DRAFT : New script BAN converter

package.json

@@ -31,7 +31,8 @@
     "migrate:up": "npx sequelize-cli db:migrate",
     "migrate:undo": "npx sequelize-cli db:migrate:undo",
     "seed:up": "npx sequelize-cli db:seed:all",
-    "seed:undo": "npx sequelize-cli db:seed:undo"
+    "seed:undo": "npx sequelize-cli db:seed:undo",
+    "exportban": "node scripts/ban-converter/index.js"
   },
   "dependencies": {
     "@ban-team/fantoir": "^0.15.0",
@@ -69,6 +70,7 @@
     "keyv": "^4.3.2",
     "leven": "^3.1.0",
     "lodash": "^4.17.21",
+    "lodash-es": "^4.17.21",
     "lru-cache": "^6.0.0",
     "minimist": "^1.2.8",
     "mongodb": "^4.7.0",
@@ -77,7 +79,7 @@
     "nanoid": "^4.0.2",
     "ndjson": "^2.0.0",
     "node-fetch": "^2.6.11",
-    "ora": "^5.4.1",
+    "ora": "7.x",
     "papaparse": "^5.4.1",
     "pg": "^8.11.0",
     "proj4": "^2.9.0",

scripts/ban-converter/ban-converter.js

@@ -0,0 +1,50 @@
+import {get} from 'lodash-es'
+
+import formaters from './formaters.js'
+import {
+  getDistrictsMap,
+  getDistrictFromAddress,
+} from './getter-district.js'
+import {
+  getMicroToponymsMap,
+  getMainMicroToponymFromAddress,
+  getSecondaryMicroToponymsFromAddress
+} from './getter-micro-toponym.js'
+
+const convertBan = async (banData, exportConfig) => {
+  const {districts, microToponyms, addresses} = banData
+  const districtsMap = getDistrictsMap(districts)
+  const microToponymsMap = getMicroToponymsMap(microToponyms)
+  const getDistrict = getDistrictFromAddress(districtsMap)
+  const getMainMicroToponym = getMainMicroToponymFromAddress(microToponymsMap)
+  const getSecondaryMicroToponyms = getSecondaryMicroToponymsFromAddress(microToponymsMap)
+  const exportConfigArray = Object.entries(exportConfig)
+
+  const result = []
+  for (const address of addresses) {
+    const __district = getDistrict(address)
+    const __microToponym = getMainMicroToponym(address)
+    const __secondaryToponyms = getSecondaryMicroToponyms(address)
+    const workingAddress = {
+      ...address,
+      __district,
+      __microToponym,
+      __secondaryToponyms,
+    }
+    result.push(
+      exportConfigArray.reduce((acc, [key, value]) => {
+        const [formaterName, path, ...args] = Array.isArray(value) ? value : [null, value]
+        const formatValue = formaters?.[formaterName] || (v => v)
+        if (value) {
+          acc[key] = formatValue(get(workingAddress, path, null), ...args)
+        }
+
+        return acc
+      }, {})
+    )
+  }
+
+  return result
+}
+
+export default convertBan

scripts/ban-converter/file-loaders.js

@@ -0,0 +1,47 @@
+import fs from 'node:fs/promises'
+import path, {dirname} from 'node:path'
+import {fileURLToPath} from 'node:url'
+
+export const loadBanFile = async path => {
+  try {
+    const response = await fs.readFile(path, 'utf8')
+    const data = JSON.parse(response)
+
+    if (data.status !== 'success') {
+      throw new Error('BAN file loading error', {values: {path}})
+    }
+
+    return data.response
+  } catch (error) {
+    throw new Error('Error on loading BAN file', {cause: error, values: {path}})
+  }
+}
+
+export const loadConfigFile = async path => {
+  try {
+    const response = await fs.readFile(path, 'utf8')
+    return JSON.parse(response)
+  } catch (error) {
+    throw new Error('Error on loading config file', {cause: error, values: {path}})
+  }
+}
+
+export const loadHelpFile = async lang => {
+  console.log(`La langue du système est: ${lang}`)
+  const __dirname = dirname(fileURLToPath(import.meta.url))
+  let helpFilePath
+  try {
+    const filePath = path.resolve(__dirname, `help.${lang}.txt`)
+    await fs.access(filePath)
+    helpFilePath = filePath
+  } catch {
+    helpFilePath = path.resolve(__dirname, 'help.en.txt')
+  }
+
+  try {
+    const response = await fs.readFile(helpFilePath, 'utf8')
+    return response
+  } catch (error) {
+    throw new Error('Error on loading help file', {cause: error, values: {path: helpFilePath}})
+  }
+}

scripts/ban-converter/formaters.js

@@ -0,0 +1,9 @@
+import normalize from '@etalab/normadresse'
+
+const formaters = {
+  NUMBER: Number,
+  NORMALIZE: normalize,
+  ARRAY_JOIN: arr => arr.join('|'),
+}
+
+export default formaters

scripts/ban-converter/getter-district.js

@@ -0,0 +1,15 @@
+export const getDistrictsMap = districts => new Map(
+  districts.map(
+    district => ([
+      district.id
+          || district.meta?.ban?.DEPRECATED_id
+          || district.meta?.insee?.cog,
+      district
+    ])
+  )
+)
+
+export const getDistrictFromAddress = districtsProp => addr => (
+  districtsProp.get(addr.districtID || addr.meta?.insee?.cog)
+)
+

scripts/ban-converter/getter-micro-toponym.js

@@ -0,0 +1,22 @@
+export const getMicroToponymsMap = microToponyms => new Map(
+  microToponyms.map(microToponym => (
+    [
+      microToponym.id
+        || microToponym.meta?.ban?.DEPRECATED_id,
+      microToponym
+    ])
+  )
+)
+
+export const getMainMicroToponymFromAddress = microToponymsProp => addr => (
+  microToponymsProp.get(
+    addr.mainMicroToponymID
+    || addr.meta?.ban?.DEPRECATED_id?.split('_').slice(0, 2).join('_')
+  )
+)
+
+export const getSecondaryMicroToponymsFromAddress = toponymsProp => addr => (
+  addr.secondaryMicroToponymIDs?.map(
+    toponymID => toponymsProp.get(toponymID)
+  )
+)

scripts/ban-converter/help.en.txt

@@ -0,0 +1,16 @@
+
+usage : ban [--version] [-h | --help] [-c=<value> | --config=<value>] 
+            <source file path> <destination file path> [-s | --silence | --quiet]
+        ban [--version] [-h | --help] [-c=<value> | --config=<value>] 
+            <source file path> <destination directory path> [-s | --silence | --quiet]
+
+parameters :
+   source file path            Path of the BAN files to convert
+   destination file path       Path to the file in which the converted data will be saved.
+   destination directory path  Path to the directory where the file with the converted data will be saved.
+
+options :
+   -v, --version           Display the version number
+   -h, --help              Display this help.
+   -c, --config            Name of the preconfiguration or path to the destination file configuration.
+   -s, --silence, --quiet  The loader and processing information are not displayed during processing.

scripts/ban-converter/help.fr.txt

@@ -0,0 +1,18 @@
+usage : ban [--version] [-h | --help] [-c=<valeur> | --config=<valeur>] 
+            <chemin du fichier source> <chemin du fichier de destination> [-s | --silence | --quiet]
+        ban [--version] [-h | --help] [-c=<valeur> | --config=<valeur>] 
+            <chemin du fichier source> <chemin du repertoir de destination> [-s | --silence | --quiet]
+
+parametres :
+   chemin du fichier source            Chemin du fichiers BAN à convertir
+   chemin du fichier de destination    Chemin vers le fichiers dans lequel seront sauvegarder les 
+                                       données converties.
+   chemin du repertoir de destination  Chemin vers le repertoir dans lequel se trouvera 
+                                       le fichier ou seront sauvegarder les données converties.
+
+options :
+   -v, --version           Affiche le numero de version
+   -h, --help              Afficher cette aide.
+   -c, --config            nom de la preconfiguration ou chemin vers la configuration du fichier de destination.
+   -s, --silence, --quiet  Le loader et les informations de traitement ne sont pas afficher lors du traitement.
+

scripts/ban-converter/index.js

@@ -0,0 +1,81 @@
+#!/usr/bin/env node
+import fs from 'node:fs/promises'
+import Path from 'node:path'
+
+import minimist from 'minimist'
+import Papa from 'papaparse'
+
+import convertBan from './ban-converter.js'
+import preconfig from './preconfig.js'
+import SpinnerLogger from './spinner-logger.js'
+import {loadBanFile, loadConfigFile, loadHelpFile} from './file-loaders.js'
+
+const main = async (inputPath, configPathOrName = 'bal', outputPath = '', options) => {
+  const {quiet} = options
+  const logger = new SpinnerLogger(!quiet)
+
+  try {
+    logger.start('Convert BAN data')
+
+    const banData = await loadBanFile(inputPath)
+    const configParam = preconfig?.[configPathOrName] || await loadConfigFile(configPathOrName)
+    const {config, name, fileExtention, csvConfig} = configParam
+    const resultData = await convertBan(banData, config)
+
+    let result
+    switch (fileExtention) {
+      case 'csv':
+        result = Papa.unparse(resultData, csvConfig)
+        break
+      case 'json':
+        result = JSON.stringify(resultData, null, 2)
+        break
+      case null:
+      case undefined:
+        throw new Error('File extention is required', {
+          cause: 'Missing file extention',
+          values: {fileExtention},
+        })
+      default:
+        throw new Error(`'.${fileExtention}' File extention is not supported`,
+          {
+            cause: 'Unsupported file extention',
+            values: {fileExtention},
+          })
+    }
+
+    const dateFile = `${(new Date()).toLocaleString('FR-fr').replace(/\//g, '-').replace(/:/, 'h').replace(/:/, 'm').replace(/\s/, '_')}s`
+    const outputFilePath = (new RegExp(`.${fileExtention}$`)).test(outputPath) ? outputPath : null
+    const resultFilePath = outputFilePath || Path.join(outputPath, `${'export'}_${name}_${dateFile}.${fileExtention}`)
+    await fs.writeFile(resultFilePath, result)
+
+    logger.succeed(`Conversion ready: ${Path.join(process.cwd(), resultFilePath)}`)
+  } catch (error) {
+    logger.fail(error)
+  }
+}
+
+const {
+  _: [inputFile, outputFile],
+  v, version,
+  h, help,
+  c, config,
+  s, silence, quiet
+} = minimist(process.argv.slice(2))
+const banFilePath = inputFile
+const options = {quiet: s || silence || quiet}
+
+if (version || v) {
+  console.log('ban-converter v0.1.0-beta.0')
+  process.exit(0)
+}
+
+if (help || h) {
+  const {env} = process
+  const sysLang = (env.LANG || env.LANGUAGE || env.LC_ALL || env.LC_MESSAGES)?.split('_')?.[0]
+  const helpText = await loadHelpFile(sysLang)
+  console.log(helpText)
+  process.exit(0)
+}
+
+main(banFilePath, c || config, outputFile, options)

scripts/ban-converter/preconfig.js

@@ -0,0 +1,77 @@
+/* eslint-disable camelcase */
+
+const ignHistoriqueAdressesConfig = {
+  dataFormat: 'csv',
+  fileExtention: 'csv',
+  csvConfig: {
+    delimiter: ';',
+  },
+  name: 'ign-historique-adresses',
+  description: 'Export des adresses BAN vers le format IGN historique adresses',
+  config: {
+    id: 'meta.ban.DEPRECATED_id',
+    id_fantoir: 'meta.dgfip.fantoir',
+    numero: 'number',
+    rep: 'suffix',
+    nom_voie: '__microToponym.labels[0].value',
+    code_postal: 'meta.laPoste.codePostal',
+    code_insee: 'meta.insee.cog',
+    nom_commune: '__district.Labels[0].value',
+    code_insee_ancienne_commune: '', // ????? // Pas encore present dans le format BAN
+    nom_ancienne_commune: '', // ????? // Pas encore present dans le format BAN
+    x: '', // ????? meta.ban.positionX
+    y: '', // ????? meta.ban.positionY
+    lon: 'positions[0].geometry.coordinates[0]',
+    lat: 'positions[0].geometry.coordinates[1]',
+    type_position: 'positions[0].type',
+    alias: null,
+    nom_ld: 'labels[0].value',
+    libelle_acheminement: 'meta.laPoste.libelleAcheminement',
+    nom_afnor: ['NORMALIZE', '__microToponym.labels[0].value'],
+    source_position: 'meta.ban.sourcePosition',
+    source_nom_voie: '__microToponym.meta.ban.sourceNomVoie',
+    certification_commune: ['NUMBER', 'certified'],
+    cad_parcelles: ['ARRAY_JOIN', 'meta.dgfip.cadastre']
+  }
+}
+
+const balConfig = {
+  dataFormat: 'csv',
+  fileExtention: 'csv',
+  csvConfig: {
+    delimiter: ';',
+  },
+  name: 'bal-1-4',
+  description: 'Export des adresses BAN vers le format IGN historique adresses',
+  config: {
+    id_ban_commune: 'districtID',
+    id_ban_toponyme: 'mainMicroToponymID',
+    id_ban_adresse: 'id',
+    cle_interop: 'meta.ban.DEPRECATED_cleInteropBAN',
+    commune_insee: 'meta.insee.cog',
+    commune_nom: '__district.Labels[0].value',
+    commune_deleguee_insee: '', // ????? // Pas encore present dans le format BAN
+    commune_deleguee_nom: '', // ????? // Pas encore present dans le format BAN
+    voie_nom: '__microToponym.labels[0].value',
+    lieudit_complement_nom: 'labels[0].value',
+    numero: 'number',
+    suffixe: 'suffix',
+    position: 'positions[0].type',
+    x: '', // ????? meta.ban.positionX
+    y: '', // ????? meta.ban.positionY
+    long: 'positions[0].geometry.coordinates[0]',
+    lat: 'positions[0].geometry.coordinates[1]',
+    cad_parcelle: 'meta.dgfip.cadastre',
+    source: 'meta.ban.sourcePosition',
+    date_der_maj: 'legalityDate',
+    certification_commune: 'certified',
+  }
+}
+
+const preconfig = {
+  ign: ignHistoriqueAdressesConfig,
+  bal: balConfig,
+  'bal-1.4': balConfig,
+}
+
+export default preconfig