diff --git a/package.json b/package.json index e7186c4a..4077d9ca 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,8 @@ "migrate:up": "npx sequelize-cli db:migrate", "migrate:undo": "npx sequelize-cli db:migrate:undo", "seed:up": "npx sequelize-cli db:seed:all", - "seed:undo": "npx sequelize-cli db:seed:undo" + "seed:undo": "npx sequelize-cli db:seed:undo", + "exportban": "node scripts/ban-converter/index.js" }, "dependencies": { "@ban-team/fantoir": "^0.15.0", @@ -69,6 +70,7 @@ "keyv": "^4.3.2", "leven": "^3.1.0", "lodash": "^4.17.21", + "lodash-es": "^4.17.21", "lru-cache": "^6.0.0", "minimist": "^1.2.8", "mongodb": "^4.7.0", @@ -77,7 +79,7 @@ "nanoid": "^4.0.2", "ndjson": "^2.0.0", "node-fetch": "^2.6.11", - "ora": "^5.4.1", + "ora": "7.x", "papaparse": "^5.4.1", "pg": "^8.11.0", "proj4": "^2.9.0", diff --git a/scripts/ban-converter/ban-converter.js b/scripts/ban-converter/ban-converter.js new file mode 100644 index 00000000..526412c9 --- /dev/null +++ b/scripts/ban-converter/ban-converter.js @@ -0,0 +1,50 @@ +import {get} from 'lodash-es' + +import formaters from './formaters.js' +import { + getDistrictsMap, + getDistrictFromAddress, +} from './getter-district.js' +import { + getMicroToponymsMap, + getMainMicroToponymFromAddress, + getSecondaryMicroToponymsFromAddress +} from './getter-micro-toponym.js' + +const convertBan = async (banData, exportConfig) => { + const {districts, microToponyms, addresses} = banData + const districtsMap = getDistrictsMap(districts) + const microToponymsMap = getMicroToponymsMap(microToponyms) + const getDistrict = getDistrictFromAddress(districtsMap) + const getMainMicroToponym = getMainMicroToponymFromAddress(microToponymsMap) + const getSecondaryMicroToponyms = getSecondaryMicroToponymsFromAddress(microToponymsMap) + const exportConfigArray = Object.entries(exportConfig) + + const result = [] + for (const address of addresses) { + const __district = getDistrict(address) + const __microToponym = getMainMicroToponym(address) + const __secondaryToponyms = getSecondaryMicroToponyms(address) + const workingAddress = { + ...address, + __district, + __microToponym, + __secondaryToponyms, + } + result.push( + exportConfigArray.reduce((acc, [key, value]) => { + const [formaterName, path, ...args] = Array.isArray(value) ? value : [null, value] + const formatValue = formaters?.[formaterName] || (v => v) + if (value) { + acc[key] = formatValue(get(workingAddress, path, null), ...args) + } + + return acc + }, {}) + ) + } + + return result +} + +export default convertBan diff --git a/scripts/ban-converter/file-loaders.js b/scripts/ban-converter/file-loaders.js new file mode 100644 index 00000000..d0a37c95 --- /dev/null +++ b/scripts/ban-converter/file-loaders.js @@ -0,0 +1,47 @@ +import fs from 'node:fs/promises' +import path, {dirname} from 'node:path' +import {fileURLToPath} from 'node:url' + +export const loadBanFile = async path => { + try { + const response = await fs.readFile(path, 'utf8') + const data = JSON.parse(response) + + if (data.status !== 'success') { + throw new Error('BAN file loading error', {values: {path}}) + } + + return data.response + } catch (error) { + throw new Error('Error on loading BAN file', {cause: error, values: {path}}) + } +} + +export const loadConfigFile = async path => { + try { + const response = await fs.readFile(path, 'utf8') + return JSON.parse(response) + } catch (error) { + throw new Error('Error on loading config file', {cause: error, values: {path}}) + } +} + +export const loadHelpFile = async lang => { + console.log(`La langue du système est: ${lang}`) + const __dirname = dirname(fileURLToPath(import.meta.url)) + let helpFilePath + try { + const filePath = path.resolve(__dirname, `help.${lang}.txt`) + await fs.access(filePath) + helpFilePath = filePath + } catch { + helpFilePath = path.resolve(__dirname, 'help.en.txt') + } + + try { + const response = await fs.readFile(helpFilePath, 'utf8') + return response + } catch (error) { + throw new Error('Error on loading help file', {cause: error, values: {path: helpFilePath}}) + } +} diff --git a/scripts/ban-converter/formaters.js b/scripts/ban-converter/formaters.js new file mode 100644 index 00000000..3d09239b --- /dev/null +++ b/scripts/ban-converter/formaters.js @@ -0,0 +1,9 @@ +import normalize from '@etalab/normadresse' + +const formaters = { + NUMBER: Number, + NORMALIZE: normalize, + ARRAY_JOIN: arr => arr.join('|'), +} + +export default formaters diff --git a/scripts/ban-converter/getter-district.js b/scripts/ban-converter/getter-district.js new file mode 100644 index 00000000..571ef1e2 --- /dev/null +++ b/scripts/ban-converter/getter-district.js @@ -0,0 +1,15 @@ +export const getDistrictsMap = districts => new Map( + districts.map( + district => ([ + district.id + || district.meta?.ban?.DEPRECATED_id + || district.meta?.insee?.cog, + district + ]) + ) +) + +export const getDistrictFromAddress = districtsProp => addr => ( + districtsProp.get(addr.districtID || addr.meta?.insee?.cog) +) + diff --git a/scripts/ban-converter/getter-micro-toponym.js b/scripts/ban-converter/getter-micro-toponym.js new file mode 100644 index 00000000..7153dbc1 --- /dev/null +++ b/scripts/ban-converter/getter-micro-toponym.js @@ -0,0 +1,22 @@ +export const getMicroToponymsMap = microToponyms => new Map( + microToponyms.map(microToponym => ( + [ + microToponym.id + || microToponym.meta?.ban?.DEPRECATED_id, + microToponym + ]) + ) +) + +export const getMainMicroToponymFromAddress = microToponymsProp => addr => ( + microToponymsProp.get( + addr.mainMicroToponymID + || addr.meta?.ban?.DEPRECATED_id?.split('_').slice(0, 2).join('_') + ) +) + +export const getSecondaryMicroToponymsFromAddress = toponymsProp => addr => ( + addr.secondaryMicroToponymIDs?.map( + toponymID => toponymsProp.get(toponymID) + ) +) diff --git a/scripts/ban-converter/help.en.txt b/scripts/ban-converter/help.en.txt new file mode 100644 index 00000000..e4a636c1 --- /dev/null +++ b/scripts/ban-converter/help.en.txt @@ -0,0 +1,16 @@ + +usage : ban [--version] [-h | --help] [-c= | --config=] + [-s | --silence | --quiet] + ban [--version] [-h | --help] [-c= | --config=] + [-s | --silence | --quiet] + +parameters : + source file path Path of the BAN files to convert + destination file path Path to the file in which the converted data will be saved. + destination directory path Path to the directory where the file with the converted data will be saved. + +options : + -v, --version Display the version number + -h, --help Display this help. + -c, --config Name of the preconfiguration or path to the destination file configuration. + -s, --silence, --quiet The loader and processing information are not displayed during processing. diff --git a/scripts/ban-converter/help.fr.txt b/scripts/ban-converter/help.fr.txt new file mode 100644 index 00000000..be402308 --- /dev/null +++ b/scripts/ban-converter/help.fr.txt @@ -0,0 +1,18 @@ +usage : ban [--version] [-h | --help] [-c= | --config=] + [-s | --silence | --quiet] + ban [--version] [-h | --help] [-c= | --config=] + [-s | --silence | --quiet] + +parametres : + chemin du fichier source Chemin du fichiers BAN à convertir + chemin du fichier de destination Chemin vers le fichiers dans lequel seront sauvegarder les + données converties. + chemin du repertoir de destination Chemin vers le repertoir dans lequel se trouvera + le fichier ou seront sauvegarder les données converties. + +options : + -v, --version Affiche le numero de version + -h, --help Afficher cette aide. + -c, --config nom de la preconfiguration ou chemin vers la configuration du fichier de destination. + -s, --silence, --quiet Le loader et les informations de traitement ne sont pas afficher lors du traitement. + diff --git a/scripts/ban-converter/index.js b/scripts/ban-converter/index.js new file mode 100644 index 00000000..d27097e2 --- /dev/null +++ b/scripts/ban-converter/index.js @@ -0,0 +1,81 @@ +#!/usr/bin/env node +import fs from 'node:fs/promises' +import Path from 'node:path' + +import minimist from 'minimist' +import Papa from 'papaparse' + +import convertBan from './ban-converter.js' +import preconfig from './preconfig.js' +import SpinnerLogger from './spinner-logger.js' +import {loadBanFile, loadConfigFile, loadHelpFile} from './file-loaders.js' + +const main = async (inputPath, configPathOrName = 'bal', outputPath = '', options) => { + const {quiet} = options + const logger = new SpinnerLogger(!quiet) + + try { + logger.start('Convert BAN data') + + const banData = await loadBanFile(inputPath) + const configParam = preconfig?.[configPathOrName] || await loadConfigFile(configPathOrName) + const {config, name, fileExtention, csvConfig} = configParam + const resultData = await convertBan(banData, config) + + let result + switch (fileExtention) { + case 'csv': + result = Papa.unparse(resultData, csvConfig) + break + case 'json': + result = JSON.stringify(resultData, null, 2) + break + case null: + case undefined: + throw new Error('File extention is required', { + cause: 'Missing file extention', + values: {fileExtention}, + }) + default: + throw new Error(`'.${fileExtention}' File extention is not supported`, + { + cause: 'Unsupported file extention', + values: {fileExtention}, + }) + } + + const dateFile = `${(new Date()).toLocaleString('FR-fr').replace(/\//g, '-').replace(/:/, 'h').replace(/:/, 'm').replace(/\s/, '_')}s` + const outputFilePath = (new RegExp(`.${fileExtention}$`)).test(outputPath) ? outputPath : null + const resultFilePath = outputFilePath || Path.join(outputPath, `${'export'}_${name}_${dateFile}.${fileExtention}`) + await fs.writeFile(resultFilePath, result) + + logger.succeed(`Conversion ready: ${Path.join(process.cwd(), resultFilePath)}`) + } catch (error) { + logger.fail(error) + } +} + +const { + _: [inputFile, outputFile], + v, version, + h, help, + c, config, + s, silence, quiet +} = minimist(process.argv.slice(2)) +const banFilePath = inputFile +const options = {quiet: s || silence || quiet} + +if (version || v) { + console.log('ban-converter v0.1.0-beta.0') + process.exit(0) +} + +if (help || h) { + const {env} = process + const sysLang = (env.LANG || env.LANGUAGE || env.LC_ALL || env.LC_MESSAGES)?.split('_')?.[0] + const helpText = await loadHelpFile(sysLang) + console.log(helpText) + process.exit(0) +} + +main(banFilePath, c || config, outputFile, options) diff --git a/scripts/ban-converter/preconfig.js b/scripts/ban-converter/preconfig.js new file mode 100644 index 00000000..1de6ffe9 --- /dev/null +++ b/scripts/ban-converter/preconfig.js @@ -0,0 +1,77 @@ +/* eslint-disable camelcase */ + +const ignHistoriqueAdressesConfig = { + dataFormat: 'csv', + fileExtention: 'csv', + csvConfig: { + delimiter: ';', + }, + name: 'ign-historique-adresses', + description: 'Export des adresses BAN vers le format IGN historique adresses', + config: { + id: 'meta.ban.DEPRECATED_id', + id_fantoir: 'meta.dgfip.fantoir', + numero: 'number', + rep: 'suffix', + nom_voie: '__microToponym.labels[0].value', + code_postal: 'meta.laPoste.codePostal', + code_insee: 'meta.insee.cog', + nom_commune: '__district.Labels[0].value', + code_insee_ancienne_commune: '', // ????? // Pas encore present dans le format BAN + nom_ancienne_commune: '', // ????? // Pas encore present dans le format BAN + x: '', // ????? meta.ban.positionX + y: '', // ????? meta.ban.positionY + lon: 'positions[0].geometry.coordinates[0]', + lat: 'positions[0].geometry.coordinates[1]', + type_position: 'positions[0].type', + alias: null, + nom_ld: 'labels[0].value', + libelle_acheminement: 'meta.laPoste.libelleAcheminement', + nom_afnor: ['NORMALIZE', '__microToponym.labels[0].value'], + source_position: 'meta.ban.sourcePosition', + source_nom_voie: '__microToponym.meta.ban.sourceNomVoie', + certification_commune: ['NUMBER', 'certified'], + cad_parcelles: ['ARRAY_JOIN', 'meta.dgfip.cadastre'] + } +} + +const balConfig = { + dataFormat: 'csv', + fileExtention: 'csv', + csvConfig: { + delimiter: ';', + }, + name: 'bal-1-4', + description: 'Export des adresses BAN vers le format IGN historique adresses', + config: { + id_ban_commune: 'districtID', + id_ban_toponyme: 'mainMicroToponymID', + id_ban_adresse: 'id', + cle_interop: 'meta.ban.DEPRECATED_cleInteropBAN', + commune_insee: 'meta.insee.cog', + commune_nom: '__district.Labels[0].value', + commune_deleguee_insee: '', // ????? // Pas encore present dans le format BAN + commune_deleguee_nom: '', // ????? // Pas encore present dans le format BAN + voie_nom: '__microToponym.labels[0].value', + lieudit_complement_nom: 'labels[0].value', + numero: 'number', + suffixe: 'suffix', + position: 'positions[0].type', + x: '', // ????? meta.ban.positionX + y: '', // ????? meta.ban.positionY + long: 'positions[0].geometry.coordinates[0]', + lat: 'positions[0].geometry.coordinates[1]', + cad_parcelle: 'meta.dgfip.cadastre', + source: 'meta.ban.sourcePosition', + date_der_maj: 'legalityDate', + certification_commune: 'certified', + } +} + +const preconfig = { + ign: ignHistoriqueAdressesConfig, + bal: balConfig, + 'bal-1.4': balConfig, +} + +export default preconfig diff --git a/scripts/ban-converter/spinner-logger.js b/scripts/ban-converter/spinner-logger.js new file mode 100644 index 00000000..d5d99b1c --- /dev/null +++ b/scripts/ban-converter/spinner-logger.js @@ -0,0 +1,31 @@ +import ora from 'ora' + +function SpinnerLogger(visualLogger) { + const spinner = ora() + const logger = { + start: message => visualLogger ? spinner.start(message) : console.log(message), + log(message) { + if (visualLogger) { + spinner.text = message + return message + } + + return console.log(message) + }, + succeed: message => visualLogger ? spinner.succeed(message) : console.log(message), + fail: message => ( + visualLogger + ? spinner.fail( + `Error >> ${ + typeof message === 'string' + ? message + : (message.message || JSON.stringify(message)) + }`) + : console.error(message) + ), + } + + return logger +} + +export default SpinnerLogger