Merge pull request #326 from BaseAdresseNationale/mmortier/legacy-csv

csv legacy without ids and new csv legacy with ids
BaseAdresseNationale · Nov 17, 2023 · 9ed063e · 9ed063e
2 parents 73d78b6 + d5cf7d8
commit 9ed063e
Show file tree

Hide file tree

Showing 3 changed files with 139 additions and 3 deletions.
diff --git a/lib/distribute/cli.cjs b/lib/distribute/cli.cjs
@@ -3,7 +3,7 @@ require('dotenv').config()
 const {getDepartements} = require('../util/cli.cjs')
 const {runInParallel} = require('../util/parallel.cjs')
 
-const DISTRIBUTIONS = ['csv', 'csv-bal', 'addok', 'geojson']
+const DISTRIBUTIONS = ['csv', 'csv-bal', 'addok', 'geojson', 'csv-with-ids']
 
 async function main() {
   const departements = getDepartements()

diff --git a/lib/distribute/writers/csv-with-ids.cjs b/lib/distribute/writers/csv-with-ids.cjs
@@ -0,0 +1,64 @@
+/* eslint camelcase: off */
+const {promisify} = require('util')
+const {join} = require('path')
+const {createWriteStream} = require('fs')
+const {createGzip} = require('zlib')
+const finished = promisify(require('stream').finished)
+const {ensureDir} = require('fs-extra')
+const csvWriter = require('csv-write-stream')
+const pumpify = require('pumpify')
+
+const {prepareAdressesWithIds, prepareLieuxDitsWithIds} = require('../../formatters/csv-legacy.cjs')
+
+function createCsvWriteStream(file) {
+  return pumpify.obj(
+    csvWriter({separator: ';'}),
+    createGzip(),
+    file
+  )
+}
+
+function waitForDrain(stream) {
+  if (stream.writableLength > stream.writableHighWaterMark) {
+    return new Promise(resolve => {
+      stream.once('drain', resolve)
+    })
+  }
+}
+
+async function createWriter(outputPath, departement) {
+  await ensureDir(outputPath)
+
+  const adressesFile = createWriteStream(join(outputPath, `adresses-with-ids-${departement}.csv.gz`))
+  const lieuxDitsFile = createWriteStream(join(outputPath, `lieux-dits-with-ids-${departement}-beta.csv.gz`))
+
+  const adressesStream = createCsvWriteStream(adressesFile)
+  const lieuxDitsStream = createCsvWriteStream(lieuxDitsFile)
+
+  return {
+    async writeAdresses({voies, numeros}) {
+      prepareLieuxDitsWithIds({voies})
+        .forEach(ld => lieuxDitsStream.write(ld))
+
+      prepareAdressesWithIds({voies, numeros})
+        .forEach(a => adressesStream.write(a))
+
+      await Promise.all([
+        waitForDrain(adressesStream),
+        waitForDrain(lieuxDitsStream)
+      ])
+    },
+
+    async finish() {
+      adressesStream.end()
+      lieuxDitsStream.end()
+
+      await Promise.all([
+        finished(adressesFile),
+        finished(lieuxDitsFile)
+      ])
+    }
+  }
+}
+
+module.exports = {createWriter}
diff --git a/lib/formatters/csv-legacy.cjs b/lib/formatters/csv-legacy.cjs
@@ -26,6 +26,38 @@ function getSource(rawSource) {
 }
 
 function adresseToRow(a) {
+  return {
+    id: a.cleInterop,
+    id_fantoir: getIdFantoirField(a.codeCommune, a.idVoie) || '',
+    numero: a.numero,
+    rep: a.suffixe || '',
+    nom_voie: a.nomVoie,
+    code_postal: a.codePostal || '',
+    code_insee: a.codeCommune,
+    nom_commune: a.nomCommune,
+    code_insee_ancienne_commune: a.codeAncienneCommune || '',
+    nom_ancienne_commune: a.nomAncienneCommune || '',
+
+    x: a.x || '',
+    y: a.y || '',
+    lon: a.lon || '',
+    lat: a.lat || '',
+    type_position: a.positionType || '',
+
+    alias: '',
+    nom_ld: a.lieuDitComplementNom,
+    libelle_acheminement: a.libelleAcheminement || '',
+    nom_afnor: normalize(a.nomVoie),
+
+    source_position: getSource(a.sourcePosition) || '',
+    source_nom_voie: getSource(a.sourceNomVoie) || '',
+
+    certification_commune: a.certifie ? '1' : '0',
+    cad_parcelles: a.parcelles ? a.parcelles.join('|') : ''
+  }
+}
+
+function adresseToRowWithIds(a) {
   return {
     id: a.cleInterop,
     id_ban_adresse: a.banId || '',
@@ -60,7 +92,7 @@ function adresseToRow(a) {
   }
 }
 
-function lieuDitToRow(a) {
+function lieuDitToRowWithIds(a) {
   return {
     id: a.idVoie,
     id_ban_toponyme: a.banId || '',
@@ -82,6 +114,26 @@ function lieuDitToRow(a) {
   }
 }
 
+function lieuDitToRow(a) {
+  return {
+    id: a.idVoie,
+    nom_lieu_dit: a.nomVoie,
+    code_postal: a.codePostal || '',
+    code_insee: a.codeCommune,
+    nom_commune: a.nomCommune,
+    code_insee_ancienne_commune: a.codeAncienneCommune || '',
+    nom_ancienne_commune: a.nomAncienneCommune || '',
+
+    x: a.x || '',
+    y: a.y || '',
+    lon: a.lon || '',
+    lat: a.lat || '',
+
+    source_position: a.source,
+    source_nom_voie: a.source
+  }
+}
+
 function prepareAdresses({voies, numeros}) {
   const voiesIndex = keyBy(voies, 'idVoie')
 
@@ -98,8 +150,28 @@ function prepareAdresses({voies, numeros}) {
     })
 }
 
+function prepareAdressesWithIds({voies, numeros}) {
+  const voiesIndex = keyBy(voies, 'idVoie')
+
+  return numeros
+    .filter(n => n.position && n.lon && n.lat)
+    .map(n => {
+      const voie = voiesIndex[n.idVoie]
+
+      if (!voie) {
+        throw new Error(`Voie ${n.idVoie} introuvable`)
+      }
+
+      return adresseToRowWithIds({...voie, ...n})
+    })
+}
+
 function prepareLieuxDits({voies}) {
   return voies.filter(v => v.type === 'lieu-dit').map(v => lieuDitToRow(v))
 }
 
-module.exports = {prepareAdresses, prepareLieuxDits}
+function prepareLieuxDitsWithIds({voies}) {
+  return voies.filter(v => v.type === 'lieu-dit').map(v => lieuDitToRowWithIds(v))
+}
+
+module.exports = {prepareAdresses, prepareLieuxDits, prepareLieuxDitsWithIds, prepareAdressesWithIds}