diff --git a/lib/distribute/cli.cjs b/lib/distribute/cli.cjs index 9e41df6f..4ee4dc82 100644 --- a/lib/distribute/cli.cjs +++ b/lib/distribute/cli.cjs @@ -3,7 +3,7 @@ require('dotenv').config() const {getDepartements} = require('../util/cli.cjs') const {runInParallel} = require('../util/parallel.cjs') -const DISTRIBUTIONS = ['csv', 'csv-bal', 'addok', 'geojson'] +const DISTRIBUTIONS = ['csv', 'csv-bal', 'addok', 'geojson', 'csv-with-ids'] async function main() { const departements = getDepartements() diff --git a/lib/distribute/writers/csv-with-ids.cjs b/lib/distribute/writers/csv-with-ids.cjs new file mode 100644 index 00000000..932c8e56 --- /dev/null +++ b/lib/distribute/writers/csv-with-ids.cjs @@ -0,0 +1,64 @@ +/* eslint camelcase: off */ +const {promisify} = require('util') +const {join} = require('path') +const {createWriteStream} = require('fs') +const {createGzip} = require('zlib') +const finished = promisify(require('stream').finished) +const {ensureDir} = require('fs-extra') +const csvWriter = require('csv-write-stream') +const pumpify = require('pumpify') + +const {prepareAdressesWithIds, prepareLieuxDitsWithIds} = require('../../formatters/csv-legacy.cjs') + +function createCsvWriteStream(file) { + return pumpify.obj( + csvWriter({separator: ';'}), + createGzip(), + file + ) +} + +function waitForDrain(stream) { + if (stream.writableLength > stream.writableHighWaterMark) { + return new Promise(resolve => { + stream.once('drain', resolve) + }) + } +} + +async function createWriter(outputPath, departement) { + await ensureDir(outputPath) + + const adressesFile = createWriteStream(join(outputPath, `adresses-with-ids-${departement}.csv.gz`)) + const lieuxDitsFile = createWriteStream(join(outputPath, `lieux-dits-with-ids-${departement}-beta.csv.gz`)) + + const adressesStream = createCsvWriteStream(adressesFile) + const lieuxDitsStream = createCsvWriteStream(lieuxDitsFile) + + return { + async writeAdresses({voies, numeros}) { + prepareLieuxDitsWithIds({voies}) + .forEach(ld => lieuxDitsStream.write(ld)) + + prepareAdressesWithIds({voies, numeros}) + .forEach(a => adressesStream.write(a)) + + await Promise.all([ + waitForDrain(adressesStream), + waitForDrain(lieuxDitsStream) + ]) + }, + + async finish() { + adressesStream.end() + lieuxDitsStream.end() + + await Promise.all([ + finished(adressesFile), + finished(lieuxDitsFile) + ]) + } + } +} + +module.exports = {createWriter} diff --git a/lib/formatters/csv-legacy.cjs b/lib/formatters/csv-legacy.cjs index b643124b..85702021 100644 --- a/lib/formatters/csv-legacy.cjs +++ b/lib/formatters/csv-legacy.cjs @@ -26,6 +26,38 @@ function getSource(rawSource) { } function adresseToRow(a) { + return { + id: a.cleInterop, + id_fantoir: getIdFantoirField(a.codeCommune, a.idVoie) || '', + numero: a.numero, + rep: a.suffixe || '', + nom_voie: a.nomVoie, + code_postal: a.codePostal || '', + code_insee: a.codeCommune, + nom_commune: a.nomCommune, + code_insee_ancienne_commune: a.codeAncienneCommune || '', + nom_ancienne_commune: a.nomAncienneCommune || '', + + x: a.x || '', + y: a.y || '', + lon: a.lon || '', + lat: a.lat || '', + type_position: a.positionType || '', + + alias: '', + nom_ld: a.lieuDitComplementNom, + libelle_acheminement: a.libelleAcheminement || '', + nom_afnor: normalize(a.nomVoie), + + source_position: getSource(a.sourcePosition) || '', + source_nom_voie: getSource(a.sourceNomVoie) || '', + + certification_commune: a.certifie ? '1' : '0', + cad_parcelles: a.parcelles ? a.parcelles.join('|') : '' + } +} + +function adresseToRowWithIds(a) { return { id: a.cleInterop, id_ban_adresse: a.banId || '', @@ -60,7 +92,7 @@ function adresseToRow(a) { } } -function lieuDitToRow(a) { +function lieuDitToRowWithIds(a) { return { id: a.idVoie, id_ban_toponyme: a.banId || '', @@ -82,6 +114,26 @@ function lieuDitToRow(a) { } } +function lieuDitToRow(a) { + return { + id: a.idVoie, + nom_lieu_dit: a.nomVoie, + code_postal: a.codePostal || '', + code_insee: a.codeCommune, + nom_commune: a.nomCommune, + code_insee_ancienne_commune: a.codeAncienneCommune || '', + nom_ancienne_commune: a.nomAncienneCommune || '', + + x: a.x || '', + y: a.y || '', + lon: a.lon || '', + lat: a.lat || '', + + source_position: a.source, + source_nom_voie: a.source + } +} + function prepareAdresses({voies, numeros}) { const voiesIndex = keyBy(voies, 'idVoie') @@ -98,8 +150,28 @@ function prepareAdresses({voies, numeros}) { }) } +function prepareAdressesWithIds({voies, numeros}) { + const voiesIndex = keyBy(voies, 'idVoie') + + return numeros + .filter(n => n.position && n.lon && n.lat) + .map(n => { + const voie = voiesIndex[n.idVoie] + + if (!voie) { + throw new Error(`Voie ${n.idVoie} introuvable`) + } + + return adresseToRowWithIds({...voie, ...n}) + }) +} + function prepareLieuxDits({voies}) { return voies.filter(v => v.type === 'lieu-dit').map(v => lieuDitToRow(v)) } -module.exports = {prepareAdresses, prepareLieuxDits} +function prepareLieuxDitsWithIds({voies}) { + return voies.filter(v => v.type === 'lieu-dit').map(v => lieuDitToRowWithIds(v)) +} + +module.exports = {prepareAdresses, prepareLieuxDits, prepareLieuxDitsWithIds, prepareAdressesWithIds}