diff --git a/lib/api/consumers/export-to-exploitation-db-consumer.js b/lib/api/consumers/export-to-exploitation-db-consumer.js index 5baffe63..edd544da 100644 --- a/lib/api/consumers/export-to-exploitation-db-consumer.js +++ b/lib/api/consumers/export-to-exploitation-db-consumer.js @@ -1,5 +1,6 @@ import {Transaction} from 'sequelize' import {createFantoirCommune} from '@ban-team/fantoir' +import {createGazetteer} from '@ban-team/gazetteer' import {findCodePostal} from 'codes-postaux/full.js' import mongo from '../../util/mongo.cjs' import {sequelize, District, CommonToponym} from '../../util/sequelize.js' @@ -16,6 +17,8 @@ const PAGE_SIZE = 100 // The path to the fantoir sqlite database const FANTOIR_PATH = process.env.FANTOIR_PATH || 'data/fantoir.sqlite' +const GAZETTEER_DB_PATH = process.env.GAZETTEER_DB_PATH || 'data/gazetteer.sqlite' + // The min and max zoom levels to use for the tiles const TILES_ZOOM_LEVELS = { commonToponym: { @@ -114,6 +117,13 @@ export default async function exportToExploitationDB({data}) { // Prepare fantoir finder from cog and fantoir sqlite database const fantoirFinder = await createFantoirCommune(cog, {fantoirPath: FANTOIR_PATH, withAnciennesCommunes: true}) + // Prepare gazetteer finder to find the ancienne commune + const gazetteerFinder = await createGazetteer({ + cacheEnabled: true, + cacheSize: 50, + dbPath: GAZETTEER_DB_PATH + }) + // Prepare pseudo code voie generator from cog const pseudoCodeVoieGenerator = await createPseudoCodeVoieGenerator(cog) @@ -190,6 +200,7 @@ export default async function exportToExploitationDB({data}) { type: sequelize.QueryTypes.SELECT, transaction, }) + const totalCommonToponymTempTableRecordsResult = Number(commonToponymTempTableCountQueryResult?.[0]?.count) const totalCommonToponymPages = Math.ceil(totalCommonToponymTempTableRecordsResult / PAGE_SIZE) @@ -208,7 +219,10 @@ export default async function exportToExploitationDB({data}) { }) // Format the data and calculate the fantoir code, tiles and postal code const pageDataWithExtraDataCalculation = pageData.map(commonToponym => calculateExtraDataForCommonToponym(commonToponym, cog, fantoirFinder, commonToponymIDFantoirCodeMap)) - const formatedPageDataForLegacy = pageDataWithExtraDataCalculation.map(commonToponym => formatCommonToponymDataForLegacy(commonToponym, district, pseudoCodeVoieGenerator, commonToponymLegacyIDCommonToponymIDMap, commonToponymLegacyIDSet)) + const formatedPageDataForLegacyPromises = pageDataWithExtraDataCalculation + .map(async commonToponym => formatCommonToponymDataForLegacy(commonToponym, {district, pseudoCodeVoieGenerator, commonToponymLegacyIDCommonToponymIDMap, commonToponymLegacyIDSet, gazetteerFinder})) + + const formatedPageDataForLegacy = await Promise.all(formatedPageDataForLegacyPromises) // Insert the data in the temp collection await tempCommonToponymCollection.insertMany(formatedPageDataForLegacy, {ordered: false}) @@ -233,6 +247,7 @@ export default async function exportToExploitationDB({data}) { type: sequelize.QueryTypes.SELECT, transaction, }) + const totalAddressRecords = Number(addressTempTableCountQueryResult?.[0]?.count) const totalAddressPages = Math.ceil(totalAddressRecords / PAGE_SIZE) @@ -251,10 +266,11 @@ export default async function exportToExploitationDB({data}) { // Format the data and calculate the fantoir code, tiles and postal code const pageDataWithExtraDataCalculation = pageData.map(address => calculateExtraDataForAddress(address, cog, commonToponymIDFantoirCodeMap)) - const formatedPageDataForLegacy = pageDataWithExtraDataCalculation - .map(address => formatAddressDataForLegacy(address, district, commonToponymLegacyIDCommonToponymIDMap, addressLegacyIDSet)) + const formatedPageDataForLegacyPromises = pageDataWithExtraDataCalculation + .map(async address => formatAddressDataForLegacy(address, {district, commonToponymLegacyIDCommonToponymIDMap, addressLegacyIDSet, gazetteerFinder})) .filter(Boolean) + const formatedPageDataForLegacy = await Promise.all(formatedPageDataForLegacyPromises) // Insert the data in the temp collection tempAddressCollection.insertMany(formatedPageDataForLegacy, {ordered: false}) } diff --git a/lib/api/consumers/format-to-legacy-helpers.js b/lib/api/consumers/format-to-legacy-helpers.js index 58199d30..f3d46406 100644 --- a/lib/api/consumers/format-to-legacy-helpers.js +++ b/lib/api/consumers/format-to-legacy-helpers.js @@ -69,7 +69,7 @@ export const formatDistrictDataForLegacy = async (district, {totalCommonToponymR } } -export const formatCommonToponymDataForLegacy = (commonToponym, district, pseudoCodeVoieGenerator, commonToponymLegacyIDCommonToponymIDMap, commonToponymLegacyIDSet) => { +export const formatCommonToponymDataForLegacy = async (commonToponym, {district, pseudoCodeVoieGenerator, commonToponymLegacyIDCommonToponymIDMap, commonToponymLegacyIDSet, gazetteerFinder}) => { const {labels: districtLabels, meta: {insee: {cog}}} = district const {id, districtID, geometry, labels, meta, updateDate, addressCount, certifiedAddressCount, bbox, addressBbox} = commonToponym @@ -83,8 +83,20 @@ export const formatCommonToponymDataForLegacy = (commonToponym, district, pseudo const legacyLabelValue = defaultLabel?.value const legacyComplementaryLabels = formatLegacyComplementatyLabels(labels, defaultLabelIsoCode) + // Geographic data + const legacyPosition = { + ...geometry, + coordinates: [round(geometry?.coordinates?.[0]), round(geometry?.coordinates?.[1])] + } + const lon = legacyPosition?.coordinates?.[0] + const lat = legacyPosition?.coordinates?.[1] + const commonToponymBbox = formatBboxForLegacy(bbox) + const commonToponymAddressBbox = formatBboxForLegacy(addressBbox) + + // Old district + const {codeAncienneCommune, nomAncienneCommune} = await calculateLegacyCommuneAncienne(cog, meta, lon, lat, gazetteerFinder) + // Ids - const codeAncienneCommune = meta?.bal?.codeAncienneCommune const legacyCommonToponymFantoirId = meta?.dgfip?.fantoir ? `${cog}_${meta?.dgfip?.fantoir}` : null let legacyCommonToponymId = legacyCommonToponymFantoirId @@ -103,16 +115,7 @@ export const formatCommonToponymDataForLegacy = (commonToponym, district, pseudo // Store all the legacy common toponym id commonToponymLegacyIDSet.add(legacyCommonToponymId) - // Geographic data - const legacyPosition = { - ...geometry, - coordinates: [round(geometry?.coordinates?.[0]), round(geometry?.coordinates?.[1])] - } - const lon = legacyPosition?.coordinates?.[0] - const lat = legacyPosition?.coordinates?.[1] - const commonToponymBbox = formatBboxForLegacy(bbox) - const commonToponymAddressBbox = formatBboxForLegacy(addressBbox) - + // Check if the common toponym is a lieu-dit const isLieuDit = meta?.bal?.isLieuDit if (isLieuDit) { @@ -129,7 +132,7 @@ export const formatCommonToponymDataForLegacy = (commonToponym, district, pseudo codeCommune: cog, nomCommune: districtLegacyLabelValue, codeAncienneCommune, - nomAncienneCommune: meta?.bal?.nomAncienneCommune, + nomAncienneCommune, codePostal: meta?.laposte?.codePostal, parcelles: meta?.cadastre?.ids || [], lon, @@ -152,7 +155,7 @@ export const formatCommonToponymDataForLegacy = (commonToponym, district, pseudo codeCommune: cog, nomCommune: districtLegacyLabelValue, codeAncienneCommune, - nomAncienneCommune: meta?.bal?.nomAncienneCommune, + nomAncienneCommune, nomVoie: legacyLabelValue, nomVoieAlt: legacyComplementaryLabels, sourceNomVoie: 'bal', @@ -170,7 +173,7 @@ export const formatCommonToponymDataForLegacy = (commonToponym, district, pseudo } } -export const formatAddressDataForLegacy = (address, district, commonToponymLegacyIDCommonToponymIDMap, addressLegacyIDSet) => { +export const formatAddressDataForLegacy = async (address, {district, commonToponymLegacyIDCommonToponymIDMap, addressLegacyIDSet, gazetteerFinder}) => { const {meta: {insee: {cog}}} = district const {id, mainCommonToponymID, secondaryCommonToponymIDs, districtID, number, suffix, positions, labels, meta, updateDate, certified, bbox} = address @@ -204,6 +207,9 @@ export const formatAddressDataForLegacy = (address, district, commonToponymLegac const banIdSecondaryCommonToponyms = secondaryCommonToponymIDs && secondaryCommonToponymIDs.length > 0 ? secondaryCommonToponymIDs : null const legacySuffix = suffix ? suffix : null + // Old district + const {codeAncienneCommune, nomAncienneCommune} = await calculateLegacyCommuneAncienne(cog, meta, lon, lat, gazetteerFinder) + return { id: legacyID, cleInterop: legacyInteropKey, @@ -213,8 +219,8 @@ export const formatAddressDataForLegacy = (address, district, commonToponymLegac banIdSecondaryCommonToponyms, idVoie: legacyCommonToponymId, codeCommune: cog, - codeAncienneCommune: meta?.bal?.codeAncienneCommune, - nomAncienneCommune: meta?.bal?.nomAncienneCommune, + codeAncienneCommune, + nomAncienneCommune, numero: number, suffixe: legacySuffix, lieuDitComplementNom: legacyLabelValue, @@ -318,3 +324,23 @@ const getAddressLegacyId = (addressLegacyIDSet, legacyInteropKey, suffix = 0) => return `${legacyInteropKey}` } + +const calculateLegacyCommuneAncienne = async (cog, meta, lon, lat, gazetteerFinder) => { + const codeAncienneCommuneFromBal = meta?.bal?.codeAncienneCommune + const nomAncienneCommuneFromBal = meta?.bal?.nomAncienneCommune + if (codeAncienneCommuneFromBal && nomAncienneCommuneFromBal) { + return {codeAncienneCommune: codeAncienneCommuneFromBal, nomAncienneCommune: nomAncienneCommuneFromBal} + } + + const gazetteerResult = await gazetteerFinder.find({lon, lat}) + if (!gazetteerResult) { + return {} + } + + const {communeAncienne, commune} = gazetteerResult + if (!communeAncienne || commune?.code !== cog) { + return {} + } + + return {codeAncienneCommune: communeAncienne.code, nomAncienneCommune: communeAncienne.nom} +}