Added calcul of old district from gazetteer db when transferring data from postgres to mongo

lib/api/consumers/export-to-exploitation-db-consumer.js

@@ -1,5 +1,6 @@
 import {Transaction} from 'sequelize'
 import {createFantoirCommune} from '@ban-team/fantoir'
+import {createGazetteer} from '@ban-team/gazetteer'
 import {findCodePostal} from 'codes-postaux/full.js'
 import mongo from '../../util/mongo.cjs'
 import {sequelize, District, CommonToponym} from '../../util/sequelize.js'
@@ -16,6 +17,8 @@ const PAGE_SIZE = 100
 // The path to the fantoir sqlite database
 const FANTOIR_PATH = process.env.FANTOIR_PATH || 'data/fantoir.sqlite'
 
+const GAZETTEER_DB_PATH = process.env.GAZETTEER_DB_PATH || 'data/gazetteer.sqlite'
+
 // The min and max zoom levels to use for the tiles
 const TILES_ZOOM_LEVELS = {
   commonToponym: {
@@ -114,6 +117,13 @@ export default async function exportToExploitationDB({data}) {
     // Prepare fantoir finder from cog and fantoir sqlite database
     const fantoirFinder = await createFantoirCommune(cog, {fantoirPath: FANTOIR_PATH, withAnciennesCommunes: true})
 
+    // Prepare gazetteer finder to find the ancienne commune
+    const gazetteerFinder = await createGazetteer({
+      cacheEnabled: true,
+      cacheSize: 50,
+      dbPath: GAZETTEER_DB_PATH
+    })
+
     // Prepare pseudo code voie generator from cog
     const pseudoCodeVoieGenerator = await createPseudoCodeVoieGenerator(cog)
 
@@ -190,6 +200,7 @@ export default async function exportToExploitationDB({data}) {
           type: sequelize.QueryTypes.SELECT,
           transaction,
         })
+
       const totalCommonToponymTempTableRecordsResult = Number(commonToponymTempTableCountQueryResult?.[0]?.count)
       const totalCommonToponymPages = Math.ceil(totalCommonToponymTempTableRecordsResult / PAGE_SIZE)
 
@@ -208,7 +219,10 @@ export default async function exportToExploitationDB({data}) {
             })
           // Format the data and calculate the fantoir code, tiles and postal code
           const pageDataWithExtraDataCalculation = pageData.map(commonToponym => calculateExtraDataForCommonToponym(commonToponym, cog, fantoirFinder, commonToponymIDFantoirCodeMap))
-          const formatedPageDataForLegacy = pageDataWithExtraDataCalculation.map(commonToponym => formatCommonToponymDataForLegacy(commonToponym, district, pseudoCodeVoieGenerator, commonToponymLegacyIDCommonToponymIDMap, commonToponymLegacyIDSet))
+          const formatedPageDataForLegacyPromises = pageDataWithExtraDataCalculation
+            .map(async commonToponym => formatCommonToponymDataForLegacy(commonToponym, {district, pseudoCodeVoieGenerator, commonToponymLegacyIDCommonToponymIDMap, commonToponymLegacyIDSet, gazetteerFinder}))
+
+          const formatedPageDataForLegacy = await Promise.all(formatedPageDataForLegacyPromises)
 
           // Insert the data in the temp collection
           await tempCommonToponymCollection.insertMany(formatedPageDataForLegacy, {ordered: false})
@@ -233,6 +247,7 @@ export default async function exportToExploitationDB({data}) {
           type: sequelize.QueryTypes.SELECT,
           transaction,
         })
+
       const totalAddressRecords = Number(addressTempTableCountQueryResult?.[0]?.count)
       const totalAddressPages = Math.ceil(totalAddressRecords / PAGE_SIZE)
 
@@ -251,10 +266,11 @@ export default async function exportToExploitationDB({data}) {
 
         // Format the data and calculate the fantoir code, tiles and postal code
         const pageDataWithExtraDataCalculation = pageData.map(address => calculateExtraDataForAddress(address, cog, commonToponymIDFantoirCodeMap))
-        const formatedPageDataForLegacy = pageDataWithExtraDataCalculation
-          .map(address => formatAddressDataForLegacy(address, district, commonToponymLegacyIDCommonToponymIDMap, addressLegacyIDSet))
+        const formatedPageDataForLegacyPromises = pageDataWithExtraDataCalculation
+          .map(async address => formatAddressDataForLegacy(address, {district, commonToponymLegacyIDCommonToponymIDMap, addressLegacyIDSet, gazetteerFinder}))
           .filter(Boolean)
 
+        const formatedPageDataForLegacy = await Promise.all(formatedPageDataForLegacyPromises)
         // Insert the data in the temp collection
         tempAddressCollection.insertMany(formatedPageDataForLegacy, {ordered: false})
       }

lib/api/consumers/format-to-legacy-helpers.js

@@ -69,7 +69,7 @@ export const formatDistrictDataForLegacy = async (district, {totalCommonToponymR
   }
 }
 
-export const formatCommonToponymDataForLegacy = (commonToponym, district, pseudoCodeVoieGenerator, commonToponymLegacyIDCommonToponymIDMap, commonToponymLegacyIDSet) => {
+export const formatCommonToponymDataForLegacy = async (commonToponym, {district, pseudoCodeVoieGenerator, commonToponymLegacyIDCommonToponymIDMap, commonToponymLegacyIDSet, gazetteerFinder}) => {
   const {labels: districtLabels, meta: {insee: {cog}}} = district
   const {id, districtID, geometry, labels, meta, updateDate, addressCount, certifiedAddressCount, bbox, addressBbox} = commonToponym
 
@@ -83,8 +83,20 @@ export const formatCommonToponymDataForLegacy = (commonToponym, district, pseudo
   const legacyLabelValue = defaultLabel?.value
   const legacyComplementaryLabels = formatLegacyComplementatyLabels(labels, defaultLabelIsoCode)
 
+  // Geographic data
+  const legacyPosition = {
+    ...geometry,
+    coordinates: [round(geometry?.coordinates?.[0]), round(geometry?.coordinates?.[1])]
+  }
+  const lon = legacyPosition?.coordinates?.[0]
+  const lat = legacyPosition?.coordinates?.[1]
+  const commonToponymBbox = formatBboxForLegacy(bbox)
+  const commonToponymAddressBbox = formatBboxForLegacy(addressBbox)
+
+  // Old district
+  const {codeAncienneCommune, nomAncienneCommune} = await calculateLegacyCommuneAncienne(cog, meta, lon, lat, gazetteerFinder)
+
   // Ids
-  const codeAncienneCommune = meta?.bal?.codeAncienneCommune
   const legacyCommonToponymFantoirId = meta?.dgfip?.fantoir ? `${cog}_${meta?.dgfip?.fantoir}` : null
 
   let legacyCommonToponymId = legacyCommonToponymFantoirId
@@ -103,16 +115,7 @@ export const formatCommonToponymDataForLegacy = (commonToponym, district, pseudo
   // Store all the legacy common toponym id
   commonToponymLegacyIDSet.add(legacyCommonToponymId)
 
-  // Geographic data
-  const legacyPosition = {
-    ...geometry,
-    coordinates: [round(geometry?.coordinates?.[0]), round(geometry?.coordinates?.[1])]
-  }
-  const lon = legacyPosition?.coordinates?.[0]
-  const lat = legacyPosition?.coordinates?.[1]
-  const commonToponymBbox = formatBboxForLegacy(bbox)
-  const commonToponymAddressBbox = formatBboxForLegacy(addressBbox)
-
+  // Check if the common toponym is a lieu-dit
   const isLieuDit = meta?.bal?.isLieuDit
 
   if (isLieuDit) {
@@ -129,7 +132,7 @@ export const formatCommonToponymDataForLegacy = (commonToponym, district, pseudo
       codeCommune: cog,
       nomCommune: districtLegacyLabelValue,
       codeAncienneCommune,
-      nomAncienneCommune: meta?.bal?.nomAncienneCommune,
+      nomAncienneCommune,
       codePostal: meta?.laposte?.codePostal,
       parcelles: meta?.cadastre?.ids || [],
       lon,
@@ -152,7 +155,7 @@ export const formatCommonToponymDataForLegacy = (commonToponym, district, pseudo
     codeCommune: cog,
     nomCommune: districtLegacyLabelValue,
     codeAncienneCommune,
-    nomAncienneCommune: meta?.bal?.nomAncienneCommune,
+    nomAncienneCommune,
     nomVoie: legacyLabelValue,
     nomVoieAlt: legacyComplementaryLabels,
     sourceNomVoie: 'bal',
@@ -170,7 +173,7 @@ export const formatCommonToponymDataForLegacy = (commonToponym, district, pseudo
   }
 }
 
-export const formatAddressDataForLegacy = (address, district, commonToponymLegacyIDCommonToponymIDMap, addressLegacyIDSet) => {
+export const formatAddressDataForLegacy = async (address, {district, commonToponymLegacyIDCommonToponymIDMap, addressLegacyIDSet, gazetteerFinder}) => {
   const {meta: {insee: {cog}}} = district
   const {id, mainCommonToponymID, secondaryCommonToponymIDs, districtID, number, suffix, positions, labels, meta, updateDate, certified, bbox} = address
 
@@ -204,6 +207,9 @@ export const formatAddressDataForLegacy = (address, district, commonToponymLegac
   const banIdSecondaryCommonToponyms = secondaryCommonToponymIDs && secondaryCommonToponymIDs.length > 0 ? secondaryCommonToponymIDs : null
   const legacySuffix = suffix ? suffix : null
 
+  // Old district
+  const {codeAncienneCommune, nomAncienneCommune} = await calculateLegacyCommuneAncienne(cog, meta, lon, lat, gazetteerFinder)
+
   return {
     id: legacyID,
     cleInterop: legacyInteropKey,
@@ -213,8 +219,8 @@ export const formatAddressDataForLegacy = (address, district, commonToponymLegac
     banIdSecondaryCommonToponyms,
     idVoie: legacyCommonToponymId,
     codeCommune: cog,
-    codeAncienneCommune: meta?.bal?.codeAncienneCommune,
-    nomAncienneCommune: meta?.bal?.nomAncienneCommune,
+    codeAncienneCommune,
+    nomAncienneCommune,
     numero: number,
     suffixe: legacySuffix,
     lieuDitComplementNom: legacyLabelValue,
@@ -318,3 +324,23 @@ const getAddressLegacyId = (addressLegacyIDSet, legacyInteropKey, suffix = 0) =>
 
   return `${legacyInteropKey}`
 }
+
+const calculateLegacyCommuneAncienne = async (cog, meta, lon, lat, gazetteerFinder) => {
+  const codeAncienneCommuneFromBal = meta?.bal?.codeAncienneCommune
+  const nomAncienneCommuneFromBal = meta?.bal?.nomAncienneCommune
+  if (codeAncienneCommuneFromBal && nomAncienneCommuneFromBal) {
+    return {codeAncienneCommune: codeAncienneCommuneFromBal, nomAncienneCommune: nomAncienneCommuneFromBal}
+  }
+
+  const gazetteerResult = await gazetteerFinder.find({lon, lat})
+  if (!gazetteerResult) {
+    return {}
+  }
+
+  const {communeAncienne, commune} = gazetteerResult
+  if (!communeAncienne || commune?.code !== cog) {
+    return {}
+  }
+
+  return {codeAncienneCommune: communeAncienne.code, nomAncienneCommune: communeAncienne.nom}
+}