Merge pull request #446 from BaseAdresseNationale/jugurtha/integrer-l…

…es-contours-postaux-dans-pg add temporary tables in export postgres to mongo
BaseAdresseNationale · Jul 16, 2024 · 6a2be3d · 6a2be3d
2 parents c0de0fc + 0b434fe
commit 6a2be3d
Showing 1 changed file with 157 additions and 83 deletions.
diff --git a/lib/api/consumers/export-to-exploitation-db-consumer.js b/lib/api/consumers/export-to-exploitation-db-consumer.js
@@ -36,21 +36,38 @@ const EXPLOITATION_DB_COLLECTION_NAMES = {
 }
 
 // QUERIES
-const commonToponymPageQuery = `
+const createCommonToponymTempTableQuery = tempTableName => `
+  CREATE TEMP TABLE ${tempTableName} AS
   SELECT
     CTV.*
   FROM
-    ban.common_toponym_view AS CTV
+    ban.common_toponym_view_cp AS CTV
   WHERE CTV."districtID" = :districtID
-  OFFSET :offset
-  LIMIT :limit
 `
-const addressPageQuery = `
+
+const createAddressTempTableQuery = tempTableName => `
+  CREATE TEMP TABLE ${tempTableName} AS
   SELECT
     AV.*
   FROM
-    ban.address_view AS AV
+    ban.address_view_cp AS AV
   WHERE AV."districtID" = :districtID
+`
+
+const commonToponymPageQuery = tempTableName => `
+  SELECT
+    *
+  FROM
+  ${tempTableName}
+  OFFSET :offset
+  LIMIT :limit
+`
+
+const addressPageQuery = tempTableName => `
+  SELECT
+    *
+  FROM
+  ${tempTableName}
   OFFSET :offset
   LIMIT :limit
 `
@@ -117,86 +134,118 @@ export default async function exportToExploitationDB({data}) {
     // Delete all data related to the district (legacy and banID)
     await deleteAllLegacyDataRelatedToCOG(cog)
 
-    // CommonToponym
-    // Count the total number of common toponyms and pages to process
-    const totalCommonToponymPages = Math.ceil(totalCommonToponymRecords / PAGE_SIZE)
-
-    const fetchAndExportDataFromCommonToponymPage = async pageNumber => {
-      const offset = (pageNumber - 1) * PAGE_SIZE
-      const [pageData] = await sequelize.query(commonToponymPageQuery, {
-        replacements: {
-          districtID,
-          offset,
-          limit: PAGE_SIZE},
+    // Generate temporary table names based on districtID
+    const tempCommonToponymTableName = `temp_common_toponym_${cog}`
+    const tempAddressTableName = `temp_address_${cog}`
+
+    // Drop temporary tables
+    try {
+      await sequelize.query(`DROP TABLE IF EXISTS ${tempCommonToponymTableName}`, {transaction})
+      await sequelize.query(`DROP TABLE IF EXISTS ${tempAddressTableName}`, {transaction})
+      // Create temporary tables
+      await sequelize.query(createCommonToponymTempTableQuery(tempCommonToponymTableName), {
+        replacements: {districtID},
         transaction,
-        raw: true,
       })
-      // Format the data and calculate the fantoir code, tiles and postal code
-      const pageDataWithExtraDataCalculation = pageData.map(commonToponym => calculateExtraDataForCommonToponym(commonToponym, cog, fantoirFinder, commonToponymIDFantoirCodeMap))
-      const formatedPageDataForLegacy = pageDataWithExtraDataCalculation.map(commonToponym => formatCommonToponymDataForLegacy(commonToponym, district, pseudoCodeVoieGenerator, commonToponymLegacyIDCommonToponymIDMap, commonToponymLegacyIDSet))
-
-      // Insert the data in the collection (legacy and banID)
-      await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.commonToponym).insertMany(formatedPageDataForLegacy, {ordered: false})
-    }
-
-    const commonToponymsExportPromises = []
-    for (let pageNumber = 1; pageNumber <= totalCommonToponymPages; pageNumber++) {
-      commonToponymsExportPromises.push(fetchAndExportDataFromCommonToponymPage(pageNumber))
-    }
-
-    await Promise.all(commonToponymsExportPromises)
-
-    // Address
-    // Count the total number of addresses and pages to process
-    const totalAddressRecords = await Address.count({
-      where: {
-        districtID,
-        isActive: true
-      },
-      transaction,
-    })
-    const totalAddressPages = Math.ceil(totalAddressRecords / PAGE_SIZE)
-
-    const fetchAndExportDataFromAddressPage = async pageNumber => {
-      const offset = (pageNumber - 1) * PAGE_SIZE
-      const [pageData] = await sequelize.query(addressPageQuery, {
-        replacements: {
+      console.log(`Temporary table ${tempCommonToponymTableName} created`)
+      await sequelize.query(createAddressTempTableQuery(tempAddressTableName), {
+        replacements: {districtID},
+        transaction,
+      })
+      console.log(`Temporary table ${tempAddressTableName} created`)
+
+      // CommonToponym
+      // Count the total number of common toponyms and pages to process
+      const totalCommonToponymPages = Math.ceil(totalCommonToponymRecords / PAGE_SIZE)
+
+      const fetchAndExportDataFromCommonToponymPage = async pageNumber => {
+        const offset = (pageNumber - 1) * PAGE_SIZE
+        const [pageData] = await sequelize.query(commonToponymPageQuery(tempCommonToponymTableName), {
+          replacements: {
+            districtID,
+            offset,
+            limit: PAGE_SIZE
+          },
+          transaction,
+          raw: true,
+        })
+        // Format the data and calculate the fantoir code, tiles and postal code
+        const pageDataWithExtraDataCalculation = pageData.map(commonToponym => calculateExtraDataForCommonToponym(commonToponym, cog, fantoirFinder, commonToponymIDFantoirCodeMap))
+        const formatedPageDataForLegacy = pageDataWithExtraDataCalculation.map(commonToponym => formatCommonToponymDataForLegacy(commonToponym, district, pseudoCodeVoieGenerator, commonToponymLegacyIDCommonToponymIDMap, commonToponymLegacyIDSet))
+
+        // Insert the data in the collection (legacy and banID)
+        await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.commonToponym).insertMany(formatedPageDataForLegacy, {ordered: false})
+      }
+
+      const commonToponymsExportPromises = []
+      for (let pageNumber = 1; pageNumber <= totalCommonToponymPages; pageNumber++) {
+        commonToponymsExportPromises.push(fetchAndExportDataFromCommonToponymPage(pageNumber))
+      }
+
+      await Promise.all(commonToponymsExportPromises)
+
+      // Address
+      // Count the total number of addresses and pages to process
+      const totalAddressRecords = await Address.count({
+        where: {
           districtID,
-          offset,
-          limit: PAGE_SIZE},
+          isActive: true
+        },
         transaction,
-        raw: true,
       })
-
-      // Format the data and calculate the fantoir code, tiles and postal code
-      const pageDataWithExtraDataCalculation = pageData.map(address => calculateExtraDataForAddress(address, cog, commonToponymIDFantoirCodeMap))
-      const formatedPageDataForLegacy = pageDataWithExtraDataCalculation.map(address => formatAddressDataForLegacy(address, district, commonToponymLegacyIDCommonToponymIDMap, addressLegacyIDSet))
-
-      // Insert the data in the collection (legacy and banID)
-      await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.address).insertMany(formatedPageDataForLegacy, {ordered: false})
-    }
-
-    const addressesExportPromises = []
-    for (let pageNumber = 1; pageNumber <= totalAddressPages; pageNumber++) {
-      addressesExportPromises.push(fetchAndExportDataFromAddressPage(pageNumber))
+      const totalAddressPages = Math.ceil(totalAddressRecords / PAGE_SIZE)
+
+      const fetchAndExportDataFromAddressPage = async pageNumber => {
+        const offset = (pageNumber - 1) * PAGE_SIZE
+        const [pageData] = await sequelize.query(addressPageQuery(tempAddressTableName), {
+          replacements: {
+            districtID,
+            offset,
+            limit: PAGE_SIZE
+          },
+          transaction,
+          raw: true,
+        })
+
+        // Format the data and calculate the fantoir code, tiles and postal code
+        const pageDataWithExtraDataCalculation = pageData.map(address => calculateExtraDataForAddress(address, cog, commonToponymIDFantoirCodeMap))
+        const formatedPageDataForLegacy = pageDataWithExtraDataCalculation.map(address => formatAddressDataForLegacy(address, district, commonToponymLegacyIDCommonToponymIDMap, addressLegacyIDSet))
+
+        // Insert the data in the collection (legacy and banID)
+        await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.address).insertMany(formatedPageDataForLegacy, {ordered: false})
+      }
+
+      const addressesExportPromises = []
+      for (let pageNumber = 1; pageNumber <= totalAddressPages; pageNumber++) {
+        addressesExportPromises.push(fetchAndExportDataFromAddressPage(pageNumber))
+      }
+
+      await Promise.all(addressesExportPromises)
+
+      // District
+      // For Legacy collections
+      const districtFormatedForLegacy = await formatDistrictDataForLegacy(district, totalCommonToponymRecords, totalAddressRecords, transaction)
+      await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.district).updateOne({codeCommune: cog}, {$set: districtFormatedForLegacy}, {upsert: true})
+
+      // Pseudo code voie generator saving data
+      await pseudoCodeVoieGenerator.save()
+
+      // Drop temporary tables
+      await sequelize.query(`DROP TABLE IF EXISTS ${tempCommonToponymTableName}`, {transaction})
+      await sequelize.query(`DROP TABLE IF EXISTS ${tempAddressTableName}`, {transaction})
+    } catch (error) {
+      await sequelize.query(`DROP TABLE IF EXISTS ${tempCommonToponymTableName}`, {transaction})
+      await sequelize.query(`DROP TABLE IF EXISTS ${tempAddressTableName}`, {transaction})
+      console.error(`Exporting districtID ${districtID} failed: ${error.message}`)
     }
-
-    await Promise.all(addressesExportPromises)
-
-    // District
-    // For Legacy collections
-    const districtFormatedForLegacy = await formatDistrictDataForLegacy(district, totalCommonToponymRecords, totalAddressRecords, transaction)
-    await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.district).updateOne({codeCommune: cog}, {$set: districtFormatedForLegacy}, {upsert: true})
-
-    // Pseudo code voie generator saving data
-    await pseudoCodeVoieGenerator.save()
-
     // Commit the transaction
+
     await transaction.commit()
     console.log(`Exporting districtID ${districtID} done`)
   } catch (error) {
     await transaction.rollback()
-    console.error(`Exporting districtID ${districtID} failed: ${error}`)
+    console.error(`Exporting districtID ${districtID} failed: ${error.message}`)
+    throw error
   }
 }
 
@@ -274,19 +323,44 @@ const calculateFantoirCode = (fantoirFinder, labelValue, codeAncienneCommune) =>
 
 // Helpers to calculate the postal code
 const calculateCommonToponymPostalCode = (commonToponymIDFantoirCodeMap, commonToponym, cog) => {
-  const fantoirCode = commonToponymIDFantoirCodeMap.get(commonToponym.id)
-  const {codePostal, libelleAcheminement} = findCodePostal(cog, fantoirCode)
-  return {codePostal, libelleAcheminement}
+  try {
+    const fantoirCode = commonToponymIDFantoirCodeMap.get(commonToponym.id)
+    let codePostal
+    let libelleAcheminement
+    if (commonToponym.postal_code === null || commonToponym.libelleAcheminement === null || commonToponym.source_cp === 'DGFIP') {
+      ({codePostal, libelleAcheminement} = findCodePostal(cog, fantoirCode))
+    } else {
+      codePostal = commonToponym.postal_code
+      libelleAcheminement = commonToponym.libelleAcheminement
+    }
+
+    return {codePostal, libelleAcheminement}
+  } catch (error) {
+    console.error('Error querying database:', error)
+    throw error
+  }
 }
 
 const calculateAddressPostalCode = (commonToponymIDFantoirCodeMap, address, cog) => {
-  const fantoirCode = commonToponymIDFantoirCodeMap.get(address.mainCommonToponymID)
-  const {number, suffix} = address
-  const {codePostal, libelleAcheminement} = findCodePostal(cog, fantoirCode, number, suffix)
-  return {codePostal, libelleAcheminement}
+  try {
+    const fantoirCode = commonToponymIDFantoirCodeMap.get(address.mainCommonToponymID)
+    const {number, suffix} = address
+    let codePostal
+    let libelleAcheminement
+    if (address.postal_code === null || address.libelleAcheminement === null || address.source_cp === 'DGFIP') {
+      ({codePostal, libelleAcheminement} = findCodePostal(cog, fantoirCode, number, suffix))
+    } else {
+      codePostal = address.postal_code
+      libelleAcheminement = address.libelleAcheminement
+    }
+
+    return {codePostal, libelleAcheminement}
+  } catch (error) {
+    console.error('Error querying database:', error)
+    throw error
+  }
 }
 
-// Helpers to calculate the tiles
 const calculateCommonToponymGeometryAndTiles = commonToponym => {
   const {geometry: geometryFromCommonToponym, centroid} = commonToponym
   let geometryFromCentroid