Merge pull request #493 from BaseAdresseNationale/antoineludeau/fix-m…

…igration-file-requirement Fix cp and datanova migration data file requirement
BaseAdresseNationale · Dec 10, 2024 · 7670269 · 7670269
2 parents ee5edf0 + 2094a0f
commit 7670269
Show file tree

Hide file tree

Showing 4 changed files with 84 additions and 84 deletions.
diff --git a/.env.sample b/.env.sample
@@ -11,6 +11,7 @@ POSTGRES_ROOT_PASSWORD=postgres
 POSTGRES_DB=base_adresse_nationale
 POSTGRES_BAN_USER=ban_plateforme
 POSTGRES_BAN_PASSWORD=ban_plateforme
+MIGRATION_DATA_FOLDER_PATH=db-migrations/data
 
 # Redis
 REDIS_URL=redis://127.0.0.1:6379

diff --git a/db-migrations/migrations/20240619135943-init-postal-area-table.cjs b/db-migrations/migrations/20240619135943-init-postal-area-table.cjs
@@ -5,7 +5,8 @@ const {Transform} = require('stream')
 const JSONStream = require('JSONStream')
 
 const {POSTGRES_BAN_USER} = process.env
-const {CP_PATH} = process.env
+const {MIGRATION_DATA_FOLDER_PATH} = process.env
+const CP_FILE_NAME = '20240619135943-contours-postaux.geojson'
 
 /** @type {import('sequelize-cli').Migration} */
 module.exports = {
@@ -76,23 +77,26 @@ module.exports = {
       })
     }
 
-    const stream = fs.createReadStream(CP_PATH)
-      .pipe(JSONStream.parse('features.*'))
-      .pipe(new Transform({
-        objectMode: true,
-        async transform(feature, encoding, callback) {
-          try {
-            await insertFeature(feature)
-            callback()
-          } catch (error) {
-            callback(error)
-          }
-        },
-      }))
-    return new Promise((resolve, reject) => {
-      stream.on('finish', resolve)
-      stream.on('error', reject)
-    })
+    const DATA_FILE_PATH = `${MIGRATION_DATA_FOLDER_PATH}/${CP_FILE_NAME}`
+    if (fs.existsSync(DATA_FILE_PATH)) {
+      const stream = fs.createReadStream(DATA_FILE_PATH)
+        .pipe(JSONStream.parse('features.*'))
+        .pipe(new Transform({
+          objectMode: true,
+          async transform(feature, encoding, callback) {
+            try {
+              await insertFeature(feature)
+              callback()
+            } catch (error) {
+              callback(error)
+            }
+          },
+        }))
+      return new Promise((resolve, reject) => {
+        stream.on('finish', resolve)
+        stream.on('error', reject)
+      })
+    }
   },
 
   async down(queryInterface, _Sequelize) {

diff --git a/db-migrations/migrations/20240621094048-init-datanova-table.cjs b/db-migrations/migrations/20240621094048-init-datanova-table.cjs
@@ -4,7 +4,8 @@ const fs = require('fs')
 const path = require('path')
 const Papa = require('papaparse')
 
-const {DATANOVA_PATH} = process.env
+const {MIGRATION_DATA_FOLDER_PATH} = process.env
+const DATANOVA_FILE_NAME = '20240621094048-datanova.csv'
 
 module.exports = {
   async up(queryInterface, Sequelize) {
@@ -33,7 +34,7 @@ module.exports = {
           allowNull: false,
         },
         libelleAcheminementWithPostalCodes: {
-          type: Sequelize.TEXT,
+          type: Sequelize.JSONB,
           allowNull: false,
         },
         createdAt: {
@@ -48,72 +49,69 @@ module.exports = {
         }
       }, {transaction})
 
-      const csvFilePath = path.resolve(DATANOVA_PATH)
-
-      const csvFileContent = fs.readFileSync(csvFilePath, 'utf8')
-
-      console.log('CSV file read successfully')
-
-      const dataRaw = Papa.parse(csvFileContent, {
-        header: true,
-        transformHeader(name) {
-          switch (name.toLowerCase()) {
-            case 'code_commune_insee':
-              return 'codeInsee'
-            case 'nom_de_la_commune':
-              return 'nomCommune'
-            case 'code_postal':
-              return 'codePostal'
-            case 'libelle_d_acheminement':
-              return 'libelleAcheminement'
-            case 'ligne_5':
-              return 'ligne5'
-            case '_geopoint':
-              return 'geopoint'
-            default:
-              return name
+      const DATA_FILE_PATH = path.resolve(MIGRATION_DATA_FOLDER_PATH, `${DATANOVA_FILE_NAME}`)
+      if (fs.existsSync(DATA_FILE_PATH)) {
+        const csvFilePath = path.resolve(DATA_FILE_PATH)
+
+        const csvFileContent = fs.readFileSync(csvFilePath, 'utf8')
+
+        const dataRaw = Papa.parse(csvFileContent, {
+          header: true,
+          transformHeader(name) {
+            switch (name.toLowerCase()) {
+              case 'code_commune_insee':
+                return 'codeInsee'
+              case 'nom_de_la_commune':
+                return 'nomCommune'
+              case 'code_postal':
+                return 'codePostal'
+              case 'libelle_d_acheminement':
+                return 'libelleAcheminement'
+              case 'ligne_5':
+                return 'ligne5'
+              case '_geopoint':
+                return 'geopoint'
+              default:
+                return name
+            }
+          },
+          skipEmptyLines: true,
+        })
+
+        const inseeDataMap = dataRaw.data.reduce((acc, {codeInsee, codePostal, libelleAcheminement}) => {
+          if (!acc[codeInsee]) {
+            acc[codeInsee] = {
+              inseeCom: codeInsee,
+              postalCodes: new Set(),
+              libelleAcheminementWithPostalCodes: {},
+              createdAt: new Date(),
+              updatedAt: new Date(),
+            }
           }
-        },
-        skipEmptyLines: true,
-      })
-
-      console.log('CSV file parsed successfully')
-
-      const inseeDataMap = dataRaw.data.reduce((acc, {codeInsee, codePostal, libelleAcheminement}) => {
-        if (!acc[codeInsee]) {
-          acc[codeInsee] = {
-            inseeCom: codeInsee,
-            postalCodes: new Set(),
-            libelleAcheminementWithPostalCodes: {},
-            createdAt: new Date(),
-            updatedAt: new Date(),
-          }
-        }
 
-        acc[codeInsee].postalCodes.add(codePostal)
-        if (!acc[codeInsee].libelleAcheminementWithPostalCodes[codePostal]) {
-          acc[codeInsee].libelleAcheminementWithPostalCodes[codePostal] = libelleAcheminement
-        }
+          acc[codeInsee].postalCodes.add(codePostal)
+          if (!acc[codeInsee].libelleAcheminementWithPostalCodes[codePostal]) {
+            acc[codeInsee].libelleAcheminementWithPostalCodes[codePostal] = libelleAcheminement
+          }
 
-        return acc
-      }, {})
+          return acc
+        }, {})
 
-      const formattedData = Object.values(inseeDataMap).map(entry => ({
-        ...entry,
-        postalCodes: [...entry.postalCodes],
-        libelleAcheminementWithPostalCodes: JSON.stringify(entry.libelleAcheminementWithPostalCodes)
-      }))
+        const formattedData = Object.values(inseeDataMap).map(entry => ({
+          ...entry,
+          postalCodes: [...entry.postalCodes],
+          libelleAcheminementWithPostalCodes: JSON.stringify(entry.libelleAcheminementWithPostalCodes)
+        }))
 
-      await queryInterface.bulkInsert({schema: 'external', tableName: 'datanova'}, formattedData, {transaction})
-      console.log('Data inserted successfully into external.datanova table')
+        await queryInterface.bulkInsert({schema: 'external', tableName: 'datanova'}, formattedData, {transaction})
 
-      // Convert the column to JSONB after insertion
-      await queryInterface.sequelize.query(`
-        ALTER TABLE external.datanova 
-        ALTER COLUMN "libelleAcheminementWithPostalCodes" 
-        TYPE JSONB USING "libelleAcheminementWithPostalCodes"::JSONB
-      `, {transaction})
-      console.log('Column libelleAcheminementWithPostalCodes converted to JSONB')
+        // Convert the column to JSONB after insertion
+        await queryInterface.sequelize.query(`
+          ALTER TABLE external.datanova 
+          ALTER COLUMN "libelleAcheminementWithPostalCodes" 
+          TYPE JSONB USING "libelleAcheminementWithPostalCodes"::JSONB
+        `, {transaction})
+      }
 
       await transaction.commit()
     } catch (error) {
@@ -126,10 +124,8 @@ module.exports = {
     const transaction = await queryInterface.sequelize.transaction()
     try {
       await queryInterface.dropTable({schema: 'external', tableName: 'datanova'}, {transaction})
-      console.log('Table external.datanova dropped successfully')
 
       await queryInterface.sequelize.query('DROP SCHEMA IF EXISTS external CASCADE', {transaction})
-      console.log('Schema external dropped successfully')
 
       await transaction.commit()
     } catch (error) {

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -78,8 +78,7 @@ services:
       - FORCE_DOWNLOAD_CONTOUR=
       - FORCE_DOWNLOAD_DATASETS=
       - IS_GENERATE_BANID_ON_ASSEMBLY=${IS_GENERATE_BANID_ON_ASSEMBLY}
-      - CP_PATH=${CP_PATH}
-      - DATANOVA_PATH=${DATANOVA_PATH}
+      - MIGRATION_DATA_FOLDER_PATH=${MIGRATION_DATA_FOLDER_PATH}
     ports:
       - "${PORT:-5000}:5000"
     volumes: