diff --git a/.env.sample b/.env.sample index 0272c40f..efe994b0 100644 --- a/.env.sample +++ b/.env.sample @@ -11,6 +11,7 @@ POSTGRES_ROOT_PASSWORD=postgres POSTGRES_DB=base_adresse_nationale POSTGRES_BAN_USER=ban_plateforme POSTGRES_BAN_PASSWORD=ban_plateforme +MIGRATION_DATA_FOLDER_PATH=db-migrations/data # Redis REDIS_URL=redis://127.0.0.1:6379 diff --git a/db-migrations/migrations/20240619135943-init-postal-area-table.cjs b/db-migrations/migrations/20240619135943-init-postal-area-table.cjs index 2f93af3e..4d4f6fa9 100644 --- a/db-migrations/migrations/20240619135943-init-postal-area-table.cjs +++ b/db-migrations/migrations/20240619135943-init-postal-area-table.cjs @@ -5,7 +5,8 @@ const {Transform} = require('stream') const JSONStream = require('JSONStream') const {POSTGRES_BAN_USER} = process.env -const {CP_PATH} = process.env +const {MIGRATION_DATA_FOLDER_PATH} = process.env +const CP_FILE_NAME = '20240619135943-contours-postaux.geojson' /** @type {import('sequelize-cli').Migration} */ module.exports = { @@ -76,23 +77,26 @@ module.exports = { }) } - const stream = fs.createReadStream(CP_PATH) - .pipe(JSONStream.parse('features.*')) - .pipe(new Transform({ - objectMode: true, - async transform(feature, encoding, callback) { - try { - await insertFeature(feature) - callback() - } catch (error) { - callback(error) - } - }, - })) - return new Promise((resolve, reject) => { - stream.on('finish', resolve) - stream.on('error', reject) - }) + const DATA_FILE_PATH = `${MIGRATION_DATA_FOLDER_PATH}/${CP_FILE_NAME}` + if (fs.existsSync(DATA_FILE_PATH)) { + const stream = fs.createReadStream(DATA_FILE_PATH) + .pipe(JSONStream.parse('features.*')) + .pipe(new Transform({ + objectMode: true, + async transform(feature, encoding, callback) { + try { + await insertFeature(feature) + callback() + } catch (error) { + callback(error) + } + }, + })) + return new Promise((resolve, reject) => { + stream.on('finish', resolve) + stream.on('error', reject) + }) + } }, async down(queryInterface, _Sequelize) { diff --git a/db-migrations/migrations/20240621094048-init-datanova-table.cjs b/db-migrations/migrations/20240621094048-init-datanova-table.cjs index 31c7d248..1f4f3e1e 100644 --- a/db-migrations/migrations/20240621094048-init-datanova-table.cjs +++ b/db-migrations/migrations/20240621094048-init-datanova-table.cjs @@ -4,7 +4,8 @@ const fs = require('fs') const path = require('path') const Papa = require('papaparse') -const {DATANOVA_PATH} = process.env +const {MIGRATION_DATA_FOLDER_PATH} = process.env +const DATANOVA_FILE_NAME = '20240621094048-datanova.csv' module.exports = { async up(queryInterface, Sequelize) { @@ -33,7 +34,7 @@ module.exports = { allowNull: false, }, libelleAcheminementWithPostalCodes: { - type: Sequelize.TEXT, + type: Sequelize.JSONB, allowNull: false, }, createdAt: { @@ -48,72 +49,69 @@ module.exports = { } }, {transaction}) - const csvFilePath = path.resolve(DATANOVA_PATH) - - const csvFileContent = fs.readFileSync(csvFilePath, 'utf8') - - console.log('CSV file read successfully') - - const dataRaw = Papa.parse(csvFileContent, { - header: true, - transformHeader(name) { - switch (name.toLowerCase()) { - case 'code_commune_insee': - return 'codeInsee' - case 'nom_de_la_commune': - return 'nomCommune' - case 'code_postal': - return 'codePostal' - case 'libelle_d_acheminement': - return 'libelleAcheminement' - case 'ligne_5': - return 'ligne5' - case '_geopoint': - return 'geopoint' - default: - return name + const DATA_FILE_PATH = path.resolve(MIGRATION_DATA_FOLDER_PATH, `${DATANOVA_FILE_NAME}`) + if (fs.existsSync(DATA_FILE_PATH)) { + const csvFilePath = path.resolve(DATA_FILE_PATH) + + const csvFileContent = fs.readFileSync(csvFilePath, 'utf8') + + const dataRaw = Papa.parse(csvFileContent, { + header: true, + transformHeader(name) { + switch (name.toLowerCase()) { + case 'code_commune_insee': + return 'codeInsee' + case 'nom_de_la_commune': + return 'nomCommune' + case 'code_postal': + return 'codePostal' + case 'libelle_d_acheminement': + return 'libelleAcheminement' + case 'ligne_5': + return 'ligne5' + case '_geopoint': + return 'geopoint' + default: + return name + } + }, + skipEmptyLines: true, + }) + + const inseeDataMap = dataRaw.data.reduce((acc, {codeInsee, codePostal, libelleAcheminement}) => { + if (!acc[codeInsee]) { + acc[codeInsee] = { + inseeCom: codeInsee, + postalCodes: new Set(), + libelleAcheminementWithPostalCodes: {}, + createdAt: new Date(), + updatedAt: new Date(), + } } - }, - skipEmptyLines: true, - }) - - console.log('CSV file parsed successfully') - - const inseeDataMap = dataRaw.data.reduce((acc, {codeInsee, codePostal, libelleAcheminement}) => { - if (!acc[codeInsee]) { - acc[codeInsee] = { - inseeCom: codeInsee, - postalCodes: new Set(), - libelleAcheminementWithPostalCodes: {}, - createdAt: new Date(), - updatedAt: new Date(), - } - } - acc[codeInsee].postalCodes.add(codePostal) - if (!acc[codeInsee].libelleAcheminementWithPostalCodes[codePostal]) { - acc[codeInsee].libelleAcheminementWithPostalCodes[codePostal] = libelleAcheminement - } + acc[codeInsee].postalCodes.add(codePostal) + if (!acc[codeInsee].libelleAcheminementWithPostalCodes[codePostal]) { + acc[codeInsee].libelleAcheminementWithPostalCodes[codePostal] = libelleAcheminement + } - return acc - }, {}) + return acc + }, {}) - const formattedData = Object.values(inseeDataMap).map(entry => ({ - ...entry, - postalCodes: [...entry.postalCodes], - libelleAcheminementWithPostalCodes: JSON.stringify(entry.libelleAcheminementWithPostalCodes) - })) + const formattedData = Object.values(inseeDataMap).map(entry => ({ + ...entry, + postalCodes: [...entry.postalCodes], + libelleAcheminementWithPostalCodes: JSON.stringify(entry.libelleAcheminementWithPostalCodes) + })) - await queryInterface.bulkInsert({schema: 'external', tableName: 'datanova'}, formattedData, {transaction}) - console.log('Data inserted successfully into external.datanova table') + await queryInterface.bulkInsert({schema: 'external', tableName: 'datanova'}, formattedData, {transaction}) - // Convert the column to JSONB after insertion - await queryInterface.sequelize.query(` - ALTER TABLE external.datanova - ALTER COLUMN "libelleAcheminementWithPostalCodes" - TYPE JSONB USING "libelleAcheminementWithPostalCodes"::JSONB - `, {transaction}) - console.log('Column libelleAcheminementWithPostalCodes converted to JSONB') + // Convert the column to JSONB after insertion + await queryInterface.sequelize.query(` + ALTER TABLE external.datanova + ALTER COLUMN "libelleAcheminementWithPostalCodes" + TYPE JSONB USING "libelleAcheminementWithPostalCodes"::JSONB + `, {transaction}) + } await transaction.commit() } catch (error) { @@ -126,10 +124,8 @@ module.exports = { const transaction = await queryInterface.sequelize.transaction() try { await queryInterface.dropTable({schema: 'external', tableName: 'datanova'}, {transaction}) - console.log('Table external.datanova dropped successfully') await queryInterface.sequelize.query('DROP SCHEMA IF EXISTS external CASCADE', {transaction}) - console.log('Schema external dropped successfully') await transaction.commit() } catch (error) { diff --git a/docker-compose.yml b/docker-compose.yml index 5292740f..cc96ac2d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -78,8 +78,7 @@ services: - FORCE_DOWNLOAD_CONTOUR= - FORCE_DOWNLOAD_DATASETS= - IS_GENERATE_BANID_ON_ASSEMBLY=${IS_GENERATE_BANID_ON_ASSEMBLY} - - CP_PATH=${CP_PATH} - - DATANOVA_PATH=${DATANOVA_PATH} + - MIGRATION_DATA_FOLDER_PATH=${MIGRATION_DATA_FOLDER_PATH} ports: - "${PORT:-5000}:5000" volumes: