From 0a36aa09abe85e84e35bcfbeee7845e84cb76934 Mon Sep 17 00:00:00 2001 From: Jugurtha Bouhadoun Date: Fri, 5 Jul 2024 09:53:53 +0200 Subject: [PATCH] use temp table to improve perf --- .../migrations/20240117135326-init-tables.cjs | 7 + ...0240618155810-init-common-toponym-view.cjs | 47 --- .../20240618155829-init-address-view.cjs | 35 --- .../20240625132013-init-postal-code-view.cjs | 93 ++++-- .../export-to-exploitation-db-consumer.js | 268 +++++++++--------- lib/util/sequelize.js | 8 +- 6 files changed, 206 insertions(+), 252 deletions(-) delete mode 100644 db-migrations/migrations/20240618155810-init-common-toponym-view.cjs delete mode 100644 db-migrations/migrations/20240618155829-init-address-view.cjs diff --git a/db-migrations/migrations/20240117135326-init-tables.cjs b/db-migrations/migrations/20240117135326-init-tables.cjs index 864e9584..834e07cf 100644 --- a/db-migrations/migrations/20240117135326-init-tables.cjs +++ b/db-migrations/migrations/20240117135326-init-tables.cjs @@ -225,6 +225,13 @@ module.exports = { // Grant permissions to ban user await queryInterface.sequelize.query(`GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA ban TO "${POSTGRES_BAN_USER}";`) + await queryInterface.sequelize.query(`GRANT USAGE ON SCHEMA ban TO "${POSTGRES_BAN_USER}";`) + await queryInterface.sequelize.query(`GRANT ALL PRIVILEGES ON SCHEMA ban TO "${POSTGRES_BAN_USER}";`) + await queryInterface.sequelize.query(`GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA ban TO "${POSTGRES_BAN_USER}";`) + await queryInterface.sequelize.query(`GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO "${POSTGRES_BAN_USER}";`) + await queryInterface.sequelize.query(`GRANT USAGE ON SCHEMA public TO "${POSTGRES_BAN_USER}";`) + await queryInterface.sequelize.query(`GRANT ALL PRIVILEGES ON SCHEMA public TO "${POSTGRES_BAN_USER}";`) + await queryInterface.sequelize.query(`GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO "${POSTGRES_BAN_USER}";`) } catch (error) { console.error(error) } diff --git a/db-migrations/migrations/20240618155810-init-common-toponym-view.cjs b/db-migrations/migrations/20240618155810-init-common-toponym-view.cjs deleted file mode 100644 index 30750228..00000000 --- a/db-migrations/migrations/20240618155810-init-common-toponym-view.cjs +++ /dev/null @@ -1,47 +0,0 @@ -'use strict' - -const {POSTGRES_BAN_USER} = process.env - -/** @type {import('sequelize-cli').Migration} */ -module.exports = { - async up(queryInterface) { - const addressBboxBuffer = 200 - const bboxBuffer = 100 - try { - // Execute the view creation - await queryInterface.sequelize.query(` - CREATE VIEW ban."common_toponym_view" AS - SELECT - CT.id, CT."districtID", CT.labels, CT.geometry, CT."updateDate", CT.meta, CT.range_validity, CT."isActive", - ST_Centroid(ST_Collect(ST_SetSRID(ST_GeomFromGeoJSON((A.positions[1])->'geometry'), 4326))) AS centroid, - ST_Transform(ST_Buffer(ST_Transform(ST_Envelope(ST_Collect(ST_SetSRID(ST_GeomFromGeoJSON((A.positions[1])->'geometry'), 4326))), 2154), ${addressBboxBuffer}, 'join=mitre endcap=square'), 4326) AS "addressBbox", - ST_Transform(ST_Buffer(ST_Transform(ST_Envelope(ST_SetSRID(ST_GeomFromGeoJSON(CT.geometry), 4326)), 2154), ${bboxBuffer}, 'join=mitre endcap=square'), 4326) AS "bbox", - COUNT(A.id) AS "addressCount", - COUNT(DISTINCT CASE WHEN A.certified = true THEN A.id ELSE NULL END) AS "certifiedAddressCount" - FROM - ban.common_toponym AS CT - LEFT JOIN - ban.address AS A - ON - (CT.id = A."mainCommonToponymID" - OR CT.id = ANY(A."secondaryCommonToponymIDs")) AND A."isActive" = true - WHERE CT."isActive" = true - GROUP BY CT.id - ORDER BY CT.id ASC ` - ) - // Grant permissions to ban user - await queryInterface.sequelize.query(`GRANT SELECT ON ban."common_toponym_view" TO "${POSTGRES_BAN_USER}";`) - } catch (error) { - console.log(error) - } - }, - - async down(queryInterface) { - try { - // Drop the view if it exists - await queryInterface.sequelize.query('DROP VIEW IF EXISTS ban."common_toponym_view" ;') - } catch (error) { - console.log(error) - } - } -} diff --git a/db-migrations/migrations/20240618155829-init-address-view.cjs b/db-migrations/migrations/20240618155829-init-address-view.cjs deleted file mode 100644 index 71145883..00000000 --- a/db-migrations/migrations/20240618155829-init-address-view.cjs +++ /dev/null @@ -1,35 +0,0 @@ -'use strict' - -const {POSTGRES_BAN_USER} = process.env - -/** @type {import('sequelize-cli').Migration} */ -module.exports = { - async up(queryInterface) { - const bboxBuffer = 50 - try { - // Create the address_view - await queryInterface.sequelize.query(` - CREATE VIEW ban."address_view" AS - SELECT - A.*, - ST_Transform(ST_Buffer(ST_Transform(ST_Envelope(ST_SetSRID(ST_GeomFromGeoJSON((A.positions[1])->'geometry'), 4326)), 2154), ${bboxBuffer}, 'join=mitre endcap=square'), 4326) AS bbox - FROM - ban.address AS A - WHERE A."isActive" = true - ORDER BY A.id ASC - `) - await queryInterface.sequelize.query(`GRANT SELECT ON ban."address_view" TO "${POSTGRES_BAN_USER}";`) - } catch (error) { - console.log(error) - } - }, - - async down(queryInterface) { - try { - // Drop the address_view if it exists - await queryInterface.sequelize.query('DROP VIEW IF EXISTS ban."address_view";') - } catch (error) { - console.log(error) - } - } -} diff --git a/db-migrations/migrations/20240625132013-init-postal-code-view.cjs b/db-migrations/migrations/20240625132013-init-postal-code-view.cjs index bed34733..b7dbae7e 100644 --- a/db-migrations/migrations/20240625132013-init-postal-code-view.cjs +++ b/db-migrations/migrations/20240625132013-init-postal-code-view.cjs @@ -5,12 +5,24 @@ const {POSTGRES_BAN_USER} = process.env /** @type {import('sequelize-cli').Migration} */ module.exports = { async up(queryInterface) { + const bboxBufferAdressView = 50 + const addressBboxBuffer = 200 + const bboxBuffer = 100 + try { // Create the address_view await queryInterface.sequelize.query(` CREATE VIEW ban."address_view_cp" AS - WITH postal_codes_array AS ( + WITH address_view AS ( + SELECT + A.*, + ST_Transform(ST_Buffer(ST_Transform(ST_Envelope(ST_SetSRID(ST_GeomFromGeoJSON((A.positions[1])->'geometry'), 4326)), 2154), ${bboxBufferAdressView}, 'join=mitre endcap=square'), 4326) AS bbox + FROM + ban.address AS A + WHERE A."isActive" = true + ORDER BY A.id ASC ), + postal_codes_array AS ( SELECT a.*, array_length(d."postalCodes", 1) AS array_length, @@ -18,7 +30,7 @@ module.exports = { b.meta->'insee'->>'cog' AS insee_com, d."libelleAcheminementWithPostalCodes" FROM - ban.address_view AS a + address_view AS a LEFT JOIN ban.district AS b ON a."districtID" = b.id @@ -69,18 +81,18 @@ module.exports = { pca.postalCodes, pca."libelleAcheminementWithPostalCodes", CASE - WHEN pca.array_length = 1 THEN 'DATANOVA' - WHEN pca.array_length > 1 THEN - CASE - WHEN EXISTS ( - SELECT 1 - FROM external.postal_area AS c - WHERE pca.insee_com = c."inseeCom" - AND ST_Intersects(ST_Transform(pca.bbox, 2154), ST_Transform(c.geometry, 2154)) - ) THEN 'CONTOURS_CP' - ELSE 'DGFIP' - END - ELSE 'DGFIP' + WHEN pca.array_length = 1 THEN 'DATANOVA' + WHEN pca.array_length > 1 THEN + CASE + WHEN EXISTS ( + SELECT 1 + FROM external.postal_area AS c + WHERE pca.insee_com = c."inseeCom" + AND ST_Intersects(ST_Transform(pca.bbox, 2154), ST_Transform(c.geometry, 2154)) + ) THEN 'CONTOURS_CP' + ELSE 'DGFIP' + END + ELSE 'DGFIP' END AS source_cp FROM postal_codes_array AS pca @@ -90,7 +102,26 @@ module.exports = { await queryInterface.sequelize.query(` CREATE VIEW ban."common_toponym_view_cp" AS - WITH postal_codes_array AS ( + WITH common_toponym_view AS( + SELECT + CT.id, CT."districtID", CT.labels, CT.geometry, CT."updateDate", CT.meta, CT.range_validity, CT."isActive", + ST_Centroid(ST_Collect(ST_SetSRID(ST_GeomFromGeoJSON((A.positions[1])->'geometry'), 4326))) AS centroid, + ST_Transform(ST_Buffer(ST_Transform(ST_Envelope(ST_Collect(ST_SetSRID(ST_GeomFromGeoJSON((A.positions[1])->'geometry'), 4326))), 2154), ${addressBboxBuffer}, 'join=mitre endcap=square'), 4326) AS "addressBbox", + ST_Transform(ST_Buffer(ST_Transform(ST_Envelope(ST_SetSRID(ST_GeomFromGeoJSON(CT.geometry), 4326)), 2154), ${bboxBuffer}, 'join=mitre endcap=square'), 4326) AS "bbox", + COUNT(A.id) AS "addressCount", + COUNT(DISTINCT CASE WHEN A.certified = true THEN A.id ELSE NULL END) AS "certifiedAddressCount" + FROM + ban.common_toponym AS CT + LEFT JOIN + ban.address AS A + ON + (CT.id = A."mainCommonToponymID" + OR CT.id = ANY(A."secondaryCommonToponymIDs")) AND A."isActive" = true + WHERE CT."isActive" = true + GROUP BY CT.id + ORDER BY CT.id ASC ), + + postal_codes_array AS ( SELECT ct.*, b.meta->'insee'->>'cog' AS insee_com, @@ -102,7 +133,7 @@ module.exports = { ELSE ct."addressBbox" END AS used_bbox FROM - ban.common_toponym_view AS ct + common_toponym_view AS ct LEFT JOIN ban.district AS b ON ct."districtID" = b.id @@ -144,27 +175,27 @@ module.exports = { SELECT pca."libelleAcheminementWithPostalCodes"->>c."postalCode" FROM external.postal_area AS c WHERE pca.insee_com = c."inseeCom" - ORDER BY ST_Area(ST_Intersection(ST_Transform(pca.used_bbox, 2154), ST_Transform(c.geometry, 2154))) DESC + ORDER BY ST_Area(ST_Intersection(ST_Transform(pca.bbox, 2154), ST_Transform(c.geometry, 2154))) DESC LIMIT 1 ) ELSE NULL END AS "libelleAcheminement", pca.postalCodes, pca."libelleAcheminementWithPostalCodes", - CASE - WHEN pca.array_length = 1 THEN 'DATANOVA' - WHEN pca.array_length > 1 THEN - CASE - WHEN EXISTS ( - SELECT 1 - FROM external.postal_area AS c - WHERE pca.insee_com = c."inseeCom" - AND ST_Intersects(ST_Transform(pca.used_bbox, 2154), ST_Transform(c.geometry, 2154)) - ) THEN 'CONTOURS_CP' - ELSE 'DGFIP' - END - ELSE 'DGFIP' - END AS source_cp, + CASE + WHEN pca.array_length = 1 THEN 'DATANOVA' + WHEN pca.array_length > 1 THEN + CASE + WHEN EXISTS ( + SELECT 1 + FROM external.postal_area AS c + WHERE pca.insee_com = c."inseeCom" + AND ST_Intersects(ST_Transform(pca.used_bbox, 2154), ST_Transform(c.geometry, 2154)) + ) THEN 'CONTOURS_CP' + ELSE 'DGFIP' + END + ELSE 'DGFIP' + END AS source_cp, pca.used_bbox FROM postal_codes_array AS pca diff --git a/lib/api/consumers/export-to-exploitation-db-consumer.js b/lib/api/consumers/export-to-exploitation-db-consumer.js index ac9891cf..9559fc7f 100644 --- a/lib/api/consumers/export-to-exploitation-db-consumer.js +++ b/lib/api/consumers/export-to-exploitation-db-consumer.js @@ -36,21 +36,38 @@ const EXPLOITATION_DB_COLLECTION_NAMES = { } // QUERIES -const commonToponymPageQuery = ` +const createCommonToponymTempTableQuery = tempTableName => ` + CREATE TEMP TABLE ${tempTableName} AS SELECT CTV.* FROM - ban.common_toponym_view AS CTV + ban.common_toponym_view_cp AS CTV WHERE CTV."districtID" = :districtID - OFFSET :offset - LIMIT :limit ` -const addressPageQuery = ` + +const createAddressTempTableQuery = tempTableName => ` + CREATE TEMP TABLE ${tempTableName} AS SELECT AV.* FROM - ban.address_view AS AV + ban.address_view_cp AS AV WHERE AV."districtID" = :districtID +` + +const commonToponymPageQuery = tempTableName => ` + SELECT + * + FROM + ${tempTableName} + OFFSET :offset + LIMIT :limit +` + +const addressPageQuery = tempTableName => ` + SELECT + * + FROM + ${tempTableName} OFFSET :offset LIMIT :limit ` @@ -67,6 +84,11 @@ export default async function exportToExploitationDB({data}) { }) try { + // Then start your transaction + const transaction = await sequelize.transaction({ + isolationLevel: Transaction.ISOLATION_LEVELS.REPEATABLE_READ + }) + // Find the district const district = await District.findOne({ where: {id: districtID}, transaction, @@ -117,86 +139,118 @@ export default async function exportToExploitationDB({data}) { // Delete all data related to the district (legacy and banID) await deleteAllLegacyDataRelatedToCOG(cog) - // CommonToponym - // Count the total number of common toponyms and pages to process - const totalCommonToponymPages = Math.ceil(totalCommonToponymRecords / PAGE_SIZE) - - const fetchAndExportDataFromCommonToponymPage = async pageNumber => { - const offset = (pageNumber - 1) * PAGE_SIZE - const [pageData] = await sequelize.query(commonToponymPageQuery, { - replacements: { - districtID, - offset, - limit: PAGE_SIZE}, + // Generate temporary table names based on districtID + const tempCommonToponymTableName = `temp_common_toponym_${cog}` + const tempAddressTableName = `temp_address_${cog}` + + // Drop temporary tables + try { + await sequelize.query(`DROP TABLE IF EXISTS ${tempCommonToponymTableName}`, {transaction}) + await sequelize.query(`DROP TABLE IF EXISTS ${tempAddressTableName}`, {transaction}) + // Create temporary tables + await sequelize.query(createCommonToponymTempTableQuery(tempCommonToponymTableName), { + replacements: {districtID}, transaction, - raw: true, }) - // Format the data and calculate the fantoir code, tiles and postal code - const pageDataWithExtraDataCalculation = await Promise.all(pageData.map(async commonToponym => calculateExtraDataForCommonToponym(commonToponym, cog, fantoirFinder, commonToponymIDFantoirCodeMap))) - const formatedPageDataForLegacy = pageDataWithExtraDataCalculation.map(commonToponym => formatCommonToponymDataForLegacy(commonToponym, district, pseudoCodeVoieGenerator, commonToponymLegacyIDCommonToponymIDMap, commonToponymLegacyIDSet)) - - // Insert the data in the collection (legacy and banID) - await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.commonToponym).insertMany(formatedPageDataForLegacy, {ordered: false}) - } + console.log(`Temporary table ${tempCommonToponymTableName} created`) + await sequelize.query(createAddressTempTableQuery(tempAddressTableName), { + replacements: {districtID}, + transaction, + }) + console.log(`Temporary table ${tempAddressTableName} created`) + + // CommonToponym + // Count the total number of common toponyms and pages to process + const totalCommonToponymPages = Math.ceil(totalCommonToponymRecords / PAGE_SIZE) + + const fetchAndExportDataFromCommonToponymPage = async pageNumber => { + const offset = (pageNumber - 1) * PAGE_SIZE + const [pageData] = await sequelize.query(commonToponymPageQuery(tempCommonToponymTableName), { + replacements: { + districtID, + offset, + limit: PAGE_SIZE + }, + transaction, + raw: true, + }) + // Format the data and calculate the fantoir code, tiles and postal code + const pageDataWithExtraDataCalculation = pageData.map(commonToponym => calculateExtraDataForCommonToponym(commonToponym, cog, fantoirFinder, commonToponymIDFantoirCodeMap)) + const formatedPageDataForLegacy = pageDataWithExtraDataCalculation.map(commonToponym => formatCommonToponymDataForLegacy(commonToponym, district, pseudoCodeVoieGenerator, commonToponymLegacyIDCommonToponymIDMap, commonToponymLegacyIDSet)) + + // Insert the data in the collection (legacy and banID) + await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.commonToponym).insertMany(formatedPageDataForLegacy, {ordered: false}) + } - const commonToponymsExportPromises = [] - for (let pageNumber = 1; pageNumber <= totalCommonToponymPages; pageNumber++) { - commonToponymsExportPromises.push(fetchAndExportDataFromCommonToponymPage(pageNumber)) - } + const commonToponymsExportPromises = [] + for (let pageNumber = 1; pageNumber <= totalCommonToponymPages; pageNumber++) { + commonToponymsExportPromises.push(fetchAndExportDataFromCommonToponymPage(pageNumber)) + } - await Promise.all(commonToponymsExportPromises) + await Promise.all(commonToponymsExportPromises) - // Address - // Count the total number of addresses and pages to process - const totalAddressRecords = await Address.count({ - where: { - districtID, - isActive: true - }, - transaction, - }) - const totalAddressPages = Math.ceil(totalAddressRecords / PAGE_SIZE) - - const fetchAndExportDataFromAddressPage = async pageNumber => { - const offset = (pageNumber - 1) * PAGE_SIZE - const [pageData] = await sequelize.query(addressPageQuery, { - replacements: { + // Address + // Count the total number of addresses and pages to process + const totalAddressRecords = await Address.count({ + where: { districtID, - offset, - limit: PAGE_SIZE}, + isActive: true + }, transaction, - raw: true, }) + const totalAddressPages = Math.ceil(totalAddressRecords / PAGE_SIZE) + + const fetchAndExportDataFromAddressPage = async pageNumber => { + const offset = (pageNumber - 1) * PAGE_SIZE + const [pageData] = await sequelize.query(addressPageQuery(tempAddressTableName), { + replacements: { + districtID, + offset, + limit: PAGE_SIZE + }, + transaction, + raw: true, + }) + + // Format the data and calculate the fantoir code, tiles and postal code + const pageDataWithExtraDataCalculation = pageData.map(address => calculateExtraDataForAddress(address, cog, commonToponymIDFantoirCodeMap)) + const formatedPageDataForLegacy = pageDataWithExtraDataCalculation.map(address => formatAddressDataForLegacy(address, district, commonToponymLegacyIDCommonToponymIDMap, addressLegacyIDSet)) + + // Insert the data in the collection (legacy and banID) + await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.address).insertMany(formatedPageDataForLegacy, {ordered: false}) + } - // Format the data and calculate the fantoir code, tiles and postal code - const pageDataWithExtraDataCalculation = await Promise.all(pageData.map(async address => calculateExtraDataForAddress(address, cog, commonToponymIDFantoirCodeMap))) - const formatedPageDataForLegacy = pageDataWithExtraDataCalculation.map(address => formatAddressDataForLegacy(address, district, commonToponymLegacyIDCommonToponymIDMap, addressLegacyIDSet)) - - // Insert the data in the collection (legacy and banID) - await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.address).insertMany(formatedPageDataForLegacy, {ordered: false}) - } - - const addressesExportPromises = [] - for (let pageNumber = 1; pageNumber <= totalAddressPages; pageNumber++) { - addressesExportPromises.push(fetchAndExportDataFromAddressPage(pageNumber)) - } + const addressesExportPromises = [] + for (let pageNumber = 1; pageNumber <= totalAddressPages; pageNumber++) { + addressesExportPromises.push(fetchAndExportDataFromAddressPage(pageNumber)) + } - await Promise.all(addressesExportPromises) + await Promise.all(addressesExportPromises) - // District - // For Legacy collections - const districtFormatedForLegacy = await formatDistrictDataForLegacy(district, totalCommonToponymRecords, totalAddressRecords, transaction) - await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.district).updateOne({codeCommune: cog}, {$set: districtFormatedForLegacy}, {upsert: true}) + // District + // For Legacy collections + const districtFormatedForLegacy = await formatDistrictDataForLegacy(district, totalCommonToponymRecords, totalAddressRecords, transaction) + await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.district).updateOne({codeCommune: cog}, {$set: districtFormatedForLegacy}, {upsert: true}) - // Pseudo code voie generator saving data - await pseudoCodeVoieGenerator.save() + // Pseudo code voie generator saving data + await pseudoCodeVoieGenerator.save() + // Drop temporary tables + await sequelize.query(`DROP TABLE IF EXISTS ${tempCommonToponymTableName}`, {transaction}) + await sequelize.query(`DROP TABLE IF EXISTS ${tempAddressTableName}`, {transaction}) + } catch (error) { + await sequelize.query(`DROP TABLE IF EXISTS ${tempCommonToponymTableName}`, {transaction}) + await sequelize.query(`DROP TABLE IF EXISTS ${tempAddressTableName}`, {transaction}) + console.error(`Exporting districtID ${districtID} failed: ${error.message}`) + } // Commit the transaction + await transaction.commit() console.log(`Exporting districtID ${districtID} done`) } catch (error) { await transaction.rollback() - console.error(`Exporting districtID ${districtID} failed: ${error}`) + console.error(`Exporting districtID ${districtID} failed: ${error.message}`) + throw error } } @@ -212,13 +266,13 @@ const deleteAllLegacyDataRelatedToCOG = async cog => { } // Helpers for calculation -export const calculateExtraDataForCommonToponym = async (commonToponym, cog, fantoirFinder, commonToponymIDFantoirCodeMap) => { +export const calculateExtraDataForCommonToponym = (commonToponym, cog, fantoirFinder, commonToponymIDFantoirCodeMap) => { // Calculate the fantoir code for each common toponym const fantoirCode = calculateCommonToponymFantoirCode(commonToponymIDFantoirCodeMap, commonToponym, fantoirFinder) // Calculate the tiles for each common toponym const {geometry, tiles, x, y} = calculateCommonToponymGeometryAndTiles(commonToponym) // Calculate the postal code for each common toponym - const {codePostal: postalCode, libelleAcheminement: deliveryLabel} = await calculateCommonToponymPostalCode(commonToponymIDFantoirCodeMap, commonToponym, cog) + const {codePostal: postalCode, libelleAcheminement: deliveryLabel} = calculateCommonToponymPostalCode(commonToponymIDFantoirCodeMap, commonToponym, cog) // Remove the centroid data from the common toponym return {...commonToponym, geometry, @@ -230,11 +284,11 @@ export const calculateExtraDataForCommonToponym = async (commonToponym, cog, fan }} } -export const calculateExtraDataForAddress = async (address, cog, commonToponymIDFantoirCodeMap) => { +export const calculateExtraDataForAddress = (address, cog, commonToponymIDFantoirCodeMap) => { // Calculate the tiles for each address const {tiles, x, y} = calculateAddressTiles(address) // Calculate the postal code for each address - const {codePostal: postalCode, libelleAcheminement: deliveryLabel} = await calculateAddressPostalCode(commonToponymIDFantoirCodeMap, address, cog) + const {codePostal: postalCode, libelleAcheminement: deliveryLabel} = calculateAddressPostalCode(commonToponymIDFantoirCodeMap, address, cog) return {...address, meta: { ...address.meta, @@ -272,41 +326,17 @@ const calculateFantoirCode = (fantoirFinder, labelValue, codeAncienneCommune) => return fantoirData.successeur?.split('-')?.[1] || fantoirData.codeFantoir } -const queryCalculateCommonToponymPostalCode = async replacements => { - const query = ` - SELECT - postal_code,"libelleAcheminement",source_cp - FROM - ban.common_toponym_view_cp - WHERE id = :id - LIMIT 1 - ` - try { - const result = await sequelize.query(query, {replacements}) - return result[0] // Assuming the first result is what you need - } catch (error) { - console.error('Error querying database:', error) - throw error - } -} - // Helpers to calculate the postal code -const calculateCommonToponymPostalCode = async (commonToponymIDFantoirCodeMap, commonToponym, cog) => { +const calculateCommonToponymPostalCode = (commonToponymIDFantoirCodeMap, commonToponym, cog) => { try { - const result = await queryCalculateCommonToponymPostalCode({id: commonToponym.id}) const fantoirCode = commonToponymIDFantoirCodeMap.get(commonToponym.id) let codePostal let libelleAcheminement - if (result.length > 0) { - const firstResult = result[0] - if (firstResult.postal_code === null || firstResult.libelleAcheminement === null || firstResult.source_cp === 'DGFIP') { - ({codePostal, libelleAcheminement} = findCodePostal(cog, fantoirCode)) - } else { - codePostal = firstResult.postal_code - libelleAcheminement = firstResult.libelleAcheminement - } - } else { + if (commonToponym.postal_code === null || commonToponym.libelleAcheminement === null || commonToponym.source_cp === 'DGFIP') { ({codePostal, libelleAcheminement} = findCodePostal(cog, fantoirCode)) + } else { + codePostal = commonToponym.postal_code + libelleAcheminement = commonToponym.libelleAcheminement } return {codePostal, libelleAcheminement} @@ -316,43 +346,17 @@ const calculateCommonToponymPostalCode = async (commonToponymIDFantoirCodeMap, c } } -const queryCalculateAddressPostalCode = async replacements => { - const query = ` - SELECT - postal_code,"libelleAcheminement",source_cp - FROM - ban.address_view_cp - WHERE id = :id - LIMIT 1 - ` +const calculateAddressPostalCode = (commonToponymIDFantoirCodeMap, address, cog) => { try { - const result = await sequelize.query(query, {replacements}) - return result[0] // Assuming the first result is what you need - } catch (error) { - console.error('Error querying database:', error) - throw error - } -} - -const calculateAddressPostalCode = async (commonToponymIDFantoirCodeMap, address, cog) => { - try { - const result = await queryCalculateAddressPostalCode({id: address.id}) const fantoirCode = commonToponymIDFantoirCodeMap.get(address.mainCommonToponymID) const {number, suffix} = address - let codePostal let libelleAcheminement - - if (result.length > 0) { - const firstResult = result[0] - if (firstResult.postal_code === null || firstResult.libelleAcheminement === null || firstResult.source_cp === 'DGFIP') { - ({codePostal, libelleAcheminement} = findCodePostal(cog, fantoirCode, number, suffix)) - } else { - codePostal = firstResult.postal_code - libelleAcheminement = firstResult.libelleAcheminement - } - } else { + if (address.postal_code === null || address.libelleAcheminement === null || address.source_cp === 'DGFIP') { ({codePostal, libelleAcheminement} = findCodePostal(cog, fantoirCode, number, suffix)) + } else { + codePostal = address.postal_code + libelleAcheminement = address.libelleAcheminement } return {codePostal, libelleAcheminement} diff --git a/lib/util/sequelize.js b/lib/util/sequelize.js index d2dc3fd1..835fd8c3 100644 --- a/lib/util/sequelize.js +++ b/lib/util/sequelize.js @@ -10,13 +10,7 @@ export const sequelize = new Sequelize(POSTGRES_DB, POSTGRES_BAN_USER, POSTGRES_ host: POSTGRES_URL, port: POSTGRES_PORT, dialect: 'postgres', - logging: false, - pool: { - max: 10, - min: 0, - acquire: 60000, - idle: 5000 - } + logging: false }) /* eslint-enable */