From cbcef64158d1b7bee88c38d48ce99827f8e96bf8 Mon Sep 17 00:00:00 2001 From: Jugurtha Bouhadoun Date: Tue, 18 Jun 2024 18:17:16 +0200 Subject: [PATCH 1/4] simplify sql request and add view --- ...0240618155810-init-common-toponym-view.cjs | 47 +++++++++++++++++ .../20240618155829-init-address-view.cjs | 35 +++++++++++++ .../export-to-exploitation-db-consumer.js | 51 +++++-------------- 3 files changed, 96 insertions(+), 37 deletions(-) create mode 100644 db-migrations/migrations/20240618155810-init-common-toponym-view.cjs create mode 100644 db-migrations/migrations/20240618155829-init-address-view.cjs diff --git a/db-migrations/migrations/20240618155810-init-common-toponym-view.cjs b/db-migrations/migrations/20240618155810-init-common-toponym-view.cjs new file mode 100644 index 00000000..195c05b5 --- /dev/null +++ b/db-migrations/migrations/20240618155810-init-common-toponym-view.cjs @@ -0,0 +1,47 @@ +'use strict' + +const {POSTGRES_BAN_USER} = process.env + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface) { + const addressBboxBuffer = 200; + const bboxBuffer = 100; + try { + // Execute the view creation + await queryInterface.sequelize.query(` + CREATE VIEW ban."common_toponym_view" AS + SELECT + CT.id, CT."districtID", CT.labels, CT.geometry, CT."updateDate", CT.meta, CT.range_validity, CT."isActive", + ST_Centroid(ST_Collect(ST_SetSRID(ST_GeomFromGeoJSON((A.positions[1])->'geometry'), 4326))) AS centroid, + ST_Transform(ST_Buffer(ST_Transform(ST_Envelope(ST_Collect(ST_SetSRID(ST_GeomFromGeoJSON((A.positions[1])->'geometry'), 4326))), 2154), ${addressBboxBuffer}, 'join=mitre endcap=square'), 4326) AS "addressBbox", + ST_Transform(ST_Buffer(ST_Transform(ST_Envelope(ST_SetSRID(ST_GeomFromGeoJSON(CT.geometry), 4326)), 2154), ${bboxBuffer}, 'join=mitre endcap=square'), 4326) AS "bbox", + COUNT(A.id) AS "addressCount", + COUNT(DISTINCT CASE WHEN A.certified = true THEN A.id ELSE NULL END) AS "certifiedAddressCount" + FROM + ban.common_toponym AS CT + LEFT JOIN + ban.address AS A + ON + (CT.id = A."mainCommonToponymID" + OR CT.id = ANY(A."secondaryCommonToponymIDs")) AND A."isActive" = true + WHERE CT."isActive" = true + GROUP BY CT.id + ORDER BY CT.id ASC ` + ) + // Grant permissions to ban user + await queryInterface.sequelize.query(`GRANT SELECT ON ban."common_toponym_view" TO "${POSTGRES_BAN_USER}";`) + } catch (error) { + console.log(error); + } + }, + + async down(queryInterface) { + try { + // Drop the view if it exists + await queryInterface.sequelize.query('DROP VIEW IF EXISTS ban."common_toponym_view" ;'); + } catch (error) { + console.log(error); + } + } +} \ No newline at end of file diff --git a/db-migrations/migrations/20240618155829-init-address-view.cjs b/db-migrations/migrations/20240618155829-init-address-view.cjs new file mode 100644 index 00000000..290c97b2 --- /dev/null +++ b/db-migrations/migrations/20240618155829-init-address-view.cjs @@ -0,0 +1,35 @@ +'use strict' + +const {POSTGRES_BAN_USER} = process.env + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface) { + const bboxBuffer = 50; + try { + // Create the address_view + await queryInterface.sequelize.query(` + CREATE VIEW ban."address_view" AS + SELECT + A.*, + ST_Transform(ST_Buffer(ST_Transform(ST_Envelope(ST_SetSRID(ST_GeomFromGeoJSON((A.positions[1])->'geometry'), 4326)), 2154), ${bboxBuffer}, 'join=mitre endcap=square'), 4326) AS bbox + FROM + ban.address AS A + WHERE A."isActive" = true + ORDER BY A.id ASC + `) + await queryInterface.sequelize.query(`GRANT SELECT ON ban."address_view" TO "${POSTGRES_BAN_USER}";`); + } catch (error) { + console.log(error); + } + }, + + async down(queryInterface) { + try { + // Drop the address_view if it exists + await queryInterface.sequelize.query('DROP VIEW IF EXISTS ban."address_view";'); + } catch (error) { + console.log(error); + } + } +} \ No newline at end of file diff --git a/lib/api/consumers/export-to-exploitation-db-consumer.js b/lib/api/consumers/export-to-exploitation-db-consumer.js index 03a877f5..76ab3c2f 100644 --- a/lib/api/consumers/export-to-exploitation-db-consumer.js +++ b/lib/api/consumers/export-to-exploitation-db-consumer.js @@ -41,46 +41,22 @@ const EXPLOITATION_DB_COLLECTION_NAMES = { address: 'numeros' } -// QUERIES & POSTGIS FUNCTIONS -// The queries are written in raw SQL to be able to use the PostGIS functions -// centroid: It calculates the centroid of a collection of geometries extracted from the "positions" column in the "address" table. -// bbox: It calculates a bounding box (envelope) for a collection of geometries from the "positions" column in the "address" table. -// The bbox result is transformed to a different coordinate system (2154 to 4326) and includes a buffer operation. - +// QUERIES const commonToponymPageQuery = ` SELECT - CT.id, CT."districtID", CT.labels, CT.geometry, CT."updateDate", CT.meta, CT.range_validity, CT."isActive", - ST_Centroid(ST_Collect(ST_SetSRID(ST_GeomFromGeoJSON((A.positions[1])->'geometry'), 4326))) AS centroid, - ST_Transform(ST_Buffer(ST_Transform(ST_Envelope(ST_Collect(ST_SetSRID(ST_GeomFromGeoJSON((A.positions[1])->'geometry'), 4326))), 2154), :addressBboxBuffer, 'join=mitre endcap=square'), 4326) AS "addressBbox", - ST_Transform(ST_Buffer(ST_Transform(ST_Envelope(ST_SetSRID(ST_GeomFromGeoJSON(CT.geometry), 4326)), 2154), :bboxBuffer, 'join=mitre endcap=square'), 4326) AS "bbox", - COUNT(A.id) AS "addressCount", - COUNT(DISTINCT CASE WHEN a.certified = true THEN a.id ELSE NULL END) AS "certifiedAddressCount" + CTV.* FROM - ban.common_toponym AS CT - LEFT JOIN - ban.address AS A - ON - (CT.id = A."mainCommonToponymID" - OR CT.id = ANY(A."secondaryCommonToponymIDs")) AND A."isActive" = true - WHERE CT."districtID" = :districtID AND CT."isActive" = true - GROUP BY CT.id - ORDER BY CT.id ASC + ban.common_toponym_view AS CTV + WHERE CTV."districtID" = :districtID OFFSET :offset LIMIT :limit ` - -// The queries are written in raw SQL to be able to use the PostGIS functions -// bbox: It calculates a bounding box (envelope) for the geometry contained in the "positions" column. -// The result is transformed from one coordinate system (2154) to another (4326) -// and includes a buffer operation with a distance of 50 units and specific parameters for joining and capping. const addressPageQuery = ` SELECT - A.*, - ST_Transform(ST_Buffer(ST_Transform(ST_Envelope(ST_SetSRID(ST_GeomFromGeoJSON((A.positions[1])->'geometry'), 4326)), 2154), :bboxBuffer, 'join=mitre endcap=square'), 4326) AS bbox + AV.* FROM - ban.address AS A - WHERE A."districtID" = :districtID AND A."isActive" = true - ORDER BY A.id ASC + ban.address_view AS AV + WHERE AV."districtID" = :districtID OFFSET :offset LIMIT :limit ` @@ -128,6 +104,10 @@ export default async function exportToExploitationDB({data}) { // Set to store the legacy address ID to avoid duplicates const addressLegacyIDSet = new Set() + // Clean collections + // Delete all data related to the district (legacy and banID) + await deleteAllLegacyDataRelatedToCOG(cog) + // Check if the district has data to export // To-fix: use the district configuration instead to check if the district has to be exported const totalCommonToponymRecords = await CommonToponym.count({ @@ -158,9 +138,7 @@ export default async function exportToExploitationDB({data}) { replacements: { districtID, offset, - limit: PAGE_SIZE, - addressBboxBuffer: COMMON_TOPONYM_BBOX_BUFFER, - bboxBuffer: SPECIFIC_COMMON_TOPONYM_BBOX_BUFFER}, + limit: PAGE_SIZE}, transaction, raw: true, }) @@ -196,8 +174,7 @@ export default async function exportToExploitationDB({data}) { replacements: { districtID, offset, - limit: PAGE_SIZE, - bboxBuffer: ADDRESS_BBOX_BUFFER}, + limit: PAGE_SIZE}, transaction, raw: true, }) @@ -344,4 +321,4 @@ const calculateAddressTiles = address => { const {positions} = address const {tiles, x, y} = derivePositionProps(positions?.[0].geometry, TILES_ZOOM_LEVELS.address.min, TILES_ZOOM_LEVELS.address.max) return {tiles, x, y} -} +} \ No newline at end of file From d5f1f31287a60890b62eab9785ce318ff24b181e Mon Sep 17 00:00:00 2001 From: Jugurtha Bouhadoun Date: Tue, 18 Jun 2024 18:54:26 +0200 Subject: [PATCH 2/4] fix xo error --- .../20240618155810-init-common-toponym-view.cjs | 12 ++++++------ .../migrations/20240618155829-init-address-view.cjs | 12 ++++++------ .../consumers/export-to-exploitation-db-consumer.js | 8 +------- 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/db-migrations/migrations/20240618155810-init-common-toponym-view.cjs b/db-migrations/migrations/20240618155810-init-common-toponym-view.cjs index 195c05b5..30750228 100644 --- a/db-migrations/migrations/20240618155810-init-common-toponym-view.cjs +++ b/db-migrations/migrations/20240618155810-init-common-toponym-view.cjs @@ -5,8 +5,8 @@ const {POSTGRES_BAN_USER} = process.env /** @type {import('sequelize-cli').Migration} */ module.exports = { async up(queryInterface) { - const addressBboxBuffer = 200; - const bboxBuffer = 100; + const addressBboxBuffer = 200 + const bboxBuffer = 100 try { // Execute the view creation await queryInterface.sequelize.query(` @@ -32,16 +32,16 @@ module.exports = { // Grant permissions to ban user await queryInterface.sequelize.query(`GRANT SELECT ON ban."common_toponym_view" TO "${POSTGRES_BAN_USER}";`) } catch (error) { - console.log(error); + console.log(error) } }, async down(queryInterface) { try { // Drop the view if it exists - await queryInterface.sequelize.query('DROP VIEW IF EXISTS ban."common_toponym_view" ;'); + await queryInterface.sequelize.query('DROP VIEW IF EXISTS ban."common_toponym_view" ;') } catch (error) { - console.log(error); + console.log(error) } } -} \ No newline at end of file +} diff --git a/db-migrations/migrations/20240618155829-init-address-view.cjs b/db-migrations/migrations/20240618155829-init-address-view.cjs index 290c97b2..71145883 100644 --- a/db-migrations/migrations/20240618155829-init-address-view.cjs +++ b/db-migrations/migrations/20240618155829-init-address-view.cjs @@ -5,7 +5,7 @@ const {POSTGRES_BAN_USER} = process.env /** @type {import('sequelize-cli').Migration} */ module.exports = { async up(queryInterface) { - const bboxBuffer = 50; + const bboxBuffer = 50 try { // Create the address_view await queryInterface.sequelize.query(` @@ -18,18 +18,18 @@ module.exports = { WHERE A."isActive" = true ORDER BY A.id ASC `) - await queryInterface.sequelize.query(`GRANT SELECT ON ban."address_view" TO "${POSTGRES_BAN_USER}";`); + await queryInterface.sequelize.query(`GRANT SELECT ON ban."address_view" TO "${POSTGRES_BAN_USER}";`) } catch (error) { - console.log(error); + console.log(error) } }, async down(queryInterface) { try { // Drop the address_view if it exists - await queryInterface.sequelize.query('DROP VIEW IF EXISTS ban."address_view";'); + await queryInterface.sequelize.query('DROP VIEW IF EXISTS ban."address_view";') } catch (error) { - console.log(error); + console.log(error) } } -} \ No newline at end of file +} diff --git a/lib/api/consumers/export-to-exploitation-db-consumer.js b/lib/api/consumers/export-to-exploitation-db-consumer.js index 76ab3c2f..cbed5e39 100644 --- a/lib/api/consumers/export-to-exploitation-db-consumer.js +++ b/lib/api/consumers/export-to-exploitation-db-consumer.js @@ -28,12 +28,6 @@ const TILES_ZOOM_LEVELS = { } } -// The buffer distance to use for the bbox calculation -const COMMON_TOPONYM_BBOX_BUFFER = 200 -// For specific common toponyms (='lieu-dit'), we use a different buffer distance -const SPECIFIC_COMMON_TOPONYM_BBOX_BUFFER = 100 -const ADDRESS_BBOX_BUFFER = 50 - // Collections names const EXPLOITATION_DB_COLLECTION_NAMES = { district: 'communes', @@ -321,4 +315,4 @@ const calculateAddressTiles = address => { const {positions} = address const {tiles, x, y} = derivePositionProps(positions?.[0].geometry, TILES_ZOOM_LEVELS.address.min, TILES_ZOOM_LEVELS.address.max) return {tiles, x, y} -} \ No newline at end of file +} From 7a2a0d4eac701584f0a0268066aad5425fcfd930 Mon Sep 17 00:00:00 2001 From: Jugurtha Bouhadoun Date: Mon, 24 Jun 2024 11:13:43 +0200 Subject: [PATCH 3/4] delete double --- lib/api/consumers/export-to-exploitation-db-consumer.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/api/consumers/export-to-exploitation-db-consumer.js b/lib/api/consumers/export-to-exploitation-db-consumer.js index cbed5e39..4b5149eb 100644 --- a/lib/api/consumers/export-to-exploitation-db-consumer.js +++ b/lib/api/consumers/export-to-exploitation-db-consumer.js @@ -97,11 +97,7 @@ export default async function exportToExploitationDB({data}) { // Set to store the legacy address ID to avoid duplicates const addressLegacyIDSet = new Set() - - // Clean collections - // Delete all data related to the district (legacy and banID) - await deleteAllLegacyDataRelatedToCOG(cog) - + // Check if the district has data to export // To-fix: use the district configuration instead to check if the district has to be exported const totalCommonToponymRecords = await CommonToponym.count({ From 50d5461e7fedadbc9539b7558a681d125b700175 Mon Sep 17 00:00:00 2001 From: Jugurtha Bouhadoun Date: Mon, 24 Jun 2024 11:16:15 +0200 Subject: [PATCH 4/4] fix xo error --- lib/api/consumers/export-to-exploitation-db-consumer.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/api/consumers/export-to-exploitation-db-consumer.js b/lib/api/consumers/export-to-exploitation-db-consumer.js index 4b5149eb..f7de0dd0 100644 --- a/lib/api/consumers/export-to-exploitation-db-consumer.js +++ b/lib/api/consumers/export-to-exploitation-db-consumer.js @@ -97,7 +97,6 @@ export default async function exportToExploitationDB({data}) { // Set to store the legacy address ID to avoid duplicates const addressLegacyIDSet = new Set() - // Check if the district has data to export // To-fix: use the district configuration instead to check if the district has to be exported const totalCommonToponymRecords = await CommonToponym.count({