From b7f59a89606d36548bc40f1f4c806f9783a8fc6d Mon Sep 17 00:00:00 2001 From: antoineludeau <52679050+antoineludeau@users.noreply.github.com> Date: Fri, 22 Nov 2024 12:39:31 +0100 Subject: [PATCH] Added dynamic views instead of stored views --- .../20241122171119-delete-views.cjs | 209 +++++++++++ ...124117-grant-rights-on-external-schema.cjs | 18 + .../export-to-exploitation-db-consumer.js | 52 +-- lib/api/consumers/format-to-legacy-helpers.js | 5 +- lib/api/consumers/sql-queries.js | 335 ++++++++++++++++++ 5 files changed, 565 insertions(+), 54 deletions(-) create mode 100644 db-migrations/migrations/20241122171119-delete-views.cjs create mode 100644 db-migrations/migrations/20241125124117-grant-rights-on-external-schema.cjs create mode 100644 lib/api/consumers/sql-queries.js diff --git a/db-migrations/migrations/20241122171119-delete-views.cjs b/db-migrations/migrations/20241122171119-delete-views.cjs new file mode 100644 index 00000000..9a8c5d79 --- /dev/null +++ b/db-migrations/migrations/20241122171119-delete-views.cjs @@ -0,0 +1,209 @@ +'use strict' + +const {POSTGRES_BAN_USER} = process.env + +const bboxBufferAdressView = 50 +const addressBboxBuffer = 200 +const bboxBuffer = 100 + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface,) { + await queryInterface.sequelize.query('DROP VIEW IF EXISTS ban."address_view_cp";') + await queryInterface.sequelize.query('DROP VIEW IF EXISTS ban."common_toponym_view_cp";') + }, + + async down(queryInterface) { + await queryInterface.sequelize.query(` + CREATE VIEW ban."address_view_cp" AS + WITH address_view AS ( + SELECT + A.*, + ST_Transform(ST_Buffer(ST_Transform(ST_Envelope(ST_SetSRID(ST_GeomFromGeoJSON((A.positions[1])->'geometry'), 4326)), 2154), ${bboxBufferAdressView}, 'join=mitre endcap=square'), 4326) AS bbox + FROM + ban.address AS A + WHERE A."isActive" = true + ORDER BY A.id ASC ), + postal_codes_array AS ( + SELECT + a.*, + array_length(d."postalCodes", 1) AS array_length, + d."postalCodes" AS postalCodes, + b.meta->'insee'->>'cog' AS insee_com, + d."libelleAcheminementWithPostalCodes" + FROM + address_view AS a + LEFT JOIN + ban.district AS b + ON a."districtID" = b.id + LEFT JOIN + external.datanova AS d + ON b.meta->'insee'->>'cog' = d."inseeCom" + ) + SELECT + pca.id, + pca."mainCommonToponymID", + pca."secondaryCommonToponymIDs", + pca."districtID", + pca."number", + pca."suffix", + pca."labels", + pca."certified", + pca."positions", + pca."updateDate", + pca."meta", + pca."range_validity", + pca."isActive", + pca."bbox", + pca.insee_com, + CASE + WHEN pca.array_length = 1 THEN pca.postalCodes[1] + WHEN pca.array_length > 1 + THEN ( + SELECT c."postalCode" + FROM external.postal_area AS c + WHERE pca.insee_com = c."inseeCom" + ORDER BY ST_Area(ST_Intersection(ST_Transform(pca.bbox, 2154), ST_Transform(c.geometry, 2154))) desc + LIMIT 1 + ) + ELSE NULL + END AS postal_code, + CASE + WHEN pca.array_length = 1 THEN pca."libelleAcheminementWithPostalCodes"->>pca.postalCodes[1] + WHEN pca.array_length > 1 + THEN ( + SELECT pca."libelleAcheminementWithPostalCodes"->>c."postalCode" + FROM external.postal_area AS c + WHERE pca.insee_com = c."inseeCom" + ORDER BY ST_Area(ST_Intersection(ST_Transform(pca.bbox, 2154), ST_Transform(c.geometry, 2154))) DESC + LIMIT 1 + ) + ELSE NULL + END AS "libelleAcheminement", + pca.postalCodes, + pca."libelleAcheminementWithPostalCodes", + CASE + WHEN pca.array_length = 1 THEN 'DATANOVA' + WHEN pca.array_length > 1 THEN + CASE + WHEN EXISTS ( + SELECT 1 + FROM external.postal_area AS c + WHERE pca.insee_com = c."inseeCom" + AND ST_Intersects(ST_Transform(pca.bbox, 2154), ST_Transform(c.geometry, 2154)) + ) THEN 'CONTOURS_CP' + ELSE 'DGFIP' + END + ELSE 'DGFIP' + END AS source_cp + FROM + postal_codes_array AS pca + ORDER BY pca.id ASC + `) + await queryInterface.sequelize.query(` + CREATE VIEW ban."common_toponym_view_cp" AS + WITH common_toponym_view AS( + SELECT + CT.id, CT."districtID", CT.labels, CT.geometry, CT."updateDate", CT.meta, CT.range_validity, CT."isActive", + ST_Centroid(ST_Collect(ST_SetSRID(ST_GeomFromGeoJSON((A.positions[1])->'geometry'), 4326))) AS centroid, + ST_Transform(ST_Buffer(ST_Transform(ST_Envelope(ST_Collect(ST_SetSRID(ST_GeomFromGeoJSON((A.positions[1])->'geometry'), 4326))), 2154), ${addressBboxBuffer}, 'join=mitre endcap=square'), 4326) AS "addressBbox", + ST_Transform(ST_Buffer(ST_Transform(ST_Envelope(ST_SetSRID(ST_GeomFromGeoJSON(CT.geometry), 4326)), 2154), ${bboxBuffer}, 'join=mitre endcap=square'), 4326) AS "bbox", + COUNT(A.id) AS "addressCount", + COUNT(DISTINCT CASE WHEN A.certified = true THEN A.id ELSE NULL END) AS "certifiedAddressCount" + FROM + ban.common_toponym AS CT + LEFT JOIN + ban.address AS A + ON + (CT.id = A."mainCommonToponymID" + OR CT.id = ANY(A."secondaryCommonToponymIDs")) AND A."isActive" = true + WHERE CT."isActive" = true + GROUP BY CT.id + ORDER BY CT.id ASC ), + + postal_codes_array AS ( + SELECT + ct.*, + b.meta->'insee'->>'cog' AS insee_com, + array_length(d."postalCodes", 1) AS array_length, + d."postalCodes" AS postalCodes, + d."libelleAcheminementWithPostalCodes", + CASE + WHEN ct."addressCount" = 0 THEN ct.bbox + ELSE ct."addressBbox" + END AS used_bbox + FROM + common_toponym_view AS ct + LEFT JOIN + ban.district AS b + ON ct."districtID" = b.id + LEFT JOIN + external.datanova AS d + ON b.meta->'insee'->>'cog' = d."inseeCom" + ) + SELECT + pca.id, + pca."districtID", + pca.labels, + pca.geometry, + pca."updateDate", + pca.meta, + pca.range_validity, + pca."isActive", + pca.centroid, + pca."addressBbox", + pca.bbox, + pca."addressCount", + pca."certifiedAddressCount", + pca.insee_com, + CASE + WHEN pca.array_length = 1 THEN pca.postalCodes[1] + WHEN pca.array_length > 1 + THEN ( + SELECT c."postalCode" + FROM external.postal_area AS c + WHERE pca.insee_com = c."inseeCom" + ORDER BY ST_Area(ST_Intersection(ST_Transform(pca.used_bbox, 2154), ST_Transform(c.geometry, 2154))) DESC + LIMIT 1 + ) + ELSE NULL + END AS postal_code, + CASE + WHEN pca.array_length = 1 THEN pca."libelleAcheminementWithPostalCodes"->>pca.postalCodes[1] + WHEN pca.array_length > 1 + THEN ( + SELECT pca."libelleAcheminementWithPostalCodes"->>c."postalCode" + FROM external.postal_area AS c + WHERE pca.insee_com = c."inseeCom" + ORDER BY ST_Area(ST_Intersection(ST_Transform(pca.bbox, 2154), ST_Transform(c.geometry, 2154))) DESC + LIMIT 1 + ) + ELSE NULL + END AS "libelleAcheminement", + pca.postalCodes, + pca."libelleAcheminementWithPostalCodes", + CASE + WHEN pca.array_length = 1 THEN 'DATANOVA' + WHEN pca.array_length > 1 THEN + CASE + WHEN EXISTS ( + SELECT 1 + FROM external.postal_area AS c + WHERE pca.insee_com = c."inseeCom" + AND ST_Intersects(ST_Transform(pca.used_bbox, 2154), ST_Transform(c.geometry, 2154)) + ) THEN 'CONTOURS_CP' + ELSE 'DGFIP' + END + ELSE 'DGFIP' + END AS source_cp, + pca.used_bbox + FROM + postal_codes_array AS pca + ORDER BY pca.id ASC + `) + + await queryInterface.sequelize.query(`GRANT SELECT ON ban."address_view_cp" TO "${POSTGRES_BAN_USER}";`) + await queryInterface.sequelize.query(`GRANT SELECT ON ban."common_toponym_view_cp" TO "${POSTGRES_BAN_USER}";`) + } +} + diff --git a/db-migrations/migrations/20241125124117-grant-rights-on-external-schema.cjs b/db-migrations/migrations/20241125124117-grant-rights-on-external-schema.cjs new file mode 100644 index 00000000..bb302763 --- /dev/null +++ b/db-migrations/migrations/20241125124117-grant-rights-on-external-schema.cjs @@ -0,0 +1,18 @@ +'use strict' + +const {POSTGRES_BAN_USER} = process.env + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface) { + await queryInterface.sequelize.query(`GRANT USAGE ON SCHEMA external TO "${POSTGRES_BAN_USER}";`) + await queryInterface.sequelize.query(`GRANT ALL PRIVILEGES ON SCHEMA external TO "${POSTGRES_BAN_USER}";`) + await queryInterface.sequelize.query(`GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA external TO "${POSTGRES_BAN_USER}";`) + }, + + async down(queryInterface) { + await queryInterface.sequelize.query(`REVOKE USAGE ON SCHEMA external FROM "${POSTGRES_BAN_USER}";`) + await queryInterface.sequelize.query(`REVOKE ALL PRIVILEGES ON SCHEMA external FROM "${POSTGRES_BAN_USER}";`) + await queryInterface.sequelize.query(`REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA external FROM "${POSTGRES_BAN_USER}";`) + } +} diff --git a/lib/api/consumers/export-to-exploitation-db-consumer.js b/lib/api/consumers/export-to-exploitation-db-consumer.js index b85c90b7..3b183d98 100644 --- a/lib/api/consumers/export-to-exploitation-db-consumer.js +++ b/lib/api/consumers/export-to-exploitation-db-consumer.js @@ -8,6 +8,7 @@ import {derivePositionProps} from '../../util/geo.cjs' import {createPseudoCodeVoieGenerator} from '../../pseudo-codes-voies.cjs' import gazetteerPromise from '../../util/gazetteer.cjs' +import {createCommonToponymTempTableQuery, createAddressTempTableQuery, countQuery, pageQuery, specificCommonToponymTempTableCountQuery, addressCertifiedTempTableCountQuery} from './sql-queries.js' import {formatCommonToponymDataForLegacy, formatAddressDataForLegacy, formatDistrictDataForLegacy} from './format-to-legacy-helpers.js' // SETTINGS @@ -37,57 +38,6 @@ const EXPLOITATION_DB_COLLECTION_NAMES = { address: 'numeros' } -// QUERIES -const createCommonToponymTempTableQuery = tempTableName => ` - CREATE TEMP TABLE ${tempTableName} AS - SELECT - CTV.* - FROM - ban.common_toponym_view_cp AS CTV - WHERE CTV."districtID" = :districtID -` - -const createAddressTempTableQuery = tempTableName => ` - CREATE TEMP TABLE ${tempTableName} AS - SELECT - AV.* - FROM - ban.address_view_cp AS AV - WHERE AV."districtID" = :districtID -` - -const pageQuery = tempTableName => ` - SELECT - * - FROM - ${tempTableName} - OFFSET :offset - LIMIT :limit -` - -const countQuery = tempTableName => ` - SELECT - COUNT(*) - FROM - ${tempTableName} -` - -const specificCommonToponymTempTableCountQuery = tempTableName => ` - SELECT - COUNT(*) - FROM - ${tempTableName} - WHERE meta->'bal'->>'isLieuDit' = 'true'; -` - -const addressCertifiedTempTableCountQuery = tempTableName => ` - SELECT - COUNT(*) - FROM - ${tempTableName} - WHERE certified = TRUE; -` - export default async function exportToExploitationDB({data}) { const {districtID} = data console.log(`Exporting districtID ${districtID} to exploitation DB...`) diff --git a/lib/api/consumers/format-to-legacy-helpers.js b/lib/api/consumers/format-to-legacy-helpers.js index 6b4554fe..087af921 100644 --- a/lib/api/consumers/format-to-legacy-helpers.js +++ b/lib/api/consumers/format-to-legacy-helpers.js @@ -71,7 +71,7 @@ export const formatDistrictDataForLegacy = async (district, {totalCommonToponymR export const formatCommonToponymDataForLegacy = async (commonToponym, {district, pseudoCodeVoieGenerator, commonToponymLegacyIDCommonToponymIDMap, commonToponymLegacyIDSet, gazetteerFinder}) => { const {labels: districtLabels, meta: {insee: {cog}}} = district - const {id, districtID, geometry, labels, meta, updateDate, addressCount, certifiedAddressCount, bbox, addressBbox} = commonToponym + const {id, districtID, geometry, labels, meta, updateDate, addressCount, certifiedAddressCount, bbox} = commonToponym // Labels // District @@ -91,7 +91,6 @@ export const formatCommonToponymDataForLegacy = async (commonToponym, {district, const lon = legacyPosition?.coordinates?.[0] const lat = legacyPosition?.coordinates?.[1] const commonToponymBbox = formatBboxForLegacy(bbox) - const commonToponymAddressBbox = formatBboxForLegacy(addressBbox) // Old district const {codeAncienneCommune, nomAncienneCommune} = await calculateLegacyCommuneAncienne(cog, meta, lon, lat, gazetteerFinder) @@ -161,7 +160,7 @@ export const formatCommonToponymDataForLegacy = async (commonToponym, {district, sourceNomVoie: 'bal', position: legacyPosition, codePostal: meta?.laposte?.codePostal, - displayBBox: commonToponymAddressBbox, + displayBBox: commonToponymBbox, lon, lat, x: meta?.geography?.x, diff --git a/lib/api/consumers/sql-queries.js b/lib/api/consumers/sql-queries.js new file mode 100644 index 00000000..b33422c0 --- /dev/null +++ b/lib/api/consumers/sql-queries.js @@ -0,0 +1,335 @@ +// QUERIES +export const createCommonToponymTempTableQuery = tempTableName => ` + CREATE TEMP TABLE ${tempTableName} AS + WITH base_common_toponym AS ( + SELECT + ct.id, + ct."districtID", + ct.labels, + ct.geometry, + ct."updateDate", + ct.meta, + ct.range_validity, + ct."isActive" + FROM ban.common_toponym ct + WHERE ct."isActive" = true + AND ct."districtID" = :districtID + ), + common_toponym_with_addresses AS ( + SELECT + bct.*, + st_centroid( + st_collect( + st_setsrid( + st_geomfromgeojson(addr.positions[1] -> 'geometry'::text), + 4326 + ) + ) + ) AS centroid, + count(addr.id) AS "addressCount", + count(DISTINCT CASE WHEN addr.certified = true THEN addr.id ELSE NULL END) AS "certifiedAddressCount", + CASE + WHEN count(addr.id) = 0 THEN + st_transform( + st_buffer( + st_transform( + st_envelope(st_setsrid(st_geomfromgeojson(bct.geometry), 4326)), + 2154 + ), + 100::double precision, + 'join=mitre endcap=square'::text + ), + 4326 + ) + ELSE + st_transform( + st_buffer( + st_transform( + st_envelope( + st_collect( + st_setsrid(st_geomfromgeojson(addr.positions[1] -> 'geometry'::text), 4326) + ) + ), + 2154 + ), + 200::double precision, + 'join=mitre endcap=square'::text + ), + 4326 + ) + END AS bbox + FROM base_common_toponym bct + LEFT JOIN ban.address addr + ON bct.id = addr."mainCommonToponymID" + AND addr."isActive" + GROUP BY + bct.id, + bct."districtID", + bct.labels, + bct.geometry, + bct."updateDate", + bct.meta, + bct.range_validity, + bct."isActive" + ), + common_toponym_with_metadata AS ( + SELECT + ctwa.*, + (distr.meta -> 'insee'::text) ->> 'cog'::text AS insee_com, + dn."postalCodes" AS postalcodes, + array_length(dn."postalCodes", 1) AS nb_postalcodes, + dn."libelleAcheminementWithPostalCodes" + FROM common_toponym_with_addresses ctwa + LEFT JOIN ban.district distr + ON ctwa."districtID" = distr.id + LEFT JOIN external.datanova dn + ON ((distr.meta -> 'insee'::text) ->> 'cog'::text) = dn."inseeCom"::text + ), + postal_matches AS ( + SELECT + ctwm.id, + ctwm.insee_com, + pa."postalCode", + pa."inseeCom", + pa.geometry, + st_area( + st_intersection( + st_transform(ctwm.bbox, 2154), + st_transform(pa.geometry, 2154) + ) + ) AS intersect_area + FROM common_toponym_with_metadata ctwm + JOIN external.postal_area pa + ON ctwm.insee_com = pa."inseeCom"::text + WHERE st_intersects( + st_transform(ctwm.bbox, 2154), + st_transform(pa.geometry, 2154) + ) + ), + ranked_postal_matches AS ( + SELECT + pm.id, + pm."postalCode", + pm.intersect_area, + ROW_NUMBER() OVER (PARTITION BY pm.id ORDER BY pm.intersect_area DESC) AS rank + FROM postal_matches pm + ), + best_postal_match AS ( + SELECT + rpm.id, + rpm."postalCode", + rpm.intersect_area + FROM ranked_postal_matches rpm + WHERE rpm.rank = 1 + ) + SELECT + ctwm.id, + ctwm."districtID", + ctwm.labels, + ctwm.geometry, + ctwm."updateDate", + ctwm.meta, + ctwm.range_validity, + ctwm."isActive", + ctwm.centroid, + ctwm.bbox, + ctwm."addressCount", + ctwm."certifiedAddressCount", + ctwm.insee_com, + CASE + WHEN ctwm.nb_postalcodes = 1 THEN ctwm.postalcodes[1] + WHEN ctwm.nb_postalcodes > 1 THEN bpm."postalCode" + ELSE NULL + END AS postal_code, + CASE + WHEN ctwm.nb_postalcodes = 1 THEN ctwm."libelleAcheminementWithPostalCodes" ->> ctwm.postalcodes[1]::text + WHEN ctwm.nb_postalcodes > 1 THEN ctwm."libelleAcheminementWithPostalCodes" ->> bpm."postalCode"::text + ELSE NULL + END AS "libelleAcheminement", + CASE + WHEN ctwm.nb_postalcodes = 1 THEN 'DATANOVA' + WHEN ctwm.nb_postalcodes > 1 THEN + CASE + WHEN bpm.intersect_area IS NOT NULL THEN 'CONTOURS_CP' + ELSE 'DGFIP' + END + ELSE 'DGFIP' + END AS source_cp + FROM common_toponym_with_metadata ctwm + LEFT JOIN best_postal_match bpm + ON ctwm.id = bpm.id + ORDER BY ctwm.id; +` + +export const createAddressTempTableQuery = tempTableName => ` + CREATE TEMP TABLE ${tempTableName} AS + WITH base_address AS ( + SELECT + addr.id, + addr."mainCommonToponymID", + addr."secondaryCommonToponymIDs", + addr."districtID", + addr.number, + addr.suffix, + addr.labels, + addr.certified, + addr.positions, + addr."updateDate", + addr.meta, + addr.range_validity, + addr."isActive" + FROM ban.address addr + WHERE addr."isActive" + AND addr."districtID" = :districtID + ), + district_metadata AS ( + SELECT + distr.id AS district_id, + (distr.meta -> 'insee'::text) ->> 'cog'::text AS insee_com + FROM ban.district distr + ), + address_with_metadata AS ( + SELECT + ba.*, + dm.insee_com + FROM base_address ba + LEFT JOIN district_metadata dm ON ba."districtID" = dm.district_id + ), + address_enriched AS ( + SELECT + awm.*, + dn."postalCodes" AS postalcodes, + array_length(dn."postalCodes", 1) AS nb_postalcodes, + dn."libelleAcheminementWithPostalCodes" + FROM address_with_metadata awm + LEFT JOIN external.datanova dn + ON awm.insee_com = dn."inseeCom"::text + ), + address_with_bbox AS ( + SELECT + ae.*, + st_transform( + st_buffer( + st_transform( + st_envelope( + st_setsrid( + st_geomfromgeojson(ae.positions[1] -> 'geometry'::text), + 4326 + ) + ), + 2154 + ), + 50::double precision, + 'join=mitre endcap=square'::text + ), + 4326 + ) AS bbox + FROM address_enriched ae + ), + postal_matches AS ( + SELECT + aab.id, + aab.insee_com, + pa."postalCode", + pa."inseeCom", + pa.geometry, + st_area( + st_intersection( + st_transform(aab.bbox, 2154), + st_transform(pa.geometry, 2154) + ) + ) AS intersect_area + FROM address_with_bbox aab + JOIN external.postal_area pa + ON aab.insee_com = pa."inseeCom"::text + WHERE st_intersects( + st_transform(aab.bbox, 2154), + st_transform(pa.geometry, 2154) + ) + ), + ranked_postal_matches AS ( + SELECT + pm.id, + pm."postalCode", + pm.intersect_area, + ROW_NUMBER() OVER (PARTITION BY pm.id ORDER BY pm.intersect_area DESC) AS rank + FROM postal_matches pm + ), + best_postal_match AS ( + SELECT + id, + "postalCode", + intersect_area + FROM ranked_postal_matches + WHERE rank = 1 + ) + SELECT + aab.id, + aab."mainCommonToponymID", + aab."secondaryCommonToponymIDs", + aab."districtID", + aab.number, + aab.suffix, + aab.labels, + aab.certified, + aab.positions, + aab."updateDate", + aab.meta, + aab.range_validity, + aab."isActive", + aab.bbox, + CASE + WHEN aab.nb_postalcodes = 1 THEN aab.postalcodes[1] + WHEN aab.nb_postalcodes > 1 THEN bpm."postalCode" + ELSE NULL + END AS postal_code, + CASE + WHEN aab.nb_postalcodes = 1 THEN aab."libelleAcheminementWithPostalCodes" ->> aab.postalcodes[1]::text + WHEN aab.nb_postalcodes > 1 THEN aab."libelleAcheminementWithPostalCodes" ->> bpm."postalCode"::text + ELSE NULL + END AS "libelleAcheminement", + CASE + WHEN aab.nb_postalcodes = 1 THEN 'DATANOVA' + WHEN aab.nb_postalcodes > 1 THEN + CASE + WHEN bpm.intersect_area IS NOT NULL THEN 'CONTOURS_CP' + ELSE 'DGFIP' + END + ELSE 'DGFIP' + END AS source_cp + FROM address_with_bbox aab + LEFT JOIN best_postal_match bpm ON aab.id = bpm.id + ORDER BY aab.id; +` + +export const pageQuery = tempTableName => ` + SELECT + * + FROM + ${tempTableName} + OFFSET :offset + LIMIT :limit +` + +export const countQuery = tempTableName => ` + SELECT + COUNT(*) + FROM + ${tempTableName} +` + +export const specificCommonToponymTempTableCountQuery = tempTableName => ` + SELECT + COUNT(*) + FROM + ${tempTableName} + WHERE meta->'bal'->>'isLieuDit' = 'true'; +` + +export const addressCertifiedTempTableCountQuery = tempTableName => ` + SELECT + COUNT(*) + FROM + ${tempTableName} + WHERE certified = TRUE; +`