From e8a3efd4c6b0ee28ac84b03ef6c7961cd270fb78 Mon Sep 17 00:00:00 2001 From: antoineludeau <52679050+antoineludeau@users.noreply.github.com> Date: Wed, 4 Oct 2023 10:17:22 +0200 Subject: [PATCH] Added legacy formatting during export from main to exploitation db --- .../export-to-exploitation-db-consumer.js | 343 +++++++++++------- lib/api/consumers/format-to-legacy-helpers.js | 313 ++++++++++++++++ lib/api/helper.js | 18 + lib/api/position-type-dictionary.json | 11 + 4 files changed, 562 insertions(+), 123 deletions(-) create mode 100644 lib/api/consumers/format-to-legacy-helpers.js create mode 100644 lib/api/position-type-dictionary.json diff --git a/lib/api/consumers/export-to-exploitation-db-consumer.js b/lib/api/consumers/export-to-exploitation-db-consumer.js index e07eacd2..8ff61bec 100644 --- a/lib/api/consumers/export-to-exploitation-db-consumer.js +++ b/lib/api/consumers/export-to-exploitation-db-consumer.js @@ -4,8 +4,11 @@ import {findCodePostal} from 'codes-postaux/full.js' import mongo from '../../util/mongo.cjs' import {sequelize, District, CommonToponym, Address} from '../../util/sequelize.js' import {derivePositionProps} from '../../util/geo.cjs' +import {createPseudoCodeVoieGenerator} from '../../pseudo-codes-voies.cjs' -// Seetings +import {formatCommonToponymDataForLegacy, formatAddressDataForLegacy, formatDistrictDataForLegacy} from './format-to-legacy-helpers.js' + +// SETTINGS // The number of records to process per page const PAGE_SIZE = 100 @@ -14,39 +17,58 @@ const PAGE_SIZE = 100 const FANTOIR_PATH = process.env.FANTOIR_PATH || 'data/fantoir.sqlite' // The min and max zoom levels to use for the tiles -const COMMON_TOPONYM_TILES_MIN_ZOOM = 10 -const COMMON_TOPONYM_TILES_MAX_ZOOM = 14 -const ADDRESS_TILES_MIN_ZOOM = 12 -const ADDRESS_TILES_MAX_ZOOM = 14 +const TILES_ZOOM_LEVELS = { + commonToponym: { + min: 10, + max: 14 + }, + address: { + min: 12, + max: 14 + } +} + +// The buffer distance to use for the bbox calculation +const COMMON_TOPONYM_BBOX_BUFFER = 200 +// For specific common toponyms (='lieu-dit'), we use a different buffer distance +const SPECIFIC_COMMON_TOPONYM_BBOX_BUFFER = 100 +const ADDRESS_BBOX_BUFFER = 50 // Collections names -const DISTRICT_COLLECTION = 'districts' -const COMMON_TOPONYM_COLLECTION = 'common_toponyms' -const ADDRESS_COLLECTION = 'addresses' - -// The priority of each position type -const POSITION_TYPES_PRIORITY = { - entrance: 1, - building: 2, - 'staircase identifier': 3, - 'unit identifier': 4, - 'utility service': 5, - 'postal delivery': 6, - parcel: 7, - segment: 8, - other: 9 +const EXPLOITATION_DB_COLLECTION_NAMES = { + legacy: { + district: 'communes_legacy', + commonToponym: 'voies_legacy', + address: 'numeros_legacy' + }, + banID: { + district: 'districts', + commonToponym: 'common_toponyms', + address: 'addresses' + } } +// QUERIES & POSTGIS FUNCTIONS +// The queries are written in raw SQL to be able to use the PostGIS functions +// centroid: It calculates the centroid of a collection of geometries extracted from the "positions" column in the "Addresses" table. +// bbox: It calculates a bounding box (envelope) for a collection of geometries from the "positions" column in the "Addresses" table. +// The bbox result is transformed to a different coordinate system (2154 to 4326) and includes a buffer operation. + const commonToponymPageQuery = ` SELECT CT.id, CT."districtID", CT.labels, CT.geometry, CT."updateDate", CT.meta, CT."createdAt", CT."updatedAt", - ST_Centroid(ST_Collect(ST_GeomFromGeoJSON((A.positions[1])->'geometry'))) AS centroid + ST_Centroid(ST_Collect(ST_SetSRID(ST_GeomFromGeoJSON((A.positions[1])->'geometry'), 4326))) AS centroid, + ST_Transform(ST_Buffer(ST_Transform(ST_Envelope(ST_Collect(ST_SetSRID(ST_GeomFromGeoJSON((A.positions[1])->'geometry'), 4326))), 2154), :addressBboxBuffer, 'join=mitre endcap=square'), 4326) AS "addressBbox", + ST_Transform(ST_Buffer(ST_Transform(ST_Envelope(ST_SetSRID(ST_GeomFromGeoJSON(CT.geometry), 4326)), 2154), :bboxBuffer, 'join=mitre endcap=square'), 4326) AS "bbox", + COUNT(A.id) AS "addressCount", + COUNT(DISTINCT CASE WHEN a.certified = true THEN a.id ELSE NULL END) AS "certifiedAddressCount" FROM "CommonToponyms" AS CT LEFT JOIN "Addresses" AS A ON CT.id = A."mainCommonToponymID" + OR CT.id = ANY(A."secondaryCommonToponymIDs") WHERE CT."districtID" = :districtID GROUP BY CT.id ORDER BY CT.id ASC @@ -54,12 +76,26 @@ const commonToponymPageQuery = ` LIMIT :limit ` -// Map to store the fantoir code for each common toponym to be able to calculate the postal codes later -const commonToponymIDFantoirCodeMap = new Map() +// The queries are written in raw SQL to be able to use the PostGIS functions +// bbox: It calculates a bounding box (envelope) for the geometry contained in the "positions" column. +// The result is transformed from one coordinate system (2154) to another (4326) +// and includes a buffer operation with a distance of 50 units and specific parameters for joining and capping. +const addressPageQuery = ` + SELECT + A.*, + ST_Transform(ST_Buffer(ST_Transform(ST_Envelope(ST_SetSRID(ST_GeomFromGeoJSON((A.positions[1])->'geometry'), 4326)), 2154), :bboxBuffer, 'join=mitre endcap=square'), 4326) AS bbox + FROM + "Addresses" AS A + WHERE A."districtID" = :districtID + ORDER BY A.id ASC + OFFSET :offset + LIMIT :limit +` export default async function exportToExploitationDB({data}) { const {districtID} = data console.log(`Exporting districtID ${districtID} to exploitation DB...`) + // Use REPEATABLE_READ isolation level to balance data consistency and concurrency // - Ensures data consistency within each table during the transaction // - Allows concurrent reads across tables, minimizing read contention @@ -78,61 +114,55 @@ export default async function exportToExploitationDB({data}) { throw new Error(`District with ID ${districtID} not found.`) } - // District - // Delete all data related to the district - await deleteAllDataRelatedToDistrict(districtID) - - // Insert the district - await mongo.db.collection(DISTRICT_COLLECTION).insertOne(district) - // Prepare data source for calculation const {meta: {insee: {cog}}} = district + // Prepare fantoir finder from cog and fantoir sqlite database const fantoirFinder = await createFantoirCommune(cog, {FANTOIR_PATH}) - // CommonToponym + // Prepare pseudo code voie generator from cog + const pseudoCodeVoieGenerator = await createPseudoCodeVoieGenerator(cog) - const fetchAndExportDataFromPage = async (type, model, collection, pageNumber) => { - const offset = (pageNumber - 1) * PAGE_SIZE - let pageData = [] - // Export the data from the page - if (type === 'commonToponym') { - [pageData] = await sequelize.query(commonToponymPageQuery, { - replacements: {districtID, offset, limit: PAGE_SIZE}, - transaction, - raw: true, - }) - } else if (type === 'adresse') { - pageData = await model.findAll({ - where: {districtID}, - order: [['id', 'ASC']], - offset, - limit: PAGE_SIZE, - transaction, - raw: true, - }) - } + // Map to store the fantoir code for each common toponym to then be able to calculate the postal codes on addresses + const commonToponymIDFantoirCodeMap = new Map() - // Format the data and calculate the fantoir code, tiles and postal code - const formatedPageData = formatPageData(pageData, type, cog, fantoirFinder) + // Map to store the common toponym ID for each legacy common toponym ID to then be able to associate it to the legacy address + const commonToponymIDlegacyCommonToponymIDMap = new Map() - // Insert the data in the collection - await mongo.db.collection(collection).insertMany(formatedPageData, {ordered: false}) - } + // Clean collections + // Delete all data related to the district (legacy and banID) + await deleteAllLegacyDataRelatedToCOG(cog) + await deleteAllDataRelatedToDistrict(districtID) + // CommonToponym // Count the total number of common toponyms and pages to process const totalCommonToponymRecords = await CommonToponym.count({ where: {districtID}, transaction, }) + const totalCommonToponymPages = Math.ceil(totalCommonToponymRecords / PAGE_SIZE) - const commonToponymsExportPromises = [] + const fetchAndExportDataFromCommonToponymPage = async pageNumber => { + const offset = (pageNumber - 1) * PAGE_SIZE + const [pageData] = await sequelize.query(commonToponymPageQuery, { + replacements: {districtID, offset, limit: PAGE_SIZE, addressBboxBuffer: COMMON_TOPONYM_BBOX_BUFFER, bboxBuffer: SPECIFIC_COMMON_TOPONYM_BBOX_BUFFER}, + transaction, + raw: true, + }) + // Format the data and calculate the fantoir code, tiles and postal code + const pageDataWithExtraDataCalculation = pageData.map(commonToponym => calculateExtraDataForCommonToponym(commonToponym, cog, fantoirFinder, commonToponymIDFantoirCodeMap)) + const formatedPageData = pageDataWithExtraDataCalculation.map(commonToponym => formatCommonToponym(commonToponym)) + const formatedPageDataForLegacy = pageDataWithExtraDataCalculation.map(commonToponym => formatCommonToponymDataForLegacy(commonToponym, district, pseudoCodeVoieGenerator, commonToponymIDlegacyCommonToponymIDMap)) + + // Insert the data in the collection (legacy and banID) + await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.legacy.commonToponym).insertMany(formatedPageDataForLegacy, {ordered: false}) + await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.banID.commonToponym).insertMany(formatedPageData, {ordered: false}) + } + const commonToponymsExportPromises = [] for (let pageNumber = 1; pageNumber <= totalCommonToponymPages; pageNumber++) { - commonToponymsExportPromises.push( - fetchAndExportDataFromPage('commonToponym', CommonToponym, COMMON_TOPONYM_COLLECTION, pageNumber) - ) + commonToponymsExportPromises.push(fetchAndExportDataFromCommonToponymPage(pageNumber)) } await Promise.all(commonToponymsExportPromises) @@ -145,21 +175,48 @@ export default async function exportToExploitationDB({data}) { }) const totalAddressPages = Math.ceil(totalAddressRecords / PAGE_SIZE) - const addressesExportPromises = [] + const fetchAndExportDataFromAddressPage = async pageNumber => { + const offset = (pageNumber - 1) * PAGE_SIZE + const [pageData] = await sequelize.query(addressPageQuery, { + replacements: {districtID, offset, limit: PAGE_SIZE, bboxBuffer: ADDRESS_BBOX_BUFFER}, + transaction, + raw: true, + }) + + // Format the data and calculate the fantoir code, tiles and postal code + const pageDataWithExtraDataCalculation = pageData.map(address => calculateExtraDataForAddress(address, cog, commonToponymIDFantoirCodeMap)) + const formatedPageData = pageDataWithExtraDataCalculation.map(address => formatAddress(address)) + const formatedPageDataForLegacy = pageDataWithExtraDataCalculation.map(address => formatAddressDataForLegacy(address, district, commonToponymIDlegacyCommonToponymIDMap)) + // Insert the data in the collection (legacy and banID) + await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.legacy.address).insertMany(formatedPageDataForLegacy, {ordered: false}) + await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.banID.address).insertMany(formatedPageData, {ordered: false}) + } + + const addressesExportPromises = [] for (let pageNumber = 1; pageNumber <= totalAddressPages; pageNumber++) { - addressesExportPromises.push( - fetchAndExportDataFromPage('adresse', Address, ADDRESS_COLLECTION, pageNumber) - ) + addressesExportPromises.push(fetchAndExportDataFromAddressPage(pageNumber)) } await Promise.all(addressesExportPromises) + // District + // For Legacy collections + const districtFormatedForLegacy = await formatDistrictDataForLegacy(district, totalCommonToponymRecords, totalAddressRecords, transaction) + await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.legacy.district).insertOne(districtFormatedForLegacy) + + // For BanID collections + await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.banID.district).insertOne(district) + + // Pseudo code voie generator saving data + await pseudoCodeVoieGenerator.save() + + // Commit the transaction await transaction.commit() console.log(`Exporting districtID ${districtID} done`) } catch (error) { await transaction.rollback() - console.error(`Exporting districtID ${districtID} failed: ${error.message}`) + console.error(`Exporting districtID ${districtID} failed: ${error}`) } } @@ -168,92 +225,132 @@ export default async function exportToExploitationDB({data}) { // Helpers for exploitation DB const deleteAllDataRelatedToDistrict = async districtID => { await Promise.all([ - mongo.db.collection(DISTRICT_COLLECTION).deleteOne({id: districtID}), - mongo.db.collection(COMMON_TOPONYM_COLLECTION).deleteMany({districtID}), - mongo.db.collection(ADDRESS_COLLECTION).deleteMany({districtID}) + mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.banID.district).deleteOne({districtID}), + mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.banID.commonToponym).deleteMany({districtID}), + mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.banID.address).deleteMany({districtID}) ]) } -// Helpers for formatting -const formatPageData = (pageData, type, cog, fantoirFinder) => { - if (type === 'commonToponym') { - return pageData.map(commonToponym => { - // Calculate the fantoir code for each common toponym - const fantoirCode = calculateCommonToponymFantoirCode(commonToponym, cog, fantoirFinder) - // Calculate the tiles for each common toponym - const {tiles, x, y} = calculateCommonToponymTiles(commonToponym) - // Calculate the postal code for each common toponym - const postalCode = calculateCommonToponymPostalCode(commonToponym, cog) - // Remove the centroid data from the common toponym - const {centroid, ...commonToponymCleaned} = commonToponym - return {...commonToponymCleaned, - meta: { - ...commonToponym.meta, - ...(fantoirCode ? {dgfip: {...commonToponym.meta?.dgfip, fantoir: fantoirCode}} : {}), - ...(tiles && x && y ? {geography: {...commonToponym.meta?.geography, tiles, x, y}} : {}), - ...(postalCode ? {laposte: {...commonToponym.meta?.laposte, codePostal: postalCode}} : {}) - }} - }) - } +const deleteAllLegacyDataRelatedToCOG = async cog => { + await Promise.all([ + mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.legacy.district).deleteOne({codeCommune: cog}), + mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.legacy.commonToponym).deleteMany({codeCommune: cog}), + mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.legacy.address).deleteMany({codeCommune: cog}), + ]) +} - if (type === 'adresse') { - return pageData.map(address => { - // Calculate the tiles for each address - const {tiles, x, y} = calculateAddressTiles(address) - // Calculate the postal code for each address - const postalCode = calculateAddressPostalCode(address, cog) - return {...address, - meta: { - ...(tiles && x && y ? {geography: {...address.meta?.geography, tiles, x, y}} : {}), - ...(postalCode ? {laposte: {...address.meta?.laposte, codePostal: postalCode}} : {}) - } - } - }) - } +// Helpers for formatting data +export const formatCommonToponym = commonToponym => { + // To-do : define the format for the common toponym + // For now, we remove data calculation used for the legacy format (centroid, addressCount, certifiedAddressCount, bbox) + const {centroid, addressCount, certifiedAddressCount, addressBbox, bbox, ...rest} = commonToponym + return rest +} + +const formatAddress = address => { + // To-do : define the format for the address + // For now, we remove data calculation used for the legacy format (bbox) + const {bbox, ...rest} = address + return rest } // Helpers for calculation +export const calculateExtraDataForCommonToponym = (commonToponym, cog, fantoirFinder, commonToponymIDFantoirCodeMap) => { + // Calculate the fantoir code for each common toponym + const fantoirCode = calculateCommonToponymFantoirCode(commonToponymIDFantoirCodeMap, commonToponym, fantoirFinder) + // Calculate the tiles for each common toponym + const {geometry, tiles, x, y} = calculateCommonToponymGeometryAndTiles(commonToponym) + // Calculate the postal code for each common toponym + const {codePostal: postalCode, libelleAcheminement: deliveryLabel} = calculateCommonToponymPostalCode(commonToponymIDFantoirCodeMap, commonToponym, cog) + // Remove the centroid data from the common toponym + return {...commonToponym, + geometry, + meta: { + ...commonToponym.meta, + ...(fantoirCode ? {dgfip: {...commonToponym.meta?.dgfip, fantoir: fantoirCode}} : {}), + ...(tiles && x && y ? {geography: {...commonToponym.meta?.geography, tiles, x, y}} : {}), + ...(postalCode && deliveryLabel ? {laposte: {...commonToponym.meta?.laposte, codePostal: postalCode, libelleAcheminement: deliveryLabel}} : {}) + }} +} + +export const calculateExtraDataForAddress = (address, cog, commonToponymIDFantoirCodeMap) => { + // Calculate the tiles for each address + const {tiles, x, y} = calculateAddressTiles(address) + // Calculate the postal code for each address + const {codePostal: postalCode, libelleAcheminement: deliveryLabel} = calculateAddressPostalCode(commonToponymIDFantoirCodeMap, address, cog) + return {...address, + meta: { + ...address.meta, + ...(tiles && x && y ? {geography: {...address.meta?.geography, tiles, x, y}} : {}), + ...(postalCode && deliveryLabel ? {laposte: {...address.meta?.laposte, codePostal: postalCode, libelleAcheminement: deliveryLabel}} : {}) + } + } +} + // Helpers to calculate the fantoir code -const calculateCommonToponymFantoirCode = (commonToponym, cog, fantoirFinder) => { +const calculateCommonToponymFantoirCode = (commonToponymIDFantoirCodeMap, commonToponym, fantoirFinder) => { + const {meta} = commonToponym // Find the label in 'fra' if possible, otherwise take the first one - const labelValue = commonToponym?.labels?.find(({isoCode}) => isoCode === 'fra')?.value || commonToponym?.labels[0]?.value - const fantoirCode = fantoirFinder.findVoie(labelValue, cog)?.codeFantoir - // Store the fantoir code for the common toponym to be able to calculate the postal codes later + const labelValue = commonToponym?.labels?.find(({isoCode}) => isoCode === 'fra')?.value || commonToponym?.labels?.[0]?.value + const fantoirCode = calculateFantoirCode(fantoirFinder, labelValue, meta?.bal?.codeAncienneCommune) + if (!fantoirCode) { + return + } + commonToponymIDFantoirCodeMap.set(commonToponym.id, fantoirCode) return fantoirCode } +const calculateFantoirCode = (fantoirFinder, labelValue, codeAncienneCommune) => { + const fantoirData = fantoirFinder.findVoie(labelValue, codeAncienneCommune) + if (!fantoirData) { + return + } + + // If the fantoir data is canceled and has no successor, we don't store it + if (fantoirData.annulee && !fantoirData.successeur) { + return + } + + return fantoirData.successeur?.split('-')?.[1] || fantoirData.codeFantoir +} + // Helpers to calculate the postal code -const calculateCommonToponymPostalCode = (commonToponym, cog) => { +const calculateCommonToponymPostalCode = (commonToponymIDFantoirCodeMap, commonToponym, cog) => { const fantoirCode = commonToponymIDFantoirCodeMap.get(commonToponym.id) - const {codePostal} = findCodePostal(cog, fantoirCode) - return codePostal + const {codePostal, libelleAcheminement} = findCodePostal(cog, fantoirCode) + return {codePostal, libelleAcheminement} } -const calculateAddressPostalCode = (address, cog) => { +const calculateAddressPostalCode = (commonToponymIDFantoirCodeMap, address, cog) => { const fantoirCode = commonToponymIDFantoirCodeMap.get(address.mainCommonToponymID) const {number, suffix} = address - const {codePostal} = findCodePostal(cog, fantoirCode, number, suffix) - return codePostal + const {codePostal, libelleAcheminement} = findCodePostal(cog, fantoirCode, number, suffix) + return {codePostal, libelleAcheminement} } // Helpers to calculate the tiles -const calculateCommonToponymTiles = commonToponym => { - const {centroid} = commonToponym - if (!centroid) { +const calculateCommonToponymGeometryAndTiles = commonToponym => { + const {geometry: geometryFromCommonToponym, centroid} = commonToponym + let geometryFromCentroid + if (centroid) { + geometryFromCentroid = { + type: centroid.type, + coordinates: centroid.coordinates + } + } + + const geometry = geometryFromCommonToponym || geometryFromCentroid + if (!geometry) { return {} } - const {crs, ...position} = centroid - const {tiles, x, y} = derivePositionProps(position, COMMON_TOPONYM_TILES_MIN_ZOOM, COMMON_TOPONYM_TILES_MAX_ZOOM) - return {tiles, x, y} + const {tiles, x, y} = derivePositionProps(geometry, TILES_ZOOM_LEVELS.commonToponym.min, TILES_ZOOM_LEVELS.commonToponym.max) + return {geometry, tiles, x, y} } const calculateAddressTiles = address => { const {positions} = address - // Find the position with the highest priority - const positionPrioritized = positions.reduce((max, item) => (POSITION_TYPES_PRIORITY[item.type] < POSITION_TYPES_PRIORITY[max.type] ? item : max), positions[0]) - // Calculate the tiles for the position with the highest priority - const {tiles, x, y} = derivePositionProps(positionPrioritized?.geometry, ADDRESS_TILES_MIN_ZOOM, ADDRESS_TILES_MAX_ZOOM) + const {tiles, x, y} = derivePositionProps(positions?.[0].geometry, TILES_ZOOM_LEVELS.address.min, TILES_ZOOM_LEVELS.address.max) return {tiles, x, y} } diff --git a/lib/api/consumers/format-to-legacy-helpers.js b/lib/api/consumers/format-to-legacy-helpers.js new file mode 100644 index 00000000..49f543a5 --- /dev/null +++ b/lib/api/consumers/format-to-legacy-helpers.js @@ -0,0 +1,313 @@ +import {readFileSync} from 'node:fs' +import {CommonToponym, Address} from '../../util/sequelize.js' +import {convertToLegacyPositionType} from '../helper.js' +import {getCommune as getDistrictFromAdminDivision, getRegion, getDepartement as getDepartment} from '../../util/cog.cjs' + +const districtsGeographicOutlineJsonURL = new URL('../../../geo.json', import.meta.url) +const districtsGeographicOutline = JSON.parse(readFileSync(districtsGeographicOutlineJsonURL)) + +const districtsAddressesExtraDataJsonURL = new URL(`../../../${process.env.COMMUNES_LOCAUX_ADRESSES_DATA_PATH}`, import.meta.url) +const districtsAddressesExtraData = JSON.parse(readFileSync(districtsAddressesExtraDataJsonURL)) +const districtsAddressesExtraDataIndex = districtsAddressesExtraData.reduce((acc, commune) => { + acc[commune.codeCommune] = commune + return acc +}, {}) + +export const formatDistrictDataForLegacy = async (district, totalCommonToponymRecords, totalAddressRecords, transaction) => { + const {id, meta, labels} = district + const {insee: {cog}} = meta + + // Count the total number of "lieu-dit" common toponym used for the district legacy format + const totalSpecifCommonToponymRecords = await CommonToponym.count({ + where: { + districtID: id, + meta: { + bal: { + isLieuDit: true + } + }, + }, + transaction, + }) + + // Count the total number of certified addresses used for the district legacy format + const totalAddressCertifiedRecords = await Address.count({ + where: {districtID: id, certified: true}, + transaction, + }) + + // District data from administrative division + const districtFromAdminDivision = getDistrictFromAdminDivision(cog) + const {population, codesPostaux: postalCodes, type} = districtFromAdminDivision + const department = getDepartment(districtFromAdminDivision.departement) + const region = getRegion(districtFromAdminDivision.region) + + // Labels + const defaultLabel = getDefaultLabel(labels) + const legacyLabelValue = defaultLabel?.value + + // Geographic data + const districtGeoData = districtsGeographicOutline[cog] + const districtBbox = districtGeoData.bbox + + let addressAnalysis + // Address analysis + if (districtsAddressesExtraDataIndex[cog]) { + const totalAddressesFromDistrict = districtsAddressesExtraDataIndex[cog].nbAdressesLocaux + const ratio = totalAddressesFromDistrict && totalAddressRecords + ? Math.round((totalAddressRecords / totalAddressesFromDistrict) * 100) + : undefined + const addressesDeficit = (population < 2000 && population > 0) ? ratio < 50 : undefined + addressAnalysis = { + nbAdressesAttendues: totalAddressesFromDistrict, + ratio, + deficitAdresses: addressesDeficit + } + } + + return { + banId: district?.id, + codeCommune: cog, + nomCommune: legacyLabelValue, + population, + departement: {nom: department?.nom, code: department?.code}, + region: {nom: region?.nom, code: region?.code}, + codesPostaux: postalCodes || [], + displayBBox: districtBbox, + typeCommune: type, + typeComposition: 'bal', + nbVoies: totalCommonToponymRecords - totalSpecifCommonToponymRecords, + nbLieuxDits: totalSpecifCommonToponymRecords, + nbNumeros: totalAddressRecords, + nbNumerosCertifies: totalAddressCertifiedRecords, + ...(addressAnalysis ? {analyseAdressage: addressAnalysis} : {}), + idRevision: meta?.bal?.idRevision, + dateRevision: meta?.bal?.dateRevision, + composedAt: new Date() + } +} + +export const formatCommonToponymDataForLegacy = (commonToponym, district, pseudoCodeVoieGenerator, commonToponymIDlegacyCommonToponymIDMap) => { + const {labels: districtLabels, meta: {insee: {cog}}} = district + const {id, districtID, geometry, labels, meta, updateDate, addressCount, certifiedAddressCount, bbox, addressBbox} = commonToponym + + // Labels + // District + const {value: districtLegacyLabelValue} = getDefaultLabel(districtLabels) + + // Common toponym + const defaultLabel = getDefaultLabel(labels) + const defaultLabelIsoCode = defaultLabel?.isoCode + const legacyLabelValue = defaultLabel?.value + const legacyComplementaryLabels = formatLegacyComplementatyLabels(labels, defaultLabelIsoCode) + + // Ids + const codeAncienneCommune = meta?.bal?.codeAncienneCommune + const legacyCommonToponymFantoirId = meta?.dgfip?.fantoir ? `${cog}_${meta?.dgfip?.fantoir}` : null + const legacyCommonToponymId = meta?.dgfip?.fantoir + ? `${cog}_${meta?.dgfip?.fantoir}` + : `${cog}_${pseudoCodeVoieGenerator.getCode(legacyLabelValue, codeAncienneCommune)}`.toLowerCase() + + // Store the legacy common toponym id for each common toponym to then be able to set it on legacy addresses + commonToponymIDlegacyCommonToponymIDMap.set(id, legacyCommonToponymId) + + // Geographic data + const legacyPosition = { + ...geometry, + coordinates: [round(geometry?.coordinates?.[0]), round(geometry?.coordinates?.[1])] + } + const lon = legacyPosition?.coordinates?.[0] + const lat = legacyPosition?.coordinates?.[1] + const commonToponymBbox = formatBboxForLegacy(bbox) + const commonToponymAddressBbox = formatBboxForLegacy(addressBbox) + + const isLieuDit = meta?.bal?.isLieuDit + + if (isLieuDit) { + // Update Date + const legacyUpdateDate = formatLegacyUpdateDate(updateDate) + return { + banId: id, + banIdDistrict: districtID, + type: 'lieu-dit', + source: 'bal', + idVoie: legacyCommonToponymId, + nomVoie: legacyLabelValue, + nomVoieAlt: legacyComplementaryLabels, + codeCommune: cog, + nomCommune: districtLegacyLabelValue, + codeAncienneCommune, + nomAncienneCommune: meta?.bal?.nomAncienneCommune, + codePostal: meta?.laposte?.codePostal, + parcelles: meta?.cadastre?.ids || [], + lon, + lat, + x: meta?.geography?.x, + y: meta?.geography?.y, + tiles: meta?.geography?.tiles, + position: legacyPosition, + displayBBox: commonToponymBbox, + dateMAJ: legacyUpdateDate + } + } + + return { + banId: id, + banIdDistrict: districtID, + type: 'voie', + idVoie: legacyCommonToponymId, + idVoieFantoir: legacyCommonToponymFantoirId, + codeCommune: cog, + nomCommune: districtLegacyLabelValue, + codeAncienneCommune, + nomAncienneCommune: meta?.bal?.nomAncienneCommune, + nomVoie: legacyLabelValue, + nomVoieAlt: legacyComplementaryLabels, + sourceNomVoie: 'bal', + position: legacyPosition, + codePostal: meta?.laposte?.codePostal, + displayBBox: commonToponymAddressBbox, + lon, + lat, + x: meta?.geography?.x, + y: meta?.geography?.y, + tiles: meta?.geography?.tiles, + sources: ['bal'], + nbNumeros: Number.parseInt(addressCount, 10), + nbNumerosCertifies: Number.parseInt(certifiedAddressCount, 10) + } +} + +export const formatAddressDataForLegacy = (address, district, commonToponymIDlegacyCommonToponymIDMap) => { + const {meta: {insee: {cog}}} = district + const {id, mainCommonToponymID, secondaryCommonToponymIDs, districtID, number, suffix, positions, labels, meta, updateDate, certified, bbox} = address + + // Labels + const defaultLabel = getDefaultLabel(labels) + const defaultLabelIsoCode = defaultLabel?.isoCode + const legacyLabelValue = defaultLabel?.value + const legacyComplementaryLabels = formatLegacyComplementatyLabels(labels, defaultLabelIsoCode) + + // Update Date + const legacyUpdateDate = formatLegacyUpdateDate(updateDate) + + // Geographic data + const legacyPositions = positions.map(position => ({position: position.geometry, positionType: convertToLegacyPositionType(position.type)})) + const legacyPosition = legacyPositions?.[0]?.position + const legacyPositionType = legacyPositions?.[0]?.positionType + const [lon, lat] = formatLegacyLonLat(legacyPosition) + const addressBbox = formatBboxForLegacy(bbox) + + // Ids + const legacyCommonToponymId = commonToponymIDlegacyCommonToponymIDMap.get(mainCommonToponymID) + const legacyInteropKey = `${legacyCommonToponymId}_${String(number).padStart(5, '0')}${suffix ? `_${suffix}` : ''}`.toLowerCase() + const legacyID = legacyInteropKey + const banIdSecondaryCommonToponyms = secondaryCommonToponymIDs && secondaryCommonToponymIDs.length > 0 ? secondaryCommonToponymIDs : null + const legacySuffix = suffix ? suffix : null + + return { + id: legacyID, + cleInterop: legacyInteropKey, + banId: id, + banIdDistrict: districtID, + banIdMainCommonToponym: mainCommonToponymID, + banIdSecondaryCommonToponyms, + idVoie: legacyCommonToponymId, + codeCommune: cog, + codeAncienneCommune: meta?.bal?.codeAncienneCommune, + nomAncienneCommune: meta?.bal?.nomAncienneCommune, + numero: number, + suffixe: legacySuffix, + lieuDitComplementNom: legacyLabelValue, + lieuDitComplementNomAlt: legacyComplementaryLabels || {}, + parcelles: meta?.cadastre?.ids || [], + positions: legacyPositions, + position: legacyPosition, + positionType: legacyPositionType, + displayBBox: addressBbox, + lon, + lat, + x: meta?.geography?.x, + y: meta?.geography?.y, + tiles: meta?.geography?.tiles, + sources: ['bal'], + sourcePosition: 'bal', + dateMAJ: legacyUpdateDate, + certifie: certified, + codePostal: meta?.laposte?.codePostal, + libelleAcheminement: meta?.laposte?.libelleAcheminement, + adressesOriginales: [address] + } +} + +// Helper for formatting bbox to legacy bbox format +const formatBboxForLegacy = bbox => { + if (!bbox) { + return + } + + const {coordinates} = bbox + const allLon = [] + const allLat = [] + coordinates[0].forEach(([lon, lat] + ) => { + allLon.push(Number.parseFloat(lon)) + allLat.push(Number.parseFloat(lat)) + }) + const lonMin = round(Math.min(...allLon), 4) + const latMin = round(Math.min(...allLat), 4) + const lonMax = round(Math.max(...allLon), 4) + const latMax = round(Math.max(...allLat), 4) + return [lonMin, latMin, lonMax, latMax] +} + +const round = (value, precision = 6) => { + if (!value) { + return + } + + return Number.parseFloat(Number.parseFloat(value).toFixed(precision)) +} + +const getDefaultLabel = (labels, icoCode = 'fr') => { + if (!labels || labels.length === 0) { + return + } + + const label = labels.find(({isoCode}) => isoCode === icoCode) + return label || labels[0] +} + +const formatLegacyComplementatyLabels = (labels, defaultLabelIsoCode) => { + if (!labels || labels.length === 0) { + return + } + + const complementaryLabels = labels?.filter(({isoCode}) => isoCode !== defaultLabelIsoCode) + return complementaryLabels.reduce((acc, {isoCode, value}) => { + acc[isoCode] = value + return acc + }, {}) +} + +const formatLegacyUpdateDate = date => { + if (!date) { + return + } + + const year = date.getFullYear() + const month = String(date.getMonth() + 1).padStart(2, '0') // Adding 1 to month because it's 0-based + const day = String(date.getDate()).padStart(2, '0') + return `${year}-${month}-${day}` +} + +const formatLegacyLonLat = position => { + if (!position) { + return + } + + const {coordinates} = position + const lon = round(coordinates?.[0]) + const lat = round(coordinates?.[1]) + return [lon, lat] +} diff --git a/lib/api/helper.js b/lib/api/helper.js index 1041e470..0adb9d2c 100644 --- a/lib/api/helper.js +++ b/lib/api/helper.js @@ -1,3 +1,4 @@ +import {readFileSync} from 'node:fs' import {getCommonToponyms} from './common-toponym/models.js' import {getDistricts} from './district/models.js' @@ -145,3 +146,20 @@ export const formatObjectWithDefaults = (inputObject, defaultValues) => { return formattedObject } + +const positionTypeDictionaryURL = new URL('position-type-dictionary.json', import.meta.url) +const positionTypeDictionary = JSON.parse(readFileSync(positionTypeDictionaryURL)) + +const positionTypeConverter = (data, langFrom, langTo) => { + // eslint-disable-next-line unicorn/prefer-object-from-entries + const dataMemorized = data.reduce((acc, val) => ( + { + ...acc, + [val[langFrom]]: val[langTo], + } + ), {}) + + return key => dataMemorized[key] +} + +export const convertToLegacyPositionType = positionTypeConverter(positionTypeDictionary, 'eng', 'fra') diff --git a/lib/api/position-type-dictionary.json b/lib/api/position-type-dictionary.json new file mode 100644 index 00000000..4f80e474 --- /dev/null +++ b/lib/api/position-type-dictionary.json @@ -0,0 +1,11 @@ +[ + {"fra": "entrée", "eng": "entrance"}, + {"fra": "bâtiment", "eng": "building"}, + {"fra": "cage d’escalier", "eng": "staircase identifier"}, + {"fra": "logement", "eng": "unit identifier"}, + {"fra": "service technique", "eng": "utility service"}, + {"fra": "délivrance postale", "eng": "postal delivery"}, + {"fra": "parcelle", "eng": "parcel"}, + {"fra": "segment", "eng": "segment"}, + {"fra": "autre", "eng": "other"} +] \ No newline at end of file