From 3760aa54944e2f403b2b3bb46a99c090e940a164 Mon Sep 17 00:00:00 2001 From: antoineludeau <52679050+antoineludeau@users.noreply.github.com> Date: Fri, 26 Jul 2024 15:51:53 +0200 Subject: [PATCH] Added bal meta clefInterop retrieval and check in export from postgres to mongo --- lib/api/consumers/format-to-legacy-helpers.js | 87 ++++++++++++++++--- .../format-to-legacy-helpers.spec.js | 36 ++++++++ 2 files changed, 112 insertions(+), 11 deletions(-) create mode 100644 lib/api/consumers/format-to-legacy-helpers.spec.js diff --git a/lib/api/consumers/format-to-legacy-helpers.js b/lib/api/consumers/format-to-legacy-helpers.js index 473b8811..4ff07ab0 100644 --- a/lib/api/consumers/format-to-legacy-helpers.js +++ b/lib/api/consumers/format-to-legacy-helpers.js @@ -8,7 +8,13 @@ const districtsGeographicOutlineJsonURL = new URL('../../../geo.json', import.me const districtsGeographicOutline = JSON.parse(readFileSync(districtsGeographicOutlineJsonURL)) const districtsAddressesExtraDataJsonURL = new URL(`../../../${process.env.COMMUNES_LOCAUX_ADRESSES_DATA_PATH}`, import.meta.url) -const districtsAddressesExtraData = JSON.parse(readFileSync(districtsAddressesExtraDataJsonURL)) +let districtsAddressesExtraData = [] +try { + districtsAddressesExtraData = JSON.parse(readFileSync(districtsAddressesExtraDataJsonURL)) +} catch (error) { + console.error('Error while reading the districts addresses extra data file', error) +} + const districtsAddressesExtraDataIndex = districtsAddressesExtraData.reduce((acc, commune) => { acc[commune.codeCommune] = commune return acc @@ -107,15 +113,7 @@ export const formatCommonToponymDataForLegacy = (commonToponym, district, pseudo const codeAncienneCommune = meta?.bal?.codeAncienneCommune const legacyCommonToponymFantoirId = meta?.dgfip?.fantoir ? `${cog}_${meta?.dgfip?.fantoir}` : null - let legacyCommonToponymId = legacyCommonToponymFantoirId - // If the legacy common toponym id is already used or not defined, we calculate a pseudo code - if (!legacyCommonToponymId || commonToponymLegacyIDSet.has(legacyCommonToponymId)) { - legacyCommonToponymId = `${cog}_${pseudoCodeVoieGenerator.getCode(legacyLabelValue, codeAncienneCommune)}`.toLowerCase() - // If the pseudo code is already used, we generate a new one with a hash from the common toponym id - if (commonToponymLegacyIDSet.has(legacyCommonToponymId)) { - legacyCommonToponymId = `${cog}_${createHmac('sha256', 'ban').update(id).digest('hex').slice(0, 6)}` - } - } + const legacyCommonToponymId = getLegacyCommonToponymId(commonToponymLegacyIDSet, legacyCommonToponymFantoirId, meta, {cog, id, pseudoCodeVoieGenerator, legacyLabelValue, codeAncienneCommune}) // Store the legacy common toponym id for each common toponym to then be able to set it on legacy addresses commonToponymLegacyIDCommonToponymIDMap.set(id, legacyCommonToponymId) @@ -212,7 +210,7 @@ export const formatAddressDataForLegacy = (address, district, commonToponymLegac // Ids const legacyCommonToponymId = commonToponymLegacyIDCommonToponymIDMap.get(mainCommonToponymID) - const legacyInteropKey = `${legacyCommonToponymId}_${String(number).padStart(5, '0')}${suffix ? `_${suffix}` : ''}`.toLowerCase() + const legacyInteropKey = getAddressLegacyInteropKey(meta, {legacyCommonToponymId, number, suffix}) const legacyID = getAddressLegacyId(addressLegacyIDSet, legacyInteropKey) addressLegacyIDSet.add(legacyID) const banIdSecondaryCommonToponyms = secondaryCommonToponymIDs && secondaryCommonToponymIDs.length > 0 ? secondaryCommonToponymIDs : null @@ -332,3 +330,70 @@ const getAddressLegacyId = (addressLegacyIDSet, legacyInteropKey, suffix = 0) => return `${legacyInteropKey}` } + +const getAddressLegacyInteropKey = (meta, {legacyCommonToponymId, number, suffix}) => { + const cleInterop = meta.bal?.cleInterop + const isValidInteropKey = checkIfCleInteropIsValid(cleInterop) + const addressLegacyInteropKey = isValidInteropKey + ? cleInterop + : `${legacyCommonToponymId}_${String(number).padStart(5, '0')}${suffix ? `_${suffix}` : ''}`.toLowerCase() + + return addressLegacyInteropKey +} + +const getLegacyCommonToponymId = ( + commonToponymLegacyIDSet, + legacyCommonToponymFantoirId, + meta, + {cog, id, pseudoCodeVoieGenerator, legacyLabelValue, codeAncienneCommune} +) => { + let legacyCommonToponymId + const cleInterop = meta.bal?.cleInterop + const isCleInteropValid = checkIfCleInteropIsValid(cleInterop) + if (isCleInteropValid) { + const cleInteropParts = cleInterop.split('_') + legacyCommonToponymId = `${cleInteropParts[0]}_${cleInteropParts[1]}` + } else { + legacyCommonToponymId = legacyCommonToponymFantoirId + } + + // If the legacy common toponym id is already used or not defined, we calculate a pseudo code + if (!legacyCommonToponymId || commonToponymLegacyIDSet.has(legacyCommonToponymId)) { + legacyCommonToponymId = `${cog}_${pseudoCodeVoieGenerator.getCode(legacyLabelValue, codeAncienneCommune)}`.toLowerCase() + // If the pseudo code is already used, we generate a new one with a hash from the common toponym id + if (commonToponymLegacyIDSet.has(legacyCommonToponymId)) { + legacyCommonToponymId = `${cog}_${createHmac('sha256', 'ban').update(id).digest('hex').slice(0, 6)}` + } + } + + return legacyCommonToponymId +} + +export const checkIfCleInteropIsValid = cleInterop => { + if (!cleInterop) { + return false + } + + // https://aitf-sig-topo.github.io/voies-adresses/files/AITF_SIG_Topo_Format_Base_Adresse_Locale_v1.3.pdf + // ● INSEE code with 5 characters + // ● Street code: the unique street identifier provided by the national unique service, + // or the FANTOIR DGFiP code, consisting of 4 to 6 alphanumeric characters + // ● Address number consisting of 5 characters, prefixed with zeros if necessary + // ● Suffix (bis / ter / qua / qui / a / b / c...). The repetition indices “bis, ter…” will + // be coded with 3 characters, and others (a, b, c, a1, b2...) will be in lowercase + // without a fixed number of characters. + // ● Each item is separated by an underscore “_” + // ● Everything is in lowercase + + const cleInteropFormatRegex = /^[a-z\d]{5}_(?:[a-z\d]{4}|[a-z\d]{6})_\d{5}(_.+)?$/ + if (!cleInteropFormatRegex.test(cleInterop)) { + return false + } + + // Check if the street code is 'xxxx' to detect the non-valid pseudo code created by "mes-adresses" service + if (cleInterop.split('_')[1] === 'xxxx') { + return false + } + + return true +} diff --git a/lib/api/consumers/format-to-legacy-helpers.spec.js b/lib/api/consumers/format-to-legacy-helpers.spec.js new file mode 100644 index 00000000..535a0051 --- /dev/null +++ b/lib/api/consumers/format-to-legacy-helpers.spec.js @@ -0,0 +1,36 @@ +import {checkIfCleInteropIsValid} from './format-to-legacy-helpers.js' + +describe('checkIfCleInteropIsValid', () => { + test('should return false for an empty string', () => { + expect(checkIfCleInteropIsValid('')).toBe(false) + }) + + test('should return false for a null value', () => { + expect(checkIfCleInteropIsValid(null)).toBe(false) + }) + + test('should return false for a string with "xxxx" as the street code', () => { + expect(checkIfCleInteropIsValid('31555_xxxx_00001')).toBe(false) + }) + + test('should return false for a string not matching the pattern', () => { + expect(checkIfCleInteropIsValid('315_7172_00001')).toBe(false) + expect(checkIfCleInteropIsValid('31555_7172a_00001')).toBe(false) + expect(checkIfCleInteropIsValid('31555_ilwhc0_000001')).toBe(false) + expect(checkIfCleInteropIsValid('31555_ilwhc0_0000a')).toBe(false) + expect(checkIfCleInteropIsValid('31555_Ilwhc0_00001')).toBe(false) + }) + + test('should return false for a string with invalid characters', () => { + expect(checkIfCleInteropIsValid('31555_ilwhc#_00001')).toBe(false) + expect(checkIfCleInteropIsValid('31555_ilwhc!_00001')).toBe(false) + }) + + test('should return true for a string matching the pattern', () => { + expect(checkIfCleInteropIsValid('31555_ilwhc0_00001')).toBe(true) + expect(checkIfCleInteropIsValid('31555_7172_00001')).toBe(true) + expect(checkIfCleInteropIsValid('31555_7172_00001_bis')).toBe(true) + expect(checkIfCleInteropIsValid('31555_7172_00001_a1')).toBe(true) + expect(checkIfCleInteropIsValid('31555_7172_00001_bât. a')).toBe(true) + }) +})