From 68d4107dcac20083042b2eed3688fb7829b536a2 Mon Sep 17 00:00:00 2001 From: antoineludeau <52679050+antoineludeau@users.noreply.github.com> Date: Tue, 20 Aug 2024 11:22:51 +0200 Subject: [PATCH] Added bal meta clefInterop and deprecatedID retrieval in export from postgres to mongo --- lib/api/consumers/format-to-legacy-helpers.js | 83 ++++++++++++++++--- .../format-to-legacy-helpers.spec.js | 49 +++++++++++ 2 files changed, 121 insertions(+), 11 deletions(-) create mode 100644 lib/api/consumers/format-to-legacy-helpers.spec.js diff --git a/lib/api/consumers/format-to-legacy-helpers.js b/lib/api/consumers/format-to-legacy-helpers.js index 473b8811..00e8f73d 100644 --- a/lib/api/consumers/format-to-legacy-helpers.js +++ b/lib/api/consumers/format-to-legacy-helpers.js @@ -8,7 +8,13 @@ const districtsGeographicOutlineJsonURL = new URL('../../../geo.json', import.me const districtsGeographicOutline = JSON.parse(readFileSync(districtsGeographicOutlineJsonURL)) const districtsAddressesExtraDataJsonURL = new URL(`../../../${process.env.COMMUNES_LOCAUX_ADRESSES_DATA_PATH}`, import.meta.url) -const districtsAddressesExtraData = JSON.parse(readFileSync(districtsAddressesExtraDataJsonURL)) +let districtsAddressesExtraData = [] +try { + districtsAddressesExtraData = JSON.parse(readFileSync(districtsAddressesExtraDataJsonURL)) +} catch (error) { + console.error('Error while reading the districts addresses extra data file', error) +} + const districtsAddressesExtraDataIndex = districtsAddressesExtraData.reduce((acc, commune) => { acc[commune.codeCommune] = commune return acc @@ -107,15 +113,7 @@ export const formatCommonToponymDataForLegacy = (commonToponym, district, pseudo const codeAncienneCommune = meta?.bal?.codeAncienneCommune const legacyCommonToponymFantoirId = meta?.dgfip?.fantoir ? `${cog}_${meta?.dgfip?.fantoir}` : null - let legacyCommonToponymId = legacyCommonToponymFantoirId - // If the legacy common toponym id is already used or not defined, we calculate a pseudo code - if (!legacyCommonToponymId || commonToponymLegacyIDSet.has(legacyCommonToponymId)) { - legacyCommonToponymId = `${cog}_${pseudoCodeVoieGenerator.getCode(legacyLabelValue, codeAncienneCommune)}`.toLowerCase() - // If the pseudo code is already used, we generate a new one with a hash from the common toponym id - if (commonToponymLegacyIDSet.has(legacyCommonToponymId)) { - legacyCommonToponymId = `${cog}_${createHmac('sha256', 'ban').update(id).digest('hex').slice(0, 6)}` - } - } + const legacyCommonToponymId = getLegacyCommonToponymId(commonToponymLegacyIDSet, legacyCommonToponymFantoirId, meta, {cog, id, pseudoCodeVoieGenerator, legacyLabelValue, codeAncienneCommune}) // Store the legacy common toponym id for each common toponym to then be able to set it on legacy addresses commonToponymLegacyIDCommonToponymIDMap.set(id, legacyCommonToponymId) @@ -212,7 +210,7 @@ export const formatAddressDataForLegacy = (address, district, commonToponymLegac // Ids const legacyCommonToponymId = commonToponymLegacyIDCommonToponymIDMap.get(mainCommonToponymID) - const legacyInteropKey = `${legacyCommonToponymId}_${String(number).padStart(5, '0')}${suffix ? `_${suffix}` : ''}`.toLowerCase() + const legacyInteropKey = getAddressLegacyInteropKey(meta, {legacyCommonToponymId, number, suffix}) const legacyID = getAddressLegacyId(addressLegacyIDSet, legacyInteropKey) addressLegacyIDSet.add(legacyID) const banIdSecondaryCommonToponyms = secondaryCommonToponymIDs && secondaryCommonToponymIDs.length > 0 ? secondaryCommonToponymIDs : null @@ -332,3 +330,66 @@ const getAddressLegacyId = (addressLegacyIDSet, legacyInteropKey, suffix = 0) => return `${legacyInteropKey}` } + +const getAddressLegacyInteropKey = (meta, {legacyCommonToponymId, number, suffix}) => { + const cleInterop = meta.bal?.cleInterop + const isValidInteropKey = checkIfDeprecatedIDIsValid(cleInterop, true) + const addressLegacyInteropKey = isValidInteropKey + ? cleInterop + : `${legacyCommonToponymId}_${String(number).padStart(5, '0')}${suffix ? `_${suffix}` : ''}`.toLowerCase() + + return addressLegacyInteropKey +} + +const getLegacyCommonToponymId = ( + commonToponymLegacyIDSet, + legacyCommonToponymFantoirId, + meta, + {cog, id, pseudoCodeVoieGenerator, legacyLabelValue, codeAncienneCommune} +) => { + const deprecatedID = meta.bal?.deprecatedID + const isDeprecatedIDValid = checkIfDeprecatedIDIsValid(deprecatedID) + let legacyCommonToponymId = isDeprecatedIDValid ? deprecatedID : legacyCommonToponymFantoirId + + // If the legacy common toponym id is already used or not defined, we calculate a pseudo code + if (!legacyCommonToponymId || commonToponymLegacyIDSet.has(legacyCommonToponymId)) { + legacyCommonToponymId = `${cog}_${pseudoCodeVoieGenerator.getCode(legacyLabelValue, codeAncienneCommune)}`.toLowerCase() + // If the pseudo code is already used, we generate a new one with a hash from the common toponym id + if (commonToponymLegacyIDSet.has(legacyCommonToponymId)) { + legacyCommonToponymId = `${cog}_${createHmac('sha256', 'ban').update(id).digest('hex').slice(0, 6)}` + } + } + + return legacyCommonToponymId +} + +export const checkIfDeprecatedIDIsValid = (id, isCleInterop = false) => { + if (!id) { + return false + } + + // https://aitf-sig-topo.github.io/voies-adresses/files/AITF_SIG_Topo_Format_Base_Adresse_Locale_v1.3.pdf + // ● INSEE code with 5 characters + // ● Street code: the unique street identifier provided by the national unique service, + // or the FANTOIR DGFiP code, consisting of 4 to 6 alphanumeric characters + // ● Address number consisting of 5 characters, prefixed with zeros if necessary + // ● Suffix (bis / ter / qua / qui / a / b / c...). The repetition indices “bis, ter…” will + // be coded with 3 characters, and others (a, b, c, a1, b2...) will be in lowercase + // without a fixed number of characters. + // ● Each item is separated by an underscore “_” + // ● Everything is in lowercase + + const deprecatedIDFormatRegex = isCleInterop + ? /^[a-z\d]{5}_(?:[a-z\d]{4}|[a-z\d]{6})_\d{5}(_[a-z\d_]+)?$/ + : /^[a-z\d]{5}_(?:[a-z\d]{4}|[a-z\d]{6})$/ + if (!deprecatedIDFormatRegex.test(id)) { + return false + } + + // Check if the street code is 'xxxx' to detect the non-valid pseudo code created by "mes-adresses" service + if (id.split('_')[1] === 'xxxx') { + return false + } + + return true +} diff --git a/lib/api/consumers/format-to-legacy-helpers.spec.js b/lib/api/consumers/format-to-legacy-helpers.spec.js new file mode 100644 index 00000000..58368d43 --- /dev/null +++ b/lib/api/consumers/format-to-legacy-helpers.spec.js @@ -0,0 +1,49 @@ +import {checkIfDeprecatedIDIsValid} from './format-to-legacy-helpers.js' + +describe('checkIfDeprecatedIDIsValid', () => { + test('should return false for an empty string', () => { + expect(checkIfDeprecatedIDIsValid('')).toBe(false) + expect(checkIfDeprecatedIDIsValid('', true)).toBe(false) + }) + + test('should return false for a null value', () => { + expect(checkIfDeprecatedIDIsValid(null)).toBe(false) + expect(checkIfDeprecatedIDIsValid(null, true)).toBe(false) + }) + + test('should return false for a string with "xxxx" as the street code', () => { + expect(checkIfDeprecatedIDIsValid('31555_xxxx_00001', true)).toBe(false) + expect(checkIfDeprecatedIDIsValid('31555_xxxx')).toBe(false) + }) + + test('should return false for a string not matching the pattern', () => { + expect(checkIfDeprecatedIDIsValid('315_7172_00001', true)).toBe(false) + expect(checkIfDeprecatedIDIsValid('31555_7172a_00001', true)).toBe(false) + expect(checkIfDeprecatedIDIsValid('31555_ilwhc0_000001', true)).toBe(false) + expect(checkIfDeprecatedIDIsValid('31555_ilwhc0_0000a', true)).toBe(false) + expect(checkIfDeprecatedIDIsValid('31555_Ilwhc0_00001', true)).toBe(false) + expect(checkIfDeprecatedIDIsValid('31555_Ilwhc0_00001_bât', true)).toBe(false) + expect(checkIfDeprecatedIDIsValid('31555_Ilwhc0_00001_bât.a', true)).toBe(false) + expect(checkIfDeprecatedIDIsValid('31555_Ilwhc0_00001_bât a', true)).toBe(false) + expect(checkIfDeprecatedIDIsValid('315_7172')).toBe(false) + expect(checkIfDeprecatedIDIsValid('31555_7172a')).toBe(false) + expect(checkIfDeprecatedIDIsValid('31555_Ilwhc0')).toBe(false) + }) + + test('should return false for a string with invalid characters', () => { + expect(checkIfDeprecatedIDIsValid('31555_ilwhc#_00001', true)).toBe(false) + expect(checkIfDeprecatedIDIsValid('31555_ilwhc!_00001', true)).toBe(false) + expect(checkIfDeprecatedIDIsValid('31555_ilwhc#')).toBe(false) + expect(checkIfDeprecatedIDIsValid('31555_ilwhc!')).toBe(false) + }) + + test('should return true for a string matching the pattern', () => { + expect(checkIfDeprecatedIDIsValid('31555_ilwhc0_00001', true)).toBe(true) + expect(checkIfDeprecatedIDIsValid('31555_7172_00001', true)).toBe(true) + expect(checkIfDeprecatedIDIsValid('31555_7172_00001_bis', true)).toBe(true) + expect(checkIfDeprecatedIDIsValid('31555_7172_00001_a1', true)).toBe(true) + expect(checkIfDeprecatedIDIsValid('31555_7172_00001_bis_a1', true)).toBe(true) + expect(checkIfDeprecatedIDIsValid('31555_ilwhc0')).toBe(true) + expect(checkIfDeprecatedIDIsValid('31555_7172')).toBe(true) + }) +})