diff --git a/lib/api/consumers/api-consumer.js b/lib/api/consumers/api-consumer.js index 68bde807..58d1f195 100644 --- a/lib/api/consumers/api-consumer.js +++ b/lib/api/consumers/api-consumer.js @@ -5,7 +5,7 @@ import {checkAddressesRequest, checkAddressesIDsRequest} from '../address/utils. import {setCommonToponyms, updateCommonToponyms, patchCommonToponyms, deleteCommonToponyms, getAllDistrictIDsFromCommonToponyms} from '../common-toponym/models.js' import {checkCommonToponymsRequest, checkCommonToponymsIDsRequest} from '../common-toponym/utils.js' import {setDistricts, updateDistricts, patchDistricts, deleteDistricts} from '../district/models.js' -import {checkDistrictsRequest, checkDistrictsIDsRequest} from '../district/utils.js' +import {checkDistrictsRequest, checkDistrictsIDsRequest, formatDataNova, formatDataNovaFromUrl} from '../district/utils.js' import {dataValidationReportFrom, formatObjectWithDefaults, addOrUpdateJob, formatPayloadDates} from '../helper.js' import {addressDefaultOptionalValues} from '../address/schema.js' import {commonToponymDefaultOptionalValues} from '../common-toponym/schema.js' @@ -216,6 +216,9 @@ const districtConsumer = async (jobType, payload, statusID) => { return checkDistrictsRequest(payload, jobType) case 'delete': return checkDistrictsIDsRequest(payload, jobType) + case 'updatePostalCode': + case 'updatePostalCodeFromUrl': + return checkDistrictsRequest(payload, 'patch') default: return dataValidationReportFrom(false, 'Unknown action type', {actionType: jobType, payload}) } @@ -229,6 +232,10 @@ const districtConsumer = async (jobType, payload, statusID) => { return formatPayloadDates(payload, jobType) case 'delete': return payload + case 'updatePostalCodeFromUrl': + return formatDataNovaFromUrl(payload) + case 'updatePostalCode': + return formatDataNova(payload) default: console.warn(`District Consumer Warn: Unknown job type : '${jobType}'`) } @@ -255,6 +262,8 @@ const districtConsumer = async (jobType, payload, statusID) => { } case 'patch': + case 'updatePostalCodeFromUrl': + case 'updatePostalCode': await patchDistricts(formattedPayload) break case 'delete': diff --git a/lib/api/district/__mocks__/district-models.js b/lib/api/district/__mocks__/district-models.js index 003be473..43f74ca4 100644 --- a/lib/api/district/__mocks__/district-models.js +++ b/lib/api/district/__mocks__/district-models.js @@ -3,3 +3,10 @@ import {bddDistrictMock} from './district-data-mock.js' export async function getDistricts(districtIDs) { return bddDistrictMock.filter(({id}) => districtIDs.includes(id)) } + +export async function getDistrictsFromCogList(districtCOGs) { + return bddDistrictMock + .map(district => [district.meta.insee.cog, district]) + .filter(([cog]) => districtCOGs.includes(cog)) + .map(([, district]) => district) +} diff --git a/lib/api/district/models.js b/lib/api/district/models.js index d1364446..7167c6fe 100644 --- a/lib/api/district/models.js +++ b/lib/api/district/models.js @@ -1,3 +1,4 @@ +import {Op} from 'sequelize' import {District} from '../../util/sequelize.js' export const getDistrict = districtID => District.findByPk(districtID, {raw: true}) @@ -6,6 +7,8 @@ export const getDistricts = districtIDs => District.findAll({where: {id: distric export const getDistrictsFromCog = cog => District.findAll({where: {meta: {insee: {cog}}}, raw: true}) +export const getDistrictsFromCogList = cogList => District.findAll({where: {meta: {insee: {cog: {[Op.or]: cogList}}}}, raw: true}) + export const setDistricts = districts => District.bulkCreate(districts) export const updateDistricts = districts => { diff --git a/lib/api/district/routes.js b/lib/api/district/routes.js index 7b667b02..8dbe9445 100644 --- a/lib/api/district/routes.js +++ b/lib/api/district/routes.js @@ -99,6 +99,69 @@ app.route('/') res.send(response) }) +app.route('/postal-codes-from-datanova-url', auth) + .put(async (req, res) => { + let response + try { + // On February 2024 the postal file url is : + // https://datanova.laposte.fr/data-fair/api/v1/datasets/laposte-hexasmal/raw + const {url} = req.query + const statusID = nanoid() + + await apiQueue.add( + {dataType: 'district', jobType: 'updatePostalCodeFromUrl', data: url, statusID}, + {jobId: statusID, removeOnComplete: true} + ) + + response = { + date: new Date(), + status: 'success', + message: `Check the status of your request : ${BAN_API_URL}/job-status/${statusID}`, + response: {statusID}, + } + } catch (error) { + const {message} = error + response = { + date: new Date(), + status: 'error', + message, + response: {}, + } + } + + res.send(response) + }) + +app.route('/postal-codes-from-datanova', auth) + .put(express.text(), async (req, res) => { + let response + try { + const postalFile = req.body + const statusID = nanoid() + + await apiQueue.add( + {dataType: 'district', jobType: 'updatePostalCode', data: postalFile, statusID}, + {jobId: statusID, removeOnComplete: true} + ) + response = { + date: new Date(), + status: 'success', + message: `Check the status of your request : ${BAN_API_URL}/job-status/${statusID}`, + response: {statusID}, + } + } catch (error) { + const {message} = error + response = { + date: new Date(), + status: 'error', + message, + response: {}, + } + } + + res.send(response) + }) + app.route('/:districtID') .get(analyticsMiddleware, async (req, res) => { let response diff --git a/lib/api/district/schema.js b/lib/api/district/schema.js index 6bea9052..896a9d4e 100644 --- a/lib/api/district/schema.js +++ b/lib/api/district/schema.js @@ -1,5 +1,5 @@ import {object, string, array, date, bool} from 'yup' -import {banID, labelSchema, balSchema} from '../schema.js' +import {banID, labelSchema, balSchema, laPosteSchema} from '../schema.js' const configSchema = object({ useBanId: bool() @@ -15,6 +15,7 @@ const inseeSchema = object({ const metaSchema = object({ insee: inseeSchema, bal: balSchema, + laPoste: laPosteSchema, }).noUnknown() export const banDistrictSchema = object({ diff --git a/lib/api/district/utils.js b/lib/api/district/utils.js index fe588d53..94966f2d 100644 --- a/lib/api/district/utils.js +++ b/lib/api/district/utils.js @@ -1,6 +1,8 @@ +import Papa from 'papaparse' +import fetch from '../../util/fetch.cjs' import {checkDataFormat, dataValidationReportFrom, checkIdsIsUniq, checkIdsIsVacant, checkIdsIsAvailable, checkDataShema, checkIdsShema} from '../helper.js' import {banID} from '../schema.js' -import {getDistricts} from './models.js' +import {getDistricts, getDistrictsFromCogList} from './models.js' import {banDistrictSchema} from './schema.js' const getExistingDistrictIDs = async districtIDs => { @@ -72,3 +74,72 @@ export const formatDistrict = district => { const lastRecordDate = rangeValidity[0].value return {...districtRest, lastRecordDate} } + +export async function formatDataNova(postalFile) { + /* eslint-disable camelcase */ + const headers = { + '#Code_commune_INSEE': 'codeInsee', + Nom_de_la_commune: 'nomCommune', + Code_postal: 'codePostal', + Libellé_d_acheminement: 'libelleAcheminement', + 'Libell�_d_acheminement': 'libelleAcheminement', // Postal file url returns wrong header charset code (UTF-8 instead of ISO-8859-1) + Ligne_5: 'ligne5', + } + /* eslint-enable camelcase */ + + const dataRaw = await Papa.parse(postalFile, { + header: true, + transformHeader: name => headers[name] || name, + skipEmptyLines: true, + }) + + const districts = await getDistrictsFromCogList( + dataRaw.data.map(({codeInsee}) => codeInsee) + ) + + const districtsByInseeCode = districts.reduce( + (acc, district) => ({ + ...acc, + ...(district?.meta?.insee?.cog + ? {[district.meta.insee.cog]: [...(acc[district.meta.insee.cog] || []), district.id]} + : {} + ), + }), {}) + + const banDistricts = Object.values( + (dataRaw?.data || []).flatMap(({codeInsee, codePostal, libelleAcheminement}) => { + const ids = districtsByInseeCode[codeInsee] || [] + return ids.map(id => ({id, codePostal: [codePostal], libelleAcheminement})) + }) + .reduce( + (acc, district) => { + if (district.id) { + if (acc[district.id]) { + acc[district.id].codePostal = [...acc[district.id].codePostal, ...district.codePostal] + } else { + acc[district.id] = district + } + } + + return acc + }, {} + ) + ) + + return banDistricts.map(({id, codePostal, libelleAcheminement}) => ({ + id, + meta: { + laPoste: { + codePostal, + libelleAcheminement, + source: 'La Poste - dataNOVA', + } + } + })) +} + +export async function formatDataNovaFromUrl(url) { + const postalFileResponse = await fetch(url) + const postalFile = await postalFileResponse.text() + return formatDataNova(postalFile) +} diff --git a/lib/api/schema.js b/lib/api/schema.js index ee9a9b85..34182848 100644 --- a/lib/api/schema.js +++ b/lib/api/schema.js @@ -24,6 +24,12 @@ export const balSchema = object({ isLieuDit: boolean() }).noUnknown() +export const laPosteSchema = object({ + source: string().trim(), + codePostal: array().of(string().trim()), + libelleAcheminement: string().trim(), +}).noUnknown() + export const idfixSchema = object({ hash: string().trim(), }).noUnknown()