From 4862d5cbf98ad1a50836c93a19bc9dc22b3ef547 Mon Sep 17 00:00:00 2001 From: Florian Loreau Date: Fri, 16 Feb 2024 10:43:21 +0100 Subject: [PATCH 1/5] Add API route to update districts postal code --- lib/api/district/models.js | 16 +++++- lib/api/district/routes.js | 111 ++++++++++++++++++++++++++++++++++++- 2 files changed, 124 insertions(+), 3 deletions(-) diff --git a/lib/api/district/models.js b/lib/api/district/models.js index d1364446..47dbbd31 100644 --- a/lib/api/district/models.js +++ b/lib/api/district/models.js @@ -1,10 +1,24 @@ +import {Op} from 'sequelize' import {District} from '../../util/sequelize.js' export const getDistrict = districtID => District.findByPk(districtID, {raw: true}) export const getDistricts = districtIDs => District.findAll({where: {id: districtIDs}, raw: true}) -export const getDistrictsFromCog = cog => District.findAll({where: {meta: {insee: {cog}}}, raw: true}) +export const getDistrictFromCog = cog => District.findAll({where: {meta: {insee: {cog}}}, raw: true}) + +export const getDistrictsFromCog = async cog => { + const districts = await District.findAll({where: {meta: {insee: {cog: {[Op.or]: cog}}}}, raw: true}) + const districtsByInseeCode = districts.reduce( + (acc, district) => { + if (!acc[district.meta?.insee?.cog]) { + acc[district.meta.insee.cog] = district + } + + return acc + }, {}) + return cog.map(codeInsee => districtsByInseeCode[codeInsee] || {}) +} export const setDistricts = districts => District.bulkCreate(districts) diff --git a/lib/api/district/routes.js b/lib/api/district/routes.js index 7b667b02..6b94cfd3 100644 --- a/lib/api/district/routes.js +++ b/lib/api/district/routes.js @@ -1,10 +1,12 @@ import 'dotenv/config.js' // eslint-disable-line import/no-unassigned-import import {customAlphabet} from 'nanoid' import express from 'express' +import Papa from 'papaparse' import queue from '../../util/queue.cjs' import auth from '../../middleware/auth.js' import analyticsMiddleware from '../../middleware/analytics.js' -import {getDistrict, getDistrictsFromCog, deleteDistrict} from './models.js' +import fetch from '../../util/fetch.cjs' +import {getDistrict, getDistrictFromCog, getDistrictsFromCog, deleteDistrict, patchDistricts} from './models.js' import {formatDistrict} from './utils.js' const apiQueue = queue('api') @@ -192,7 +194,7 @@ app.get('/cog/:cog', analyticsMiddleware, async (req, res) => { let response try { const {cog} = req.params - const districts = await getDistrictsFromCog(cog) + const districts = await getDistrictFromCog(cog) if (!districts || districts.length === 0) { res.status(404).send('Request ID unknown') @@ -219,4 +221,109 @@ app.get('/cog/:cog', analyticsMiddleware, async (req, res) => { res.send(response) }) +async function updateDbFromDataNova(postalFile) { + /* eslint-disable camelcase */ + const headers = { + '#Code_commune_INSEE': 'codeInsee', + Nom_de_la_commune: 'nomCommune', + Code_postal: 'codePostal', + Libellé_d_acheminement: 'libelleAcheminement', + 'Libell�_d_acheminement': 'libelleAcheminement', // Postal file url returns wrong header charset code (UTF-8 instead of ISO-8859-1) + Ligne_5: 'ligne5', + } + /* eslint-enable camelcase */ + + const dataRaw = await Papa.parse(postalFile, { + header: true, + transformHeader: name => headers[name] || name, + skipEmptyLines: true, + }) + + const districts = await getDistrictsFromCog( + dataRaw.data.map(({codeInsee}) => codeInsee) + ) + + const districtsByInseeCode = districts.reduce( + (acc, district) => ({ + ...acc, + ...(!district?.meta || acc[district.meta?.insee?.cog] ? {} : {[district.meta.insee.cog]: district}), + }), {}) + + const banDistricts = Object.values( + (dataRaw?.data || []).map(({codeInsee, codePostal, libelleAcheminement}) => { + const {id} = districtsByInseeCode[codeInsee] || {} + return {id, codePostal: [codePostal], libelleAcheminement} + }) + .reduce( + (acc, district) => { + if (district.id) { + if (acc[district.id]) { + acc[district.id].codePostal = [...acc[district.id].codePostal, ...district.codePostal] + } else { + acc[district.id] = district + } + } + + return acc + }, {} + ) + ) + + const patchBulkOperations = banDistricts.map(({id, codePostal, libelleAcheminement}) => ({ + id, + meta: { + laPoste: { + codePostal, + libelleAcheminement, + source: 'La Poste - dataNOVA', + } + } + })) + + patchDistricts(patchBulkOperations) + + return banDistricts +} + +app.route('/codePostal') + .get(async (req, res) => { + let response + try { + // On February 2024 the postal file url is : + // https://datanova.laposte.fr/data-fair/api/v1/datasets/laposte-hexasmal/raw + const {url} = req.query + const postalFile = await fetch(url) + + response = await updateDbFromDataNova(postalFile) + } catch (error) { + const {message} = error + response = { + date: new Date(), + status: 'error', + message, + response: {}, + } + } + + res.send(response) + }) + .post(express.text(), async (req, res) => { + let response + try { + const postalFile = req.body + + response = await updateDbFromDataNova(postalFile) + } catch (error) { + const {message} = error + response = { + date: new Date(), + status: 'error', + message, + response: {}, + } + } + + res.send(response) + }) + export default app From 3d10b58d3372dafaed4a57e78b332c3f57e01ef1 Mon Sep 17 00:00:00 2001 From: Florian Loreau Date: Fri, 23 Feb 2024 15:59:08 +0100 Subject: [PATCH 2/5] Make district postal code route asynchronous --- lib/api/consumers/api-consumer.js | 11 ++- lib/api/district/__mocks__/district-models.js | 7 ++ lib/api/district/models.js | 6 +- lib/api/district/routes.js | 94 +++++-------------- lib/api/district/schema.js | 3 +- lib/api/district/utils.js | 66 ++++++++++++- lib/api/schema.js | 6 ++ 7 files changed, 118 insertions(+), 75 deletions(-) diff --git a/lib/api/consumers/api-consumer.js b/lib/api/consumers/api-consumer.js index 68bde807..58d1f195 100644 --- a/lib/api/consumers/api-consumer.js +++ b/lib/api/consumers/api-consumer.js @@ -5,7 +5,7 @@ import {checkAddressesRequest, checkAddressesIDsRequest} from '../address/utils. import {setCommonToponyms, updateCommonToponyms, patchCommonToponyms, deleteCommonToponyms, getAllDistrictIDsFromCommonToponyms} from '../common-toponym/models.js' import {checkCommonToponymsRequest, checkCommonToponymsIDsRequest} from '../common-toponym/utils.js' import {setDistricts, updateDistricts, patchDistricts, deleteDistricts} from '../district/models.js' -import {checkDistrictsRequest, checkDistrictsIDsRequest} from '../district/utils.js' +import {checkDistrictsRequest, checkDistrictsIDsRequest, formatDataNova, formatDataNovaFromUrl} from '../district/utils.js' import {dataValidationReportFrom, formatObjectWithDefaults, addOrUpdateJob, formatPayloadDates} from '../helper.js' import {addressDefaultOptionalValues} from '../address/schema.js' import {commonToponymDefaultOptionalValues} from '../common-toponym/schema.js' @@ -216,6 +216,9 @@ const districtConsumer = async (jobType, payload, statusID) => { return checkDistrictsRequest(payload, jobType) case 'delete': return checkDistrictsIDsRequest(payload, jobType) + case 'updatePostalCode': + case 'updatePostalCodeFromUrl': + return checkDistrictsRequest(payload, 'patch') default: return dataValidationReportFrom(false, 'Unknown action type', {actionType: jobType, payload}) } @@ -229,6 +232,10 @@ const districtConsumer = async (jobType, payload, statusID) => { return formatPayloadDates(payload, jobType) case 'delete': return payload + case 'updatePostalCodeFromUrl': + return formatDataNovaFromUrl(payload) + case 'updatePostalCode': + return formatDataNova(payload) default: console.warn(`District Consumer Warn: Unknown job type : '${jobType}'`) } @@ -255,6 +262,8 @@ const districtConsumer = async (jobType, payload, statusID) => { } case 'patch': + case 'updatePostalCodeFromUrl': + case 'updatePostalCode': await patchDistricts(formattedPayload) break case 'delete': diff --git a/lib/api/district/__mocks__/district-models.js b/lib/api/district/__mocks__/district-models.js index 003be473..13b4ae3c 100644 --- a/lib/api/district/__mocks__/district-models.js +++ b/lib/api/district/__mocks__/district-models.js @@ -3,3 +3,10 @@ import {bddDistrictMock} from './district-data-mock.js' export async function getDistricts(districtIDs) { return bddDistrictMock.filter(({id}) => districtIDs.includes(id)) } + +export async function getDistrictsFromCog(districtCOGs) { + return bddDistrictMock + .map(district => [district.meta.insee.cog, district]) + .filter(([cog]) => districtCOGs.includes(cog)) + .map(([, district]) => district) +} diff --git a/lib/api/district/models.js b/lib/api/district/models.js index 47dbbd31..3f2c112b 100644 --- a/lib/api/district/models.js +++ b/lib/api/district/models.js @@ -7,8 +7,8 @@ export const getDistricts = districtIDs => District.findAll({where: {id: distric export const getDistrictFromCog = cog => District.findAll({where: {meta: {insee: {cog}}}, raw: true}) -export const getDistrictsFromCog = async cog => { - const districts = await District.findAll({where: {meta: {insee: {cog: {[Op.or]: cog}}}}, raw: true}) +export const getDistrictsFromCog = async cogList => { + const districts = await District.findAll({where: {meta: {insee: {cog: {[Op.or]: cogList}}}}, raw: true}) const districtsByInseeCode = districts.reduce( (acc, district) => { if (!acc[district.meta?.insee?.cog]) { @@ -17,7 +17,7 @@ export const getDistrictsFromCog = async cog => { return acc }, {}) - return cog.map(codeInsee => districtsByInseeCode[codeInsee] || {}) + return cogList.map(codeInsee => districtsByInseeCode[codeInsee] || {}) } export const setDistricts = districts => District.bulkCreate(districts) diff --git a/lib/api/district/routes.js b/lib/api/district/routes.js index 6b94cfd3..757c0425 100644 --- a/lib/api/district/routes.js +++ b/lib/api/district/routes.js @@ -1,12 +1,11 @@ import 'dotenv/config.js' // eslint-disable-line import/no-unassigned-import import {customAlphabet} from 'nanoid' import express from 'express' -import Papa from 'papaparse' import queue from '../../util/queue.cjs' import auth from '../../middleware/auth.js' import analyticsMiddleware from '../../middleware/analytics.js' import fetch from '../../util/fetch.cjs' -import {getDistrict, getDistrictFromCog, getDistrictsFromCog, deleteDistrict, patchDistricts} from './models.js' +import {getDistrict, getDistrictsFromCog, deleteDistrict} from './models.js' import {formatDistrict} from './utils.js' const apiQueue = queue('api') @@ -194,7 +193,7 @@ app.get('/cog/:cog', analyticsMiddleware, async (req, res) => { let response try { const {cog} = req.params - const districts = await getDistrictFromCog(cog) + const districts = await getDistrictsFromCog(cog) if (!districts || districts.length === 0) { res.status(404).send('Request ID unknown') @@ -221,70 +220,6 @@ app.get('/cog/:cog', analyticsMiddleware, async (req, res) => { res.send(response) }) -async function updateDbFromDataNova(postalFile) { - /* eslint-disable camelcase */ - const headers = { - '#Code_commune_INSEE': 'codeInsee', - Nom_de_la_commune: 'nomCommune', - Code_postal: 'codePostal', - Libellé_d_acheminement: 'libelleAcheminement', - 'Libell�_d_acheminement': 'libelleAcheminement', // Postal file url returns wrong header charset code (UTF-8 instead of ISO-8859-1) - Ligne_5: 'ligne5', - } - /* eslint-enable camelcase */ - - const dataRaw = await Papa.parse(postalFile, { - header: true, - transformHeader: name => headers[name] || name, - skipEmptyLines: true, - }) - - const districts = await getDistrictsFromCog( - dataRaw.data.map(({codeInsee}) => codeInsee) - ) - - const districtsByInseeCode = districts.reduce( - (acc, district) => ({ - ...acc, - ...(!district?.meta || acc[district.meta?.insee?.cog] ? {} : {[district.meta.insee.cog]: district}), - }), {}) - - const banDistricts = Object.values( - (dataRaw?.data || []).map(({codeInsee, codePostal, libelleAcheminement}) => { - const {id} = districtsByInseeCode[codeInsee] || {} - return {id, codePostal: [codePostal], libelleAcheminement} - }) - .reduce( - (acc, district) => { - if (district.id) { - if (acc[district.id]) { - acc[district.id].codePostal = [...acc[district.id].codePostal, ...district.codePostal] - } else { - acc[district.id] = district - } - } - - return acc - }, {} - ) - ) - - const patchBulkOperations = banDistricts.map(({id, codePostal, libelleAcheminement}) => ({ - id, - meta: { - laPoste: { - codePostal, - libelleAcheminement, - source: 'La Poste - dataNOVA', - } - } - })) - - patchDistricts(patchBulkOperations) - - return banDistricts -} - app.route('/codePostal') .get(async (req, res) => { let response @@ -293,8 +228,19 @@ app.route('/codePostal') // https://datanova.laposte.fr/data-fair/api/v1/datasets/laposte-hexasmal/raw const {url} = req.query const postalFile = await fetch(url) + const statusID = nanoid() + + await apiQueue.add( + {dataType: 'district', jobType: 'updatePostalCode', data: postalFile, statusID}, + {jobId: statusID, removeOnComplete: true} + ) - response = await updateDbFromDataNova(postalFile) + response = { + date: new Date(), + status: 'success', + message: `Check the status of your request : ${BAN_API_URL}/job-status/${statusID}`, + response: {statusID}, + } } catch (error) { const {message} = error response = { @@ -311,8 +257,18 @@ app.route('/codePostal') let response try { const postalFile = req.body + const statusID = nanoid() - response = await updateDbFromDataNova(postalFile) + await apiQueue.add( + {dataType: 'district', jobType: 'updatePostalCode', data: postalFile, statusID}, + {jobId: statusID, removeOnComplete: true} + ) + response = { + date: new Date(), + status: 'success', + message: `Check the status of your request : ${BAN_API_URL}/job-status/${statusID}`, + response: {statusID}, + } } catch (error) { const {message} = error response = { diff --git a/lib/api/district/schema.js b/lib/api/district/schema.js index 6bea9052..896a9d4e 100644 --- a/lib/api/district/schema.js +++ b/lib/api/district/schema.js @@ -1,5 +1,5 @@ import {object, string, array, date, bool} from 'yup' -import {banID, labelSchema, balSchema} from '../schema.js' +import {banID, labelSchema, balSchema, laPosteSchema} from '../schema.js' const configSchema = object({ useBanId: bool() @@ -15,6 +15,7 @@ const inseeSchema = object({ const metaSchema = object({ insee: inseeSchema, bal: balSchema, + laPoste: laPosteSchema, }).noUnknown() export const banDistrictSchema = object({ diff --git a/lib/api/district/utils.js b/lib/api/district/utils.js index fe588d53..9e495601 100644 --- a/lib/api/district/utils.js +++ b/lib/api/district/utils.js @@ -1,6 +1,7 @@ +import Papa from 'papaparse' import {checkDataFormat, dataValidationReportFrom, checkIdsIsUniq, checkIdsIsVacant, checkIdsIsAvailable, checkDataShema, checkIdsShema} from '../helper.js' import {banID} from '../schema.js' -import {getDistricts} from './models.js' +import {getDistricts, getDistrictsFromCog} from './models.js' import {banDistrictSchema} from './schema.js' const getExistingDistrictIDs = async districtIDs => { @@ -72,3 +73,66 @@ export const formatDistrict = district => { const lastRecordDate = rangeValidity[0].value return {...districtRest, lastRecordDate} } + +export async function formatDataNova(postalFile) { + /* eslint-disable camelcase */ + const headers = { + '#Code_commune_INSEE': 'codeInsee', + Nom_de_la_commune: 'nomCommune', + Code_postal: 'codePostal', + Libellé_d_acheminement: 'libelleAcheminement', + 'Libell�_d_acheminement': 'libelleAcheminement', // Postal file url returns wrong header charset code (UTF-8 instead of ISO-8859-1) + Ligne_5: 'ligne5', + } + /* eslint-enable camelcase */ + + const dataRaw = await Papa.parse(postalFile, { + header: true, + transformHeader: name => headers[name] || name, + skipEmptyLines: true, + }) + + const districts = await getDistrictsFromCog( + dataRaw.data.map(({codeInsee}) => codeInsee) + ) + + const districtsByInseeCode = districts.reduce( + (acc, district) => ({ + ...acc, + ...(district?.meta?.insee?.cog + ? {[district.meta.insee.cog]: [...acc[district.meta.insee.cog], district.id]} + : {} + ), + }), {}) + + const banDistricts = Object.values( + (dataRaw?.data || []).flatMap(({codeInsee, codePostal, libelleAcheminement}) => { + const ids = districtsByInseeCode[codeInsee] || [] + return ids.map(id => ({id, codePostal: [codePostal], libelleAcheminement})) + }) + .reduce( + (acc, district) => { + if (district.id) { + if (acc[district.id]) { + acc[district.id].codePostal = [...acc[district.id].codePostal, ...district.codePostal] + } else { + acc[district.id] = district + } + } + + return acc + }, {} + ) + ) + + return banDistricts.map(({id, codePostal, libelleAcheminement}) => ({ + id, + meta: { + laPoste: { + codePostal, + libelleAcheminement, + source: 'La Poste - dataNOVA', + } + } + })) +} diff --git a/lib/api/schema.js b/lib/api/schema.js index ee9a9b85..34182848 100644 --- a/lib/api/schema.js +++ b/lib/api/schema.js @@ -24,6 +24,12 @@ export const balSchema = object({ isLieuDit: boolean() }).noUnknown() +export const laPosteSchema = object({ + source: string().trim(), + codePostal: array().of(string().trim()), + libelleAcheminement: string().trim(), +}).noUnknown() + export const idfixSchema = object({ hash: string().trim(), }).noUnknown() From 64f09a8374ea56ad1fbd725b86a5bb833dde81f7 Mon Sep 17 00:00:00 2001 From: Florian Loreau Date: Fri, 12 Apr 2024 14:45:37 +0200 Subject: [PATCH 3/5] Fix: rename getDistrict functions --- lib/api/district/__mocks__/district-models.js | 2 +- lib/api/district/models.js | 17 +++-------------- lib/api/district/utils.js | 4 ++-- 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/lib/api/district/__mocks__/district-models.js b/lib/api/district/__mocks__/district-models.js index 13b4ae3c..43f74ca4 100644 --- a/lib/api/district/__mocks__/district-models.js +++ b/lib/api/district/__mocks__/district-models.js @@ -4,7 +4,7 @@ export async function getDistricts(districtIDs) { return bddDistrictMock.filter(({id}) => districtIDs.includes(id)) } -export async function getDistrictsFromCog(districtCOGs) { +export async function getDistrictsFromCogList(districtCOGs) { return bddDistrictMock .map(district => [district.meta.insee.cog, district]) .filter(([cog]) => districtCOGs.includes(cog)) diff --git a/lib/api/district/models.js b/lib/api/district/models.js index 3f2c112b..7167c6fe 100644 --- a/lib/api/district/models.js +++ b/lib/api/district/models.js @@ -5,20 +5,9 @@ export const getDistrict = districtID => District.findByPk(districtID, {raw: tru export const getDistricts = districtIDs => District.findAll({where: {id: districtIDs}, raw: true}) -export const getDistrictFromCog = cog => District.findAll({where: {meta: {insee: {cog}}}, raw: true}) - -export const getDistrictsFromCog = async cogList => { - const districts = await District.findAll({where: {meta: {insee: {cog: {[Op.or]: cogList}}}}, raw: true}) - const districtsByInseeCode = districts.reduce( - (acc, district) => { - if (!acc[district.meta?.insee?.cog]) { - acc[district.meta.insee.cog] = district - } - - return acc - }, {}) - return cogList.map(codeInsee => districtsByInseeCode[codeInsee] || {}) -} +export const getDistrictsFromCog = cog => District.findAll({where: {meta: {insee: {cog}}}, raw: true}) + +export const getDistrictsFromCogList = cogList => District.findAll({where: {meta: {insee: {cog: {[Op.or]: cogList}}}}, raw: true}) export const setDistricts = districts => District.bulkCreate(districts) diff --git a/lib/api/district/utils.js b/lib/api/district/utils.js index 9e495601..c60ea463 100644 --- a/lib/api/district/utils.js +++ b/lib/api/district/utils.js @@ -1,7 +1,7 @@ import Papa from 'papaparse' import {checkDataFormat, dataValidationReportFrom, checkIdsIsUniq, checkIdsIsVacant, checkIdsIsAvailable, checkDataShema, checkIdsShema} from '../helper.js' import {banID} from '../schema.js' -import {getDistricts, getDistrictsFromCog} from './models.js' +import {getDistricts, getDistrictsFromCogList} from './models.js' import {banDistrictSchema} from './schema.js' const getExistingDistrictIDs = async districtIDs => { @@ -92,7 +92,7 @@ export async function formatDataNova(postalFile) { skipEmptyLines: true, }) - const districts = await getDistrictsFromCog( + const districts = await getDistrictsFromCogList( dataRaw.data.map(({codeInsee}) => codeInsee) ) From ba2a17c36ff73eba3ae35131f1796fa4b6a08814 Mon Sep 17 00:00:00 2001 From: Florian Loreau Date: Fri, 23 Feb 2024 16:03:04 +0100 Subject: [PATCH 4/5] Add authentification middleware to district postal code route --- lib/api/district/routes.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/api/district/routes.js b/lib/api/district/routes.js index 757c0425..11158d1a 100644 --- a/lib/api/district/routes.js +++ b/lib/api/district/routes.js @@ -220,7 +220,7 @@ app.get('/cog/:cog', analyticsMiddleware, async (req, res) => { res.send(response) }) -app.route('/codePostal') +app.route('/codePostal', auth) .get(async (req, res) => { let response try { From 2cfb46eca7f7740107da5a9d55370916c2331889 Mon Sep 17 00:00:00 2001 From: Florian Loreau Date: Fri, 12 Apr 2024 14:47:57 +0200 Subject: [PATCH 5/5] Fix : change route name --- lib/api/district/routes.js | 126 ++++++++++++++++++------------------- lib/api/district/utils.js | 9 ++- 2 files changed, 71 insertions(+), 64 deletions(-) diff --git a/lib/api/district/routes.js b/lib/api/district/routes.js index 11158d1a..8dbe9445 100644 --- a/lib/api/district/routes.js +++ b/lib/api/district/routes.js @@ -4,7 +4,6 @@ import express from 'express' import queue from '../../util/queue.cjs' import auth from '../../middleware/auth.js' import analyticsMiddleware from '../../middleware/analytics.js' -import fetch from '../../util/fetch.cjs' import {getDistrict, getDistrictsFromCog, deleteDistrict} from './models.js' import {formatDistrict} from './utils.js' @@ -100,6 +99,69 @@ app.route('/') res.send(response) }) +app.route('/postal-codes-from-datanova-url', auth) + .put(async (req, res) => { + let response + try { + // On February 2024 the postal file url is : + // https://datanova.laposte.fr/data-fair/api/v1/datasets/laposte-hexasmal/raw + const {url} = req.query + const statusID = nanoid() + + await apiQueue.add( + {dataType: 'district', jobType: 'updatePostalCodeFromUrl', data: url, statusID}, + {jobId: statusID, removeOnComplete: true} + ) + + response = { + date: new Date(), + status: 'success', + message: `Check the status of your request : ${BAN_API_URL}/job-status/${statusID}`, + response: {statusID}, + } + } catch (error) { + const {message} = error + response = { + date: new Date(), + status: 'error', + message, + response: {}, + } + } + + res.send(response) + }) + +app.route('/postal-codes-from-datanova', auth) + .put(express.text(), async (req, res) => { + let response + try { + const postalFile = req.body + const statusID = nanoid() + + await apiQueue.add( + {dataType: 'district', jobType: 'updatePostalCode', data: postalFile, statusID}, + {jobId: statusID, removeOnComplete: true} + ) + response = { + date: new Date(), + status: 'success', + message: `Check the status of your request : ${BAN_API_URL}/job-status/${statusID}`, + response: {statusID}, + } + } catch (error) { + const {message} = error + response = { + date: new Date(), + status: 'error', + message, + response: {}, + } + } + + res.send(response) + }) + app.route('/:districtID') .get(analyticsMiddleware, async (req, res) => { let response @@ -220,66 +282,4 @@ app.get('/cog/:cog', analyticsMiddleware, async (req, res) => { res.send(response) }) -app.route('/codePostal', auth) - .get(async (req, res) => { - let response - try { - // On February 2024 the postal file url is : - // https://datanova.laposte.fr/data-fair/api/v1/datasets/laposte-hexasmal/raw - const {url} = req.query - const postalFile = await fetch(url) - const statusID = nanoid() - - await apiQueue.add( - {dataType: 'district', jobType: 'updatePostalCode', data: postalFile, statusID}, - {jobId: statusID, removeOnComplete: true} - ) - - response = { - date: new Date(), - status: 'success', - message: `Check the status of your request : ${BAN_API_URL}/job-status/${statusID}`, - response: {statusID}, - } - } catch (error) { - const {message} = error - response = { - date: new Date(), - status: 'error', - message, - response: {}, - } - } - - res.send(response) - }) - .post(express.text(), async (req, res) => { - let response - try { - const postalFile = req.body - const statusID = nanoid() - - await apiQueue.add( - {dataType: 'district', jobType: 'updatePostalCode', data: postalFile, statusID}, - {jobId: statusID, removeOnComplete: true} - ) - response = { - date: new Date(), - status: 'success', - message: `Check the status of your request : ${BAN_API_URL}/job-status/${statusID}`, - response: {statusID}, - } - } catch (error) { - const {message} = error - response = { - date: new Date(), - status: 'error', - message, - response: {}, - } - } - - res.send(response) - }) - export default app diff --git a/lib/api/district/utils.js b/lib/api/district/utils.js index c60ea463..94966f2d 100644 --- a/lib/api/district/utils.js +++ b/lib/api/district/utils.js @@ -1,4 +1,5 @@ import Papa from 'papaparse' +import fetch from '../../util/fetch.cjs' import {checkDataFormat, dataValidationReportFrom, checkIdsIsUniq, checkIdsIsVacant, checkIdsIsAvailable, checkDataShema, checkIdsShema} from '../helper.js' import {banID} from '../schema.js' import {getDistricts, getDistrictsFromCogList} from './models.js' @@ -100,7 +101,7 @@ export async function formatDataNova(postalFile) { (acc, district) => ({ ...acc, ...(district?.meta?.insee?.cog - ? {[district.meta.insee.cog]: [...acc[district.meta.insee.cog], district.id]} + ? {[district.meta.insee.cog]: [...(acc[district.meta.insee.cog] || []), district.id]} : {} ), }), {}) @@ -136,3 +137,9 @@ export async function formatDataNova(postalFile) { } })) } + +export async function formatDataNovaFromUrl(url) { + const postalFileResponse = await fetch(url) + const postalFile = await postalFileResponse.text() + return formatDataNova(postalFile) +}