Create api route to update district postal code with datanova

lib/api/consumers/api-consumer.js

@@ -5,7 +5,7 @@ import {checkAddressesRequest, checkAddressesIDsRequest} from '../address/utils.
 import {setCommonToponyms, updateCommonToponyms, patchCommonToponyms, deleteCommonToponyms, getAllDistrictIDsFromCommonToponyms} from '../common-toponym/models.js'
 import {checkCommonToponymsRequest, checkCommonToponymsIDsRequest} from '../common-toponym/utils.js'
 import {setDistricts, updateDistricts, patchDistricts, deleteDistricts} from '../district/models.js'
-import {checkDistrictsRequest, checkDistrictsIDsRequest} from '../district/utils.js'
+import {checkDistrictsRequest, checkDistrictsIDsRequest, formatDataNova, formatDataNovaFromUrl} from '../district/utils.js'
 import {dataValidationReportFrom, formatObjectWithDefaults, addOrUpdateJob, formatPayloadDates} from '../helper.js'
 import {addressDefaultOptionalValues} from '../address/schema.js'
 import {commonToponymDefaultOptionalValues} from '../common-toponym/schema.js'
@@ -216,6 +216,9 @@ const districtConsumer = async (jobType, payload, statusID) => {
         return checkDistrictsRequest(payload, jobType)
       case 'delete':
         return checkDistrictsIDsRequest(payload, jobType)
+      case 'updatePostalCode':
+      case 'updatePostalCodeFromUrl':
+        return checkDistrictsRequest(payload, 'patch')
       default:
         return dataValidationReportFrom(false, 'Unknown action type', {actionType: jobType, payload})
     }
@@ -229,6 +232,10 @@ const districtConsumer = async (jobType, payload, statusID) => {
         return formatPayloadDates(payload, jobType)
       case 'delete':
         return payload
+      case 'updatePostalCodeFromUrl':
+        return formatDataNovaFromUrl(payload)
+      case 'updatePostalCode':
+        return formatDataNova(payload)
       default:
         console.warn(`District Consumer Warn: Unknown job type : '${jobType}'`)
     }
@@ -255,6 +262,8 @@ const districtConsumer = async (jobType, payload, statusID) => {
       }
 
       case 'patch':
+      case 'updatePostalCodeFromUrl':
+      case 'updatePostalCode':
         await patchDistricts(formattedPayload)
         break
       case 'delete':

lib/api/district/__mocks__/district-models.js

@@ -3,3 +3,10 @@ import {bddDistrictMock} from './district-data-mock.js'
 export async function getDistricts(districtIDs) {
   return bddDistrictMock.filter(({id}) => districtIDs.includes(id))
 }
+
+export async function getDistrictsFromCogList(districtCOGs) {
+  return bddDistrictMock
+    .map(district => [district.meta.insee.cog, district])
+    .filter(([cog]) => districtCOGs.includes(cog))
+    .map(([, district]) => district)
+}

lib/api/district/models.js

@@ -1,3 +1,4 @@
+import {Op} from 'sequelize'
 import {District} from '../../util/sequelize.js'
 
 export const getDistrict = districtID => District.findByPk(districtID, {raw: true})
@@ -6,6 +7,8 @@ export const getDistricts = districtIDs => District.findAll({where: {id: distric
 
 export const getDistrictsFromCog = cog => District.findAll({where: {meta: {insee: {cog}}}, raw: true})
 
+export const getDistrictsFromCogList = cogList => District.findAll({where: {meta: {insee: {cog: {[Op.or]: cogList}}}}, raw: true})
+
 export const setDistricts = districts => District.bulkCreate(districts)
 
 export const updateDistricts = districts => {

lib/api/district/routes.js

@@ -99,6 +99,69 @@ app.route('/')
     res.send(response)
   })
 
+app.route('/postal-codes-from-datanova-url', auth)
+  .put(async (req, res) => {
+    let response
+    try {
+      // On February 2024 the postal file url is :
+      // https://datanova.laposte.fr/data-fair/api/v1/datasets/laposte-hexasmal/raw
+      const {url} = req.query
+      const statusID = nanoid()
+
+      await apiQueue.add(
+        {dataType: 'district', jobType: 'updatePostalCodeFromUrl', data: url, statusID},
+        {jobId: statusID, removeOnComplete: true}
+      )
+
+      response = {
+        date: new Date(),
+        status: 'success',
+        message: `Check the status of your request : ${BAN_API_URL}/job-status/${statusID}`,
+        response: {statusID},
+      }
+    } catch (error) {
+      const {message} = error
+      response = {
+        date: new Date(),
+        status: 'error',
+        message,
+        response: {},
+      }
+    }
+
+    res.send(response)
+  })
+
+app.route('/postal-codes-from-datanova', auth)
+  .put(express.text(), async (req, res) => {
+    let response
+    try {
+      const postalFile = req.body
+      const statusID = nanoid()
+
+      await apiQueue.add(
+        {dataType: 'district', jobType: 'updatePostalCode', data: postalFile, statusID},
+        {jobId: statusID, removeOnComplete: true}
+      )
+      response = {
+        date: new Date(),
+        status: 'success',
+        message: `Check the status of your request : ${BAN_API_URL}/job-status/${statusID}`,
+        response: {statusID},
+      }
+    } catch (error) {
+      const {message} = error
+      response = {
+        date: new Date(),
+        status: 'error',
+        message,
+        response: {},
+      }
+    }
+
+    res.send(response)
+  })
+
 app.route('/:districtID')
   .get(analyticsMiddleware, async (req, res) => {
     let response

lib/api/district/schema.js

@@ -1,5 +1,5 @@
 import {object, string, array, date, bool} from 'yup'
-import {banID, labelSchema, balSchema} from '../schema.js'
+import {banID, labelSchema, balSchema, laPosteSchema} from '../schema.js'
 
 const configSchema = object({
   useBanId: bool()
@@ -15,6 +15,7 @@ const inseeSchema = object({
 const metaSchema = object({
   insee: inseeSchema,
   bal: balSchema,
+  laPoste: laPosteSchema,
 }).noUnknown()
 
 export const banDistrictSchema = object({

lib/api/district/utils.js

@@ -1,6 +1,8 @@
+import Papa from 'papaparse'
+import fetch from '../../util/fetch.cjs'
 import {checkDataFormat, dataValidationReportFrom, checkIdsIsUniq, checkIdsIsVacant, checkIdsIsAvailable, checkDataShema, checkIdsShema} from '../helper.js'
 import {banID} from '../schema.js'
-import {getDistricts} from './models.js'
+import {getDistricts, getDistrictsFromCogList} from './models.js'
 import {banDistrictSchema} from './schema.js'
 
 const getExistingDistrictIDs = async districtIDs => {
@@ -72,3 +74,72 @@ export const formatDistrict = district => {
   const lastRecordDate = rangeValidity[0].value
   return {...districtRest, lastRecordDate}
 }
+
+export async function formatDataNova(postalFile) {
+  /* eslint-disable camelcase */
+  const headers = {
+    '#Code_commune_INSEE': 'codeInsee',
+    Nom_de_la_commune: 'nomCommune',
+    Code_postal: 'codePostal',
+    Libellé_d_acheminement: 'libelleAcheminement',
+    'Libell�_d_acheminement': 'libelleAcheminement', // Postal file url returns wrong header charset code (UTF-8 instead of ISO-8859-1)
+    Ligne_5: 'ligne5',
+  }
+  /* eslint-enable camelcase */
+
+  const dataRaw = await Papa.parse(postalFile, {
+    header: true,
+    transformHeader: name => headers[name] || name,
+    skipEmptyLines: true,
+  })
+
+  const districts = await getDistrictsFromCogList(
+    dataRaw.data.map(({codeInsee}) => codeInsee)
+  )
+
+  const districtsByInseeCode = districts.reduce(
+    (acc, district) => ({
+      ...acc,
+      ...(district?.meta?.insee?.cog
+        ? {[district.meta.insee.cog]: [...(acc[district.meta.insee.cog] || []), district.id]}
+        : {}
+      ),
+    }), {})
+
+  const banDistricts = Object.values(
+    (dataRaw?.data || []).flatMap(({codeInsee, codePostal, libelleAcheminement}) => {
+      const ids = districtsByInseeCode[codeInsee] || []
+      return ids.map(id => ({id, codePostal: [codePostal], libelleAcheminement}))
+    })
+      .reduce(
+        (acc, district) => {
+          if (district.id) {
+            if (acc[district.id]) {
+              acc[district.id].codePostal = [...acc[district.id].codePostal, ...district.codePostal]
+            } else {
+              acc[district.id] = district
+            }
+          }
+
+          return acc
+        }, {}
+      )
+  )
+
+  return banDistricts.map(({id, codePostal, libelleAcheminement}) => ({
+    id,
+    meta: {
+      laPoste: {
+        codePostal,
+        libelleAcheminement,
+        source: 'La Poste - dataNOVA',
+      }
+    }
+  }))
+}
+
+export async function formatDataNovaFromUrl(url) {
+  const postalFileResponse = await fetch(url)
+  const postalFile = await postalFileResponse.text()
+  return formatDataNova(postalFile)
+}

lib/api/schema.js

@@ -24,6 +24,12 @@ export const balSchema = object({
   isLieuDit: boolean()
 }).noUnknown()
 
+export const laPosteSchema = object({
+  source: string().trim(),
+  codePostal: array().of(string().trim()),
+  libelleAcheminement: string().trim(),
+}).noUnknown()
+
 export const idfixSchema = object({
   hash: string().trim(),
 }).noUnknown()