Added report mecanism

lib/api/consumers/build-reports.js

@@ -0,0 +1,22 @@
+import mongo from '../../util/mongo.cjs'
+import {formatAndUpdateReports} from '../report/utils.js'
+
+const buildReportsConsumer = async () => {
+  try {
+    console.log('Building reports...')
+    const filters = {
+      status: null,
+      preProcessingStatusKey: 0,
+      'meta.targetedPlateform': 'ban',
+      preProcessingResponse: {$ne: {}}
+    }
+
+    const reportsNotCompletelyBuilt = await mongo.db.collection('processing-reports').find({...filters}).toArray()
+    await formatAndUpdateReports(reportsNotCompletelyBuilt)
+    console.log('Reports built successfully')
+  } catch (error) {
+    console.error(error)
+  }
+}
+
+export default buildReportsConsumer

lib/api/report/routes.js

@@ -1,26 +1,51 @@
 import 'dotenv/config.js' // eslint-disable-line import/no-unassigned-import
 import express from 'express'
+import {getDistrict} from '../district/models.js'
 import mongo from '../../util/mongo.cjs'
 import auth from '../../middleware/auth.js'
 import analyticsMiddleware from '../../middleware/analytics.js'
+import {formatAndUpdateReports} from './utils.js'
 
 const app = new express.Router()
 
-app.route('/idfix-processing')
+app.route('/')
+  .get(auth, analyticsMiddleware, async (req, res) => {
+    const {districtID, cog, statusKey, preProcessingStatusKey} = req.query
+    const filters = {
+      ...(districtID && {districtID}),
+      ...(cog && {'meta.cog': cog}),
+      ...(statusKey && {statusKey: Number.parseInt(statusKey, 10)}),
+      ...(preProcessingStatusKey && {preProcessingStatusKey: Number.parseInt(preProcessingStatusKey, 10)}),
+    }
+
+    const reports = await mongo.db.collection('processing-reports').find({...filters}).toArray()
+    const formattedReports = await formatAndUpdateReports(reports)
+    res.send(formattedReports)
+  })
+
+app.route('/district/:districtID')
   .post(auth, analyticsMiddleware, async (req, res) => {
     let response
     try {
+      const {districtID} = req.params
+      const district = await getDistrict(districtID)
+
+      if (!district) {
+        res.status(404).send('Request ID unknown')
+        return
+      }
+
       const report = req.body
-      const {cog} = report
-      if (!cog) {
-        throw new Error('Missing cog')
+      const dataToInsert = {
+        districtID,
+        ...report,
       }
 
-      await mongo.db.collection('report').updateOne({cog}, {$set: report}, {upsert: true})
+      await mongo.db.collection('processing-reports').insertOne(dataToInsert, {upsert: true})
       response = {
         date: new Date(),
         status: 'success',
-        message: 'Id-fix processing report created successfully',
+        message: 'Processing report created successfully',
         response: {},
       }
     } catch (error) {

lib/api/report/utils.js

@@ -0,0 +1,119 @@
+import mongo from '../../util/mongo.cjs'
+import {getJobStatus} from '../job-status/models.js'
+
+const categories = ['addresses', 'commonToponyms', 'districts']
+const operations = ['add', 'update', 'delete']
+
+export const formatAndUpdateReports = async reports => {
+  const formattedReportsPromises = reports.map(async report => formatAndUpdateReport(report))
+  return Promise.all(formattedReportsPromises)
+}
+
+const formatAndUpdateReport = async report => {
+  const {_id, ...reportRest} = report
+  const {status, preProcessingStatusKey, preProcessingResponse, meta: {targetedPlateform}} = reportRest
+  // If the report does not have a final status yet and the pre-processing status is in success, we need to reconstruct.
+  // the flag 'targetedPlateform' is used to determine if the pre-processing data has been sent to the ban APIs or the legacy compose API
+  // Our ban APIs are asynchronous so the preprocessing report need to be reconstructed to get the final status
+  if (!status && preProcessingStatusKey === 0
+      && targetedPlateform === 'ban'
+      && Object.keys(preProcessingResponse).length > 0
+  ) {
+    try {
+      let errorCount = 0
+      let mostRecentDate = new Date(0)
+      const formattedResponse = {}
+
+      for (const category of categories) {
+        for (const operation of operations) {
+          if (!preProcessingResponse[category] || !preProcessingResponse[category][operation]) { // eslint-disable-line max-depth
+            continue
+          }
+
+          formattedResponse[category] = {}
+          const jobStatusArr = await Promise.all( // eslint-disable-line no-await-in-loop
+            preProcessingResponse[category][operation].map(async report => {
+              const statusID = report?.response?.statusID
+              if (!statusID) {
+                throw new Error('Missing statusID in pre-processing report response')
+              }
+
+              const jobStatus = await getJobStatus(statusID)
+              if (!jobStatus) {
+                throw new Error(`Job status ${report.response.statusID} not found : either pending or expired`)
+              }
+
+              return jobStatus
+            })
+          )
+
+          formattedResponse[category][operation] = jobStatusArr.reduce((acc, jobStatus) => {
+            const {status, count, report, updatedAt} = jobStatus
+
+            const date = new Date(updatedAt)
+            if (date > mostRecentDate) {
+              mostRecentDate = date
+            }
+
+            if (status === 'error') {
+              errorCount++
+              acc.error = {
+                count: acc.error ? acc.error.count + count : count,
+                report: [...acc.error.report, report],
+              }
+            }
+
+            if (status === 'success') {
+              acc.success = {
+                count: acc.success ? acc.success.count + count : count,
+              }
+            }
+
+            return acc
+          }, {
+            success: {
+              count: 0,
+            },
+            error: {
+              count: 0,
+              report: []
+            },
+          })
+        }
+      }
+
+      const processingReport = {
+        statusKey: errorCount ? 1 : 0,
+        status: errorCount ? 'error' : 'success',
+        message: errorCount ? 'Processed with errors' : 'Processed successfully',
+        response: formattedResponse,
+        date: mostRecentDate
+      }
+
+      // Update the report with the processing report
+      await mongo.db.collection('processing-reports').updateOne({_id}, {$set: {...processingReport}})
+
+      return {
+        ...reportRest,
+        ...processingReport
+      }
+    } catch (error) {
+      const processingReport = {
+        statusKey: 1,
+        status: 'error',
+        message: `Could not process the report : ${error.message}`,
+        response: {},
+        date: new Date()
+      }
+
+      const {_id, ...reportRest} = report
+
+      return {
+        ...reportRest,
+        ...processingReport
+      }
+    }
+  }
+
+  return report
+}

worker.js

@@ -6,6 +6,7 @@ import ms from 'ms'
 import apiConsumer from './lib/api/consumers/api-consumer.js'
 import exportToExploitationDBConsumer from './lib/api/consumers/export-to-exploitation-db-consumer.js'
 import cleanJobStatusConsumer from './lib/api/consumers/clean-job-status-consumer.js'
+import buildReportsConsumer from './lib/api/consumers/build-reports.js'
 
 import mongo from './lib/util/mongo.cjs'
 import queue from './lib/util/queue.cjs'
@@ -36,6 +37,8 @@ async function main() {
   queue('export-to-exploitation-db').process(1, exportToExploitationDBConsumer)
   queue('clean-job-status').process(1, cleanJobStatusConsumer)
   queue('clean-job-status').add({}, {jobId: 'cleanJobStatusJobId', repeat: {every: ms('1d')}, removeOnComplete: true})
+  queue('build-reports').process(1, buildReportsConsumer)
+  queue('build-reports').add({}, {jobId: 'buildReportsJobId', repeat: {every: ms('10s')}, removeOnComplete: true})
 }
 
 main().catch(error => {