fix: flux import: optimisation reprise sur erreur + notifications sla…

…ck (#1689)

* fix: flux import: optimisation reprise sur erreur + notifications slack

* fix: tests

* fix: tests

server/src/jobs/offrePartenaire/__snapshots__/importRHAlternance.test.ts.snap

@@ -14,6 +14,7 @@ exports[`import RH Alternance > rawRhAlternanceToComputedMapper > should test th
   ],
   "created_at": null,
   "errors": [],
+  "jobs_in_success": [],
   "offer_access_conditions": [],
   "offer_creation": 2024-11-14T11:11:08.000Z,
   "offer_description": "Détail du poste : QUE FEREZ-VOUS ?<br><br>Rattaché(e) à Arnaud, Responsable HSE Outremer, vous contribuerez au déploiement d'une démarche HSE amenant les équipes vers une évolution de la maturité HSE de l'entreprise.<br><br>Vos missions principales seront :<br><br>* Participer au déploiement de la feuille de route HSE 2024-25,<br><br>* Participer à l'animation de la culture HSE dans l'établissement (1/4 sécurité/ accueil sécurité / sensibilisation quotidienne, communication «Safety First »...),<br><br>* La mise à jour de l'évaluation des risques professionnels et risques d'exposition,<br><br>* La mise en place des indicateurs et tableaux de bord,<br><br>* Veille documentaires (rédaction de documents / consignes/ mise à jour des FDS / process),<br><br>* Participer à l'analyse environnementale du site et proposition d'un plan d'action associé,<br><br>* La recherche de réduction de nos impacts, comme la réduction et/ou la valorisation de nos déchets (emballages plastiques, cartons...).<br><br>NOUS SOMMES FIERS DE POUVOIR PROPOSER DEPUIS DES ANNÉES DES PARCOURS D'ÉVOLUTION À NOS SALARIÉS.

server/src/jobs/offrePartenaire/fillComputedJobsPartners.ts

@@ -57,4 +57,5 @@ export const blankComputedJobPartner: Omit<IComputedJobsPartners, "_id" | "partn
   workplace_siret: null,
   workplace_size: null,
   workplace_website: null,
+  jobs_in_success: [],
 }

server/src/jobs/offrePartenaire/fillFieldsForPartnersFactory.ts

@@ -7,6 +7,7 @@ import { COMPUTED_ERROR_SOURCE, IComputedJobsPartners } from "shared/models/jobs
 import { logger as globalLogger } from "@/common/logger"
 import { getDbCollection } from "@/common/utils/mongodbUtils"
 import { sentryCaptureException } from "@/common/utils/sentryUtils"
+import { notifyToSlack } from "@/common/utils/slackUtils"
 import { streamGroupByCount } from "@/common/utils/streamUtils"
 
 /**
@@ -47,6 +48,7 @@ export const fillFieldsForPartnersFactory = async <SourceFields extends keyof IJ
       },
       {
         business_error: null,
+        jobs_in_success: { $nin: [job] },
       },
     ],
   }
@@ -70,20 +72,32 @@ export const fillFieldsForPartnersFactory = async <SourceFields extends keyof IJ
         try {
           const responses = await getData(documents)
 
-          const dataToWrite = responses.map((document) => {
+          const dataToWrite = responses.flatMap((document) => {
             const { _id, ...newFields } = document
-            return {
-              updateOne: {
-                filter: { _id },
-                update: { $set: newFields },
+            return [
+              {
+                updateOne: {
+                  filter: { _id },
+                  update: { $set: newFields },
+                },
+              },
+              {
+                updateOne: {
+                  filter: { _id },
+                  update: {
+                    $push: {
+                      jobs_in_success: job,
+                    },
+                  },
+                },
               },
-            }
+            ]
           })
           if (dataToWrite.length) {
             await getDbCollection("computed_jobs_partners").bulkWrite(dataToWrite, {
               ordered: false,
             })
-            counters.success += dataToWrite.length
+            counters.success += responses.length
           }
           const documentsWithMissingData = documents.filter((_document, index) => !responses[index])
           if (documentsWithMissingData.length) {
@@ -142,5 +156,11 @@ export const fillFieldsForPartnersFactory = async <SourceFields extends keyof IJ
       { parallel: 1 }
     )
   )
-  logger.info(`job ${job} : enrichissement terminé`, counters)
+  const message = `job ${job} : enrichissement terminé. total=${counters.total}, success=${counters.success}, errors=${counters.error}`
+  logger.info(message)
+  await notifyToSlack({
+    subject: `computedJobPartners: enrichissement de données`,
+    message,
+    error: counters.error > 0,
+  })
 }

server/src/jobs/offrePartenaire/helloWorkMapper.test.ts

@@ -109,6 +109,7 @@ describe("helloWorkJobToJobsPartners", () => {
       errors: [],
       validated: false,
       business_error: null,
+      jobs_in_success: [],
     })
   })
 })

server/src/jobs/offrePartenaire/importRHAlternance.ts

@@ -10,6 +10,7 @@ import { z } from "zod"
 
 import { logger } from "@/common/logger"
 import { getDbCollection } from "@/common/utils/mongodbUtils"
+import { notifyToSlack } from "@/common/utils/slackUtils"
 import config from "@/config"
 import dayjs from "@/services/dayjs.service"
 
@@ -64,7 +65,12 @@ export const importRHAlternanceRaw = async () => {
     }
     jobCount += savedJobs.length
   }
-  logger.info(`import done: ${jobCount} jobs imported`)
+  const message = `import RH Alternance terminé : ${jobCount} offres importées`
+  logger.info(message)
+  await notifyToSlack({
+    subject: `import des offres RH Alternance dans raw`,
+    message,
+  })
 }
 
 export const importRHAlternanceToComputed = async () => {

server/src/jobs/offrePartenaire/rawToComputedJobsPartners.ts

@@ -9,6 +9,7 @@ import { AnyZodObject } from "zod"
 import { logger } from "@/common/logger"
 import { getDbCollection } from "@/common/utils/mongodbUtils"
 import { sentryCaptureException } from "@/common/utils/sentryUtils"
+import { notifyToSlack } from "@/common/utils/slackUtils"
 
 export const rawToComputedJobsPartners = async <ZodInput extends AnyZodObject>({
   collectionSource,
@@ -56,5 +57,11 @@ export const rawToComputedJobsPartners = async <ZodInput extends AnyZodObject>({
       { parallel: 10 }
     )
   )
-  logger.info(`import dans computed_jobs_partners pour partner_label=${partnerLabel} terminé`, counters)
+  const message = `import dans computed_jobs_partners pour partner_label=${partnerLabel} terminé. total=${counters.total}, success=${counters.success}, errors=${counters.error}`
+  logger.info(message)
+  await notifyToSlack({
+    subject: `mapping Raw => computed_jobs_partners`,
+    message,
+    error: counters.error > 0,
+  })
 }

server/tests/utils/jobsPartners.test.utils.ts

@@ -1,4 +1,4 @@
-import { generateJobsPartnersOfferPrivate } from "shared/fixtures/jobPartners.fixture"
+import { generateComputedJobsPartnersFixture, generateJobsPartnersOfferPrivate } from "shared/fixtures/jobPartners.fixture"
 import { IJobsPartnersOfferPrivate, ZJobsPartnersOfferPrivate } from "shared/models/jobsPartners.model"
 import { IComputedJobsPartners } from "shared/models/jobsPartnersComputed.model"
 
@@ -8,10 +8,7 @@ import { saveDbEntity } from "./user.test.utils"
 
 export async function createComputedJobPartner(data: Partial<IComputedJobsPartners> = {}) {
   const computedJobPartner = {
-    errors: [],
-    validated: true,
-    business_error: null,
-    ...generateJobsPartnersOfferPrivate(),
+    ...generateComputedJobsPartnersFixture(),
     ...data,
   }
   await getDbCollection("computed_jobs_partners").insertOne(computedJobPartner)

shared/fixtures/jobPartners.fixture.ts

@@ -65,6 +65,8 @@ export function generateComputedJobsPartnersFixture(data: Partial<IComputedJobsP
     errors: [],
     validated: false,
     business_error: null,
+    jobs_in_success: [],
+
     _id: new ObjectId(),
     workplace_siret: null,
     workplace_website: null,
@@ -114,7 +116,6 @@ export function generateComputedJobsPartnersFixture(data: Partial<IComputedJobsP
 
     created_at: new Date("2021-01-28T15:00:00.000Z"),
     updated_at: new Date("2021-01-28T15:00:00.000Z"),
-
     ...data,
   }
 }

shared/models/jobsPartnersComputed.model.ts

@@ -35,6 +35,7 @@ export const ZComputedJobsPartners = extensions
   })
   .extend({
     _id: zObjectId,
+    jobs_in_success: z.array(extensions.buildEnum(COMPUTED_ERROR_SOURCE)),
     errors: z.array(
       z
         .object({
@@ -59,6 +60,7 @@ export default {
     [{ errors: 1 }, {}],
     [{ partner_label: 1, partner_job_id: 1 }, { unique: true }],
     [{ workplace_siret: 1 }, {}],
+    [{ jobs_in_success: 1 }, {}],
     [{ "duplicates.otherOfferId": 1 }, {}],
   ],
   collectionName,