fix: mise a jour des computed organisme des effectifs (#3901)

mission-apprentissage · Nov 14, 2024 · e296b08 · e296b08
1 parent 6691203
commit e296b08
Show file tree

Hide file tree

Showing 6 changed files with 82 additions and 3 deletions.
diff --git a/server/src/commands.ts b/server/src/commands.ts
@@ -718,6 +718,12 @@ program
   .option("-q, --queued", "Run job asynchronously", false)
   .action(createJobAction("tmp:patches:update-effectifs-source"));
 
+program
+  .command("tmp:patches:update_effectifs_computed_organisme")
+  .description("Mise a jour des computed organisme des effectifs")
+  .option("-q, --queued", "Run job asynchronously", false)
+  .action(createJobAction("tmp:patches:update_effectifs_computed_organisme"));
+
 program
   .command("dev:list-http-endpoints")
   .description("Liste les routes du serveur HTTP")

diff --git a/server/src/db/migrations/20241114091932-tmp:patches:update_effectifs_computed_organisme.ts b/server/src/db/migrations/20241114091932-tmp:patches:update_effectifs_computed_organisme.ts
@@ -0,0 +1,5 @@
+import { addJob } from "job-processor";
+
+export const up = async () => {
+  await addJob({ name: "tmp:patches:update_effectifs_computed_organisme", queued: true });
+};
diff --git a/server/src/jobs/hydrate/effectifs/hydrate-effectifs-computed-organisme.ts b/server/src/jobs/hydrate/effectifs/hydrate-effectifs-computed-organisme.ts
@@ -0,0 +1,45 @@
+import { ObjectId } from "mongodb";
+import { IOrganisme } from "shared/models";
+
+import { generateOrganismeComputed } from "@/common/actions/organismes/organismes.actions";
+import logger from "@/common/logger";
+import { organismesDb, effectifsDb } from "@/common/model/collections";
+
+export async function hydrateEffectifsComputedOrganisme() {
+  const batchSize = 100;
+  let processedCount = 0;
+
+  const cursor = organismesDb().find({});
+  while (await cursor.hasNext()) {
+    const batch: IOrganisme[] = [];
+    for (let i = 0; i < batchSize; i++) {
+      if (await cursor.hasNext()) {
+        const organisme = await cursor.next();
+        if (organisme) {
+          batch.push(organisme);
+        }
+      } else {
+        break;
+      }
+    }
+
+    await Promise.all(
+      batch.map(async (organisme) => {
+        const computedResult = generateOrganismeComputed(organisme);
+        await effectifsDb().updateMany(
+          { organisme_id: new ObjectId(organisme._id) },
+          {
+            $set: {
+              "_computed.organisme": computedResult,
+            },
+          }
+        );
+      })
+    );
+
+    processedCount += batch.length;
+    logger.info(`${processedCount} organismes traités jusqu'à présent.`);
+  }
+
+  logger.info(`Traitement de ${processedCount} organismes terminé.`);
+}
diff --git a/server/src/jobs/ingestion/process-ingestion.ts b/server/src/jobs/ingestion/process-ingestion.ts
@@ -290,16 +290,15 @@ async function transformEffectifQueueV3ToEffectif(rawEffectifQueued: IEffectifQu
             effectifQueued?.etablissement_lieu_de_formation_uai,
             effectifQueued?.etablissement_lieu_de_formation_siret
           );
-          organismeTarget = organisme;
           Object.assign(itemProcessingInfos, addPrefixToProperties("organisme_lieu_", stats));
           return organisme;
         })(),
         (async () => {
           const { organisme, stats } = await findOrganismeWithStats(
             effectifQueued?.etablissement_formateur_uai,
-            effectifQueued?.etablissement_formateur_siret,
-            { _id: 1 }
+            effectifQueued?.etablissement_formateur_siret
           );
+          organismeTarget = organisme;
           Object.assign(itemProcessingInfos, addPrefixToProperties("organisme_formateur_", stats));
           return organisme;
         })(),

diff --git a/server/src/jobs/jobs.ts b/server/src/jobs/jobs.ts
@@ -31,6 +31,7 @@ import {
   hydrateEffectifsComputedOpcos,
   hydrateEffectifsComputedReseaux,
 } from "./hydrate/effectifs/hydrate-effectifs-computed";
+import { hydrateEffectifsComputedOrganisme } from "./hydrate/effectifs/hydrate-effectifs-computed-organisme";
 import { hydrateEffectifsComputedTypes } from "./hydrate/effectifs/hydrate-effectifs-computed-types";
 import { hydrateEffectifsFormationsNiveaux } from "./hydrate/effectifs/hydrate-effectifs-formations-niveaux";
 import {
@@ -514,6 +515,11 @@ export async function setupJobProcessor() {
           return initOpcos(name, rncpList);
         },
       },
+      "tmp:patches:update_effectifs_computed_organisme": {
+        handler: async () => {
+          return hydrateEffectifsComputedOrganisme();
+        },
+      },
       "process:effectifs-queue:remove-duplicates": {
         handler: async () => {
           return removeDuplicatesEffectifsQueue();

diff --git a/server/tests/integration/jobs/ingestion/process-ingestion.test.ts b/server/tests/integration/jobs/ingestion/process-ingestion.test.ts
@@ -619,7 +619,13 @@ describe("Processus d'ingestion", () => {
           validation_errors: [],
           _computed: {
             organisme: {
+              academie: "10",
+              departement: "01",
               fiable: false,
+              region: "84",
+              reseaux: [],
+              siret: "77937827200016",
+              uai: "0802004U",
             },
             formation: {
               opcos: [],
@@ -715,7 +721,13 @@ describe("Processus d'ingestion", () => {
           validation_errors: [],
           _computed: {
             organisme: {
+              academie: "10",
+              departement: "01",
               fiable: false,
+              region: "84",
+              reseaux: [],
+              siret: "77937827200016",
+              uai: "0802004U",
             },
             formation: {
               opcos: [],
@@ -889,7 +901,13 @@ describe("Processus d'ingestion", () => {
           validation_errors: [],
           _computed: {
             organisme: {
+              academie: "10",
+              departement: "01",
               fiable: false,
+              region: "84",
+              reseaux: [],
+              siret: "77937827200016",
+              uai: "0802004U",
             },
             formation: {
               opcos: [],