diff --git a/server/src/commands.ts b/server/src/commands.ts index 14bfd2209..e8af6e95d 100644 --- a/server/src/commands.ts +++ b/server/src/commands.ts @@ -718,6 +718,12 @@ program .option("-q, --queued", "Run job asynchronously", false) .action(createJobAction("tmp:patches:update-effectifs-source")); +program + .command("tmp:patches:update_effectifs_computed_organisme") + .description("Mise a jour des computed organisme des effectifs") + .option("-q, --queued", "Run job asynchronously", false) + .action(createJobAction("tmp:patches:update_effectifs_computed_organisme")); + program .command("dev:list-http-endpoints") .description("Liste les routes du serveur HTTP") diff --git a/server/src/db/migrations/20241114091932-tmp:patches:update_effectifs_computed_organisme.ts b/server/src/db/migrations/20241114091932-tmp:patches:update_effectifs_computed_organisme.ts new file mode 100644 index 000000000..85c92fb8a --- /dev/null +++ b/server/src/db/migrations/20241114091932-tmp:patches:update_effectifs_computed_organisme.ts @@ -0,0 +1,5 @@ +import { addJob } from "job-processor"; + +export const up = async () => { + await addJob({ name: "tmp:patches:update_effectifs_computed_organisme", queued: true }); +}; diff --git a/server/src/jobs/hydrate/effectifs/hydrate-effectifs-computed-organisme.ts b/server/src/jobs/hydrate/effectifs/hydrate-effectifs-computed-organisme.ts new file mode 100644 index 000000000..9fb648b1f --- /dev/null +++ b/server/src/jobs/hydrate/effectifs/hydrate-effectifs-computed-organisme.ts @@ -0,0 +1,45 @@ +import { ObjectId } from "mongodb"; +import { IOrganisme } from "shared/models"; + +import { generateOrganismeComputed } from "@/common/actions/organismes/organismes.actions"; +import logger from "@/common/logger"; +import { organismesDb, effectifsDb } from "@/common/model/collections"; + +export async function hydrateEffectifsComputedOrganisme() { + const batchSize = 100; + let processedCount = 0; + + const cursor = organismesDb().find({}); + while (await cursor.hasNext()) { + const batch: IOrganisme[] = []; + for (let i = 0; i < batchSize; i++) { + if (await cursor.hasNext()) { + const organisme = await cursor.next(); + if (organisme) { + batch.push(organisme); + } + } else { + break; + } + } + + await Promise.all( + batch.map(async (organisme) => { + const computedResult = generateOrganismeComputed(organisme); + await effectifsDb().updateMany( + { organisme_id: new ObjectId(organisme._id) }, + { + $set: { + "_computed.organisme": computedResult, + }, + } + ); + }) + ); + + processedCount += batch.length; + logger.info(`${processedCount} organismes traités jusqu'à présent.`); + } + + logger.info(`Traitement de ${processedCount} organismes terminé.`); +} diff --git a/server/src/jobs/ingestion/process-ingestion.ts b/server/src/jobs/ingestion/process-ingestion.ts index 52d71e723..025dcc0ae 100644 --- a/server/src/jobs/ingestion/process-ingestion.ts +++ b/server/src/jobs/ingestion/process-ingestion.ts @@ -290,16 +290,15 @@ async function transformEffectifQueueV3ToEffectif(rawEffectifQueued: IEffectifQu effectifQueued?.etablissement_lieu_de_formation_uai, effectifQueued?.etablissement_lieu_de_formation_siret ); - organismeTarget = organisme; Object.assign(itemProcessingInfos, addPrefixToProperties("organisme_lieu_", stats)); return organisme; })(), (async () => { const { organisme, stats } = await findOrganismeWithStats( effectifQueued?.etablissement_formateur_uai, - effectifQueued?.etablissement_formateur_siret, - { _id: 1 } + effectifQueued?.etablissement_formateur_siret ); + organismeTarget = organisme; Object.assign(itemProcessingInfos, addPrefixToProperties("organisme_formateur_", stats)); return organisme; })(), diff --git a/server/src/jobs/jobs.ts b/server/src/jobs/jobs.ts index dee872819..b20cc3ab3 100644 --- a/server/src/jobs/jobs.ts +++ b/server/src/jobs/jobs.ts @@ -31,6 +31,7 @@ import { hydrateEffectifsComputedOpcos, hydrateEffectifsComputedReseaux, } from "./hydrate/effectifs/hydrate-effectifs-computed"; +import { hydrateEffectifsComputedOrganisme } from "./hydrate/effectifs/hydrate-effectifs-computed-organisme"; import { hydrateEffectifsComputedTypes } from "./hydrate/effectifs/hydrate-effectifs-computed-types"; import { hydrateEffectifsFormationsNiveaux } from "./hydrate/effectifs/hydrate-effectifs-formations-niveaux"; import { @@ -514,6 +515,11 @@ export async function setupJobProcessor() { return initOpcos(name, rncpList); }, }, + "tmp:patches:update_effectifs_computed_organisme": { + handler: async () => { + return hydrateEffectifsComputedOrganisme(); + }, + }, "process:effectifs-queue:remove-duplicates": { handler: async () => { return removeDuplicatesEffectifsQueue(); diff --git a/server/tests/integration/jobs/ingestion/process-ingestion.test.ts b/server/tests/integration/jobs/ingestion/process-ingestion.test.ts index f1cc14c27..bf52a9d8e 100644 --- a/server/tests/integration/jobs/ingestion/process-ingestion.test.ts +++ b/server/tests/integration/jobs/ingestion/process-ingestion.test.ts @@ -619,7 +619,13 @@ describe("Processus d'ingestion", () => { validation_errors: [], _computed: { organisme: { + academie: "10", + departement: "01", fiable: false, + region: "84", + reseaux: [], + siret: "77937827200016", + uai: "0802004U", }, formation: { opcos: [], @@ -715,7 +721,13 @@ describe("Processus d'ingestion", () => { validation_errors: [], _computed: { organisme: { + academie: "10", + departement: "01", fiable: false, + region: "84", + reseaux: [], + siret: "77937827200016", + uai: "0802004U", }, formation: { opcos: [], @@ -889,7 +901,13 @@ describe("Processus d'ingestion", () => { validation_errors: [], _computed: { organisme: { + academie: "10", + departement: "01", fiable: false, + region: "84", + reseaux: [], + siret: "77937827200016", + uai: "0802004U", }, formation: { opcos: [],