From f2944a50b929e6cafa9f406415b03eda66c02f27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Auricoste?= Date: Wed, 11 Dec 2024 10:05:12 +0100 Subject: [PATCH] fix(lbac-2298): ajout de la verif code postal dans les duplicates (#1697) * fix(lbac-2298): ajout de la verif code postal dans les duplicates * fix: improve tests * fix: tests --- .talismanrc | 14 +- .../detectDuplicateJobPartners.test.ts | 142 +++++++++++++++--- .../detectDuplicateJobPartners.ts | 90 +++++++---- .../fillComputedJobsPartners.ts | 2 + .../validateComputedJobPartners.ts | 33 ++-- .../formulaire.service.test.ts.snap | 19 +++ shared/fixtures/recruiter.fixture.ts | 21 ++- shared/models/jobsPartnersComputed.model.ts | 4 +- 8 files changed, 254 insertions(+), 71 deletions(-) diff --git a/.talismanrc b/.talismanrc index 8712e48f7..4444256f8 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,4 +1,6 @@ fileignoreconfig: +- filename: .infra/files/configs/mongodb/seed.gpg + checksum: c0d065fd21e851774bfb4f6dad2d4af7e339d38d3c62241ae40491ff4f8b8543 - filename: .infra/vault/vault.yml checksum: 40717e8b277d9c9812c89d62fd3e09d0d70cdd0bb76a460f09486a31b6dd9cc7 - filename: cypress/e2e/manual/create-many-applications.cy.ts @@ -37,6 +39,8 @@ fileignoreconfig: checksum: 72502c3acc46f9b6903b69095518c3d865cf437f311ed14585da04d8440b4ac0 - filename: server/src/http/controllers/v2/jobs.controller.v2.test.ts checksum: a99bfdcb5a8f4c66e9d055b44160adbc7ec2bcaf87209ff76fa9a9f674b9ca86 +- filename: server/src/http/controllers/v2/jobs.controller.v2.ts + checksum: da8baa573172fa97c21f9123e04032718249dcec1194a51d7a1f1757070594ce - filename: server/src/http/controllers/v3/jobs/jobs.controller.v3.test.ts checksum: 4b24ff09463f476fd0b64bef104da6e17153ffdfff2d628c96b16d3ea33779f7 - filename: server/src/http/routes/appointmentRequest.controller.ts @@ -57,6 +61,8 @@ fileignoreconfig: checksum: 3cd111d8c109cfec357bae48af70d0cf5644d02cd2c4b9afc5b8aa07bccbd535 - filename: server/src/jobs/offrePartenaire/__snapshots__/importRHAlternance.test.ts.snap checksum: 41e5eadcdaa0cb1eb95e4cb3859f4b7b1d176bbb95ee8451d46a6b2c9021c3fd +- filename: server/src/jobs/offrePartenaire/detectDuplicateJobPartners.test.ts + checksum: 82d624da5034d7e482cb7b2a99a134796e3a2b3c3a2a89a10da2e49242854c79 - filename: server/src/jobs/offrePartenaire/importFromUrlInXml.ts checksum: 18bd149922151536ad0e239b082309697cee6046b33512d5eecc54b2d25b465a - filename: server/src/jobs/offrePartenaire/importHelloWork.test.input.xml @@ -85,6 +91,8 @@ fileignoreconfig: checksum: a263b92a95e69de7efb856d73122b8f05ef98140c96dd0212213f26dce85056e - filename: server/src/services/formulaire.service.ts checksum: e93ff7ce146d35e70eedcbe9b66097750e7754650468a5ada6b0ed72f0fdbc49 +- filename: server/src/services/jobs/jobOpportunity/__snapshots__/jobOpportunity.service.test.ts.snap + checksum: da3aa00d2041f36cd4b24eb4c98d04eb236cc80eab82a4e5aaf2e7e82549c613 - filename: server/src/services/jobs/jobOpportunity/jobOpportunity.service.test.ts checksum: 8dd77bbcf687b50239b4958fbb957c069dda342551b76cc797af5349e0bd7024 - filename: server/src/services/recruteurLba.service.test.ts @@ -239,12 +247,6 @@ fileignoreconfig: checksum: 324cd501354cfff65447c2599c4cc8966aa8aac30dda7854623dd6f7f7b0d34e - filename: yarn.lock checksum: 21c8c4f9064194196622ad215768893c1756eec1cbc175efffcb1428d8821c9f -- filename: server/src/http/controllers/v2/jobs.controller.v2.ts - checksum: da8baa573172fa97c21f9123e04032718249dcec1194a51d7a1f1757070594ce -- filename: server/src/services/jobs/jobOpportunity/__snapshots__/jobOpportunity.service.test.ts.snap - checksum: da3aa00d2041f36cd4b24eb4c98d04eb236cc80eab82a4e5aaf2e7e82549c613 -- filename: .infra/files/configs/mongodb/seed.gpg - checksum: c0d065fd21e851774bfb4f6dad2d4af7e339d38d3c62241ae40491ff4f8b8543 scopeconfig: - scope: node custom_patterns: diff --git a/server/src/jobs/offrePartenaire/detectDuplicateJobPartners.test.ts b/server/src/jobs/offrePartenaire/detectDuplicateJobPartners.test.ts index b1cf4f813..870cc6d13 100644 --- a/server/src/jobs/offrePartenaire/detectDuplicateJobPartners.test.ts +++ b/server/src/jobs/offrePartenaire/detectDuplicateJobPartners.test.ts @@ -1,16 +1,18 @@ import { givenSomeComputedJobPartners } from "@tests/fixture/givenSomeComputedJobPartners" import { useMongo } from "@tests/utils/mongo.test.utils" import { saveRecruiter } from "@tests/utils/user.test.utils" +import { ObjectId } from "bson" import { RECRUITER_STATUS } from "shared/constants" import { generateJobFixture, generateRecruiterFixture } from "shared/fixtures/recruiter.fixture" import { JOB_STATUS } from "shared/models" import { JOBPARTNERS_LABEL } from "shared/models/jobsPartners.model" +import { JOB_PARTNER_BUSINESS_ERROR } from "shared/models/jobsPartnersComputed.model" import { beforeEach, describe, expect, it } from "vitest" import { getPairs } from "@/common/utils/array" import { getDbCollection } from "@/common/utils/mongodbUtils" -import { checkSimilarity, detectDuplicateJobPartners } from "./detectDuplicateJobPartners" +import { checkSimilarity, detectDuplicateJobPartners, isCanonicalForDuplicate, OfferRef } from "./detectDuplicateJobPartners" const siret = "42476141900045" @@ -29,14 +31,18 @@ describe("detectDuplicateJobPartners", () => { // given await givenSomeComputedJobPartners([ { + _id: new ObjectId("60646425184afd00e017c1ab"), partner_label: JOBPARTNERS_LABEL.HELLOWORK, workplace_siret: siret, offer_title: offerTitle, + workplace_address_zipcode: "75007", }, { + _id: new ObjectId("62646425184afd00e017c1ab"), partner_label: JOBPARTNERS_LABEL.RH_ALTERNANCE, workplace_siret: siret, offer_title: offerTitle, + workplace_address_zipcode: "75007", }, ]) // when @@ -51,6 +57,7 @@ describe("detectDuplicateJobPartners", () => { reason: "identical workplace_siret, identical offer_title", }, ]) + expect.soft(job.business_error).toEqual(null) expect.soft(job2.duplicates).toEqual([ { otherOfferId: job._id, @@ -58,20 +65,28 @@ describe("detectDuplicateJobPartners", () => { reason: "identical workplace_siret, identical offer_title", }, ]) + expect.soft(job2.business_error).toEqual(JOB_PARTNER_BUSINESS_ERROR.DUPLICATE) }) it("should detect a duplicate with an exact match in the offer title and the siret (recruiter)", async () => { // given await givenSomeComputedJobPartners([ { + _id: new ObjectId("60646425184afd00e017c1ab"), partner_label: JOBPARTNERS_LABEL.RH_ALTERNANCE, workplace_siret: siret, offer_title: offerTitle, + workplace_address_zipcode: "75007", }, ]) const recruiter = await saveRecruiter( generateRecruiterFixture({ + _id: new ObjectId("62646425184afd00e017c1ab"), establishment_siret: siret, status: RECRUITER_STATUS.ACTIF, + address_detail: { + ...generateRecruiterFixture().address_detail, + code_postal: "75007", + }, jobs: [ generateJobFixture({ rome_appellation_label: offerTitle, @@ -92,6 +107,64 @@ describe("detectDuplicateJobPartners", () => { reason: "identical workplace_siret, identical offer_title", }, ]) + expect.soft(job.business_error).toEqual(JOB_PARTNER_BUSINESS_ERROR.DUPLICATE) + }) + it("should not detect a duplicate with an exact match in the offer title and the siret but a different zip code (job_partner)", async () => { + // given + await givenSomeComputedJobPartners([ + { + partner_label: JOBPARTNERS_LABEL.HELLOWORK, + workplace_siret: siret, + offer_title: offerTitle, + workplace_address_zipcode: "75002", + }, + { + partner_label: JOBPARTNERS_LABEL.RH_ALTERNANCE, + workplace_siret: siret, + offer_title: offerTitle, + workplace_address_zipcode: "75007", + }, + ]) + // when + await detectDuplicateJobPartners() + // then + const jobs = await getDbCollection("computed_jobs_partners").find({}).toArray() + const [job, job2] = jobs + expect.soft(job.duplicates).toEqual([]) + expect.soft(job2.duplicates).toEqual([]) + }) + it("should not detect a duplicate with an exact match in the offer title and the siret but a different zip code (recruiter)", async () => { + // given + await givenSomeComputedJobPartners([ + { + partner_label: JOBPARTNERS_LABEL.RH_ALTERNANCE, + workplace_siret: siret, + offer_title: offerTitle, + workplace_address_zipcode: "75002", + }, + ]) + await saveRecruiter( + generateRecruiterFixture({ + establishment_siret: siret, + status: RECRUITER_STATUS.ACTIF, + address_detail: { + ...generateRecruiterFixture().address_detail, + code_postal: "75007", + }, + jobs: [ + generateJobFixture({ + rome_appellation_label: offerTitle, + job_status: JOB_STATUS.ACTIVE, + }), + ], + }) + ) + // when + await detectDuplicateJobPartners() + // then + const jobs = await getDbCollection("computed_jobs_partners").find({}).toArray() + const [job] = jobs + expect.soft(job.duplicates).toEqual([]) }) describe("checkSimilarity", () => { it("should not detect when one string is empty", async () => { @@ -106,29 +179,54 @@ describe("detectDuplicateJobPartners", () => { expect.soft(checkSimilarity(offerTitle, offerTitle)).toEqual("identical") }) - const coiffeurGroup = ["H/F Coiffeur", "Coiffeur H/F", "Coiffeur", "Coiffeuse", "Coiffeur.se", "Coiffeur (H/F)"] - it.each(getPairs(coiffeurGroup))("should detect similar strings: %s === %s", (item1, item2) => { - expect.soft(checkSimilarity(item1, item2)).toEqual(expect.stringContaining("similar")) - }) - - const directionGroup = ["Direction interministérielle du numérique", "Dir. interministérielle du numérique"] - it.each(getPairs(directionGroup))("should detect similar strings: %s === %s", (item1, item2) => { - expect.soft(checkSimilarity(item1, item2)).toEqual(expect.stringContaining("similar")) - }) - - const sigleGroup = ["MFR la pignerie", "Maison familiale rurale la pignerie"] - it.each(getPairs(sigleGroup))("should detect similar strings: %s === %s", (item1, item2) => { - expect.soft(checkSimilarity(item1, item2)).toEqual(expect.stringContaining("similar")) - }) - - const sensSimilaireGroup = ["Chargé de mission en Ressources Humaines (RH) (H/F)", "Chargé de mission RH", "Chargé.e de mission RH"] - it.each(getPairs(sensSimilaireGroup))("should detect similar strings: %s === %s", (item1, item2) => { - expect.soft(checkSimilarity(item1, item2)).toEqual(expect.stringContaining("similar")) + const groups = [ + ["H/F Coiffeur", "Coiffeur H/F", "Coiffeur", "Coiffeuse", "Coiffeur.se", "Coiffeur (H/F)"], + ["Direction interministérielle du numérique", "Dir. interministérielle du numérique"], + ["MFR la pignerie", "Maison familiale rurale la pignerie"], + ["Chargé de mission en Ressources Humaines (RH) (H/F)", "Chargé de mission RH", "Chargé.e de mission RH"], + // ["Drysoft", "dyrsoft"] + ] + groups.forEach((group) => { + it.each(getPairs(group))("should detect similar strings: %s === %s", (item1, item2) => { + expect.soft(checkSimilarity(item1, item2)).toEqual(expect.stringContaining("similar")) + }) }) + }) + describe("isCanonicalForDuplicate", () => { + // cf https://www.math.univ-toulouse.fr/~msablik/Cours/MathDiscretes/Slide4-Relation.pdf + describe("doit être une relation d ordre", () => { + const offerRefs: OfferRef[] = [ + { collectionName: "recruiters", _id: new ObjectId("60646425184afd00e017c1ab") }, + { collectionName: "recruiters", _id: new ObjectId("642605b7c52247005267027e") }, + { collectionName: "recruiters", _id: new ObjectId("682605b7c52247005267027e") }, + { collectionName: "computed_jobs_partners", _id: new ObjectId("6525231132fe045f240e4bcd") }, + { collectionName: "computed_jobs_partners", _id: new ObjectId("67347c0c9af58cb6e6ebbb65") }, + { collectionName: "computed_jobs_partners", _id: new ObjectId("63347c0c9af58cb6e6ebbb65") }, + ] - const typoGroup = ["Drysoft", "dyrsoft"] - it.skip.each(getPairs(typoGroup))("should detect similar strings: %s === %s", (item1, item2) => { - expect.soft(checkSimilarity(item1, item2)).toEqual(expect.stringContaining("similar")) + it("la relation doit être reflexive (xRx)", () => { + offerRefs.forEach((offerRef) => expect(isCanonicalForDuplicate(offerRef, offerRef)).toBe(true)) + }) + it("la relation doit être antisymétrique (xRy and yRx => x = y)", () => { + getPairs(offerRefs) + .filter(([item, item2]) => isCanonicalForDuplicate(item, item2) && isCanonicalForDuplicate(item2, item)) + .forEach(([item, item2]) => expect(item).toBe(item2)) + }) + it("la relation doit être transitive (xRy and yRz => xRz)", () => { + const validPairs = getPairs(offerRefs).flatMap(([item, item2]) => { + const results: [OfferRef, OfferRef][] = [] + if (isCanonicalForDuplicate(item, item2)) { + results.push([item, item2]) + } + if (isCanonicalForDuplicate(item2, item)) { + results.push([item2, item]) + } + return results + }) + getPairs(validPairs) + .filter(([[_item, item2], [item3, _item4]]) => item2 === item3) + .forEach(([[item, _item2], [_item3, item4]]) => expect(isCanonicalForDuplicate(item, item4)).toBe(true)) + }) }) }) }) diff --git a/server/src/jobs/offrePartenaire/detectDuplicateJobPartners.ts b/server/src/jobs/offrePartenaire/detectDuplicateJobPartners.ts index bfe906a6d..728f841b9 100644 --- a/server/src/jobs/offrePartenaire/detectDuplicateJobPartners.ts +++ b/server/src/jobs/offrePartenaire/detectDuplicateJobPartners.ts @@ -1,9 +1,9 @@ import { groupBy } from "lodash-es" -import { ObjectId } from "mongodb" +import { AnyBulkWriteOperation, Filter, ObjectId } from "mongodb" import { oleoduc, writeData } from "oleoduc" import { RECRUITER_STATUS } from "shared/constants" -import { IJob, JOB_STATUS } from "shared/models" -import jobsPartnersComputedModel, { IComputedJobPartnersDuplicateRef, IComputedJobsPartners } from "shared/models/jobsPartnersComputed.model" +import { IJob, JOB_STATUS, ZGlobalAddress } from "shared/models" +import jobsPartnersComputedModel, { IComputedJobPartnersDuplicateRef, IComputedJobsPartners, JOB_PARTNER_BUSINESS_ERROR } from "shared/models/jobsPartnersComputed.model" import recruiterModel, { IRecruiter } from "shared/models/recruiter.model" import { removeAccents } from "shared/utils" import * as stringSimilarity from "string-similarity" @@ -14,8 +14,8 @@ import { asyncForEach } from "@/common/utils/asyncUtils" import { getDbCollection } from "@/common/utils/mongodbUtils" // champs utilisés pour les projections -const fieldsRead = ["_id", "partner_label", "offer_title", "duplicates"] as const satisfies (keyof IComputedJobsPartners)[] -const recruiterFieldsRead = ["_id", "status"] as const satisfies (keyof IRecruiter)[] +const fieldsRead = ["_id", "partner_label", "offer_title", "duplicates", "workplace_address_zipcode"] as const satisfies (keyof IComputedJobsPartners)[] +const recruiterFieldsRead = ["_id", "status", "address_detail"] as const satisfies (keyof IRecruiter)[] const jobFieldsRead = ["_id", "rome_appellation_label", "job_status"] as const satisfies (keyof IJob)[] const FAKE_RECRUITERS_JOB_PARTNER = "recruiters" @@ -32,8 +32,10 @@ type AggregationResult = { recruiters?: ProjectedIRecruiter[] } +const computedJobPartnersFilter: Filter = { business_error: null } + export const detectDuplicateJobPartners = async () => { - await getDbCollection("computed_jobs_partners").updateMany({}, { $set: { duplicates: [] } }) + await getDbCollection("computed_jobs_partners").updateMany(computedJobPartnersFilter, { $set: { duplicates: [] } }) const jobPartnerFields: (keyof IComputedJobsPartners)[] = ["workplace_siret", "workplace_brand", "workplace_legal_name", "workplace_name"] const jobPartnerVsRecruiterFields: { jobPartnerField: keyof IComputedJobsPartners; recruiterField: keyof IRecruiter }[] = [ { jobPartnerField: "workplace_siret", recruiterField: "establishment_siret" }, @@ -52,6 +54,7 @@ const jobPartnerStreamFactory = (groupField: keyof IComputedJobsPartners) => { logger.info(`début de detectDuplicateJobPartners groupé par le champ ${groupField}`) return getDbCollection("computed_jobs_partners") .aggregate([ + { $match: computedJobPartnersFilter }, { $group: { _id: `$${groupField}`, documents: { $push: "$$ROOT" } } }, { $match: { _id: { $ne: null }, "documents.1": { $exists: true } } }, { @@ -74,6 +77,7 @@ const jobPartnerVsRecruiterStreamFactory = (jobPartnerField: keyof IComputedJobs logger.info(`début de detectDuplicateJobPartners entre computedJobPartners et recruiters, pour les champs jobPartnerField=${jobPartnerField} et recruiterField=${recruiterField}`) return getDbCollection("computed_jobs_partners") .aggregate([ + { $match: computedJobPartnersFilter }, { $group: { _id: `$${jobPartnerField}`, documents: { $push: "$$ROOT" } } }, { $match: { _id: { $ne: null } } }, { @@ -131,10 +135,11 @@ const detectDuplicateJobPartnersFactory = async (groupField: keyof IComputedJobs documentStream, writeData(async (aggregationResult: AggregationResult) => { const convertedRecruiters: TreatedDocument[] = (aggregationResult?.recruiters ?? []).flatMap((recruiter) => { - const { jobs, status } = recruiter + const { jobs, status, address_detail } = recruiter if (status !== RECRUITER_STATUS.ACTIF) { return [] } + const parsedAddress = ZGlobalAddress.safeParse(address_detail) return jobs.flatMap((job) => { const { _id, rome_appellation_label, job_status } = job if (job_status !== JOB_STATUS.ACTIVE) { @@ -144,6 +149,7 @@ const detectDuplicateJobPartnersFactory = async (groupField: keyof IComputedJobs _id, partner_label: FAKE_RECRUITERS_JOB_PARTNER, offer_title: rome_appellation_label, + workplace_address_zipcode: parsedAddress.data?.code_postal || null, } return [mapped] }) @@ -166,6 +172,9 @@ const detectDuplicateJobPartnersFactory = async (groupField: keyof IComputedJobs maxOfferPairCount = Math.max(maxOfferPairCount, offerPairs.length) offerPairCount += offerPairs.length const updates = offerPairs.flatMap(([offer1, offer2]) => { + if (offer1.workplace_address_zipcode !== offer2.workplace_address_zipcode) { + return [] + } const reasons: string[] = [`identical ${groupField}`] const similarityOpt = checkSimilarity(offer1.offer_title, offer2.offer_title) if (similarityOpt) { @@ -215,7 +224,7 @@ export const checkSimilarity = (string1: string | null | undefined, string2: str } } -type OfferRef = { +export type OfferRef = { _id: ObjectId collectionName: IComputedJobPartnersDuplicateRef["collectionName"] } @@ -227,7 +236,17 @@ const treatedDocumentToOfferRef = (doc: TreatedDocument): OfferRef => { } } -const duplicateInfosToMongoUpdates = (offer1: OfferRef, offer2: OfferRef, reasons: string[]) => { +// in case of duplicates, returns true if offer1 is selected over offer2 +export const isCanonicalForDuplicate = (offer1: OfferRef, offer2: OfferRef) => { + if (offer1.collectionName === offer2.collectionName) return offer1._id.toString() <= offer2._id.toString() + if (offer2.collectionName === "recruiters") return false + if (offer1.collectionName === "recruiters") return true + return offer1._id.toString() <= offer2._id.toString() +} + +type BulkOperation = AnyBulkWriteOperation + +const duplicateInfosToMongoUpdates = (offer1: OfferRef, offer2: OfferRef, reasons: string[]): BulkOperation[] => { const reason = reasons.join(", ") const duplicateObject1: IComputedJobPartnersDuplicateRef = { otherOfferId: offer2._id, @@ -239,24 +258,41 @@ const duplicateInfosToMongoUpdates = (offer1: OfferRef, offer2: OfferRef, reason collectionName: offer1.collectionName, reason, } - return [ - offer1.collectionName === jobsPartnersComputedModel.collectionName - ? { - updateOne: { - filter: { _id: offer1._id }, - update: { $push: { duplicates: duplicateObject1 } }, - }, - } - : null, - offer2.collectionName === jobsPartnersComputedModel.collectionName - ? { - updateOne: { - filter: { _id: offer2._id }, - update: { $push: { duplicates: duplicateObject2 } }, - }, - } - : null, - ].flatMap((x) => (x ? [x] : [])) + const isOffer1canonical = isCanonicalForDuplicate(offer1, offer2) + const operations: BulkOperation[] = [] + if (offer1.collectionName === jobsPartnersComputedModel.collectionName) { + operations.push({ + updateOne: { + filter: { _id: offer1._id }, + update: { $push: { duplicates: duplicateObject1 } }, + }, + }) + if (!isOffer1canonical) { + operations.push({ + updateOne: { + filter: { _id: offer1._id }, + update: { $set: { business_error: JOB_PARTNER_BUSINESS_ERROR.DUPLICATE } }, + }, + }) + } + } + if (offer2.collectionName === jobsPartnersComputedModel.collectionName) { + operations.push({ + updateOne: { + filter: { _id: offer2._id }, + update: { $push: { duplicates: duplicateObject2 } }, + }, + }) + if (isOffer1canonical) { + operations.push({ + updateOne: { + filter: { _id: offer2._id }, + update: { $set: { business_error: JOB_PARTNER_BUSINESS_ERROR.DUPLICATE } }, + }, + }) + } + } + return operations } const acronymes = { diff --git a/server/src/jobs/offrePartenaire/fillComputedJobsPartners.ts b/server/src/jobs/offrePartenaire/fillComputedJobsPartners.ts index e81acd0f1..14bbf5276 100644 --- a/server/src/jobs/offrePartenaire/fillComputedJobsPartners.ts +++ b/server/src/jobs/offrePartenaire/fillComputedJobsPartners.ts @@ -1,6 +1,7 @@ import { JOB_STATUS_ENGLISH } from "shared/models" import { IComputedJobsPartners } from "shared/models/jobsPartnersComputed.model" +import { detectDuplicateJobPartners } from "./detectDuplicateJobPartners" import { fillLocationInfosForPartners } from "./fillLocationInfosForPartners" import { fillOpcoInfosForPartners } from "./fillOpcoInfosForPartners" import { fillRomeForPartners } from "./fillRomeForPartners" @@ -12,6 +13,7 @@ export const fillComputedJobsPartners = async () => { await fillSiretInfosForPartners() await fillLocationInfosForPartners() await fillRomeForPartners() + await detectDuplicateJobPartners() await validateComputedJobPartners() } diff --git a/server/src/jobs/offrePartenaire/validateComputedJobPartners.ts b/server/src/jobs/offrePartenaire/validateComputedJobPartners.ts index 930aa37aa..b1823488e 100644 --- a/server/src/jobs/offrePartenaire/validateComputedJobPartners.ts +++ b/server/src/jobs/offrePartenaire/validateComputedJobPartners.ts @@ -1,6 +1,7 @@ +import { AnyBulkWriteOperation } from "mongodb" import { oleoduc, writeData } from "oleoduc" import jobsPartnersModel from "shared/models/jobsPartners.model" -import { COMPUTED_ERROR_SOURCE, IComputedJobsPartners } from "shared/models/jobsPartnersComputed.model" +import { COMPUTED_ERROR_SOURCE, IComputedJobsPartners, JOB_PARTNER_BUSINESS_ERROR } from "shared/models/jobsPartnersComputed.model" import { logger } from "@/common/logger" import { getDbCollection } from "@/common/utils/mongodbUtils" @@ -9,6 +10,8 @@ import { streamGroupByCount } from "@/common/utils/streamUtils" const groupSize = 100 const zodModel = jobsPartnersModel.zod +type BulkOperation = AnyBulkWriteOperation + export const validateComputedJobPartners = async () => { logger.info(`validation des computed_job_partners`) const toUpdateCount = await getDbCollection("computed_jobs_partners").countDocuments({}) @@ -21,14 +24,11 @@ export const validateComputedJobPartners = async () => { async (documents: IComputedJobsPartners[]) => { counters.total += documents.length counters.total % 1000 === 0 && logger.info(`processing document ${counters.total}`) - const bulkWriteFunction = getDbCollection("computed_jobs_partners").bulkWrite - type Operation = Parameters[0][number] - const setOperations: Operation[] = [] - const pushOperations: Operation[] = [] + const operations: BulkOperation[] = [] documents.map((document) => { const { success: validated, error } = zodModel.safeParse(document) - setOperations.push({ + operations.push({ updateOne: { filter: { _id: document._id }, update: { @@ -39,7 +39,7 @@ export const validateComputedJobPartners = async () => { if (error) { counters.error++ - pushOperations.push({ + operations.push({ updateOne: { filter: { _id: document._id }, update: { @@ -52,17 +52,22 @@ export const validateComputedJobPartners = async () => { }, }, }) + operations.push({ + updateOne: { + filter: { _id: document._id }, + update: { + $set: { + business_error: JOB_PARTNER_BUSINESS_ERROR.ZOD_VALIDATION, + }, + }, + }, + }) } else { counters.success++ } }) - if (setOperations?.length) { - await getDbCollection("computed_jobs_partners").bulkWrite(setOperations, { - ordered: false, - }) - } - if (pushOperations?.length) { - await getDbCollection("computed_jobs_partners").bulkWrite(pushOperations, { + if (operations?.length) { + await getDbCollection("computed_jobs_partners").bulkWrite(operations, { ordered: false, }) } diff --git a/server/src/services/__snapshots__/formulaire.service.test.ts.snap b/server/src/services/__snapshots__/formulaire.service.test.ts.snap index bfd7d9a58..c2e17387e 100644 --- a/server/src/services/__snapshots__/formulaire.service.test.ts.snap +++ b/server/src/services/__snapshots__/formulaire.service.test.ts.snap @@ -3,6 +3,25 @@ exports[`createJob > should insert a job 1`] = ` { "_id": "670ce30b57a50d6875c141f9", + "address_detail": { + "acheminement_postal": { + "l1": null, + "l2": null, + "l3": null, + "l4": "15 RUE DES LOUPS", + "l5": null, + "l6": "75002 PARIS", + "l7": "FRANCE", + }, + "cedex": null, + "code_insee_localite": "75002", + "code_postal": "75002", + "complement_adresse": null, + "localite": "PARIS", + "nom_voie": "DES LOUPS", + "numero_voie": "15", + "type_voie": "RUE", + }, "cfa_delegated_siret": null, "createdAt": 2021-01-28T15:00:00.000Z, "email": "entreprise@mail.fr", diff --git a/shared/fixtures/recruiter.fixture.ts b/shared/fixtures/recruiter.fixture.ts index b4e5db216..d873d8c8d 100644 --- a/shared/fixtures/recruiter.fixture.ts +++ b/shared/fixtures/recruiter.fixture.ts @@ -24,7 +24,7 @@ type RecruiterFixtureInput = Partial< } > -export function generateRecruiterFixture(data: RecruiterFixtureInput): IRecruiter { +export function generateRecruiterFixture(data: RecruiterFixtureInput = {}): IRecruiter { return { _id: new ObjectId(), establishment_id: "xxxx-xxxx-xxxx-xxxx", @@ -36,6 +36,25 @@ export function generateRecruiterFixture(data: RecruiterFixtureInput): IRecruite status: RECRUITER_STATUS.ACTIF, createdAt: new Date("2021-01-28T15:00:00.000Z"), updatedAt: new Date("2021-02-03T17:00:00.000Z"), + address_detail: { + numero_voie: "15", + type_voie: "RUE", + nom_voie: "DES LOUPS", + complement_adresse: null, + code_postal: "75002", + localite: "PARIS", + code_insee_localite: "75002", + cedex: null, + acheminement_postal: { + l1: null, + l2: null, + l3: null, + l4: "15 RUE DES LOUPS", + l5: null, + l6: "75002 PARIS", + l7: "FRANCE", + }, + }, ...data, jobs: data.jobs ? data.jobs.map((job) => generateJobFixture(job)) : [generateJobFixture({})], } diff --git a/shared/models/jobsPartnersComputed.model.ts b/shared/models/jobsPartnersComputed.model.ts index 6705ca7cb..6eb44d1bd 100644 --- a/shared/models/jobsPartnersComputed.model.ts +++ b/shared/models/jobsPartnersComputed.model.ts @@ -18,6 +18,8 @@ export enum COMPUTED_ERROR_SOURCE { export enum JOB_PARTNER_BUSINESS_ERROR { CLOSED_COMPANY = "CLOSED_COMPANY", + DUPLICATE = "DUPLICATE", + ZOD_VALIDATION = "ZOD_VALIDATION", } export const ZComputedJobPartnersDuplicateRef = z.object({ @@ -45,7 +47,7 @@ export const ZComputedJobsPartners = extensions .nullable() .describe("Détail des erreurs rencontrées lors de la récupération des données obligatoires") ), - validated: z.boolean().default(false).describe("Toutes les données nécessaires au passage vers jobs_partners sont présentes et valides"), + validated: z.boolean().default(false).describe("Toutes les données nécessaires au passage vers jobs_partners sont présentes et valides (validation zod)"), business_error: z.string().nullable().default(null), duplicates: z.array(ZComputedJobPartnersDuplicateRef).nullish().describe("Référence les autres offres en duplicata avec celle-ci"), })