diff --git a/.talismanrc b/.talismanrc index f40f108100..087732ceae 100644 --- a/.talismanrc +++ b/.talismanrc @@ -24,7 +24,7 @@ fileignoreconfig: - filename: server/.env.test checksum: 2534c2dae48c1464b97489263621dcd516a676b28fdbb34e98267a10e00fd839 - filename: server/src/security/accessTokenService.ts - checksum: 671dbfb4d2d85ce2050c8c0755384028619e2ebb98616aca5bc7cd36704bb343 + checksum: f05cafd17797362fc9bfb53062af2095ead2cbe2fa967fad23bd61b756052004 - filename: server/src/common/model/schema/_shared/mongoose-paginate.ts checksum: b6762a7cb5df9bbee1f0ce893827f0991ad01514f7122a848b3b5d49b620f238 - filename: server/src/config.ts @@ -43,8 +43,6 @@ fileignoreconfig: checksum: d716e214d828109181a138f0ae253d5489a3c544b2625917b458d1e07886c408 - filename: server/src/jobs/lba_recruteur/formulaire/misc/removeVersionKeyFromRecruiters.ts checksum: 3cd111d8c109cfec357bae48af70d0cf5644d02cd2c4b9afc5b8aa07bccbd535 -- filename: server/src/security/accessTokenService.ts - checksum: f05cafd17797362fc9bfb53062af2095ead2cbe2fa967fad23bd61b756052004 - filename: server/src/services/application.service.ts checksum: 935cd8f213565ba7bcc2925fca149aaa6cbe9bb5e393a13ab3525dff6ad17234 - filename: server/tests/integration/http/formationV1.test.ts diff --git a/server/src/commands.ts b/server/src/commands.ts index 649acb2309..a6a237ccf2 100644 --- a/server/src/commands.ts +++ b/server/src/commands.ts @@ -557,6 +557,12 @@ program .option("-q, --queued", "Run job asynchronously", false) .action(createJobAction("user-recruters:data-validation:fix")) +program + .command("fix-data-validation-user-recruteurs-cfa") + .description("Répare les data des userrecruteurs CFA") + .option("-q, --queued", "Run job asynchronously", false) + .action(createJobAction("user-recruters-cfa:data-validation:fix")) + program .command("anonymize-user-recruteurs") .description("Anonymize les userrecruteurs qui ne se sont pas connectés depuis plus de 2 ans") diff --git a/server/src/common/model/schema/_shared/shared.types.ts b/server/src/common/model/schema/_shared/shared.types.ts deleted file mode 100644 index d56a87df44..0000000000 --- a/server/src/common/model/schema/_shared/shared.types.ts +++ /dev/null @@ -1,77 +0,0 @@ -interface Academie { - code: string - nom: string -} - -interface Geojson { - geometry: IGeometry - properties: IProperties - type: string -} - -interface IGeometry { - coordinates: number[] - type: string -} - -interface IProperties { - score: number - source: string -} - -interface AcheminementPostal { - l1: string // raison_social | enseigne - l2: string - l3: string // lieu dit - l4: string // numéro et rue - l5: string - l6: string // code postal et ville - l7: string // pays -} - -interface IAdresseCFA { - academie: Academie - code_insee: string - code_postal: string - departement: Academie - geojson: Geojson - label: string - localite: string - region: Academie - acheminement_postal?: AcheminementPostal -} - -interface IAdresseV2 extends AcheminementPostal { - numero_voie: string - type_voie: string - nom_voie: string - complement_adresse: string - code_postal: string - localite: string - code_insee_localite: string - cedex: null - acheminement_postal?: AcheminementPostal -} - -interface IAdresseV3 { - status_diffusion: string - complement_adresse: string | null - numero_voie: string - indice_repetition_voie: string - type_voie: string - libelle_voie: string - code_postal: string - libelle_commune: string - libelle_commune_etranger: string | null - distribution_speciale: string | null - code_commune: string - code_cedex: string | null - libelle_cedex: string | null - code_pays_etranger: string | null - libelle_pays_etranger: string | null - acheminement_postal?: AcheminementPostal -} - -type IGlobalAddress = IAdresseCFA | IAdresseV2 | IAdresseV3 - -export type { IAdresseCFA, IAdresseV2, IAdresseV3, IGlobalAddress } diff --git a/server/src/jobs/jobs.ts b/server/src/jobs/jobs.ts index d8bc0c1787..492cf7ff35 100644 --- a/server/src/jobs/jobs.ts +++ b/server/src/jobs/jobs.ts @@ -40,6 +40,7 @@ import { importReferentielOpcoFromConstructys } from "./lba_recruteur/opco/const import { relanceOpco } from "./lba_recruteur/opco/relanceOpco" import { createOffreCollection } from "./lba_recruteur/seed/createOffre" import { fillRecruiterRaisonSociale } from "./lba_recruteur/user/misc/fillRecruiterRaisonSociale" +import { fixUserRecruiterCfaDataValidation } from "./lba_recruteur/user/misc/fixUserRecruteurCfaDataValidation" import { fixUserRecruiterDataValidation } from "./lba_recruteur/user/misc/fixUserRecruteurDataValidation" import { checkAwaitingCompaniesValidation } from "./lba_recruteur/user/misc/updateMissingActivationState" import { updateSiretInfosInError } from "./lba_recruteur/user/misc/updateSiretInfosInError" @@ -356,6 +357,8 @@ export async function runJob(job: IInternalJobsCronTask | IInternalJobsSimple): return fixRecruiterDataValidation() case "user-recruters:data-validation:fix": return fixUserRecruiterDataValidation() + case "user-recruters-cfa:data-validation:fix": + return fixUserRecruiterCfaDataValidation() case "referentiel-opco:constructys:import": { const { parallelism } = job.payload return importReferentielOpcoFromConstructys(parseInt(parallelism)) diff --git a/server/src/jobs/lba_recruteur/user/misc/fixUserRecruteurCfaDataValidation.ts b/server/src/jobs/lba_recruteur/user/misc/fixUserRecruteurCfaDataValidation.ts new file mode 100644 index 0000000000..703e49177b --- /dev/null +++ b/server/src/jobs/lba_recruteur/user/misc/fixUserRecruteurCfaDataValidation.ts @@ -0,0 +1,59 @@ +import Boom from "boom" +import { ZCfaReferentielData } from "shared/models" + +import { logger } from "@/common/logger" +import { UserRecruteur } from "@/common/model" +import { asyncForEach } from "@/common/utils/asyncUtils" +import { sentryCaptureException } from "@/common/utils/sentryUtils" +import { notifyToSlack } from "@/common/utils/slackUtils" +import { getOrganismeDeFormationDataFromSiret } from "@/services/etablissement.service" +import { updateUser } from "@/services/userRecruteur.service" + +export const fixUserRecruiterCfaDataValidation = async () => { + const subject = "Fix data validations pour les userrecruteurs CFA : address_detail" + const userRecruteurs = await UserRecruteur.find({ type: "CFA" }).lean() + const stats = { success: 0, failure: 0, skip: 0 } + logger.info(`${subject}: ${userRecruteurs.length} user recruteurs à mettre à jour...`) + await asyncForEach(userRecruteurs, async (userRecruiter, index) => { + try { + index % 100 === 0 && logger.info("index", index) + const { establishment_siret, is_qualiopi, establishment_raison_sociale, address_detail, address, geo_coordinates } = userRecruiter + if ( + !ZCfaReferentielData.pick({ + is_qualiopi: true, + establishment_siret: true, + establishment_raison_sociale: true, + address_detail: true, + address: true, + geo_coordinates: true, + }).safeParse({ + is_qualiopi, + establishment_siret, + establishment_raison_sociale, + address_detail, + address, + geo_coordinates, + }).success + ) { + if (!establishment_siret) { + throw Boom.internal("Missing establishment_siret", { _id: userRecruiter._id }) + } + const cfaData = await getOrganismeDeFormationDataFromSiret(establishment_siret, false) + await updateUser({ _id: userRecruiter._id }, cfaData) + stats.success++ + } else { + stats.skip++ + } + } catch (err) { + logger.error(err) + sentryCaptureException(err) + stats.failure++ + } + }) + await notifyToSlack({ + subject, + message: `${stats.failure} erreurs. ${stats.success} mises à jour. ${stats.skip} ignorés.`, + error: stats.failure > 0, + }) + return stats +} diff --git a/server/src/services/etablissement.service.ts b/server/src/services/etablissement.service.ts index 0e16da0261..263e733be6 100644 --- a/server/src/services/etablissement.service.ts +++ b/server/src/services/etablissement.service.ts @@ -1,7 +1,7 @@ import { AxiosResponse } from "axios" import Boom from "boom" import type { FilterQuery } from "mongoose" -import { IEtablissement, ILbaCompany, IRecruiter, IReferentielData, IReferentielOpco, IUserRecruteur, ZUserRecruteurReferentielData } from "shared" +import { ICfaReferentielData, IEtablissement, IGeometry, ILbaCompany, IRecruiter, IReferentielOpco, IUserRecruteur, ZCfaReferentielData, assertUnreachable } from "shared" import { EDiffusibleStatus } from "shared/constants/diffusibleStatus" import { BusinessErrorCodes } from "shared/constants/errorCodes" import { ETAT_UTILISATEUR } from "shared/constants/recruteur" @@ -439,8 +439,6 @@ function getRaisonSocialeFromGouvResponse(d: IEtablissementGouv): string | undef /** * @description Format Entreprise data - * @param {IEtablissementGouv} data - * @returns {IFormatAPIEntreprise} */ export const formatEntrepriseData = (d: IEtablissementGouv): IFormatAPIEntreprise => { if (!d.adresse) { @@ -461,12 +459,27 @@ export const formatEntrepriseData = (d: IEtablissementGouv): IFormatAPIEntrepris } } +function geometryToGeoCoord(geometry: IGeometry): [number, number] { + const { type } = geometry + if (type === "Point") { + return geometry.coordinates + } else if (type === "Polygon") { + return geometry.coordinates[0][0] + } else { + assertUnreachable(type) + } +} + /** * @description Format Referentiel data - * @param {IReferentiel} d - * @returns {Object} */ -export const formatReferentielData = (d: IReferentiel): IReferentielData => { +export const formatReferentielData = (d: IReferentiel): ICfaReferentielData => { + const geojson = d.adresse?.geojson ?? d.lieux_de_formation.at(0)?.adresse?.geojson + if (!geojson) { + throw Boom.internal("impossible de lire la geometry") + } + const coords = geometryToGeoCoord(geojson.geometry) + const referentielData = { establishment_state: d.etat_administratif, is_qualiopi: d.qualiopi, @@ -475,14 +488,11 @@ export const formatReferentielData = (d: IReferentiel): IReferentielData => { contacts: d.contacts, address_detail: d.adresse, address: d.adresse?.label, - geo_coordinates: d.adresse - ? `${d.adresse?.geojson.geometry.coordinates[1]},${d.adresse?.geojson.geometry.coordinates[0]}` - : `${d.lieux_de_formation[0]?.adresse?.geojson?.geometry.coordinates[0]},${d.lieux_de_formation[0]?.adresse?.geojson?.geometry.coordinates[1]}`, + geo_coordinates: `${coords[1]},${coords[0]}`, } - - const validation = ZUserRecruteurReferentielData.safeParse(referentielData) + const validation = ZCfaReferentielData.safeParse(referentielData) if (!validation.success) { - sentryCaptureException(Boom.internal(`address format error for siret=${d.siret}.`, { validationError: validation.error })) + sentryCaptureException(Boom.internal(`erreur de validation sur les données du référentiel CFA pour le siret=${d.siret}.`, { validationError: validation.error })) } return referentielData } @@ -651,16 +661,18 @@ export const getEntrepriseDataFromSiret = async ({ siret, cfa_delegated_siret }: return { ...entrepriseData, geo_coordinates: `${latitude},${longitude}` } } -export const getOrganismeDeFormationDataFromSiret = async (siret: string) => { - const cfaUserRecruteurOpt = await getEtablissement({ establishment_siret: siret, type: CFA }) - if (cfaUserRecruteurOpt) { - throw Boom.forbidden("Ce numéro siret est déjà associé à un compte utilisateur.", { reason: BusinessErrorCodes.ALREADY_EXISTS }) +export const getOrganismeDeFormationDataFromSiret = async (siret: string, shouldValidate = true) => { + if (shouldValidate) { + const cfaUserRecruteurOpt = await getEtablissement({ establishment_siret: siret, type: CFA }) + if (cfaUserRecruteurOpt) { + throw Boom.forbidden("Ce numéro siret est déjà associé à un compte utilisateur.", { reason: BusinessErrorCodes.ALREADY_EXISTS }) + } } const referentiel = await getEtablissementFromReferentiel(siret) if (!referentiel) { throw Boom.badRequest("Le numéro siret n'est pas référencé comme centre de formation.", { reason: BusinessErrorCodes.UNKNOWN }) } - if (referentiel.etat_administratif === "fermé") { + if (shouldValidate && referentiel.etat_administratif === "fermé") { throw Boom.badRequest("Le numéro siret indique un établissement fermé.", { reason: BusinessErrorCodes.CLOSED }) } if (!referentiel.adresse) { @@ -669,7 +681,7 @@ export const getOrganismeDeFormationDataFromSiret = async (siret: string) => { }) } const formattedReferentiel = formatReferentielData(referentiel) - if (!formattedReferentiel.is_qualiopi) { + if (shouldValidate && !formattedReferentiel.is_qualiopi) { throw Boom.badRequest("L’organisme rattaché à ce SIRET n’est pas certifié Qualiopi", { reason: BusinessErrorCodes.NOT_QUALIOPI, ...formattedReferentiel }) } return formattedReferentiel diff --git a/server/src/services/etablissement.service.types.ts b/server/src/services/etablissement.service.types.ts index 60ab4d97f4..68ed5bef46 100644 --- a/server/src/services/etablissement.service.types.ts +++ b/server/src/services/etablissement.service.types.ts @@ -1,8 +1,6 @@ -import { IRecruiter, IUserRecruteur } from "shared" +import { IAdresseCFA, IAdresseV3, IRecruiter, IUserRecruteur } from "shared" import { Jsonify } from "type-fest" -import { IAdresseCFA, IAdresseV3 } from "../common/model/schema/_shared/shared.types" - export interface IFormatAPIReferentiel extends Pick { establishment_state: string diff --git a/shared/models/address.model.ts b/shared/models/address.model.ts index edc404c74c..8d649c5c24 100644 --- a/shared/models/address.model.ts +++ b/shared/models/address.model.ts @@ -7,16 +7,29 @@ const ZAcademie = z }) .strict() -const ZGeometry = z +const Z2DCoord = z.tuple([z.number(), z.number()]) + +const ZPointGeometry = z .object({ - coordinates: z.array(z.number()), - type: z.string(), + coordinates: Z2DCoord, + type: z.literal("Point"), }) .strict() +const ZPolygonGeometry = z + .object({ + coordinates: z.array(z.array(Z2DCoord)), // oui, il y a un niveau de tableau en trop mais ce sont les data qu'on récupère + type: z.literal("Polygon"), + }) + .strict() + +const ZGeometry = z.union([ZPointGeometry, ZPolygonGeometry]) + +export type IGeometry = z.input + const ZProperties = z .object({ - score: z.number(), + score: z.number().nullish(), source: z.string(), }) .strict() @@ -92,3 +105,7 @@ const ZAdresseV3 = z .openapi("AdresseV3") export const ZGlobalAddress = z.union([ZAdresseCFA, ZAdresseV2, ZAdresseV3]) + +export type IAdresseV3 = z.input +export type IAdresseCFA = z.input +export type IGlobalAddress = z.input diff --git a/shared/models/cfa.model.ts b/shared/models/cfa.model.ts deleted file mode 100644 index aba270a3bf..0000000000 --- a/shared/models/cfa.model.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { z } from "../helpers/zodWithOpenApi" - -export const zCFA = z - .object({ - establishment_state: z.string(), - is_qualiopi: z.string(), - establishment_siret: z.string(), - establishment_raison_sociale: z.string(), - contacts: z.string(), - address_detail: z.string(), - address: z.string(), - geo_coordinates: z.string(), - }) - .strict() - .openapi("Model") diff --git a/shared/models/index.ts b/shared/models/index.ts index b8844e9fc8..1f764797ef 100644 --- a/shared/models/index.ts +++ b/shared/models/index.ts @@ -1,7 +1,6 @@ export * from "./address.model" export * from "./applications.model" export * from "./appointments.model" -export * from "./cfa.model" export * from "./credentials.model" export * from "./elligibleTraining.model" export * from "./emailBlacklist.model" diff --git a/shared/models/usersRecruteur.model.ts b/shared/models/usersRecruteur.model.ts index 8cc7051b0f..04623be52b 100644 --- a/shared/models/usersRecruteur.model.ts +++ b/shared/models/usersRecruteur.model.ts @@ -59,7 +59,7 @@ export const ZUserRecruteur = ZUserRecruteurWritable.omit({ is_qualiopi: ZUserRecruteurWritable.shape.is_qualiopi.nullish(), }) -export const zReferentielData = z +export const ZCfaReferentielData = z .object({ establishment_state: z.string(), is_qualiopi: z.boolean(), @@ -77,11 +77,11 @@ export const zReferentielData = z ), address_detail: ZGlobalAddress, address: z.string(), - geo_coordinates: z.string().nullish(), + geo_coordinates: z.string().max(40).nullish(), }) .strict() -export type IReferentielData = z.output +export type ICfaReferentielData = z.output export type IUserStatusValidation = z.output export type IUserStatusValidationJson = Jsonify> @@ -161,14 +161,3 @@ export const ZAnonymizedUserRecruteur = ZUserRecruteur.pick({ }) export type IAnonymizedUserRecruteur = z.output - -export const ZUserRecruteurReferentielData = ZUserRecruteur.pick({ - establishment_state: true, - is_qualiopi: true, - establishment_siret: true, - establishment_raison_sociale: true, - contacts: true, - address_detail: true, - address: true, - geo_coordinates: true, -}).strict() diff --git a/shared/routes/recruiters.routes.ts b/shared/routes/recruiters.routes.ts index 3d4eaa202a..7af92941de 100644 --- a/shared/routes/recruiters.routes.ts +++ b/shared/routes/recruiters.routes.ts @@ -4,7 +4,7 @@ import { extensions } from "../helpers/zodHelpers/zodPrimitives" import { z } from "../helpers/zodWithOpenApi" import { ZRecruiter } from "../models" import { zObjectId } from "../models/common" -import { ZUserRecruteur, ZUserRecruteurPublic, ZUserRecruteurWritable, zReferentielData } from "../models/usersRecruteur.model" +import { ZUserRecruteur, ZUserRecruteurPublic, ZUserRecruteurWritable, ZCfaReferentielData } from "../models/usersRecruteur.model" import { IRoutesDef } from "./common.routes" @@ -99,7 +99,7 @@ export const zRecruiterRoutes = { // TODO_SECURITY_FIX faire en sorte que le back refasse l'appel params: z.object({ siret: extensions.siret }).strict(), response: { - "2xx": zReferentielData, + "2xx": ZCfaReferentielData, }, securityScheme: null, },