feat: validate models (#902)

* feat: validate models

* fix: appointment route & user update

* fix: model

* feat: recovery script

* fix: remove comment

* feat: add comment to siret

server/src/commands.ts

@@ -166,6 +166,15 @@ program
   .action(createJobAction("fix-diffusible-companies"))
 
 program.command("check-diffusible-companies").description("Check companies are diffusible").action(createJobAction("check-diffusible-companies"))
+program
+  .command("fix:duplicate:users")
+  .description("Fix duplicated users in users collections and update appointment collection accordingly")
+  .action(createJobAction("fix:duplicate:users"))
+
+program
+  .command("migration:correctionRDVA")
+  .description("Corrige les erreurs de données ne correspondant pas aux modèles associés")
+  .action(createJobAction("migration:correctionRDVA"))
 
 program.command("db:obfuscate").description("Pseudonymisation des documents").option("-q, --queued", "Run job asynchronously", false).action(createJobAction("db:obfuscate"))
 

server/src/common/model/schema/emailBlacklist/emailBlacklist.schema.ts

@@ -1,6 +1,6 @@
-import { model, Schema } from "../../../mongodb"
+import { IEmailBlacklist } from "shared"
 
-import { IEmailBlacklist } from "./emailBlacklist.types"
+import { model, Schema } from "../../../mongodb"
 
 export const emailBlacklistSchema = new Schema<IEmailBlacklist>(
   {

server/src/http/routes/appointmentRequest.controller.ts

@@ -44,7 +44,7 @@ export default (server: Server) => {
         throw Boom.badRequest("Formation introuvable.")
       }
 
-      let user = await users.getUser(email)
+      let user = await users.getUserByMail(email)
 
       // Updates firstname and last name if the user already exists
       if (user) {
@@ -103,7 +103,7 @@ export default (server: Server) => {
       await sendFormateurAppointmentEmail(user, createdAppointement, eligibleTrainingsForAppointment, referrerObj, etablissement)
       await sendCandidateAppointmentEmail(user, createdAppointement, eligibleTrainingsForAppointment, referrerObj)
 
-      const appointmentUpdated = await Appointment.findById(createdAppointement._id)
+      const appointmentUpdated = await Appointment.findById(createdAppointement._id).lean()
 
       res.status(200).send({
         userId: user._id,

server/src/jobs/database/temp/fixRDVACollections.ts

@@ -0,0 +1,26 @@
+import dayjs from "shared/helpers/dayjs"
+
+import { EmailBlacklist, Etablissement } from "../../../common/model"
+
+// SIRET number that does not comply with LUHN algorythm in etablissements collection
+const SIRET_TO_REMOVE_FROM_ETABLISSEMENT = [
+  "13002799900132",
+  "99999999999920",
+  "78144401300000",
+  "33778063900000",
+  "78151651300000",
+  "19331424200000",
+  "19330028200000",
+  "19470020900017",
+  "52407208900175",
+]
+
+const DATE = dayjs().toDate()
+
+const removeObsoleteEtablissement = async () => await Etablissement.deleteMany({ formateur_siret: { $in: SIRET_TO_REMOVE_FROM_ETABLISSEMENT } })
+const addMissingDateFieldToEmailBlacklist = async () => await EmailBlacklist.updateMany({ created_at: { $exists: false } }, { $set: { created_at: DATE } })
+
+export const fixRDVACollections = async () => {
+  await removeObsoleteEtablissement()
+  await addMissingDateFieldToEmailBlacklist()
+}

server/src/jobs/database/validateModels.ts

@@ -5,6 +5,7 @@ import {
   ZAppointment,
   ZCredential,
   ZEligibleTrainingsForAppointmentSchema,
+  ZEmailBlacklist,
   ZEtablissement,
   ZJob,
   ZLbaCompany,
@@ -28,6 +29,7 @@ import {
   AppointmentDetailed,
   Credential,
   EligibleTrainingsForAppointment,
+  EmailBlacklist,
   Etablissement,
   FormationCatalogue,
   Job,
@@ -95,7 +97,7 @@ export async function validateModels(): Promise<void> {
   //  await validateModel(DiplomesMetiers, ZDiplomesMetiers)
   //  await validateModel(DomainesMetiers, ZDomainesMetiers)
   await validateModel(EligibleTrainingsForAppointment, ZEligibleTrainingsForAppointmentSchema)
-  // await validateModel(EmailBlacklist, ZEmailBlacklist)
+  await validateModel(EmailBlacklist, ZEmailBlacklist)
   await validateModel(Etablissement, ZEtablissement)
   await validateModel(FormationCatalogue, zFormationCatalogueSchema)
   //  await validateModel(GeoLocation, ZGeoLocation)

server/src/jobs/jobs.ts

@@ -11,6 +11,7 @@ import { cronsInit, cronsScheduler } from "./crons_actions"
 import { checkDiffusibleCompanies, fixDiffusibleCompanies } from "./database/fixDiffusibleCompanies"
 import { obfuscateCollections } from "./database/obfuscateCollections"
 import { removeVersionKeyFromAllCollections } from "./database/removeVersionKeyFromAllCollections"
+import { fixRDVACollections } from "./database/temp/fixRDVACollections"
 import { validateModels } from "./database/validateModels"
 import updateDiplomesMetiers from "./diplomesMetiers/updateDiplomesMetiers"
 import updateDomainesMetiers from "./domainesMetiers/updateDomainesMetiers"
@@ -55,6 +56,7 @@ import { inviteEtablissementToPremium } from "./rdv/inviteEtablissementToPremium
 import { inviteEtablissementAffelnetToPremium } from "./rdv/inviteEtablissementToPremiumAffelnet"
 import { inviteEtablissementToPremiumFollowUp } from "./rdv/inviteEtablissementToPremiumFollowUp"
 import { inviteEtablissementAffelnetToPremiumFollowUp } from "./rdv/inviteEtablissementToPremiumFollowUpAffelnet"
+import { fixDuplicateUsers } from "./rdv/oneTimeJob/fixDuplicateUsers"
 import { repriseEmailRdvs } from "./rdv/oneTimeJob/repriseEmailsRdv"
 import { premiumActivatedReminder } from "./rdv/premiumActivatedReminder"
 import { premiumInviteOneShot } from "./rdv/premiumInviteOneShot"
@@ -212,6 +214,10 @@ export async function runJob(job: IInternalJobsCronTask | IInternalJobsSimple):
       return CronsMap[job.name].handler()
     }
     switch (job.name) {
+      case "fix:duplicate:users": // Temporaire, doit tourner une fois en production
+        return fixDuplicateUsers()
+      case "migration:correctionRDVA": // Temporaire, doit tourner une fois en recette et production
+        return fixRDVACollections()
       case "control:applications":
         return controlApplications()
       case "control:appointments":

server/src/jobs/rdv/oneTimeJob/fixDuplicateUsers.ts

@@ -0,0 +1,39 @@
+import { IUser } from "shared"
+
+import { logger } from "../../../common/logger"
+import { Appointment, User } from "../../../common/model"
+
+const findDuplicates = (users) => {
+  const emailMap: { [key: string]: IUser[] } = {}
+
+  users.forEach((user) => {
+    const { email } = user
+
+    if (!emailMap[email]) {
+      emailMap[email] = [user]
+    } else {
+      emailMap[email].push(user)
+    }
+  })
+
+  const duplicateGroups = Object.values(emailMap)
+    .filter((group) => group.length > 1)
+    .map((group) => group.sort((a, b) => new Date(b.last_action_date).getTime() - new Date(a.last_action_date).getTime()))
+
+  return duplicateGroups
+}
+
+export const fixDuplicateUsers = async () => {
+  logger.info(`Start user deduplication`)
+  const users = await User.find({}).lean()
+  const duplicates: Iterable<IUser[]> = findDuplicates(users)
+
+  for await (const groupOfUsers of duplicates) {
+    const userToKeep = groupOfUsers.shift()
+    for await (const group of groupOfUsers) {
+      await Appointment.updateMany({ applicant_id: group._id.toString() }, { $set: { applicant_id: userToKeep?._id.toString() } })
+      await User.findByIdAndDelete(group._id)
+    }
+  }
+  logger.info(`End user deduplication`)
+}

server/src/services/user.service.ts

@@ -4,13 +4,6 @@ import { ETAT_UTILISATEUR } from "shared/constants/recruteur"
 
 import { Recruiter, User, UserRecruteur } from "../common/model/index"
 
-/**
- * @description Returns user from its username.
- * @param {string} username
- * @returns {Promise<IUser>}
- */
-const getUser = (username: string) => User.findOne({ username })
-
 /**
  * @description Returns user from its email.
  * @param {string} email
@@ -147,4 +140,4 @@ const getUserAndRecruitersDataForOpcoUser = async (opco: string): Promise<TRetur
   return results
 }
 
-export { createUser, find, findOne, getUser, getUserAndRecruitersDataForOpcoUser, getUserById, getUserByMail, update }
+export { createUser, find, findOne, getUserAndRecruitersDataForOpcoUser, getUserById, getUserByMail, update }

shared/models/elligibleTraining.model.ts

@@ -22,8 +22,8 @@ export const ZEligibleTrainingsForAppointmentSchema = z
     parcoursup_id: z.string().nullable(),
     cle_ministere_educatif: ZAppointment.shape.cle_ministere_educatif,
     etablissement_formateur_raison_sociale: z.string(),
-    etablissement_formateur_street: z.string(),
-    departement_etablissement_formateur: z.string(),
+    etablissement_formateur_street: z.string().nullable(),
+    departement_etablissement_formateur: z.string().nullable(),
     etablissement_formateur_city: ZEtablissement.shape.formateur_city,
     lieu_formation_street: z.string(),
     lieu_formation_city: z.string(),

shared/models/emailBlacklist.model.ts

@@ -0,0 +1,17 @@
+import { Jsonify } from "type-fest"
+
+import { z } from "../helpers/zodWithOpenApi"
+
+import { zObjectId } from "./common"
+
+export const ZEmailBlacklist = z
+  .object({
+    _id: zObjectId,
+    email: z.string(),
+    blacklisting_origin: z.string(),
+    created_at: z.coerce.date(),
+  })
+  .strict()
+
+export type IEmailBlacklist = z.output<typeof ZEmailBlacklist>
+export type IEmailBlacklistJson = Jsonify<z.input<typeof ZEmailBlacklist>>

shared/models/etablissement.model.ts

@@ -29,7 +29,6 @@ export const ZEtablissement = z
     optout_activation_scheduled_date: z.date().nullish(),
     optout_activation_date: z.date().nullish(),
     optout_refusal_date: z.date().nullish(),
-    mailing: z.array(ZMailing).nullish(),
     last_catalogue_sync_date: z.date().nullish(),
     created_at: z.date().nullish(),
     affelnet_perimetre: z.boolean().nullish(),

shared/models/index.ts

@@ -4,6 +4,7 @@ export * from "./appointments.model"
 export * from "./cfa.model"
 export * from "./credentials.model"
 export * from "./elligibleTraining.model"
+export * from "./emailBlacklist.model"
 export * from "./etablissement.model"
 export * from "./formation.model"
 export * from "./job.model"
@@ -19,8 +20,8 @@ export * from "./referentielOpco.model"
 export * from "./rncpRomes.model"
 export * from "./rome.model"
 export * from "./session.model"
+export * from "./siretDiffusibleStatus.model"
 export * from "./unsubscribeOF.model"
 export * from "./unsubscribedLbaCompany.model"
 export * from "./user.model"
 export * from "./usersRecruteur.model"
-export * from "./siretDiffusibleStatus.model"

shared/models/user.model.ts

@@ -6,8 +6,6 @@ import { zObjectId } from "./common"
 export const ZUser = z
   .object({
     _id: zObjectId,
-    username: z.string(),
-    password: z.string(),
     firstname: z.string(),
     lastname: z.string(),
     phone: z.string(),

shared/routes/appointments.routes.ts

@@ -266,14 +266,12 @@ export const zAppointmentsRoute = {
         })
         .strict(),
       response: {
-        // TODO ANY TO BE FIXED
-        "200": z.any(),
-        // "200": z
-        //   .object({
-        //     userId: z.string(),
-        //     appointment: z.union([ZAppointment, z.null()]),
-        //   })
-        //   .strict(),
+        "200": z
+          .object({
+            userId: zObjectId,
+            appointment: z.union([ZAppointment, z.null()]),
+          })
+          .strict(),
       },
       securityScheme: null,
     },

ui/components/RDV/DemandeDeContact.tsx

@@ -260,7 +260,7 @@ const DemandeDeContact = (props: Props) => {
                 onBlur={formik.handleBlur}
                 value={formik.values.applicantMessageToCfa}
               />
-              <FormErrorMessage>{formik.errors.lastname}</FormErrorMessage>
+              <FormErrorMessage>{formik.errors.applicantMessageToCfa}</FormErrorMessage>
             </FormControl>
           )}
         </FormControl>