Merge pull request #508 from mission-apprentissage/fix/dedoublonner-cs

feat: dédoublonnage + changement des demandes concernées

server/.env.test

@@ -14,7 +14,7 @@ PQSL_USER=postgres
 PSQL_PWD=password
 PSQL_URI=postgresql://postgres:[email protected]:5432/orion-test-VITEST_POOL_ID
 PSQL_CA=
-PSQL_LOG_LEVEL=query
+PSQL_LOG_LEVEL=error
 # Auth
 AUTH_JWT_SECRET=abcdef
 ACTIVATION_JWT_SECRET=abcdef

server/seed/generate-seed.sh

@@ -2,32 +2,36 @@
 
 readonly VAULT_PASSWORD_CLIENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/vault/get-vault-password-client.sh"
 
-echo "Chemin de VAULT_PASSWORD_CLIENT : $VAULT_PASSWORD_CLIENT"
+echo ""
+echo "🔀 Chemin de VAULT_PASSWORD_CLIENT : $VAULT_PASSWORD_CLIENT"
+echo ""
 
 readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly ANSIBLE_DIR="${SCRIPT_DIR}/../../.infra/ansible"
 readonly INFRA_DIR="${SCRIPT_DIR}/../../.infra/vault"
 readonly VAULT_FILE="${1:-${INFRA_DIR}/vault.yml}"
 readonly VAULT_PASSWORD_FILE="${SCRIPT_DIR}/../../.bin/scripts/get-vault-password-client.sh"
 
-echo "Lancement de la génération de la seed"
-
+echo "🚀 Lancement de la génération de la seed"
+echo ""
 # Récupérer les informations d'identification de la base de données à partir du Vault
 
 DB_URL=$(ansible-vault view $VAULT_FILE --vault-password-file "${VAULT_PASSWORD_FILE}" | grep PSQL_URI | head -n 1 | awk '{print $2}' | xargs)
 SCHEMA_DUMP_FILE="seed_schema.dump"
 DATA_DUMP_FILE="seed_data.dump"
 
-echo "DB_URL : $DB_URL"
+echo "🔑 DB_URL : $DB_URL"
 echo ""
 
 # Générer le fichier de dump
-echo "Génération de la seed contenant le schéma de la DB"
+echo "🌱 Génération de la seed contenant le schéma de la DB"
 echo ""
-pg_dump $DB_URL -Fc --clean --if-exists --create --schema-only --file $SCHEMA_DUMP_FILE
+pg_dump $DB_URL --format=custom --clean --if-exists --create --schema-only --file $SCHEMA_DUMP_FILE
 
-echo "Génération de la seed contenant les données de la DB"
-pg_dump $DB_URL -Fc --data-only \
+echo "🤖 Génération de la seed contenant les données de la DB"
+echo ""
+
+pg_dump $DB_URL --format=custom --data-only \
   --exclude-table='public."changeLog"' \
   --exclude-table='public."changementStatut"' \
   --exclude-table='public."demande"' \
@@ -40,7 +44,8 @@ pg_dump $DB_URL -Fc --data-only \
   --file $DATA_DUMP_FILE
 
 if [ $? -eq 0 ]; then
-  echo "Dump de la base de données généré avec succès"
+  echo "✅ Dump de la base de données généré avec succès"
 else
-  echo "Échec de la génération du dump de la base de données."
+  echo "❌ Échec de la génération du dump de la base de données."
 fi
+

server/src/db/schema.ts

@@ -223,7 +223,6 @@ export interface Demande {
   campagneId: string | null;
   id: Generated<string>;
   numeroHistorique: string | null;
-  autreBesoinRH: string | null;
   amiCmaValide: boolean | null;
   amiCmaValideAnnee: string | null;
   recrutementRH: boolean | null;
@@ -242,6 +241,7 @@ export interface Demande {
   nbFormationRH: number | null;
   discipline1FormationRH: string | null;
   discipline2FormationRH: string | null;
+  autreBesoinRH: string | null;
   amiCmaEnCoursValidation: boolean | null;
   updatedBy: string | null;
   capaciteScolaireColoreeActuelle: Generated<number | null>;
@@ -405,8 +405,8 @@ export interface FormationEtablissement {
 }
 
 export interface FormationHistorique {
-  cfd: string;
   ancienCFD: string;
+  cfd: string;
   voie: string;
 }
 
@@ -444,8 +444,8 @@ export interface IndicateurEntree {
   formationEtablissementId: string;
   rentreeScolaire: string;
   effectifs: Json | null;
-  capacites: Json | null;
   anneeDebut: number | null;
+  capacites: Json | null;
   premiersVoeux: Json | null;
 }
 
@@ -686,7 +686,6 @@ export interface LatestDemandeNonMaterializedView {
   campagneId: string | null;
   id: string | null;
   numeroHistorique: string | null;
-  autreBesoinRH: string | null;
   amiCmaValide: boolean | null;
   amiCmaValideAnnee: string | null;
   recrutementRH: boolean | null;
@@ -705,6 +704,7 @@ export interface LatestDemandeNonMaterializedView {
   nbFormationRH: number | null;
   discipline1FormationRH: string | null;
   discipline2FormationRH: string | null;
+  autreBesoinRH: string | null;
   amiCmaEnCoursValidation: boolean | null;
   updatedBy: string | null;
   capaciteScolaireColoreeActuelle: number | null;
@@ -819,7 +819,7 @@ export interface PositionFormationRegionaleQuadrant {
 export interface RawData {
   type: string;
   data: Json | null;
-  id: Generated<string>;
+  id: Generated<string | null>;
 }
 
 export interface Region {

server/src/migrations/index.ts

@@ -97,6 +97,7 @@ import * as migration_1731666807035 from "./migration_1731666807035";
 import * as migration_1733743141284 from "./migration_1733743141284";
 import * as migration_1734527074984 from "./migration_1734527074984";
 import * as migration_1734527078957 from "./migration_1734527078957";
+import * as migration_1734618351550 from "./migration_1734618351550";
 
 type Migration = {
   up: (db: Kysely<any>) => Promise<void>;
@@ -201,4 +202,5 @@ export const migrations: Migrations = {
   migration_1733743141284,
   migration_1734527074984,
   migration_1734527078957,
+  migration_1734618351550,
 };

server/src/migrations/migration_1734618351550.ts

@@ -0,0 +1,162 @@
+import type { Kysely } from "kysely";
+import { sql } from "kysely";
+
+import { getKbdClient } from "@/db/db";
+
+const correspondancesCFD: Record<string, string> = {
+  "56122101": "56122103",
+  "56122102": "56122105",
+  "56124201": "56124201",
+  "46122701": "46122703",
+  "46131101": "46131101",
+  "56134201": "56134201",
+  "46125001": "46125002",
+  "46125002": "46125003",
+  "46122001": "46122001",
+  "46131201": "46131201",
+  "46133503": "46133503",
+  "46132601": "46132601",
+  "46133504": "46133504",
+  "46133501": "46133501",
+  "46133502": "46133502",
+  "46133505": "46133505",
+  "46125101": "46125122",
+  "56122106": "56122112",
+  "46122301": "46122306",
+  "46123301": "46123304",
+  "46125502": "46125509",
+  "56122103": "56122107",
+  "56122105": "56122109",
+  "56122107": "56122111",
+  "56122301": "56122304",
+  "56122302": "56122307",
+  "56122401": "56122405",
+  "56122701": "56122706",
+  "56123201": "56123204",
+  "56123301": "56123307",
+  "56125101": "56125118",
+  "56133001": "56133002",
+  "56133401": "56133411",
+  "56133601": "56133605",
+  "46122702": "46122704",
+  "46123302": "46123306",
+  "46125003": "46125004",
+  "46125102": "46125123",
+  "46125301": "46125308",
+  "46125302": "46125309",
+  "46125303": "46125310",
+  "46125304": "46125311",
+  "46125305": "46125312",
+  "46125401": "46125406",
+  "46125402": "46125407",
+  "46125403": "46125408",
+  "46122703": "46122705",
+  "46125404": "46125409",
+  "46125501": "46125508",
+  "46133401": "46133412",
+  "56122104": "56122113",
+  "46125503": "46125510",
+  "46133403": "46133414",
+  "46133402": "46133413",
+  "56123302": "56123308",
+  "56123303": "56123309",
+  "56122109": "56122114",
+  "56123202": "56123205",
+};
+
+const listeDesCfdsNonReutilises = [
+  "56122101",
+  "56122102",
+  "46122701",
+  "46125001",
+  "46125101",
+  "56122106",
+  "46122301",
+  "46123301",
+  "46125502",
+  "56122301",
+  "56122302",
+  "56122401",
+  "56122701",
+  "56123201",
+  "56123301",
+  "56125101",
+  "56133001",
+  "56133401",
+  "56133601",
+  "46122702",
+  "46123302",
+  "46125102",
+  "46125301",
+  "46125302",
+  "46125303",
+  "46125304",
+  "46125305",
+  "46125401",
+  "46125402",
+  "46125403",
+  "46125404",
+  "46125501",
+  "46133401",
+  "56122104",
+  "46125503",
+  "46133403",
+  "46133402",
+  "56123302",
+  "56123303",
+  "56123202",
+];
+
+export const up = async (db: Kysely<unknown>) => {
+  let sqlQuery = "";
+
+  Object.entries(correspondancesCFD).map(async ([ancienCFD, nouveauCFD]) => {
+    sqlQuery += `UPDATE demande SET cfd = '${nouveauCFD}' WHERE cfd = '${ancienCFD}' AND id in (SELECT id FROM "latestDemandeView");\n`;
+    sqlQuery += `UPDATE intention SET cfd = '${nouveauCFD}' WHERE cfd = '${ancienCFD}' AND id in (SELECT id FROM "latestIntentionView");\n`;
+  });
+
+  // disable les triggers qui utilisent trop de mémoire sur des opérations massives
+  await getKbdClient().executeQuery(
+    sql`
+    ALTER TABLE demande DISABLE TRIGGER ALL;
+    ALTER TABLE intention DISABLE TRIGGER ALL;
+    `.compile(db)
+  );
+  await getKbdClient().executeQuery(sql.raw(sqlQuery).compile(db));
+  // enable les triggers
+  await getKbdClient().executeQuery(
+    sql`
+    ALTER TABLE demande ENABLE TRIGGER ALL;
+    ALTER TABLE intention ENABLE TRIGGER ALL;
+    `.compile(db)
+  );
+
+  await getKbdClient().executeQuery(
+    sql`
+    REFRESH MATERIALIZED VIEW "latestDemandeView" WITH DATA;
+    REFRESH MATERIALIZED VIEW "latestIntentionView" WITH DATA;
+    REFRESH MATERIALIZED VIEW "demandeIntentionView" WITH DATA;
+    REFRESH MATERIALIZED VIEW "latestDemandeIntentionView" WITH DATA;
+    `.compile(db)
+  );
+
+  await getKbdClient().deleteFrom("formationRome").where("cfd", "in", listeDesCfdsNonReutilises).execute();
+  await getKbdClient().deleteFrom("dataFormation").where("cfd", "in", listeDesCfdsNonReutilises).execute();
+
+  await getKbdClient().executeQuery(
+    sql`
+    REFRESH MATERIALIZED VIEW "formationView" WITH DATA;
+    REFRESH MATERIALIZED VIEW "formationScolaireView" WITH DATA;
+    REFRESH MATERIALIZED VIEW "formationApprentissageView" WITH DATA;
+    `.compile(db)
+  );
+};
+
+export const down = async (_db: Kysely<unknown>) => {};
+
+// codes temporaires
+// ('56122103','56122105','56124201','46122703','46131101','56134201','46125002','46125003','46122001','46131201','46133503','46132601','46133504','46133501','46133502','46133505','46125122','56122112','46122306','46123304','46125509','56122107','56122109','56122111','56122304','56122307','56122405','56122706','56123204','56123307','56125118','56133002','56133411','56133605','46122704','46123306','46125004','46125123','46125308','46125309','46125310','46125311','46125312','46125406','46125407','46125408','46122705','46125409','46125508','46133412','56122113','46125510','46133414','46133413','56123308','56123309','56122114','56123205')
+// codes actuels
+// ('56122101','56122102','56124201','46122701','46131101','56134201','46125001','46125002','46122001','46131201','46133503','46132601','46133504','46133501','46133502','46133505','46125101','56122106','46122301','46123301','46125502','56122103','56122105','56122107','56122301','56122302','56122401','56122701','56123201','56123301','56125101','56133001','56133401','56133601','46122702','46123302','46125003','46125102','46125301','46125302','46125303','46125304','46125305','46125401','46125402','46125403','46122703','46125404','46125501','46133401','56122104','46125503','46133403','46133402','56123302','56123303','56122109','56123202')
+// codes à supprimer
+// ('56122101','56122102','46122701','46125001','46125101','56122106','46122301','46123301','46125502','56122301','56122302','56122401','56122701','56123201','56123301','56125101','56133001','56133401','56133601','46122702','46123302','46125102','46125301','46125302','46125303','46125304','46125305','46125401','46125402','46125403','46125404','46125501','46133401','56122104','46125503','46133403','46133402','56123302','56123303','56123202')

server/tests/pg.test.utils.ts

@@ -22,6 +22,8 @@ export const startAndConnectPg = async () => {
 
   await connectToPgDb(dbUri);
 
+  await refreshViews();
+
   await migrateToLatest(true, false);
   console.log("Migration terminée");
   console.log("Rafraichissement des vues matérialisées");