feat: Ajout du continuum pour les diplomes 1<=>1 (#206)

feat: ajout d'une route bcn
feat: ajout des stats manquantes (continuum) pour les anciens/nouveaux diplomes (1 pour 1)

.husky/pre-commit

@@ -6,7 +6,7 @@ set -eu
 function validate_commit() {
   cd .
   local sensible_files_pattern="\.(csv|xls|xls(x?)|json|env)$"
-  local exception="package.json|tsconfig.json$"
+  local exception="package.json|tsconfig.json|bcn/n_mef.csv$"
   local files
   files=$(git diff --cached --name-only | grep -v -E "$exception" | grep -E "$sensible_files_pattern") || true
 

docker-compose.override.yml

@@ -8,13 +8,15 @@ services:
 
   ui:
     build:
-      args:
-        TRAJECTOIRES_PRO_ENV: dev
+      dockerfile: ./Dockerfile.dev
     mem_limit: 1g
     stdin_open: true
-
+    volumes:
+      - trajectoires_pro_ui_data:/data:z
+      - /app/node_modules
+      - ./ui:/app:z
   server:
-    command: [ "yarn", "debug" ]
+    command: ["yarn", "debug"]
     ports:
       - "127.0.0.1:9229:9229" # Allows remote debugging
     volumes:
@@ -47,6 +49,9 @@ volumes:
   trajectoires_pro_server_data:
     driver: local
     name: trajectoires_pro_server_data
+  trajectoires_pro_ui_data:
+    driver: local
+    name: trajectoires_pro_ui_data
   trajectoires_pro_metabase_data:
     driver: local
     name: trajectoires_pro_metabase_data

server/migrations/20230802092319-ajout_de_champs_dans_la_collection_BCN.js

@@ -0,0 +1,45 @@
+import { omit, without } from "lodash-es";
+import { arrayOf, date, string } from "../src/common/db/collections/jsonSchema/jsonSchemaTypes.js";
+import * as MongoDB from "../src/common/db/mongodb.js";
+
+const schema = {
+  properties: {
+    libelle_long: string(),
+    date_ouverture: date(),
+    date_premiere_session: string(),
+    date_derniere_session: string(),
+    ancien_diplome: arrayOf(string()),
+    nouveau_diplome: arrayOf(string()),
+  },
+  required: ["libelle_long"],
+};
+
+export const up = async (db, client) => {
+  MongoDB.setMongoDBClient(client);
+  return MongoDB.mergeSchema("bcn", schema);
+};
+
+export const down = async (db, client) => {
+  const collectionInfos = await db.listCollections({ name: "bcn" }).toArray();
+  const validator = collectionInfos[0].options.validator;
+
+  if (!validator) {
+    return;
+  }
+
+  const oldSchema = validator.$jsonSchema;
+  const newSchema = {
+    ...oldSchema,
+    properties: omit(oldSchema.properties, Object.keys(schema.properties)),
+    required: without(oldSchema.required, ...schema.required),
+  };
+
+  return db.command({
+    collMod: "bcn",
+    validationLevel: "strict",
+    validationAction: "error",
+    validator: {
+      $jsonSchema: newSchema,
+    },
+  });
+};

server/migrations/20230807122559-ajout_des_champs_pour_le_continuum_dans_la_collection_BCN.js

@@ -0,0 +1,54 @@
+import { omit, without } from "lodash-es";
+import { object, string, enumOf } from "../src/common/db/collections/jsonSchema/jsonSchemaTypes.js";
+import * as MongoDB from "../src/common/db/mongodb.js";
+
+const schema = {
+  properties: {
+    donnee_source: object(
+      {
+        code_certification: string(),
+        type: enumOf(["self", "ancienne", "nouvelle"]),
+      },
+      { required: ["code_certification", "type"] }
+    ),
+  },
+  required: ["donnee_source"],
+};
+
+export const up = async (db, client) => {
+  MongoDB.setMongoDBClient(client);
+  return Promise.all([
+    MongoDB.mergeSchema("certificationsStats", schema),
+    MongoDB.mergeSchema("formationsStats", schema),
+    MongoDB.mergeSchema("regionalesStats", schema),
+  ]);
+};
+
+export const down = async (db, client) => {
+  return Promise.all(
+    ["certificationsStats", "formationsStats", "regionalesStats"].map(async (collection) => {
+      const collectionInfos = await db.listCollections({ name: collection }).toArray();
+      const validator = collectionInfos[0].options.validator;
+
+      if (!validator) {
+        return;
+      }
+
+      const oldSchema = validator.$jsonSchema;
+      const newSchema = {
+        ...oldSchema,
+        properties: omit(oldSchema.properties, Object.keys(schema.properties)),
+        required: without(oldSchema.required || [], ...schema.required),
+      };
+
+      return db.command({
+        collMod: collection,
+        validationLevel: "strict",
+        validationAction: "error",
+        validator: {
+          $jsonSchema: newSchema,
+        },
+      });
+    })
+  );
+};

server/migrations/20230809131650-ajout_de_la_collection_bcn_mef.js

@@ -0,0 +1,91 @@
+import { logger } from "../src/common/logger.js";
+import { object, objectId, string, date } from "../src/common/db/collections/jsonSchema/jsonSchemaTypes.js";
+
+const name = "bcn_mef";
+
+function indexes() {
+  return [[{ mef_stat_11: 1 }, { unique: true }], [{ mef: 1 }]];
+}
+
+function schema() {
+  return object(
+    {
+      _id: objectId(),
+      mef_stat_11: string(),
+      mef: string(),
+      dispositif_formation: string(),
+      formation_diplome: string(),
+      duree_dispositif: string(),
+      annee_dispositif: string(),
+
+      libelle_court: string(),
+      libelle_long: string(),
+
+      date_ouverture: date(),
+      date_fermeture: date(),
+
+      statut_mef: string(),
+      nb_option_obligatoire: string(),
+      nb_option_facultatif: string(),
+      renforcement_langue: string(),
+      duree_projet: string(),
+      duree_stage: string(),
+      horaire: string(),
+      mef_inscription_scolarite: string(),
+      mef_stat_9: string(),
+
+      date_intervention: date(),
+      libelle_edition: string(),
+      commentaire: string(),
+      _meta: object(
+        {
+          created_on: date(),
+          updated_on: date(),
+          date_import: date(),
+        },
+        { required: ["date_import"] }
+      ),
+    },
+    {
+      required: [
+        "mef",
+        "dispositif_formation",
+        "formation_diplome",
+        "duree_dispositif",
+        "annee_dispositif",
+        "mef_stat_11",
+        "libelle_court",
+        "libelle_long",
+        "_meta",
+      ],
+      additionalProperties: true,
+    }
+  );
+}
+
+export const up = async (db, client) => {
+  await db.createCollection(name);
+
+  logger.debug(`Configuring indexes for collection ${name}...`);
+  let dbCol = db.collection(name);
+
+  await Promise.all(
+    indexes().map(([index, options]) => {
+      return dbCol.createIndex(index, options);
+    })
+  );
+
+  logger.debug(`Configuring validation for collection ${name}...`);
+  await db.command({
+    collMod: name,
+    validationLevel: "strict",
+    validationAction: "error",
+    validator: {
+      $jsonSchema: schema(),
+    },
+  });
+};
+
+export const down = async (db, client) => {
+  // We do not remove the collection to avoid deleting data by error
+};

server/package.json

@@ -11,7 +11,7 @@
   "scripts": {
     "start": "migrate-mongo up && nodemon --ignore tests/ src/index.js",
     "debug": "nodemon --inspect=0.0.0.0 --signal SIGINT --ignore tests/ src/index.js",
-    "test": "TRAJECTOIRES_PRO_LOG_LEVEL=fatal mocha --inline-diffs --exit tests/**/*-test.js",
+    "test": "TRAJECTOIRES_PRO_LOG_LEVEL=fatal mocha --inline-diffs --exit tests/**/*-test.js tests/*-test.js",
     "cli": "TRAJECTOIRES_PRO_LOG_DESTINATIONS=stderr node src/cli.js",
     "coverage": "nyc --temp-dir .coverage/.nyc_output --report-dir .coverage --reporter=lcov --reporter=html yarn test",
     "lint": "eslint src/ tests/",
@@ -42,14 +42,17 @@
     "luxon": "2.3.2",
     "migrate-mongo": "10.0.0",
     "mockdate": "3.0.5",
+    "moment": "2.29.4",
+    "moment-timezone": "0.5.43",
     "mongodb": "4.5.0",
-    "oleoduc": "0.7.0",
+    "oleoduc": "0.8.0",
     "passport": "0.5.2",
     "passport-headerapikey": "1.2.2",
     "pretty-ms": "7.0.1",
     "rate-limiter-flexible": "2.3.7",
     "sinon": "15.0.2",
     "stream-json": "1.7.4",
+    "stream-to-array": "2.3.0",
     "swagger-ui-express": "4.3.0",
     "uuid": "8.3.2",
     "yamljs": "0.3.0"

server/src/cli.js

@@ -5,16 +5,25 @@ import { runScript } from "./common/runScript.js";
 import { writeToStdout } from "oleoduc";
 import { exportCodeCertifications } from "./jobs/exportCodeCertifications.js";
 import { importBCN } from "./jobs/importBCN.js";
+import { importBCNMEF } from "./jobs/importBCNMEF.js";
+import { importBCNContinuum } from "./jobs/importBCNContinuum.js";
+import { computeBCNMEFContinuum } from "./jobs/computeBCNMEFContinuum.js";
 import { importStats } from "./jobs/importStats.js";
 import { backfillMetrics } from "./jobs/backfillMetrics.js";
 import { asArray } from "./common/utils/stringUtils.js";
+import { computeContinuumStats } from "./jobs/computeContinuumStats.js";
 
 cli
   .command("importBCN")
   .description("Import les CFD et MEF depuis la BCN")
   .action(() => {
-    runScript(() => {
-      return importBCN();
+    runScript(async () => {
+      const statsBCN = await importBCN();
+      const statsMef = await importBCNMEF();
+      const statsContinuum = await importBCNContinuum();
+      const statsMefContinuum = await computeBCNMEFContinuum();
+
+      return { statsBCN, statsMef, statsContinuum, statsMefContinuum };
     });
   });
 
@@ -28,6 +37,16 @@ cli
     });
   });
 
+cli
+  .command("computeContinuumStats")
+  .description("Calcule les données statistiques manquantes pour les anciens/nouveaux diplomes")
+  .argument("[stats]", "Le nom des stats à importer (formations,certifications,regionales)", asArray)
+  .action((stats) => {
+    runScript(() => {
+      return computeContinuumStats({ stats });
+    });
+  });
+
 cli
   .command("exportCodeCertifications")
   .description("Permet de savoir si des codes certifications sont fermés")

server/src/common/bcn.js

@@ -2,8 +2,6 @@ import { compose, transformData, writeData, oleoduc } from "oleoduc";
 import { fetchStream } from "./utils/httpUtils.js";
 import iconv from "iconv-lite";
 import { parseCsv } from "./utils/csvUtils.js";
-import { bcn } from "./db/collections/collections.js";
-import { castArray } from "lodash-es";
 
 const ANCIENS_NIVEAUX_MAPPER = {
   5: "3", // CAP
@@ -50,18 +48,6 @@ export async function getNiveauxDiplome(options) {
   return niveauxDiplome;
 }
 
-export async function findCodeFormationDiplome(codeCertification) {
-  const found = await bcn().findOne({ code_certification: { $in: castArray(codeCertification) } });
-  return found?.code_formation_diplome;
-}
-
-export async function findCodesFormationDiplome(codeCertification) {
-  const found = await bcn()
-    .find({ code_certification: { $in: castArray(codeCertification) } })
-    .toArray();
-  return found.map((f) => f.code_formation_diplome);
-}
-
 export async function getBCNTable(tableName, options = {}) {
   let stream =
     options[tableName] ||

server/src/common/db/collections/bcn.js

@@ -1,4 +1,4 @@
-import { date, object, objectId, string } from "./jsonSchema/jsonSchemaTypes.js";
+import { date, object, objectId, string, arrayOf } from "./jsonSchema/jsonSchemaTypes.js";
 import { diplomeSchema } from "./jsonSchema/diplomeSchema.js";
 import { metaSchema, metaBCNSchema } from "./jsonSchema/metaSchema.js";
 
@@ -17,11 +17,17 @@ export function schema() {
       code_formation_diplome: string(),
       date_fermeture: date(),
       libelle: string(),
+      libelle_long: string(),
       diplome: diplomeSchema(),
       _meta: metaSchema([metaBCNSchema()]),
+      date_ouverture: date(),
+      date_premiere_session: string(),
+      date_derniere_session: string(),
+      nouveau_diplome: arrayOf(string()),
+      ancien_diplome: arrayOf(string()),
     },
     {
-      required: ["type", "code_certification", "code_formation_diplome", "libelle", "_meta"],
+      required: ["type", "code_certification", "code_formation_diplome", "libelle", "libelle_long", "_meta"],
       additionalProperties: true,
     }
   );