[WIP] Ajout de la gestion des liens opcos / établissements

common/models/establishmentSchema.js

@@ -454,6 +454,17 @@ const establishmentSchema = {
   },
   //////////////////
 
+  opcos: {
+    type: [String],
+    default: [],
+    description: "Liste des opcos rattachés à l'établissement",
+  },
+  opcos_formations: {
+    type: [Object],
+    default: {},
+    description: "Liste des formations avec leurs opcos rattachés",
+  },
+
   catalogue_published: {
     type: Boolean,
     default: false,

front/package.json

@@ -22,6 +22,7 @@
     "connected-react-router": "^6.6.1",
     "cross-env": "^7.0.2",
     "formik": "^2.1.2",
+    "lodash": "^4.17.20",
     "node-less-chokidar": "^0.4.1",
     "npm-run-all": "^4.1.5",
     "react": "^16.12.0",

front/src/pages/Etablissement/Etablissement.js

@@ -6,13 +6,31 @@ import { useFormik } from "formik";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import { faPen, faCheck } from "@fortawesome/free-solid-svg-icons";
 import { push } from "connected-react-router";
+import { groupBy, map, orderBy } from "lodash";
 
 //import Section from "./components/Section";
 import routes from "../../routes.json";
 
 import "./etablissement.css";
+import { get } from "env-var";
 
-const sleep = m => new Promise(r => setTimeout(r, m));
+const sleep = (m) => new Promise((r) => setTimeout(r, m));
+
+const getOpcosFormatted = (opcos_formations) => {
+  const allOpcos = opcos_formations
+    .map((x) => x.opcos)
+    .join(",")
+    .split(",");
+  const opcosForNbFormations = orderBy(
+    map(groupBy(allOpcos), (val) => ({
+      opcos: val[0],
+      nb_formations: val.length,
+    })),
+    "nb_formations",
+    "desc"
+  );
+  return opcosForNbFormations;
+};
 
 const checkIfHasRightToEdit = (item, userAcm) => {
   let hasRightToEdit = userAcm.all;
@@ -58,7 +76,7 @@ const EditSection = ({ edition, onEdit, handleSubmit }) => {
 };
 
 const Etablissement = ({ etablissement, edition, onEdit, handleChange, handleSubmit, values }) => {
-  const { acm: userAcm } = useSelector(state => state.user);
+  const { acm: userAcm } = useSelector((state) => state.user);
   const hasRightToEdit = checkIfHasRightToEdit(etablissement, userAcm);
 
   return (
@@ -180,6 +198,24 @@ const Etablissement = ({ etablissement, edition, onEdit, handleChange, handleSub
             </div>
           </div>
         )}
+        {etablissement.opcos_formations.length > 0 && (
+          <div className="sidebar-section info">
+            <div className="field">
+              <h2>OPCOs</h2>
+              {/* {etablissement.opcos_formations.map((x, i) => (
+                <p key={i}>{x.opcos}</p>
+              ))} */}
+              {getOpcosFormatted(etablissement.opcos_formations).map((x, i) => (
+                <p key={i}>
+                  {x.opcos} :{" "}
+                  <b>
+                    <i>{x.nb_formations} formations</i>
+                  </b>
+                </p>
+              ))}
+            </div>
+          </div>
+        )}
       </Col>
     </Row>
   );

front/src/pages/Formation/Formation.js

@@ -201,8 +201,8 @@ const Formation = ({ formation, edition, onEdit, handleChange, handleSubmit, val
             {formation.opcos.length > 0 && (
               <>
                 <h3>OPCOs liés à la formation</h3>
-                {formation.opcos.map((x) => (
-                  <p>{x}</p>
+                {formation.opcos.map((x, i) => (
+                  <p key={i}>{x}</p>
                 ))}
               </>
             )}

front/src/pages/Search/constantsEtablissements.js

@@ -1,4 +1,4 @@
-const FILTERS = ["QUERYBUILDER", "SEARCH", "num_departement", "nom_academie"];
+const FILTERS = ["QUERYBUILDER", "SEARCH", "num_departement", "nom_academie", "opcos"];
 
 const columnsDefinition = [
   {
@@ -246,6 +246,12 @@ const columnsDefinition = [
     exportOnly: true,
     editable: false,
   },
+  {
+    Header: "OPCOs",
+    accessor: "opcos",
+    width: 200,
+    editable: false,
+  },
 ];
 
 const queryBuilderField = [
@@ -275,6 +281,14 @@ const facetDefinition = [
     selectAllLabel: "Tous",
     sortBy: "asc",
   },
+  {
+    componentId: "opcos",
+    dataField: "opcos.keyword",
+    title: "OPCOs",
+    filterLabel: "opcos",
+    selectAllLabel: "Tout OPCOs",
+    sortBy: "asc",
+  },
 ];
 
 const dataSearch = {

jobs/features/opcos/src/importer/opcoImporter.js

@@ -1,58 +1,145 @@
 const { pipeline, writeObject } = require("../../../../../common/streamUtils");
 const logger = require("../../../../common-jobs/Logger").mainLogger;
-const { Formation } = require("../../../../../common/models2");
+const { Formation, Establishment } = require("../../../../../common/models2");
 const createReferentiel = require("../utils/referentiel");
 const { infosCodes, computeCodes } = require("../utils/constants");
 
 module.exports = async () => {
-  logger.info(" -- Start of OPCOs Importer -- ");
-  const referentiel = await createReferentiel();
+  const importToTrainings = async options => {
+    logger.info(" -- Starting Import OPCOs to trainings -- ");
+    const referentiel = await createReferentiel();
 
-  let stats = {
-    opcosUpdated: 0,
-    opcosNotFound: 0,
-    errors: 0,
-  };
+    let stats = {
+      trainings: {
+        updated: 0,
+        noCodeEn: 0,
+        noIdccsFound: 0,
+        noOpcosFound: 0,
+        errors: 0,
+      },
+    };
+
+    logger.info("Updating formations...");
+    await pipeline(
+      await Formation.find({}).cursor(),
+      writeObject(
+        async f => {
+          try {
+            const overrideMode = options.override ? options.override : false;
+            if (f.opcos.length === 0 || overrideMode) {
+              if (!f.educ_nat_code) {
+                f.info_opcos = infosCodes.NoCodeEn;
+                f.info_opcos_intitule = computeCodes[infosCodes.NoCodeEn];
+                stats.trainings.noCodeEn++;
+              } else {
+                const opcosForFormations = await referentiel.findOpcosFromCodeEn(f.educ_nat_code);
+
+                if (opcosForFormations.length > 0) {
+                  logger.info(
+                    `Adding OPCOs ${opcosForFormations.map(x => x.Opérateurdecompétences)} for formation ${
+                      f._id
+                    } for educ_nat_code ${f.educ_nat_code}`
+                  );
+                  f.opcos = opcosForFormations.map(x => x.Opérateurdecompétences);
+                  f.info_opcos = infosCodes.Found;
+                  f.info_opcos_intitule = computeCodes[infosCodes.Found];
+                  stats.trainings.updated++;
+                } else {
+                  logger.info(`No OPCOs found for formation ${f._id} for educ_nat_code ${f.educ_nat_code}`);
 
-  logger.info("Updating formations...");
-  await pipeline(
-    await Formation.find({}).cursor(),
-    writeObject(
-      async f => {
-        try {
-          if (!f.educ_nat_code) {
-            f.info_opcos = infosCodes.NotFoundable;
-            f.info_opcos_intitule = computeCodes[infosCodes.NotFoundable];
-          } else {
-            const opcosForFormations = await referentiel.findOpcosFromCodeEn(f.educ_nat_code);
-
-            if (opcosForFormations.length > 0) {
-              logger.info(
-                `Adding OPCOs ${opcosForFormations.map(x => x.Opérateurdecompétences)} for formation ${
-                  f._id
-                } for educ_nat_code ${f.educ_nat_code}`
-              );
-              f.opcos = opcosForFormations.map(x => x.Opérateurdecompétences);
-              f.info_opcos = infosCodes.Found;
-              f.info_opcos_intitule = computeCodes[infosCodes.Found];
-              stats.opcosUpdated++;
-            } else {
-              logger.info(`No OPCOs found for formation ${f._id} for educ_nat_code ${f.educ_nat_code}`);
-              f.info_opcos = infosCodes.NotFound;
-              f.info_opcos_intitule = computeCodes[infosCodes.NotFound];
-              stats.opcosNotFound++;
+                  if ((await referentiel.findIdccsFromCodeEn(f.educ_nat_code).length) === 0) {
+                    f.info_opcos = infosCodes.NoIdccsFound;
+                    f.info_opcos_intitule = computeCodes[infosCodes.NoIdccsFound];
+                    stats.trainings.noIdccsFound++;
+                  } else {
+                    f.info_opcos = infosCodes.NoOpcosFound;
+                    f.info_opcos_intitule = computeCodes[infosCodes.NoOpcosFound];
+                    stats.trainings.noOpcosFound++;
+                  }
+                }
+              }
+
+              await f.save();
             }
+          } catch (e) {
+            stats.trainings.errors++;
+            logger.error(e);
           }
+        },
+        { parallel: 5 }
+      )
+    );
+    logger.info(" -- End of Import OPCOs to trainings -- ");
+    return stats;
+  };
+
+  const importToEtablishments = async options => {
+    logger.info(" -- Starting Import OPCOs to etablishments -- ");
 
-          await f.save();
-        } catch (e) {
-          stats.errors++;
-          logger.error(e);
-        }
+    let stats = {
+      etablishments: {
+        updated: 0,
+        notFound: 0,
+        errors: 0,
       },
-      { parallel: 5 }
-    )
-  );
-  logger.info(" -- End of OPCOs Importer -- ");
-  return stats;
+    };
+
+    logger.info("Updating etablishments...");
+    await pipeline(
+      await Establishment.find({ uai: { $nin: [null, ""] } })
+        .batchSize(5000)
+        .cursor(),
+      writeObject(
+        async e => {
+          try {
+            const overrideMode = options.override ? options.override : false;
+            if (e.opcos.length === 0 || overrideMode) {
+              // Gets all formations having opcos linked to uai on etablissement_formateur_uai
+              const formationsOpcosForUai = await (
+                await Formation.find({
+                  etablissement_formateur_uai: `${e.uai}`,
+                })
+              ).filter(x => x.opcos.length > 0);
+
+              if (formationsOpcosForUai.length === 0) {
+                stats.etablishments.notFound++;
+                logger.info(`No OPCOs found linked to etablissement ${e._id}`);
+              } else {
+                e.opcos = [...new Set(formationsOpcosForUai.flatMap(x => x.opcos))]; // Unique opcos
+                e.opcos_formations = formationsOpcosForUai.map(x => {
+                  return {
+                    formation_id: x._id,
+                    opcos: x.opcos,
+                  };
+                }); // List of couple formation / opcos
+                stats.etablishments.updated++;
+                logger.info(`Adding OPCOs found ${e.opcos} for formations linked to etablissement ${e._id}`);
+              }
+              await e.save();
+            }
+          } catch (e) {
+            stats.etablishments.errors++;
+            logger.error(e);
+          }
+        },
+        { parallel: 5 }
+      )
+    );
+    logger.info(" -- End of Import OPCOs to etablishments -- ");
+    return stats;
+  };
+
+  return {
+    importOpcosToTrainings: importToTrainings,
+    importOpcosToEtablishments: importToEtablishments,
+    importOpcos: async options => {
+      logger.info(" -- Import OPCOs --");
+      const importTrainingStats = await importToTrainings(options);
+      const importEtablishmentsStats = await importToEtablishments(options);
+      const stats = { ...importTrainingStats, ...importEtablishmentsStats };
+      logger.info(" -- Stats -- ");
+      logger.info(JSON.stringify(stats));
+      logger.info(" -- End of Import OPCOs --");
+    },
+  };
 };

jobs/features/opcos/src/index.js

@@ -2,8 +2,11 @@ const { execute } = require("../../../../common/scriptWrapper");
 const opcoImporter = require("./importer/opcoImporter");
 
 const run = async () => {
+  const importer = await opcoImporter();
   await execute(() => {
-    return opcoImporter();
+    return importer.importOpcos({
+      override: process.env.OVERRIDE_MODE === "true",
+    });
   });
 };
 

jobs/features/opcos/src/stats.js

@@ -1,12 +1,14 @@
 const logger = require("../../../common-jobs/Logger").mainLogger;
 const { connectToMongo, closeMongoConnection } = require("../../../../common/mongo");
-const { Formation } = require("../../../../common/models2");
+const { Formation, Establishment } = require("../../../../common/models2");
 
 const run = async () => {
   try {
     await connectToMongo();
     logger.info(" -- Stats of OPCO Linker -- ");
     const formations = await Formation.find();
+
+    // Stats classiques - Formations
     const formationsAvecOpco = await Formation.find({ opcos: { $ne: [] } });
     const formationsSansOpcos = await Formation.find({ opcos: [] });
     const formationsSansOpcosEtSansCodeEducNat = await Formation.find({
@@ -15,11 +17,33 @@ const run = async () => {
     const formationsSansOpcosEtAvecCodeEducNat = await Formation.find({
       $and: [{ opcos: [] }, { educ_nat_code: { $nin: [null, ""] } }],
     });
+
+    // Stats avec info_opcos
+    const formationsOpcosEmpty = await Formation.find({ info_opcos: 0 });
+    const formationsOpcosFound = await Formation.find({ info_opcos: 1 });
+    const formationsOpcosNoCodeEn = await Formation.find({ info_opcos: 2 });
+    const formationsOpcosNotFoundNoIdccs = await Formation.find({ info_opcos: 3 });
+    const formationsOpcosNotFoundNoOpcos = await Formation.find({ info_opcos: 4 });
+
+    // Stats Etablissements
+    const etablissements = await Establishment.find();
+    const etablissementsAvecOpcos = await Establishment.find({ opcos: { $nin: [[], null] } });
+
     logger.info(`${formations.length} formations`);
-    logger.info(`${formationsAvecOpco.length} formations avec opcos`);
-    logger.info(`${formationsSansOpcos.length} formations sans opcos`);
-    logger.info(`${formationsSansOpcosEtSansCodeEducNat.length} formations sans opcos et sans educNatCode`);
-    logger.info(`${formationsSansOpcosEtAvecCodeEducNat.length} formations sans opcos et avec educNatCode`);
+    logger.info(`${formationsAvecOpco.length} formations avec OPCOs`);
+    logger.info(`${formationsSansOpcos.length} formations sans OPCOs`);
+    logger.info(`${formationsSansOpcosEtSansCodeEducNat.length} formations sans OPCOs et sans code diplome`);
+    logger.info(`${formationsSansOpcosEtAvecCodeEducNat.length} formations sans OPCOs et avec code diplome`);
+
+    logger.info(`${formationsOpcosEmpty.length} formations avec OPCOs vides`);
+    logger.info(`${formationsOpcosFound.length} formations avec OPCOs trouvés`);
+    logger.info(`${formationsOpcosNoCodeEn.length} formations sans OPCOs et sans code diplome`);
+    logger.info(`${formationsOpcosNotFoundNoIdccs.length} formations sans OPCOs - pas de lien code diplome / IDCCs`);
+    logger.info(`${formationsOpcosNotFoundNoOpcos.length} formations sans OPCOs - pas de lien IDCCs / OPCOs`);
+
+    logger.info(`${etablissements.length} établissements`);
+    logger.info(`${etablissementsAvecOpcos.length} établissements avec OPCOs`);
+
     logger.info(" -- End Stats of OPCO Linker -- ");
     closeMongoConnection();
   } catch (err) {