diff --git a/common/models/establishmentSchema.js b/common/models/establishmentSchema.js index 6e5d9828..6ea1ae10 100644 --- a/common/models/establishmentSchema.js +++ b/common/models/establishmentSchema.js @@ -454,6 +454,17 @@ const establishmentSchema = { }, ////////////////// + opcos: { + type: [String], + default: [], + description: "Liste des opcos rattachés à l'établissement", + }, + opcos_formations: { + type: [Object], + default: {}, + description: "Liste des formations avec leurs opcos rattachés", + }, + catalogue_published: { type: Boolean, default: false, diff --git a/front/package.json b/front/package.json index 329bb1c8..c7fb8684 100644 --- a/front/package.json +++ b/front/package.json @@ -22,6 +22,7 @@ "connected-react-router": "^6.6.1", "cross-env": "^7.0.2", "formik": "^2.1.2", + "lodash": "^4.17.20", "node-less-chokidar": "^0.4.1", "npm-run-all": "^4.1.5", "react": "^16.12.0", diff --git a/front/src/pages/Etablissement/Etablissement.js b/front/src/pages/Etablissement/Etablissement.js index 02c6ca7d..cf05616e 100644 --- a/front/src/pages/Etablissement/Etablissement.js +++ b/front/src/pages/Etablissement/Etablissement.js @@ -6,13 +6,31 @@ import { useFormik } from "formik"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { faPen, faCheck } from "@fortawesome/free-solid-svg-icons"; import { push } from "connected-react-router"; +import { groupBy, map, orderBy } from "lodash"; //import Section from "./components/Section"; import routes from "../../routes.json"; import "./etablissement.css"; +import { get } from "env-var"; -const sleep = m => new Promise(r => setTimeout(r, m)); +const sleep = (m) => new Promise((r) => setTimeout(r, m)); + +const getOpcosFormatted = (opcos_formations) => { + const allOpcos = opcos_formations + .map((x) => x.opcos) + .join(",") + .split(","); + const opcosForNbFormations = orderBy( + map(groupBy(allOpcos), (val) => ({ + opcos: val[0], + nb_formations: val.length, + })), + "nb_formations", + "desc" + ); + return opcosForNbFormations; +}; const checkIfHasRightToEdit = (item, userAcm) => { let hasRightToEdit = userAcm.all; @@ -58,7 +76,7 @@ const EditSection = ({ edition, onEdit, handleSubmit }) => { }; const Etablissement = ({ etablissement, edition, onEdit, handleChange, handleSubmit, values }) => { - const { acm: userAcm } = useSelector(state => state.user); + const { acm: userAcm } = useSelector((state) => state.user); const hasRightToEdit = checkIfHasRightToEdit(etablissement, userAcm); return ( @@ -180,6 +198,24 @@ const Etablissement = ({ etablissement, edition, onEdit, handleChange, handleSub )} + {etablissement.opcos_formations.length > 0 && ( +
+
+

OPCOs

+ {/* {etablissement.opcos_formations.map((x, i) => ( +

{x.opcos}

+ ))} */} + {getOpcosFormatted(etablissement.opcos_formations).map((x, i) => ( +

+ {x.opcos} :{" "} + + {x.nb_formations} formations + +

+ ))} +
+
+ )} ); diff --git a/front/src/pages/Formation/Formation.js b/front/src/pages/Formation/Formation.js index e9dfa809..b99392c6 100644 --- a/front/src/pages/Formation/Formation.js +++ b/front/src/pages/Formation/Formation.js @@ -201,8 +201,8 @@ const Formation = ({ formation, edition, onEdit, handleChange, handleSubmit, val {formation.opcos.length > 0 && ( <>

OPCOs liés à la formation

- {formation.opcos.map((x) => ( -

{x}

+ {formation.opcos.map((x, i) => ( +

{x}

))} )} diff --git a/front/src/pages/Search/constantsEtablissements.js b/front/src/pages/Search/constantsEtablissements.js index 4ac051f1..15b1d376 100644 --- a/front/src/pages/Search/constantsEtablissements.js +++ b/front/src/pages/Search/constantsEtablissements.js @@ -1,4 +1,4 @@ -const FILTERS = ["QUERYBUILDER", "SEARCH", "num_departement", "nom_academie"]; +const FILTERS = ["QUERYBUILDER", "SEARCH", "num_departement", "nom_academie", "opcos"]; const columnsDefinition = [ { @@ -246,6 +246,12 @@ const columnsDefinition = [ exportOnly: true, editable: false, }, + { + Header: "OPCOs", + accessor: "opcos", + width: 200, + editable: false, + }, ]; const queryBuilderField = [ @@ -275,6 +281,14 @@ const facetDefinition = [ selectAllLabel: "Tous", sortBy: "asc", }, + { + componentId: "opcos", + dataField: "opcos.keyword", + title: "OPCOs", + filterLabel: "opcos", + selectAllLabel: "Tout OPCOs", + sortBy: "asc", + }, ]; const dataSearch = { diff --git a/jobs/features/opcos/src/importer/opcoImporter.js b/jobs/features/opcos/src/importer/opcoImporter.js index 111f8bef..00940153 100644 --- a/jobs/features/opcos/src/importer/opcoImporter.js +++ b/jobs/features/opcos/src/importer/opcoImporter.js @@ -1,58 +1,145 @@ const { pipeline, writeObject } = require("../../../../../common/streamUtils"); const logger = require("../../../../common-jobs/Logger").mainLogger; -const { Formation } = require("../../../../../common/models2"); +const { Formation, Establishment } = require("../../../../../common/models2"); const createReferentiel = require("../utils/referentiel"); const { infosCodes, computeCodes } = require("../utils/constants"); module.exports = async () => { - logger.info(" -- Start of OPCOs Importer -- "); - const referentiel = await createReferentiel(); + const importToTrainings = async options => { + logger.info(" -- Starting Import OPCOs to trainings -- "); + const referentiel = await createReferentiel(); - let stats = { - opcosUpdated: 0, - opcosNotFound: 0, - errors: 0, - }; + let stats = { + trainings: { + updated: 0, + noCodeEn: 0, + noIdccsFound: 0, + noOpcosFound: 0, + errors: 0, + }, + }; + + logger.info("Updating formations..."); + await pipeline( + await Formation.find({}).cursor(), + writeObject( + async f => { + try { + const overrideMode = options.override ? options.override : false; + if (f.opcos.length === 0 || overrideMode) { + if (!f.educ_nat_code) { + f.info_opcos = infosCodes.NoCodeEn; + f.info_opcos_intitule = computeCodes[infosCodes.NoCodeEn]; + stats.trainings.noCodeEn++; + } else { + const opcosForFormations = await referentiel.findOpcosFromCodeEn(f.educ_nat_code); + + if (opcosForFormations.length > 0) { + logger.info( + `Adding OPCOs ${opcosForFormations.map(x => x.Opérateurdecompétences)} for formation ${ + f._id + } for educ_nat_code ${f.educ_nat_code}` + ); + f.opcos = opcosForFormations.map(x => x.Opérateurdecompétences); + f.info_opcos = infosCodes.Found; + f.info_opcos_intitule = computeCodes[infosCodes.Found]; + stats.trainings.updated++; + } else { + logger.info(`No OPCOs found for formation ${f._id} for educ_nat_code ${f.educ_nat_code}`); - logger.info("Updating formations..."); - await pipeline( - await Formation.find({}).cursor(), - writeObject( - async f => { - try { - if (!f.educ_nat_code) { - f.info_opcos = infosCodes.NotFoundable; - f.info_opcos_intitule = computeCodes[infosCodes.NotFoundable]; - } else { - const opcosForFormations = await referentiel.findOpcosFromCodeEn(f.educ_nat_code); - - if (opcosForFormations.length > 0) { - logger.info( - `Adding OPCOs ${opcosForFormations.map(x => x.Opérateurdecompétences)} for formation ${ - f._id - } for educ_nat_code ${f.educ_nat_code}` - ); - f.opcos = opcosForFormations.map(x => x.Opérateurdecompétences); - f.info_opcos = infosCodes.Found; - f.info_opcos_intitule = computeCodes[infosCodes.Found]; - stats.opcosUpdated++; - } else { - logger.info(`No OPCOs found for formation ${f._id} for educ_nat_code ${f.educ_nat_code}`); - f.info_opcos = infosCodes.NotFound; - f.info_opcos_intitule = computeCodes[infosCodes.NotFound]; - stats.opcosNotFound++; + if ((await referentiel.findIdccsFromCodeEn(f.educ_nat_code).length) === 0) { + f.info_opcos = infosCodes.NoIdccsFound; + f.info_opcos_intitule = computeCodes[infosCodes.NoIdccsFound]; + stats.trainings.noIdccsFound++; + } else { + f.info_opcos = infosCodes.NoOpcosFound; + f.info_opcos_intitule = computeCodes[infosCodes.NoOpcosFound]; + stats.trainings.noOpcosFound++; + } + } + } + + await f.save(); } + } catch (e) { + stats.trainings.errors++; + logger.error(e); } + }, + { parallel: 5 } + ) + ); + logger.info(" -- End of Import OPCOs to trainings -- "); + return stats; + }; + + const importToEtablishments = async options => { + logger.info(" -- Starting Import OPCOs to etablishments -- "); - await f.save(); - } catch (e) { - stats.errors++; - logger.error(e); - } + let stats = { + etablishments: { + updated: 0, + notFound: 0, + errors: 0, }, - { parallel: 5 } - ) - ); - logger.info(" -- End of OPCOs Importer -- "); - return stats; + }; + + logger.info("Updating etablishments..."); + await pipeline( + await Establishment.find({ uai: { $nin: [null, ""] } }) + .batchSize(5000) + .cursor(), + writeObject( + async e => { + try { + const overrideMode = options.override ? options.override : false; + if (e.opcos.length === 0 || overrideMode) { + // Gets all formations having opcos linked to uai on etablissement_formateur_uai + const formationsOpcosForUai = await ( + await Formation.find({ + etablissement_formateur_uai: `${e.uai}`, + }) + ).filter(x => x.opcos.length > 0); + + if (formationsOpcosForUai.length === 0) { + stats.etablishments.notFound++; + logger.info(`No OPCOs found linked to etablissement ${e._id}`); + } else { + e.opcos = [...new Set(formationsOpcosForUai.flatMap(x => x.opcos))]; // Unique opcos + e.opcos_formations = formationsOpcosForUai.map(x => { + return { + formation_id: x._id, + opcos: x.opcos, + }; + }); // List of couple formation / opcos + stats.etablishments.updated++; + logger.info(`Adding OPCOs found ${e.opcos} for formations linked to etablissement ${e._id}`); + } + await e.save(); + } + } catch (e) { + stats.etablishments.errors++; + logger.error(e); + } + }, + { parallel: 5 } + ) + ); + logger.info(" -- End of Import OPCOs to etablishments -- "); + return stats; + }; + + return { + importOpcosToTrainings: importToTrainings, + importOpcosToEtablishments: importToEtablishments, + importOpcos: async options => { + logger.info(" -- Import OPCOs --"); + const importTrainingStats = await importToTrainings(options); + const importEtablishmentsStats = await importToEtablishments(options); + const stats = { ...importTrainingStats, ...importEtablishmentsStats }; + logger.info(" -- Stats -- "); + logger.info(JSON.stringify(stats)); + logger.info(" -- End of Import OPCOs --"); + }, + }; }; diff --git a/jobs/features/opcos/src/index.js b/jobs/features/opcos/src/index.js index f666bcf5..0d5e293c 100644 --- a/jobs/features/opcos/src/index.js +++ b/jobs/features/opcos/src/index.js @@ -2,8 +2,11 @@ const { execute } = require("../../../../common/scriptWrapper"); const opcoImporter = require("./importer/opcoImporter"); const run = async () => { + const importer = await opcoImporter(); await execute(() => { - return opcoImporter(); + return importer.importOpcos({ + override: process.env.OVERRIDE_MODE === "true", + }); }); }; diff --git a/jobs/features/opcos/src/stats.js b/jobs/features/opcos/src/stats.js index 2c0536f0..b5e6ba5a 100644 --- a/jobs/features/opcos/src/stats.js +++ b/jobs/features/opcos/src/stats.js @@ -1,12 +1,14 @@ const logger = require("../../../common-jobs/Logger").mainLogger; const { connectToMongo, closeMongoConnection } = require("../../../../common/mongo"); -const { Formation } = require("../../../../common/models2"); +const { Formation, Establishment } = require("../../../../common/models2"); const run = async () => { try { await connectToMongo(); logger.info(" -- Stats of OPCO Linker -- "); const formations = await Formation.find(); + + // Stats classiques - Formations const formationsAvecOpco = await Formation.find({ opcos: { $ne: [] } }); const formationsSansOpcos = await Formation.find({ opcos: [] }); const formationsSansOpcosEtSansCodeEducNat = await Formation.find({ @@ -15,11 +17,33 @@ const run = async () => { const formationsSansOpcosEtAvecCodeEducNat = await Formation.find({ $and: [{ opcos: [] }, { educ_nat_code: { $nin: [null, ""] } }], }); + + // Stats avec info_opcos + const formationsOpcosEmpty = await Formation.find({ info_opcos: 0 }); + const formationsOpcosFound = await Formation.find({ info_opcos: 1 }); + const formationsOpcosNoCodeEn = await Formation.find({ info_opcos: 2 }); + const formationsOpcosNotFoundNoIdccs = await Formation.find({ info_opcos: 3 }); + const formationsOpcosNotFoundNoOpcos = await Formation.find({ info_opcos: 4 }); + + // Stats Etablissements + const etablissements = await Establishment.find(); + const etablissementsAvecOpcos = await Establishment.find({ opcos: { $nin: [[], null] } }); + logger.info(`${formations.length} formations`); - logger.info(`${formationsAvecOpco.length} formations avec opcos`); - logger.info(`${formationsSansOpcos.length} formations sans opcos`); - logger.info(`${formationsSansOpcosEtSansCodeEducNat.length} formations sans opcos et sans educNatCode`); - logger.info(`${formationsSansOpcosEtAvecCodeEducNat.length} formations sans opcos et avec educNatCode`); + logger.info(`${formationsAvecOpco.length} formations avec OPCOs`); + logger.info(`${formationsSansOpcos.length} formations sans OPCOs`); + logger.info(`${formationsSansOpcosEtSansCodeEducNat.length} formations sans OPCOs et sans code diplome`); + logger.info(`${formationsSansOpcosEtAvecCodeEducNat.length} formations sans OPCOs et avec code diplome`); + + logger.info(`${formationsOpcosEmpty.length} formations avec OPCOs vides`); + logger.info(`${formationsOpcosFound.length} formations avec OPCOs trouvés`); + logger.info(`${formationsOpcosNoCodeEn.length} formations sans OPCOs et sans code diplome`); + logger.info(`${formationsOpcosNotFoundNoIdccs.length} formations sans OPCOs - pas de lien code diplome / IDCCs`); + logger.info(`${formationsOpcosNotFoundNoOpcos.length} formations sans OPCOs - pas de lien IDCCs / OPCOs`); + + logger.info(`${etablissements.length} établissements`); + logger.info(`${etablissementsAvecOpcos.length} établissements avec OPCOs`); + logger.info(" -- End Stats of OPCO Linker -- "); closeMongoConnection(); } catch (err) { diff --git a/jobs/features/opcos/src/utils/constants.js b/jobs/features/opcos/src/utils/constants.js index e6797b79..d5859ded 100644 --- a/jobs/features/opcos/src/utils/constants.js +++ b/jobs/features/opcos/src/utils/constants.js @@ -1,10 +1,12 @@ const infosCodes = { - NotFound: 0, + Empty: 0, Found: 1, - NotFoundable: 2, + NoCodeEn: 2, + NoIdccsFound: 3, + NoOpcosFound: 4, }; -const computeCodes = ["Non trouvés", "Trouvés", "Non trouvables"]; +const computeCodes = ["NC", "Trouvés", "Aucun code diplôme", "Non trouvés - pas d'idcc", "Non trouvés - pas d'OPCOs"]; module.exports = { infosCodes, diff --git a/jobs/features/opcos/test/integration/opco-test.js b/jobs/features/opcos/test/integration/opco-test.js deleted file mode 100644 index 7351227a..00000000 --- a/jobs/features/opcos/test/integration/opco-test.js +++ /dev/null @@ -1,10 +0,0 @@ -const assert = require("assert"); -const fs = require("fs"); -const path = require("path"); -const createReferentiel = require("../../src/utils/referentiel"); - -describe(__filename, () => { - it("Vérifie", async () => { - assert.strictEqual(true, true); - }); -}); diff --git a/jobs/test/data/fixtures.js b/jobs/test/data/fixtures.js index d8799d06..81a5dbec 100644 --- a/jobs/test/data/fixtures.js +++ b/jobs/test/data/fixtures.js @@ -71,6 +71,9 @@ module.exports = { computed_bcn_intitule: "Erreur", computed_bcn_niveau: "", computed_bcn_diplome: "", + opcos: [], + info_opcos: 0, + info_opcos_intitule: "NC", source: "TEST", commentaires: "", last_modification: "", @@ -138,6 +141,8 @@ module.exports = { computed_declare_prefecture: "OUI", computed_conventionne: "OUI", computed_info_datadock: "datadocké", + opcos: [], + opcos_formations: [], published: true, created_at: new Date(), last_update_at: new Date(),