From aad91f55a6888ff5e341d628bba2f5553e1b0b84 Mon Sep 17 00:00:00 2001 From: Samir Date: Mon, 17 Aug 2020 13:57:43 +0200 Subject: [PATCH 1/6] =?UTF-8?q?Ajout=20de=20la=20gestion=20des=20liens=20o?= =?UTF-8?q?pcos=20/=20=C3=A9tablissements?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/models/establishmentSchema.js | 11 ++ .../opcos/src/importer/opcoImporter.js | 154 +++++++++++++----- jobs/features/opcos/src/index.js | 3 +- jobs/features/opcos/src/stats.js | 19 ++- .../opcos/test/integration/opco-test.js | 3 - 5 files changed, 137 insertions(+), 53 deletions(-) diff --git a/common/models/establishmentSchema.js b/common/models/establishmentSchema.js index 6e5d9828..6ea1ae10 100644 --- a/common/models/establishmentSchema.js +++ b/common/models/establishmentSchema.js @@ -454,6 +454,17 @@ const establishmentSchema = { }, ////////////////// + opcos: { + type: [String], + default: [], + description: "Liste des opcos rattachés à l'établissement", + }, + opcos_formations: { + type: [Object], + default: {}, + description: "Liste des formations avec leurs opcos rattachés", + }, + catalogue_published: { type: Boolean, default: false, diff --git a/jobs/features/opcos/src/importer/opcoImporter.js b/jobs/features/opcos/src/importer/opcoImporter.js index 111f8bef..923b4908 100644 --- a/jobs/features/opcos/src/importer/opcoImporter.js +++ b/jobs/features/opcos/src/importer/opcoImporter.js @@ -1,58 +1,124 @@ const { pipeline, writeObject } = require("../../../../../common/streamUtils"); const logger = require("../../../../common-jobs/Logger").mainLogger; -const { Formation } = require("../../../../../common/models2"); +const { Formation, Establishment } = require("../../../../../common/models2"); const createReferentiel = require("../utils/referentiel"); const { infosCodes, computeCodes } = require("../utils/constants"); +const { map, groupBy, orderBy } = require("lodash"); module.exports = async () => { - logger.info(" -- Start of OPCOs Importer -- "); - const referentiel = await createReferentiel(); + const importToTrainings = async () => { + logger.info(" -- Starting Import OPCOs to trainings -- "); + const referentiel = await createReferentiel(); - let stats = { - opcosUpdated: 0, - opcosNotFound: 0, - errors: 0, + let stats = { + opcosForTrainingUpdated: 0, + opcosForTrainingNotFound: 0, + opcosForTrainingErrors: 0, + }; + + logger.info("Updating formations..."); + await pipeline( + await Formation.find({}).cursor(), + writeObject( + async f => { + try { + if (!f.educ_nat_code) { + f.info_opcos = infosCodes.NotFoundable; + f.info_opcos_intitule = computeCodes[infosCodes.NotFoundable]; + } else { + const opcosForFormations = await referentiel.findOpcosFromCodeEn(f.educ_nat_code); + + if (opcosForFormations.length > 0) { + logger.info( + `Adding OPCOs ${opcosForFormations.map(x => x.Opérateurdecompétences)} for formation ${ + f._id + } for educ_nat_code ${f.educ_nat_code}` + ); + f.opcos = opcosForFormations.map(x => x.Opérateurdecompétences); + f.info_opcos = infosCodes.Found; + f.info_opcos_intitule = computeCodes[infosCodes.Found]; + stats.opcosForTrainingUpdated++; + } else { + logger.info(`No OPCOs found for formation ${f._id} for educ_nat_code ${f.educ_nat_code}`); + f.info_opcos = infosCodes.NotFound; + f.info_opcos_intitule = computeCodes[infosCodes.NotFound]; + stats.opcosForTrainingNotFound++; + } + } + + await f.save(); + } catch (e) { + stats.opcosForTrainingErrors++; + logger.error(e); + } + }, + { parallel: 5 } + ) + ); + logger.info(" -- End of Import OPCOs to trainings -- "); + return stats; }; - logger.info("Updating formations..."); - await pipeline( - await Formation.find({}).cursor(), - writeObject( - async f => { - try { - if (!f.educ_nat_code) { - f.info_opcos = infosCodes.NotFoundable; - f.info_opcos_intitule = computeCodes[infosCodes.NotFoundable]; - } else { - const opcosForFormations = await referentiel.findOpcosFromCodeEn(f.educ_nat_code); - - if (opcosForFormations.length > 0) { - logger.info( - `Adding OPCOs ${opcosForFormations.map(x => x.Opérateurdecompétences)} for formation ${ - f._id - } for educ_nat_code ${f.educ_nat_code}` - ); - f.opcos = opcosForFormations.map(x => x.Opérateurdecompétences); - f.info_opcos = infosCodes.Found; - f.info_opcos_intitule = computeCodes[infosCodes.Found]; - stats.opcosUpdated++; + const importToEtablishments = async () => { + logger.info(" -- Starting Import OPCOs to etablishments -- "); + + let stats = { + opcosForEtablishmentUpdated: 0, + opcosForEtablishmentNotFound: 0, + opcosForEtablishmentError: 0, + }; + + logger.info("Updating etablishments..."); + await pipeline( + await Establishment.find({ uai: { $nin: [null, ""] } }).cursor(), + writeObject( + async e => { + try { + // Gets all formations having opcos linked to uai on etablissement_formateur_uai + const formationsOpcosForUai = await ( + await Formation.find({ + etablissement_formateur_uai: `${e.uai}`, + }) + ).filter(x => x.opcos.length > 0); + + if (formationsOpcosForUai.length === 0) { + stats.opcosForEtablishmentNotFound++; + logger.info(`No OPCOs found linked to etablissement ${e._id}`); } else { - logger.info(`No OPCOs found for formation ${f._id} for educ_nat_code ${f.educ_nat_code}`); - f.info_opcos = infosCodes.NotFound; - f.info_opcos_intitule = computeCodes[infosCodes.NotFound]; - stats.opcosNotFound++; + e.opcos = [...new Set(formationsOpcosForUai.flatMap(x => x.opcos))]; // Unique opcos + e.opcos_formations = formationsOpcosForUai.map(x => { + return { + formation_id: x._id, + opcos: x.opcos, + }; + }); // List of couple formation / opcos + stats.opcosForEtablishmentUpdated++; + logger.info(`Adding OPCOs found ${e.opcos} for formations linked to etablissement ${e._id}`); } + await e.save(); + } catch (e) { + stats.opcosForEtablishmentError++; + logger.error(e); } + }, + { parallel: 5 } + ) + ); + logger.info(" -- End of Import OPCOs to etablishments -- "); + return stats; + }; - await f.save(); - } catch (e) { - stats.errors++; - logger.error(e); - } - }, - { parallel: 5 } - ) - ); - logger.info(" -- End of OPCOs Importer -- "); - return stats; + return { + importOpcosToTrainings: importToTrainings, + importOpcosToEtablishments: importToEtablishments, + importOpcos: async () => { + logger.info(" -- Import OPCOs --"); + const importTrainingStats = await importToTrainings(); + const importEtablishmentsStats = await importToEtablishments(); + const stats = { ...importTrainingStats, ...importEtablishmentsStats }; + logger.info(" -- Stats -- "); + logger.info(stats); + logger.info(" -- End of Import OPCOs --"); + }, + }; }; diff --git a/jobs/features/opcos/src/index.js b/jobs/features/opcos/src/index.js index f666bcf5..e409d914 100644 --- a/jobs/features/opcos/src/index.js +++ b/jobs/features/opcos/src/index.js @@ -2,8 +2,9 @@ const { execute } = require("../../../../common/scriptWrapper"); const opcoImporter = require("./importer/opcoImporter"); const run = async () => { + const importer = await opcoImporter(); await execute(() => { - return opcoImporter(); + return importer.importOpcos(); }); }; diff --git a/jobs/features/opcos/src/stats.js b/jobs/features/opcos/src/stats.js index 2c0536f0..0ea9a3f9 100644 --- a/jobs/features/opcos/src/stats.js +++ b/jobs/features/opcos/src/stats.js @@ -1,12 +1,13 @@ const logger = require("../../../common-jobs/Logger").mainLogger; const { connectToMongo, closeMongoConnection } = require("../../../../common/mongo"); -const { Formation } = require("../../../../common/models2"); +const { Formation, Establishment } = require("../../../../common/models2"); const run = async () => { try { await connectToMongo(); logger.info(" -- Stats of OPCO Linker -- "); const formations = await Formation.find(); + const formationsAvecOpco = await Formation.find({ opcos: { $ne: [] } }); const formationsSansOpcos = await Formation.find({ opcos: [] }); const formationsSansOpcosEtSansCodeEducNat = await Formation.find({ @@ -15,11 +16,19 @@ const run = async () => { const formationsSansOpcosEtAvecCodeEducNat = await Formation.find({ $and: [{ opcos: [] }, { educ_nat_code: { $nin: [null, ""] } }], }); + + const etablissements = await Establishment.find(); + const etablissementsAvecOpcos = await Establishment.find({ opcos: { $ne: [] } }); + logger.info(`${formations.length} formations`); - logger.info(`${formationsAvecOpco.length} formations avec opcos`); - logger.info(`${formationsSansOpcos.length} formations sans opcos`); - logger.info(`${formationsSansOpcosEtSansCodeEducNat.length} formations sans opcos et sans educNatCode`); - logger.info(`${formationsSansOpcosEtAvecCodeEducNat.length} formations sans opcos et avec educNatCode`); + logger.info(`${formationsAvecOpco.length} formations avec OPCOs`); + logger.info(`${formationsSansOpcos.length} formations sans OPCOs`); + logger.info(`${formationsSansOpcosEtSansCodeEducNat.length} formations sans OPCOs et sans educNatCode`); + logger.info(`${formationsSansOpcosEtAvecCodeEducNat.length} formations sans OPCOs et avec educNatCode`); + + logger.info(`${etablissements.length} établissements`); + logger.info(`${etablissementsAvecOpcos.length} établissements avec OPCOs`); + logger.info(" -- End Stats of OPCO Linker -- "); closeMongoConnection(); } catch (err) { diff --git a/jobs/features/opcos/test/integration/opco-test.js b/jobs/features/opcos/test/integration/opco-test.js index 7351227a..92fa40c4 100644 --- a/jobs/features/opcos/test/integration/opco-test.js +++ b/jobs/features/opcos/test/integration/opco-test.js @@ -1,7 +1,4 @@ const assert = require("assert"); -const fs = require("fs"); -const path = require("path"); -const createReferentiel = require("../../src/utils/referentiel"); describe(__filename, () => { it("Vérifie", async () => { From cdc92dda751188a5e4e3534bce419f14f89e4254 Mon Sep 17 00:00:00 2001 From: Samir Date: Mon, 17 Aug 2020 14:01:21 +0200 Subject: [PATCH 2/6] Remove lodash --- jobs/features/opcos/src/importer/opcoImporter.js | 1 - 1 file changed, 1 deletion(-) diff --git a/jobs/features/opcos/src/importer/opcoImporter.js b/jobs/features/opcos/src/importer/opcoImporter.js index 923b4908..320ba56e 100644 --- a/jobs/features/opcos/src/importer/opcoImporter.js +++ b/jobs/features/opcos/src/importer/opcoImporter.js @@ -3,7 +3,6 @@ const logger = require("../../../../common-jobs/Logger").mainLogger; const { Formation, Establishment } = require("../../../../../common/models2"); const createReferentiel = require("../utils/referentiel"); const { infosCodes, computeCodes } = require("../utils/constants"); -const { map, groupBy, orderBy } = require("lodash"); module.exports = async () => { const importToTrainings = async () => { From ce1ac633c0ae9ef2937a3cd89a9f267a82b3536f Mon Sep 17 00:00:00 2001 From: Samir Date: Tue, 18 Aug 2020 09:58:53 +0200 Subject: [PATCH 3/6] Update info_opcos on training management --- .../opcos/src/importer/opcoImporter.js | 122 ++++++++++-------- jobs/features/opcos/src/utils/constants.js | 9 +- 2 files changed, 75 insertions(+), 56 deletions(-) diff --git a/jobs/features/opcos/src/importer/opcoImporter.js b/jobs/features/opcos/src/importer/opcoImporter.js index 320ba56e..1c51f099 100644 --- a/jobs/features/opcos/src/importer/opcoImporter.js +++ b/jobs/features/opcos/src/importer/opcoImporter.js @@ -5,14 +5,16 @@ const createReferentiel = require("../utils/referentiel"); const { infosCodes, computeCodes } = require("../utils/constants"); module.exports = async () => { - const importToTrainings = async () => { + const importToTrainings = async options => { logger.info(" -- Starting Import OPCOs to trainings -- "); const referentiel = await createReferentiel(); let stats = { - opcosForTrainingUpdated: 0, - opcosForTrainingNotFound: 0, - opcosForTrainingErrors: 0, + updated: 0, + noCodeEn: 0, + noIdccsFound: 0, + noOpcosFound: 0, + errors: 0, }; logger.info("Updating formations..."); @@ -21,33 +23,44 @@ module.exports = async () => { writeObject( async f => { try { - if (!f.educ_nat_code) { - f.info_opcos = infosCodes.NotFoundable; - f.info_opcos_intitule = computeCodes[infosCodes.NotFoundable]; - } else { - const opcosForFormations = await referentiel.findOpcosFromCodeEn(f.educ_nat_code); - - if (opcosForFormations.length > 0) { - logger.info( - `Adding OPCOs ${opcosForFormations.map(x => x.Opérateurdecompétences)} for formation ${ - f._id - } for educ_nat_code ${f.educ_nat_code}` - ); - f.opcos = opcosForFormations.map(x => x.Opérateurdecompétences); - f.info_opcos = infosCodes.Found; - f.info_opcos_intitule = computeCodes[infosCodes.Found]; - stats.opcosForTrainingUpdated++; + const overrideMode = options.override ? options.override : false; + if (f.opcos.length === 0 || overrideMode) { + if (!f.educ_nat_code) { + f.info_opcos = infosCodes.NoCodeEn; + f.info_opcos_intitule = computeCodes[infosCodes.NoCodeEn]; + stats.noCodeEn++; } else { - logger.info(`No OPCOs found for formation ${f._id} for educ_nat_code ${f.educ_nat_code}`); - f.info_opcos = infosCodes.NotFound; - f.info_opcos_intitule = computeCodes[infosCodes.NotFound]; - stats.opcosForTrainingNotFound++; + const opcosForFormations = await referentiel.findOpcosFromCodeEn(f.educ_nat_code); + + if (opcosForFormations.length > 0) { + logger.info( + `Adding OPCOs ${opcosForFormations.map(x => x.Opérateurdecompétences)} for formation ${ + f._id + } for educ_nat_code ${f.educ_nat_code}` + ); + f.opcos = opcosForFormations.map(x => x.Opérateurdecompétences); + f.info_opcos = infosCodes.Found; + f.info_opcos_intitule = computeCodes[infosCodes.Found]; + stats.updated++; + } else { + logger.info(`No OPCOs found for formation ${f._id} for educ_nat_code ${f.educ_nat_code}`); + + if ((await referentiel.findIdccsFromCodeEn(f.educ_nat_code).length) === 0) { + f.info_opcos = infosCodes.NoIdccsFound; + f.info_opcos_intitule = computeCodes[infosCodes.NoIdccsFound]; + stats.noIdccsFound++; + } else { + f.info_opcos = infosCodes.NoOpcosFound; + f.info_opcos_intitule = computeCodes[infosCodes.NoOpcosFound]; + stats.noOpcosFound++; + } + } } - } - await f.save(); + await f.save(); + } } catch (e) { - stats.opcosForTrainingErrors++; + stats.errors++; logger.error(e); } }, @@ -58,7 +71,7 @@ module.exports = async () => { return stats; }; - const importToEtablishments = async () => { + const importToEtablishments = async options => { logger.info(" -- Starting Import OPCOs to etablishments -- "); let stats = { @@ -69,32 +82,37 @@ module.exports = async () => { logger.info("Updating etablishments..."); await pipeline( - await Establishment.find({ uai: { $nin: [null, ""] } }).cursor(), + await Establishment.find({ uai: { $nin: [null, ""] } }) + .batchSize(5000) + .cursor(), writeObject( async e => { try { - // Gets all formations having opcos linked to uai on etablissement_formateur_uai - const formationsOpcosForUai = await ( - await Formation.find({ - etablissement_formateur_uai: `${e.uai}`, - }) - ).filter(x => x.opcos.length > 0); + const overrideMode = options.override ? options.override : false; + if (e.opcos.length === 0 || overrideMode) { + // Gets all formations having opcos linked to uai on etablissement_formateur_uai + const formationsOpcosForUai = await ( + await Formation.find({ + etablissement_formateur_uai: `${e.uai}`, + }) + ).filter(x => x.opcos.length > 0); - if (formationsOpcosForUai.length === 0) { - stats.opcosForEtablishmentNotFound++; - logger.info(`No OPCOs found linked to etablissement ${e._id}`); - } else { - e.opcos = [...new Set(formationsOpcosForUai.flatMap(x => x.opcos))]; // Unique opcos - e.opcos_formations = formationsOpcosForUai.map(x => { - return { - formation_id: x._id, - opcos: x.opcos, - }; - }); // List of couple formation / opcos - stats.opcosForEtablishmentUpdated++; - logger.info(`Adding OPCOs found ${e.opcos} for formations linked to etablissement ${e._id}`); + if (formationsOpcosForUai.length === 0) { + stats.opcosForEtablishmentNotFound++; + logger.info(`No OPCOs found linked to etablissement ${e._id}`); + } else { + e.opcos = [...new Set(formationsOpcosForUai.flatMap(x => x.opcos))]; // Unique opcos + e.opcos_formations = formationsOpcosForUai.map(x => { + return { + formation_id: x._id, + opcos: x.opcos, + }; + }); // List of couple formation / opcos + stats.opcosForEtablishmentUpdated++; + logger.info(`Adding OPCOs found ${e.opcos} for formations linked to etablissement ${e._id}`); + } + await e.save(); } - await e.save(); } catch (e) { stats.opcosForEtablishmentError++; logger.error(e); @@ -110,10 +128,10 @@ module.exports = async () => { return { importOpcosToTrainings: importToTrainings, importOpcosToEtablishments: importToEtablishments, - importOpcos: async () => { + importOpcos: async options => { logger.info(" -- Import OPCOs --"); - const importTrainingStats = await importToTrainings(); - const importEtablishmentsStats = await importToEtablishments(); + const importTrainingStats = await importToTrainings(options); + const importEtablishmentsStats = await importToEtablishments(options); const stats = { ...importTrainingStats, ...importEtablishmentsStats }; logger.info(" -- Stats -- "); logger.info(stats); diff --git a/jobs/features/opcos/src/utils/constants.js b/jobs/features/opcos/src/utils/constants.js index e6797b79..588b7719 100644 --- a/jobs/features/opcos/src/utils/constants.js +++ b/jobs/features/opcos/src/utils/constants.js @@ -1,10 +1,11 @@ const infosCodes = { - NotFound: 0, - Found: 1, - NotFoundable: 2, + Found: 0, + NoCodeEn: 1, + NoIdccsFound: 2, + NoOpcosFound: 2, }; -const computeCodes = ["Non trouvés", "Trouvés", "Non trouvables"]; +const computeCodes = ["Trouvés", "Aucun code diplôme", "Non trouvés - pas d'idcc", "Non trouvés - pas d'OPCOs"]; module.exports = { infosCodes, From b2e0d663ddb8e90fbad9125b3ba7ee9e7d172466 Mon Sep 17 00:00:00 2001 From: Samir Date: Tue, 18 Aug 2020 11:03:47 +0200 Subject: [PATCH 4/6] Fixes & updates --- .../opcos/src/importer/opcoImporter.js | 36 ++++++++++--------- jobs/features/opcos/src/index.js | 4 ++- jobs/features/opcos/src/stats.js | 21 +++++++++-- jobs/features/opcos/src/utils/constants.js | 11 +++--- .../opcos/test/integration/opco-test.js | 7 ---- jobs/test/data/fixtures.js | 5 +++ 6 files changed, 52 insertions(+), 32 deletions(-) delete mode 100644 jobs/features/opcos/test/integration/opco-test.js diff --git a/jobs/features/opcos/src/importer/opcoImporter.js b/jobs/features/opcos/src/importer/opcoImporter.js index 1c51f099..876ba79a 100644 --- a/jobs/features/opcos/src/importer/opcoImporter.js +++ b/jobs/features/opcos/src/importer/opcoImporter.js @@ -10,11 +10,13 @@ module.exports = async () => { const referentiel = await createReferentiel(); let stats = { - updated: 0, - noCodeEn: 0, - noIdccsFound: 0, - noOpcosFound: 0, - errors: 0, + trainings: { + updated: 0, + noCodeEn: 0, + noIdccsFound: 0, + noOpcosFound: 0, + errors: 0, + }, }; logger.info("Updating formations..."); @@ -28,7 +30,7 @@ module.exports = async () => { if (!f.educ_nat_code) { f.info_opcos = infosCodes.NoCodeEn; f.info_opcos_intitule = computeCodes[infosCodes.NoCodeEn]; - stats.noCodeEn++; + stats.trainings.noCodeEn++; } else { const opcosForFormations = await referentiel.findOpcosFromCodeEn(f.educ_nat_code); @@ -41,18 +43,18 @@ module.exports = async () => { f.opcos = opcosForFormations.map(x => x.Opérateurdecompétences); f.info_opcos = infosCodes.Found; f.info_opcos_intitule = computeCodes[infosCodes.Found]; - stats.updated++; + stats.trainings.updated++; } else { logger.info(`No OPCOs found for formation ${f._id} for educ_nat_code ${f.educ_nat_code}`); if ((await referentiel.findIdccsFromCodeEn(f.educ_nat_code).length) === 0) { f.info_opcos = infosCodes.NoIdccsFound; f.info_opcos_intitule = computeCodes[infosCodes.NoIdccsFound]; - stats.noIdccsFound++; + stats.trainings.noIdccsFound++; } else { f.info_opcos = infosCodes.NoOpcosFound; f.info_opcos_intitule = computeCodes[infosCodes.NoOpcosFound]; - stats.noOpcosFound++; + stats.trainings.noOpcosFound++; } } } @@ -60,7 +62,7 @@ module.exports = async () => { await f.save(); } } catch (e) { - stats.errors++; + stats.trainings.errors++; logger.error(e); } }, @@ -75,9 +77,11 @@ module.exports = async () => { logger.info(" -- Starting Import OPCOs to etablishments -- "); let stats = { - opcosForEtablishmentUpdated: 0, - opcosForEtablishmentNotFound: 0, - opcosForEtablishmentError: 0, + etablishments: { + updated: 0, + notFound: 0, + errors: 0, + }, }; logger.info("Updating etablishments..."); @@ -98,7 +102,7 @@ module.exports = async () => { ).filter(x => x.opcos.length > 0); if (formationsOpcosForUai.length === 0) { - stats.opcosForEtablishmentNotFound++; + stats.etablishments.notFound++; logger.info(`No OPCOs found linked to etablissement ${e._id}`); } else { e.opcos = [...new Set(formationsOpcosForUai.flatMap(x => x.opcos))]; // Unique opcos @@ -108,13 +112,13 @@ module.exports = async () => { opcos: x.opcos, }; }); // List of couple formation / opcos - stats.opcosForEtablishmentUpdated++; + stats.etablishments.updated++; logger.info(`Adding OPCOs found ${e.opcos} for formations linked to etablissement ${e._id}`); } await e.save(); } } catch (e) { - stats.opcosForEtablishmentError++; + stats.etablishments.errors++; logger.error(e); } }, diff --git a/jobs/features/opcos/src/index.js b/jobs/features/opcos/src/index.js index e409d914..3db24cc9 100644 --- a/jobs/features/opcos/src/index.js +++ b/jobs/features/opcos/src/index.js @@ -4,7 +4,9 @@ const opcoImporter = require("./importer/opcoImporter"); const run = async () => { const importer = await opcoImporter(); await execute(() => { - return importer.importOpcos(); + return importer.importOpcosToEtablishments({ + override: process.env.OVERRIDE_MODE === "true", + }); }); }; diff --git a/jobs/features/opcos/src/stats.js b/jobs/features/opcos/src/stats.js index 0ea9a3f9..b5e6ba5a 100644 --- a/jobs/features/opcos/src/stats.js +++ b/jobs/features/opcos/src/stats.js @@ -8,6 +8,7 @@ const run = async () => { logger.info(" -- Stats of OPCO Linker -- "); const formations = await Formation.find(); + // Stats classiques - Formations const formationsAvecOpco = await Formation.find({ opcos: { $ne: [] } }); const formationsSansOpcos = await Formation.find({ opcos: [] }); const formationsSansOpcosEtSansCodeEducNat = await Formation.find({ @@ -17,14 +18,28 @@ const run = async () => { $and: [{ opcos: [] }, { educ_nat_code: { $nin: [null, ""] } }], }); + // Stats avec info_opcos + const formationsOpcosEmpty = await Formation.find({ info_opcos: 0 }); + const formationsOpcosFound = await Formation.find({ info_opcos: 1 }); + const formationsOpcosNoCodeEn = await Formation.find({ info_opcos: 2 }); + const formationsOpcosNotFoundNoIdccs = await Formation.find({ info_opcos: 3 }); + const formationsOpcosNotFoundNoOpcos = await Formation.find({ info_opcos: 4 }); + + // Stats Etablissements const etablissements = await Establishment.find(); - const etablissementsAvecOpcos = await Establishment.find({ opcos: { $ne: [] } }); + const etablissementsAvecOpcos = await Establishment.find({ opcos: { $nin: [[], null] } }); logger.info(`${formations.length} formations`); logger.info(`${formationsAvecOpco.length} formations avec OPCOs`); logger.info(`${formationsSansOpcos.length} formations sans OPCOs`); - logger.info(`${formationsSansOpcosEtSansCodeEducNat.length} formations sans OPCOs et sans educNatCode`); - logger.info(`${formationsSansOpcosEtAvecCodeEducNat.length} formations sans OPCOs et avec educNatCode`); + logger.info(`${formationsSansOpcosEtSansCodeEducNat.length} formations sans OPCOs et sans code diplome`); + logger.info(`${formationsSansOpcosEtAvecCodeEducNat.length} formations sans OPCOs et avec code diplome`); + + logger.info(`${formationsOpcosEmpty.length} formations avec OPCOs vides`); + logger.info(`${formationsOpcosFound.length} formations avec OPCOs trouvés`); + logger.info(`${formationsOpcosNoCodeEn.length} formations sans OPCOs et sans code diplome`); + logger.info(`${formationsOpcosNotFoundNoIdccs.length} formations sans OPCOs - pas de lien code diplome / IDCCs`); + logger.info(`${formationsOpcosNotFoundNoOpcos.length} formations sans OPCOs - pas de lien IDCCs / OPCOs`); logger.info(`${etablissements.length} établissements`); logger.info(`${etablissementsAvecOpcos.length} établissements avec OPCOs`); diff --git a/jobs/features/opcos/src/utils/constants.js b/jobs/features/opcos/src/utils/constants.js index 588b7719..d5859ded 100644 --- a/jobs/features/opcos/src/utils/constants.js +++ b/jobs/features/opcos/src/utils/constants.js @@ -1,11 +1,12 @@ const infosCodes = { - Found: 0, - NoCodeEn: 1, - NoIdccsFound: 2, - NoOpcosFound: 2, + Empty: 0, + Found: 1, + NoCodeEn: 2, + NoIdccsFound: 3, + NoOpcosFound: 4, }; -const computeCodes = ["Trouvés", "Aucun code diplôme", "Non trouvés - pas d'idcc", "Non trouvés - pas d'OPCOs"]; +const computeCodes = ["NC", "Trouvés", "Aucun code diplôme", "Non trouvés - pas d'idcc", "Non trouvés - pas d'OPCOs"]; module.exports = { infosCodes, diff --git a/jobs/features/opcos/test/integration/opco-test.js b/jobs/features/opcos/test/integration/opco-test.js deleted file mode 100644 index 92fa40c4..00000000 --- a/jobs/features/opcos/test/integration/opco-test.js +++ /dev/null @@ -1,7 +0,0 @@ -const assert = require("assert"); - -describe(__filename, () => { - it("Vérifie", async () => { - assert.strictEqual(true, true); - }); -}); diff --git a/jobs/test/data/fixtures.js b/jobs/test/data/fixtures.js index d8799d06..81a5dbec 100644 --- a/jobs/test/data/fixtures.js +++ b/jobs/test/data/fixtures.js @@ -71,6 +71,9 @@ module.exports = { computed_bcn_intitule: "Erreur", computed_bcn_niveau: "", computed_bcn_diplome: "", + opcos: [], + info_opcos: 0, + info_opcos_intitule: "NC", source: "TEST", commentaires: "", last_modification: "", @@ -138,6 +141,8 @@ module.exports = { computed_declare_prefecture: "OUI", computed_conventionne: "OUI", computed_info_datadock: "datadocké", + opcos: [], + opcos_formations: [], published: true, created_at: new Date(), last_update_at: new Date(), From 4a9fceb3b811e48f7c0f4446ab4542c18b37d609 Mon Sep 17 00:00:00 2001 From: Samir Date: Tue, 18 Aug 2020 15:44:03 +0200 Subject: [PATCH 5/6] Add opcos to Etablishments front page --- front/package.json | 1 + .../src/pages/Etablissement/Etablissement.js | 40 ++++++++++++++++++- front/src/pages/Formation/Formation.js | 4 +- .../opcos/src/importer/opcoImporter.js | 2 +- jobs/features/opcos/src/index.js | 2 +- 5 files changed, 43 insertions(+), 6 deletions(-) diff --git a/front/package.json b/front/package.json index 329bb1c8..c7fb8684 100644 --- a/front/package.json +++ b/front/package.json @@ -22,6 +22,7 @@ "connected-react-router": "^6.6.1", "cross-env": "^7.0.2", "formik": "^2.1.2", + "lodash": "^4.17.20", "node-less-chokidar": "^0.4.1", "npm-run-all": "^4.1.5", "react": "^16.12.0", diff --git a/front/src/pages/Etablissement/Etablissement.js b/front/src/pages/Etablissement/Etablissement.js index 02c6ca7d..cf05616e 100644 --- a/front/src/pages/Etablissement/Etablissement.js +++ b/front/src/pages/Etablissement/Etablissement.js @@ -6,13 +6,31 @@ import { useFormik } from "formik"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { faPen, faCheck } from "@fortawesome/free-solid-svg-icons"; import { push } from "connected-react-router"; +import { groupBy, map, orderBy } from "lodash"; //import Section from "./components/Section"; import routes from "../../routes.json"; import "./etablissement.css"; +import { get } from "env-var"; -const sleep = m => new Promise(r => setTimeout(r, m)); +const sleep = (m) => new Promise((r) => setTimeout(r, m)); + +const getOpcosFormatted = (opcos_formations) => { + const allOpcos = opcos_formations + .map((x) => x.opcos) + .join(",") + .split(","); + const opcosForNbFormations = orderBy( + map(groupBy(allOpcos), (val) => ({ + opcos: val[0], + nb_formations: val.length, + })), + "nb_formations", + "desc" + ); + return opcosForNbFormations; +}; const checkIfHasRightToEdit = (item, userAcm) => { let hasRightToEdit = userAcm.all; @@ -58,7 +76,7 @@ const EditSection = ({ edition, onEdit, handleSubmit }) => { }; const Etablissement = ({ etablissement, edition, onEdit, handleChange, handleSubmit, values }) => { - const { acm: userAcm } = useSelector(state => state.user); + const { acm: userAcm } = useSelector((state) => state.user); const hasRightToEdit = checkIfHasRightToEdit(etablissement, userAcm); return ( @@ -180,6 +198,24 @@ const Etablissement = ({ etablissement, edition, onEdit, handleChange, handleSub )} + {etablissement.opcos_formations.length > 0 && ( +
+
+

OPCOs

+ {/* {etablissement.opcos_formations.map((x, i) => ( +

{x.opcos}

+ ))} */} + {getOpcosFormatted(etablissement.opcos_formations).map((x, i) => ( +

+ {x.opcos} :{" "} + + {x.nb_formations} formations + +

+ ))} +
+
+ )} ); diff --git a/front/src/pages/Formation/Formation.js b/front/src/pages/Formation/Formation.js index e9dfa809..b99392c6 100644 --- a/front/src/pages/Formation/Formation.js +++ b/front/src/pages/Formation/Formation.js @@ -201,8 +201,8 @@ const Formation = ({ formation, edition, onEdit, handleChange, handleSubmit, val {formation.opcos.length > 0 && ( <>

OPCOs liés à la formation

- {formation.opcos.map((x) => ( -

{x}

+ {formation.opcos.map((x, i) => ( +

{x}

))} )} diff --git a/jobs/features/opcos/src/importer/opcoImporter.js b/jobs/features/opcos/src/importer/opcoImporter.js index 876ba79a..00940153 100644 --- a/jobs/features/opcos/src/importer/opcoImporter.js +++ b/jobs/features/opcos/src/importer/opcoImporter.js @@ -138,7 +138,7 @@ module.exports = async () => { const importEtablishmentsStats = await importToEtablishments(options); const stats = { ...importTrainingStats, ...importEtablishmentsStats }; logger.info(" -- Stats -- "); - logger.info(stats); + logger.info(JSON.stringify(stats)); logger.info(" -- End of Import OPCOs --"); }, }; diff --git a/jobs/features/opcos/src/index.js b/jobs/features/opcos/src/index.js index 3db24cc9..0d5e293c 100644 --- a/jobs/features/opcos/src/index.js +++ b/jobs/features/opcos/src/index.js @@ -4,7 +4,7 @@ const opcoImporter = require("./importer/opcoImporter"); const run = async () => { const importer = await opcoImporter(); await execute(() => { - return importer.importOpcosToEtablishments({ + return importer.importOpcos({ override: process.env.OVERRIDE_MODE === "true", }); }); From bce2461ee63b2c10de912488267325de2a7b13ea Mon Sep 17 00:00:00 2001 From: Samir Date: Tue, 18 Aug 2020 15:46:58 +0200 Subject: [PATCH 6/6] Add filter for OPCOs in Etablishements search --- .../src/pages/Search/constantsEtablissements.js | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/front/src/pages/Search/constantsEtablissements.js b/front/src/pages/Search/constantsEtablissements.js index 4ac051f1..15b1d376 100644 --- a/front/src/pages/Search/constantsEtablissements.js +++ b/front/src/pages/Search/constantsEtablissements.js @@ -1,4 +1,4 @@ -const FILTERS = ["QUERYBUILDER", "SEARCH", "num_departement", "nom_academie"]; +const FILTERS = ["QUERYBUILDER", "SEARCH", "num_departement", "nom_academie", "opcos"]; const columnsDefinition = [ { @@ -246,6 +246,12 @@ const columnsDefinition = [ exportOnly: true, editable: false, }, + { + Header: "OPCOs", + accessor: "opcos", + width: 200, + editable: false, + }, ]; const queryBuilderField = [ @@ -275,6 +281,14 @@ const facetDefinition = [ selectAllLabel: "Tous", sortBy: "asc", }, + { + componentId: "opcos", + dataField: "opcos.keyword", + title: "OPCOs", + filterLabel: "opcos", + selectAllLabel: "Tout OPCOs", + sortBy: "asc", + }, ]; const dataSearch = {