diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 8030fdd5a1..6d59b6add1 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -37,7 +37,7 @@ jobs: strategy: fail-fast: false matrix: - language: ["javascript"] + language: ["javascript-typescript"] steps: - name: Checkout repository @@ -53,3 +53,5 @@ jobs: - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{matrix.language}}" diff --git a/.infra/ansible/deploy.yml b/.infra/ansible/deploy.yml index 008de626d6..8fc45528ac 100644 --- a/.infra/ansible/deploy.yml +++ b/.infra/ansible/deploy.yml @@ -91,6 +91,11 @@ chdir: /opt/app cmd: "sudo /opt/app/tools/reload-proxy.sh" + - name: "Désactivation du mode maintenance" + shell: + chdir: /opt/app + cmd: "sudo /opt/app/tools/maintenance/maintenance-off.sh" + - name: "Verification des certificats SSL" shell: chdir: /opt/app @@ -121,8 +126,3 @@ shell: chdir: /opt/app cmd: "sudo docker system prune --all --force" - - - name: "Désactivation du mode maintenance" - shell: - chdir: /opt/app - cmd: "sudo /opt/app/tools/maintenance/maintenance-off.sh" diff --git a/.infra/docker-compose.preview-system.yml b/.infra/docker-compose.preview-system.yml index 27a2d94afa..25946b2bd8 100644 --- a/.infra/docker-compose.preview-system.yml +++ b/.infra/docker-compose.preview-system.yml @@ -43,7 +43,7 @@ services: mongodb: <<: *default - image: mongo:6.0.2-focal + image: mongo:6.0.11 hostname: mongodb container_name: lba_mongodb deploy: @@ -72,7 +72,7 @@ services: resources: limits: memory: 2g - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.6 + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.15 environment: - ES_JAVA_OPTS=-Xmx512m -Xms512m - discovery.type=single-node @@ -108,7 +108,7 @@ services: smtp: <<: *default - image: axllent/mailpit:v1.5.5 + image: axllent/mailpit:v1.10.1 container_name: lba_smtp ports: - 1025:1025 @@ -126,7 +126,7 @@ services: nodeexporter: <<: *default - image: prom/node-exporter:v1.5.0 + image: prom/node-exporter:v1.7.0 hostname: "{{host_name}}" user: root command: @@ -150,7 +150,7 @@ services: cadvisor: <<: *default - image: gcr.io/cadvisor/cadvisor:v0.46.0 + image: gcr.io/cadvisor/cadvisor:v0.47.2 hostname: "{{host_name}}" privileged: true devices: diff --git a/.infra/docker-compose.production.yml b/.infra/docker-compose.production.yml index 0d4d6df5b6..1b736f4eb4 100644 --- a/.infra/docker-compose.production.yml +++ b/.infra/docker-compose.production.yml @@ -85,7 +85,7 @@ services: metabase: <<: *default - image: metabase/metabase:v0.46.6.4 + image: metabase/metabase:v0.47.8 deploy: <<: *deploy-default resources: @@ -122,7 +122,7 @@ services: elasticsearch: <<: *default - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.6 + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.15 deploy: <<: *deploy-default resources: diff --git a/.infra/docker-compose.recette.yml b/.infra/docker-compose.recette.yml index 113990d2a2..d786992d17 100644 --- a/.infra/docker-compose.recette.yml +++ b/.infra/docker-compose.recette.yml @@ -2,7 +2,7 @@ version: "3.8" services: smtp: - image: axllent/mailpit:v1.5.5 + image: axllent/mailpit:v1.10.1 deploy: resources: limits: @@ -29,6 +29,7 @@ services: environment: - MP_DATA_FILE=/data/mailpit.db - MP_UI_AUTH_FILE=/auth + - MP_WEBROOT=/smtp/ logging: driver: "fluentd" options: diff --git a/.infra/files/configs/reverse_proxy/locations/70_smtp.conf.template b/.infra/files/configs/reverse_proxy/locations/70_smtp.conf.template index e18079c9fb..70042afb6b 100644 --- a/.infra/files/configs/reverse_proxy/locations/70_smtp.conf.template +++ b/.infra/files/configs/reverse_proxy/locations/70_smtp.conf.template @@ -1,16 +1,4 @@ -location ~ ^/smtp/(.*)$ { +location /smtp { set $upstream http://smtp:8025; - proxy_pass $upstream/$1$is_args$args; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-Host $server_name; - proxy_set_header X-Forwarded-Proto $scheme; - - # Websocket configuration - # See: - # - https://www.nginx.com/blog/websocket-nginx/ - # - https://github.com/mailhog/MailHog/issues/117 - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $connection_upgrade; + include includes/proxy.conf; } diff --git a/docker-compose.yml b/docker-compose.yml index 73af7f48be..0f1ecd96c4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.8" services: mongodb: - image: mongo:6.0.2-focal + image: mongo:6.0.11 restart: unless-stopped hostname: mongodb mem_limit: 5g @@ -21,7 +21,7 @@ services: start_period: 10s elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.6 + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.15 container_name: lba_elasticsearch environment: - ES_JAVA_OPTS=-Xmx512m -Xms512m @@ -55,7 +55,7 @@ services: - lba_clamav_data:/var/lib/clamav smtp: - image: axllent/mailpit:v1.5.5 + image: axllent/mailpit:v1.10.1 restart: unless-stopped ports: - 1025:1025 diff --git a/server/src/commands.ts b/server/src/commands.ts index 48d55d03d8..c444539993 100644 --- a/server/src/commands.ts +++ b/server/src/commands.ts @@ -158,7 +158,13 @@ function createJobAction(name) { program.command("db:validate").description("Validate Documents").option("-q, --queued", "Run job asynchronously", false).action(createJobAction("db:validate")) -program.command("fix-diffusible-companies").description("Clean companies not diffusible").action(createJobAction("fix-diffusible-companies")) +program + .command("fix-diffusible-companies") + .description("Clean companies not diffusible.") + .option("-c, --collection_list ", " est la liste des collections à réparer séparées par des ,") + .option("-q, --queued", "Run job asynchronously", false) + .action(createJobAction("fix-diffusible-companies")) + program.command("check-diffusible-companies").description("Check companies are diffusible").action(createJobAction("check-diffusible-companies")) program.command("fiab:kevin").description("Run migrations up").action(createJobAction("fiab:kevin")) program.command("db:obfuscate").description("Pseudonymisation des documents").option("-q, --queued", "Run job asynchronously", false).action(createJobAction("db:obfuscate")) diff --git a/server/src/http/controllers/metiers/metiers.controller.ts b/server/src/http/controllers/metiers/metiers.controller.ts index dba6aaeec7..bdc88c2772 100644 --- a/server/src/http/controllers/metiers/metiers.controller.ts +++ b/server/src/http/controllers/metiers/metiers.controller.ts @@ -1,6 +1,6 @@ import { zRoutes } from "shared/index.js" -import { getCoupleAppellationRomeIntitule, getMetiers, getMetiersPourCfd, getMetiersPourEtablissement, getTousLesMetiers } from "../../../services/metiers.service" +import { getCoupleAppellationRomeIntitule, getMetiers, getMetiersPourCfd, getTousLesMetiers } from "../../../services/metiers.service" import { Server } from "../../server" const config = { @@ -24,19 +24,6 @@ export default (server: Server) => { } ) - server.get( - "/v1/metiers/metiersParEtablissement/:siret", - { - schema: zRoutes.get["/v1/metiers/metiersParEtablissement/:siret"], - config, - }, - async (req, res) => { - const { siret } = req.params - const result = await getMetiersPourEtablissement({ siret }) - return res.send(result) - } - ) - server.get( "/v1/metiers/all", { diff --git a/server/src/jobs/database/fixDiffusibleCompanies.ts b/server/src/jobs/database/fixDiffusibleCompanies.ts index 0517b81a07..ef52da624d 100644 --- a/server/src/jobs/database/fixDiffusibleCompanies.ts +++ b/server/src/jobs/database/fixDiffusibleCompanies.ts @@ -1,15 +1,18 @@ import { setTimeout } from "timers/promises" import Boom from "boom" -import { ILbaCompany } from "shared" +import { ILbaCompany, IRecruiter, IUserRecruteur, JOB_STATUS } from "shared" import { EDiffusibleStatus } from "shared/constants/diffusibleStatus" +import { ETAT_UTILISATEUR, RECRUITER_STATUS, VALIDATION_UTILISATEUR } from "shared/constants/recruteur" import { logger } from "@/common/logger" +import { Recruiter, UserRecruteur } from "@/common/model" import { db } from "@/common/mongodb" import { getEtablissementDiffusionStatus } from "@/services/etablissement.service" const MAX_RETRY = 100 const DELAY = 100 +const ANONYMIZED = "anonymized" const getDiffusionStatus = async (siret: string, count = 1) => { const isDiffusible = await getEtablissementDiffusionStatus(siret) @@ -51,8 +54,111 @@ const fixLbaCompanies = async () => { logger.info(`Fixing lba companies done`) } -export async function fixDiffusibleCompanies(): Promise { - await fixLbaCompanies() +const deactivateRecruiter = async (recruiter: IRecruiter) => { + console.log("deactivating non diffusible recruiter : ", recruiter.establishment_siret) + recruiter.status = RECRUITER_STATUS.ARCHIVE + recruiter.address = ANONYMIZED + recruiter.geo_coordinates = ANONYMIZED + recruiter.address_detail = recruiter.address_detail + ? { status_diffusion: recruiter.address_detail.status_diffusion, libelle_commune: ANONYMIZED } + : { libelle_commune: ANONYMIZED } + + for await (const job of recruiter.jobs) { + job.job_status = JOB_STATUS.ACTIVE ? JOB_STATUS.ANNULEE : job.job_status + } + + await Recruiter.updateOne({ _id: recruiter._id }, { $set: { ...recruiter } }) +} + +const deactivateUserRecruteur = async (userRecruteur: IUserRecruteur) => { + console.log("deactivating non diffusible userRecruteur : ", userRecruteur.establishment_siret) + + const userStatus = { + user: "SERVEUR", + validation_type: VALIDATION_UTILISATEUR.AUTO, + status: ETAT_UTILISATEUR.DESACTIVE, + reason: "Anonymization des données", + date: new Date(), + } + if (!userRecruteur.status) { + userRecruteur.status = [] + } + userRecruteur.status.push(userStatus) + + userRecruteur.address = ANONYMIZED + userRecruteur.geo_coordinates = ANONYMIZED + + if (userRecruteur.address_detail) { + userRecruteur.address_detail = { libelle_commune: ANONYMIZED } + } + + await UserRecruteur.updateOne({ _id: userRecruteur._id }, { $set: { ...userRecruteur } }) +} + +const fixRecruiters = async () => { + logger.info(`Fixing diffusible recruiters and offers`) + const recruiters: AsyncIterable = await db.collection("recruiters").find({}) + + let count = 0 + let deactivatedCount = 0 + let errorCount = 0 + for await (const recruiter of recruiters) { + if (count % 100 === 0) { + logger.info(`${count} recruiters checked. ${deactivatedCount} removed. ${errorCount} errors`) + } + count++ + try { + const isDiffusible = await getDiffusionStatus(recruiter.establishment_siret) + + if (isDiffusible !== EDiffusibleStatus.DIFFUSIBLE) { + deactivateRecruiter(recruiter) + + deactivatedCount++ + } + } catch (err) { + errorCount++ + console.log(err) + break + } + } + + const userRecruteurs: AsyncIterable = await db.collection("userrecruteurs").find({}) + + count = 0 + deactivatedCount = 0 + errorCount = 0 + for await (const userRecruteur of userRecruteurs) { + if (count % 100 === 0) { + logger.info(`${count} userRecruteurs checked. ${deactivatedCount} removed. ${errorCount} errors`) + } + count++ + try { + const isDiffusible = userRecruteur.establishment_siret ? await getDiffusionStatus(userRecruteur.establishment_siret) : EDiffusibleStatus.NOT_FOUND + + if (isDiffusible !== EDiffusibleStatus.DIFFUSIBLE) { + deactivateUserRecruteur(userRecruteur) + + deactivatedCount++ + } + } catch (err) { + errorCount++ + console.log(err) + break + } + } +} + +export async function fixDiffusibleCompanies(payload: { collection_list?: string }): Promise { + const collectionList = payload?.collection_list ?? "lbacompanies,recruiters" + const list = collectionList.split(",") + + if (list.includes("lbacompanies")) { + await fixLbaCompanies() + } + + if (list.includes("recruiters")) { + await fixRecruiters() + } } export async function checkDiffusibleCompanies(): Promise { diff --git a/server/src/jobs/jobs.ts b/server/src/jobs/jobs.ts index 63e74174e4..8b95d8d209 100644 --- a/server/src/jobs/jobs.ts +++ b/server/src/jobs/jobs.ts @@ -356,7 +356,7 @@ export async function runJob(job: IInternalJobsCronTask | IInternalJobsSimple): case "mongodb:indexes:create": return createMongoDBIndexes() case "fix-diffusible-companies": - return fixDiffusibleCompanies() + return fixDiffusibleCompanies(job.payload) case "check-diffusible-companies": return checkDiffusibleCompanies() case "db:validate": diff --git a/server/src/services/metiers.service.ts b/server/src/services/metiers.service.ts index 06e4f3bf92..9c1246c1ec 100644 --- a/server/src/services/metiers.service.ts +++ b/server/src/services/metiers.service.ts @@ -325,18 +325,6 @@ export const getMetiersPourCfd = async ({ cfd }: { cfd: string }): Promise} - */ -export const getMetiersPourEtablissement = async ({ siret }: { siret: string }): Promise => { - const romeResponse = await getRomesFromCatalogue({ siret }) - const { romes } = romeResponse - const metiers = await getMetiersFromRomes(romes) - return metiers -} - /** * Récupère la liste des métiers dans la table domaines / métiers correspondant à un tableau de codes ROME * @param {string[]} romes diff --git a/server/tests/integration/http/romesFromCatalogue.test.ts b/server/tests/integration/http/romesFromCatalogue.test.ts index c91473e13b..9768f72ace 100644 --- a/server/tests/integration/http/romesFromCatalogue.test.ts +++ b/server/tests/integration/http/romesFromCatalogue.test.ts @@ -18,22 +18,6 @@ describe.skip("romesFromCatalogue", () => { assert.ok(JSON.parse(response.body).error.length > 0) }) - it("Vérifie que la route métiers par établissement répond", async () => { - const response = await httpClient().inject({ method: "GET", path: "/api/v1/metiers/metiersParEtablissement/a" }) - - if (response.statusCode !== 500) { - // test en local avec es bien renseigné - - expect(response.statusCode).toBe(200) - - assert.ok(JSON.parse(response.body).metiers instanceof Array) - assert.ok(JSON.parse(response.body).metiers.length === 0) - assert.ok(JSON.parse(response.body).error.length > 0) - } else { - expect(response.statusCode).toBe(500) - } - }) - it("Vérifie que la requête metiersParFormation répond avec des résultats", async () => { const response = await httpClient().inject({ method: "GET", path: "/api/v1/metiers/metiersParFormation/50022137" }) @@ -47,18 +31,6 @@ describe.skip("romesFromCatalogue", () => { } }) - it("Vérifie que la requête metiersParEtablissement répond avec des résultats", async () => { - const response = await httpClient().inject({ method: "GET", path: "/api/v1/metiers/metiersParEtablissement/77566202600225" }) - - if (response.statusCode !== 500) { - // test en local avec es bien renseigné - expect(response.statusCode).toBe(200) - assert.ok(JSON.parse(response.body).metiers instanceof Array) - } else { - expect(response.statusCode).toBe(500) - } - }) - it("Vérifie que la requête tous les métiers répond avec des résultats", async () => { const response = await httpClient().inject({ method: "GET", path: "/api/v1/metiers/all" }) diff --git a/shared/helpers/openapi/__snapshots__/generateOpenapi.test.ts.snap b/shared/helpers/openapi/__snapshots__/generateOpenapi.test.ts.snap index 747bceaeb0..6caf3e45d2 100644 --- a/shared/helpers/openapi/__snapshots__/generateOpenapi.test.ts.snap +++ b/shared/helpers/openapi/__snapshots__/generateOpenapi.test.ts.snap @@ -5672,57 +5672,6 @@ L'email est envoyé depuis l'adresse générique \\"Ne pas répondre\\" de La bo ], }, }, - "/v1/metiers/metiersParEtablissement/{siret}": { - "get": { - "description": "Récupérer la liste des noms des métiers du référentiel de La bonne alternance pour un établissement de formation", - "operationId": "getMetiersParEtablissement", - "parameters": [ - { - "description": "Le numéro de SIRET de l'établissement", - "in": "path", - "name": "siret", - "required": true, - "schema": { - "description": "Le numéro de SIRET de l'établissement", - "example": "78424186100011", - "pattern": "^[0-9]{14}$", - "type": "string", - }, - }, - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "additionalProperties": false, - "properties": { - "metiers": { - "description": "Un tableau des noms des métiers triés par ordre alphabétique", - "items": { - "description": "Un nom de métier du référentiel La bonne alternance", - "example": "Accueil touristique", - "type": "string", - }, - "type": "array", - }, - }, - "required": [ - "metiers", - ], - "type": "object", - }, - }, - }, - "description": "", - }, - }, - "security": [], - "tags": [ - "Metiers", - ], - }, - }, "/v1/metiers/metiersParFormation/{cfd}": { "get": { "description": "Récupérer la liste des noms des métiers du référentiel de La bonne alternance pour une formation donnée", diff --git a/shared/routes/metiers.routes.ts b/shared/routes/metiers.routes.ts index 519970ed4b..054148aa9c 100644 --- a/shared/routes/metiers.routes.ts +++ b/shared/routes/metiers.routes.ts @@ -1,4 +1,3 @@ -import { extensions } from "../helpers/zodHelpers/zodPrimitives" import { z } from "../helpers/zodWithOpenApi" import { ZAppellationsRomes, ZMetierEnrichiArray, ZMetiers } from "../models/metiers.model" @@ -29,24 +28,6 @@ export const zMetiersRoutes = { operationId: "getMetiersParCfd", }, }, - "/v1/metiers/metiersParEtablissement/:siret": { - method: "get", - path: "/v1/metiers/metiersParEtablissement/:siret", - params: z - .object({ - siret: extensions.siret, - }) - .strict(), - response: { - 200: ZMetiers, - }, - securityScheme: null, - openapi: { - description: "Récupérer la liste des noms des métiers du référentiel de La bonne alternance pour un établissement de formation", - tags: ["Metiers"] as string[], - operationId: "getMetiersParEtablissement", - }, - }, "/v1/metiers/all": { method: "get", path: "/v1/metiers/all",