diff --git a/targets/alert-cli/src/dares/__mocks__/getAgreementsData.ts b/targets/alert-cli/src/dares/__mocks__/getAgreementsData.ts new file mode 100644 index 000000000..8ceafdbc5 --- /dev/null +++ b/targets/alert-cli/src/dares/__mocks__/getAgreementsData.ts @@ -0,0 +1,22 @@ +import { GetAgreementData } from "../getAgreementsData"; + +export async function getAgreements(): Promise { + return Promise.resolve([ + { + name: "Convention collective nationale des transports routiers et activités auxiliaires du transport", + num: 16, + }, + { + name: "Convention collective nationale de l'industrie textile du 1er février 1951. Etendue par arrêté du 17 décembre 1951, rectificatif du 13 janvier 1952, mise à jour le 29 mai 1979, en vigueur le 1er octobre 1979. Etendue par arrêté du 23 octobre 1979. JONC 12 janvier 1980 ", + num: 18, + }, + { + name: "Convention collective nationale des établissements privés d'hospitalisation, de soins, de cure et de garde à but non lucratif (FEHAP, convention de 1951)", + num: 29, + }, + { + name: "Convention collective nationale des industries chimiques et connexes", + num: 44, + }, + ]); +} diff --git a/targets/alert-cli/src/dares/__mocks__/getDaresData.ts b/targets/alert-cli/src/dares/__mocks__/getDaresData.ts new file mode 100644 index 000000000..2c33a8367 --- /dev/null +++ b/targets/alert-cli/src/dares/__mocks__/getDaresData.ts @@ -0,0 +1,19 @@ +import { Agreement } from "../types"; + +export const getDaresData = async (): Promise => { + return Promise.resolve([ + { + name: "Convention collective nationale des transports routiers et activités auxiliaires du transport", + num: 16, + }, + { name: "Convention collective nationale de l'industrie textile", num: 18 }, + { + name: "Convention collective nationale des établissements privés d'hospitalisation, de soins, de cure et de garde à but non lucratif (FEHAP, convention de 1951)", + num: 29, + }, + { + name: "Convention collective nationale de l'Import-export et du Commerce international", + num: 43, + }, + ]); +}; diff --git a/targets/alert-cli/src/dares/__tests__/difference.test.ts b/targets/alert-cli/src/dares/__tests__/difference.test.ts new file mode 100644 index 000000000..6c774dcaa --- /dev/null +++ b/targets/alert-cli/src/dares/__tests__/difference.test.ts @@ -0,0 +1,24 @@ +import { getDifferenceBetweenIndexAndDares } from "../difference"; + +jest.mock("../getAgreementsData"); +jest.mock("../getDaresData"); + +describe("getDifferenceBetweenIndexAndDares", () => { + it("should work", async () => { + const result = await getDifferenceBetweenIndexAndDares(); + expect(result).toEqual({ + addedAgreementsFromDares: [ + { + name: "Convention collective nationale de l'Import-export et du Commerce international", + num: 43, + }, + ], + removedAgreementsFromDares: [ + { + name: "Convention collective nationale des industries chimiques et connexes", + num: 44, + }, + ], + }); + }); +}); diff --git a/targets/alert-cli/src/dares/__tests__/getDaresData.test.ts b/targets/alert-cli/src/dares/__tests__/getDaresData.test.ts new file mode 100644 index 000000000..b419d7e4d --- /dev/null +++ b/targets/alert-cli/src/dares/__tests__/getDaresData.test.ts @@ -0,0 +1,53 @@ +import { getDaresData } from "../getDaresData"; + +jest.mock("../fetchDaresXlsx", () => { + return { + fetchDaresXlsx: () => { + return [ + { + name: "Liste IDCC-Publication", + data: [ + ["IDENTIFIANT DE CONVENTION COLLECTIVE (IDCC) : "], + [ + null, + "CODES EN VIGUEUR POUR LE REMPLISSAGE de la DADS-U et de la DSN", + ], + [], + ["IDCC", "TITRE DE LA CONVENTION"], + [ + "0016", + "Convention collective nationale des transports routiers et activités auxiliaires du transport", + ], + ["0018", "Convention collective nationale de l'industrie textile"], + [ + "0029", + "Convention collective nationale des établissements privés d'hospitalisation, de soins, de cure et de garde à but non lucratif (FEHAP, convention de 1951)", + ], + ["9998", "___Convention non encore en vigueur___"], + ["9999", "___Sans convention collective___"], + ], + }, + ]; + }, + }; +}); + +describe("getDaresData", () => { + it("should work", async () => { + const result = await getDaresData(); + expect(result).toEqual([ + { + name: "Convention collective nationale des transports routiers et activités auxiliaires du transport", + num: 16, + }, + { + name: "Convention collective nationale de l'industrie textile", + num: 18, + }, + { + name: "Convention collective nationale des établissements privés d'hospitalisation, de soins, de cure et de garde à but non lucratif (FEHAP, convention de 1951)", + num: 29, + }, + ]); + }); +}); diff --git a/targets/alert-cli/src/dares/difference.ts b/targets/alert-cli/src/dares/difference.ts index 7ff9dbd6a..8bd0a9fc3 100644 --- a/targets/alert-cli/src/dares/difference.ts +++ b/targets/alert-cli/src/dares/difference.ts @@ -1,46 +1,22 @@ -import xlsx from "node-xlsx"; import { Diff, Agreement } from "./types"; -import fs from "fs"; - -export function getDifferenceBetweenIndexAndDares( - pathDares: string, - pathIndex: string -): Diff { - const workSheetsFromFile = xlsx.parse(pathDares); - - const supportedCcXlsx: Agreement[] = []; - - workSheetsFromFile[0].data.forEach((row: string[]) => { - const ccNumber = parseInt(row[0]); - const ccName = row[1]; - if (ccNumber && ccName) { - const ccNameWithoutParenthesis = ccName - .replace(/\(.*annexée.*\)/gi, "") - .trim(); - supportedCcXlsx.push({ - name: ccNameWithoutParenthesis, - num: ccNumber, - }); - } - }); - - const dataJson = JSON.parse(fs.readFileSync(pathIndex, "utf8")); - - const supportedCcIndexJson: Agreement[] = dataJson.map((cc: any) => { - return { - name: cc.title, - num: cc.num, - }; - }); - - const missingAgreementsFromDares: Agreement[] = supportedCcXlsx.filter( - (ccIndex) => - !supportedCcIndexJson.find((ccXlsx) => ccXlsx.num === ccIndex.num) +import { getDaresData } from "./getDaresData"; +import { getAgreements } from "./getAgreementsData"; + +export async function getDifferenceBetweenIndexAndDares(): Promise { + const daresDataList = await getDaresData(); + const AgreementDataList = await getAgreements(); + + const addedAgreementsFromDares: Agreement[] = daresDataList.filter( + (daresData) => + !AgreementDataList.find( + (agreementData) => agreementData.num === daresData.num + ) ); - const exceedingAgreementsFromKali = supportedCcIndexJson.filter( - (ccXlsx) => !supportedCcXlsx.find((ccIndex) => ccIndex.num === ccXlsx.num) + const removedAgreementsFromDares = AgreementDataList.filter( + (agreementData) => + !daresDataList.find((daresData) => daresData.num === agreementData.num) ); - return { missingAgreementsFromDares, exceedingAgreementsFromKali }; + return { addedAgreementsFromDares, removedAgreementsFromDares }; } diff --git a/targets/alert-cli/src/dares/fetchDaresXlsx.ts b/targets/alert-cli/src/dares/fetchDaresXlsx.ts new file mode 100644 index 000000000..efe4f7e82 --- /dev/null +++ b/targets/alert-cli/src/dares/fetchDaresXlsx.ts @@ -0,0 +1,13 @@ +import { extractDaresXlsxFromMT } from "./scrapping"; +import { downloadFileInTempFolder } from "./download"; +import xlsx from "node-xlsx"; + +export const fetchDaresXlsx = async () => { + const xlsxUrl = await extractDaresXlsxFromMT(); + const xlsxPath = await downloadFileInTempFolder(xlsxUrl, "dares.xlsx"); + const workSheetsFromFile: { + name: string; + data: any[][]; + }[] = xlsx.parse(xlsxPath); + return workSheetsFromFile; +}; diff --git a/targets/alert-cli/src/dares/getAgreementsData.ts b/targets/alert-cli/src/dares/getAgreementsData.ts new file mode 100644 index 000000000..6b2724205 --- /dev/null +++ b/targets/alert-cli/src/dares/getAgreementsData.ts @@ -0,0 +1,42 @@ +import { gqlClient } from "@shared/utils"; + +const getAgreementQuery = ` +query queryAgreements { + agreements: agreement_agreements( + where: { id : { _neq: "0000" }}, + order_by: { id: asc }) { + id + name + } +} +`; + +export interface GetAgreementData { + num: number; + name: string; +} + +interface GetAgreementQueryResult { + agreements: { + id: string; + name: string; + }[]; +} + +export async function getAgreements(): Promise { + const client = gqlClient(); + const result = await client + .query(getAgreementQuery, {}) + .toPromise(); + if ( + !result || + !result.data || + !result.data?.agreements || + !result.data?.agreements.length + ) { + throw new Error("An error occured on hasura agreement loading"); + } + return result.data.agreements.map(({ name, id }) => { + return { name, num: parseInt(id) }; + }); +} diff --git a/targets/alert-cli/src/dares/getDaresData.ts b/targets/alert-cli/src/dares/getDaresData.ts new file mode 100644 index 000000000..aec3c0202 --- /dev/null +++ b/targets/alert-cli/src/dares/getDaresData.ts @@ -0,0 +1,30 @@ +import { Agreement } from "./types"; +import { fetchDaresXlsx } from "./fetchDaresXlsx"; + +export const getDaresData = async () => { + const workSheetsFromFile: { + name: string; + data: any[][]; + }[] = await fetchDaresXlsx(); + + return workSheetsFromFile[0].data.reduce((arr, row) => { + const ccNumber = parseInt(row[0]); + if ([9998, 9999].indexOf(ccNumber) !== -1) { + return arr; + } + const ccName = row[1] as string; + if (ccNumber && ccName) { + const ccNameWithoutParenthesis = ccName + .replace(/\(.*annexée.*\)/gi, "") + .trim(); + return [ + ...arr, + { + name: ccNameWithoutParenthesis, + num: ccNumber, + }, + ]; + } + return arr; + }, []); +}; diff --git a/targets/alert-cli/src/dares/index.ts b/targets/alert-cli/src/dares/index.ts index 250512a07..04a7079b2 100644 --- a/targets/alert-cli/src/dares/index.ts +++ b/targets/alert-cli/src/dares/index.ts @@ -1,13 +1,7 @@ -import { URL_KALI } from "./config"; import { getDifferenceBetweenIndexAndDares } from "./difference"; -import { downloadFileInTempFolder } from "./download"; import { saveDiff } from "./save"; -import { extractDaresXlsxFromMT } from "./scrapping"; export const runDares = async () => { - const xlsxUrl = await extractDaresXlsxFromMT(); - const xlsxPath = await downloadFileInTempFolder(xlsxUrl, "dares.xlsx"); - const indexPath = await downloadFileInTempFolder(URL_KALI, "index.json"); - const diff = await getDifferenceBetweenIndexAndDares(xlsxPath, indexPath); + const diff = await getDifferenceBetweenIndexAndDares(); await saveDiff(diff); }; diff --git a/targets/alert-cli/src/dares/save.ts b/targets/alert-cli/src/dares/save.ts index a81cd2195..170ee1e82 100644 --- a/targets/alert-cli/src/dares/save.ts +++ b/targets/alert-cli/src/dares/save.ts @@ -6,7 +6,7 @@ export const saveDiff = async (diff: Diff) => { const alertRepository = new AlertRepository(gqlClient()); const alertsRemovedToSave: DaresAlertInsert[] = - diff.exceedingAgreementsFromKali.map((agreement) => ({ + diff.removedAgreementsFromDares.map((agreement) => ({ info: { id: agreement.num, }, @@ -31,7 +31,7 @@ export const saveDiff = async (diff: Diff) => { })); const alertsAddedToSave: DaresAlertInsert[] = - diff.missingAgreementsFromDares.map((agreement) => ({ + diff.addedAgreementsFromDares.map((agreement) => ({ info: { id: agreement.num, }, diff --git a/targets/alert-cli/src/dares/types.ts b/targets/alert-cli/src/dares/types.ts index 875b0b45b..f221eef2e 100644 --- a/targets/alert-cli/src/dares/types.ts +++ b/targets/alert-cli/src/dares/types.ts @@ -1,8 +1,8 @@ import { DaresAlert } from "@socialgouv/cdtn-types"; export interface Diff { - missingAgreementsFromDares: Agreement[]; - exceedingAgreementsFromKali: Agreement[]; + addedAgreementsFromDares: Agreement[]; + removedAgreementsFromDares: Agreement[]; } export interface Agreement { diff --git a/targets/hasura/migrations/default/1727689859510_add_table_to_audit/down.sql b/targets/hasura/migrations/default/1727689859510_add_table_to_audit/down.sql new file mode 100644 index 000000000..70007d2c6 --- /dev/null +++ b/targets/hasura/migrations/default/1727689859510_add_table_to_audit/down.sql @@ -0,0 +1,10 @@ +DROP TRIGGER IF EXISTS audit_trigger_row on search.prequalified_documents; +DROP TRIGGER IF EXISTS audit_trigger_row on search.prequalified; +DROP TRIGGER IF EXISTS audit_trigger_row on information.informations_references; +DROP TRIGGER IF EXISTS audit_trigger_row on information.informations_contents_references; +DROP TRIGGER IF EXISTS audit_trigger_row on information.informations_contents_blocks_contents; +DROP TRIGGER IF EXISTS audit_trigger_row on information.informations_contents_blocks; +DROP TRIGGER IF EXISTS audit_trigger_row on information.informations_contents; +DROP TRIGGER IF EXISTS audit_trigger_row on information.informations; +DROP TRIGGER IF EXISTS audit_trigger_row on agreement.agreements; + diff --git a/targets/hasura/migrations/default/1727689859510_add_table_to_audit/up.sql b/targets/hasura/migrations/default/1727689859510_add_table_to_audit/up.sql new file mode 100644 index 000000000..2712bfd14 --- /dev/null +++ b/targets/hasura/migrations/default/1727689859510_add_table_to_audit/up.sql @@ -0,0 +1,9 @@ +select audit.audit_table('agreement.agreements'::regclass, true, true); +select audit.audit_table('information.informations'::regclass, true, true); +select audit.audit_table('information.informations_contents'::regclass, true, true); +select audit.audit_table('information.informations_contents_blocks'::regclass, true, true); +select audit.audit_table('information.informations_contents_blocks_contents'::regclass, true, true); +select audit.audit_table('information.informations_contents_references'::regclass, true, true); +select audit.audit_table('information.informations_references'::regclass, true, true); +select audit.audit_table('search.prequalified'::regclass, true, true); +select audit.audit_table('search.prequalified_documents'::regclass, true, true); diff --git a/targets/hasura/migrations/default/1727795072418_clean_dares_alerts/down.sql b/targets/hasura/migrations/default/1727795072418_clean_dares_alerts/down.sql new file mode 100644 index 000000000..e69de29bb diff --git a/targets/hasura/migrations/default/1727795072418_clean_dares_alerts/up.sql b/targets/hasura/migrations/default/1727795072418_clean_dares_alerts/up.sql new file mode 100644 index 000000000..85fabba2b --- /dev/null +++ b/targets/hasura/migrations/default/1727795072418_clean_dares_alerts/up.sql @@ -0,0 +1,3 @@ +delete from alerts +where repository = 'dares' +and status = 'todo'; \ No newline at end of file