feat: suppression de mongo-migrate

.infra/ansible/deploy.yml

@@ -31,7 +31,7 @@
         cmd: "sudo docker service scale tdb_queue_processor=0 tdb_jobs_processor=0"
       when:
         - stack_status.stdout != ""
-        - '"migrations-status=synced" not in migration_status_output.stdout_lines'
+        - '"migrations-status=require-shutdown" not in migration_status_output.stdout_lines'
 
     - name: "Activation du mode maintenance pour effectuer la migration"
       shell:
@@ -47,7 +47,7 @@
         cmd: "sudo docker service scale tdb_server=0"
       when:
         - stack_status.stdout != ""
-        - '"migrations-status=synced" not in migration_status_output.stdout_lines'
+        - '"migrations-status=require-shutdown" not in migration_status_output.stdout_lines'
 
     - name: Lancement des migrations
       shell:

server/package.json

@@ -62,7 +62,6 @@
     "lodash-es": "4.17.21",
     "lodash.pick": "4.4.0",
     "luxon": "^2.5.2",
-    "migrate-mongo": "10.0.0",
     "mjml": "^4.15.3",
     "mongodb": "^5.9.2",
     "multer": "^1.4.5-lts.1",

server/src/common/utils/errorUtils.ts

@@ -1,3 +1,5 @@
+import { captureException } from "@sentry/node";
+
 class AppError extends Error {
   details: any;
 
@@ -27,3 +29,9 @@ export const formatError = (error: any) => {
 
   return error;
 };
+
+export function withCause<T extends Error>(error: T, cause: Error, level: "fatal" | "error" | "warning" = "error"): T {
+  error.cause = cause;
+  captureException(cause, { level });
+  return error;
+}

server/src/jobs/jobs.ts

@@ -323,8 +323,12 @@ export async function setupJobProcessor() {
       },
       "migrations:status": {
         handler: async () => {
-          const pendingMigrations = await statusMigration();
-          console.log(`migrations-status=${pendingMigrations === 0 ? "synced" : "pending"}`);
+          const { count, requireShutdown } = await statusMigration();
+          if (count === 0) {
+            console.log("migrations-status=synced");
+          } else {
+            console.log(`migrations-status=${requireShutdown ? "require-shutdown" : "pending"}`);
+          }
           return;
         },
       },

server/src/jobs/migrations/migrations.ts

@@ -1,14 +1,17 @@
-import { readFile, writeFile } from "node:fs/promises";
+import { readdir, writeFile } from "node:fs/promises";
 import path from "path";
 
-import { config, create as mcreate, status as mstatus, up as mup } from "migrate-mongo";
+import Boom from "boom";
+import { format } from "date-fns";
 
-import { getMongodbClient } from "@/common/mongodb";
+import { getDatabase, getMongodbClient } from "@/common/mongodb";
+import { withCause } from "@/common/utils/errorUtils";
 import { __dirname } from "@/common/utils/esmUtils";
+import config from "@/config";
 
 const myConfig = {
   mongodb: {
-    url: process.env.FLUX_RETOUR_CFAS_MONGODB_URI as string,
+    url: config.mongodb.uri,
 
     // in URL
     databaseName: "",
@@ -22,7 +25,7 @@ const myConfig = {
   },
 
   // The migrations dir, can be an relative or absolute path. Only edit this when really necessary.
-  migrationsDir: path.join(__dirname(import.meta.url), "./db/migrations"),
+  migrationsDir: path.join(__dirname(import.meta.url), "./migrations"),
 
   // The mongodb collection where the applied changes are stored. Only edit this when really necessary.
   changelogCollectionName: "changelog",
@@ -38,43 +41,84 @@ const myConfig = {
   moduleSystem: "esm",
 };
 
-export async function up() {
-  config.set(myConfig);
+async function listMigrationFiles(): Promise<string[]> {
+  const files = await readdir(myConfig.migrationsDir, { withFileTypes: true });
 
-  await status();
-
-  const client = getMongodbClient();
-  await mup(client.db(), client);
+  return files
+    .filter((file) => file.isFile() && file.name.endsWith(myConfig.migrationFileExtension))
+    .map((file) => file.name);
 }
 
-// Show migration status and returns number of pending migrations
-export async function status(): Promise<number> {
-  // @ts-ignore
-  config.set(myConfig);
-  const client = getMongodbClient();
+async function getAppliedMigrations(): Promise<Map<string, Date>> {
+  const db = getDatabase();
+  const appliedMigrations = await db
+    .collection("changelog")
+    .find({}, { sort: { fileName: 1 } })
+    .toArray();
 
-  // @ts-ignore
-  const migrationStatus = await mstatus(client.db());
-  migrationStatus.forEach(({ fileName, appliedAt }) => console.log(fileName, ":", appliedAt));
+  return new Map(appliedMigrations.map(({ fileName, appliedAt }) => [fileName, appliedAt]));
+}
 
-  return migrationStatus.filter(({ appliedAt }) => appliedAt === "PENDING").length;
+export async function up(): Promise<number> {
+  const migrationFiles = await listMigrationFiles();
+  const appliedMigrationsFiles = await getAppliedMigrations();
+
+  let count = 0;
+  for (const migrationFile of migrationFiles) {
+    if (!appliedMigrationsFiles.has(migrationFile)) {
+      count++;
+      try {
+        const { up } = await import(path.join(myConfig.migrationsDir, migrationFile));
+        await up(getDatabase(), getMongodbClient());
+        await getDatabase()
+          .collection(myConfig.changelogCollectionName)
+          .insertOne({ fileName: migrationFile, appliedAt: new Date() });
+        console.log(`${migrationFile} : APPLIED`);
+      } catch (e) {
+        throw withCause(Boom.internal("Error applying migration", { migrationFile }), e as Error);
+      }
+    }
+  }
+
+  return count;
+}
+
+// Show migration status and returns number of pending migrations
+export async function status(): Promise<{ count: number; requireShutdown: boolean }> {
+  const migrationFiles = await listMigrationFiles();
+  const appliedMigrationsFiles = await getAppliedMigrations();
+
+  const result = {
+    requireShutdown: false,
+    count: 0,
+  };
+
+  for (const migrationFile of migrationFiles) {
+    if (!appliedMigrationsFiles.has(migrationFile)) {
+      result.count++;
+    }
+    const { requireShutdown = false } = await import(path.join(myConfig.migrationsDir, migrationFile));
+    result.requireShutdown = result.requireShutdown || requireShutdown;
+
+    const appliedAt = appliedMigrationsFiles.get(migrationFile) ?? "PENDING";
+    console.log(`${migrationFile} : ${appliedAt}`);
+  }
+
+  return result;
 }
 
 export async function create({ description }: { description: string }) {
-  config.set({
-    ...myConfig,
-    migrationsDir: "src/db/migrations",
-    migrationFileExtension: ".ts",
-  });
-  const fileName = await mcreate(description);
-  const file = `src/db/migrations/${fileName}`;
-  const content = await readFile(file, {
-    encoding: "utf-8",
-  });
-  // ajoute le typage TS et supprime la migration down inutile
-  const newContent = `import { Db } from "mongodb";
-
-${content.replaceAll("async (db, client)", "async (db: Db)").replace(/\nexport const down =[\s\S]+/, "")}`;
+  const fileName = `${format(new Date(), "yyyyMMddHHmmss")}-${description.replaceAll(" ", "_")}.ts`;
+  const file = `src/migrations/${fileName}`;
+  const newContent = `
+import { getDbCollection } from "@/common/utils/mongodbUtils";
+
+export const up = async () => {
+await getDbCollection("").updateMany({}, { $set: { "profile.newField": "defaultValue" } });
+};
+
+// set to false ONLY IF migration does not imply a breaking change (ex: update field value or add index)
+export const requireShutdown: boolean = true;`;
 
   await writeFile(file, newContent, { encoding: "utf-8" });
   console.log("Created:", fileName);

yarn.lock

@@ -6003,7 +6003,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"cli-table3@npm:^0.6.1, cli-table3@npm:^0.6.3":
+"cli-table3@npm:^0.6.3":
   version: 0.6.3
   resolution: "cli-table3@npm:0.6.3"
   dependencies:
@@ -6201,7 +6201,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"commander@npm:^9.1.0, commander@npm:^9.5.0":
+"commander@npm:^9.5.0":
   version: 9.5.0
   resolution: "commander@npm:9.5.0"
   checksum: c7a3e27aa59e913b54a1bafd366b88650bc41d6651f0cbe258d4ff09d43d6a7394232a4dadd0bf518b3e696fdf595db1028a0d82c785b88bd61f8a440cecfade
@@ -6818,7 +6818,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"date-fns@npm:^2.28.0, date-fns@npm:^2.30.0":
+"date-fns@npm:^2.30.0":
   version: 2.30.0
   resolution: "date-fns@npm:2.30.0"
   dependencies:
@@ -8696,13 +8696,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"fn-args@npm:^5.0.0":
-  version: 5.0.0
-  resolution: "fn-args@npm:5.0.0"
-  checksum: 51af66d5188a6243a33a9afc7309bcf24908331b16d8df2c10afad8dea0f39985e50ab9b53fd4bc4dfebcdc3cf7e1cb80cd46c76d69d34e4fd9deaa170b19765
-  languageName: node
-  linkType: hard
-
 "focus-lock@npm:^1.2.0":
   version: 1.2.0
   resolution: "focus-lock@npm:1.2.0"
@@ -8839,17 +8832,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"fs-extra@npm:^10.0.1":
-  version: 10.1.0
-  resolution: "fs-extra@npm:10.1.0"
-  dependencies:
-    graceful-fs: ^4.2.0
-    jsonfile: ^6.0.1
-    universalify: ^2.0.0
-  checksum: dc94ab37096f813cc3ca12f0f1b5ad6744dfed9ed21e953d72530d103cea193c2f81584a39e9dee1bea36de5ee66805678c0dddc048e8af1427ac19c00fffc50
-  languageName: node
-  linkType: hard
-
 "fs-extra@npm:^11.0.0":
   version: 11.1.1
   resolution: "fs-extra@npm:11.1.1"
@@ -12334,25 +12316,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"migrate-mongo@npm:10.0.0":
-  version: 10.0.0
-  resolution: "migrate-mongo@npm:10.0.0"
-  dependencies:
-    cli-table3: ^0.6.1
-    commander: ^9.1.0
-    date-fns: ^2.28.0
-    fn-args: ^5.0.0
-    fs-extra: ^10.0.1
-    lodash: ^4.17.21
-    p-each-series: ^2.2.0
-  peerDependencies:
-    mongodb: ^4.4.1 || ^5.0.0
-  bin:
-    migrate-mongo: bin/migrate-mongo.js
-  checksum: 0696db54624b01bc342793dc05be3b62a6cf5a22e23091bf779ed1d42cb9e0292c3130ecc2a1452c2b085a58dc43d7322185d2aa5e0d201da2de173b550bff4e
-  languageName: node
-  linkType: hard
-
 "mime-db@npm:1.52.0":
   version: 1.52.0
   resolution: "mime-db@npm:1.52.0"
@@ -14042,13 +14005,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"p-each-series@npm:^2.2.0":
-  version: 2.2.0
-  resolution: "p-each-series@npm:2.2.0"
-  checksum: 5fbe2f1f1966f55833bd401fe36f7afe410707d5e9fb6032c6dde8aa716d50521c3bb201fdb584130569b5941d5e84993e09e0b3f76a474288e0ede8f632983c
-  languageName: node
-  linkType: hard
-
 "p-each-series@npm:^3.0.0":
   version: 3.0.0
   resolution: "p-each-series@npm:3.0.0"
@@ -16238,7 +16194,6 @@ __metadata:
     lodash-es: 4.17.21
     lodash.pick: 4.4.0
     luxon: ^2.5.2
-    migrate-mongo: 10.0.0
     mjml: ^4.15.3
     mongodb: ^5.9.2
     multer: ^1.4.5-lts.1