Merge pull request #1479 from argos-ci/knex-scripts

chore: move knex-scripts to a local package

apps/backend/package.json

@@ -28,6 +28,7 @@
   },
   "dependencies": {
     "@apollo/server": "^4.11.2",
+    "@argos/knex-scripts": "workspace:*",
     "@argos/tsconfig": "workspace:*",
     "@argos/util": "workspace:*",
     "@aws-sdk/client-s3": "^3.712.0",
@@ -89,6 +90,7 @@
   },
   "devDependencies": {
     "@argos/config-types": "workspace:*",
+    "@argos/knex-scripts": "workspace:*",
     "@types/amqplib": "^0.10.6",
     "@types/auth-header": "^1.0.6",
     "@types/convict": "^6.1.6",
@@ -103,7 +105,6 @@
     "@types/tmp": "^0.2.6",
     "concurrently": "^9.1.0",
     "factory-girl-ts": "^2.3.1",
-    "knex-scripts": "^0.3.6",
     "moment": "^2.30.1",
     "openapi3-ts": "^4.4.0",
     "rimraf": "^6.0.1",

eslint.config.mjs

@@ -4,7 +4,7 @@ import tseslint from "typescript-eslint";
 
 const config = tseslint.config(
   {
-    name: "argos/global-ignoes",
+    name: "argos/global-ignores",
     ignores: [
       "**/dist",
       "apps/backend/src/graphql/__generated__",
@@ -16,6 +16,7 @@ const config = tseslint.config(
   {
     name: "argos/custom-ts-rules",
     rules: {
+      curly: "error",
       "@typescript-eslint/no-explicit-any": "off",
       "@typescript-eslint/no-unused-vars": [
         "error",

packages/knex-scripts/.swcrc

@@ -0,0 +1,14 @@
+{
+  "$schema": "https://json.schemastore.org/swcrc",
+  "jsc": {
+    "parser": {
+      "syntax": "typescript"
+    },
+    "target": "es2022"
+  },
+  "module": {
+    "type": "es6",
+    "resolveFully": true
+  },
+  "sourceMaps": false
+}

packages/knex-scripts/bin/knex-scripts.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+
+import "../dist/cli.js";

packages/knex-scripts/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "@argos/knex-scripts",
+  "version": "2.0.0",
+  "private": true,
+  "bin": {
+    "knex-scripts": "./bin/knex-scripts.js"
+  },
+  "type": "module",
+  "exports": {
+    "./package.json": "./package.json"
+  },
+  "scripts": {
+    "build": "rm -rf dist && swc src -d dist --strip-leading-paths",
+    "watch-build": "pnpm run build -- --watch --quiet",
+    "check-types": "tsc --noEmit",
+    "check-format": "prettier --check --cache --ignore-path=../../.gitignore --ignore-path=../../.prettierignore .",
+    "lint": "eslint ."
+  },
+  "sideEffects": false,
+  "dependencies": {
+    "commander": "^12.1.0",
+    "fast-glob": "^3.3.2",
+    "ora": "^8.1.1"
+  },
+  "peerDependencies": {
+    "knex": "^3.0.0"
+  },
+  "devDependencies": {
+    "@argos/tsconfig": "workspace:*"
+  }
+}

packages/knex-scripts/src/check-structure.ts

@@ -0,0 +1,63 @@
+import { readFile } from "node:fs/promises";
+import { Command } from "commander";
+import { oraPromise } from "ora";
+
+import { getConfig } from "./config.js";
+import { getInsertsFromMigrations } from "./utils.js";
+
+/**
+ * Get inserts from the structure file.
+ */
+async function getInsertsFromStructure(input: {
+  structurePath: string;
+}): Promise<string[]> {
+  const structure = await readFile(input.structurePath, "utf-8");
+  const regex =
+    /INSERT INTO public\.knex_migrations\(name, batch, migration_time\) VALUES \('.*', 1, NOW\(\)\);/g;
+  const inserts = [];
+
+  let match: RegExpExecArray | null;
+  while ((match = regex.exec(structure))) {
+    inserts.push(match[0]);
+  }
+
+  return inserts;
+}
+
+/**
+ * Check if the structure is up to date.
+ */
+async function checkIsStructureUpToDate() {
+  const config = await getConfig();
+  const [migrationsInFolder, migrationsInStructure] = await Promise.all([
+    getInsertsFromMigrations(config),
+    getInsertsFromStructure(config),
+  ]);
+
+  if (migrationsInFolder.length !== migrationsInStructure.length) {
+    return false;
+  }
+
+  return migrationsInFolder.every(
+    (insert, index) => migrationsInStructure[index] === insert,
+  );
+}
+
+export function addCheckStructureCommand(program: Command) {
+  program
+    .command("check-structure")
+    .description(
+      "Compare the dumped structure with the structure in the database.",
+    )
+    .action(async () => {
+      await oraPromise(
+        (async () => {
+          const isUpToDate = await checkIsStructureUpToDate();
+          if (!isUpToDate) {
+            throw new Error("Structure is outdated.");
+          }
+        })(),
+        "Checking structure...",
+      );
+    });
+}

packages/knex-scripts/src/cli.ts

@@ -0,0 +1,34 @@
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import { fileURLToPath, URL } from "node:url";
+import { program } from "commander";
+
+import { addCheckStructureCommand } from "./check-structure.js";
+import { addCreateCommand } from "./create.js";
+import { addDropCommand } from "./drop.js";
+import { addDumpCommand } from "./dump.js";
+import { addLoadCommand } from "./load.js";
+import { addTruncateCommand } from "./truncate.js";
+
+const __dirname = fileURLToPath(new URL(".", import.meta.url));
+
+const rawPkg = await readFile(resolve(__dirname, "..", "package.json"), "utf8");
+const pkg = JSON.parse(rawPkg);
+
+program
+  .name(pkg.name)
+  .version(pkg.version)
+  .description("CLI tool to manage PostgresSQL database over Knex.js.");
+
+addCheckStructureCommand(program);
+addCreateCommand(program);
+addDropCommand(program);
+addDumpCommand(program);
+addLoadCommand(program);
+addTruncateCommand(program);
+
+if (!process.argv.slice(2).length) {
+  program.outputHelp();
+} else {
+  program.parse(process.argv);
+}

packages/knex-scripts/src/config.ts

@@ -0,0 +1,35 @@
+import { join } from "node:path";
+import type { Knex } from "knex";
+
+type Config = {
+  knexConfig: Knex.Config;
+  structurePath: string;
+  migrationsPath: string;
+};
+
+async function readKnexConfig(): Promise<Knex.Config> {
+  const knexFile = join(process.cwd(), "knexfile.js");
+  try {
+    const config: unknown = await import(knexFile);
+    if (
+      !config ||
+      typeof config !== "object" ||
+      !("default" in config) ||
+      !config.default
+    ) {
+      throw new Error(`Invalid knexfile.js`);
+    }
+    return config.default;
+  } catch {
+    throw new Error(`Could not find ${knexFile}`);
+  }
+}
+
+export async function getConfig(): Promise<Config> {
+  const knexConfig = await readKnexConfig();
+  return {
+    knexConfig,
+    structurePath: join(process.cwd(), "db/structure.sql"),
+    migrationsPath: join(process.cwd(), "db/migrations"),
+  };
+}

packages/knex-scripts/src/create.ts

@@ -0,0 +1,36 @@
+import { Command } from "commander";
+import { oraPromise } from "ora";
+
+import { getConfig } from "./config.js";
+import {
+  getCommandEnv,
+  getPostgresCommand,
+  preventRunningInProduction,
+  runCommand,
+} from "./utils.js";
+
+/**
+ * Create the database based on the configuration in the knexfile.
+ */
+async function createDatabase() {
+  preventRunningInProduction();
+
+  const config = await getConfig();
+  const env = getCommandEnv(config);
+  const { command, args } = getPostgresCommand(config, "createdb");
+  await runCommand({ command, args, env });
+}
+
+/**
+ * Add the "create" command to the program.
+ */
+export function addCreateCommand(program: Command) {
+  program
+    .command("create")
+    .description(
+      "Create the database based on the configuration in the knexfile.",
+    )
+    .action(async () => {
+      await oraPromise(createDatabase(), "Creating database...");
+    });
+}

packages/knex-scripts/src/drop.ts

@@ -0,0 +1,38 @@
+import { Command } from "commander";
+import { oraPromise } from "ora";
+
+import { getConfig } from "./config.js";
+import {
+  getCommandEnv,
+  getPostgresCommand,
+  preventRunningInProduction,
+  runCommand,
+} from "./utils.js";
+
+/**
+ * Drop the database based on the configuration in the knexfile.
+ */
+async function dropDatabase() {
+  preventRunningInProduction();
+
+  const config = await getConfig();
+  const env = getCommandEnv(config);
+  const { command, args } = getPostgresCommand(config, "dropdb", [
+    "--if-exists",
+  ]);
+  await runCommand({ command, args, env });
+}
+
+/**
+ * Add the "drop" command to the program.
+ */
+export function addDropCommand(program: Command) {
+  program
+    .command("drop")
+    .description(
+      "Drop the database based on the configuration in the knexfile.",
+    )
+    .action(async () => {
+      await oraPromise(dropDatabase(), "Dropping database...");
+    });
+}

packages/knex-scripts/src/dump.ts

@@ -0,0 +1,50 @@
+import { appendFile, mkdir } from "fs/promises";
+import { dirname } from "path";
+import { Command } from "commander";
+import { oraPromise } from "ora";
+
+import { getConfig } from "./config.js";
+import {
+  getCommandEnv,
+  getInsertsFromMigrations,
+  getPostgresCommand,
+  requireEnv,
+  runCommand,
+} from "./utils.js";
+
+/**
+ * Dump the database schema to a file.
+ */
+async function dumpDatabaseSchema() {
+  const config = await getConfig();
+  requireEnv("development");
+
+  await mkdir(dirname(config.structurePath), { recursive: true });
+
+  const env = getCommandEnv(config);
+  const { command, args } = getPostgresCommand(config, "pg_dump", [
+    "--schema-only",
+    "-f",
+    config.structurePath,
+  ]);
+
+  await runCommand({ command, args, env });
+
+  const migrationInserts = await getInsertsFromMigrations(config);
+  await appendFile(
+    config.structurePath,
+    `-- Knex migrations\n\n${migrationInserts.join("\n")}`,
+  );
+}
+
+/**
+ * Add the "dump" command to the program.
+ */
+export function addDumpCommand(program: Command) {
+  program
+    .command("dump")
+    .description("Dump the database schema to a file.")
+    .action(async () => {
+      await oraPromise(dumpDatabaseSchema(), "Dumping database schema...");
+    });
+}

packages/knex-scripts/src/load.ts

@@ -0,0 +1,42 @@
+import { Command } from "commander";
+import { oraPromise } from "ora";
+
+import { getConfig } from "./config.js";
+import {
+  getCommandEnv,
+  getPostgresCommand,
+  preventRunningInProduction,
+  runCommand,
+} from "./utils.js";
+
+/**
+ * Load database schema from file.
+ */
+async function loadDatabaseSchema() {
+  preventRunningInProduction();
+
+  const config = await getConfig();
+
+  const env = getCommandEnv(config);
+
+  const { command, args } = getPostgresCommand(config, "psql", [
+    "-v",
+    "ON_ERROR_STOP=1",
+    "-f",
+    config.structurePath,
+  ]);
+
+  await runCommand({ command, args, env });
+}
+
+/**
+ * Add the "load" command to the program.
+ */
+export function addLoadCommand(program: Command) {
+  program
+    .command("load")
+    .description("Load the database schema from a file.")
+    .action(async () => {
+      await oraPromise(loadDatabaseSchema(), "Loading database schema...");
+    });
+}