Multiple worker support

cardstack · Jan 13, 2025 · c55ed0c · c55ed0c
1 parent e03162c
commit c55ed0c
Show file tree

Hide file tree

Showing 17 changed files with 445 additions and 80 deletions.
diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml
@@ -74,10 +74,21 @@ jobs:
     with:
       repository: "boxel-realm-server-${{ inputs.environment }}"
       environment: ${{ inputs.environment }}
-      dockerfile: "packages/realm-server/Dockerfile"
+      dockerfile: "packages/realm-server/realm-server.Dockerfile"
       build-args: |
         "realm_server_script=start:${{ inputs.environment }}"
 
+  build-worker:
+    name: Build worker Docker image
+    uses: cardstack/gh-actions/.github/workflows/docker-ecr.yml@main
+    secrets: inherit
+    with:
+      repository: "boxel-realm-server-${{ inputs.environment }}"
+      environment: ${{ inputs.environment }}
+      dockerfile: "packages/realm-server/worker.Dockerfile"
+      build-args: |
+        "worker_script=start:worker-${{ inputs.environment }}"
+
   build-pg-migration:
     name: Build pg-migration Docker image
     uses: cardstack/gh-actions/.github/workflows/docker-ecr.yml@main
@@ -110,9 +121,22 @@ jobs:
     steps:
       - run: sleep 240
 
+  deploy-worker:
+    name: Deploy worker
+    needs: [build-realm-server, deploy-host, post-migrate-db]
+    uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
+    secrets: inherit
+    with:
+      container-name: "boxel-worker"
+      environment: ${{ inputs.environment }}
+      cluster: ${{ inputs.environment }}
+      service-name: "boxel-worker-${{ inputs.environment }}"
+      image: ${{ needs.build-worker.outputs.image }}
+      wait-for-service-stability: false
+
   deploy-realm-server:
     name: Deploy realm server
-    needs: [build-realm-server, deploy-host, post-migrate-db]
+    needs: [deploy-worker]
     uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
     secrets: inherit
     with:

diff --git a/README.md b/README.md
@@ -73,7 +73,7 @@ Live reloads are not available in this mode, however, if you use start the serve
 
 #### Using `start:all`
 
-Instead of running `pnpm start:base`, you can alternatively use `pnpm start:all` which also serves a few other realms on other ports--this is convenient if you wish to switch between the app and the tests without having to restart servers. Here's what is spun up with `start:all`:
+Instead of running `pnpm start:base`, you can alternatively use `pnpm start:all` which also serves a few other realms on other ports--this is convenient if you wish to switch between the app and the tests without having to restart servers. Use the environment variable `WORKER_COUNT` to add additional workers. By default there is 1 worker for each realm server. Here's what is spun up with `start:all`:
 
 | Port  | Description                                                   | Running `start:all` | Running `start:base` |
 | ----- | ------------------------------------------------------------- | ------------------- | -------------------- |
@@ -82,13 +82,16 @@ Instead of running `pnpm start:base`, you can alternatively use `pnpm start:all`
 | :4201 | `/seed` seed realm                                            | ✅                  | 🚫                   |
 | :4202 | `/test` host test realm, `/node-test` node test realm         | ✅                  | 🚫                   |
 | :4205 | `/test` realm for matrix client tests (playwright controlled) | 🚫                  | 🚫                   |
+| :4210 | Development Worker Manager (spins up 1 worker by default)     | ✅                  | 🚫                   |
+| :4211 | Test Worker Manager (spins up 1 worker by default)            | ✅                  | 🚫                   |
+| :4212 | Test Worker Manager for matrix client tests (playwright controlled - 1 worker) | ✅                  | 🚫                   |
 | :5001 | Mail user interface for viewing emails sent to local SMTP     | ✅                  | 🚫                   |
 | :5435 | Postgres DB                                                   | ✅                  | 🚫                   |
 | :8008 | Matrix synapse server                                         | ✅                  | 🚫                   |
 
 #### Using `start:development`
 
-You can also use `start:development` if you want the functionality of `start:all`, but without running the test realms. `start:development` will enable you to open http://localhost:4201 and allow to select between the cards in the /base and /experiments realm.
+You can also use `start:development` if you want the functionality of `start:all`, but without running the test realms. `start:development` will enable you to open http://localhost:4201 and allow to select between the cards in the /base and /experiments realm. In order to use `start:development` you must also make sure to run `start:worker-development` in order to start the workers (which are normally started in `start:all`.
 
 ### Card Pre-rendering
 

diff --git a/packages/matrix/helpers/isolated-realm-server.ts b/packages/matrix/helpers/isolated-realm-server.ts
@@ -31,6 +31,36 @@ export async function startServer() {
   process.env.MATRIX_URL = 'http://localhost:8008';
   process.env.REALM_SERVER_MATRIX_USERNAME = 'realm_server';
 
+  let worker = spawn(
+    'ts-node',
+    [
+      `--transpileOnly`,
+      'worker-manager',
+      `--port=4212`,
+      `--matrixURL='http://localhost:8008'`,
+      `--distURL="${process.env.HOST_URL ?? 'http://localhost:4200'}"`,
+
+      `--fromUrl='http://localhost:4205/test/'`,
+      `--toUrl='http://localhost:4205/test/'`,
+      `--fromUrl='https://cardstack.com/base/'`,
+      `--toUrl='http://localhost:4201/base/'`,
+    ],
+    {
+      cwd: realmServerDir,
+      stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
+    },
+  );
+  if (worker.stdout) {
+    worker.stdout.on('data', (data: Buffer) =>
+      console.log(`worker: ${data.toString()}`),
+    );
+  }
+  if (worker.stderr) {
+    worker.stderr.on('data', (data: Buffer) =>
+      console.error(`worker: ${data.toString()}`),
+    );
+  }
+
   let realmServer = spawn(
     'ts-node',
     [
@@ -40,13 +70,14 @@ export async function startServer() {
       `--matrixURL='http://localhost:8008'`,
       `--realmsRootPath='${dir.name}'`,
       `--seedPath='${seedPath}'`,
+      `--workerManagerPort=4212`,
       `--migrateDB`,
       `--useRegistrationSecretFunction`,
 
       `--path='${testRealmDir}'`,
       `--username='test_realm'`,
-      `--fromUrl='/test/'`,
-      `--toUrl='/test/'`,
+      `--fromUrl='http://localhost:4205/test/'`,
+      `--toUrl='http://localhost:4205/test/'`,
       `--fromUrl='https://cardstack.com/base/'`,
       `--toUrl='http://localhost:4201/base/'`,
     ],

diff --git a/packages/realm-server/main.ts b/packages/realm-server/main.ts
@@ -11,13 +11,12 @@ import { NodeAdapter } from './node-realm';
 import yargs from 'yargs';
 import { RealmServer } from './server';
 import { resolve } from 'path';
-import { spawn } from 'child_process';
+import { createConnection, type Socket } from 'net';
 import { makeFastBootIndexRunner } from './fastboot';
 import { shimExternals } from './lib/externals';
 import * as Sentry from '@sentry/node';
 import { PgAdapter, PgQueuePublisher } from '@cardstack/postgres';
 import { MatrixClient } from '@cardstack/runtime-common/matrix-client';
-import flattenDeep from 'lodash/flattenDeep';
 import 'decorator-transforms/globals';
 
 let log = logger('main');
@@ -68,6 +67,7 @@ let {
   useRegistrationSecretFunction,
   seedPath,
   migrateDB,
+  workerManagerPort,
 } = yargs(process.argv.slice(2))
   .usage('Start realm server')
   .options({
@@ -130,6 +130,11 @@ let {
         'The flag should be set when running matrix tests where the synapse instance is torn down and restarted multiple times during the life of the realm server.',
       type: 'boolean',
     },
+    workerManagerPort: {
+      description:
+        'The port the worker manager is running on. used to wait for the workers to be ready',
+      type: 'number',
+    },
   })
   .parseSync();
 
@@ -165,8 +170,8 @@ let virtualNetwork = new VirtualNetwork();
 shimExternals(virtualNetwork);
 
 let urlMappings = fromUrls.map((fromUrl, i) => [
-  new URL(String(fromUrl), `http://localhost:${port}`),
-  new URL(String(toUrls[i]), `http://localhost:${port}`),
+  new URL(String(fromUrl)),
+  new URL(String(toUrls[i])),
 ]);
 for (let [from, to] of urlMappings) {
   virtualNetwork.addURLMapping(from, to);
@@ -185,7 +190,9 @@ let autoMigrate = migrateDB || undefined;
     manager.getOptions.bind(manager),
   );
 
-  await startWorker({ autoMigrate });
+  if (workerManagerPort != null) {
+    await waitForWorkerManager(workerManagerPort);
+  }
 
   for (let [i, path] of paths.entries()) {
     let url = hrefs[i][0];
@@ -324,51 +331,49 @@ let autoMigrate = migrateDB || undefined;
   process.exit(-3);
 });
 
-async function startWorker(opts?: { autoMigrate?: true }) {
-  let worker = spawn(
-    'ts-node',
-    [
-      '--transpileOnly',
-      'worker',
-      `--port=${port}`,
-      `--matrixURL='${matrixURL}'`,
-      `--distURL='${distURL}'`,
-      ...(opts?.autoMigrate ? [`--migrateDB`] : []),
-      ...flattenDeep(
-        urlMappings.map(([from, to]) => [
-          `--fromUrl='${from}'`,
-          `--toUrl='${to}'`,
-        ]),
-      ),
-    ],
-    {
-      stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
-    },
-  );
+let workerReadyDeferred: Deferred<boolean> | undefined;
+async function waitForWorkerManager(port: number) {
+  const workerManager = await new Promise<Socket>((r) => {
+    let socket = createConnection({ port }, () => {
+      log.info(`Connected to worker manager on port ${port}`);
+      r(socket);
+    });
+  });
 
-  if (worker.stdout) {
-    worker.stdout.on('data', (data: Buffer) =>
-      log.info(`worker: ${data.toString()}`),
-    );
-  }
-  if (worker.stderr) {
-    worker.stderr.on('data', (data: Buffer) =>
-      console.error(`worker: ${data.toString()}`),
-    );
-  }
+  workerManager.on('data', (data) => {
+    let res = data.toString();
+    if (!workerReadyDeferred) {
+      throw new Error(
+        `received unsolicited message from worker manager on port ${port}`,
+      );
+    }
+    switch (res) {
+      case 'ready':
+      case 'not-ready':
+        workerReadyDeferred.fulfill(res === 'ready' ? true : false);
+        break;
+      default:
+        workerReadyDeferred.reject(
+          `unexpected response from worker manager: ${res}`,
+        );
+    }
+  });
 
-  let timeout = await Promise.race([
-    new Promise<void>((r) => {
-      worker.on('message', (message) => {
-        if (message === 'ready') {
-          r();
-        }
-      });
-    }),
-    new Promise<true>((r) => setTimeout(() => r(true), 30_000)),
-  ]);
-  if (timeout) {
-    console.error(`timed-out waiting for worker to start. Stopping server`);
-    process.exit(-2);
+  try {
+    let isReady = false;
+    let timeout = Date.now() + 30_000;
+    do {
+      workerReadyDeferred = new Deferred();
+      workerManager.write('ready?');
+      isReady = await workerReadyDeferred.promise;
+    } while (!isReady && Date.now() < timeout);
+    if (!isReady) {
+      throw new Error(
+        `timed out trying to connect to worker manager on port ${port}`,
+      );
+    }
+  } finally {
+    workerManager.end();
   }
+  log.info('workers are ready');
 }
diff --git a/packages/realm-server/package.json b/packages/realm-server/package.json
@@ -24,6 +24,7 @@
     "@types/lodash": "^4.14.182",
     "@types/mime-types": "^2.1.1",
     "@types/node": "^18.18.5",
+    "@types/pluralize": "^0.0.30",
     "@types/qs": "^6.9.17",
     "@types/qunit": "^2.11.3",
     "@types/sane": "^2.0.1",
@@ -52,6 +53,7 @@
     "loglevel": "^1.8.1",
     "mime-types": "^2.1.35",
     "npm-run-all": "^4.1.5",
+    "pluralize": "^8.0.0",
     "prettier": "^2.8.4",
     "prettier-plugin-ember-template-tag": "^1.1.0",
     "qs": "^6.13.0",
@@ -85,12 +87,16 @@
     "setup:catalog-in-deployment": "mkdir -p /persistent/catalog && rsync --dry-run --itemize-changes --size-only --recursive --delete ../catalog-realm/. /persistent/catalog/ && rsync --size-only --recursive --delete ../catalog-realm/. /persistent/catalog/",
     "start": "PGPORT=5435 NODE_NO_WARNINGS=1 ts-node --transpileOnly main",
     "start:base": "./scripts/start-base.sh",
-    "start:test-realms": "./scripts/start-test-realms.sh",
+    "start:test-realms": "./scripts/start-test-realms.sh --workerManagerPort=4211",
     "start:all": "./scripts/start-all.sh",
     "start:without-matrix": "./scripts/start-without-matrix.sh",
     "start:staging": "./scripts/start-staging.sh",
-    "start:development": "./scripts/start-development.sh",
+    "start:development": "./scripts/start-development.sh --workerManagerPort=4210",
     "start:production": "./scripts/start-production.sh",
+    "start:worker-development": "./scripts/start-worker-development.sh",
+    "start:worker-test": "./scripts/start-worker-test.sh",
+    "start:worker-staging": "./scripts/start-worker-staging.sh",
+    "start:worker-production": "./scripts/start-worker-production.sh",
     "start:services-for-matrix-tests": "./scripts/start-services-for-matrix-tests.sh",
     "wait": "sleep 10000000",
     "lint": "concurrently \"pnpm:lint:*(!fix)\" --names \"lint:\"",

diff --git a/packages/realm-server/Dockerfile → ...ages/realm-server/realm-server.Dockerfile b/packages/realm-server/Dockerfile → ...ages/realm-server/realm-server.Dockerfile
diff --git a/packages/realm-server/scripts/start-all.sh b/packages/realm-server/scripts/start-all.sh
@@ -1,8 +1,8 @@
 #! /bin/sh
 
 NODE_NO_WARNINGS=1 start-server-and-test \
-  'run-p start:pg start:matrix start:smtp start:development' \
+  'run-p start:pg start:matrix start:smtp start:worker-development start:development' \
   'http-get://localhost:4201/base/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson|http-get://localhost:4201/experiments/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson|http://localhost:8008|http://localhost:5001' \
-  'run-p start:test-realms' \
+  'run-p start:worker-test start:test-realms' \
   'http-get://localhost:4202/node-test/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson' \
   'wait'
diff --git a/packages/realm-server/scripts/start-development.sh b/packages/realm-server/scripts/start-development.sh
@@ -24,6 +24,7 @@ NODE_ENV=development \
   --realmsRootPath='./realms/localhost_4201' \
   --seedPath='../seed-realm' \
   --migrateDB \
+  $1 \
   \
   --path='../base' \
   --username='base_realm' \

diff --git a/packages/realm-server/scripts/start-test-realms.sh b/packages/realm-server/scripts/start-test-realms.sh
@@ -23,15 +23,16 @@ NODE_ENV=test \
   --realmsRootPath='./realms/localhost_4202' \
   --matrixRegistrationSecretFile='../matrix/registration_secret.txt' \
   --migrateDB \
+  $1 \
   \
   --path='./tests/cards' \
   --username='node-test_realm' \
-  --fromUrl='/node-test/' \
-  --toUrl='/node-test/' \
+  --fromUrl='http://localhost:4202/node-test/' \
+  --toUrl='http://localhost:4202/node-test/' \
   \
   --path='../host/tests/cards' \
   --username='test_realm' \
-  --fromUrl='/test/' \
-  --toUrl='/test/' \
+  --fromUrl='http://localhost:4202/test/' \
+  --toUrl='http://localhost:4202/test/' \
   --fromUrl='https://cardstack.com/base/' \
   --toUrl='http://localhost:4201/base/'
diff --git a/packages/realm-server/scripts/start-worker-development.sh b/packages/realm-server/scripts/start-worker-development.sh
@@ -0,0 +1,30 @@
+#! /bin/sh
+SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)"
+. "$SCRIPTS_DIR/wait-for-pg.sh"
+
+wait_for_postgres
+
+NODE_ENV=development \
+  NODE_NO_WARNINGS=1 \
+  PGPORT=5435 \
+  PGDATABASE=boxel \
+  LOG_LEVELS='*=info' \
+  REALM_SECRET_SEED="shhh! it's a secret" \
+  ts-node \
+  --transpileOnly worker-manager \
+  --count="${WORKER_COUNT:-1}" \
+  --port=4210 \
+  --matrixURL='http://localhost:8008' \
+  --distURL="${HOST_URL:-http://localhost:4200}" \
+  \
+  --fromUrl='https://cardstack.com/base/' \
+  --toUrl='http://localhost:4201/base/' \
+  \
+  --fromUrl='http://localhost:4201/experiments/' \
+  --toUrl='http://localhost:4201/experiments/' \
+  \
+  --fromUrl='http://localhost:4201/seed/' \
+  --toUrl='http://localhost:4201/seed/' \
+  \
+  --fromUrl='http://localhost:4201/catalog/' \
+  --toUrl='http://localhost:4201/catalog/'