Merge pull request #2015 from cardstack/shard-server-tests

shard realm server tests

.github/workflows/ci.yaml

@@ -234,8 +234,24 @@ jobs:
     name: Realm Server Tests
     runs-on: ubuntu-latest
     concurrency:
-      group: realm-server-test-${{ github.head_ref || github.run_id }}
+      group: realm-server-test-${{ matrix.testModule }}-${{ github.head_ref || github.run_id }}
       cancel-in-progress: true
+    strategy:
+      fail-fast: false
+      matrix:
+        testModule:
+          [
+            "auth-client-test.ts",
+            "billing-test.ts",
+            "index-query-engine-test.ts",
+            "index-writer-test.ts",
+            "indexing-test.ts",
+            "loader-test.ts",
+            "module-syntax-test.ts",
+            "queue-test.ts",
+            "realm-server-test.ts",
+            "virtual-network-test.ts",
+          ]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/init
@@ -265,11 +281,13 @@ jobs:
       - name: realm server test suite
         run: pnpm test:wait-for-servers
         working-directory: packages/realm-server
+        env:
+          TEST_MODULE: ${{matrix.testModule}}
       - name: Upload realm server log
         uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: realm-server-log
+          name: realm-server-log-${{matrix.testModule}}
           path: /tmp/server.log
           retention-days: 30
 

packages/realm-server/package.json

@@ -14,6 +14,7 @@
     "@types/eventsource": "^1.1.11",
     "@types/flat": "^5.0.5",
     "@types/fs-extra": "^9.0.13",
+    "@types/js-yaml": "^4.0.9",
     "@types/jsdom": "^21.1.1",
     "@types/jsonwebtoken": "^9.0.5",
     "@types/koa": "^2.13.5",
@@ -41,6 +42,7 @@
     "flat": "^5.0.2",
     "fs-extra": "^10.1.0",
     "http-server": "^14.1.1",
+    "js-yaml": "^4.1.0",
     "jsdom": "^21.1.1",
     "jsonwebtoken": "^9.0.2",
     "koa": "^2.14.1",
@@ -71,11 +73,12 @@
   },
   "scripts": {
     "test": "./scripts/remove-test-dbs.sh; LOG_LEVELS=\"pg-adapter=warn,realm:requests=warn,current-run=error${LOG_LEVELS:+,}${LOG_LEVELS}\" NODE_NO_WARNINGS=1 PGPORT=5435 STRIPE_WEBHOOK_SECRET=stripe-webhook-secret STRIPE_API_KEY=stripe-api-key qunit --require ts-node/register/transpile-only tests/index.ts",
+    "test-module": "./scripts/remove-test-dbs.sh; LOG_LEVELS=\"pg-adapter=warn,realm:requests=warn,current-run=error${LOG_LEVELS:+,}${LOG_LEVELS}\" NODE_NO_WARNINGS=1 PGPORT=5435 STRIPE_WEBHOOK_SECRET=stripe-webhook-secret STRIPE_API_KEY=stripe-api-key qunit --require ts-node/register/transpile-only --module ${TEST_MODULE} tests/index.ts",
     "start:matrix": "cd ../matrix && pnpm assert-synapse-running",
     "start:smtp": "cd ../matrix && pnpm assert-smtp-running",
     "start:pg": "./scripts/start-pg.sh",
     "stop:pg": "./scripts/stop-pg.sh",
-    "test:wait-for-servers": "NODE_NO_WARNINGS=1 start-server-and-test 'pnpm run wait' 'http-get://localhost:4201/base/fields/boolean-field?acceptHeader=application%2Fvnd.card%2Bjson' 'pnpm run wait' 'http-get://localhost:4202/node-test/person-1?acceptHeader=application%2Fvnd.card%2Bjson|http://localhost:8008|http://localhost:5001' 'test'",
+    "test:wait-for-servers": "NODE_NO_WARNINGS=1 start-server-and-test 'pnpm run wait' 'http-get://localhost:4201/base/fields/boolean-field?acceptHeader=application%2Fvnd.card%2Bjson' 'pnpm run wait' 'http-get://localhost:4202/node-test/person-1?acceptHeader=application%2Fvnd.card%2Bjson|http://localhost:8008|http://localhost:5001' 'test-module'",
     "setup:base-in-deployment": "mkdir -p /persistent/base && rsync --dry-run --itemize-changes --size-only --recursive --delete ../base/. /persistent/base/ && rsync --size-only --recursive --delete ../base/. /persistent/base/",
     "setup:experiments-in-deployment": "mkdir -p /persistent/experiments && rsync --dry-run --itemize-changes --size-only --recursive ../experiments-realm/. /persistent/experiments/ && rsync --size-only --recursive ../experiments-realm/. /persistent/experiments/",
     "setup:seed-in-deployment": "mkdir -p /persistent/seed && rsync --dry-run --itemize-changes --size-only --recursive --delete ../seed-realm/. /persistent/seed/ && rsync --size-only --recursive --delete ../seed-realm/. /persistent/seed/",
@@ -95,6 +98,7 @@
     "lint:js": "eslint . --cache",
     "lint:js:fix": "eslint . --fix",
     "lint:glint": "glint",
+    "lint:test-shards": "ts-node --transpileOnly scripts/lint-test-shards.ts",
     "full-reset": "./scripts/full-reset.sh",
     "sync-stripe-products": "NODE_NO_WARNINGS=1 PGDATABASE=boxel PGPORT=5435 ts-node --transpileOnly scripts/sync-stripe-products.ts",
     "stripe": "docker run --rm --add-host=host.docker.internal:host-gateway -it stripe/stripe-cli:latest"

packages/realm-server/scripts/lint-test-shards.ts

@@ -0,0 +1,83 @@
+import { readFileSync, readdirSync } from 'fs-extra';
+import yaml from 'js-yaml';
+import { join, basename } from 'path';
+
+const YAML_FILE = join(
+  __dirname,
+  '..',
+  '..',
+  '..',
+  '.github',
+  'workflows',
+  'ci.yaml',
+);
+const TEST_DIR = join(__dirname, '..', 'tests');
+
+function getCiTestModules(yamlFilePath: string) {
+  try {
+    const yamlContent = readFileSync(yamlFilePath, 'utf8');
+    const yamlData = yaml.load(yamlContent) as Record<string, any>;
+
+    const shardIndexes: string[] =
+      yamlData?.jobs?.['realm-server-test']?.strategy?.matrix?.testModule;
+
+    if (!Array.isArray(shardIndexes)) {
+      throw new Error(
+        `Invalid 'jobs.realm-server-test.strategy.matrix.testModule' format in the YAML file.`,
+      );
+    }
+
+    return shardIndexes;
+  } catch (error: any) {
+    console.error(`Error reading shardIndex from YAML file: ${error.message}`);
+    process.exit(1);
+  }
+}
+
+function getFilesystemTestModules(testDir: string) {
+  try {
+    const files = readdirSync(testDir);
+    return files
+      .filter((file) => file.endsWith('-test.ts'))
+      .map((file) => basename(file));
+  } catch (error: any) {
+    console.error(
+      `Error reading test files from dir ${testDir}: ${error.message}`,
+    );
+    process.exit(1);
+  }
+}
+
+function validateTestFiles(yamlFilePath: string, testDir: string) {
+  const ciTestModules = getCiTestModules(yamlFilePath);
+  const filesystemTestModules = getFilesystemTestModules(testDir);
+
+  let errorFound = false;
+
+  for (let filename of filesystemTestModules) {
+    if (!ciTestModules.includes(filename)) {
+      console.error(
+        `Error: Test file '${filename}' exists in the filesystem but not in the ${yamlFilePath} file.`,
+      );
+      errorFound = true;
+    }
+  }
+  for (let filename of ciTestModules) {
+    if (!filesystemTestModules.includes(filename)) {
+      console.error(
+        `Error: Test file '${filename}' exists in the YAML file but not in the filesystem.`,
+      );
+      errorFound = true;
+    }
+  }
+
+  if (errorFound) {
+    process.exit(1);
+  } else {
+    console.log(
+      `All test files are accounted for in the ${yamlFilePath} file for the realm-server matrix strategy.`,
+    );
+  }
+}
+
+validateTestFiles(YAML_FILE, TEST_DIR);

packages/realm-server/tests/auth-client-test.ts

@@ -7,93 +7,104 @@ import {
 } from '@cardstack/runtime-common/realm-auth-client';
 import { VirtualNetwork } from '@cardstack/runtime-common';
 import jwt from 'jsonwebtoken';
+import { basename } from 'path';
 
 function createJWT(expiresIn: string | number) {
   return jwt.sign({}, 'secret', { expiresIn });
 }
 
-module('realm-auth-client', function (assert) {
-  let client: RealmAuthClient;
+module(basename(__filename), function () {
+  module('realm-auth-client', function (assert) {
+    let client: RealmAuthClient;
 
-  assert.beforeEach(function () {
-    let mockMatrixClient = {
-      isLoggedIn() {
-        return true;
-      },
-      getUserId() {
-        return 'userId';
-      },
-      async getJoinedRooms() {
-        return Promise.resolve({ joined_rooms: [] });
-      },
-      async joinRoom() {
-        return Promise.resolve();
-      },
-      async sendEvent() {
-        return Promise.resolve();
-      },
-      async hashMessageWithSecret(_message: string): Promise<string> {
-        throw new Error('Method not implemented.');
-      },
-    } as RealmAuthMatrixClientInterface;
+    assert.beforeEach(function () {
+      let mockMatrixClient = {
+        isLoggedIn() {
+          return true;
+        },
+        getUserId() {
+          return 'userId';
+        },
+        async getJoinedRooms() {
+          return Promise.resolve({ joined_rooms: [] });
+        },
+        async joinRoom() {
+          return Promise.resolve();
+        },
+        async sendEvent() {
+          return Promise.resolve();
+        },
+        async hashMessageWithSecret(_message: string): Promise<string> {
+          throw new Error('Method not implemented.');
+        },
+      } as RealmAuthMatrixClientInterface;
 
-    let virtualNetwork = new VirtualNetwork();
+      let virtualNetwork = new VirtualNetwork();
 
-    client = new RealmAuthClient(
-      new URL('http://testrealm.com/'),
-      mockMatrixClient,
-      virtualNetwork.fetch,
-    ) as any;
+      client = new RealmAuthClient(
+        new URL('http://testrealm.com/'),
+        mockMatrixClient,
+        virtualNetwork.fetch,
+      ) as any;
 
-    // [] notation is a hack to make TS happy so we can set private properties with mocks
-    client['initiateSessionRequest'] = async function (): Promise<Response> {
-      return {
-        status: 401,
-        json() {
-          return Promise.resolve({
-            room: 'room',
-            challenge: 'challenge',
-          });
-        },
-      } as Response;
-    };
-    client['challengeRequest'] = async function (): Promise<Response> {
-      return {
-        ok: true,
-        headers: {
-          get() {
-            return createJWT('1h');
+      // [] notation is a hack to make TS happy so we can set private properties with mocks
+      client['initiateSessionRequest'] = async function (): Promise<Response> {
+        return {
+          status: 401,
+          json() {
+            return Promise.resolve({
+              room: 'room',
+              challenge: 'challenge',
+            });
           },
-        },
-      } as unknown as Response;
-    };
-  });
+        } as Response;
+      };
+      client['challengeRequest'] = async function (): Promise<Response> {
+        return {
+          ok: true,
+          headers: {
+            get() {
+              return createJWT('1h');
+            },
+          },
+        } as unknown as Response;
+      };
+    });
 
-  test('it authenticates and caches the jwt until it expires', async function (assert) {
-    let jwtFromClient = await client.getJWT();
+    test('it authenticates and caches the jwt until it expires', async function (assert) {
+      let jwtFromClient = await client.getJWT();
 
-    assert.strictEqual(
-      jwtFromClient.split('.').length,
-      3,
-      'jwtFromClient looks like a jwt',
-    );
+      assert.strictEqual(
+        jwtFromClient.split('.').length,
+        3,
+        'jwtFromClient looks like a jwt',
+      );
 
-    assert.strictEqual(
-      jwtFromClient,
-      await client.getJWT(),
-      'jwt is the same which means it is cached until it is about to expire',
-    );
-  });
+      assert.strictEqual(
+        jwtFromClient,
+        await client.getJWT(),
+        'jwt is the same which means it is cached until it is about to expire',
+      );
+    });
 
-  test('it refreshes the jwt if it is about to expire in the client', async function (assert) {
-    let jwtFromClient = createJWT('10s'); // Expires very soon, so the client will first refresh it
-    client['_jwt'] = jwtFromClient;
-    assert.notEqual(jwtFromClient, await client.getJWT(), 'jwt got refreshed');
-  });
+    test('it refreshes the jwt if it is about to expire in the client', async function (assert) {
+      let jwtFromClient = createJWT('10s'); // Expires very soon, so the client will first refresh it
+      client['_jwt'] = jwtFromClient;
+      assert.notEqual(
+        jwtFromClient,
+        await client.getJWT(),
+        'jwt got refreshed',
+      );
+    });
 
-  test('it refreshes the jwt if it expired in the client', async function (assert) {
-    let jwtFromClient = createJWT(-1); // Expired 1 second ago
-    client['_jwt'] = jwtFromClient;
-    assert.notEqual(jwtFromClient, await client.getJWT(), 'jwt got refreshed');
+    test('it refreshes the jwt if it expired in the client', async function (assert) {
+      let jwtFromClient = createJWT(-1); // Expired 1 second ago
+      client['_jwt'] = jwtFromClient;
+      assert.notEqual(
+        jwtFromClient,
+        await client.getJWT(),
+        'jwt got refreshed',
+      );
+    });
   });
 });