Speed up tests runs by sharding using tenbin

.github/workflows/end-to-end-tests.yaml

@@ -54,6 +54,17 @@ jobs:
               with:
                   repository: element-hq/element-web
 
+            # Restore playwright-results.json file, which records the execution time of each test file.
+            # The splitTests function uses this file for sharding.
+            # We do this in the build stage to ensure it remains consistent if a playwright test got manually restarted
+            - name: Restore playwright-results.json
+              uses: actions/cache/restore@v4
+              with:
+                  path: playwright-results.json
+                  key: playwright-results
+                  restore-keys: |
+                      playwright-results-*
+
             - uses: actions/setup-node@v4
               with:
                   cache: "yarn"
@@ -77,15 +88,17 @@ jobs:
               env:
                   CI_PACKAGE: true
                   VERSION: "${{ steps.layered_build.outputs.VERSION }}"
-              run: |
-                  yarn build
+              run: yarn build
 
             - name: Upload Artifact
               uses: actions/upload-artifact@v4
               with:
-                  name: webapp
-                  path: webapp
+                  name: build
+                  path: |
+                      webapp
+                      playwright-results.json
                   retention-days: 1
+                  if-no-files-found: error
 
             - name: Calculate runner variables
               id: runner-vars
@@ -122,6 +135,8 @@ jobs:
                       project: Firefox
                     - runAllTests: false
                       project: WebKit
+        env:
+            SHARD_BLOB_NAME: blob-report-${{ matrix.project }}-${{ matrix.runner }}
         steps:
             - uses: actions/checkout@v4
               with:
@@ -131,8 +146,7 @@ jobs:
             - name: 📥 Download artifact
               uses: actions/download-artifact@v4
               with:
-                  name: webapp
-                  path: webapp
+                  name: build
 
             - uses: actions/setup-node@v4
               with:
@@ -168,63 +182,77 @@ jobs:
             - name: Run Playwright tests
               run: |
                   yarn playwright test \
-                      --shard "${{ matrix.runner }}/${{ needs.build.outputs.num-runners }}" \
                       --project="${{ matrix.project }}" \
                       ${{ (github.event_name == 'pull_request' && matrix.runAllTests == false ) && '--grep-invert @mergequeue' || '' }}
+              env:
+                  SHARD: "${{ matrix.runner }}/${{ needs.build.outputs.num-runners }}"
+                  PLAYWRIGHT_BLOB_OUTPUT_FILE: ${{ env.SHARD_BLOB_NAME }}.zip
 
             - name: Upload blob report to GitHub Actions Artifacts
               if: always()
               uses: actions/upload-artifact@v4
               with:
-                  name: all-blob-reports-${{ matrix.project }}-${{ matrix.runner }}
-                  path: blob-report
+                  name: ${{ env.SHARD_BLOB_NAME }}
+                  path: ${{ env.SHARD_BLOB_NAME }}.zip
                   retention-days: 1
+                  if-no-files-found: error
 
-    complete:
-        name: end-to-end-tests
+    report:
+        name: Report results
         needs: playwright
-        if: always()
+        if: ${{ !cancelled() && inputs.skip != true }}
         runs-on: ubuntu-24.04
         steps:
             - uses: actions/checkout@v4
-              if: inputs.skip != true
               with:
                   persist-credentials: false
                   repository: element-hq/element-web
 
             - uses: actions/setup-node@v4
-              if: inputs.skip != true
               with:
                   cache: "yarn"
                   node-version: "lts/*"
 
             - name: Install dependencies
-              if: inputs.skip != true
               run: yarn install --frozen-lockfile
 
             - name: Download blob reports from GitHub Actions Artifacts
-              if: inputs.skip != true
               uses: actions/download-artifact@v4
               with:
-                  pattern: all-blob-reports-*
+                  pattern: blob-report-*
                   path: all-blob-reports
                   merge-multiple: true
 
             - name: Merge into HTML Report
-              if: inputs.skip != true
-              run: yarn playwright merge-reports --reporter=html,./playwright/flaky-reporter.ts,./playwright/stale-screenshot-reporter.ts ./all-blob-reports
+              run: yarn playwright merge-reports --reporter=html,json,./playwright/flaky-reporter.ts,./playwright/stale-screenshot-reporter.ts ./all-blob-reports
               env:
                   # Only pass creds to the flaky-reporter on main branch runs
                   GITHUB_TOKEN: ${{ github.ref_name == 'develop' && secrets.ELEMENT_BOT_TOKEN || '' }}
+                  PLAYWRIGHT_JSON_OUTPUT_NAME: playwright-results.json
+
+            - name: Cache playwright-results.json
+              uses: actions/cache/save@v4
+              with:
+                  path: playwright-results.json
+                  key: playwright-results-${{ github.run_id }}
 
             # Upload the HTML report even if one of our reporters fails, this can happen when stale screenshots are detected
             - name: Upload HTML report
-              if: always() && inputs.skip != true
+              if: always()
               uses: actions/upload-artifact@v4
               with:
                   name: html-report
                   path: playwright-report
                   retention-days: 14
+                  if-no-files-found: error
 
-            - if: needs.playwright.result != 'skipped' && needs.playwright.result != 'success'
+    complete:
+        name: end-to-end-tests
+        needs:
+            - playwright
+            - report
+        if: ${{ !cancelled() }}
+        runs-on: ubuntu-24.04
+        steps:
+            - if: needs.playwright.result == 'failure' || needs.report.result == 'failure'
               run: exit 1

.github/workflows/tests.yml

@@ -64,6 +64,16 @@ jobs:
               id: cpu-cores
               uses: SimenB/github-actions-cpu-cores@97ba232459a8e02ff6121db9362b09661c875ab8 # v2
 
+            # Restore tenbin-report.json file, which records the execution time of each test file.
+            # @tenbin/jest/sequencer uses this file for sharding.
+            - name: Restore tenbin-report.json
+              uses: actions/cache/restore@v4
+              with:
+                  path: tenbin-report.json
+                  key: tenbin-report
+                  restore-keys: |
+                      tenbin-report-*
+
             - name: Run tests
               run: |
                   yarn test \
@@ -78,6 +88,13 @@ jobs:
                   # tell jest to use coloured output
                   FORCE_COLOR: true
 
+            # @tenbin/jest/reporter generates tenbin-report.json for each shard.
+            - name: Upload tenbin-report.json
+              uses: actions/upload-artifact@v4
+              with:
+                  name: tenbin-report-${{ matrix.runner }}
+                  path: tenbin-report.json
+
             - name: Move coverage files into place
               if: env.ENABLE_COVERAGE == 'true'
               run: mv coverage/lcov.info coverage/${{ steps.setupNode.outputs.node-version }}-${{ matrix.runner }}.lcov.info
@@ -112,3 +129,16 @@ jobs:
                   context: SonarCloud Code Analysis
                   sha: ${{ github.sha }}
                   target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+            # Download, merge & upload tenbin-report.json
+            - uses: actions/download-artifact@v4
+              with:
+                  path: tenbin-report
+                  pattern: tenbin-report-*
+            - name: Merge  tenbin-report
+              run: jq -s add tenbin-report/**/tenbin-report.json > tenbin-report.json
+            - name: Cache tenbin-report.json
+              uses: actions/cache/save@v4
+              with:
+                  path: tenbin-report.json
+                  key: tenbin-report-${{ github.run_id }}

jest.config.ts

@@ -52,11 +52,12 @@ const config: Config = {
     testResultsProcessor: "@casualbot/jest-sonar-reporter",
     prettierPath: null,
     moduleDirectories: ["node_modules", "test/test-utils"],
+    testSequencer: "@tenbin/jest/sequencer",
 };
 
 // if we're running under GHA, enable the GHA reporter
 if (env["GITHUB_ACTIONS"] !== undefined) {
-    const reporters: Config["reporters"] = [["github-actions", { silent: false }], "summary"];
+    const reporters: Config["reporters"] = [["github-actions", { silent: false }], "summary", "@tenbin/jest/reporter"];
 
     // if we're running against the develop branch, also enable the slow test reporter
     if (env["GITHUB_REF"] == "refs/heads/develop") {

package.json

@@ -181,6 +181,8 @@
         "@sentry/webpack-plugin": "^2.7.1",
         "@stylistic/eslint-plugin": "^2.9.0",
         "@svgr/webpack": "^8.0.0",
+        "@tenbin/jest": "^0.5.0",
+        "@tenbin/playwright": "^0.5.0",
         "@testcontainers/postgresql": "^10.16.0",
         "@testing-library/dom": "^10.4.0",
         "@testing-library/jest-dom": "^6.4.8",

playwright.config.ts

@@ -7,6 +7,7 @@ Please see LICENSE files in the repository root for full details.
 */
 
 import { defineConfig, devices } from "@playwright/test";
+import { splitTests } from "@tenbin/playwright";
 
 const baseURL = process.env["BASE_URL"] ?? "http://localhost:8080";
 
@@ -70,4 +71,11 @@ export default defineConfig({
     snapshotDir: "playwright/snapshots",
     snapshotPathTemplate: "{snapshotDir}/{testFilePath}/{arg}-{platform}{ext}",
     forbidOnly: !!process.env.CI,
+    testMatch: process.env.SHARD
+        ? splitTests({
+              shard: process.env.SHARD,
+              pattern: ["playwright/e2e/**/*.spec.ts"],
+              reportFile: "playwright-results.json",
+          })
+        : undefined,
 });

playwright/e2e/crypto/backups-mas.spec.ts

@@ -19,19 +19,19 @@ test.use(masHomeserver);
 test.describe("Encryption state after registration", () => {
     test.skip(isDendrite, "does not yet support MAS");
 
-    test("Key backup is enabled by default", async ({ page, mailhogClient, app }) => {
+    test("Key backup is enabled by default", async ({ page, mailhogClient, app }, testInfo) => {
         await page.goto("/#/login");
         await page.getByRole("button", { name: "Continue" }).click();
-        await registerAccountMas(page, mailhogClient, "alice", "[email protected]", "Pa$sW0rD!");
+        await registerAccountMas(page, mailhogClient, `alice_${testInfo.testId}`, "[email protected]", "Pa$sW0rD!");
 
         await app.settings.openUserSettings("Security & Privacy");
         await expect(page.getByText("This session is backing up your keys.")).toBeVisible();
     });
 
-    test("user is prompted to set up recovery", async ({ page, mailhogClient, app }) => {
+    test("user is prompted to set up recovery", async ({ page, mailhogClient, app }, testInfo) => {
         await page.goto("/#/login");
         await page.getByRole("button", { name: "Continue" }).click();
-        await registerAccountMas(page, mailhogClient, "alice", "[email protected]", "Pa$sW0rD!");
+        await registerAccountMas(page, mailhogClient, `alice_${testInfo.testId}`, "[email protected]", "Pa$sW0rD!");
 
         await page.getByRole("button", { name: "Add room" }).click();
         await page.getByRole("menuitem", { name: "New room" }).click();
@@ -45,8 +45,13 @@ test.describe("Encryption state after registration", () => {
 test.describe("Key backup reset from elsewhere", () => {
     test.skip(isDendrite, "does not yet support MAS");
 
-    test("Key backup is disabled when reset from elsewhere", async ({ page, mailhogClient, request, homeserver }) => {
-        const testUsername = "alice";
+    test("Key backup is disabled when reset from elsewhere", async ({
+        page,
+        mailhogClient,
+        request,
+        homeserver,
+    }, testInfo) => {
+        const testUsername = `alice_${testInfo.testId}`;
         const testPassword = "Pa$sW0rD!";
 
         // there's a delay before keys are uploaded so the error doesn't appear immediately: use a fake
@@ -62,8 +67,7 @@ test.describe("Key backup reset from elsewhere", () => {
         await page.getByRole("textbox", { name: "Name" }).fill("test room");
         await page.getByRole("button", { name: "Create room" }).click();
 
-        // @ts-ignore - this runs in the browser scope where mxMatrixClientPeg is a thing. Here, it is not.
-        const accessToken = await page.evaluate(() => mxMatrixClientPeg.get().getAccessToken());
+        const accessToken = await page.evaluate(() => window.mxMatrixClientPeg.get().getAccessToken());
 
         const csAPI = new TestClientServerAPI(request, homeserver, accessToken);
 

playwright/e2e/csAPI.ts

@@ -9,40 +9,31 @@ import { APIRequestContext } from "playwright-core";
 import { KeyBackupInfo } from "matrix-js-sdk/src/crypto-api";
 
 import { HomeserverInstance } from "../plugins/homeserver";
+import { ClientServerApi } from "../testcontainers/utils.ts";
 
 /**
  * A small subset of the Client-Server API used to manipulate the state of the
  * account on the homeserver independently of the client under test.
  */
-export class TestClientServerAPI {
+export class TestClientServerAPI extends ClientServerApi {
     public constructor(
-        private request: APIRequestContext,
-        private homeserver: HomeserverInstance,
+        request: APIRequestContext,
+        homeserver: HomeserverInstance,
         private accessToken: string,
-    ) {}
+    ) {
+        super(homeserver.baseUrl);
+        this.setRequest(request);
+    }
 
     public async getCurrentBackupInfo(): Promise<KeyBackupInfo | null> {
-        const res = await this.request.get(`${this.homeserver.baseUrl}/_matrix/client/v3/room_keys/version`, {
-            headers: { Authorization: `Bearer ${this.accessToken}` },
-        });
-
-        return await res.json();
+        return this.request("GET", `/v3/room_keys/version`, this.accessToken);
     }
 
     /**
      * Calls the API directly to delete the given backup version
      * @param version The version to delete
      */
     public async deleteBackupVersion(version: string): Promise<void> {
-        const res = await this.request.delete(
-            `${this.homeserver.baseUrl}/_matrix/client/v3/room_keys/version/${version}`,
-            {
-                headers: { Authorization: `Bearer ${this.accessToken}` },
-            },
-        );
-
-        if (!res.ok) {
-            throw new Error(`Failed to delete backup version: ${res.status}`);
-        }
+        await this.request("DELETE", `/v3/room_keys/version/${version}`, this.accessToken);
     }
 }

playwright/e2e/login/login-consent.spec.ts

@@ -91,6 +91,11 @@ test.use({
             },
         },
     },
+    context: async ({ context, homeserver }, use) => {
+        // Restart the homeserver to wipe its in-memory db so we can reuse the same user ID without cross-signing prompts
+        await homeserver.restart();
+        await use(context);
+    },
     credentials: async ({ context, homeserver }, use) => {
         const displayName = "Dave";
         const credentials = await homeserver.registerUser(username, password, displayName);

playwright/e2e/login/login-sso.spec.ts

@@ -17,13 +17,13 @@ test.use(legacyOAuthHomeserver);
 test.describe("SSO login", () => {
     test.skip(isDendrite, "does not yet support SSO");
 
-    test("logs in with SSO and lands on the home screen", async ({ page, homeserver }) => {
+    test("logs in with SSO and lands on the home screen", async ({ page, homeserver }, testInfo) => {
         // If this test fails with a screen showing "Timeout connecting to remote server", it is most likely due to
         // your firewall settings: Synapse is unable to reach the OIDC server.
         //
         // If you are using ufw, try something like:
         //    sudo ufw allow in on docker0
         //
-        await doTokenRegistration(page, homeserver);
+        await doTokenRegistration(page, homeserver, testInfo);
     });
 });

playwright/e2e/login/soft_logout_oauth.spec.ts

@@ -26,8 +26,8 @@ test.use({
 test.use(legacyOAuthHomeserver);
 test.describe("Soft logout with SSO user", () => {
     test.use({
-        user: async ({ page, homeserver }, use) => {
-            const user = await doTokenRegistration(page, homeserver);
+        user: async ({ page, homeserver }, use, testInfo) => {
+            const user = await doTokenRegistration(page, homeserver, testInfo);
 
             // Eventually, we should end up at the home screen.
             await expect(page).toHaveURL(/\/#\/home$/);