Fix flaky playwright tests (#28957)

* Docs Signed-off-by: Michael Telatynski <[email protected]> * Avoid reusing user1234 Signed-off-by: Michael Telatynski <[email protected]> * Fix stale-screenshot-reporter.ts Signed-off-by: Michael Telatynski <[email protected]> * Clean up public rooms between tests on reused homeserver Signed-off-by: Michael Telatynski <[email protected]> * Deflake spotlight when homeserver is reused Signed-off-by: Michael Telatynski <[email protected]> * Deflake more tests using existing username Signed-off-by: Michael Telatynski <[email protected]> * Clean mailhog between tests Signed-off-by: Michael Telatynski <[email protected]> * Fix more flakes Signed-off-by: Michael Telatynski <[email protected]> * Fix missing _request Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Fix playwright flaky tests Signed-off-by: Michael Telatynski <[email protected]> * Wipe mailhog between test runs Signed-off-by: Michael Telatynski <[email protected]> * Delint Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * delint Signed-off-by: Michael Telatynski <[email protected]> * Deflake more tests Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Fix flaky tests Signed-off-by: Michael Telatynski <[email protected]> * Fix flaky tests Signed-off-by: Michael Telatynski <[email protected]> * Fix mas config Signed-off-by: Michael Telatynski <[email protected]> * Fix another flaky test Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> --------- Signed-off-by: Michael Telatynski <[email protected]>
element-hq · Jan 14, 2025 · 5882b00 · 5882b00
1 parent 37f8d70
commit 5882b00
Show file tree

Hide file tree

Showing 10 changed files with 444 additions and 202 deletions.
diff --git a/playwright/e2e/csAPI.ts b/playwright/e2e/csAPI.ts
@@ -9,40 +9,31 @@ import { APIRequestContext } from "playwright-core";
 import { KeyBackupInfo } from "matrix-js-sdk/src/crypto-api";
 
 import { HomeserverInstance } from "../plugins/homeserver";
+import { ClientServerApi } from "../plugins/utils/api.ts";
 
 /**
  * A small subset of the Client-Server API used to manipulate the state of the
  * account on the homeserver independently of the client under test.
  */
-export class TestClientServerAPI {
+export class TestClientServerAPI extends ClientServerApi {
     public constructor(
-        private request: APIRequestContext,
-        private homeserver: HomeserverInstance,
+        request: APIRequestContext,
+        homeserver: HomeserverInstance,
         private accessToken: string,
-    ) {}
+    ) {
+        super(homeserver.baseUrl);
+        this.setRequest(request);
+    }
 
     public async getCurrentBackupInfo(): Promise<KeyBackupInfo | null> {
-        const res = await this.request.get(`${this.homeserver.baseUrl}/_matrix/client/v3/room_keys/version`, {
-            headers: { Authorization: `Bearer ${this.accessToken}` },
-        });
-
-        return await res.json();
+        return this.request("GET", `/v3/room_keys/version`, this.accessToken);
     }
 
     /**
      * Calls the API directly to delete the given backup version
      * @param version The version to delete
      */
     public async deleteBackupVersion(version: string): Promise<void> {
-        const res = await this.request.delete(
-            `${this.homeserver.baseUrl}/_matrix/client/v3/room_keys/version/${version}`,
-            {
-                headers: { Authorization: `Bearer ${this.accessToken}` },
-            },
-        );
-
-        if (!res.ok) {
-            throw new Error(`Failed to delete backup version: ${res.status}`);
-        }
+        await this.request("DELETE", `/v3/room_keys/version/${version}`, this.accessToken);
     }
 }
diff --git a/playwright/e2e/spotlight/spotlight.spec.ts b/playwright/e2e/spotlight/spotlight.spec.ts
diff --git a/playwright/element-web-test.ts b/playwright/element-web-test.ts
@@ -49,7 +49,7 @@ const CONFIG_JSON: Partial<IConfigOptions> = {
     },
 };
 
-interface CredentialsWithDisplayName extends Credentials {
+export interface CredentialsWithDisplayName extends Credentials {
     displayName: string;
 }
 

diff --git a/playwright/plugins/homeserver/index.ts b/playwright/plugins/homeserver/index.ts
@@ -6,8 +6,11 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com
 Please see LICENSE files in the repository root for full details.
 */
 
+import { ClientServerApi } from "../utils/api.ts";
+
 export interface HomeserverInstance {
     readonly baseUrl: string;
+    readonly csApi: ClientServerApi;
 
     /**
      * Register a user on the given Homeserver using the shared registration secret.

diff --git a/playwright/plugins/utils/api.ts b/playwright/plugins/utils/api.ts
@@ -0,0 +1,76 @@
+/*
+Copyright 2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE files in the repository root for full details.
+*/
+
+import { APIRequestContext } from "@playwright/test";
+
+import { Credentials } from "../homeserver";
+
+export type Verb = "GET" | "POST" | "PUT" | "DELETE";
+
+export class Api {
+    private _request?: APIRequestContext;
+
+    constructor(private readonly baseUrl: string) {}
+
+    public setRequest(request: APIRequestContext): void {
+        this._request = request;
+    }
+
+    public async request<R extends {}>(verb: "GET", path: string, token?: string, data?: never): Promise<R>;
+    public async request<R extends {}>(verb: Verb, path: string, token?: string, data?: object): Promise<R>;
+    public async request<R extends {}>(verb: Verb, path: string, token?: string, data?: object): Promise<R> {
+        const url = `${this.baseUrl}${path}`;
+        const res = await this._request.fetch(url, {
+            data,
+            method: verb,
+            headers: token
+                ? {
+                      Authorization: `Bearer ${token}`,
+                  }
+                : undefined,
+        });
+
+        if (!res.ok()) {
+            throw new Error(
+                `Request to ${url} failed with status ${res.status()}: ${JSON.stringify(await res.json())}`,
+            );
+        }
+
+        return res.json();
+    }
+}
+
+export class ClientServerApi extends Api {
+    constructor(baseUrl: string) {
+        super(`${baseUrl}/_matrix/client`);
+    }
+
+    public async loginUser(userId: string, password: string): Promise<Credentials> {
+        const json = await this.request<{
+            access_token: string;
+            user_id: string;
+            device_id: string;
+            home_server: string;
+        }>("POST", "/v3/login", undefined, {
+            type: "m.login.password",
+            identifier: {
+                type: "m.id.user",
+                user: userId,
+            },
+            password: password,
+        });
+
+        return {
+            password,
+            accessToken: json.access_token,
+            userId: json.user_id,
+            deviceId: json.device_id,
+            homeServer: json.home_server || json.user_id.split(":").slice(1).join(":"),
+            username: userId.slice(1).split(":")[0],
+        };
+    }
+}
diff --git a/playwright/services.ts b/playwright/services.ts
@@ -119,6 +119,7 @@ export const test = base.extend<TestFixtures, Services & Options>({
                 .withNetworkAliases("homeserver")
                 .withLogConsumer(logger.getConsumer("synapse"))
                 .withConfig(synapseConfig)
+                .withMatrixAuthenticationService(mas)
                 .start();
 
             await use(container);
@@ -141,5 +142,6 @@ export const test = base.extend<TestFixtures, Services & Options>({
         await logger.onTestStarted(context);
         await use(context);
         await logger.onTestFinished(testInfo);
+        await homeserver.onTestFinished(testInfo);
     },
 });
diff --git a/playwright/testcontainers/HomeserverContainer.ts b/playwright/testcontainers/HomeserverContainer.ts
@@ -6,17 +6,19 @@ Please see LICENSE files in the repository root for full details.
 */
 
 import { AbstractStartedContainer, GenericContainer } from "testcontainers";
-import { APIRequestContext } from "@playwright/test";
+import { APIRequestContext, TestInfo } from "@playwright/test";
 
-import { StartedSynapseContainer } from "./synapse.ts";
 import { HomeserverInstance } from "../plugins/homeserver";
+import { StartedMatrixAuthenticationServiceContainer } from "./mas.ts";
 
 export interface HomeserverContainer<Config> extends GenericContainer {
     withConfigField(key: string, value: any): this;
     withConfig(config: Partial<Config>): this;
-    start(): Promise<StartedSynapseContainer>;
+    withMatrixAuthenticationService(mas?: StartedMatrixAuthenticationServiceContainer): this;
+    start(): Promise<StartedHomeserverContainer>;
 }
 
 export interface StartedHomeserverContainer extends AbstractStartedContainer, HomeserverInstance {
     setRequest(request: APIRequestContext): void;
+    onTestFinished(testInfo: TestInfo): Promise<void>;
 }
diff --git a/playwright/testcontainers/dendrite.ts b/playwright/testcontainers/dendrite.ts
@@ -13,6 +13,7 @@ import { randB64Bytes } from "../plugins/utils/rand.ts";
 import { StartedSynapseContainer } from "./synapse.ts";
 import { deepCopy } from "../plugins/utils/object.ts";
 import { HomeserverContainer } from "./HomeserverContainer.ts";
+import { StartedMatrixAuthenticationServiceContainer } from "./mas.ts";
 
 const DEFAULT_CONFIG = {
     version: 2,
@@ -235,7 +236,11 @@ export class DendriteContainer extends GenericContainer implements HomeserverCon
         return this;
     }
 
-    public override async start(): Promise<StartedSynapseContainer> {
+    public withMatrixAuthenticationService(mas?: StartedMatrixAuthenticationServiceContainer): this {
+        throw new Error("Dendrite does not support MAS.");
+    }
+
+    public override async start(): Promise<StartedDendriteContainer> {
         this.withCopyContentToContainer([
             {
                 target: "/etc/dendrite/dendrite.yaml",
@@ -244,8 +249,7 @@ export class DendriteContainer extends GenericContainer implements HomeserverCon
         ]);
 
         const container = await super.start();
-        // Surprisingly, Dendrite implements the same register user Admin API Synapse, so we can just extend it
-        return new StartedSynapseContainer(
+        return new StartedDendriteContainer(
             container,
             `http://${container.getHost()}:${container.getMappedPort(8008)}`,
             this.config.client_api.registration_shared_secret,
@@ -258,3 +262,6 @@ export class PineconeContainer extends DendriteContainer {
         super("matrixdotorg/dendrite-demo-pinecone:main", "/usr/bin/dendrite-demo-pinecone");
     }
 }
+
+// Surprisingly, Dendrite implements the same register user Synapse Admin API, so we can just extend it
+export class StartedDendriteContainer extends StartedSynapseContainer {}
-Original file line number
+Diff line change
@@ Expand Up / @@ -49,7 +49,7 @@ const CONFIG_JSON: Partial<IConfigOptions> = { @@
         },
     };
-    interface CredentialsWithDisplayName extends Credentials {
+    export interface CredentialsWithDisplayName extends Credentials {
         displayName: string;
     }
@@ Expand Down @@