diff --git a/docs/playwright.md b/docs/playwright.md index 315033955b2..2c26b7ab2be 100644 --- a/docs/playwright.md +++ b/docs/playwright.md @@ -77,6 +77,9 @@ test.use({ ``` The appropriate homeserver will be launched by the Playwright worker and reused for all tests which match the worker configuration. +Due to homeservers being reused between tests, please use unique names for any rooms put into the room directory as +they may be visible from other tests, the suggested approach is to use `testInfo.testId` within the name or lodash's uniqueId. +We remove public rooms from the room directory between tests but deleting users doesn't have a homeserver agnostic solution. The logs from testcontainers will be attached to any reports output from Playwright. ## Writing Tests diff --git a/playwright/e2e/forgot-password/forgot-password.spec.ts b/playwright/e2e/forgot-password/forgot-password.spec.ts index 71475e892eb..af4e6def7ed 100644 --- a/playwright/e2e/forgot-password/forgot-password.spec.ts +++ b/playwright/e2e/forgot-password/forgot-password.spec.ts @@ -6,16 +6,25 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com Please see LICENSE files in the repository root for full details. */ -import { expect, test } from "../../element-web-test"; +import { expect, test as base } from "../../element-web-test"; import { selectHomeserver } from "../utils"; import { emailHomeserver } from "../../plugins/homeserver/synapse/emailHomeserver.ts"; import { isDendrite } from "../../plugins/homeserver/dendrite"; +import { Credentials } from "../../plugins/homeserver"; -const username = "user1234"; -// this has to be password-like enough to please zxcvbn. Needless to say it's just from pwgen. -const password = "oETo7MPf0o"; const email = "user@nowhere.dummy"; +const test = base.extend<{ credentials: Pick }>({ + // eslint-disable-next-line no-empty-pattern + credentials: async ({}, use, testInfo) => { + await use({ + username: `user_${testInfo.testId}`, + // this has to be password-like enough to please zxcvbn. Needless to say it's just from pwgen. + password: "oETo7MPf0o", + }); + }, +}); + test.use(emailHomeserver); test.use({ config: { @@ -45,31 +54,35 @@ test.describe("Forgot Password", () => { await expect(page.getByRole("main")).toMatchScreenshot("forgot-password.png"); }); - test("renders email verification dialog properly", { tag: "@screenshot" }, async ({ page, homeserver }) => { - const user = await homeserver.registerUser(username, password); + test( + "renders email verification dialog properly", + { tag: "@screenshot" }, + async ({ page, homeserver, credentials }) => { + const user = await homeserver.registerUser(credentials.username, credentials.password); - await homeserver.setThreepid(user.userId, "email", email); + await homeserver.setThreepid(user.userId, "email", email); - await page.goto("/"); + await page.goto("/"); - await page.getByRole("link", { name: "Sign in" }).click(); - await selectHomeserver(page, homeserver.baseUrl); + await page.getByRole("link", { name: "Sign in" }).click(); + await selectHomeserver(page, homeserver.baseUrl); - await page.getByRole("button", { name: "Forgot password?" }).click(); + await page.getByRole("button", { name: "Forgot password?" }).click(); - await page.getByRole("textbox", { name: "Email address" }).fill(email); + await page.getByRole("textbox", { name: "Email address" }).fill(email); - await page.getByRole("button", { name: "Send email" }).click(); + await page.getByRole("button", { name: "Send email" }).click(); - await page.getByRole("button", { name: "Next" }).click(); + await page.getByRole("button", { name: "Next" }).click(); - await page.getByRole("textbox", { name: "New Password", exact: true }).fill(password); - await page.getByRole("textbox", { name: "Confirm new password", exact: true }).fill(password); + await page.getByRole("textbox", { name: "New Password", exact: true }).fill(credentials.password); + await page.getByRole("textbox", { name: "Confirm new password", exact: true }).fill(credentials.password); - await page.getByRole("button", { name: "Reset password" }).click(); + await page.getByRole("button", { name: "Reset password" }).click(); - await expect(page.getByRole("button", { name: "Resend" })).toBeInViewport(); + await expect(page.getByRole("button", { name: "Resend" })).toBeInViewport(); - await expect(page.locator(".mx_Dialog")).toMatchScreenshot("forgot-password-verify-email.png"); - }); + await expect(page.locator(".mx_Dialog")).toMatchScreenshot("forgot-password-verify-email.png"); + }, + ); }); diff --git a/playwright/e2e/login/login-consent.spec.ts b/playwright/e2e/login/login-consent.spec.ts index ab70e1d1869..d7d5861a02e 100644 --- a/playwright/e2e/login/login-consent.spec.ts +++ b/playwright/e2e/login/login-consent.spec.ts @@ -77,6 +77,9 @@ async function login(page: Page, homeserver: HomeserverInstance, credentials: Cr await page.getByRole("button", { name: "Sign in" }).click(); } +// This test suite uses the same userId for all tests in the suite +// due to DEVICE_SIGNING_KEYS_BODY being specific to that userId, +// so we restart the Synapse container to make it forget everything. test.use(consentHomeserver); test.use({ config: { @@ -97,6 +100,9 @@ test.use({ ...credentials, displayName, }); + + // Restart the homeserver to wipe its in-memory db so we can reuse the same user ID without cross-signing prompts + await homeserver.restart(); }, }); diff --git a/playwright/e2e/oidc/index.ts b/playwright/e2e/oidc/index.ts index 7e9b03ee6ac..bfd49b496a0 100644 --- a/playwright/e2e/oidc/index.ts +++ b/playwright/e2e/oidc/index.ts @@ -33,7 +33,7 @@ export async function registerAccountMas( expect(messages.items).toHaveLength(1); }).toPass(); expect(messages.items[0].to).toEqual(`${username} <${email}>`); - const [code] = messages.items[0].text.match(/(\d{6})/); + const [, code] = messages.items[0].text.match(/Your verification code to confirm this email address is: (\d{6})/); await page.getByRole("textbox", { name: "6-digit code" }).fill(code); await page.getByRole("button", { name: "Continue" }).click(); diff --git a/playwright/e2e/oidc/oidc-native.spec.ts b/playwright/e2e/oidc/oidc-native.spec.ts index 63cf0a5b59f..a50730ce747 100644 --- a/playwright/e2e/oidc/oidc-native.spec.ts +++ b/playwright/e2e/oidc/oidc-native.spec.ts @@ -17,7 +17,15 @@ test.describe("OIDC Native", { tag: ["@no-firefox", "@no-webkit"] }, () => { test.skip(isDendrite, "does not yet support MAS"); test.slow(); // trace recording takes a while here - test("can register the oauth2 client and an account", async ({ context, page, homeserver, mailhogClient, mas }) => { + test("can register the oauth2 client and an account", async ({ + context, + page, + homeserver, + mailhogClient, + mas, + }, testInfo) => { + await page.clock.install(); + const tokenUri = `${mas.baseUrl}/oauth2/token`; const tokenApiPromise = page.waitForRequest( (request) => request.url() === tokenUri && request.postDataJSON()["grant_type"] === "authorization_code", @@ -25,11 +33,14 @@ test.describe("OIDC Native", { tag: ["@no-firefox", "@no-webkit"] }, () => { await page.goto("/#/login"); await page.getByRole("button", { name: "Continue" }).click(); - await registerAccountMas(page, mailhogClient, "alice", "alice@email.com", "Pa$sW0rD!"); + + const userId = `alice_${testInfo.testId}`; + await registerAccountMas(page, mailhogClient, userId, "alice@email.com", "Pa$sW0rD!"); // Eventually, we should end up at the home screen. await expect(page).toHaveURL(/\/#\/home$/, { timeout: 10000 }); - await expect(page.getByRole("heading", { name: "Welcome alice", exact: true })).toBeVisible(); + await expect(page.getByRole("heading", { name: `Welcome ${userId}`, exact: true })).toBeVisible(); + await page.clock.runFor(20000); // run the timer so we see the token request const tokenApiRequest = await tokenApiPromise; expect(tokenApiRequest.postDataJSON()["grant_type"]).toBe("authorization_code"); diff --git a/playwright/e2e/one-to-one-chat/one-to-one-chat.spec.ts b/playwright/e2e/one-to-one-chat/one-to-one-chat.spec.ts index deefb305dbc..8a4401f5f23 100644 --- a/playwright/e2e/one-to-one-chat/one-to-one-chat.spec.ts +++ b/playwright/e2e/one-to-one-chat/one-to-one-chat.spec.ts @@ -17,8 +17,8 @@ const test = base.extend<{ test.describe("1:1 chat room", () => { test.use({ displayName: "Jeff", - user2: async ({ homeserver }, use) => { - const credentials = await homeserver.registerUser("user1234", "p4s5W0rD", "Timmy"); + user2: async ({ homeserver }, use, testInfo) => { + const credentials = await homeserver.registerUser(`user2_${testInfo.testId}`, "p4s5W0rD", "Timmy"); await use(credentials); }, }); diff --git a/playwright/e2e/share-dialog/share-dialog.spec.ts b/playwright/e2e/share-dialog/share-dialog.spec.ts index d5424a681d6..58574a46ffe 100644 --- a/playwright/e2e/share-dialog/share-dialog.spec.ts +++ b/playwright/e2e/share-dialog/share-dialog.spec.ts @@ -23,7 +23,7 @@ test.describe("Share dialog", () => { const dialog = page.getByRole("dialog", { name: "Share room" }); await expect(dialog.getByText(`https://matrix.to/#/${room.roomId}`)).toBeVisible(); - expect(dialog).toMatchScreenshot("share-dialog-room.png", { + await expect(dialog).toMatchScreenshot("share-dialog-room.png", { // QRCode and url changes at every run mask: [page.locator(".mx_QRCode"), page.locator(".mx_ShareDialog_top > span")], }); @@ -40,7 +40,7 @@ test.describe("Share dialog", () => { const dialog = page.getByRole("dialog", { name: "Share User" }); await expect(dialog.getByText(`https://matrix.to/#/${user.userId}`)).toBeVisible(); - expect(dialog).toMatchScreenshot("share-dialog-user.png", { + await expect(dialog).toMatchScreenshot("share-dialog-user.png", { // QRCode changes at every run mask: [page.locator(".mx_QRCode")], }); @@ -57,7 +57,7 @@ test.describe("Share dialog", () => { const dialog = page.getByRole("dialog", { name: "Share Room Message" }); await expect(dialog.getByRole("checkbox", { name: "Link to selected message" })).toBeChecked(); - expect(dialog).toMatchScreenshot("share-dialog-event.png", { + await expect(dialog).toMatchScreenshot("share-dialog-event.png", { // QRCode and url changes at every run mask: [page.locator(".mx_QRCode"), page.locator(".mx_ShareDialog_top > span")], }); diff --git a/playwright/e2e/sliding-sync/sliding-sync.spec.ts b/playwright/e2e/sliding-sync/sliding-sync.spec.ts index 29a612ccd32..1ab7909a478 100644 --- a/playwright/e2e/sliding-sync/sliding-sync.spec.ts +++ b/playwright/e2e/sliding-sync/sliding-sync.spec.ts @@ -108,7 +108,6 @@ test.describe("Sliding Sync", () => { await page.getByRole("menuitemradio", { name: "A-Z" }).dispatchEvent("click"); await expect(page.locator(".mx_StyledRadioButton_checked").getByText("A-Z")).toBeVisible(); - await page.pause(); await checkOrder(["Apple", "Orange", "Pineapple", "Test Room"], page); }); diff --git a/playwright/services.ts b/playwright/services.ts index b480cbc4054..5e2679953e7 100644 --- a/playwright/services.ts +++ b/playwright/services.ts @@ -7,22 +7,25 @@ Please see LICENSE files in the repository root for full details. import { test as base } from "@playwright/test"; import mailhog from "mailhog"; -import { GenericContainer, Network, StartedNetwork, StartedTestContainer, Wait } from "testcontainers"; +import { Network, StartedNetwork } from "testcontainers"; import { PostgreSqlContainer, StartedPostgreSqlContainer } from "@testcontainers/postgresql"; import { SynapseConfigOptions, SynapseContainer } from "./testcontainers/synapse.ts"; import { ContainerLogger } from "./testcontainers/utils.ts"; import { StartedMatrixAuthenticationServiceContainer } from "./testcontainers/mas.ts"; import { HomeserverContainer, StartedHomeserverContainer } from "./testcontainers/HomeserverContainer.ts"; +import { MailhogContainer, StartedMailhogContainer } from "./testcontainers/mailhog.ts"; + +interface TestFixtures { + mailhogClient: mailhog.API; +} export interface Services { logger: ContainerLogger; network: StartedNetwork; postgres: StartedPostgreSqlContainer; - - mailhog: StartedTestContainer; - mailhogClient: mailhog.API; + mailhog: StartedMailhogContainer; synapseConfigOptions: SynapseConfigOptions; _homeserver: HomeserverContainer; @@ -30,7 +33,7 @@ export interface Services { mas?: StartedMatrixAuthenticationServiceContainer; } -export const test = base.extend<{}, Services>({ +export const test = base.extend({ logger: [ // eslint-disable-next-line no-empty-pattern async ({}, use) => { @@ -79,24 +82,20 @@ export const test = base.extend<{}, Services>({ mailhog: [ async ({ logger, network }, use) => { - const container = await new GenericContainer("mailhog/mailhog:latest") + const container = await new MailhogContainer() .withNetwork(network) .withNetworkAliases("mailhog") - .withExposedPorts(8025) .withLogConsumer(logger.getConsumer("mailhog")) - .withWaitStrategy(Wait.forListeningPorts()) .start(); await use(container); await container.stop(); }, { scope: "worker" }, ], - mailhogClient: [ - async ({ mailhog: container }, use) => { - await use(mailhog({ host: container.getHost(), port: container.getMappedPort(8025) })); - }, - { scope: "worker" }, - ], + mailhogClient: async ({ mailhog: container }, use) => { + await use(container.client); + await container.client.deleteAll(); + }, synapseConfigOptions: [{}, { option: true, scope: "worker" }], _homeserver: [ diff --git a/playwright/snapshots/share-dialog/share-dialog.spec.ts/share-dialog-event-linux.png b/playwright/snapshots/share-dialog/share-dialog.spec.ts/share-dialog-event-linux.png index 541eaa2fa55..6a55618c78d 100644 Binary files a/playwright/snapshots/share-dialog/share-dialog.spec.ts/share-dialog-event-linux.png and b/playwright/snapshots/share-dialog/share-dialog.spec.ts/share-dialog-event-linux.png differ diff --git a/playwright/testcontainers/mailhog.ts b/playwright/testcontainers/mailhog.ts new file mode 100644 index 00000000000..c3305607d89 --- /dev/null +++ b/playwright/testcontainers/mailhog.ts @@ -0,0 +1,30 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE files in the repository root for full details. +*/ + +import { AbstractStartedContainer, GenericContainer, StartedTestContainer, Wait } from "testcontainers"; +import mailhog from "mailhog"; + +export class MailhogContainer extends GenericContainer { + constructor() { + super("mailhog/mailhog:latest"); + + this.withExposedPorts(8025).withWaitStrategy(Wait.forListeningPorts()); + } + + public override async start(): Promise { + return new StartedMailhogContainer(await super.start()); + } +} + +export class StartedMailhogContainer extends AbstractStartedContainer { + public readonly client: mailhog.API; + + constructor(container: StartedTestContainer) { + super(container); + this.client = mailhog({ host: container.getHost(), port: container.getMappedPort(8025) }); + } +}