From 195bc19ab329c27de78d53190b695c493f57f7bc Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Tue, 30 Sep 2025 13:48:35 +1000 Subject: [PATCH 01/29] feat: max message length tests --- run/test/specs/locators/conversation.ts | 36 ++++++++ run/test/specs/message_length.spec.ts | 105 ++++++++++++++++++++++++ run/types/allure.ts | 4 +- run/types/testing.ts | 4 + 4 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 run/test/specs/message_length.spec.ts diff --git a/run/test/specs/locators/conversation.ts b/run/test/specs/locators/conversation.ts index 960237075..eefe45093 100644 --- a/run/test/specs/locators/conversation.ts +++ b/run/test/specs/locators/conversation.ts @@ -585,3 +585,39 @@ export class EmojiReactsCount extends LocatorsInterface { } } } + +export class MessageLengthCountdown extends LocatorsInterface { + constructor( + device: DeviceWrapper, + private length?: string + ) { + super(device); + } + public build(): StrategyExtractionObj { + switch (this.platform) { + case 'android': + return { + strategy: 'id', + selector: 'network.loki.messenger.qa:id/characterLimitText', + text: this.length, + } as const; + case 'ios': + return { + strategy: 'xpath', + selector: `//XCUIElementTypeStaticText[@name="${this.length}"]`, + text: this.length, + } as const; + } + } +} + +export class MessageLengthOkayButton extends LocatorsInterface { + public build(): StrategyExtractionObj { + switch (this.platform) { + case 'android': + return { strategy: 'id', selector: 'Okay' } as const; + case 'ios': + return { strategy: 'xpath', selector: '//XCUIElementTypeButton[@name="Okay"]' } as const; + } + } +} diff --git a/run/test/specs/message_length.spec.ts b/run/test/specs/message_length.spec.ts new file mode 100644 index 000000000..e2c1fea4f --- /dev/null +++ b/run/test/specs/message_length.spec.ts @@ -0,0 +1,105 @@ +import { test, type TestInfo } from '@playwright/test'; + +import { englishStrippedStr } from '../../localizer/englishStrippedStr'; +import { TestSteps } from '../../types/allure'; +import { bothPlatformsIt } from '../../types/sessionIt'; +import { USERNAME } from '../../types/testing'; +import { + MessageBody, + MessageInput, + MessageLengthCountdown, + MessageLengthOkayButton, + SendButton, +} from './locators/conversation'; +import { PlusButton } from './locators/home'; +import { EnterAccountID, NewMessageOption, NextButton } from './locators/start_conversation'; +import { newUser } from './utils/create_account'; +import { closeApp, openAppOnPlatformSingleDevice, SupportedPlatformsType } from './utils/open_app'; + +const maxChars = 2000; +const countdownThreshold = 1800; + +const messageLengthTestCases = [ + { + length: 1799, + char: 'a', + shouldSend: true, + description: 'no countdown shows, message sends', + }, + { length: 1800, char: 'b', shouldSend: true, description: 'countdown shows 200, message sends' }, + { length: 2000, char: 'c', shouldSend: true, description: 'countdown shows 0, message sends' }, + { + length: 2001, + char: 'd', + shouldSend: false, + description: 'countdown shows -1, cannot send message', + }, +]; + +for (const testCase of messageLengthTestCases) { + bothPlatformsIt({ + title: `Message length limit (${testCase.length} chars)`, + risk: 'high', + countOfDevicesNeeded: 1, + allureSuites: { + parent: 'Sending Messages', + suite: 'Rules', + }, + allureDescription: `Verifies message length behavior at ${testCase.length} characters - ${testCase.description}`, + testCb: async (platform: SupportedPlatformsType, testInfo: TestInfo) => { + const { device, alice } = await test.step(TestSteps.SETUP.NEW_USER, async () => { + const { device } = await openAppOnPlatformSingleDevice(platform, testInfo); + const alice = await newUser(device, USERNAME.ALICE); + return { device, alice }; + }); + + // Send message to self to bring up Note to Self conversation + await test.step(TestSteps.OPEN.NTS, async () => { + await device.clickOnElementAll(new PlusButton(device)); + await device.clickOnElementAll(new NewMessageOption(device)); + await device.inputText(alice.accountID, new EnterAccountID(device)); + await device.scrollDown(); + await device.clickOnElementAll(new NextButton(device)); + }); + + await test.step(`Type ${testCase.length} chars, check countdown`, async () => { + const expectedCount = + testCase.length < countdownThreshold ? null : (maxChars - testCase.length).toString(); + + // Construct the string of desired length + const message = testCase.char.repeat(testCase.length); + await device.inputText(message, new MessageInput(device)); + + // Does the countdown appear? + if (expectedCount) { + await device.waitForTextElementToBePresent( + new MessageLengthCountdown(device, expectedCount) + ); + } else { + await device.verifyElementNotPresent(new MessageLengthCountdown(device)); + } + + await device.clickOnElementAll(new SendButton(device)); + + // Is the message short enough to send? + if (testCase.shouldSend) { + await device.waitForTextElementToBePresent(new MessageBody(device, message)); + } else { + // Modal appears, verify and dismiss + await device.checkModalStrings( + englishStrippedStr('modalMessageTooLongTitle').toString(), + englishStrippedStr('modalMessageTooLongDescription') + .withArgs({ limit: maxChars.toString() }) + .toString() + ); + await device.clickOnElementAll(new MessageLengthOkayButton(device)); + await device.verifyElementNotPresent(new MessageBody(device, message)); + } + }); + + await test.step(TestSteps.SETUP.CLOSE_APP, async () => { + await closeApp(device); + }); + }, + }); +} diff --git a/run/types/allure.ts b/run/types/allure.ts index 7ee2167be..9cc5134ec 100644 --- a/run/types/allure.ts +++ b/run/types/allure.ts @@ -25,7 +25,7 @@ export type AllureSuiteConfig = | { parent: 'In-App Review Prompt'; suite: 'Flows' | 'Triggers' } | { parent: 'Linkouts' } | { parent: 'New Conversation'; suite: 'Join Community' | 'New Message' } - | { parent: 'Sending Messages'; suite: 'Attachments' | 'Emoji reacts' } + | { parent: 'Sending Messages'; suite: 'Attachments' | 'Emoji reacts' | 'Rules' } | { parent: 'Settings'; suite: 'App Disguise' } | { parent: 'User Actions'; @@ -77,7 +77,7 @@ export const TestSteps = { }, // Open/Navigate steps OPEN: { - NTS: 'Open Note to Self', + NTS: 'Open Note to Self', UPDATE_GROUP_INFO: `Open 'Update Group Information' modal`, PATH: 'Open Path screen', APPEARANCE: 'Open Appearance settings', diff --git a/run/types/testing.ts b/run/types/testing.ts index 78233f623..f4a18cb76 100644 --- a/run/types/testing.ts +++ b/run/types/testing.ts @@ -140,6 +140,7 @@ export type XPath = | `//android.widget.TextView[@text="${string}"]` | `//XCUIElementTypeAlert//*//XCUIElementTypeButton` | `//XCUIElementTypeButton[@name="Continue"]` + | `//XCUIElementTypeButton[@name="Okay"]` | `//XCUIElementTypeButton[@name="Settings"]` | `//XCUIElementTypeCell[.//XCUIElementTypeOther[@label="${string}"]]//XCUIElementTypeStaticText[@value="😂"]` | `//XCUIElementTypeCell[.//XCUIElementTypeOther[@label="${string}"]]//XCUIElementTypeStaticText[@value="${string}"]` @@ -148,6 +149,7 @@ export type XPath = | `//XCUIElementTypeCell[@name="Session"]` | `//XCUIElementTypeImage` | `//XCUIElementTypeOther[contains(@name, "Hey,")][1]` + | `//XCUIElementTypeStaticText[@name="${string}"]` | `//XCUIElementTypeStaticText[@name="Paste"]` | `//XCUIElementTypeStaticText[@name="Videos"]` | `//XCUIElementTypeStaticText[contains(@name, '00:')]` @@ -508,6 +510,7 @@ export type Id = | 'network.loki.messenger.qa:id/callInProgress' | 'network.loki.messenger.qa:id/callSubtitle' | 'network.loki.messenger.qa:id/callTitle' + | 'network.loki.messenger.qa:id/characterLimitText' | 'network.loki.messenger.qa:id/crop_image_menu_crop' | 'network.loki.messenger.qa:id/emptyStateContainer' | 'network.loki.messenger.qa:id/endCallButton' @@ -535,6 +538,7 @@ export type Id = | 'nickname-input' | 'not-now-button' | 'Notifications' + | 'Okay' | 'open-survey-button' | 'Open' | 'Open URL' From 984a17771090ef4a229709f9a7cffedf83ece084 Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Wed, 8 Oct 2025 09:28:19 +1100 Subject: [PATCH 02/29] feat: add more visual regression tests reintroduce full page screenshots with blur tidy up unused code --- run/screenshots/android/app_disguise.png | 4 +- .../android/landingpage_new_account.png | 4 +- .../android/landingpage_restore_account.png | 4 +- .../messagebody_incoming_reply_message.png | 3 + .../messagebody_incoming_short_message.png | 3 + .../messagebody_outgoing_reply_message.png | 3 + .../messagebody_outgoing_short_message.png | 3 + .../android/settings_appearance.png | 3 + .../android/settings_conversations.png | 3 + .../android/settings_notifications.png | 3 + run/screenshots/android/settings_privacy.png | 3 + run/screenshots/ios/app_disguise.png | 4 +- .../ios/landingpage_new_account.png | 4 +- .../ios/landingpage_restore_account.png | 4 +- .../messagebody_incoming_reply_message.png | 3 + .../messagebody_incoming_short_message.png | 3 + .../messagebody_outgoing_reply_message.png | 3 + .../messagebody_outgoing_short_message.png | 3 + run/screenshots/ios/settings_appearance.png | 3 + .../ios/settings_conversations.png | 3 + .../ios/settings_notifications.png | 3 + run/screenshots/ios/settings_privacy.png | 3 + run/test/specs/app_disguise_icons.spec.ts | 4 +- run/test/specs/check_avatar_color.spec.ts | 3 +- .../specs/landing_page_new_account.spec.ts | 7 +- .../landing_page_restore_account.spec.ts | 7 +- run/test/specs/locators/conversation.ts | 16 +- run/test/specs/utils/screenshot_paths.ts | 14 ++ run/test/specs/utils/utilities.ts | 38 ---- run/test/specs/utils/verify_screenshots.ts | 176 +++++++++++++----- run/test/specs/visual_message_bubbles.spec.ts | 104 +++++++++++ run/test/specs/visual_settings.spec.ts | 88 +++++++++ run/types/DeviceWrapper.ts | 2 +- run/types/allure.ts | 7 +- run/types/testing.ts | 10 +- 35 files changed, 432 insertions(+), 116 deletions(-) create mode 100644 run/screenshots/android/messagebody_incoming_reply_message.png create mode 100644 run/screenshots/android/messagebody_incoming_short_message.png create mode 100644 run/screenshots/android/messagebody_outgoing_reply_message.png create mode 100644 run/screenshots/android/messagebody_outgoing_short_message.png create mode 100644 run/screenshots/android/settings_appearance.png create mode 100644 run/screenshots/android/settings_conversations.png create mode 100644 run/screenshots/android/settings_notifications.png create mode 100644 run/screenshots/android/settings_privacy.png create mode 100644 run/screenshots/ios/messagebody_incoming_reply_message.png create mode 100644 run/screenshots/ios/messagebody_incoming_short_message.png create mode 100644 run/screenshots/ios/messagebody_outgoing_reply_message.png create mode 100644 run/screenshots/ios/messagebody_outgoing_short_message.png create mode 100644 run/screenshots/ios/settings_appearance.png create mode 100644 run/screenshots/ios/settings_conversations.png create mode 100644 run/screenshots/ios/settings_notifications.png create mode 100644 run/screenshots/ios/settings_privacy.png create mode 100644 run/test/specs/visual_message_bubbles.spec.ts create mode 100644 run/test/specs/visual_settings.spec.ts diff --git a/run/screenshots/android/app_disguise.png b/run/screenshots/android/app_disguise.png index 6349657d9..211502d3a 100644 --- a/run/screenshots/android/app_disguise.png +++ b/run/screenshots/android/app_disguise.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c979e249014dc0c7083020ef91d922e44a63aace080176c8551f40e6ccf8b223 -size 125751 +oid sha256:4e12299e64f2c1cf4d067efab6b2c64c54a4581e19bd01b46a0f2b74f46f92f5 +size 172617 diff --git a/run/screenshots/android/landingpage_new_account.png b/run/screenshots/android/landingpage_new_account.png index 03808de9a..646bfa47e 100644 --- a/run/screenshots/android/landingpage_new_account.png +++ b/run/screenshots/android/landingpage_new_account.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:67df5e6fbc1b9c3be58275db41b3c6bc64911c0755e38196082062a585df457b -size 81721 +oid sha256:66411453c10cb273ccc36c73d65ab80ad30a4a0b170c62dd849e0397480f7bab +size 124569 diff --git a/run/screenshots/android/landingpage_restore_account.png b/run/screenshots/android/landingpage_restore_account.png index 19e9d6978..78202c5b8 100644 --- a/run/screenshots/android/landingpage_restore_account.png +++ b/run/screenshots/android/landingpage_restore_account.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:774a934a8ce00a720d29d5b2306aa1f257c290b064c65b0afd8a60cfbd10d5a4 -size 57649 +oid sha256:b91c51f0fefff264e464cfee3df2b283b6bf52aae8aeafa47498dfae613fc51a +size 88133 diff --git a/run/screenshots/android/messagebody_incoming_reply_message.png b/run/screenshots/android/messagebody_incoming_reply_message.png new file mode 100644 index 000000000..d13048f9a --- /dev/null +++ b/run/screenshots/android/messagebody_incoming_reply_message.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed707da45a0d20692ce60311a28dea03fb6d081b903c5877821ea4726057d0d7 +size 31612 diff --git a/run/screenshots/android/messagebody_incoming_short_message.png b/run/screenshots/android/messagebody_incoming_short_message.png new file mode 100644 index 000000000..f0d115097 --- /dev/null +++ b/run/screenshots/android/messagebody_incoming_short_message.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9910e433e7f161c4bdf17b1367c3616ac1ce95399240a5ccf6930bfd89fd1332 +size 10431 diff --git a/run/screenshots/android/messagebody_outgoing_reply_message.png b/run/screenshots/android/messagebody_outgoing_reply_message.png new file mode 100644 index 000000000..383955e8a --- /dev/null +++ b/run/screenshots/android/messagebody_outgoing_reply_message.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7ccbf17d4e6f2ba0ca490462e8464c72c357fc1e950722178caab12acfb2c14 +size 38690 diff --git a/run/screenshots/android/messagebody_outgoing_short_message.png b/run/screenshots/android/messagebody_outgoing_short_message.png new file mode 100644 index 000000000..a53664175 --- /dev/null +++ b/run/screenshots/android/messagebody_outgoing_short_message.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e79bf04d5d933a2009d51e002e3210bd71ae8bf3b4f8a6425af557a620b81407 +size 12809 diff --git a/run/screenshots/android/settings_appearance.png b/run/screenshots/android/settings_appearance.png new file mode 100644 index 000000000..c8058b8a0 --- /dev/null +++ b/run/screenshots/android/settings_appearance.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:867ebc1c445305f15ef6ac99b7d2662818c15fd8db91c6b26e93093684249cc4 +size 175739 diff --git a/run/screenshots/android/settings_conversations.png b/run/screenshots/android/settings_conversations.png new file mode 100644 index 000000000..b9a0ec72b --- /dev/null +++ b/run/screenshots/android/settings_conversations.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96fecdfd6b84ee9a1a8d14c97de141f3da09d649134c8fbb3a699bbbcded74fb +size 194821 diff --git a/run/screenshots/android/settings_notifications.png b/run/screenshots/android/settings_notifications.png new file mode 100644 index 000000000..e7179c989 --- /dev/null +++ b/run/screenshots/android/settings_notifications.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ffb70b714367d130cfb7a11063a3a8bf627ab269fc1ea3db25dafc698095346 +size 180821 diff --git a/run/screenshots/android/settings_privacy.png b/run/screenshots/android/settings_privacy.png new file mode 100644 index 000000000..1c7954756 --- /dev/null +++ b/run/screenshots/android/settings_privacy.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73508116cab27eb1964c5928a1f6c7b62b766d567bc78234d27799aa3828b9df +size 249297 diff --git a/run/screenshots/ios/app_disguise.png b/run/screenshots/ios/app_disguise.png index 7950444a3..7ac619e06 100644 --- a/run/screenshots/ios/app_disguise.png +++ b/run/screenshots/ios/app_disguise.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f215acd1deb44135705057cc83a0f3997ce999c456783fa3c1209a33b1ca3c48 -size 477016 +oid sha256:c52f1ca9a0d2b9bd7963e0079794f9937f254d4f3cec2d06c0210570912b7d30 +size 306456 diff --git a/run/screenshots/ios/landingpage_new_account.png b/run/screenshots/ios/landingpage_new_account.png index 28afdcb85..477230ec7 100644 --- a/run/screenshots/ios/landingpage_new_account.png +++ b/run/screenshots/ios/landingpage_new_account.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c920f37c84d06f6d90fdb2b5cf5867863af7321bf3df8c33b010536d91cbe2da -size 159637 +oid sha256:ed49daf2bbafa9149844023697c84d29d80fd42c757571035bd915c0939427ea +size 151942 diff --git a/run/screenshots/ios/landingpage_restore_account.png b/run/screenshots/ios/landingpage_restore_account.png index c5252cbf5..16d398874 100644 --- a/run/screenshots/ios/landingpage_restore_account.png +++ b/run/screenshots/ios/landingpage_restore_account.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0f64723ba961cc8574cff8f6dd20ec40c12b3b2e3049511074529d39016f3b69 -size 143198 +oid sha256:67c060cc837cc24aede99498085e7bcc96408270e4d20c69569871cbf07ec94e +size 122304 diff --git a/run/screenshots/ios/messagebody_incoming_reply_message.png b/run/screenshots/ios/messagebody_incoming_reply_message.png new file mode 100644 index 000000000..2e401f1c2 --- /dev/null +++ b/run/screenshots/ios/messagebody_incoming_reply_message.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:543e5b07647a363cae67168fcc59e839a87d819ade255dd1059447e950598db7 +size 52588 diff --git a/run/screenshots/ios/messagebody_incoming_short_message.png b/run/screenshots/ios/messagebody_incoming_short_message.png new file mode 100644 index 000000000..bbe86ae01 --- /dev/null +++ b/run/screenshots/ios/messagebody_incoming_short_message.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2840b29abde3d1f9319dffee80a4c357e65dae1a8fb0153b8bc2650aa4da6a73 +size 14734 diff --git a/run/screenshots/ios/messagebody_outgoing_reply_message.png b/run/screenshots/ios/messagebody_outgoing_reply_message.png new file mode 100644 index 000000000..8a6fdfde7 --- /dev/null +++ b/run/screenshots/ios/messagebody_outgoing_reply_message.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f8af6b29078b62f6dfb1bd1699ed60df6d0d73d4fe5fa770ba20324b53cfa09 +size 57028 diff --git a/run/screenshots/ios/messagebody_outgoing_short_message.png b/run/screenshots/ios/messagebody_outgoing_short_message.png new file mode 100644 index 000000000..a10c50614 --- /dev/null +++ b/run/screenshots/ios/messagebody_outgoing_short_message.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd63d32fcb4c50b7f0c3050299fe471ab3359116fb2a5ad12d65faf9de415b15 +size 15902 diff --git a/run/screenshots/ios/settings_appearance.png b/run/screenshots/ios/settings_appearance.png new file mode 100644 index 000000000..07481b37c --- /dev/null +++ b/run/screenshots/ios/settings_appearance.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45321422bfc943e4167a778ccd5c0fc8c901ab6a9f6eed9d9bad07988f1e43a5 +size 251228 diff --git a/run/screenshots/ios/settings_conversations.png b/run/screenshots/ios/settings_conversations.png new file mode 100644 index 000000000..fc8652648 --- /dev/null +++ b/run/screenshots/ios/settings_conversations.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:344ab1487cbd0c9a53476104e70ef7196e91e03725f1bb9d8897d6655c78164d +size 185064 diff --git a/run/screenshots/ios/settings_notifications.png b/run/screenshots/ios/settings_notifications.png new file mode 100644 index 000000000..9cdafce0d --- /dev/null +++ b/run/screenshots/ios/settings_notifications.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27d6f3a9926c32a93157b320fc4360f3d919a5c34cff99f36030df11a088fa0c +size 199664 diff --git a/run/screenshots/ios/settings_privacy.png b/run/screenshots/ios/settings_privacy.png new file mode 100644 index 000000000..12e68681a --- /dev/null +++ b/run/screenshots/ios/settings_privacy.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8cbe958c8c8b203707f7e9ce1fbffd4e450eb816583bec4c31a77772c849100 +size 340584 diff --git a/run/test/specs/app_disguise_icons.spec.ts b/run/test/specs/app_disguise_icons.spec.ts index 312c9a5d2..41820680f 100644 --- a/run/test/specs/app_disguise_icons.spec.ts +++ b/run/test/specs/app_disguise_icons.spec.ts @@ -10,7 +10,7 @@ import { AppDisguisePageScreenshot } from './utils/screenshot_paths'; import { verifyElementScreenshot } from './utils/verify_screenshots'; bothPlatformsIt({ - title: 'App disguise icons', + title: 'Check app disguise icon layout', risk: 'medium', countOfDevicesNeeded: 1, testCb: appDisguiseIcons, @@ -31,7 +31,7 @@ async function appDisguiseIcons(platform: SupportedPlatformsType, testInfo: Test await device.clickOnElementAll(new UserSettings(device)); await device.clickOnElementAll(new AppearanceMenuItem(device)); }); - await test.step(TestSteps.VERIFY.ELEMENT_SCREENSHOT('app disguise icons'), async () => { + await test.step(TestSteps.VERIFY.SCREENSHOT('app disguise icons'), async () => { await device.clickOnElementAll(new SelectAppIcon(device)); await verifyElementScreenshot(device, new AppDisguisePageScreenshot(device), testInfo); }); diff --git a/run/test/specs/check_avatar_color.spec.ts b/run/test/specs/check_avatar_color.spec.ts index 80dbbb1ae..786e892cd 100644 --- a/run/test/specs/check_avatar_color.spec.ts +++ b/run/test/specs/check_avatar_color.spec.ts @@ -10,12 +10,13 @@ import { isSameColor } from './utils/check_colour'; import { closeApp, SupportedPlatformsType } from './utils/open_app'; bothPlatformsIt({ - title: 'Avatar color', + title: 'Check placeholder avatar color', risk: 'medium', countOfDevicesNeeded: 2, testCb: avatarColor, allureSuites: { parent: 'Visual Checks', + suite: 'Settings', }, allureDescription: `Verifies that a user's placeholder avatar color appears the same to a contact`, }); diff --git a/run/test/specs/landing_page_new_account.spec.ts b/run/test/specs/landing_page_new_account.spec.ts index cb9904970..5099abfc1 100644 --- a/run/test/specs/landing_page_new_account.spec.ts +++ b/run/test/specs/landing_page_new_account.spec.ts @@ -8,10 +8,15 @@ import { EmptyLandingPageScreenshot } from './utils/screenshot_paths'; import { verifyElementScreenshot } from './utils/verify_screenshots'; bothPlatformsIt({ - title: 'Landing page new account', + title: 'Check landing page (new account) layout', risk: 'low', testCb: landingPageNewAccount, countOfDevicesNeeded: 1, + allureSuites: { + parent: 'Visual Checks', + suite: 'Onboarding', + }, + allureDescription: `Verifies that the landing page for a new account matches the expected baseline`, }); async function landingPageNewAccount(platform: SupportedPlatformsType, testInfo: TestInfo) { diff --git a/run/test/specs/landing_page_restore_account.spec.ts b/run/test/specs/landing_page_restore_account.spec.ts index 70f36b639..d3c175ed1 100644 --- a/run/test/specs/landing_page_restore_account.spec.ts +++ b/run/test/specs/landing_page_restore_account.spec.ts @@ -9,10 +9,15 @@ import { EmptyLandingPageScreenshot } from './utils/screenshot_paths'; import { verifyElementScreenshot } from './utils/verify_screenshots'; bothPlatformsIt({ - title: 'Landing page restore account', + title: 'Check landing page (restored account) layout', risk: 'low', testCb: landingPageRestoreAccount, countOfDevicesNeeded: 2, + allureSuites: { + parent: 'Visual Checks', + suite: 'Onboarding', + }, + allureDescription: `Verifies that the landing page for a restored account matches the expected baseline`, }); async function landingPageRestoreAccount(platform: SupportedPlatformsType, testInfo: TestInfo) { diff --git a/run/test/specs/locators/conversation.ts b/run/test/specs/locators/conversation.ts index 960237075..e93b27216 100644 --- a/run/test/specs/locators/conversation.ts +++ b/run/test/specs/locators/conversation.ts @@ -236,10 +236,18 @@ export class ConversationHeaderName extends LocatorsInterface { export class NotificationSettings extends LocatorsInterface { public build() { - return { - strategy: 'accessibility id', - selector: 'Notifications', - } as const; + switch (this.platform) { + case 'android': + return { + strategy: 'id', + selector: 'Notifications', + } as const; + case 'ios': + return { + strategy: 'accessibility id', + selector: 'Notifications', + } as const; + } } } diff --git a/run/test/specs/utils/screenshot_paths.ts b/run/test/specs/utils/screenshot_paths.ts index 1bff8b8e5..81392617f 100644 --- a/run/test/specs/utils/screenshot_paths.ts +++ b/run/test/specs/utils/screenshot_paths.ts @@ -1,5 +1,6 @@ import path from 'path'; +import { MessageBody } from '../locators/conversation'; import { EmptyLandingPage } from '../locators/home'; import { AppDisguisePage } from '../locators/settings'; @@ -18,3 +19,16 @@ export class AppDisguisePageScreenshot extends AppDisguisePage { return path.join('run', 'screenshots', this.platform, 'app_disguise.png'); } } + +export class MessageBodyScreenshot extends MessageBody { + // The message body locator can appear in different states depending on the message content + public screenshotFileName( + state: + | 'incoming_reply_message' + | 'incoming_short_message' + | 'outgoing_reply_message' + | 'outgoing_short_message' + ): string { + return path.join('run', 'screenshots', this.platform, `messagebody_${state}.png`); + } +} diff --git a/run/test/specs/utils/utilities.ts b/run/test/specs/utils/utilities.ts index b89a2464b..1511b1d4d 100644 --- a/run/test/specs/utils/utilities.ts +++ b/run/test/specs/utils/utilities.ts @@ -2,12 +2,8 @@ import { exec as execNotPromised } from 'child_process'; import * as fs from 'fs'; import { pick } from 'lodash'; import path from 'path'; -import sharp from 'sharp'; import * as util from 'util'; -import { v4 as uuidv4 } from 'uuid'; -import { DeviceWrapper } from '../../../types/DeviceWrapper'; -import { Suffix } from '../../../types/testing'; const exec = util.promisify(execNotPromised); export async function runScriptAndLog(toRun: string, verbose = false): Promise { @@ -69,40 +65,6 @@ export function ensureHttpsURL(url: string): string { return url.startsWith('https://') ? url : `https://${url}`; } -export async function cropScreenshot(device: DeviceWrapper, screenshotBuffer: Buffer) { - const { width, height } = await device.getWindowRect(); - const cropTop = 250; - const cropLeft = 0; - const cropWidth = width; - const cropHeight = height - 250; - - const croppedBuf = await sharp(screenshotBuffer) - .extract({ left: cropLeft, top: cropTop, width: cropWidth, height: cropHeight }) - .png() - .toBuffer(); - - return croppedBuf; -} - -export async function saveImage( - source: Buffer | { save: (fullPath: string) => Promise }, - directory: string, - suffix: Suffix, - customFileName?: string -) { - const name = customFileName ?? `${uuidv4()}_${suffix}.png`; - const fullPath = path.join(directory, name); - - fs.mkdirSync(path.dirname(fullPath), { recursive: true }); - - if (Buffer.isBuffer(source)) { - fs.writeFileSync(fullPath, source); - } else { - await source.save(fullPath); - } - return fullPath; -} - export function getDiffDirectory() { const diffsDir = path.join('test-results', 'diffs'); fs.mkdirSync(diffsDir, { recursive: true }); diff --git a/run/test/specs/utils/verify_screenshots.ts b/run/test/specs/utils/verify_screenshots.ts index f949acce4..fc5df3862 100644 --- a/run/test/specs/utils/verify_screenshots.ts +++ b/run/test/specs/utils/verify_screenshots.ts @@ -2,10 +2,12 @@ import { TestInfo } from '@playwright/test'; import * as fs from 'fs'; import looksSame from 'looks-same'; import * as path from 'path'; +import sharp from 'sharp'; import { v4 as uuidv4 } from 'uuid'; import { DeviceWrapper } from '../../../types/DeviceWrapper'; import { LocatorsInterfaceScreenshot } from '../locators'; +import { SupportedPlatformsType } from './open_app'; import { getDiffDirectory } from './utilities'; type Attachment = { @@ -24,53 +26,55 @@ export async function pushAttachmentsToReport( } /** - * Takes a screenshot of a UI element and verifies it against a saved baseline image. - * - * Requires Playwright's `testInfo` for attaching visual comparison artifacts to the test report. - * Supports locators with multiple states; enforces correct state usage via type constraints. - * If no baseline image exists, the element screenshot is saved and an error is thrown. - * On mismatch, a pixel-by-pixel comparison is performed and a visual diff is attached (when CI + ALLURE_ENABLED). - * Baseline screenshots are assumed to have been taken on:Pixel 6 (1080x2061) and iPhone 16 Pro Max (1320x2868) - * - * Example usage: - * // Locator with multiple states: - * await verifyElementScreenshot(device, new EmptyLandingPageScreenshot(device), testInfo, 'new_account'); - * - * // Locator with a single state: - * await verifyElementScreenshot(device, new SomeSimpleLocatorScreenshot(device), testInfo); + * Crops screenshot to remove dynamic status bar elements (time, battery, signal) + * and other variable UI elements that cause false positives in visual comparisons + * NOTE: If cropping becomes unreliable there are methods to manipulate iOS and Android status bars */ +async function cropScreenshot(_device: DeviceWrapper, screenshotBuffer: Buffer): Promise { + const image = sharp(screenshotBuffer); + const { width, height } = await image.metadata(); + const cropTop = 150; + const cropLeft = 5; // I was getting weird rendering artifacts on the edges + const cropRight = 5; + const cropWidth = width - cropRight; + const cropHeight = height - cropTop; -export async function verifyElementScreenshot< - T extends LocatorsInterfaceScreenshot & { screenshotFileName: (...args: any[]) => string }, ->( - device: DeviceWrapper, - element: T, + return sharp(screenshotBuffer) + .extract({ left: cropLeft, top: cropTop, width: cropWidth, height: cropHeight }) + .png() + .blur() // This blur is imperceptible but gets rid of all the antialiasing issues + .toBuffer(); +} + +/** + * Shared logic for screenshot comparison and Allure reporting + */ +async function compareScreenshots( + actualScreenshotBuffer: Buffer, + baselineScreenshotPath: string, + uuid: string, testInfo: TestInfo, - ...args: Parameters // Enforces states when mandatory + tolerance?: number ): Promise { - // Declaring a UUID in advance so that the diff and screenshot files are matched alphanumerically - const uuid = uuidv4(); - // Using Playwright's default test-results folder ensures cleanup at the beginning of each run const diffsDir = getDiffDirectory(); - // Get the element screenshot as base64 - const elementToScreenshot = await device.waitForTextElementToBePresent(element); - const elementScreenshotBase64: string = await device.getElementScreenshot( - elementToScreenshot.ELEMENT - ); - // Convert the base64 string to a Buffer and save it to disk as a png - const elementScreenshotPath = path.join(diffsDir, `${uuid}_screenshot.png`); - const screenshotBuffer = Buffer.from(elementScreenshotBase64, 'base64'); - fs.writeFileSync(elementScreenshotPath, screenshotBuffer); - // Check if baseline screenshot exists - const baselineScreenshotPath = element.screenshotFileName(...args); + const actualScreenshotPath = path.join(diffsDir, `${uuid}_screenshot.png`); + fs.writeFileSync(actualScreenshotPath, actualScreenshotBuffer); + + // Check if baseline exists if (!fs.existsSync(baselineScreenshotPath)) { + // For local development you can uncomment these lines to auto-save baselines at the correct location + // fs.mkdirSync(path.dirname(baselineScreenshotPath), { recursive: true }); + // fs.writeFileSync(baselineScreenshotPath, actualScreenshotBuffer); throw new Error( - `No baseline image found at: ${baselineScreenshotPath}. A new screenshot has been saved at: ${elementScreenshotPath}` + `No baseline image found at: ${baselineScreenshotPath}. A new screenshot has been saved at: ${actualScreenshotPath}` ); } - // Use looks-same to verify the element screenshot against the baseline - const { equal, diffImage } = await looksSame(elementScreenshotPath, baselineScreenshotPath, { + + // Compare screenshots + console.log('Attempting visual comparison...'); + const { equal, diffImage } = await looksSame(actualScreenshotPath, baselineScreenshotPath, { createDiffImage: true, + tolerance: tolerance, }); if (!equal) { const diffImagePath = path.join(diffsDir, `${uuid}_diffImage.png`); @@ -81,10 +85,10 @@ export async function verifyElementScreenshot< // Load baseline and diff images const baselineBase64 = fs.readFileSync(baselineScreenshotPath).toString('base64'); const diffBase64 = fs.readFileSync(diffImagePath).toString('base64'); + const actualBase64 = actualScreenshotBuffer.toString('base64'); - // Wrap them in the Allure visual diff format const visualDiffPayload = { - actual: `data:image/png;base64,${elementScreenshotBase64}`, + actual: `data:image/png;base64,${actualBase64}`, expected: `data:image/png;base64,${baselineBase64}`, diff: `data:image/png;base64,${diffBase64}`, }; @@ -102,7 +106,7 @@ export async function verifyElementScreenshot< }, { name: 'Actual Screenshot', - body: Buffer.from(elementScreenshotBase64, 'base64'), + body: actualScreenshotBuffer, contentType: 'image/png', }, { @@ -111,18 +115,90 @@ export async function verifyElementScreenshot< contentType: 'image/png', }, ]); - console.log(`Visual comparison failed. The diff has been saved to ${diffImagePath}`); - throw new Error(`The UI doesn't match expected appearance`); } - // Cleanup of element screenshot file on success - try { - fs.unlinkSync(elementScreenshotPath); - console.log('Temporary screenshot deleted successfully'); - } catch (err) { - if (err instanceof Error) { - console.error(`Error deleting file: ${err.message}`); - } + console.log(`Visual comparison failed. The diff has been saved to ${diffImagePath}`); + throw new Error(`The UI doesn't match expected appearance`); + } + + // Cleanup on success + try { + fs.unlinkSync(actualScreenshotPath); + console.log('Temporary screenshot deleted successfully'); + } catch (err) { + if (err instanceof Error) { + console.error(`Error deleting file: ${err.message}`); } } } + +/** + * Takes a screenshot of a UI element and verifies it against a saved baseline image. + * + * Requires Playwright's `testInfo` for attaching visual comparison artifacts to the test report. + * Supports locators with multiple states; enforces correct state usage via type constraints. + * If no baseline image exists, the element screenshot is saved and an error is thrown. + * On mismatch, a pixel-by-pixel comparison is performed and a visual diff is attached (when CI + ALLURE_ENABLED). + * Baseline screenshots are assumed to have been taken on: Pixel 6 (1080x2061) and iPhone 16 Pro Max (1320x2868) + * + * Example usage: + * // Locator with multiple states: + * await verifyElementScreenshot(device, new EmptyLandingPageScreenshot(device), testInfo, 'new_account'); + * + * // Locator with a single state: + * await verifyElementScreenshot(device, new SomeSimpleLocatorScreenshot(device), testInfo); + */ +export async function verifyElementScreenshot< + T extends LocatorsInterfaceScreenshot & { screenshotFileName: (...args: any[]) => string }, +>( + device: DeviceWrapper, + element: T, + testInfo: TestInfo, + ...args: Parameters // Enforces states when mandatory +): Promise { + const uuid = uuidv4(); + + // Get the element screenshot as base64 + const elementToScreenshot = await device.waitForTextElementToBePresent(element); + const elementScreenshotBase64: string = await device.getElementScreenshot( + elementToScreenshot.ELEMENT + ); + + // Convert the base64 string to a Buffer + let screenshotBuffer: Buffer = Buffer.from(elementScreenshotBase64, 'base64'); + screenshotBuffer = await sharp(screenshotBuffer).blur().png().toBuffer(); // Imperceptible blur to reduce antialiasing issues + + // Use shared comparison logic + const baselineScreenshotPath = element.screenshotFileName(...args); + await compareScreenshots(screenshotBuffer, baselineScreenshotPath, uuid, testInfo); +} + +/** + * Takes a full page screenshot and verifies it against a saved baseline image. + * + * Uses the same comparison logic as verifyElementScreenshot but captures the entire + * viewport and applies cropping to remove dynamic elements like status bar indicators. + * + * Requires Playwright's `testInfo` for attaching visual comparison artifacts to the test report. + * If no baseline image exists, the page screenshot is saved and an error is thrown. + * On mismatch, a pixel-by-pixel comparison is performed and a visual diff is attached (when CI + ALLURE_ENABLED). + * + * Example usage: + * await verifyPageScreenshot(device, platform, 'screenshotName', testInfo); + */ +export async function verifyPageScreenshot( + device: DeviceWrapper, + platform: SupportedPlatformsType, + screenshotName: string, + testInfo: TestInfo +): Promise { + const uuid = uuidv4(); + const baselineScreenshotPath = path.join('run', 'screenshots', platform, `${screenshotName}.png`); + + // Get full page screenshot and crop it + const pageScreenshotBase64 = await device.getScreenshot(); + const screenshotBuffer = Buffer.from(pageScreenshotBase64, 'base64'); + const croppedBuffer = await cropScreenshot(device, screenshotBuffer); + + await compareScreenshots(croppedBuffer, baselineScreenshotPath, uuid, testInfo, 5); // Slightly higher tolerance for full page screenshots +} diff --git a/run/test/specs/visual_message_bubbles.spec.ts b/run/test/specs/visual_message_bubbles.spec.ts new file mode 100644 index 000000000..23d392811 --- /dev/null +++ b/run/test/specs/visual_message_bubbles.spec.ts @@ -0,0 +1,104 @@ +import { test, type TestInfo } from '@playwright/test'; + +import { TestSteps } from '../../types/allure'; +import { bothPlatformsIt } from '../../types/sessionIt'; +import { + MessageBody, + MessageInput, + OutgoingMessageStatusSent, + SendButton, +} from './locators/conversation'; +import { open_Alice1_Bob1_friends } from './state_builder'; +import { closeApp, SupportedPlatformsType } from './utils/open_app'; +import { MessageBodyScreenshot } from './utils/screenshot_paths'; +import { verifyElementScreenshot } from './utils/verify_screenshots'; + +bothPlatformsIt({ + title: 'Check message bubble layout', + risk: 'high', + countOfDevicesNeeded: 2, + testCb: messageBubbleAppearance, + allureSuites: { + parent: 'Visual Checks', + suite: 'Conversation', + }, + allureDescription: `Verifies that message bubbles appear as expected (oneline and multiline messages, reply layout, outgoing/incoming)`, +}); + +async function messageBubbleAppearance(platform: SupportedPlatformsType, testInfo: TestInfo) { + const { + devices: { alice1, bob1 }, + prebuilt: { alice, bob }, + } = await test.step(TestSteps.SETUP.QA_SEEDER, async () => { + return open_Alice1_Bob1_friends({ + platform, + focusFriendsConvo: true, + testInfo, + }); + }); + // Alice sends a short message to Bob and Bob replies with a longer message + // This lets us verify four different message bubbles + const shortMessage = 'This is a short message'; + const replyMessage = + 'This is a longer reply message that is likely to span multiple lines which is great for testing'; + + await test.step(TestSteps.SEND.MESSAGE(alice.userName, bob.userName), async () => { + await alice1.sendMessage(shortMessage); + await alice1.waitForTextElementToBePresent(new MessageBody(alice1, shortMessage)); + }); + await test.step(TestSteps.VERIFY.SCREENSHOT('outgoing one-line message bubble'), async () => { + await verifyElementScreenshot( + alice1, + new MessageBodyScreenshot(alice1, shortMessage), + testInfo, + 'outgoing_short_message' + ); + }); + await test.step(TestSteps.VERIFY.SCREENSHOT('incoming one-line message bubble'), async () => { + await bob1.waitForTextElementToBePresent(new MessageBody(bob1, shortMessage)); + await verifyElementScreenshot( + bob1, + new MessageBodyScreenshot(bob1, shortMessage), + testInfo, + 'incoming_short_message' + ); + }); + await test.step(TestSteps.SEND.REPLY(bob.userName, alice.userName), async () => { + // Bob replies with a longer message + await bob1.longPressMessage(shortMessage); + await bob1.clickOnByAccessibilityID('Reply to message'); + await bob1.inputText(replyMessage, new MessageInput(bob1)); + await bob1.clickOnElementAll(new SendButton(bob1)); + await bob1.waitForTextElementToBePresent({ + ...new OutgoingMessageStatusSent(bob1).build(), + maxWait: 20_000, + }); + }); + await test.step( + TestSteps.VERIFY.SCREENSHOT('outgoing multiline reply message bubble'), + async () => { + await bob1.waitForTextElementToBePresent(new MessageBody(bob1, replyMessage)); // Otherwise there were stale element errors + await verifyElementScreenshot( + bob1, + new MessageBodyScreenshot(bob1, replyMessage), + testInfo, + 'outgoing_reply_message' + ); + } + ); + await test.step( + TestSteps.VERIFY.SCREENSHOT('incoming multiline reply message bubble'), + async () => { + await alice1.waitForTextElementToBePresent(new MessageBody(alice1, replyMessage)); + await verifyElementScreenshot( + alice1, + new MessageBodyScreenshot(alice1, replyMessage), + testInfo, + 'incoming_reply_message' + ); + } + ); + await test.step(TestSteps.SETUP.CLOSE_APP, async () => { + await closeApp(alice1, bob1); + }); +} diff --git a/run/test/specs/visual_settings.spec.ts b/run/test/specs/visual_settings.spec.ts new file mode 100644 index 000000000..5b6dd7852 --- /dev/null +++ b/run/test/specs/visual_settings.spec.ts @@ -0,0 +1,88 @@ +import { test, type TestInfo } from '@playwright/test'; + +import { TestSteps } from '../../types/allure'; +import { DeviceWrapper } from '../../types/DeviceWrapper'; +import { bothPlatformsIt } from '../../types/sessionIt'; +import { USERNAME } from '../../types/testing'; +import { NotificationSettings } from './locators/conversation'; +import { + AppearanceMenuItem, + ConversationsMenuItem, + PrivacyMenuItem, + UserSettings, +} from './locators/settings'; +import { sleepFor } from './utils'; +import { newUser } from './utils/create_account'; +import { closeApp, openAppOnPlatformSingleDevice, SupportedPlatformsType } from './utils/open_app'; +import { verifyPageScreenshot } from './utils/verify_screenshots'; + +const testCases = [ + { + screenName: 'Privacy settings', + screenshotFile: 'settings_privacy', + navigation: async (device: DeviceWrapper) => { + await device.clickOnElementAll(new UserSettings(device)); + await device.clickOnElementAll(new PrivacyMenuItem(device)); + }, + }, + { + screenName: 'Conversations settings', + screenshotFile: 'settings_conversations', + navigation: async (device: DeviceWrapper) => { + await device.clickOnElementAll(new UserSettings(device)); + await device.clickOnElementAll(new ConversationsMenuItem(device)); + }, + }, + { + screenName: 'Notifications settings', + screenshotFile: 'settings_notifications', + navigation: async (device: DeviceWrapper) => { + await device.clickOnElementAll(new UserSettings(device)); + await device.clickOnElementAll(new NotificationSettings(device)); + await sleepFor(1_000); // This one otherwise captures a black screen + }, + }, + { + screenName: 'Appearance settings', + screenshotFile: 'settings_appearance', + navigation: async (device: DeviceWrapper) => { + await device.clickOnElementAll(new UserSettings(device)); + await device.clickOnElementAll(new AppearanceMenuItem(device)); + }, + }, +]; + +for (const { screenName, screenshotFile, navigation } of testCases) { + bothPlatformsIt({ + title: `Check ${screenName} layout`, + risk: 'high', + countOfDevicesNeeded: 1, + allureSuites: { + parent: 'Visual Checks', + suite: 'Settings', + }, + allureDescription: `Verifies that the ${screenName} screen layout matches the expected baseline`, + testCb: async (platform: SupportedPlatformsType, testInfo: TestInfo) => { + const { device } = await test.step(TestSteps.SETUP.NEW_USER, async () => { + const { device } = await openAppOnPlatformSingleDevice(platform, testInfo); + await newUser(device, USERNAME.ALICE, { + saveUserData: false, + allowNotificationPermissions: false, + }); + return { device }; + }); + + await test.step(TestSteps.OPEN.GENERIC(screenName), async () => { + await navigation(device); + }); + + await test.step(TestSteps.VERIFY.SCREENSHOT(screenName), async () => { + await verifyPageScreenshot(device, platform, screenshotFile, testInfo); + }); + + await test.step(TestSteps.SETUP.CLOSE_APP, async () => { + await closeApp(device); + }); + }, + }); +} diff --git a/run/types/DeviceWrapper.ts b/run/types/DeviceWrapper.ts index 201f5cac3..c07f9e7aa 100644 --- a/run/types/DeviceWrapper.ts +++ b/run/types/DeviceWrapper.ts @@ -2375,7 +2375,7 @@ export class DeviceWrapper { if (this.isAndroid()) { const permissions = await this.doesElementExist({ ...locatorConfig, - maxWait: 2_000, + maxWait: 5_000, }); if (permissions) { diff --git a/run/types/allure.ts b/run/types/allure.ts index 606b5f40b..3de7b21d7 100644 --- a/run/types/allure.ts +++ b/run/types/allure.ts @@ -39,7 +39,7 @@ export type AllureSuiteConfig = | 'Set Nickname' | 'Share to Session'; } - | { parent: 'Visual Checks' } + | { parent: 'Visual Checks'; suite: 'Conversation' | 'Onboarding' | 'Settings' } | { parent: 'Voice Calls' }; /** @@ -71,12 +71,14 @@ export const TestSteps = { SEND: { MESSAGE: (sender: UserNameType, recipient: string) => `${sender} sends a message to ${recipient}`, + REPLY: (sender: UserNameType, recipient: string) => `${sender} replies to ${recipient}`, LINK: 'Send Link', IMAGE: 'Send Image', EMOJI_REACT: `Send an emoji react`, }, // Open/Navigate steps OPEN: { + GENERIC: (string: string) => `Open ${string}`, NTS: 'Open Note to Self', UPDATE_GROUP_INFO: `Open 'Update Group Information' modal`, PATH: 'Open Path screen', @@ -98,8 +100,7 @@ export const TestSteps = { }, // Verify steps VERIFY: { - ELEMENT_SCREENSHOT: (elementDesc: string) => - `Verify ${elementDesc} element screenshot matches baseline`, + SCREENSHOT: (desc: string) => `Verify ${desc} screenshot matches baseline`, GENERIC_MODAL: 'Verify modal strings', SPECIFIC_MODAL: (modalDesc: string) => `Verify ${modalDesc} modal strings`, MESSAGE_SYNCED: 'Verify message synced to linked device', diff --git a/run/types/testing.ts b/run/types/testing.ts index 0938cb918..c66b5b1bd 100644 --- a/run/types/testing.ts +++ b/run/types/testing.ts @@ -574,8 +574,12 @@ export type Id = export type TestRisk = 'high' | 'low' | 'medium'; -export type ElementStates = 'new_account' | 'restore_account'; - -export type Suffix = 'diff' | 'screenshot'; +export type ElementStates = + | 'incoming_reply_message' + | 'incoming_short_message' + | 'new_account' + | 'outgoing_reply_message' + | 'outgoing_short_message' + | 'restore_account'; export type AppName = 'Session AQA' | 'Session QA'; From 03455ddff90bbfbcc09612b71341e2eda66a857f Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Thu, 9 Oct 2025 11:43:02 +1100 Subject: [PATCH 03/29] feat: add ssim.js --- package.json | 1 + yarn.lock | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/package.json b/package.json index 45eb54c94..19b408ae6 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "prettier": "^3.3.3", "sharp": "^0.34.2", "sinon": "^19.0.2", + "ssim.js": "^3.5.0", "sync-request-curl": "^3.3.3", "ts-node": "^10.9.1", "typescript": "^5.6.3", diff --git a/yarn.lock b/yarn.lock index 82c08a783..63ede4e6f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7250,6 +7250,7 @@ __metadata: prettier: "npm:^3.3.3" sharp: "npm:^0.34.2" sinon: "npm:^19.0.2" + ssim.js: "npm:^3.5.0" sync-request-curl: "npm:^3.3.3" ts-node: "npm:^10.9.1" typescript: "npm:^5.6.3" @@ -7632,6 +7633,13 @@ __metadata: languageName: node linkType: hard +"ssim.js@npm:^3.5.0": + version: 3.5.0 + resolution: "ssim.js@npm:3.5.0" + checksum: 10c0/9e7101da17395d3acbd417aac712d8f156522e79059a27cb38882eedd5a8868e31871c8f58bec3a150f8cf0660883cf22310cbd2fd63b408c1fd0ab02fda9fbc + languageName: node + linkType: hard + "ssri@npm:^10.0.0": version: 10.0.6 resolution: "ssri@npm:10.0.6" From 69c1c65f5254e619cf39983fe354e0af273a8200 Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Thu, 9 Oct 2025 11:55:46 +1100 Subject: [PATCH 04/29] feat: use SSIM instead of pixel matching for screenshot comparison --- run/screenshots/android/app_disguise.png | 4 +- .../android/conversation_alice.png | 3 + run/screenshots/android/conversation_bob.png | 3 + .../android/landingpage_new_account.png | 4 +- .../android/landingpage_restore_account.png | 4 +- .../messagebody_incoming_reply_message.png | 3 - .../messagebody_incoming_short_message.png | 3 - .../messagebody_outgoing_reply_message.png | 3 - .../messagebody_outgoing_short_message.png | 3 - .../android/settings_appearance.png | 4 +- .../android/settings_conversations.png | 4 +- .../android/settings_notifications.png | 4 +- run/screenshots/android/settings_privacy.png | 4 +- run/screenshots/ios/app_disguise.png | 4 +- run/screenshots/ios/conversation_alice.png | 3 + run/screenshots/ios/conversation_bob.png | 3 + .../ios/landingpage_new_account.png | 4 +- .../ios/landingpage_restore_account.png | 4 +- .../messagebody_incoming_reply_message.png | 3 - .../messagebody_incoming_short_message.png | 3 - .../messagebody_outgoing_reply_message.png | 3 - .../messagebody_outgoing_short_message.png | 3 - run/screenshots/ios/settings_appearance.png | 4 +- .../ios/settings_conversations.png | 4 +- .../ios/settings_notifications.png | 4 +- run/screenshots/ios/settings_privacy.png | 4 +- run/test/specs/app_disguise_icons.spec.ts | 5 +- .../specs/landing_page_new_account.spec.ts | 12 +- .../landing_page_restore_account.spec.ts | 10 +- run/test/specs/utils/screenshot_paths.ts | 34 -- run/test/specs/utils/verify_screenshots.ts | 342 ++++++++++-------- run/test/specs/visual_message_bubbles.spec.ts | 52 +-- 32 files changed, 254 insertions(+), 293 deletions(-) create mode 100644 run/screenshots/android/conversation_alice.png create mode 100644 run/screenshots/android/conversation_bob.png delete mode 100644 run/screenshots/android/messagebody_incoming_reply_message.png delete mode 100644 run/screenshots/android/messagebody_incoming_short_message.png delete mode 100644 run/screenshots/android/messagebody_outgoing_reply_message.png delete mode 100644 run/screenshots/android/messagebody_outgoing_short_message.png create mode 100644 run/screenshots/ios/conversation_alice.png create mode 100644 run/screenshots/ios/conversation_bob.png delete mode 100644 run/screenshots/ios/messagebody_incoming_reply_message.png delete mode 100644 run/screenshots/ios/messagebody_incoming_short_message.png delete mode 100644 run/screenshots/ios/messagebody_outgoing_reply_message.png delete mode 100644 run/screenshots/ios/messagebody_outgoing_short_message.png delete mode 100644 run/test/specs/utils/screenshot_paths.ts diff --git a/run/screenshots/android/app_disguise.png b/run/screenshots/android/app_disguise.png index 211502d3a..605c2d421 100644 --- a/run/screenshots/android/app_disguise.png +++ b/run/screenshots/android/app_disguise.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4e12299e64f2c1cf4d067efab6b2c64c54a4581e19bd01b46a0f2b74f46f92f5 -size 172617 +oid sha256:0174d2fcb331bf2bc8876fdff2f024116ef721d58df6a75c8cb78cceec0df574 +size 147441 diff --git a/run/screenshots/android/conversation_alice.png b/run/screenshots/android/conversation_alice.png new file mode 100644 index 000000000..0ee18c700 --- /dev/null +++ b/run/screenshots/android/conversation_alice.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:075707255598a95252744a10f63b009aab26f51261fd1886b30bfe8ec37e8873 +size 88351 diff --git a/run/screenshots/android/conversation_bob.png b/run/screenshots/android/conversation_bob.png new file mode 100644 index 000000000..151c57736 --- /dev/null +++ b/run/screenshots/android/conversation_bob.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aec8729a484575aa8392ceec5f7642992ace478c68f1066b83d4cb364375cbd8 +size 162455 diff --git a/run/screenshots/android/landingpage_new_account.png b/run/screenshots/android/landingpage_new_account.png index 646bfa47e..5a5de8b36 100644 --- a/run/screenshots/android/landingpage_new_account.png +++ b/run/screenshots/android/landingpage_new_account.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:66411453c10cb273ccc36c73d65ab80ad30a4a0b170c62dd849e0397480f7bab -size 124569 +oid sha256:aec8609e7d0013725d3be235b552a9399e94e2143f3edbeda4c7172c1a29f0ff +size 103496 diff --git a/run/screenshots/android/landingpage_restore_account.png b/run/screenshots/android/landingpage_restore_account.png index 78202c5b8..f979b2f26 100644 --- a/run/screenshots/android/landingpage_restore_account.png +++ b/run/screenshots/android/landingpage_restore_account.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b91c51f0fefff264e464cfee3df2b283b6bf52aae8aeafa47498dfae613fc51a -size 88133 +oid sha256:9cfdee6c04fc5c6e395e77fc92b03ef674d31254fe75fff72a326c9a1ab3ec30 +size 79441 diff --git a/run/screenshots/android/messagebody_incoming_reply_message.png b/run/screenshots/android/messagebody_incoming_reply_message.png deleted file mode 100644 index d13048f9a..000000000 --- a/run/screenshots/android/messagebody_incoming_reply_message.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ed707da45a0d20692ce60311a28dea03fb6d081b903c5877821ea4726057d0d7 -size 31612 diff --git a/run/screenshots/android/messagebody_incoming_short_message.png b/run/screenshots/android/messagebody_incoming_short_message.png deleted file mode 100644 index f0d115097..000000000 --- a/run/screenshots/android/messagebody_incoming_short_message.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:9910e433e7f161c4bdf17b1367c3616ac1ce95399240a5ccf6930bfd89fd1332 -size 10431 diff --git a/run/screenshots/android/messagebody_outgoing_reply_message.png b/run/screenshots/android/messagebody_outgoing_reply_message.png deleted file mode 100644 index 383955e8a..000000000 --- a/run/screenshots/android/messagebody_outgoing_reply_message.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b7ccbf17d4e6f2ba0ca490462e8464c72c357fc1e950722178caab12acfb2c14 -size 38690 diff --git a/run/screenshots/android/messagebody_outgoing_short_message.png b/run/screenshots/android/messagebody_outgoing_short_message.png deleted file mode 100644 index a53664175..000000000 --- a/run/screenshots/android/messagebody_outgoing_short_message.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e79bf04d5d933a2009d51e002e3210bd71ae8bf3b4f8a6425af557a620b81407 -size 12809 diff --git a/run/screenshots/android/settings_appearance.png b/run/screenshots/android/settings_appearance.png index c8058b8a0..7803ea423 100644 --- a/run/screenshots/android/settings_appearance.png +++ b/run/screenshots/android/settings_appearance.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:867ebc1c445305f15ef6ac99b7d2662818c15fd8db91c6b26e93093684249cc4 -size 175739 +oid sha256:d513f037ba20f8b3026f159678e3bf5eeb049e09b539f836993b8cafa3ec8cda +size 128602 diff --git a/run/screenshots/android/settings_conversations.png b/run/screenshots/android/settings_conversations.png index b9a0ec72b..983c94eab 100644 --- a/run/screenshots/android/settings_conversations.png +++ b/run/screenshots/android/settings_conversations.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:96fecdfd6b84ee9a1a8d14c97de141f3da09d649134c8fbb3a699bbbcded74fb -size 194821 +oid sha256:183f7aac856c61eb5f46638fc25dbb1755215712752b50f721698f3301a232d8 +size 156289 diff --git a/run/screenshots/android/settings_notifications.png b/run/screenshots/android/settings_notifications.png index e7179c989..0cdc813c4 100644 --- a/run/screenshots/android/settings_notifications.png +++ b/run/screenshots/android/settings_notifications.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5ffb70b714367d130cfb7a11063a3a8bf627ab269fc1ea3db25dafc698095346 -size 180821 +oid sha256:846ea6ce60f9014e0fa36823e176da62f6f27fc63026ea1291f66b69cebf0730 +size 142963 diff --git a/run/screenshots/android/settings_privacy.png b/run/screenshots/android/settings_privacy.png index 1c7954756..7afe312b0 100644 --- a/run/screenshots/android/settings_privacy.png +++ b/run/screenshots/android/settings_privacy.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:73508116cab27eb1964c5928a1f6c7b62b766d567bc78234d27799aa3828b9df -size 249297 +oid sha256:2db83338c651b6e56b70f23b7faeae92dc98bb652e24b52c0ac961ec6d7b3e47 +size 200618 diff --git a/run/screenshots/ios/app_disguise.png b/run/screenshots/ios/app_disguise.png index 7ac619e06..c7609276a 100644 --- a/run/screenshots/ios/app_disguise.png +++ b/run/screenshots/ios/app_disguise.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c52f1ca9a0d2b9bd7963e0079794f9937f254d4f3cec2d06c0210570912b7d30 -size 306456 +oid sha256:47b1c7cb55a03ffa0818b71661eeb2cac6c98f70766d5735944b9d8203e4e037 +size 501359 diff --git a/run/screenshots/ios/conversation_alice.png b/run/screenshots/ios/conversation_alice.png new file mode 100644 index 000000000..75576e6c8 --- /dev/null +++ b/run/screenshots/ios/conversation_alice.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bc4e1026534f5885ddb6648472f01becbc16b0065e9b514278bdd04e5257d53 +size 313525 diff --git a/run/screenshots/ios/conversation_bob.png b/run/screenshots/ios/conversation_bob.png new file mode 100644 index 000000000..43e60ac88 --- /dev/null +++ b/run/screenshots/ios/conversation_bob.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ffebd513e3d787dd57ea7cbf6c5b6cd323bac20f936bff5024fb06c7992d433 +size 315152 diff --git a/run/screenshots/ios/landingpage_new_account.png b/run/screenshots/ios/landingpage_new_account.png index 477230ec7..0fc079315 100644 --- a/run/screenshots/ios/landingpage_new_account.png +++ b/run/screenshots/ios/landingpage_new_account.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ed49daf2bbafa9149844023697c84d29d80fd42c757571035bd915c0939427ea -size 151942 +oid sha256:643b7aaf13a7f4dd456a5f9331948acfc02ced68a2492de20d10a54a0054c8db +size 190440 diff --git a/run/screenshots/ios/landingpage_restore_account.png b/run/screenshots/ios/landingpage_restore_account.png index 16d398874..5ece9b270 100644 --- a/run/screenshots/ios/landingpage_restore_account.png +++ b/run/screenshots/ios/landingpage_restore_account.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:67c060cc837cc24aede99498085e7bcc96408270e4d20c69569871cbf07ec94e -size 122304 +oid sha256:f3c636dc9de34f2490c2242c260e7c6d9ee37d905a4786de1522db8b1f76a81a +size 175782 diff --git a/run/screenshots/ios/messagebody_incoming_reply_message.png b/run/screenshots/ios/messagebody_incoming_reply_message.png deleted file mode 100644 index 2e401f1c2..000000000 --- a/run/screenshots/ios/messagebody_incoming_reply_message.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:543e5b07647a363cae67168fcc59e839a87d819ade255dd1059447e950598db7 -size 52588 diff --git a/run/screenshots/ios/messagebody_incoming_short_message.png b/run/screenshots/ios/messagebody_incoming_short_message.png deleted file mode 100644 index bbe86ae01..000000000 --- a/run/screenshots/ios/messagebody_incoming_short_message.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2840b29abde3d1f9319dffee80a4c357e65dae1a8fb0153b8bc2650aa4da6a73 -size 14734 diff --git a/run/screenshots/ios/messagebody_outgoing_reply_message.png b/run/screenshots/ios/messagebody_outgoing_reply_message.png deleted file mode 100644 index 8a6fdfde7..000000000 --- a/run/screenshots/ios/messagebody_outgoing_reply_message.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5f8af6b29078b62f6dfb1bd1699ed60df6d0d73d4fe5fa770ba20324b53cfa09 -size 57028 diff --git a/run/screenshots/ios/messagebody_outgoing_short_message.png b/run/screenshots/ios/messagebody_outgoing_short_message.png deleted file mode 100644 index a10c50614..000000000 --- a/run/screenshots/ios/messagebody_outgoing_short_message.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:cd63d32fcb4c50b7f0c3050299fe471ab3359116fb2a5ad12d65faf9de415b15 -size 15902 diff --git a/run/screenshots/ios/settings_appearance.png b/run/screenshots/ios/settings_appearance.png index 07481b37c..dce164016 100644 --- a/run/screenshots/ios/settings_appearance.png +++ b/run/screenshots/ios/settings_appearance.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:45321422bfc943e4167a778ccd5c0fc8c901ab6a9f6eed9d9bad07988f1e43a5 -size 251228 +oid sha256:d0e1b0f39d07df89d46e9b796a9177e123079081167aead1c679343f660d1f47 +size 200131 diff --git a/run/screenshots/ios/settings_conversations.png b/run/screenshots/ios/settings_conversations.png index fc8652648..c7f853cf7 100644 --- a/run/screenshots/ios/settings_conversations.png +++ b/run/screenshots/ios/settings_conversations.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:344ab1487cbd0c9a53476104e70ef7196e91e03725f1bb9d8897d6655c78164d -size 185064 +oid sha256:ef3370a4a1296aa734982c76ac6e989b9fb8acdada5ca585c47e9cf1f94cad0d +size 162443 diff --git a/run/screenshots/ios/settings_notifications.png b/run/screenshots/ios/settings_notifications.png index 9cdafce0d..dcbd1172f 100644 --- a/run/screenshots/ios/settings_notifications.png +++ b/run/screenshots/ios/settings_notifications.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:27d6f3a9926c32a93157b320fc4360f3d919a5c34cff99f36030df11a088fa0c -size 199664 +oid sha256:2c286f4276d71c13261f0991dbbcfd3a6ddbf74714a907c8b1a1f8bfa52bb3ba +size 170000 diff --git a/run/screenshots/ios/settings_privacy.png b/run/screenshots/ios/settings_privacy.png index 12e68681a..910221e13 100644 --- a/run/screenshots/ios/settings_privacy.png +++ b/run/screenshots/ios/settings_privacy.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c8cbe958c8c8b203707f7e9ce1fbffd4e450eb816583bec4c31a77772c849100 -size 340584 +oid sha256:49e67fe26e57958963d97190d339e181983ee974ccf1d0a108d283d529bb5e3e +size 257612 diff --git a/run/test/specs/app_disguise_icons.spec.ts b/run/test/specs/app_disguise_icons.spec.ts index 41820680f..c2d0807b0 100644 --- a/run/test/specs/app_disguise_icons.spec.ts +++ b/run/test/specs/app_disguise_icons.spec.ts @@ -6,8 +6,7 @@ import { USERNAME } from '../../types/testing'; import { AppearanceMenuItem, SelectAppIcon, UserSettings } from './locators/settings'; import { newUser } from './utils/create_account'; import { closeApp, openAppOnPlatformSingleDevice, SupportedPlatformsType } from './utils/open_app'; -import { AppDisguisePageScreenshot } from './utils/screenshot_paths'; -import { verifyElementScreenshot } from './utils/verify_screenshots'; +import { verifyPageScreenshot } from './utils/verify_screenshots'; bothPlatformsIt({ title: 'Check app disguise icon layout', @@ -33,7 +32,7 @@ async function appDisguiseIcons(platform: SupportedPlatformsType, testInfo: Test }); await test.step(TestSteps.VERIFY.SCREENSHOT('app disguise icons'), async () => { await device.clickOnElementAll(new SelectAppIcon(device)); - await verifyElementScreenshot(device, new AppDisguisePageScreenshot(device), testInfo); + await verifyPageScreenshot(device, platform, 'app_disguise', testInfo); }); await test.step(TestSteps.SETUP.CLOSE_APP, async () => { await closeApp(device); diff --git a/run/test/specs/landing_page_new_account.spec.ts b/run/test/specs/landing_page_new_account.spec.ts index 5099abfc1..bcd4c7d2c 100644 --- a/run/test/specs/landing_page_new_account.spec.ts +++ b/run/test/specs/landing_page_new_account.spec.ts @@ -1,11 +1,10 @@ -import type { TestInfo } from '@playwright/test'; +import { type TestInfo } from '@playwright/test'; import { bothPlatformsIt } from '../../types/sessionIt'; import { USERNAME } from '../../types/testing'; import { newUser } from './utils/create_account'; import { closeApp, openAppOnPlatformSingleDevice, SupportedPlatformsType } from './utils/open_app'; -import { EmptyLandingPageScreenshot } from './utils/screenshot_paths'; -import { verifyElementScreenshot } from './utils/verify_screenshots'; +import { verifyPageScreenshot } from './utils/verify_screenshots'; bothPlatformsIt({ title: 'Check landing page (new account) layout', @@ -23,11 +22,6 @@ async function landingPageNewAccount(platform: SupportedPlatformsType, testInfo: const { device } = await openAppOnPlatformSingleDevice(platform, testInfo); await newUser(device, USERNAME.ALICE); // Verify that the party popper is shown on the landing page - await verifyElementScreenshot( - device, - new EmptyLandingPageScreenshot(device), - testInfo, - 'new_account' - ); + await verifyPageScreenshot(device, platform, 'landingpage_new_account', testInfo); await closeApp(device); } diff --git a/run/test/specs/landing_page_restore_account.spec.ts b/run/test/specs/landing_page_restore_account.spec.ts index d3c175ed1..9b92d7c7d 100644 --- a/run/test/specs/landing_page_restore_account.spec.ts +++ b/run/test/specs/landing_page_restore_account.spec.ts @@ -5,8 +5,7 @@ import { USERNAME } from '@session-foundation/qa-seeder'; import { bothPlatformsIt } from '../../types/sessionIt'; import { linkedDevice } from './utils/link_device'; import { closeApp, openAppTwoDevices, SupportedPlatformsType } from './utils/open_app'; -import { EmptyLandingPageScreenshot } from './utils/screenshot_paths'; -import { verifyElementScreenshot } from './utils/verify_screenshots'; +import { verifyPageScreenshot } from './utils/verify_screenshots'; bothPlatformsIt({ title: 'Check landing page (restored account) layout', @@ -25,11 +24,6 @@ async function landingPageRestoreAccount(platform: SupportedPlatformsType, testI const { device1: alice1, device2: alice2 } = await openAppTwoDevices(platform, testInfo); await linkedDevice(alice1, alice2, USERNAME.ALICE); // Verify that the Session logo is shown on the landing page - await verifyElementScreenshot( - alice2, - new EmptyLandingPageScreenshot(alice2), - testInfo, - 'restore_account' - ); + await verifyPageScreenshot(alice2, platform, 'landingpage_restore_account', testInfo); await closeApp(alice1, alice2); } diff --git a/run/test/specs/utils/screenshot_paths.ts b/run/test/specs/utils/screenshot_paths.ts deleted file mode 100644 index 81392617f..000000000 --- a/run/test/specs/utils/screenshot_paths.ts +++ /dev/null @@ -1,34 +0,0 @@ -import path from 'path'; - -import { MessageBody } from '../locators/conversation'; -import { EmptyLandingPage } from '../locators/home'; -import { AppDisguisePage } from '../locators/settings'; - -// Extends locator classes with baseline screenshot paths for visual regression testing -// If a locator appears in multiple states, a state argument must be provided to screenshotFileName() - -export class EmptyLandingPageScreenshot extends EmptyLandingPage { - // The landing page has two different states depending on the onboarding flow taken - public screenshotFileName(state: 'new_account' | 'restore_account'): string { - return path.join('run', 'screenshots', this.platform, `landingpage_${state}.png`); - } -} - -export class AppDisguisePageScreenshot extends AppDisguisePage { - public screenshotFileName(): string { - return path.join('run', 'screenshots', this.platform, 'app_disguise.png'); - } -} - -export class MessageBodyScreenshot extends MessageBody { - // The message body locator can appear in different states depending on the message content - public screenshotFileName( - state: - | 'incoming_reply_message' - | 'incoming_short_message' - | 'outgoing_reply_message' - | 'outgoing_short_message' - ): string { - return path.join('run', 'screenshots', this.platform, `messagebody_${state}.png`); - } -} diff --git a/run/test/specs/utils/verify_screenshots.ts b/run/test/specs/utils/verify_screenshots.ts index fc5df3862..4db70c1a8 100644 --- a/run/test/specs/utils/verify_screenshots.ts +++ b/run/test/specs/utils/verify_screenshots.ts @@ -3,12 +3,12 @@ import * as fs from 'fs'; import looksSame from 'looks-same'; import * as path from 'path'; import sharp from 'sharp'; +import { ssim } from 'ssim.js'; import { v4 as uuidv4 } from 'uuid'; import { DeviceWrapper } from '../../../types/DeviceWrapper'; -import { LocatorsInterfaceScreenshot } from '../locators'; import { SupportedPlatformsType } from './open_app'; -import { getDiffDirectory } from './utilities'; +import { getDiffDirectory, runScriptAndLog } from './utilities'; type Attachment = { name: string; @@ -16,6 +16,67 @@ type Attachment = { contentType: string; }; +interface ImageData { + data: Uint8ClampedArray; + width: number; + height: number; +} + +/** + * Eliminate any potential mismatches by mocking the status bar to always be the same + */ +async function setConsistentStatusBar(device: DeviceWrapper): Promise { + if (device.isIOS()) { + // Time: 4:20, 100% battery, full wifi signal + await runScriptAndLog( + `xcrun simctl status_bar ${device.udid} override --time "04:20" --batteryLevel 100 --batteryState charged --wifiBars 3`, + true + ); + } else if (device.isAndroid()) { + // Enable demo mode to set consistent status bar elements + await runScriptAndLog( + `adb -s ${device.udid} shell settings put global sysui_demo_allowed 1`, + true + ); + // Dismiss notifications + await runScriptAndLog( + `adb -s ${device.udid} shell am broadcast -a com.android.systemui.demo -e command notifications -e visible false`, + true + ); + // Time: 4:20 + await runScriptAndLog( + `adb -s ${device.udid} shell am broadcast -a com.android.systemui.demo -e command clock -e hhmm 0420`, + true + ); + // 100% battery + await runScriptAndLog( + `adb -s ${device.udid} shell am broadcast -a com.android.systemui.demo -e command battery -e level 100 -e plugged false`, + true + ); + // Full wifi (for some reason shows an ! next to the icon but that's fine) + await runScriptAndLog( + `adb -s ${device.udid} shell am broadcast -a com.android.systemui.demo -e command network -e wifi show -e level 4`, + true + ); + } +} + +async function clearStatusBarOverrides(device: DeviceWrapper): Promise { + try { + if (device.isIOS()) { + await runScriptAndLog(`xcrun simctl status_bar ${device.udid} clear`, true); + } else if (device.isAndroid()) { + await runScriptAndLog( + `adb -s ${device.udid} shell am broadcast -a com.android.systemui.demo -e command exit`, + true + ); + } + } catch (error) { + console.warn('Failed to clear status bar overrides:', error); + // Don't throw - this is cleanup, shouldn't fail the test + } +} + export async function pushAttachmentsToReport( testInfo: TestInfo, attachments: Attachment[] @@ -26,165 +87,148 @@ export async function pushAttachmentsToReport( } /** - * Crops screenshot to remove dynamic status bar elements (time, battery, signal) - * and other variable UI elements that cause false positives in visual comparisons - * NOTE: If cropping becomes unreliable there are methods to manipulate iOS and Android status bars + * Converts image buffer to SSIM-compatible ImageData format + */ +async function bufferToImageData(imageBuffer: Buffer): Promise { + const image = sharp(imageBuffer); + const { width, height } = await image.metadata(); + const rawBuffer = await image.raw().toBuffer(); + + return { + data: new Uint8ClampedArray(rawBuffer), + width: width, + height: height, + }; +} + +/** + * Converts file path to SSIM-compatible ImageData format */ -async function cropScreenshot(_device: DeviceWrapper, screenshotBuffer: Buffer): Promise { - const image = sharp(screenshotBuffer); +async function fileToImageData(filePath: string): Promise { + const image = sharp(filePath); const { width, height } = await image.metadata(); - const cropTop = 150; - const cropLeft = 5; // I was getting weird rendering artifacts on the edges - const cropRight = 5; - const cropWidth = width - cropRight; - const cropHeight = height - cropTop; - - return sharp(screenshotBuffer) - .extract({ left: cropLeft, top: cropTop, width: cropWidth, height: cropHeight }) - .png() - .blur() // This blur is imperceptible but gets rid of all the antialiasing issues - .toBuffer(); + const rawBuffer = await image.raw().toBuffer(); + + return { + data: new Uint8ClampedArray(rawBuffer), + width: width, + height: height, + }; } /** - * Shared logic for screenshot comparison and Allure reporting + * Performs SSIM comparison with optional fallback to looks-same for diff generation + * SSIM focuses on structural similarity rather than pixel-perfect matching, making it + * robust to minor rendering differences while still catching layout changes */ -async function compareScreenshots( - actualScreenshotBuffer: Buffer, - baselineScreenshotPath: string, - uuid: string, +async function compareWithSSIM( + actualBuffer: Buffer, + baselineImagePath: string, testInfo: TestInfo, - tolerance?: number + threshold: number = 0.95 ): Promise { - const diffsDir = getDiffDirectory(); - const actualScreenshotPath = path.join(diffsDir, `${uuid}_screenshot.png`); - fs.writeFileSync(actualScreenshotPath, actualScreenshotBuffer); - - // Check if baseline exists - if (!fs.existsSync(baselineScreenshotPath)) { - // For local development you can uncomment these lines to auto-save baselines at the correct location - // fs.mkdirSync(path.dirname(baselineScreenshotPath), { recursive: true }); - // fs.writeFileSync(baselineScreenshotPath, actualScreenshotBuffer); + const actualImageData = await bufferToImageData(actualBuffer); + const baselineImageData = await fileToImageData(baselineImagePath); + + // Check dimensions match + if ( + actualImageData.width !== baselineImageData.width || + actualImageData.height !== baselineImageData.height + ) { throw new Error( - `No baseline image found at: ${baselineScreenshotPath}. A new screenshot has been saved at: ${actualScreenshotPath}` + `Image dimensions don't match: actual ${actualImageData.width}x${actualImageData.height}, \n + baseline ${baselineImageData.width}x${baselineImageData.height}` ); } - // Compare screenshots - console.log('Attempting visual comparison...'); - const { equal, diffImage } = await looksSame(actualScreenshotPath, baselineScreenshotPath, { - createDiffImage: true, - tolerance: tolerance, - }); - if (!equal) { - const diffImagePath = path.join(diffsDir, `${uuid}_diffImage.png`); - await diffImage.save(diffImagePath); - - // For the CI, create a visual diff that renders in the Allure report - if (process.env.ALLURE_ENABLED === 'true' && process.env.CI === '1') { - // Load baseline and diff images - const baselineBase64 = fs.readFileSync(baselineScreenshotPath).toString('base64'); - const diffBase64 = fs.readFileSync(diffImagePath).toString('base64'); - const actualBase64 = actualScreenshotBuffer.toString('base64'); - - const visualDiffPayload = { - actual: `data:image/png;base64,${actualBase64}`, - expected: `data:image/png;base64,${baselineBase64}`, - diff: `data:image/png;base64,${diffBase64}`, - }; - - await pushAttachmentsToReport(testInfo, [ - { - name: 'Visual Comparison', - body: Buffer.from(JSON.stringify(visualDiffPayload), 'utf-8'), - contentType: 'application/vnd.allure.image.diff', - }, - { - name: 'Baseline Screenshot', - body: Buffer.from(baselineBase64, 'base64'), - contentType: 'image/png', - }, - { - name: 'Actual Screenshot', - body: actualScreenshotBuffer, - contentType: 'image/png', - }, - { - name: 'Diff Screenshot', - body: Buffer.from(diffBase64, 'base64'), - contentType: 'image/png', - }, - ]); - } + const { mssim } = ssim(actualImageData, baselineImageData); + console.log(`SSIM similarity score: ${mssim.toFixed(4)}`); - console.log(`Visual comparison failed. The diff has been saved to ${diffImagePath}`); - throw new Error(`The UI doesn't match expected appearance`); - } + if (mssim < threshold) { + // Generate visual diff for debugging + const uuid = uuidv4(); + const diffsDir = getDiffDirectory(); + const actualPath = path.join(diffsDir, `${uuid}_actual.png`); + const diffPath = path.join(diffsDir, `${uuid}_diff.png`); - // Cleanup on success - try { - fs.unlinkSync(actualScreenshotPath); - console.log('Temporary screenshot deleted successfully'); - } catch (err) { - if (err instanceof Error) { - console.error(`Error deleting file: ${err.message}`); + fs.writeFileSync(actualPath, actualBuffer); + + try { + const { diffImage } = await looksSame(actualPath, baselineImagePath, { + createDiffImage: true, + }); + + if (diffImage) { + await diffImage.save(diffPath); + console.log(`Visual diff saved to: ${diffPath}`); + } + + // Attach artifacts to report + if (process.env.ALLURE_ENABLED === 'true' && process.env.CI === '1') { + const baselineBase64 = fs.readFileSync(baselineImagePath).toString('base64'); + const diffBase64 = fs.readFileSync(diffPath).toString('base64'); + const actualBase64 = actualBuffer.toString('base64'); + const visualDiffPayload = { + actual: `data:image/png;base64,${actualBase64}`, + expected: `data:image/png;base64,${baselineBase64}`, + diff: `data:image/png;base64,${diffBase64}`, + }; + + await pushAttachmentsToReport(testInfo, [ + { + name: 'Visual Comparison', + body: Buffer.from(JSON.stringify(visualDiffPayload), 'utf-8'), + contentType: 'application/vnd.allure.image.diff', + }, + { + name: 'Baseline Screenshot', + body: Buffer.from(baselineBase64, 'base64'), + contentType: 'image/png', + }, + { + name: 'Actual Screenshot', + body: Buffer.from(actualBase64, 'base64'), + contentType: 'image/png', + }, + { + name: 'Diff Screenshot', + body: Buffer.from(diffBase64, 'base64'), + contentType: 'image/png', + }, + ]); + } + } catch (error) { + console.warn('Error processing visual diff', error); } + + console.log(`SSIM similarity score ${mssim.toFixed(4)} below threshold ${threshold}`); + throw new Error('The observed UI does not match the expected baseline'); } } /** - * Takes a screenshot of a UI element and verifies it against a saved baseline image. - * - * Requires Playwright's `testInfo` for attaching visual comparison artifacts to the test report. - * Supports locators with multiple states; enforces correct state usage via type constraints. - * If no baseline image exists, the element screenshot is saved and an error is thrown. - * On mismatch, a pixel-by-pixel comparison is performed and a visual diff is attached (when CI + ALLURE_ENABLED). - * Baseline screenshots are assumed to have been taken on: Pixel 6 (1080x2061) and iPhone 16 Pro Max (1320x2868) - * - * Example usage: - * // Locator with multiple states: - * await verifyElementScreenshot(device, new EmptyLandingPageScreenshot(device), testInfo, 'new_account'); - * - * // Locator with a single state: - * await verifyElementScreenshot(device, new SomeSimpleLocatorScreenshot(device), testInfo); + * Handles baseline creation for development */ -export async function verifyElementScreenshot< - T extends LocatorsInterfaceScreenshot & { screenshotFileName: (...args: any[]) => string }, ->( - device: DeviceWrapper, - element: T, - testInfo: TestInfo, - ...args: Parameters // Enforces states when mandatory -): Promise { - const uuid = uuidv4(); - - // Get the element screenshot as base64 - const elementToScreenshot = await device.waitForTextElementToBePresent(element); - const elementScreenshotBase64: string = await device.getElementScreenshot( - elementToScreenshot.ELEMENT - ); +function ensureBaseline(actualBuffer: Buffer, baselinePath: string): void { + if (!fs.existsSync(baselinePath)) { + const diffsDir = getDiffDirectory(); + const uuid = uuidv4(); + const tempPath = path.join(diffsDir, `${uuid}_new_baseline.png`); + fs.writeFileSync(tempPath, actualBuffer); - // Convert the base64 string to a Buffer - let screenshotBuffer: Buffer = Buffer.from(elementScreenshotBase64, 'base64'); - screenshotBuffer = await sharp(screenshotBuffer).blur().png().toBuffer(); // Imperceptible blur to reduce antialiasing issues + // Uncomment these lines for local development to auto-create baselines + // fs.mkdirSync(path.dirname(baselinePath), { recursive: true }); + // fs.writeFileSync(baselinePath, actualBuffer); - // Use shared comparison logic - const baselineScreenshotPath = element.screenshotFileName(...args); - await compareScreenshots(screenshotBuffer, baselineScreenshotPath, uuid, testInfo); + throw new Error( + `No baseline image found at: ${baselinePath}. \n + A new screenshot has been saved at: ${tempPath}` + ); + } } /** - * Takes a full page screenshot and verifies it against a saved baseline image. - * - * Uses the same comparison logic as verifyElementScreenshot but captures the entire - * viewport and applies cropping to remove dynamic elements like status bar indicators. - * - * Requires Playwright's `testInfo` for attaching visual comparison artifacts to the test report. - * If no baseline image exists, the page screenshot is saved and an error is thrown. - * On mismatch, a pixel-by-pixel comparison is performed and a visual diff is attached (when CI + ALLURE_ENABLED). - * - * Example usage: - * await verifyPageScreenshot(device, platform, 'screenshotName', testInfo); + * Takes a full page screenshot and verifies it against a saved baseline image using SSIM. */ export async function verifyPageScreenshot( device: DeviceWrapper, @@ -192,13 +236,25 @@ export async function verifyPageScreenshot( screenshotName: string, testInfo: TestInfo ): Promise { - const uuid = uuidv4(); - const baselineScreenshotPath = path.join('run', 'screenshots', platform, `${screenshotName}.png`); + await setConsistentStatusBar(device); + try { + // Get full page screenshot and crop it + const pageScreenshotBase64 = await device.getScreenshot(); + const screenshotBuffer = Buffer.from(pageScreenshotBase64, 'base64'); + // const croppedBuffer = await cropScreenshot(device, screenshotBuffer); - // Get full page screenshot and crop it - const pageScreenshotBase64 = await device.getScreenshot(); - const screenshotBuffer = Buffer.from(pageScreenshotBase64, 'base64'); - const croppedBuffer = await cropScreenshot(device, screenshotBuffer); + // Get baseline path and ensure it exists + const baselineScreenshotPath = path.join( + 'run', + 'screenshots', + platform, + `${screenshotName}.png` + ); + ensureBaseline(screenshotBuffer, baselineScreenshotPath); - await compareScreenshots(croppedBuffer, baselineScreenshotPath, uuid, testInfo, 5); // Slightly higher tolerance for full page screenshots + // Perform SSIM comparison + await compareWithSSIM(screenshotBuffer, baselineScreenshotPath, testInfo, 0.99); + } finally { + await clearStatusBarOverrides(device); + } } diff --git a/run/test/specs/visual_message_bubbles.spec.ts b/run/test/specs/visual_message_bubbles.spec.ts index 23d392811..4e4901b89 100644 --- a/run/test/specs/visual_message_bubbles.spec.ts +++ b/run/test/specs/visual_message_bubbles.spec.ts @@ -10,8 +10,7 @@ import { } from './locators/conversation'; import { open_Alice1_Bob1_friends } from './state_builder'; import { closeApp, SupportedPlatformsType } from './utils/open_app'; -import { MessageBodyScreenshot } from './utils/screenshot_paths'; -import { verifyElementScreenshot } from './utils/verify_screenshots'; +import { verifyPageScreenshot } from './utils/verify_screenshots'; bothPlatformsIt({ title: 'Check message bubble layout', @@ -44,24 +43,6 @@ async function messageBubbleAppearance(platform: SupportedPlatformsType, testInf await test.step(TestSteps.SEND.MESSAGE(alice.userName, bob.userName), async () => { await alice1.sendMessage(shortMessage); - await alice1.waitForTextElementToBePresent(new MessageBody(alice1, shortMessage)); - }); - await test.step(TestSteps.VERIFY.SCREENSHOT('outgoing one-line message bubble'), async () => { - await verifyElementScreenshot( - alice1, - new MessageBodyScreenshot(alice1, shortMessage), - testInfo, - 'outgoing_short_message' - ); - }); - await test.step(TestSteps.VERIFY.SCREENSHOT('incoming one-line message bubble'), async () => { - await bob1.waitForTextElementToBePresent(new MessageBody(bob1, shortMessage)); - await verifyElementScreenshot( - bob1, - new MessageBodyScreenshot(bob1, shortMessage), - testInfo, - 'incoming_short_message' - ); }); await test.step(TestSteps.SEND.REPLY(bob.userName, alice.userName), async () => { // Bob replies with a longer message @@ -74,30 +55,13 @@ async function messageBubbleAppearance(platform: SupportedPlatformsType, testInf maxWait: 20_000, }); }); - await test.step( - TestSteps.VERIFY.SCREENSHOT('outgoing multiline reply message bubble'), - async () => { - await bob1.waitForTextElementToBePresent(new MessageBody(bob1, replyMessage)); // Otherwise there were stale element errors - await verifyElementScreenshot( - bob1, - new MessageBodyScreenshot(bob1, replyMessage), - testInfo, - 'outgoing_reply_message' - ); - } - ); - await test.step( - TestSteps.VERIFY.SCREENSHOT('incoming multiline reply message bubble'), - async () => { - await alice1.waitForTextElementToBePresent(new MessageBody(alice1, replyMessage)); - await verifyElementScreenshot( - alice1, - new MessageBodyScreenshot(alice1, replyMessage), - testInfo, - 'incoming_reply_message' - ); - } - ); + await alice1.waitForTextElementToBePresent(new MessageBody(alice1, replyMessage)); + await test.step(TestSteps.VERIFY.SCREENSHOT('conversation screen (Alice)'), async () => { + await verifyPageScreenshot(alice1, platform, 'conversation_alice', testInfo); + }); + await test.step(TestSteps.VERIFY.SCREENSHOT('conversation screen (Bob)'), async () => { + await verifyPageScreenshot(bob1, platform, 'conversation_bob', testInfo); + }); await test.step(TestSteps.SETUP.CLOSE_APP, async () => { await closeApp(alice1, bob1); }); From daad111b46a54ac3f83e1f53a2d81c9641926ea2 Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Thu, 9 Oct 2025 13:12:24 +1100 Subject: [PATCH 05/29] fix: renamed android locators --- run/test/specs/locators/onboarding.ts | 4 ++-- run/types/testing.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/run/test/specs/locators/onboarding.ts b/run/test/specs/locators/onboarding.ts index dcf7c1fe1..3540363c7 100644 --- a/run/test/specs/locators/onboarding.ts +++ b/run/test/specs/locators/onboarding.ts @@ -105,7 +105,7 @@ export class TermsOfServiceButton extends LocatorsInterface { case 'android': return { strategy: 'id', - selector: 'Terms of Service', + selector: 'Terms of service button', } as const; case 'ios': return { @@ -122,7 +122,7 @@ export class PrivacyPolicyButton extends LocatorsInterface { case 'android': return { strategy: 'id', - selector: 'Privacy Policy', + selector: 'Privacy policy button', } as const; case 'ios': return { diff --git a/run/types/testing.ts b/run/types/testing.ts index 5444917aa..2a755670c 100644 --- a/run/types/testing.ts +++ b/run/types/testing.ts @@ -544,7 +544,7 @@ export type Id = | 'Open URL' | 'preferred-display-name' | 'Privacy' - | 'Privacy Policy' + | 'Privacy policy button' | 'pro-badge-text' | 'Quit' | 'rate-app-button' @@ -565,7 +565,7 @@ export type Id = | 'show-nts-confirm-button' | 'Show' | 'Slow mode notifications button' - | 'Terms of Service' + | 'Terms of service button' | 'update-group-info-confirm-button' | 'update-group-info-description-input' | 'update-group-info-name-input' From 140b6cba897431ebe76694765bd04d26abccf41b Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Thu, 9 Oct 2025 13:30:49 +1100 Subject: [PATCH 06/29] feat: add retry loop to devnet check --- run/test/specs/utils/devnet.ts | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/run/test/specs/utils/devnet.ts b/run/test/specs/utils/devnet.ts index 26fbc527a..7908a5886 100644 --- a/run/test/specs/utils/devnet.ts +++ b/run/test/specs/utils/devnet.ts @@ -1,4 +1,5 @@ import { buildStateForTest } from '@session-foundation/qa-seeder'; +import { execSync } from 'child_process'; import request from 'sync-request-curl'; import type { SupportedPlatformsType } from './open_app'; @@ -14,18 +15,32 @@ type NetworkType = Parameters[2]; // Using sync HTTP here to avoid cascading async changes through test init // This runs at test startup, so blocking is acceptable function canReachDevnet(): boolean { + const isCI = process.env.CI === '1'; + const maxAttempts = isCI ? 3 : 1; + const timeout = isCI ? 10_000 : 2_000; // Check if devnet is available - try { - const response = request('GET', DEVNET_URL, { - timeout: 2000, - }); + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + if (maxAttempts > 1) { + console.log(`Checking devnet accessibility (attempt ${attempt}/${maxAttempts})...`); + } - console.log(`Internal devnet is accessible (HTTP ${response.statusCode})`); - return true; - } catch { - console.log('Internal devnet is not accessible'); - return false; + const response = request('GET', DEVNET_URL, { timeout }); + console.log(`Internal devnet is accessible (HTTP ${response.statusCode})`); + return true; + } catch (error) { + const errorMsg = error instanceof Error ? error.message : 'Unknown error'; + + if (attempt === maxAttempts) { + console.log(`Internal devnet is not accessible: ${errorMsg}`); + } else { + console.log(`Attempt ${attempt} failed: ${errorMsg}, retrying...`); + execSync(`sleep ${attempt}`); + } + } } + + return false; } function isAutomaticQABuildAndroid(apkPath: string): boolean { // Check env var first (for CI), then filename (for local) From fc548aaa7a9d27ad5b8c2a235727f3075719c08a Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Thu, 9 Oct 2025 13:46:08 +1100 Subject: [PATCH 07/29] fix: use correct baseline screenshots --- run/screenshots/ios/conversation_alice.png | 4 ++-- run/screenshots/ios/conversation_bob.png | 4 ++-- run/screenshots/ios/settings_appearance.png | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/run/screenshots/ios/conversation_alice.png b/run/screenshots/ios/conversation_alice.png index 75576e6c8..86e13bfe5 100644 --- a/run/screenshots/ios/conversation_alice.png +++ b/run/screenshots/ios/conversation_alice.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1bc4e1026534f5885ddb6648472f01becbc16b0065e9b514278bdd04e5257d53 -size 313525 +oid sha256:be209f73e146af31ae72d5555881144d27a66b3e638e1417056c29ebcb019896 +size 316129 diff --git a/run/screenshots/ios/conversation_bob.png b/run/screenshots/ios/conversation_bob.png index 43e60ac88..b660c3370 100644 --- a/run/screenshots/ios/conversation_bob.png +++ b/run/screenshots/ios/conversation_bob.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0ffebd513e3d787dd57ea7cbf6c5b6cd323bac20f936bff5024fb06c7992d433 -size 315152 +oid sha256:7164fb001a04afa53c685cec45f0115cb0562becf0af23da3d7f3d7aa4d915be +size 319475 diff --git a/run/screenshots/ios/settings_appearance.png b/run/screenshots/ios/settings_appearance.png index dce164016..b611304d7 100644 --- a/run/screenshots/ios/settings_appearance.png +++ b/run/screenshots/ios/settings_appearance.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d0e1b0f39d07df89d46e9b796a9177e123079081167aead1c679343f660d1f47 -size 200131 +oid sha256:0c216cc7d671847e2d9a340027d2456a13f7836447fa7eaee0ccb12c9213b454 +size 206412 From 1f7a32d0e282f7f777cd01b4df423e6b12adf44f Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Thu, 9 Oct 2025 13:46:46 +1100 Subject: [PATCH 08/29] fix: also check devnet availability at workflow start --- .github/workflows/android-regression.yml | 50 +++++++++++++++++++++--- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/.github/workflows/android-regression.yml b/.github/workflows/android-regression.yml index ac5d80671..e5ac1343b 100644 --- a/.github/workflows/android-regression.yml +++ b/.github/workflows/android-regression.yml @@ -95,18 +95,58 @@ jobs: - name: Download APK & extract it run: | pwd + + # Check if devnet is accessible before choosing APK + echo "Checking devnet accessibility for APK selection..." + DEVNET_ACCESSIBLE=false + + # Retry logic matching your TypeScript function + for attempt in 1 2 3; do + echo "Devnet check attempt $attempt/3..." + if curl -s --connect-timeout 5 --max-time 10 http://sesh-net.local:1280 >/dev/null 2>&1; then + echo "Devnet is accessible on attempt $attempt" + DEVNET_ACCESSIBLE=true + break + else + echo "Attempt $attempt failed" + if [ $attempt -lt 3 ]; then + echo "Waiting ${attempt}s before retry..." + sleep $attempt + fi + fi + done + + if [ "$DEVNET_ACCESSIBLE" = "false" ]; then + echo "Devnet is not accessible after 3 attempts" + fi + + # Download and extract APK wget -q -O session-android.apk.tar.xz ${{ github.event.inputs.APK_URL }} tar xf session-android.apk.tar.xz mv session-android-*universal extracted - if ls extracted/*automaticQa.apk; then - mv extracted/*automaticQa.apk extracted/session-android.apk - echo "IS_AUTOMATIC_QA=true" >> $GITHUB_ENV - elif ls extracted/*qa.apk; then + # Choose APK based on devnet accessibility + if ls extracted/*automaticQa.apk 1>/dev/null 2>&1; then + if [ "$DEVNET_ACCESSIBLE" = "true" ]; then + echo "Using AQA build (devnet accessible)" + mv extracted/*automaticQa.apk extracted/session-android.apk + echo "IS_AUTOMATIC_QA=true" >> $GITHUB_ENV + else + echo "AQA build available but devnet not accessible - falling back to regular QA build" + if ls extracted/*qa.apk 1>/dev/null 2>&1; then + mv extracted/*qa.apk extracted/session-android.apk + echo "IS_AUTOMATIC_QA=false" >> $GITHUB_ENV + else + echo "No regular QA build found as fallback" + exit 1 + fi + fi + elif ls extracted/*qa.apk 1>/dev/null 2>&1; then + echo "Using regular QA build" mv extracted/*qa.apk extracted/session-android.apk echo "IS_AUTOMATIC_QA=false" >> $GITHUB_ENV else - echo "Error: No .qa APK found (only .qa builds are supported)" + echo "No suitable APK found" exit 1 fi From c5abac4502226ce9b6dba8112d5ab7e780523eb1 Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Thu, 9 Oct 2025 13:56:25 +1100 Subject: [PATCH 09/29] fix: threshold is parametrized --- run/test/specs/utils/verify_screenshots.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run/test/specs/utils/verify_screenshots.ts b/run/test/specs/utils/verify_screenshots.ts index 4db70c1a8..789035b1d 100644 --- a/run/test/specs/utils/verify_screenshots.ts +++ b/run/test/specs/utils/verify_screenshots.ts @@ -125,8 +125,8 @@ async function compareWithSSIM( actualBuffer: Buffer, baselineImagePath: string, testInfo: TestInfo, - threshold: number = 0.95 ): Promise { + const threshold = 0.99 // Very strict matching since this doesn't rely on pixelmatching anymore const actualImageData = await bufferToImageData(actualBuffer); const baselineImageData = await fileToImageData(baselineImagePath); @@ -253,7 +253,7 @@ export async function verifyPageScreenshot( ensureBaseline(screenshotBuffer, baselineScreenshotPath); // Perform SSIM comparison - await compareWithSSIM(screenshotBuffer, baselineScreenshotPath, testInfo, 0.99); + await compareWithSSIM(screenshotBuffer, baselineScreenshotPath, testInfo); } finally { await clearStatusBarOverrides(device); } From 37e74b89c72d63e5913d26bcd5d55d0f0bc22549 Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Thu, 9 Oct 2025 15:09:04 +1100 Subject: [PATCH 10/29] fix: dismiss kb for message bubble test --- .github/workflows/android-regression.yml | 4 +- run/screenshots/android/conversation_bob.png | 4 +- run/test/specs/utils/utilities.ts | 48 ++++++++++++++ run/test/specs/utils/verify_screenshots.ts | 65 ++----------------- run/test/specs/visual_message_bubbles.spec.ts | 3 +- 5 files changed, 59 insertions(+), 65 deletions(-) diff --git a/.github/workflows/android-regression.yml b/.github/workflows/android-regression.yml index e5ac1343b..c6899bce4 100644 --- a/.github/workflows/android-regression.yml +++ b/.github/workflows/android-regression.yml @@ -99,7 +99,7 @@ jobs: # Check if devnet is accessible before choosing APK echo "Checking devnet accessibility for APK selection..." DEVNET_ACCESSIBLE=false - + # Retry logic matching your TypeScript function for attempt in 1 2 3; do echo "Devnet check attempt $attempt/3..." @@ -115,7 +115,7 @@ jobs: fi fi done - + if [ "$DEVNET_ACCESSIBLE" = "false" ]; then echo "Devnet is not accessible after 3 attempts" fi diff --git a/run/screenshots/android/conversation_bob.png b/run/screenshots/android/conversation_bob.png index 151c57736..cb618ed78 100644 --- a/run/screenshots/android/conversation_bob.png +++ b/run/screenshots/android/conversation_bob.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:aec8729a484575aa8392ceec5f7642992ace478c68f1066b83d4cb364375cbd8 -size 162455 +oid sha256:0f4197c0c9eea91fcbc5bdf200a18b6b3701a2d5e78ed6b8f22aebaa696fb43b +size 93516 diff --git a/run/test/specs/utils/utilities.ts b/run/test/specs/utils/utilities.ts index 1511b1d4d..b2f82561d 100644 --- a/run/test/specs/utils/utilities.ts +++ b/run/test/specs/utils/utilities.ts @@ -4,6 +4,8 @@ import { pick } from 'lodash'; import path from 'path'; import * as util from 'util'; +import { DeviceWrapper } from '../../../types/DeviceWrapper'; + const exec = util.promisify(execNotPromised); export async function runScriptAndLog(toRun: string, verbose = false): Promise { @@ -77,3 +79,49 @@ export async function assertUrlIsReachable(url: string): Promise { throw new Error(`Expected status 200 but got ${response.status} for URL: ${url}`); } } + +/** + * Eliminate any potential mismatches by mocking the status bar to always be the same + */ +export async function setConsistentStatusBar(device: DeviceWrapper): Promise { + if (device.isIOS()) { + // Time: 4:20, 100% battery, full wifi signal + await runScriptAndLog( + `xcrun simctl status_bar ${device.udid} override --time "04:20" --batteryLevel 100 --batteryState charged --wifiBars 3` + ); + } else if (device.isAndroid()) { + // Enable demo mode to set consistent status bar elements + await runScriptAndLog(`adb -s ${device.udid} shell settings put global sysui_demo_allowed 1`); + // Dismiss notifications + await runScriptAndLog( + `adb -s ${device.udid} shell am broadcast -a com.android.systemui.demo -e command notifications -e visible false` + ); + // Time: 4:20 + await runScriptAndLog( + `adb -s ${device.udid} shell am broadcast -a com.android.systemui.demo -e command clock -e hhmm 0420` + ); + // 100% battery + await runScriptAndLog( + `adb -s ${device.udid} shell am broadcast -a com.android.systemui.demo -e command battery -e level 100 -e plugged false` + ); + // Full wifi (for some reason shows an ! next to the icon but that's fine) + await runScriptAndLog( + `adb -s ${device.udid} shell am broadcast -a com.android.systemui.demo -e command network -e wifi show -e level 4` + ); + } +} + +export async function clearStatusBarOverrides(device: DeviceWrapper): Promise { + try { + if (device.isIOS()) { + await runScriptAndLog(`xcrun simctl status_bar ${device.udid} clear`); + } else if (device.isAndroid()) { + await runScriptAndLog( + `adb -s ${device.udid} shell am broadcast -a com.android.systemui.demo -e command exit` + ); + } + } catch (error) { + console.warn('Failed to clear status bar overrides:', error); + // Don't throw - this is cleanup, shouldn't fail the test + } +} \ No newline at end of file diff --git a/run/test/specs/utils/verify_screenshots.ts b/run/test/specs/utils/verify_screenshots.ts index 789035b1d..72e962bf6 100644 --- a/run/test/specs/utils/verify_screenshots.ts +++ b/run/test/specs/utils/verify_screenshots.ts @@ -8,7 +8,8 @@ import { v4 as uuidv4 } from 'uuid'; import { DeviceWrapper } from '../../../types/DeviceWrapper'; import { SupportedPlatformsType } from './open_app'; -import { getDiffDirectory, runScriptAndLog } from './utilities'; +import { getDiffDirectory } from './utilities'; +import { clearStatusBarOverrides, setConsistentStatusBar } from './utilities'; type Attachment = { name: string; @@ -22,62 +23,7 @@ interface ImageData { height: number; } -/** - * Eliminate any potential mismatches by mocking the status bar to always be the same - */ -async function setConsistentStatusBar(device: DeviceWrapper): Promise { - if (device.isIOS()) { - // Time: 4:20, 100% battery, full wifi signal - await runScriptAndLog( - `xcrun simctl status_bar ${device.udid} override --time "04:20" --batteryLevel 100 --batteryState charged --wifiBars 3`, - true - ); - } else if (device.isAndroid()) { - // Enable demo mode to set consistent status bar elements - await runScriptAndLog( - `adb -s ${device.udid} shell settings put global sysui_demo_allowed 1`, - true - ); - // Dismiss notifications - await runScriptAndLog( - `adb -s ${device.udid} shell am broadcast -a com.android.systemui.demo -e command notifications -e visible false`, - true - ); - // Time: 4:20 - await runScriptAndLog( - `adb -s ${device.udid} shell am broadcast -a com.android.systemui.demo -e command clock -e hhmm 0420`, - true - ); - // 100% battery - await runScriptAndLog( - `adb -s ${device.udid} shell am broadcast -a com.android.systemui.demo -e command battery -e level 100 -e plugged false`, - true - ); - // Full wifi (for some reason shows an ! next to the icon but that's fine) - await runScriptAndLog( - `adb -s ${device.udid} shell am broadcast -a com.android.systemui.demo -e command network -e wifi show -e level 4`, - true - ); - } -} - -async function clearStatusBarOverrides(device: DeviceWrapper): Promise { - try { - if (device.isIOS()) { - await runScriptAndLog(`xcrun simctl status_bar ${device.udid} clear`, true); - } else if (device.isAndroid()) { - await runScriptAndLog( - `adb -s ${device.udid} shell am broadcast -a com.android.systemui.demo -e command exit`, - true - ); - } - } catch (error) { - console.warn('Failed to clear status bar overrides:', error); - // Don't throw - this is cleanup, shouldn't fail the test - } -} - -export async function pushAttachmentsToReport( +async function pushAttachmentsToReport( testInfo: TestInfo, attachments: Attachment[] ): Promise { @@ -124,9 +70,9 @@ async function fileToImageData(filePath: string): Promise { async function compareWithSSIM( actualBuffer: Buffer, baselineImagePath: string, - testInfo: TestInfo, + testInfo: TestInfo ): Promise { - const threshold = 0.99 // Very strict matching since this doesn't rely on pixelmatching anymore + const threshold = 0.99; // Very strict matching since this doesn't rely on pixelmatching anymore const actualImageData = await bufferToImageData(actualBuffer); const baselineImageData = await fileToImageData(baselineImagePath); @@ -241,7 +187,6 @@ export async function verifyPageScreenshot( // Get full page screenshot and crop it const pageScreenshotBase64 = await device.getScreenshot(); const screenshotBuffer = Buffer.from(pageScreenshotBase64, 'base64'); - // const croppedBuffer = await cropScreenshot(device, screenshotBuffer); // Get baseline path and ensure it exists const baselineScreenshotPath = path.join( diff --git a/run/test/specs/visual_message_bubbles.spec.ts b/run/test/specs/visual_message_bubbles.spec.ts index 4e4901b89..eb0014939 100644 --- a/run/test/specs/visual_message_bubbles.spec.ts +++ b/run/test/specs/visual_message_bubbles.spec.ts @@ -55,11 +55,12 @@ async function messageBubbleAppearance(platform: SupportedPlatformsType, testInf maxWait: 20_000, }); }); - await alice1.waitForTextElementToBePresent(new MessageBody(alice1, replyMessage)); await test.step(TestSteps.VERIFY.SCREENSHOT('conversation screen (Alice)'), async () => { + await alice1.waitForTextElementToBePresent(new MessageBody(alice1, replyMessage)); await verifyPageScreenshot(alice1, platform, 'conversation_alice', testInfo); }); await test.step(TestSteps.VERIFY.SCREENSHOT('conversation screen (Bob)'), async () => { + await bob1.onAndroid().back(); // dismiss keyboard await verifyPageScreenshot(bob1, platform, 'conversation_bob', testInfo); }); await test.step(TestSteps.SETUP.CLOSE_APP, async () => { From e0401af7567b343fa76a1d96bb6746d5b0da6793 Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Thu, 9 Oct 2025 15:39:28 +1100 Subject: [PATCH 11/29] feat: add settings page visual check decrease tolerance to 97 fix ambiguous notificationssettings locator move statusbar faking utils to utilities --- run/screenshots/android/settings.png | 3 ++ run/screenshots/ios/settings.png | 3 ++ run/test/specs/disappearing_call.spec.ts | 4 +- run/test/specs/locators/conversation.ts | 4 +- run/test/specs/locators/settings.ts | 17 +++++++++ run/test/specs/utils/open_app.ts | 43 ++++++++++++---------- run/test/specs/utils/utilities.ts | 2 +- run/test/specs/utils/verify_screenshots.ts | 6 +-- run/test/specs/visual_settings.spec.ts | 11 +++++- run/test/specs/voice_calls.spec.ts | 6 +-- 10 files changed, 66 insertions(+), 33 deletions(-) create mode 100644 run/screenshots/android/settings.png create mode 100644 run/screenshots/ios/settings.png diff --git a/run/screenshots/android/settings.png b/run/screenshots/android/settings.png new file mode 100644 index 000000000..e4bbcc15f --- /dev/null +++ b/run/screenshots/android/settings.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e290aecdb8ba4e13606c65c3cf549b381c20abcdb3dbbfb882c7bceb328b41d0 +size 134845 diff --git a/run/screenshots/ios/settings.png b/run/screenshots/ios/settings.png new file mode 100644 index 000000000..61fe0874b --- /dev/null +++ b/run/screenshots/ios/settings.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c34aed56b3a5be48831fab0309da694272237905f5d6ae7614a1bc7494fdd295 +size 223059 diff --git a/run/test/specs/disappearing_call.spec.ts b/run/test/specs/disappearing_call.spec.ts index 1ce067e4b..93a901080 100644 --- a/run/test/specs/disappearing_call.spec.ts +++ b/run/test/specs/disappearing_call.spec.ts @@ -2,7 +2,7 @@ import type { TestInfo } from '@playwright/test'; import { bothPlatformsItSeparate } from '../../types/sessionIt'; import { DISAPPEARING_TIMES } from '../../types/testing'; -import { CallButton, NotificationSettings, NotificationSwitch } from './locators/conversation'; +import { CallButton, NotificationsModalButton, NotificationSwitch } from './locators/conversation'; import { ContinueButton } from './locators/global'; import { open_Alice1_Bob1_friends } from './state_builder'; import { sleepFor } from './utils'; @@ -123,7 +123,7 @@ async function disappearingCallMessage1o1Android( await alice1.clickOnElementById( 'com.android.permissioncontroller:id/permission_allow_foreground_only_button' ); - await alice1.clickOnElementAll(new NotificationSettings(alice1)); + await alice1.clickOnElementAll(new NotificationsModalButton(alice1)); await alice1.clickOnElementAll(new NotificationSwitch(alice1)); // Return to conversation await alice1.navigateBack(false); diff --git a/run/test/specs/locators/conversation.ts b/run/test/specs/locators/conversation.ts index 47c5b3c15..620b96a2b 100644 --- a/run/test/specs/locators/conversation.ts +++ b/run/test/specs/locators/conversation.ts @@ -234,12 +234,12 @@ export class ConversationHeaderName extends LocatorsInterface { } } -export class NotificationSettings extends LocatorsInterface { +export class NotificationsModalButton extends LocatorsInterface { public build() { switch (this.platform) { case 'android': return { - strategy: 'id', + strategy: 'accessibility id', selector: 'Notifications', } as const; case 'ios': diff --git a/run/test/specs/locators/settings.ts b/run/test/specs/locators/settings.ts index b485be8aa..a7093e2f7 100644 --- a/run/test/specs/locators/settings.ts +++ b/run/test/specs/locators/settings.ts @@ -177,6 +177,23 @@ export class ConversationsMenuItem extends LocatorsInterface { } } +export class NotificationsMenuItem extends LocatorsInterface { + public build() { + switch (this.platform) { + case 'android': + return { + strategy: 'id', + selector: 'Notifications', + } as const; + case 'ios': + return { + strategy: 'accessibility id', + selector: 'Notifications', + } as const; + } + } +} + export class AppearanceMenuItem extends LocatorsInterface { public build() { switch (this.platform) { diff --git a/run/test/specs/utils/open_app.ts b/run/test/specs/utils/open_app.ts index f523109ca..723237e83 100644 --- a/run/test/specs/utils/open_app.ts +++ b/run/test/specs/utils/open_app.ts @@ -291,27 +291,30 @@ const openiOSApp = async ( device: DeviceWrapper; }> => { console.info('openiOSApp'); - + let actualCapabilitiesIndex: number; const parallelIndex = parseInt(process.env.TEST_PARALLEL_INDEX || '0'); - - // NOTE: This assumes DEVICES_PER_TEST_COUNT=4 is set in CI for iOS (not applicable to Android) - // Worker pools are fixed at 4 devices each regardless of actual test size: - // Worker 0: devices 0-3, Worker 1: devices 4-7, Worker 2: devices 8-11 - const devicesPerWorker = getDevicesPerTestCount(); - const workerBaseOffset = devicesPerWorker * parallelIndex; - - // Apply retry offset, but wrap within the worker's device pool only - // This means when retrying, alice/bob etc won't be the same device as before within a worker's pool - // This is to avoid any issues where a device might be in a bad state for some reason - // (e.g. not accessing photo library on iOS) - const retryOffset = testInfo.retry || 0; - const deviceIndexWithinWorker = (capabilitiesIndex + retryOffset) % devicesPerWorker; - const actualCapabilitiesIndex = workerBaseOffset + deviceIndexWithinWorker; - - if (retryOffset > 0) { - console.info( - `Retry offset applied (#${retryOffset}), rotating device allocations within worker` - ); + if (process.env.CI === '1') { + // NOTE: This assumes DEVICES_PER_TEST_COUNT=4 is set in CI for iOS (not applicable to Android) + // Worker pools are fixed at 4 devices each regardless of actual test size: + // Worker 0: devices 0-3, Worker 1: devices 4-7, Worker 2: devices 8-11 + const devicesPerWorker = getDevicesPerTestCount(); + const workerBaseOffset = devicesPerWorker * parallelIndex; + + // Apply retry offset, but wrap within the worker's device pool only + // This means when retrying, alice/bob etc won't be the same device as before within a worker's pool + // This is to avoid any issues where a device might be in a bad state for some reason + // (e.g. not accessing photo library on iOS) + const retryOffset = testInfo.retry || 0; + const deviceIndexWithinWorker = (capabilitiesIndex + retryOffset) % devicesPerWorker; + actualCapabilitiesIndex = workerBaseOffset + deviceIndexWithinWorker; + + if (retryOffset > 0) { + console.info( + `Retry offset applied (#${retryOffset}), rotating device allocations within worker` + ); + } + } else { + actualCapabilitiesIndex = capabilitiesIndex; } const opts: XCUITestDriverOpts = { diff --git a/run/test/specs/utils/utilities.ts b/run/test/specs/utils/utilities.ts index b2f82561d..809a113dc 100644 --- a/run/test/specs/utils/utilities.ts +++ b/run/test/specs/utils/utilities.ts @@ -124,4 +124,4 @@ export async function clearStatusBarOverrides(device: DeviceWrapper): Promise { - const threshold = 0.99; // Very strict matching since this doesn't rely on pixelmatching anymore + const threshold = 0.97; // Strict matching since this doesn't rely on pixelmatching anymore const actualImageData = await bufferToImageData(actualBuffer); const baselineImageData = await fileToImageData(baselineImagePath); @@ -163,8 +163,8 @@ function ensureBaseline(actualBuffer: Buffer, baselinePath: string): void { fs.writeFileSync(tempPath, actualBuffer); // Uncomment these lines for local development to auto-create baselines - // fs.mkdirSync(path.dirname(baselinePath), { recursive: true }); - // fs.writeFileSync(baselinePath, actualBuffer); + fs.mkdirSync(path.dirname(baselinePath), { recursive: true }); + fs.writeFileSync(baselinePath, actualBuffer); throw new Error( `No baseline image found at: ${baselinePath}. \n diff --git a/run/test/specs/visual_settings.spec.ts b/run/test/specs/visual_settings.spec.ts index 5b6dd7852..6bbee5e5a 100644 --- a/run/test/specs/visual_settings.spec.ts +++ b/run/test/specs/visual_settings.spec.ts @@ -4,7 +4,7 @@ import { TestSteps } from '../../types/allure'; import { DeviceWrapper } from '../../types/DeviceWrapper'; import { bothPlatformsIt } from '../../types/sessionIt'; import { USERNAME } from '../../types/testing'; -import { NotificationSettings } from './locators/conversation'; +import { NotificationsMenuItem } from './locators/settings'; import { AppearanceMenuItem, ConversationsMenuItem, @@ -17,6 +17,13 @@ import { closeApp, openAppOnPlatformSingleDevice, SupportedPlatformsType } from import { verifyPageScreenshot } from './utils/verify_screenshots'; const testCases = [ + { + screenName: 'Settings page', + screenshotFile: 'settings', + navigation: async (device: DeviceWrapper) => { + await device.clickOnElementAll(new UserSettings(device)); + }, + }, { screenName: 'Privacy settings', screenshotFile: 'settings_privacy', @@ -38,7 +45,7 @@ const testCases = [ screenshotFile: 'settings_notifications', navigation: async (device: DeviceWrapper) => { await device.clickOnElementAll(new UserSettings(device)); - await device.clickOnElementAll(new NotificationSettings(device)); + await device.clickOnElementAll(new NotificationsMenuItem(device)); await sleepFor(1_000); // This one otherwise captures a black screen }, }, diff --git a/run/test/specs/voice_calls.spec.ts b/run/test/specs/voice_calls.spec.ts index 0df4e28b6..300c563b5 100644 --- a/run/test/specs/voice_calls.spec.ts +++ b/run/test/specs/voice_calls.spec.ts @@ -4,7 +4,7 @@ import { englishStrippedStr } from '../../localizer/englishStrippedStr'; import { TestSteps } from '../../types/allure'; import { bothPlatformsItSeparate } from '../../types/sessionIt'; import { CloseSettings } from './locators'; -import { CallButton, NotificationSettings, NotificationSwitch } from './locators/conversation'; +import { CallButton, NotificationsModalButton, NotificationSwitch } from './locators/conversation'; import { open_Alice1_Bob1_friends } from './state_builder'; import { sleepFor } from './utils/index'; import { closeApp, SupportedPlatformsType } from './utils/open_app'; @@ -194,7 +194,7 @@ async function voiceCallAndroid(platform: SupportedPlatformsType, testInfo: Test englishStrippedStr('sessionNotifications').toString(), englishStrippedStr('callsNotificationsRequired').toString() ); - await alice1.clickOnElementAll(new NotificationSettings(alice1)); + await alice1.clickOnElementAll(new NotificationsModalButton(alice1)); await alice1.clickOnElementAll(new NotificationSwitch(alice1)); }); await alice1.navigateBack(false); @@ -241,7 +241,7 @@ async function voiceCallAndroid(platform: SupportedPlatformsType, testInfo: Test await bob1.clickOnElementById( 'com.android.permissioncontroller:id/permission_allow_foreground_only_button' ); - await bob1.clickOnElementAll(new NotificationSettings(bob1)); + await bob1.clickOnElementAll(new NotificationsModalButton(bob1)); await bob1.clickOnElementAll(new NotificationSwitch(bob1)); await bob1.navigateBack(false); await bob1.navigateBack(false); From 78ac0a665b29e37c12c189573235a8b6ecc271d8 Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Thu, 9 Oct 2025 16:30:33 +1100 Subject: [PATCH 12/29] fix: adjust sent indicator android --- run/test/specs/locators/conversation.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/run/test/specs/locators/conversation.ts b/run/test/specs/locators/conversation.ts index 620b96a2b..61666aa65 100644 --- a/run/test/specs/locators/conversation.ts +++ b/run/test/specs/locators/conversation.ts @@ -180,9 +180,10 @@ export class OutgoingMessageStatusSent extends LocatorsInterface { switch (this.platform) { case 'android': return { - strategy: '-android uiautomator', + strategy: 'id', selector: - 'new UiSelector().resourceId("network.loki.messenger.qa:id/messageStatusTextView").text("Sent")', + 'network.loki.messenger.qa:id/messageStatusTextView', + text: 'Sent' } as const; case 'ios': return { From ab7947c4d630218a03d86fe66efb095890e0a762 Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Fri, 10 Oct 2025 11:49:28 +1100 Subject: [PATCH 13/29] fix: 1.28.2 still has the old locators --- run/test/specs/community_emoji_react.spec.ts | 3 +++ run/test/specs/locators/index.ts | 6 +----- run/test/specs/locators/onboarding.ts | 4 ++-- run/types/testing.ts | 12 ++---------- 4 files changed, 8 insertions(+), 17 deletions(-) diff --git a/run/test/specs/community_emoji_react.spec.ts b/run/test/specs/community_emoji_react.spec.ts index 5d3dbca26..d95a22f22 100644 --- a/run/test/specs/community_emoji_react.spec.ts +++ b/run/test/specs/community_emoji_react.spec.ts @@ -18,6 +18,9 @@ bothPlatformsIt({ suite: 'Emoji reacts', }, allureDescription: 'Verifies that an emoji reaction can be sent and is received in a community', + allureLinks: { + android: 'SES-4608' + } }); async function sendEmojiReactionCommunity(platform: SupportedPlatformsType, testInfo: TestInfo) { diff --git a/run/test/specs/locators/index.ts b/run/test/specs/locators/index.ts index 8a61169cc..3c72cde5b 100644 --- a/run/test/specs/locators/index.ts +++ b/run/test/specs/locators/index.ts @@ -1,6 +1,6 @@ import { ANDROID_XPATHS, IOS_XPATHS } from '../../../constants'; import { DeviceWrapper } from '../../../types/DeviceWrapper'; -import { ElementStates, StrategyExtractionObj } from '../../../types/testing'; +import { StrategyExtractionObj } from '../../../types/testing'; import { getAppDisplayName } from '../utils/devnet'; import { SupportedPlatformsType } from '../utils/open_app'; @@ -44,10 +44,6 @@ export function describeLocator(locator: StrategyExtractionObj & { text?: string return text ? `${base} and text "${text}"` : base; } -// Returns the expected screenshot path for a locator, optionally varying by state -export abstract class LocatorsInterfaceScreenshot extends LocatorsInterface { - abstract screenshotFileName(state?: ElementStates): string; -} export class ApplyChanges extends LocatorsInterface { public build() { diff --git a/run/test/specs/locators/onboarding.ts b/run/test/specs/locators/onboarding.ts index 3540363c7..d54798c5b 100644 --- a/run/test/specs/locators/onboarding.ts +++ b/run/test/specs/locators/onboarding.ts @@ -105,7 +105,7 @@ export class TermsOfServiceButton extends LocatorsInterface { case 'android': return { strategy: 'id', - selector: 'Terms of service button', + selector: 'Terms of Service', // will be Privacy policy button with Pro Settings } as const; case 'ios': return { @@ -122,7 +122,7 @@ export class PrivacyPolicyButton extends LocatorsInterface { case 'android': return { strategy: 'id', - selector: 'Privacy policy button', + selector: 'Privacy Policy', // will be Privacy policy button with Pro Settings } as const; case 'ios': return { diff --git a/run/types/testing.ts b/run/types/testing.ts index 2a755670c..488039a79 100644 --- a/run/types/testing.ts +++ b/run/types/testing.ts @@ -544,7 +544,7 @@ export type Id = | 'Open URL' | 'preferred-display-name' | 'Privacy' - | 'Privacy policy button' + | 'Privacy Policy' | 'pro-badge-text' | 'Quit' | 'rate-app-button' @@ -565,7 +565,7 @@ export type Id = | 'show-nts-confirm-button' | 'Show' | 'Slow mode notifications button' - | 'Terms of service button' + | 'Terms of Service' | 'update-group-info-confirm-button' | 'update-group-info-description-input' | 'update-group-info-name-input' @@ -578,12 +578,4 @@ export type Id = export type TestRisk = 'high' | 'low' | 'medium'; -export type ElementStates = - | 'incoming_reply_message' - | 'incoming_short_message' - | 'new_account' - | 'outgoing_reply_message' - | 'outgoing_short_message' - | 'restore_account'; - export type AppName = 'Session AQA' | 'Session QA'; From 8a10bb3df9b0fa06dd4c179c9f4ce5e6c810944a Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Fri, 10 Oct 2025 16:01:58 +1100 Subject: [PATCH 14/29] fix: refine screenshot threshold handling --- run/screenshots/ios/landingpage_new_account.png | 4 ++-- run/test/specs/app_disguise_icons.spec.ts | 2 +- run/test/specs/community_emoji_react.spec.ts | 4 ++-- run/test/specs/landing_page_new_account.spec.ts | 2 +- .../specs/landing_page_restore_account.spec.ts | 2 +- run/test/specs/locators/conversation.ts | 5 ++--- run/test/specs/locators/index.ts | 1 - run/test/specs/utils/verify_screenshots.ts | 16 +++++++++++----- run/test/specs/visual_settings.spec.ts | 2 +- run/types/testing.ts | 12 ++++++++++++ 10 files changed, 33 insertions(+), 17 deletions(-) diff --git a/run/screenshots/ios/landingpage_new_account.png b/run/screenshots/ios/landingpage_new_account.png index 0fc079315..06d6e2cf7 100644 --- a/run/screenshots/ios/landingpage_new_account.png +++ b/run/screenshots/ios/landingpage_new_account.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:643b7aaf13a7f4dd456a5f9331948acfc02ced68a2492de20d10a54a0054c8db -size 190440 +oid sha256:8b86a2977d43b32cdc619aecd01d9deb4cdc069a9540aed82ffb8fd3b1c741cd +size 197547 diff --git a/run/test/specs/app_disguise_icons.spec.ts b/run/test/specs/app_disguise_icons.spec.ts index c2d0807b0..d64fc2996 100644 --- a/run/test/specs/app_disguise_icons.spec.ts +++ b/run/test/specs/app_disguise_icons.spec.ts @@ -32,7 +32,7 @@ async function appDisguiseIcons(platform: SupportedPlatformsType, testInfo: Test }); await test.step(TestSteps.VERIFY.SCREENSHOT('app disguise icons'), async () => { await device.clickOnElementAll(new SelectAppIcon(device)); - await verifyPageScreenshot(device, platform, 'app_disguise', testInfo); + await verifyPageScreenshot(device, platform, 'app_disguise', testInfo, 0.99); }); await test.step(TestSteps.SETUP.CLOSE_APP, async () => { await closeApp(device); diff --git a/run/test/specs/community_emoji_react.spec.ts b/run/test/specs/community_emoji_react.spec.ts index d95a22f22..e3619ecd9 100644 --- a/run/test/specs/community_emoji_react.spec.ts +++ b/run/test/specs/community_emoji_react.spec.ts @@ -19,8 +19,8 @@ bothPlatformsIt({ }, allureDescription: 'Verifies that an emoji reaction can be sent and is received in a community', allureLinks: { - android: 'SES-4608' - } + android: 'SES-4608', + }, }); async function sendEmojiReactionCommunity(platform: SupportedPlatformsType, testInfo: TestInfo) { diff --git a/run/test/specs/landing_page_new_account.spec.ts b/run/test/specs/landing_page_new_account.spec.ts index bcd4c7d2c..fe1389ecf 100644 --- a/run/test/specs/landing_page_new_account.spec.ts +++ b/run/test/specs/landing_page_new_account.spec.ts @@ -22,6 +22,6 @@ async function landingPageNewAccount(platform: SupportedPlatformsType, testInfo: const { device } = await openAppOnPlatformSingleDevice(platform, testInfo); await newUser(device, USERNAME.ALICE); // Verify that the party popper is shown on the landing page - await verifyPageScreenshot(device, platform, 'landingpage_new_account', testInfo); + await verifyPageScreenshot(device, platform, 'landingpage_new_account', testInfo, 0.995); await closeApp(device); } diff --git a/run/test/specs/landing_page_restore_account.spec.ts b/run/test/specs/landing_page_restore_account.spec.ts index 9b92d7c7d..c030c12d8 100644 --- a/run/test/specs/landing_page_restore_account.spec.ts +++ b/run/test/specs/landing_page_restore_account.spec.ts @@ -24,6 +24,6 @@ async function landingPageRestoreAccount(platform: SupportedPlatformsType, testI const { device1: alice1, device2: alice2 } = await openAppTwoDevices(platform, testInfo); await linkedDevice(alice1, alice2, USERNAME.ALICE); // Verify that the Session logo is shown on the landing page - await verifyPageScreenshot(alice2, platform, 'landingpage_restore_account', testInfo); + await verifyPageScreenshot(alice2, platform, 'landingpage_restore_account', testInfo, 0.995); await closeApp(alice1, alice2); } diff --git a/run/test/specs/locators/conversation.ts b/run/test/specs/locators/conversation.ts index 61666aa65..862d47ec3 100644 --- a/run/test/specs/locators/conversation.ts +++ b/run/test/specs/locators/conversation.ts @@ -181,9 +181,8 @@ export class OutgoingMessageStatusSent extends LocatorsInterface { case 'android': return { strategy: 'id', - selector: - 'network.loki.messenger.qa:id/messageStatusTextView', - text: 'Sent' + selector: 'network.loki.messenger.qa:id/messageStatusTextView', + text: 'Sent', } as const; case 'ios': return { diff --git a/run/test/specs/locators/index.ts b/run/test/specs/locators/index.ts index 3c72cde5b..955f7c001 100644 --- a/run/test/specs/locators/index.ts +++ b/run/test/specs/locators/index.ts @@ -44,7 +44,6 @@ export function describeLocator(locator: StrategyExtractionObj & { text?: string return text ? `${base} and text "${text}"` : base; } - export class ApplyChanges extends LocatorsInterface { public build() { switch (this.platform) { diff --git a/run/test/specs/utils/verify_screenshots.ts b/run/test/specs/utils/verify_screenshots.ts index be15aa382..ecdcd5dc6 100644 --- a/run/test/specs/utils/verify_screenshots.ts +++ b/run/test/specs/utils/verify_screenshots.ts @@ -7,6 +7,7 @@ import { ssim } from 'ssim.js'; import { v4 as uuidv4 } from 'uuid'; import { DeviceWrapper } from '../../../types/DeviceWrapper'; +import { ScreenshotFileNames } from '../../../types/testing'; import { SupportedPlatformsType } from './open_app'; import { getDiffDirectory } from './utilities'; import { clearStatusBarOverrides, setConsistentStatusBar } from './utilities'; @@ -70,9 +71,9 @@ async function fileToImageData(filePath: string): Promise { async function compareWithSSIM( actualBuffer: Buffer, baselineImagePath: string, - testInfo: TestInfo + testInfo: TestInfo, + threshold: number ): Promise { - const threshold = 0.97; // Strict matching since this doesn't rely on pixelmatching anymore const actualImageData = await bufferToImageData(actualBuffer); const baselineImageData = await fileToImageData(baselineImagePath); @@ -179,9 +180,14 @@ function ensureBaseline(actualBuffer: Buffer, baselinePath: string): void { export async function verifyPageScreenshot( device: DeviceWrapper, platform: SupportedPlatformsType, - screenshotName: string, - testInfo: TestInfo + screenshotName: ScreenshotFileNames, + testInfo: TestInfo, + threshold: number = 0.97 // Strict tolerance by default ): Promise { + // Validate threshold range + if (threshold < 0 || threshold > 1) { + throw new Error(`SSIM threshold must be between 0 and 1, got: ${threshold}`); + } await setConsistentStatusBar(device); try { // Get full page screenshot and crop it @@ -198,7 +204,7 @@ export async function verifyPageScreenshot( ensureBaseline(screenshotBuffer, baselineScreenshotPath); // Perform SSIM comparison - await compareWithSSIM(screenshotBuffer, baselineScreenshotPath, testInfo); + await compareWithSSIM(screenshotBuffer, baselineScreenshotPath, testInfo, threshold); } finally { await clearStatusBarOverrides(device); } diff --git a/run/test/specs/visual_settings.spec.ts b/run/test/specs/visual_settings.spec.ts index 6bbee5e5a..fe72a8caf 100644 --- a/run/test/specs/visual_settings.spec.ts +++ b/run/test/specs/visual_settings.spec.ts @@ -57,7 +57,7 @@ const testCases = [ await device.clickOnElementAll(new AppearanceMenuItem(device)); }, }, -]; +] as const; for (const { screenName, screenshotFile, navigation } of testCases) { bothPlatformsIt({ diff --git a/run/types/testing.ts b/run/types/testing.ts index 488039a79..a97aee4c9 100644 --- a/run/types/testing.ts +++ b/run/types/testing.ts @@ -579,3 +579,15 @@ export type Id = export type TestRisk = 'high' | 'low' | 'medium'; export type AppName = 'Session AQA' | 'Session QA'; + +export type ScreenshotFileNames = + | 'app_disguise' + | 'conversation_alice' + | 'conversation_bob' + | 'landingpage_new_account' + | 'landingpage_restore_account' + | 'settings_appearance' + | 'settings_conversations' + | 'settings_notifications' + | 'settings_privacy' + | 'settings'; From e960d51338f6f2629dd71ec75bd29ea7dc27731b Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Wed, 15 Oct 2025 17:10:46 +1100 Subject: [PATCH 15/29] feat: UPM home screen test --- run/screenshots/android/upm_home.png | 3 ++ run/test/specs/upm_homescreen.spec.ts | 60 +++++++++++++++++++++++++++ run/test/specs/utils/devnet.ts | 8 ++++ run/types/testing.ts | 4 +- 4 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 run/screenshots/android/upm_home.png create mode 100644 run/test/specs/upm_homescreen.spec.ts diff --git a/run/screenshots/android/upm_home.png b/run/screenshots/android/upm_home.png new file mode 100644 index 000000000..f5b2a0fa7 --- /dev/null +++ b/run/screenshots/android/upm_home.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4059b3b2f68ba36ebd6c86d33a720ece3f6bdf4ba069bf6ed6e28d1a5e033a9 +size 117288 diff --git a/run/test/specs/upm_homescreen.spec.ts b/run/test/specs/upm_homescreen.spec.ts new file mode 100644 index 000000000..4025aebd8 --- /dev/null +++ b/run/test/specs/upm_homescreen.spec.ts @@ -0,0 +1,60 @@ +import { test, type TestInfo } from '@playwright/test'; + +import { TestSteps } from '../../types/allure'; +import { androidIt } from '../../types/sessionIt'; +import { open_Alice1_Bob1_friends } from './state_builder'; +import { closeApp, SupportedPlatformsType } from './utils/open_app'; +import { verifyPageScreenshot } from './utils/verify_screenshots'; + +androidIt({ + title: 'User Profile Modal Home Screen', + risk: 'high', + testCb: upmHomeScreen, + countOfDevicesNeeded: 2, +}); + +async function upmHomeScreen(platform: SupportedPlatformsType, testInfo: TestInfo) { + const { + devices: { alice1, bob1 }, + prebuilt: { bob }, + } = await test.step(TestSteps.SETUP.QA_SEEDER, async () => { + return open_Alice1_Bob1_friends({ + platform, + focusFriendsConvo: false, + testInfo, + }); + }); + await test.step('Open User Profile Modal on home screen', async () => { + await alice1.longPressConversation(bob.userName); + await alice1.clickOnElementAll({ + strategy: 'accessibility id', + selector: 'Details', + }); + }); + await test.step(TestSteps.VERIFY.SCREENSHOT('user profile modal'), async () => { + await verifyPageScreenshot(alice1, platform, 'upm_home', testInfo); + }); + await test.step(`Verify ${bob.userName} display name in user profile modal`, async () => { + await alice1.waitForTextElementToBePresent({ + strategy: 'id', + selector: 'pro-badge-text', + text: bob.userName, + }); + }); + await test.step(`Verify ${bob.userName} account id in user profile modal`, async () => { + const el = await alice1.waitForTextElementToBePresent({ + strategy: 'id', + selector: 'account-id', + }); + const eltext = await alice1.getTextFromElement(el); + const normalized = eltext.replace(/\s+/g, ''); // account id comes in two lines + if (eltext !== bob.sessionId) { + throw new Error(`Account ID does not match. + Expected: ${bob.sessionId} + Observed: ${normalized}`); + } + }); + await test.step(TestSteps.SETUP.CLOSE_APP, async () => { + await closeApp(alice1, bob1); + }); +} diff --git a/run/test/specs/utils/devnet.ts b/run/test/specs/utils/devnet.ts index 7908a5886..150d85ed6 100644 --- a/run/test/specs/utils/devnet.ts +++ b/run/test/specs/utils/devnet.ts @@ -64,6 +64,14 @@ export function getNetworkTarget(platform: SupportedPlatformsType): NetworkType const apkPath = getAndroidApk(); const isAQA = isAutomaticQABuildAndroid(apkPath); + + // Early exit for non AQA builds - no need to check devnet + if (!isAQA) { + process.env.DETECTED_NETWORK_TARGET = 'mainnet'; + console.log('Network target: mainnet'); + return 'mainnet'; + } + const canAccessDevnet = canReachDevnet(); // If you pass an AQA build in the .env but can't access devnet, tests will fail if (isAQA && !canAccessDevnet) { diff --git a/run/types/testing.ts b/run/types/testing.ts index a97aee4c9..0acebc899 100644 --- a/run/types/testing.ts +++ b/run/types/testing.ts @@ -420,6 +420,7 @@ export type AccessibilityId = export type Id = | DISAPPEARING_TIMES + | 'account-id' | 'Account ID' | 'android:id/aerr_close' | 'android:id/aerr_wait' @@ -590,4 +591,5 @@ export type ScreenshotFileNames = | 'settings_conversations' | 'settings_notifications' | 'settings_privacy' - | 'settings'; + | 'settings' + | 'upm_home'; From 6745af8f3f53080ff24efb6e9a4f24e975d503ee Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Thu, 16 Oct 2025 17:00:52 +1100 Subject: [PATCH 16/29] chore: add more allure suites and descriptions --- run/test/specs/community_tests_image.spec.ts | 2 +- run/test/specs/group_message_delete.spec.ts | 8 +++++++- run/test/specs/group_message_document.spec.ts | 6 ++++++ run/test/specs/group_message_gif.spec.ts | 6 ++++++ run/test/specs/group_message_image.spec.ts | 6 ++++++ run/test/specs/group_message_link_preview.spec.ts | 6 ++++++ run/test/specs/group_message_long_text.spec.ts | 4 ++++ run/test/specs/group_message_unsend.spec.ts | 8 +++++++- run/test/specs/group_message_video.spec.ts | 6 ++++++ run/test/specs/group_message_voice.spec.ts | 6 ++++++ run/test/specs/group_tests_add_contact.spec.ts | 5 +++++ .../specs/group_tests_change_group_description.spec.ts | 2 ++ run/test/specs/group_tests_change_group_name.spec.ts | 5 +++++ run/types/allure.ts | 2 +- 14 files changed, 68 insertions(+), 4 deletions(-) diff --git a/run/test/specs/community_tests_image.spec.ts b/run/test/specs/community_tests_image.spec.ts index 6aca09d0a..d6e83c3a4 100644 --- a/run/test/specs/community_tests_image.spec.ts +++ b/run/test/specs/community_tests_image.spec.ts @@ -14,7 +14,7 @@ bothPlatformsIt({ risk: 'medium', countOfDevicesNeeded: 2, testCb: sendImageCommunity, - allureSuites: { parent: 'Sending Messages', suite: 'Attachments' }, + allureSuites: { parent: 'Sending Messages', suite: 'Message types' }, allureDescription: 'Verifies that an image can be sent and received in a community', }); diff --git a/run/test/specs/group_message_delete.spec.ts b/run/test/specs/group_message_delete.spec.ts index 9d80bfd3a..a11dac5ec 100644 --- a/run/test/specs/group_message_delete.spec.ts +++ b/run/test/specs/group_message_delete.spec.ts @@ -8,10 +8,16 @@ import { open_Alice1_Bob1_Charlie1_friends_group } from './state_builder'; import { closeApp, SupportedPlatformsType } from './utils/open_app'; bothPlatformsIt({ - title: 'Delete message in group', + title: 'Delete message locally in group', risk: 'high', countOfDevicesNeeded: 3, testCb: deleteMessageGroup, + allureSuites: { + parent: 'User Actions', + suite: 'Delete Message', + }, + allureDescription: + 'Verifies that local deletion in a group does not delete a message for other participants.', }); async function deleteMessageGroup(platform: SupportedPlatformsType, testInfo: TestInfo) { diff --git a/run/test/specs/group_message_document.spec.ts b/run/test/specs/group_message_document.spec.ts index edd6378b2..1d5bbd552 100644 --- a/run/test/specs/group_message_document.spec.ts +++ b/run/test/specs/group_message_document.spec.ts @@ -16,6 +16,12 @@ bothPlatformsItSeparate({ android: { testCb: sendDocumentGroupAndroid, }, + allureSuites: { + parent: 'Sending Messages', + suite: 'Message types', + }, + allureDescription: + 'Verifies that a PDF can be sent to a group, all members receive the document, and replying to a document works as expected', }); async function sendDocumentGroupiOS(platform: SupportedPlatformsType, testInfo: TestInfo) { diff --git a/run/test/specs/group_message_gif.spec.ts b/run/test/specs/group_message_gif.spec.ts index ae3a5f5b6..d1ec33d2d 100644 --- a/run/test/specs/group_message_gif.spec.ts +++ b/run/test/specs/group_message_gif.spec.ts @@ -16,6 +16,12 @@ bothPlatformsItSeparate({ android: { testCb: sendGifGroupAndroid, }, + allureSuites: { + parent: 'Sending Messages', + suite: 'Message types', + }, + allureDescription: + 'Verifies that a GIF can be sent to a group, all members receive the document, and replying to a document works as expected', }); async function sendGifGroupiOS(platform: SupportedPlatformsType, testInfo: TestInfo) { diff --git a/run/test/specs/group_message_image.spec.ts b/run/test/specs/group_message_image.spec.ts index 61fa50522..63dc2c2f8 100644 --- a/run/test/specs/group_message_image.spec.ts +++ b/run/test/specs/group_message_image.spec.ts @@ -16,6 +16,12 @@ bothPlatformsItSeparate({ android: { testCb: sendImageGroupAndroid, }, + allureSuites: { + parent: 'Sending Messages', + suite: 'Message types', + }, + allureDescription: + 'Verifies that an image can be sent to a group, all members receive the document, and replying to a document works as expected', }); async function sendImageGroupiOS(platform: SupportedPlatformsType, testInfo: TestInfo) { diff --git a/run/test/specs/group_message_link_preview.spec.ts b/run/test/specs/group_message_link_preview.spec.ts index f3b73a5e1..07b3f0a32 100644 --- a/run/test/specs/group_message_link_preview.spec.ts +++ b/run/test/specs/group_message_link_preview.spec.ts @@ -24,6 +24,12 @@ bothPlatformsItSeparate({ android: { testCb: sendLinkGroupAndroid, }, + allureSuites: { + parent: 'Sending Messages', + suite: 'Message types', + }, + allureDescription: + 'Verifies that a link with preview can be sent to a group, all members receive the document, and replying to a document works as expected', }); async function sendLinkGroupiOS(platform: SupportedPlatformsType, testInfo: TestInfo) { diff --git a/run/test/specs/group_message_long_text.spec.ts b/run/test/specs/group_message_long_text.spec.ts index 9122cb964..3f4b103da 100644 --- a/run/test/specs/group_message_long_text.spec.ts +++ b/run/test/specs/group_message_long_text.spec.ts @@ -11,6 +11,10 @@ bothPlatformsIt({ risk: 'low', countOfDevicesNeeded: 3, testCb: sendLongMessageGroup, + allureSuites: { + parent: 'Sending Messages', + suite: 'Message types', + }, allureDescription: 'Verifies that a long message can be sent to a group', }); diff --git a/run/test/specs/group_message_unsend.spec.ts b/run/test/specs/group_message_unsend.spec.ts index 3ecd759fa..6d4176067 100644 --- a/run/test/specs/group_message_unsend.spec.ts +++ b/run/test/specs/group_message_unsend.spec.ts @@ -8,10 +8,16 @@ import { open_Alice1_Bob1_Charlie1_friends_group } from './state_builder'; import { closeApp, SupportedPlatformsType } from './utils/open_app'; bothPlatformsIt({ - title: 'Unsend message in group', + title: 'Delete message for all in group', risk: 'high', countOfDevicesNeeded: 3, testCb: unsendMessageGroup, + allureSuites: { + parent: 'User Actions', + suite: 'Delete Message', + }, + allureDescription: + 'Verifies that global deletion in a group deletes a message for every participant.', }); async function unsendMessageGroup(platform: SupportedPlatformsType, testInfo: TestInfo) { diff --git a/run/test/specs/group_message_video.spec.ts b/run/test/specs/group_message_video.spec.ts index c3d7af6e8..e97d37dbf 100644 --- a/run/test/specs/group_message_video.spec.ts +++ b/run/test/specs/group_message_video.spec.ts @@ -15,6 +15,12 @@ bothPlatformsItSeparate({ android: { testCb: sendVideoGroupAndroid, }, + allureSuites: { + parent: 'Sending Messages', + suite: 'Message types', + }, + allureDescription: + 'Verifies that a video can be sent to a group, all members receive the document, and replying to a document works as expected', }); async function sendVideoGroupiOS(platform: SupportedPlatformsType, testInfo: TestInfo) { diff --git a/run/test/specs/group_message_voice.spec.ts b/run/test/specs/group_message_voice.spec.ts index 2efd9bf0a..32115f500 100644 --- a/run/test/specs/group_message_voice.spec.ts +++ b/run/test/specs/group_message_voice.spec.ts @@ -10,6 +10,12 @@ bothPlatformsIt({ risk: 'high', countOfDevicesNeeded: 3, testCb: sendVoiceMessageGroup, + allureSuites: { + parent: 'Sending Messages', + suite: 'Message types', + }, + allureDescription: + 'Verifies that a voice message can be sent to a group, all members receive the document, and replying to a document works as expected', }); async function sendVoiceMessageGroup(platform: SupportedPlatformsType, testInfo: TestInfo) { diff --git a/run/test/specs/group_tests_add_contact.spec.ts b/run/test/specs/group_tests_add_contact.spec.ts index 1981a6331..ce4cecae8 100644 --- a/run/test/specs/group_tests_add_contact.spec.ts +++ b/run/test/specs/group_tests_add_contact.spec.ts @@ -19,6 +19,11 @@ bothPlatformsIt({ risk: 'high', testCb: addContactToGroup, countOfDevicesNeeded: 4, + allureSuites: { + parent: 'Groups', + suite: 'Edit Group', + }, + allureDescription: 'Create four accounts, create a group with three, add the fourth member', }); async function addContactToGroup(platform: SupportedPlatformsType, testInfo: TestInfo) { const testGroupName = 'Group to test adding contact'; diff --git a/run/test/specs/group_tests_change_group_description.spec.ts b/run/test/specs/group_tests_change_group_description.spec.ts index 6e76b38ed..2678ef778 100644 --- a/run/test/specs/group_tests_change_group_description.spec.ts +++ b/run/test/specs/group_tests_change_group_description.spec.ts @@ -32,6 +32,8 @@ bothPlatformsItSeparate({ parent: 'Groups', suite: 'Edit Group', }, + allureDescription: + 'Verifies that a group description can be at most 200 chars and that every member can see a valid change.', }); // Setup diff --git a/run/test/specs/group_tests_change_group_name.spec.ts b/run/test/specs/group_tests_change_group_name.spec.ts index d505178c0..4e50f94e2 100644 --- a/run/test/specs/group_tests_change_group_name.spec.ts +++ b/run/test/specs/group_tests_change_group_name.spec.ts @@ -22,6 +22,11 @@ bothPlatformsItSeparate({ android: { testCb: changeGroupNameAndroid, }, + allureSuites: { + parent: 'Groups', + suite: 'Edit Group', + }, + allureDescription: 'Verifies that a group name change syncs to every member.', }); async function changeGroupNameIos(platform: SupportedPlatformsType, testInfo: TestInfo) { diff --git a/run/types/allure.ts b/run/types/allure.ts index 7ff5eedbf..5c09d0d3d 100644 --- a/run/types/allure.ts +++ b/run/types/allure.ts @@ -25,7 +25,7 @@ export type AllureSuiteConfig = | { parent: 'In-App Review Prompt'; suite: 'Flows' | 'Triggers' } | { parent: 'Linkouts' } | { parent: 'New Conversation'; suite: 'Join Community' | 'New Message' } - | { parent: 'Sending Messages'; suite: 'Attachments' | 'Emoji reacts' | 'Rules' } + | { parent: 'Sending Messages'; suite: 'Emoji reacts' | 'Message types' | 'Rules' } | { parent: 'Settings'; suite: 'App Disguise' } | { parent: 'User Actions'; From a7597c4785a7514fde7e51094beb18244f093899 Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Fri, 17 Oct 2025 15:23:24 +1100 Subject: [PATCH 17/29] feat: add blinded message request test --- run/test/specs/community_requests_off.spec.ts | 75 +++++++++++ run/test/specs/community_requests_on.spec.ts | 118 ++++++++++++++++++ run/test/specs/utils/capabilities_ios.ts | 2 +- run/types/testing.ts | 6 + 4 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 run/test/specs/community_requests_off.spec.ts create mode 100644 run/test/specs/community_requests_on.spec.ts diff --git a/run/test/specs/community_requests_off.spec.ts b/run/test/specs/community_requests_off.spec.ts new file mode 100644 index 000000000..a7743300e --- /dev/null +++ b/run/test/specs/community_requests_off.spec.ts @@ -0,0 +1,75 @@ +import { expect, TestInfo } from '@playwright/test'; +import { USERNAME } from '@session-foundation/qa-seeder'; + +import { testCommunityLink, testCommunityName } from '../../constants/community'; +import { bothPlatformsIt } from '../../types/sessionIt'; +import { newUser } from './utils/create_account'; +import { joinCommunity } from './utils/join_community'; +import { closeApp, openAppTwoDevices, SupportedPlatformsType } from './utils/open_app'; + +bothPlatformsIt({ + title: 'Blinded message request off', + risk: 'medium', + testCb: blindedMessageRequests, + countOfDevicesNeeded: 2, + allureDescription: + 'Verifies that a message request cannot be sent when Community Message Requests are off.', +}); + +// TODO: tidy this up with neat locators +async function blindedMessageRequests(platform: SupportedPlatformsType, testInfo: TestInfo) { + const { device1, device2 } = await openAppTwoDevices(platform, testInfo); + await Promise.all([ + newUser(device1, USERNAME.ALICE, { saveUserData: false }), + newUser(device2, USERNAME.BOB, { saveUserData: false }), + ]); + await Promise.all( + [device1, device2].map(async device => { + await joinCommunity(device, testCommunityLink, testCommunityName); + }) + ); + const message = `I do not accept blinded message requests + ${platform} + ${Date.now()}`; + await device2.sendMessage(message); + // Click on profile picture (Android) or sender name (iOS) + await device1 + .onAndroid() + .clickOnElementXPath( + `//android.view.ViewGroup[@resource-id='network.loki.messenger.qa:id/mainContainer'][.//android.widget.TextView[contains(@text,'${message}')]]//androidx.compose.ui.platform.ComposeView[@resource-id='network.loki.messenger.qa:id/profilePictureView']` + ); + await device1 + .onIOS() + .clickOnElementXPath( + `//XCUIElementTypeCell[.//XCUIElementTypeOther[@name='Message body' and contains(@label,'${message}')]]//XCUIElementTypeStaticText[contains(@value,'(15')]` + ); + + let attr; + + if (platform === 'android') { + const el = await device1.waitForTextElementToBePresent({ + strategy: 'id', + selector: 'account-id', + }); + const elText = await device1.getTextFromElement(el); + expect(elText).toMatch(/^15/); + const messageButton = await device1.waitForTextElementToBePresent({ + strategy: 'xpath', + selector: `//android.widget.TextView[@text="Message"]/parent::android.view.View`, + }); + attr = await device1.getAttribute('enabled', messageButton.ELEMENT); + } else { + await device1.waitForTextElementToBePresent({ + strategy: 'accessibility id', + selector: 'Blinded ID', + }); + const messageButton = await device1.waitForTextElementToBePresent({ + strategy: 'accessibility id', + selector: 'Message', + }); + attr = await device1.getAttribute('enabled', messageButton.ELEMENT); + } + if (attr !== 'false') { + device1.log(`Message button attribute is 'enabled = ${attr}'`); + throw new Error(`Message button should be disabled but it is not`); + } + await closeApp(device1, device2); +} diff --git a/run/test/specs/community_requests_on.spec.ts b/run/test/specs/community_requests_on.spec.ts new file mode 100644 index 000000000..6a3e9245d --- /dev/null +++ b/run/test/specs/community_requests_on.spec.ts @@ -0,0 +1,118 @@ +import { expect, TestInfo } from '@playwright/test'; +import { USERNAME } from '@session-foundation/qa-seeder'; + +import { testCommunityLink, testCommunityName } from '../../constants/community'; +import { englishStrippedStr } from '../../localizer/englishStrippedStr'; +import { bothPlatformsIt } from '../../types/sessionIt'; +import { CloseSettings } from './locators'; +import { ConversationHeaderName, MessageBody } from './locators/conversation'; +import { MessageRequestsBanner } from './locators/home'; +import { PrivacyMenuItem, UserSettings } from './locators/settings'; +import { newUser } from './utils/create_account'; +import { joinCommunity } from './utils/join_community'; +import { closeApp, openAppTwoDevices, SupportedPlatformsType } from './utils/open_app'; + +bothPlatformsIt({ + title: 'Blinded message request', + risk: 'medium', + testCb: blindedMessageRequests, + countOfDevicesNeeded: 2, + allureDescription: + 'Verifies that a message request can be sent when Community Message Requests are on.', + allureLinks: { + ios: 'SES-4722', + }, +}); + +// TODO: tidy this up with neat locators +async function blindedMessageRequests(platform: SupportedPlatformsType, testInfo: TestInfo) { + const { device1, device2 } = await openAppTwoDevices(platform, testInfo); + const [alice, bob] = await Promise.all([ + newUser(device1, USERNAME.ALICE, { saveUserData: false }), + newUser(device2, USERNAME.BOB, { saveUserData: false }), + ]); + await device2.clickOnElementAll(new UserSettings(device2)); + await device2.clickOnElementAll(new PrivacyMenuItem(device2)); + await device2.onAndroid().clickOnElementAll({ + strategy: '-android uiautomator', + selector: 'new UiSelector().text("Community Message Requests")', + }); + await device2.onIOS().clickOnElementAll({ + strategy: 'accessibility id', + selector: 'Community Message Requests', + }); + await device2.navigateBack(); + await device2.clickOnElementAll(new CloseSettings(device2)); + await Promise.all( + [device1, device2].map(async device => { + await joinCommunity(device, testCommunityLink, testCommunityName); + }) + ); + const message = `I accept blinded message requests + ${platform} + ${Date.now()}`; + await device2.sendMessage(message); + await device2.navigateBack(); + // Click on profile picture (Android) or sender name (iOS) + await device1 + .onAndroid() + .clickOnElementXPath( + `//android.view.ViewGroup[@resource-id='network.loki.messenger.qa:id/mainContainer'][.//android.widget.TextView[contains(@text,'${message}')]]//androidx.compose.ui.platform.ComposeView[@resource-id='network.loki.messenger.qa:id/profilePictureView']` + ); + await device1 + .onIOS() + .clickOnElementXPath( + `//XCUIElementTypeCell[.//XCUIElementTypeOther[@name='Message body' and contains(@label,'${message}')]]//XCUIElementTypeStaticText[contains(@value,'(15')]` + ); + if (platform === 'android') { + const el = await device1.waitForTextElementToBePresent({ + strategy: 'id', + selector: 'account-id', + }); + const elText = await device1.getTextFromElement(el); + expect(elText).toMatch(/^15/); + await device1.clickOnElementAll({ + strategy: '-android uiautomator', + selector: 'new UiSelector().text("Message")', + }); + } else { + await device1.waitForTextElementToBePresent({ + strategy: 'accessibility id', + selector: 'Blinded ID', + }); + await device1.clickOnByAccessibilityID('Message'); + } + + await device1.clickOnElementAll(new ConversationHeaderName(device1, bob.userName)); + const messageRequestPendingDescription = englishStrippedStr( + 'messageRequestPendingDescription' + ).toString(); + await device1.onIOS().waitForTextElementToBePresent({ + strategy: 'accessibility id', + selector: 'Control message', + text: messageRequestPendingDescription, + }); + await device1.onAndroid().waitForTextElementToBePresent({ + strategy: 'id', + selector: 'network.loki.messenger.qa:id/textSendAfterApproval', + text: messageRequestPendingDescription, + }); + await device1.sendMessage('Howdy partner'); + await device2.clickOnElementAll(new MessageRequestsBanner(device2)); + // Bob clicks on request conversation item + await device2.clickOnByAccessibilityID('Message request'); + await device2.waitForTextElementToBePresent(new ConversationHeaderName(device2, alice.userName)); + const messageRequestsAcceptDescription = englishStrippedStr( + 'messageRequestsAcceptDescription' + ).toString(); + await device2.onIOS().waitForControlMessageToBePresent(messageRequestsAcceptDescription); + await device2.onAndroid().waitForTextElementToBePresent({ + strategy: 'id', + selector: 'network.loki.messenger.qa:id/sendAcceptsTextView', + text: messageRequestsAcceptDescription, + }); + + // Send message from Bob to Alice + const acceptMessage = 'Howdy back'; + await device2.sendMessage(acceptMessage); + await device1.waitForTextElementToBePresent(new MessageBody(device1, acceptMessage)); + await closeApp(device1, device2); +} diff --git a/run/test/specs/utils/capabilities_ios.ts b/run/test/specs/utils/capabilities_ios.ts index 713457a2a..e355be25b 100644 --- a/run/test/specs/utils/capabilities_ios.ts +++ b/run/test/specs/utils/capabilities_ios.ts @@ -31,7 +31,7 @@ const sharediOSCapabilities: AppiumXCUITestCapabilities = { 'appium:processArguments': { env: { debugDisappearingMessageDurations: 'true', - communityPollLimit: 5, + communityPollLimit: 3, }, }, } as AppiumXCUITestCapabilities; diff --git a/run/types/testing.ts b/run/types/testing.ts index 0acebc899..238e20e29 100644 --- a/run/types/testing.ts +++ b/run/types/testing.ts @@ -134,16 +134,19 @@ export type XPath = | `//*[./*[@name='${DISAPPEARING_TIMES}']]/*[2]` | `//*[@resource-id='network.loki.messenger.qa:id/callTitle' and contains(@text, ':')]` | `//*[starts-with(@content-desc, "Photo taken on")]` + | `//android.view.ViewGroup[@resource-id='network.loki.messenger.qa:id/mainContainer'][.//android.widget.TextView[contains(@text,'${string}')]]//androidx.compose.ui.platform.ComposeView[@resource-id='network.loki.messenger.qa:id/profilePictureView']` | `//android.view.ViewGroup[@resource-id="network.loki.messenger.qa:id/mainContainer"][.//android.widget.TextView[contains(@text,"${string}")]]//android.view.ViewGroup[@resource-id="network.loki.messenger.qa:id/layout_emoji_container"]` | `//android.view.ViewGroup[@resource-id="network.loki.messenger.qa:id/mainContainer"][.//android.widget.TextView[contains(@text,"${string}")]]//android.widget.TextView[@resource-id="network.loki.messenger.qa:id/reactions_pill_count"][@text="${string}"]` | `//android.widget.LinearLayout[.//android.widget.TextView[@content-desc="Conversation list item" and @text="${string}"]]//android.widget.TextView[@resource-id="network.loki.messenger.qa:id/snippetTextView" and @text="${string}"]` | `//android.widget.TextView[@text="${string}"]` + | `//android.widget.TextView[@text="Message"]/parent::android.view.View` | `//XCUIElementTypeAlert//*//XCUIElementTypeButton` | `//XCUIElementTypeButton[@name="Continue"]` | `//XCUIElementTypeButton[@name="Okay"]` | `//XCUIElementTypeButton[@name="Settings"]` | `//XCUIElementTypeCell[.//XCUIElementTypeOther[@label="${string}"]]//XCUIElementTypeStaticText[@value="😂"]` | `//XCUIElementTypeCell[.//XCUIElementTypeOther[@label="${string}"]]//XCUIElementTypeStaticText[@value="${string}"]` + | `//XCUIElementTypeCell[.//XCUIElementTypeOther[@name='Message body' and contains(@label,'${string}')]]//XCUIElementTypeStaticText[contains(@value,'(15')]` | `//XCUIElementTypeCell[@name="${string}"]` | `//XCUIElementTypeCell[@name="Conversation list item" and @label="${string}"]//XCUIElementTypeStaticText[@name="${string}"]` | `//XCUIElementTypeCell[@name="Session"]` @@ -197,6 +200,7 @@ export type AccessibilityId = | 'Awaiting Recipient Answer... 4/6' | 'back' | 'Back' + | 'Blinded ID' | 'Block' | 'Block contacts - Navigation' | 'blocked-banner' @@ -212,6 +216,7 @@ export type AccessibilityId = | 'Close' | 'Close button' | 'Community invitation' + | 'Community Message Requests' | 'Configuration message' | 'Confirm' | 'Confirm block' @@ -313,6 +318,7 @@ export type AccessibilityId = | 'MeetingSE' | 'Meetings option' | 'Mentions list' + | 'Message' | 'Message body' | 'Message composition' | 'Message input box' From f52b8b4975e10600a2d04338508f10ea0da84643 Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Mon, 20 Oct 2025 16:19:17 +1100 Subject: [PATCH 18/29] fix: upm test compares trimmed values --- run/test/specs/community_requests_off.spec.ts | 2 +- run/test/specs/community_requests_on.spec.ts | 2 +- run/test/specs/upm_homescreen.spec.ts | 9 +++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/run/test/specs/community_requests_off.spec.ts b/run/test/specs/community_requests_off.spec.ts index a7743300e..e43a5ba1c 100644 --- a/run/test/specs/community_requests_off.spec.ts +++ b/run/test/specs/community_requests_off.spec.ts @@ -8,7 +8,7 @@ import { joinCommunity } from './utils/join_community'; import { closeApp, openAppTwoDevices, SupportedPlatformsType } from './utils/open_app'; bothPlatformsIt({ - title: 'Blinded message request off', + title: 'Community message requests off', risk: 'medium', testCb: blindedMessageRequests, countOfDevicesNeeded: 2, diff --git a/run/test/specs/community_requests_on.spec.ts b/run/test/specs/community_requests_on.spec.ts index 6a3e9245d..aa7fe6ac9 100644 --- a/run/test/specs/community_requests_on.spec.ts +++ b/run/test/specs/community_requests_on.spec.ts @@ -13,7 +13,7 @@ import { joinCommunity } from './utils/join_community'; import { closeApp, openAppTwoDevices, SupportedPlatformsType } from './utils/open_app'; bothPlatformsIt({ - title: 'Blinded message request', + title: 'Community message requests on', risk: 'medium', testCb: blindedMessageRequests, countOfDevicesNeeded: 2, diff --git a/run/test/specs/upm_homescreen.spec.ts b/run/test/specs/upm_homescreen.spec.ts index 4025aebd8..a87021631 100644 --- a/run/test/specs/upm_homescreen.spec.ts +++ b/run/test/specs/upm_homescreen.spec.ts @@ -48,10 +48,11 @@ async function upmHomeScreen(platform: SupportedPlatformsType, testInfo: TestInf }); const eltext = await alice1.getTextFromElement(el); const normalized = eltext.replace(/\s+/g, ''); // account id comes in two lines - if (eltext !== bob.sessionId) { - throw new Error(`Account ID does not match. - Expected: ${bob.sessionId} - Observed: ${normalized}`); + const expected = bob.sessionId.trim(); + if (normalized !== expected) { + console.log(`Expected: ${expected} + Observed: ${normalized}`); + throw new Error('Incorrect Account ID in the User Profile Modal'); } }); await test.step(TestSteps.SETUP.CLOSE_APP, async () => { From ba8eaa7e8cbbc24bdbed41313862b629efcdd09a Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Tue, 21 Oct 2025 15:30:50 +1100 Subject: [PATCH 19/29] feat: add optional tolerance to color matching --- .../linked_device_profile_picture_syncs.spec.ts | 4 +++- run/test/specs/upm_homescreen.spec.ts | 4 ++-- .../user_actions_change_profile_picture.spec.ts | 4 +++- run/test/specs/utils/check_colour.ts | 12 +++++++----- run/types/DeviceWrapper.ts | 5 +++-- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/run/test/specs/linked_device_profile_picture_syncs.spec.ts b/run/test/specs/linked_device_profile_picture_syncs.spec.ts index 447da819e..d0247023b 100644 --- a/run/test/specs/linked_device_profile_picture_syncs.spec.ts +++ b/run/test/specs/linked_device_profile_picture_syncs.spec.ts @@ -21,6 +21,7 @@ bothPlatformsIt({ async function avatarRestored(platform: SupportedPlatformsType, testInfo: TestInfo) { const expectedPixelHexColor = '04cbfe'; // This is the color of the profile picture image stored in the repo + const tolerance = 5; // Slightly higher than default tolerance because of jpeg compression const { devices: { alice1, alice2 }, } = await open_Alice2({ platform, testInfo }); @@ -29,7 +30,8 @@ async function avatarRestored(platform: SupportedPlatformsType, testInfo: TestIn await test.step(TestSteps.VERIFY.PROFILE_PICTURE_CHANGED, async () => { await alice2.waitForElementColorMatch( { ...new UserAvatar(alice2).build(), maxWait: 20_000 }, - expectedPixelHexColor + expectedPixelHexColor, + tolerance ); }); await closeApp(alice1, alice2); diff --git a/run/test/specs/upm_homescreen.spec.ts b/run/test/specs/upm_homescreen.spec.ts index a87021631..ce36da7b5 100644 --- a/run/test/specs/upm_homescreen.spec.ts +++ b/run/test/specs/upm_homescreen.spec.ts @@ -46,8 +46,8 @@ async function upmHomeScreen(platform: SupportedPlatformsType, testInfo: TestInf strategy: 'id', selector: 'account-id', }); - const eltext = await alice1.getTextFromElement(el); - const normalized = eltext.replace(/\s+/g, ''); // account id comes in two lines + const elText = await alice1.getTextFromElement(el); + const normalized = elText.replace(/\s+/g, ''); // account id comes in two lines const expected = bob.sessionId.trim(); if (normalized !== expected) { console.log(`Expected: ${expected} diff --git a/run/test/specs/user_actions_change_profile_picture.spec.ts b/run/test/specs/user_actions_change_profile_picture.spec.ts index bdca8f7fd..5c3ed7c59 100644 --- a/run/test/specs/user_actions_change_profile_picture.spec.ts +++ b/run/test/specs/user_actions_change_profile_picture.spec.ts @@ -22,6 +22,7 @@ bothPlatformsIt({ async function changeProfilePicture(platform: SupportedPlatformsType, testInfo: TestInfo) { const expectedPixelHexColor = '04cbfe'; // This is the color of the profile picture image stored in the repo + const tolerance = 5; // Slightly higher than default tolerance because of jpeg compression const { device } = await test.step(TestSteps.SETUP.NEW_USER, async () => { const { device } = await openAppOnPlatformSingleDevice(platform, testInfo); await newUser(device, USERNAME.ALICE, { saveUserData: false }); @@ -33,7 +34,8 @@ async function changeProfilePicture(platform: SupportedPlatformsType, testInfo: await test.step(TestSteps.VERIFY.PROFILE_PICTURE_CHANGED, async () => { await device.waitForElementColorMatch( { ...new UserAvatar(device).build(), maxWait: 10_000 }, - expectedPixelHexColor + expectedPixelHexColor, + tolerance ); }); await test.step(TestSteps.SETUP.CLOSE_APP, async () => { diff --git a/run/test/specs/utils/check_colour.ts b/run/test/specs/utils/check_colour.ts index 21fd8a9f9..5b09032b3 100644 --- a/run/test/specs/utils/check_colour.ts +++ b/run/test/specs/utils/check_colour.ts @@ -38,13 +38,15 @@ export async function parseDataImage(base64: string) { return hexColor; } - // Determines if two colors look "the same" for humans even if they are not an exact match -export function isSameColor(hex1: string, hex2: string) { +export function isSameColor(hex1: string, hex2: string, tolerance?: number) { // Convert the hex strings to RGB objects const rgb1 = hexToRgbObject(hex1); const rgb2 = hexToRgbObject(hex2); - // Perform the color comparison using the looks-same library - const isSameColor = colors(rgb1, rgb2); - return isSameColor; + // Only pass options if tolerance is provided + if (tolerance !== undefined) { + return colors(rgb1, rgb2, { tolerance }); + } + // Call without options for default behavior + return colors(rgb1, rgb2); } diff --git a/run/types/DeviceWrapper.ts b/run/types/DeviceWrapper.ts index c07f9e7aa..4e81fabfc 100644 --- a/run/types/DeviceWrapper.ts +++ b/run/types/DeviceWrapper.ts @@ -1641,7 +1641,8 @@ export class DeviceWrapper { public async waitForElementColorMatch( args: { text?: string; maxWait?: number } & (LocatorsInterface | StrategyExtractionObj), - expectedColor: string + expectedColor: string, + tolerance?: number ): Promise { const locator = args instanceof LocatorsInterface ? args.build() : args; const description = describeLocator({ ...locator, text: args.text }); @@ -1661,7 +1662,7 @@ export class DeviceWrapper { const base64 = await this.getElementScreenshot(element.ELEMENT); const actualColor = await parseDataImage(base64); - const matches = isSameColor(expectedColor, actualColor); + const matches = isSameColor(expectedColor, actualColor, tolerance); return { success: matches, From 5b470de10a5a9f3c382be89b4fde4f70326de06f Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Tue, 21 Oct 2025 17:16:53 +1100 Subject: [PATCH 20/29] chore: decrease settings tolerance from 97 to 96 --- run/test/specs/utils/utilities.ts | 4 +++- run/test/specs/visual_settings.spec.ts | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/run/test/specs/utils/utilities.ts b/run/test/specs/utils/utilities.ts index 809a113dc..82bc0cd6f 100644 --- a/run/test/specs/utils/utilities.ts +++ b/run/test/specs/utils/utilities.ts @@ -86,9 +86,11 @@ export async function assertUrlIsReachable(url: string): Promise { export async function setConsistentStatusBar(device: DeviceWrapper): Promise { if (device.isIOS()) { // Time: 4:20, 100% battery, full wifi signal + device.log(`[DEBUG]: Attempting to set fake status bar`); await runScriptAndLog( - `xcrun simctl status_bar ${device.udid} override --time "04:20" --batteryLevel 100 --batteryState charged --wifiBars 3` + `xcrun simctl status_bar ${device.udid} override --time "04:20" --batteryLevel 100 --batteryState charged --wifiBars 3`, true ); + device.log(`[DEBUG]: Fake status bar command has been sent`); } else if (device.isAndroid()) { // Enable demo mode to set consistent status bar elements await runScriptAndLog(`adb -s ${device.udid} shell settings put global sysui_demo_allowed 1`); diff --git a/run/test/specs/visual_settings.spec.ts b/run/test/specs/visual_settings.spec.ts index fe72a8caf..ac32322a2 100644 --- a/run/test/specs/visual_settings.spec.ts +++ b/run/test/specs/visual_settings.spec.ts @@ -84,7 +84,7 @@ for (const { screenName, screenshotFile, navigation } of testCases) { }); await test.step(TestSteps.VERIFY.SCREENSHOT(screenName), async () => { - await verifyPageScreenshot(device, platform, screenshotFile, testInfo); + await verifyPageScreenshot(device, platform, screenshotFile, testInfo, 0.96); }); await test.step(TestSteps.SETUP.CLOSE_APP, async () => { From 3c86fc9a3689571ec344b3674ba8f27f5cb83977 Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Fri, 24 Oct 2025 15:18:15 +1100 Subject: [PATCH 21/29] feat: add allure rollback workflow --- .github/workflows/allure-rollback.yml | 43 +++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/allure-rollback.yml diff --git a/.github/workflows/allure-rollback.yml b/.github/workflows/allure-rollback.yml new file mode 100644 index 000000000..19c9e432e --- /dev/null +++ b/.github/workflows/allure-rollback.yml @@ -0,0 +1,43 @@ +name: Roll-back Allure Report + +# This workflow removes (unpublishes) the last Allure report from gh-pages. +# +# When to use: +# - Did not intend to create a new report (e.g. accidentally published a debug run) +# - Incorrect inputs (wrong build, metadta, Appium branch, etc.) +# - Test is corrupted by any other reason, don't want to muddy the result history + + +on: + workflow_dispatch: + +jobs: + cleanup: + runs-on: ubuntu-latest + steps: + - name: Checkout gh-pages branch + uses: actions/checkout@v4 + with: + ref: gh-pages + lfs: true + fetch-depth: 0 # Need full history for other workflows that rely on file ages + + - name: Configure git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Reset, force push and prune + run: | + echo "Resetting gh-pages by 1 commit..." + git reset HEAD~1 + git push --force-with-lease + git lfs prune --verify-remote || echo "LFS prune failed (non-fatal)" + + - name: Summary + run: | + echo "Successfully removed last commit from gh-pages" >> $GITHUB_STEP_SUMMARY + echo "Pruned orphaned LFS objects" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Triggered by: ${{ github.actor }}" >> $GITHUB_STEP_SUMMARY + echo "Current HEAD: $(git rev-parse HEAD)" >> $GITHUB_STEP_SUMMARY \ No newline at end of file From aab5ae3780bfdf8c1a1f510d6a5c4176afe90a66 Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Fri, 31 Oct 2025 16:52:21 +1100 Subject: [PATCH 22/29] feat: add delete group test --- .github/workflows/allure-rollback.yml | 13 ++- .../specs/group_tests_create_group.spec.ts | 5 ++ .../group_tests_create_group_banner.spec.ts | 6 ++ .../specs/group_tests_delete_group.spec.ts | 80 +++++++++++++++++++ .../group_tests_edit_group_banner.spec.ts | 5 ++ .../group_tests_invite_contact_banner.spec.ts | 6 ++ .../specs/group_tests_leave_group.spec.ts | 5 ++ run/test/specs/locators/groups.ts | 32 ++++++++ run/test/specs/utils/utilities.ts | 3 +- run/types/allure.ts | 2 +- run/types/testing.ts | 3 + 11 files changed, 151 insertions(+), 9 deletions(-) create mode 100644 run/test/specs/group_tests_delete_group.spec.ts diff --git a/.github/workflows/allure-rollback.yml b/.github/workflows/allure-rollback.yml index 19c9e432e..5fb2b4c98 100644 --- a/.github/workflows/allure-rollback.yml +++ b/.github/workflows/allure-rollback.yml @@ -1,13 +1,12 @@ name: Roll-back Allure Report # This workflow removes (unpublishes) the last Allure report from gh-pages. -# +# # When to use: # - Did not intend to create a new report (e.g. accidentally published a debug run) # - Incorrect inputs (wrong build, metadta, Appium branch, etc.) # - Test is corrupted by any other reason, don't want to muddy the result history - on: workflow_dispatch: @@ -20,24 +19,24 @@ jobs: with: ref: gh-pages lfs: true - fetch-depth: 0 # Need full history for other workflows that rely on file ages - + fetch-depth: 0 # Need full history for other workflows that rely on file ages + - name: Configure git run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - + - name: Reset, force push and prune run: | echo "Resetting gh-pages by 1 commit..." git reset HEAD~1 git push --force-with-lease git lfs prune --verify-remote || echo "LFS prune failed (non-fatal)" - + - name: Summary run: | echo "Successfully removed last commit from gh-pages" >> $GITHUB_STEP_SUMMARY echo "Pruned orphaned LFS objects" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "Triggered by: ${{ github.actor }}" >> $GITHUB_STEP_SUMMARY - echo "Current HEAD: $(git rev-parse HEAD)" >> $GITHUB_STEP_SUMMARY \ No newline at end of file + echo "Current HEAD: $(git rev-parse HEAD)" >> $GITHUB_STEP_SUMMARY diff --git a/run/test/specs/group_tests_create_group.spec.ts b/run/test/specs/group_tests_create_group.spec.ts index b2158f3f0..440f797cb 100644 --- a/run/test/specs/group_tests_create_group.spec.ts +++ b/run/test/specs/group_tests_create_group.spec.ts @@ -11,6 +11,11 @@ bothPlatformsIt({ risk: 'high', testCb: groupCreation, countOfDevicesNeeded: 3, + allureSuites: { + parent: 'Groups', + suite: 'Create Group', + }, + allureDescription: 'Verifies that a group of 3 can be created successfully via the UI', }); async function groupCreation(platform: SupportedPlatformsType, testInfo: TestInfo) { diff --git a/run/test/specs/group_tests_create_group_banner.spec.ts b/run/test/specs/group_tests_create_group_banner.spec.ts index 9d192d5ea..a5f57c4a8 100644 --- a/run/test/specs/group_tests_create_group_banner.spec.ts +++ b/run/test/specs/group_tests_create_group_banner.spec.ts @@ -13,6 +13,12 @@ androidIt({ risk: 'high', testCb: createGroupBanner, countOfDevicesNeeded: 2, + allureSuites: { + parent: 'Groups', + suite: 'Create Group', + }, + allureDescription: + 'Verifies that the latest release banner is present on the Create Group screen', }); async function createGroupBanner(platform: SupportedPlatformsType, testInfo: TestInfo) { diff --git a/run/test/specs/group_tests_delete_group.spec.ts b/run/test/specs/group_tests_delete_group.spec.ts new file mode 100644 index 000000000..031b89656 --- /dev/null +++ b/run/test/specs/group_tests_delete_group.spec.ts @@ -0,0 +1,80 @@ +import { test, type TestInfo } from '@playwright/test'; + +import { englishStrippedStr } from '../../localizer/englishStrippedStr'; +import { TestSteps } from '../../types/allure'; +import { bothPlatformsIt } from '../../types/sessionIt'; +import { ConversationSettings } from './locators/conversation'; +import { DeleteGroupConfirm, DeleteGroupMenuItem } from './locators/groups'; +import { ConversationItem } from './locators/home'; +import { open_Alice1_Bob1_Charlie1_friends_group } from './state_builder'; +import { closeApp, SupportedPlatformsType } from './utils/open_app'; + +bothPlatformsIt({ + title: 'Delete group', + risk: 'high', + testCb: deleteGroup, + countOfDevicesNeeded: 3, + allureSuites: { + parent: 'Groups', + suite: 'Leave/Delete Group', + }, + allureDescription: `Verifies that an admin can delete a group successfully via the UI. + The group members see the empty state control message, and the admin's conversation disappears from the home screen.`, +}); + +async function deleteGroup(platform: SupportedPlatformsType, testInfo: TestInfo) { + const testGroupName = 'Delete group'; + const { + devices: { alice1, bob1, charlie1 }, + } = await test.step(TestSteps.SETUP.QA_SEEDER, async () => { + return open_Alice1_Bob1_Charlie1_friends_group({ + platform, + groupName: testGroupName, + focusGroupConvo: true, + testInfo, + }); + }); + await test.step('Admin deletes group', async () => { + await alice1.clickOnElementAll(new ConversationSettings(alice1)); + await alice1.clickOnElementAll(new DeleteGroupMenuItem(alice1)); + await test.step(TestSteps.VERIFY.SPECIFIC_MODAL('Delete Group'), async () => { + await alice1.checkModalStrings( + englishStrippedStr('groupDelete').toString(), + englishStrippedStr('groupDeleteDescription') + .withArgs({ group_name: testGroupName }) + .toString() + ); + }); + await alice1.clickOnElementAll(new DeleteGroupConfirm(alice1)); + }); + await test.step('Verify group is deleted for all members', async () => { + if (platform === 'ios') { + await Promise.all( + [bob1, charlie1].map(device => + device.waitForControlMessageToBePresent( + englishStrippedStr('groupDeletedMemberDescription') + .withArgs({ group_name: testGroupName }) + .toString() + ) + ) + ); + } else { + // Android uses the empty state for this "control message" + await Promise.all( + [bob1, charlie1].map(device => + device.waitForTextElementToBePresent({ + strategy: 'accessibility id', + selector: 'Empty list', + text: englishStrippedStr('groupDeletedMemberDescription') + .withArgs({ group_name: testGroupName }) + .toString(), + }) + ) + ); + } + await alice1.verifyElementNotPresent(new ConversationItem(alice1, testGroupName).build()); + }); + await test.step(TestSteps.SETUP.CLOSE_APP, async () => { + await closeApp(alice1, bob1, charlie1); + }); +} diff --git a/run/test/specs/group_tests_edit_group_banner.spec.ts b/run/test/specs/group_tests_edit_group_banner.spec.ts index a06ce76d3..0e9d55c69 100644 --- a/run/test/specs/group_tests_edit_group_banner.spec.ts +++ b/run/test/specs/group_tests_edit_group_banner.spec.ts @@ -12,6 +12,11 @@ androidIt({ risk: 'medium', testCb: editGroupBanner, countOfDevicesNeeded: 3, + allureSuites: { + parent: 'Groups', + suite: 'Edit Group', + }, + allureDescription: 'Verifies that the latest release banner is present on the Edit Group screen', }); async function editGroupBanner(platform: SupportedPlatformsType, testInfo: TestInfo) { diff --git a/run/test/specs/group_tests_invite_contact_banner.spec.ts b/run/test/specs/group_tests_invite_contact_banner.spec.ts index ccabaebe7..a6bd351e4 100644 --- a/run/test/specs/group_tests_invite_contact_banner.spec.ts +++ b/run/test/specs/group_tests_invite_contact_banner.spec.ts @@ -13,6 +13,12 @@ androidIt({ risk: 'medium', testCb: inviteContactGroupBanner, countOfDevicesNeeded: 3, + allureSuites: { + parent: 'Groups', + suite: 'Edit Group', + }, + allureDescription: + 'Verifies that the latest release banner is present on the Invite Contacts screen', }); async function inviteContactGroupBanner(platform: SupportedPlatformsType, testInfo: TestInfo) { diff --git a/run/test/specs/group_tests_leave_group.spec.ts b/run/test/specs/group_tests_leave_group.spec.ts index 21adc4fa0..e3dbde48a 100644 --- a/run/test/specs/group_tests_leave_group.spec.ts +++ b/run/test/specs/group_tests_leave_group.spec.ts @@ -14,6 +14,11 @@ bothPlatformsIt({ risk: 'high', testCb: leaveGroup, countOfDevicesNeeded: 3, + allureSuites: { + parent: 'Groups', + suite: 'Leave/Delete Group', + }, + allureDescription: 'Verifies that a non-admin member can leave a group successfully via the UI', }); async function leaveGroup(platform: SupportedPlatformsType, testInfo: TestInfo) { diff --git a/run/test/specs/locators/groups.ts b/run/test/specs/locators/groups.ts index f994e7e9f..89f1439f9 100644 --- a/run/test/specs/locators/groups.ts +++ b/run/test/specs/locators/groups.ts @@ -196,6 +196,38 @@ export class LeaveGroupConfirm extends LocatorsInterface { } } } +export class DeleteGroupMenuItem extends LocatorsInterface { + public build(): StrategyExtractionObj { + switch (this.platform) { + case 'android': + return { + strategy: 'id', + selector: 'delete-group-menu-option', + } as const; + case 'ios': + return { + strategy: 'accessibility id', + selector: 'Leave group', // yep this is leave even for the delete option + } as const; + } + } +} +export class DeleteGroupConfirm extends LocatorsInterface { + public build(): StrategyExtractionObj { + switch (this.platform) { + case 'android': + return { + strategy: 'id', + selector: 'delete-group-confirm-button', + } as const; + case 'ios': + return { + strategy: 'accessibility id', + selector: 'Delete', + } as const; + } + } +} export class LatestReleaseBanner extends LocatorsInterface { public build() { switch (this.platform) { diff --git a/run/test/specs/utils/utilities.ts b/run/test/specs/utils/utilities.ts index 82bc0cd6f..bc07a9e30 100644 --- a/run/test/specs/utils/utilities.ts +++ b/run/test/specs/utils/utilities.ts @@ -88,7 +88,8 @@ export async function setConsistentStatusBar(device: DeviceWrapper): Promise Date: Fri, 31 Oct 2025 17:07:04 +1100 Subject: [PATCH 23/29] chore: add more allure suites and descriptions --- run/test/specs/invite_a_friend_share.spec.ts | 15 +++++++++------ run/test/specs/linked_device_avatar_color.spec.ts | 12 ++++++++---- .../specs/linked_device_change_username.spec.ts | 5 +++++ run/test/specs/linked_device_create_group.spec.ts | 6 ++++++ .../specs/linked_device_delete_message.spec.ts | 6 ++++++ .../specs/linked_device_hide_note_to_self.spec.ts | 1 + .../specs/linked_device_restore_group.spec.ts | 6 ++++++ run/types/allure.ts | 3 ++- 8 files changed, 43 insertions(+), 11 deletions(-) diff --git a/run/test/specs/invite_a_friend_share.spec.ts b/run/test/specs/invite_a_friend_share.spec.ts index 88ae0d575..a1afa55d0 100644 --- a/run/test/specs/invite_a_friend_share.spec.ts +++ b/run/test/specs/invite_a_friend_share.spec.ts @@ -14,6 +14,11 @@ bothPlatformsIt({ risk: 'medium', testCb: inviteAFriend, countOfDevicesNeeded: 1, + allureSuites: { + parent: 'New Conversation', + suite: 'Invite a Friend', + }, + allureDescription: `Verifies that the 'Invite a Friend' share functionality opens the native share sheet and the user's Account ID is present in the message.`, }); async function inviteAFriend(platform: SupportedPlatformsType, testInfo: TestInfo) { @@ -43,12 +48,10 @@ async function inviteAFriend(platform: SupportedPlatformsType, testInfo: TestInf } // Retrieve the Share message and validate that it contains the user's Account ID const retrievedShareMessage = await device.getTextFromElement(messageElement); - if (retrievedShareMessage.includes(user.accountID)) { - device.log("The Invite a Friend message snippet contains the user's Account ID"); - } else { - throw new Error( - `The Invite a Friend message snippet does not contain the user's Account ID\nThe message goes ${retrievedShareMessage}` - ); + if (!retrievedShareMessage.includes(user.accountID)) { + console.log(`Expected Share Message to contain Account ID: ${user.accountID}`); + console.log(`Actual Share Message: ${retrievedShareMessage}`); + throw new Error(`The Invite a Friend message snippet does not contain the user's Account ID.`); } await closeApp(device); } diff --git a/run/test/specs/linked_device_avatar_color.spec.ts b/run/test/specs/linked_device_avatar_color.spec.ts index cb8136b2e..cd411b340 100644 --- a/run/test/specs/linked_device_avatar_color.spec.ts +++ b/run/test/specs/linked_device_avatar_color.spec.ts @@ -11,11 +11,15 @@ bothPlatformsIt({ risk: 'medium', testCb: avatarColorLinkedDevice, countOfDevicesNeeded: 2, + allureSuites: { + parent: 'Visual Checks', + suite: 'Onboarding', + }, + allureDescription: `Verifies that a user's avatar color is consistent across linked devices.`, }); async function avatarColorLinkedDevice(platform: SupportedPlatformsType, testInfo: TestInfo) { const { devices: { alice1, alice2 }, - prebuilt: { alice }, } = await open_Alice2({ platform, testInfo }); // Get Alice's avatar color on device 1 (Home Screen avatar) and turn it into a hex value @@ -25,9 +29,9 @@ async function avatarColorLinkedDevice(platform: SupportedPlatformsType, testInf // Color matching devices 1 and 2 const colorMatch = isSameColor(alice1PixelColor, alice2PixelColor); if (!colorMatch) { - throw new Error( - `The avatar color of ${alice.userName} does not match across devices. The colors are ${alice1PixelColor} and ${alice2PixelColor}` - ); + console.log(`Device 1 pixel color: ${alice1PixelColor}`); + console.log(`Device 2 pixel color: ${alice2PixelColor}`); + throw new Error(`The user's placeholder avatar color does not match across linked devices.`); } await closeApp(alice1, alice2); } diff --git a/run/test/specs/linked_device_change_username.spec.ts b/run/test/specs/linked_device_change_username.spec.ts index 38e93bb34..7193e883f 100644 --- a/run/test/specs/linked_device_change_username.spec.ts +++ b/run/test/specs/linked_device_change_username.spec.ts @@ -12,6 +12,11 @@ bothPlatformsIt({ risk: 'medium', countOfDevicesNeeded: 2, testCb: changeUsernameLinked, + allureSuites: { + parent: 'User Actions', + suite: 'Change Username', + }, + allureDescription: `Verifies that a username change syncs to a linked device.`, }); async function changeUsernameLinked(platform: SupportedPlatformsType, testInfo: TestInfo) { diff --git a/run/test/specs/linked_device_create_group.spec.ts b/run/test/specs/linked_device_create_group.spec.ts index 40899c26d..64931e5dc 100644 --- a/run/test/specs/linked_device_create_group.spec.ts +++ b/run/test/specs/linked_device_create_group.spec.ts @@ -26,6 +26,12 @@ bothPlatformsItSeparate({ android: { testCb: linkedGroupAndroid, }, + allureSuites: { + parent: 'Groups', + suite: 'Create Group', + }, + allureDescription: + 'Verifies that a group created on one device appears on a linked device, and that a group name change syncs to all members.', }); async function linkedGroupiOS(platform: SupportedPlatformsType, testInfo: TestInfo) { diff --git a/run/test/specs/linked_device_delete_message.spec.ts b/run/test/specs/linked_device_delete_message.spec.ts index 1f21683f6..46a4d612d 100644 --- a/run/test/specs/linked_device_delete_message.spec.ts +++ b/run/test/specs/linked_device_delete_message.spec.ts @@ -13,6 +13,12 @@ bothPlatformsIt({ risk: 'high', testCb: deletedMessageLinkedDevice, countOfDevicesNeeded: 3, + allureSuites: { + parent: 'User Actions', + suite: 'Delete Message', + }, + allureDescription: + 'Verifies that when a message is deleted on one device, it shows as deleted on a linked device too.', }); async function deletedMessageLinkedDevice(platform: SupportedPlatformsType, testInfo: TestInfo) { const { diff --git a/run/test/specs/linked_device_hide_note_to_self.spec.ts b/run/test/specs/linked_device_hide_note_to_self.spec.ts index 19db9dff1..e6e02d740 100644 --- a/run/test/specs/linked_device_hide_note_to_self.spec.ts +++ b/run/test/specs/linked_device_hide_note_to_self.spec.ts @@ -18,6 +18,7 @@ bothPlatformsIt({ parent: 'User Actions', suite: 'Hide Note to Self', }, + allureDescription: 'Verifies that Hide Note To Self syncs to a linked device.', }); async function hideNoteToSelf(platform: SupportedPlatformsType, testInfo: TestInfo) { diff --git a/run/test/specs/linked_device_restore_group.spec.ts b/run/test/specs/linked_device_restore_group.spec.ts index e90c6e485..289e53d83 100644 --- a/run/test/specs/linked_device_restore_group.spec.ts +++ b/run/test/specs/linked_device_restore_group.spec.ts @@ -14,6 +14,12 @@ bothPlatformsIt({ risk: 'high', testCb: restoreGroup, countOfDevicesNeeded: 4, + allureSuites: { + parent: 'Groups', + suite: 'Create Group', + }, + allureDescription: + 'Verifies that a group (and its messages) created on one device appears when restoring on a second device.', }); async function restoreGroup(platform: SupportedPlatformsType, testInfo: TestInfo) { const testGroupName = 'Restore group'; diff --git a/run/types/allure.ts b/run/types/allure.ts index dbefa1928..0bcb6bffd 100644 --- a/run/types/allure.ts +++ b/run/types/allure.ts @@ -24,7 +24,7 @@ export type AllureSuiteConfig = | { parent: 'Groups'; suite: 'Create Group' | 'Edit Group' | 'Leave/Delete Group' } | { parent: 'In-App Review Prompt'; suite: 'Flows' | 'Triggers' } | { parent: 'Linkouts' } - | { parent: 'New Conversation'; suite: 'Join Community' | 'New Message' } + | { parent: 'New Conversation'; suite: 'Invite a Friend' | 'Join Community' | 'New Message' } | { parent: 'Sending Messages'; suite: 'Emoji reacts' | 'Message types' | 'Rules' } | { parent: 'Settings'; suite: 'App Disguise' } | { @@ -32,6 +32,7 @@ export type AllureSuiteConfig = suite: | 'Block/Unblock' | 'Change Profile Picture' + | 'Change Username' | 'Delete Contact' | 'Delete Conversation' | 'Delete Message' From 904277535e30c84ca6faa0460318c7946f0d1f6e Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Mon, 3 Nov 2025 11:10:37 +1100 Subject: [PATCH 24/29] fix: make delete group a 4 device test --- .../specs/group_tests_delete_group.spec.ts | 25 +++++--- run/test/specs/state_builder/index.ts | 61 +++++++++++++++++++ 2 files changed, 77 insertions(+), 9 deletions(-) diff --git a/run/test/specs/group_tests_delete_group.spec.ts b/run/test/specs/group_tests_delete_group.spec.ts index 031b89656..d63a96f75 100644 --- a/run/test/specs/group_tests_delete_group.spec.ts +++ b/run/test/specs/group_tests_delete_group.spec.ts @@ -5,29 +5,29 @@ import { TestSteps } from '../../types/allure'; import { bothPlatformsIt } from '../../types/sessionIt'; import { ConversationSettings } from './locators/conversation'; import { DeleteGroupConfirm, DeleteGroupMenuItem } from './locators/groups'; -import { ConversationItem } from './locators/home'; -import { open_Alice1_Bob1_Charlie1_friends_group } from './state_builder'; +import { ConversationItem, PlusButton } from './locators/home'; +import { open_Alice2_Bob1_Charlie1_friends_group } from './state_builder'; import { closeApp, SupportedPlatformsType } from './utils/open_app'; bothPlatformsIt({ - title: 'Delete group', + title: 'Delete group linked device', risk: 'high', testCb: deleteGroup, - countOfDevicesNeeded: 3, + countOfDevicesNeeded: 4, allureSuites: { parent: 'Groups', suite: 'Leave/Delete Group', }, allureDescription: `Verifies that an admin can delete a group successfully via the UI. - The group members see the empty state control message, and the admin's conversation disappears from the home screen.`, + The group members see the empty state control message, and the admin's conversation disappears from the home screen, even on a linked device.`, }); async function deleteGroup(platform: SupportedPlatformsType, testInfo: TestInfo) { const testGroupName = 'Delete group'; const { - devices: { alice1, bob1, charlie1 }, + devices: { alice1, bob1, charlie1, alice2}, } = await test.step(TestSteps.SETUP.QA_SEEDER, async () => { - return open_Alice1_Bob1_Charlie1_friends_group({ + return open_Alice2_Bob1_Charlie1_friends_group({ platform, groupName: testGroupName, focusGroupConvo: true, @@ -48,6 +48,7 @@ async function deleteGroup(platform: SupportedPlatformsType, testInfo: TestInfo) await alice1.clickOnElementAll(new DeleteGroupConfirm(alice1)); }); await test.step('Verify group is deleted for all members', async () => { + // Members if (platform === 'ios') { await Promise.all( [bob1, charlie1].map(device => @@ -72,9 +73,15 @@ async function deleteGroup(platform: SupportedPlatformsType, testInfo: TestInfo) ) ); } - await alice1.verifyElementNotPresent(new ConversationItem(alice1, testGroupName).build()); + // Admins + await Promise.all( + [alice1, alice2].map(async device => { + await device.waitForTextElementToBePresent(new PlusButton(device)); // Ensure we're on the home screen + await device.verifyElementNotPresent(new ConversationItem(device, testGroupName).build()); + }) + ); }); await test.step(TestSteps.SETUP.CLOSE_APP, async () => { - await closeApp(alice1, bob1, charlie1); + await closeApp(alice1, bob1, charlie1, alice2); }); } diff --git a/run/test/specs/state_builder/index.ts b/run/test/specs/state_builder/index.ts index 885ddea05..40cb2df3f 100644 --- a/run/test/specs/state_builder/index.ts +++ b/run/test/specs/state_builder/index.ts @@ -203,6 +203,67 @@ export async function open_Alice1_Bob1_Charlie1_friends_group({ }; } +export async function open_Alice2_Bob1_Charlie1_friends_group({ + platform, + groupName, + focusGroupConvo, + testInfo, +}: WithPlatform & + WithFocusGroupConvo & { + groupName: string; + testInfo: TestInfo; + }) { + const stateToBuildKey = '3friendsInGroup'; + const appsToOpen = 4; + const result = await openAppsWithState({ + platform, + appsToOpen, + stateToBuildKey, + groupName, + testInfo, + }); + result.devices[0].setDeviceIdentity('alice1'); + result.devices[1].setDeviceIdentity('bob1'); + result.devices[2].setDeviceIdentity('charlie1'); + result.devices[3].setDeviceIdentity('alice2'); + + const alice = result.prebuilt.users[0]; + const bob = result.prebuilt.users[1]; + const charlie = result.prebuilt.users[2]; + + const seedPhrases = [alice.seedPhrase, bob.seedPhrase, charlie.seedPhrase, alice.seedPhrase]; + await linkDevices(result.devices, seedPhrases); + + const alice1 = result.devices[0]; + const bob1 = result.devices[1]; + const charlie1 = result.devices[2]; + const alice2 = result.devices[3]; + + const formattedGroup = { group: result.prebuilt.group }; + const formattedDevices = { + alice1, + bob1, + charlie1, + alice2, + }; + const formattedUsers: WithUsers<3> = { + alice, + bob, + charlie, + }; + if (focusGroupConvo) { + await focusConvoOnDevices({ + devices: result.devices, + convoName: result.prebuilt.group.groupName, + }); + } + + return { + devices: formattedDevices, + prebuilt: { ...formattedUsers, ...formattedGroup }, + }; +} + /** * Open 4 devices, one for Alice, one for Bob, one for Charlie, and one extra, unlinked. * This function is used for testing that we can do a bunch of actions without having a linked device, From 528927e21853e145088f44b49b21dfe864dcfc72 Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Wed, 5 Nov 2025 09:40:32 +1100 Subject: [PATCH 25/29] feat: add network page fetch test --- .../specs/group_tests_delete_group.spec.ts | 2 +- run/test/specs/locators/network_page.ts | 81 +++++++++++++++++++ run/test/specs/network_page_values.spec.ts | 57 +++++++++++++ run/test/specs/state_builder/index.ts | 2 +- run/types/testing.ts | 6 ++ 5 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 run/test/specs/network_page_values.spec.ts diff --git a/run/test/specs/group_tests_delete_group.spec.ts b/run/test/specs/group_tests_delete_group.spec.ts index d63a96f75..c260824a3 100644 --- a/run/test/specs/group_tests_delete_group.spec.ts +++ b/run/test/specs/group_tests_delete_group.spec.ts @@ -25,7 +25,7 @@ bothPlatformsIt({ async function deleteGroup(platform: SupportedPlatformsType, testInfo: TestInfo) { const testGroupName = 'Delete group'; const { - devices: { alice1, bob1, charlie1, alice2}, + devices: { alice1, bob1, charlie1, alice2 }, } = await test.step(TestSteps.SETUP.QA_SEEDER, async () => { return open_Alice2_Bob1_Charlie1_friends_group({ platform, diff --git a/run/test/specs/locators/network_page.ts b/run/test/specs/locators/network_page.ts index dfb736cc9..148e81d30 100644 --- a/run/test/specs/locators/network_page.ts +++ b/run/test/specs/locators/network_page.ts @@ -1,4 +1,5 @@ import { LocatorsInterface } from '.'; +import { DeviceWrapper } from '../../../types/DeviceWrapper'; export class SessionNetworkMenuItem extends LocatorsInterface { public build() { @@ -84,3 +85,83 @@ export class OpenLinkButton extends LocatorsInterface { } } } +export class SESHPrice extends LocatorsInterface { + private expectedText: string; + + constructor(device: DeviceWrapper, priceValue: number) { + super(device); + this.expectedText = `$${priceValue.toFixed(2)} USD`; + } + + public build() { + switch (this.platform) { + case 'android': + return { + strategy: 'id', + selector: 'SESH price', + text: this.expectedText, + } as const; + case 'ios': + return { + strategy: 'accessibility id', + selector: 'SENT price', + text: this.expectedText, + } as const; + } + } +} + +export class StakingRewardPoolAmount extends LocatorsInterface { + private expectedText: string; + + constructor(device: DeviceWrapper, amount: number) { + super(device); + // Format with commas and SESH suffix + this.expectedText = `${amount.toLocaleString('en-US')} SESH`; + } + + public build() { + switch (this.platform) { + case 'android': + return { + strategy: 'id', + selector: 'Staking reward pool amount', + text: this.expectedText, + } as const; + case 'ios': + return { + strategy: 'accessibility id', + selector: 'Staking reward pool amount', + text: this.expectedText, + } as const; + } + } +} + +export class MarketCapAmount extends LocatorsInterface { + private expectedText: string; + + constructor(device: DeviceWrapper, amount: number) { + super(device); + // Round to whole number, then format with commas and USD suffix + const rounded = Math.round(amount); + this.expectedText = `$${rounded.toLocaleString('en-US')} USD`; + } + + public build() { + switch (this.platform) { + case 'android': + return { + strategy: 'id', + selector: 'Market cap amount', + text: this.expectedText, + } as const; + case 'ios': + return { + strategy: 'accessibility id', + selector: 'Market cap amount', + text: this.expectedText, + } as const; + } + } +} diff --git a/run/test/specs/network_page_values.spec.ts b/run/test/specs/network_page_values.spec.ts new file mode 100644 index 000000000..d4246ba47 --- /dev/null +++ b/run/test/specs/network_page_values.spec.ts @@ -0,0 +1,57 @@ +import type { TestInfo } from '@playwright/test'; + +import { bothPlatformsIt } from '../../types/sessionIt'; +import { USERNAME } from '../../types/testing'; +import { MarketCapAmount, SESHPrice, SessionNetworkMenuItem, StakingRewardPoolAmount } from './locators/network_page'; +import { UserSettings } from './locators/settings'; +import { newUser } from './utils/create_account'; +import { closeApp, openAppOnPlatformSingleDevice, SupportedPlatformsType } from './utils/open_app'; + +bothPlatformsIt({ + title: 'Network page values', + risk: 'medium', + testCb: networkPageValues, + countOfDevicesNeeded: 1, +}); + +function validateNetworkData(data: any): asserts data is { + price: { usd: number; usd_market_cap: number }; + token: { staking_reward_pool: number }; +} { + if ( + typeof data?.price?.usd !== 'number' || + typeof data?.token?.staking_reward_pool !== 'number' || + typeof data?.price?.usd_market_cap !== 'number' + ) { + throw new Error('Network API response missing or invalid numeric fields'); + } +} + +async function networkPageValues(platform: SupportedPlatformsType, testInfo: TestInfo) { + const { device } = await openAppOnPlatformSingleDevice(platform, testInfo); + await newUser(device, USERNAME.ALICE, { saveUserData: false }); + await device.clickOnElementAll(new UserSettings(device)); + await device.clickOnElementAll(new SessionNetworkMenuItem(device)); + + const response = await fetch('http://networkv1.getsession.org/info'); + if (!response.ok) { + throw new Error(`Network API returned ${response.status}`); + } + const data = await response.json(); + validateNetworkData(data); + + // SESH Price + await device.waitForTextElementToBePresent(new SESHPrice(device, data.price.usd)); + + // Staking Reward Pool + await device.waitForTextElementToBePresent( + new StakingRewardPoolAmount(device, data.token.staking_reward_pool) + ); + + // Market Cap + await device.waitForTextElementToBePresent( + new MarketCapAmount(device, data.price.usd_market_cap) + ); + + await closeApp(device); +} diff --git a/run/test/specs/state_builder/index.ts b/run/test/specs/state_builder/index.ts index 40cb2df3f..975cdaf5a 100644 --- a/run/test/specs/state_builder/index.ts +++ b/run/test/specs/state_builder/index.ts @@ -225,7 +225,7 @@ export async function open_Alice2_Bob1_Charlie1_friends_group({ result.devices[0].setDeviceIdentity('alice1'); result.devices[1].setDeviceIdentity('bob1'); result.devices[2].setDeviceIdentity('charlie1'); - result.devices[3].setDeviceIdentity('alice2'); + result.devices[3].setDeviceIdentity('alice2'); const alice = result.prebuilt.users[0]; const bob = result.prebuilt.users[1]; diff --git a/run/types/testing.ts b/run/types/testing.ts index 294805dec..52b9823a8 100644 --- a/run/types/testing.ts +++ b/run/types/testing.ts @@ -315,6 +315,7 @@ export type AccessibilityId = | 'Loading animation' | 'Local Network Permission - Switch' | 'Manage Members' + | 'Market cap amount' | 'Media message' | 'MeetingSE' | 'Meetings option' @@ -386,6 +387,7 @@ export type AccessibilityId = | 'Select alternate app icon' | 'Send' | 'Send message button' + | 'SENT price' | 'Session' | 'Session | Send Messages, Not Metadata. | Private Messenger' | 'Session ID generated' @@ -402,6 +404,7 @@ export type AccessibilityId = | 'Show roots' | 'Slow mode notifications button' | 'space' + | 'Staking reward pool amount' | 'TabBarItemTitle' | 'Terms of Service' | 'test_file, pdf' @@ -509,6 +512,7 @@ export type Id = | 'Leave' | 'Loading animation' | 'manage-members-menu-option' + | 'Market cap amount' | 'MeetingSE option' | 'Modal description' | 'Modal heading' @@ -567,6 +571,7 @@ export type Id = | 'Reveal recovery phrase button' | 'Save' | 'Select All' + | 'SESH price' | 'session-network-menu-item' | 'Session id input box' | 'set-nickname-confirm-button' @@ -575,6 +580,7 @@ export type Id = | 'show-nts-confirm-button' | 'Show' | 'Slow mode notifications button' + | 'Staking reward pool amount' | 'Terms of Service' | 'update-group-info-confirm-button' | 'update-group-info-description-input' From 0b4149d685e0762001a64a78cfa2c4646496aa21 Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Wed, 5 Nov 2025 12:02:46 +1100 Subject: [PATCH 26/29] fix: add another self healing blacklist item --- run/test/specs/network_page_values.spec.ts | 11 ++++++++--- run/types/DeviceWrapper.ts | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/run/test/specs/network_page_values.spec.ts b/run/test/specs/network_page_values.spec.ts index d4246ba47..5bd0d8d66 100644 --- a/run/test/specs/network_page_values.spec.ts +++ b/run/test/specs/network_page_values.spec.ts @@ -2,7 +2,12 @@ import type { TestInfo } from '@playwright/test'; import { bothPlatformsIt } from '../../types/sessionIt'; import { USERNAME } from '../../types/testing'; -import { MarketCapAmount, SESHPrice, SessionNetworkMenuItem, StakingRewardPoolAmount } from './locators/network_page'; +import { + MarketCapAmount, + SESHPrice, + SessionNetworkMenuItem, + StakingRewardPoolAmount, +} from './locators/network_page'; import { UserSettings } from './locators/settings'; import { newUser } from './utils/create_account'; import { closeApp, openAppOnPlatformSingleDevice, SupportedPlatformsType } from './utils/open_app'; @@ -42,12 +47,12 @@ async function networkPageValues(platform: SupportedPlatformsType, testInfo: Tes // SESH Price await device.waitForTextElementToBePresent(new SESHPrice(device, data.price.usd)); - + // Staking Reward Pool await device.waitForTextElementToBePresent( new StakingRewardPoolAmount(device, data.token.staking_reward_pool) ); - + // Market Cap await device.waitForTextElementToBePresent( new MarketCapAmount(device, data.price.usd_market_cap) diff --git a/run/types/DeviceWrapper.ts b/run/types/DeviceWrapper.ts index 4e81fabfc..160b50869 100644 --- a/run/types/DeviceWrapper.ts +++ b/run/types/DeviceWrapper.ts @@ -333,6 +333,7 @@ export class DeviceWrapper { { from: 'Voice message', to: 'New voice message' }, { from: 'Message sent status: Sent', to: 'Message sent status: Sending' }, { from: 'Done', to: 'Donate' }, + { from: 'New conversation button', to: 'conversation-options-avatar' }, ]; // System locators such as 'network.loki.messenger.qa:id' can cause false positives with too high similarity scores From 1a16a48b92d2a7931fe2b14aa54bb201e7cd22d5 Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Wed, 5 Nov 2025 13:47:54 +1100 Subject: [PATCH 27/29] chore: add allure to network page tests --- .../specs/network_page_link_network.spec.ts | 7 +- .../specs/network_page_link_staking.spec.ts | 7 +- .../specs/network_page_refresh_page.spec.ts | 4 + run/test/specs/network_page_values.spec.ts | 74 +++++++++++++------ run/types/allure.ts | 1 + 5 files changed, 66 insertions(+), 27 deletions(-) diff --git a/run/test/specs/network_page_link_network.spec.ts b/run/test/specs/network_page_link_network.spec.ts index b85fb25e3..574cadf80 100644 --- a/run/test/specs/network_page_link_network.spec.ts +++ b/run/test/specs/network_page_link_network.spec.ts @@ -20,6 +20,11 @@ bothPlatformsIt({ risk: 'medium', testCb: networkPageLearnMore, countOfDevicesNeeded: 1, + allureSuites: { + parent: 'Network Page', + }, + allureDescription: + 'Verifies that the "Learn More" link on the Network Page opens the correct URL in the device browser.', }); async function networkPageLearnMore(platform: SupportedPlatformsType, testInfo: TestInfo) { @@ -50,8 +55,6 @@ async function networkPageLearnMore(platform: SupportedPlatformsType, testInfo: throw new Error( `The retrieved URL does not match the expected. The retrieved URL is ${fullRetrievedURL}` ); - } else { - device.log('The URLs match.'); } await assertUrlIsReachable(linkURL); // Close browser and app diff --git a/run/test/specs/network_page_link_staking.spec.ts b/run/test/specs/network_page_link_staking.spec.ts index 2a8e9a254..114dcc2bf 100644 --- a/run/test/specs/network_page_link_staking.spec.ts +++ b/run/test/specs/network_page_link_staking.spec.ts @@ -20,6 +20,11 @@ bothPlatformsIt({ risk: 'medium', testCb: networkPageLearnMore, countOfDevicesNeeded: 1, + allureSuites: { + parent: 'Network Page', + }, + allureDescription: + 'Verifies that the "Learn More" link on the Network Page for Staking opens the correct URL in the device browser.', }); async function networkPageLearnMore(platform: SupportedPlatformsType, testInfo: TestInfo) { @@ -51,8 +56,6 @@ async function networkPageLearnMore(platform: SupportedPlatformsType, testInfo: throw new Error( `The retrieved URL does not match the expected. The retrieved URL is ${fullRetrievedURL}` ); - } else { - device.log('The URLs match.'); } await assertUrlIsReachable(linkURL); // Close browser and app diff --git a/run/test/specs/network_page_refresh_page.spec.ts b/run/test/specs/network_page_refresh_page.spec.ts index 55201d690..0340fede0 100644 --- a/run/test/specs/network_page_refresh_page.spec.ts +++ b/run/test/specs/network_page_refresh_page.spec.ts @@ -12,6 +12,10 @@ bothPlatformsIt({ risk: 'low', testCb: refreshNetworkPage, countOfDevicesNeeded: 1, + allureSuites: { + parent: 'Network Page', + }, + allureDescription: `Verifies that the Network Page refreshes and updates the "Last updated" timestamp correctly.`, }); async function refreshNetworkPage(platform: SupportedPlatformsType, testInfo: TestInfo) { diff --git a/run/test/specs/network_page_values.spec.ts b/run/test/specs/network_page_values.spec.ts index 5bd0d8d66..10721ee71 100644 --- a/run/test/specs/network_page_values.spec.ts +++ b/run/test/specs/network_page_values.spec.ts @@ -1,5 +1,6 @@ -import type { TestInfo } from '@playwright/test'; +import { test, type TestInfo } from '@playwright/test'; +import { TestSteps } from '../../types/allure'; import { bothPlatformsIt } from '../../types/sessionIt'; import { USERNAME } from '../../types/testing'; import { @@ -17,6 +18,11 @@ bothPlatformsIt({ risk: 'medium', testCb: networkPageValues, countOfDevicesNeeded: 1, + allureSuites: { + parent: 'Network Page', + }, + allureDescription: + 'Verifies that the Session Network Page displays the values fetched from the network API correctly.', }); function validateNetworkData(data: any): asserts data is { @@ -33,30 +39,52 @@ function validateNetworkData(data: any): asserts data is { } async function networkPageValues(platform: SupportedPlatformsType, testInfo: TestInfo) { - const { device } = await openAppOnPlatformSingleDevice(platform, testInfo); - await newUser(device, USERNAME.ALICE, { saveUserData: false }); - await device.clickOnElementAll(new UserSettings(device)); - await device.clickOnElementAll(new SessionNetworkMenuItem(device)); - - const response = await fetch('http://networkv1.getsession.org/info'); - if (!response.ok) { - throw new Error(`Network API returned ${response.status}`); - } - const data = await response.json(); - validateNetworkData(data); + let data: { + price: { usd: number; usd_market_cap: number }; + token: { staking_reward_pool: number }; + }; + + const { device } = await test.step(TestSteps.SETUP.NEW_USER, async () => { + const { device } = await openAppOnPlatformSingleDevice(platform, testInfo); + await newUser(device, USERNAME.ALICE, { saveUserData: false }); + return { device }; + }); + + await test.step(TestSteps.OPEN.GENERIC('Session Network Page'), async () => { + await device.clickOnElementAll(new UserSettings(device)); + await device.clickOnElementAll(new SessionNetworkMenuItem(device)); + }); + + await test.step('Fetch and validate Network API data', async () => { + const response = await fetch('http://networkv1.getsession.org/info'); + if (!response.ok) { + throw new Error(`Network API returned ${response.status}`); + } + data = await response.json(); + validateNetworkData(data); + + console.log(`Price: ${data.price.usd}`); + console.log(`Staking Reward Pool: ${data.token.staking_reward_pool}`); + console.log(`Market Cap: ${data.price.usd_market_cap}`); + }); - // SESH Price - await device.waitForTextElementToBePresent(new SESHPrice(device, data.price.usd)); + await test.step('Verify SESH price is displayed correctly', async () => { + await device.waitForTextElementToBePresent(new SESHPrice(device, data.price.usd)); + }); - // Staking Reward Pool - await device.waitForTextElementToBePresent( - new StakingRewardPoolAmount(device, data.token.staking_reward_pool) - ); + await test.step('Verify Staking Reward Pool is displayed correctly', async () => { + await device.waitForTextElementToBePresent( + new StakingRewardPoolAmount(device, data.token.staking_reward_pool) + ); + }); - // Market Cap - await device.waitForTextElementToBePresent( - new MarketCapAmount(device, data.price.usd_market_cap) - ); + await test.step('Verify Market Cap is displayed correctly', async () => { + await device.waitForTextElementToBePresent( + new MarketCapAmount(device, data.price.usd_market_cap) + ); + }); - await closeApp(device); + await test.step(TestSteps.SETUP.CLOSE_APP, async () => { + await closeApp(device); + }); } diff --git a/run/types/allure.ts b/run/types/allure.ts index 0bcb6bffd..a7ab333a9 100644 --- a/run/types/allure.ts +++ b/run/types/allure.ts @@ -24,6 +24,7 @@ export type AllureSuiteConfig = | { parent: 'Groups'; suite: 'Create Group' | 'Edit Group' | 'Leave/Delete Group' } | { parent: 'In-App Review Prompt'; suite: 'Flows' | 'Triggers' } | { parent: 'Linkouts' } + | { parent: 'Network Page' } | { parent: 'New Conversation'; suite: 'Invite a Friend' | 'Join Community' | 'New Message' } | { parent: 'Sending Messages'; suite: 'Emoji reacts' | 'Message types' | 'Rules' } | { parent: 'Settings'; suite: 'App Disguise' } From 3caa7260b781bb49e973d6c00673be4e2a71818f Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Fri, 7 Nov 2025 10:20:46 +1100 Subject: [PATCH 28/29] fix: tidy up community mr tests --- .github/workflows/android-regression.yml | 2 +- run/test/specs/community_requests_off.spec.ts | 79 ++++------ run/test/specs/community_requests_on.spec.ts | 145 +++++++----------- run/test/specs/locators/conversation.ts | 86 +++++++++++ run/test/specs/locators/settings.ts | 17 ++ run/types/allure.ts | 2 +- 6 files changed, 193 insertions(+), 138 deletions(-) diff --git a/.github/workflows/android-regression.yml b/.github/workflows/android-regression.yml index c6899bce4..171f7bce9 100644 --- a/.github/workflows/android-regression.yml +++ b/.github/workflows/android-regression.yml @@ -100,7 +100,7 @@ jobs: echo "Checking devnet accessibility for APK selection..." DEVNET_ACCESSIBLE=false - # Retry logic matching your TypeScript function + # Retry logic for attempt in 1 2 3; do echo "Devnet check attempt $attempt/3..." if curl -s --connect-timeout 5 --max-time 10 http://sesh-net.local:1280 >/dev/null 2>&1; then diff --git a/run/test/specs/community_requests_off.spec.ts b/run/test/specs/community_requests_off.spec.ts index e43a5ba1c..075d5a55d 100644 --- a/run/test/specs/community_requests_off.spec.ts +++ b/run/test/specs/community_requests_off.spec.ts @@ -1,8 +1,10 @@ -import { expect, TestInfo } from '@playwright/test'; +import { test, type TestInfo } from '@playwright/test'; import { USERNAME } from '@session-foundation/qa-seeder'; import { testCommunityLink, testCommunityName } from '../../constants/community'; +import { TestSteps } from '../../types/allure'; import { bothPlatformsIt } from '../../types/sessionIt'; +import { CommunityMessageAuthor, UPMMessageButton } from './locators/conversation'; import { newUser } from './utils/create_account'; import { joinCommunity } from './utils/join_community'; import { closeApp, openAppTwoDevices, SupportedPlatformsType } from './utils/open_app'; @@ -12,64 +14,41 @@ bothPlatformsIt({ risk: 'medium', testCb: blindedMessageRequests, countOfDevicesNeeded: 2, + allureSuites: { parent: 'Settings', suite: 'Community Message Requests' }, allureDescription: 'Verifies that a message request cannot be sent when Community Message Requests are off.', }); -// TODO: tidy this up with neat locators async function blindedMessageRequests(platform: SupportedPlatformsType, testInfo: TestInfo) { - const { device1, device2 } = await openAppTwoDevices(platform, testInfo); - await Promise.all([ - newUser(device1, USERNAME.ALICE, { saveUserData: false }), - newUser(device2, USERNAME.BOB, { saveUserData: false }), - ]); - await Promise.all( - [device1, device2].map(async device => { - await joinCommunity(device, testCommunityLink, testCommunityName); - }) - ); const message = `I do not accept blinded message requests + ${platform} + ${Date.now()}`; - await device2.sendMessage(message); - // Click on profile picture (Android) or sender name (iOS) - await device1 - .onAndroid() - .clickOnElementXPath( - `//android.view.ViewGroup[@resource-id='network.loki.messenger.qa:id/mainContainer'][.//android.widget.TextView[contains(@text,'${message}')]]//androidx.compose.ui.platform.ComposeView[@resource-id='network.loki.messenger.qa:id/profilePictureView']` + const { device1, device2 } = await test.step(TestSteps.SETUP.NEW_USER, async () => { + const { device1, device2 } = await openAppTwoDevices(platform, testInfo); + await Promise.all([ + newUser(device1, USERNAME.ALICE, { saveUserData: false }), + newUser(device2, USERNAME.BOB, { saveUserData: false }), + ]); + return { device1, device2 }; + }); + await test.step(TestSteps.NEW_CONVERSATION.JOIN_COMMUNITY, async () => { + await Promise.all( + [device1, device2].map(async device => { + await joinCommunity(device, testCommunityLink, testCommunityName); + }) ); - await device1 - .onIOS() - .clickOnElementXPath( - `//XCUIElementTypeCell[.//XCUIElementTypeOther[@name='Message body' and contains(@label,'${message}')]]//XCUIElementTypeStaticText[contains(@value,'(15')]` - ); - - let attr; - - if (platform === 'android') { - const el = await device1.waitForTextElementToBePresent({ - strategy: 'id', - selector: 'account-id', - }); - const elText = await device1.getTextFromElement(el); - expect(elText).toMatch(/^15/); - const messageButton = await device1.waitForTextElementToBePresent({ - strategy: 'xpath', - selector: `//android.widget.TextView[@text="Message"]/parent::android.view.View`, - }); - attr = await device1.getAttribute('enabled', messageButton.ELEMENT); - } else { - await device1.waitForTextElementToBePresent({ - strategy: 'accessibility id', - selector: 'Blinded ID', - }); - const messageButton = await device1.waitForTextElementToBePresent({ - strategy: 'accessibility id', - selector: 'Message', - }); - attr = await device1.getAttribute('enabled', messageButton.ELEMENT); - } + }); + await test.step(TestSteps.SEND.MESSAGE(USERNAME.BOB, testCommunityName), async () => { + await device2.sendMessage(message); + }); + await device1.clickOnElementAll(new CommunityMessageAuthor(device1, message)); + await test.step(`Verify the 'Message' button in the User Profile Modal is disabled`, async () => { + const messageButton = await device1.waitForTextElementToBePresent(new UPMMessageButton(device1)); + const attr = await device1.getAttribute('enabled', messageButton.ELEMENT); if (attr !== 'false') { device1.log(`Message button attribute is 'enabled = ${attr}'`); throw new Error(`Message button should be disabled but it is not`); } - await closeApp(device1, device2); +}); + await test.step(TestSteps.SETUP.CLOSE_APP, async () => { + await closeApp(device1, device2); + }); } diff --git a/run/test/specs/community_requests_on.spec.ts b/run/test/specs/community_requests_on.spec.ts index aa7fe6ac9..de0c98ece 100644 --- a/run/test/specs/community_requests_on.spec.ts +++ b/run/test/specs/community_requests_on.spec.ts @@ -1,13 +1,20 @@ -import { expect, TestInfo } from '@playwright/test'; +import { test, type TestInfo } from '@playwright/test'; import { USERNAME } from '@session-foundation/qa-seeder'; import { testCommunityLink, testCommunityName } from '../../constants/community'; -import { englishStrippedStr } from '../../localizer/englishStrippedStr'; +import { TestSteps } from '../../types/allure'; import { bothPlatformsIt } from '../../types/sessionIt'; import { CloseSettings } from './locators'; -import { ConversationHeaderName, MessageBody } from './locators/conversation'; +import { + CommunityMessageAuthor, + ConversationHeaderName, + MessageBody, + MessageRequestAcceptDescription, + MessageRequestPendingDescription, + UPMMessageButton, +} from './locators/conversation'; import { MessageRequestsBanner } from './locators/home'; -import { PrivacyMenuItem, UserSettings } from './locators/settings'; +import { CommunityMessageRequestSwitch, PrivacyMenuItem, UserSettings } from './locators/settings'; import { newUser } from './utils/create_account'; import { joinCommunity } from './utils/join_community'; import { closeApp, openAppTwoDevices, SupportedPlatformsType } from './utils/open_app'; @@ -17,6 +24,7 @@ bothPlatformsIt({ risk: 'medium', testCb: blindedMessageRequests, countOfDevicesNeeded: 2, + allureSuites: { parent: 'Settings', suite: 'Community Message Requests' }, allureDescription: 'Verifies that a message request can be sent when Community Message Requests are on.', allureLinks: { @@ -24,95 +32,60 @@ bothPlatformsIt({ }, }); -// TODO: tidy this up with neat locators async function blindedMessageRequests(platform: SupportedPlatformsType, testInfo: TestInfo) { - const { device1, device2 } = await openAppTwoDevices(platform, testInfo); - const [alice, bob] = await Promise.all([ - newUser(device1, USERNAME.ALICE, { saveUserData: false }), - newUser(device2, USERNAME.BOB, { saveUserData: false }), - ]); - await device2.clickOnElementAll(new UserSettings(device2)); - await device2.clickOnElementAll(new PrivacyMenuItem(device2)); - await device2.onAndroid().clickOnElementAll({ - strategy: '-android uiautomator', - selector: 'new UiSelector().text("Community Message Requests")', + const message = `I accept blinded message requests + ${platform} + ${Date.now()}`; + const messageRequestMessage = 'Howdy'; + const messageRequestReply = 'Howdy back'; + const { device1, device2, alice, bob } = await test.step(TestSteps.SETUP.NEW_USER, async () => { + const { device1, device2 } = await openAppTwoDevices(platform, testInfo); + const [alice, bob] = await Promise.all([ + newUser(device1, USERNAME.ALICE, { saveUserData: false }), + newUser(device2, USERNAME.BOB, { saveUserData: false }), + ]); + return { device1, device2, alice, bob }; }); - await device2.onIOS().clickOnElementAll({ - strategy: 'accessibility id', - selector: 'Community Message Requests', + await test.step('Bob enables Community Message Requests', async () => { + await device2.clickOnElementAll(new UserSettings(device2)); + await device2.clickOnElementAll(new PrivacyMenuItem(device2)); + await device2.clickOnElementAll(new CommunityMessageRequestSwitch(device2)); + await device2.navigateBack(); + await device2.clickOnElementAll(new CloseSettings(device2)); }); - await device2.navigateBack(); - await device2.clickOnElementAll(new CloseSettings(device2)); - await Promise.all( - [device1, device2].map(async device => { - await joinCommunity(device, testCommunityLink, testCommunityName); - }) - ); - const message = `I accept blinded message requests + ${platform} + ${Date.now()}`; - await device2.sendMessage(message); - await device2.navigateBack(); - // Click on profile picture (Android) or sender name (iOS) - await device1 - .onAndroid() - .clickOnElementXPath( - `//android.view.ViewGroup[@resource-id='network.loki.messenger.qa:id/mainContainer'][.//android.widget.TextView[contains(@text,'${message}')]]//androidx.compose.ui.platform.ComposeView[@resource-id='network.loki.messenger.qa:id/profilePictureView']` + await test.step(TestSteps.NEW_CONVERSATION.JOIN_COMMUNITY, async () => { + await Promise.all( + [device1, device2].map(async device => { + await joinCommunity(device, testCommunityLink, testCommunityName); + }) ); - await device1 - .onIOS() - .clickOnElementXPath( - `//XCUIElementTypeCell[.//XCUIElementTypeOther[@name='Message body' and contains(@label,'${message}')]]//XCUIElementTypeStaticText[contains(@value,'(15')]` - ); - if (platform === 'android') { - const el = await device1.waitForTextElementToBePresent({ - strategy: 'id', - selector: 'account-id', - }); - const elText = await device1.getTextFromElement(el); - expect(elText).toMatch(/^15/); - await device1.clickOnElementAll({ - strategy: '-android uiautomator', - selector: 'new UiSelector().text("Message")', - }); - } else { - await device1.waitForTextElementToBePresent({ - strategy: 'accessibility id', - selector: 'Blinded ID', - }); - await device1.clickOnByAccessibilityID('Message'); - } + }); - await device1.clickOnElementAll(new ConversationHeaderName(device1, bob.userName)); - const messageRequestPendingDescription = englishStrippedStr( - 'messageRequestPendingDescription' - ).toString(); - await device1.onIOS().waitForTextElementToBePresent({ - strategy: 'accessibility id', - selector: 'Control message', - text: messageRequestPendingDescription, + await test.step(TestSteps.SEND.MESSAGE(bob.userName, testCommunityName), async () => { + await device2.sendMessage(message); + await device2.navigateBack(); }); - await device1.onAndroid().waitForTextElementToBePresent({ - strategy: 'id', - selector: 'network.loki.messenger.qa:id/textSendAfterApproval', - text: messageRequestPendingDescription, + await test.step(TestSteps.SEND.MESSAGE(alice.userName, bob.userName), async () => { + await device1.clickOnElementAll(new CommunityMessageAuthor(device1, message)); + await device1.clickOnElementAll(new UPMMessageButton(device1)); + await device1.clickOnElementAll(new ConversationHeaderName(device1, bob.userName)); + await device1.waitForTextElementToBePresent(new MessageRequestPendingDescription(device1)); + await device1.sendMessage(messageRequestMessage); }); - await device1.sendMessage('Howdy partner'); - await device2.clickOnElementAll(new MessageRequestsBanner(device2)); - // Bob clicks on request conversation item - await device2.clickOnByAccessibilityID('Message request'); - await device2.waitForTextElementToBePresent(new ConversationHeaderName(device2, alice.userName)); - const messageRequestsAcceptDescription = englishStrippedStr( - 'messageRequestsAcceptDescription' - ).toString(); - await device2.onIOS().waitForControlMessageToBePresent(messageRequestsAcceptDescription); - await device2.onAndroid().waitForTextElementToBePresent({ - strategy: 'id', - selector: 'network.loki.messenger.qa:id/sendAcceptsTextView', - text: messageRequestsAcceptDescription, + await test.step(`${bob.userName} accepts message request from ${alice.userName}`, async () => { + await device2.clickOnElementAll(new MessageRequestsBanner(device2)); + // Bob clicks on request conversation item + await device2.clickOnByAccessibilityID('Message request'); + await device2.waitForTextElementToBePresent( + new ConversationHeaderName(device2, alice.userName) + ); + await device2.waitForTextElementToBePresent(new MessageBody(device2, messageRequestMessage)); + await device2.waitForTextElementToBePresent(new MessageRequestAcceptDescription(device2)); }); - // Send message from Bob to Alice - const acceptMessage = 'Howdy back'; - await device2.sendMessage(acceptMessage); - await device1.waitForTextElementToBePresent(new MessageBody(device1, acceptMessage)); - await closeApp(device1, device2); + await test.step(TestSteps.SEND.MESSAGE(bob.userName, alice.userName), async () => { + await device2.sendMessage(messageRequestReply); + await device1.waitForTextElementToBePresent(new MessageBody(device1, messageRequestReply)); + }); + await test.step(TestSteps.SETUP.CLOSE_APP, async () => { + await closeApp(device1, device2); + }); } diff --git a/run/test/specs/locators/conversation.ts b/run/test/specs/locators/conversation.ts index 862d47ec3..1b5249d7b 100644 --- a/run/test/specs/locators/conversation.ts +++ b/run/test/specs/locators/conversation.ts @@ -1,6 +1,7 @@ import type { DeviceWrapper } from '../../../types/DeviceWrapper'; import { testCommunityName } from '../../../constants/community'; +import { englishStrippedStr } from '../../../localizer/englishStrippedStr'; import { StrategyExtractionObj } from '../../../types/testing'; import { getAppDisplayName } from '../utils/devnet'; import { LocatorsInterface } from './index'; @@ -629,3 +630,88 @@ export class MessageLengthOkayButton extends LocatorsInterface { } } } + +export class CommunityMessageAuthor extends LocatorsInterface { + public text: string; + constructor(device: DeviceWrapper, text: string) { + super(device); + this.text = text; + } + public build(): StrategyExtractionObj { + switch (this.platform) { + case 'android': + // Identify the profile picture of a message with a specific text + return { + strategy: 'xpath', + selector: `//android.view.ViewGroup[@resource-id='network.loki.messenger.qa:id/mainContainer'][.//android.widget.TextView[contains(@text,'${this.text}')]]//androidx.compose.ui.platform.ComposeView[@resource-id='network.loki.messenger.qa:id/profilePictureView']`, + } as const; + case 'ios': + // Identify the display name of a blinded sender of a message with a specific text + return { + strategy: 'xpath', + selector: `//XCUIElementTypeCell[.//XCUIElementTypeOther[@name='Message body' and contains(@label,'${this.text}')]]//XCUIElementTypeStaticText[contains(@value,'(15')]`, + } as const; + } + } +} + +export class UPMMessageButton extends LocatorsInterface { + public build(): StrategyExtractionObj { + switch (this.platform) { + case 'android': + return { + strategy: 'xpath', + selector: `//android.widget.TextView[@text="Message"]/parent::android.view.View`, + } as const; + case 'ios': + return { + strategy: 'accessibility id', + selector: 'Message', + } as const; + } + } +} + +export class MessageRequestPendingDescription extends LocatorsInterface { + public build() { + const messageRequestPendingDescription = englishStrippedStr( + 'messageRequestPendingDescription' + ).toString(); + switch (this.platform) { + case 'android': + return { + strategy: 'id', + selector: 'network.loki.messenger.qa:id/textSendAfterApproval', + text: messageRequestPendingDescription, + } as const; + case 'ios': + return { + strategy: 'accessibility id', + selector: 'Control message', + text: messageRequestPendingDescription, + } as const; + } + } +} + +export class MessageRequestAcceptDescription extends LocatorsInterface { + public build() { + const messageRequestsAcceptDescription = englishStrippedStr( + 'messageRequestsAcceptDescription' + ).toString(); + switch (this.platform) { + case 'android': + return { + strategy: 'id', + selector: 'network.loki.messenger.qa:id/sendAcceptsTextView', + text: messageRequestsAcceptDescription, + } as const; + case 'ios': + return { + strategy: 'accessibility id', + selector: 'Control message', + text: messageRequestsAcceptDescription, + } as const; + } + } +} diff --git a/run/test/specs/locators/settings.ts b/run/test/specs/locators/settings.ts index a7093e2f7..02ae1349b 100644 --- a/run/test/specs/locators/settings.ts +++ b/run/test/specs/locators/settings.ts @@ -112,6 +112,23 @@ export class RecoveryPhraseContainer extends LocatorsInterface { } } +export class CommunityMessageRequestSwitch extends LocatorsInterface { + public build(): StrategyExtractionObj { + switch (this.platform) { + case 'android': + return { + strategy: '-android uiautomator', + selector: 'new UiSelector().text("Community Message Requests")', + } as const; + case 'ios': + return { + strategy: 'accessibility id', + selector: 'Community Message Requests', + } as const; + } + } +} + export class SaveProfilePictureButton extends LocatorsInterface { public build(): StrategyExtractionObj { switch (this.platform) { diff --git a/run/types/allure.ts b/run/types/allure.ts index a7ab333a9..c9d5cb235 100644 --- a/run/types/allure.ts +++ b/run/types/allure.ts @@ -27,7 +27,7 @@ export type AllureSuiteConfig = | { parent: 'Network Page' } | { parent: 'New Conversation'; suite: 'Invite a Friend' | 'Join Community' | 'New Message' } | { parent: 'Sending Messages'; suite: 'Emoji reacts' | 'Message types' | 'Rules' } - | { parent: 'Settings'; suite: 'App Disguise' } + | { parent: 'Settings'; suite: 'App Disguise' | 'Community Message Requests' } | { parent: 'User Actions'; suite: From fab8b1ccc8e7997230e808cf38396e74f0290250 Mon Sep 17 00:00:00 2001 From: Miklos Mandoki Date: Fri, 7 Nov 2025 11:00:45 +1100 Subject: [PATCH 29/29] fix: network page refresh test actually waits and refreshes --- run/test/specs/community_requests_off.spec.ts | 16 ++++++---- run/test/specs/locators/conversation.ts | 4 --- run/test/specs/locators/network_page.ts | 9 ++++++ run/test/specs/locators/onboarding.ts | 4 +-- .../specs/network_page_refresh_page.spec.ts | 26 +++++++-------- run/test/specs/network_page_values.spec.ts | 14 +------- run/test/specs/utils/network_api.ts | 32 +++++++++++++++++++ run/types/DeviceWrapper.ts | 17 ++-------- run/types/testing.ts | 2 -- 9 files changed, 68 insertions(+), 56 deletions(-) create mode 100644 run/test/specs/utils/network_api.ts diff --git a/run/test/specs/community_requests_off.spec.ts b/run/test/specs/community_requests_off.spec.ts index 075d5a55d..89057bdf3 100644 --- a/run/test/specs/community_requests_off.spec.ts +++ b/run/test/specs/community_requests_off.spec.ts @@ -41,13 +41,15 @@ async function blindedMessageRequests(platform: SupportedPlatformsType, testInfo }); await device1.clickOnElementAll(new CommunityMessageAuthor(device1, message)); await test.step(`Verify the 'Message' button in the User Profile Modal is disabled`, async () => { - const messageButton = await device1.waitForTextElementToBePresent(new UPMMessageButton(device1)); - const attr = await device1.getAttribute('enabled', messageButton.ELEMENT); - if (attr !== 'false') { - device1.log(`Message button attribute is 'enabled = ${attr}'`); - throw new Error(`Message button should be disabled but it is not`); - } -}); + const messageButton = await device1.waitForTextElementToBePresent( + new UPMMessageButton(device1) + ); + const attr = await device1.getAttribute('enabled', messageButton.ELEMENT); + if (attr !== 'false') { + device1.log(`Message button attribute is 'enabled = ${attr}'`); + throw new Error(`Message button should be disabled but it is not`); + } + }); await test.step(TestSteps.SETUP.CLOSE_APP, async () => { await closeApp(device1, device2); }); diff --git a/run/test/specs/locators/conversation.ts b/run/test/specs/locators/conversation.ts index 1b5249d7b..66cfce859 100644 --- a/run/test/specs/locators/conversation.ts +++ b/run/test/specs/locators/conversation.ts @@ -239,10 +239,6 @@ export class NotificationsModalButton extends LocatorsInterface { public build() { switch (this.platform) { case 'android': - return { - strategy: 'accessibility id', - selector: 'Notifications', - } as const; case 'ios': return { strategy: 'accessibility id', diff --git a/run/test/specs/locators/network_page.ts b/run/test/specs/locators/network_page.ts index 148e81d30..6b577f89e 100644 --- a/run/test/specs/locators/network_page.ts +++ b/run/test/specs/locators/network_page.ts @@ -1,4 +1,5 @@ import { LocatorsInterface } from '.'; +import { englishStrippedStr } from '../../../localizer/englishStrippedStr'; import { DeviceWrapper } from '../../../types/DeviceWrapper'; export class SessionNetworkMenuItem extends LocatorsInterface { @@ -53,17 +54,25 @@ export class SessionNetworkLearnMoreStaking extends LocatorsInterface { } export class LastUpdatedTimeStamp extends LocatorsInterface { + private expectedText: string; + + constructor(device: DeviceWrapper, relative_time: string) { + super(device); + this.expectedText = englishStrippedStr('updated').withArgs({ relative_time }).toString(); + } public build() { switch (this.platform) { case 'android': return { strategy: 'id', selector: 'Last updated timestamp', + text: this.expectedText, } as const; case 'ios': return { strategy: 'accessibility id', selector: 'Last updated timestamp', + text: this.expectedText, } as const; } } diff --git a/run/test/specs/locators/onboarding.ts b/run/test/specs/locators/onboarding.ts index d54798c5b..869e16c8b 100644 --- a/run/test/specs/locators/onboarding.ts +++ b/run/test/specs/locators/onboarding.ts @@ -105,7 +105,7 @@ export class TermsOfServiceButton extends LocatorsInterface { case 'android': return { strategy: 'id', - selector: 'Terms of Service', // will be Privacy policy button with Pro Settings + selector: 'Terms of Service', // will be Terms of service *button* past 1.29.0 } as const; case 'ios': return { @@ -122,7 +122,7 @@ export class PrivacyPolicyButton extends LocatorsInterface { case 'android': return { strategy: 'id', - selector: 'Privacy Policy', // will be Privacy policy button with Pro Settings + selector: 'Privacy Policy', // will be Privacy policy *button* past 1.29.0 } as const; case 'ios': return { diff --git a/run/test/specs/network_page_refresh_page.spec.ts b/run/test/specs/network_page_refresh_page.spec.ts index 0340fede0..2bce3961c 100644 --- a/run/test/specs/network_page_refresh_page.spec.ts +++ b/run/test/specs/network_page_refresh_page.spec.ts @@ -4,11 +4,12 @@ import { bothPlatformsIt } from '../../types/sessionIt'; import { USERNAME } from '../../types/testing'; import { LastUpdatedTimeStamp, SessionNetworkMenuItem } from './locators/network_page'; import { UserSettings } from './locators/settings'; +import { sleepFor } from './utils'; import { newUser } from './utils/create_account'; -import { openAppOnPlatformSingleDevice, SupportedPlatformsType } from './utils/open_app'; +import { closeApp, openAppOnPlatformSingleDevice, SupportedPlatformsType } from './utils/open_app'; bothPlatformsIt({ - title: 'Refresh network page', + title: 'Network page refresh', risk: 'low', testCb: refreshNetworkPage, countOfDevicesNeeded: 1, @@ -19,23 +20,20 @@ bothPlatformsIt({ }); async function refreshNetworkPage(platform: SupportedPlatformsType, testInfo: TestInfo) { - const { device } = await openAppOnPlatformSingleDevice(platform, testInfo); + const zeroMinutesAgo = '0m'; + const oneMinuteAgo = '1m'; - const lastUpdatedExpected = 'Last updated 0m ago'; + const { device } = await openAppOnPlatformSingleDevice(platform, testInfo); await newUser(device, USERNAME.ALICE, { saveUserData: false }); await device.clickOnElementAll(new UserSettings(device)); await device.onAndroid().scrollDown(); await device.clickOnElementAll(new SessionNetworkMenuItem(device)); - // Check for loading states - await device.waitForLoadingMedia(); + await device.waitForLoadingMedia(); // Wait for fetch to complete + await device.waitForTextElementToBePresent(new LastUpdatedTimeStamp(device, zeroMinutesAgo)); + await sleepFor(65_000); // 60+5 seconds to ensure the last updated value changes + await device.waitForTextElementToBePresent(new LastUpdatedTimeStamp(device, oneMinuteAgo)); await device.pullToRefresh(); await device.waitForLoadingMedia(); - await device.onAndroid().scrollDown(); - const timeStampEl = await device.waitForTextElementToBePresent(new LastUpdatedTimeStamp(device)); - const lastUpdatedActual = await device.getTextFromElement(timeStampEl); - if (lastUpdatedActual !== lastUpdatedExpected) { - throw new Error( - `The retrieved last updated time does not match the expected. The retrieved last updated time is ${lastUpdatedActual}` - ); - } + await device.waitForTextElementToBePresent(new LastUpdatedTimeStamp(device, zeroMinutesAgo)); + await closeApp(device); } diff --git a/run/test/specs/network_page_values.spec.ts b/run/test/specs/network_page_values.spec.ts index 10721ee71..e8558cb5c 100644 --- a/run/test/specs/network_page_values.spec.ts +++ b/run/test/specs/network_page_values.spec.ts @@ -11,6 +11,7 @@ import { } from './locators/network_page'; import { UserSettings } from './locators/settings'; import { newUser } from './utils/create_account'; +import { validateNetworkData } from './utils/network_api'; import { closeApp, openAppOnPlatformSingleDevice, SupportedPlatformsType } from './utils/open_app'; bothPlatformsIt({ @@ -25,19 +26,6 @@ bothPlatformsIt({ 'Verifies that the Session Network Page displays the values fetched from the network API correctly.', }); -function validateNetworkData(data: any): asserts data is { - price: { usd: number; usd_market_cap: number }; - token: { staking_reward_pool: number }; -} { - if ( - typeof data?.price?.usd !== 'number' || - typeof data?.token?.staking_reward_pool !== 'number' || - typeof data?.price?.usd_market_cap !== 'number' - ) { - throw new Error('Network API response missing or invalid numeric fields'); - } -} - async function networkPageValues(platform: SupportedPlatformsType, testInfo: TestInfo) { let data: { price: { usd: number; usd_market_cap: number }; diff --git a/run/test/specs/utils/network_api.ts b/run/test/specs/utils/network_api.ts new file mode 100644 index 000000000..598baa705 --- /dev/null +++ b/run/test/specs/utils/network_api.ts @@ -0,0 +1,32 @@ +import { isFinite } from 'lodash'; + +export type NetworkData = { + price: { usd: number; usd_market_cap: number }; + token: { staking_reward_pool: number }; +}; + +function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} + +function isPositiveFiniteNumber(n: unknown): n is number { + return typeof n === 'number' && isFinite(n) && n > 0; +} + +export function validateNetworkData(data: unknown): asserts data is NetworkData { + if (!isObject(data)) { + throw new Error('Invalid network API response: not an object'); + } + + if (!isObject(data.price) || !isObject(data.token)) { + throw new Error('Invalid network API response: missing price or token'); + } + + if ( + !isPositiveFiniteNumber(data.price.usd) || + !isPositiveFiniteNumber(data.price.usd_market_cap) || + !isPositiveFiniteNumber(data.token.staking_reward_pool) + ) { + throw new Error('Invalid network API response: numeric fields must be positive and finite'); + } +} diff --git a/run/types/DeviceWrapper.ts b/run/types/DeviceWrapper.ts index 160b50869..6b5dd0583 100644 --- a/run/types/DeviceWrapper.ts +++ b/run/types/DeviceWrapper.ts @@ -2310,20 +2310,9 @@ export class DeviceWrapper { this.info('Scroll button not found, continuing'); } } - public async pullToRefresh() { - if (this.isAndroid()) { - await this.pressCoordinates( - InteractionPoints.NetworkPageAndroid.x, - InteractionPoints.NetworkPageAndroid.y, - true - ); - } else { - await this.pressCoordinates( - InteractionPoints.NetworkPageIOS.x, - InteractionPoints.NetworkPageIOS.y, - true - ); - } + public async pullToRefresh(): Promise { + const { width, height } = await this.getWindowRect(); + await this.scroll({ x: width / 2, y: height * 0.15 }, { x: width / 2, y: height * 0.55 }, 200); } public async navigateBack(newAndroid: boolean = true) { diff --git a/run/types/testing.ts b/run/types/testing.ts index 52b9823a8..b1e78dda5 100644 --- a/run/types/testing.ts +++ b/run/types/testing.ts @@ -52,8 +52,6 @@ export const InteractionPoints: Record = { GifButtonKeyboardClosed: { x: 36, y: 689 }, DocumentKeyboardOpen: { x: 36, y: 476 }, DocumentKeyboardClosed: { x: 36, y: 740 }, - NetworkPageAndroid: { x: 880, y: 1150 }, - NetworkPageIOS: { x: 308, y: 220 }, BackToSession: { x: 42, y: 42 }, };