From 9b2f0e1f47c1d0765cd7931f4708cbbafba6aae9 Mon Sep 17 00:00:00 2001 From: Niklas Berglund Date: Wed, 24 Apr 2024 14:56:31 +0200 Subject: [PATCH] Move iOS end to end tests to staging environment --- .../actions/ios-end-to-end-tests/action.yml | 38 ++++- ...os-end-to-end-tests-settings-migration.yml | 4 + .github/workflows/ios-end-to-end-tests.yml | 22 ++- ios/Configurations/Api.xcconfig.template | 2 +- ios/Configurations/UITests.xcconfig.template | 10 +- ios/MullvadVPN.xcodeproj/project.pbxproj | 8 + .../xcschemes/MullvadVPNUITests.xcscheme | 4 +- .../Classes/AccessbilityIdentifier.swift | 10 ++ .../Cells/SwitchCellContentView.swift | 2 +- ios/MullvadVPN/Supporting Files/Info.plist | 5 + .../ProblemReportReviewViewController.swift | 5 + ...mReportViewController+ViewManagement.swift | 3 +- .../RelayFilter/RelayFilterChipView.swift | 4 + .../Tunnel/TunnelControlView.swift | 2 + .../VPNSettings/CustomDNSViewController.swift | 1 + ios/MullvadVPN/Views/StatusActivityView.swift | 3 + ios/MullvadVPN/Views/StatusImageView.swift | 17 +++ ios/MullvadVPNUITests/AccountTests.swift | 70 ++++++--- ios/MullvadVPNUITests/ConnectivityTests.swift | 54 ++++++- ios/MullvadVPNUITests/CustomListsTests.swift | 13 -- ios/MullvadVPNUITests/Info.plist | 8 +- .../Networking/MullvadAPIWrapper.swift | 141 ++++++++++++++---- .../Networking/Networking.swift | 46 ++++-- .../Networking/PartnerAPIClient.swift | 123 +++++++++++++++ .../Pages/APIAccessPage.swift | 2 +- .../Pages/AccountDeletionPage.swift | 9 +- ios/MullvadVPNUITests/Pages/AccountPage.swift | 12 +- .../Pages/AddAccessMethodPage.swift | 6 +- ios/MullvadVPNUITests/Pages/Alert.swift | 2 +- ios/MullvadVPNUITests/Pages/AppLogsPage.swift | 36 +++++ .../Pages/ChangeLogAlert.swift | 2 +- .../Pages/CustomListPage.swift | 4 +- .../Pages/DNSSettingsPage.swift | 5 +- .../Pages/DeviceManagementPage.swift | 5 +- .../Pages/EditAccessMethodPage.swift | 3 +- .../Pages/EditCustomListLocationsPage.swift | 2 +- ios/MullvadVPNUITests/Pages/HeaderBar.swift | 11 +- .../Pages/ListCustomListsPage.swift | 2 +- ios/MullvadVPNUITests/Pages/LoginPage.swift | 49 ++++-- .../Pages/OutOfTimePage.swift | 2 +- ios/MullvadVPNUITests/Pages/Page.swift | 13 +- .../Pages/ProblemReportPage.swift | 5 +- .../Pages/ProblemReportSubmittedPage.swift | 2 +- .../Pages/RevokedDevicePage.swift | 2 +- .../Pages/SelectLocationPage.swift | 16 +- .../Pages/SettingsPage.swift | 2 +- .../Pages/TermsOfServicePage.swift | 2 +- .../Pages/TunnelControlPage.swift | 33 +++- .../Pages/VPNSettingsPage.swift | 2 +- ios/MullvadVPNUITests/Pages/WelcomePage.swift | 2 +- ios/MullvadVPNUITests/README.md | 3 + ios/MullvadVPNUITests/RelayTests.swift | 71 ++++++--- .../SettingsMigrationTests.swift | 12 ++ .../Test base classes/BaseUITestCase.swift | 141 ++++++++++++++++-- .../LoggedInWithTimeUITestCase.swift | 22 +++ 55 files changed, 877 insertions(+), 198 deletions(-) create mode 100644 ios/MullvadVPNUITests/Networking/PartnerAPIClient.swift create mode 100644 ios/MullvadVPNUITests/Pages/AppLogsPage.swift diff --git a/.github/actions/ios-end-to-end-tests/action.yml b/.github/actions/ios-end-to-end-tests/action.yml index 9d771d151c5a..fa2d198a44a5 100644 --- a/.github/actions/ios-end-to-end-tests/action.yml +++ b/.github/actions/ios-end-to-end-tests/action.yml @@ -19,10 +19,22 @@ inputs: xcode_test_plan: description: 'Xcode Test Plan to run' required: true + partner_api_token: + description: 'Partner API Token' + required: true + test_name: + description: 'Test case/suite name. Will run all tests in the test plan if not provided.' + required: false runs: using: 'composite' steps: + - name: Make sure app is not installed + run: ios-deploy --id $IOS_TEST_DEVICE_UDID --uninstall_only --bundle_id net.mullvad.MullvadVPN + shell: bash + env: + IOS_TEST_DEVICE_UDID: ${{ inputs.test_device_udid }} + - name: Configure Xcode project run: | for file in *.xcconfig.template ; do cp $file ${file//.template/} ; done @@ -34,8 +46,12 @@ runs: sed -i "" \ "/TEST_DEVICE_IDENTIFIER_UUID =/ s/= .*/= $TEST_DEVICE_IDENTIFIER_UUID/" \ UITests.xcconfig - echo -e "\nHAS_TIME_ACCOUNT_NUMBER = $HAS_TIME_ACCOUNT_NUMBER" >> UITests.xcconfig - echo "NO_TIME_ACCOUNT_NUMBER = $NO_TIME_ACCOUNT_NUMBER" >> UITests.xcconfig + sed -i "" \ + "/PARTNER_API_TOKEN =/ s#= .*#= $PARTNER_API_TOKEN#" \ + UITests.xcconfig + sed -i "" \ + "/ATTACH_APP_LOGS_ON_FAILURE =/ s#= .*#= 1#" \ + UITests.xcconfig shell: bash working-directory: ios/Configurations env: @@ -43,17 +59,31 @@ runs: TEST_DEVICE_IDENTIFIER_UUID: ${{ inputs.test_device_identifier_uuid }} HAS_TIME_ACCOUNT_NUMBER: ${{ inputs.has_time_account_number }} NO_TIME_ACCOUNT_NUMBER: ${{ inputs.no_time_account_number }} + PARTNER_API_TOKEN: ${{ inputs.partner_api_token }} - name: Run end-to-end-tests run: | + if [ -n "$TEST_NAME" ]; then + TEST_NAME_ARGUMENT=" -only-testing $TEST_NAME" + else + TEST_NAME_ARGUMENT="" + fi set -o pipefail && env NSUnbufferedIO=YES xcodebuild \ -project MullvadVPN.xcodeproj \ -scheme MullvadVPNUITests \ - -testPlan $XCODE_TEST_PLAN \ + -testPlan $XCODE_TEST_PLAN $TEST_NAME_ARGUMENT \ + -resultBundlePath xcode-test-report \ -destination "platform=iOS,id=$TEST_DEVICE_UDID" \ - clean test 2>&1 | xcbeautify --report junit --report-path test-report + clean test 2>&1 | xcbeautify --report junit --report-path junit-test-report shell: bash working-directory: ios/ env: XCODE_TEST_PLAN: ${{ inputs.xcode_test_plan }} TEST_DEVICE_UDID: ${{ inputs.test_device_udid }} + TEST_NAME: ${{ inputs.test_name }} + + - name: Uninstall app if still installed + run: ios-deploy --id $IOS_TEST_DEVICE_UDID --uninstall_only --bundle_id net.mullvad.MullvadVPN + shell: bash + env: + IOS_TEST_DEVICE_UDID: ${{ inputs.test_device_udid }} diff --git a/.github/workflows/ios-end-to-end-tests-settings-migration.yml b/.github/workflows/ios-end-to-end-tests-settings-migration.yml index a5a27fd6454e..2ff8e1a2bad2 100644 --- a/.github/workflows/ios-end-to-end-tests-settings-migration.yml +++ b/.github/workflows/ios-end-to-end-tests-settings-migration.yml @@ -42,6 +42,7 @@ jobs: no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} xcode_test_plan: 'MullvadVPNUITestsChangeDNSSettings' + partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }} - name: Store test report for changing DNS settings uses: actions/upload-artifact@v4 @@ -62,6 +63,7 @@ jobs: has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }} no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} + partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }} xcode_test_plan: 'MullvadVPNUITestsVerifyDNSSettingsChanged' - name: Store test report for verifying DNS settings @@ -85,6 +87,7 @@ jobs: has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }} no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} + partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }} xcode_test_plan: 'MullvadVPNUITestsChangeSettings' - name: Store test report for changing all settings @@ -106,6 +109,7 @@ jobs: has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }} no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} + partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }} xcode_test_plan: 'MullvadVPNUITestsVerifySettingsChanged' - name: Store test report for verifying all other settings diff --git a/.github/workflows/ios-end-to-end-tests.yml b/.github/workflows/ios-end-to-end-tests.yml index 64d494b8ab29..8d1c8178d072 100644 --- a/.github/workflows/ios-end-to-end-tests.yml +++ b/.github/workflows/ios-end-to-end-tests.yml @@ -14,6 +14,12 @@ on: - .github/workflows/ios-end-to-end-tests.yml - ios/** workflow_dispatch: + inputs: + # Optionally specify a test case or suite to run. + # Must be in the format MullvadVPNUITest// where test case name is optional. + test_name: + description: 'Only run test case/suite' + required: false schedule: # At midnight every day. # Notifications for scheduled workflows are sent to the user who last modified the cron @@ -26,6 +32,7 @@ jobs: if: github.event.pull_request.merged || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' name: End to end tests runs-on: [self-hosted, macOS, ios-test] + timeout-minutes: 60 steps: - name: Configure Rust uses: actions-rs/toolchain@v1 @@ -50,11 +57,13 @@ jobs: uses: ./.github/actions/ios-end-to-end-tests with: xcode_test_plan: ${{ env.XCODE_TEST_PLAN }} + test_name: ${{ github.event.inputs.test_name }} ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }} test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }} has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }} no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} + partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }} - name: Comment PR on test failure if: failure() && github.event_name == 'pull_request' @@ -76,5 +85,14 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: test-report - path: ios/test-report/junit.xml + name: test-results + path: | + ios/junit-test-report/junit.xml + ios/xcode-test-report.xcresult + + - name: Store app log artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: app-logs + path: ios/xcode-test-report/**/app-log-*.log diff --git a/ios/Configurations/Api.xcconfig.template b/ios/Configurations/Api.xcconfig.template index 99d1fc1fc13a..ff39567be69c 100644 --- a/ios/Configurations/Api.xcconfig.template +++ b/ios/Configurations/Api.xcconfig.template @@ -6,4 +6,4 @@ API_HOST_NAME[config=Staging] = api.stagemole.eu API_ENDPOINT[config=Debug] = 45.83.223.196:443 API_ENDPOINT[config=Release] = 45.83.223.196:443 API_ENDPOINT[config=MockRelease] = 45.83.223.196:443 -API_ENDPOINT[config=Staging] = 85.203.53.95:443 +API_ENDPOINT[config=Staging] = 185.217.116.129:443 diff --git a/ios/Configurations/UITests.xcconfig.template b/ios/Configurations/UITests.xcconfig.template index e39224eedfe0..030fe46d3129 100644 --- a/ios/Configurations/UITests.xcconfig.template +++ b/ios/Configurations/UITests.xcconfig.template @@ -6,10 +6,12 @@ IOS_DEVICE_PIN_CODE = // UUID to identify test runs. Should be unique per test device. Generate with for example uuidgen on macOS. TEST_DEVICE_IDENTIFIER_UUID = +// Base64 encoded token for the partner API. Will only be used if account numbers are not configured. +PARTNER_API_TOKEN = + // Mullvad accounts used by UI tests HAS_TIME_ACCOUNT_NUMBER[config=Debug] = HAS_TIME_ACCOUNT_NUMBER[config=Staging] = -FIVE_WIREGUARD_KEYS_ACCOUNT_NUMBER = // Ad serving domain used when testing ad blocking. Note that we are assuming there's an HTTP server running on the host. AD_SERVING_DOMAIN = vpnlist.to @@ -19,3 +21,9 @@ SHOULD_BE_REACHABLE_DOMAIN = mullvad.net // Base URL for the firewall API, Note that // will be treated as a comment, therefor you need to insert a ${} between the slashes for example http:/${}/8.8.8.8 FIREWALL_API_BASE_URL = http:/${}/8.8.8.8 + +// URL for Mullvad provided JSON data with information about the connection. https://am.i.mullvad.net/json for production, https://am.i.stagemole.eu/json for staging. +AM_I_JSON_URL = https:/${}/am.i.stagemole.eu/json + +// Specify whether app logs should be extracted and attached to test report for failing tests +ATTACH_APP_LOGS_ON_FAILURE = 0 diff --git a/ios/MullvadVPN.xcodeproj/project.pbxproj b/ios/MullvadVPN.xcodeproj/project.pbxproj index a5e202fa9a0e..3ec4efe6ed60 100644 --- a/ios/MullvadVPN.xcodeproj/project.pbxproj +++ b/ios/MullvadVPN.xcodeproj/project.pbxproj @@ -620,6 +620,7 @@ 850201DB2B503D7700EF8C96 /* RelayTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 850201DA2B503D7700EF8C96 /* RelayTests.swift */; }; 850201DD2B503D8C00EF8C96 /* SelectLocationPage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 850201DC2B503D8C00EF8C96 /* SelectLocationPage.swift */; }; 850201DF2B5040A500EF8C96 /* TunnelControlPage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 850201DE2B5040A500EF8C96 /* TunnelControlPage.swift */; }; + 85021CAE2BDBC4290098B400 /* AppLogsPage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 85021CAD2BDBC4290098B400 /* AppLogsPage.swift */; }; 85139B2D2B84B4A700734217 /* OutOfTimePage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 85139B2C2B84B4A700734217 /* OutOfTimePage.swift */; }; 852969282B4D9C1F007EAD4C /* AccountTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 852969272B4D9C1F007EAD4C /* AccountTests.swift */; }; 852969332B4E9232007EAD4C /* Page.swift in Sources */ = {isa = PBXBuildFile; fileRef = 852969322B4E9232007EAD4C /* Page.swift */; }; @@ -647,6 +648,7 @@ 8556EB542B9A1D7100D26DD4 /* BridgingHeader.h in Headers */ = {isa = PBXBuildFile; fileRef = 8556EB532B9A1D7100D26DD4 /* BridgingHeader.h */; }; 8556EB562B9B0AC500D26DD4 /* RevokedDevicePage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8556EB552B9B0AC500D26DD4 /* RevokedDevicePage.swift */; }; 855D9F5B2B63E56B00D7C64D /* ProblemReportPage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 855D9F5A2B63E56B00D7C64D /* ProblemReportPage.swift */; }; + 856952DC2BD2922A008C1F84 /* PartnerAPIClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 856952DB2BD2922A008C1F84 /* PartnerAPIClient.swift */; }; 856952E22BD6B04C008C1F84 /* XCUIElement+Extensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 856952E12BD6B04C008C1F84 /* XCUIElement+Extensions.swift */; }; 8585CBE32BC684180015B6A4 /* EditAccessMethodPage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8585CBE22BC684180015B6A4 /* EditAccessMethodPage.swift */; }; 8587A05D2B84D43100152938 /* ChangeLogAlert.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8587A05C2B84D43100152938 /* ChangeLogAlert.swift */; }; @@ -1950,6 +1952,7 @@ 850201DC2B503D8C00EF8C96 /* SelectLocationPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SelectLocationPage.swift; sourceTree = ""; }; 850201DE2B5040A500EF8C96 /* TunnelControlPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TunnelControlPage.swift; sourceTree = ""; }; 850201E22B51A93C00EF8C96 /* SettingsPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsPage.swift; sourceTree = ""; }; + 85021CAD2BDBC4290098B400 /* AppLogsPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppLogsPage.swift; sourceTree = ""; }; 85139B2C2B84B4A700734217 /* OutOfTimePage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OutOfTimePage.swift; sourceTree = ""; }; 852969252B4D9C1F007EAD4C /* MullvadVPNUITests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = MullvadVPNUITests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; 852969272B4D9C1F007EAD4C /* AccountTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AccountTests.swift; sourceTree = ""; }; @@ -1981,6 +1984,7 @@ 8556EB532B9A1D7100D26DD4 /* BridgingHeader.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BridgingHeader.h; sourceTree = ""; }; 8556EB552B9B0AC500D26DD4 /* RevokedDevicePage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RevokedDevicePage.swift; sourceTree = ""; }; 855D9F5A2B63E56B00D7C64D /* ProblemReportPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProblemReportPage.swift; sourceTree = ""; }; + 856952DB2BD2922A008C1F84 /* PartnerAPIClient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PartnerAPIClient.swift; sourceTree = ""; }; 856952E12BD6B04C008C1F84 /* XCUIElement+Extensions.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "XCUIElement+Extensions.swift"; sourceTree = ""; }; 8585CBE22BC684180015B6A4 /* EditAccessMethodPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EditAccessMethodPage.swift; sourceTree = ""; }; 8587A05C2B84D43100152938 /* ChangeLogAlert.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChangeLogAlert.swift; sourceTree = ""; }; @@ -3928,6 +3932,7 @@ 852D054C2BC3DE3A008578D2 /* APIAccessPage.swift */, 852D054E2BC43DF7008578D2 /* AddAccessMethodPage.swift */, 8585CBE22BC684180015B6A4 /* EditAccessMethodPage.swift */, + 85021CAD2BDBC4290098B400 /* AppLogsPage.swift */, ); path = Pages; sourceTree = ""; @@ -3939,6 +3944,7 @@ 85557B0F2B59215F00795FE1 /* FirewallRule.swift */, 85557B132B5983CF00795FE1 /* MullvadAPIWrapper.swift */, 85E3BDE42B70E18C00FA71FD /* Networking.swift */, + 856952DB2BD2922A008C1F84 /* PartnerAPIClient.swift */, ); path = Networking; sourceTree = ""; @@ -6066,6 +6072,7 @@ 8542F7532BCFBD050035C042 /* SelectLocationFilterPage.swift in Sources */, 850201DD2B503D8C00EF8C96 /* SelectLocationPage.swift in Sources */, 85D039982BA4711800940E7F /* SettingsMigrationTests.swift in Sources */, + 85021CAE2BDBC4290098B400 /* AppLogsPage.swift in Sources */, 850201DB2B503D7700EF8C96 /* RelayTests.swift in Sources */, 852D054D2BC3DE3A008578D2 /* APIAccessPage.swift in Sources */, 85139B2D2B84B4A700734217 /* OutOfTimePage.swift in Sources */, @@ -6084,6 +6091,7 @@ 852969282B4D9C1F007EAD4C /* AccountTests.swift in Sources */, 8587A05D2B84D43100152938 /* ChangeLogAlert.swift in Sources */, 85FB5A102B6960A30015DCED /* AccountDeletionPage.swift in Sources */, + 856952DC2BD2922A008C1F84 /* PartnerAPIClient.swift in Sources */, 85557B162B5ABBBE00795FE1 /* XCUIElementQuery+Extensions.swift in Sources */, 855D9F5B2B63E56B00D7C64D /* ProblemReportPage.swift in Sources */, 8529693A2B4F0238007EAD4C /* TermsOfServicePage.swift in Sources */, diff --git a/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme b/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme index a206bbdc7147..3258ef6f4722 100644 --- a/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme +++ b/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme @@ -7,7 +7,7 @@ buildImplicitDependencies = "YES"> @@ -46,7 +46,7 @@ NSExceptionDomains + 185.217.116.129 + + NSExceptionAllowsInsecureHTTPLoads + + 127.0.0.1 NSExceptionAllowsInsecureHTTPLoads diff --git a/ios/MullvadVPN/View controllers/ProblemReport/ProblemReportReviewViewController.swift b/ios/MullvadVPN/View controllers/ProblemReport/ProblemReportReviewViewController.swift index b999eb32b458..ce513e8ec432 100644 --- a/ios/MullvadVPN/View controllers/ProblemReport/ProblemReportReviewViewController.swift +++ b/ios/MullvadVPN/View controllers/ProblemReport/ProblemReportReviewViewController.swift @@ -24,6 +24,8 @@ class ProblemReportReviewViewController: UIViewController { override func viewDidLoad() { super.viewDidLoad() + view.accessibilityIdentifier = .appLogsView + navigationItem.title = NSLocalizedString( "NAVIGATION_TITLE", tableName: "ProblemReportReview", @@ -37,6 +39,7 @@ class ProblemReportReviewViewController: UIViewController { self?.dismiss(animated: true) }) ) + navigationItem.rightBarButtonItem?.accessibilityIdentifier = .appLogsDoneButton #if DEBUG navigationItem.leftBarButtonItem = UIBarButtonItem( @@ -45,8 +48,10 @@ class ProblemReportReviewViewController: UIViewController { self?.share() }) ) + navigationItem.leftBarButtonItem?.accessibilityIdentifier = .appLogsShareButton #endif + textView.accessibilityIdentifier = .problemReportAppLogsTextView textView.translatesAutoresizingMaskIntoConstraints = false textView.text = reportString textView.isEditable = false diff --git a/ios/MullvadVPN/View controllers/ProblemReport/ProblemReportViewController+ViewManagement.swift b/ios/MullvadVPN/View controllers/ProblemReport/ProblemReportViewController+ViewManagement.swift index 488c48370fec..6d98e9c83bc6 100644 --- a/ios/MullvadVPN/View controllers/ProblemReport/ProblemReportViewController+ViewManagement.swift +++ b/ios/MullvadVPN/View controllers/ProblemReport/ProblemReportViewController+ViewManagement.swift @@ -88,6 +88,7 @@ extension ProblemReportViewController { func makeViewLogsButton() -> AppButton { let button = AppButton(style: .default) + button.accessibilityIdentifier = .problemReportAppLogsButton button.translatesAutoresizingMaskIntoConstraints = false button.setTitle(Self.persistentViewModel.viewLogsButtonTitle, for: .normal) button.addTarget(self, action: #selector(handleViewLogsButtonTap), for: .touchUpInside) @@ -96,7 +97,7 @@ extension ProblemReportViewController { func makeSendButton() -> AppButton { let button = AppButton(style: .success) - button.accessibilityIdentifier = AccessibilityIdentifier.problemReportSendButton + button.accessibilityIdentifier = .problemReportSendButton button.translatesAutoresizingMaskIntoConstraints = false button.setTitle(Self.persistentViewModel.sendLogsButtonTitle, for: .normal) button.addTarget(self, action: #selector(handleSendButtonTap), for: .touchUpInside) diff --git a/ios/MullvadVPN/View controllers/RelayFilter/RelayFilterChipView.swift b/ios/MullvadVPN/View controllers/RelayFilter/RelayFilterChipView.swift index 986281c9d617..67c73b0339b9 100644 --- a/ios/MullvadVPN/View controllers/RelayFilter/RelayFilterChipView.swift +++ b/ios/MullvadVPN/View controllers/RelayFilter/RelayFilterChipView.swift @@ -11,6 +11,7 @@ import UIKit class RelayFilterChipView: UIView { private let titleLabel: UILabel = { let label = UILabel() + label.accessibilityIdentifier = .relayFilterChipLabel label.font = UIFont.preferredFont(forTextStyle: .caption1) label.adjustsFontForContentSizeCategory = true label.textColor = .white @@ -22,11 +23,14 @@ class RelayFilterChipView: UIView { init() { super.init(frame: .zero) + self.accessibilityIdentifier = .relayFilterChipView + let closeButton = IncreasedHitButton() closeButton.setImage( UIImage(named: "IconCloseSml")?.withTintColor(.white.withAlphaComponent(0.6)), for: .normal ) + closeButton.accessibilityIdentifier = .relayFilterChipCloseButton closeButton.addTarget(self, action: #selector(didTapButton(_:)), for: .touchUpInside) let container = UIStackView(arrangedSubviews: [titleLabel, closeButton]) diff --git a/ios/MullvadVPN/View controllers/Tunnel/TunnelControlView.swift b/ios/MullvadVPN/View controllers/Tunnel/TunnelControlView.swift index 92acdb7a5542..16fd69b25b3a 100644 --- a/ios/MullvadVPN/View controllers/Tunnel/TunnelControlView.swift +++ b/ios/MullvadVPN/View controllers/Tunnel/TunnelControlView.swift @@ -189,6 +189,8 @@ final class TunnelControlView: UIView { switch tunnelState { case .connected: secureLabel.accessibilityIdentifier = .connectionStatusConnectedLabel + case .connecting: + secureLabel.accessibilityIdentifier = .connectionStatusConnectingLabel default: secureLabel.accessibilityIdentifier = .connectionStatusNotConnectedLabel } diff --git a/ios/MullvadVPN/View controllers/VPNSettings/CustomDNSViewController.swift b/ios/MullvadVPN/View controllers/VPNSettings/CustomDNSViewController.swift index 67e6ffce19ab..e13dffc0b732 100644 --- a/ios/MullvadVPN/View controllers/VPNSettings/CustomDNSViewController.swift +++ b/ios/MullvadVPN/View controllers/VPNSettings/CustomDNSViewController.swift @@ -32,6 +32,7 @@ class CustomDNSViewController: UITableViewController, VPNSettingsDataSourceDeleg override func viewDidLoad() { super.viewDidLoad() + tableView.accessibilityIdentifier = .dnsSettingsTableView tableView.backgroundColor = .secondaryColor tableView.separatorColor = .secondaryColor tableView.rowHeight = UITableView.automaticDimension diff --git a/ios/MullvadVPN/Views/StatusActivityView.swift b/ios/MullvadVPN/Views/StatusActivityView.swift index c44cbf82fb2f..9dc9dcca6923 100644 --- a/ios/MullvadVPN/Views/StatusActivityView.swift +++ b/ios/MullvadVPN/Views/StatusActivityView.swift @@ -42,9 +42,12 @@ class StatusActivityView: UIView { addSubview(activityIndicator) NSLayoutConstraint.activate([ + activityIndicator.widthAnchor.constraint(equalTo: statusImageView.widthAnchor), activityIndicator.heightAnchor.constraint(equalTo: statusImageView.heightAnchor), statusImageView.topAnchor.constraint(equalTo: topAnchor), statusImageView.bottomAnchor.constraint(equalTo: bottomAnchor), + statusImageView.leadingAnchor.constraint(equalTo: leadingAnchor), + statusImageView.trailingAnchor.constraint(equalTo: trailingAnchor), statusImageView.centerXAnchor.constraint(equalTo: centerXAnchor), statusImageView.centerYAnchor.constraint(equalTo: centerYAnchor), diff --git a/ios/MullvadVPN/Views/StatusImageView.swift b/ios/MullvadVPN/Views/StatusImageView.swift index 7665daa5809e..51f603c3825d 100644 --- a/ios/MullvadVPN/Views/StatusImageView.swift +++ b/ios/MullvadVPN/Views/StatusImageView.swift @@ -29,6 +29,22 @@ class StatusImageView: UIImageView { } } + override var accessibilityValue: String? { + get { + switch style { + case .success: + return "success" + case .failure: + return "fail" + } + } + + // swiftlint:disable:next unused_setter_value + set { + fatalError("This accessibilityValue property is get only") + } + } + override var intrinsicContentSize: CGSize { CGSize(width: 60, height: 60) } @@ -42,6 +58,7 @@ class StatusImageView: UIImageView { self.style = style super.init(image: style.image) image = style.image + accessibilityIdentifier = .statusImageView } required init?(coder: NSCoder) { diff --git a/ios/MullvadVPNUITests/AccountTests.swift b/ios/MullvadVPNUITests/AccountTests.swift index d417c66d19ec..18ca3be7419e 100644 --- a/ios/MullvadVPNUITests/AccountTests.swift +++ b/ios/MullvadVPNUITests/AccountTests.swift @@ -8,14 +8,14 @@ import XCTest -class AccountApi: XCTestCase { - func testApi() throws { - let temporaryAccountNumber = try MullvadAPIWrapper().createAccount() - try MullvadAPIWrapper().addDevices(5, account: temporaryAccountNumber) - } -} - class AccountTests: LoggedOutUITestCase { + lazy var mullvadAPIWrapper: MullvadAPIWrapper = { + do { + // swiftlint:disable:next force_try + return try! MullvadAPIWrapper() + } + }() + override func setUpWithError() throws { continueAfterFailure = false @@ -29,11 +29,11 @@ class AccountTests: LoggedOutUITestCase { // Verify welcome page is shown and get account number from it let accountNumber = WelcomePage(app).getAccountNumber() - try MullvadAPIWrapper().deleteAccount(accountNumber) + mullvadAPIWrapper.deleteAccount(accountNumber) } func testDeleteAccount() throws { - let accountNumber = try MullvadAPIWrapper().createAccount() + let accountNumber = mullvadAPIWrapper.createAccount() LoginPage(app) .tapAccountNumberTextField() @@ -60,12 +60,31 @@ class AccountTests: LoggedOutUITestCase { .verifyFailIconShown() } + /// Verify logging in works. Will retry x number of times since login request sometimes time out. func testLogin() throws { + let hasTimeAccountNumber = getAccountWithTime() + + addTeardownBlock { + self.returnAccountWithTime(accountNumber: hasTimeAccountNumber) + } + + var successIconShown = false + var retryCount = 0 + let maxRetryCount = 3 + LoginPage(app) .tapAccountNumberTextField() .enterText(hasTimeAccountNumber) - .tapAccountNumberSubmitButton() - .verifySuccessIconShown() + + repeat { + successIconShown = LoginPage(app) + .tapAccountNumberSubmitButton() + .getSuccessIconShown() + + retryCount += 1 + } while successIconShown == false && retryCount < maxRetryCount + + HeaderBar(app) .verifyDeviceLabelShown() } @@ -80,16 +99,12 @@ class AccountTests: LoggedOutUITestCase { func testLoginToAccountWithTooManyDevices() throws { // Setup - let temporaryAccountNumber = try MullvadAPIWrapper().createAccount() - try MullvadAPIWrapper().addDevices(5, account: temporaryAccountNumber) + let temporaryAccountNumber = mullvadAPIWrapper.createAccount() + mullvadAPIWrapper.addDevices(5, account: temporaryAccountNumber) // Teardown addTeardownBlock { - do { - try MullvadAPIWrapper().deleteAccount(temporaryAccountNumber) - } catch { - XCTFail("Failed to delete account using app API") - } + self.mullvadAPIWrapper.deleteAccount(temporaryAccountNumber) } LoginPage(app) @@ -109,6 +124,8 @@ class AccountTests: LoggedOutUITestCase { // First taken back to login page and automatically being logged in LoginPage(app) .verifySuccessIconShown() + + HeaderBar(app) .verifyDeviceLabelShown() // And then taken to out of time page because this account don't have any time added to it @@ -116,26 +133,33 @@ class AccountTests: LoggedOutUITestCase { } func testLogOut() throws { - let newAccountNumber = try MullvadAPIWrapper().createAccount() + let newAccountNumber = mullvadAPIWrapper.createAccount() login(accountNumber: newAccountNumber) - XCTAssertEqual(try MullvadAPIWrapper().getDevices(newAccountNumber).count, 1) + XCTAssertEqual(try mullvadAPIWrapper.getDevices(newAccountNumber).count, 1, "Account has one device") HeaderBar(app) .tapAccountButton() AccountPage(app) .tapLogOutButton() + .waitForLogoutSpinnerToDisappear() LoginPage(app) - XCTAssertEqual(try MullvadAPIWrapper().getDevices(newAccountNumber).count, 0) - try MullvadAPIWrapper().deleteAccount(newAccountNumber) + XCTAssertEqual(try mullvadAPIWrapper.getDevices(newAccountNumber).count, 0, "Account has 0 devices") + mullvadAPIWrapper.deleteAccount(newAccountNumber) } func testTimeLeft() throws { + let hasTimeAccountNumber = getAccountWithTime() + + addTeardownBlock { + self.returnAccountWithTime(accountNumber: hasTimeAccountNumber) + } + login(accountNumber: hasTimeAccountNumber) - let accountExpiry = try MullvadAPIWrapper().getAccountExpiry(hasTimeAccountNumber) + let accountExpiry = try mullvadAPIWrapper.getAccountExpiry(hasTimeAccountNumber) HeaderBar(app) .tapAccountButton() diff --git a/ios/MullvadVPNUITests/ConnectivityTests.swift b/ios/MullvadVPNUITests/ConnectivityTests.swift index 64438308f940..30936e85bd5c 100644 --- a/ios/MullvadVPNUITests/ConnectivityTests.swift +++ b/ios/MullvadVPNUITests/ConnectivityTests.swift @@ -15,7 +15,11 @@ class ConnectivityTests: LoggedOutUITestCase { /// Verifies that the app still functions when API has been blocked func testAPIConnectionViaBridges() throws { + firewallAPIClient.removeRules() + let hasTimeAccountNumber = getAccountWithTime() + addTeardownBlock { + self.returnAccountWithTime(accountNumber: hasTimeAccountNumber) self.firewallAPIClient.removeRules() } @@ -25,25 +29,57 @@ class ConnectivityTests: LoggedOutUITestCase { LoginPage(app) .tapAccountNumberTextField() - .enterText(self.hasTimeAccountNumber) + .enterText(hasTimeAccountNumber) .tapAccountNumberSubmitButton() // After creating firewall rule first login attempt might fail. One more attempt is allowed since the app is cycling between two methods. - if isLoggedIn() { - LoginPage(app) - .verifySuccessIconShown() + let successIconShown = LoginPage(app) + .getSuccessIconShown() + + if successIconShown { + HeaderBar(app) .verifyDeviceLabelShown() } else { LoginPage(app) .verifyFailIconShown() .tapAccountNumberSubmitButton() .verifySuccessIconShown() + + HeaderBar(app) .verifyDeviceLabelShown() } } /// Get the app into a blocked state by connecting to a relay then applying a filter which don't find this relay, then verify that app can still communicate by logging out and verifying that the device was successfully removed func testAPIReachableWhenBlocked() throws { + let hasTimeAccountNumber = getAccountWithTime() + addTeardownBlock { + // Reset any filters + self.login(accountNumber: hasTimeAccountNumber) + + TunnelControlPage(self.app) + .tapSelectLocationButton() + + let filterCloseButtons = self.app.buttons + .matching(identifier: AccessibilityIdentifier.relayFilterChipCloseButton.rawValue) + .allElementsBoundByIndex + + for filterCloseButton in filterCloseButtons where filterCloseButton.isHittable { + filterCloseButton.tap() + } + + // Reset selected location to Sweden + SelectLocationPage(self.app) + .tapLocationCell(withName: BaseUITestCase.appDefaultCountry) + + self.allowAddVPNConfigurationsIfAsked() + + TunnelControlPage(self.app) + .tapCancelOrDisconnectButton() + + self.returnAccountWithTime(accountNumber: hasTimeAccountNumber) + } + // Setup. Enter blocked state by connecting to relay and applying filter which relay isn't part of. login(accountNumber: hasTimeAccountNumber) @@ -88,6 +124,7 @@ class ConnectivityTests: LoggedOutUITestCase { AccountPage(app) .tapLogOutButton() + .waitForLogoutSpinnerToDisappear() LoginPage(app) @@ -97,6 +134,8 @@ class ConnectivityTests: LoggedOutUITestCase { // swiftlint:disable function_body_length /// Test that the app is functioning when API is down. To simulate API being down we create a dummy access method func testAppStillFunctioningWhenAPIDown() throws { + let hasTimeAccountNumber = getAccountWithTime() + addTeardownBlock { HeaderBar(self.app) .tapSettingsButton() @@ -105,12 +144,13 @@ class ConnectivityTests: LoggedOutUITestCase { .tapAPIAccessCell() self.toggleAllAccessMethodsEnabledSwitchesIfOff() + self.returnAccountWithTime(accountNumber: hasTimeAccountNumber) } // Setup. Create a dummy access method to simulate API being down(unreachable) LoginPage(app) .tapAccountNumberTextField() - .enterText(self.hasTimeAccountNumber) + .enterText(hasTimeAccountNumber) .tapAccountNumberSubmitButton() TunnelControlPage(app) @@ -163,12 +203,12 @@ class ConnectivityTests: LoggedOutUITestCase { // Log out will take long because API cannot be reached AccountPage(app) .tapLogOutButton() - .waitForSpinnerNoLongerShown() + .waitForLogoutSpinnerToDisappear() // Verify API cannot be reached by doing a login attempt which should fail LoginPage(app) .tapAccountNumberTextField() - .enterText(self.hasTimeAccountNumber) + .enterText(hasTimeAccountNumber) .tapAccountNumberSubmitButton() .verifyFailIconShown() } diff --git a/ios/MullvadVPNUITests/CustomListsTests.swift b/ios/MullvadVPNUITests/CustomListsTests.swift index 6a3f9f1db725..0d86b4dc091e 100644 --- a/ios/MullvadVPNUITests/CustomListsTests.swift +++ b/ios/MullvadVPNUITests/CustomListsTests.swift @@ -38,7 +38,6 @@ class CustomListsTests: LoggedInWithTimeUITestCase { let customListName = createCustomListName() createCustomList(named: customListName) - workaroundOpenCustomListMenuBug() deleteCustomList(named: customListName) SelectLocationPage(app) @@ -55,11 +54,9 @@ class CustomListsTests: LoggedInWithTimeUITestCase { createCustomList(named: customListName) addTeardownBlock { - self.workaroundOpenCustomListMenuBug() self.deleteCustomList(named: customListName) } - workaroundOpenCustomListMenuBug() startEditingCustomList(named: customListName) EditCustomListLocationsPage(app) @@ -84,11 +81,9 @@ class CustomListsTests: LoggedInWithTimeUITestCase { createCustomList(named: customListName) addTeardownBlock { - self.workaroundOpenCustomListMenuBug() self.deleteCustomList(named: customListName) } - workaroundOpenCustomListMenuBug() startEditingCustomList(named: customListName) EditCustomListLocationsPage(app) @@ -125,14 +120,6 @@ class CustomListsTests: LoggedInWithTimeUITestCase { .tapCreateListButton() } - func workaroundOpenCustomListMenuBug() { - // In order to avoid a bug where the open custom list button cannot be found, the location view is closed and then reopened - SelectLocationPage(app) - .tapDoneButton() - TunnelControlPage(app) - .tapSelectLocationButton() - } - func startEditingCustomList(named customListName: String) { SelectLocationPage(app) .tapWhereStatusBarShouldBeToScrollToTopMostPosition() diff --git a/ios/MullvadVPNUITests/Info.plist b/ios/MullvadVPNUITests/Info.plist index 0bed909d3f3e..a81366999153 100644 --- a/ios/MullvadVPNUITests/Info.plist +++ b/ios/MullvadVPNUITests/Info.plist @@ -4,22 +4,26 @@ AdServingDomain $(AD_SERVING_DOMAIN) + AmIJSONUrl + $(AM_I_JSON_URL) ApiEndpoint $(API_ENDPOINT) ApiHostName $(API_HOST_NAME) + AttachAppLogsOnFailure + ${ATTACH_APP_LOGS_ON_FAILURE} DisplayName $(DISPLAY_NAME) FirewallApiBaseURL $(FIREWALL_API_BASE_URL) - FiveWireGuardKeysAccountNumber - $(FIVE_WIREGUARD_KEYS_ACCOUNT_NUMBER) HasTimeAccountNumber $(HAS_TIME_ACCOUNT_NUMBER) IOSDevicePinCode $(IOS_DEVICE_PIN_CODE) NoTimeAccountNumber $(NO_TIME_ACCOUNT_NUMBER) + PartnerApiToken + $(PARTNER_API_TOKEN) ShouldBeReachableDomain $(SHOULD_BE_REACHABLE_DOMAIN) TestDeviceIdentifier diff --git a/ios/MullvadVPNUITests/Networking/MullvadAPIWrapper.swift b/ios/MullvadVPNUITests/Networking/MullvadAPIWrapper.swift index 588b692a7a3b..d0983b3ee201 100644 --- a/ios/MullvadVPNUITests/Networking/MullvadAPIWrapper.swift +++ b/ios/MullvadVPNUITests/Networking/MullvadAPIWrapper.swift @@ -16,12 +16,16 @@ enum MullvadAPIError: Error { } class MullvadAPIWrapper { + private var mullvadAPI: MullvadApi + private let throttleQueue = DispatchQueue(label: "MullvadAPIWrapperThrottleQueue", qos: .userInitiated) + private var lastCallDate: Date? + private let throttleDelay: TimeInterval = 0.25 + private let throttleWaitTimeout: TimeInterval = 5.0 + // swiftlint:disable force_cast static let hostName = Bundle(for: MullvadAPIWrapper.self) .infoDictionary?["ApiHostName"] as! String - private var mullvadAPI: MullvadApi - /// API endpoint configuration value in the format : static let endpoint = Bundle(for: MullvadAPIWrapper.self) .infoDictionary?["ApiEndpoint"] as! String @@ -33,6 +37,27 @@ class MullvadAPIWrapper { mullvadAPI = try MullvadApi(apiAddress: apiAddress, hostname: hostname) } + /// Throttle what's in the callback. This is used for throttling requests to the app API. All requests should be throttled or else we might be rate limited. 5 requests per second allowed. + private func throttle(callback: @escaping () -> Void) { + throttleQueue.async { + let now = Date() + var delay: TimeInterval = 0 + + if let lastCallDate = self.lastCallDate { + let timeSinceLastCall = now.timeIntervalSince(lastCallDate) + + if timeSinceLastCall < self.throttleDelay { + delay = self.throttleDelay - timeSinceLastCall + } + } + + self.throttleQueue.asyncAfter(deadline: .now() + delay) { + callback() + self.lastCallDate = Date() + } + } + } + public static func getAPIIPAddress() throws -> String { guard let ipAddress = endpoint.components(separatedBy: ":").first else { throw MullvadAPIError.invalidEndpointFormatError @@ -59,56 +84,116 @@ class MullvadAPIWrapper { } func createAccount() -> String { - do { - let accountNumber = try mullvadAPI.createAccount() - return accountNumber - } catch { - XCTFail("Failed to create account using app API") - return String() + var accountNumber = String() + var requestError: Error? + let requestCompletedExpectation = XCTestExpectation(description: "Create account request completed") + + throttle { + do { + accountNumber = try self.mullvadAPI.createAccount() + } catch { + requestError = MullvadAPIError.requestError + } + + requestCompletedExpectation.fulfill() } + + let waitResult = XCTWaiter().wait(for: [requestCompletedExpectation], timeout: throttleWaitTimeout) + XCTAssertEqual(waitResult, .completed, "Create account request completed") + XCTAssertNil(requestError, "Create account error is nil") + + return accountNumber } func deleteAccount(_ accountNumber: String) { + var requestError: Error? + let requestCompletedExpectation = XCTestExpectation(description: "Delete account request completed") + do { try mullvadAPI.delete(account: accountNumber) } catch { - XCTFail("Failed to delete account using app API") + requestError = MullvadAPIError.requestError } + + requestCompletedExpectation.fulfill() + + let waitResult = XCTWaiter().wait(for: [requestCompletedExpectation], timeout: throttleWaitTimeout) + XCTAssertEqual(waitResult, .completed, "Delete account request completed") + XCTAssertNil(requestError, "Delete account error is nil") } /// Add another device to specified account. A dummy WireGuard key will be generated. - func addDevice(_ account: String) throws { - let devicePublicKey = generateMockWireGuardKey() + func addDevice(_ account: String) { + var addDeviceError: Error? + let requestCompletedExpectation = XCTestExpectation(description: "Add device request completed") - do { - try mullvadAPI.addDevice(forAccount: account, publicKey: devicePublicKey) - } catch { - throw MullvadAPIError.requestError + throttle { + let devicePublicKey = self.generateMockWireGuardKey() + + do { + try self.mullvadAPI.addDevice(forAccount: account, publicKey: devicePublicKey) + } catch { + addDeviceError = MullvadAPIError.requestError + } + + requestCompletedExpectation.fulfill() } + + let waitResult = XCTWaiter().wait(for: [requestCompletedExpectation], timeout: throttleWaitTimeout) + XCTAssertEqual(waitResult, .completed, "Add device request completed") + XCTAssertNil(addDeviceError, "Add device error is nil") } /// Add multiple devices to specified account. Dummy WireGuard keys will be generated. - func addDevices(_ numberOfDevices: Int, account: String) throws { - for _ in 0 ..< numberOfDevices { - try self.addDevice(account) + func addDevices(_ numberOfDevices: Int, account: String) { + for i in 0 ..< numberOfDevices { + self.addDevice(account) + print("Created \(i + 1) devices") } } func getAccountExpiry(_ account: String) throws -> Date { - do { - let accountExpiryTimestamp = Double(try mullvadAPI.getExpiry(forAccount: account)) - let accountExpiryDate = Date(timeIntervalSince1970: accountExpiryTimestamp) - return accountExpiryDate - } catch { - throw MullvadAPIError.requestError + var accountExpiryDate: Date = .distantPast + var requestError: Error? + let requestCompletedExpectation = XCTestExpectation(description: "Get account expiry request completed") + + throttle { + do { + let accountExpiryTimestamp = Double(try self.mullvadAPI.getExpiry(forAccount: account)) + accountExpiryDate = Date(timeIntervalSince1970: accountExpiryTimestamp) + } catch { + requestError = MullvadAPIError.requestError + } + + requestCompletedExpectation.fulfill() } + + let waitResult = XCTWaiter().wait(for: [requestCompletedExpectation], timeout: throttleWaitTimeout) + XCTAssertEqual(waitResult, .completed, "Get account expiry request completed") + XCTAssertNil(requestError, "Get account expiry error is nil") + + return accountExpiryDate } func getDevices(_ account: String) throws -> [Device] { - do { - return try mullvadAPI.listDevices(forAccount: account) - } catch { - throw MullvadAPIError.requestError + var devices: [Device] = [] + var requestError: Error? + let requestCompletedExpectation = XCTestExpectation(description: "Get devices request completed") + + throttle { + do { + devices = try self.mullvadAPI.listDevices(forAccount: account) + } catch { + requestError = MullvadAPIError.requestError + } + + requestCompletedExpectation.fulfill() } + + let waitResult = XCTWaiter.wait(for: [requestCompletedExpectation], timeout: throttleWaitTimeout) + XCTAssertEqual(waitResult, .completed, "Get devices request completed") + XCTAssertNil(requestError, "Get devices error is nil") + + return devices } } diff --git a/ios/MullvadVPNUITests/Networking/Networking.swift b/ios/MullvadVPNUITests/Networking/Networking.swift index 003cc4386c3d..6e75925d940a 100644 --- a/ios/MullvadVPNUITests/Networking/Networking.swift +++ b/ios/MullvadVPNUITests/Networking/Networking.swift @@ -129,34 +129,52 @@ class Networking { public static func verifyCanAccessAPI() throws { let apiIPAddress = try MullvadAPIWrapper.getAPIIPAddress() let apiPort = try MullvadAPIWrapper.getAPIPort() - XCTAssertTrue(try canConnectSocket(host: apiIPAddress, port: apiPort)) + XCTAssertTrue( + try canConnectSocket(host: apiIPAddress, port: apiPort), + "Failed to verify that API can be accessed" + ) } /// Verify API cannot be accessed by attempting to connect a socket to the configured API host and port public static func verifyCannotAccessAPI() throws { let apiIPAddress = try MullvadAPIWrapper.getAPIIPAddress() let apiPort = try MullvadAPIWrapper.getAPIPort() - XCTAssertFalse(try canConnectSocket(host: apiIPAddress, port: apiPort)) + XCTAssertFalse( + try canConnectSocket(host: apiIPAddress, port: apiPort), + "Failed to verify that API cannot be accessed" + ) } /// Verify that the device has Internet connectivity public static func verifyCanAccessInternet() throws { - XCTAssertTrue(try canConnectSocket(host: getAlwaysReachableDomain(), port: "80")) + XCTAssertTrue( + try canConnectSocket(host: getAlwaysReachableDomain(), port: "80"), + "Failed to verify that the Internet can be acccessed" + ) } /// Verify that the device does not have Internet connectivity public static func verifyCannotAccessInternet() throws { - XCTAssertFalse(try canConnectSocket(host: getAlwaysReachableDomain(), port: "80")) + XCTAssertFalse( + try canConnectSocket(host: getAlwaysReachableDomain(), port: "80"), + "Failed to verify that the Internet cannot be accessed" + ) } /// Verify that an ad serving domain is reachable by making sure a connection can be established on port 80 public static func verifyCanReachAdServingDomain() throws { - XCTAssertTrue(try Self.canConnectSocket(host: try Self.getAdServingDomain(), port: "80")) + XCTAssertTrue( + try Self.canConnectSocket(host: try Self.getAdServingDomain(), port: "80"), + "Failed to verify that ad serving domain can be accessed" + ) } /// Verify that an ad serving domain is NOT reachable by making sure a connection can not be established on port 80 public static func verifyCannotReachAdServingDomain() throws { - XCTAssertFalse(try Self.canConnectSocket(host: try Self.getAdServingDomain(), port: "80")) + XCTAssertFalse( + try Self.canConnectSocket(host: try Self.getAdServingDomain(), port: "80"), + "Failed to verify that ad serving domain cannot be accessed" + ) } /// Verify that the expected DNS server is used by verifying provider name and whether it is a Mullvad DNS server or not @@ -205,8 +223,12 @@ class Networking { XCTAssertGreaterThanOrEqual(dnsServerEntries.count, 1) for dnsServerEntry in dnsServerEntries { - XCTAssertEqual(dnsServerEntry.organization, providerName) - XCTAssertEqual(dnsServerEntry.mullvad_dns, isMullvad) + XCTAssertEqual(dnsServerEntry.organization, providerName, "Expected organization name") + XCTAssertEqual( + dnsServerEntry.mullvad_dns, + isMullvad, + "Verifying that it is or isn't a Mullvad DNS server" + ) } } } @@ -215,8 +237,12 @@ class Networking { } } - public static func verifyConnectedThroughMullvad() { - let mullvadConnectionJsonEndpoint = "https://am.i.mullvad.net/json" + public static func verifyConnectedThroughMullvad() throws { + let mullvadConnectionJsonEndpoint = try XCTUnwrap( + Bundle(for: Networking.self) + .infoDictionary?["AmIJSONUrl"] as? String, + "Read am I JSON URL from Info" + ) guard let url = URL(string: mullvadConnectionJsonEndpoint) else { XCTFail("Failed to unwrap URL") return diff --git a/ios/MullvadVPNUITests/Networking/PartnerAPIClient.swift b/ios/MullvadVPNUITests/Networking/PartnerAPIClient.swift new file mode 100644 index 000000000000..67d1d7acdddb --- /dev/null +++ b/ios/MullvadVPNUITests/Networking/PartnerAPIClient.swift @@ -0,0 +1,123 @@ +// +// PartnerAPIClient.swift +// MullvadVPNUITests +// +// Created by Niklas Berglund on 2024-04-19. +// Copyright © 2024 Mullvad VPN AB. All rights reserved. +// + +import Foundation +import XCTest + +class PartnerAPIClient { + let baseURL = URL(string: "https://partner.stagemole.eu/v1/")! + + lazy var accessToken: String = { + guard let token = Bundle(for: BaseUITestCase.self).infoDictionary?["PartnerApiToken"] as? String else { + fatalError("Failed to retrieve partner API token from config") + } + return token + }() + + /// Add time to an account + /// - Parameters: + /// - accountNumber: Account number + /// - days: Number of days to add. Needs to be between 1 and 31. + func addTime(accountNumber: String, days: Int) -> Date { + let jsonResponse = sendRequest( + method: "POST", + endpoint: "accounts/\(accountNumber)/extend", + jsonObject: ["days": "\(days)"] + ) + + guard let newExpiryString = jsonResponse["new_expiry"] as? String else { + XCTFail("Failed to read new account expiry from response") + return Date() + } + + let dateFormatter = ISO8601DateFormatter() + guard let newExpiryDate = dateFormatter.date(from: newExpiryString) else { + XCTFail("Failed to create Date object from date string") + return Date() + } + + return newExpiryDate + } + + func createAccount() -> String { + let jsonResponse = sendRequest(method: "POST", endpoint: "accounts", jsonObject: nil) + + guard let accountNumber = jsonResponse["id"] as? String else { + XCTFail("Failed to read created account number") + return String() + } + + return accountNumber + } + + func deleteAccount(accountNumber: String) { + _ = sendRequest(method: "DELETE", endpoint: "accounts/\(accountNumber)", jsonObject: nil) + } + + private func sendRequest(method: String, endpoint: String, jsonObject: [String: Any]?) -> [String: Any] { + let url = baseURL.appendingPathComponent(endpoint) + var request = URLRequest(url: url) + request.httpMethod = method + request.setValue("Basic \(accessToken)", forHTTPHeaderField: "Authorization") + + var jsonResponse: [String: Any] = [:] + + do { + if let jsonObject = jsonObject { + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + request.httpBody = try JSONSerialization.data(withJSONObject: jsonObject, options: []) + } + } catch { + XCTFail("Failed to serialize JSON object") + return [:] + } + + let completionHandlerInvokedExpectation = XCTestExpectation( + description: "Completion handler for the request is invoked" + ) + + var requestError: Error? + + let task = URLSession.shared.dataTask(with: request) { data, response, error in + requestError = error + + guard let data = data, + let response = response as? HTTPURLResponse, + error == nil else { + XCTFail("Error: \(error?.localizedDescription ?? "Unknown error")") + completionHandlerInvokedExpectation.fulfill() + return + } + + if 200 ... 204 ~= response.statusCode { + print("Request successful") + do { + if data.isEmpty { + // Not all requests return JSON data + jsonResponse = [:] + } else { + jsonResponse = try JSONSerialization.jsonObject(with: data) as? [String: Any] ?? [:] + } + } catch { + XCTFail("Failed to deserialize JSON response") + } + } else { + XCTFail("Request failed") + } + + completionHandlerInvokedExpectation.fulfill() + } + + task.resume() + let waitResult = XCTWaiter().wait(for: [completionHandlerInvokedExpectation], timeout: 10) + XCTAssertEqual(waitResult, .completed, "Waiting for partner API request expectation completed") + XCTAssertNil(requestError) + + return jsonResponse + } +} diff --git a/ios/MullvadVPNUITests/Pages/APIAccessPage.swift b/ios/MullvadVPNUITests/Pages/APIAccessPage.swift index 33d8be3393fa..bd1f5850ebc2 100644 --- a/ios/MullvadVPNUITests/Pages/APIAccessPage.swift +++ b/ios/MullvadVPNUITests/Pages/APIAccessPage.swift @@ -12,7 +12,7 @@ import XCTest class APIAccessPage: Page { override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .apiAccessView + self.pageElement = app.otherElements[.apiAccessView] waitForPageToBeShown() } diff --git a/ios/MullvadVPNUITests/Pages/AccountDeletionPage.swift b/ios/MullvadVPNUITests/Pages/AccountDeletionPage.swift index 30bf01034ebd..b92b51a7ed33 100644 --- a/ios/MullvadVPNUITests/Pages/AccountDeletionPage.swift +++ b/ios/MullvadVPNUITests/Pages/AccountDeletionPage.swift @@ -13,7 +13,7 @@ class AccountDeletionPage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .deleteAccountView + self.pageElement = app.otherElements[.deleteAccountView] waitForPageToBeShown() } @@ -23,12 +23,7 @@ class AccountDeletionPage: Page { } @discardableResult func tapDeleteAccountButton() -> Self { - guard let pageAccessibilityIdentifier = self.pageAccessibilityIdentifier else { - XCTFail("Page's accessibility identifier not set") - return self - } - - app.otherElements[pageAccessibilityIdentifier].buttons[AccessibilityIdentifier.deleteButton].tap() + app.otherElements[.deleteAccountView].buttons[AccessibilityIdentifier.deleteButton].tap() return self } diff --git a/ios/MullvadVPNUITests/Pages/AccountPage.swift b/ios/MullvadVPNUITests/Pages/AccountPage.swift index 83082f339577..fd3d5d448e60 100644 --- a/ios/MullvadVPNUITests/Pages/AccountPage.swift +++ b/ios/MullvadVPNUITests/Pages/AccountPage.swift @@ -13,7 +13,7 @@ class AccountPage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .accountView + self.pageElement = app.otherElements[.accountView] waitForPageToBeShown() } @@ -67,13 +67,13 @@ class AccountPage: Page { return self } - XCTAssertEqual(strippedDate, paidUntilLabelDate) + XCTAssertEqual(strippedDate, paidUntilLabelDate, "Paid until date correct") return self } - @discardableResult func waitForSpinnerNoLongerShown() -> Self { - app.otherElements[AccessibilityIdentifier.logOutSpinnerAlertView] - .waitForNonExistence(timeout: BaseUITestCase.veryLongTimeout) - return self + func waitForLogoutSpinnerToDisappear() { + let spinnerDisappeared = app.otherElements[.logOutSpinnerAlertView] + .waitForNonExistence(timeout: BaseUITestCase.extremelyLongTimeout) + XCTAssertTrue(spinnerDisappeared, "Log out spinner disappeared") } } diff --git a/ios/MullvadVPNUITests/Pages/AddAccessMethodPage.swift b/ios/MullvadVPNUITests/Pages/AddAccessMethodPage.swift index 698397bbbcdd..d961822226ed 100644 --- a/ios/MullvadVPNUITests/Pages/AddAccessMethodPage.swift +++ b/ios/MullvadVPNUITests/Pages/AddAccessMethodPage.swift @@ -12,8 +12,7 @@ import XCTest class AddAccessMethodPage: Page { override init(_ app: XCUIApplication) { super.init(app) - - self.pageAccessibilityIdentifier = .addAccessMethodTableView + self.pageElement = app.tables[.addAccessMethodTableView] waitForPageToBeShown() } @@ -75,8 +74,7 @@ class AddAccessMethodPage: Page { class AddAccessMethodAPIUnreachableAlert: Page { override init(_ app: XCUIApplication) { super.init(app) - - self.pageAccessibilityIdentifier = .accessMethodUnreachableAlert + self.pageElement = app.otherElements[.accessMethodUnreachableAlert] waitForPageToBeShown() } diff --git a/ios/MullvadVPNUITests/Pages/Alert.swift b/ios/MullvadVPNUITests/Pages/Alert.swift index dce7c3bf11c1..0f723af612de 100644 --- a/ios/MullvadVPNUITests/Pages/Alert.swift +++ b/ios/MullvadVPNUITests/Pages/Alert.swift @@ -14,7 +14,7 @@ class Alert: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .alertContainerView + self.pageElement = app.otherElements[.alertContainerView] waitForPageToBeShown() } diff --git a/ios/MullvadVPNUITests/Pages/AppLogsPage.swift b/ios/MullvadVPNUITests/Pages/AppLogsPage.swift new file mode 100644 index 000000000000..32d157e9e2d3 --- /dev/null +++ b/ios/MullvadVPNUITests/Pages/AppLogsPage.swift @@ -0,0 +1,36 @@ +// +// AppLogsPage.swift +// MullvadVPNUITests +// +// Created by Niklas Berglund on 2024-04-26. +// Copyright © 2024 Mullvad VPN AB. All rights reserved. +// + +import XCTest + +class AppLogsPage: Page { + override init(_ app: XCUIApplication) { + super.init(app) + self.pageElement = app.otherElements[.appLogsView] + waitForPageToBeShown() + } + + @discardableResult func tapShareButton() -> Self { + app.buttons[.appLogsShareButton].tap() + return self + } + + @discardableResult func tapDoneButton() -> Self { + app.buttons[.appLogsDoneButton].tap() + return self + } + + func getAppLogText() -> String { + guard let logText = app.textViews[.problemReportAppLogsTextView].value as? String else { + XCTFail("Failed to extract app log text") + return String() + } + + return logText + } +} diff --git a/ios/MullvadVPNUITests/Pages/ChangeLogAlert.swift b/ios/MullvadVPNUITests/Pages/ChangeLogAlert.swift index 70d67d0294fc..61e59ec953f3 100644 --- a/ios/MullvadVPNUITests/Pages/ChangeLogAlert.swift +++ b/ios/MullvadVPNUITests/Pages/ChangeLogAlert.swift @@ -13,7 +13,7 @@ class ChangeLogAlert: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .changeLogAlert + self.pageElement = app.otherElements[.changeLogAlert] waitForPageToBeShown() } diff --git a/ios/MullvadVPNUITests/Pages/CustomListPage.swift b/ios/MullvadVPNUITests/Pages/CustomListPage.swift index 17ebf363c343..b4e6dc1311b0 100644 --- a/ios/MullvadVPNUITests/Pages/CustomListPage.swift +++ b/ios/MullvadVPNUITests/Pages/CustomListPage.swift @@ -12,13 +12,13 @@ class CustomListPage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .newCustomListView + self.pageElement = app.otherElements[.newCustomListView] waitForPageToBeShown() } @discardableResult func verifyCreateButtonIs(enabled: Bool) -> Self { let saveOrCreateButton = app.buttons[.saveCreateCustomListButton] - XCTAssertTrue(saveOrCreateButton.isEnabled == enabled) + XCTAssertTrue(saveOrCreateButton.isEnabled == enabled, "Verify state of create button") return self } diff --git a/ios/MullvadVPNUITests/Pages/DNSSettingsPage.swift b/ios/MullvadVPNUITests/Pages/DNSSettingsPage.swift index e592862cbf33..88cd45bb541c 100644 --- a/ios/MullvadVPNUITests/Pages/DNSSettingsPage.swift +++ b/ios/MullvadVPNUITests/Pages/DNSSettingsPage.swift @@ -13,7 +13,8 @@ class DNSSettingsPage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .dnsSettings + self.pageElement = app.tables[.dnsSettingsTableView] + waitForPageToBeShown() } private func assertSwitchOn(accessibilityIdentifier: AccessibilityIdentifier) -> Self { @@ -38,7 +39,7 @@ class DNSSettingsPage: Page { @discardableResult func tapDNSContentBlockersHeaderExpandButton() -> Self { let headerView = app.otherElements[AccessibilityIdentifier.dnsContentBlockersHeaderView] - let expandButton = headerView.buttons[AccessibilityIdentifier.collapseButton] + let expandButton = headerView.buttons[AccessibilityIdentifier.expandButton] expandButton.tap() return self diff --git a/ios/MullvadVPNUITests/Pages/DeviceManagementPage.swift b/ios/MullvadVPNUITests/Pages/DeviceManagementPage.swift index 571fb1cfd6ec..68340a260ff2 100644 --- a/ios/MullvadVPNUITests/Pages/DeviceManagementPage.swift +++ b/ios/MullvadVPNUITests/Pages/DeviceManagementPage.swift @@ -13,7 +13,7 @@ class DeviceManagementPage: Page { override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .deviceManagementView + self.pageElement = app.otherElements[.deviceManagementView] waitForPageToBeShown() } @@ -36,8 +36,7 @@ class DeviceManagementPage: Page { class DeviceManagementLogOutDeviceConfirmationAlert: Page { override init(_ app: XCUIApplication) { super.init(app) - - self.pageAccessibilityIdentifier = .alertContainerView + self.pageElement = app.otherElements[.alertContainerView] waitForPageToBeShown() } diff --git a/ios/MullvadVPNUITests/Pages/EditAccessMethodPage.swift b/ios/MullvadVPNUITests/Pages/EditAccessMethodPage.swift index 68f21d35cca5..93aad69c1769 100644 --- a/ios/MullvadVPNUITests/Pages/EditAccessMethodPage.swift +++ b/ios/MullvadVPNUITests/Pages/EditAccessMethodPage.swift @@ -12,8 +12,7 @@ import XCTest class EditAccessMethodPage: Page { override init(_ app: XCUIApplication) { super.init(app) - - self.pageAccessibilityIdentifier = .editAccessMethodView + self.pageElement = app.tables[.editAccessMethodView] waitForPageToBeShown() } diff --git a/ios/MullvadVPNUITests/Pages/EditCustomListLocationsPage.swift b/ios/MullvadVPNUITests/Pages/EditCustomListLocationsPage.swift index 08d3d6fa9fef..7d7050c14800 100644 --- a/ios/MullvadVPNUITests/Pages/EditCustomListLocationsPage.swift +++ b/ios/MullvadVPNUITests/Pages/EditCustomListLocationsPage.swift @@ -12,7 +12,7 @@ class EditCustomListLocationsPage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .editCustomListEditLocationsView + self.pageElement = app.otherElements[.editCustomListEditLocationsView] waitForPageToBeShown() } diff --git a/ios/MullvadVPNUITests/Pages/HeaderBar.swift b/ios/MullvadVPNUITests/Pages/HeaderBar.swift index 129d721b3745..0d46fe961680 100644 --- a/ios/MullvadVPNUITests/Pages/HeaderBar.swift +++ b/ios/MullvadVPNUITests/Pages/HeaderBar.swift @@ -16,7 +16,7 @@ class HeaderBar: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .headerBarView + self.pageElement = app.otherElements[.headerBarView] waitForPageToBeShown() } @@ -29,4 +29,13 @@ class HeaderBar: Page { settingsButton.tap() return self } + + @discardableResult public func verifyDeviceLabelShown() -> Self { + XCTAssertTrue( + app.staticTexts[AccessibilityIdentifier.headerDeviceNameLabel] + .waitForExistence(timeout: BaseUITestCase.defaultTimeout), "Device name displayed in header" + ) + + return self + } } diff --git a/ios/MullvadVPNUITests/Pages/ListCustomListsPage.swift b/ios/MullvadVPNUITests/Pages/ListCustomListsPage.swift index 2af421be7a20..8f62bf755286 100644 --- a/ios/MullvadVPNUITests/Pages/ListCustomListsPage.swift +++ b/ios/MullvadVPNUITests/Pages/ListCustomListsPage.swift @@ -12,7 +12,7 @@ class ListCustomListsPage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .listCustomListsView + self.pageElement = app.otherElements[.listCustomListsView] waitForPageToBeShown() } diff --git a/ios/MullvadVPNUITests/Pages/LoginPage.swift b/ios/MullvadVPNUITests/Pages/LoginPage.swift index f7b204b344bc..284e5ac7e72b 100644 --- a/ios/MullvadVPNUITests/Pages/LoginPage.swift +++ b/ios/MullvadVPNUITests/Pages/LoginPage.swift @@ -13,7 +13,7 @@ class LoginPage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .loginView + self.pageElement = app.otherElements[.loginView] waitForPageToBeShown() } @@ -22,6 +22,13 @@ class LoginPage: Page { return self } + @discardableResult public func waitForAccountNumberSubmitButton() -> Self { + let submitButtonExist = app.buttons[AccessibilityIdentifier.loginTextFieldButton] + .waitForExistence(timeout: BaseUITestCase.defaultTimeout) + XCTAssertTrue(submitButtonExist, "Account number submit button shown") + return self + } + @discardableResult public func tapAccountNumberSubmitButton() -> Self { app.buttons[AccessibilityIdentifier.loginTextFieldButton].tap() return self @@ -32,24 +39,40 @@ class LoginPage: Page { return self } - @discardableResult public func verifyDeviceLabelShown() -> Self { - XCTAssertTrue( - app.staticTexts[AccessibilityIdentifier.headerDeviceNameLabel] - .waitForExistence(timeout: BaseUITestCase.defaultTimeout) - ) + @discardableResult public func verifySuccessIconShown() -> Self { + // Success icon is only shown very briefly, since another view is presented after success icon is shown. + // Therefore we need to poll faster than waitForElement function. + let successIconDisplayedExpectation = XCTestExpectation(description: "Success icon shown") + let timer = Timer.scheduledTimer(withTimeInterval: 0.2, repeats: true) { [self] _ in + let statusImageView = self.app.images[.statusImageView] - return self - } + if statusImageView.exists { + if statusImageView.value as? String == "success" { + successIconDisplayedExpectation.fulfill() + } + } + } + + let waitResult = XCTWaiter.wait(for: [successIconDisplayedExpectation], timeout: BaseUITestCase.longTimeout) + XCTAssertEqual(waitResult, .completed, "Success icon shown") + timer.invalidate() - @discardableResult public func verifySuccessIconShown() -> Self { - _ = app.images.element(matching: .image, identifier: "IconSuccess") - .waitForExistence(timeout: BaseUITestCase.defaultTimeout) return self } @discardableResult public func verifyFailIconShown() -> Self { - _ = app.images.element(matching: .image, identifier: "IconFail") - .waitForExistence(timeout: BaseUITestCase.longTimeout) + let predicate = NSPredicate(format: "identifier == 'statusImageView' AND value == 'fail'") + let elementQuery = app.images.containing(predicate) + let elementExists = elementQuery.firstMatch.waitForExistence(timeout: BaseUITestCase.longTimeout) + XCTAssertTrue(elementExists, "Fail icon shown") return self } + + /// Checks whether success icon is being shown + func getSuccessIconShown() -> Bool { + let predicate = NSPredicate(format: "identifier == 'statusImageView' AND value == 'success'") + let elementQuery = app.images.containing(predicate) + let elementExists = elementQuery.firstMatch.waitForExistence(timeout: BaseUITestCase.defaultTimeout) + return elementExists + } } diff --git a/ios/MullvadVPNUITests/Pages/OutOfTimePage.swift b/ios/MullvadVPNUITests/Pages/OutOfTimePage.swift index 0950aa9147ce..22e287ad1463 100644 --- a/ios/MullvadVPNUITests/Pages/OutOfTimePage.swift +++ b/ios/MullvadVPNUITests/Pages/OutOfTimePage.swift @@ -13,7 +13,7 @@ class OutOfTimePage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .outOfTimeView + self.pageElement = app.otherElements[.outOfTimeView] waitForPageToBeShown() } } diff --git a/ios/MullvadVPNUITests/Pages/Page.swift b/ios/MullvadVPNUITests/Pages/Page.swift index 09a6b5b2ae3d..92a63d888458 100644 --- a/ios/MullvadVPNUITests/Pages/Page.swift +++ b/ios/MullvadVPNUITests/Pages/Page.swift @@ -11,18 +11,19 @@ import XCTest class Page { let app: XCUIApplication - var pageAccessibilityIdentifier: AccessibilityIdentifier? + + /// Element in the page used to verify that the page is currently being shown, usually accessibilityIdentifier of the view controller's main view + var pageElement: XCUIElement? @discardableResult init(_ app: XCUIApplication) { self.app = app } func waitForPageToBeShown() { - if let pageAccessibilityIdentifier = self.pageAccessibilityIdentifier { - XCTAssert( - self.app.descendants(matching: .any).matching(identifier: pageAccessibilityIdentifier.rawValue) - .firstMatch - .waitForExistence(timeout: BaseUITestCase.defaultTimeout) + if let pageElement { + XCTAssertTrue( + pageElement.waitForExistence(timeout: BaseUITestCase.defaultTimeout), + "Page is shown" ) } } diff --git a/ios/MullvadVPNUITests/Pages/ProblemReportPage.swift b/ios/MullvadVPNUITests/Pages/ProblemReportPage.swift index d02a0ec9917e..6c9cbabff9e4 100644 --- a/ios/MullvadVPNUITests/Pages/ProblemReportPage.swift +++ b/ios/MullvadVPNUITests/Pages/ProblemReportPage.swift @@ -13,8 +13,7 @@ class ProblemReportPage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - pageAccessibilityIdentifier = .problemReportView - + self.pageElement = app.otherElements[.problemReportView] waitForPageToBeShown() } @@ -33,7 +32,7 @@ class ProblemReportPage: Page { } @discardableResult func tapViewAppLogsButton() -> Self { - app.otherElements[AccessibilityIdentifier.problemReportAppLogsButton] + app.buttons[AccessibilityIdentifier.problemReportAppLogsButton] .tap() return self diff --git a/ios/MullvadVPNUITests/Pages/ProblemReportSubmittedPage.swift b/ios/MullvadVPNUITests/Pages/ProblemReportSubmittedPage.swift index a90c93b3ff77..ee0cdfd2cdbe 100644 --- a/ios/MullvadVPNUITests/Pages/ProblemReportSubmittedPage.swift +++ b/ios/MullvadVPNUITests/Pages/ProblemReportSubmittedPage.swift @@ -13,7 +13,7 @@ class ProblemReportSubmittedPage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - pageAccessibilityIdentifier = .problemReportSubmittedView + self.pageElement = app.otherElements[.problemReportSubmittedView] waitForPageToBeShown() } } diff --git a/ios/MullvadVPNUITests/Pages/RevokedDevicePage.swift b/ios/MullvadVPNUITests/Pages/RevokedDevicePage.swift index 4b8dd6dd7206..8225cb008fa9 100644 --- a/ios/MullvadVPNUITests/Pages/RevokedDevicePage.swift +++ b/ios/MullvadVPNUITests/Pages/RevokedDevicePage.swift @@ -13,7 +13,7 @@ class RevokedDevicePage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .revokedDeviceView + self.pageElement = app.otherElements[.revokedDeviceView] waitForPageToBeShown() } diff --git a/ios/MullvadVPNUITests/Pages/SelectLocationPage.swift b/ios/MullvadVPNUITests/Pages/SelectLocationPage.swift index d4126083ca08..8c87309a04fd 100644 --- a/ios/MullvadVPNUITests/Pages/SelectLocationPage.swift +++ b/ios/MullvadVPNUITests/Pages/SelectLocationPage.swift @@ -13,7 +13,7 @@ class SelectLocationPage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .selectLocationView + self.pageElement = app.otherElements[.selectLocationView] waitForPageToBeShown() } @@ -68,8 +68,18 @@ class SelectLocationPage: Page { } @discardableResult func tapCustomListEllipsisButton() -> Self { - let customListEllipsisButton = app.buttons[AccessibilityIdentifier.openCustomListsMenuButton] - customListEllipsisButton.tap() + let customListEllipsisButtons = app.buttons + .matching(identifier: AccessibilityIdentifier.openCustomListsMenuButton.rawValue).allElementsBoundByIndex + + // This is a workaround for an issue we have with the ellipsis showing up multiple times in the accessibility hieararchy even though in the view hierarchy there is only one + // Only the actually visual one is hittable, so only the visible button will be tapped + for ellipsisButton in customListEllipsisButtons where ellipsisButton.isHittable { + ellipsisButton.tap() + return self + } + + XCTFail("Found no hittable custom list ellipsis button") + return self } diff --git a/ios/MullvadVPNUITests/Pages/SettingsPage.swift b/ios/MullvadVPNUITests/Pages/SettingsPage.swift index db28fb33c929..8d40154abf7d 100644 --- a/ios/MullvadVPNUITests/Pages/SettingsPage.swift +++ b/ios/MullvadVPNUITests/Pages/SettingsPage.swift @@ -13,7 +13,7 @@ class SettingsPage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .settingsContainerView + self.pageElement = app.otherElements[.settingsContainerView] waitForPageToBeShown() } diff --git a/ios/MullvadVPNUITests/Pages/TermsOfServicePage.swift b/ios/MullvadVPNUITests/Pages/TermsOfServicePage.swift index 0f32bf7be6be..2654602494be 100644 --- a/ios/MullvadVPNUITests/Pages/TermsOfServicePage.swift +++ b/ios/MullvadVPNUITests/Pages/TermsOfServicePage.swift @@ -13,7 +13,7 @@ class TermsOfServicePage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .termsOfServiceView + self.pageElement = app.otherElements[.termsOfServiceView] waitForPageToBeShown() } diff --git a/ios/MullvadVPNUITests/Pages/TunnelControlPage.swift b/ios/MullvadVPNUITests/Pages/TunnelControlPage.swift index 4a2eeee4c790..5a52c3129716 100644 --- a/ios/MullvadVPNUITests/Pages/TunnelControlPage.swift +++ b/ios/MullvadVPNUITests/Pages/TunnelControlPage.swift @@ -70,7 +70,7 @@ class TunnelControlPage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .tunnelControlView + self.pageElement = app.otherElements[.tunnelControlView] waitForPageToBeShown() } @@ -94,9 +94,25 @@ class TunnelControlPage: Page { return self } + /// Tap either cancel or disconnect button, depending on the current connection state. Use this function sparingly when it's irrelevant whether the app is currently connecting to a relay or already connected. + @discardableResult func tapCancelOrDisconnectButton() -> Self { + let cancelButton = app.buttons[.cancelButton] + let disconnectButton = app.buttons[.disconnectButton] + + if disconnectButton.exists && disconnectButton.isHittable { + disconnectButton.tap() + } else { + cancelButton.tap() + } + + return self + } + @discardableResult func waitForSecureConnectionLabel() -> Self { - _ = app.staticTexts[AccessibilityIdentifier.connectionStatusConnectedLabel] - .waitForExistence(timeout: BaseUITestCase.defaultTimeout) + let labelFound = app.staticTexts[.connectionStatusConnectedLabel] + .waitForExistence(timeout: BaseUITestCase.extremelyLongTimeout) + XCTAssertTrue(labelFound, "Secure connection label presented") + return self } @@ -193,4 +209,15 @@ class TunnelControlPage: Page { return String() } } + + func getCurrentRelayName() -> String { + let relayExpandButton = app.buttons[.relayStatusCollapseButton] + + guard let relayName = relayExpandButton.value as? String else { + XCTFail("Failed to read relay name from tunnel control page") + return String() + } + + return relayName + } } diff --git a/ios/MullvadVPNUITests/Pages/VPNSettingsPage.swift b/ios/MullvadVPNUITests/Pages/VPNSettingsPage.swift index 71adb3eaaa3f..a3c854ee8032 100644 --- a/ios/MullvadVPNUITests/Pages/VPNSettingsPage.swift +++ b/ios/MullvadVPNUITests/Pages/VPNSettingsPage.swift @@ -17,7 +17,7 @@ class VPNSettingsPage: Page { private func cellExpandCollapseButton(_ cellAccessiblityIdentifier: AccessibilityIdentifier) -> XCUIElement { let table = app.tables[AccessibilityIdentifier.vpnSettingsTableView] let matchingCells = table.otherElements.containing(.any, identifier: cellAccessiblityIdentifier.rawValue) - let expandButton = matchingCells.buttons[AccessibilityIdentifier.collapseButton] + let expandButton = matchingCells.buttons[AccessibilityIdentifier.expandButton] return expandButton } diff --git a/ios/MullvadVPNUITests/Pages/WelcomePage.swift b/ios/MullvadVPNUITests/Pages/WelcomePage.swift index d6e8048413b1..2a1ca9df2967 100644 --- a/ios/MullvadVPNUITests/Pages/WelcomePage.swift +++ b/ios/MullvadVPNUITests/Pages/WelcomePage.swift @@ -13,7 +13,7 @@ class WelcomePage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .welcomeView + self.pageElement = app.otherElements[.welcomeView] waitForPageToBeShown() } diff --git a/ios/MullvadVPNUITests/README.md b/ios/MullvadVPNUITests/README.md index a8146b24d91d..7fbbc33e80ec 100644 --- a/ios/MullvadVPNUITests/README.md +++ b/ios/MullvadVPNUITests/README.md @@ -4,6 +4,8 @@ 1. Make sure device is added to provisioning profiles 2. Disable passcode in iOS settings - otherwise tests cannot be started without manually entering passcode 3. Make sure device is configured in GitHub secrets(see *GitHub setup* below) +4. Make sure the test device is connected to the WiFi `app-team-ios-tests` +5. Make sure iCloud syncing of keychain is off on the device so that the device isn't getting WiFi passwords from another device causing it to sometimes connect to another WiFi. ## Set up of runner environment 1. Install Xcode @@ -34,6 +36,7 @@ - `IOS_NO_TIME_ACCOUNT_NUMBER` - Production server account with time added to it - `IOS_TEST_DEVICE_IDENTIFIER_UUID` - unique identifier for the test device. Create new identifier with `uuidgen`. - `IOS_TEST_DEVICE_UDID` - the iOS device's UDID. + - `PARTNER_API_TOKEN` - secret token for partner API. Optional and only intended to be used in CI when running tests against staging environment. ## Test plans There are a few different test plans which are mainly to be triggered by GitHub action workflows but can also be triggered manually with Xcode: diff --git a/ios/MullvadVPNUITests/RelayTests.swift b/ios/MullvadVPNUITests/RelayTests.swift index c8848e2a46cb..13cf9f31cd05 100644 --- a/ios/MullvadVPNUITests/RelayTests.swift +++ b/ios/MullvadVPNUITests/RelayTests.swift @@ -26,17 +26,16 @@ class RelayTests: LoggedInWithTimeUITestCase { } } - func testAppConnection() throws { - TunnelControlPage(app) - .tapSecureConnectionButton() - - allowAddVPNConfigurationsIfAsked() + /// Restore default country by selecting it in location selector and immediately disconnecting when app starts connecting to relay in it + private func restoreDefaultCountry() { + TunnelControlPage(self.app) + .tapSelectLocationButton() - TunnelControlPage(app) - .waitForSecureConnectionLabel() + SelectLocationPage(self.app) + .tapLocationCell(withName: BaseUITestCase.appDefaultCountry) - try Networking.verifyCanAccessInternet() - Networking.verifyConnectedThroughMullvad() + TunnelControlPage(self.app) + .tapCancelOrDisconnectButton() } func testAdBlockingViaDNS() throws { @@ -84,12 +83,29 @@ class RelayTests: LoggedInWithTimeUITestCase { .tapDisconnectButton() } + func testAppConnection() throws { + TunnelControlPage(app) + .tapSecureConnectionButton() + + allowAddVPNConfigurationsIfAsked() + + TunnelControlPage(app) + .waitForSecureConnectionLabel() + + try Networking.verifyCanAccessInternet() + try Networking.verifyConnectedThroughMullvad() + } + func testConnectionRetryLogic() throws { FirewallAPIClient().removeRules() removeFirewallRulesInTearDown = true + addTeardownBlock { + self.restoreDefaultCountry() + } + // First get relay IP address - let relayIPAddress = getGot001WireGuardRelayIPAddress() + let relayIPAddress = getDefaultRelayIPAddress() // Run actual test try FirewallAPIClient().createRule( @@ -97,7 +113,10 @@ class RelayTests: LoggedInWithTimeUITestCase { ) TunnelControlPage(app) - .tapSecureConnectionButton() + .tapSelectLocationButton() + + SelectLocationPage(app) + .tapLocationCell(withName: BaseUITestCase.testsDefaultRelay) // Should be two UDP connection attempts but sometimes only one is shown in the UI TunnelControlPage(app) @@ -151,8 +170,12 @@ class RelayTests: LoggedInWithTimeUITestCase { FirewallAPIClient().removeRules() removeFirewallRulesInTearDown = true + addTeardownBlock { + self.restoreDefaultCountry() + } + // First get relay IP address - let relayIPAddress = getGot001WireGuardRelayIPAddress() + let relayIPAddress = getDefaultRelayIPAddress() // Run actual test try FirewallAPIClient().createRule( @@ -174,7 +197,10 @@ class RelayTests: LoggedInWithTimeUITestCase { .tapDoneButton() TunnelControlPage(app) - .tapSecureConnectionButton() + .tapSelectLocationButton() + + SelectLocationPage(app) + .tapLocationCell(withName: BaseUITestCase.testsDefaultRelay) // Should be two UDP connection attempts but sometimes only one is shown in the UI TunnelControlPage(app) @@ -208,25 +234,24 @@ class RelayTests: LoggedInWithTimeUITestCase { TunnelControlPage(app) .tapRelayStatusExpandCollapseButton() .verifyConnectingToPort("4001") + .waitForSecureConnectionLabel() .tapDisconnectButton() } - /// Get got001 WireGuard relay IP address by connecting to it and checking which IP address the app connects to. Assumes user is logged on and at tunnel control page. - private func getGot001WireGuardRelayIPAddress() -> String { - let wireGuardGot001RelayName = "se-got-wg-001" - + /// Get Gothenburg relay IP address and name by connecting to any Gothenburg relay and checking relay and IP address the app connects to. Assumes user is logged on and at tunnel control page. + private func getDefaultRelayIPAddress() -> String { TunnelControlPage(app) .tapSelectLocationButton() - if SelectLocationPage(app).locationCellIsExpanded("Sweden") { - // Already expanded - just make sure correct relay is selected + if SelectLocationPage(app).locationCellIsExpanded(BaseUITestCase.testsDefaultCountry) { + // Already expanded - just make sure the correct city cell is selected SelectLocationPage(app) - .tapLocationCell(withName: wireGuardGot001RelayName) + .tapLocationCell(withName: BaseUITestCase.testsDefaultCity) } else { SelectLocationPage(app) - .tapLocationCellExpandButton(withName: "Sweden") - .tapLocationCellExpandButton(withName: "Gothenburg") - .tapLocationCell(withName: wireGuardGot001RelayName) + .tapLocationCellExpandButton(withName: BaseUITestCase.testsDefaultCountry) + .tapLocationCellExpandButton(withName: BaseUITestCase.testsDefaultCity) + .tapLocationCell(withName: BaseUITestCase.testsDefaultRelay) } allowAddVPNConfigurationsIfAsked() diff --git a/ios/MullvadVPNUITests/SettingsMigrationTests.swift b/ios/MullvadVPNUITests/SettingsMigrationTests.swift index 35d46b8c87cb..83e142f202ee 100644 --- a/ios/MullvadVPNUITests/SettingsMigrationTests.swift +++ b/ios/MullvadVPNUITests/SettingsMigrationTests.swift @@ -29,6 +29,12 @@ class SettingsMigrationTests: BaseUITestCase { } func testChangeCustomDNSSettings() { + let hasTimeAccountNumber = getAccountWithTime() + + addTeardownBlock { + self.returnAccountWithTime(accountNumber: hasTimeAccountNumber) + } + logoutIfLoggedIn() login(accountNumber: hasTimeAccountNumber) @@ -52,6 +58,12 @@ class SettingsMigrationTests: BaseUITestCase { } func testChangeSettings() { + let hasTimeAccountNumber = getAccountWithTime() + + addTeardownBlock { + self.returnAccountWithTime(accountNumber: hasTimeAccountNumber) + } + logoutIfLoggedIn() login(accountNumber: hasTimeAccountNumber) diff --git a/ios/MullvadVPNUITests/Test base classes/BaseUITestCase.swift b/ios/MullvadVPNUITests/Test base classes/BaseUITestCase.swift index 3551f482feec..f834c0dc97a2 100644 --- a/ios/MullvadVPNUITests/Test base classes/BaseUITestCase.swift +++ b/ios/MullvadVPNUITests/Test base classes/BaseUITestCase.swift @@ -13,20 +13,86 @@ class BaseUITestCase: XCTestCase { let app = XCUIApplication() static let defaultTimeout = 5.0 static let longTimeout = 15.0 - static let veryLongTimeout = 60.0 + static let veryLongTimeout = 20.0 + static let extremelyLongTimeout = 180.0 static let shortTimeout = 1.0 + /// The apps default country - the preselected country location after fresh install + static let appDefaultCountry = "Sweden" + + /// Default country to use in tests. + static let testsDefaultCountry = "Germany" + + /// Default city to use in tests + static let testsDefaultCity = "Frankfurt" + + /// Default relay to use in tests + static let testsDefaultRelay = "de-fra-wg-001" + // swiftlint:disable force_cast let displayName = Bundle(for: BaseUITestCase.self) .infoDictionary?["DisplayName"] as! String - let hasTimeAccountNumber = Bundle(for: BaseUITestCase.self) - .infoDictionary?["HasTimeAccountNumber"] as! String - let fiveWireGuardKeysAccountNumber = Bundle(for: BaseUITestCase.self) - .infoDictionary?["FiveWireGuardKeysAccountNumber"] as! String + private let bundleHasTimeAccountNumber = Bundle(for: BaseUITestCase.self) + .infoDictionary?["HasTimeAccountNumber"] as? String + private let bundleNoTimeAccountNumber = Bundle(for: BaseUITestCase.self) + .infoDictionary?["NoTimeAccountNumber"] as? String let iOSDevicePinCode = Bundle(for: BaseUITestCase.self) .infoDictionary?["IOSDevicePinCode"] as! String + let attachAppLogsOnFailure = Bundle(for: BaseUITestCase.self) + .infoDictionary?["AttachAppLogsOnFailure"] as! String == "1" // swiftlint:enable force_cast + /// Get an account number with time. If an account with time is specified in the configuration file that account will be used, else a temporary account will be created. + func getAccountWithTime() -> String { + if let configuredAccountWithTime = bundleHasTimeAccountNumber, !configuredAccountWithTime.isEmpty { + return configuredAccountWithTime + } else { + let partnerAPIClient = PartnerAPIClient() + let accountNumber = partnerAPIClient.createAccount() + _ = partnerAPIClient.addTime(accountNumber: accountNumber, days: 1) + return accountNumber + } + } + + /// Return account with time after done using it This is neccessary because if a temporary account was created we want to delete it. + func returnAccountWithTime(accountNumber: String) { + if bundleHasTimeAccountNumber?.isEmpty == true { + PartnerAPIClient().deleteAccount(accountNumber: accountNumber) + } + } + + /// Get an account number without time. If an account without time is specified in the configuration file that account will be used, else a temporary account will be created. + func getAccountWithoutTime() -> String { + if let configuredAccountWithoutTime = bundleNoTimeAccountNumber, !configuredAccountWithoutTime.isEmpty { + return configuredAccountWithoutTime + } else { + let partnerAPIClient = PartnerAPIClient() + let accountNumber = partnerAPIClient.createAccount() + return accountNumber + } + } + + func returnAccountWithoutTime(accountNumber: String) { + if bundleNoTimeAccountNumber?.isEmpty == true { + PartnerAPIClient().deleteAccount(accountNumber: accountNumber) + } + } + + /// Handle iOS add VPN configuration permission alert - allow and enter device PIN code + func allowAddVPNConfigurations() { + let springboard = XCUIApplication(bundleIdentifier: "com.apple.springboard") + + let alertAllowButton = springboard.buttons.element(boundBy: 0) + if alertAllowButton.waitForExistence(timeout: Self.defaultTimeout) { + alertAllowButton.tap() + } + + if iOSDevicePinCode.isEmpty == false { + _ = springboard.buttons["1"].waitForExistence(timeout: Self.defaultTimeout) + springboard.typeText(iOSDevicePinCode) + } + } + /// Handle iOS add VPN configuration permission alert if presented, otherwise ignore func allowAddVPNConfigurationsIfAsked() { let springboard = XCUIApplication(bundleIdentifier: "com.apple.springboard") @@ -63,7 +129,7 @@ class BaseUITestCase: XCTestCase { return true } - /// Suite level teardown ran after test have executed + /// Suite level teardown ran after all tests in suite have been executed override class func tearDown() { if shouldUninstallAppInTeardown() { uninstallApp() @@ -79,6 +145,32 @@ class BaseUITestCase: XCTestCase { /// Test level teardown override func tearDown() { app.terminate() + + if let testRun = self.testRun, testRun.failureCount > 0, attachAppLogsOnFailure == true { + app.launch() + + HeaderBar(app) + .tapSettingsButton() + + SettingsPage(app) + .tapReportAProblemCell() + + ProblemReportPage(app) + .tapViewAppLogsButton() + + let logText = AppLogsPage(app) + .getAppLogText() + + // Attach app log to result + let dateFormatter = DateFormatter() + dateFormatter.dateFormat = "yyyy-MM-dd_HH-mm-ss" + let dateString = dateFormatter.string(from: Date()) + let attachment = XCTAttachment(string: logText) + attachment.name = "app-log-\(dateString).log" + add(attachment) + + app.terminate() + } } /// Check if currently logged on to an account. Note that it is assumed that we are logged in if login view isn't currently shown. @@ -96,10 +188,9 @@ class BaseUITestCase: XCTestCase { func agreeToTermsOfServiceIfShown() { let termsOfServiceIsShown = app.otherElements[ - AccessibilityIdentifier - .termsOfServiceView.rawValue + .termsOfServiceView ] - .waitForExistence(timeout: 1) + .waitForExistence(timeout: Self.shortTimeout) if termsOfServiceIsShown { TermsOfServicePage(app) @@ -109,8 +200,8 @@ class BaseUITestCase: XCTestCase { func dismissChangeLogIfShown() { let changeLogIsShown = app - .otherElements[AccessibilityIdentifier.changeLogAlert.rawValue] - .waitForExistence(timeout: 1.0) + .otherElements[.changeLogAlert] + .waitForExistence(timeout: Self.shortTimeout) if changeLogIsShown { ChangeLogAlert(app) @@ -125,10 +216,28 @@ class BaseUITestCase: XCTestCase { /// Login with specified account number. It is a prerequisite that the login page is currently shown. func login(accountNumber: String) { + var successIconShown = false + var retryCount = 0 + let maxRetryCount = 3 + LoginPage(app) .tapAccountNumberTextField() .enterText(accountNumber) - .tapAccountNumberSubmitButton() + + repeat { + successIconShown = LoginPage(app) + .tapAccountNumberSubmitButton() + .getSuccessIconShown() + + if successIconShown == false { + // Give it some time to show up. App might be waiting for a network connection to timeout. + LoginPage(app).waitForAccountNumberSubmitButton() + } + + retryCount += 1 + } while successIconShown == false && retryCount < maxRetryCount + + HeaderBar(app) .verifyDeviceLabelShown() } @@ -145,6 +254,7 @@ class BaseUITestCase: XCTestCase { .tapAccountButton() AccountPage(app) .tapLogOutButton() + .waitForLogoutSpinnerToDisappear() } else { // Workaround for revoked device view not showing account button RevokedDevicePage(app) @@ -157,13 +267,18 @@ class BaseUITestCase: XCTestCase { static func uninstallApp() { let appName = "Mullvad VPN" + let searchQuery = appName + .replacingOccurrences( + of: " ", + with: "" + ) // With space in the query Spotlight search sometimes don't match the Mullvad VPN app let timeout = TimeInterval(5) let springboard = XCUIApplication(bundleIdentifier: "com.apple.springboard") let spotlight = XCUIApplication(bundleIdentifier: "com.apple.Spotlight") springboard.swipeDown() - spotlight.textFields["SpotlightSearchField"].typeText(appName) + spotlight.textFields["SpotlightSearchField"].typeText(searchQuery) let appIcon = spotlight.icons[appName].firstMatch if appIcon.waitForExistence(timeout: timeout) { diff --git a/ios/MullvadVPNUITests/Test base classes/LoggedInWithTimeUITestCase.swift b/ios/MullvadVPNUITests/Test base classes/LoggedInWithTimeUITestCase.swift index fcf6d0f5e2cb..f5c1c011c564 100644 --- a/ios/MullvadVPNUITests/Test base classes/LoggedInWithTimeUITestCase.swift +++ b/ios/MullvadVPNUITests/Test base classes/LoggedInWithTimeUITestCase.swift @@ -7,20 +7,42 @@ // import Foundation +import XCTest /// Base class for tests that should start from a state of being logged on to an account with time left class LoggedInWithTimeUITestCase: BaseUITestCase { + var hasTimeAccountNumber: String? + override func setUp() { super.setUp() + hasTimeAccountNumber = getAccountWithTime() + agreeToTermsOfServiceIfShown() dismissChangeLogIfShown() logoutIfLoggedIn() + guard let hasTimeAccountNumber = self.hasTimeAccountNumber else { + XCTFail("hasTimeAccountNumber unexpectedly not set") + return + } + login(accountNumber: hasTimeAccountNumber) // Relaunch app so that tests start from a deterministic state app.terminate() app.launch() } + + override func tearDown() { + super.tearDown() + + guard let hasTimeAccountNumber = self.hasTimeAccountNumber else { + XCTFail("hasTimeAccountNumber unexpectedly not set") + return + } + + app.terminate() // Terminate app to make sure we have Internet connectivity + self.returnAccountWithTime(accountNumber: hasTimeAccountNumber) + } }