From b683f6131f4a88960f616ace118f10f02a0362f1 Mon Sep 17 00:00:00 2001 From: Niklas Berglund Date: Wed, 24 Apr 2024 14:56:31 +0200 Subject: [PATCH] Move iOS end to end tests to staging environment --- .../actions/ios-end-to-end-tests/action.yml | 38 ++++- ...os-end-to-end-tests-settings-migration.yml | 4 + .github/workflows/ios-end-to-end-tests.yml | 22 ++- ios/Configurations/Api.xcconfig.template | 2 +- ios/Configurations/UITests.xcconfig.template | 10 +- ios/MullvadVPN.xcodeproj/project.pbxproj | 10 +- .../xcschemes/MullvadVPNUITests.xcscheme | 4 +- .../Classes/AccessbilityIdentifier.swift | 10 ++ .../Cells/SwitchCellContentView.swift | 2 +- ios/MullvadVPN/Supporting Files/Info.plist | 5 + .../ProblemReportReviewViewController.swift | 5 + ...mReportViewController+ViewManagement.swift | 3 +- .../RelayFilter/RelayFilterChipView.swift | 4 + .../Tunnel/TunnelControlView.swift | 2 + .../VPNSettings/CustomDNSViewController.swift | 1 + ios/MullvadVPN/Views/StatusActivityView.swift | 3 + ios/MullvadVPN/Views/StatusImageView.swift | 17 +++ ios/MullvadVPNUITests/AccountTests.swift | 70 ++++++--- ios/MullvadVPNUITests/ConnectivityTests.swift | 54 ++++++- ios/MullvadVPNUITests/CustomListsTests.swift | 13 -- ios/MullvadVPNUITests/Info.plist | 8 +- .../Networking/MullvadAPIWrapper.swift | 141 ++++++++++++++---- .../Networking/Networking.swift | 46 ++++-- .../Networking/PartnerAPIClient.swift | 123 +++++++++++++++ .../Pages/APIAccessPage.swift | 2 +- .../Pages/AccountDeletionPage.swift | 9 +- ios/MullvadVPNUITests/Pages/AccountPage.swift | 12 +- .../Pages/AddAccessMethodPage.swift | 6 +- ios/MullvadVPNUITests/Pages/Alert.swift | 2 +- ios/MullvadVPNUITests/Pages/AppLogsPage.swift | 36 +++++ .../Pages/ChangeLogAlert.swift | 2 +- .../Pages/CustomListPage.swift | 4 +- .../Pages/DNSSettingsPage.swift | 5 +- .../Pages/DeviceManagementPage.swift | 5 +- .../Pages/EditAccessMethodPage.swift | 3 +- .../Pages/EditCustomListLocationsPage.swift | 2 +- ios/MullvadVPNUITests/Pages/HeaderBar.swift | 11 +- .../Pages/ListCustomListsPage.swift | 2 +- ios/MullvadVPNUITests/Pages/LoginPage.swift | 49 ++++-- .../Pages/OutOfTimePage.swift | 2 +- ios/MullvadVPNUITests/Pages/Page.swift | 13 +- .../Pages/ProblemReportPage.swift | 5 +- .../Pages/ProblemReportSubmittedPage.swift | 2 +- .../Pages/RevokedDevicePage.swift | 2 +- .../Pages/SelectLocationPage.swift | 16 +- .../Pages/SettingsPage.swift | 2 +- .../Pages/TermsOfServicePage.swift | 2 +- .../Pages/TunnelControlPage.swift | 33 +++- .../Pages/VPNSettingsPage.swift | 2 +- ios/MullvadVPNUITests/Pages/WelcomePage.swift | 2 +- ios/MullvadVPNUITests/README.md | 3 + ios/MullvadVPNUITests/RelayTests.swift | 89 +++++++---- .../SettingsMigrationTests.swift | 12 ++ .../Test base classes/BaseUITestCase.swift | 141 ++++++++++++++++-- .../LoggedInWithTimeUITestCase.swift | 22 +++ 55 files changed, 891 insertions(+), 204 deletions(-) create mode 100644 ios/MullvadVPNUITests/Networking/PartnerAPIClient.swift create mode 100644 ios/MullvadVPNUITests/Pages/AppLogsPage.swift diff --git a/.github/actions/ios-end-to-end-tests/action.yml b/.github/actions/ios-end-to-end-tests/action.yml index 9d771d151c5a..fa2d198a44a5 100644 --- a/.github/actions/ios-end-to-end-tests/action.yml +++ b/.github/actions/ios-end-to-end-tests/action.yml @@ -19,10 +19,22 @@ inputs: xcode_test_plan: description: 'Xcode Test Plan to run' required: true + partner_api_token: + description: 'Partner API Token' + required: true + test_name: + description: 'Test case/suite name. Will run all tests in the test plan if not provided.' + required: false runs: using: 'composite' steps: + - name: Make sure app is not installed + run: ios-deploy --id $IOS_TEST_DEVICE_UDID --uninstall_only --bundle_id net.mullvad.MullvadVPN + shell: bash + env: + IOS_TEST_DEVICE_UDID: ${{ inputs.test_device_udid }} + - name: Configure Xcode project run: | for file in *.xcconfig.template ; do cp $file ${file//.template/} ; done @@ -34,8 +46,12 @@ runs: sed -i "" \ "/TEST_DEVICE_IDENTIFIER_UUID =/ s/= .*/= $TEST_DEVICE_IDENTIFIER_UUID/" \ UITests.xcconfig - echo -e "\nHAS_TIME_ACCOUNT_NUMBER = $HAS_TIME_ACCOUNT_NUMBER" >> UITests.xcconfig - echo "NO_TIME_ACCOUNT_NUMBER = $NO_TIME_ACCOUNT_NUMBER" >> UITests.xcconfig + sed -i "" \ + "/PARTNER_API_TOKEN =/ s#= .*#= $PARTNER_API_TOKEN#" \ + UITests.xcconfig + sed -i "" \ + "/ATTACH_APP_LOGS_ON_FAILURE =/ s#= .*#= 1#" \ + UITests.xcconfig shell: bash working-directory: ios/Configurations env: @@ -43,17 +59,31 @@ runs: TEST_DEVICE_IDENTIFIER_UUID: ${{ inputs.test_device_identifier_uuid }} HAS_TIME_ACCOUNT_NUMBER: ${{ inputs.has_time_account_number }} NO_TIME_ACCOUNT_NUMBER: ${{ inputs.no_time_account_number }} + PARTNER_API_TOKEN: ${{ inputs.partner_api_token }} - name: Run end-to-end-tests run: | + if [ -n "$TEST_NAME" ]; then + TEST_NAME_ARGUMENT=" -only-testing $TEST_NAME" + else + TEST_NAME_ARGUMENT="" + fi set -o pipefail && env NSUnbufferedIO=YES xcodebuild \ -project MullvadVPN.xcodeproj \ -scheme MullvadVPNUITests \ - -testPlan $XCODE_TEST_PLAN \ + -testPlan $XCODE_TEST_PLAN $TEST_NAME_ARGUMENT \ + -resultBundlePath xcode-test-report \ -destination "platform=iOS,id=$TEST_DEVICE_UDID" \ - clean test 2>&1 | xcbeautify --report junit --report-path test-report + clean test 2>&1 | xcbeautify --report junit --report-path junit-test-report shell: bash working-directory: ios/ env: XCODE_TEST_PLAN: ${{ inputs.xcode_test_plan }} TEST_DEVICE_UDID: ${{ inputs.test_device_udid }} + TEST_NAME: ${{ inputs.test_name }} + + - name: Uninstall app if still installed + run: ios-deploy --id $IOS_TEST_DEVICE_UDID --uninstall_only --bundle_id net.mullvad.MullvadVPN + shell: bash + env: + IOS_TEST_DEVICE_UDID: ${{ inputs.test_device_udid }} diff --git a/.github/workflows/ios-end-to-end-tests-settings-migration.yml b/.github/workflows/ios-end-to-end-tests-settings-migration.yml index a5a27fd6454e..2ff8e1a2bad2 100644 --- a/.github/workflows/ios-end-to-end-tests-settings-migration.yml +++ b/.github/workflows/ios-end-to-end-tests-settings-migration.yml @@ -42,6 +42,7 @@ jobs: no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} xcode_test_plan: 'MullvadVPNUITestsChangeDNSSettings' + partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }} - name: Store test report for changing DNS settings uses: actions/upload-artifact@v4 @@ -62,6 +63,7 @@ jobs: has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }} no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} + partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }} xcode_test_plan: 'MullvadVPNUITestsVerifyDNSSettingsChanged' - name: Store test report for verifying DNS settings @@ -85,6 +87,7 @@ jobs: has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }} no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} + partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }} xcode_test_plan: 'MullvadVPNUITestsChangeSettings' - name: Store test report for changing all settings @@ -106,6 +109,7 @@ jobs: has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }} no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} + partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }} xcode_test_plan: 'MullvadVPNUITestsVerifySettingsChanged' - name: Store test report for verifying all other settings diff --git a/.github/workflows/ios-end-to-end-tests.yml b/.github/workflows/ios-end-to-end-tests.yml index 64d494b8ab29..8d1c8178d072 100644 --- a/.github/workflows/ios-end-to-end-tests.yml +++ b/.github/workflows/ios-end-to-end-tests.yml @@ -14,6 +14,12 @@ on: - .github/workflows/ios-end-to-end-tests.yml - ios/** workflow_dispatch: + inputs: + # Optionally specify a test case or suite to run. + # Must be in the format MullvadVPNUITest// where test case name is optional. + test_name: + description: 'Only run test case/suite' + required: false schedule: # At midnight every day. # Notifications for scheduled workflows are sent to the user who last modified the cron @@ -26,6 +32,7 @@ jobs: if: github.event.pull_request.merged || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' name: End to end tests runs-on: [self-hosted, macOS, ios-test] + timeout-minutes: 60 steps: - name: Configure Rust uses: actions-rs/toolchain@v1 @@ -50,11 +57,13 @@ jobs: uses: ./.github/actions/ios-end-to-end-tests with: xcode_test_plan: ${{ env.XCODE_TEST_PLAN }} + test_name: ${{ github.event.inputs.test_name }} ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }} test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }} has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }} no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} + partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }} - name: Comment PR on test failure if: failure() && github.event_name == 'pull_request' @@ -76,5 +85,14 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: test-report - path: ios/test-report/junit.xml + name: test-results + path: | + ios/junit-test-report/junit.xml + ios/xcode-test-report.xcresult + + - name: Store app log artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: app-logs + path: ios/xcode-test-report/**/app-log-*.log diff --git a/ios/Configurations/Api.xcconfig.template b/ios/Configurations/Api.xcconfig.template index 99d1fc1fc13a..ff39567be69c 100644 --- a/ios/Configurations/Api.xcconfig.template +++ b/ios/Configurations/Api.xcconfig.template @@ -6,4 +6,4 @@ API_HOST_NAME[config=Staging] = api.stagemole.eu API_ENDPOINT[config=Debug] = 45.83.223.196:443 API_ENDPOINT[config=Release] = 45.83.223.196:443 API_ENDPOINT[config=MockRelease] = 45.83.223.196:443 -API_ENDPOINT[config=Staging] = 85.203.53.95:443 +API_ENDPOINT[config=Staging] = 185.217.116.129:443 diff --git a/ios/Configurations/UITests.xcconfig.template b/ios/Configurations/UITests.xcconfig.template index e39224eedfe0..030fe46d3129 100644 --- a/ios/Configurations/UITests.xcconfig.template +++ b/ios/Configurations/UITests.xcconfig.template @@ -6,10 +6,12 @@ IOS_DEVICE_PIN_CODE = // UUID to identify test runs. Should be unique per test device. Generate with for example uuidgen on macOS. TEST_DEVICE_IDENTIFIER_UUID = +// Base64 encoded token for the partner API. Will only be used if account numbers are not configured. +PARTNER_API_TOKEN = + // Mullvad accounts used by UI tests HAS_TIME_ACCOUNT_NUMBER[config=Debug] = HAS_TIME_ACCOUNT_NUMBER[config=Staging] = -FIVE_WIREGUARD_KEYS_ACCOUNT_NUMBER = // Ad serving domain used when testing ad blocking. Note that we are assuming there's an HTTP server running on the host. AD_SERVING_DOMAIN = vpnlist.to @@ -19,3 +21,9 @@ SHOULD_BE_REACHABLE_DOMAIN = mullvad.net // Base URL for the firewall API, Note that // will be treated as a comment, therefor you need to insert a ${} between the slashes for example http:/${}/8.8.8.8 FIREWALL_API_BASE_URL = http:/${}/8.8.8.8 + +// URL for Mullvad provided JSON data with information about the connection. https://am.i.mullvad.net/json for production, https://am.i.stagemole.eu/json for staging. +AM_I_JSON_URL = https:/${}/am.i.stagemole.eu/json + +// Specify whether app logs should be extracted and attached to test report for failing tests +ATTACH_APP_LOGS_ON_FAILURE = 0 diff --git a/ios/MullvadVPN.xcodeproj/project.pbxproj b/ios/MullvadVPN.xcodeproj/project.pbxproj index 8e0a79894885..0b5adee78a0a 100644 --- a/ios/MullvadVPN.xcodeproj/project.pbxproj +++ b/ios/MullvadVPN.xcodeproj/project.pbxproj @@ -621,6 +621,7 @@ 850201DB2B503D7700EF8C96 /* RelayTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 850201DA2B503D7700EF8C96 /* RelayTests.swift */; }; 850201DD2B503D8C00EF8C96 /* SelectLocationPage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 850201DC2B503D8C00EF8C96 /* SelectLocationPage.swift */; }; 850201DF2B5040A500EF8C96 /* TunnelControlPage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 850201DE2B5040A500EF8C96 /* TunnelControlPage.swift */; }; + 85021CAE2BDBC4290098B400 /* AppLogsPage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 85021CAD2BDBC4290098B400 /* AppLogsPage.swift */; }; 85139B2D2B84B4A700734217 /* OutOfTimePage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 85139B2C2B84B4A700734217 /* OutOfTimePage.swift */; }; 852969282B4D9C1F007EAD4C /* AccountTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 852969272B4D9C1F007EAD4C /* AccountTests.swift */; }; 852969332B4E9232007EAD4C /* Page.swift in Sources */ = {isa = PBXBuildFile; fileRef = 852969322B4E9232007EAD4C /* Page.swift */; }; @@ -648,6 +649,7 @@ 8556EB542B9A1D7100D26DD4 /* BridgingHeader.h in Headers */ = {isa = PBXBuildFile; fileRef = 8556EB532B9A1D7100D26DD4 /* BridgingHeader.h */; }; 8556EB562B9B0AC500D26DD4 /* RevokedDevicePage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8556EB552B9B0AC500D26DD4 /* RevokedDevicePage.swift */; }; 855D9F5B2B63E56B00D7C64D /* ProblemReportPage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 855D9F5A2B63E56B00D7C64D /* ProblemReportPage.swift */; }; + 856952DC2BD2922A008C1F84 /* PartnerAPIClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 856952DB2BD2922A008C1F84 /* PartnerAPIClient.swift */; }; 856952E22BD6B04C008C1F84 /* XCUIElement+Extensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 856952E12BD6B04C008C1F84 /* XCUIElement+Extensions.swift */; }; 8585CBE32BC684180015B6A4 /* EditAccessMethodPage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8585CBE22BC684180015B6A4 /* EditAccessMethodPage.swift */; }; 8587A05D2B84D43100152938 /* ChangeLogAlert.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8587A05C2B84D43100152938 /* ChangeLogAlert.swift */; }; @@ -1952,6 +1954,7 @@ 850201DC2B503D8C00EF8C96 /* SelectLocationPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SelectLocationPage.swift; sourceTree = ""; }; 850201DE2B5040A500EF8C96 /* TunnelControlPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TunnelControlPage.swift; sourceTree = ""; }; 850201E22B51A93C00EF8C96 /* SettingsPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsPage.swift; sourceTree = ""; }; + 85021CAD2BDBC4290098B400 /* AppLogsPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppLogsPage.swift; sourceTree = ""; }; 85139B2C2B84B4A700734217 /* OutOfTimePage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OutOfTimePage.swift; sourceTree = ""; }; 852969252B4D9C1F007EAD4C /* MullvadVPNUITests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = MullvadVPNUITests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; 852969272B4D9C1F007EAD4C /* AccountTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AccountTests.swift; sourceTree = ""; }; @@ -1983,6 +1986,7 @@ 8556EB532B9A1D7100D26DD4 /* BridgingHeader.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BridgingHeader.h; sourceTree = ""; }; 8556EB552B9B0AC500D26DD4 /* RevokedDevicePage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RevokedDevicePage.swift; sourceTree = ""; }; 855D9F5A2B63E56B00D7C64D /* ProblemReportPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProblemReportPage.swift; sourceTree = ""; }; + 856952DB2BD2922A008C1F84 /* PartnerAPIClient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PartnerAPIClient.swift; sourceTree = ""; }; 856952E12BD6B04C008C1F84 /* XCUIElement+Extensions.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "XCUIElement+Extensions.swift"; sourceTree = ""; }; 8585CBE22BC684180015B6A4 /* EditAccessMethodPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EditAccessMethodPage.swift; sourceTree = ""; }; 8587A05C2B84D43100152938 /* ChangeLogAlert.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChangeLogAlert.swift; sourceTree = ""; }; @@ -3901,10 +3905,10 @@ 85557B0C2B591B0F00795FE1 /* Networking */, 852969312B4E9220007EAD4C /* Pages */, 850201DA2B503D7700EF8C96 /* RelayTests.swift */, - 856952E12BD6B04C008C1F84 /* XCUIElement+Extensions.swift */, 85D039972BA4711800940E7F /* SettingsMigrationTests.swift */, 85C7A2E82B89024B00035D5A /* SettingsTests.swift */, 8518F6392B601910009EB113 /* Test base classes */, + 856952E12BD6B04C008C1F84 /* XCUIElement+Extensions.swift */, 85557B152B5ABBBE00795FE1 /* XCUIElementQuery+Extensions.swift */, ); path = MullvadVPNUITests; @@ -3939,6 +3943,7 @@ 852D054C2BC3DE3A008578D2 /* APIAccessPage.swift */, 852D054E2BC43DF7008578D2 /* AddAccessMethodPage.swift */, 8585CBE22BC684180015B6A4 /* EditAccessMethodPage.swift */, + 85021CAD2BDBC4290098B400 /* AppLogsPage.swift */, ); path = Pages; sourceTree = ""; @@ -3950,6 +3955,7 @@ 85557B0F2B59215F00795FE1 /* FirewallRule.swift */, 85557B132B5983CF00795FE1 /* MullvadAPIWrapper.swift */, 85E3BDE42B70E18C00FA71FD /* Networking.swift */, + 856952DB2BD2922A008C1F84 /* PartnerAPIClient.swift */, ); path = Networking; sourceTree = ""; @@ -6078,6 +6084,7 @@ 8542F7532BCFBD050035C042 /* SelectLocationFilterPage.swift in Sources */, 850201DD2B503D8C00EF8C96 /* SelectLocationPage.swift in Sources */, 85D039982BA4711800940E7F /* SettingsMigrationTests.swift in Sources */, + 85021CAE2BDBC4290098B400 /* AppLogsPage.swift in Sources */, 850201DB2B503D7700EF8C96 /* RelayTests.swift in Sources */, 852D054D2BC3DE3A008578D2 /* APIAccessPage.swift in Sources */, 85139B2D2B84B4A700734217 /* OutOfTimePage.swift in Sources */, @@ -6096,6 +6103,7 @@ 852969282B4D9C1F007EAD4C /* AccountTests.swift in Sources */, 8587A05D2B84D43100152938 /* ChangeLogAlert.swift in Sources */, 85FB5A102B6960A30015DCED /* AccountDeletionPage.swift in Sources */, + 856952DC2BD2922A008C1F84 /* PartnerAPIClient.swift in Sources */, 85557B162B5ABBBE00795FE1 /* XCUIElementQuery+Extensions.swift in Sources */, 855D9F5B2B63E56B00D7C64D /* ProblemReportPage.swift in Sources */, 8529693A2B4F0238007EAD4C /* TermsOfServicePage.swift in Sources */, diff --git a/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme b/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme index a206bbdc7147..3258ef6f4722 100644 --- a/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme +++ b/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme @@ -7,7 +7,7 @@ buildImplicitDependencies = "YES"> @@ -46,7 +46,7 @@ NSExceptionDomains + 185.217.116.129 + + NSExceptionAllowsInsecureHTTPLoads + + 127.0.0.1 NSExceptionAllowsInsecureHTTPLoads diff --git a/ios/MullvadVPN/View controllers/ProblemReport/ProblemReportReviewViewController.swift b/ios/MullvadVPN/View controllers/ProblemReport/ProblemReportReviewViewController.swift index 603b6b34bbba..a12d32362a74 100644 --- a/ios/MullvadVPN/View controllers/ProblemReport/ProblemReportReviewViewController.swift +++ b/ios/MullvadVPN/View controllers/ProblemReport/ProblemReportReviewViewController.swift @@ -24,6 +24,8 @@ class ProblemReportReviewViewController: UIViewController { override func viewDidLoad() { super.viewDidLoad() + view.accessibilityIdentifier = .appLogsView + navigationItem.title = NSLocalizedString( "NAVIGATION_TITLE", tableName: "ProblemReportReview", @@ -37,6 +39,7 @@ class ProblemReportReviewViewController: UIViewController { self?.dismiss(animated: true) }) ) + navigationItem.rightBarButtonItem?.accessibilityIdentifier = .appLogsDoneButton #if DEBUG navigationItem.leftBarButtonItem = UIBarButtonItem( @@ -45,8 +48,10 @@ class ProblemReportReviewViewController: UIViewController { self?.share() }) ) + navigationItem.leftBarButtonItem?.accessibilityIdentifier = .appLogsShareButton #endif + textView.accessibilityIdentifier = .problemReportAppLogsTextView textView.translatesAutoresizingMaskIntoConstraints = false textView.isEditable = false textView.font = UIFont.monospacedSystemFont( diff --git a/ios/MullvadVPN/View controllers/ProblemReport/ProblemReportViewController+ViewManagement.swift b/ios/MullvadVPN/View controllers/ProblemReport/ProblemReportViewController+ViewManagement.swift index 488c48370fec..6d98e9c83bc6 100644 --- a/ios/MullvadVPN/View controllers/ProblemReport/ProblemReportViewController+ViewManagement.swift +++ b/ios/MullvadVPN/View controllers/ProblemReport/ProblemReportViewController+ViewManagement.swift @@ -88,6 +88,7 @@ extension ProblemReportViewController { func makeViewLogsButton() -> AppButton { let button = AppButton(style: .default) + button.accessibilityIdentifier = .problemReportAppLogsButton button.translatesAutoresizingMaskIntoConstraints = false button.setTitle(Self.persistentViewModel.viewLogsButtonTitle, for: .normal) button.addTarget(self, action: #selector(handleViewLogsButtonTap), for: .touchUpInside) @@ -96,7 +97,7 @@ extension ProblemReportViewController { func makeSendButton() -> AppButton { let button = AppButton(style: .success) - button.accessibilityIdentifier = AccessibilityIdentifier.problemReportSendButton + button.accessibilityIdentifier = .problemReportSendButton button.translatesAutoresizingMaskIntoConstraints = false button.setTitle(Self.persistentViewModel.sendLogsButtonTitle, for: .normal) button.addTarget(self, action: #selector(handleSendButtonTap), for: .touchUpInside) diff --git a/ios/MullvadVPN/View controllers/RelayFilter/RelayFilterChipView.swift b/ios/MullvadVPN/View controllers/RelayFilter/RelayFilterChipView.swift index 986281c9d617..67c73b0339b9 100644 --- a/ios/MullvadVPN/View controllers/RelayFilter/RelayFilterChipView.swift +++ b/ios/MullvadVPN/View controllers/RelayFilter/RelayFilterChipView.swift @@ -11,6 +11,7 @@ import UIKit class RelayFilterChipView: UIView { private let titleLabel: UILabel = { let label = UILabel() + label.accessibilityIdentifier = .relayFilterChipLabel label.font = UIFont.preferredFont(forTextStyle: .caption1) label.adjustsFontForContentSizeCategory = true label.textColor = .white @@ -22,11 +23,14 @@ class RelayFilterChipView: UIView { init() { super.init(frame: .zero) + self.accessibilityIdentifier = .relayFilterChipView + let closeButton = IncreasedHitButton() closeButton.setImage( UIImage(named: "IconCloseSml")?.withTintColor(.white.withAlphaComponent(0.6)), for: .normal ) + closeButton.accessibilityIdentifier = .relayFilterChipCloseButton closeButton.addTarget(self, action: #selector(didTapButton(_:)), for: .touchUpInside) let container = UIStackView(arrangedSubviews: [titleLabel, closeButton]) diff --git a/ios/MullvadVPN/View controllers/Tunnel/TunnelControlView.swift b/ios/MullvadVPN/View controllers/Tunnel/TunnelControlView.swift index 92acdb7a5542..16fd69b25b3a 100644 --- a/ios/MullvadVPN/View controllers/Tunnel/TunnelControlView.swift +++ b/ios/MullvadVPN/View controllers/Tunnel/TunnelControlView.swift @@ -189,6 +189,8 @@ final class TunnelControlView: UIView { switch tunnelState { case .connected: secureLabel.accessibilityIdentifier = .connectionStatusConnectedLabel + case .connecting: + secureLabel.accessibilityIdentifier = .connectionStatusConnectingLabel default: secureLabel.accessibilityIdentifier = .connectionStatusNotConnectedLabel } diff --git a/ios/MullvadVPN/View controllers/VPNSettings/CustomDNSViewController.swift b/ios/MullvadVPN/View controllers/VPNSettings/CustomDNSViewController.swift index 67e6ffce19ab..e13dffc0b732 100644 --- a/ios/MullvadVPN/View controllers/VPNSettings/CustomDNSViewController.swift +++ b/ios/MullvadVPN/View controllers/VPNSettings/CustomDNSViewController.swift @@ -32,6 +32,7 @@ class CustomDNSViewController: UITableViewController, VPNSettingsDataSourceDeleg override func viewDidLoad() { super.viewDidLoad() + tableView.accessibilityIdentifier = .dnsSettingsTableView tableView.backgroundColor = .secondaryColor tableView.separatorColor = .secondaryColor tableView.rowHeight = UITableView.automaticDimension diff --git a/ios/MullvadVPN/Views/StatusActivityView.swift b/ios/MullvadVPN/Views/StatusActivityView.swift index c44cbf82fb2f..9dc9dcca6923 100644 --- a/ios/MullvadVPN/Views/StatusActivityView.swift +++ b/ios/MullvadVPN/Views/StatusActivityView.swift @@ -42,9 +42,12 @@ class StatusActivityView: UIView { addSubview(activityIndicator) NSLayoutConstraint.activate([ + activityIndicator.widthAnchor.constraint(equalTo: statusImageView.widthAnchor), activityIndicator.heightAnchor.constraint(equalTo: statusImageView.heightAnchor), statusImageView.topAnchor.constraint(equalTo: topAnchor), statusImageView.bottomAnchor.constraint(equalTo: bottomAnchor), + statusImageView.leadingAnchor.constraint(equalTo: leadingAnchor), + statusImageView.trailingAnchor.constraint(equalTo: trailingAnchor), statusImageView.centerXAnchor.constraint(equalTo: centerXAnchor), statusImageView.centerYAnchor.constraint(equalTo: centerYAnchor), diff --git a/ios/MullvadVPN/Views/StatusImageView.swift b/ios/MullvadVPN/Views/StatusImageView.swift index 7665daa5809e..51f603c3825d 100644 --- a/ios/MullvadVPN/Views/StatusImageView.swift +++ b/ios/MullvadVPN/Views/StatusImageView.swift @@ -29,6 +29,22 @@ class StatusImageView: UIImageView { } } + override var accessibilityValue: String? { + get { + switch style { + case .success: + return "success" + case .failure: + return "fail" + } + } + + // swiftlint:disable:next unused_setter_value + set { + fatalError("This accessibilityValue property is get only") + } + } + override var intrinsicContentSize: CGSize { CGSize(width: 60, height: 60) } @@ -42,6 +58,7 @@ class StatusImageView: UIImageView { self.style = style super.init(image: style.image) image = style.image + accessibilityIdentifier = .statusImageView } required init?(coder: NSCoder) { diff --git a/ios/MullvadVPNUITests/AccountTests.swift b/ios/MullvadVPNUITests/AccountTests.swift index d417c66d19ec..18ca3be7419e 100644 --- a/ios/MullvadVPNUITests/AccountTests.swift +++ b/ios/MullvadVPNUITests/AccountTests.swift @@ -8,14 +8,14 @@ import XCTest -class AccountApi: XCTestCase { - func testApi() throws { - let temporaryAccountNumber = try MullvadAPIWrapper().createAccount() - try MullvadAPIWrapper().addDevices(5, account: temporaryAccountNumber) - } -} - class AccountTests: LoggedOutUITestCase { + lazy var mullvadAPIWrapper: MullvadAPIWrapper = { + do { + // swiftlint:disable:next force_try + return try! MullvadAPIWrapper() + } + }() + override func setUpWithError() throws { continueAfterFailure = false @@ -29,11 +29,11 @@ class AccountTests: LoggedOutUITestCase { // Verify welcome page is shown and get account number from it let accountNumber = WelcomePage(app).getAccountNumber() - try MullvadAPIWrapper().deleteAccount(accountNumber) + mullvadAPIWrapper.deleteAccount(accountNumber) } func testDeleteAccount() throws { - let accountNumber = try MullvadAPIWrapper().createAccount() + let accountNumber = mullvadAPIWrapper.createAccount() LoginPage(app) .tapAccountNumberTextField() @@ -60,12 +60,31 @@ class AccountTests: LoggedOutUITestCase { .verifyFailIconShown() } + /// Verify logging in works. Will retry x number of times since login request sometimes time out. func testLogin() throws { + let hasTimeAccountNumber = getAccountWithTime() + + addTeardownBlock { + self.returnAccountWithTime(accountNumber: hasTimeAccountNumber) + } + + var successIconShown = false + var retryCount = 0 + let maxRetryCount = 3 + LoginPage(app) .tapAccountNumberTextField() .enterText(hasTimeAccountNumber) - .tapAccountNumberSubmitButton() - .verifySuccessIconShown() + + repeat { + successIconShown = LoginPage(app) + .tapAccountNumberSubmitButton() + .getSuccessIconShown() + + retryCount += 1 + } while successIconShown == false && retryCount < maxRetryCount + + HeaderBar(app) .verifyDeviceLabelShown() } @@ -80,16 +99,12 @@ class AccountTests: LoggedOutUITestCase { func testLoginToAccountWithTooManyDevices() throws { // Setup - let temporaryAccountNumber = try MullvadAPIWrapper().createAccount() - try MullvadAPIWrapper().addDevices(5, account: temporaryAccountNumber) + let temporaryAccountNumber = mullvadAPIWrapper.createAccount() + mullvadAPIWrapper.addDevices(5, account: temporaryAccountNumber) // Teardown addTeardownBlock { - do { - try MullvadAPIWrapper().deleteAccount(temporaryAccountNumber) - } catch { - XCTFail("Failed to delete account using app API") - } + self.mullvadAPIWrapper.deleteAccount(temporaryAccountNumber) } LoginPage(app) @@ -109,6 +124,8 @@ class AccountTests: LoggedOutUITestCase { // First taken back to login page and automatically being logged in LoginPage(app) .verifySuccessIconShown() + + HeaderBar(app) .verifyDeviceLabelShown() // And then taken to out of time page because this account don't have any time added to it @@ -116,26 +133,33 @@ class AccountTests: LoggedOutUITestCase { } func testLogOut() throws { - let newAccountNumber = try MullvadAPIWrapper().createAccount() + let newAccountNumber = mullvadAPIWrapper.createAccount() login(accountNumber: newAccountNumber) - XCTAssertEqual(try MullvadAPIWrapper().getDevices(newAccountNumber).count, 1) + XCTAssertEqual(try mullvadAPIWrapper.getDevices(newAccountNumber).count, 1, "Account has one device") HeaderBar(app) .tapAccountButton() AccountPage(app) .tapLogOutButton() + .waitForLogoutSpinnerToDisappear() LoginPage(app) - XCTAssertEqual(try MullvadAPIWrapper().getDevices(newAccountNumber).count, 0) - try MullvadAPIWrapper().deleteAccount(newAccountNumber) + XCTAssertEqual(try mullvadAPIWrapper.getDevices(newAccountNumber).count, 0, "Account has 0 devices") + mullvadAPIWrapper.deleteAccount(newAccountNumber) } func testTimeLeft() throws { + let hasTimeAccountNumber = getAccountWithTime() + + addTeardownBlock { + self.returnAccountWithTime(accountNumber: hasTimeAccountNumber) + } + login(accountNumber: hasTimeAccountNumber) - let accountExpiry = try MullvadAPIWrapper().getAccountExpiry(hasTimeAccountNumber) + let accountExpiry = try mullvadAPIWrapper.getAccountExpiry(hasTimeAccountNumber) HeaderBar(app) .tapAccountButton() diff --git a/ios/MullvadVPNUITests/ConnectivityTests.swift b/ios/MullvadVPNUITests/ConnectivityTests.swift index 64438308f940..30936e85bd5c 100644 --- a/ios/MullvadVPNUITests/ConnectivityTests.swift +++ b/ios/MullvadVPNUITests/ConnectivityTests.swift @@ -15,7 +15,11 @@ class ConnectivityTests: LoggedOutUITestCase { /// Verifies that the app still functions when API has been blocked func testAPIConnectionViaBridges() throws { + firewallAPIClient.removeRules() + let hasTimeAccountNumber = getAccountWithTime() + addTeardownBlock { + self.returnAccountWithTime(accountNumber: hasTimeAccountNumber) self.firewallAPIClient.removeRules() } @@ -25,25 +29,57 @@ class ConnectivityTests: LoggedOutUITestCase { LoginPage(app) .tapAccountNumberTextField() - .enterText(self.hasTimeAccountNumber) + .enterText(hasTimeAccountNumber) .tapAccountNumberSubmitButton() // After creating firewall rule first login attempt might fail. One more attempt is allowed since the app is cycling between two methods. - if isLoggedIn() { - LoginPage(app) - .verifySuccessIconShown() + let successIconShown = LoginPage(app) + .getSuccessIconShown() + + if successIconShown { + HeaderBar(app) .verifyDeviceLabelShown() } else { LoginPage(app) .verifyFailIconShown() .tapAccountNumberSubmitButton() .verifySuccessIconShown() + + HeaderBar(app) .verifyDeviceLabelShown() } } /// Get the app into a blocked state by connecting to a relay then applying a filter which don't find this relay, then verify that app can still communicate by logging out and verifying that the device was successfully removed func testAPIReachableWhenBlocked() throws { + let hasTimeAccountNumber = getAccountWithTime() + addTeardownBlock { + // Reset any filters + self.login(accountNumber: hasTimeAccountNumber) + + TunnelControlPage(self.app) + .tapSelectLocationButton() + + let filterCloseButtons = self.app.buttons + .matching(identifier: AccessibilityIdentifier.relayFilterChipCloseButton.rawValue) + .allElementsBoundByIndex + + for filterCloseButton in filterCloseButtons where filterCloseButton.isHittable { + filterCloseButton.tap() + } + + // Reset selected location to Sweden + SelectLocationPage(self.app) + .tapLocationCell(withName: BaseUITestCase.appDefaultCountry) + + self.allowAddVPNConfigurationsIfAsked() + + TunnelControlPage(self.app) + .tapCancelOrDisconnectButton() + + self.returnAccountWithTime(accountNumber: hasTimeAccountNumber) + } + // Setup. Enter blocked state by connecting to relay and applying filter which relay isn't part of. login(accountNumber: hasTimeAccountNumber) @@ -88,6 +124,7 @@ class ConnectivityTests: LoggedOutUITestCase { AccountPage(app) .tapLogOutButton() + .waitForLogoutSpinnerToDisappear() LoginPage(app) @@ -97,6 +134,8 @@ class ConnectivityTests: LoggedOutUITestCase { // swiftlint:disable function_body_length /// Test that the app is functioning when API is down. To simulate API being down we create a dummy access method func testAppStillFunctioningWhenAPIDown() throws { + let hasTimeAccountNumber = getAccountWithTime() + addTeardownBlock { HeaderBar(self.app) .tapSettingsButton() @@ -105,12 +144,13 @@ class ConnectivityTests: LoggedOutUITestCase { .tapAPIAccessCell() self.toggleAllAccessMethodsEnabledSwitchesIfOff() + self.returnAccountWithTime(accountNumber: hasTimeAccountNumber) } // Setup. Create a dummy access method to simulate API being down(unreachable) LoginPage(app) .tapAccountNumberTextField() - .enterText(self.hasTimeAccountNumber) + .enterText(hasTimeAccountNumber) .tapAccountNumberSubmitButton() TunnelControlPage(app) @@ -163,12 +203,12 @@ class ConnectivityTests: LoggedOutUITestCase { // Log out will take long because API cannot be reached AccountPage(app) .tapLogOutButton() - .waitForSpinnerNoLongerShown() + .waitForLogoutSpinnerToDisappear() // Verify API cannot be reached by doing a login attempt which should fail LoginPage(app) .tapAccountNumberTextField() - .enterText(self.hasTimeAccountNumber) + .enterText(hasTimeAccountNumber) .tapAccountNumberSubmitButton() .verifyFailIconShown() } diff --git a/ios/MullvadVPNUITests/CustomListsTests.swift b/ios/MullvadVPNUITests/CustomListsTests.swift index 6a3f9f1db725..0d86b4dc091e 100644 --- a/ios/MullvadVPNUITests/CustomListsTests.swift +++ b/ios/MullvadVPNUITests/CustomListsTests.swift @@ -38,7 +38,6 @@ class CustomListsTests: LoggedInWithTimeUITestCase { let customListName = createCustomListName() createCustomList(named: customListName) - workaroundOpenCustomListMenuBug() deleteCustomList(named: customListName) SelectLocationPage(app) @@ -55,11 +54,9 @@ class CustomListsTests: LoggedInWithTimeUITestCase { createCustomList(named: customListName) addTeardownBlock { - self.workaroundOpenCustomListMenuBug() self.deleteCustomList(named: customListName) } - workaroundOpenCustomListMenuBug() startEditingCustomList(named: customListName) EditCustomListLocationsPage(app) @@ -84,11 +81,9 @@ class CustomListsTests: LoggedInWithTimeUITestCase { createCustomList(named: customListName) addTeardownBlock { - self.workaroundOpenCustomListMenuBug() self.deleteCustomList(named: customListName) } - workaroundOpenCustomListMenuBug() startEditingCustomList(named: customListName) EditCustomListLocationsPage(app) @@ -125,14 +120,6 @@ class CustomListsTests: LoggedInWithTimeUITestCase { .tapCreateListButton() } - func workaroundOpenCustomListMenuBug() { - // In order to avoid a bug where the open custom list button cannot be found, the location view is closed and then reopened - SelectLocationPage(app) - .tapDoneButton() - TunnelControlPage(app) - .tapSelectLocationButton() - } - func startEditingCustomList(named customListName: String) { SelectLocationPage(app) .tapWhereStatusBarShouldBeToScrollToTopMostPosition() diff --git a/ios/MullvadVPNUITests/Info.plist b/ios/MullvadVPNUITests/Info.plist index 0bed909d3f3e..a81366999153 100644 --- a/ios/MullvadVPNUITests/Info.plist +++ b/ios/MullvadVPNUITests/Info.plist @@ -4,22 +4,26 @@ AdServingDomain $(AD_SERVING_DOMAIN) + AmIJSONUrl + $(AM_I_JSON_URL) ApiEndpoint $(API_ENDPOINT) ApiHostName $(API_HOST_NAME) + AttachAppLogsOnFailure + ${ATTACH_APP_LOGS_ON_FAILURE} DisplayName $(DISPLAY_NAME) FirewallApiBaseURL $(FIREWALL_API_BASE_URL) - FiveWireGuardKeysAccountNumber - $(FIVE_WIREGUARD_KEYS_ACCOUNT_NUMBER) HasTimeAccountNumber $(HAS_TIME_ACCOUNT_NUMBER) IOSDevicePinCode $(IOS_DEVICE_PIN_CODE) NoTimeAccountNumber $(NO_TIME_ACCOUNT_NUMBER) + PartnerApiToken + $(PARTNER_API_TOKEN) ShouldBeReachableDomain $(SHOULD_BE_REACHABLE_DOMAIN) TestDeviceIdentifier diff --git a/ios/MullvadVPNUITests/Networking/MullvadAPIWrapper.swift b/ios/MullvadVPNUITests/Networking/MullvadAPIWrapper.swift index 588b692a7a3b..d0983b3ee201 100644 --- a/ios/MullvadVPNUITests/Networking/MullvadAPIWrapper.swift +++ b/ios/MullvadVPNUITests/Networking/MullvadAPIWrapper.swift @@ -16,12 +16,16 @@ enum MullvadAPIError: Error { } class MullvadAPIWrapper { + private var mullvadAPI: MullvadApi + private let throttleQueue = DispatchQueue(label: "MullvadAPIWrapperThrottleQueue", qos: .userInitiated) + private var lastCallDate: Date? + private let throttleDelay: TimeInterval = 0.25 + private let throttleWaitTimeout: TimeInterval = 5.0 + // swiftlint:disable force_cast static let hostName = Bundle(for: MullvadAPIWrapper.self) .infoDictionary?["ApiHostName"] as! String - private var mullvadAPI: MullvadApi - /// API endpoint configuration value in the format : static let endpoint = Bundle(for: MullvadAPIWrapper.self) .infoDictionary?["ApiEndpoint"] as! String @@ -33,6 +37,27 @@ class MullvadAPIWrapper { mullvadAPI = try MullvadApi(apiAddress: apiAddress, hostname: hostname) } + /// Throttle what's in the callback. This is used for throttling requests to the app API. All requests should be throttled or else we might be rate limited. 5 requests per second allowed. + private func throttle(callback: @escaping () -> Void) { + throttleQueue.async { + let now = Date() + var delay: TimeInterval = 0 + + if let lastCallDate = self.lastCallDate { + let timeSinceLastCall = now.timeIntervalSince(lastCallDate) + + if timeSinceLastCall < self.throttleDelay { + delay = self.throttleDelay - timeSinceLastCall + } + } + + self.throttleQueue.asyncAfter(deadline: .now() + delay) { + callback() + self.lastCallDate = Date() + } + } + } + public static func getAPIIPAddress() throws -> String { guard let ipAddress = endpoint.components(separatedBy: ":").first else { throw MullvadAPIError.invalidEndpointFormatError @@ -59,56 +84,116 @@ class MullvadAPIWrapper { } func createAccount() -> String { - do { - let accountNumber = try mullvadAPI.createAccount() - return accountNumber - } catch { - XCTFail("Failed to create account using app API") - return String() + var accountNumber = String() + var requestError: Error? + let requestCompletedExpectation = XCTestExpectation(description: "Create account request completed") + + throttle { + do { + accountNumber = try self.mullvadAPI.createAccount() + } catch { + requestError = MullvadAPIError.requestError + } + + requestCompletedExpectation.fulfill() } + + let waitResult = XCTWaiter().wait(for: [requestCompletedExpectation], timeout: throttleWaitTimeout) + XCTAssertEqual(waitResult, .completed, "Create account request completed") + XCTAssertNil(requestError, "Create account error is nil") + + return accountNumber } func deleteAccount(_ accountNumber: String) { + var requestError: Error? + let requestCompletedExpectation = XCTestExpectation(description: "Delete account request completed") + do { try mullvadAPI.delete(account: accountNumber) } catch { - XCTFail("Failed to delete account using app API") + requestError = MullvadAPIError.requestError } + + requestCompletedExpectation.fulfill() + + let waitResult = XCTWaiter().wait(for: [requestCompletedExpectation], timeout: throttleWaitTimeout) + XCTAssertEqual(waitResult, .completed, "Delete account request completed") + XCTAssertNil(requestError, "Delete account error is nil") } /// Add another device to specified account. A dummy WireGuard key will be generated. - func addDevice(_ account: String) throws { - let devicePublicKey = generateMockWireGuardKey() + func addDevice(_ account: String) { + var addDeviceError: Error? + let requestCompletedExpectation = XCTestExpectation(description: "Add device request completed") - do { - try mullvadAPI.addDevice(forAccount: account, publicKey: devicePublicKey) - } catch { - throw MullvadAPIError.requestError + throttle { + let devicePublicKey = self.generateMockWireGuardKey() + + do { + try self.mullvadAPI.addDevice(forAccount: account, publicKey: devicePublicKey) + } catch { + addDeviceError = MullvadAPIError.requestError + } + + requestCompletedExpectation.fulfill() } + + let waitResult = XCTWaiter().wait(for: [requestCompletedExpectation], timeout: throttleWaitTimeout) + XCTAssertEqual(waitResult, .completed, "Add device request completed") + XCTAssertNil(addDeviceError, "Add device error is nil") } /// Add multiple devices to specified account. Dummy WireGuard keys will be generated. - func addDevices(_ numberOfDevices: Int, account: String) throws { - for _ in 0 ..< numberOfDevices { - try self.addDevice(account) + func addDevices(_ numberOfDevices: Int, account: String) { + for i in 0 ..< numberOfDevices { + self.addDevice(account) + print("Created \(i + 1) devices") } } func getAccountExpiry(_ account: String) throws -> Date { - do { - let accountExpiryTimestamp = Double(try mullvadAPI.getExpiry(forAccount: account)) - let accountExpiryDate = Date(timeIntervalSince1970: accountExpiryTimestamp) - return accountExpiryDate - } catch { - throw MullvadAPIError.requestError + var accountExpiryDate: Date = .distantPast + var requestError: Error? + let requestCompletedExpectation = XCTestExpectation(description: "Get account expiry request completed") + + throttle { + do { + let accountExpiryTimestamp = Double(try self.mullvadAPI.getExpiry(forAccount: account)) + accountExpiryDate = Date(timeIntervalSince1970: accountExpiryTimestamp) + } catch { + requestError = MullvadAPIError.requestError + } + + requestCompletedExpectation.fulfill() } + + let waitResult = XCTWaiter().wait(for: [requestCompletedExpectation], timeout: throttleWaitTimeout) + XCTAssertEqual(waitResult, .completed, "Get account expiry request completed") + XCTAssertNil(requestError, "Get account expiry error is nil") + + return accountExpiryDate } func getDevices(_ account: String) throws -> [Device] { - do { - return try mullvadAPI.listDevices(forAccount: account) - } catch { - throw MullvadAPIError.requestError + var devices: [Device] = [] + var requestError: Error? + let requestCompletedExpectation = XCTestExpectation(description: "Get devices request completed") + + throttle { + do { + devices = try self.mullvadAPI.listDevices(forAccount: account) + } catch { + requestError = MullvadAPIError.requestError + } + + requestCompletedExpectation.fulfill() } + + let waitResult = XCTWaiter.wait(for: [requestCompletedExpectation], timeout: throttleWaitTimeout) + XCTAssertEqual(waitResult, .completed, "Get devices request completed") + XCTAssertNil(requestError, "Get devices error is nil") + + return devices } } diff --git a/ios/MullvadVPNUITests/Networking/Networking.swift b/ios/MullvadVPNUITests/Networking/Networking.swift index 003cc4386c3d..6e75925d940a 100644 --- a/ios/MullvadVPNUITests/Networking/Networking.swift +++ b/ios/MullvadVPNUITests/Networking/Networking.swift @@ -129,34 +129,52 @@ class Networking { public static func verifyCanAccessAPI() throws { let apiIPAddress = try MullvadAPIWrapper.getAPIIPAddress() let apiPort = try MullvadAPIWrapper.getAPIPort() - XCTAssertTrue(try canConnectSocket(host: apiIPAddress, port: apiPort)) + XCTAssertTrue( + try canConnectSocket(host: apiIPAddress, port: apiPort), + "Failed to verify that API can be accessed" + ) } /// Verify API cannot be accessed by attempting to connect a socket to the configured API host and port public static func verifyCannotAccessAPI() throws { let apiIPAddress = try MullvadAPIWrapper.getAPIIPAddress() let apiPort = try MullvadAPIWrapper.getAPIPort() - XCTAssertFalse(try canConnectSocket(host: apiIPAddress, port: apiPort)) + XCTAssertFalse( + try canConnectSocket(host: apiIPAddress, port: apiPort), + "Failed to verify that API cannot be accessed" + ) } /// Verify that the device has Internet connectivity public static func verifyCanAccessInternet() throws { - XCTAssertTrue(try canConnectSocket(host: getAlwaysReachableDomain(), port: "80")) + XCTAssertTrue( + try canConnectSocket(host: getAlwaysReachableDomain(), port: "80"), + "Failed to verify that the Internet can be acccessed" + ) } /// Verify that the device does not have Internet connectivity public static func verifyCannotAccessInternet() throws { - XCTAssertFalse(try canConnectSocket(host: getAlwaysReachableDomain(), port: "80")) + XCTAssertFalse( + try canConnectSocket(host: getAlwaysReachableDomain(), port: "80"), + "Failed to verify that the Internet cannot be accessed" + ) } /// Verify that an ad serving domain is reachable by making sure a connection can be established on port 80 public static func verifyCanReachAdServingDomain() throws { - XCTAssertTrue(try Self.canConnectSocket(host: try Self.getAdServingDomain(), port: "80")) + XCTAssertTrue( + try Self.canConnectSocket(host: try Self.getAdServingDomain(), port: "80"), + "Failed to verify that ad serving domain can be accessed" + ) } /// Verify that an ad serving domain is NOT reachable by making sure a connection can not be established on port 80 public static func verifyCannotReachAdServingDomain() throws { - XCTAssertFalse(try Self.canConnectSocket(host: try Self.getAdServingDomain(), port: "80")) + XCTAssertFalse( + try Self.canConnectSocket(host: try Self.getAdServingDomain(), port: "80"), + "Failed to verify that ad serving domain cannot be accessed" + ) } /// Verify that the expected DNS server is used by verifying provider name and whether it is a Mullvad DNS server or not @@ -205,8 +223,12 @@ class Networking { XCTAssertGreaterThanOrEqual(dnsServerEntries.count, 1) for dnsServerEntry in dnsServerEntries { - XCTAssertEqual(dnsServerEntry.organization, providerName) - XCTAssertEqual(dnsServerEntry.mullvad_dns, isMullvad) + XCTAssertEqual(dnsServerEntry.organization, providerName, "Expected organization name") + XCTAssertEqual( + dnsServerEntry.mullvad_dns, + isMullvad, + "Verifying that it is or isn't a Mullvad DNS server" + ) } } } @@ -215,8 +237,12 @@ class Networking { } } - public static func verifyConnectedThroughMullvad() { - let mullvadConnectionJsonEndpoint = "https://am.i.mullvad.net/json" + public static func verifyConnectedThroughMullvad() throws { + let mullvadConnectionJsonEndpoint = try XCTUnwrap( + Bundle(for: Networking.self) + .infoDictionary?["AmIJSONUrl"] as? String, + "Read am I JSON URL from Info" + ) guard let url = URL(string: mullvadConnectionJsonEndpoint) else { XCTFail("Failed to unwrap URL") return diff --git a/ios/MullvadVPNUITests/Networking/PartnerAPIClient.swift b/ios/MullvadVPNUITests/Networking/PartnerAPIClient.swift new file mode 100644 index 000000000000..67d1d7acdddb --- /dev/null +++ b/ios/MullvadVPNUITests/Networking/PartnerAPIClient.swift @@ -0,0 +1,123 @@ +// +// PartnerAPIClient.swift +// MullvadVPNUITests +// +// Created by Niklas Berglund on 2024-04-19. +// Copyright © 2024 Mullvad VPN AB. All rights reserved. +// + +import Foundation +import XCTest + +class PartnerAPIClient { + let baseURL = URL(string: "https://partner.stagemole.eu/v1/")! + + lazy var accessToken: String = { + guard let token = Bundle(for: BaseUITestCase.self).infoDictionary?["PartnerApiToken"] as? String else { + fatalError("Failed to retrieve partner API token from config") + } + return token + }() + + /// Add time to an account + /// - Parameters: + /// - accountNumber: Account number + /// - days: Number of days to add. Needs to be between 1 and 31. + func addTime(accountNumber: String, days: Int) -> Date { + let jsonResponse = sendRequest( + method: "POST", + endpoint: "accounts/\(accountNumber)/extend", + jsonObject: ["days": "\(days)"] + ) + + guard let newExpiryString = jsonResponse["new_expiry"] as? String else { + XCTFail("Failed to read new account expiry from response") + return Date() + } + + let dateFormatter = ISO8601DateFormatter() + guard let newExpiryDate = dateFormatter.date(from: newExpiryString) else { + XCTFail("Failed to create Date object from date string") + return Date() + } + + return newExpiryDate + } + + func createAccount() -> String { + let jsonResponse = sendRequest(method: "POST", endpoint: "accounts", jsonObject: nil) + + guard let accountNumber = jsonResponse["id"] as? String else { + XCTFail("Failed to read created account number") + return String() + } + + return accountNumber + } + + func deleteAccount(accountNumber: String) { + _ = sendRequest(method: "DELETE", endpoint: "accounts/\(accountNumber)", jsonObject: nil) + } + + private func sendRequest(method: String, endpoint: String, jsonObject: [String: Any]?) -> [String: Any] { + let url = baseURL.appendingPathComponent(endpoint) + var request = URLRequest(url: url) + request.httpMethod = method + request.setValue("Basic \(accessToken)", forHTTPHeaderField: "Authorization") + + var jsonResponse: [String: Any] = [:] + + do { + if let jsonObject = jsonObject { + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + request.httpBody = try JSONSerialization.data(withJSONObject: jsonObject, options: []) + } + } catch { + XCTFail("Failed to serialize JSON object") + return [:] + } + + let completionHandlerInvokedExpectation = XCTestExpectation( + description: "Completion handler for the request is invoked" + ) + + var requestError: Error? + + let task = URLSession.shared.dataTask(with: request) { data, response, error in + requestError = error + + guard let data = data, + let response = response as? HTTPURLResponse, + error == nil else { + XCTFail("Error: \(error?.localizedDescription ?? "Unknown error")") + completionHandlerInvokedExpectation.fulfill() + return + } + + if 200 ... 204 ~= response.statusCode { + print("Request successful") + do { + if data.isEmpty { + // Not all requests return JSON data + jsonResponse = [:] + } else { + jsonResponse = try JSONSerialization.jsonObject(with: data) as? [String: Any] ?? [:] + } + } catch { + XCTFail("Failed to deserialize JSON response") + } + } else { + XCTFail("Request failed") + } + + completionHandlerInvokedExpectation.fulfill() + } + + task.resume() + let waitResult = XCTWaiter().wait(for: [completionHandlerInvokedExpectation], timeout: 10) + XCTAssertEqual(waitResult, .completed, "Waiting for partner API request expectation completed") + XCTAssertNil(requestError) + + return jsonResponse + } +} diff --git a/ios/MullvadVPNUITests/Pages/APIAccessPage.swift b/ios/MullvadVPNUITests/Pages/APIAccessPage.swift index 33d8be3393fa..bd1f5850ebc2 100644 --- a/ios/MullvadVPNUITests/Pages/APIAccessPage.swift +++ b/ios/MullvadVPNUITests/Pages/APIAccessPage.swift @@ -12,7 +12,7 @@ import XCTest class APIAccessPage: Page { override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .apiAccessView + self.pageElement = app.otherElements[.apiAccessView] waitForPageToBeShown() } diff --git a/ios/MullvadVPNUITests/Pages/AccountDeletionPage.swift b/ios/MullvadVPNUITests/Pages/AccountDeletionPage.swift index 30bf01034ebd..b92b51a7ed33 100644 --- a/ios/MullvadVPNUITests/Pages/AccountDeletionPage.swift +++ b/ios/MullvadVPNUITests/Pages/AccountDeletionPage.swift @@ -13,7 +13,7 @@ class AccountDeletionPage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .deleteAccountView + self.pageElement = app.otherElements[.deleteAccountView] waitForPageToBeShown() } @@ -23,12 +23,7 @@ class AccountDeletionPage: Page { } @discardableResult func tapDeleteAccountButton() -> Self { - guard let pageAccessibilityIdentifier = self.pageAccessibilityIdentifier else { - XCTFail("Page's accessibility identifier not set") - return self - } - - app.otherElements[pageAccessibilityIdentifier].buttons[AccessibilityIdentifier.deleteButton].tap() + app.otherElements[.deleteAccountView].buttons[AccessibilityIdentifier.deleteButton].tap() return self } diff --git a/ios/MullvadVPNUITests/Pages/AccountPage.swift b/ios/MullvadVPNUITests/Pages/AccountPage.swift index 83082f339577..fd3d5d448e60 100644 --- a/ios/MullvadVPNUITests/Pages/AccountPage.swift +++ b/ios/MullvadVPNUITests/Pages/AccountPage.swift @@ -13,7 +13,7 @@ class AccountPage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .accountView + self.pageElement = app.otherElements[.accountView] waitForPageToBeShown() } @@ -67,13 +67,13 @@ class AccountPage: Page { return self } - XCTAssertEqual(strippedDate, paidUntilLabelDate) + XCTAssertEqual(strippedDate, paidUntilLabelDate, "Paid until date correct") return self } - @discardableResult func waitForSpinnerNoLongerShown() -> Self { - app.otherElements[AccessibilityIdentifier.logOutSpinnerAlertView] - .waitForNonExistence(timeout: BaseUITestCase.veryLongTimeout) - return self + func waitForLogoutSpinnerToDisappear() { + let spinnerDisappeared = app.otherElements[.logOutSpinnerAlertView] + .waitForNonExistence(timeout: BaseUITestCase.extremelyLongTimeout) + XCTAssertTrue(spinnerDisappeared, "Log out spinner disappeared") } } diff --git a/ios/MullvadVPNUITests/Pages/AddAccessMethodPage.swift b/ios/MullvadVPNUITests/Pages/AddAccessMethodPage.swift index 698397bbbcdd..d961822226ed 100644 --- a/ios/MullvadVPNUITests/Pages/AddAccessMethodPage.swift +++ b/ios/MullvadVPNUITests/Pages/AddAccessMethodPage.swift @@ -12,8 +12,7 @@ import XCTest class AddAccessMethodPage: Page { override init(_ app: XCUIApplication) { super.init(app) - - self.pageAccessibilityIdentifier = .addAccessMethodTableView + self.pageElement = app.tables[.addAccessMethodTableView] waitForPageToBeShown() } @@ -75,8 +74,7 @@ class AddAccessMethodPage: Page { class AddAccessMethodAPIUnreachableAlert: Page { override init(_ app: XCUIApplication) { super.init(app) - - self.pageAccessibilityIdentifier = .accessMethodUnreachableAlert + self.pageElement = app.otherElements[.accessMethodUnreachableAlert] waitForPageToBeShown() } diff --git a/ios/MullvadVPNUITests/Pages/Alert.swift b/ios/MullvadVPNUITests/Pages/Alert.swift index dce7c3bf11c1..0f723af612de 100644 --- a/ios/MullvadVPNUITests/Pages/Alert.swift +++ b/ios/MullvadVPNUITests/Pages/Alert.swift @@ -14,7 +14,7 @@ class Alert: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .alertContainerView + self.pageElement = app.otherElements[.alertContainerView] waitForPageToBeShown() } diff --git a/ios/MullvadVPNUITests/Pages/AppLogsPage.swift b/ios/MullvadVPNUITests/Pages/AppLogsPage.swift new file mode 100644 index 000000000000..32d157e9e2d3 --- /dev/null +++ b/ios/MullvadVPNUITests/Pages/AppLogsPage.swift @@ -0,0 +1,36 @@ +// +// AppLogsPage.swift +// MullvadVPNUITests +// +// Created by Niklas Berglund on 2024-04-26. +// Copyright © 2024 Mullvad VPN AB. All rights reserved. +// + +import XCTest + +class AppLogsPage: Page { + override init(_ app: XCUIApplication) { + super.init(app) + self.pageElement = app.otherElements[.appLogsView] + waitForPageToBeShown() + } + + @discardableResult func tapShareButton() -> Self { + app.buttons[.appLogsShareButton].tap() + return self + } + + @discardableResult func tapDoneButton() -> Self { + app.buttons[.appLogsDoneButton].tap() + return self + } + + func getAppLogText() -> String { + guard let logText = app.textViews[.problemReportAppLogsTextView].value as? String else { + XCTFail("Failed to extract app log text") + return String() + } + + return logText + } +} diff --git a/ios/MullvadVPNUITests/Pages/ChangeLogAlert.swift b/ios/MullvadVPNUITests/Pages/ChangeLogAlert.swift index 70d67d0294fc..61e59ec953f3 100644 --- a/ios/MullvadVPNUITests/Pages/ChangeLogAlert.swift +++ b/ios/MullvadVPNUITests/Pages/ChangeLogAlert.swift @@ -13,7 +13,7 @@ class ChangeLogAlert: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .changeLogAlert + self.pageElement = app.otherElements[.changeLogAlert] waitForPageToBeShown() } diff --git a/ios/MullvadVPNUITests/Pages/CustomListPage.swift b/ios/MullvadVPNUITests/Pages/CustomListPage.swift index 17ebf363c343..b4e6dc1311b0 100644 --- a/ios/MullvadVPNUITests/Pages/CustomListPage.swift +++ b/ios/MullvadVPNUITests/Pages/CustomListPage.swift @@ -12,13 +12,13 @@ class CustomListPage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .newCustomListView + self.pageElement = app.otherElements[.newCustomListView] waitForPageToBeShown() } @discardableResult func verifyCreateButtonIs(enabled: Bool) -> Self { let saveOrCreateButton = app.buttons[.saveCreateCustomListButton] - XCTAssertTrue(saveOrCreateButton.isEnabled == enabled) + XCTAssertTrue(saveOrCreateButton.isEnabled == enabled, "Verify state of create button") return self } diff --git a/ios/MullvadVPNUITests/Pages/DNSSettingsPage.swift b/ios/MullvadVPNUITests/Pages/DNSSettingsPage.swift index e592862cbf33..88cd45bb541c 100644 --- a/ios/MullvadVPNUITests/Pages/DNSSettingsPage.swift +++ b/ios/MullvadVPNUITests/Pages/DNSSettingsPage.swift @@ -13,7 +13,8 @@ class DNSSettingsPage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .dnsSettings + self.pageElement = app.tables[.dnsSettingsTableView] + waitForPageToBeShown() } private func assertSwitchOn(accessibilityIdentifier: AccessibilityIdentifier) -> Self { @@ -38,7 +39,7 @@ class DNSSettingsPage: Page { @discardableResult func tapDNSContentBlockersHeaderExpandButton() -> Self { let headerView = app.otherElements[AccessibilityIdentifier.dnsContentBlockersHeaderView] - let expandButton = headerView.buttons[AccessibilityIdentifier.collapseButton] + let expandButton = headerView.buttons[AccessibilityIdentifier.expandButton] expandButton.tap() return self diff --git a/ios/MullvadVPNUITests/Pages/DeviceManagementPage.swift b/ios/MullvadVPNUITests/Pages/DeviceManagementPage.swift index 571fb1cfd6ec..68340a260ff2 100644 --- a/ios/MullvadVPNUITests/Pages/DeviceManagementPage.swift +++ b/ios/MullvadVPNUITests/Pages/DeviceManagementPage.swift @@ -13,7 +13,7 @@ class DeviceManagementPage: Page { override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .deviceManagementView + self.pageElement = app.otherElements[.deviceManagementView] waitForPageToBeShown() } @@ -36,8 +36,7 @@ class DeviceManagementPage: Page { class DeviceManagementLogOutDeviceConfirmationAlert: Page { override init(_ app: XCUIApplication) { super.init(app) - - self.pageAccessibilityIdentifier = .alertContainerView + self.pageElement = app.otherElements[.alertContainerView] waitForPageToBeShown() } diff --git a/ios/MullvadVPNUITests/Pages/EditAccessMethodPage.swift b/ios/MullvadVPNUITests/Pages/EditAccessMethodPage.swift index 68f21d35cca5..93aad69c1769 100644 --- a/ios/MullvadVPNUITests/Pages/EditAccessMethodPage.swift +++ b/ios/MullvadVPNUITests/Pages/EditAccessMethodPage.swift @@ -12,8 +12,7 @@ import XCTest class EditAccessMethodPage: Page { override init(_ app: XCUIApplication) { super.init(app) - - self.pageAccessibilityIdentifier = .editAccessMethodView + self.pageElement = app.tables[.editAccessMethodView] waitForPageToBeShown() } diff --git a/ios/MullvadVPNUITests/Pages/EditCustomListLocationsPage.swift b/ios/MullvadVPNUITests/Pages/EditCustomListLocationsPage.swift index 08d3d6fa9fef..7d7050c14800 100644 --- a/ios/MullvadVPNUITests/Pages/EditCustomListLocationsPage.swift +++ b/ios/MullvadVPNUITests/Pages/EditCustomListLocationsPage.swift @@ -12,7 +12,7 @@ class EditCustomListLocationsPage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .editCustomListEditLocationsView + self.pageElement = app.otherElements[.editCustomListEditLocationsView] waitForPageToBeShown() } diff --git a/ios/MullvadVPNUITests/Pages/HeaderBar.swift b/ios/MullvadVPNUITests/Pages/HeaderBar.swift index 129d721b3745..0d46fe961680 100644 --- a/ios/MullvadVPNUITests/Pages/HeaderBar.swift +++ b/ios/MullvadVPNUITests/Pages/HeaderBar.swift @@ -16,7 +16,7 @@ class HeaderBar: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .headerBarView + self.pageElement = app.otherElements[.headerBarView] waitForPageToBeShown() } @@ -29,4 +29,13 @@ class HeaderBar: Page { settingsButton.tap() return self } + + @discardableResult public func verifyDeviceLabelShown() -> Self { + XCTAssertTrue( + app.staticTexts[AccessibilityIdentifier.headerDeviceNameLabel] + .waitForExistence(timeout: BaseUITestCase.defaultTimeout), "Device name displayed in header" + ) + + return self + } } diff --git a/ios/MullvadVPNUITests/Pages/ListCustomListsPage.swift b/ios/MullvadVPNUITests/Pages/ListCustomListsPage.swift index 2af421be7a20..8f62bf755286 100644 --- a/ios/MullvadVPNUITests/Pages/ListCustomListsPage.swift +++ b/ios/MullvadVPNUITests/Pages/ListCustomListsPage.swift @@ -12,7 +12,7 @@ class ListCustomListsPage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .listCustomListsView + self.pageElement = app.otherElements[.listCustomListsView] waitForPageToBeShown() } diff --git a/ios/MullvadVPNUITests/Pages/LoginPage.swift b/ios/MullvadVPNUITests/Pages/LoginPage.swift index f7b204b344bc..284e5ac7e72b 100644 --- a/ios/MullvadVPNUITests/Pages/LoginPage.swift +++ b/ios/MullvadVPNUITests/Pages/LoginPage.swift @@ -13,7 +13,7 @@ class LoginPage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .loginView + self.pageElement = app.otherElements[.loginView] waitForPageToBeShown() } @@ -22,6 +22,13 @@ class LoginPage: Page { return self } + @discardableResult public func waitForAccountNumberSubmitButton() -> Self { + let submitButtonExist = app.buttons[AccessibilityIdentifier.loginTextFieldButton] + .waitForExistence(timeout: BaseUITestCase.defaultTimeout) + XCTAssertTrue(submitButtonExist, "Account number submit button shown") + return self + } + @discardableResult public func tapAccountNumberSubmitButton() -> Self { app.buttons[AccessibilityIdentifier.loginTextFieldButton].tap() return self @@ -32,24 +39,40 @@ class LoginPage: Page { return self } - @discardableResult public func verifyDeviceLabelShown() -> Self { - XCTAssertTrue( - app.staticTexts[AccessibilityIdentifier.headerDeviceNameLabel] - .waitForExistence(timeout: BaseUITestCase.defaultTimeout) - ) + @discardableResult public func verifySuccessIconShown() -> Self { + // Success icon is only shown very briefly, since another view is presented after success icon is shown. + // Therefore we need to poll faster than waitForElement function. + let successIconDisplayedExpectation = XCTestExpectation(description: "Success icon shown") + let timer = Timer.scheduledTimer(withTimeInterval: 0.2, repeats: true) { [self] _ in + let statusImageView = self.app.images[.statusImageView] - return self - } + if statusImageView.exists { + if statusImageView.value as? String == "success" { + successIconDisplayedExpectation.fulfill() + } + } + } + + let waitResult = XCTWaiter.wait(for: [successIconDisplayedExpectation], timeout: BaseUITestCase.longTimeout) + XCTAssertEqual(waitResult, .completed, "Success icon shown") + timer.invalidate() - @discardableResult public func verifySuccessIconShown() -> Self { - _ = app.images.element(matching: .image, identifier: "IconSuccess") - .waitForExistence(timeout: BaseUITestCase.defaultTimeout) return self } @discardableResult public func verifyFailIconShown() -> Self { - _ = app.images.element(matching: .image, identifier: "IconFail") - .waitForExistence(timeout: BaseUITestCase.longTimeout) + let predicate = NSPredicate(format: "identifier == 'statusImageView' AND value == 'fail'") + let elementQuery = app.images.containing(predicate) + let elementExists = elementQuery.firstMatch.waitForExistence(timeout: BaseUITestCase.longTimeout) + XCTAssertTrue(elementExists, "Fail icon shown") return self } + + /// Checks whether success icon is being shown + func getSuccessIconShown() -> Bool { + let predicate = NSPredicate(format: "identifier == 'statusImageView' AND value == 'success'") + let elementQuery = app.images.containing(predicate) + let elementExists = elementQuery.firstMatch.waitForExistence(timeout: BaseUITestCase.defaultTimeout) + return elementExists + } } diff --git a/ios/MullvadVPNUITests/Pages/OutOfTimePage.swift b/ios/MullvadVPNUITests/Pages/OutOfTimePage.swift index 0950aa9147ce..22e287ad1463 100644 --- a/ios/MullvadVPNUITests/Pages/OutOfTimePage.swift +++ b/ios/MullvadVPNUITests/Pages/OutOfTimePage.swift @@ -13,7 +13,7 @@ class OutOfTimePage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .outOfTimeView + self.pageElement = app.otherElements[.outOfTimeView] waitForPageToBeShown() } } diff --git a/ios/MullvadVPNUITests/Pages/Page.swift b/ios/MullvadVPNUITests/Pages/Page.swift index 09a6b5b2ae3d..92a63d888458 100644 --- a/ios/MullvadVPNUITests/Pages/Page.swift +++ b/ios/MullvadVPNUITests/Pages/Page.swift @@ -11,18 +11,19 @@ import XCTest class Page { let app: XCUIApplication - var pageAccessibilityIdentifier: AccessibilityIdentifier? + + /// Element in the page used to verify that the page is currently being shown, usually accessibilityIdentifier of the view controller's main view + var pageElement: XCUIElement? @discardableResult init(_ app: XCUIApplication) { self.app = app } func waitForPageToBeShown() { - if let pageAccessibilityIdentifier = self.pageAccessibilityIdentifier { - XCTAssert( - self.app.descendants(matching: .any).matching(identifier: pageAccessibilityIdentifier.rawValue) - .firstMatch - .waitForExistence(timeout: BaseUITestCase.defaultTimeout) + if let pageElement { + XCTAssertTrue( + pageElement.waitForExistence(timeout: BaseUITestCase.defaultTimeout), + "Page is shown" ) } } diff --git a/ios/MullvadVPNUITests/Pages/ProblemReportPage.swift b/ios/MullvadVPNUITests/Pages/ProblemReportPage.swift index d02a0ec9917e..6c9cbabff9e4 100644 --- a/ios/MullvadVPNUITests/Pages/ProblemReportPage.swift +++ b/ios/MullvadVPNUITests/Pages/ProblemReportPage.swift @@ -13,8 +13,7 @@ class ProblemReportPage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - pageAccessibilityIdentifier = .problemReportView - + self.pageElement = app.otherElements[.problemReportView] waitForPageToBeShown() } @@ -33,7 +32,7 @@ class ProblemReportPage: Page { } @discardableResult func tapViewAppLogsButton() -> Self { - app.otherElements[AccessibilityIdentifier.problemReportAppLogsButton] + app.buttons[AccessibilityIdentifier.problemReportAppLogsButton] .tap() return self diff --git a/ios/MullvadVPNUITests/Pages/ProblemReportSubmittedPage.swift b/ios/MullvadVPNUITests/Pages/ProblemReportSubmittedPage.swift index a90c93b3ff77..ee0cdfd2cdbe 100644 --- a/ios/MullvadVPNUITests/Pages/ProblemReportSubmittedPage.swift +++ b/ios/MullvadVPNUITests/Pages/ProblemReportSubmittedPage.swift @@ -13,7 +13,7 @@ class ProblemReportSubmittedPage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - pageAccessibilityIdentifier = .problemReportSubmittedView + self.pageElement = app.otherElements[.problemReportSubmittedView] waitForPageToBeShown() } } diff --git a/ios/MullvadVPNUITests/Pages/RevokedDevicePage.swift b/ios/MullvadVPNUITests/Pages/RevokedDevicePage.swift index 4b8dd6dd7206..8225cb008fa9 100644 --- a/ios/MullvadVPNUITests/Pages/RevokedDevicePage.swift +++ b/ios/MullvadVPNUITests/Pages/RevokedDevicePage.swift @@ -13,7 +13,7 @@ class RevokedDevicePage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .revokedDeviceView + self.pageElement = app.otherElements[.revokedDeviceView] waitForPageToBeShown() } diff --git a/ios/MullvadVPNUITests/Pages/SelectLocationPage.swift b/ios/MullvadVPNUITests/Pages/SelectLocationPage.swift index d4126083ca08..8c87309a04fd 100644 --- a/ios/MullvadVPNUITests/Pages/SelectLocationPage.swift +++ b/ios/MullvadVPNUITests/Pages/SelectLocationPage.swift @@ -13,7 +13,7 @@ class SelectLocationPage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .selectLocationView + self.pageElement = app.otherElements[.selectLocationView] waitForPageToBeShown() } @@ -68,8 +68,18 @@ class SelectLocationPage: Page { } @discardableResult func tapCustomListEllipsisButton() -> Self { - let customListEllipsisButton = app.buttons[AccessibilityIdentifier.openCustomListsMenuButton] - customListEllipsisButton.tap() + let customListEllipsisButtons = app.buttons + .matching(identifier: AccessibilityIdentifier.openCustomListsMenuButton.rawValue).allElementsBoundByIndex + + // This is a workaround for an issue we have with the ellipsis showing up multiple times in the accessibility hieararchy even though in the view hierarchy there is only one + // Only the actually visual one is hittable, so only the visible button will be tapped + for ellipsisButton in customListEllipsisButtons where ellipsisButton.isHittable { + ellipsisButton.tap() + return self + } + + XCTFail("Found no hittable custom list ellipsis button") + return self } diff --git a/ios/MullvadVPNUITests/Pages/SettingsPage.swift b/ios/MullvadVPNUITests/Pages/SettingsPage.swift index db28fb33c929..8d40154abf7d 100644 --- a/ios/MullvadVPNUITests/Pages/SettingsPage.swift +++ b/ios/MullvadVPNUITests/Pages/SettingsPage.swift @@ -13,7 +13,7 @@ class SettingsPage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .settingsContainerView + self.pageElement = app.otherElements[.settingsContainerView] waitForPageToBeShown() } diff --git a/ios/MullvadVPNUITests/Pages/TermsOfServicePage.swift b/ios/MullvadVPNUITests/Pages/TermsOfServicePage.swift index 0f32bf7be6be..2654602494be 100644 --- a/ios/MullvadVPNUITests/Pages/TermsOfServicePage.swift +++ b/ios/MullvadVPNUITests/Pages/TermsOfServicePage.swift @@ -13,7 +13,7 @@ class TermsOfServicePage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .termsOfServiceView + self.pageElement = app.otherElements[.termsOfServiceView] waitForPageToBeShown() } diff --git a/ios/MullvadVPNUITests/Pages/TunnelControlPage.swift b/ios/MullvadVPNUITests/Pages/TunnelControlPage.swift index 4a2eeee4c790..5a52c3129716 100644 --- a/ios/MullvadVPNUITests/Pages/TunnelControlPage.swift +++ b/ios/MullvadVPNUITests/Pages/TunnelControlPage.swift @@ -70,7 +70,7 @@ class TunnelControlPage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .tunnelControlView + self.pageElement = app.otherElements[.tunnelControlView] waitForPageToBeShown() } @@ -94,9 +94,25 @@ class TunnelControlPage: Page { return self } + /// Tap either cancel or disconnect button, depending on the current connection state. Use this function sparingly when it's irrelevant whether the app is currently connecting to a relay or already connected. + @discardableResult func tapCancelOrDisconnectButton() -> Self { + let cancelButton = app.buttons[.cancelButton] + let disconnectButton = app.buttons[.disconnectButton] + + if disconnectButton.exists && disconnectButton.isHittable { + disconnectButton.tap() + } else { + cancelButton.tap() + } + + return self + } + @discardableResult func waitForSecureConnectionLabel() -> Self { - _ = app.staticTexts[AccessibilityIdentifier.connectionStatusConnectedLabel] - .waitForExistence(timeout: BaseUITestCase.defaultTimeout) + let labelFound = app.staticTexts[.connectionStatusConnectedLabel] + .waitForExistence(timeout: BaseUITestCase.extremelyLongTimeout) + XCTAssertTrue(labelFound, "Secure connection label presented") + return self } @@ -193,4 +209,15 @@ class TunnelControlPage: Page { return String() } } + + func getCurrentRelayName() -> String { + let relayExpandButton = app.buttons[.relayStatusCollapseButton] + + guard let relayName = relayExpandButton.value as? String else { + XCTFail("Failed to read relay name from tunnel control page") + return String() + } + + return relayName + } } diff --git a/ios/MullvadVPNUITests/Pages/VPNSettingsPage.swift b/ios/MullvadVPNUITests/Pages/VPNSettingsPage.swift index 71adb3eaaa3f..a3c854ee8032 100644 --- a/ios/MullvadVPNUITests/Pages/VPNSettingsPage.swift +++ b/ios/MullvadVPNUITests/Pages/VPNSettingsPage.swift @@ -17,7 +17,7 @@ class VPNSettingsPage: Page { private func cellExpandCollapseButton(_ cellAccessiblityIdentifier: AccessibilityIdentifier) -> XCUIElement { let table = app.tables[AccessibilityIdentifier.vpnSettingsTableView] let matchingCells = table.otherElements.containing(.any, identifier: cellAccessiblityIdentifier.rawValue) - let expandButton = matchingCells.buttons[AccessibilityIdentifier.collapseButton] + let expandButton = matchingCells.buttons[AccessibilityIdentifier.expandButton] return expandButton } diff --git a/ios/MullvadVPNUITests/Pages/WelcomePage.swift b/ios/MullvadVPNUITests/Pages/WelcomePage.swift index d6e8048413b1..2a1ca9df2967 100644 --- a/ios/MullvadVPNUITests/Pages/WelcomePage.swift +++ b/ios/MullvadVPNUITests/Pages/WelcomePage.swift @@ -13,7 +13,7 @@ class WelcomePage: Page { @discardableResult override init(_ app: XCUIApplication) { super.init(app) - self.pageAccessibilityIdentifier = .welcomeView + self.pageElement = app.otherElements[.welcomeView] waitForPageToBeShown() } diff --git a/ios/MullvadVPNUITests/README.md b/ios/MullvadVPNUITests/README.md index b1fa392f2a38..703beee9f25e 100644 --- a/ios/MullvadVPNUITests/README.md +++ b/ios/MullvadVPNUITests/README.md @@ -4,6 +4,8 @@ 1. Make sure device is added to provisioning profiles 2. Disable passcode in iOS settings - otherwise tests cannot be started without manually entering passcode 3. Make sure device is configured in GitHub secrets(see *GitHub setup* below) +4. Make sure the test device is connected to the WiFi `app-team-ios-tests` +5. Make sure iCloud syncing of keychain is off on the device so that the device isn't getting WiFi passwords from another device causing it to sometimes connect to another WiFi. ## Set up of runner environment 1. Install Xcode @@ -34,6 +36,7 @@ - `IOS_NO_TIME_ACCOUNT_NUMBER` - Production server account with time added to it - `IOS_TEST_DEVICE_IDENTIFIER_UUID` - unique identifier for the test device. Create new identifier with `uuidgen`. - `IOS_TEST_DEVICE_UDID` - the iOS device's UDID. + - `PARTNER_API_TOKEN` - secret token for partner API. Optional and only intended to be used in CI when running tests against staging environment. ## Test plans There are a few different test plans which are mainly to be triggered by GitHub action workflows but can also be triggered manually with Xcode: diff --git a/ios/MullvadVPNUITests/RelayTests.swift b/ios/MullvadVPNUITests/RelayTests.swift index c8848e2a46cb..a08bb3f8105e 100644 --- a/ios/MullvadVPNUITests/RelayTests.swift +++ b/ios/MullvadVPNUITests/RelayTests.swift @@ -9,6 +9,11 @@ import Foundation import XCTest +private struct RelayInfo { + var name: String + var ipAddress: String +} + class RelayTests: LoggedInWithTimeUITestCase { var removeFirewallRulesInTearDown = false @@ -26,17 +31,16 @@ class RelayTests: LoggedInWithTimeUITestCase { } } - func testAppConnection() throws { - TunnelControlPage(app) - .tapSecureConnectionButton() - - allowAddVPNConfigurationsIfAsked() + /// Restore default country by selecting it in location selector and immediately disconnecting when app starts connecting to relay in it + private func restoreDefaultCountry() { + TunnelControlPage(self.app) + .tapSelectLocationButton() - TunnelControlPage(app) - .waitForSecureConnectionLabel() + SelectLocationPage(self.app) + .tapLocationCell(withName: BaseUITestCase.appDefaultCountry) - try Networking.verifyCanAccessInternet() - Networking.verifyConnectedThroughMullvad() + TunnelControlPage(self.app) + .tapCancelOrDisconnectButton() } func testAdBlockingViaDNS() throws { @@ -84,20 +88,41 @@ class RelayTests: LoggedInWithTimeUITestCase { .tapDisconnectButton() } + func testAppConnection() throws { + TunnelControlPage(app) + .tapSecureConnectionButton() + + allowAddVPNConfigurationsIfAsked() + + TunnelControlPage(app) + .waitForSecureConnectionLabel() + + try Networking.verifyCanAccessInternet() + try Networking.verifyConnectedThroughMullvad() + } + func testConnectionRetryLogic() throws { FirewallAPIClient().removeRules() removeFirewallRulesInTearDown = true - // First get relay IP address - let relayIPAddress = getGot001WireGuardRelayIPAddress() + addTeardownBlock { + self.restoreDefaultCountry() + } + + // First get relay info + let relayInfo = getDefaultRelayInfo() // Run actual test try FirewallAPIClient().createRule( - FirewallRule.makeBlockAllTrafficRule(toIPAddress: relayIPAddress) + FirewallRule.makeBlockAllTrafficRule(toIPAddress: relayInfo.ipAddress) ) TunnelControlPage(app) - .tapSecureConnectionButton() + .tapSelectLocationButton() + + SelectLocationPage(app) + .tapLocationCellExpandButton(withName: BaseUITestCase.testsDefaultCity) + .tapLocationCell(withName: relayInfo.name) // Should be two UDP connection attempts but sometimes only one is shown in the UI TunnelControlPage(app) @@ -151,12 +176,16 @@ class RelayTests: LoggedInWithTimeUITestCase { FirewallAPIClient().removeRules() removeFirewallRulesInTearDown = true - // First get relay IP address - let relayIPAddress = getGot001WireGuardRelayIPAddress() + addTeardownBlock { + self.restoreDefaultCountry() + } + + // First get relay info + let relayInfo = getDefaultRelayInfo() // Run actual test try FirewallAPIClient().createRule( - FirewallRule.makeBlockUDPTrafficRule(toIPAddress: relayIPAddress) + FirewallRule.makeBlockUDPTrafficRule(toIPAddress: relayInfo.ipAddress) ) HeaderBar(app) @@ -174,7 +203,11 @@ class RelayTests: LoggedInWithTimeUITestCase { .tapDoneButton() TunnelControlPage(app) - .tapSecureConnectionButton() + .tapSelectLocationButton() + + SelectLocationPage(app) + .tapLocationCellExpandButton(withName: BaseUITestCase.testsDefaultCity) + .tapLocationCell(withName: relayInfo.name) // Should be two UDP connection attempts but sometimes only one is shown in the UI TunnelControlPage(app) @@ -208,25 +241,23 @@ class RelayTests: LoggedInWithTimeUITestCase { TunnelControlPage(app) .tapRelayStatusExpandCollapseButton() .verifyConnectingToPort("4001") + .waitForSecureConnectionLabel() .tapDisconnectButton() } - /// Get got001 WireGuard relay IP address by connecting to it and checking which IP address the app connects to. Assumes user is logged on and at tunnel control page. - private func getGot001WireGuardRelayIPAddress() -> String { - let wireGuardGot001RelayName = "se-got-wg-001" - + /// Connect to a relay in the default country and city, get name and IP address of the relay the app successfully connects to. Assumes user is logged on and at tunnel control page. + private func getDefaultRelayInfo() -> RelayInfo { TunnelControlPage(app) .tapSelectLocationButton() - if SelectLocationPage(app).locationCellIsExpanded("Sweden") { - // Already expanded - just make sure correct relay is selected + if SelectLocationPage(app).locationCellIsExpanded(BaseUITestCase.testsDefaultCountry) { + // Already expanded - just make sure the correct city cell is selected SelectLocationPage(app) - .tapLocationCell(withName: wireGuardGot001RelayName) + .tapLocationCell(withName: BaseUITestCase.testsDefaultCity) } else { SelectLocationPage(app) - .tapLocationCellExpandButton(withName: "Sweden") - .tapLocationCellExpandButton(withName: "Gothenburg") - .tapLocationCell(withName: wireGuardGot001RelayName) + .tapLocationCellExpandButton(withName: BaseUITestCase.testsDefaultCountry) + .tapLocationCell(withName: BaseUITestCase.testsDefaultCity) } allowAddVPNConfigurationsIfAsked() @@ -236,10 +267,12 @@ class RelayTests: LoggedInWithTimeUITestCase { .tapRelayStatusExpandCollapseButton() .getInIPAddressFromConnectionStatus() + let relayName = TunnelControlPage(app).getCurrentRelayName() + TunnelControlPage(app) .tapDisconnectButton() - return relayIPAddress + return RelayInfo(name: relayName, ipAddress: relayIPAddress) } func testCustomDNS() throws { diff --git a/ios/MullvadVPNUITests/SettingsMigrationTests.swift b/ios/MullvadVPNUITests/SettingsMigrationTests.swift index 35d46b8c87cb..83e142f202ee 100644 --- a/ios/MullvadVPNUITests/SettingsMigrationTests.swift +++ b/ios/MullvadVPNUITests/SettingsMigrationTests.swift @@ -29,6 +29,12 @@ class SettingsMigrationTests: BaseUITestCase { } func testChangeCustomDNSSettings() { + let hasTimeAccountNumber = getAccountWithTime() + + addTeardownBlock { + self.returnAccountWithTime(accountNumber: hasTimeAccountNumber) + } + logoutIfLoggedIn() login(accountNumber: hasTimeAccountNumber) @@ -52,6 +58,12 @@ class SettingsMigrationTests: BaseUITestCase { } func testChangeSettings() { + let hasTimeAccountNumber = getAccountWithTime() + + addTeardownBlock { + self.returnAccountWithTime(accountNumber: hasTimeAccountNumber) + } + logoutIfLoggedIn() login(accountNumber: hasTimeAccountNumber) diff --git a/ios/MullvadVPNUITests/Test base classes/BaseUITestCase.swift b/ios/MullvadVPNUITests/Test base classes/BaseUITestCase.swift index 3551f482feec..15fd0ded2bae 100644 --- a/ios/MullvadVPNUITests/Test base classes/BaseUITestCase.swift +++ b/ios/MullvadVPNUITests/Test base classes/BaseUITestCase.swift @@ -13,20 +13,86 @@ class BaseUITestCase: XCTestCase { let app = XCUIApplication() static let defaultTimeout = 5.0 static let longTimeout = 15.0 - static let veryLongTimeout = 60.0 + static let veryLongTimeout = 20.0 + static let extremelyLongTimeout = 180.0 static let shortTimeout = 1.0 + /// The apps default country - the preselected country location after fresh install + static let appDefaultCountry = "Sweden" + + /// Default country to use in tests. + static let testsDefaultCountry = "Sweden" + + /// Default city to use in tests + static let testsDefaultCity = "Gothenburg" + + /// Default relay to use in tests + static let testsDefaultRelay = "se-got-wg-001" + // swiftlint:disable force_cast let displayName = Bundle(for: BaseUITestCase.self) .infoDictionary?["DisplayName"] as! String - let hasTimeAccountNumber = Bundle(for: BaseUITestCase.self) - .infoDictionary?["HasTimeAccountNumber"] as! String - let fiveWireGuardKeysAccountNumber = Bundle(for: BaseUITestCase.self) - .infoDictionary?["FiveWireGuardKeysAccountNumber"] as! String + private let bundleHasTimeAccountNumber = Bundle(for: BaseUITestCase.self) + .infoDictionary?["HasTimeAccountNumber"] as? String + private let bundleNoTimeAccountNumber = Bundle(for: BaseUITestCase.self) + .infoDictionary?["NoTimeAccountNumber"] as? String let iOSDevicePinCode = Bundle(for: BaseUITestCase.self) .infoDictionary?["IOSDevicePinCode"] as! String + let attachAppLogsOnFailure = Bundle(for: BaseUITestCase.self) + .infoDictionary?["AttachAppLogsOnFailure"] as! String == "1" // swiftlint:enable force_cast + /// Get an account number with time. If an account with time is specified in the configuration file that account will be used, else a temporary account will be created. + func getAccountWithTime() -> String { + if let configuredAccountWithTime = bundleHasTimeAccountNumber, !configuredAccountWithTime.isEmpty { + return configuredAccountWithTime + } else { + let partnerAPIClient = PartnerAPIClient() + let accountNumber = partnerAPIClient.createAccount() + _ = partnerAPIClient.addTime(accountNumber: accountNumber, days: 1) + return accountNumber + } + } + + /// Return account with time after done using it This is neccessary because if a temporary account was created we want to delete it. + func returnAccountWithTime(accountNumber: String) { + if bundleHasTimeAccountNumber?.isEmpty == true { + PartnerAPIClient().deleteAccount(accountNumber: accountNumber) + } + } + + /// Get an account number without time. If an account without time is specified in the configuration file that account will be used, else a temporary account will be created. + func getAccountWithoutTime() -> String { + if let configuredAccountWithoutTime = bundleNoTimeAccountNumber, !configuredAccountWithoutTime.isEmpty { + return configuredAccountWithoutTime + } else { + let partnerAPIClient = PartnerAPIClient() + let accountNumber = partnerAPIClient.createAccount() + return accountNumber + } + } + + func returnAccountWithoutTime(accountNumber: String) { + if bundleNoTimeAccountNumber?.isEmpty == true { + PartnerAPIClient().deleteAccount(accountNumber: accountNumber) + } + } + + /// Handle iOS add VPN configuration permission alert - allow and enter device PIN code + func allowAddVPNConfigurations() { + let springboard = XCUIApplication(bundleIdentifier: "com.apple.springboard") + + let alertAllowButton = springboard.buttons.element(boundBy: 0) + if alertAllowButton.waitForExistence(timeout: Self.defaultTimeout) { + alertAllowButton.tap() + } + + if iOSDevicePinCode.isEmpty == false { + _ = springboard.buttons["1"].waitForExistence(timeout: Self.defaultTimeout) + springboard.typeText(iOSDevicePinCode) + } + } + /// Handle iOS add VPN configuration permission alert if presented, otherwise ignore func allowAddVPNConfigurationsIfAsked() { let springboard = XCUIApplication(bundleIdentifier: "com.apple.springboard") @@ -63,7 +129,7 @@ class BaseUITestCase: XCTestCase { return true } - /// Suite level teardown ran after test have executed + /// Suite level teardown ran after all tests in suite have been executed override class func tearDown() { if shouldUninstallAppInTeardown() { uninstallApp() @@ -79,6 +145,32 @@ class BaseUITestCase: XCTestCase { /// Test level teardown override func tearDown() { app.terminate() + + if let testRun = self.testRun, testRun.failureCount > 0, attachAppLogsOnFailure == true { + app.launch() + + HeaderBar(app) + .tapSettingsButton() + + SettingsPage(app) + .tapReportAProblemCell() + + ProblemReportPage(app) + .tapViewAppLogsButton() + + let logText = AppLogsPage(app) + .getAppLogText() + + // Attach app log to result + let dateFormatter = DateFormatter() + dateFormatter.dateFormat = "yyyy-MM-dd_HH-mm-ss" + let dateString = dateFormatter.string(from: Date()) + let attachment = XCTAttachment(string: logText) + attachment.name = "app-log-\(dateString).log" + add(attachment) + + app.terminate() + } } /// Check if currently logged on to an account. Note that it is assumed that we are logged in if login view isn't currently shown. @@ -96,10 +188,9 @@ class BaseUITestCase: XCTestCase { func agreeToTermsOfServiceIfShown() { let termsOfServiceIsShown = app.otherElements[ - AccessibilityIdentifier - .termsOfServiceView.rawValue + .termsOfServiceView ] - .waitForExistence(timeout: 1) + .waitForExistence(timeout: Self.shortTimeout) if termsOfServiceIsShown { TermsOfServicePage(app) @@ -109,8 +200,8 @@ class BaseUITestCase: XCTestCase { func dismissChangeLogIfShown() { let changeLogIsShown = app - .otherElements[AccessibilityIdentifier.changeLogAlert.rawValue] - .waitForExistence(timeout: 1.0) + .otherElements[.changeLogAlert] + .waitForExistence(timeout: Self.shortTimeout) if changeLogIsShown { ChangeLogAlert(app) @@ -125,10 +216,28 @@ class BaseUITestCase: XCTestCase { /// Login with specified account number. It is a prerequisite that the login page is currently shown. func login(accountNumber: String) { + var successIconShown = false + var retryCount = 0 + let maxRetryCount = 3 + LoginPage(app) .tapAccountNumberTextField() .enterText(accountNumber) - .tapAccountNumberSubmitButton() + + repeat { + successIconShown = LoginPage(app) + .tapAccountNumberSubmitButton() + .getSuccessIconShown() + + if successIconShown == false { + // Give it some time to show up. App might be waiting for a network connection to timeout. + LoginPage(app).waitForAccountNumberSubmitButton() + } + + retryCount += 1 + } while successIconShown == false && retryCount < maxRetryCount + + HeaderBar(app) .verifyDeviceLabelShown() } @@ -145,6 +254,7 @@ class BaseUITestCase: XCTestCase { .tapAccountButton() AccountPage(app) .tapLogOutButton() + .waitForLogoutSpinnerToDisappear() } else { // Workaround for revoked device view not showing account button RevokedDevicePage(app) @@ -157,13 +267,18 @@ class BaseUITestCase: XCTestCase { static func uninstallApp() { let appName = "Mullvad VPN" + let searchQuery = appName + .replacingOccurrences( + of: " ", + with: "" + ) // With space in the query Spotlight search sometimes don't match the Mullvad VPN app let timeout = TimeInterval(5) let springboard = XCUIApplication(bundleIdentifier: "com.apple.springboard") let spotlight = XCUIApplication(bundleIdentifier: "com.apple.Spotlight") springboard.swipeDown() - spotlight.textFields["SpotlightSearchField"].typeText(appName) + spotlight.textFields["SpotlightSearchField"].typeText(searchQuery) let appIcon = spotlight.icons[appName].firstMatch if appIcon.waitForExistence(timeout: timeout) { diff --git a/ios/MullvadVPNUITests/Test base classes/LoggedInWithTimeUITestCase.swift b/ios/MullvadVPNUITests/Test base classes/LoggedInWithTimeUITestCase.swift index fcf6d0f5e2cb..f5c1c011c564 100644 --- a/ios/MullvadVPNUITests/Test base classes/LoggedInWithTimeUITestCase.swift +++ b/ios/MullvadVPNUITests/Test base classes/LoggedInWithTimeUITestCase.swift @@ -7,20 +7,42 @@ // import Foundation +import XCTest /// Base class for tests that should start from a state of being logged on to an account with time left class LoggedInWithTimeUITestCase: BaseUITestCase { + var hasTimeAccountNumber: String? + override func setUp() { super.setUp() + hasTimeAccountNumber = getAccountWithTime() + agreeToTermsOfServiceIfShown() dismissChangeLogIfShown() logoutIfLoggedIn() + guard let hasTimeAccountNumber = self.hasTimeAccountNumber else { + XCTFail("hasTimeAccountNumber unexpectedly not set") + return + } + login(accountNumber: hasTimeAccountNumber) // Relaunch app so that tests start from a deterministic state app.terminate() app.launch() } + + override func tearDown() { + super.tearDown() + + guard let hasTimeAccountNumber = self.hasTimeAccountNumber else { + XCTFail("hasTimeAccountNumber unexpectedly not set") + return + } + + app.terminate() // Terminate app to make sure we have Internet connectivity + self.returnAccountWithTime(accountNumber: hasTimeAccountNumber) + } }