From 6b07ad3fb68a32b250df2f628d3df02afc1f4b83 Mon Sep 17 00:00:00 2001 From: Niklas Berglund Date: Mon, 25 Mar 2024 14:54:24 +0100 Subject: [PATCH 1/2] Add settings migration tests for iOS --- .../actions/ios-end-to-end-tests/action.yml | 59 +++++++ ...os-end-to-end-tests-settings-migration.yml | 116 ++++++++++++ .github/workflows/ios-end-to-end-tests.yml | 69 ++++---- ios/MullvadVPN.xcodeproj/project.pbxproj | 34 +++- .../xcschemes/MullvadVPNUITests.xcscheme | 12 ++ .../Classes/AccessbilityIdentifier.swift | 6 + .../Settings/SettingsDNSTextCell.swift | 1 + .../VPNSettings/CustomDNSCellFactory.swift | 2 + .../VPNSettings/CustomDNSViewController.swift | 2 + .../VPNSettings/VPNSettingsCellFactory.swift | 2 +- .../VPNSettings/VPNSettingsDataSource.swift | 2 +- ios/MullvadVPNUITests/ConnectivityTests.swift | 4 - .../Pages/DNSSettingsPage.swift | 165 ++++++++++++++++++ .../Pages/SettingsPage.swift | 24 --- .../Pages/VPNSettingsPage.swift | 80 ++++++++- ios/MullvadVPNUITests/README.md | 22 ++- ios/MullvadVPNUITests/RelayTests.swift | 6 +- .../SettingsMigrationTests.swift | 138 +++++++++++++++ .../Test base classes/BaseUITestCase.swift | 14 +- .../XCUIElement+Extensions.swift | 18 ++ ios/TestPlans/MullvadVPNUITestsAll.xctestplan | 3 +- ...lvadVPNUITestsChangeDNSSettings.xctestplan | 28 +++ ...MullvadVPNUITestsChangeSettings.xctestplan | 28 +++ ...UITestsVerifyDNSSettingsChanged.xctestplan | 28 +++ ...VPNUITestsVerifySettingsChanged.xctestplan | 28 +++ 25 files changed, 811 insertions(+), 80 deletions(-) create mode 100644 .github/actions/ios-end-to-end-tests/action.yml create mode 100644 .github/workflows/ios-end-to-end-tests-settings-migration.yml create mode 100644 ios/MullvadVPNUITests/Pages/DNSSettingsPage.swift create mode 100644 ios/MullvadVPNUITests/SettingsMigrationTests.swift create mode 100644 ios/MullvadVPNUITests/XCUIElement+Extensions.swift create mode 100644 ios/TestPlans/MullvadVPNUITestsChangeDNSSettings.xctestplan create mode 100644 ios/TestPlans/MullvadVPNUITestsChangeSettings.xctestplan create mode 100644 ios/TestPlans/MullvadVPNUITestsVerifyDNSSettingsChanged.xctestplan create mode 100644 ios/TestPlans/MullvadVPNUITestsVerifySettingsChanged.xctestplan diff --git a/.github/actions/ios-end-to-end-tests/action.yml b/.github/actions/ios-end-to-end-tests/action.yml new file mode 100644 index 000000000000..9d771d151c5a --- /dev/null +++ b/.github/actions/ios-end-to-end-tests/action.yml @@ -0,0 +1,59 @@ +name: 'iOS end to end tests action' +description: 'Prepares and runs end to end tests on iOS device' +inputs: + ios_device_pin_code: + description: 'iOS Device Pin Code' + required: true + test_device_identifier_uuid: + description: 'Test Device Identifier UUID' + required: true + has_time_account_number: + description: 'Has Time Account Number' + required: true + no_time_account_number: + description: 'No Time Account Number' + required: true + test_device_udid: + description: 'Test Device UDID' + required: true + xcode_test_plan: + description: 'Xcode Test Plan to run' + required: true + +runs: + using: 'composite' + steps: + - name: Configure Xcode project + run: | + for file in *.xcconfig.template ; do cp $file ${file//.template/} ; done + sed -i "" "/^HAS_TIME_ACCOUNT_NUMBER/d" UITests.xcconfig + sed -i "" "/^NO_TIME_ACCOUNT_NUMBER/d" UITests.xcconfig + sed -i "" \ + "/IOS_DEVICE_PIN_CODE =/ s/= .*/= $IOS_DEVICE_PIN_CODE/" \ + UITests.xcconfig + sed -i "" \ + "/TEST_DEVICE_IDENTIFIER_UUID =/ s/= .*/= $TEST_DEVICE_IDENTIFIER_UUID/" \ + UITests.xcconfig + echo -e "\nHAS_TIME_ACCOUNT_NUMBER = $HAS_TIME_ACCOUNT_NUMBER" >> UITests.xcconfig + echo "NO_TIME_ACCOUNT_NUMBER = $NO_TIME_ACCOUNT_NUMBER" >> UITests.xcconfig + shell: bash + working-directory: ios/Configurations + env: + IOS_DEVICE_PIN_CODE: ${{ inputs.ios_device_pin_code }} + TEST_DEVICE_IDENTIFIER_UUID: ${{ inputs.test_device_identifier_uuid }} + HAS_TIME_ACCOUNT_NUMBER: ${{ inputs.has_time_account_number }} + NO_TIME_ACCOUNT_NUMBER: ${{ inputs.no_time_account_number }} + + - name: Run end-to-end-tests + run: | + set -o pipefail && env NSUnbufferedIO=YES xcodebuild \ + -project MullvadVPN.xcodeproj \ + -scheme MullvadVPNUITests \ + -testPlan $XCODE_TEST_PLAN \ + -destination "platform=iOS,id=$TEST_DEVICE_UDID" \ + clean test 2>&1 | xcbeautify --report junit --report-path test-report + shell: bash + working-directory: ios/ + env: + XCODE_TEST_PLAN: ${{ inputs.xcode_test_plan }} + TEST_DEVICE_UDID: ${{ inputs.test_device_udid }} diff --git a/.github/workflows/ios-end-to-end-tests-settings-migration.yml b/.github/workflows/ios-end-to-end-tests-settings-migration.yml new file mode 100644 index 000000000000..180f454cecfe --- /dev/null +++ b/.github/workflows/ios-end-to-end-tests-settings-migration.yml @@ -0,0 +1,116 @@ +--- +name: iOS settings migration tests +permissions: + contents: read +on: + workflow_dispatch: + schedule: + # At midnight every day. + # Notifications for scheduled workflows are sent to the user who last modified the cron + # syntax in the workflow file. If you update this you must have notifications for + # Github Actions enabled, so these don't go unnoticed. + # https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/notifications-for-workflow-runs + - cron: '0 0 * * *' +jobs: + test: + name: Settings migration end to end tests + runs-on: [self-hosted, macOS, ios-test] + env: + OLD_APP_COMMIT_HASH: f741bf159d5719301e4d1fd099b3d353457532fd + steps: + - name: Configure Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + target: aarch64-apple-ios + + - name: Uninstall app + run: ios-deploy --id ${{ secrets.IOS_TEST_DEVICE_UDID }} --uninstall_only --bundle_id net.mullvad.MullvadVPN + + - name: Checkout old repository version + uses: actions/checkout@v4 + with: + ref: ${{ env.OLD_APP_COMMIT_HASH }} + + - name: Change DNS settings on old app version + uses: ./.github/actions/ios-end-to-end-tests + with: + ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }} + test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }} + has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }} + no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} + test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} + xcode_test_plan: 'MullvadVPNUITestsChangeDNSSettings' + + - name: Store test report for changing DNS settings + uses: actions/upload-artifact@v4 + if: always() + with: + name: test-report-change-dns-settings + path: ios/test-report/junit.xml + + - name: Checkout repository to get the current app version + uses: actions/checkout@v4 + + - name: Verify DNS settings still changed on current app version + uses: ./.github/actions/ios-end-to-end-tests + if: always() + with: + ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }} + test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }} + has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }} + no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} + test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} + xcode_test_plan: 'MullvadVPNUITestsVerifyDNSSettingsChanged' + + - name: Store test report for verifying DNS settings + uses: actions/upload-artifact@v4 + if: always() + with: + name: test-report-verify-dns-settings + path: ios/test-report/junit.xml + + - name: Checkout old repository version + uses: actions/checkout@v4 + with: + ref: ${{ env.OLD_APP_COMMIT_HASH }} + + - name: Change all other settings on old app version + uses: ./.github/actions/ios-end-to-end-tests + if: always() + with: + ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }} + test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }} + has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }} + no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} + test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} + xcode_test_plan: 'MullvadVPNUITestsChangeSettings' + + - name: Store test report for changing all settings + uses: actions/upload-artifact@v4 + if: always() + with: + name: test-report-change-all-other-settings + path: ios/test-report/junit.xml + + - name: Checkout repository to get the current app version + uses: actions/checkout@v4 + + - name: Verify all other settings still changed on current app version + uses: ./.github/actions/ios-end-to-end-tests + if: always() + with: + ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }} + test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }} + has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }} + no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} + test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} + xcode_test_plan: 'MullvadVPNUITestsVerifySettingsChanged' + + - name: Store test report for verifying all other settings + uses: actions/upload-artifact@v4 + if: always() + with: + name: test-report-verify-all-other-settings + path: ios/test-report/junit.xml diff --git a/.github/workflows/ios-end-to-end-tests.yml b/.github/workflows/ios-end-to-end-tests.yml index 6f953cd34612..b44665649f40 100644 --- a/.github/workflows/ios-end-to-end-tests.yml +++ b/.github/workflows/ios-end-to-end-tests.yml @@ -14,56 +14,51 @@ on: - .github/workflows/ios-end-to-end-tests.yml - ios/** workflow_dispatch: + schedule: + # At midnight every day. + # Notifications for scheduled workflows are sent to the user who last modified the cron + # syntax in the workflow file. If you update this you must have notifications for + # Github Actions enabled, so these don't go unnoticed. + # https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/notifications-for-workflow-runs + - cron: '0 0 * * *' jobs: test: if: github.event.pull_request.merged || github.event_name == 'workflow_dispatch' name: End to end tests runs-on: [self-hosted, macOS, ios-test] - env: - IOS_DEVICE_PIN_CODE: ${{ secrets.IOS_DEVICE_PIN_CODE }} - TEST_DEVICE_IDENTIFIER_UUID: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }} - TEST_DEVICE_UDID: ${{ secrets.IOS_TEST_DEVICE_UDID }} - HAS_TIME_ACCOUNT_NUMBER: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }} - NO_TIME_ACCOUNT_NUMBER: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} steps: + - name: Configure Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + target: aarch64-apple-ios + - name: Checkout repository uses: actions/checkout@v4 - - name: Configure Rust - run: | - rustup default stable - rustup update stable - rustup target add aarch64-apple-ios aarch64-apple-ios-sim - - - name: Configure Xcode project + - name: Select test plan to execute run: | - for file in *.xcconfig.template ; do cp $file ${file//.template/} ; done - sed -i "" "/^HAS_TIME_ACCOUNT_NUMBER/d" UITests.xcconfig - sed -i "" "/^NO_TIME_ACCOUNT_NUMBER/d" UITests.xcconfig - sed -i "" \ - "/IOS_DEVICE_PIN_CODE =/ s/= .*/= $IOS_DEVICE_PIN_CODE/" \ - UITests.xcconfig - sed -i "" \ - "/TEST_DEVICE_IDENTIFIER_UUID =/ s/= .*/= $TEST_DEVICE_IDENTIFIER_UUID/" \ - UITests.xcconfig - echo -e "\nHAS_TIME_ACCOUNT_NUMBER = $HAS_TIME_ACCOUNT_NUMBER" >> UITests.xcconfig - echo "NO_TIME_ACCOUNT_NUMBER = $NO_TIME_ACCOUNT_NUMBER" >> UITests.xcconfig - working-directory: ios/Configurations + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + echo "XCODE_TEST_PLAN=MullvadVPNUITestsSmoke" >> $GITHUB_ENV + elif [[ "${{ github.event_name }}" == "schedule" ]]; then + echo "XCODE_TEST_PLAN=MullvadVPNUITestsAll" >> $GITHUB_ENV + elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "XCODE_TEST_PLAN=MullvadVPNUITestsAll" >> $GITHUB_ENV + fi - - name: Run end-to-end-tests - run: | - set -o pipefail && env NSUnbufferedIO=YES xcodebuild \ - -project MullvadVPN.xcodeproj \ - -scheme MullvadVPNUITests \ - -configuration Debug \ - -testPlan MullvadVPNUITestsSmoke \ - -destination "platform=iOS,id=$TEST_DEVICE_UDID" \ - -disableAutomaticPackageResolution \ - test 2>&1 | xcbeautify --report junit --report-path test-report - working-directory: ios/ + - name: iOS end to end tests action + uses: ./.github/actions/ios-end-to-end-tests + with: + xcode_test_plan: ${{ env.XCODE_TEST_PLAN }} + ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }} + test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }} + has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }} + no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} + test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} - name: Comment PR on test failure - if: failure() && github.event_name != 'workflow_dispatch' + if: failure() && github.event_name == 'pull_request' uses: actions/github-script@v7 with: github-token: ${{secrets.GITHUB_TOKEN}} diff --git a/ios/MullvadVPN.xcodeproj/project.pbxproj b/ios/MullvadVPN.xcodeproj/project.pbxproj index f1ee25f1a1e3..af62482d1283 100644 --- a/ios/MullvadVPN.xcodeproj/project.pbxproj +++ b/ios/MullvadVPN.xcodeproj/project.pbxproj @@ -617,6 +617,11 @@ 852969362B4E9724007EAD4C /* AccessbilityIdentifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7A0B311D2B303A0D004B12E0 /* AccessbilityIdentifier.swift */; }; 8529693A2B4F0238007EAD4C /* TermsOfServicePage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 852969392B4F0238007EAD4C /* TermsOfServicePage.swift */; }; 8529693C2B4F0257007EAD4C /* Alert.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8529693B2B4F0257007EAD4C /* Alert.swift */; }; + 852A26462BA9C9CB006EB9C8 /* DNSSettingsPage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 852A26452BA9C9CB006EB9C8 /* DNSSettingsPage.swift */; }; + 852BC66F2BAB44F500A47558 /* MullvadVPNUITestsVerifySettingsChanged.xctestplan in Resources */ = {isa = PBXBuildFile; fileRef = 852BC66C2BAB44F500A47558 /* MullvadVPNUITestsVerifySettingsChanged.xctestplan */; }; + 852BC6702BAB44F500A47558 /* MullvadVPNUITestsChangeSettings.xctestplan in Resources */ = {isa = PBXBuildFile; fileRef = 852BC66D2BAB44F500A47558 /* MullvadVPNUITestsChangeSettings.xctestplan */; }; + 852BC6712BAB44F500A47558 /* MullvadVPNUITestsVerifyDNSSettingsChanged.xctestplan in Resources */ = {isa = PBXBuildFile; fileRef = 852BC66E2BAB44F500A47558 /* MullvadVPNUITestsVerifyDNSSettingsChanged.xctestplan */; }; + 852BC6732BAB450B00A47558 /* MullvadVPNUITestsChangeDNSSettings.xctestplan in Resources */ = {isa = PBXBuildFile; fileRef = 852BC6722BAB450B00A47558 /* MullvadVPNUITestsChangeDNSSettings.xctestplan */; }; 8532E6872B8CCED600ACECD1 /* ProblemReportSubmittedPage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8532E6862B8CCED600ACECD1 /* ProblemReportSubmittedPage.swift */; }; 8542CE242B95F7B9006FCA14 /* VPNSettingsPage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8542CE232B95F7B9006FCA14 /* VPNSettingsPage.swift */; }; 85557B0E2B591B2600795FE1 /* FirewallAPIClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 85557B0D2B591B2600795FE1 /* FirewallAPIClient.swift */; }; @@ -632,8 +637,10 @@ 8587A05D2B84D43100152938 /* ChangeLogAlert.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8587A05C2B84D43100152938 /* ChangeLogAlert.swift */; }; 8590896C2B61763B003AF5F5 /* LoggedInWithoutTimeUITestCase.swift in Sources */ = {isa = PBXBuildFile; fileRef = 859089682B61763B003AF5F5 /* LoggedInWithoutTimeUITestCase.swift */; }; 8590896F2B61763B003AF5F5 /* LoggedOutUITestCase.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8590896B2B61763B003AF5F5 /* LoggedOutUITestCase.swift */; }; + 85A42B862BB1D627007BABF7 /* XCUIElement+Extensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 85A42B852BB1D627007BABF7 /* XCUIElement+Extensions.swift */; }; 85B267612B849ADB0098E3CD /* mullvad-api.h in Headers */ = {isa = PBXBuildFile; fileRef = 85B267602B849ADB0098E3CD /* mullvad-api.h */; }; 85C7A2E92B89024B00035D5A /* SettingsTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 85C7A2E82B89024B00035D5A /* SettingsTests.swift */; }; + 85D039982BA4711800940E7F /* SettingsMigrationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 85D039972BA4711800940E7F /* SettingsMigrationTests.swift */; }; 85D2B0B12B6BD32400DF9DA7 /* BaseUITestCase.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8590896A2B61763B003AF5F5 /* BaseUITestCase.swift */; }; 85E3BDE52B70E18C00FA71FD /* Networking.swift in Sources */ = {isa = PBXBuildFile; fileRef = 85E3BDE42B70E18C00FA71FD /* Networking.swift */; }; 85EC620C2B838D10005AFFB5 /* MullvadAPIWrapper.swift in Sources */ = {isa = PBXBuildFile; fileRef = 85557B132B5983CF00795FE1 /* MullvadAPIWrapper.swift */; }; @@ -1861,6 +1868,11 @@ 852969382B4ED818007EAD4C /* UITests.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = UITests.xcconfig; sourceTree = ""; }; 852969392B4F0238007EAD4C /* TermsOfServicePage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TermsOfServicePage.swift; sourceTree = ""; }; 8529693B2B4F0257007EAD4C /* Alert.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Alert.swift; sourceTree = ""; }; + 852A26452BA9C9CB006EB9C8 /* DNSSettingsPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DNSSettingsPage.swift; sourceTree = ""; }; + 852BC66C2BAB44F500A47558 /* MullvadVPNUITestsVerifySettingsChanged.xctestplan */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = MullvadVPNUITestsVerifySettingsChanged.xctestplan; sourceTree = ""; }; + 852BC66D2BAB44F500A47558 /* MullvadVPNUITestsChangeSettings.xctestplan */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = MullvadVPNUITestsChangeSettings.xctestplan; sourceTree = ""; }; + 852BC66E2BAB44F500A47558 /* MullvadVPNUITestsVerifyDNSSettingsChanged.xctestplan */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = MullvadVPNUITestsVerifyDNSSettingsChanged.xctestplan; sourceTree = ""; }; + 852BC6722BAB450B00A47558 /* MullvadVPNUITestsChangeDNSSettings.xctestplan */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = MullvadVPNUITestsChangeDNSSettings.xctestplan; sourceTree = ""; }; 8532E6862B8CCED600ACECD1 /* ProblemReportSubmittedPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProblemReportSubmittedPage.swift; sourceTree = ""; }; 8542CE232B95F7B9006FCA14 /* VPNSettingsPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VPNSettingsPage.swift; sourceTree = ""; }; 85557B0D2B591B2600795FE1 /* FirewallAPIClient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FirewallAPIClient.swift; sourceTree = ""; }; @@ -1879,8 +1891,10 @@ 859089692B61763B003AF5F5 /* LoggedInWithTimeUITestCase.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = LoggedInWithTimeUITestCase.swift; sourceTree = ""; }; 8590896A2B61763B003AF5F5 /* BaseUITestCase.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = BaseUITestCase.swift; sourceTree = ""; }; 8590896B2B61763B003AF5F5 /* LoggedOutUITestCase.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = LoggedOutUITestCase.swift; sourceTree = ""; }; + 85A42B852BB1D627007BABF7 /* XCUIElement+Extensions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "XCUIElement+Extensions.swift"; sourceTree = ""; }; 85B267602B849ADB0098E3CD /* mullvad-api.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = "mullvad-api.h"; path = "../../mullvad-api/include/mullvad-api.h"; sourceTree = ""; }; 85C7A2E82B89024B00035D5A /* SettingsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsTests.swift; sourceTree = ""; }; + 85D039972BA4711800940E7F /* SettingsMigrationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsMigrationTests.swift; sourceTree = ""; }; 85E3BDE42B70E18C00FA71FD /* Networking.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Networking.swift; sourceTree = ""; }; 85FB5A0B2B6903990015DCED /* WelcomePage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WelcomePage.swift; sourceTree = ""; }; 85FB5A0F2B6960A30015DCED /* AccountDeletionPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AccountDeletionPage.swift; sourceTree = ""; }; @@ -3508,11 +3522,15 @@ 7A83C3FC2A55B39500DFB83A /* TestPlans */ = { isa = PBXGroup; children = ( - 852969302B4D9E70007EAD4C /* MullvadVPNUITestsAll.xctestplan */, - 85006A8E2B73EF67004AD8FB /* MullvadVPNUITestsSmoke.xctestplan */, 7A83C3FE2A55B72E00DFB83A /* MullvadVPNApp.xctestplan */, 7A83C4002A55B81A00DFB83A /* MullvadVPNCI.xctestplan */, 7A02D4EA2A9CEC7A00C19E31 /* MullvadVPNScreenshots.xctestplan */, + 852969302B4D9E70007EAD4C /* MullvadVPNUITestsAll.xctestplan */, + 852BC6722BAB450B00A47558 /* MullvadVPNUITestsChangeDNSSettings.xctestplan */, + 852BC66D2BAB44F500A47558 /* MullvadVPNUITestsChangeSettings.xctestplan */, + 85006A8E2B73EF67004AD8FB /* MullvadVPNUITestsSmoke.xctestplan */, + 852BC66E2BAB44F500A47558 /* MullvadVPNUITestsVerifyDNSSettingsChanged.xctestplan */, + 852BC66C2BAB44F500A47558 /* MullvadVPNUITestsVerifySettingsChanged.xctestplan */, ); path = TestPlans; sourceTree = ""; @@ -3587,7 +3605,9 @@ 850201DA2B503D7700EF8C96 /* RelayTests.swift */, 8518F6392B601910009EB113 /* Test base classes */, 85557B152B5ABBBE00795FE1 /* XCUIElementQuery+Extensions.swift */, + 85A42B852BB1D627007BABF7 /* XCUIElement+Extensions.swift */, 85C7A2E82B89024B00035D5A /* SettingsTests.swift */, + 85D039972BA4711800940E7F /* SettingsMigrationTests.swift */, ); path = MullvadVPNUITests; sourceTree = ""; @@ -3599,6 +3619,7 @@ 85557B1F2B5FBBD700795FE1 /* AccountPage.swift */, 8529693B2B4F0257007EAD4C /* Alert.swift */, 8587A05C2B84D43100152938 /* ChangeLogAlert.swift */, + 852A26452BA9C9CB006EB9C8 /* DNSSettingsPage.swift */, 85557B1D2B5FB8C700795FE1 /* HeaderBar.swift */, 852969342B4E9270007EAD4C /* LoginPage.swift */, 85139B2C2B84B4A700734217 /* OutOfTimePage.swift */, @@ -3610,8 +3631,8 @@ 850201E22B51A93C00EF8C96 /* SettingsPage.swift */, 852969392B4F0238007EAD4C /* TermsOfServicePage.swift */, 850201DE2B5040A500EF8C96 /* TunnelControlPage.swift */, - 85FB5A0B2B6903990015DCED /* WelcomePage.swift */, 8542CE232B95F7B9006FCA14 /* VPNSettingsPage.swift */, + 85FB5A0B2B6903990015DCED /* WelcomePage.swift */, ); path = Pages; sourceTree = ""; @@ -4564,6 +4585,10 @@ isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; files = ( + 852BC66F2BAB44F500A47558 /* MullvadVPNUITestsVerifySettingsChanged.xctestplan in Resources */, + 852BC6712BAB44F500A47558 /* MullvadVPNUITestsVerifyDNSSettingsChanged.xctestplan in Resources */, + 852BC6702BAB44F500A47558 /* MullvadVPNUITestsChangeSettings.xctestplan in Resources */, + 852BC6732BAB450B00A47558 /* MullvadVPNUITestsChangeDNSSettings.xctestplan in Resources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -5580,8 +5605,10 @@ 85D2B0B12B6BD32400DF9DA7 /* BaseUITestCase.swift in Sources */, 8529693C2B4F0257007EAD4C /* Alert.swift in Sources */, 850201DD2B503D8C00EF8C96 /* SelectLocationPage.swift in Sources */, + 85D039982BA4711800940E7F /* SettingsMigrationTests.swift in Sources */, 850201DB2B503D7700EF8C96 /* RelayTests.swift in Sources */, 85139B2D2B84B4A700734217 /* OutOfTimePage.swift in Sources */, + 85A42B862BB1D627007BABF7 /* XCUIElement+Extensions.swift in Sources */, 852969362B4E9724007EAD4C /* AccessbilityIdentifier.swift in Sources */, 85E3BDE52B70E18C00FA71FD /* Networking.swift in Sources */, 85C7A2E92B89024B00035D5A /* SettingsTests.swift in Sources */, @@ -5600,6 +5627,7 @@ 8529693A2B4F0238007EAD4C /* TermsOfServicePage.swift in Sources */, 8532E6872B8CCED600ACECD1 /* ProblemReportSubmittedPage.swift in Sources */, 85FB5A0C2B6903990015DCED /* WelcomePage.swift in Sources */, + 852A26462BA9C9CB006EB9C8 /* DNSSettingsPage.swift in Sources */, 850201DF2B5040A500EF8C96 /* TunnelControlPage.swift in Sources */, 8542CE242B95F7B9006FCA14 /* VPNSettingsPage.swift in Sources */, 85557B1E2B5FB8C700795FE1 /* HeaderBar.swift in Sources */, diff --git a/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme b/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme index 7603c9373776..a206bbdc7147 100644 --- a/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme +++ b/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme @@ -19,6 +19,18 @@ + + + + + + + + Self { + let switchElement = app.cells[accessibilityIdentifier] + .switches[AccessibilityIdentifier.customSwitch] + + guard let switchValue = switchElement.value as? String else { + XCTFail("Failed to read switch state") + return self + } + + XCTAssertEqual(switchValue, "1") + + return self + } + + @discardableResult func tapBackButton() -> Self { + // Workaround for setting accessibility identifier on navigation bar button being non-trivial + app.buttons.matching(identifier: "VPN settings").allElementsBoundByIndex.last?.tap() + return self + } + + @discardableResult func tapDNSContentBlockersHeaderExpandButton() -> Self { + let headerView = app.otherElements[AccessibilityIdentifier.dnsContentBlockersHeaderView] + let expandButton = headerView.buttons[AccessibilityIdentifier.collapseButton] + expandButton.tap() + + return self + } + + @discardableResult func tapUseCustomDNSSwitch() -> Self { + app.cells[AccessibilityIdentifier.dnsSettingsUseCustomDNSCell] + .switches[AccessibilityIdentifier.customSwitch] + .tap() + + return self + } + + @discardableResult func tapBlockAdsSwitch() -> Self { + app.cells[AccessibilityIdentifier.blockAdvertising] + .switches[AccessibilityIdentifier.customSwitch] + .tap() + + return self + } + + @discardableResult func tapBlockTrackerSwitch() -> Self { + app.cells[AccessibilityIdentifier.blockTracking] + .switches[AccessibilityIdentifier.customSwitch] + .tap() + + return self + } + + @discardableResult func tapBlockMalwareSwitch() -> Self { + app.cells[AccessibilityIdentifier.blockMalware] + .switches[AccessibilityIdentifier.customSwitch] + .tap() + + return self + } + + @discardableResult func tapBlockAdultContentSwitch() -> Self { + app.cells[AccessibilityIdentifier.blockAdultContent] + .switches[AccessibilityIdentifier.customSwitch] + .tap() + + return self + } + + @discardableResult func tapBlockGamblingSwitch() -> Self { + app.cells[AccessibilityIdentifier.blockGambling] + .switches[AccessibilityIdentifier.customSwitch] + .tap() + + return self + } + + @discardableResult func tapBlockSocialMediaSwitch() -> Self { + app.cells[AccessibilityIdentifier.blockSocialMedia] + .switches[AccessibilityIdentifier.customSwitch] + .tap() + + return self + } + + @discardableResult func tapEditButton() -> Self { + app.buttons[AccessibilityIdentifier.dnsSettingsEditButton] + .tap() + return self + } + + @discardableResult func tapDoneButton() -> Self { + return self.tapEditButton() + } + + @discardableResult func tapAddAServer() -> Self { + app.cells[AccessibilityIdentifier.dnsSettingsAddServerCell] + .tap() + return self + } + + @discardableResult func tapEnterIPAddressTextField() -> Self { + app.textFields[AccessibilityIdentifier.dnsSettingsEnterIPAddressTextField] + .tap() + return self + } + + @discardableResult func verifyBlockAdsSwitchOn() -> Self { + return assertSwitchOn(accessibilityIdentifier: .blockAdvertising) + } + + @discardableResult func verifyBlockTrackerSwitchOn() -> Self { + return assertSwitchOn(accessibilityIdentifier: .blockTracking) + } + + @discardableResult func verifyBlockMalwareSwitchOn() -> Self { + return assertSwitchOn(accessibilityIdentifier: .blockMalware) + } + + @discardableResult func verifyBlockAdultContentSwitchOn() -> Self { + return assertSwitchOn(accessibilityIdentifier: .blockAdultContent) + } + + @discardableResult func verifyBlockGamblingSwitchOn() -> Self { + return assertSwitchOn(accessibilityIdentifier: .blockGambling) + } + + @discardableResult func verifyBlockSocialMediaSwitchOn() -> Self { + return assertSwitchOn(accessibilityIdentifier: .blockSocialMedia) + } + + @discardableResult func verifyUseCustomDNSSwitchOn() -> Self { + return assertSwitchOn(accessibilityIdentifier: .dnsSettingsUseCustomDNSCell) + } + + /// Verify that the UI shows stored DNS server IP address same as `ipAddress`. Note that this function assumes there is only one custom DNS server IP address stored. + @discardableResult func verifyCustomDNSIPAddress(_ ipAddress: String) -> Self { + let textField = app.textFields[AccessibilityIdentifier.dnsSettingsEnterIPAddressTextField] + + guard let settingIPAddress = textField.value as? String else { + XCTFail("Failed to read configured DNS IP address") + return self + } + + XCTAssertEqual(ipAddress, settingIPAddress) + return self + } +} diff --git a/ios/MullvadVPNUITests/Pages/SettingsPage.swift b/ios/MullvadVPNUITests/Pages/SettingsPage.swift index b65ca02f817e..6ed383afa987 100644 --- a/ios/MullvadVPNUITests/Pages/SettingsPage.swift +++ b/ios/MullvadVPNUITests/Pages/SettingsPage.swift @@ -39,28 +39,4 @@ class SettingsPage: Page { return self } - - @discardableResult func tapDNSSettingsCell() -> Self { - app.tables - .cells[AccessibilityIdentifier.dnsSettings] - .tap() - - return self - } - - @discardableResult func tapDNSContentBlockingHeaderExpandButton() -> Self { - let headerView = app.otherElements[AccessibilityIdentifier.dnsContentBlockersHeaderView] - let expandButton = headerView.buttons[AccessibilityIdentifier.collapseButton] - expandButton.tap() - - return self - } - - @discardableResult func tapBlockAdsSwitch() -> Self { - app.cells[AccessibilityIdentifier.blockAdvertising] - .switches[AccessibilityIdentifier.customSwitch] - .tap() - - return self - } } diff --git a/ios/MullvadVPNUITests/Pages/VPNSettingsPage.swift b/ios/MullvadVPNUITests/Pages/VPNSettingsPage.swift index 22ad20540bb5..71adb3eaaa3f 100644 --- a/ios/MullvadVPNUITests/Pages/VPNSettingsPage.swift +++ b/ios/MullvadVPNUITests/Pages/VPNSettingsPage.swift @@ -28,9 +28,16 @@ class VPNSettingsPage: Page { return self } + @discardableResult func tapDNSSettingsCell() -> Self { + app.tables + .cells[AccessibilityIdentifier.dnsSettings] + .tap() + + return self + } + @discardableResult func tapWireGuardPortsExpandButton() -> Self { cellExpandCollapseButton(AccessibilityIdentifier.wireGuardPortsCell).tap() - return self } @@ -46,16 +53,51 @@ class VPNSettingsPage: Page { return self } + @discardableResult func tapUDPOverTCPPortAutomaticCell() -> Self { + app.cells["\(AccessibilityIdentifier.wireGuardObfuscationPort)Automatic"] + .tap() + return self + } + + @discardableResult func tapUDPOverTCPPort80Cell() -> Self { + app.cells["\(AccessibilityIdentifier.wireGuardObfuscationPort)80"] + .tap() + return self + } + + @discardableResult func tapUDPOverTCPPort5001Cell() -> Self { + app.cells["\(AccessibilityIdentifier.wireGuardObfuscationPort)5001"] + .tap() + return self + } + @discardableResult func tapQuantumResistantTunnelExpandButton() -> Self { cellExpandCollapseButton(AccessibilityIdentifier.quantumResistantTunnelCell).tap() return self } + @discardableResult func tapQuantumResistantTunnelAutomaticCell() -> Self { + app.cells[AccessibilityIdentifier.quantumResistanceAutomatic] + .tap() + return self + } + + @discardableResult func tapQuantumResistantTunnelOnCell() -> Self { + app.cells[AccessibilityIdentifier.quantumResistanceOn] + .tap() + return self + } + + @discardableResult func tapQuantumResistantTunnelOffCell() -> Self { + app.cells[AccessibilityIdentifier.quantumResistanceOff] + .tap() + return self + } + @discardableResult func tapWireGuardObfuscationAutomaticCell() -> Self { app.cells[AccessibilityIdentifier.wireGuardObfuscationAutomatic] .tap() - return self } @@ -72,8 +114,40 @@ class VPNSettingsPage: Page { } @discardableResult func tapCustomWireGuardPortTextField() -> Self { - app.textFields[AccessibilityIdentifier.customWireGuardPortTextField].tap() + app.textFields[AccessibilityIdentifier.customWireGuardPortTextField] + .tap() + return self + } + + @discardableResult func verifyCustomWireGuardPortSelected(portNumber: String) -> Self { + let cell = app.cells[AccessibilityIdentifier.wireGuardCustomPort] + XCTAssertTrue(cell.isSelected) + let textField = app.textFields[AccessibilityIdentifier.customWireGuardPortTextField] + + guard let textFieldValue = textField.value as? String else { + XCTFail("Failed to read custom port text field value") + return self + } + + XCTAssertEqual(textFieldValue, portNumber) + return self + } + + @discardableResult func verifyWireGuardObfuscationOnSelected() -> Self { + let onCell = app.cells[AccessibilityIdentifier.wireGuardObfuscationOn] + XCTAssertTrue(onCell.isSelected) + return self + } + + @discardableResult func verifyUDPOverTCPPort80Selected() -> Self { + let cell = app.cells["\(AccessibilityIdentifier.wireGuardObfuscationPort)80"] + XCTAssertTrue(cell.isSelected) + return self + } + @discardableResult func verifyQuantumResistantTunnelOffSelected() -> Self { + let cell = app.cells[AccessibilityIdentifier.quantumResistanceOff] + XCTAssertTrue(cell.isSelected) return self } } diff --git a/ios/MullvadVPNUITests/README.md b/ios/MullvadVPNUITests/README.md index 6f2b4f0f085e..a8146b24d91d 100644 --- a/ios/MullvadVPNUITests/README.md +++ b/ios/MullvadVPNUITests/README.md @@ -13,15 +13,17 @@ 5. Install yeetd - `wget https://github.com/biscuitehh/yeetd/releases/download/1.0/yeetd-normal.pkg` - `sudo installer -pkg yeetd-normal.pkg -target yeetd` -6. Install Homebrew and dependencies +6. Install ios-deploy + - `brew install ios-deploy` +7. Install Homebrew and dependencies - `/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"` - `brew install xcbeautify wget swiftlint` -7. Install Ruby +8. Install Ruby - `\curl -sSL https://get.rvm.io | bash` -8. Install Rust and add iOS targets +9. Install Rust and add iOS targets - `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh` - `rustup target install aarch64-apple-ios aarch64-apple-ios-sim` -9. Install Go 1.19 +10. Install Go 1.19 - `brew install go@1.19` ## GitHub runner setup @@ -32,3 +34,15 @@ - `IOS_NO_TIME_ACCOUNT_NUMBER` - Production server account with time added to it - `IOS_TEST_DEVICE_IDENTIFIER_UUID` - unique identifier for the test device. Create new identifier with `uuidgen`. - `IOS_TEST_DEVICE_UDID` - the iOS device's UDID. + +## Test plans +There are a few different test plans which are mainly to be triggered by GitHub action workflows but can also be triggered manually with Xcode: +* `MullvadVPNUITestsAll` - All tests except settings migration tests which are in separate test plan and workflow +* `MullvadVPNUITestsSmoke` - A few tests for smoke testing when merge:ing to `main` + +And also the following test plans which are used for testing settings migration(`ios-end-to-end-tests-settings-migration`): + +* `MullvadVPNUITestsChangeDNSSettings` - Change settings for using custom DNS +* `MullvadVPNUITestsVerifyDNSSettingsChanged` - Verify custom DNS settings still changed +* `MullvadVPNUITestsChangeSettings` - Change all settings except custom DNS setting +* `MullvadVPNUITestsVerifySettingsChanged` - Verify all settings except custom DNS setting still changed diff --git a/ios/MullvadVPNUITests/RelayTests.swift b/ios/MullvadVPNUITests/RelayTests.swift index b90d158ba72a..04df1e345438 100644 --- a/ios/MullvadVPNUITests/RelayTests.swift +++ b/ios/MullvadVPNUITests/RelayTests.swift @@ -32,8 +32,12 @@ class RelayTests: LoggedInWithTimeUITestCase { SettingsPage(app) .tapVPNSettingsCell() + + VPNSettingsPage(app) .tapDNSSettingsCell() - .tapDNSContentBlockingHeaderExpandButton() + + DNSSettingsPage(app) + .tapDNSContentBlockersHeaderExpandButton() .tapBlockAdsSwitch() .swipeDownToDismissModal() diff --git a/ios/MullvadVPNUITests/SettingsMigrationTests.swift b/ios/MullvadVPNUITests/SettingsMigrationTests.swift new file mode 100644 index 000000000000..35d46b8c87cb --- /dev/null +++ b/ios/MullvadVPNUITests/SettingsMigrationTests.swift @@ -0,0 +1,138 @@ +// +// SettingsMigrationTests.swift +// MullvadVPNUITests +// +// Created by Niklas Berglund on 2024-03-15. +// Copyright © 2024 Mullvad VPN AB. All rights reserved. +// + +import Foundation +import XCTest + +class SettingsMigrationTests: BaseUITestCase { + let customDNSServerIPAddress = "123.123.123.123" + let wireGuardPort = "4001" + + override class func shouldUninstallAppInTeardown() -> Bool { + return false + } + + override func setUp() { + super.setUp() + + agreeToTermsOfServiceIfShown() + dismissChangeLogIfShown() + + // Relaunch app so that tests start from a deterministic state + app.terminate() + app.launch() + } + + func testChangeCustomDNSSettings() { + logoutIfLoggedIn() + login(accountNumber: hasTimeAccountNumber) + + HeaderBar(app) + .tapSettingsButton() + + SettingsPage(app) + .tapVPNSettingsCell() + + VPNSettingsPage(app) + .tapDNSSettingsCell() + + DNSSettingsPage(app) + .tapEditButton() + .tapAddAServer() + .tapEnterIPAddressTextField() + .enterText(customDNSServerIPAddress) + .dismissKeyboard() + .tapUseCustomDNSSwitch() + .tapDoneButton() + } + + func testChangeSettings() { + logoutIfLoggedIn() + login(accountNumber: hasTimeAccountNumber) + + HeaderBar(app) + .tapSettingsButton() + + SettingsPage(app) + .tapVPNSettingsCell() + + VPNSettingsPage(app) + .tapDNSSettingsCell() + + DNSSettingsPage(app) + .tapDNSContentBlockersHeaderExpandButton() + .tapBlockAdsSwitch() + .tapBlockTrackerSwitch() + .tapBlockMalwareSwitch() + .tapBlockAdultContentSwitch() + .tapBlockGamblingSwitch() + .tapBlockSocialMediaSwitch() + .tapBackButton() + + VPNSettingsPage(app) + .tapWireGuardPortsExpandButton() + .tapCustomWireGuardPortTextField() + .enterText(wireGuardPort) + .dismissKeyboard() + .tapWireGuardPortsExpandButton() + .tapWireGuardObfuscationExpandButton() + .tapWireGuardObfuscationOnCell() + .tapWireGuardObfuscationExpandButton() + .tapUDPOverTCPPortExpandButton() + .tapUDPOverTCPPort80Cell() + .tapUDPOverTCPPortExpandButton() + } + + func testVerifyCustomDNSSettingsStillChanged() { + HeaderBar(app) + .tapSettingsButton() + + SettingsPage(app) + .tapVPNSettingsCell() + + VPNSettingsPage(app) + .tapDNSSettingsCell() + + DNSSettingsPage(app) + .verifyUseCustomDNSSwitchOn() + .verifyCustomDNSIPAddress(customDNSServerIPAddress) + } + + func testVerifySettingsStillChanged() { + HeaderBar(app) + .tapSettingsButton() + + SettingsPage(app) + .tapVPNSettingsCell() + + VPNSettingsPage(app) + .tapDNSSettingsCell() + + DNSSettingsPage(app) + .tapDNSContentBlockersHeaderExpandButton() + .verifyBlockAdsSwitchOn() + .verifyBlockTrackerSwitchOn() + .verifyBlockMalwareSwitchOn() + .verifyBlockAdultContentSwitchOn() + .verifyBlockGamblingSwitchOn() + .verifyBlockSocialMediaSwitchOn() + .tapBackButton() + + VPNSettingsPage(app) + .tapWireGuardPortsExpandButton() + .verifyCustomWireGuardPortSelected(portNumber: wireGuardPort) + .tapWireGuardPortsExpandButton() + .tapWireGuardObfuscationExpandButton() + .tapWireGuardObfuscationOnCell() + .verifyWireGuardObfuscationOnSelected() + .tapWireGuardObfuscationExpandButton() + .tapUDPOverTCPPortExpandButton() + .verifyUDPOverTCPPort80Selected() + .tapBackButton() + } +} diff --git a/ios/MullvadVPNUITests/Test base classes/BaseUITestCase.swift b/ios/MullvadVPNUITests/Test base classes/BaseUITestCase.swift index be7d517d5418..11b0cf6ee5f0 100644 --- a/ios/MullvadVPNUITests/Test base classes/BaseUITestCase.swift +++ b/ios/MullvadVPNUITests/Test base classes/BaseUITestCase.swift @@ -53,9 +53,16 @@ class BaseUITestCase: XCTestCase { // MARK: - Setup & teardown + /// Override this class function to change the uninstall behaviour in suite level teardown + class func shouldUninstallAppInTeardown() -> Bool { + return true + } + /// Suite level teardown ran after test have executed override class func tearDown() { - uninstallApp() + if shouldUninstallAppInTeardown() { + uninstallApp() + } } /// Test level setup @@ -104,6 +111,11 @@ class BaseUITestCase: XCTestCase { ChangeLogAlert(app) .tapOkay() } + + // Ensure changelog is no longer shown + _ = app + .otherElements[AccessibilityIdentifier.changeLogAlert.rawValue] + .waitForNonExistence(timeout: Self.shortTimeout) } /// Login with specified account number. It is a prerequisite that the login page is currently shown. diff --git a/ios/MullvadVPNUITests/XCUIElement+Extensions.swift b/ios/MullvadVPNUITests/XCUIElement+Extensions.swift new file mode 100644 index 000000000000..9f0aa87188e7 --- /dev/null +++ b/ios/MullvadVPNUITests/XCUIElement+Extensions.swift @@ -0,0 +1,18 @@ +// +// XCUIElement+Extensions.swift +// MullvadVPNUITests +// +// Created by Niklas Berglund on 2024-03-25. +// Copyright © 2024 Mullvad VPN AB. All rights reserved. +// + +import XCTest + +extension XCUIElement { + func waitForNonExistence(timeout: TimeInterval) -> Bool { + let predicate = NSPredicate(format: "exists == FALSE") + let expectation = XCTNSPredicateExpectation(predicate: predicate, object: self) + _ = XCTWaiter.wait(for: [expectation], timeout: timeout) + return !exists + } +} diff --git a/ios/TestPlans/MullvadVPNUITestsAll.xctestplan b/ios/TestPlans/MullvadVPNUITestsAll.xctestplan index 14cb0990dc0c..74265b9ddad7 100644 --- a/ios/TestPlans/MullvadVPNUITestsAll.xctestplan +++ b/ios/TestPlans/MullvadVPNUITestsAll.xctestplan @@ -21,7 +21,8 @@ "BaseUITestCase", "LoggedInWithTimeUITestCase", "LoggedInWithoutTimeUITestCase", - "LoggedOutUITestCase" + "LoggedOutUITestCase", + "SettingsMigrationTests" ], "target" : { "containerPath" : "container:MullvadVPN.xcodeproj", diff --git a/ios/TestPlans/MullvadVPNUITestsChangeDNSSettings.xctestplan b/ios/TestPlans/MullvadVPNUITestsChangeDNSSettings.xctestplan new file mode 100644 index 000000000000..92d92d1974a6 --- /dev/null +++ b/ios/TestPlans/MullvadVPNUITestsChangeDNSSettings.xctestplan @@ -0,0 +1,28 @@ +{ + "configurations" : [ + { + "id" : "29A4F7B5-B97B-4F31-9EB0-3C09615D0ACF", + "name" : "Configuration 1", + "options" : { + + } + } + ], + "defaultOptions" : { + "testTimeoutsEnabled" : true + }, + "testTargets" : [ + { + "selectedTests" : [ + "MigrationTests\/testChangeCustomDNSSettings()", + "SettingsMigrationTests\/testChangeCustomDNSSettings()" + ], + "target" : { + "containerPath" : "container:MullvadVPN.xcodeproj", + "identifier" : "852969242B4D9C1F007EAD4C", + "name" : "MullvadVPNUITests" + } + } + ], + "version" : 1 +} diff --git a/ios/TestPlans/MullvadVPNUITestsChangeSettings.xctestplan b/ios/TestPlans/MullvadVPNUITestsChangeSettings.xctestplan new file mode 100644 index 000000000000..366e2e597b76 --- /dev/null +++ b/ios/TestPlans/MullvadVPNUITestsChangeSettings.xctestplan @@ -0,0 +1,28 @@ +{ + "configurations" : [ + { + "id" : "29A4F7B5-B97B-4F31-9EB0-3C09615D0ACF", + "name" : "Configuration 1", + "options" : { + + } + } + ], + "defaultOptions" : { + "testTimeoutsEnabled" : true + }, + "testTargets" : [ + { + "selectedTests" : [ + "MigrationTests\/testChangeSettings()", + "SettingsMigrationTests\/testChangeSettings()" + ], + "target" : { + "containerPath" : "container:MullvadVPN.xcodeproj", + "identifier" : "852969242B4D9C1F007EAD4C", + "name" : "MullvadVPNUITests" + } + } + ], + "version" : 1 +} diff --git a/ios/TestPlans/MullvadVPNUITestsVerifyDNSSettingsChanged.xctestplan b/ios/TestPlans/MullvadVPNUITestsVerifyDNSSettingsChanged.xctestplan new file mode 100644 index 000000000000..7b2c070a3d3a --- /dev/null +++ b/ios/TestPlans/MullvadVPNUITestsVerifyDNSSettingsChanged.xctestplan @@ -0,0 +1,28 @@ +{ + "configurations" : [ + { + "id" : "29A4F7B5-B97B-4F31-9EB0-3C09615D0ACF", + "name" : "Configuration 1", + "options" : { + + } + } + ], + "defaultOptions" : { + "testTimeoutsEnabled" : true + }, + "testTargets" : [ + { + "selectedTests" : [ + "MigrationTests\/testVerifyCustomDNSSettingsStillChanged()", + "SettingsMigrationTests\/testVerifyCustomDNSSettingsStillChanged()" + ], + "target" : { + "containerPath" : "container:MullvadVPN.xcodeproj", + "identifier" : "852969242B4D9C1F007EAD4C", + "name" : "MullvadVPNUITests" + } + } + ], + "version" : 1 +} diff --git a/ios/TestPlans/MullvadVPNUITestsVerifySettingsChanged.xctestplan b/ios/TestPlans/MullvadVPNUITestsVerifySettingsChanged.xctestplan new file mode 100644 index 000000000000..872d198dd9da --- /dev/null +++ b/ios/TestPlans/MullvadVPNUITestsVerifySettingsChanged.xctestplan @@ -0,0 +1,28 @@ +{ + "configurations" : [ + { + "id" : "29A4F7B5-B97B-4F31-9EB0-3C09615D0ACF", + "name" : "Configuration 1", + "options" : { + + } + } + ], + "defaultOptions" : { + "testTimeoutsEnabled" : true + }, + "testTargets" : [ + { + "selectedTests" : [ + "MigrationTests\/testVerifySettingsStillChanged()", + "SettingsMigrationTests\/testVerifySettingsStillChanged()" + ], + "target" : { + "containerPath" : "container:MullvadVPN.xcodeproj", + "identifier" : "852969242B4D9C1F007EAD4C", + "name" : "MullvadVPNUITests" + } + } + ], + "version" : 1 +} From 302ce388f6d57d1daddc6a83e91e6669933524a6 Mon Sep 17 00:00:00 2001 From: Niklas Berglund Date: Thu, 4 Apr 2024 13:20:53 +0200 Subject: [PATCH 2/2] Update old app commit hash --- .github/workflows/ios-end-to-end-tests-settings-migration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ios-end-to-end-tests-settings-migration.yml b/.github/workflows/ios-end-to-end-tests-settings-migration.yml index 180f454cecfe..a5a27fd6454e 100644 --- a/.github/workflows/ios-end-to-end-tests-settings-migration.yml +++ b/.github/workflows/ios-end-to-end-tests-settings-migration.yml @@ -16,7 +16,7 @@ jobs: name: Settings migration end to end tests runs-on: [self-hosted, macOS, ios-test] env: - OLD_APP_COMMIT_HASH: f741bf159d5719301e4d1fd099b3d353457532fd + OLD_APP_COMMIT_HASH: 895b7d98825e678f5d7023d5ea3c9b7beee89280 steps: - name: Configure Rust uses: actions-rs/toolchain@v1