Remove settings migration test ios 866

.github/workflows/ios-end-to-end-tests.yml

@@ -6,7 +6,7 @@
   contents: read
   issues: write
   pull-requests: write
 on:
   workflow_call:
     inputs:
       arg_tests_json_key:
@@ -22,6 +22,10 @@
       user_supplied_test_name:
         description: 'Only run test case/suite'
         required: false
+      commit_hash:
+        description: >
+         Specifies the repository commit to use; defaults to the current event's commit if not provided.
+        required: false
 jobs:
   set-up-outputs-directory:
     name: Set up outputs directory

ios/MullvadVPN.xcodeproj/project.pbxproj

@@ -644,10 +644,6 @@
 		8529693A2B4F0238007EAD4C /* TermsOfServicePage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 852969392B4F0238007EAD4C /* TermsOfServicePage.swift */; };
 		8529693C2B4F0257007EAD4C /* Alert.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8529693B2B4F0257007EAD4C /* Alert.swift */; };
 		852A26462BA9C9CB006EB9C8 /* DNSSettingsPage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 852A26452BA9C9CB006EB9C8 /* DNSSettingsPage.swift */; };
-		852BC66F2BAB44F500A47558 /* MullvadVPNUITestsVerifySettingsChanged.xctestplan in Resources */ = {isa = PBXBuildFile; fileRef = 852BC66C2BAB44F500A47558 /* MullvadVPNUITestsVerifySettingsChanged.xctestplan */; };
-		852BC6702BAB44F500A47558 /* MullvadVPNUITestsChangeSettings.xctestplan in Resources */ = {isa = PBXBuildFile; fileRef = 852BC66D2BAB44F500A47558 /* MullvadVPNUITestsChangeSettings.xctestplan */; };
-		852BC6712BAB44F500A47558 /* MullvadVPNUITestsVerifyDNSSettingsChanged.xctestplan in Resources */ = {isa = PBXBuildFile; fileRef = 852BC66E2BAB44F500A47558 /* MullvadVPNUITestsVerifyDNSSettingsChanged.xctestplan */; };
-		852BC6732BAB450B00A47558 /* MullvadVPNUITestsChangeDNSSettings.xctestplan in Resources */ = {isa = PBXBuildFile; fileRef = 852BC6722BAB450B00A47558 /* MullvadVPNUITestsChangeDNSSettings.xctestplan */; };
 		852D054D2BC3DE3A008578D2 /* APIAccessPage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 852D054C2BC3DE3A008578D2 /* APIAccessPage.swift */; };
 		852D054F2BC43DF7008578D2 /* AddAccessMethodPage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 852D054E2BC43DF7008578D2 /* AddAccessMethodPage.swift */; };
 		8532E6872B8CCED600ACECD1 /* ProblemReportSubmittedPage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8532E6862B8CCED600ACECD1 /* ProblemReportSubmittedPage.swift */; };
@@ -1979,10 +1975,6 @@
 		852969392B4F0238007EAD4C /* TermsOfServicePage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TermsOfServicePage.swift; sourceTree = "<group>"; };
 		8529693B2B4F0257007EAD4C /* Alert.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Alert.swift; sourceTree = "<group>"; };
 		852A26452BA9C9CB006EB9C8 /* DNSSettingsPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DNSSettingsPage.swift; sourceTree = "<group>"; };
-		852BC66C2BAB44F500A47558 /* MullvadVPNUITestsVerifySettingsChanged.xctestplan */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = MullvadVPNUITestsVerifySettingsChanged.xctestplan; sourceTree = "<group>"; };
-		852BC66D2BAB44F500A47558 /* MullvadVPNUITestsChangeSettings.xctestplan */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = MullvadVPNUITestsChangeSettings.xctestplan; sourceTree = "<group>"; };
-		852BC66E2BAB44F500A47558 /* MullvadVPNUITestsVerifyDNSSettingsChanged.xctestplan */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = MullvadVPNUITestsVerifyDNSSettingsChanged.xctestplan; sourceTree = "<group>"; };
-		852BC6722BAB450B00A47558 /* MullvadVPNUITestsChangeDNSSettings.xctestplan */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = MullvadVPNUITestsChangeDNSSettings.xctestplan; sourceTree = "<group>"; };
 		852D054C2BC3DE3A008578D2 /* APIAccessPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = APIAccessPage.swift; sourceTree = "<group>"; };
 		852D054E2BC43DF7008578D2 /* AddAccessMethodPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AddAccessMethodPage.swift; sourceTree = "<group>"; };
 		8532E6862B8CCED600ACECD1 /* ProblemReportSubmittedPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProblemReportSubmittedPage.swift; sourceTree = "<group>"; };
@@ -3908,11 +3900,7 @@
 				7A83C4002A55B81A00DFB83A /* MullvadVPNCI.xctestplan */,
 				7A02D4EA2A9CEC7A00C19E31 /* MullvadVPNScreenshots.xctestplan */,
 				852969302B4D9E70007EAD4C /* MullvadVPNUITestsAll.xctestplan */,
-				852BC6722BAB450B00A47558 /* MullvadVPNUITestsChangeDNSSettings.xctestplan */,
-				852BC66D2BAB44F500A47558 /* MullvadVPNUITestsChangeSettings.xctestplan */,
 				85006A8E2B73EF67004AD8FB /* MullvadVPNUITestsSmoke.xctestplan */,
-				852BC66E2BAB44F500A47558 /* MullvadVPNUITestsVerifyDNSSettingsChanged.xctestplan */,
-				852BC66C2BAB44F500A47558 /* MullvadVPNUITestsVerifySettingsChanged.xctestplan */,
 			);
 			path = TestPlans;
 			sourceTree = "<group>";
@@ -5123,10 +5111,6 @@
 			isa = PBXResourcesBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
-				852BC66F2BAB44F500A47558 /* MullvadVPNUITestsVerifySettingsChanged.xctestplan in Resources */,
-				852BC6712BAB44F500A47558 /* MullvadVPNUITestsVerifyDNSSettingsChanged.xctestplan in Resources */,
-				852BC6702BAB44F500A47558 /* MullvadVPNUITestsChangeSettings.xctestplan in Resources */,
-				852BC6732BAB450B00A47558 /* MullvadVPNUITestsChangeDNSSettings.xctestplan in Resources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};

ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme

@@ -16,18 +16,6 @@
             reference = "container:TestPlans/MullvadVPNUITestsAll.xctestplan"
             default = "YES">
          </TestPlanReference>
-         <TestPlanReference
-            reference = "container:TestPlans/MullvadVPNUITestsVerifyDNSSettingsChanged.xctestplan">
-         </TestPlanReference>
-         <TestPlanReference
-            reference = "container:TestPlans/MullvadVPNUITestsChangeSettings.xctestplan">
-         </TestPlanReference>
-         <TestPlanReference
-            reference = "container:TestPlans/MullvadVPNUITestsVerifySettingsChanged.xctestplan">
-         </TestPlanReference>
-         <TestPlanReference
-            reference = "container:TestPlans/MullvadVPNUITestsChangeDNSSettings.xctestplan">
-         </TestPlanReference>
       </TestPlans>
       <Testables>
          <TestableReference

ios/MullvadVPNUITests/Pages/VPNSettingsPage.swift

@@ -133,21 +133,6 @@ class VPNSettingsPage: Page {
         return self
     }
 
-    @discardableResult func tapDaitaSwitch() -> Self {
-        app.cells[AccessibilityIdentifier.daitaSwitch]
-            .switches[AccessibilityIdentifier.customSwitch]
-            .tap()
-        let promptIsShown = app
-            .otherElements[AccessibilityIdentifier.daitaPromptAlert.rawValue]
-            .waitForExistence(timeout: 1.0)
-
-        if promptIsShown {
-            DaitaPromptAlert(app)
-                .tapEnableAnyway()
-        }
-        return self
-    }
-
     @discardableResult func verifyCustomWireGuardPortSelected(portNumber: String) -> Self {
         let cell = app.cells[AccessibilityIdentifier.wireGuardCustomPort]
         XCTAssertTrue(cell.isSelected)
@@ -199,28 +184,6 @@ class VPNSettingsPage: Page {
         return self
     }
 
-    @discardableResult func verifyDaitaSwitchOn() -> Self {
-        let switchElement = app.cells[.daitaSwitch]
-            .switches[AccessibilityIdentifier.customSwitch]
-
-        guard let switchValue = switchElement.value as? String else {
-            XCTFail("Failed to read switch state")
-            return self
-        }
-
-        XCTAssertEqual(switchValue, "1")
-        return self
-    }
-
-    @discardableResult func tapDaitaSwitchIfOn() -> Self {
-        let switchElement = app.cells[.daitaSwitch].switches[AccessibilityIdentifier.customSwitch]
-
-        if switchElement.value as? String == "1" {
-            tapDaitaSwitch()
-        }
-        return self
-    }
-
     @discardableResult func tapMultihopSwitchIfOn() -> Self {
         let switchElement = app.cells[.multihopSwitch].switches[AccessibilityIdentifier.customSwitch]
 

ios/MullvadVPNUITests/README.md

@@ -9,7 +9,6 @@ There are five workflows running tests:
  - [ios-end-to-end-tests-nightly.yml](https://github.com/mullvad/mullvadvpn-app/actions/workflows/ios-end-to-end-tests-nightly.yml) - scheduled nightly test run, running all tests.
  - [ios-end-to-end-tests-merge-to-main.yml](https://github.com/mullvad/mullvadvpn-app/actions/workflows/ios-end-to-end-tests-merge-to-main.yml) - automatically triggered by a PR merge to `main`.
  - [ios-end-to-end-tests-api.yml](https://github.com/mullvad/mullvadvpn-app/actions/workflows/ios-end-to-end-tests-api.yml) - manually triggered tests focusing on making sure the API is functioning as intended on stagemole.
- - [ios-end-to-end-tests-settings-migration.yml](https://github.com/mullvad/mullvadvpn-app/actions/workflows/ios-end-to-end-tests-settings-migration.yml) - for now this is still manually triggered. Tests installing older version of the app, changing settings, upgrading the app and verifying that settings were correctly migrated.
 
 ## Adding more tests
 When adding more files with test suites they must be added to the `MullvadVPNUITestsAll` test plan and also added to the appropriate node(s) in `ios/MullvadVPNUITests/tests.json` file in order to run in CI. For new test cases in already existing test suite nothing needs to be done. The test case/suite values in `tests.json` translate to input for `xcodebuild -only-testing` which is in the format `<target-name>/<test-suite-name>/<test-case-name>`. The GitHub actions workflow will add the `<target-name>` part so only `<test-suite-name>/<test-case-name>` is required, where `<test-case-name>` is optional. So for example `AccountTests` and `AccountTests/testLogin` are both valid values.
@@ -33,7 +32,7 @@ The test device must be on the office WiFi `app-team-ios-tests` in order to be a
 1. Make sure device is added to provisioning profiles
 2. Enable developer mode
 3. Disable passcode in iOS settings - otherwise tests cannot be started without manually entering passcode
-4. Set the value of `TEST_DEVICE_UDID` to the UDID of the test device in `ios-end-to-end-tests.yml` and `ios-end-to-end-tests-settings-migration.yml`.
+4. Set the value of `TEST_DEVICE_UDID` to the UDID of the test device in `ios-end-to-end-tests.yml`.
 5. Make sure the test device is connected to the WiFi `app-team-ios-tests`
 6. Make sure iCloud syncing of keychain is off on the device so that the device isn't getting WiFi passwords from another device causing it to sometimes connect to another WiFi.
 7. After the device is set up download updated provisioning profiles on the GitHub runner computer(Download manual profiles in Xcode settings)
@@ -69,14 +68,10 @@ The test device must be on the office WiFi `app-team-ios-tests` in order to be a
   - `PARTNER_API_TOKEN` - secret token for partner API. Optional and only intended to be used in CI when running tests against staging environment.
 
 ### Specifying which tests run when in CI
-Which tests run when is specified in `tests.json`(See _Adding more tests_). Settings migration is an exception, it uses four different test plans and a separate workflow `ios-end-to-end-tests-settings-migration.yml` which executes the test plans in order, do not reinstall the app in between runs but upgrades the app after changing settings:
-* `MullvadVPNUITestsChangeDNSSettings` - Change settings for using custom DNS
-* `MullvadVPNUITestsVerifyDNSSettingsChanged` - Verify custom DNS settings still changed
-* `MullvadVPNUITestsChangeSettings` - Change all settings except custom DNS setting
-* `MullvadVPNUITestsVerifySettingsChanged` - Verify all settings except custom DNS setting still changed
+Which tests run when is specified in `tests.json`(See _Adding more tests_).
 
 ### Current test devices
-Currently we are using an iPhone 15 Pro(UDID `00008130-0019181022F3803A`) running iOS 17.
+Currently we are using an iPhone 15 Pro(UDID `00008130-0019181022F3803A`) running iOS 17.3.1.
 
 ## APIs used
 The iOS team NUC is hosting APIs consumed by tests:

ios/MullvadVPNUITests/SettingsMigrationTests.swift

@@ -9,15 +9,32 @@
 import Foundation
 import XCTest
 
-/*
- Settings migration is an exception, it uses four different test plans and a separate workflow
-   `ios-end-to-end-tests-settings-migration.yml` which executes the test plans in order,
- do not reinstall the app in between runs but upgrades the app after changing settings:
- * `MullvadVPNUITestsChangeDNSSettings` - Change settings for using custom DNS
- * `MullvadVPNUITestsVerifyDNSSettingsChanged` - Verify custom DNS settings still changed
- * `MullvadVPNUITestsChangeSettings` - Change all settings except custom DNS setting
- * `MullvadVPNUITestsVerifySettingsChanged` - Verify all settings except custom DNS setting still changed
- */
+/// Pre-Release iOS Settings Migration Testing Instructions
+///
+/// Before releasing a new version, ensure the settings migration process works as expected.
+/// Follow these steps to validate that user settings persist correctly across app updates:
+///
+/// 1. Remove the installed app:
+///    Uninstall the current app from the test device to ensure a clean environment.
+/// 2. Switch to an older released version:
+///    Checkout an app version released approximately 6 months ago for testing migration over a meaningful time span.
+/// 3. Run `testChangeCustomDNSSettings`:
+///    Modify DNS settings in the older app version to simulate real-world user interactions.
+/// 4. Checkout the release branch:
+///    Switch to the branch containing the new app version to be released.
+///    - Run `testVerifyCustomDNSSettingsStillChanged`:
+///      Verify that DNS settings changed in step 3 persist after upgrading.
+/// 5. Return to the older version:
+///    Checkout the same older version used in step 2 to continue testing additional settings.
+/// 6. Run `testChangeVPNSettings`:
+///    Modify VPN-related settings in the older app version.
+/// 7. Switch back to the release branch:
+///    Return to the branch checked out in step 4.
+///    - Run `testVerifySettingsStillChanged`:
+///      Confirm that VPN settings changed in step 6 persist after upgrading.
+///
+/// These steps ensure the app's settings migration logic is robust and reliable,
+/// providing a seamless user experience during upgrades.
 class SettingsMigrationTests: BaseUITestCase {
     let customDNSServerIPAddress = "123.123.123.123"
     let wireGuardPort = "4001"
@@ -121,7 +138,6 @@ class SettingsMigrationTests: BaseUITestCase {
             .tapUDPOverTCPPort80Cell()
             .tapQuantumResistantTunnelExpandButton()
             .tapQuantumResistantTunnelOnCell()
-            .tapDaitaSwitch()
             .tapMultihopSwitch()
     }
 
@@ -154,7 +170,6 @@ class SettingsMigrationTests: BaseUITestCase {
             .verifyUDPOverTCPPort80Selected()
             .tapQuantumResistantTunnelExpandButton()
             .verifyQuantumResistantTunnelOnSelected()
-            .verifyDaitaSwitchOn()
             .verifyMultihopSwitchOn()
     }
 }