Add commit_hash to end to end test

mullvad · Oct 28, 2024 · cf1e688 · cf1e688
1 parent 9dc7ff7
commit cf1e688
Show file tree

Hide file tree

Showing 3 changed files with 82 additions and 117 deletions.
diff --git a/.github/actions/build-ios-e2e-tests/action.yml b/.github/actions/build-ios-e2e-tests/action.yml
@@ -1,74 +1,85 @@
----
-name: Build iOS end-to-end tests action
-description: Prepares and builds end-to-end tests on an iOS device
+name: 'Build iOS end to end tests action'
+description: 'Prepares and builds end to end tests on iOS device'
 inputs:
   ios_device_pin_code:
-    description: iOS Device Pin Code
+    description: 'iOS Device Pin Code'
     required: true
   test_device_identifier_uuid:
-    description: Test Device Identifier UUID
+    description: 'Test Device Identifier UUID'
     required: true
   has_time_account_number:
-    description: Has Time Account Number
+    description: 'Has Time Account Number'
     required: true
   no_time_account_number:
-    description: No Time Account Number
+    description: 'No Time Account Number'
     required: true
   test_device_udid:
-    description: Test Device UDID
+    description: 'Test Device UDID'
     required: true
   partner_api_token:
-    description: Partner API Token
+    description: 'Partner API Token'
     required: true
   test_name:
-    description: Test case/suite name. Will run all tests in the test plan if not provided.
+    description: 'Test case/suite name. Will run all tests in the test plan if not provided.'
     required: false
   outputs_path:
-    description: Path to store outputs. This should be unique for each job run in order to avoid concurrency issues.
+    description: 'Path to store outputs. This should be unique for each job run in order to avoid concurrency issues.'
     required: true
-  base_directory:
-    description: Base directory for the working paths. Defaults to the current directory (".").
-    required: false
-    default: .
 
 runs:
-  using: composite
+  using: 'composite'
   steps:
-    - name: Set working-directory or default
-      id: set-working-directory
+    - name: Configure Xcode project
       run: |
-        if [ -z "${{ inputs.base_directory }}" ]; then
-          echo "base_directory=." >> $GITHUB_ENV
-        else
-          echo "base_directory=${{ inputs.base_directory }}" >> $GITHUB_ENV
-        fi
+        for file in *.xcconfig.template ; do cp $file ${file//.template/} ; done
+          sed -i "" "/^HAS_TIME_ACCOUNT_NUMBER/d" UITests.xcconfig
+          sed -i "" "/^NO_TIME_ACCOUNT_NUMBER/d" UITests.xcconfig
+          sed -i "" \
+            "/IOS_DEVICE_PIN_CODE =/ s/= .*/= $IOS_DEVICE_PIN_CODE/" \
+            UITests.xcconfig
+          sed -i "" \
+            "/TEST_DEVICE_IDENTIFIER_UUID =/ s/= .*/= $TEST_DEVICE_IDENTIFIER_UUID/" \
+            UITests.xcconfig
+          sed -i "" \
+            "s#^// PARTNER_API_TOKEN =#PARTNER_API_TOKEN =#" \
+            UITests.xcconfig
+          sed -i "" \
+            "/PARTNER_API_TOKEN =/ s#= .*#= $PARTNER_API_TOKEN#" \
+            UITests.xcconfig
+          sed -i "" \
+            "/ATTACH_APP_LOGS_ON_FAILURE =/ s#= .*#= 1#" \
+            UITests.xcconfig
+          sed -i "" \
+            "/TEST_DEVICE_IS_IPAD =/ s#= .*#= 0#" \
+            UITests.xcconfig
+          sed -i "" \
+            "/UNINSTALL_APP_IN_TEST_SUITE_TEAR_DOWN =/ s#= .*#= 0#" \
+            UITests.xcconfig
       shell: bash
+      working-directory: ios/Configurations
+      env:
+        IOS_DEVICE_PIN_CODE: ${{ inputs.ios_device_pin_code }}
+        TEST_DEVICE_IDENTIFIER_UUID: ${{ inputs.test_device_identifier_uuid }}
+        HAS_TIME_ACCOUNT_NUMBER: ${{ inputs.has_time_account_number }}
+        NO_TIME_ACCOUNT_NUMBER: ${{ inputs.no_time_account_number }}
+        PARTNER_API_TOKEN: ${{ inputs.partner_api_token }}
 
-    - name: Configure Xcode project
+    - name: Build app and tests for testing
       run: |
-        for file in *.xcconfig.template; do cp $file ${file//.template/}; done
-        sed -i "" "/^HAS_TIME_ACCOUNT_NUMBER/d" UITests.xcconfig
-        sed -i "" "/^NO_TIME_ACCOUNT_NUMBER/d" UITests.xcconfig
-        sed -i "" \
-          "/IOS_DEVICE_PIN_CODE =/ s/= .*/= $IOS_DEVICE_PIN_CODE/" \
-          UITests.xcconfig
-        sed -i "" \
-          "/TEST_DEVICE_IDENTIFIER_UUID =/ s/= .*/= $TEST_DEVICE_IDENTIFIER_UUID/" \
-          UITests.xcconfig
-        sed -i "" \
-          "s#^// PARTNER_API_TOKEN =#PARTNER_API_TOKEN =#" \
-          UITests.xcconfig
-        sed -i "" \
-          "/PARTNER_API_TOKEN =/ s#= .*#= $PARTNER_API_TOKEN#" \
-          UITests.xcconfig
-        sed -i "" \
-          "/ATTACH_APP_LOGS_ON_FAILURE =/ s#= .*#= 1#" \
-          UITests.xcconfig
-        sed -i "" \
-          "/TEST_DEVICE_IS_IPAD =/ s#= .*#= 0#" \
-          UITests.xcconfig
-        sed -i "" \
-          "/UNINSTALL_APP_IN_TEST_SUITE_TEAR_DOWN =/ s#= .*#= 0#" \
-          UITests.xcconfig
+        if [ -n "$TEST_NAME" ]; then
+          TEST_NAME_ARGUMENT=" -only-testing $TEST_NAME"
+        else
+          TEST_NAME_ARGUMENT=""
+        fi
+        set -o pipefail && env NSUnbufferedIO=YES xcodebuild \
+          -project MullvadVPN.xcodeproj \
+          -scheme MullvadVPNUITests \
+          -testPlan MullvadVPNUITestsAll $TEST_NAME_ARGUMENT \
+          -destination "platform=iOS,id=$TEST_DEVICE_UDID" \
+          -derivedDataPath derived-data \
+          clean build-for-testing 2>&1
       shell: bash
-      working-directory: ${{ env.base_directory }}/ios/
+      working-directory: ios/
+      env:
+        TEST_DEVICE_UDID: ${{ inputs.test_device_udid }}
+        TEST_NAME: ${{ inputs.test_name }}
diff --git a/.github/actions/run-ios-e2e-tests/action.yml b/.github/actions/run-ios-e2e-tests/action.yml
@@ -1,51 +1,36 @@
----
-name: Run iOS end-to-end tests action
-description: Runs end-to-end tests on iOS device
+name: 'Run iOS end to end tests action'
+description: 'Runs end to end tests on iOS device'
 inputs:
   test_name:
-    description: Test case/suite name. Will run all tests in the test plan if not provided.
+    description: 'Test case/suite name. Will run all tests in the test plan if not provided.'
     required: false
   test_device_udid:
-    description: Test Device UDID
+    description: 'Test Device UDID'
     required: true
   outputs_path:
     description: >
-      Path to where outputs are stored - both build outputs and outputs
-      from running tests. This should be unique for each job run in order to
-      avoid concurrency issues.
+      Path to where outputs are stored - both build outputs and outputs from running tests.
+      This should be unique for each job run in order to avoid concurrency issues.
     required: true
-  base_directory:
-    description: Base directory for the working paths.
-    required: false
-    default: .
 
 runs:
-  using: composite
+  using: 'composite'
   steps:
-    - name: Set base directory or default
-      id: set-base-directory
-      run: |
-        if [ -z "${{ inputs.base_directory }}" ]; then
-          echo "base_directory=." >> $GITHUB_ENV
-        else
-          echo "base_directory=${{ inputs.base_directory }}" >> $GITHUB_ENV
-        fi
-      shell: bash
-
+    # Set up a unique output directory
     - name: Set up outputs directory
       run: |
         # Forcing the filesystem buffers to be flushed to ensure the
         # directory tree is updated
         sync
-
         if [ -n "$TEST_NAME" ]; then
           # Strip slashes to avoid creating subdirectories
           test_name_sanitized=$(printf "$TEST_NAME" | sed 's/\//_/g')
-          echo "Setting output directory: tests-output-$test_name_sanitized"
-          test_output_directory="${{ inputs.outputs_path }}/tests-output-$test_name_sanitized"
+          echo "Setting output directory tests-output-test-name-sanitized"
+          echo "$test_name_sanitized"
+          test_output_directory="${{ env.OUTPUTS_PATH }}/tests-output-$test_name_sanitized"
         else
-          echo "Setting output directory: tests-output"
-          test_output_directory="${{ inputs.outputs_path }}/tests-output"
+          echo "Setting output directory output"
+          test_output_directory="${{ env.OUTPUTS_PATH }}/tests-output"
         fi
 
         echo "TEST_OUTPUT_DIRECTORY=$test_output_directory" >> $GITHUB_ENV
@@ -56,26 +41,21 @@ runs:
         OUTPUTS_PATH: ${{ inputs.outputs_path }}
 
     - name: Uninstall app
-      run: |
-        echo "Uninstalling app from device with UDID: $TEST_DEVICE_UDID"
-        ios-deploy --id $TEST_DEVICE_UDID --uninstall_only --bundle_id net.mullvad.MullvadVPN
+      run: ios-deploy --id $TEST_DEVICE_UDID --uninstall_only --bundle_id net.mullvad.MullvadVPN
       shell: bash
       env:
         TEST_DEVICE_UDID: ${{ inputs.test_device_udid }}
 
-    - name: Run end-to-end tests
+    - name: Run end-to-end-tests
       run: |
         # Forcing the filesystem buffers to be flushed to ensure the
         # directory tree is updated
         sync
-
         if [ -n "$TEST_NAME" ]; then
           TEST_NAME_ARGUMENT=" -only-testing $TEST_NAME"
         else
           TEST_NAME_ARGUMENT=""
         fi
-
-        echo "Running end-to-end tests in directory: ${{ env.base_directory }}/mullvadvpn-app/ios"
         set -o pipefail && env NSUnbufferedIO=YES xcodebuild \
           -project MullvadVPN.xcodeproj \
           -scheme MullvadVPNUITests \
@@ -86,7 +66,7 @@ runs:
           test-without-building 2>&1 | xcbeautify --report junit \
           --report-path ${{ env.TEST_OUTPUT_DIRECTORY }}/junit-test-report
       shell: bash
-      working-directory: ${{ env.base_directory }}/mullvadvpn-app/ios
+      working-directory: ${{ inputs.outputs_path }}/mullvadvpn-app/ios
       env:
         TEST_NAME: ${{ inputs.test_name }}
         TEST_DEVICE_UDID: ${{ inputs.test_device_udid }}

diff --git a/.github/workflows/ios-end-to-end-tests.yml b/.github/workflows/ios-end-to-end-tests.yml
@@ -24,7 +24,6 @@ on:
         description: 'Only run test case/suite'
         required: false
 jobs:
-  # Set up outputs directory
   set-up-outputs-directory:
     name: Set up outputs directory
     runs-on: [self-hosted, macOS, ios-test]
@@ -40,20 +39,6 @@ jobs:
 
         shell: bash
 
-  # Get Branch Name
-  get-branch-name:
-    name: Get Branch Name
-    runs-on: [self-hosted, macOS, ios-test]
-    outputs:
-      branch_name: ${{ steps.get_branch.outputs.branch_name }}
-    steps:
-      - name: Get Branch Name
-        id: get_branch
-        run: |
-          # Extract branch name from GITHUB_REF, which contains refs/heads/branch_name
-          BRANCH_NAME="${GITHUB_REF##*/}"
-          echo "branch_name=${BRANCH_NAME}" >> $GITHUB_OUTPUT
-
   # Define the set of tests to run based on the event type and input
   define-test-suites-matrix:
     name: Define test suites matrix
@@ -91,10 +76,7 @@ jobs:
   build:
     name: Build for end to end testing
     runs-on: [self-hosted, macOS, ios-test]
-    needs: [set-up-outputs-directory,get-branch-name]
-    env:
-      BRANCH_NAME: ${{ needs.get-branch-name.outputs.branch_name }}
-      OUTPUTS_DIR: ${{ needs.set-up-outputs-directory.outputs.job_outputs_directory }}
+    needs: set-up-outputs-directory
     timeout-minutes: 20
     steps:
       - name: Set commit hash or default to github.sha
@@ -111,10 +93,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           clean: true
-          fetch-depth: 1
           ref: ${{ env.COMMIT_HASH }}
-          path: $HOME/${{ env.COMMIT_HASH }}
-
 
       - name: Configure Rust
         uses: actions-rs/[email protected]
@@ -124,7 +103,7 @@ jobs:
           target: aarch64-apple-ios
 
       - name: Build iOS end to end tests action
-        uses: mullvad/mullvadvpn-app/.github/actions/build-ios-e2e-tests/${{ env.BRANCH_NAME }}
+        uses: ./.github/actions/build-ios-e2e-tests
         with:
           test_name: ${{ github.event.inputs.user_supplied_test_name }}
           ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }}
@@ -133,8 +112,7 @@ jobs:
           no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
           test_device_udid: ${{ env.TEST_DEVICE_UDID }}
           partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }}
-          outputs_path: ${{ env.OUTPUTS_DIR }}
-          base_directory: $HOME/${{ env.COMMIT_HASH }}
+          outputs_path: ${{ needs.set-up-outputs-directory.outputs.job_outputs_directory }}
 
       - name: Debug print job output directory
         run: |
@@ -159,23 +137,19 @@ jobs:
   test:
     name: Run tests
     runs-on: [self-hosted, macOS, ios-test]
-    needs: [build, define-test-suites-matrix, set-up-outputs-directory, get-branch-name]
-    env:
-      BRANCH_NAME: ${{ needs.get-branch-name.outputs.branch_name }}
-      OUTPUTS_DIR: ${{ needs.set-up-outputs-directory.outputs.job_outputs_directory }}]
+    needs: [build, define-test-suites-matrix, set-up-outputs-directory]
     timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
         test_suite: ${{fromJson(needs.define-test-suites-matrix.outputs.test_suites_json)}}
     steps:
       - name: Run iOS end to end tests action
-        uses:  mullvad/mullvadvpn-app/.github/actions/run-ios-e2e-tests/${{ env.BRANCH_NAME }}
+        uses: ./.github/actions/run-ios-e2e-tests
         with:
           test_name: "MullvadVPNUITests/${{ matrix.test_suite }}"
           test_device_udid: ${{ env.TEST_DEVICE_UDID }}
-          outputs_path: ${{ env.OUTPUTS_DIR }}
-          base_directory: $HOME/${{ env.COMMIT_HASH }}
+          outputs_path: ${{ needs.set-up-outputs-directory.outputs.job_outputs_directory }}
 
   clean-up-outputs-directory:
     if: always()