Run sync tests in matrix (#2263)

Co-authored-by: Dominik Kapusta <[email protected]>
duckduckgo · Dec 18, 2023 · b5ee323 · b5ee323
1 parent 5f8ab87
commit b5ee323
Show file tree

Hide file tree

Showing 7 changed files with 62 additions and 47 deletions.
diff --git a/.github/workflows/adhoc.yml b/.github/workflows/adhoc.yml
@@ -58,13 +58,13 @@ jobs:
           echo "dsyms_path=${{ github.workspace }}/${{ env.dsyms_filename }}" >> $GITHUB_ENV
 
       - name: Upload IPA artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ env.ipa_filename }}
           path: ${{ env.ipa_path }}
 
       - name: Upload dSYMs artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ env.dsyms_filename }}
           path: ${{ env.dsyms_path }}

diff --git a/.github/workflows/alpha.yml b/.github/workflows/alpha.yml
@@ -97,7 +97,7 @@ jobs:
         echo "build_version=${build_version}" >> $GITHUB_ENV
 
     - name: Upload dSYMs artifact
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: DuckDuckGo-Alpha-dSYM-${{ env.app_version }}
         path: ${{ env.dsyms_path }}

diff --git a/.github/workflows/end-to-end.yml b/.github/workflows/end-to-end.yml
@@ -79,7 +79,7 @@ jobs:
           --data ' { "data": { "name": "GH Workflow Failure - End to end tests", "workspace": "${{ vars.GH_ASANA_WORKSPACE_ID }}", "projects": [ "${{ vars.GH_ASANA_IOS_APP_PROJECT_ID }}" ], "notes" : "The end to end workflow has failed. See https://github.com/duckduckgo/iOS/actions/runs/${{ github.run_id }}" } }'
 
     - name: Upload logs when workflow failed
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       if: failure()
       with:
         name: BuildLogs

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -51,7 +51,7 @@ jobs:
           | xcbeautify --report junit --report-path . --junit-report-filename unittests.xml
 
     - name: Upload logs if workflow failed
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       if: failure()
       with:
         name: BuildLogs

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -87,7 +87,7 @@ jobs:
           | xcbeautify --report junit --report-path . --junit-report-filename unittests.xml
 
     - name: Upload logs if workflow failed
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       if: failure()
       with:
         name: BuildLogs

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -80,7 +80,7 @@ jobs:
 
     - name: Upload dSYMs artifact
       if: always()
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: DuckDuckGo-${{ steps.destination.outputs.destination }}-dSYM-${{ env.app_version }}
         path: ${{ env.dsyms_path }}

diff --git a/.github/workflows/sync-end-to-end.yml b/.github/workflows/sync-end-to-end.yml
@@ -5,9 +5,10 @@ on:
     - cron: '0 5 * * *' # run at 5 AM UTC
 
 jobs:
-  sync-end-to-end-tests:
-    name: Sync End to end Tests
+  build-for-sync-end-to-end-tests:
+    name: Build for Sync End To End Tests
     runs-on: macos-13
+    timeout-minutes: 30
 
     steps:
     - name: Check out the code
@@ -44,62 +45,76 @@ jobs:
           -derivedDataPath "DerivedData" \
           -skipPackagePluginValidation \
         | tee xcodebuild.log
+
+    - name: Store Binary
+      uses: actions/upload-artifact@v4
+      with:
+        name: duckduckgo-ios-app
+        path: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app
+
+    - name: Upload logs when workflow failed
+      uses: actions/upload-artifact@v4
+      if: failure()
+      with:
+        name: BuildLogs
+        path: |
+          xcodebuild.log
+          DerivedData/Logs/Test/*.xcresult
+        retention-days: 7
 
+  sync-end-to-end-tests:
+    name: Sync End To End Tests
+    needs: build-for-sync-end-to-end-tests
+    runs-on: macos-13
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        os-version: [15, 16, 17]
+      #max-parallel: 1 # Uncomment this line to run tests sequentially. 
+      fail-fast: false
+
+    steps:
+    - name: Check out the code
+      uses: actions/checkout@v3 # Don't need submodules here as this is only for the tests folder
+
     - name: Create test account for Sync and return the recovery code
       uses: duckduckgo/sync_crypto/action@main
       id: sync-recovery-code
       with:
         debug: true
 
-    - name: Sync e2e tests - iOS 15
-      uses: mobile-dev-inc/[email protected]
-      with:
-        api-key: ${{ secrets.MAESTRO_CLOUD_API_KEY }}
-        app-file: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app
-        workspace: .maestro
-        include-tags: sync
-        env: |
-            CODE=${{ steps.sync-recovery-code.outputs.recovery-code }}
-    
-    - name: Sync e2e tests - iOS 16
-      uses: mobile-dev-inc/[email protected]
+    - name: Retrieve Binary
+      uses: actions/download-artifact@v4
       with:
-        api-key: ${{ secrets.MAESTRO_CLOUD_API_KEY }}
-        app-file: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app
-        ios-version: 16
-        workspace: .maestro
-        include-tags: sync
-        env: |
-            CODE=${{ steps.sync-recovery-code.outputs.recovery-code }}
+        name: duckduckgo-ios-app
+        path: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app
 
-    - name: Sync e2e tests - iOS 17
+    - name: Sync e2e tests 
       uses: mobile-dev-inc/[email protected]
       with:
         api-key: ${{ secrets.MAESTRO_CLOUD_API_KEY }}
         app-file: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app
-        ios-version: 17
+        ios-version: ${{ matrix.os-version }}
         workspace: .maestro
         include-tags: sync
         env: |
             CODE=${{ steps.sync-recovery-code.outputs.recovery-code }}
     
-    - name: Create Asana task when workflow failed
-      if: ${{ failure() }}
-      run: |
-        curl -s "https://app.asana.com/api/1.0/tasks" \
-          --header "Accept: application/json" \
-          --header "Authorization: Bearer ${{ secrets.ASANA_ACCESS_TOKEN }}" \
-          --header "Content-Type: application/json" \
-          --data ' { "data": { "name": "GH Workflow Failure - Sync End to end tests", "workspace": "${{ vars.GH_ASANA_WORKSPACE_ID }}", "projects": [ "${{ vars.GH_ASANA_IOS_APP_PROJECT_ID }}" ], "notes" : "The end to end workflow has failed. See https://github.com/duckduckgo/iOS/actions/runs/${{ github.run_id }}" } }'
+  notify-failure:
+    name: Notify on failure
+    if: ${{ always() && contains(join(needs.*.result, ','), 'failure') }}
+    needs: [build-for-sync-end-to-end-tests, sync-end-to-end-tests]
+    runs-on: ubuntu-latest
 
-    - name: Upload logs when workflow failed
-      uses: actions/upload-artifact@v3
-      if: failure()
-      with:
-        name: BuildLogs
-        path: |
-          xcodebuild.log
-          DerivedData/Logs/Test/*.xcresult
-        retention-days: 7
+    steps:
+      - name: Create Asana task when workflow failed
+        run: |
+          curl -s "https://app.asana.com/api/1.0/tasks" \
+            --header "Accept: application/json" \
+            --header "Authorization: Bearer ${{ secrets.ASANA_ACCESS_TOKEN }}" \
+            --header "Content-Type: application/json" \
+            --data ' { "data": { "name": "GH Workflow Failure - Sync End to end tests", "workspace": "${{ vars.GH_ASANA_WORKSPACE_ID }}", "projects": [ "${{ vars.GH_ASANA_IOS_APP_PROJECT_ID }}" ], "notes" : "The end to end workflow has failed. See https://github.com/duckduckgo/iOS/actions/runs/${{ github.run_id }}" } }'
+
+