From a840f5a3a29e5e20d1de7ff59638c038c7001af7 Mon Sep 17 00:00:00 2001 From: Lorenzo Mattei Date: Wed, 13 Dec 2023 09:10:14 +0100 Subject: [PATCH 01/17] Run tests on multiple OS using a matrix --- .github/workflows/sync-end-to-end.yml | 63 ++++++++++++--------------- 1 file changed, 29 insertions(+), 34 deletions(-) diff --git a/.github/workflows/sync-end-to-end.yml b/.github/workflows/sync-end-to-end.yml index 0125ab669a..657d31c66c 100644 --- a/.github/workflows/sync-end-to-end.yml +++ b/.github/workflows/sync-end-to-end.yml @@ -5,8 +5,8 @@ on: - cron: '0 5 * * *' # run at 5 AM UTC jobs: - sync-end-to-end-tests: - name: Sync End to end Tests + build-for-sync-end-to-end-tests: + name: Build for Sync End To End Tests runs-on: macos-13 steps: @@ -44,44 +44,47 @@ jobs: -derivedDataPath "DerivedData" \ | tee xcodebuild.log + - name: Upload logs when workflow failed + uses: actions/upload-artifact@v3 + if: failure() + with: + name: BuildLogs + path: | + xcodebuild.log + DerivedData/Logs/Test/*.xcresult + retention-days: 7 + + sync-end-to-end-tests: + name: Sync End To End Tests + needs: build-for-sync-end-to-end-tests + runs-on: macos-13 + strategy: + matrix: + os-version: [15, 16, 17] + + steps: - name: Create test account for Sync and return the recovery code uses: duckduckgo/sync_crypto/action@main id: sync-recovery-code with: debug: true - - name: Sync e2e tests - iOS 15 + - name: Sync e2e tests uses: mobile-dev-inc/action-maestro-cloud@v1.8.0 with: api-key: ${{ secrets.MAESTRO_CLOUD_API_KEY }} app-file: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app + ios-version: ${{ matrix.os-version }} workspace: .maestro include-tags: sync env: | CODE=${{ steps.sync-recovery-code.outputs.recovery-code }} - - name: Sync e2e tests - iOS 16 - uses: mobile-dev-inc/action-maestro-cloud@v1.8.0 - with: - api-key: ${{ secrets.MAESTRO_CLOUD_API_KEY }} - app-file: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app - ios-version: 16 - workspace: .maestro - include-tags: sync - env: | - CODE=${{ steps.sync-recovery-code.outputs.recovery-code }} - - - name: Sync e2e tests - iOS 17 - uses: mobile-dev-inc/action-maestro-cloud@v1.8.0 - with: - api-key: ${{ secrets.MAESTRO_CLOUD_API_KEY }} - app-file: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app - ios-version: 17 - workspace: .maestro - include-tags: sync - env: | - CODE=${{ steps.sync-recovery-code.outputs.recovery-code }} - + notify-failure: + name: Notify on failure + if: ${{ always() && contains(join(needs.*.result, ','), 'failure') }} + needs: [build-for-sync-end-to-end-tests, sync-end-to-end-tests] + runs-on: ubuntu-latest - name: Create Asana task when workflow failed if: ${{ failure() }} run: | @@ -91,14 +94,6 @@ jobs: --header "Content-Type: application/json" \ --data ' { "data": { "name": "GH Workflow Failure - Sync End to end tests", "workspace": "${{ vars.GH_ASANA_WORKSPACE_ID }}", "projects": [ "${{ vars.GH_ASANA_IOS_APP_PROJECT_ID }}" ], "notes" : "The end to end workflow has failed. See https://github.com/duckduckgo/iOS/actions/runs/${{ github.run_id }}" } }' - - name: Upload logs when workflow failed - uses: actions/upload-artifact@v3 - if: failure() - with: - name: BuildLogs - path: | - xcodebuild.log - DerivedData/Logs/Test/*.xcresult - retention-days: 7 + From 239fac2bb9f077d14a8e9847076ee6c1cc1020d2 Mon Sep 17 00:00:00 2001 From: Lorenzo Mattei Date: Wed, 13 Dec 2023 09:10:37 +0100 Subject: [PATCH 02/17] Add push trigger for test --- .github/workflows/sync-end-to-end.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sync-end-to-end.yml b/.github/workflows/sync-end-to-end.yml index 657d31c66c..eaa3fda03c 100644 --- a/.github/workflows/sync-end-to-end.yml +++ b/.github/workflows/sync-end-to-end.yml @@ -1,6 +1,7 @@ name: Sync-End-to-End tests on: + push: schedule: - cron: '0 5 * * *' # run at 5 AM UTC From 6820513413abd48611a690c5f6028c36e40b573d Mon Sep 17 00:00:00 2001 From: Lorenzo Mattei Date: Wed, 13 Dec 2023 09:18:56 +0100 Subject: [PATCH 03/17] Reuse binary across jobs --- .github/workflows/sync-end-to-end.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/sync-end-to-end.yml b/.github/workflows/sync-end-to-end.yml index eaa3fda03c..d236d6b015 100644 --- a/.github/workflows/sync-end-to-end.yml +++ b/.github/workflows/sync-end-to-end.yml @@ -44,6 +44,12 @@ jobs: -destination "platform=iOS Simulator,name=iPhone 14" \ -derivedDataPath "DerivedData" \ | tee xcodebuild.log + + - name: Store Binary + uses: actions/upload-artifact@v3 + with: + name: duckduckgo-ios-app + path: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app - name: Upload logs when workflow failed uses: actions/upload-artifact@v3 @@ -70,6 +76,12 @@ jobs: with: debug: true + - name: Retrieve Binary + uses: actions/download-artifact@v3 + with: + name: duckduckgo-ios-app + path: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app + - name: Sync e2e tests uses: mobile-dev-inc/action-maestro-cloud@v1.8.0 with: From 189b020a68d74250f3e07a488f3695af0c5e1902 Mon Sep 17 00:00:00 2001 From: Lorenzo Mattei Date: Wed, 13 Dec 2023 10:05:14 +0100 Subject: [PATCH 04/17] Fix yaml syntax --- .github/workflows/sync-end-to-end.yml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/sync-end-to-end.yml b/.github/workflows/sync-end-to-end.yml index d236d6b015..4f7f63f476 100644 --- a/.github/workflows/sync-end-to-end.yml +++ b/.github/workflows/sync-end-to-end.yml @@ -93,19 +93,21 @@ jobs: env: | CODE=${{ steps.sync-recovery-code.outputs.recovery-code }} - notify-failure: + notify-failure: name: Notify on failure if: ${{ always() && contains(join(needs.*.result, ','), 'failure') }} needs: [build-for-sync-end-to-end-tests, sync-end-to-end-tests] runs-on: ubuntu-latest - - name: Create Asana task when workflow failed - if: ${{ failure() }} - run: | - curl -s "https://app.asana.com/api/1.0/tasks" \ - --header "Accept: application/json" \ - --header "Authorization: Bearer ${{ secrets.ASANA_ACCESS_TOKEN }}" \ - --header "Content-Type: application/json" \ - --data ' { "data": { "name": "GH Workflow Failure - Sync End to end tests", "workspace": "${{ vars.GH_ASANA_WORKSPACE_ID }}", "projects": [ "${{ vars.GH_ASANA_IOS_APP_PROJECT_ID }}" ], "notes" : "The end to end workflow has failed. See https://github.com/duckduckgo/iOS/actions/runs/${{ github.run_id }}" } }' + + steps: + - name: Create Asana task when workflow failed + if: ${{ failure() }} + run: | + curl -s "https://app.asana.com/api/1.0/tasks" \ + --header "Accept: application/json" \ + --header "Authorization: Bearer ${{ secrets.ASANA_ACCESS_TOKEN }}" \ + --header "Content-Type: application/json" \ + --data ' { "data": { "name": "GH Workflow Failure - Sync End to end tests", "workspace": "${{ vars.GH_ASANA_WORKSPACE_ID }}", "projects": [ "${{ vars.GH_ASANA_IOS_APP_PROJECT_ID }}" ], "notes" : "The end to end workflow has failed. See https://github.com/duckduckgo/iOS/actions/runs/${{ github.run_id }}" } }' From 801f13b4e1d53b4cf2f8bfce27e7e030be87a0f8 Mon Sep 17 00:00:00 2001 From: Lorenzo Mattei Date: Wed, 13 Dec 2023 10:23:09 +0100 Subject: [PATCH 05/17] Fix Asana notification --- .github/workflows/sync-end-to-end.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/sync-end-to-end.yml b/.github/workflows/sync-end-to-end.yml index 4f7f63f476..b0d274d614 100644 --- a/.github/workflows/sync-end-to-end.yml +++ b/.github/workflows/sync-end-to-end.yml @@ -101,7 +101,6 @@ jobs: steps: - name: Create Asana task when workflow failed - if: ${{ failure() }} run: | curl -s "https://app.asana.com/api/1.0/tasks" \ --header "Accept: application/json" \ From 2c3a8d56a96477cb5b5c45c8d4c138ab6c5c6296 Mon Sep 17 00:00:00 2001 From: Lorenzo Mattei Date: Wed, 13 Dec 2023 10:41:18 +0100 Subject: [PATCH 06/17] Checkout repo to make tests available to maestro cloud --- .github/workflows/sync-end-to-end.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/sync-end-to-end.yml b/.github/workflows/sync-end-to-end.yml index b0d274d614..eece75339a 100644 --- a/.github/workflows/sync-end-to-end.yml +++ b/.github/workflows/sync-end-to-end.yml @@ -70,6 +70,9 @@ jobs: os-version: [15, 16, 17] steps: + - name: Check out the code + uses: actions/checkout@v3 # Don't need submodules here as this is only for the tests folder + - name: Create test account for Sync and return the recovery code uses: duckduckgo/sync_crypto/action@main id: sync-recovery-code From 2ed59abb81c11752b988005203380765ecac9b6c Mon Sep 17 00:00:00 2001 From: Lorenzo Mattei Date: Wed, 13 Dec 2023 12:16:44 +0100 Subject: [PATCH 07/17] Avoid job parallelization --- .github/workflows/sync-end-to-end.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sync-end-to-end.yml b/.github/workflows/sync-end-to-end.yml index eece75339a..d2a669c2bd 100644 --- a/.github/workflows/sync-end-to-end.yml +++ b/.github/workflows/sync-end-to-end.yml @@ -68,6 +68,7 @@ jobs: strategy: matrix: os-version: [15, 16, 17] + max-parallel: 1 steps: - name: Check out the code From f92957b0ac1dac655d28c4e0d719c1721d34b1a2 Mon Sep 17 00:00:00 2001 From: Lorenzo Mattei Date: Thu, 14 Dec 2023 15:03:27 +0100 Subject: [PATCH 08/17] Test running in parallel --- .github/workflows/sync-end-to-end.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-end-to-end.yml b/.github/workflows/sync-end-to-end.yml index d2a669c2bd..98640fcb58 100644 --- a/.github/workflows/sync-end-to-end.yml +++ b/.github/workflows/sync-end-to-end.yml @@ -68,7 +68,7 @@ jobs: strategy: matrix: os-version: [15, 16, 17] - max-parallel: 1 + #max-parallel: 1 steps: - name: Check out the code From 94e67bbb84d0b1dcca3caf11b63e5a20faa1a9b0 Mon Sep 17 00:00:00 2001 From: Lorenzo Mattei Date: Thu, 14 Dec 2023 18:23:23 +0100 Subject: [PATCH 09/17] Add comment --- .github/workflows/sync-end-to-end.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-end-to-end.yml b/.github/workflows/sync-end-to-end.yml index 98640fcb58..dd7578c77f 100644 --- a/.github/workflows/sync-end-to-end.yml +++ b/.github/workflows/sync-end-to-end.yml @@ -68,7 +68,7 @@ jobs: strategy: matrix: os-version: [15, 16, 17] - #max-parallel: 1 + #max-parallel: 1 # Uncomment this line to run tests sequentially. steps: - name: Check out the code From 6ee0cafb33f3dd1027a160ab0f71c47cc5450b70 Mon Sep 17 00:00:00 2001 From: Lorenzo Mattei Date: Thu, 14 Dec 2023 18:24:14 +0100 Subject: [PATCH 10/17] Remove triggering GHA on push --- .github/workflows/sync-end-to-end.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/sync-end-to-end.yml b/.github/workflows/sync-end-to-end.yml index dd7578c77f..9830c28a7b 100644 --- a/.github/workflows/sync-end-to-end.yml +++ b/.github/workflows/sync-end-to-end.yml @@ -1,7 +1,6 @@ name: Sync-End-to-End tests on: - push: schedule: - cron: '0 5 * * *' # run at 5 AM UTC From bab1dfdeea12acfaea12aeb0812c1088271f2d69 Mon Sep 17 00:00:00 2001 From: Lorenzo Mattei Date: Fri, 15 Dec 2023 16:18:12 +0100 Subject: [PATCH 11/17] Let tests on other OS run even if one fails --- .github/workflows/sync-end-to-end.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sync-end-to-end.yml b/.github/workflows/sync-end-to-end.yml index 9830c28a7b..5312d05153 100644 --- a/.github/workflows/sync-end-to-end.yml +++ b/.github/workflows/sync-end-to-end.yml @@ -68,6 +68,7 @@ jobs: matrix: os-version: [15, 16, 17] #max-parallel: 1 # Uncomment this line to run tests sequentially. + fail-fast: false steps: - name: Check out the code From 3f969fe2ebaef691748c677d3fbc9d75336504e8 Mon Sep 17 00:00:00 2001 From: Dominik Kapusta Date: Fri, 15 Dec 2023 16:50:18 +0100 Subject: [PATCH 12/17] Run on push --- .github/workflows/sync-end-to-end.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sync-end-to-end.yml b/.github/workflows/sync-end-to-end.yml index 5312d05153..0ae65ebca2 100644 --- a/.github/workflows/sync-end-to-end.yml +++ b/.github/workflows/sync-end-to-end.yml @@ -1,6 +1,7 @@ name: Sync-End-to-End tests on: + push: schedule: - cron: '0 5 * * *' # run at 5 AM UTC From 6e54f125e272eb9e7c42d35fff4558e59dca76e3 Mon Sep 17 00:00:00 2001 From: Dominik Kapusta Date: Fri, 15 Dec 2023 17:22:56 +0100 Subject: [PATCH 13/17] Use upload/download action v4 (much faster than v3) --- .github/workflows/adhoc.yml | 4 ++-- .github/workflows/alpha.yml | 2 +- .github/workflows/end-to-end.yml | 2 +- .github/workflows/nightly.yml | 2 +- .github/workflows/pr.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/sync-end-to-end.yml | 6 +++--- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/adhoc.yml b/.github/workflows/adhoc.yml index 5a2e33c100..0b78830657 100644 --- a/.github/workflows/adhoc.yml +++ b/.github/workflows/adhoc.yml @@ -58,13 +58,13 @@ jobs: echo "dsyms_path=${{ github.workspace }}/${{ env.dsyms_filename }}" >> $GITHUB_ENV - name: Upload IPA artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ env.ipa_filename }} path: ${{ env.ipa_path }} - name: Upload dSYMs artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ env.dsyms_filename }} path: ${{ env.dsyms_path }} diff --git a/.github/workflows/alpha.yml b/.github/workflows/alpha.yml index 5d39e3648f..e32bb92231 100644 --- a/.github/workflows/alpha.yml +++ b/.github/workflows/alpha.yml @@ -97,7 +97,7 @@ jobs: echo "build_version=${build_version}" >> $GITHUB_ENV - name: Upload dSYMs artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: DuckDuckGo-Alpha-dSYM-${{ env.app_version }} path: ${{ env.dsyms_path }} diff --git a/.github/workflows/end-to-end.yml b/.github/workflows/end-to-end.yml index de34f5fd98..d15375f1e1 100644 --- a/.github/workflows/end-to-end.yml +++ b/.github/workflows/end-to-end.yml @@ -78,7 +78,7 @@ jobs: --data ' { "data": { "name": "GH Workflow Failure - End to end tests", "workspace": "${{ vars.GH_ASANA_WORKSPACE_ID }}", "projects": [ "${{ vars.GH_ASANA_IOS_APP_PROJECT_ID }}" ], "notes" : "The end to end workflow has failed. See https://github.com/duckduckgo/iOS/actions/runs/${{ github.run_id }}" } }' - name: Upload logs when workflow failed - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: failure() with: name: BuildLogs diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5a67fdb265..ed86f8de79 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -50,7 +50,7 @@ jobs: | xcbeautify --report junit --report-path . --junit-report-filename unittests.xml - name: Upload logs if workflow failed - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: failure() with: name: BuildLogs diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 1b2e4bba26..a93b4d49e4 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -86,7 +86,7 @@ jobs: | xcbeautify --report junit --report-path . --junit-report-filename unittests.xml - name: Upload logs if workflow failed - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: failure() with: name: BuildLogs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 58f13d0b12..eed601f957 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -80,7 +80,7 @@ jobs: - name: Upload dSYMs artifact if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: DuckDuckGo-${{ steps.destination.outputs.destination }}-dSYM-${{ env.app_version }} path: ${{ env.dsyms_path }} diff --git a/.github/workflows/sync-end-to-end.yml b/.github/workflows/sync-end-to-end.yml index 0ae65ebca2..4885ee97c7 100644 --- a/.github/workflows/sync-end-to-end.yml +++ b/.github/workflows/sync-end-to-end.yml @@ -46,13 +46,13 @@ jobs: | tee xcodebuild.log - name: Store Binary - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: duckduckgo-ios-app path: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app - name: Upload logs when workflow failed - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: failure() with: name: BuildLogs @@ -82,7 +82,7 @@ jobs: debug: true - name: Retrieve Binary - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: duckduckgo-ios-app path: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app From 003789a7bfe62ca37cd20c747fdcb0a22fe632e1 Mon Sep 17 00:00:00 2001 From: Dominik Kapusta Date: Fri, 15 Dec 2023 17:23:15 +0100 Subject: [PATCH 14/17] Use XL runner and set a timeout --- .github/workflows/sync-end-to-end.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sync-end-to-end.yml b/.github/workflows/sync-end-to-end.yml index 4885ee97c7..924b65f68c 100644 --- a/.github/workflows/sync-end-to-end.yml +++ b/.github/workflows/sync-end-to-end.yml @@ -64,7 +64,8 @@ jobs: sync-end-to-end-tests: name: Sync End To End Tests needs: build-for-sync-end-to-end-tests - runs-on: macos-13 + runs-on: macos-13-xlarge + timeout-minutes: 30 strategy: matrix: os-version: [15, 16, 17] From fb13306f1eb0f28d95c83c15f1b70582bd506e4e Mon Sep 17 00:00:00 2001 From: Dominik Kapusta Date: Fri, 15 Dec 2023 17:25:50 +0100 Subject: [PATCH 15/17] Use XL runner and set a timeout also for the build job --- .github/workflows/sync-end-to-end.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sync-end-to-end.yml b/.github/workflows/sync-end-to-end.yml index 924b65f68c..0a1d1a2ae2 100644 --- a/.github/workflows/sync-end-to-end.yml +++ b/.github/workflows/sync-end-to-end.yml @@ -8,7 +8,8 @@ on: jobs: build-for-sync-end-to-end-tests: name: Build for Sync End To End Tests - runs-on: macos-13 + runs-on: macos-13-xlarge + timeout-minutes: 30 steps: - name: Check out the code From 474221f02ad1e9e8c9fdfa058cfcf341514b5a64 Mon Sep 17 00:00:00 2001 From: Dominik Kapusta Date: Fri, 15 Dec 2023 17:31:30 +0100 Subject: [PATCH 16/17] Don't use XL runners --- .github/workflows/sync-end-to-end.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync-end-to-end.yml b/.github/workflows/sync-end-to-end.yml index 0a1d1a2ae2..801df7d598 100644 --- a/.github/workflows/sync-end-to-end.yml +++ b/.github/workflows/sync-end-to-end.yml @@ -8,7 +8,7 @@ on: jobs: build-for-sync-end-to-end-tests: name: Build for Sync End To End Tests - runs-on: macos-13-xlarge + runs-on: macos-13 timeout-minutes: 30 steps: @@ -65,7 +65,7 @@ jobs: sync-end-to-end-tests: name: Sync End To End Tests needs: build-for-sync-end-to-end-tests - runs-on: macos-13-xlarge + runs-on: macos-13 timeout-minutes: 30 strategy: matrix: From a463515484945e69f758f6ab275320bf28cbeba3 Mon Sep 17 00:00:00 2001 From: Dominik Kapusta Date: Fri, 15 Dec 2023 18:02:48 +0100 Subject: [PATCH 17/17] Revert "Run on push" This reverts commit 3f969fe2ebaef691748c677d3fbc9d75336504e8. --- .github/workflows/sync-end-to-end.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/sync-end-to-end.yml b/.github/workflows/sync-end-to-end.yml index 801df7d598..4bacae1e17 100644 --- a/.github/workflows/sync-end-to-end.yml +++ b/.github/workflows/sync-end-to-end.yml @@ -1,7 +1,6 @@ name: Sync-End-to-End tests on: - push: schedule: - cron: '0 5 * * *' # run at 5 AM UTC