UI Test workflows

.github/workflows/sync_end_to_end_legacy_os.yml

@@ -153,7 +153,7 @@ jobs:
  #          --header "Accept: application/json" \
  #          --header "Authorization: Bearer ${{ secrets.ASANA_ACCESS_TOKEN }}" \
  #          --header "Content-Type: application/json" \
- #          --data ' { "data": { "name": "GH Workflow Failure - Sync End to end tests", "projects": [ "${{ vars.MACOS_APP_DEVELOPMENT_ASANA_PROJECT_ID }}" ], "notes" : "The end to end workflow has failed. See https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" } }'
+ #          --data ' { "data": { "name": "GH Workflow Failure - Sync End to end tests Legacy", "projects": [ "${{ vars.MACOS_APP_DEVELOPMENT_ASANA_PROJECT_ID }}" ], "notes" : "The end to end workflow has failed. See https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" } }'
 
     - name: Publish tests report
       uses: mikepenz/action-junit-report@v3

.github/workflows/ui_tests.yml

@@ -0,0 +1,107 @@
+name: UI Tests
+
+on:
+  workflow_dispatch:
+  schedule:
+     - cron: '0 3 * * 1-5' # 3AM UTC offsetted to legacy to avoid action-junit-report@v4 bug
+
+jobs:
+  ui-tests:
+    name: UI tests
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        runner: [macos-13-xlarge, macos-14-xlarge]
+
+    timeout-minutes: 120
+
+    steps:
+    - name: Check out the code
+      uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - name: Set cache key hash
+      run: |
+        has_only_tags=$(jq '[ .pins[].state | has("version") ] | all' DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved)
+        if [[ "$has_only_tags" == "true" ]]; then
+          echo "cache_key_hash=${{ hashFiles('DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }}" >> $GITHUB_ENV
+        else
+          echo "Package.resolved contains dependencies specified by branch or commit, skipping cache."
+        fi
+
+    - name: Cache SPM
+      if: env.cache_key_hash
+      uses: actions/cache@v3
+      with:
+        path: DerivedData/SourcePackages
+        key: ${{ runner.os }}-spm-${{ env.cache_key_hash }}
+        restore-keys: |
+          ${{ runner.os }}-spm-
+
+    - name: Install Apple Developer ID Application certificate
+      uses: ./.github/actions/install-certs-and-profiles
+      with:
+        BUILD_CERTIFICATE_BASE64: ${{ secrets.BUILD_CERTIFICATE_BASE64 }}
+        P12_PASSWORD: ${{ secrets.P12_PASSWORD }}
+        KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
+        REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.REVIEW_PROVISION_PROFILE_BASE64 }}
+        RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.RELEASE_PROVISION_PROFILE_BASE64 }}
+        DBP_AGENT_RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.DBP_AGENT_RELEASE_PROVISION_PROFILE_BASE64 }}
+        DBP_AGENT_REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.DBP_AGENT_REVIEW_PROVISION_PROFILE_BASE64 }}
+        NETP_SYSEX_RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_SYSEX_RELEASE_PROVISION_PROFILE_BASE64_V2 }}
+        NETP_SYSEX_REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_SYSEX_REVIEW_PROVISION_PROFILE_BASE64_V2 }}
+        NETP_AGENT_RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_AGENT_RELEASE_PROVISION_PROFILE_BASE64_V2 }}
+        NETP_AGENT_REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_AGENT_REVIEW_PROVISION_PROFILE_BASE64_V2 }}
+        NETP_NOTIFICATIONS_RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_NOTIFICATIONS_RELEASE_PROVISION_PROFILE_BASE64 }}
+        NETP_NOTIFICATIONS_REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_NOTIFICATIONS_REVIEW_PROVISION_PROFILE_BASE64 }}
+
+    - name: Select Xcode
+      run: sudo xcode-select -s /Applications/Xcode_$(<.xcode-version).app/Contents/Developer
+
+    - name: Build and run UI Testing
+      run: |
+        defaults write com.duckduckgo.macos.browser.review moveToApplicationsFolderAlertSuppress 1
+        defaults write com.duckduckgo.macos.browser.review onboarding.finished -bool true
+        export CI=false
+        export CONFIGURATION=Review 
+        export DERIVED_DATA_PATH="$(pwd)/DerivedData"
+        set -o pipefail && xcodebuild test \
+          -scheme "UI Tests" \
+          -configuration $CONFIGURATION \
+          -derivedDataPath $DERIVED_DATA_PATH \
+          -skipPackagePluginValidation \
+          -skipMacroValidation \
+          -test-iterations 2 \
+          -retry-tests-on-failure \
+        | tee xcodebuild.log \
+        | xcbeautify --report junit --report-path . --junit-report-filename ui-tests.xml
+
+     #   - name: Create Asana task when workflow failed
+ #     if: ${{ failure() }} && github.ref == 'refs/heads/main'
+ #     run: |
+ #       curl -s "https://app.asana.com/api/1.0/tasks" \
+ #          --header "Accept: application/json" \
+ #          --header "Authorization: Bearer ${{ secrets.ASANA_ACCESS_TOKEN }}" \
+ #          --header "Content-Type: application/json" \
+ #          --data ' { "data": { "name": "GH Workflow Failure - UI Tests", "projects": [ "${{ vars.MACOS_APP_DEVELOPMENT_ASANA_PROJECT_ID }}" ], "notes" : "The end to end workflow has failed. See https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" } }'
+
+
+    - name: Publish tests report
+      uses: mikepenz/action-junit-report@v4
+      if: always() 
+      with:
+        check_name: "Test Report ${{ matrix.runner }}"
+        report_paths: ui-tests.xml
+
+    - name: Upload logs when workflow failed
+      uses: actions/upload-artifact@v4
+      if: failure()
+      with:
+        name: "BuildLogs ${{ matrix.runner }}"
+        path: |
+          xcodebuild.log
+          DerivedData/Logs/Test/*.xcresult
+          ~/Library/Logs/DiagnosticReports/*
+        retention-days: 7

.github/workflows/ui_tests_legacy_os.yml

@@ -0,0 +1,183 @@
+name: UI Tests MacOS11/12
+
+on:
+  workflow_dispatch:
+  schedule:
+     - cron: '0 4 * * 1-5' # 4 AM UTC
+
+jobs:
+  # create-notarized-app:
+  #   name: Prepare DMG Release
+  #   uses: ./.github/workflows/build_notarized.yml
+  #   with:
+  #     release-type: review
+  #     create-dmg: false
+  #     branch: ${{ github.ref_name }}
+  #   secrets:
+  #     BUILD_CERTIFICATE_BASE64: ${{ secrets.BUILD_CERTIFICATE_BASE64 }}
+  #     P12_PASSWORD: ${{ secrets.P12_PASSWORD }}
+  #     KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
+  #     REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.REVIEW_PROVISION_PROFILE_BASE64 }}
+  #     RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.RELEASE_PROVISION_PROFILE_BASE64 }}
+  #     DBP_AGENT_RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.DBP_AGENT_RELEASE_PROVISION_PROFILE_BASE64 }}
+  #     DBP_AGENT_REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.DBP_AGENT_REVIEW_PROVISION_PROFILE_BASE64 }}
+  #     NETP_SYSEX_RELEASE_PROVISION_PROFILE_BASE64_V2: ${{ secrets.NETP_SYSEX_RELEASE_PROVISION_PROFILE_BASE64_V2 }}
+  #     NETP_SYSEX_REVIEW_PROVISION_PROFILE_BASE64_V2: ${{ secrets.NETP_SYSEX_REVIEW_PROVISION_PROFILE_BASE64_V2 }}
+  #     NETP_AGENT_RELEASE_PROVISION_PROFILE_BASE64_V2: ${{ secrets.NETP_AGENT_RELEASE_PROVISION_PROFILE_BASE64_V2 }}
+  #     NETP_AGENT_REVIEW_PROVISION_PROFILE_BASE64_V2: ${{ secrets.NETP_AGENT_REVIEW_PROVISION_PROFILE_BASE64_V2 }}
+  #     NETP_NOTIFICATIONS_RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_NOTIFICATIONS_RELEASE_PROVISION_PROFILE_BASE64 }}
+  #     NETP_NOTIFICATIONS_REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_NOTIFICATIONS_REVIEW_PROVISION_PROFILE_BASE64 }}
+  #     APPLE_API_KEY_BASE64: ${{ secrets.APPLE_API_KEY_BASE64 }}
+  #     APPLE_API_KEY_ID: ${{ secrets.APPLE_API_KEY_ID }}
+  #     APPLE_API_KEY_ISSUER: ${{ secrets.APPLE_API_KEY_ISSUER }}
+  #     ASANA_ACCESS_TOKEN: ${{ secrets.ASANA_ACCESS_TOKEN }}
+  #     MM_HANDLES_BASE64: ${{ secrets.MM_HANDLES_BASE64 }}
+  #     MM_WEBHOOK_URL: ${{ secrets.MM_WEBHOOK_URL }}
+  #     AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  #     AWS_ACCESS_KEY_ID_RELEASE_S3: ${{ secrets.AWS_ACCESS_KEY_ID_RELEASE_S3 }}
+  #     AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  #     AWS_SECRET_ACCESS_KEY_RELEASE_S3: ${{ secrets.AWS_SECRET_ACCESS_KEY_RELEASE_S3 }}
+
+  export-testserver:
+    name: Export tests server
+    runs-on: macos-13-xlarge
+    steps:
+    - name: Check out the code
+      uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - name: Build test server
+      run: |
+        set -o pipefail && xcodebuild build \
+          -scheme "tests-server" \
+          -derivedDataPath DerivedData \
+          -configuration Review 
+
+    - name: Upload tests-server
+      uses: actions/upload-artifact@v4
+      with:
+        name: tests-server
+        path: DerivedData/Build/Products/Review/tests-server
+
+  ui-tests-legacy-os:
+    name: UI Tests legacy OS
+    needs: [ export-testserver ]
+
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        runner: [ macos-11, macos-12-large ]
+
+    timeout-minutes: 120
+    steps:
+    - name: Check out the code
+      uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - name: Download and unzip artifact
+      uses: actions/download-artifact@v4
+
+    - name: Set cache key hash
+      run: |
+        has_only_tags=$(jq '[ .pins[].state | has("version") ] | all' DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved)
+        if [[ "$has_only_tags" == "true" ]]; then
+          echo "cache_key_hash=${{ hashFiles('DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }}" >> $GITHUB_ENV
+        else
+          echo "Package.resolved contains dependencies specified by branch or commit, skipping cache."
+        fi
+
+    - name: Cache SPM
+      if: env.cache_key_hash
+      uses: actions/cache@v3
+      with:
+        path: DerivedData/SourcePackages
+        key: ${{ runner.os }}-spm-${{ env.cache_key_hash }}
+        restore-keys: |
+          ${{ runner.os }}-spm-
+
+    - name: Install Apple Developer ID Application certificate
+      uses: ./.github/actions/install-certs-and-profiles
+      with:
+        BUILD_CERTIFICATE_BASE64: ${{ secrets.BUILD_CERTIFICATE_BASE64 }}
+        P12_PASSWORD: ${{ secrets.P12_PASSWORD }}
+        KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
+        REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.REVIEW_PROVISION_PROFILE_BASE64 }}
+        RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.RELEASE_PROVISION_PROFILE_BASE64 }}
+        DBP_AGENT_RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.DBP_AGENT_RELEASE_PROVISION_PROFILE_BASE64 }}
+        DBP_AGENT_REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.DBP_AGENT_REVIEW_PROVISION_PROFILE_BASE64 }}
+        NETP_SYSEX_RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_SYSEX_RELEASE_PROVISION_PROFILE_BASE64_V2 }}
+        NETP_SYSEX_REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_SYSEX_REVIEW_PROVISION_PROFILE_BASE64_V2 }}
+        NETP_AGENT_RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_AGENT_RELEASE_PROVISION_PROFILE_BASE64_V2 }}
+        NETP_AGENT_REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_AGENT_REVIEW_PROVISION_PROFILE_BASE64_V2 }}
+        NETP_NOTIFICATIONS_RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_NOTIFICATIONS_RELEASE_PROVISION_PROFILE_BASE64 }}
+        NETP_NOTIFICATIONS_REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_NOTIFICATIONS_REVIEW_PROVISION_PROFILE_BASE64 }}
+
+    # This first bit is a hack to stop the app building package dependencies that isn't needed by the test runner
+    - name: Build test runner 
+      run: |
+        rm -rf LocalPackages
+        rm -rf /Users/runner/work/macos-browser/macos-browser/DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved 
+        sed -i '' '/\/\* Begin XCSwiftPackageProductDependency section \*\//,/\/\* End XCSwiftPackageProductDependency section \*\//d' DuckDuckGo.xcodeproj/project.pbxproj
+        sed -i '' '/\/\* Begin XCRemoteSwiftPackageReference section \*\//,/\/\* End XCRemoteSwiftPackageReference section \*\//d' DuckDuckGo.xcodeproj/project.pbxproj
+        set -o pipefail && xcodebuild build-for-testing \
+          -scheme "UI Tests Legacy" \
+          -derivedDataPath DerivedData \
+          -configuration Review 
+
+    - name: Unzip and Copy tests-server to /DerivedData                         
+      run: |
+        ls -R
+        cp -R tests-server/tests-server "../DerivedData/Build/Products/Review/"
+
+    - name: Unzip and Copy app to /DerivedData                         
+      run: |
+        cd DuckDuckGo-review-*.app && unzip DuckDuckGo-*.zip
+        cp -R "DuckDuckGo Review.app" "../DerivedData/Build/Products/Review/DuckDuckGo Review.app"
+
+    # xcpretty does nothing here
+    - name: Run tests
+      run: |
+        defaults write com.duckduckgo.macos.browser.review moveToApplicationsFolderAlertSuppress 1
+        defaults write com.duckduckgo.macos.browser.review onboarding.finished -bool true
+        set -o pipefail && xcodebuild test \
+          -scheme "UI Tests Legacy" \
+          -derivedDataPath DerivedData \
+          -configuration Review \
+          -test-iterations 2 \
+          -retry-tests-on-failure \
+        | tee xcodebuild.log \
+        | (if [ "${{ matrix.runner }}" = "macos-11" ]; then
+            xcpretty --report junit --output ui-tests.xml 
+          else
+            xcbeautify --report junit --report-path . --junit-report-filename ui-tests.xml
+          fi)
+
+ #   - name: Create Asana task when workflow failed
+ #     if: ${{ failure() }} && github.ref == 'refs/heads/main'
+ #     run: |
+ #       curl -s "https://app.asana.com/api/1.0/tasks" \
+ #          --header "Accept: application/json" \
+ #          --header "Authorization: Bearer ${{ secrets.ASANA_ACCESS_TOKEN }}" \
+ #          --header "Content-Type: application/json" \
+ #          --data ' { "data": { "name": "GH Workflow Failure - UI Tests Legacy OS", "projects": [ "${{ vars.MACOS_APP_DEVELOPMENT_ASANA_PROJECT_ID }}" ], "notes" : "The end to end workflow has failed. See https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" } }'
+
+    - name: Publish tests report
+      uses: mikepenz/action-junit-report@v4
+      if: always() 
+      with:
+        check_name: "Test Report ${{ matrix.runner }}"
+        report_paths: ui-tests.xml
+
+    - name: Upload logs when workflow failed
+      uses: actions/upload-artifact@v4
+      if: always() 
+      with:
+        name: "BuildLogs ${{ matrix.runner }}"
+        path: |
+          xcodebuild.log
+          DerivedData/Logs/Test/*.xcresult
+          ~/Library/Logs/DiagnosticReports/*
+        retention-days: 7

Configuration/Tests/TestsServer.xcconfig

@@ -18,3 +18,11 @@
 #include "../Common.xcconfig"
 
 PRODUCT_NAME = $(TARGET_NAME);
+CODE_SIGN_IDENTITY[config=CI][sdk=macosx*] =
+CODE_SIGN_IDENTITY[config=Review][sdk=macosx*] =
+CODE_SIGN_STYLE[config=Review][sdk=*] = Manual
+CODE_SIGN_STYLE[config=CI][sdk=*] = Manual
+PROVISIONING_PROFILE_SPECIFIER[config=Review][sdk=macosx*] =
+PROVISIONING_PROFILE_SPECIFIER[config=CI][sdk=macosx*] =
+CODE_SIGNING_ALLOWED[config=Review][sdk=macosx*] = NO
+CODE_SIGNING_ALLOWED[config=CI][sdk=macosx*] = NO

DuckDuckGo.xcodeproj/project.pbxproj

@@ -3487,13 +3487,6 @@
 			remoteGlobalIDString = B6EC37E729B5DA2A001ACE79;
 			remoteInfo = "tests-server";
 		};
-		EE02D41D2BB460B500DBE6B3 /* PBXContainerItemProxy */ = {
-			isa = PBXContainerItemProxy;
-			containerPortal = AA585D76248FD31100E9A3E2 /* Project object */;
-			proxyType = 1;
-			remoteGlobalIDString = B6EC37E729B5DA2A001ACE79;
-			remoteInfo = "tests-server";
-		};
 /* End PBXContainerItemProxy section */
 
 /* Begin PBXCopyFilesBuildPhase section */
@@ -9252,7 +9245,6 @@
 			buildRules = (
 			);
 			dependencies = (
-				EE02D41E2BB460B500DBE6B3 /* PBXTargetDependency */,
 				B6F997BD2B8F35EF00476735 /* PBXTargetDependency */,
 			);
 			name = "UI Tests";
@@ -13581,11 +13573,6 @@
 			isa = PBXTargetDependency;
 			productRef = B6F997C02B8F35F800476735 /* SwiftLintPlugin */;
 		};
-		EE02D41E2BB460B500DBE6B3 /* PBXTargetDependency */ = {
-			isa = PBXTargetDependency;
-			target = B6EC37E729B5DA2A001ACE79 /* tests-server */;
-			targetProxy = EE02D41D2BB460B500DBE6B3 /* PBXContainerItemProxy */;
-		};
 /* End PBXTargetDependency section */
 
 /* Begin PBXVariantGroup section */