CT-2856 | Backward compatibility testing (#227)

* Backward compatibility testing setup * Fix formatting and test on current branch * Skip collecting simulator logs * Add missing job to workflow * Add pre-reqs * Explicitly specify devices * Run regular tests on latest iOS available * Bring preauth payment options into view * Run against master
Judopay · Jan 22, 2024 · 989d025 · 989d025
1 parent ebf1dd4
commit 989d025
Show file tree

Hide file tree

Showing 3 changed files with 127 additions and 17 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -86,8 +86,84 @@ commands:
               ]
             }
 
+  notify_slack_pass_weekly:
+    steps:
+      - slack/notify:
+          event: pass
+          custom: |
+            {
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": "CircleCI has just *successfully* ran the :aapl: iOS E2E mobile tests for backward compatibility, here are the results: \n *<$CIRCLE_BUILD_URL | :circleci: Open build>* \n *<https://console.firebase.google.com/u/0/project/$GOOGLE_PROJECT_ID/testlab/histories/ | :firebase: Firebase results>*"
+                  }
+                },
+                {
+                  "type": "section",
+                  "fields": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Project:*\n$CIRCLE_PROJECT_REPONAME"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Branch:*\n$CIRCLE_BRANCH"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Commit:* \n <https://github.com/Judopay/$CIRCLE_PROJECT_REPONAME/commit/${CIRCLE_SHA1} | ${CIRCLE_SHA1:0:7}>"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Author:*\n$CIRCLE_USERNAME"
+                    }
+                  ]
+                }
+              ]
+            }
+
+  notify_slack_error_weekly:
+    steps:
+      - slack/notify:
+          event: fail
+          custom: |
+            {
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": "CircleCI ran the :aapl: iOS E2E mobile tests for backward compatibility but they *failed*, here are the results: \n *<$CIRCLE_BUILD_URL | :circleci: Open build>* \n *<https://console.firebase.google.com/u/0/project/$GOOGLE_PROJECT_ID/testlab/histories/ | :firebase: Firebase results>*"
+                  }
+                },
+                {
+                  "type": "section",
+                  "fields": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Project:*\n$CIRCLE_PROJECT_REPONAME"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Branch:*\n$CIRCLE_BRANCH"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Commit:* \n <https://github.com/Judopay/$CIRCLE_PROJECT_REPONAME/commit/${CIRCLE_SHA1} | ${CIRCLE_SHA1:0:7}>"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Author:*\n$CIRCLE_USERNAME"
+                    }
+                  ]
+                }
+              ]
+            }
+
 parameters:
-  nightly:
+  weekly:
     type: boolean
     default: false
 
@@ -265,9 +341,39 @@ jobs:
             gcloud firebase test ios run \
               --test ./Output/ObjectiveCExampleAppUITests.zip \
               --xcode-version 15.0 \
+              --device model=iphone14pro,version=16.6 \
               --results-bucket $FIREBASE_TEST_RESULTS_BUCKET
       - notify_slack_error
       - notify_slack_pass
+  instrumented_tests_sample_multiple:
+    executor: macos
+    steps:
+      - attach_workspace:
+          at: ~/project
+      - run:
+          name: Install Google Cloud CLI
+          command: |
+            brew install google-cloud-sdk --cask
+      - run:
+          name: Store Google Service Account
+          command: echo $GCLOUD_SERVICE_KEY > ${HOME}/gcloud-service-key.json
+      - run:
+          name: Authorize gcloud and set config defaults
+          command: |
+            gcloud auth activate-service-account --key-file=${HOME}/gcloud-service-key.json
+            gcloud --quiet config set project ${GOOGLE_PROJECT_ID}
+      - run:
+          name: Test with Firebase Test Lab
+          no_output_timeout: 30m
+          command: >
+            gcloud firebase test ios run \
+              --test ./Output/ObjectiveCExampleAppUITests.zip \
+              --xcode-version 15.0 \
+              --device model=iphone13pro,version=15.2 \
+              --device model=iphone11pro,version=14.7 \
+              --results-bucket $FIREBASE_TEST_RESULTS_BUCKET
+      - notify_slack_error_weekly
+      - notify_slack_pass_weekly
   run_instrumented_tests:
     executor: macos
     steps:
@@ -343,7 +449,7 @@ jobs:
 workflows:  
   build:
     when:
-      not: << pipeline.parameters.nightly >>
+      not: << pipeline.parameters.weekly >>
     jobs:
       - checkout_project:
           filters:
@@ -414,17 +520,18 @@ workflows:
           environment: prod
           requires:
             - release_sdk
-  e2e-nightly:
-    when: << pipeline.parameters.nightly >>
-    jobs:
-      - checkout_project
-      - build_instrumented_tests_package:
-          context: shared-secrets
-      - run_instrumented_tests:      
-          context: shared-secrets
-          requires:
-            - build_instrumented_tests_package
-          filters:
-            branches:
-              only:
-                - master
+  e2e-backward-compat-weekly:
+      when: << pipeline.parameters.weekly >>
+      jobs:
+        - checkout_project
+        - build_instrumented_tests_package:
+            requires:
+              - checkout_project
+        - instrumented_tests_sample_multiple:
+            context: shared-secrets
+            requires:
+              - build_instrumented_tests_package
+            filters:
+              branches:
+                only:
+                  - master
diff --git a/...les/ObjectiveCExampleApp/ObjectiveCExampleAppUITests/CardPayment/CardPaymentUITests.swift b/...les/ObjectiveCExampleApp/ObjectiveCExampleAppUITests/CardPayment/CardPaymentUITests.swift
@@ -304,6 +304,8 @@ final class CardPaymentUITests: XCTestCase {
 
         app.launch()
 
+        app.swipeUp()
+
         app.cellWithIdentifier(TestData.PREAUTH_METHODS_LABEL)?.tap()
 
         app.addCard?.tap()

diff --git a/fastlane/Fastfile b/fastlane/Fastfile
@@ -46,7 +46,8 @@ platform :ios do
       scheme: "ObjectiveCExampleAppUITests",
       destination: "generic/platform=iOS",
       sdk: "iphoneos",
-      xcodebuild_command: "env NSUnbufferedIO=YES xcodebuild -allowProvisioningUpdates TEST_API_JUDO_ID=$JUDO_API_ID TEST_API_TOKEN=$JUDO_API_TOKEN TEST_API_SECRET=$JUDO_API_SECRET"
+      xcodebuild_command: "env NSUnbufferedIO=YES xcodebuild -allowProvisioningUpdates TEST_API_JUDO_ID=$JUDO_API_ID TEST_API_TOKEN=$JUDO_API_TOKEN TEST_API_SECRET=$JUDO_API_SECRET",
+      include_simulator_logs: false
     }
 
     sync_code_signing sync_code_signing_options