Merge pull request #1107 from bugsnag/release-v6.9.4

Release v6.9.4
bugsnag · Jun 2, 2021 · ede9f5b · ede9f5b
2 parents b0b619e + 3fdeab6
commit ede9f5b
Show file tree

Hide file tree

Showing 46 changed files with 527 additions and 532 deletions.
diff --git a/.buildkite/pipeline.full.yml b/.buildkite/pipeline.full.yml
@@ -2,10 +2,42 @@ env:
   LANG: "en_GB.UTF-8"
 
 steps:
-  - label: ':ios: iOS 13 full end-to-end tests'
+  ##############################################################################
+  #
+  # E2E tests
+  #
+
+  - label: 'iOS 13 E2E tests batch 1'
+    depends_on:
+      - cocoa_fixture
+    timeout_in_minutes: 45
+    agents:
+      queue: opensource
+    plugins:
+      artifacts#v1.3.0:
+        download: ["features/fixtures/ios-swift-cocoapods/output/iOSTestApp.ipa"]
+      docker-compose#v3.3.0:
+        run: cocoa-maze-runner
+        command:
+          - "--app=/app/build/iOSTestApp.ipa"
+          - "--farm=bs"
+          - "--device=IOS_13"
+          - "--resilient"
+          - "--appium-version=1.17.0"
+          - "--fail-fast"
+          - "--exclude=features/[h-z].*.feature"
+          - "--order=random"
+    concurrency: 9
+    concurrency_group: browserstack-app
+    retry:
+      automatic:
+        - exit_status: -1  # Agent was lost
+          limit: 2
+
+  - label: 'iOS 13 E2E tests batch 2'
     depends_on:
       - cocoa_fixture
-    timeout_in_minutes: 120
+    timeout_in_minutes: 45
     agents:
       queue: opensource
     plugins:
@@ -20,17 +52,46 @@ steps:
           - "--resilient"
           - "--appium-version=1.17.0"
           - "--fail-fast"
+          - "--exclude=features/[a-g].*.feature"
+          - "--order=random"
+    concurrency: 9
+    concurrency_group: browserstack-app
+    retry:
+      automatic:
+        - exit_status: -1  # Agent was lost
+          limit: 2
+
+  - label: 'iOS 12 E2E tests batch 1'
+    depends_on:
+      - cocoa_fixture
+    timeout_in_minutes: 45
+    agents:
+      queue: opensource
+    plugins:
+      artifacts#v1.3.0:
+        download: ["features/fixtures/ios-swift-cocoapods/output/iOSTestApp.ipa"]
+      docker-compose#v3.3.0:
+        run: cocoa-maze-runner
+        command:
+          - "--app=/app/build/iOSTestApp.ipa"
+          - "--farm=bs"
+          - "--device=IOS_12"
+          - "--resilient"
+          - "--appium-version=1.17.0"
+          - "--fail-fast"
+          - "--exclude=features/[h-z].*.feature"
+          - "--order=random"
     concurrency: 9
     concurrency_group: browserstack-app
     retry:
       automatic:
         - exit_status: -1  # Agent was lost
           limit: 2
 
-  - label: ':ios: iOS 12 full end-to-end tests'
+  - label: 'iOS 12 E2E tests batch 2'
     depends_on:
       - cocoa_fixture
-    timeout_in_minutes: 120
+    timeout_in_minutes: 45
     agents:
       queue: opensource
     plugins:
@@ -45,19 +106,49 @@ steps:
           - "--resilient"
           - "--appium-version=1.17.0"
           - "--fail-fast"
+          - "--exclude=features/[a-g].*.feature"
+          - "--order=random"
+    concurrency: 9
+    concurrency_group: browserstack-app
+    retry:
+      automatic:
+        - exit_status: -1  # Agent was lost
+          limit: 2
+
+  - label: 'iOS 11 E2E tests batch 1'
+    depends_on:
+      - cocoa_fixture
+    # More time than other steps as the BrowserStack iOS 11 devices seem particularly unstable and
+    # sessions need resetting frequently, taking a minute or more each time.
+    timeout_in_minutes: 50
+    agents:
+      queue: opensource
+    plugins:
+      artifacts#v1.3.0:
+        download: ["features/fixtures/ios-swift-cocoapods/output/iOSTestApp.ipa"]
+      docker-compose#v3.3.0:
+        run: cocoa-maze-runner
+        command:
+          - "--app=/app/build/iOSTestApp.ipa"
+          - "--farm=bs"
+          - "--device=IOS_11_0_IPHONE_8_PLUS"
+          - "--appium-version=1.8.0"
+          - "--fail-fast"
+          - "--exclude=features/[h-z].*.feature"
+          - "--order=random"
     concurrency: 9
     concurrency_group: browserstack-app
     retry:
       automatic:
         - exit_status: -1  # Agent was lost
           limit: 2
 
-  - label: ':ios: iOS 11 full end-to-end tests'
+  - label: 'iOS 11 E2E tests batch 2'
     depends_on:
       - cocoa_fixture
     # More time than other steps as the BrowserStack iOS 11 devices seem particularly unstable and
     # sessions need resetting frequently, taking a minute or more each time.
-    timeout_in_minutes: 120
+    timeout_in_minutes: 50
     agents:
       queue: opensource
     plugins:
@@ -71,14 +162,16 @@ steps:
           - "--device=IOS_11_0_IPHONE_8_PLUS"
           - "--appium-version=1.8.0"
           - "--fail-fast"
+          - "--exclude=features/[a-g].*.feature"
+          - "--order=random"
     concurrency: 9
     concurrency_group: browserstack-app
     retry:
       automatic:
         - exit_status: -1  # Agent was lost
           limit: 2
 
-  - label: ':apple: macOS 11.0 full end-to-end tests'
+  - label: 'macOS 11.0 E2E tests'
     depends_on:
       - cocoa_fixture
     timeout_in_minutes: 60
@@ -97,8 +190,9 @@ steps:
         --app=macOSTestApp
         --tags='not @skip_macos'
         --fail-fast
+        --order=random
 
-  - label: ':apple: macOS 10.13 full end-to-end tests'
+  - label: 'macOS 10.13 E2E tests'
     depends_on:
       - cocoa_fixture
     timeout_in_minutes: 60
@@ -117,8 +211,9 @@ steps:
         --app=macOSTestApp
         --tags='not @skip_macos'
         --fail-fast
+        --order=random
 
-  - label: ':apple: macOS 10.14 full end-to-end tests'
+  - label: 'macOS 10.14 E2E tests'
     depends_on:
       - cocoa_fixture
     timeout_in_minutes: 60
@@ -137,6 +232,12 @@ steps:
         --app=macOSTestApp
         --tags='not @skip_macos'
         --fail-fast
+        --order=random
+
+  ##############################################################################
+  #
+  # Build example apps
+  #
 
   - label: 'examples/objective-c-ios'
     agents:

diff --git a/.buildkite/pipeline.quick.yml b/.buildkite/pipeline.quick.yml
@@ -2,6 +2,11 @@ env:
   LANG: "en_GB.UTF-8"
 
 steps:
+  ##############################################################################
+  #
+  # Unit tests
+  #
+
   - label: macOS 11 unit tests
     timeout_in_minutes: 10
     agents:
@@ -104,7 +109,12 @@ steps:
     artifact_paths:
       - logs/*
 
-  - label: ':apple: macOS 10.15 full end-to-end tests'
+  ##############################################################################
+  #
+  # E2E tests
+  #
+
+  - label: 'macOS 10.15 E2E tests'
     depends_on:
       - cocoa_fixture
     timeout_in_minutes: 60
@@ -123,11 +133,12 @@ steps:
         --app=macOSTestApp
         --tags='not @skip_macos'
         --fail-fast
+        --order=random
 
-  - label: ':ios: iOS 14 full end-to-end tests'
+  - label: 'iOS 14 E2E tests batch 1'
     depends_on:
       - cocoa_fixture
-    timeout_in_minutes: 120
+    timeout_in_minutes: 45
     agents:
       queue: opensource
     plugins:
@@ -142,17 +153,46 @@ steps:
           - "--resilient"
           - "--appium-version=1.17.0"
           - "--fail-fast"
+          - "--exclude=features/[h-z].*.feature"
+          - "--order=random"
     concurrency: 9
     concurrency_group: browserstack-app
     retry:
       automatic:
         - exit_status: -1  # Agent was lost
           limit: 2
 
-  - label: ':ios: iOS 10 full end-to-end tests - batch 1'
+  - label: 'iOS 14 E2E tests batch 2'
     depends_on:
       - cocoa_fixture
-    timeout_in_minutes: 60
+    timeout_in_minutes: 45
+    agents:
+      queue: opensource
+    plugins:
+      artifacts#v1.3.0:
+        download: ["features/fixtures/ios-swift-cocoapods/output/iOSTestApp.ipa"]
+      docker-compose#v3.3.0:
+        run: cocoa-maze-runner
+        command:
+          - "--app=/app/build/iOSTestApp.ipa"
+          - "--farm=bs"
+          - "--device=IOS_14"
+          - "--resilient"
+          - "--appium-version=1.17.0"
+          - "--fail-fast"
+          - "--exclude=features/[a-g].*.feature"
+          - "--order=random"
+    concurrency: 9
+    concurrency_group: browserstack-app
+    retry:
+      automatic:
+        - exit_status: -1  # Agent was lost
+          limit: 2
+
+  - label: 'iOS 10 E2E tests batch 1'
+    depends_on:
+      - cocoa_fixture
+    timeout_in_minutes: 45
     agents:
       queue: opensource
     plugins:
@@ -166,19 +206,19 @@ steps:
           - "--device=IOS_10"
           - "--appium-version=1.8.0"
           - "--fail-fast"
-          - "-e"
-          - "features/[o-z].*.feature"
+          - "--exclude=features/[h-z].*.feature"
+          - "--order=random"
     concurrency: 9
     concurrency_group: browserstack-app
     retry:
       automatic:
         - exit_status: -1  # Agent was lost
           limit: 2
 
-  - label: ':ios: iOS 10 full end-to-end tests - batch 2'
+  - label: 'iOS 10 E2E tests batch 2'
     depends_on:
       - cocoa_fixture
-    timeout_in_minutes: 60
+    timeout_in_minutes: 45
     agents:
       queue: opensource
     plugins:
@@ -192,8 +232,8 @@ steps:
           - "--device=IOS_10"
           - "--appium-version=1.8.0"
           - "--fail-fast"
-          - "-e"
-          - "features/[a-n].*.feature"
+          - "--exclude=features/[a-g].*.feature"
+          - "--order=random"
     concurrency: 9
     concurrency_group: browserstack-app
     retry: