From e0e9cab201e7ab04f0ca2a213267fef1a8d80218 Mon Sep 17 00:00:00 2001 From: Josh Edney Date: Wed, 9 Oct 2024 11:36:45 +0100 Subject: [PATCH 1/9] ios 17 tests on bb --- .buildkite/pipeline.yml | 48 ++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 6ab8c3f78..2aad7211b 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -36,16 +36,14 @@ steps: - features/fixtures/macos/output/macOSTestApp_Release.zip - features/fixtures/macos/output/macOSTestApp_Debug.zip - features/fixtures/ios/output/ipa_url_bb_release.txt - - features/fixtures/ios/output/ipa_url_bs_release.txt + - features/fixtures/ios/output/ipa_url_bb_release.txt + - features/fixtures/ios/output/ipa_url_bb_debug.txt - features/fixtures/ios/output/ipa_url_bb_debug.txt - - features/fixtures/ios/output/ipa_url_bs_debug.txt commands: - bundle install - make test-fixtures - bundle exec upload-app --farm=bb --app=./features/fixtures/ios/output/iOSTestApp_Release.ipa --app-id-file=./features/fixtures/ios/output/ipa_url_bb_release.txt - - bundle exec upload-app --farm=bs --app=./features/fixtures/ios/output/iOSTestApp_Release.ipa --app-id-file=./features/fixtures/ios/output/ipa_url_bs_release.txt - bundle exec upload-app --farm=bb --app=./features/fixtures/ios/output/iOSTestApp_Debug.ipa --app-id-file=./features/fixtures/ios/output/ipa_url_bb_debug.txt - - bundle exec upload-app --farm=bs --app=./features/fixtures/ios/output/iOSTestApp_Debug.ipa --app-id-file=./features/fixtures/ios/output/ipa_url_bs_debug.txt - label: Carthage timeout_in_minutes: 15 @@ -367,7 +365,7 @@ steps: # # BrowserStack # - - label: ':browserstack: iOS 17 E2E tests batch 1' + - label: ':bitbar: iOS 17 E2E tests batch 1' depends_on: - cocoa_fixture timeout_in_minutes: 60 @@ -375,18 +373,20 @@ steps: queue: opensource plugins: artifacts#v1.5.0: - download: "features/fixtures/ios/output/ipa_url_bs_release.txt" + download: "features/fixtures/ios/output/ipa_url_bb_release.txt" upload: "maze_output/failed/**/*" docker-compose#v3.7.0: pull: cocoa-maze-runner run: cocoa-maze-runner service-ports: true command: - - "--app=@/app/build/ipa_url_bs_release.txt" - - "--farm=bs" + - "--app=@/app/build/ipa_url_bb_release.txt" + - "--farm=bb" - "--device=IOS_17" - "--appium-version=1.21.0" - "--a11y-locator" + - "--no-tunnel" + - "--aws-public-ip" - "--fail-fast" - "features/release" - "--exclude=features/release/[e-z].*.feature$" @@ -402,7 +402,7 @@ steps: - exit_status: -1 # Agent was lost limit: 2 - - label: ':browserstack: iOS 17 E2E tests batch 2' + - label: ':bitbar: iOS 17 E2E tests batch 2' depends_on: - cocoa_fixture timeout_in_minutes: 60 @@ -410,18 +410,19 @@ steps: queue: opensource plugins: artifacts#v1.5.0: - download: "features/fixtures/ios/output/ipa_url_bs_release.txt" + download: "features/fixtures/ios/output/ipa_url_bb_release.txt" upload: "maze_output/failed/**/*" docker-compose#v3.7.0: pull: cocoa-maze-runner run: cocoa-maze-runner service-ports: true command: - - "--app=@/app/build/ipa_url_bs_release.txt" - - "--farm=bs" + - "--app=@/app/build/ipa_url_bb_release.txt" + - "--farm=bb" - "--device=IOS_17" - - "--appium-version=1.21.0" - "--a11y-locator" + - "--no-tunnel" + - "--aws-public-ip" - "--fail-fast" - "features/release" - "--exclude=features/release/[a-d].*.feature$" @@ -438,7 +439,7 @@ steps: limit: 2 # PLAT-11155: App hang tests run on BrowserStack (Appium 1.x) for now - - label: ':browserstack: iOS 17 app hang tests' + - label: ':bitbar: iOS 17 app hang tests' depends_on: - cocoa_fixture timeout_in_minutes: 30 @@ -446,16 +447,17 @@ steps: queue: opensource plugins: artifacts#v1.5.0: - download: "features/fixtures/ios/output/ipa_url_bs_release.txt" + download: "features/fixtures/ios/output/ipa_url_bb_release.txt" upload: "maze_output/failed/**/*" docker-compose#v3.7.0: pull: cocoa-maze-runner run: cocoa-maze-runner command: - - "--app=@build/ipa_url_bs_release.txt" - - "--farm=bs" + - "--app=@build/ipa_url_bb_release.txt" + - "--farm=bb" - "--device=IOS_17" - - "--appium-version=1.21.0" + - "--no-tunnel" + - "--aws-public-ip" - "--fail-fast" - "features/app_hangs.feature" test-collector#v1.10.2: @@ -622,7 +624,7 @@ steps: # Debug configration E2E tests # - - label: ':browserstack: iOS 17 debug configuration tests' + - label: ':bitbar: iOS 17 debug configuration tests' depends_on: - cocoa_fixture timeout_in_minutes: 60 @@ -630,15 +632,17 @@ steps: queue: opensource plugins: artifacts#v1.9.3: - download: "features/fixtures/ios/output/ipa_url_bs_debug.txt" + download: "features/fixtures/ios/output/ipa_url_bb_debug.txt" upload: "maze_output/failed/**/*" docker-compose#v4.7.0: pull: cocoa-maze-runner run: cocoa-maze-runner command: - - "--app=@/app/build/ipa_url_bs_debug.txt" - - "--farm=bs" + - "--app=@/app/build/ipa_url_bb_debug.txt" + - "--farm=bb" - "--device=IOS_17" + - "--no-tunnel" + - "--aws-public-ip" - "--fail-fast" - "features/debug" test-collector#v1.10.2: From 3719078d59bf00859d1f8d0f19f07b45eeeb3506 Mon Sep 17 00:00:00 2001 From: Josh Edney Date: Wed, 9 Oct 2024 11:45:28 +0100 Subject: [PATCH 2/9] ios 17 tests on bb --- .buildkite/pipeline.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 2aad7211b..4d2dfea8e 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -376,8 +376,8 @@ steps: download: "features/fixtures/ios/output/ipa_url_bb_release.txt" upload: "maze_output/failed/**/*" docker-compose#v3.7.0: - pull: cocoa-maze-runner - run: cocoa-maze-runner + pull: cocoa-maze-runner-bitbar + run: cocoa-maze-runner-bitbar service-ports: true command: - "--app=@/app/build/ipa_url_bb_release.txt" @@ -395,7 +395,7 @@ steps: format: "junit" branch: "^master|next$$" concurrency: 25 - concurrency_group: 'browserstack-app' + concurrency_group: 'bitbar' concurrency_method: eager retry: automatic: @@ -413,8 +413,8 @@ steps: download: "features/fixtures/ios/output/ipa_url_bb_release.txt" upload: "maze_output/failed/**/*" docker-compose#v3.7.0: - pull: cocoa-maze-runner - run: cocoa-maze-runner + pull: cocoa-maze-runner-bitbar + run: cocoa-maze-runner-bitbar service-ports: true command: - "--app=@/app/build/ipa_url_bb_release.txt" @@ -431,7 +431,7 @@ steps: format: "junit" branch: "^master|next$$" concurrency: 25 - concurrency_group: 'browserstack-app' + concurrency_group: 'bitbar' concurrency_method: eager retry: automatic: @@ -450,8 +450,8 @@ steps: download: "features/fixtures/ios/output/ipa_url_bb_release.txt" upload: "maze_output/failed/**/*" docker-compose#v3.7.0: - pull: cocoa-maze-runner - run: cocoa-maze-runner + pull: cocoa-maze-runner-bitbar + run: cocoa-maze-runner-bitbar command: - "--app=@build/ipa_url_bb_release.txt" - "--farm=bb" @@ -465,7 +465,7 @@ steps: format: "junit" branch: "^master|next$$" concurrency: 5 - concurrency_group: 'browserstack-app' + concurrency_group: 'bitbar' concurrency_method: eager retry: automatic: @@ -635,8 +635,8 @@ steps: download: "features/fixtures/ios/output/ipa_url_bb_debug.txt" upload: "maze_output/failed/**/*" docker-compose#v4.7.0: - pull: cocoa-maze-runner - run: cocoa-maze-runner + pull: cocoa-maze-runner-bitbar + run: cocoa-maze-runner-bitbar command: - "--app=@/app/build/ipa_url_bb_debug.txt" - "--farm=bb" @@ -650,7 +650,7 @@ steps: format: "junit" branch: "^master|next$$" concurrency: 5 - concurrency_group: 'browserstack-app' + concurrency_group: 'bitbar' concurrency_method: eager retry: automatic: From d990821dbadef99b7cac85d727ee63580a35491e Mon Sep 17 00:00:00 2001 From: Josh Edney Date: Wed, 9 Oct 2024 14:00:36 +0100 Subject: [PATCH 3/9] ios 17 tests on bb --- .buildkite/pipeline.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 4d2dfea8e..5901e96b8 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -36,8 +36,6 @@ steps: - features/fixtures/macos/output/macOSTestApp_Release.zip - features/fixtures/macos/output/macOSTestApp_Debug.zip - features/fixtures/ios/output/ipa_url_bb_release.txt - - features/fixtures/ios/output/ipa_url_bb_release.txt - - features/fixtures/ios/output/ipa_url_bb_debug.txt - features/fixtures/ios/output/ipa_url_bb_debug.txt commands: - bundle install @@ -383,7 +381,6 @@ steps: - "--app=@/app/build/ipa_url_bb_release.txt" - "--farm=bb" - "--device=IOS_17" - - "--appium-version=1.21.0" - "--a11y-locator" - "--no-tunnel" - "--aws-public-ip" From 13057033874d677120dc876f345f96563825ea8c Mon Sep 17 00:00:00 2001 From: Josh Edney Date: Wed, 9 Oct 2024 16:19:56 +0100 Subject: [PATCH 4/9] ios 17 tests on bb --- .buildkite/pipeline.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 5901e96b8..a1cf76ad6 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -370,10 +370,10 @@ steps: agents: queue: opensource plugins: - artifacts#v1.5.0: + artifacts#v1.9.3: download: "features/fixtures/ios/output/ipa_url_bb_release.txt" upload: "maze_output/failed/**/*" - docker-compose#v3.7.0: + docker-compose#v4.7.0: pull: cocoa-maze-runner-bitbar run: cocoa-maze-runner-bitbar service-ports: true @@ -381,7 +381,6 @@ steps: - "--app=@/app/build/ipa_url_bb_release.txt" - "--farm=bb" - "--device=IOS_17" - - "--a11y-locator" - "--no-tunnel" - "--aws-public-ip" - "--fail-fast" From fcbcce7a6669e1cfb9b52497a0be8b7ad485343d Mon Sep 17 00:00:00 2001 From: Josh Edney Date: Thu, 10 Oct 2024 09:22:44 +0100 Subject: [PATCH 5/9] ios 17 tests on bb --- .buildkite/pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index a1cf76ad6..fe14c4af6 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -380,6 +380,7 @@ steps: command: - "--app=@/app/build/ipa_url_bb_release.txt" - "--farm=bb" + - "--appium-version=1.22" - "--device=IOS_17" - "--no-tunnel" - "--aws-public-ip" From 35aa45fa6943163e34dbe1182137b3d79a8c1e6a Mon Sep 17 00:00:00 2001 From: Josh Edney Date: Thu, 10 Oct 2024 09:39:11 +0100 Subject: [PATCH 6/9] ios 17 tests on bb --- .buildkite/pipeline.yml | 680 ++++++++++++++++++++-------------------- 1 file changed, 340 insertions(+), 340 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index fe14c4af6..b75dfa399 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -380,11 +380,11 @@ steps: command: - "--app=@/app/build/ipa_url_bb_release.txt" - "--farm=bb" - - "--appium-version=1.22" - "--device=IOS_17" - "--no-tunnel" - "--aws-public-ip" - "--fail-fast" + - "--appium-version=2.0" - "features/release" - "--exclude=features/release/[e-z].*.feature$" test-collector#v1.10.2: @@ -398,342 +398,342 @@ steps: automatic: - exit_status: -1 # Agent was lost limit: 2 - - - label: ':bitbar: iOS 17 E2E tests batch 2' - depends_on: - - cocoa_fixture - timeout_in_minutes: 60 - agents: - queue: opensource - plugins: - artifacts#v1.5.0: - download: "features/fixtures/ios/output/ipa_url_bb_release.txt" - upload: "maze_output/failed/**/*" - docker-compose#v3.7.0: - pull: cocoa-maze-runner-bitbar - run: cocoa-maze-runner-bitbar - service-ports: true - command: - - "--app=@/app/build/ipa_url_bb_release.txt" - - "--farm=bb" - - "--device=IOS_17" - - "--a11y-locator" - - "--no-tunnel" - - "--aws-public-ip" - - "--fail-fast" - - "features/release" - - "--exclude=features/release/[a-d].*.feature$" - test-collector#v1.10.2: - files: "reports/TEST-*.xml" - format: "junit" - branch: "^master|next$$" - concurrency: 25 - concurrency_group: 'bitbar' - concurrency_method: eager - retry: - automatic: - - exit_status: -1 # Agent was lost - limit: 2 - - # PLAT-11155: App hang tests run on BrowserStack (Appium 1.x) for now - - label: ':bitbar: iOS 17 app hang tests' - depends_on: - - cocoa_fixture - timeout_in_minutes: 30 - agents: - queue: opensource - plugins: - artifacts#v1.5.0: - download: "features/fixtures/ios/output/ipa_url_bb_release.txt" - upload: "maze_output/failed/**/*" - docker-compose#v3.7.0: - pull: cocoa-maze-runner-bitbar - run: cocoa-maze-runner-bitbar - command: - - "--app=@build/ipa_url_bb_release.txt" - - "--farm=bb" - - "--device=IOS_17" - - "--no-tunnel" - - "--aws-public-ip" - - "--fail-fast" - - "features/app_hangs.feature" - test-collector#v1.10.2: - files: "reports/TEST-*.xml" - format: "junit" - branch: "^master|next$$" - concurrency: 5 - concurrency_group: 'bitbar' - concurrency_method: eager - retry: - automatic: - - exit_status: -1 # Agent was lost - limit: 2 - - exit_status: 104 # App hang related error - limit: 2 - - ############################################################################## - # - # Basic build E2E tests - Release Configuration - # - - # - # BitBar - # - - - label: ':bitbar: iOS 16 barebone tests' - depends_on: - - cocoa_fixture - timeout_in_minutes: 60 - agents: - queue: opensource - plugins: - artifacts#v1.9.3: - download: "features/fixtures/ios/output/ipa_url_bb_release.txt" - upload: "maze_output/failed/**/*" - docker-compose#v4.7.0: - pull: cocoa-maze-runner-bitbar - run: cocoa-maze-runner-bitbar - service-ports: true - command: - - "--app=@/app/build/ipa_url_bb_release.txt" - - "--farm=bb" - - "--device=IOS_16" - - "--no-tunnel" - - "--aws-public-ip" - - "--fail-fast" - - "features/release/barebone_tests.feature" - test-collector#v1.10.2: - files: "reports/TEST-*.xml" - format: "junit" - branch: "^master|next$$" - concurrency: 25 - concurrency_group: 'bitbar' - concurrency_method: eager - retry: - automatic: - - exit_status: -1 # Agent was lost - limit: 2 - - - label: ':bitbar: iOS 15 barebone tests' - depends_on: - - cocoa_fixture - timeout_in_minutes: 60 - agents: - queue: opensource - plugins: - artifacts#v1.9.3: - download: "features/fixtures/ios/output/ipa_url_bb_release.txt" - upload: "maze_output/failed/**/*" - docker-compose#v4.7.0: - pull: cocoa-maze-runner-bitbar - run: cocoa-maze-runner-bitbar - service-ports: true - command: - - "--app=@/app/build/ipa_url_bb_release.txt" - - "--farm=bb" - - "--device=IOS_15" - - "--no-tunnel" - - "--aws-public-ip" - - "--fail-fast" - - "features/release/barebone_tests.feature" - test-collector#v1.10.2: - files: "reports/TEST-*.xml" - format: "junit" - branch: "^master|next$$" - concurrency: 25 - concurrency_group: 'bitbar' - concurrency_method: eager - retry: - automatic: - - exit_status: -1 # Agent was lost - limit: 2 - - - label: ':bitbar: iOS 14 barebone tests' - depends_on: - - cocoa_fixture - timeout_in_minutes: 60 - agents: - queue: opensource - plugins: - artifacts#v1.9.3: - download: "features/fixtures/ios/output/ipa_url_bb_release.txt" - upload: "maze_output/failed/**/*" - docker-compose#v4.7.0: - pull: cocoa-maze-runner-bitbar - run: cocoa-maze-runner-bitbar - service-ports: true - command: - - "--app=@/app/build/ipa_url_bb_release.txt" - - "--farm=bb" - - "--device=IOS_14" - - "--no-tunnel" - - "--aws-public-ip" - - "--fail-fast" - - "features/release/barebone_tests.feature" - test-collector#v1.10.2: - files: "reports/TEST-*.xml" - format: "junit" - branch: "^master|next$$" - concurrency: 25 - concurrency_group: 'bitbar' - concurrency_method: eager - retry: - automatic: - - exit_status: -1 # Agent was lost - limit: 2 - - - label: ':bitbar: iOS 13 barebone tests' - depends_on: - - cocoa_fixture - timeout_in_minutes: 60 - agents: - queue: opensource - plugins: - artifacts#v1.9.3: - download: "features/fixtures/ios/output/ipa_url_bb_release.txt" - upload: "maze_output/failed/**/*" - docker-compose#v4.7.0: - pull: cocoa-maze-runner-bitbar - run: cocoa-maze-runner-bitbar - service-ports: true - command: - - "--app=@/app/build/ipa_url_bb_release.txt" - - "--farm=bb" - - "--device=IOS_13" - - "--no-tunnel" - - "--aws-public-ip" - - "--fail-fast" - - "features/release/barebone_tests.feature" - test-collector#v1.10.2: - files: "reports/TEST-*.xml" - format: "junit" - branch: "^master|next$$" - concurrency: 25 - concurrency_group: 'bitbar' - concurrency_method: eager - retry: - automatic: - - exit_status: -1 # Agent was lost - limit: 2 - - ############################################################################## - # - # Debug configration E2E tests - # - - - label: ':bitbar: iOS 17 debug configuration tests' - depends_on: - - cocoa_fixture - timeout_in_minutes: 60 - agents: - queue: opensource - plugins: - artifacts#v1.9.3: - download: "features/fixtures/ios/output/ipa_url_bb_debug.txt" - upload: "maze_output/failed/**/*" - docker-compose#v4.7.0: - pull: cocoa-maze-runner-bitbar - run: cocoa-maze-runner-bitbar - command: - - "--app=@/app/build/ipa_url_bb_debug.txt" - - "--farm=bb" - - "--device=IOS_17" - - "--no-tunnel" - - "--aws-public-ip" - - "--fail-fast" - - "features/debug" - test-collector#v1.10.2: - files: "reports/TEST-*.xml" - format: "junit" - branch: "^master|next$$" - concurrency: 5 - concurrency_group: 'bitbar' - concurrency_method: eager - retry: - automatic: - - exit_status: -1 # Agent was lost - limit: 2 - - - label: ':bitbar: iOS 13 debug configuration tests' - depends_on: - - cocoa_fixture - timeout_in_minutes: 60 - agents: - queue: opensource - plugins: - artifacts#v1.9.3: - download: "features/fixtures/ios/output/ipa_url_bb_debug.txt" - upload: "maze_output/failed/**/*" - docker-compose#v4.7.0: - pull: cocoa-maze-runner-bitbar - run: cocoa-maze-runner-bitbar - service-ports: true - command: - - "--app=@/app/build/ipa_url_bb_debug.txt" - - "--farm=bb" - - "--device=IOS_13" - - "--no-tunnel" - - "--aws-public-ip" - - "--fail-fast" - - "features/debug" - test-collector#v1.10.2: - files: "reports/TEST-*.xml" - format: "junit" - branch: "^master|next$$" - concurrency: 25 - concurrency_group: 'bitbar' - concurrency_method: eager - retry: - automatic: - - exit_status: -1 # Agent was lost - limit: 2 - - - label: 'macOS 14 debug E2E tests' - depends_on: - - cocoa_fixture - timeout_in_minutes: 10 - agents: - queue: macos-14 - plugins: - artifacts#v1.5.0: - download: "features/fixtures/macos/output/macOSTestApp_Debug.zip" - upload: - - "macOSTestApp.log" - - "maze_output/failed/**/*" - test-collector#v1.10.2: - files: "reports/TEST-*.xml" - format: "junit" - branch: "^master|next$$" - commands: - - bundle install - - bundle exec maze-runner - features/debug - --os=macos - --fail-fast - - - label: 'macOS 10.13 debug E2E tests' - depends_on: - - cocoa_fixture - timeout_in_minutes: 10 - agents: - queue: opensource-mac-cocoa-10.13 - plugins: - artifacts#v1.5.0: - download: "features/fixtures/macos/output/macOSTestApp_Debug.zip" - upload: - - "macOSTestApp.log" - - "maze_output/failed/**/*" - commands: - - bundle install - - bundle exec maze-runner - features/debug - --os=macos - --fail-fast - - ############################################################################## - # - # Trigger more tests - # - - - label: 'Conditionally trigger full set of tests' - command: sh -c .buildkite/pipeline_trigger.sh +# +# - label: ':bitbar: iOS 17 E2E tests batch 2' +# depends_on: +# - cocoa_fixture +# timeout_in_minutes: 60 +# agents: +# queue: opensource +# plugins: +# artifacts#v1.5.0: +# download: "features/fixtures/ios/output/ipa_url_bb_release.txt" +# upload: "maze_output/failed/**/*" +# docker-compose#v3.7.0: +# pull: cocoa-maze-runner-bitbar +# run: cocoa-maze-runner-bitbar +# service-ports: true +# command: +# - "--app=@/app/build/ipa_url_bb_release.txt" +# - "--farm=bb" +# - "--device=IOS_17" +# - "--a11y-locator" +# - "--no-tunnel" +# - "--aws-public-ip" +# - "--fail-fast" +# - "features/release" +# - "--exclude=features/release/[a-d].*.feature$" +# test-collector#v1.10.2: +# files: "reports/TEST-*.xml" +# format: "junit" +# branch: "^master|next$$" +# concurrency: 25 +# concurrency_group: 'bitbar' +# concurrency_method: eager +# retry: +# automatic: +# - exit_status: -1 # Agent was lost +# limit: 2 +# +# # PLAT-11155: App hang tests run on BrowserStack (Appium 1.x) for now +# - label: ':bitbar: iOS 17 app hang tests' +# depends_on: +# - cocoa_fixture +# timeout_in_minutes: 30 +# agents: +# queue: opensource +# plugins: +# artifacts#v1.5.0: +# download: "features/fixtures/ios/output/ipa_url_bb_release.txt" +# upload: "maze_output/failed/**/*" +# docker-compose#v3.7.0: +# pull: cocoa-maze-runner-bitbar +# run: cocoa-maze-runner-bitbar +# command: +# - "--app=@build/ipa_url_bb_release.txt" +# - "--farm=bb" +# - "--device=IOS_17" +# - "--no-tunnel" +# - "--aws-public-ip" +# - "--fail-fast" +# - "features/app_hangs.feature" +# test-collector#v1.10.2: +# files: "reports/TEST-*.xml" +# format: "junit" +# branch: "^master|next$$" +# concurrency: 5 +# concurrency_group: 'bitbar' +# concurrency_method: eager +# retry: +# automatic: +# - exit_status: -1 # Agent was lost +# limit: 2 +# - exit_status: 104 # App hang related error +# limit: 2 +# +# ############################################################################## +# # +# # Basic build E2E tests - Release Configuration +# # +# +# # +# # BitBar +# # +# +# - label: ':bitbar: iOS 16 barebone tests' +# depends_on: +# - cocoa_fixture +# timeout_in_minutes: 60 +# agents: +# queue: opensource +# plugins: +# artifacts#v1.9.3: +# download: "features/fixtures/ios/output/ipa_url_bb_release.txt" +# upload: "maze_output/failed/**/*" +# docker-compose#v4.7.0: +# pull: cocoa-maze-runner-bitbar +# run: cocoa-maze-runner-bitbar +# service-ports: true +# command: +# - "--app=@/app/build/ipa_url_bb_release.txt" +# - "--farm=bb" +# - "--device=IOS_16" +# - "--no-tunnel" +# - "--aws-public-ip" +# - "--fail-fast" +# - "features/release/barebone_tests.feature" +# test-collector#v1.10.2: +# files: "reports/TEST-*.xml" +# format: "junit" +# branch: "^master|next$$" +# concurrency: 25 +# concurrency_group: 'bitbar' +# concurrency_method: eager +# retry: +# automatic: +# - exit_status: -1 # Agent was lost +# limit: 2 +# +# - label: ':bitbar: iOS 15 barebone tests' +# depends_on: +# - cocoa_fixture +# timeout_in_minutes: 60 +# agents: +# queue: opensource +# plugins: +# artifacts#v1.9.3: +# download: "features/fixtures/ios/output/ipa_url_bb_release.txt" +# upload: "maze_output/failed/**/*" +# docker-compose#v4.7.0: +# pull: cocoa-maze-runner-bitbar +# run: cocoa-maze-runner-bitbar +# service-ports: true +# command: +# - "--app=@/app/build/ipa_url_bb_release.txt" +# - "--farm=bb" +# - "--device=IOS_15" +# - "--no-tunnel" +# - "--aws-public-ip" +# - "--fail-fast" +# - "features/release/barebone_tests.feature" +# test-collector#v1.10.2: +# files: "reports/TEST-*.xml" +# format: "junit" +# branch: "^master|next$$" +# concurrency: 25 +# concurrency_group: 'bitbar' +# concurrency_method: eager +# retry: +# automatic: +# - exit_status: -1 # Agent was lost +# limit: 2 +# +# - label: ':bitbar: iOS 14 barebone tests' +# depends_on: +# - cocoa_fixture +# timeout_in_minutes: 60 +# agents: +# queue: opensource +# plugins: +# artifacts#v1.9.3: +# download: "features/fixtures/ios/output/ipa_url_bb_release.txt" +# upload: "maze_output/failed/**/*" +# docker-compose#v4.7.0: +# pull: cocoa-maze-runner-bitbar +# run: cocoa-maze-runner-bitbar +# service-ports: true +# command: +# - "--app=@/app/build/ipa_url_bb_release.txt" +# - "--farm=bb" +# - "--device=IOS_14" +# - "--no-tunnel" +# - "--aws-public-ip" +# - "--fail-fast" +# - "features/release/barebone_tests.feature" +# test-collector#v1.10.2: +# files: "reports/TEST-*.xml" +# format: "junit" +# branch: "^master|next$$" +# concurrency: 25 +# concurrency_group: 'bitbar' +# concurrency_method: eager +# retry: +# automatic: +# - exit_status: -1 # Agent was lost +# limit: 2 +# +# - label: ':bitbar: iOS 13 barebone tests' +# depends_on: +# - cocoa_fixture +# timeout_in_minutes: 60 +# agents: +# queue: opensource +# plugins: +# artifacts#v1.9.3: +# download: "features/fixtures/ios/output/ipa_url_bb_release.txt" +# upload: "maze_output/failed/**/*" +# docker-compose#v4.7.0: +# pull: cocoa-maze-runner-bitbar +# run: cocoa-maze-runner-bitbar +# service-ports: true +# command: +# - "--app=@/app/build/ipa_url_bb_release.txt" +# - "--farm=bb" +# - "--device=IOS_13" +# - "--no-tunnel" +# - "--aws-public-ip" +# - "--fail-fast" +# - "features/release/barebone_tests.feature" +# test-collector#v1.10.2: +# files: "reports/TEST-*.xml" +# format: "junit" +# branch: "^master|next$$" +# concurrency: 25 +# concurrency_group: 'bitbar' +# concurrency_method: eager +# retry: +# automatic: +# - exit_status: -1 # Agent was lost +# limit: 2 +# +# ############################################################################## +# # +# # Debug configration E2E tests +# # +# +# - label: ':bitbar: iOS 17 debug configuration tests' +# depends_on: +# - cocoa_fixture +# timeout_in_minutes: 60 +# agents: +# queue: opensource +# plugins: +# artifacts#v1.9.3: +# download: "features/fixtures/ios/output/ipa_url_bb_debug.txt" +# upload: "maze_output/failed/**/*" +# docker-compose#v4.7.0: +# pull: cocoa-maze-runner-bitbar +# run: cocoa-maze-runner-bitbar +# command: +# - "--app=@/app/build/ipa_url_bb_debug.txt" +# - "--farm=bb" +# - "--device=IOS_17" +# - "--no-tunnel" +# - "--aws-public-ip" +# - "--fail-fast" +# - "features/debug" +# test-collector#v1.10.2: +# files: "reports/TEST-*.xml" +# format: "junit" +# branch: "^master|next$$" +# concurrency: 5 +# concurrency_group: 'bitbar' +# concurrency_method: eager +# retry: +# automatic: +# - exit_status: -1 # Agent was lost +# limit: 2 +# +# - label: ':bitbar: iOS 13 debug configuration tests' +# depends_on: +# - cocoa_fixture +# timeout_in_minutes: 60 +# agents: +# queue: opensource +# plugins: +# artifacts#v1.9.3: +# download: "features/fixtures/ios/output/ipa_url_bb_debug.txt" +# upload: "maze_output/failed/**/*" +# docker-compose#v4.7.0: +# pull: cocoa-maze-runner-bitbar +# run: cocoa-maze-runner-bitbar +# service-ports: true +# command: +# - "--app=@/app/build/ipa_url_bb_debug.txt" +# - "--farm=bb" +# - "--device=IOS_13" +# - "--no-tunnel" +# - "--aws-public-ip" +# - "--fail-fast" +# - "features/debug" +# test-collector#v1.10.2: +# files: "reports/TEST-*.xml" +# format: "junit" +# branch: "^master|next$$" +# concurrency: 25 +# concurrency_group: 'bitbar' +# concurrency_method: eager +# retry: +# automatic: +# - exit_status: -1 # Agent was lost +# limit: 2 +# +# - label: 'macOS 14 debug E2E tests' +# depends_on: +# - cocoa_fixture +# timeout_in_minutes: 10 +# agents: +# queue: macos-14 +# plugins: +# artifacts#v1.5.0: +# download: "features/fixtures/macos/output/macOSTestApp_Debug.zip" +# upload: +# - "macOSTestApp.log" +# - "maze_output/failed/**/*" +# test-collector#v1.10.2: +# files: "reports/TEST-*.xml" +# format: "junit" +# branch: "^master|next$$" +# commands: +# - bundle install +# - bundle exec maze-runner +# features/debug +# --os=macos +# --fail-fast +# +# - label: 'macOS 10.13 debug E2E tests' +# depends_on: +# - cocoa_fixture +# timeout_in_minutes: 10 +# agents: +# queue: opensource-mac-cocoa-10.13 +# plugins: +# artifacts#v1.5.0: +# download: "features/fixtures/macos/output/macOSTestApp_Debug.zip" +# upload: +# - "macOSTestApp.log" +# - "maze_output/failed/**/*" +# commands: +# - bundle install +# - bundle exec maze-runner +# features/debug +# --os=macos +# --fail-fast +# +# ############################################################################## +# # +# # Trigger more tests +# # +# +# - label: 'Conditionally trigger full set of tests' +# command: sh -c .buildkite/pipeline_trigger.sh From f93af5e37bf79974c8ac3c8bba03b3e8e95315ad Mon Sep 17 00:00:00 2001 From: Josh Edney Date: Fri, 11 Oct 2024 09:18:41 +0100 Subject: [PATCH 7/9] adjust options for ios_17 bb tests --- .buildkite/pipeline.yml | 679 ++++++++++++++++++++-------------------- 1 file changed, 339 insertions(+), 340 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index b75dfa399..a1cf76ad6 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -384,7 +384,6 @@ steps: - "--no-tunnel" - "--aws-public-ip" - "--fail-fast" - - "--appium-version=2.0" - "features/release" - "--exclude=features/release/[e-z].*.feature$" test-collector#v1.10.2: @@ -398,342 +397,342 @@ steps: automatic: - exit_status: -1 # Agent was lost limit: 2 -# -# - label: ':bitbar: iOS 17 E2E tests batch 2' -# depends_on: -# - cocoa_fixture -# timeout_in_minutes: 60 -# agents: -# queue: opensource -# plugins: -# artifacts#v1.5.0: -# download: "features/fixtures/ios/output/ipa_url_bb_release.txt" -# upload: "maze_output/failed/**/*" -# docker-compose#v3.7.0: -# pull: cocoa-maze-runner-bitbar -# run: cocoa-maze-runner-bitbar -# service-ports: true -# command: -# - "--app=@/app/build/ipa_url_bb_release.txt" -# - "--farm=bb" -# - "--device=IOS_17" -# - "--a11y-locator" -# - "--no-tunnel" -# - "--aws-public-ip" -# - "--fail-fast" -# - "features/release" -# - "--exclude=features/release/[a-d].*.feature$" -# test-collector#v1.10.2: -# files: "reports/TEST-*.xml" -# format: "junit" -# branch: "^master|next$$" -# concurrency: 25 -# concurrency_group: 'bitbar' -# concurrency_method: eager -# retry: -# automatic: -# - exit_status: -1 # Agent was lost -# limit: 2 -# -# # PLAT-11155: App hang tests run on BrowserStack (Appium 1.x) for now -# - label: ':bitbar: iOS 17 app hang tests' -# depends_on: -# - cocoa_fixture -# timeout_in_minutes: 30 -# agents: -# queue: opensource -# plugins: -# artifacts#v1.5.0: -# download: "features/fixtures/ios/output/ipa_url_bb_release.txt" -# upload: "maze_output/failed/**/*" -# docker-compose#v3.7.0: -# pull: cocoa-maze-runner-bitbar -# run: cocoa-maze-runner-bitbar -# command: -# - "--app=@build/ipa_url_bb_release.txt" -# - "--farm=bb" -# - "--device=IOS_17" -# - "--no-tunnel" -# - "--aws-public-ip" -# - "--fail-fast" -# - "features/app_hangs.feature" -# test-collector#v1.10.2: -# files: "reports/TEST-*.xml" -# format: "junit" -# branch: "^master|next$$" -# concurrency: 5 -# concurrency_group: 'bitbar' -# concurrency_method: eager -# retry: -# automatic: -# - exit_status: -1 # Agent was lost -# limit: 2 -# - exit_status: 104 # App hang related error -# limit: 2 -# -# ############################################################################## -# # -# # Basic build E2E tests - Release Configuration -# # -# -# # -# # BitBar -# # -# -# - label: ':bitbar: iOS 16 barebone tests' -# depends_on: -# - cocoa_fixture -# timeout_in_minutes: 60 -# agents: -# queue: opensource -# plugins: -# artifacts#v1.9.3: -# download: "features/fixtures/ios/output/ipa_url_bb_release.txt" -# upload: "maze_output/failed/**/*" -# docker-compose#v4.7.0: -# pull: cocoa-maze-runner-bitbar -# run: cocoa-maze-runner-bitbar -# service-ports: true -# command: -# - "--app=@/app/build/ipa_url_bb_release.txt" -# - "--farm=bb" -# - "--device=IOS_16" -# - "--no-tunnel" -# - "--aws-public-ip" -# - "--fail-fast" -# - "features/release/barebone_tests.feature" -# test-collector#v1.10.2: -# files: "reports/TEST-*.xml" -# format: "junit" -# branch: "^master|next$$" -# concurrency: 25 -# concurrency_group: 'bitbar' -# concurrency_method: eager -# retry: -# automatic: -# - exit_status: -1 # Agent was lost -# limit: 2 -# -# - label: ':bitbar: iOS 15 barebone tests' -# depends_on: -# - cocoa_fixture -# timeout_in_minutes: 60 -# agents: -# queue: opensource -# plugins: -# artifacts#v1.9.3: -# download: "features/fixtures/ios/output/ipa_url_bb_release.txt" -# upload: "maze_output/failed/**/*" -# docker-compose#v4.7.0: -# pull: cocoa-maze-runner-bitbar -# run: cocoa-maze-runner-bitbar -# service-ports: true -# command: -# - "--app=@/app/build/ipa_url_bb_release.txt" -# - "--farm=bb" -# - "--device=IOS_15" -# - "--no-tunnel" -# - "--aws-public-ip" -# - "--fail-fast" -# - "features/release/barebone_tests.feature" -# test-collector#v1.10.2: -# files: "reports/TEST-*.xml" -# format: "junit" -# branch: "^master|next$$" -# concurrency: 25 -# concurrency_group: 'bitbar' -# concurrency_method: eager -# retry: -# automatic: -# - exit_status: -1 # Agent was lost -# limit: 2 -# -# - label: ':bitbar: iOS 14 barebone tests' -# depends_on: -# - cocoa_fixture -# timeout_in_minutes: 60 -# agents: -# queue: opensource -# plugins: -# artifacts#v1.9.3: -# download: "features/fixtures/ios/output/ipa_url_bb_release.txt" -# upload: "maze_output/failed/**/*" -# docker-compose#v4.7.0: -# pull: cocoa-maze-runner-bitbar -# run: cocoa-maze-runner-bitbar -# service-ports: true -# command: -# - "--app=@/app/build/ipa_url_bb_release.txt" -# - "--farm=bb" -# - "--device=IOS_14" -# - "--no-tunnel" -# - "--aws-public-ip" -# - "--fail-fast" -# - "features/release/barebone_tests.feature" -# test-collector#v1.10.2: -# files: "reports/TEST-*.xml" -# format: "junit" -# branch: "^master|next$$" -# concurrency: 25 -# concurrency_group: 'bitbar' -# concurrency_method: eager -# retry: -# automatic: -# - exit_status: -1 # Agent was lost -# limit: 2 -# -# - label: ':bitbar: iOS 13 barebone tests' -# depends_on: -# - cocoa_fixture -# timeout_in_minutes: 60 -# agents: -# queue: opensource -# plugins: -# artifacts#v1.9.3: -# download: "features/fixtures/ios/output/ipa_url_bb_release.txt" -# upload: "maze_output/failed/**/*" -# docker-compose#v4.7.0: -# pull: cocoa-maze-runner-bitbar -# run: cocoa-maze-runner-bitbar -# service-ports: true -# command: -# - "--app=@/app/build/ipa_url_bb_release.txt" -# - "--farm=bb" -# - "--device=IOS_13" -# - "--no-tunnel" -# - "--aws-public-ip" -# - "--fail-fast" -# - "features/release/barebone_tests.feature" -# test-collector#v1.10.2: -# files: "reports/TEST-*.xml" -# format: "junit" -# branch: "^master|next$$" -# concurrency: 25 -# concurrency_group: 'bitbar' -# concurrency_method: eager -# retry: -# automatic: -# - exit_status: -1 # Agent was lost -# limit: 2 -# -# ############################################################################## -# # -# # Debug configration E2E tests -# # -# -# - label: ':bitbar: iOS 17 debug configuration tests' -# depends_on: -# - cocoa_fixture -# timeout_in_minutes: 60 -# agents: -# queue: opensource -# plugins: -# artifacts#v1.9.3: -# download: "features/fixtures/ios/output/ipa_url_bb_debug.txt" -# upload: "maze_output/failed/**/*" -# docker-compose#v4.7.0: -# pull: cocoa-maze-runner-bitbar -# run: cocoa-maze-runner-bitbar -# command: -# - "--app=@/app/build/ipa_url_bb_debug.txt" -# - "--farm=bb" -# - "--device=IOS_17" -# - "--no-tunnel" -# - "--aws-public-ip" -# - "--fail-fast" -# - "features/debug" -# test-collector#v1.10.2: -# files: "reports/TEST-*.xml" -# format: "junit" -# branch: "^master|next$$" -# concurrency: 5 -# concurrency_group: 'bitbar' -# concurrency_method: eager -# retry: -# automatic: -# - exit_status: -1 # Agent was lost -# limit: 2 -# -# - label: ':bitbar: iOS 13 debug configuration tests' -# depends_on: -# - cocoa_fixture -# timeout_in_minutes: 60 -# agents: -# queue: opensource -# plugins: -# artifacts#v1.9.3: -# download: "features/fixtures/ios/output/ipa_url_bb_debug.txt" -# upload: "maze_output/failed/**/*" -# docker-compose#v4.7.0: -# pull: cocoa-maze-runner-bitbar -# run: cocoa-maze-runner-bitbar -# service-ports: true -# command: -# - "--app=@/app/build/ipa_url_bb_debug.txt" -# - "--farm=bb" -# - "--device=IOS_13" -# - "--no-tunnel" -# - "--aws-public-ip" -# - "--fail-fast" -# - "features/debug" -# test-collector#v1.10.2: -# files: "reports/TEST-*.xml" -# format: "junit" -# branch: "^master|next$$" -# concurrency: 25 -# concurrency_group: 'bitbar' -# concurrency_method: eager -# retry: -# automatic: -# - exit_status: -1 # Agent was lost -# limit: 2 -# -# - label: 'macOS 14 debug E2E tests' -# depends_on: -# - cocoa_fixture -# timeout_in_minutes: 10 -# agents: -# queue: macos-14 -# plugins: -# artifacts#v1.5.0: -# download: "features/fixtures/macos/output/macOSTestApp_Debug.zip" -# upload: -# - "macOSTestApp.log" -# - "maze_output/failed/**/*" -# test-collector#v1.10.2: -# files: "reports/TEST-*.xml" -# format: "junit" -# branch: "^master|next$$" -# commands: -# - bundle install -# - bundle exec maze-runner -# features/debug -# --os=macos -# --fail-fast -# -# - label: 'macOS 10.13 debug E2E tests' -# depends_on: -# - cocoa_fixture -# timeout_in_minutes: 10 -# agents: -# queue: opensource-mac-cocoa-10.13 -# plugins: -# artifacts#v1.5.0: -# download: "features/fixtures/macos/output/macOSTestApp_Debug.zip" -# upload: -# - "macOSTestApp.log" -# - "maze_output/failed/**/*" -# commands: -# - bundle install -# - bundle exec maze-runner -# features/debug -# --os=macos -# --fail-fast -# -# ############################################################################## -# # -# # Trigger more tests -# # -# -# - label: 'Conditionally trigger full set of tests' -# command: sh -c .buildkite/pipeline_trigger.sh + + - label: ':bitbar: iOS 17 E2E tests batch 2' + depends_on: + - cocoa_fixture + timeout_in_minutes: 60 + agents: + queue: opensource + plugins: + artifacts#v1.5.0: + download: "features/fixtures/ios/output/ipa_url_bb_release.txt" + upload: "maze_output/failed/**/*" + docker-compose#v3.7.0: + pull: cocoa-maze-runner-bitbar + run: cocoa-maze-runner-bitbar + service-ports: true + command: + - "--app=@/app/build/ipa_url_bb_release.txt" + - "--farm=bb" + - "--device=IOS_17" + - "--a11y-locator" + - "--no-tunnel" + - "--aws-public-ip" + - "--fail-fast" + - "features/release" + - "--exclude=features/release/[a-d].*.feature$" + test-collector#v1.10.2: + files: "reports/TEST-*.xml" + format: "junit" + branch: "^master|next$$" + concurrency: 25 + concurrency_group: 'bitbar' + concurrency_method: eager + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + + # PLAT-11155: App hang tests run on BrowserStack (Appium 1.x) for now + - label: ':bitbar: iOS 17 app hang tests' + depends_on: + - cocoa_fixture + timeout_in_minutes: 30 + agents: + queue: opensource + plugins: + artifacts#v1.5.0: + download: "features/fixtures/ios/output/ipa_url_bb_release.txt" + upload: "maze_output/failed/**/*" + docker-compose#v3.7.0: + pull: cocoa-maze-runner-bitbar + run: cocoa-maze-runner-bitbar + command: + - "--app=@build/ipa_url_bb_release.txt" + - "--farm=bb" + - "--device=IOS_17" + - "--no-tunnel" + - "--aws-public-ip" + - "--fail-fast" + - "features/app_hangs.feature" + test-collector#v1.10.2: + files: "reports/TEST-*.xml" + format: "junit" + branch: "^master|next$$" + concurrency: 5 + concurrency_group: 'bitbar' + concurrency_method: eager + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + - exit_status: 104 # App hang related error + limit: 2 + + ############################################################################## + # + # Basic build E2E tests - Release Configuration + # + + # + # BitBar + # + + - label: ':bitbar: iOS 16 barebone tests' + depends_on: + - cocoa_fixture + timeout_in_minutes: 60 + agents: + queue: opensource + plugins: + artifacts#v1.9.3: + download: "features/fixtures/ios/output/ipa_url_bb_release.txt" + upload: "maze_output/failed/**/*" + docker-compose#v4.7.0: + pull: cocoa-maze-runner-bitbar + run: cocoa-maze-runner-bitbar + service-ports: true + command: + - "--app=@/app/build/ipa_url_bb_release.txt" + - "--farm=bb" + - "--device=IOS_16" + - "--no-tunnel" + - "--aws-public-ip" + - "--fail-fast" + - "features/release/barebone_tests.feature" + test-collector#v1.10.2: + files: "reports/TEST-*.xml" + format: "junit" + branch: "^master|next$$" + concurrency: 25 + concurrency_group: 'bitbar' + concurrency_method: eager + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + + - label: ':bitbar: iOS 15 barebone tests' + depends_on: + - cocoa_fixture + timeout_in_minutes: 60 + agents: + queue: opensource + plugins: + artifacts#v1.9.3: + download: "features/fixtures/ios/output/ipa_url_bb_release.txt" + upload: "maze_output/failed/**/*" + docker-compose#v4.7.0: + pull: cocoa-maze-runner-bitbar + run: cocoa-maze-runner-bitbar + service-ports: true + command: + - "--app=@/app/build/ipa_url_bb_release.txt" + - "--farm=bb" + - "--device=IOS_15" + - "--no-tunnel" + - "--aws-public-ip" + - "--fail-fast" + - "features/release/barebone_tests.feature" + test-collector#v1.10.2: + files: "reports/TEST-*.xml" + format: "junit" + branch: "^master|next$$" + concurrency: 25 + concurrency_group: 'bitbar' + concurrency_method: eager + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + + - label: ':bitbar: iOS 14 barebone tests' + depends_on: + - cocoa_fixture + timeout_in_minutes: 60 + agents: + queue: opensource + plugins: + artifacts#v1.9.3: + download: "features/fixtures/ios/output/ipa_url_bb_release.txt" + upload: "maze_output/failed/**/*" + docker-compose#v4.7.0: + pull: cocoa-maze-runner-bitbar + run: cocoa-maze-runner-bitbar + service-ports: true + command: + - "--app=@/app/build/ipa_url_bb_release.txt" + - "--farm=bb" + - "--device=IOS_14" + - "--no-tunnel" + - "--aws-public-ip" + - "--fail-fast" + - "features/release/barebone_tests.feature" + test-collector#v1.10.2: + files: "reports/TEST-*.xml" + format: "junit" + branch: "^master|next$$" + concurrency: 25 + concurrency_group: 'bitbar' + concurrency_method: eager + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + + - label: ':bitbar: iOS 13 barebone tests' + depends_on: + - cocoa_fixture + timeout_in_minutes: 60 + agents: + queue: opensource + plugins: + artifacts#v1.9.3: + download: "features/fixtures/ios/output/ipa_url_bb_release.txt" + upload: "maze_output/failed/**/*" + docker-compose#v4.7.0: + pull: cocoa-maze-runner-bitbar + run: cocoa-maze-runner-bitbar + service-ports: true + command: + - "--app=@/app/build/ipa_url_bb_release.txt" + - "--farm=bb" + - "--device=IOS_13" + - "--no-tunnel" + - "--aws-public-ip" + - "--fail-fast" + - "features/release/barebone_tests.feature" + test-collector#v1.10.2: + files: "reports/TEST-*.xml" + format: "junit" + branch: "^master|next$$" + concurrency: 25 + concurrency_group: 'bitbar' + concurrency_method: eager + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + + ############################################################################## + # + # Debug configration E2E tests + # + + - label: ':bitbar: iOS 17 debug configuration tests' + depends_on: + - cocoa_fixture + timeout_in_minutes: 60 + agents: + queue: opensource + plugins: + artifacts#v1.9.3: + download: "features/fixtures/ios/output/ipa_url_bb_debug.txt" + upload: "maze_output/failed/**/*" + docker-compose#v4.7.0: + pull: cocoa-maze-runner-bitbar + run: cocoa-maze-runner-bitbar + command: + - "--app=@/app/build/ipa_url_bb_debug.txt" + - "--farm=bb" + - "--device=IOS_17" + - "--no-tunnel" + - "--aws-public-ip" + - "--fail-fast" + - "features/debug" + test-collector#v1.10.2: + files: "reports/TEST-*.xml" + format: "junit" + branch: "^master|next$$" + concurrency: 5 + concurrency_group: 'bitbar' + concurrency_method: eager + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + + - label: ':bitbar: iOS 13 debug configuration tests' + depends_on: + - cocoa_fixture + timeout_in_minutes: 60 + agents: + queue: opensource + plugins: + artifacts#v1.9.3: + download: "features/fixtures/ios/output/ipa_url_bb_debug.txt" + upload: "maze_output/failed/**/*" + docker-compose#v4.7.0: + pull: cocoa-maze-runner-bitbar + run: cocoa-maze-runner-bitbar + service-ports: true + command: + - "--app=@/app/build/ipa_url_bb_debug.txt" + - "--farm=bb" + - "--device=IOS_13" + - "--no-tunnel" + - "--aws-public-ip" + - "--fail-fast" + - "features/debug" + test-collector#v1.10.2: + files: "reports/TEST-*.xml" + format: "junit" + branch: "^master|next$$" + concurrency: 25 + concurrency_group: 'bitbar' + concurrency_method: eager + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + + - label: 'macOS 14 debug E2E tests' + depends_on: + - cocoa_fixture + timeout_in_minutes: 10 + agents: + queue: macos-14 + plugins: + artifacts#v1.5.0: + download: "features/fixtures/macos/output/macOSTestApp_Debug.zip" + upload: + - "macOSTestApp.log" + - "maze_output/failed/**/*" + test-collector#v1.10.2: + files: "reports/TEST-*.xml" + format: "junit" + branch: "^master|next$$" + commands: + - bundle install + - bundle exec maze-runner + features/debug + --os=macos + --fail-fast + + - label: 'macOS 10.13 debug E2E tests' + depends_on: + - cocoa_fixture + timeout_in_minutes: 10 + agents: + queue: opensource-mac-cocoa-10.13 + plugins: + artifacts#v1.5.0: + download: "features/fixtures/macos/output/macOSTestApp_Debug.zip" + upload: + - "macOSTestApp.log" + - "maze_output/failed/**/*" + commands: + - bundle install + - bundle exec maze-runner + features/debug + --os=macos + --fail-fast + + ############################################################################## + # + # Trigger more tests + # + + - label: 'Conditionally trigger full set of tests' + command: sh -c .buildkite/pipeline_trigger.sh From 821c65c048845dbb3f265cf92d7b416689dca6dc Mon Sep 17 00:00:00 2001 From: Josh Edney Date: Tue, 15 Oct 2024 17:12:01 +0100 Subject: [PATCH 8/9] update bk docker-compose module --- .buildkite/pipeline.full.yml | 6 +++--- .buildkite/pipeline.yml | 24 ++++++++++++++---------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/.buildkite/pipeline.full.yml b/.buildkite/pipeline.full.yml index 8a196e0d6..b27509d6e 100644 --- a/.buildkite/pipeline.full.yml +++ b/.buildkite/pipeline.full.yml @@ -391,7 +391,7 @@ steps: artifacts#v1.5.0: download: "features/fixtures/ios/output/ipa_url_bs_release.txt" upload: "maze_output/failed/**/*" - docker-compose#v3.7.0: + docker-compose#v4.7.0: pull: cocoa-maze-runner run: cocoa-maze-runner command: @@ -426,7 +426,7 @@ steps: artifacts#v1.5.0: download: "features/fixtures/ios/output/ipa_url_bs_release.txt" upload: "maze_output/failed/**/*" - docker-compose#v3.7.0: + docker-compose#v4.7.0: pull: cocoa-maze-runner run: cocoa-maze-runner command: @@ -459,7 +459,7 @@ steps: artifacts#v1.5.0: download: "features/fixtures/ios/output/ipa_url_bs_release.txt" upload: "maze_output/failed/**/*" - docker-compose#v3.7.0: + docker-compose#v4.7.0: pull: cocoa-maze-runner run: cocoa-maze-runner command: diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index a1cf76ad6..9ac77c1d8 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -35,13 +35,18 @@ steps: - features/fixtures/ios/output/iOSTestApp_Debug.ipa - features/fixtures/macos/output/macOSTestApp_Release.zip - features/fixtures/macos/output/macOSTestApp_Debug.zip + - features/fixtures/ios/output/ipa_url_bb_release.txt + - features/fixtures/ios/output/ipa_url_bs_release.txt - features/fixtures/ios/output/ipa_url_bb_debug.txt + - features/fixtures/ios/output/ipa_url_bs_debug.txt commands: - bundle install - make test-fixtures - bundle exec upload-app --farm=bb --app=./features/fixtures/ios/output/iOSTestApp_Release.ipa --app-id-file=./features/fixtures/ios/output/ipa_url_bb_release.txt + - bundle exec upload-app --farm=bs --app=./features/fixtures/ios/output/iOSTestApp_Release.ipa --app-id-file=./features/fixtures/ios/output/ipa_url_bs_release.txt - bundle exec upload-app --farm=bb --app=./features/fixtures/ios/output/iOSTestApp_Debug.ipa --app-id-file=./features/fixtures/ios/output/ipa_url_bb_debug.txt + - bundle exec upload-app --farm=bs --app=./features/fixtures/ios/output/iOSTestApp_Debug.ipa --app-id-file=./features/fixtures/ios/output/ipa_url_bs_debug.txt - label: Carthage timeout_in_minutes: 15 @@ -408,7 +413,7 @@ steps: artifacts#v1.5.0: download: "features/fixtures/ios/output/ipa_url_bb_release.txt" upload: "maze_output/failed/**/*" - docker-compose#v3.7.0: + docker-compose#v4.7.0: pull: cocoa-maze-runner-bitbar run: cocoa-maze-runner-bitbar service-ports: true @@ -416,7 +421,6 @@ steps: - "--app=@/app/build/ipa_url_bb_release.txt" - "--farm=bb" - "--device=IOS_17" - - "--a11y-locator" - "--no-tunnel" - "--aws-public-ip" - "--fail-fast" @@ -435,7 +439,7 @@ steps: limit: 2 # PLAT-11155: App hang tests run on BrowserStack (Appium 1.x) for now - - label: ':bitbar: iOS 17 app hang tests' + - label: ':browserstack: iOS 17 app hang tests' depends_on: - cocoa_fixture timeout_in_minutes: 30 @@ -445,12 +449,12 @@ steps: artifacts#v1.5.0: download: "features/fixtures/ios/output/ipa_url_bb_release.txt" upload: "maze_output/failed/**/*" - docker-compose#v3.7.0: - pull: cocoa-maze-runner-bitbar - run: cocoa-maze-runner-bitbar + docker-compose#v4.7.0: + pull: cocoa-maze-runner + run: cocoa-maze-runner command: - - "--app=@build/ipa_url_bb_release.txt" - - "--farm=bb" + - "--app=@build/ipa_url_bs_release.txt" + - "--farm=bs" - "--device=IOS_17" - "--no-tunnel" - "--aws-public-ip" @@ -461,7 +465,7 @@ steps: format: "junit" branch: "^master|next$$" concurrency: 5 - concurrency_group: 'bitbar' + concurrency_group: 'browserstack-app' concurrency_method: eager retry: automatic: @@ -645,7 +649,7 @@ steps: files: "reports/TEST-*.xml" format: "junit" branch: "^master|next$$" - concurrency: 5 + concurrency: 25 concurrency_group: 'bitbar' concurrency_method: eager retry: From 9850820d51170eaae91b2c86acd62010572474be Mon Sep 17 00:00:00 2001 From: Josh Edney Date: Tue, 15 Oct 2024 17:20:56 +0100 Subject: [PATCH 9/9] update bs app hang test url file --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 9ac77c1d8..9e9617d5e 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -447,7 +447,7 @@ steps: queue: opensource plugins: artifacts#v1.5.0: - download: "features/fixtures/ios/output/ipa_url_bb_release.txt" + download: "features/fixtures/ios/output/ipa_url_bs_release.txt" upload: "maze_output/failed/**/*" docker-compose#v4.7.0: pull: cocoa-maze-runner