Migrate mobile tests to BrowserStack (#28)

Picovoice · Nov 21, 2024 · 4ff1e27 · 4ff1e27
1 parent 434cb91
commit 4ff1e27
Show file tree

Hide file tree

Showing 16 changed files with 428 additions and 310 deletions.
diff --git a/.github/workflows/android-appcenter.yml → .github/workflows/android-browserstack.yml b/.github/workflows/android-appcenter.yml → .github/workflows/android-browserstack.yml
@@ -1,18 +1,18 @@
-name: Android AppCenter Tests
+name: Android BrowserStack Tests
 
 on:
   workflow_dispatch:
   push:
     branches: [ main ]
     paths:
       - 'binding/android/FalconTestApp/**'
-      - '.github/workflows/android-appcenter.yml'
+      - '.github/workflows/android-browserstack.yml'
       - 'resources/audio_samples/**'
   pull_request:
     branches: [ main, 'v[0-9]+.[0-9]+' ]
     paths:
       - 'binding/android/FalconTestApp/**'
-      - '.github/workflows/android-appcenter.yml'
+      - '.github/workflows/android-browserstack.yml'
       - 'resources/audio_samples/**'
 
 defaults:
@@ -21,19 +21,18 @@ defaults:
 
 jobs:
   build:
-    name: Run Android Tests on AppCenter
+    name: Run Android Tests on BrowserStack
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v3
 
-      - name: Set up Node.js LTS
-        uses: actions/setup-node@v3
+      - name: Installing Python
+        uses: actions/setup-python@v5
         with:
-          node-version: lts/*
-
-      - name: Install AppCenter CLI
-        run: npm install -g appcenter-cli
+          python-version: '3.10'
+      - run:
+          pip3 install requests
 
       - name: set up JDK 11
         uses: actions/setup-java@v3
@@ -66,30 +65,29 @@ jobs:
       - name: Build androidTest
         run: ./gradlew assembleDebugAndroidTest
 
-      - name: Run tests on AppCenter
-        run: appcenter test run espresso
-          --token ${{secrets.APPCENTERAPITOKEN}}
-          --app "Picovoice/Falcon-Android"
-          --devices "Picovoice/android-min-max"
-          --app-path falcon-test-app/build/outputs/apk/debug/falcon-test-app-debug.apk
-          --test-series "falcon-android"
-          --locale "en_US"
-          --build-dir falcon-test-app/build/outputs/apk/androidTest/debug
+      - name: Run tests on BrowserStack
+        run: python3 ../../../script/automation/browserstack.py
+          --type espresso
+          --username "${{secrets.BROWSERSTACK_USERNAME}}"
+          --access_key "${{secrets.BROWSERSTACK_ACCESS_KEY}}"
+          --project_name "Falcon-Android"
+          --devices "android-min-max"
+          --app_path "falcon-test-app/build/outputs/apk/debug/falcon-test-app-debug.apk"
+          --test_path "falcon-test-app/build/outputs/apk/androidTest/debug/falcon-test-app-debug-androidTest.apk"
 
   build-integ:
-    name: Run Android Integration Tests on AppCenter
+    name: Run Android Integration Tests on BrowserStack
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v3
 
-      - name: Set up Node.js LTS
-        uses: actions/setup-node@v3
+      - name: Installing Python
+        uses: actions/setup-python@v5
         with:
-          node-version: lts/*
-
-      - name: Install AppCenter CLI
-        run: npm install -g appcenter-cli
+          python-version: '3.10'
+      - run:
+          pip3 install requests
 
       - name: set up JDK 11
         uses: actions/setup-java@v3
@@ -122,12 +120,12 @@ jobs:
       - name: Build androidTest
         run: ./gradlew assembleReleaseAndroidTest -DtestBuildType=integ
 
-      - name: Run tests on AppCenter
-        run: appcenter test run espresso
-          --token ${{secrets.APPCENTERAPITOKEN}}
-          --app "Picovoice/Falcon-Android"
-          --devices "Picovoice/android-min-max"
-          --app-path falcon-test-app/build/outputs/apk/release/falcon-test-app-release.apk
-          --test-series "falcon-android"
-          --locale "en_US"
-          --build-dir falcon-test-app/build/outputs/apk/androidTest/release
+      - name: Run tests on BrowserStack
+        run: python3 ../../../script/automation/browserstack.py
+          --type espresso
+          --username "${{secrets.BROWSERSTACK_USERNAME}}"
+          --access_key "${{secrets.BROWSERSTACK_ACCESS_KEY}}"
+          --project_name "Falcon-Android-Integration"
+          --devices "android-min-max"
+          --app_path "falcon-test-app/build/outputs/apk/release/falcon-test-app-release.apk"
+          --test_path "falcon-test-app/build/outputs/apk/androidTest/release/falcon-test-app-release-androidTest.apk"
diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml
@@ -21,28 +21,26 @@ defaults:
 
 jobs:
   build:
-    name: Run Android Speed Tests on AppCenter
+    name: Run Android Speed Tests on BrowserStack
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        device: [single-android, 32bit-android]
+        device: [ android-perf ]
         include:
-          - device: single-android
+          - device: android-perf
             initPerformanceThresholdSec: 1.0
-            procPerformanceThresholdSec: 1.0
-          - device: 32bit-android
-            initPerformanceThresholdSec: 1.0
-            procPerformanceThresholdSec: 7.0
+            procPerformanceThresholdSec: 2.0
 
     steps:
       - uses: actions/checkout@v3
 
-      - name: Setup Node.js environment
-        uses: actions/setup-node@v3
-
-      - name: Install AppCenter CLI
-        run: npm install -g appcenter-cli
+      - name: Installing Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - run:
+          pip3 install requests
 
       - name: set up JDK 11
         uses: actions/setup-java@v3
@@ -84,12 +82,12 @@ jobs:
       - name: Build androidTest
         run: ./gradlew assembleDebugAndroidTest -DtestBuildType=perf
 
-      - name: Run tests on AppCenter
-        run: appcenter test run espresso
-          --token ${{secrets.APPCENTERAPITOKEN}}
-          --app "Picovoice/Falcon-Android"
-          --devices "Picovoice/${{ matrix.device }}"
-          --app-path falcon-test-app/build/outputs/apk/debug/falcon-test-app-debug.apk
-          --test-series "falcon-android"
-          --locale "en_US"
-          --build-dir falcon-test-app/build/outputs/apk/androidTest/debug
+      - name: Run tests on BrowserStack
+        run: python3 ../../../script/automation/browserstack.py
+          --type espresso
+          --username "${{secrets.BROWSERSTACK_USERNAME}}"
+          --access_key "${{secrets.BROWSERSTACK_ACCESS_KEY}}"
+          --project_name "Falcon-Android-Performance"
+          --devices "${{ matrix.device }}"
+          --app_path "falcon-test-app/build/outputs/apk/debug/falcon-test-app-debug.apk"
+          --test_path "falcon-test-app/build/outputs/apk/androidTest/debug/falcon-test-app-debug-androidTest.apk"
diff --git a/.github/workflows/ios-appcenter.yml b/.github/workflows/ios-appcenter.yml
diff --git a/.github/workflows/ios-browserstack.yml b/.github/workflows/ios-browserstack.yml
@@ -0,0 +1,82 @@
+name: iOS BrowserStack Tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ main ]
+    paths:
+      - '.github/workflows/ios-browserstack.yml'
+      - 'binding/ios/FalconAppTest/**'
+      - 'resources/audio_samples/**'
+  pull_request:
+    branches: [ main, 'v[0-9]+.[0-9]+' ]
+    paths:
+      - '.github/workflows/ios-browserstack.yml'
+      - 'binding/ios/FalconAppTest/**'
+      - 'resources/audio_samples/**'
+
+defaults:
+  run:
+    working-directory: binding/ios/FalconAppTest
+
+jobs:
+  build:
+    name: Run iOS Tests on BrowserStack
+    runs-on: macos-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Installing Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - run:
+          pip3 install requests
+
+      - name: Install Cocoapods
+        run: gem install cocoapods
+
+      - name: Make build dir
+        run: mkdir ddp
+
+      - name: Copy test_resources
+        run: ./copy_test_resources.sh
+
+      - name: Run Cocoapods
+        run: pod install
+
+      - name: Inject AccessKey
+        run: sed -i '.bak' 's:{TESTING_ACCESS_KEY_HERE}:${{secrets.PV_VALID_ACCESS_KEY}}:'
+          FalconAppTestUITests/FalconAppTestUITests.swift
+
+      - name: XCode Build
+        run: xcrun xcodebuild build-for-testing
+          -configuration Debug
+          -workspace FalconAppTest.xcworkspace
+          -sdk iphoneos
+          -scheme FalconAppTest
+          -derivedDataPath ddp
+          CODE_SIGNING_ALLOWED=NO
+
+      - name: Generating ipa
+        run: cd ddp/Build/Products/Debug-iphoneos/ &&
+          mkdir Payload &&
+          cp -r FalconAppTest.app Payload &&
+          zip --symlinks -r FalconAppTest.ipa Payload &&
+          rm -r Payload
+
+      - name: Zipping Tests
+        run: cd ddp/Build/Products/Debug-iphoneos/ &&
+          zip --symlinks -r FalconAppTestUITests.zip FalconAppTestUITests-Runner.app
+
+      - name: Run tests on BrowserStack
+        run: python3 ../../../script/automation/browserstack.py
+          --type xcuitest
+          --username "${{secrets.BROWSERSTACK_USERNAME}}"
+          --access_key "${{secrets.BROWSERSTACK_ACCESS_KEY}}"
+          --project_name "Falcon-iOS"
+          --devices "ios-min-max"
+          --app_path "ddp/Build/Products/Debug-iphoneos/FalconAppTest.ipa"
+          --test_path "ddp/Build/Products/Debug-iphoneos/FalconAppTestUITests.zip"
diff --git a/.github/workflows/ios-demos.yml b/.github/workflows/ios-demos.yml
@@ -33,9 +33,6 @@ jobs:
       - name: Install Cocoapods
         run: gem install cocoapods
 
-      - name: Install AppCenter CLI
-        run: npm install -g appcenter-cli
-
       - name: Make build dir
         run: mkdir ddp