From daffd870fe1e9c44d8a0f06f33bc6045d84d1835 Mon Sep 17 00:00:00 2001 From: Albert Ho Date: Wed, 20 Nov 2024 16:27:34 -0800 Subject: [PATCH 1/6] replace App Center with BrowserStack --- ...appcenter.yml => android-browserstack.yml} | 68 +++++---- .github/workflows/android-perf.yml | 38 +++-- .github/workflows/ios-appcenter.yml | 73 ---------- .github/workflows/ios-browserstack.yml | 83 +++++++++++ .github/workflows/ios-demos.yml | 3 - .github/workflows/ios-perf.yml | 56 ++++---- .../leopard-test-app/build.gradle | 2 +- .../picovoice/leopard/testapp/BaseTest.java | 13 -- .../leopard/testapp/IntegrationTest.java | 11 -- .../ios/LeopardAppTest/copy_test_resources.sh | 3 - resources/spell-check/dict.txt | 2 - script/automation/browserstack.py | 135 ++++++++++++++++++ 12 files changed, 301 insertions(+), 186 deletions(-) rename .github/workflows/{android-appcenter.yml => android-browserstack.yml} (63%) delete mode 100644 .github/workflows/ios-appcenter.yml create mode 100644 .github/workflows/ios-browserstack.yml create mode 100644 script/automation/browserstack.py diff --git a/.github/workflows/android-appcenter.yml b/.github/workflows/android-browserstack.yml similarity index 63% rename from .github/workflows/android-appcenter.yml rename to .github/workflows/android-browserstack.yml index 258df155..2af1f39e 100644 --- a/.github/workflows/android-appcenter.yml +++ b/.github/workflows/android-browserstack.yml @@ -1,4 +1,4 @@ -name: Android AppCenter Tests +name: Android BrowserStack Tests on: workflow_dispatch: @@ -6,14 +6,14 @@ on: branches: [ master ] paths: - 'binding/android/LeopardTestApp/**' - - '.github/workflows/android-appcenter.yml' + - '.github/workflows/android-browserstack.yml' - 'resources/audio_samples/**' - 'resources/.test/**' pull_request: branches: [ master, 'v[0-9]+.[0-9]+' ] paths: - 'binding/android/LeopardTestApp/**' - - '.github/workflows/android-appcenter.yml' + - '.github/workflows/android-browserstack.yml' - 'resources/audio_samples/**' - 'resources/.test/**' @@ -23,19 +23,18 @@ defaults: jobs: build: - name: Run Android Tests on AppCenter + name: Run Android Tests on BrowserStack runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Node.js LTS - uses: actions/setup-node@v3 + - name: Installing Python + uses: actions/setup-python@v5 with: - node-version: lts/* - - - name: Install AppCenter CLI - run: npm install -g appcenter-cli + python-version: '3.10' + - run: + pip3 install requests - name: set up JDK 11 uses: actions/setup-java@v3 @@ -68,30 +67,29 @@ jobs: - name: Build androidTest run: ./gradlew assembleEnDebugAndroidTest - - name: Run tests on AppCenter - run: appcenter test run espresso - --token ${{secrets.APPCENTERAPITOKEN}} - --app "Picovoice/Leopard-Android" - --devices "Picovoice/android-min-max" - --app-path leopard-test-app/build/outputs/apk/en/debug/leopard-test-app-en-debug.apk - --test-series "leopard-android" - --locale "en_US" - --build-dir leopard-test-app/build/outputs/apk/androidTest/en/debug + - name: Run tests on BrowserStack + run: python3 ../../../script/automation/browserstack.py + --type espresso + --username "${{secrets.BROWSERSTACK_USERNAME}}" + --access_key "${{secrets.BROWSERSTACK_ACCESS_KEY}}" + --project_name "Leopard-Android" + --devices "android-min-max" + --app_path "leopard-test-app/build/outputs/apk/en/debug/leopard-test-app-en-debug.apk" + --test_path "leopard-test-app/build/outputs/apk/en/androidTest/debug/leopard-test-app-en-debug-androidTest.apk" build-integ: - name: Run Android Integration Tests on AppCenter + name: Run Android Integration Tests on BrowserStack runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Node.js LTS - uses: actions/setup-node@v3 + - name: Installing Python + uses: actions/setup-python@v5 with: - node-version: lts/* - - - name: Install AppCenter CLI - run: npm install -g appcenter-cli + python-version: '3.10' + - run: + pip3 install requests - name: set up JDK 11 uses: actions/setup-java@v3 @@ -124,13 +122,13 @@ jobs: - name: Build androidTest run: ./gradlew assembleEnReleaseAndroidTest -DtestBuildType=integ - - name: Run tests on AppCenter - run: appcenter test run espresso - --token ${{secrets.APPCENTERAPITOKEN}} - --app "Picovoice/Leopard-Android" - --devices "Picovoice/android-min-max" - --app-path leopard-test-app/build/outputs/apk/en/release/leopard-test-app-en-release.apk - --test-series "leopard-android" - --locale "en_US" - --build-dir leopard-test-app/build/outputs/apk/androidTest/en/release + - name: Run tests on BrowserStack + run: python3 ../../../script/automation/browserstack.py + --type espresso + --username "${{secrets.BROWSERSTACK_USERNAME}}" + --access_key "${{secrets.BROWSERSTACK_ACCESS_KEY}}" + --project_name "Leopard-Android-Integration" + --devices "android-min-max" + --app_path "leopard-test-app/build/outputs/apk/en/release/leopard-test-app-en-release.apk" + --test_path "leopard-test-app/build/outputs/apk/androidTest/en/release/leopard-test-app-en-release-androidTest.apk" diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml index f0cc6ca8..c822769e 100644 --- a/.github/workflows/android-perf.yml +++ b/.github/workflows/android-perf.yml @@ -21,30 +21,26 @@ defaults: jobs: build: - name: Run Android Speed Tests on AppCenter + name: Run Android Speed Tests on BrowserStack runs-on: ubuntu-latest strategy: matrix: - device: [single-android, 32bit-android] + device: [ android-perf ] include: - - device: single-android + - device: android-perf initPerformanceThresholdSec: 6.0 procPerformanceThresholdSec: 1.0 - - device: 32bit-android - initPerformanceThresholdSec: 18.0 - procPerformanceThresholdSec: 7.0 steps: - uses: actions/checkout@v3 - - name: Set up Node.js LTS - uses: actions/setup-node@v3 + - name: Installing Python + uses: actions/setup-python@v5 with: - node-version: lts/* - - - name: Install AppCenter CLI - run: npm install -g appcenter-cli + python-version: '3.10' + - run: + pip3 install requests - name: set up JDK 11 uses: actions/setup-java@v3 @@ -86,12 +82,12 @@ jobs: - name: Build androidTest run: ./gradlew assembleEnDebugAndroidTest -DtestBuildType=perf - - name: Run tests on AppCenter - run: appcenter test run espresso - --token ${{secrets.APPCENTERAPITOKEN}} - --app "Picovoice/Leopard-Android" - --devices "Picovoice/${{ matrix.device }}" - --app-path leopard-test-app/build/outputs/apk/en/debug/leopard-test-app-en-debug.apk - --test-series "leopard-android" - --locale "en_US" - --build-dir leopard-test-app/build/outputs/apk/androidTest/en/debug + - name: Run tests on BrowserStack + run: python3 ../../../script/automation/browserstack.py + --type espresso + --username "${{secrets.BROWSERSTACK_USERNAME}}" + --access_key "${{secrets.BROWSERSTACK_ACCESS_KEY}}" + --project_name "Leopard-Android-Performance" + --devices "${{ matrix.device }}" + --app_path "leopard-test-app/build/outputs/apk/en/debug/leopard-test-app-en-debug.apk" + --test_path "leopard-test-app/build/outputs/apk/androidTest/en/debug/leopard-test-app-en-debug-androidTest.apk" diff --git a/.github/workflows/ios-appcenter.yml b/.github/workflows/ios-appcenter.yml deleted file mode 100644 index 1c805dc1..00000000 --- a/.github/workflows/ios-appcenter.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: iOS AppCenter Tests - -on: - workflow_dispatch: - push: - branches: [ master ] - paths: - - 'binding/ios/LeopardAppTest/Podfile.lock' - - '.github/workflows/ios-appcenter.yml' - pull_request: - branches: [ master, 'v[0-9]+.[0-9]+' ] - paths: - - 'binding/ios/LeopardAppTest/Podfile.lock' - - '.github/workflows/ios-appcenter.yml' - -defaults: - run: - working-directory: binding/ios/LeopardAppTest - -jobs: - build: - name: Run iOS Tests on AppCenter - runs-on: macos-latest - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Node.js LTS - uses: actions/setup-node@v3 - with: - node-version: lts/* - - - name: Install Cocoapods - run: gem install cocoapods - - - name: Install AppCenter CLI - run: npm install -g appcenter-cli - - - name: Make build dir - run: mkdir ddp - - - name: Install resource script dependency - run: | - brew update - brew install convmv - - name: Copy test_resources - run: ./copy_test_resources.sh - - - name: Run Cocoapods - run: pod install - - - name: Inject AppID - run: sed -i '.bak' 's:{TESTING_ACCESS_KEY_HERE}:${{secrets.PV_VALID_ACCESS_KEY}}:' - LeopardAppTestUITests/LeopardAppTestUITests.swift - - - name: XCode Build - run: xcrun xcodebuild build-for-testing - -configuration Debug - -workspace LeopardAppTest.xcworkspace - -sdk iphoneos - -scheme LeopardAppTest - -derivedDataPath ddp - CODE_SIGNING_ALLOWED=NO - - - name: Run Tests on AppCenter - run: appcenter test run xcuitest - --token ${{secrets.APPCENTERAPITOKEN}} - --app "Picovoice/Leopard-iOS" - --devices "Picovoice/ios-min-max" - --test-series "leopard-ios" - --locale "en_US" - --build-dir ddp/Build/Products/Debug-iphoneos diff --git a/.github/workflows/ios-browserstack.yml b/.github/workflows/ios-browserstack.yml new file mode 100644 index 00000000..b187e52b --- /dev/null +++ b/.github/workflows/ios-browserstack.yml @@ -0,0 +1,83 @@ +name: iOS BrowserStack Tests + +on: + workflow_dispatch: + push: + branches: [ master ] + paths: + - 'binding/ios/LeopardAppTest/Podfile.lock' + - '.github/workflows/ios-browserstack.yml' + pull_request: + branches: [ master, 'v[0-9]+.[0-9]+' ] + paths: + - 'binding/ios/LeopardAppTest/Podfile.lock' + - '.github/workflows/ios-browserstack.yml' + +defaults: + run: + working-directory: binding/ios/LeopardAppTest + +jobs: + build: + name: Run iOS Tests on BrowserStack + runs-on: macos-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Installing Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + - run: + pip3 install requests + + - name: Install Cocoapods + run: gem install cocoapods + + - name: Install AppCenter CLI + run: npm install -g appcenter-cli + + - name: Make build dir + run: mkdir ddp + + - name: Copy test_resources + run: ./copy_test_resources.sh + + - name: Run Cocoapods + run: pod install + + - name: Inject AccessKey + run: sed -i '.bak' 's:{TESTING_ACCESS_KEY_HERE}:${{secrets.PV_VALID_ACCESS_KEY}}:' + LeopardAppTestUITests/LeopardAppTestUITests.swift + + - name: XCode Build + run: xcrun xcodebuild build-for-testing + -configuration Debug + -workspace LeopardAppTest.xcworkspace + -sdk iphoneos + -scheme LeopardAppTest + -derivedDataPath ddp + CODE_SIGNING_ALLOWED=NO + + - name: Generating ipa + run: cd ddp/Build/Products/Debug-iphoneos/ && + mkdir Payload && + cp -r LeopardAppTest.app Payload && + zip --symlinks -r LeopardAppTest.ipa Payload && + rm -r Payload + + - name: Zipping Tests + run: cd ddp/Build/Products/Debug-iphoneos/ && + zip --symlinks -r LeopardAppTestUITests.zip LeopardAppTestUITests-Runner.app + + - name: Run tests on BrowserStack + run: python3 ../../../script/automation/browserstack.py + --type xcuitest + --username "${{secrets.BROWSERSTACK_USERNAME}}" + --access_key "${{secrets.BROWSERSTACK_ACCESS_KEY}}" + --project_name "Leopard-iOS" + --devices "ios-min-max" + --app_path "ddp/Build/Products/Debug-iphoneos/LeopardAppTest.ipa" + --test_path "ddp/Build/Products/Debug-iphoneos/LeopardAppTestUITests.zip" diff --git a/.github/workflows/ios-demos.yml b/.github/workflows/ios-demos.yml index 8f04ef37..d4670f10 100644 --- a/.github/workflows/ios-demos.yml +++ b/.github/workflows/ios-demos.yml @@ -33,9 +33,6 @@ jobs: - name: Install Cocoapods run: gem install cocoapods - - name: Install AppCenter CLI - run: npm install -g appcenter-cli - - name: Make build dir run: mkdir ddp diff --git a/.github/workflows/ios-perf.yml b/.github/workflows/ios-perf.yml index 03e4f2d1..9c34a261 100644 --- a/.github/workflows/ios-perf.yml +++ b/.github/workflows/ios-perf.yml @@ -21,12 +21,12 @@ defaults: jobs: build: - name: Run iOS Tests on AppCenter + name: Run iOS Tests on BrowserStack runs-on: macos-latest strategy: matrix: - device: [ios-perf] + device: [ ios-perf ] include: - device: ios-perf initPerformanceThresholdSec: 3.0 @@ -36,31 +36,25 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - name: Set up Node.js LTS - uses: actions/setup-node@v3 - with: - node-version: lts/* + - name: Installing Python + uses: actions/setup-python@v5 + python-version: '3.10' + - run: + pip3 install requests - name: Install Cocoapods run: gem install cocoapods - - name: Install AppCenter CLI - run: npm install -g appcenter-cli - - name: Make build dir run: mkdir ddp - - name: Install resource script dependency - run: | - brew update - brew install convmv - name: Copy test_resources run: ./copy_test_resources.sh - name: Run Cocoapods run: pod install - - name: Inject AppID + - name: Inject AccessKey run: sed -i '.bak' 's:{TESTING_ACCESS_KEY_HERE}:${{secrets.PV_VALID_ACCESS_KEY}}:' PerformanceTest/PerformanceTest.swift @@ -69,11 +63,13 @@ jobs: PerformanceTest/PerformanceTest.swift - name: Inject Performance Threshold - run: sed -i '.bak' 's:{INIT_PERFORMANCE_THRESHOLD_SEC}:${{ matrix.initPerformanceThresholdSec }}:' + run: sed -i '.bak' + '1,/{INIT_PERFORMANCE_THRESHOLD_SEC}/s/{INIT_PERFORMANCE_THRESHOLD_SEC}/${{ matrix.initPerformanceThresholdSec }}/' PerformanceTest/PerformanceTest.swift - name: Inject Performance Threshold - run: sed -i '.bak' 's:{PROC_PERFORMANCE_THRESHOLD_SEC}:${{ matrix.procPerformanceThresholdSec }}:' + run: sed -i '.bak' + '1,/{PROC_PERFORMANCE_THRESHOLD_SEC}/s/{PROC_PERFORMANCE_THRESHOLD_SEC}/${{ matrix.procPerformanceThresholdSec }}/' PerformanceTest/PerformanceTest.swift - name: XCode Build @@ -85,11 +81,23 @@ jobs: -derivedDataPath ddp CODE_SIGNING_ALLOWED=NO - - name: Run Tests on AppCenter - run: appcenter test run xcuitest - --token ${{secrets.APPCENTERAPITOKEN}} - --app "Picovoice/Leopard-iOS" - --devices "Picovoice/${{ matrix.device }}" - --test-series "leopard-ios" - --locale "en_US" - --build-dir ddp/Build/Products/Debug-iphoneos + - name: Generating ipa + run: cd ddp/Build/Products/Debug-iphoneos/ && + mkdir Payload && + cp -r LeopardAppTest.app Payload && + zip --symlinks -r LeopardAppTest.ipa Payload && + rm -r Payload + + - name: Zipping Tests + run: cd ddp/Build/Products/Debug-iphoneos/ && + zip --symlinks -r PerformanceTest.zip PerformanceTest-Runner.app + + - name: Run tests on BrowserStack + run: python3 ../../../script/automation/browserstack.py + --type xcuitest + --username "${{secrets.BROWSERSTACK_USERNAME}}" + --access_key "${{secrets.BROWSERSTACK_ACCESS_KEY}}" + --project_name "Leopard-iOS-Performance" + --devices "${{ matrix.device }}" + --app_path "ddp/Build/Products/Debug-iphoneos/LeopardAppTest.ipa" + --test_path "ddp/Build/Products/Debug-iphoneos/PerformanceTest.zip" diff --git a/binding/android/LeopardTestApp/leopard-test-app/build.gradle b/binding/android/LeopardTestApp/leopard-test-app/build.gradle index db4cfc0f..3e9312d1 100644 --- a/binding/android/LeopardTestApp/leopard-test-app/build.gradle +++ b/binding/android/LeopardTestApp/leopard-test-app/build.gradle @@ -140,6 +140,7 @@ dependencies { implementation 'androidx.appcompat:appcompat:1.6.1' implementation 'com.google.android.material:material:1.8.0' implementation 'androidx.constraintlayout:constraintlayout:2.1.4' + implementation 'com.google.code.gson:gson:2.10' implementation 'ai.picovoice:leopard-android:2.0.1' // Espresso UI Testing @@ -147,7 +148,6 @@ dependencies { androidTestImplementation('androidx.test.espresso:espresso-core:3.2.0', { exclude group: 'com.android.support', module: 'support-annotations' }) - androidTestImplementation('com.microsoft.appcenter:espresso-test-extension:1.4') androidTestImplementation('androidx.test.espresso:espresso-intents:3.5.1') } diff --git a/binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/BaseTest.java b/binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/BaseTest.java index c4a04845..6998d712 100644 --- a/binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/BaseTest.java +++ b/binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/BaseTest.java @@ -20,12 +20,7 @@ import androidx.test.platform.app.InstrumentationRegistry; -import com.microsoft.appcenter.espresso.Factory; -import com.microsoft.appcenter.espresso.ReportHelper; - -import org.junit.After; import org.junit.Before; -import org.junit.Rule; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; @@ -44,9 +39,6 @@ public class BaseTest { - @Rule - public ReportHelper reportHelper = Factory.getReportHelper(); - Context testContext; Context appContext; AssetManager assetManager; @@ -55,11 +47,6 @@ public class BaseTest { String accessKey; - @After - public void TearDown() { - reportHelper.label("Stopping App"); - } - @Before public void Setup() throws IOException { testContext = InstrumentationRegistry.getInstrumentation().getContext(); diff --git a/binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/IntegrationTest.java b/binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/IntegrationTest.java index 8f92cb19..af91e891 100644 --- a/binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/IntegrationTest.java +++ b/binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/IntegrationTest.java @@ -16,9 +16,6 @@ import androidx.test.ext.junit.rules.ActivityScenarioRule; import androidx.test.ext.junit.runners.AndroidJUnit4; -import com.microsoft.appcenter.espresso.Factory; -import com.microsoft.appcenter.espresso.ReportHelper; - import org.hamcrest.Matcher; import org.junit.After; import org.junit.Before; @@ -74,9 +71,6 @@ public void perform(UiController uiController, View view) { @RunWith(AndroidJUnit4.class) public class IntegrationTest { - @Rule - public ReportHelper reportHelper = Factory.getReportHelper(); - @Rule public ActivityScenarioRule activityScenarioRule = new ActivityScenarioRule<>(MainActivity.class); @@ -91,11 +85,6 @@ public void intentsTeardown() { Intents.release(); } - @After - public void TearDown() { - reportHelper.label("Stopping App"); - } - @Test public void testPorcupine() { onView(withId(R.id.testButton)).perform(click()); diff --git a/binding/ios/LeopardAppTest/copy_test_resources.sh b/binding/ios/LeopardAppTest/copy_test_resources.sh index 00bee125..d9bfa663 100755 --- a/binding/ios/LeopardAppTest/copy_test_resources.sh +++ b/binding/ios/LeopardAppTest/copy_test_resources.sh @@ -15,6 +15,3 @@ cp ${LIB_DIR}/common/*.pv ${ASSETS_DIR}/model_files echo "Copying test data file..." cp ${RESOURCE_DIR}/.test/test_data.json ${ASSETS_DIR} - -echo "Fixing filename encodings for Appcenter compatibility" -convmv --notest -f utf8 -t utf8 --nfd -r ${ASSETS_DIR} diff --git a/resources/spell-check/dict.txt b/resources/spell-check/dict.txt index 896773de..fc5dbcc4 100644 --- a/resources/spell-check/dict.txt +++ b/resources/spell-check/dict.txt @@ -4,7 +4,6 @@ alcune alla anche androidx -Appcenter aprender arbovm armv @@ -30,7 +29,6 @@ conseiller conservateur constraintlayout consulat -convmv cstring cword devient diff --git a/script/automation/browserstack.py b/script/automation/browserstack.py new file mode 100644 index 00000000..4a8d2378 --- /dev/null +++ b/script/automation/browserstack.py @@ -0,0 +1,135 @@ +import argparse +import requests +import time + +APP_URI = 'https://api-cloud.browserstack.com/app-automate/{}/v2/app' +TEST_URI = 'https://api-cloud.browserstack.com/app-automate/{}/v2/test-suite' +BUILD_URI = 'https://api-cloud.browserstack.com/app-automate/{}/v2/build' +STATUS_URI = 'https://api-cloud.browserstack.com/app-automate/{}/v2/builds/{}' + +devices_dict = { + 'android-min-max': [ + 'Samsung Galaxy S8-7.0', + 'Samsung Galaxy M52-11.0', + 'Google Pixel 9-15.0' + ], + 'android-perf': [ + 'Google Pixel 6 Pro-15.0' + ], + 'ios-min-max': [ + 'iPhone SE 2020-13', + 'iPhone 14 Pro-16', + 'iPhone 14-18' + ], + 'ios-perf': [ + 'iPhone 13-18', + ] +} + + +def main(args: argparse.Namespace) -> None: + app_files = { + 'file': open(args.app_path, 'rb') + } + + app_response = requests.post( + APP_URI.format(args.type), + files=app_files, + auth=(args.username, args.access_key) + ) + app_response_json = app_response.json() + + if not app_response.ok: + print('App Upload Failed', app_response_json) + exit(1) + + test_files = { + 'file': open(args.test_path, 'rb') + } + test_response = requests.post( + TEST_URI.format(args.type), + files=test_files, + auth=(args.username, args.access_key) + ) + test_response_json = test_response.json() + + if not test_response.ok: + print('Test Upload Failed', test_response_json) + exit(1) + + build_headers = { + 'Content-Type': 'application/json' + } + build_data = { + 'app': app_response_json['app_url'], + 'testSuite': test_response_json['test_suite_url'], + 'project': args.project_name, + 'devices': devices_dict[args.devices] + } + + while True: + build_response = requests.post( + BUILD_URI.format(args.type), + headers=build_headers, + json=build_data, + auth=(args.username, args.access_key) + ) + if (build_response is not None and 'message' in build_response.json() and '[BROWSERSTACK_ALL_PARALLELS_IN_USE]' + in build_response.json()['message']): + print('Parallel threads limit reached. Waiting...', flush=True) + time.sleep(60) + else: + break + + if build_response is None: + print('Build Failed') + exit(1) + + build_response_json = build_response.json() + + if not build_response.ok: + print('Build Failed', build_response.json()) + exit(1) + + if build_response_json['message'] != 'Success': + print('Build Unsuccessful') + exit(1) + + print( + 'View build results at https://app-automate.browserstack.com/dashboard/v2/builds/{}' + .format(build_response_json['build_id'])) + + while True: + time.sleep(60) + status_response = requests.get( + STATUS_URI.format(args.type, build_response_json['build_id']), + auth=(args.username, args.access_key) + ) + status_response_json = status_response.json() + status = status_response_json['status'] + + if not status_response.ok: + print('Status Request Failed', status_response_json) + exit(1) + + if status != 'queued' and status != 'running': + break + + print('Status:', status) + if status != 'passed': + exit(1) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--type', choices=['espresso', 'xcuitest'], required=True) + parser.add_argument('--username', required=True) + parser.add_argument('--access_key', required=True) + + parser.add_argument('--project_name', required=True) + parser.add_argument('--devices', choices=devices_dict.keys(), required=True) + parser.add_argument('--app_path', required=True) + parser.add_argument('--test_path', required=True) + args = parser.parse_args() + + main(args) From e302669b8931028b0cccedb386a8a3fb3eb40043 Mon Sep 17 00:00:00 2001 From: Albert Ho Date: Thu, 21 Nov 2024 14:00:30 -0800 Subject: [PATCH 2/6] fix path --- .github/workflows/android-browserstack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/android-browserstack.yml b/.github/workflows/android-browserstack.yml index 2af1f39e..7acf45c9 100644 --- a/.github/workflows/android-browserstack.yml +++ b/.github/workflows/android-browserstack.yml @@ -75,7 +75,7 @@ jobs: --project_name "Leopard-Android" --devices "android-min-max" --app_path "leopard-test-app/build/outputs/apk/en/debug/leopard-test-app-en-debug.apk" - --test_path "leopard-test-app/build/outputs/apk/en/androidTest/debug/leopard-test-app-en-debug-androidTest.apk" + --test_path "leopard-test-app/build/outputs/apk/androidTest/en/debug/leopard-test-app-en-debug-androidTest.apk" build-integ: name: Run Android Integration Tests on BrowserStack From 10fd1f546a10e2112cbdda7fded9c90a5f9b031e Mon Sep 17 00:00:00 2001 From: Albert Ho Date: Mon, 25 Nov 2024 12:10:31 -0800 Subject: [PATCH 3/6] try fix --- .github/workflows/android-browserstack.yml | 8 +- .github/workflows/android-demos.yml | 4 +- .github/workflows/android-perf.yml | 4 +- binding/android/LeopardTestApp/build.gradle | 2 +- .../android/LeopardTestApp/gradle.properties | 3 + .../gradle/wrapper/gradle-wrapper.properties | 2 +- .../leopard-test-app/build.gradle | 1 + .../picovoice/leopard/testapp/BaseTest.java | 3 +- .../leopard/testapp/DiarizationTests.java | 123 +++++ .../leopard/testapp/LanguageTests.java | 216 +++++++++ .../leopard/testapp/LeopardTest.java | 426 ------------------ .../leopard/testapp/StandardTests.java | 137 ++++++ .../src/main/AndroidManifest.xml | 3 +- resources/spell-check/dict.txt | 1 + script/automation/browserstack.py | 5 +- 15 files changed, 496 insertions(+), 442 deletions(-) create mode 100644 binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/DiarizationTests.java create mode 100644 binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/LanguageTests.java delete mode 100644 binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/LeopardTest.java create mode 100644 binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/StandardTests.java diff --git a/.github/workflows/android-browserstack.yml b/.github/workflows/android-browserstack.yml index 7acf45c9..683e62f5 100644 --- a/.github/workflows/android-browserstack.yml +++ b/.github/workflows/android-browserstack.yml @@ -36,10 +36,10 @@ jobs: - run: pip3 install requests - - name: set up JDK 11 + - name: set up JDK 17 uses: actions/setup-java@v3 with: - java-version: '11' + java-version: '17' distribution: 'temurin' - name: Copy test_resources @@ -91,10 +91,10 @@ jobs: - run: pip3 install requests - - name: set up JDK 11 + - name: set up JDK 17 uses: actions/setup-java@v3 with: - java-version: '11' + java-version: '17' distribution: 'temurin' - name: Copy test_resources diff --git a/.github/workflows/android-demos.yml b/.github/workflows/android-demos.yml index 8dbf29de..88d2d276 100644 --- a/.github/workflows/android-demos.yml +++ b/.github/workflows/android-demos.yml @@ -25,10 +25,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: set up JDK 11 + - name: set up JDK 17 uses: actions/setup-java@v3 with: - java-version: '11' + java-version: '17' distribution: 'temurin' - name: Build diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml index c822769e..f8e2440c 100644 --- a/.github/workflows/android-perf.yml +++ b/.github/workflows/android-perf.yml @@ -42,10 +42,10 @@ jobs: - run: pip3 install requests - - name: set up JDK 11 + - name: set up JDK 17 uses: actions/setup-java@v3 with: - java-version: '11' + java-version: '17' distribution: 'temurin' - name: Copy test_resources diff --git a/binding/android/LeopardTestApp/build.gradle b/binding/android/LeopardTestApp/build.gradle index d1001bd5..f688e154 100644 --- a/binding/android/LeopardTestApp/build.gradle +++ b/binding/android/LeopardTestApp/build.gradle @@ -8,7 +8,7 @@ buildscript { mavenCentral() } dependencies { - classpath 'com.android.tools.build:gradle:7.2.2' + classpath 'com.android.tools.build:gradle:8.2.2' } } diff --git a/binding/android/LeopardTestApp/gradle.properties b/binding/android/LeopardTestApp/gradle.properties index c09e1e3b..683c3de0 100644 --- a/binding/android/LeopardTestApp/gradle.properties +++ b/binding/android/LeopardTestApp/gradle.properties @@ -15,3 +15,6 @@ org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8 # Android operating system, and which are packaged with your app"s APK # https://developer.android.com/topic/libraries/support-library/androidx-rn android.useAndroidX=true +android.defaults.buildfeatures.buildconfig=true +android.nonTransitiveRClass=false +android.nonFinalResIds=false diff --git a/binding/android/LeopardTestApp/gradle/wrapper/gradle-wrapper.properties b/binding/android/LeopardTestApp/gradle/wrapper/gradle-wrapper.properties index 740ab489..3d63a902 100644 --- a/binding/android/LeopardTestApp/gradle/wrapper/gradle-wrapper.properties +++ b/binding/android/LeopardTestApp/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ #Tue Jun 29 23:02:09 PDT 2021 distributionBase=GRADLE_USER_HOME -distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip distributionPath=wrapper/dists zipStorePath=wrapper/dists zipStoreBase=GRADLE_USER_HOME diff --git a/binding/android/LeopardTestApp/leopard-test-app/build.gradle b/binding/android/LeopardTestApp/leopard-test-app/build.gradle index 3e9312d1..7b15042f 100644 --- a/binding/android/LeopardTestApp/leopard-test-app/build.gradle +++ b/binding/android/LeopardTestApp/leopard-test-app/build.gradle @@ -133,6 +133,7 @@ android { lint { abortOnError false } + namespace 'ai.picovoice.leopard.testapp' } dependencies { diff --git a/binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/BaseTest.java b/binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/BaseTest.java index 6998d712..edf4bd87 100644 --- a/binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/BaseTest.java +++ b/binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/BaseTest.java @@ -1,5 +1,5 @@ /* - Copyright 2022-2023 Picovoice Inc. + Copyright 2022-2024 Picovoice Inc. You may not use this file except in compliance with the license. A copy of the license is located in the "LICENSE" file accompanying this source. @@ -12,7 +12,6 @@ package ai.picovoice.leopard.testapp; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertEquals; import android.content.Context; diff --git a/binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/DiarizationTests.java b/binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/DiarizationTests.java new file mode 100644 index 00000000..cb14d172 --- /dev/null +++ b/binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/DiarizationTests.java @@ -0,0 +1,123 @@ +/* + Copyright 2022-2024 Picovoice Inc. + + You may not use this file except in compliance with the license. A copy of the license is + located in the "LICENSE" file accompanying this source. + + Unless required by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + express or implied. See the License for the specific language governing permissions and + limitations under the License. +*/ + +package ai.picovoice.leopard.testapp; + +import static org.junit.Assert.*; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import ai.picovoice.leopard.Leopard; +import ai.picovoice.leopard.LeopardTranscript; + +@RunWith(Parameterized.class) +public class DiarizationTests extends BaseTest { + @Parameterized.Parameter(value = 0) + public String language; + + @Parameterized.Parameter(value = 1) + public String modelFile; + + @Parameterized.Parameter(value = 2) + public String testAudioFile; + + @Parameterized.Parameter(value = 3) + public LeopardTranscript.Word[] expectedWords; + + @Parameterized.Parameters(name = "{0}") + public static Collection initParameters() throws IOException { + String testDataJsonString = getTestDataString(); + + JsonParser parser = new JsonParser(); + JsonObject testDataJson = parser.parse(testDataJsonString).getAsJsonObject(); + JsonArray languageTests = testDataJson + .getAsJsonObject("tests") + .getAsJsonArray("diarization_tests"); + + List parameters = new ArrayList<>(); + for (int i = 0; i < languageTests.size(); i++) { + JsonObject testData = languageTests.get(i).getAsJsonObject(); + + String language = testData.get("language").getAsString(); + String audioFile = testData.get("audio_file").getAsString(); + JsonArray words = testData.get("words").getAsJsonArray(); + + String modelFile; + if (language.equals("en")) { + modelFile = "model_files/leopard_params.pv"; + } else { + modelFile = String.format("model_files/leopard_params_%s.pv", language); + } + + String testAudioFile = String.format("audio_samples/%s", audioFile); + + LeopardTranscript.Word[] paramWords = new LeopardTranscript.Word[words.size()]; + for (int j = 0; j < words.size(); j++) { + JsonObject wordObject = words.get(j).getAsJsonObject(); + + String word = wordObject.get("word").getAsString(); + int speakerTag = wordObject.get("speaker_tag").getAsInt(); + + paramWords[j] = new LeopardTranscript.Word( + word, + 0.f, + 0.f, + 0.f, + speakerTag + ); + } + + parameters.add(new Object[]{ + language, + modelFile, + testAudioFile, + paramWords + }); + } + + return parameters; + } + + @Test + public void testDiarizationMultipleSpeakers() throws Exception { + String modelPath = new File(testResourcesPath, modelFile).getAbsolutePath(); + Leopard leopard = new Leopard.Builder() + .setAccessKey(accessKey) + .setModelPath(modelPath) + .setEnableDiarization(true) + .build(appContext); + + File audioFile = new File(testResourcesPath, testAudioFile); + short[] pcm = readAudioFile(audioFile.getAbsolutePath()); + + LeopardTranscript result = leopard.process(pcm); + + assertEquals(result.getWordArray().length, expectedWords.length); + for (int i = 0; i < result.getWordArray().length; i++) { + assertEquals(result.getWordArray()[i].getWord(), expectedWords[i].getWord()); + assertEquals(result.getWordArray()[i].getSpeakerTag(), expectedWords[i].getSpeakerTag()); + } + leopard.delete(); + } +} diff --git a/binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/LanguageTests.java b/binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/LanguageTests.java new file mode 100644 index 00000000..fdea0e5c --- /dev/null +++ b/binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/LanguageTests.java @@ -0,0 +1,216 @@ +/* + Copyright 2022-2024 Picovoice Inc. + + You may not use this file except in compliance with the license. A copy of the license is + located in the "LICENSE" file accompanying this source. + + Unless required by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + express or implied. See the License for the specific language governing permissions and + limitations under the License. +*/ + +package ai.picovoice.leopard.testapp; + +import static org.junit.Assert.*; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import ai.picovoice.leopard.Leopard; +import ai.picovoice.leopard.LeopardTranscript; + +@RunWith(Parameterized.class) +public class LanguageTests extends BaseTest { + @Parameterized.Parameter(value = 0) + public String language; + + @Parameterized.Parameter(value = 1) + public String modelFile; + + @Parameterized.Parameter(value = 2) + public String testAudioFile; + + @Parameterized.Parameter(value = 3) + public String expectedTranscript; + + @Parameterized.Parameter(value = 4) + public String expectedTranscriptWithPunctuation; + + @Parameterized.Parameter(value = 5) + public float errorRate; + + @Parameterized.Parameter(value = 6) + public LeopardTranscript.Word[] expectedWords; + + @Parameterized.Parameters(name = "{0}") + public static Collection initParameters() throws IOException { + String testDataJsonString = getTestDataString(); + + JsonParser parser = new JsonParser(); + JsonObject testDataJson = parser.parse(testDataJsonString).getAsJsonObject(); + JsonArray languageTests = testDataJson + .getAsJsonObject("tests") + .getAsJsonArray("language_tests"); + + List parameters = new ArrayList<>(); + for (int i = 0; i < languageTests.size(); i++) { + JsonObject testData = languageTests.get(i).getAsJsonObject(); + + String language = testData.get("language").getAsString(); + String audioFile = testData.get("audio_file").getAsString(); + String transcript = testData.get("transcript").getAsString(); + String transcriptWithPunctuation = testData.get("transcript_with_punctuation").getAsString(); + float errorRate = testData.get("error_rate").getAsFloat(); + JsonArray words = testData.get("words").getAsJsonArray(); + + String modelFile; + if (language.equals("en")) { + modelFile = "model_files/leopard_params.pv"; + } else { + modelFile = String.format("model_files/leopard_params_%s.pv", language); + } + + String testAudioFile = String.format("audio_samples/%s", audioFile); + + LeopardTranscript.Word[] paramWords = new LeopardTranscript.Word[words.size()]; + for (int j = 0; j < words.size(); j++) { + JsonObject wordObject = words.get(j).getAsJsonObject(); + + String word = wordObject.get("word").getAsString(); + float confidence = wordObject.get("confidence").getAsFloat(); + float startSec = wordObject.get("start_sec").getAsFloat(); + float endSec = wordObject.get("end_sec").getAsFloat(); + int speakerTag = wordObject.get("speaker_tag").getAsInt(); + + paramWords[j] = new LeopardTranscript.Word( + word, + confidence, + startSec, + endSec, + speakerTag + ); + } + + parameters.add(new Object[]{ + language, + modelFile, + testAudioFile, + transcript, + transcriptWithPunctuation, + errorRate, + paramWords + }); + } + + return parameters; + } + + @Test + public void testTranscribeAudioFile() throws Exception { + String modelPath = new File(testResourcesPath, modelFile).getAbsolutePath(); + Leopard leopard = new Leopard.Builder() + .setAccessKey(accessKey) + .setModelPath(modelPath) + .build(appContext); + + File audioFile = new File(testResourcesPath, testAudioFile); + boolean useCER = language.equals("ja"); + + LeopardTranscript result = leopard.processFile(audioFile.getAbsolutePath()); + + assertTrue(getWordErrorRate(result.getTranscriptString(), expectedTranscript, useCER) < errorRate); + validateMetadata( + result.getWordArray(), + expectedWords, + false + ); + + leopard.delete(); + } + + @Test + public void testTranscribeAudioFileWithPunctuation() throws Exception { + String modelPath = new File(testResourcesPath, modelFile).getAbsolutePath(); + Leopard leopard = new Leopard.Builder() + .setAccessKey(accessKey) + .setModelPath(modelPath) + .setEnableAutomaticPunctuation(true) + .build(appContext); + + File audioFile = new File(testResourcesPath, testAudioFile); + boolean useCER = language.equals("ja"); + + LeopardTranscript result = leopard.processFile(audioFile.getAbsolutePath()); + assertTrue(getWordErrorRate( + result.getTranscriptString(), expectedTranscriptWithPunctuation, useCER) < errorRate); + + validateMetadata( + result.getWordArray(), + expectedWords, + false + ); + + leopard.delete(); + } + + @Test + public void testTranscribeAudioData() throws Exception { + String modelPath = new File(testResourcesPath, modelFile).getAbsolutePath(); + Leopard leopard = new Leopard.Builder() + .setAccessKey(accessKey) + .setModelPath(modelPath) + .build(appContext); + + File audioFile = new File(testResourcesPath, testAudioFile); + short[] pcm = readAudioFile(audioFile.getAbsolutePath()); + + LeopardTranscript result = leopard.process(pcm); + boolean useCER = language.equals("ja"); + + assertTrue(getWordErrorRate(result.getTranscriptString(), expectedTranscript, useCER) < errorRate); + validateMetadata( + result.getWordArray(), + expectedWords, + false + ); + + leopard.delete(); + } + + @Test + public void testTranscribeAudioDataWithDiarization() throws Exception { + String modelPath = new File(testResourcesPath, modelFile).getAbsolutePath(); + Leopard leopard = new Leopard.Builder() + .setAccessKey(accessKey) + .setModelPath(modelPath) + .setEnableDiarization(true) + .build(appContext); + + File audioFile = new File(testResourcesPath, testAudioFile); + short[] pcm = readAudioFile(audioFile.getAbsolutePath()); + + LeopardTranscript result = leopard.process(pcm); + boolean useCER = language.equals("ja"); + + assertTrue(getWordErrorRate(result.getTranscriptString(), expectedTranscript, useCER) < errorRate); + validateMetadata( + result.getWordArray(), + expectedWords, + true + ); + + leopard.delete(); + } +} diff --git a/binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/LeopardTest.java b/binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/LeopardTest.java deleted file mode 100644 index 893a3e84..00000000 --- a/binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/LeopardTest.java +++ /dev/null @@ -1,426 +0,0 @@ -/* - Copyright 2022-2023 Picovoice Inc. - - You may not use this file except in compliance with the license. A copy of the license is - located in the "LICENSE" file accompanying this source. - - Unless required by applicable law or agreed to in writing, software distributed under the - License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - express or implied. See the License for the specific language governing permissions and - limitations under the License. -*/ - -package ai.picovoice.leopard.testapp; - -import static org.junit.Assert.*; - -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; - -import org.junit.Test; -import org.junit.experimental.runners.Enclosed; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -import ai.picovoice.leopard.Leopard; -import ai.picovoice.leopard.LeopardException; -import ai.picovoice.leopard.LeopardTranscript; - - -@RunWith(Enclosed.class) -public class LeopardTest { - - public static class StandardTests extends BaseTest { - - @Test - public void testInitFailWithInvalidAccessKey() { - boolean didFail = false; - try { - new Leopard.Builder() - .setAccessKey("") - .setModelPath(defaultModelPath) - .build(appContext); - } catch (LeopardException e) { - didFail = true; - } - - assertTrue(didFail); - } - - @Test - public void testInitFailWithMissingAccessKey() { - boolean didFail = false; - try { - new Leopard.Builder() - .setModelPath(defaultModelPath) - .build(appContext); - } catch (LeopardException e) { - didFail = true; - } - - assertTrue(didFail); - } - - @Test - public void testInitFailWithInvalidModelPath() { - boolean didFail = false; - File modelPath = new File(testResourcesPath, "bad_path/bad_path.pv"); - try { - new Leopard.Builder() - .setAccessKey(accessKey) - .setModelPath(modelPath.getAbsolutePath()) - .build(appContext); - } catch (LeopardException e) { - didFail = true; - } - - assertTrue(didFail); - } - - @Test - public void testInitFailWithMissingModelPath() { - boolean didFail = false; - try { - new Leopard.Builder() - .setAccessKey(accessKey) - .build(appContext); - } catch (LeopardException e) { - didFail = true; - } - - assertTrue(didFail); - } - - @Test - public void getVersion() throws LeopardException { - Leopard leopard = new Leopard.Builder().setAccessKey(accessKey) - .setModelPath(defaultModelPath) - .build(appContext); - - assertTrue(leopard.getVersion() != null && !leopard.getVersion().equals("")); - - leopard.delete(); - } - - @Test - public void getSampleRate() throws LeopardException { - Leopard leopard = new Leopard.Builder().setAccessKey(accessKey) - .setModelPath(defaultModelPath) - .build(appContext); - - assertTrue(leopard.getSampleRate() > 0); - - leopard.delete(); - } - - @Test - public void testErrorStack() { - String[] error = {}; - try { - new Leopard.Builder() - .setAccessKey("invalid") - .setModelPath(defaultModelPath) - .build(appContext); - } catch (LeopardException e) { - error = e.getMessageStack(); - } - - assertTrue(0 < error.length); - assertTrue(error.length <= 8); - - try { - new Leopard.Builder() - .setAccessKey("invalid") - .setModelPath(defaultModelPath) - .build(appContext); - } catch (LeopardException e) { - for (int i = 0; i < error.length; i++) { - assertEquals(e.getMessageStack()[i], error[i]); - } - } - } - } - - @RunWith(Parameterized.class) - public static class LanguageTests extends BaseTest { - @Parameterized.Parameter(value = 0) - public String language; - - @Parameterized.Parameter(value = 1) - public String modelFile; - - @Parameterized.Parameter(value = 2) - public String testAudioFile; - - @Parameterized.Parameter(value = 3) - public String expectedTranscript; - - @Parameterized.Parameter(value = 4) - public String expectedTranscriptWithPunctuation; - - @Parameterized.Parameter(value = 5) - public float errorRate; - - @Parameterized.Parameter(value = 6) - public LeopardTranscript.Word[] expectedWords; - - @Parameterized.Parameters(name = "{0}") - public static Collection initParameters() throws IOException { - String testDataJsonString = getTestDataString(); - - JsonParser parser = new JsonParser(); - JsonObject testDataJson = parser.parse(testDataJsonString).getAsJsonObject(); - JsonArray languageTests = testDataJson - .getAsJsonObject("tests") - .getAsJsonArray("language_tests"); - - List parameters = new ArrayList<>(); - for (int i = 0; i < languageTests.size(); i++) { - JsonObject testData = languageTests.get(i).getAsJsonObject(); - - String language = testData.get("language").getAsString(); - String audioFile = testData.get("audio_file").getAsString(); - String transcript = testData.get("transcript").getAsString(); - String transcriptWithPunctuation = testData.get("transcript_with_punctuation").getAsString(); - float errorRate = testData.get("error_rate").getAsFloat(); - JsonArray words = testData.get("words").getAsJsonArray(); - - String modelFile; - if (language.equals("en")) { - modelFile = "model_files/leopard_params.pv"; - } else { - modelFile = String.format("model_files/leopard_params_%s.pv", language); - } - - String testAudioFile = String.format("audio_samples/%s", audioFile); - - LeopardTranscript.Word[] paramWords = new LeopardTranscript.Word[words.size()]; - for (int j = 0; j < words.size(); j++) { - JsonObject wordObject = words.get(j).getAsJsonObject(); - - String word = wordObject.get("word").getAsString(); - float confidence = wordObject.get("confidence").getAsFloat(); - float startSec = wordObject.get("start_sec").getAsFloat(); - float endSec = wordObject.get("end_sec").getAsFloat(); - int speakerTag = wordObject.get("speaker_tag").getAsInt(); - - paramWords[j] = new LeopardTranscript.Word( - word, - confidence, - startSec, - endSec, - speakerTag - ); - } - - parameters.add(new Object[]{ - language, - modelFile, - testAudioFile, - transcript, - transcriptWithPunctuation, - errorRate, - paramWords - }); - } - - return parameters; - } - - - @Test - public void testTranscribeAudioFile() throws Exception { - String modelPath = new File(testResourcesPath, modelFile).getAbsolutePath(); - Leopard leopard = new Leopard.Builder() - .setAccessKey(accessKey) - .setModelPath(modelPath) - .build(appContext); - - File audioFile = new File(testResourcesPath, testAudioFile); - boolean useCER = language.equals("ja"); - - LeopardTranscript result = leopard.processFile(audioFile.getAbsolutePath()); - - assertTrue(getWordErrorRate(result.getTranscriptString(), expectedTranscript, useCER) < errorRate); - validateMetadata( - result.getWordArray(), - expectedWords, - false - ); - - leopard.delete(); - } - - @Test - public void testTranscribeAudioFileWithPunctuation() throws Exception { - String modelPath = new File(testResourcesPath, modelFile).getAbsolutePath(); - Leopard leopard = new Leopard.Builder() - .setAccessKey(accessKey) - .setModelPath(modelPath) - .setEnableAutomaticPunctuation(true) - .build(appContext); - - File audioFile = new File(testResourcesPath, testAudioFile); - boolean useCER = language.equals("ja"); - - LeopardTranscript result = leopard.processFile(audioFile.getAbsolutePath()); - assertTrue(getWordErrorRate( - result.getTranscriptString(), expectedTranscriptWithPunctuation, useCER) < errorRate); - - validateMetadata( - result.getWordArray(), - expectedWords, - false - ); - - leopard.delete(); - } - - @Test - public void testTranscribeAudioData() throws Exception { - String modelPath = new File(testResourcesPath, modelFile).getAbsolutePath(); - Leopard leopard = new Leopard.Builder() - .setAccessKey(accessKey) - .setModelPath(modelPath) - .build(appContext); - - File audioFile = new File(testResourcesPath, testAudioFile); - short[] pcm = readAudioFile(audioFile.getAbsolutePath()); - - LeopardTranscript result = leopard.process(pcm); - boolean useCER = language.equals("ja"); - - assertTrue(getWordErrorRate(result.getTranscriptString(), expectedTranscript, useCER) < errorRate); - validateMetadata( - result.getWordArray(), - expectedWords, - false - ); - - leopard.delete(); - } - - @Test - public void testTranscribeAudioDataWithDiarization() throws Exception { - String modelPath = new File(testResourcesPath, modelFile).getAbsolutePath(); - Leopard leopard = new Leopard.Builder() - .setAccessKey(accessKey) - .setModelPath(modelPath) - .setEnableDiarization(true) - .build(appContext); - - File audioFile = new File(testResourcesPath, testAudioFile); - short[] pcm = readAudioFile(audioFile.getAbsolutePath()); - - LeopardTranscript result = leopard.process(pcm); - boolean useCER = language.equals("ja"); - - assertTrue(getWordErrorRate(result.getTranscriptString(), expectedTranscript, useCER) < errorRate); - validateMetadata( - result.getWordArray(), - expectedWords, - true - ); - - leopard.delete(); - } - } - - @RunWith(Parameterized.class) - public static class DiarizationTests extends BaseTest { - @Parameterized.Parameter(value = 0) - public String language; - - @Parameterized.Parameter(value = 1) - public String modelFile; - - @Parameterized.Parameter(value = 2) - public String testAudioFile; - - @Parameterized.Parameter(value = 3) - public LeopardTranscript.Word[] expectedWords; - - @Parameterized.Parameters(name = "{0}") - public static Collection initParameters() throws IOException { - String testDataJsonString = getTestDataString(); - - JsonParser parser = new JsonParser(); - JsonObject testDataJson = parser.parse(testDataJsonString).getAsJsonObject(); - JsonArray languageTests = testDataJson - .getAsJsonObject("tests") - .getAsJsonArray("diarization_tests"); - - List parameters = new ArrayList<>(); - for (int i = 0; i < languageTests.size(); i++) { - JsonObject testData = languageTests.get(i).getAsJsonObject(); - - String language = testData.get("language").getAsString(); - String audioFile = testData.get("audio_file").getAsString(); - JsonArray words = testData.get("words").getAsJsonArray(); - - String modelFile; - if (language.equals("en")) { - modelFile = "model_files/leopard_params.pv"; - } else { - modelFile = String.format("model_files/leopard_params_%s.pv", language); - } - - String testAudioFile = String.format("audio_samples/%s", audioFile); - - LeopardTranscript.Word[] paramWords = new LeopardTranscript.Word[words.size()]; - for (int j = 0; j < words.size(); j++) { - JsonObject wordObject = words.get(j).getAsJsonObject(); - - String word = wordObject.get("word").getAsString(); - int speakerTag = wordObject.get("speaker_tag").getAsInt(); - - paramWords[j] = new LeopardTranscript.Word( - word, - 0.f, - 0.f, - 0.f, - speakerTag - ); - } - - parameters.add(new Object[]{ - language, - modelFile, - testAudioFile, - paramWords - }); - } - - return parameters; - } - - @Test - public void testDiarizationMultipleSpeakers() throws Exception { - String modelPath = new File(testResourcesPath, modelFile).getAbsolutePath(); - Leopard leopard = new Leopard.Builder() - .setAccessKey(accessKey) - .setModelPath(modelPath) - .setEnableDiarization(true) - .build(appContext); - - File audioFile = new File(testResourcesPath, testAudioFile); - short[] pcm = readAudioFile(audioFile.getAbsolutePath()); - - LeopardTranscript result = leopard.process(pcm); - - assertEquals(result.getWordArray().length, expectedWords.length); - for (int i = 0; i < result.getWordArray().length; i++) { - assertEquals(result.getWordArray()[i].getWord(), expectedWords[i].getWord()); - assertEquals(result.getWordArray()[i].getSpeakerTag(), expectedWords[i].getSpeakerTag()); - } - leopard.delete(); - } - } -} diff --git a/binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/StandardTests.java b/binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/StandardTests.java new file mode 100644 index 00000000..d04500d4 --- /dev/null +++ b/binding/android/LeopardTestApp/leopard-test-app/src/androidTest/java/ai/picovoice/leopard/testapp/StandardTests.java @@ -0,0 +1,137 @@ +/* + Copyright 2022-2024 Picovoice Inc. + + You may not use this file except in compliance with the license. A copy of the license is + located in the "LICENSE" file accompanying this source. + + Unless required by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + express or implied. See the License for the specific language governing permissions and + limitations under the License. +*/ + +package ai.picovoice.leopard.testapp; + +import static org.junit.Assert.*; + +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.io.File; + +import ai.picovoice.leopard.Leopard; +import ai.picovoice.leopard.LeopardException; + +import androidx.test.ext.junit.runners.AndroidJUnit4; + +@RunWith(AndroidJUnit4.class) +public class StandardTests extends BaseTest { + + @Test + public void testInitFailWithInvalidAccessKey() { + boolean didFail = false; + try { + new Leopard.Builder() + .setAccessKey("") + .setModelPath(defaultModelPath) + .build(appContext); + } catch (LeopardException e) { + didFail = true; + } + + assertTrue(didFail); + } + + @Test + public void testInitFailWithMissingAccessKey() { + boolean didFail = false; + try { + new Leopard.Builder() + .setModelPath(defaultModelPath) + .build(appContext); + } catch (LeopardException e) { + didFail = true; + } + + assertTrue(didFail); + } + + @Test + public void testInitFailWithInvalidModelPath() { + boolean didFail = false; + File modelPath = new File(testResourcesPath, "bad_path/bad_path.pv"); + try { + new Leopard.Builder() + .setAccessKey(accessKey) + .setModelPath(modelPath.getAbsolutePath()) + .build(appContext); + } catch (LeopardException e) { + didFail = true; + } + + assertTrue(didFail); + } + + @Test + public void testInitFailWithMissingModelPath() { + boolean didFail = false; + try { + new Leopard.Builder() + .setAccessKey(accessKey) + .build(appContext); + } catch (LeopardException e) { + didFail = true; + } + + assertTrue(didFail); + } + + @Test + public void getVersion() throws LeopardException { + Leopard leopard = new Leopard.Builder().setAccessKey(accessKey) + .setModelPath(defaultModelPath) + .build(appContext); + + assertTrue(leopard.getVersion() != null && !leopard.getVersion().equals("")); + + leopard.delete(); + } + + @Test + public void getSampleRate() throws LeopardException { + Leopard leopard = new Leopard.Builder().setAccessKey(accessKey) + .setModelPath(defaultModelPath) + .build(appContext); + + assertTrue(leopard.getSampleRate() > 0); + + leopard.delete(); + } + + @Test + public void testErrorStack() { + String[] error = {}; + try { + new Leopard.Builder() + .setAccessKey("invalid") + .setModelPath(defaultModelPath) + .build(appContext); + } catch (LeopardException e) { + error = e.getMessageStack(); + } + + assertTrue(0 < error.length); + assertTrue(error.length <= 8); + + try { + new Leopard.Builder() + .setAccessKey("invalid") + .setModelPath(defaultModelPath) + .build(appContext); + } catch (LeopardException e) { + for (int i = 0; i < error.length; i++) { + assertEquals(e.getMessageStack()[i], error[i]); + } + } + } +} \ No newline at end of file diff --git a/binding/android/LeopardTestApp/leopard-test-app/src/main/AndroidManifest.xml b/binding/android/LeopardTestApp/leopard-test-app/src/main/AndroidManifest.xml index fb3b5b9f..1c836d92 100644 --- a/binding/android/LeopardTestApp/leopard-test-app/src/main/AndroidManifest.xml +++ b/binding/android/LeopardTestApp/leopard-test-app/src/main/AndroidManifest.xml @@ -1,6 +1,5 @@ - + diff --git a/resources/spell-check/dict.txt b/resources/spell-check/dict.txt index fc5dbcc4..11cfc5eb 100644 --- a/resources/spell-check/dict.txt +++ b/resources/spell-check/dict.txt @@ -158,5 +158,6 @@ xcodeproj xcscheme xcschemes xcshareddata +xcuitest xcworkspace ytdlp \ No newline at end of file diff --git a/script/automation/browserstack.py b/script/automation/browserstack.py index 4a8d2378..7e0cd25b 100644 --- a/script/automation/browserstack.py +++ b/script/automation/browserstack.py @@ -17,7 +17,7 @@ 'Google Pixel 6 Pro-15.0' ], 'ios-min-max': [ - 'iPhone SE 2020-13', + 'iPhone SE 2022-15', 'iPhone 14 Pro-16', 'iPhone 14-18' ], @@ -64,7 +64,8 @@ def main(args: argparse.Namespace) -> None: 'app': app_response_json['app_url'], 'testSuite': test_response_json['test_suite_url'], 'project': args.project_name, - 'devices': devices_dict[args.devices] + 'devices': devices_dict[args.devices], + 'deviceLogs': True } while True: From 0beb32715373624c02077a5dfa9c0a971bb2a4e2 Mon Sep 17 00:00:00 2001 From: Albert Ho Date: Mon, 25 Nov 2024 17:18:38 -0800 Subject: [PATCH 4/6] try fix --- .../android/LeopardTestApp/leopard-test-app/build.gradle | 5 +++++ .../LeopardTestApp/leopard-test-app/proguard-rules.pro | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/binding/android/LeopardTestApp/leopard-test-app/build.gradle b/binding/android/LeopardTestApp/leopard-test-app/build.gradle index 7b15042f..4e89db07 100644 --- a/binding/android/LeopardTestApp/leopard-test-app/build.gradle +++ b/binding/android/LeopardTestApp/leopard-test-app/build.gradle @@ -142,6 +142,7 @@ dependencies { implementation 'com.google.android.material:material:1.8.0' implementation 'androidx.constraintlayout:constraintlayout:2.1.4' implementation 'com.google.code.gson:gson:2.10' + implementation 'com.google.errorprone:error_prone_annotations:2.36.0' implementation 'ai.picovoice:leopard-android:2.0.1' // Espresso UI Testing @@ -157,7 +158,11 @@ afterEvaluate { flavor -> tasks."merge${flavor.name.capitalize()}DebugAssets".dependsOn "${flavor.name}CopyParams" tasks."merge${flavor.name.capitalize()}ReleaseAssets".dependsOn "${flavor.name}CopyParams" + tasks."generate${flavor.name.capitalize()}ReleaseLintVitalReportModel".dependsOn "${flavor.name}copyParams" + tasks."lintVitalAnalyze${flavor.name.capitalize()}Release".dependsOn "${flavor.name}copyParams" tasks."merge${flavor.name.capitalize()}DebugAssets".dependsOn "${flavor.name}CopyAudio" tasks."merge${flavor.name.capitalize()}ReleaseAssets".dependsOn "${flavor.name}CopyAudio" + tasks."generate${flavor.name.capitalize()}ReleaseLintVitalReportModel".dependsOn "${flavor.name}copyAudio" + tasks."lintVitalAnalyze${flavor.name.capitalize()}Release".dependsOn "${flavor.name}copyAudio" } } diff --git a/binding/android/LeopardTestApp/leopard-test-app/proguard-rules.pro b/binding/android/LeopardTestApp/leopard-test-app/proguard-rules.pro index 158caf35..280693da 100644 --- a/binding/android/LeopardTestApp/leopard-test-app/proguard-rules.pro +++ b/binding/android/LeopardTestApp/leopard-test-app/proguard-rules.pro @@ -20,4 +20,8 @@ # hide the original source file name. #-renamesourcefileattribute SourceFile -keep class com.google.** { *; } --keep class com.microsoft.** { *; } \ No newline at end of file +-keep class com.microsoft.** { *; } + +-dontwarn com.google.errorprone.annotations.CheckReturnValue +-dontwarn com.google.errorprone.annotations.MustBeClosed +-dontwarn javax.lang.model.element.Modifier \ No newline at end of file From 2e43590992b326a271a5411859457d45ceb29972 Mon Sep 17 00:00:00 2001 From: Albert Ho Date: Mon, 25 Nov 2024 17:22:14 -0800 Subject: [PATCH 5/6] try fix --- .../android/LeopardTestApp/leopard-test-app/build.gradle | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/binding/android/LeopardTestApp/leopard-test-app/build.gradle b/binding/android/LeopardTestApp/leopard-test-app/build.gradle index 4e89db07..262eb03f 100644 --- a/binding/android/LeopardTestApp/leopard-test-app/build.gradle +++ b/binding/android/LeopardTestApp/leopard-test-app/build.gradle @@ -158,11 +158,11 @@ afterEvaluate { flavor -> tasks."merge${flavor.name.capitalize()}DebugAssets".dependsOn "${flavor.name}CopyParams" tasks."merge${flavor.name.capitalize()}ReleaseAssets".dependsOn "${flavor.name}CopyParams" - tasks."generate${flavor.name.capitalize()}ReleaseLintVitalReportModel".dependsOn "${flavor.name}copyParams" - tasks."lintVitalAnalyze${flavor.name.capitalize()}Release".dependsOn "${flavor.name}copyParams" + tasks."generate${flavor.name.capitalize()}ReleaseLintVitalReportModel".dependsOn "${flavor.name}CopyParams" + tasks."lintVitalAnalyze${flavor.name.capitalize()}Release".dependsOn "${flavor.name}CopyParams" tasks."merge${flavor.name.capitalize()}DebugAssets".dependsOn "${flavor.name}CopyAudio" tasks."merge${flavor.name.capitalize()}ReleaseAssets".dependsOn "${flavor.name}CopyAudio" - tasks."generate${flavor.name.capitalize()}ReleaseLintVitalReportModel".dependsOn "${flavor.name}copyAudio" - tasks."lintVitalAnalyze${flavor.name.capitalize()}Release".dependsOn "${flavor.name}copyAudio" + tasks."generate${flavor.name.capitalize()}ReleaseLintVitalReportModel".dependsOn "${flavor.name}CopyAudio" + tasks."lintVitalAnalyze${flavor.name.capitalize()}Release".dependsOn "${flavor.name}CopyAudio" } } From 2296196a5733f5af19f292e1e6bfb6b26bab794e Mon Sep 17 00:00:00 2001 From: Albert Ho Date: Tue, 26 Nov 2024 11:45:54 -0800 Subject: [PATCH 6/6] try fix --- .github/workflows/android-perf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml index f8e2440c..56296b36 100644 --- a/.github/workflows/android-perf.yml +++ b/.github/workflows/android-perf.yml @@ -30,7 +30,7 @@ jobs: include: - device: android-perf initPerformanceThresholdSec: 6.0 - procPerformanceThresholdSec: 1.0 + procPerformanceThresholdSec: 1.5 steps: - uses: actions/checkout@v3