From 042919850a469dcfc4f305299575c2c8a76904c3 Mon Sep 17 00:00:00 2001 From: Kwangsoo Yeo Date: Mon, 2 Dec 2024 15:28:33 -0800 Subject: [PATCH] v2.1 web --- .github/workflows/web-demos.yml | 6 ++ binding/web/.gitignore | 1 + binding/web/README.md | 8 +++ binding/web/package.json | 4 +- binding/web/scripts/setup_test.js | 12 ++++ binding/web/src/cheetah.ts | 17 +++-- binding/web/test/cheetah.test.ts | 115 ++++++++++++++++-------------- binding/web/yarn.lock | 20 +++--- demo/web/.gitignore | 2 +- demo/web/index.html | 4 +- demo/web/package.json | 5 +- demo/web/scripts/run_demo.js | 64 +++++++++++++++++ demo/web/yarn.lock | 26 ++++--- 13 files changed, 201 insertions(+), 83 deletions(-) create mode 100644 demo/web/scripts/run_demo.js diff --git a/.github/workflows/web-demos.yml b/.github/workflows/web-demos.yml index 7b09159c..6530a106 100644 --- a/.github/workflows/web-demos.yml +++ b/.github/workflows/web-demos.yml @@ -38,5 +38,11 @@ jobs: - name: Pre-build dependencies run: npm install yarn + # ************** REMOVE AFTER RELEASE ******************** + - name: Build Local Packages + run: yarn && yarn build + working-directory: binding/web + # ******************************************************** + - name: Install dependencies run: yarn install diff --git a/binding/web/.gitignore b/binding/web/.gitignore index ee7e40b6..169eea56 100644 --- a/binding/web/.gitignore +++ b/binding/web/.gitignore @@ -4,3 +4,4 @@ lib/pv_cheetah*.wasm cypress/fixtures/audio_samples/* test/cheetah_params*.js test/cheetah_params*.pv +test/test_data.json diff --git a/binding/web/README.md b/binding/web/README.md index 15041382..d5c85e94 100644 --- a/binding/web/README.md +++ b/binding/web/README.md @@ -191,6 +191,14 @@ Terminate `CheetahWorker` instance: await handle.terminate(); ``` +### Language Model + +The Cheetah Web SDK comes preloaded with a default English language model (`.pv` file). +Default models for other supported languages can be found in [lib/common](../../lib/common). + +Create custom language models using the [Picovoice Console](https://console.picovoice.ai/). Here you can train +language models with custom vocabulary and boost words in the existing vocabulary. + ## Demo For example usage refer to our [Web demo application](https://github.com/Picovoice/cheetah/tree/master/demo/web). diff --git a/binding/web/package.json b/binding/web/package.json index b10c163f..0dd5ac50 100644 --- a/binding/web/package.json +++ b/binding/web/package.json @@ -3,7 +3,7 @@ "description": "Cheetah Speech-to-Text engine for web browsers (via WebAssembly)", "author": "Picovoice Inc", "license": "Apache-2.0", - "version": "2.0.0", + "version": "2.1.0", "keywords": [ "cheetah", "web", @@ -35,7 +35,7 @@ "test-perf": "cypress run --spec test/cheetah_perf.test.ts" }, "dependencies": { - "@picovoice/web-utils": "=1.3.1" + "@picovoice/web-utils": "=1.4.3" }, "devDependencies": { "@babel/core": "^7.21.3", diff --git a/binding/web/scripts/setup_test.js b/binding/web/scripts/setup_test.js index 33134657..dd9f3014 100644 --- a/binding/web/scripts/setup_test.js +++ b/binding/web/scripts/setup_test.js @@ -15,6 +15,16 @@ const paramsSourceDirectory = join( 'common', ); +const testDataSource = join( + __dirname, + '..', + '..', + '..', + 'resources', + '.test', + 'test_data.json' +); + const sourceDirectory = join( __dirname, "..", @@ -30,6 +40,8 @@ try { fs.copyFileSync(join(paramsSourceDirectory, file), join(testDirectory, file)); }); + fs.copyFileSync(testDataSource, join(testDirectory, 'test_data.json')); + fs.mkdirSync(join(fixturesDirectory, 'audio_samples'), { recursive: true }); fs.readdirSync(join(sourceDirectory, 'audio_samples')).forEach(file => { fs.copyFileSync(join(sourceDirectory, 'audio_samples', file), join(fixturesDirectory, 'audio_samples', file)); diff --git a/binding/web/src/cheetah.ts b/binding/web/src/cheetah.ts index d83eb5d8..1adffc29 100644 --- a/binding/web/src/cheetah.ts +++ b/binding/web/src/cheetah.ts @@ -452,7 +452,7 @@ export class Cheetah { // A WebAssembly page has a constant size of 64KiB. -> 1MiB ~= 16 pages const memory = new WebAssembly.Memory({ initial: 3700 }); - const memoryBufferUint8 = new Uint8Array(memory.buffer); + let memoryBufferUint8 = new Uint8Array(memory.buffer); const pvError = new PvError(); @@ -551,7 +551,7 @@ export class Cheetah { throw new CheetahErrors.CheetahOutOfMemoryError('malloc failed: Cannot allocate memory'); } - const memoryBufferView = new DataView(memory.buffer); + let memoryBufferView = new DataView(memory.buffer); const status = await pv_cheetah_init( accessKeyAddress, @@ -559,6 +559,15 @@ export class Cheetah { endpointDurationSec, (enableAutomaticPunctuation) ? 1 : 0, objectAddressAddress); + + if (memoryBufferView.buffer.byteLength === 0) { + memoryBufferView = new DataView(memory.buffer); + } + + if (memoryBufferUint8.buffer.byteLength === 0) { + memoryBufferUint8 = new Uint8Array(memory.buffer); + } + if (status !== PV_STATUS_SUCCESS) { const messageStack = await Cheetah.getMessageStack( pv_get_error_stack, @@ -599,7 +608,7 @@ export class Cheetah { frameLength: frameLength, sampleRate: sampleRate, version: version, - + objectAddress: objectAddress, inputBufferAddress: inputBufferAddress, isEndpointAddress: isEndpointAddress, @@ -625,7 +634,7 @@ export class Cheetah { memoryBufferUint8: Uint8Array, ): Promise { const status = await pv_get_error_stack(messageStackAddressAddressAddress, messageStackDepthAddress); - if (status != PvStatus.SUCCESS) { + if (status !== PvStatus.SUCCESS) { throw pvStatusToException(status, "Unable to get Cheetah error state"); } diff --git a/binding/web/test/cheetah.test.ts b/binding/web/test/cheetah.test.ts index 2cfd9fc3..2496ecc6 100644 --- a/binding/web/test/cheetah.test.ts +++ b/binding/web/test/cheetah.test.ts @@ -1,19 +1,12 @@ import { Cheetah, CheetahWorker } from "../"; -import { CheetahError } from "../dist/types/cheetah_error"; +import { CheetahError } from "../dist/types/cheetah_errors"; +import testData from './test_data.json'; // @ts-ignore import cheetahParams from "./cheetah_params"; import { PvModel } from '@picovoice/web-utils'; -const ACCESS_KEY: string = Cypress.env("ACCESS_KEY"); - -const testParam = { - language: 'en', - audio_file: 'test.wav', - transcript: 'Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.', - punctuations: ['.'], - error_rate: 0.025, -}; +const ACCESS_KEY: string = Cypress.env('ACCESS_KEY'); const levenshteinDistance = (words1: string[], words2: string[]) => { const res = Array.from(Array(words1.length + 1), () => new Array(words2.length + 1)); @@ -37,8 +30,10 @@ const levenshteinDistance = (words1: string[], words2: string[]) => { const wordErrorRate = (reference: string, hypothesis: string, useCER = false): number => { const splitter = (useCER) ? '' : ' '; - const ed = levenshteinDistance(reference.split(splitter), hypothesis.split(splitter)); - return ed / reference.length; + const refWords = reference.split(splitter); + const hypWords = hypothesis.split(splitter); + const ed = levenshteinDistance(refWords, hypWords); + return ed / refWords.length; }; function delay(time: number) { @@ -131,7 +126,7 @@ const runProcTest = async ( model, { enableAutomaticPunctuation: enablePunctuation, - processErrorCallback: (error: string) => { + processErrorCallback: (error: CheetahError) => { reject(error); } } @@ -166,7 +161,7 @@ const runProcTest = async ( describe("Cheetah Binding", function () { it(`should return process and flush error message stack`, async () => { - let errors: [CheetahError] = []; + let errors: CheetahError[] = []; const runProcess = () => new Promise(async resolve => { const cheetah = await Cheetah.create( @@ -285,45 +280,59 @@ describe("Cheetah Binding", function () { }); }); - it(`should be able to process (${testParam.language}) (${instanceString})`, () => { - try { - cy.getFramesFromFile(`audio_samples/${testParam.audio_file}`).then( async pcm => { - const suffix = (testParam.language === 'en') ? '' : `_${testParam.language}`; - await runProcTest( - instance, - pcm, - testParam.punctuations, - testParam.transcript, - testParam.error_rate, - { - model: { publicPath: `/test/cheetah_params${suffix}.pv`, forceWrite: true }, - enablePunctuation: false, - useCER: (testParam.language === 'ja') - }); - }); - } catch (e) { - expect(e).to.be.undefined; - } - }); + for (const testParam of testData.tests.language_tests) { + it(`should be able to process (${testParam.language}) (${instanceString})`, () => { + try { + cy.getFramesFromFile(`audio_samples/${testParam.audio_file}`).then( + async pcm => { + const suffix = + testParam.language === 'en' ? '' : `_${testParam.language}`; + await runProcTest( + instance, + pcm, + testParam.punctuations, + testParam.transcript, + testParam.error_rate, + { + model: { + publicPath: `/test/cheetah_params${suffix}.pv`, + forceWrite: true, + }, + } + ); + } + ); + } catch (e) { + expect(e).to.be.undefined; + } + }); - it(`should be able to process with punctuation (${testParam.language}) (${instanceString})`, () => { - try { - cy.getFramesFromFile(`audio_samples/${testParam.audio_file}`).then( async pcm => { - const suffix = (testParam.language === 'en') ? '' : `_${testParam.language}`; - await runProcTest( - instance, - pcm, - testParam.punctuations, - testParam.transcript, - testParam.error_rate, - { - model: { publicPath: `/test/cheetah_params${suffix}.pv`, forceWrite: true }, - useCER: (testParam.language === 'ja') - }); - }); - } catch (e) { - expect(e).to.be.undefined; - } - }); + it(`should be able to process with punctuation (${testParam.language}) (${instanceString})`, () => { + try { + cy.getFramesFromFile(`audio_samples/${testParam.audio_file}`).then( + async pcm => { + const suffix = + testParam.language === 'en' ? '' : `_${testParam.language}`; + await runProcTest( + instance, + pcm, + testParam.punctuations, + testParam.transcript, + testParam.error_rate, + { + model: { + publicPath: `/test/cheetah_params${suffix}.pv`, + forceWrite: true, + }, + enablePunctuation: true, + } + ); + } + ); + } catch (e) { + expect(e).to.be.undefined; + } + }); + } } }); diff --git a/binding/web/yarn.lock b/binding/web/yarn.lock index 666e46ec..ea08a359 100644 --- a/binding/web/yarn.lock +++ b/binding/web/yarn.lock @@ -1100,12 +1100,12 @@ "@nodelib/fs.scandir" "2.1.5" fastq "^1.6.0" -"@picovoice/web-utils@=1.3.1": - version "1.3.1" - resolved "https://registry.yarnpkg.com/@picovoice/web-utils/-/web-utils-1.3.1.tgz#d417e98604a650b54a8e03669015ecf98c2383ec" - integrity sha512-jcDqdULtTm+yJrnHDjg64hARup+Z4wNkYuXHNx6EM8+qZkweBq9UA6XJrHAlUkPnlkso4JWjaIKhz3x8vZcd3g== +"@picovoice/web-utils@=1.4.3": + version "1.4.3" + resolved "https://registry.yarnpkg.com/@picovoice/web-utils/-/web-utils-1.4.3.tgz#1de0b20d6080c18d295c6df37c09d88bf7c4f555" + integrity sha512-7JN3YYsSD9Gtce6YKG3XqpX49dkeu7jTdbox7rHQA/X/Q3zxopXA9zlCKSq6EIjFbiX2iuzDKUx1XrFa3d8c0w== dependencies: - commander "^9.2.0" + commander "^10.0.1" "@rollup/plugin-babel@^6.0.3": version "6.0.4" @@ -1679,6 +1679,11 @@ combined-stream@^1.0.6, combined-stream@~1.0.6: dependencies: delayed-stream "~1.0.0" +commander@^10.0.1: + version "10.0.1" + resolved "https://registry.yarnpkg.com/commander/-/commander-10.0.1.tgz#881ee46b4f77d1c1dccc5823433aa39b022cbe06" + integrity sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug== + commander@^2.20.0: version "2.20.3" resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" @@ -1689,11 +1694,6 @@ commander@^5.1.0: resolved "https://registry.yarnpkg.com/commander/-/commander-5.1.0.tgz#46abbd1652f8e059bddaef99bbdcb2ad9cf179ae" integrity sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg== -commander@^9.2.0: - version "9.5.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-9.5.0.tgz#bc08d1eb5cedf7ccb797a96199d41c7bc3e60d30" - integrity sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ== - common-tags@^1.8.0: version "1.8.2" resolved "https://registry.yarnpkg.com/common-tags/-/common-tags-1.8.2.tgz#94ebb3c076d26032745fd54face7f688ef5ac9c6" diff --git a/demo/web/.gitignore b/demo/web/.gitignore index bdb2b261..20e5a64c 100644 --- a/demo/web/.gitignore +++ b/demo/web/.gitignore @@ -4,4 +4,4 @@ node_modules dist/ *.log .DS_Store -cheetah_params.js +models/* diff --git a/demo/web/index.html b/demo/web/index.html index 66800d85..50eea5c6 100644 --- a/demo/web/index.html +++ b/demo/web/index.html @@ -3,7 +3,7 @@ - +