From 67d9d02df657e55c33cd156972051d20bd86b064 Mon Sep 17 00:00:00 2001 From: Hiroshiba Date: Tue, 20 Jun 2023 04:20:56 +0900 Subject: [PATCH 1/3] =?UTF-8?q?Github=20Actions=E3=81=A7shell:=20bash?= =?UTF-8?q?=E3=82=92=E6=AF=8E=E5=9B=9E=E6=9B=B8=E3=81=8B=E3=81=AA=E3=81=8F?= =?UTF-8?q?=E3=81=A6=E3=82=82=E8=89=AF=E3=81=84=E3=82=88=E3=81=86=E3=81=AB?= =?UTF-8?q?=E3=81=99=E3=82=8B=20(#528)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build_and_deploy.yml | 19 +++-------------- .github/workflows/cargo-deny.yml | 3 +++ .github/workflows/download_test.yml | 7 +++---- .github/workflows/generate_document.yml | 3 +++ .github/workflows/labeler.yml | 3 +++ .github/workflows/psscriptanalyzer.yml | 3 +++ .github/workflows/test.yml | 23 ++++++++++----------- .github/workflows/typos.yml | 4 ++++ .github/workflows/update_rust_toolchain.yml | 4 +++- 9 files changed, 36 insertions(+), 33 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 5f48330e7..5c258922f 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -24,6 +24,9 @@ env: # Raw character weights are not public. # Skip uploading to GitHub Release on public repo. SKIP_UPLOADING_RELEASE_ASSET: ${{ secrets.SKIP_UPLOADING_RELEASE_ASSET || '1' }} +defaults: + run: + shell: bash jobs: build_and_deploy: environment: ${{ github.event.inputs.code_signing == 'true' && 'code_signing' || '' }} # コード署名用のenvironment @@ -124,7 +127,6 @@ jobs: targets: ${{ matrix.target }} - name: Install cross compiler for aarch64-unknown-linux-gnu if: matrix.target == 'aarch64-unknown-linux-gnu' - shell: bash run: | sudo apt update sudo apt install gcc-aarch64-linux-gnu g++-aarch64-linux-gnu @@ -134,17 +136,14 @@ jobs: ndk-version: r25b - name: Set path for android if: endsWith(matrix.target, '-linux-android') - shell: bash run: | echo "$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin" >> "$GITHUB_PATH" echo "AR_${{ matrix.target }}=llvm-ar" >> "$GITHUB_ENV" - name: Install cargo-binstall uses: taiki-e/install-action@cargo-binstall - name: Install cargo-edit - shell: bash run: cargo binstall cargo-edit@^0.11 --no-confirm --log-level debug - name: set cargo version - shell: bash run: | cargo set-version "$VERSION" --exclude voicevox_core_python_api --exclude download --exclude xtask if ${{ !!matrix.whl_local_version }}; then cargo set-version "$VERSION+"${{ matrix.whl_local_version }} -p voicevox_core_python_api; fi @@ -156,7 +155,6 @@ jobs: - name: build voicevox_core_python_api if: matrix.whl_local_version id: build-voicevox-core-python-api - shell: bash run: | pip install -r ./crates/voicevox_core_python_api/requirements.txt maturin build --manifest-path ./crates/voicevox_core_python_api/Cargo.toml --features ${{ matrix.features }}, --target ${{ matrix.target }} --release @@ -164,10 +162,8 @@ jobs: env: ORT_USE_CUDA: ${{ matrix.use_cuda }} - name: Set ASSET_NAME env var - shell: bash run: echo "ASSET_NAME=voicevox_core-${{ matrix.artifact_name }}-${{ env.VERSION }}" >> "$GITHUB_ENV" - name: Organize artifact - shell: bash run: | mkdir -p "artifact/${{ env.ASSET_NAME }}" cp -v crates/voicevox_core_c_api/include/voicevox_core.h "artifact/${{ env.ASSET_NAME }}" @@ -181,7 +177,6 @@ jobs: echo "${{ env.VERSION }}" > "artifact/${{ env.ASSET_NAME }}/VERSION" - name: Code signing (Windows) if: startsWith(matrix.os, 'windows') && github.event.inputs.code_signing == 'true' - shell: bash run: | bash build_util/codesign.bash "artifact/${{ env.ASSET_NAME }}/voicevox_core.dll" env: @@ -194,7 +189,6 @@ jobs: name: voicevox_core-${{ matrix.target }} path: artifact/${{ env.ASSET_NAME }} - name: Archive artifact - shell: bash run: | cd artifact 7z a "../${{ env.ASSET_NAME }}.zip" "${{ env.ASSET_NAME }}" @@ -221,7 +215,6 @@ jobs: runs-on: macos-12 steps: - name: Set ASSET_NAME env var - shell: bash run: echo "ASSET_NAME=voicevox_core-ios-xcframework-cpu-${{ env.VERSION }}" >> "$GITHUB_ENV" - uses: actions/download-artifact@v2 with: @@ -236,12 +229,10 @@ jobs: name: voicevox_core-aarch64-apple-ios path: artifact/voicevox_core-aarch64-apple-ios - name: Create fat binary - shell: bash run: | mkdir -p "artifact/voicevox_core-sim" lipo -create "artifact/voicevox_core-x86_64-apple-ios/libvoicevox_core.dylib" "artifact/voicevox_core-aarch64-apple-ios-sim/libvoicevox_core.dylib" -output "artifact/voicevox_core-sim/libvoicevox_core.dylib" - name: Create XCFramework - shell: bash run: | mkdir -p "artifact/${{ env.ASSET_NAME }}" xcodebuild -create-xcframework \ @@ -251,7 +242,6 @@ jobs: -headers "artifact/voicevox_core-aarch64-apple-ios/voicevox_core.h" \ -output "artifact/${{ env.ASSET_NAME }}/voicevox_core.xcframework" - name: Archive artifact - shell: bash run: | cd artifact/${{ env.ASSET_NAME }} 7z a "../../${{ env.ASSET_NAME }}.zip" "voicevox_core.xcframework" @@ -302,7 +292,6 @@ jobs: - uses: actions/checkout@v3 - name: Install cross compiler for aarch64-unknown-linux-gnu if: matrix.target == 'aarch64-unknown-linux-gnu' - shell: bash run: | sudo apt update sudo apt install gcc-aarch64-linux-gnu g++-aarch64-linux-gnu @@ -313,7 +302,6 @@ jobs: - name: Build downloader run: cargo build -vv --release -p download --target ${{ matrix.target }} - name: Rename the binary - shell: bash run: | case "$OS" in Windows) exe_suffix=.exe;; @@ -322,7 +310,6 @@ jobs: mv $"target/${{ matrix.target }}/release/download$exe_suffix" ./${{ matrix.name }} - name: Code signing (Windows) if: startsWith(matrix.os, 'windows') && github.event.inputs.code_signing == 'true' - shell: bash run: | bash build_util/codesign.bash ./${{ matrix.name }} env: diff --git a/.github/workflows/cargo-deny.yml b/.github/workflows/cargo-deny.yml index 76c9e412d..ac4978b9a 100644 --- a/.github/workflows/cargo-deny.yml +++ b/.github/workflows/cargo-deny.yml @@ -2,6 +2,9 @@ name: cargo-deny on: push: pull_request: +defaults: + run: + shell: bash jobs: cargo-deny: runs-on: ubuntu-20.04 diff --git a/.github/workflows/download_test.yml b/.github/workflows/download_test.yml index 161c9d6b2..4ae0f972d 100644 --- a/.github/workflows/download_test.yml +++ b/.github/workflows/download_test.yml @@ -9,6 +9,9 @@ on: - "crates/download/**" - "scripts/downloads/*" - ".github/workflows/download_test.yml" +defaults: + run: + shell: bash jobs: download-releases: strategy: @@ -296,18 +299,15 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Get latest version if: ${{ env.EXPECTED_VOICEVOX_CORE_VERSION == 'latest' }} - shell: bash run: | echo "EXPECTED_VOICEVOX_CORE_VERSION=$(gh release view --repo VOICEVOX/voicevox_core --json 'tagName' --jq '.tagName')" >> "$GITHUB_ENV" env: GITHUB_TOKEN: ${{ github.token }} - name: Check downloaded version - shell: bash run: | [ -e "${{ matrix.download_dir }}/VERSION" ] [ "$(cat "${{ matrix.download_dir }}/VERSION")" = "${{ env.EXPECTED_VOICEVOX_CORE_VERSION }}" ] - name: Check downloaded files - shell: bash run: | mapfile -t items < <(echo -n '${{ matrix.check_items }}') for item in "${items[@]}" @@ -317,7 +317,6 @@ jobs: [ -e "${{ matrix.download_dir }}"/${item} ] done - name: Check should not exists files - shell: bash run: | mapfile -t items < <(echo -n '${{ matrix.check_not_exists_items }}') for item in "${items[@]}" diff --git a/.github/workflows/generate_document.yml b/.github/workflows/generate_document.yml index 05cc825de..40fc29c24 100644 --- a/.github/workflows/generate_document.yml +++ b/.github/workflows/generate_document.yml @@ -4,6 +4,9 @@ on: branches: - main pull_request: +defaults: + run: + shell: bash jobs: generate_api_document: runs-on: ubuntu-latest diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index bc48a76a7..25d78ec6a 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -2,6 +2,9 @@ name: Issue Labeler on: issues: types: [opened] +defaults: + run: + shell: bash jobs: triage: diff --git a/.github/workflows/psscriptanalyzer.yml b/.github/workflows/psscriptanalyzer.yml index 5643c7cc0..148efcb43 100644 --- a/.github/workflows/psscriptanalyzer.yml +++ b/.github/workflows/psscriptanalyzer.yml @@ -6,6 +6,9 @@ on: paths: - '**.ps1' - '.github/workflows/psscriptanalyzer.yml' +defaults: + run: + shell: bash jobs: psscriptanalyzer: runs-on: ubuntu-22.04 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4d9c53f04..9e97cc22a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,6 +5,9 @@ on: - "*" - "**/*" pull_request: +defaults: + run: + shell: bash jobs: shellcheck: runs-on: ubuntu-22.04 @@ -102,7 +105,6 @@ jobs: # cargoのキャッシュが原因でテストが失敗する場合はバージョン部分をカウントアップすること key: "v2-cargo-test-cache-${{ matrix.features }}-${{ matrix.os }}" - name: Run cargo test - shell: bash run: cargo test -vv --features ,${{ matrix.features }} -- --include-ignored c-header: @@ -173,12 +175,10 @@ jobs: uses: jwlawson/actions-setup-cmake@v1.13 - name: Install build dependencies if: startsWith(matrix.os, 'ubuntu') - shell: bash run: | sudo apt-get update sudo apt-get install -y cmake - name: Build - shell: bash run: | cd example/cpp/unix cmake -S . -B build @@ -193,6 +193,9 @@ jobs: SOLUTION_FILE_PATH: example\cpp\windows\windows_example.sln # Configuration type to build. BUILD_CONFIGURATION: Debug + defaults: + run: + shell: pwsh steps: - uses: actions/checkout@v3 @@ -204,7 +207,7 @@ jobs: run: cargo build -p voicevox_core_c_api -vv - name: 必要なfileをexampleのディレクトリに移動させる run: | - mkdir example/cpp/windows/simple_tts/lib/x64 + mkdir -p example/cpp/windows/simple_tts/lib/x64 cp -v crates/voicevox_core_c_api/include/voicevox_core.h example/cpp/windows/simple_tts/ cp target/debug/voicevox_core.dll.lib example/cpp/windows/simple_tts/lib/x64/voicevox_core.lib @@ -237,14 +240,10 @@ jobs: uses: ./.github/actions/rust-toolchain-from-file - name: venv作成 uses: ./.github/actions/create-venv - - shell: bash - run: pip install -r ./crates/voicevox_core_python_api/requirements.txt - - shell: bash - run: cargo build -p voicevox_core_c_api -vv - - shell: bash - run: maturin build --manifest-path ./crates/voicevox_core_python_api/Cargo.toml --locked - - shell: bash - run: maturin develop --manifest-path ./crates/voicevox_core_python_api/Cargo.toml --locked + - run: pip install -r ./crates/voicevox_core_python_api/requirements.txt + - run: cargo build -p voicevox_core_c_api -vv + - run: maturin build --manifest-path ./crates/voicevox_core_python_api/Cargo.toml --locked + - run: maturin develop --manifest-path ./crates/voicevox_core_python_api/Cargo.toml --locked - name: 必要なDLLをカレントディレクトリにコピー run: | cp -v target/debug/build/onnxruntime-sys-*/out/onnxruntime_*/onnxruntime-*/lib/onnxruntime.dll . || true diff --git a/.github/workflows/typos.yml b/.github/workflows/typos.yml index 511e94c45..a42e05dcf 100644 --- a/.github/workflows/typos.yml +++ b/.github/workflows/typos.yml @@ -7,6 +7,10 @@ on: - "**" workflow_dispatch: +defaults: + run: + shell: bash + jobs: typos: runs-on: ubuntu-latest diff --git a/.github/workflows/update_rust_toolchain.yml b/.github/workflows/update_rust_toolchain.yml index f9002cf00..5fd567377 100644 --- a/.github/workflows/update_rust_toolchain.yml +++ b/.github/workflows/update_rust_toolchain.yml @@ -5,6 +5,9 @@ on: - main schedule: - cron: "0 0 * * *" +defaults: + run: + shell: bash jobs: update-rust-toolchain: runs-on: ubuntu-latest @@ -14,7 +17,6 @@ jobs: - name: set up stable rust uses: dtolnay/rust-toolchain@stable - name: update rust toolchain - shell: bash run: | rust_version=$(rustup run stable rustc --version | sed -r "s/.*([0-9]+\.[0-9]+\.[0-9]+[^ ]*).*/\1/") echo "$rust_version" > ./rust-toolchain From 00c43153d42032ad3044ebf9e780d6af5c3f2bf0 Mon Sep 17 00:00:00 2001 From: Hiroshiba Date: Wed, 21 Jun 2023 01:41:23 +0900 Subject: [PATCH 2/3] =?UTF-8?q?=E3=81=93=E3=81=AE=E3=83=AA=E3=83=9D?= =?UTF-8?q?=E3=82=B8=E3=83=88=E3=83=AA=E3=81=A7=E8=A3=BD=E5=93=81=E3=83=93?= =?UTF-8?q?=E3=83=AB=E3=83=89=E3=82=92=E5=8F=AF=E8=83=BD=E3=81=AB=E3=81=99?= =?UTF-8?q?=E3=82=8B=20(#527)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build_and_deploy.yml | 88 ++++++++++++++++++++++---- 1 file changed, 75 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 5c258922f..cbb3ab2ec 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -1,4 +1,10 @@ name: build and deploy workflow + +# 製品版もビルドできる。製品版ビルド時の違いは以下の3点 +# 1. production環境を使う +# 2. 製品版リポジトリのコードをmergeする +# 3. RESOURCEリポジトリからモデルをダウンロードして置き換える + on: workflow_dispatch: inputs: @@ -9,6 +15,12 @@ on: description: "コード署名する" type: boolean required: false + default: false + is_production: + description: "製品版をビルドする" + type: boolean + required: false + default: false release: types: - published @@ -18,18 +30,17 @@ on: - "*" - "**/*" env: + VOICEVOX_RESOURCE_VERSION: "0.15.0-preview.1" + VOICEVOX_FAT_RESOURCE_VERSION: "0.15.0-preview.0" # releaseタグ名か、workflow_dispatchでのバージョン名か、'0.0.0'が入る VERSION: ${{ github.event.release.tag_name || github.event.inputs.version || '0.0.0' }} - - # Raw character weights are not public. - # Skip uploading to GitHub Release on public repo. - SKIP_UPLOADING_RELEASE_ASSET: ${{ secrets.SKIP_UPLOADING_RELEASE_ASSET || '1' }} + PRODUCTION_REPOSITORY_TAG: "0.15.0-preview.0" # 製品版のタグ名 defaults: run: shell: bash jobs: build_and_deploy: - environment: ${{ github.event.inputs.code_signing == 'true' && 'code_signing' || '' }} # コード署名用のenvironment + environment: ${{ github.event.inputs.is_production == 'true' && 'production' || '' }} # 製品版のenvironment strategy: matrix: include: @@ -114,7 +125,22 @@ jobs: use_cuda: false runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v3 # 製品版ではない場合 + if: ${{ github.event.inputs.is_production != 'true' }} + - uses: actions/checkout@v3 # 製品版の場合 + if: ${{ github.event.inputs.is_production == 'true' }} + with: + fetch-depth: 0 # 全履歴取得 + token: ${{ secrets.PRODUCTION_GITHUB_TOKEN }} + - name: Merge production branch + if: github.event.inputs.is_production == 'true' + shell: bash + run: | + ( + git remote add private ${{ secrets.PRODUCTION_REPOSITORY_URL }} + git fetch private refs/tags/${{ env.PRODUCTION_REPOSITORY_TAG }} + git -c user.name=dummy -c user.email=dummy@dummy.dummy merge FETCH_HEAD + ) > /dev/null 2>&1 - name: Set up Python 3.8 if: matrix.whl_local_version uses: actions/setup-python@v4 @@ -139,6 +165,26 @@ jobs: run: | echo "$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin" >> "$GITHUB_PATH" echo "AR_${{ matrix.target }}=llvm-ar" >> "$GITHUB_ENV" + - name: Checkout VOICEVOX RESOURCE + if: github.event.inputs.is_production == 'true' + uses: actions/checkout@v3 + with: + repository: VOICEVOX/voicevox_resource + ref: ${{ env.VOICEVOX_RESOURCE_VERSION }} + path: download/resource + - name: Checkout VOICEVOX FAT RESOURCE + if: github.event.inputs.is_production == 'true' + uses: actions/checkout@v3 + with: + repository: VOICEVOX/voicevox_fat_resource + ref: ${{ env.VOICEVOX_FAT_RESOURCE_VERSION }} + path: download/fat_resource + - name: Raplace resource + if: github.event.inputs.is_production == 'true' + shell: bash + run: | + mv -f download/resource/core/README.md ./README.md + rm -r ./model; mv download/fat_resource/core/model ./model - name: Install cargo-binstall uses: taiki-e/install-action@cargo-binstall - name: Install cargo-edit @@ -148,7 +194,16 @@ jobs: cargo set-version "$VERSION" --exclude voicevox_core_python_api --exclude download --exclude xtask if ${{ !!matrix.whl_local_version }}; then cargo set-version "$VERSION+"${{ matrix.whl_local_version }} -p voicevox_core_python_api; fi - name: build voicevox_core_c_api - run: cargo build -p voicevox_core_c_api -vv --features ${{ matrix.features }}, --target ${{ matrix.target }} --release + shell: bash + run: | + function build() { + cargo build -p voicevox_core_c_api -vv --features ${{ matrix.features }}, --target ${{ matrix.target }} --release + } + if ${{ github.event.inputs.is_production != 'true' }}; then + build + else + build > /dev/null 2>&1 + fi env: RUSTFLAGS: -C panic=abort ORT_USE_CUDA: ${{ matrix.use_cuda }} @@ -157,7 +212,14 @@ jobs: id: build-voicevox-core-python-api run: | pip install -r ./crates/voicevox_core_python_api/requirements.txt - maturin build --manifest-path ./crates/voicevox_core_python_api/Cargo.toml --features ${{ matrix.features }}, --target ${{ matrix.target }} --release + function build() { + maturin build --manifest-path ./crates/voicevox_core_python_api/Cargo.toml --features ${{ matrix.features }}, --target ${{ matrix.target }} --release + } + if ${{ github.event.inputs.is_production != 'true' }}; then + build + else + build > /dev/null 2>&1 + fi echo "whl=$(find ./target/wheels -type f)" >> "$GITHUB_OUTPUT" env: ORT_USE_CUDA: ${{ matrix.use_cuda }} @@ -183,7 +245,7 @@ jobs: CERT_BASE64: ${{ secrets.CERT_BASE64 }} CERT_PASSWORD: ${{ secrets.CERT_PASSWORD }} - name: Upload artifact to build XCFramework - if: contains(matrix.target, 'ios') + if: contains(matrix.target, 'ios') uses: actions/upload-artifact@v2 with: name: voicevox_core-${{ matrix.target }} @@ -202,7 +264,7 @@ jobs: ${{ env.ASSET_NAME }}.zip target_commitish: ${{ github.sha }} - name: Upload Python whl to Release - if: env.VERSION != '0.0.0' && env.SKIP_UPLOADING_RELEASE_ASSET == '0' && matrix.whl_local_version + if: env.VERSION != '0.0.0' && matrix.whl_local_version uses: softprops/action-gh-release@v1 with: prerelease: true @@ -259,7 +321,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Upload to Release - if: env.VERSION != '0.0.0' && env.SKIP_UPLOADING_RELEASE_ASSET == '0' + if: env.VERSION != '0.0.0' uses: softprops/action-gh-release@v1 with: prerelease: true @@ -268,7 +330,7 @@ jobs: scripts/downloads/* target_commitish: ${{ github.sha }} deploy_precompiled_downloader: - environment: ${{ github.event.inputs.code_signing == 'true' && 'code_signing' || '' }} # コード署名用のenvironment + environment: ${{ github.event.inputs.is_production == 'true' && 'production' || '' }} # コード署名用のenvironment strategy: matrix: include: @@ -316,7 +378,7 @@ jobs: CERT_BASE64: ${{ secrets.CERT_BASE64 }} CERT_PASSWORD: ${{ secrets.CERT_PASSWORD }} - name: Upload to Release - if: env.VERSION != '0.0.0' && env.SKIP_UPLOADING_RELEASE_ASSET == '0' + if: env.VERSION != '0.0.0' uses: softprops/action-gh-release@v1 with: prerelease: true From 82d781c7d560c3bf0f78468024e1597401489e6e Mon Sep 17 00:00:00 2001 From: Nanashi Date: Mon, 3 Jul 2023 06:02:42 +0900 Subject: [PATCH 3/3] =?UTF-8?q?Add:=20Python=20API=E3=81=AE=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=82=92=E8=BF=BD=E5=8A=A0=20(#484)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Hiroshiba Co-authored-by: Hiroshiba Co-authored-by: Ryo Yamashita Co-authored-by: Qryxip Co-authored-by: qryxip --- .github/workflows/test.yml | 17 ++-- Cargo.lock | 6 ++ crates/test_util/.gitignore | 1 + crates/test_util/Cargo.toml | 8 ++ crates/test_util/build.rs | 86 ++++++++++++++++++- crates/test_util/src/lib.rs | 18 +++- crates/test_util/src/typing.rs | 41 +++++++++ crates/voicevox_core_c_api/Cargo.toml | 1 + .../tests/e2e/snapshots.toml | 2 - .../tests/e2e/testcases/compatible_engine.rs | 78 ++++++----------- crates/voicevox_core_python_api/Cargo.toml | 1 + .../voicevox_core_python_api/pyproject.toml | 3 + .../python/test/conftest.py | 49 +++++++++++ .../python/test/test_engine.py | 63 ++++++++++++++ .../python/voicevox_core/_rust.pyi | 18 ++-- .../requirements-dev.txt | 2 + .../requirements-test.txt | 2 + 17 files changed, 322 insertions(+), 74 deletions(-) create mode 100644 crates/test_util/.gitignore create mode 100644 crates/test_util/src/typing.rs create mode 100644 crates/voicevox_core_python_api/python/test/conftest.py create mode 100644 crates/voicevox_core_python_api/python/test/test_engine.py create mode 100644 crates/voicevox_core_python_api/requirements-dev.txt create mode 100644 crates/voicevox_core_python_api/requirements-test.txt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9e97cc22a..66dcfc1ba 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -244,16 +244,15 @@ jobs: - run: cargo build -p voicevox_core_c_api -vv - run: maturin build --manifest-path ./crates/voicevox_core_python_api/Cargo.toml --locked - run: maturin develop --manifest-path ./crates/voicevox_core_python_api/Cargo.toml --locked - - name: 必要なDLLをカレントディレクトリにコピー + - name: 必要なDLLをコピーしてpytestを実行 + working-directory: crates/voicevox_core_python_api run: | - cp -v target/debug/build/onnxruntime-sys-*/out/onnxruntime_*/onnxruntime-*/lib/onnxruntime.dll . || true - cp -v target/debug/build/onnxruntime-sys-*/out/onnxruntime_*/onnxruntime-*/lib/libonnxruntime.so.* . || true - cp -v target/debug/build/onnxruntime-sys-*/out/onnxruntime_*/onnxruntime-*/lib/libonnxruntime.*.dylib . || true - - name: '`maturin develop`でインストールした`voicevox_core_python_api`を実行' - shell: python - run: | - import voicevox_core - print(voicevox_core) + cp -v ../../target/debug/build/onnxruntime-sys-*/out/onnxruntime_*/onnxruntime-*/lib/onnxruntime.dll . || true + cp -v ../../target/debug/build/onnxruntime-sys-*/out/onnxruntime_*/onnxruntime-*/lib/libonnxruntime.so.* . || true + cp -v ../../target/debug/build/onnxruntime-sys-*/out/onnxruntime_*/onnxruntime-*/lib/libonnxruntime.*.dylib . || true + + pip install -r requirements-test.txt + pytest env: CARGO_TERM_COLOR: always diff --git a/Cargo.lock b/Cargo.lock index ecff628ce..14bc1bccd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3346,6 +3346,10 @@ dependencies = [ "anyhow", "async-std", "flate2", + "fs-err", + "once_cell", + "serde", + "serde_json", "surf", "tar", ] @@ -3852,6 +3856,7 @@ dependencies = [ "rstest", "serde", "serde_json", + "test_util", "thiserror", "toml 0.7.2", "tracing-subscriber", @@ -3872,6 +3877,7 @@ dependencies = [ "pyo3-log", "serde", "serde_json", + "test_util", "tracing", "voicevox_core", ] diff --git a/crates/test_util/.gitignore b/crates/test_util/.gitignore new file mode 100644 index 000000000..1269488f7 --- /dev/null +++ b/crates/test_util/.gitignore @@ -0,0 +1 @@ +data diff --git a/crates/test_util/Cargo.toml b/crates/test_util/Cargo.toml index 70af48a5a..ebf1712d7 100644 --- a/crates/test_util/Cargo.toml +++ b/crates/test_util/Cargo.toml @@ -4,9 +4,17 @@ version = "0.0.0" edition.workspace = true publish.workspace = true +[dependencies] +serde.workspace = true +serde_json.workspace = true +once_cell.workspace = true + [build-dependencies] anyhow.workspace = true async-std = { version = "1.12.0", features = ["attributes"] } +fs-err.workspace = true flate2 = "1.0.24" +serde.workspace = true +serde_json.workspace = true surf = "2.3.2" tar = "0.4.38" diff --git a/crates/test_util/build.rs b/crates/test_util/build.rs index 2f4f3597c..1d667cd9e 100644 --- a/crates/test_util/build.rs +++ b/crates/test_util/build.rs @@ -8,15 +8,22 @@ use async_std::io::ReadExt as _; use flate2::read::GzDecoder; use tar::Archive; +#[path = "src/typing.rs"] +mod typing; + const DIC_DIR_NAME: &str = "open_jtalk_dic_utf_8-1.11"; #[async_std::main] async fn main() -> anyhow::Result<()> { - let out_dir = &PathBuf::from(env::var_os("OUT_DIR").unwrap()); - download_open_jtalk_dict(out_dir).await + let mut dist = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); + dist.push("data"); + download_open_jtalk_dict(&dist).await?; + generate_example_data_json(&dist)?; + Ok(()) } -async fn download_open_jtalk_dict(out_dir: &Path) -> anyhow::Result<()> { +/// OpenJTalkの辞書をダウンロードして展開する。 +async fn download_open_jtalk_dict(dist: &Path) -> anyhow::Result<()> { let download_url = format!( "https://github.com/r9y9/open_jtalk/releases/download/v1.11.1/{DIC_DIR_NAME}.tar.gz" ); @@ -30,6 +37,77 @@ async fn download_open_jtalk_dict(out_dir: &Path) -> anyhow::Result<()> { let dict_tar = GzDecoder::new(&body_bytes[..]); let mut dict_archive = Archive::new(dict_tar); - dict_archive.unpack(out_dir)?; + dict_archive.unpack(dist)?; + Ok(()) +} + +/// テストデータのJSONを生成する。 +fn generate_example_data_json(dist: &Path) -> anyhow::Result<()> { + let test_data = typing::ExampleData { + speaker_id: 0, + + duration: typing::DurationExampleData { + length: 8, + // 「t e s u t o」 + phoneme_vector: vec![0, 37, 14, 35, 6, 37, 30, 0], + result: vec![ + 0.9537022, + 0.046877652, + 0.11338878, + 0.06429571, + 0.07507616, + 0.08266081, + 0.1571679, + 0.64980185, + ], + }, + intonation: typing::IntonationExampleData { + length: 5, + + vowel_phoneme_vector: vec![0, 14, 6, 30, 0], + consonant_phoneme_vector: vec![-1, 37, 35, 37, -1], + start_accent_vector: vec![0, 1, 0, 0, 0], + end_accent_vector: vec![0, 1, 0, 0, 0], + + start_accent_phrase_vector: vec![0, 1, 0, 0, 0], + + end_accent_phrase_vector: vec![0, 0, 0, 1, 0], + + result: vec![5.0591826, 5.905218, 5.846999, 5.565851, 5.528879], + }, + decode: typing::DecodeExampleData { + f0_length: 69, + phoneme_size: 45, + f0_vector: { + let mut f0 = [0.; 69]; + f0[9..24].fill(5.905218); + f0[37..60].fill(5.565851); + f0.to_vec() + }, + phoneme_vector: { + let mut phoneme = [0.; 45 * 69]; + let mut set_one = |index, range| { + for i in range { + phoneme[(i * 45 + index) as usize] = 1.; + } + }; + set_one(0, 0..9); + set_one(37, 9..13); + set_one(14, 13..24); + set_one(35, 24..30); + set_one(6, 30..37); + set_one(37, 37..45); + set_one(30, 45..60); + set_one(0, 60..69); + phoneme.to_vec() + }, + }, + }; + + fs_err::write( + dist.join("example_data.json"), + serde_json::to_string(&test_data)?, + )?; + Ok(()) } diff --git a/crates/test_util/src/lib.rs b/crates/test_util/src/lib.rs index 981d68346..15259b8d3 100644 --- a/crates/test_util/src/lib.rs +++ b/crates/test_util/src/lib.rs @@ -1 +1,17 @@ -pub const OPEN_JTALK_DIC_DIR: &str = concat!(env!("OUT_DIR"), "/open_jtalk_dic_utf_8-1.11"); +mod typing; +use once_cell::sync::Lazy; +pub use typing::*; + +pub const OPEN_JTALK_DIC_DIR: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/data/open_jtalk_dic_utf_8-1.11" +); + +pub const EXAMPLE_DATA_JSON: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/data/example_data.json" +)); + +pub static EXAMPLE_DATA: Lazy = Lazy::new(|| { + serde_json::from_str(EXAMPLE_DATA_JSON).expect("failed to parse example_data.json") +}); diff --git a/crates/test_util/src/typing.rs b/crates/test_util/src/typing.rs new file mode 100644 index 000000000..1d10c9cb9 --- /dev/null +++ b/crates/test_util/src/typing.rs @@ -0,0 +1,41 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize)] +pub struct DurationExampleData { + pub length: i64, + + pub phoneme_vector: Vec, + + pub result: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct IntonationExampleData { + pub length: i64, + + pub vowel_phoneme_vector: Vec, + pub consonant_phoneme_vector: Vec, + pub start_accent_vector: Vec, + pub end_accent_vector: Vec, + pub start_accent_phrase_vector: Vec, + pub end_accent_phrase_vector: Vec, + + pub result: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct DecodeExampleData { + pub f0_length: i64, + pub phoneme_size: i64, + pub f0_vector: Vec, + pub phoneme_vector: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ExampleData { + pub speaker_id: i64, + + pub duration: DurationExampleData, + pub intonation: IntonationExampleData, + pub decode: DecodeExampleData, +} diff --git a/crates/voicevox_core_c_api/Cargo.toml b/crates/voicevox_core_c_api/Cargo.toml index ac529588a..b8e75a0f0 100644 --- a/crates/voicevox_core_c_api/Cargo.toml +++ b/crates/voicevox_core_c_api/Cargo.toml @@ -45,5 +45,6 @@ process_path.workspace = true regex.workspace = true rstest = "0.15.0" serde.workspace = true +test_util.workspace = true toml = "0.7.2" typetag = "0.2.5" diff --git a/crates/voicevox_core_c_api/tests/e2e/snapshots.toml b/crates/voicevox_core_c_api/tests/e2e/snapshots.toml index f7dd1bf7d..de0e53bff 100644 --- a/crates/voicevox_core_c_api/tests/e2e/snapshots.toml +++ b/crates/voicevox_core_c_api/tests/e2e/snapshots.toml @@ -1,6 +1,4 @@ [compatible_engine] -yukarin_s_forward = [0.9537022, 0.046877652, 0.11338878, 0.06429571, 0.07507616, 0.08266081, 0.1571679, 0.64980185] -yukarin_sa_forward = [5.0591826, 5.905218, 5.846999, 5.565851, 5.528879] stderr.windows = ''' {windows-video-cards} ''' diff --git a/crates/voicevox_core_c_api/tests/e2e/testcases/compatible_engine.rs b/crates/voicevox_core_c_api/tests/e2e/testcases/compatible_engine.rs index 971951f15..3a369539e 100644 --- a/crates/voicevox_core_c_api/tests/e2e/testcases/compatible_engine.rs +++ b/crates/voicevox_core_c_api/tests/e2e/testcases/compatible_engine.rs @@ -7,6 +7,8 @@ use libloading::Library; use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; +use test_util::EXAMPLE_DATA; + use crate::{ assert_cdylib::{self, case, Utf8Output}, float_assert, snapshots, @@ -50,18 +52,17 @@ impl assert_cdylib::TestCase for TestCase { assert!(initialize(false, 0, false)); - assert!(!is_model_loaded(SPEAKER_ID)); - assert!(load_model(SPEAKER_ID)); - assert!(is_model_loaded(SPEAKER_ID)); + assert!(!is_model_loaded(EXAMPLE_DATA.speaker_id)); + assert!(load_model(EXAMPLE_DATA.speaker_id)); + assert!(is_model_loaded(EXAMPLE_DATA.speaker_id)); // テスト用テキストは"t e s u t o" - let phoneme_length = { let mut phoneme_length = [0.; 8]; assert!(yukarin_s_forward( - 8, - [0, 37, 14, 35, 6, 37, 30, 0].as_mut_ptr(), - &mut { SPEAKER_ID } as *mut i64, + EXAMPLE_DATA.duration.length, + EXAMPLE_DATA.duration.phoneme_vector.as_ptr() as *mut i64, + &mut { EXAMPLE_DATA.speaker_id } as *mut i64, phoneme_length.as_mut_ptr(), )); phoneme_length @@ -70,50 +71,27 @@ impl assert_cdylib::TestCase for TestCase { let intonation_list = { let mut intonation_list = [0.; 5]; assert!(yukarin_sa_forward( - 5, - [0, 14, 6, 30, 0].as_mut_ptr(), - [-1, 37, 35, 37, -1].as_mut_ptr(), - [0, 1, 0, 0, 0].as_mut_ptr(), - [0, 1, 0, 0, 0].as_mut_ptr(), - [0, 1, 0, 0, 0].as_mut_ptr(), - [0, 0, 0, 1, 0].as_mut_ptr(), - &mut { SPEAKER_ID } as *mut i64, + EXAMPLE_DATA.intonation.length, + EXAMPLE_DATA.intonation.vowel_phoneme_vector.as_ptr() as *mut i64, + EXAMPLE_DATA.intonation.consonant_phoneme_vector.as_ptr() as *mut i64, + EXAMPLE_DATA.intonation.start_accent_vector.as_ptr() as *mut i64, + EXAMPLE_DATA.intonation.end_accent_vector.as_ptr() as *mut i64, + EXAMPLE_DATA.intonation.start_accent_phrase_vector.as_ptr() as *mut i64, + EXAMPLE_DATA.intonation.end_accent_phrase_vector.as_ptr() as *mut i64, + &mut { EXAMPLE_DATA.speaker_id } as *mut i64, intonation_list.as_mut_ptr(), )); intonation_list }; let wave = { - let mut wave = [0.; 256 * F0_LENGTH]; + let mut wave = vec![0.; 256 * EXAMPLE_DATA.decode.f0_length as usize]; assert!(decode_forward( - F0_LENGTH as _, - PHONEME_SIZE as _, - { - let mut f0 = [0.; F0_LENGTH]; - f0[9..24].fill(5.905218); - f0[37..60].fill(5.565851); - f0 - } - .as_mut_ptr(), - { - let mut phoneme = [0.; PHONEME_SIZE * F0_LENGTH]; - let mut set_one = |index, range| { - for i in range { - phoneme[i * PHONEME_SIZE + index] = 1.; - } - }; - set_one(0, 0..9); - set_one(37, 9..13); - set_one(14, 13..24); - set_one(35, 24..30); - set_one(6, 30..37); - set_one(37, 37..45); - set_one(30, 45..60); - set_one(0, 60..69); - phoneme - } - .as_mut_ptr(), - &mut { SPEAKER_ID } as *mut i64, + EXAMPLE_DATA.decode.f0_length, + EXAMPLE_DATA.decode.phoneme_size, + EXAMPLE_DATA.decode.f0_vector.as_ptr() as *mut f32, + EXAMPLE_DATA.decode.phoneme_vector.as_ptr() as *mut f32, + &mut { EXAMPLE_DATA.speaker_id } as *mut i64, wave.as_mut_ptr(), )); wave @@ -125,17 +103,13 @@ impl assert_cdylib::TestCase for TestCase { supported_devices, ); - float_assert::close_l1(&phoneme_length, &SNAPSHOTS.yukarin_s_forward, 0.01); - float_assert::close_l1(&intonation_list, &SNAPSHOTS.yukarin_sa_forward, 0.01); + float_assert::close_l1(&phoneme_length, &EXAMPLE_DATA.duration.result, 0.01); + float_assert::close_l1(&intonation_list, &EXAMPLE_DATA.intonation.result, 0.01); assert!(wave.iter().copied().all(f32::is_normal)); finalize(); - return Ok(()); - - const SPEAKER_ID: i64 = 0; - const F0_LENGTH: usize = 69; - const PHONEME_SIZE: usize = 45; + Ok(()) } fn assert_output(&self, output: Utf8Output) -> AssertResult { @@ -153,8 +127,6 @@ static SNAPSHOTS: Lazy = snapshots::section!(compatible_engine); #[derive(Deserialize)] struct Snapshots { - pub(crate) yukarin_s_forward: [f32; 8], - pub(crate) yukarin_sa_forward: [f32; 5], #[serde(deserialize_with = "snapshots::deserialize_platform_specific_snapshot")] stderr: String, } diff --git a/crates/voicevox_core_python_api/Cargo.toml b/crates/voicevox_core_python_api/Cargo.toml index 87aec1341..37d97bc20 100644 --- a/crates/voicevox_core_python_api/Cargo.toml +++ b/crates/voicevox_core_python_api/Cargo.toml @@ -24,6 +24,7 @@ serde.workspace = true serde_json.workspace = true tracing.workspace = true voicevox_core.workspace = true +test_util.workspace = true [build-dependencies] anyhow.workspace = true diff --git a/crates/voicevox_core_python_api/pyproject.toml b/crates/voicevox_core_python_api/pyproject.toml index 7c143ded1..0cba4aeed 100644 --- a/crates/voicevox_core_python_api/pyproject.toml +++ b/crates/voicevox_core_python_api/pyproject.toml @@ -7,6 +7,9 @@ dependencies = ["numpy", "pydantic>=1.9.2,<2"] requires = ["maturin>=0.13.2,<0.14"] build-backend = "maturin" +[tool.isort] +profile = "black" + [tool.maturin] bindings = "pyo3" skip-auditwheel = true # Linuxでlibonnxruntime.so.*の不在を許してもらう diff --git a/crates/voicevox_core_python_api/python/test/conftest.py b/crates/voicevox_core_python_api/python/test/conftest.py new file mode 100644 index 000000000..ca947f850 --- /dev/null +++ b/crates/voicevox_core_python_api/python/test/conftest.py @@ -0,0 +1,49 @@ +import json +import os +from dataclasses import dataclass +from pathlib import Path +from typing import List, TypedDict + +import numpy as np +import pytest + +root_dir = Path(os.path.dirname(os.path.abspath(__file__))) + + +class DurationExampleData(TypedDict): + length: int + phoneme_vector: List[int] + result: List[float] + + +class IntonationExampleData(TypedDict): + length: int + vowel_phoneme_vector: List[int] + consonant_phoneme_vector: List[int] + start_accent_vector: List[int] + end_accent_vector: List[int] + start_accent_phrase_vector: List[int] + end_accent_phrase_vector: List[int] + result: List[float] + + +class DecodeExampleData(TypedDict): + f0_length: int + phoneme_size: int + f0_vector: List[float] + phoneme_vector: List[float] + + +class ExampleData(TypedDict): + speaker_id: int + duration: DurationExampleData + intonation: IntonationExampleData + decode: DecodeExampleData + + +@pytest.fixture(scope="session") +def example_data() -> ExampleData: + with ( + root_dir.parent.parent.parent / "test_util" / "data" / "example_data.json" + ).open() as f: + return json.load(f) diff --git a/crates/voicevox_core_python_api/python/test/test_engine.py b/crates/voicevox_core_python_api/python/test/test_engine.py new file mode 100644 index 000000000..94212c1dd --- /dev/null +++ b/crates/voicevox_core_python_api/python/test/test_engine.py @@ -0,0 +1,63 @@ +import numpy as np +from conftest import ExampleData +from voicevox_core import VoicevoxCore + + +# crates/voicevox_core_c_api/tests/e2e/testcases/compatible_engine.rs と同じ。 +# crates/voicevox_core_c_api/tests/e2e/testcases/compatible_engine_load_model_before_initialize.rs +# （コア初期化前にモデルをロードするとエラーになる）は、コアを初期化せずにモデルをロードすることが出来ないため、 +# Python API版でのテストはしない。 +def test_engine(example_data: ExampleData): + core = VoicevoxCore() + speaker_id = example_data["speaker_id"] + assert not core.is_model_loaded(speaker_id) + core.load_model(speaker_id) + assert core.is_model_loaded(speaker_id) + + duration = core.predict_duration( + np.array(example_data["duration"]["phoneme_vector"], dtype=np.int64), + speaker_id, + ) + + intonation = core.predict_intonation( + example_data["intonation"]["length"], + np.array(example_data["intonation"]["vowel_phoneme_vector"], dtype=np.int64), + np.array( + example_data["intonation"]["consonant_phoneme_vector"], dtype=np.int64 + ), + np.array(example_data["intonation"]["start_accent_vector"], dtype=np.int64), + np.array(example_data["intonation"]["end_accent_vector"], dtype=np.int64), + np.array( + example_data["intonation"]["start_accent_phrase_vector"], dtype=np.int64 + ), + np.array( + example_data["intonation"]["end_accent_phrase_vector"], dtype=np.int64 + ), + speaker_id, + ) + + wave = core.decode( + example_data["decode"]["f0_length"], + example_data["decode"]["phoneme_size"], + np.array(example_data["decode"]["f0_vector"], dtype=np.float32), + np.array(example_data["decode"]["phoneme_vector"], dtype=np.float32), + speaker_id, + ) + + check_float_array_near( + duration, np.array(example_data["duration"]["result"], dtype=np.float32), 0.01 + ) + check_float_array_near( + intonation, + np.array(example_data["intonation"]["result"], dtype=np.float32), + 0.01, + ) + + assert not np.isnan(wave).any() + assert not np.isinf(wave).any() + + +def check_float_array_near(a: np.ndarray, b: np.ndarray, max_abs_diff: float): + assert a.dtype == b.dtype + assert a.shape == b.shape + assert np.max(np.abs(a - b)) <= max_abs_diff diff --git a/crates/voicevox_core_python_api/python/voicevox_core/_rust.pyi b/crates/voicevox_core_python_api/python/voicevox_core/_rust.pyi index c34e407f1..b83f5e128 100644 --- a/crates/voicevox_core_python_api/python/voicevox_core/_rust.pyi +++ b/crates/voicevox_core_python_api/python/voicevox_core/_rust.pyi @@ -3,8 +3,13 @@ from typing import Final, List, Literal, Union import numpy as np from numpy.typing import NDArray - -from voicevox_core import AccelerationMode, AccentPhrase, AudioQuery, Meta, SupportedDevices +from voicevox_core import ( + AccelerationMode, + AccentPhrase, + AudioQuery, + Meta, + SupportedDevices, +) METAS: Final[List[Meta]] SUPPORTED_DEVICES: Final[SupportedDevices] @@ -188,7 +193,8 @@ class VoicevoxCore: :class:`List` [:class:`AccentPhrase`] """ ... - def mora_length( self, + def mora_length( + self, accent_phrases: List[AccentPhrase], speaker_id: int, ) -> List[AccentPhrase]: @@ -206,7 +212,8 @@ class VoicevoxCore: :class:`List` [:class:`AccentPhrase`] """ ... - def mora_pitch( self, + def mora_pitch( + self, accent_phrases: List[AccentPhrase], speaker_id: int, ) -> List[AccentPhrase]: @@ -224,7 +231,8 @@ class VoicevoxCore: :class:`List` [:class:`AccentPhrase`] """ ... - def mora_data( self, + def mora_data( + self, accent_phrases: List[AccentPhrase], speaker_id: int, ) -> List[AccentPhrase]: diff --git a/crates/voicevox_core_python_api/requirements-dev.txt b/crates/voicevox_core_python_api/requirements-dev.txt new file mode 100644 index 000000000..72fde7f15 --- /dev/null +++ b/crates/voicevox_core_python_api/requirements-dev.txt @@ -0,0 +1,2 @@ +black==23.3.0 +isort==5.12.0 diff --git a/crates/voicevox_core_python_api/requirements-test.txt b/crates/voicevox_core_python_api/requirements-test.txt new file mode 100644 index 000000000..9e24dd984 --- /dev/null +++ b/crates/voicevox_core_python_api/requirements-test.txt @@ -0,0 +1,2 @@ +pytest==7.3.1 +numpy==1.24.3