Update fork with latest changes from Google master

.github/workflows/cifuzz.yml

@@ -0,0 +1,26 @@
+name: CIFuzz
+on: [pull_request]
+jobs:
+  Fuzzing:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Build Fuzzers
+      id: build
+      uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
+      with:
+        oss-fuzz-project-name: 'sentencepiece'
+        dry-run: false
+        language: c++
+    - name: Run Fuzzers
+      uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
+      with:
+        oss-fuzz-project-name: 'sentencepiece'
+        fuzz-seconds: 300
+        dry-run: false
+        language: c++
+    - name: Upload Crash
+      uses: actions/upload-artifact@v3
+      if: failure() && steps.build.outcome == 'success'
+      with:
+        name: artifacts
+        path: ./out/artifacts

.github/workflows/cmake.yml

@@ -0,0 +1,77 @@
+name: CI for general build
+
+on:
+  push:
+    branches: [ master ]
+    tags:
+      - 'v*'
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, ubuntu-20.04, windows-latest, macOS-11 ]
+        arch: [ x64 ]
+        include:
+          - os: windows-latest
+            arch: x86
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.x'
+        architecture: ${{matrix.arch}}
+
+    - name: Config for Windows
+      if: runner.os == 'Windows'
+      run: |
+        if ("${{matrix.arch}}" -eq "x64") {
+          $msbuildPlatform = "x64"
+        } else {
+          $msbuildPlatform = "Win32"
+        }
+        cmake -A $msbuildPlatform -B ${{github.workspace}}/build -DSPM_BUILD_TEST=ON -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root
+
+    - name: Config for Unix
+      if: runner.os != 'Windows'
+      run: cmake -B ${{github.workspace}}/build -DSPM_BUILD_TEST=ON -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root
+
+    - name: Build
+      run: cmake --build ${{github.workspace}}/build --config Release --target install --parallel 8
+
+    - name: Test
+      working-directory: ${{github.workspace}}/build
+      run: ctest -C Release --output-on-failure
+
+    - name: Package
+      working-directory: ${{github.workspace}}/build
+      run: cpack
+
+    - name: Build Python wrapper
+      working-directory: ${{github.workspace}}/python
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel twine
+        python setup.py test
+        python setup.py bdist_wheel
+
+    - name: Upload artifcacts
+      uses: actions/upload-artifact@v3
+      with:
+        path: ./build/*.7z
+
+    - name: Upload Release Assets
+      if: startsWith(github.ref, 'refs/tags/')
+      uses: svenstaro/upload-release-action@v2
+      with:
+        repo_token: ${{ secrets.GITHUB_TOKEN }}
+        file: ./build/*.7z
+        tag: ${{ github.ref }}
+        overwrite: true
+        prerelease: true
+        file_glob: true
+        body: "This is my release text"

.github/workflows/cross_build.yml

@@ -0,0 +1,41 @@
+name: CrossBuild
+
+on:
+  push:
+    branches: [ master ]
+    tags:
+      - 'v*'
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [ i686, arm, aarch64, riscv64, powerpc, powerpc64, powerpc64le, s390x, sparc64, m68k, sh4, alpha ]
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Install cross tools
+      run: |
+        sudo apt-get install -y sudo qemu-user gdb zstd dwarfdump {gcc,g++}-10-{i686,aarch64,riscv64,powerpc,powerpc64,powerpc64le,s390x,sparc64,m68k,sh4,alpha}-linux-gnu {gcc,g++}-10-arm-linux-gnueabihf
+        sudo ln -sf /usr/bin/arm-linux-gnueabihf-gcc-10 /usr/bin/arm-linux-gnu-gcc-10
+        sudo ln -sf /usr/bin/arm-linux-gnueabihf-g++-10 /usr/bin/arm-linux-gnu-g++-10
+        sudo ln -sf /usr/arm-linux-gnueabihf /usr/arm-linux-gnu
+
+    - name: Build
+      run: |
+        mkdir -p ${{github.workspace}}/build
+        cd ${{github.workspace}}/build
+        env CXX=/usr/bin/${{matrix.arch}}-linux-gnu-g++-10 CC=/usr/bin/${{matrix.arch}}-linux-gnu-gcc-10 cmake .. -DSPM_BUILD_TEST=ON -DSPM_ENABLE_SHARED=OFF -DCMAKE_FIND_ROOT_PATH=/usr/${{matrix.arch}}-linux-gnu -DSPM_CROSS_SYSTEM_PROCESSOR=${{matrix.arch}}
+        make -j$(nproc)
+
+    - name: Test on QEMU
+      if: matrix.arch != 'sparc64' && matrix.arch != 'm68k' && matrix.arch != 'sh4'
+      run: |
+        cd ${{github.workspace}}/build
+        qemu_arch=`echo ${{matrix.arch}} | sed -e s/powerpc/ppc/ -e s/686/386/`
+        qemu-${qemu_arch} -L /usr/${{matrix.arch}}-linux-gnu src/spm_test

.github/workflows/wheel.yml

@@ -0,0 +1,147 @@
+name: Build Wheels
+
+on:
+  push:
+    branches: [ master ]
+    tags:
+      - 'v*'
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build_wheels:
+    outputs:
+      digests-linux: ${{ steps.hash-linux.outputs.digests }}
+      digests-macos: ${{ steps.hash-macos.outputs.digests }}
+      digests-windows: ${{ steps.hash-windows.outputs.digests }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macOS-11]
+    runs-on: ${{ matrix.os }}
+    name: Build wheels on ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
+
+      - name: Set up QEMU
+        if: runner.os == 'Linux'
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: arm64
+
+      - name: Build for Windows
+        if: runner.os == 'Windows'
+        run: |
+          cmake -A Win32 -B ${{github.workspace}}/build_win32 -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root_win32
+          cmake --build ${{github.workspace}}/build_win32 --config Release --target install --parallel 8
+          cmake -A x64   -B ${{github.workspace}}/build_amd64 -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root_amd64
+          cmake --build ${{github.workspace}}/build_amd64 --config Release --target install --parallel 8
+
+      - name: Build for Mac
+        if: runner.os == 'macOS'
+        run: |
+          cmake -B ${{github.workspace}}/build -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root
+          cmake --build ${{github.workspace}}/build --config Release --target install --parallel 8
+        env:
+          CMAKE_OSX_ARCHITECTURES: arm64;x86_64
+
+      - name: Install cibuildwheel
+        working-directory: ${{github.workspace}}/python
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine
+          python -m pip install cibuildwheel==2.12.0
+
+      - name: Build wheels
+        working-directory: ${{github.workspace}}/python
+        run: python -m cibuildwheel --output-dir wheelhouse
+        env:
+          CIBW_ARCHS_LINUX: auto aarch64
+          CIBW_ARCHS_MACOS: x86_64 universal2 arm64
+          CIBW_SKIP: "pp* *-musllinux_*"
+          CIBW_BUILD_VERBOSITY: 1
+
+      - name: Build sdist archive
+        working-directory: ${{github.workspace}}/python
+        run: sh build_sdist.sh
+
+      - name: Fetch sdist archive
+        uses: tj-actions/glob@v17
+        id: sdist
+        with:
+          files: ./python/dist/*.tar.gz
+
+      - name: Build wheel from sdist
+        run: python -m pip wheel "${{ steps.sdist.outputs.paths }}" --verbose
+
+      - name: Copy sdist
+        working-directory: ${{github.workspace}}/python
+        if: runner.os == 'macOS'
+        run: cp -f dist/*.tar.gz wheelhouse/
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v3
+        with:
+          path: |
+            ./python/wheelhouse/*.whl
+            ./python/wheelhouse/*.tar.gz
+
+      - name: Upload wheel release
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: ./python/wheelhouse/*
+          tag: ${{ github.ref }}
+          overwrite: true
+          prerelease: true
+          file_glob: true
+
+      - name: Generate SLSA subjects - Macos
+        id: hash-macos
+        if: runner.os == 'macOS'
+        run: echo "digests=$(shasum -a 256 ./python/wheelhouse/* | base64)" >> $GITHUB_OUTPUT
+
+      - name: Generate SLSA subjects - Linux
+        id: hash-linux
+        if: runner.os == 'Linux'
+        run: echo "digests=$(sha256sum ./python/wheelhouse/* | base64 -w0)" >> $GITHUB_OUTPUT
+
+      - name: Generate SLSA subjects - Windows
+        id: hash-windows
+        if: runner.os == 'Windows'
+        run: echo "digests=$(sha256sum ./python/wheelhouse/* | base64 -w0)" >> $GITHUB_OUTPUT
+
+  gather-disgests:
+    needs: [build_wheels]
+    outputs:
+      digests: ${{ steps.hash.outputs.digests }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Merge results
+        id: hash
+        env:
+          LINUX_DIGESTS: "${{ needs.build_wheels.outputs.digests-linux }}"
+          MACOS_DIGESTS: "${{ needs.build_wheels.outputs.digests-macos }}"
+          WINDOWS_DIGESTS: "${{ needs.build_wheels.outputs.digests-windows }}"
+        run: |
+          set -euo pipefail
+          echo "$LINUX_DIGESTS" | base64 -d > checksums.txt
+          echo "$MACOS_DIGESTS" | base64 -d >> checksums.txt
+          echo "$WINDOWS_DIGESTS" | base64 -d >> checksums.txt
+          echo "digests=$(cat checksums.txt | base64 -w0)" >> $GITHUB_OUTPUT
+
+  provenance:
+    if: startsWith(github.ref, 'refs/tags/')
+    needs: [build_wheels, gather-disgests]
+    permissions:
+      actions: read   # To read the workflow path.
+      id-token: write # To sign the provenance.
+      contents: write # To add assets to a release.
+    uses: slsa-framework/slsa-github-generator/.github/workflows/[email protected]
+    with:
+      base64-subjects: "${{ needs.gather-disgests.outputs.digests }}"
+      upload-assets: true # Optional: Upload to a new release

.gitignore

@@ -72,3 +72,6 @@ libsentencepiece.so*
 libsentencepiece_train.so*
 python/bundled
 _sentencepiece.*.so
+third_party/abseil-cpp
+
+python/sentencepiece