Support torch 2.1.0 (#1249)

k2-fsa · Oct 23, 2023 · d12eec7 · d12eec7
1 parent c83b5cd
commit d12eec7
Show file tree

Hide file tree

Showing 27 changed files with 275 additions and 222 deletions.
diff --git a/.github/workflows/build-cpu-macos.yml b/.github/workflows/build-cpu-macos.yml
@@ -36,6 +36,7 @@ on:
       - 'cmake/**'
       - 'k2/csrc/**'
       - 'k2/python/**'
+  workflow_dispatch:
 
 concurrency:
   group: build-cpu-macos-${{ github.ref }}

diff --git a/.github/workflows/build-cpu-ubuntu.yml b/.github/workflows/build-cpu-ubuntu.yml
@@ -36,6 +36,7 @@ on:
       - 'cmake/**'
       - 'k2/csrc/**'
       - 'k2/python/**'
+  workflow_dispatch:
 
 concurrency:
   group: build-cpu-ubuntu-${{ github.ref }}

diff --git a/.github/workflows/build-cpu-windows.yml b/.github/workflows/build-cpu-windows.yml
@@ -35,6 +35,7 @@ on:
       - 'cmake/**'
       - 'k2/csrc/**'
       - 'k2/python/**'
+  workflow_dispatch:
 
 concurrency:
   group: build-cpu-windows-${{ github.ref }}

diff --git a/.github/workflows/build-cuda-ubuntu.yml b/.github/workflows/build-cuda-ubuntu.yml
@@ -36,6 +36,7 @@ on:
       - 'cmake/**'
       - 'k2/csrc/**'
       - 'k2/python/**'
+  workflow_dispatch:
 
 concurrency:
   group: build-cuda-ubuntu-${{ github.ref }}

diff --git a/.github/workflows/macos-cpu-m1-wheels.yml b/.github/workflows/macos-cpu-m1-wheels.yml
diff --git a/.github/workflows/macos-cpu-wheels.yml b/.github/workflows/macos-cpu-wheels.yml
@@ -18,7 +18,7 @@ jobs:
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Generating build matrix
@@ -39,7 +39,7 @@ jobs:
         ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -59,7 +59,7 @@ jobs:
           ls -lh ./wheelhouse/
 
       - name: Upload Wheel
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-macos-latest-cpu
           path: wheelhouse/*.whl

diff --git a/.github/workflows/run-tests-cpu.yml b/.github/workflows/run-tests-cpu.yml
@@ -38,6 +38,7 @@ on:
       - 'cmake/**'
       - 'k2/csrc/**'
       - 'k2/python/**'
+  workflow_dispatch:
 
 concurrency:
   group: run-tests-cpu-${{ github.ref }}
@@ -47,16 +48,14 @@ jobs:
   run-tests-cpu:
     if: github.event.label.name == 'ready' || github.event.label.name == 'cpp-test' || github.event_name == 'push'
     runs-on: ${{ matrix.os }}
+    name: ${{ matrix.python-version }} ${{ matrix.build_type }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
-        torch: ["1.13.1"]
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        torch: ["2.1.0"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
         build_type: ["Release", "Debug"]
-        exclude:
-          - os: macos-latest
-            python-version: "3.11"
 
     steps:
       # refer to https://github.com/actions/checkout
@@ -114,7 +113,7 @@ jobs:
           ./scripts/github_actions/fix_torch.sh
           mkdir build
           cd build
-          cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DK2_WITH_CUDA=OFF ..
+          cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DK2_WITH_CUDA=OFF -DCMAKE_CXX_STANDARD=17 ..
           cat k2/csrc/version.h
 
       - name: ${{ matrix.build_type }} Build

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
@@ -26,6 +26,7 @@ on:
       - '.github/workflows/run-tests.yml'
       - 'CMakeLists.txt'
       - 'cmake/**'
+      - 'scripts/github_actions/**'
       - 'k2/csrc/**'
       - 'k2/python/**'
   pull_request:
@@ -34,8 +35,10 @@ on:
       - '.github/workflows/run-tests.yml'
       - 'CMakeLists.txt'
       - 'cmake/**'
+      - 'scripts/github_actions/**'
       - 'k2/csrc/**'
       - 'k2/python/**'
+  workflow_dispatch:
 
 concurrency:
   group: run-tests-${{ github.ref }}
@@ -45,107 +48,80 @@ jobs:
   run-tests:
     if: github.event.label.name == 'ready' || github.event_name == 'push'
     runs-on: ${{ matrix.os }}
+    name: ${{ matrix.build_type }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        cuda: ["11.7"]
-        torch: ["1.13.1"]
+        cuda: ["12.1"]
+        torch: ["2.1.0"]
         python-version: ["3.11"]
-        build_type: ["Release", "Debug"]
 
     steps:
       # refer to https://github.com/actions/checkout
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
-      - name: Install CUDA Toolkit ${{ matrix.cuda }}
-        env:
-          cuda: ${{ matrix.cuda }}
-        run: |
-          source ./scripts/github_actions/install_cuda.sh
-          echo "CUDA_HOME=${CUDA_HOME}" >> $GITHUB_ENV
-          echo "${CUDA_HOME}/bin" >> $GITHUB_PATH
-          echo "LD_LIBRARY_PATH=${CUDA_HOME}/lib:${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
-        shell: bash
-
-      - name: Display NVCC version
-        run: |
-          which nvcc
-          nvcc --version
-
-      - name: Display GCC version
-        run: |
-          gcc --version
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+      - name: Run the build process with Docker
+        uses: addnab/docker-run-action@v3
         with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Display Python version
-        run: python -c "import sys; print(sys.version)"
-
-      - name: Install PyTorch ${{ matrix.torch }}
-        env:
-          cuda: ${{ matrix.cuda }}
-          torch: ${{ matrix.torch }}
+            image: "pytorch/manylinux-builder:cuda12.1"
+            options: -v ${{ github.workspace }}:/var/www -e PYTHON_VERSION=${{ matrix.python-version }} -e TORCH_VERSION=${{ matrix.torch }} -e CUDA_VERSION=${{ matrix.cuda }}
+            run: |
+              echo "pwd: $PWD"
+              uname -a
+              id
+              cat /etc/*release
+              gcc --version
+              python3 --version
+              which python3
+
+              pushd /usr/local
+              rm cuda
+              ln -s cuda-$CUDA_VERSION cuda
+              popd
+              which nvcc
+              nvcc --version
+
+              cp /var/www/scripts/github_actions/install_torch.sh .
+              chmod +x install_torch.sh
+
+              /var/www/scripts/github_actions/build-ubuntu-cuda.sh
+
+      - name: Display wheels
         shell: bash
         run: |
-          python3 -m pip install -qq --upgrade pip six
-          python3 -m pip install -qq bs4 requests tqdm typing_extensions
-          python3 -m pip install -qq dataclasses graphviz
-          sudo apt-get -qq install graphviz
-
-          ./scripts/github_actions/install_torch.sh
-          python3 -c "import torch; print('torch version:', torch.__version__)"
-
-      - name: Install git lfs
-        run: |
-          sudo apt-get install -y git-lfs
+          ls -lh ./wheelhouse/
 
-      - name: Download cudnn 8.0
-        env:
-          cuda: ${{ matrix.cuda }}
-        run: |
-          ./scripts/github_actions/install_cudnn.sh
+      - name: Upload Wheel
+        uses: actions/upload-artifact@v3
+        with:
+          name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cuda
+          path: wheelhouse/*.whl
 
-      - name: Configure CMake
-        shell: bash
+      # https://huggingface.co/docs/hub/spaces-github-actions
+      - name: Publish to huggingface
         env:
-          torch: ${{ matrix.torch }}
-        run: |
-          pwd
-          ./scripts/github_actions/fix_torch.sh
-          mkdir build
-          cd build
-          cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} ..
-          cat k2/csrc/version.h
-
-      - name: ${{ matrix.build_type }} Build
-        shell: bash
-        run: |
-          echo "number of cores: $(nproc)"
-          cd build
-          # we cannot use -j here because of limited RAM
-          # of the VM provided by GitHub actions
-          make VERBOSE=1 -j2
-
-      - name: Display Build Information
-        shell: bash
-        run: |
-          export PYTHONPATH=$PWD/k2/python:$PWD/build/lib:$PYTHONPATH
-          python3 -m k2.version
-
-      - name: Run Tests
-        shell: bash
-        run: |
-          cd build
-          ctest --output-on-failure
-          # default log level is INFO
-          ./bin/cu_log_test --gtest_filter="Log.Cpu"
-          K2_LOG_LEVEL=TRACE ./bin/cu_log_test --gtest_filter="Log.Cpu"
-          K2_LOG_LEVEL=DEBUG ./bin/cu_log_test --gtest_filter="Log.Cpu"
-          K2_LOG_LEVEL=INFO ./bin/cu_log_test --gtest_filter="Log.Cpu"
-          K2_LOG_LEVEL=WARNING ./bin/cu_log_test --gtest_filter="Log.Cpu"
-          K2_LOG_LEVEL=ERROR ./bin/cu_log_test --gtest_filter="Log.Cpu"
-          K2_LOG_LEVEL=FATAL ./bin/cu_log_test --gtest_filter="Log.Cpu"
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        uses: nick-fields/retry@v2
+        with:
+          max_attempts: 20
+          timeout_seconds: 200
+          shell: bash
+          command: |
+            git config --global user.email "[email protected]"
+            git config --global user.name "Fangjun Kuang"
+
+            rm -rf huggingface
+            export GIT_LFS_SKIP_SMUDGE=1
+
+            git clone https://huggingface.co/csukuangfj/k2 huggingface
+            cd huggingface
+            git pull
+
+            mkdir -p ubuntu-cuda
+            cp -v ../wheelhouse/*.whl ./ubuntu-cuda
+            git status
+            git lfs track "*.whl"
+            git add .
+            git commit -m "upload ubuntu-cuda wheel for torch ${{ matrix.torch }} python ${{ matrix.python-version }}"
+            git push https://csukuangfj:[email protected]/csukuangfj/k2 main
diff --git a/.github/workflows/style_check.yml b/.github/workflows/style_check.yml
@@ -31,6 +31,7 @@ on:
       - '.github/workflows/style_check.yml'
       - 'k2/csrc/**'
       - 'k2/python/**'
+  workflow_dispatch:
 
 concurrency:
   group: style_check-${{ github.ref }}

diff --git a/.github/workflows/test-k2-as-third-party-lib-cpu-macos.yml b/.github/workflows/test-k2-as-third-party-lib-cpu-macos.yml
@@ -38,6 +38,7 @@ on:
       - 'k2/csrc/**'
       - 'k2/python/**'
       - 'scripts/github_actions/k2-torch-api-test/**'
+  workflow_dispatch:
 
 concurrency:
   group: test-k2-as-third-party-lib-cpu-macos-${{ github.ref }}