Merge pull request #61 from jllllll/pr

Build wheels with pre-compiled CUDA kernels
turboderp-org · Sep 26, 2023 · 2a70028 · 2a70028
2 parents ba5f619 + 2012864
commit 2a70028
Show file tree

Hide file tree

Showing 7 changed files with 577 additions and 79 deletions.
diff --git a/.github/workflows/build-wheels-release-rocm.yml b/.github/workflows/build-wheels-release-rocm.yml
@@ -0,0 +1,109 @@
+name: Build ROCm 5.4.2 Wheels & Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      pypi:
+        description: 'Upload wheels to PyPI? 1 = yes, 0 = no'
+        default: '1'
+        required: true
+        type: string
+  workflow_call:
+    inputs:
+      pypi:
+        description: 'Upload wheels to PyPI? 1 = yes, 0 = no'
+        default: '0'
+        required: true
+        type: string
+
+permissions:
+  contents: write
+
+jobs:
+  build_wheels:
+    name: Build ROCm wheel for Python ${{ matrix.pyver }}
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        pyver: ["3.8", "3.9", "3.10", "3.11"]
+
+    steps:
+      - name: Free Disk Space
+        uses: jlumbroso/[email protected]
+        with:
+          tool-cache: false
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: false
+          swap-storage: false
+
+      - uses: actions/checkout@v3
+
+      - name: Install ROCm SDK
+        run: |
+          export ROCM_VERSION=5.6
+          
+          [ ! -d /etc/apt/keyrings ] && sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+          wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
+          echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/$ROCM_VERSION focal main" | sudo tee --append /etc/apt/sources.list.d/rocm.list
+          echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | sudo tee /etc/apt/preferences.d/rocm-pin-600
+          sudo apt update
+          sudo apt install rocm-hip-sdk -y
+          echo "/opt/rocm/bin" >> $GITHUB_PATH
+          echo "ROCM_PATH=/opt/rocm" >> $GITHUB_ENV
+          echo "ROCM_VERSION=$ROCM_VERSION" >> $GITHUB_ENV
+          echo "USE_ROCM=1" >> $GITHUB_ENV
+          
+      - uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.pyver }}
+
+      - name: Install Dependencies
+        run: |
+          pip3 install torch\<=2.1.0 --pre --index-url="https://download.pytorch.org/whl/rocm$ROCM_VERSION" --extra-index-url="https://download.pytorch.org/whl/nightly/rocm$ROCM_VERSION"
+          pip3 install build wheel safetensors sentencepiece ninja
+        
+      - name: Build Wheel
+        id: build-wheel
+        run: |
+          if ($(Get-Content 'setup.py' -raw) -match 'version = "(\d+\.(?:\d+\.?)*)" \+ \(') {Write-Output $('::notice file=build-wheels-release-rocm.yml,line=54,title=Package Version::Detected package version is: {0}' -f $Matches[1]); Write-Output "PACKAGE_VERSION=$($Matches[1])" >> "$env:GITHUB_OUTPUT"} else {Write-Output '::error file=build-wheels-release.yml,line=41::Could not parse version from setup.py! You must upload wheels manually!'; Write-Output "PACKAGE_VERSION=None" >> "$env:GITHUB_OUTPUT"}
+          $BUILDTAG = "+rocm$env:ROCM_VERSION"
+          python3 -m build -n --wheel -C--build-option=egg_info "-C--build-option=--tag-build=$BUILDTAG"
+        shell: pwsh
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: 'wheels'
+          path: ./dist/*.whl
+
+      - name: Upload files to a GitHub release
+        if: steps.build-wheel.outputs.PACKAGE_VERSION != 'None'
+        uses: svenstaro/[email protected]
+        with:
+          file: ./dist/*.whl
+          tag: ${{ steps.build-wheel.outputs.PACKAGE_VERSION }}
+          file_glob: true
+          overwrite: true
+          release_name: ${{ steps.build-wheel.outputs.PACKAGE_VERSION }}
+
+  publish-to-pypi:
+    name: Publish Python distribution to PyPI
+    if: inputs.pypi == '1' && github.event_name != 'workflow_call'
+    needs: ['build_wheels','build_rocm']
+    runs-on: ubuntu-latest
+
+    environment:
+      name: pypi
+      url: https://pypi.org/p/exllamav2
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+      - name: Download all the wheels
+        uses: actions/download-artifact@v3
+        with:
+          name: wheels
+          path: dist/
+      - name: Publish distribution to PyPI
+        uses: pypa/[email protected]
diff --git a/.github/workflows/build-wheels-release.yml b/.github/workflows/build-wheels-release.yml
@@ -0,0 +1,192 @@
+name: Build Wheels & Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      pypi:
+        description: 'Upload wheels to PyPI? 1 = yes, 0 = no'
+        default: '1'
+        required: true
+        type: string
+
+permissions:
+  contents: write
+
+jobs:
+  build_wheels:
+    name: ${{ matrix.os }} Python ${{ matrix.pyver }} CUDA ${{ matrix.cuda }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-20.04, windows-latest]
+        pyver: ["3.8", "3.9", "3.10", "3.11"]
+        cuda: ["11.7.0", "11.8.0", "12.1.1"]
+    defaults:
+      run:
+        shell: pwsh
+    env:
+      CUDAVER: ${{ matrix.cuda }}
+      PYVER: ${{ matrix.pyver }}
+
+    steps:
+      - name: Free Disk Space
+        uses: jlumbroso/[email protected]
+        if: runner.os == 'Linux'
+        with:
+          tool-cache: false
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: false
+          swap-storage: false
+
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.pyver }}
+
+      - name: Setup Mamba
+        uses: conda-incubator/[email protected]
+        with:
+          activate-environment: "build"
+          python-version: ${{ matrix.pyver }}
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          use-mamba: true
+          add-pip-as-python-dependency: true
+          auto-activate-base: false
+
+      - name: Install Dependencies
+        run: |
+          $cudaVersion = $env:CUDAVER
+          $cudaVersionPytorch = $env:CUDAVER.Remove($env:CUDAVER.LastIndexOf('.')).Replace('.','')
+          
+          $cudaChannels = ''
+          $cudaNum = [int]$cudaVersion.substring($cudaVersion.LastIndexOf('.')+1)
+          while ($cudaNum -ge 0) { $cudaChannels += '-c nvidia/label/cuda-' + $cudaVersion.Remove($cudaVersion.LastIndexOf('.')+1) + $cudaNum + ' '; $cudaNum-- }
+          mamba install -y 'cuda' $cudaChannels.TrimEnd().Split()
+          
+          if ([version]$env:CUDAVER -gt [version]'11.8.0') {$torchver = "torch==2.1.0.* --pre --index-url https://download.pytorch.org/whl/nightly/cu$cudaVersionPytorch"} else {$torchver = "torch --index-url https://download.pytorch.org/whl/cu$cudaVersionPytorch"}
+          python -m pip install $torchver.split()
+          
+          python -m pip install build wheel safetensors sentencepiece ninja
+        
+      - name: Build Wheel
+        id: build-wheel
+        run: |
+          if ($(Get-Content 'setup.py' -raw) -match 'version = "(\d+\.(?:\d+\.?)*)" \+ \(') {Write-Output $('::notice file=build-wheels-release.yml,line=53,title=Package Version::Detected package version is: {0}' -f $Matches[1]); Write-Output "PACKAGE_VERSION=$($Matches[1])" >> "$env:GITHUB_OUTPUT"} else {Write-Output '::error file=build-wheels-release.yml,line=41::Could not parse version from setup.py! You must upload wheels manually!'; Write-Output "PACKAGE_VERSION=None" >> "$env:GITHUB_OUTPUT"}
+          
+          $env:CUDA_PATH = $env:CONDA_PREFIX
+          $env:CUDA_HOME = $env:CONDA_PREFIX
+          
+          $cudaVersion = $env:CUDAVER
+          $cudaVersionPytorch = $env:CUDAVER.Remove($env:CUDAVER.LastIndexOf('.')).Replace('.','')
+          $BUILDTAG = "+cu$cudaVersionPytorch"
+          
+          if ($IsLinux) {$env:LD_LIBRARY_PATH = $env:CONDA_PREFIX + '/lib:' + $env:LD_LIBRARY_PATH}
+          $env:TORCH_CUDA_ARCH_LIST = if ([version]$env:CUDAVER -lt [version]'11.8') {'6.0 6.1 7.0 7.5 8.0 8.6+PTX'} else {'6.0 6.1 7.0 7.5 8.0 8.6 8.9 9.0+PTX'}
+          python -m build -n --wheel -C--build-option=egg_info "-C--build-option=--tag-build=$BUILDTAG"
+        
+      - uses: actions/upload-artifact@v3
+        with:
+          name: 'wheels'
+          path: ./dist/*
+
+      - name: Upload files to a GitHub release
+        if: steps.build-wheel.outputs.PACKAGE_VERSION != 'None'
+        uses: svenstaro/[email protected]
+        with:
+          file: ./dist/*.whl
+          tag: ${{ steps.build-wheel.outputs.PACKAGE_VERSION }}
+          file_glob: true
+          overwrite: true
+          release_name: ${{ steps.build-wheel.outputs.PACKAGE_VERSION }}
+
+  build_rocm:
+    name: Build ROCm Wheels & Release
+    needs: build_wheels
+    uses: ./.github/workflows/build-wheels-release-rocm.yml
+    with:
+      pypi: '0'
+
+  publish-wheels-to-pypi:
+    name: Publish Python distribution to PyPI
+    if: inputs.pypi == '1'
+    needs: ['build_wheels','build_rocm']
+    runs-on: ubuntu-latest
+
+    environment:
+      name: pypi
+      url: https://pypi.org/p/exllamav2
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+      - name: Download all the wheels
+        uses: actions/download-artifact@v3
+        with:
+          name: wheels
+          path: dist/
+      - name: Publish distribution to PyPI
+        uses: pypa/[email protected]
+
+  build_sdist:
+    name: Build sdist
+    needs: publish-wheels-to-pypi
+    runs-on: ubuntu-20.04
+    defaults:
+      run:
+        shell: pwsh
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v3
+        with:
+          python-version: "3.11"
+
+      - name: Install Dependencies
+        run: |
+          python -m pip install build wheel ninja
+        
+      - name: Build Wheel
+        id: build-wheel-sdist
+        run: |
+          if ($(Get-Content 'setup.py' -raw) -match 'version = "(\d+\.(?:\d+\.?)*)" \+ \(') {Write-Output $('::notice file=build-wheels-release.yml,line=53,title=Package Version::Detected package version is: {0}' -f $Matches[1]); Write-Output "PACKAGE_VERSION=$($Matches[1])" >> "$env:GITHUB_OUTPUT"} else {Write-Output '::error file=build-wheels-release.yml,line=41::Could not parse version from setup.py! You must upload wheels manually!'; Write-Output "PACKAGE_VERSION=None" >> "$env:GITHUB_OUTPUT"}
+          $env:EXLLAMA_NOCOMPILE=1
+          python -m build
+        
+      - uses: actions/upload-artifact@v3
+        with:
+          name: 'sdist'
+          path: ./dist/*
+
+      - name: Upload files to a GitHub release
+        if: steps.build-wheel-sdist.outputs.PACKAGE_VERSION != 'None'
+        uses: svenstaro/[email protected]
+        with:
+          file: ./dist/*.whl
+          tag: ${{ steps.build-wheel.outputs.PACKAGE_VERSION }}
+          file_glob: true
+          overwrite: true
+          release_name: ${{ steps.build-wheel.outputs.PACKAGE_VERSION }}
+
+  publish-sdist-to-pypi:
+    name: Publish Python distribution to PyPI
+    if: inputs.pypi == '1'
+    needs: build_sdist
+    runs-on: ubuntu-latest
+
+    environment:
+      name: pypi
+      url: https://pypi.org/p/exllamav2
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+      - name: Download the sdist
+        uses: actions/download-artifact@v3
+        with:
+          name: sdist
+          path: dist/
+      - name: Publish distribution to PyPI
+        uses: pypa/[email protected]
diff --git a/.github/workflows/build-wheels-rocm.yml b/.github/workflows/build-wheels-rocm.yml
@@ -0,0 +1,60 @@
+name: Build ROCm 5.4.2 Wheels
+
+on:
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  build_wheels:
+    name: Build ROCm wheel for Python ${{ matrix.pyver }}
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        pyver: ["3.8", "3.9", "3.10", "3.11"]
+
+    steps:
+      - name: Free Disk Space
+        uses: jlumbroso/[email protected]
+        with:
+          tool-cache: false
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: false
+          swap-storage: false
+
+      - uses: actions/checkout@v3
+
+      - name: Install ROCm SDK
+        run: |
+          export ROCM_VERSION=5.6
+          
+          [ ! -d /etc/apt/keyrings ] && sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+          wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
+          echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/$ROCM_VERSION focal main" | sudo tee --append /etc/apt/sources.list.d/rocm.list
+          echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | sudo tee /etc/apt/preferences.d/rocm-pin-600
+          sudo apt update
+          sudo apt install rocm-hip-sdk -y
+          echo "/opt/rocm/bin" >> $GITHUB_PATH
+          echo "ROCM_PATH=/opt/rocm" >> $GITHUB_ENV
+          echo "ROCM_VERSION=$ROCM_VERSION" >> $GITHUB_ENV
+          echo "USE_ROCM=1" >> $GITHUB_ENV
+          
+      - uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.pyver }}
+
+      - name: Install Dependencies
+        run: |
+          pip3 install torch\<=2.1.0 --pre --index-url="https://download.pytorch.org/whl/rocm$ROCM_VERSION" --extra-index-url="https://download.pytorch.org/whl/nightly/rocm$ROCM_VERSION"
+          pip3 install build wheel safetensors sentencepiece ninja
+        
+      - name: Build Wheel
+        run: |
+          BUILDTAG="+rocm$ROCM_VERSION"
+          python3 -m build -n --wheel -C--build-option=egg_info -C--build-option=--tag-build=$BUILDTAG
+        
+      - uses: actions/upload-artifact@v3
+        with:
+          name: 'wheels'
+          path: ./dist/*.whl