test-gpu.yaml

name: Test on GPU

on:
  pull_request:
    paths:
      - "crates/**"
      - "Cargo.toml"
      - "Cargo.lock"
      - ".github/workflows/test-gpu.yaml"

jobs:
  zksync-crypto-gpu-build:
    runs-on: [ ubuntu-latest ]
    strategy:
      matrix:
        cuda: [ "12.0.0-devel-ubuntu20.04", "12.5.0-devel-ubuntu20.04" ]
        # TODO: Building the whole workspace with `--test` currently fails with link-time errors,
        # presumably due to either `gpu-ffi` or `gpu-prover` crates.
        # So for now we point at specific packages we want to test.
        package: [ "era_cudart", "boojum-cuda", "shivini" ]
    container:
      image: nvidia/cuda:${{ matrix.cuda }}
    env:
      BELLMAN_CUDA_DIR: ${{ github.workspace }}/bellman-cuda
    steps:
      - name: Prepare environment
        env:
          DEBIAN_FRONTEND: noninteractive
        run: |
          apt update && apt install -y \
            pkg-config libclang-dev build-essential lldb lld \
            clang openssl libssl-dev gcc g++ wget curl jq
          echo "/usr/local/nvidia/bin:/usr/local/cuda/bin" >> $GITHUB_PATH

      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3

      - name: Setup rustup
        run: |
          wget -q -O - https://sh.rustup.rs | bash -s -- -y
          echo "${HOME}/.cargo/bin" >> "${GITHUB_PATH}"
          echo "export PATH=\"$HOME/.cargo/bin:\$PATH\"" >> "${HOME}/.bash_profile"

      - name: Setup CMake
        run: |
          curl -LO https://github.com/Kitware/CMake/releases/download/v3.24.3/cmake-3.24.3-linux-x86_64.sh && \
            chmod +x cmake-3.24.3-linux-x86_64.sh && \
            ./cmake-3.24.3-linux-x86_64.sh --skip-license --prefix=/usr/local

      - name: Prepare bellman-cuda directory
        shell: bash
        # Curl ugliness is required because gh can't work with authentication: https://github.com/cli/cli/issues/2680.
        run: |
          release=($(curl --silent https://api.github.com/repos/matter-labs/era-bellman-cuda/releases | jq -r '.[0] | .name, .tarball_url, .assets[0].browser_download_url'))
          curl --silent -L "${release[1]}" --output bellman-cuda-source.tar.gz
          curl --silent -L "${release[2]}" --output bellman-cuda.tar.gz
          mkdir -p bellman-cuda
          tar xvfz bellman-cuda.tar.gz -C ./bellman-cuda
          tar xvfz bellman-cuda-source.tar.gz -C ./bellman-cuda --strip-components=1 --wildcards \*/src/

      - name: Check CMake version
        run: |
          cmake --version

      - name: Check CUDA version
        run: |
          nvcc --version

      - name: Setup rust
        run: |
          rustup set profile minimal
          rustup toolchain install nightly-2024-08-01
          rustup default nightly-2024-08-01

      - name: Build tests and copy binaries to a separate dir
        shell: bash
        run: |
          mkdir artifacts
          CUDAARCHS=80 CARGO_TARGET_DIR=./build \
          cargo +nightly-2024-08-01 test -p ${{ matrix.package }} --no-run --release --message-format=json -q \
          | jq -r 'select(.executable != null) | .executable' \
          | while read binary; do
            cp "$binary" artifacts/
          done

      - name: Upload test binaries as a single artifact
        uses: actions/upload-artifact@v3
        with:
          name: zksync-crypto-gpu-${{ matrix.cuda }}-test-binaries
          path: artifacts/
          if-no-files-found: error

  zksync-crypto-gpu-test:
    runs-on: [ matterlabs-ci-gpu-runner ]
    needs: zksync-crypto-gpu-build
    env:
      BELLMAN_CUDA_DIR: ${{ github.workspace }}/bellman-cuda
    steps:
      - name: Prepare environment
        run: |
          echo "/usr/local/nvidia/bin:/usr/local/cuda/bin" >> $GITHUB_PATH

      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3

      - name: Check Nvidia driver version
        run: |
          nvidia-smi

      - name: Prepare bellman-cuda directory
        shell: bash
        # Curl ugliness is required because gh can't work with authentication: https://github.com/cli/cli/issues/2680.
        run: |
          release=($(curl --silent https://api.github.com/repos/matter-labs/era-bellman-cuda/releases | jq -r '.[0] | .name, .tarball_url, .assets[0].browser_download_url'))
          curl --silent -L "${release[1]}" --output bellman-cuda-source.tar.gz
          curl --silent -L "${release[2]}" --output bellman-cuda.tar.gz
          mkdir -p bellman-cuda
          tar xvfz bellman-cuda.tar.gz -C ./bellman-cuda
          tar xvfz bellman-cuda-source.tar.gz -C ./bellman-cuda --strip-components=1 --wildcards \*/src/

      - name: Download test binaries built with CUDA 12.0
        uses: actions/download-artifact@v3
        with:
          name: zksync-crypto-gpu-12.0.0-devel-ubuntu20.04-test-binaries
          path: zksync-crypto-gpu-test-binaries/12.0

      - name: Download test binaries built with CUDA 12.5
        uses: actions/download-artifact@v3
        with:
          name: zksync-crypto-gpu-12.5.0-devel-ubuntu20.04-test-binaries
          path: zksync-crypto-gpu-test-binaries/12.5

      - name: Run test binaries built with CUDA 12.5
        id: test_cuda_12_5
        continue-on-error: true
        run: |
          for binary in zksync-crypto-gpu-test-binaries/12.5/*; do
            chmod +x $binary
            $binary
          done

      - name: Run test binaries built with CUDA 12.0
        if: steps.test_cuda_12_5.outcome == 'failure' || steps.test_cuda_12_5.outcome == 'success'
        run: |
          for binary in zksync-crypto-gpu-test-binaries/12.0/*; do
            chmod +x $binary
            $binary
          done