build-wheels.yml

name: Build Wheels

on:
  push:
    tags:
      - '*'
    branches:
      - develop
      - ci/*

  pull_request:
    types:
      - opened
      - synchronize
    branches:
      - develop

env:
  PYTHON_VERSION: "3.12"
  FORCE_COLOR: "1"

jobs:
  build-wheels:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ ubuntu-latest, windows-latest, macos-12 ]
    env:
      CIBW_SKIP: "*-musllinux*"
      CIBW_ARCHS_MACOS: "x86_64 arm64"
      CIBW_ARCHS_LINUX: "x86_64"
      # CIBW_ARCHS_LINUX: "x86_64 aarch64"  # https://github.com/pypa/cibuildwheel/issues/1771#issuecomment-1973003145
      CIBW_MANYLINUX_X86_64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest
      CIBW_MANYLINUX_AARCH64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_aarch64:latest
      CIBW_TEST_SKIP: "*-macosx_arm64 *-manylinux_aarch64"    # ARM64 wheels cannot be tested
      CIBW_REPAIR_WHEEL_COMMAND_MACOS: >-
        DYLD_LIBRARY_PATH=$LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}
      CIBW_BEFORE_BUILD_WINDOWS: "python -m pip install delvewheel"
      CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: >-
        delvewheel repair --add-path C:\vcpkg\installed\x64-windows\bin -w {dest_dir} {wheel}

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Cache Vcpkg
        uses: actions/cache@v4
        id: cache-vcpkg
        if: runner.os == 'macOS' || runner.os == 'Windows'
        with:
          path: |
            /usr/local/share/vcpkg/installed
            C:\vcpkg\installed
          key: ${{ runner.os }}-vcpkg-15     # INCREMENT ME!!

      - name: Install Vcpkg Dependencies
        if: (runner.os == 'macOS' || runner.os == 'Windows') && steps.cache-vcpkg.outputs.cache-hit != 'true'
        shell: bash
        run: |
          set -e

          VCPKG_CMD="vcpkg --overlay-ports .vcpkg/ports --overlay-triplets .vcpkg/triplets install"
          PKG_LIST="lexbor lz4 re2 uchardet zlib"

          if [ -d /usr/local/share/vcpkg/ports ]; then
            # MacOS
            $VCPKG_CMD --triplet=x64-osx $PKG_LIST
            $VCPKG_CMD --triplet=arm64-osx $PKG_LIST
          elif [ -d /c/vcpkg ]; then
            # Windows
            $VCPKG_CMD --triplet=x64-windows $PKG_LIST
          else
            echo "Unsupported platform." >&2
            exit 1
          fi

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      # - name: Set up QEMU
      #   if: runner.os == 'Linux'
      #   uses: docker/setup-qemu-action@v3
      #   with:
      #     platforms: aarch64

      - name: Build FastWARC
        uses: pypa/cibuildwheel@v2.16.5
        with:
          package-dir: fastwarc
          output-dir: wheelhouse
        env:
          CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/fastwarc

      - name: Build Resiliparse
        uses: pypa/cibuildwheel@v2.16.5
        with:
          package-dir: resiliparse
          output-dir: wheelhouse
        env:
          CIBW_BEFORE_TEST: >-
            python -c "import glob, platform; open('fastwarc.txt', 'w').write(glob.glob('wheelhouse/FastWARC-*cp' + ''.join(map(str, platform.python_version_tuple()[:2])) + '-*_' + platform.machine().lower() + '.whl')[0])" &&
            python -m pip install -r fastwarc.txt
          CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/resiliparse

      - name: Upload Wheels
        uses: actions/upload-artifact@v4
        with:
          name: wheels-${{ matrix.os }}
          path: ./wheelhouse/*.whl

  # To be merged with the above when https://github.com/pypa/cibuildwheel/issues/1771#issuecomment-1973003145 is fixed
  build-wheels-aarch64:
    runs-on: ubuntu-latest
    if: startsWith(github.ref, 'refs/tags/v')
    env:
      CIBW_SKIP: "*cp38* *cp39* *-musllinux*"
      CIBW_MANYLINUX_AARCH64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_aarch64:latest
      CIBW_ARCHS_LINUX: "aarch64"
      CIBW_TEST_SKIP: "*"
      CIBW_CONTAINER_ENGINE: "docker; create_args: --platform linux/arm64/v8"

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
        with:
          platforms: all

      - name: Build FastWARC
        uses: pypa/cibuildwheel@v2.16.5
        with:
          package-dir: fastwarc
          output-dir: wheelhouse
        env:
          CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/fastwarc

      - name: Build Resiliparse
        uses: pypa/cibuildwheel@v2.16.5
        with:
          package-dir: resiliparse
          output-dir: wheelhouse
        env:
          CIBW_BEFORE_TEST: >-
            python -c "import glob, platform; open('fastwarc.txt', 'w').write(glob.glob('wheelhouse/FastWARC-*cp' + ''.join(map(str, platform.python_version_tuple()[:2])) + '-*_' + platform.machine().lower() + '.whl')[0])" &&
            python -m pip install -r fastwarc.txt
          CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/resiliparse

      - name: Upload Wheels
        uses: actions/upload-artifact@v4
        with:
          name: wheels-ubuntu-latest-aarch64
          path: ./wheelhouse/*.whl

  build-sdist:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install Build Module
        run: python -m pip install build

      - name: Build FastWARC Source Dist
        run: python -m build --sdist --outdir dist fastwarc

      - name: Build Resiliparse Source Dist
        run: python -m build --sdist --outdir dist resiliparse

      - name: Upload Source Dists
        uses: actions/upload-artifact@v4
        with:
          name: sdist
          path: ./dist/*.tar.gz

  build-asan:
    runs-on: ubuntu-latest
    container:
      image: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest
    env:
      DEBUG: "1"
      ASAN: "1"
      ASAN_OPTIONS: leak_check_at_exit=0

    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Build Extensions
        run: |
          set -e
          python${PYTHON_VERSION} -m pip install -e "fastwarc[all,test]"
          python${PYTHON_VERSION} -m pip install -e "resiliparse[all,test]"

      - name: Run Tests
        run: |
          export LD_PRELOAD="$(ldconfig -p | grep libasan | head -n1 | awk '{print $4}')"
          python${PYTHON_VERSION} -m pytest --capture=sys --verbose tests/

  build-coverage:
    runs-on: ubuntu-latest
    container:
      image: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest
    env:
      TRACE: "1"

    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Build Extensions
        run: |
          set -e
          python${PYTHON_VERSION} -m pip install cython codecov-cli
          python${PYTHON_VERSION} -m pip install -e "fastwarc[all,test]"
          python${PYTHON_VERSION} -m pip install -e "resiliparse[all,test]"

      - name: Run Tests
        run: python${PYTHON_VERSION} -m pytest --cov --cov-report xml --junitxml=report.junit.xml tests/

      # We cannot use the Codecov GitHub action due to incompatible Glibc versions
      - name: Upload to Codecov
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
          TOKENLESS: ${{ github.event.pull_request.head.label }}
        run: |
          set -e

          PR=""
          if [ -n '${{ github.event.pull_request.head.sha }}' ]; then
            PR="--pr ${{ github.event.number }}"
          fi

          python${PYTHON_VERSION} -m codecov_cli.main do-upload \
            --file report.junit.xml \
            --report-type test_results \
            --sha ${{ github.event.pull_request.head.sha || github.sha }} \
            --branch ${{ github.event.pull_request.head.label || github.ref_name }} \
            $PR \
            --slug ${{ github.repository }} \
            --git-service github \
            --fail-on-error

          python${PYTHON_VERSION} -m codecov_cli.main upload-process \
            --file coverage.xml \
            --report-type coverage \
            --sha ${{ github.event.pull_request.head.sha || github.sha }} \
            --branch ${{ github.event.pull_request.head.label || github.ref_name }} \
            $PR \
            --slug ${{ github.repository }} \
            --git-service github \
            --fail-on-error

      - name: Upload Artifacts
        uses: actions/upload-artifact@v4
        with:
          name: coverage
          path: |
            report.junit.xml
            coverage.xml

  build-documentation:
    runs-on: ubuntu-latest
    needs: build-wheels
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Download Artifacts
        uses: actions/download-artifact@v4
        with:
          name: wheels-ubuntu-latest
          path: wheelhouse

      - name: Build Documentation
        run: |
          set -e

          python -m pip install twine
          grep -vE "fastwarc|resiliparse" docs/requirements.txt | xargs python -m pip install

          PYTHON_ABI="cp${PYTHON_VERSION/./}"
          find wheelhouse -name "FastWARC-*-${PYTHON_ABI}-*-manylinux_x86_64.whl" | xargs -I% python -m pip install "%[all]"
          find wheelhouse -name "Resiliparse-*-${PYTHON_ABI}-*-manylinux_x86_64.whl" | xargs -I% python -m pip install "%[all]"

          cd docs
          make html

      - name: Trigger Readthedocs Build
        if: github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/tags/v')
        uses: dfm/rtds-action@v1
        with:
          webhook_url: ${{ secrets.RTDS_WEBHOOK_URL }}
          webhook_token: ${{ secrets.RTDS_WEBHOOK_TOKEN }}
          commit_ref: ${{ github.ref }}

  publish-wheels:
    runs-on: ubuntu-latest
    if: startsWith(github.ref, 'refs/tags/v')
    needs: [ build-wheels, build-wheels-aarch64, build-asan, build-sdist, build-documentation ]
    steps:
      - name: Download Wheels
        uses: actions/download-artifact@v4
        with:
          pattern: wheels-*
          merge-multiple: true

      - name: Download Source Dist
        uses: actions/download-artifact@v4
        with:
          name: sdist

      - name: Publish to PyPi
        env:
          TWINE_USERNAME: __token__
          TWINE_PASSWORD1: ${{ secrets.PYPI_FASTWARC_API_TOKEN }}
          TWINE_PASSWORD2: ${{ secrets.PYPI_RESILIPARSE_API_TOKEN }}
        run: |
          set -e

          python3 -m pip install twine

          TWINE_PASSWORD="$TWINE_PASSWORD1" python3 -m twine upload FastWARC-*.whl fastwarc-*.tar.gz
          TWINE_PASSWORD="$TWINE_PASSWORD2" python3 -m twine upload Resiliparse-*.whl resiliparse-*.tar.gz

      - name: Wait
        run: sleep 30

      - name: Trigger Readthedocs Build
        uses: dfm/rtds-action@v1
        with:
          webhook_url: ${{ secrets.RTDS_WEBHOOK_URL }}
          webhook_token: ${{ secrets.RTDS_WEBHOOK_TOKEN }}
          commit_ref: ${{ github.ref }}