Use vcpkg to build dependencies #847
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Wheels | |
| on: | |
| push: | |
| tags: | |
| - '*' | |
| branches: | |
| - develop | |
| - ci/* | |
| pull_request: | |
| types: | |
| - opened | |
| - synchronize | |
| branches: | |
| - develop | |
| env: | |
| PYTHON_VERSION: "3.12" | |
| FORCE_COLOR: "1" | |
| MACOSX_DEPLOYMENT_TARGET: "12.0" | |
| jobs: | |
| build-wheels: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ ubuntu-latest, windows-latest, macos-latest ] | |
| env: | |
| CIBW_SKIP: "*-musllinux*" | |
| CIBW_TEST_SKIP: "*-manylinux_aarch64 *-macosx_x86_64" # x86_64 tests are flaky on arm64 | |
| CIBW_ARCHS_MACOS: "x86_64 arm64" | |
| CIBW_ARCHS_LINUX: "x86_64" | |
| # CIBW_ARCHS_LINUX: "x86_64 aarch64" # https://github.com/pypa/cibuildwheel/issues/1771#issuecomment-1973003145 | |
| CIBW_MANYLINUX_X86_64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest | |
| CIBW_MANYLINUX_AARCH64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_aarch64:latest | |
| CIBW_REPAIR_WHEEL_COMMAND_MACOS: >- | |
| DYLD_LIBRARY_PATH=$(pwd)/vcpkg_installed/$(echo {delocate_archs} | sed s/x86_64/x64/)-osx/lib | |
| delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} | |
| CIBW_BEFORE_BUILD_WINDOWS: "python -m pip install delvewheel" | |
| CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: >- | |
| delvewheel repair --add-path vcpkg_installed\x64-windows\bin -w {dest_dir} {wheel} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Cache Vcpkg | |
| uses: actions/cache@v4 | |
| id: cache-vcpkg | |
| if: runner.os == 'macOS' || runner.os == 'Windows' | |
| with: | |
| path: ./vcpkg_installed | |
| key: ${{ runner.os }}-vcpkg-19 # INCREMENT ME!! | |
| - name: Install Vcpkg Dependencies | |
| if: (runner.os == 'macOS' || runner.os == 'Windows') && steps.cache-vcpkg.outputs.cache-hit != 'true' | |
| shell: bash | |
| run: | | |
| set -xe | |
| # Windows | |
| if [ -d /c/vcpkg ]; then | |
| vcpkg install --triplet=x64-windows | |
| # MacOS | |
| elif [ "$(uname -s)" == "Darwin" ]; then | |
| git clone https://github.com/Microsoft/vcpkg | |
| ./vcpkg/bootstrap-vcpkg.sh | |
| # Use to two install roots and merge manually, otherwise the second install clears out the previous | |
| ./vcpkg/vcpkg install --triplet=x64-osx | |
| ./vcpkg/vcpkg install --triplet=arm64-osx --x-install-root vcpkg_installed2 | |
| cp -a vcpkg_installed2/* vcpkg_installed/ | |
| rm -r vcpkg_installed2 | |
| else | |
| echo "Unsupported platform." >&2 | |
| exit 1 | |
| fi | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| # - name: Set up QEMU | |
| # if: runner.os == 'Linux' | |
| # uses: docker/setup-qemu-action@v3 | |
| # with: | |
| # platforms: aarch64 | |
| - name: Build FastWARC | |
| uses: pypa/[email protected] | |
| with: | |
| package-dir: fastwarc | |
| output-dir: wheelhouse | |
| env: | |
| CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/fastwarc | |
| - name: Build Resiliparse | |
| uses: pypa/[email protected] | |
| with: | |
| package-dir: resiliparse | |
| output-dir: wheelhouse | |
| env: | |
| CIBW_BEFORE_TEST: >- | |
| python -c "import glob, platform; open('fastwarc.txt', 'w').write(glob.glob('wheelhouse/FastWARC-*cp' + ''.join(map(str, platform.python_version_tuple()[:2])) + '-*_' + platform.machine().lower() + '.whl')[0])" && | |
| python -m pip install -r fastwarc.txt | |
| CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/resiliparse | |
| - name: Upload Wheels | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: wheels-${{ matrix.os }} | |
| path: ./wheelhouse/*.whl | |
| # To be merged with the above when https://github.com/pypa/cibuildwheel/issues/1771#issuecomment-1973003145 is fixed | |
| build-wheels-aarch64: | |
| runs-on: ubuntu-latest | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| env: | |
| CIBW_SKIP: "*cp38* *cp39* *-musllinux*" | |
| CIBW_MANYLINUX_AARCH64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_aarch64:latest | |
| CIBW_ARCHS_LINUX: "aarch64" | |
| CIBW_TEST_SKIP: "*" | |
| CIBW_CONTAINER_ENGINE: "docker; create_args: --platform linux/arm64/v8" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| with: | |
| platforms: all | |
| - name: Build FastWARC | |
| uses: pypa/[email protected] | |
| with: | |
| package-dir: fastwarc | |
| output-dir: wheelhouse | |
| env: | |
| CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/fastwarc | |
| - name: Build Resiliparse | |
| uses: pypa/[email protected] | |
| with: | |
| package-dir: resiliparse | |
| output-dir: wheelhouse | |
| env: | |
| CIBW_BEFORE_TEST: >- | |
| python -c "import glob, platform; open('fastwarc.txt', 'w').write(glob.glob('wheelhouse/FastWARC-*cp' + ''.join(map(str, platform.python_version_tuple()[:2])) + '-*_' + platform.machine().lower() + '.whl')[0])" && | |
| python -m pip install -r fastwarc.txt | |
| CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/resiliparse | |
| - name: Upload Wheels | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: wheels-ubuntu-latest-aarch64 | |
| path: ./wheelhouse/*.whl | |
| build-sdist: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install Build Module | |
| run: python -m pip install build | |
| - name: Build FastWARC Source Dist | |
| run: python -m build --sdist --outdir dist fastwarc | |
| - name: Build Resiliparse Source Dist | |
| run: python -m build --sdist --outdir dist resiliparse | |
| - name: Upload Source Dists | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: sdist | |
| path: ./dist/*.tar.gz | |
| build-asan: | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest | |
| env: | |
| DEBUG: "1" | |
| ASAN: "1" | |
| ASAN_OPTIONS: leak_check_at_exit=0 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Build Extensions | |
| run: | | |
| set -e | |
| python${PYTHON_VERSION} -m pip install -e "fastwarc[all,test]" | |
| python${PYTHON_VERSION} -m pip install -e "resiliparse[all,test]" | |
| - name: Run Tests | |
| run: | | |
| export LD_PRELOAD="$(ldconfig -p | grep libasan | head -n1 | awk '{print $4}')" | |
| python${PYTHON_VERSION} -m pytest --capture=sys --verbose tests/ | |
| build-coverage: | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest | |
| env: | |
| TRACE: "1" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Build Extensions | |
| run: | | |
| set -e | |
| python${PYTHON_VERSION} -m pip install cython codecov-cli | |
| python${PYTHON_VERSION} -m pip install -e "fastwarc[all,test]" | |
| python${PYTHON_VERSION} -m pip install -e "resiliparse[all,test]" | |
| - name: Run Tests | |
| run: python${PYTHON_VERSION} -m pytest --cov --cov-report xml --junitxml=report.junit.xml tests/ | |
| # We cannot use the Codecov GitHub action due to incompatible Glibc versions | |
| - name: Upload to Codecov | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| TOKENLESS: ${{ github.event.pull_request.head.label }} | |
| run: | | |
| set -e | |
| PR="" | |
| if [ -n '${{ github.event.pull_request.head.sha }}' ]; then | |
| PR="--pr ${{ github.event.number }}" | |
| fi | |
| python${PYTHON_VERSION} -m codecov_cli.main do-upload \ | |
| --file report.junit.xml \ | |
| --report-type test_results \ | |
| --sha ${{ github.event.pull_request.head.sha || github.sha }} \ | |
| --branch ${{ github.event.pull_request.head.label || github.ref_name }} \ | |
| $PR \ | |
| --slug ${{ github.repository }} \ | |
| --git-service github \ | |
| --fail-on-error | |
| python${PYTHON_VERSION} -m codecov_cli.main upload-process \ | |
| --file coverage.xml \ | |
| --report-type coverage \ | |
| --sha ${{ github.event.pull_request.head.sha || github.sha }} \ | |
| --branch ${{ github.event.pull_request.head.label || github.ref_name }} \ | |
| $PR \ | |
| --slug ${{ github.repository }} \ | |
| --git-service github \ | |
| --fail-on-error | |
| - name: Upload Artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage | |
| path: | | |
| report.junit.xml | |
| coverage.xml | |
| build-documentation: | |
| runs-on: ubuntu-latest | |
| needs: build-wheels | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Download Artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: wheels-ubuntu-latest | |
| path: wheelhouse | |
| - name: Build Documentation | |
| run: | | |
| set -e | |
| python -m pip install twine | |
| grep -vE "fastwarc|resiliparse" docs/requirements.txt | xargs python -m pip install | |
| PYTHON_ABI="cp${PYTHON_VERSION/./}" | |
| find wheelhouse -name "FastWARC-*-${PYTHON_ABI}-*-manylinux_x86_64.whl" | xargs -I% python -m pip install "%[all]" | |
| find wheelhouse -name "Resiliparse-*-${PYTHON_ABI}-*-manylinux_x86_64.whl" | xargs -I% python -m pip install "%[all]" | |
| cd docs | |
| make html | |
| - name: Trigger Readthedocs Build | |
| if: github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/tags/v') | |
| uses: dfm/rtds-action@v1 | |
| with: | |
| webhook_url: ${{ secrets.RTDS_WEBHOOK_URL }} | |
| webhook_token: ${{ secrets.RTDS_WEBHOOK_TOKEN }} | |
| commit_ref: ${{ github.ref }} | |
| publish-wheels: | |
| runs-on: ubuntu-latest | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| needs: [ build-wheels, build-wheels-aarch64, build-asan, build-sdist, build-documentation ] | |
| steps: | |
| - name: Download Wheels | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: wheels-* | |
| merge-multiple: true | |
| - name: Download Source Dist | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: sdist | |
| - name: Publish to PyPi | |
| env: | |
| TWINE_USERNAME: __token__ | |
| TWINE_PASSWORD1: ${{ secrets.PYPI_FASTWARC_API_TOKEN }} | |
| TWINE_PASSWORD2: ${{ secrets.PYPI_RESILIPARSE_API_TOKEN }} | |
| run: | | |
| set -e | |
| python3 -m pip install twine | |
| TWINE_PASSWORD="$TWINE_PASSWORD1" python3 -m twine upload FastWARC-*.whl fastwarc-*.tar.gz | |
| TWINE_PASSWORD="$TWINE_PASSWORD2" python3 -m twine upload Resiliparse-*.whl resiliparse-*.tar.gz | |
| - name: Wait | |
| run: sleep 30 | |
| - name: Trigger Readthedocs Build | |
| uses: dfm/rtds-action@v1 | |
| with: | |
| webhook_url: ${{ secrets.RTDS_WEBHOOK_URL }} | |
| webhook_token: ${{ secrets.RTDS_WEBHOOK_TOKEN }} | |
| commit_ref: ${{ github.ref }} |