Skip to content

Remove dead code

Remove dead code #797

Workflow file for this run

name: Build Wheels
on:
push:
tags:
- '*'
branches:
- develop
- ci/*
pull_request:
types:
- opened
- synchronize
branches:
- develop
env:
PYTHON_VERSION: "3.12"
FORCE_COLOR: "1"
jobs:
build-wheels:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ ubuntu-latest, windows-latest, macos-12 ]
env:
CIBW_SKIP: "*-musllinux*"
CIBW_ARCHS_MACOS: "x86_64 arm64"
CIBW_ARCHS_LINUX: "x86_64"
# CIBW_ARCHS_LINUX: "x86_64 aarch64" # https://github.com/pypa/cibuildwheel/issues/1771#issuecomment-1973003145
CIBW_MANYLINUX_X86_64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest
CIBW_MANYLINUX_AARCH64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_aarch64:latest
CIBW_TEST_SKIP: "*-macosx_arm64 *-manylinux_aarch64" # ARM64 wheels cannot be tested
CIBW_REPAIR_WHEEL_COMMAND_MACOS: >-
DYLD_LIBRARY_PATH=$LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}
CIBW_BEFORE_BUILD_WINDOWS: "python -m pip install delvewheel"
CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: >-
delvewheel repair --add-path C:\vcpkg\installed\x64-windows\bin -w {dest_dir} {wheel}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Cache Vcpkg
uses: actions/cache@v4
id: cache-vcpkg
if: runner.os == 'macOS' || runner.os == 'Windows'
with:
path: |
/usr/local/share/vcpkg/installed
C:\vcpkg\installed
key: ${{ runner.os }}-vcpkg-15 # INCREMENT ME!!
- name: Install Vcpkg Dependencies
if: (runner.os == 'macOS' || runner.os == 'Windows') && steps.cache-vcpkg.outputs.cache-hit != 'true'
shell: bash
run: |
set -e
VCPKG_CMD="vcpkg --overlay-ports .vcpkg/ports --overlay-triplets .vcpkg/triplets install"
PKG_LIST="lexbor lz4 re2 uchardet zlib"
if [ -d /usr/local/share/vcpkg/ports ]; then
# MacOS
$VCPKG_CMD --triplet=x64-osx $PKG_LIST
$VCPKG_CMD --triplet=arm64-osx $PKG_LIST
elif [ -d /c/vcpkg ]; then
# Windows
$VCPKG_CMD --triplet=x64-windows $PKG_LIST
else
echo "Unsupported platform." >&2
exit 1
fi
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
# - name: Set up QEMU
# if: runner.os == 'Linux'
# uses: docker/setup-qemu-action@v3
# with:
# platforms: aarch64
- name: Build FastWARC
uses: pypa/[email protected]
with:
package-dir: fastwarc
output-dir: wheelhouse
env:
CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/fastwarc
- name: Build Resiliparse
uses: pypa/[email protected]
with:
package-dir: resiliparse
output-dir: wheelhouse
env:
CIBW_BEFORE_TEST: >-
python -c "import glob, platform; open('fastwarc.txt', 'w').write(glob.glob('wheelhouse/FastWARC-*cp' + ''.join(map(str, platform.python_version_tuple()[:2])) + '-*_' + platform.machine().lower() + '.whl')[0])" &&
python -m pip install -r fastwarc.txt
CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/resiliparse
- name: Upload Wheels
uses: actions/upload-artifact@v4
with:
name: wheels-${{ matrix.os }}
path: ./wheelhouse/*.whl
# To be merged with the above when https://github.com/pypa/cibuildwheel/issues/1771#issuecomment-1973003145 is fixed
build-wheels-aarch64:
runs-on: ubuntu-latest
if: startsWith(github.ref, 'refs/tags/v')
env:
CIBW_SKIP: "*cp38* *cp39* *-musllinux*"
CIBW_MANYLINUX_AARCH64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_aarch64:latest
CIBW_ARCHS_LINUX: "aarch64"
CIBW_TEST_SKIP: "*"
CIBW_CONTAINER_ENGINE: "docker; create_args: --platform linux/arm64/v8"
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: all
- name: Build FastWARC
uses: pypa/[email protected]
with:
package-dir: fastwarc
output-dir: wheelhouse
env:
CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/fastwarc
- name: Build Resiliparse
uses: pypa/[email protected]
with:
package-dir: resiliparse
output-dir: wheelhouse
env:
CIBW_BEFORE_TEST: >-
python -c "import glob, platform; open('fastwarc.txt', 'w').write(glob.glob('wheelhouse/FastWARC-*cp' + ''.join(map(str, platform.python_version_tuple()[:2])) + '-*_' + platform.machine().lower() + '.whl')[0])" &&
python -m pip install -r fastwarc.txt
CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/resiliparse
- name: Upload Wheels
uses: actions/upload-artifact@v4
with:
name: wheels-ubuntu-latest-aarch64
path: ./wheelhouse/*.whl
build-sdist:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Install Build Module
run: python -m pip install build
- name: Build FastWARC Source Dist
run: python -m build --sdist --outdir dist fastwarc
- name: Build Resiliparse Source Dist
run: python -m build --sdist --outdir dist resiliparse
- name: Upload Source Dists
uses: actions/upload-artifact@v4
with:
name: sdist
path: ./dist/*.tar.gz
build-asan:
runs-on: ubuntu-latest
container:
image: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest
env:
DEBUG: "1"
ASAN: "1"
ASAN_OPTIONS: leak_check_at_exit=0
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Build Extensions
run: |
set -e
python${PYTHON_VERSION} -m pip install -e "fastwarc[all,test]"
python${PYTHON_VERSION} -m pip install -e "resiliparse[all,test]"
- name: Run Tests
run: |
export LD_PRELOAD="$(ldconfig -p | grep libasan | head -n1 | awk '{print $4}')"
python${PYTHON_VERSION} -m pytest --capture=sys --verbose tests/
build-coverage:
runs-on: ubuntu-latest
container:
image: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest
env:
TRACE: "1"
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Build Extensions
run: |
set -e
python${PYTHON_VERSION} -m pip install cython codecov-cli
python${PYTHON_VERSION} -m pip install -e "fastwarc[all,test]"
python${PYTHON_VERSION} -m pip install -e "resiliparse[all,test]"
- name: Run Tests
run: python${PYTHON_VERSION} -m pytest --cov --cov-report xml --junitxml=report.junit.xml tests/
# We cannot use the Codecov GitHub action due to incompatible Glibc versions
- name: Upload to Codecov
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
TOKENLESS: ${{ github.event.pull_request.head.label }}
run: |
set -e
PR=""
if [ -n '${{ github.event.pull_request.head.sha }}' ]; then
PR="--pr ${{ github.event.number }}"
fi
python${PYTHON_VERSION} -m codecov_cli.main do-upload \
--file report.junit.xml \
--report-type test_results \
--sha ${{ github.event.pull_request.head.sha || github.sha }} \
--branch ${{ github.event.pull_request.head.label || github.ref_name }} \
$PR \
--slug ${{ github.repository }} \
--git-service github \
--fail-on-error
python${PYTHON_VERSION} -m codecov_cli.main upload-process \
--file coverage.xml \
--report-type coverage \
--sha ${{ github.event.pull_request.head.sha || github.sha }} \
--branch ${{ github.event.pull_request.head.label || github.ref_name }} \
$PR \
--slug ${{ github.repository }} \
--git-service github \
--fail-on-error
- name: Upload Artifacts
uses: actions/upload-artifact@v4
with:
name: coverage
path: |
report.junit.xml
coverage.xml
build-documentation:
runs-on: ubuntu-latest
needs: build-wheels
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Download Artifacts
uses: actions/download-artifact@v4
with:
name: wheels-ubuntu-latest
path: wheelhouse
- name: Build Documentation
run: |
set -e
python -m pip install twine
grep -vE "fastwarc|resiliparse" docs/requirements.txt | xargs python -m pip install
PYTHON_ABI="cp${PYTHON_VERSION/./}"
find wheelhouse -name "FastWARC-*-${PYTHON_ABI}-*-manylinux_x86_64.whl" | xargs -I% python -m pip install "%[all]"
find wheelhouse -name "Resiliparse-*-${PYTHON_ABI}-*-manylinux_x86_64.whl" | xargs -I% python -m pip install "%[all]"
cd docs
make html
- name: Trigger Readthedocs Build
if: github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/tags/v')
uses: dfm/rtds-action@v1
with:
webhook_url: ${{ secrets.RTDS_WEBHOOK_URL }}
webhook_token: ${{ secrets.RTDS_WEBHOOK_TOKEN }}
commit_ref: ${{ github.ref }}
publish-wheels:
runs-on: ubuntu-latest
if: startsWith(github.ref, 'refs/tags/v')
needs: [ build-wheels, build-wheels-aarch64, build-asan, build-sdist, build-documentation ]
steps:
- name: Download Wheels
uses: actions/download-artifact@v4
with:
pattern: wheels-*
merge-multiple: true
- name: Download Source Dist
uses: actions/download-artifact@v4
with:
name: sdist
- name: Publish to PyPi
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD1: ${{ secrets.PYPI_FASTWARC_API_TOKEN }}
TWINE_PASSWORD2: ${{ secrets.PYPI_RESILIPARSE_API_TOKEN }}
run: |
set -e
python3 -m pip install twine
TWINE_PASSWORD="$TWINE_PASSWORD1" python3 -m twine upload FastWARC-*.whl fastwarc-*.tar.gz
TWINE_PASSWORD="$TWINE_PASSWORD2" python3 -m twine upload Resiliparse-*.whl resiliparse-*.tar.gz
- name: Wait
run: sleep 30
- name: Trigger Readthedocs Build
uses: dfm/rtds-action@v1
with:
webhook_url: ${{ secrets.RTDS_WEBHOOK_URL }}
webhook_token: ${{ secrets.RTDS_WEBHOOK_TOKEN }}
commit_ref: ${{ github.ref }}