Add first draft of Warc S3 store #128
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: "CI" | |
on: | |
push: | |
jobs: | |
python-build: | |
name: "ποΈ Build Python wheels" | |
strategy: | |
matrix: | |
python: | |
- "3.10" | |
runs-on: ubuntu-latest | |
steps: | |
- name: "π₯ Check-out" | |
uses: actions/checkout@v3 | |
- name: "π§° Install Protoc" | |
run: sudo apt install protobuf-compiler | |
- name: "π§° Install Python" | |
uses: actions/setup-python@v4 | |
with: | |
python-version: ${{ matrix.python }} | |
cache: pip | |
cache-dependency-path: pyproject.toml | |
- name: "π§° Install dependencies" | |
run: pip install build twine | |
- name: "ποΈ Build Python wheels" | |
run: python -m build | |
- name: "π§ͺ Check package bundles" | |
run: twine check dist/* | |
- name: "π€ Upload Python wheels" | |
uses: actions/upload-artifact@v3 | |
if: matrix.python == '3.10' | |
with: | |
name: wheels | |
path: dist | |
python-code-format: | |
name: "π Check Python code format" | |
strategy: | |
matrix: | |
python: | |
- "3.10" | |
runs-on: ubuntu-latest | |
steps: | |
- name: "π₯ Check-out" | |
uses: actions/checkout@v3 | |
- name: "π§° Install Protoc" | |
run: sudo apt install protobuf-compiler | |
- name: "π§° Install Python" | |
uses: actions/setup-python@v4 | |
with: | |
python-version: ${{ matrix.python }} | |
cache: pip | |
cache-dependency-path: pyproject.toml | |
- name: "π§° Install dependencies" | |
run: pip install .[tests] | |
- name: "π Check Python code format" | |
run: flake8 archive_query_log | |
python-lint: | |
name: "π Lint Python code" | |
strategy: | |
matrix: | |
python: | |
- "3.10" | |
runs-on: ubuntu-latest | |
steps: | |
- name: "π₯ Check-out" | |
uses: actions/checkout@v3 | |
- name: "π§° Install Protoc" | |
run: sudo apt install protobuf-compiler | |
- name: "π§° Install Python" | |
uses: actions/setup-python@v4 | |
with: | |
python-version: ${{ matrix.python }} | |
cache: pip | |
cache-dependency-path: pyproject.toml | |
- name: "π§° Install dependencies" | |
run: pip install .[tests] | |
- name: "π Lint Python code" | |
run: pylint archive_query_log | |
python-typing: | |
name: "π Check Python static typing" | |
strategy: | |
matrix: | |
python: | |
- "3.10" | |
runs-on: ubuntu-latest | |
steps: | |
- name: "π₯ Check-out" | |
uses: actions/checkout@v3 | |
- name: "π§° Install Protoc" | |
run: sudo apt install protobuf-compiler | |
- name: "π§° Install Python" | |
uses: actions/setup-python@v4 | |
with: | |
python-version: ${{ matrix.python }} | |
cache: pip | |
cache-dependency-path: pyproject.toml | |
- name: "π§° Install dependencies" | |
run: pip install .[tests] | |
- name: "π Check Python static typing" | |
run: mypy archive_query_log | |
python-security: | |
name: "π Check Python code security" | |
strategy: | |
matrix: | |
python: | |
- "3.10" | |
runs-on: ubuntu-latest | |
steps: | |
- name: "π₯ Check-out" | |
uses: actions/checkout@v3 | |
- name: "π§° Install Protoc" | |
run: sudo apt install protobuf-compiler | |
- name: "π§° Install Python" | |
uses: actions/setup-python@v4 | |
with: | |
python-version: ${{ matrix.python }} | |
cache: pip | |
cache-dependency-path: pyproject.toml | |
- name: "π§° Install dependencies" | |
run: pip install .[tests] | |
- name: "π Check Python code security" | |
run: bandit -c pyproject.toml -r archive_query_log | |
python-test: | |
name: "π§ͺ Test Python code" | |
strategy: | |
matrix: | |
python: | |
- "3.10" | |
runs-on: ubuntu-latest | |
steps: | |
- name: "π₯ Check-out" | |
uses: actions/checkout@v3 | |
- name: "π§° Install Protoc" | |
run: sudo apt install protobuf-compiler | |
- name: "π§° Install Python" | |
uses: actions/setup-python@v4 | |
with: | |
python-version: ${{ matrix.python }} | |
cache: pip | |
cache-dependency-path: pyproject.toml | |
- name: "π§° Install dependencies" | |
run: pip install .[tests] | |
- name: "π§ͺ Test Python code" | |
run: pytest --cov --cov-report=xml archive_query_log | |
- name: "π€ Upload coverage to Codecov" | |
uses: codecov/codecov-action@v3 | |
if: matrix.python == '3.10' | |
with: | |
fail_ci_if_error: true | |
token: ${{ secrets.CODECOV_TOKEN }} | |
docker-build: | |
name: "ποΈ Build Docker image" | |
runs-on: ubuntu-latest | |
steps: | |
- name: "π₯ Check-out" | |
uses: actions/checkout@v3 | |
- name: "π§° Set up QEMU" | |
uses: docker/setup-qemu-action@v2 | |
- name: "π§° Set up Docker Buildx" | |
uses: docker/setup-buildx-action@v2 | |
- name: "ποΈ Build Docker image" | |
uses: docker/build-push-action@v4 | |
with: | |
context: . | |
push: false | |
python-publish: | |
name: "π Publish Python wheels" | |
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') | |
needs: | |
- python-build | |
- python-code-format | |
- python-lint | |
- python-typing | |
- python-security | |
- python-test | |
- docker-build | |
runs-on: ubuntu-latest | |
permissions: | |
id-token: write | |
steps: | |
- name: "π₯ Check-out" | |
uses: actions/checkout@v3 | |
- name: "π₯ Download Python wheels" | |
uses: actions/download-artifact@v3 | |
with: | |
name: wheels | |
path: dist | |
- name: "π Publish Python wheels" | |
uses: pypa/gh-action-pypi-publish@release/v1 | |
docker-publish: | |
name: "π Publish Docker image" | |
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') | |
permissions: | |
packages: write | |
needs: | |
- python-build | |
- python-code-format | |
- python-lint | |
- python-typing | |
- python-security | |
- python-test | |
- docker-build | |
runs-on: ubuntu-latest | |
steps: | |
- name: "π₯ Check-out" | |
uses: actions/checkout@v3 | |
- name: "π§° Set up QEMU" | |
uses: docker/setup-qemu-action@v2 | |
- name: "π§° Set up Docker Buildx" | |
uses: docker/setup-buildx-action@v2 | |
#- name: "π Login to Docker Hub" | |
# uses: docker/login-action@v2 | |
# with: | |
# username: ${{ secrets.DOCKERHUB_USERNAME }} | |
# password: ${{ secrets.DOCKERHUB_TOKEN }} | |
- name: "π Login to GitHub Packages" | |
uses: docker/login-action@v2 | |
with: | |
registry: ghcr.io | |
username: ${{ github.actor }} | |
password: ${{ secrets.GITHUB_TOKEN }} | |
- name: "βΉοΈ Extract image metadata" | |
id: meta | |
uses: docker/metadata-action@v4 | |
with: | |
#images: | | |
# webis/archive-query-log | |
# ghcr.io/${{ github.repository }} | |
images: | | |
ghcr.io/${{ github.repository }} | |
- name: "π Build and push image" | |
uses: docker/build-push-action@v4 | |
with: | |
context: . | |
push: true | |
tags: ${{ steps.meta.outputs.tags }} | |
labels: ${{ steps.meta.outputs.labels }} | |
github-release: | |
name: "π Create GitHub release" | |
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') | |
needs: | |
- python-build | |
- python-code-format | |
- python-lint | |
- python-typing | |
- python-security | |
- python-test | |
- docker-build | |
runs-on: ubuntu-latest | |
steps: | |
- name: "π₯ Check-out" | |
uses: actions/checkout@v3 | |
- name: "π₯ Download Python wheels" | |
uses: actions/download-artifact@v3 | |
with: | |
name: wheels | |
path: dist | |
- name: "π Create GitHub release" | |
uses: softprops/action-gh-release@v1 | |
with: | |
name: Release ${{ github.ref_name }} | |
files: dist/* | |
fail_on_unmatched_files: true | |
draft: false | |
prerelease: false | |
generate_release_notes: true |