Skip to content

Add first draft of Warc S3 store #128

Add first draft of Warc S3 store

Add first draft of Warc S3 store #128

Workflow file for this run

name: "CI"
on:
push:
jobs:
python-build:
name: "πŸ—οΈ Build Python wheels"
strategy:
matrix:
python:
- "3.10"
runs-on: ubuntu-latest
steps:
- name: "πŸ“₯ Check-out"
uses: actions/checkout@v3
- name: "🧰 Install Protoc"
run: sudo apt install protobuf-compiler
- name: "🧰 Install Python"
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python }}
cache: pip
cache-dependency-path: pyproject.toml
- name: "🧰 Install dependencies"
run: pip install build twine
- name: "πŸ—οΈ Build Python wheels"
run: python -m build
- name: "πŸ§ͺ Check package bundles"
run: twine check dist/*
- name: "πŸ“€ Upload Python wheels"
uses: actions/upload-artifact@v3
if: matrix.python == '3.10'
with:
name: wheels
path: dist
python-code-format:
name: "πŸ” Check Python code format"
strategy:
matrix:
python:
- "3.10"
runs-on: ubuntu-latest
steps:
- name: "πŸ“₯ Check-out"
uses: actions/checkout@v3
- name: "🧰 Install Protoc"
run: sudo apt install protobuf-compiler
- name: "🧰 Install Python"
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python }}
cache: pip
cache-dependency-path: pyproject.toml
- name: "🧰 Install dependencies"
run: pip install .[tests]
- name: "πŸ” Check Python code format"
run: flake8 archive_query_log
python-lint:
name: "πŸ” Lint Python code"
strategy:
matrix:
python:
- "3.10"
runs-on: ubuntu-latest
steps:
- name: "πŸ“₯ Check-out"
uses: actions/checkout@v3
- name: "🧰 Install Protoc"
run: sudo apt install protobuf-compiler
- name: "🧰 Install Python"
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python }}
cache: pip
cache-dependency-path: pyproject.toml
- name: "🧰 Install dependencies"
run: pip install .[tests]
- name: "πŸ” Lint Python code"
run: pylint archive_query_log
python-typing:
name: "πŸ” Check Python static typing"
strategy:
matrix:
python:
- "3.10"
runs-on: ubuntu-latest
steps:
- name: "πŸ“₯ Check-out"
uses: actions/checkout@v3
- name: "🧰 Install Protoc"
run: sudo apt install protobuf-compiler
- name: "🧰 Install Python"
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python }}
cache: pip
cache-dependency-path: pyproject.toml
- name: "🧰 Install dependencies"
run: pip install .[tests]
- name: "πŸ” Check Python static typing"
run: mypy archive_query_log
python-security:
name: "πŸ” Check Python code security"
strategy:
matrix:
python:
- "3.10"
runs-on: ubuntu-latest
steps:
- name: "πŸ“₯ Check-out"
uses: actions/checkout@v3
- name: "🧰 Install Protoc"
run: sudo apt install protobuf-compiler
- name: "🧰 Install Python"
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python }}
cache: pip
cache-dependency-path: pyproject.toml
- name: "🧰 Install dependencies"
run: pip install .[tests]
- name: "πŸ” Check Python code security"
run: bandit -c pyproject.toml -r archive_query_log
python-test:
name: "πŸ§ͺ Test Python code"
strategy:
matrix:
python:
- "3.10"
runs-on: ubuntu-latest
steps:
- name: "πŸ“₯ Check-out"
uses: actions/checkout@v3
- name: "🧰 Install Protoc"
run: sudo apt install protobuf-compiler
- name: "🧰 Install Python"
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python }}
cache: pip
cache-dependency-path: pyproject.toml
- name: "🧰 Install dependencies"
run: pip install .[tests]
- name: "πŸ§ͺ Test Python code"
run: pytest --cov --cov-report=xml archive_query_log
- name: "πŸ“€ Upload coverage to Codecov"
uses: codecov/codecov-action@v3
if: matrix.python == '3.10'
with:
fail_ci_if_error: true
token: ${{ secrets.CODECOV_TOKEN }}
docker-build:
name: "πŸ—οΈ Build Docker image"
runs-on: ubuntu-latest
steps:
- name: "πŸ“₯ Check-out"
uses: actions/checkout@v3
- name: "🧰 Set up QEMU"
uses: docker/setup-qemu-action@v2
- name: "🧰 Set up Docker Buildx"
uses: docker/setup-buildx-action@v2
- name: "πŸ—οΈ Build Docker image"
uses: docker/build-push-action@v4
with:
context: .
push: false
python-publish:
name: "πŸš€ Publish Python wheels"
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
needs:
- python-build
- python-code-format
- python-lint
- python-typing
- python-security
- python-test
- docker-build
runs-on: ubuntu-latest
permissions:
id-token: write
steps:
- name: "πŸ“₯ Check-out"
uses: actions/checkout@v3
- name: "πŸ“₯ Download Python wheels"
uses: actions/download-artifact@v3
with:
name: wheels
path: dist
- name: "πŸš€ Publish Python wheels"
uses: pypa/gh-action-pypi-publish@release/v1
docker-publish:
name: "πŸš€ Publish Docker image"
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
permissions:
packages: write
needs:
- python-build
- python-code-format
- python-lint
- python-typing
- python-security
- python-test
- docker-build
runs-on: ubuntu-latest
steps:
- name: "πŸ“₯ Check-out"
uses: actions/checkout@v3
- name: "🧰 Set up QEMU"
uses: docker/setup-qemu-action@v2
- name: "🧰 Set up Docker Buildx"
uses: docker/setup-buildx-action@v2
#- name: "πŸ”‘ Login to Docker Hub"
# uses: docker/login-action@v2
# with:
# username: ${{ secrets.DOCKERHUB_USERNAME }}
# password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: "πŸ”‘ Login to GitHub Packages"
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: "ℹ️ Extract image metadata"
id: meta
uses: docker/metadata-action@v4
with:
#images: |
# webis/archive-query-log
# ghcr.io/${{ github.repository }}
images: |
ghcr.io/${{ github.repository }}
- name: "πŸš€ Build and push image"
uses: docker/build-push-action@v4
with:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
github-release:
name: "πŸš€ Create GitHub release"
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
needs:
- python-build
- python-code-format
- python-lint
- python-typing
- python-security
- python-test
- docker-build
runs-on: ubuntu-latest
steps:
- name: "πŸ“₯ Check-out"
uses: actions/checkout@v3
- name: "πŸ“₯ Download Python wheels"
uses: actions/download-artifact@v3
with:
name: wheels
path: dist
- name: "πŸš€ Create GitHub release"
uses: softprops/action-gh-release@v1
with:
name: Release ${{ github.ref_name }}
files: dist/*
fail_on_unmatched_files: true
draft: false
prerelease: false
generate_release_notes: true