Skip to content
Python CI

Python CI #863

Sign in to view logs

Python CI

Python CI #863

Workflow file for this run

.github/workflows/ci_python.yml at 2770902
# This file was initially generated by maturin v1.2.0, using:
#
# maturin generate-ci github --pytest -o ../.github/workflows/ci_python.yml
#
# TODO:
# - `safety`
# - `typeguard`
# - `xdoctest`
#
# Currently, `mypy` and `pytest` are running as part of building the rules.
# We could (in theory) use a matrix to do these in parallel fetching the
# built wheels, but it doesn't seem likely to provide a significant speedup.
name: Python CI
# Only one job per-ref
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
# See https://matklad.github.io/2021/09/04/fast-rust-builds.html#CI-Workflow.
# Speeds up CI builds, which won't have previous results anyway.
CARGO_INCREMENTAL: 0
on:
push:
branches:
- main
tags:
- '*'
pull_request:
branches:
- main
merge_group:
branches:
- main
defaults:
run:
shell: bash
working-directory: python
permissions:
actions: write
# Used by clippy action to report lint errors.
checks: write
contents: read
jobs:
# Run lint separately from the build -- it doesn't need the built Rust code.
#
# Also, only do it on one machine.
python-lint:
if: github.event_name == 'pull_request' || github.event_name == 'merge_group'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Install poetry
run: |
pipx install poetry
poetry config virtualenvs.create true --local
poetry config virtualenvs.in-project true --local
- uses: actions/setup-python@v4
with:
python-version: 3.11
cache: poetry
- name: install deps
run: |
poetry install --only=main --only=lint
- name: black
run: |
poetry run black --diff pysrc pytests docs
- name: flake8
run: |
poetry run flake8 pysrc pytests docs
- name: isort
run: |
poetry run isort --filter-files --diff pysrc pytests docs
- name: pydocstyle
run: |
poetry run pydocstyle pysrc
rust-build-test:
if: |
github.event_name == 'pull_request' ||
github.event_name == 'merge_group' ||
(github.event_name == 'push' && github.ref == 'refs/heads/main')
runs-on:
group: linux-rust-builds
steps:
- uses: actions/checkout@v3
- name: Install stable Rust toolchain
id: toolchain
uses: dtolnay/rust-toolchain@master
with:
toolchain: stable
components: clippy
- uses: awalsh128/cache-apt-pkgs-action@latest
with:
packages: protobuf-compiler lld
version: 1.0
# For caching, we use a special `rust-cache` action, which ensures we
# cache based on the architecture, and also deletes stuff that shouldn't
# be put in the cache (such as the toolchain).
#
# See
# https://matklad.github.io/2021/09/04/fast-rust-builds.html#CI-Caching.
#
# This currently saves the cache on each branch. This may exceed GitHub's
# cache size limit. If we start hitting that, we could choose to only
# cache the `main` branch, which would provide caching for cases where the
# dependencies of crates haven't changed. See
# https://github.com/Swatinem/rust-cache/issues/95.
- name: Cache Rust Dependencies
uses: Swatinem/rust-cache@v2
with:
workspaces:
.
python
# Cache the rocksdb crate. https://github.com/MystenLabs/sui/pull/1164
cache-directories: "~/.cargo/registry/src/**/librocksdb-sys-*"
- name: Rust Tests
uses: actions-rs/cargo@v1
with:
command: test
args: --all-features -- --nocapture --quiet
# Clippy needs to build the code, so we run it as part of the main CI since
# we've already spent the time building it.
- name: Cargo Clippy
uses: actions-rs/clippy-check@v1
with:
token: ${{ secrets.GITHUB_TOKEN }}
# The `lib.rs` or `bin.rs` files for each create *should* specify the same
# flags to ensure vscode, etc. reports warnings. But, specifying them in the
# CI protects against cases where we forgot to do that.
args: >
--all-features
-- -Dwarnings -Drust-2018-idioms -Dnonstandard-style -Dfuture-incompatible
-Dclippy::mod_module_files
-Dclippy::print-stdout
-Dclippy::print-stderr
-Dclippy::undocumented-unsafe-blocks
python-build-test:
if: |
github.event_name == 'pull_request' ||
github.event_name == 'merge_group' ||
(github.event_name == 'push' && github.ref == 'refs/heads/main')
runs-on:
group: linux-rust-builds
steps:
- uses: actions/checkout@v3
- name: Install poetry
run: |
pipx install poetry
poetry config virtualenvs.create true --local
poetry config virtualenvs.in-project true --local
- uses: actions/setup-python@v4
with:
python-version: |
3.9
3.10
3.11
cache: poetry
- name: Install stable Rust toolchain
id: toolchain
uses: dtolnay/rust-toolchain@master
with:
toolchain: stable
components: clippy
- uses: awalsh128/cache-apt-pkgs-action@latest
with:
packages: protobuf-compiler lld
version: 1.0
# For caching, we use a special `rust-cache` action, which ensures we
# cache based on the architecture, and also deletes stuff that shouldn't
# be put in the cache (such as the toolchain).
#
# See
# https://matklad.github.io/2021/09/04/fast-rust-builds.html#CI-Caching.
#
# This currently saves the cache on each branch. This may exceed GitHub's
# cache size limit. If we start hitting that, we could choose to only
# cache the `main` branch, which would provide caching for cases where the
# dependencies of crates haven't changed. See
# https://github.com/Swatinem/rust-cache/issues/95.
- name: Cache Rust Dependencies
uses: Swatinem/rust-cache@v2
with:
workspaces:
.
python
# Cache the rocksdb crate. https://github.com/MystenLabs/sui/pull/1164
cache-directories: "~/.cargo/registry/src/**/librocksdb-sys-*"
- name: Build debug wheels
uses: messense/maturin-action@v1
with:
args: --out dist --profile dev
# Build on host instead of a manylinux container.
container: off
working-directory: python
- name: Determine Version
# The wheel suffix is determined by seeing what `maturin build` printed.
run: |
pip install tomlkit packaging
VERSION=$(python ../scripts/version.py get --normalize pyproject.toml project.version)
WHEEL=dist/kaskada-${VERSION}-cp38-abi3-manylinux_2_31_x86_64.whl
echo "WHEEL: ${WHEEL}"
echo "WHEEL=${WHEEL}" >> $GITHUB_ENV
- name: pytest and mypy
# This installs the kaskada package using the wheel.
# This ensures that we don't accidentally install the version from pypi.
run: |
for V in 3.9 3.10 3.11; do
echo "::group::Install for Python $V"
poetry env use $V
poetry env info
source $(poetry env info --path)/bin/activate
poetry install --only=test --only=typecheck
pip install ${WHEEL} --force-reinstall
echo "::endgroup::"
echo "::group::Test Python $V"
poetry run pytest pytests -S minio
echo "::endgroup::"
echo "::group::MyPy Python $V"
poetry run mypy -- --install-types --non-interactive pysrc pytests
echo "::endgroup::"
deactivate
done
- name: Lint reference docs
run: |
poetry env use 3.11
source $(poetry env info --path)/bin/activate
poetry install --with=docs
pip install ${WHEEL} --force-reinstall
cd docs
python _scripts/lint_reference.py
deactivate
- name: Set up Quarto
uses: quarto-dev/quarto-actions/setup@v2
- name: Build docs
run: |
poetry env use 3.11
source $(poetry env info --path)/bin/activate
poetry install --with=docs
pip install ${WHEEL} --force-reinstall
cd docs
python _scripts/gen_reference.py
python -m quartodoc interlinks
quarto render --output-dir _site
deactivate
- name: Upload docs
uses: actions/upload-pages-artifact@v2
with:
# Automatically uploads an artifact from the './_site' directory by default
path: ${{ github.workspace }}/python/docs/_site
rust-format:
runs-on: ubuntu-20.04
steps:
- name: Checkout sources
uses: actions/checkout@v3
- name: Install stable toolchain
id: toolchain
uses: dtolnay/rust-toolchain@master
with:
toolchain: stable
components: rustfmt
# `cargo fmt` doesn't need any dependencies, so no need to cache.
# Restoring the cache takes 2m. Running rustfmt takes 10s.
- name: Run cargo fmt (main Rust code)
uses: actions-rs/cargo@v1
with:
command: fmt
args: --all -- --check
- name: Run cargo fmt (Python Rust code)
uses: actions-rs/cargo@v1
with:
command: fmt
args: --all --manifest-path python/Cargo.toml -- --check
rust-audit:
runs-on: ubuntu-20.04
# Audits don't need a toolchain or any of the dependencies, so we don't cache.
# Restoring the cache takes 2m. Running `cargo deny` takes 10s.
strategy:
matrix:
checks:
- advisories
- bans licenses sources
# Prevent sudden announcement of a new advisory from failing ci:
continue-on-error: ${{ matrix.checks == 'advisories' }}
steps:
- uses: actions/checkout@v3
- uses: EmbarkStudios/cargo-deny-action@v1
with:
command: check ${{ matrix.checks }}
rust-unused-deps:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
# This doesn't use the `cargo-machete` action since that doesn't cache
# the machete binary.env:
- name: Install cargo-machete from crates.io
uses: baptiste0928/cargo-install@v2
with:
crate: cargo-machete
- name: Machete (Sparrow)
working-directory:
run:
cargo machete
- name: Machete (Python)
run:
cargo machete
docs-deploy:
# Deploy docs on push to main.
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
# Deployment job
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
# Grant GITHUB_TOKEN the permissions required to make a Pages deployment
permissions:
pages: write # to deploy to Pages
id-token: write # to verify the deployment originates from an appropriate source
runs-on: ubuntu-latest
needs: [python-build-test]
steps:
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v2