tests: Fix session tests (#846) #853
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# This file was initially generated by maturin v1.2.0, using: | |
# | |
# maturin generate-ci github --pytest -o ../.github/workflows/ci_python.yml | |
# | |
# TODO: | |
# - `safety` | |
# - `typeguard` | |
# - `xdoctest` | |
# | |
# Currently, `mypy` and `pytest` are running as part of building the rules. | |
# We could (in theory) use a matrix to do these in parallel fetching the | |
# built wheels, but it doesn't seem likely to provide a significant speedup. | |
name: Python CI | |
# Only one job per-ref | |
concurrency: | |
group: ${{ github.workflow }}-${{ github.ref }} | |
cancel-in-progress: true | |
env: | |
# See https://matklad.github.io/2021/09/04/fast-rust-builds.html#CI-Workflow. | |
# Speeds up CI builds, which won't have previous results anyway. | |
CARGO_INCREMENTAL: 0 | |
on: | |
push: | |
branches: | |
- main | |
tags: | |
- '*' | |
pull_request: | |
branches: | |
- main | |
merge_group: | |
branches: | |
- main | |
defaults: | |
run: | |
shell: bash | |
working-directory: python | |
permissions: | |
actions: write | |
# Used by clippy action to report lint errors. | |
checks: write | |
contents: read | |
jobs: | |
# Run lint separately from the build -- it doesn't need the built Rust code. | |
# | |
# Also, only do it on one machine. | |
python-lint: | |
if: github.event_name == 'pull_request' || github.event_name == 'merge_group' | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v2 | |
- name: Install poetry | |
run: | | |
pipx install poetry | |
poetry config virtualenvs.create true --local | |
poetry config virtualenvs.in-project true --local | |
- uses: actions/setup-python@v4 | |
with: | |
python-version: 3.11 | |
cache: poetry | |
- name: install deps | |
run: | | |
poetry install --only=main --only=lint | |
- name: black | |
run: | | |
poetry run black --diff pysrc pytests docs | |
- name: flake8 | |
run: | | |
poetry run flake8 pysrc pytests docs | |
- name: isort | |
run: | | |
poetry run isort --filter-files --diff pysrc pytests docs | |
- name: pydocstyle | |
run: | | |
poetry run pydocstyle pysrc | |
rust-build-test: | |
if: | | |
github.event_name == 'pull_request' || | |
github.event_name == 'merge_group' || | |
(github.event_name == 'push' && github.ref == 'refs/heads/main') | |
runs-on: | |
group: linux-rust-builds | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Install stable Rust toolchain | |
id: toolchain | |
uses: dtolnay/rust-toolchain@master | |
with: | |
toolchain: stable | |
components: clippy | |
- uses: awalsh128/cache-apt-pkgs-action@latest | |
with: | |
packages: protobuf-compiler lld | |
version: 1.0 | |
# For caching, we use a special `rust-cache` action, which ensures we | |
# cache based on the architecture, and also deletes stuff that shouldn't | |
# be put in the cache (such as the toolchain). | |
# | |
# See | |
# https://matklad.github.io/2021/09/04/fast-rust-builds.html#CI-Caching. | |
# | |
# This currently saves the cache on each branch. This may exceed GitHub's | |
# cache size limit. If we start hitting that, we could choose to only | |
# cache the `main` branch, which would provide caching for cases where the | |
# dependencies of crates haven't changed. See | |
# https://github.com/Swatinem/rust-cache/issues/95. | |
- name: Cache Rust Dependencies | |
uses: Swatinem/rust-cache@v2 | |
with: | |
workspaces: | |
. | |
python | |
# Cache the rocksdb crate. https://github.com/MystenLabs/sui/pull/1164 | |
cache-directories: "~/.cargo/registry/src/**/librocksdb-sys-*" | |
- name: Rust Tests | |
uses: actions-rs/cargo@v1 | |
with: | |
command: test | |
args: --all-features -- --nocapture --quiet | |
# Clippy needs to build the code, so we run it as part of the main CI since | |
# we've already spent the time building it. | |
- name: Cargo Clippy | |
uses: actions-rs/clippy-check@v1 | |
with: | |
token: ${{ secrets.GITHUB_TOKEN }} | |
# The `lib.rs` or `bin.rs` files for each create *should* specify the same | |
# flags to ensure vscode, etc. reports warnings. But, specifying them in the | |
# CI protects against cases where we forgot to do that. | |
args: > | |
--all-features | |
-- -Dwarnings -Drust-2018-idioms -Dnonstandard-style -Dfuture-incompatible | |
-Dclippy::mod_module_files | |
-Dclippy::print-stdout | |
-Dclippy::print-stderr | |
-Dclippy::undocumented-unsafe-blocks | |
python-build-test: | |
if: | | |
github.event_name == 'pull_request' || | |
github.event_name == 'merge_group' || | |
(github.event_name == 'push' && github.ref == 'refs/heads/main') | |
runs-on: | |
group: linux-rust-builds | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Install poetry | |
run: | | |
pipx install poetry | |
poetry config virtualenvs.create true --local | |
poetry config virtualenvs.in-project true --local | |
- uses: actions/setup-python@v4 | |
with: | |
python-version: | | |
3.9 | |
3.10 | |
3.11 | |
cache: poetry | |
- name: Install stable Rust toolchain | |
id: toolchain | |
uses: dtolnay/rust-toolchain@master | |
with: | |
toolchain: stable | |
components: clippy | |
- uses: awalsh128/cache-apt-pkgs-action@latest | |
with: | |
packages: protobuf-compiler lld | |
version: 1.0 | |
# For caching, we use a special `rust-cache` action, which ensures we | |
# cache based on the architecture, and also deletes stuff that shouldn't | |
# be put in the cache (such as the toolchain). | |
# | |
# See | |
# https://matklad.github.io/2021/09/04/fast-rust-builds.html#CI-Caching. | |
# | |
# This currently saves the cache on each branch. This may exceed GitHub's | |
# cache size limit. If we start hitting that, we could choose to only | |
# cache the `main` branch, which would provide caching for cases where the | |
# dependencies of crates haven't changed. See | |
# https://github.com/Swatinem/rust-cache/issues/95. | |
- name: Cache Rust Dependencies | |
uses: Swatinem/rust-cache@v2 | |
with: | |
workspaces: | |
. | |
python | |
# Cache the rocksdb crate. https://github.com/MystenLabs/sui/pull/1164 | |
cache-directories: "~/.cargo/registry/src/**/librocksdb-sys-*" | |
- name: Build debug wheels | |
uses: messense/maturin-action@v1 | |
with: | |
args: --out dist --profile dev | |
# Build on host instead of a manylinux container. | |
container: off | |
working-directory: python | |
- name: Determine Version | |
# The wheel suffix is determined by seeing what `maturin build` printed. | |
run: | | |
pip install tomlkit packaging | |
VERSION=$(python ../scripts/version.py get --normalize pyproject.toml project.version) | |
WHEEL=dist/kaskada-${VERSION}-cp38-abi3-manylinux_2_31_x86_64.whl | |
echo "WHEEL: ${WHEEL}" | |
echo "WHEEL=${WHEEL}" >> $GITHUB_ENV | |
- name: pytest and mypy | |
# This installs the kaskada package using the wheel. | |
# This ensures that we don't accidentally install the version from pypi. | |
run: | | |
for V in 3.9 3.10 3.11; do | |
echo "::group::Install for Python $V" | |
poetry env use $V | |
poetry env info | |
source $(poetry env info --path)/bin/activate | |
poetry install --only=test --only=typecheck | |
pip install ${WHEEL} --force-reinstall | |
echo "::endgroup::" | |
echo "::group::Test Python $V" | |
poetry run pytest pytests -S minio | |
echo "::endgroup::" | |
echo "::group::MyPy Python $V" | |
poetry run mypy -- --install-types --non-interactive pysrc pytests | |
echo "::endgroup::" | |
deactivate | |
done | |
- name: Lint reference docs | |
run: | | |
poetry env use 3.11 | |
source $(poetry env info --path)/bin/activate | |
poetry install --with=docs | |
cd docs | |
python _scripts/lint_reference.py | |
deactivate | |
- name: Set up Quarto | |
uses: quarto-dev/quarto-actions/setup@v2 | |
- name: Build docs | |
run: | | |
poetry env use 3.11 | |
source $(poetry env info --path)/bin/activate | |
cd docs | |
python _scripts/gen_reference.py | |
python -m quartodoc interlinks | |
quarto render --output-dir _site | |
deactivate | |
- name: Upload docs | |
uses: actions/upload-pages-artifact@v2 | |
with: | |
# Automatically uploads an artifact from the './_site' directory by default | |
path: ${{ github.workspace }}/python/docs/_site | |
rust-format: | |
runs-on: ubuntu-20.04 | |
steps: | |
- name: Checkout sources | |
uses: actions/checkout@v3 | |
- name: Install stable toolchain | |
id: toolchain | |
uses: dtolnay/rust-toolchain@master | |
with: | |
toolchain: stable | |
components: rustfmt | |
# `cargo fmt` doesn't need any dependencies, so no need to cache. | |
# Restoring the cache takes 2m. Running rustfmt takes 10s. | |
- name: Run cargo fmt (main Rust code) | |
uses: actions-rs/cargo@v1 | |
with: | |
command: fmt | |
args: --all -- --check | |
- name: Run cargo fmt (Python Rust code) | |
uses: actions-rs/cargo@v1 | |
with: | |
command: fmt | |
args: --all --manifest-path python/Cargo.toml -- --check | |
rust-audit: | |
runs-on: ubuntu-20.04 | |
# Audits don't need a toolchain or any of the dependencies, so we don't cache. | |
# Restoring the cache takes 2m. Running `cargo deny` takes 10s. | |
strategy: | |
matrix: | |
checks: | |
- advisories | |
- bans licenses sources | |
# Prevent sudden announcement of a new advisory from failing ci: | |
continue-on-error: ${{ matrix.checks == 'advisories' }} | |
steps: | |
- uses: actions/checkout@v3 | |
- uses: EmbarkStudios/cargo-deny-action@v1 | |
with: | |
command: check ${{ matrix.checks }} | |
rust-unused-deps: | |
runs-on: ubuntu-latest | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v3 | |
# This doesn't use the `cargo-machete` action since that doesn't cache | |
# the machete binary.env: | |
- name: Install cargo-machete from crates.io | |
uses: baptiste0928/cargo-install@v2 | |
with: | |
crate: cargo-machete | |
- name: Machete (Sparrow) | |
working-directory: | |
run: | |
cargo machete | |
- name: Machete (Python) | |
run: | |
cargo machete | |
docs-deploy: | |
# Deploy docs on push to main. | |
if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
# Deployment job | |
environment: | |
name: github-pages | |
url: ${{ steps.deployment.outputs.page_url }} | |
# Grant GITHUB_TOKEN the permissions required to make a Pages deployment | |
permissions: | |
pages: write # to deploy to Pages | |
id-token: write # to verify the deployment originates from an appropriate source | |
runs-on: ubuntu-latest | |
needs: [python-build-test] | |
steps: | |
- name: Deploy to GitHub Pages | |
id: deployment | |
uses: actions/deploy-pages@v2 |