ci_python.yml

# This file was initially generated by maturin v1.2.0, using:
#
#    maturin generate-ci github --pytest -o ../.github/workflows/ci_python.yml
#
# TODO:
# - `safety`
# - `typeguard`
# - `xdoctest`
#
# Currently, `mypy` and `pytest` are running as part of building the rules.
# We could (in theory) use a matrix to do these in parallel fetching the
# built wheels, but it doesn't seem likely to provide a significant speedup.
name: Python CI

# Only one job per-ref
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  # See https://matklad.github.io/2021/09/04/fast-rust-builds.html#CI-Workflow.
  # Speeds up CI builds, which won't have previous results anyway.
  CARGO_INCREMENTAL: 0

on:
  push:
    branches:
      - main
    tags:
      - '*'
  pull_request:
    branches:
      - main
  merge_group:
    branches:
      - main

defaults:
  run:
    shell: bash
    working-directory: python

permissions:
  actions: write
  # Used by clippy action to report lint errors.
  checks: write
  contents: read

jobs:
  # Run lint separately from the build -- it doesn't need the built Rust code.
  #
  # Also, only do it on one machine.
  python-lint:
    if: github.event_name == 'pull_request' || github.event_name == 'merge_group'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - name: Install poetry
        run: |
          pipx install poetry
          poetry config virtualenvs.create true --local
          poetry config virtualenvs.in-project true --local
      - uses: actions/setup-python@v4
        with:
          python-version: 3.11
          cache: poetry
      - name: install deps
        run: |
          poetry install --only=main --only=lint
      - name: black
        run: |
          poetry run black --diff pysrc pytests docs
      - name: flake8
        run: |
          poetry run flake8 pysrc pytests docs
      - name: isort
        run: |
          poetry run isort --filter-files --diff pysrc pytests docs
      - name: pydocstyle
        run: |
          poetry run pydocstyle pysrc

  rust-build-test:
    if: |
      github.event_name == 'pull_request' ||
      github.event_name == 'merge_group' ||
      (github.event_name == 'push' && github.ref == 'refs/heads/main')
    runs-on:
        group: linux-rust-builds
    steps:
      - uses: actions/checkout@v3
      - name: Install stable Rust toolchain
        id: toolchain
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: stable
          components: clippy
      - uses: awalsh128/cache-apt-pkgs-action@latest
        with:
          packages: protobuf-compiler lld
          version: 1.0
      # For caching, we use a special `rust-cache` action, which ensures we
      # cache based on the architecture, and also deletes stuff that shouldn't
      # be put in the cache (such as the toolchain).
      #
      # See
      # https://matklad.github.io/2021/09/04/fast-rust-builds.html#CI-Caching.
      #
      # This currently saves the cache on each branch. This may exceed GitHub's
      # cache size limit. If we start hitting that, we could choose to only
      # cache the `main` branch, which would provide caching for cases where the
      # dependencies of crates haven't changed. See
      # https://github.com/Swatinem/rust-cache/issues/95.
      - name: Cache Rust Dependencies
        uses: Swatinem/rust-cache@v2
        with:
          workspaces:
            .
            python
          # Cache the rocksdb crate. https://github.com/MystenLabs/sui/pull/1164
          cache-directories: "~/.cargo/registry/src/**/librocksdb-sys-*"
      - name: Rust Tests
        uses: actions-rs/cargo@v1
        with:
          command: test
          args: --all-features -- --nocapture --quiet
      # Clippy needs to build the code, so we run it as part of the main CI since
      # we've already spent the time building it.
      - name: Cargo Clippy
        uses: actions-rs/clippy-check@v1
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          # The `lib.rs` or `bin.rs` files for each create *should* specify the same
          # flags to ensure vscode, etc. reports warnings. But, specifying them in the
          # CI protects against cases where we forgot to do that.
          args: >
            --all-features
            -- -Dwarnings -Drust-2018-idioms -Dnonstandard-style -Dfuture-incompatible
            -Dclippy::mod_module_files
            -Dclippy::print-stdout
            -Dclippy::print-stderr
            -Dclippy::undocumented-unsafe-blocks

  python-build-test:
    if: |
      github.event_name == 'pull_request' ||
      github.event_name == 'merge_group' ||
      (github.event_name == 'push' && github.ref == 'refs/heads/main')
    runs-on:
        group: linux-rust-builds
    steps:
      - uses: actions/checkout@v3
      - name: Install poetry
        run: |
          pipx install poetry
          poetry config virtualenvs.create true --local
          poetry config virtualenvs.in-project true --local
      - uses: actions/setup-python@v4
        with:
          python-version: |
            3.9
            3.10
            3.11
          cache: poetry
      - name: Install stable Rust toolchain
        id: toolchain
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: stable
          components: clippy
      - uses: awalsh128/cache-apt-pkgs-action@latest
        with:
          packages: protobuf-compiler lld
          version: 1.0
      # For caching, we use a special `rust-cache` action, which ensures we
      # cache based on the architecture, and also deletes stuff that shouldn't
      # be put in the cache (such as the toolchain).
      #
      # See
      # https://matklad.github.io/2021/09/04/fast-rust-builds.html#CI-Caching.
      #
      # This currently saves the cache on each branch. This may exceed GitHub's
      # cache size limit. If we start hitting that, we could choose to only
      # cache the `main` branch, which would provide caching for cases where the
      # dependencies of crates haven't changed. See
      # https://github.com/Swatinem/rust-cache/issues/95.
      - name: Cache Rust Dependencies
        uses: Swatinem/rust-cache@v2
        with:
          workspaces:
            .
            python
          # Cache the rocksdb crate. https://github.com/MystenLabs/sui/pull/1164
          cache-directories: "~/.cargo/registry/src/**/librocksdb-sys-*"
      - name: Build debug wheels
        uses: messense/maturin-action@v1
        with:
          args: --out dist --profile dev
          # Build on host instead of a manylinux container.
          container: off
          working-directory: python
      - name: Determine Version
        #  The wheel suffix is determined by seeing what `maturin build` printed.
        run: |
          pip install tomlkit packaging
          VERSION=$(python ../scripts/version.py get --normalize pyproject.toml project.version)
          WHEEL=dist/kaskada-${VERSION}-cp38-abi3-manylinux_2_31_x86_64.whl
          echo "WHEEL: ${WHEEL}"
          echo "WHEEL=${WHEEL}" >> $GITHUB_ENV
      - name: pytest and mypy
        # This installs the kaskada package using the wheel.
        # This ensures that we don't accidentally install the version from pypi.
        run: |
          for V in 3.9 3.10 3.11; do
            echo "::group::Install for Python $V"
            poetry env use $V
            poetry env info
            source $(poetry env info --path)/bin/activate
            poetry install --only=test --only=typecheck
            pip install ${WHEEL} --force-reinstall
            echo "::endgroup::"
            echo "::group::Test Python $V"
            poetry run pytest pytests -S minio
            echo "::endgroup::"
            echo "::group::MyPy Python $V"
            poetry run mypy -- --install-types --non-interactive pysrc pytests
            echo "::endgroup::"
            deactivate
          done
      - name: Lint reference docs
        run: |
          poetry env use 3.11
          source $(poetry env info --path)/bin/activate
          poetry install --with=docs
          pip install ${WHEEL} --force-reinstall
          cd docs
          python _scripts/lint_reference.py
          deactivate
      - name: Set up Quarto
        uses: quarto-dev/quarto-actions/setup@v2
      - name: Build docs
        run: |
          poetry env use 3.11
          source $(poetry env info --path)/bin/activate
          poetry install --with=docs
          pip install ${WHEEL} --force-reinstall
          cd docs
          python _scripts/gen_reference.py
          python -m quartodoc interlinks
          quarto render --output-dir _site
          deactivate
      - name: Upload docs
        uses: actions/upload-pages-artifact@v2
        with:
          # Automatically uploads an artifact from the './_site' directory by default
          path: ${{ github.workspace }}/python/docs/_site

  rust-format:
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3

      - name: Install stable toolchain
        id: toolchain
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: stable
          components: rustfmt

      # `cargo fmt` doesn't need any dependencies, so no need to cache.
      # Restoring the cache takes 2m. Running rustfmt takes 10s.

      - name: Run cargo fmt (main Rust code)
        uses: actions-rs/cargo@v1
        with:
          command: fmt
          args: --all -- --check

      - name: Run cargo fmt (Python Rust code)
        uses: actions-rs/cargo@v1
        with:
          command: fmt
          args: --all --manifest-path python/Cargo.toml -- --check

  rust-audit:
    runs-on: ubuntu-20.04

    # Audits don't need a toolchain or any of the dependencies, so we don't cache.
    # Restoring the cache takes 2m. Running `cargo deny` takes 10s.

    strategy:
      matrix:
        checks:
          - advisories
          - bans licenses sources

    # Prevent sudden announcement of a new advisory from failing ci:
    continue-on-error: ${{ matrix.checks == 'advisories' }}

    steps:
      - uses: actions/checkout@v3
      - uses: EmbarkStudios/cargo-deny-action@v1
        with:
          command: check ${{ matrix.checks }}

  rust-unused-deps:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      # This doesn't use the `cargo-machete` action since that doesn't cache
      # the machete binary.env:
      - name: Install cargo-machete from crates.io
        uses: baptiste0928/cargo-install@v2
        with:
          crate: cargo-machete
      - name: Machete (Sparrow)
        working-directory:
        run:
          cargo machete
      - name: Machete (Python)
        run:
          cargo machete

  docs-deploy:
    # Deploy docs on push to main.
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    # Deployment job
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}

      # Grant GITHUB_TOKEN the permissions required to make a Pages deployment
    permissions:
      pages: write      # to deploy to Pages
      id-token: write   # to verify the deployment originates from an appropriate source

    runs-on: ubuntu-latest
    needs: [python-build-test]
    steps:
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v2