Merge branch 'main' into partitioned/source-trait

.github/workflows/ci_python.yml

@@ -17,6 +17,11 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  # See https://matklad.github.io/2021/09/04/fast-rust-builds.html#CI-Workflow.
+  # Speeds up CI builds, which won't have previous results anyway.
+  CARGO_INCREMENTAL: 0
+
 on:
   push:
     branches:
@@ -36,13 +41,16 @@ defaults:
     working-directory: python
 
 permissions:
+  actions: write
+  # Used by clippy action to report lint errors.
+  checks: write
   contents: read
 
 jobs:
   # Run lint separately from the build -- it doesn't need the built Rust code.
   #
   # Also, only do it on one machine.
-  lint:
+  python-lint:
     if: github.event_name == 'pull_request' || github.event_name == 'merge_group'
     runs-on: ubuntu-latest
     steps:
@@ -72,12 +80,74 @@ jobs:
         run: |
           poetry run pydocstyle pysrc
 
-  debug:
+  rust-build-test:
     if: |
       github.event_name == 'pull_request' ||
       github.event_name == 'merge_group' ||
       (github.event_name == 'push' && github.ref == 'refs/heads/main')
-    runs-on: ubuntu-latest
+    runs-on:
+        group: linux-rust-builds
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install stable Rust toolchain
+        id: toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+          components: clippy
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: protobuf-compiler lld
+          version: 1.0
+      # For caching, we use a special `rust-cache` action, which ensures we
+      # cache based on the architecture, and also deletes stuff that shouldn't
+      # be put in the cache (such as the toolchain).
+      #
+      # See
+      # https://matklad.github.io/2021/09/04/fast-rust-builds.html#CI-Caching.
+      #
+      # This currently saves the cache on each branch. This may exceed GitHub's
+      # cache size limit. If we start hitting that, we could choose to only
+      # cache the `main` branch, which would provide caching for cases where the
+      # dependencies of crates haven't changed. See
+      # https://github.com/Swatinem/rust-cache/issues/95.
+      - name: Cache Rust Dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces:
+            .
+            python
+          # Cache the rocksdb crate. https://github.com/MystenLabs/sui/pull/1164
+          cache-directories: "~/.cargo/registry/src/**/librocksdb-sys-*"
+      - name: Rust Tests
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --all-features -- --nocapture --quiet
+      # Clippy needs to build the code, so we run it as part of the main CI since
+      # we've already spent the time building it.
+      - name: Cargo Clippy
+        uses: actions-rs/clippy-check@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          # The `lib.rs` or `bin.rs` files for each create *should* specify the same
+          # flags to ensure vscode, etc. reports warnings. But, specifying them in the
+          # CI protects against cases where we forgot to do that.
+          args: >
+            --all-features
+            -- -Dwarnings -Drust-2018-idioms -Dnonstandard-style -Dfuture-incompatible
+            -Dclippy::mod_module_files
+            -Dclippy::print-stdout
+            -Dclippy::print-stderr
+            -Dclippy::undocumented-unsafe-blocks
+
+  python-build-test:
+    if: |
+      github.event_name == 'pull_request' ||
+      github.event_name == 'merge_group' ||
+      (github.event_name == 'push' && github.ref == 'refs/heads/main')
+    runs-on:
+        group: linux-rust-builds
     steps:
       - uses: actions/checkout@v3
       - name: Install poetry
@@ -92,24 +162,49 @@ jobs:
             3.10
             3.11
           cache: poetry
+      - name: Install stable Rust toolchain
+        id: toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+          components: clippy
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: protobuf-compiler lld
+          version: 1.0
+      # For caching, we use a special `rust-cache` action, which ensures we
+      # cache based on the architecture, and also deletes stuff that shouldn't
+      # be put in the cache (such as the toolchain).
+      #
+      # See
+      # https://matklad.github.io/2021/09/04/fast-rust-builds.html#CI-Caching.
+      #
+      # This currently saves the cache on each branch. This may exceed GitHub's
+      # cache size limit. If we start hitting that, we could choose to only
+      # cache the `main` branch, which would provide caching for cases where the
+      # dependencies of crates haven't changed. See
+      # https://github.com/Swatinem/rust-cache/issues/95.
+      - name: Cache Rust Dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces:
+            .
+            python
+          # Cache the rocksdb crate. https://github.com/MystenLabs/sui/pull/1164
+          cache-directories: "~/.cargo/registry/src/**/librocksdb-sys-*"
       - name: Build debug wheels
         uses: messense/maturin-action@v1
         with:
-          target: x86_64
-          manylinux: 2_28
           args: --out dist --profile dev
+          # Build on host instead of a manylinux container.
+          container: off
           working-directory: python
-          before-script-linux: |
-            set -e
-            dnf -y install clang protobuf-devel lld
-            clang --version
-            protoc --version
       - name: Determine Version
-        if: github.event_name != 'release'
+        #  The wheel suffix is determined by seeing what `maturin build` printed.
         run: |
           pip install tomlkit packaging
           VERSION=$(python ../scripts/version.py get --normalize pyproject.toml project.version)
-          WHEEL=dist/kaskada-${VERSION}-cp38-abi3-manylinux_2_28_x86_64.whl
+          WHEEL=dist/kaskada-${VERSION}-cp38-abi3-manylinux_2_31_x86_64.whl
           echo "WHEEL: ${WHEEL}"
           echo "WHEEL=${WHEEL}" >> $GITHUB_ENV
       - name: pytest and mypy
@@ -155,6 +250,70 @@ jobs:
           # Automatically uploads an artifact from the './_site' directory by default
           path: ${{ github.workspace }}/python/docs/_build
 
+  rust-format:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+
+      - name: Install stable toolchain
+        id: toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+          components: rustfmt
+
+      # `cargo fmt` doesn't need any dependencies, so no need to cache.
+      # Restoring the cache takes 2m. Running rustfmt takes 10s.
+
+      - name: Run cargo fmt (main Rust code)
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+      - name: Run cargo fmt (Python Rust code)
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all --manifest-path python/Cargo.toml -- --check
+
+  rust-audit:
+    runs-on: ubuntu-20.04
+
+    # Audits don't need a toolchain or any of the dependencies, so we don't cache.
+    # Restoring the cache takes 2m. Running `cargo deny` takes 10s.
+
+    strategy:
+      matrix:
+        checks:
+          - advisories
+          - bans licenses sources
+
+    # Prevent sudden announcement of a new advisory from failing ci:
+    continue-on-error: ${{ matrix.checks == 'advisories' }}
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: EmbarkStudios/cargo-deny-action@v1
+        with:
+          command: check ${{ matrix.checks }}
+
+  rust-unused-deps:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      # This doesn't use the `cargo-machete` action since that doesn't cache
+      # the machete binary.env:
+      - name: Install cargo-machete from crates.io
+        uses: baptiste0928/cargo-install@v2
+        with:
+          crate: cargo-machete
+      - name: Machete
+        run:
+          cargo machete
+
   docs-deploy:
     # Deploy docs on push to main.
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
@@ -169,7 +328,7 @@ jobs:
       id-token: write   # to verify the deployment originates from an appropriate source
 
     runs-on: ubuntu-latest
-    needs: [debug]
+    needs: [python-build-test]
     steps:
       - name: Deploy to GitHub Pages
         id: deployment

.github/workflows/ci_with_rust_nightly.yml

@@ -13,6 +13,11 @@ permissions:
   checks: write
   contents: read
 
+env:
+  # See https://matklad.github.io/2021/09/04/fast-rust-builds.html#CI-Workflow.
+  # Speeds up CI builds, which won't have previous results anyway.
+  CARGO_INCREMENTAL: 0
+
 jobs:
   rust_check:
     name: Nightly Rust Check
@@ -50,7 +55,7 @@ jobs:
       # dependencies of crates haven't changed. See
       # https://github.com/Swatinem/rust-cache/issues/95.
       - name: Nightly Cache Rust Dependencies
-        uses: Swatinem/rust-cache@ce325b60658c1b38465c06cc965b79baf32c1e72
+        uses: Swatinem/rust-cache@v2
         with:
           # Cache the rocksdb crate. https://github.com/MystenLabs/sui/pull/1164
           cache-directories: "~/.cargo/registry/src/**/librocksdb-sys-*"