ci(rust): faster CI jobs (#3089)

* Add `Cargo.lock` so that the cache is more effective. * Added one CI job that runs `check` without this, to make sure a fresh solve still works. * Separated `fmt` and `clippy` into their own jobs for faster feedback. * Fixed `rust-cache` to actually be used in builds. * Added [mold](https://github.com/rui314/mold) linker to Linux builds for faster builds * Separated build and test steps so it's easier to see where time is spend * Remote single-threaded requirement from `linux-arm` CI job, so those tests run faster.
lancedb · Nov 5, 2024 · 387c98c · 387c98c
1 parent 98f642c
commit 387c98c
Show file tree

Hide file tree

Showing 5 changed files with 19,097 additions and 59 deletions.
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -1,4 +1,4 @@
-name: Build and run Rust tests
+name: Rust
 on:
   push:
     branches:
@@ -28,6 +28,26 @@ env:
   CARGO_BUILD_JOBS: "1"
 
 jobs:
+  format:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check formatting
+        run: cargo fmt -- --check
+  clippy:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: Swatinem/rust-cache@v2
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Run clippy
+        run: |
+          cargo clippy --version
+          cargo clippy --all-features --tests --benches -- -D warnings
   linux-build:
     runs-on: "ubuntu-24.04"
     timeout-minutes: 45
@@ -44,36 +64,16 @@ jobs:
       - uses: actions/checkout@v4
       # pin the toolchain version to avoid surprises
       - uses: actions-rust-lang/setup-rust-toolchain@v1
-        if: ${{ matrix.toolchain == 'stable' }}
-        with:
-          toolchain: "stable"
-          cache-workspaces: "src/rust"
-          # Disable full debug symbol generation to speed up CI build and keep memory down
-          # "1" means line tables only, which is useful for panic tracebacks.
-          rustflags: "-C debuginfo=1"
-          components: "rustfmt,clippy"
-      - uses: actions-rust-lang/setup-rust-toolchain@v1
-        if: ${{ matrix.toolchain != 'stable' }}
         with:
-          toolchain: "nightly"
-          cache-workspaces: "src/rust"
-          # Disable full debug symbol generation to speed up CI build and keep memory down
-          # "1" means line tables only, which is useful for panic tracebacks.
-          rustflags: "-C debuginfo=1"
-          components: "rustfmt,clippy"
+          toolchain: ${{ matrix.toolchain }}
+      - uses: rui314/setup-mold@v1
       - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
       - name: Install dependencies
         run: |
           sudo apt update
           sudo apt install -y protobuf-compiler libssl-dev
           rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }}
-          rustup component add rustfmt
-      - name: Run cargo fmt
-        run: cargo fmt --check
       - name: Start DynamoDB local for tests
-        if: ${{ matrix.toolchain == 'stable' }}
         run: |
           docker run -d -e AWS_ACCESS_KEY_ID=DUMMYKEY -e AWS_SECRET_ACCESS_KEY=DUMMYKEY -p 8000:8000 amazon/dynamodb-local
       - name: Install cargo-llvm-cov
@@ -82,10 +82,12 @@ jobs:
         if: ${{ matrix.toolchain == 'stable' }}
         run: |
           cargo llvm-cov --workspace --codecov --output-path coverage.codecov --all-features
+      - name: Build tests (nightly)
+        run: cargo test --all-features --workspace --no-run
       - name: Run tests (nightly)
         if: ${{ matrix.toolchain != 'stable' }}
         run: |
-          cargo test
+          cargo test --all-features --workspace
       - name: Upload coverage to Codecov
         if: ${{ matrix.toolchain == 'stable' }}
         uses: codecov/codecov-action@v4
@@ -100,31 +102,25 @@ jobs:
     timeout-minutes: 45
     steps:
       - uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
       - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
           toolchain: "stable"
-          cache-workspaces: "src/rust"
-          # Disable full debug symbol generation to speed up CI build and keep memory down
-          # "1" means line tables only, which is useful for panic tracebacks.
-          rustflags: "-C debuginfo=1"
-          components: "rustfmt,clippy"
+      - uses: rui314/setup-mold@v1
+      - uses: Swatinem/rust-cache@v2
       - name: Install dependencies
         run: |
           sudo apt -y -qq update
           sudo apt install -y protobuf-compiler libssl-dev pkg-config
-      - name: Run cargo fmt
-        run: cargo fmt --check
-        shell: bash
+      - name: Build tests
+        run: |
+          cargo test --all-features --no-run
       - name: Start DynamoDB local for tests
         run: |
           docker run -d -e AWS_ACCESS_KEY_ID=DUMMYKEY -e AWS_SECRET_ACCESS_KEY=DUMMYKEY -p 8000:8000 amazon/dynamodb-local
       - name: Run tests
         run: |
-          cargo test --all-features -- --test-threads 1
-  clippy_and_benchmark:
+          cargo test --all-features
+  build-no-lock:
     runs-on: ubuntu-24.04
     timeout-minutes: 30
     env:
@@ -133,19 +129,17 @@ jobs:
       CXX: clang
     steps:
       - uses: actions/checkout@v4
+      # Remote cargo.lock to force a fresh build
+      - name: Remove Cargo.lock
+        run: rm -f Cargo.lock
+      - uses: rui314/setup-mold@v1
       - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
       - name: Install dependencies
         run: |
           sudo apt update
           sudo apt install -y protobuf-compiler libssl-dev
-      - name: Run clippy
-        run: |
-          cargo clippy --version
-          cargo clippy --all-features --tests --benches -- -D warnings
-      - name: Build benchmarks
-        run: cargo build --benches
+      - name: Build all
+        run: cargo build --benches --all-features --tests
   mac-build:
     runs-on: "macos-14"
     timeout-minutes: 45
@@ -160,8 +154,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
       - name: Select new xcode
         # Default XCode right now is 15.0.1, which contains a bug that causes
         # backtraces to not show properly. See:
@@ -172,12 +164,12 @@ jobs:
       - name: Set up Rust
         run: |
           rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }}
-          rustup component add rustfmt
+      - name: Build tests
+        run: cargo test --all-features --no-run
       - name: Run tests
-        # Check all benches, even though we aren't going to run them.
-        run: |
-          cargo build --tests --benches --all-features --workspace
-          cargo test --all-features
+        run: cargo test --all-features
+      - name: Check benchmarks
+        run: cargo check --benches --all-features
   windows-build:
     runs-on: windows-latest
     defaults:
@@ -186,8 +178,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
       - name: Install Protoc v21.12
         working-directory: C:\
         run: |
@@ -197,11 +187,12 @@ jobs:
           7z x protoc.zip
           Add-Content $env:GITHUB_PATH "C:\protoc\bin"
         shell: powershell
+      - name: Build tests
+        run: cargo test --no-run
       - name: Run tests
-        # Check all benches, even though we aren't going to run them.
-        run: |
-          cargo build --tests --benches --all-features --workspace
-          cargo test
+        run: cargo test
+      - name: Check benchmarks
+        run: cargo check --benches
   msrv:
     # Check the minimum supported Rust version
     name: MSRV Check - Rust v${{ matrix.msrv }}
@@ -217,6 +208,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: true
+      - uses: Swatinem/rust-cache@v2
       - name: Install dependencies
         run: |
           sudo apt update

diff --git a/.gitignore b/.gitignore
@@ -89,7 +89,6 @@ wheelhouse
 
 # Rust
 target
-Cargo.lock
 **/sccache.log
 
 # c++ lsp