Merge branch 'main' into fix/use_require_files

.github/codecov.yml

@@ -0,0 +1,17 @@
+
+coverage:
+  status:
+    project:
+      default:
+        # allow some leniency on the deviation of pull requests
+        threshold: '1%'
+        informational: true
+    patch:
+      default:
+        informational: true
+
+
+ignore:
+  - "delta-inspect/"
+  - "proofs/"
+  - "**/*.toml"

.github/dependabot.yml

@@ -11,3 +11,4 @@ updates:
       # arrow and datafusion are bumped manually
       - dependency-name: "arrow*"
       - dependency-name: "datafusion*"
+      - dependency-name: "parquet"

.github/workflows/build.yml

@@ -26,31 +26,6 @@ jobs:
       - name: Format
         run: cargo fmt -- --check
 
-  coverage:
-    runs-on: ubuntu-latest
-    env:
-      CARGO_TERM_COLOR: always
-    steps:
-      - uses: actions/checkout@v4
-      - name: Install rust
-        uses: actions-rs/toolchain@v1
-        with:
-          profile: default
-          toolchain: '1.80'
-          override: true
-      - name: Install cargo-llvm-cov
-        uses: taiki-e/install-action@cargo-llvm-cov
-      - uses: Swatinem/rust-cache@v2
-      - name: Generate code coverage
-        run: cargo llvm-cov --features ${DEFAULT_FEATURES} --workspace --codecov --output-path codecov.json -- --skip read_table_version_hdfs
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v4
-        with:
-          files: codecov.json
-          fail_ci_if_error: true
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-
   build:
     strategy:
       fail-fast: false

.github/workflows/codecov.yml

@@ -0,0 +1,36 @@
+name: coverage
+
+on:
+  push:
+    branches: [main, "rust-v*"]
+  pull_request:
+    branches: [main, "rust-v*"]
+
+env:
+  DEFAULT_FEATURES: "azure,datafusion,s3,gcs,glue,hdfs "
+
+jobs:
+  coverage:
+    runs-on: ubuntu-latest
+    env:
+      CARGO_TERM_COLOR: always
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install rust
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: default
+          toolchain: '1.80'
+          override: true
+      - name: Install cargo-llvm-cov
+        uses: taiki-e/install-action@cargo-llvm-cov
+      - uses: Swatinem/rust-cache@v2
+      - name: Generate code coverage
+        run: cargo llvm-cov --features ${DEFAULT_FEATURES} --workspace --codecov --output-path codecov.json -- --skip read_table_version_hdfs
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          files: codecov.json
+          fail_ci_if_error: true
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/python_benchmark.yml

@@ -0,0 +1,54 @@
+name: python_benchmark
+
+
+# This is separate from the python_build so that it doesn't need to run on the merge group
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+defaults:
+  run:
+    working-directory: ./python
+
+jobs:
+  benchmark:
+    name: Python Benchmark
+    runs-on: ubuntu-latest
+    env:
+      RUSTFLAGS: "-C debuginfo=line-tables-only"
+      CARGO_INCREMENTAL: 0
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup Environment
+        uses: ./.github/actions/setup-env
+
+      - name: Build deltalake in release mode
+        run: |
+          python -m venv venv
+          source venv/bin/activate
+          MATURIN_EXTRA_ARGS=--release make develop
+
+      # Download previous benchmark result from cache (if exists)
+      - name: Download previous benchmark data
+        uses: actions/cache@v2
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+
+      - name: Run benchmark
+        run: |
+          source venv/bin/activate
+          pytest tests/test_benchmark.py -m benchmark --benchmark-json output.json
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: "pytest"
+          output-file-path: python/output.json
+          external-data-json-path: ./cache/benchmark-data.json
+          fail-on-alert: true
+

.github/workflows/python_build.yml

@@ -97,45 +97,6 @@ jobs:
           python -m pytest -m "not pandas and not integration and not benchmark"
           pip install pandas
 
-  benchmark:
-    name: Python Benchmark
-    runs-on: ubuntu-latest
-    env:
-      RUSTFLAGS: "-C debuginfo=line-tables-only"
-      CARGO_INCREMENTAL: 0
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Setup Environment
-        uses: ./.github/actions/setup-env
-
-      - name: Build deltalake in release mode
-        run: |
-          python -m venv venv
-          source venv/bin/activate
-          MATURIN_EXTRA_ARGS=--release make develop
-
-      # Download previous benchmark result from cache (if exists)
-      - name: Download previous benchmark data
-        uses: actions/cache@v2
-        with:
-          path: ./cache
-          key: ${{ runner.os }}-benchmark
-
-      - name: Run benchmark
-        run: |
-          source venv/bin/activate
-          pytest tests/test_benchmark.py -m benchmark --benchmark-json output.json
-
-      - name: Store benchmark result
-        uses: benchmark-action/github-action-benchmark@v1
-        with:
-          tool: "pytest"
-          output-file-path: python/output.json
-          external-data-json-path: ./cache/benchmark-data.json
-          fail-on-alert: true
-
   test-pyspark:
     name: PySpark Integration Tests
     runs-on: ubuntu-latest

Cargo.toml

@@ -26,7 +26,7 @@ debug = true
 debug = "line-tables-only"
 
 [workspace.dependencies]
-delta_kernel = { version = "0.3.0" }
+delta_kernel = { version = "=0.3.0" }
 # delta_kernel = { path = "../delta-kernel-rs/kernel", version = "0.3.0" }
 
 # arrow

crates/aws/Cargo.toml

@@ -12,7 +12,7 @@ repository.workspace = true
 rust-version.workspace = true
 
 [dependencies]
-deltalake-core = { version = "0.19.0", path = "../core" }
+deltalake-core = { version = "0.19.1", path = "../core" }
 aws-smithy-runtime-api = { version="1.1.7" }
 aws-smithy-runtime = { version="1.1.7", optional = true}
 aws-credential-types = { version="1.1.7", features = ["hardcoded-credentials"]}

crates/azure/Cargo.toml

@@ -12,7 +12,7 @@ repository.workspace = true
 rust-version.workspace = true
 
 [dependencies]
-deltalake-core = { version = "0.19.0", path = "../core" }
+deltalake-core = { version = "0.19.1", path = "../core" }
 lazy_static = "1"
 
 # workspace depenndecies

crates/catalog-glue/Cargo.toml

@@ -15,7 +15,7 @@ rust-version.workspace = true
 async-trait = { workspace = true }
 aws-config = "1"
 aws-sdk-glue = "1"
-deltalake-core = { version = "0.19.0", path = "../core" }
+deltalake-core = { version = "0.19.1", path = "../core" }
 thiserror = { workspace = true }
 
 [dev-dependencies]

crates/gcp/Cargo.toml

@@ -12,7 +12,7 @@ repository.workspace = true
 rust-version.workspace = true
 
 [dependencies]
-deltalake-core = { version = "0.19.0", path = "../core" }
+deltalake-core = { version = "0.19.1", path = "../core" }
 lazy_static = "1"
 
 # workspace depenndecies

crates/hdfs/Cargo.toml

@@ -12,7 +12,7 @@ repository.workspace = true
 rust-version.workspace = true
 
 [dependencies]
-deltalake-core = { version = "0.19.0", path = "../core" }
+deltalake-core = { version = "0.19.1", path = "../core" }
 hdfs-native-object-store = "0.11"
 
 # workspace dependecies

crates/mount/Cargo.toml

@@ -12,7 +12,7 @@ repository.workspace = true
 rust-version.workspace = true
 
 [dependencies]
-deltalake-core = { version = "0.19.0", path = "../core", features = [
+deltalake-core = { version = "0.19.1", path = "../core", features = [
     "datafusion",
 ] }
 lazy_static = "1"