From 2944dfa0ef3e5cedd3352f5cd7f54f42ed05e9f2 Mon Sep 17 00:00:00 2001 From: "R. Tyler Croy" Date: Wed, 11 Sep 2024 00:34:40 +0000 Subject: [PATCH 1/5] fix: pin the delta_kernel version to 0.3.0 specifically 0.3.1 upgrades arrow to major version 53 and is therefore incompatible with arrow version 52 which we depend upon --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 74a4c73597..ccbb766e0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ debug = true debug = "line-tables-only" [workspace.dependencies] -delta_kernel = { version = "0.3.0" } +delta_kernel = { version = "=0.3.0" } # delta_kernel = { path = "../delta-kernel-rs/kernel", version = "0.3.0" } # arrow From cba7f6871cfc842f8cd97d7e32c35a93ff2e5b6e Mon Sep 17 00:00:00 2001 From: "R. Tyler Croy" Date: Wed, 11 Sep 2024 00:53:43 +0000 Subject: [PATCH 2/5] fix: pin all dependencies to deltalake-core 0.19.1 since 0.19.0 must be yanked Dependencies which might resolve 0.19.0 could inadvertently pull in kernel 0.3.1 and receive a broken build --- crates/aws/Cargo.toml | 2 +- crates/azure/Cargo.toml | 2 +- crates/catalog-glue/Cargo.toml | 2 +- crates/gcp/Cargo.toml | 2 +- crates/hdfs/Cargo.toml | 2 +- crates/mount/Cargo.toml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/aws/Cargo.toml b/crates/aws/Cargo.toml index a62652ef52..9fe0c05934 100644 --- a/crates/aws/Cargo.toml +++ b/crates/aws/Cargo.toml @@ -12,7 +12,7 @@ repository.workspace = true rust-version.workspace = true [dependencies] -deltalake-core = { version = "0.19.0", path = "../core" } +deltalake-core = { version = "0.19.1", path = "../core" } aws-smithy-runtime-api = { version="1.1.7" } aws-smithy-runtime = { version="1.1.7", optional = true} aws-credential-types = { version="1.1.7", features = ["hardcoded-credentials"]} diff --git a/crates/azure/Cargo.toml b/crates/azure/Cargo.toml index 598dadec8d..56181a9f0c 100644 --- a/crates/azure/Cargo.toml +++ b/crates/azure/Cargo.toml @@ -12,7 +12,7 @@ repository.workspace = true rust-version.workspace = true [dependencies] -deltalake-core = { version = "0.19.0", path = "../core" } +deltalake-core = { version = "0.19.1", path = "../core" } lazy_static = "1" # workspace depenndecies diff --git a/crates/catalog-glue/Cargo.toml b/crates/catalog-glue/Cargo.toml index b8598bf7e6..b107d1955a 100644 --- a/crates/catalog-glue/Cargo.toml +++ b/crates/catalog-glue/Cargo.toml @@ -15,7 +15,7 @@ rust-version.workspace = true async-trait = { workspace = true } aws-config = "1" aws-sdk-glue = "1" -deltalake-core = { version = "0.19.0", path = "../core" } +deltalake-core = { version = "0.19.1", path = "../core" } thiserror = { workspace = true } [dev-dependencies] diff --git a/crates/gcp/Cargo.toml b/crates/gcp/Cargo.toml index 8538697d22..623868a6aa 100644 --- a/crates/gcp/Cargo.toml +++ b/crates/gcp/Cargo.toml @@ -12,7 +12,7 @@ repository.workspace = true rust-version.workspace = true [dependencies] -deltalake-core = { version = "0.19.0", path = "../core" } +deltalake-core = { version = "0.19.1", path = "../core" } lazy_static = "1" # workspace depenndecies diff --git a/crates/hdfs/Cargo.toml b/crates/hdfs/Cargo.toml index 7118ea0231..bb9525c302 100644 --- a/crates/hdfs/Cargo.toml +++ b/crates/hdfs/Cargo.toml @@ -12,7 +12,7 @@ repository.workspace = true rust-version.workspace = true [dependencies] -deltalake-core = { version = "0.19.0", path = "../core" } +deltalake-core = { version = "0.19.1", path = "../core" } hdfs-native-object-store = "0.11" # workspace dependecies diff --git a/crates/mount/Cargo.toml b/crates/mount/Cargo.toml index 87e2b9d4e1..2e8e42eccf 100644 --- a/crates/mount/Cargo.toml +++ b/crates/mount/Cargo.toml @@ -12,7 +12,7 @@ repository.workspace = true rust-version.workspace = true [dependencies] -deltalake-core = { version = "0.19.0", path = "../core", features = [ +deltalake-core = { version = "0.19.1", path = "../core", features = [ "datafusion", ] } lazy_static = "1" From 45bda3f36ce64795fbd74ad1fd197158bc3448d9 Mon Sep 17 00:00:00 2001 From: "R. Tyler Croy" Date: Tue, 10 Sep 2024 13:11:47 +0000 Subject: [PATCH 3/5] chore: exclude parquet from dependabot as well follows the same release cadence as arrow --- .github/dependabot.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f9dbc3dc7e..1e5b6b27a4 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,3 +11,4 @@ updates: # arrow and datafusion are bumped manually - dependency-name: "arrow*" - dependency-name: "datafusion*" + - dependency-name: "parquet" From 3ad0b4e21fdf5e5c296263ac94b6621d41de892b Mon Sep 17 00:00:00 2001 From: "R. Tyler Croy" Date: Mon, 9 Sep 2024 13:49:37 +0000 Subject: [PATCH 4/5] chore: rearrange github actions a bit This ensures that the benchmarking and codecov code doesn't run in the merge queue where it doesn't make sense (and also causes unsightly :x: ) --- .github/workflows/build.yml | 25 ------------ .github/workflows/codecov.yml | 36 +++++++++++++++++ .github/workflows/python_benchmark.yml | 54 ++++++++++++++++++++++++++ .github/workflows/python_build.yml | 39 ------------------- 4 files changed, 90 insertions(+), 64 deletions(-) create mode 100644 .github/workflows/codecov.yml create mode 100644 .github/workflows/python_benchmark.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ff650eff98..65d0fcb147 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,31 +26,6 @@ jobs: - name: Format run: cargo fmt -- --check - coverage: - runs-on: ubuntu-latest - env: - CARGO_TERM_COLOR: always - steps: - - uses: actions/checkout@v4 - - name: Install rust - uses: actions-rs/toolchain@v1 - with: - profile: default - toolchain: '1.80' - override: true - - name: Install cargo-llvm-cov - uses: taiki-e/install-action@cargo-llvm-cov - - uses: Swatinem/rust-cache@v2 - - name: Generate code coverage - run: cargo llvm-cov --features ${DEFAULT_FEATURES} --workspace --codecov --output-path codecov.json -- --skip read_table_version_hdfs - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 - with: - files: codecov.json - fail_ci_if_error: true - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - build: strategy: fail-fast: false diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml new file mode 100644 index 0000000000..a8d9beabcd --- /dev/null +++ b/.github/workflows/codecov.yml @@ -0,0 +1,36 @@ +name: coverage + +on: + push: + branches: [main, "rust-v*"] + pull_request: + branches: [main, "rust-v*"] + +env: + DEFAULT_FEATURES: "azure,datafusion,s3,gcs,glue,hdfs " + +jobs: + coverage: + runs-on: ubuntu-latest + env: + CARGO_TERM_COLOR: always + steps: + - uses: actions/checkout@v4 + - name: Install rust + uses: actions-rs/toolchain@v1 + with: + profile: default + toolchain: '1.80' + override: true + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + - uses: Swatinem/rust-cache@v2 + - name: Generate code coverage + run: cargo llvm-cov --features ${DEFAULT_FEATURES} --workspace --codecov --output-path codecov.json -- --skip read_table_version_hdfs + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + files: codecov.json + fail_ci_if_error: true + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/python_benchmark.yml b/.github/workflows/python_benchmark.yml new file mode 100644 index 0000000000..896c5cc412 --- /dev/null +++ b/.github/workflows/python_benchmark.yml @@ -0,0 +1,54 @@ +name: python_benchmark + + +# This is separate from the python_build so that it doesn't need to run on the merge group +on: + push: + branches: [main] + pull_request: + branches: [main] + +defaults: + run: + working-directory: ./python + +jobs: + benchmark: + name: Python Benchmark + runs-on: ubuntu-latest + env: + RUSTFLAGS: "-C debuginfo=line-tables-only" + CARGO_INCREMENTAL: 0 + + steps: + - uses: actions/checkout@v2 + + - name: Setup Environment + uses: ./.github/actions/setup-env + + - name: Build deltalake in release mode + run: | + python -m venv venv + source venv/bin/activate + MATURIN_EXTRA_ARGS=--release make develop + + # Download previous benchmark result from cache (if exists) + - name: Download previous benchmark data + uses: actions/cache@v2 + with: + path: ./cache + key: ${{ runner.os }}-benchmark + + - name: Run benchmark + run: | + source venv/bin/activate + pytest tests/test_benchmark.py -m benchmark --benchmark-json output.json + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: "pytest" + output-file-path: python/output.json + external-data-json-path: ./cache/benchmark-data.json + fail-on-alert: true + diff --git a/.github/workflows/python_build.yml b/.github/workflows/python_build.yml index d4ed2d4f7d..dc5483e091 100644 --- a/.github/workflows/python_build.yml +++ b/.github/workflows/python_build.yml @@ -97,45 +97,6 @@ jobs: python -m pytest -m "not pandas and not integration and not benchmark" pip install pandas - benchmark: - name: Python Benchmark - runs-on: ubuntu-latest - env: - RUSTFLAGS: "-C debuginfo=line-tables-only" - CARGO_INCREMENTAL: 0 - - steps: - - uses: actions/checkout@v2 - - - name: Setup Environment - uses: ./.github/actions/setup-env - - - name: Build deltalake in release mode - run: | - python -m venv venv - source venv/bin/activate - MATURIN_EXTRA_ARGS=--release make develop - - # Download previous benchmark result from cache (if exists) - - name: Download previous benchmark data - uses: actions/cache@v2 - with: - path: ./cache - key: ${{ runner.os }}-benchmark - - - name: Run benchmark - run: | - source venv/bin/activate - pytest tests/test_benchmark.py -m benchmark --benchmark-json output.json - - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - tool: "pytest" - output-file-path: python/output.json - external-data-json-path: ./cache/benchmark-data.json - fail-on-alert: true - test-pyspark: name: PySpark Integration Tests runs-on: ubuntu-latest From 8ba4fe0c87a23e0559b42e4db425cca85d3fa4f6 Mon Sep 17 00:00:00 2001 From: "R. Tyler Croy" Date: Tue, 10 Sep 2024 15:18:46 +0000 Subject: [PATCH 5/5] chore: cleanup codecov defaults --- .github/codecov.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/codecov.yml diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 0000000000..dd93c3b7cf --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,17 @@ + +coverage: + status: + project: + default: + # allow some leniency on the deviation of pull requests + threshold: '1%' + informational: true + patch: + default: + informational: true + + +ignore: + - "delta-inspect/" + - "proofs/" + - "**/*.toml"