From 45bda3f36ce64795fbd74ad1fd197158bc3448d9 Mon Sep 17 00:00:00 2001 From: "R. Tyler Croy" Date: Tue, 10 Sep 2024 13:11:47 +0000 Subject: [PATCH 1/3] chore: exclude parquet from dependabot as well follows the same release cadence as arrow --- .github/dependabot.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f9dbc3dc7e..1e5b6b27a4 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,3 +11,4 @@ updates: # arrow and datafusion are bumped manually - dependency-name: "arrow*" - dependency-name: "datafusion*" + - dependency-name: "parquet" From 3ad0b4e21fdf5e5c296263ac94b6621d41de892b Mon Sep 17 00:00:00 2001 From: "R. Tyler Croy" Date: Mon, 9 Sep 2024 13:49:37 +0000 Subject: [PATCH 2/3] chore: rearrange github actions a bit This ensures that the benchmarking and codecov code doesn't run in the merge queue where it doesn't make sense (and also causes unsightly :x: ) --- .github/workflows/build.yml | 25 ------------ .github/workflows/codecov.yml | 36 +++++++++++++++++ .github/workflows/python_benchmark.yml | 54 ++++++++++++++++++++++++++ .github/workflows/python_build.yml | 39 ------------------- 4 files changed, 90 insertions(+), 64 deletions(-) create mode 100644 .github/workflows/codecov.yml create mode 100644 .github/workflows/python_benchmark.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ff650eff98..65d0fcb147 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,31 +26,6 @@ jobs: - name: Format run: cargo fmt -- --check - coverage: - runs-on: ubuntu-latest - env: - CARGO_TERM_COLOR: always - steps: - - uses: actions/checkout@v4 - - name: Install rust - uses: actions-rs/toolchain@v1 - with: - profile: default - toolchain: '1.80' - override: true - - name: Install cargo-llvm-cov - uses: taiki-e/install-action@cargo-llvm-cov - - uses: Swatinem/rust-cache@v2 - - name: Generate code coverage - run: cargo llvm-cov --features ${DEFAULT_FEATURES} --workspace --codecov --output-path codecov.json -- --skip read_table_version_hdfs - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 - with: - files: codecov.json - fail_ci_if_error: true - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - build: strategy: fail-fast: false diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml new file mode 100644 index 0000000000..a8d9beabcd --- /dev/null +++ b/.github/workflows/codecov.yml @@ -0,0 +1,36 @@ +name: coverage + +on: + push: + branches: [main, "rust-v*"] + pull_request: + branches: [main, "rust-v*"] + +env: + DEFAULT_FEATURES: "azure,datafusion,s3,gcs,glue,hdfs " + +jobs: + coverage: + runs-on: ubuntu-latest + env: + CARGO_TERM_COLOR: always + steps: + - uses: actions/checkout@v4 + - name: Install rust + uses: actions-rs/toolchain@v1 + with: + profile: default + toolchain: '1.80' + override: true + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + - uses: Swatinem/rust-cache@v2 + - name: Generate code coverage + run: cargo llvm-cov --features ${DEFAULT_FEATURES} --workspace --codecov --output-path codecov.json -- --skip read_table_version_hdfs + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + files: codecov.json + fail_ci_if_error: true + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/python_benchmark.yml b/.github/workflows/python_benchmark.yml new file mode 100644 index 0000000000..896c5cc412 --- /dev/null +++ b/.github/workflows/python_benchmark.yml @@ -0,0 +1,54 @@ +name: python_benchmark + + +# This is separate from the python_build so that it doesn't need to run on the merge group +on: + push: + branches: [main] + pull_request: + branches: [main] + +defaults: + run: + working-directory: ./python + +jobs: + benchmark: + name: Python Benchmark + runs-on: ubuntu-latest + env: + RUSTFLAGS: "-C debuginfo=line-tables-only" + CARGO_INCREMENTAL: 0 + + steps: + - uses: actions/checkout@v2 + + - name: Setup Environment + uses: ./.github/actions/setup-env + + - name: Build deltalake in release mode + run: | + python -m venv venv + source venv/bin/activate + MATURIN_EXTRA_ARGS=--release make develop + + # Download previous benchmark result from cache (if exists) + - name: Download previous benchmark data + uses: actions/cache@v2 + with: + path: ./cache + key: ${{ runner.os }}-benchmark + + - name: Run benchmark + run: | + source venv/bin/activate + pytest tests/test_benchmark.py -m benchmark --benchmark-json output.json + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: "pytest" + output-file-path: python/output.json + external-data-json-path: ./cache/benchmark-data.json + fail-on-alert: true + diff --git a/.github/workflows/python_build.yml b/.github/workflows/python_build.yml index d4ed2d4f7d..dc5483e091 100644 --- a/.github/workflows/python_build.yml +++ b/.github/workflows/python_build.yml @@ -97,45 +97,6 @@ jobs: python -m pytest -m "not pandas and not integration and not benchmark" pip install pandas - benchmark: - name: Python Benchmark - runs-on: ubuntu-latest - env: - RUSTFLAGS: "-C debuginfo=line-tables-only" - CARGO_INCREMENTAL: 0 - - steps: - - uses: actions/checkout@v2 - - - name: Setup Environment - uses: ./.github/actions/setup-env - - - name: Build deltalake in release mode - run: | - python -m venv venv - source venv/bin/activate - MATURIN_EXTRA_ARGS=--release make develop - - # Download previous benchmark result from cache (if exists) - - name: Download previous benchmark data - uses: actions/cache@v2 - with: - path: ./cache - key: ${{ runner.os }}-benchmark - - - name: Run benchmark - run: | - source venv/bin/activate - pytest tests/test_benchmark.py -m benchmark --benchmark-json output.json - - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - tool: "pytest" - output-file-path: python/output.json - external-data-json-path: ./cache/benchmark-data.json - fail-on-alert: true - test-pyspark: name: PySpark Integration Tests runs-on: ubuntu-latest From 8ba4fe0c87a23e0559b42e4db425cca85d3fa4f6 Mon Sep 17 00:00:00 2001 From: "R. Tyler Croy" Date: Tue, 10 Sep 2024 15:18:46 +0000 Subject: [PATCH 3/3] chore: cleanup codecov defaults --- .github/codecov.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/codecov.yml diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 0000000000..dd93c3b7cf --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,17 @@ + +coverage: + status: + project: + default: + # allow some leniency on the deviation of pull requests + threshold: '1%' + informational: true + patch: + default: + informational: true + + +ignore: + - "delta-inspect/" + - "proofs/" + - "**/*.toml"