refactor: remove the queue in LanceArrowWriter to reduce memory usage for spark sink #3399
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Run Java JNI Tests | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| paths: | |
| - java/** | |
| - rust/** | |
| - .github/workflows/java.yml | |
| env: | |
| # This env var is used by Swatinem/rust-cache@v2 for the cache | |
| # key, so we set it to make sure it is always consistent. | |
| CARGO_TERM_COLOR: always | |
| # Disable full debug symbol generation to speed up CI build and keep memory down | |
| # "1" means line tables only, which is useful for panic tracebacks. | |
| RUSTFLAGS: "-C debuginfo=1" | |
| RUST_BACKTRACE: "1" | |
| # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html | |
| # CI builds are faster with incremental disabled. | |
| CARGO_INCREMENTAL: "0" | |
| CARGO_BUILD_JOBS: "1" | |
| jobs: | |
| rust-clippy-fmt: | |
| runs-on: ubuntu-22.04 | |
| name: Rust Clippy and Fmt Check | |
| defaults: | |
| run: | |
| working-directory: ./java/core/lance-jni | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: java/core/lance-jni | |
| - name: Install dependencies | |
| run: | | |
| sudo apt update | |
| sudo apt install -y protobuf-compiler libssl-dev | |
| - name: Run cargo fmt | |
| run: cargo fmt --check | |
| - name: Rust Clippy | |
| run: cargo clippy --all-targets -- -D warnings | |
| build-and-test-java: | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| matrix: | |
| java-version: [8, 11, 17] | |
| name: Build and Test with Java ${{ matrix.java-version }} | |
| defaults: | |
| run: | |
| working-directory: ./java | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: java/core/lance-jni | |
| - name: Install dependencies | |
| run: | | |
| sudo apt update | |
| sudo apt install -y protobuf-compiler libssl-dev | |
| - name: Set up Java ${{ matrix.java-version }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: ${{ matrix.java-version }} | |
| cache: "maven" | |
| - name: Running tests with Java ${{ matrix.java-version }} | |
| run: | | |
| if [ "${{ matrix.java-version }}" == "17" ]; then | |
| export JAVA_TOOL_OPTIONS="$JAVA_TOOL_OPTIONS \ | |
| -XX:+IgnoreUnrecognizedVMOptions \ | |
| --add-opens=java.base/java.lang=ALL-UNNAMED \ | |
| --add-opens=java.base/java.lang.invoke=ALL-UNNAMED \ | |
| --add-opens=java.base/java.lang.reflect=ALL-UNNAMED \ | |
| --add-opens=java.base/java.io=ALL-UNNAMED \ | |
| --add-opens=java.base/java.net=ALL-UNNAMED \ | |
| --add-opens=java.base/java.nio=ALL-UNNAMED \ | |
| --add-opens=java.base/java.util=ALL-UNNAMED \ | |
| --add-opens=java.base/java.util.concurrent=ALL-UNNAMED \ | |
| --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED \ | |
| --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED \ | |
| --add-opens=java.base/sun.nio.ch=ALL-UNNAMED \ | |
| --add-opens=java.base/sun.nio.cs=ALL-UNNAMED \ | |
| --add-opens=java.base/sun.security.action=ALL-UNNAMED \ | |
| --add-opens=java.base/sun.util.calendar=ALL-UNNAMED \ | |
| --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED \ | |
| -Djdk.reflect.useDirectMethodHandle=false \ | |
| -Dio.netty.tryReflectionSetAccessible=true" | |
| fi | |
| mvn clean install |