feat: arrow based table state and checkpoint handling #4607
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build | |
| on: | |
| push: | |
| branches: [main, "rust-v*"] | |
| pull_request: | |
| branches: [main, "rust-v*"] | |
| jobs: | |
| format: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install minimal stable with clippy and rustfmt | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: default | |
| toolchain: stable | |
| override: true | |
| - name: Format | |
| run: cargo fmt -- --check | |
| build: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| - macos-11 | |
| - windows-latest | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install minimal stable with clippy and rustfmt | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: default | |
| toolchain: stable | |
| override: true | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: build and lint with clippy | |
| run: cargo clippy --features azure,datafusion,s3,gcs,glue --tests | |
| - name: Spot-check build for native-tls features | |
| run: cargo clippy --no-default-features --features azure,datafusion,s3-native-tls,gcs,glue-native-tls --tests | |
| - name: Check docs | |
| run: cargo doc --features azure,datafusion,s3,gcs,glue | |
| - name: Check no default features (except rustls) | |
| run: cargo check --no-default-features --features rustls | |
| test: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| - macos-11 | |
| - windows-latest | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| # Disable full debug symbol generation to speed up CI build and keep memory down | |
| RUSTFLAGS: -C debuginfo=line-tables-only | |
| # Disable incremental builds by cargo for CI which should save disk space | |
| # and hopefully avoid final link "No space left on device" | |
| CARGO_INCREMENTAL: 0 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install minimal stable with clippy and rustfmt | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: default | |
| toolchain: "stable" | |
| override: true | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Run tests | |
| run: cargo test --verbose --features datafusion,azure | |
| integration_test: | |
| name: Integration Tests | |
| runs-on: ubuntu-latest | |
| env: | |
| CARGO_INCREMENTAL: 0 | |
| # Disable full debug symbol generation to speed up CI build and keep memory down | |
| # <https://doc.rust-lang.org/cargo/reference/profiles.html> | |
| RUSTFLAGS: "-C debuginfo=line-tables-only" | |
| # https://github.com/rust-lang/cargo/issues/10280 | |
| CARGO_NET_GIT_FETCH_WITH_CLI: "true" | |
| RUST_BACKTRACE: "1" | |
| AWS_DEFAULT_REGION: "us-east-1" | |
| AWS_ACCESS_KEY_ID: deltalake | |
| AWS_SECRET_ACCESS_KEY: weloverust | |
| AWS_ENDPOINT_URL: http://localhost:4566 | |
| AWS_ALLOW_HTTP: "1" | |
| AZURE_USE_EMULATOR: "1" | |
| AZURE_STORAGE_ALLOW_HTTP: "1" | |
| AZURITE_BLOB_STORAGE_URL: "http://localhost:10000" | |
| AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install minimal stable with clippy and rustfmt | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: default | |
| toolchain: stable | |
| override: true | |
| # - uses: actions/setup-java@v3 | |
| # with: | |
| # distribution: "zulu" | |
| # java-version: "17" | |
| # - uses: beyondstorage/setup-hdfs@master | |
| # with: | |
| # hdfs-version: "3.3.2" | |
| # - name: Set Hadoop env | |
| # run: | | |
| # echo "CLASSPATH=$CLASSPATH:`hadoop classpath --glob`" >> $GITHUB_ENV | |
| # echo "LD_LIBRARY_PATH=$JAVA_HOME/lib/server" >> $GITHUB_ENV | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Start emulated services | |
| run: docker-compose up -d | |
| - name: Run tests with rustls (default) | |
| run: | | |
| cargo test --features integration_test,azure,s3,gcs,datafusion | |
| - name: Run tests with native-tls | |
| run: | | |
| cargo test --no-default-features --features integration_test,s3-native-tls,datafusion | |
| parquet2_test: | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTFLAGS: "-C debuginfo=line-tables-only" | |
| CARGO_INCREMENTAL: 0 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install minimal stable with clippy and rustfmt | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: default | |
| toolchain: stable | |
| override: true | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Run tests | |
| working-directory: crates/deltalake-core | |
| run: cargo test --no-default-features --features=parquet2 |