feat(test): integration test for S3 log store with pyspark #7045
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: python_build | |
| on: | |
| merge_group: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| defaults: | |
| run: | |
| working-directory: ./python | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-env | |
| - name: Check Python | |
| run: | | |
| python -m venv venv | |
| source venv/bin/activate | |
| pip install ruff==0.5.2 mypy==1.10.1 types-dataclasses typing-extensions | |
| make check-python | |
| - name: Check Rust | |
| run: make check-rust | |
| test-minimal: | |
| name: Python Build (Python 3.8 PyArrow 8.0.0) | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTFLAGS: "-C debuginfo=line-tables-only" | |
| CARGO_INCREMENTAL: 0 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-env | |
| with: | |
| python-version: 3.8 | |
| - name: Build and install deltalake | |
| run: | | |
| python -m venv venv | |
| source venv/bin/activate | |
| make setup | |
| # Install minimum PyArrow version | |
| pip install -e .[pandas,devel] pyarrow==8.0.0 | |
| env: | |
| RUSTFLAGS: "-C debuginfo=line-tables-only" | |
| - name: Run tests | |
| run: | | |
| source venv/bin/activate | |
| make unit-test | |
| # - name: Run Integration tests | |
| # run: | | |
| # py.test --cov tests -m integration | |
| test: | |
| name: Python Build (Python 3.10 PyArrow latest) | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTFLAGS: "-C debuginfo=1" | |
| CARGO_INCREMENTAL: 0 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-env | |
| - name: Start emulated services | |
| run: docker compose up -d | |
| - name: Build and install deltalake | |
| run: | | |
| python -m venv venv | |
| source venv/bin/activate | |
| make develop | |
| - name: Download Data Acceptance Tests (DAT) files | |
| run: make setup-dat | |
| - name: Run tests | |
| run: | | |
| source venv/bin/activate | |
| python -m pytest -m '((s3 or azure) and integration) or not integration and not benchmark' --doctest-modules | |
| - name: Test without pandas | |
| run: | | |
| source venv/bin/activate | |
| pip uninstall --yes pandas | |
| python -m pytest -m "not pandas and not integration and not benchmark" | |
| pip install pandas | |
| benchmark: | |
| name: Python Benchmark | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTFLAGS: "-C debuginfo=line-tables-only" | |
| CARGO_INCREMENTAL: 0 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-env | |
| - name: Build deltalake in release mode | |
| run: | | |
| python -m venv venv | |
| source venv/bin/activate | |
| MATURIN_EXTRA_ARGS=--release make develop | |
| # Download previous benchmark result from cache (if exists) | |
| - name: Download previous benchmark data | |
| uses: actions/cache@v2 | |
| with: | |
| path: ./cache | |
| key: ${{ runner.os }}-benchmark | |
| - name: Run benchmark | |
| run: | | |
| source venv/bin/activate | |
| pytest tests/test_benchmark.py -m benchmark --benchmark-json output.json | |
| - name: Store benchmark result | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| tool: "pytest" | |
| output-file-path: python/output.json | |
| external-data-json-path: ./cache/benchmark-data.json | |
| fail-on-alert: true | |
| test-pyspark: | |
| name: PySpark Integration Tests | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTFLAGS: "-C debuginfo=line-tables-only" | |
| CARGO_INCREMENTAL: 0 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-env | |
| - uses: actions/setup-java@v2 | |
| with: | |
| distribution: "zulu" | |
| java-version: "11" | |
| - name: Build and install deltalake | |
| run: | | |
| python -m venv venv | |
| source venv/bin/activate | |
| make develop-pyspark | |
| - name: Run tests | |
| run: | | |
| source venv/bin/activate | |
| make test-pyspark | |
| multi-python-running: | |
| name: Running with Python ${{ matrix.python-version }} | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTFLAGS: "-C debuginfo=line-tables-only" | |
| CARGO_INCREMENTAL: 0 | |
| strategy: | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-env | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Build and install deltalake | |
| run: | | |
| python -m venv venv | |
| source venv/bin/activate | |
| make setup | |
| maturin develop | |
| - name: Run deltalake | |
| run: | | |
| source venv/bin/activate | |
| python -c 'import deltalake' |