-
Notifications
You must be signed in to change notification settings - Fork 418
158 lines (127 loc) · 3.69 KB
/
python_build.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
name: python_build
on:
merge_group:
push:
branches: [main]
pull_request:
branches: [main]
defaults:
run:
working-directory: ./python
jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Setup Environment
uses: ./.github/actions/setup-env
- name: Check Python
run: |
python -m venv venv
source venv/bin/activate
pip install ruff==0.5.2 mypy==1.10.1 types-dataclasses typing-extensions
make check-python
- name: Check Rust
run: make check-rust
test-minimal:
name: Python Build (Python 3.8 PyArrow 16.0.0)
runs-on: ubuntu-latest
env:
RUSTFLAGS: "-C debuginfo=line-tables-only"
CARGO_INCREMENTAL: 0
steps:
- uses: actions/checkout@v3
- name: Setup Environment
uses: ./.github/actions/setup-env
with:
python-version: 3.8
- name: Build and install deltalake
run: |
python -m venv venv
source venv/bin/activate
make setup
# Install minimum PyArrow version
pip install -e .[pandas,devel] pyarrow==16.0.0
env:
RUSTFLAGS: "-C debuginfo=line-tables-only"
- name: Run tests
run: |
source venv/bin/activate
make unit-test
test:
name: Python Build (Python 3.10 PyArrow latest)
runs-on: ubuntu-latest
env:
RUSTFLAGS: "-C debuginfo=1"
CARGO_INCREMENTAL: 0
steps:
- uses: actions/checkout@v3
- name: Setup Environment
uses: ./.github/actions/setup-env
- name: Start emulated services
run: docker compose up -d
- name: Build and install deltalake
run: |
python -m venv venv
source venv/bin/activate
make develop
- name: Download Data Acceptance Tests (DAT) files
run: make setup-dat
- name: Run tests
run: |
source venv/bin/activate
python -m pytest -m '((s3 or azure) and integration) or not integration and not benchmark' --doctest-modules
- name: Test without pandas
run: |
source venv/bin/activate
pip uninstall --yes pandas
python -m pytest -m "not pandas and not integration and not benchmark"
pip install pandas
test-pyspark:
name: PySpark Integration Tests
runs-on: ubuntu-latest
env:
RUSTFLAGS: "-C debuginfo=line-tables-only"
CARGO_INCREMENTAL: 0
steps:
- uses: actions/checkout@v3
- name: Setup Environment
uses: ./.github/actions/setup-env
- uses: actions/setup-java@v2
with:
distribution: "zulu"
java-version: "11"
- name: Build and install deltalake
run: |
python -m venv venv
source venv/bin/activate
make develop-pyspark
- name: Run tests
run: |
source venv/bin/activate
make test-pyspark
multi-python-running:
name: Running with Python ${{ matrix.python-version }}
runs-on: ubuntu-latest
env:
RUSTFLAGS: "-C debuginfo=line-tables-only"
CARGO_INCREMENTAL: 0
strategy:
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
steps:
- uses: actions/checkout@v3
- name: Setup Environment
uses: ./.github/actions/setup-env
with:
python-version: ${{ matrix.python-version }}
- name: Build and install deltalake
run: |
python -m venv venv
source venv/bin/activate
make setup
maturin develop
- name: Run deltalake
run: |
source venv/bin/activate
python -c 'import deltalake'