-
Notifications
You must be signed in to change notification settings - Fork 15
335 lines (313 loc) · 11.2 KB
/
ci_python.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
# This file was initially generated by maturin v1.2.0, using:
#
# maturin generate-ci github --pytest -o ../.github/workflows/ci_python.yml
#
# TODO:
# - `safety`
# - `typeguard`
# - `xdoctest`
#
# Currently, `mypy` and `pytest` are running as part of building the rules.
# We could (in theory) use a matrix to do these in parallel fetching the
# built wheels, but it doesn't seem likely to provide a significant speedup.
name: Python CI
# Only one job per-ref
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
# See https://matklad.github.io/2021/09/04/fast-rust-builds.html#CI-Workflow.
# Speeds up CI builds, which won't have previous results anyway.
CARGO_INCREMENTAL: 0
on:
push:
branches:
- main
tags:
- '*'
pull_request:
branches:
- main
merge_group:
branches:
- main
defaults:
run:
shell: bash
working-directory: python
permissions:
actions: write
# Used by clippy action to report lint errors.
checks: write
contents: read
jobs:
# Run lint separately from the build -- it doesn't need the built Rust code.
#
# Also, only do it on one machine.
python-lint:
if: github.event_name == 'pull_request' || github.event_name == 'merge_group'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Install poetry
run: |
pipx install poetry
poetry config virtualenvs.create true --local
poetry config virtualenvs.in-project true --local
- uses: actions/setup-python@v4
with:
python-version: 3.11
cache: poetry
- name: install deps
run: |
poetry install --only=main --only=lint
- name: black
run: |
poetry run black --diff pysrc pytests docs/source
- name: flake8
run: |
poetry run flake8 pysrc pytests docs/source
- name: isort
run: |
poetry run isort --filter-files --diff pysrc pytests docs/source
- name: pydocstyle
run: |
poetry run pydocstyle pysrc
rust-build-test:
if: |
github.event_name == 'pull_request' ||
github.event_name == 'merge_group' ||
(github.event_name == 'push' && github.ref == 'refs/heads/main')
runs-on:
group: linux-rust-builds
steps:
- uses: actions/checkout@v3
- name: Install stable Rust toolchain
id: toolchain
uses: dtolnay/rust-toolchain@master
with:
toolchain: stable
components: clippy
- uses: awalsh128/cache-apt-pkgs-action@latest
with:
packages: protobuf-compiler lld
version: 1.0
# For caching, we use a special `rust-cache` action, which ensures we
# cache based on the architecture, and also deletes stuff that shouldn't
# be put in the cache (such as the toolchain).
#
# See
# https://matklad.github.io/2021/09/04/fast-rust-builds.html#CI-Caching.
#
# This currently saves the cache on each branch. This may exceed GitHub's
# cache size limit. If we start hitting that, we could choose to only
# cache the `main` branch, which would provide caching for cases where the
# dependencies of crates haven't changed. See
# https://github.com/Swatinem/rust-cache/issues/95.
- name: Cache Rust Dependencies
uses: Swatinem/rust-cache@v2
with:
workspaces:
.
python
# Cache the rocksdb crate. https://github.com/MystenLabs/sui/pull/1164
cache-directories: "~/.cargo/registry/src/**/librocksdb-sys-*"
- name: Rust Tests
uses: actions-rs/cargo@v1
with:
command: test
args: --all-features -- --nocapture --quiet
# Clippy needs to build the code, so we run it as part of the main CI since
# we've already spent the time building it.
- name: Cargo Clippy
uses: actions-rs/clippy-check@v1
with:
token: ${{ secrets.GITHUB_TOKEN }}
# The `lib.rs` or `bin.rs` files for each create *should* specify the same
# flags to ensure vscode, etc. reports warnings. But, specifying them in the
# CI protects against cases where we forgot to do that.
args: >
--all-features
-- -Dwarnings -Drust-2018-idioms -Dnonstandard-style -Dfuture-incompatible
-Dclippy::mod_module_files
-Dclippy::print-stdout
-Dclippy::print-stderr
-Dclippy::undocumented-unsafe-blocks
python-build-test:
if: |
github.event_name == 'pull_request' ||
github.event_name == 'merge_group' ||
(github.event_name == 'push' && github.ref == 'refs/heads/main')
runs-on:
group: linux-rust-builds
steps:
- uses: actions/checkout@v3
- name: Install poetry
run: |
pipx install poetry
poetry config virtualenvs.create true --local
poetry config virtualenvs.in-project true --local
- uses: actions/setup-python@v4
with:
python-version: |
3.9
3.10
3.11
cache: poetry
- name: Install stable Rust toolchain
id: toolchain
uses: dtolnay/rust-toolchain@master
with:
toolchain: stable
components: clippy
- uses: awalsh128/cache-apt-pkgs-action@latest
with:
packages: protobuf-compiler lld
version: 1.0
# For caching, we use a special `rust-cache` action, which ensures we
# cache based on the architecture, and also deletes stuff that shouldn't
# be put in the cache (such as the toolchain).
#
# See
# https://matklad.github.io/2021/09/04/fast-rust-builds.html#CI-Caching.
#
# This currently saves the cache on each branch. This may exceed GitHub's
# cache size limit. If we start hitting that, we could choose to only
# cache the `main` branch, which would provide caching for cases where the
# dependencies of crates haven't changed. See
# https://github.com/Swatinem/rust-cache/issues/95.
- name: Cache Rust Dependencies
uses: Swatinem/rust-cache@v2
with:
workspaces:
.
python
# Cache the rocksdb crate. https://github.com/MystenLabs/sui/pull/1164
cache-directories: "~/.cargo/registry/src/**/librocksdb-sys-*"
- name: Build debug wheels
uses: messense/maturin-action@v1
with:
args: --out dist --profile dev
# Build on host instead of a manylinux container.
container: off
working-directory: python
- name: Determine Version
# The wheel suffix is determined by seeing what `maturin build` printed.
run: |
pip install tomlkit packaging
VERSION=$(python ../scripts/version.py get --normalize pyproject.toml project.version)
WHEEL=dist/kaskada-${VERSION}-cp38-abi3-manylinux_2_31_x86_64.whl
echo "WHEEL: ${WHEEL}"
echo "WHEEL=${WHEEL}" >> $GITHUB_ENV
- name: pytest and mypy
# This installs the kaskada package using the wheel.
# This ensures that we don't accidentally install the version from pypi.
run: |
for V in 3.9 3.10 3.11; do
echo "::group::Install for Python $V"
poetry env use $V
poetry env info
source $(poetry env info --path)/bin/activate
poetry install --only=test --only=typecheck
pip install ${WHEEL} --force-reinstall
echo "::endgroup::"
echo "::group::Test Python $V"
poetry run pytest pytests -S minio
echo "::endgroup::"
echo "::group::MyPy Python $V"
poetry run mypy -- --install-types --non-interactive pysrc pytests
echo "::endgroup::"
deactivate
done
- name: Setup QT
# Needed by sphinx-social-cards.
# https://github.com/2bndy5/sphinx-social-cards/blob/main/.github/workflows/build.yml#L54
run: |
sudo apt-get install -y libgl1-mesa-dev libxkbcommon-x11-0
echo "QT_QPA_PLATFORM=offscreen" >> "$GITHUB_ENV"
- name: Build docs
# ablog doesn't currently indicate whether it supports parallel reads,
# leading to a warning.
# when possible, add `"-j", "auto",` to do parallel builds (and in nox).
run: |
poetry env use 3.11
source $(poetry env info --path)/bin/activate
poetry install --with=docs
pip install ${WHEEL} --force-reinstall
sphinx-build docs/source docs/_build -W # -j auto
deactivate
- name: Upload docs
uses: actions/upload-pages-artifact@v2
with:
# Automatically uploads an artifact from the './_site' directory by default
path: ${{ github.workspace }}/python/docs/_build
rust-format:
runs-on: ubuntu-20.04
steps:
- name: Checkout sources
uses: actions/checkout@v3
- name: Install stable toolchain
id: toolchain
uses: dtolnay/rust-toolchain@master
with:
toolchain: stable
components: rustfmt
# `cargo fmt` doesn't need any dependencies, so no need to cache.
# Restoring the cache takes 2m. Running rustfmt takes 10s.
- name: Run cargo fmt (main Rust code)
uses: actions-rs/cargo@v1
with:
command: fmt
args: --all -- --check
- name: Run cargo fmt (Python Rust code)
uses: actions-rs/cargo@v1
with:
command: fmt
args: --all --manifest-path python/Cargo.toml -- --check
rust-audit:
runs-on: ubuntu-20.04
# Audits don't need a toolchain or any of the dependencies, so we don't cache.
# Restoring the cache takes 2m. Running `cargo deny` takes 10s.
strategy:
matrix:
checks:
- advisories
- bans licenses sources
# Prevent sudden announcement of a new advisory from failing ci:
continue-on-error: ${{ matrix.checks == 'advisories' }}
steps:
- uses: actions/checkout@v3
- uses: EmbarkStudios/cargo-deny-action@v1
with:
command: check ${{ matrix.checks }}
rust-unused-deps:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
# This doesn't use the `cargo-machete` action since that doesn't cache
# the machete binary.env:
- name: Install cargo-machete from crates.io
uses: baptiste0928/cargo-install@v2
with:
crate: cargo-machete
- name: Machete
run:
cargo machete
docs-deploy:
# Deploy docs on push to main.
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
# Deployment job
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
# Grant GITHUB_TOKEN the permissions required to make a Pages deployment
permissions:
pages: write # to deploy to Pages
id-token: write # to verify the deployment originates from an appropriate source
runs-on: ubuntu-latest
needs: [python-build-test]
steps:
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v2