Skip to content

Commit 1861d57

Browse files
hemidactylushemidactylus
authored
Publish-to-PyPI automated flow (#379)
* wip test release flow * move and rename flow * simplify wip + adapt to run on PRs * TEMP rename other workflows * TEMP remove other flows * fix yaml * fix yaml 2 * publish to test pypi with env * test remove skip-existing for test pypi * revert last commit * refactor uv+env setup into action * add shell 1 * broader publish flow wip 1 * work around passing the boolean flag * make adjustments for flow * add the silly secrest to unit tests (eeeh) * add least py unit test * try again with the lowest-py unit tests * try again (3) with the lowest-py unit tests * checkout * least-python unit tests use the built artifact * add publish step (now on test pypi) and release step (draft, prerelease) * cleanup of working-directory leftovers * restore all other workflows, release process ready to go to main with safety ON
1 parent d912647 commit 1861d57

File tree

4 files changed

+355
-2
lines changed

4 files changed

+355
-2
lines changed

‎.github/actions/uv_setup/action.yml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
name: uv-install
2+
description: Set up Python and uv
3+
4+
inputs:
5+
python-version:
6+
description: Python version, supporting MAJOR.MINOR only
7+
required: true
8+
9+
env:
10+
UV_VERSION: "0.8.11"
11+
12+
runs:
13+
using: composite
14+
steps:
15+
- name: Install uv and set the python version
16+
uses: astral-sh/setup-uv@v6
17+
with:
18+
version: ${{ env.UV_VERSION }}
19+
python-version: ${{ inputs.python-version }}

‎.github/workflows/_test_release.yml‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
name: test-release
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
dangerous-nonmaster-release:
7+
required: false
8+
type: boolean
9+
default: false
10+
description: "Release from a non-master branch (danger!)"
11+
12+
env:
13+
PYTHON_VERSION: "3.12"
14+
UV_FROZEN: "true"
15+
16+
jobs:
17+
build:
18+
if: github.ref == 'refs/heads/main' || inputs.dangerous-nonmaster-release
19+
runs-on: ubuntu-latest
20+
21+
outputs:
22+
pkg-name: ${{ steps.check-version.outputs.pkg-name }}
23+
version: ${{ steps.check-version.outputs.version }}
24+
25+
steps:
26+
- uses: actions/checkout@v4
27+
28+
- name: Set up Python + uv
29+
uses: "./.github/actions/uv_setup"
30+
with:
31+
python-version: ${{ env.PYTHON_VERSION }}
32+
33+
# Separation of build and release, to minimize permissions to the former.
34+
# See: https://github.com/pypa/gh-action-pypi-publish#non-goals
35+
- name: Build project for distribution
36+
run: uv build
37+
38+
- name: Upload build
39+
uses: actions/upload-artifact@v4
40+
with:
41+
name: test-dist
42+
path: dist/
43+
44+
- name: Check Version
45+
id: check-version
46+
shell: python
47+
run: |
48+
import os
49+
import tomllib
50+
with open("pyproject.toml", "rb") as f:
51+
data = tomllib.load(f)
52+
pkg_name = data["project"]["name"]
53+
version = data["project"]["version"]
54+
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
55+
f.write(f"pkg-name={pkg_name}\n")
56+
f.write(f"version={version}\n")
57+
58+
publish:
59+
needs:
60+
- build
61+
runs-on: ubuntu-latest
62+
# This requires an 'environment' with this name on the github repo (and is best practice to restrict permissions)
63+
environment: pypi
64+
permissions:
65+
# Needed by trusted publish: https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/
66+
# Must be configured on (test) PyPI, see https://docs.pypi.org/trusted-publishers/adding-a-publisher/
67+
id-token: write
68+
69+
steps:
70+
- uses: actions/checkout@v4
71+
72+
- uses: actions/download-artifact@v5
73+
with:
74+
name: test-dist
75+
path: dist/
76+
77+
- name: Publish to test PyPI
78+
uses: pypa/gh-action-pypi-publish@release/v1
79+
with:
80+
packages-dir: dist/
81+
verbose: true
82+
print-hash: true
83+
repository-url: https://test.pypi.org/legacy/
84+
# This setting ONLY IN CI AND ON TEST PYPI! See https://github.com/pypa/gh-action-pypi-publish#tolerating-release-package-file-duplicates
85+
skip-existing: true
86+
# TODO determine whether to enable attestations later on, and how
87+
attestations: false

‎.github/workflows/release.yml‎

Lines changed: 247 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,247 @@
1+
name: release
2+
run-name: Release by @${{ github.actor }}
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
dangerous-nonmaster-release:
7+
required: false
8+
type: boolean
9+
default: false
10+
description: "Release from a non-master branch (danger!)"
11+
12+
env:
13+
PYTHON_VERSION: "3.12"
14+
LEAST_PYTHON_VERSION: "3.8"
15+
UV_FROZEN: "true"
16+
UV_NO_SYNC: "true"
17+
18+
jobs:
19+
build:
20+
if: github.ref == 'refs/heads/main' || inputs.dangerous-nonmaster-release
21+
runs-on: ubuntu-latest
22+
23+
outputs:
24+
pkg-name: ${{ steps.check-version.outputs.pkg-name }}
25+
version: ${{ steps.check-version.outputs.version }}
26+
27+
steps:
28+
- uses: actions/checkout@v4
29+
30+
- name: Set up Python + uv
31+
uses: "./.github/actions/uv_setup"
32+
with:
33+
python-version: ${{ env.PYTHON_VERSION }}
34+
35+
- name: Install dependencies including dev
36+
run: uv sync --dev
37+
shell: bash
38+
39+
# Separation of build and release, to minimize permissions to the former.
40+
# See: https://github.com/pypa/gh-action-pypi-publish#non-goals
41+
- name: Build project for distribution
42+
run: uv build
43+
44+
- name: Upload build
45+
uses: actions/upload-artifact@v4
46+
with:
47+
name: dist
48+
path: dist/
49+
50+
- name: Check Version
51+
id: check-version
52+
shell: python
53+
run: |
54+
import os
55+
import tomllib
56+
with open("pyproject.toml", "rb") as f:
57+
data = tomllib.load(f)
58+
pkg_name = data["project"]["name"]
59+
version = data["project"]["version"]
60+
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
61+
f.write(f"pkg-name={pkg_name}\n")
62+
f.write(f"version={version}\n")
63+
64+
test-pypi-publish:
65+
needs:
66+
- build
67+
uses:
68+
./.github/workflows/_test_release.yml
69+
permissions: write-all
70+
with:
71+
dangerous-nonmaster-release: true
72+
dangerous-nonmaster-release: ${{ inputs.dangerous-nonmaster-release }}
73+
secrets: inherit
74+
75+
pre-release-checks:
76+
needs:
77+
- build
78+
- test-pypi-publish
79+
runs-on: ubuntu-latest
80+
steps:
81+
- uses: actions/checkout@v4
82+
83+
- name: Set up Python + uv
84+
uses: "./.github/actions/uv_setup"
85+
id: setup-python
86+
with:
87+
python-version: ${{ env.PYTHON_VERSION }}
88+
89+
- uses: actions/download-artifact@v5
90+
with:
91+
name: dist
92+
path: dist/
93+
94+
- name: Import dist package
95+
shell: bash
96+
env:
97+
PKG_NAME: ${{ needs.build.outputs.pkg-name }}
98+
VERSION: ${{ needs.build.outputs.version }}
99+
run: |
100+
uv venv
101+
VIRTUAL_ENV=.venv uv pip install dist/*.whl
102+
103+
# Make the package (file)name into the imported package name
104+
IMPORT_NAME="$(echo "$PKG_NAME" | sed s/-/_/g)"
105+
106+
uv run python -c "import $IMPORT_NAME; print(dir($IMPORT_NAME))"
107+
108+
- name: Install dependencies including dev
109+
run: uv sync --dev
110+
shell: bash
111+
112+
# Overwrite the local version of the package with the built version
113+
- name: Override-import built package to use for tests
114+
shell: bash
115+
run: |
116+
VIRTUAL_ENV=.venv uv pip install dist/*.whl
117+
118+
- name: Run unit tests
119+
env:
120+
ASTRA_DB_APPLICATION_TOKEN: ${{ secrets.ASTRA_DB_APPLICATION_TOKEN }}
121+
ASTRA_DB_API_ENDPOINT: ${{ secrets.ASTRA_DB_API_ENDPOINT }}
122+
run: make test
123+
124+
# TODO restore integration tests for final
125+
# - name: Run integration tests
126+
# env:
127+
# ASTRA_DB_APPLICATION_TOKEN: ${{ secrets.ASTRA_DB_APPLICATION_TOKEN }}
128+
# ASTRA_DB_API_ENDPOINT: ${{ secrets.ASTRA_DB_API_ENDPOINT }}
129+
# HEADER_EMBEDDING_API_KEY_OPENAI: ${{ secrets.HEADER_EMBEDDING_API_KEY_OPENAI }}
130+
# run: make test-integration
131+
132+
pre-release-unit-lowest-python:
133+
needs:
134+
- build
135+
- test-pypi-publish
136+
runs-on: ubuntu-latest
137+
steps:
138+
- uses: actions/checkout@v4
139+
140+
- name: Set up Python + uv
141+
uses: "./.github/actions/uv_setup"
142+
with:
143+
python-version: ${{ env.LEAST_PYTHON_VERSION }}
144+
145+
- uses: actions/download-artifact@v5
146+
with:
147+
name: dist
148+
path: dist/
149+
150+
- name: Install dependencies including dev
151+
run: |
152+
uv sync --dev
153+
shell: bash
154+
155+
# Overwrite the local version of the package with the built version
156+
- name: Override-import built package to use for tests
157+
shell: bash
158+
run: |
159+
VIRTUAL_ENV=.venv uv pip install dist/*.whl
160+
161+
- name: Run unit test on least Python version
162+
env:
163+
ASTRA_DB_APPLICATION_TOKEN: ${{ secrets.ASTRA_DB_APPLICATION_TOKEN }}
164+
ASTRA_DB_API_ENDPOINT: ${{ secrets.ASTRA_DB_API_ENDPOINT }}
165+
run: make test
166+
167+
publish:
168+
needs:
169+
- build
170+
- test-pypi-publish
171+
- pre-release-checks
172+
- pre-release-unit-lowest-python
173+
runs-on: ubuntu-latest
174+
# This requires an 'environment' with this name on the github repo (and is best practice to restrict permissions)
175+
environment: pypi
176+
permissions:
177+
# Needed by trusted publish: https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/
178+
# Must be configured on (test) PyPI, see https://docs.pypi.org/trusted-publishers/adding-a-publisher/
179+
id-token: write
180+
181+
steps:
182+
- uses: actions/checkout@v4
183+
184+
- name: Set up Python + uv
185+
uses: "./.github/actions/uv_setup"
186+
with:
187+
python-version: ${{ env.PYTHON_VERSION }}
188+
189+
- uses: actions/download-artifact@v5
190+
with:
191+
name: dist
192+
path: dist/
193+
194+
# TODO: retarget prod PyPI (remove 'repository-url' and skip-existing)
195+
- name: Publish package distributions to PyPI - TEST FOR NOW
196+
uses: pypa/gh-action-pypi-publish@release/v1
197+
with:
198+
packages-dir: dist/
199+
verbose: true
200+
print-hash: true
201+
repository-url: https://test.pypi.org/legacy/
202+
# This setting ONLY IN CI AND ON TEST PYPI! See https://github.com/pypa/gh-action-pypi-publish#tolerating-release-package-file-duplicates
203+
skip-existing: true
204+
# TODO determine whether to enable attestations later on, and how
205+
attestations: false
206+
207+
mark-release:
208+
needs:
209+
- build
210+
- test-pypi-publish
211+
- pre-release-checks
212+
- pre-release-unit-lowest-python
213+
- publish
214+
runs-on: ubuntu-latest
215+
permissions:
216+
# Needed by `ncipollo/release-action` for creating the release
217+
contents: write
218+
219+
steps:
220+
- uses: actions/checkout@v4
221+
222+
- name: Set up Python + uv
223+
uses: "./.github/actions/uv_setup"
224+
id: setup-python
225+
with:
226+
python-version: ${{ env.PYTHON_VERSION }}
227+
228+
- uses: actions/download-artifact@v5
229+
with:
230+
name: dist
231+
path: dist/
232+
233+
- name: Create release (TMP draft, prerelease)
234+
uses: ncipollo/release-action@v1
235+
with:
236+
artifacts: "dist/*"
237+
token: ${{ secrets.GITHUB_TOKEN }}
238+
# TODO: restore to false
239+
draft: true
240+
generateReleaseNotes: true
241+
# TODO: restore `v${{ needs.build.outputs.version }}`
242+
tag: test-v${{ needs.build.outputs.version }}
243+
# TODO: restore (no 'test-'')
244+
name: "test-Release v${{ needs.build.outputs.version }}"
245+
commit: ${{ github.sha }}
246+
# TODO: restore false
247+
prerelease: true

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,13 +38,13 @@ format-fix-tests: FMT_FLAGS=
3838
format-fix-tests: format-tests
3939

4040
test-integration:
41-
uv run $(VENV_FLAGS) pytest tests/base -vv
41+
uv run $(VENV_FLAGS) pytest tests/base/integration -vv
4242

4343
test:
4444
uv run $(VENV_FLAGS) pytest tests/base/unit -vv
4545

4646
docker-test-integration:
47-
DOCKER_COMPOSE_LOCAL_DATA_API="yes" uv run pytest tests/base -vv
47+
DOCKER_COMPOSE_LOCAL_DATA_API="yes" uv run pytest tests/base/integration -vv
4848

4949
build:
5050
rm -f dist/astrapy*

0 commit comments

Comments
 (0)