Python Release #21

Workflow file for this run

.github/workflows/release_python.yml at 5ab9279

	# Release workflow for the Kaskada Python library.
	#
	# This flow is triggered by the creation of a new GitHub release.
	# Generally, the new release should be marked as a "Pre-Release".
	#
	# The creation of the release will cause this to determine the version
	# and build the Python wheels on multiple platforms, before combining
	# them and uploading the new Python library to PyPI.
	#
	# Ideas for improvement
	# ---------------------
	# 1. After releasing we should update the `latest` tag. We don't do this
	# yet because the old version of the Kaskada Python client downloads
	# the latest engine release.
	# 2. After releasing, we should update the version in the checked in
	# project configurations to be a `-dev` pre-release of the next version.
	# 3. We could add a step that uploads the wheels to Test PyPi for verification.
	# 4. We could build the wheels on merge to main, allowing the release process
	# to retrieve them rather than building them. This would also let us do the
	# the bulk of the multi-platform testing regularly, as well as making it
	# less likely that we'd need to recreate a release to deal with problems.
	# 5. Schedule a weekly, automatic release.
	name: Python Release

	# Only one job per-ref
	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: true

	on:
	release:
	types:
	- published

	defaults:
	run:
	shell: bash
	working-directory: python

	permissions:
	contents: read

	jobs:
	version:
	runs-on: ubuntu-latest
	outputs:
	version: ${{ steps.set-version.outputs.version }}
	steps:
	- uses: actions/checkout@v3
	- uses: actions/setup-python@v4
	with:
	python-version: 3.11
	- name: Determine version (current)
	if: github.event_name != 'release'
	run: \|
	pip install tomlkit packaging
	VERSION=$(python ../scripts/version.py get pyproject.toml project.version)
	echo "VERSION=$VERSION" >> $GITHUB_ENV
	- name: Determine version (release)
	if: github.event_name == 'release'
	run: \|
	echo "GITHUB_REF=${GITHUB_REF}"
	VERSION=${GITHUB_REF#refs/tags/v}
	pip install tomlkit packaging
	echo "VERSION=${VERSION}"
	echo "VERSION=$VERSION" >> $GITHUB_ENV
	- name: Set version
	id: set-version
	run: \|
	echo "version=$VERSION" >> $GITHUB_OUTPUT

	build-wheel-macos:
	# Build wheels during release.
	if: github.event_name == 'release'
	runs-on: macos-latest
	needs: [version]
	strategy:
	matrix:
	include:
	- target: x86_64
	wheel_suffix: macosx_11_0_x86_64
	# - target: arm64
	# wheel_suffix: macosx_11_0_arm64
	- target: universal2
	wheel_suffix: macosx_11_0_x86_64.macosx_11_0_arm64.macosx_11_0_universal2
	steps:
	- uses: actions/checkout@v3
	- name: Install poetry
	run: \|
	pipx install poetry
	poetry config virtualenvs.create true --local
	poetry config virtualenvs.in-project true --local
	- uses: actions/setup-python@v4
	with:
	python-version: \|
	3.9
	3.10
	3.11
	architecture: x64
	cache: poetry
	- name: Install Protoc
	uses: arduino/setup-protoc@v1
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- name: Set Version (For Release)
	if: github.event_name == 'release'
	run: \|
	pip install tomlkit packaging
	python ../scripts/version.py set ${{ needs.version.outputs.version }} \
	pyproject.toml:project.version \
	pyproject.toml:tool.poetry.version \
	Cargo.toml:package.version
	- name: Build wheel
	uses: messense/maturin-action@v1
	with:
	target: ${{ matrix.target }}
	args: --release --out dist --sdist
	working-directory: python
	- name: pytest and mypy
	if: matrix.target != 'arm64'
	run: \|
	pip install tomlkit packaging
	VERSION=$(python ../scripts/version.py normalize ${{ needs.version.outputs.version }})
	WHEEL="dist/kaskada-${VERSION}-cp38-abi3-${{ matrix.wheel_suffix }}.whl"
	echo "WHEEL:${WHEEL}"
	for V in 3.9 3.10 3.11; do
	echo "::group::Install for Python $V"
	poetry env use $V
	source $(poetry env info --path)/bin/activate
	poetry install --only=test --only=typecheck
	pip install ${WHEEL} --force-reinstall
	echo "::endgroup::"
	echo "::group::Test Python $V"
	poetry run pytest
	echo "::endgroup::"
	echo "::group::MyPy Python $V"
	poetry run mypy -- --install-types --non-interactive pysrc pytests
	echo "::endgroup::"
	deactivate
	done
	- name: Upload wheels
	uses: actions/upload-artifact@v2
	with:
	name: wheels
	path: ${{ github.workspace }}/python/dist

	build-wheel-windows:
	if: github.event_name == 'release'
	needs: [version]
	runs-on: windows-latest
	steps:
	- uses: actions/checkout@v3
	- name: Install poetry
	run: \|
	pipx install poetry
	poetry config virtualenvs.create true --local
	poetry config virtualenvs.in-project true --local
	- uses: actions/setup-python@v4
	with:
	python-version: 3.11
	architecture: x64
	cache: poetry
	- name: Install Protoc
	uses: arduino/setup-protoc@v1
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- name: Set Version (For Release)
	if: github.event_name == 'release'
	run: \|
	pip install tomlkit packaging
	python ../scripts/version.py set ${{ needs.version.outputs.version }} \
	pyproject.toml:project.version \
	pyproject.toml:tool.poetry.version \
	Cargo.toml:package.version
	- name: Build wheels
	uses: messense/maturin-action@v1
	with:
	target: x64
	args: --release --out dist
	working-directory: python
	- name: pytest and mypy (Windows x64)
	shell: bash
	run: \|
	echo "::group::Install for Python 3.11"
	source $(poetry env info --path)\\Scripts\\activate
	poetry install --only=test --only=typecheck
	pip install tomlkit packaging
	VERSION=$(python ../scripts/version.py normalize ${{ needs.version.outputs.version }})
	WHEEL="dist/kaskada-${VERSION}-cp38-abi3-win_amd64.whl"
	echo "WHEEL:${WHEEL}"
	pip install ${WHEEL} --force-reinstall
	echo "::endgroup::"
	echo "::group::Test Python 3.11"
	poetry run pytest
	echo "::endgroup::"
	echo "::group::MyPy Python 3.11"
	poetry run mypy -- --install-types --non-interactive pysrc pytests
	echo "::endgroup::"
	deactivate
	- name: Upload wheels
	uses: actions/upload-artifact@v2
	with:
	name: wheels
	path: ${{ github.workspace }}/python/dist

	build-wheel-linux:
	if: github.event_name == 'release'
	needs: [version]
	runs-on: ubuntu-latest
	strategy:
	matrix:
	include:
	- target: x86_64
	steps:
	- uses: actions/checkout@v3
	- name: Install poetry
	run: \|
	pipx install poetry
	poetry config virtualenvs.create true --local
	poetry config virtualenvs.in-project true --local
	- uses: actions/setup-python@v4
	with:
	python-version: \|
	3.9
	3.10
	3.11
	cache: poetry
	- name: Set Version (For Release)
	if: github.event_name == 'release'
	run: \|
	pip install tomlkit packaging
	python ../scripts/version.py set ${{ needs.version.outputs.version }} \
	pyproject.toml:project.version \
	pyproject.toml:tool.poetry.version \
	Cargo.toml:package.version
	- name: Build wheels
	uses: messense/maturin-action@v1
	with:
	target: ${{ matrix.target }}
	manylinux: 2_28
	args: --release --out dist
	working-directory: python
	before-script-linux: \|
	set -e
	dnf -y install clang protobuf-devel lld
	clang --version
	protoc --version
	- name: pytest and mypy (Linux x86_64)
	if: matrix.target == 'x86_64'
	run: \|
	pip install tomlkit packaging
	VERSION=$(python ../scripts/version.py normalize ${{ needs.version.outputs.version }})
	WHEEL="dist/kaskada-${VERSION}-cp38-abi3-manylinux_2_28_${{ matrix.target }}.whl"
	echo "WHEEL:${WHEEL}"
	for V in 3.9 3.10 3.11; do
	echo "::group::Install for Python $V"
	poetry env use $V
	poetry env info
	source $(poetry env info --path)/bin/activate
	poetry install --only=test --only=typecheck
	pip install ${WHEEL} --force-reinstall
	echo "::endgroup::"
	echo "::group::Test Python $V"
	poetry run pytest
	echo "::endgroup::"
	echo "::group::MyPy Python $V"
	poetry run mypy -- --install-types --non-interactive pysrc pytests
	echo "::endgroup::"
	deactivate
	done
	- name: Upload wheels
	uses: actions/upload-artifact@v2
	with:
	name: wheels
	path: ${{ github.workspace }}/python/dist

	# Make the source distribution
	sdist:
	if: github.event_name == 'release'
	needs: [version]
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	- name: Set Version (For Release)
	if: github.event_name == 'release'
	run: \|
	pip install tomlkit packaging
	python ../scripts/version.py set ${{ needs.version.outputs.version }} \
	pyproject.toml:project.version \
	pyproject.toml:tool.poetry.version \
	Cargo.toml:package.version
	- name: Build sdist
	uses: PyO3/maturin-action@v1
	with:
	command: sdist
	args: --out dist
	working-directory: python
	- name: Upload sdist
	uses: actions/upload-artifact@v3
	with:
	name: wheels
	path: ${{ github.workspace }}/python/dist

	release:
	name: Release
	runs-on: ubuntu-latest
	environment: pypi
	needs: [build-wheel-linux, build-wheel-windows, build-wheel-macos, sdist]
	permissions:
	id-token: write
	if: github.event_name == 'release' && github.event.action == 'published'
	steps:
	- uses: actions/download-artifact@v3
	with:
	name: wheels
	path: dist

	- uses: pypa/gh-action-pypi-publish@release/v1
	# with:
	# repository-url: https://test.pypi.org/legacy/

	- name: Publish release
	# We don't need the source for the release, so we don't currently check it out.
	# However, we set a default working-directory of `python` which is within the source.
	# To avoid problems, we explicitly set the working-directory to the root here.
	working-directory:
	run: \|
	gh release edit ${{ github.ref_name }} \
	--draft=false --latest --discussion-category "Announcements"
	env:
	GH_TOKEN: ${{ github.token }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Python Release #21

Workflow file

Python Release #21

Jobs

Run details

Workflow file for this run