Build wheels #309

Workflow file for this run

.github/workflows/build-wheels.yml at 4527c16

	name: Build wheels

	# Builds wheels and sends to QE and Aerospike artifactory
	on:
	workflow_dispatch:
	inputs:
	use-server-rc:
	type: boolean
	required: true
	default: false

	jobs:
	update-version:
	runs-on: ubuntu-latest
	permissions:
	# Give the default GITHUB_TOKEN write permission to commit and push the
	# added or changed files to the repository.
	contents: write
	steps:
	- uses: actions/checkout@v2
	with:
	fetch-depth: 0

	- name: Get new version and build number
	run: \|
	pip install python-semantic-release
	latest_tag=$(cat VERSION)
	echo "Bumping off of latest tag $latest_tag"
	new_tag=$(pysemver bump prerelease $latest_tag)
	echo "NEW_TAG=$new_tag" >> $GITHUB_ENV
	echo "The new tag for this build is $new_tag"

	- name: Update version
	run: \|
	sed -i "s/const char version\[] = \".*\";/const char version\[] = \"${{ env.NEW_TAG }}\";/" src/main/aerospike.c
	echo -e "${{ env.NEW_TAG }}" > VERSION

	- name: Commit new version
	uses: stefanzweifel/git-auto-commit-action@v4
	with:
	commit_message: Bump version to ${{ env.NEW_TAG }}

	build-sdist:
	name: Build and install sdist
	runs-on: ubuntu-latest
	needs: update-version
	steps:
	- uses: actions/checkout@v2
	with:
	submodules: recursive
	# Use branch instead of hash that triggered workflow
	ref: ${{ github.ref_name }}

	- uses: actions/setup-python@v2
	with:
	python-version: 3.9
	architecture: 'x64'

	- run: sudo apt update
	- name: Install build dependencies (apt packages)
	run: sudo apt install python3-dev libssl-dev -y
	- name: Install build dependencies (pip packages)
	run: python3 -m pip install build

	- name: Build source distribution
	run: python3 -m build --sdist

	- name: Upload sdist to GitHub
	uses: actions/upload-artifact@v3
	with:
	path: ./dist/*.tar.gz
	# Artifact name, not the file name
	name: sdist

	source-installs:
	strategy:
	matrix:
	distros: [
	# Image name, build-<name> and container name
	["alpine", "alpine"],
	["redhat/ubi9", "rpm-based"],
	["amazonlinux:2023", "rpm-based"]
	]
	fail-fast: false
	runs-on: ubuntu-latest
	steps:
	- name: Run Aerospike server release candidate with latest tag
	if: ${{ inputs.use-server-rc }}
	run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike.jfrog.io/docker/aerospike/aerospike-server-rc:latest

	- name: Run Aerospike server
	if: ${{ !inputs.use-server-rc }}
	run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server

	- uses: actions/checkout@v2
	with:
	submodules: recursive

	- name: Run Alpine container
	run: docker run --name ${{ matrix.distros[1] }} --network host --detach ${{ matrix.distros[0] }} tail -f /dev/null

	- name: Copy repo to container
	run: docker cp . ${{ matrix.distros[1] }}:/aerospike-client-python

	- name: Build and test
	run: docker exec -w /aerospike-client-python ${{ matrix.distros[1] }} ./.github/workflows/build-${{ matrix.distros[1] }}.sh

	manylinux_arm64:
	runs-on: ubuntu-latest
	needs: update-version
	strategy:
	fail-fast: false
	matrix:
	# Python versions to build wheels on
	python: [
	"cp37",
	"cp38",
	"cp39",
	"cp310",
	"cp311"
	]

	steps:
	- uses: actions/checkout@v2
	with:
	submodules: recursive
	ref: ${{ github.ref_name }}

	- name: Set up QEMU for cross compiling arm64
	uses: docker/setup-qemu-action@v2
	with:
	platforms: all

	- uses: ./.github/actions/run-ee-server
	with:
	use-server-rc: ${{ inputs.use-server-rc }}

	- name: Set config.conf to use Docker IP address of Aerospike server
	# config.conf should be copied into the cibuildwheel Docker container
	run: \|
	export SERVER_DOCKER_IP=$(docker container inspect -f '{{ .NetworkSettings.IPAddress }}' aerospike)
	pip install crudini
	sed -i "s/127.0.0.1:3000//" config.conf
	crudini --set config.conf enterprise-edition hosts ${SERVER_DOCKER_IP}:3000
	working-directory: test

	- name: Build wheel
	uses: pypa/[email protected]
	env:
	CIBW_BUILD: ${{ matrix.python }}-manylinux_aarch64
	CIBW_BUILD_FRONTEND: build
	CIBW_BEFORE_ALL_LINUX: >
	yum install openssl-devel -y &&
	yum install python-devel -y &&
	yum install python-setuptools -y
	CIBW_ARCHS: "aarch64"
	CIBW_TEST_COMMAND: >
	cd {project}/test/ &&
	pip install -r requirements.txt &&
	python -m pytest new_tests/

	- name: Upload wheels to GitHub
	uses: actions/upload-artifact@v3
	with:
	path: ./wheelhouse/*.whl
	# Artifact name, not the file name
	name: manylinux-aarch64-${{ matrix.python }}

	manylinux_x86_64:
	runs-on: ubuntu-latest
	needs: update-version
	strategy:
	fail-fast: false
	matrix:
	# Python versions to build wheels on
	python: [
	["cp37", "3.7"],
	["cp38", "3.8"],
	["cp39", "3.9"],
	["cp310", "3.10"],
	["cp311", "3.11"]
	]

	steps:
	- uses: actions/checkout@v2
	with:
	submodules: recursive
	ref: ${{ github.ref_name }}

	- uses: ./.github/actions/run-ee-server
	with:
	use-server-rc: ${{ inputs.use-server-rc }}

	- name: Wait for server to start
	run: sleep 5

	- name: Set config.conf to use Docker IP address of Aerospike server
	# config.conf should be copied into the cibuildwheel Docker container
	run: \|
	export SERVER_DOCKER_IP=$(docker container inspect -f '{{ .NetworkSettings.IPAddress }}' aerospike)
	pip install crudini
	sed -i "s/127.0.0.1:3000//" config.conf
	crudini --set config.conf enterprise-edition hosts ${SERVER_DOCKER_IP}:3000
	working-directory: test

	- name: Build wheel
	uses: pypa/[email protected]
	env:
	CIBW_BUILD: ${{ matrix.python[0] }}-manylinux_x86_64
	CIBW_BUILD_FRONTEND: build
	CIBW_BEFORE_ALL_LINUX: >
	yum install openssl-devel -y &&
	yum install python-devel -y &&
	yum install python-setuptools -y
	CIBW_ARCHS: "x86_64"
	CIBW_TEST_COMMAND: >
	cd {project}/test/ &&
	pip install -r requirements.txt &&
	python -m pytest new_tests/

	- name: Upload wheels to GitHub
	uses: actions/upload-artifact@v3
	with:
	path: ./wheelhouse/*.whl
	# Artifact name, not the file name
	name: manylinux-x86_64-${{ matrix.python[0] }}

	manylinux_x86_64_no_test:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2
	with:
	submodules: recursive

	- name: Build wheel
	uses: pypa/[email protected]
	env:
	CIBW_BUILD: cp39-manylinux_x86_64
	CIBW_BUILD_FRONTEND: build
	CIBW_BEFORE_ALL_LINUX: >
	yum install openssl-devel -y &&
	yum install python-devel -y &&
	yum install python-setuptools -y
	CIBW_ARCHS: "x86_64"

	- name: Upload wheels to GitHub
	uses: actions/upload-artifact@v3
	with:
	path: ./wheelhouse/*.whl
	# Artifact name, not the file name
	name: manylinux-x86_64-cp39-distros

	# Test RPM-based Linux OS's
	manylinux_x86_64_rpm_based:
	needs: manylinux_x86_64_no_test
	strategy:
	matrix:
	image: [
	"redhat/ubi8",
	"amazonlinux:2023"
	]
	fail-fast: false
	runs-on: ubuntu-latest
	services:
	aerospike:
	image: ${{ inputs.use-server-rc && 'aerospike.jfrog.io/docker/aerospike/aerospike-server-rc:latest' \|\| 'aerospike/aerospike-server' }}
	container:
	image: ${{ matrix.image }}
	steps:
	- run: dnf -y update
	- run: dnf -y install git
	- uses: actions/checkout@v3
	with:
	submodules: recursive

	- uses: actions/download-artifact@v3
	with:
	name: manylinux-x86_64-cp39-distros

	- run: dnf -y install python3.9
	# Amazon Linux 2023 doesn't come with pip
	- run: python3.9 -m ensurepip
	- run: python3.9 -m pip install *.whl

	- name: Install test dependencies
	run: python3.9 -m pip install -r requirements.txt
	working-directory: test

	- run: sed -i "s/127.0.0.1/$AEROSPIKE_HOST/" config.conf
	working-directory: test
	env:
	AEROSPIKE_HOST: aerospike

	- name: Run tests
	run: python3.9 -m pytest new_tests/
	working-directory: test

	manylinux_x86_64_debian10:
	needs: manylinux_x86_64_no_test
	runs-on: ubuntu-latest
	services:
	aerospike:
	image: ${{ inputs.use-server-rc && 'aerospike.jfrog.io/docker/aerospike/aerospike-server-rc:latest' \|\| 'aerospike/aerospike-server' }}
	container:
	image: python:3.9-buster
	steps:
	- uses: actions/checkout@v2

	- uses: actions/download-artifact@v3
	with:
	name: manylinux-x86_64-cp39-distros

	- run: python3 -m pip install *.whl

	- name: Install test dependencies
	run: python3 -m pip install -r requirements.txt
	working-directory: test

	- run: sed -i "s/127.0.0.1/$AEROSPIKE_HOST/" config.conf
	working-directory: test
	env:
	AEROSPIKE_HOST: aerospike

	- name: Run tests
	run: python3 -m pytest new_tests/
	working-directory: test

	manylinux_x86_64_ubuntu2204:
	needs: manylinux_x86_64_no_test
	runs-on: ubuntu-22.04
	services:
	aerospike:
	image: ${{ inputs.use-server-rc && 'aerospike.jfrog.io/docker/aerospike/aerospike-server-rc:latest' \|\| 'aerospike/aerospike-server' }}
	ports:
	- 3000:3000
	- 3001:3001
	- 3002:3002
	steps:
	- uses: actions/checkout@v2

	- uses: actions/download-artifact@v3
	with:
	name: manylinux-x86_64-cp39-distros

	- name: Set up Python 3.9
	uses: actions/setup-python@v3
	with:
	python-version: "3.9"

	- run: python3 -m pip install *.whl

	- name: Install test dependencies
	run: python3 -m pip install -r requirements.txt
	working-directory: test

	- name: Run tests
	run: python3 -m pytest new_tests/
	working-directory: test

	macOS-x86:
	needs: update-version
	strategy:
	fail-fast: false
	matrix:
	python-version: [
	["cp37", "3.7"],
	["cp38", "3.8"],
	["cp39", "3.9"],
	["cp310", "3.10"],
	["cp311", "3.11"]
	]
	os: [macos-latest]
	runs-on: ${{ matrix.os }}
	steps:
	- uses: actions/checkout@v3
	with:
	submodules: recursive
	ref: ${{ github.ref_name }}

	- name: Set up Python ${{ matrix.python-version[1] }}
	uses: actions/setup-python@v3
	with:
	python-version: ${{ matrix.python-version[1] }}

	- name: Build wheel
	uses: pypa/[email protected]
	env:
	CIBW_BUILD: ${{ matrix.python-version[0] }}-macosx_x86_64
	CIBW_BUILD_FRONTEND: build
	CIBW_ENVIRONMENT: SSL_LIB_PATH="$(brew --prefix [email protected])/lib/" CPATH="$(brew --prefix [email protected])/include/" STATIC_SSL=1
	CIBW_ARCHS: "x86_64"
	CIBW_BEFORE_TEST: >
	export USE_SERVER_RC=${{ inputs.use-server-rc }} &&
	vagrant up &&
	sleep 3 &&
	pip install -r test/requirements.txt
	CIBW_TEST_COMMAND: >
	cd {project}/test/ &&
	python -m pytest new_tests/

	- name: Save macOS wheel
	uses: actions/upload-artifact@v3
	with:
	name: ${{ matrix.os }}-x86_64-${{ matrix.python-version[0] }}-wheel
	path: wheelhouse/*.whl

	macOS-m1:
	runs-on: [self-hosted, macOS, ARM64]
	needs: update-version
	strategy:
	fail-fast: false
	matrix:
	python-version: [
	"3.8",
	"3.9",
	"3.10",
	"3.11"
	]
	steps:
	- uses: actions/checkout@v3
	with:
	submodules: recursive
	ref: ${{ github.ref_name }}

	# Update dependencies if needed
	- name: Add brew to path
	run: echo PATH=$PATH:/opt/homebrew/bin/ >> $GITHUB_ENV
	- run: brew install python@${{ matrix.python-version }}
	- run: brew install [email protected]

	- run: \|
	openssl_path=$(brew --prefix [email protected])
	echo SSL_LIB_PATH="$openssl_path/lib/" >> $GITHUB_ENV
	echo CPATH="$openssl_path/include/" >> $GITHUB_ENV
	echo STATIC_SSL=1 >> $GITHUB_ENV

	- run: python${{ matrix.python-version }} -m pip install build delocate

	- name: Use server rc
	if: ${{ inputs.use-server-rc }}
	run: echo IMAGE_NAME="aerospike.jfrog.io/docker/aerospike/aerospike-server-rc:latest" >> $GITHUB_ENV

	- name: Use server release
	if: ${{ !inputs.use-server-rc }}
	run: echo IMAGE_NAME="aerospike/aerospike-server" >> $GITHUB_ENV

	- name: Run server
	run: docker run -d -p 3000:3000 --name aerospike ${{ env.IMAGE_NAME }}

	- run: python${{ matrix.python-version }} -m build
	- run: delocate-wheel --require-archs "arm64" -w wheelhouse/ -v dist/*.whl
	- run: python${{ matrix.python-version }} -m pip install --find-links=wheelhouse/ --no-index --force-reinstall aerospike

	- run: python${{ matrix.python-version }} -m pip install -r requirements.txt
	working-directory: test

	- run: python${{ matrix.python-version }} -m pytest new_tests/
	working-directory: test

	- name: Save macOS wheel
	uses: actions/upload-artifact@v3
	with:
	name: macos-amd64-${{ matrix.python-version }}-wheel
	path: wheelhouse/*.whl

	- name: Stop server
	if: ${{ always() }}
	run: \|
	docker container stop aerospike
	docker container prune -f

	send-to-qe:
	needs: [manylinux_x86_64, manylinux_arm64]
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@master
	- run: echo "Upload to QE"
	# - uses: shallwefootball/s3-upload-action@master
	# with:
	# aws_key_id: ${{ secrets.AWS_KEY_ID }}
	# aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
	# aws_bucket: ${{ secrets.AWS_BUCKET }}
	# # Send all distributions to QE build system
	# source_dir: './wheelhouse'

	upload-artifacts-to-jfrog:
	needs: [
	# Cat 1 tests
	source-installs,
	# Wheel installs (cat 2 tests)
	manylinux_x86_64_rpm_based,
	manylinux_x86_64_ubuntu2204,
	manylinux_x86_64_debian10,
	# CentOS 7 Linux cat 2 tests (for all wheels)
	manylinux_x86_64,
	manylinux_arm64,
	macOS-x86,
	macOS-m1,
	]
	runs-on: ubuntu-latest
	steps:
	- uses: actions/download-artifact@v3

	- name: Reorganize artifacts
	run: \|
	mkdir artifacts
	find . -type f -exec mv {} artifacts \;

	- name: Publish package distributions to PyPI
	uses: pypa/gh-action-pypi-publish@release/v1
	with:
	repository-url: ${{ secrets.JFROG_URL }}
	user: ${{ secrets.JFROG_USERNAME }}
	password: ${{ secrets.JFROG_ACCESS_TOKEN }}
	packages-dir: artifacts/
	verbose: true

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Build wheels #309

Workflow file

Build wheels #309

Jobs

Run details

Workflow file for this run