Build wheels #309
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Build wheels | |
# Builds wheels and sends to QE and Aerospike artifactory | |
on: | |
workflow_dispatch: | |
inputs: | |
use-server-rc: | |
type: boolean | |
required: true | |
default: false | |
jobs: | |
update-version: | |
runs-on: ubuntu-latest | |
permissions: | |
# Give the default GITHUB_TOKEN write permission to commit and push the | |
# added or changed files to the repository. | |
contents: write | |
steps: | |
- uses: actions/checkout@v2 | |
with: | |
fetch-depth: 0 | |
- name: Get new version and build number | |
run: | | |
pip install python-semantic-release | |
latest_tag=$(cat VERSION) | |
echo "Bumping off of latest tag $latest_tag" | |
new_tag=$(pysemver bump prerelease $latest_tag) | |
echo "NEW_TAG=$new_tag" >> $GITHUB_ENV | |
echo "The new tag for this build is $new_tag" | |
- name: Update version | |
run: | | |
sed -i "s/const char version\[] = \".*\";/const char version\[] = \"${{ env.NEW_TAG }}\";/" src/main/aerospike.c | |
echo -e "${{ env.NEW_TAG }}" > VERSION | |
- name: Commit new version | |
uses: stefanzweifel/git-auto-commit-action@v4 | |
with: | |
commit_message: Bump version to ${{ env.NEW_TAG }} | |
build-sdist: | |
name: Build and install sdist | |
runs-on: ubuntu-latest | |
needs: update-version | |
steps: | |
- uses: actions/checkout@v2 | |
with: | |
submodules: recursive | |
# Use branch instead of hash that triggered workflow | |
ref: ${{ github.ref_name }} | |
- uses: actions/setup-python@v2 | |
with: | |
python-version: 3.9 | |
architecture: 'x64' | |
- run: sudo apt update | |
- name: Install build dependencies (apt packages) | |
run: sudo apt install python3-dev libssl-dev -y | |
- name: Install build dependencies (pip packages) | |
run: python3 -m pip install build | |
- name: Build source distribution | |
run: python3 -m build --sdist | |
- name: Upload sdist to GitHub | |
uses: actions/upload-artifact@v3 | |
with: | |
path: ./dist/*.tar.gz | |
# Artifact name, not the file name | |
name: sdist | |
source-installs: | |
strategy: | |
matrix: | |
distros: [ | |
# Image name, build-<name> and container name | |
["alpine", "alpine"], | |
["redhat/ubi9", "rpm-based"], | |
["amazonlinux:2023", "rpm-based"] | |
] | |
fail-fast: false | |
runs-on: ubuntu-latest | |
steps: | |
- name: Run Aerospike server release candidate with latest tag | |
if: ${{ inputs.use-server-rc }} | |
run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike.jfrog.io/docker/aerospike/aerospike-server-rc:latest | |
- name: Run Aerospike server | |
if: ${{ !inputs.use-server-rc }} | |
run: docker run -d --name aerospike -p 3000-3002:3000-3002 aerospike/aerospike-server | |
- uses: actions/checkout@v2 | |
with: | |
submodules: recursive | |
- name: Run Alpine container | |
run: docker run --name ${{ matrix.distros[1] }} --network host --detach ${{ matrix.distros[0] }} tail -f /dev/null | |
- name: Copy repo to container | |
run: docker cp . ${{ matrix.distros[1] }}:/aerospike-client-python | |
- name: Build and test | |
run: docker exec -w /aerospike-client-python ${{ matrix.distros[1] }} ./.github/workflows/build-${{ matrix.distros[1] }}.sh | |
manylinux_arm64: | |
runs-on: ubuntu-latest | |
needs: update-version | |
strategy: | |
fail-fast: false | |
matrix: | |
# Python versions to build wheels on | |
python: [ | |
"cp37", | |
"cp38", | |
"cp39", | |
"cp310", | |
"cp311" | |
] | |
steps: | |
- uses: actions/checkout@v2 | |
with: | |
submodules: recursive | |
ref: ${{ github.ref_name }} | |
- name: Set up QEMU for cross compiling arm64 | |
uses: docker/setup-qemu-action@v2 | |
with: | |
platforms: all | |
- uses: ./.github/actions/run-ee-server | |
with: | |
use-server-rc: ${{ inputs.use-server-rc }} | |
- name: Set config.conf to use Docker IP address of Aerospike server | |
# config.conf should be copied into the cibuildwheel Docker container | |
run: | | |
export SERVER_DOCKER_IP=$(docker container inspect -f '{{ .NetworkSettings.IPAddress }}' aerospike) | |
pip install crudini | |
sed -i "s/127.0.0.1:3000//" config.conf | |
crudini --set config.conf enterprise-edition hosts ${SERVER_DOCKER_IP}:3000 | |
working-directory: test | |
- name: Build wheel | |
uses: pypa/[email protected] | |
env: | |
CIBW_BUILD: ${{ matrix.python }}-manylinux_aarch64 | |
CIBW_BUILD_FRONTEND: build | |
CIBW_BEFORE_ALL_LINUX: > | |
yum install openssl-devel -y && | |
yum install python-devel -y && | |
yum install python-setuptools -y | |
CIBW_ARCHS: "aarch64" | |
CIBW_TEST_COMMAND: > | |
cd {project}/test/ && | |
pip install -r requirements.txt && | |
python -m pytest new_tests/ | |
- name: Upload wheels to GitHub | |
uses: actions/upload-artifact@v3 | |
with: | |
path: ./wheelhouse/*.whl | |
# Artifact name, not the file name | |
name: manylinux-aarch64-${{ matrix.python }} | |
manylinux_x86_64: | |
runs-on: ubuntu-latest | |
needs: update-version | |
strategy: | |
fail-fast: false | |
matrix: | |
# Python versions to build wheels on | |
python: [ | |
["cp37", "3.7"], | |
["cp38", "3.8"], | |
["cp39", "3.9"], | |
["cp310", "3.10"], | |
["cp311", "3.11"] | |
] | |
steps: | |
- uses: actions/checkout@v2 | |
with: | |
submodules: recursive | |
ref: ${{ github.ref_name }} | |
- uses: ./.github/actions/run-ee-server | |
with: | |
use-server-rc: ${{ inputs.use-server-rc }} | |
- name: Wait for server to start | |
run: sleep 5 | |
- name: Set config.conf to use Docker IP address of Aerospike server | |
# config.conf should be copied into the cibuildwheel Docker container | |
run: | | |
export SERVER_DOCKER_IP=$(docker container inspect -f '{{ .NetworkSettings.IPAddress }}' aerospike) | |
pip install crudini | |
sed -i "s/127.0.0.1:3000//" config.conf | |
crudini --set config.conf enterprise-edition hosts ${SERVER_DOCKER_IP}:3000 | |
working-directory: test | |
- name: Build wheel | |
uses: pypa/[email protected] | |
env: | |
CIBW_BUILD: ${{ matrix.python[0] }}-manylinux_x86_64 | |
CIBW_BUILD_FRONTEND: build | |
CIBW_BEFORE_ALL_LINUX: > | |
yum install openssl-devel -y && | |
yum install python-devel -y && | |
yum install python-setuptools -y | |
CIBW_ARCHS: "x86_64" | |
CIBW_TEST_COMMAND: > | |
cd {project}/test/ && | |
pip install -r requirements.txt && | |
python -m pytest new_tests/ | |
- name: Upload wheels to GitHub | |
uses: actions/upload-artifact@v3 | |
with: | |
path: ./wheelhouse/*.whl | |
# Artifact name, not the file name | |
name: manylinux-x86_64-${{ matrix.python[0] }} | |
manylinux_x86_64_no_test: | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v2 | |
with: | |
submodules: recursive | |
- name: Build wheel | |
uses: pypa/[email protected] | |
env: | |
CIBW_BUILD: cp39-manylinux_x86_64 | |
CIBW_BUILD_FRONTEND: build | |
CIBW_BEFORE_ALL_LINUX: > | |
yum install openssl-devel -y && | |
yum install python-devel -y && | |
yum install python-setuptools -y | |
CIBW_ARCHS: "x86_64" | |
- name: Upload wheels to GitHub | |
uses: actions/upload-artifact@v3 | |
with: | |
path: ./wheelhouse/*.whl | |
# Artifact name, not the file name | |
name: manylinux-x86_64-cp39-distros | |
# Test RPM-based Linux OS's | |
manylinux_x86_64_rpm_based: | |
needs: manylinux_x86_64_no_test | |
strategy: | |
matrix: | |
image: [ | |
"redhat/ubi8", | |
"amazonlinux:2023" | |
] | |
fail-fast: false | |
runs-on: ubuntu-latest | |
services: | |
aerospike: | |
image: ${{ inputs.use-server-rc && 'aerospike.jfrog.io/docker/aerospike/aerospike-server-rc:latest' || 'aerospike/aerospike-server' }} | |
container: | |
image: ${{ matrix.image }} | |
steps: | |
- run: dnf -y update | |
- run: dnf -y install git | |
- uses: actions/checkout@v3 | |
with: | |
submodules: recursive | |
- uses: actions/download-artifact@v3 | |
with: | |
name: manylinux-x86_64-cp39-distros | |
- run: dnf -y install python3.9 | |
# Amazon Linux 2023 doesn't come with pip | |
- run: python3.9 -m ensurepip | |
- run: python3.9 -m pip install *.whl | |
- name: Install test dependencies | |
run: python3.9 -m pip install -r requirements.txt | |
working-directory: test | |
- run: sed -i "s/127.0.0.1/$AEROSPIKE_HOST/" config.conf | |
working-directory: test | |
env: | |
AEROSPIKE_HOST: aerospike | |
- name: Run tests | |
run: python3.9 -m pytest new_tests/ | |
working-directory: test | |
manylinux_x86_64_debian10: | |
needs: manylinux_x86_64_no_test | |
runs-on: ubuntu-latest | |
services: | |
aerospike: | |
image: ${{ inputs.use-server-rc && 'aerospike.jfrog.io/docker/aerospike/aerospike-server-rc:latest' || 'aerospike/aerospike-server' }} | |
container: | |
image: python:3.9-buster | |
steps: | |
- uses: actions/checkout@v2 | |
- uses: actions/download-artifact@v3 | |
with: | |
name: manylinux-x86_64-cp39-distros | |
- run: python3 -m pip install *.whl | |
- name: Install test dependencies | |
run: python3 -m pip install -r requirements.txt | |
working-directory: test | |
- run: sed -i "s/127.0.0.1/$AEROSPIKE_HOST/" config.conf | |
working-directory: test | |
env: | |
AEROSPIKE_HOST: aerospike | |
- name: Run tests | |
run: python3 -m pytest new_tests/ | |
working-directory: test | |
manylinux_x86_64_ubuntu2204: | |
needs: manylinux_x86_64_no_test | |
runs-on: ubuntu-22.04 | |
services: | |
aerospike: | |
image: ${{ inputs.use-server-rc && 'aerospike.jfrog.io/docker/aerospike/aerospike-server-rc:latest' || 'aerospike/aerospike-server' }} | |
ports: | |
- 3000:3000 | |
- 3001:3001 | |
- 3002:3002 | |
steps: | |
- uses: actions/checkout@v2 | |
- uses: actions/download-artifact@v3 | |
with: | |
name: manylinux-x86_64-cp39-distros | |
- name: Set up Python 3.9 | |
uses: actions/setup-python@v3 | |
with: | |
python-version: "3.9" | |
- run: python3 -m pip install *.whl | |
- name: Install test dependencies | |
run: python3 -m pip install -r requirements.txt | |
working-directory: test | |
- name: Run tests | |
run: python3 -m pytest new_tests/ | |
working-directory: test | |
macOS-x86: | |
needs: update-version | |
strategy: | |
fail-fast: false | |
matrix: | |
python-version: [ | |
["cp37", "3.7"], | |
["cp38", "3.8"], | |
["cp39", "3.9"], | |
["cp310", "3.10"], | |
["cp311", "3.11"] | |
] | |
os: [macos-latest] | |
runs-on: ${{ matrix.os }} | |
steps: | |
- uses: actions/checkout@v3 | |
with: | |
submodules: recursive | |
ref: ${{ github.ref_name }} | |
- name: Set up Python ${{ matrix.python-version[1] }} | |
uses: actions/setup-python@v3 | |
with: | |
python-version: ${{ matrix.python-version[1] }} | |
- name: Build wheel | |
uses: pypa/[email protected] | |
env: | |
CIBW_BUILD: ${{ matrix.python-version[0] }}-macosx_x86_64 | |
CIBW_BUILD_FRONTEND: build | |
CIBW_ENVIRONMENT: SSL_LIB_PATH="$(brew --prefix [email protected])/lib/" CPATH="$(brew --prefix [email protected])/include/" STATIC_SSL=1 | |
CIBW_ARCHS: "x86_64" | |
CIBW_BEFORE_TEST: > | |
export USE_SERVER_RC=${{ inputs.use-server-rc }} && | |
vagrant up && | |
sleep 3 && | |
pip install -r test/requirements.txt | |
CIBW_TEST_COMMAND: > | |
cd {project}/test/ && | |
python -m pytest new_tests/ | |
- name: Save macOS wheel | |
uses: actions/upload-artifact@v3 | |
with: | |
name: ${{ matrix.os }}-x86_64-${{ matrix.python-version[0] }}-wheel | |
path: wheelhouse/*.whl | |
macOS-m1: | |
runs-on: [self-hosted, macOS, ARM64] | |
needs: update-version | |
strategy: | |
fail-fast: false | |
matrix: | |
python-version: [ | |
"3.8", | |
"3.9", | |
"3.10", | |
"3.11" | |
] | |
steps: | |
- uses: actions/checkout@v3 | |
with: | |
submodules: recursive | |
ref: ${{ github.ref_name }} | |
# Update dependencies if needed | |
- name: Add brew to path | |
run: echo PATH=$PATH:/opt/homebrew/bin/ >> $GITHUB_ENV | |
- run: brew install python@${{ matrix.python-version }} | |
- run: brew install [email protected] | |
- run: | | |
openssl_path=$(brew --prefix [email protected]) | |
echo SSL_LIB_PATH="$openssl_path/lib/" >> $GITHUB_ENV | |
echo CPATH="$openssl_path/include/" >> $GITHUB_ENV | |
echo STATIC_SSL=1 >> $GITHUB_ENV | |
- run: python${{ matrix.python-version }} -m pip install build delocate | |
- name: Use server rc | |
if: ${{ inputs.use-server-rc }} | |
run: echo IMAGE_NAME="aerospike.jfrog.io/docker/aerospike/aerospike-server-rc:latest" >> $GITHUB_ENV | |
- name: Use server release | |
if: ${{ !inputs.use-server-rc }} | |
run: echo IMAGE_NAME="aerospike/aerospike-server" >> $GITHUB_ENV | |
- name: Run server | |
run: docker run -d -p 3000:3000 --name aerospike ${{ env.IMAGE_NAME }} | |
- run: python${{ matrix.python-version }} -m build | |
- run: delocate-wheel --require-archs "arm64" -w wheelhouse/ -v dist/*.whl | |
- run: python${{ matrix.python-version }} -m pip install --find-links=wheelhouse/ --no-index --force-reinstall aerospike | |
- run: python${{ matrix.python-version }} -m pip install -r requirements.txt | |
working-directory: test | |
- run: python${{ matrix.python-version }} -m pytest new_tests/ | |
working-directory: test | |
- name: Save macOS wheel | |
uses: actions/upload-artifact@v3 | |
with: | |
name: macos-amd64-${{ matrix.python-version }}-wheel | |
path: wheelhouse/*.whl | |
- name: Stop server | |
if: ${{ always() }} | |
run: | | |
docker container stop aerospike | |
docker container prune -f | |
send-to-qe: | |
needs: [manylinux_x86_64, manylinux_arm64] | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@master | |
- run: echo "Upload to QE" | |
# - uses: shallwefootball/s3-upload-action@master | |
# with: | |
# aws_key_id: ${{ secrets.AWS_KEY_ID }} | |
# aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}} | |
# aws_bucket: ${{ secrets.AWS_BUCKET }} | |
# # Send all distributions to QE build system | |
# source_dir: './wheelhouse' | |
upload-artifacts-to-jfrog: | |
needs: [ | |
# Cat 1 tests | |
source-installs, | |
# Wheel installs (cat 2 tests) | |
manylinux_x86_64_rpm_based, | |
manylinux_x86_64_ubuntu2204, | |
manylinux_x86_64_debian10, | |
# CentOS 7 Linux cat 2 tests (for all wheels) | |
manylinux_x86_64, | |
manylinux_arm64, | |
macOS-x86, | |
macOS-m1, | |
] | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/download-artifact@v3 | |
- name: Reorganize artifacts | |
run: | | |
mkdir artifacts | |
find . -type f -exec mv {} artifacts \; | |
- name: Publish package distributions to PyPI | |
uses: pypa/gh-action-pypi-publish@release/v1 | |
with: | |
repository-url: ${{ secrets.JFROG_URL }} | |
user: ${{ secrets.JFROG_USERNAME }} | |
password: ${{ secrets.JFROG_ACCESS_TOKEN }} | |
packages-dir: artifacts/ | |
verbose: true |