Skip to content

Merge remote-tracking branch 'origin/main' into last_zenodo_check #2354

Merge remote-tracking branch 'origin/main' into last_zenodo_check

Merge remote-tracking branch 'origin/main' into last_zenodo_check #2354

Workflow file for this run

name: Build PyPop
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: true
on:
workflow_dispatch:
pull_request:
paths-ignore:
- '**.md'
- '**.rst'
- 'CITATION.cff'
- '.zenodo.json'
- 'website/**'
- '.github/dependabot.yml'
- '.github/release-drafter.yml'
- '.github/ISSUE_TEMPLATE/**'
- '.github/workflows/documentation.yaml'
- '.github/workflows/buildjet_arm64.yml'
- '.github/workflows/release-drafter.yml'
- '.github/workflows/codeql.yml'
- '.gitattributes'
push:
paths-ignore:
- '**.md'
- '**.rst'
- 'CITATION.cff'
- '.zenodo.json'
- 'website/**'
- '.github/dependabot.yml'
- '.github/release-drafter.yml'
- '.github/ISSUE_TEMPLATE/**'
- '.github/workflows/documentation.yaml'
- '.github/workflows/buildjet_arm64.yml'
- '.github/workflows/release-drafter.yml'
- '.github/workflows/codeql.yml'
- '.gitattributes'
release:
types:
- published
jobs:
skip_publish_zenodo:
name: Non publication, Zenodo skip
runs-on: ubuntu-latest
# FIXME: this is the default action, the logical inverse of the
# below "if" in publish_zenodo: a bit hacky, but it works
if: (github.event_name == 'release' && github.event.action == 'published' && github.event.release.prerelease == false) == false
steps:
- name: Skip Zenodo step unless publishing release
run: |
echo "Not a publication release, skipping Zenodo"
echo "github.ref: ${{ toJson(github.ref) }}"
echo "github.event_name: ${{ toJson(github.event_name) }}"
echo "github.event.action: ${{ toJson(github.event.action) }}"
publish_zenodo:
name: Production release, Zenodo publication
runs-on: ubuntu-latest
# "else" we *are* publishing something, only works for non-pre-releases on the main branch
if: github.event.release.target_commitish == 'last_zenodo_check' && github.event_name == 'release' && github.event.action == 'published' && github.event.release.prerelease == false
steps:
- name: Checkout the contents of your repository
uses: actions/checkout@v4
- name: Upsert the version and related ids in the metadata in .zenodo.json
run: |
echo "GITHUB_REF_NAME = $GITHUB_REF_NAME | GITHUB_REF = $GITHUB_REF | GITHUB_EVENT_PATH = $GITHUB_EVENT_PATH"
echo "github.event.release.target_commitish = ${{ github.event.release.target_commitish }}"
# convert semver to Python-style version
VERSION_PYTHON_STYLE=$(GITHUB_REF_NAME=$GITHUB_REF_NAME python -c 'import os; from packaging import version; print(str(version.Version(os.getenv("GITHUB_REF_NAME"))))')
echo "VERSION_PYTHON_STYLE = $VERSION_PYTHON_STYLE"
# cp .zenodo.json .zenodo.orig.json
# update the 'related_identifiers' to use exact tree and pypi packaage
cat .zenodo.json | jq "(.related_identifiers[] | (select (.relation == \"isSourceOf\" and .resource_type == \"software\")) | select (.identifier | startswith(\"https://pypi.org/project/pypop-genomics\")).identifier) |= \"https://pypi.org/project/pypop-genomics/$VERSION_PYTHON_STYLE\"" | jq "(.related_identifiers[] | (select (.relation == \"isSupplementTo\" and .resource_type == \"software\")) | select (.identifier | startswith(\"https://github.com/alexlancaster/pypop\")).identifier) |= \"https://github.com/alexlancaster/pypop/tree/$GITHUB_REF_NAME\"" > .zenodo.mod.json
# update the version number
echo "{\"version\": \"$GITHUB_REF_NAME\"}" | cat .zenodo.mod.json - | jq -s add > .zenodo.json
# debugging only
cat .zenodo.json
rm .zenodo.mod.json
- name: Upsert the version in the metadata in CITATION.cff
run: |
yq -i ".version = \"$GITHUB_REF_NAME\"" CITATION.cff
- name: Push changes back to repo files
run: |
git fetch origin
git config --global user.name "ci-pypop"
git config --global user.email "[email protected]"
git checkout -b $GITHUB_REF_NAME-with-upserting-changes-pre-upload
git add .zenodo.json CITATION.cff
git commit -m "Upserted version into CITATION.cff and into .zenodo.json"
git checkout ${{ github.event.release.target_commitish }}
git merge $GITHUB_REF_NAME-with-upserting-changes-pre-upload
git push origin ${{ github.event.release.target_commitish }}
# FIXME: removing files marked with 'export-ignore' 'set' in .gitattributes
# before zenodraft creates the zipball using https://stackoverflow.com/a/30573319
# to ensure that we handle filenames with potential ':' or newlines
git ls-files -z|git check-attr --stdin -z export-ignore |sed -zne 'x;n;n;s/^set$//;t print;b;:print;x;p'|xargs --null rm
# FIXME: need to also remove orphaned empty directories resulting from above
find . -empty -type d -delete
- name: Create a draft snapshot of your repository contents as a new
version in test collection on Zenodo using metadata
from repository file .zenodo.json
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
uses: zenodraft/[email protected]
# uses: alexlancaster/[email protected]
with:
concept: 10080667
metadata: .zenodo.json
publish: false
sandbox: false
# sandbox: true
compression: zip
upsert-doi: true
upsert-location: doi
verbose: true
generate-wheels-matrix:
# Create a matrix of all architectures & versions to build.
# This enables the next step to run cibuildwheel in parallel.
# From https://iscinumpy.dev/post/cibuildwheel-2-10-0/#only-210
name: Generate wheels matrix
runs-on: ubuntu-latest
outputs:
include: ${{ steps.set-matrix.outputs.include }}
steps:
- uses: actions/checkout@v4
- name: Install cibuildwheel
# Nb. keep cibuildwheel version pin consistent with job below
run: pipx install cibuildwheel==2.21.3
- id: set-matrix
run: |
echo "CI_ONLY" $CI_ONLY
if [ "$CI_ONLY" == "true" ]; then
MATRIX="[{'only':'cp313-manylinux_x86_64','os':'ubuntu-20.04'},{'only':'cp313-win_amd64','os':'windows-2019'}, {'only':'cp313-macosx_x86_64','os':'macos-13'}, {'only':'cp313-macosx_arm64','os':'macos-14'}]"
else
MATRIX=$(
{
cibuildwheel --print-build-identifiers --platform linux \
| jq -nRc '{"only": inputs, "os": "ubuntu-20.04"}' \
&& cibuildwheel --print-build-identifiers --platform macos --archs x86_64 \
| jq -nRc '{"only": inputs, "os": "macos-13"}' \
&& cibuildwheel --print-build-identifiers --platform macos --archs arm64 \
| jq -nRc '{"only": inputs, "os": "macos-14"}' \
&& cibuildwheel --print-build-identifiers --platform windows \
| jq -nRc '{"only": inputs, "os": "windows-2019"}'
} | jq -sc
)
fi
echo "include=$MATRIX" >> $GITHUB_OUTPUT
echo "platform matrix:" $MATRIX
env:
# only generate all wheels on releases, pull requests or commits to the main branch
# otherwise just generate a test wheel Python 3.12 for 4 platform
CI_ONLY: ${{ github.event_name != 'release' && github.event_name != 'pull_request' && github.ref_name != 'main' && github.event_name != 'workflow_dispatch' }}
build_wheels:
name: Build wheels on ${{ matrix.only }}
# depend on zenodo, to make sure updated CITATION.cff gets into builds
# this only results in changes upon production releases
needs: [publish_zenodo, skip_publish_zenodo, generate-wheels-matrix]
# trying solution here: https://github.com/actions/runner/issues/491#issuecomment-1507495166
# so that it works if either job in "needs" completes
if: always() && !cancelled() && !failure()
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.generate-wheels-matrix.outputs.include) }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
# FIXME: if on a release, use the tag name, not original SHA because it might be changed (somewhat hacky)
ref: ${{ (github.event_name == 'release' && github.event.action == 'published') && github.ref_name || '' }}
- name: Set up QEMU
if: runner.os == 'Linux'
uses: docker/setup-qemu-action@v3
with:
platforms: all
- name: Install a recent stable Python to handle Python deps
uses: actions/setup-python@v4
with:
python-version: 3.12
- name: Query version with setuptools_scm
id: version
shell: bash
run: |
python -m pip install setuptools_scm
VERSION=$(python -c "from src.PyPop import __version_scheme__; import setuptools_scm; print(setuptools_scm.get_version(version_scheme=__version_scheme__))")
echo "VERSION=${VERSION}"
echo "VERSION=${VERSION}" >> $GITHUB_ENV
- name: Install toml and remove cffconvert from pyproject.toml
run: |
python -m pip install toml
python -c "
import toml
with open('pyproject.toml', 'r') as f:
config = toml.load(f)
if 'build-system' in config and 'requires' in config['build-system']:
config['build-system']['requires'] = [
dep for dep in config['build-system']['requires'] if 'cffconvert' not in dep.lower()
]
with open('pyproject.toml', 'w') as f:
toml.dump(config, f)
"
- name: Generate citation formats
run: |
python --version
python -m pip install git+https://github.com/alexlancaster/cffconvert.git@combine_features#egg=cffconvert
python src/PyPop/citation.py
- name: Build and test wheels
uses: pypa/[email protected]
env:
# FIXME: only run the slow tests when doing regular pushes, or manual - not for PRs
CIBW_TEST_COMMAND: "pytest -v {package}/tests ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '--runslow' || '' }}"
SETUPTOOLS_SCM_PRETEND_VERSION: ${{ env.VERSION }}
CIBW_ENVIRONMENT_PASS_LINUX: SETUPTOOLS_SCM_PRETEND_VERSION
with:
only: ${{ matrix.only }}
package-dir: .
output-dir: wheelhouse
config-file: "{package}/pyproject.toml"
- uses: actions/upload-artifact@v4
with:
name: wheels-${{ matrix.only }}
path: ./wheelhouse/*.whl
# path: ./wheelhouse/* use if more than wheels are wanted
build_sdist:
name: Build source distribution
# depend on zenodo, to make sure updated CITATION.cff gets into builds
needs: [publish_zenodo, skip_publish_zenodo]
if: always() && !cancelled() && !failure()
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with: # FIXME: if on a release, use the tag name, not original SHA because it might be changed (somewhat hacky)
ref: ${{ (github.event_name == 'release' && github.event.action == 'published') && github.ref_name || '' }}
- name: Build sdist
run: pipx run build --sdist
- uses: actions/upload-artifact@v4
with:
name: sdist
path: dist/*.tar.gz
upload_gh_release:
name: Upload binary wheels and sdist to GH release page
needs: [build_wheels, build_sdist]
runs-on: ubuntu-latest
# alternatively, to publish when a GitHub Release is created, use the following rule:
if: github.event_name == 'release' && github.event.action == 'published' && always() && !cancelled() && !failure()
steps:
- uses: actions/download-artifact@v4
with:
# unpacks default artifact into dist/
path: dist
merge-multiple: true
- uses: softprops/action-gh-release@v2
name: Uploading binaries to release page
with:
files: dist/*
upload_test_pypi:
name: Upload to Test_PyPI
needs: [build_wheels, build_sdist]
runs-on: ubuntu-latest
environment: test_pypi
permissions:
id-token: write
# drop requirement of 'main', release to test_pypi to test releases
if: github.event_name == 'release' && github.event.action == 'published' && always() && !cancelled() && !failure()
steps:
- uses: actions/download-artifact@v4
with:
# unpacks default artifact into dist/
path: dist
merge-multiple: true
- uses: pypa/gh-action-pypi-publish@release/v1
with:
repository_url: https://test.pypi.org/legacy/
upload_pypi:
name: Upload to PyPI
needs: [build_wheels, build_sdist]
runs-on: ubuntu-latest
environment: pypi
permissions:
id-token: write
if: github.event.release.target_commitish == 'main' && github.event_name == 'release' && github.event.action == 'published' && always() && !cancelled() && !failure()
steps:
- uses: actions/download-artifact@v4
with:
# unpacks default artifact into dist/
path: dist
merge-multiple: true
- name: Publish package distributions to PyPI
uses: pypa/gh-action-pypi-publish@release/v1