Merge branch 'main' into arm64-testing #2387
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build PyPop | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} | |
| cancel-in-progress: true | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| paths-ignore: | |
| - '**.md' | |
| - '**.rst' | |
| - 'CITATION.cff' | |
| - '.zenodo.json' | |
| - 'website/**' | |
| - '.github/dependabot.yml' | |
| - '.github/release-drafter.yml' | |
| - '.github/ISSUE_TEMPLATE/**' | |
| - '.github/workflows/documentation.yaml' | |
| - '.github/workflows/buildjet_arm64.yml' | |
| - '.github/workflows/release-drafter.yml' | |
| - '.github/workflows/codeql.yml' | |
| - '.gitattributes' | |
| push: | |
| paths-ignore: | |
| - '**.md' | |
| - '**.rst' | |
| - 'CITATION.cff' | |
| - '.zenodo.json' | |
| - 'website/**' | |
| - '.github/dependabot.yml' | |
| - '.github/release-drafter.yml' | |
| - '.github/ISSUE_TEMPLATE/**' | |
| - '.github/workflows/documentation.yaml' | |
| - '.github/workflows/buildjet_arm64.yml' | |
| - '.github/workflows/release-drafter.yml' | |
| - '.github/workflows/codeql.yml' | |
| - '.gitattributes' | |
| release: | |
| types: | |
| - published | |
| jobs: | |
| skip_publish_zenodo: | |
| name: Non publication, Zenodo skip | |
| runs-on: ubuntu-latest | |
| # FIXME: this is the default action, the logical inverse of the | |
| # below "if" in publish_zenodo: a bit hacky, but it works | |
| if: (github.event_name == 'release' && github.event.action == 'published' && github.event.release.prerelease == false) == false | |
| steps: | |
| - name: Skip Zenodo step unless publishing release | |
| run: | | |
| echo "Not a publication release, skipping Zenodo" | |
| echo "github.ref: ${{ toJson(github.ref) }}" | |
| echo "github.event_name: ${{ toJson(github.event_name) }}" | |
| echo "github.event.action: ${{ toJson(github.event.action) }}" | |
| publish_zenodo: | |
| name: Production release, Zenodo publication | |
| runs-on: ubuntu-latest | |
| # "else" we *are* publishing something, only works for non-pre-releases on the main branch | |
| if: github.event.release.target_commitish == 'main' && github.event_name == 'release' && github.event.action == 'published' && github.event.release.prerelease == false | |
| steps: | |
| - name: Checkout the contents of your repository | |
| uses: actions/checkout@v4 | |
| - name: Upsert the version and related ids in the metadata in .zenodo.json | |
| run: | | |
| echo "GITHUB_REF_NAME = $GITHUB_REF_NAME | GITHUB_REF = $GITHUB_REF | GITHUB_EVENT_PATH = $GITHUB_EVENT_PATH" | |
| echo "github.event.release.target_commitish = ${{ github.event.release.target_commitish }}" | |
| # convert semver to Python-style version | |
| VERSION_PYTHON_STYLE=$(GITHUB_REF_NAME=$GITHUB_REF_NAME python -c 'import os; from packaging import version; print(str(version.Version(os.getenv("GITHUB_REF_NAME"))))') | |
| echo "VERSION_PYTHON_STYLE = $VERSION_PYTHON_STYLE" | |
| # cp .zenodo.json .zenodo.orig.json | |
| # update the 'related_identifiers' to use exact tree and pypi packaage | |
| cat .zenodo.json | jq "(.related_identifiers[] | (select (.relation == \"isSourceOf\" and .resource_type == \"software\")) | select (.identifier | startswith(\"https://pypi.org/project/pypop-genomics\")).identifier) |= \"https://pypi.org/project/pypop-genomics/$VERSION_PYTHON_STYLE\"" | jq "(.related_identifiers[] | (select (.relation == \"isSupplementTo\" and .resource_type == \"software\")) | select (.identifier | startswith(\"https://github.com/alexlancaster/pypop\")).identifier) |= \"https://github.com/alexlancaster/pypop/tree/$GITHUB_REF_NAME\"" > .zenodo.mod.json | |
| # update the version number | |
| echo "{\"version\": \"$GITHUB_REF_NAME\"}" | cat .zenodo.mod.json - | jq -s add > .zenodo.json | |
| # debugging only | |
| cat .zenodo.json | |
| rm .zenodo.mod.json | |
| - name: Upsert the version in the metadata in CITATION.cff | |
| run: | | |
| yq -i ".version = \"$GITHUB_REF_NAME\"" CITATION.cff | |
| - name: Push changes back to repo files | |
| run: | | |
| git fetch origin | |
| git config --global user.name "ci-pypop" | |
| git config --global user.email "[email protected]" | |
| git checkout -b $GITHUB_REF_NAME-with-upserting-changes-pre-upload | |
| git add .zenodo.json CITATION.cff | |
| git commit -m "Upserted version into CITATION.cff and into .zenodo.json" | |
| git checkout ${{ github.event.release.target_commitish }} | |
| git merge $GITHUB_REF_NAME-with-upserting-changes-pre-upload | |
| git push origin ${{ github.event.release.target_commitish }} | |
| # FIXME: removing files marked with 'export-ignore' 'set' in .gitattributes | |
| # before zenodraft creates the zipball using https://stackoverflow.com/a/30573319 | |
| # to ensure that we handle filenames with potential ':' or newlines | |
| git ls-files -z|git check-attr --stdin -z export-ignore |sed -zne 'x;n;n;s/^set$//;t print;b;:print;x;p'|xargs --null rm | |
| # FIXME: need to also remove orphaned empty directories resulting from above | |
| find . -empty -type d -delete | |
| - name: Create a draft snapshot of your repository contents as a new | |
| version in test collection on Zenodo using metadata | |
| from repository file .zenodo.json | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }} | |
| uses: zenodraft/[email protected] | |
| # uses: alexlancaster/[email protected] | |
| with: | |
| concept: 10080667 | |
| metadata: .zenodo.json | |
| publish: false | |
| sandbox: false | |
| compression: zip | |
| upsert-doi: true | |
| upsert-location: doi | |
| verbose: true | |
| generate-wheels-matrix: | |
| # Create a matrix of all architectures & versions to build. | |
| # This enables the next step to run cibuildwheel in parallel. | |
| # From https://iscinumpy.dev/post/cibuildwheel-2-10-0/#only-210 | |
| name: Generate wheels matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| include: ${{ steps.set-matrix.outputs.include }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install cibuildwheel | |
| # Nb. keep cibuildwheel version pin consistent with job below | |
| run: pipx install cibuildwheel==2.22.0 | |
| - id: set-matrix | |
| run: | | |
| echo "CI_ONLY" $CI_ONLY | |
| if [ "$CI_ONLY" == "true" ]; then | |
| MATRIX="[{'only':'cp313-manylinux_x86_64','os':'ubuntu-20.04'},{'only':'cp313-win_amd64','os':'windows-2019'}, {'only':'cp313-macosx_x86_64','os':'macos-13'}, {'only':'cp313-macosx_arm64','os':'macos-14'}]" | |
| else | |
| MATRIX=$( | |
| { | |
| cibuildwheel --print-build-identifiers --platform linux \ | |
| | jq -nRc '{"only": inputs, "os": "ubuntu-20.04"}' \ | |
| && cibuildwheel --print-build-identifiers --platform macos --archs x86_64 \ | |
| | jq -nRc '{"only": inputs, "os": "macos-13"}' \ | |
| && cibuildwheel --print-build-identifiers --platform macos --archs arm64 \ | |
| | jq -nRc '{"only": inputs, "os": "macos-14"}' \ | |
| && cibuildwheel --print-build-identifiers --platform windows \ | |
| | jq -nRc '{"only": inputs, "os": "windows-2019"}' | |
| } | jq -sc | |
| ) | |
| fi | |
| echo "include=$MATRIX" >> $GITHUB_OUTPUT | |
| echo "platform matrix:" $MATRIX | |
| env: | |
| # only generate all wheels on releases, pull requests or commits to the main branch | |
| # otherwise just generate a test wheel Python 3.12 for 4 platform | |
| CI_ONLY: ${{ github.event_name != 'release' && github.event_name != 'pull_request' && github.ref_name != 'main' && github.event_name != 'workflow_dispatch' }} | |
| build_wheels: | |
| name: Build wheels on ${{ matrix.only }} | |
| # depend on zenodo, to make sure updated CITATION.cff gets into builds | |
| # this only results in changes upon production releases | |
| needs: [publish_zenodo, skip_publish_zenodo, generate-wheels-matrix] | |
| # trying solution here: https://github.com/actions/runner/issues/491#issuecomment-1507495166 | |
| # so that it works if either job in "needs" completes | |
| if: always() && !cancelled() && !failure() | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson(needs.generate-wheels-matrix.outputs.include) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # FIXME: if on a release, use the tag name, not original SHA because it might be changed (somewhat hacky) | |
| ref: ${{ (github.event_name == 'release' && github.event.action == 'published') && github.ref_name || '' }} | |
| - name: Set up QEMU | |
| if: runner.os == 'Linux' | |
| uses: docker/setup-qemu-action@v3 | |
| with: | |
| platforms: all | |
| - name: Install a recent stable Python to handle Python deps | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.12 | |
| - name: Query version with setuptools_scm | |
| id: version | |
| shell: bash | |
| run: | | |
| python -m pip install setuptools_scm | |
| VERSION=$(python -c "from src.PyPop import __version_scheme__; import setuptools_scm; print(setuptools_scm.get_version(version_scheme=__version_scheme__))") | |
| echo "VERSION=${VERSION}" | |
| echo "VERSION=${VERSION}" >> $GITHUB_ENV | |
| - name: Install toml and remove cffconvert from pyproject.toml | |
| run: | | |
| python -m pip install toml | |
| python -c " | |
| import toml | |
| with open('pyproject.toml', 'r') as f: | |
| config = toml.load(f) | |
| if 'build-system' in config and 'requires' in config['build-system']: | |
| config['build-system']['requires'] = [ | |
| dep for dep in config['build-system']['requires'] if 'cffconvert' not in dep.lower() | |
| ] | |
| with open('pyproject.toml', 'w') as f: | |
| toml.dump(config, f) | |
| " | |
| - name: Generate citation formats | |
| run: | | |
| python --version | |
| python -m pip install git+https://github.com/alexlancaster/cffconvert.git@combine_features#egg=cffconvert | |
| python src/PyPop/citation.py | |
| - name: Build and test wheels | |
| uses: pypa/[email protected] | |
| env: | |
| # FIXME: only run the slow tests when doing regular pushes, or manual - not for PRs | |
| CIBW_TEST_COMMAND: "pytest -v {package}/tests ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '--runslow' || '' }}" | |
| SETUPTOOLS_SCM_PRETEND_VERSION: ${{ env.VERSION }} | |
| CIBW_ENVIRONMENT_PASS_LINUX: SETUPTOOLS_SCM_PRETEND_VERSION | |
| with: | |
| only: ${{ matrix.only }} | |
| package-dir: . | |
| output-dir: wheelhouse | |
| config-file: "{package}/pyproject.toml" | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: wheels-${{ matrix.only }} | |
| path: ./wheelhouse/*.whl | |
| # path: ./wheelhouse/* use if more than wheels are wanted | |
| build_sdist: | |
| name: Build source distribution | |
| # depend on zenodo, to make sure updated CITATION.cff gets into builds | |
| needs: [publish_zenodo, skip_publish_zenodo] | |
| if: always() && !cancelled() && !failure() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: # FIXME: if on a release, use the tag name, not original SHA because it might be changed (somewhat hacky) | |
| ref: ${{ (github.event_name == 'release' && github.event.action == 'published') && github.ref_name || '' }} | |
| - name: Build sdist | |
| run: pipx run build --sdist | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: sdist | |
| path: dist/*.tar.gz | |
| upload_gh_release: | |
| name: Upload binary wheels and sdist to GH release page | |
| needs: [build_wheels, build_sdist] | |
| runs-on: ubuntu-latest | |
| # alternatively, to publish when a GitHub Release is created, use the following rule: | |
| if: github.event_name == 'release' && github.event.action == 'published' && always() && !cancelled() && !failure() | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| # unpacks default artifact into dist/ | |
| path: dist | |
| merge-multiple: true | |
| - uses: softprops/action-gh-release@v2 | |
| name: Uploading binaries to release page | |
| with: | |
| files: dist/* | |
| upload_test_pypi: | |
| name: Upload to Test_PyPI | |
| needs: [build_wheels, build_sdist] | |
| runs-on: ubuntu-latest | |
| environment: test_pypi | |
| permissions: | |
| id-token: write | |
| # drop requirement of 'main', release to test_pypi to test releases | |
| if: github.event_name == 'release' && github.event.action == 'published' && always() && !cancelled() && !failure() | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| # unpacks default artifact into dist/ | |
| path: dist | |
| merge-multiple: true | |
| - uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| repository_url: https://test.pypi.org/legacy/ | |
| upload_pypi: | |
| name: Upload to PyPI | |
| needs: [build_wheels, build_sdist] | |
| runs-on: ubuntu-latest | |
| environment: pypi | |
| permissions: | |
| id-token: write | |
| if: github.event.release.target_commitish == 'main' && github.event_name == 'release' && github.event.action == 'published' && always() && !cancelled() && !failure() | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| # unpacks default artifact into dist/ | |
| path: dist | |
| merge-multiple: true | |
| - name: Publish package distributions to PyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 |