build_wheels.yml

name: Build PyPop

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

on:
  workflow_dispatch:
  pull_request:
    paths-ignore:
      - '**.md'
      - '**.rst'
      - 'CITATION.cff'
      - '.zenodo.extras.json'
      - 'website/**'
      - '.github/dependabot.yml'
      - '.github/release-drafter.yml'
      - '.github/ISSUE_TEMPLATE/**'
      - '.github/workflows/documentation.yaml'
      - '.github/workflows/buildjet_arm64.yml'
      - '.github/workflows/release-drafter.yml'
      - '.github/workflows/codeql.yml'
      - '.gitattributes'
  push:
    paths-ignore:
      - '**.md'
      - '**.rst'
      - 'CITATION.cff'
      - '.zenodo.extras.json'
      - 'website/**'
      - '.github/dependabot.yml'
      - '.github/release-drafter.yml'
      - '.github/ISSUE_TEMPLATE/**'
      - '.github/workflows/documentation.yaml'
      - '.github/workflows/buildjet_arm64.yml'      
      - '.github/workflows/release-drafter.yml'
      - '.github/workflows/codeql.yml'
      - '.gitattributes'
  release:
    types:
      - published

jobs:
  skip_publish_zenodo:
    name: Non publication, Zenodo skip
    runs-on: ubuntu-latest
    # FIXME: this is the default action, the logical inverse of the
    # below "if" in publish_zenodo: a bit hacky, but it works
    if: (github.event_name == 'release' && github.event.action == 'published' && github.event.release.prerelease == false) == false
    steps:
      - name: Skip Zenodo step unless publishing release
        run: |
          echo "Not a publication release, skipping Zenodo"
          echo "github.ref: ${{ toJson(github.ref) }}"
          echo "github.event_name: ${{ toJson(github.event_name) }}"
          echo "github.event.action: ${{ toJson(github.event.action) }}"

  publish_zenodo:
    name: Production release, Zenodo publication
    runs-on: ubuntu-latest
    # "else" we *are* publishing something, only works for non-pre-releases on the main branch
    if: github.event.release.target_commitish == 'citation-cff-zenodo' && github.event_name == 'release' && github.event.action == 'published' && github.event.release.prerelease == false
    steps:
      - name: Checkout the contents of your repository
        uses: actions/checkout@v4
      - name: Upsert the version and related ids in the metadata in CITATION.cff
        run: |
          echo "GITHUB_REF_NAME = $GITHUB_REF_NAME | GITHUB_REF = $GITHUB_REF | GITHUB_EVENT_PATH = $GITHUB_EVENT_PATH"
          echo "github.event.release.target_commitish = ${{ github.event.release.target_commitish }}"
          # convert semver to Python-style version
          VERSION_PYTHON_STYLE=$(GITHUB_REF_NAME=$GITHUB_REF_NAME python -c 'import os; from  packaging import version; print(str(version.Version(os.getenv("GITHUB_REF_NAME"))))')
          echo "VERSION_PYTHON_STYLE = $VERSION_PYTHON_STYLE"

          # insert version
          yq -i ".version = \"$GITHUB_REF_NAME\"" CITATION.cff
          # update the 'related_identifiers' to use exact tree and pypi packaage
          yq -i "(.identifiers[] | select(.value == \"https://github.com/alexlancaster/pypop/tree*\") | .value) |= \"https://github.com/alexlancaster/pypop/tree/$GITHUB_REF_NAME\"" CITATION.cff
          yq -i "(.identifiers[] | select(.value == \"https://pypi.org/project/pypop-genomics/*\") | .value) |= \"https://pypi.org/project/pypop-genomics/$VERSION_PYTHON_STYLE\"" CITATION.cff 
          # temporarily remove DOI
          yq -i eval "del(.doi)" CITATION.cff  

          # use pre-prelease of cffconvert 3.0.0
          pip install -U pip
          pip install 'git+https://github.com/alexlancaster/cffconvert.git@combine_features#egg=cffconvert'
          # convert to json, but remove creators
          cffconvert --format zenodo --infile CITATION.cff | jq 'del(.creators)' > .zenodo.citation.json
          # save creators separately, if they are included in the merge they get re-sorted
          cffconvert --format zenodo --infile CITATION.cff | jq '{creators, creators}' > .zenodo.creators.json
          
          # need to do a "deep merge" to make sure 'related_identifiers' get merged properly
          # including updating the version number, followed by merging back with creators
          cat .zenodo.citation.json .zenodo.extras.json |jq -s 'def deepmerge(a;b):
            reduce b[] as $item (a;
              reduce ($item | keys_unsorted[]) as $key (.;
                $item[$key] as $val | ($val | type) as $type | .[$key] = if ($type == "object") then
                  deepmerge({}; [if .[$key] == null then {} else .[$key] end, $val])
                elif ($type == "array") then
                  (.[$key] + $val | unique)
                else
                  $val
                end)
              );
            deepmerge({}; .)' -S --indent 4 | cat .zenodo.creators.json - | jq -s add --indent 4 > .zenodo.json
            
          # debugging only
          cat .zenodo.citation.json
          cat .zenodo.creators.json
          cat .zenodo.json 
          rm .zenodo.citation.json .zenodo.creators.json
      - name: Push changes back to repo
        run: |
          git fetch origin
          git config --global user.name "ci-pypop"
          git config --global user.email "username@users.noreply.github.com"
          git checkout -b $GITHUB_REF_NAME-with-upserting-changes-pre-upload
          git add CITATION.cff
          git commit -m "Upserted version into CITATION.cff"
          git checkout ${{ github.event.release.target_commitish }}
          git merge $GITHUB_REF_NAME-with-upserting-changes-pre-upload
          git push origin ${{ github.event.release.target_commitish }}
      - name: Clean-up and remove files not for export
        run: |
          # FIXME: removing files marked with 'export-ignore' 'set' in .gitattributes
          # before zenodraft creates the zipball using https://stackoverflow.com/a/30573319
          # to ensure that we handle filenames with potential ':' or newlines
          git ls-files -z|git check-attr --stdin -z export-ignore |sed -zne 'x;n;n;s/^set$//;t print;b;:print;x;p'|xargs --null rm
          # FIXME: need to also remove orphaned empty directories resulting from above
          find . -empty -type d -delete
      - name: Create a draft snapshot of your repository contents as a new
              version using metadata from generated .zenodo.json
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
        uses: zenodraft/action@0.13.3
        # uses: alexlancaster/zenodraft-action@0.10.2
        with:
          concept: 10080667
          metadata: .zenodo.json
          publish: false
          sandbox: false
          compression: zip
          upsert-doi: true
          upsert-location: doi
          verbose: true

  generate-wheels-matrix:
    # Create a matrix of all architectures & versions to build.
    # This enables the next step to run cibuildwheel in parallel.
    # From https://iscinumpy.dev/post/cibuildwheel-2-10-0/#only-210
    name: Generate wheels matrix
    runs-on: ubuntu-latest
    outputs:
      include: ${{ steps.set-matrix.outputs.include }}
    steps:
      - uses: actions/checkout@v4
      - name: Install cibuildwheel
        # Nb. keep cibuildwheel version pin consistent with job below
        run: pipx install cibuildwheel==2.21.3
      - id: set-matrix
        run: |
          echo "CI_ONLY" $CI_ONLY
          if [ "$CI_ONLY" == "true" ]; then
            MATRIX="[{'only':'cp313-manylinux_x86_64','os':'ubuntu-20.04'},{'only':'cp313-win_amd64','os':'windows-2019'}, {'only':'cp313-macosx_x86_64','os':'macos-13'}, {'only':'cp313-macosx_arm64','os':'macos-14'}]"
          else
            MATRIX=$(
              {
              cibuildwheel --print-build-identifiers --platform linux \
              | jq -nRc '{"only": inputs, "os": "ubuntu-20.04"}' \
              && cibuildwheel --print-build-identifiers --platform macos --archs x86_64 \
              | jq -nRc '{"only": inputs, "os": "macos-13"}' \
              && cibuildwheel --print-build-identifiers --platform macos --archs arm64 \
              | jq -nRc '{"only": inputs, "os": "macos-14"}' \
              && cibuildwheel --print-build-identifiers --platform windows \
              | jq -nRc '{"only": inputs, "os": "windows-2019"}'
            } | jq -sc
            )
          fi
          echo "include=$MATRIX" >> $GITHUB_OUTPUT
          echo "platform matrix:" $MATRIX
        env:
           # only generate all wheels on releases, pull requests or commits to the main branch
           # otherwise just generate a test wheel Python 3.12 for 4 platform
           CI_ONLY: ${{ github.event_name != 'release' && github.event_name != 'pull_request' && github.ref_name != 'main' && github.event_name != 'workflow_dispatch' }}
          
  build_wheels:
    name: Build wheels on ${{ matrix.only }}
    # depend on zenodo, to make sure updated CITATION.cff gets into builds
    # this only results in changes upon production releases
    needs: [publish_zenodo, skip_publish_zenodo, generate-wheels-matrix]
    # trying solution here: https://github.com/actions/runner/issues/491#issuecomment-1507495166
    # so that it works if either job in "needs" completes
    if: always() && !cancelled() && !failure()
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.generate-wheels-matrix.outputs.include) }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          # FIXME: if on a release, use the tag name, not original SHA because it might be changed (somewhat hacky)
          ref: ${{ (github.event_name == 'release' && github.event.action == 'published') && github.ref_name || '' }}
      - name: Set up QEMU
        if: runner.os == 'Linux'
        uses: docker/setup-qemu-action@v3
        with:
          platforms: all
      - name: Install a recent stable Python to handle Python deps
        uses: actions/setup-python@v5
        with:
          python-version: 3.12
      - name: Query version with setuptools_scm
        id: version
        shell: bash
        run: |
          python -m pip install setuptools_scm
          VERSION=$(python -c "from src.PyPop import __version_scheme__; import setuptools_scm; print(setuptools_scm.get_version(version_scheme=__version_scheme__))")
          echo "VERSION=${VERSION}"
          echo "VERSION=${VERSION}" >> $GITHUB_ENV
      - name: Install toml and remove cffconvert from pyproject.toml
        run: |
          python -m pip install toml
          python -c "
          import toml
          with open('pyproject.toml', 'r') as f:
             config = toml.load(f)
          if 'build-system' in config and 'requires' in config['build-system']:
             config['build-system']['requires'] = [
                 dep for dep in config['build-system']['requires'] if 'cffconvert' not in dep.lower()
             ]
          with open('pyproject.toml', 'w') as f:
             toml.dump(config, f)
          "
      - name: Generate citation formats
        run: |
          python --version
          python -m pip install git+https://github.com/alexlancaster/cffconvert.git@combine_features#egg=cffconvert
          python src/PyPop/citation.py
      - name: Build and test wheels
        uses: pypa/cibuildwheel@v2.21.3
        env:
          # FIXME: only run the slow tests when doing regular pushes, or manual - not for PRs
          CIBW_TEST_COMMAND: "pytest -v {package}/tests ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '--runslow' || '' }}"
          SETUPTOOLS_SCM_PRETEND_VERSION: ${{ env.VERSION }}
          CIBW_ENVIRONMENT_PASS_LINUX: SETUPTOOLS_SCM_PRETEND_VERSION
        with:
          only: ${{ matrix.only }}
          package-dir: .
          output-dir: wheelhouse
          config-file: "{package}/pyproject.toml"
      - uses: actions/upload-artifact@v4
        with:
          name: wheels-${{ matrix.only }}
          path: ./wheelhouse/*.whl
          # path: ./wheelhouse/*  use if more than wheels are wanted

  build_sdist:
    name: Build source distribution
    # depend on zenodo, to make sure updated CITATION.cff gets into builds
    needs: [publish_zenodo, skip_publish_zenodo]
    if: always() && !cancelled() && !failure()
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:  # FIXME: if on a release, use the tag name, not original SHA because it might be changed (somewhat hacky)
          ref: ${{ (github.event_name == 'release' && github.event.action == 'published') && github.ref_name || '' }}
      - name: Build sdist
        run: pipx run build --sdist
      - uses: actions/upload-artifact@v4
        with:
          name: sdist
          path: dist/*.tar.gz

  upload_gh_release:
    name: Upload binary wheels and sdist to GH release page
    needs: [build_wheels, build_sdist]
    runs-on: ubuntu-latest
    # alternatively, to publish when a GitHub Release is created, use the following rule:
    if: github.event_name == 'release' && github.event.action == 'published' && always() && !cancelled() && !failure()
    steps:
      - uses: actions/download-artifact@v4
        with:
          # unpacks default artifact into dist/
          path: dist
          merge-multiple: true
      - uses: softprops/action-gh-release@v2
        name: Uploading binaries to release page
        with:
          files: dist/*

  upload_test_pypi:
    name: Upload to Test_PyPI
    needs: [build_wheels, build_sdist]
    runs-on: ubuntu-latest
    environment: test_pypi
    permissions:
      id-token: write
    # drop requirement of 'main', release to test_pypi to test releases
    if: github.event_name == 'release' && github.event.action == 'published' && always() && !cancelled() && !failure()
    steps:
      - uses: actions/download-artifact@v4
        with:
          # unpacks default artifact into dist/
          path: dist
          merge-multiple: true          
      - uses: pypa/gh-action-pypi-publish@release/v1
        with:
          repository_url: https://test.pypi.org/legacy/

  upload_pypi:
    name: Upload to PyPI
    needs: [build_wheels, build_sdist]
    runs-on: ubuntu-latest
    environment: pypi
    permissions:
      id-token: write
    if: github.event.release.target_commitish == 'main' && github.event_name == 'release' && github.event.action == 'published' && always()  && !cancelled() && !failure()
    steps:
      - uses: actions/download-artifact@v4
        with:
          # unpacks default artifact into dist/
          path: dist
          merge-multiple: true          
      - name: Publish package distributions to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1