Add GitHub Workflow to publish conda package (#340)

* Add Packages workflow * Remove release job from postmerge workflow * Remove package build from test workflows * Use branch-name action to select branch name for tests * Update conda recipe to noarch release * Exclude `treelite_runtime` from conda package dependencies * Update conda recipe to match pattern from other Merlin packages * Fix typo in comment * Remove check-manifest from build-pypi job * Add wildcard to match package artifact correctly * Add tag condition back to the conda release job --------- Co-authored-by: Karl Higley <[email protected]>
NVIDIA-Merlin · May 3, 2023 · b722bd6 · b722bd6
1 parent 45c0595
commit b722bd6
Show file tree

Hide file tree

Showing 4 changed files with 138 additions and 61 deletions.
diff --git a/.github/workflows/cpu-ci.yml b/.github/workflows/cpu-ci.yml
@@ -33,24 +33,13 @@ jobs:
       - name: Install and upgrade python packages
         run: |
           python -m pip install --upgrade pip setuptools==59.4.0 wheel tox
+      - name: Get Branch name
+        id: get-branch-name
+        uses: NVIDIA-Merlin/.github/actions/branch-name@main
       - name: Run tests
         run: |
-          ref_type=${{ github.ref_type }}
-          branch=main
-          if [[ $ref_type == "tag"* ]]
-          then
-            raw=$(git branch -r --contains ${{ github.ref_name }})
-            branch=${raw/origin\/}
-          fi
+          branch="${{ steps.get-branch-name.outputs.branch }}"
           tox -e test-cpu -- $branch
-      - name: Generate package for pypi
-        run: |
-          python setup.py sdist
-      - name: Upload artifacts to github
-        uses: actions/upload-artifact@v3
-        with:
-          name: dist
-          path: dist
       - name: Building docs
         run: |
           tox -e docs

diff --git a/.github/workflows/packages.yaml b/.github/workflows/packages.yaml
@@ -0,0 +1,118 @@
+name: Packages
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - release*
+    tags:
+      - v*
+  pull_request:
+    branches: [main]
+
+jobs:
+  build-pypi:
+    name: Build PyPI Package
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+      - name: Install and upgrade python packages
+        run: |
+          python -m pip install --upgrade pip setuptools==59.4.0 wheel
+      - name: Generate package for pypi
+        run: |
+          python setup.py sdist
+      - name: Upload pypi artifacts to github
+        uses: actions/upload-artifact@v3
+        with:
+          name: dist
+          path: dist
+
+  build-conda:
+    name: Build Conda Package
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+      - name: Generate package for conda
+        id: conda_build
+        run: |
+          echo "conda pkgs dir $CONDA_PKGS_DIRS"
+          conda install -c conda-forge mamba
+          mamba install -c conda-forge conda-build boa conda-verify
+          conda mambabuild . -c defaults -c conda-forge -c numba -c rapidsai -c nvidia --output-folder ./conda_packages
+          conda_package=$(find ./conda_packages/ -name "*.tar.bz2")
+          export CONDA_PACKAGE=$conda_package
+          echo "conda_package : $conda_package"
+          echo "conda_package=$conda_package" >> "$GITHUB_OUTPUT"
+      - name: Upload conda artifacts to github
+        uses: actions/upload-artifact@v3
+        with:
+          name: conda
+          path: ${{ steps.conda_build.outputs.conda_package }}
+
+  release-pypi:
+    name: Release PyPI Package
+    runs-on: ubuntu-latest
+    if: "startsWith(github.ref, 'refs/tags/')"
+    needs: [build-pypi]
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          name: dist
+          path: dist
+      - name: Create GitHub Release
+        uses: fnkr/[email protected]
+        env:
+          GHR_PATH: ./dist
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+      - name: Push to PyPI
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: |
+          pip install --upgrade wheel pip setuptools twine
+          twine upload dist/*
+
+  release-conda:
+    name: Release Conda Package
+    runs-on: ubuntu-latest
+    if: "startsWith(github.ref, 'refs/tags/')"
+    needs: [build-conda]
+    steps:
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - uses: actions/download-artifact@v3
+        with:
+          name: conda
+          path: conda
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+      - name: Install conda dependencies
+        shell: bash -l {0}
+        run: |
+          conda install -y anaconda-client conda-build
+      - name: Push to anaconda
+        shell: bash -l {0}
+        env:
+          ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }}
+        run: |
+          anaconda -t $ANACONDA_TOKEN upload -u nvidia conda/*.tar.bz2
diff --git a/.github/workflows/postmerge-cpu.yml b/.github/workflows/postmerge-cpu.yml
@@ -30,24 +30,13 @@ jobs:
       - name: Install and upgrade python packages
         run: |
           python -m pip install --upgrade pip setuptools==59.4.0 wheel tox
+      - name: Get Branch name
+        id: get-branch-name
+        uses: NVIDIA-Merlin/.github/actions/branch-name@main
       - name: Run tests
         run: |
-          ref_type=${{ github.ref_type }}
-          branch=main
-          if [[ $ref_type == "tag"* ]]
-          then
-            raw=$(git branch -r --contains ${{ github.ref_name }})
-            branch=${raw/origin\/}
-          fi
+          branch="${{ steps.get-branch-name.outputs.branch }}"
           tox -e test-cpu-postmerge -- $branch
-      - name: Generate package for pypi
-        run: |
-          python setup.py sdist
-      - name: Upload artifacts to github
-        uses: actions/upload-artifact@v3
-        with:
-          name: dist
-          path: dist
       - name: Building docs
         run: |
           tox -e docs
@@ -69,28 +58,3 @@ jobs:
         with:
           name: pr
           path: pr/
-
-  release:
-    name: Release
-    runs-on: ubuntu-latest
-    if: "startsWith(github.ref, 'refs/tags/')"
-    needs: [cpu-ci-postmerge]
-    steps:
-      - uses: actions/download-artifact@v2
-        with:
-          name: dist
-      - name: Create GitHub Release
-        uses: fnkr/[email protected]
-        env:
-          GHR_PATH: .
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-      - name: Push to PyPi
-        env:
-          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-        run: |
-          pip install --upgrade wheel pip setuptools twine
-          twine upload *
diff --git a/conda/recipes/meta.yaml b/conda/recipes/meta.yaml
@@ -1,12 +1,10 @@
-# Copyright (c) 2021, NVIDIA CORPORATION.
+# Copyright (c) 2023, NVIDIA CORPORATION.
 
 # Usage:
 #   conda build . -c defaults -c conda-forge -c numba -c rapidsai
 
 {% set version = environ.get('GIT_DESCRIBE_TAG', '0.1').lstrip('v') + environ.get('VERSION_SUFFIX', '') %}
-{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %}
 {% set git_revision_count=environ.get('GIT_DESCRIBE_NUMBER', 0) %}
-{% set py_version=environ.get('CONDA_PY', 36) %}
 {% set setup_py_data = load_setup_py_data() %}
 
 package:
@@ -18,14 +16,22 @@ source:
 
 build:
   number: {{ git_revision_count }}
+  noarch: python
   script: python -m pip install . -vvv
 
 requirements:
+  build:
+    - python
+    - setuptools
   run:
-  {% for req in setup_py_data.get('install_requires', []) %}
-    - {{ req }}
-  {% endfor %}
     - python
+    {% for req in setup_py_data.get('install_requires', []) %}
+      # the treelite_runtime pip package is included in the treelite conda package
+      # and not available to install separately
+      {% if not req.startswith("treelite_runtime") %}
+    - {{ req }}
+      {% endif %}
+    {% endfor %}
 
 about:
   home: https://github.com/NVIDIA-Merlin/systems