Automations for sparc.client

.github/workflows/CI.yml

@@ -1,19 +1,23 @@
 # This workflow will install Python dependencies, run tests and lint with a single version of Python
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: CI 
+name: CI
 
 on:
-  [push]
-
+  push:
+    tags:
+      - 'v*'  # Execute if version is tagged
+    branches:
+      - main  # Or if it is push to main
+  pull_request:  # Or if it is any pull_request
+  workflow_call: # Or if the call comes from another workflow (reusability)
+
 permissions:
   contents: read
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
-
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python 3.10
@@ -36,21 +40,19 @@ jobs:
     - name: Generate coverage report
       run: |
         PYTHONPATH=src
-        pytest --cov=./src tests/
+        pytest --cov=./src tests/ --cov-report term --cov-report=xml:coverage.xml
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v3
       with:
         verbose: true
-    - name: Install pypa/build
-      run: >-
-        python -m
-        pip install
-        build
-        --user
-    - name: Build a binary wheel and a source tarball
-      run: >-
-        python -m
-        build
-        --sdist
-        --wheel
-        --outdir dist/
+        directory: ./coverage/reports/
+        env_vars: OS,PYTHON
+        fail_ci_if_error: true
+        files: ./coverage.xml
+        flags: unittests
+        name: codecov-umbrella
+#        token: ${{ secrets.CODECOV_TOKEN }}
+    - name: Check if the package builds
+      run: |
+        python -m pip install -U pip build
+        python -m build

.github/workflows/release.yml

@@ -0,0 +1,108 @@
+# Build the package, release new version and publish it to PyPI after tests pass
+# References:
+# - https://discourse.jupyter.org/t/use-github-workflows-to-automatically-publish-to-pypi-when-new-tags-are-created/14941/2
+# - https://dev.to/this-is-learning/manually-trigger-a-github-action-with-workflowdispatch-3mga
+# - https://maciek.land/blog/automatic-releases-with-github-actions
+# - https://www.loopwerk.io/articles/2021/automating-changelog/
+
+
+name: Release new version
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Please specify version, e.g.: v0.3.5"
+        default: "v0.0.0"
+jobs:
+  verify:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check permissions
+        uses: actions-cool/check-user-permission@v2
+        id: checkUser
+        with:
+          require: 'admin'
+  check:
+    needs: verify
+    uses: ./.github/workflows/CI.yml
+#    if: steps.checkUser.outputs.check-result == 'true'
+  publish:
+    needs: check
+    runs-on: ubuntu-latest
+
+    # Specifying a GitHub environment for PyPI release is optional, but strongly encouraged
+    environment: 
+      name: pypi
+      url: https://pypi.org/p/sparc.client
+    permissions:
+#      # IMPORTANT: this permission is mandatory for trusted publishing on PyPI
+      id-token: write
+      contents: write
+      packages: write
+    steps:
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Checkout source
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Configure git user
+        run: |
+          git config user.email "[email protected]"
+          git config user.name "GitHub Actions (run by ${{ github.actor }})"
+
+      - name: Tag commit
+        uses: tvdias/[email protected]
+        with:
+          repo-token: ${{ github.token }}
+          tag: ${{ github.event.inputs.version }}
+
+      - name: Update CHANGELOG
+        id: changelog
+        uses: requarks/changelog-action@v1
+        with:
+          token: ${{ github.token }}
+          tag: ${{ github.event.inputs.version }}
+
+      - name: Commit and push CHANGELOG.md
+        uses: EndBug/add-and-commit@v9
+        with:
+          add: CHANGELOG.md
+          message: "chore: Update CHANGELOG.md"
+          pull: '--rebase --autostash'
+          tag: '-a ${{ github.event.inputs.version }} -m ${{ github.event.inputs.version }} --force'
+          tag_push: '--force'
+
+      - name: Build package
+        run: |
+          python -m pip install -U pip build
+          python -m build
+
+#      - name: Publish Distribution to TestPyPI
+#        uses: pypa/[email protected]
+#        with:
+#          user: __token__
+#          password: ${{ secrets.TEST_PYPI_TOKEN }}
+#          repository_url: https://test.pypi.org/legacy/
+#          skip_existing: true
+
+      - name: Publish Distribution to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+#        with: #not needed with new PyPI publishing
+#          user: __token__
+#          password: ${{ secrets.PYPI_TOKEN }}
+
+      - name: Create Release
+        uses: ncipollo/[email protected]
+        with:
+          allowUpdates: true
+          draft: false
+          makeLatest: true
+          name: ${{ github.event.inputs.version }}
+          tag: ${{ github.event.inputs.version }}
+          body: ${{ steps.changelog.outputs.changes }}
+          token: ${{ github.token }}

.github/workflows/sphinx.yml

@@ -1,10 +1,21 @@
 name: Sphinx build
 
-on: push
+on: 
+  push:
+    branches:
+      - main  # Run if it is a push to main
+  pull_request:
+    branches:
+      - main  # Or if it is a pull request to main
 
 jobs:
+  check:
+    uses: ./.github/workflows/CI.yml
+
   build:
+    needs: check
     runs-on: ubuntu-latest
+
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python 3.10

.gitignore

@@ -27,6 +27,9 @@ share/python-wheels/
 *.egg
 MANIFEST
 
+# dynamic versioning in _version.py (automatically built for PyPI only)
+src/sparc/client/_version.py
+
 # PyInstaller
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.

CHANGELOG.md

@@ -1,51 +1,67 @@
-v0.2.0
-======
-New functionalities:
-* Added O2SparcService service:
-   * Introduced a `O2SparcSolver` class which is the main class for running computational jobs on o²S²PARC. This class holds the following methods:
-   * `submit_job`
-   * `get_job_progress`
-   * `job_done`
-   * `get_results`
-   * `get_job_log`
-* Introduced `get_solver` method to `O2SparcService` which returns a `O2SparcSolver` object
-* Scaffold Retrieval:
-   * Introducing the ability to use `sparc.client` to retrieve scaffolds or scaffold descriptions.
-   * The retrieved scaffold or scaffold description files can now be converted to a commonly used mesh format, such as VTK.
-   * Reuse of packages from the mapping tools codebase ensures efficient and standardized mesh conversion.
-* MBF Segmentation Export:
-   * Added support for exporting MBF Segmentation data to a commonly used mesh format, like VTK.
-* Segmentation Data Analysis:
-   * New functionality to analyze a given segmentation data file for suitability in the mapping tools fitting workflow, and provide a clear and informative report.
-* Updated Documentation:
-   * Added the [SPARC Python Zinc Client tutorial](https://github.com/nih-sparc/sparc.client/blob/main/docs/tutorial-zinc.ipynb) to reflect the features related to Zinc.
-
-
-v0.1.0
-======
-Fixes:
-* download multiple files from Pennsieve #12
-* pennsieve Download file API #14
-* Github action updates: Reviewdog should run whenever a PR is modified after opening #15
-* new tutorial in Jupyter Notebook
-* README.md update
-
-
-v0.0.2
-======
+# CHANGELOG
+
+## [v0.2.0]
+
+### :sparkles: New features
+
+- Added O2SparcService service:
+
+  * Introduced a `O2SparcSolver` class which is the main class for running computational jobs on o²S²PARC. This class holds the following methods:
+  * `submit_job`
+  * `get_job_progress`
+  * `job_done`
+  * `get_results`
+  * `get_job_log`
+
+- Introduced `get_solver` method to `O2SparcService` which returns a `O2SparcSolver` object
+
+- Scaffold Retrieval:
+
+  * Introducing the ability to use `sparc.client` to retrieve scaffolds or scaffold descriptions.
+  * The retrieved scaffold or scaffold description files can now be converted to a commonly used mesh format, such as VTK.
+  * Reuse of packages from the mapping tools codebase ensures efficient and standardized mesh conversion.
+
+- MBF Segmentation Export:
+
+  * Added support for exporting MBF Segmentation data to a commonly used mesh format, like VTK.
+
+- Segmentation Data Analysis:
+
+  * New functionality to analyze a given segmentation data file for suitability in the mapping tools fitting workflow, and provide a clear and informative report.
+
+- Updated Documentation:
+
+  * Added the [SPARC Python Zinc Client tutorial](https://github.com/nih-sparc/sparc.client/blob/main/docs/tutorial-zinc.ipynb) to reflect the features related to Zinc.
+
+
+## [v0.1.0]
+
+### :bug: Bug Fixes
+
+- download multiple files from Pennsieve #12
+- pennsieve Download file API #14
+- Github action updates: Reviewdog should run whenever a PR is modified after opening #15
+- new tutorial in Jupyter Notebook
+- README.md update
+
+## [v0.0.2]
+
 Alpha2 release of Python Sparc Client.
-Major updates:
-* Code coverage at 100%
-* Sphinx documentation with Github Pages
 
+### :sparkles: New features
+
+- Code coverage at 100%
+- Sphinx documentation with Github Pages
+
+## [v0.0.1]
 
-v0.0.1
-======
 Alpha release of Python Sparc Client.
-Basic functionalities:
-* automatic/manual module loading
-* ServiceBase class for adding new modules
-* Pennsieve module with basic functionalities: 
-   * listing datasets, files, records
-   * downloading files
-   * Basic API support (GET/POST) 
+
+### :sparkles: New features
+
+- automatic/manual module loading
+- ServiceBase class for adding new modules
+- Pennsieve module with basic functionalities:
+  * listing datasets, files, records
+  * downloading files
+  * Basic API support (GET/POST)

README.md

@@ -94,3 +94,48 @@ Run `pip install -e '.[test]'` to install the dependencies needed for a developm
 Run `pytest --cov=./src` to run the tests and get a test coverage summary.
 
 Run `pytest --cov-report html --cov=./src` to run the tests and get a full HTML coverage report output to `htmlcov`.
+
+Run `python -m build` to check if your package builds successfully.
+
+
+The process is currently automated using Github Action in CI.yml.
+
+
+# Software releasing guidelines
+
+The process of releasing new version of the software is fully automated.
+
+This means that `CHANGELOG.md` as well as release commands are automatically generated.
+
+The versioning is fully dynamic using git tags.
+
+Please also note that there is no package/software version pyproject.toml. We use dynamic versioning provided by setuptools_scm
+
+
+Also file sparc.client/_version.py should not be committed to the repository.
+
+
+
+## How commits should look like?
+
+We are using [Semantic Commit Messages](https://gist.github.com/joshbuchea/6f47e86d2510bce28f8e7f42ae84c716) for commits. 
+
+Basically this means that important commits should start with one of the following prefixes: *chore:*, *docs:*, *feat:*, *fix:*, *refactor:*, *style:*, or *test:*.
+
+Additionally, we ask to refer to the issue number on Github (by adding a reference, e.g. `#24234`)
+[Refer to issues](https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/autolinked-references-and-urls)
+
+
+## Releasing a new version
+
+
+1. In order to release a new version, an action 'Create new release' needs to be launched from Github Actions menu. 
+Please navigate to 'Actions' and click 'Create new release' on the left hand side. On the right hand side, you can click 'Run workflow'
+
+2. After launching a workflow, specify manually a version. The version needs to start with 'v', e.g. 'v0.0.34'.
+
+3. Launching a workflow checks for the user permission (needs to be admin to the repository) and runs CI in order to verify integrity of the software.
+
+4. If the CI/CD test passes, a temporary tag is created. The commits which follow the symantic versioning naming convention are then used to create and update CHANGELOG.md.
+
+5. Once CHANGELOG.md is pushed to the main branch, the new version is tagged again and the software is released to Github and to PyPI.