Update tutorials, workflows, and project configuration (#60)

* Removes branch selection when downloading data

* Generates the notebooks from scripts

* Auto-rename all cell ids using nbstripout

* Update project version to 0.6.0 and adjust dependencies

* Renames GitHub workflows for clarity

* Adds workflow to delete old GitHub Actions runs

* Add GitHub actions for automated release drafting

* Add workflow to perform security scanning

* Add workflow to enforce PR label verification

* Add GitHub Actions workflow for PyPI publishing

* Enhances project configuration in pyproject.toml

* Applies ruff linting

* Apply ruff linting with --unsafe-fixes

* Fixes dependency installation

* Fixes install command spacing in workflows

* Adds YAPF for code formatting and updates Ruff config

* Applies ruff formatting

* Apply more ruff formatting

* Apply prettier formatting

* Enhances CI/CD with testing workflow and notebook conversion

* Refines workflow scripts for improved readability

* Refactors notebook handling in CI workflow

* Adds Jupytext to documentation dependencies

* Updates workflows to enhance package testing

* Standardizes workflow job names and increases timeout

* Refines tutorial parameters for performance

* Refines Jupyter notebook conversion in workflow

* Refines CI workflow naming and formatting

* Updates workflow job names and adds shell specifications

* Reformats YAML for improved readability

* Reorganizes Python setup in GitHub workflow

* Adds Windows-specific conda setup to CI workflow

* Try fixing Unicode issues on Windows

* Enhances CI workflows and separates tutorial testing

* Refines workflows and dependencies handling

* Adds file overwrite option to download utility

* Updates tutorials

* Restricts security scan trigger and updates Python matrix

* Adds SPDX headers to all source files

* Updates copyright information

* Add contributing and development guidelines

* Corrects GitHub URL in SPDX headers

* Update tools/update_spdx-headers.py [ci skip]

Co-authored-by: Copilot <[email protected]>

---------

Co-authored-by: Copilot <[email protected]>

Author: AndrewSazonov

.github/release-drafter.yml

@@ -0,0 +1,34 @@
+# This file is used to configure the Release Drafter GitHub Action
+# https://github.com/marketplace/actions/release-drafter
+
+name-template: 'easydiffraction $RESOLVED_VERSION'
+tag-template: 'v$RESOLVED_VERSION'
+categories:
+  - title: 'Added'
+    labels: # Labels to use to categorize a pull request as a feature
+      - '[scope] significant'
+      - '[scope] enhancement'
+      - '[scope] documentation'
+  - title: 'Fixed'
+    labels: # Labels to use to categorize a pull request as a bug fix
+      - '[scope] bug'
+  - title: 'Changed'
+    labels: # Labels to use to categorize a pull request as a maintenance task
+      - '[scope] maintenance'
+change-template: '- $TITLE (#$NUMBER)'
+change-title-escapes: '\<*_&' # You can add # and @ to disable mentions, and add ` to disable code blocks.
+version-resolver:
+  major:
+    labels:
+      - '[scope] significant'
+  minor:
+    labels:
+      - '[scope] enhancement'
+  patch:
+    labels:
+      - '[scope] bug'
+      - '[scope] maintenance'
+      - '[scope] documentation'
+  default: patch
+template: |
+  $CHANGES

.github/workflows/building-deploying-docs.yml renamed to .github/workflows/build-docs.yml

@@ -1,14 +1,20 @@
-name: Building and deploying docs
+name: Build and deploy docs
 
 on:
   # Trigger the workflow on push
   push:
-    # To the develop and master branches
-    branches: [develop, master, docs, d-spacing]
-
+    # Selected branches
+    branches: [develop, master, docs]
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
+# Allow only one concurrent workflow, skipping runs queued between the run
+# in-progress and latest queued. And cancel in-progress runs.
+concurrency:
+  group:
+    ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   # Set the environment variables to be used in all jobs defined in this workflow
   # Set the CI_BRANCH environment variable to be the branch name
@@ -18,25 +24,25 @@ env:
 
 jobs:
   # Job 1: Build the static files for the documentation site
-  building-docs:
-    runs-on: macos-14 # Use macOS to switch to dark mode for Plotly charts
+  build-docs:
+    strategy:
+      matrix:
+        os: [macos-14] # Use macOS to switch to dark mode for Plotly charts
+        python-version: ['3.13']
 
-    steps:
-      - name: Cancel previous workflow runs
-        uses: n1hility/cancel-previous-runs@v2
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
+    runs-on: ${{ matrix.os }}
 
+    steps:
       # Without this step, GITHUB_REPOSITORY is not accessible from mkdocs.yml
       - name: Get GitHub repository
         run: echo "GITHUB_REPOSITORY=$GITHUB_REPOSITORY" >> $GITHUB_ENV
 
-      # Save the latest release version of easyscience/EasyDiffractionLib to RELEASE_VERSION
+      # Save the latest release version of easyscience/diffraction-lib to RELEASE_VERSION
       # RELEASE_VERSION is used in the mkdocs.yml file to set release_version.
       # The release_version is then needed to display the latest release version in the index.md file
-      - name: Get the latest release version of EasyDiffraction Library
+      - name: Get the latest release version of easydiffraction library
         run: |
-          git clone --depth 1 https://github.com/easyscience/EasyDiffractionLib .
+          git clone --depth 1 https://github.com/easyscience/${{ github.event.repository.name }} .
           git fetch --tags
           echo "RELEASE_VERSION=$(git describe --tags --abbrev=0)" >> $GITHUB_ENV
 
@@ -54,18 +60,19 @@ jobs:
       - name: Check-out repository
         uses: actions/checkout@v4
 
-      - name: Set up Python environment
+      - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
-          python-version: '3.12'
+          python-version: ${{ matrix.python-version }}
 
       - name: Upgrade package installer for Python
+        shell: bash
         run: python -m pip install --upgrade pip
 
       # Install EasyDiffraction Library to run Jupyter notebooks
-      # Install with the 'charts' and 'docs' extras
+      # Install with the 'docs' and 'visualization' extras
       - name: Install EasyDiffraction Library and its dependencies
-        run: python -m pip install .
+        run: python -m pip install .'[dev,docs,visualization]'
 
       # Clone assets extra from:
       # - easyscience/assets-docs
@@ -96,19 +103,23 @@ jobs:
           cp ../assets-branding/easydiffraction/icons/bw.svg overrides/.icons/easydiffraction.svg
           cp ../assets-branding/easyscience-org/icons/eso-icon_bw.svg overrides/.icons/easyscience.svg
 
-      # Copy Jupyter notebooks from the project to the docs folder
+      # Convert python scripts in the notebooks directory to Jupyter notebooks
+      # Strip output from the notebooks and simpify cell ids
       # The notebooks are used to generate the documentation
-      - name: Convert ${{ env.NOTEBOOKS_DIR }}/*.py to docs/${{env.NOTEBOOKS_DIR }}/*.ipynb
+      - name:
+          Convert ${{ env.NOTEBOOKS_DIR }}/*.py to docs/${{env.NOTEBOOKS_DIR
+          }}/*.ipynb
         run: |
           cp -R ${{ env.NOTEBOOKS_DIR }}/data docs/${{ env.NOTEBOOKS_DIR }}/
           jupytext ${{ env.NOTEBOOKS_DIR }}/*.py --from py:percent --to ipynb
+          nbstripout ${{ env.NOTEBOOKS_DIR }}/*.ipynb
           mv ${{ env.NOTEBOOKS_DIR }}/*.ipynb docs/${{ env.NOTEBOOKS_DIR }}/
 
       # The following step is needed to avoid the following message during the build:
       # "Matplotlib is building the font cache; this may take a moment"
       - name: Pre-build site step
         run: |
-          export PYTHONPATH=$(pwd)/src:$PYTHONPATH
+          export PYTHONPATH="$(pwd)/src${PYTHONPATH:+:$PYTHONPATH}"
           python -c "import easydiffraction"
 
       # Create the mkdocs.yml configuration file
@@ -125,7 +136,7 @@ jobs:
         run: |
           export JUPYTER_PLATFORM_DIRS=1
           export PYTHONWARNINGS="ignore::RuntimeWarning"
-          export PYTHONPATH=$(pwd)/src:$PYTHONPATH
+          export PYTHONPATH="$(pwd)/src${PYTHONPATH:+:$PYTHONPATH}"
           mkdocs build
 
       # Set up the Pages action to configure the static files to be deployed
@@ -140,17 +151,16 @@ jobs:
       # The artifact is then used in the next job by actions/deploy-pages to deploy the static files to GitHub Pages
       # Unfortunately, the artifact is not available for download, so extra steps below are needed to do similar things
       - name:
-          Upload built site as artifact for
-          easyscience.github.io/EasyDiffractionLib (all branches)
+          Upload built site as artifact for easyscience.github.io/${{
+          github.event.repository.name }} (all branches)
         uses: actions/upload-pages-artifact@v3
         with:
           path: site/
 
       # Upload the static files from the site/ directory to be used in the next job
       # This extra step is needed to allow the download of the artifact in the next job
       # for pushing its content to the branch named 'gh_pages'
-      - name:
-          Upload built site as artifact for gh_pages (master branch)
+      - name: Upload built site as artifact for gh_pages (master branch)
         if: ${{ env.CI_BRANCH == 'master' }}
         uses: actions/upload-artifact@v4
         with:
@@ -160,8 +170,8 @@ jobs:
           compression-level: 0
 
   # Job 2: Deploy the static files
-  deploying-docs:
-    needs: building-docs # previous job 'build-docs' need to be finished first
+  deploy-docs:
+    needs: build-docs # previous job 'build-docs' need to be finished first
 
     # Grant GITHUB_TOKEN the permissions required to make a Pages deployment
     permissions:
@@ -189,7 +199,9 @@ jobs:
       # https://github.com/easyscience/diffraction-lib/settings/environments
       # Currently, only develop and master branches are allowed to deploy to GitHub Pages
       # Deployed pages are available at https://easyscience.github.io/diffraction-lib
-      - name: Deploy to easyscience.github.io/diffraction-lib (all branches)
+      - name:
+          Deploy to easyscience.github.io/${{ github.event.repository.name }}
+          (all branches)
         uses: actions/deploy-pages@v4
 
       # Download built site as artifact from a previous job for gh_pages (master branch)
@@ -209,7 +221,8 @@ jobs:
       # https://github.com/easyscience/diffraction-lib/settings/hooks
       # This is done for the gh_pages branch when the site is tested with a step above
       - name:
-          Deploy to gh_pages branch to trigger deployment to custom domain (master branch)
+          Deploy to gh_pages branch to trigger deployment to custom domain
+          (master branch)
         if: ${{ env.CI_BRANCH == 'master' }}
         uses: s0/git-publish-subdir-action@develop
         env:

.github/workflows/delete-old-runs.yml

@@ -0,0 +1,80 @@
+# This workflow will delete old workflow runs based on the input
+# parameters.
+# https://github.com/Mattraks/delete-workflow-runs
+
+name: Delete old workflow runs
+
+on:
+  # Run monthly, at 00:00 on the 1st day of month.
+  schedule:
+    - cron: '0 0 1 * *'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      days:
+        description: 'Number of days.'
+        required: true
+        default: 30
+      minimum_runs:
+        description: 'The minimum runs to keep for each workflow.'
+        required: true
+        default: 6
+      delete_workflow_pattern:
+        description:
+          'The name or filename of the workflow. if not set then it will target
+          all workflows.'
+        required: false
+      delete_workflow_by_state_pattern:
+        description:
+          'Remove workflow by state: active, deleted, disabled_fork,
+          disabled_inactivity, disabled_manually'
+        required: true
+        default: 'All'
+        type: choice
+        options:
+          - 'All'
+          - active
+          - deleted
+          - disabled_inactivity
+          - disabled_manually
+      delete_run_by_conclusion_pattern:
+        description:
+          'Remove workflow by conclusion: action_required, cancelled, failure,
+          skipped, success'
+        required: true
+        default: 'All'
+        type: choice
+        options:
+          - 'All'
+          - action_required
+          - cancelled
+          - failure
+          - skipped
+          - success
+      dry_run:
+        description: 'Only log actions, do not perform any delete operations.'
+        required: false
+
+jobs:
+  del-runs:
+    runs-on: ubuntu-latest
+
+    permissions:
+      actions: write
+
+    steps:
+      - name: Delete workflow runs
+        uses: Mattraks/delete-workflow-runs@v2
+        with:
+          token: ${{ github.token }}
+          repository: ${{ github.repository }}
+          retain_days: ${{ github.event.inputs.days }}
+          keep_minimum_runs: ${{ github.event.inputs.minimum_runs }}
+          delete_workflow_pattern:
+            ${{ github.event.inputs.delete_workflow_pattern }}
+          delete_workflow_by_state_pattern:
+            ${{ github.event.inputs.delete_workflow_by_state_pattern }}
+          delete_run_by_conclusion_pattern:
+            ${{ github.event.inputs.delete_run_by_conclusion_pattern }}
+          dry_run: ${{ github.event.inputs.dry_run }}

.github/workflows/publish-pypi.yml

@@ -0,0 +1,41 @@
+# Builds a Python package and publish it to PyPI when a new tag is
+# created.
+
+name: Publish to PyPI
+
+on:
+  # Runs on creating a new tag starting with 'v', e.g. 'v1.0.3'
+  push:
+    tags:
+      - 'v*'
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  pypi-publish:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check-out repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: '0' # full history with tags to get the version number by versioningit
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Upgrade package installer for Python
+        run: pip install --upgrade pip
+
+      - name: Install Python dependencies
+        run: pip install '.[dev]'
+
+      - name: Create Python package
+        run: python -m build
+
+      - name: Publish distribution 📦 to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_PASSWORD }}

.github/workflows/scan-security.yml

@@ -0,0 +1,41 @@
+# Integrates a collection of open source static analysis tools with
+# GitHub code scanning.
+# https://github.com/github/ossar-action
+
+name: Scan security (static analysis)
+
+on:
+  # Trigger the workflow on pull request
+  pull_request:
+    branches:
+      - '**'
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  scan-security-ossar:
+    # OSSAR runs on windows-latest.
+    # ubuntu-latest and macos-latest support coming soon
+    runs-on: windows-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          # We must fetch at least the immediate parents so that if this is
+          # a pull request then we can checkout the head.
+          fetch-depth: 2
+
+      # If this run was triggered by a pull request event, then checkout
+      # the head of the pull request instead of the merge commit.
+      - run: git checkout HEAD^2
+        if: ${{ github.event_name == 'pull_request' }}
+
+      - name: Run open source static analysis tools
+        uses: github/ossar-action@main
+        id: ossar
+
+      - name: Upload results to Security tab
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: ${{ steps.ossar.outputs.sarifFile }}