Merge branch 'master' into prepare-8.9.10

.github/workflows/1_create_release_pr.yml

@@ -47,9 +47,7 @@ jobs:
         uses: cylc/release-actions/build-python-package@v1
 
       - name: Generate changelog
-        run: |
-          python3 -m pip install -q towncrier
-          towncrier build --yes
+        uses: MetRonnie/release-actions/stage-1/towncrier-build@changelog
 
       - name: Create pull request
         uses: cylc/release-actions/stage-1/create-release-pr@v1

.github/workflows/2_auto_publish_release.yml

@@ -45,6 +45,14 @@ jobs:
         # # Can try using this for testing:
         # repository_url: https://test.pypi.org/legacy/
 
+    - name: Write release notes
+      id: release-notes
+      uses: MetRonnie/release-actions/stage-2/write-release-notes@changelog
+      with:
+        footer: |
+          Cylc 8 can be installed via pypi or Conda - you don't need to download this release directly.
+          See the [installation](https://cylc.github.io/cylc-doc/latest/html/installation.html) section of the documentation.
+
     - name: Publish GitHub release
       id: create-release
       uses: cylc/release-actions/create-release@v1
@@ -55,12 +63,7 @@ jobs:
         tag_name: ${{ env.VERSION }}
         release_name: cylc-flow-${{ env.VERSION }}
         prerelease: ${{ env.PRERELEASE }}
-        body: |
-          See [${{ env.CHANGELOG_FILE }}](https://github.com/${{ github.repository }}/blob/master/${{ env.CHANGELOG_FILE }}) for detail.
-
-          Cylc 8 can be installed via pypi or Conda - you don't need to download this release directly.
-          See the [installation](https://cylc.github.io/cylc-doc/latest/html/installation.html) section of the documentation.
-        # TODO: Get topmost changelog section somehow and use that as the body?
+        body_path: ${{ steps.release-notes.outputs.filepath }}
 
     - name: Comment on the release PR with the results & next steps
       if: always()

.github/workflows/branch_sync.yml

@@ -14,109 +14,7 @@ on:
 
 jobs:
   sync:
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    env:
-      BASE_BRANCH: master
-      HEAD_BRANCH: ${{ inputs.head_branch || github.ref_name }}
-      STATUS_JSON: https://raw.githubusercontent.com/cylc/cylc-admin/master/docs/status/branches.json
-      FORCE_COLOR: 2
-    steps:
-      - name: Check branch name
-        shell: python
-        run: |
-          import os
-          import json
-          import sys
-          from urllib.request import urlopen
-
-          if os.environ['GITHUB_EVENT_NAME'] == 'schedule':
-            # Get branch from status page
-            meta = json.loads(
-              urlopen(os.environ['STATUS_JSON']).read()
-            )['meta_releases']
-            version = min(meta)
-            branch = meta[version][os.environ['GITHUB_REPOSITORY']]
-          else:
-            branch = os.environ['HEAD_BRANCH'].strip()
-
-          if branch.endswith('-sync'):
-            sys.exit("::error::Do not run this workflow for already-created sync branches")
-
-          with open(os.environ['GITHUB_ENV'], 'a') as F:
-            print(f'HEAD_BRANCH={branch}', file=F)
-            print(f'SYNC_BRANCH={branch}-sync', file=F)
-
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: master
-
-      - name: Configure git
-        uses: cylc/release-actions/configure-git@v1
-
-      - name: Checkout sync branch if it exists
-        continue-on-error: true
-        run: |
-          git switch -c "$SYNC_BRANCH" "origin/${SYNC_BRANCH}"
-          echo "BASE_BRANCH=$SYNC_BRANCH" >> "$GITHUB_ENV"
-
-      - name: Attempt fast-forward
-        id: ff
-        run: |
-          if git merge "origin/${HEAD_BRANCH}" --ff-only; then
-            if [[ "$(git rev-parse HEAD)" == "$(git rev-parse "origin/${BASE_BRANCH}")" ]]; then
-              echo "::notice::$BASE_BRANCH is up to date with $HEAD_BRANCH"
-              exit 0
-            fi
-            git push origin "$BASE_BRANCH"
-          elif [[ "$BASE_BRANCH" == "$SYNC_BRANCH" ]]; then
-            echo "::notice::Cannot fast-forward $BASE_BRANCH to $HEAD_BRANCH; merge existing PR first"
-          else
-            echo "continue=true" >> "$GITHUB_OUTPUT"
-          fi
-
-      - name: Attempt merge into master
-        id: merge
-        if: steps.ff.outputs.continue
-        run: |
-          git switch master
-          if git merge "origin/${HEAD_BRANCH}"; then
-            if git diff HEAD^ --exit-code --stat; then
-              echo "::notice::No diff between master and $HEAD_BRANCH"
-              exit 0
-            fi
-          else
-            git merge --abort
-          fi
-          echo "continue=true" >> $GITHUB_OUTPUT
-
-      - name: Push sync branch
-        id: push
-        if: steps.merge.outputs.continue
-        run: |
-          git switch -c "$SYNC_BRANCH" "origin/${HEAD_BRANCH}"
-          git push origin "$SYNC_BRANCH"
-          echo "continue=true" >> $GITHUB_OUTPUT
-
-      - name: Open PR
-        if: steps.push.outputs.continue
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          BODY: |
-            Please do a **normal merge**, not squash merge.
-            Please fix conflicts if necessary.
-
-            ---
-
-            Triggered by `${{ github.event_name }}`
-        run: |
-          url="$(
-            gh pr create --head "$SYNC_BRANCH" \
-            --title "🤖 Merge ${SYNC_BRANCH} into master" \
-            --body "$BODY"
-          )"
-          echo "::notice::PR created at ${url}"
-
-          gh pr edit "$SYNC_BRANCH" --add-label "sync" || true
+    uses: cylc/release-actions/.github/workflows/branch-sync.yml@v1
+    with:
+      head_branch: ${{ inputs.head_branch }}
+    secrets: inherit

.github/workflows/test_fast.yml

@@ -17,10 +17,10 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 20
     strategy:
-      fail-fast: false # Don't let a failed MacOS run stop the Ubuntu runs
+      fail-fast: false # don't stop on first failure
       matrix:
         os: ['ubuntu-latest']
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3']
         include:
           - os: 'macos-latest'
             python-version: '3.7'
@@ -39,7 +39,7 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu')
         run: |
           sudo apt-get update
-          sudo apt-get install -y shellcheck sqlite3
+          sudo apt-get install -y sqlite3
 
       - name: Install
         run: |
@@ -48,37 +48,10 @@ jobs:
       - name: Configure git  # Needed by the odd test
         uses: cylc/release-actions/configure-git@v1
 
-      - name: Check changelog
-        if: startsWith(matrix.os, 'ubuntu')
-        run: towncrier build --draft
-
-      - name: Style
-        if: startsWith(matrix.os, 'ubuntu')
-        run: |
-          flake8
-          etc/bin/shellchecker
-
-      # note: exclude python 3.10+ from mypy checks as these produce false
-      # positives in installed libraries for python 3.7
-      - name: Typing
-        if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python-version, 3.9)
-        run: mypy
-
-      - name: Doctests
-        timeout-minutes: 4
-        run: |
-          pytest cylc/flow
-
       - name: Unit Tests
-        timeout-minutes: 4
-        run: |
-          pytest tests/unit
-
-      - name: Bandit
-        if: ${{ matrix.python-version == '3.7' }}
-        # https://github.com/PyCQA/bandit/issues/658
+        timeout-minutes: 5
         run: |
-          bandit -r --ini .bandit cylc/flow
+          pytest cylc/flow tests/unit
 
       - name: Integration Tests
         timeout-minutes: 6
@@ -104,9 +77,47 @@ jobs:
           path: coverage.xml
           retention-days: 7
 
+  lint:
+    runs-on: 'ubuntu-latest'
+    timeout-minutes: 10
+    steps:
+      - name: Apt-Get Install
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y shellcheck
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # note: exclude python 3.10+ from mypy checks as these produce false
+      # positives in installed libraries for python 3.7
+      - name: Configure Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Install
+        run: |
+          pip install -e ."[tests]"
+
+      - name: Flake8
+        run: flake8
+
+      - name: Bandit
+        run: |
+          bandit -r --ini .bandit cylc/flow
+
+      - name: Shellchecker
+        run: etc/bin/shellchecker
+
+      - name: MyPy
+        run: mypy
+
+      - name: Towncrier
+        run: towncrier build --draft
+
       - name: Linkcheck
-        if: startsWith(matrix.python-version, '3.10')
-        run: pytest -m linkcheck --dist=load tests/unit
+        run: pytest -m linkcheck --dist=load --color=yes -n 10 tests/unit/test_links.py
 
   codecov:
     needs: test

.github/workflows/test_functional.yml

@@ -46,9 +46,9 @@ jobs:
         # NOTE: includes must define ALL of the matrix values
         include:
           # latest python
-          - name: 'py-3.11'
+          - name: 'py-3-latest'
             os: 'ubuntu-latest'
-            python-version: '3.11'
+            python-version: '3'
             test-base: 'tests/f'
             chunk: '1/4'
             platform: '_local_background*'
@@ -108,7 +108,7 @@ jobs:
         run: |
           # install system deps
           brew update
-          brew install bash coreutils gnu-sed
+          brew install bash coreutils gnu-sed grep
 
           # add GNU coreutils and sed to the user PATH
           # (see instructions in brew install output)
@@ -118,6 +118,9 @@ jobs:
           echo \
             "/usr/local/opt/gnu-sed/libexec/gnubin" \
             >> "${GITHUB_PATH}"
+          echo \
+            "/usr/local/opt/grep/libexec/gnubin" \
+            >> "${GITHUB_PATH}"
 
           # add coreutils to the bashrc too (for jobs)
           cat >> "${HOME}/.bashrc" <<__HERE__

CHANGES.md

@@ -19,10 +19,18 @@ $ towncrier create <PR-number>.<break|feat|fix>.md --content "Short description"
 
 ### 🔧 Fixes
 
+WHATEVER
+## __cylc-8.2.3 (Released 2023-11-02)__
+
+### 🔧 Fixes
+
+[#5660](https://github.com/cylc/cylc-flow/pull/5660) - Re-worked graph n-window algorithm for better efficiency.
 [#5753](https://github.com/cylc/cylc-flow/pull/5753) - Fixed bug where execution time limit polling intervals could end up incorrectly applied
 
 [#5776](https://github.com/cylc/cylc-flow/pull/5776) - Ensure that submit-failed tasks are marked as incomplete (so remain visible) when running in back-compat mode.
 
+[#5791](https://github.com/cylc/cylc-flow/pull/5791) - fix a bug where if multiple clock triggers are set for a task only one was being satisfied.
+
 ## __cylc-8.2.2 (Released 2023-10-05)__
 
 ### 🚀 Enhancements

CONTRIBUTING.md

@@ -64,7 +64,7 @@ requests_).
  - Prasanna Challuri
  - David Matthews
  - Tim Whitcomb
- - (Scott Wales)
+ - Scott Wales
  - Tomek Trzeciak
  - Thomas Coleman
  - Bruno Kinoshita

changes.d/5709.feat.md

@@ -0,0 +1 @@
+Forward arbitrary environment variables over SSH connections

changes.d/5727.break.md

@@ -0,0 +1 @@
+Cylc now ignores `PYTHONPATH` to make it more robust to task environments which set this value. If you want to add to the Cylc environment itself, e.g. to install a Cylc extension, use `CYLC_PYTHONPATH`.

changes.d/5731.feat.md

@@ -0,0 +1 @@
+Major upgrade to `cylc tui` which now supports larger workflows and can browse installed workflows.

changes.d/5772.feat.md

@@ -0,0 +1 @@
+Add a check for indentation being 4N spaces.

changes.d/5794.break.md

@@ -0,0 +1 @@
+Remove `cylc report-timings` from automatic installation with `pip install cylc-flow[all]`. If you now wish to install it use `pip install cylc-flow[report-timings]`. `cylc report-timings` is incompatible with Python 3.12.

changes.d/5801.fix.md

@@ -0,0 +1 @@
+Fix traceback when using parentheses on right hand side of graph trigger.

changes.d/5803.feat.md

@@ -0,0 +1 @@
+Updated 'reinstall' functionality to support multiple workflows

changes.d/5821.fix.md

@@ -0,0 +1 @@
+Fixed issue where large uncommitted changes could cause `cylc install` to hang.

changes.d/5836.break.md

@@ -0,0 +1 @@
+Removed the 'CYLC_TASK_DEPENDENCIES' environment variable

changes.d/5838.feat.md

@@ -0,0 +1 @@
+`cylc lint`: added rule to check for `rose date` usage (should be replaced with `isodatetime`).

changes.d/5841.fix.md

@@ -0,0 +1 @@
+Improve handling of S011 to not warn if the # is '#$' (e.g. shell base arithmetic)

conda-environment.yml

@@ -9,14 +9,14 @@ dependencies:
   - graphviz  # for static graphing
   # Note: can't pin jinja2 any higher than this until we give up on Cylc 7 back-compat
   - jinja2 >=3.0,<3.1
-  - metomi-isodatetime >=1!3.0.0, <1!3.1.0
+  - metomi-isodatetime >=1!3.0.0, <1!3.2.0
+  - packaging
   # Constrain protobuf version for compatible Scheduler-UIS comms across hosts
-  - protobuf >=4.21.2,<4.22.0
+  - protobuf >=4.24.4,<4.25.0
   - psutil >=5.6.0
   - python
   - pyzmq >=22
-  - setuptools >=49,!=67.*
-  - importlib_metadata # [py<3.8]
+  - importlib_metadata >=5.0 # [py<3.12]
   - urwid >=2,<3
   - tomli >=2 # [py<3.11]
 

cylc/flow/__init__.py

@@ -58,11 +58,19 @@ def environ_init():
 
 def iter_entry_points(entry_point_name):
     """Iterate over Cylc entry points."""
-    import pkg_resources
+    import sys
+    if sys.version_info[:2] > (3, 11):
+        from importlib.metadata import entry_points
+    else:
+        # BACK COMPAT: importlib_metadata
+        #   importlib.metadata was added in Python 3.8. The required interfaces
+        #   were completed by 3.12. For lower versions we must use the
+        #   importlib_metadata backport.
+        # FROM: Python 3.7
+        # TO: Python: 3.12
+        from importlib_metadata import entry_points
     yield from (
         entry_point
-        for entry_point in pkg_resources.iter_entry_points(entry_point_name)
-        # Filter out the cylc namespace as it should be empty.
-        # All cylc packages should take the form cylc-<name>
-        if entry_point.dist.key != 'cylc'
+        # for entry_point in entry_points()[entry_point_name]
+        for entry_point in entry_points().select(group=entry_point_name)
     )