Merge branch 'master' into 8.3.x-sync

.github/workflows/2_auto_publish_release.yml

@@ -38,7 +38,7 @@ jobs:
       uses: cylc/release-actions/build-python-package@v1
 
     - name: Publish distribution to PyPI
-      uses: pypa/[email protected].1
+      uses: pypa/[email protected].3
       with:
         user: __token__ # uses the API token feature of PyPI - least permissions possible
         password: ${{ secrets.PYPI_TOKEN }}

CHANGES.md

@@ -73,8 +73,6 @@ $ towncrier create <PR-number>.<break|feat|fix>.md --content "Short description"
 
 ### 🔧 Fixes
 
-[#6178](https://github.com/cylc/cylc-flow/pull/6178) - Fix an issue where Tui could hang when closing.
-
 [#6186](https://github.com/cylc/cylc-flow/pull/6186) - Fixed bug where using flow numbers with `cylc set` would not work correctly.
 
 [#6200](https://github.com/cylc/cylc-flow/pull/6200) - Fixed bug where a stalled paused workflow would be incorrectly reported as running, not paused
@@ -91,6 +89,8 @@ $ towncrier create <PR-number>.<break|feat|fix>.md --content "Short description"
 
 [#6176](https://github.com/cylc/cylc-flow/pull/6176) - Fix bug where jobs which fail to submit are not shown in GUI/TUI if submission retries are set.
 
+[#6178](https://github.com/cylc/cylc-flow/pull/6178) - Fix an issue where Tui could hang when closing.
+
 ## __cylc-8.3.0 (Released 2024-06-18)__
 
 ### ⚠ Breaking Changes

CONTRIBUTING.md

@@ -96,6 +96,7 @@ requests_).
  - Diquan Jabbour
  - Shixian Sheng
  - Utheri Wagura
+ - Paul Earnshaw
 <!-- end-shortlog -->
 
 (All contributors are identifiable with email addresses in the git version

changes.d/6137.feat.md

@@ -0,0 +1 @@
+New Cylc lint rule: S014: Don't use job runner specific execution time limit directives, use execution time limit.

changes.d/6168.feat.md

@@ -0,0 +1 @@
+Allow symlinking log/job separately from log

changes.d/6289.feat.md

@@ -0,0 +1 @@
+Made the errors resulting from Jinja2 `raise` and `assert` statements more straight forward.

changes.d/6364.fix.md

@@ -0,0 +1 @@
+Fixed bug where `cylc clean <workflow> --rm share` would not take care of removing the target of the `share/cycle` symlink directory.

changes.d/6376.fix.md

@@ -0,0 +1 @@
+Fixes an issue that could cause Cylc to ignore the remaining hosts in a platform in response to an `ssh` error in some niche circumstances.

cylc/flow/__init__.py

@@ -15,9 +15,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """Set up the cylc environment."""
 
-import os
 import logging
-
+import os
 
 CYLC_LOG = 'cylc'
 
@@ -53,7 +52,7 @@ def environ_init():
 
 environ_init()
 
-__version__ = '8.3.6.dev'
+__version__ = '8.4.0.dev'
 
 
 def iter_entry_points(entry_point_name):

cylc/flow/broadcast_report.py

@@ -72,7 +72,7 @@ def get_broadcast_change_iter(modified_settings, is_cancel=False):
         value = setting
         keys_str = ""
         while isinstance(value, dict):
-            key, value = list(value.items())[0]
+            key, value = next(iter(value.items()))
             if isinstance(value, dict):
                 keys_str += "[" + key + "]"
             else:

cylc/flow/cfgspec/globalcfg.py

@@ -30,6 +30,7 @@
 from cylc.flow.exceptions import GlobalConfigError
 from cylc.flow.hostuserutil import get_user_home
 from cylc.flow.network.client_factory import CommsMeth
+from cylc.flow.pathutil import SYMLINKABLE_LOCATIONS
 from cylc.flow.parsec.config import (
     ConfigNode as Conf,
     ParsecConfig,
@@ -1148,55 +1149,21 @@ def default_for(
 
                     .. versionadded:: 8.0.0
                 """)
-                Conf('log', VDR.V_STRING, None, desc="""
-                    Alternative location for the log dir.
-
-                    If specified the workflow log directory will be created in
-                    ``<this-path>/cylc-run/<workflow-id>/log`` and a
-                    symbolic link will be created from
-                    ``$HOME/cylc-run/<workflow-id>/log``. If not specified
-                    the workflow log directory will be created in
-                    ``$HOME/cylc-run/<workflow-id>/log``.
-
-                    .. versionadded:: 8.0.0
-                """)
-                Conf('share', VDR.V_STRING, None, desc="""
-                    Alternative location for the share dir.
-
-                    If specified the workflow share directory will be
-                    created in ``<this-path>/cylc-run/<workflow-id>/share``
-                    and a symbolic link will be created from
-                    ``<$HOME/cylc-run/<workflow-id>/share``. If not specified
-                    the workflow share directory will be created in
-                    ``$HOME/cylc-run/<workflow-id>/share``.
-
-                    .. versionadded:: 8.0.0
-                """)
-                Conf('share/cycle', VDR.V_STRING, None, desc="""
-                    Alternative directory for the share/cycle dir.
-
-                    If specified the workflow share/cycle directory
-                    will be created in
-                    ``<this-path>/cylc-run/<workflow-id>/share/cycle``
-                    and a symbolic link will be created from
-                    ``$HOME/cylc-run/<workflow-id>/share/cycle``. If not
-                    specified the workflow share/cycle directory will be
-                    created in ``$HOME/cylc-run/<workflow-id>/share/cycle``.
-
-                    .. versionadded:: 8.0.0
-                """)
-                Conf('work', VDR.V_STRING, None, desc="""
-                    Alternative directory for the work dir.
-
-                    If specified the workflow work directory will be created in
-                    ``<this-path>/cylc-run/<workflow-id>/work`` and a
-                    symbolic link will be created from
-                    ``$HOME/cylc-run/<workflow-id>/work``. If not specified
-                    the workflow work directory will be created in
-                    ``$HOME/cylc-run/<workflow-id>/work``.
-
-                    .. versionadded:: 8.0.0
-                """)
+                for folder, versionadded in SYMLINKABLE_LOCATIONS.items():
+                    Conf(folder, VDR.V_STRING, None, desc=f"""
+                        Alternative location for the log dir.
+
+                        If specified the workflow {folder} directory will
+                        be created in
+                        ``<this-path>/cylc-run/<workflow-id>/{folder}``
+                        and a symbolic link will be created from
+                        ``$HOME/cylc-run/<workflow-id>/{folder}``.
+                        If not specified the workflow log directory will
+                        be created in
+                        ``$HOME/cylc-run/<workflow-id>/{folder}``.
+
+                        .. versionadded:: {versionadded}
+                    """)
     with Conf('platforms', desc='''
         Platforms allow you to define compute resources available at your
         site.
@@ -1311,7 +1278,7 @@ def default_for(
                 The means by which task progress messages are reported back to
                 the running workflow.
 
-                Options:
+                ..rubric:: Options:
 
                 zmq
                    Direct client-server TCP communication via network ports
@@ -1320,6 +1287,8 @@ def default_for(
                 ssh
                    Use non-interactive ssh for task communications
 
+                For more information, see :ref:`TaskComms`.
+
                 .. versionchanged:: 8.0.0
 
                    {REPLACES}``global.rc[hosts][<host>]task communication

cylc/flow/clean.py

@@ -57,6 +57,7 @@
 )
 from cylc.flow.pathutil import (
     get_workflow_run_dir,
+    is_relative_to,
     parse_rm_dirs,
     remove_dir_and_target,
     remove_dir_or_file,
@@ -116,20 +117,16 @@ def _clean_check(opts: 'Values', id_: str, run_dir: Path) -> None:
     # Thing to clean must be a dir or broken symlink:
     if not run_dir.is_dir() and not run_dir.is_symlink():
         raise FileNotFoundError(f"No directory to clean at {run_dir}")
-    db_path = (
-        run_dir / WorkflowFiles.Service.DIRNAME / WorkflowFiles.Service.DB
-    )
-    if opts.local_only and not db_path.is_file():
-        # Will reach here if this is cylc clean re-invoked on remote host
-        # (workflow DB only exists on scheduler host); don't need to worry
+    if opts.no_scan:
+        # This is cylc clean re-invoked on remote host; don't need to worry
         # about contact file.
         return
     try:
         detect_old_contact_file(id_)
     except ContactFileExists as exc:
         raise ServiceFileError(
             f"Cannot clean running workflow {id_}.\n\n{exc}"
-        )
+        ) from None
 
 
 def init_clean(id_: str, opts: 'Values') -> None:
@@ -173,7 +170,7 @@ def init_clean(id_: str, opts: 'Values') -> None:
             try:
                 platform_names = get_platforms_from_db(local_run_dir)
             except ServiceFileError as exc:
-                raise ServiceFileError(f"Cannot clean {id_} - {exc}")
+                raise ServiceFileError(f"Cannot clean {id_} - {exc}") from None
             except sqlite3.OperationalError as exc:
                 # something went wrong with the query
                 # e.g. the table/field we need isn't there
@@ -186,7 +183,7 @@ def init_clean(id_: str, opts: 'Values') -> None:
                     ' local files (you may need to remove files on other'
                     ' platforms manually).'
                 )
-                raise ServiceFileError(f"Cannot clean {id_} - {exc}")
+                raise ServiceFileError(f"Cannot clean {id_} - {exc}") from exc
 
         if platform_names and platform_names != {'localhost'}:
             remote_clean(
@@ -259,7 +256,7 @@ def glob_in_run_dir(
     """Execute a (recursive) glob search in the given run directory.
 
     Returns list of any absolute paths that match the pattern. However:
-    * Does not follow symlinks (apart from the spcedified symlink dirs).
+    * Does not follow symlinks (apart from the specified symlink dirs).
     * Also does not return matching subpaths of matching directories (because
         that would be redundant).
 
@@ -281,6 +278,9 @@ def glob_in_run_dir(
     results: List[Path] = []
     subpath_excludes: Set[Path] = set()
     for path in matches:
+        # Iterate down through ancestors (starting at the run dir) to
+        # weed out redundant subpaths of matched directories and subpaths of
+        # non-standard symlinks
         for rel_ancestor in reversed(path.relative_to(run_dir).parents):
             ancestor = run_dir / rel_ancestor
             if ancestor in subpath_excludes:
@@ -326,13 +326,19 @@ def _clean_using_glob(
         LOG.info(f"No files matching '{pattern}' in {run_dir}")
         return
     # First clean any matching symlink dirs
-    for path in abs_symlink_dirs:
-        if path in matches:
-            remove_dir_and_target(path)
-            if path == run_dir:
+    for symlink_dir in abs_symlink_dirs:
+        # Note: must clean e.g. share/cycle/ before share/ if the former
+        # is a symlink even if only the latter was specified.
+        if (
+            any(is_relative_to(symlink_dir, path) for path in matches)
+            and symlink_dir.is_symlink()
+        ):
+            remove_dir_and_target(symlink_dir)
+            if symlink_dir == run_dir:
                 # We have deleted the run dir
                 return
-            matches.remove(path)
+            if symlink_dir in matches:
+                matches.remove(symlink_dir)
     # Now clean the rest
     for path in matches:
         remove_dir_or_file(path)
@@ -361,7 +367,8 @@ def remote_clean(
     except PlatformLookupError as exc:
         raise PlatformLookupError(
             f"Cannot clean {id_} on remote platforms as the workflow database "
-            f"is out of date/inconsistent with the global config - {exc}")
+            f"is out of date/inconsistent with the global config - {exc}"
+        ) from None
 
     queue: Deque[RemoteCleanQueueTuple] = deque()
     remote_clean_cmd = partial(

cylc/flow/command_validation.py

@@ -77,7 +77,7 @@ def flow_opts(flows: List[str], flow_wait: bool) -> None:
             try:
                 int(val)
             except ValueError:
-                raise InputError(ERR_OPT_FLOW_VAL)
+                raise InputError(ERR_OPT_FLOW_VAL) from None
 
     if flow_wait and flows[0] in {FLOW_NEW, FLOW_NONE}:
         raise InputError(ERR_OPT_FLOW_WAIT)

cylc/flow/commands.py

@@ -211,7 +211,7 @@ async def stop(
         try:
             mode = StopMode(mode)
         except ValueError:
-            raise CommandFailedError(f"Invalid stop mode: '{mode}'")
+            raise CommandFailedError(f"Invalid stop mode: '{mode}'") from None
         schd._set_stop(mode)
         if mode is StopMode.REQUEST_KILL:
             schd.time_next_kill = time()
@@ -308,7 +308,7 @@ async def set_verbosity(schd: 'Scheduler', level: Union[int, str]):
         lvl = int(level)
         LOG.setLevel(lvl)
     except (TypeError, ValueError) as exc:
-        raise CommandFailedError(exc)
+        raise CommandFailedError(exc) from None
     cylc.flow.flags.verbosity = log_level_to_verbosity(lvl)
     yield