speed up import time (#5770)

* Don't import modules for type checking purposes.
* Move some light weight functions out of a heavy weight module to
  reduce the import penalty.
* Make a couple of heavy weight imports dynamic to avoid importing them
  when not necessary.
* This reduces the import time of cylc.flow.scripts.message by ~66%.
* This reduces the task runtime (`script=true`) by ~1s.

cylc/flow/config.py

@@ -68,7 +68,7 @@
 import cylc.flow.flags
 from cylc.flow.graph_parser import GraphParser
 from cylc.flow.listify import listify
-from cylc.flow.option_parsers import verbosity_to_env
+from cylc.flow.log_level import verbosity_to_env
 from cylc.flow.graphnode import GraphNodeParser
 from cylc.flow.param_expand import NameExpander
 from cylc.flow.parsec.exceptions import ItemNotFoundError

cylc/flow/exceptions.py

@@ -26,11 +26,14 @@
     Tuple,
     Type,
     Union,
+    TYPE_CHECKING,
 )
 
-from cylc.flow.subprocctx import SubFuncContext
 from cylc.flow.util import format_cmd
 
+if TYPE_CHECKING:
+    from cylc.flow.subprocctx import SubFuncContext
+
 
 class CylcError(Exception):
     """Generic exception for Cylc errors.
@@ -198,7 +201,7 @@ def __init__(
         message: str,
         platform_name: str,
         *,
-        ctx: Optional[SubFuncContext] = None,
+        ctx: 'Optional[SubFuncContext]' = None,
         cmd: Optional[Union[str, Iterable]] = None,
         ret_code: Optional[int] = None,
         out: Optional[str] = None,

cylc/flow/flow_mgr.py

@@ -16,11 +16,14 @@
 
 """Manage flow counter and flow metadata."""
 
-from typing import Dict, Set, Optional
+from typing import Dict, Set, Optional, TYPE_CHECKING
 import datetime
 
 from cylc.flow import LOG
-from cylc.flow.workflow_db_mgr import WorkflowDatabaseManager
+
+
+if TYPE_CHECKING:
+    from cylc.flow.workflow_db_mgr import WorkflowDatabaseManager
 
 
 FlowNums = Set[int]

cylc/flow/id_cli.py

@@ -31,11 +31,6 @@
     upgrade_legacy_ids,
 )
 from cylc.flow.pathutil import EXPLICIT_RELATIVE_PATH_REGEX
-from cylc.flow.network.scan import (
-    filter_name,
-    is_active,
-    scan,
-)
 from cylc.flow.workflow_files import (
     check_flow_file,
     detect_both_flow_and_suite,
@@ -487,6 +482,12 @@ async def _expand_workflow_tokens_impl(tokens, match_active=True):
             'currently supported.'
         )
 
+    # import only when needed to avoid slowing CLI unnecessarily
+    from cylc.flow.network.scan import (
+        filter_name,
+        is_active,
+        scan,
+    )
     # construct the pipe
     pipe = scan | filter_name(fnmatch.translate(tokens['workflow']))
     if match_active is not None:

cylc/flow/job_file.py

@@ -25,7 +25,7 @@
 from cylc.flow import __version__ as CYLC_VERSION
 from cylc.flow.job_runner_mgr import JobRunnerManager
 import cylc.flow.flags
-from cylc.flow.option_parsers import verbosity_to_env
+from cylc.flow.log_level import verbosity_to_env
 from cylc.flow.config import interpolate_template, ParamExpandError
 
 # the maximum number of task dependencies which Cylc will list before

cylc/flow/job_runner_handlers/documentation.py

@@ -22,13 +22,16 @@
 not intended to be subclassed.
 """
 
-import re
 from typing import (
     Iterable,
     List,
     Tuple,
+    TYPE_CHECKING,
 )
 
+if TYPE_CHECKING:
+    import re
+
 
 class ExampleHandler():
     """Documentation for writing job runner handlers.
@@ -257,15 +260,15 @@ class QSUBHandler(PBSHandler):
 
     """
 
-    REC_ID_FROM_SUBMIT_OUT: re.Pattern
+    REC_ID_FROM_SUBMIT_OUT: 're.Pattern'
     """Regular expression to extract job IDs from submission stderr.
 
     A regular expression (compiled) to extract the job "id" from the standard
     output or standard error of the job submission command.
 
     """
 
-    REC_ID_FROM_SUBMIT_ERR: re.Pattern
+    REC_ID_FROM_SUBMIT_ERR: 're.Pattern'
     """Regular expression to extract job IDs from submission stderr.
 
     See :py:attr:`ExampleHandler.REC_ID_FROM_SUBMIT_OUT`.

cylc/flow/log_level.py

@@ -0,0 +1,116 @@
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Utilities for configuring logging level via the CLI."""
+
+import logging
+from typing import List, Dict, Union, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import os
+
+
+def verbosity_to_log_level(verb: int) -> int:
+    """Convert Cylc verbosity to log severity level."""
+    if verb < 0:
+        return logging.WARNING
+    if verb > 0:
+        return logging.DEBUG
+    return logging.INFO
+
+
+def log_level_to_verbosity(lvl: int) -> int:
+    """Convert log severity level to Cylc verbosity.
+
+    Examples:
+        >>> log_level_to_verbosity(logging.NOTSET)
+        2
+        >>> log_level_to_verbosity(logging.DEBUG)
+        1
+        >>> log_level_to_verbosity(logging.INFO)
+        0
+        >>> log_level_to_verbosity(logging.WARNING)
+        -1
+        >>> log_level_to_verbosity(logging.ERROR)
+        -1
+    """
+    if lvl < logging.DEBUG:
+        return 2
+    if lvl < logging.INFO:
+        return 1
+    if lvl == logging.INFO:
+        return 0
+    return -1
+
+
+def verbosity_to_opts(verb: int) -> List[str]:
+    """Convert Cylc verbosity to the CLI opts required to replicate it.
+
+    Examples:
+        >>> verbosity_to_opts(0)
+        []
+        >>> verbosity_to_opts(-2)
+        ['-q', '-q']
+        >>> verbosity_to_opts(2)
+        ['-v', '-v']
+
+    """
+    return [
+        '-q'
+        for _ in range(verb, 0)
+    ] + [
+        '-v'
+        for _ in range(0, verb)
+    ]
+
+
+def verbosity_to_env(verb: int) -> Dict[str, str]:
+    """Convert Cylc verbosity to the env vars required to replicate it.
+
+    Examples:
+        >>> verbosity_to_env(0)
+        {'CYLC_VERBOSE': 'false', 'CYLC_DEBUG': 'false'}
+        >>> verbosity_to_env(1)
+        {'CYLC_VERBOSE': 'true', 'CYLC_DEBUG': 'false'}
+        >>> verbosity_to_env(2)
+        {'CYLC_VERBOSE': 'true', 'CYLC_DEBUG': 'true'}
+
+    """
+    return {
+        'CYLC_VERBOSE': str((verb > 0)).lower(),
+        'CYLC_DEBUG': str((verb > 1)).lower(),
+    }
+
+
+def env_to_verbosity(env: 'Union[Dict, os._Environ]') -> int:
+    """Extract verbosity from environment variables.
+
+    Examples:
+        >>> env_to_verbosity({})
+        0
+        >>> env_to_verbosity({'CYLC_VERBOSE': 'true'})
+        1
+        >>> env_to_verbosity({'CYLC_DEBUG': 'true'})
+        2
+        >>> env_to_verbosity({'CYLC_DEBUG': 'TRUE'})
+        2
+
+    """
+    return (
+        2 if env.get('CYLC_DEBUG', '').lower() == 'true'
+        else 1 if env.get('CYLC_VERBOSE', '').lower() == 'true'
+        else 0
+    )

cylc/flow/network/server.py

@@ -21,7 +21,6 @@
 from time import sleep
 from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
 
-from graphql.execution import ExecutionResult
 from graphql.execution.executors.asyncio import AsyncioExecutor
 import zmq
 from zmq.auth.thread import ThreadAuthenticator
@@ -41,6 +40,7 @@
 
 if TYPE_CHECKING:
     from cylc.flow.scheduler import Scheduler
+    from graphql.execution import ExecutionResult
 
 
 # maps server methods to the protobuf message (for client/UIS import)
@@ -368,7 +368,7 @@ def graphql(
             object: Execution result, or a list with errors.
         """
         try:
-            executed: ExecutionResult = schema.execute(
+            executed: 'ExecutionResult' = schema.execute(
                 request_string,
                 variable_values=variables,
                 context_value={

cylc/flow/option_parsers.py

@@ -33,7 +33,7 @@
 
 import sys
 from textwrap import dedent
-from typing import Any, Dict, Optional, List, Tuple, Union
+from typing import Any, Dict, Optional, List, Tuple
 
 from cylc.flow import LOG
 from cylc.flow.terminal import supports_color, DIM
@@ -42,6 +42,10 @@
     CylcLogFormatter,
     setup_segregated_log_streams,
 )
+from cylc.flow.log_level import (
+    env_to_verbosity,
+    verbosity_to_log_level
+)
 
 WORKFLOW_ID_ARG_DOC = ('WORKFLOW', 'Workflow ID')
 WORKFLOW_ID_MULTI_ARG_DOC = ('WORKFLOW ...', 'Workflow ID(s)')
@@ -177,99 +181,6 @@ def format_help_headings(string):
     )
 
 
-def verbosity_to_log_level(verb: int) -> int:
-    """Convert Cylc verbosity to log severity level."""
-    if verb < 0:
-        return logging.WARNING
-    if verb > 0:
-        return logging.DEBUG
-    return logging.INFO
-
-
-def log_level_to_verbosity(lvl: int) -> int:
-    """Convert log severity level to Cylc verbosity.
-
-    Examples:
-        >>> log_level_to_verbosity(logging.NOTSET)
-        2
-        >>> log_level_to_verbosity(logging.DEBUG)
-        1
-        >>> log_level_to_verbosity(logging.INFO)
-        0
-        >>> log_level_to_verbosity(logging.WARNING)
-        -1
-        >>> log_level_to_verbosity(logging.ERROR)
-        -1
-    """
-    if lvl < logging.DEBUG:
-        return 2
-    if lvl < logging.INFO:
-        return 1
-    if lvl == logging.INFO:
-        return 0
-    return -1
-
-
-def verbosity_to_opts(verb: int) -> List[str]:
-    """Convert Cylc verbosity to the CLI opts required to replicate it.
-
-    Examples:
-        >>> verbosity_to_opts(0)
-        []
-        >>> verbosity_to_opts(-2)
-        ['-q', '-q']
-        >>> verbosity_to_opts(2)
-        ['-v', '-v']
-
-    """
-    return [
-        '-q'
-        for _ in range(verb, 0)
-    ] + [
-        '-v'
-        for _ in range(0, verb)
-    ]
-
-
-def verbosity_to_env(verb: int) -> Dict[str, str]:
-    """Convert Cylc verbosity to the env vars required to replicate it.
-
-    Examples:
-        >>> verbosity_to_env(0)
-        {'CYLC_VERBOSE': 'false', 'CYLC_DEBUG': 'false'}
-        >>> verbosity_to_env(1)
-        {'CYLC_VERBOSE': 'true', 'CYLC_DEBUG': 'false'}
-        >>> verbosity_to_env(2)
-        {'CYLC_VERBOSE': 'true', 'CYLC_DEBUG': 'true'}
-
-    """
-    return {
-        'CYLC_VERBOSE': str((verb > 0)).lower(),
-        'CYLC_DEBUG': str((verb > 1)).lower(),
-    }
-
-
-def env_to_verbosity(env: Union[Dict, os._Environ]) -> int:
-    """Extract verbosity from environment variables.
-
-    Examples:
-        >>> env_to_verbosity({})
-        0
-        >>> env_to_verbosity({'CYLC_VERBOSE': 'true'})
-        1
-        >>> env_to_verbosity({'CYLC_DEBUG': 'true'})
-        2
-        >>> env_to_verbosity({'CYLC_DEBUG': 'TRUE'})
-        2
-
-    """
-    return (
-        2 if env.get('CYLC_DEBUG', '').lower() == 'true'
-        else 1 if env.get('CYLC_VERBOSE', '').lower() == 'true'
-        else 0
-    )
-
-
 class CylcOption(Option):
     """Optparse option which adds a decrement action."""