diff --git a/cylc/flow/__init__.py b/cylc/flow/__init__.py index bc9f24aa573..001d55ef05e 100644 --- a/cylc/flow/__init__.py +++ b/cylc/flow/__init__.py @@ -58,11 +58,9 @@ def environ_init(): def iter_entry_points(entry_point_name): """Iterate over Cylc entry points.""" - import pkg_resources + from importlib_metadata import entry_points yield from ( entry_point - for entry_point in pkg_resources.iter_entry_points(entry_point_name) - # Filter out the cylc namespace as it should be empty. - # All cylc packages should take the form cylc- - if entry_point.dist.key != 'cylc' + # for entry_point in entry_points()[entry_point_name] + for entry_point in entry_points().select(group=entry_point_name) ) diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py index 86c639af113..33fe2729c61 100644 --- a/cylc/flow/cfgspec/globalcfg.py +++ b/cylc/flow/cfgspec/globalcfg.py @@ -22,7 +22,7 @@ from typing import List, Optional, Tuple, Any, Union from contextlib import suppress -from pkg_resources import parse_version +from packaging.version import parse as parse_version, Version from cylc.flow import LOG from cylc.flow import __version__ as CYLC_VERSION @@ -1858,8 +1858,7 @@ def get_version_hierarchy(version: str) -> List[str]: ['', '8', '8.0', '8.0.1', '8.0.1a2', '8.0.1a2.dev'] """ - smart_ver: Any = parse_version(version) - # (No type anno. yet for Version in pkg_resources.extern.packaging.version) + smart_ver: Version = parse_version(version) base = [str(i) for i in smart_ver.release] hierarchy = [''] hierarchy += ['.'.join(base[:i]) for i in range(1, len(base) + 1)] diff --git a/cylc/flow/network/scan.py b/cylc/flow/network/scan.py index 54222a510c7..c2202f3f31e 100644 --- a/cylc/flow/network/scan.py +++ b/cylc/flow/network/scan.py @@ -50,10 +50,8 @@ import re from typing import AsyncGenerator, Dict, Iterable, List, Optional, Tuple, Union -from pkg_resources import ( - parse_requirements, - parse_version -) +from packaging.version import parse as parse_version +from packaging.specifiers import SpecifierSet from cylc.flow import LOG from cylc.flow.async_util import ( @@ -354,11 +352,7 @@ async def validate_contact_info(flow): def parse_requirement(requirement_string): """Parse a requirement from a requirement string.""" - # we have to give the requirement a name but what we call it doesn't - # actually matter - for req in parse_requirements(f'x {requirement_string}'): - # there should only be one requirement - return (req,), {} + return (SpecifierSet(requirement_string),), {} @pipe(preproc=parse_requirement) @@ -373,7 +367,7 @@ async def cylc_version(flow, requirement): flow (dict): Flow information dictionary, provided by scan through the pipe. requirement (str): - Requirement specifier in pkg_resources format e.g. ``> 8, < 9`` + Requirement specifier in PEP 440 format e.g. ``> 8, < 9`` """ return parse_version(flow[ContactFileFields.VERSION]) in requirement @@ -391,7 +385,7 @@ async def api_version(flow, requirement): flow (dict): Flow information dictionary, provided by scan through the pipe. requirement (str): - Requirement specifier in pkg_resources format e.g. ``> 8, < 9`` + Requirement specifier in PEP 440 format e.g. ``> 8, < 9`` """ return parse_version(flow[ContactFileFields.API]) in requirement diff --git a/cylc/flow/parsec/fileparse.py b/cylc/flow/parsec/fileparse.py index f4f8d723f0b..42e8a4aa40b 100644 --- a/cylc/flow/parsec/fileparse.py +++ b/cylc/flow/parsec/fileparse.py @@ -277,7 +277,7 @@ def process_plugins(fpath, opts): try: # If you want it to work on sourcedirs you need to get the options # to here. - plugin_result = entry_point.resolve()( + plugin_result = entry_point.load()( srcdir=fpath, opts=opts ) except Exception as exc: diff --git a/cylc/flow/scheduler_cli.py b/cylc/flow/scheduler_cli.py index 5c58b1eddba..e2f2d98f46a 100644 --- a/cylc/flow/scheduler_cli.py +++ b/cylc/flow/scheduler_cli.py @@ -25,7 +25,7 @@ import sys from typing import TYPE_CHECKING -from pkg_resources import parse_version +from packaging.version import parse as parse_version from cylc.flow import LOG, __version__ from cylc.flow.exceptions import ( diff --git a/cylc/flow/scripts/completion_server.py b/cylc/flow/scripts/completion_server.py index 2ac87f87b1b..ead311955da 100644 --- a/cylc/flow/scripts/completion_server.py +++ b/cylc/flow/scripts/completion_server.py @@ -46,10 +46,8 @@ import sys import typing as t -from pkg_resources import ( - parse_requirements, - parse_version -) +from packaging.version import parse as parse_version +from packaging.specifiers import SpecifierSet from cylc.flow.cfgspec.glbl_cfg import glbl_cfg from cylc.flow.id import tokenise, IDTokens, Tokens @@ -74,7 +72,7 @@ # set the compatibility range for completion scripts with this server # I.E. if we change the server interface, change this compatibility range. # User's will be presented with an upgrade notice if this happens. -REQUIRED_SCRIPT_VERSION = 'completion-script >=1.0.0, <2.0.0' +REQUIRED_SCRIPT_VERSION = '>=1.0.0, <2.0.0' # register the psudo "help" and "version" commands COMMAND_LIST = list(COMMANDS) + ['help', 'version'] @@ -317,7 +315,7 @@ def list_options(command: str) -> t.List[str]: if command in COMMAND_OPTION_MAP: return COMMAND_OPTION_MAP[command] try: - entry_point = COMMANDS[command].resolve() + entry_point = COMMANDS[command].load() except KeyError: return [] parser = entry_point.parser_function() @@ -637,15 +635,13 @@ def check_completion_script_compatibility( # check that the installed completion script is compabile with this # completion server version - for requirement in parse_requirements(REQUIRED_SCRIPT_VERSION): - # NOTE: there's only one requirement but we have to iterate to get it - if installed_version not in requirement: - is_compatible = False - print( - f'The Cylc {completion_lang} script needs to be updated to' - ' work with this version of Cylc.', - file=sys.stderr, - ) + if installed_version not in SpecifierSet(REQUIRED_SCRIPT_VERSION): + is_compatible = False + print( + f'The Cylc {completion_lang} script needs to be updated to' + ' work with this version of Cylc.', + file=sys.stderr, + ) # check for completion script updates if installed_version < current_version: diff --git a/cylc/flow/scripts/cylc.py b/cylc/flow/scripts/cylc.py index 60abd1913b1..db595584b20 100644 --- a/cylc/flow/scripts/cylc.py +++ b/cylc/flow/scripts/cylc.py @@ -43,12 +43,29 @@ def pythonpath_manip(): pythonpath_manip() +if sys.version_info[:2] > (3, 11): + from importlib.metadata import ( + entry_points, + files, + ) +else: + # BACK COMPAT: importlib_metadata + # importlib.metadata was added in Python 3.8. The required interfaces + # were completed by 3.12. For lower versions we must use the + # importlib_metadata backport. + # FROM: Python 3.7 + # TO: Python: 3.12 + from importlib_metadata import ( + entry_points, + files, + ) + import argparse from contextlib import contextmanager +import os from typing import Iterator, NoReturn, Optional, Tuple from ansimarkup import parse as cparse -import pkg_resources from cylc.flow import __version__, iter_entry_points from cylc.flow.option_parsers import ( @@ -306,9 +323,9 @@ def execute_cmd(cmd: str, *args: str) -> NoReturn: args: Command line arguments to pass to that command. """ - entry_point: pkg_resources.EntryPoint = COMMANDS[cmd] + entry_point = COMMANDS[cmd] try: - entry_point.resolve()(*args) + entry_point.load()(*args) except ModuleNotFoundError as exc: msg = handle_missing_dependency(entry_point, exc) print(msg, file=sys.stderr) @@ -417,18 +434,10 @@ def print_id_help(): def print_license() -> None: - try: - from importlib.metadata import files - except ImportError: - # BACK COMPAT: importlib_metadata - # importlib.metadata was added in Python 3.8 - # FROM: Python 3.7 - # TO: Python: 3.8 - from importlib_metadata import files # type: ignore[no-redef] - license_file = next(filter( - lambda f: f.name == 'COPYING', files('cylc-flow') - )) - print(license_file.read_text()) + for file in files('cylc-flow') or []: + if file.name == 'COPYING': + print(file.read_text()) + return def print_command_list(commands=None, indent=0): @@ -467,54 +476,55 @@ def cli_version(long_fmt=False): """Wrapper for get_version.""" print(get_version(long_fmt)) if long_fmt: - print(list_plugins()) + print(cparse(list_plugins())) sys.exit(0) def list_plugins(): - entry_point_names = [ - entry_point_name - for entry_point_name - in pkg_resources.get_entry_map('cylc-flow').keys() - if entry_point_name.startswith('cylc.') - ] - - entry_point_groups = { - entry_point_name: [ - entry_point - for entry_point - in iter_entry_points(entry_point_name) - if not entry_point.module_name.startswith('cylc.flow') - ] - for entry_point_name in entry_point_names - } - - dists = { - entry_point.dist - for entry_points in entry_point_groups.values() - for entry_point in entry_points - } - - lines = [] - if dists: - lines.append('\nPlugins:') - maxlen1 = max(len(dist.project_name) for dist in dists) + 2 - maxlen2 = max(len(dist.version) for dist in dists) + 2 - for dist in dists: - lines.append( - f' {dist.project_name.ljust(maxlen1)}' - f' {dist.version.ljust(maxlen2)}' - f' {dist.module_path}' - ) - - lines.append('\nEntry Points:') - for entry_point_name, entry_points in entry_point_groups.items(): - if entry_points: - lines.append(f' {entry_point_name}:') - for entry_point in entry_points: - lines.append(f' {entry_point}') - - return '\n'.join(lines) + from cylc.flow.terminal import DIM, format_grid + # go through all Cylc entry points + _dists = set() + __entry_points = {} + for entry_point in entry_points(): + if ( + # all Cylc entry points are under the "cylc" namespace + entry_point.group.startswith('cylc.') + # don't list cylc-flow entry-points (i.e. built-ins) + and not entry_point.value.startswith('cylc.flow') + ): + _dists.add(entry_point.dist) + __entry_points.setdefault(entry_point.group, []) + __entry_points[entry_point.group].append(entry_point) + + # list all the distriutions which provide Cylc entry points + _plugins = [] + for dist in _dists: + _plugins.append(( + '', + f'{dist.name}', + dist.version, + f'<{DIM}>{dist.locate_file("__init__.py").parent}', + )) + + # list all of the entry points by "group" (e.g. "cylc.command") + _entry_points = [] + for group, points in sorted(__entry_points.items()): + _entry_points.append((f' {group}:', '', '')) + for entry_point in points: + _entry_points.append(( + f' {entry_point.name}', + f'{entry_point.dist.name}', + f'<{DIM}>{entry_point.value}', + )) + + return '\n'.join(( + '\nPlugins:', + *format_grid(_plugins), + '\nEntry Points:', + *format_grid( + _entry_points + ), + )) @contextmanager @@ -686,7 +696,7 @@ def main(): def handle_missing_dependency( - entry_point: pkg_resources.EntryPoint, + entry_point, err: ModuleNotFoundError ) -> str: """Return a suitable error message for a missing optional dependency. @@ -698,12 +708,8 @@ def handle_missing_dependency( Re-raises the given ModuleNotFoundError if it is unexpected. """ - try: - # Check for missing optional dependencies - entry_point.require() - except pkg_resources.DistributionNotFound as exc: - # Confirmed missing optional dependencies - return f"cylc {entry_point.name}: {exc}" - else: - # Error not due to missing optional dependencies; this is unexpected - raise err + msg = f'"cylc {entry_point.name}" requires "{entry_point.dist.name}' + if entry_point.extras: + msg += f'[{",".join(entry_point.extras)}]' + msg += f'"\n\n{err.__class__.__name__}: {err}' + return msg diff --git a/cylc/flow/scripts/install.py b/cylc/flow/scripts/install.py index 45010e4c714..be9583137e9 100755 --- a/cylc/flow/scripts/install.py +++ b/cylc/flow/scripts/install.py @@ -301,7 +301,7 @@ def install( 'cylc.pre_configure' ): try: - entry_point.resolve()(srcdir=source, opts=opts) + entry_point.load()(srcdir=source, opts=opts) except Exception as exc: # NOTE: except Exception (purposefully vague) # this is to separate plugin from core Cylc errors @@ -329,7 +329,7 @@ def install( 'cylc.post_install' ): try: - entry_point.resolve()( + entry_point.load()( srcdir=source_dir, opts=opts, rundir=str(rundir) diff --git a/cylc/flow/scripts/reinstall.py b/cylc/flow/scripts/reinstall.py index c3b48b23c74..5e1c75056aa 100644 --- a/cylc/flow/scripts/reinstall.py +++ b/cylc/flow/scripts/reinstall.py @@ -338,7 +338,7 @@ def pre_configure(opts: 'Values', src_dir: Path) -> None: 'cylc.pre_configure' ): try: - entry_point.resolve()(srcdir=src_dir, opts=opts) + entry_point.load()(srcdir=src_dir, opts=opts) except Exception as exc: # NOTE: except Exception (purposefully vague) # this is to separate plugin from core Cylc errors @@ -355,7 +355,7 @@ def post_install(opts: 'Values', src_dir: Path, run_dir: Path) -> None: 'cylc.post_install' ): try: - entry_point.resolve()( + entry_point.load()( srcdir=src_dir, opts=opts, rundir=str(run_dir) diff --git a/cylc/flow/terminal.py b/cylc/flow/terminal.py index bc45f6d65d3..e848691252e 100644 --- a/cylc/flow/terminal.py +++ b/cylc/flow/terminal.py @@ -94,6 +94,48 @@ def print_contents(contents, padding=5, char='.', indent=0): print(f'{indent} {" " * title_width}{" " * padding}{line}') +def format_grid(rows, gutter=2): + """Format gridded text. + + This takes a 2D table of text and formats it to the maximum width of each + column and adds a bit of space between them. + + Args: + rows: + 2D list containing the text to format. + gutter: + The width of the gutter between columns. + + Examples: + >>> format_grid([ + ... ['a', 'b', 'ccccc'], + ... ['ddddd', 'e', 'f'], + ... ]) + ['a b ccccc ', + 'ddddd e f '] + + >>> format_grid([]) + [] + + """ + if not rows: + return rows + templ = [ + '{col:%d}' % (max( + len(row[ind]) + for row in rows + ) + gutter) + for ind in range(len(rows[0])) + ] + lines = [] + for row in rows: + ret = '' + for ind, col in enumerate(row): + ret += templ[ind].format(col=col) + lines.append(ret) + return lines + + def supports_color(): """Determine if running in a terminal which supports color. diff --git a/cylc/flow/workflow_db_mgr.py b/cylc/flow/workflow_db_mgr.py index 091d1712429..e9402ba058b 100644 --- a/cylc/flow/workflow_db_mgr.py +++ b/cylc/flow/workflow_db_mgr.py @@ -25,7 +25,6 @@ import json import os -from pkg_resources import parse_version from shutil import copy, rmtree from sqlite3 import OperationalError from tempfile import mkstemp @@ -33,6 +32,8 @@ Any, AnyStr, Dict, List, Optional, Set, TYPE_CHECKING, Tuple, Union ) +from packaging.version import parse as parse_version + from cylc.flow import LOG from cylc.flow.broadcast_report import get_broadcast_change_iter from cylc.flow.rundb import CylcWorkflowDAO diff --git a/setup.cfg b/setup.cfg index 6581f341571..355204b481c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -69,12 +69,11 @@ install_requires = jinja2==3.0.* metomi-isodatetime>=1!3.0.0,<1!3.2.0 # Constrain protobuf version for compatible Scheduler-UIS comms across hosts + packaging protobuf>=4.21.2,<4.22.0 psutil>=5.6.0 pyzmq>=22 - # https://github.com/pypa/setuptools/issues/3802 - setuptools>=49,!=67.* - importlib_metadata; python_version < "3.8" + importlib_metadata>=5.0; python_version < "3.12" urwid==2.* # unpinned transient dependencies used for type checking rx @@ -127,7 +126,6 @@ tests = # Type annotation stubs # http://mypy-lang.blogspot.com/2021/05/the-upcoming-switch-to-modular-typeshed.html types-Jinja2>=0.1.3 - types-pkg_resources>=0.1.2 types-protobuf>=0.1.10 types-six>=0.1.6 typing-extensions>=4 diff --git a/tests/integration/test_install.py b/tests/integration/test_install.py index cbac55c5361..f681c5a2a01 100644 --- a/tests/integration/test_install.py +++ b/tests/integration/test_install.py @@ -162,7 +162,7 @@ def test_install_gets_back_compat_mode_for_plugins( class failIfDeprecated: """A fake Cylc Plugin entry point""" @staticmethod - def resolve(): + def load(): return failIfDeprecated.raiser @staticmethod diff --git a/tests/unit/plugins/test_pre_configure.py b/tests/unit/plugins/test_pre_configure.py index 717648ea594..e9f91054e12 100644 --- a/tests/unit/plugins/test_pre_configure.py +++ b/tests/unit/plugins/test_pre_configure.py @@ -31,7 +31,7 @@ def __init__(self, fcn): self.name = fcn.__name__ self.fcn = fcn - def resolve(self): + def load(self): return self.fcn diff --git a/tests/unit/scripts/test_completion_server.py b/tests/unit/scripts/test_completion_server.py index d50e46b7039..186e13b7272 100644 --- a/tests/unit/scripts/test_completion_server.py +++ b/tests/unit/scripts/test_completion_server.py @@ -398,18 +398,17 @@ def test_list_options(monkeypatch): assert list_options('zz9+za') == [] # patch the logic to turn off the auto_add behaviour of CylcOptionParser - def _resolve(): - def _parser_function(): - parser = get_option_parser() - del parser.auto_add - return parser - - return SimpleNamespace(parser_function=_parser_function) - - monkeypatch.setattr( - COMMANDS['trigger'], - 'resolve', - _resolve + class EntryPoint: + def load(self): + def _parser_function(): + parser = get_option_parser() + del parser.auto_add + return parser + return SimpleNamespace(parser_function=_parser_function) + monkeypatch.setitem( + COMMANDS, + 'trigger', + EntryPoint(), ) # with auto_add turned off the --color option should be absent @@ -674,7 +673,7 @@ def _get_current_completion_script_version(_script, lang): # set the completion script compatibility range to >=1.0.0, <2.0.0 monkeypatch.setattr( 'cylc.flow.scripts.completion_server.REQUIRED_SCRIPT_VERSION', - 'completion-script >=1.0.0, <2.0.0', + '>=1.0.0, <2.0.0', ) monkeypatch.setattr( 'cylc.flow.scripts.completion_server' diff --git a/tests/unit/scripts/test_cylc.py b/tests/unit/scripts/test_cylc.py index 9f461367377..4b844dafdc7 100644 --- a/tests/unit/scripts/test_cylc.py +++ b/tests/unit/scripts/test_cylc.py @@ -16,7 +16,6 @@ # along with this program. If not, see . import os -import pkg_resources import sys from types import SimpleNamespace from typing import Callable @@ -32,12 +31,9 @@ @pytest.fixture def mock_entry_points(monkeypatch: pytest.MonkeyPatch): """Mock a range of entry points.""" - def _resolve_fail(*args, **kwargs): + def _load_fail(*args, **kwargs): raise ModuleNotFoundError('foo') - def _require_fail(*args, **kwargs): - raise pkg_resources.DistributionNotFound('foo', ['my_extras']) - def _resolve_ok(*args, **kwargs): return Mock() @@ -50,23 +46,17 @@ def _mocked_entry_points(include_bad: bool = False): 'good': SimpleNamespace( name='good', module_name='os.path', - resolve=_resolve_ok, - require=_require_ok, + load=_resolve_ok, + extras=[], + dist=SimpleNamespace(name='a'), ), # an entry point with optional dependencies missing: 'missing': SimpleNamespace( name='missing', module_name='not.a.python.module', # force an import error - resolve=_resolve_fail, - require=_require_fail, - ), - # an entry point with optional dependencies missing, but they - # are not needed for the core functionality of the entry point: - 'partial': SimpleNamespace( - name='partial', - module_name='os.path', - resolve=_resolve_ok, - require=_require_fail, + load=_load_fail, + extras=[], + dist=SimpleNamespace(name='foo'), ), } if include_bad: @@ -75,8 +65,10 @@ def _mocked_entry_points(include_bad: bool = False): commands['bad'] = SimpleNamespace( name='bad', module_name='not.a.python.module', - resolve=_resolve_fail, + load=_load_fail, require=_require_ok, + extras=[], + dist=SimpleNamespace(name='d'), ) monkeypatch.setattr('cylc.flow.scripts.cylc.COMMANDS', commands) @@ -90,14 +82,13 @@ def test_iter_commands(mock_entry_points): """ mock_entry_points() commands = list(iter_commands()) - assert [i[0] for i in commands] == ['good', 'partial'] + assert [i[0] for i in commands] == ['good'] def test_iter_commands_bad(mock_entry_points): - """Test listing commands fails if there is an unexpected import error.""" + """Test listing commands doesn't fail on import error.""" mock_entry_points(include_bad=True) - with pytest.raises(ModuleNotFoundError): - list(iter_commands()) + list(iter_commands()) def test_execute_cmd( @@ -125,19 +116,16 @@ def test_execute_cmd( execute_cmd('missing') capexit.assert_any_call(1) assert capsys.readouterr().err.strip() == ( - "cylc missing: The 'foo' distribution was not found and is" - " required by my_extras" + '"cylc missing" requires "foo"\n\nModuleNotFoundError: foo' ) - # the "partial" entry point should exit 0 - capexit.reset_mock() - execute_cmd('partial') - capexit.assert_called_once_with() - assert capsys.readouterr().err == '' + # the "bad" entry point should log an error + execute_cmd('bad') + capexit.assert_any_call(1) - # the "bad" entry point should raise an exception - with pytest.raises(ModuleNotFoundError): - execute_cmd('bad') + stderr = capsys.readouterr().err.strip() + assert '"cylc bad" requires "d"' in stderr + assert 'ModuleNotFoundError: foo' in stderr def test_pythonpath_manip(monkeypatch):