diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 41c5fe05c2..6800263b6b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -28,9 +28,6 @@ updates: # Allow both direct and indirect updates for all packages - dependency-type: "all" ignore: - # #6668 - - dependency-name: "setuptools" - update-types: ["version-update:semver-major"] - dependency-name: "pytest-inmanta-extensions" versions: [">=0.0.1"] - dependency-name: "pydantic" diff --git a/changelogs/unreleased/drop-pkg-resources-first-part.yml b/changelogs/unreleased/drop-pkg-resources-first-part.yml new file mode 100644 index 0000000000..a8f3b79bf0 --- /dev/null +++ b/changelogs/unreleased/drop-pkg-resources-first-part.yml @@ -0,0 +1,4 @@ +--- +description: First part of `pkg_resources` library deprecation. +change-type: patch +destination-branches: [iso7] diff --git a/docs/conf.py b/docs/conf.py index 0959468460..e679b93db2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,7 +12,7 @@ # serve to show the default. import importlib.metadata import shutil -import sys, os, pkg_resources, datetime +import sys, os, datetime from importlib.metadata import PackageNotFoundError from sphinx.errors import ConfigError diff --git a/mypy-baseline.txt b/mypy-baseline.txt index b5379aae38..d03828d25a 100644 --- a/mypy-baseline.txt +++ b/mypy-baseline.txt @@ -940,13 +940,8 @@ src/inmanta/execute/scheduler.py:0: error: Missing type parameters for generic t src/inmanta/execute/scheduler.py:0: error: "" has no attribute "attribute" [attr-defined] src/inmanta/execute/scheduler.py:0: error: "" has no attribute "attribute" [attr-defined] src/inmanta/execute/scheduler.py:0: error: "" has no attribute "attribute" [attr-defined] +src/inmanta/env.py:0: error: Argument 1 to "parse_requirements" has incompatible type "Sequence[CanonicalRequirement]"; expected "Sequence[str]" [arg-type] src/inmanta/env.py:0: error: "type[WorkingSet]" has no attribute "_build_master" [attr-defined] -src/inmanta/env.py:0: error: Incompatible types in assignment (expression has type "WorkingSet", variable has type "Iterable[DistInfoDistribution]") [assignment] -src/inmanta/env.py:0: note: Following member(s) of "WorkingSet" have conflicts: -src/inmanta/env.py:0: note: Expected: -src/inmanta/env.py:0: note: def __iter__(self) -> Iterator[DistInfoDistribution] -src/inmanta/env.py:0: note: Got: -src/inmanta/env.py:0: note: def __iter__(self) -> Iterator[Distribution] src/inmanta/env.py:0: error: Incompatible return value type (got "tuple[str | None, Loader | None] | None", expected "tuple[str | None, Loader] | None") [return-value] src/inmanta/env.py:0: error: Item "None" of "ModuleSpec | None" has no attribute "submodule_search_locations" [union-attr] src/inmanta/env.py:0: error: Item "None" of "ModuleSpec | None" has no attribute "submodule_search_locations" [union-attr] @@ -957,7 +952,6 @@ src/inmanta/compiler/__init__.py:0: error: Incompatible types in assignment (exp src/inmanta/compiler/__init__.py:0: error: Incompatible types in assignment (expression has type "Namespace | None", variable has type "Namespace") [assignment] src/inmanta/compiler/__init__.py:0: error: Missing type parameters for generic type "ResultVariable" [type-arg] src/inmanta/module.py:0: error: Missing type parameters for generic type Module [type-arg] -src/inmanta/module.py:0: error: Incompatible return value type (got "SpecifierSet", expected "str") [return-value] src/inmanta/module.py:0: error: Returning Any from function declared to return "Mapping[str, object]" [no-any-return] src/inmanta/module.py:0: error: Argument 1 to "Metadata" has incompatible type "**Mapping[str, object]"; expected "str" [arg-type] src/inmanta/module.py:0: error: Argument 1 to "Metadata" has incompatible type "**Mapping[str, object]"; expected "str | None" [arg-type] @@ -974,18 +968,12 @@ src/inmanta/module.py:0: error: Missing type parameters for generic type Module src/inmanta/module.py:0: error: Missing type parameters for generic type Module [type-arg] src/inmanta/module.py:0: error: Missing type parameters for generic type Module [type-arg] src/inmanta/module.py:0: error: Missing type parameters for generic type Module [type-arg] -src/inmanta/module.py:0: error: Unsupported operand types for in ("Version" and "InmantaModuleRequirement") [operator] src/inmanta/module.py:0: error: Missing type parameters for generic type Module [type-arg] src/inmanta/module.py:0: error: Missing type parameters for generic type Module [type-arg] src/inmanta/module.py:0: error: Missing type parameters for generic type Module [type-arg] src/inmanta/module.py:0: error: Missing type parameters for generic type Module [type-arg] src/inmanta/module.py:0: error: Missing type parameters for generic type Module [type-arg] src/inmanta/module.py:0: error: Missing type parameters for generic type Module [type-arg] -src/inmanta/module.py:0: error: Incompatible return value type (got "pkg_resources._vendored_packaging.version.Version", expected "packaging.version.Version | None") [return-value] -src/inmanta/module.py:0: error: "str" has no attribute "filter" [attr-defined] -src/inmanta/module.py:0: error: Argument 4 to "__best_for_compiler_version" of "ModuleV1" has incompatible type "pkg_resources._vendored_packaging.version.Version"; expected "packaging.version.Version" [arg-type] -src/inmanta/module.py:0: error: Incompatible return value type (got "pkg_resources._vendored_packaging.version.Version", expected "packaging.version.Version | None") [return-value] -src/inmanta/module.py:0: error: Incompatible return value type (got "pkg_resources._vendored_packaging.version.Version", expected "packaging.version.Version | None") [return-value] src/inmanta/moduletool.py:0: error: Skipping analyzing "cookiecutter.main": module is installed, but missing library stubs or py.typed marker [import-untyped] src/inmanta/moduletool.py:0: error: Missing type parameters for generic type "ModuleLike" [type-arg] src/inmanta/moduletool.py:0: error: Missing type parameters for generic type Module [type-arg] diff --git a/setup.py b/setup.py index 57de5e7c92..dea58e132c 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ "pynacl~=1.5", "python-dateutil~=2.0", "pyyaml~=6.0", - "setuptools<71", + "setuptools", "texttable~=1.0", "tornado~=6.0", # lower bound because of ilevkivskyi/typing_inspect#100 diff --git a/src/inmanta/agent/agent.py b/src/inmanta/agent/agent.py index 1ab84219a0..719cc211e3 100644 --- a/src/inmanta/agent/agent.py +++ b/src/inmanta/agent/agent.py @@ -35,8 +35,7 @@ from logging import Logger from typing import Any, Dict, Optional, Union, cast -import pkg_resources - +import inmanta.util from inmanta import const, data, env, protocol from inmanta.agent import config as cfg from inmanta.agent import handler @@ -1460,7 +1459,7 @@ async def _install(self, sources: list[ModuleSource], requirements: Sequence[str await loop.run_in_executor( self.thread_pool, self._env.install_for_config, - list(pkg_resources.parse_requirements(requirements)), + inmanta.util.parse_requirements(requirements), pip_config, ) await loop.run_in_executor(self.thread_pool, self._loader.deploy_version, sources) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 20dac51721..73f36697ca 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -17,6 +17,7 @@ """ import enum +import importlib.metadata import importlib.util import json import logging @@ -40,15 +41,17 @@ from typing import Callable, NamedTuple, Optional, TypeVar import pkg_resources -from pkg_resources import DistInfoDistribution, Distribution, Requirement, WorkingSet +import inmanta.util +import packaging.requirements +import packaging.utils +import packaging.version from inmanta import const from inmanta.ast import CompilerException from inmanta.data.model import LEGACY_PIP_DEFAULT, PipConfig from inmanta.server.bootloader import InmantaBootloader from inmanta.stable_api import stable_api from inmanta.util import strtobool -from packaging import version LOGGER = logging.getLogger(__name__) LOGGER_PIP = logging.getLogger("inmanta.pip") # Use this logger to log pip commands or data related to pip commands. @@ -75,8 +78,8 @@ class VersionConflict: :param owner: The package from which the constraint originates """ - requirement: Requirement - installed_version: Optional[version.Version] = None + requirement: inmanta.util.CanonicalRequirement + installed_version: Optional[packaging.version.Version] = None owner: Optional[str] = None def __str__(self) -> str: @@ -132,7 +135,7 @@ def get_conflicts_string(self) -> Optional[str]: if not self.conflicts: return None msg = "" - for current_conflict in sorted(self.conflicts, key=lambda x: x.requirement.key): + for current_conflict in sorted(self.conflicts, key=lambda x: x.requirement.name): msg += f"\n\t* {current_conflict}" return msg @@ -160,17 +163,17 @@ def get_advice(self) -> Optional[str]: ) -req_list = TypeVar("req_list", Sequence[str], Sequence[Requirement]) +req_list = TypeVar("req_list", Sequence[str], Sequence[inmanta.util.CanonicalRequirement]) class PythonWorkingSet: @classmethod - def _get_as_requirements_type(cls, requirements: req_list) -> Sequence[Requirement]: + def _get_as_requirements_type(cls, requirements: req_list) -> Sequence[inmanta.util.CanonicalRequirement]: """ Convert requirements from Union[Sequence[str], Sequence[Requirement]] to Sequence[Requirement] """ if isinstance(requirements[0], str): - return [Requirement.parse(r) for r in requirements] + return inmanta.util.parse_requirements(requirements) else: return requirements @@ -181,11 +184,11 @@ def are_installed(cls, requirements: req_list) -> bool: """ if not requirements: return True - installed_packages: dict[str, version.Version] = cls.get_packages_in_working_set() + installed_packages: dict[str, packaging.version.Version] = cls.get_packages_in_working_set() def _are_installed_recursive( - reqs: Sequence[Requirement], - seen_requirements: Sequence[Requirement], + reqs: Sequence[inmanta.util.CanonicalRequirement], + seen_requirements: Sequence[inmanta.util.CanonicalRequirement], contained_in_extra: Optional[str] = None, ) -> bool: """ @@ -201,7 +204,8 @@ def _are_installed_recursive( for r in reqs: if r in seen_requirements: continue - # Requirements created by the `Distribution.requires()` method have the extra, the Requirement was created from, + # Requirements created by the `Distribution.requires()` method have the extra, the Requirement was created + # from, # set as a marker. The line below makes sure that the "extra" marker matches. The marker is not set by # `Distribution.requires()` when the package is installed in editable mode, but setting it always doesn't make # the marker evaluation fail. @@ -209,16 +213,19 @@ def _are_installed_recursive( if r.marker and not r.marker.evaluate(environment=environment_marker_evaluation): # The marker of the requirement doesn't apply on this environment continue - if r.key not in installed_packages or str(installed_packages[r.key]) not in r: + if r.name not in installed_packages or not r.specifier.contains(installed_packages[r.name], prereleases=True): return False if r.extras: for extra in r.extras: - distribution: Optional[Distribution] = pkg_resources.working_set.find(r) + distribution: Optional[pkg_resources.Distribution] = pkg_resources.working_set.find( + pkg_resources.Requirement.parse(r.name) + ) if distribution is None: return False - pkgs_required_by_extra: set[Requirement] = set(distribution.requires(extras=(extra,))) - set( - distribution.requires(extras=()) - ) + + pkgs_required_by_extra: set[inmanta.util.CanonicalRequirement] = set( + [inmanta.util.parse_requirement(str(e)) for e in distribution.requires(extras=(extra,))] + ) - set([inmanta.util.parse_requirement(str(e)) for e in distribution.requires(extras=())]) if not _are_installed_recursive( reqs=list(pkgs_required_by_extra), seen_requirements=list(seen_requirements) + list(reqs), @@ -227,18 +234,19 @@ def _are_installed_recursive( return False return True - reqs_as_requirements: Sequence[Requirement] = cls._get_as_requirements_type(requirements) + reqs_as_requirements: Sequence[inmanta.util.CanonicalRequirement] = cls._get_as_requirements_type(requirements) return _are_installed_recursive(reqs_as_requirements, seen_requirements=[]) @classmethod - def get_packages_in_working_set(cls, inmanta_modules_only: bool = False) -> dict[str, version.Version]: + def get_packages_in_working_set(cls, inmanta_modules_only: bool = False) -> dict[str, packaging.version.Version]: """ - Return all packages present in `pkg_resources.working_set` together with the version of the package. + Return all packages (under the canonicalized form) present in `pkg_resources.working_set` together with the version + of the package. :param inmanta_modules_only: Only return inmanta modules from the working set """ return { - dist_info.key: version.Version(dist_info.version) + packaging.utils.canonicalize_name(dist_info.key): packaging.version.Version(dist_info.version) for dist_info in pkg_resources.working_set if not inmanta_modules_only or dist_info.key.startswith(const.MODULE_PKG_NAME_PREFIX) } @@ -258,7 +266,7 @@ def get_dependency_tree(cls, dists: abc.Iterable[str]) -> abc.Set[str]: :param dists: The keys for the distributions to get the dependency tree for. """ # create dict for O(1) lookup - installed_distributions: abc.Mapping[str, Distribution] = { + installed_distributions: abc.Mapping[str, pkg_resources.Distribution] = { dist_info.key: dist_info for dist_info in pkg_resources.working_set } @@ -277,7 +285,10 @@ def _get_tree_recursive_single(acc: abc.Set[str], dist: str) -> abc.Set[str]: # recurse on direct dependencies return _get_tree_recursive( - (requirement.key for requirement in installed_distributions[dist].requires()), + ( + packaging.utils.canonicalize_name(requirement.key) + for requirement in installed_distributions[dist].requires() + ), acc=acc | {dist}, ) @@ -370,7 +381,7 @@ def run_pip_install_command_from_config( cls, python_path: str, config: PipConfig, - requirements: Optional[Sequence[Requirement]] = None, + requirements: Optional[Sequence[packaging.requirements.Requirement]] = None, requirements_files: Optional[list[str]] = None, upgrade: bool = False, upgrade_strategy: PipUpgradeStrategy = PipUpgradeStrategy.ONLY_IF_NEEDED, @@ -602,7 +613,7 @@ def get_env_path_for_python_path(cls, python_path: str) -> str: """ return os.path.dirname(os.path.dirname(python_path)) - def get_installed_packages(self, only_editable: bool = False) -> dict[str, version.Version]: + def get_installed_packages(self, only_editable: bool = False) -> dict[str, packaging.version.Version]: """ Return a list of all installed packages in the site-packages of a python interpreter. @@ -611,11 +622,11 @@ def get_installed_packages(self, only_editable: bool = False) -> dict[str, versi """ cmd = PipCommandBuilder.compose_list_command(self.python_path, format=PipListFormat.json, only_editable=only_editable) output = CommandRunner(LOGGER_PIP).run_command_and_log_output(cmd, stderr=subprocess.DEVNULL, env=os.environ.copy()) - return {r["name"]: version.Version(r["version"]) for r in json.loads(output)} + return {r["name"]: packaging.version.Version(r["version"]) for r in json.loads(output)} def install_for_config( self, - requirements: list[Requirement], + requirements: list[inmanta.util.CanonicalRequirement], config: PipConfig, upgrade: bool = False, constraint_files: Optional[list[str]] = None, @@ -656,7 +667,7 @@ def install_for_config( def install_from_index( self, - requirements: list[Requirement], + requirements: list[inmanta.util.CanonicalRequirement], index_urls: Optional[list[str]] = None, upgrade: bool = False, allow_pre_releases: bool = False, @@ -727,7 +738,7 @@ def install_from_list( use_pip_config was ignored on ISO6 and it still is """ self.install_from_index( - requirements=[Requirement.parse(r) for r in requirements_list], + requirements=inmanta.util.parse_requirements(requirements_list), upgrade=upgrade, upgrade_strategy=upgrade_strategy, use_pip_config=True, @@ -737,24 +748,32 @@ def install_from_list( def get_protected_inmanta_packages(cls) -> list[str]: """ Returns the list of packages that should not be installed/updated by any operation on a Python environment. + This list of packages will be under the canonical form. """ return [ # Protect product packages - "inmanta", - "inmanta-service-orchestrator", + packaging.utils.canonicalize_name("inmanta"), + packaging.utils.canonicalize_name("inmanta-service-orchestrator"), # Protect all server extensions - *(f"inmanta-{ext_name}" for ext_name in InmantaBootloader.get_available_extensions().keys()), + *( + packaging.utils.canonicalize_name(f"inmanta-{ext_name}") + for ext_name in InmantaBootloader.get_available_extensions().keys() + ), ] @classmethod - def _get_requirements_on_inmanta_package(cls) -> Sequence[Requirement]: + def _get_requirements_on_inmanta_package(cls) -> Sequence[inmanta.util.CanonicalRequirement]: """ Returns the content of the requirement file that should be supplied to each `pip install` invocation to make sure that no Inmanta packages gets overridden. """ protected_inmanta_packages: list[str] = cls.get_protected_inmanta_packages() - workingset: dict[str, version.Version] = PythonWorkingSet.get_packages_in_working_set() - return [Requirement.parse(f"{pkg}=={workingset[pkg]}") for pkg in workingset if pkg in protected_inmanta_packages] + workingset: dict[str, packaging.version.Version] = PythonWorkingSet.get_packages_in_working_set() + return [ + inmanta.util.parse_requirement(requirement=f"{pkg}=={workingset[pkg]}") + for pkg in workingset + if pkg in protected_inmanta_packages + ] class CommandRunner: @@ -848,7 +867,7 @@ def are_installed(self, requirements: req_list) -> bool: def install_for_config( self, - requirements: list[Requirement], + requirements: list[inmanta.util.CanonicalRequirement], config: PipConfig, upgrade: bool = False, constraint_files: Optional[list[str]] = None, @@ -868,7 +887,7 @@ def install_for_config( def get_constraint_violations_for_check( self, strict_scope: Optional[Pattern[str]] = None, - constraints: Optional[list[Requirement]] = None, + constraints: Optional[list[inmanta.util.CanonicalRequirement]] = None, ) -> tuple[set[VersionConflict], set[VersionConflict]]: """ Return the constraint violations that exist in this venv. Returns a tuple of non-strict and strict violations, @@ -876,7 +895,7 @@ def get_constraint_violations_for_check( """ class OwnedRequirement(NamedTuple): - requirement: Requirement + requirement: inmanta.util.CanonicalRequirement owner: Optional[str] = None def is_owned_by(self, owners: abc.Set[str]) -> bool: @@ -884,10 +903,11 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: # all requirements of all packages installed in this environment installed_constraints: abc.Set[OwnedRequirement] = frozenset( - OwnedRequirement(requirement, dist_info.key) + OwnedRequirement(inmanta.util.parse_requirement(requirement=str(requirement)), dist_info.key) for dist_info in pkg_resources.working_set for requirement in dist_info.requires() ) + inmanta_constraints: abc.Set[OwnedRequirement] = frozenset( OwnedRequirement(r, owner="inmanta-core") for r in self._get_requirements_on_inmanta_package() ) @@ -897,30 +917,32 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: all_constraints: abc.Set[OwnedRequirement] = installed_constraints | inmanta_constraints | extra_constraints - full_strict_scope: abc.Set[str] = PythonWorkingSet.get_dependency_tree( + parameters = list( chain( ( [] if strict_scope is None else (dist_info.key for dist_info in pkg_resources.working_set if strict_scope.fullmatch(dist_info.key)) ), - (requirement.requirement.key for requirement in inmanta_constraints), - (requirement.requirement.key for requirement in extra_constraints), + (requirement.requirement.name for requirement in inmanta_constraints), + (requirement.requirement.name for requirement in extra_constraints), ) ) + full_strict_scope: abc.Set[str] = PythonWorkingSet.get_dependency_tree(parameters) - installed_versions: dict[str, version.Version] = PythonWorkingSet.get_packages_in_working_set() + installed_versions: dict[str, packaging.version.Version] = PythonWorkingSet.get_packages_in_working_set() constraint_violations: set[VersionConflict] = set() constraint_violations_strict: set[VersionConflict] = set() for c in all_constraints: requirement = c.requirement - if (requirement.key not in installed_versions or str(installed_versions[requirement.key]) not in requirement) and ( - not requirement.marker or (requirement.marker and requirement.marker.evaluate()) + if requirement.name not in installed_versions or ( + not requirement.specifier.contains(installed_versions[requirement.name], prereleases=True) + and (not requirement.marker or (requirement.marker and requirement.marker.evaluate())) ): version_conflict = VersionConflict( requirement=requirement, - installed_version=installed_versions.get(requirement.key, None), + installed_version=installed_versions.get(requirement.name, None), owner=c.owner, ) if c.is_owned_by(full_strict_scope): @@ -933,7 +955,7 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: def check( self, strict_scope: Optional[Pattern[str]] = None, - constraints: Optional[list[Requirement]] = None, + constraints: Optional[list[inmanta.util.CanonicalRequirement]] = None, ) -> None: """ Check this Python environment for incompatible dependencies in installed packages. @@ -958,7 +980,9 @@ def check( for violation in constraint_violations: LOGGER.warning("%s", violation) - def check_legacy(self, in_scope: Pattern[str], constraints: Optional[list[Requirement]] = None) -> bool: + def check_legacy( + self, in_scope: Pattern[str], constraints: Optional[list[inmanta.util.CanonicalRequirement]] = None + ) -> bool: """ Check this Python environment for incompatible dependencies in installed packages. This method is a legacy method in the sense that it has been replaced with a more correct check defined in self.check(). This method is invoked @@ -975,20 +999,21 @@ def check_legacy(self, in_scope: Pattern[str], constraints: Optional[list[Requir in_scope, constraints ) - working_set: abc.Iterable[DistInfoDistribution] = pkg_resources.working_set + working_set: abc.Iterable[importlib.metadata.Distribution] = importlib.metadata.distributions() # add all requirements of all in scope packages installed in this environment - all_constraints: set[Requirement] = set(constraints if constraints is not None else []).union( - requirement + all_constraints: set[inmanta.util.CanonicalRequirement] = set(constraints if constraints is not None else []).union( + inmanta.util.parse_requirement(requirement=requirement) for dist_info in working_set - if in_scope.fullmatch(dist_info.key) - for requirement in dist_info.requires() + if in_scope.fullmatch(dist_info.name) + for requirement in dist_info.requires or [] ) - installed_versions: dict[str, version.Version] = PythonWorkingSet.get_packages_in_working_set() + installed_versions: dict[str, packaging.version.Version] = PythonWorkingSet.get_packages_in_working_set() constraint_violations: set[VersionConflict] = { - VersionConflict(constraint, installed_versions.get(constraint.key, None)) + VersionConflict(constraint, installed_versions.get(constraint.name, None)) for constraint in all_constraints - if constraint.key not in installed_versions or str(installed_versions[constraint.key]) not in constraint + if constraint.name not in installed_versions + or not constraint.specifier.contains(installed_versions[constraint.name], prereleases=True) } all_violations = constraint_violations_non_strict | constraint_violations_strict | constraint_violations @@ -1289,7 +1314,7 @@ def _activate_that(self) -> None: def install_for_config( self, - requirements: list[Requirement], + requirements: list[inmanta.util.CanonicalRequirement], config: PipConfig, upgrade: bool = False, constraint_files: Optional[list[str]] = None, @@ -1327,7 +1352,7 @@ class VenvSnapshot: old_os_venv: Optional[str] old_process_env_path: str old_process_env: ActiveEnv - old_working_set: WorkingSet + old_working_set: pkg_resources.WorkingSet def restore(self) -> None: os.environ["PATH"] = self.old_os_path diff --git a/src/inmanta/file_parser.py b/src/inmanta/file_parser.py index 900844b5fb..35783545fc 100644 --- a/src/inmanta/file_parser.py +++ b/src/inmanta/file_parser.py @@ -18,8 +18,8 @@ import os -from pkg_resources import Requirement - +import inmanta.util +import packaging.utils from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap @@ -56,11 +56,11 @@ class RequirementsTxtParser: """ @classmethod - def parse(cls, filename: str) -> list[Requirement]: + def parse(cls, filename: str) -> list[inmanta.util.CanonicalRequirement]: """ Get all the requirements in `filename` as a list of `Requirement` instances. """ - return [Requirement.parse(r) for r in cls.parse_requirements_as_strs(filename)] + return [inmanta.util.parse_requirement(requirement=r) for r in cls.parse_requirements_as_strs(filename)] @classmethod def parse_requirements_as_strs(cls, filename: str) -> list[str]: @@ -86,6 +86,7 @@ def get_content_with_dep_removed(cls, filename: str, remove_dep_on_pkg: str) -> if not os.path.exists(filename): raise Exception(f"File {filename} doesn't exist") + removed_dependency = packaging.utils.canonicalize_name(remove_dep_on_pkg) result = "" line_continuation_buffer = "" with open(filename, encoding="utf-8") as fd: @@ -93,14 +94,22 @@ def get_content_with_dep_removed(cls, filename: str, remove_dep_on_pkg: str) -> if line_continuation_buffer: line_continuation_buffer += line if not line.endswith("\\"): - if Requirement.parse(line_continuation_buffer).key != remove_dep_on_pkg: + if ( + inmanta.util.parse_requirement( + requirement=inmanta.util.remove_comment_part_from_specifier(line_continuation_buffer) + ).name + != removed_dependency + ): result += line_continuation_buffer line_continuation_buffer = "" elif not line.strip() or line.strip().startswith("#"): result += line elif line.endswith("\\"): line_continuation_buffer = line - elif Requirement.parse(line).key != remove_dep_on_pkg.lower(): + elif ( + inmanta.util.parse_requirement(requirement=inmanta.util.remove_comment_part_from_specifier(line)).name + != removed_dependency + ): result += line else: # Dependency matches `remove_dep_on_pkg` => Remove line from result diff --git a/src/inmanta/main.py b/src/inmanta/main.py index 4171a51d02..ab90f86215 100644 --- a/src/inmanta/main.py +++ b/src/inmanta/main.py @@ -26,17 +26,15 @@ from typing import Any, Callable, Optional, Union, cast import click +import importlib_metadata import texttable -from click_plugins import with_plugins -from pkg_resources import iter_entry_points -from inmanta import protocol +from inmanta import protocol, util from inmanta.config import Config, cmdline_rest_transport from inmanta.const import AgentAction, AgentTriggerMethod, ResourceAction from inmanta.data.model import ResourceVersionIdStr from inmanta.resources import Id from inmanta.types import JsonType -from inmanta.util import parse_timestamp class Client: @@ -182,7 +180,7 @@ def get_table(header: list[str], rows: list[list[str]], data_type: Optional[list return table.draw() -@with_plugins(iter_entry_points("inmanta.cli_plugins")) +@util.click_group_with_plugins(iter(importlib_metadata.entry_points(group="inmanta.cli_plugins"))) @click.group(help="Base command") @click.option("--host", help="The server hostname to connect to") @click.option("--port", help="The server port to connect to") @@ -668,7 +666,7 @@ def param(ctx: click.Context) -> None: def param_list(client: Client, environment: str) -> None: result = client.get_dict("list_params", arguments=dict(tid=client.to_environment_id(environment))) expire = int(result["expire"]) - now = parse_timestamp(result["now"]) + now = util.parse_timestamp(result["now"]) when = now - datetime.timedelta(0, expire) data = [] @@ -680,7 +678,7 @@ def param_list(client: Client, environment: str) -> None: p["name"], p["source"], p["updated"], - str(float(parse_timestamp(p["updated"]) < when)), + str(float(util.parse_timestamp(p["updated"]) < when)), ] ) diff --git a/src/inmanta/module.py b/src/inmanta/module.py index accebd8a4f..e56065366c 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -39,6 +39,7 @@ from enum import Enum from functools import reduce from importlib.abc import Loader +from importlib.metadata import PackageNotFoundError from io import BytesIO, TextIOBase from itertools import chain from subprocess import CalledProcessError @@ -47,13 +48,13 @@ from typing import Annotated, ClassVar, Dict, Generic, List, NewType, Optional, TextIO, TypeVar, Union, cast import more_itertools -import pkg_resources import pydantic import yaml -from pkg_resources import Distribution, DistributionNotFound, Requirement, parse_requirements, parse_version from pydantic import BaseModel, Field, NameEmail, StringConstraints, ValidationError, field_validator import inmanta.data.model +import packaging.requirements +import packaging.utils import packaging.version from inmanta import RUNNING_TESTS, const, env, loader, plugins from inmanta.ast import CompilerException, LocatableString, Location, Namespace, Range, WrappingRuntimeException @@ -67,7 +68,8 @@ from inmanta.stable_api import stable_api from inmanta.util import get_compiler_version from inmanta.warnings import InmantaWarning -from packaging import version +from packaging.specifiers import SpecifierSet +from packaging.version import Version from ruamel.yaml.comments import CommentedMap try: @@ -96,26 +98,31 @@ class InmantaModuleRequirement: used by distinguishing the two on a type level. """ - def __init__(self, requirement: Requirement) -> None: - if requirement.project_name.startswith(ModuleV2.PKG_NAME_PREFIX): + def __init__(self, requirement: inmanta.util.CanonicalRequirement) -> None: + if requirement.name.startswith(ModuleV2.PKG_NAME_PREFIX): raise ValueError( f"InmantaModuleRequirement instances work with inmanta module names, not python package names. " f"Problematic case: {str(requirement)}" ) - self._requirement: Requirement = requirement + self._requirement = requirement @property def project_name(self) -> str: - # Requirement converts all "_" to "-". Inmanta modules use "_" - return self._requirement.project_name.replace("-", "_") + warnings.warn(InmantaWarning("The `project_name` property has been deprecated in favor of `name`")) + return self.name + + @property + def name(self) -> str: + return self._requirement.name.replace("-", "_") @property def key(self) -> str: # Requirement converts all "_" to "-". Inmanta modules use "_" - return self._requirement.key.replace("-", "_") + warnings.warn(InmantaWarning("The `key` property has been deprecated in favor of `name`")) + return self.name @property - def specifier(self) -> str: + def specifier(self) -> SpecifierSet: return self._requirement.specifier def __eq__(self, other: object) -> bool: @@ -123,8 +130,8 @@ def __eq__(self, other: object) -> bool: return NotImplemented return self._requirement == other._requirement - def __contains__(self, version: str) -> bool: - return version in self._requirement + def __contains__(self, version: packaging.version.Version | str) -> bool: + return self._requirement.specifier.contains(version, prereleases=True) def __str__(self) -> str: return str(self._requirement).replace("-", "_") @@ -132,16 +139,6 @@ def __str__(self) -> str: def __hash__(self) -> int: return self._requirement.__hash__() - @property - def specs(self) -> Sequence[tuple[str, str]]: - return self._requirement.specs - - def version_spec_str(self) -> str: - """ - Returns a string representation of this module requirement's version spec. Includes only the version part. - """ - return ",".join("".join(spec) for spec in self.specs) - @classmethod def parse(cls: type[TInmantaModuleRequirement], spec: str) -> TInmantaModuleRequirement: if spec.startswith(ModuleV2.PKG_NAME_PREFIX): @@ -150,16 +147,16 @@ def parse(cls: type[TInmantaModuleRequirement], spec: str) -> TInmantaModuleRequ ) if "-" in spec: raise ValueError("Invalid Inmanta module requirement: Inmanta module names use '_', not '-'.") - return cls(Requirement.parse(spec)) + return cls(inmanta.util.parse_requirement(requirement=spec)) - def get_python_package_requirement(self) -> Requirement: + def get_python_package_requirement(self) -> inmanta.util.CanonicalRequirement: """ Return a Requirement with the name of the Python distribution package for this module requirement. """ - module_name = self.project_name + module_name = self.name pkg_name = ModuleV2Source.get_package_name_for(module_name) pkg_req_str = str(self).replace(module_name, pkg_name, 1) # Replace max 1 occurrence - return Requirement.parse(pkg_req_str) + return inmanta.util.parse_requirement(requirement=pkg_req_str) class CompilerExceptionWithExtendedTrace(CompilerException): @@ -341,7 +338,7 @@ def get_all_tags(self, repo: str) -> list[str]: pass @abstractmethod - def get_version_tags(self, repo: str, only_return_stable_versions: bool = False) -> list[version.Version]: + def get_version_tags(self, repo: str, only_return_stable_versions: bool = False) -> list[packaging.version.Version]: pass @abstractmethod @@ -422,7 +419,7 @@ def status(self, repo: str, untracked_files_mode: Optional[UntrackedFilesMode] = def get_all_tags(self, repo: str) -> list[str]: return subprocess.check_output(["git", "tag"], cwd=repo).decode("utf-8").splitlines() - def get_version_tags(self, repo: str, only_return_stable_versions: bool = False) -> list[version.Version]: + def get_version_tags(self, repo: str, only_return_stable_versions: bool = False) -> list[packaging.version.Version]: """ Return the Git tags that represent version numbers as version.Version objects. Only PEP440 compliant versions will be returned. @@ -434,8 +431,8 @@ def get_version_tags(self, repo: str, only_return_stable_versions: bool = False) all_tags: list[str] = sorted(self.get_all_tags(repo)) for tag in all_tags: try: - parsed_version: version.Version = version.Version(tag) - except version.InvalidVersion: + parsed_version: packaging.version.Version = packaging.version.Version(tag) + except packaging.version.InvalidVersion: continue if not only_return_stable_versions or not parsed_version.is_prerelease: result.append(parsed_version) @@ -595,7 +592,9 @@ def _format_constraints(self, module_name: str, module_spec: list[InmantaModuleR :param module_name: The name of the module. :param module_spec: List of inmanta requirements in which to look for the module. """ - constraints_on_module: list[str] = [str(req) for req in module_spec if module_name == req.key and req.specs] + constraints_on_module: list[str] = [ + str(req) for req in module_spec if module_name == req.name and len(req.specifier) > 0 + ] if constraints_on_module: from_constraints = f"(with constraints {' '.join(constraints_on_module)})" else: @@ -611,18 +610,21 @@ def log_pre_install_information(self, module_name: str, module_spec: list[Inmant """ raise NotImplementedError("Abstract method") - def _log_version_snapshot(self, header: Optional[str], version_snapshot: dict[str, version.Version]) -> None: + def _log_version_snapshot(self, header: Optional[str], version_snapshot: dict[str, packaging.version.Version]) -> None: if version_snapshot: out = [header] if header is not None else [] out.extend(f"{mod}: {version}" for mod, version in version_snapshot.items()) LOGGER.debug("\n".join(out)) def _log_snapshot_difference( - self, version_snapshot: dict[str, version.Version], previous_snapshot: dict[str, version.Version], header: Optional[str] + self, + version_snapshot: dict[str, packaging.version.Version], + previous_snapshot: dict[str, packaging.version.Version], + header: Optional[str], ) -> None: - set_pre_install: set[tuple[str, version.Version]] = set(previous_snapshot.items()) - set_post_install: set[tuple[str, version.Version]] = set(version_snapshot.items()) - updates_and_additions: set[tuple[str, version.Version]] = set_post_install - set_pre_install + set_pre_install: set[tuple[str, packaging.version.Version]] = set(previous_snapshot.items()) + set_post_install: set[tuple[str, packaging.version.Version]] = set(version_snapshot.items()) + updates_and_additions: set[tuple[str, packaging.version.Version]] = set_post_install - set_pre_install if version_snapshot: out = [header] if header is not None else [] @@ -665,7 +667,7 @@ def from_path(cls, project: Optional["Project"], module_name: str, path: str) -> raise NotImplementedError("Abstract method") def _get_module_name(self, module_spec: list[InmantaModuleRequirement]) -> str: - module_names: set[str] = {req.project_name for req in module_spec} + module_names: set[str] = {req.name for req in module_spec} module_name: str = more_itertools.one( module_names, too_short=ValueError("module_spec should contain at least one requirement"), @@ -683,19 +685,16 @@ def __init__(self, urls: Sequence[str] = []) -> None: self.urls: list[str] = [url if not os.path.exists(url) else os.path.abspath(url) for url in urls] @classmethod - def get_installed_version(cls, module_name: str) -> Optional[version.Version]: + def get_installed_version(cls, module_name: str) -> Optional[packaging.version.Version]: """ Returns the version for a module if it is installed. """ if module_name.startswith(ModuleV2.PKG_NAME_PREFIX): raise ValueError("PythonRepo instances work with inmanta module names, not Python package names.") try: - dist: Distribution = pkg_resources.get_distribution(ModuleV2Source.get_package_name_for(module_name)) - return version.Version(dist.version) - except DistributionNotFound: + return packaging.version.Version(importlib.metadata.version(ModuleV2Source.get_package_name_for(module_name))) + except PackageNotFoundError: return None - except version.InvalidVersion: - raise InvalidModuleException(f"Package {dist.project_name} was installed but it has no valid version.") @classmethod def get_inmanta_module_name(cls, python_package_name: str) -> str: @@ -720,14 +719,14 @@ def install(self, project: "Project", module_spec: list[InmantaModuleRequirement assert_pip_has_source(project.metadata.pip, f"a v2 module {module_name}") - requirements: list[Requirement] = [req.get_python_package_requirement() for req in module_spec] + requirements: list[inmanta.util.CanonicalRequirement] = [req.get_python_package_requirement() for req in module_spec] preinstalled: Optional[ModuleV2] = self.get_installed_module(project, module_name) # Get known requires and add them to prevent invalidating constraints through updates # These could be constraints (-c) as well, but that requires additional sanitation # Because for pip not every valid -r is a valid -c current_requires = project.get_strict_python_requirements_as_list() - requirements += [Requirement.parse(r) for r in current_requires] + requirements += inmanta.util.parse_requirements(current_requires) if preinstalled is not None: # log warning if preinstalled version does not match constraints @@ -737,7 +736,7 @@ def install(self, project: "Project", module_spec: list[InmantaModuleRequirement "Currently installed %s-%s does not match constraint %s: updating to compatible version.", module_name, preinstalled_version, - ",".join(constraint.version_spec_str() for constraint in module_spec if constraint.specs), + ",".join(str(constraint.specifier) for constraint in module_spec if len(constraint.specifier) > 0), ) try: self.log_pre_install_information(module_name, module_spec) @@ -760,7 +759,7 @@ def install(self, project: "Project", module_spec: list[InmantaModuleRequirement def log_pre_install_information(self, module_name: str, module_spec: list[InmantaModuleRequirement]) -> None: LOGGER.debug("Installing module %s (v2) %s.", module_name, super()._format_constraints(module_name, module_spec)) - def take_v2_modules_snapshot(self, header: Optional[str] = None) -> dict[str, version.Version]: + def take_v2_modules_snapshot(self, header: Optional[str] = None) -> dict[str, packaging.version.Version]: """ Log and return a dictionary containing currently installed v2 modules and their versions. @@ -772,7 +771,7 @@ def take_v2_modules_snapshot(self, header: Optional[str] = None) -> dict[str, ve return version_snapshot def log_snapshot_difference_v2_modules( - self, previous_snapshot: dict[str, version.Version], header: Optional[str] = None + self, previous_snapshot: dict[str, packaging.version.Version], header: Optional[str] = None ) -> None: """ Logs a diff view of v2 inmanta modules currently installed (in alphabetical order) and their version. @@ -792,7 +791,7 @@ def log_post_install_information(self, module_name: str) -> None: :param module_name: The module's name. """ - installed_version: Optional[version.Version] = self.get_installed_version(module_name) + installed_version: Optional[packaging.version.Version] = self.get_installed_version(module_name) LOGGER.debug("Successfully installed module %s (v2) version %s", module_name, installed_version) def path_for(self, name: str) -> Optional[str]: @@ -857,7 +856,7 @@ def __init__(self, local_repo: "ModuleRepo", remote_repo: "ModuleRepo") -> None: def log_pre_install_information(self, module_name: str, module_spec: list[InmantaModuleRequirement]) -> None: LOGGER.debug("Installing module %s (v1) %s.", module_name, super()._format_constraints(module_name, module_spec)) - def take_modules_snapshot(self, project: "Project", header: Optional[str] = None) -> dict[str, version.Version]: + def take_modules_snapshot(self, project: "Project", header: Optional[str] = None) -> dict[str, packaging.version.Version]: """ Log and return a dictionary containing currently loaded modules and their versions. @@ -869,7 +868,7 @@ def take_modules_snapshot(self, project: "Project", header: Optional[str] = None return version_snapshot def log_snapshot_difference_v1_modules( - self, project: "Project", previous_snapshot: dict[str, version.Version], header: Optional[str] = None + self, project: "Project", previous_snapshot: dict[str, packaging.version.Version], header: Optional[str] = None ) -> None: """ Logs a diff view on inmanta modules (both v1 and v2) currently loaded (in alphabetical order) and their version. @@ -912,7 +911,7 @@ def install(self, project: "Project", module_spec: list[InmantaModuleRequirement "Currently installed %s-%s does not match constraint %s: updating to compatible version.", module_name, preinstalled_version, - ",".join(constraint.version_spec_str() for constraint in module_spec if constraint.specs), + ",".join(str(constraint.specifier) for constraint in module_spec if len(constraint.specifier) > 0), ) self.log_pre_install_information(module_name, module_spec) modules_pre_install = self.take_modules_snapshot(project, header="Modules versions before installation:") @@ -1055,7 +1054,7 @@ def make_repo(path: str, root: Optional[str] = None) -> Union[LocalFileRepo, Rem def merge_specs(mainspec: "Dict[str, List[InmantaModuleRequirement]]", new: "List[InmantaModuleRequirement]") -> None: """Merge two maps str->[TMetadata] by concatting their lists.""" for req in new: - key = req.project_name + key = req.name if key not in mainspec: mainspec[key] = [req] else: @@ -1175,14 +1174,17 @@ def has_requirement_for(self, pkg_name: str) -> bool: Returns True iff this requirements.txt file contains the given package name. The given `pkg_name` is matched case insensitive against the requirements in this RequirementsTxtFile. """ - return any(r.key == pkg_name.lower() for r in self._requirements) + canonicalized: str = packaging.utils.canonicalize_name(pkg_name) + return any(r.name == canonicalized for r in self._requirements) - def set_requirement_and_write(self, requirement: Requirement) -> None: + def set_requirement_and_write(self, requirement: inmanta.util.CanonicalRequirement) -> None: """ Add the given requirement to the requirements.txt file and update the file on disk, replacing any existing constraints on this package. """ - new_content_file = RequirementsTxtParser.get_content_with_dep_removed(self._filename, remove_dep_on_pkg=requirement.key) + new_content_file = RequirementsTxtParser.get_content_with_dep_removed( + self._filename, remove_dep_on_pkg=requirement.name + ) new_content_file = new_content_file.rstrip() if new_content_file: new_content_file = f"{new_content_file}\n{requirement}" @@ -1270,8 +1272,8 @@ class ModuleMetadata(ABC, Metadata): @classmethod def is_pep440_version(cls, v: str) -> str: try: - version.Version(v) - except version.InvalidVersion as e: + packaging.version.Version(v) + except packaging.version.InvalidVersion as e: raise ValueError(f"Version {v} is not PEP440 compliant") from e return v @@ -1322,9 +1324,9 @@ def get_full_version(self) -> packaging.version.Version: @classmethod def _compose_full_version(cls, v: str, version_tag: str) -> packaging.version.Version: if not version_tag: - return version.Version(v) + return packaging.version.Version(v) normalized_tag: str = version_tag.lstrip(".") - return version.Version(f"{v}.{normalized_tag}") + return packaging.version.Version(f"{v}.{normalized_tag}") @stable_api @@ -1358,11 +1360,11 @@ def is_pep440_version_v1(cls, v: str) -> str: @classmethod def _substitute_version(cls: type[TModuleMetadata], source: str, new_version: str, version_tag: str = "") -> str: - new_version_obj: version.Version = cls._compose_full_version(new_version, version_tag) + new_version_obj: packaging.version.Version = cls._compose_full_version(new_version, version_tag) return re.sub(r"([\s]version\s*:\s*['\"\s]?)[^\"'}\s]+(['\"]?)", rf"\g<1>{new_version_obj}\g<2>", source) def get_full_version(self) -> packaging.version.Version: - return version.Version(self.version) + return packaging.version.Version(self.version) def to_v2(self) -> "ModuleV2Metadata": values = self.dict() @@ -1372,7 +1374,7 @@ def to_v2(self) -> "ModuleV2Metadata": install_requires = [ModuleV2Source.get_package_name_for(r) for r in values["requires"]] del values["requires"] values["name"] = ModuleV2Source.get_package_name_for(values["name"]) - values["version"], values["version_tag"] = ModuleV2Metadata.split_version(version.Version(values["version"])) + values["version"], values["version_tag"] = ModuleV2Metadata.split_version(packaging.version.Version(values["version"])) return ModuleV2Metadata(**values, install_requires=install_requires) @@ -1408,7 +1410,7 @@ class ModuleV2Metadata(ModuleMetadata): @field_validator("version") @classmethod def is_base_version(cls, v: str) -> str: - version_obj: version.Version = version.Version(v) + version_obj: packaging.version.Version = packaging.version.Version(v) if str(version_obj) != version_obj.base_version: raise ValueError( "setup.cfg version should be a base version without tag. Use egg_info.tag_build to configure a tag" @@ -1439,7 +1441,7 @@ def get_version_tag(v: packaging.version.Version) -> str: def is_valid_version_tag(cls, v: str) -> str: try: cls._compose_full_version("1.0.0", v) - except version.InvalidVersion as e: + except packaging.version.InvalidVersion as e: raise ValueError(f"Version tag {v} is not PEP440 compliant") from e return v @@ -1830,7 +1832,7 @@ def has_module_requirement(self, module_name: str) -> bool: declare dependencies module dependencies. This could include the requirements.txt file next to the metadata file of the project or module. """ - return any(module_name == InmantaModuleRequirement.parse(req).key for req in self.get_module_requirements()) + return any(module_name == InmantaModuleRequirement.parse(req).name for req in self.get_module_requirements()) def _load_file(self, ns: Namespace, file: str) -> tuple[list[Statement], BasicBlock]: ns.location = Location(file, 1) @@ -1902,7 +1904,7 @@ def add_module_requirement_to_requires_and_write(self, requirement: InmantaModul # Update requires if "requires" in content and content["requires"]: existing_matching_reqs: list[str] = [ - r for r in content["requires"] if InmantaModuleRequirement.parse(r).key == requirement.key + r for r in content["requires"] if InmantaModuleRequirement.parse(r).name == requirement.name ] for r in existing_matching_reqs: content["requires"].remove(r) @@ -1919,7 +1921,7 @@ def has_module_requirement_in_requires(self, module_name: str) -> bool: content: CommentedMap = PreservativeYamlParser.parse(self.get_metadata_file_path()) if "requires" not in content: return False - return any(r for r in content["requires"] if InmantaModuleRequirement.parse(r).key == module_name) + return any(r for r in content["requires"] if InmantaModuleRequirement.parse(r).name == module_name) def remove_module_requirement_from_requires_and_write(self, module_name: str) -> None: """ @@ -1930,7 +1932,7 @@ def remove_module_requirement_from_requires_and_write(self, module_name: str) -> if not self.has_module_requirement_in_requires(module_name): return content: CommentedMap = PreservativeYamlParser.parse(self.get_metadata_file_path()) - content["requires"] = [r for r in content["requires"] if InmantaModuleRequirement.parse(r).key != module_name] + content["requires"] = [r for r in content["requires"] if InmantaModuleRequirement.parse(r).name != module_name] PreservativeYamlParser.dump(self.get_metadata_file_path(), content) @@ -2124,7 +2126,7 @@ def install_modules(self, *, bypass_module_cache: bool = False, update_dependenc self.verify_module_version_compatibility() # do python install - pyreq: list[Requirement] = [Requirement.parse(x) for x in self.collect_python_requirements()] + pyreq: list[inmanta.util.CanonicalRequirement] = inmanta.util.parse_requirements(self.collect_python_requirements()) if len(pyreq) > 0: # upgrade both direct and transitive module dependencies: eager upgrade strategy @@ -2316,13 +2318,13 @@ def load_module_v2_requirements(module_like: ModuleLike) -> None: for requirement in module_like.get_module_v2_requirements(): # load module self.get_module( - requirement.key, + requirement.name, allow_v1=False, install_v2=install, bypass_module_cache=bypass_module_cache, ) # queue AST reload - require_v2(requirement.key) + require_v2(requirement.name) def setup_module(module: Module) -> None: """ @@ -2520,10 +2522,10 @@ def verify_module_version_compatibility(self) -> None: LOGGER.warning("Module %s is present in requires but it is not used by the model.", name) continue module = self.modules[name] - version = parse_version(str(module.version)) + current_version = Version(version=str(module.version)) for r in spec: - if version not in r: - exc_message += f"\n\t* requirement {r} on module {name} not fulfilled, now at version {version}." + if current_version not in r: + exc_message += f"\n\t* requirement {r} on module {name} not fulfilled, now at version {current_version}." if exc_message: exc_message = f"The following requirements were not satisfied:{exc_message}" @@ -2542,7 +2544,9 @@ def verify_python_requires(self) -> None: Verifies no incompatibilities exist within the Python environment with respect to installed module v2 requirements. """ if self.strict_deps_check: - constraints: list[Requirement] = [Requirement.parse(item) for item in self.collect_python_requirements()] + constraints: list[inmanta.util.CanonicalRequirement] = inmanta.util.parse_requirements( + self.collect_python_requirements() + ) env.process_env.check(strict_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*"), constraints=constraints) else: if not env.process_env.check_legacy(in_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*")): @@ -2661,7 +2665,7 @@ def add_module_requirement_persistent(self, requirement: InmantaModuleRequiremen requirements_txt_file.set_requirement_and_write(requirement.get_python_package_requirement()) elif os.path.exists(requirements_txt_file_path): requirements_txt_file = RequirementsTxtFile(requirements_txt_file_path) - requirements_txt_file.remove_requirement_and_write(requirement.get_python_package_requirement().key) + requirements_txt_file.remove_requirement_and_write(requirement.get_python_package_requirement().name) def get_module_requirements(self) -> list[str]: return [*self.metadata.requires, *(str(req) for req in self.get_module_v2_requirements())] @@ -2673,7 +2677,7 @@ def requires(self) -> "List[InmantaModuleRequirement]": # filter on import stmt reqs = [] for spec in self._metadata.requires: - req = [x for x in parse_requirements(spec)] + req = [inmanta.util.parse_requirement(requirement=spec)] if len(req) > 1: print(f"Module file for {self._path} has bad line in requirements specification {spec}") reqe = InmantaModuleRequirement(req[0]) @@ -2704,7 +2708,7 @@ def get_spec(name: str) -> "List[InmantaModuleRequirement]": return {name: get_spec(name) for name in imports} - def collect_python_requirements(self) -> list[str]: + def collect_python_requirements(self) -> Sequence[str]: """ Collect the list of all python requirements of all modules in this project, excluding those on inmanta modules. """ @@ -2810,7 +2814,7 @@ def requires(self) -> "List[InmantaModuleRequirement]": """ reqs = [] for spec in self.get_module_requirements(): - req = [x for x in parse_requirements(spec)] + req = [inmanta.util.parse_requirement(requirement=spec)] if len(req) > 1: print(f"Module file for {self._path} has bad line in requirements specification {spec}") reqe = InmantaModuleRequirement(req[0]) @@ -2838,12 +2842,12 @@ def rewrite_version(self, new_version: str, version_tag: str = "") -> None: fd.write(new_module_def) self._metadata = new_metadata - def get_version(self) -> version.Version: + def get_version(self) -> packaging.version.Version: """ Return the version of this module. This is the actually installed version, which might differ from the version declared in its metadata (e.g. by a .dev0 tag). """ - return version.Version(self._metadata.version) + return packaging.version.Version(self._metadata.version) version = property(get_version) @@ -2898,7 +2902,7 @@ def get_freeze(self, submodule: str, recursive: bool = False, mode: str = ">=") if impor not in out: v1_mode: bool = self.GENERATION == ModuleGeneration.V1 mainmod = self._project.get_module(impor, install_v1=v1_mode, allow_v1=v1_mode) - vers: version.Version = mainmod.version + vers: packaging.version.Version = mainmod.version # track submodules for cycle avoidance out[impor] = mode + " " + str(vers) if recursive: @@ -3154,7 +3158,7 @@ def get_all_requires(self) -> list[InmantaModuleRequirement]: :return: all modules required by an import from any sub-modules, with all constraints applied """ # get all constraints - spec: dict[str, InmantaModuleRequirement] = {req.project_name: req for req in self.requires()} + spec: dict[str, InmantaModuleRequirement] = {req.name: req for req in self.requires()} # find all imports imports = {imp.name.split("::")[0] for subm in sorted(self.get_all_submodules()) for imp in self.get_imports(subm)} return [spec[r] if spec.get(r) else InmantaModuleRequirement.parse(r) for r in imports] @@ -3203,30 +3207,30 @@ def update( @classmethod def get_suitable_version_for( cls, modulename: str, requirements: Iterable[InmantaModuleRequirement], path: str, release_only: bool = True - ) -> Optional[version.Version]: + ) -> Optional[packaging.version.Version]: versions_str = gitprovider.get_all_tags(path) - def try_parse(x: str) -> Optional[version.Version]: + def try_parse(x: str) -> Optional[packaging.version.Version]: try: - return parse_version(x) + return Version(version=x) except Exception: return None - versions: list[version.Version] = [x for x in [try_parse(v) for v in versions_str] if x is not None] + versions: list[packaging.version.Version] = [x for x in [try_parse(v) for v in versions_str] if x is not None] versions = sorted(versions, reverse=True) for r in requirements: - versions = [x for x in r.specifier.filter(versions, not release_only)] + versions = [x for x in r.specifier.filter(iterable=versions, prereleases=not release_only)] comp_version_raw = get_compiler_version() - comp_version = parse_version(comp_version_raw) + comp_version = Version(version=comp_version_raw) return cls.__best_for_compiler_version(modulename, versions, path, comp_version) @classmethod def __best_for_compiler_version( - cls, modulename: str, versions: list[version.Version], path: str, comp_version: version.Version - ) -> Optional[version.Version]: - def get_cv_for(best: version.Version) -> Optional[version.Version]: + cls, modulename: str, versions: list[packaging.version.Version], path: str, comp_version: packaging.version.Version + ) -> Optional[packaging.version.Version]: + def get_cv_for(best: packaging.version.Version) -> Optional[packaging.version.Version]: cfg_text: str = gitprovider.get_file_for_version(path, str(best), cls.MODULE_FILE) metadata: ModuleV1Metadata = cls.get_metadata_file_schema_type().parse(cfg_text) if metadata.compiler_version is None: @@ -3234,7 +3238,7 @@ def get_cv_for(best: version.Version) -> Optional[version.Version]: v = metadata.compiler_version if isinstance(v, (int, float)): v = str(v) - return parse_version(v) + return Version(version=v) if not versions: return None @@ -3286,23 +3290,23 @@ def add_module_requirement_persistent(self, requirement: InmantaModuleRequiremen # Remove requirement from requirements.txt file if os.path.exists(requirements_txt_file_path): requirements_txt_file = RequirementsTxtFile(requirements_txt_file_path) - requirements_txt_file.remove_requirement_and_write(requirement.get_python_package_requirement().key) + requirements_txt_file.remove_requirement_and_write(requirement.get_python_package_requirement().name) else: # Add requirement to requirements.txt requirements_txt_file = RequirementsTxtFile(requirements_txt_file_path, create_file_if_not_exists=True) requirements_txt_file.set_requirement_and_write(requirement.get_python_package_requirement()) # Remove requirement from module.yml file - self.remove_module_requirement_from_requires_and_write(requirement.key) + self.remove_module_requirement_from_requires_and_write(requirement.name) - def versions(self) -> list[version.Version]: + def versions(self) -> list[packaging.version.Version]: """ Provide a list of all versions available in the repository """ versions_str: list[str] = gitprovider.get_all_tags(self._path) - def try_parse(x: str) -> Optional[version.Version]: + def try_parse(x: str) -> Optional[packaging.version.Version]: try: - return parse_version(x) + return Version(version=x) except Exception: return None @@ -3358,10 +3362,10 @@ def __init__( project: Optional[Project], path: str, is_editable_install: bool = False, - installed_version: Optional[version.Version] = None, + installed_version: Optional[packaging.version.Version] = None, ) -> None: self._is_editable_install = is_editable_install - self._version: Optional[version.Version] = installed_version + self._version: Optional[packaging.version.Version] = installed_version super().__init__(project, path) if not os.path.exists(os.path.join(self.model_dir, "_init.cf")): @@ -3380,7 +3384,7 @@ def from_path(cls: type[TModule], path: str) -> Optional[TModule]: # setup.cfg is a generic Python config file: if the metadata does not match an inmanta module's, return None return None - def get_version(self) -> version.Version: + def get_version(self) -> packaging.version.Version: return self._version if self._version is not None else self._metadata.get_full_version() version = property(get_version) @@ -3427,12 +3431,12 @@ def add_module_requirement_persistent(self, requirement: InmantaModuleRequiremen # Parse config file config_parser = ConfigParser() config_parser.read(self.get_metadata_file_path()) - python_pkg_requirement: Requirement = requirement.get_python_package_requirement() + python_pkg_requirement: inmanta.util.CanonicalRequirement = requirement.get_python_package_requirement() if config_parser.has_option("options", "install_requires"): new_install_requires = [ r for r in config_parser.get("options", "install_requires").split("\n") - if r and Requirement.parse(r).key != python_pkg_requirement.key + if r and inmanta.util.parse_requirement(requirement=r).name != python_pkg_requirement.name ] new_install_requires.append(str(python_pkg_requirement)) else: diff --git a/src/inmanta/moduletool.py b/src/inmanta/moduletool.py index 2abbfec73a..83921e19d4 100644 --- a/src/inmanta/moduletool.py +++ b/src/inmanta/moduletool.py @@ -38,18 +38,18 @@ from configparser import ConfigParser from functools import total_ordering from re import Pattern -from typing import IO, TYPE_CHECKING, Any, Literal, Optional +from typing import IO, Any, Literal, Optional import click import more_itertools import texttable import yaml from cookiecutter.main import cookiecutter -from pkg_resources import Requirement import build import inmanta import inmanta.warnings +import packaging.requirements import toml from build.env import DefaultIsolatedEnv from inmanta import const, env @@ -78,11 +78,6 @@ from inmanta.stable_api import stable_api from packaging.version import Version -if TYPE_CHECKING: - from packaging.requirements import InvalidRequirement -else: - from pkg_resources.extern.packaging.requirements import InvalidRequirement - LOGGER = logging.getLogger(__name__) @@ -477,7 +472,7 @@ def update( def do_update(specs: Mapping[str, Sequence[InmantaModuleRequirement]], modules: list[str]) -> None: v2_modules = {module for module in modules if my_project.module_source.path_for(module) is not None} - v2_python_specs: list[Requirement] = [ + v2_python_specs: list[inmanta.util.CanonicalRequirement] = [ module_spec.get_python_package_requirement() for module, module_specs in specs.items() for module_spec in module_specs @@ -489,7 +484,7 @@ def do_update(specs: Mapping[str, Sequence[InmantaModuleRequirement]], modules: # Because for pip not every valid -r is a valid -c current_requires = my_project.get_strict_python_requirements_as_list() env.process_env.install_for_config( - v2_python_specs + [Requirement.parse(r) for r in current_requires], + v2_python_specs + inmanta.util.parse_requirements(current_requires), my_project.metadata.pip, upgrade=True, ) @@ -809,7 +804,7 @@ def add(self, module_req: str, v1: bool = False, v2: bool = False, override: boo raise CLIException("Current working directory doesn't contain an Inmanta module or project", exitcode=1) try: module_requirement = InmantaModuleRequirement.parse(module_req) - except InvalidRequirement: + except packaging.requirements.InvalidRequirement: raise CLIException(f"'{module_req}' is not a valid requirement", exitcode=1) if not override and module_like.has_module_requirement(module_requirement.key): raise CLIException( @@ -1007,7 +1002,7 @@ def show_bool(b: bool) -> str: matches = version == reqv editable = True else: - reqv = ",".join(req.version_spec_str() for req in specs[name] if req.specs) or "*" + reqv = ",".join(str(req.specifier) for req in specs[name] if len(req.specifier) > 0) or "*" matches = all(version in req for req in specs[name]) editable = mod.is_editable() @@ -1953,7 +1948,7 @@ def get_setup_cfg(self, in_folder: str, warn_on_merge: bool = False) -> configpa # add requirements module_requirements: list[InmantaModuleRequirement] = [ - req for req in self._module.get_all_requires() if req.project_name != self._module.name + req for req in self._module.get_all_requires() if req.name != self._module.name ] python_requirements: list[str] = self._module.get_strict_python_requirements_as_list() if module_requirements or python_requirements: diff --git a/src/inmanta/server/extensions.py b/src/inmanta/server/extensions.py index 63069313cb..5a5e680af9 100644 --- a/src/inmanta/server/extensions.py +++ b/src/inmanta/server/extensions.py @@ -16,12 +16,12 @@ Contact: code@inmanta.com """ +import importlib.metadata import logging import os from collections import defaultdict from typing import Any, Generic, Optional, TypeVar -import pkg_resources import yaml from inmanta import data @@ -147,8 +147,8 @@ def _get_product_version(self) -> str: packages = ["inmanta-oss", "inmanta", "inmanta-core"] for package in packages: try: - return pkg_resources.get_distribution(package).version - except pkg_resources.DistributionNotFound: + return importlib.metadata.version(package) + except importlib.metadata.PackageNotFoundError: pass LOGGER.warning("Couldn't determine product version. Make sure inmanta is properly installed.") diff --git a/src/inmanta/util/__init__.py b/src/inmanta/util/__init__.py index 2e639a10ec..e9988968db 100644 --- a/src/inmanta/util/__init__.py +++ b/src/inmanta/util/__init__.py @@ -27,24 +27,31 @@ import itertools import logging import os +import pathlib import socket import threading import time +import typing import uuid import warnings from abc import ABC, abstractmethod from asyncio import CancelledError, Lock, Task, ensure_future, gather from collections import abc, defaultdict -from collections.abc import Coroutine, Iterator +from collections.abc import Coroutine, Iterable, Iterator from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from logging import Logger from types import TracebackType -from typing import BinaryIO, Callable, Generic, Optional, TypeVar, Union +from typing import BinaryIO, Callable, Generic, Optional, Sequence, TypeVar, Union import asyncpg +import click +import importlib_metadata from tornado import gen +import packaging +import packaging.requirements +import packaging.utils from crontab import CronTab from inmanta import COMPILER_VERSION, const from inmanta.stable_api import stable_api @@ -864,3 +871,108 @@ def check_for_pool_exhaustion(self) -> None: def _reset_counter(self) -> None: self._exhausted_pool_events_count = 0 + + +def remove_comment_part_from_specifier(to_clean: str) -> str: + """ + Remove the comment part of a requirement specifier + + :param to_clean: The requirement specifier to clean + :return: A cleaned requirement specifier + """ + # Refer to PEP 508. A requirement could contain a hashtag + to_clean = to_clean.strip() + drop_comment, _, _ = to_clean.partition(" #") + # We make sure whitespaces are not counted in the length of this string, e.g. " #" + drop_comment = drop_comment.strip() + return drop_comment + + +CanonicalRequirement = typing.NewType("CanonicalRequirement", packaging.requirements.Requirement) +""" +A CanonicalRequirement is a packaging.requirements.Requirement except that the name of this Requirement is canonicalized, which +allows us to compare names without dealing afterwards with the format of these requirements. +""" + + +def parse_requirement(requirement: str) -> CanonicalRequirement: + """ + Parse the given requirement string into a requirement object with a canonicalized name, meaning that we are sure that + every CanonicalRequirement will follow the same convention regarding the name. This will allow us compare requirements. + This function supposes to receive an actual requirement. + + :param requirement: The requirement's name + :return: A new requirement instance + """ + # We canonicalize the name of the requirement to be able to compare requirements and check if the requirement is + # already installed + # The following line could cause issue because we are not supposed to modify fields of an existing instance + # The version of packaging is constrained to ensure this can not cause problems in production. + requirement_instance = packaging.requirements.Requirement(requirement_string=requirement) + requirement_instance.name = packaging.utils.canonicalize_name(requirement_instance.name) + canonical_requirement_instance = CanonicalRequirement(requirement_instance) + return canonical_requirement_instance + + +def parse_requirements(requirements: Sequence[str]) -> list[CanonicalRequirement]: + """ + Parse the given requirements (sequence of strings) into requirement objects with a canonicalized name, meaning that we + are sure that every CanonicalRequirement will follow the same convention regarding the name. This will allow us compare + requirements. This function supposes to receive actual requirements. Commented strings will not be handled and result in + a ValueError + + :param requirements: The names of the different requirements + :return: list[CanonicalRequirement] + """ + return [parse_requirement(requirement=e) for e in requirements] + + +def parse_requirements_from_file(file_path: pathlib.Path) -> list[CanonicalRequirement]: + """ + Parse the given requirements (line by line) from a file into requirement objects with a canonicalized name, meaning that we + are sure that every CanonicalRequirement will follow the same convention regarding the name. This will allow us compare + requirements. + + :param file_path: The path to the read the requirements from + :return: list[CanonicalRequirement] + """ + if not file_path.exists(): + raise RuntimeError(f"The provided path does not exist: `{file_path}`!") + + with open(file_path) as f: + file_contents: list[str] = f.readlines() + requirements = [ + parse_requirement(remove_comment_part_from_specifier(line)) + for line in file_contents + if (stripped := line.lstrip()) and not stripped.startswith("#") # preprocessing + ] + + return requirements + + +# Retaken from the `click-plugins` repo which is now unmaintained +def click_group_with_plugins(plugins: Iterable[importlib_metadata.EntryPoint]) -> Callable[[click.Group], click.Group]: + """ + A decorator to register external CLI commands to an instance of `click.Group()`. + + :param plugins: An iterable producing one `pkg_resources.EntryPoint()` per iteration + :return: The provided click group with the new commands + """ + + def decorator(group: click.Group) -> click.Group: + for entry_point in plugins: + try: + group.add_command(entry_point.load()) + except Exception as e: + # Catch this so a busted plugin doesn't take down the CLI. + # Handled by registering a dummy command that does nothing + # other than explain the error. + def print_error(error: Exception) -> None: + click.echo(f"Error: could not load this plugin for the following reason: {error}") + + new_print_error = functools.partial(print_error, e) + group.add_command(click.Command(name=entry_point.name, callback=new_print_error)) + + return group + + return decorator diff --git a/tests/conftest.py b/tests/conftest.py index ea00e8cdea..962ef67287 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,6 +20,7 @@ import warnings from re import Pattern +import pkg_resources from tornado.httpclient import AsyncHTTPClient import _pytest.logging @@ -28,6 +29,7 @@ from inmanta.logging import InmantaLoggerConfig from inmanta.protocol import auth from inmanta.util import ScheduledTask, Scheduler, TaskMethod, TaskSchedule +from packaging.requirements import Requirement """ About the use of @parametrize_any and @slowtest: @@ -96,14 +98,12 @@ from typing import Callable, Dict, Optional, Union import asyncpg -import pkg_resources import psutil import py import pyformance import pytest from asyncpg.exceptions import DuplicateDatabaseError from click import testing -from pkg_resources import Requirement from pyformance.registry import MetricsRegistry from tornado import netutil @@ -1936,8 +1936,11 @@ def index_with_pkgs_containing_optional_deps() -> str: path=os.path.join(tmpdirname, "pkg"), publish_index=pip_index, optional_dependencies={ - "optional-a": [Requirement.parse("dep-a")], - "optional-b": [Requirement.parse("dep-b"), Requirement.parse("dep-c")], + "optional-a": [inmanta.util.parse_requirement(requirement="dep-a")], + "optional-b": [ + inmanta.util.parse_requirement(requirement="dep-b"), + inmanta.util.parse_requirement(requirement="dep-c"), + ], }, ) for pkg_name in ["dep-a", "dep-b", "dep-c"]: diff --git a/tests/moduletool/conftest.py b/tests/moduletool/conftest.py index a4aef66f1c..7db8d7b78f 100644 --- a/tests/moduletool/conftest.py +++ b/tests/moduletool/conftest.py @@ -204,17 +204,17 @@ def modules_repo(git_modules_dir) -> str: E-> H D-> F,G """ - make_module_simple_deps(reporoot, "A", ["B", "C", "D"], project=True) - make_module_simple_deps(reporoot, "B") - c = make_module_simple_deps(reporoot, "C", ["E", "F", "E::a"], version="3.0") - add_file(c, "model/a.cf", "import modI", "add mod C::a", "3.2") - make_module_simple_deps(reporoot, "D", ["F", "G"]) - e = make_module_simple_deps(reporoot, "E", ["H"], version="3.0") - add_file(e, "model/a.cf", "import modJ", "add mod E::a", "3.2") - make_module_simple_deps(reporoot, "F") - make_module_simple_deps(reporoot, "G") - make_module_simple_deps(reporoot, "H") - make_module_simple_deps(reporoot, "I") - make_module_simple_deps(reporoot, "J") + make_module_simple_deps(reporoot, "a", ["b", "c", "d"], project=True) + make_module_simple_deps(reporoot, "b") + c = make_module_simple_deps(reporoot, "c", ["e", "f", "e::a"], version="3.0") + add_file(c, "model/a.cf", "import modi", "add mod c::a", "3.2") + make_module_simple_deps(reporoot, "d", ["f", "g"]) + e = make_module_simple_deps(reporoot, "e", ["h"], version="3.0") + add_file(e, "model/a.cf", "import modj", "add mod e::a", "3.2") + make_module_simple_deps(reporoot, "f") + make_module_simple_deps(reporoot, "g") + make_module_simple_deps(reporoot, "h") + make_module_simple_deps(reporoot, "i") + make_module_simple_deps(reporoot, "j") return reporoot diff --git a/tests/moduletool/test_add.py b/tests/moduletool/test_add.py index 63621a86aa..2594bf15f1 100644 --- a/tests/moduletool/test_add.py +++ b/tests/moduletool/test_add.py @@ -23,8 +23,8 @@ import py import pytest -from pkg_resources import Requirement +import inmanta.util from inmanta import env from inmanta.command import CLIException from inmanta.module import ModuleV1, ModuleV1Metadata, ModuleV2, ModuleV2Source, Project, ProjectMetadata @@ -89,7 +89,7 @@ def test_module_add_v2_module_to_project( dest_dir=os.path.join(tmpdir, f"elaboratev2module-v{version}"), new_version=Version(version), publish_index=pip_index, - new_extras={"optional": [Requirement.parse("inmanta-module-minimalv2module")]}, + new_extras={"optional": [inmanta.util.parse_requirement(requirement="inmanta-module-minimalv2module")]}, ) # Create project diff --git a/tests/moduletool/test_convert_v1_v2.py b/tests/moduletool/test_convert_v1_v2.py index e1b07f9608..de5526686a 100644 --- a/tests/moduletool/test_convert_v1_v2.py +++ b/tests/moduletool/test_convert_v1_v2.py @@ -25,9 +25,9 @@ import py import pytest -from pkg_resources import Requirement from pytest import MonkeyPatch +import inmanta.util import toml from inmanta import moduletool from inmanta.command import CLIException @@ -114,7 +114,7 @@ def test_issue_3159_conversion_std_module_add_self_to_dependencies(tmpdir): parser = configparser.ConfigParser() parser.read(setup_cfg_file) assert parser.has_option("options", "install_requires") - install_requires = [Requirement.parse(r) for r in parser.get("options", "install_requires").split("\n") if r] + install_requires = inmanta.util.parse_requirements(parser.get("options", "install_requires").split("\n")) pkg_names = [r.name for r in install_requires] assert "inmanta-module-std" not in pkg_names diff --git a/tests/moduletool/test_freeze.py b/tests/moduletool/test_freeze.py index 6cda52f9eb..7e3f047ebf 100644 --- a/tests/moduletool/test_freeze.py +++ b/tests/moduletool/test_freeze.py @@ -32,42 +32,42 @@ @pytest.mark.slowtest def test_freeze_basic(git_modules_dir: str, modules_repo: str, tmpdir): - install_project(git_modules_dir, "projectA", tmpdir) + install_project(git_modules_dir, "projecta", tmpdir) modtool = ModuleTool() - cmod = modtool.get_module("modC") - assert cmod.get_freeze("modC", recursive=False, mode="==") == {"std": "== 3.2", "modE": "== 3.2", "modF": "== 3.2"} - assert cmod.get_freeze("modC", recursive=True, mode="==") == { + cmod = modtool.get_module("modc") + assert cmod.get_freeze("modc", recursive=False, mode="==") == {"std": "== 3.2", "mode": "== 3.2", "modf": "== 3.2"} + assert cmod.get_freeze("modc", recursive=True, mode="==") == { "std": "== 3.2", - "modE": "== 3.2", - "modF": "== 3.2", - "modH": "== 3.2", - "modJ": "== 3.2", + "mode": "== 3.2", + "modf": "== 3.2", + "modh": "== 3.2", + "modj": "== 3.2", } - assert cmod.get_freeze("modC::a", recursive=False, mode="==") == {"std": "== 3.2", "modI": "== 3.2"} + assert cmod.get_freeze("modc::a", recursive=False, mode="==") == {"std": "== 3.2", "modi": "== 3.2"} @pytest.mark.slowtest def test_project_freeze_basic(git_modules_dir: str, modules_repo: str, tmpdir): - install_project(git_modules_dir, "projectA", tmpdir) + install_project(git_modules_dir, "projecta", tmpdir) modtool = ModuleTool() proj = modtool.get_project() assert proj.get_freeze(recursive=False, mode="==") == { "std": "== 3.2", - "modB": "== 3.2", - "modC": "== 3.2", - "modD": "== 3.2", + "modb": "== 3.2", + "modc": "== 3.2", + "modd": "== 3.2", } assert proj.get_freeze(recursive=True, mode="==") == { "std": "== 3.2", - "modB": "== 3.2", - "modC": "== 3.2", - "modD": "== 3.2", - "modE": "== 3.2", - "modF": "== 3.2", - "modG": "== 3.2", - "modH": "== 3.2", - "modJ": "== 3.2", + "modb": "== 3.2", + "modc": "== 3.2", + "modd": "== 3.2", + "mode": "== 3.2", + "modf": "== 3.2", + "modg": "== 3.2", + "modh": "== 3.2", + "modj": "== 3.2", } @@ -85,7 +85,7 @@ def test_project_freeze_bad(git_modules_dir: str, modules_repo: str, tmpdir): @pytest.mark.slowtest def test_project_freeze(git_modules_dir: str, modules_repo: str, capsys, tmpdir): - coroot = install_project(git_modules_dir, "projectA", tmpdir) + coroot = install_project(git_modules_dir, "projecta", tmpdir) app(["project", "freeze", "-o", "-"]) @@ -95,16 +95,16 @@ def test_project_freeze(git_modules_dir: str, modules_repo: str, capsys, tmpdir) assert len(err) == 0, err assert ( out - == """name: projectA + == """name: projecta license: Apache 2.0 version: 0.0.1 modulepath: libs downloadpath: libs repo: %s requires: -- modB ~= 3.2 -- modC ~= 3.2 -- modD ~= 3.2 +- modb ~= 3.2 +- modc ~= 3.2 +- modd ~= 3.2 - std ~= 3.2 """ % modules_repo @@ -113,7 +113,7 @@ def test_project_freeze(git_modules_dir: str, modules_repo: str, capsys, tmpdir) @pytest.mark.slowtest def test_project_freeze_disk(git_modules_dir: str, modules_repo: str, capsys, tmpdir): - coroot = install_project(git_modules_dir, "projectA", tmpdir) + coroot = install_project(git_modules_dir, "projecta", tmpdir) app(["project", "freeze"]) @@ -125,16 +125,16 @@ def test_project_freeze_disk(git_modules_dir: str, modules_repo: str, capsys, tm with open(os.path.join(coroot, "project.yml"), encoding="utf-8") as fh: assert ( fh.read() - == """name: projectA + == """name: projecta license: Apache 2.0 version: 0.0.1 modulepath: libs downloadpath: libs repo: %s requires: -- modB ~= 3.2 -- modC ~= 3.2 -- modD ~= 3.2 +- modb ~= 3.2 +- modc ~= 3.2 +- modd ~= 3.2 - std ~= 3.2 """ % modules_repo @@ -143,7 +143,7 @@ def test_project_freeze_disk(git_modules_dir: str, modules_repo: str, capsys, tm @pytest.mark.slowtest def test_project_freeze_odd_opperator(git_modules_dir: str, modules_repo: str, tmpdir): - coroot = install_project(git_modules_dir, "projectA", tmpdir) + coroot = install_project(git_modules_dir, "projecta", tmpdir) # Start a new subprocess, because inmanta-cli executes sys.exit() when an invalid argument is used. process = subprocess.Popen( @@ -165,10 +165,10 @@ def test_project_freeze_odd_opperator(git_modules_dir: str, modules_repo: str, t def test_project_options_in_config(git_modules_dir: str, modules_repo: str, capsys, tmpdir): coroot = install_project( git_modules_dir, - "projectA", + "projecta", tmpdir, config_content=f""" -name: projectA +name: projecta license: Apache 2.0 version: 0.0.1 modulepath: libs @@ -188,7 +188,7 @@ def verify(): with open("project.yml", encoding="utf-8") as fh: assert fh.read() == ( - """name: projectA + """name: projecta license: Apache 2.0 version: 0.0.1 modulepath: libs @@ -197,14 +197,14 @@ def verify(): freeze_recursive: true freeze_operator: == requires: -- modB == 3.2 -- modC == 3.2 -- modD == 3.2 -- modE == 3.2 -- modF == 3.2 -- modG == 3.2 -- modH == 3.2 -- modJ == 3.2 +- modb == 3.2 +- modc == 3.2 +- modd == 3.2 +- mode == 3.2 +- modf == 3.2 +- modg == 3.2 +- modh == 3.2 +- modj == 3.2 - std == 3.2 """ % modules_repo @@ -218,7 +218,7 @@ def verify(): @pytest.mark.slowtest def test_module_freeze(git_modules_dir: str, modules_repo: str, capsys, tmpdir): - coroot = install_project(git_modules_dir, "projectA", tmpdir) + coroot = install_project(git_modules_dir, "projecta", tmpdir) def verify(): out, err = capsys.readouterr() @@ -226,24 +226,24 @@ def verify(): assert os.path.getsize(os.path.join(coroot, "project.yml")) != 0 assert len(err) == 0, err assert out == ( - """name: modC + """name: modc license: Apache 2.0 version: '3.2' requires: -- modE ~= 3.2 -- modF ~= 3.2 -- modI ~= 3.2 +- mode ~= 3.2 +- modf ~= 3.2 +- modi ~= 3.2 - std ~= 3.2 """ ) - app(["module", "-m", "modC", "freeze", "-o", "-"]) + app(["module", "-m", "modc", "freeze", "-o", "-"]) verify() @pytest.mark.slowtest def test_module_freeze_self_disk(git_modules_dir: str, modules_repo: str, capsys, tmpdir): - coroot = install_project(git_modules_dir, "projectA", tmpdir) + coroot = install_project(git_modules_dir, "projecta", tmpdir) def verify(): out, err = capsys.readouterr() @@ -251,25 +251,25 @@ def verify(): assert len(err) == 0, err assert len(out) == 0, out - modpath = os.path.join(coroot, "libs/modC/module.yml") + modpath = os.path.join(coroot, "libs/modc/module.yml") assert os.path.getsize(os.path.join(coroot, "project.yml")) != 0 assert os.path.getsize(modpath) != 0 with open(modpath, encoding="utf-8") as fh: outf = fh.read() assert outf == ( - """name: modC + """name: modc license: Apache 2.0 version: '3.2' requires: -- modE ~= 3.2 -- modF ~= 3.2 -- modI ~= 3.2 +- mode ~= 3.2 +- modf ~= 3.2 +- modi ~= 3.2 - std ~= 3.2 """ ) - modp = os.path.join(coroot, "libs/modC") + modp = os.path.join(coroot, "libs/modc") app(["project", "install"]) os.chdir(modp) app(["module", "freeze"]) diff --git a/tests/moduletool/test_install.py b/tests/moduletool/test_install.py index 01dc8f0693..66d5b5041f 100644 --- a/tests/moduletool/test_install.py +++ b/tests/moduletool/test_install.py @@ -30,8 +30,8 @@ import py import pytest import yaml -from pkg_resources import Requirement +import inmanta.util from inmanta import compiler, const, env, loader, module from inmanta.ast import CompilerException from inmanta.command import CLIException @@ -252,14 +252,14 @@ def test_module_install_conflicting_requirements(tmpdir: py.path.local, snippetc os.path.join(modules_v2_dir, "minimalv2module"), os.path.join(str(tmpdir), "modone"), new_name="modone", - new_requirements=[Requirement.parse("lorem~=0.0.1")], + new_requirements=[inmanta.util.parse_requirement(requirement="lorem~=0.0.1")], install=True, ) module_from_template( os.path.join(modules_v2_dir, "minimalv2module"), os.path.join(str(tmpdir), "modtwo"), new_name="modtwo", - new_requirements=[Requirement.parse("lorem~=0.1.0")], + new_requirements=[inmanta.util.parse_requirement(requirement="lorem~=0.1.0")], install=True, ) @@ -509,7 +509,10 @@ def test_project_install( index_url=local_module_package_index, # We add tornado, as there is a code path in update for the case where the project has python requires python_requires=["tornado"] - + [Requirement.parse(module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names], + + [ + inmanta.util.parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) + for mod in install_module_names + ], install_project=False, ) @@ -541,7 +544,10 @@ def test_project_install( "\n".join(f"import {mod}" for mod in ["std", *install_module_names]), autostd=False, python_package_sources=[local_module_package_index], - python_requires=[Requirement.parse(module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names] + python_requires=[ + inmanta.util.parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) + for mod in install_module_names + ] + ["lorem"], install_project=False, ) @@ -676,7 +682,7 @@ def test_project_install_modules_cache_invalid( index_url=index.url, extra_index_url=[local_module_package_index], # make sure main module gets installed, pulling in newest version of dependency module - python_requires=[Requirement.parse(module.ModuleV2Source.get_package_name_for(main_module))], + python_requires=[inmanta.util.parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(main_module))], ) # populate project.modules[dependency_module] to force the error conditions in this simplified example @@ -761,7 +767,7 @@ def test_project_install_incompatible_versions( install_project=False, add_to_module_path=[v1_modules_path], index_url=index.url, - python_requires=[Requirement.parse(module.ModuleV2Source.get_package_name_for(v2_mod_name))], + python_requires=[inmanta.util.parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(v2_mod_name))], ) # install project @@ -814,14 +820,14 @@ def test_project_install_incompatible_dependencies( v2_template_path, os.path.join(str(tmpdir), "v2mod2"), new_name="v2mod2", - new_requirements=[Requirement.parse("inmanta-module-v2mod1~=1.0.0")], + new_requirements=[inmanta.util.parse_requirement(requirement="inmanta-module-v2mod1~=1.0.0")], publish_index=index, ) v2mod3: module.ModuleV2Metadata = module_from_template( v2_template_path, os.path.join(str(tmpdir), "v2mod3"), new_name="v2mod3", - new_requirements=[Requirement.parse("inmanta-module-v2mod1~=2.0.0")], + new_requirements=[inmanta.util.parse_requirement(requirement="inmanta-module-v2mod1~=2.0.0")], publish_index=index, ) @@ -835,7 +841,9 @@ def test_project_install_incompatible_dependencies( install_project=False, index_url=index.url, python_requires=[ - Requirement.parse(module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata))) + inmanta.util.parse_requirement( + requirement=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) + ) for metadata in [v2mod2, v2mod3] ], ) @@ -917,7 +925,9 @@ def test_install_from_index_dont_leak_pip_index( # Installing a V2 module requires a python package source. index_url="unknown", python_requires=[ - Requirement.parse(module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata))) + inmanta.util.parse_requirement( + requirement=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) + ) for metadata in [v2mod1] ], ) @@ -977,7 +987,9 @@ def test_install_with_use_config( index_url=index.url if not use_pip_config else None, use_pip_config_file=use_pip_config, python_requires=[ - Requirement.parse(module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata))) + inmanta.util.parse_requirement( + requirement=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) + ) for metadata in [v2mod1] ], ) @@ -1044,7 +1056,9 @@ def test_install_with_use_config_extra_index( extra_index_url=[index2.url], use_pip_config_file=True, python_requires=[ - Requirement.parse(module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata))) + inmanta.util.parse_requirement( + requirement=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) + ) for metadata in [v2mod1, v2mod2] ], ) @@ -1078,7 +1092,7 @@ def test_install_with_use_config_but_PIP_CONFIG_FILE_not_set( autostd=False, install_project=False, use_pip_config_file=True, - python_requires=[Requirement.parse("inmanta-module-dummy-module")], + python_requires=[inmanta.util.parse_requirement(requirement="inmanta-module-dummy-module")], ) # install project @@ -1195,7 +1209,7 @@ def test_install_project_with_install_mode_master(tmpdir: py.path.local, snippet autostd=False, install_project=False, add_to_module_path=[str(tmpdir)], - project_requires=[InmantaModuleRequirement(Requirement.parse("mod11==3.2.1"))], + project_requires=[InmantaModuleRequirement(inmanta.util.parse_requirement(requirement="mod11==3.2.1"))], install_mode=InstallMode.master, ) @@ -1223,7 +1237,7 @@ def test_module_install_logging(local_module_package_index: str, snippetcompiler v2_module = "minimalv2module" - v2_requirements = [Requirement.parse(module.ModuleV2Source.get_package_name_for(v2_module))] + v2_requirements = [inmanta.util.parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(v2_module))] # set up project and modules project: module.Project = snippetcompiler_clean.setup_for_snippet( @@ -1324,7 +1338,9 @@ def test_pip_output(local_module_package_index: str, snippetcompiler_clean, capl ) modules = ["modone", "modtwo"] - v2_requirements = [Requirement.parse(module.ModuleV2Source.get_package_name_for(mod)) for mod in modules] + v2_requirements = [ + inmanta.util.parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in modules + ] snippetcompiler_clean.setup_for_snippet( f""" @@ -1410,7 +1426,9 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[Requirement.parse(module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[ + inmanta.util.parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module")) + ], install_project=True, ) log_contains( @@ -1442,7 +1460,9 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[Requirement.parse(module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[ + inmanta.util.parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module")) + ], install_project=True, ) @@ -1475,7 +1495,9 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[Requirement.parse(module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[ + inmanta.util.parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module")) + ], install_project=True, ) log_contains( @@ -1544,7 +1566,7 @@ def test_version_snapshot(local_module_package_index: str, snippetcompiler_clean autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[Requirement.parse(module.ModuleV2Source.get_package_name_for("module_b"))], + python_requires=[inmanta.util.parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("module_b"))], install_project=True, ) @@ -1570,7 +1592,7 @@ def test_version_snapshot(local_module_package_index: str, snippetcompiler_clean autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[Requirement.parse(module.ModuleV2Source.get_package_name_for("module_c"))], + python_requires=[inmanta.util.parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("module_c"))], install_project=True, ) @@ -1641,7 +1663,8 @@ def test_constraints_logging_v2(modules_v2_dir, tmpdir, caplog, snippetcompiler_ index_url=local_module_package_index, extra_index_url=[index.url], python_requires=[ - Requirement.parse(module.ModuleV2Source.get_package_name_for(mod)) for mod in ["module_b", "module_a"] + inmanta.util.parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) + for mod in ["module_b", "module_a"] ], install_project=True, project_requires=[ diff --git a/tests/moduletool/test_update.py b/tests/moduletool/test_update.py index 00ed58f59d..1c9f9bc9e8 100644 --- a/tests/moduletool/test_update.py +++ b/tests/moduletool/test_update.py @@ -20,8 +20,8 @@ import py.path import pytest -from pkg_resources import Requirement +import inmanta.util from inmanta import env from inmanta.config import Config from inmanta.data.model import PipConfig @@ -127,7 +127,9 @@ def assert_version_installed(module_name: str, version: str) -> None: new_version=Version(current_version), new_name=module_name, new_requirements=( - [InmantaModuleRequirement(Requirement.parse("module2<3.0.0"))] if module_name == "module1" else None + [InmantaModuleRequirement(inmanta.util.parse_requirement(requirement="module2<3.0.0"))] + if module_name == "module1" + else None ), install=False, publish_index=pip_index, @@ -142,7 +144,7 @@ def assert_version_installed(module_name: str, version: str) -> None: # Add a dependency on module2, without setting an explicit version constraint. Later version of module1 # do set a version constraint on the dependency on module2. This way it is verified whether the module update # command takes into account the version constraints set in a new version of a module. - new_requirements=[InmantaModuleRequirement(Requirement.parse("module2"))], + new_requirements=[InmantaModuleRequirement(inmanta.util.parse_requirement(requirement="module2"))], install=False, publish_index=pip_index, new_content_init_cf="entity" if corrupt_module else None, # Introduce syntax error in the module @@ -239,7 +241,11 @@ def test_module_update_dependencies( create_python_package("a", Version("1.0.0"), str(tmpdir.join("a-1.0.0")), publish_index=index) for v in ("1.0.0", "1.0.1", "2.0.0"): create_python_package( - "b", Version(v), str(tmpdir.join(f"b-{v}")), requirements=[Requirement.parse("c")], publish_index=index + "b", + Version(v), + str(tmpdir.join(f"b-{v}")), + requirements=[inmanta.util.parse_requirement(requirement="c")], + publish_index=index, ) for v in ("1.0.0", "2.0.0"): create_python_package("c", Version(v), str(tmpdir.join(f"c-{v}")), publish_index=index) @@ -254,7 +260,7 @@ def test_module_update_dependencies( # install b-1.0.0 and c-1.0.0 env.process_env.install_for_config( - [Requirement.parse(req) for req in ("b==1.0.0", "c==1.0.0")], + [inmanta.util.parse_requirement(requirement=req) for req in ("b==1.0.0", "c==1.0.0")], config=PipConfig( index_url=index.url, use_system_config=False, @@ -266,7 +272,7 @@ def test_module_update_dependencies( source_dir=os.path.join(modules_dir, "minimalv1module"), dest_dir=str(tmpdir.join("modules", "my_mod")), new_name="my_mod", - new_requirements=[Requirement.parse(req) for req in ("a", "b~=1.0.0")], + new_requirements=inmanta.util.parse_requirements(["a", "b~=1.0.0"]), ) # run `inmanta project update` without running install first diff --git a/tests/server/test_compilerservice.py b/tests/server/test_compilerservice.py index c030026f8e..5d06c574bf 100644 --- a/tests/server/test_compilerservice.py +++ b/tests/server/test_compilerservice.py @@ -30,13 +30,13 @@ from collections import abc from typing import TYPE_CHECKING, Optional -import pkg_resources import py.path import pytest from pytest import approx import inmanta.ast.export as ast_export import inmanta.data.model as model +import inmanta.util import utils from inmanta import config, data from inmanta.const import INMANTA_REMOVED_SET_ID, ParameterSource @@ -1807,7 +1807,7 @@ def patch_get_protected_inmanta_packages(): venv = PythonEnvironment(env_path=venv_path) assert name_protected_pkg not in venv.get_installed_packages() venv.install_for_config( - requirements=[pkg_resources.Requirement.parse(name_protected_pkg)], + requirements=[inmanta.util.parse_requirement(requirement=name_protected_pkg)], config=PipConfig( index_url=local_module_package_index, ), diff --git a/tests/test_env.py b/tests/test_env.py index 3fbb1b1704..8372c8bf33 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -30,11 +30,10 @@ from typing import Callable, LiteralString, Optional from unittest.mock import patch -import pkg_resources import py import pytest -from pkg_resources import Requirement +import inmanta.util from inmanta import env, loader, module from inmanta.data.model import PipConfig from inmanta.env import Pip @@ -178,7 +177,7 @@ def test_gen_req_file(): # make sure they all parse for req in reqs: - pkg_resources.parse_requirements(req) + inmanta.util.parse_requirement(requirement=req) def test_environment_python_version_multi_digit(tmpdir: py.path.local) -> None: @@ -207,7 +206,7 @@ def test_process_env_install_from_index( package_name: str = "more-itertools" assert package_name not in env.process_env.get_installed_packages() env.process_env.install_for_config( - [Requirement.parse(package_name + (f"=={version}" if version is not None else ""))], + [inmanta.util.parse_requirement(requirement=package_name + (f"=={version}" if version is not None else ""))], config=PipConfig( use_system_config=True, # we need an upstream for some packages ), @@ -222,7 +221,7 @@ def test_process_env_install_from_index( # It should hit the cache there and return here. # Cheap and fast test env.process_env.install_from_index( - [Requirement.parse(package_name + (f"=={version}" if version is not None else ""))], + [inmanta.util.parse_requirement(requirement=package_name + (f"=={version}" if version is not None else ""))], use_pip_config=True, ) @@ -272,7 +271,7 @@ def test_process_env_install_from_index_not_found_env_var( with pytest.raises(env.PackageNotFound, match=re.escape(expected)): env.process_env.install_for_config( - [Requirement.parse("this-package-does-not-exist")], + [inmanta.util.parse_requirement(requirement="this-package-does-not-exist")], config=PipConfig( index_url=index_urls[0], # The first element should only be passed to the index_url. If there are indexes in the environment @@ -309,7 +308,7 @@ def test_process_env_install_no_index(tmpdir: py.path.local, monkeypatch, use_sy with pytest.raises(env.PackageNotFound, match=re.escape(expected)): env.process_env.install_for_config( - requirements=[Requirement.parse("this-package-does-not-exist")], + requirements=[inmanta.util.parse_requirement(requirement="this-package-does-not-exist")], paths=[env.LocalPackagePath(path=str(tmpdir))], config=PipConfig(use_system_config=use_system_config), ) @@ -326,7 +325,7 @@ def test_process_env_install_from_index_conflicting_reqs( package_name: str = "more-itertools" with pytest.raises(env.ConflictingRequirements) as e: env.process_env.install_for_config( - [Requirement.parse(f"{package_name}{version}") for version in [">8.5", "<=8"]], + [inmanta.util.parse_requirement(requirement=f"{package_name}{version}") for version in [">8.5", "<=8"]], config=PipConfig( use_system_config=True, # we need an upstream for some packages ), @@ -396,8 +395,7 @@ def test_active_env_get_module_file( loader.PluginModuleFinder.configure_module_finder([os.path.join(str(tmpdir), "libs")]) assert env.ActiveEnv.get_module_file(module_name) is None - - env.process_env.install_for_config([Requirement.parse(package_name)], pip_config, add_inmanta_requires=False) + env.process_env.install_for_config([inmanta.util.parse_requirement(requirement=package_name)], pip_config) assert package_name in env.process_env.get_installed_packages() module_info: Optional[tuple[Optional[str], Loader]] = env.ActiveEnv.get_module_file(module_name) assert module_info is not None @@ -454,7 +452,7 @@ def test_active_env_get_module_file_editable_namespace_package( def create_install_package( - name: str, version: version.Version, requirements: list[Requirement], local_module_package_index: str + name: str, version: version.Version, requirements: list[inmanta.util.CanonicalRequirement], local_module_package_index: str ) -> None: """ Creates and installs a simple package with specified requirements. Creates package in a temporary directory and @@ -542,7 +540,10 @@ def assert_all_checks(expect_test: tuple[bool, str] = (True, ""), expect_nonext: create_install_package("test-package-one", version.Version("1.0.0"), [], local_module_package_index) assert_all_checks() create_install_package( - "test-package-two", version.Version("1.0.0"), [Requirement.parse("test-package-one~=1.0")], local_module_package_index + "test-package-two", + version.Version("1.0.0"), + [inmanta.util.parse_requirement(requirement="test-package-one~=1.0")], + local_module_package_index, ) assert_all_checks() create_install_package("test-package-one", version.Version("2.0.0"), [], local_module_package_index) @@ -562,7 +563,7 @@ def test_active_env_check_constraints(caplog, tmpvenv_active_inherit: str, local """ caplog.set_level(logging.WARNING) in_scope: Pattern[str] = re.compile("test-package-.*") - constraints: list[Requirement] = [Requirement.parse("test-package-one~=1.0")] + constraints: list[inmanta.util.CanonicalRequirement] = [inmanta.util.parse_requirement(requirement="test-package-one~=1.0")] env.process_env.check(in_scope) @@ -579,7 +580,10 @@ def test_active_env_check_constraints(caplog, tmpvenv_active_inherit: str, local # setup for #4761 caplog.clear() create_install_package( - "ext-package-one", version.Version("1.0.0"), [Requirement.parse("test-package-one==1.0")], local_module_package_index + "ext-package-one", + version.Version("1.0.0"), + [inmanta.util.parse_requirement(requirement="test-package-one==1.0")], + local_module_package_index, ) env.process_env.check(in_scope, constraints) assert "Incompatibility between constraint" not in caplog.text @@ -609,7 +613,7 @@ def test_override_inmanta_package(tmpvenv_active_inherit: env.VirtualEnv) -> Non installed_pkgs = tmpvenv_active_inherit.get_installed_packages() assert "inmanta-core" in installed_pkgs, "The inmanta-core package should be installed to run the tests" - inmanta_requirements = Requirement.parse("inmanta-core==4.0.0") + inmanta_requirements = inmanta.util.parse_requirement(requirement="inmanta-core==4.0.0") with pytest.raises(env.ConflictingRequirements) as excinfo: tmpvenv_active_inherit.install_for_config( requirements=[inmanta_requirements], @@ -649,13 +653,13 @@ def test_cache_on_active_env(tmpvenv_active_inherit: env.ActiveEnv, local_module """ def _assert_install(requirement: str, installed: bool) -> None: - parsed_requirement = Requirement.parse(requirement) + parsed_requirement = inmanta.util.parse_requirement(requirement=requirement) for r in [requirement, parsed_requirement]: assert tmpvenv_active_inherit.are_installed(requirements=[r]) == installed _assert_install("inmanta-module-elaboratev2module==1.2.3", installed=False) tmpvenv_active_inherit.install_for_config( - requirements=[Requirement.parse("inmanta-module-elaboratev2module==1.2.3")], + requirements=[inmanta.util.parse_requirement(requirement="inmanta-module-elaboratev2module==1.2.3")], config=PipConfig( index_url=local_module_package_index, ), @@ -699,7 +703,7 @@ def test_are_installed_dependency_cycle_on_extra(tmpdir, tmpvenv_active_inherit: path=os.path.join(tmpdir, "pkg"), publish_index=pip_index, optional_dependencies={ - "optional-pkg": [Requirement.parse("dep[optional-dep]")], + "optional-pkg": [inmanta.util.parse_requirement(requirement="dep[optional-dep]")], }, ) create_python_package( @@ -708,11 +712,11 @@ def test_are_installed_dependency_cycle_on_extra(tmpdir, tmpvenv_active_inherit: path=os.path.join(tmpdir, "dep"), publish_index=pip_index, optional_dependencies={ - "optional-dep": [Requirement.parse("pkg[optional-pkg]")], + "optional-dep": [inmanta.util.parse_requirement(requirement="pkg[optional-pkg]")], }, ) - requirements = [Requirement.parse("pkg[optional-pkg]")] + requirements = [inmanta.util.parse_requirement(requirement="pkg[optional-pkg]")] tmpvenv_active_inherit.install_for_config( requirements=requirements, config=PipConfig( diff --git a/tests/test_file_parser.py b/tests/test_file_parser.py index 3103971dca..65d53b4da7 100644 --- a/tests/test_file_parser.py +++ b/tests/test_file_parser.py @@ -17,15 +17,19 @@ """ import os +import pathlib -from pkg_resources import Requirement +import pytest +import inmanta.util +import packaging.requirements from inmanta.file_parser import RequirementsTxtParser def test_requirements_txt_parser(tmpdir) -> None: content = """ test==1.2.3 + # A comment other-dep~=2.0.0 third-dep<5.0.0 # another comment @@ -40,27 +44,45 @@ def test_requirements_txt_parser(tmpdir) -> None: fd.write(content) expected_requirements = ["test==1.2.3", "other-dep~=2.0.0", "third-dep<5.0.0", "splitteddep", "Capital"] - requirements: list[Requirement] = RequirementsTxtParser().parse(requirements_txt_file) - assert requirements == [Requirement.parse(r) for r in expected_requirements] + requirements: list[inmanta.util.CanonicalRequirement] = RequirementsTxtParser().parse(requirements_txt_file) + assert requirements == inmanta.util.parse_requirements(expected_requirements) requirements_as_str = RequirementsTxtParser.parse_requirements_as_strs(requirements_txt_file) assert requirements_as_str == expected_requirements + parsed_canonical_requirements_from_file = inmanta.util.parse_requirements_from_file(pathlib.Path(requirements_txt_file)) + assert parsed_canonical_requirements_from_file == requirements + + problematic_requirements = [ + "test==1.2.3", + "other-dep~=2.0.0", + "third-dep<5.0.0 # another comment", + "splitteddep", + "Capital", + ] + + parsed_canonical_requirements = inmanta.util.parse_requirements(expected_requirements) + assert parsed_canonical_requirements == requirements + + with pytest.raises(Exception) as e: + inmanta.util.parse_requirements(problematic_requirements) + assert "Expected end or semicolon (after version specifier)\n third-dep<5.0.0 # another comment\n" in str(e.value) + new_content = RequirementsTxtParser.get_content_with_dep_removed(requirements_txt_file, remove_dep_on_pkg="test") - assert ( - new_content - == """ + expected_content = """ + # A comment other-dep~=2.0.0 third-dep<5.0.0 # another comment splitteddep Capital """ - ) + assert new_content == expected_content new_content = RequirementsTxtParser.get_content_with_dep_removed(requirements_txt_file, remove_dep_on_pkg="third-dep") assert ( new_content == """ test==1.2.3 + # A comment other-dep~=2.0.0 splitteddep @@ -72,6 +94,7 @@ def test_requirements_txt_parser(tmpdir) -> None: new_content == """ test==1.2.3 + # A comment other-dep~=2.0.0 third-dep<5.0.0 # another comment @@ -83,9 +106,56 @@ def test_requirements_txt_parser(tmpdir) -> None: new_content == """ test==1.2.3 + # A comment other-dep~=2.0.0 third-dep<5.0.0 # another comment splitteddep """ ) + + +@pytest.mark.parametrize( + "iteration", + [ + ("", True), + ("#", True), + (" # ", True), + ("#this is a comment", True), + ("test==1.2.3", False), + ("other-dep~=2.0.0", False), + ], +) +def test_canonical_requirement(iteration) -> None: + """ + Ensure that empty name requirements are not allowed in `Requirement` + """ + name, should_fail = iteration + if should_fail: + with pytest.raises(packaging.requirements.InvalidRequirement): + inmanta.util.parse_requirement(requirement=name) + else: + inmanta.util.parse_requirement(requirement=name) + + +@pytest.mark.parametrize( + "iteration", + [ + ("", ""), + ("#", "#"), + (" # ", "#"), + ("#this is a comment", "#this is a comment"), + ("test==1.2.3", "test==1.2.3"), + ("other-dep~=2.0.0", "other-dep~=2.0.0"), + ("test==1.2.3 # a command", "test==1.2.3"), + ("other-dep #~=2.0.0", "other-dep"), + ("other-dep#~=2.0.0", "other-dep#~=2.0.0"), + ], +) +def test_drop_comment_part(iteration) -> None: + """ + Ensure that empty name requirements are not allowed in `Requirement` + """ + value, expected_value = iteration + current_value = inmanta.util.remove_comment_part_from_specifier(value) + assert current_value == expected_value diff --git a/tests/test_module_loader.py b/tests/test_module_loader.py index ee5975a87c..a9f7a11d13 100644 --- a/tests/test_module_loader.py +++ b/tests/test_module_loader.py @@ -28,8 +28,8 @@ import py import pytest -from pkg_resources import Requirement +import inmanta.util from inmanta import compiler, const, env, loader, plugins, resources from inmanta.ast import CompilerException from inmanta.const import CF_CACHE_DIR @@ -346,7 +346,7 @@ def test_load_module_recursive_v2_module_depends_on_v1( project = snippetcompiler.setup_for_snippet( snippet="import v2_depends_on_v1", index_url=local_module_package_index, - python_requires=[Requirement.parse("inmanta-module-v2-depends-on-v1")], + python_requires=[inmanta.util.parse_requirement(requirement="inmanta-module-v2-depends-on-v1")], install_project=False, ) if preload_v1_module: @@ -373,7 +373,7 @@ def test_load_module_recursive_complex_module_dependencies(local_module_package_ snippet="import complex_module_dependencies_mod1", autostd=False, index_url=local_module_package_index, - python_requires=[Requirement.parse("inmanta-module-complex-module-dependencies-mod1")], + python_requires=[inmanta.util.parse_requirement(requirement="inmanta-module-complex-module-dependencies-mod1")], install_project=False, ) assert "complex_module_dependencies_mod1" not in project.modules @@ -397,7 +397,7 @@ def test_load_import_based_v2_project(local_module_package_index: str, snippetco """ module_name: str = "minimalv2module" - def load(requires: Optional[list[Requirement]] = None) -> None: + def load(requires: Optional[list[inmanta.util.CanonicalRequirement]] = None) -> None: project: Project = snippetcompiler_clean.setup_for_snippet( f"import {module_name}", autostd=False, @@ -412,7 +412,7 @@ def load(requires: Optional[list[Requirement]] = None) -> None: with pytest.raises(ModuleLoadingException, match=f"Failed to load module {module_name}"): load() # assert that it doesn't raise an error with explicit requirements set - load([Requirement.parse(ModuleV2Source.get_package_name_for(module_name))]) + load([inmanta.util.parse_requirement(requirement=ModuleV2Source.get_package_name_for(module_name))]) @pytest.mark.parametrize("v1", [True, False]) @@ -469,7 +469,9 @@ def test_load_import_based_v2_module( extra_index_url=[index.url], # make sure that even listing the requirement in project.yml does not suffice project_requires=[InmantaModuleRequirement.parse(dependency_module_name)], - python_requires=[] if v1 else [Requirement.parse(ModuleV2Source.get_package_name_for(main_module_name))], + python_requires=( + [] if v1 else [inmanta.util.parse_requirement(requirement=ModuleV2Source.get_package_name_for(main_module_name))] + ), ) if explicit_dependency: @@ -609,7 +611,9 @@ def test_project_requirements_dont_overwrite_core_requirements_source( module_name: str = "minimalv2module" module_path: str = str(tmpdir.join(module_name)) module_from_template( - os.path.join(modules_v2_dir, module_name), module_path, new_requirements=[Requirement.parse("Jinja2==2.11.3")] + os.path.join(modules_v2_dir, module_name), + module_path, + new_requirements=[inmanta.util.parse_requirement(requirement="Jinja2==2.11.3")], ) # Activate the snippetcompiler venv @@ -653,7 +657,7 @@ def test_project_requirements_dont_overwrite_core_requirements_index( module_from_template( os.path.join(modules_v2_dir, module_name), module_path, - new_requirements=[Requirement.parse("Jinja2==2.11.3")], + new_requirements=[inmanta.util.parse_requirement(requirement="Jinja2==2.11.3")], publish_index=index, ) @@ -704,7 +708,11 @@ def test_module_conflicting_dependencies_with_v2_modules( # Create a python package y with version 1.0.0 that depends on x~=1.0.0 create_python_package( - "y", Version("1.0.0"), str(tmpdir.join("y-1.0.0")), requirements=[Requirement.parse("x~=1.0.0")], publish_index=index + "y", + Version("1.0.0"), + str(tmpdir.join("y-1.0.0")), + requirements=[inmanta.util.parse_requirement(requirement="x~=1.0.0")], + publish_index=index, ) # Create the first module @@ -713,7 +721,7 @@ def test_module_conflicting_dependencies_with_v2_modules( module_from_template( os.path.join(modules_v2_dir, module_name1), module_path1, - new_requirements=[Requirement.parse("y~=1.0.0")], + new_requirements=[inmanta.util.parse_requirement(requirement="y~=1.0.0")], publish_index=index, ) @@ -724,7 +732,7 @@ def test_module_conflicting_dependencies_with_v2_modules( os.path.join(modules_v2_dir, "minimalv2module"), module_path2, new_name="minimalv2module2", - new_requirements=[Requirement.parse("x~=2.0.0")], + new_requirements=[inmanta.util.parse_requirement(requirement="x~=2.0.0")], publish_index=index, ) @@ -784,7 +792,7 @@ def test_module_conflicting_dependencies_with_v1_module( os.path.join(modules_dir, module_name1), module_path1, new_name="modulev1", - new_requirements=[Requirement.parse("y~=1.0.0")], + new_requirements=[inmanta.util.parse_requirement(requirement="y~=1.0.0")], ) # Create the second module @@ -793,7 +801,7 @@ def test_module_conflicting_dependencies_with_v1_module( module_from_template( os.path.join(modules_v2_dir, module_name2), module_path2, - new_requirements=[Requirement.parse("y~=2.0.0")], + new_requirements=[inmanta.util.parse_requirement(requirement="y~=2.0.0")], publish_index=index, ) @@ -841,11 +849,13 @@ def test_module_install_extra_on_project_level_v2_dep( new_name="mymod", new_requirements=[], new_extras={ - "myfeature": [Requirement.parse(package_name_extra)], + "myfeature": [inmanta.util.parse_requirement(requirement=package_name_extra)], }, publish_index=index, ) - package_with_extra: Requirement = InmantaModuleRequirement.parse("mymod[myfeature]").get_python_package_requirement() + package_with_extra: inmanta.inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( + "mymod[myfeature]" + ).get_python_package_requirement() package_name: str = f"{ModuleV2.PKG_NAME_PREFIX}mymod" # project with dependency on mymod with extra @@ -890,7 +900,7 @@ def test_module_install_extra_on_dep_of_v2_module( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [Requirement.parse(package_name_extra)], + "myfeature": [inmanta.util.parse_requirement(requirement=package_name_extra)], }, publish_index=index, ) @@ -910,7 +920,7 @@ def test_module_install_extra_on_dep_of_v2_module( install_project=True, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[Requirement.parse("inmanta-module-myv2mod")], + python_requires=[inmanta.util.parse_requirement(requirement="inmanta-module-myv2mod")], autostd=False, ) @@ -947,7 +957,7 @@ def test_module_install_extra_on_dep_of_v1_module( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [Requirement.parse(package_name_extra)], + "myfeature": [inmanta.util.parse_requirement(requirement=package_name_extra)], }, publish_index=index, ) @@ -997,12 +1007,16 @@ def test_module_install_extra_on_project_level_v2_dep_update_scenario( new_name="mymod", new_requirements=[], new_extras={ - "myfeature": [Requirement.parse(package_name_extra)], + "myfeature": [inmanta.util.parse_requirement(requirement=package_name_extra)], }, publish_index=index, ) - package_without_extra: Requirement = InmantaModuleRequirement.parse("mymod").get_python_package_requirement() - package_with_extra: Requirement = InmantaModuleRequirement.parse("mymod[myfeature]").get_python_package_requirement() + package_without_extra: inmanta.inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( + "mymod" + ).get_python_package_requirement() + package_with_extra: inmanta.inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( + "mymod[myfeature]" + ).get_python_package_requirement() package_name: str = str(package_without_extra) def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> None: @@ -1066,12 +1080,16 @@ def test_module_install_extra_on_dep_of_v2_module_update_scenario( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [Requirement.parse(package_name_extra)], + "myfeature": [inmanta.util.parse_requirement(requirement=package_name_extra)], }, publish_index=index, ) - package_without_extra: Requirement = InmantaModuleRequirement.parse("depmod").get_python_package_requirement() - package_with_extra: Requirement = InmantaModuleRequirement.parse("depmod[myfeature]").get_python_package_requirement() + package_without_extra: inmanta.inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( + "depmod" + ).get_python_package_requirement() + package_with_extra: inmanta.inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( + "depmod[myfeature]" + ).get_python_package_requirement() package_name: str = str(package_without_extra) def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> None: @@ -1095,7 +1113,7 @@ def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> install_project=True, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[Requirement.parse("inmanta-module-myv2mod==1.0.0")], + python_requires=[inmanta.util.parse_requirement(requirement="inmanta-module-myv2mod==1.0.0")], autostd=False, ) assert_installed(extra_installed=False) @@ -1117,7 +1135,7 @@ def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> install_project=not do_project_update, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[Requirement.parse("inmanta-module-myv2mod==2.0.0")], + python_requires=[inmanta.util.parse_requirement(requirement="inmanta-module-myv2mod==2.0.0")], autostd=False, ) if do_project_update: @@ -1151,8 +1169,12 @@ def test_module_install_extra_on_dep_of_v1_module_update_scenario( index: PipIndex = PipIndex(artifact_dir=str(tmpdir.join(".index"))) # Publish dependency of V1 module (depmod) to python package repo - package_without_extra: Requirement = InmantaModuleRequirement.parse("depmod").get_python_package_requirement() - package_with_extra: Requirement = InmantaModuleRequirement.parse("depmod[myfeature]").get_python_package_requirement() + package_without_extra: inmanta.inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( + "depmod" + ).get_python_package_requirement() + package_with_extra: inmanta.inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( + "depmod[myfeature]" + ).get_python_package_requirement() package_name: str = str(package_without_extra) module_from_template( @@ -1161,7 +1183,7 @@ def test_module_install_extra_on_dep_of_v1_module_update_scenario( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [Requirement.parse(package_name_extra)], + "myfeature": [inmanta.util.parse_requirement(requirement=package_name_extra)], }, publish_index=index, ) @@ -1225,7 +1247,7 @@ async def test_v1_module_depends_on_third_party_dep_with_extra( os.path.join(tmpdir, "myv1mod"), new_name="myv1mod", new_content_init_cf="", - new_requirements=[Requirement.parse("pkg[optional-a]")], + new_requirements=[inmanta.util.parse_requirement(requirement="pkg[optional-a]")], ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv1mod", @@ -1245,7 +1267,7 @@ async def test_v1_module_depends_on_third_party_dep_with_extra( os.path.join(tmpdir, "myv1mod"), new_name="myv1mod", new_content_init_cf="", - new_requirements=[Requirement.parse("pkg[optional-a,optional-b]")], + new_requirements=[inmanta.util.parse_requirement(requirement="pkg[optional-a,optional-b]")], ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv1mod", @@ -1272,13 +1294,13 @@ async def test_v2_module_depends_on_third_party_dep_with_extra( str(tmpdir.join("myv2mod")), new_name="myv2mod", new_version=Version("1.0.0"), - new_requirements=[Requirement.parse("pkg[optional-a]")], + new_requirements=[inmanta.util.parse_requirement(requirement="pkg[optional-a]")], publish_index=index, ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv2mod", install_project=True, - python_requires=[Requirement.parse("inmanta-module-myv2mod==1.0.0")], + python_requires=[inmanta.util.parse_requirement(requirement="inmanta-module-myv2mod==1.0.0")], index_url=index.url, extra_index_url=[index_with_pkgs_containing_optional_deps], autostd=False, @@ -1294,13 +1316,13 @@ async def test_v2_module_depends_on_third_party_dep_with_extra( str(tmpdir.join("myv2mod")), new_name="myv2mod", new_version=Version("2.0.0"), - new_requirements=[Requirement.parse("pkg[optional-a,optional-b]")], + new_requirements=[inmanta.util.parse_requirement(requirement="pkg[optional-a,optional-b]")], publish_index=index, ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv2mod", install_project=True, - python_requires=[Requirement.parse("inmanta-module-myv2mod==2.0.0")], + python_requires=[inmanta.util.parse_requirement(requirement="inmanta-module-myv2mod==2.0.0")], index_url=index.url, extra_index_url=[index_with_pkgs_containing_optional_deps], autostd=False, diff --git a/tests/utils.py b/tests/utils.py index 4e4203fed5..4ba84551ec 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -35,10 +35,12 @@ import pytest import yaml -from pkg_resources import Requirement, parse_version import build import build.env +import inmanta.util +import packaging.requirements +import packaging.version from _pytest.mark import MarkDecorator from inmanta import config, const, data, env, module, protocol, util from inmanta.data import ResourceIdStr @@ -49,7 +51,6 @@ from inmanta.server.extensions import ProductMetadata from inmanta.util import get_compiler_version, hash_file from libpip2pi.commands import dir2pi -from packaging import version T = TypeVar("T") @@ -425,7 +426,7 @@ def get_product_meta_data() -> ProductMetadata: def product_version_lower_or_equal_than(version: str) -> bool: - return parse_version(get_product_meta_data().version) <= parse_version(version) + return packaging.version.Version(version=get_product_meta_data().version) <= packaging.version.Version(version=version) def mark_only_for_version_higher_than(version: str) -> "MarkDecorator": @@ -454,14 +455,14 @@ def publish(self) -> None: def create_python_package( name: str, - pkg_version: version.Version, + pkg_version: packaging.version.Version, path: str, *, - requirements: Optional[Sequence[Requirement]] = None, + requirements: Optional[Sequence[inmanta.util.CanonicalRequirement]] = None, install: bool = False, editable: bool = False, publish_index: Optional[PipIndex] = None, - optional_dependencies: Optional[dict[str, Sequence[Requirement]]] = None, + optional_dependencies: Optional[dict[str, Sequence[inmanta.util.CanonicalRequirement]]] = None, ) -> None: """ Creates an empty Python package. @@ -544,10 +545,12 @@ def module_from_template( source_dir: str, dest_dir: Optional[str] = None, *, - new_version: Optional[version.Version] = None, + new_version: Optional[packaging.version.Version] = None, new_name: Optional[str] = None, - new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, Requirement]]] = None, - new_extras: Optional[abc.Mapping[str, abc.Sequence[Union[module.InmantaModuleRequirement, Requirement]]]] = None, + new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, inmanta.util.CanonicalRequirement]]] = None, + new_extras: Optional[ + abc.Mapping[str, abc.Sequence[Union[module.InmantaModuleRequirement, inmanta.util.CanonicalRequirement]]] + ] = None, install: bool = False, editable: bool = False, publish_index: Optional[PipIndex] = None, @@ -576,9 +579,12 @@ def module_from_template( """ def to_python_requires( - requires: abc.Sequence[Union[module.InmantaModuleRequirement, Requirement]] - ) -> abc.Iterator[Requirement]: - return (str(req if isinstance(req, Requirement) else req.get_python_package_requirement()) for req in requires) + requires: abc.Sequence[Union[module.InmantaModuleRequirement, inmanta.util.CanonicalRequirement]] + ) -> list[str]: + return [ + str(req) if isinstance(req, packaging.requirements.Requirement) else str(req.get_python_package_requirement()) + for req in requires + ] if (dest_dir is None) != in_place: raise ValueError("Either dest_dir or in_place must be set, never both.") @@ -657,9 +663,9 @@ def v1_module_from_template( source_dir: str, dest_dir: str, *, - new_version: Optional[version.Version] = None, + new_version: Optional[packaging.version.Version] = None, new_name: Optional[str] = None, - new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, Requirement]]] = None, + new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, inmanta.util.CanonicalRequirement]]] = None, new_content_init_cf: Optional[str] = None, new_content_init_py: Optional[str] = None, ) -> module.ModuleV2Metadata: @@ -699,7 +705,11 @@ def v1_module_from_template( with open(os.path.join(dest_dir, "requirements.txt"), "w") as fd: fd.write( "\n".join( - str(req if isinstance(req, Requirement) else req.get_python_package_requirement()) + ( + str(req) + if isinstance(req, packaging.requirements.Requirement) + else str(req.get_python_package_requirement()) + ) for req in new_requirements ) )