From 37ee4b12c8c78bb0b6f9983192a6904e5168d532 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 31 Jul 2024 13:57:50 +0200 Subject: [PATCH 01/74] first commit --- docs/conf.py | 2 +- src/inmanta/agent/agent.py | 5 +- src/inmanta/agent/executor.py | 5 +- src/inmanta/env.py | 68 ++++++++++++++------------ src/inmanta/file_parser.py | 9 ++-- src/inmanta/main.py | 4 +- src/inmanta/module.py | 52 +++++++++++--------- src/inmanta/moduletool.py | 11 ++--- src/inmanta/server/extensions.py | 6 +-- tests/conftest.py | 9 ++-- tests/moduletool/test_add.py | 4 +- tests/moduletool/test_convert_v1_v2.py | 4 +- tests/moduletool/test_install.py | 56 ++++++++++++--------- tests/moduletool/test_update.py | 14 +++--- tests/server/test_compilerservice.py | 4 +- tests/test_env.py | 42 +++++++++------- tests/test_file_parser.py | 5 +- tests/test_module_loader.py | 60 +++++++++++++---------- tests/utils.py | 5 +- 19 files changed, 196 insertions(+), 169 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 0959468460..e679b93db2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,7 +12,7 @@ # serve to show the default. import importlib.metadata import shutil -import sys, os, pkg_resources, datetime +import sys, os, datetime from importlib.metadata import PackageNotFoundError from sphinx.errors import ConfigError diff --git a/src/inmanta/agent/agent.py b/src/inmanta/agent/agent.py index 2e4a3a4800..c7b1f86633 100644 --- a/src/inmanta/agent/agent.py +++ b/src/inmanta/agent/agent.py @@ -33,9 +33,8 @@ from logging import Logger from typing import Any, Collection, Dict, Optional, Union, cast -import pkg_resources - import inmanta.agent.executor +import packaging.requirements from inmanta import config, const, data, env, protocol from inmanta.agent import config as cfg from inmanta.agent import executor, forking_executor, in_process_executor @@ -1341,7 +1340,7 @@ async def _install(self, blueprint: executor.ExecutorBlueprint) -> None: await loop.run_in_executor( self.thread_pool, self._env.install_for_config, - list(pkg_resources.parse_requirements(blueprint.requirements)), + [packaging.requirements.Requirement(requirement_string=e) for e in blueprint.requirements], blueprint.pip_config, ) await loop.run_in_executor(self.thread_pool, self._loader.deploy_version, blueprint.sources) diff --git a/src/inmanta/agent/executor.py b/src/inmanta/agent/executor.py index 5ce4ec852a..c0258da9cd 100644 --- a/src/inmanta/agent/executor.py +++ b/src/inmanta/agent/executor.py @@ -32,9 +32,8 @@ from dataclasses import dataclass from typing import Any, Dict, Optional, Sequence -import pkg_resources - import inmanta.types +import packaging.requirements from inmanta.agent import config as cfg from inmanta.data.model import PipConfig, ResourceIdStr, ResourceType, ResourceVersionIdStr from inmanta.env import PythonEnvironment @@ -270,7 +269,7 @@ async def create_and_install_environment(self, blueprint: EnvBlueprint) -> None: if len(req): # install_for_config expects at least 1 requirement or a path to install install_for_config = functools.partial( self.install_for_config, - requirements=list(pkg_resources.parse_requirements(req)), + requirements=[packaging.requirements.Requirement(requirement_string=e) for e in req], config=blueprint.pip_config, ) await loop.run_in_executor(self.thread_pool, install_for_config) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 1645864b73..f83545f4db 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -32,15 +32,13 @@ from functools import reduce from importlib.abc import Loader from importlib.machinery import ModuleSpec +from importlib.metadata import Distribution from itertools import chain from re import Pattern from subprocess import CalledProcessError from textwrap import indent from typing import NamedTuple, Optional, TypeVar -import pkg_resources -from pkg_resources import DistInfoDistribution, Distribution, Requirement - from inmanta import const from inmanta.ast import CompilerException from inmanta.data.model import LEGACY_PIP_DEFAULT, PipConfig @@ -48,6 +46,7 @@ from inmanta.stable_api import stable_api from inmanta.util import strtobool from packaging import version +from packaging.requirements import Requirement LOGGER = logging.getLogger(__name__) LOGGER_PIP = logging.getLogger("inmanta.pip") # Use this logger to log pip commands or data related to pip commands. @@ -166,7 +165,7 @@ def _get_as_requirements_type(cls, requirements: req_list) -> Sequence[Requireme Convert requirements from Union[Sequence[str], Sequence[Requirement]] to Sequence[Requirement] """ if isinstance(requirements[0], str): - return [Requirement.parse(r) for r in requirements] + return [Requirement(requirement_string=r) for r in requirements] else: return requirements @@ -205,15 +204,21 @@ def _are_installed_recursive( if r.marker and not r.marker.evaluate(environment=environment_marker_evaluation): # The marker of the requirement doesn't apply on this environment continue - if r.key not in installed_packages or str(installed_packages[r.key]) not in r: + if r.name not in installed_packages or str(installed_packages[r.name]) not in r: return False if r.extras: for extra in r.extras: - distribution: Optional[Distribution] = pkg_resources.working_set.find(r) + search_dist = [e for e in importlib.metadata.distributions() if e.name == extra] + assert len(search_dist) <= 1 + distribution: Optional[Distribution] = None if len(search_dist) == 0 else search_dist[0] + extra_packages = [ + Requirement(requirement_string=e) for e in distribution.metadata.json["provides_extra"] + ] + if distribution is None: return False - pkgs_required_by_extra: set[Requirement] = set(distribution.requires(extras=(extra,))) - set( - distribution.requires(extras=()) + pkgs_required_by_extra: set[Requirement] = set(extra_packages) - set( + [Requirement(requirement_string=e) for e in distribution.requires] ) if not _are_installed_recursive( reqs=list(pkgs_required_by_extra), @@ -229,20 +234,16 @@ def _are_installed_recursive( @classmethod def get_packages_in_working_set(cls, inmanta_modules_only: bool = False) -> dict[str, version.Version]: """ - Return all packages present in `pkg_resources.working_set` together with the version of the package. + Return all packages present in `importlib.metadata.distributions()` together with the version of the package. :param inmanta_modules_only: Only return inmanta modules from the working set """ return { - dist_info.key: version.Version(dist_info.version) - for dist_info in pkg_resources.working_set - if not inmanta_modules_only or dist_info.key.startswith(const.MODULE_PKG_NAME_PREFIX) + dist_info.name: version.Version(dist_info.version) + for dist_info in importlib.metadata.distributions() + if not inmanta_modules_only or dist_info.name.startswith(const.MODULE_PKG_NAME_PREFIX) } - @classmethod - def rebuild_working_set(cls) -> None: - pkg_resources.working_set = pkg_resources.WorkingSet._build_master() - @classmethod def get_dependency_tree(cls, dists: abc.Iterable[str]) -> abc.Set[str]: """ @@ -255,7 +256,7 @@ def get_dependency_tree(cls, dists: abc.Iterable[str]) -> abc.Set[str]: """ # create dict for O(1) lookup installed_distributions: abc.Mapping[str, Distribution] = { - dist_info.key: dist_info for dist_info in pkg_resources.working_set + dist_info.name: dist_info for dist_info in importlib.metadata.distributions() } def _get_tree_recursive(dists: abc.Iterable[str], acc: abc.Set[str] = frozenset()) -> abc.Set[str]: @@ -840,7 +841,7 @@ def install_from_list( use_pip_config was ignored on ISO6 and it still is """ self.install_from_index( - requirements=[Requirement.parse(r) for r in requirements_list], + requirements=[Requirement(requirement_string=r) for r in requirements_list], upgrade=upgrade, upgrade_strategy=upgrade_strategy, use_pip_config=True, @@ -867,7 +868,11 @@ def _get_requirements_on_inmanta_package(cls) -> Sequence[Requirement]: """ protected_inmanta_packages: list[str] = cls.get_protected_inmanta_packages() workingset: dict[str, version.Version] = PythonWorkingSet.get_packages_in_working_set() - return [Requirement.parse(f"{pkg}=={workingset[pkg]}") for pkg in workingset if pkg in protected_inmanta_packages] + return [ + Requirement(requirement_string=f"{pkg}=={workingset[pkg]}") + for pkg in workingset + if pkg in protected_inmanta_packages + ] class CommandRunner: @@ -995,9 +1000,9 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: # all requirements of all packages installed in this environment installed_constraints: abc.Set[OwnedRequirement] = frozenset( - OwnedRequirement(requirement, dist_info.key) - for dist_info in pkg_resources.working_set - for requirement in dist_info.requires() + OwnedRequirement(requirement, dist_info.name) + for dist_info in importlib.metadata.distributions() + for requirement in dist_info.requires ) inmanta_constraints: abc.Set[OwnedRequirement] = frozenset( OwnedRequirement(r, owner="inmanta-core") for r in cls._get_requirements_on_inmanta_package() @@ -1013,10 +1018,14 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: ( [] if strict_scope is None - else (dist_info.key for dist_info in pkg_resources.working_set if strict_scope.fullmatch(dist_info.key)) + else ( + dist_info.name + for dist_info in importlib.metadata.distributions() + if strict_scope.fullmatch(dist_info.name) + ) ), - (requirement.requirement.key for requirement in inmanta_constraints), - (requirement.requirement.key for requirement in extra_constraints), + (requirement.requirement.name for requirement in inmanta_constraints), + (requirement.requirement.name for requirement in extra_constraints), ) ) @@ -1086,13 +1095,13 @@ def check_legacy(cls, in_scope: Pattern[str], constraints: Optional[list[Require in_scope, constraints ) - working_set: abc.Iterable[DistInfoDistribution] = pkg_resources.working_set + working_set: abc.Iterable[Distribution] = importlib.metadata.distributions() # add all requirements of all in scope packages installed in this environment all_constraints: set[Requirement] = set(constraints if constraints is not None else []).union( - requirement + Requirement(requirement_string=requirement) for dist_info in working_set - if in_scope.fullmatch(dist_info.key) - for requirement in dist_info.requires() + if in_scope.fullmatch(dist_info.name) + for requirement in dist_info.requires ) installed_versions: dict[str, version.Version] = PythonWorkingSet.get_packages_in_working_set() @@ -1167,7 +1176,6 @@ def notify_change(self) -> None: are executed in a subprocess. """ importlib.reload(mod) - PythonWorkingSet.rebuild_working_set() process_env: ActiveEnv = ActiveEnv(python_path=sys.executable) diff --git a/src/inmanta/file_parser.py b/src/inmanta/file_parser.py index 900844b5fb..0df7316247 100644 --- a/src/inmanta/file_parser.py +++ b/src/inmanta/file_parser.py @@ -18,8 +18,7 @@ import os -from pkg_resources import Requirement - +from packaging.requirements import Requirement from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap @@ -60,7 +59,7 @@ def parse(cls, filename: str) -> list[Requirement]: """ Get all the requirements in `filename` as a list of `Requirement` instances. """ - return [Requirement.parse(r) for r in cls.parse_requirements_as_strs(filename)] + return [Requirement(requirement_string=r) for r in cls.parse_requirements_as_strs(filename)] @classmethod def parse_requirements_as_strs(cls, filename: str) -> list[str]: @@ -93,14 +92,14 @@ def get_content_with_dep_removed(cls, filename: str, remove_dep_on_pkg: str) -> if line_continuation_buffer: line_continuation_buffer += line if not line.endswith("\\"): - if Requirement.parse(line_continuation_buffer).key != remove_dep_on_pkg: + if Requirement(requirement_string=line_continuation_buffer).key != remove_dep_on_pkg: result += line_continuation_buffer line_continuation_buffer = "" elif not line.strip() or line.strip().startswith("#"): result += line elif line.endswith("\\"): line_continuation_buffer = line - elif Requirement.parse(line).key != remove_dep_on_pkg.lower(): + elif Requirement(requirement_string=line).key != remove_dep_on_pkg.lower(): result += line else: # Dependency matches `remove_dep_on_pkg` => Remove line from result diff --git a/src/inmanta/main.py b/src/inmanta/main.py index 4171a51d02..47335d4139 100644 --- a/src/inmanta/main.py +++ b/src/inmanta/main.py @@ -26,9 +26,9 @@ from typing import Any, Callable, Optional, Union, cast import click +import importlib_metadata import texttable from click_plugins import with_plugins -from pkg_resources import iter_entry_points from inmanta import protocol from inmanta.config import Config, cmdline_rest_transport @@ -182,7 +182,7 @@ def get_table(header: list[str], rows: list[list[str]], data_type: Optional[list return table.draw() -@with_plugins(iter_entry_points("inmanta.cli_plugins")) +@with_plugins(importlib_metadata.entry_points(group="inmanta.cli_plugins")) @click.group(help="Base command") @click.option("--host", help="The server hostname to connect to") @click.option("--port", help="The server port to connect to") diff --git a/src/inmanta/module.py b/src/inmanta/module.py index 0edbe2d4b7..670fcc28e9 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -39,6 +39,7 @@ from enum import Enum from functools import reduce from importlib.abc import Loader +from importlib.metadata import Distribution, PackageNotFoundError from io import BytesIO, TextIOBase from itertools import chain from subprocess import CalledProcessError @@ -47,10 +48,8 @@ from typing import Annotated, ClassVar, Dict, Generic, List, NewType, Optional, TextIO, TypeVar, Union, cast import more_itertools -import pkg_resources import pydantic import yaml -from pkg_resources import Distribution, DistributionNotFound, Requirement, parse_requirements, parse_version from pydantic import BaseModel, Field, NameEmail, StringConstraints, ValidationError, field_validator import inmanta.data.model @@ -68,6 +67,8 @@ from inmanta.util import get_compiler_version from inmanta.warnings import InmantaWarning from packaging import version +from packaging.requirements import Requirement +from packaging.version import Version from ruamel.yaml.comments import CommentedMap try: @@ -97,7 +98,7 @@ class InmantaModuleRequirement: """ def __init__(self, requirement: Requirement) -> None: - if requirement.project_name.startswith(ModuleV2.PKG_NAME_PREFIX): + if requirement.name.startswith(ModuleV2.PKG_NAME_PREFIX): raise ValueError( f"InmantaModuleRequirement instances work with inmanta module names, not python package names. " f"Problematic case: {str(requirement)}" @@ -107,7 +108,7 @@ def __init__(self, requirement: Requirement) -> None: @property def project_name(self) -> str: # Requirement converts all "_" to "-". Inmanta modules use "_" - return self._requirement.project_name.replace("-", "_") + return self._requirement.name.replace("-", "_") @property def key(self) -> str: @@ -150,7 +151,7 @@ def parse(cls: type[TInmantaModuleRequirement], spec: str) -> TInmantaModuleRequ ) if "-" in spec: raise ValueError("Invalid Inmanta module requirement: Inmanta module names use '_', not '-'.") - return cls(Requirement.parse(spec)) + return cls(Requirement(requirement_string=spec)) def get_python_package_requirement(self) -> Requirement: """ @@ -159,7 +160,7 @@ def get_python_package_requirement(self) -> Requirement: module_name = self.project_name pkg_name = ModuleV2Source.get_package_name_for(module_name) pkg_req_str = str(self).replace(module_name, pkg_name, 1) # Replace max 1 occurrence - return Requirement.parse(pkg_req_str) + return Requirement(requirement_string=pkg_req_str) class CompilerExceptionWithExtendedTrace(CompilerException): @@ -690,12 +691,13 @@ def get_installed_version(cls, module_name: str) -> Optional[version.Version]: if module_name.startswith(ModuleV2.PKG_NAME_PREFIX): raise ValueError("PythonRepo instances work with inmanta module names, not Python package names.") try: - dist: Distribution = pkg_resources.get_distribution(ModuleV2Source.get_package_name_for(module_name)) - return version.Version(dist.version) - except DistributionNotFound: + dist: Distribution = Distribution.from_name(ModuleV2Source.get_package_name_for(module_name)) + try: + return version.Version(dist.version) + except version.InvalidVersion: + raise InvalidModuleException(f"Package {dist.name} was installed but it has no valid version.") + except PackageNotFoundError: return None - except version.InvalidVersion: - raise InvalidModuleException(f"Package {dist.project_name} was installed but it has no valid version.") @classmethod def get_inmanta_module_name(cls, python_package_name: str) -> str: @@ -727,7 +729,7 @@ def install(self, project: "Project", module_spec: list[InmantaModuleRequirement # These could be constraints (-c) as well, but that requires additional sanitation # Because for pip not every valid -r is a valid -c current_requires = project.get_strict_python_requirements_as_list() - requirements += [Requirement.parse(r) for r in current_requires] + requirements += [Requirement(requirement_string=r) for r in current_requires] if preinstalled is not None: # log warning if preinstalled version does not match constraints @@ -2124,7 +2126,7 @@ def install_modules(self, *, bypass_module_cache: bool = False, update_dependenc self.verify_module_version_compatibility() # do python install - pyreq: list[Requirement] = [Requirement.parse(x) for x in self.collect_python_requirements()] + pyreq: list[Requirement] = [Requirement(requirement_string=x) for x in self.collect_python_requirements()] if len(pyreq) > 0: # upgrade both direct and transitive module dependencies: eager upgrade strategy @@ -2520,10 +2522,10 @@ def verify_module_version_compatibility(self) -> None: LOGGER.warning("Module %s is present in requires but it is not used by the model.", name) continue module = self.modules[name] - version = parse_version(str(module.version)) + current_version = Version(version=str(module.version)) for r in spec: - if version not in r: - exc_message += f"\n\t* requirement {r} on module {name} not fulfilled, now at version {version}." + if current_version not in r: + exc_message += f"\n\t* requirement {r} on module {name} not fulfilled, now at version {current_version}." if exc_message: exc_message = f"The following requirements were not satisfied:{exc_message}" @@ -2542,7 +2544,9 @@ def verify_python_requires(self) -> None: Verifies no incompatibilities exist within the Python environment with respect to installed module v2 requirements. """ if self.strict_deps_check: - constraints: list[Requirement] = [Requirement.parse(item) for item in self.collect_python_requirements()] + constraints: list[Requirement] = [ + Requirement(requirement_string=item) for item in self.collect_python_requirements() + ] env.ActiveEnv.check(strict_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*"), constraints=constraints) else: if not env.ActiveEnv.check_legacy(in_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*")): @@ -2673,7 +2677,7 @@ def requires(self) -> "List[InmantaModuleRequirement]": # filter on import stmt reqs = [] for spec in self._metadata.requires: - req = [x for x in parse_requirements(spec)] + req = [Requirement(requirement_string=e) for e in spec] if len(req) > 1: print(f"Module file for {self._path} has bad line in requirements specification {spec}") reqe = InmantaModuleRequirement(req[0]) @@ -2810,7 +2814,7 @@ def requires(self) -> "List[InmantaModuleRequirement]": """ reqs = [] for spec in self.get_module_requirements(): - req = [x for x in parse_requirements(spec)] + req = [Requirement(requirement_string=e) for e in spec] if len(req) > 1: print(f"Module file for {self._path} has bad line in requirements specification {spec}") reqe = InmantaModuleRequirement(req[0]) @@ -3202,7 +3206,7 @@ def get_suitable_version_for( def try_parse(x: str) -> Optional[version.Version]: try: - return parse_version(x) + return Version(version=x) except Exception: return None @@ -3213,7 +3217,7 @@ def try_parse(x: str) -> Optional[version.Version]: versions = [x for x in r.specifier.filter(versions, not release_only)] comp_version_raw = get_compiler_version() - comp_version = parse_version(comp_version_raw) + comp_version = Version(version=comp_version_raw) return cls.__best_for_compiler_version(modulename, versions, path, comp_version) @classmethod @@ -3228,7 +3232,7 @@ def get_cv_for(best: version.Version) -> Optional[version.Version]: v = metadata.compiler_version if isinstance(v, (int, float)): v = str(v) - return parse_version(v) + return Version(version=v) if not versions: return None @@ -3296,7 +3300,7 @@ def versions(self) -> list[version.Version]: def try_parse(x: str) -> Optional[version.Version]: try: - return parse_version(x) + return Version(version=x) except Exception: return None @@ -3426,7 +3430,7 @@ def add_module_requirement_persistent(self, requirement: InmantaModuleRequiremen new_install_requires = [ r for r in config_parser.get("options", "install_requires").split("\n") - if r and Requirement.parse(r).key != python_pkg_requirement.key + if r and Requirement(requirement_string=r).key != python_pkg_requirement.key ] new_install_requires.append(str(python_pkg_requirement)) else: diff --git a/src/inmanta/moduletool.py b/src/inmanta/moduletool.py index 403191db99..e318ccd97e 100644 --- a/src/inmanta/moduletool.py +++ b/src/inmanta/moduletool.py @@ -38,14 +38,13 @@ from configparser import ConfigParser from functools import total_ordering from re import Pattern -from typing import IO, TYPE_CHECKING, Any, Optional +from typing import IO, Any, Optional import click import more_itertools import texttable import yaml from cookiecutter.main import cookiecutter -from pkg_resources import Requirement import build import inmanta @@ -76,13 +75,9 @@ gitprovider, ) from inmanta.stable_api import stable_api +from packaging.requirements import InvalidRequirement, Requirement from packaging.version import Version -if TYPE_CHECKING: - from packaging.requirements import InvalidRequirement -else: - from pkg_resources.extern.packaging.requirements import InvalidRequirement - LOGGER = logging.getLogger(__name__) @@ -489,7 +484,7 @@ def do_update(specs: Mapping[str, Sequence[InmantaModuleRequirement]], modules: # Because for pip not every valid -r is a valid -c current_requires = my_project.get_strict_python_requirements_as_list() env.process_env.install_for_config( - v2_python_specs + [Requirement.parse(r) for r in current_requires], + v2_python_specs + [Requirement(requirement_string=r) for r in current_requires], my_project.metadata.pip, upgrade=True, ) diff --git a/src/inmanta/server/extensions.py b/src/inmanta/server/extensions.py index 63069313cb..5a5e680af9 100644 --- a/src/inmanta/server/extensions.py +++ b/src/inmanta/server/extensions.py @@ -16,12 +16,12 @@ Contact: code@inmanta.com """ +import importlib.metadata import logging import os from collections import defaultdict from typing import Any, Generic, Optional, TypeVar -import pkg_resources import yaml from inmanta import data @@ -147,8 +147,8 @@ def _get_product_version(self) -> str: packages = ["inmanta-oss", "inmanta", "inmanta-core"] for package in packages: try: - return pkg_resources.get_distribution(package).version - except pkg_resources.DistributionNotFound: + return importlib.metadata.version(package) + except importlib.metadata.PackageNotFoundError: pass LOGGER.warning("Couldn't determine product version. Make sure inmanta is properly installed.") diff --git a/tests/conftest.py b/tests/conftest.py index 5e651fe12a..7ddb559c80 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -27,6 +27,7 @@ from inmanta.logging import InmantaLoggerConfig from inmanta.protocol import auth from inmanta.util import ScheduledTask, Scheduler, TaskMethod, TaskSchedule +from packaging.requirements import Requirement """ About the use of @parametrize_any and @slowtest: @@ -101,7 +102,6 @@ import pytest from asyncpg.exceptions import DuplicateDatabaseError from click import testing -from pkg_resources import Requirement from pyformance.registry import MetricsRegistry from tornado import netutil @@ -510,11 +510,11 @@ def deactive_venv(): old_pythonpath = os.environ.get("PYTHONPATH", None) old_os_venv: Optional[str] = os.environ.get("VIRTUAL_ENV", None) old_process_env: str = env.process_env.python_path - old_working_set = pkg_resources.working_set old_available_extensions = ( dict(InmantaBootloader.AVAILABLE_EXTENSIONS) if InmantaBootloader.AVAILABLE_EXTENSIONS is not None else None ) + # TODO h we remove some magic here yield os.environ["PATH"] = old_os_path @@ -527,7 +527,6 @@ def deactive_venv(): sys.path_hooks.extend(old_path_hooks) # Clear cache for sys.path_hooks sys.path_importer_cache.clear() - pkg_resources.working_set = old_working_set # Restore PYTHONPATH if old_pythonpath is not None: os.environ["PYTHONPATH"] = old_pythonpath @@ -1913,8 +1912,8 @@ def index_with_pkgs_containing_optional_deps() -> str: path=os.path.join(tmpdirname, "pkg"), publish_index=pip_index, optional_dependencies={ - "optional-a": [Requirement.parse("dep-a")], - "optional-b": [Requirement.parse("dep-b"), Requirement.parse("dep-c")], + "optional-a": [Requirement(requirement_string="dep-a")], + "optional-b": [Requirement(requirement_string="dep-b"), Requirement(requirement_string="dep-c")], }, ) for pkg_name in ["dep-a", "dep-b", "dep-c"]: diff --git a/tests/moduletool/test_add.py b/tests/moduletool/test_add.py index 012b1dffbe..5a84124d61 100644 --- a/tests/moduletool/test_add.py +++ b/tests/moduletool/test_add.py @@ -23,12 +23,12 @@ import py import pytest -from pkg_resources import Requirement from inmanta.command import CLIException from inmanta.env import process_env from inmanta.module import ModuleV1, ModuleV1Metadata, ModuleV2, ModuleV2Source, Project, ProjectMetadata from inmanta.moduletool import ModuleTool +from packaging.requirements import Requirement from packaging.version import Version from utils import PipIndex, module_from_template @@ -89,7 +89,7 @@ def test_module_add_v2_module_to_project( dest_dir=os.path.join(tmpdir, f"elaboratev2module-v{version}"), new_version=Version(version), publish_index=pip_index, - new_extras={"optional": [Requirement.parse("inmanta-module-minimalv2module")]}, + new_extras={"optional": [Requirement(requirement_string="inmanta-module-minimalv2module")]}, ) # Create project diff --git a/tests/moduletool/test_convert_v1_v2.py b/tests/moduletool/test_convert_v1_v2.py index e1b07f9608..de082ac7da 100644 --- a/tests/moduletool/test_convert_v1_v2.py +++ b/tests/moduletool/test_convert_v1_v2.py @@ -25,7 +25,6 @@ import py import pytest -from pkg_resources import Requirement from pytest import MonkeyPatch import toml @@ -34,6 +33,7 @@ from inmanta.module import DummyProject, ModuleV1, ModuleV2, ModuleV2Metadata from inmanta.moduletool import ModuleConverter, ModuleVersionException from packaging import version +from packaging.requirements import Requirement from utils import log_contains, v1_module_from_template @@ -114,7 +114,7 @@ def test_issue_3159_conversion_std_module_add_self_to_dependencies(tmpdir): parser = configparser.ConfigParser() parser.read(setup_cfg_file) assert parser.has_option("options", "install_requires") - install_requires = [Requirement.parse(r) for r in parser.get("options", "install_requires").split("\n") if r] + install_requires = [Requirement(requirement_string=r) for r in parser.get("options", "install_requires").split("\n") if r] pkg_names = [r.name for r in install_requires] assert "inmanta-module-std" not in pkg_names diff --git a/tests/moduletool/test_install.py b/tests/moduletool/test_install.py index 01dc8f0693..137bab4031 100644 --- a/tests/moduletool/test_install.py +++ b/tests/moduletool/test_install.py @@ -30,7 +30,6 @@ import py import pytest import yaml -from pkg_resources import Requirement from inmanta import compiler, const, env, loader, module from inmanta.ast import CompilerException @@ -41,6 +40,7 @@ from inmanta.moduletool import DummyProject, ModuleConverter, ModuleTool, ProjectTool from moduletool.common import BadModProvider, install_project from packaging import version +from packaging.requirements import Requirement from utils import LogSequence, PipIndex, log_contains, module_from_template LOGGER = logging.getLogger(__name__) @@ -252,14 +252,14 @@ def test_module_install_conflicting_requirements(tmpdir: py.path.local, snippetc os.path.join(modules_v2_dir, "minimalv2module"), os.path.join(str(tmpdir), "modone"), new_name="modone", - new_requirements=[Requirement.parse("lorem~=0.0.1")], + new_requirements=[Requirement(requirement_string="lorem~=0.0.1")], install=True, ) module_from_template( os.path.join(modules_v2_dir, "minimalv2module"), os.path.join(str(tmpdir), "modtwo"), new_name="modtwo", - new_requirements=[Requirement.parse("lorem~=0.1.0")], + new_requirements=[Requirement(requirement_string="lorem~=0.1.0")], install=True, ) @@ -509,7 +509,7 @@ def test_project_install( index_url=local_module_package_index, # We add tornado, as there is a code path in update for the case where the project has python requires python_requires=["tornado"] - + [Requirement.parse(module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names], + + [Requirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names], install_project=False, ) @@ -541,7 +541,9 @@ def test_project_install( "\n".join(f"import {mod}" for mod in ["std", *install_module_names]), autostd=False, python_package_sources=[local_module_package_index], - python_requires=[Requirement.parse(module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names] + python_requires=[ + Requirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names + ] + ["lorem"], install_project=False, ) @@ -676,7 +678,7 @@ def test_project_install_modules_cache_invalid( index_url=index.url, extra_index_url=[local_module_package_index], # make sure main module gets installed, pulling in newest version of dependency module - python_requires=[Requirement.parse(module.ModuleV2Source.get_package_name_for(main_module))], + python_requires=[Requirement(requirement_string=module.ModuleV2Source.get_package_name_for(main_module))], ) # populate project.modules[dependency_module] to force the error conditions in this simplified example @@ -761,7 +763,7 @@ def test_project_install_incompatible_versions( install_project=False, add_to_module_path=[v1_modules_path], index_url=index.url, - python_requires=[Requirement.parse(module.ModuleV2Source.get_package_name_for(v2_mod_name))], + python_requires=[Requirement(requirement_string=module.ModuleV2Source.get_package_name_for(v2_mod_name))], ) # install project @@ -814,14 +816,14 @@ def test_project_install_incompatible_dependencies( v2_template_path, os.path.join(str(tmpdir), "v2mod2"), new_name="v2mod2", - new_requirements=[Requirement.parse("inmanta-module-v2mod1~=1.0.0")], + new_requirements=[Requirement(requirement_string="inmanta-module-v2mod1~=1.0.0")], publish_index=index, ) v2mod3: module.ModuleV2Metadata = module_from_template( v2_template_path, os.path.join(str(tmpdir), "v2mod3"), new_name="v2mod3", - new_requirements=[Requirement.parse("inmanta-module-v2mod1~=2.0.0")], + new_requirements=[Requirement(requirement_string="inmanta-module-v2mod1~=2.0.0")], publish_index=index, ) @@ -835,7 +837,9 @@ def test_project_install_incompatible_dependencies( install_project=False, index_url=index.url, python_requires=[ - Requirement.parse(module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata))) + Requirement( + requirement_string=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) + ) for metadata in [v2mod2, v2mod3] ], ) @@ -917,7 +921,9 @@ def test_install_from_index_dont_leak_pip_index( # Installing a V2 module requires a python package source. index_url="unknown", python_requires=[ - Requirement.parse(module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata))) + Requirement( + requirement_string=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) + ) for metadata in [v2mod1] ], ) @@ -977,7 +983,9 @@ def test_install_with_use_config( index_url=index.url if not use_pip_config else None, use_pip_config_file=use_pip_config, python_requires=[ - Requirement.parse(module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata))) + Requirement( + requirement_string=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) + ) for metadata in [v2mod1] ], ) @@ -1044,7 +1052,9 @@ def test_install_with_use_config_extra_index( extra_index_url=[index2.url], use_pip_config_file=True, python_requires=[ - Requirement.parse(module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata))) + Requirement( + requirement_string=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) + ) for metadata in [v2mod1, v2mod2] ], ) @@ -1078,7 +1088,7 @@ def test_install_with_use_config_but_PIP_CONFIG_FILE_not_set( autostd=False, install_project=False, use_pip_config_file=True, - python_requires=[Requirement.parse("inmanta-module-dummy-module")], + python_requires=[Requirement(requirement_string="inmanta-module-dummy-module")], ) # install project @@ -1195,7 +1205,7 @@ def test_install_project_with_install_mode_master(tmpdir: py.path.local, snippet autostd=False, install_project=False, add_to_module_path=[str(tmpdir)], - project_requires=[InmantaModuleRequirement(Requirement.parse("mod11==3.2.1"))], + project_requires=[InmantaModuleRequirement(Requirement(requirement_string="mod11==3.2.1"))], install_mode=InstallMode.master, ) @@ -1223,7 +1233,7 @@ def test_module_install_logging(local_module_package_index: str, snippetcompiler v2_module = "minimalv2module" - v2_requirements = [Requirement.parse(module.ModuleV2Source.get_package_name_for(v2_module))] + v2_requirements = [Requirement(requirement_string=module.ModuleV2Source.get_package_name_for(v2_module))] # set up project and modules project: module.Project = snippetcompiler_clean.setup_for_snippet( @@ -1324,7 +1334,7 @@ def test_pip_output(local_module_package_index: str, snippetcompiler_clean, capl ) modules = ["modone", "modtwo"] - v2_requirements = [Requirement.parse(module.ModuleV2Source.get_package_name_for(mod)) for mod in modules] + v2_requirements = [Requirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in modules] snippetcompiler_clean.setup_for_snippet( f""" @@ -1410,7 +1420,7 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[Requirement.parse(module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[Requirement(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module"))], install_project=True, ) log_contains( @@ -1442,7 +1452,7 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[Requirement.parse(module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[Requirement(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module"))], install_project=True, ) @@ -1475,7 +1485,7 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[Requirement.parse(module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[Requirement(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module"))], install_project=True, ) log_contains( @@ -1544,7 +1554,7 @@ def test_version_snapshot(local_module_package_index: str, snippetcompiler_clean autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[Requirement.parse(module.ModuleV2Source.get_package_name_for("module_b"))], + python_requires=[Requirement(requirement_string=module.ModuleV2Source.get_package_name_for("module_b"))], install_project=True, ) @@ -1570,7 +1580,7 @@ def test_version_snapshot(local_module_package_index: str, snippetcompiler_clean autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[Requirement.parse(module.ModuleV2Source.get_package_name_for("module_c"))], + python_requires=[Requirement(requirement_string=module.ModuleV2Source.get_package_name_for("module_c"))], install_project=True, ) @@ -1641,7 +1651,7 @@ def test_constraints_logging_v2(modules_v2_dir, tmpdir, caplog, snippetcompiler_ index_url=local_module_package_index, extra_index_url=[index.url], python_requires=[ - Requirement.parse(module.ModuleV2Source.get_package_name_for(mod)) for mod in ["module_b", "module_a"] + Requirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in ["module_b", "module_a"] ], install_project=True, project_requires=[ diff --git a/tests/moduletool/test_update.py b/tests/moduletool/test_update.py index ee9240d032..d18219e811 100644 --- a/tests/moduletool/test_update.py +++ b/tests/moduletool/test_update.py @@ -20,7 +20,6 @@ import py.path import pytest -from pkg_resources import Requirement from inmanta.config import Config from inmanta.data.model import PipConfig @@ -29,6 +28,7 @@ from inmanta.moduletool import ProjectTool from inmanta.parser import ParserException from moduletool.common import add_file, clone_repo +from packaging.requirements import Requirement from packaging.version import Version from utils import PipIndex, create_python_package, module_from_template, v1_module_from_template @@ -126,7 +126,9 @@ def assert_version_installed(module_name: str, version: str) -> None: new_version=Version(current_version), new_name=module_name, new_requirements=( - [InmantaModuleRequirement(Requirement.parse("module2<3.0.0"))] if module_name == "module1" else None + [InmantaModuleRequirement(Requirement(requirement_string="module2<3.0.0"))] + if module_name == "module1" + else None ), install=False, publish_index=pip_index, @@ -141,7 +143,7 @@ def assert_version_installed(module_name: str, version: str) -> None: # Add a dependency on module2, without setting an explicit version constraint. Later version of module1 # do set a version constraint on the dependency on module2. This way it is verified whether the module update # command takes into account the version constraints set in a new version of a module. - new_requirements=[InmantaModuleRequirement(Requirement.parse("module2"))], + new_requirements=[InmantaModuleRequirement(Requirement(requirement_string="module2"))], install=False, publish_index=pip_index, new_content_init_cf="entity" if corrupt_module else None, # Introduce syntax error in the module @@ -238,7 +240,7 @@ def test_module_update_dependencies( create_python_package("a", Version("1.0.0"), str(tmpdir.join("a-1.0.0")), publish_index=index) for v in ("1.0.0", "1.0.1", "2.0.0"): create_python_package( - "b", Version(v), str(tmpdir.join(f"b-{v}")), requirements=[Requirement.parse("c")], publish_index=index + "b", Version(v), str(tmpdir.join(f"b-{v}")), requirements=[Requirement(requirement_string="c")], publish_index=index ) for v in ("1.0.0", "2.0.0"): create_python_package("c", Version(v), str(tmpdir.join(f"c-{v}")), publish_index=index) @@ -253,7 +255,7 @@ def test_module_update_dependencies( # install b-1.0.0 and c-1.0.0 process_env.install_for_config( - [Requirement.parse(req) for req in ("b==1.0.0", "c==1.0.0")], + [Requirement(requirement_string=req) for req in ("b==1.0.0", "c==1.0.0")], config=PipConfig( index_url=index.url, use_system_config=False, @@ -265,7 +267,7 @@ def test_module_update_dependencies( source_dir=os.path.join(modules_dir, "minimalv1module"), dest_dir=str(tmpdir.join("modules", "my_mod")), new_name="my_mod", - new_requirements=[Requirement.parse(req) for req in ("a", "b~=1.0.0")], + new_requirements=[Requirement(requirement_string=req) for req in ("a", "b~=1.0.0")], ) # run `inmanta project update` without running install first diff --git a/tests/server/test_compilerservice.py b/tests/server/test_compilerservice.py index c030026f8e..a035ee399e 100644 --- a/tests/server/test_compilerservice.py +++ b/tests/server/test_compilerservice.py @@ -30,7 +30,6 @@ from collections import abc from typing import TYPE_CHECKING, Optional -import pkg_resources import py.path import pytest from pytest import approx @@ -51,6 +50,7 @@ from inmanta.server.services.compilerservice import CompilerService, CompileRun, CompileStateListener from inmanta.server.services.notificationservice import NotificationService from inmanta.util import ensure_directory_exist +from packaging.requirements import Requirement from server.conftest import EnvironmentFactory from utils import LogSequence, report_db_index_usage, retry_limited, wait_for_version @@ -1807,7 +1807,7 @@ def patch_get_protected_inmanta_packages(): venv = PythonEnvironment(env_path=venv_path) assert name_protected_pkg not in venv.get_installed_packages() venv.install_for_config( - requirements=[pkg_resources.Requirement.parse(name_protected_pkg)], + requirements=[Requirement(requirement_string=name_protected_pkg)], config=PipConfig( index_url=local_module_package_index, ), diff --git a/tests/test_env.py b/tests/test_env.py index 937ee6b58a..6e0ec185e7 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -30,15 +30,14 @@ from typing import Callable, LiteralString, Optional from unittest.mock import patch -import pkg_resources import py import pytest -from pkg_resources import Requirement from inmanta import env, loader, module from inmanta.data.model import PipConfig from inmanta.env import Pip from packaging import version +from packaging.requirements import Requirement from utils import LogSequence, PipIndex, create_python_package if "inmanta-core" in env.process_env.get_installed_packages(only_editable=True): @@ -153,6 +152,7 @@ def _list_dir(path: str, ignore: list[str]) -> list[str]: subprocess.check_output([os.path.join(venv.env_path, "bin/pip"), "list"]) +# TODO h FROM Requirement.parse( TO Requirement(requirement_string= def test_gen_req_file(): """ These are all examples used in older testcases that did not work correctly before @@ -178,7 +178,7 @@ def test_gen_req_file(): # make sure they all parse for req in reqs: - pkg_resources.parse_requirements(req) + Requirement(requirement_string=req) def test_environment_python_version_multi_digit(tmpdir: py.path.local) -> None: @@ -207,7 +207,7 @@ def test_process_env_install_from_index( package_name: str = "more-itertools" assert package_name not in env.process_env.get_installed_packages() env.process_env.install_for_config( - [Requirement.parse(package_name + (f"=={version}" if version is not None else ""))], + [Requirement(requirement_string=package_name + (f"=={version}" if version is not None else ""))], config=PipConfig( use_system_config=True, # we need an upstream for some packages ), @@ -222,7 +222,7 @@ def test_process_env_install_from_index( # It should hit the cache there and return here. # Cheap and fast test env.process_env.install_from_index( - [Requirement.parse(package_name + (f"=={version}" if version is not None else ""))], + [Requirement(requirement_string=package_name + (f"=={version}" if version is not None else ""))], use_pip_config=True, ) @@ -272,7 +272,7 @@ def test_process_env_install_from_index_not_found_env_var( with pytest.raises(env.PackageNotFound, match=re.escape(expected)): env.process_env.install_for_config( - [Requirement.parse("this-package-does-not-exist")], + [Requirement(requirement_string="this-package-does-not-exist")], config=PipConfig( index_url=index_urls[0], # The first element should only be passed to the index_url. If there are indexes in the environment @@ -309,7 +309,7 @@ def test_process_env_install_no_index(tmpdir: py.path.local, monkeypatch, use_sy with pytest.raises(env.PackageNotFound, match=re.escape(expected)): env.process_env.install_for_config( - requirements=[Requirement.parse("this-package-does-not-exist")], + requirements=[Requirement(requirement_string="this-package-does-not-exist")], paths=[env.LocalPackagePath(path=str(tmpdir))], config=PipConfig(use_system_config=use_system_config), ) @@ -326,7 +326,7 @@ def test_process_env_install_from_index_conflicting_reqs( package_name: str = "more-itertools" with pytest.raises(env.ConflictingRequirements) as e: env.process_env.install_for_config( - [Requirement.parse(f"{package_name}{version}") for version in [">8.5", "<=8"]], + [Requirement(requirement_string=f"{package_name}{version}") for version in [">8.5", "<=8"]], config=PipConfig( use_system_config=True, # we need an upstream for some packages ), @@ -396,7 +396,7 @@ def test_active_env_get_module_file( loader.PluginModuleFinder.configure_module_finder([os.path.join(str(tmpdir), "libs")]) assert env.ActiveEnv.get_module_file(module_name) is None - env.process_env.install_for_config([Requirement.parse(package_name)], pip_config) + env.process_env.install_for_config([Requirement(requirement_string=package_name)], pip_config) assert package_name in env.process_env.get_installed_packages() module_info: Optional[tuple[Optional[str], Loader]] = env.ActiveEnv.get_module_file(module_name) assert module_info is not None @@ -540,7 +540,10 @@ def assert_all_checks(expect_test: tuple[bool, str] = (True, ""), expect_nonext: create_install_package("test-package-one", version.Version("1.0.0"), [], local_module_package_index) assert_all_checks() create_install_package( - "test-package-two", version.Version("1.0.0"), [Requirement.parse("test-package-one~=1.0")], local_module_package_index + "test-package-two", + version.Version("1.0.0"), + [Requirement(requirement_string="test-package-one~=1.0")], + local_module_package_index, ) assert_all_checks() create_install_package("test-package-one", version.Version("2.0.0"), [], local_module_package_index) @@ -560,7 +563,7 @@ def test_active_env_check_constraints(caplog, tmpvenv_active_inherit: str, local """ caplog.set_level(logging.WARNING) in_scope: Pattern[str] = re.compile("test-package-.*") - constraints: list[Requirement] = [Requirement.parse("test-package-one~=1.0")] + constraints: list[Requirement] = [Requirement(requirement_string="test-package-one~=1.0")] env.ActiveEnv.check(in_scope) @@ -577,7 +580,10 @@ def test_active_env_check_constraints(caplog, tmpvenv_active_inherit: str, local # setup for #4761 caplog.clear() create_install_package( - "ext-package-one", version.Version("1.0.0"), [Requirement.parse("test-package-one==1.0")], local_module_package_index + "ext-package-one", + version.Version("1.0.0"), + [Requirement(requirement_string="test-package-one==1.0")], + local_module_package_index, ) env.ActiveEnv.check(in_scope, constraints) assert "Incompatibility between constraint" not in caplog.text @@ -607,7 +613,7 @@ def test_override_inmanta_package(tmpvenv_active_inherit: env.VirtualEnv) -> Non installed_pkgs = tmpvenv_active_inherit.get_installed_packages() assert "inmanta-core" in installed_pkgs, "The inmanta-core package should be installed to run the tests" - inmanta_requirements = Requirement.parse("inmanta-core==4.0.0") + inmanta_requirements = Requirement(requirement_string="inmanta-core==4.0.0") with pytest.raises(env.ConflictingRequirements) as excinfo: tmpvenv_active_inherit.install_for_config( requirements=[inmanta_requirements], @@ -647,13 +653,13 @@ def test_cache_on_active_env(tmpvenv_active_inherit: env.ActiveEnv, local_module """ def _assert_install(requirement: str, installed: bool) -> None: - parsed_requirement = Requirement.parse(requirement) + parsed_requirement = Requirement(requirement_string=requirement) for r in [requirement, parsed_requirement]: assert tmpvenv_active_inherit.are_installed(requirements=[r]) == installed _assert_install("inmanta-module-elaboratev2module==1.2.3", installed=False) tmpvenv_active_inherit.install_for_config( - requirements=[Requirement.parse("inmanta-module-elaboratev2module==1.2.3")], + requirements=[Requirement(requirement_string="inmanta-module-elaboratev2module==1.2.3")], config=PipConfig( index_url=local_module_package_index, ), @@ -697,7 +703,7 @@ def test_are_installed_dependency_cycle_on_extra(tmpdir, tmpvenv_active_inherit: path=os.path.join(tmpdir, "pkg"), publish_index=pip_index, optional_dependencies={ - "optional-pkg": [Requirement.parse("dep[optional-dep]")], + "optional-pkg": [Requirement(requirement_string="dep[optional-dep]")], }, ) create_python_package( @@ -706,11 +712,11 @@ def test_are_installed_dependency_cycle_on_extra(tmpdir, tmpvenv_active_inherit: path=os.path.join(tmpdir, "dep"), publish_index=pip_index, optional_dependencies={ - "optional-dep": [Requirement.parse("pkg[optional-pkg]")], + "optional-dep": [Requirement(requirement_string="pkg[optional-pkg]")], }, ) - requirements = [Requirement.parse("pkg[optional-pkg]")] + requirements = [Requirement(requirement_string="pkg[optional-pkg]")] tmpvenv_active_inherit.install_for_config( requirements=requirements, config=PipConfig( diff --git a/tests/test_file_parser.py b/tests/test_file_parser.py index 3103971dca..34554b5093 100644 --- a/tests/test_file_parser.py +++ b/tests/test_file_parser.py @@ -18,9 +18,8 @@ import os -from pkg_resources import Requirement - from inmanta.file_parser import RequirementsTxtParser +from packaging.requirements import Requirement def test_requirements_txt_parser(tmpdir) -> None: @@ -41,7 +40,7 @@ def test_requirements_txt_parser(tmpdir) -> None: expected_requirements = ["test==1.2.3", "other-dep~=2.0.0", "third-dep<5.0.0", "splitteddep", "Capital"] requirements: list[Requirement] = RequirementsTxtParser().parse(requirements_txt_file) - assert requirements == [Requirement.parse(r) for r in expected_requirements] + assert requirements == [Requirement(requirement_string=r) for r in expected_requirements] requirements_as_str = RequirementsTxtParser.parse_requirements_as_strs(requirements_txt_file) assert requirements_as_str == expected_requirements diff --git a/tests/test_module_loader.py b/tests/test_module_loader.py index 5da7865d0d..995a297b16 100644 --- a/tests/test_module_loader.py +++ b/tests/test_module_loader.py @@ -28,7 +28,6 @@ import py import pytest -from pkg_resources import Requirement from inmanta import compiler, const, env, loader, plugins, resources from inmanta.ast import CompilerException @@ -46,6 +45,7 @@ Project, ) from inmanta.moduletool import ModuleConverter, ModuleTool, ProjectTool +from packaging.requirements import Requirement from packaging.version import Version from utils import PipIndex, create_python_package, log_contains, module_from_template, v1_module_from_template @@ -346,7 +346,7 @@ def test_load_module_recursive_v2_module_depends_on_v1( project = snippetcompiler.setup_for_snippet( snippet="import v2_depends_on_v1", index_url=local_module_package_index, - python_requires=[Requirement.parse("inmanta-module-v2-depends-on-v1")], + python_requires=[Requirement(requirement_string="inmanta-module-v2-depends-on-v1")], install_project=False, ) if preload_v1_module: @@ -373,7 +373,7 @@ def test_load_module_recursive_complex_module_dependencies(local_module_package_ snippet="import complex_module_dependencies_mod1", autostd=False, index_url=local_module_package_index, - python_requires=[Requirement.parse("inmanta-module-complex-module-dependencies-mod1")], + python_requires=[Requirement(requirement_string="inmanta-module-complex-module-dependencies-mod1")], install_project=False, ) assert "complex_module_dependencies_mod1" not in project.modules @@ -412,7 +412,7 @@ def load(requires: Optional[list[Requirement]] = None) -> None: with pytest.raises(ModuleLoadingException, match=f"Failed to load module {module_name}"): load() # assert that it doesn't raise an error with explicit requirements set - load([Requirement.parse(ModuleV2Source.get_package_name_for(module_name))]) + load([Requirement(requirement_string=ModuleV2Source.get_package_name_for(module_name))]) @pytest.mark.parametrize("v1", [True, False]) @@ -469,7 +469,7 @@ def test_load_import_based_v2_module( extra_index_url=[index.url], # make sure that even listing the requirement in project.yml does not suffice project_requires=[InmantaModuleRequirement.parse(dependency_module_name)], - python_requires=[] if v1 else [Requirement.parse(ModuleV2Source.get_package_name_for(main_module_name))], + python_requires=[] if v1 else [Requirement(requirement_string=ModuleV2Source.get_package_name_for(main_module_name))], ) if explicit_dependency: @@ -609,7 +609,9 @@ def test_project_requirements_dont_overwrite_core_requirements_source( module_name: str = "minimalv2module" module_path: str = str(tmpdir.join(module_name)) module_from_template( - os.path.join(modules_v2_dir, module_name), module_path, new_requirements=[Requirement.parse("Jinja2==2.11.3")] + os.path.join(modules_v2_dir, module_name), + module_path, + new_requirements=[Requirement(requirement_string="Jinja2==2.11.3")], ) # Activate the snippetcompiler venv @@ -653,7 +655,7 @@ def test_project_requirements_dont_overwrite_core_requirements_index( module_from_template( os.path.join(modules_v2_dir, module_name), module_path, - new_requirements=[Requirement.parse("Jinja2==2.11.3")], + new_requirements=[Requirement(requirement_string="Jinja2==2.11.3")], publish_index=index, ) @@ -704,7 +706,11 @@ def test_module_conflicting_dependencies_with_v2_modules( # Create a python package y with version 1.0.0 that depends on x~=1.0.0 create_python_package( - "y", Version("1.0.0"), str(tmpdir.join("y-1.0.0")), requirements=[Requirement.parse("x~=1.0.0")], publish_index=index + "y", + Version("1.0.0"), + str(tmpdir.join("y-1.0.0")), + requirements=[Requirement(requirement_string="x~=1.0.0")], + publish_index=index, ) # Create the first module @@ -713,7 +719,7 @@ def test_module_conflicting_dependencies_with_v2_modules( module_from_template( os.path.join(modules_v2_dir, module_name1), module_path1, - new_requirements=[Requirement.parse("y~=1.0.0")], + new_requirements=[Requirement(requirement_string="y~=1.0.0")], publish_index=index, ) @@ -724,7 +730,7 @@ def test_module_conflicting_dependencies_with_v2_modules( os.path.join(modules_v2_dir, "minimalv2module"), module_path2, new_name="minimalv2module2", - new_requirements=[Requirement.parse("x~=2.0.0")], + new_requirements=[Requirement(requirement_string="x~=2.0.0")], publish_index=index, ) @@ -784,7 +790,7 @@ def test_module_conflicting_dependencies_with_v1_module( os.path.join(modules_dir, module_name1), module_path1, new_name="modulev1", - new_requirements=[Requirement.parse("y~=1.0.0")], + new_requirements=[Requirement(requirement_string="y~=1.0.0")], ) # Create the second module @@ -793,7 +799,7 @@ def test_module_conflicting_dependencies_with_v1_module( module_from_template( os.path.join(modules_v2_dir, module_name2), module_path2, - new_requirements=[Requirement.parse("y~=2.0.0")], + new_requirements=[Requirement(requirement_string="y~=2.0.0")], publish_index=index, ) @@ -841,7 +847,7 @@ def test_module_install_extra_on_project_level_v2_dep( new_name="mymod", new_requirements=[], new_extras={ - "myfeature": [Requirement.parse(package_name_extra)], + "myfeature": [Requirement(requirement_string=package_name_extra)], }, publish_index=index, ) @@ -890,7 +896,7 @@ def test_module_install_extra_on_dep_of_v2_module( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [Requirement.parse(package_name_extra)], + "myfeature": [Requirement(requirement_string=package_name_extra)], }, publish_index=index, ) @@ -910,7 +916,7 @@ def test_module_install_extra_on_dep_of_v2_module( install_project=True, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[Requirement.parse("inmanta-module-myv2mod")], + python_requires=[Requirement(requirement_string="inmanta-module-myv2mod")], autostd=False, ) @@ -947,7 +953,7 @@ def test_module_install_extra_on_dep_of_v1_module( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [Requirement.parse(package_name_extra)], + "myfeature": [Requirement(requirement_string=package_name_extra)], }, publish_index=index, ) @@ -997,7 +1003,7 @@ def test_module_install_extra_on_project_level_v2_dep_update_scenario( new_name="mymod", new_requirements=[], new_extras={ - "myfeature": [Requirement.parse(package_name_extra)], + "myfeature": [Requirement(requirement_string=package_name_extra)], }, publish_index=index, ) @@ -1066,7 +1072,7 @@ def test_module_install_extra_on_dep_of_v2_module_update_scenario( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [Requirement.parse(package_name_extra)], + "myfeature": [Requirement(requirement_string=package_name_extra)], }, publish_index=index, ) @@ -1095,7 +1101,7 @@ def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> install_project=True, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[Requirement.parse("inmanta-module-myv2mod==1.0.0")], + python_requires=[Requirement(requirement_string="inmanta-module-myv2mod==1.0.0")], autostd=False, ) assert_installed(extra_installed=False) @@ -1117,7 +1123,7 @@ def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> install_project=not do_project_update, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[Requirement.parse("inmanta-module-myv2mod==2.0.0")], + python_requires=[Requirement(requirement_string="inmanta-module-myv2mod==2.0.0")], autostd=False, ) if do_project_update: @@ -1161,7 +1167,7 @@ def test_module_install_extra_on_dep_of_v1_module_update_scenario( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [Requirement.parse(package_name_extra)], + "myfeature": [Requirement(requirement_string=package_name_extra)], }, publish_index=index, ) @@ -1225,7 +1231,7 @@ async def test_v1_module_depends_on_third_party_dep_with_extra( os.path.join(tmpdir, "myv1mod"), new_name="myv1mod", new_content_init_cf="", - new_requirements=[Requirement.parse("pkg[optional-a]")], + new_requirements=[Requirement(requirement_string="pkg[optional-a]")], ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv1mod", @@ -1245,7 +1251,7 @@ async def test_v1_module_depends_on_third_party_dep_with_extra( os.path.join(tmpdir, "myv1mod"), new_name="myv1mod", new_content_init_cf="", - new_requirements=[Requirement.parse("pkg[optional-a,optional-b]")], + new_requirements=[Requirement(requirement_string="pkg[optional-a,optional-b]")], ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv1mod", @@ -1272,13 +1278,13 @@ async def test_v2_module_depends_on_third_party_dep_with_extra( str(tmpdir.join("myv2mod")), new_name="myv2mod", new_version=Version("1.0.0"), - new_requirements=[Requirement.parse("pkg[optional-a]")], + new_requirements=[Requirement(requirement_string="pkg[optional-a]")], publish_index=index, ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv2mod", install_project=True, - python_requires=[Requirement.parse("inmanta-module-myv2mod==1.0.0")], + python_requires=[Requirement(requirement_string="inmanta-module-myv2mod==1.0.0")], index_url=index.url, extra_index_url=[index_with_pkgs_containing_optional_deps], autostd=False, @@ -1294,13 +1300,13 @@ async def test_v2_module_depends_on_third_party_dep_with_extra( str(tmpdir.join("myv2mod")), new_name="myv2mod", new_version=Version("2.0.0"), - new_requirements=[Requirement.parse("pkg[optional-a,optional-b]")], + new_requirements=[Requirement(requirement_string="pkg[optional-a,optional-b]")], publish_index=index, ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv2mod", install_project=True, - python_requires=[Requirement.parse("inmanta-module-myv2mod==2.0.0")], + python_requires=[Requirement(requirement_string="inmanta-module-myv2mod==2.0.0")], index_url=index.url, extra_index_url=[index_with_pkgs_containing_optional_deps], autostd=False, diff --git a/tests/utils.py b/tests/utils.py index 47a1a535bb..a7997d8d53 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -37,7 +37,6 @@ import pytest import yaml -from pkg_resources import Requirement, parse_version import build import build.env @@ -52,6 +51,8 @@ from inmanta.util import get_compiler_version, hash_file from libpip2pi.commands import dir2pi from packaging import version +from packaging.requirements import Requirement +from packaging.version import Version T = TypeVar("T") @@ -455,7 +456,7 @@ def get_product_meta_data() -> ProductMetadata: def product_version_lower_or_equal_than(version: str) -> bool: - return parse_version(get_product_meta_data().version) <= parse_version(version) + return Version(version=get_product_meta_data().version) <= Version(version=version) def mark_only_for_version_higher_than(version: str) -> "MarkDecorator": From 021e0094d716578cef305359969840741948ef53 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 31 Jul 2024 16:12:32 +0200 Subject: [PATCH 02/74] wip --- src/inmanta/env.py | 32 +++++++++++++++++--------------- src/inmanta/file_parser.py | 4 ++-- src/inmanta/module.py | 20 +++++++++++--------- 3 files changed, 30 insertions(+), 26 deletions(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index f83545f4db..2a7ea73616 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -127,7 +127,7 @@ def get_conflicts_string(self) -> Optional[str]: if not self.conflicts: return None msg = "" - for current_conflict in sorted(self.conflicts, key=lambda x: x.requirement.key): + for current_conflict in sorted(self.conflicts, key=lambda x: x.requirement.name): msg += f"\n\t* {current_conflict}" return msg @@ -204,21 +204,22 @@ def _are_installed_recursive( if r.marker and not r.marker.evaluate(environment=environment_marker_evaluation): # The marker of the requirement doesn't apply on this environment continue - if r.name not in installed_packages or str(installed_packages[r.name]) not in r: + if r.name not in installed_packages or str(installed_packages[r.name]) not in r.specifier: return False if r.extras: for extra in r.extras: search_dist = [e for e in importlib.metadata.distributions() if e.name == extra] assert len(search_dist) <= 1 distribution: Optional[Distribution] = None if len(search_dist) == 0 else search_dist[0] + if distribution is None: + return False + extra_packages = [ Requirement(requirement_string=e) for e in distribution.metadata.json["provides_extra"] ] - if distribution is None: - return False pkgs_required_by_extra: set[Requirement] = set(extra_packages) - set( - [Requirement(requirement_string=e) for e in distribution.requires] + [Requirement(requirement_string=e) for e in distribution.requires or []] ) if not _are_installed_recursive( reqs=list(pkgs_required_by_extra), @@ -274,7 +275,7 @@ def _get_tree_recursive_single(acc: abc.Set[str], dist: str) -> abc.Set[str]: # recurse on direct dependencies return _get_tree_recursive( - (requirement.key for requirement in installed_distributions[dist].requires()), + (requirement for requirement in installed_distributions[dist].requires or []), acc=acc | {dist}, ) @@ -1000,9 +1001,9 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: # all requirements of all packages installed in this environment installed_constraints: abc.Set[OwnedRequirement] = frozenset( - OwnedRequirement(requirement, dist_info.name) + OwnedRequirement(Requirement(requirement_string=requirement), dist_info.name) for dist_info in importlib.metadata.distributions() - for requirement in dist_info.requires + for requirement in dist_info.requires or [] ) inmanta_constraints: abc.Set[OwnedRequirement] = frozenset( OwnedRequirement(r, owner="inmanta-core") for r in cls._get_requirements_on_inmanta_package() @@ -1035,12 +1036,13 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: constraint_violations_strict: set[VersionConflict] = set() for c in all_constraints: requirement = c.requirement - if (requirement.key not in installed_versions or str(installed_versions[requirement.key]) not in requirement) and ( - not requirement.marker or (requirement.marker and requirement.marker.evaluate()) - ): + if ( + requirement.name not in installed_versions + or str(installed_versions[requirement.name]) not in requirement.specifier + ) and (not requirement.marker or (requirement.marker and requirement.marker.evaluate())): version_conflict = VersionConflict( requirement=requirement, - installed_version=installed_versions.get(requirement.key, None), + installed_version=installed_versions.get(requirement.name, None), owner=c.owner, ) if c.is_owned_by(full_strict_scope): @@ -1101,14 +1103,14 @@ def check_legacy(cls, in_scope: Pattern[str], constraints: Optional[list[Require Requirement(requirement_string=requirement) for dist_info in working_set if in_scope.fullmatch(dist_info.name) - for requirement in dist_info.requires + for requirement in dist_info.requires or [] ) installed_versions: dict[str, version.Version] = PythonWorkingSet.get_packages_in_working_set() constraint_violations: set[VersionConflict] = { - VersionConflict(constraint, installed_versions.get(constraint.key, None)) + VersionConflict(constraint, installed_versions.get(constraint.name, None)) for constraint in all_constraints - if constraint.key not in installed_versions or str(installed_versions[constraint.key]) not in constraint + if constraint.name not in installed_versions or str(installed_versions[constraint.name]) not in constraint.specifier } all_violations = constraint_violations_non_strict | constraint_violations_strict | constraint_violations diff --git a/src/inmanta/file_parser.py b/src/inmanta/file_parser.py index 0df7316247..2d56045a08 100644 --- a/src/inmanta/file_parser.py +++ b/src/inmanta/file_parser.py @@ -92,14 +92,14 @@ def get_content_with_dep_removed(cls, filename: str, remove_dep_on_pkg: str) -> if line_continuation_buffer: line_continuation_buffer += line if not line.endswith("\\"): - if Requirement(requirement_string=line_continuation_buffer).key != remove_dep_on_pkg: + if Requirement(requirement_string=line_continuation_buffer).name != remove_dep_on_pkg: result += line_continuation_buffer line_continuation_buffer = "" elif not line.strip() or line.strip().startswith("#"): result += line elif line.endswith("\\"): line_continuation_buffer = line - elif Requirement(requirement_string=line).key != remove_dep_on_pkg.lower(): + elif Requirement(requirement_string=line).name != remove_dep_on_pkg.lower(): result += line else: # Dependency matches `remove_dep_on_pkg` => Remove line from result diff --git a/src/inmanta/module.py b/src/inmanta/module.py index 670fcc28e9..71def63dc2 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -113,11 +113,11 @@ def project_name(self) -> str: @property def key(self) -> str: # Requirement converts all "_" to "-". Inmanta modules use "_" - return self._requirement.key.replace("-", "_") + return self._requirement.name.replace("-", "_") @property def specifier(self) -> str: - return self._requirement.specifier + return str(self._requirement.specifier) def __eq__(self, other: object) -> bool: if not isinstance(other, InmantaModuleRequirement): @@ -125,7 +125,7 @@ def __eq__(self, other: object) -> bool: return self._requirement == other._requirement def __contains__(self, version: str) -> bool: - return version in self._requirement + return version in self._requirement.specifier def __str__(self) -> str: return str(self._requirement).replace("-", "_") @@ -135,7 +135,7 @@ def __hash__(self) -> int: @property def specs(self) -> Sequence[tuple[str, str]]: - return self._requirement.specs + return [(e.operator, e.version) for e in self._requirement.specifier] def version_spec_str(self) -> str: """ @@ -1177,14 +1177,16 @@ def has_requirement_for(self, pkg_name: str) -> bool: Returns True iff this requirements.txt file contains the given package name. The given `pkg_name` is matched case insensitive against the requirements in this RequirementsTxtFile. """ - return any(r.key == pkg_name.lower() for r in self._requirements) + return any(r.name == pkg_name.lower() for r in self._requirements) def set_requirement_and_write(self, requirement: Requirement) -> None: """ Add the given requirement to the requirements.txt file and update the file on disk, replacing any existing constraints on this package. """ - new_content_file = RequirementsTxtParser.get_content_with_dep_removed(self._filename, remove_dep_on_pkg=requirement.key) + new_content_file = RequirementsTxtParser.get_content_with_dep_removed( + self._filename, remove_dep_on_pkg=requirement.name + ) new_content_file = new_content_file.rstrip() if new_content_file: new_content_file = f"{new_content_file}\n{requirement}" @@ -2665,7 +2667,7 @@ def add_module_requirement_persistent(self, requirement: InmantaModuleRequiremen requirements_txt_file.set_requirement_and_write(requirement.get_python_package_requirement()) elif os.path.exists(requirements_txt_file_path): requirements_txt_file = RequirementsTxtFile(requirements_txt_file_path) - requirements_txt_file.remove_requirement_and_write(requirement.get_python_package_requirement().key) + requirements_txt_file.remove_requirement_and_write(requirement.get_python_package_requirement().name) def get_module_requirements(self) -> list[str]: return [*self.metadata.requires, *(str(req) for req in self.get_module_v2_requirements())] @@ -3284,7 +3286,7 @@ def add_module_requirement_persistent(self, requirement: InmantaModuleRequiremen # Remove requirement from requirements.txt file if os.path.exists(requirements_txt_file_path): requirements_txt_file = RequirementsTxtFile(requirements_txt_file_path) - requirements_txt_file.remove_requirement_and_write(requirement.get_python_package_requirement().key) + requirements_txt_file.remove_requirement_and_write(requirement.get_python_package_requirement().name) else: # Add requirement to requirements.txt requirements_txt_file = RequirementsTxtFile(requirements_txt_file_path, create_file_if_not_exists=True) @@ -3430,7 +3432,7 @@ def add_module_requirement_persistent(self, requirement: InmantaModuleRequiremen new_install_requires = [ r for r in config_parser.get("options", "install_requires").split("\n") - if r and Requirement(requirement_string=r).key != python_pkg_requirement.key + if r and Requirement(requirement_string=r).name != python_pkg_requirement.name ] new_install_requires.append(str(python_pkg_requirement)) else: From 17791058c137962ff16e0c2e702cf53872c1ea95 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 31 Jul 2024 16:15:13 +0200 Subject: [PATCH 03/74] mypy --- src/inmanta/env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 2a7ea73616..4069970f24 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -165,7 +165,7 @@ def _get_as_requirements_type(cls, requirements: req_list) -> Sequence[Requireme Convert requirements from Union[Sequence[str], Sequence[Requirement]] to Sequence[Requirement] """ if isinstance(requirements[0], str): - return [Requirement(requirement_string=r) for r in requirements] + return [Requirement(requirement_string=r) for r in requirements if isinstance(r, str)] else: return requirements From 871e18d650f238f757971bb9690438a75d25b316 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Fri, 2 Aug 2024 17:39:43 +0200 Subject: [PATCH 04/74] refactor --- src/inmanta/agent/agent.py | 4 +- src/inmanta/agent/executor.py | 5 +- src/inmanta/env.py | 112 ++++++++++++++++--------- src/inmanta/file_parser.py | 12 +-- src/inmanta/module.py | 50 ++++++----- src/inmanta/moduletool.py | 7 +- tests/conftest.py | 15 ++-- tests/moduletool/test_add.py | 5 +- tests/moduletool/test_convert_v1_v2.py | 6 +- tests/moduletool/test_install.py | 48 +++++------ tests/moduletool/test_update.py | 17 ++-- tests/server/test_compilerservice.py | 5 +- tests/test_env.py | 37 ++++---- tests/test_file_parser.py | 6 +- tests/test_module_loader.py | 73 ++++++++-------- tests/utils.py | 20 ++--- 16 files changed, 231 insertions(+), 191 deletions(-) diff --git a/src/inmanta/agent/agent.py b/src/inmanta/agent/agent.py index c7b1f86633..8fc49a4378 100644 --- a/src/inmanta/agent/agent.py +++ b/src/inmanta/agent/agent.py @@ -34,7 +34,6 @@ from typing import Any, Collection, Dict, Optional, Union, cast import inmanta.agent.executor -import packaging.requirements from inmanta import config, const, data, env, protocol from inmanta.agent import config as cfg from inmanta.agent import executor, forking_executor, in_process_executor @@ -48,6 +47,7 @@ ResourceType, ResourceVersionIdStr, ) +from inmanta.env import SafeRequirement from inmanta.loader import CodeLoader, ModuleSource from inmanta.protocol import SessionEndpoint, SyncClient, methods, methods_v2 from inmanta.resources import Id @@ -1340,7 +1340,7 @@ async def _install(self, blueprint: executor.ExecutorBlueprint) -> None: await loop.run_in_executor( self.thread_pool, self._env.install_for_config, - [packaging.requirements.Requirement(requirement_string=e) for e in blueprint.requirements], + [SafeRequirement(requirement_string=e) for e in blueprint.requirements], blueprint.pip_config, ) await loop.run_in_executor(self.thread_pool, self._loader.deploy_version, blueprint.sources) diff --git a/src/inmanta/agent/executor.py b/src/inmanta/agent/executor.py index c0258da9cd..7450a9e55a 100644 --- a/src/inmanta/agent/executor.py +++ b/src/inmanta/agent/executor.py @@ -33,10 +33,9 @@ from typing import Any, Dict, Optional, Sequence import inmanta.types -import packaging.requirements from inmanta.agent import config as cfg from inmanta.data.model import PipConfig, ResourceIdStr, ResourceType, ResourceVersionIdStr -from inmanta.env import PythonEnvironment +from inmanta.env import PythonEnvironment, SafeRequirement from inmanta.loader import ModuleSource from inmanta.resources import Id from inmanta.types import JsonType @@ -269,7 +268,7 @@ async def create_and_install_environment(self, blueprint: EnvBlueprint) -> None: if len(req): # install_for_config expects at least 1 requirement or a path to install install_for_config = functools.partial( self.install_for_config, - requirements=[packaging.requirements.Requirement(requirement_string=e) for e in req], + requirements=[SafeRequirement(requirement_string=e) for e in req], config=blueprint.pip_config, ) await loop.run_in_executor(self.thread_pool, install_for_config) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 4069970f24..5fa454d67c 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -37,7 +37,7 @@ from re import Pattern from subprocess import CalledProcessError from textwrap import indent -from typing import NamedTuple, Optional, TypeVar +from typing import NamedTuple, Optional, TypeVar, cast from inmanta import const from inmanta.ast import CompilerException @@ -47,6 +47,7 @@ from inmanta.util import strtobool from packaging import version from packaging.requirements import Requirement +from packaging.utils import InvalidName, NormalizedName LOGGER = logging.getLogger(__name__) LOGGER_PIP = logging.getLogger("inmanta.pip") # Use this logger to log pip commands or data related to pip commands. @@ -60,6 +61,28 @@ class PipInstallError(Exception): pass +# Core metadata spec for `Name` +_validate_regex = re.compile(r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$", re.IGNORECASE) +_canonicalize_regex = re.compile(r"[-_.]+") +_normalized_regex = re.compile(r"^([a-z0-9]|[a-z0-9]([a-z0-9-](?!--))*[a-z0-9])$") +# PEP 427: The build number must start with a digit. +_build_tag_regex = re.compile(r"(\d+)(.*)") + + +def canonicalize_name(name: str, *, validate: bool = False) -> NormalizedName: + if validate and not _validate_regex.match(name): + raise InvalidName(f"name is invalid: {name!r}") + # This is taken from PEP 503. + value = _canonicalize_regex.sub("-", name) + return cast(NormalizedName, value) + + +class SafeRequirement(Requirement): + def __init__(self, requirement_string: str) -> None: + super().__init__(requirement_string=requirement_string) + self.name = canonicalize_name(self.name) + + @dataclass(eq=True, frozen=True) class VersionConflict: """ @@ -70,7 +93,7 @@ class VersionConflict: :param owner: The package from which the constraint originates """ - requirement: Requirement + requirement: SafeRequirement installed_version: Optional[version.Version] = None owner: Optional[str] = None @@ -78,7 +101,7 @@ def __str__(self) -> str: owner = "" if self.owner: # Cfr pip - # Requirement already satisfied: certifi>=2017.4.17 in /[...]/site-packages + # SafeRequirement already satisfied: certifi>=2017.4.17 in /[...]/site-packages # (from requests>=2.23.0->cookiecutter<3,>=1->inmanta-core==7.0.0) (2022.6.15) owner = f" (from {self.owner})" if self.installed_version: @@ -155,17 +178,19 @@ def get_advice(self) -> Optional[str]: ) -req_list = TypeVar("req_list", Sequence[str], Sequence[Requirement]) +req_list = TypeVar("req_list", Sequence[str], Sequence[SafeRequirement]) + +import importlib.metadata class PythonWorkingSet: @classmethod - def _get_as_requirements_type(cls, requirements: req_list) -> Sequence[Requirement]: + def _get_as_requirements_type(cls, requirements: req_list) -> Sequence[SafeRequirement]: """ - Convert requirements from Union[Sequence[str], Sequence[Requirement]] to Sequence[Requirement] + Convert requirements from Union[Sequence[str], Sequence[SafeRequirement]] to SequeSafeRequirementment] """ if isinstance(requirements[0], str): - return [Requirement(requirement_string=r) for r in requirements if isinstance(r, str)] + return [SafeRequirement(requirement_string=r) for r in requirements if isinstance(r, str)] else: return requirements @@ -179,8 +204,8 @@ def are_installed(cls, requirements: req_list) -> bool: installed_packages: dict[str, version.Version] = cls.get_packages_in_working_set() def _are_installed_recursive( - reqs: Sequence[Requirement], - seen_requirements: Sequence[Requirement], + reqs: Sequence[SafeRequirement], + seen_requirements: Sequence[SafeRequirement], contained_in_extra: Optional[str] = None, ) -> bool: """ @@ -196,7 +221,8 @@ def _are_installed_recursive( for r in reqs: if r in seen_requirements: continue - # Requirements created by the `Distribution.requires()` method have the extra, the Requirement was created from, + # Requirements created by the `Distribution.requires()` method have the extra, the SafeRequirement was created + # from, TODO h update this comment # set as a marker. The line below makes sure that the "extra" marker matches. The marker is not set by # `Distribution.requires()` when the package is installed in editable mode, but setting it always doesn't make # the marker evaluation fail. @@ -204,22 +230,20 @@ def _are_installed_recursive( if r.marker and not r.marker.evaluate(environment=environment_marker_evaluation): # The marker of the requirement doesn't apply on this environment continue - if r.name not in installed_packages or str(installed_packages[r.name]) not in r.specifier: + if r.name not in installed_packages or ( + len(r.specifier) > 0 and str(installed_packages[r.name]) not in r.specifier + ): return False if r.extras: for extra in r.extras: - search_dist = [e for e in importlib.metadata.distributions() if e.name == extra] + search_dist = [e for e in importlib.metadata.distributions() if e.name == r.name] assert len(search_dist) <= 1 distribution: Optional[Distribution] = None if len(search_dist) == 0 else search_dist[0] if distribution is None: return False - extra_packages = [ - Requirement(requirement_string=e) for e in distribution.metadata.json["provides_extra"] - ] - - pkgs_required_by_extra: set[Requirement] = set(extra_packages) - set( - [Requirement(requirement_string=e) for e in distribution.requires or []] + pkgs_required_by_extra: set[SafeRequirement] = set( + [SafeRequirement(requirement_string=e) for e in distribution.requires or []] ) if not _are_installed_recursive( reqs=list(pkgs_required_by_extra), @@ -229,7 +253,7 @@ def _are_installed_recursive( return False return True - reqs_as_requirements: Sequence[Requirement] = cls._get_as_requirements_type(requirements) + reqs_as_requirements: Sequence[SafeRequirement] = cls._get_as_requirements_type(requirements) return _are_installed_recursive(reqs_as_requirements, seen_requirements=[]) @classmethod @@ -240,7 +264,7 @@ def get_packages_in_working_set(cls, inmanta_modules_only: bool = False) -> dict :param inmanta_modules_only: Only return inmanta modules from the working set """ return { - dist_info.name: version.Version(dist_info.version) + canonicalize_name(dist_info.name).lower(): version.Version(dist_info.version) for dist_info in importlib.metadata.distributions() if not inmanta_modules_only or dist_info.name.startswith(const.MODULE_PKG_NAME_PREFIX) } @@ -275,7 +299,10 @@ def _get_tree_recursive_single(acc: abc.Set[str], dist: str) -> abc.Set[str]: # recurse on direct dependencies return _get_tree_recursive( - (requirement for requirement in installed_distributions[dist].requires or []), + ( + SafeRequirement(requirement_string=requirement).name + for requirement in installed_distributions[dist].requires or [] + ), acc=acc | {dist}, ) @@ -368,7 +395,7 @@ def run_pip_install_command_from_config( cls, python_path: str, config: PipConfig, - requirements: Optional[Sequence[Requirement]] = None, + requirements: Optional[Sequence[SafeRequirement]] = None, requirements_files: Optional[list[str]] = None, upgrade: bool = False, upgrade_strategy: PipUpgradeStrategy = PipUpgradeStrategy.ONLY_IF_NEEDED, @@ -734,7 +761,7 @@ def get_installed_packages(self, only_editable: bool = False) -> dict[str, versi def install_for_config( self, - requirements: list[Requirement], + requirements: list[SafeRequirement], config: PipConfig, upgrade: bool = False, constraint_files: Optional[list[str]] = None, @@ -771,7 +798,7 @@ def install_for_config( def install_from_index( self, - requirements: list[Requirement], + requirements: list[SafeRequirement], index_urls: Optional[list[str]] = None, upgrade: bool = False, allow_pre_releases: bool = False, @@ -842,7 +869,7 @@ def install_from_list( use_pip_config was ignored on ISO6 and it still is """ self.install_from_index( - requirements=[Requirement(requirement_string=r) for r in requirements_list], + requirements=[SafeRequirement(requirement_string=r) for r in requirements_list], upgrade=upgrade, upgrade_strategy=upgrade_strategy, use_pip_config=True, @@ -862,7 +889,7 @@ def get_protected_inmanta_packages(cls) -> list[str]: ] @classmethod - def _get_requirements_on_inmanta_package(cls) -> Sequence[Requirement]: + def _get_requirements_on_inmanta_package(cls) -> Sequence[SafeRequirement]: """ Returns the content of the requirement file that should be supplied to each `pip install` invocation to make sure that no Inmanta packages gets overridden. @@ -870,7 +897,7 @@ def _get_requirements_on_inmanta_package(cls) -> Sequence[Requirement]: protected_inmanta_packages: list[str] = cls.get_protected_inmanta_packages() workingset: dict[str, version.Version] = PythonWorkingSet.get_packages_in_working_set() return [ - Requirement(requirement_string=f"{pkg}=={workingset[pkg]}") + SafeRequirement(requirement_string=f"{pkg}=={workingset[pkg]}") for pkg in workingset if pkg in protected_inmanta_packages ] @@ -967,7 +994,7 @@ def are_installed(self, requirements: req_list) -> bool: def install_for_config( self, - requirements: list[Requirement], + requirements: list[SafeRequirement], config: PipConfig, upgrade: bool = False, constraint_files: Optional[list[str]] = None, @@ -985,7 +1012,7 @@ def install_for_config( def get_constraint_violations_for_check( cls, strict_scope: Optional[Pattern[str]] = None, - constraints: Optional[list[Requirement]] = None, + constraints: Optional[list[SafeRequirement]] = None, ) -> tuple[set[VersionConflict], set[VersionConflict]]: """ Return the constraint violations that exist in this venv. Returns a tuple of non-strict and strict violations, @@ -993,7 +1020,7 @@ def get_constraint_violations_for_check( """ class OwnedRequirement(NamedTuple): - requirement: Requirement + requirement: SafeRequirement owner: Optional[str] = None def is_owned_by(self, owners: abc.Set[str]) -> bool: @@ -1001,10 +1028,14 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: # all requirements of all packages installed in this environment installed_constraints: abc.Set[OwnedRequirement] = frozenset( - OwnedRequirement(Requirement(requirement_string=requirement), dist_info.name) + OwnedRequirement(SafeRequirement(requirement_string=requirement), canonicalize_name(dist_info.name)) for dist_info in importlib.metadata.distributions() for requirement in dist_info.requires or [] + if SafeRequirement(requirement).marker is None ) + # for e in installed_constraints: + # e.requirement.name = canonicalize_name(e.requirement.name).lower() + inmanta_constraints: abc.Set[OwnedRequirement] = frozenset( OwnedRequirement(r, owner="inmanta-core") for r in cls._get_requirements_on_inmanta_package() ) @@ -1014,7 +1045,7 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: all_constraints: abc.Set[OwnedRequirement] = installed_constraints | inmanta_constraints | extra_constraints - full_strict_scope: abc.Set[str] = PythonWorkingSet.get_dependency_tree( + parameters = list( chain( ( [] @@ -1029,6 +1060,7 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: (requirement.requirement.name for requirement in extra_constraints), ) ) + full_strict_scope: abc.Set[str] = PythonWorkingSet.get_dependency_tree(parameters) installed_versions: dict[str, version.Version] = PythonWorkingSet.get_packages_in_working_set() @@ -1037,12 +1069,12 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: for c in all_constraints: requirement = c.requirement if ( - requirement.name not in installed_versions - or str(installed_versions[requirement.name]) not in requirement.specifier + requirement.name.lower() not in installed_versions + or str(installed_versions[requirement.name.lower()]) not in requirement.specifier ) and (not requirement.marker or (requirement.marker and requirement.marker.evaluate())): version_conflict = VersionConflict( requirement=requirement, - installed_version=installed_versions.get(requirement.name, None), + installed_version=installed_versions.get(requirement.name.lower(), None), # TODO h do something owner=c.owner, ) if c.is_owned_by(full_strict_scope): @@ -1056,7 +1088,7 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: def check( cls, strict_scope: Optional[Pattern[str]] = None, - constraints: Optional[list[Requirement]] = None, + constraints: Optional[list[SafeRequirement]] = None, ) -> None: """ Check this Python environment for incompatible dependencies in installed packages. @@ -1080,7 +1112,7 @@ def check( LOGGER.warning("%s", violation) @classmethod - def check_legacy(cls, in_scope: Pattern[str], constraints: Optional[list[Requirement]] = None) -> bool: + def check_legacy(cls, in_scope: Pattern[str], constraints: Optional[list[SafeRequirement]] = None) -> bool: """ Check this Python environment for incompatible dependencies in installed packages. This method is a legacy method in the sense that it has been replaced with a more correct check defined in self.check(). This method is invoked @@ -1099,8 +1131,8 @@ def check_legacy(cls, in_scope: Pattern[str], constraints: Optional[list[Require working_set: abc.Iterable[Distribution] = importlib.metadata.distributions() # add all requirements of all in scope packages installed in this environment - all_constraints: set[Requirement] = set(constraints if constraints is not None else []).union( - Requirement(requirement_string=requirement) + all_constraints: set[SafeRequirement] = set(constraints if constraints is not None else []).union( + SafeRequirement(requirement_string=requirement) for dist_info in working_set if in_scope.fullmatch(dist_info.name) for requirement in dist_info.requires or [] @@ -1269,7 +1301,7 @@ def _activate_that(self) -> None: def install_for_config( self, - requirements: list[Requirement], + requirements: list[SafeRequirement], config: PipConfig, upgrade: bool = False, constraint_files: Optional[list[str]] = None, diff --git a/src/inmanta/file_parser.py b/src/inmanta/file_parser.py index 2d56045a08..9a50a2ec20 100644 --- a/src/inmanta/file_parser.py +++ b/src/inmanta/file_parser.py @@ -18,7 +18,7 @@ import os -from packaging.requirements import Requirement +from inmanta.env import SafeRequirement from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap @@ -55,11 +55,11 @@ class RequirementsTxtParser: """ @classmethod - def parse(cls, filename: str) -> list[Requirement]: + def parse(cls, filename: str) -> list[SafeRequirement]: """ - Get all the requirements in `filename` as a list of `Requirement` instances. + Get all the requirements in `filename` as a list of `SafeRequirement` instances. """ - return [Requirement(requirement_string=r) for r in cls.parse_requirements_as_strs(filename)] + return [SafeRequirement(requirement_string=r) for r in cls.parse_requirements_as_strs(filename)] @classmethod def parse_requirements_as_strs(cls, filename: str) -> list[str]: @@ -92,14 +92,14 @@ def get_content_with_dep_removed(cls, filename: str, remove_dep_on_pkg: str) -> if line_continuation_buffer: line_continuation_buffer += line if not line.endswith("\\"): - if Requirement(requirement_string=line_continuation_buffer).name != remove_dep_on_pkg: + if SafeRequirement(requirement_string=line_continuation_buffer.split("#")[0]).name != remove_dep_on_pkg: result += line_continuation_buffer line_continuation_buffer = "" elif not line.strip() or line.strip().startswith("#"): result += line elif line.endswith("\\"): line_continuation_buffer = line - elif Requirement(requirement_string=line).name != remove_dep_on_pkg.lower(): + elif SafeRequirement(requirement_string=line.split("#")[0]).name != remove_dep_on_pkg: # .lower() result += line else: # Dependency matches `remove_dep_on_pkg` => Remove line from result diff --git a/src/inmanta/module.py b/src/inmanta/module.py index 71def63dc2..b7ac35b3c9 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -59,7 +59,7 @@ from inmanta.ast.blocks import BasicBlock from inmanta.ast.statements import BiStatement, DefinitionStatement, DynamicStatement, Statement from inmanta.ast.statements.define import DefineImport -from inmanta.env import assert_pip_has_source +from inmanta.env import SafeRequirement, assert_pip_has_source from inmanta.file_parser import PreservativeYamlParser, RequirementsTxtParser from inmanta.parser import plyInmantaParser from inmanta.parser.plyInmantaParser import cache_manager @@ -67,7 +67,7 @@ from inmanta.util import get_compiler_version from inmanta.warnings import InmantaWarning from packaging import version -from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet from packaging.version import Version from ruamel.yaml.comments import CommentedMap @@ -97,13 +97,13 @@ class InmantaModuleRequirement: used by distinguishing the two on a type level. """ - def __init__(self, requirement: Requirement) -> None: + def __init__(self, requirement: SafeRequirement) -> None: if requirement.name.startswith(ModuleV2.PKG_NAME_PREFIX): raise ValueError( f"InmantaModuleRequirement instances work with inmanta module names, not python package names. " f"Problematic case: {str(requirement)}" ) - self._requirement: Requirement = requirement + self._requirement: SafeRequirement = requirement @property def project_name(self) -> str: @@ -116,8 +116,8 @@ def key(self) -> str: return self._requirement.name.replace("-", "_") @property - def specifier(self) -> str: - return str(self._requirement.specifier) + def specifier(self) -> SpecifierSet: + return self._requirement.specifier def __eq__(self, other: object) -> bool: if not isinstance(other, InmantaModuleRequirement): @@ -151,16 +151,16 @@ def parse(cls: type[TInmantaModuleRequirement], spec: str) -> TInmantaModuleRequ ) if "-" in spec: raise ValueError("Invalid Inmanta module requirement: Inmanta module names use '_', not '-'.") - return cls(Requirement(requirement_string=spec)) + return cls(SafeRequirement(requirement_string=spec)) - def get_python_package_requirement(self) -> Requirement: + def get_python_package_requirement(self) -> SafeRequirement: """ - Return a Requirement with the name of the Python distribution package for this module requirement. + Return a SafeRequirement with the name of the Python distribution package for this module requirement. """ module_name = self.project_name pkg_name = ModuleV2Source.get_package_name_for(module_name) pkg_req_str = str(self).replace(module_name, pkg_name, 1) # Replace max 1 occurrence - return Requirement(requirement_string=pkg_req_str) + return SafeRequirement(requirement_string=pkg_req_str) class CompilerExceptionWithExtendedTrace(CompilerException): @@ -722,14 +722,14 @@ def install(self, project: "Project", module_spec: list[InmantaModuleRequirement assert_pip_has_source(project.metadata.pip, f"a v2 module {module_name}") - requirements: list[Requirement] = [req.get_python_package_requirement() for req in module_spec] + requirements: list[SafeRequirement] = [req.get_python_package_requirement() for req in module_spec] preinstalled: Optional[ModuleV2] = self.get_installed_module(project, module_name) # Get known requires and add them to prevent invalidating constraints through updates # These could be constraints (-c) as well, but that requires additional sanitation # Because for pip not every valid -r is a valid -c current_requires = project.get_strict_python_requirements_as_list() - requirements += [Requirement(requirement_string=r) for r in current_requires] + requirements += [SafeRequirement(requirement_string=r) for r in current_requires] if preinstalled is not None: # log warning if preinstalled version does not match constraints @@ -1177,9 +1177,9 @@ def has_requirement_for(self, pkg_name: str) -> bool: Returns True iff this requirements.txt file contains the given package name. The given `pkg_name` is matched case insensitive against the requirements in this RequirementsTxtFile. """ - return any(r.name == pkg_name.lower() for r in self._requirements) + return any(r.name == pkg_name for r in self._requirements) # .lower() - def set_requirement_and_write(self, requirement: Requirement) -> None: + def set_requirement_and_write(self, requirement: SafeRequirement) -> None: """ Add the given requirement to the requirements.txt file and update the file on disk, replacing any existing constraints on this package. @@ -2128,7 +2128,7 @@ def install_modules(self, *, bypass_module_cache: bool = False, update_dependenc self.verify_module_version_compatibility() # do python install - pyreq: list[Requirement] = [Requirement(requirement_string=x) for x in self.collect_python_requirements()] + pyreq: list[SafeRequirement] = [SafeRequirement(requirement_string=x) for x in self.collect_python_requirements()] if len(pyreq) > 0: # upgrade both direct and transitive module dependencies: eager upgrade strategy @@ -2546,8 +2546,8 @@ def verify_python_requires(self) -> None: Verifies no incompatibilities exist within the Python environment with respect to installed module v2 requirements. """ if self.strict_deps_check: - constraints: list[Requirement] = [ - Requirement(requirement_string=item) for item in self.collect_python_requirements() + constraints: list[SafeRequirement] = [ + SafeRequirement(requirement_string=item) for item in self.collect_python_requirements() ] env.ActiveEnv.check(strict_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*"), constraints=constraints) else: @@ -2679,7 +2679,10 @@ def requires(self) -> "List[InmantaModuleRequirement]": # filter on import stmt reqs = [] for spec in self._metadata.requires: - req = [Requirement(requirement_string=e) for e in spec] + if isinstance(spec, list): + req = [SafeRequirement(requirement_string=e) for e in spec] + else: + req = [SafeRequirement(requirement_string=spec)] if len(req) > 1: print(f"Module file for {self._path} has bad line in requirements specification {spec}") reqe = InmantaModuleRequirement(req[0]) @@ -2816,7 +2819,10 @@ def requires(self) -> "List[InmantaModuleRequirement]": """ reqs = [] for spec in self.get_module_requirements(): - req = [Requirement(requirement_string=e) for e in spec] + if isinstance(spec, list): + req = [SafeRequirement(requirement_string=e) for e in spec] + else: + req = [SafeRequirement(requirement_string=spec)] if len(req) > 1: print(f"Module file for {self._path} has bad line in requirements specification {spec}") reqe = InmantaModuleRequirement(req[0]) @@ -3216,7 +3222,7 @@ def try_parse(x: str) -> Optional[version.Version]: versions = sorted(versions, reverse=True) for r in requirements: - versions = [x for x in r.specifier.filter(versions, not release_only)] + versions = [x for x in r.specifier.filter(iterable=versions, prereleases=not release_only)] comp_version_raw = get_compiler_version() comp_version = Version(version=comp_version_raw) @@ -3427,12 +3433,12 @@ def add_module_requirement_persistent(self, requirement: InmantaModuleRequiremen # Parse config file config_parser = ConfigParser() config_parser.read(self.get_metadata_file_path()) - python_pkg_requirement: Requirement = requirement.get_python_package_requirement() + python_pkg_requirement: SafeRequirement = requirement.get_python_package_requirement() if config_parser.has_option("options", "install_requires"): new_install_requires = [ r for r in config_parser.get("options", "install_requires").split("\n") - if r and Requirement(requirement_string=r).name != python_pkg_requirement.name + if r and SafeRequirement(requirement_string=r).name != python_pkg_requirement.name ] new_install_requires.append(str(python_pkg_requirement)) else: diff --git a/src/inmanta/moduletool.py b/src/inmanta/moduletool.py index e318ccd97e..35dcefb1d6 100644 --- a/src/inmanta/moduletool.py +++ b/src/inmanta/moduletool.py @@ -55,6 +55,7 @@ from inmanta.ast import CompilerException from inmanta.command import CLIException, ShowUsageException from inmanta.const import CF_CACHE_DIR, MAX_UPDATE_ATTEMPT +from inmanta.env import SafeRequirement from inmanta.module import ( DummyProject, FreezeOperator, @@ -75,7 +76,7 @@ gitprovider, ) from inmanta.stable_api import stable_api -from packaging.requirements import InvalidRequirement, Requirement +from packaging.requirements import InvalidRequirement from packaging.version import Version LOGGER = logging.getLogger(__name__) @@ -472,7 +473,7 @@ def update( def do_update(specs: Mapping[str, Sequence[InmantaModuleRequirement]], modules: list[str]) -> None: v2_modules = {module for module in modules if my_project.module_source.path_for(module) is not None} - v2_python_specs: list[Requirement] = [ + v2_python_specs: list[SafeRequirement] = [ module_spec.get_python_package_requirement() for module, module_specs in specs.items() for module_spec in module_specs @@ -484,7 +485,7 @@ def do_update(specs: Mapping[str, Sequence[InmantaModuleRequirement]], modules: # Because for pip not every valid -r is a valid -c current_requires = my_project.get_strict_python_requirements_as_list() env.process_env.install_for_config( - v2_python_specs + [Requirement(requirement_string=r) for r in current_requires], + v2_python_specs + [SafeRequirement(requirement_string=r) for r in current_requires], my_project.metadata.pip, upgrade=True, ) diff --git a/tests/conftest.py b/tests/conftest.py index 7ddb559c80..d593172b17 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -27,7 +27,6 @@ from inmanta.logging import InmantaLoggerConfig from inmanta.protocol import auth from inmanta.util import ScheduledTask, Scheduler, TaskMethod, TaskSchedule -from packaging.requirements import Requirement """ About the use of @parametrize_any and @slowtest: @@ -102,6 +101,7 @@ import pytest from asyncpg.exceptions import DuplicateDatabaseError from click import testing +from pkg_resources import Requirement from pyformance.registry import MetricsRegistry from tornado import netutil @@ -120,7 +120,7 @@ from inmanta.ast import CompilerException from inmanta.data.schema import SCHEMA_VERSION_TABLE from inmanta.db import util as db_util -from inmanta.env import CommandRunner, LocalPackagePath, VirtualEnv, mock_process_env +from inmanta.env import CommandRunner, LocalPackagePath, SafeRequirement, VirtualEnv, mock_process_env from inmanta.export import ResourceDict, cfg_env, unknown_parameters from inmanta.module import InmantaModuleRequirement, InstallMode, Project, RelationPrecedenceRule from inmanta.moduletool import DefaultIsolatedEnvCached, ModuleTool, V2ModuleBuilder @@ -510,11 +510,11 @@ def deactive_venv(): old_pythonpath = os.environ.get("PYTHONPATH", None) old_os_venv: Optional[str] = os.environ.get("VIRTUAL_ENV", None) old_process_env: str = env.process_env.python_path + old_working_set = pkg_resources.working_set old_available_extensions = ( dict(InmantaBootloader.AVAILABLE_EXTENSIONS) if InmantaBootloader.AVAILABLE_EXTENSIONS is not None else None ) - # TODO h we remove some magic here yield os.environ["PATH"] = old_os_path @@ -527,6 +527,7 @@ def deactive_venv(): sys.path_hooks.extend(old_path_hooks) # Clear cache for sys.path_hooks sys.path_importer_cache.clear() + pkg_resources.working_set = old_working_set # Restore PYTHONPATH if old_pythonpath is not None: os.environ["PYTHONPATH"] = old_pythonpath @@ -1147,7 +1148,7 @@ def setup_for_snippet( add_to_module_path: Optional[list[str]] = None, python_package_sources: Optional[list[str]] = None, project_requires: Optional[list[InmantaModuleRequirement]] = None, - python_requires: Optional[list[Requirement]] = None, + python_requires: Optional[list[SafeRequirement]] = None, install_mode: Optional[InstallMode] = None, relation_precedence_rules: Optional[list[RelationPrecedenceRule]] = None, strict_deps_check: Optional[bool] = None, @@ -1249,7 +1250,7 @@ def setup_for_snippet_external( add_to_module_path: Optional[list[str]] = None, python_package_sources: Optional[list[str]] = None, project_requires: Optional[list[InmantaModuleRequirement]] = None, - python_requires: Optional[list[Requirement]] = None, + python_requires: Optional[list[SafeRequirement]] = None, install_mode: Optional[InstallMode] = None, relation_precedence_rules: Optional[list[RelationPrecedenceRule]] = None, use_pip_config_file: bool = False, @@ -1912,8 +1913,8 @@ def index_with_pkgs_containing_optional_deps() -> str: path=os.path.join(tmpdirname, "pkg"), publish_index=pip_index, optional_dependencies={ - "optional-a": [Requirement(requirement_string="dep-a")], - "optional-b": [Requirement(requirement_string="dep-b"), Requirement(requirement_string="dep-c")], + "optional-a": [Requirement.parse("dep-a")], + "optional-b": [Requirement.parse("dep-b"), Requirement.parse("dep-c")], }, ) for pkg_name in ["dep-a", "dep-b", "dep-c"]: diff --git a/tests/moduletool/test_add.py b/tests/moduletool/test_add.py index 5a84124d61..458bfa3cec 100644 --- a/tests/moduletool/test_add.py +++ b/tests/moduletool/test_add.py @@ -25,10 +25,9 @@ import pytest from inmanta.command import CLIException -from inmanta.env import process_env +from inmanta.env import SafeRequirement, process_env from inmanta.module import ModuleV1, ModuleV1Metadata, ModuleV2, ModuleV2Source, Project, ProjectMetadata from inmanta.moduletool import ModuleTool -from packaging.requirements import Requirement from packaging.version import Version from utils import PipIndex, module_from_template @@ -89,7 +88,7 @@ def test_module_add_v2_module_to_project( dest_dir=os.path.join(tmpdir, f"elaboratev2module-v{version}"), new_version=Version(version), publish_index=pip_index, - new_extras={"optional": [Requirement(requirement_string="inmanta-module-minimalv2module")]}, + new_extras={"optional": [SafeRequirement(requirement_string="inmanta-module-minimalv2module")]}, ) # Create project diff --git a/tests/moduletool/test_convert_v1_v2.py b/tests/moduletool/test_convert_v1_v2.py index de082ac7da..b24e5793ef 100644 --- a/tests/moduletool/test_convert_v1_v2.py +++ b/tests/moduletool/test_convert_v1_v2.py @@ -30,10 +30,10 @@ import toml from inmanta import moduletool from inmanta.command import CLIException +from inmanta.env import SafeRequirement from inmanta.module import DummyProject, ModuleV1, ModuleV2, ModuleV2Metadata from inmanta.moduletool import ModuleConverter, ModuleVersionException from packaging import version -from packaging.requirements import Requirement from utils import log_contains, v1_module_from_template @@ -114,7 +114,9 @@ def test_issue_3159_conversion_std_module_add_self_to_dependencies(tmpdir): parser = configparser.ConfigParser() parser.read(setup_cfg_file) assert parser.has_option("options", "install_requires") - install_requires = [Requirement(requirement_string=r) for r in parser.get("options", "install_requires").split("\n") if r] + install_requires = [ + SafeRequirement(requirement_string=r) for r in parser.get("options", "install_requires").split("\n") if r + ] pkg_names = [r.name for r in install_requires] assert "inmanta-module-std" not in pkg_names diff --git a/tests/moduletool/test_install.py b/tests/moduletool/test_install.py index 137bab4031..91d92c6e5f 100644 --- a/tests/moduletool/test_install.py +++ b/tests/moduletool/test_install.py @@ -35,12 +35,11 @@ from inmanta.ast import CompilerException from inmanta.command import CLIException from inmanta.config import Config -from inmanta.env import CommandRunner, ConflictingRequirements, PipConfig +from inmanta.env import CommandRunner, ConflictingRequirements, PipConfig, SafeRequirement from inmanta.module import InmantaModuleRequirement, InstallMode, ModuleLoadingException, ModuleNotFoundException from inmanta.moduletool import DummyProject, ModuleConverter, ModuleTool, ProjectTool from moduletool.common import BadModProvider, install_project from packaging import version -from packaging.requirements import Requirement from utils import LogSequence, PipIndex, log_contains, module_from_template LOGGER = logging.getLogger(__name__) @@ -252,14 +251,14 @@ def test_module_install_conflicting_requirements(tmpdir: py.path.local, snippetc os.path.join(modules_v2_dir, "minimalv2module"), os.path.join(str(tmpdir), "modone"), new_name="modone", - new_requirements=[Requirement(requirement_string="lorem~=0.0.1")], + new_requirements=[SafeRequirement(requirement_string="lorem~=0.0.1")], install=True, ) module_from_template( os.path.join(modules_v2_dir, "minimalv2module"), os.path.join(str(tmpdir), "modtwo"), new_name="modtwo", - new_requirements=[Requirement(requirement_string="lorem~=0.1.0")], + new_requirements=[SafeRequirement(requirement_string="lorem~=0.1.0")], install=True, ) @@ -509,7 +508,7 @@ def test_project_install( index_url=local_module_package_index, # We add tornado, as there is a code path in update for the case where the project has python requires python_requires=["tornado"] - + [Requirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names], + + [SafeRequirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names], install_project=False, ) @@ -542,7 +541,7 @@ def test_project_install( autostd=False, python_package_sources=[local_module_package_index], python_requires=[ - Requirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names + SafeRequirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names ] + ["lorem"], install_project=False, @@ -678,7 +677,7 @@ def test_project_install_modules_cache_invalid( index_url=index.url, extra_index_url=[local_module_package_index], # make sure main module gets installed, pulling in newest version of dependency module - python_requires=[Requirement(requirement_string=module.ModuleV2Source.get_package_name_for(main_module))], + python_requires=[SafeRequirement(requirement_string=module.ModuleV2Source.get_package_name_for(main_module))], ) # populate project.modules[dependency_module] to force the error conditions in this simplified example @@ -763,7 +762,7 @@ def test_project_install_incompatible_versions( install_project=False, add_to_module_path=[v1_modules_path], index_url=index.url, - python_requires=[Requirement(requirement_string=module.ModuleV2Source.get_package_name_for(v2_mod_name))], + python_requires=[SafeRequirement(requirement_string=module.ModuleV2Source.get_package_name_for(v2_mod_name))], ) # install project @@ -816,14 +815,14 @@ def test_project_install_incompatible_dependencies( v2_template_path, os.path.join(str(tmpdir), "v2mod2"), new_name="v2mod2", - new_requirements=[Requirement(requirement_string="inmanta-module-v2mod1~=1.0.0")], + new_requirements=[SafeRequirement(requirement_string="inmanta-module-v2mod1~=1.0.0")], publish_index=index, ) v2mod3: module.ModuleV2Metadata = module_from_template( v2_template_path, os.path.join(str(tmpdir), "v2mod3"), new_name="v2mod3", - new_requirements=[Requirement(requirement_string="inmanta-module-v2mod1~=2.0.0")], + new_requirements=[SafeRequirement(requirement_string="inmanta-module-v2mod1~=2.0.0")], publish_index=index, ) @@ -837,7 +836,7 @@ def test_project_install_incompatible_dependencies( install_project=False, index_url=index.url, python_requires=[ - Requirement( + SafeRequirement( requirement_string=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod2, v2mod3] @@ -921,7 +920,7 @@ def test_install_from_index_dont_leak_pip_index( # Installing a V2 module requires a python package source. index_url="unknown", python_requires=[ - Requirement( + SafeRequirement( requirement_string=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod1] @@ -983,7 +982,7 @@ def test_install_with_use_config( index_url=index.url if not use_pip_config else None, use_pip_config_file=use_pip_config, python_requires=[ - Requirement( + SafeRequirement( requirement_string=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod1] @@ -1052,7 +1051,7 @@ def test_install_with_use_config_extra_index( extra_index_url=[index2.url], use_pip_config_file=True, python_requires=[ - Requirement( + SafeRequirement( requirement_string=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod1, v2mod2] @@ -1088,7 +1087,7 @@ def test_install_with_use_config_but_PIP_CONFIG_FILE_not_set( autostd=False, install_project=False, use_pip_config_file=True, - python_requires=[Requirement(requirement_string="inmanta-module-dummy-module")], + python_requires=[SafeRequirement(requirement_string="inmanta-module-dummy-module")], ) # install project @@ -1205,7 +1204,7 @@ def test_install_project_with_install_mode_master(tmpdir: py.path.local, snippet autostd=False, install_project=False, add_to_module_path=[str(tmpdir)], - project_requires=[InmantaModuleRequirement(Requirement(requirement_string="mod11==3.2.1"))], + project_requires=[InmantaModuleRequirement(SafeRequirement(requirement_string="mod11==3.2.1"))], install_mode=InstallMode.master, ) @@ -1233,7 +1232,7 @@ def test_module_install_logging(local_module_package_index: str, snippetcompiler v2_module = "minimalv2module" - v2_requirements = [Requirement(requirement_string=module.ModuleV2Source.get_package_name_for(v2_module))] + v2_requirements = [SafeRequirement(requirement_string=module.ModuleV2Source.get_package_name_for(v2_module))] # set up project and modules project: module.Project = snippetcompiler_clean.setup_for_snippet( @@ -1334,7 +1333,7 @@ def test_pip_output(local_module_package_index: str, snippetcompiler_clean, capl ) modules = ["modone", "modtwo"] - v2_requirements = [Requirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in modules] + v2_requirements = [SafeRequirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in modules] snippetcompiler_clean.setup_for_snippet( f""" @@ -1420,7 +1419,7 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[Requirement(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[SafeRequirement(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module"))], install_project=True, ) log_contains( @@ -1452,7 +1451,7 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[Requirement(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[SafeRequirement(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module"))], install_project=True, ) @@ -1485,7 +1484,7 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[Requirement(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[SafeRequirement(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module"))], install_project=True, ) log_contains( @@ -1554,7 +1553,7 @@ def test_version_snapshot(local_module_package_index: str, snippetcompiler_clean autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[Requirement(requirement_string=module.ModuleV2Source.get_package_name_for("module_b"))], + python_requires=[SafeRequirement(requirement_string=module.ModuleV2Source.get_package_name_for("module_b"))], install_project=True, ) @@ -1580,7 +1579,7 @@ def test_version_snapshot(local_module_package_index: str, snippetcompiler_clean autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[Requirement(requirement_string=module.ModuleV2Source.get_package_name_for("module_c"))], + python_requires=[SafeRequirement(requirement_string=module.ModuleV2Source.get_package_name_for("module_c"))], install_project=True, ) @@ -1651,7 +1650,8 @@ def test_constraints_logging_v2(modules_v2_dir, tmpdir, caplog, snippetcompiler_ index_url=local_module_package_index, extra_index_url=[index.url], python_requires=[ - Requirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in ["module_b", "module_a"] + SafeRequirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) + for mod in ["module_b", "module_a"] ], install_project=True, project_requires=[ diff --git a/tests/moduletool/test_update.py b/tests/moduletool/test_update.py index d18219e811..07369f0a85 100644 --- a/tests/moduletool/test_update.py +++ b/tests/moduletool/test_update.py @@ -23,12 +23,11 @@ from inmanta.config import Config from inmanta.data.model import PipConfig -from inmanta.env import LocalPackagePath, process_env +from inmanta.env import LocalPackagePath, SafeRequirement, process_env from inmanta.module import InmantaModuleRequirement, InstallMode, ModuleV1, ModuleV2Source from inmanta.moduletool import ProjectTool from inmanta.parser import ParserException from moduletool.common import add_file, clone_repo -from packaging.requirements import Requirement from packaging.version import Version from utils import PipIndex, create_python_package, module_from_template, v1_module_from_template @@ -126,7 +125,7 @@ def assert_version_installed(module_name: str, version: str) -> None: new_version=Version(current_version), new_name=module_name, new_requirements=( - [InmantaModuleRequirement(Requirement(requirement_string="module2<3.0.0"))] + [InmantaModuleRequirement(SafeRequirement(requirement_string="module2<3.0.0"))] if module_name == "module1" else None ), @@ -143,7 +142,7 @@ def assert_version_installed(module_name: str, version: str) -> None: # Add a dependency on module2, without setting an explicit version constraint. Later version of module1 # do set a version constraint on the dependency on module2. This way it is verified whether the module update # command takes into account the version constraints set in a new version of a module. - new_requirements=[InmantaModuleRequirement(Requirement(requirement_string="module2"))], + new_requirements=[InmantaModuleRequirement(SafeRequirement(requirement_string="module2"))], install=False, publish_index=pip_index, new_content_init_cf="entity" if corrupt_module else None, # Introduce syntax error in the module @@ -240,7 +239,11 @@ def test_module_update_dependencies( create_python_package("a", Version("1.0.0"), str(tmpdir.join("a-1.0.0")), publish_index=index) for v in ("1.0.0", "1.0.1", "2.0.0"): create_python_package( - "b", Version(v), str(tmpdir.join(f"b-{v}")), requirements=[Requirement(requirement_string="c")], publish_index=index + "b", + Version(v), + str(tmpdir.join(f"b-{v}")), + requirements=[SafeRequirement(requirement_string="c")], + publish_index=index, ) for v in ("1.0.0", "2.0.0"): create_python_package("c", Version(v), str(tmpdir.join(f"c-{v}")), publish_index=index) @@ -255,7 +258,7 @@ def test_module_update_dependencies( # install b-1.0.0 and c-1.0.0 process_env.install_for_config( - [Requirement(requirement_string=req) for req in ("b==1.0.0", "c==1.0.0")], + [SafeRequirement(requirement_string=req) for req in ("b==1.0.0", "c==1.0.0")], config=PipConfig( index_url=index.url, use_system_config=False, @@ -267,7 +270,7 @@ def test_module_update_dependencies( source_dir=os.path.join(modules_dir, "minimalv1module"), dest_dir=str(tmpdir.join("modules", "my_mod")), new_name="my_mod", - new_requirements=[Requirement(requirement_string=req) for req in ("a", "b~=1.0.0")], + new_requirements=[SafeRequirement(requirement_string=req) for req in ("a", "b~=1.0.0")], ) # run `inmanta project update` without running install first diff --git a/tests/server/test_compilerservice.py b/tests/server/test_compilerservice.py index a035ee399e..9dd286324a 100644 --- a/tests/server/test_compilerservice.py +++ b/tests/server/test_compilerservice.py @@ -41,7 +41,7 @@ from inmanta.const import INMANTA_REMOVED_SET_ID, ParameterSource from inmanta.data import APILIMIT, Compile, Report from inmanta.data.model import PipConfig -from inmanta.env import PythonEnvironment +from inmanta.env import PythonEnvironment, SafeRequirement from inmanta.export import cfg_env from inmanta.protocol import Result from inmanta.server import SLICE_COMPILER, SLICE_SERVER, protocol @@ -50,7 +50,6 @@ from inmanta.server.services.compilerservice import CompilerService, CompileRun, CompileStateListener from inmanta.server.services.notificationservice import NotificationService from inmanta.util import ensure_directory_exist -from packaging.requirements import Requirement from server.conftest import EnvironmentFactory from utils import LogSequence, report_db_index_usage, retry_limited, wait_for_version @@ -1807,7 +1806,7 @@ def patch_get_protected_inmanta_packages(): venv = PythonEnvironment(env_path=venv_path) assert name_protected_pkg not in venv.get_installed_packages() venv.install_for_config( - requirements=[Requirement(requirement_string=name_protected_pkg)], + requirements=[SafeRequirement(requirement_string=name_protected_pkg)], config=PipConfig( index_url=local_module_package_index, ), diff --git a/tests/test_env.py b/tests/test_env.py index 6e0ec185e7..b389e3d818 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -35,9 +35,8 @@ from inmanta import env, loader, module from inmanta.data.model import PipConfig -from inmanta.env import Pip +from inmanta.env import Pip, SafeRequirement from packaging import version -from packaging.requirements import Requirement from utils import LogSequence, PipIndex, create_python_package if "inmanta-core" in env.process_env.get_installed_packages(only_editable=True): @@ -178,7 +177,7 @@ def test_gen_req_file(): # make sure they all parse for req in reqs: - Requirement(requirement_string=req) + SafeRequirement(requirement_string=req) def test_environment_python_version_multi_digit(tmpdir: py.path.local) -> None: @@ -207,7 +206,7 @@ def test_process_env_install_from_index( package_name: str = "more-itertools" assert package_name not in env.process_env.get_installed_packages() env.process_env.install_for_config( - [Requirement(requirement_string=package_name + (f"=={version}" if version is not None else ""))], + [SafeRequirement(requirement_string=package_name + (f"=={version}" if version is not None else ""))], config=PipConfig( use_system_config=True, # we need an upstream for some packages ), @@ -222,7 +221,7 @@ def test_process_env_install_from_index( # It should hit the cache there and return here. # Cheap and fast test env.process_env.install_from_index( - [Requirement(requirement_string=package_name + (f"=={version}" if version is not None else ""))], + [SafeRequirement(requirement_string=package_name + (f"=={version}" if version is not None else ""))], use_pip_config=True, ) @@ -272,7 +271,7 @@ def test_process_env_install_from_index_not_found_env_var( with pytest.raises(env.PackageNotFound, match=re.escape(expected)): env.process_env.install_for_config( - [Requirement(requirement_string="this-package-does-not-exist")], + [SafeRequirement(requirement_string="this-package-does-not-exist")], config=PipConfig( index_url=index_urls[0], # The first element should only be passed to the index_url. If there are indexes in the environment @@ -309,7 +308,7 @@ def test_process_env_install_no_index(tmpdir: py.path.local, monkeypatch, use_sy with pytest.raises(env.PackageNotFound, match=re.escape(expected)): env.process_env.install_for_config( - requirements=[Requirement(requirement_string="this-package-does-not-exist")], + requirements=[SafeRequirement(requirement_string="this-package-does-not-exist")], paths=[env.LocalPackagePath(path=str(tmpdir))], config=PipConfig(use_system_config=use_system_config), ) @@ -326,7 +325,7 @@ def test_process_env_install_from_index_conflicting_reqs( package_name: str = "more-itertools" with pytest.raises(env.ConflictingRequirements) as e: env.process_env.install_for_config( - [Requirement(requirement_string=f"{package_name}{version}") for version in [">8.5", "<=8"]], + [SafeRequirement(requirement_string=f"{package_name}{version}") for version in [">8.5", "<=8"]], config=PipConfig( use_system_config=True, # we need an upstream for some packages ), @@ -396,7 +395,7 @@ def test_active_env_get_module_file( loader.PluginModuleFinder.configure_module_finder([os.path.join(str(tmpdir), "libs")]) assert env.ActiveEnv.get_module_file(module_name) is None - env.process_env.install_for_config([Requirement(requirement_string=package_name)], pip_config) + env.process_env.install_for_config([SafeRequirement(requirement_string=package_name)], pip_config) assert package_name in env.process_env.get_installed_packages() module_info: Optional[tuple[Optional[str], Loader]] = env.ActiveEnv.get_module_file(module_name) assert module_info is not None @@ -452,7 +451,7 @@ def test_active_env_get_module_file_editable_namespace_package( def create_install_package( - name: str, version: version.Version, requirements: list[Requirement], local_module_package_index: str + name: str, version: version.Version, requirements: list[SafeRequirement], local_module_package_index: str ) -> None: """ Creates and installs a simple package with specified requirements. Creates package in a temporary directory and @@ -542,7 +541,7 @@ def assert_all_checks(expect_test: tuple[bool, str] = (True, ""), expect_nonext: create_install_package( "test-package-two", version.Version("1.0.0"), - [Requirement(requirement_string="test-package-one~=1.0")], + [SafeRequirement(requirement_string="test-package-one~=1.0")], local_module_package_index, ) assert_all_checks() @@ -563,7 +562,7 @@ def test_active_env_check_constraints(caplog, tmpvenv_active_inherit: str, local """ caplog.set_level(logging.WARNING) in_scope: Pattern[str] = re.compile("test-package-.*") - constraints: list[Requirement] = [Requirement(requirement_string="test-package-one~=1.0")] + constraints: list[SafeRequirement] = [SafeRequirement(requirement_string="test-package-one~=1.0")] env.ActiveEnv.check(in_scope) @@ -582,7 +581,7 @@ def test_active_env_check_constraints(caplog, tmpvenv_active_inherit: str, local create_install_package( "ext-package-one", version.Version("1.0.0"), - [Requirement(requirement_string="test-package-one==1.0")], + [SafeRequirement(requirement_string="test-package-one==1.0")], local_module_package_index, ) env.ActiveEnv.check(in_scope, constraints) @@ -613,7 +612,7 @@ def test_override_inmanta_package(tmpvenv_active_inherit: env.VirtualEnv) -> Non installed_pkgs = tmpvenv_active_inherit.get_installed_packages() assert "inmanta-core" in installed_pkgs, "The inmanta-core package should be installed to run the tests" - inmanta_requirements = Requirement(requirement_string="inmanta-core==4.0.0") + inmanta_requirements = SafeRequirement(requirement_string="inmanta-core==4.0.0") with pytest.raises(env.ConflictingRequirements) as excinfo: tmpvenv_active_inherit.install_for_config( requirements=[inmanta_requirements], @@ -653,13 +652,13 @@ def test_cache_on_active_env(tmpvenv_active_inherit: env.ActiveEnv, local_module """ def _assert_install(requirement: str, installed: bool) -> None: - parsed_requirement = Requirement(requirement_string=requirement) + parsed_requirement = SafeRequirement(requirement_string=requirement) for r in [requirement, parsed_requirement]: assert tmpvenv_active_inherit.are_installed(requirements=[r]) == installed _assert_install("inmanta-module-elaboratev2module==1.2.3", installed=False) tmpvenv_active_inherit.install_for_config( - requirements=[Requirement(requirement_string="inmanta-module-elaboratev2module==1.2.3")], + requirements=[SafeRequirement(requirement_string="inmanta-module-elaboratev2module==1.2.3")], config=PipConfig( index_url=local_module_package_index, ), @@ -703,7 +702,7 @@ def test_are_installed_dependency_cycle_on_extra(tmpdir, tmpvenv_active_inherit: path=os.path.join(tmpdir, "pkg"), publish_index=pip_index, optional_dependencies={ - "optional-pkg": [Requirement(requirement_string="dep[optional-dep]")], + "optional-pkg": [SafeRequirement(requirement_string="dep[optional-dep]")], }, ) create_python_package( @@ -712,11 +711,11 @@ def test_are_installed_dependency_cycle_on_extra(tmpdir, tmpvenv_active_inherit: path=os.path.join(tmpdir, "dep"), publish_index=pip_index, optional_dependencies={ - "optional-dep": [Requirement(requirement_string="pkg[optional-pkg]")], + "optional-dep": [SafeRequirement(requirement_string="pkg[optional-pkg]")], }, ) - requirements = [Requirement(requirement_string="pkg[optional-pkg]")] + requirements = [SafeRequirement(requirement_string="pkg[optional-pkg]")] tmpvenv_active_inherit.install_for_config( requirements=requirements, config=PipConfig( diff --git a/tests/test_file_parser.py b/tests/test_file_parser.py index 34554b5093..0904c34d03 100644 --- a/tests/test_file_parser.py +++ b/tests/test_file_parser.py @@ -18,8 +18,8 @@ import os +from inmanta.env import SafeRequirement from inmanta.file_parser import RequirementsTxtParser -from packaging.requirements import Requirement def test_requirements_txt_parser(tmpdir) -> None: @@ -39,8 +39,8 @@ def test_requirements_txt_parser(tmpdir) -> None: fd.write(content) expected_requirements = ["test==1.2.3", "other-dep~=2.0.0", "third-dep<5.0.0", "splitteddep", "Capital"] - requirements: list[Requirement] = RequirementsTxtParser().parse(requirements_txt_file) - assert requirements == [Requirement(requirement_string=r) for r in expected_requirements] + requirements: list[SafeRequirement] = RequirementsTxtParser().parse(requirements_txt_file) + assert requirements == [SafeRequirement(requirement_string=r) for r in expected_requirements] requirements_as_str = RequirementsTxtParser.parse_requirements_as_strs(requirements_txt_file) assert requirements_as_str == expected_requirements diff --git a/tests/test_module_loader.py b/tests/test_module_loader.py index 995a297b16..2992687d0f 100644 --- a/tests/test_module_loader.py +++ b/tests/test_module_loader.py @@ -33,7 +33,7 @@ from inmanta.ast import CompilerException from inmanta.const import CF_CACHE_DIR from inmanta.data.model import PipConfig -from inmanta.env import ConflictingRequirements, LocalPackagePath, PackageNotFound, process_env +from inmanta.env import ConflictingRequirements, LocalPackagePath, PackageNotFound, SafeRequirement, process_env from inmanta.module import ( DummyProject, InmantaModuleRequirement, @@ -45,7 +45,6 @@ Project, ) from inmanta.moduletool import ModuleConverter, ModuleTool, ProjectTool -from packaging.requirements import Requirement from packaging.version import Version from utils import PipIndex, create_python_package, log_contains, module_from_template, v1_module_from_template @@ -346,7 +345,7 @@ def test_load_module_recursive_v2_module_depends_on_v1( project = snippetcompiler.setup_for_snippet( snippet="import v2_depends_on_v1", index_url=local_module_package_index, - python_requires=[Requirement(requirement_string="inmanta-module-v2-depends-on-v1")], + python_requires=[SafeRequirement(requirement_string="inmanta-module-v2-depends-on-v1")], install_project=False, ) if preload_v1_module: @@ -373,7 +372,7 @@ def test_load_module_recursive_complex_module_dependencies(local_module_package_ snippet="import complex_module_dependencies_mod1", autostd=False, index_url=local_module_package_index, - python_requires=[Requirement(requirement_string="inmanta-module-complex-module-dependencies-mod1")], + python_requires=[SafeRequirement(requirement_string="inmanta-module-complex-module-dependencies-mod1")], install_project=False, ) assert "complex_module_dependencies_mod1" not in project.modules @@ -397,7 +396,7 @@ def test_load_import_based_v2_project(local_module_package_index: str, snippetco """ module_name: str = "minimalv2module" - def load(requires: Optional[list[Requirement]] = None) -> None: + def load(requires: Optional[list[SafeRequirement]] = None) -> None: project: Project = snippetcompiler_clean.setup_for_snippet( f"import {module_name}", autostd=False, @@ -412,7 +411,7 @@ def load(requires: Optional[list[Requirement]] = None) -> None: with pytest.raises(ModuleLoadingException, match=f"Failed to load module {module_name}"): load() # assert that it doesn't raise an error with explicit requirements set - load([Requirement(requirement_string=ModuleV2Source.get_package_name_for(module_name))]) + load([SafeRequirement(requirement_string=ModuleV2Source.get_package_name_for(module_name))]) @pytest.mark.parametrize("v1", [True, False]) @@ -469,7 +468,9 @@ def test_load_import_based_v2_module( extra_index_url=[index.url], # make sure that even listing the requirement in project.yml does not suffice project_requires=[InmantaModuleRequirement.parse(dependency_module_name)], - python_requires=[] if v1 else [Requirement(requirement_string=ModuleV2Source.get_package_name_for(main_module_name))], + python_requires=( + [] if v1 else [SafeRequirement(requirement_string=ModuleV2Source.get_package_name_for(main_module_name))] + ), ) if explicit_dependency: @@ -611,7 +612,7 @@ def test_project_requirements_dont_overwrite_core_requirements_source( module_from_template( os.path.join(modules_v2_dir, module_name), module_path, - new_requirements=[Requirement(requirement_string="Jinja2==2.11.3")], + new_requirements=[SafeRequirement(requirement_string="Jinja2==2.11.3")], ) # Activate the snippetcompiler venv @@ -655,7 +656,7 @@ def test_project_requirements_dont_overwrite_core_requirements_index( module_from_template( os.path.join(modules_v2_dir, module_name), module_path, - new_requirements=[Requirement(requirement_string="Jinja2==2.11.3")], + new_requirements=[SafeRequirement(requirement_string="Jinja2==2.11.3")], publish_index=index, ) @@ -709,7 +710,7 @@ def test_module_conflicting_dependencies_with_v2_modules( "y", Version("1.0.0"), str(tmpdir.join("y-1.0.0")), - requirements=[Requirement(requirement_string="x~=1.0.0")], + requirements=[SafeRequirement(requirement_string="x~=1.0.0")], publish_index=index, ) @@ -719,7 +720,7 @@ def test_module_conflicting_dependencies_with_v2_modules( module_from_template( os.path.join(modules_v2_dir, module_name1), module_path1, - new_requirements=[Requirement(requirement_string="y~=1.0.0")], + new_requirements=[SafeRequirement(requirement_string="y~=1.0.0")], publish_index=index, ) @@ -730,7 +731,7 @@ def test_module_conflicting_dependencies_with_v2_modules( os.path.join(modules_v2_dir, "minimalv2module"), module_path2, new_name="minimalv2module2", - new_requirements=[Requirement(requirement_string="x~=2.0.0")], + new_requirements=[SafeRequirement(requirement_string="x~=2.0.0")], publish_index=index, ) @@ -790,7 +791,7 @@ def test_module_conflicting_dependencies_with_v1_module( os.path.join(modules_dir, module_name1), module_path1, new_name="modulev1", - new_requirements=[Requirement(requirement_string="y~=1.0.0")], + new_requirements=[SafeRequirement(requirement_string="y~=1.0.0")], ) # Create the second module @@ -799,7 +800,7 @@ def test_module_conflicting_dependencies_with_v1_module( module_from_template( os.path.join(modules_v2_dir, module_name2), module_path2, - new_requirements=[Requirement(requirement_string="y~=2.0.0")], + new_requirements=[SafeRequirement(requirement_string="y~=2.0.0")], publish_index=index, ) @@ -847,11 +848,11 @@ def test_module_install_extra_on_project_level_v2_dep( new_name="mymod", new_requirements=[], new_extras={ - "myfeature": [Requirement(requirement_string=package_name_extra)], + "myfeature": [SafeRequirement(requirement_string=package_name_extra)], }, publish_index=index, ) - package_with_extra: Requirement = InmantaModuleRequirement.parse("mymod[myfeature]").get_python_package_requirement() + package_with_extra: SafeRequirement = InmantaModuleRequirement.parse("mymod[myfeature]").get_python_package_requirement() package_name: str = f"{ModuleV2.PKG_NAME_PREFIX}mymod" # project with dependency on mymod with extra @@ -896,7 +897,7 @@ def test_module_install_extra_on_dep_of_v2_module( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [Requirement(requirement_string=package_name_extra)], + "myfeature": [SafeRequirement(requirement_string=package_name_extra)], }, publish_index=index, ) @@ -916,7 +917,7 @@ def test_module_install_extra_on_dep_of_v2_module( install_project=True, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[Requirement(requirement_string="inmanta-module-myv2mod")], + python_requires=[SafeRequirement(requirement_string="inmanta-module-myv2mod")], autostd=False, ) @@ -953,7 +954,7 @@ def test_module_install_extra_on_dep_of_v1_module( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [Requirement(requirement_string=package_name_extra)], + "myfeature": [SafeRequirement(requirement_string=package_name_extra)], }, publish_index=index, ) @@ -1003,12 +1004,12 @@ def test_module_install_extra_on_project_level_v2_dep_update_scenario( new_name="mymod", new_requirements=[], new_extras={ - "myfeature": [Requirement(requirement_string=package_name_extra)], + "myfeature": [SafeRequirement(requirement_string=package_name_extra)], }, publish_index=index, ) - package_without_extra: Requirement = InmantaModuleRequirement.parse("mymod").get_python_package_requirement() - package_with_extra: Requirement = InmantaModuleRequirement.parse("mymod[myfeature]").get_python_package_requirement() + package_without_extra: SafeRequirement = InmantaModuleRequirement.parse("mymod").get_python_package_requirement() + package_with_extra: SafeRequirement = InmantaModuleRequirement.parse("mymod[myfeature]").get_python_package_requirement() package_name: str = str(package_without_extra) def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> None: @@ -1072,12 +1073,12 @@ def test_module_install_extra_on_dep_of_v2_module_update_scenario( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [Requirement(requirement_string=package_name_extra)], + "myfeature": [SafeRequirement(requirement_string=package_name_extra)], }, publish_index=index, ) - package_without_extra: Requirement = InmantaModuleRequirement.parse("depmod").get_python_package_requirement() - package_with_extra: Requirement = InmantaModuleRequirement.parse("depmod[myfeature]").get_python_package_requirement() + package_without_extra: SafeRequirement = InmantaModuleRequirement.parse("depmod").get_python_package_requirement() + package_with_extra: SafeRequirement = InmantaModuleRequirement.parse("depmod[myfeature]").get_python_package_requirement() package_name: str = str(package_without_extra) def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> None: @@ -1101,7 +1102,7 @@ def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> install_project=True, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[Requirement(requirement_string="inmanta-module-myv2mod==1.0.0")], + python_requires=[SafeRequirement(requirement_string="inmanta-module-myv2mod==1.0.0")], autostd=False, ) assert_installed(extra_installed=False) @@ -1123,7 +1124,7 @@ def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> install_project=not do_project_update, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[Requirement(requirement_string="inmanta-module-myv2mod==2.0.0")], + python_requires=[SafeRequirement(requirement_string="inmanta-module-myv2mod==2.0.0")], autostd=False, ) if do_project_update: @@ -1157,8 +1158,8 @@ def test_module_install_extra_on_dep_of_v1_module_update_scenario( index: PipIndex = PipIndex(artifact_dir=str(tmpdir.join(".index"))) # Publish dependency of V1 module (depmod) to python package repo - package_without_extra: Requirement = InmantaModuleRequirement.parse("depmod").get_python_package_requirement() - package_with_extra: Requirement = InmantaModuleRequirement.parse("depmod[myfeature]").get_python_package_requirement() + package_without_extra: SafeRequirement = InmantaModuleRequirement.parse("depmod").get_python_package_requirement() + package_with_extra: SafeRequirement = InmantaModuleRequirement.parse("depmod[myfeature]").get_python_package_requirement() package_name: str = str(package_without_extra) module_from_template( @@ -1167,7 +1168,7 @@ def test_module_install_extra_on_dep_of_v1_module_update_scenario( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [Requirement(requirement_string=package_name_extra)], + "myfeature": [SafeRequirement(requirement_string=package_name_extra)], }, publish_index=index, ) @@ -1231,7 +1232,7 @@ async def test_v1_module_depends_on_third_party_dep_with_extra( os.path.join(tmpdir, "myv1mod"), new_name="myv1mod", new_content_init_cf="", - new_requirements=[Requirement(requirement_string="pkg[optional-a]")], + new_requirements=[SafeRequirement(requirement_string="pkg[optional-a]")], ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv1mod", @@ -1251,7 +1252,7 @@ async def test_v1_module_depends_on_third_party_dep_with_extra( os.path.join(tmpdir, "myv1mod"), new_name="myv1mod", new_content_init_cf="", - new_requirements=[Requirement(requirement_string="pkg[optional-a,optional-b]")], + new_requirements=[SafeRequirement(requirement_string="pkg[optional-a,optional-b]")], ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv1mod", @@ -1278,13 +1279,13 @@ async def test_v2_module_depends_on_third_party_dep_with_extra( str(tmpdir.join("myv2mod")), new_name="myv2mod", new_version=Version("1.0.0"), - new_requirements=[Requirement(requirement_string="pkg[optional-a]")], + new_requirements=[SafeRequirement(requirement_string="pkg[optional-a]")], publish_index=index, ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv2mod", install_project=True, - python_requires=[Requirement(requirement_string="inmanta-module-myv2mod==1.0.0")], + python_requires=[SafeRequirement(requirement_string="inmanta-module-myv2mod==1.0.0")], index_url=index.url, extra_index_url=[index_with_pkgs_containing_optional_deps], autostd=False, @@ -1300,13 +1301,13 @@ async def test_v2_module_depends_on_third_party_dep_with_extra( str(tmpdir.join("myv2mod")), new_name="myv2mod", new_version=Version("2.0.0"), - new_requirements=[Requirement(requirement_string="pkg[optional-a,optional-b]")], + new_requirements=[SafeRequirement(requirement_string="pkg[optional-a,optional-b]")], publish_index=index, ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv2mod", install_project=True, - python_requires=[Requirement(requirement_string="inmanta-module-myv2mod==2.0.0")], + python_requires=[SafeRequirement(requirement_string="inmanta-module-myv2mod==2.0.0")], index_url=index.url, extra_index_url=[index_with_pkgs_containing_optional_deps], autostd=False, diff --git a/tests/utils.py b/tests/utils.py index a7997d8d53..6902e478a1 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -44,6 +44,7 @@ from inmanta import config, const, data, env, module, protocol, util from inmanta.data import ResourceIdStr from inmanta.data.model import PipConfig +from inmanta.env import SafeRequirement from inmanta.moduletool import ModuleTool from inmanta.protocol import Client from inmanta.server.bootloader import InmantaBootloader @@ -51,7 +52,6 @@ from inmanta.util import get_compiler_version, hash_file from libpip2pi.commands import dir2pi from packaging import version -from packaging.requirements import Requirement from packaging.version import Version T = TypeVar("T") @@ -488,11 +488,11 @@ def create_python_package( pkg_version: version.Version, path: str, *, - requirements: Optional[Sequence[Requirement]] = None, + requirements: Optional[Sequence[SafeRequirement]] = None, install: bool = False, editable: bool = False, publish_index: Optional[PipIndex] = None, - optional_dependencies: Optional[dict[str, Sequence[Requirement]]] = None, + optional_dependencies: Optional[dict[str, Sequence[SafeRequirement]]] = None, ) -> None: """ Creates an empty Python package. @@ -577,8 +577,8 @@ def module_from_template( *, new_version: Optional[version.Version] = None, new_name: Optional[str] = None, - new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, Requirement]]] = None, - new_extras: Optional[abc.Mapping[str, abc.Sequence[Union[module.InmantaModuleRequirement, Requirement]]]] = None, + new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, SafeRequirement]]] = None, + new_extras: Optional[abc.Mapping[str, abc.Sequence[Union[module.InmantaModuleRequirement, SafeRequirement]]]] = None, install: bool = False, editable: bool = False, publish_index: Optional[PipIndex] = None, @@ -606,10 +606,8 @@ def module_from_template( :param four_digit_version: if the version uses 4 digits (3 by default) """ - def to_python_requires( - requires: abc.Sequence[Union[module.InmantaModuleRequirement, Requirement]] - ) -> abc.Iterator[Requirement]: - return (str(req if isinstance(req, Requirement) else req.get_python_package_requirement()) for req in requires) + def to_python_requires(requires: abc.Sequence[Union[module.InmantaModuleRequirement, SafeRequirement]]) -> list[str]: + return [str(req if isinstance(req, SafeRequirement) else str(req.get_python_package_requirement())) for req in requires] if (dest_dir is None) != in_place: raise ValueError("Either dest_dir or in_place must be set, never both.") @@ -690,7 +688,7 @@ def v1_module_from_template( *, new_version: Optional[version.Version] = None, new_name: Optional[str] = None, - new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, Requirement]]] = None, + new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, SafeRequirement]]] = None, new_content_init_cf: Optional[str] = None, new_content_init_py: Optional[str] = None, ) -> module.ModuleV2Metadata: @@ -730,7 +728,7 @@ def v1_module_from_template( with open(os.path.join(dest_dir, "requirements.txt"), "w") as fd: fd.write( "\n".join( - str(req if isinstance(req, Requirement) else req.get_python_package_requirement()) + str(req if isinstance(req, SafeRequirement) else req.get_python_package_requirement()) for req in new_requirements ) ) From 3a5743ac95d8608dd7c5e97b76fcb7d173be6496 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Thu, 8 Aug 2024 16:03:37 +0200 Subject: [PATCH 05/74] lower name --- src/inmanta/env.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 5fa454d67c..73d3a127fb 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -230,8 +230,8 @@ def _are_installed_recursive( if r.marker and not r.marker.evaluate(environment=environment_marker_evaluation): # The marker of the requirement doesn't apply on this environment continue - if r.name not in installed_packages or ( - len(r.specifier) > 0 and str(installed_packages[r.name]) not in r.specifier + if r.name.lower() not in installed_packages or ( + len(r.specifier) > 0 and str(installed_packages[r.name.lower()]) not in r.specifier ): return False if r.extras: From a6a6ddea1b9664373d2bfdab30aeb1019ca7d0f6 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Fri, 9 Aug 2024 10:25:59 +0200 Subject: [PATCH 06/74] test --- src/inmanta/env.py | 3 --- tests/conftest.py | 10 +++------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 73d3a127fb..4e20d7f632 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -1033,8 +1033,6 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: for requirement in dist_info.requires or [] if SafeRequirement(requirement).marker is None ) - # for e in installed_constraints: - # e.requirement.name = canonicalize_name(e.requirement.name).lower() inmanta_constraints: abc.Set[OwnedRequirement] = frozenset( OwnedRequirement(r, owner="inmanta-core") for r in cls._get_requirements_on_inmanta_package() @@ -1265,7 +1263,6 @@ def use_virtual_env(self) -> None: self._activate_that() mock_process_env(python_path=self.python_path) - # patch up pkg self.notify_change() self._using_venv = True diff --git a/tests/conftest.py b/tests/conftest.py index d593172b17..a339601c16 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,7 +15,7 @@ Contact: code@inmanta.com """ - +import importlib.metadata import logging.config import warnings @@ -94,14 +94,12 @@ from typing import Callable, Dict, Optional, Union import asyncpg -import pkg_resources import psutil import py import pyformance import pytest from asyncpg.exceptions import DuplicateDatabaseError from click import testing -from pkg_resources import Requirement from pyformance.registry import MetricsRegistry from tornado import netutil @@ -510,7 +508,6 @@ def deactive_venv(): old_pythonpath = os.environ.get("PYTHONPATH", None) old_os_venv: Optional[str] = os.environ.get("VIRTUAL_ENV", None) old_process_env: str = env.process_env.python_path - old_working_set = pkg_resources.working_set old_available_extensions = ( dict(InmantaBootloader.AVAILABLE_EXTENSIONS) if InmantaBootloader.AVAILABLE_EXTENSIONS is not None else None ) @@ -527,7 +524,6 @@ def deactive_venv(): sys.path_hooks.extend(old_path_hooks) # Clear cache for sys.path_hooks sys.path_importer_cache.clear() - pkg_resources.working_set = old_working_set # Restore PYTHONPATH if old_pythonpath is not None: os.environ["PYTHONPATH"] = old_pythonpath @@ -1084,7 +1080,7 @@ def __init__(self, env_path: str) -> None: def deactivate(self): if self._using_venv: self._using_venv = False - self.working_set = pkg_resources.working_set + self.working_set = importlib.metadata.distributions() def use_virtual_env(self) -> None: """ @@ -1101,7 +1097,7 @@ def use_virtual_env(self) -> None: # Later run self._activate_that() mock_process_env(python_path=self.python_path) - pkg_resources.working_set = self.working_set + self.working_set = importlib.metadata.distributions() self._using_venv = True From a32e38340e7074b24e88fc333f71990c6edb9d7e Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Fri, 9 Aug 2024 14:58:58 +0200 Subject: [PATCH 07/74] refactor --- src/inmanta/agent/agent.py | 5 +---- src/inmanta/agent/in_process_executor.py | 5 ++--- tests/conftest.py | 5 +++-- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/inmanta/agent/agent.py b/src/inmanta/agent/agent.py index 9a5a2fdf6a..8e66dfc934 100644 --- a/src/inmanta/agent/agent.py +++ b/src/inmanta/agent/agent.py @@ -33,8 +33,6 @@ from typing import Any, Collection, Dict, Optional, Union, cast from inmanta import config, const, data, protocol -import inmanta.agent.executor -from inmanta import config, const, data, env, protocol from inmanta.agent import config as cfg from inmanta.agent import executor, forking_executor, in_process_executor from inmanta.agent.executor import ResourceDetails, ResourceInstallSpec @@ -47,8 +45,7 @@ ResourceType, ResourceVersionIdStr, ) -from inmanta.env import SafeRequirement -from inmanta.loader import CodeLoader, ModuleSource +from inmanta.loader import ModuleSource from inmanta.protocol import SessionEndpoint, SyncClient, methods, methods_v2 from inmanta.resources import Id from inmanta.types import Apireturn, JsonType diff --git a/src/inmanta/agent/in_process_executor.py b/src/inmanta/agent/in_process_executor.py index 04d4817f06..bb9c9389f4 100644 --- a/src/inmanta/agent/in_process_executor.py +++ b/src/inmanta/agent/in_process_executor.py @@ -23,8 +23,6 @@ from concurrent.futures.thread import ThreadPoolExecutor from typing import Any, Optional -import pkg_resources - import inmanta.agent.cache import inmanta.protocol import inmanta.util @@ -36,6 +34,7 @@ from inmanta.agent.io.remote import ChannelClosedException from inmanta.const import ParameterSource from inmanta.data.model import AttributeStateChange, ResourceIdStr, ResourceVersionIdStr +from inmanta.env import SafeRequirement from inmanta.loader import CodeLoader from inmanta.resources import Id, Resource from inmanta.types import Apireturn @@ -586,7 +585,7 @@ async def _install(self, blueprint: executor.ExecutorBlueprint) -> None: await loop.run_in_executor( self.process.thread_pool, self._env.install_for_config, - list(pkg_resources.parse_requirements(blueprint.requirements)), + [SafeRequirement(requirement_string=e) for e in blueprint.requirements], blueprint.pip_config, ) await loop.run_in_executor(self.process.thread_pool, self._loader.deploy_version, blueprint.sources) diff --git a/tests/conftest.py b/tests/conftest.py index 327feb766b..3013150f0e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,6 +15,7 @@ Contact: code@inmanta.com """ + import importlib.metadata import logging.config import warnings @@ -1915,8 +1916,8 @@ def index_with_pkgs_containing_optional_deps() -> str: path=os.path.join(tmpdirname, "pkg"), publish_index=pip_index, optional_dependencies={ - "optional-a": [Requirement.parse("dep-a")], - "optional-b": [Requirement.parse("dep-b"), Requirement.parse("dep-c")], + "optional-a": [SafeRequirement(requirement_string="dep-a")], + "optional-b": [SafeRequirement(requirement_string="dep-b"), SafeRequirement(requirement_string="dep-c")], }, ) for pkg_name in ["dep-a", "dep-b", "dep-c"]: From 33779967d9d207476ba02b64f3525fa7c450c826 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Tue, 20 Aug 2024 15:37:48 +0200 Subject: [PATCH 08/74] rollback changes --- src/inmanta/env.py | 8 +++++++- tests/conftest.py | 7 +++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 4e20d7f632..76d94f26f8 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -39,6 +39,8 @@ from textwrap import indent from typing import NamedTuple, Optional, TypeVar, cast +import pkg_resources + from inmanta import const from inmanta.ast import CompilerException from inmanta.data.model import LEGACY_PIP_DEFAULT, PipConfig @@ -269,6 +271,10 @@ def get_packages_in_working_set(cls, inmanta_modules_only: bool = False) -> dict if not inmanta_modules_only or dist_info.name.startswith(const.MODULE_PKG_NAME_PREFIX) } + @classmethod + def rebuild_working_set(cls) -> None: + pkg_resources.working_set = pkg_resources.WorkingSet._build_master() + @classmethod def get_dependency_tree(cls, dists: abc.Iterable[str]) -> abc.Set[str]: """ @@ -1208,7 +1214,7 @@ def notify_change(self) -> None: are executed in a subprocess. """ importlib.reload(mod) - + PythonWorkingSet.rebuild_working_set() process_env: ActiveEnv = ActiveEnv(python_path=sys.executable) """ diff --git a/tests/conftest.py b/tests/conftest.py index 3013150f0e..d0b876fcb0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,6 +20,7 @@ import logging.config import warnings +import pkg_resources from tornado.httpclient import AsyncHTTPClient import _pytest.logging @@ -509,6 +510,7 @@ def deactive_venv(): old_pythonpath = os.environ.get("PYTHONPATH", None) old_os_venv: Optional[str] = os.environ.get("VIRTUAL_ENV", None) old_process_env: str = env.process_env.python_path + old_working_set = pkg_resources.working_set old_available_extensions = ( dict(InmantaBootloader.AVAILABLE_EXTENSIONS) if InmantaBootloader.AVAILABLE_EXTENSIONS is not None else None ) @@ -525,6 +527,7 @@ def deactive_venv(): sys.path_hooks.extend(old_path_hooks) # Clear cache for sys.path_hooks sys.path_importer_cache.clear() + pkg_resources.working_set = old_working_set # Restore PYTHONPATH if old_pythonpath is not None: os.environ["PYTHONPATH"] = old_pythonpath @@ -1087,7 +1090,7 @@ def __init__(self, env_path: str) -> None: def deactivate(self): if self._using_venv: self._using_venv = False - self.working_set = importlib.metadata.distributions() + self.working_set = pkg_resources.working_set def use_virtual_env(self) -> None: """ @@ -1104,7 +1107,7 @@ def use_virtual_env(self) -> None: # Later run self._activate_that() mock_process_env(python_path=self.python_path) - self.working_set = importlib.metadata.distributions() + pkg_resources.working_set = self.working_set self._using_venv = True From 880d9da00771bbbf50f4a961a945f4c07cc66f0b Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Tue, 20 Aug 2024 15:53:51 +0200 Subject: [PATCH 09/74] rollback changes --- src/inmanta/env.py | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 76d94f26f8..329a4ec633 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -238,15 +238,12 @@ def _are_installed_recursive( return False if r.extras: for extra in r.extras: - search_dist = [e for e in importlib.metadata.distributions() if e.name == r.name] - assert len(search_dist) <= 1 - distribution: Optional[Distribution] = None if len(search_dist) == 0 else search_dist[0] + distribution: Optional[Distribution] = pkg_resources.working_set.find(r) if distribution is None: return False - pkgs_required_by_extra: set[SafeRequirement] = set( - [SafeRequirement(requirement_string=e) for e in distribution.requires or []] - ) + pkgs_required_by_extra: set[Requirement] = set(distribution.requires(extras=(extra,))) - set( + distribution.requires(extras=())) if not _are_installed_recursive( reqs=list(pkgs_required_by_extra), seen_requirements=list(seen_requirements) + list(reqs), @@ -266,9 +263,9 @@ def get_packages_in_working_set(cls, inmanta_modules_only: bool = False) -> dict :param inmanta_modules_only: Only return inmanta modules from the working set """ return { - canonicalize_name(dist_info.name).lower(): version.Version(dist_info.version) - for dist_info in importlib.metadata.distributions() - if not inmanta_modules_only or dist_info.name.startswith(const.MODULE_PKG_NAME_PREFIX) + dist_info.key: version.Version(dist_info.version) + for dist_info in pkg_resources.working_set + if not inmanta_modules_only or dist_info.key.startswith(const.MODULE_PKG_NAME_PREFIX) } @classmethod @@ -287,7 +284,7 @@ def get_dependency_tree(cls, dists: abc.Iterable[str]) -> abc.Set[str]: """ # create dict for O(1) lookup installed_distributions: abc.Mapping[str, Distribution] = { - dist_info.name: dist_info for dist_info in importlib.metadata.distributions() + dist_info.key: dist_info for dist_info in pkg_resources.working_set } def _get_tree_recursive(dists: abc.Iterable[str], acc: abc.Set[str] = frozenset()) -> abc.Set[str]: @@ -307,7 +304,7 @@ def _get_tree_recursive_single(acc: abc.Set[str], dist: str) -> abc.Set[str]: return _get_tree_recursive( ( SafeRequirement(requirement_string=requirement).name - for requirement in installed_distributions[dist].requires or [] + for requirement in installed_distributions[dist].requires() ), acc=acc | {dist}, ) @@ -1034,10 +1031,9 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: # all requirements of all packages installed in this environment installed_constraints: abc.Set[OwnedRequirement] = frozenset( - OwnedRequirement(SafeRequirement(requirement_string=requirement), canonicalize_name(dist_info.name)) - for dist_info in importlib.metadata.distributions() - for requirement in dist_info.requires or [] - if SafeRequirement(requirement).marker is None + OwnedRequirement(requirement, dist_info.key) + for dist_info in pkg_resources.working_set + for requirement in dist_info.requires() ) inmanta_constraints: abc.Set[OwnedRequirement] = frozenset( @@ -1054,12 +1050,7 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: ( [] if strict_scope is None - else ( - dist_info.name - for dist_info in importlib.metadata.distributions() - if strict_scope.fullmatch(dist_info.name) - ) - ), + else (dist_info.key for dist_info in pkg_resources.working_set if strict_scope.fullmatch(dist_info.key)) ), (requirement.requirement.name for requirement in inmanta_constraints), (requirement.requirement.name for requirement in extra_constraints), ) From c5187f8a741215c6d16b832d544f5a6e8f79e1b3 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 21 Aug 2024 10:39:23 +0200 Subject: [PATCH 10/74] refactor --- src/inmanta/env.py | 26 +++++++++++++++++--------- tests/conftest.py | 1 - 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 329a4ec633..94d964fca5 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -238,12 +238,15 @@ def _are_installed_recursive( return False if r.extras: for extra in r.extras: - distribution: Optional[Distribution] = pkg_resources.working_set.find(r) + distribution: Optional[pkg_resources.Distribution] = pkg_resources.working_set.find( + pkg_resources.Requirement.parse(r.name) + ) if distribution is None: return False - pkgs_required_by_extra: set[Requirement] = set(distribution.requires(extras=(extra,))) - set( - distribution.requires(extras=())) + pkgs_required_by_extra: set[SafeRequirement] = set( + [SafeRequirement(e.key) for e in distribution.requires(extras=(extra,))] + ) - set([SafeRequirement(e.key) for e in distribution.requires(extras=())]) if not _are_installed_recursive( reqs=list(pkgs_required_by_extra), seen_requirements=list(seen_requirements) + list(reqs), @@ -263,7 +266,7 @@ def get_packages_in_working_set(cls, inmanta_modules_only: bool = False) -> dict :param inmanta_modules_only: Only return inmanta modules from the working set """ return { - dist_info.key: version.Version(dist_info.version) + canonicalize_name(dist_info.key): version.Version(dist_info.version) for dist_info in pkg_resources.working_set if not inmanta_modules_only or dist_info.key.startswith(const.MODULE_PKG_NAME_PREFIX) } @@ -283,7 +286,7 @@ def get_dependency_tree(cls, dists: abc.Iterable[str]) -> abc.Set[str]: :param dists: The keys for the distributions to get the dependency tree for. """ # create dict for O(1) lookup - installed_distributions: abc.Mapping[str, Distribution] = { + installed_distributions: abc.Mapping[str, pkg_resources.Distribution] = { dist_info.key: dist_info for dist_info in pkg_resources.working_set } @@ -303,7 +306,7 @@ def _get_tree_recursive_single(acc: abc.Set[str], dist: str) -> abc.Set[str]: # recurse on direct dependencies return _get_tree_recursive( ( - SafeRequirement(requirement_string=requirement).name + SafeRequirement(requirement_string=requirement.key).name for requirement in installed_distributions[dist].requires() ), acc=acc | {dist}, @@ -1031,7 +1034,7 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: # all requirements of all packages installed in this environment installed_constraints: abc.Set[OwnedRequirement] = frozenset( - OwnedRequirement(requirement, dist_info.key) + OwnedRequirement(SafeRequirement(requirement_string=requirement.key), dist_info.key) for dist_info in pkg_resources.working_set for requirement in dist_info.requires() ) @@ -1050,7 +1053,8 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: ( [] if strict_scope is None - else (dist_info.key for dist_info in pkg_resources.working_set if strict_scope.fullmatch(dist_info.key)) ), + else (dist_info.key for dist_info in pkg_resources.working_set if strict_scope.fullmatch(dist_info.key)) + ), (requirement.requirement.name for requirement in inmanta_constraints), (requirement.requirement.name for requirement in extra_constraints), ) @@ -1065,7 +1069,10 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: requirement = c.requirement if ( requirement.name.lower() not in installed_versions - or str(installed_versions[requirement.name.lower()]) not in requirement.specifier + or ( + len(requirement.specifier) > 0 + and str(installed_versions[requirement.name.lower()]) not in requirement.specifier + ) ) and (not requirement.marker or (requirement.marker and requirement.marker.evaluate())): version_conflict = VersionConflict( requirement=requirement, @@ -1207,6 +1214,7 @@ def notify_change(self) -> None: importlib.reload(mod) PythonWorkingSet.rebuild_working_set() + process_env: ActiveEnv = ActiveEnv(python_path=sys.executable) """ Singleton representing the Python environment this process is running in. diff --git a/tests/conftest.py b/tests/conftest.py index d0b876fcb0..3cdbd5eab0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,7 +16,6 @@ Contact: code@inmanta.com """ -import importlib.metadata import logging.config import warnings From a239a06daaccda9fb818821a90061f8af239e8e6 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 21 Aug 2024 15:38:53 +0200 Subject: [PATCH 11/74] refactor --- src/inmanta/env.py | 45 ++++++++++++-------------------------- src/inmanta/file_parser.py | 4 ++-- src/inmanta/main.py | 2 +- src/inmanta/module.py | 12 +++------- tests/test_env.py | 1 - 5 files changed, 20 insertions(+), 44 deletions(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 94d964fca5..85ccb0f310 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -37,7 +37,7 @@ from re import Pattern from subprocess import CalledProcessError from textwrap import indent -from typing import NamedTuple, Optional, TypeVar, cast +from typing import NamedTuple, Optional, TypeVar import pkg_resources @@ -47,9 +47,8 @@ from inmanta.server.bootloader import InmantaBootloader from inmanta.stable_api import stable_api from inmanta.util import strtobool -from packaging import version +from packaging import utils, version from packaging.requirements import Requirement -from packaging.utils import InvalidName, NormalizedName LOGGER = logging.getLogger(__name__) LOGGER_PIP = logging.getLogger("inmanta.pip") # Use this logger to log pip commands or data related to pip commands. @@ -63,26 +62,10 @@ class PipInstallError(Exception): pass -# Core metadata spec for `Name` -_validate_regex = re.compile(r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$", re.IGNORECASE) -_canonicalize_regex = re.compile(r"[-_.]+") -_normalized_regex = re.compile(r"^([a-z0-9]|[a-z0-9]([a-z0-9-](?!--))*[a-z0-9])$") -# PEP 427: The build number must start with a digit. -_build_tag_regex = re.compile(r"(\d+)(.*)") - - -def canonicalize_name(name: str, *, validate: bool = False) -> NormalizedName: - if validate and not _validate_regex.match(name): - raise InvalidName(f"name is invalid: {name!r}") - # This is taken from PEP 503. - value = _canonicalize_regex.sub("-", name) - return cast(NormalizedName, value) - - class SafeRequirement(Requirement): def __init__(self, requirement_string: str) -> None: super().__init__(requirement_string=requirement_string) - self.name = canonicalize_name(self.name) + self.name = utils.canonicalize_name(self.name) @dataclass(eq=True, frozen=True) @@ -224,7 +207,7 @@ def _are_installed_recursive( if r in seen_requirements: continue # Requirements created by the `Distribution.requires()` method have the extra, the SafeRequirement was created - # from, TODO h update this comment + # from, # set as a marker. The line below makes sure that the "extra" marker matches. The marker is not set by # `Distribution.requires()` when the package is installed in editable mode, but setting it always doesn't make # the marker evaluation fail. @@ -232,8 +215,10 @@ def _are_installed_recursive( if r.marker and not r.marker.evaluate(environment=environment_marker_evaluation): # The marker of the requirement doesn't apply on this environment continue - if r.name.lower() not in installed_packages or ( - len(r.specifier) > 0 and str(installed_packages[r.name.lower()]) not in r.specifier + if ( + r.name.lower() not in installed_packages + or (len(r.specifier) > 0 and str(installed_packages[r.name.lower()]) not in r.specifier) + # If no specifiers are provided, the `in` operation will return `False` ): return False if r.extras: @@ -261,12 +246,12 @@ def _are_installed_recursive( @classmethod def get_packages_in_working_set(cls, inmanta_modules_only: bool = False) -> dict[str, version.Version]: """ - Return all packages present in `importlib.metadata.distributions()` together with the version of the package. + Return all packages present in `pkg_resources.working_set` together with the version of the package. :param inmanta_modules_only: Only return inmanta modules from the working set """ return { - canonicalize_name(dist_info.key): version.Version(dist_info.version) + utils.canonicalize_name(dist_info.key): version.Version(dist_info.version) for dist_info in pkg_resources.working_set if not inmanta_modules_only or dist_info.key.startswith(const.MODULE_PKG_NAME_PREFIX) } @@ -1068,15 +1053,12 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: for c in all_constraints: requirement = c.requirement if ( - requirement.name.lower() not in installed_versions - or ( - len(requirement.specifier) > 0 - and str(installed_versions[requirement.name.lower()]) not in requirement.specifier - ) + requirement.name not in installed_versions + or (len(requirement.specifier) > 0 and str(installed_versions[requirement.name]) not in requirement.specifier) ) and (not requirement.marker or (requirement.marker and requirement.marker.evaluate())): version_conflict = VersionConflict( requirement=requirement, - installed_version=installed_versions.get(requirement.name.lower(), None), # TODO h do something + installed_version=installed_versions.get(requirement.name, None), owner=c.owner, ) if c.is_owned_by(full_strict_scope): @@ -1268,6 +1250,7 @@ def use_virtual_env(self) -> None: self._activate_that() mock_process_env(python_path=self.python_path) + # patch up pkg self.notify_change() self._using_venv = True diff --git a/src/inmanta/file_parser.py b/src/inmanta/file_parser.py index 9a50a2ec20..3ec4425296 100644 --- a/src/inmanta/file_parser.py +++ b/src/inmanta/file_parser.py @@ -92,14 +92,14 @@ def get_content_with_dep_removed(cls, filename: str, remove_dep_on_pkg: str) -> if line_continuation_buffer: line_continuation_buffer += line if not line.endswith("\\"): - if SafeRequirement(requirement_string=line_continuation_buffer.split("#")[0]).name != remove_dep_on_pkg: + if SafeRequirement(requirement_string=line_continuation_buffer).name != remove_dep_on_pkg: result += line_continuation_buffer line_continuation_buffer = "" elif not line.strip() or line.strip().startswith("#"): result += line elif line.endswith("\\"): line_continuation_buffer = line - elif SafeRequirement(requirement_string=line.split("#")[0]).name != remove_dep_on_pkg: # .lower() + elif SafeRequirement(requirement_string=line).name != remove_dep_on_pkg: result += line else: # Dependency matches `remove_dep_on_pkg` => Remove line from result diff --git a/src/inmanta/main.py b/src/inmanta/main.py index 47335d4139..f1a4395290 100644 --- a/src/inmanta/main.py +++ b/src/inmanta/main.py @@ -182,7 +182,7 @@ def get_table(header: list[str], rows: list[list[str]], data_type: Optional[list return table.draw() -@with_plugins(importlib_metadata.entry_points(group="inmanta.cli_plugins")) +@with_plugins(importlib_metadata.entry_points().select(group="inmanta.cli_plugins")) @click.group(help="Base command") @click.option("--host", help="The server hostname to connect to") @click.option("--port", help="The server port to connect to") diff --git a/src/inmanta/module.py b/src/inmanta/module.py index b7ac35b3c9..4af5c8e103 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -1177,7 +1177,7 @@ def has_requirement_for(self, pkg_name: str) -> bool: Returns True iff this requirements.txt file contains the given package name. The given `pkg_name` is matched case insensitive against the requirements in this RequirementsTxtFile. """ - return any(r.name == pkg_name for r in self._requirements) # .lower() + return any(r.name == pkg_name for r in self._requirements) def set_requirement_and_write(self, requirement: SafeRequirement) -> None: """ @@ -2679,10 +2679,7 @@ def requires(self) -> "List[InmantaModuleRequirement]": # filter on import stmt reqs = [] for spec in self._metadata.requires: - if isinstance(spec, list): - req = [SafeRequirement(requirement_string=e) for e in spec] - else: - req = [SafeRequirement(requirement_string=spec)] + req = [SafeRequirement(requirement_string=spec)] if len(req) > 1: print(f"Module file for {self._path} has bad line in requirements specification {spec}") reqe = InmantaModuleRequirement(req[0]) @@ -2819,10 +2816,7 @@ def requires(self) -> "List[InmantaModuleRequirement]": """ reqs = [] for spec in self.get_module_requirements(): - if isinstance(spec, list): - req = [SafeRequirement(requirement_string=e) for e in spec] - else: - req = [SafeRequirement(requirement_string=spec)] + req = [SafeRequirement(requirement_string=spec)] if len(req) > 1: print(f"Module file for {self._path} has bad line in requirements specification {spec}") reqe = InmantaModuleRequirement(req[0]) diff --git a/tests/test_env.py b/tests/test_env.py index b389e3d818..59a454c892 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -151,7 +151,6 @@ def _list_dir(path: str, ignore: list[str]) -> list[str]: subprocess.check_output([os.path.join(venv.env_path, "bin/pip"), "list"]) -# TODO h FROM Requirement.parse( TO Requirement(requirement_string= def test_gen_req_file(): """ These are all examples used in older testcases that did not work correctly before From 06ea0cf252428029562e7b4ef8a264cc99321b0d Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 21 Aug 2024 15:43:26 +0200 Subject: [PATCH 12/74] refactor --- src/inmanta/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inmanta/main.py b/src/inmanta/main.py index f1a4395290..47335d4139 100644 --- a/src/inmanta/main.py +++ b/src/inmanta/main.py @@ -182,7 +182,7 @@ def get_table(header: list[str], rows: list[list[str]], data_type: Optional[list return table.draw() -@with_plugins(importlib_metadata.entry_points().select(group="inmanta.cli_plugins")) +@with_plugins(importlib_metadata.entry_points(group="inmanta.cli_plugins")) @click.group(help="Base command") @click.option("--host", help="The server hostname to connect to") @click.option("--port", help="The server port to connect to") From aeb68cbfe079b160d98b12cade38146002a7c575 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 21 Aug 2024 16:23:13 +0200 Subject: [PATCH 13/74] fix broken test --- src/inmanta/env.py | 4 +++- src/inmanta/file_parser.py | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 85ccb0f310..7e4333ec12 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -64,7 +64,9 @@ class PipInstallError(Exception): class SafeRequirement(Requirement): def __init__(self, requirement_string: str) -> None: - super().__init__(requirement_string=requirement_string) + # Packaging Requirement is not able to parse requirements with comment. Therefore, we need to remove the `comment` part + drop_comment = requirement_string.split("#")[0] + super().__init__(requirement_string=drop_comment) self.name = utils.canonicalize_name(self.name) diff --git a/src/inmanta/file_parser.py b/src/inmanta/file_parser.py index 3ec4425296..90bde89be6 100644 --- a/src/inmanta/file_parser.py +++ b/src/inmanta/file_parser.py @@ -85,6 +85,7 @@ def get_content_with_dep_removed(cls, filename: str, remove_dep_on_pkg: str) -> if not os.path.exists(filename): raise Exception(f"File {filename} doesn't exist") + removed_dependency = SafeRequirement(requirement_string=remove_dep_on_pkg) result = "" line_continuation_buffer = "" with open(filename, encoding="utf-8") as fd: @@ -92,14 +93,14 @@ def get_content_with_dep_removed(cls, filename: str, remove_dep_on_pkg: str) -> if line_continuation_buffer: line_continuation_buffer += line if not line.endswith("\\"): - if SafeRequirement(requirement_string=line_continuation_buffer).name != remove_dep_on_pkg: + if SafeRequirement(requirement_string=line_continuation_buffer).name != removed_dependency.name: result += line_continuation_buffer line_continuation_buffer = "" elif not line.strip() or line.strip().startswith("#"): result += line elif line.endswith("\\"): line_continuation_buffer = line - elif SafeRequirement(requirement_string=line).name != remove_dep_on_pkg: + elif SafeRequirement(requirement_string=line).name != removed_dependency.name: result += line else: # Dependency matches `remove_dep_on_pkg` => Remove line from result From 87e381a3fa3b57e746a7a311705c72a220a1cc28 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Thu, 29 Aug 2024 10:48:41 +0200 Subject: [PATCH 14/74] refactor --- src/inmanta/env.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 7e4333ec12..7712024543 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -67,6 +67,8 @@ def __init__(self, requirement_string: str) -> None: # Packaging Requirement is not able to parse requirements with comment. Therefore, we need to remove the `comment` part drop_comment = requirement_string.split("#")[0] super().__init__(requirement_string=drop_comment) + # We canonicalize the name of the requirement to be able to compare requirements and check if the requirement is + # already installed self.name = utils.canonicalize_name(self.name) @@ -174,7 +176,7 @@ class PythonWorkingSet: @classmethod def _get_as_requirements_type(cls, requirements: req_list) -> Sequence[SafeRequirement]: """ - Convert requirements from Union[Sequence[str], Sequence[SafeRequirement]] to SequeSafeRequirementment] + Convert requirements from Union[Sequence[str], Sequence[SafeRequirement]] to Sequence[SafeRequirement] """ if isinstance(requirements[0], str): return [SafeRequirement(requirement_string=r) for r in requirements if isinstance(r, str)] @@ -218,8 +220,8 @@ def _are_installed_recursive( # The marker of the requirement doesn't apply on this environment continue if ( - r.name.lower() not in installed_packages - or (len(r.specifier) > 0 and str(installed_packages[r.name.lower()]) not in r.specifier) + r.name not in installed_packages + or (str(installed_packages[r.name]) not in r.specifier) # TODO # If no specifiers are provided, the `in` operation will return `False` ): return False From ae4915b413f2d78a6b10136d0f62bcf87479dff1 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Thu, 29 Aug 2024 10:49:06 +0200 Subject: [PATCH 15/74] refactor --- src/inmanta/env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 7712024543..eaee32c472 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -221,7 +221,7 @@ def _are_installed_recursive( continue if ( r.name not in installed_packages - or (str(installed_packages[r.name]) not in r.specifier) # TODO + or (str(installed_packages[r.name]) not in r.specifier) # If no specifiers are provided, the `in` operation will return `False` ): return False From 0768b2bd5180109700e7c6f0bbdbef02d6a9c4f7 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Thu, 29 Aug 2024 15:41:58 +0200 Subject: [PATCH 16/74] refactor --- src/inmanta/env.py | 2 +- src/inmanta/main.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index eaee32c472..ec02333c5a 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -255,7 +255,7 @@ def get_packages_in_working_set(cls, inmanta_modules_only: bool = False) -> dict :param inmanta_modules_only: Only return inmanta modules from the working set """ return { - utils.canonicalize_name(dist_info.key): version.Version(dist_info.version) + SafeRequirement(dist_info.key).name: version.Version(dist_info.version) for dist_info in pkg_resources.working_set if not inmanta_modules_only or dist_info.key.startswith(const.MODULE_PKG_NAME_PREFIX) } diff --git a/src/inmanta/main.py b/src/inmanta/main.py index 47335d4139..c52c6d6ed1 100644 --- a/src/inmanta/main.py +++ b/src/inmanta/main.py @@ -182,7 +182,7 @@ def get_table(header: list[str], rows: list[list[str]], data_type: Optional[list return table.draw() -@with_plugins(importlib_metadata.entry_points(group="inmanta.cli_plugins")) +@with_plugins(iter(importlib_metadata.entry_points(group="inmanta.cli_plugins"))) @click.group(help="Base command") @click.option("--host", help="The server hostname to connect to") @click.option("--port", help="The server port to connect to") From 8d71c665932b8ff8535887f223eaa58815d27158 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Thu, 29 Aug 2024 15:52:13 +0200 Subject: [PATCH 17/74] add test safe requirement --- src/inmanta/env.py | 3 ++- tests/compiler/test_basics.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index ec02333c5a..32a45ea61b 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -65,7 +65,8 @@ class PipInstallError(Exception): class SafeRequirement(Requirement): def __init__(self, requirement_string: str) -> None: # Packaging Requirement is not able to parse requirements with comment. Therefore, we need to remove the `comment` part - drop_comment = requirement_string.split("#")[0] + drop_comment = requirement_string.split("#")[0].strip() + assert len(drop_comment) > 0, "The name of the requirement cannot be an empty string!" super().__init__(requirement_string=drop_comment) # We canonicalize the name of the requirement to be able to compare requirements and check if the requirement is # already installed diff --git a/tests/compiler/test_basics.py b/tests/compiler/test_basics.py index 292be3ded0..2b4b193992 100644 --- a/tests/compiler/test_basics.py +++ b/tests/compiler/test_basics.py @@ -24,6 +24,7 @@ from inmanta import compiler, const from inmanta.ast import DoubleSetException, RuntimeException +from inmanta.env import SafeRequirement from inmanta.plugins import PluginDeprecationWarning from utils import module_from_template, v1_module_from_template @@ -731,3 +732,12 @@ def test_implementation_import_missing_error(snippetcompiler) -> None: assert "could not find type tests::length in namespace __config__" in exception.value.msg assert exception.value.location.lnr == 6 assert exception.value.location.start_char == 20 + + +@pytest.mark.parametrize("name", ["", "#", " # ", "#this is a comment"]) +def test_safe_requirement(name) -> None: + """ + Ensure that empty name requirements are not allowed in `SafeRequirement` + """ + with pytest.raises(AssertionError): + SafeRequirement(requirement_string=name) From f8847327a7385e7e1a923a2c3a5f314cc006d66f Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Thu, 29 Aug 2024 16:47:00 +0200 Subject: [PATCH 18/74] Remove SafeRequirement --- src/inmanta/agent/executor.py | 4 +- src/inmanta/agent/in_process_executor.py | 4 +- src/inmanta/env.py | 78 +++++++++++------------- src/inmanta/file_parser.py | 15 ++--- src/inmanta/module.py | 25 ++++---- src/inmanta/moduletool.py | 4 +- tests/compiler/test_basics.py | 4 +- tests/conftest.py | 6 +- tests/moduletool/test_add.py | 4 +- tests/moduletool/test_convert_v1_v2.py | 6 +- tests/moduletool/test_install.py | 47 +++++++------- tests/moduletool/test_update.py | 12 ++-- tests/server/test_compilerservice.py | 4 +- tests/test_env.py | 34 +++++------ tests/test_file_parser.py | 4 +- tests/test_module_loader.py | 56 ++++++++--------- 16 files changed, 147 insertions(+), 160 deletions(-) diff --git a/src/inmanta/agent/executor.py b/src/inmanta/agent/executor.py index 9a1dc8ace2..3ec61870bf 100644 --- a/src/inmanta/agent/executor.py +++ b/src/inmanta/agent/executor.py @@ -41,7 +41,7 @@ from inmanta.agent import config as cfg from inmanta.agent import resourcepool from inmanta.data.model import PipConfig, ResourceIdStr, ResourceType, ResourceVersionIdStr -from inmanta.env import PythonEnvironment, SafeRequirement +from inmanta.env import PythonEnvironment, safe_parse from inmanta.loader import ModuleSource from inmanta.resources import Id from inmanta.types import JsonType @@ -294,7 +294,7 @@ def create_and_install_environment(self, blueprint: EnvBlueprint) -> None: self.init_env() if len(req): # install_for_config expects at least 1 requirement or a path to install self.install_for_config( - requirements=[SafeRequirement(requirement_string=e) for e in req], + requirements=[safe_parse(requirement=e) for e in req], config=blueprint.pip_config, ) diff --git a/src/inmanta/agent/in_process_executor.py b/src/inmanta/agent/in_process_executor.py index 29a6813c00..db6684457e 100644 --- a/src/inmanta/agent/in_process_executor.py +++ b/src/inmanta/agent/in_process_executor.py @@ -33,7 +33,7 @@ from inmanta.agent.handler import HandlerAPI, SkipResource from inmanta.const import ParameterSource from inmanta.data.model import AttributeStateChange, ResourceIdStr, ResourceVersionIdStr -from inmanta.env import SafeRequirement +from inmanta.env import safe_parse from inmanta.loader import CodeLoader from inmanta.resources import Id, Resource from inmanta.types import Apireturn @@ -571,7 +571,7 @@ async def _install(self, blueprint: executor.ExecutorBlueprint) -> None: await loop.run_in_executor( self.process.thread_pool, self._env.install_for_config, - [SafeRequirement(requirement_string=e) for e in blueprint.requirements], + [safe_parse(requirement=e) for e in blueprint.requirements], blueprint.pip_config, ) await loop.run_in_executor(self.process.thread_pool, self._loader.deploy_version, blueprint.sources) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 32a45ea61b..161a46cb74 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -62,15 +62,14 @@ class PipInstallError(Exception): pass -class SafeRequirement(Requirement): - def __init__(self, requirement_string: str) -> None: - # Packaging Requirement is not able to parse requirements with comment. Therefore, we need to remove the `comment` part - drop_comment = requirement_string.split("#")[0].strip() - assert len(drop_comment) > 0, "The name of the requirement cannot be an empty string!" - super().__init__(requirement_string=drop_comment) - # We canonicalize the name of the requirement to be able to compare requirements and check if the requirement is - # already installed - self.name = utils.canonicalize_name(self.name) +def safe_parse(requirement: str) -> Requirement: + # Packaging Requirement is not able to parse requirements with comment. Therefore, we need to remove the `comment` part + drop_comment = requirement.split("#")[0].strip() + assert len(drop_comment) > 0, "The name of the requirement cannot be an empty string!" + # We canonicalize the name of the requirement to be able to compare requirements and check if the requirement is + # already installed + canonicalized_requirement_name = utils.canonicalize_name(drop_comment) + return Requirement(requirement_string=canonicalized_requirement_name) @dataclass(eq=True, frozen=True) @@ -83,7 +82,7 @@ class VersionConflict: :param owner: The package from which the constraint originates """ - requirement: SafeRequirement + requirement: Requirement installed_version: Optional[version.Version] = None owner: Optional[str] = None @@ -168,19 +167,19 @@ def get_advice(self) -> Optional[str]: ) -req_list = TypeVar("req_list", Sequence[str], Sequence[SafeRequirement]) +req_list = TypeVar("req_list", Sequence[str], Sequence[Requirement]) import importlib.metadata class PythonWorkingSet: @classmethod - def _get_as_requirements_type(cls, requirements: req_list) -> Sequence[SafeRequirement]: + def _get_as_requirements_type(cls, requirements: req_list) -> Sequence[Requirement]: """ Convert requirements from Union[Sequence[str], Sequence[SafeRequirement]] to Sequence[SafeRequirement] """ if isinstance(requirements[0], str): - return [SafeRequirement(requirement_string=r) for r in requirements if isinstance(r, str)] + return [safe_parse(requirement=r) for r in requirements if isinstance(r, str)] else: return requirements @@ -194,8 +193,8 @@ def are_installed(cls, requirements: req_list) -> bool: installed_packages: dict[str, version.Version] = cls.get_packages_in_working_set() def _are_installed_recursive( - reqs: Sequence[SafeRequirement], - seen_requirements: Sequence[SafeRequirement], + reqs: Sequence[Requirement], + seen_requirements: Sequence[Requirement], contained_in_extra: Optional[str] = None, ) -> bool: """ @@ -234,9 +233,9 @@ def _are_installed_recursive( if distribution is None: return False - pkgs_required_by_extra: set[SafeRequirement] = set( - [SafeRequirement(e.key) for e in distribution.requires(extras=(extra,))] - ) - set([SafeRequirement(e.key) for e in distribution.requires(extras=())]) + pkgs_required_by_extra: set[Requirement] = set( + [safe_parse(e.key) for e in distribution.requires(extras=(extra,))] + ) - set([safe_parse(e.key) for e in distribution.requires(extras=())]) if not _are_installed_recursive( reqs=list(pkgs_required_by_extra), seen_requirements=list(seen_requirements) + list(reqs), @@ -245,7 +244,7 @@ def _are_installed_recursive( return False return True - reqs_as_requirements: Sequence[SafeRequirement] = cls._get_as_requirements_type(requirements) + reqs_as_requirements: Sequence[Requirement] = cls._get_as_requirements_type(requirements) return _are_installed_recursive(reqs_as_requirements, seen_requirements=[]) @classmethod @@ -256,7 +255,7 @@ def get_packages_in_working_set(cls, inmanta_modules_only: bool = False) -> dict :param inmanta_modules_only: Only return inmanta modules from the working set """ return { - SafeRequirement(dist_info.key).name: version.Version(dist_info.version) + safe_parse(dist_info.key).name: version.Version(dist_info.version) for dist_info in pkg_resources.working_set if not inmanta_modules_only or dist_info.key.startswith(const.MODULE_PKG_NAME_PREFIX) } @@ -295,10 +294,7 @@ def _get_tree_recursive_single(acc: abc.Set[str], dist: str) -> abc.Set[str]: # recurse on direct dependencies return _get_tree_recursive( - ( - SafeRequirement(requirement_string=requirement.key).name - for requirement in installed_distributions[dist].requires() - ), + (safe_parse(requirement=requirement.key).name for requirement in installed_distributions[dist].requires()), acc=acc | {dist}, ) @@ -391,7 +387,7 @@ def run_pip_install_command_from_config( cls, python_path: str, config: PipConfig, - requirements: Optional[Sequence[SafeRequirement]] = None, + requirements: Optional[Sequence[Requirement]] = None, requirements_files: Optional[list[str]] = None, upgrade: bool = False, upgrade_strategy: PipUpgradeStrategy = PipUpgradeStrategy.ONLY_IF_NEEDED, @@ -757,7 +753,7 @@ def get_installed_packages(self, only_editable: bool = False) -> dict[str, versi def install_for_config( self, - requirements: list[SafeRequirement], + requirements: list[Requirement], config: PipConfig, upgrade: bool = False, constraint_files: Optional[list[str]] = None, @@ -794,7 +790,7 @@ def install_for_config( def install_from_index( self, - requirements: list[SafeRequirement], + requirements: list[Requirement], index_urls: Optional[list[str]] = None, upgrade: bool = False, allow_pre_releases: bool = False, @@ -865,7 +861,7 @@ def install_from_list( use_pip_config was ignored on ISO6 and it still is """ self.install_from_index( - requirements=[SafeRequirement(requirement_string=r) for r in requirements_list], + requirements=[safe_parse(requirement=r) for r in requirements_list], upgrade=upgrade, upgrade_strategy=upgrade_strategy, use_pip_config=True, @@ -885,18 +881,14 @@ def get_protected_inmanta_packages(cls) -> list[str]: ] @classmethod - def _get_requirements_on_inmanta_package(cls) -> Sequence[SafeRequirement]: + def _get_requirements_on_inmanta_package(cls) -> Sequence[Requirement]: """ Returns the content of the requirement file that should be supplied to each `pip install` invocation to make sure that no Inmanta packages gets overridden. """ protected_inmanta_packages: list[str] = cls.get_protected_inmanta_packages() workingset: dict[str, version.Version] = PythonWorkingSet.get_packages_in_working_set() - return [ - SafeRequirement(requirement_string=f"{pkg}=={workingset[pkg]}") - for pkg in workingset - if pkg in protected_inmanta_packages - ] + return [safe_parse(requirement=f"{pkg}=={workingset[pkg]}") for pkg in workingset if pkg in protected_inmanta_packages] class CommandRunner: @@ -990,7 +982,7 @@ def are_installed(self, requirements: req_list) -> bool: def install_for_config( self, - requirements: list[SafeRequirement], + requirements: list[Requirement], config: PipConfig, upgrade: bool = False, constraint_files: Optional[list[str]] = None, @@ -1008,7 +1000,7 @@ def install_for_config( def get_constraint_violations_for_check( cls, strict_scope: Optional[Pattern[str]] = None, - constraints: Optional[list[SafeRequirement]] = None, + constraints: Optional[list[Requirement]] = None, ) -> tuple[set[VersionConflict], set[VersionConflict]]: """ Return the constraint violations that exist in this venv. Returns a tuple of non-strict and strict violations, @@ -1016,7 +1008,7 @@ def get_constraint_violations_for_check( """ class OwnedRequirement(NamedTuple): - requirement: SafeRequirement + requirement: Requirement owner: Optional[str] = None def is_owned_by(self, owners: abc.Set[str]) -> bool: @@ -1024,7 +1016,7 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: # all requirements of all packages installed in this environment installed_constraints: abc.Set[OwnedRequirement] = frozenset( - OwnedRequirement(SafeRequirement(requirement_string=requirement.key), dist_info.key) + OwnedRequirement(safe_parse(requirement=requirement.key), dist_info.key) for dist_info in pkg_resources.working_set for requirement in dist_info.requires() ) @@ -1077,7 +1069,7 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: def check( cls, strict_scope: Optional[Pattern[str]] = None, - constraints: Optional[list[SafeRequirement]] = None, + constraints: Optional[list[Requirement]] = None, ) -> None: """ Check this Python environment for incompatible dependencies in installed packages. @@ -1101,7 +1093,7 @@ def check( LOGGER.warning("%s", violation) @classmethod - def check_legacy(cls, in_scope: Pattern[str], constraints: Optional[list[SafeRequirement]] = None) -> bool: + def check_legacy(cls, in_scope: Pattern[str], constraints: Optional[list[Requirement]] = None) -> bool: """ Check this Python environment for incompatible dependencies in installed packages. This method is a legacy method in the sense that it has been replaced with a more correct check defined in self.check(). This method is invoked @@ -1120,8 +1112,8 @@ def check_legacy(cls, in_scope: Pattern[str], constraints: Optional[list[SafeReq working_set: abc.Iterable[Distribution] = importlib.metadata.distributions() # add all requirements of all in scope packages installed in this environment - all_constraints: set[SafeRequirement] = set(constraints if constraints is not None else []).union( - SafeRequirement(requirement_string=requirement) + all_constraints: set[Requirement] = set(constraints if constraints is not None else []).union( + safe_parse(requirement=requirement) for dist_info in working_set if in_scope.fullmatch(dist_info.name) for requirement in dist_info.requires or [] @@ -1291,7 +1283,7 @@ def _activate_that(self) -> None: def install_for_config( self, - requirements: list[SafeRequirement], + requirements: list[Requirement], config: PipConfig, upgrade: bool = False, constraint_files: Optional[list[str]] = None, diff --git a/src/inmanta/file_parser.py b/src/inmanta/file_parser.py index 90bde89be6..9682e2151c 100644 --- a/src/inmanta/file_parser.py +++ b/src/inmanta/file_parser.py @@ -18,7 +18,8 @@ import os -from inmanta.env import SafeRequirement +from inmanta.env import safe_parse +from packaging.requirements import Requirement from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap @@ -55,11 +56,11 @@ class RequirementsTxtParser: """ @classmethod - def parse(cls, filename: str) -> list[SafeRequirement]: + def parse(cls, filename: str) -> list[Requirement]: """ - Get all the requirements in `filename` as a list of `SafeRequirement` instances. + Get all the requirements in `filename` as a list of `Requirement` instances. """ - return [SafeRequirement(requirement_string=r) for r in cls.parse_requirements_as_strs(filename)] + return [safe_parse(requirement_string=r) for r in cls.parse_requirements_as_strs(filename)] @classmethod def parse_requirements_as_strs(cls, filename: str) -> list[str]: @@ -85,7 +86,7 @@ def get_content_with_dep_removed(cls, filename: str, remove_dep_on_pkg: str) -> if not os.path.exists(filename): raise Exception(f"File {filename} doesn't exist") - removed_dependency = SafeRequirement(requirement_string=remove_dep_on_pkg) + removed_dependency = safe_parse(requirement=remove_dep_on_pkg) result = "" line_continuation_buffer = "" with open(filename, encoding="utf-8") as fd: @@ -93,14 +94,14 @@ def get_content_with_dep_removed(cls, filename: str, remove_dep_on_pkg: str) -> if line_continuation_buffer: line_continuation_buffer += line if not line.endswith("\\"): - if SafeRequirement(requirement_string=line_continuation_buffer).name != removed_dependency.name: + if safe_parse(requirement=line_continuation_buffer).name != removed_dependency.name: result += line_continuation_buffer line_continuation_buffer = "" elif not line.strip() or line.strip().startswith("#"): result += line elif line.endswith("\\"): line_continuation_buffer = line - elif SafeRequirement(requirement_string=line).name != removed_dependency.name: + elif safe_parse(requirement=line).name != removed_dependency.name: result += line else: # Dependency matches `remove_dep_on_pkg` => Remove line from result diff --git a/src/inmanta/module.py b/src/inmanta/module.py index 4af5c8e103..b4864b8c3d 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -59,7 +59,7 @@ from inmanta.ast.blocks import BasicBlock from inmanta.ast.statements import BiStatement, DefinitionStatement, DynamicStatement, Statement from inmanta.ast.statements.define import DefineImport -from inmanta.env import SafeRequirement, assert_pip_has_source +from inmanta.env import SafeRequirement, assert_pip_has_source, safe_parse from inmanta.file_parser import PreservativeYamlParser, RequirementsTxtParser from inmanta.parser import plyInmantaParser from inmanta.parser.plyInmantaParser import cache_manager @@ -67,6 +67,7 @@ from inmanta.util import get_compiler_version from inmanta.warnings import InmantaWarning from packaging import version +from packaging.requirements import Requirement from packaging.specifiers import SpecifierSet from packaging.version import Version from ruamel.yaml.comments import CommentedMap @@ -97,13 +98,13 @@ class InmantaModuleRequirement: used by distinguishing the two on a type level. """ - def __init__(self, requirement: SafeRequirement) -> None: + def __init__(self, requirement: Requirement) -> None: if requirement.name.startswith(ModuleV2.PKG_NAME_PREFIX): raise ValueError( f"InmantaModuleRequirement instances work with inmanta module names, not python package names. " f"Problematic case: {str(requirement)}" ) - self._requirement: SafeRequirement = requirement + self._requirement: Requirement = requirement @property def project_name(self) -> str: @@ -151,7 +152,7 @@ def parse(cls: type[TInmantaModuleRequirement], spec: str) -> TInmantaModuleRequ ) if "-" in spec: raise ValueError("Invalid Inmanta module requirement: Inmanta module names use '_', not '-'.") - return cls(SafeRequirement(requirement_string=spec)) + return cls(safe_parse(requirement=spec)) def get_python_package_requirement(self) -> SafeRequirement: """ @@ -160,7 +161,7 @@ def get_python_package_requirement(self) -> SafeRequirement: module_name = self.project_name pkg_name = ModuleV2Source.get_package_name_for(module_name) pkg_req_str = str(self).replace(module_name, pkg_name, 1) # Replace max 1 occurrence - return SafeRequirement(requirement_string=pkg_req_str) + return safe_parse(requirement=pkg_req_str) class CompilerExceptionWithExtendedTrace(CompilerException): @@ -729,7 +730,7 @@ def install(self, project: "Project", module_spec: list[InmantaModuleRequirement # These could be constraints (-c) as well, but that requires additional sanitation # Because for pip not every valid -r is a valid -c current_requires = project.get_strict_python_requirements_as_list() - requirements += [SafeRequirement(requirement_string=r) for r in current_requires] + requirements += [safe_parse(requirement=r) for r in current_requires] if preinstalled is not None: # log warning if preinstalled version does not match constraints @@ -2128,7 +2129,7 @@ def install_modules(self, *, bypass_module_cache: bool = False, update_dependenc self.verify_module_version_compatibility() # do python install - pyreq: list[SafeRequirement] = [SafeRequirement(requirement_string=x) for x in self.collect_python_requirements()] + pyreq: list[SafeRequirement] = [safe_parse(requirement=x) for x in self.collect_python_requirements()] if len(pyreq) > 0: # upgrade both direct and transitive module dependencies: eager upgrade strategy @@ -2546,9 +2547,7 @@ def verify_python_requires(self) -> None: Verifies no incompatibilities exist within the Python environment with respect to installed module v2 requirements. """ if self.strict_deps_check: - constraints: list[SafeRequirement] = [ - SafeRequirement(requirement_string=item) for item in self.collect_python_requirements() - ] + constraints: list[SafeRequirement] = [safe_parse(requirement=item) for item in self.collect_python_requirements()] env.ActiveEnv.check(strict_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*"), constraints=constraints) else: if not env.ActiveEnv.check_legacy(in_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*")): @@ -2679,7 +2678,7 @@ def requires(self) -> "List[InmantaModuleRequirement]": # filter on import stmt reqs = [] for spec in self._metadata.requires: - req = [SafeRequirement(requirement_string=spec)] + req = [safe_parse(requirement=spec)] if len(req) > 1: print(f"Module file for {self._path} has bad line in requirements specification {spec}") reqe = InmantaModuleRequirement(req[0]) @@ -2816,7 +2815,7 @@ def requires(self) -> "List[InmantaModuleRequirement]": """ reqs = [] for spec in self.get_module_requirements(): - req = [SafeRequirement(requirement_string=spec)] + req = [safe_parse(requirement=spec)] if len(req) > 1: print(f"Module file for {self._path} has bad line in requirements specification {spec}") reqe = InmantaModuleRequirement(req[0]) @@ -3432,7 +3431,7 @@ def add_module_requirement_persistent(self, requirement: InmantaModuleRequiremen new_install_requires = [ r for r in config_parser.get("options", "install_requires").split("\n") - if r and SafeRequirement(requirement_string=r).name != python_pkg_requirement.name + if r and safe_parse(requirement=r).name != python_pkg_requirement.name ] new_install_requires.append(str(python_pkg_requirement)) else: diff --git a/src/inmanta/moduletool.py b/src/inmanta/moduletool.py index 35dcefb1d6..555f6b0c99 100644 --- a/src/inmanta/moduletool.py +++ b/src/inmanta/moduletool.py @@ -55,7 +55,7 @@ from inmanta.ast import CompilerException from inmanta.command import CLIException, ShowUsageException from inmanta.const import CF_CACHE_DIR, MAX_UPDATE_ATTEMPT -from inmanta.env import SafeRequirement +from inmanta.env import SafeRequirement, safe_parse from inmanta.module import ( DummyProject, FreezeOperator, @@ -485,7 +485,7 @@ def do_update(specs: Mapping[str, Sequence[InmantaModuleRequirement]], modules: # Because for pip not every valid -r is a valid -c current_requires = my_project.get_strict_python_requirements_as_list() env.process_env.install_for_config( - v2_python_specs + [SafeRequirement(requirement_string=r) for r in current_requires], + v2_python_specs + [safe_parse(requirement=r) for r in current_requires], my_project.metadata.pip, upgrade=True, ) diff --git a/tests/compiler/test_basics.py b/tests/compiler/test_basics.py index 2b4b193992..de12f5db25 100644 --- a/tests/compiler/test_basics.py +++ b/tests/compiler/test_basics.py @@ -24,7 +24,7 @@ from inmanta import compiler, const from inmanta.ast import DoubleSetException, RuntimeException -from inmanta.env import SafeRequirement +from inmanta.env import safe_parse from inmanta.plugins import PluginDeprecationWarning from utils import module_from_template, v1_module_from_template @@ -740,4 +740,4 @@ def test_safe_requirement(name) -> None: Ensure that empty name requirements are not allowed in `SafeRequirement` """ with pytest.raises(AssertionError): - SafeRequirement(requirement_string=name) + safe_parse(requirement=name) diff --git a/tests/conftest.py b/tests/conftest.py index 3cdbd5eab0..edb8a53fe5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -119,7 +119,7 @@ from inmanta.ast import CompilerException from inmanta.data.schema import SCHEMA_VERSION_TABLE from inmanta.db import util as db_util -from inmanta.env import CommandRunner, LocalPackagePath, SafeRequirement, VirtualEnv, mock_process_env +from inmanta.env import CommandRunner, LocalPackagePath, SafeRequirement, VirtualEnv, mock_process_env, safe_parse from inmanta.export import ResourceDict, cfg_env, unknown_parameters from inmanta.module import InmantaModuleRequirement, InstallMode, Project, RelationPrecedenceRule from inmanta.moduletool import DefaultIsolatedEnvCached, ModuleTool, V2ModuleBuilder @@ -1918,8 +1918,8 @@ def index_with_pkgs_containing_optional_deps() -> str: path=os.path.join(tmpdirname, "pkg"), publish_index=pip_index, optional_dependencies={ - "optional-a": [SafeRequirement(requirement_string="dep-a")], - "optional-b": [SafeRequirement(requirement_string="dep-b"), SafeRequirement(requirement_string="dep-c")], + "optional-a": [safe_parse(requirement="dep-a")], + "optional-b": [safe_parse(requirement="dep-b"), safe_parse(requirement="dep-c")], }, ) for pkg_name in ["dep-a", "dep-b", "dep-c"]: diff --git a/tests/moduletool/test_add.py b/tests/moduletool/test_add.py index 458bfa3cec..dfb2f3c452 100644 --- a/tests/moduletool/test_add.py +++ b/tests/moduletool/test_add.py @@ -25,7 +25,7 @@ import pytest from inmanta.command import CLIException -from inmanta.env import SafeRequirement, process_env +from inmanta.env import process_env, safe_parse from inmanta.module import ModuleV1, ModuleV1Metadata, ModuleV2, ModuleV2Source, Project, ProjectMetadata from inmanta.moduletool import ModuleTool from packaging.version import Version @@ -88,7 +88,7 @@ def test_module_add_v2_module_to_project( dest_dir=os.path.join(tmpdir, f"elaboratev2module-v{version}"), new_version=Version(version), publish_index=pip_index, - new_extras={"optional": [SafeRequirement(requirement_string="inmanta-module-minimalv2module")]}, + new_extras={"optional": [safe_parse(requirement="inmanta-module-minimalv2module")]}, ) # Create project diff --git a/tests/moduletool/test_convert_v1_v2.py b/tests/moduletool/test_convert_v1_v2.py index b24e5793ef..5e1af3431f 100644 --- a/tests/moduletool/test_convert_v1_v2.py +++ b/tests/moduletool/test_convert_v1_v2.py @@ -30,7 +30,7 @@ import toml from inmanta import moduletool from inmanta.command import CLIException -from inmanta.env import SafeRequirement +from inmanta.env import safe_parse from inmanta.module import DummyProject, ModuleV1, ModuleV2, ModuleV2Metadata from inmanta.moduletool import ModuleConverter, ModuleVersionException from packaging import version @@ -114,9 +114,7 @@ def test_issue_3159_conversion_std_module_add_self_to_dependencies(tmpdir): parser = configparser.ConfigParser() parser.read(setup_cfg_file) assert parser.has_option("options", "install_requires") - install_requires = [ - SafeRequirement(requirement_string=r) for r in parser.get("options", "install_requires").split("\n") if r - ] + install_requires = [safe_parse(requirement=r) for r in parser.get("options", "install_requires").split("\n") if r] pkg_names = [r.name for r in install_requires] assert "inmanta-module-std" not in pkg_names diff --git a/tests/moduletool/test_install.py b/tests/moduletool/test_install.py index 91d92c6e5f..4bcd10edc4 100644 --- a/tests/moduletool/test_install.py +++ b/tests/moduletool/test_install.py @@ -35,7 +35,7 @@ from inmanta.ast import CompilerException from inmanta.command import CLIException from inmanta.config import Config -from inmanta.env import CommandRunner, ConflictingRequirements, PipConfig, SafeRequirement +from inmanta.env import CommandRunner, ConflictingRequirements, PipConfig, safe_parse from inmanta.module import InmantaModuleRequirement, InstallMode, ModuleLoadingException, ModuleNotFoundException from inmanta.moduletool import DummyProject, ModuleConverter, ModuleTool, ProjectTool from moduletool.common import BadModProvider, install_project @@ -251,14 +251,14 @@ def test_module_install_conflicting_requirements(tmpdir: py.path.local, snippetc os.path.join(modules_v2_dir, "minimalv2module"), os.path.join(str(tmpdir), "modone"), new_name="modone", - new_requirements=[SafeRequirement(requirement_string="lorem~=0.0.1")], + new_requirements=[safe_parse(requiremement="lorem~=0.0.1")], install=True, ) module_from_template( os.path.join(modules_v2_dir, "minimalv2module"), os.path.join(str(tmpdir), "modtwo"), new_name="modtwo", - new_requirements=[SafeRequirement(requirement_string="lorem~=0.1.0")], + new_requirements=[safe_parse(requirement_string="lorem~=0.1.0")], install=True, ) @@ -508,7 +508,7 @@ def test_project_install( index_url=local_module_package_index, # We add tornado, as there is a code path in update for the case where the project has python requires python_requires=["tornado"] - + [SafeRequirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names], + + [safe_parse(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names], install_project=False, ) @@ -541,7 +541,7 @@ def test_project_install( autostd=False, python_package_sources=[local_module_package_index], python_requires=[ - SafeRequirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names + safe_parse(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names ] + ["lorem"], install_project=False, @@ -677,7 +677,7 @@ def test_project_install_modules_cache_invalid( index_url=index.url, extra_index_url=[local_module_package_index], # make sure main module gets installed, pulling in newest version of dependency module - python_requires=[SafeRequirement(requirement_string=module.ModuleV2Source.get_package_name_for(main_module))], + python_requires=[safe_parse(requirement_string=module.ModuleV2Source.get_package_name_for(main_module))], ) # populate project.modules[dependency_module] to force the error conditions in this simplified example @@ -762,7 +762,7 @@ def test_project_install_incompatible_versions( install_project=False, add_to_module_path=[v1_modules_path], index_url=index.url, - python_requires=[SafeRequirement(requirement_string=module.ModuleV2Source.get_package_name_for(v2_mod_name))], + python_requires=[safe_parse(requirement_string=module.ModuleV2Source.get_package_name_for(v2_mod_name))], ) # install project @@ -815,14 +815,14 @@ def test_project_install_incompatible_dependencies( v2_template_path, os.path.join(str(tmpdir), "v2mod2"), new_name="v2mod2", - new_requirements=[SafeRequirement(requirement_string="inmanta-module-v2mod1~=1.0.0")], + new_requirements=[safe_parse(requirement_string="inmanta-module-v2mod1~=1.0.0")], publish_index=index, ) v2mod3: module.ModuleV2Metadata = module_from_template( v2_template_path, os.path.join(str(tmpdir), "v2mod3"), new_name="v2mod3", - new_requirements=[SafeRequirement(requirement_string="inmanta-module-v2mod1~=2.0.0")], + new_requirements=[safe_parse(requirement_string="inmanta-module-v2mod1~=2.0.0")], publish_index=index, ) @@ -836,7 +836,7 @@ def test_project_install_incompatible_dependencies( install_project=False, index_url=index.url, python_requires=[ - SafeRequirement( + safe_parse( requirement_string=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod2, v2mod3] @@ -920,7 +920,7 @@ def test_install_from_index_dont_leak_pip_index( # Installing a V2 module requires a python package source. index_url="unknown", python_requires=[ - SafeRequirement( + safe_parse( requirement_string=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod1] @@ -982,7 +982,7 @@ def test_install_with_use_config( index_url=index.url if not use_pip_config else None, use_pip_config_file=use_pip_config, python_requires=[ - SafeRequirement( + safe_parse( requirement_string=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod1] @@ -1051,7 +1051,7 @@ def test_install_with_use_config_extra_index( extra_index_url=[index2.url], use_pip_config_file=True, python_requires=[ - SafeRequirement( + safe_parse( requirement_string=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod1, v2mod2] @@ -1087,7 +1087,7 @@ def test_install_with_use_config_but_PIP_CONFIG_FILE_not_set( autostd=False, install_project=False, use_pip_config_file=True, - python_requires=[SafeRequirement(requirement_string="inmanta-module-dummy-module")], + python_requires=[safe_parse(requirement_string="inmanta-module-dummy-module")], ) # install project @@ -1204,7 +1204,7 @@ def test_install_project_with_install_mode_master(tmpdir: py.path.local, snippet autostd=False, install_project=False, add_to_module_path=[str(tmpdir)], - project_requires=[InmantaModuleRequirement(SafeRequirement(requirement_string="mod11==3.2.1"))], + project_requires=[InmantaModuleRequirement(safe_parse(requirement_string="mod11==3.2.1"))], install_mode=InstallMode.master, ) @@ -1232,7 +1232,7 @@ def test_module_install_logging(local_module_package_index: str, snippetcompiler v2_module = "minimalv2module" - v2_requirements = [SafeRequirement(requirement_string=module.ModuleV2Source.get_package_name_for(v2_module))] + v2_requirements = [safe_parse(requirement_string=module.ModuleV2Source.get_package_name_for(v2_module))] # set up project and modules project: module.Project = snippetcompiler_clean.setup_for_snippet( @@ -1333,7 +1333,7 @@ def test_pip_output(local_module_package_index: str, snippetcompiler_clean, capl ) modules = ["modone", "modtwo"] - v2_requirements = [SafeRequirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in modules] + v2_requirements = [safe_parse(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in modules] snippetcompiler_clean.setup_for_snippet( f""" @@ -1419,7 +1419,7 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[SafeRequirement(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[safe_parse(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module"))], install_project=True, ) log_contains( @@ -1451,7 +1451,7 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[SafeRequirement(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[safe_parse(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module"))], install_project=True, ) @@ -1484,7 +1484,7 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[SafeRequirement(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[safe_parse(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module"))], install_project=True, ) log_contains( @@ -1553,7 +1553,7 @@ def test_version_snapshot(local_module_package_index: str, snippetcompiler_clean autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[SafeRequirement(requirement_string=module.ModuleV2Source.get_package_name_for("module_b"))], + python_requires=[safe_parse(requirement_string=module.ModuleV2Source.get_package_name_for("module_b"))], install_project=True, ) @@ -1579,7 +1579,7 @@ def test_version_snapshot(local_module_package_index: str, snippetcompiler_clean autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[SafeRequirement(requirement_string=module.ModuleV2Source.get_package_name_for("module_c"))], + python_requires=[safe_parse(requirement_string=module.ModuleV2Source.get_package_name_for("module_c"))], install_project=True, ) @@ -1650,8 +1650,7 @@ def test_constraints_logging_v2(modules_v2_dir, tmpdir, caplog, snippetcompiler_ index_url=local_module_package_index, extra_index_url=[index.url], python_requires=[ - SafeRequirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) - for mod in ["module_b", "module_a"] + safe_parse(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in ["module_b", "module_a"] ], install_project=True, project_requires=[ diff --git a/tests/moduletool/test_update.py b/tests/moduletool/test_update.py index 07369f0a85..cd584c9984 100644 --- a/tests/moduletool/test_update.py +++ b/tests/moduletool/test_update.py @@ -23,7 +23,7 @@ from inmanta.config import Config from inmanta.data.model import PipConfig -from inmanta.env import LocalPackagePath, SafeRequirement, process_env +from inmanta.env import LocalPackagePath, process_env, safe_parse from inmanta.module import InmantaModuleRequirement, InstallMode, ModuleV1, ModuleV2Source from inmanta.moduletool import ProjectTool from inmanta.parser import ParserException @@ -125,7 +125,7 @@ def assert_version_installed(module_name: str, version: str) -> None: new_version=Version(current_version), new_name=module_name, new_requirements=( - [InmantaModuleRequirement(SafeRequirement(requirement_string="module2<3.0.0"))] + [InmantaModuleRequirement(safe_parse(requirement_string="module2<3.0.0"))] if module_name == "module1" else None ), @@ -142,7 +142,7 @@ def assert_version_installed(module_name: str, version: str) -> None: # Add a dependency on module2, without setting an explicit version constraint. Later version of module1 # do set a version constraint on the dependency on module2. This way it is verified whether the module update # command takes into account the version constraints set in a new version of a module. - new_requirements=[InmantaModuleRequirement(SafeRequirement(requirement_string="module2"))], + new_requirements=[InmantaModuleRequirement(safe_parse(requirement_string="module2"))], install=False, publish_index=pip_index, new_content_init_cf="entity" if corrupt_module else None, # Introduce syntax error in the module @@ -242,7 +242,7 @@ def test_module_update_dependencies( "b", Version(v), str(tmpdir.join(f"b-{v}")), - requirements=[SafeRequirement(requirement_string="c")], + requirements=[safe_parse(requirement_string="c")], publish_index=index, ) for v in ("1.0.0", "2.0.0"): @@ -258,7 +258,7 @@ def test_module_update_dependencies( # install b-1.0.0 and c-1.0.0 process_env.install_for_config( - [SafeRequirement(requirement_string=req) for req in ("b==1.0.0", "c==1.0.0")], + [safe_parse(requirement_string=req) for req in ("b==1.0.0", "c==1.0.0")], config=PipConfig( index_url=index.url, use_system_config=False, @@ -270,7 +270,7 @@ def test_module_update_dependencies( source_dir=os.path.join(modules_dir, "minimalv1module"), dest_dir=str(tmpdir.join("modules", "my_mod")), new_name="my_mod", - new_requirements=[SafeRequirement(requirement_string=req) for req in ("a", "b~=1.0.0")], + new_requirements=[safe_parse(requirement_string=req) for req in ("a", "b~=1.0.0")], ) # run `inmanta project update` without running install first diff --git a/tests/server/test_compilerservice.py b/tests/server/test_compilerservice.py index 9dd286324a..7ccf796993 100644 --- a/tests/server/test_compilerservice.py +++ b/tests/server/test_compilerservice.py @@ -41,7 +41,7 @@ from inmanta.const import INMANTA_REMOVED_SET_ID, ParameterSource from inmanta.data import APILIMIT, Compile, Report from inmanta.data.model import PipConfig -from inmanta.env import PythonEnvironment, SafeRequirement +from inmanta.env import PythonEnvironment, safe_parse from inmanta.export import cfg_env from inmanta.protocol import Result from inmanta.server import SLICE_COMPILER, SLICE_SERVER, protocol @@ -1806,7 +1806,7 @@ def patch_get_protected_inmanta_packages(): venv = PythonEnvironment(env_path=venv_path) assert name_protected_pkg not in venv.get_installed_packages() venv.install_for_config( - requirements=[SafeRequirement(requirement_string=name_protected_pkg)], + requirements=[safe_parse(requirement_string=name_protected_pkg)], config=PipConfig( index_url=local_module_package_index, ), diff --git a/tests/test_env.py b/tests/test_env.py index 59a454c892..e74e368532 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -35,7 +35,7 @@ from inmanta import env, loader, module from inmanta.data.model import PipConfig -from inmanta.env import Pip, SafeRequirement +from inmanta.env import Pip, SafeRequirement, safe_parse from packaging import version from utils import LogSequence, PipIndex, create_python_package @@ -176,7 +176,7 @@ def test_gen_req_file(): # make sure they all parse for req in reqs: - SafeRequirement(requirement_string=req) + safe_parse(requirement_string=req) def test_environment_python_version_multi_digit(tmpdir: py.path.local) -> None: @@ -205,7 +205,7 @@ def test_process_env_install_from_index( package_name: str = "more-itertools" assert package_name not in env.process_env.get_installed_packages() env.process_env.install_for_config( - [SafeRequirement(requirement_string=package_name + (f"=={version}" if version is not None else ""))], + [safe_parse(requirement_string=package_name + (f"=={version}" if version is not None else ""))], config=PipConfig( use_system_config=True, # we need an upstream for some packages ), @@ -220,7 +220,7 @@ def test_process_env_install_from_index( # It should hit the cache there and return here. # Cheap and fast test env.process_env.install_from_index( - [SafeRequirement(requirement_string=package_name + (f"=={version}" if version is not None else ""))], + [safe_parse(requirement_string=package_name + (f"=={version}" if version is not None else ""))], use_pip_config=True, ) @@ -270,7 +270,7 @@ def test_process_env_install_from_index_not_found_env_var( with pytest.raises(env.PackageNotFound, match=re.escape(expected)): env.process_env.install_for_config( - [SafeRequirement(requirement_string="this-package-does-not-exist")], + [safe_parse(requirement_string="this-package-does-not-exist")], config=PipConfig( index_url=index_urls[0], # The first element should only be passed to the index_url. If there are indexes in the environment @@ -307,7 +307,7 @@ def test_process_env_install_no_index(tmpdir: py.path.local, monkeypatch, use_sy with pytest.raises(env.PackageNotFound, match=re.escape(expected)): env.process_env.install_for_config( - requirements=[SafeRequirement(requirement_string="this-package-does-not-exist")], + requirements=[safe_parse(requirement_string="this-package-does-not-exist")], paths=[env.LocalPackagePath(path=str(tmpdir))], config=PipConfig(use_system_config=use_system_config), ) @@ -324,7 +324,7 @@ def test_process_env_install_from_index_conflicting_reqs( package_name: str = "more-itertools" with pytest.raises(env.ConflictingRequirements) as e: env.process_env.install_for_config( - [SafeRequirement(requirement_string=f"{package_name}{version}") for version in [">8.5", "<=8"]], + [safe_parse(requirement_string=f"{package_name}{version}") for version in [">8.5", "<=8"]], config=PipConfig( use_system_config=True, # we need an upstream for some packages ), @@ -394,7 +394,7 @@ def test_active_env_get_module_file( loader.PluginModuleFinder.configure_module_finder([os.path.join(str(tmpdir), "libs")]) assert env.ActiveEnv.get_module_file(module_name) is None - env.process_env.install_for_config([SafeRequirement(requirement_string=package_name)], pip_config) + env.process_env.install_for_config([safe_parse(requirement_string=package_name)], pip_config) assert package_name in env.process_env.get_installed_packages() module_info: Optional[tuple[Optional[str], Loader]] = env.ActiveEnv.get_module_file(module_name) assert module_info is not None @@ -540,7 +540,7 @@ def assert_all_checks(expect_test: tuple[bool, str] = (True, ""), expect_nonext: create_install_package( "test-package-two", version.Version("1.0.0"), - [SafeRequirement(requirement_string="test-package-one~=1.0")], + [safe_parse(requirement_string="test-package-one~=1.0")], local_module_package_index, ) assert_all_checks() @@ -561,7 +561,7 @@ def test_active_env_check_constraints(caplog, tmpvenv_active_inherit: str, local """ caplog.set_level(logging.WARNING) in_scope: Pattern[str] = re.compile("test-package-.*") - constraints: list[SafeRequirement] = [SafeRequirement(requirement_string="test-package-one~=1.0")] + constraints: list[SafeRequirement] = [safe_parse(requirement_string="test-package-one~=1.0")] env.ActiveEnv.check(in_scope) @@ -580,7 +580,7 @@ def test_active_env_check_constraints(caplog, tmpvenv_active_inherit: str, local create_install_package( "ext-package-one", version.Version("1.0.0"), - [SafeRequirement(requirement_string="test-package-one==1.0")], + [safe_parse(requirement_string="test-package-one==1.0")], local_module_package_index, ) env.ActiveEnv.check(in_scope, constraints) @@ -611,7 +611,7 @@ def test_override_inmanta_package(tmpvenv_active_inherit: env.VirtualEnv) -> Non installed_pkgs = tmpvenv_active_inherit.get_installed_packages() assert "inmanta-core" in installed_pkgs, "The inmanta-core package should be installed to run the tests" - inmanta_requirements = SafeRequirement(requirement_string="inmanta-core==4.0.0") + inmanta_requirements = safe_parse(requirement_string="inmanta-core==4.0.0") with pytest.raises(env.ConflictingRequirements) as excinfo: tmpvenv_active_inherit.install_for_config( requirements=[inmanta_requirements], @@ -651,13 +651,13 @@ def test_cache_on_active_env(tmpvenv_active_inherit: env.ActiveEnv, local_module """ def _assert_install(requirement: str, installed: bool) -> None: - parsed_requirement = SafeRequirement(requirement_string=requirement) + parsed_requirement = safe_parse(requirement_string=requirement) for r in [requirement, parsed_requirement]: assert tmpvenv_active_inherit.are_installed(requirements=[r]) == installed _assert_install("inmanta-module-elaboratev2module==1.2.3", installed=False) tmpvenv_active_inherit.install_for_config( - requirements=[SafeRequirement(requirement_string="inmanta-module-elaboratev2module==1.2.3")], + requirements=[safe_parse(requirement_string="inmanta-module-elaboratev2module==1.2.3")], config=PipConfig( index_url=local_module_package_index, ), @@ -701,7 +701,7 @@ def test_are_installed_dependency_cycle_on_extra(tmpdir, tmpvenv_active_inherit: path=os.path.join(tmpdir, "pkg"), publish_index=pip_index, optional_dependencies={ - "optional-pkg": [SafeRequirement(requirement_string="dep[optional-dep]")], + "optional-pkg": [safe_parse(requirement_string="dep[optional-dep]")], }, ) create_python_package( @@ -710,11 +710,11 @@ def test_are_installed_dependency_cycle_on_extra(tmpdir, tmpvenv_active_inherit: path=os.path.join(tmpdir, "dep"), publish_index=pip_index, optional_dependencies={ - "optional-dep": [SafeRequirement(requirement_string="pkg[optional-pkg]")], + "optional-dep": [safe_parse(requirement_string="pkg[optional-pkg]")], }, ) - requirements = [SafeRequirement(requirement_string="pkg[optional-pkg]")] + requirements = [safe_parse(requirement_string="pkg[optional-pkg]")] tmpvenv_active_inherit.install_for_config( requirements=requirements, config=PipConfig( diff --git a/tests/test_file_parser.py b/tests/test_file_parser.py index 0904c34d03..e3353cf905 100644 --- a/tests/test_file_parser.py +++ b/tests/test_file_parser.py @@ -18,7 +18,7 @@ import os -from inmanta.env import SafeRequirement +from inmanta.env import SafeRequirement, safe_parse from inmanta.file_parser import RequirementsTxtParser @@ -40,7 +40,7 @@ def test_requirements_txt_parser(tmpdir) -> None: expected_requirements = ["test==1.2.3", "other-dep~=2.0.0", "third-dep<5.0.0", "splitteddep", "Capital"] requirements: list[SafeRequirement] = RequirementsTxtParser().parse(requirements_txt_file) - assert requirements == [SafeRequirement(requirement_string=r) for r in expected_requirements] + assert requirements == [safe_parse(requirement_string=r) for r in expected_requirements] requirements_as_str = RequirementsTxtParser.parse_requirements_as_strs(requirements_txt_file) assert requirements_as_str == expected_requirements diff --git a/tests/test_module_loader.py b/tests/test_module_loader.py index 2992687d0f..b7c596184a 100644 --- a/tests/test_module_loader.py +++ b/tests/test_module_loader.py @@ -33,7 +33,7 @@ from inmanta.ast import CompilerException from inmanta.const import CF_CACHE_DIR from inmanta.data.model import PipConfig -from inmanta.env import ConflictingRequirements, LocalPackagePath, PackageNotFound, SafeRequirement, process_env +from inmanta.env import ConflictingRequirements, LocalPackagePath, PackageNotFound, SafeRequirement, process_env, safe_parse from inmanta.module import ( DummyProject, InmantaModuleRequirement, @@ -345,7 +345,7 @@ def test_load_module_recursive_v2_module_depends_on_v1( project = snippetcompiler.setup_for_snippet( snippet="import v2_depends_on_v1", index_url=local_module_package_index, - python_requires=[SafeRequirement(requirement_string="inmanta-module-v2-depends-on-v1")], + python_requires=[safe_parse(requirement_string="inmanta-module-v2-depends-on-v1")], install_project=False, ) if preload_v1_module: @@ -372,7 +372,7 @@ def test_load_module_recursive_complex_module_dependencies(local_module_package_ snippet="import complex_module_dependencies_mod1", autostd=False, index_url=local_module_package_index, - python_requires=[SafeRequirement(requirement_string="inmanta-module-complex-module-dependencies-mod1")], + python_requires=[safe_parse(requirement_string="inmanta-module-complex-module-dependencies-mod1")], install_project=False, ) assert "complex_module_dependencies_mod1" not in project.modules @@ -411,7 +411,7 @@ def load(requires: Optional[list[SafeRequirement]] = None) -> None: with pytest.raises(ModuleLoadingException, match=f"Failed to load module {module_name}"): load() # assert that it doesn't raise an error with explicit requirements set - load([SafeRequirement(requirement_string=ModuleV2Source.get_package_name_for(module_name))]) + load([safe_parse(requirement_string=ModuleV2Source.get_package_name_for(module_name))]) @pytest.mark.parametrize("v1", [True, False]) @@ -468,9 +468,7 @@ def test_load_import_based_v2_module( extra_index_url=[index.url], # make sure that even listing the requirement in project.yml does not suffice project_requires=[InmantaModuleRequirement.parse(dependency_module_name)], - python_requires=( - [] if v1 else [SafeRequirement(requirement_string=ModuleV2Source.get_package_name_for(main_module_name))] - ), + python_requires=([] if v1 else [safe_parse(requirement_string=ModuleV2Source.get_package_name_for(main_module_name))]), ) if explicit_dependency: @@ -612,7 +610,7 @@ def test_project_requirements_dont_overwrite_core_requirements_source( module_from_template( os.path.join(modules_v2_dir, module_name), module_path, - new_requirements=[SafeRequirement(requirement_string="Jinja2==2.11.3")], + new_requirements=[safe_parse(requirement_string="Jinja2==2.11.3")], ) # Activate the snippetcompiler venv @@ -656,7 +654,7 @@ def test_project_requirements_dont_overwrite_core_requirements_index( module_from_template( os.path.join(modules_v2_dir, module_name), module_path, - new_requirements=[SafeRequirement(requirement_string="Jinja2==2.11.3")], + new_requirements=[safe_parse(requirement_string="Jinja2==2.11.3")], publish_index=index, ) @@ -710,7 +708,7 @@ def test_module_conflicting_dependencies_with_v2_modules( "y", Version("1.0.0"), str(tmpdir.join("y-1.0.0")), - requirements=[SafeRequirement(requirement_string="x~=1.0.0")], + requirements=[safe_parse(requirement_string="x~=1.0.0")], publish_index=index, ) @@ -720,7 +718,7 @@ def test_module_conflicting_dependencies_with_v2_modules( module_from_template( os.path.join(modules_v2_dir, module_name1), module_path1, - new_requirements=[SafeRequirement(requirement_string="y~=1.0.0")], + new_requirements=[safe_parse(requirement_string="y~=1.0.0")], publish_index=index, ) @@ -731,7 +729,7 @@ def test_module_conflicting_dependencies_with_v2_modules( os.path.join(modules_v2_dir, "minimalv2module"), module_path2, new_name="minimalv2module2", - new_requirements=[SafeRequirement(requirement_string="x~=2.0.0")], + new_requirements=[safe_parse(requirement_string="x~=2.0.0")], publish_index=index, ) @@ -791,7 +789,7 @@ def test_module_conflicting_dependencies_with_v1_module( os.path.join(modules_dir, module_name1), module_path1, new_name="modulev1", - new_requirements=[SafeRequirement(requirement_string="y~=1.0.0")], + new_requirements=[safe_parse(requirement_string="y~=1.0.0")], ) # Create the second module @@ -800,7 +798,7 @@ def test_module_conflicting_dependencies_with_v1_module( module_from_template( os.path.join(modules_v2_dir, module_name2), module_path2, - new_requirements=[SafeRequirement(requirement_string="y~=2.0.0")], + new_requirements=[safe_parse(requirement_string="y~=2.0.0")], publish_index=index, ) @@ -848,7 +846,7 @@ def test_module_install_extra_on_project_level_v2_dep( new_name="mymod", new_requirements=[], new_extras={ - "myfeature": [SafeRequirement(requirement_string=package_name_extra)], + "myfeature": [safe_parse(requirement_string=package_name_extra)], }, publish_index=index, ) @@ -897,7 +895,7 @@ def test_module_install_extra_on_dep_of_v2_module( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [SafeRequirement(requirement_string=package_name_extra)], + "myfeature": [safe_parse(requirement_string=package_name_extra)], }, publish_index=index, ) @@ -917,7 +915,7 @@ def test_module_install_extra_on_dep_of_v2_module( install_project=True, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[SafeRequirement(requirement_string="inmanta-module-myv2mod")], + python_requires=[safe_parse(requirement_string="inmanta-module-myv2mod")], autostd=False, ) @@ -954,7 +952,7 @@ def test_module_install_extra_on_dep_of_v1_module( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [SafeRequirement(requirement_string=package_name_extra)], + "myfeature": [safe_parse(requirement_string=package_name_extra)], }, publish_index=index, ) @@ -1004,7 +1002,7 @@ def test_module_install_extra_on_project_level_v2_dep_update_scenario( new_name="mymod", new_requirements=[], new_extras={ - "myfeature": [SafeRequirement(requirement_string=package_name_extra)], + "myfeature": [safe_parse(requirement_string=package_name_extra)], }, publish_index=index, ) @@ -1073,7 +1071,7 @@ def test_module_install_extra_on_dep_of_v2_module_update_scenario( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [SafeRequirement(requirement_string=package_name_extra)], + "myfeature": [safe_parse(requirement_string=package_name_extra)], }, publish_index=index, ) @@ -1102,7 +1100,7 @@ def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> install_project=True, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[SafeRequirement(requirement_string="inmanta-module-myv2mod==1.0.0")], + python_requires=[safe_parse(requirement_string="inmanta-module-myv2mod==1.0.0")], autostd=False, ) assert_installed(extra_installed=False) @@ -1124,7 +1122,7 @@ def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> install_project=not do_project_update, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[SafeRequirement(requirement_string="inmanta-module-myv2mod==2.0.0")], + python_requires=[safe_parse(requirement_string="inmanta-module-myv2mod==2.0.0")], autostd=False, ) if do_project_update: @@ -1168,7 +1166,7 @@ def test_module_install_extra_on_dep_of_v1_module_update_scenario( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [SafeRequirement(requirement_string=package_name_extra)], + "myfeature": [safe_parse(requirement_string=package_name_extra)], }, publish_index=index, ) @@ -1232,7 +1230,7 @@ async def test_v1_module_depends_on_third_party_dep_with_extra( os.path.join(tmpdir, "myv1mod"), new_name="myv1mod", new_content_init_cf="", - new_requirements=[SafeRequirement(requirement_string="pkg[optional-a]")], + new_requirements=[safe_parse(requirement_string="pkg[optional-a]")], ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv1mod", @@ -1252,7 +1250,7 @@ async def test_v1_module_depends_on_third_party_dep_with_extra( os.path.join(tmpdir, "myv1mod"), new_name="myv1mod", new_content_init_cf="", - new_requirements=[SafeRequirement(requirement_string="pkg[optional-a,optional-b]")], + new_requirements=[safe_parse(requirement_string="pkg[optional-a,optional-b]")], ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv1mod", @@ -1279,13 +1277,13 @@ async def test_v2_module_depends_on_third_party_dep_with_extra( str(tmpdir.join("myv2mod")), new_name="myv2mod", new_version=Version("1.0.0"), - new_requirements=[SafeRequirement(requirement_string="pkg[optional-a]")], + new_requirements=[safe_parse(requirement_string="pkg[optional-a]")], publish_index=index, ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv2mod", install_project=True, - python_requires=[SafeRequirement(requirement_string="inmanta-module-myv2mod==1.0.0")], + python_requires=[safe_parse(requirement_string="inmanta-module-myv2mod==1.0.0")], index_url=index.url, extra_index_url=[index_with_pkgs_containing_optional_deps], autostd=False, @@ -1301,13 +1299,13 @@ async def test_v2_module_depends_on_third_party_dep_with_extra( str(tmpdir.join("myv2mod")), new_name="myv2mod", new_version=Version("2.0.0"), - new_requirements=[SafeRequirement(requirement_string="pkg[optional-a,optional-b]")], + new_requirements=[safe_parse(requirement_string="pkg[optional-a,optional-b]")], publish_index=index, ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv2mod", install_project=True, - python_requires=[SafeRequirement(requirement_string="inmanta-module-myv2mod==2.0.0")], + python_requires=[safe_parse(requirement_string="inmanta-module-myv2mod==2.0.0")], index_url=index.url, extra_index_url=[index_with_pkgs_containing_optional_deps], autostd=False, From e523023fc806377fdb27141f411dedbd8a235541 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Thu, 29 Aug 2024 16:49:28 +0200 Subject: [PATCH 19/74] Remove SafeRequirement --- src/inmanta/file_parser.py | 2 +- src/inmanta/module.py | 14 +++++++------- src/inmanta/moduletool.py | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/inmanta/file_parser.py b/src/inmanta/file_parser.py index 9682e2151c..45fbaaa366 100644 --- a/src/inmanta/file_parser.py +++ b/src/inmanta/file_parser.py @@ -60,7 +60,7 @@ def parse(cls, filename: str) -> list[Requirement]: """ Get all the requirements in `filename` as a list of `Requirement` instances. """ - return [safe_parse(requirement_string=r) for r in cls.parse_requirements_as_strs(filename)] + return [safe_parse(requirement=r) for r in cls.parse_requirements_as_strs(filename)] @classmethod def parse_requirements_as_strs(cls, filename: str) -> list[str]: diff --git a/src/inmanta/module.py b/src/inmanta/module.py index b4864b8c3d..4c3cf5c5b0 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -59,7 +59,7 @@ from inmanta.ast.blocks import BasicBlock from inmanta.ast.statements import BiStatement, DefinitionStatement, DynamicStatement, Statement from inmanta.ast.statements.define import DefineImport -from inmanta.env import SafeRequirement, assert_pip_has_source, safe_parse +from inmanta.env import assert_pip_has_source, safe_parse from inmanta.file_parser import PreservativeYamlParser, RequirementsTxtParser from inmanta.parser import plyInmantaParser from inmanta.parser.plyInmantaParser import cache_manager @@ -154,7 +154,7 @@ def parse(cls: type[TInmantaModuleRequirement], spec: str) -> TInmantaModuleRequ raise ValueError("Invalid Inmanta module requirement: Inmanta module names use '_', not '-'.") return cls(safe_parse(requirement=spec)) - def get_python_package_requirement(self) -> SafeRequirement: + def get_python_package_requirement(self) -> Requirement: """ Return a SafeRequirement with the name of the Python distribution package for this module requirement. """ @@ -723,7 +723,7 @@ def install(self, project: "Project", module_spec: list[InmantaModuleRequirement assert_pip_has_source(project.metadata.pip, f"a v2 module {module_name}") - requirements: list[SafeRequirement] = [req.get_python_package_requirement() for req in module_spec] + requirements: list[Requirement] = [req.get_python_package_requirement() for req in module_spec] preinstalled: Optional[ModuleV2] = self.get_installed_module(project, module_name) # Get known requires and add them to prevent invalidating constraints through updates @@ -1180,7 +1180,7 @@ def has_requirement_for(self, pkg_name: str) -> bool: """ return any(r.name == pkg_name for r in self._requirements) - def set_requirement_and_write(self, requirement: SafeRequirement) -> None: + def set_requirement_and_write(self, requirement: Requirement) -> None: """ Add the given requirement to the requirements.txt file and update the file on disk, replacing any existing constraints on this package. @@ -2129,7 +2129,7 @@ def install_modules(self, *, bypass_module_cache: bool = False, update_dependenc self.verify_module_version_compatibility() # do python install - pyreq: list[SafeRequirement] = [safe_parse(requirement=x) for x in self.collect_python_requirements()] + pyreq: list[Requirement] = [safe_parse(requirement=x) for x in self.collect_python_requirements()] if len(pyreq) > 0: # upgrade both direct and transitive module dependencies: eager upgrade strategy @@ -2547,7 +2547,7 @@ def verify_python_requires(self) -> None: Verifies no incompatibilities exist within the Python environment with respect to installed module v2 requirements. """ if self.strict_deps_check: - constraints: list[SafeRequirement] = [safe_parse(requirement=item) for item in self.collect_python_requirements()] + constraints: list[Requirement] = [safe_parse(requirement=item) for item in self.collect_python_requirements()] env.ActiveEnv.check(strict_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*"), constraints=constraints) else: if not env.ActiveEnv.check_legacy(in_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*")): @@ -3426,7 +3426,7 @@ def add_module_requirement_persistent(self, requirement: InmantaModuleRequiremen # Parse config file config_parser = ConfigParser() config_parser.read(self.get_metadata_file_path()) - python_pkg_requirement: SafeRequirement = requirement.get_python_package_requirement() + python_pkg_requirement: Requirement = requirement.get_python_package_requirement() if config_parser.has_option("options", "install_requires"): new_install_requires = [ r diff --git a/src/inmanta/moduletool.py b/src/inmanta/moduletool.py index 555f6b0c99..c9934531cc 100644 --- a/src/inmanta/moduletool.py +++ b/src/inmanta/moduletool.py @@ -55,7 +55,7 @@ from inmanta.ast import CompilerException from inmanta.command import CLIException, ShowUsageException from inmanta.const import CF_CACHE_DIR, MAX_UPDATE_ATTEMPT -from inmanta.env import SafeRequirement, safe_parse +from inmanta.env import safe_parse from inmanta.module import ( DummyProject, FreezeOperator, @@ -76,7 +76,7 @@ gitprovider, ) from inmanta.stable_api import stable_api -from packaging.requirements import InvalidRequirement +from packaging.requirements import InvalidRequirement, Requirement from packaging.version import Version LOGGER = logging.getLogger(__name__) @@ -473,7 +473,7 @@ def update( def do_update(specs: Mapping[str, Sequence[InmantaModuleRequirement]], modules: list[str]) -> None: v2_modules = {module for module in modules if my_project.module_source.path_for(module) is not None} - v2_python_specs: list[SafeRequirement] = [ + v2_python_specs: list[Requirement] = [ module_spec.get_python_package_requirement() for module, module_specs in specs.items() for module_spec in module_specs From b0f0d8b5d82632b8b51177705f12c5044c3f4fbf Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Thu, 29 Aug 2024 16:50:15 +0200 Subject: [PATCH 20/74] renaming --- src/inmanta/agent/executor.py | 4 +- src/inmanta/agent/in_process_executor.py | 4 +- src/inmanta/env.py | 20 ++++----- src/inmanta/file_parser.py | 10 ++--- src/inmanta/module.py | 18 ++++---- src/inmanta/moduletool.py | 4 +- tests/compiler/test_basics.py | 4 +- tests/conftest.py | 6 +-- tests/moduletool/test_add.py | 4 +- tests/moduletool/test_convert_v1_v2.py | 4 +- tests/moduletool/test_install.py | 46 ++++++++++---------- tests/moduletool/test_update.py | 12 +++--- tests/server/test_compilerservice.py | 4 +- tests/test_env.py | 34 +++++++-------- tests/test_file_parser.py | 4 +- tests/test_module_loader.py | 54 ++++++++++++------------ 16 files changed, 116 insertions(+), 116 deletions(-) diff --git a/src/inmanta/agent/executor.py b/src/inmanta/agent/executor.py index 3ec61870bf..9e09eac480 100644 --- a/src/inmanta/agent/executor.py +++ b/src/inmanta/agent/executor.py @@ -41,7 +41,7 @@ from inmanta.agent import config as cfg from inmanta.agent import resourcepool from inmanta.data.model import PipConfig, ResourceIdStr, ResourceType, ResourceVersionIdStr -from inmanta.env import PythonEnvironment, safe_parse +from inmanta.env import PythonEnvironment, safe_parse_requirement from inmanta.loader import ModuleSource from inmanta.resources import Id from inmanta.types import JsonType @@ -294,7 +294,7 @@ def create_and_install_environment(self, blueprint: EnvBlueprint) -> None: self.init_env() if len(req): # install_for_config expects at least 1 requirement or a path to install self.install_for_config( - requirements=[safe_parse(requirement=e) for e in req], + requirements=[safe_parse_requirement(requirement=e) for e in req], config=blueprint.pip_config, ) diff --git a/src/inmanta/agent/in_process_executor.py b/src/inmanta/agent/in_process_executor.py index db6684457e..0911235c75 100644 --- a/src/inmanta/agent/in_process_executor.py +++ b/src/inmanta/agent/in_process_executor.py @@ -33,7 +33,7 @@ from inmanta.agent.handler import HandlerAPI, SkipResource from inmanta.const import ParameterSource from inmanta.data.model import AttributeStateChange, ResourceIdStr, ResourceVersionIdStr -from inmanta.env import safe_parse +from inmanta.env import safe_parse_requirement from inmanta.loader import CodeLoader from inmanta.resources import Id, Resource from inmanta.types import Apireturn @@ -571,7 +571,7 @@ async def _install(self, blueprint: executor.ExecutorBlueprint) -> None: await loop.run_in_executor( self.process.thread_pool, self._env.install_for_config, - [safe_parse(requirement=e) for e in blueprint.requirements], + [safe_parse_requirement(requirement=e) for e in blueprint.requirements], blueprint.pip_config, ) await loop.run_in_executor(self.process.thread_pool, self._loader.deploy_version, blueprint.sources) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 161a46cb74..b2068631a9 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -62,7 +62,7 @@ class PipInstallError(Exception): pass -def safe_parse(requirement: str) -> Requirement: +def safe_parse_requirement(requirement: str) -> Requirement: # Packaging Requirement is not able to parse requirements with comment. Therefore, we need to remove the `comment` part drop_comment = requirement.split("#")[0].strip() assert len(drop_comment) > 0, "The name of the requirement cannot be an empty string!" @@ -179,7 +179,7 @@ def _get_as_requirements_type(cls, requirements: req_list) -> Sequence[Requireme Convert requirements from Union[Sequence[str], Sequence[SafeRequirement]] to Sequence[SafeRequirement] """ if isinstance(requirements[0], str): - return [safe_parse(requirement=r) for r in requirements if isinstance(r, str)] + return [safe_parse_requirement(requirement=r) for r in requirements if isinstance(r, str)] else: return requirements @@ -234,8 +234,8 @@ def _are_installed_recursive( return False pkgs_required_by_extra: set[Requirement] = set( - [safe_parse(e.key) for e in distribution.requires(extras=(extra,))] - ) - set([safe_parse(e.key) for e in distribution.requires(extras=())]) + [safe_parse_requirement(e.key) for e in distribution.requires(extras=(extra,))] + ) - set([safe_parse_requirement(e.key) for e in distribution.requires(extras=())]) if not _are_installed_recursive( reqs=list(pkgs_required_by_extra), seen_requirements=list(seen_requirements) + list(reqs), @@ -255,7 +255,7 @@ def get_packages_in_working_set(cls, inmanta_modules_only: bool = False) -> dict :param inmanta_modules_only: Only return inmanta modules from the working set """ return { - safe_parse(dist_info.key).name: version.Version(dist_info.version) + safe_parse_requirement(dist_info.key).name: version.Version(dist_info.version) for dist_info in pkg_resources.working_set if not inmanta_modules_only or dist_info.key.startswith(const.MODULE_PKG_NAME_PREFIX) } @@ -294,7 +294,7 @@ def _get_tree_recursive_single(acc: abc.Set[str], dist: str) -> abc.Set[str]: # recurse on direct dependencies return _get_tree_recursive( - (safe_parse(requirement=requirement.key).name for requirement in installed_distributions[dist].requires()), + (safe_parse_requirement(requirement=requirement.key).name for requirement in installed_distributions[dist].requires()), acc=acc | {dist}, ) @@ -861,7 +861,7 @@ def install_from_list( use_pip_config was ignored on ISO6 and it still is """ self.install_from_index( - requirements=[safe_parse(requirement=r) for r in requirements_list], + requirements=[safe_parse_requirement(requirement=r) for r in requirements_list], upgrade=upgrade, upgrade_strategy=upgrade_strategy, use_pip_config=True, @@ -888,7 +888,7 @@ def _get_requirements_on_inmanta_package(cls) -> Sequence[Requirement]: """ protected_inmanta_packages: list[str] = cls.get_protected_inmanta_packages() workingset: dict[str, version.Version] = PythonWorkingSet.get_packages_in_working_set() - return [safe_parse(requirement=f"{pkg}=={workingset[pkg]}") for pkg in workingset if pkg in protected_inmanta_packages] + return [safe_parse_requirement(requirement=f"{pkg}=={workingset[pkg]}") for pkg in workingset if pkg in protected_inmanta_packages] class CommandRunner: @@ -1016,7 +1016,7 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: # all requirements of all packages installed in this environment installed_constraints: abc.Set[OwnedRequirement] = frozenset( - OwnedRequirement(safe_parse(requirement=requirement.key), dist_info.key) + OwnedRequirement(safe_parse_requirement(requirement=requirement.key), dist_info.key) for dist_info in pkg_resources.working_set for requirement in dist_info.requires() ) @@ -1113,7 +1113,7 @@ def check_legacy(cls, in_scope: Pattern[str], constraints: Optional[list[Require working_set: abc.Iterable[Distribution] = importlib.metadata.distributions() # add all requirements of all in scope packages installed in this environment all_constraints: set[Requirement] = set(constraints if constraints is not None else []).union( - safe_parse(requirement=requirement) + safe_parse_requirement(requirement=requirement) for dist_info in working_set if in_scope.fullmatch(dist_info.name) for requirement in dist_info.requires or [] diff --git a/src/inmanta/file_parser.py b/src/inmanta/file_parser.py index 45fbaaa366..4bdebbfd11 100644 --- a/src/inmanta/file_parser.py +++ b/src/inmanta/file_parser.py @@ -18,7 +18,7 @@ import os -from inmanta.env import safe_parse +from inmanta.env import safe_parse_requirement from packaging.requirements import Requirement from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap @@ -60,7 +60,7 @@ def parse(cls, filename: str) -> list[Requirement]: """ Get all the requirements in `filename` as a list of `Requirement` instances. """ - return [safe_parse(requirement=r) for r in cls.parse_requirements_as_strs(filename)] + return [safe_parse_requirement(requirement=r) for r in cls.parse_requirements_as_strs(filename)] @classmethod def parse_requirements_as_strs(cls, filename: str) -> list[str]: @@ -86,7 +86,7 @@ def get_content_with_dep_removed(cls, filename: str, remove_dep_on_pkg: str) -> if not os.path.exists(filename): raise Exception(f"File {filename} doesn't exist") - removed_dependency = safe_parse(requirement=remove_dep_on_pkg) + removed_dependency = safe_parse_requirement(requirement=remove_dep_on_pkg) result = "" line_continuation_buffer = "" with open(filename, encoding="utf-8") as fd: @@ -94,14 +94,14 @@ def get_content_with_dep_removed(cls, filename: str, remove_dep_on_pkg: str) -> if line_continuation_buffer: line_continuation_buffer += line if not line.endswith("\\"): - if safe_parse(requirement=line_continuation_buffer).name != removed_dependency.name: + if safe_parse_requirement(requirement=line_continuation_buffer).name != removed_dependency.name: result += line_continuation_buffer line_continuation_buffer = "" elif not line.strip() or line.strip().startswith("#"): result += line elif line.endswith("\\"): line_continuation_buffer = line - elif safe_parse(requirement=line).name != removed_dependency.name: + elif safe_parse_requirement(requirement=line).name != removed_dependency.name: result += line else: # Dependency matches `remove_dep_on_pkg` => Remove line from result diff --git a/src/inmanta/module.py b/src/inmanta/module.py index 4c3cf5c5b0..0c6ce977f7 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -59,7 +59,7 @@ from inmanta.ast.blocks import BasicBlock from inmanta.ast.statements import BiStatement, DefinitionStatement, DynamicStatement, Statement from inmanta.ast.statements.define import DefineImport -from inmanta.env import assert_pip_has_source, safe_parse +from inmanta.env import assert_pip_has_source, safe_parse_requirement from inmanta.file_parser import PreservativeYamlParser, RequirementsTxtParser from inmanta.parser import plyInmantaParser from inmanta.parser.plyInmantaParser import cache_manager @@ -152,7 +152,7 @@ def parse(cls: type[TInmantaModuleRequirement], spec: str) -> TInmantaModuleRequ ) if "-" in spec: raise ValueError("Invalid Inmanta module requirement: Inmanta module names use '_', not '-'.") - return cls(safe_parse(requirement=spec)) + return cls(safe_parse_requirement(requirement=spec)) def get_python_package_requirement(self) -> Requirement: """ @@ -161,7 +161,7 @@ def get_python_package_requirement(self) -> Requirement: module_name = self.project_name pkg_name = ModuleV2Source.get_package_name_for(module_name) pkg_req_str = str(self).replace(module_name, pkg_name, 1) # Replace max 1 occurrence - return safe_parse(requirement=pkg_req_str) + return safe_parse_requirement(requirement=pkg_req_str) class CompilerExceptionWithExtendedTrace(CompilerException): @@ -730,7 +730,7 @@ def install(self, project: "Project", module_spec: list[InmantaModuleRequirement # These could be constraints (-c) as well, but that requires additional sanitation # Because for pip not every valid -r is a valid -c current_requires = project.get_strict_python_requirements_as_list() - requirements += [safe_parse(requirement=r) for r in current_requires] + requirements += [safe_parse_requirement(requirement=r) for r in current_requires] if preinstalled is not None: # log warning if preinstalled version does not match constraints @@ -2129,7 +2129,7 @@ def install_modules(self, *, bypass_module_cache: bool = False, update_dependenc self.verify_module_version_compatibility() # do python install - pyreq: list[Requirement] = [safe_parse(requirement=x) for x in self.collect_python_requirements()] + pyreq: list[Requirement] = [safe_parse_requirement(requirement=x) for x in self.collect_python_requirements()] if len(pyreq) > 0: # upgrade both direct and transitive module dependencies: eager upgrade strategy @@ -2547,7 +2547,7 @@ def verify_python_requires(self) -> None: Verifies no incompatibilities exist within the Python environment with respect to installed module v2 requirements. """ if self.strict_deps_check: - constraints: list[Requirement] = [safe_parse(requirement=item) for item in self.collect_python_requirements()] + constraints: list[Requirement] = [safe_parse_requirement(requirement=item) for item in self.collect_python_requirements()] env.ActiveEnv.check(strict_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*"), constraints=constraints) else: if not env.ActiveEnv.check_legacy(in_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*")): @@ -2678,7 +2678,7 @@ def requires(self) -> "List[InmantaModuleRequirement]": # filter on import stmt reqs = [] for spec in self._metadata.requires: - req = [safe_parse(requirement=spec)] + req = [safe_parse_requirement(requirement=spec)] if len(req) > 1: print(f"Module file for {self._path} has bad line in requirements specification {spec}") reqe = InmantaModuleRequirement(req[0]) @@ -2815,7 +2815,7 @@ def requires(self) -> "List[InmantaModuleRequirement]": """ reqs = [] for spec in self.get_module_requirements(): - req = [safe_parse(requirement=spec)] + req = [safe_parse_requirement(requirement=spec)] if len(req) > 1: print(f"Module file for {self._path} has bad line in requirements specification {spec}") reqe = InmantaModuleRequirement(req[0]) @@ -3431,7 +3431,7 @@ def add_module_requirement_persistent(self, requirement: InmantaModuleRequiremen new_install_requires = [ r for r in config_parser.get("options", "install_requires").split("\n") - if r and safe_parse(requirement=r).name != python_pkg_requirement.name + if r and safe_parse_requirement(requirement=r).name != python_pkg_requirement.name ] new_install_requires.append(str(python_pkg_requirement)) else: diff --git a/src/inmanta/moduletool.py b/src/inmanta/moduletool.py index c9934531cc..099a27263c 100644 --- a/src/inmanta/moduletool.py +++ b/src/inmanta/moduletool.py @@ -55,7 +55,7 @@ from inmanta.ast import CompilerException from inmanta.command import CLIException, ShowUsageException from inmanta.const import CF_CACHE_DIR, MAX_UPDATE_ATTEMPT -from inmanta.env import safe_parse +from inmanta.env import safe_parse_requirement from inmanta.module import ( DummyProject, FreezeOperator, @@ -485,7 +485,7 @@ def do_update(specs: Mapping[str, Sequence[InmantaModuleRequirement]], modules: # Because for pip not every valid -r is a valid -c current_requires = my_project.get_strict_python_requirements_as_list() env.process_env.install_for_config( - v2_python_specs + [safe_parse(requirement=r) for r in current_requires], + v2_python_specs + [safe_parse_requirement(requirement=r) for r in current_requires], my_project.metadata.pip, upgrade=True, ) diff --git a/tests/compiler/test_basics.py b/tests/compiler/test_basics.py index de12f5db25..1ee2a21c0f 100644 --- a/tests/compiler/test_basics.py +++ b/tests/compiler/test_basics.py @@ -24,7 +24,7 @@ from inmanta import compiler, const from inmanta.ast import DoubleSetException, RuntimeException -from inmanta.env import safe_parse +from inmanta.env import safe_parse_requirement from inmanta.plugins import PluginDeprecationWarning from utils import module_from_template, v1_module_from_template @@ -740,4 +740,4 @@ def test_safe_requirement(name) -> None: Ensure that empty name requirements are not allowed in `SafeRequirement` """ with pytest.raises(AssertionError): - safe_parse(requirement=name) + safe_parse_requirement(requirement=name) diff --git a/tests/conftest.py b/tests/conftest.py index edb8a53fe5..78b62218c6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -119,7 +119,7 @@ from inmanta.ast import CompilerException from inmanta.data.schema import SCHEMA_VERSION_TABLE from inmanta.db import util as db_util -from inmanta.env import CommandRunner, LocalPackagePath, SafeRequirement, VirtualEnv, mock_process_env, safe_parse +from inmanta.env import CommandRunner, LocalPackagePath, SafeRequirement, VirtualEnv, mock_process_env, safe_parse_requirement from inmanta.export import ResourceDict, cfg_env, unknown_parameters from inmanta.module import InmantaModuleRequirement, InstallMode, Project, RelationPrecedenceRule from inmanta.moduletool import DefaultIsolatedEnvCached, ModuleTool, V2ModuleBuilder @@ -1918,8 +1918,8 @@ def index_with_pkgs_containing_optional_deps() -> str: path=os.path.join(tmpdirname, "pkg"), publish_index=pip_index, optional_dependencies={ - "optional-a": [safe_parse(requirement="dep-a")], - "optional-b": [safe_parse(requirement="dep-b"), safe_parse(requirement="dep-c")], + "optional-a": [safe_parse_requirement(requirement="dep-a")], + "optional-b": [safe_parse_requirement(requirement="dep-b"), safe_parse_requirement(requirement="dep-c")], }, ) for pkg_name in ["dep-a", "dep-b", "dep-c"]: diff --git a/tests/moduletool/test_add.py b/tests/moduletool/test_add.py index dfb2f3c452..cb93dc2aa7 100644 --- a/tests/moduletool/test_add.py +++ b/tests/moduletool/test_add.py @@ -25,7 +25,7 @@ import pytest from inmanta.command import CLIException -from inmanta.env import process_env, safe_parse +from inmanta.env import process_env, safe_parse_requirement from inmanta.module import ModuleV1, ModuleV1Metadata, ModuleV2, ModuleV2Source, Project, ProjectMetadata from inmanta.moduletool import ModuleTool from packaging.version import Version @@ -88,7 +88,7 @@ def test_module_add_v2_module_to_project( dest_dir=os.path.join(tmpdir, f"elaboratev2module-v{version}"), new_version=Version(version), publish_index=pip_index, - new_extras={"optional": [safe_parse(requirement="inmanta-module-minimalv2module")]}, + new_extras={"optional": [safe_parse_requirement(requirement="inmanta-module-minimalv2module")]}, ) # Create project diff --git a/tests/moduletool/test_convert_v1_v2.py b/tests/moduletool/test_convert_v1_v2.py index 5e1af3431f..a54625eeb7 100644 --- a/tests/moduletool/test_convert_v1_v2.py +++ b/tests/moduletool/test_convert_v1_v2.py @@ -30,7 +30,7 @@ import toml from inmanta import moduletool from inmanta.command import CLIException -from inmanta.env import safe_parse +from inmanta.env import safe_parse_requirement from inmanta.module import DummyProject, ModuleV1, ModuleV2, ModuleV2Metadata from inmanta.moduletool import ModuleConverter, ModuleVersionException from packaging import version @@ -114,7 +114,7 @@ def test_issue_3159_conversion_std_module_add_self_to_dependencies(tmpdir): parser = configparser.ConfigParser() parser.read(setup_cfg_file) assert parser.has_option("options", "install_requires") - install_requires = [safe_parse(requirement=r) for r in parser.get("options", "install_requires").split("\n") if r] + install_requires = [safe_parse_requirement(requirement=r) for r in parser.get("options", "install_requires").split("\n") if r] pkg_names = [r.name for r in install_requires] assert "inmanta-module-std" not in pkg_names diff --git a/tests/moduletool/test_install.py b/tests/moduletool/test_install.py index 4bcd10edc4..896c74d104 100644 --- a/tests/moduletool/test_install.py +++ b/tests/moduletool/test_install.py @@ -35,7 +35,7 @@ from inmanta.ast import CompilerException from inmanta.command import CLIException from inmanta.config import Config -from inmanta.env import CommandRunner, ConflictingRequirements, PipConfig, safe_parse +from inmanta.env import CommandRunner, ConflictingRequirements, PipConfig, safe_parse_requirement from inmanta.module import InmantaModuleRequirement, InstallMode, ModuleLoadingException, ModuleNotFoundException from inmanta.moduletool import DummyProject, ModuleConverter, ModuleTool, ProjectTool from moduletool.common import BadModProvider, install_project @@ -251,14 +251,14 @@ def test_module_install_conflicting_requirements(tmpdir: py.path.local, snippetc os.path.join(modules_v2_dir, "minimalv2module"), os.path.join(str(tmpdir), "modone"), new_name="modone", - new_requirements=[safe_parse(requiremement="lorem~=0.0.1")], + new_requirements=[safe_parse_requirement(requiremement="lorem~=0.0.1")], install=True, ) module_from_template( os.path.join(modules_v2_dir, "minimalv2module"), os.path.join(str(tmpdir), "modtwo"), new_name="modtwo", - new_requirements=[safe_parse(requirement_string="lorem~=0.1.0")], + new_requirements=[safe_parse_requirement(requirement_string="lorem~=0.1.0")], install=True, ) @@ -508,7 +508,7 @@ def test_project_install( index_url=local_module_package_index, # We add tornado, as there is a code path in update for the case where the project has python requires python_requires=["tornado"] - + [safe_parse(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names], + + [safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names], install_project=False, ) @@ -541,7 +541,7 @@ def test_project_install( autostd=False, python_package_sources=[local_module_package_index], python_requires=[ - safe_parse(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names + safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names ] + ["lorem"], install_project=False, @@ -677,7 +677,7 @@ def test_project_install_modules_cache_invalid( index_url=index.url, extra_index_url=[local_module_package_index], # make sure main module gets installed, pulling in newest version of dependency module - python_requires=[safe_parse(requirement_string=module.ModuleV2Source.get_package_name_for(main_module))], + python_requires=[safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for(main_module))], ) # populate project.modules[dependency_module] to force the error conditions in this simplified example @@ -762,7 +762,7 @@ def test_project_install_incompatible_versions( install_project=False, add_to_module_path=[v1_modules_path], index_url=index.url, - python_requires=[safe_parse(requirement_string=module.ModuleV2Source.get_package_name_for(v2_mod_name))], + python_requires=[safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for(v2_mod_name))], ) # install project @@ -815,14 +815,14 @@ def test_project_install_incompatible_dependencies( v2_template_path, os.path.join(str(tmpdir), "v2mod2"), new_name="v2mod2", - new_requirements=[safe_parse(requirement_string="inmanta-module-v2mod1~=1.0.0")], + new_requirements=[safe_parse_requirement(requirement_string="inmanta-module-v2mod1~=1.0.0")], publish_index=index, ) v2mod3: module.ModuleV2Metadata = module_from_template( v2_template_path, os.path.join(str(tmpdir), "v2mod3"), new_name="v2mod3", - new_requirements=[safe_parse(requirement_string="inmanta-module-v2mod1~=2.0.0")], + new_requirements=[safe_parse_requirement(requirement_string="inmanta-module-v2mod1~=2.0.0")], publish_index=index, ) @@ -836,7 +836,7 @@ def test_project_install_incompatible_dependencies( install_project=False, index_url=index.url, python_requires=[ - safe_parse( + safe_parse_requirement( requirement_string=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod2, v2mod3] @@ -920,7 +920,7 @@ def test_install_from_index_dont_leak_pip_index( # Installing a V2 module requires a python package source. index_url="unknown", python_requires=[ - safe_parse( + safe_parse_requirement( requirement_string=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod1] @@ -982,7 +982,7 @@ def test_install_with_use_config( index_url=index.url if not use_pip_config else None, use_pip_config_file=use_pip_config, python_requires=[ - safe_parse( + safe_parse_requirement( requirement_string=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod1] @@ -1051,7 +1051,7 @@ def test_install_with_use_config_extra_index( extra_index_url=[index2.url], use_pip_config_file=True, python_requires=[ - safe_parse( + safe_parse_requirement( requirement_string=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod1, v2mod2] @@ -1087,7 +1087,7 @@ def test_install_with_use_config_but_PIP_CONFIG_FILE_not_set( autostd=False, install_project=False, use_pip_config_file=True, - python_requires=[safe_parse(requirement_string="inmanta-module-dummy-module")], + python_requires=[safe_parse_requirement(requirement_string="inmanta-module-dummy-module")], ) # install project @@ -1204,7 +1204,7 @@ def test_install_project_with_install_mode_master(tmpdir: py.path.local, snippet autostd=False, install_project=False, add_to_module_path=[str(tmpdir)], - project_requires=[InmantaModuleRequirement(safe_parse(requirement_string="mod11==3.2.1"))], + project_requires=[InmantaModuleRequirement(safe_parse_requirement(requirement_string="mod11==3.2.1"))], install_mode=InstallMode.master, ) @@ -1232,7 +1232,7 @@ def test_module_install_logging(local_module_package_index: str, snippetcompiler v2_module = "minimalv2module" - v2_requirements = [safe_parse(requirement_string=module.ModuleV2Source.get_package_name_for(v2_module))] + v2_requirements = [safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for(v2_module))] # set up project and modules project: module.Project = snippetcompiler_clean.setup_for_snippet( @@ -1333,7 +1333,7 @@ def test_pip_output(local_module_package_index: str, snippetcompiler_clean, capl ) modules = ["modone", "modtwo"] - v2_requirements = [safe_parse(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in modules] + v2_requirements = [safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in modules] snippetcompiler_clean.setup_for_snippet( f""" @@ -1419,7 +1419,7 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[safe_parse(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module"))], install_project=True, ) log_contains( @@ -1451,7 +1451,7 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[safe_parse(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module"))], install_project=True, ) @@ -1484,7 +1484,7 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[safe_parse(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module"))], install_project=True, ) log_contains( @@ -1553,7 +1553,7 @@ def test_version_snapshot(local_module_package_index: str, snippetcompiler_clean autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[safe_parse(requirement_string=module.ModuleV2Source.get_package_name_for("module_b"))], + python_requires=[safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for("module_b"))], install_project=True, ) @@ -1579,7 +1579,7 @@ def test_version_snapshot(local_module_package_index: str, snippetcompiler_clean autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[safe_parse(requirement_string=module.ModuleV2Source.get_package_name_for("module_c"))], + python_requires=[safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for("module_c"))], install_project=True, ) @@ -1650,7 +1650,7 @@ def test_constraints_logging_v2(modules_v2_dir, tmpdir, caplog, snippetcompiler_ index_url=local_module_package_index, extra_index_url=[index.url], python_requires=[ - safe_parse(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in ["module_b", "module_a"] + safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in ["module_b", "module_a"] ], install_project=True, project_requires=[ diff --git a/tests/moduletool/test_update.py b/tests/moduletool/test_update.py index cd584c9984..826b44b5b0 100644 --- a/tests/moduletool/test_update.py +++ b/tests/moduletool/test_update.py @@ -23,7 +23,7 @@ from inmanta.config import Config from inmanta.data.model import PipConfig -from inmanta.env import LocalPackagePath, process_env, safe_parse +from inmanta.env import LocalPackagePath, process_env, safe_parse_requirement from inmanta.module import InmantaModuleRequirement, InstallMode, ModuleV1, ModuleV2Source from inmanta.moduletool import ProjectTool from inmanta.parser import ParserException @@ -125,7 +125,7 @@ def assert_version_installed(module_name: str, version: str) -> None: new_version=Version(current_version), new_name=module_name, new_requirements=( - [InmantaModuleRequirement(safe_parse(requirement_string="module2<3.0.0"))] + [InmantaModuleRequirement(safe_parse_requirement(requirement_string="module2<3.0.0"))] if module_name == "module1" else None ), @@ -142,7 +142,7 @@ def assert_version_installed(module_name: str, version: str) -> None: # Add a dependency on module2, without setting an explicit version constraint. Later version of module1 # do set a version constraint on the dependency on module2. This way it is verified whether the module update # command takes into account the version constraints set in a new version of a module. - new_requirements=[InmantaModuleRequirement(safe_parse(requirement_string="module2"))], + new_requirements=[InmantaModuleRequirement(safe_parse_requirement(requirement_string="module2"))], install=False, publish_index=pip_index, new_content_init_cf="entity" if corrupt_module else None, # Introduce syntax error in the module @@ -242,7 +242,7 @@ def test_module_update_dependencies( "b", Version(v), str(tmpdir.join(f"b-{v}")), - requirements=[safe_parse(requirement_string="c")], + requirements=[safe_parse_requirement(requirement_string="c")], publish_index=index, ) for v in ("1.0.0", "2.0.0"): @@ -258,7 +258,7 @@ def test_module_update_dependencies( # install b-1.0.0 and c-1.0.0 process_env.install_for_config( - [safe_parse(requirement_string=req) for req in ("b==1.0.0", "c==1.0.0")], + [safe_parse_requirement(requirement_string=req) for req in ("b==1.0.0", "c==1.0.0")], config=PipConfig( index_url=index.url, use_system_config=False, @@ -270,7 +270,7 @@ def test_module_update_dependencies( source_dir=os.path.join(modules_dir, "minimalv1module"), dest_dir=str(tmpdir.join("modules", "my_mod")), new_name="my_mod", - new_requirements=[safe_parse(requirement_string=req) for req in ("a", "b~=1.0.0")], + new_requirements=[safe_parse_requirement(requirement_string=req) for req in ("a", "b~=1.0.0")], ) # run `inmanta project update` without running install first diff --git a/tests/server/test_compilerservice.py b/tests/server/test_compilerservice.py index 7ccf796993..39ca6775dc 100644 --- a/tests/server/test_compilerservice.py +++ b/tests/server/test_compilerservice.py @@ -41,7 +41,7 @@ from inmanta.const import INMANTA_REMOVED_SET_ID, ParameterSource from inmanta.data import APILIMIT, Compile, Report from inmanta.data.model import PipConfig -from inmanta.env import PythonEnvironment, safe_parse +from inmanta.env import PythonEnvironment, safe_parse_requirement from inmanta.export import cfg_env from inmanta.protocol import Result from inmanta.server import SLICE_COMPILER, SLICE_SERVER, protocol @@ -1806,7 +1806,7 @@ def patch_get_protected_inmanta_packages(): venv = PythonEnvironment(env_path=venv_path) assert name_protected_pkg not in venv.get_installed_packages() venv.install_for_config( - requirements=[safe_parse(requirement_string=name_protected_pkg)], + requirements=[safe_parse_requirement(requirement_string=name_protected_pkg)], config=PipConfig( index_url=local_module_package_index, ), diff --git a/tests/test_env.py b/tests/test_env.py index e74e368532..0b15f1cd2e 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -35,7 +35,7 @@ from inmanta import env, loader, module from inmanta.data.model import PipConfig -from inmanta.env import Pip, SafeRequirement, safe_parse +from inmanta.env import Pip, SafeRequirement, safe_parse_requirement from packaging import version from utils import LogSequence, PipIndex, create_python_package @@ -176,7 +176,7 @@ def test_gen_req_file(): # make sure they all parse for req in reqs: - safe_parse(requirement_string=req) + safe_parse_requirement(requirement_string=req) def test_environment_python_version_multi_digit(tmpdir: py.path.local) -> None: @@ -205,7 +205,7 @@ def test_process_env_install_from_index( package_name: str = "more-itertools" assert package_name not in env.process_env.get_installed_packages() env.process_env.install_for_config( - [safe_parse(requirement_string=package_name + (f"=={version}" if version is not None else ""))], + [safe_parse_requirement(requirement_string=package_name + (f"=={version}" if version is not None else ""))], config=PipConfig( use_system_config=True, # we need an upstream for some packages ), @@ -220,7 +220,7 @@ def test_process_env_install_from_index( # It should hit the cache there and return here. # Cheap and fast test env.process_env.install_from_index( - [safe_parse(requirement_string=package_name + (f"=={version}" if version is not None else ""))], + [safe_parse_requirement(requirement_string=package_name + (f"=={version}" if version is not None else ""))], use_pip_config=True, ) @@ -270,7 +270,7 @@ def test_process_env_install_from_index_not_found_env_var( with pytest.raises(env.PackageNotFound, match=re.escape(expected)): env.process_env.install_for_config( - [safe_parse(requirement_string="this-package-does-not-exist")], + [safe_parse_requirement(requirement_string="this-package-does-not-exist")], config=PipConfig( index_url=index_urls[0], # The first element should only be passed to the index_url. If there are indexes in the environment @@ -307,7 +307,7 @@ def test_process_env_install_no_index(tmpdir: py.path.local, monkeypatch, use_sy with pytest.raises(env.PackageNotFound, match=re.escape(expected)): env.process_env.install_for_config( - requirements=[safe_parse(requirement_string="this-package-does-not-exist")], + requirements=[safe_parse_requirement(requirement_string="this-package-does-not-exist")], paths=[env.LocalPackagePath(path=str(tmpdir))], config=PipConfig(use_system_config=use_system_config), ) @@ -324,7 +324,7 @@ def test_process_env_install_from_index_conflicting_reqs( package_name: str = "more-itertools" with pytest.raises(env.ConflictingRequirements) as e: env.process_env.install_for_config( - [safe_parse(requirement_string=f"{package_name}{version}") for version in [">8.5", "<=8"]], + [safe_parse_requirement(requirement_string=f"{package_name}{version}") for version in [">8.5", "<=8"]], config=PipConfig( use_system_config=True, # we need an upstream for some packages ), @@ -394,7 +394,7 @@ def test_active_env_get_module_file( loader.PluginModuleFinder.configure_module_finder([os.path.join(str(tmpdir), "libs")]) assert env.ActiveEnv.get_module_file(module_name) is None - env.process_env.install_for_config([safe_parse(requirement_string=package_name)], pip_config) + env.process_env.install_for_config([safe_parse_requirement(requirement_string=package_name)], pip_config) assert package_name in env.process_env.get_installed_packages() module_info: Optional[tuple[Optional[str], Loader]] = env.ActiveEnv.get_module_file(module_name) assert module_info is not None @@ -540,7 +540,7 @@ def assert_all_checks(expect_test: tuple[bool, str] = (True, ""), expect_nonext: create_install_package( "test-package-two", version.Version("1.0.0"), - [safe_parse(requirement_string="test-package-one~=1.0")], + [safe_parse_requirement(requirement_string="test-package-one~=1.0")], local_module_package_index, ) assert_all_checks() @@ -561,7 +561,7 @@ def test_active_env_check_constraints(caplog, tmpvenv_active_inherit: str, local """ caplog.set_level(logging.WARNING) in_scope: Pattern[str] = re.compile("test-package-.*") - constraints: list[SafeRequirement] = [safe_parse(requirement_string="test-package-one~=1.0")] + constraints: list[SafeRequirement] = [safe_parse_requirement(requirement_string="test-package-one~=1.0")] env.ActiveEnv.check(in_scope) @@ -580,7 +580,7 @@ def test_active_env_check_constraints(caplog, tmpvenv_active_inherit: str, local create_install_package( "ext-package-one", version.Version("1.0.0"), - [safe_parse(requirement_string="test-package-one==1.0")], + [safe_parse_requirement(requirement_string="test-package-one==1.0")], local_module_package_index, ) env.ActiveEnv.check(in_scope, constraints) @@ -611,7 +611,7 @@ def test_override_inmanta_package(tmpvenv_active_inherit: env.VirtualEnv) -> Non installed_pkgs = tmpvenv_active_inherit.get_installed_packages() assert "inmanta-core" in installed_pkgs, "The inmanta-core package should be installed to run the tests" - inmanta_requirements = safe_parse(requirement_string="inmanta-core==4.0.0") + inmanta_requirements = safe_parse_requirement(requirement_string="inmanta-core==4.0.0") with pytest.raises(env.ConflictingRequirements) as excinfo: tmpvenv_active_inherit.install_for_config( requirements=[inmanta_requirements], @@ -651,13 +651,13 @@ def test_cache_on_active_env(tmpvenv_active_inherit: env.ActiveEnv, local_module """ def _assert_install(requirement: str, installed: bool) -> None: - parsed_requirement = safe_parse(requirement_string=requirement) + parsed_requirement = safe_parse_requirement(requirement_string=requirement) for r in [requirement, parsed_requirement]: assert tmpvenv_active_inherit.are_installed(requirements=[r]) == installed _assert_install("inmanta-module-elaboratev2module==1.2.3", installed=False) tmpvenv_active_inherit.install_for_config( - requirements=[safe_parse(requirement_string="inmanta-module-elaboratev2module==1.2.3")], + requirements=[safe_parse_requirement(requirement_string="inmanta-module-elaboratev2module==1.2.3")], config=PipConfig( index_url=local_module_package_index, ), @@ -701,7 +701,7 @@ def test_are_installed_dependency_cycle_on_extra(tmpdir, tmpvenv_active_inherit: path=os.path.join(tmpdir, "pkg"), publish_index=pip_index, optional_dependencies={ - "optional-pkg": [safe_parse(requirement_string="dep[optional-dep]")], + "optional-pkg": [safe_parse_requirement(requirement_string="dep[optional-dep]")], }, ) create_python_package( @@ -710,11 +710,11 @@ def test_are_installed_dependency_cycle_on_extra(tmpdir, tmpvenv_active_inherit: path=os.path.join(tmpdir, "dep"), publish_index=pip_index, optional_dependencies={ - "optional-dep": [safe_parse(requirement_string="pkg[optional-pkg]")], + "optional-dep": [safe_parse_requirement(requirement_string="pkg[optional-pkg]")], }, ) - requirements = [safe_parse(requirement_string="pkg[optional-pkg]")] + requirements = [safe_parse_requirement(requirement_string="pkg[optional-pkg]")] tmpvenv_active_inherit.install_for_config( requirements=requirements, config=PipConfig( diff --git a/tests/test_file_parser.py b/tests/test_file_parser.py index e3353cf905..1486821b96 100644 --- a/tests/test_file_parser.py +++ b/tests/test_file_parser.py @@ -18,7 +18,7 @@ import os -from inmanta.env import SafeRequirement, safe_parse +from inmanta.env import SafeRequirement, safe_parse_requirement from inmanta.file_parser import RequirementsTxtParser @@ -40,7 +40,7 @@ def test_requirements_txt_parser(tmpdir) -> None: expected_requirements = ["test==1.2.3", "other-dep~=2.0.0", "third-dep<5.0.0", "splitteddep", "Capital"] requirements: list[SafeRequirement] = RequirementsTxtParser().parse(requirements_txt_file) - assert requirements == [safe_parse(requirement_string=r) for r in expected_requirements] + assert requirements == [safe_parse_requirement(requirement_string=r) for r in expected_requirements] requirements_as_str = RequirementsTxtParser.parse_requirements_as_strs(requirements_txt_file) assert requirements_as_str == expected_requirements diff --git a/tests/test_module_loader.py b/tests/test_module_loader.py index b7c596184a..1b5eec51bd 100644 --- a/tests/test_module_loader.py +++ b/tests/test_module_loader.py @@ -33,7 +33,7 @@ from inmanta.ast import CompilerException from inmanta.const import CF_CACHE_DIR from inmanta.data.model import PipConfig -from inmanta.env import ConflictingRequirements, LocalPackagePath, PackageNotFound, SafeRequirement, process_env, safe_parse +from inmanta.env import ConflictingRequirements, LocalPackagePath, PackageNotFound, SafeRequirement, process_env, safe_parse_requirement from inmanta.module import ( DummyProject, InmantaModuleRequirement, @@ -345,7 +345,7 @@ def test_load_module_recursive_v2_module_depends_on_v1( project = snippetcompiler.setup_for_snippet( snippet="import v2_depends_on_v1", index_url=local_module_package_index, - python_requires=[safe_parse(requirement_string="inmanta-module-v2-depends-on-v1")], + python_requires=[safe_parse_requirement(requirement_string="inmanta-module-v2-depends-on-v1")], install_project=False, ) if preload_v1_module: @@ -372,7 +372,7 @@ def test_load_module_recursive_complex_module_dependencies(local_module_package_ snippet="import complex_module_dependencies_mod1", autostd=False, index_url=local_module_package_index, - python_requires=[safe_parse(requirement_string="inmanta-module-complex-module-dependencies-mod1")], + python_requires=[safe_parse_requirement(requirement_string="inmanta-module-complex-module-dependencies-mod1")], install_project=False, ) assert "complex_module_dependencies_mod1" not in project.modules @@ -411,7 +411,7 @@ def load(requires: Optional[list[SafeRequirement]] = None) -> None: with pytest.raises(ModuleLoadingException, match=f"Failed to load module {module_name}"): load() # assert that it doesn't raise an error with explicit requirements set - load([safe_parse(requirement_string=ModuleV2Source.get_package_name_for(module_name))]) + load([safe_parse_requirement(requirement_string=ModuleV2Source.get_package_name_for(module_name))]) @pytest.mark.parametrize("v1", [True, False]) @@ -468,7 +468,7 @@ def test_load_import_based_v2_module( extra_index_url=[index.url], # make sure that even listing the requirement in project.yml does not suffice project_requires=[InmantaModuleRequirement.parse(dependency_module_name)], - python_requires=([] if v1 else [safe_parse(requirement_string=ModuleV2Source.get_package_name_for(main_module_name))]), + python_requires=([] if v1 else [safe_parse_requirement(requirement_string=ModuleV2Source.get_package_name_for(main_module_name))]), ) if explicit_dependency: @@ -610,7 +610,7 @@ def test_project_requirements_dont_overwrite_core_requirements_source( module_from_template( os.path.join(modules_v2_dir, module_name), module_path, - new_requirements=[safe_parse(requirement_string="Jinja2==2.11.3")], + new_requirements=[safe_parse_requirement(requirement_string="Jinja2==2.11.3")], ) # Activate the snippetcompiler venv @@ -654,7 +654,7 @@ def test_project_requirements_dont_overwrite_core_requirements_index( module_from_template( os.path.join(modules_v2_dir, module_name), module_path, - new_requirements=[safe_parse(requirement_string="Jinja2==2.11.3")], + new_requirements=[safe_parse_requirement(requirement_string="Jinja2==2.11.3")], publish_index=index, ) @@ -708,7 +708,7 @@ def test_module_conflicting_dependencies_with_v2_modules( "y", Version("1.0.0"), str(tmpdir.join("y-1.0.0")), - requirements=[safe_parse(requirement_string="x~=1.0.0")], + requirements=[safe_parse_requirement(requirement_string="x~=1.0.0")], publish_index=index, ) @@ -718,7 +718,7 @@ def test_module_conflicting_dependencies_with_v2_modules( module_from_template( os.path.join(modules_v2_dir, module_name1), module_path1, - new_requirements=[safe_parse(requirement_string="y~=1.0.0")], + new_requirements=[safe_parse_requirement(requirement_string="y~=1.0.0")], publish_index=index, ) @@ -729,7 +729,7 @@ def test_module_conflicting_dependencies_with_v2_modules( os.path.join(modules_v2_dir, "minimalv2module"), module_path2, new_name="minimalv2module2", - new_requirements=[safe_parse(requirement_string="x~=2.0.0")], + new_requirements=[safe_parse_requirement(requirement_string="x~=2.0.0")], publish_index=index, ) @@ -789,7 +789,7 @@ def test_module_conflicting_dependencies_with_v1_module( os.path.join(modules_dir, module_name1), module_path1, new_name="modulev1", - new_requirements=[safe_parse(requirement_string="y~=1.0.0")], + new_requirements=[safe_parse_requirement(requirement_string="y~=1.0.0")], ) # Create the second module @@ -798,7 +798,7 @@ def test_module_conflicting_dependencies_with_v1_module( module_from_template( os.path.join(modules_v2_dir, module_name2), module_path2, - new_requirements=[safe_parse(requirement_string="y~=2.0.0")], + new_requirements=[safe_parse_requirement(requirement_string="y~=2.0.0")], publish_index=index, ) @@ -846,7 +846,7 @@ def test_module_install_extra_on_project_level_v2_dep( new_name="mymod", new_requirements=[], new_extras={ - "myfeature": [safe_parse(requirement_string=package_name_extra)], + "myfeature": [safe_parse_requirement(requirement_string=package_name_extra)], }, publish_index=index, ) @@ -895,7 +895,7 @@ def test_module_install_extra_on_dep_of_v2_module( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [safe_parse(requirement_string=package_name_extra)], + "myfeature": [safe_parse_requirement(requirement_string=package_name_extra)], }, publish_index=index, ) @@ -915,7 +915,7 @@ def test_module_install_extra_on_dep_of_v2_module( install_project=True, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[safe_parse(requirement_string="inmanta-module-myv2mod")], + python_requires=[safe_parse_requirement(requirement_string="inmanta-module-myv2mod")], autostd=False, ) @@ -952,7 +952,7 @@ def test_module_install_extra_on_dep_of_v1_module( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [safe_parse(requirement_string=package_name_extra)], + "myfeature": [safe_parse_requirement(requirement_string=package_name_extra)], }, publish_index=index, ) @@ -1002,7 +1002,7 @@ def test_module_install_extra_on_project_level_v2_dep_update_scenario( new_name="mymod", new_requirements=[], new_extras={ - "myfeature": [safe_parse(requirement_string=package_name_extra)], + "myfeature": [safe_parse_requirement(requirement_string=package_name_extra)], }, publish_index=index, ) @@ -1071,7 +1071,7 @@ def test_module_install_extra_on_dep_of_v2_module_update_scenario( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [safe_parse(requirement_string=package_name_extra)], + "myfeature": [safe_parse_requirement(requirement_string=package_name_extra)], }, publish_index=index, ) @@ -1100,7 +1100,7 @@ def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> install_project=True, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[safe_parse(requirement_string="inmanta-module-myv2mod==1.0.0")], + python_requires=[safe_parse_requirement(requirement_string="inmanta-module-myv2mod==1.0.0")], autostd=False, ) assert_installed(extra_installed=False) @@ -1122,7 +1122,7 @@ def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> install_project=not do_project_update, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[safe_parse(requirement_string="inmanta-module-myv2mod==2.0.0")], + python_requires=[safe_parse_requirement(requirement_string="inmanta-module-myv2mod==2.0.0")], autostd=False, ) if do_project_update: @@ -1166,7 +1166,7 @@ def test_module_install_extra_on_dep_of_v1_module_update_scenario( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [safe_parse(requirement_string=package_name_extra)], + "myfeature": [safe_parse_requirement(requirement_string=package_name_extra)], }, publish_index=index, ) @@ -1230,7 +1230,7 @@ async def test_v1_module_depends_on_third_party_dep_with_extra( os.path.join(tmpdir, "myv1mod"), new_name="myv1mod", new_content_init_cf="", - new_requirements=[safe_parse(requirement_string="pkg[optional-a]")], + new_requirements=[safe_parse_requirement(requirement_string="pkg[optional-a]")], ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv1mod", @@ -1250,7 +1250,7 @@ async def test_v1_module_depends_on_third_party_dep_with_extra( os.path.join(tmpdir, "myv1mod"), new_name="myv1mod", new_content_init_cf="", - new_requirements=[safe_parse(requirement_string="pkg[optional-a,optional-b]")], + new_requirements=[safe_parse_requirement(requirement_string="pkg[optional-a,optional-b]")], ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv1mod", @@ -1277,13 +1277,13 @@ async def test_v2_module_depends_on_third_party_dep_with_extra( str(tmpdir.join("myv2mod")), new_name="myv2mod", new_version=Version("1.0.0"), - new_requirements=[safe_parse(requirement_string="pkg[optional-a]")], + new_requirements=[safe_parse_requirement(requirement_string="pkg[optional-a]")], publish_index=index, ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv2mod", install_project=True, - python_requires=[safe_parse(requirement_string="inmanta-module-myv2mod==1.0.0")], + python_requires=[safe_parse_requirement(requirement_string="inmanta-module-myv2mod==1.0.0")], index_url=index.url, extra_index_url=[index_with_pkgs_containing_optional_deps], autostd=False, @@ -1299,13 +1299,13 @@ async def test_v2_module_depends_on_third_party_dep_with_extra( str(tmpdir.join("myv2mod")), new_name="myv2mod", new_version=Version("2.0.0"), - new_requirements=[safe_parse(requirement_string="pkg[optional-a,optional-b]")], + new_requirements=[safe_parse_requirement(requirement_string="pkg[optional-a,optional-b]")], publish_index=index, ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv2mod", install_project=True, - python_requires=[safe_parse(requirement_string="inmanta-module-myv2mod==2.0.0")], + python_requires=[safe_parse_requirement(requirement_string="inmanta-module-myv2mod==2.0.0")], index_url=index.url, extra_index_url=[index_with_pkgs_containing_optional_deps], autostd=False, From b797205dd32ae31de8e1cb50b0a458d0e6e2f81b Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Thu, 29 Aug 2024 16:56:51 +0200 Subject: [PATCH 21/74] Remove SafeRequirement --- src/inmanta/env.py | 17 ++++++++++----- src/inmanta/module.py | 6 ++++-- tests/compiler/test_basics.py | 2 +- tests/conftest.py | 7 ++++--- tests/moduletool/test_convert_v1_v2.py | 4 +++- tests/moduletool/test_install.py | 27 +++++++++++++++++------- tests/test_env.py | 6 +++--- tests/test_file_parser.py | 4 ++-- tests/test_module_loader.py | 29 +++++++++++++++++--------- tests/utils.py | 18 ++++++++-------- 10 files changed, 77 insertions(+), 43 deletions(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index b2068631a9..79deff50eb 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -90,7 +90,7 @@ def __str__(self) -> str: owner = "" if self.owner: # Cfr pip - # SafeRequirement already satisfied: certifi>=2017.4.17 in /[...]/site-packages + # Requirement already satisfied: certifi>=2017.4.17 in /[...]/site-packages # (from requests>=2.23.0->cookiecutter<3,>=1->inmanta-core==7.0.0) (2022.6.15) owner = f" (from {self.owner})" if self.installed_version: @@ -176,7 +176,7 @@ class PythonWorkingSet: @classmethod def _get_as_requirements_type(cls, requirements: req_list) -> Sequence[Requirement]: """ - Convert requirements from Union[Sequence[str], Sequence[SafeRequirement]] to Sequence[SafeRequirement] + Convert requirements from Union[Sequence[str], Sequence[Requirement]] to Sequence[Requirement] """ if isinstance(requirements[0], str): return [safe_parse_requirement(requirement=r) for r in requirements if isinstance(r, str)] @@ -210,7 +210,7 @@ def _are_installed_recursive( for r in reqs: if r in seen_requirements: continue - # Requirements created by the `Distribution.requires()` method have the extra, the SafeRequirement was created + # Requirements created by the `Distribution.requires()` method have the extra, the Requirement was created # from, # set as a marker. The line below makes sure that the "extra" marker matches. The marker is not set by # `Distribution.requires()` when the package is installed in editable mode, but setting it always doesn't make @@ -294,7 +294,10 @@ def _get_tree_recursive_single(acc: abc.Set[str], dist: str) -> abc.Set[str]: # recurse on direct dependencies return _get_tree_recursive( - (safe_parse_requirement(requirement=requirement.key).name for requirement in installed_distributions[dist].requires()), + ( + safe_parse_requirement(requirement=requirement.key).name + for requirement in installed_distributions[dist].requires() + ), acc=acc | {dist}, ) @@ -888,7 +891,11 @@ def _get_requirements_on_inmanta_package(cls) -> Sequence[Requirement]: """ protected_inmanta_packages: list[str] = cls.get_protected_inmanta_packages() workingset: dict[str, version.Version] = PythonWorkingSet.get_packages_in_working_set() - return [safe_parse_requirement(requirement=f"{pkg}=={workingset[pkg]}") for pkg in workingset if pkg in protected_inmanta_packages] + return [ + safe_parse_requirement(requirement=f"{pkg}=={workingset[pkg]}") + for pkg in workingset + if pkg in protected_inmanta_packages + ] class CommandRunner: diff --git a/src/inmanta/module.py b/src/inmanta/module.py index 0c6ce977f7..9f4e179567 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -156,7 +156,7 @@ def parse(cls: type[TInmantaModuleRequirement], spec: str) -> TInmantaModuleRequ def get_python_package_requirement(self) -> Requirement: """ - Return a SafeRequirement with the name of the Python distribution package for this module requirement. + Return a Requirement with the name of the Python distribution package for this module requirement. """ module_name = self.project_name pkg_name = ModuleV2Source.get_package_name_for(module_name) @@ -2547,7 +2547,9 @@ def verify_python_requires(self) -> None: Verifies no incompatibilities exist within the Python environment with respect to installed module v2 requirements. """ if self.strict_deps_check: - constraints: list[Requirement] = [safe_parse_requirement(requirement=item) for item in self.collect_python_requirements()] + constraints: list[Requirement] = [ + safe_parse_requirement(requirement=item) for item in self.collect_python_requirements() + ] env.ActiveEnv.check(strict_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*"), constraints=constraints) else: if not env.ActiveEnv.check_legacy(in_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*")): diff --git a/tests/compiler/test_basics.py b/tests/compiler/test_basics.py index 1ee2a21c0f..e0e50fdcf8 100644 --- a/tests/compiler/test_basics.py +++ b/tests/compiler/test_basics.py @@ -737,7 +737,7 @@ def test_implementation_import_missing_error(snippetcompiler) -> None: @pytest.mark.parametrize("name", ["", "#", " # ", "#this is a comment"]) def test_safe_requirement(name) -> None: """ - Ensure that empty name requirements are not allowed in `SafeRequirement` + Ensure that empty name requirements are not allowed in `Requirement` """ with pytest.raises(AssertionError): safe_parse_requirement(requirement=name) diff --git a/tests/conftest.py b/tests/conftest.py index 78b62218c6..221424fd8e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,6 +28,7 @@ from inmanta.logging import InmantaLoggerConfig from inmanta.protocol import auth from inmanta.util import ScheduledTask, Scheduler, TaskMethod, TaskSchedule +from packaging.requirements import Requirement """ About the use of @parametrize_any and @slowtest: @@ -119,7 +120,7 @@ from inmanta.ast import CompilerException from inmanta.data.schema import SCHEMA_VERSION_TABLE from inmanta.db import util as db_util -from inmanta.env import CommandRunner, LocalPackagePath, SafeRequirement, VirtualEnv, mock_process_env, safe_parse_requirement +from inmanta.env import CommandRunner, LocalPackagePath, VirtualEnv, mock_process_env, safe_parse_requirement from inmanta.export import ResourceDict, cfg_env, unknown_parameters from inmanta.module import InmantaModuleRequirement, InstallMode, Project, RelationPrecedenceRule from inmanta.moduletool import DefaultIsolatedEnvCached, ModuleTool, V2ModuleBuilder @@ -1153,7 +1154,7 @@ def setup_for_snippet( add_to_module_path: Optional[list[str]] = None, python_package_sources: Optional[list[str]] = None, project_requires: Optional[list[InmantaModuleRequirement]] = None, - python_requires: Optional[list[SafeRequirement]] = None, + python_requires: Optional[list[Requirement]] = None, install_mode: Optional[InstallMode] = None, relation_precedence_rules: Optional[list[RelationPrecedenceRule]] = None, strict_deps_check: Optional[bool] = None, @@ -1255,7 +1256,7 @@ def setup_for_snippet_external( add_to_module_path: Optional[list[str]] = None, python_package_sources: Optional[list[str]] = None, project_requires: Optional[list[InmantaModuleRequirement]] = None, - python_requires: Optional[list[SafeRequirement]] = None, + python_requires: Optional[list[Requirement]] = None, install_mode: Optional[InstallMode] = None, relation_precedence_rules: Optional[list[RelationPrecedenceRule]] = None, use_pip_config_file: bool = False, diff --git a/tests/moduletool/test_convert_v1_v2.py b/tests/moduletool/test_convert_v1_v2.py index a54625eeb7..1b81bc8aaf 100644 --- a/tests/moduletool/test_convert_v1_v2.py +++ b/tests/moduletool/test_convert_v1_v2.py @@ -114,7 +114,9 @@ def test_issue_3159_conversion_std_module_add_self_to_dependencies(tmpdir): parser = configparser.ConfigParser() parser.read(setup_cfg_file) assert parser.has_option("options", "install_requires") - install_requires = [safe_parse_requirement(requirement=r) for r in parser.get("options", "install_requires").split("\n") if r] + install_requires = [ + safe_parse_requirement(requirement=r) for r in parser.get("options", "install_requires").split("\n") if r + ] pkg_names = [r.name for r in install_requires] assert "inmanta-module-std" not in pkg_names diff --git a/tests/moduletool/test_install.py b/tests/moduletool/test_install.py index 896c74d104..69f09e3f8c 100644 --- a/tests/moduletool/test_install.py +++ b/tests/moduletool/test_install.py @@ -508,7 +508,10 @@ def test_project_install( index_url=local_module_package_index, # We add tornado, as there is a code path in update for the case where the project has python requires python_requires=["tornado"] - + [safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names], + + [ + safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) + for mod in install_module_names + ], install_project=False, ) @@ -541,7 +544,8 @@ def test_project_install( autostd=False, python_package_sources=[local_module_package_index], python_requires=[ - safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names + safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) + for mod in install_module_names ] + ["lorem"], install_project=False, @@ -1333,7 +1337,9 @@ def test_pip_output(local_module_package_index: str, snippetcompiler_clean, capl ) modules = ["modone", "modtwo"] - v2_requirements = [safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in modules] + v2_requirements = [ + safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in modules + ] snippetcompiler_clean.setup_for_snippet( f""" @@ -1419,7 +1425,9 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[ + safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module")) + ], install_project=True, ) log_contains( @@ -1451,7 +1459,9 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[ + safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module")) + ], install_project=True, ) @@ -1484,7 +1494,9 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[ + safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module")) + ], install_project=True, ) log_contains( @@ -1650,7 +1662,8 @@ def test_constraints_logging_v2(modules_v2_dir, tmpdir, caplog, snippetcompiler_ index_url=local_module_package_index, extra_index_url=[index.url], python_requires=[ - safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in ["module_b", "module_a"] + safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) + for mod in ["module_b", "module_a"] ], install_project=True, project_requires=[ diff --git a/tests/test_env.py b/tests/test_env.py index 0b15f1cd2e..4d89126bd2 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -35,7 +35,7 @@ from inmanta import env, loader, module from inmanta.data.model import PipConfig -from inmanta.env import Pip, SafeRequirement, safe_parse_requirement +from inmanta.env import Pip, Requirement, safe_parse_requirement from packaging import version from utils import LogSequence, PipIndex, create_python_package @@ -450,7 +450,7 @@ def test_active_env_get_module_file_editable_namespace_package( def create_install_package( - name: str, version: version.Version, requirements: list[SafeRequirement], local_module_package_index: str + name: str, version: version.Version, requirements: list[Requirement], local_module_package_index: str ) -> None: """ Creates and installs a simple package with specified requirements. Creates package in a temporary directory and @@ -561,7 +561,7 @@ def test_active_env_check_constraints(caplog, tmpvenv_active_inherit: str, local """ caplog.set_level(logging.WARNING) in_scope: Pattern[str] = re.compile("test-package-.*") - constraints: list[SafeRequirement] = [safe_parse_requirement(requirement_string="test-package-one~=1.0")] + constraints: list[Requirement] = [safe_parse_requirement(requirement_string="test-package-one~=1.0")] env.ActiveEnv.check(in_scope) diff --git a/tests/test_file_parser.py b/tests/test_file_parser.py index 1486821b96..37c855c232 100644 --- a/tests/test_file_parser.py +++ b/tests/test_file_parser.py @@ -18,7 +18,7 @@ import os -from inmanta.env import SafeRequirement, safe_parse_requirement +from inmanta.env import Requirement, safe_parse_requirement from inmanta.file_parser import RequirementsTxtParser @@ -39,7 +39,7 @@ def test_requirements_txt_parser(tmpdir) -> None: fd.write(content) expected_requirements = ["test==1.2.3", "other-dep~=2.0.0", "third-dep<5.0.0", "splitteddep", "Capital"] - requirements: list[SafeRequirement] = RequirementsTxtParser().parse(requirements_txt_file) + requirements: list[Requirement] = RequirementsTxtParser().parse(requirements_txt_file) assert requirements == [safe_parse_requirement(requirement_string=r) for r in expected_requirements] requirements_as_str = RequirementsTxtParser.parse_requirements_as_strs(requirements_txt_file) assert requirements_as_str == expected_requirements diff --git a/tests/test_module_loader.py b/tests/test_module_loader.py index 1b5eec51bd..8f3bc15fab 100644 --- a/tests/test_module_loader.py +++ b/tests/test_module_loader.py @@ -33,7 +33,14 @@ from inmanta.ast import CompilerException from inmanta.const import CF_CACHE_DIR from inmanta.data.model import PipConfig -from inmanta.env import ConflictingRequirements, LocalPackagePath, PackageNotFound, SafeRequirement, process_env, safe_parse_requirement +from inmanta.env import ( + ConflictingRequirements, + LocalPackagePath, + PackageNotFound, + Requirement, + process_env, + safe_parse_requirement, +) from inmanta.module import ( DummyProject, InmantaModuleRequirement, @@ -396,7 +403,7 @@ def test_load_import_based_v2_project(local_module_package_index: str, snippetco """ module_name: str = "minimalv2module" - def load(requires: Optional[list[SafeRequirement]] = None) -> None: + def load(requires: Optional[list[Requirement]] = None) -> None: project: Project = snippetcompiler_clean.setup_for_snippet( f"import {module_name}", autostd=False, @@ -468,7 +475,9 @@ def test_load_import_based_v2_module( extra_index_url=[index.url], # make sure that even listing the requirement in project.yml does not suffice project_requires=[InmantaModuleRequirement.parse(dependency_module_name)], - python_requires=([] if v1 else [safe_parse_requirement(requirement_string=ModuleV2Source.get_package_name_for(main_module_name))]), + python_requires=( + [] if v1 else [safe_parse_requirement(requirement_string=ModuleV2Source.get_package_name_for(main_module_name))] + ), ) if explicit_dependency: @@ -850,7 +859,7 @@ def test_module_install_extra_on_project_level_v2_dep( }, publish_index=index, ) - package_with_extra: SafeRequirement = InmantaModuleRequirement.parse("mymod[myfeature]").get_python_package_requirement() + package_with_extra: Requirement = InmantaModuleRequirement.parse("mymod[myfeature]").get_python_package_requirement() package_name: str = f"{ModuleV2.PKG_NAME_PREFIX}mymod" # project with dependency on mymod with extra @@ -1006,8 +1015,8 @@ def test_module_install_extra_on_project_level_v2_dep_update_scenario( }, publish_index=index, ) - package_without_extra: SafeRequirement = InmantaModuleRequirement.parse("mymod").get_python_package_requirement() - package_with_extra: SafeRequirement = InmantaModuleRequirement.parse("mymod[myfeature]").get_python_package_requirement() + package_without_extra: Requirement = InmantaModuleRequirement.parse("mymod").get_python_package_requirement() + package_with_extra: Requirement = InmantaModuleRequirement.parse("mymod[myfeature]").get_python_package_requirement() package_name: str = str(package_without_extra) def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> None: @@ -1075,8 +1084,8 @@ def test_module_install_extra_on_dep_of_v2_module_update_scenario( }, publish_index=index, ) - package_without_extra: SafeRequirement = InmantaModuleRequirement.parse("depmod").get_python_package_requirement() - package_with_extra: SafeRequirement = InmantaModuleRequirement.parse("depmod[myfeature]").get_python_package_requirement() + package_without_extra: Requirement = InmantaModuleRequirement.parse("depmod").get_python_package_requirement() + package_with_extra: Requirement = InmantaModuleRequirement.parse("depmod[myfeature]").get_python_package_requirement() package_name: str = str(package_without_extra) def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> None: @@ -1156,8 +1165,8 @@ def test_module_install_extra_on_dep_of_v1_module_update_scenario( index: PipIndex = PipIndex(artifact_dir=str(tmpdir.join(".index"))) # Publish dependency of V1 module (depmod) to python package repo - package_without_extra: SafeRequirement = InmantaModuleRequirement.parse("depmod").get_python_package_requirement() - package_with_extra: SafeRequirement = InmantaModuleRequirement.parse("depmod[myfeature]").get_python_package_requirement() + package_without_extra: Requirement = InmantaModuleRequirement.parse("depmod").get_python_package_requirement() + package_with_extra: Requirement = InmantaModuleRequirement.parse("depmod[myfeature]").get_python_package_requirement() package_name: str = str(package_without_extra) module_from_template( diff --git a/tests/utils.py b/tests/utils.py index 6902e478a1..c0159b48b2 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -44,7 +44,6 @@ from inmanta import config, const, data, env, module, protocol, util from inmanta.data import ResourceIdStr from inmanta.data.model import PipConfig -from inmanta.env import SafeRequirement from inmanta.moduletool import ModuleTool from inmanta.protocol import Client from inmanta.server.bootloader import InmantaBootloader @@ -52,6 +51,7 @@ from inmanta.util import get_compiler_version, hash_file from libpip2pi.commands import dir2pi from packaging import version +from packaging.requirements import Requirement from packaging.version import Version T = TypeVar("T") @@ -488,11 +488,11 @@ def create_python_package( pkg_version: version.Version, path: str, *, - requirements: Optional[Sequence[SafeRequirement]] = None, + requirements: Optional[Sequence[Requirement]] = None, install: bool = False, editable: bool = False, publish_index: Optional[PipIndex] = None, - optional_dependencies: Optional[dict[str, Sequence[SafeRequirement]]] = None, + optional_dependencies: Optional[dict[str, Sequence[Requirement]]] = None, ) -> None: """ Creates an empty Python package. @@ -577,8 +577,8 @@ def module_from_template( *, new_version: Optional[version.Version] = None, new_name: Optional[str] = None, - new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, SafeRequirement]]] = None, - new_extras: Optional[abc.Mapping[str, abc.Sequence[Union[module.InmantaModuleRequirement, SafeRequirement]]]] = None, + new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, Requirement]]] = None, + new_extras: Optional[abc.Mapping[str, abc.Sequence[Union[module.InmantaModuleRequirement, Requirement]]]] = None, install: bool = False, editable: bool = False, publish_index: Optional[PipIndex] = None, @@ -606,8 +606,8 @@ def module_from_template( :param four_digit_version: if the version uses 4 digits (3 by default) """ - def to_python_requires(requires: abc.Sequence[Union[module.InmantaModuleRequirement, SafeRequirement]]) -> list[str]: - return [str(req if isinstance(req, SafeRequirement) else str(req.get_python_package_requirement())) for req in requires] + def to_python_requires(requires: abc.Sequence[Union[module.InmantaModuleRequirement, Requirement]]) -> list[str]: + return [str(req if isinstance(req, Requirement) else str(req.get_python_package_requirement())) for req in requires] if (dest_dir is None) != in_place: raise ValueError("Either dest_dir or in_place must be set, never both.") @@ -688,7 +688,7 @@ def v1_module_from_template( *, new_version: Optional[version.Version] = None, new_name: Optional[str] = None, - new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, SafeRequirement]]] = None, + new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, Requirement]]] = None, new_content_init_cf: Optional[str] = None, new_content_init_py: Optional[str] = None, ) -> module.ModuleV2Metadata: @@ -728,7 +728,7 @@ def v1_module_from_template( with open(os.path.join(dest_dir, "requirements.txt"), "w") as fd: fd.write( "\n".join( - str(req if isinstance(req, SafeRequirement) else req.get_python_package_requirement()) + str(req if isinstance(req, Requirement) else req.get_python_package_requirement()) for req in new_requirements ) ) From 259626f3ab535f20242ba36c73437fab167821db Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Thu, 29 Aug 2024 17:06:58 +0200 Subject: [PATCH 22/74] fix parsing --- src/inmanta/env.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 79deff50eb..28fe9c45a6 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -68,8 +68,9 @@ def safe_parse_requirement(requirement: str) -> Requirement: assert len(drop_comment) > 0, "The name of the requirement cannot be an empty string!" # We canonicalize the name of the requirement to be able to compare requirements and check if the requirement is # already installed - canonicalized_requirement_name = utils.canonicalize_name(drop_comment) - return Requirement(requirement_string=canonicalized_requirement_name) + requirement = Requirement(requirement_string=drop_comment) + requirement.name = utils.canonicalize_name(requirement.name) + return requirement @dataclass(eq=True, frozen=True) From 26420b2ee7447209b381a6ad118bc9e3abb17435 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Fri, 30 Aug 2024 10:59:11 +0200 Subject: [PATCH 23/74] fix name parameter --- src/inmanta/agent/executor.py | 2 +- src/inmanta/agent/in_process_executor.py | 2 +- src/inmanta/env.py | 16 ++++---- src/inmanta/file_parser.py | 8 ++-- src/inmanta/module.py | 16 ++++---- src/inmanta/moduletool.py | 2 +- tests/compiler/test_basics.py | 2 +- tests/conftest.py | 7 +++- tests/moduletool/test_add.py | 2 +- tests/moduletool/test_convert_v1_v2.py | 2 +- tests/moduletool/test_install.py | 36 ++++++++-------- tests/moduletool/test_update.py | 10 ++--- tests/server/test_compilerservice.py | 2 +- tests/test_env.py | 32 +++++++-------- tests/test_file_parser.py | 2 +- tests/test_module_loader.py | 52 ++++++++++++------------ 16 files changed, 97 insertions(+), 96 deletions(-) diff --git a/src/inmanta/agent/executor.py b/src/inmanta/agent/executor.py index 9e09eac480..80ecfe494f 100644 --- a/src/inmanta/agent/executor.py +++ b/src/inmanta/agent/executor.py @@ -294,7 +294,7 @@ def create_and_install_environment(self, blueprint: EnvBlueprint) -> None: self.init_env() if len(req): # install_for_config expects at least 1 requirement or a path to install self.install_for_config( - requirements=[safe_parse_requirement(requirement=e) for e in req], + requirements=[safe_parse_requirement(requirement_name=e) for e in req], config=blueprint.pip_config, ) diff --git a/src/inmanta/agent/in_process_executor.py b/src/inmanta/agent/in_process_executor.py index 0911235c75..d27b21692b 100644 --- a/src/inmanta/agent/in_process_executor.py +++ b/src/inmanta/agent/in_process_executor.py @@ -571,7 +571,7 @@ async def _install(self, blueprint: executor.ExecutorBlueprint) -> None: await loop.run_in_executor( self.process.thread_pool, self._env.install_for_config, - [safe_parse_requirement(requirement=e) for e in blueprint.requirements], + [safe_parse_requirement(requirement_name=e) for e in blueprint.requirements], blueprint.pip_config, ) await loop.run_in_executor(self.process.thread_pool, self._loader.deploy_version, blueprint.sources) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 28fe9c45a6..a302002883 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -62,9 +62,9 @@ class PipInstallError(Exception): pass -def safe_parse_requirement(requirement: str) -> Requirement: +def safe_parse_requirement(requirement_name: str) -> Requirement: # Packaging Requirement is not able to parse requirements with comment. Therefore, we need to remove the `comment` part - drop_comment = requirement.split("#")[0].strip() + drop_comment = requirement_name.split("#")[0].strip() assert len(drop_comment) > 0, "The name of the requirement cannot be an empty string!" # We canonicalize the name of the requirement to be able to compare requirements and check if the requirement is # already installed @@ -180,7 +180,7 @@ def _get_as_requirements_type(cls, requirements: req_list) -> Sequence[Requireme Convert requirements from Union[Sequence[str], Sequence[Requirement]] to Sequence[Requirement] """ if isinstance(requirements[0], str): - return [safe_parse_requirement(requirement=r) for r in requirements if isinstance(r, str)] + return [safe_parse_requirement(requirement_name=r) for r in requirements if isinstance(r, str)] else: return requirements @@ -296,7 +296,7 @@ def _get_tree_recursive_single(acc: abc.Set[str], dist: str) -> abc.Set[str]: # recurse on direct dependencies return _get_tree_recursive( ( - safe_parse_requirement(requirement=requirement.key).name + safe_parse_requirement(requirement_name=requirement.key).name for requirement in installed_distributions[dist].requires() ), acc=acc | {dist}, @@ -865,7 +865,7 @@ def install_from_list( use_pip_config was ignored on ISO6 and it still is """ self.install_from_index( - requirements=[safe_parse_requirement(requirement=r) for r in requirements_list], + requirements=[safe_parse_requirement(requirement_name=r) for r in requirements_list], upgrade=upgrade, upgrade_strategy=upgrade_strategy, use_pip_config=True, @@ -893,7 +893,7 @@ def _get_requirements_on_inmanta_package(cls) -> Sequence[Requirement]: protected_inmanta_packages: list[str] = cls.get_protected_inmanta_packages() workingset: dict[str, version.Version] = PythonWorkingSet.get_packages_in_working_set() return [ - safe_parse_requirement(requirement=f"{pkg}=={workingset[pkg]}") + safe_parse_requirement(requirement_name=f"{pkg}=={workingset[pkg]}") for pkg in workingset if pkg in protected_inmanta_packages ] @@ -1024,7 +1024,7 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: # all requirements of all packages installed in this environment installed_constraints: abc.Set[OwnedRequirement] = frozenset( - OwnedRequirement(safe_parse_requirement(requirement=requirement.key), dist_info.key) + OwnedRequirement(safe_parse_requirement(requirement_name=requirement.key), dist_info.key) for dist_info in pkg_resources.working_set for requirement in dist_info.requires() ) @@ -1121,7 +1121,7 @@ def check_legacy(cls, in_scope: Pattern[str], constraints: Optional[list[Require working_set: abc.Iterable[Distribution] = importlib.metadata.distributions() # add all requirements of all in scope packages installed in this environment all_constraints: set[Requirement] = set(constraints if constraints is not None else []).union( - safe_parse_requirement(requirement=requirement) + safe_parse_requirement(requirement_name=requirement) for dist_info in working_set if in_scope.fullmatch(dist_info.name) for requirement in dist_info.requires or [] diff --git a/src/inmanta/file_parser.py b/src/inmanta/file_parser.py index 4bdebbfd11..458959fbce 100644 --- a/src/inmanta/file_parser.py +++ b/src/inmanta/file_parser.py @@ -60,7 +60,7 @@ def parse(cls, filename: str) -> list[Requirement]: """ Get all the requirements in `filename` as a list of `Requirement` instances. """ - return [safe_parse_requirement(requirement=r) for r in cls.parse_requirements_as_strs(filename)] + return [safe_parse_requirement(requirement_name=r) for r in cls.parse_requirements_as_strs(filename)] @classmethod def parse_requirements_as_strs(cls, filename: str) -> list[str]: @@ -86,7 +86,7 @@ def get_content_with_dep_removed(cls, filename: str, remove_dep_on_pkg: str) -> if not os.path.exists(filename): raise Exception(f"File {filename} doesn't exist") - removed_dependency = safe_parse_requirement(requirement=remove_dep_on_pkg) + removed_dependency = safe_parse_requirement(requirement_name=remove_dep_on_pkg) result = "" line_continuation_buffer = "" with open(filename, encoding="utf-8") as fd: @@ -94,14 +94,14 @@ def get_content_with_dep_removed(cls, filename: str, remove_dep_on_pkg: str) -> if line_continuation_buffer: line_continuation_buffer += line if not line.endswith("\\"): - if safe_parse_requirement(requirement=line_continuation_buffer).name != removed_dependency.name: + if safe_parse_requirement(requirement_name=line_continuation_buffer).name != removed_dependency.name: result += line_continuation_buffer line_continuation_buffer = "" elif not line.strip() or line.strip().startswith("#"): result += line elif line.endswith("\\"): line_continuation_buffer = line - elif safe_parse_requirement(requirement=line).name != removed_dependency.name: + elif safe_parse_requirement(requirement_name=line).name != removed_dependency.name: result += line else: # Dependency matches `remove_dep_on_pkg` => Remove line from result diff --git a/src/inmanta/module.py b/src/inmanta/module.py index 9f4e179567..451554d60c 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -152,7 +152,7 @@ def parse(cls: type[TInmantaModuleRequirement], spec: str) -> TInmantaModuleRequ ) if "-" in spec: raise ValueError("Invalid Inmanta module requirement: Inmanta module names use '_', not '-'.") - return cls(safe_parse_requirement(requirement=spec)) + return cls(safe_parse_requirement(requirement_name=spec)) def get_python_package_requirement(self) -> Requirement: """ @@ -161,7 +161,7 @@ def get_python_package_requirement(self) -> Requirement: module_name = self.project_name pkg_name = ModuleV2Source.get_package_name_for(module_name) pkg_req_str = str(self).replace(module_name, pkg_name, 1) # Replace max 1 occurrence - return safe_parse_requirement(requirement=pkg_req_str) + return safe_parse_requirement(requirement_name=pkg_req_str) class CompilerExceptionWithExtendedTrace(CompilerException): @@ -730,7 +730,7 @@ def install(self, project: "Project", module_spec: list[InmantaModuleRequirement # These could be constraints (-c) as well, but that requires additional sanitation # Because for pip not every valid -r is a valid -c current_requires = project.get_strict_python_requirements_as_list() - requirements += [safe_parse_requirement(requirement=r) for r in current_requires] + requirements += [safe_parse_requirement(requirement_name=r) for r in current_requires] if preinstalled is not None: # log warning if preinstalled version does not match constraints @@ -2129,7 +2129,7 @@ def install_modules(self, *, bypass_module_cache: bool = False, update_dependenc self.verify_module_version_compatibility() # do python install - pyreq: list[Requirement] = [safe_parse_requirement(requirement=x) for x in self.collect_python_requirements()] + pyreq: list[Requirement] = [safe_parse_requirement(requirement_name=x) for x in self.collect_python_requirements()] if len(pyreq) > 0: # upgrade both direct and transitive module dependencies: eager upgrade strategy @@ -2548,7 +2548,7 @@ def verify_python_requires(self) -> None: """ if self.strict_deps_check: constraints: list[Requirement] = [ - safe_parse_requirement(requirement=item) for item in self.collect_python_requirements() + safe_parse_requirement(requirement_name=item) for item in self.collect_python_requirements() ] env.ActiveEnv.check(strict_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*"), constraints=constraints) else: @@ -2680,7 +2680,7 @@ def requires(self) -> "List[InmantaModuleRequirement]": # filter on import stmt reqs = [] for spec in self._metadata.requires: - req = [safe_parse_requirement(requirement=spec)] + req = [safe_parse_requirement(requirement_name=spec)] if len(req) > 1: print(f"Module file for {self._path} has bad line in requirements specification {spec}") reqe = InmantaModuleRequirement(req[0]) @@ -2817,7 +2817,7 @@ def requires(self) -> "List[InmantaModuleRequirement]": """ reqs = [] for spec in self.get_module_requirements(): - req = [safe_parse_requirement(requirement=spec)] + req = [safe_parse_requirement(requirement_name=spec)] if len(req) > 1: print(f"Module file for {self._path} has bad line in requirements specification {spec}") reqe = InmantaModuleRequirement(req[0]) @@ -3433,7 +3433,7 @@ def add_module_requirement_persistent(self, requirement: InmantaModuleRequiremen new_install_requires = [ r for r in config_parser.get("options", "install_requires").split("\n") - if r and safe_parse_requirement(requirement=r).name != python_pkg_requirement.name + if r and safe_parse_requirement(requirement_name=r).name != python_pkg_requirement.name ] new_install_requires.append(str(python_pkg_requirement)) else: diff --git a/src/inmanta/moduletool.py b/src/inmanta/moduletool.py index 099a27263c..cb23480a23 100644 --- a/src/inmanta/moduletool.py +++ b/src/inmanta/moduletool.py @@ -485,7 +485,7 @@ def do_update(specs: Mapping[str, Sequence[InmantaModuleRequirement]], modules: # Because for pip not every valid -r is a valid -c current_requires = my_project.get_strict_python_requirements_as_list() env.process_env.install_for_config( - v2_python_specs + [safe_parse_requirement(requirement=r) for r in current_requires], + v2_python_specs + [safe_parse_requirement(requirement_name=r) for r in current_requires], my_project.metadata.pip, upgrade=True, ) diff --git a/tests/compiler/test_basics.py b/tests/compiler/test_basics.py index e0e50fdcf8..ae90c52680 100644 --- a/tests/compiler/test_basics.py +++ b/tests/compiler/test_basics.py @@ -740,4 +740,4 @@ def test_safe_requirement(name) -> None: Ensure that empty name requirements are not allowed in `Requirement` """ with pytest.raises(AssertionError): - safe_parse_requirement(requirement=name) + safe_parse_requirement(requirement_name=name) diff --git a/tests/conftest.py b/tests/conftest.py index 221424fd8e..082b1366aa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1919,8 +1919,11 @@ def index_with_pkgs_containing_optional_deps() -> str: path=os.path.join(tmpdirname, "pkg"), publish_index=pip_index, optional_dependencies={ - "optional-a": [safe_parse_requirement(requirement="dep-a")], - "optional-b": [safe_parse_requirement(requirement="dep-b"), safe_parse_requirement(requirement="dep-c")], + "optional-a": [safe_parse_requirement(requirement_name="dep-a")], + "optional-b": [ + safe_parse_requirement(requirement_name="dep-b"), + safe_parse_requirement(requirement_name="dep-c"), + ], }, ) for pkg_name in ["dep-a", "dep-b", "dep-c"]: diff --git a/tests/moduletool/test_add.py b/tests/moduletool/test_add.py index cb93dc2aa7..7b9a038401 100644 --- a/tests/moduletool/test_add.py +++ b/tests/moduletool/test_add.py @@ -88,7 +88,7 @@ def test_module_add_v2_module_to_project( dest_dir=os.path.join(tmpdir, f"elaboratev2module-v{version}"), new_version=Version(version), publish_index=pip_index, - new_extras={"optional": [safe_parse_requirement(requirement="inmanta-module-minimalv2module")]}, + new_extras={"optional": [safe_parse_requirement(requirement_name="inmanta-module-minimalv2module")]}, ) # Create project diff --git a/tests/moduletool/test_convert_v1_v2.py b/tests/moduletool/test_convert_v1_v2.py index 1b81bc8aaf..e5751e5dcd 100644 --- a/tests/moduletool/test_convert_v1_v2.py +++ b/tests/moduletool/test_convert_v1_v2.py @@ -115,7 +115,7 @@ def test_issue_3159_conversion_std_module_add_self_to_dependencies(tmpdir): parser.read(setup_cfg_file) assert parser.has_option("options", "install_requires") install_requires = [ - safe_parse_requirement(requirement=r) for r in parser.get("options", "install_requires").split("\n") if r + safe_parse_requirement(requirement_name=r) for r in parser.get("options", "install_requires").split("\n") if r ] pkg_names = [r.name for r in install_requires] assert "inmanta-module-std" not in pkg_names diff --git a/tests/moduletool/test_install.py b/tests/moduletool/test_install.py index 69f09e3f8c..72395a9e16 100644 --- a/tests/moduletool/test_install.py +++ b/tests/moduletool/test_install.py @@ -258,7 +258,7 @@ def test_module_install_conflicting_requirements(tmpdir: py.path.local, snippetc os.path.join(modules_v2_dir, "minimalv2module"), os.path.join(str(tmpdir), "modtwo"), new_name="modtwo", - new_requirements=[safe_parse_requirement(requirement_string="lorem~=0.1.0")], + new_requirements=[safe_parse_requirement(requirement_name="lorem~=0.1.0")], install=True, ) @@ -509,7 +509,7 @@ def test_project_install( # We add tornado, as there is a code path in update for the case where the project has python requires python_requires=["tornado"] + [ - safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) + safe_parse_requirement(requirement_name=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names ], install_project=False, @@ -544,7 +544,7 @@ def test_project_install( autostd=False, python_package_sources=[local_module_package_index], python_requires=[ - safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) + safe_parse_requirement(requirement_name=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names ] + ["lorem"], @@ -681,7 +681,7 @@ def test_project_install_modules_cache_invalid( index_url=index.url, extra_index_url=[local_module_package_index], # make sure main module gets installed, pulling in newest version of dependency module - python_requires=[safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for(main_module))], + python_requires=[safe_parse_requirement(requirement_name=module.ModuleV2Source.get_package_name_for(main_module))], ) # populate project.modules[dependency_module] to force the error conditions in this simplified example @@ -766,7 +766,7 @@ def test_project_install_incompatible_versions( install_project=False, add_to_module_path=[v1_modules_path], index_url=index.url, - python_requires=[safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for(v2_mod_name))], + python_requires=[safe_parse_requirement(requirement_name=module.ModuleV2Source.get_package_name_for(v2_mod_name))], ) # install project @@ -819,14 +819,14 @@ def test_project_install_incompatible_dependencies( v2_template_path, os.path.join(str(tmpdir), "v2mod2"), new_name="v2mod2", - new_requirements=[safe_parse_requirement(requirement_string="inmanta-module-v2mod1~=1.0.0")], + new_requirements=[safe_parse_requirement(requirement_name="inmanta-module-v2mod1~=1.0.0")], publish_index=index, ) v2mod3: module.ModuleV2Metadata = module_from_template( v2_template_path, os.path.join(str(tmpdir), "v2mod3"), new_name="v2mod3", - new_requirements=[safe_parse_requirement(requirement_string="inmanta-module-v2mod1~=2.0.0")], + new_requirements=[safe_parse_requirement(requirement_name="inmanta-module-v2mod1~=2.0.0")], publish_index=index, ) @@ -1091,7 +1091,7 @@ def test_install_with_use_config_but_PIP_CONFIG_FILE_not_set( autostd=False, install_project=False, use_pip_config_file=True, - python_requires=[safe_parse_requirement(requirement_string="inmanta-module-dummy-module")], + python_requires=[safe_parse_requirement(requirement_name="inmanta-module-dummy-module")], ) # install project @@ -1208,7 +1208,7 @@ def test_install_project_with_install_mode_master(tmpdir: py.path.local, snippet autostd=False, install_project=False, add_to_module_path=[str(tmpdir)], - project_requires=[InmantaModuleRequirement(safe_parse_requirement(requirement_string="mod11==3.2.1"))], + project_requires=[InmantaModuleRequirement(safe_parse_requirement(requirement_name="mod11==3.2.1"))], install_mode=InstallMode.master, ) @@ -1236,7 +1236,7 @@ def test_module_install_logging(local_module_package_index: str, snippetcompiler v2_module = "minimalv2module" - v2_requirements = [safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for(v2_module))] + v2_requirements = [safe_parse_requirement(requirement_name=module.ModuleV2Source.get_package_name_for(v2_module))] # set up project and modules project: module.Project = snippetcompiler_clean.setup_for_snippet( @@ -1338,7 +1338,7 @@ def test_pip_output(local_module_package_index: str, snippetcompiler_clean, capl modules = ["modone", "modtwo"] v2_requirements = [ - safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) for mod in modules + safe_parse_requirement(requirement_name=module.ModuleV2Source.get_package_name_for(mod)) for mod in modules ] snippetcompiler_clean.setup_for_snippet( @@ -1426,7 +1426,7 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil index_url=local_module_package_index, extra_index_url=[index.url], python_requires=[ - safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module")) + safe_parse_requirement(requirement_name=module.ModuleV2Source.get_package_name_for("parent_module")) ], install_project=True, ) @@ -1460,7 +1460,7 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil index_url=local_module_package_index, extra_index_url=[index.url], python_requires=[ - safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module")) + safe_parse_requirement(requirement_name=module.ModuleV2Source.get_package_name_for("parent_module")) ], install_project=True, ) @@ -1494,9 +1494,7 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[ - safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for("parent_module")) - ], + python_requires=[safe_parse_requirement(requirement_name=module.ModuleV2Source.get_package_name_for("parent_module"))], install_project=True, ) log_contains( @@ -1565,7 +1563,7 @@ def test_version_snapshot(local_module_package_index: str, snippetcompiler_clean autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for("module_b"))], + python_requires=[safe_parse_requirement(requirement_name=module.ModuleV2Source.get_package_name_for("module_b"))], install_project=True, ) @@ -1591,7 +1589,7 @@ def test_version_snapshot(local_module_package_index: str, snippetcompiler_clean autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for("module_c"))], + python_requires=[safe_parse_requirement(requirement_name=module.ModuleV2Source.get_package_name_for("module_c"))], install_project=True, ) @@ -1662,7 +1660,7 @@ def test_constraints_logging_v2(modules_v2_dir, tmpdir, caplog, snippetcompiler_ index_url=local_module_package_index, extra_index_url=[index.url], python_requires=[ - safe_parse_requirement(requirement_string=module.ModuleV2Source.get_package_name_for(mod)) + safe_parse_requirement(requirement_name=module.ModuleV2Source.get_package_name_for(mod)) for mod in ["module_b", "module_a"] ], install_project=True, diff --git a/tests/moduletool/test_update.py b/tests/moduletool/test_update.py index 826b44b5b0..8d8169cd4e 100644 --- a/tests/moduletool/test_update.py +++ b/tests/moduletool/test_update.py @@ -125,7 +125,7 @@ def assert_version_installed(module_name: str, version: str) -> None: new_version=Version(current_version), new_name=module_name, new_requirements=( - [InmantaModuleRequirement(safe_parse_requirement(requirement_string="module2<3.0.0"))] + [InmantaModuleRequirement(safe_parse_requirement(requirement_name="module2<3.0.0"))] if module_name == "module1" else None ), @@ -142,7 +142,7 @@ def assert_version_installed(module_name: str, version: str) -> None: # Add a dependency on module2, without setting an explicit version constraint. Later version of module1 # do set a version constraint on the dependency on module2. This way it is verified whether the module update # command takes into account the version constraints set in a new version of a module. - new_requirements=[InmantaModuleRequirement(safe_parse_requirement(requirement_string="module2"))], + new_requirements=[InmantaModuleRequirement(safe_parse_requirement(requirement_name="module2"))], install=False, publish_index=pip_index, new_content_init_cf="entity" if corrupt_module else None, # Introduce syntax error in the module @@ -242,7 +242,7 @@ def test_module_update_dependencies( "b", Version(v), str(tmpdir.join(f"b-{v}")), - requirements=[safe_parse_requirement(requirement_string="c")], + requirements=[safe_parse_requirement(requirement_name="c")], publish_index=index, ) for v in ("1.0.0", "2.0.0"): @@ -258,7 +258,7 @@ def test_module_update_dependencies( # install b-1.0.0 and c-1.0.0 process_env.install_for_config( - [safe_parse_requirement(requirement_string=req) for req in ("b==1.0.0", "c==1.0.0")], + [safe_parse_requirement(requirement_name=req) for req in ("b==1.0.0", "c==1.0.0")], config=PipConfig( index_url=index.url, use_system_config=False, @@ -270,7 +270,7 @@ def test_module_update_dependencies( source_dir=os.path.join(modules_dir, "minimalv1module"), dest_dir=str(tmpdir.join("modules", "my_mod")), new_name="my_mod", - new_requirements=[safe_parse_requirement(requirement_string=req) for req in ("a", "b~=1.0.0")], + new_requirements=[safe_parse_requirement(requirement_name=req) for req in ("a", "b~=1.0.0")], ) # run `inmanta project update` without running install first diff --git a/tests/server/test_compilerservice.py b/tests/server/test_compilerservice.py index 39ca6775dc..622470f474 100644 --- a/tests/server/test_compilerservice.py +++ b/tests/server/test_compilerservice.py @@ -1806,7 +1806,7 @@ def patch_get_protected_inmanta_packages(): venv = PythonEnvironment(env_path=venv_path) assert name_protected_pkg not in venv.get_installed_packages() venv.install_for_config( - requirements=[safe_parse_requirement(requirement_string=name_protected_pkg)], + requirements=[safe_parse_requirement(requirement_name=name_protected_pkg)], config=PipConfig( index_url=local_module_package_index, ), diff --git a/tests/test_env.py b/tests/test_env.py index 4d89126bd2..d358e81329 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -176,7 +176,7 @@ def test_gen_req_file(): # make sure they all parse for req in reqs: - safe_parse_requirement(requirement_string=req) + safe_parse_requirement(requirement_name=req) def test_environment_python_version_multi_digit(tmpdir: py.path.local) -> None: @@ -205,7 +205,7 @@ def test_process_env_install_from_index( package_name: str = "more-itertools" assert package_name not in env.process_env.get_installed_packages() env.process_env.install_for_config( - [safe_parse_requirement(requirement_string=package_name + (f"=={version}" if version is not None else ""))], + [safe_parse_requirement(requirement_name=package_name + (f"=={version}" if version is not None else ""))], config=PipConfig( use_system_config=True, # we need an upstream for some packages ), @@ -220,7 +220,7 @@ def test_process_env_install_from_index( # It should hit the cache there and return here. # Cheap and fast test env.process_env.install_from_index( - [safe_parse_requirement(requirement_string=package_name + (f"=={version}" if version is not None else ""))], + [safe_parse_requirement(requirement_name=package_name + (f"=={version}" if version is not None else ""))], use_pip_config=True, ) @@ -270,7 +270,7 @@ def test_process_env_install_from_index_not_found_env_var( with pytest.raises(env.PackageNotFound, match=re.escape(expected)): env.process_env.install_for_config( - [safe_parse_requirement(requirement_string="this-package-does-not-exist")], + [safe_parse_requirement(requirement_name="this-package-does-not-exist")], config=PipConfig( index_url=index_urls[0], # The first element should only be passed to the index_url. If there are indexes in the environment @@ -307,7 +307,7 @@ def test_process_env_install_no_index(tmpdir: py.path.local, monkeypatch, use_sy with pytest.raises(env.PackageNotFound, match=re.escape(expected)): env.process_env.install_for_config( - requirements=[safe_parse_requirement(requirement_string="this-package-does-not-exist")], + requirements=[safe_parse_requirement(requirement_name="this-package-does-not-exist")], paths=[env.LocalPackagePath(path=str(tmpdir))], config=PipConfig(use_system_config=use_system_config), ) @@ -324,7 +324,7 @@ def test_process_env_install_from_index_conflicting_reqs( package_name: str = "more-itertools" with pytest.raises(env.ConflictingRequirements) as e: env.process_env.install_for_config( - [safe_parse_requirement(requirement_string=f"{package_name}{version}") for version in [">8.5", "<=8"]], + [safe_parse_requirement(requirement_name=f"{package_name}{version}") for version in [">8.5", "<=8"]], config=PipConfig( use_system_config=True, # we need an upstream for some packages ), @@ -394,7 +394,7 @@ def test_active_env_get_module_file( loader.PluginModuleFinder.configure_module_finder([os.path.join(str(tmpdir), "libs")]) assert env.ActiveEnv.get_module_file(module_name) is None - env.process_env.install_for_config([safe_parse_requirement(requirement_string=package_name)], pip_config) + env.process_env.install_for_config([safe_parse_requirement(requirement_name=package_name)], pip_config) assert package_name in env.process_env.get_installed_packages() module_info: Optional[tuple[Optional[str], Loader]] = env.ActiveEnv.get_module_file(module_name) assert module_info is not None @@ -540,7 +540,7 @@ def assert_all_checks(expect_test: tuple[bool, str] = (True, ""), expect_nonext: create_install_package( "test-package-two", version.Version("1.0.0"), - [safe_parse_requirement(requirement_string="test-package-one~=1.0")], + [safe_parse_requirement(requirement_name="test-package-one~=1.0")], local_module_package_index, ) assert_all_checks() @@ -561,7 +561,7 @@ def test_active_env_check_constraints(caplog, tmpvenv_active_inherit: str, local """ caplog.set_level(logging.WARNING) in_scope: Pattern[str] = re.compile("test-package-.*") - constraints: list[Requirement] = [safe_parse_requirement(requirement_string="test-package-one~=1.0")] + constraints: list[Requirement] = [safe_parse_requirement(requirement_name="test-package-one~=1.0")] env.ActiveEnv.check(in_scope) @@ -580,7 +580,7 @@ def test_active_env_check_constraints(caplog, tmpvenv_active_inherit: str, local create_install_package( "ext-package-one", version.Version("1.0.0"), - [safe_parse_requirement(requirement_string="test-package-one==1.0")], + [safe_parse_requirement(requirement_name="test-package-one==1.0")], local_module_package_index, ) env.ActiveEnv.check(in_scope, constraints) @@ -611,7 +611,7 @@ def test_override_inmanta_package(tmpvenv_active_inherit: env.VirtualEnv) -> Non installed_pkgs = tmpvenv_active_inherit.get_installed_packages() assert "inmanta-core" in installed_pkgs, "The inmanta-core package should be installed to run the tests" - inmanta_requirements = safe_parse_requirement(requirement_string="inmanta-core==4.0.0") + inmanta_requirements = safe_parse_requirement(requirement_name="inmanta-core==4.0.0") with pytest.raises(env.ConflictingRequirements) as excinfo: tmpvenv_active_inherit.install_for_config( requirements=[inmanta_requirements], @@ -651,13 +651,13 @@ def test_cache_on_active_env(tmpvenv_active_inherit: env.ActiveEnv, local_module """ def _assert_install(requirement: str, installed: bool) -> None: - parsed_requirement = safe_parse_requirement(requirement_string=requirement) + parsed_requirement = safe_parse_requirement(requirement_name=requirement) for r in [requirement, parsed_requirement]: assert tmpvenv_active_inherit.are_installed(requirements=[r]) == installed _assert_install("inmanta-module-elaboratev2module==1.2.3", installed=False) tmpvenv_active_inherit.install_for_config( - requirements=[safe_parse_requirement(requirement_string="inmanta-module-elaboratev2module==1.2.3")], + requirements=[safe_parse_requirement(requirement_name="inmanta-module-elaboratev2module==1.2.3")], config=PipConfig( index_url=local_module_package_index, ), @@ -701,7 +701,7 @@ def test_are_installed_dependency_cycle_on_extra(tmpdir, tmpvenv_active_inherit: path=os.path.join(tmpdir, "pkg"), publish_index=pip_index, optional_dependencies={ - "optional-pkg": [safe_parse_requirement(requirement_string="dep[optional-dep]")], + "optional-pkg": [safe_parse_requirement(requirement_name="dep[optional-dep]")], }, ) create_python_package( @@ -710,11 +710,11 @@ def test_are_installed_dependency_cycle_on_extra(tmpdir, tmpvenv_active_inherit: path=os.path.join(tmpdir, "dep"), publish_index=pip_index, optional_dependencies={ - "optional-dep": [safe_parse_requirement(requirement_string="pkg[optional-pkg]")], + "optional-dep": [safe_parse_requirement(requirement_name="pkg[optional-pkg]")], }, ) - requirements = [safe_parse_requirement(requirement_string="pkg[optional-pkg]")] + requirements = [safe_parse_requirement(requirement_name="pkg[optional-pkg]")] tmpvenv_active_inherit.install_for_config( requirements=requirements, config=PipConfig( diff --git a/tests/test_file_parser.py b/tests/test_file_parser.py index 37c855c232..98cef35040 100644 --- a/tests/test_file_parser.py +++ b/tests/test_file_parser.py @@ -40,7 +40,7 @@ def test_requirements_txt_parser(tmpdir) -> None: expected_requirements = ["test==1.2.3", "other-dep~=2.0.0", "third-dep<5.0.0", "splitteddep", "Capital"] requirements: list[Requirement] = RequirementsTxtParser().parse(requirements_txt_file) - assert requirements == [safe_parse_requirement(requirement_string=r) for r in expected_requirements] + assert requirements == [safe_parse_requirement(requirement_name=r) for r in expected_requirements] requirements_as_str = RequirementsTxtParser.parse_requirements_as_strs(requirements_txt_file) assert requirements_as_str == expected_requirements diff --git a/tests/test_module_loader.py b/tests/test_module_loader.py index 8f3bc15fab..c78f935bd2 100644 --- a/tests/test_module_loader.py +++ b/tests/test_module_loader.py @@ -352,7 +352,7 @@ def test_load_module_recursive_v2_module_depends_on_v1( project = snippetcompiler.setup_for_snippet( snippet="import v2_depends_on_v1", index_url=local_module_package_index, - python_requires=[safe_parse_requirement(requirement_string="inmanta-module-v2-depends-on-v1")], + python_requires=[safe_parse_requirement(requirement_name="inmanta-module-v2-depends-on-v1")], install_project=False, ) if preload_v1_module: @@ -379,7 +379,7 @@ def test_load_module_recursive_complex_module_dependencies(local_module_package_ snippet="import complex_module_dependencies_mod1", autostd=False, index_url=local_module_package_index, - python_requires=[safe_parse_requirement(requirement_string="inmanta-module-complex-module-dependencies-mod1")], + python_requires=[safe_parse_requirement(requirement_name="inmanta-module-complex-module-dependencies-mod1")], install_project=False, ) assert "complex_module_dependencies_mod1" not in project.modules @@ -418,7 +418,7 @@ def load(requires: Optional[list[Requirement]] = None) -> None: with pytest.raises(ModuleLoadingException, match=f"Failed to load module {module_name}"): load() # assert that it doesn't raise an error with explicit requirements set - load([safe_parse_requirement(requirement_string=ModuleV2Source.get_package_name_for(module_name))]) + load([safe_parse_requirement(requirement_name=ModuleV2Source.get_package_name_for(module_name))]) @pytest.mark.parametrize("v1", [True, False]) @@ -476,7 +476,7 @@ def test_load_import_based_v2_module( # make sure that even listing the requirement in project.yml does not suffice project_requires=[InmantaModuleRequirement.parse(dependency_module_name)], python_requires=( - [] if v1 else [safe_parse_requirement(requirement_string=ModuleV2Source.get_package_name_for(main_module_name))] + [] if v1 else [safe_parse_requirement(requirement_name=ModuleV2Source.get_package_name_for(main_module_name))] ), ) @@ -619,7 +619,7 @@ def test_project_requirements_dont_overwrite_core_requirements_source( module_from_template( os.path.join(modules_v2_dir, module_name), module_path, - new_requirements=[safe_parse_requirement(requirement_string="Jinja2==2.11.3")], + new_requirements=[safe_parse_requirement(requirement_name="Jinja2==2.11.3")], ) # Activate the snippetcompiler venv @@ -663,7 +663,7 @@ def test_project_requirements_dont_overwrite_core_requirements_index( module_from_template( os.path.join(modules_v2_dir, module_name), module_path, - new_requirements=[safe_parse_requirement(requirement_string="Jinja2==2.11.3")], + new_requirements=[safe_parse_requirement(requirement_name="Jinja2==2.11.3")], publish_index=index, ) @@ -717,7 +717,7 @@ def test_module_conflicting_dependencies_with_v2_modules( "y", Version("1.0.0"), str(tmpdir.join("y-1.0.0")), - requirements=[safe_parse_requirement(requirement_string="x~=1.0.0")], + requirements=[safe_parse_requirement(requirement_name="x~=1.0.0")], publish_index=index, ) @@ -727,7 +727,7 @@ def test_module_conflicting_dependencies_with_v2_modules( module_from_template( os.path.join(modules_v2_dir, module_name1), module_path1, - new_requirements=[safe_parse_requirement(requirement_string="y~=1.0.0")], + new_requirements=[safe_parse_requirement(requirement_name="y~=1.0.0")], publish_index=index, ) @@ -738,7 +738,7 @@ def test_module_conflicting_dependencies_with_v2_modules( os.path.join(modules_v2_dir, "minimalv2module"), module_path2, new_name="minimalv2module2", - new_requirements=[safe_parse_requirement(requirement_string="x~=2.0.0")], + new_requirements=[safe_parse_requirement(requirement_name="x~=2.0.0")], publish_index=index, ) @@ -798,7 +798,7 @@ def test_module_conflicting_dependencies_with_v1_module( os.path.join(modules_dir, module_name1), module_path1, new_name="modulev1", - new_requirements=[safe_parse_requirement(requirement_string="y~=1.0.0")], + new_requirements=[safe_parse_requirement(requirement_name="y~=1.0.0")], ) # Create the second module @@ -807,7 +807,7 @@ def test_module_conflicting_dependencies_with_v1_module( module_from_template( os.path.join(modules_v2_dir, module_name2), module_path2, - new_requirements=[safe_parse_requirement(requirement_string="y~=2.0.0")], + new_requirements=[safe_parse_requirement(requirement_name="y~=2.0.0")], publish_index=index, ) @@ -855,7 +855,7 @@ def test_module_install_extra_on_project_level_v2_dep( new_name="mymod", new_requirements=[], new_extras={ - "myfeature": [safe_parse_requirement(requirement_string=package_name_extra)], + "myfeature": [safe_parse_requirement(requirement_name=package_name_extra)], }, publish_index=index, ) @@ -904,7 +904,7 @@ def test_module_install_extra_on_dep_of_v2_module( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [safe_parse_requirement(requirement_string=package_name_extra)], + "myfeature": [safe_parse_requirement(requirement_name=package_name_extra)], }, publish_index=index, ) @@ -924,7 +924,7 @@ def test_module_install_extra_on_dep_of_v2_module( install_project=True, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[safe_parse_requirement(requirement_string="inmanta-module-myv2mod")], + python_requires=[safe_parse_requirement(requirement_name="inmanta-module-myv2mod")], autostd=False, ) @@ -961,7 +961,7 @@ def test_module_install_extra_on_dep_of_v1_module( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [safe_parse_requirement(requirement_string=package_name_extra)], + "myfeature": [safe_parse_requirement(requirement_name=package_name_extra)], }, publish_index=index, ) @@ -1011,7 +1011,7 @@ def test_module_install_extra_on_project_level_v2_dep_update_scenario( new_name="mymod", new_requirements=[], new_extras={ - "myfeature": [safe_parse_requirement(requirement_string=package_name_extra)], + "myfeature": [safe_parse_requirement(requirement_name=package_name_extra)], }, publish_index=index, ) @@ -1080,7 +1080,7 @@ def test_module_install_extra_on_dep_of_v2_module_update_scenario( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [safe_parse_requirement(requirement_string=package_name_extra)], + "myfeature": [safe_parse_requirement(requirement_name=package_name_extra)], }, publish_index=index, ) @@ -1109,7 +1109,7 @@ def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> install_project=True, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[safe_parse_requirement(requirement_string="inmanta-module-myv2mod==1.0.0")], + python_requires=[safe_parse_requirement(requirement_name="inmanta-module-myv2mod==1.0.0")], autostd=False, ) assert_installed(extra_installed=False) @@ -1131,7 +1131,7 @@ def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> install_project=not do_project_update, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[safe_parse_requirement(requirement_string="inmanta-module-myv2mod==2.0.0")], + python_requires=[safe_parse_requirement(requirement_name="inmanta-module-myv2mod==2.0.0")], autostd=False, ) if do_project_update: @@ -1175,7 +1175,7 @@ def test_module_install_extra_on_dep_of_v1_module_update_scenario( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [safe_parse_requirement(requirement_string=package_name_extra)], + "myfeature": [safe_parse_requirement(requirement_name=package_name_extra)], }, publish_index=index, ) @@ -1239,7 +1239,7 @@ async def test_v1_module_depends_on_third_party_dep_with_extra( os.path.join(tmpdir, "myv1mod"), new_name="myv1mod", new_content_init_cf="", - new_requirements=[safe_parse_requirement(requirement_string="pkg[optional-a]")], + new_requirements=[safe_parse_requirement(requirement_name="pkg[optional-a]")], ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv1mod", @@ -1259,7 +1259,7 @@ async def test_v1_module_depends_on_third_party_dep_with_extra( os.path.join(tmpdir, "myv1mod"), new_name="myv1mod", new_content_init_cf="", - new_requirements=[safe_parse_requirement(requirement_string="pkg[optional-a,optional-b]")], + new_requirements=[safe_parse_requirement(requirement_name="pkg[optional-a,optional-b]")], ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv1mod", @@ -1286,13 +1286,13 @@ async def test_v2_module_depends_on_third_party_dep_with_extra( str(tmpdir.join("myv2mod")), new_name="myv2mod", new_version=Version("1.0.0"), - new_requirements=[safe_parse_requirement(requirement_string="pkg[optional-a]")], + new_requirements=[safe_parse_requirement(requirement_name="pkg[optional-a]")], publish_index=index, ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv2mod", install_project=True, - python_requires=[safe_parse_requirement(requirement_string="inmanta-module-myv2mod==1.0.0")], + python_requires=[safe_parse_requirement(requirement_name="inmanta-module-myv2mod==1.0.0")], index_url=index.url, extra_index_url=[index_with_pkgs_containing_optional_deps], autostd=False, @@ -1308,13 +1308,13 @@ async def test_v2_module_depends_on_third_party_dep_with_extra( str(tmpdir.join("myv2mod")), new_name="myv2mod", new_version=Version("2.0.0"), - new_requirements=[safe_parse_requirement(requirement_string="pkg[optional-a,optional-b]")], + new_requirements=[safe_parse_requirement(requirement_name="pkg[optional-a,optional-b]")], publish_index=index, ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv2mod", install_project=True, - python_requires=[safe_parse_requirement(requirement_string="inmanta-module-myv2mod==2.0.0")], + python_requires=[safe_parse_requirement(requirement_name="inmanta-module-myv2mod==2.0.0")], index_url=index.url, extra_index_url=[index_with_pkgs_containing_optional_deps], autostd=False, From b4435550d2e8f0e887042ce1d380504a5fae0a30 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Fri, 30 Aug 2024 17:31:48 +0200 Subject: [PATCH 24/74] refactor --- src/inmanta/agent/executor.py | 4 +- src/inmanta/agent/in_process_executor.py | 4 +- src/inmanta/env.py | 98 +++++++++++++------- src/inmanta/file_parser.py | 9 +- src/inmanta/module.py | 112 +++++++++++------------ src/inmanta/moduletool.py | 4 +- tests/compiler/test_basics.py | 2 +- tests/conftest.py | 6 +- tests/moduletool/test_add.py | 2 +- tests/moduletool/test_convert_v1_v2.py | 6 +- tests/moduletool/test_install.py | 44 ++++----- tests/moduletool/test_update.py | 12 +-- tests/server/test_compilerservice.py | 2 +- tests/test_env.py | 32 +++---- tests/test_file_parser.py | 7 +- tests/test_module_loader.py | 52 +++++------ tests/utils.py | 8 +- 17 files changed, 210 insertions(+), 194 deletions(-) diff --git a/src/inmanta/agent/executor.py b/src/inmanta/agent/executor.py index 80ecfe494f..d079edb332 100644 --- a/src/inmanta/agent/executor.py +++ b/src/inmanta/agent/executor.py @@ -41,7 +41,7 @@ from inmanta.agent import config as cfg from inmanta.agent import resourcepool from inmanta.data.model import PipConfig, ResourceIdStr, ResourceType, ResourceVersionIdStr -from inmanta.env import PythonEnvironment, safe_parse_requirement +from inmanta.env import PythonEnvironment, safe_parse_requirements from inmanta.loader import ModuleSource from inmanta.resources import Id from inmanta.types import JsonType @@ -294,7 +294,7 @@ def create_and_install_environment(self, blueprint: EnvBlueprint) -> None: self.init_env() if len(req): # install_for_config expects at least 1 requirement or a path to install self.install_for_config( - requirements=[safe_parse_requirement(requirement_name=e) for e in req], + requirements=safe_parse_requirements(req), config=blueprint.pip_config, ) diff --git a/src/inmanta/agent/in_process_executor.py b/src/inmanta/agent/in_process_executor.py index d27b21692b..0d0e927a92 100644 --- a/src/inmanta/agent/in_process_executor.py +++ b/src/inmanta/agent/in_process_executor.py @@ -33,7 +33,7 @@ from inmanta.agent.handler import HandlerAPI, SkipResource from inmanta.const import ParameterSource from inmanta.data.model import AttributeStateChange, ResourceIdStr, ResourceVersionIdStr -from inmanta.env import safe_parse_requirement +from inmanta.env import safe_parse_requirements from inmanta.loader import CodeLoader from inmanta.resources import Id, Resource from inmanta.types import Apireturn @@ -571,7 +571,7 @@ async def _install(self, blueprint: executor.ExecutorBlueprint) -> None: await loop.run_in_executor( self.process.thread_pool, self._env.install_for_config, - [safe_parse_requirement(requirement_name=e) for e in blueprint.requirements], + safe_parse_requirements(blueprint.requirements), blueprint.pip_config, ) await loop.run_in_executor(self.process.thread_pool, self._loader.deploy_version, blueprint.sources) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index a302002883..6b07aff18e 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -17,10 +17,12 @@ """ import enum +import importlib.metadata import importlib.util import json import logging import os +import pathlib import re import site import subprocess @@ -41,13 +43,14 @@ import pkg_resources +import packaging.utils +import packaging.version from inmanta import const from inmanta.ast import CompilerException from inmanta.data.model import LEGACY_PIP_DEFAULT, PipConfig from inmanta.server.bootloader import InmantaBootloader from inmanta.stable_api import stable_api from inmanta.util import strtobool -from packaging import utils, version from packaging.requirements import Requirement LOGGER = logging.getLogger(__name__) @@ -62,15 +65,49 @@ class PipInstallError(Exception): pass -def safe_parse_requirement(requirement_name: str) -> Requirement: +def safe_parse_requirement(requirement: str) -> Requirement: + """ + To be able to compare requirements, we need to make sure that every requirement's name is canonicalized otherwise issues + could arise when checking if packages are installed in a particular Venv. + + :param requirement: The requirement's name + :return: A new requirement instance + """ # Packaging Requirement is not able to parse requirements with comment. Therefore, we need to remove the `comment` part - drop_comment = requirement_name.split("#")[0].strip() - assert len(drop_comment) > 0, "The name of the requirement cannot be an empty string!" + drop_comment, _, _ = requirement.partition("#") + if len(drop_comment) == 0: + raise ValueError("The name of the requirement cannot be an empty string!") # We canonicalize the name of the requirement to be able to compare requirements and check if the requirement is # already installed - requirement = Requirement(requirement_string=drop_comment) - requirement.name = utils.canonicalize_name(requirement.name) - return requirement + requirement_instance = Requirement(requirement_string=drop_comment) + requirement_instance.name = packaging.utils.canonicalize_name(requirement_instance.name) + return requirement_instance + + +def safe_parse_requirements(requirements: Sequence[str]) -> Sequence[Requirement]: + """ + To be able to compare requirements, we need to make sure that every requirement's name is canonicalized otherwise issues + could arise when checking if packages are installed in a particular Venv. + + :param requirements: The names of the different requirements + :return: Sequence[Requirement] + """ + return [safe_parse_requirement(requirement=e) for e in requirements] + + +def safe_parse_requirements_from_file(file_path: pathlib.Path) -> Sequence[Requirement]: + """ + To be able to compare requirements, we need to make sure that every requirement's name is canonicalized otherwise issues + could arise when checking if packages are installed in a particular Venv. + + :param file_path: The path to the read the requirements from + :return: Sequence[Requirement] + """ + requirements = [] + with open(file_path) as f: + for line in f.readlines(): + requirements.append(safe_parse_requirement(line)) + return requirements @dataclass(eq=True, frozen=True) @@ -84,7 +121,7 @@ class VersionConflict: """ requirement: Requirement - installed_version: Optional[version.Version] = None + installed_version: Optional[packaging.version.Version] = None owner: Optional[str] = None def __str__(self) -> str: @@ -170,8 +207,6 @@ def get_advice(self) -> Optional[str]: req_list = TypeVar("req_list", Sequence[str], Sequence[Requirement]) -import importlib.metadata - class PythonWorkingSet: @classmethod @@ -180,7 +215,7 @@ def _get_as_requirements_type(cls, requirements: req_list) -> Sequence[Requireme Convert requirements from Union[Sequence[str], Sequence[Requirement]] to Sequence[Requirement] """ if isinstance(requirements[0], str): - return [safe_parse_requirement(requirement_name=r) for r in requirements if isinstance(r, str)] + return safe_parse_requirement(requirements) else: return requirements @@ -191,7 +226,7 @@ def are_installed(cls, requirements: req_list) -> bool: """ if not requirements: return True - installed_packages: dict[str, version.Version] = cls.get_packages_in_working_set() + installed_packages: dict[str, packaging.version.Version] = cls.get_packages_in_working_set() def _are_installed_recursive( reqs: Sequence[Requirement], @@ -220,11 +255,7 @@ def _are_installed_recursive( if r.marker and not r.marker.evaluate(environment=environment_marker_evaluation): # The marker of the requirement doesn't apply on this environment continue - if ( - r.name not in installed_packages - or (str(installed_packages[r.name]) not in r.specifier) - # If no specifiers are provided, the `in` operation will return `False` - ): + if r.name not in installed_packages or (installed_packages[r.name] not in r.specifier): return False if r.extras: for extra in r.extras: @@ -249,14 +280,14 @@ def _are_installed_recursive( return _are_installed_recursive(reqs_as_requirements, seen_requirements=[]) @classmethod - def get_packages_in_working_set(cls, inmanta_modules_only: bool = False) -> dict[str, version.Version]: + def get_packages_in_working_set(cls, inmanta_modules_only: bool = False) -> dict[str, packaging.version.Version]: """ Return all packages present in `pkg_resources.working_set` together with the version of the package. :param inmanta_modules_only: Only return inmanta modules from the working set """ return { - safe_parse_requirement(dist_info.key).name: version.Version(dist_info.version) + packaging.utils.canonicalize_name(dist_info.key): packaging.version.Version(dist_info.version) for dist_info in pkg_resources.working_set if not inmanta_modules_only or dist_info.key.startswith(const.MODULE_PKG_NAME_PREFIX) } @@ -296,7 +327,7 @@ def _get_tree_recursive_single(acc: abc.Set[str], dist: str) -> abc.Set[str]: # recurse on direct dependencies return _get_tree_recursive( ( - safe_parse_requirement(requirement_name=requirement.key).name + packaging.utils.canonicalize_name(requirement.key) for requirement in installed_distributions[dist].requires() ), acc=acc | {dist}, @@ -744,7 +775,7 @@ def _write_pth_file(self) -> None: with open(self._path_pth_file, "w", encoding="utf-8") as fd: fd.write(script_as_oneliner) - def get_installed_packages(self, only_editable: bool = False) -> dict[str, version.Version]: + def get_installed_packages(self, only_editable: bool = False) -> dict[str, packaging.version.Version]: """ Return a list of all installed packages in the site-packages of a python interpreter. @@ -753,7 +784,7 @@ def get_installed_packages(self, only_editable: bool = False) -> dict[str, versi """ cmd = PipCommandBuilder.compose_list_command(self.python_path, format=PipListFormat.json, only_editable=only_editable) output = CommandRunner(LOGGER_PIP).run_command_and_log_output(cmd, stderr=subprocess.DEVNULL, env=os.environ.copy()) - return {r["name"]: version.Version(r["version"]) for r in json.loads(output)} + return {r["name"]: packaging.version.Version(r["version"]) for r in json.loads(output)} def install_for_config( self, @@ -865,7 +896,7 @@ def install_from_list( use_pip_config was ignored on ISO6 and it still is """ self.install_from_index( - requirements=[safe_parse_requirement(requirement_name=r) for r in requirements_list], + requirements=safe_parse_requirements(requirements_list), upgrade=upgrade, upgrade_strategy=upgrade_strategy, use_pip_config=True, @@ -891,9 +922,9 @@ def _get_requirements_on_inmanta_package(cls) -> Sequence[Requirement]: to make sure that no Inmanta packages gets overridden. """ protected_inmanta_packages: list[str] = cls.get_protected_inmanta_packages() - workingset: dict[str, version.Version] = PythonWorkingSet.get_packages_in_working_set() + workingset: dict[str, packaging.version.Version] = PythonWorkingSet.get_packages_in_working_set() return [ - safe_parse_requirement(requirement_name=f"{pkg}=={workingset[pkg]}") + safe_parse_requirement(requirement=f"{pkg}=={workingset[pkg]}") for pkg in workingset if pkg in protected_inmanta_packages ] @@ -1024,7 +1055,7 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: # all requirements of all packages installed in this environment installed_constraints: abc.Set[OwnedRequirement] = frozenset( - OwnedRequirement(safe_parse_requirement(requirement_name=requirement.key), dist_info.key) + OwnedRequirement(safe_parse_requirement(requirement=requirement.key), dist_info.key) for dist_info in pkg_resources.working_set for requirement in dist_info.requires() ) @@ -1051,16 +1082,15 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: ) full_strict_scope: abc.Set[str] = PythonWorkingSet.get_dependency_tree(parameters) - installed_versions: dict[str, version.Version] = PythonWorkingSet.get_packages_in_working_set() + installed_versions: dict[str, packaging.version.Version] = PythonWorkingSet.get_packages_in_working_set() constraint_violations: set[VersionConflict] = set() constraint_violations_strict: set[VersionConflict] = set() for c in all_constraints: requirement = c.requirement - if ( - requirement.name not in installed_versions - or (len(requirement.specifier) > 0 and str(installed_versions[requirement.name]) not in requirement.specifier) - ) and (not requirement.marker or (requirement.marker and requirement.marker.evaluate())): + if (installed_versions[requirement.name] not in requirement.specifier) and ( + not requirement.marker or (requirement.marker and requirement.marker.evaluate()) + ): version_conflict = VersionConflict( requirement=requirement, installed_version=installed_versions.get(requirement.name, None), @@ -1121,17 +1151,17 @@ def check_legacy(cls, in_scope: Pattern[str], constraints: Optional[list[Require working_set: abc.Iterable[Distribution] = importlib.metadata.distributions() # add all requirements of all in scope packages installed in this environment all_constraints: set[Requirement] = set(constraints if constraints is not None else []).union( - safe_parse_requirement(requirement_name=requirement) + safe_parse_requirement(requirement=requirement) for dist_info in working_set if in_scope.fullmatch(dist_info.name) for requirement in dist_info.requires or [] ) - installed_versions: dict[str, version.Version] = PythonWorkingSet.get_packages_in_working_set() + installed_versions: dict[str, packaging.version.Version] = PythonWorkingSet.get_packages_in_working_set() constraint_violations: set[VersionConflict] = { VersionConflict(constraint, installed_versions.get(constraint.name, None)) for constraint in all_constraints - if constraint.name not in installed_versions or str(installed_versions[constraint.name]) not in constraint.specifier + if constraint.name not in installed_versions or installed_versions[constraint.name] not in constraint.specifier } all_violations = constraint_violations_non_strict | constraint_violations_strict | constraint_violations diff --git a/src/inmanta/file_parser.py b/src/inmanta/file_parser.py index 458959fbce..9d94309890 100644 --- a/src/inmanta/file_parser.py +++ b/src/inmanta/file_parser.py @@ -18,6 +18,7 @@ import os +import packaging.utils from inmanta.env import safe_parse_requirement from packaging.requirements import Requirement from ruamel.yaml import YAML @@ -60,7 +61,7 @@ def parse(cls, filename: str) -> list[Requirement]: """ Get all the requirements in `filename` as a list of `Requirement` instances. """ - return [safe_parse_requirement(requirement_name=r) for r in cls.parse_requirements_as_strs(filename)] + return [safe_parse_requirement(requirement=r) for r in cls.parse_requirements_as_strs(filename)] @classmethod def parse_requirements_as_strs(cls, filename: str) -> list[str]: @@ -86,7 +87,7 @@ def get_content_with_dep_removed(cls, filename: str, remove_dep_on_pkg: str) -> if not os.path.exists(filename): raise Exception(f"File {filename} doesn't exist") - removed_dependency = safe_parse_requirement(requirement_name=remove_dep_on_pkg) + removed_dependency = packaging.utils.canonicalize_name(remove_dep_on_pkg) result = "" line_continuation_buffer = "" with open(filename, encoding="utf-8") as fd: @@ -94,14 +95,14 @@ def get_content_with_dep_removed(cls, filename: str, remove_dep_on_pkg: str) -> if line_continuation_buffer: line_continuation_buffer += line if not line.endswith("\\"): - if safe_parse_requirement(requirement_name=line_continuation_buffer).name != removed_dependency.name: + if safe_parse_requirement(requirement=line_continuation_buffer).name != removed_dependency: result += line_continuation_buffer line_continuation_buffer = "" elif not line.strip() or line.strip().startswith("#"): result += line elif line.endswith("\\"): line_continuation_buffer = line - elif safe_parse_requirement(requirement_name=line).name != removed_dependency.name: + elif packaging.utils.canonicalize_name(line) != removed_dependency: result += line else: # Dependency matches `remove_dep_on_pkg` => Remove line from result diff --git a/src/inmanta/module.py b/src/inmanta/module.py index 451554d60c..155c5e73bb 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -39,7 +39,7 @@ from enum import Enum from functools import reduce from importlib.abc import Loader -from importlib.metadata import Distribution, PackageNotFoundError +from importlib.metadata import PackageNotFoundError from io import BytesIO, TextIOBase from itertools import chain from subprocess import CalledProcessError @@ -59,14 +59,13 @@ from inmanta.ast.blocks import BasicBlock from inmanta.ast.statements import BiStatement, DefinitionStatement, DynamicStatement, Statement from inmanta.ast.statements.define import DefineImport -from inmanta.env import assert_pip_has_source, safe_parse_requirement +from inmanta.env import assert_pip_has_source, safe_parse_requirement, safe_parse_requirements from inmanta.file_parser import PreservativeYamlParser, RequirementsTxtParser from inmanta.parser import plyInmantaParser from inmanta.parser.plyInmantaParser import cache_manager from inmanta.stable_api import stable_api from inmanta.util import get_compiler_version from inmanta.warnings import InmantaWarning -from packaging import version from packaging.requirements import Requirement from packaging.specifiers import SpecifierSet from packaging.version import Version @@ -125,7 +124,7 @@ def __eq__(self, other: object) -> bool: return NotImplemented return self._requirement == other._requirement - def __contains__(self, version: str) -> bool: + def __contains__(self, version: packaging.version.Version | str) -> bool: return version in self._requirement.specifier def __str__(self) -> str: @@ -152,7 +151,7 @@ def parse(cls: type[TInmantaModuleRequirement], spec: str) -> TInmantaModuleRequ ) if "-" in spec: raise ValueError("Invalid Inmanta module requirement: Inmanta module names use '_', not '-'.") - return cls(safe_parse_requirement(requirement_name=spec)) + return cls(safe_parse_requirement(requirement=spec)) def get_python_package_requirement(self) -> Requirement: """ @@ -161,7 +160,7 @@ def get_python_package_requirement(self) -> Requirement: module_name = self.project_name pkg_name = ModuleV2Source.get_package_name_for(module_name) pkg_req_str = str(self).replace(module_name, pkg_name, 1) # Replace max 1 occurrence - return safe_parse_requirement(requirement_name=pkg_req_str) + return safe_parse_requirement(requirement=pkg_req_str) class CompilerExceptionWithExtendedTrace(CompilerException): @@ -343,7 +342,7 @@ def get_all_tags(self, repo: str) -> list[str]: pass @abstractmethod - def get_version_tags(self, repo: str, only_return_stable_versions: bool = False) -> list[version.Version]: + def get_version_tags(self, repo: str, only_return_stable_versions: bool = False) -> list[packaging.version.Version]: pass @abstractmethod @@ -424,7 +423,7 @@ def status(self, repo: str, untracked_files_mode: Optional[UntrackedFilesMode] = def get_all_tags(self, repo: str) -> list[str]: return subprocess.check_output(["git", "tag"], cwd=repo).decode("utf-8").splitlines() - def get_version_tags(self, repo: str, only_return_stable_versions: bool = False) -> list[version.Version]: + def get_version_tags(self, repo: str, only_return_stable_versions: bool = False) -> list[packaging.version.Version]: """ Return the Git tags that represent version numbers as version.Version objects. Only PEP440 compliant versions will be returned. @@ -436,8 +435,8 @@ def get_version_tags(self, repo: str, only_return_stable_versions: bool = False) all_tags: list[str] = sorted(self.get_all_tags(repo)) for tag in all_tags: try: - parsed_version: version.Version = version.Version(tag) - except version.InvalidVersion: + parsed_version: packaging.version.Version = packaging.version.Version(tag) + except packaging.version.InvalidVersion: continue if not only_return_stable_versions or not parsed_version.is_prerelease: result.append(parsed_version) @@ -613,18 +612,21 @@ def log_pre_install_information(self, module_name: str, module_spec: list[Inmant """ raise NotImplementedError("Abstract method") - def _log_version_snapshot(self, header: Optional[str], version_snapshot: dict[str, version.Version]) -> None: + def _log_version_snapshot(self, header: Optional[str], version_snapshot: dict[str, packaging.version.Version]) -> None: if version_snapshot: out = [header] if header is not None else [] out.extend(f"{mod}: {version}" for mod, version in version_snapshot.items()) LOGGER.debug("\n".join(out)) def _log_snapshot_difference( - self, version_snapshot: dict[str, version.Version], previous_snapshot: dict[str, version.Version], header: Optional[str] + self, + version_snapshot: dict[str, packaging.version.Version], + previous_snapshot: dict[str, packaging.version.Version], + header: Optional[str], ) -> None: - set_pre_install: set[tuple[str, version.Version]] = set(previous_snapshot.items()) - set_post_install: set[tuple[str, version.Version]] = set(version_snapshot.items()) - updates_and_additions: set[tuple[str, version.Version]] = set_post_install - set_pre_install + set_pre_install: set[tuple[str, packaging.version.Version]] = set(previous_snapshot.items()) + set_post_install: set[tuple[str, packaging.version.Version]] = set(version_snapshot.items()) + updates_and_additions: set[tuple[str, packaging.version.Version]] = set_post_install - set_pre_install if version_snapshot: out = [header] if header is not None else [] @@ -685,18 +687,14 @@ def __init__(self, urls: Sequence[str] = []) -> None: self.urls: list[str] = [url if not os.path.exists(url) else os.path.abspath(url) for url in urls] @classmethod - def get_installed_version(cls, module_name: str) -> Optional[version.Version]: + def get_installed_version(cls, module_name: str) -> Optional[packaging.version.Version]: """ Returns the version for a module if it is installed. """ if module_name.startswith(ModuleV2.PKG_NAME_PREFIX): raise ValueError("PythonRepo instances work with inmanta module names, not Python package names.") try: - dist: Distribution = Distribution.from_name(ModuleV2Source.get_package_name_for(module_name)) - try: - return version.Version(dist.version) - except version.InvalidVersion: - raise InvalidModuleException(f"Package {dist.name} was installed but it has no valid version.") + return packaging.version.Version(importlib.metadata.version(ModuleV2Source.get_package_name_for(module_name))) except PackageNotFoundError: return None @@ -730,7 +728,7 @@ def install(self, project: "Project", module_spec: list[InmantaModuleRequirement # These could be constraints (-c) as well, but that requires additional sanitation # Because for pip not every valid -r is a valid -c current_requires = project.get_strict_python_requirements_as_list() - requirements += [safe_parse_requirement(requirement_name=r) for r in current_requires] + requirements += safe_parse_requirements(current_requires) if preinstalled is not None: # log warning if preinstalled version does not match constraints @@ -763,7 +761,7 @@ def install(self, project: "Project", module_spec: list[InmantaModuleRequirement def log_pre_install_information(self, module_name: str, module_spec: list[InmantaModuleRequirement]) -> None: LOGGER.debug("Installing module %s (v2) %s.", module_name, super()._format_constraints(module_name, module_spec)) - def take_v2_modules_snapshot(self, header: Optional[str] = None) -> dict[str, version.Version]: + def take_v2_modules_snapshot(self, header: Optional[str] = None) -> dict[str, packaging.version.Version]: """ Log and return a dictionary containing currently installed v2 modules and their versions. @@ -775,7 +773,7 @@ def take_v2_modules_snapshot(self, header: Optional[str] = None) -> dict[str, ve return version_snapshot def log_snapshot_difference_v2_modules( - self, previous_snapshot: dict[str, version.Version], header: Optional[str] = None + self, previous_snapshot: dict[str, packaging.version.Version], header: Optional[str] = None ) -> None: """ Logs a diff view of v2 inmanta modules currently installed (in alphabetical order) and their version. @@ -795,7 +793,7 @@ def log_post_install_information(self, module_name: str) -> None: :param module_name: The module's name. """ - installed_version: Optional[version.Version] = self.get_installed_version(module_name) + installed_version: Optional[packaging.version.Version] = self.get_installed_version(module_name) LOGGER.debug("Successfully installed module %s (v2) version %s", module_name, installed_version) def path_for(self, name: str) -> Optional[str]: @@ -860,7 +858,7 @@ def __init__(self, local_repo: "ModuleRepo", remote_repo: "ModuleRepo") -> None: def log_pre_install_information(self, module_name: str, module_spec: list[InmantaModuleRequirement]) -> None: LOGGER.debug("Installing module %s (v1) %s.", module_name, super()._format_constraints(module_name, module_spec)) - def take_modules_snapshot(self, project: "Project", header: Optional[str] = None) -> dict[str, version.Version]: + def take_modules_snapshot(self, project: "Project", header: Optional[str] = None) -> dict[str, packaging.version.Version]: """ Log and return a dictionary containing currently loaded modules and their versions. @@ -872,7 +870,7 @@ def take_modules_snapshot(self, project: "Project", header: Optional[str] = None return version_snapshot def log_snapshot_difference_v1_modules( - self, project: "Project", previous_snapshot: dict[str, version.Version], header: Optional[str] = None + self, project: "Project", previous_snapshot: dict[str, packaging.version.Version], header: Optional[str] = None ) -> None: """ Logs a diff view on inmanta modules (both v1 and v2) currently loaded (in alphabetical order) and their version. @@ -1178,7 +1176,7 @@ def has_requirement_for(self, pkg_name: str) -> bool: Returns True iff this requirements.txt file contains the given package name. The given `pkg_name` is matched case insensitive against the requirements in this RequirementsTxtFile. """ - return any(r.name == pkg_name for r in self._requirements) + return any(r.name == pkg_name.lower() for r in self._requirements) def set_requirement_and_write(self, requirement: Requirement) -> None: """ @@ -1275,8 +1273,8 @@ class ModuleMetadata(ABC, Metadata): @classmethod def is_pep440_version(cls, v: str) -> str: try: - version.Version(v) - except version.InvalidVersion as e: + packaging.version.Version(v) + except packaging.version.InvalidVersion as e: raise ValueError(f"Version {v} is not PEP440 compliant") from e return v @@ -1327,9 +1325,9 @@ def get_full_version(self) -> packaging.version.Version: @classmethod def _compose_full_version(cls, v: str, version_tag: str) -> packaging.version.Version: if not version_tag: - return version.Version(v) + return packaging.version.Version(v) normalized_tag: str = version_tag.lstrip(".") - return version.Version(f"{v}.{normalized_tag}") + return packaging.version.Version(f"{v}.{normalized_tag}") @stable_api @@ -1363,11 +1361,11 @@ def is_pep440_version_v1(cls, v: str) -> str: @classmethod def _substitute_version(cls: type[TModuleMetadata], source: str, new_version: str, version_tag: str = "") -> str: - new_version_obj: version.Version = cls._compose_full_version(new_version, version_tag) + new_version_obj: packaging.version.Version = cls._compose_full_version(new_version, version_tag) return re.sub(r"([\s]version\s*:\s*['\"\s]?)[^\"'}\s]+(['\"]?)", rf"\g<1>{new_version_obj}\g<2>", source) def get_full_version(self) -> packaging.version.Version: - return version.Version(self.version) + return packaging.version.Version(self.version) def to_v2(self) -> "ModuleV2Metadata": values = self.dict() @@ -1377,7 +1375,7 @@ def to_v2(self) -> "ModuleV2Metadata": install_requires = [ModuleV2Source.get_package_name_for(r) for r in values["requires"]] del values["requires"] values["name"] = ModuleV2Source.get_package_name_for(values["name"]) - values["version"], values["version_tag"] = ModuleV2Metadata.split_version(version.Version(values["version"])) + values["version"], values["version_tag"] = ModuleV2Metadata.split_version(packaging.version.Version(values["version"])) return ModuleV2Metadata(**values, install_requires=install_requires) @@ -1413,7 +1411,7 @@ class ModuleV2Metadata(ModuleMetadata): @field_validator("version") @classmethod def is_base_version(cls, v: str) -> str: - version_obj: version.Version = version.Version(v) + version_obj: packaging.version.Version = packaging.version.Version(v) if str(version_obj) != version_obj.base_version: raise ValueError( "setup.cfg version should be a base version without tag. Use egg_info.tag_build to configure a tag" @@ -1444,7 +1442,7 @@ def get_version_tag(v: packaging.version.Version) -> str: def is_valid_version_tag(cls, v: str) -> str: try: cls._compose_full_version("1.0.0", v) - except version.InvalidVersion as e: + except packaging.version.InvalidVersion as e: raise ValueError(f"Version tag {v} is not PEP440 compliant") from e return v @@ -2129,7 +2127,7 @@ def install_modules(self, *, bypass_module_cache: bool = False, update_dependenc self.verify_module_version_compatibility() # do python install - pyreq: list[Requirement] = [safe_parse_requirement(requirement_name=x) for x in self.collect_python_requirements()] + pyreq: list[Requirement] = safe_parse_requirement(self.collect_python_requirements()) if len(pyreq) > 0: # upgrade both direct and transitive module dependencies: eager upgrade strategy @@ -2547,9 +2545,7 @@ def verify_python_requires(self) -> None: Verifies no incompatibilities exist within the Python environment with respect to installed module v2 requirements. """ if self.strict_deps_check: - constraints: list[Requirement] = [ - safe_parse_requirement(requirement_name=item) for item in self.collect_python_requirements() - ] + constraints: list[Requirement] = [safe_parse_requirements(self.collect_python_requirements())] env.ActiveEnv.check(strict_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*"), constraints=constraints) else: if not env.ActiveEnv.check_legacy(in_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*")): @@ -2680,7 +2676,7 @@ def requires(self) -> "List[InmantaModuleRequirement]": # filter on import stmt reqs = [] for spec in self._metadata.requires: - req = [safe_parse_requirement(requirement_name=spec)] + req = [safe_parse_requirement(requirement=spec)] if len(req) > 1: print(f"Module file for {self._path} has bad line in requirements specification {spec}") reqe = InmantaModuleRequirement(req[0]) @@ -2817,7 +2813,7 @@ def requires(self) -> "List[InmantaModuleRequirement]": """ reqs = [] for spec in self.get_module_requirements(): - req = [safe_parse_requirement(requirement_name=spec)] + req = [safe_parse_requirement(requirement=spec)] if len(req) > 1: print(f"Module file for {self._path} has bad line in requirements specification {spec}") reqe = InmantaModuleRequirement(req[0]) @@ -2845,12 +2841,12 @@ def rewrite_version(self, new_version: str, version_tag: str = "") -> None: fd.write(new_module_def) self._metadata = new_metadata - def get_version(self) -> version.Version: + def get_version(self) -> packaging.version.Version: """ Return the version of this module. This is the actually installed version, which might differ from the version declared in its metadata (e.g. by a .dev0 tag). """ - return version.Version(self._metadata.version) + return packaging.version.Version(self._metadata.version) version = property(get_version) @@ -2905,7 +2901,7 @@ def get_freeze(self, submodule: str, recursive: bool = False, mode: str = ">=") if impor not in out: v1_mode: bool = self.GENERATION == ModuleGeneration.V1 mainmod = self._project.get_module(impor, install_v1=v1_mode, allow_v1=v1_mode) - vers: version.Version = mainmod.version + vers: packaging.version.Version = mainmod.version # track submodules for cycle avoidance out[impor] = mode + " " + str(vers) if recursive: @@ -3204,16 +3200,16 @@ def update( @classmethod def get_suitable_version_for( cls, modulename: str, requirements: Iterable[InmantaModuleRequirement], path: str, release_only: bool = True - ) -> Optional[version.Version]: + ) -> Optional[packaging.version.Version]: versions_str = gitprovider.get_all_tags(path) - def try_parse(x: str) -> Optional[version.Version]: + def try_parse(x: str) -> Optional[packaging.version.Version]: try: return Version(version=x) except Exception: return None - versions: list[version.Version] = [x for x in [try_parse(v) for v in versions_str] if x is not None] + versions: list[packaging.version.Version] = [x for x in [try_parse(v) for v in versions_str] if x is not None] versions = sorted(versions, reverse=True) for r in requirements: @@ -3225,9 +3221,9 @@ def try_parse(x: str) -> Optional[version.Version]: @classmethod def __best_for_compiler_version( - cls, modulename: str, versions: list[version.Version], path: str, comp_version: version.Version - ) -> Optional[version.Version]: - def get_cv_for(best: version.Version) -> Optional[version.Version]: + cls, modulename: str, versions: list[packaging.version.Version], path: str, comp_version: packaging.version.Version + ) -> Optional[packaging.version.Version]: + def get_cv_for(best: packaging.version.Version) -> Optional[packaging.version.Version]: cfg_text: str = gitprovider.get_file_for_version(path, str(best), cls.MODULE_FILE) metadata: ModuleV1Metadata = cls.get_metadata_file_schema_type().parse(cfg_text) if metadata.compiler_version is None: @@ -3295,13 +3291,13 @@ def add_module_requirement_persistent(self, requirement: InmantaModuleRequiremen # Remove requirement from module.yml file self.remove_module_requirement_from_requires_and_write(requirement.key) - def versions(self) -> list[version.Version]: + def versions(self) -> list[packaging.version.Version]: """ Provide a list of all versions available in the repository """ versions_str: list[str] = gitprovider.get_all_tags(self._path) - def try_parse(x: str) -> Optional[version.Version]: + def try_parse(x: str) -> Optional[packaging.version.Version]: try: return Version(version=x) except Exception: @@ -3359,10 +3355,10 @@ def __init__( project: Optional[Project], path: str, is_editable_install: bool = False, - installed_version: Optional[version.Version] = None, + installed_version: Optional[packaging.version.Version] = None, ) -> None: self._is_editable_install = is_editable_install - self._version: Optional[version.Version] = installed_version + self._version: Optional[packaging.version.Version] = installed_version super().__init__(project, path) if not os.path.exists(os.path.join(self.model_dir, "_init.cf")): @@ -3381,7 +3377,7 @@ def from_path(cls: type[TModule], path: str) -> Optional[TModule]: # setup.cfg is a generic Python config file: if the metadata does not match an inmanta module's, return None return None - def get_version(self) -> version.Version: + def get_version(self) -> packaging.version.Version: return self._version if self._version is not None else self._metadata.get_full_version() version = property(get_version) @@ -3433,7 +3429,7 @@ def add_module_requirement_persistent(self, requirement: InmantaModuleRequiremen new_install_requires = [ r for r in config_parser.get("options", "install_requires").split("\n") - if r and safe_parse_requirement(requirement_name=r).name != python_pkg_requirement.name + if r and safe_parse_requirement(requirement=r).name != python_pkg_requirement.name ] new_install_requires.append(str(python_pkg_requirement)) else: diff --git a/src/inmanta/moduletool.py b/src/inmanta/moduletool.py index cb23480a23..fe8f050580 100644 --- a/src/inmanta/moduletool.py +++ b/src/inmanta/moduletool.py @@ -55,7 +55,7 @@ from inmanta.ast import CompilerException from inmanta.command import CLIException, ShowUsageException from inmanta.const import CF_CACHE_DIR, MAX_UPDATE_ATTEMPT -from inmanta.env import safe_parse_requirement +from inmanta.env import safe_parse_requirements from inmanta.module import ( DummyProject, FreezeOperator, @@ -485,7 +485,7 @@ def do_update(specs: Mapping[str, Sequence[InmantaModuleRequirement]], modules: # Because for pip not every valid -r is a valid -c current_requires = my_project.get_strict_python_requirements_as_list() env.process_env.install_for_config( - v2_python_specs + [safe_parse_requirement(requirement_name=r) for r in current_requires], + v2_python_specs + safe_parse_requirements(current_requires), my_project.metadata.pip, upgrade=True, ) diff --git a/tests/compiler/test_basics.py b/tests/compiler/test_basics.py index ae90c52680..e0e50fdcf8 100644 --- a/tests/compiler/test_basics.py +++ b/tests/compiler/test_basics.py @@ -740,4 +740,4 @@ def test_safe_requirement(name) -> None: Ensure that empty name requirements are not allowed in `Requirement` """ with pytest.raises(AssertionError): - safe_parse_requirement(requirement_name=name) + safe_parse_requirement(requirement=name) diff --git a/tests/conftest.py b/tests/conftest.py index 082b1366aa..390e2f58f0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1919,10 +1919,10 @@ def index_with_pkgs_containing_optional_deps() -> str: path=os.path.join(tmpdirname, "pkg"), publish_index=pip_index, optional_dependencies={ - "optional-a": [safe_parse_requirement(requirement_name="dep-a")], + "optional-a": [safe_parse_requirement(requirement="dep-a")], "optional-b": [ - safe_parse_requirement(requirement_name="dep-b"), - safe_parse_requirement(requirement_name="dep-c"), + safe_parse_requirement(requirement="dep-b"), + safe_parse_requirement(requirement="dep-c"), ], }, ) diff --git a/tests/moduletool/test_add.py b/tests/moduletool/test_add.py index 7b9a038401..cb93dc2aa7 100644 --- a/tests/moduletool/test_add.py +++ b/tests/moduletool/test_add.py @@ -88,7 +88,7 @@ def test_module_add_v2_module_to_project( dest_dir=os.path.join(tmpdir, f"elaboratev2module-v{version}"), new_version=Version(version), publish_index=pip_index, - new_extras={"optional": [safe_parse_requirement(requirement_name="inmanta-module-minimalv2module")]}, + new_extras={"optional": [safe_parse_requirement(requirement="inmanta-module-minimalv2module")]}, ) # Create project diff --git a/tests/moduletool/test_convert_v1_v2.py b/tests/moduletool/test_convert_v1_v2.py index e5751e5dcd..c634410785 100644 --- a/tests/moduletool/test_convert_v1_v2.py +++ b/tests/moduletool/test_convert_v1_v2.py @@ -30,7 +30,7 @@ import toml from inmanta import moduletool from inmanta.command import CLIException -from inmanta.env import safe_parse_requirement +from inmanta.env import safe_parse_requirements from inmanta.module import DummyProject, ModuleV1, ModuleV2, ModuleV2Metadata from inmanta.moduletool import ModuleConverter, ModuleVersionException from packaging import version @@ -114,9 +114,7 @@ def test_issue_3159_conversion_std_module_add_self_to_dependencies(tmpdir): parser = configparser.ConfigParser() parser.read(setup_cfg_file) assert parser.has_option("options", "install_requires") - install_requires = [ - safe_parse_requirement(requirement_name=r) for r in parser.get("options", "install_requires").split("\n") if r - ] + install_requires = safe_parse_requirements(parser.get("options", "install_requires").split("\n")) pkg_names = [r.name for r in install_requires] assert "inmanta-module-std" not in pkg_names diff --git a/tests/moduletool/test_install.py b/tests/moduletool/test_install.py index 72395a9e16..bea488bd7f 100644 --- a/tests/moduletool/test_install.py +++ b/tests/moduletool/test_install.py @@ -258,7 +258,7 @@ def test_module_install_conflicting_requirements(tmpdir: py.path.local, snippetc os.path.join(modules_v2_dir, "minimalv2module"), os.path.join(str(tmpdir), "modtwo"), new_name="modtwo", - new_requirements=[safe_parse_requirement(requirement_name="lorem~=0.1.0")], + new_requirements=[safe_parse_requirement(requirement="lorem~=0.1.0")], install=True, ) @@ -508,10 +508,7 @@ def test_project_install( index_url=local_module_package_index, # We add tornado, as there is a code path in update for the case where the project has python requires python_requires=["tornado"] - + [ - safe_parse_requirement(requirement_name=module.ModuleV2Source.get_package_name_for(mod)) - for mod in install_module_names - ], + + [safe_parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names], install_project=False, ) @@ -544,8 +541,7 @@ def test_project_install( autostd=False, python_package_sources=[local_module_package_index], python_requires=[ - safe_parse_requirement(requirement_name=module.ModuleV2Source.get_package_name_for(mod)) - for mod in install_module_names + safe_parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names ] + ["lorem"], install_project=False, @@ -681,7 +677,7 @@ def test_project_install_modules_cache_invalid( index_url=index.url, extra_index_url=[local_module_package_index], # make sure main module gets installed, pulling in newest version of dependency module - python_requires=[safe_parse_requirement(requirement_name=module.ModuleV2Source.get_package_name_for(main_module))], + python_requires=[safe_parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(main_module))], ) # populate project.modules[dependency_module] to force the error conditions in this simplified example @@ -766,7 +762,7 @@ def test_project_install_incompatible_versions( install_project=False, add_to_module_path=[v1_modules_path], index_url=index.url, - python_requires=[safe_parse_requirement(requirement_name=module.ModuleV2Source.get_package_name_for(v2_mod_name))], + python_requires=[safe_parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(v2_mod_name))], ) # install project @@ -819,14 +815,14 @@ def test_project_install_incompatible_dependencies( v2_template_path, os.path.join(str(tmpdir), "v2mod2"), new_name="v2mod2", - new_requirements=[safe_parse_requirement(requirement_name="inmanta-module-v2mod1~=1.0.0")], + new_requirements=[safe_parse_requirement(requirement="inmanta-module-v2mod1~=1.0.0")], publish_index=index, ) v2mod3: module.ModuleV2Metadata = module_from_template( v2_template_path, os.path.join(str(tmpdir), "v2mod3"), new_name="v2mod3", - new_requirements=[safe_parse_requirement(requirement_name="inmanta-module-v2mod1~=2.0.0")], + new_requirements=[safe_parse_requirement(requirement="inmanta-module-v2mod1~=2.0.0")], publish_index=index, ) @@ -1091,7 +1087,7 @@ def test_install_with_use_config_but_PIP_CONFIG_FILE_not_set( autostd=False, install_project=False, use_pip_config_file=True, - python_requires=[safe_parse_requirement(requirement_name="inmanta-module-dummy-module")], + python_requires=[safe_parse_requirement(requirement="inmanta-module-dummy-module")], ) # install project @@ -1208,7 +1204,7 @@ def test_install_project_with_install_mode_master(tmpdir: py.path.local, snippet autostd=False, install_project=False, add_to_module_path=[str(tmpdir)], - project_requires=[InmantaModuleRequirement(safe_parse_requirement(requirement_name="mod11==3.2.1"))], + project_requires=[InmantaModuleRequirement(safe_parse_requirement(requirement="mod11==3.2.1"))], install_mode=InstallMode.master, ) @@ -1236,7 +1232,7 @@ def test_module_install_logging(local_module_package_index: str, snippetcompiler v2_module = "minimalv2module" - v2_requirements = [safe_parse_requirement(requirement_name=module.ModuleV2Source.get_package_name_for(v2_module))] + v2_requirements = [safe_parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(v2_module))] # set up project and modules project: module.Project = snippetcompiler_clean.setup_for_snippet( @@ -1337,9 +1333,7 @@ def test_pip_output(local_module_package_index: str, snippetcompiler_clean, capl ) modules = ["modone", "modtwo"] - v2_requirements = [ - safe_parse_requirement(requirement_name=module.ModuleV2Source.get_package_name_for(mod)) for mod in modules - ] + v2_requirements = [safe_parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in modules] snippetcompiler_clean.setup_for_snippet( f""" @@ -1425,9 +1419,7 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[ - safe_parse_requirement(requirement_name=module.ModuleV2Source.get_package_name_for("parent_module")) - ], + python_requires=[safe_parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module"))], install_project=True, ) log_contains( @@ -1459,9 +1451,7 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[ - safe_parse_requirement(requirement_name=module.ModuleV2Source.get_package_name_for("parent_module")) - ], + python_requires=[safe_parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module"))], install_project=True, ) @@ -1494,7 +1484,7 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[safe_parse_requirement(requirement_name=module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[safe_parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module"))], install_project=True, ) log_contains( @@ -1563,7 +1553,7 @@ def test_version_snapshot(local_module_package_index: str, snippetcompiler_clean autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[safe_parse_requirement(requirement_name=module.ModuleV2Source.get_package_name_for("module_b"))], + python_requires=[safe_parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("module_b"))], install_project=True, ) @@ -1589,7 +1579,7 @@ def test_version_snapshot(local_module_package_index: str, snippetcompiler_clean autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[safe_parse_requirement(requirement_name=module.ModuleV2Source.get_package_name_for("module_c"))], + python_requires=[safe_parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("module_c"))], install_project=True, ) @@ -1660,7 +1650,7 @@ def test_constraints_logging_v2(modules_v2_dir, tmpdir, caplog, snippetcompiler_ index_url=local_module_package_index, extra_index_url=[index.url], python_requires=[ - safe_parse_requirement(requirement_name=module.ModuleV2Source.get_package_name_for(mod)) + safe_parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in ["module_b", "module_a"] ], install_project=True, diff --git a/tests/moduletool/test_update.py b/tests/moduletool/test_update.py index 8d8169cd4e..fc25fec1a1 100644 --- a/tests/moduletool/test_update.py +++ b/tests/moduletool/test_update.py @@ -23,7 +23,7 @@ from inmanta.config import Config from inmanta.data.model import PipConfig -from inmanta.env import LocalPackagePath, process_env, safe_parse_requirement +from inmanta.env import LocalPackagePath, process_env, safe_parse_requirement, safe_parse_requirements from inmanta.module import InmantaModuleRequirement, InstallMode, ModuleV1, ModuleV2Source from inmanta.moduletool import ProjectTool from inmanta.parser import ParserException @@ -125,7 +125,7 @@ def assert_version_installed(module_name: str, version: str) -> None: new_version=Version(current_version), new_name=module_name, new_requirements=( - [InmantaModuleRequirement(safe_parse_requirement(requirement_name="module2<3.0.0"))] + [InmantaModuleRequirement(safe_parse_requirement(requirement="module2<3.0.0"))] if module_name == "module1" else None ), @@ -142,7 +142,7 @@ def assert_version_installed(module_name: str, version: str) -> None: # Add a dependency on module2, without setting an explicit version constraint. Later version of module1 # do set a version constraint on the dependency on module2. This way it is verified whether the module update # command takes into account the version constraints set in a new version of a module. - new_requirements=[InmantaModuleRequirement(safe_parse_requirement(requirement_name="module2"))], + new_requirements=[InmantaModuleRequirement(safe_parse_requirement(requirement="module2"))], install=False, publish_index=pip_index, new_content_init_cf="entity" if corrupt_module else None, # Introduce syntax error in the module @@ -242,7 +242,7 @@ def test_module_update_dependencies( "b", Version(v), str(tmpdir.join(f"b-{v}")), - requirements=[safe_parse_requirement(requirement_name="c")], + requirements=[safe_parse_requirement(requirement="c")], publish_index=index, ) for v in ("1.0.0", "2.0.0"): @@ -258,7 +258,7 @@ def test_module_update_dependencies( # install b-1.0.0 and c-1.0.0 process_env.install_for_config( - [safe_parse_requirement(requirement_name=req) for req in ("b==1.0.0", "c==1.0.0")], + [safe_parse_requirement(requirement=req) for req in ("b==1.0.0", "c==1.0.0")], config=PipConfig( index_url=index.url, use_system_config=False, @@ -270,7 +270,7 @@ def test_module_update_dependencies( source_dir=os.path.join(modules_dir, "minimalv1module"), dest_dir=str(tmpdir.join("modules", "my_mod")), new_name="my_mod", - new_requirements=[safe_parse_requirement(requirement_name=req) for req in ("a", "b~=1.0.0")], + new_requirements=safe_parse_requirements(["a", "b~=1.0.0"]), ) # run `inmanta project update` without running install first diff --git a/tests/server/test_compilerservice.py b/tests/server/test_compilerservice.py index 622470f474..921d0cdc43 100644 --- a/tests/server/test_compilerservice.py +++ b/tests/server/test_compilerservice.py @@ -1806,7 +1806,7 @@ def patch_get_protected_inmanta_packages(): venv = PythonEnvironment(env_path=venv_path) assert name_protected_pkg not in venv.get_installed_packages() venv.install_for_config( - requirements=[safe_parse_requirement(requirement_name=name_protected_pkg)], + requirements=[safe_parse_requirement(requirement=name_protected_pkg)], config=PipConfig( index_url=local_module_package_index, ), diff --git a/tests/test_env.py b/tests/test_env.py index d358e81329..7d8950eba5 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -176,7 +176,7 @@ def test_gen_req_file(): # make sure they all parse for req in reqs: - safe_parse_requirement(requirement_name=req) + safe_parse_requirement(requirement=req) def test_environment_python_version_multi_digit(tmpdir: py.path.local) -> None: @@ -205,7 +205,7 @@ def test_process_env_install_from_index( package_name: str = "more-itertools" assert package_name not in env.process_env.get_installed_packages() env.process_env.install_for_config( - [safe_parse_requirement(requirement_name=package_name + (f"=={version}" if version is not None else ""))], + [safe_parse_requirement(requirement=package_name + (f"=={version}" if version is not None else ""))], config=PipConfig( use_system_config=True, # we need an upstream for some packages ), @@ -220,7 +220,7 @@ def test_process_env_install_from_index( # It should hit the cache there and return here. # Cheap and fast test env.process_env.install_from_index( - [safe_parse_requirement(requirement_name=package_name + (f"=={version}" if version is not None else ""))], + [safe_parse_requirement(requirement=package_name + (f"=={version}" if version is not None else ""))], use_pip_config=True, ) @@ -270,7 +270,7 @@ def test_process_env_install_from_index_not_found_env_var( with pytest.raises(env.PackageNotFound, match=re.escape(expected)): env.process_env.install_for_config( - [safe_parse_requirement(requirement_name="this-package-does-not-exist")], + [safe_parse_requirement(requirement="this-package-does-not-exist")], config=PipConfig( index_url=index_urls[0], # The first element should only be passed to the index_url. If there are indexes in the environment @@ -307,7 +307,7 @@ def test_process_env_install_no_index(tmpdir: py.path.local, monkeypatch, use_sy with pytest.raises(env.PackageNotFound, match=re.escape(expected)): env.process_env.install_for_config( - requirements=[safe_parse_requirement(requirement_name="this-package-does-not-exist")], + requirements=[safe_parse_requirement(requirement="this-package-does-not-exist")], paths=[env.LocalPackagePath(path=str(tmpdir))], config=PipConfig(use_system_config=use_system_config), ) @@ -324,7 +324,7 @@ def test_process_env_install_from_index_conflicting_reqs( package_name: str = "more-itertools" with pytest.raises(env.ConflictingRequirements) as e: env.process_env.install_for_config( - [safe_parse_requirement(requirement_name=f"{package_name}{version}") for version in [">8.5", "<=8"]], + [safe_parse_requirement(requirement=f"{package_name}{version}") for version in [">8.5", "<=8"]], config=PipConfig( use_system_config=True, # we need an upstream for some packages ), @@ -394,7 +394,7 @@ def test_active_env_get_module_file( loader.PluginModuleFinder.configure_module_finder([os.path.join(str(tmpdir), "libs")]) assert env.ActiveEnv.get_module_file(module_name) is None - env.process_env.install_for_config([safe_parse_requirement(requirement_name=package_name)], pip_config) + env.process_env.install_for_config([safe_parse_requirement(requirement=package_name)], pip_config) assert package_name in env.process_env.get_installed_packages() module_info: Optional[tuple[Optional[str], Loader]] = env.ActiveEnv.get_module_file(module_name) assert module_info is not None @@ -540,7 +540,7 @@ def assert_all_checks(expect_test: tuple[bool, str] = (True, ""), expect_nonext: create_install_package( "test-package-two", version.Version("1.0.0"), - [safe_parse_requirement(requirement_name="test-package-one~=1.0")], + [safe_parse_requirement(requirement="test-package-one~=1.0")], local_module_package_index, ) assert_all_checks() @@ -561,7 +561,7 @@ def test_active_env_check_constraints(caplog, tmpvenv_active_inherit: str, local """ caplog.set_level(logging.WARNING) in_scope: Pattern[str] = re.compile("test-package-.*") - constraints: list[Requirement] = [safe_parse_requirement(requirement_name="test-package-one~=1.0")] + constraints: list[Requirement] = [safe_parse_requirement(requirement="test-package-one~=1.0")] env.ActiveEnv.check(in_scope) @@ -580,7 +580,7 @@ def test_active_env_check_constraints(caplog, tmpvenv_active_inherit: str, local create_install_package( "ext-package-one", version.Version("1.0.0"), - [safe_parse_requirement(requirement_name="test-package-one==1.0")], + [safe_parse_requirement(requirement="test-package-one==1.0")], local_module_package_index, ) env.ActiveEnv.check(in_scope, constraints) @@ -611,7 +611,7 @@ def test_override_inmanta_package(tmpvenv_active_inherit: env.VirtualEnv) -> Non installed_pkgs = tmpvenv_active_inherit.get_installed_packages() assert "inmanta-core" in installed_pkgs, "The inmanta-core package should be installed to run the tests" - inmanta_requirements = safe_parse_requirement(requirement_name="inmanta-core==4.0.0") + inmanta_requirements = safe_parse_requirement(requirement="inmanta-core==4.0.0") with pytest.raises(env.ConflictingRequirements) as excinfo: tmpvenv_active_inherit.install_for_config( requirements=[inmanta_requirements], @@ -651,13 +651,13 @@ def test_cache_on_active_env(tmpvenv_active_inherit: env.ActiveEnv, local_module """ def _assert_install(requirement: str, installed: bool) -> None: - parsed_requirement = safe_parse_requirement(requirement_name=requirement) + parsed_requirement = safe_parse_requirement(requirement=requirement) for r in [requirement, parsed_requirement]: assert tmpvenv_active_inherit.are_installed(requirements=[r]) == installed _assert_install("inmanta-module-elaboratev2module==1.2.3", installed=False) tmpvenv_active_inherit.install_for_config( - requirements=[safe_parse_requirement(requirement_name="inmanta-module-elaboratev2module==1.2.3")], + requirements=[safe_parse_requirement(requirement="inmanta-module-elaboratev2module==1.2.3")], config=PipConfig( index_url=local_module_package_index, ), @@ -701,7 +701,7 @@ def test_are_installed_dependency_cycle_on_extra(tmpdir, tmpvenv_active_inherit: path=os.path.join(tmpdir, "pkg"), publish_index=pip_index, optional_dependencies={ - "optional-pkg": [safe_parse_requirement(requirement_name="dep[optional-dep]")], + "optional-pkg": [safe_parse_requirement(requirement="dep[optional-dep]")], }, ) create_python_package( @@ -710,11 +710,11 @@ def test_are_installed_dependency_cycle_on_extra(tmpdir, tmpvenv_active_inherit: path=os.path.join(tmpdir, "dep"), publish_index=pip_index, optional_dependencies={ - "optional-dep": [safe_parse_requirement(requirement_name="pkg[optional-pkg]")], + "optional-dep": [safe_parse_requirement(requirement="pkg[optional-pkg]")], }, ) - requirements = [safe_parse_requirement(requirement_name="pkg[optional-pkg]")] + requirements = [safe_parse_requirement(requirement="pkg[optional-pkg]")] tmpvenv_active_inherit.install_for_config( requirements=requirements, config=PipConfig( diff --git a/tests/test_file_parser.py b/tests/test_file_parser.py index 98cef35040..868b8f3064 100644 --- a/tests/test_file_parser.py +++ b/tests/test_file_parser.py @@ -18,7 +18,8 @@ import os -from inmanta.env import Requirement, safe_parse_requirement +import packaging.requirements +from inmanta.env import safe_parse_requirements from inmanta.file_parser import RequirementsTxtParser @@ -39,8 +40,8 @@ def test_requirements_txt_parser(tmpdir) -> None: fd.write(content) expected_requirements = ["test==1.2.3", "other-dep~=2.0.0", "third-dep<5.0.0", "splitteddep", "Capital"] - requirements: list[Requirement] = RequirementsTxtParser().parse(requirements_txt_file) - assert requirements == [safe_parse_requirement(requirement_name=r) for r in expected_requirements] + requirements: list[packaging.requirements.Requirement] = RequirementsTxtParser().parse(requirements_txt_file) + assert requirements == safe_parse_requirements(expected_requirements) requirements_as_str = RequirementsTxtParser.parse_requirements_as_strs(requirements_txt_file) assert requirements_as_str == expected_requirements diff --git a/tests/test_module_loader.py b/tests/test_module_loader.py index c78f935bd2..b4b2cae886 100644 --- a/tests/test_module_loader.py +++ b/tests/test_module_loader.py @@ -352,7 +352,7 @@ def test_load_module_recursive_v2_module_depends_on_v1( project = snippetcompiler.setup_for_snippet( snippet="import v2_depends_on_v1", index_url=local_module_package_index, - python_requires=[safe_parse_requirement(requirement_name="inmanta-module-v2-depends-on-v1")], + python_requires=[safe_parse_requirement(requirement="inmanta-module-v2-depends-on-v1")], install_project=False, ) if preload_v1_module: @@ -379,7 +379,7 @@ def test_load_module_recursive_complex_module_dependencies(local_module_package_ snippet="import complex_module_dependencies_mod1", autostd=False, index_url=local_module_package_index, - python_requires=[safe_parse_requirement(requirement_name="inmanta-module-complex-module-dependencies-mod1")], + python_requires=[safe_parse_requirement(requirement="inmanta-module-complex-module-dependencies-mod1")], install_project=False, ) assert "complex_module_dependencies_mod1" not in project.modules @@ -418,7 +418,7 @@ def load(requires: Optional[list[Requirement]] = None) -> None: with pytest.raises(ModuleLoadingException, match=f"Failed to load module {module_name}"): load() # assert that it doesn't raise an error with explicit requirements set - load([safe_parse_requirement(requirement_name=ModuleV2Source.get_package_name_for(module_name))]) + load([safe_parse_requirement(requirement=ModuleV2Source.get_package_name_for(module_name))]) @pytest.mark.parametrize("v1", [True, False]) @@ -476,7 +476,7 @@ def test_load_import_based_v2_module( # make sure that even listing the requirement in project.yml does not suffice project_requires=[InmantaModuleRequirement.parse(dependency_module_name)], python_requires=( - [] if v1 else [safe_parse_requirement(requirement_name=ModuleV2Source.get_package_name_for(main_module_name))] + [] if v1 else [safe_parse_requirement(requirement=ModuleV2Source.get_package_name_for(main_module_name))] ), ) @@ -619,7 +619,7 @@ def test_project_requirements_dont_overwrite_core_requirements_source( module_from_template( os.path.join(modules_v2_dir, module_name), module_path, - new_requirements=[safe_parse_requirement(requirement_name="Jinja2==2.11.3")], + new_requirements=[safe_parse_requirement(requirement="Jinja2==2.11.3")], ) # Activate the snippetcompiler venv @@ -663,7 +663,7 @@ def test_project_requirements_dont_overwrite_core_requirements_index( module_from_template( os.path.join(modules_v2_dir, module_name), module_path, - new_requirements=[safe_parse_requirement(requirement_name="Jinja2==2.11.3")], + new_requirements=[safe_parse_requirement(requirement="Jinja2==2.11.3")], publish_index=index, ) @@ -717,7 +717,7 @@ def test_module_conflicting_dependencies_with_v2_modules( "y", Version("1.0.0"), str(tmpdir.join("y-1.0.0")), - requirements=[safe_parse_requirement(requirement_name="x~=1.0.0")], + requirements=[safe_parse_requirement(requirement="x~=1.0.0")], publish_index=index, ) @@ -727,7 +727,7 @@ def test_module_conflicting_dependencies_with_v2_modules( module_from_template( os.path.join(modules_v2_dir, module_name1), module_path1, - new_requirements=[safe_parse_requirement(requirement_name="y~=1.0.0")], + new_requirements=[safe_parse_requirement(requirement="y~=1.0.0")], publish_index=index, ) @@ -738,7 +738,7 @@ def test_module_conflicting_dependencies_with_v2_modules( os.path.join(modules_v2_dir, "minimalv2module"), module_path2, new_name="minimalv2module2", - new_requirements=[safe_parse_requirement(requirement_name="x~=2.0.0")], + new_requirements=[safe_parse_requirement(requirement="x~=2.0.0")], publish_index=index, ) @@ -798,7 +798,7 @@ def test_module_conflicting_dependencies_with_v1_module( os.path.join(modules_dir, module_name1), module_path1, new_name="modulev1", - new_requirements=[safe_parse_requirement(requirement_name="y~=1.0.0")], + new_requirements=[safe_parse_requirement(requirement="y~=1.0.0")], ) # Create the second module @@ -807,7 +807,7 @@ def test_module_conflicting_dependencies_with_v1_module( module_from_template( os.path.join(modules_v2_dir, module_name2), module_path2, - new_requirements=[safe_parse_requirement(requirement_name="y~=2.0.0")], + new_requirements=[safe_parse_requirement(requirement="y~=2.0.0")], publish_index=index, ) @@ -855,7 +855,7 @@ def test_module_install_extra_on_project_level_v2_dep( new_name="mymod", new_requirements=[], new_extras={ - "myfeature": [safe_parse_requirement(requirement_name=package_name_extra)], + "myfeature": [safe_parse_requirement(requirement=package_name_extra)], }, publish_index=index, ) @@ -904,7 +904,7 @@ def test_module_install_extra_on_dep_of_v2_module( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [safe_parse_requirement(requirement_name=package_name_extra)], + "myfeature": [safe_parse_requirement(requirement=package_name_extra)], }, publish_index=index, ) @@ -924,7 +924,7 @@ def test_module_install_extra_on_dep_of_v2_module( install_project=True, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[safe_parse_requirement(requirement_name="inmanta-module-myv2mod")], + python_requires=[safe_parse_requirement(requirement="inmanta-module-myv2mod")], autostd=False, ) @@ -961,7 +961,7 @@ def test_module_install_extra_on_dep_of_v1_module( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [safe_parse_requirement(requirement_name=package_name_extra)], + "myfeature": [safe_parse_requirement(requirement=package_name_extra)], }, publish_index=index, ) @@ -1011,7 +1011,7 @@ def test_module_install_extra_on_project_level_v2_dep_update_scenario( new_name="mymod", new_requirements=[], new_extras={ - "myfeature": [safe_parse_requirement(requirement_name=package_name_extra)], + "myfeature": [safe_parse_requirement(requirement=package_name_extra)], }, publish_index=index, ) @@ -1080,7 +1080,7 @@ def test_module_install_extra_on_dep_of_v2_module_update_scenario( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [safe_parse_requirement(requirement_name=package_name_extra)], + "myfeature": [safe_parse_requirement(requirement=package_name_extra)], }, publish_index=index, ) @@ -1109,7 +1109,7 @@ def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> install_project=True, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[safe_parse_requirement(requirement_name="inmanta-module-myv2mod==1.0.0")], + python_requires=[safe_parse_requirement(requirement="inmanta-module-myv2mod==1.0.0")], autostd=False, ) assert_installed(extra_installed=False) @@ -1131,7 +1131,7 @@ def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> install_project=not do_project_update, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[safe_parse_requirement(requirement_name="inmanta-module-myv2mod==2.0.0")], + python_requires=[safe_parse_requirement(requirement="inmanta-module-myv2mod==2.0.0")], autostd=False, ) if do_project_update: @@ -1175,7 +1175,7 @@ def test_module_install_extra_on_dep_of_v1_module_update_scenario( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [safe_parse_requirement(requirement_name=package_name_extra)], + "myfeature": [safe_parse_requirement(requirement=package_name_extra)], }, publish_index=index, ) @@ -1239,7 +1239,7 @@ async def test_v1_module_depends_on_third_party_dep_with_extra( os.path.join(tmpdir, "myv1mod"), new_name="myv1mod", new_content_init_cf="", - new_requirements=[safe_parse_requirement(requirement_name="pkg[optional-a]")], + new_requirements=[safe_parse_requirement(requirement="pkg[optional-a]")], ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv1mod", @@ -1259,7 +1259,7 @@ async def test_v1_module_depends_on_third_party_dep_with_extra( os.path.join(tmpdir, "myv1mod"), new_name="myv1mod", new_content_init_cf="", - new_requirements=[safe_parse_requirement(requirement_name="pkg[optional-a,optional-b]")], + new_requirements=[safe_parse_requirement(requirement="pkg[optional-a,optional-b]")], ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv1mod", @@ -1286,13 +1286,13 @@ async def test_v2_module_depends_on_third_party_dep_with_extra( str(tmpdir.join("myv2mod")), new_name="myv2mod", new_version=Version("1.0.0"), - new_requirements=[safe_parse_requirement(requirement_name="pkg[optional-a]")], + new_requirements=[safe_parse_requirement(requirement="pkg[optional-a]")], publish_index=index, ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv2mod", install_project=True, - python_requires=[safe_parse_requirement(requirement_name="inmanta-module-myv2mod==1.0.0")], + python_requires=[safe_parse_requirement(requirement="inmanta-module-myv2mod==1.0.0")], index_url=index.url, extra_index_url=[index_with_pkgs_containing_optional_deps], autostd=False, @@ -1308,13 +1308,13 @@ async def test_v2_module_depends_on_third_party_dep_with_extra( str(tmpdir.join("myv2mod")), new_name="myv2mod", new_version=Version("2.0.0"), - new_requirements=[safe_parse_requirement(requirement_name="pkg[optional-a,optional-b]")], + new_requirements=[safe_parse_requirement(requirement="pkg[optional-a,optional-b]")], publish_index=index, ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv2mod", install_project=True, - python_requires=[safe_parse_requirement(requirement_name="inmanta-module-myv2mod==2.0.0")], + python_requires=[safe_parse_requirement(requirement="inmanta-module-myv2mod==2.0.0")], index_url=index.url, extra_index_url=[index_with_pkgs_containing_optional_deps], autostd=False, diff --git a/tests/utils.py b/tests/utils.py index c0159b48b2..2a08526074 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -40,6 +40,7 @@ import build import build.env +import packaging.version from _pytest.mark import MarkDecorator from inmanta import config, const, data, env, module, protocol, util from inmanta.data import ResourceIdStr @@ -50,7 +51,6 @@ from inmanta.server.extensions import ProductMetadata from inmanta.util import get_compiler_version, hash_file from libpip2pi.commands import dir2pi -from packaging import version from packaging.requirements import Requirement from packaging.version import Version @@ -485,7 +485,7 @@ def publish(self) -> None: def create_python_package( name: str, - pkg_version: version.Version, + pkg_version: packaging.version.Version, path: str, *, requirements: Optional[Sequence[Requirement]] = None, @@ -575,7 +575,7 @@ def module_from_template( source_dir: str, dest_dir: Optional[str] = None, *, - new_version: Optional[version.Version] = None, + new_version: Optional[packaging.version.Version] = None, new_name: Optional[str] = None, new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, Requirement]]] = None, new_extras: Optional[abc.Mapping[str, abc.Sequence[Union[module.InmantaModuleRequirement, Requirement]]]] = None, @@ -686,7 +686,7 @@ def v1_module_from_template( source_dir: str, dest_dir: str, *, - new_version: Optional[version.Version] = None, + new_version: Optional[packaging.version.Version] = None, new_name: Optional[str] = None, new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, Requirement]]] = None, new_content_init_cf: Optional[str] = None, From fca2bfcdc1e85275898cfb5ad2af581d18041ac6 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Fri, 30 Aug 2024 17:35:49 +0200 Subject: [PATCH 25/74] format mypy --- src/inmanta/env.py | 4 ++-- src/inmanta/module.py | 6 +++--- tests/compiler/test_basics.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 6b07aff18e..b1a4174f84 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -215,7 +215,7 @@ def _get_as_requirements_type(cls, requirements: req_list) -> Sequence[Requireme Convert requirements from Union[Sequence[str], Sequence[Requirement]] to Sequence[Requirement] """ if isinstance(requirements[0], str): - return safe_parse_requirement(requirements) + return safe_parse_requirements(requirements) else: return requirements @@ -825,7 +825,7 @@ def install_for_config( def install_from_index( self, - requirements: list[Requirement], + requirements: Sequence[Requirement], index_urls: Optional[list[str]] = None, upgrade: bool = False, allow_pre_releases: bool = False, diff --git a/src/inmanta/module.py b/src/inmanta/module.py index 36a561c9a2..d54abc5600 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -2127,7 +2127,7 @@ def install_modules(self, *, bypass_module_cache: bool = False, update_dependenc self.verify_module_version_compatibility() # do python install - pyreq: list[Requirement] = safe_parse_requirement(self.collect_python_requirements()) + pyreq: Sequence[Requirement] = safe_parse_requirements(self.collect_python_requirements()) if len(pyreq) > 0: # upgrade both direct and transitive module dependencies: eager upgrade strategy @@ -2545,7 +2545,7 @@ def verify_python_requires(self) -> None: Verifies no incompatibilities exist within the Python environment with respect to installed module v2 requirements. """ if self.strict_deps_check: - constraints: list[Requirement] = [safe_parse_requirements(self.collect_python_requirements())] + constraints: Sequence[Requirement] = safe_parse_requirements(self.collect_python_requirements()) env.ActiveEnv.check(strict_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*"), constraints=constraints) else: if not env.ActiveEnv.check_legacy(in_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*")): @@ -2707,7 +2707,7 @@ def get_spec(name: str) -> "List[InmantaModuleRequirement]": return {name: get_spec(name) for name in imports} - def collect_python_requirements(self) -> list[str]: + def collect_python_requirements(self) -> Sequence[str]: """ Collect the list of all python requirements of all modules in this project, excluding those on inmanta modules. """ diff --git a/tests/compiler/test_basics.py b/tests/compiler/test_basics.py index 288bcdd8e3..2c3e5034a7 100644 --- a/tests/compiler/test_basics.py +++ b/tests/compiler/test_basics.py @@ -26,8 +26,8 @@ from inmanta import compiler, const, module from inmanta.ast import DoubleSetException, RuntimeException -from inmanta.module import InstallMode from inmanta.env import safe_parse_requirement +from inmanta.module import InstallMode from inmanta.plugins import PluginDeprecationWarning from packaging import version from utils import module_from_template, v1_module_from_template From db0591c095f0e50f9c5f32da36cea9ee7c670825 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Mon, 2 Sep 2024 09:14:53 +0200 Subject: [PATCH 26/74] fix comparing versions --- src/inmanta/env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index b1a4174f84..c6ab7f6e7d 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -1088,7 +1088,7 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: constraint_violations_strict: set[VersionConflict] = set() for c in all_constraints: requirement = c.requirement - if (installed_versions[requirement.name] not in requirement.specifier) and ( + if (len(requirement.specifier) > 0 and installed_versions[requirement.name] not in requirement.specifier) and ( not requirement.marker or (requirement.marker and requirement.marker.evaluate()) ): version_conflict = VersionConflict( From 10b41bd63c7d9ff11c474428a224fee914f94894 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Mon, 2 Sep 2024 10:24:48 +0200 Subject: [PATCH 27/74] refactor + fix broken calls --- src/inmanta/env.py | 1 + src/inmanta/file_parser.py | 4 ++-- tests/compiler/test_basics.py | 2 +- tests/moduletool/test_install.py | 10 +++++----- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index c6ab7f6e7d..ea4467dc20 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -1088,6 +1088,7 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: constraint_violations_strict: set[VersionConflict] = set() for c in all_constraints: requirement = c.requirement + # If no specifiers are provided, the `in` operation will return `False` if (len(requirement.specifier) > 0 and installed_versions[requirement.name] not in requirement.specifier) and ( not requirement.marker or (requirement.marker and requirement.marker.evaluate()) ): diff --git a/src/inmanta/file_parser.py b/src/inmanta/file_parser.py index 9d94309890..3bfd6b3558 100644 --- a/src/inmanta/file_parser.py +++ b/src/inmanta/file_parser.py @@ -102,8 +102,8 @@ def get_content_with_dep_removed(cls, filename: str, remove_dep_on_pkg: str) -> result += line elif line.endswith("\\"): line_continuation_buffer = line - elif packaging.utils.canonicalize_name(line) != removed_dependency: - result += line + elif safe_parse_requirement(requirement=line).name != removed_dependency: + result += safe_parse_requirement(requirement=line).name else: # Dependency matches `remove_dep_on_pkg` => Remove line from result pass diff --git a/tests/compiler/test_basics.py b/tests/compiler/test_basics.py index 2c3e5034a7..a65518e8ab 100644 --- a/tests/compiler/test_basics.py +++ b/tests/compiler/test_basics.py @@ -743,7 +743,7 @@ def test_safe_requirement(name) -> None: """ Ensure that empty name requirements are not allowed in `Requirement` """ - with pytest.raises(AssertionError): + with pytest.raises(ValueError): safe_parse_requirement(requirement=name) diff --git a/tests/moduletool/test_install.py b/tests/moduletool/test_install.py index bea488bd7f..2f2220f9e4 100644 --- a/tests/moduletool/test_install.py +++ b/tests/moduletool/test_install.py @@ -251,7 +251,7 @@ def test_module_install_conflicting_requirements(tmpdir: py.path.local, snippetc os.path.join(modules_v2_dir, "minimalv2module"), os.path.join(str(tmpdir), "modone"), new_name="modone", - new_requirements=[safe_parse_requirement(requiremement="lorem~=0.0.1")], + new_requirements=[safe_parse_requirement(requirement="lorem~=0.0.1")], install=True, ) module_from_template( @@ -837,7 +837,7 @@ def test_project_install_incompatible_dependencies( index_url=index.url, python_requires=[ safe_parse_requirement( - requirement_string=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) + requirement=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod2, v2mod3] ], @@ -921,7 +921,7 @@ def test_install_from_index_dont_leak_pip_index( index_url="unknown", python_requires=[ safe_parse_requirement( - requirement_string=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) + requirement=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod1] ], @@ -983,7 +983,7 @@ def test_install_with_use_config( use_pip_config_file=use_pip_config, python_requires=[ safe_parse_requirement( - requirement_string=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) + requirement=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod1] ], @@ -1052,7 +1052,7 @@ def test_install_with_use_config_extra_index( use_pip_config_file=True, python_requires=[ safe_parse_requirement( - requirement_string=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) + requirement=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod1, v2mod2] ], From 0c8d2924a4115164c2fad4f99034eb95ce18be72 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Mon, 2 Sep 2024 14:50:59 +0200 Subject: [PATCH 28/74] rollback + fix broken tests --- src/inmanta/env.py | 20 ++++++++++++++++---- src/inmanta/file_parser.py | 2 +- tests/test_file_parser.py | 6 ++---- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index ea4467dc20..d85171668f 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -65,6 +65,19 @@ class PipInstallError(Exception): pass +def remove_comment_part(to_clean: str) -> str: + """ + Remove the comment part of a given string and ensure that the lenght of the string is greater than 0 + + :param to_clean: The string to clean + :return: A cleaned string + """ + drop_comment, _, _ = to_clean.partition("#") + if len(drop_comment) == 0: + raise ValueError("The name of the requirement cannot be an empty string!") + return drop_comment + + def safe_parse_requirement(requirement: str) -> Requirement: """ To be able to compare requirements, we need to make sure that every requirement's name is canonicalized otherwise issues @@ -74,9 +87,7 @@ def safe_parse_requirement(requirement: str) -> Requirement: :return: A new requirement instance """ # Packaging Requirement is not able to parse requirements with comment. Therefore, we need to remove the `comment` part - drop_comment, _, _ = requirement.partition("#") - if len(drop_comment) == 0: - raise ValueError("The name of the requirement cannot be an empty string!") + drop_comment = remove_comment_part(to_clean=requirement) # We canonicalize the name of the requirement to be able to compare requirements and check if the requirement is # already installed requirement_instance = Requirement(requirement_string=drop_comment) @@ -255,7 +266,8 @@ def _are_installed_recursive( if r.marker and not r.marker.evaluate(environment=environment_marker_evaluation): # The marker of the requirement doesn't apply on this environment continue - if r.name not in installed_packages or (installed_packages[r.name] not in r.specifier): + # If no specifiers are provided, the `in` operation will return `False` + if r.name not in installed_packages or (len(r.specifier) > 0 and installed_packages[r.name] not in r.specifier): return False if r.extras: for extra in r.extras: diff --git a/src/inmanta/file_parser.py b/src/inmanta/file_parser.py index 3bfd6b3558..378e3db079 100644 --- a/src/inmanta/file_parser.py +++ b/src/inmanta/file_parser.py @@ -103,7 +103,7 @@ def get_content_with_dep_removed(cls, filename: str, remove_dep_on_pkg: str) -> elif line.endswith("\\"): line_continuation_buffer = line elif safe_parse_requirement(requirement=line).name != removed_dependency: - result += safe_parse_requirement(requirement=line).name + result += line else: # Dependency matches `remove_dep_on_pkg` => Remove line from result pass diff --git a/tests/test_file_parser.py b/tests/test_file_parser.py index 868b8f3064..d9fc160583 100644 --- a/tests/test_file_parser.py +++ b/tests/test_file_parser.py @@ -46,16 +46,14 @@ def test_requirements_txt_parser(tmpdir) -> None: assert requirements_as_str == expected_requirements new_content = RequirementsTxtParser.get_content_with_dep_removed(requirements_txt_file, remove_dep_on_pkg="test") - assert ( - new_content - == """ + expected_content = """ # A comment other-dep~=2.0.0 third-dep<5.0.0 # another comment splitteddep Capital """ - ) + assert new_content == expected_content new_content = RequirementsTxtParser.get_content_with_dep_removed(requirements_txt_file, remove_dep_on_pkg="third-dep") assert ( new_content From 7962876827dd46c79e734f5db98369fff300e00b Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Mon, 2 Sep 2024 15:26:58 +0200 Subject: [PATCH 29/74] refactor --- Makefile | 2 +- src/inmanta/env.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 3934bd17ff..ad41cc59a9 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ black = black src tests tests_common .PHONY: install install: pip install -U setuptools pip uv -c requirements.txt - uv pip install -U -e . -c requirements.txt -r requirements.dev.txt + uv pip install -U -e . -c requirements.txt -r requirements.txt -r requirements.dev.txt .PHONY: install-tests install-tests: diff --git a/src/inmanta/env.py b/src/inmanta/env.py index d85171668f..e262ee86ac 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -278,8 +278,8 @@ def _are_installed_recursive( return False pkgs_required_by_extra: set[Requirement] = set( - [safe_parse_requirement(e.key) for e in distribution.requires(extras=(extra,))] - ) - set([safe_parse_requirement(e.key) for e in distribution.requires(extras=())]) + [safe_parse_requirement(str(e)) for e in distribution.requires(extras=(extra,))] + ) - set([safe_parse_requirement(str(e)) for e in distribution.requires(extras=())]) if not _are_installed_recursive( reqs=list(pkgs_required_by_extra), seen_requirements=list(seen_requirements) + list(reqs), @@ -1047,6 +1047,8 @@ def install_for_config( finally: self.notify_change() + from pkg_resources.__init__ import Requirement + @classmethod def get_constraint_violations_for_check( cls, @@ -1067,7 +1069,7 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: # all requirements of all packages installed in this environment installed_constraints: abc.Set[OwnedRequirement] = frozenset( - OwnedRequirement(safe_parse_requirement(requirement=requirement.key), dist_info.key) + OwnedRequirement(safe_parse_requirement(requirement=str(requirement)), dist_info.key) for dist_info in pkg_resources.working_set for requirement in dist_info.requires() ) From f902c997cb7e385203397b5626daa1db362ea1c9 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Mon, 2 Sep 2024 15:37:51 +0200 Subject: [PATCH 30/74] fix broken tests --- src/inmanta/env.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index e262ee86ac..774d1f1a9d 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -1103,8 +1103,9 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: for c in all_constraints: requirement = c.requirement # If no specifiers are provided, the `in` operation will return `False` - if (len(requirement.specifier) > 0 and installed_versions[requirement.name] not in requirement.specifier) and ( - not requirement.marker or (requirement.marker and requirement.marker.evaluate()) + if requirement.name not in installed_versions or ( + (len(requirement.specifier) > 0 and installed_versions[requirement.name] not in requirement.specifier) + and (not requirement.marker or (requirement.marker and requirement.marker.evaluate())) ): version_conflict = VersionConflict( requirement=requirement, From 00c5506e31f90ec46887205364244f6be83d1266 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Fri, 6 Sep 2024 17:19:33 +0200 Subject: [PATCH 31/74] refactor --- src/inmanta/agent/executor.py | 5 +- src/inmanta/agent/in_process_executor.py | 4 +- src/inmanta/env.py | 73 +++--------------------- src/inmanta/file_parser.py | 8 +-- src/inmanta/module.py | 19 +++--- src/inmanta/moduletool.py | 4 +- src/inmanta/util/__init__.py | 64 ++++++++++++++++++++- tests/compiler/test_basics.py | 4 +- tests/conftest.py | 9 +-- tests/moduletool/test_add.py | 4 +- tests/moduletool/test_convert_v1_v2.py | 4 +- tests/moduletool/test_install.py | 46 +++++++-------- tests/moduletool/test_update.py | 12 ++-- tests/server/test_compilerservice.py | 4 +- tests/test_env.py | 35 ++++++------ tests/test_file_parser.py | 4 +- tests/test_module_loader.py | 54 +++++++++--------- 17 files changed, 181 insertions(+), 172 deletions(-) diff --git a/src/inmanta/agent/executor.py b/src/inmanta/agent/executor.py index 1442ea80de..7be0f46d61 100644 --- a/src/inmanta/agent/executor.py +++ b/src/inmanta/agent/executor.py @@ -41,7 +41,8 @@ from inmanta.agent import config as cfg from inmanta.agent import resourcepool from inmanta.data.model import PipConfig, ResourceIdStr, ResourceType, ResourceVersionIdStr -from inmanta.env import PythonEnvironment, safe_parse_requirements +from inmanta.env import PythonEnvironment +from inmanta.util import parse_canonical_requirements from inmanta.loader import ModuleSource from inmanta.resources import Id from inmanta.types import JsonType @@ -294,7 +295,7 @@ def create_and_install_environment(self, blueprint: EnvBlueprint) -> None: self.init_env() if len(req): # install_for_config expects at least 1 requirement or a path to install self.install_for_config( - requirements=safe_parse_requirements(req), + requirements=parse_canonical_requirements(req), config=blueprint.pip_config, ) diff --git a/src/inmanta/agent/in_process_executor.py b/src/inmanta/agent/in_process_executor.py index 0d0e927a92..764a0717cb 100644 --- a/src/inmanta/agent/in_process_executor.py +++ b/src/inmanta/agent/in_process_executor.py @@ -33,7 +33,7 @@ from inmanta.agent.handler import HandlerAPI, SkipResource from inmanta.const import ParameterSource from inmanta.data.model import AttributeStateChange, ResourceIdStr, ResourceVersionIdStr -from inmanta.env import safe_parse_requirements +from inmanta.util import parse_canonical_requirements from inmanta.loader import CodeLoader from inmanta.resources import Id, Resource from inmanta.types import Apireturn @@ -571,7 +571,7 @@ async def _install(self, blueprint: executor.ExecutorBlueprint) -> None: await loop.run_in_executor( self.process.thread_pool, self._env.install_for_config, - safe_parse_requirements(blueprint.requirements), + parse_canonical_requirements(blueprint.requirements), blueprint.pip_config, ) await loop.run_in_executor(self.process.thread_pool, self._loader.deploy_version, blueprint.sources) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 774d1f1a9d..6f3d3a5175 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -22,7 +22,6 @@ import json import logging import os -import pathlib import re import site import subprocess @@ -45,7 +44,7 @@ import packaging.utils import packaging.version -from inmanta import const +from inmanta import const, util from inmanta.ast import CompilerException from inmanta.data.model import LEGACY_PIP_DEFAULT, PipConfig from inmanta.server.bootloader import InmantaBootloader @@ -65,62 +64,6 @@ class PipInstallError(Exception): pass -def remove_comment_part(to_clean: str) -> str: - """ - Remove the comment part of a given string and ensure that the lenght of the string is greater than 0 - - :param to_clean: The string to clean - :return: A cleaned string - """ - drop_comment, _, _ = to_clean.partition("#") - if len(drop_comment) == 0: - raise ValueError("The name of the requirement cannot be an empty string!") - return drop_comment - - -def safe_parse_requirement(requirement: str) -> Requirement: - """ - To be able to compare requirements, we need to make sure that every requirement's name is canonicalized otherwise issues - could arise when checking if packages are installed in a particular Venv. - - :param requirement: The requirement's name - :return: A new requirement instance - """ - # Packaging Requirement is not able to parse requirements with comment. Therefore, we need to remove the `comment` part - drop_comment = remove_comment_part(to_clean=requirement) - # We canonicalize the name of the requirement to be able to compare requirements and check if the requirement is - # already installed - requirement_instance = Requirement(requirement_string=drop_comment) - requirement_instance.name = packaging.utils.canonicalize_name(requirement_instance.name) - return requirement_instance - - -def safe_parse_requirements(requirements: Sequence[str]) -> Sequence[Requirement]: - """ - To be able to compare requirements, we need to make sure that every requirement's name is canonicalized otherwise issues - could arise when checking if packages are installed in a particular Venv. - - :param requirements: The names of the different requirements - :return: Sequence[Requirement] - """ - return [safe_parse_requirement(requirement=e) for e in requirements] - - -def safe_parse_requirements_from_file(file_path: pathlib.Path) -> Sequence[Requirement]: - """ - To be able to compare requirements, we need to make sure that every requirement's name is canonicalized otherwise issues - could arise when checking if packages are installed in a particular Venv. - - :param file_path: The path to the read the requirements from - :return: Sequence[Requirement] - """ - requirements = [] - with open(file_path) as f: - for line in f.readlines(): - requirements.append(safe_parse_requirement(line)) - return requirements - - @dataclass(eq=True, frozen=True) class VersionConflict: """ @@ -226,7 +169,7 @@ def _get_as_requirements_type(cls, requirements: req_list) -> Sequence[Requireme Convert requirements from Union[Sequence[str], Sequence[Requirement]] to Sequence[Requirement] """ if isinstance(requirements[0], str): - return safe_parse_requirements(requirements) + return util.parse_canonical_requirements(requirements) else: return requirements @@ -278,8 +221,8 @@ def _are_installed_recursive( return False pkgs_required_by_extra: set[Requirement] = set( - [safe_parse_requirement(str(e)) for e in distribution.requires(extras=(extra,))] - ) - set([safe_parse_requirement(str(e)) for e in distribution.requires(extras=())]) + [util.parse_canonical_requirement(str(e)) for e in distribution.requires(extras=(extra,))] + ) - set([util.parse_canonical_requirement(str(e)) for e in distribution.requires(extras=())]) if not _are_installed_recursive( reqs=list(pkgs_required_by_extra), seen_requirements=list(seen_requirements) + list(reqs), @@ -908,7 +851,7 @@ def install_from_list( use_pip_config was ignored on ISO6 and it still is """ self.install_from_index( - requirements=safe_parse_requirements(requirements_list), + requirements=util.parse_canonical_requirements(requirements_list), upgrade=upgrade, upgrade_strategy=upgrade_strategy, use_pip_config=True, @@ -936,7 +879,7 @@ def _get_requirements_on_inmanta_package(cls) -> Sequence[Requirement]: protected_inmanta_packages: list[str] = cls.get_protected_inmanta_packages() workingset: dict[str, packaging.version.Version] = PythonWorkingSet.get_packages_in_working_set() return [ - safe_parse_requirement(requirement=f"{pkg}=={workingset[pkg]}") + util.parse_canonical_requirement(requirement=f"{pkg}=={workingset[pkg]}") for pkg in workingset if pkg in protected_inmanta_packages ] @@ -1069,7 +1012,7 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: # all requirements of all packages installed in this environment installed_constraints: abc.Set[OwnedRequirement] = frozenset( - OwnedRequirement(safe_parse_requirement(requirement=str(requirement)), dist_info.key) + OwnedRequirement(util.parse_canonical_requirement(requirement=str(requirement)), dist_info.key) for dist_info in pkg_resources.working_set for requirement in dist_info.requires() ) @@ -1167,7 +1110,7 @@ def check_legacy(cls, in_scope: Pattern[str], constraints: Optional[list[Require working_set: abc.Iterable[Distribution] = importlib.metadata.distributions() # add all requirements of all in scope packages installed in this environment all_constraints: set[Requirement] = set(constraints if constraints is not None else []).union( - safe_parse_requirement(requirement=requirement) + util.parse_canonical_requirement(requirement=requirement) for dist_info in working_set if in_scope.fullmatch(dist_info.name) for requirement in dist_info.requires or [] diff --git a/src/inmanta/file_parser.py b/src/inmanta/file_parser.py index 378e3db079..8ee4696212 100644 --- a/src/inmanta/file_parser.py +++ b/src/inmanta/file_parser.py @@ -19,7 +19,7 @@ import os import packaging.utils -from inmanta.env import safe_parse_requirement +from inmanta.util import parse_canonical_requirement from packaging.requirements import Requirement from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap @@ -61,7 +61,7 @@ def parse(cls, filename: str) -> list[Requirement]: """ Get all the requirements in `filename` as a list of `Requirement` instances. """ - return [safe_parse_requirement(requirement=r) for r in cls.parse_requirements_as_strs(filename)] + return [parse_canonical_requirement(requirement=r) for r in cls.parse_requirements_as_strs(filename)] @classmethod def parse_requirements_as_strs(cls, filename: str) -> list[str]: @@ -95,14 +95,14 @@ def get_content_with_dep_removed(cls, filename: str, remove_dep_on_pkg: str) -> if line_continuation_buffer: line_continuation_buffer += line if not line.endswith("\\"): - if safe_parse_requirement(requirement=line_continuation_buffer).name != removed_dependency: + if parse_canonical_requirement(requirement=line_continuation_buffer).name != removed_dependency: result += line_continuation_buffer line_continuation_buffer = "" elif not line.strip() or line.strip().startswith("#"): result += line elif line.endswith("\\"): line_continuation_buffer = line - elif safe_parse_requirement(requirement=line).name != removed_dependency: + elif parse_canonical_requirement(requirement=line).name != removed_dependency: result += line else: # Dependency matches `remove_dep_on_pkg` => Remove line from result diff --git a/src/inmanta/module.py b/src/inmanta/module.py index d54abc5600..c4b3553d0c 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -59,7 +59,8 @@ from inmanta.ast.blocks import BasicBlock from inmanta.ast.statements import BiStatement, DefinitionStatement, DynamicStatement, Statement from inmanta.ast.statements.define import DefineImport -from inmanta.env import assert_pip_has_source, safe_parse_requirement, safe_parse_requirements +from inmanta.env import assert_pip_has_source +from inmanta.util import parse_canonical_requirements, parse_canonical_requirement from inmanta.file_parser import PreservativeYamlParser, RequirementsTxtParser from inmanta.parser import plyInmantaParser from inmanta.parser.plyInmantaParser import cache_manager @@ -151,7 +152,7 @@ def parse(cls: type[TInmantaModuleRequirement], spec: str) -> TInmantaModuleRequ ) if "-" in spec: raise ValueError("Invalid Inmanta module requirement: Inmanta module names use '_', not '-'.") - return cls(safe_parse_requirement(requirement=spec)) + return cls(parse_canonical_requirement(requirement=spec)) def get_python_package_requirement(self) -> Requirement: """ @@ -160,7 +161,7 @@ def get_python_package_requirement(self) -> Requirement: module_name = self.project_name pkg_name = ModuleV2Source.get_package_name_for(module_name) pkg_req_str = str(self).replace(module_name, pkg_name, 1) # Replace max 1 occurrence - return safe_parse_requirement(requirement=pkg_req_str) + return parse_canonical_requirement(requirement=pkg_req_str) class CompilerExceptionWithExtendedTrace(CompilerException): @@ -728,7 +729,7 @@ def install(self, project: "Project", module_spec: list[InmantaModuleRequirement # These could be constraints (-c) as well, but that requires additional sanitation # Because for pip not every valid -r is a valid -c current_requires = project.get_strict_python_requirements_as_list() - requirements += safe_parse_requirements(current_requires) + requirements += parse_canonical_requirements(current_requires) if preinstalled is not None: # log warning if preinstalled version does not match constraints @@ -2127,7 +2128,7 @@ def install_modules(self, *, bypass_module_cache: bool = False, update_dependenc self.verify_module_version_compatibility() # do python install - pyreq: Sequence[Requirement] = safe_parse_requirements(self.collect_python_requirements()) + pyreq: Sequence[Requirement] = parse_canonical_requirements(self.collect_python_requirements()) if len(pyreq) > 0: # upgrade both direct and transitive module dependencies: eager upgrade strategy @@ -2545,7 +2546,7 @@ def verify_python_requires(self) -> None: Verifies no incompatibilities exist within the Python environment with respect to installed module v2 requirements. """ if self.strict_deps_check: - constraints: Sequence[Requirement] = safe_parse_requirements(self.collect_python_requirements()) + constraints: Sequence[Requirement] = parse_canonical_requirements(self.collect_python_requirements()) env.ActiveEnv.check(strict_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*"), constraints=constraints) else: if not env.ActiveEnv.check_legacy(in_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*")): @@ -2676,7 +2677,7 @@ def requires(self) -> "List[InmantaModuleRequirement]": # filter on import stmt reqs = [] for spec in self._metadata.requires: - req = [safe_parse_requirement(requirement=spec)] + req = [parse_canonical_requirement(requirement=spec)] if len(req) > 1: print(f"Module file for {self._path} has bad line in requirements specification {spec}") reqe = InmantaModuleRequirement(req[0]) @@ -2813,7 +2814,7 @@ def requires(self) -> "List[InmantaModuleRequirement]": """ reqs = [] for spec in self.get_module_requirements(): - req = [safe_parse_requirement(requirement=spec)] + req = [parse_canonical_requirement(requirement=spec)] if len(req) > 1: print(f"Module file for {self._path} has bad line in requirements specification {spec}") reqe = InmantaModuleRequirement(req[0]) @@ -3435,7 +3436,7 @@ def add_module_requirement_persistent(self, requirement: InmantaModuleRequiremen new_install_requires = [ r for r in config_parser.get("options", "install_requires").split("\n") - if r and safe_parse_requirement(requirement=r).name != python_pkg_requirement.name + if r and parse_canonical_requirement(requirement=r).name != python_pkg_requirement.name ] new_install_requires.append(str(python_pkg_requirement)) else: diff --git a/src/inmanta/moduletool.py b/src/inmanta/moduletool.py index 323f5fe679..6af7212250 100644 --- a/src/inmanta/moduletool.py +++ b/src/inmanta/moduletool.py @@ -55,7 +55,7 @@ from inmanta.ast import CompilerException from inmanta.command import CLIException, ShowUsageException from inmanta.const import CF_CACHE_DIR, MAX_UPDATE_ATTEMPT -from inmanta.env import safe_parse_requirements +from inmanta.util import parse_canonical_requirements from inmanta.module import ( DummyProject, FreezeOperator, @@ -485,7 +485,7 @@ def do_update(specs: Mapping[str, Sequence[InmantaModuleRequirement]], modules: # Because for pip not every valid -r is a valid -c current_requires = my_project.get_strict_python_requirements_as_list() env.process_env.install_for_config( - v2_python_specs + safe_parse_requirements(current_requires), + v2_python_specs + parse_canonical_requirements(current_requires), my_project.metadata.pip, upgrade=True, ) diff --git a/src/inmanta/util/__init__.py b/src/inmanta/util/__init__.py index c6070f1f88..1a66705164 100644 --- a/src/inmanta/util/__init__.py +++ b/src/inmanta/util/__init__.py @@ -27,6 +27,7 @@ import itertools import logging import os +import pathlib import socket import threading import time @@ -40,15 +41,18 @@ from dataclasses import dataclass from logging import Logger from types import TracebackType -from typing import BinaryIO, Callable, Generic, Optional, TypeVar, Union +from typing import BinaryIO, Callable, Generic, Optional, Sequence, TypeVar, Union import asyncpg from tornado import gen +import packaging +import packaging.utils from crontab import CronTab from inmanta import COMPILER_VERSION, const from inmanta.stable_api import stable_api from inmanta.types import JsonType, PrimitiveTypes, ReturnTypes +from packaging.requirements import Requirement from pydantic_core import Url LOGGER = logging.getLogger(__name__) @@ -863,3 +867,61 @@ def check_for_pool_exhaustion(self) -> None: def _reset_counter(self) -> None: self._exhausted_pool_events_count = 0 + + +def remove_comment_part(to_clean: str) -> str: + """ + Remove the comment part of a given string and ensure that the lenght of the string is greater than 0 + + :param to_clean: The string to clean + :return: A cleaned string + """ + drop_comment, _, _ = to_clean.partition("#") + if len(drop_comment) == 0: + raise ValueError("The name of the requirement cannot be an empty string!") + return drop_comment + + +def parse_canonical_requirement(requirement: str) -> Requirement: + """ + To be able to compare requirements, we need to make sure that every requirement's name is canonicalized otherwise issues + could arise when checking if packages are installed in a particular Venv. + + :param requirement: The requirement's name + :return: A new requirement instance + """ + # Packaging Requirement is not able to parse requirements with comment. Therefore, we need to remove the `comment` part + drop_comment = remove_comment_part(to_clean=requirement) + # We canonicalize the name of the requirement to be able to compare requirements and check if the requirement is + # already installed + # This instance is considered as doomed because we are not supposed to modify directly the instance + doomed_requirement_instance = Requirement(requirement_string=drop_comment) + canonical_name = str(doomed_requirement_instance).replace(doomed_requirement_instance.name, packaging.utils.canonicalize_name(doomed_requirement_instance.name)) + requirement_instance = Requirement(requirement_string=canonical_name) + return requirement_instance + + +def parse_canonical_requirements(requirements: Sequence[str]) -> Sequence[Requirement]: + """ + To be able to compare requirements, we need to make sure that every requirement's name is canonicalized otherwise issues + could arise when checking if packages are installed in a particular Venv. + + :param requirements: The names of the different requirements + :return: Sequence[Requirement] + """ + return [parse_canonical_requirement(requirement=e) for e in requirements] + + +def parse_canonical_requirements_from_file(file_path: pathlib.Path) -> Sequence[Requirement]: + """ + To be able to compare requirements, we need to make sure that every requirement's name is canonicalized otherwise issues + could arise when checking if packages are installed in a particular Venv. + + :param file_path: The path to the read the requirements from + :return: Sequence[Requirement] + """ + requirements = [] + with open(file_path) as f: + for line in f.readlines(): + requirements.append(parse_canonical_requirement(line)) + return requirements diff --git a/tests/compiler/test_basics.py b/tests/compiler/test_basics.py index a65518e8ab..234a4753a0 100644 --- a/tests/compiler/test_basics.py +++ b/tests/compiler/test_basics.py @@ -26,7 +26,7 @@ from inmanta import compiler, const, module from inmanta.ast import DoubleSetException, RuntimeException -from inmanta.env import safe_parse_requirement +from inmanta.env import parse_canonical_requirement from inmanta.module import InstallMode from inmanta.plugins import PluginDeprecationWarning from packaging import version @@ -744,7 +744,7 @@ def test_safe_requirement(name) -> None: Ensure that empty name requirements are not allowed in `Requirement` """ with pytest.raises(ValueError): - safe_parse_requirement(requirement=name) + parse_canonical_requirement(requirement=name) @pytest.mark.slowtest diff --git a/tests/conftest.py b/tests/conftest.py index 390e2f58f0..8a665a7f83 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -120,7 +120,8 @@ from inmanta.ast import CompilerException from inmanta.data.schema import SCHEMA_VERSION_TABLE from inmanta.db import util as db_util -from inmanta.env import CommandRunner, LocalPackagePath, VirtualEnv, mock_process_env, safe_parse_requirement +from inmanta.env import CommandRunner, LocalPackagePath, VirtualEnv, mock_process_env +from inmanta.util import parse_canonical_requirement from inmanta.export import ResourceDict, cfg_env, unknown_parameters from inmanta.module import InmantaModuleRequirement, InstallMode, Project, RelationPrecedenceRule from inmanta.moduletool import DefaultIsolatedEnvCached, ModuleTool, V2ModuleBuilder @@ -1919,10 +1920,10 @@ def index_with_pkgs_containing_optional_deps() -> str: path=os.path.join(tmpdirname, "pkg"), publish_index=pip_index, optional_dependencies={ - "optional-a": [safe_parse_requirement(requirement="dep-a")], + "optional-a": [parse_canonical_requirement(requirement="dep-a")], "optional-b": [ - safe_parse_requirement(requirement="dep-b"), - safe_parse_requirement(requirement="dep-c"), + parse_canonical_requirement(requirement="dep-b"), + parse_canonical_requirement(requirement="dep-c"), ], }, ) diff --git a/tests/moduletool/test_add.py b/tests/moduletool/test_add.py index cb93dc2aa7..98d5ca8873 100644 --- a/tests/moduletool/test_add.py +++ b/tests/moduletool/test_add.py @@ -25,7 +25,7 @@ import pytest from inmanta.command import CLIException -from inmanta.env import process_env, safe_parse_requirement +from inmanta.env import process_env, parse_canonical_requirement from inmanta.module import ModuleV1, ModuleV1Metadata, ModuleV2, ModuleV2Source, Project, ProjectMetadata from inmanta.moduletool import ModuleTool from packaging.version import Version @@ -88,7 +88,7 @@ def test_module_add_v2_module_to_project( dest_dir=os.path.join(tmpdir, f"elaboratev2module-v{version}"), new_version=Version(version), publish_index=pip_index, - new_extras={"optional": [safe_parse_requirement(requirement="inmanta-module-minimalv2module")]}, + new_extras={"optional": [parse_canonical_requirement(requirement="inmanta-module-minimalv2module")]}, ) # Create project diff --git a/tests/moduletool/test_convert_v1_v2.py b/tests/moduletool/test_convert_v1_v2.py index c634410785..a7a1f6bf50 100644 --- a/tests/moduletool/test_convert_v1_v2.py +++ b/tests/moduletool/test_convert_v1_v2.py @@ -30,7 +30,7 @@ import toml from inmanta import moduletool from inmanta.command import CLIException -from inmanta.env import safe_parse_requirements +from inmanta.env import parse_canonical_requirements from inmanta.module import DummyProject, ModuleV1, ModuleV2, ModuleV2Metadata from inmanta.moduletool import ModuleConverter, ModuleVersionException from packaging import version @@ -114,7 +114,7 @@ def test_issue_3159_conversion_std_module_add_self_to_dependencies(tmpdir): parser = configparser.ConfigParser() parser.read(setup_cfg_file) assert parser.has_option("options", "install_requires") - install_requires = safe_parse_requirements(parser.get("options", "install_requires").split("\n")) + install_requires = parse_canonical_requirements(parser.get("options", "install_requires").split("\n")) pkg_names = [r.name for r in install_requires] assert "inmanta-module-std" not in pkg_names diff --git a/tests/moduletool/test_install.py b/tests/moduletool/test_install.py index 2f2220f9e4..0d65a97c91 100644 --- a/tests/moduletool/test_install.py +++ b/tests/moduletool/test_install.py @@ -35,7 +35,7 @@ from inmanta.ast import CompilerException from inmanta.command import CLIException from inmanta.config import Config -from inmanta.env import CommandRunner, ConflictingRequirements, PipConfig, safe_parse_requirement +from inmanta.env import CommandRunner, ConflictingRequirements, PipConfig, parse_canonical_requirement from inmanta.module import InmantaModuleRequirement, InstallMode, ModuleLoadingException, ModuleNotFoundException from inmanta.moduletool import DummyProject, ModuleConverter, ModuleTool, ProjectTool from moduletool.common import BadModProvider, install_project @@ -251,14 +251,14 @@ def test_module_install_conflicting_requirements(tmpdir: py.path.local, snippetc os.path.join(modules_v2_dir, "minimalv2module"), os.path.join(str(tmpdir), "modone"), new_name="modone", - new_requirements=[safe_parse_requirement(requirement="lorem~=0.0.1")], + new_requirements=[parse_canonical_requirement(requirement="lorem~=0.0.1")], install=True, ) module_from_template( os.path.join(modules_v2_dir, "minimalv2module"), os.path.join(str(tmpdir), "modtwo"), new_name="modtwo", - new_requirements=[safe_parse_requirement(requirement="lorem~=0.1.0")], + new_requirements=[parse_canonical_requirement(requirement="lorem~=0.1.0")], install=True, ) @@ -508,7 +508,7 @@ def test_project_install( index_url=local_module_package_index, # We add tornado, as there is a code path in update for the case where the project has python requires python_requires=["tornado"] - + [safe_parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names], + + [parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names], install_project=False, ) @@ -541,7 +541,7 @@ def test_project_install( autostd=False, python_package_sources=[local_module_package_index], python_requires=[ - safe_parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names + parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names ] + ["lorem"], install_project=False, @@ -677,7 +677,7 @@ def test_project_install_modules_cache_invalid( index_url=index.url, extra_index_url=[local_module_package_index], # make sure main module gets installed, pulling in newest version of dependency module - python_requires=[safe_parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(main_module))], + python_requires=[parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for(main_module))], ) # populate project.modules[dependency_module] to force the error conditions in this simplified example @@ -762,7 +762,7 @@ def test_project_install_incompatible_versions( install_project=False, add_to_module_path=[v1_modules_path], index_url=index.url, - python_requires=[safe_parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(v2_mod_name))], + python_requires=[parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for(v2_mod_name))], ) # install project @@ -815,14 +815,14 @@ def test_project_install_incompatible_dependencies( v2_template_path, os.path.join(str(tmpdir), "v2mod2"), new_name="v2mod2", - new_requirements=[safe_parse_requirement(requirement="inmanta-module-v2mod1~=1.0.0")], + new_requirements=[parse_canonical_requirement(requirement="inmanta-module-v2mod1~=1.0.0")], publish_index=index, ) v2mod3: module.ModuleV2Metadata = module_from_template( v2_template_path, os.path.join(str(tmpdir), "v2mod3"), new_name="v2mod3", - new_requirements=[safe_parse_requirement(requirement="inmanta-module-v2mod1~=2.0.0")], + new_requirements=[parse_canonical_requirement(requirement="inmanta-module-v2mod1~=2.0.0")], publish_index=index, ) @@ -836,7 +836,7 @@ def test_project_install_incompatible_dependencies( install_project=False, index_url=index.url, python_requires=[ - safe_parse_requirement( + parse_canonical_requirement( requirement=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod2, v2mod3] @@ -920,7 +920,7 @@ def test_install_from_index_dont_leak_pip_index( # Installing a V2 module requires a python package source. index_url="unknown", python_requires=[ - safe_parse_requirement( + parse_canonical_requirement( requirement=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod1] @@ -982,7 +982,7 @@ def test_install_with_use_config( index_url=index.url if not use_pip_config else None, use_pip_config_file=use_pip_config, python_requires=[ - safe_parse_requirement( + parse_canonical_requirement( requirement=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod1] @@ -1051,7 +1051,7 @@ def test_install_with_use_config_extra_index( extra_index_url=[index2.url], use_pip_config_file=True, python_requires=[ - safe_parse_requirement( + parse_canonical_requirement( requirement=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod1, v2mod2] @@ -1087,7 +1087,7 @@ def test_install_with_use_config_but_PIP_CONFIG_FILE_not_set( autostd=False, install_project=False, use_pip_config_file=True, - python_requires=[safe_parse_requirement(requirement="inmanta-module-dummy-module")], + python_requires=[parse_canonical_requirement(requirement="inmanta-module-dummy-module")], ) # install project @@ -1204,7 +1204,7 @@ def test_install_project_with_install_mode_master(tmpdir: py.path.local, snippet autostd=False, install_project=False, add_to_module_path=[str(tmpdir)], - project_requires=[InmantaModuleRequirement(safe_parse_requirement(requirement="mod11==3.2.1"))], + project_requires=[InmantaModuleRequirement(parse_canonical_requirement(requirement="mod11==3.2.1"))], install_mode=InstallMode.master, ) @@ -1232,7 +1232,7 @@ def test_module_install_logging(local_module_package_index: str, snippetcompiler v2_module = "minimalv2module" - v2_requirements = [safe_parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(v2_module))] + v2_requirements = [parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for(v2_module))] # set up project and modules project: module.Project = snippetcompiler_clean.setup_for_snippet( @@ -1333,7 +1333,7 @@ def test_pip_output(local_module_package_index: str, snippetcompiler_clean, capl ) modules = ["modone", "modtwo"] - v2_requirements = [safe_parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in modules] + v2_requirements = [parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in modules] snippetcompiler_clean.setup_for_snippet( f""" @@ -1419,7 +1419,7 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[safe_parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module"))], install_project=True, ) log_contains( @@ -1451,7 +1451,7 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[safe_parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module"))], install_project=True, ) @@ -1484,7 +1484,7 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[safe_parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module"))], install_project=True, ) log_contains( @@ -1553,7 +1553,7 @@ def test_version_snapshot(local_module_package_index: str, snippetcompiler_clean autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[safe_parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("module_b"))], + python_requires=[parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for("module_b"))], install_project=True, ) @@ -1579,7 +1579,7 @@ def test_version_snapshot(local_module_package_index: str, snippetcompiler_clean autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[safe_parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("module_c"))], + python_requires=[parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for("module_c"))], install_project=True, ) @@ -1650,7 +1650,7 @@ def test_constraints_logging_v2(modules_v2_dir, tmpdir, caplog, snippetcompiler_ index_url=local_module_package_index, extra_index_url=[index.url], python_requires=[ - safe_parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) + parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in ["module_b", "module_a"] ], install_project=True, diff --git a/tests/moduletool/test_update.py b/tests/moduletool/test_update.py index fc25fec1a1..63978fafc4 100644 --- a/tests/moduletool/test_update.py +++ b/tests/moduletool/test_update.py @@ -23,7 +23,7 @@ from inmanta.config import Config from inmanta.data.model import PipConfig -from inmanta.env import LocalPackagePath, process_env, safe_parse_requirement, safe_parse_requirements +from inmanta.env import LocalPackagePath, process_env, parse_canonical_requirement, parse_canonical_requirements from inmanta.module import InmantaModuleRequirement, InstallMode, ModuleV1, ModuleV2Source from inmanta.moduletool import ProjectTool from inmanta.parser import ParserException @@ -125,7 +125,7 @@ def assert_version_installed(module_name: str, version: str) -> None: new_version=Version(current_version), new_name=module_name, new_requirements=( - [InmantaModuleRequirement(safe_parse_requirement(requirement="module2<3.0.0"))] + [InmantaModuleRequirement(parse_canonical_requirement(requirement="module2<3.0.0"))] if module_name == "module1" else None ), @@ -142,7 +142,7 @@ def assert_version_installed(module_name: str, version: str) -> None: # Add a dependency on module2, without setting an explicit version constraint. Later version of module1 # do set a version constraint on the dependency on module2. This way it is verified whether the module update # command takes into account the version constraints set in a new version of a module. - new_requirements=[InmantaModuleRequirement(safe_parse_requirement(requirement="module2"))], + new_requirements=[InmantaModuleRequirement(parse_canonical_requirement(requirement="module2"))], install=False, publish_index=pip_index, new_content_init_cf="entity" if corrupt_module else None, # Introduce syntax error in the module @@ -242,7 +242,7 @@ def test_module_update_dependencies( "b", Version(v), str(tmpdir.join(f"b-{v}")), - requirements=[safe_parse_requirement(requirement="c")], + requirements=[parse_canonical_requirement(requirement="c")], publish_index=index, ) for v in ("1.0.0", "2.0.0"): @@ -258,7 +258,7 @@ def test_module_update_dependencies( # install b-1.0.0 and c-1.0.0 process_env.install_for_config( - [safe_parse_requirement(requirement=req) for req in ("b==1.0.0", "c==1.0.0")], + [parse_canonical_requirement(requirement=req) for req in ("b==1.0.0", "c==1.0.0")], config=PipConfig( index_url=index.url, use_system_config=False, @@ -270,7 +270,7 @@ def test_module_update_dependencies( source_dir=os.path.join(modules_dir, "minimalv1module"), dest_dir=str(tmpdir.join("modules", "my_mod")), new_name="my_mod", - new_requirements=safe_parse_requirements(["a", "b~=1.0.0"]), + new_requirements=parse_canonical_requirements(["a", "b~=1.0.0"]), ) # run `inmanta project update` without running install first diff --git a/tests/server/test_compilerservice.py b/tests/server/test_compilerservice.py index 921d0cdc43..c0787b65c5 100644 --- a/tests/server/test_compilerservice.py +++ b/tests/server/test_compilerservice.py @@ -41,7 +41,7 @@ from inmanta.const import INMANTA_REMOVED_SET_ID, ParameterSource from inmanta.data import APILIMIT, Compile, Report from inmanta.data.model import PipConfig -from inmanta.env import PythonEnvironment, safe_parse_requirement +from inmanta.env import PythonEnvironment, parse_canonical_requirement from inmanta.export import cfg_env from inmanta.protocol import Result from inmanta.server import SLICE_COMPILER, SLICE_SERVER, protocol @@ -1806,7 +1806,7 @@ def patch_get_protected_inmanta_packages(): venv = PythonEnvironment(env_path=venv_path) assert name_protected_pkg not in venv.get_installed_packages() venv.install_for_config( - requirements=[safe_parse_requirement(requirement=name_protected_pkg)], + requirements=[parse_canonical_requirement(requirement=name_protected_pkg)], config=PipConfig( index_url=local_module_package_index, ), diff --git a/tests/test_env.py b/tests/test_env.py index 7d8950eba5..70d3b911b1 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -35,7 +35,8 @@ from inmanta import env, loader, module from inmanta.data.model import PipConfig -from inmanta.env import Pip, Requirement, safe_parse_requirement +from inmanta.env import Pip, Requirement +from inmanta.util import parse_canonical_requirement from packaging import version from utils import LogSequence, PipIndex, create_python_package @@ -176,7 +177,7 @@ def test_gen_req_file(): # make sure they all parse for req in reqs: - safe_parse_requirement(requirement=req) + parse_canonical_requirement(requirement=req) def test_environment_python_version_multi_digit(tmpdir: py.path.local) -> None: @@ -205,7 +206,7 @@ def test_process_env_install_from_index( package_name: str = "more-itertools" assert package_name not in env.process_env.get_installed_packages() env.process_env.install_for_config( - [safe_parse_requirement(requirement=package_name + (f"=={version}" if version is not None else ""))], + [parse_canonical_requirement(requirement=package_name + (f"=={version}" if version is not None else ""))], config=PipConfig( use_system_config=True, # we need an upstream for some packages ), @@ -220,7 +221,7 @@ def test_process_env_install_from_index( # It should hit the cache there and return here. # Cheap and fast test env.process_env.install_from_index( - [safe_parse_requirement(requirement=package_name + (f"=={version}" if version is not None else ""))], + [parse_canonical_requirement(requirement=package_name + (f"=={version}" if version is not None else ""))], use_pip_config=True, ) @@ -270,7 +271,7 @@ def test_process_env_install_from_index_not_found_env_var( with pytest.raises(env.PackageNotFound, match=re.escape(expected)): env.process_env.install_for_config( - [safe_parse_requirement(requirement="this-package-does-not-exist")], + [parse_canonical_requirement(requirement="this-package-does-not-exist")], config=PipConfig( index_url=index_urls[0], # The first element should only be passed to the index_url. If there are indexes in the environment @@ -307,7 +308,7 @@ def test_process_env_install_no_index(tmpdir: py.path.local, monkeypatch, use_sy with pytest.raises(env.PackageNotFound, match=re.escape(expected)): env.process_env.install_for_config( - requirements=[safe_parse_requirement(requirement="this-package-does-not-exist")], + requirements=[parse_canonical_requirement(requirement="this-package-does-not-exist")], paths=[env.LocalPackagePath(path=str(tmpdir))], config=PipConfig(use_system_config=use_system_config), ) @@ -324,7 +325,7 @@ def test_process_env_install_from_index_conflicting_reqs( package_name: str = "more-itertools" with pytest.raises(env.ConflictingRequirements) as e: env.process_env.install_for_config( - [safe_parse_requirement(requirement=f"{package_name}{version}") for version in [">8.5", "<=8"]], + [parse_canonical_requirement(requirement=f"{package_name}{version}") for version in [">8.5", "<=8"]], config=PipConfig( use_system_config=True, # we need an upstream for some packages ), @@ -394,7 +395,7 @@ def test_active_env_get_module_file( loader.PluginModuleFinder.configure_module_finder([os.path.join(str(tmpdir), "libs")]) assert env.ActiveEnv.get_module_file(module_name) is None - env.process_env.install_for_config([safe_parse_requirement(requirement=package_name)], pip_config) + env.process_env.install_for_config([parse_canonical_requirement(requirement=package_name)], pip_config) assert package_name in env.process_env.get_installed_packages() module_info: Optional[tuple[Optional[str], Loader]] = env.ActiveEnv.get_module_file(module_name) assert module_info is not None @@ -540,7 +541,7 @@ def assert_all_checks(expect_test: tuple[bool, str] = (True, ""), expect_nonext: create_install_package( "test-package-two", version.Version("1.0.0"), - [safe_parse_requirement(requirement="test-package-one~=1.0")], + [parse_canonical_requirement(requirement="test-package-one~=1.0")], local_module_package_index, ) assert_all_checks() @@ -561,7 +562,7 @@ def test_active_env_check_constraints(caplog, tmpvenv_active_inherit: str, local """ caplog.set_level(logging.WARNING) in_scope: Pattern[str] = re.compile("test-package-.*") - constraints: list[Requirement] = [safe_parse_requirement(requirement="test-package-one~=1.0")] + constraints: list[Requirement] = [parse_canonical_requirement(requirement="test-package-one~=1.0")] env.ActiveEnv.check(in_scope) @@ -580,7 +581,7 @@ def test_active_env_check_constraints(caplog, tmpvenv_active_inherit: str, local create_install_package( "ext-package-one", version.Version("1.0.0"), - [safe_parse_requirement(requirement="test-package-one==1.0")], + [parse_canonical_requirement(requirement="test-package-one==1.0")], local_module_package_index, ) env.ActiveEnv.check(in_scope, constraints) @@ -611,7 +612,7 @@ def test_override_inmanta_package(tmpvenv_active_inherit: env.VirtualEnv) -> Non installed_pkgs = tmpvenv_active_inherit.get_installed_packages() assert "inmanta-core" in installed_pkgs, "The inmanta-core package should be installed to run the tests" - inmanta_requirements = safe_parse_requirement(requirement="inmanta-core==4.0.0") + inmanta_requirements = parse_canonical_requirement(requirement="inmanta-core==4.0.0") with pytest.raises(env.ConflictingRequirements) as excinfo: tmpvenv_active_inherit.install_for_config( requirements=[inmanta_requirements], @@ -651,13 +652,13 @@ def test_cache_on_active_env(tmpvenv_active_inherit: env.ActiveEnv, local_module """ def _assert_install(requirement: str, installed: bool) -> None: - parsed_requirement = safe_parse_requirement(requirement=requirement) + parsed_requirement = parse_canonical_requirement(requirement=requirement) for r in [requirement, parsed_requirement]: assert tmpvenv_active_inherit.are_installed(requirements=[r]) == installed _assert_install("inmanta-module-elaboratev2module==1.2.3", installed=False) tmpvenv_active_inherit.install_for_config( - requirements=[safe_parse_requirement(requirement="inmanta-module-elaboratev2module==1.2.3")], + requirements=[parse_canonical_requirement(requirement="inmanta-module-elaboratev2module==1.2.3")], config=PipConfig( index_url=local_module_package_index, ), @@ -701,7 +702,7 @@ def test_are_installed_dependency_cycle_on_extra(tmpdir, tmpvenv_active_inherit: path=os.path.join(tmpdir, "pkg"), publish_index=pip_index, optional_dependencies={ - "optional-pkg": [safe_parse_requirement(requirement="dep[optional-dep]")], + "optional-pkg": [parse_canonical_requirement(requirement="dep[optional-dep]")], }, ) create_python_package( @@ -710,11 +711,11 @@ def test_are_installed_dependency_cycle_on_extra(tmpdir, tmpvenv_active_inherit: path=os.path.join(tmpdir, "dep"), publish_index=pip_index, optional_dependencies={ - "optional-dep": [safe_parse_requirement(requirement="pkg[optional-pkg]")], + "optional-dep": [parse_canonical_requirement(requirement="pkg[optional-pkg]")], }, ) - requirements = [safe_parse_requirement(requirement="pkg[optional-pkg]")] + requirements = [parse_canonical_requirement(requirement="pkg[optional-pkg]")] tmpvenv_active_inherit.install_for_config( requirements=requirements, config=PipConfig( diff --git a/tests/test_file_parser.py b/tests/test_file_parser.py index d9fc160583..45bd373609 100644 --- a/tests/test_file_parser.py +++ b/tests/test_file_parser.py @@ -19,7 +19,7 @@ import os import packaging.requirements -from inmanta.env import safe_parse_requirements +from inmanta.env import parse_canonical_requirements from inmanta.file_parser import RequirementsTxtParser @@ -41,7 +41,7 @@ def test_requirements_txt_parser(tmpdir) -> None: expected_requirements = ["test==1.2.3", "other-dep~=2.0.0", "third-dep<5.0.0", "splitteddep", "Capital"] requirements: list[packaging.requirements.Requirement] = RequirementsTxtParser().parse(requirements_txt_file) - assert requirements == safe_parse_requirements(expected_requirements) + assert requirements == parse_canonical_requirements(expected_requirements) requirements_as_str = RequirementsTxtParser.parse_requirements_as_strs(requirements_txt_file) assert requirements_as_str == expected_requirements diff --git a/tests/test_module_loader.py b/tests/test_module_loader.py index b4b2cae886..3900573e62 100644 --- a/tests/test_module_loader.py +++ b/tests/test_module_loader.py @@ -39,7 +39,7 @@ PackageNotFound, Requirement, process_env, - safe_parse_requirement, + parse_canonical_requirement, ) from inmanta.module import ( DummyProject, @@ -352,7 +352,7 @@ def test_load_module_recursive_v2_module_depends_on_v1( project = snippetcompiler.setup_for_snippet( snippet="import v2_depends_on_v1", index_url=local_module_package_index, - python_requires=[safe_parse_requirement(requirement="inmanta-module-v2-depends-on-v1")], + python_requires=[parse_canonical_requirement(requirement="inmanta-module-v2-depends-on-v1")], install_project=False, ) if preload_v1_module: @@ -379,7 +379,7 @@ def test_load_module_recursive_complex_module_dependencies(local_module_package_ snippet="import complex_module_dependencies_mod1", autostd=False, index_url=local_module_package_index, - python_requires=[safe_parse_requirement(requirement="inmanta-module-complex-module-dependencies-mod1")], + python_requires=[parse_canonical_requirement(requirement="inmanta-module-complex-module-dependencies-mod1")], install_project=False, ) assert "complex_module_dependencies_mod1" not in project.modules @@ -418,7 +418,7 @@ def load(requires: Optional[list[Requirement]] = None) -> None: with pytest.raises(ModuleLoadingException, match=f"Failed to load module {module_name}"): load() # assert that it doesn't raise an error with explicit requirements set - load([safe_parse_requirement(requirement=ModuleV2Source.get_package_name_for(module_name))]) + load([parse_canonical_requirement(requirement=ModuleV2Source.get_package_name_for(module_name))]) @pytest.mark.parametrize("v1", [True, False]) @@ -476,7 +476,7 @@ def test_load_import_based_v2_module( # make sure that even listing the requirement in project.yml does not suffice project_requires=[InmantaModuleRequirement.parse(dependency_module_name)], python_requires=( - [] if v1 else [safe_parse_requirement(requirement=ModuleV2Source.get_package_name_for(main_module_name))] + [] if v1 else [parse_canonical_requirement(requirement=ModuleV2Source.get_package_name_for(main_module_name))] ), ) @@ -619,7 +619,7 @@ def test_project_requirements_dont_overwrite_core_requirements_source( module_from_template( os.path.join(modules_v2_dir, module_name), module_path, - new_requirements=[safe_parse_requirement(requirement="Jinja2==2.11.3")], + new_requirements=[parse_canonical_requirement(requirement="Jinja2==2.11.3")], ) # Activate the snippetcompiler venv @@ -663,7 +663,7 @@ def test_project_requirements_dont_overwrite_core_requirements_index( module_from_template( os.path.join(modules_v2_dir, module_name), module_path, - new_requirements=[safe_parse_requirement(requirement="Jinja2==2.11.3")], + new_requirements=[parse_canonical_requirement(requirement="Jinja2==2.11.3")], publish_index=index, ) @@ -717,7 +717,7 @@ def test_module_conflicting_dependencies_with_v2_modules( "y", Version("1.0.0"), str(tmpdir.join("y-1.0.0")), - requirements=[safe_parse_requirement(requirement="x~=1.0.0")], + requirements=[parse_canonical_requirement(requirement="x~=1.0.0")], publish_index=index, ) @@ -727,7 +727,7 @@ def test_module_conflicting_dependencies_with_v2_modules( module_from_template( os.path.join(modules_v2_dir, module_name1), module_path1, - new_requirements=[safe_parse_requirement(requirement="y~=1.0.0")], + new_requirements=[parse_canonical_requirement(requirement="y~=1.0.0")], publish_index=index, ) @@ -738,7 +738,7 @@ def test_module_conflicting_dependencies_with_v2_modules( os.path.join(modules_v2_dir, "minimalv2module"), module_path2, new_name="minimalv2module2", - new_requirements=[safe_parse_requirement(requirement="x~=2.0.0")], + new_requirements=[parse_canonical_requirement(requirement="x~=2.0.0")], publish_index=index, ) @@ -798,7 +798,7 @@ def test_module_conflicting_dependencies_with_v1_module( os.path.join(modules_dir, module_name1), module_path1, new_name="modulev1", - new_requirements=[safe_parse_requirement(requirement="y~=1.0.0")], + new_requirements=[parse_canonical_requirement(requirement="y~=1.0.0")], ) # Create the second module @@ -807,7 +807,7 @@ def test_module_conflicting_dependencies_with_v1_module( module_from_template( os.path.join(modules_v2_dir, module_name2), module_path2, - new_requirements=[safe_parse_requirement(requirement="y~=2.0.0")], + new_requirements=[parse_canonical_requirement(requirement="y~=2.0.0")], publish_index=index, ) @@ -855,7 +855,7 @@ def test_module_install_extra_on_project_level_v2_dep( new_name="mymod", new_requirements=[], new_extras={ - "myfeature": [safe_parse_requirement(requirement=package_name_extra)], + "myfeature": [parse_canonical_requirement(requirement=package_name_extra)], }, publish_index=index, ) @@ -904,7 +904,7 @@ def test_module_install_extra_on_dep_of_v2_module( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [safe_parse_requirement(requirement=package_name_extra)], + "myfeature": [parse_canonical_requirement(requirement=package_name_extra)], }, publish_index=index, ) @@ -924,7 +924,7 @@ def test_module_install_extra_on_dep_of_v2_module( install_project=True, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[safe_parse_requirement(requirement="inmanta-module-myv2mod")], + python_requires=[parse_canonical_requirement(requirement="inmanta-module-myv2mod")], autostd=False, ) @@ -961,7 +961,7 @@ def test_module_install_extra_on_dep_of_v1_module( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [safe_parse_requirement(requirement=package_name_extra)], + "myfeature": [parse_canonical_requirement(requirement=package_name_extra)], }, publish_index=index, ) @@ -1011,7 +1011,7 @@ def test_module_install_extra_on_project_level_v2_dep_update_scenario( new_name="mymod", new_requirements=[], new_extras={ - "myfeature": [safe_parse_requirement(requirement=package_name_extra)], + "myfeature": [parse_canonical_requirement(requirement=package_name_extra)], }, publish_index=index, ) @@ -1080,7 +1080,7 @@ def test_module_install_extra_on_dep_of_v2_module_update_scenario( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [safe_parse_requirement(requirement=package_name_extra)], + "myfeature": [parse_canonical_requirement(requirement=package_name_extra)], }, publish_index=index, ) @@ -1109,7 +1109,7 @@ def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> install_project=True, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[safe_parse_requirement(requirement="inmanta-module-myv2mod==1.0.0")], + python_requires=[parse_canonical_requirement(requirement="inmanta-module-myv2mod==1.0.0")], autostd=False, ) assert_installed(extra_installed=False) @@ -1131,7 +1131,7 @@ def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> install_project=not do_project_update, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[safe_parse_requirement(requirement="inmanta-module-myv2mod==2.0.0")], + python_requires=[parse_canonical_requirement(requirement="inmanta-module-myv2mod==2.0.0")], autostd=False, ) if do_project_update: @@ -1175,7 +1175,7 @@ def test_module_install_extra_on_dep_of_v1_module_update_scenario( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [safe_parse_requirement(requirement=package_name_extra)], + "myfeature": [parse_canonical_requirement(requirement=package_name_extra)], }, publish_index=index, ) @@ -1239,7 +1239,7 @@ async def test_v1_module_depends_on_third_party_dep_with_extra( os.path.join(tmpdir, "myv1mod"), new_name="myv1mod", new_content_init_cf="", - new_requirements=[safe_parse_requirement(requirement="pkg[optional-a]")], + new_requirements=[parse_canonical_requirement(requirement="pkg[optional-a]")], ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv1mod", @@ -1259,7 +1259,7 @@ async def test_v1_module_depends_on_third_party_dep_with_extra( os.path.join(tmpdir, "myv1mod"), new_name="myv1mod", new_content_init_cf="", - new_requirements=[safe_parse_requirement(requirement="pkg[optional-a,optional-b]")], + new_requirements=[parse_canonical_requirement(requirement="pkg[optional-a,optional-b]")], ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv1mod", @@ -1286,13 +1286,13 @@ async def test_v2_module_depends_on_third_party_dep_with_extra( str(tmpdir.join("myv2mod")), new_name="myv2mod", new_version=Version("1.0.0"), - new_requirements=[safe_parse_requirement(requirement="pkg[optional-a]")], + new_requirements=[parse_canonical_requirement(requirement="pkg[optional-a]")], publish_index=index, ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv2mod", install_project=True, - python_requires=[safe_parse_requirement(requirement="inmanta-module-myv2mod==1.0.0")], + python_requires=[parse_canonical_requirement(requirement="inmanta-module-myv2mod==1.0.0")], index_url=index.url, extra_index_url=[index_with_pkgs_containing_optional_deps], autostd=False, @@ -1308,13 +1308,13 @@ async def test_v2_module_depends_on_third_party_dep_with_extra( str(tmpdir.join("myv2mod")), new_name="myv2mod", new_version=Version("2.0.0"), - new_requirements=[safe_parse_requirement(requirement="pkg[optional-a,optional-b]")], + new_requirements=[parse_canonical_requirement(requirement="pkg[optional-a,optional-b]")], publish_index=index, ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv2mod", install_project=True, - python_requires=[safe_parse_requirement(requirement="inmanta-module-myv2mod==2.0.0")], + python_requires=[parse_canonical_requirement(requirement="inmanta-module-myv2mod==2.0.0")], index_url=index.url, extra_index_url=[index_with_pkgs_containing_optional_deps], autostd=False, From e6975c5533628fe78a9d36dd8ffd246b46eb8a79 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Mon, 16 Sep 2024 11:18:35 +0200 Subject: [PATCH 32/74] merge master + introduce new type --- src/inmanta/agent/executor.py | 4 +- src/inmanta/agent/in_process_executor.py | 5 +- src/inmanta/env.py | 66 ++++++++++++------------ src/inmanta/file_parser.py | 8 +-- src/inmanta/module.py | 33 ++++++------ src/inmanta/moduletool.py | 6 +-- src/inmanta/util/__init__.py | 24 +++++---- tests/conftest.py | 9 ++-- tests/moduletool/test_add.py | 2 +- tests/moduletool/test_install.py | 20 +++++-- tests/moduletool/test_update.py | 2 +- tests/test_env.py | 34 ++++++------ tests/test_module_loader.py | 9 +--- 13 files changed, 114 insertions(+), 108 deletions(-) diff --git a/src/inmanta/agent/executor.py b/src/inmanta/agent/executor.py index 7a2615e591..066009c89f 100644 --- a/src/inmanta/agent/executor.py +++ b/src/inmanta/agent/executor.py @@ -41,10 +41,10 @@ from inmanta.const import ResourceState from inmanta.data.model import PipConfig, ResourceIdStr, ResourceType, ResourceVersionIdStr from inmanta.env import PythonEnvironment -from inmanta.util import parse_canonical_requirements from inmanta.loader import ModuleSource from inmanta.resources import Id from inmanta.types import JsonType +from inmanta.util import parse_requirements LOGGER = logging.getLogger(__name__) @@ -296,7 +296,7 @@ async def create_and_install_environment(self, blueprint: EnvBlueprint) -> None: await asyncio.get_running_loop().run_in_executor(self.io_threadpool, self.init_env) if len(req): # install_for_config expects at least 1 requirement or a path to install await self.async_install_for_config( - requirements=list(parse_canonical_requirements(req)), + requirements=parse_requirements(req), config=blueprint.pip_config, ) diff --git a/src/inmanta/agent/in_process_executor.py b/src/inmanta/agent/in_process_executor.py index 520ab8ff7a..30e734c807 100644 --- a/src/inmanta/agent/in_process_executor.py +++ b/src/inmanta/agent/in_process_executor.py @@ -32,11 +32,10 @@ from inmanta.agent.handler import HandlerAPI, SkipResource from inmanta.const import ParameterSource, ResourceState from inmanta.data.model import AttributeStateChange, ResourceIdStr, ResourceVersionIdStr -from inmanta.util import parse_canonical_requirements from inmanta.loader import CodeLoader from inmanta.resources import Id, Resource from inmanta.types import Apireturn -from inmanta.util import NamedLock, join_threadpools +from inmanta.util import NamedLock, join_threadpools, parse_requirements class InProcessExecutor(executor.Executor, executor.AgentInstance): @@ -602,7 +601,7 @@ async def _install(self, blueprint: executor.ExecutorBlueprint) -> None: await loop.run_in_executor( self.thread_pool, self._env.install_for_config, - parse_canonical_requirements(blueprint.requirements), + parse_requirements(blueprint.requirements), blueprint.pip_config, ) await loop.run_in_executor(self.thread_pool, self._loader.deploy_version, blueprint.sources) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 7202b15df2..825aad8e3d 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -35,7 +35,6 @@ from functools import reduce from importlib.abc import Loader from importlib.machinery import ModuleSpec -from importlib.metadata import Distribution from itertools import chain from re import Pattern from subprocess import CalledProcessError @@ -43,8 +42,8 @@ from typing import Callable, NamedTuple, Optional, Tuple, TypeVar import pkg_resources -from pkg_resources import DistInfoDistribution, Distribution, Requirement, WorkingSet +import packaging.requirements import packaging.utils import packaging.version from inmanta import const, util @@ -53,7 +52,6 @@ from inmanta.server.bootloader import InmantaBootloader from inmanta.stable_api import stable_api from inmanta.util import strtobool -from packaging.requirements import Requirement LOGGER = logging.getLogger(__name__) LOGGER_PIP = logging.getLogger("inmanta.pip") # Use this logger to log pip commands or data related to pip commands. @@ -80,7 +78,7 @@ class VersionConflict: :param owner: The package from which the constraint originates """ - requirement: Requirement + requirement: packaging.requirements.Requirement installed_version: Optional[packaging.version.Version] = None owner: Optional[str] = None @@ -165,17 +163,17 @@ def get_advice(self) -> Optional[str]: ) -req_list = TypeVar("req_list", Sequence[str], Sequence[Requirement]) +req_list = TypeVar("req_list", Sequence[str], Sequence[packaging.requirements.Requirement]) class PythonWorkingSet: @classmethod - def _get_as_requirements_type(cls, requirements: req_list) -> Sequence[Requirement]: + def _get_as_requirements_type(cls, requirements: req_list) -> Sequence[packaging.requirements.Requirement]: """ Convert requirements from Union[Sequence[str], Sequence[Requirement]] to Sequence[Requirement] """ if isinstance(requirements[0], str): - return util.parse_canonical_requirements(requirements) + return util.parse_requirements(requirements) else: return requirements @@ -189,8 +187,8 @@ def are_installed(cls, requirements: req_list) -> bool: installed_packages: dict[str, packaging.version.Version] = cls.get_packages_in_working_set() def _are_installed_recursive( - reqs: Sequence[Requirement], - seen_requirements: Sequence[Requirement], + reqs: Sequence[packaging.requirements.Requirement], + seen_requirements: Sequence[packaging.requirements.Requirement], contained_in_extra: Optional[str] = None, ) -> bool: """ @@ -226,9 +224,9 @@ def _are_installed_recursive( if distribution is None: return False - pkgs_required_by_extra: set[Requirement] = set( - [util.parse_canonical_requirement(str(e)) for e in distribution.requires(extras=(extra,))] - ) - set([util.parse_canonical_requirement(str(e)) for e in distribution.requires(extras=())]) + pkgs_required_by_extra: set[packaging.requirements.Requirement] = set( + [util.parse_requirement(str(e)) for e in distribution.requires(extras=(extra,))] + ) - set([util.parse_requirement(str(e)) for e in distribution.requires(extras=())]) if not _are_installed_recursive( reqs=list(pkgs_required_by_extra), seen_requirements=list(seen_requirements) + list(reqs), @@ -237,7 +235,7 @@ def _are_installed_recursive( return False return True - reqs_as_requirements: Sequence[Requirement] = cls._get_as_requirements_type(requirements) + reqs_as_requirements: Sequence[packaging.requirements.Requirement] = cls._get_as_requirements_type(requirements) return _are_installed_recursive(reqs_as_requirements, seen_requirements=[]) @classmethod @@ -383,7 +381,7 @@ def run_pip_install_command_from_config( cls, python_path: str, config: PipConfig, - requirements: Optional[Sequence[Requirement]] = None, + requirements: Optional[Sequence[packaging.requirements.Requirement]] = None, requirements_files: Optional[list[str]] = None, upgrade: bool = False, upgrade_strategy: PipUpgradeStrategy = PipUpgradeStrategy.ONLY_IF_NEEDED, @@ -424,7 +422,7 @@ async def async_run_pip_install_command_from_config( cls, python_path: str, config: PipConfig, - requirements: Optional[Sequence[Requirement]] = None, + requirements: Optional[Sequence[packaging.requirements.Requirement]] = None, requirements_files: Optional[list[str]] = None, upgrade: bool = False, upgrade_strategy: PipUpgradeStrategy = PipUpgradeStrategy.ONLY_IF_NEEDED, @@ -464,7 +462,7 @@ def _prepare_command( cls, python_path: str, config: PipConfig, - requirements: Optional[Sequence[Requirement]] = None, + requirements: Optional[Sequence[packaging.requirements.Requirement]] = None, requirements_files: Optional[list[str]] = None, upgrade: bool = False, upgrade_strategy: PipUpgradeStrategy = PipUpgradeStrategy.ONLY_IF_NEEDED, @@ -818,7 +816,7 @@ def get_installed_packages(self, only_editable: bool = False) -> dict[str, packa def install_for_config( self, - requirements: list[Requirement], + requirements: list[packaging.requirements.Requirement], config: PipConfig, upgrade: bool = False, constraint_files: Optional[list[str]] = None, @@ -859,7 +857,7 @@ def install_for_config( async def async_install_for_config( self, - requirements: list[Requirement], + requirements: list[packaging.requirements.Requirement], config: PipConfig, upgrade: bool = False, constraint_files: Optional[list[str]] = None, @@ -896,7 +894,7 @@ async def async_install_for_config( def install_from_index( self, - requirements: Sequence[Requirement], + requirements: list[packaging.requirements.Requirement], index_urls: Optional[list[str]] = None, upgrade: bool = False, allow_pre_releases: bool = False, @@ -967,7 +965,7 @@ def install_from_list( use_pip_config was ignored on ISO6 and it still is """ self.install_from_index( - requirements=util.parse_canonical_requirements(requirements_list), + requirements=util.parse_requirements(requirements_list), upgrade=upgrade, upgrade_strategy=upgrade_strategy, use_pip_config=True, @@ -987,7 +985,7 @@ def get_protected_inmanta_packages(cls) -> list[str]: ] @classmethod - def _get_requirements_on_inmanta_package(cls) -> Sequence[Requirement]: + def _get_requirements_on_inmanta_package(cls) -> Sequence[packaging.requirements.Requirement]: """ Returns the content of the requirement file that should be supplied to each `pip install` invocation to make sure that no Inmanta packages gets overridden. @@ -995,7 +993,7 @@ def _get_requirements_on_inmanta_package(cls) -> Sequence[Requirement]: protected_inmanta_packages: list[str] = cls.get_protected_inmanta_packages() workingset: dict[str, packaging.version.Version] = PythonWorkingSet.get_packages_in_working_set() return [ - util.parse_canonical_requirement(requirement=f"{pkg}=={workingset[pkg]}") + util.parse_requirement(requirement=f"{pkg}=={workingset[pkg]}") for pkg in workingset if pkg in protected_inmanta_packages ] @@ -1123,7 +1121,7 @@ def are_installed(self, requirements: req_list) -> bool: def install_for_config( self, - requirements: list[Requirement], + requirements: list[packaging.requirements.Requirement], config: PipConfig, upgrade: bool = False, constraint_files: Optional[list[str]] = None, @@ -1143,7 +1141,7 @@ def install_for_config( def get_constraint_violations_for_check( self, strict_scope: Optional[Pattern[str]] = None, - constraints: Optional[list[Requirement]] = None, + constraints: Optional[list[packaging.requirements.Requirement]] = None, ) -> tuple[set[VersionConflict], set[VersionConflict]]: """ Return the constraint violations that exist in this venv. Returns a tuple of non-strict and strict violations, @@ -1151,7 +1149,7 @@ def get_constraint_violations_for_check( """ class OwnedRequirement(NamedTuple): - requirement: Requirement + requirement: packaging.requirements.Requirement owner: Optional[str] = None def is_owned_by(self, owners: abc.Set[str]) -> bool: @@ -1159,7 +1157,7 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: # all requirements of all packages installed in this environment installed_constraints: abc.Set[OwnedRequirement] = frozenset( - OwnedRequirement(util.parse_canonical_requirement(requirement=str(requirement)), dist_info.key) + OwnedRequirement(util.parse_requirement(requirement=str(requirement)), dist_info.key) for dist_info in pkg_resources.working_set for requirement in dist_info.requires() ) @@ -1212,7 +1210,7 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: def check( self, strict_scope: Optional[Pattern[str]] = None, - constraints: Optional[list[Requirement]] = None, + constraints: Optional[list[packaging.requirements.Requirement]] = None, ) -> None: """ Check this Python environment for incompatible dependencies in installed packages. @@ -1237,7 +1235,9 @@ def check( for violation in constraint_violations: LOGGER.warning("%s", violation) - def check_legacy(self, in_scope: Pattern[str], constraints: Optional[list[Requirement]] = None) -> bool: + def check_legacy( + self, in_scope: Pattern[str], constraints: Optional[list[packaging.requirements.Requirement]] = None + ) -> bool: """ Check this Python environment for incompatible dependencies in installed packages. This method is a legacy method in the sense that it has been replaced with a more correct check defined in self.check(). This method is invoked @@ -1254,10 +1254,10 @@ def check_legacy(self, in_scope: Pattern[str], constraints: Optional[list[Requir in_scope, constraints ) - working_set: abc.Iterable[Distribution] = importlib.metadata.distributions() + working_set: abc.Iterable[importlib.metadata.Distribution] = importlib.metadata.distributions() # add all requirements of all in scope packages installed in this environment - all_constraints: set[Requirement] = set(constraints if constraints is not None else []).union( - util.parse_canonical_requirement(requirement=requirement) + all_constraints: set[packaging.requirements.Requirement] = set(constraints if constraints is not None else []).union( + util.parse_requirement(requirement=requirement) for dist_info in working_set if in_scope.fullmatch(dist_info.name) for requirement in dist_info.requires or [] @@ -1444,7 +1444,7 @@ def _activate_that(self) -> None: def install_for_config( self, - requirements: list[Requirement], + requirements: list[packaging.requirements.Requirement], config: PipConfig, upgrade: bool = False, constraint_files: Optional[list[str]] = None, @@ -1482,7 +1482,7 @@ class VenvSnapshot: old_os_venv: Optional[str] old_process_env_path: str old_process_env: ActiveEnv - old_working_set: WorkingSet + old_working_set: pkg_resources.WorkingSet def restore(self) -> None: os.environ["PATH"] = self.old_os_path diff --git a/src/inmanta/file_parser.py b/src/inmanta/file_parser.py index 8ee4696212..2129fe8590 100644 --- a/src/inmanta/file_parser.py +++ b/src/inmanta/file_parser.py @@ -19,7 +19,7 @@ import os import packaging.utils -from inmanta.util import parse_canonical_requirement +from inmanta.util import parse_requirement from packaging.requirements import Requirement from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap @@ -61,7 +61,7 @@ def parse(cls, filename: str) -> list[Requirement]: """ Get all the requirements in `filename` as a list of `Requirement` instances. """ - return [parse_canonical_requirement(requirement=r) for r in cls.parse_requirements_as_strs(filename)] + return [parse_requirement(requirement=r) for r in cls.parse_requirements_as_strs(filename)] @classmethod def parse_requirements_as_strs(cls, filename: str) -> list[str]: @@ -95,14 +95,14 @@ def get_content_with_dep_removed(cls, filename: str, remove_dep_on_pkg: str) -> if line_continuation_buffer: line_continuation_buffer += line if not line.endswith("\\"): - if parse_canonical_requirement(requirement=line_continuation_buffer).name != removed_dependency: + if parse_requirement(requirement=line_continuation_buffer).name != removed_dependency: result += line_continuation_buffer line_continuation_buffer = "" elif not line.strip() or line.strip().startswith("#"): result += line elif line.endswith("\\"): line_continuation_buffer = line - elif parse_canonical_requirement(requirement=line).name != removed_dependency: + elif parse_requirement(requirement=line).name != removed_dependency: result += line else: # Dependency matches `remove_dep_on_pkg` => Remove line from result diff --git a/src/inmanta/module.py b/src/inmanta/module.py index 89d67b73f7..43faf1b687 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -53,6 +53,7 @@ from pydantic import BaseModel, Field, NameEmail, StringConstraints, ValidationError, field_validator import inmanta.data.model +import packaging.requirements import packaging.version from inmanta import RUNNING_TESTS, const, env, loader, plugins from inmanta.ast import CompilerException, LocatableString, Location, Namespace, Range, WrappingRuntimeException @@ -60,14 +61,12 @@ from inmanta.ast.statements import BiStatement, DefinitionStatement, DynamicStatement, Statement from inmanta.ast.statements.define import DefineImport from inmanta.env import assert_pip_has_source -from inmanta.util import parse_canonical_requirements, parse_canonical_requirement from inmanta.file_parser import PreservativeYamlParser, RequirementsTxtParser from inmanta.parser import plyInmantaParser from inmanta.parser.plyInmantaParser import cache_manager from inmanta.stable_api import stable_api -from inmanta.util import get_compiler_version +from inmanta.util import get_compiler_version, parse_requirement, parse_requirements from inmanta.warnings import InmantaWarning -from packaging.requirements import Requirement from packaging.specifiers import SpecifierSet from packaging.version import Version from ruamel.yaml.comments import CommentedMap @@ -98,13 +97,13 @@ class InmantaModuleRequirement: used by distinguishing the two on a type level. """ - def __init__(self, requirement: Requirement) -> None: + def __init__(self, requirement: packaging.requirements.Requirement) -> None: if requirement.name.startswith(ModuleV2.PKG_NAME_PREFIX): raise ValueError( f"InmantaModuleRequirement instances work with inmanta module names, not python package names. " f"Problematic case: {str(requirement)}" ) - self._requirement: Requirement = requirement + self._requirement: packaging.requirements.Requirement = requirement @property def project_name(self) -> str: @@ -152,16 +151,16 @@ def parse(cls: type[TInmantaModuleRequirement], spec: str) -> TInmantaModuleRequ ) if "-" in spec: raise ValueError("Invalid Inmanta module requirement: Inmanta module names use '_', not '-'.") - return cls(parse_canonical_requirement(requirement=spec)) + return cls(parse_requirement(requirement=spec)) - def get_python_package_requirement(self) -> Requirement: + def get_python_package_requirement(self) -> packaging.requirements.Requirement: """ Return a Requirement with the name of the Python distribution package for this module requirement. """ module_name = self.project_name pkg_name = ModuleV2Source.get_package_name_for(module_name) pkg_req_str = str(self).replace(module_name, pkg_name, 1) # Replace max 1 occurrence - return parse_canonical_requirement(requirement=pkg_req_str) + return parse_requirement(requirement=pkg_req_str) class CompilerExceptionWithExtendedTrace(CompilerException): @@ -722,14 +721,14 @@ def install(self, project: "Project", module_spec: list[InmantaModuleRequirement assert_pip_has_source(project.metadata.pip, f"a v2 module {module_name}") - requirements: list[Requirement] = [req.get_python_package_requirement() for req in module_spec] + requirements: list[packaging.requirements.Requirement] = [req.get_python_package_requirement() for req in module_spec] preinstalled: Optional[ModuleV2] = self.get_installed_module(project, module_name) # Get known requires and add them to prevent invalidating constraints through updates # These could be constraints (-c) as well, but that requires additional sanitation # Because for pip not every valid -r is a valid -c current_requires = project.get_strict_python_requirements_as_list() - requirements += parse_canonical_requirements(current_requires) + requirements += parse_requirements(current_requires) if preinstalled is not None: # log warning if preinstalled version does not match constraints @@ -1179,7 +1178,7 @@ def has_requirement_for(self, pkg_name: str) -> bool: """ return any(r.name == pkg_name.lower() for r in self._requirements) - def set_requirement_and_write(self, requirement: Requirement) -> None: + def set_requirement_and_write(self, requirement: packaging.requirements.Requirement) -> None: """ Add the given requirement to the requirements.txt file and update the file on disk, replacing any existing constraints on this package. @@ -2128,7 +2127,7 @@ def install_modules(self, *, bypass_module_cache: bool = False, update_dependenc self.verify_module_version_compatibility() # do python install - pyreq: Sequence[Requirement] = parse_canonical_requirements(self.collect_python_requirements()) + pyreq: list[packaging.requirements.Requirement] = parse_requirements(self.collect_python_requirements()) if len(pyreq) > 0: # upgrade both direct and transitive module dependencies: eager upgrade strategy @@ -2546,7 +2545,7 @@ def verify_python_requires(self) -> None: Verifies no incompatibilities exist within the Python environment with respect to installed module v2 requirements. """ if self.strict_deps_check: - constraints: Sequence[Requirement] = parse_canonical_requirements(self.collect_python_requirements()) + constraints: list[packaging.requirements.Requirement] = parse_requirements(self.collect_python_requirements()) env.process_env.check(strict_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*"), constraints=constraints) else: if not env.process_env.check_legacy(in_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*")): @@ -2677,7 +2676,7 @@ def requires(self) -> "List[InmantaModuleRequirement]": # filter on import stmt reqs = [] for spec in self._metadata.requires: - req = [parse_canonical_requirement(requirement=spec)] + req = [parse_requirement(requirement=spec)] if len(req) > 1: print(f"Module file for {self._path} has bad line in requirements specification {spec}") reqe = InmantaModuleRequirement(req[0]) @@ -2814,7 +2813,7 @@ def requires(self) -> "List[InmantaModuleRequirement]": """ reqs = [] for spec in self.get_module_requirements(): - req = [parse_canonical_requirement(requirement=spec)] + req = [parse_requirement(requirement=spec)] if len(req) > 1: print(f"Module file for {self._path} has bad line in requirements specification {spec}") reqe = InmantaModuleRequirement(req[0]) @@ -3431,12 +3430,12 @@ def add_module_requirement_persistent(self, requirement: InmantaModuleRequiremen # Parse config file config_parser = ConfigParser() config_parser.read(self.get_metadata_file_path()) - python_pkg_requirement: Requirement = requirement.get_python_package_requirement() + python_pkg_requirement: packaging.requirements.Requirement = requirement.get_python_package_requirement() if config_parser.has_option("options", "install_requires"): new_install_requires = [ r for r in config_parser.get("options", "install_requires").split("\n") - if r and parse_canonical_requirement(requirement=r).name != python_pkg_requirement.name + if r and parse_requirement(requirement=r).name != python_pkg_requirement.name ] new_install_requires.append(str(python_pkg_requirement)) else: diff --git a/src/inmanta/moduletool.py b/src/inmanta/moduletool.py index ac26c39854..54466e5ee1 100644 --- a/src/inmanta/moduletool.py +++ b/src/inmanta/moduletool.py @@ -38,7 +38,7 @@ from configparser import ConfigParser from functools import total_ordering from re import Pattern -from typing import IO, TYPE_CHECKING, Any, Literal, Optional +from typing import IO, Any, Literal, Optional import click import more_itertools @@ -55,7 +55,6 @@ from inmanta.ast import CompilerException from inmanta.command import CLIException, ShowUsageException from inmanta.const import CF_CACHE_DIR, MAX_UPDATE_ATTEMPT -from inmanta.util import parse_canonical_requirements from inmanta.module import ( DummyProject, FreezeOperator, @@ -76,6 +75,7 @@ gitprovider, ) from inmanta.stable_api import stable_api +from inmanta.util import parse_requirements from packaging.requirements import InvalidRequirement, Requirement from packaging.version import Version @@ -485,7 +485,7 @@ def do_update(specs: Mapping[str, Sequence[InmantaModuleRequirement]], modules: # Because for pip not every valid -r is a valid -c current_requires = my_project.get_strict_python_requirements_as_list() env.process_env.install_for_config( - v2_python_specs + parse_canonical_requirements(current_requires), + v2_python_specs + parse_requirements(current_requires), my_project.metadata.pip, upgrade=True, ) diff --git a/src/inmanta/util/__init__.py b/src/inmanta/util/__init__.py index 1a66705164..43b3605a3a 100644 --- a/src/inmanta/util/__init__.py +++ b/src/inmanta/util/__init__.py @@ -31,6 +31,7 @@ import socket import threading import time +import typing import uuid import warnings from abc import ABC, abstractmethod @@ -47,12 +48,12 @@ from tornado import gen import packaging +import packaging.requirements import packaging.utils from crontab import CronTab from inmanta import COMPILER_VERSION, const from inmanta.stable_api import stable_api from inmanta.types import JsonType, PrimitiveTypes, ReturnTypes -from packaging.requirements import Requirement from pydantic_core import Url LOGGER = logging.getLogger(__name__) @@ -882,7 +883,10 @@ def remove_comment_part(to_clean: str) -> str: return drop_comment -def parse_canonical_requirement(requirement: str) -> Requirement: +CanonicalRequirement = typing.NewType("CanonicalRequirement", packaging.requirements.Requirement) + + +def parse_requirement(requirement: str) -> packaging.requirements.Requirement: """ To be able to compare requirements, we need to make sure that every requirement's name is canonicalized otherwise issues could arise when checking if packages are installed in a particular Venv. @@ -895,13 +899,15 @@ def parse_canonical_requirement(requirement: str) -> Requirement: # We canonicalize the name of the requirement to be able to compare requirements and check if the requirement is # already installed # This instance is considered as doomed because we are not supposed to modify directly the instance - doomed_requirement_instance = Requirement(requirement_string=drop_comment) - canonical_name = str(doomed_requirement_instance).replace(doomed_requirement_instance.name, packaging.utils.canonicalize_name(doomed_requirement_instance.name)) - requirement_instance = Requirement(requirement_string=canonical_name) + doomed_requirement_instance = packaging.requirements.Requirement(requirement_string=drop_comment) + canonical_name = str(doomed_requirement_instance).replace( + doomed_requirement_instance.name, packaging.utils.canonicalize_name(doomed_requirement_instance.name) + ) + requirement_instance = packaging.requirements.Requirement(requirement_string=canonical_name) return requirement_instance -def parse_canonical_requirements(requirements: Sequence[str]) -> Sequence[Requirement]: +def parse_requirements(requirements: Sequence[str]) -> list[packaging.requirements.Requirement]: """ To be able to compare requirements, we need to make sure that every requirement's name is canonicalized otherwise issues could arise when checking if packages are installed in a particular Venv. @@ -909,10 +915,10 @@ def parse_canonical_requirements(requirements: Sequence[str]) -> Sequence[Requir :param requirements: The names of the different requirements :return: Sequence[Requirement] """ - return [parse_canonical_requirement(requirement=e) for e in requirements] + return [parse_requirement(requirement=e) for e in requirements] -def parse_canonical_requirements_from_file(file_path: pathlib.Path) -> Sequence[Requirement]: +def parse_canonical_requirements_from_file(file_path: pathlib.Path) -> list[packaging.requirements.Requirement]: """ To be able to compare requirements, we need to make sure that every requirement's name is canonicalized otherwise issues could arise when checking if packages are installed in a particular Venv. @@ -923,5 +929,5 @@ def parse_canonical_requirements_from_file(file_path: pathlib.Path) -> Sequence[ requirements = [] with open(file_path) as f: for line in f.readlines(): - requirements.append(parse_canonical_requirement(line)) + requirements.append(parse_requirement(line)) return requirements diff --git a/tests/conftest.py b/tests/conftest.py index 021a45d7cd..2934ff0def 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -122,8 +122,6 @@ from inmanta.data.schema import SCHEMA_VERSION_TABLE from inmanta.db import util as db_util from inmanta.env import ActiveEnv, CommandRunner, LocalPackagePath, VirtualEnv, swap_process_env -from inmanta.env import CommandRunner, LocalPackagePath, VirtualEnv, mock_process_env -from inmanta.util import parse_requirement from inmanta.export import ResourceDict, cfg_env, unknown_parameters from inmanta.module import InmantaModuleRequirement, InstallMode, Project, RelationPrecedenceRule from inmanta.moduletool import DefaultIsolatedEnvCached, ModuleTool, V2ModuleBuilder @@ -135,6 +133,7 @@ from inmanta.server.services import orchestrationservice from inmanta.server.services.compilerservice import CompilerService, CompileRun from inmanta.types import JsonType +from inmanta.util import parse_requirement from inmanta.warnings import WarningsManager from libpip2pi.commands import dir2pi from packaging.version import Version @@ -1923,10 +1922,10 @@ def index_with_pkgs_containing_optional_deps() -> str: path=os.path.join(tmpdirname, "pkg"), publish_index=pip_index, optional_dependencies={ - "optional-a": [parse_canonical_requirement(requirement="dep-a")], + "optional-a": [parse_requirement(requirement="dep-a")], "optional-b": [ - parse_canonical_requirement(requirement="dep-b"), - parse_canonical_requirement(requirement="dep-c"), + parse_requirement(requirement="dep-b"), + parse_requirement(requirement="dep-c"), ], }, ) diff --git a/tests/moduletool/test_add.py b/tests/moduletool/test_add.py index da19a69b48..68916a22bc 100644 --- a/tests/moduletool/test_add.py +++ b/tests/moduletool/test_add.py @@ -26,7 +26,7 @@ from inmanta import env from inmanta.command import CLIException -from inmanta.env import process_env, parse_canonical_requirement +from inmanta.env import parse_canonical_requirement from inmanta.module import ModuleV1, ModuleV1Metadata, ModuleV2, ModuleV2Source, Project, ProjectMetadata from inmanta.moduletool import ModuleTool from packaging.version import Version diff --git a/tests/moduletool/test_install.py b/tests/moduletool/test_install.py index 0d65a97c91..d154792289 100644 --- a/tests/moduletool/test_install.py +++ b/tests/moduletool/test_install.py @@ -508,7 +508,10 @@ def test_project_install( index_url=local_module_package_index, # We add tornado, as there is a code path in update for the case where the project has python requires python_requires=["tornado"] - + [parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names], + + [ + parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) + for mod in install_module_names + ], install_project=False, ) @@ -541,7 +544,8 @@ def test_project_install( autostd=False, python_package_sources=[local_module_package_index], python_requires=[ - parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names + parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) + for mod in install_module_names ] + ["lorem"], install_project=False, @@ -1333,7 +1337,9 @@ def test_pip_output(local_module_package_index: str, snippetcompiler_clean, capl ) modules = ["modone", "modtwo"] - v2_requirements = [parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in modules] + v2_requirements = [ + parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in modules + ] snippetcompiler_clean.setup_for_snippet( f""" @@ -1419,7 +1425,9 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[ + parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module")) + ], install_project=True, ) log_contains( @@ -1451,7 +1459,9 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[ + parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module")) + ], install_project=True, ) diff --git a/tests/moduletool/test_update.py b/tests/moduletool/test_update.py index b85b27e48e..659d58587a 100644 --- a/tests/moduletool/test_update.py +++ b/tests/moduletool/test_update.py @@ -24,7 +24,7 @@ from inmanta import env from inmanta.config import Config from inmanta.data.model import PipConfig -from inmanta.env import LocalPackagePath, process_env, parse_canonical_requirement, parse_canonical_requirements +from inmanta.env import LocalPackagePath, parse_canonical_requirement, parse_canonical_requirements from inmanta.module import InmantaModuleRequirement, InstallMode, ModuleV1, ModuleV2Source from inmanta.moduletool import ProjectTool from inmanta.parser import ParserException diff --git a/tests/test_env.py b/tests/test_env.py index d64280aa42..db08b7c202 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -36,7 +36,7 @@ from inmanta import env, loader, module from inmanta.data.model import PipConfig from inmanta.env import Pip, Requirement -from inmanta.util import parse_canonical_requirement +from inmanta.util import parse_requirement from packaging import version from utils import LogSequence, PipIndex, create_python_package @@ -177,7 +177,7 @@ def test_gen_req_file(): # make sure they all parse for req in reqs: - parse_canonical_requirement(requirement=req) + parse_requirement(requirement=req) def test_environment_python_version_multi_digit(tmpdir: py.path.local) -> None: @@ -206,7 +206,7 @@ def test_process_env_install_from_index( package_name: str = "more-itertools" assert package_name not in env.process_env.get_installed_packages() env.process_env.install_for_config( - [parse_canonical_requirement(requirement=package_name + (f"=={version}" if version is not None else ""))], + [parse_requirement(requirement=package_name + (f"=={version}" if version is not None else ""))], config=PipConfig( use_system_config=True, # we need an upstream for some packages ), @@ -221,7 +221,7 @@ def test_process_env_install_from_index( # It should hit the cache there and return here. # Cheap and fast test env.process_env.install_from_index( - [parse_canonical_requirement(requirement=package_name + (f"=={version}" if version is not None else ""))], + [parse_requirement(requirement=package_name + (f"=={version}" if version is not None else ""))], use_pip_config=True, ) @@ -271,7 +271,7 @@ def test_process_env_install_from_index_not_found_env_var( with pytest.raises(env.PackageNotFound, match=re.escape(expected)): env.process_env.install_for_config( - [parse_canonical_requirement(requirement="this-package-does-not-exist")], + [parse_requirement(requirement="this-package-does-not-exist")], config=PipConfig( index_url=index_urls[0], # The first element should only be passed to the index_url. If there are indexes in the environment @@ -308,7 +308,7 @@ def test_process_env_install_no_index(tmpdir: py.path.local, monkeypatch, use_sy with pytest.raises(env.PackageNotFound, match=re.escape(expected)): env.process_env.install_for_config( - requirements=[parse_canonical_requirement(requirement="this-package-does-not-exist")], + requirements=[parse_requirement(requirement="this-package-does-not-exist")], paths=[env.LocalPackagePath(path=str(tmpdir))], config=PipConfig(use_system_config=use_system_config), ) @@ -325,7 +325,7 @@ def test_process_env_install_from_index_conflicting_reqs( package_name: str = "more-itertools" with pytest.raises(env.ConflictingRequirements) as e: env.process_env.install_for_config( - [parse_canonical_requirement(requirement=f"{package_name}{version}") for version in [">8.5", "<=8"]], + [parse_requirement(requirement=f"{package_name}{version}") for version in [">8.5", "<=8"]], config=PipConfig( use_system_config=True, # we need an upstream for some packages ), @@ -395,7 +395,7 @@ def test_active_env_get_module_file( loader.PluginModuleFinder.configure_module_finder([os.path.join(str(tmpdir), "libs")]) assert env.ActiveEnv.get_module_file(module_name) is None - env.process_env.install_for_config([parse_canonical_requirement(requirement=package_name)], pip_config) + env.process_env.install_for_config([parse_requirement(requirement=package_name)], pip_config) assert package_name in env.process_env.get_installed_packages() module_info: Optional[tuple[Optional[str], Loader]] = env.ActiveEnv.get_module_file(module_name) assert module_info is not None @@ -542,7 +542,7 @@ def assert_all_checks(expect_test: tuple[bool, str] = (True, ""), expect_nonext: create_install_package( "test-package-two", version.Version("1.0.0"), - [parse_canonical_requirement(requirement="test-package-one~=1.0")], + [parse_requirement(requirement="test-package-one~=1.0")], local_module_package_index, ) assert_all_checks() @@ -563,7 +563,7 @@ def test_active_env_check_constraints(caplog, tmpvenv_active_inherit: str, local """ caplog.set_level(logging.WARNING) in_scope: Pattern[str] = re.compile("test-package-.*") - constraints: list[Requirement] = [parse_canonical_requirement(requirement="test-package-one~=1.0")] + constraints: list[Requirement] = [parse_requirement(requirement="test-package-one~=1.0")] env.process_env.check(in_scope) @@ -582,7 +582,7 @@ def test_active_env_check_constraints(caplog, tmpvenv_active_inherit: str, local create_install_package( "ext-package-one", version.Version("1.0.0"), - [parse_canonical_requirement(requirement="test-package-one==1.0")], + [parse_requirement(requirement="test-package-one==1.0")], local_module_package_index, ) env.process_env.check(in_scope, constraints) @@ -613,7 +613,7 @@ def test_override_inmanta_package(tmpvenv_active_inherit: env.VirtualEnv) -> Non installed_pkgs = tmpvenv_active_inherit.get_installed_packages() assert "inmanta-core" in installed_pkgs, "The inmanta-core package should be installed to run the tests" - inmanta_requirements = parse_canonical_requirement(requirement="inmanta-core==4.0.0") + inmanta_requirements = parse_requirement(requirement="inmanta-core==4.0.0") with pytest.raises(env.ConflictingRequirements) as excinfo: tmpvenv_active_inherit.install_for_config( requirements=[inmanta_requirements], @@ -653,13 +653,13 @@ def test_cache_on_active_env(tmpvenv_active_inherit: env.ActiveEnv, local_module """ def _assert_install(requirement: str, installed: bool) -> None: - parsed_requirement = parse_canonical_requirement(requirement=requirement) + parsed_requirement = parse_requirement(requirement=requirement) for r in [requirement, parsed_requirement]: assert tmpvenv_active_inherit.are_installed(requirements=[r]) == installed _assert_install("inmanta-module-elaboratev2module==1.2.3", installed=False) tmpvenv_active_inherit.install_for_config( - requirements=[parse_canonical_requirement(requirement="inmanta-module-elaboratev2module==1.2.3")], + requirements=[parse_requirement(requirement="inmanta-module-elaboratev2module==1.2.3")], config=PipConfig( index_url=local_module_package_index, ), @@ -703,7 +703,7 @@ def test_are_installed_dependency_cycle_on_extra(tmpdir, tmpvenv_active_inherit: path=os.path.join(tmpdir, "pkg"), publish_index=pip_index, optional_dependencies={ - "optional-pkg": [parse_canonical_requirement(requirement="dep[optional-dep]")], + "optional-pkg": [parse_requirement(requirement="dep[optional-dep]")], }, ) create_python_package( @@ -712,11 +712,11 @@ def test_are_installed_dependency_cycle_on_extra(tmpdir, tmpvenv_active_inherit: path=os.path.join(tmpdir, "dep"), publish_index=pip_index, optional_dependencies={ - "optional-dep": [parse_canonical_requirement(requirement="pkg[optional-pkg]")], + "optional-dep": [parse_requirement(requirement="pkg[optional-pkg]")], }, ) - requirements = [parse_canonical_requirement(requirement="pkg[optional-pkg]")] + requirements = [parse_requirement(requirement="pkg[optional-pkg]")] tmpvenv_active_inherit.install_for_config( requirements=requirements, config=PipConfig( diff --git a/tests/test_module_loader.py b/tests/test_module_loader.py index 72c5fceaeb..0543d10a0f 100644 --- a/tests/test_module_loader.py +++ b/tests/test_module_loader.py @@ -33,14 +33,7 @@ from inmanta.ast import CompilerException from inmanta.const import CF_CACHE_DIR from inmanta.data.model import PipConfig -from inmanta.env import ( - ConflictingRequirements, - LocalPackagePath, - PackageNotFound, - Requirement, - process_env, - parse_canonical_requirement, -) +from inmanta.env import ConflictingRequirements, LocalPackagePath, PackageNotFound, Requirement, parse_canonical_requirement from inmanta.module import ( DummyProject, InmantaModuleRequirement, From c8ec8772c6170637614e8c73e74facf7b725b98a Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Mon, 16 Sep 2024 11:47:54 +0200 Subject: [PATCH 33/74] fix broken tests --- src/inmanta/env.py | 20 ++++----- src/inmanta/module.py | 10 ++--- src/inmanta/moduletool.py | 8 ++-- src/inmanta/util/__init__.py | 8 ++-- tests/compiler/test_basics.py | 5 +-- tests/moduletool/test_add.py | 4 +- tests/moduletool/test_convert_v1_v2.py | 4 +- tests/moduletool/test_install.py | 58 +++++++++++--------------- tests/moduletool/test_update.py | 13 +++--- tests/server/test_compilerservice.py | 6 +-- tests/test_env.py | 7 ++-- tests/test_file_parser.py | 4 +- tests/test_module_loader.py | 56 ++++++++++++------------- tests/utils.py | 25 +++++++---- 14 files changed, 111 insertions(+), 117 deletions(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 825aad8e3d..118cd9e18b 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -224,7 +224,7 @@ def _are_installed_recursive( if distribution is None: return False - pkgs_required_by_extra: set[packaging.requirements.Requirement] = set( + pkgs_required_by_extra: set[util.CanonicalRequirement] = set( [util.parse_requirement(str(e)) for e in distribution.requires(extras=(extra,))] ) - set([util.parse_requirement(str(e)) for e in distribution.requires(extras=())]) if not _are_installed_recursive( @@ -816,7 +816,7 @@ def get_installed_packages(self, only_editable: bool = False) -> dict[str, packa def install_for_config( self, - requirements: list[packaging.requirements.Requirement], + requirements: list[util.CanonicalRequirement], config: PipConfig, upgrade: bool = False, constraint_files: Optional[list[str]] = None, @@ -857,7 +857,7 @@ def install_for_config( async def async_install_for_config( self, - requirements: list[packaging.requirements.Requirement], + requirements: list[util.CanonicalRequirement], config: PipConfig, upgrade: bool = False, constraint_files: Optional[list[str]] = None, @@ -894,7 +894,7 @@ async def async_install_for_config( def install_from_index( self, - requirements: list[packaging.requirements.Requirement], + requirements: list[util.CanonicalRequirement], index_urls: Optional[list[str]] = None, upgrade: bool = False, allow_pre_releases: bool = False, @@ -1121,7 +1121,7 @@ def are_installed(self, requirements: req_list) -> bool: def install_for_config( self, - requirements: list[packaging.requirements.Requirement], + requirements: list[util.CanonicalRequirement], config: PipConfig, upgrade: bool = False, constraint_files: Optional[list[str]] = None, @@ -1141,7 +1141,7 @@ def install_for_config( def get_constraint_violations_for_check( self, strict_scope: Optional[Pattern[str]] = None, - constraints: Optional[list[packaging.requirements.Requirement]] = None, + constraints: Optional[list[util.CanonicalRequirement]] = None, ) -> tuple[set[VersionConflict], set[VersionConflict]]: """ Return the constraint violations that exist in this venv. Returns a tuple of non-strict and strict violations, @@ -1210,7 +1210,7 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: def check( self, strict_scope: Optional[Pattern[str]] = None, - constraints: Optional[list[packaging.requirements.Requirement]] = None, + constraints: Optional[list[util.CanonicalRequirement]] = None, ) -> None: """ Check this Python environment for incompatible dependencies in installed packages. @@ -1235,9 +1235,7 @@ def check( for violation in constraint_violations: LOGGER.warning("%s", violation) - def check_legacy( - self, in_scope: Pattern[str], constraints: Optional[list[packaging.requirements.Requirement]] = None - ) -> bool: + def check_legacy(self, in_scope: Pattern[str], constraints: Optional[list[util.CanonicalRequirement]] = None) -> bool: """ Check this Python environment for incompatible dependencies in installed packages. This method is a legacy method in the sense that it has been replaced with a more correct check defined in self.check(). This method is invoked @@ -1444,7 +1442,7 @@ def _activate_that(self) -> None: def install_for_config( self, - requirements: list[packaging.requirements.Requirement], + requirements: list[util.CanonicalRequirement], config: PipConfig, upgrade: bool = False, constraint_files: Optional[list[str]] = None, diff --git a/src/inmanta/module.py b/src/inmanta/module.py index 43faf1b687..e6dbc59715 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -55,7 +55,7 @@ import inmanta.data.model import packaging.requirements import packaging.version -from inmanta import RUNNING_TESTS, const, env, loader, plugins +from inmanta import RUNNING_TESTS, const, env, loader, plugins, util from inmanta.ast import CompilerException, LocatableString, Location, Namespace, Range, WrappingRuntimeException from inmanta.ast.blocks import BasicBlock from inmanta.ast.statements import BiStatement, DefinitionStatement, DynamicStatement, Statement @@ -153,7 +153,7 @@ def parse(cls: type[TInmantaModuleRequirement], spec: str) -> TInmantaModuleRequ raise ValueError("Invalid Inmanta module requirement: Inmanta module names use '_', not '-'.") return cls(parse_requirement(requirement=spec)) - def get_python_package_requirement(self) -> packaging.requirements.Requirement: + def get_python_package_requirement(self) -> util.CanonicalRequirement: """ Return a Requirement with the name of the Python distribution package for this module requirement. """ @@ -721,7 +721,7 @@ def install(self, project: "Project", module_spec: list[InmantaModuleRequirement assert_pip_has_source(project.metadata.pip, f"a v2 module {module_name}") - requirements: list[packaging.requirements.Requirement] = [req.get_python_package_requirement() for req in module_spec] + requirements: list[util.CanonicalRequirement] = [req.get_python_package_requirement() for req in module_spec] preinstalled: Optional[ModuleV2] = self.get_installed_module(project, module_name) # Get known requires and add them to prevent invalidating constraints through updates @@ -2127,7 +2127,7 @@ def install_modules(self, *, bypass_module_cache: bool = False, update_dependenc self.verify_module_version_compatibility() # do python install - pyreq: list[packaging.requirements.Requirement] = parse_requirements(self.collect_python_requirements()) + pyreq: list[util.CanonicalRequirement] = parse_requirements(self.collect_python_requirements()) if len(pyreq) > 0: # upgrade both direct and transitive module dependencies: eager upgrade strategy @@ -2545,7 +2545,7 @@ def verify_python_requires(self) -> None: Verifies no incompatibilities exist within the Python environment with respect to installed module v2 requirements. """ if self.strict_deps_check: - constraints: list[packaging.requirements.Requirement] = parse_requirements(self.collect_python_requirements()) + constraints: list[util.CanonicalRequirement] = parse_requirements(self.collect_python_requirements()) env.process_env.check(strict_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*"), constraints=constraints) else: if not env.process_env.check_legacy(in_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*")): diff --git a/src/inmanta/moduletool.py b/src/inmanta/moduletool.py index 54466e5ee1..36a163b7d2 100644 --- a/src/inmanta/moduletool.py +++ b/src/inmanta/moduletool.py @@ -49,9 +49,10 @@ import build import inmanta import inmanta.warnings +import packaging.requirements import toml from build.env import DefaultIsolatedEnv -from inmanta import const, env +from inmanta import const, env, util from inmanta.ast import CompilerException from inmanta.command import CLIException, ShowUsageException from inmanta.const import CF_CACHE_DIR, MAX_UPDATE_ATTEMPT @@ -76,7 +77,6 @@ ) from inmanta.stable_api import stable_api from inmanta.util import parse_requirements -from packaging.requirements import InvalidRequirement, Requirement from packaging.version import Version LOGGER = logging.getLogger(__name__) @@ -473,7 +473,7 @@ def update( def do_update(specs: Mapping[str, Sequence[InmantaModuleRequirement]], modules: list[str]) -> None: v2_modules = {module for module in modules if my_project.module_source.path_for(module) is not None} - v2_python_specs: list[Requirement] = [ + v2_python_specs: list[util.CanonicalRequirement] = [ module_spec.get_python_package_requirement() for module, module_specs in specs.items() for module_spec in module_specs @@ -805,7 +805,7 @@ def add(self, module_req: str, v1: bool = False, v2: bool = False, override: boo raise CLIException("Current working directory doesn't contain an Inmanta module or project", exitcode=1) try: module_requirement = InmantaModuleRequirement.parse(module_req) - except InvalidRequirement: + except packaging.requirements.InvalidRequirement: raise CLIException(f"'{module_req}' is not a valid requirement", exitcode=1) if not override and module_like.has_module_requirement(module_requirement.key): raise CLIException( diff --git a/src/inmanta/util/__init__.py b/src/inmanta/util/__init__.py index 43b3605a3a..fa1b5c87b1 100644 --- a/src/inmanta/util/__init__.py +++ b/src/inmanta/util/__init__.py @@ -886,7 +886,7 @@ def remove_comment_part(to_clean: str) -> str: CanonicalRequirement = typing.NewType("CanonicalRequirement", packaging.requirements.Requirement) -def parse_requirement(requirement: str) -> packaging.requirements.Requirement: +def parse_requirement(requirement: str) -> CanonicalRequirement: """ To be able to compare requirements, we need to make sure that every requirement's name is canonicalized otherwise issues could arise when checking if packages are installed in a particular Venv. @@ -903,11 +903,11 @@ def parse_requirement(requirement: str) -> packaging.requirements.Requirement: canonical_name = str(doomed_requirement_instance).replace( doomed_requirement_instance.name, packaging.utils.canonicalize_name(doomed_requirement_instance.name) ) - requirement_instance = packaging.requirements.Requirement(requirement_string=canonical_name) + requirement_instance = CanonicalRequirement(packaging.requirements.Requirement(requirement_string=canonical_name)) return requirement_instance -def parse_requirements(requirements: Sequence[str]) -> list[packaging.requirements.Requirement]: +def parse_requirements(requirements: Sequence[str]) -> list[CanonicalRequirement]: """ To be able to compare requirements, we need to make sure that every requirement's name is canonicalized otherwise issues could arise when checking if packages are installed in a particular Venv. @@ -918,7 +918,7 @@ def parse_requirements(requirements: Sequence[str]) -> list[packaging.requiremen return [parse_requirement(requirement=e) for e in requirements] -def parse_canonical_requirements_from_file(file_path: pathlib.Path) -> list[packaging.requirements.Requirement]: +def parse_requirements_from_file(file_path: pathlib.Path) -> list[CanonicalRequirement]: """ To be able to compare requirements, we need to make sure that every requirement's name is canonicalized otherwise issues could arise when checking if packages are installed in a particular Venv. diff --git a/tests/compiler/test_basics.py b/tests/compiler/test_basics.py index 762fa7e867..b2ab1611cb 100644 --- a/tests/compiler/test_basics.py +++ b/tests/compiler/test_basics.py @@ -24,9 +24,8 @@ import py import pytest -from inmanta import compiler, const, module +from inmanta import compiler, const, module, util from inmanta.ast import DoubleSetException, RuntimeException -from inmanta.env import parse_canonical_requirement from inmanta.module import InstallMode from inmanta.plugins import PluginDeprecationWarning from packaging import version @@ -750,7 +749,7 @@ def test_safe_requirement(name) -> None: Ensure that empty name requirements are not allowed in `Requirement` """ with pytest.raises(ValueError): - parse_canonical_requirement(requirement=name) + util.parse_requirement(requirement=name) @pytest.mark.slowtest diff --git a/tests/moduletool/test_add.py b/tests/moduletool/test_add.py index 68916a22bc..7e72afc3da 100644 --- a/tests/moduletool/test_add.py +++ b/tests/moduletool/test_add.py @@ -26,9 +26,9 @@ from inmanta import env from inmanta.command import CLIException -from inmanta.env import parse_canonical_requirement from inmanta.module import ModuleV1, ModuleV1Metadata, ModuleV2, ModuleV2Source, Project, ProjectMetadata from inmanta.moduletool import ModuleTool +from inmanta.util import parse_requirement from packaging.version import Version from utils import PipIndex, module_from_template @@ -89,7 +89,7 @@ def test_module_add_v2_module_to_project( dest_dir=os.path.join(tmpdir, f"elaboratev2module-v{version}"), new_version=Version(version), publish_index=pip_index, - new_extras={"optional": [parse_canonical_requirement(requirement="inmanta-module-minimalv2module")]}, + new_extras={"optional": [parse_requirement(requirement="inmanta-module-minimalv2module")]}, ) # Create project diff --git a/tests/moduletool/test_convert_v1_v2.py b/tests/moduletool/test_convert_v1_v2.py index a7a1f6bf50..0c42fb60a8 100644 --- a/tests/moduletool/test_convert_v1_v2.py +++ b/tests/moduletool/test_convert_v1_v2.py @@ -30,9 +30,9 @@ import toml from inmanta import moduletool from inmanta.command import CLIException -from inmanta.env import parse_canonical_requirements from inmanta.module import DummyProject, ModuleV1, ModuleV2, ModuleV2Metadata from inmanta.moduletool import ModuleConverter, ModuleVersionException +from inmanta.util import parse_requirements from packaging import version from utils import log_contains, v1_module_from_template @@ -114,7 +114,7 @@ def test_issue_3159_conversion_std_module_add_self_to_dependencies(tmpdir): parser = configparser.ConfigParser() parser.read(setup_cfg_file) assert parser.has_option("options", "install_requires") - install_requires = parse_canonical_requirements(parser.get("options", "install_requires").split("\n")) + install_requires = parse_requirements(parser.get("options", "install_requires").split("\n")) pkg_names = [r.name for r in install_requires] assert "inmanta-module-std" not in pkg_names diff --git a/tests/moduletool/test_install.py b/tests/moduletool/test_install.py index d154792289..d6d4f4540b 100644 --- a/tests/moduletool/test_install.py +++ b/tests/moduletool/test_install.py @@ -35,9 +35,10 @@ from inmanta.ast import CompilerException from inmanta.command import CLIException from inmanta.config import Config -from inmanta.env import CommandRunner, ConflictingRequirements, PipConfig, parse_canonical_requirement +from inmanta.env import CommandRunner, ConflictingRequirements, PipConfig from inmanta.module import InmantaModuleRequirement, InstallMode, ModuleLoadingException, ModuleNotFoundException from inmanta.moduletool import DummyProject, ModuleConverter, ModuleTool, ProjectTool +from inmanta.util import parse_requirement from moduletool.common import BadModProvider, install_project from packaging import version from utils import LogSequence, PipIndex, log_contains, module_from_template @@ -251,14 +252,14 @@ def test_module_install_conflicting_requirements(tmpdir: py.path.local, snippetc os.path.join(modules_v2_dir, "minimalv2module"), os.path.join(str(tmpdir), "modone"), new_name="modone", - new_requirements=[parse_canonical_requirement(requirement="lorem~=0.0.1")], + new_requirements=[parse_requirement(requirement="lorem~=0.0.1")], install=True, ) module_from_template( os.path.join(modules_v2_dir, "minimalv2module"), os.path.join(str(tmpdir), "modtwo"), new_name="modtwo", - new_requirements=[parse_canonical_requirement(requirement="lorem~=0.1.0")], + new_requirements=[parse_requirement(requirement="lorem~=0.1.0")], install=True, ) @@ -508,10 +509,7 @@ def test_project_install( index_url=local_module_package_index, # We add tornado, as there is a code path in update for the case where the project has python requires python_requires=["tornado"] - + [ - parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) - for mod in install_module_names - ], + + [parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names], install_project=False, ) @@ -544,8 +542,7 @@ def test_project_install( autostd=False, python_package_sources=[local_module_package_index], python_requires=[ - parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) - for mod in install_module_names + parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names ] + ["lorem"], install_project=False, @@ -681,7 +678,7 @@ def test_project_install_modules_cache_invalid( index_url=index.url, extra_index_url=[local_module_package_index], # make sure main module gets installed, pulling in newest version of dependency module - python_requires=[parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for(main_module))], + python_requires=[parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(main_module))], ) # populate project.modules[dependency_module] to force the error conditions in this simplified example @@ -766,7 +763,7 @@ def test_project_install_incompatible_versions( install_project=False, add_to_module_path=[v1_modules_path], index_url=index.url, - python_requires=[parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for(v2_mod_name))], + python_requires=[parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(v2_mod_name))], ) # install project @@ -819,14 +816,14 @@ def test_project_install_incompatible_dependencies( v2_template_path, os.path.join(str(tmpdir), "v2mod2"), new_name="v2mod2", - new_requirements=[parse_canonical_requirement(requirement="inmanta-module-v2mod1~=1.0.0")], + new_requirements=[parse_requirement(requirement="inmanta-module-v2mod1~=1.0.0")], publish_index=index, ) v2mod3: module.ModuleV2Metadata = module_from_template( v2_template_path, os.path.join(str(tmpdir), "v2mod3"), new_name="v2mod3", - new_requirements=[parse_canonical_requirement(requirement="inmanta-module-v2mod1~=2.0.0")], + new_requirements=[parse_requirement(requirement="inmanta-module-v2mod1~=2.0.0")], publish_index=index, ) @@ -840,7 +837,7 @@ def test_project_install_incompatible_dependencies( install_project=False, index_url=index.url, python_requires=[ - parse_canonical_requirement( + parse_requirement( requirement=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod2, v2mod3] @@ -924,7 +921,7 @@ def test_install_from_index_dont_leak_pip_index( # Installing a V2 module requires a python package source. index_url="unknown", python_requires=[ - parse_canonical_requirement( + parse_requirement( requirement=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod1] @@ -986,7 +983,7 @@ def test_install_with_use_config( index_url=index.url if not use_pip_config else None, use_pip_config_file=use_pip_config, python_requires=[ - parse_canonical_requirement( + parse_requirement( requirement=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod1] @@ -1055,7 +1052,7 @@ def test_install_with_use_config_extra_index( extra_index_url=[index2.url], use_pip_config_file=True, python_requires=[ - parse_canonical_requirement( + parse_requirement( requirement=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod1, v2mod2] @@ -1091,7 +1088,7 @@ def test_install_with_use_config_but_PIP_CONFIG_FILE_not_set( autostd=False, install_project=False, use_pip_config_file=True, - python_requires=[parse_canonical_requirement(requirement="inmanta-module-dummy-module")], + python_requires=[parse_requirement(requirement="inmanta-module-dummy-module")], ) # install project @@ -1208,7 +1205,7 @@ def test_install_project_with_install_mode_master(tmpdir: py.path.local, snippet autostd=False, install_project=False, add_to_module_path=[str(tmpdir)], - project_requires=[InmantaModuleRequirement(parse_canonical_requirement(requirement="mod11==3.2.1"))], + project_requires=[InmantaModuleRequirement(parse_requirement(requirement="mod11==3.2.1"))], install_mode=InstallMode.master, ) @@ -1236,7 +1233,7 @@ def test_module_install_logging(local_module_package_index: str, snippetcompiler v2_module = "minimalv2module" - v2_requirements = [parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for(v2_module))] + v2_requirements = [parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(v2_module))] # set up project and modules project: module.Project = snippetcompiler_clean.setup_for_snippet( @@ -1337,9 +1334,7 @@ def test_pip_output(local_module_package_index: str, snippetcompiler_clean, capl ) modules = ["modone", "modtwo"] - v2_requirements = [ - parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in modules - ] + v2_requirements = [parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in modules] snippetcompiler_clean.setup_for_snippet( f""" @@ -1425,9 +1420,7 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[ - parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module")) - ], + python_requires=[parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module"))], install_project=True, ) log_contains( @@ -1459,9 +1452,7 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[ - parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module")) - ], + python_requires=[parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module"))], install_project=True, ) @@ -1494,7 +1485,7 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module"))], install_project=True, ) log_contains( @@ -1563,7 +1554,7 @@ def test_version_snapshot(local_module_package_index: str, snippetcompiler_clean autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for("module_b"))], + python_requires=[parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("module_b"))], install_project=True, ) @@ -1589,7 +1580,7 @@ def test_version_snapshot(local_module_package_index: str, snippetcompiler_clean autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for("module_c"))], + python_requires=[parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("module_c"))], install_project=True, ) @@ -1660,8 +1651,7 @@ def test_constraints_logging_v2(modules_v2_dir, tmpdir, caplog, snippetcompiler_ index_url=local_module_package_index, extra_index_url=[index.url], python_requires=[ - parse_canonical_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) - for mod in ["module_b", "module_a"] + parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in ["module_b", "module_a"] ], install_project=True, project_requires=[ diff --git a/tests/moduletool/test_update.py b/tests/moduletool/test_update.py index 659d58587a..6daef61242 100644 --- a/tests/moduletool/test_update.py +++ b/tests/moduletool/test_update.py @@ -24,10 +24,11 @@ from inmanta import env from inmanta.config import Config from inmanta.data.model import PipConfig -from inmanta.env import LocalPackagePath, parse_canonical_requirement, parse_canonical_requirements +from inmanta.env import LocalPackagePath from inmanta.module import InmantaModuleRequirement, InstallMode, ModuleV1, ModuleV2Source from inmanta.moduletool import ProjectTool from inmanta.parser import ParserException +from inmanta.util import parse_requirement, parse_requirements from moduletool.common import add_file, clone_repo from packaging.version import Version from utils import PipIndex, create_python_package, module_from_template, v1_module_from_template @@ -126,7 +127,7 @@ def assert_version_installed(module_name: str, version: str) -> None: new_version=Version(current_version), new_name=module_name, new_requirements=( - [InmantaModuleRequirement(parse_canonical_requirement(requirement="module2<3.0.0"))] + [InmantaModuleRequirement(parse_requirement(requirement="module2<3.0.0"))] if module_name == "module1" else None ), @@ -143,7 +144,7 @@ def assert_version_installed(module_name: str, version: str) -> None: # Add a dependency on module2, without setting an explicit version constraint. Later version of module1 # do set a version constraint on the dependency on module2. This way it is verified whether the module update # command takes into account the version constraints set in a new version of a module. - new_requirements=[InmantaModuleRequirement(parse_canonical_requirement(requirement="module2"))], + new_requirements=[InmantaModuleRequirement(parse_requirement(requirement="module2"))], install=False, publish_index=pip_index, new_content_init_cf="entity" if corrupt_module else None, # Introduce syntax error in the module @@ -243,7 +244,7 @@ def test_module_update_dependencies( "b", Version(v), str(tmpdir.join(f"b-{v}")), - requirements=[parse_canonical_requirement(requirement="c")], + requirements=[parse_requirement(requirement="c")], publish_index=index, ) for v in ("1.0.0", "2.0.0"): @@ -259,7 +260,7 @@ def test_module_update_dependencies( # install b-1.0.0 and c-1.0.0 env.process_env.install_for_config( - [parse_canonical_requirement(requirement=req) for req in ("b==1.0.0", "c==1.0.0")], + [parse_requirement(requirement=req) for req in ("b==1.0.0", "c==1.0.0")], config=PipConfig( index_url=index.url, use_system_config=False, @@ -271,7 +272,7 @@ def test_module_update_dependencies( source_dir=os.path.join(modules_dir, "minimalv1module"), dest_dir=str(tmpdir.join("modules", "my_mod")), new_name="my_mod", - new_requirements=parse_canonical_requirements(["a", "b~=1.0.0"]), + new_requirements=parse_requirements(["a", "b~=1.0.0"]), ) # run `inmanta project update` without running install first diff --git a/tests/server/test_compilerservice.py b/tests/server/test_compilerservice.py index c0787b65c5..4440939828 100644 --- a/tests/server/test_compilerservice.py +++ b/tests/server/test_compilerservice.py @@ -41,7 +41,7 @@ from inmanta.const import INMANTA_REMOVED_SET_ID, ParameterSource from inmanta.data import APILIMIT, Compile, Report from inmanta.data.model import PipConfig -from inmanta.env import PythonEnvironment, parse_canonical_requirement +from inmanta.env import PythonEnvironment from inmanta.export import cfg_env from inmanta.protocol import Result from inmanta.server import SLICE_COMPILER, SLICE_SERVER, protocol @@ -49,7 +49,7 @@ from inmanta.server.protocol import Server from inmanta.server.services.compilerservice import CompilerService, CompileRun, CompileStateListener from inmanta.server.services.notificationservice import NotificationService -from inmanta.util import ensure_directory_exist +from inmanta.util import ensure_directory_exist, parse_requirement from server.conftest import EnvironmentFactory from utils import LogSequence, report_db_index_usage, retry_limited, wait_for_version @@ -1806,7 +1806,7 @@ def patch_get_protected_inmanta_packages(): venv = PythonEnvironment(env_path=venv_path) assert name_protected_pkg not in venv.get_installed_packages() venv.install_for_config( - requirements=[parse_canonical_requirement(requirement=name_protected_pkg)], + requirements=[parse_requirement(requirement=name_protected_pkg)], config=PipConfig( index_url=local_module_package_index, ), diff --git a/tests/test_env.py b/tests/test_env.py index db08b7c202..9329cf64ac 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -33,9 +33,10 @@ import py import pytest +import packaging.requirements from inmanta import env, loader, module from inmanta.data.model import PipConfig -from inmanta.env import Pip, Requirement +from inmanta.env import Pip from inmanta.util import parse_requirement from packaging import version from utils import LogSequence, PipIndex, create_python_package @@ -452,7 +453,7 @@ def test_active_env_get_module_file_editable_namespace_package( def create_install_package( - name: str, version: version.Version, requirements: list[Requirement], local_module_package_index: str + name: str, version: version.Version, requirements: list[packaging.requirements.Requirement], local_module_package_index: str ) -> None: """ Creates and installs a simple package with specified requirements. Creates package in a temporary directory and @@ -563,7 +564,7 @@ def test_active_env_check_constraints(caplog, tmpvenv_active_inherit: str, local """ caplog.set_level(logging.WARNING) in_scope: Pattern[str] = re.compile("test-package-.*") - constraints: list[Requirement] = [parse_requirement(requirement="test-package-one~=1.0")] + constraints: list[packaging.requirements.Requirement] = [parse_requirement(requirement="test-package-one~=1.0")] env.process_env.check(in_scope) diff --git a/tests/test_file_parser.py b/tests/test_file_parser.py index 45bd373609..57eefe440c 100644 --- a/tests/test_file_parser.py +++ b/tests/test_file_parser.py @@ -19,8 +19,8 @@ import os import packaging.requirements -from inmanta.env import parse_canonical_requirements from inmanta.file_parser import RequirementsTxtParser +from inmanta.util import parse_requirements def test_requirements_txt_parser(tmpdir) -> None: @@ -41,7 +41,7 @@ def test_requirements_txt_parser(tmpdir) -> None: expected_requirements = ["test==1.2.3", "other-dep~=2.0.0", "third-dep<5.0.0", "splitteddep", "Capital"] requirements: list[packaging.requirements.Requirement] = RequirementsTxtParser().parse(requirements_txt_file) - assert requirements == parse_canonical_requirements(expected_requirements) + assert requirements == parse_requirements(expected_requirements) requirements_as_str = RequirementsTxtParser.parse_requirements_as_strs(requirements_txt_file) assert requirements_as_str == expected_requirements diff --git a/tests/test_module_loader.py b/tests/test_module_loader.py index 0543d10a0f..f802abd670 100644 --- a/tests/test_module_loader.py +++ b/tests/test_module_loader.py @@ -33,7 +33,7 @@ from inmanta.ast import CompilerException from inmanta.const import CF_CACHE_DIR from inmanta.data.model import PipConfig -from inmanta.env import ConflictingRequirements, LocalPackagePath, PackageNotFound, Requirement, parse_canonical_requirement +from inmanta.env import ConflictingRequirements, LocalPackagePath, PackageNotFound, Requirement, parse_requirement from inmanta.module import ( DummyProject, InmantaModuleRequirement, @@ -345,7 +345,7 @@ def test_load_module_recursive_v2_module_depends_on_v1( project = snippetcompiler.setup_for_snippet( snippet="import v2_depends_on_v1", index_url=local_module_package_index, - python_requires=[parse_canonical_requirement(requirement="inmanta-module-v2-depends-on-v1")], + python_requires=[parse_requirement(requirement="inmanta-module-v2-depends-on-v1")], install_project=False, ) if preload_v1_module: @@ -372,7 +372,7 @@ def test_load_module_recursive_complex_module_dependencies(local_module_package_ snippet="import complex_module_dependencies_mod1", autostd=False, index_url=local_module_package_index, - python_requires=[parse_canonical_requirement(requirement="inmanta-module-complex-module-dependencies-mod1")], + python_requires=[parse_requirement(requirement="inmanta-module-complex-module-dependencies-mod1")], install_project=False, ) assert "complex_module_dependencies_mod1" not in project.modules @@ -411,7 +411,7 @@ def load(requires: Optional[list[Requirement]] = None) -> None: with pytest.raises(ModuleLoadingException, match=f"Failed to load module {module_name}"): load() # assert that it doesn't raise an error with explicit requirements set - load([parse_canonical_requirement(requirement=ModuleV2Source.get_package_name_for(module_name))]) + load([parse_requirement(requirement=ModuleV2Source.get_package_name_for(module_name))]) @pytest.mark.parametrize("v1", [True, False]) @@ -468,9 +468,7 @@ def test_load_import_based_v2_module( extra_index_url=[index.url], # make sure that even listing the requirement in project.yml does not suffice project_requires=[InmantaModuleRequirement.parse(dependency_module_name)], - python_requires=( - [] if v1 else [parse_canonical_requirement(requirement=ModuleV2Source.get_package_name_for(main_module_name))] - ), + python_requires=([] if v1 else [parse_requirement(requirement=ModuleV2Source.get_package_name_for(main_module_name))]), ) if explicit_dependency: @@ -612,7 +610,7 @@ def test_project_requirements_dont_overwrite_core_requirements_source( module_from_template( os.path.join(modules_v2_dir, module_name), module_path, - new_requirements=[parse_canonical_requirement(requirement="Jinja2==2.11.3")], + new_requirements=[parse_requirement(requirement="Jinja2==2.11.3")], ) # Activate the snippetcompiler venv @@ -656,7 +654,7 @@ def test_project_requirements_dont_overwrite_core_requirements_index( module_from_template( os.path.join(modules_v2_dir, module_name), module_path, - new_requirements=[parse_canonical_requirement(requirement="Jinja2==2.11.3")], + new_requirements=[parse_requirement(requirement="Jinja2==2.11.3")], publish_index=index, ) @@ -710,7 +708,7 @@ def test_module_conflicting_dependencies_with_v2_modules( "y", Version("1.0.0"), str(tmpdir.join("y-1.0.0")), - requirements=[parse_canonical_requirement(requirement="x~=1.0.0")], + requirements=[parse_requirement(requirement="x~=1.0.0")], publish_index=index, ) @@ -720,7 +718,7 @@ def test_module_conflicting_dependencies_with_v2_modules( module_from_template( os.path.join(modules_v2_dir, module_name1), module_path1, - new_requirements=[parse_canonical_requirement(requirement="y~=1.0.0")], + new_requirements=[parse_requirement(requirement="y~=1.0.0")], publish_index=index, ) @@ -731,7 +729,7 @@ def test_module_conflicting_dependencies_with_v2_modules( os.path.join(modules_v2_dir, "minimalv2module"), module_path2, new_name="minimalv2module2", - new_requirements=[parse_canonical_requirement(requirement="x~=2.0.0")], + new_requirements=[parse_requirement(requirement="x~=2.0.0")], publish_index=index, ) @@ -791,7 +789,7 @@ def test_module_conflicting_dependencies_with_v1_module( os.path.join(modules_dir, module_name1), module_path1, new_name="modulev1", - new_requirements=[parse_canonical_requirement(requirement="y~=1.0.0")], + new_requirements=[parse_requirement(requirement="y~=1.0.0")], ) # Create the second module @@ -800,7 +798,7 @@ def test_module_conflicting_dependencies_with_v1_module( module_from_template( os.path.join(modules_v2_dir, module_name2), module_path2, - new_requirements=[parse_canonical_requirement(requirement="y~=2.0.0")], + new_requirements=[parse_requirement(requirement="y~=2.0.0")], publish_index=index, ) @@ -848,7 +846,7 @@ def test_module_install_extra_on_project_level_v2_dep( new_name="mymod", new_requirements=[], new_extras={ - "myfeature": [parse_canonical_requirement(requirement=package_name_extra)], + "myfeature": [parse_requirement(requirement=package_name_extra)], }, publish_index=index, ) @@ -897,7 +895,7 @@ def test_module_install_extra_on_dep_of_v2_module( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [parse_canonical_requirement(requirement=package_name_extra)], + "myfeature": [parse_requirement(requirement=package_name_extra)], }, publish_index=index, ) @@ -917,7 +915,7 @@ def test_module_install_extra_on_dep_of_v2_module( install_project=True, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[parse_canonical_requirement(requirement="inmanta-module-myv2mod")], + python_requires=[parse_requirement(requirement="inmanta-module-myv2mod")], autostd=False, ) @@ -954,7 +952,7 @@ def test_module_install_extra_on_dep_of_v1_module( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [parse_canonical_requirement(requirement=package_name_extra)], + "myfeature": [parse_requirement(requirement=package_name_extra)], }, publish_index=index, ) @@ -1004,7 +1002,7 @@ def test_module_install_extra_on_project_level_v2_dep_update_scenario( new_name="mymod", new_requirements=[], new_extras={ - "myfeature": [parse_canonical_requirement(requirement=package_name_extra)], + "myfeature": [parse_requirement(requirement=package_name_extra)], }, publish_index=index, ) @@ -1073,7 +1071,7 @@ def test_module_install_extra_on_dep_of_v2_module_update_scenario( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [parse_canonical_requirement(requirement=package_name_extra)], + "myfeature": [parse_requirement(requirement=package_name_extra)], }, publish_index=index, ) @@ -1102,7 +1100,7 @@ def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> install_project=True, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[parse_canonical_requirement(requirement="inmanta-module-myv2mod==1.0.0")], + python_requires=[parse_requirement(requirement="inmanta-module-myv2mod==1.0.0")], autostd=False, ) assert_installed(extra_installed=False) @@ -1124,7 +1122,7 @@ def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> install_project=not do_project_update, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[parse_canonical_requirement(requirement="inmanta-module-myv2mod==2.0.0")], + python_requires=[parse_requirement(requirement="inmanta-module-myv2mod==2.0.0")], autostd=False, ) if do_project_update: @@ -1168,7 +1166,7 @@ def test_module_install_extra_on_dep_of_v1_module_update_scenario( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [parse_canonical_requirement(requirement=package_name_extra)], + "myfeature": [parse_requirement(requirement=package_name_extra)], }, publish_index=index, ) @@ -1232,7 +1230,7 @@ async def test_v1_module_depends_on_third_party_dep_with_extra( os.path.join(tmpdir, "myv1mod"), new_name="myv1mod", new_content_init_cf="", - new_requirements=[parse_canonical_requirement(requirement="pkg[optional-a]")], + new_requirements=[parse_requirement(requirement="pkg[optional-a]")], ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv1mod", @@ -1252,7 +1250,7 @@ async def test_v1_module_depends_on_third_party_dep_with_extra( os.path.join(tmpdir, "myv1mod"), new_name="myv1mod", new_content_init_cf="", - new_requirements=[parse_canonical_requirement(requirement="pkg[optional-a,optional-b]")], + new_requirements=[parse_requirement(requirement="pkg[optional-a,optional-b]")], ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv1mod", @@ -1279,13 +1277,13 @@ async def test_v2_module_depends_on_third_party_dep_with_extra( str(tmpdir.join("myv2mod")), new_name="myv2mod", new_version=Version("1.0.0"), - new_requirements=[parse_canonical_requirement(requirement="pkg[optional-a]")], + new_requirements=[parse_requirement(requirement="pkg[optional-a]")], publish_index=index, ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv2mod", install_project=True, - python_requires=[parse_canonical_requirement(requirement="inmanta-module-myv2mod==1.0.0")], + python_requires=[parse_requirement(requirement="inmanta-module-myv2mod==1.0.0")], index_url=index.url, extra_index_url=[index_with_pkgs_containing_optional_deps], autostd=False, @@ -1301,13 +1299,13 @@ async def test_v2_module_depends_on_third_party_dep_with_extra( str(tmpdir.join("myv2mod")), new_name="myv2mod", new_version=Version("2.0.0"), - new_requirements=[parse_canonical_requirement(requirement="pkg[optional-a,optional-b]")], + new_requirements=[parse_requirement(requirement="pkg[optional-a,optional-b]")], publish_index=index, ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv2mod", install_project=True, - python_requires=[parse_canonical_requirement(requirement="inmanta-module-myv2mod==2.0.0")], + python_requires=[parse_requirement(requirement="inmanta-module-myv2mod==2.0.0")], index_url=index.url, extra_index_url=[index_with_pkgs_containing_optional_deps], autostd=False, diff --git a/tests/utils.py b/tests/utils.py index cb6a821fce..30a6061235 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -40,6 +40,7 @@ import build import build.env +import packaging.requirements import packaging.version from _pytest.mark import MarkDecorator from inmanta import config, const, data, env, module, protocol, util @@ -51,7 +52,6 @@ from inmanta.server.extensions import ProductMetadata from inmanta.util import get_compiler_version, hash_file from libpip2pi.commands import dir2pi -from packaging.requirements import Requirement from packaging.version import Version T = TypeVar("T") @@ -491,11 +491,11 @@ def create_python_package( pkg_version: packaging.version.Version, path: str, *, - requirements: Optional[Sequence[Requirement]] = None, + requirements: Optional[Sequence[packaging.requirements.Requirement]] = None, install: bool = False, editable: bool = False, publish_index: Optional[PipIndex] = None, - optional_dependencies: Optional[dict[str, Sequence[Requirement]]] = None, + optional_dependencies: Optional[dict[str, Sequence[packaging.requirements.Requirement]]] = None, ) -> None: """ Creates an empty Python package. @@ -580,8 +580,10 @@ def module_from_template( *, new_version: Optional[packaging.version.Version] = None, new_name: Optional[str] = None, - new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, Requirement]]] = None, - new_extras: Optional[abc.Mapping[str, abc.Sequence[Union[module.InmantaModuleRequirement, Requirement]]]] = None, + new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, packaging.requirements.Requirement]]] = None, + new_extras: Optional[ + abc.Mapping[str, abc.Sequence[Union[module.InmantaModuleRequirement, packaging.requirements.Requirement]]] + ] = None, install: bool = False, editable: bool = False, publish_index: Optional[PipIndex] = None, @@ -609,8 +611,13 @@ def module_from_template( :param four_digit_version: if the version uses 4 digits (3 by default) """ - def to_python_requires(requires: abc.Sequence[Union[module.InmantaModuleRequirement, Requirement]]) -> list[str]: - return [str(req if isinstance(req, Requirement) else str(req.get_python_package_requirement())) for req in requires] + def to_python_requires( + requires: abc.Sequence[Union[module.InmantaModuleRequirement, packaging.requirements.Requirement]] + ) -> list[str]: + return [ + str(req if isinstance(req, packaging.requirements.Requirement) else str(req.get_python_package_requirement())) + for req in requires + ] if (dest_dir is None) != in_place: raise ValueError("Either dest_dir or in_place must be set, never both.") @@ -691,7 +698,7 @@ def v1_module_from_template( *, new_version: Optional[packaging.version.Version] = None, new_name: Optional[str] = None, - new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, Requirement]]] = None, + new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, packaging.requirements.Requirement]]] = None, new_content_init_cf: Optional[str] = None, new_content_init_py: Optional[str] = None, ) -> module.ModuleV2Metadata: @@ -731,7 +738,7 @@ def v1_module_from_template( with open(os.path.join(dest_dir, "requirements.txt"), "w") as fd: fd.write( "\n".join( - str(req if isinstance(req, Requirement) else req.get_python_package_requirement()) + str(req if isinstance(req, packaging.requirements.Requirement) else req.get_python_package_requirement()) for req in new_requirements ) ) From 43a64c0741fdcadba48c7da92282a8f8f3f9cc48 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Mon, 16 Sep 2024 13:27:57 +0200 Subject: [PATCH 34/74] fix broken tests --- src/inmanta/module.py | 28 ++++++++++++---------------- src/inmanta/moduletool.py | 2 +- tests/test_module_loader.py | 27 ++++++++++++++++++--------- 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/src/inmanta/module.py b/src/inmanta/module.py index e6dbc59715..b240d6b7f2 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -97,19 +97,23 @@ class InmantaModuleRequirement: used by distinguishing the two on a type level. """ - def __init__(self, requirement: packaging.requirements.Requirement) -> None: + def __init__(self, requirement: util.CanonicalRequirement) -> None: if requirement.name.startswith(ModuleV2.PKG_NAME_PREFIX): raise ValueError( f"InmantaModuleRequirement instances work with inmanta module names, not python package names. " f"Problematic case: {str(requirement)}" ) - self._requirement: packaging.requirements.Requirement = requirement + self._requirement: util.CanonicalRequirement = requirement @property def project_name(self) -> str: # Requirement converts all "_" to "-". Inmanta modules use "_" return self._requirement.name.replace("-", "_") + @property + def name(self) -> str: + return self._requirement.name + @property def key(self) -> str: # Requirement converts all "_" to "-". Inmanta modules use "_" @@ -125,7 +129,7 @@ def __eq__(self, other: object) -> bool: return self._requirement == other._requirement def __contains__(self, version: packaging.version.Version | str) -> bool: - return version in self._requirement.specifier + return version in self._requirement.specifier if len(self._requirement.specifier) > 0 else True def __str__(self) -> str: return str(self._requirement).replace("-", "_") @@ -133,16 +137,6 @@ def __str__(self) -> str: def __hash__(self) -> int: return self._requirement.__hash__() - @property - def specs(self) -> Sequence[tuple[str, str]]: - return [(e.operator, e.version) for e in self._requirement.specifier] - - def version_spec_str(self) -> str: - """ - Returns a string representation of this module requirement's version spec. Includes only the version part. - """ - return ",".join("".join(spec) for spec in self.specs) - @classmethod def parse(cls: type[TInmantaModuleRequirement], spec: str) -> TInmantaModuleRequirement: if spec.startswith(ModuleV2.PKG_NAME_PREFIX): @@ -596,7 +590,9 @@ def _format_constraints(self, module_name: str, module_spec: list[InmantaModuleR :param module_name: The name of the module. :param module_spec: List of inmanta requirements in which to look for the module. """ - constraints_on_module: list[str] = [str(req) for req in module_spec if module_name == req.key and req.specs] + constraints_on_module: list[str] = [ + str(req) for req in module_spec if module_name == req.key and len(req.specifier) > 0 + ] if constraints_on_module: from_constraints = f"(with constraints {' '.join(constraints_on_module)})" else: @@ -738,7 +734,7 @@ def install(self, project: "Project", module_spec: list[InmantaModuleRequirement "Currently installed %s-%s does not match constraint %s: updating to compatible version.", module_name, preinstalled_version, - ",".join(constraint.version_spec_str() for constraint in module_spec if constraint.specs), + ",".join(str(constraint.specifier) for constraint in module_spec if len(constraint.specifier) > 0), ) try: self.log_pre_install_information(module_name, module_spec) @@ -913,7 +909,7 @@ def install(self, project: "Project", module_spec: list[InmantaModuleRequirement "Currently installed %s-%s does not match constraint %s: updating to compatible version.", module_name, preinstalled_version, - ",".join(constraint.version_spec_str() for constraint in module_spec if constraint.specs), + ",".join(str(constraint.specifier) for constraint in module_spec if len(constraint.specifier) > 0), ) self.log_pre_install_information(module_name, module_spec) modules_pre_install = self.take_modules_snapshot(project, header="Modules versions before installation:") diff --git a/src/inmanta/moduletool.py b/src/inmanta/moduletool.py index 36a163b7d2..f02cd87b7a 100644 --- a/src/inmanta/moduletool.py +++ b/src/inmanta/moduletool.py @@ -1003,7 +1003,7 @@ def show_bool(b: bool) -> str: matches = version == reqv editable = True else: - reqv = ",".join(req.version_spec_str() for req in specs[name] if req.specs) or "*" + reqv = ",".join(str(req.specifier) for req in specs[name] if len(req.specifier) > 0) or "*" matches = all(version in req for req in specs[name]) editable = mod.is_editable() diff --git a/tests/test_module_loader.py b/tests/test_module_loader.py index f802abd670..2ccc38a755 100644 --- a/tests/test_module_loader.py +++ b/tests/test_module_loader.py @@ -33,7 +33,7 @@ from inmanta.ast import CompilerException from inmanta.const import CF_CACHE_DIR from inmanta.data.model import PipConfig -from inmanta.env import ConflictingRequirements, LocalPackagePath, PackageNotFound, Requirement, parse_requirement +from inmanta.env import ConflictingRequirements, LocalPackagePath, PackageNotFound from inmanta.module import ( DummyProject, InmantaModuleRequirement, @@ -45,6 +45,7 @@ Project, ) from inmanta.moduletool import ModuleConverter, ModuleTool, ProjectTool +from inmanta.util import CanonicalRequirement, parse_requirement from packaging.version import Version from utils import PipIndex, create_python_package, log_contains, module_from_template, v1_module_from_template @@ -396,7 +397,7 @@ def test_load_import_based_v2_project(local_module_package_index: str, snippetco """ module_name: str = "minimalv2module" - def load(requires: Optional[list[Requirement]] = None) -> None: + def load(requires: Optional[list[CanonicalRequirement]] = None) -> None: project: Project = snippetcompiler_clean.setup_for_snippet( f"import {module_name}", autostd=False, @@ -850,7 +851,9 @@ def test_module_install_extra_on_project_level_v2_dep( }, publish_index=index, ) - package_with_extra: Requirement = InmantaModuleRequirement.parse("mymod[myfeature]").get_python_package_requirement() + package_with_extra: CanonicalRequirement = InmantaModuleRequirement.parse( + "mymod[myfeature]" + ).get_python_package_requirement() package_name: str = f"{ModuleV2.PKG_NAME_PREFIX}mymod" # project with dependency on mymod with extra @@ -1006,8 +1009,10 @@ def test_module_install_extra_on_project_level_v2_dep_update_scenario( }, publish_index=index, ) - package_without_extra: Requirement = InmantaModuleRequirement.parse("mymod").get_python_package_requirement() - package_with_extra: Requirement = InmantaModuleRequirement.parse("mymod[myfeature]").get_python_package_requirement() + package_without_extra: CanonicalRequirement = InmantaModuleRequirement.parse("mymod").get_python_package_requirement() + package_with_extra: CanonicalRequirement = InmantaModuleRequirement.parse( + "mymod[myfeature]" + ).get_python_package_requirement() package_name: str = str(package_without_extra) def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> None: @@ -1075,8 +1080,10 @@ def test_module_install_extra_on_dep_of_v2_module_update_scenario( }, publish_index=index, ) - package_without_extra: Requirement = InmantaModuleRequirement.parse("depmod").get_python_package_requirement() - package_with_extra: Requirement = InmantaModuleRequirement.parse("depmod[myfeature]").get_python_package_requirement() + package_without_extra: CanonicalRequirement = InmantaModuleRequirement.parse("depmod").get_python_package_requirement() + package_with_extra: CanonicalRequirement = InmantaModuleRequirement.parse( + "depmod[myfeature]" + ).get_python_package_requirement() package_name: str = str(package_without_extra) def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> None: @@ -1156,8 +1163,10 @@ def test_module_install_extra_on_dep_of_v1_module_update_scenario( index: PipIndex = PipIndex(artifact_dir=str(tmpdir.join(".index"))) # Publish dependency of V1 module (depmod) to python package repo - package_without_extra: Requirement = InmantaModuleRequirement.parse("depmod").get_python_package_requirement() - package_with_extra: Requirement = InmantaModuleRequirement.parse("depmod[myfeature]").get_python_package_requirement() + package_without_extra: CanonicalRequirement = InmantaModuleRequirement.parse("depmod").get_python_package_requirement() + package_with_extra: CanonicalRequirement = InmantaModuleRequirement.parse( + "depmod[myfeature]" + ).get_python_package_requirement() package_name: str = str(package_without_extra) module_from_template( From 917f9efa655748f68b95d518cf63c4c6da455288 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Mon, 16 Sep 2024 13:39:30 +0200 Subject: [PATCH 35/74] add warnings + use new name property --- src/inmanta/module.py | 27 +++++++++++++++------------ src/inmanta/moduletool.py | 2 +- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/inmanta/module.py b/src/inmanta/module.py index b240d6b7f2..b7d2e8e189 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -108,6 +108,8 @@ def __init__(self, requirement: util.CanonicalRequirement) -> None: @property def project_name(self) -> str: # Requirement converts all "_" to "-". Inmanta modules use "_" + warnings.warn(InmantaWarning("The `project_name` property has been deprecated in favor of `name`")) + return self._requirement.name.replace("-", "_") @property @@ -117,6 +119,7 @@ def name(self) -> str: @property def key(self) -> str: # Requirement converts all "_" to "-". Inmanta modules use "_" + warnings.warn(InmantaWarning("The `key` property has been deprecated in favor of `name`")) return self._requirement.name.replace("-", "_") @property @@ -151,7 +154,7 @@ def get_python_package_requirement(self) -> util.CanonicalRequirement: """ Return a Requirement with the name of the Python distribution package for this module requirement. """ - module_name = self.project_name + module_name = self.name pkg_name = ModuleV2Source.get_package_name_for(module_name) pkg_req_str = str(self).replace(module_name, pkg_name, 1) # Replace max 1 occurrence return parse_requirement(requirement=pkg_req_str) @@ -591,7 +594,7 @@ def _format_constraints(self, module_name: str, module_spec: list[InmantaModuleR :param module_spec: List of inmanta requirements in which to look for the module. """ constraints_on_module: list[str] = [ - str(req) for req in module_spec if module_name == req.key and len(req.specifier) > 0 + str(req) for req in module_spec if module_name == req.name and len(req.specifier) > 0 ] if constraints_on_module: from_constraints = f"(with constraints {' '.join(constraints_on_module)})" @@ -665,7 +668,7 @@ def from_path(cls, project: Optional["Project"], module_name: str, path: str) -> raise NotImplementedError("Abstract method") def _get_module_name(self, module_spec: list[InmantaModuleRequirement]) -> str: - module_names: set[str] = {req.project_name for req in module_spec} + module_names: set[str] = {req.name for req in module_spec} module_name: str = more_itertools.one( module_names, too_short=ValueError("module_spec should contain at least one requirement"), @@ -1052,7 +1055,7 @@ def make_repo(path: str, root: Optional[str] = None) -> Union[LocalFileRepo, Rem def merge_specs(mainspec: "Dict[str, List[InmantaModuleRequirement]]", new: "List[InmantaModuleRequirement]") -> None: """Merge two maps str->[TMetadata] by concatting their lists.""" for req in new: - key = req.project_name + key = req.name if key not in mainspec: mainspec[key] = [req] else: @@ -1829,7 +1832,7 @@ def has_module_requirement(self, module_name: str) -> bool: declare dependencies module dependencies. This could include the requirements.txt file next to the metadata file of the project or module. """ - return any(module_name == InmantaModuleRequirement.parse(req).key for req in self.get_module_requirements()) + return any(module_name == InmantaModuleRequirement.parse(req).name for req in self.get_module_requirements()) def _load_file(self, ns: Namespace, file: str) -> tuple[list[Statement], BasicBlock]: ns.location = Location(file, 1) @@ -1901,7 +1904,7 @@ def add_module_requirement_to_requires_and_write(self, requirement: InmantaModul # Update requires if "requires" in content and content["requires"]: existing_matching_reqs: list[str] = [ - r for r in content["requires"] if InmantaModuleRequirement.parse(r).key == requirement.key + r for r in content["requires"] if InmantaModuleRequirement.parse(r).name == requirement.name ] for r in existing_matching_reqs: content["requires"].remove(r) @@ -1918,7 +1921,7 @@ def has_module_requirement_in_requires(self, module_name: str) -> bool: content: CommentedMap = PreservativeYamlParser.parse(self.get_metadata_file_path()) if "requires" not in content: return False - return any(r for r in content["requires"] if InmantaModuleRequirement.parse(r).key == module_name) + return any(r for r in content["requires"] if InmantaModuleRequirement.parse(r).name == module_name) def remove_module_requirement_from_requires_and_write(self, module_name: str) -> None: """ @@ -1929,7 +1932,7 @@ def remove_module_requirement_from_requires_and_write(self, module_name: str) -> if not self.has_module_requirement_in_requires(module_name): return content: CommentedMap = PreservativeYamlParser.parse(self.get_metadata_file_path()) - content["requires"] = [r for r in content["requires"] if InmantaModuleRequirement.parse(r).key != module_name] + content["requires"] = [r for r in content["requires"] if InmantaModuleRequirement.parse(r).name != module_name] PreservativeYamlParser.dump(self.get_metadata_file_path(), content) @@ -2315,13 +2318,13 @@ def load_module_v2_requirements(module_like: ModuleLike) -> None: for requirement in module_like.get_module_v2_requirements(): # load module self.get_module( - requirement.key, + requirement.name, allow_v1=False, install_v2=install, bypass_module_cache=bypass_module_cache, ) # queue AST reload - require_v2(requirement.key) + require_v2(requirement.name) def setup_module(module: Module) -> None: """ @@ -3153,7 +3156,7 @@ def get_all_requires(self) -> list[InmantaModuleRequirement]: :return: all modules required by an import from any sub-modules, with all constraints applied """ # get all constraints - spec: dict[str, InmantaModuleRequirement] = {req.project_name: req for req in self.requires()} + spec: dict[str, InmantaModuleRequirement] = {req.name: req for req in self.requires()} # find all imports imports = {imp.name.split("::")[0] for subm in sorted(self.get_all_submodules()) for imp in self.get_imports(subm)} return [spec[r] if spec.get(r) else InmantaModuleRequirement.parse(r) for r in imports] @@ -3291,7 +3294,7 @@ def add_module_requirement_persistent(self, requirement: InmantaModuleRequiremen requirements_txt_file = RequirementsTxtFile(requirements_txt_file_path, create_file_if_not_exists=True) requirements_txt_file.set_requirement_and_write(requirement.get_python_package_requirement()) # Remove requirement from module.yml file - self.remove_module_requirement_from_requires_and_write(requirement.key) + self.remove_module_requirement_from_requires_and_write(requirement.name) def versions(self) -> list[packaging.version.Version]: """ diff --git a/src/inmanta/moduletool.py b/src/inmanta/moduletool.py index f02cd87b7a..086279dce0 100644 --- a/src/inmanta/moduletool.py +++ b/src/inmanta/moduletool.py @@ -1949,7 +1949,7 @@ def get_setup_cfg(self, in_folder: str, warn_on_merge: bool = False) -> configpa # add requirements module_requirements: list[InmantaModuleRequirement] = [ - req for req in self._module.get_all_requires() if req.project_name != self._module.name + req for req in self._module.get_all_requires() if req.name != self._module.name ] python_requirements: list[str] = self._module.get_strict_python_requirements_as_list() if module_requirements or python_requirements: From 1c43dc79630c819e5d150ce05d82ca016c5e2b70 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Mon, 16 Sep 2024 14:23:34 +0200 Subject: [PATCH 36/74] fix + mypy --- mypy-baseline.txt | 24 +++++++++--------------- src/inmanta/module.py | 2 +- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/mypy-baseline.txt b/mypy-baseline.txt index 7a903b0947..53a9832261 100644 --- a/mypy-baseline.txt +++ b/mypy-baseline.txt @@ -103,6 +103,7 @@ src/inmanta/ast/type.py:0: error: No overload variant of "reversed" matches argu src/inmanta/ast/type.py:0: note: Possible overload variants: src/inmanta/ast/type.py:0: note: def [_T] __new__(cls, Reversible[_T], /) -> reversed[_T] src/inmanta/ast/type.py:0: note: def [_T] __new__(cls, SupportsLenAndGetItem[_T], /) -> reversed[_T] +src/inmanta/server/services/databaseservice.py:0: error: Incompatible return value type (got "dict[str, object]", expected "dict[str, BaseModel | UUID | bool | int | float | datetime | str | Sequence[BaseModel | UUID | bool | int | float | datetime | str] | Mapping[str, BaseModel | UUID | bool | int | float | datetime | str]]") [return-value] src/inmanta/protocol/decorators.py:0: error: "FuncT" has no attribute "__protocol_method__" [attr-defined] src/inmanta/protocol/decorators.py:0: error: "FuncT" has no attribute "__protocol_mapping__" [attr-defined] src/inmanta/protocol/decorators.py:0: error: "FuncT" has no attribute "__api_version__" [attr-defined] @@ -892,13 +893,8 @@ src/inmanta/execute/scheduler.py:0: error: Missing type parameters for generic t src/inmanta/execute/scheduler.py:0: error: "" has no attribute "attribute" [attr-defined] src/inmanta/execute/scheduler.py:0: error: "" has no attribute "attribute" [attr-defined] src/inmanta/execute/scheduler.py:0: error: "" has no attribute "attribute" [attr-defined] +src/inmanta/env.py:0: error: Argument 1 to "parse_requirements" has incompatible type "Sequence[Requirement]"; expected "Sequence[str]" [arg-type] src/inmanta/env.py:0: error: "type[WorkingSet]" has no attribute "_build_master" [attr-defined] -src/inmanta/env.py:0: error: Incompatible types in assignment (expression has type "WorkingSet", variable has type "Iterable[DistInfoDistribution]") [assignment] -src/inmanta/env.py:0: note: Following member(s) of "WorkingSet" have conflicts: -src/inmanta/env.py:0: note: Expected: -src/inmanta/env.py:0: note: def __iter__(self) -> Iterator[DistInfoDistribution] -src/inmanta/env.py:0: note: Got: -src/inmanta/env.py:0: note: def __iter__(self) -> Iterator[Distribution] src/inmanta/env.py:0: error: Incompatible return value type (got "tuple[str | None, Loader | None] | None", expected "tuple[str | None, Loader] | None") [return-value] src/inmanta/env.py:0: error: Item "None" of "ModuleSpec | None" has no attribute "submodule_search_locations" [union-attr] src/inmanta/env.py:0: error: Item "None" of "ModuleSpec | None" has no attribute "submodule_search_locations" [union-attr] @@ -909,7 +905,6 @@ src/inmanta/compiler/__init__.py:0: error: Incompatible types in assignment (exp src/inmanta/compiler/__init__.py:0: error: Incompatible types in assignment (expression has type "Namespace | None", variable has type "Namespace") [assignment] src/inmanta/compiler/__init__.py:0: error: Missing type parameters for generic type "ResultVariable" [type-arg] src/inmanta/module.py:0: error: Missing type parameters for generic type Module [type-arg] -src/inmanta/module.py:0: error: Incompatible return value type (got "SpecifierSet", expected "str") [return-value] src/inmanta/module.py:0: error: Returning Any from function declared to return "Mapping[str, object]" [no-any-return] src/inmanta/module.py:0: error: Argument 1 to "Metadata" has incompatible type "**Mapping[str, object]"; expected "str" [arg-type] src/inmanta/module.py:0: error: Argument 1 to "Metadata" has incompatible type "**Mapping[str, object]"; expected "str | None" [arg-type] @@ -926,18 +921,12 @@ src/inmanta/module.py:0: error: Missing type parameters for generic type Module src/inmanta/module.py:0: error: Missing type parameters for generic type Module [type-arg] src/inmanta/module.py:0: error: Missing type parameters for generic type Module [type-arg] src/inmanta/module.py:0: error: Missing type parameters for generic type Module [type-arg] -src/inmanta/module.py:0: error: Unsupported operand types for in ("Version" and "InmantaModuleRequirement") [operator] src/inmanta/module.py:0: error: Missing type parameters for generic type Module [type-arg] src/inmanta/module.py:0: error: Missing type parameters for generic type Module [type-arg] src/inmanta/module.py:0: error: Missing type parameters for generic type Module [type-arg] src/inmanta/module.py:0: error: Missing type parameters for generic type Module [type-arg] src/inmanta/module.py:0: error: Missing type parameters for generic type Module [type-arg] src/inmanta/module.py:0: error: Missing type parameters for generic type Module [type-arg] -src/inmanta/module.py:0: error: Incompatible return value type (got "pkg_resources._vendored_packaging.version.Version", expected "packaging.version.Version | None") [return-value] -src/inmanta/module.py:0: error: "str" has no attribute "filter" [attr-defined] -src/inmanta/module.py:0: error: Argument 4 to "__best_for_compiler_version" of "ModuleV1" has incompatible type "pkg_resources._vendored_packaging.version.Version"; expected "packaging.version.Version" [arg-type] -src/inmanta/module.py:0: error: Incompatible return value type (got "pkg_resources._vendored_packaging.version.Version", expected "packaging.version.Version | None") [return-value] -src/inmanta/module.py:0: error: Incompatible return value type (got "pkg_resources._vendored_packaging.version.Version", expected "packaging.version.Version | None") [return-value] src/inmanta/moduletool.py:0: error: Skipping analyzing "cookiecutter.main": module is installed, but missing library stubs or py.typed marker [import-untyped] src/inmanta/moduletool.py:0: error: Missing type parameters for generic type "ModuleLike" [type-arg] src/inmanta/moduletool.py:0: error: Missing type parameters for generic type Module [type-arg] @@ -1011,7 +1000,6 @@ src/inmanta/server/services/environment_metrics_service.py:0: error: "Sequence[E src/inmanta/server/services/environment_metrics_service.py:0: error: Argument 1 to "create_metrics_timer" has incompatible type "Sequence[MetricValue]"; expected "Sequence[MetricValueTimer]" [arg-type] src/inmanta/server/services/environment_metrics_service.py:0: error: "str" has no attribute "value" [attr-defined] src/inmanta/server/services/environment_metrics_service.py:0: error: Unsupported target for indexed assignment ("float | dict[str, float] | None") [index] -src/inmanta/server/services/databaseservice.py:0: error: Incompatible return value type (got "dict[str, object]", expected "dict[str, BaseModel | UUID | bool | int | float | datetime | str | Sequence[BaseModel | UUID | bool | int | float | datetime | str] | Mapping[str, BaseModel | UUID | bool | int | float | datetime | str]]") [return-value] src/inmanta/server/diff.py:0: error: "object" has no attribute "__iter__"; maybe "__dir__" or "__str__"? (not iterable) [attr-defined] src/inmanta/server/diff.py:0: error: Incompatible return value type (got "str | None", expected "str") [return-value] src/inmanta/protocol/openapi/converter.py:0: error: Returning Any from function declared to return "BaseModel | UUID | bool | int | float | datetime | str | Sequence[BaseModel | UUID | bool | int | float | datetime | str] | Mapping[str, BaseModel | UUID | bool | int | float | datetime | str] | None | JSONSerializable" [no-any-return] @@ -1060,6 +1048,12 @@ src/inmanta/model.py:0: error: Call to untyped function "ReferenceValue" in type src/inmanta/model.py:0: error: "Value" has no attribute "to_dict" [attr-defined] src/inmanta/model.py:0: error: "Value" has no attribute "to_dict" [attr-defined] src/inmanta/main.py:0: error: Returning Any from function declared to return "dict[str, Any] | None" [no-any-return] +src/inmanta/main.py:0: error: Argument 1 to "iter" has incompatible type "EntryPoints"; expected "SupportsIter[Generator[EntryPoint, None, None]]" [arg-type] +src/inmanta/main.py:0: note: Following member(s) of "EntryPoints" have conflicts: +src/inmanta/main.py:0: note: Expected: +src/inmanta/main.py:0: note: def __iter__(self) -> Generator[EntryPoint, None, None] +src/inmanta/main.py:0: note: Got: +src/inmanta/main.py:0: note: def __iter__(self) -> Iterator[Any] src/inmanta/main.py:0: error: Value of type "object" is not indexable [index] src/inmanta/main.py:0: error: Invalid index type "str" for "str"; expected type "SupportsIndex | slice" [index] src/inmanta/main.py:0: error: Value of type "dict[str, Any] | None" is not indexable [index] @@ -1163,8 +1157,8 @@ src/inmanta/server/services/orchestrationservice.py:0: error: Incompatible types src/inmanta/server/services/orchestrationservice.py:0: error: Incompatible types in assignment (expression has type "set[str | None]", variable has type "set[str]") [assignment] src/inmanta/server/services/orchestrationservice.py:0: error: Argument "agents" to "_trigger_auto_deploy" of "OrchestrationService" has incompatible type "Collection[str]"; expected "Sequence[str] | None" [arg-type] src/inmanta/server/services/orchestrationservice.py:0: error: Incompatible return value type (got "ReturnValueWithMeta[Sequence[DesiredStateVersion]]", expected "ReturnValue[list[DesiredStateVersion]]") [return-value] -src/inmanta/server/agentmanager.py:0: error: Argument 1 to "_ensure_scheduler" of "AutostartedAgentManager" has incompatible type "Environment | None"; expected "Environment" [arg-type] src/inmanta/server/agentmanager.py:0: error: Argument 1 to "ensure_agent_registered" of "AgentManager" has incompatible type "Environment | None"; expected "Environment" [arg-type] +src/inmanta/server/agentmanager.py:0: error: Argument 1 to "_ensure_scheduler" of "AutostartedAgentManager" has incompatible type "Environment | None"; expected "Environment" [arg-type] src/inmanta/server/services/compilerservice.py:0: error: Incompatible types in assignment (expression has type "bool | int | float | str | dict[str, str | int | bool]", variable has type "int") [assignment] src/inmanta/server/services/paramservice.py:0: error: Unsupported operand types for + ("None" and "timedelta") [operator] src/inmanta/server/services/paramservice.py:0: note: Left operand is of type "datetime | None" diff --git a/src/inmanta/module.py b/src/inmanta/module.py index b7d2e8e189..9fc119331e 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -114,7 +114,7 @@ def project_name(self) -> str: @property def name(self) -> str: - return self._requirement.name + return self._requirement.name.replace("-", "_") @property def key(self) -> str: From 1dce9865e6106fcdf8b5b7b817ab71f0f8730633 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Mon, 16 Sep 2024 15:35:46 +0200 Subject: [PATCH 37/74] refactor --- src/inmanta/env.py | 9 +++++---- src/inmanta/util/__init__.py | 16 +++++++++++++--- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 118cd9e18b..665890932b 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -213,8 +213,7 @@ def _are_installed_recursive( if r.marker and not r.marker.evaluate(environment=environment_marker_evaluation): # The marker of the requirement doesn't apply on this environment continue - # If no specifiers are provided, the `in` operation will return `False` - if r.name not in installed_packages or (len(r.specifier) > 0 and installed_packages[r.name] not in r.specifier): + if r.name not in installed_packages or not r.specifier.contains(installed_packages[r.name], prereleases=True): return False if r.extras: for extra in r.extras: @@ -1190,9 +1189,11 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: constraint_violations_strict: set[VersionConflict] = set() for c in all_constraints: requirement = c.requirement - # If no specifiers are provided, the `in` operation will return `False` if requirement.name not in installed_versions or ( - (len(requirement.specifier) > 0 and installed_versions[requirement.name] not in requirement.specifier) + ( + len(requirement.specifier) > 0 + and not requirement.specifier.contains(installed_versions[requirement.name], prereleases=True) + ) and (not requirement.marker or (requirement.marker and requirement.marker.evaluate())) ): version_conflict = VersionConflict( diff --git a/src/inmanta/util/__init__.py b/src/inmanta/util/__init__.py index fa1b5c87b1..75eae68a57 100644 --- a/src/inmanta/util/__init__.py +++ b/src/inmanta/util/__init__.py @@ -872,12 +872,18 @@ def _reset_counter(self) -> None: def remove_comment_part(to_clean: str) -> str: """ - Remove the comment part of a given string and ensure that the lenght of the string is greater than 0 + Remove the comment part of a given string and ensure that the length of the string is greater than 0 :param to_clean: The string to clean :return: A cleaned string """ - drop_comment, _, _ = to_clean.partition("#") + # Refer to PEP 508. A requirement could contain a hashtag + to_clean = to_clean.strip() + if to_clean.startswith("#"): + raise ValueError("The name of the requirement cannot be a comment!") + drop_comment, _, _ = to_clean.partition(" #") + # We make sure whitespaces are not counted in the length of this string, e.g. " #" + drop_comment = drop_comment.strip() if len(drop_comment) == 0: raise ValueError("The name of the requirement cannot be an empty string!") return drop_comment @@ -929,5 +935,9 @@ def parse_requirements_from_file(file_path: pathlib.Path) -> list[CanonicalRequi requirements = [] with open(file_path) as f: for line in f.readlines(): - requirements.append(parse_requirement(line)) + try: + requirements.append(parse_requirement(line)) + except ValueError: + # This line was empty or only containing a comment + continue return requirements From d2414333369649936968720f881e9bee99ef858e Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Mon, 16 Sep 2024 16:49:30 +0200 Subject: [PATCH 38/74] retake with plugins --- setup.py | 1 + src/inmanta/main.py | 10 ++++---- src/inmanta/util/__init__.py | 45 +++++++++++++++++++++++++++++++++++- 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 12754ed8ec..6ab4db1c88 100644 --- a/setup.py +++ b/setup.py @@ -95,6 +95,7 @@ "inmanta-cli = inmanta.main:main", "inmanta = inmanta.app:app", "inmanta-initial-user-setup = inmanta.user_setup:main", + "inmanta-cli_plugins = inmanta.main:cmd", ], }, ) diff --git a/src/inmanta/main.py b/src/inmanta/main.py index c52c6d6ed1..699152a904 100644 --- a/src/inmanta/main.py +++ b/src/inmanta/main.py @@ -28,15 +28,13 @@ import click import importlib_metadata import texttable -from click_plugins import with_plugins -from inmanta import protocol +from inmanta import protocol, util from inmanta.config import Config, cmdline_rest_transport from inmanta.const import AgentAction, AgentTriggerMethod, ResourceAction from inmanta.data.model import ResourceVersionIdStr from inmanta.resources import Id from inmanta.types import JsonType -from inmanta.util import parse_timestamp class Client: @@ -182,7 +180,7 @@ def get_table(header: list[str], rows: list[list[str]], data_type: Optional[list return table.draw() -@with_plugins(iter(importlib_metadata.entry_points(group="inmanta.cli_plugins"))) +@util.with_plugins(iter(importlib_metadata.entry_points(group="inmanta.cli_plugins"))) @click.group(help="Base command") @click.option("--host", help="The server hostname to connect to") @click.option("--port", help="The server port to connect to") @@ -668,7 +666,7 @@ def param(ctx: click.Context) -> None: def param_list(client: Client, environment: str) -> None: result = client.get_dict("list_params", arguments=dict(tid=client.to_environment_id(environment))) expire = int(result["expire"]) - now = parse_timestamp(result["now"]) + now = util.parse_timestamp(result["now"]) when = now - datetime.timedelta(0, expire) data = [] @@ -680,7 +678,7 @@ def param_list(client: Client, environment: str) -> None: p["name"], p["source"], p["updated"], - str(float(parse_timestamp(p["updated"]) < when)), + str(float(util.parse_timestamp(p["updated"]) < when)), ] ) diff --git a/src/inmanta/util/__init__.py b/src/inmanta/util/__init__.py index 75eae68a57..a17eb3d2f0 100644 --- a/src/inmanta/util/__init__.py +++ b/src/inmanta/util/__init__.py @@ -45,6 +45,9 @@ from typing import BinaryIO, Callable, Generic, Optional, Sequence, TypeVar, Union import asyncpg +import click +import importlib_metadata +from click import Group from tornado import gen import packaging @@ -904,7 +907,7 @@ def parse_requirement(requirement: str) -> CanonicalRequirement: drop_comment = remove_comment_part(to_clean=requirement) # We canonicalize the name of the requirement to be able to compare requirements and check if the requirement is # already installed - # This instance is considered as doomed because we are not supposed to modify directly the instance + # This instance is considered as doomed because the requirement name that this instance is using is not "canonicalized" doomed_requirement_instance = packaging.requirements.Requirement(requirement_string=drop_comment) canonical_name = str(doomed_requirement_instance).replace( doomed_requirement_instance.name, packaging.utils.canonicalize_name(doomed_requirement_instance.name) @@ -941,3 +944,43 @@ def parse_requirements_from_file(file_path: pathlib.Path) -> list[CanonicalRequi # This line was empty or only containing a comment continue return requirements + + +# Retaken from the `click-plugins` repo which is now unmaintained +def with_plugins(plugins: Iterator[importlib_metadata.EntryPoint]) -> Callable[[Group], Group]: + """ + A decorator to register external CLI commands to an instance of + `click.Group()`. + + Parameters + ---------- + plugins : iter + An iterable producing one `pkg_resources.EntryPoint()` per iteration. + attrs : **kwargs, optional + Additional keyword arguments for instantiating `click.Group()`. + + Returns + ------- + click.Group() + """ + + def decorator(group: click.Group) -> click.Group: + if not isinstance(group, click.Group): + raise TypeError("Plugins can only be attached to an instance of click.Group()") + + for entry_point in plugins or (): + try: + group.add_command(entry_point.load()) + except Exception as e: + # Catch this so a busted plugin doesn't take down the CLI. + # Handled by registering a dummy command that does nothing + # other than explain the error. + def print_error(error: Exception) -> None: + click.echo(f"Error: could not load this plugin for the following reason: {error}") + + new_print_error = functools.partial(print_error, e) + group.add_command(click.Command(name=entry_point.name, callback=new_print_error)) + + return group + + return decorator From 8a880935f79677b91e610f141f5eb4369ec09413 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Tue, 17 Sep 2024 11:40:04 +0200 Subject: [PATCH 39/74] rollback old changes --- setup.py | 4 +--- src/inmanta/util/__init__.py | 10 ++++------ 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 6ab4db1c88..86e8891cf4 100644 --- a/setup.py +++ b/setup.py @@ -18,8 +18,7 @@ "logfire~=0.46", "more-itertools>=8,<11", "opentelemetry-instrumentation-asyncpg~=0.46b0", - # leave upper bound floating for fast-moving and extremely stable packaging - "packaging>=21.3", + "packaging>=21.3,23", # pip>=21.3 required for editable pyproject.toml + setup.cfg based install support "pip>=21.3", "ply~=3.0", @@ -95,7 +94,6 @@ "inmanta-cli = inmanta.main:main", "inmanta = inmanta.app:app", "inmanta-initial-user-setup = inmanta.user_setup:main", - "inmanta-cli_plugins = inmanta.main:cmd", ], }, ) diff --git a/src/inmanta/util/__init__.py b/src/inmanta/util/__init__.py index a17eb3d2f0..a78b70039f 100644 --- a/src/inmanta/util/__init__.py +++ b/src/inmanta/util/__init__.py @@ -908,12 +908,10 @@ def parse_requirement(requirement: str) -> CanonicalRequirement: # We canonicalize the name of the requirement to be able to compare requirements and check if the requirement is # already installed # This instance is considered as doomed because the requirement name that this instance is using is not "canonicalized" - doomed_requirement_instance = packaging.requirements.Requirement(requirement_string=drop_comment) - canonical_name = str(doomed_requirement_instance).replace( - doomed_requirement_instance.name, packaging.utils.canonicalize_name(doomed_requirement_instance.name) - ) - requirement_instance = CanonicalRequirement(packaging.requirements.Requirement(requirement_string=canonical_name)) - return requirement_instance + requirement_instance = packaging.requirements.Requirement(requirement_string=drop_comment) + requirement_instance.name = packaging.utils.canonicalize_name(requirement_instance.name) + canonical_requirement_instance = CanonicalRequirement(requirement_instance) + return canonical_requirement_instance def parse_requirements(requirements: Sequence[str]) -> list[CanonicalRequirement]: From a046cf871a236f8bf8681238291d8e286df3b1cc Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Tue, 17 Sep 2024 11:48:29 +0200 Subject: [PATCH 40/74] add missing test --- src/inmanta/util/__init__.py | 4 ++++ tests/test_file_parser.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/src/inmanta/util/__init__.py b/src/inmanta/util/__init__.py index a78b70039f..de5d70783a 100644 --- a/src/inmanta/util/__init__.py +++ b/src/inmanta/util/__init__.py @@ -933,12 +933,16 @@ def parse_requirements_from_file(file_path: pathlib.Path) -> list[CanonicalRequi :param file_path: The path to the read the requirements from :return: Sequence[Requirement] """ + if not file_path.exists(): + raise RuntimeError(f"The provided path does not exist: `{file_path}`!") + requirements = [] with open(file_path) as f: for line in f.readlines(): try: requirements.append(parse_requirement(line)) except ValueError: + LOGGER.warning("This line was skipped because the requirement could not be parsed: %s", line) # This line was empty or only containing a comment continue return requirements diff --git a/tests/test_file_parser.py b/tests/test_file_parser.py index 57eefe440c..8d8970989a 100644 --- a/tests/test_file_parser.py +++ b/tests/test_file_parser.py @@ -17,8 +17,10 @@ """ import os +import pathlib import packaging.requirements +from inmanta import util from inmanta.file_parser import RequirementsTxtParser from inmanta.util import parse_requirements @@ -45,6 +47,9 @@ def test_requirements_txt_parser(tmpdir) -> None: requirements_as_str = RequirementsTxtParser.parse_requirements_as_strs(requirements_txt_file) assert requirements_as_str == expected_requirements + parsed_canonical_requirements = util.parse_requirements_from_file(pathlib.Path(requirements_txt_file)) + assert len(parsed_canonical_requirements) == len(expected_requirements) + new_content = RequirementsTxtParser.get_content_with_dep_removed(requirements_txt_file, remove_dep_on_pkg="test") expected_content = """ # A comment From 1896d2e8e18a1f89035e2ab8f75a734802d1830c Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Tue, 17 Sep 2024 12:02:26 +0200 Subject: [PATCH 41/74] missing < --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 86e8891cf4..80f251aef5 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ "logfire~=0.46", "more-itertools>=8,<11", "opentelemetry-instrumentation-asyncpg~=0.46b0", - "packaging>=21.3,23", + "packaging>=21.3,<23", # pip>=21.3 required for editable pyproject.toml + setup.cfg based install support "pip>=21.3", "ply~=3.0", From 22e4d1d6ba7c1842e2d50dc51320d2aa85a6a12b Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Tue, 17 Sep 2024 13:11:51 +0200 Subject: [PATCH 42/74] wrong number --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 80f251aef5..a9d8e6fbfa 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ "logfire~=0.46", "more-itertools>=8,<11", "opentelemetry-instrumentation-asyncpg~=0.46b0", - "packaging>=21.3,<23", + "packaging>=21.3,<25", # pip>=21.3 required for editable pyproject.toml + setup.cfg based install support "pip>=21.3", "ply~=3.0", From 0c3ab6eea2c6a4074141583b7fa7c09b09f244ec Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Tue, 17 Sep 2024 13:38:03 +0200 Subject: [PATCH 43/74] resolve mypy problems --- mypy-baseline.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/mypy-baseline.txt b/mypy-baseline.txt index 53a9832261..76f7a946e1 100644 --- a/mypy-baseline.txt +++ b/mypy-baseline.txt @@ -1048,12 +1048,6 @@ src/inmanta/model.py:0: error: Call to untyped function "ReferenceValue" in type src/inmanta/model.py:0: error: "Value" has no attribute "to_dict" [attr-defined] src/inmanta/model.py:0: error: "Value" has no attribute "to_dict" [attr-defined] src/inmanta/main.py:0: error: Returning Any from function declared to return "dict[str, Any] | None" [no-any-return] -src/inmanta/main.py:0: error: Argument 1 to "iter" has incompatible type "EntryPoints"; expected "SupportsIter[Generator[EntryPoint, None, None]]" [arg-type] -src/inmanta/main.py:0: note: Following member(s) of "EntryPoints" have conflicts: -src/inmanta/main.py:0: note: Expected: -src/inmanta/main.py:0: note: def __iter__(self) -> Generator[EntryPoint, None, None] -src/inmanta/main.py:0: note: Got: -src/inmanta/main.py:0: note: def __iter__(self) -> Iterator[Any] src/inmanta/main.py:0: error: Value of type "object" is not indexable [index] src/inmanta/main.py:0: error: Invalid index type "str" for "str"; expected type "SupportsIndex | slice" [index] src/inmanta/main.py:0: error: Value of type "dict[str, Any] | None" is not indexable [index] From 4f13c5df8b551e622e47ea10d413eef89223315c Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Tue, 17 Sep 2024 17:24:03 +0200 Subject: [PATCH 44/74] refactor --- src/inmanta/env.py | 8 ++--- src/inmanta/module.py | 2 +- src/inmanta/util/__init__.py | 31 ++++++++++-------- tests/compiler/test_basics.py | 11 +------ tests/test_file_parser.py | 61 +++++++++++++++++++++++++++++++++-- 5 files changed, 81 insertions(+), 32 deletions(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 665890932b..e1fd696b34 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -1190,10 +1190,7 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: for c in all_constraints: requirement = c.requirement if requirement.name not in installed_versions or ( - ( - len(requirement.specifier) > 0 - and not requirement.specifier.contains(installed_versions[requirement.name], prereleases=True) - ) + not requirement.specifier.contains(installed_versions[requirement.name], prereleases=True) and (not requirement.marker or (requirement.marker and requirement.marker.evaluate())) ): version_conflict = VersionConflict( @@ -1266,7 +1263,8 @@ def check_legacy(self, in_scope: Pattern[str], constraints: Optional[list[util.C constraint_violations: set[VersionConflict] = { VersionConflict(constraint, installed_versions.get(constraint.name, None)) for constraint in all_constraints - if constraint.name not in installed_versions or installed_versions[constraint.name] not in constraint.specifier + if constraint.name not in installed_versions + or not constraint.specifier.contains(installed_versions[constraint.name], prereleases=True) } all_violations = constraint_violations_non_strict | constraint_violations_strict | constraint_violations diff --git a/src/inmanta/module.py b/src/inmanta/module.py index 1c6f55b0d9..e0ee85562d 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -132,7 +132,7 @@ def __eq__(self, other: object) -> bool: return self._requirement == other._requirement def __contains__(self, version: packaging.version.Version | str) -> bool: - return version in self._requirement.specifier if len(self._requirement.specifier) > 0 else True + return self._requirement.specifier.contains(version, prereleases=True) def __str__(self) -> str: return str(self._requirement).replace("-", "_") diff --git a/src/inmanta/util/__init__.py b/src/inmanta/util/__init__.py index de5d70783a..06d87335b6 100644 --- a/src/inmanta/util/__init__.py +++ b/src/inmanta/util/__init__.py @@ -879,6 +879,7 @@ def remove_comment_part(to_clean: str) -> str: :param to_clean: The string to clean :return: A cleaned string + :raise: When the provided string to clean doesn't contain a requirement name """ # Refer to PEP 508. A requirement could contain a hashtag to_clean = to_clean.strip() @@ -899,9 +900,11 @@ def parse_requirement(requirement: str) -> CanonicalRequirement: """ To be able to compare requirements, we need to make sure that every requirement's name is canonicalized otherwise issues could arise when checking if packages are installed in a particular Venv. + This function supposes to receive an actual requirement. Commented strings will not be handled and result in a ValueError :param requirement: The requirement's name :return: A new requirement instance + :raise: ValueError when commented or empty strings are provided """ # Packaging Requirement is not able to parse requirements with comment. Therefore, we need to remove the `comment` part drop_comment = remove_comment_part(to_clean=requirement) @@ -922,7 +925,16 @@ def parse_requirements(requirements: Sequence[str]) -> list[CanonicalRequirement :param requirements: The names of the different requirements :return: Sequence[Requirement] """ - return [parse_requirement(requirement=e) for e in requirements] + canonical_requirements = [] + for e in requirements: + try: + canonical_requirements.append(parse_requirement(requirement=e)) + except ValueError: + LOGGER.debug("This line was skipped because the requirement could not be parsed: %s", e) + # This line was empty or only containing a comment + continue + + return canonical_requirements def parse_requirements_from_file(file_path: pathlib.Path) -> list[CanonicalRequirement]: @@ -942,7 +954,7 @@ def parse_requirements_from_file(file_path: pathlib.Path) -> list[CanonicalRequi try: requirements.append(parse_requirement(line)) except ValueError: - LOGGER.warning("This line was skipped because the requirement could not be parsed: %s", line) + LOGGER.debug("This line was skipped because the requirement could not be parsed: %s", line) # This line was empty or only containing a comment continue return requirements @@ -951,19 +963,10 @@ def parse_requirements_from_file(file_path: pathlib.Path) -> list[CanonicalRequi # Retaken from the `click-plugins` repo which is now unmaintained def with_plugins(plugins: Iterator[importlib_metadata.EntryPoint]) -> Callable[[Group], Group]: """ - A decorator to register external CLI commands to an instance of - `click.Group()`. - - Parameters - ---------- - plugins : iter - An iterable producing one `pkg_resources.EntryPoint()` per iteration. - attrs : **kwargs, optional - Additional keyword arguments for instantiating `click.Group()`. + A decorator to register external CLI commands to an instance of `click.Group()`. - Returns - ------- - click.Group() + :param plugins: An iterable producing one `pkg_resources.EntryPoint()` per iteration + :return: The provided click group with the new commands """ def decorator(group: click.Group) -> click.Group: diff --git a/tests/compiler/test_basics.py b/tests/compiler/test_basics.py index b2ab1611cb..f55073b486 100644 --- a/tests/compiler/test_basics.py +++ b/tests/compiler/test_basics.py @@ -24,7 +24,7 @@ import py import pytest -from inmanta import compiler, const, module, util +from inmanta import compiler, const, module from inmanta.ast import DoubleSetException, RuntimeException from inmanta.module import InstallMode from inmanta.plugins import PluginDeprecationWarning @@ -743,15 +743,6 @@ def test_implementation_import_missing_error(snippetcompiler) -> None: assert exception.value.location.start_char == 20 -@pytest.mark.parametrize("name", ["", "#", " # ", "#this is a comment"]) -def test_safe_requirement(name) -> None: - """ - Ensure that empty name requirements are not allowed in `Requirement` - """ - with pytest.raises(ValueError): - util.parse_requirement(requirement=name) - - @pytest.mark.slowtest def test_moduletool_failing( capsys, diff --git a/tests/test_file_parser.py b/tests/test_file_parser.py index 8d8970989a..ffbdc62848 100644 --- a/tests/test_file_parser.py +++ b/tests/test_file_parser.py @@ -19,6 +19,8 @@ import os import pathlib +import pytest + import packaging.requirements from inmanta import util from inmanta.file_parser import RequirementsTxtParser @@ -47,8 +49,13 @@ def test_requirements_txt_parser(tmpdir) -> None: requirements_as_str = RequirementsTxtParser.parse_requirements_as_strs(requirements_txt_file) assert requirements_as_str == expected_requirements - parsed_canonical_requirements = util.parse_requirements_from_file(pathlib.Path(requirements_txt_file)) - assert len(parsed_canonical_requirements) == len(expected_requirements) + parsed_canonical_requirements_from_file = util.parse_requirements_from_file(pathlib.Path(requirements_txt_file)) + assert parsed_canonical_requirements_from_file == requirements + + parsed_canonical_requirements = util.parse_requirements( + ["test==1.2.3", "# A comment", "other-dep~=2.0.0", "third-dep<5.0.0 # another comment", "splitteddep", "Capital"] + ) + assert parsed_canonical_requirements == requirements new_content = RequirementsTxtParser.get_content_with_dep_removed(requirements_txt_file, remove_dep_on_pkg="test") expected_content = """ @@ -92,3 +99,53 @@ def test_requirements_txt_parser(tmpdir) -> None: splitteddep """ ) + + +@pytest.mark.parametrize( + "iteration", + [ + ("", True), + ("#", True), + (" # ", True), + ("#this is a comment", True), + ("test==1.2.3", False), + ("other-dep~=2.0.0", False), + ], +) +def test_canonical_requirement(iteration) -> None: + """ + Ensure that empty name requirements are not allowed in `Requirement` + """ + name, should_fail = iteration + if should_fail: + with pytest.raises(ValueError): + util.parse_requirement(requirement=name) + else: + util.parse_requirement(requirement=name) + + +@pytest.mark.parametrize( + "iteration", + [ + ("", "", True), + ("#", "#", True), + (" # ", " # ", True), + ("#this is a comment", "#this is a comment", True), + ("test==1.2.3", "test==1.2.3", False), + ("other-dep~=2.0.0", "other-dep~=2.0.0", False), + ("test==1.2.3 # a command", "test==1.2.3", False), + ("other-dep #~=2.0.0", "other-dep", False), + ("other-dep#~=2.0.0", "other-dep#~=2.0.0", False), + ], +) +def test_drop_comment_part(iteration) -> None: + """ + Ensure that empty name requirements are not allowed in `Requirement` + """ + value, expected_value, should_fail = iteration + if should_fail: + with pytest.raises(ValueError): + util.remove_comment_part(value) + else: + current_value = util.remove_comment_part(value) + assert current_value == expected_value From 8e15960f6b7c69628dcc09064339dba5a9fc4609 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Tue, 17 Sep 2024 17:29:18 +0200 Subject: [PATCH 45/74] more cases --- tests/test_file_parser.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/test_file_parser.py b/tests/test_file_parser.py index ffbdc62848..6abfdf76a7 100644 --- a/tests/test_file_parser.py +++ b/tests/test_file_parser.py @@ -30,6 +30,7 @@ def test_requirements_txt_parser(tmpdir) -> None: content = """ test==1.2.3 + # A comment other-dep~=2.0.0 third-dep<5.0.0 # another comment @@ -53,12 +54,22 @@ def test_requirements_txt_parser(tmpdir) -> None: assert parsed_canonical_requirements_from_file == requirements parsed_canonical_requirements = util.parse_requirements( - ["test==1.2.3", "# A comment", "other-dep~=2.0.0", "third-dep<5.0.0 # another comment", "splitteddep", "Capital"] + [ + "test==1.2.3", + "# A comment", + " ", + "", + "other-dep~=2.0.0", + "third-dep<5.0.0 # another comment", + "splitteddep", + "Capital", + ] ) assert parsed_canonical_requirements == requirements new_content = RequirementsTxtParser.get_content_with_dep_removed(requirements_txt_file, remove_dep_on_pkg="test") expected_content = """ + # A comment other-dep~=2.0.0 third-dep<5.0.0 # another comment @@ -71,6 +82,7 @@ def test_requirements_txt_parser(tmpdir) -> None: new_content == """ test==1.2.3 + # A comment other-dep~=2.0.0 splitteddep @@ -82,6 +94,7 @@ def test_requirements_txt_parser(tmpdir) -> None: new_content == """ test==1.2.3 + # A comment other-dep~=2.0.0 third-dep<5.0.0 # another comment @@ -93,6 +106,7 @@ def test_requirements_txt_parser(tmpdir) -> None: new_content == """ test==1.2.3 + # A comment other-dep~=2.0.0 third-dep<5.0.0 # another comment From d8d03ac3a8a953a9d2f52cf630dd46e51d048101 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 18 Sep 2024 08:05:53 +0200 Subject: [PATCH 46/74] refactor --- src/inmanta/util/__init__.py | 11 +++++------ tests/test_file_parser.py | 28 ++++++++++++---------------- 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/src/inmanta/util/__init__.py b/src/inmanta/util/__init__.py index 06d87335b6..c8987fd3a2 100644 --- a/src/inmanta/util/__init__.py +++ b/src/inmanta/util/__init__.py @@ -875,21 +875,16 @@ def _reset_counter(self) -> None: def remove_comment_part(to_clean: str) -> str: """ - Remove the comment part of a given string and ensure that the length of the string is greater than 0 + Remove the comment part of a given string :param to_clean: The string to clean :return: A cleaned string - :raise: When the provided string to clean doesn't contain a requirement name """ # Refer to PEP 508. A requirement could contain a hashtag to_clean = to_clean.strip() - if to_clean.startswith("#"): - raise ValueError("The name of the requirement cannot be a comment!") drop_comment, _, _ = to_clean.partition(" #") # We make sure whitespaces are not counted in the length of this string, e.g. " #" drop_comment = drop_comment.strip() - if len(drop_comment) == 0: - raise ValueError("The name of the requirement cannot be an empty string!") return drop_comment @@ -908,6 +903,10 @@ def parse_requirement(requirement: str) -> CanonicalRequirement: """ # Packaging Requirement is not able to parse requirements with comment. Therefore, we need to remove the `comment` part drop_comment = remove_comment_part(to_clean=requirement) + + if drop_comment.startswith("#") or len(drop_comment) == 0: + raise ValueError(f"The requirement is invalid: Cannot be a comment or empty -> `{drop_comment}`!") + # We canonicalize the name of the requirement to be able to compare requirements and check if the requirement is # already installed # This instance is considered as doomed because the requirement name that this instance is using is not "canonicalized" diff --git a/tests/test_file_parser.py b/tests/test_file_parser.py index 6abfdf76a7..0f970b7163 100644 --- a/tests/test_file_parser.py +++ b/tests/test_file_parser.py @@ -141,25 +141,21 @@ def test_canonical_requirement(iteration) -> None: @pytest.mark.parametrize( "iteration", [ - ("", "", True), - ("#", "#", True), - (" # ", " # ", True), - ("#this is a comment", "#this is a comment", True), - ("test==1.2.3", "test==1.2.3", False), - ("other-dep~=2.0.0", "other-dep~=2.0.0", False), - ("test==1.2.3 # a command", "test==1.2.3", False), - ("other-dep #~=2.0.0", "other-dep", False), - ("other-dep#~=2.0.0", "other-dep#~=2.0.0", False), + ("", ""), + ("#", "#"), + (" # ", "#"), + ("#this is a comment", "#this is a comment"), + ("test==1.2.3", "test==1.2.3"), + ("other-dep~=2.0.0", "other-dep~=2.0.0"), + ("test==1.2.3 # a command", "test==1.2.3"), + ("other-dep #~=2.0.0", "other-dep"), + ("other-dep#~=2.0.0", "other-dep#~=2.0.0"), ], ) def test_drop_comment_part(iteration) -> None: """ Ensure that empty name requirements are not allowed in `Requirement` """ - value, expected_value, should_fail = iteration - if should_fail: - with pytest.raises(ValueError): - util.remove_comment_part(value) - else: - current_value = util.remove_comment_part(value) - assert current_value == expected_value + value, expected_value = iteration + current_value = util.remove_comment_part(value) + assert current_value == expected_value From cc46563b685b1935bcc3eb1ea992d042ba07e2c7 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 18 Sep 2024 08:14:23 +0200 Subject: [PATCH 47/74] refactor wrong type --- src/inmanta/env.py | 24 ++++++++++++------------ src/inmanta/file_parser.py | 4 ++-- src/inmanta/module.py | 4 ++-- tests/test_env.py | 7 +++---- tests/test_file_parser.py | 3 +-- tests/utils.py | 16 ++++++++-------- 6 files changed, 28 insertions(+), 30 deletions(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index e1fd696b34..643ba03c8d 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -78,7 +78,7 @@ class VersionConflict: :param owner: The package from which the constraint originates """ - requirement: packaging.requirements.Requirement + requirement: util.CanonicalRequirement installed_version: Optional[packaging.version.Version] = None owner: Optional[str] = None @@ -163,12 +163,12 @@ def get_advice(self) -> Optional[str]: ) -req_list = TypeVar("req_list", Sequence[str], Sequence[packaging.requirements.Requirement]) +req_list = TypeVar("req_list", Sequence[str], Sequence[util.CanonicalRequirement]) class PythonWorkingSet: @classmethod - def _get_as_requirements_type(cls, requirements: req_list) -> Sequence[packaging.requirements.Requirement]: + def _get_as_requirements_type(cls, requirements: req_list) -> Sequence[util.CanonicalRequirement]: """ Convert requirements from Union[Sequence[str], Sequence[Requirement]] to Sequence[Requirement] """ @@ -187,8 +187,8 @@ def are_installed(cls, requirements: req_list) -> bool: installed_packages: dict[str, packaging.version.Version] = cls.get_packages_in_working_set() def _are_installed_recursive( - reqs: Sequence[packaging.requirements.Requirement], - seen_requirements: Sequence[packaging.requirements.Requirement], + reqs: Sequence[util.CanonicalRequirement], + seen_requirements: Sequence[util.CanonicalRequirement], contained_in_extra: Optional[str] = None, ) -> bool: """ @@ -234,7 +234,7 @@ def _are_installed_recursive( return False return True - reqs_as_requirements: Sequence[packaging.requirements.Requirement] = cls._get_as_requirements_type(requirements) + reqs_as_requirements: Sequence[util.CanonicalRequirement] = cls._get_as_requirements_type(requirements) return _are_installed_recursive(reqs_as_requirements, seen_requirements=[]) @classmethod @@ -380,7 +380,7 @@ def run_pip_install_command_from_config( cls, python_path: str, config: PipConfig, - requirements: Optional[Sequence[packaging.requirements.Requirement]] = None, + requirements: Optional[Sequence[util.CanonicalRequirement]] = None, requirements_files: Optional[list[str]] = None, upgrade: bool = False, upgrade_strategy: PipUpgradeStrategy = PipUpgradeStrategy.ONLY_IF_NEEDED, @@ -421,7 +421,7 @@ async def async_run_pip_install_command_from_config( cls, python_path: str, config: PipConfig, - requirements: Optional[Sequence[packaging.requirements.Requirement]] = None, + requirements: Optional[Sequence[util.CanonicalRequirement]] = None, requirements_files: Optional[list[str]] = None, upgrade: bool = False, upgrade_strategy: PipUpgradeStrategy = PipUpgradeStrategy.ONLY_IF_NEEDED, @@ -461,7 +461,7 @@ def _prepare_command( cls, python_path: str, config: PipConfig, - requirements: Optional[Sequence[packaging.requirements.Requirement]] = None, + requirements: Optional[Sequence[util.CanonicalRequirement]] = None, requirements_files: Optional[list[str]] = None, upgrade: bool = False, upgrade_strategy: PipUpgradeStrategy = PipUpgradeStrategy.ONLY_IF_NEEDED, @@ -984,7 +984,7 @@ def get_protected_inmanta_packages(cls) -> list[str]: ] @classmethod - def _get_requirements_on_inmanta_package(cls) -> Sequence[packaging.requirements.Requirement]: + def _get_requirements_on_inmanta_package(cls) -> Sequence[util.CanonicalRequirement]: """ Returns the content of the requirement file that should be supplied to each `pip install` invocation to make sure that no Inmanta packages gets overridden. @@ -1148,7 +1148,7 @@ def get_constraint_violations_for_check( """ class OwnedRequirement(NamedTuple): - requirement: packaging.requirements.Requirement + requirement: util.CanonicalRequirement owner: Optional[str] = None def is_owned_by(self, owners: abc.Set[str]) -> bool: @@ -1252,7 +1252,7 @@ def check_legacy(self, in_scope: Pattern[str], constraints: Optional[list[util.C working_set: abc.Iterable[importlib.metadata.Distribution] = importlib.metadata.distributions() # add all requirements of all in scope packages installed in this environment - all_constraints: set[packaging.requirements.Requirement] = set(constraints if constraints is not None else []).union( + all_constraints: set[util.CanonicalRequirement] = set(constraints if constraints is not None else []).union( util.parse_requirement(requirement=requirement) for dist_info in working_set if in_scope.fullmatch(dist_info.name) diff --git a/src/inmanta/file_parser.py b/src/inmanta/file_parser.py index 2129fe8590..6ab2ed7cd0 100644 --- a/src/inmanta/file_parser.py +++ b/src/inmanta/file_parser.py @@ -19,8 +19,8 @@ import os import packaging.utils +from inmanta import util from inmanta.util import parse_requirement -from packaging.requirements import Requirement from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap @@ -57,7 +57,7 @@ class RequirementsTxtParser: """ @classmethod - def parse(cls, filename: str) -> list[Requirement]: + def parse(cls, filename: str) -> list[util.CanonicalRequirement]: """ Get all the requirements in `filename` as a list of `Requirement` instances. """ diff --git a/src/inmanta/module.py b/src/inmanta/module.py index e0ee85562d..bf29a32d44 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -1177,7 +1177,7 @@ def has_requirement_for(self, pkg_name: str) -> bool: """ return any(r.name == pkg_name.lower() for r in self._requirements) - def set_requirement_and_write(self, requirement: packaging.requirements.Requirement) -> None: + def set_requirement_and_write(self, requirement: util.CanonicalRequirement) -> None: """ Add the given requirement to the requirements.txt file and update the file on disk, replacing any existing constraints on this package. @@ -3423,7 +3423,7 @@ def add_module_requirement_persistent(self, requirement: InmantaModuleRequiremen # Parse config file config_parser = ConfigParser() config_parser.read(self.get_metadata_file_path()) - python_pkg_requirement: packaging.requirements.Requirement = requirement.get_python_package_requirement() + python_pkg_requirement: util.CanonicalRequirement = requirement.get_python_package_requirement() if config_parser.has_option("options", "install_requires"): new_install_requires = [ r diff --git a/tests/test_env.py b/tests/test_env.py index 9329cf64ac..7aba8674e9 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -33,8 +33,7 @@ import py import pytest -import packaging.requirements -from inmanta import env, loader, module +from inmanta import env, loader, module, util from inmanta.data.model import PipConfig from inmanta.env import Pip from inmanta.util import parse_requirement @@ -453,7 +452,7 @@ def test_active_env_get_module_file_editable_namespace_package( def create_install_package( - name: str, version: version.Version, requirements: list[packaging.requirements.Requirement], local_module_package_index: str + name: str, version: version.Version, requirements: list[util.CanonicalRequirement], local_module_package_index: str ) -> None: """ Creates and installs a simple package with specified requirements. Creates package in a temporary directory and @@ -564,7 +563,7 @@ def test_active_env_check_constraints(caplog, tmpvenv_active_inherit: str, local """ caplog.set_level(logging.WARNING) in_scope: Pattern[str] = re.compile("test-package-.*") - constraints: list[packaging.requirements.Requirement] = [parse_requirement(requirement="test-package-one~=1.0")] + constraints: list[util.CanonicalRequirement] = [parse_requirement(requirement="test-package-one~=1.0")] env.process_env.check(in_scope) diff --git a/tests/test_file_parser.py b/tests/test_file_parser.py index 0f970b7163..b52f7e2131 100644 --- a/tests/test_file_parser.py +++ b/tests/test_file_parser.py @@ -21,7 +21,6 @@ import pytest -import packaging.requirements from inmanta import util from inmanta.file_parser import RequirementsTxtParser from inmanta.util import parse_requirements @@ -45,7 +44,7 @@ def test_requirements_txt_parser(tmpdir) -> None: fd.write(content) expected_requirements = ["test==1.2.3", "other-dep~=2.0.0", "third-dep<5.0.0", "splitteddep", "Capital"] - requirements: list[packaging.requirements.Requirement] = RequirementsTxtParser().parse(requirements_txt_file) + requirements: list[util.CanonicalRequirement] = RequirementsTxtParser().parse(requirements_txt_file) assert requirements == parse_requirements(expected_requirements) requirements_as_str = RequirementsTxtParser.parse_requirements_as_strs(requirements_txt_file) assert requirements_as_str == expected_requirements diff --git a/tests/utils.py b/tests/utils.py index 30a6061235..11dccb6baf 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -491,11 +491,11 @@ def create_python_package( pkg_version: packaging.version.Version, path: str, *, - requirements: Optional[Sequence[packaging.requirements.Requirement]] = None, + requirements: Optional[Sequence[util.CanonicalRequirement]] = None, install: bool = False, editable: bool = False, publish_index: Optional[PipIndex] = None, - optional_dependencies: Optional[dict[str, Sequence[packaging.requirements.Requirement]]] = None, + optional_dependencies: Optional[dict[str, Sequence[util.CanonicalRequirement]]] = None, ) -> None: """ Creates an empty Python package. @@ -580,9 +580,9 @@ def module_from_template( *, new_version: Optional[packaging.version.Version] = None, new_name: Optional[str] = None, - new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, packaging.requirements.Requirement]]] = None, + new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, util.CanonicalRequirement]]] = None, new_extras: Optional[ - abc.Mapping[str, abc.Sequence[Union[module.InmantaModuleRequirement, packaging.requirements.Requirement]]] + abc.Mapping[str, abc.Sequence[Union[module.InmantaModuleRequirement, util.CanonicalRequirement]]] ] = None, install: bool = False, editable: bool = False, @@ -612,10 +612,10 @@ def module_from_template( """ def to_python_requires( - requires: abc.Sequence[Union[module.InmantaModuleRequirement, packaging.requirements.Requirement]] + requires: abc.Sequence[Union[module.InmantaModuleRequirement, util.CanonicalRequirement]] ) -> list[str]: return [ - str(req if isinstance(req, packaging.requirements.Requirement) else str(req.get_python_package_requirement())) + str(req if isinstance(req, util.CanonicalRequirement) else str(req.get_python_package_requirement())) for req in requires ] @@ -698,7 +698,7 @@ def v1_module_from_template( *, new_version: Optional[packaging.version.Version] = None, new_name: Optional[str] = None, - new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, packaging.requirements.Requirement]]] = None, + new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, util.CanonicalRequirement]]] = None, new_content_init_cf: Optional[str] = None, new_content_init_py: Optional[str] = None, ) -> module.ModuleV2Metadata: @@ -738,7 +738,7 @@ def v1_module_from_template( with open(os.path.join(dest_dir, "requirements.txt"), "w") as fd: fd.write( "\n".join( - str(req if isinstance(req, packaging.requirements.Requirement) else req.get_python_package_requirement()) + str(req if isinstance(req, util.CanonicalRequirement) else req.get_python_package_requirement()) for req in new_requirements ) ) From 5fd461a6f46e5be9dd9ece87a5edca096ab67533 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 18 Sep 2024 08:24:39 +0200 Subject: [PATCH 48/74] mypy-sync --- mypy-baseline.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy-baseline.txt b/mypy-baseline.txt index 76f7a946e1..a18e922e89 100644 --- a/mypy-baseline.txt +++ b/mypy-baseline.txt @@ -893,7 +893,7 @@ src/inmanta/execute/scheduler.py:0: error: Missing type parameters for generic t src/inmanta/execute/scheduler.py:0: error: "" has no attribute "attribute" [attr-defined] src/inmanta/execute/scheduler.py:0: error: "" has no attribute "attribute" [attr-defined] src/inmanta/execute/scheduler.py:0: error: "" has no attribute "attribute" [attr-defined] -src/inmanta/env.py:0: error: Argument 1 to "parse_requirements" has incompatible type "Sequence[Requirement]"; expected "Sequence[str]" [arg-type] +src/inmanta/env.py:0: error: Argument 1 to "parse_requirements" has incompatible type "Sequence[CanonicalRequirement]"; expected "Sequence[str]" [arg-type] src/inmanta/env.py:0: error: "type[WorkingSet]" has no attribute "_build_master" [attr-defined] src/inmanta/env.py:0: error: Incompatible return value type (got "tuple[str | None, Loader | None] | None", expected "tuple[str | None, Loader] | None") [return-value] src/inmanta/env.py:0: error: Item "None" of "ModuleSpec | None" has no attribute "submodule_search_locations" [union-attr] From 90371008e18871ad04ad67fcc035def7afb4714e Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 18 Sep 2024 08:54:21 +0200 Subject: [PATCH 49/74] refactor --- src/inmanta/util/__init__.py | 16 +++------------- tests/test_file_parser.py | 27 +++++++++++++++------------ 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/src/inmanta/util/__init__.py b/src/inmanta/util/__init__.py index c8987fd3a2..a64ffc2038 100644 --- a/src/inmanta/util/__init__.py +++ b/src/inmanta/util/__init__.py @@ -918,22 +918,12 @@ def parse_requirement(requirement: str) -> CanonicalRequirement: def parse_requirements(requirements: Sequence[str]) -> list[CanonicalRequirement]: """ - To be able to compare requirements, we need to make sure that every requirement's name is canonicalized otherwise issues - could arise when checking if packages are installed in a particular Venv. + This function supposes to receive actual requirements. Commented strings will not be handled and result in a ValueError :param requirements: The names of the different requirements - :return: Sequence[Requirement] + :return: list[CanonicalRequirement] """ - canonical_requirements = [] - for e in requirements: - try: - canonical_requirements.append(parse_requirement(requirement=e)) - except ValueError: - LOGGER.debug("This line was skipped because the requirement could not be parsed: %s", e) - # This line was empty or only containing a comment - continue - - return canonical_requirements + return [parse_requirement(requirement=e) for e in requirements] def parse_requirements_from_file(file_path: pathlib.Path) -> list[CanonicalRequirement]: diff --git a/tests/test_file_parser.py b/tests/test_file_parser.py index b52f7e2131..ee765377ce 100644 --- a/tests/test_file_parser.py +++ b/tests/test_file_parser.py @@ -52,20 +52,23 @@ def test_requirements_txt_parser(tmpdir) -> None: parsed_canonical_requirements_from_file = util.parse_requirements_from_file(pathlib.Path(requirements_txt_file)) assert parsed_canonical_requirements_from_file == requirements - parsed_canonical_requirements = util.parse_requirements( - [ - "test==1.2.3", - "# A comment", - " ", - "", - "other-dep~=2.0.0", - "third-dep<5.0.0 # another comment", - "splitteddep", - "Capital", - ] - ) + problematic_requirements = [ + "test==1.2.3", + "# A comment", + " ", + "", + "other-dep~=2.0.0", + "third-dep<5.0.0 # another comment", + "splitteddep", + "Capital", + ] + + parsed_canonical_requirements = util.parse_requirements(expected_requirements) assert parsed_canonical_requirements == requirements + with pytest.raises(ValueError): + util.parse_requirements(problematic_requirements) + new_content = RequirementsTxtParser.get_content_with_dep_removed(requirements_txt_file, remove_dep_on_pkg="test") expected_content = """ From 84270ae33189d17349679fec9472394810fcec6c Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 18 Sep 2024 11:34:43 +0200 Subject: [PATCH 50/74] fix broken tests --- tests/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index 11dccb6baf..8bb9d68188 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -615,7 +615,7 @@ def to_python_requires( requires: abc.Sequence[Union[module.InmantaModuleRequirement, util.CanonicalRequirement]] ) -> list[str]: return [ - str(req if isinstance(req, util.CanonicalRequirement) else str(req.get_python_package_requirement())) + str(req) if isinstance(req, packaging.requirements.Requirement) else str(req.get_python_package_requirement()) for req in requires ] @@ -738,7 +738,11 @@ def v1_module_from_template( with open(os.path.join(dest_dir, "requirements.txt"), "w") as fd: fd.write( "\n".join( - str(req if isinstance(req, util.CanonicalRequirement) else req.get_python_package_requirement()) + ( + str(req) + if isinstance(req, packaging.requirements.Requirement) + else str(req.get_python_package_requirement()) + ) for req in new_requirements ) ) From 6134daa21e527f9cd8008a93f680e726a3a4fe8d Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 18 Sep 2024 15:34:34 +0200 Subject: [PATCH 51/74] refactor --- Makefile | 2 +- setup.py | 3 +- src/inmanta/agent/executor.py | 3 +- src/inmanta/agent/in_process_executor.py | 4 +- src/inmanta/env.py | 18 +++--- src/inmanta/file_parser.py | 8 +-- src/inmanta/main.py | 2 +- src/inmanta/module.py | 24 ++++---- src/inmanta/moduletool.py | 3 +- src/inmanta/util/__init__.py | 58 ++++++++++-------- tests/conftest.py | 7 +-- tests/moduletool/test_add.py | 4 +- tests/moduletool/test_convert_v1_v2.py | 4 +- tests/moduletool/test_install.py | 59 +++++++++++------- tests/moduletool/test_update.py | 12 ++-- tests/test_env.py | 34 +++++------ tests/test_file_parser.py | 19 +++--- tests/test_module_loader.py | 78 +++++++++++++----------- 18 files changed, 186 insertions(+), 156 deletions(-) diff --git a/Makefile b/Makefile index aaa151d10f..03bd871fe5 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ black = black src tests tests_common .PHONY: install install: pip install -U setuptools pip uv -c requirements.txt - uv pip install -U -e . -c requirements.txt -r requirements.txt -r requirements.dev.txt + uv pip install -U -e . -c requirements.txt -r requirements.dev.txt .PHONY: install-tests install-tests: diff --git a/setup.py b/setup.py index a9d8e6fbfa..a95a24b0d6 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,8 @@ "logfire~=0.46", "more-itertools>=8,<11", "opentelemetry-instrumentation-asyncpg~=0.46b0", - "packaging>=21.3,<25", + # upper bound on packaging because we use a non-public API that might change in any (non-SemVer) version + "packaging>=21.3,<24.2", # pip>=21.3 required for editable pyproject.toml + setup.cfg based install support "pip>=21.3", "ply~=3.0", diff --git a/src/inmanta/agent/executor.py b/src/inmanta/agent/executor.py index 066009c89f..66ee2970a9 100644 --- a/src/inmanta/agent/executor.py +++ b/src/inmanta/agent/executor.py @@ -44,7 +44,6 @@ from inmanta.loader import ModuleSource from inmanta.resources import Id from inmanta.types import JsonType -from inmanta.util import parse_requirements LOGGER = logging.getLogger(__name__) @@ -296,7 +295,7 @@ async def create_and_install_environment(self, blueprint: EnvBlueprint) -> None: await asyncio.get_running_loop().run_in_executor(self.io_threadpool, self.init_env) if len(req): # install_for_config expects at least 1 requirement or a path to install await self.async_install_for_config( - requirements=parse_requirements(req), + requirements=inmanta.util.parse_requirements(req), config=blueprint.pip_config, ) diff --git a/src/inmanta/agent/in_process_executor.py b/src/inmanta/agent/in_process_executor.py index 30e734c807..ffb1a9d390 100644 --- a/src/inmanta/agent/in_process_executor.py +++ b/src/inmanta/agent/in_process_executor.py @@ -35,7 +35,7 @@ from inmanta.loader import CodeLoader from inmanta.resources import Id, Resource from inmanta.types import Apireturn -from inmanta.util import NamedLock, join_threadpools, parse_requirements +from inmanta.util import NamedLock, join_threadpools class InProcessExecutor(executor.Executor, executor.AgentInstance): @@ -601,7 +601,7 @@ async def _install(self, blueprint: executor.ExecutorBlueprint) -> None: await loop.run_in_executor( self.thread_pool, self._env.install_for_config, - parse_requirements(blueprint.requirements), + inmanta.util.parse_requirements(blueprint.requirements), blueprint.pip_config, ) await loop.run_in_executor(self.thread_pool, self._loader.deploy_version, blueprint.sources) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 643ba03c8d..4ff86c6ab5 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -43,6 +43,7 @@ import pkg_resources +import inmanta.util import packaging.requirements import packaging.utils import packaging.version @@ -173,7 +174,7 @@ def _get_as_requirements_type(cls, requirements: req_list) -> Sequence[util.Cano Convert requirements from Union[Sequence[str], Sequence[Requirement]] to Sequence[Requirement] """ if isinstance(requirements[0], str): - return util.parse_requirements(requirements) + return inmanta.util.parse_requirements(requirements) else: return requirements @@ -224,8 +225,8 @@ def _are_installed_recursive( return False pkgs_required_by_extra: set[util.CanonicalRequirement] = set( - [util.parse_requirement(str(e)) for e in distribution.requires(extras=(extra,))] - ) - set([util.parse_requirement(str(e)) for e in distribution.requires(extras=())]) + [inmanta.util.parse_requirement(str(e)) for e in distribution.requires(extras=(extra,))] + ) - set([inmanta.util.parse_requirement(str(e)) for e in distribution.requires(extras=())]) if not _are_installed_recursive( reqs=list(pkgs_required_by_extra), seen_requirements=list(seen_requirements) + list(reqs), @@ -240,7 +241,8 @@ def _are_installed_recursive( @classmethod def get_packages_in_working_set(cls, inmanta_modules_only: bool = False) -> dict[str, packaging.version.Version]: """ - Return all packages present in `pkg_resources.working_set` together with the version of the package. + Return all packages (under the canonicalized form) present in `pkg_resources.working_set` together with the version + of the package. :param inmanta_modules_only: Only return inmanta modules from the working set """ @@ -964,7 +966,7 @@ def install_from_list( use_pip_config was ignored on ISO6 and it still is """ self.install_from_index( - requirements=util.parse_requirements(requirements_list), + requirements=inmanta.util.parse_requirements(requirements_list), upgrade=upgrade, upgrade_strategy=upgrade_strategy, use_pip_config=True, @@ -992,7 +994,7 @@ def _get_requirements_on_inmanta_package(cls) -> Sequence[util.CanonicalRequirem protected_inmanta_packages: list[str] = cls.get_protected_inmanta_packages() workingset: dict[str, packaging.version.Version] = PythonWorkingSet.get_packages_in_working_set() return [ - util.parse_requirement(requirement=f"{pkg}=={workingset[pkg]}") + inmanta.util.parse_requirement(requirement=f"{pkg}=={workingset[pkg]}") for pkg in workingset if pkg in protected_inmanta_packages ] @@ -1156,7 +1158,7 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: # all requirements of all packages installed in this environment installed_constraints: abc.Set[OwnedRequirement] = frozenset( - OwnedRequirement(util.parse_requirement(requirement=str(requirement)), dist_info.key) + OwnedRequirement(inmanta.util.parse_requirement(requirement=str(requirement)), dist_info.key) for dist_info in pkg_resources.working_set for requirement in dist_info.requires() ) @@ -1253,7 +1255,7 @@ def check_legacy(self, in_scope: Pattern[str], constraints: Optional[list[util.C working_set: abc.Iterable[importlib.metadata.Distribution] = importlib.metadata.distributions() # add all requirements of all in scope packages installed in this environment all_constraints: set[util.CanonicalRequirement] = set(constraints if constraints is not None else []).union( - util.parse_requirement(requirement=requirement) + inmanta.util.parse_requirement(requirement=requirement) for dist_info in working_set if in_scope.fullmatch(dist_info.name) for requirement in dist_info.requires or [] diff --git a/src/inmanta/file_parser.py b/src/inmanta/file_parser.py index 6ab2ed7cd0..a7e5170702 100644 --- a/src/inmanta/file_parser.py +++ b/src/inmanta/file_parser.py @@ -18,9 +18,9 @@ import os +import inmanta.util import packaging.utils from inmanta import util -from inmanta.util import parse_requirement from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap @@ -61,7 +61,7 @@ def parse(cls, filename: str) -> list[util.CanonicalRequirement]: """ Get all the requirements in `filename` as a list of `Requirement` instances. """ - return [parse_requirement(requirement=r) for r in cls.parse_requirements_as_strs(filename)] + return [inmanta.util.parse_requirement(requirement=r) for r in cls.parse_requirements_as_strs(filename)] @classmethod def parse_requirements_as_strs(cls, filename: str) -> list[str]: @@ -95,14 +95,14 @@ def get_content_with_dep_removed(cls, filename: str, remove_dep_on_pkg: str) -> if line_continuation_buffer: line_continuation_buffer += line if not line.endswith("\\"): - if parse_requirement(requirement=line_continuation_buffer).name != removed_dependency: + if inmanta.util.parse_requirement(requirement=line_continuation_buffer).name != removed_dependency: result += line_continuation_buffer line_continuation_buffer = "" elif not line.strip() or line.strip().startswith("#"): result += line elif line.endswith("\\"): line_continuation_buffer = line - elif parse_requirement(requirement=line).name != removed_dependency: + elif inmanta.util.parse_requirement(requirement=line).name != removed_dependency: result += line else: # Dependency matches `remove_dep_on_pkg` => Remove line from result diff --git a/src/inmanta/main.py b/src/inmanta/main.py index 699152a904..ab90f86215 100644 --- a/src/inmanta/main.py +++ b/src/inmanta/main.py @@ -180,7 +180,7 @@ def get_table(header: list[str], rows: list[list[str]], data_type: Optional[list return table.draw() -@util.with_plugins(iter(importlib_metadata.entry_points(group="inmanta.cli_plugins"))) +@util.click_group_with_plugins(iter(importlib_metadata.entry_points(group="inmanta.cli_plugins"))) @click.group(help="Base command") @click.option("--host", help="The server hostname to connect to") @click.option("--port", help="The server port to connect to") diff --git a/src/inmanta/module.py b/src/inmanta/module.py index bf29a32d44..ad2b2eab9b 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -54,6 +54,7 @@ import inmanta.data.model import packaging.requirements +import packaging.utils import packaging.version from inmanta import RUNNING_TESTS, const, env, loader, plugins, util from inmanta.ast import CompilerException, LocatableString, Location, Namespace, Range, WrappingRuntimeException @@ -65,7 +66,7 @@ from inmanta.parser import plyInmantaParser from inmanta.parser.plyInmantaParser import cache_manager from inmanta.stable_api import stable_api -from inmanta.util import get_compiler_version, parse_requirement, parse_requirements +from inmanta.util import get_compiler_version from inmanta.warnings import InmantaWarning from packaging.specifiers import SpecifierSet from packaging.version import Version @@ -110,7 +111,7 @@ def project_name(self) -> str: # Requirement converts all "_" to "-". Inmanta modules use "_" warnings.warn(InmantaWarning("The `project_name` property has been deprecated in favor of `name`")) - return self._requirement.name.replace("-", "_") + return self.name @property def name(self) -> str: @@ -148,7 +149,7 @@ def parse(cls: type[TInmantaModuleRequirement], spec: str) -> TInmantaModuleRequ ) if "-" in spec: raise ValueError("Invalid Inmanta module requirement: Inmanta module names use '_', not '-'.") - return cls(parse_requirement(requirement=spec)) + return cls(inmanta.util.parse_requirement(requirement=spec)) def get_python_package_requirement(self) -> util.CanonicalRequirement: """ @@ -157,7 +158,7 @@ def get_python_package_requirement(self) -> util.CanonicalRequirement: module_name = self.name pkg_name = ModuleV2Source.get_package_name_for(module_name) pkg_req_str = str(self).replace(module_name, pkg_name, 1) # Replace max 1 occurrence - return parse_requirement(requirement=pkg_req_str) + return inmanta.util.parse_requirement(requirement=pkg_req_str) class CompilerExceptionWithExtendedTrace(CompilerException): @@ -727,7 +728,7 @@ def install(self, project: "Project", module_spec: list[InmantaModuleRequirement # These could be constraints (-c) as well, but that requires additional sanitation # Because for pip not every valid -r is a valid -c current_requires = project.get_strict_python_requirements_as_list() - requirements += parse_requirements(current_requires) + requirements += inmanta.util.parse_requirements(current_requires) if preinstalled is not None: # log warning if preinstalled version does not match constraints @@ -1175,7 +1176,8 @@ def has_requirement_for(self, pkg_name: str) -> bool: Returns True iff this requirements.txt file contains the given package name. The given `pkg_name` is matched case insensitive against the requirements in this RequirementsTxtFile. """ - return any(r.name == pkg_name.lower() for r in self._requirements) + canonicalized: str = packaging.utils.canonicalize_name(pkg_name) + return any(r.name == canonicalized for r in self._requirements) def set_requirement_and_write(self, requirement: util.CanonicalRequirement) -> None: """ @@ -2126,7 +2128,7 @@ def install_modules(self, *, bypass_module_cache: bool = False, update_dependenc self.verify_module_version_compatibility() # do python install - pyreq: list[util.CanonicalRequirement] = parse_requirements(self.collect_python_requirements()) + pyreq: list[util.CanonicalRequirement] = inmanta.util.parse_requirements(self.collect_python_requirements()) if len(pyreq) > 0: # upgrade both direct and transitive module dependencies: eager upgrade strategy @@ -2544,7 +2546,7 @@ def verify_python_requires(self) -> None: Verifies no incompatibilities exist within the Python environment with respect to installed module v2 requirements. """ if self.strict_deps_check: - constraints: list[util.CanonicalRequirement] = parse_requirements(self.collect_python_requirements()) + constraints: list[util.CanonicalRequirement] = inmanta.util.parse_requirements(self.collect_python_requirements()) env.process_env.check(strict_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*"), constraints=constraints) else: if not env.process_env.check_legacy(in_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*")): @@ -2675,7 +2677,7 @@ def requires(self) -> "List[InmantaModuleRequirement]": # filter on import stmt reqs = [] for spec in self._metadata.requires: - req = [parse_requirement(requirement=spec)] + req = [inmanta.util.parse_requirement(requirement=spec)] if len(req) > 1: print(f"Module file for {self._path} has bad line in requirements specification {spec}") reqe = InmantaModuleRequirement(req[0]) @@ -2812,7 +2814,7 @@ def requires(self) -> "List[InmantaModuleRequirement]": """ reqs = [] for spec in self.get_module_requirements(): - req = [parse_requirement(requirement=spec)] + req = [inmanta.util.parse_requirement(requirement=spec)] if len(req) > 1: print(f"Module file for {self._path} has bad line in requirements specification {spec}") reqe = InmantaModuleRequirement(req[0]) @@ -3428,7 +3430,7 @@ def add_module_requirement_persistent(self, requirement: InmantaModuleRequiremen new_install_requires = [ r for r in config_parser.get("options", "install_requires").split("\n") - if r and parse_requirement(requirement=r).name != python_pkg_requirement.name + if r and inmanta.util.parse_requirement(requirement=r).name != python_pkg_requirement.name ] new_install_requires.append(str(python_pkg_requirement)) else: diff --git a/src/inmanta/moduletool.py b/src/inmanta/moduletool.py index 8195b36ddc..a2129e40a9 100644 --- a/src/inmanta/moduletool.py +++ b/src/inmanta/moduletool.py @@ -76,7 +76,6 @@ gitprovider, ) from inmanta.stable_api import stable_api -from inmanta.util import parse_requirements from packaging.version import Version LOGGER = logging.getLogger(__name__) @@ -485,7 +484,7 @@ def do_update(specs: Mapping[str, Sequence[InmantaModuleRequirement]], modules: # Because for pip not every valid -r is a valid -c current_requires = my_project.get_strict_python_requirements_as_list() env.process_env.install_for_config( - v2_python_specs + parse_requirements(current_requires), + v2_python_specs + inmanta.util.parse_requirements(current_requires), my_project.metadata.pip, upgrade=True, ) diff --git a/src/inmanta/util/__init__.py b/src/inmanta/util/__init__.py index a64ffc2038..3d643408be 100644 --- a/src/inmanta/util/__init__.py +++ b/src/inmanta/util/__init__.py @@ -28,6 +28,7 @@ import logging import os import pathlib +import re import socket import threading import time @@ -37,7 +38,7 @@ from abc import ABC, abstractmethod from asyncio import CancelledError, Lock, Task, ensure_future, gather from collections import abc, defaultdict -from collections.abc import Coroutine, Iterator +from collections.abc import Coroutine, Iterable, Iterator from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from logging import Logger @@ -47,7 +48,6 @@ import asyncpg import click import importlib_metadata -from click import Group from tornado import gen import packaging @@ -66,6 +66,8 @@ T = TypeVar("T") S = TypeVar("S") +REQUIREMENT_SPECIFIER = re.compile(r"^\s*#*(\S+).*") + def get_compiler_version() -> str: return COMPILER_VERSION @@ -873,43 +875,48 @@ def _reset_counter(self) -> None: self._exhausted_pool_events_count = 0 -def remove_comment_part(to_clean: str) -> str: +def remove_comment_part_from_specifier(to_clean: str) -> str: """ - Remove the comment part of a given string + Remove the comment part of a requirement specifier - :param to_clean: The string to clean - :return: A cleaned string + :param to_clean: The requirement specifier to clean + :return: A cleaned requirement specifier """ # Refer to PEP 508. A requirement could contain a hashtag - to_clean = to_clean.strip() - drop_comment, _, _ = to_clean.partition(" #") - # We make sure whitespaces are not counted in the length of this string, e.g. " #" - drop_comment = drop_comment.strip() - return drop_comment + groups = REQUIREMENT_SPECIFIER.search(to_clean) + if not groups: + return "" + + return groups.group(1) +""" +A CanonicalRequirement is a packaging.requirements.Requirement except that the name of this Requirement is canonicalized, which +allows us to compare requirements without dealing afterwards with the format of these requirements. +""" CanonicalRequirement = typing.NewType("CanonicalRequirement", packaging.requirements.Requirement) def parse_requirement(requirement: str) -> CanonicalRequirement: """ - To be able to compare requirements, we need to make sure that every requirement's name is canonicalized otherwise issues - could arise when checking if packages are installed in a particular Venv. + Parse the given requirement string into a requirement object with a canonicalized name, meaning that we are sure that + every CanonicalRequirement will follow the same convention regarding the name. This will allow us compare requirements. This function supposes to receive an actual requirement. Commented strings will not be handled and result in a ValueError :param requirement: The requirement's name :return: A new requirement instance :raise: ValueError when commented or empty strings are provided """ - # Packaging Requirement is not able to parse requirements with comment. Therefore, we need to remove the `comment` part - drop_comment = remove_comment_part(to_clean=requirement) + # packaging.Requirement is not able to parse requirements with comment (if there is one). + # Therefore, we need to make sure that the provided requirement doesn't contain any `comment` part + drop_comment = remove_comment_part_from_specifier(to_clean=requirement) if drop_comment.startswith("#") or len(drop_comment) == 0: raise ValueError(f"The requirement is invalid: Cannot be a comment or empty -> `{drop_comment}`!") # We canonicalize the name of the requirement to be able to compare requirements and check if the requirement is # already installed - # This instance is considered as doomed because the requirement name that this instance is using is not "canonicalized" + # /!\ The following line could cause issue because we are not supposed to modify fields of an existing instance requirement_instance = packaging.requirements.Requirement(requirement_string=drop_comment) requirement_instance.name = packaging.utils.canonicalize_name(requirement_instance.name) canonical_requirement_instance = CanonicalRequirement(requirement_instance) @@ -918,7 +925,10 @@ def parse_requirement(requirement: str) -> CanonicalRequirement: def parse_requirements(requirements: Sequence[str]) -> list[CanonicalRequirement]: """ - This function supposes to receive actual requirements. Commented strings will not be handled and result in a ValueError + Parse the given requirements (sequence of strings) into requirement objects with a canonicalized name, meaning that we + are sure that every CanonicalRequirement will follow the same convention regarding the name. This will allow us compare + requirements. This function supposes to receive actual requirements. Commented strings will not be handled and result in + a ValueError :param requirements: The names of the different requirements :return: list[CanonicalRequirement] @@ -928,11 +938,12 @@ def parse_requirements(requirements: Sequence[str]) -> list[CanonicalRequirement def parse_requirements_from_file(file_path: pathlib.Path) -> list[CanonicalRequirement]: """ - To be able to compare requirements, we need to make sure that every requirement's name is canonicalized otherwise issues - could arise when checking if packages are installed in a particular Venv. + Parse the given requirements (line by line) from a file into requirement objects with a canonicalized name, meaning that we + are sure that every CanonicalRequirement will follow the same convention regarding the name. This will allow us compare + requirements. :param file_path: The path to the read the requirements from - :return: Sequence[Requirement] + :return: list[CanonicalRequirement] """ if not file_path.exists(): raise RuntimeError(f"The provided path does not exist: `{file_path}`!") @@ -950,7 +961,7 @@ def parse_requirements_from_file(file_path: pathlib.Path) -> list[CanonicalRequi # Retaken from the `click-plugins` repo which is now unmaintained -def with_plugins(plugins: Iterator[importlib_metadata.EntryPoint]) -> Callable[[Group], Group]: +def click_group_with_plugins(plugins: Iterable[importlib_metadata.EntryPoint]) -> Callable[[click.Group], click.Group]: """ A decorator to register external CLI commands to an instance of `click.Group()`. @@ -959,10 +970,7 @@ def with_plugins(plugins: Iterator[importlib_metadata.EntryPoint]) -> Callable[[ """ def decorator(group: click.Group) -> click.Group: - if not isinstance(group, click.Group): - raise TypeError("Plugins can only be attached to an instance of click.Group()") - - for entry_point in plugins or (): + for entry_point in plugins: try: group.add_command(entry_point.load()) except Exception as e: diff --git a/tests/conftest.py b/tests/conftest.py index 2934ff0def..1d2841ac28 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -133,7 +133,6 @@ from inmanta.server.services import orchestrationservice from inmanta.server.services.compilerservice import CompilerService, CompileRun from inmanta.types import JsonType -from inmanta.util import parse_requirement from inmanta.warnings import WarningsManager from libpip2pi.commands import dir2pi from packaging.version import Version @@ -1922,10 +1921,10 @@ def index_with_pkgs_containing_optional_deps() -> str: path=os.path.join(tmpdirname, "pkg"), publish_index=pip_index, optional_dependencies={ - "optional-a": [parse_requirement(requirement="dep-a")], + "optional-a": [inmanta.util.parse_requirement(requirement="dep-a")], "optional-b": [ - parse_requirement(requirement="dep-b"), - parse_requirement(requirement="dep-c"), + inmanta.util.parse_requirement(requirement="dep-b"), + inmanta.util.parse_requirement(requirement="dep-c"), ], }, ) diff --git a/tests/moduletool/test_add.py b/tests/moduletool/test_add.py index 7e72afc3da..2594bf15f1 100644 --- a/tests/moduletool/test_add.py +++ b/tests/moduletool/test_add.py @@ -24,11 +24,11 @@ import py import pytest +import inmanta.util from inmanta import env from inmanta.command import CLIException from inmanta.module import ModuleV1, ModuleV1Metadata, ModuleV2, ModuleV2Source, Project, ProjectMetadata from inmanta.moduletool import ModuleTool -from inmanta.util import parse_requirement from packaging.version import Version from utils import PipIndex, module_from_template @@ -89,7 +89,7 @@ def test_module_add_v2_module_to_project( dest_dir=os.path.join(tmpdir, f"elaboratev2module-v{version}"), new_version=Version(version), publish_index=pip_index, - new_extras={"optional": [parse_requirement(requirement="inmanta-module-minimalv2module")]}, + new_extras={"optional": [inmanta.util.parse_requirement(requirement="inmanta-module-minimalv2module")]}, ) # Create project diff --git a/tests/moduletool/test_convert_v1_v2.py b/tests/moduletool/test_convert_v1_v2.py index 0c42fb60a8..de5526686a 100644 --- a/tests/moduletool/test_convert_v1_v2.py +++ b/tests/moduletool/test_convert_v1_v2.py @@ -27,12 +27,12 @@ import pytest from pytest import MonkeyPatch +import inmanta.util import toml from inmanta import moduletool from inmanta.command import CLIException from inmanta.module import DummyProject, ModuleV1, ModuleV2, ModuleV2Metadata from inmanta.moduletool import ModuleConverter, ModuleVersionException -from inmanta.util import parse_requirements from packaging import version from utils import log_contains, v1_module_from_template @@ -114,7 +114,7 @@ def test_issue_3159_conversion_std_module_add_self_to_dependencies(tmpdir): parser = configparser.ConfigParser() parser.read(setup_cfg_file) assert parser.has_option("options", "install_requires") - install_requires = parse_requirements(parser.get("options", "install_requires").split("\n")) + install_requires = inmanta.util.parse_requirements(parser.get("options", "install_requires").split("\n")) pkg_names = [r.name for r in install_requires] assert "inmanta-module-std" not in pkg_names diff --git a/tests/moduletool/test_install.py b/tests/moduletool/test_install.py index d6d4f4540b..66d5b5041f 100644 --- a/tests/moduletool/test_install.py +++ b/tests/moduletool/test_install.py @@ -31,6 +31,7 @@ import pytest import yaml +import inmanta.util from inmanta import compiler, const, env, loader, module from inmanta.ast import CompilerException from inmanta.command import CLIException @@ -38,7 +39,6 @@ from inmanta.env import CommandRunner, ConflictingRequirements, PipConfig from inmanta.module import InmantaModuleRequirement, InstallMode, ModuleLoadingException, ModuleNotFoundException from inmanta.moduletool import DummyProject, ModuleConverter, ModuleTool, ProjectTool -from inmanta.util import parse_requirement from moduletool.common import BadModProvider, install_project from packaging import version from utils import LogSequence, PipIndex, log_contains, module_from_template @@ -252,14 +252,14 @@ def test_module_install_conflicting_requirements(tmpdir: py.path.local, snippetc os.path.join(modules_v2_dir, "minimalv2module"), os.path.join(str(tmpdir), "modone"), new_name="modone", - new_requirements=[parse_requirement(requirement="lorem~=0.0.1")], + new_requirements=[inmanta.util.parse_requirement(requirement="lorem~=0.0.1")], install=True, ) module_from_template( os.path.join(modules_v2_dir, "minimalv2module"), os.path.join(str(tmpdir), "modtwo"), new_name="modtwo", - new_requirements=[parse_requirement(requirement="lorem~=0.1.0")], + new_requirements=[inmanta.util.parse_requirement(requirement="lorem~=0.1.0")], install=True, ) @@ -509,7 +509,10 @@ def test_project_install( index_url=local_module_package_index, # We add tornado, as there is a code path in update for the case where the project has python requires python_requires=["tornado"] - + [parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names], + + [ + inmanta.util.parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) + for mod in install_module_names + ], install_project=False, ) @@ -542,7 +545,8 @@ def test_project_install( autostd=False, python_package_sources=[local_module_package_index], python_requires=[ - parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in install_module_names + inmanta.util.parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) + for mod in install_module_names ] + ["lorem"], install_project=False, @@ -678,7 +682,7 @@ def test_project_install_modules_cache_invalid( index_url=index.url, extra_index_url=[local_module_package_index], # make sure main module gets installed, pulling in newest version of dependency module - python_requires=[parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(main_module))], + python_requires=[inmanta.util.parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(main_module))], ) # populate project.modules[dependency_module] to force the error conditions in this simplified example @@ -763,7 +767,7 @@ def test_project_install_incompatible_versions( install_project=False, add_to_module_path=[v1_modules_path], index_url=index.url, - python_requires=[parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(v2_mod_name))], + python_requires=[inmanta.util.parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(v2_mod_name))], ) # install project @@ -816,14 +820,14 @@ def test_project_install_incompatible_dependencies( v2_template_path, os.path.join(str(tmpdir), "v2mod2"), new_name="v2mod2", - new_requirements=[parse_requirement(requirement="inmanta-module-v2mod1~=1.0.0")], + new_requirements=[inmanta.util.parse_requirement(requirement="inmanta-module-v2mod1~=1.0.0")], publish_index=index, ) v2mod3: module.ModuleV2Metadata = module_from_template( v2_template_path, os.path.join(str(tmpdir), "v2mod3"), new_name="v2mod3", - new_requirements=[parse_requirement(requirement="inmanta-module-v2mod1~=2.0.0")], + new_requirements=[inmanta.util.parse_requirement(requirement="inmanta-module-v2mod1~=2.0.0")], publish_index=index, ) @@ -837,7 +841,7 @@ def test_project_install_incompatible_dependencies( install_project=False, index_url=index.url, python_requires=[ - parse_requirement( + inmanta.util.parse_requirement( requirement=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod2, v2mod3] @@ -921,7 +925,7 @@ def test_install_from_index_dont_leak_pip_index( # Installing a V2 module requires a python package source. index_url="unknown", python_requires=[ - parse_requirement( + inmanta.util.parse_requirement( requirement=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod1] @@ -983,7 +987,7 @@ def test_install_with_use_config( index_url=index.url if not use_pip_config else None, use_pip_config_file=use_pip_config, python_requires=[ - parse_requirement( + inmanta.util.parse_requirement( requirement=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod1] @@ -1052,7 +1056,7 @@ def test_install_with_use_config_extra_index( extra_index_url=[index2.url], use_pip_config_file=True, python_requires=[ - parse_requirement( + inmanta.util.parse_requirement( requirement=module.ModuleV2Source.get_package_name_for(module.ModuleV2.get_name_from_metadata(metadata)) ) for metadata in [v2mod1, v2mod2] @@ -1088,7 +1092,7 @@ def test_install_with_use_config_but_PIP_CONFIG_FILE_not_set( autostd=False, install_project=False, use_pip_config_file=True, - python_requires=[parse_requirement(requirement="inmanta-module-dummy-module")], + python_requires=[inmanta.util.parse_requirement(requirement="inmanta-module-dummy-module")], ) # install project @@ -1205,7 +1209,7 @@ def test_install_project_with_install_mode_master(tmpdir: py.path.local, snippet autostd=False, install_project=False, add_to_module_path=[str(tmpdir)], - project_requires=[InmantaModuleRequirement(parse_requirement(requirement="mod11==3.2.1"))], + project_requires=[InmantaModuleRequirement(inmanta.util.parse_requirement(requirement="mod11==3.2.1"))], install_mode=InstallMode.master, ) @@ -1233,7 +1237,7 @@ def test_module_install_logging(local_module_package_index: str, snippetcompiler v2_module = "minimalv2module" - v2_requirements = [parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(v2_module))] + v2_requirements = [inmanta.util.parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(v2_module))] # set up project and modules project: module.Project = snippetcompiler_clean.setup_for_snippet( @@ -1334,7 +1338,9 @@ def test_pip_output(local_module_package_index: str, snippetcompiler_clean, capl ) modules = ["modone", "modtwo"] - v2_requirements = [parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in modules] + v2_requirements = [ + inmanta.util.parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in modules + ] snippetcompiler_clean.setup_for_snippet( f""" @@ -1420,7 +1426,9 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[ + inmanta.util.parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module")) + ], install_project=True, ) log_contains( @@ -1452,7 +1460,9 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[ + inmanta.util.parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module")) + ], install_project=True, ) @@ -1485,7 +1495,9 @@ def test_no_matching_distribution(local_module_package_index: str, snippetcompil autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module"))], + python_requires=[ + inmanta.util.parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("parent_module")) + ], install_project=True, ) log_contains( @@ -1554,7 +1566,7 @@ def test_version_snapshot(local_module_package_index: str, snippetcompiler_clean autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("module_b"))], + python_requires=[inmanta.util.parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("module_b"))], install_project=True, ) @@ -1580,7 +1592,7 @@ def test_version_snapshot(local_module_package_index: str, snippetcompiler_clean autostd=False, index_url=local_module_package_index, extra_index_url=[index.url], - python_requires=[parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("module_c"))], + python_requires=[inmanta.util.parse_requirement(requirement=module.ModuleV2Source.get_package_name_for("module_c"))], install_project=True, ) @@ -1651,7 +1663,8 @@ def test_constraints_logging_v2(modules_v2_dir, tmpdir, caplog, snippetcompiler_ index_url=local_module_package_index, extra_index_url=[index.url], python_requires=[ - parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) for mod in ["module_b", "module_a"] + inmanta.util.parse_requirement(requirement=module.ModuleV2Source.get_package_name_for(mod)) + for mod in ["module_b", "module_a"] ], install_project=True, project_requires=[ diff --git a/tests/moduletool/test_update.py b/tests/moduletool/test_update.py index 6daef61242..1c9f9bc9e8 100644 --- a/tests/moduletool/test_update.py +++ b/tests/moduletool/test_update.py @@ -21,6 +21,7 @@ import py.path import pytest +import inmanta.util from inmanta import env from inmanta.config import Config from inmanta.data.model import PipConfig @@ -28,7 +29,6 @@ from inmanta.module import InmantaModuleRequirement, InstallMode, ModuleV1, ModuleV2Source from inmanta.moduletool import ProjectTool from inmanta.parser import ParserException -from inmanta.util import parse_requirement, parse_requirements from moduletool.common import add_file, clone_repo from packaging.version import Version from utils import PipIndex, create_python_package, module_from_template, v1_module_from_template @@ -127,7 +127,7 @@ def assert_version_installed(module_name: str, version: str) -> None: new_version=Version(current_version), new_name=module_name, new_requirements=( - [InmantaModuleRequirement(parse_requirement(requirement="module2<3.0.0"))] + [InmantaModuleRequirement(inmanta.util.parse_requirement(requirement="module2<3.0.0"))] if module_name == "module1" else None ), @@ -144,7 +144,7 @@ def assert_version_installed(module_name: str, version: str) -> None: # Add a dependency on module2, without setting an explicit version constraint. Later version of module1 # do set a version constraint on the dependency on module2. This way it is verified whether the module update # command takes into account the version constraints set in a new version of a module. - new_requirements=[InmantaModuleRequirement(parse_requirement(requirement="module2"))], + new_requirements=[InmantaModuleRequirement(inmanta.util.parse_requirement(requirement="module2"))], install=False, publish_index=pip_index, new_content_init_cf="entity" if corrupt_module else None, # Introduce syntax error in the module @@ -244,7 +244,7 @@ def test_module_update_dependencies( "b", Version(v), str(tmpdir.join(f"b-{v}")), - requirements=[parse_requirement(requirement="c")], + requirements=[inmanta.util.parse_requirement(requirement="c")], publish_index=index, ) for v in ("1.0.0", "2.0.0"): @@ -260,7 +260,7 @@ def test_module_update_dependencies( # install b-1.0.0 and c-1.0.0 env.process_env.install_for_config( - [parse_requirement(requirement=req) for req in ("b==1.0.0", "c==1.0.0")], + [inmanta.util.parse_requirement(requirement=req) for req in ("b==1.0.0", "c==1.0.0")], config=PipConfig( index_url=index.url, use_system_config=False, @@ -272,7 +272,7 @@ def test_module_update_dependencies( source_dir=os.path.join(modules_dir, "minimalv1module"), dest_dir=str(tmpdir.join("modules", "my_mod")), new_name="my_mod", - new_requirements=parse_requirements(["a", "b~=1.0.0"]), + new_requirements=inmanta.util.parse_requirements(["a", "b~=1.0.0"]), ) # run `inmanta project update` without running install first diff --git a/tests/test_env.py b/tests/test_env.py index 7aba8674e9..f72ac1b003 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -33,10 +33,10 @@ import py import pytest +import inmanta.util from inmanta import env, loader, module, util from inmanta.data.model import PipConfig from inmanta.env import Pip -from inmanta.util import parse_requirement from packaging import version from utils import LogSequence, PipIndex, create_python_package @@ -177,7 +177,7 @@ def test_gen_req_file(): # make sure they all parse for req in reqs: - parse_requirement(requirement=req) + inmanta.util.parse_requirement(requirement=req) def test_environment_python_version_multi_digit(tmpdir: py.path.local) -> None: @@ -206,7 +206,7 @@ def test_process_env_install_from_index( package_name: str = "more-itertools" assert package_name not in env.process_env.get_installed_packages() env.process_env.install_for_config( - [parse_requirement(requirement=package_name + (f"=={version}" if version is not None else ""))], + [inmanta.util.parse_requirement(requirement=package_name + (f"=={version}" if version is not None else ""))], config=PipConfig( use_system_config=True, # we need an upstream for some packages ), @@ -221,7 +221,7 @@ def test_process_env_install_from_index( # It should hit the cache there and return here. # Cheap and fast test env.process_env.install_from_index( - [parse_requirement(requirement=package_name + (f"=={version}" if version is not None else ""))], + [inmanta.util.parse_requirement(requirement=package_name + (f"=={version}" if version is not None else ""))], use_pip_config=True, ) @@ -271,7 +271,7 @@ def test_process_env_install_from_index_not_found_env_var( with pytest.raises(env.PackageNotFound, match=re.escape(expected)): env.process_env.install_for_config( - [parse_requirement(requirement="this-package-does-not-exist")], + [inmanta.util.parse_requirement(requirement="this-package-does-not-exist")], config=PipConfig( index_url=index_urls[0], # The first element should only be passed to the index_url. If there are indexes in the environment @@ -308,7 +308,7 @@ def test_process_env_install_no_index(tmpdir: py.path.local, monkeypatch, use_sy with pytest.raises(env.PackageNotFound, match=re.escape(expected)): env.process_env.install_for_config( - requirements=[parse_requirement(requirement="this-package-does-not-exist")], + requirements=[inmanta.util.parse_requirement(requirement="this-package-does-not-exist")], paths=[env.LocalPackagePath(path=str(tmpdir))], config=PipConfig(use_system_config=use_system_config), ) @@ -325,7 +325,7 @@ def test_process_env_install_from_index_conflicting_reqs( package_name: str = "more-itertools" with pytest.raises(env.ConflictingRequirements) as e: env.process_env.install_for_config( - [parse_requirement(requirement=f"{package_name}{version}") for version in [">8.5", "<=8"]], + [inmanta.util.parse_requirement(requirement=f"{package_name}{version}") for version in [">8.5", "<=8"]], config=PipConfig( use_system_config=True, # we need an upstream for some packages ), @@ -395,7 +395,7 @@ def test_active_env_get_module_file( loader.PluginModuleFinder.configure_module_finder([os.path.join(str(tmpdir), "libs")]) assert env.ActiveEnv.get_module_file(module_name) is None - env.process_env.install_for_config([parse_requirement(requirement=package_name)], pip_config) + env.process_env.install_for_config([inmanta.util.parse_requirement(requirement=package_name)], pip_config) assert package_name in env.process_env.get_installed_packages() module_info: Optional[tuple[Optional[str], Loader]] = env.ActiveEnv.get_module_file(module_name) assert module_info is not None @@ -542,7 +542,7 @@ def assert_all_checks(expect_test: tuple[bool, str] = (True, ""), expect_nonext: create_install_package( "test-package-two", version.Version("1.0.0"), - [parse_requirement(requirement="test-package-one~=1.0")], + [inmanta.util.parse_requirement(requirement="test-package-one~=1.0")], local_module_package_index, ) assert_all_checks() @@ -563,7 +563,7 @@ def test_active_env_check_constraints(caplog, tmpvenv_active_inherit: str, local """ caplog.set_level(logging.WARNING) in_scope: Pattern[str] = re.compile("test-package-.*") - constraints: list[util.CanonicalRequirement] = [parse_requirement(requirement="test-package-one~=1.0")] + constraints: list[util.CanonicalRequirement] = [inmanta.util.parse_requirement(requirement="test-package-one~=1.0")] env.process_env.check(in_scope) @@ -582,7 +582,7 @@ def test_active_env_check_constraints(caplog, tmpvenv_active_inherit: str, local create_install_package( "ext-package-one", version.Version("1.0.0"), - [parse_requirement(requirement="test-package-one==1.0")], + [inmanta.util.parse_requirement(requirement="test-package-one==1.0")], local_module_package_index, ) env.process_env.check(in_scope, constraints) @@ -613,7 +613,7 @@ def test_override_inmanta_package(tmpvenv_active_inherit: env.VirtualEnv) -> Non installed_pkgs = tmpvenv_active_inherit.get_installed_packages() assert "inmanta-core" in installed_pkgs, "The inmanta-core package should be installed to run the tests" - inmanta_requirements = parse_requirement(requirement="inmanta-core==4.0.0") + inmanta_requirements = inmanta.util.parse_requirement(requirement="inmanta-core==4.0.0") with pytest.raises(env.ConflictingRequirements) as excinfo: tmpvenv_active_inherit.install_for_config( requirements=[inmanta_requirements], @@ -653,13 +653,13 @@ def test_cache_on_active_env(tmpvenv_active_inherit: env.ActiveEnv, local_module """ def _assert_install(requirement: str, installed: bool) -> None: - parsed_requirement = parse_requirement(requirement=requirement) + parsed_requirement = inmanta.util.parse_requirement(requirement=requirement) for r in [requirement, parsed_requirement]: assert tmpvenv_active_inherit.are_installed(requirements=[r]) == installed _assert_install("inmanta-module-elaboratev2module==1.2.3", installed=False) tmpvenv_active_inherit.install_for_config( - requirements=[parse_requirement(requirement="inmanta-module-elaboratev2module==1.2.3")], + requirements=[inmanta.util.parse_requirement(requirement="inmanta-module-elaboratev2module==1.2.3")], config=PipConfig( index_url=local_module_package_index, ), @@ -703,7 +703,7 @@ def test_are_installed_dependency_cycle_on_extra(tmpdir, tmpvenv_active_inherit: path=os.path.join(tmpdir, "pkg"), publish_index=pip_index, optional_dependencies={ - "optional-pkg": [parse_requirement(requirement="dep[optional-dep]")], + "optional-pkg": [inmanta.util.parse_requirement(requirement="dep[optional-dep]")], }, ) create_python_package( @@ -712,11 +712,11 @@ def test_are_installed_dependency_cycle_on_extra(tmpdir, tmpvenv_active_inherit: path=os.path.join(tmpdir, "dep"), publish_index=pip_index, optional_dependencies={ - "optional-dep": [parse_requirement(requirement="pkg[optional-pkg]")], + "optional-dep": [inmanta.util.parse_requirement(requirement="pkg[optional-pkg]")], }, ) - requirements = [parse_requirement(requirement="pkg[optional-pkg]")] + requirements = [inmanta.util.parse_requirement(requirement="pkg[optional-pkg]")] tmpvenv_active_inherit.install_for_config( requirements=requirements, config=PipConfig( diff --git a/tests/test_file_parser.py b/tests/test_file_parser.py index ee765377ce..67c8c7458b 100644 --- a/tests/test_file_parser.py +++ b/tests/test_file_parser.py @@ -21,9 +21,8 @@ import pytest -from inmanta import util +import inmanta.util from inmanta.file_parser import RequirementsTxtParser -from inmanta.util import parse_requirements def test_requirements_txt_parser(tmpdir) -> None: @@ -44,12 +43,12 @@ def test_requirements_txt_parser(tmpdir) -> None: fd.write(content) expected_requirements = ["test==1.2.3", "other-dep~=2.0.0", "third-dep<5.0.0", "splitteddep", "Capital"] - requirements: list[util.CanonicalRequirement] = RequirementsTxtParser().parse(requirements_txt_file) - assert requirements == parse_requirements(expected_requirements) + requirements: list[inmanta.util.CanonicalRequirement] = RequirementsTxtParser().parse(requirements_txt_file) + assert requirements == inmanta.util.parse_requirements(expected_requirements) requirements_as_str = RequirementsTxtParser.parse_requirements_as_strs(requirements_txt_file) assert requirements_as_str == expected_requirements - parsed_canonical_requirements_from_file = util.parse_requirements_from_file(pathlib.Path(requirements_txt_file)) + parsed_canonical_requirements_from_file = inmanta.util.parse_requirements_from_file(pathlib.Path(requirements_txt_file)) assert parsed_canonical_requirements_from_file == requirements problematic_requirements = [ @@ -63,11 +62,11 @@ def test_requirements_txt_parser(tmpdir) -> None: "Capital", ] - parsed_canonical_requirements = util.parse_requirements(expected_requirements) + parsed_canonical_requirements = inmanta.util.parse_requirements(expected_requirements) assert parsed_canonical_requirements == requirements with pytest.raises(ValueError): - util.parse_requirements(problematic_requirements) + inmanta.util.parse_requirements(problematic_requirements) new_content = RequirementsTxtParser.get_content_with_dep_removed(requirements_txt_file, remove_dep_on_pkg="test") expected_content = """ @@ -135,9 +134,9 @@ def test_canonical_requirement(iteration) -> None: name, should_fail = iteration if should_fail: with pytest.raises(ValueError): - util.parse_requirement(requirement=name) + inmanta.util.parse_requirement(requirement=name) else: - util.parse_requirement(requirement=name) + inmanta.util.parse_requirement(requirement=name) @pytest.mark.parametrize( @@ -159,5 +158,5 @@ def test_drop_comment_part(iteration) -> None: Ensure that empty name requirements are not allowed in `Requirement` """ value, expected_value = iteration - current_value = util.remove_comment_part(value) + current_value = inmanta.util.remove_comment_part_from_specifier(value) assert current_value == expected_value diff --git a/tests/test_module_loader.py b/tests/test_module_loader.py index 2ccc38a755..6cbfecfb59 100644 --- a/tests/test_module_loader.py +++ b/tests/test_module_loader.py @@ -29,6 +29,7 @@ import py import pytest +import inmanta.util from inmanta import compiler, const, env, loader, plugins, resources from inmanta.ast import CompilerException from inmanta.const import CF_CACHE_DIR @@ -45,7 +46,6 @@ Project, ) from inmanta.moduletool import ModuleConverter, ModuleTool, ProjectTool -from inmanta.util import CanonicalRequirement, parse_requirement from packaging.version import Version from utils import PipIndex, create_python_package, log_contains, module_from_template, v1_module_from_template @@ -346,7 +346,7 @@ def test_load_module_recursive_v2_module_depends_on_v1( project = snippetcompiler.setup_for_snippet( snippet="import v2_depends_on_v1", index_url=local_module_package_index, - python_requires=[parse_requirement(requirement="inmanta-module-v2-depends-on-v1")], + python_requires=[inmanta.util.parse_requirement(requirement="inmanta-module-v2-depends-on-v1")], install_project=False, ) if preload_v1_module: @@ -373,7 +373,7 @@ def test_load_module_recursive_complex_module_dependencies(local_module_package_ snippet="import complex_module_dependencies_mod1", autostd=False, index_url=local_module_package_index, - python_requires=[parse_requirement(requirement="inmanta-module-complex-module-dependencies-mod1")], + python_requires=[inmanta.util.parse_requirement(requirement="inmanta-module-complex-module-dependencies-mod1")], install_project=False, ) assert "complex_module_dependencies_mod1" not in project.modules @@ -397,7 +397,7 @@ def test_load_import_based_v2_project(local_module_package_index: str, snippetco """ module_name: str = "minimalv2module" - def load(requires: Optional[list[CanonicalRequirement]] = None) -> None: + def load(requires: Optional[list[inmanta.util.CanonicalRequirement]] = None) -> None: project: Project = snippetcompiler_clean.setup_for_snippet( f"import {module_name}", autostd=False, @@ -412,7 +412,7 @@ def load(requires: Optional[list[CanonicalRequirement]] = None) -> None: with pytest.raises(ModuleLoadingException, match=f"Failed to load module {module_name}"): load() # assert that it doesn't raise an error with explicit requirements set - load([parse_requirement(requirement=ModuleV2Source.get_package_name_for(module_name))]) + load([inmanta.util.parse_requirement(requirement=ModuleV2Source.get_package_name_for(module_name))]) @pytest.mark.parametrize("v1", [True, False]) @@ -469,7 +469,9 @@ def test_load_import_based_v2_module( extra_index_url=[index.url], # make sure that even listing the requirement in project.yml does not suffice project_requires=[InmantaModuleRequirement.parse(dependency_module_name)], - python_requires=([] if v1 else [parse_requirement(requirement=ModuleV2Source.get_package_name_for(main_module_name))]), + python_requires=( + [] if v1 else [inmanta.util.parse_requirement(requirement=ModuleV2Source.get_package_name_for(main_module_name))] + ), ) if explicit_dependency: @@ -611,7 +613,7 @@ def test_project_requirements_dont_overwrite_core_requirements_source( module_from_template( os.path.join(modules_v2_dir, module_name), module_path, - new_requirements=[parse_requirement(requirement="Jinja2==2.11.3")], + new_requirements=[inmanta.util.parse_requirement(requirement="Jinja2==2.11.3")], ) # Activate the snippetcompiler venv @@ -655,7 +657,7 @@ def test_project_requirements_dont_overwrite_core_requirements_index( module_from_template( os.path.join(modules_v2_dir, module_name), module_path, - new_requirements=[parse_requirement(requirement="Jinja2==2.11.3")], + new_requirements=[inmanta.util.parse_requirement(requirement="Jinja2==2.11.3")], publish_index=index, ) @@ -709,7 +711,7 @@ def test_module_conflicting_dependencies_with_v2_modules( "y", Version("1.0.0"), str(tmpdir.join("y-1.0.0")), - requirements=[parse_requirement(requirement="x~=1.0.0")], + requirements=[inmanta.util.parse_requirement(requirement="x~=1.0.0")], publish_index=index, ) @@ -719,7 +721,7 @@ def test_module_conflicting_dependencies_with_v2_modules( module_from_template( os.path.join(modules_v2_dir, module_name1), module_path1, - new_requirements=[parse_requirement(requirement="y~=1.0.0")], + new_requirements=[inmanta.util.parse_requirement(requirement="y~=1.0.0")], publish_index=index, ) @@ -730,7 +732,7 @@ def test_module_conflicting_dependencies_with_v2_modules( os.path.join(modules_v2_dir, "minimalv2module"), module_path2, new_name="minimalv2module2", - new_requirements=[parse_requirement(requirement="x~=2.0.0")], + new_requirements=[inmanta.util.parse_requirement(requirement="x~=2.0.0")], publish_index=index, ) @@ -790,7 +792,7 @@ def test_module_conflicting_dependencies_with_v1_module( os.path.join(modules_dir, module_name1), module_path1, new_name="modulev1", - new_requirements=[parse_requirement(requirement="y~=1.0.0")], + new_requirements=[inmanta.util.parse_requirement(requirement="y~=1.0.0")], ) # Create the second module @@ -799,7 +801,7 @@ def test_module_conflicting_dependencies_with_v1_module( module_from_template( os.path.join(modules_v2_dir, module_name2), module_path2, - new_requirements=[parse_requirement(requirement="y~=2.0.0")], + new_requirements=[inmanta.util.parse_requirement(requirement="y~=2.0.0")], publish_index=index, ) @@ -847,11 +849,11 @@ def test_module_install_extra_on_project_level_v2_dep( new_name="mymod", new_requirements=[], new_extras={ - "myfeature": [parse_requirement(requirement=package_name_extra)], + "myfeature": [inmanta.util.parse_requirement(requirement=package_name_extra)], }, publish_index=index, ) - package_with_extra: CanonicalRequirement = InmantaModuleRequirement.parse( + package_with_extra: inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( "mymod[myfeature]" ).get_python_package_requirement() package_name: str = f"{ModuleV2.PKG_NAME_PREFIX}mymod" @@ -898,7 +900,7 @@ def test_module_install_extra_on_dep_of_v2_module( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [parse_requirement(requirement=package_name_extra)], + "myfeature": [inmanta.util.parse_requirement(requirement=package_name_extra)], }, publish_index=index, ) @@ -918,7 +920,7 @@ def test_module_install_extra_on_dep_of_v2_module( install_project=True, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[parse_requirement(requirement="inmanta-module-myv2mod")], + python_requires=[inmanta.util.parse_requirement(requirement="inmanta-module-myv2mod")], autostd=False, ) @@ -955,7 +957,7 @@ def test_module_install_extra_on_dep_of_v1_module( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [parse_requirement(requirement=package_name_extra)], + "myfeature": [inmanta.util.parse_requirement(requirement=package_name_extra)], }, publish_index=index, ) @@ -1005,12 +1007,14 @@ def test_module_install_extra_on_project_level_v2_dep_update_scenario( new_name="mymod", new_requirements=[], new_extras={ - "myfeature": [parse_requirement(requirement=package_name_extra)], + "myfeature": [inmanta.util.parse_requirement(requirement=package_name_extra)], }, publish_index=index, ) - package_without_extra: CanonicalRequirement = InmantaModuleRequirement.parse("mymod").get_python_package_requirement() - package_with_extra: CanonicalRequirement = InmantaModuleRequirement.parse( + package_without_extra: inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( + "mymod" + ).get_python_package_requirement() + package_with_extra: inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( "mymod[myfeature]" ).get_python_package_requirement() package_name: str = str(package_without_extra) @@ -1076,12 +1080,14 @@ def test_module_install_extra_on_dep_of_v2_module_update_scenario( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [parse_requirement(requirement=package_name_extra)], + "myfeature": [inmanta.util.parse_requirement(requirement=package_name_extra)], }, publish_index=index, ) - package_without_extra: CanonicalRequirement = InmantaModuleRequirement.parse("depmod").get_python_package_requirement() - package_with_extra: CanonicalRequirement = InmantaModuleRequirement.parse( + package_without_extra: inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( + "depmod" + ).get_python_package_requirement() + package_with_extra: inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( "depmod[myfeature]" ).get_python_package_requirement() package_name: str = str(package_without_extra) @@ -1107,7 +1113,7 @@ def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> install_project=True, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[parse_requirement(requirement="inmanta-module-myv2mod==1.0.0")], + python_requires=[inmanta.util.parse_requirement(requirement="inmanta-module-myv2mod==1.0.0")], autostd=False, ) assert_installed(extra_installed=False) @@ -1129,7 +1135,7 @@ def assert_installed(*, module_installed: bool = True, extra_installed: bool) -> install_project=not do_project_update, index_url=index.url, extra_index_url=[local_module_package_index, "https://pypi.org/simple"], - python_requires=[parse_requirement(requirement="inmanta-module-myv2mod==2.0.0")], + python_requires=[inmanta.util.parse_requirement(requirement="inmanta-module-myv2mod==2.0.0")], autostd=False, ) if do_project_update: @@ -1163,8 +1169,10 @@ def test_module_install_extra_on_dep_of_v1_module_update_scenario( index: PipIndex = PipIndex(artifact_dir=str(tmpdir.join(".index"))) # Publish dependency of V1 module (depmod) to python package repo - package_without_extra: CanonicalRequirement = InmantaModuleRequirement.parse("depmod").get_python_package_requirement() - package_with_extra: CanonicalRequirement = InmantaModuleRequirement.parse( + package_without_extra: inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( + "depmod" + ).get_python_package_requirement() + package_with_extra: inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( "depmod[myfeature]" ).get_python_package_requirement() package_name: str = str(package_without_extra) @@ -1175,7 +1183,7 @@ def test_module_install_extra_on_dep_of_v1_module_update_scenario( new_name="depmod", new_requirements=[], new_extras={ - "myfeature": [parse_requirement(requirement=package_name_extra)], + "myfeature": [inmanta.util.parse_requirement(requirement=package_name_extra)], }, publish_index=index, ) @@ -1239,7 +1247,7 @@ async def test_v1_module_depends_on_third_party_dep_with_extra( os.path.join(tmpdir, "myv1mod"), new_name="myv1mod", new_content_init_cf="", - new_requirements=[parse_requirement(requirement="pkg[optional-a]")], + new_requirements=[inmanta.util.parse_requirement(requirement="pkg[optional-a]")], ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv1mod", @@ -1259,7 +1267,7 @@ async def test_v1_module_depends_on_third_party_dep_with_extra( os.path.join(tmpdir, "myv1mod"), new_name="myv1mod", new_content_init_cf="", - new_requirements=[parse_requirement(requirement="pkg[optional-a,optional-b]")], + new_requirements=[inmanta.util.parse_requirement(requirement="pkg[optional-a,optional-b]")], ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv1mod", @@ -1286,13 +1294,13 @@ async def test_v2_module_depends_on_third_party_dep_with_extra( str(tmpdir.join("myv2mod")), new_name="myv2mod", new_version=Version("1.0.0"), - new_requirements=[parse_requirement(requirement="pkg[optional-a]")], + new_requirements=[inmanta.util.parse_requirement(requirement="pkg[optional-a]")], publish_index=index, ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv2mod", install_project=True, - python_requires=[parse_requirement(requirement="inmanta-module-myv2mod==1.0.0")], + python_requires=[inmanta.util.parse_requirement(requirement="inmanta-module-myv2mod==1.0.0")], index_url=index.url, extra_index_url=[index_with_pkgs_containing_optional_deps], autostd=False, @@ -1308,13 +1316,13 @@ async def test_v2_module_depends_on_third_party_dep_with_extra( str(tmpdir.join("myv2mod")), new_name="myv2mod", new_version=Version("2.0.0"), - new_requirements=[parse_requirement(requirement="pkg[optional-a,optional-b]")], + new_requirements=[inmanta.util.parse_requirement(requirement="pkg[optional-a,optional-b]")], publish_index=index, ) project: Project = snippetcompiler_clean.setup_for_snippet( "import myv2mod", install_project=True, - python_requires=[parse_requirement(requirement="inmanta-module-myv2mod==2.0.0")], + python_requires=[inmanta.util.parse_requirement(requirement="inmanta-module-myv2mod==2.0.0")], index_url=index.url, extra_index_url=[index_with_pkgs_containing_optional_deps], autostd=False, From 45ef63fb5fe03d8ac32d65f5b5f8727546bac25e Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 18 Sep 2024 15:44:28 +0200 Subject: [PATCH 52/74] refactor --- src/inmanta/env.py | 46 +++++++++++++++++++------------------ src/inmanta/file_parser.py | 3 +-- src/inmanta/module.py | 20 ++++++++-------- src/inmanta/moduletool.py | 4 ++-- tests/test_env.py | 6 ++--- tests/test_file_parser.py | 2 +- tests/test_module_loader.py | 16 ++++++------- tests/utils.py | 13 ++++++----- 8 files changed, 57 insertions(+), 53 deletions(-) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 4ff86c6ab5..419634d1cd 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -47,7 +47,7 @@ import packaging.requirements import packaging.utils import packaging.version -from inmanta import const, util +from inmanta import const from inmanta.ast import CompilerException from inmanta.data.model import LEGACY_PIP_DEFAULT, PipConfig from inmanta.server.bootloader import InmantaBootloader @@ -79,7 +79,7 @@ class VersionConflict: :param owner: The package from which the constraint originates """ - requirement: util.CanonicalRequirement + requirement: inmanta.util.CanonicalRequirement installed_version: Optional[packaging.version.Version] = None owner: Optional[str] = None @@ -164,12 +164,12 @@ def get_advice(self) -> Optional[str]: ) -req_list = TypeVar("req_list", Sequence[str], Sequence[util.CanonicalRequirement]) +req_list = TypeVar("req_list", Sequence[str], Sequence[inmanta.util.CanonicalRequirement]) class PythonWorkingSet: @classmethod - def _get_as_requirements_type(cls, requirements: req_list) -> Sequence[util.CanonicalRequirement]: + def _get_as_requirements_type(cls, requirements: req_list) -> Sequence[inmanta.util.CanonicalRequirement]: """ Convert requirements from Union[Sequence[str], Sequence[Requirement]] to Sequence[Requirement] """ @@ -188,8 +188,8 @@ def are_installed(cls, requirements: req_list) -> bool: installed_packages: dict[str, packaging.version.Version] = cls.get_packages_in_working_set() def _are_installed_recursive( - reqs: Sequence[util.CanonicalRequirement], - seen_requirements: Sequence[util.CanonicalRequirement], + reqs: Sequence[inmanta.util.CanonicalRequirement], + seen_requirements: Sequence[inmanta.util.CanonicalRequirement], contained_in_extra: Optional[str] = None, ) -> bool: """ @@ -224,7 +224,7 @@ def _are_installed_recursive( if distribution is None: return False - pkgs_required_by_extra: set[util.CanonicalRequirement] = set( + pkgs_required_by_extra: set[inmanta.util.CanonicalRequirement] = set( [inmanta.util.parse_requirement(str(e)) for e in distribution.requires(extras=(extra,))] ) - set([inmanta.util.parse_requirement(str(e)) for e in distribution.requires(extras=())]) if not _are_installed_recursive( @@ -235,7 +235,7 @@ def _are_installed_recursive( return False return True - reqs_as_requirements: Sequence[util.CanonicalRequirement] = cls._get_as_requirements_type(requirements) + reqs_as_requirements: Sequence[inmanta.util.CanonicalRequirement] = cls._get_as_requirements_type(requirements) return _are_installed_recursive(reqs_as_requirements, seen_requirements=[]) @classmethod @@ -382,7 +382,7 @@ def run_pip_install_command_from_config( cls, python_path: str, config: PipConfig, - requirements: Optional[Sequence[util.CanonicalRequirement]] = None, + requirements: Optional[Sequence[inmanta.util.CanonicalRequirement]] = None, requirements_files: Optional[list[str]] = None, upgrade: bool = False, upgrade_strategy: PipUpgradeStrategy = PipUpgradeStrategy.ONLY_IF_NEEDED, @@ -423,7 +423,7 @@ async def async_run_pip_install_command_from_config( cls, python_path: str, config: PipConfig, - requirements: Optional[Sequence[util.CanonicalRequirement]] = None, + requirements: Optional[Sequence[inmanta.util.CanonicalRequirement]] = None, requirements_files: Optional[list[str]] = None, upgrade: bool = False, upgrade_strategy: PipUpgradeStrategy = PipUpgradeStrategy.ONLY_IF_NEEDED, @@ -463,7 +463,7 @@ def _prepare_command( cls, python_path: str, config: PipConfig, - requirements: Optional[Sequence[util.CanonicalRequirement]] = None, + requirements: Optional[Sequence[inmanta.util.CanonicalRequirement]] = None, requirements_files: Optional[list[str]] = None, upgrade: bool = False, upgrade_strategy: PipUpgradeStrategy = PipUpgradeStrategy.ONLY_IF_NEEDED, @@ -817,7 +817,7 @@ def get_installed_packages(self, only_editable: bool = False) -> dict[str, packa def install_for_config( self, - requirements: list[util.CanonicalRequirement], + requirements: list[inmanta.util.CanonicalRequirement], config: PipConfig, upgrade: bool = False, constraint_files: Optional[list[str]] = None, @@ -858,7 +858,7 @@ def install_for_config( async def async_install_for_config( self, - requirements: list[util.CanonicalRequirement], + requirements: list[inmanta.util.CanonicalRequirement], config: PipConfig, upgrade: bool = False, constraint_files: Optional[list[str]] = None, @@ -895,7 +895,7 @@ async def async_install_for_config( def install_from_index( self, - requirements: list[util.CanonicalRequirement], + requirements: list[inmanta.util.CanonicalRequirement], index_urls: Optional[list[str]] = None, upgrade: bool = False, allow_pre_releases: bool = False, @@ -986,7 +986,7 @@ def get_protected_inmanta_packages(cls) -> list[str]: ] @classmethod - def _get_requirements_on_inmanta_package(cls) -> Sequence[util.CanonicalRequirement]: + def _get_requirements_on_inmanta_package(cls) -> Sequence[inmanta.util.CanonicalRequirement]: """ Returns the content of the requirement file that should be supplied to each `pip install` invocation to make sure that no Inmanta packages gets overridden. @@ -1122,7 +1122,7 @@ def are_installed(self, requirements: req_list) -> bool: def install_for_config( self, - requirements: list[util.CanonicalRequirement], + requirements: list[inmanta.util.CanonicalRequirement], config: PipConfig, upgrade: bool = False, constraint_files: Optional[list[str]] = None, @@ -1142,7 +1142,7 @@ def install_for_config( def get_constraint_violations_for_check( self, strict_scope: Optional[Pattern[str]] = None, - constraints: Optional[list[util.CanonicalRequirement]] = None, + constraints: Optional[list[inmanta.util.CanonicalRequirement]] = None, ) -> tuple[set[VersionConflict], set[VersionConflict]]: """ Return the constraint violations that exist in this venv. Returns a tuple of non-strict and strict violations, @@ -1150,7 +1150,7 @@ def get_constraint_violations_for_check( """ class OwnedRequirement(NamedTuple): - requirement: util.CanonicalRequirement + requirement: inmanta.util.CanonicalRequirement owner: Optional[str] = None def is_owned_by(self, owners: abc.Set[str]) -> bool: @@ -1210,7 +1210,7 @@ def is_owned_by(self, owners: abc.Set[str]) -> bool: def check( self, strict_scope: Optional[Pattern[str]] = None, - constraints: Optional[list[util.CanonicalRequirement]] = None, + constraints: Optional[list[inmanta.util.CanonicalRequirement]] = None, ) -> None: """ Check this Python environment for incompatible dependencies in installed packages. @@ -1235,7 +1235,9 @@ def check( for violation in constraint_violations: LOGGER.warning("%s", violation) - def check_legacy(self, in_scope: Pattern[str], constraints: Optional[list[util.CanonicalRequirement]] = None) -> bool: + def check_legacy( + self, in_scope: Pattern[str], constraints: Optional[list[inmanta.util.CanonicalRequirement]] = None + ) -> bool: """ Check this Python environment for incompatible dependencies in installed packages. This method is a legacy method in the sense that it has been replaced with a more correct check defined in self.check(). This method is invoked @@ -1254,7 +1256,7 @@ def check_legacy(self, in_scope: Pattern[str], constraints: Optional[list[util.C working_set: abc.Iterable[importlib.metadata.Distribution] = importlib.metadata.distributions() # add all requirements of all in scope packages installed in this environment - all_constraints: set[util.CanonicalRequirement] = set(constraints if constraints is not None else []).union( + all_constraints: set[inmanta.util.CanonicalRequirement] = set(constraints if constraints is not None else []).union( inmanta.util.parse_requirement(requirement=requirement) for dist_info in working_set if in_scope.fullmatch(dist_info.name) @@ -1443,7 +1445,7 @@ def _activate_that(self) -> None: def install_for_config( self, - requirements: list[util.CanonicalRequirement], + requirements: list[inmanta.util.CanonicalRequirement], config: PipConfig, upgrade: bool = False, constraint_files: Optional[list[str]] = None, diff --git a/src/inmanta/file_parser.py b/src/inmanta/file_parser.py index a7e5170702..4f0c6d2e23 100644 --- a/src/inmanta/file_parser.py +++ b/src/inmanta/file_parser.py @@ -20,7 +20,6 @@ import inmanta.util import packaging.utils -from inmanta import util from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap @@ -57,7 +56,7 @@ class RequirementsTxtParser: """ @classmethod - def parse(cls, filename: str) -> list[util.CanonicalRequirement]: + def parse(cls, filename: str) -> list[inmanta.util.CanonicalRequirement]: """ Get all the requirements in `filename` as a list of `Requirement` instances. """ diff --git a/src/inmanta/module.py b/src/inmanta/module.py index ad2b2eab9b..fd73a6a4b2 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -56,7 +56,7 @@ import packaging.requirements import packaging.utils import packaging.version -from inmanta import RUNNING_TESTS, const, env, loader, plugins, util +from inmanta import RUNNING_TESTS, const, env, loader, plugins from inmanta.ast import CompilerException, LocatableString, Location, Namespace, Range, WrappingRuntimeException from inmanta.ast.blocks import BasicBlock from inmanta.ast.statements import BiStatement, DefinitionStatement, DynamicStatement, Statement @@ -98,13 +98,13 @@ class InmantaModuleRequirement: used by distinguishing the two on a type level. """ - def __init__(self, requirement: util.CanonicalRequirement) -> None: + def __init__(self, requirement: inmanta.util.CanonicalRequirement) -> None: if requirement.name.startswith(ModuleV2.PKG_NAME_PREFIX): raise ValueError( f"InmantaModuleRequirement instances work with inmanta module names, not python package names. " f"Problematic case: {str(requirement)}" ) - self._requirement: util.CanonicalRequirement = requirement + self._requirement: inmanta.util.CanonicalRequirement = requirement @property def project_name(self) -> str: @@ -151,7 +151,7 @@ def parse(cls: type[TInmantaModuleRequirement], spec: str) -> TInmantaModuleRequ raise ValueError("Invalid Inmanta module requirement: Inmanta module names use '_', not '-'.") return cls(inmanta.util.parse_requirement(requirement=spec)) - def get_python_package_requirement(self) -> util.CanonicalRequirement: + def get_python_package_requirement(self) -> inmanta.util.CanonicalRequirement: """ Return a Requirement with the name of the Python distribution package for this module requirement. """ @@ -721,7 +721,7 @@ def install(self, project: "Project", module_spec: list[InmantaModuleRequirement assert_pip_has_source(project.metadata.pip, f"a v2 module {module_name}") - requirements: list[util.CanonicalRequirement] = [req.get_python_package_requirement() for req in module_spec] + requirements: list[inmanta.util.CanonicalRequirement] = [req.get_python_package_requirement() for req in module_spec] preinstalled: Optional[ModuleV2] = self.get_installed_module(project, module_name) # Get known requires and add them to prevent invalidating constraints through updates @@ -1179,7 +1179,7 @@ def has_requirement_for(self, pkg_name: str) -> bool: canonicalized: str = packaging.utils.canonicalize_name(pkg_name) return any(r.name == canonicalized for r in self._requirements) - def set_requirement_and_write(self, requirement: util.CanonicalRequirement) -> None: + def set_requirement_and_write(self, requirement: inmanta.util.CanonicalRequirement) -> None: """ Add the given requirement to the requirements.txt file and update the file on disk, replacing any existing constraints on this package. @@ -2128,7 +2128,7 @@ def install_modules(self, *, bypass_module_cache: bool = False, update_dependenc self.verify_module_version_compatibility() # do python install - pyreq: list[util.CanonicalRequirement] = inmanta.util.parse_requirements(self.collect_python_requirements()) + pyreq: list[inmanta.util.CanonicalRequirement] = inmanta.util.parse_requirements(self.collect_python_requirements()) if len(pyreq) > 0: # upgrade both direct and transitive module dependencies: eager upgrade strategy @@ -2546,7 +2546,9 @@ def verify_python_requires(self) -> None: Verifies no incompatibilities exist within the Python environment with respect to installed module v2 requirements. """ if self.strict_deps_check: - constraints: list[util.CanonicalRequirement] = inmanta.util.parse_requirements(self.collect_python_requirements()) + constraints: list[inmanta.util.CanonicalRequirement] = inmanta.util.parse_requirements( + self.collect_python_requirements() + ) env.process_env.check(strict_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*"), constraints=constraints) else: if not env.process_env.check_legacy(in_scope=re.compile(f"{ModuleV2.PKG_NAME_PREFIX}.*")): @@ -3425,7 +3427,7 @@ def add_module_requirement_persistent(self, requirement: InmantaModuleRequiremen # Parse config file config_parser = ConfigParser() config_parser.read(self.get_metadata_file_path()) - python_pkg_requirement: util.CanonicalRequirement = requirement.get_python_package_requirement() + python_pkg_requirement: inmanta.util.CanonicalRequirement = requirement.get_python_package_requirement() if config_parser.has_option("options", "install_requires"): new_install_requires = [ r diff --git a/src/inmanta/moduletool.py b/src/inmanta/moduletool.py index a2129e40a9..73f03df89d 100644 --- a/src/inmanta/moduletool.py +++ b/src/inmanta/moduletool.py @@ -52,7 +52,7 @@ import packaging.requirements import toml from build.env import DefaultIsolatedEnv -from inmanta import const, env, util +from inmanta import const, env from inmanta.ast import CompilerException from inmanta.command import CLIException, ShowUsageException from inmanta.const import CF_CACHE_DIR, MAX_UPDATE_ATTEMPT @@ -472,7 +472,7 @@ def update( def do_update(specs: Mapping[str, Sequence[InmantaModuleRequirement]], modules: list[str]) -> None: v2_modules = {module for module in modules if my_project.module_source.path_for(module) is not None} - v2_python_specs: list[util.CanonicalRequirement] = [ + v2_python_specs: list[inmanta.util.CanonicalRequirement] = [ module_spec.get_python_package_requirement() for module, module_specs in specs.items() for module_spec in module_specs diff --git a/tests/test_env.py b/tests/test_env.py index f72ac1b003..faa059a248 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -34,7 +34,7 @@ import pytest import inmanta.util -from inmanta import env, loader, module, util +from inmanta import env, loader, module from inmanta.data.model import PipConfig from inmanta.env import Pip from packaging import version @@ -452,7 +452,7 @@ def test_active_env_get_module_file_editable_namespace_package( def create_install_package( - name: str, version: version.Version, requirements: list[util.CanonicalRequirement], local_module_package_index: str + name: str, version: version.Version, requirements: list[inmanta.util.CanonicalRequirement], local_module_package_index: str ) -> None: """ Creates and installs a simple package with specified requirements. Creates package in a temporary directory and @@ -563,7 +563,7 @@ def test_active_env_check_constraints(caplog, tmpvenv_active_inherit: str, local """ caplog.set_level(logging.WARNING) in_scope: Pattern[str] = re.compile("test-package-.*") - constraints: list[util.CanonicalRequirement] = [inmanta.util.parse_requirement(requirement="test-package-one~=1.0")] + constraints: list[inmanta.util.CanonicalRequirement] = [inmanta.util.parse_requirement(requirement="test-package-one~=1.0")] env.process_env.check(in_scope) diff --git a/tests/test_file_parser.py b/tests/test_file_parser.py index 67c8c7458b..259a1318e5 100644 --- a/tests/test_file_parser.py +++ b/tests/test_file_parser.py @@ -43,7 +43,7 @@ def test_requirements_txt_parser(tmpdir) -> None: fd.write(content) expected_requirements = ["test==1.2.3", "other-dep~=2.0.0", "third-dep<5.0.0", "splitteddep", "Capital"] - requirements: list[inmanta.util.CanonicalRequirement] = RequirementsTxtParser().parse(requirements_txt_file) + requirements: list[inmanta.inmanta.util.CanonicalRequirement] = RequirementsTxtParser().parse(requirements_txt_file) assert requirements == inmanta.util.parse_requirements(expected_requirements) requirements_as_str = RequirementsTxtParser.parse_requirements_as_strs(requirements_txt_file) assert requirements_as_str == expected_requirements diff --git a/tests/test_module_loader.py b/tests/test_module_loader.py index 6cbfecfb59..99c2276c0a 100644 --- a/tests/test_module_loader.py +++ b/tests/test_module_loader.py @@ -397,7 +397,7 @@ def test_load_import_based_v2_project(local_module_package_index: str, snippetco """ module_name: str = "minimalv2module" - def load(requires: Optional[list[inmanta.util.CanonicalRequirement]] = None) -> None: + def load(requires: Optional[list[inmanta.inmanta.util.CanonicalRequirement]] = None) -> None: project: Project = snippetcompiler_clean.setup_for_snippet( f"import {module_name}", autostd=False, @@ -853,7 +853,7 @@ def test_module_install_extra_on_project_level_v2_dep( }, publish_index=index, ) - package_with_extra: inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( + package_with_extra: inmanta.inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( "mymod[myfeature]" ).get_python_package_requirement() package_name: str = f"{ModuleV2.PKG_NAME_PREFIX}mymod" @@ -1011,10 +1011,10 @@ def test_module_install_extra_on_project_level_v2_dep_update_scenario( }, publish_index=index, ) - package_without_extra: inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( + package_without_extra: inmanta.inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( "mymod" ).get_python_package_requirement() - package_with_extra: inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( + package_with_extra: inmanta.inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( "mymod[myfeature]" ).get_python_package_requirement() package_name: str = str(package_without_extra) @@ -1084,10 +1084,10 @@ def test_module_install_extra_on_dep_of_v2_module_update_scenario( }, publish_index=index, ) - package_without_extra: inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( + package_without_extra: inmanta.inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( "depmod" ).get_python_package_requirement() - package_with_extra: inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( + package_with_extra: inmanta.inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( "depmod[myfeature]" ).get_python_package_requirement() package_name: str = str(package_without_extra) @@ -1169,10 +1169,10 @@ def test_module_install_extra_on_dep_of_v1_module_update_scenario( index: PipIndex = PipIndex(artifact_dir=str(tmpdir.join(".index"))) # Publish dependency of V1 module (depmod) to python package repo - package_without_extra: inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( + package_without_extra: inmanta.inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( "depmod" ).get_python_package_requirement() - package_with_extra: inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( + package_with_extra: inmanta.inmanta.util.CanonicalRequirement = InmantaModuleRequirement.parse( "depmod[myfeature]" ).get_python_package_requirement() package_name: str = str(package_without_extra) diff --git a/tests/utils.py b/tests/utils.py index 8bb9d68188..8e9ac2b9d4 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -40,6 +40,7 @@ import build import build.env +import inmanta.util import packaging.requirements import packaging.version from _pytest.mark import MarkDecorator @@ -491,11 +492,11 @@ def create_python_package( pkg_version: packaging.version.Version, path: str, *, - requirements: Optional[Sequence[util.CanonicalRequirement]] = None, + requirements: Optional[Sequence[inmanta.util.CanonicalRequirement]] = None, install: bool = False, editable: bool = False, publish_index: Optional[PipIndex] = None, - optional_dependencies: Optional[dict[str, Sequence[util.CanonicalRequirement]]] = None, + optional_dependencies: Optional[dict[str, Sequence[inmanta.util.CanonicalRequirement]]] = None, ) -> None: """ Creates an empty Python package. @@ -580,9 +581,9 @@ def module_from_template( *, new_version: Optional[packaging.version.Version] = None, new_name: Optional[str] = None, - new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, util.CanonicalRequirement]]] = None, + new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, inmanta.util.CanonicalRequirement]]] = None, new_extras: Optional[ - abc.Mapping[str, abc.Sequence[Union[module.InmantaModuleRequirement, util.CanonicalRequirement]]] + abc.Mapping[str, abc.Sequence[Union[module.InmantaModuleRequirement, inmanta.util.CanonicalRequirement]]] ] = None, install: bool = False, editable: bool = False, @@ -612,7 +613,7 @@ def module_from_template( """ def to_python_requires( - requires: abc.Sequence[Union[module.InmantaModuleRequirement, util.CanonicalRequirement]] + requires: abc.Sequence[Union[module.InmantaModuleRequirement, inmanta.util.CanonicalRequirement]] ) -> list[str]: return [ str(req) if isinstance(req, packaging.requirements.Requirement) else str(req.get_python_package_requirement()) @@ -698,7 +699,7 @@ def v1_module_from_template( *, new_version: Optional[packaging.version.Version] = None, new_name: Optional[str] = None, - new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, util.CanonicalRequirement]]] = None, + new_requirements: Optional[Sequence[Union[module.InmantaModuleRequirement, inmanta.util.CanonicalRequirement]]] = None, new_content_init_cf: Optional[str] = None, new_content_init_py: Optional[str] = None, ) -> module.ModuleV2Metadata: From ab6f0980b078decf3d3e84c5f87f05561bf1f913 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 18 Sep 2024 15:52:20 +0200 Subject: [PATCH 53/74] refactor --- tests/server/test_compilerservice.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/server/test_compilerservice.py b/tests/server/test_compilerservice.py index 4440939828..f814588c28 100644 --- a/tests/server/test_compilerservice.py +++ b/tests/server/test_compilerservice.py @@ -30,6 +30,7 @@ from collections import abc from typing import TYPE_CHECKING, Optional +import inmanta.util import py.path import pytest from pytest import approx @@ -49,7 +50,7 @@ from inmanta.server.protocol import Server from inmanta.server.services.compilerservice import CompilerService, CompileRun, CompileStateListener from inmanta.server.services.notificationservice import NotificationService -from inmanta.util import ensure_directory_exist, parse_requirement +from inmanta.util import ensure_directory_exist from server.conftest import EnvironmentFactory from utils import LogSequence, report_db_index_usage, retry_limited, wait_for_version @@ -1806,7 +1807,7 @@ def patch_get_protected_inmanta_packages(): venv = PythonEnvironment(env_path=venv_path) assert name_protected_pkg not in venv.get_installed_packages() venv.install_for_config( - requirements=[parse_requirement(requirement=name_protected_pkg)], + requirements=[inmanta.util.parse_requirement(requirement=name_protected_pkg)], config=PipConfig( index_url=local_module_package_index, ), From 4fb89020f482a190f8a9c01ba386a41a8cb3061c Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 18 Sep 2024 17:23:37 +0200 Subject: [PATCH 54/74] regex broke tests --- src/inmanta/util/__init__.py | 20 ++++++++++---------- tests/server/test_compilerservice.py | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/inmanta/util/__init__.py b/src/inmanta/util/__init__.py index 3d643408be..d33438e7a4 100644 --- a/src/inmanta/util/__init__.py +++ b/src/inmanta/util/__init__.py @@ -875,19 +875,19 @@ def _reset_counter(self) -> None: self._exhausted_pool_events_count = 0 -def remove_comment_part_from_specifier(to_clean: str) -> str: +def remove_comment_part(to_clean: str) -> str: """ - Remove the comment part of a requirement specifier + Remove the comment part of a given string - :param to_clean: The requirement specifier to clean - :return: A cleaned requirement specifier + :param to_clean: The string to clean + :return: A cleaned string """ # Refer to PEP 508. A requirement could contain a hashtag - groups = REQUIREMENT_SPECIFIER.search(to_clean) - if not groups: - return "" - - return groups.group(1) + to_clean = to_clean.strip() + drop_comment, _, _ = to_clean.partition(" #") + # We make sure whitespaces are not counted in the length of this string, e.g. " #" + drop_comment = drop_comment.strip() + return drop_comment """ @@ -909,7 +909,7 @@ def parse_requirement(requirement: str) -> CanonicalRequirement: """ # packaging.Requirement is not able to parse requirements with comment (if there is one). # Therefore, we need to make sure that the provided requirement doesn't contain any `comment` part - drop_comment = remove_comment_part_from_specifier(to_clean=requirement) + drop_comment = remove_comment_part(to_clean=requirement) if drop_comment.startswith("#") or len(drop_comment) == 0: raise ValueError(f"The requirement is invalid: Cannot be a comment or empty -> `{drop_comment}`!") diff --git a/tests/server/test_compilerservice.py b/tests/server/test_compilerservice.py index f814588c28..5d06c574bf 100644 --- a/tests/server/test_compilerservice.py +++ b/tests/server/test_compilerservice.py @@ -30,13 +30,13 @@ from collections import abc from typing import TYPE_CHECKING, Optional -import inmanta.util import py.path import pytest from pytest import approx import inmanta.ast.export as ast_export import inmanta.data.model as model +import inmanta.util import utils from inmanta import config, data from inmanta.const import INMANTA_REMOVED_SET_ID, ParameterSource From 784e5334eab279700c0ca58889833fd9a1569076 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 18 Sep 2024 17:24:19 +0200 Subject: [PATCH 55/74] remove old regex --- src/inmanta/util/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/inmanta/util/__init__.py b/src/inmanta/util/__init__.py index d33438e7a4..dbe9fd2f56 100644 --- a/src/inmanta/util/__init__.py +++ b/src/inmanta/util/__init__.py @@ -28,7 +28,6 @@ import logging import os import pathlib -import re import socket import threading import time @@ -66,8 +65,6 @@ T = TypeVar("T") S = TypeVar("S") -REQUIREMENT_SPECIFIER = re.compile(r"^\s*#*(\S+).*") - def get_compiler_version() -> str: return COMPILER_VERSION From c750b49e228cf8f31e54922fc0dc1eb5377ee3e2 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 18 Sep 2024 17:58:25 +0200 Subject: [PATCH 56/74] incomplete refactor --- src/inmanta/util/__init__.py | 35 +++++++++++++---------------------- tests/test_file_parser.py | 11 +++++------ tests/utils.py | 3 +-- 3 files changed, 19 insertions(+), 30 deletions(-) diff --git a/src/inmanta/util/__init__.py b/src/inmanta/util/__init__.py index dbe9fd2f56..70c0e552e1 100644 --- a/src/inmanta/util/__init__.py +++ b/src/inmanta/util/__init__.py @@ -872,12 +872,12 @@ def _reset_counter(self) -> None: self._exhausted_pool_events_count = 0 -def remove_comment_part(to_clean: str) -> str: +def remove_comment_part_from_specifier(to_clean: str) -> str: """ - Remove the comment part of a given string + Remove the comment part of a requirement specifier - :param to_clean: The string to clean - :return: A cleaned string + :param to_clean: The requirement specifier to clean + :return: A cleaned requirement specifier """ # Refer to PEP 508. A requirement could contain a hashtag to_clean = to_clean.strip() @@ -898,23 +898,15 @@ def parse_requirement(requirement: str) -> CanonicalRequirement: """ Parse the given requirement string into a requirement object with a canonicalized name, meaning that we are sure that every CanonicalRequirement will follow the same convention regarding the name. This will allow us compare requirements. - This function supposes to receive an actual requirement. Commented strings will not be handled and result in a ValueError + This function supposes to receive an actual requirement. :param requirement: The requirement's name :return: A new requirement instance - :raise: ValueError when commented or empty strings are provided """ - # packaging.Requirement is not able to parse requirements with comment (if there is one). - # Therefore, we need to make sure that the provided requirement doesn't contain any `comment` part - drop_comment = remove_comment_part(to_clean=requirement) - - if drop_comment.startswith("#") or len(drop_comment) == 0: - raise ValueError(f"The requirement is invalid: Cannot be a comment or empty -> `{drop_comment}`!") - # We canonicalize the name of the requirement to be able to compare requirements and check if the requirement is # already installed # /!\ The following line could cause issue because we are not supposed to modify fields of an existing instance - requirement_instance = packaging.requirements.Requirement(requirement_string=drop_comment) + requirement_instance = packaging.requirements.Requirement(requirement_string=requirement) requirement_instance.name = packaging.utils.canonicalize_name(requirement_instance.name) canonical_requirement_instance = CanonicalRequirement(requirement_instance) return canonical_requirement_instance @@ -945,15 +937,14 @@ def parse_requirements_from_file(file_path: pathlib.Path) -> list[CanonicalRequi if not file_path.exists(): raise RuntimeError(f"The provided path does not exist: `{file_path}`!") - requirements = [] with open(file_path) as f: - for line in f.readlines(): - try: - requirements.append(parse_requirement(line)) - except ValueError: - LOGGER.debug("This line was skipped because the requirement could not be parsed: %s", line) - # This line was empty or only containing a comment - continue + file_contents: list[str] = f.readlines() + requirements = [ + parse_requirement(remove_comment_part_from_specifier(line)) + for line in file_contents + if (stripped := line.lstrip()) and not stripped.startswith("#") # preprocessing + ] + return requirements diff --git a/tests/test_file_parser.py b/tests/test_file_parser.py index 259a1318e5..1b3f19c482 100644 --- a/tests/test_file_parser.py +++ b/tests/test_file_parser.py @@ -19,6 +19,7 @@ import os import pathlib +import packaging.requirements import pytest import inmanta.util @@ -43,7 +44,7 @@ def test_requirements_txt_parser(tmpdir) -> None: fd.write(content) expected_requirements = ["test==1.2.3", "other-dep~=2.0.0", "third-dep<5.0.0", "splitteddep", "Capital"] - requirements: list[inmanta.inmanta.util.CanonicalRequirement] = RequirementsTxtParser().parse(requirements_txt_file) + requirements: list[inmanta.util.CanonicalRequirement] = RequirementsTxtParser().parse(requirements_txt_file) assert requirements == inmanta.util.parse_requirements(expected_requirements) requirements_as_str = RequirementsTxtParser.parse_requirements_as_strs(requirements_txt_file) assert requirements_as_str == expected_requirements @@ -53,9 +54,6 @@ def test_requirements_txt_parser(tmpdir) -> None: problematic_requirements = [ "test==1.2.3", - "# A comment", - " ", - "", "other-dep~=2.0.0", "third-dep<5.0.0 # another comment", "splitteddep", @@ -65,8 +63,9 @@ def test_requirements_txt_parser(tmpdir) -> None: parsed_canonical_requirements = inmanta.util.parse_requirements(expected_requirements) assert parsed_canonical_requirements == requirements - with pytest.raises(ValueError): + with pytest.raises(Exception) as e: inmanta.util.parse_requirements(problematic_requirements) + assert 'Expected end or semicolon (after version specifier)\n third-dep<5.0.0 # another comment\n' in str(e.value) new_content = RequirementsTxtParser.get_content_with_dep_removed(requirements_txt_file, remove_dep_on_pkg="test") expected_content = """ @@ -133,7 +132,7 @@ def test_canonical_requirement(iteration) -> None: """ name, should_fail = iteration if should_fail: - with pytest.raises(ValueError): + with pytest.raises(packaging.requirements.InvalidRequirement): inmanta.util.parse_requirement(requirement=name) else: inmanta.util.parse_requirement(requirement=name) diff --git a/tests/utils.py b/tests/utils.py index 8e9ac2b9d4..1d514703bd 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -53,7 +53,6 @@ from inmanta.server.extensions import ProductMetadata from inmanta.util import get_compiler_version, hash_file from libpip2pi.commands import dir2pi -from packaging.version import Version T = TypeVar("T") @@ -460,7 +459,7 @@ def get_product_meta_data() -> ProductMetadata: def product_version_lower_or_equal_than(version: str) -> bool: - return Version(version=get_product_meta_data().version) <= Version(version=version) + return packaging.version.Version(version=get_product_meta_data().version) <= packaging.version.Version(version=version) def mark_only_for_version_higher_than(version: str) -> "MarkDecorator": From 0993635b2f2ddd16764afdf78f02dff0c3603cc1 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 18 Sep 2024 18:00:36 +0200 Subject: [PATCH 57/74] refactor --- src/inmanta/file_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/inmanta/file_parser.py b/src/inmanta/file_parser.py index 4f0c6d2e23..a622d9687a 100644 --- a/src/inmanta/file_parser.py +++ b/src/inmanta/file_parser.py @@ -94,14 +94,14 @@ def get_content_with_dep_removed(cls, filename: str, remove_dep_on_pkg: str) -> if line_continuation_buffer: line_continuation_buffer += line if not line.endswith("\\"): - if inmanta.util.parse_requirement(requirement=line_continuation_buffer).name != removed_dependency: + if inmanta.util.parse_requirement(requirement=inmanta.util.remove_comment_part_from_specifier(line_continuation_buffer)).name != removed_dependency: result += line_continuation_buffer line_continuation_buffer = "" elif not line.strip() or line.strip().startswith("#"): result += line elif line.endswith("\\"): line_continuation_buffer = line - elif inmanta.util.parse_requirement(requirement=line).name != removed_dependency: + elif inmanta.util.parse_requirement(requirement=inmanta.util.remove_comment_part_from_specifier(line)).name != removed_dependency: result += line else: # Dependency matches `remove_dep_on_pkg` => Remove line from result From cd118751c7d87662b7b6fce6deb14d0b458829ce Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 18 Sep 2024 18:01:06 +0200 Subject: [PATCH 58/74] refactor --- src/inmanta/file_parser.py | 12 ++++++++++-- tests/test_file_parser.py | 4 ++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/inmanta/file_parser.py b/src/inmanta/file_parser.py index a622d9687a..35783545fc 100644 --- a/src/inmanta/file_parser.py +++ b/src/inmanta/file_parser.py @@ -94,14 +94,22 @@ def get_content_with_dep_removed(cls, filename: str, remove_dep_on_pkg: str) -> if line_continuation_buffer: line_continuation_buffer += line if not line.endswith("\\"): - if inmanta.util.parse_requirement(requirement=inmanta.util.remove_comment_part_from_specifier(line_continuation_buffer)).name != removed_dependency: + if ( + inmanta.util.parse_requirement( + requirement=inmanta.util.remove_comment_part_from_specifier(line_continuation_buffer) + ).name + != removed_dependency + ): result += line_continuation_buffer line_continuation_buffer = "" elif not line.strip() or line.strip().startswith("#"): result += line elif line.endswith("\\"): line_continuation_buffer = line - elif inmanta.util.parse_requirement(requirement=inmanta.util.remove_comment_part_from_specifier(line)).name != removed_dependency: + elif ( + inmanta.util.parse_requirement(requirement=inmanta.util.remove_comment_part_from_specifier(line)).name + != removed_dependency + ): result += line else: # Dependency matches `remove_dep_on_pkg` => Remove line from result diff --git a/tests/test_file_parser.py b/tests/test_file_parser.py index 1b3f19c482..65d53b4da7 100644 --- a/tests/test_file_parser.py +++ b/tests/test_file_parser.py @@ -19,10 +19,10 @@ import os import pathlib -import packaging.requirements import pytest import inmanta.util +import packaging.requirements from inmanta.file_parser import RequirementsTxtParser @@ -65,7 +65,7 @@ def test_requirements_txt_parser(tmpdir) -> None: with pytest.raises(Exception) as e: inmanta.util.parse_requirements(problematic_requirements) - assert 'Expected end or semicolon (after version specifier)\n third-dep<5.0.0 # another comment\n' in str(e.value) + assert "Expected end or semicolon (after version specifier)\n third-dep<5.0.0 # another comment\n" in str(e.value) new_content = RequirementsTxtParser.get_content_with_dep_removed(requirements_txt_file, remove_dep_on_pkg="test") expected_content = """ From c40fe2b1db4a16bb10b11f84bd41071c42edc0ba Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Thu, 19 Sep 2024 08:05:37 +0200 Subject: [PATCH 59/74] fix --- tests/test_module_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_module_loader.py b/tests/test_module_loader.py index 99c2276c0a..a9f7a11d13 100644 --- a/tests/test_module_loader.py +++ b/tests/test_module_loader.py @@ -397,7 +397,7 @@ def test_load_import_based_v2_project(local_module_package_index: str, snippetco """ module_name: str = "minimalv2module" - def load(requires: Optional[list[inmanta.inmanta.util.CanonicalRequirement]] = None) -> None: + def load(requires: Optional[list[inmanta.util.CanonicalRequirement]] = None) -> None: project: Project = snippetcompiler_clean.setup_for_snippet( f"import {module_name}", autostd=False, From 63f0a5f90bfcab94204e98c1f1d11fc7f1ed1ec0 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Thu, 19 Sep 2024 11:12:40 +0200 Subject: [PATCH 60/74] refactor --- src/inmanta/agent/executor.py | 3 ++- src/inmanta/env.py | 17 ++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/inmanta/agent/executor.py b/src/inmanta/agent/executor.py index 66ee2970a9..0ce65e10f5 100644 --- a/src/inmanta/agent/executor.py +++ b/src/inmanta/agent/executor.py @@ -35,6 +35,7 @@ import inmanta.types import inmanta.util +import packaging.requirements from inmanta import const from inmanta.agent import config as cfg from inmanta.agent import resourcepool @@ -295,7 +296,7 @@ async def create_and_install_environment(self, blueprint: EnvBlueprint) -> None: await asyncio.get_running_loop().run_in_executor(self.io_threadpool, self.init_env) if len(req): # install_for_config expects at least 1 requirement or a path to install await self.async_install_for_config( - requirements=inmanta.util.parse_requirements(req), + requirements=[packaging.requirements.Requirement(requirement_string=e) for e in req], config=blueprint.pip_config, ) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 419634d1cd..f06696ac2e 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -382,7 +382,7 @@ def run_pip_install_command_from_config( cls, python_path: str, config: PipConfig, - requirements: Optional[Sequence[inmanta.util.CanonicalRequirement]] = None, + requirements: Optional[Sequence[packaging.requirements.Requirement]] = None, requirements_files: Optional[list[str]] = None, upgrade: bool = False, upgrade_strategy: PipUpgradeStrategy = PipUpgradeStrategy.ONLY_IF_NEEDED, @@ -423,7 +423,7 @@ async def async_run_pip_install_command_from_config( cls, python_path: str, config: PipConfig, - requirements: Optional[Sequence[inmanta.util.CanonicalRequirement]] = None, + requirements: Optional[Sequence[packaging.requirements.Requirement]] = None, requirements_files: Optional[list[str]] = None, upgrade: bool = False, upgrade_strategy: PipUpgradeStrategy = PipUpgradeStrategy.ONLY_IF_NEEDED, @@ -463,7 +463,7 @@ def _prepare_command( cls, python_path: str, config: PipConfig, - requirements: Optional[Sequence[inmanta.util.CanonicalRequirement]] = None, + requirements: Optional[Sequence[packaging.requirements.Requirement]] = None, requirements_files: Optional[list[str]] = None, upgrade: bool = False, upgrade_strategy: PipUpgradeStrategy = PipUpgradeStrategy.ONLY_IF_NEEDED, @@ -858,7 +858,7 @@ def install_for_config( async def async_install_for_config( self, - requirements: list[inmanta.util.CanonicalRequirement], + requirements: list[packaging.requirements.Requirement], config: PipConfig, upgrade: bool = False, constraint_files: Optional[list[str]] = None, @@ -979,10 +979,13 @@ def get_protected_inmanta_packages(cls) -> list[str]: """ return [ # Protect product packages - "inmanta", - "inmanta-service-orchestrator", + packaging.utils.canonicalize_name("inmanta"), + packaging.utils.canonicalize_name("inmanta-service-orchestrator"), # Protect all server extensions - *(f"inmanta-{ext_name}" for ext_name in InmantaBootloader.get_available_extensions().keys()), + *( + packaging.utils.canonicalize_name(f"inmanta-{ext_name}") + for ext_name in InmantaBootloader.get_available_extensions().keys() + ), ] @classmethod From 8facdcfd19ba0237601b634e8e24d67f0a79329f Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Thu, 19 Sep 2024 11:22:37 +0200 Subject: [PATCH 61/74] docstring --- src/inmanta/env.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/inmanta/env.py b/src/inmanta/env.py index f06696ac2e..c047c28eb5 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -976,6 +976,7 @@ def install_from_list( def get_protected_inmanta_packages(cls) -> list[str]: """ Returns the list of packages that should not be installed/updated by any operation on a Python environment. + This list of packages will be under the canonical form. """ return [ # Protect product packages From bc43a12e530c8a41328b0bc09d8bf6b93003e20f Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Thu, 19 Sep 2024 19:01:43 +0200 Subject: [PATCH 62/74] refactor --- src/inmanta/util/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/inmanta/util/__init__.py b/src/inmanta/util/__init__.py index 70c0e552e1..4f561bd73d 100644 --- a/src/inmanta/util/__init__.py +++ b/src/inmanta/util/__init__.py @@ -887,11 +887,11 @@ def remove_comment_part_from_specifier(to_clean: str) -> str: return drop_comment +CanonicalRequirement = typing.NewType("CanonicalRequirement", packaging.requirements.Requirement) """ A CanonicalRequirement is a packaging.requirements.Requirement except that the name of this Requirement is canonicalized, which -allows us to compare requirements without dealing afterwards with the format of these requirements. +allows us to compare names without dealing afterwards with the format of these requirements. """ -CanonicalRequirement = typing.NewType("CanonicalRequirement", packaging.requirements.Requirement) def parse_requirement(requirement: str) -> CanonicalRequirement: @@ -905,7 +905,8 @@ def parse_requirement(requirement: str) -> CanonicalRequirement: """ # We canonicalize the name of the requirement to be able to compare requirements and check if the requirement is # already installed - # /!\ The following line could cause issue because we are not supposed to modify fields of an existing instance + # The following line could cause issue because we are not supposed to modify fields of an existing instance + # The version of packaging is constrained to ensure this can not cause problems in production. requirement_instance = packaging.requirements.Requirement(requirement_string=requirement) requirement_instance.name = packaging.utils.canonicalize_name(requirement_instance.name) canonical_requirement_instance = CanonicalRequirement(requirement_instance) From ccab505cbfcdfefb5e736a88545442cf40418e9d Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Fri, 20 Sep 2024 11:04:51 +0200 Subject: [PATCH 63/74] add changelog --- changelogs/unreleased/drop-pkg-resources-first-part.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelogs/unreleased/drop-pkg-resources-first-part.yml diff --git a/changelogs/unreleased/drop-pkg-resources-first-part.yml b/changelogs/unreleased/drop-pkg-resources-first-part.yml new file mode 100644 index 0000000000..c8eedda700 --- /dev/null +++ b/changelogs/unreleased/drop-pkg-resources-first-part.yml @@ -0,0 +1,4 @@ +--- +description: First part of `pkg_resources` library deprecation. +change-type: patch +destination-branches: [master, iso7] From ae80eb8e62bd8762926a28c8e0c7be3154fec6d4 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Mon, 23 Sep 2024 16:01:45 +0200 Subject: [PATCH 64/74] refactor --- src/inmanta/module.py | 7 ++----- src/inmanta/util/__init__.py | 22 ++++++++++++---------- tests/moduletool/test_update.py | 2 +- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/inmanta/module.py b/src/inmanta/module.py index fd73a6a4b2..8717dd0b9b 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -108,10 +108,7 @@ def __init__(self, requirement: inmanta.util.CanonicalRequirement) -> None: @property def project_name(self) -> str: - # Requirement converts all "_" to "-". Inmanta modules use "_" - warnings.warn(InmantaWarning("The `project_name` property has been deprecated in favor of `name`")) - - return self.name + return self._requirement.project_name.replace("-", "_") @property def name(self) -> str: @@ -669,7 +666,7 @@ def from_path(cls, project: Optional["Project"], module_name: str, path: str) -> raise NotImplementedError("Abstract method") def _get_module_name(self, module_spec: list[InmantaModuleRequirement]) -> str: - module_names: set[str] = {req.name for req in module_spec} + module_names: set[str] = {req.project_name for req in module_spec} module_name: str = more_itertools.one( module_names, too_short=ValueError("module_spec should contain at least one requirement"), diff --git a/src/inmanta/util/__init__.py b/src/inmanta/util/__init__.py index 4f561bd73d..a1ce90ed99 100644 --- a/src/inmanta/util/__init__.py +++ b/src/inmanta/util/__init__.py @@ -31,7 +31,6 @@ import socket import threading import time -import typing import uuid import warnings from abc import ABC, abstractmethod @@ -887,11 +886,17 @@ def remove_comment_part_from_specifier(to_clean: str) -> str: return drop_comment -CanonicalRequirement = typing.NewType("CanonicalRequirement", packaging.requirements.Requirement) -""" -A CanonicalRequirement is a packaging.requirements.Requirement except that the name of this Requirement is canonicalized, which -allows us to compare names without dealing afterwards with the format of these requirements. -""" +class CanonicalRequirement(packaging.requirements.Requirement): + """ + A CanonicalRequirement is a packaging.requirements.Requirement except that the name of this Requirement is canonicalized, + which allows us to compare names without dealing afterwards with the format of these requirements. + """ + + def __init__(self, requirement_string: str): + super().__init__(requirement_string) + upper_cases = {i for i in range(len(requirement_string[: len(self.name)])) if requirement_string[i].isupper()} + self.project_name = "".join([self.name[i].upper() if i in upper_cases else self.name[i] for i in range(len(self.name))]) + self.name = packaging.utils.canonicalize_name(self.name) def parse_requirement(requirement: str) -> CanonicalRequirement: @@ -907,10 +912,7 @@ def parse_requirement(requirement: str) -> CanonicalRequirement: # already installed # The following line could cause issue because we are not supposed to modify fields of an existing instance # The version of packaging is constrained to ensure this can not cause problems in production. - requirement_instance = packaging.requirements.Requirement(requirement_string=requirement) - requirement_instance.name = packaging.utils.canonicalize_name(requirement_instance.name) - canonical_requirement_instance = CanonicalRequirement(requirement_instance) - return canonical_requirement_instance + return CanonicalRequirement(requirement_string=requirement) def parse_requirements(requirements: Sequence[str]) -> list[CanonicalRequirement]: diff --git a/tests/moduletool/test_update.py b/tests/moduletool/test_update.py index 1c9f9bc9e8..aa787395e2 100644 --- a/tests/moduletool/test_update.py +++ b/tests/moduletool/test_update.py @@ -127,7 +127,7 @@ def assert_version_installed(module_name: str, version: str) -> None: new_version=Version(current_version), new_name=module_name, new_requirements=( - [InmantaModuleRequirement(inmanta.util.parse_requirement(requirement="module2<3.0.0"))] + [InmantaModuleRequirement(inmanta.util.parse_requirement(requirement="module2<3.0.0"), "module2")] if module_name == "module1" else None ), From ba749a68adf497871b774230e88236a6efa52f6a Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Tue, 24 Sep 2024 09:51:14 +0200 Subject: [PATCH 65/74] fix broken tests --- tests/moduletool/test_update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/moduletool/test_update.py b/tests/moduletool/test_update.py index aa787395e2..1c9f9bc9e8 100644 --- a/tests/moduletool/test_update.py +++ b/tests/moduletool/test_update.py @@ -127,7 +127,7 @@ def assert_version_installed(module_name: str, version: str) -> None: new_version=Version(current_version), new_name=module_name, new_requirements=( - [InmantaModuleRequirement(inmanta.util.parse_requirement(requirement="module2<3.0.0"), "module2")] + [InmantaModuleRequirement(inmanta.util.parse_requirement(requirement="module2<3.0.0"))] if module_name == "module1" else None ), From 6aae7ae510cc164f3d8bb45c754ebd8c2b35edcd Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 25 Sep 2024 08:25:05 +0200 Subject: [PATCH 66/74] revert change --- src/inmanta/util/__init__.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/inmanta/util/__init__.py b/src/inmanta/util/__init__.py index a1ce90ed99..4f561bd73d 100644 --- a/src/inmanta/util/__init__.py +++ b/src/inmanta/util/__init__.py @@ -31,6 +31,7 @@ import socket import threading import time +import typing import uuid import warnings from abc import ABC, abstractmethod @@ -886,17 +887,11 @@ def remove_comment_part_from_specifier(to_clean: str) -> str: return drop_comment -class CanonicalRequirement(packaging.requirements.Requirement): - """ - A CanonicalRequirement is a packaging.requirements.Requirement except that the name of this Requirement is canonicalized, - which allows us to compare names without dealing afterwards with the format of these requirements. - """ - - def __init__(self, requirement_string: str): - super().__init__(requirement_string) - upper_cases = {i for i in range(len(requirement_string[: len(self.name)])) if requirement_string[i].isupper()} - self.project_name = "".join([self.name[i].upper() if i in upper_cases else self.name[i] for i in range(len(self.name))]) - self.name = packaging.utils.canonicalize_name(self.name) +CanonicalRequirement = typing.NewType("CanonicalRequirement", packaging.requirements.Requirement) +""" +A CanonicalRequirement is a packaging.requirements.Requirement except that the name of this Requirement is canonicalized, which +allows us to compare names without dealing afterwards with the format of these requirements. +""" def parse_requirement(requirement: str) -> CanonicalRequirement: @@ -912,7 +907,10 @@ def parse_requirement(requirement: str) -> CanonicalRequirement: # already installed # The following line could cause issue because we are not supposed to modify fields of an existing instance # The version of packaging is constrained to ensure this can not cause problems in production. - return CanonicalRequirement(requirement_string=requirement) + requirement_instance = packaging.requirements.Requirement(requirement_string=requirement) + requirement_instance.name = packaging.utils.canonicalize_name(requirement_instance.name) + canonical_requirement_instance = CanonicalRequirement(requirement_instance) + return canonical_requirement_instance def parse_requirements(requirements: Sequence[str]) -> list[CanonicalRequirement]: From ba01c4d2873280e58fec0788d2f5a95cba7cfcee Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Mon, 30 Sep 2024 11:05:35 +0200 Subject: [PATCH 67/74] rollback --- src/inmanta/module.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/inmanta/module.py b/src/inmanta/module.py index 8717dd0b9b..78b4dc3a14 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -48,6 +48,7 @@ from typing import Annotated, ClassVar, Dict, Generic, List, NewType, Optional, TextIO, TypeVar, Union, cast import more_itertools +import pkg_resources import pydantic import yaml from pydantic import BaseModel, Field, NameEmail, StringConstraints, ValidationError, field_validator @@ -72,6 +73,7 @@ from packaging.version import Version from ruamel.yaml.comments import CommentedMap +from pkg_resources import Requirement try: from typing import TYPE_CHECKING except ImportError: @@ -98,17 +100,18 @@ class InmantaModuleRequirement: used by distinguishing the two on a type level. """ - def __init__(self, requirement: inmanta.util.CanonicalRequirement) -> None: + def __init__(self, requirement: packaging.requirements.Requirement) -> None: if requirement.name.startswith(ModuleV2.PKG_NAME_PREFIX): raise ValueError( f"InmantaModuleRequirement instances work with inmanta module names, not python package names. " f"Problematic case: {str(requirement)}" ) - self._requirement: inmanta.util.CanonicalRequirement = requirement + self._requirement: inmanta.util.CanonicalRequirement = inmanta.util.parse_requirement(str(requirement)) + self._project_name = re.sub('[^A-Za-z0-9.]+', '-', requirement.name) @property def project_name(self) -> str: - return self._requirement.project_name.replace("-", "_") + return self._project_name.replace("-", "_") @property def name(self) -> str: @@ -146,7 +149,7 @@ def parse(cls: type[TInmantaModuleRequirement], spec: str) -> TInmantaModuleRequ ) if "-" in spec: raise ValueError("Invalid Inmanta module requirement: Inmanta module names use '_', not '-'.") - return cls(inmanta.util.parse_requirement(requirement=spec)) + return cls(packaging.requirements.Requirement(requirement_string=spec)) def get_python_package_requirement(self) -> inmanta.util.CanonicalRequirement: """ From 24692f8630ed03125023295499a1a8adaaafcb97 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Mon, 30 Sep 2024 11:13:06 +0200 Subject: [PATCH 68/74] refactor --- src/inmanta/module.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/inmanta/module.py b/src/inmanta/module.py index 78b4dc3a14..c250834961 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -48,7 +48,6 @@ from typing import Annotated, ClassVar, Dict, Generic, List, NewType, Optional, TextIO, TypeVar, Union, cast import more_itertools -import pkg_resources import pydantic import yaml from pydantic import BaseModel, Field, NameEmail, StringConstraints, ValidationError, field_validator @@ -73,7 +72,6 @@ from packaging.version import Version from ruamel.yaml.comments import CommentedMap -from pkg_resources import Requirement try: from typing import TYPE_CHECKING except ImportError: @@ -107,7 +105,7 @@ def __init__(self, requirement: packaging.requirements.Requirement) -> None: f"Problematic case: {str(requirement)}" ) self._requirement: inmanta.util.CanonicalRequirement = inmanta.util.parse_requirement(str(requirement)) - self._project_name = re.sub('[^A-Za-z0-9.]+', '-', requirement.name) + self._project_name = re.sub("[^A-Za-z0-9.]+", "-", requirement.name) @property def project_name(self) -> str: From dc95c996699dde6f4fbeaa27122f9d0a348666c3 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Mon, 30 Sep 2024 12:54:37 +0200 Subject: [PATCH 69/74] refactor --- src/inmanta/module.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/inmanta/module.py b/src/inmanta/module.py index c250834961..21bdb9931b 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -86,6 +86,7 @@ TModule = TypeVar("TModule", bound="Module") TProject = TypeVar("TProject", bound="Project") TInmantaModuleRequirement = TypeVar("TInmantaModuleRequirement", bound="InmantaModuleRequirement") +REGEX_DISTRIBUTION_NAME = re.compile(r"[^A-Za-z0-9.]+") @stable_api @@ -104,8 +105,11 @@ def __init__(self, requirement: packaging.requirements.Requirement) -> None: f"InmantaModuleRequirement instances work with inmanta module names, not python package names. " f"Problematic case: {str(requirement)}" ) - self._requirement: inmanta.util.CanonicalRequirement = inmanta.util.parse_requirement(str(requirement)) - self._project_name = re.sub("[^A-Za-z0-9.]+", "-", requirement.name) + # Retaken from pkg_resources.safe_name() -> Convert an arbitrary string to a standard distribution name + self._project_name = REGEX_DISTRIBUTION_NAME.sub("-", requirement.name) + # We canonicalize the name directly here instead of having to parse a second time the requirement + requirement.name = packaging.utils.canonicalize_name(requirement.name) + self._requirement: inmanta.util.CanonicalRequirement = inmanta.util.CanonicalRequirement(requirement) @property def project_name(self) -> str: @@ -119,7 +123,7 @@ def name(self) -> str: def key(self) -> str: # Requirement converts all "_" to "-". Inmanta modules use "_" warnings.warn(InmantaWarning("The `key` property has been deprecated in favor of `name`")) - return self._requirement.name.replace("-", "_") + return self.name @property def specifier(self) -> SpecifierSet: From 3e71ebf580dacf0ec78e3360334caae935e50655 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Mon, 30 Sep 2024 14:47:15 +0200 Subject: [PATCH 70/74] refactor to make sure to not share this instance --- src/inmanta/module.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/inmanta/module.py b/src/inmanta/module.py index 21bdb9931b..46d4a46ff8 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -107,9 +107,7 @@ def __init__(self, requirement: packaging.requirements.Requirement) -> None: ) # Retaken from pkg_resources.safe_name() -> Convert an arbitrary string to a standard distribution name self._project_name = REGEX_DISTRIBUTION_NAME.sub("-", requirement.name) - # We canonicalize the name directly here instead of having to parse a second time the requirement - requirement.name = packaging.utils.canonicalize_name(requirement.name) - self._requirement: inmanta.util.CanonicalRequirement = inmanta.util.CanonicalRequirement(requirement) + self._requirement: inmanta.util.CanonicalRequirement = inmanta.util.parse_requirement(str(requirement)) @property def project_name(self) -> str: From d77a95d4d2088535f86b32a536b27e7375fb99cc Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 2 Oct 2024 09:54:49 +0200 Subject: [PATCH 71/74] refactor --- src/inmanta/module.py | 8 +-- tests/moduletool/conftest.py | 24 +++---- tests/moduletool/test_freeze.py | 112 ++++++++++++++++---------------- 3 files changed, 71 insertions(+), 73 deletions(-) diff --git a/src/inmanta/module.py b/src/inmanta/module.py index 46d4a46ff8..80dbf76633 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -86,7 +86,6 @@ TModule = TypeVar("TModule", bound="Module") TProject = TypeVar("TProject", bound="Project") TInmantaModuleRequirement = TypeVar("TInmantaModuleRequirement", bound="InmantaModuleRequirement") -REGEX_DISTRIBUTION_NAME = re.compile(r"[^A-Za-z0-9.]+") @stable_api @@ -105,13 +104,12 @@ def __init__(self, requirement: packaging.requirements.Requirement) -> None: f"InmantaModuleRequirement instances work with inmanta module names, not python package names. " f"Problematic case: {str(requirement)}" ) - # Retaken from pkg_resources.safe_name() -> Convert an arbitrary string to a standard distribution name - self._project_name = REGEX_DISTRIBUTION_NAME.sub("-", requirement.name) self._requirement: inmanta.util.CanonicalRequirement = inmanta.util.parse_requirement(str(requirement)) @property def project_name(self) -> str: - return self._project_name.replace("-", "_") + warnings.warn(InmantaWarning("The `project_name` property has been deprecated in favor of `name`")) + return self.name @property def name(self) -> str: @@ -149,7 +147,7 @@ def parse(cls: type[TInmantaModuleRequirement], spec: str) -> TInmantaModuleRequ ) if "-" in spec: raise ValueError("Invalid Inmanta module requirement: Inmanta module names use '_', not '-'.") - return cls(packaging.requirements.Requirement(requirement_string=spec)) + return cls(inmanta.util.parse_requirement(requirement=spec)) def get_python_package_requirement(self) -> inmanta.util.CanonicalRequirement: """ diff --git a/tests/moduletool/conftest.py b/tests/moduletool/conftest.py index a4aef66f1c..7db8d7b78f 100644 --- a/tests/moduletool/conftest.py +++ b/tests/moduletool/conftest.py @@ -204,17 +204,17 @@ def modules_repo(git_modules_dir) -> str: E-> H D-> F,G """ - make_module_simple_deps(reporoot, "A", ["B", "C", "D"], project=True) - make_module_simple_deps(reporoot, "B") - c = make_module_simple_deps(reporoot, "C", ["E", "F", "E::a"], version="3.0") - add_file(c, "model/a.cf", "import modI", "add mod C::a", "3.2") - make_module_simple_deps(reporoot, "D", ["F", "G"]) - e = make_module_simple_deps(reporoot, "E", ["H"], version="3.0") - add_file(e, "model/a.cf", "import modJ", "add mod E::a", "3.2") - make_module_simple_deps(reporoot, "F") - make_module_simple_deps(reporoot, "G") - make_module_simple_deps(reporoot, "H") - make_module_simple_deps(reporoot, "I") - make_module_simple_deps(reporoot, "J") + make_module_simple_deps(reporoot, "a", ["b", "c", "d"], project=True) + make_module_simple_deps(reporoot, "b") + c = make_module_simple_deps(reporoot, "c", ["e", "f", "e::a"], version="3.0") + add_file(c, "model/a.cf", "import modi", "add mod c::a", "3.2") + make_module_simple_deps(reporoot, "d", ["f", "g"]) + e = make_module_simple_deps(reporoot, "e", ["h"], version="3.0") + add_file(e, "model/a.cf", "import modj", "add mod e::a", "3.2") + make_module_simple_deps(reporoot, "f") + make_module_simple_deps(reporoot, "g") + make_module_simple_deps(reporoot, "h") + make_module_simple_deps(reporoot, "i") + make_module_simple_deps(reporoot, "j") return reporoot diff --git a/tests/moduletool/test_freeze.py b/tests/moduletool/test_freeze.py index 6cda52f9eb..7e3f047ebf 100644 --- a/tests/moduletool/test_freeze.py +++ b/tests/moduletool/test_freeze.py @@ -32,42 +32,42 @@ @pytest.mark.slowtest def test_freeze_basic(git_modules_dir: str, modules_repo: str, tmpdir): - install_project(git_modules_dir, "projectA", tmpdir) + install_project(git_modules_dir, "projecta", tmpdir) modtool = ModuleTool() - cmod = modtool.get_module("modC") - assert cmod.get_freeze("modC", recursive=False, mode="==") == {"std": "== 3.2", "modE": "== 3.2", "modF": "== 3.2"} - assert cmod.get_freeze("modC", recursive=True, mode="==") == { + cmod = modtool.get_module("modc") + assert cmod.get_freeze("modc", recursive=False, mode="==") == {"std": "== 3.2", "mode": "== 3.2", "modf": "== 3.2"} + assert cmod.get_freeze("modc", recursive=True, mode="==") == { "std": "== 3.2", - "modE": "== 3.2", - "modF": "== 3.2", - "modH": "== 3.2", - "modJ": "== 3.2", + "mode": "== 3.2", + "modf": "== 3.2", + "modh": "== 3.2", + "modj": "== 3.2", } - assert cmod.get_freeze("modC::a", recursive=False, mode="==") == {"std": "== 3.2", "modI": "== 3.2"} + assert cmod.get_freeze("modc::a", recursive=False, mode="==") == {"std": "== 3.2", "modi": "== 3.2"} @pytest.mark.slowtest def test_project_freeze_basic(git_modules_dir: str, modules_repo: str, tmpdir): - install_project(git_modules_dir, "projectA", tmpdir) + install_project(git_modules_dir, "projecta", tmpdir) modtool = ModuleTool() proj = modtool.get_project() assert proj.get_freeze(recursive=False, mode="==") == { "std": "== 3.2", - "modB": "== 3.2", - "modC": "== 3.2", - "modD": "== 3.2", + "modb": "== 3.2", + "modc": "== 3.2", + "modd": "== 3.2", } assert proj.get_freeze(recursive=True, mode="==") == { "std": "== 3.2", - "modB": "== 3.2", - "modC": "== 3.2", - "modD": "== 3.2", - "modE": "== 3.2", - "modF": "== 3.2", - "modG": "== 3.2", - "modH": "== 3.2", - "modJ": "== 3.2", + "modb": "== 3.2", + "modc": "== 3.2", + "modd": "== 3.2", + "mode": "== 3.2", + "modf": "== 3.2", + "modg": "== 3.2", + "modh": "== 3.2", + "modj": "== 3.2", } @@ -85,7 +85,7 @@ def test_project_freeze_bad(git_modules_dir: str, modules_repo: str, tmpdir): @pytest.mark.slowtest def test_project_freeze(git_modules_dir: str, modules_repo: str, capsys, tmpdir): - coroot = install_project(git_modules_dir, "projectA", tmpdir) + coroot = install_project(git_modules_dir, "projecta", tmpdir) app(["project", "freeze", "-o", "-"]) @@ -95,16 +95,16 @@ def test_project_freeze(git_modules_dir: str, modules_repo: str, capsys, tmpdir) assert len(err) == 0, err assert ( out - == """name: projectA + == """name: projecta license: Apache 2.0 version: 0.0.1 modulepath: libs downloadpath: libs repo: %s requires: -- modB ~= 3.2 -- modC ~= 3.2 -- modD ~= 3.2 +- modb ~= 3.2 +- modc ~= 3.2 +- modd ~= 3.2 - std ~= 3.2 """ % modules_repo @@ -113,7 +113,7 @@ def test_project_freeze(git_modules_dir: str, modules_repo: str, capsys, tmpdir) @pytest.mark.slowtest def test_project_freeze_disk(git_modules_dir: str, modules_repo: str, capsys, tmpdir): - coroot = install_project(git_modules_dir, "projectA", tmpdir) + coroot = install_project(git_modules_dir, "projecta", tmpdir) app(["project", "freeze"]) @@ -125,16 +125,16 @@ def test_project_freeze_disk(git_modules_dir: str, modules_repo: str, capsys, tm with open(os.path.join(coroot, "project.yml"), encoding="utf-8") as fh: assert ( fh.read() - == """name: projectA + == """name: projecta license: Apache 2.0 version: 0.0.1 modulepath: libs downloadpath: libs repo: %s requires: -- modB ~= 3.2 -- modC ~= 3.2 -- modD ~= 3.2 +- modb ~= 3.2 +- modc ~= 3.2 +- modd ~= 3.2 - std ~= 3.2 """ % modules_repo @@ -143,7 +143,7 @@ def test_project_freeze_disk(git_modules_dir: str, modules_repo: str, capsys, tm @pytest.mark.slowtest def test_project_freeze_odd_opperator(git_modules_dir: str, modules_repo: str, tmpdir): - coroot = install_project(git_modules_dir, "projectA", tmpdir) + coroot = install_project(git_modules_dir, "projecta", tmpdir) # Start a new subprocess, because inmanta-cli executes sys.exit() when an invalid argument is used. process = subprocess.Popen( @@ -165,10 +165,10 @@ def test_project_freeze_odd_opperator(git_modules_dir: str, modules_repo: str, t def test_project_options_in_config(git_modules_dir: str, modules_repo: str, capsys, tmpdir): coroot = install_project( git_modules_dir, - "projectA", + "projecta", tmpdir, config_content=f""" -name: projectA +name: projecta license: Apache 2.0 version: 0.0.1 modulepath: libs @@ -188,7 +188,7 @@ def verify(): with open("project.yml", encoding="utf-8") as fh: assert fh.read() == ( - """name: projectA + """name: projecta license: Apache 2.0 version: 0.0.1 modulepath: libs @@ -197,14 +197,14 @@ def verify(): freeze_recursive: true freeze_operator: == requires: -- modB == 3.2 -- modC == 3.2 -- modD == 3.2 -- modE == 3.2 -- modF == 3.2 -- modG == 3.2 -- modH == 3.2 -- modJ == 3.2 +- modb == 3.2 +- modc == 3.2 +- modd == 3.2 +- mode == 3.2 +- modf == 3.2 +- modg == 3.2 +- modh == 3.2 +- modj == 3.2 - std == 3.2 """ % modules_repo @@ -218,7 +218,7 @@ def verify(): @pytest.mark.slowtest def test_module_freeze(git_modules_dir: str, modules_repo: str, capsys, tmpdir): - coroot = install_project(git_modules_dir, "projectA", tmpdir) + coroot = install_project(git_modules_dir, "projecta", tmpdir) def verify(): out, err = capsys.readouterr() @@ -226,24 +226,24 @@ def verify(): assert os.path.getsize(os.path.join(coroot, "project.yml")) != 0 assert len(err) == 0, err assert out == ( - """name: modC + """name: modc license: Apache 2.0 version: '3.2' requires: -- modE ~= 3.2 -- modF ~= 3.2 -- modI ~= 3.2 +- mode ~= 3.2 +- modf ~= 3.2 +- modi ~= 3.2 - std ~= 3.2 """ ) - app(["module", "-m", "modC", "freeze", "-o", "-"]) + app(["module", "-m", "modc", "freeze", "-o", "-"]) verify() @pytest.mark.slowtest def test_module_freeze_self_disk(git_modules_dir: str, modules_repo: str, capsys, tmpdir): - coroot = install_project(git_modules_dir, "projectA", tmpdir) + coroot = install_project(git_modules_dir, "projecta", tmpdir) def verify(): out, err = capsys.readouterr() @@ -251,25 +251,25 @@ def verify(): assert len(err) == 0, err assert len(out) == 0, out - modpath = os.path.join(coroot, "libs/modC/module.yml") + modpath = os.path.join(coroot, "libs/modc/module.yml") assert os.path.getsize(os.path.join(coroot, "project.yml")) != 0 assert os.path.getsize(modpath) != 0 with open(modpath, encoding="utf-8") as fh: outf = fh.read() assert outf == ( - """name: modC + """name: modc license: Apache 2.0 version: '3.2' requires: -- modE ~= 3.2 -- modF ~= 3.2 -- modI ~= 3.2 +- mode ~= 3.2 +- modf ~= 3.2 +- modi ~= 3.2 - std ~= 3.2 """ ) - modp = os.path.join(coroot, "libs/modC") + modp = os.path.join(coroot, "libs/modc") app(["project", "install"]) os.chdir(modp) app(["module", "freeze"]) From af785c48129e7bec0bd22232b51c6e86484ae7f4 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 2 Oct 2024 10:14:24 +0200 Subject: [PATCH 72/74] remove constraint on setuptools --- .github/dependabot.yml | 3 --- setup.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index bee0272417..c2cfbfe7b1 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -31,9 +31,6 @@ updates: # Allow both direct and indirect updates for all packages - dependency-type: "all" ignore: - # #6668 - - dependency-name: "setuptools" - update-types: ["version-update:semver-major"] - dependency-name: "pytest-inmanta-extensions" versions: [">=0.0.1"] - dependency-name: "pydantic" diff --git a/setup.py b/setup.py index a95a24b0d6..ff6affe7cd 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ "pynacl~=1.5", "python-dateutil~=2.0", "pyyaml~=6.0", - "setuptools<71", + "setuptools", "texttable~=1.0", "tornado~=6.0", # lower bound because of ilevkivskyi/typing_inspect#100 From 8e4ae5766dd0853a260953cc480f468e98289dc5 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 2 Oct 2024 13:48:36 +0200 Subject: [PATCH 73/74] missing revert changes --- src/inmanta/module.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/inmanta/module.py b/src/inmanta/module.py index 80dbf76633..7be822016d 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -98,13 +98,13 @@ class InmantaModuleRequirement: used by distinguishing the two on a type level. """ - def __init__(self, requirement: packaging.requirements.Requirement) -> None: + def __init__(self, requirement: inmanta.util.CanonicalRequirement) -> None: if requirement.name.startswith(ModuleV2.PKG_NAME_PREFIX): raise ValueError( f"InmantaModuleRequirement instances work with inmanta module names, not python package names. " f"Problematic case: {str(requirement)}" ) - self._requirement: inmanta.util.CanonicalRequirement = inmanta.util.parse_requirement(str(requirement)) + self._requirement = inmanta.util.parse_requirement(str(requirement)) @property def project_name(self) -> str: @@ -667,7 +667,7 @@ def from_path(cls, project: Optional["Project"], module_name: str, path: str) -> raise NotImplementedError("Abstract method") def _get_module_name(self, module_spec: list[InmantaModuleRequirement]) -> str: - module_names: set[str] = {req.project_name for req in module_spec} + module_names: set[str] = {req.name for req in module_spec} module_name: str = more_itertools.one( module_names, too_short=ValueError("module_spec should contain at least one requirement"), From 73d9cf3f186c6d0cfb8d5b725c21c5ff5afcd0e0 Mon Sep 17 00:00:00 2001 From: Hugo Lloreda Date: Wed, 2 Oct 2024 14:39:41 +0200 Subject: [PATCH 74/74] missing revert changes --- src/inmanta/module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inmanta/module.py b/src/inmanta/module.py index 7be822016d..3bcd206c1b 100644 --- a/src/inmanta/module.py +++ b/src/inmanta/module.py @@ -104,7 +104,7 @@ def __init__(self, requirement: inmanta.util.CanonicalRequirement) -> None: f"InmantaModuleRequirement instances work with inmanta module names, not python package names. " f"Problematic case: {str(requirement)}" ) - self._requirement = inmanta.util.parse_requirement(str(requirement)) + self._requirement = requirement @property def project_name(self) -> str: