diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8b085e791..fdf83c188 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: hooks: - id: mypy exclude: (docs/.*) - additional_dependencies: ["types-filelock", "types-freezegun", "types-pkg_resources"] + additional_dependencies: ["attrs", "types-filelock", "types-freezegun", "types-pkg_resources"] - repo: https://github.com/PyCQA/flake8 rev: 7.0.0 diff --git a/CHANGES.md b/CHANGES.md index 6f84930bf..e91086c4d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,11 @@ - Updated documentation for `[mirror]` configuration options `PR #1669` +## Bug Fixes + +- Don't write a diff file unless the 'diff-file' option is set in the configuration file `PR #1694` +- Correctly interpret references with surrounding text when using custom reference syntax in 'diff-file' `PR #1694` + # 6.5.0 ## New Features diff --git a/src/bandersnatch/config/__init__.py b/src/bandersnatch/config/__init__.py new file mode 100644 index 000000000..565b68aad --- /dev/null +++ b/src/bandersnatch/config/__init__.py @@ -0,0 +1,4 @@ +# flake8: noqa +from .core import BandersnatchConfig +from .errors import ConfigurationError, InvalidValueError, MissingOptionError +from .mirror_options import MirrorOptions diff --git a/src/bandersnatch/config/attrs_utils.py b/src/bandersnatch/config/attrs_utils.py new file mode 100644 index 000000000..c47e7cdcc --- /dev/null +++ b/src/bandersnatch/config/attrs_utils.py @@ -0,0 +1,72 @@ +from collections.abc import Callable +from configparser import ConfigParser +from typing import Any, TypeAlias, TypeVar + +import attrs +from attrs import Attribute + +from . import errors + +AttrsConverter: TypeAlias = Callable[[Any], Any] +AttrsFieldTransformer: TypeAlias = Callable[[type, list[Attribute]], list[Attribute]] + +_V = TypeVar("_V") +AttrsValidator: TypeAlias = Callable[[Any, Attribute, _V], _V] + + +def only_if_str(converter_fn: AttrsConverter) -> AttrsConverter: + """Wrap an attrs converter function so it is only applied to strings. + + 'converter' functions on attrs fields are applied to all values set to the field, + *including* the default value. This causes problems if the default value is already + the desired type but the converter only handles strings. + + :param AttrsConverter converter_fn: any attrs converter + :return AttrsConverter: converter function that uses `converter_fn` if the passed + value is a string, and otherwise returns the value unmodified. + """ + + def _apply_if_str(value: Any) -> Any: + if isinstance(value, str): + return converter_fn(value) + else: + return value + + return _apply_if_str + + +def get_name_value_for_option( + config: ConfigParser, section_name: str, option: Attribute +) -> tuple[str, object | None]: + option_name = config.optionxform(option.alias or option.name) + + if option.default is attrs.NOTHING and not config.has_option( + section_name, option_name + ): + raise errors.MissingOptionError.for_option(section_name, option_name) + + getter: Callable[..., Any] + if option.converter is not None: + getter = config.get + elif option.type == bool: + getter = config.getboolean + elif option.type == float: + getter = config.getfloat + elif option.type == int: + getter = config.getint + else: + getter = config.get + + try: + option_value = getter(section_name, option_name, fallback=None) + except ValueError as conversion_error: + type_name = option.type.__name__ if option.type else "???" + message = f"can't convert option name '{option_name}' to expected type '{type_name}': {conversion_error!s}" + raise errors.InvalidValueError.for_option( + section_name, option_name, message + ) from conversion_error + + return option_name, option_value + + +validate_not_empty: AttrsValidator = attrs.validators.min_len(1) diff --git a/src/bandersnatch/config/comparison_method.py b/src/bandersnatch/config/comparison_method.py new file mode 100644 index 000000000..f2a356c8b --- /dev/null +++ b/src/bandersnatch/config/comparison_method.py @@ -0,0 +1,30 @@ +"""Enumeration of supported file comparison strategies""" + +import sys + +if sys.version_info >= (3, 11): + from enum import StrEnum +else: + from bandersnatch.utils import StrEnum + + +class ComparisonMethod(StrEnum): + HASH = "hash" + STAT = "stat" + + +class InvalidComparisonMethod(ValueError): + """We don't have a valid comparison method choice from configuration""" + + pass + + +def get_comparison_value(method: str) -> ComparisonMethod: + try: + return ComparisonMethod(method) + except ValueError: + valid_methods = sorted(v.value for v in ComparisonMethod) + raise InvalidComparisonMethod( + f"{method} is not a valid file comparison method. " + + f"Valid options are: {valid_methods}" + ) diff --git a/src/bandersnatch/config/core.py b/src/bandersnatch/config/core.py new file mode 100644 index 000000000..e2e07549f --- /dev/null +++ b/src/bandersnatch/config/core.py @@ -0,0 +1,63 @@ +import importlib.resources +import shutil +import sys +from collections.abc import Mapping +from configparser import ConfigParser, ExtendedInterpolation +from pathlib import Path +from typing import Protocol, TypeVar, cast + +if sys.version_info >= (3, 11): + from typing import Self +else: + Self = TypeVar("Self", bound="ConfigModel") + + +class ConfigModel(Protocol): + + @classmethod + def from_config_parser(cls: type[Self], source: ConfigParser) -> Self: ... + + +_C = TypeVar("_C", bound=ConfigModel) + + +class BandersnatchConfig(ConfigParser): + + def __init__(self, defaults: Mapping[str, str] | None = None) -> None: + super().__init__( + defaults=defaults, + delimiters=("=",), + strict=True, + interpolation=ExtendedInterpolation(), + ) + + self._validate_config_models: dict[str, ConfigModel] = {} + + # This allows writing option names in the config file with either '_' or '-' as word separators + def optionxform(self, optionstr: str) -> str: + return optionstr.lower().replace("-", "_") + + def read_path(self, file_path: Path) -> None: + with file_path.open() as cfg_file: + self.read_file(cfg_file) + + def get_validated(self, model: type[_C]) -> _C: + name = model.__qualname__ + if name not in self._validate_config_models: + self._validate_config_models[name] = model.from_config_parser(self) + return cast(_C, self._validate_config_models[name]) + + @classmethod + def from_path( + cls, source_file: Path, *, defaults: Mapping[str, str] | None = None + ) -> "BandersnatchConfig": + config = cls(defaults=defaults) + config.read_path(source_file) + return config + + +def write_default_config_file(dest: Path) -> None: + with importlib.resources.path( + "bandersnatch", "default.conf" + ) as default_config_path: + shutil.copy(default_config_path, dest) diff --git a/src/bandersnatch/config/diff_file_reference.py b/src/bandersnatch/config/diff_file_reference.py new file mode 100644 index 000000000..2d7a02ec4 --- /dev/null +++ b/src/bandersnatch/config/diff_file_reference.py @@ -0,0 +1,60 @@ +""" +Custom configparser section/option reference syntax for the diff-file option. + +diff-file supports a "section reference" syntax for it's value: + + [mirror] + ... + diff-file = /folder{{
_