Merge pull request #234 from hermes-hmc/feature/219-declarative-appro…

…ach-to-settings

Declarative approach to settings

docs/source/conf.py

@@ -49,6 +49,7 @@
     'sphinx.ext.ifconfig',
     'sphinx.ext.githubpages',
     'sphinx.ext.autodoc',
+    'sphinx.ext.napoleon',
     'sphinx-favicon',
     'sphinxcontrib.contentui',
     'sphinxcontrib.images',

hermes.toml

@@ -3,10 +3,10 @@
 # SPDX-License-Identifier: CC0-1.0
 
 [harvest]
-from = [ "cff", "git" ]
+sources = [ "cff", "git" ]
 
 [harvest.cff]
-validate = false
+enable_validation = false
 
 [deposit]
 target = "invenio_rdm"

pyproject.toml

@@ -40,8 +40,9 @@ cffconvert = "^2.0.0"
 toml = "^0.10.2"
 pyparsing = "^3.0.9"
 requests = "^2.28.1"
+pydantic = "^2.5.1"
+pydantic-settings = "^2.1.0"
 
-# Packages for developers
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.1.1"
 pytest-cov = "^3.0.0"

src/hermes/cli.py

@@ -8,20 +8,24 @@
 """
 This module provides the main entry point for the HERMES command line application.
 """
+import sys
 import logging
 import typing as t
 import pathlib
 from importlib import metadata
 
 import click
+import toml
+from pydantic import ValidationError
 
-from hermes import config
+import hermes.logger as logger
 from hermes.commands import workflow
-from hermes.config import configure, init_logging
+from hermes.logger import configure, init_logging
+from hermes.settings import HermesSettings
 
 
 def log_header(header, summary=None):
-    _log = config.getLogger('cli')
+    _log = logger.getLogger('cli')
 
     dist = metadata.distribution('hermes')
     meta = dist.metadata
@@ -86,8 +90,22 @@ def invoke(self, ctx: click.Context) -> t.Any:
 
         # Get the user provided working dir from the --path option or default to current working directory.
         working_path = ctx.params.get('path').absolute()
-
-        configure(ctx.params.get('config').absolute(), working_path)
+        config_path = ctx.params.get('config').absolute()
+        try:
+            with open(config_path, 'r') as config_file:
+                config = HermesSettings.model_validate(toml.load(config_file))
+        except ValidationError as e:
+            print(e, file=sys.stderr)
+            sys.exit(1)
+        except FileNotFoundError:
+            if config_path.name != 'hermes.toml':
+                # An explicit filename (different from default) was given, so the file should be available...
+                print(f"Configuration not present at {config_path}.", file=sys.stderr)
+                sys.exit(1)
+            else:
+                print("No 'hermes.toml' found, falling back to default configuration (might not be what you want).")
+                config = HermesSettings()
+        configure(config, working_path)
         init_logging()
         log_header(None)
 

src/hermes/commands/deposit/file.py

@@ -7,7 +7,6 @@
 
 import json
 
-from hermes import config
 from hermes.commands.deposit.base import BaseDepositPlugin
 from hermes.model.path import ContextPath
 
@@ -17,7 +16,7 @@ def map_metadata(self) -> None:
         self.ctx.update(ContextPath.parse('deposit.file'), self.ctx['codemeta'])
 
     def publish(self) -> None:
-        file_config = config.get("deposit").get("file", {})
+        file_config = self.ctx.config.deposit.file
         output_data = self.ctx['deposit.file']
 
         with open(file_config.get('filename', 'hermes.json'), 'w') as deposition_file:

src/hermes/commands/deposit/invenio.py

@@ -16,7 +16,6 @@
 import click
 import requests
 
-from hermes import config
 from hermes.commands.deposit.base import BaseDepositPlugin
 from hermes.commands.deposit.error import DepositionUnauthorizedError
 from hermes.error import MisconfigurationError
@@ -43,11 +42,11 @@ def __init__(self, auth_token=None, platform_name=None) -> None:
         if platform_name is not None:
             self.platform_name = platform_name
 
-        self.config = config.get("deposit").get(self.platform_name, {})
+        self.config = getattr(self.ctx.config.deposit, self.platform_name)
         self.headers.update({"User-Agent": hermes_user_agent})
 
         self.auth_token = auth_token
-        self.site_url = self.config.get("site_url")
+        self.site_url = self.config.site_url
         if self.site_url is None:
             raise MisconfigurationError(f"deposit.{self.platform_name}.site_url is not configured")
 
@@ -81,7 +80,7 @@ def new_deposit(self):
 
     @property
     def api_paths(self):
-        return self.config.get("api_paths", {})
+        return self.config.api_paths
 
     @property
     def licenses_api_path(self):
@@ -251,7 +250,7 @@ def __init__(self, click_ctx: click.Context, ctx: CodeMetaContext, client=None,
             self.client = client
 
         self.resolver = resolver or self.invenio_resolver_class(self.client)
-        self.config = config.get("deposit").get(self.platform_name, {})
+        self.config = getattr(self.ctx.config.deposit, self.platform_name)
         self.links = {}
 
     # TODO: Populate some data structure here? Or move more of this into __init__?
@@ -270,8 +269,8 @@ def prepare(self) -> None:
         - update ``self.ctx`` with metadata collected during the checks
         """
 
-        rec_id = self.config.get('record_id')
-        doi = self.config.get('doi')
+        rec_id = self.config.record_id
+        doi = self.config.doi
 
         try:
             codemeta_identifier = self.ctx["codemeta.identifier"]
@@ -564,7 +563,7 @@ def _get_community_identifiers(self):
         raised.
         """
 
-        communities = self.config.get("communities")
+        communities = self.config.communities
         if communities is None:
             return None
 
@@ -596,7 +595,7 @@ def _get_access_modalities(self, license):
         This function also makes sure that the given embargo date can be parsed as an ISO
         8601 string representation and that the access rights are given as a string.
         """
-        access_right = self.config.get("access_right")
+        access_right = self.config.access_right
         if access_right is None:
             raise MisconfigurationError(f"deposit.{self.platform_name}.access_right is not configured")
 
@@ -607,7 +606,7 @@ def _get_access_modalities(self, license):
                 f"{', '.join(access_right_options)}"
             )
 
-        embargo_date = self.config.get("embargo_date")
+        embargo_date = self.config.embargo_date
         if access_right == "embargoed" and embargo_date is None:
             raise MisconfigurationError(
                 f"With access_right {access_right}, "
@@ -623,7 +622,7 @@ def _get_access_modalities(self, license):
                     "Must be in ISO 8601 format."
                 )
 
-        access_conditions = self.config.get("access_conditions")
+        access_conditions = self.config.access_conditions
         if access_right == "restricted" and access_conditions is None:
             raise MisconfigurationError(
                 f"With access_right {access_right}, "

src/hermes/commands/harvest/cff.py

@@ -59,7 +59,7 @@ def harvest_cff(click_ctx: click.Context, ctx: HermesHarvestContext):
 
     # Validate the content to be correct CFF
     cff_dict = _load_cff_from_file(cff_data)
-    if ctx.config.get('validate', True) and not _validate(cff_file, cff_dict):
+    if ctx.config.cff_validate and not _validate(cff_file, cff_dict):
         raise HermesValidationError(cff_file)
 
     # Convert to CodeMeta using cffconvert

src/hermes/commands/postprocess/invenio.py

@@ -11,8 +11,6 @@
 import toml
 from ruamel import yaml
 
-from hermes import config
-
 
 _log = logging.getLogger('deposit.invenio')
 
@@ -21,9 +19,9 @@ def config_record_id(ctx):
     deposition_path = ctx.get_cache('deposit', 'deposit')
     with deposition_path.open("r") as deposition_file:
         deposition = json.load(deposition_file)
-    conf = config.get('hermes')
+    conf = ctx.config.hermes
     try:
-        conf['deposit']['invenio']['record_id'] = deposition['record_id']
+        conf.deposit.invenio.record_id = deposition['record_id']
         toml.dump(conf, open('hermes.toml', 'w'))
     except KeyError:
         raise RuntimeError("No deposit.invenio configuration available to store record id in") from None

src/hermes/commands/postprocess/invenio_rdm.py

@@ -10,8 +10,6 @@
 
 import toml
 
-from hermes import config
-
 
 _log = logging.getLogger('deposit.invenio_rdm')
 
@@ -20,7 +18,7 @@ def config_record_id(ctx):
     deposition_path = ctx.get_cache('deposit', 'deposit')
     with deposition_path.open("r") as deposition_file:
         deposition = json.load(deposition_file)
-    conf = config.get('hermes')
+    conf = ctx.config.hermes
     try:
         conf['deposit']['invenio_rdm']['record_id'] = deposition['record_id']
         toml.dump(conf, open('hermes.toml', 'w'))

src/hermes/commands/workflow.py

@@ -15,7 +15,6 @@
 
 import click
 
-from hermes import config
 from hermes.commands.deposit.base import BaseDepositPlugin
 from hermes.error import MisconfigurationError
 from hermes.model.context import HermesContext, HermesHarvestContext, CodeMetaContext
@@ -40,8 +39,10 @@ def harvest(click_ctx: click.Context):
     ctx.init_cache("harvest")
 
     # Get all harvesters
-    harvest_config = config.get("harvest")
-    harvester_names = harvest_config.get('from', [ep.name for ep in metadata.entry_points(group='hermes.harvest')])
+    harvest_config = ctx.config.harvest
+    harvester_names = harvest_config.sources if type(harvest_config.sources) else [ep.name for ep in
+                                                                                   metadata.entry_points(
+                                                                                       group='hermes.harvest')]
 
     for harvester_name in harvester_names:
         harvesters = metadata.entry_points(group='hermes.harvest', name=harvester_name)
@@ -55,7 +56,7 @@ def harvest(click_ctx: click.Context):
         _log.debug(". Loading harvester from %s", harvester.value)
         harvest = harvester.load()
 
-        with HermesHarvestContext(ctx, harvester, harvest_config.get(harvester.name, {})) as harvest_ctx:
+        with HermesHarvestContext(ctx, harvester, harvest_config) as harvest_ctx:
             harvest(click_ctx, harvest_ctx)
             for _key, ((_value, _tag), *_trace) in harvest_ctx._data.items():
                 if any(v != _value and t == _tag for v, t in _trace):
@@ -83,8 +84,10 @@ def process(click_ctx: click.Context):
         click_ctx.exit(1)
 
     # Get all harvesters
-    harvest_config = config.get("harvest")
-    harvester_names = harvest_config.get('from', [ep.name for ep in metadata.entry_points(group='hermes.harvest')])
+    harvest_config = ctx.config.harvest
+    harvester_names = harvest_config.sources if type(harvest_config.sources) else [ep.name for ep in
+                                                                                   metadata.entry_points(
+                                                                                       group='hermes.harvest')]
 
     for harvester_name in harvester_names:
         harvesters = metadata.entry_points(group='hermes.harvest', name=harvester_name)
@@ -188,12 +191,12 @@ def deposit(click_ctx: click.Context, initial, auth_token, file):
     with open(codemeta_file) as codemeta_fh:
         ctx.update(codemeta_path, json.load(codemeta_fh))
 
-    deposit_config = config.get("deposit")
+    deposit_config = ctx.config.deposit
 
     plugin_group = "hermes.deposit"
     # TODO: Is having a default a good idea?
     # TODO: Should we allow a list here so that multiple plugins are run?
-    plugin_name = deposit_config.get("target", "invenio")
+    plugin_name = deposit_config.target
 
     try:
         ep, *eps = metadata.entry_points(group=plugin_group, name=plugin_name)
@@ -238,8 +241,8 @@ def postprocess(click_ctx: click.Context):
         click_ctx.exit(1)
 
     # Get all postprocessors
-    postprocess_config = config.get("postprocess")
-    postprocess_names = postprocess_config.get('execute', [])
+    postprocess_config = ctx.config.postprocess
+    postprocess_names = postprocess_config.execute
 
     for postprocess_name in postprocess_names:
         postprocessors = metadata.entry_points(group='hermes.postprocess', name=postprocess_name)

src/hermes/config.py → src/hermes/logger.py

@@ -6,12 +6,11 @@
 
 import logging
 import pathlib
-import sys
-
-import toml
 
+import hermes.settings
 from hermes.model.context import HermesContext
 
+config = None
 # This is the default logging configuration, required to see log output at all.
 #  - Maybe it could possibly somehow be a somewhat good idea to move this into an own module ... later perhaps
 _logging_config = {
@@ -61,7 +60,7 @@
 }
 
 
-def configure(config_path: pathlib.Path, working_path: pathlib.Path):
+def configure(settings: hermes.settings.HermesSettings, working_path: pathlib.Path):
     """
     Load the configuration from the given path as global hermes configuration.
 
@@ -76,52 +75,14 @@ def configure(config_path: pathlib.Path, working_path: pathlib.Path):
     _config['logging']['handlers']['auditfile']['filename'] = \
         working_path / HermesContext.hermes_cache_name / "audit.log"
 
-    # Load configuration if not present
-    try:
-        with open(config_path, 'r') as config_file:
-            hermes_config = toml.load(config_file)
-            _config['hermes'] = hermes_config
-            _config['logging'] = hermes_config.get('logging', _config['logging'])
-
-    except FileNotFoundError:
-        if config_path.name != 'hermes.toml':
-            # An explicit filename (different from default) was given, so the file should be available...
-            print(f"Configuration not present at {config_path}.", file=sys.stderr)
-            sys.exit(1)
-
-
-def get(name: str) -> dict:
-    """
-    Retrieve the configuration dict for a certain sub-system (i.e., a section from the config file).
-
-    The returned dict comes directly from the cache.
-    I.e., it is possible to do the following *stunt* to inject default values:
-
-    .. code: python
+    _config['hermes'] = settings
+    global config
+    config = settings
+    _config['logging'] = settings.logging if settings.logging != {} else _config['logging']
 
-        my_config = config.get('my-config')
-        my_config.update({ 'default': 'values' })
-
-    :param name: The section to retrieve.
-    :return: The loaded configuration data or an empty dictionary.
-    """
-
-    if name not in _config:
-        # If configuration is not present, create it.
-        if 'hermes' not in _config:
-            _config['hermes'] = {}
-        if name not in _config['hermes']:
-            _config['hermes'][name] = {}
-        _config[name] = _config['hermes'][name]
-
-    elif name != 'hermes' and _config['hermes'][name] is not _config[name]:
-        # If a configuration was loaded, after the defaults were set, update it.
-        _config[name].update(_config['hermes'].get('name', {}))
-
-    return _config.get(name)
+# Might be a good idea to move somewhere else (see comment for _logging_config)?
 
 
-# Might be a good idea to move somewhere else (see comment for _logging_config)?
 _loggers = {}
 
 

src/hermes/model/context.py

@@ -57,6 +57,10 @@ def __init__(self, project_dir: t.Optional[Path] = None):
         self._errors = []
         self.contexts = {self.hermes_lod_context}
 
+        # HACK this needs to be done differently
+        from hermes import logger
+        self.config = logger.config
+
     def __getitem__(self, key: ContextPath | str) -> t.Any:
         """
         Access a single entry from the context.