Implement 'simplified' approach

- Model 'build' and 'solve' no longer use a 'temporary' configuration
- Simplified the pydantic configuration schema, and made it fully 'frozen' and non extensible
- Moved data table loading into the Data generation portion
- Fixed most tests

src/calliope/config.py

@@ -5,14 +5,14 @@
 from collections.abc import Hashable
 from datetime import datetime
 from pathlib import Path
-from typing import Annotated, Literal, Self, TypeVar
+from typing import Annotated, Literal, TypeVar
 
 import jsonref
 from pydantic import AfterValidator, BaseModel, Field, model_validator
 from pydantic_core import PydanticCustomError
+from typing_extensions import Self
 
 from calliope.attrdict import AttrDict
-from calliope.util import tools
 
 MODES_T = Literal["plan", "operate", "spores"]
 CONFIG_T = Literal["init", "build", "solve"]
@@ -54,13 +54,16 @@ def _hide_from_schema(schema: dict):
 class ConfigBaseModel(BaseModel):
     """A base class for creating pydantic models for Calliope configuration options."""
 
-    _kwargs: dict = {}
+    model_config = {
+        "extra": "forbid",
+        "frozen": True,
+        "revalidate_instances": "always",
+        "use_attribute_docstrings": True,
+    }
 
     def update(self, update_dict: dict, deep: bool = False) -> Self:
         """Return a new iteration of the model with updated fields.
 
-        Updates are validated and stored in the parent class in the `_kwargs` key.
-
         Args:
             update_dict (dict): Dictionary with which to update the base model.
             deep (bool, optional): Set to True to make a deep copy of the model. Defaults to False.
@@ -74,12 +77,10 @@ def update(self, update_dict: dict, deep: bool = False) -> Self:
             key_class = getattr(self, key)
             if isinstance(key_class, ConfigBaseModel):
                 new_dict[key] = key_class.update(val)
-                key_class._kwargs = val
             else:
                 new_dict[key] = val
         updated = super().model_copy(update=new_dict, deep=deep)
         updated.model_validate(updated)
-        self._kwargs = update_dict
         return updated
 
     def model_no_ref_schema(self) -> AttrDict:
@@ -93,31 +94,10 @@ def model_no_ref_schema(self) -> AttrDict:
         schema_dict.del_key("$defs")
         return schema_dict
 
-    @property
-    def applied_keyword_overrides(self) -> dict:
-        """Most recently applied keyword overrides used to update this configuration.
-
-        Returns:
-            dict: Description of applied overrides.
-        """
-        return self._kwargs
-
 
 class Init(ConfigBaseModel):
     """All configuration options used when initialising a Calliope model."""
 
-    model_config = {
-        "title": "init",
-        "extra": "forbid",
-        "frozen": True,
-        "json_schema_extra": hide_from_schema(["def_path"]),
-        "revalidate_instances": "always",
-        "use_attribute_docstrings": True,
-    }
-
-    def_path: Path = Field(default=".", repr=False, exclude=True)
-    """The path to the main model definition YAML file, if one has been used to instantiate the Calliope Model class."""
-
     name: str | None = Field(default=None)
     """Model name"""
 
@@ -142,7 +122,7 @@ class Init(ConfigBaseModel):
     time_resample: str | None = Field(default=None, pattern="^[0-9]+[a-zA-Z]")
     """Setting to adjust time resolution, e.g. '2h' for 2-hourly"""
 
-    time_cluster: Path | None = Field(default=None)
+    time_cluster: str | None = Field(default=None)
     """
     Setting to cluster the timeseries.
     Must be a path to a file where each date is linked to a representative date that also exists in the timeseries.
@@ -160,24 +140,16 @@ class Init(ConfigBaseModel):
     Automatically derived distances from lat/lon coordinates will be given in this unit.
     """
 
-    @model_validator(mode="before")
-    @classmethod
-    def abs_path(cls, data):
-        """Add model definition path."""
-        if data.get("time_cluster", None) is not None:
-            data["time_cluster"] = tools.relative_path(
-                data["def_path"], data["time_cluster"]
-            )
-        return data
-
 
 class BuildOperate(ConfigBaseModel):
     """Operate mode configuration options used when building a Calliope optimisation problem (`calliope.Model.build`)."""
 
     model_config = {
-        "title": "operate",
         "extra": "forbid",
-        "json_schema_extra": hide_from_schema(["start_window_idx"]),
+        "frozen": True,
+        "json_schema_extra": hide_from_schema(
+            ["start_window_idx"]
+        ),  # FIXME-remove, config should not be altered by our code
         "revalidate_instances": "always",
         "use_attribute_docstrings": True,
     }
@@ -205,12 +177,6 @@ class BuildOperate(ConfigBaseModel):
 class Build(ConfigBaseModel):
     """Base configuration options used when building a Calliope optimisation problem (`calliope.Model.build`)."""
 
-    model_config = {
-        "title": "build",
-        "extra": "forbid",
-        "revalidate_instances": "always",
-    }
-
     mode: MODES_T = Field(default="plan")
     """Mode in which to run the optimisation."""
 
@@ -267,7 +233,7 @@ class SolveSpores(ConfigBaseModel):
     If False, will consolidate all iterations into one dataset after completion of N iterations (defined by `number`) and save that one dataset.
     """
 
-    save_per_spore_path: Path | None = Field(default=None)
+    save_per_spore_path: str | None = Field(default=None)
     """If saving per spore, the path to save to."""
 
     skip_cost_op: bool = Field(default=False)
@@ -281,21 +247,15 @@ def require_save_per_spore_path(self) -> Self:
                 raise ValueError(
                     "Must define `save_per_spore_path` if you want to save each SPORES result separately."
                 )
-            elif not self.save_per_spore_path.is_dir():
+            elif not Path(self.save_per_spore_path).is_dir():
                 raise ValueError("`save_per_spore_path` must be a directory.")
         return self
 
 
 class Solve(ConfigBaseModel):
     """Base configuration options used when solving a Calliope optimisation problem (`calliope.Model.solve`)."""
 
-    model_config = {
-        "title": "solve",
-        "extra": "forbid",
-        "revalidate_instances": "always",
-    }
-
-    save_logs: Path | None = Field(default=None)
+    save_logs: str | None = Field(default=None)
     """If given, should be a path to a directory in which to save optimisation logs."""
 
     solver_io: str | None = Field(default=None)
@@ -322,7 +282,6 @@ class Solve(ConfigBaseModel):
 class CalliopeConfig(ConfigBaseModel):
     """Calliope configuration class."""
 
-    model_config = {"title": "config"}
     init: Init = Init()
     build: Build = Build()
     solve: Solve = Solve()

src/calliope/model.py

@@ -15,7 +15,6 @@
 from calliope import backend, config, exceptions, io, preprocess
 from calliope.attrdict import AttrDict
 from calliope.postprocess import postprocess as postprocess_results
-from calliope.preprocess.data_tables import DataTable
 from calliope.preprocess.model_data import ModelDataFactory
 from calliope.util.logging import log_time
 from calliope.util.schema import (
@@ -41,7 +40,7 @@ class Model:
     """A Calliope Model."""
 
     _TS_OFFSET = pd.Timedelta(1, unit="nanoseconds")
-    ATTRS_SAVED = ("applied_math", "config")
+    ATTRS_SAVED = ("applied_math", "config", "def_path")
 
     def __init__(
         self,
@@ -76,6 +75,7 @@ def __init__(
         self.defaults: AttrDict
         self.applied_math: preprocess.CalliopeMath
         self.backend: BackendModel
+        self.def_path: str | None = None
         self._is_built: bool = False
         self._is_solved: bool = False
 
@@ -93,7 +93,7 @@ def __init__(
         else:
             if not isinstance(model_definition, dict):
                 # Only file definitions allow relative files.
-                kwargs["def_path"] = str(model_definition)
+                self.def_path = str(model_definition)
             self._init_from_model_definition(
                 model_definition, scenario, override_dict, data_table_dfs, **kwargs
             )
@@ -163,23 +163,23 @@ def _init_from_model_definition(
             comment="Model: preprocessing stage 1 (model_run)",
         )
         model_config = config.CalliopeConfig(**model_def_full.pop("config"))
-        init_config = model_config.init
 
         param_metadata = {"default": extract_from_schema(MODEL_SCHEMA, "default")}
         attributes = {
-            "calliope_version_defined": init_config.calliope_version,
+            "calliope_version_defined": model_config.init.calliope_version,
             "calliope_version_initialised": calliope.__version__,
             "applied_overrides": applied_overrides,
             "scenario": scenario,
             "defaults": param_metadata["default"],
         }
-        data_tables: list[DataTable] = []
-        for table_name, table_dict in model_def_full.pop("data_tables", {}).items():
-            data_tables.append(
-                DataTable(table_name, table_dict, data_table_dfs, init_config.def_path)
-            )
+        # FIXME-config: remove config input once model_def_full uses pydantic
         model_data_factory = ModelDataFactory(
-            init_config, model_def_full, data_tables, attributes, param_metadata
+            model_config.init,
+            model_def_full,
+            self.def_path,
+            data_table_dfs,
+            attributes,
+            param_metadata,
         )
         model_data_factory.build()
 
@@ -192,10 +192,7 @@ def _init_from_model_definition(
             comment="Model: preprocessing stage 2 (model_data)",
         )
 
-        self._model_data.attrs["name"] = init_config.name
-
-        # Unlike at the build and solve phases, we store the init config overrides in the main model config.
-        model_config.init = init_config  # FIXME-config: unnecessary?
+        self._model_data.attrs["name"] = model_config.init.name
         self.config = model_config
 
         log_time(
@@ -220,7 +217,6 @@ def _init_from_model_data(self, model_data: xr.Dataset) -> None:
             )
         if "config" in model_data.attrs:
             self.config = config.CalliopeConfig(**model_data.attrs.pop("config"))
-            self.config.update(model_data.attrs.pop("config_kwarg_overrides"))
 
         self._model_data = model_data
 
@@ -260,26 +256,26 @@ def build(
             comment="Model: backend build starting",
         )
 
-        build_config = self.config.update({"build": kwargs}).build
-        mode = build_config.mode
+        self.config = self.config.update({"build": kwargs})
+        mode = self.config.build.mode
         if mode == "operate":
             if not self._model_data.attrs["allow_operate_mode"]:
                 raise exceptions.ModelError(
                     "Unable to run this model in operate (i.e. dispatch) mode, probably because "
                     "there exist non-uniform timesteps (e.g. from time clustering)"
                 )
-            backend_input = self._prepare_operate_mode_inputs(build_config.operate)
+            backend_input = self._prepare_operate_mode_inputs(self.config.build.operate)
         else:
             backend_input = self._model_data
 
-        init_math_list = [] if build_config.ignore_mode_math else [mode]
+        init_math_list = [] if self.config.build.ignore_mode_math else [mode]
         end_math_list = [] if add_math_dict is None else [add_math_dict]
-        full_math_list = init_math_list + build_config.add_math + end_math_list
+        full_math_list = init_math_list + self.config.build.add_math + end_math_list
         LOGGER.debug(f"Math preprocessing | Loading math: {full_math_list}")
-        model_math = preprocess.CalliopeMath(full_math_list, self.config.init.def_path)
+        model_math = preprocess.CalliopeMath(full_math_list, self.def_path)
 
         self.backend = backend.get_model_backend(
-            build_config, backend_input, model_math
+            self.config.build, backend_input, model_math
         )
         self.backend.add_optimisation_components()
 
@@ -334,24 +330,23 @@ def solve(self, force: bool = False, warmstart: bool = False, **kwargs) -> None:
         else:
             to_drop = []
 
-        solve_config = self.config.update({"solve": kwargs}).solve
-        # FIXME: find a way to avoid overcomplicated passing of settings between modes
-        mode = self.config.update(self.config.applied_keyword_overrides).build.mode
+        self.config = self.config.update({"solve": kwargs})
+
+        shadow_prices = self.config.solve.shadow_prices
+        self.backend.shadow_prices.track_constraints(shadow_prices)
+
+        mode = self.config.build.mode
         self._model_data.attrs["timestamp_solve_start"] = log_time(
             LOGGER,
             self._timings,
             "solve_start",
             comment=f"Optimisation model | starting model in {mode} mode.",
         )
-
-        shadow_prices = solve_config.shadow_prices
-        self.backend.shadow_prices.track_constraints(shadow_prices)
-
         if mode == "operate":
-            results = self._solve_operate(**solve_config.model_dump())
+            results = self._solve_operate(**self.config.solve.model_dump())
         else:
             results = self.backend._solve(
-                warmstart=warmstart, **solve_config.model_dump()
+                warmstart=warmstart, **self.config.solve.model_dump()
             )
 
         log_time(
@@ -393,12 +388,10 @@ def solve(self, force: bool = False, warmstart: bool = False, **kwargs) -> None:
 
         self._is_solved = True
 
-    def run(self, force_rerun=False, **kwargs):
+    def run(self, force_rerun=False):
         """Run the model.
 
         If ``force_rerun`` is True, any existing results will be overwritten.
-
-        Additional kwargs are passed to the backend.
         """
         exceptions.warn(
             "`run()` is deprecated and will be removed in a "
@@ -412,11 +405,12 @@ def to_netcdf(self, path):
         """Save complete model data (inputs and, if available, results) to a NetCDF file at the given `path`."""
         saved_attrs = {}
         for attr in set(self.ATTRS_SAVED) & set(self.__dict__.keys()):
-            if not isinstance(getattr(self, attr), str | list | None):
+            if attr == "config":
+                saved_attrs[attr] = self.config.model_dump()
+            elif not isinstance(getattr(self, attr), str | list | None):
                 saved_attrs[attr] = dict(getattr(self, attr))
             else:
                 saved_attrs[attr] = getattr(self, attr)
-        saved_attrs["config_kwarg_overrides"] = self.config.applied_keyword_overrides
 
         io.save_netcdf(self._model_data, path, **saved_attrs)
 
@@ -509,7 +503,7 @@ def _solve_operate(self, **solver_config) -> xr.Dataset:
         """
         if self.backend.inputs.timesteps[0] != self._model_data.timesteps[0]:
             LOGGER.info("Optimisation model | Resetting model to first time window.")
-            self.build(force=True, **self.config.build.applied_keyword_overrides)
+            self.build(force=True)
 
         LOGGER.info("Optimisation model | Running first time window.")
 
@@ -536,7 +530,7 @@ def _solve_operate(self, **solver_config) -> xr.Dataset:
                     "Optimisation model | Reaching the end of the timeseries. "
                     "Re-building model with shorter time horizon."
                 )
-                build_kwargs = AttrDict(self.config.build.applied_keyword_overrides)
+                build_kwargs = AttrDict()
                 build_kwargs.set_key("operate.start_window_idx", idx + 1)
                 self.build(force=True, **build_kwargs)
             else:

src/calliope/preprocess/data_tables.py

@@ -53,7 +53,7 @@ def __init__(
         table_name: str,
         data_table: DataTableDict,
         data_table_dfs: dict[str, pd.DataFrame] | None = None,
-        model_definition_path: Path = Path("."),
+        model_definition_path: str | Path | None = None,
     ):
         """Load and format a data table from file / in-memory object.