Merge pull request #1360 from metno/generalize-model_naming_convention

Pydantic ObsEntry

pyaerocom/aeroval/_processing_base.py

@@ -79,29 +79,6 @@ class HasColocator(HasConfig):
     Config class that also has the ability to co-locate
     """
 
-    def _get_diurnal_only(self, obs_name):
-        """
-        Check if colocated data is flagged for only diurnal processing
-
-        Parameters
-        ----------
-        obs_name : string
-            Name of observational subset
-        colocated_data : ColocatedData
-            A ColocatedData object that will be checked for suitability of
-            diurnal processing.
-
-        Returns
-        -------
-        diurnal_only : bool
-        """
-        entry = self.cfg.get_obs_entry(obs_name)
-        try:
-            diurnal_only = entry["diurnal_only"]
-        except KeyError:
-            diurnal_only = False
-        return diurnal_only
-
     def get_colocator(self, model_name: str = None, obs_name: str = None) -> Colocator:
         """
         Instantiate colocation engine
@@ -130,7 +107,7 @@ def get_colocator(self, model_name: str = None, obs_name: str = None) -> Colocat
             mod_cfg = self.cfg.get_model_entry(model_name)
             col_cfg["model_cfg"] = mod_cfg
 
-            # LB: Hack and at what lowlevel_helpers's import_from was doing
+            # Hack and at what lowlevel_helpers's import_from was doing
             for key, val in mod_cfg.items():
                 if key in ColocationSetup.model_fields:
                     col_cfg[key] = val
@@ -139,12 +116,12 @@ def get_colocator(self, model_name: str = None, obs_name: str = None) -> Colocat
             pyaro_config = obs_cfg["obs_config"] if "obs_config" in obs_cfg else None
             col_cfg["obs_config"] = pyaro_config
 
-            # LB: Hack and at what lowlevel_helpers's import_from was doing
-            for key, val in obs_cfg.items():
+            # Hack and at what lowlevel_helpers's import_from was doing
+            for key, val in obs_cfg.model_dump().items():
                 if key in ColocationSetup.model_fields:
                     col_cfg[key] = val
 
-            col_cfg["add_meta"].update(diurnal_only=self._get_diurnal_only(obs_name))
+            col_cfg["add_meta"].update(diurnal_only=self.cfg.get_obs_entry(obs_name).diurnal_only)
 
         col_stp = ColocationSetup(**col_cfg)
         col = Colocator(col_stp)

pyaerocom/aeroval/collections.py

@@ -11,8 +11,9 @@ class BaseCollection(BrowseDict, abc.ABC):
     #: maximum length of entry names
     MAXLEN_KEYS = 25
     #: Invalid chars in entry names
-    FORBIDDEN_CHARS_KEYS = ["_"]
+    FORBIDDEN_CHARS_KEYS = []
 
+    # TODO: Wait a few release cycles after v0.23.0 and see if this can be removed
     def _check_entry_name(self, key):
         if any([x in key for x in self.FORBIDDEN_CHARS_KEYS]):
             raise EvalEntryNameError(
@@ -22,8 +23,6 @@ def _check_entry_name(self, key):
 
     def __setitem__(self, key, value):
         self._check_entry_name(key)
-        if "web_interface_name" in value:
-            self._check_entry_name(value["web_interface_name"])
         super().__setitem__(key, value)
 
     def keylist(self, name_or_pattern: str = None) -> list:
@@ -70,7 +69,7 @@ def get_entry(self, key) -> object:
 
     @property
     @abc.abstractmethod
-    def web_iface_names(self) -> list:
+    def web_interface_names(self) -> list:
         """
         List of webinterface names for
         """
@@ -107,7 +106,7 @@ def get_entry(self, key) -> object:
         """
         try:
             entry = self[key]
-            entry["obs_name"] = self.get_web_iface_name(key)
+            entry.obs_name = self.get_web_interface_name(key)
             return entry
         except (KeyError, AttributeError):
             raise EntryNotAvailable(f"no such entry {key}")
@@ -127,7 +126,7 @@ def get_all_vars(self) -> list[str]:
             vars.extend(ocfg.get_all_vars())
         return sorted(list(set(vars)))
 
-    def get_web_iface_name(self, key):
+    def get_web_interface_name(self, key):
         """
         Get webinterface name for entry
 
@@ -148,26 +147,23 @@ def get_web_iface_name(self, key):
             corresponding name
 
         """
-        entry = self[key]
-        if "web_interface_name" not in entry:
-            return key
-        return entry["web_interface_name"]
+        return self[key].web_interface_name if self[key].web_interface_name is not None else key
 
     @property
-    def web_iface_names(self) -> list:
+    def web_interface_names(self) -> list:
         """
         List of web interface names for each obs entry
 
         Returns
         -------
         list
         """
-        return [self.get_web_iface_name(key) for key in self.keylist()]
+        return [self.get_web_interface_name(key) for key in self.keylist()]
 
     @property
     def all_vert_types(self):
         """List of unique vertical types specified in this collection"""
-        return list({x["obs_vert_type"] for x in self.values()})
+        return list({x.obs_vert_type for x in self.values()})
 
 
 class ModelCollection(BaseCollection):
@@ -224,7 +220,7 @@ def get_entry(self, key) -> ModelEntry:
             raise EntryNotAvailable(f"no such entry {key}")
 
     @property
-    def web_iface_names(self) -> list:
+    def web_interface_names(self) -> list:
         """
         List of web interface names for each obs entry
 

pyaerocom/aeroval/experiment_output.py

@@ -425,7 +425,7 @@ def _check_clean_ts_file(self, fp) -> bool:
                 return True
 
             models_avail = list(data)
-            models_in_exp = self.cfg.model_cfg.web_iface_names
+            models_in_exp = self.cfg.model_cfg.web_interface_names
             if all([mod in models_in_exp for mod in models_avail]):
                 # nothing to clean up
                 return False
@@ -512,7 +512,7 @@ def get_model_order_menu(self) -> list:
                 )
             order.extend(self.cfg.webdisp_opts.model_order_menu)
         elif self.cfg.webdisp_opts.obsorder_from_config:
-            order.extend(self.cfg.model_cfg.web_iface_names)
+            order.extend(self.cfg.model_cfg.web_interface_names)
         return order
 
     def get_obs_order_menu(self) -> list:
@@ -526,7 +526,7 @@ def get_obs_order_menu(self) -> list:
                 )
             order.extend(self.cfg.webdisp_opts.obs_order_menu)
         elif self.cfg.webdisp_opts.obsorder_from_config:
-            order.extend(self.cfg.obs_cfg.web_iface_names)
+            order.extend(self.cfg.obs_cfg.web_interface_names)
         return order
 
     def _get_json_output_files(self, dirname) -> list[str]:
@@ -752,7 +752,7 @@ def _is_part_of_experiment(self, obs_name, obs_var, mod_name, mod_var) -> bool:
         allobs = self.cfg.obs_cfg
         obs_matches = []
         for key, ocfg in allobs.items():
-            if obs_name == allobs.get_web_iface_name(key):
+            if obs_name == allobs.get_web_interface_name(key):
                 obs_matches.append(ocfg)
         if len(obs_matches) == 0:
             self._invalid["obs"].append(obs_name)

pyaerocom/aeroval/experiment_processor.py

@@ -34,7 +34,7 @@ def _run_single_entry(self, model_name, obs_name, var_list):
             logger.info(msg)
             return
         ocfg = self.cfg.get_obs_entry(obs_name)
-        if ocfg["is_superobs"]:
+        if ocfg.is_superobs:
             try:
                 engine = SuperObsEngine(self.cfg)
                 engine.run(
@@ -47,19 +47,19 @@ def _run_single_entry(self, model_name, obs_name, var_list):
                 if self.raise_exceptions:
                     raise
                 logger.warning("failed to process superobs...")
-        elif ocfg["only_superobs"]:
+        elif ocfg.only_superobs:
             logger.info(
                 f"Skipping json processing of {obs_name}, as this is "
                 f"marked to be used only as part of a superobs "
                 f"network"
             )
-        elif ocfg["only_json"]:
-            if not ocfg["coldata_dir"]:
+        elif ocfg.only_json:
+            if not ocfg.coldata_dir:
                 raise Exception(
                     "No coldata_dir provided for an obs network for which only_json=True. The assumption of setting only_json=True is that colocated files already exist, and so a directory for these files must be provided."
                 )
             else:
-                preprocessed_coldata_dir = ocfg["coldata_dir"]
+                preprocessed_coldata_dir = ocfg.coldata_dir
                 mask = f"{preprocessed_coldata_dir}/{model_name}/*.nc"
                 files_to_convert = glob.glob(mask)
                 engine = ColdataToJsonEngine(self.cfg)
@@ -69,7 +69,7 @@ def _run_single_entry(self, model_name, obs_name, var_list):
             # If a var_list is given, only run on the obs networks which contain that variable
             if var_list:
                 var_list_asked = var_list
-                obs_vars = ocfg["obs_vars"]
+                obs_vars = ocfg.obs_vars
                 var_list = list(set(obs_vars) & set(var_list))
                 if not var_list:
                     logger.warning(

pyaerocom/aeroval/helpers.py

@@ -162,7 +162,7 @@ def make_dummy_model(obs_list: list, cfg) -> str:
     tmp_var_obj = Variable()
     # Loops over variables in obs
     for obs in obs_list:
-        for var in cfg.obs_cfg[obs]["obs_vars"]:
+        for var in cfg.obs_cfg[obs].obs_vars:
             # Create dummy cube
 
             dummy_cube = make_dummy_cube(var, start_yr=start, stop_yr=stop, freq=freq)
@@ -185,7 +185,7 @@ def make_dummy_model(obs_list: list, cfg) -> str:
             for dummy_grid_yr in yr_gen:
                 # Add to netcdf
                 yr = dummy_grid_yr.years_avail()[0]
-                vert_code = cfg.obs_cfg[obs]["obs_vert_type"]
+                vert_code = cfg.obs_cfg[obs].obs_vert_type
 
                 save_name = dummy_grid_yr.aerocom_savename(model_id, var, vert_code, yr, freq)
                 dummy_grid_yr.to_netcdf(outdir, savename=save_name)