From f3e1d993193cd8607350f5153c168181a9781c97 Mon Sep 17 00:00:00 2001 From: Daniel Heinesen Date: Tue, 18 Jun 2024 11:18:04 +0200 Subject: [PATCH 1/2] First try at fixing col setup without start stop. Model validator for pydating is also removed --- pyaerocom/colocation/colocation_setup.py | 51 ++++++++++++++---------- pyaerocom/colocation/colocator.py | 8 +++- 2 files changed, 36 insertions(+), 23 deletions(-) diff --git a/pyaerocom/colocation/colocation_setup.py b/pyaerocom/colocation/colocation_setup.py index 3203679e0..399d864fa 100644 --- a/pyaerocom/colocation/colocation_setup.py +++ b/pyaerocom/colocation/colocation_setup.py @@ -463,6 +463,36 @@ def validate_no_forbidden_keys(self): for key in self.FORBIDDEN_KEYS: if key in self.model_fields: raise ValidationError + return self + + @model_validator(mode="after") + # @classmethod + def validate_start_stop_xand(self): + if not (self.start and self.stop): + if self.start or self.stop: + raise ValueError("Both start and stop need to be provided or both not provided.") + return self + + # @model_validator(mode="after") + # @classmethod + # def validate_obs_config(cls, v: PyaroConfig): + # if v is not None and cls.obs.config.name != cls.obs_id: + # logger.info( + # f"Data ID in Pyaro config {v.name} does not match obs_id {cls.obs_id}. Setting Pyaro config to None!" + # ) + # v = None + # if v is not None: + # if isinstance(v, dict): + # logger.info("Obs config was given as dict. Will try to convert to PyaroConfig") + # v = PyaroConfig(**v) + # if v.name != cls.obs_id: + # logger.info( + # f"Data ID in Pyaro config {v.name} does not match obs_id {cls.obs_id}. Setting Obs ID to match Pyaro Config!" + # ) + # cls.obs_id = v.name + # if cls.obs_id is None: + # cls.obs_id = v.name + # return v @cached_property def basedir_logfiles(self): @@ -471,27 +501,6 @@ def basedir_logfiles(self): p.mkdir(parents=True, exist_ok=True) return str(p) - @model_validator(mode="after") - @classmethod - def validate_obs_config(cls, v: PyaroConfig): - if v is not None and cls.obs.config.name != cls.obs_id: - logger.info( - f"Data ID in Pyaro config {v.name} does not match obs_id {cls.obs_id}. Setting Pyaro config to None!" - ) - v = None - if v is not None: - if isinstance(v, dict): - logger.info("Obs config was given as dict. Will try to convert to PyaroConfig") - v = PyaroConfig(**v) - if v.name != cls.obs_id: - logger.info( - f"Data ID in Pyaro config {v.name} does not match obs_id {cls.obs_id}. Setting Obs ID to match Pyaro Config!" - ) - cls.obs_id = v.name - if cls.obs_id is None: - cls.obs_id = v.name - return v - def add_glob_meta(self, **kwargs): """ Add global metadata to :attr:`add_meta` diff --git a/pyaerocom/colocation/colocator.py b/pyaerocom/colocation/colocator.py index c3e5283fd..ce6ea960d 100644 --- a/pyaerocom/colocation/colocator.py +++ b/pyaerocom/colocation/colocator.py @@ -914,8 +914,13 @@ def _infer_start_stop_yr_from_model_reader(self): self.stop = last def _check_set_start_stop(self): - if self.colocation_setup.start is None: + if self.colocation_setup.start is None and self.colocation_setup.stop is None: self._infer_start_stop_yr_from_model_reader() + self.start, self.stop = start_stop(self.start, self.stop) + else: + self.start, self.stop = start_stop( + self.colocation_setup.start, self.colocation_setup.stop + ) if self.colocation_setup.model_use_climatology: if self.colocation_setup.stop is not None or not isinstance( self.colocation_setup.start, int @@ -925,7 +930,6 @@ def _check_set_start_stop(self): 'climatology fields, please specify "start" as integer ' 'denoting the year, and set "stop"=None' ) - self.start, self.stop = start_stop(self.colocation_setup.start, self.colocation_setup.stop) def _coldata_savename(self, obs_var, mod_var, ts_type, **kwargs): """Get filename of colocated data file for saving""" From 160a794122b4252f4e7cfd55f6aed2661ec18769 Mon Sep 17 00:00:00 2001 From: Daniel Heinesen Date: Tue, 18 Jun 2024 13:21:01 +0200 Subject: [PATCH 2/2] Makes some more tests works --- pyaerocom/colocation/colocation_setup.py | 44 +++++++++++++----------- tests/colocation/test_colocator.py | 2 ++ 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/pyaerocom/colocation/colocation_setup.py b/pyaerocom/colocation/colocation_setup.py index 399d864fa..c30e83276 100644 --- a/pyaerocom/colocation/colocation_setup.py +++ b/pyaerocom/colocation/colocation_setup.py @@ -463,6 +463,7 @@ def validate_no_forbidden_keys(self): for key in self.FORBIDDEN_KEYS: if key in self.model_fields: raise ValidationError + return self @model_validator(mode="after") @@ -471,28 +472,29 @@ def validate_start_stop_xand(self): if not (self.start and self.stop): if self.start or self.stop: raise ValueError("Both start and stop need to be provided or both not provided.") - return self - # @model_validator(mode="after") - # @classmethod - # def validate_obs_config(cls, v: PyaroConfig): - # if v is not None and cls.obs.config.name != cls.obs_id: - # logger.info( - # f"Data ID in Pyaro config {v.name} does not match obs_id {cls.obs_id}. Setting Pyaro config to None!" - # ) - # v = None - # if v is not None: - # if isinstance(v, dict): - # logger.info("Obs config was given as dict. Will try to convert to PyaroConfig") - # v = PyaroConfig(**v) - # if v.name != cls.obs_id: - # logger.info( - # f"Data ID in Pyaro config {v.name} does not match obs_id {cls.obs_id}. Setting Obs ID to match Pyaro Config!" - # ) - # cls.obs_id = v.name - # if cls.obs_id is None: - # cls.obs_id = v.name - # return v + # return self + + @model_validator(mode="after") + @classmethod + def validate_obs_config(cls, v: PyaroConfig): + if v is not None and cls.obs.config.name != cls.obs_id: + logger.info( + f"Data ID in Pyaro config {v.name} does not match obs_id {cls.obs_id}. Setting Pyaro config to None!" + ) + v = None + if v is not None: + if isinstance(v, dict): + logger.info("Obs config was given as dict. Will try to convert to PyaroConfig") + v = PyaroConfig(**v) + if v.name != cls.obs_id: + logger.info( + f"Data ID in Pyaro config {v.name} does not match obs_id {cls.obs_id}. Setting Obs ID to match Pyaro Config!" + ) + cls.obs_id = v.name + if cls.obs_id is None: + cls.obs_id = v.name + return v @cached_property def basedir_logfiles(self): diff --git a/tests/colocation/test_colocator.py b/tests/colocation/test_colocator.py index 1431ffd7d..17c8450c5 100644 --- a/tests/colocation/test_colocator.py +++ b/tests/colocation/test_colocator.py @@ -69,6 +69,7 @@ def setup(): obs_id="AeronetSunV3L2Subset.daily", obs_vars="od550aer", start=2010, + stop=2011, raise_exceptions=True, reanalyse_existing=True, ) @@ -180,6 +181,7 @@ def test_Colocator__coldata_savename(setup): setup["model_name"] = "model" setup["filter_name"] = ALL_REGION_NAME setup["start"] = 2015 + setup["stop"] = 2016 col_stp = ColocationSetup(**setup) col = Colocator(col_stp) col._check_set_start_stop()