From 61854163dfa71fd6463c660211a5b3b137595671 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?thorbjoernl=20=28Thorbj=C3=B8rn=29?= Date: Mon, 8 Jul 2024 14:09:32 +0000 Subject: [PATCH 1/7] fix: mda8 --- pyaerocom/aeroval/experiment_output.py | 3 +++ pyaerocom/aeroval/glob_defaults.py | 1 + pyaerocom/colocation/colocator.py | 10 +++++++--- pyaerocom/data/variables.ini | 4 ++++ pyaerocom/stats/mda8/mda8.py | 8 ++++++++ 5 files changed, 23 insertions(+), 3 deletions(-) diff --git a/pyaerocom/aeroval/experiment_output.py b/pyaerocom/aeroval/experiment_output.py index 7c975b46f..75226b8f3 100644 --- a/pyaerocom/aeroval/experiment_output.py +++ b/pyaerocom/aeroval/experiment_output.py @@ -21,6 +21,7 @@ from pyaerocom.aeroval.setupclasses import EvalSetup from pyaerocom.aeroval.varinfo_web import VarinfoWeb from pyaerocom.exceptions import EntryNotAvailable, VariableDefinitionError +from pyaerocom.stats.mda8.const import MDA_VARS from pyaerocom.stats.stats import _init_stats_dummy from pyaerocom.variable_helpers import get_aliases @@ -698,6 +699,8 @@ def _is_part_of_experiment(self, obs_name, obs_var, mod_name, mod_var) -> bool: True if this combination is valid, else False. """ + # if obs_var in ["vmro3mda8"] and mod_var in ["vmro3mda8"]: + # return True # get model entry for model name try: mcfg = self.cfg.model_cfg.get_entry(mod_name) diff --git a/pyaerocom/aeroval/glob_defaults.py b/pyaerocom/aeroval/glob_defaults.py index ff3aca270..28c1d87d5 100644 --- a/pyaerocom/aeroval/glob_defaults.py +++ b/pyaerocom/aeroval/glob_defaults.py @@ -629,6 +629,7 @@ vmrso2=["SO2", "3D", "Gas volume mixing ratio"], concso4=["SO4", "3D", "Particle concentrations"], vmro3=["O3", "3D", "Volume mixing ratios"], + vmro3mda8=["O3 (MDA8)", "3D", "Valume mixing ratios"], vmro3max=["O3Max", "3D", "Volume mixing ratios"], vmrox=["OX", "3D", "Gas volume mixing ratio"], concco=["CO", "3D", "Particle concentration"], diff --git a/pyaerocom/colocation/colocator.py b/pyaerocom/colocation/colocator.py index 444a38a70..cd783c94d 100644 --- a/pyaerocom/colocation/colocator.py +++ b/pyaerocom/colocation/colocator.py @@ -384,16 +384,20 @@ def run(self, var_list: list = None): data_out[mod_var][obs_var] = coldata if obs_var in MDA_VARS: - mda8 = None try: mda8 = mda8_colocated_data( coldata, obs_var=f"{obs_var}mda8", mod_var=f"{mod_var}mda8" ) - except ValueError: + except ValueError as e: logger.warning( "Tried calculating mda8 for [%s, %s], but failed.", obs_var, mod_var ) - finally: + logger.error(e) + else: + self._save_coldata(mda8) + logger.warning( + "Successfully calculated mda8 for [%s, %s].", obs_var, mod_var + ) data_out[f"{mod_var}mda8"][f"{obs_var}mda8"] = mda8 self._processing_status.append([mod_var, obs_var, 1]) diff --git a/pyaerocom/data/variables.ini b/pyaerocom/data/variables.ini index 4d1477281..047db9f41 100644 --- a/pyaerocom/data/variables.ini +++ b/pyaerocom/data/variables.ini @@ -3105,6 +3105,10 @@ unit = ug m-3 description=Mass concentration of ozone unit = ug m-3 +[vmro3mda8] +description=MDA8 of o3 VMR. +unit = nmol mol-1 + [concco] description=Mass concentration of organic carbon unit = ug m-3 diff --git a/pyaerocom/stats/mda8/mda8.py b/pyaerocom/stats/mda8/mda8.py index 471708fae..924d7d9e4 100644 --- a/pyaerocom/stats/mda8/mda8.py +++ b/pyaerocom/stats/mda8/mda8.py @@ -1,8 +1,12 @@ +import logging + import numpy as np import xarray as xr from pyaerocom.colocation.colocated_data import ColocatedData +logger = logging.getLogger(__name__) + def min_periods_max(x: np.ndarray, /, min_periods=1) -> float: """Calculates the max of a 1-dimensional array, returning @@ -30,20 +34,24 @@ def mda8_colocated_data(coldat: ColocatedData, /, obs_var: str, mod_var: str) -> :return: Colocated data object containing """ if not isinstance(coldat, ColocatedData): + logger.warning(f"Unexpected data type {type(coldat)}, expected ColocatedData.") raise ValueError(f"Unexpected type {type(coldat)}. Expected ColocatedData") if coldat.ts_type != "hourly": + logger.warning(f"Unexpected ts_type {coldat.ts_type}, expected 'hourly'.") raise ValueError(f"Expected hourly timeseries. Got {coldat.ts_type}.") # TODO: Currently order of dims matter in the implementation, so this check is # stricter than it probably should be. if coldat.dims != ("data_source", "time", "station_name"): + logger.info(f"Unexpected dimensions, {coldat.dims}.") raise ValueError( f"Unexpected dimensions. Got {coldat.dims}, expected ['data_source', 'time', 'station_name']." ) cd = ColocatedData(_calc_mda8(coldat.data)) cd.data.attrs["var_name"] = [obs_var, mod_var] + cd.metadata["var_name_input"] = [obs_var, mod_var] return cd From 83be0b5c69245aba5142c4cd8bc593aebc82f971 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?thorbjoernl=20=28Thorbj=C3=B8rn=29?= Date: Tue, 9 Jul 2024 07:36:45 +0000 Subject: [PATCH 2/7] . --- pyaerocom/aeroval/experiment_output.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyaerocom/aeroval/experiment_output.py b/pyaerocom/aeroval/experiment_output.py index 75226b8f3..f4219fd49 100644 --- a/pyaerocom/aeroval/experiment_output.py +++ b/pyaerocom/aeroval/experiment_output.py @@ -699,8 +699,8 @@ def _is_part_of_experiment(self, obs_name, obs_var, mod_name, mod_var) -> bool: True if this combination is valid, else False. """ - # if obs_var in ["vmro3mda8"] and mod_var in ["vmro3mda8"]: - # return True + if obs_var in ["vmro3mda8"] and mod_var in ["vmro3mda8"]: + return True # get model entry for model name try: mcfg = self.cfg.model_cfg.get_entry(mod_name) From 33a2a007945d2ad507e1f9b2a7970897aa94fea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?thorbjoernl=20=28Thorbj=C3=B8rn=29?= Date: Tue, 9 Jul 2024 07:57:29 +0000 Subject: [PATCH 3/7] fix: Log message level --- pyaerocom/colocation/colocator.py | 5 +---- pyaerocom/stats/mda8/mda8.py | 3 --- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/pyaerocom/colocation/colocator.py b/pyaerocom/colocation/colocator.py index cd783c94d..5f9553ad5 100644 --- a/pyaerocom/colocation/colocator.py +++ b/pyaerocom/colocation/colocator.py @@ -389,10 +389,7 @@ def run(self, var_list: list = None): coldata, obs_var=f"{obs_var}mda8", mod_var=f"{mod_var}mda8" ) except ValueError as e: - logger.warning( - "Tried calculating mda8 for [%s, %s], but failed.", obs_var, mod_var - ) - logger.error(e) + logger.debug(e) else: self._save_coldata(mda8) logger.warning( diff --git a/pyaerocom/stats/mda8/mda8.py b/pyaerocom/stats/mda8/mda8.py index 924d7d9e4..c7a63fd9b 100644 --- a/pyaerocom/stats/mda8/mda8.py +++ b/pyaerocom/stats/mda8/mda8.py @@ -34,17 +34,14 @@ def mda8_colocated_data(coldat: ColocatedData, /, obs_var: str, mod_var: str) -> :return: Colocated data object containing """ if not isinstance(coldat, ColocatedData): - logger.warning(f"Unexpected data type {type(coldat)}, expected ColocatedData.") raise ValueError(f"Unexpected type {type(coldat)}. Expected ColocatedData") if coldat.ts_type != "hourly": - logger.warning(f"Unexpected ts_type {coldat.ts_type}, expected 'hourly'.") raise ValueError(f"Expected hourly timeseries. Got {coldat.ts_type}.") # TODO: Currently order of dims matter in the implementation, so this check is # stricter than it probably should be. if coldat.dims != ("data_source", "time", "station_name"): - logger.info(f"Unexpected dimensions, {coldat.dims}.") raise ValueError( f"Unexpected dimensions. Got {coldat.dims}, expected ['data_source', 'time', 'station_name']." ) From 201636db8ed8b2a45b9c847881909a9884b53de7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?thorbjoernl=20=28Thorbj=C3=B8rn=29?= Date: Tue, 9 Jul 2024 08:05:41 +0000 Subject: [PATCH 4/7] warning->info --- pyaerocom/colocation/colocator.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyaerocom/colocation/colocator.py b/pyaerocom/colocation/colocator.py index 5f9553ad5..794802028 100644 --- a/pyaerocom/colocation/colocator.py +++ b/pyaerocom/colocation/colocator.py @@ -392,9 +392,7 @@ def run(self, var_list: list = None): logger.debug(e) else: self._save_coldata(mda8) - logger.warning( - "Successfully calculated mda8 for [%s, %s].", obs_var, mod_var - ) + logger.info("Successfully calculated mda8 for [%s, %s].", obs_var, mod_var) data_out[f"{mod_var}mda8"][f"{obs_var}mda8"] = mda8 self._processing_status.append([mod_var, obs_var, 1]) From dcb95c1edc032a4bb1020d0fa232224fa39f555b Mon Sep 17 00:00:00 2001 From: Lewis Blake Date: Tue, 9 Jul 2024 10:40:53 +0200 Subject: [PATCH 5/7] review fix up --- pyaerocom/aeroval/experiment_output.py | 4 ++-- pyaerocom/colocation/colocator.py | 4 ++-- pyaerocom/data/variables.ini | 2 +- pyaerocom/stats/mda8/const.py | 3 ++- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pyaerocom/aeroval/experiment_output.py b/pyaerocom/aeroval/experiment_output.py index f4219fd49..3331b2c00 100644 --- a/pyaerocom/aeroval/experiment_output.py +++ b/pyaerocom/aeroval/experiment_output.py @@ -21,7 +21,7 @@ from pyaerocom.aeroval.setupclasses import EvalSetup from pyaerocom.aeroval.varinfo_web import VarinfoWeb from pyaerocom.exceptions import EntryNotAvailable, VariableDefinitionError -from pyaerocom.stats.mda8.const import MDA_VARS +from pyaerocom.stats.mda8.const import MDA_OUTPUT_VARS from pyaerocom.stats.stats import _init_stats_dummy from pyaerocom.variable_helpers import get_aliases @@ -699,7 +699,7 @@ def _is_part_of_experiment(self, obs_name, obs_var, mod_name, mod_var) -> bool: True if this combination is valid, else False. """ - if obs_var in ["vmro3mda8"] and mod_var in ["vmro3mda8"]: + if obs_var in MDA_OUTPUT_VARS and mod_var in MDA_OUTPUT_VARS: return True # get model entry for model name try: diff --git a/pyaerocom/colocation/colocator.py b/pyaerocom/colocation/colocator.py index 794802028..6993ece3a 100644 --- a/pyaerocom/colocation/colocator.py +++ b/pyaerocom/colocation/colocator.py @@ -31,7 +31,7 @@ from pyaerocom.io import ReadCAMS2_83, ReadGridded, ReadUngridded from pyaerocom.io.helpers import get_all_supported_ids_ungridded from pyaerocom.io.mscw_ctm.reader import ReadMscwCtm -from pyaerocom.stats.mda8.const import MDA_VARS +from pyaerocom.stats.mda8.const import MDA8_INPUT_VARS from pyaerocom.stats.mda8.mda8 import mda8_colocated_data from .colocated_data import ColocatedData @@ -383,7 +383,7 @@ def run(self, var_list: list = None): ) # note this can be ColocatedData or ColocatedDataLists data_out[mod_var][obs_var] = coldata - if obs_var in MDA_VARS: + if obs_var in MDA8_INPUT_VARS: try: mda8 = mda8_colocated_data( coldata, obs_var=f"{obs_var}mda8", mod_var=f"{mod_var}mda8" diff --git a/pyaerocom/data/variables.ini b/pyaerocom/data/variables.ini index 047db9f41..23dd82cb5 100644 --- a/pyaerocom/data/variables.ini +++ b/pyaerocom/data/variables.ini @@ -3106,7 +3106,7 @@ description=Mass concentration of ozone unit = ug m-3 [vmro3mda8] -description=MDA8 of o3 VMR. +description=MDA8 of O3 VMR. unit = nmol mol-1 [concco] diff --git a/pyaerocom/stats/mda8/const.py b/pyaerocom/stats/mda8/const.py index 8c32ac409..e0e6dc2fd 100644 --- a/pyaerocom/stats/mda8/const.py +++ b/pyaerocom/stats/mda8/const.py @@ -1,2 +1,3 @@ # Variable names for which mda8 values are to be calculated. -MDA_VARS = ("conco3", "vmro3") +MDA8_INPUT_VARS = ("conco3", "vmro3") +MDA8_OUTPUT_VARS = ("vmro3mda8",) From 6be91d48b0fffdffad46891c970bab1dfcda0732 Mon Sep 17 00:00:00 2001 From: Lewis Blake Date: Tue, 9 Jul 2024 10:42:11 +0200 Subject: [PATCH 6/7] Should be MDA8 --- pyaerocom/aeroval/experiment_output.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyaerocom/aeroval/experiment_output.py b/pyaerocom/aeroval/experiment_output.py index 3331b2c00..39111eb0b 100644 --- a/pyaerocom/aeroval/experiment_output.py +++ b/pyaerocom/aeroval/experiment_output.py @@ -21,7 +21,7 @@ from pyaerocom.aeroval.setupclasses import EvalSetup from pyaerocom.aeroval.varinfo_web import VarinfoWeb from pyaerocom.exceptions import EntryNotAvailable, VariableDefinitionError -from pyaerocom.stats.mda8.const import MDA_OUTPUT_VARS +from pyaerocom.stats.mda8.const import MDA8_OUTPUT_VARS from pyaerocom.stats.stats import _init_stats_dummy from pyaerocom.variable_helpers import get_aliases @@ -699,7 +699,7 @@ def _is_part_of_experiment(self, obs_name, obs_var, mod_name, mod_var) -> bool: True if this combination is valid, else False. """ - if obs_var in MDA_OUTPUT_VARS and mod_var in MDA_OUTPUT_VARS: + if obs_var in MDA8_OUTPUT_VARS and mod_var in MDA8_OUTPUT_VARS: return True # get model entry for model name try: From 2dad9c7890096535c9ae4ad23915150e1198ad99 Mon Sep 17 00:00:00 2001 From: Lewis Blake Date: Tue, 9 Jul 2024 10:55:55 +0200 Subject: [PATCH 7/7] comment why MDA8 output is specical case --- pyaerocom/aeroval/experiment_output.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyaerocom/aeroval/experiment_output.py b/pyaerocom/aeroval/experiment_output.py index 39111eb0b..cffdab848 100644 --- a/pyaerocom/aeroval/experiment_output.py +++ b/pyaerocom/aeroval/experiment_output.py @@ -21,7 +21,7 @@ from pyaerocom.aeroval.setupclasses import EvalSetup from pyaerocom.aeroval.varinfo_web import VarinfoWeb from pyaerocom.exceptions import EntryNotAvailable, VariableDefinitionError -from pyaerocom.stats.mda8.const import MDA8_OUTPUT_VARS +from pyaerocom.stats.mda8.const import MDA8_INPUT_VARS, MDA8_OUTPUT_VARS from pyaerocom.stats.stats import _init_stats_dummy from pyaerocom.variable_helpers import get_aliases @@ -699,8 +699,12 @@ def _is_part_of_experiment(self, obs_name, obs_var, mod_name, mod_var) -> bool: True if this combination is valid, else False. """ + + # MDA8 is computed on-the-fly ONLY if a MDA8_INPUT_VAR at hourly freq is detected. + # Consequently, it is not specified in a config but should be included as part of the experiment. if obs_var in MDA8_OUTPUT_VARS and mod_var in MDA8_OUTPUT_VARS: return True + # get model entry for model name try: mcfg = self.cfg.model_cfg.get_entry(mod_name)