From 81c3d970a23816bf7be4d04bc9c22e475560f8a5 Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Mon, 14 Oct 2024 15:42:28 +0200 Subject: [PATCH 01/14] add ratpm10pm25 to EMEP calculated variables --- .../aeroval/config/ratpm10pm25/__init__.py | 0 .../aeroval/config/ratpm10pm25/base_config.py | 600 ++++++++++++++++++ pyaerocom/io/mscw_ctm/additional_variables.py | 22 + pyaerocom/io/mscw_ctm/reader.py | 6 +- 4 files changed, 626 insertions(+), 2 deletions(-) create mode 100644 pyaerocom/aeroval/config/ratpm10pm25/__init__.py create mode 100644 pyaerocom/aeroval/config/ratpm10pm25/base_config.py diff --git a/pyaerocom/aeroval/config/ratpm10pm25/__init__.py b/pyaerocom/aeroval/config/ratpm10pm25/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyaerocom/aeroval/config/ratpm10pm25/base_config.py b/pyaerocom/aeroval/config/ratpm10pm25/base_config.py new file mode 100644 index 000000000..0d7beac41 --- /dev/null +++ b/pyaerocom/aeroval/config/ratpm10pm25/base_config.py @@ -0,0 +1,600 @@ +""" +Global config for ratio pm10 vs pm25 +""" + +import copy +import logging +import os + +logger = logging.getLogger(__name__) + +# Constraints +DEFAULT_RESAMPLE_CONSTRAINTS = dict( + yearly=dict(monthly=9), + monthly=dict( + daily=21, + weekly=3, + ), + daily=dict(hourly=18), +) + +DEFAULT_RESAMPLE_CONSTRAINTS_DAILY = dict( + daily=dict(hourly=18), +) + +# ODCSFUN_EEANRT = "EEAAQeRep.NRT;concpm10/EEAAQeRep.NRT;concpm25" +ODCSFUN_EEAV2 = "EEAAQeRep.v2;concpm10/EEAAQeRep.v2;concpm25" +ODCSFUN_EBAS = "EBASMC;concpm10/EBASMC;concpm25" + + +def get_CFG(reportyear, year, model_dir) -> dict: + """create aeroval configuration dict to run the variable + ratpm10pm25 (ratio pm10 vspm25) + + :returns: a dict of a model configuration usable for EvalSetup + """ + # get current path for reference to local gridded_io_aux.py + base_conf_path = os.path.dirname(__file__) + + CFG = dict( + json_basedir=os.path.abspath("/home/jang/data/aeroval-local-web/data"), + coldata_basedir=os.path.abspath("/home/jang/data/aeroval-local-web/coldata"), + # io_aux_file=os.path.abspath("/home/jang/data/aeroval-local-web/gridded_io_aux.py"), not needed for ReadMscwCtm + # io_aux_file=os.path.join(base_conf_path, "gridded_io_aux.py"), + # var_scale_colmap_file=os.path.abspath( + # "/home/jang/data/aeroval-local-web/pyaerocom-config/config_files/CAMEO/user_var_scale_colmap.ini" + # ), + # if True, existing colocated data files will be deleted and contours will be overwritten + reanalyse_existing=True, + only_json=False, + add_model_maps=True, + only_model_maps=False, + modelmaps_opts=dict(maps_freq="yearly", maps_res_deg=5), + clear_existing_json=False, + # if True, the analysis will stop whenever an error occurs (else, errors that + # occurred will be written into the logfiles) + raise_exceptions=True, + # Regional filter for analysis + filter_name="ALL-wMOUNTAINS", + # colocation frequency (no statistics in higher resolution can be computed) + ts_type="daily", + map_zoom="Europe", + freqs=["yearly", "monthly", "weekly", "daily", "hourly"], + periods=[f"{year}"], + main_freq="daily", + zeros_to_nan=False, + use_diurnal=True, + min_num_obs=DEFAULT_RESAMPLE_CONSTRAINTS, + colocate_time=True, + resample_how={"vmro3max": {"daily": {"hourly": "max"}}}, + obs_remove_outliers=False, + model_remove_outliers=False, + harmonise_units=True, + regions_how="country", + annual_stats_constrained=True, + proj_id="emep", + exp_id=f"{reportyear}-reporting", + exp_name=f"Evaluation of EMEP runs for {reportyear} EMEP reporting", + exp_descr=( + f"Evaluation of EMEP runs for {reportyear} EMEP reporting. The EMEP model, simulated for {year}, is compared against observations from EEA and EBAS." + ), + exp_pi="emep.mscw@met.no", + public=True, + # directory where colocated data files are supposed to be stored + weighted_stats=True, + var_order_menu=[ + # Gases + "ratpm10pm25", + "concNno", + "concNno2", + "concNtno3", + "concNhno3", + "concNtnh", + "concNnh3", + "concnh4", + "concSso2", + "concso4t", + "concso4c", + "vmro3", + "vmro3max", + "vmro3mda8", + "vmrox", + "vmrco", + # PMs + "concpm10", + "concpm25", + "concno3pm10", + "concno3pm25", + "concnh4pm25", + "concso4pm25", + "concCecpm10", + "concCecpm25", + "concCocpm10", # SURF_ugC_PM_OMCOARSE missing in model-output + "concCocpm25", + "concsspm10", + "concsspm25", + # Depositions + "wetrdn", + "wetoxs", + "wetoxn", + "prmm", + ], + ) + + CFG["model_cfg"] = { + "EMEP.cameo": dict( + model_id="EMEP,", + model_data_dir=model_dir, + # gridded_reader_id={"model": "ReadMscwCtm"}, + # model_read_aux={}, + # model_ts_type_read="daily", + ), + } + + """ + Filters + """ + + # OBS SPECIFIC FILTERS (combination of the above and more) + EEA_RURAL_FILTER = { + "station_classification": ["background"], + "area_classification": [ + "rural", + "rural-nearcity", + "rural-regional", + "rural-remote", + ], + } + + BASE_FILTER = { + "latitude": [30, 82], + "longitude": [-30, 90], + } + + EBAS_FILTER = { + **BASE_FILTER, + "data_level": [None, 2], + "set_flags_nan": True, + } + + EEA_FILTER = { + **BASE_FILTER, + **EEA_RURAL_FILTER, + } + + EEA_FILTER_ALL = { + **BASE_FILTER, + } + + AERONET_FILTER = { + **BASE_FILTER, # Forandring fra Daniel + "altitude": [-20, 1000], + } + + # Station filters + + eea_species = [ + "concpm10", + "concpm25", + "ratpm10pm25", + ] + + ebas_species = [ + "concpm10", + "concpm25", + "ratpm10pm25", + ] + + # no new sites with 2021 observations (comment Svetlana T.) + height_ignore_ebas = [ + "AT0034G", + "AT0038R", + "AT0049R", + "BG0001R", + "CH0001G", + "CH0018R", + "CH0024R", + "CH0031R", + "CH0033R", + "DE0054R", + "DE0057G", + "DE0075R", + "ES0005R", + "ES0022R", + "FR0019R", + "FR0030R", + "FR0031R", + "FR0038U", + "FR0039U", + "FR0100G", + "GR0003R", + "GR0101R", + "HR0002R", + "HR0004R", + "IT0002R", + "IT0009R", + "IT0019R", + "IT0020U", + "IT0021U", + "IT0024R", + "KG0001R", + "KG0002U", + "NO0036R", + "NO0039R", + "NO0211R", + "NO0214R", + "NO0225R", + "NO0226R", + "NO0227R", + "NO0229R", + "NO0796R", + "NO0802R", + "NO0907R", + "NO2073R", + "NO2079R", + "NO2085R", + "NO2096R", + "NO2156R", + "NO2210R", + "NO2216R", + "NO2219R", + "NO2233R", + "NO2239R", + "NO2257R", + "NO2263R", + "NO2274R", + "NO2280R", + "NO2288R", + "NO2362R", + "NO2380R", + "NO2397R", + "NO2411R", + "PL0003R", + "PT0005R", + "PT0007R", + "PT0012R", + "RO0002R", + "RO0003R", + "SE0093R", + "SE0094R", + "SI0032R", + "SK0002R", + ] + + height_ignore_eea = [ + "FR33220", + "TR0047A", + "AT72619", + "ES1982A", + "IT0983A", + "IS0040A", + "IT2099A", + "BG0080A", + "IT2159A", + "IT0906A", + "AT72821", + "IT1190A", + "IT1976A", + "AT56072", + "IT2178A", + "IS0044A", + "IT1335A", + "AT0SON1", + "IT0703A", + "AT72227", + "DEUB044", + "AT55032", + "HR0013A", + "FR33120", + "AT60182", + "IT0908A", + "ES1673A", + "AT55019", + "SK0042A", + "SI0032R", + "ES0005R", + "FR33720", + "DEBY196", + "AT60177", + "IT2128A", + "AT2SP18", + "FR15045", + "R160421", + "IT2234A", + "TR0118A", + "DEST039", + "E165168", + "AT72110", + "FR15013", + "ES1348A", + "E165169", + "AL0206A", + "AT72822", + "DEBY123", + "FR15031", + "AT72538", + "IS0042A", + "FR33114", + "AT52300", + "IT1859A", + "FR33232", + "IT2239A", + "IS0043A", + "PL0003R", + "FR31027", + "FR33113", + "FR15048", + "AT54057", + "TR0046A", + "FR33111", + "IT2284A", + "AT72550", + "IT1037A", + "FR33121", + "E165167", + "IT1847A", + "AT72912", + "RS0047A", + "R610613", + "TR0110A", + "R160512", + "IT1191A", + "IT1963A", + "FR15053", + "RO0009R", + "IT0508A", + "IT2233A", + "MK0041A", + "AT72519", + "BG0079A", + "IT1696A", + "IT1619A", + "IT2267A", + "TR0107A", + "AT56071", + "FR29440", + "AT4S235", + "AD0945A", + "IS0038A", + "E165166", + "PT01047", + "AT55018", + "SK0002R", + "IT0499A", + "HR0014A", + "IT0591A", + "IT0507A", + "AT72315", + "E165170", + "ES1432A", + "IT1166A", + "AT4S254", + "IT1967A", + "AT2VL52", + "IT1930A", + "AT72115", + "AT82708", + "IT0988A", + "FR15038", + "AT82801", + "IT2285A", + "NO0039R", + "TR0020A", + "IT2096A", + "AD0942A", + "TR0071A", + "E165165", + "ES0354A", + "AT72910", + "ES1882A", + "IT1725A", + "AT60150", + "CH0024A", + "IT1114A", + "AT72113", + "IT1852A", + "IS0048A", + "FR15017", + "FR15039", + "IT0980A", + "IT0502A", + "IT1678A", + "IT1334A", + "IT0978A", + "FR15043", + "IT2279A", + "IT0775A", + "IT1539A", + "AT72123", + "IT2014A", + "XK0005A", + "AT2WO15", + "FR33122", + "XK0007A", + "AT60196", + "CH0033A", + "IT1385A", + "GR0405A", + "AT52000", + "IT2266A", + "FR15046", + "AT72223", + "FR24024", + "IT0979A", + "AT2SP10", + "IT2179A", + "IT0977A", + "AT72530", + "ES1248A", + "AT72106", + "IT0753A", + ] + + EEA_FILTER_ALL = { + key: dict( + **EEA_FILTER_ALL, + station_name=height_ignore_eea, + negate="station_name", + ) + for key in eea_species + } + + OBS_GROUNDBASED = { + ################## + # EBAS + ################## + "EBAS-m": dict( + obs_id="EBASMC", + web_interface_name="EBAS-m", + obs_vars=ebas_species, + obs_vert_type="Surface", + colocate_time=True, + ts_type="monthly", + obs_filters=EBAS_FILTER, + obs_merge_how={ + "ratpm10pm25": "eval", + }, + obs_aux_requires={ + "ratpm10pm25": { + "EBASMC": [ + "concpm10", + "concpm25", + ], + } + }, + obs_aux_funs={ + "vmrox": + # variables used in computation method need to be based on AeroCom + # units, since the colocated StationData objects (from which the + # new UngriddedData is computed, will perform AeroCom unit check + # and conversion) + "(EBASMC;concpm10/EBASMC;concpm25)" + }, + obs_aux_units={"ratpm10pm25": "1"}, + ), + "EBAS-d": dict( + obs_id="EBASMC", + web_interface_name="EBAS-d", + obs_vars=ebas_species, + obs_vert_type="Surface", + colocate_time=True, + min_num_obs=DEFAULT_RESAMPLE_CONSTRAINTS, + ts_type="daily", + obs_filters=EBAS_FILTER, + obs_merge_how={ + "ratpm10pm25": "eval", + }, + obs_aux_requires={ + "ratpm10pm25": { + "EBASMC": [ + "concpm10", + "concpm25", + ], + } + }, + obs_aux_funs={ + "vmrox": + # variables used in computation method need to be based on AeroCom + # units, since the colocated StationData objects (from which the + # new UngriddedData is computed, will perform AeroCom unit check + # and conversion) + "(EBASMC;concpm10/EBASMC;concpm25)" + }, + obs_aux_units={"ratpm10pm25": "1"}, + ), + # Diurnal + # "EBAS-h-diurnal": dict( + # obs_id="EBASMC", + # web_interface_name="EBAS-h", + # obs_vars=[ + # "concNno2", + # "concNno", + # "vmro3", + # "concpm10", + # "concpm25", + # ], + # obs_vert_type="Surface", + # ts_type="hourly", + # # diurnal_only=True, + # resample_how="mean", + # obs_filters={**EBAS_FILTER, "ts_type": "hourly"}, + # ), + # OX + ################ + # EEA-rural + ################ + "EEA-d-rural": dict( + obs_id="EEAAQeRep.v2", + obs_vars=[ + "concpm10", + "concpm25", + "ratpm10pm25", + ], + web_interface_name="EEA-rural", + obs_vert_type="Surface", + obs_filters=EEA_FILTER, + obs_merge_how={ + "ratpm10pm25": "eval", + }, + obs_aux_requires={ + "ratpm10pm25": { + "EBASMC": [ + "concpm10", + "concpm25", + ], + } + }, + obs_aux_funs={ + "vmrox": + # variables used in computation method need to be based on AeroCom + # units, since the colocated StationData objects (from which the + # new UngriddedData is computed, will perform AeroCom unit check + # and conversion) + "(EBASMC;concpm10/EBASMC;concpm25)" + }, + obs_aux_units={"ratpm10pm25": "1"}, + ), + ################ + # EEA-all + ################ + "EEA-d-all": dict( + obs_id="EEAAQeRep.v2", + obs_vars=[ + "concpm10", + "concpm25", + "ratpm10pm25", + ], + web_interface_name="EEA-all", + obs_vert_type="Surface", + obs_filters=EEA_FILTER_ALL, + obs_merge_how={ + "ratpm10pm25": "eval", + }, + obs_aux_requires={ + "ratpm10pm25": { + "EBASMC": [ + "concpm10", + "concpm25", + ], + } + }, + obs_aux_funs={ + "vmrox": + # variables used in computation method need to be based on AeroCom + # units, since the colocated StationData objects (from which the + # new UngriddedData is computed, will perform AeroCom unit check + # and conversion) + "(EBASMC;concpm10/EBASMC;concpm25)" + }, + obs_aux_units={"ratpm10pm25": "1"}, + ), + } + + # Setup for supported satellite evaluations + OBS_SAT = {} + + OBS_CFG = {**OBS_GROUNDBASED, **OBS_SAT} + + CFG["obs_cfg"] = OBS_CFG + + return copy.deepcopy(CFG) diff --git a/pyaerocom/io/mscw_ctm/additional_variables.py b/pyaerocom/io/mscw_ctm/additional_variables.py index 883550722..bf893ae2c 100644 --- a/pyaerocom/io/mscw_ctm/additional_variables.py +++ b/pyaerocom/io/mscw_ctm/additional_variables.py @@ -382,3 +382,25 @@ def calc_concpolyol(concspores): concpolyol = concspores.copy(deep=True) * factor concpolyol.attrs["units"] = "ug m-3" return concpolyol + + +def calc_ratpm10pm25(concpm10: xr.DataArray, concpm25: xr.DataArray) -> xr.DataArray: + """ + Calculate ratio of pm10 and pm25 + + Parameters + ---------- + concpm10 : xr.DataArray + mass concentration pm10 + concpm25 : xr.DataArray + mass concentration of pm25 + + Returns + ------- + xr.DataArray + ratio of concpm10 / concpm25 in units of 1 + + """ + ratpm10pm25 = concpm10 / concpm25 + ratpm10pm25.attrs["units"] = "1" + return ratpm10pm25 diff --git a/pyaerocom/io/mscw_ctm/reader.py b/pyaerocom/io/mscw_ctm/reader.py index 6fb3c016e..4b332edd6 100755 --- a/pyaerocom/io/mscw_ctm/reader.py +++ b/pyaerocom/io/mscw_ctm/reader.py @@ -13,7 +13,6 @@ from pyaerocom.io.gridded_reader import GriddedReader from pyaerocom.projection_information import ProjectionInformation from pyaerocom.units_helpers import UALIASES - from .additional_variables import ( add_dataarrays, calc_concNhno3, @@ -36,6 +35,7 @@ identity, subtract_dataarrays, update_EC_units, + calc_ratpm10pm25, ) from .model_variables import emep_variables @@ -101,6 +101,7 @@ class ReadMscwCtm(GriddedReader): "concNno2": ["concno2"], "concSso2": ["concso2"], "vmro3": ["conco3"], + "ratpm10pm25": ["concpm10", "concpm25"], # For Pollen # "concpolyol": ["concspores"], } @@ -145,6 +146,7 @@ class ReadMscwCtm(GriddedReader): "concNno2": calc_concNno2, "concSso2": calc_concSso2, "vmro3": calc_vmro3, + "ratpm10pm25": calc_ratpm10pm25, # "concpolyol": calc_concpolyol, } @@ -702,7 +704,7 @@ def read_var(self, var_name, ts_type=None, **kwargs): proj_info=proj_info, ) - #!obsolete + # !obsolete # if var.is_deposition: # implicit_to_explicit_rates(gridded, ts_type) From 5bfec97d0edde728d3a9a11c963804c97255b157 Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Wed, 16 Oct 2024 15:08:30 +0200 Subject: [PATCH 02/14] add config example for ratpm10pm25 and ratpm25pm10 --- .../aeroval/config/ratpm10pm25/base_config.py | 356 +++--------------- pyaerocom/io/mscw_ctm/additional_variables.py | 22 ++ pyaerocom/io/mscw_ctm/reader.py | 5 +- 3 files changed, 74 insertions(+), 309 deletions(-) diff --git a/pyaerocom/aeroval/config/ratpm10pm25/base_config.py b/pyaerocom/aeroval/config/ratpm10pm25/base_config.py index 0d7beac41..748d78c22 100644 --- a/pyaerocom/aeroval/config/ratpm10pm25/base_config.py +++ b/pyaerocom/aeroval/config/ratpm10pm25/base_config.py @@ -47,26 +47,29 @@ def get_CFG(reportyear, year, model_dir) -> dict: # if True, existing colocated data files will be deleted and contours will be overwritten reanalyse_existing=True, only_json=False, - add_model_maps=True, + add_model_maps=False, only_model_maps=False, - modelmaps_opts=dict(maps_freq="yearly", maps_res_deg=5), + modelmaps_opts=dict(maps_freq="monthly", maps_res_deg=5), clear_existing_json=False, # if True, the analysis will stop whenever an error occurs (else, errors that # occurred will be written into the logfiles) - raise_exceptions=True, + raise_exceptions=False, # Regional filter for analysis filter_name="ALL-wMOUNTAINS", # colocation frequency (no statistics in higher resolution can be computed) ts_type="daily", map_zoom="Europe", - freqs=["yearly", "monthly", "weekly", "daily", "hourly"], + freqs=[ + "yearly", + "monthly", + "daily", + ], periods=[f"{year}"], - main_freq="daily", + main_freq="monthly", zeros_to_nan=False, use_diurnal=True, min_num_obs=DEFAULT_RESAMPLE_CONSTRAINTS, colocate_time=True, - resample_how={"vmro3max": {"daily": {"hourly": "max"}}}, obs_remove_outliers=False, model_remove_outliers=False, harmonise_units=True, @@ -85,6 +88,7 @@ def get_CFG(reportyear, year, model_dir) -> dict: var_order_menu=[ # Gases "ratpm10pm25", + "ratpm25pm10", "concNno", "concNno2", "concNtno3", @@ -122,30 +126,28 @@ def get_CFG(reportyear, year, model_dir) -> dict: ) CFG["model_cfg"] = { - "EMEP.cameo": dict( + "EMEPcameo": dict( model_id="EMEP,", model_data_dir=model_dir, - # gridded_reader_id={"model": "ReadMscwCtm"}, + gridded_reader_id={"model": "ReadMscwCtm"}, # model_read_aux={}, - # model_ts_type_read="daily", + model_ts_type_read="daily", ), + # "EMEP": dict( + # model_id="EMEP.ratpm25pm10.testing", + # # model_read_aux={ + # + # # "ratpm10pm25": dict( + # # vars_required=["sconcpm10", "sconcpm25"], fun="calc_ratpm10pm25" + # # ) + # # }, + # ), } """ Filters """ - # OBS SPECIFIC FILTERS (combination of the above and more) - EEA_RURAL_FILTER = { - "station_classification": ["background"], - "area_classification": [ - "rural", - "rural-nearcity", - "rural-regional", - "rural-remote", - ], - } - BASE_FILTER = { "latitude": [30, 82], "longitude": [-30, 90], @@ -157,15 +159,6 @@ def get_CFG(reportyear, year, model_dir) -> dict: "set_flags_nan": True, } - EEA_FILTER = { - **BASE_FILTER, - **EEA_RURAL_FILTER, - } - - EEA_FILTER_ALL = { - **BASE_FILTER, - } - AERONET_FILTER = { **BASE_FILTER, # Forandring fra Daniel "altitude": [-20, 1000], @@ -173,16 +166,11 @@ def get_CFG(reportyear, year, model_dir) -> dict: # Station filters - eea_species = [ - "concpm10", - "concpm25", - "ratpm10pm25", - ] - ebas_species = [ "concpm10", "concpm25", "ratpm10pm25", + "ratpm25pm10", ] # no new sites with 2021 observations (comment Svetlana T.) @@ -261,196 +249,20 @@ def get_CFG(reportyear, year, model_dir) -> dict: "SK0002R", ] - height_ignore_eea = [ - "FR33220", - "TR0047A", - "AT72619", - "ES1982A", - "IT0983A", - "IS0040A", - "IT2099A", - "BG0080A", - "IT2159A", - "IT0906A", - "AT72821", - "IT1190A", - "IT1976A", - "AT56072", - "IT2178A", - "IS0044A", - "IT1335A", - "AT0SON1", - "IT0703A", - "AT72227", - "DEUB044", - "AT55032", - "HR0013A", - "FR33120", - "AT60182", - "IT0908A", - "ES1673A", - "AT55019", - "SK0042A", - "SI0032R", - "ES0005R", - "FR33720", - "DEBY196", - "AT60177", - "IT2128A", - "AT2SP18", - "FR15045", - "R160421", - "IT2234A", - "TR0118A", - "DEST039", - "E165168", - "AT72110", - "FR15013", - "ES1348A", - "E165169", - "AL0206A", - "AT72822", - "DEBY123", - "FR15031", - "AT72538", - "IS0042A", - "FR33114", - "AT52300", - "IT1859A", - "FR33232", - "IT2239A", - "IS0043A", - "PL0003R", - "FR31027", - "FR33113", - "FR15048", - "AT54057", - "TR0046A", - "FR33111", - "IT2284A", - "AT72550", - "IT1037A", - "FR33121", - "E165167", - "IT1847A", - "AT72912", - "RS0047A", - "R610613", - "TR0110A", - "R160512", - "IT1191A", - "IT1963A", - "FR15053", - "RO0009R", - "IT0508A", - "IT2233A", - "MK0041A", - "AT72519", - "BG0079A", - "IT1696A", - "IT1619A", - "IT2267A", - "TR0107A", - "AT56071", - "FR29440", - "AT4S235", - "AD0945A", - "IS0038A", - "E165166", - "PT01047", - "AT55018", - "SK0002R", - "IT0499A", - "HR0014A", - "IT0591A", - "IT0507A", - "AT72315", - "E165170", - "ES1432A", - "IT1166A", - "AT4S254", - "IT1967A", - "AT2VL52", - "IT1930A", - "AT72115", - "AT82708", - "IT0988A", - "FR15038", - "AT82801", - "IT2285A", - "NO0039R", - "TR0020A", - "IT2096A", - "AD0942A", - "TR0071A", - "E165165", - "ES0354A", - "AT72910", - "ES1882A", - "IT1725A", - "AT60150", - "CH0024A", - "IT1114A", - "AT72113", - "IT1852A", - "IS0048A", - "FR15017", - "FR15039", - "IT0980A", - "IT0502A", - "IT1678A", - "IT1334A", - "IT0978A", - "FR15043", - "IT2279A", - "IT0775A", - "IT1539A", - "AT72123", - "IT2014A", - "XK0005A", - "AT2WO15", - "FR33122", - "XK0007A", - "AT60196", - "CH0033A", - "IT1385A", - "GR0405A", - "AT52000", - "IT2266A", - "FR15046", - "AT72223", - "FR24024", - "IT0979A", - "AT2SP10", - "IT2179A", - "IT0977A", - "AT72530", - "ES1248A", - "AT72106", - "IT0753A", - ] - - EEA_FILTER_ALL = { - key: dict( - **EEA_FILTER_ALL, - station_name=height_ignore_eea, - negate="station_name", - ) - for key in eea_species - } - OBS_GROUNDBASED = { ################## # EBAS ################## - "EBAS-m": dict( - obs_id="EBASMC", - web_interface_name="EBAS-m", - obs_vars=ebas_species, + "EBAS-d-10": dict( + obs_id="EBASratd10", + web_interface_name="EBAS-d", + obs_vars=["ratpm10pm25"], obs_vert_type="Surface", colocate_time=True, - ts_type="monthly", + min_num_obs=DEFAULT_RESAMPLE_CONSTRAINTS, + ts_type="daily", obs_filters=EBAS_FILTER, + obs_type="ungridded", obs_merge_how={ "ratpm10pm25": "eval", }, @@ -463,81 +275,30 @@ def get_CFG(reportyear, year, model_dir) -> dict: } }, obs_aux_funs={ - "vmrox": + "ratpm10pm25": # variables used in computation method need to be based on AeroCom # units, since the colocated StationData objects (from which the # new UngriddedData is computed, will perform AeroCom unit check # and conversion) - "(EBASMC;concpm10/EBASMC;concpm25)" + "(EBASMC;concpm10/EBASMC;concpm25)" }, obs_aux_units={"ratpm10pm25": "1"}, ), - "EBAS-d": dict( - obs_id="EBASMC", + "EBAS-d-25": dict( + obs_id="EBASratd25", web_interface_name="EBAS-d", - obs_vars=ebas_species, + obs_vars=["ratpm25pm10"], obs_vert_type="Surface", colocate_time=True, min_num_obs=DEFAULT_RESAMPLE_CONSTRAINTS, ts_type="daily", obs_filters=EBAS_FILTER, + obs_type="ungridded", obs_merge_how={ - "ratpm10pm25": "eval", - }, - obs_aux_requires={ - "ratpm10pm25": { - "EBASMC": [ - "concpm10", - "concpm25", - ], - } - }, - obs_aux_funs={ - "vmrox": - # variables used in computation method need to be based on AeroCom - # units, since the colocated StationData objects (from which the - # new UngriddedData is computed, will perform AeroCom unit check - # and conversion) - "(EBASMC;concpm10/EBASMC;concpm25)" - }, - obs_aux_units={"ratpm10pm25": "1"}, - ), - # Diurnal - # "EBAS-h-diurnal": dict( - # obs_id="EBASMC", - # web_interface_name="EBAS-h", - # obs_vars=[ - # "concNno2", - # "concNno", - # "vmro3", - # "concpm10", - # "concpm25", - # ], - # obs_vert_type="Surface", - # ts_type="hourly", - # # diurnal_only=True, - # resample_how="mean", - # obs_filters={**EBAS_FILTER, "ts_type": "hourly"}, - # ), - # OX - ################ - # EEA-rural - ################ - "EEA-d-rural": dict( - obs_id="EEAAQeRep.v2", - obs_vars=[ - "concpm10", - "concpm25", - "ratpm10pm25", - ], - web_interface_name="EEA-rural", - obs_vert_type="Surface", - obs_filters=EEA_FILTER, - obs_merge_how={ - "ratpm10pm25": "eval", + "ratpm25pm10": "eval", }, obs_aux_requires={ - "ratpm10pm25": { + "ratpm25pm10": { "EBASMC": [ "concpm10", "concpm25", @@ -545,48 +306,27 @@ def get_CFG(reportyear, year, model_dir) -> dict: } }, obs_aux_funs={ - "vmrox": + "ratpm25pm10": # variables used in computation method need to be based on AeroCom # units, since the colocated StationData objects (from which the # new UngriddedData is computed, will perform AeroCom unit check # and conversion) - "(EBASMC;concpm10/EBASMC;concpm25)" + "(EBASMC;concpm25/EBASMC;concpm10)" }, - obs_aux_units={"ratpm10pm25": "1"}, + obs_aux_units={"ratpm25pm10": "1"}, ), - ################ - # EEA-all - ################ - "EEA-d-all": dict( - obs_id="EEAAQeRep.v2", + "EBAS-d-tc": dict( + obs_id="EBASMC", + web_interface_name="EBAS-d", obs_vars=[ "concpm10", "concpm25", - "ratpm10pm25", ], - web_interface_name="EEA-all", obs_vert_type="Surface", - obs_filters=EEA_FILTER_ALL, - obs_merge_how={ - "ratpm10pm25": "eval", - }, - obs_aux_requires={ - "ratpm10pm25": { - "EBASMC": [ - "concpm10", - "concpm25", - ], - } - }, - obs_aux_funs={ - "vmrox": - # variables used in computation method need to be based on AeroCom - # units, since the colocated StationData objects (from which the - # new UngriddedData is computed, will perform AeroCom unit check - # and conversion) - "(EBASMC;concpm10/EBASMC;concpm25)" - }, - obs_aux_units={"ratpm10pm25": "1"}, + colocate_time=True, + min_num_obs=DEFAULT_RESAMPLE_CONSTRAINTS, + ts_type="daily", + obs_filters=EBAS_FILTER, ), } diff --git a/pyaerocom/io/mscw_ctm/additional_variables.py b/pyaerocom/io/mscw_ctm/additional_variables.py index bf893ae2c..9208141b7 100644 --- a/pyaerocom/io/mscw_ctm/additional_variables.py +++ b/pyaerocom/io/mscw_ctm/additional_variables.py @@ -404,3 +404,25 @@ def calc_ratpm10pm25(concpm10: xr.DataArray, concpm25: xr.DataArray) -> xr.DataA ratpm10pm25 = concpm10 / concpm25 ratpm10pm25.attrs["units"] = "1" return ratpm10pm25 + + +def calc_ratpm25pm10(concpm25: xr.DataArray, concpm10: xr.DataArray) -> xr.DataArray: + """ + Calculate ratio of pm10 and pm25 + + Parameters + ---------- + concpm10 : xr.DataArray + mass concentration pm10 + concpm25 : xr.DataArray + mass concentration of pm25 + + Returns + ------- + xr.DataArray + ratio of concpm25 / concpm10 in units of 1 + + """ + ratpm25pm10 = concpm25 / concpm10 + ratpm25pm10.attrs["units"] = "1" + return ratpm25pm10 diff --git a/pyaerocom/io/mscw_ctm/reader.py b/pyaerocom/io/mscw_ctm/reader.py index 4b332edd6..ed4790081 100755 --- a/pyaerocom/io/mscw_ctm/reader.py +++ b/pyaerocom/io/mscw_ctm/reader.py @@ -6,13 +6,13 @@ import numpy as np import xarray as xr - from pyaerocom import const from pyaerocom.exceptions import VarNotAvailableError from pyaerocom.griddeddata import GriddedData from pyaerocom.io.gridded_reader import GriddedReader from pyaerocom.projection_information import ProjectionInformation from pyaerocom.units_helpers import UALIASES + from .additional_variables import ( add_dataarrays, calc_concNhno3, @@ -36,6 +36,7 @@ subtract_dataarrays, update_EC_units, calc_ratpm10pm25, + calc_ratpm25pm10, ) from .model_variables import emep_variables @@ -102,6 +103,7 @@ class ReadMscwCtm(GriddedReader): "concSso2": ["concso2"], "vmro3": ["conco3"], "ratpm10pm25": ["concpm10", "concpm25"], + "ratpm25pm10": ["concpm25", "concpm10"], # For Pollen # "concpolyol": ["concspores"], } @@ -147,6 +149,7 @@ class ReadMscwCtm(GriddedReader): "concSso2": calc_concSso2, "vmro3": calc_vmro3, "ratpm10pm25": calc_ratpm10pm25, + "ratpm25pm10": calc_ratpm25pm10, # "concpolyol": calc_concpolyol, } From d009166c986a7fdababdca70bcbf05a8af204f20 Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Wed, 16 Oct 2024 15:20:01 +0200 Subject: [PATCH 03/14] rename directory for clearness --- pyaerocom/aeroval/config/{ratpm10pm25 => pmratios}/__init__.py | 0 pyaerocom/aeroval/config/{ratpm10pm25 => pmratios}/base_config.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename pyaerocom/aeroval/config/{ratpm10pm25 => pmratios}/__init__.py (100%) rename pyaerocom/aeroval/config/{ratpm10pm25 => pmratios}/base_config.py (100%) diff --git a/pyaerocom/aeroval/config/ratpm10pm25/__init__.py b/pyaerocom/aeroval/config/pmratios/__init__.py similarity index 100% rename from pyaerocom/aeroval/config/ratpm10pm25/__init__.py rename to pyaerocom/aeroval/config/pmratios/__init__.py diff --git a/pyaerocom/aeroval/config/ratpm10pm25/base_config.py b/pyaerocom/aeroval/config/pmratios/base_config.py similarity index 100% rename from pyaerocom/aeroval/config/ratpm10pm25/base_config.py rename to pyaerocom/aeroval/config/pmratios/base_config.py From 678bb3b96c56fd4d91d9228b2e3a7639f3e0ed86 Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Wed, 16 Oct 2024 15:53:44 +0200 Subject: [PATCH 04/14] add some documentation --- docs/_static/aeroval/sample_pm_ratios.py | 45 ++++++++++++++++++++++++ docs/aeroval-examples.rst | 7 +++- 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 docs/_static/aeroval/sample_pm_ratios.py diff --git a/docs/_static/aeroval/sample_pm_ratios.py b/docs/_static/aeroval/sample_pm_ratios.py new file mode 100644 index 000000000..044146a7b --- /dev/null +++ b/docs/_static/aeroval/sample_pm_ratios.py @@ -0,0 +1,45 @@ +# Example aeroval configuration for pm ratios +# using the EMEP reader (which has a built in pm ratio calculation) + +import os +from getpass import getuser + +from pyaerocom.aeroval import EvalSetup, ExperimentProcessor +from pyaerocom.aeroval.config.pmratios.base_config import get_CFG + +reportyear = year = 2019 +CFG = get_CFG(reportyear=reportyear, year=year, + # model directory + model_dir="/lustre/storeB/project/fou/kl/CAMEO/u8_cams0201/" + ) +user = getuser() + +CFG.update(dict( + json_basedir=os.path.abspath(f"/home/{user}/data/aeroval-local-web/data"), # always adjust to your environment + coldata_basedir=os.path.abspath(f"/home/{user}/data/aeroval-local-web/coldata"), + # always adjust to your environment + clear_existing_json=True, + add_model_maps=True, + # if True, the analysis will stop whenever an error occurs (else, errors that + # occurred will be written into the logfiles) + raise_exceptions=False, + modelmaps_opts=dict(maps_freq="monthly", maps_res_deg=5), + # Regional filter for analysis + periods=[f"{year}"], + proj_id="RATPM", + exp_id=f"ratpm testing {year}", + exp_name=f"Evaluation of EMEP runs for {year}", + exp_descr=( + f"Evaluation of EMEP runs for {year} for CAMEO. The EMEP model, is compared against observations from EEA and EBAS." + ), + exp_pi="jang@met.no", + public=True, + # directory where colocated data files are supposed to be stored + weighted_stats=True, )) + +stp = EvalSetup(**CFG) + +ana = ExperimentProcessor(stp) +ana.update_interface() + +res = ana.run() diff --git a/docs/aeroval-examples.rst b/docs/aeroval-examples.rst index 713b3e82f..b86ac101e 100644 --- a/docs/aeroval-examples.rst +++ b/docs/aeroval-examples.rst @@ -8,7 +8,7 @@ detailed explanations of the setup parameters. A configuration could be run as t The code blocks below are the Python configuruation files *cfg_examples_example1.py* and *sample_gridded_io_aux.py*. -Example 1 +Example 1: NorESM, CAMS reanalysis against AERONET --------- NorESM2 and CAMS reanalysis vs AERONET and merged satellite AOD dataset. @@ -21,3 +21,8 @@ Example IO aux file for model reading .. literalinclude:: _static/aeroval/sample_gridded_io_aux.py +Example for pm ratios compared to EMEP mode data +--------------------- + +.. literalinclude:: _static/aeroval/sample_pm_ratios.py + From 7d635fa248cd60503437593615ce68cf7379b4a1 Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Wed, 16 Oct 2024 16:09:38 +0200 Subject: [PATCH 05/14] add pm ratio tests --- tests/io/mscw_ctm/test_additional_variables.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/io/mscw_ctm/test_additional_variables.py b/tests/io/mscw_ctm/test_additional_variables.py index 99dccba70..eaf137443 100644 --- a/tests/io/mscw_ctm/test_additional_variables.py +++ b/tests/io/mscw_ctm/test_additional_variables.py @@ -6,6 +6,8 @@ calc_concNno3pm25, calc_concNtnh, calc_conNtno3, + calc_ratpm10pm25, + calc_ratpm25pm10, update_EC_units, ) from tests.fixtures.mscw_ctm import create_fake_MSCWCtm_data @@ -80,6 +82,22 @@ def test_calc_concNtnh(): assert (concNtnh == concNtnh_from_func).all() +def test_calc_concpm10pm25(): + concpm10 = create_fake_MSCWCtm_data() + concpm25 = create_fake_MSCWCtm_data() + + ratpm10pm25_from_func = calc_ratpm10pm25(concpm10, concpm25) + assert ratpm10pm25_from_func.attrs["units"] == "1" + + +def test_calc_concpm25pm10(): + concpm10 = create_fake_MSCWCtm_data() + concpm25 = create_fake_MSCWCtm_data() + + ratpm10pm25_from_func = calc_ratpm25pm10(concpm25, concpm10) + assert ratpm10pm25_from_func.attrs["units"] == "1" + + def test_calc_concNnh3(): concnh3 = create_fake_MSCWCtm_data() From bd8a2ee1705b446fe7bfa7b217c102abc49ba85c Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Wed, 16 Oct 2024 16:12:28 +0200 Subject: [PATCH 06/14] linting --- tests/io/mscw_ctm/test_additional_variables.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/io/mscw_ctm/test_additional_variables.py b/tests/io/mscw_ctm/test_additional_variables.py index eaf137443..bccbb057f 100644 --- a/tests/io/mscw_ctm/test_additional_variables.py +++ b/tests/io/mscw_ctm/test_additional_variables.py @@ -94,8 +94,8 @@ def test_calc_concpm25pm10(): concpm10 = create_fake_MSCWCtm_data() concpm25 = create_fake_MSCWCtm_data() - ratpm10pm25_from_func = calc_ratpm25pm10(concpm25, concpm10) - assert ratpm10pm25_from_func.attrs["units"] == "1" + ratpm25pm10_from_func = calc_ratpm25pm10(concpm25, concpm10) + assert ratpm25pm10_from_func.attrs["units"] == "1" def test_calc_concNnh3(): From 80a22f89e1fdeac79f4cc5db4abd0007d2db7ef7 Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Wed, 16 Oct 2024 16:14:50 +0200 Subject: [PATCH 07/14] linting --- pyaerocom/aeroval/config/pmratios/base_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyaerocom/aeroval/config/pmratios/base_config.py b/pyaerocom/aeroval/config/pmratios/base_config.py index 748d78c22..6973e2a60 100644 --- a/pyaerocom/aeroval/config/pmratios/base_config.py +++ b/pyaerocom/aeroval/config/pmratios/base_config.py @@ -280,7 +280,7 @@ def get_CFG(reportyear, year, model_dir) -> dict: # units, since the colocated StationData objects (from which the # new UngriddedData is computed, will perform AeroCom unit check # and conversion) - "(EBASMC;concpm10/EBASMC;concpm25)" + "(EBASMC;concpm10/EBASMC;concpm25)" }, obs_aux_units={"ratpm10pm25": "1"}, ), @@ -311,7 +311,7 @@ def get_CFG(reportyear, year, model_dir) -> dict: # units, since the colocated StationData objects (from which the # new UngriddedData is computed, will perform AeroCom unit check # and conversion) - "(EBASMC;concpm25/EBASMC;concpm10)" + "(EBASMC;concpm25/EBASMC;concpm10)" }, obs_aux_units={"ratpm25pm10": "1"}, ), From 7d2dc95d16f27a684c5094790b25717b765e2343 Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Wed, 16 Oct 2024 16:20:12 +0200 Subject: [PATCH 08/14] ruff --- .../aeroval/config/pmratios/base_config.py | 104 +----------------- 1 file changed, 2 insertions(+), 102 deletions(-) diff --git a/pyaerocom/aeroval/config/pmratios/base_config.py b/pyaerocom/aeroval/config/pmratios/base_config.py index 6973e2a60..4f0526520 100644 --- a/pyaerocom/aeroval/config/pmratios/base_config.py +++ b/pyaerocom/aeroval/config/pmratios/base_config.py @@ -34,7 +34,6 @@ def get_CFG(reportyear, year, model_dir) -> dict: :returns: a dict of a model configuration usable for EvalSetup """ # get current path for reference to local gridded_io_aux.py - base_conf_path = os.path.dirname(__file__) CFG = dict( json_basedir=os.path.abspath("/home/jang/data/aeroval-local-web/data"), @@ -133,15 +132,6 @@ def get_CFG(reportyear, year, model_dir) -> dict: # model_read_aux={}, model_ts_type_read="daily", ), - # "EMEP": dict( - # model_id="EMEP.ratpm25pm10.testing", - # # model_read_aux={ - # - # # "ratpm10pm25": dict( - # # vars_required=["sconcpm10", "sconcpm25"], fun="calc_ratpm10pm25" - # # ) - # # }, - # ), } """ @@ -159,96 +149,6 @@ def get_CFG(reportyear, year, model_dir) -> dict: "set_flags_nan": True, } - AERONET_FILTER = { - **BASE_FILTER, # Forandring fra Daniel - "altitude": [-20, 1000], - } - - # Station filters - - ebas_species = [ - "concpm10", - "concpm25", - "ratpm10pm25", - "ratpm25pm10", - ] - - # no new sites with 2021 observations (comment Svetlana T.) - height_ignore_ebas = [ - "AT0034G", - "AT0038R", - "AT0049R", - "BG0001R", - "CH0001G", - "CH0018R", - "CH0024R", - "CH0031R", - "CH0033R", - "DE0054R", - "DE0057G", - "DE0075R", - "ES0005R", - "ES0022R", - "FR0019R", - "FR0030R", - "FR0031R", - "FR0038U", - "FR0039U", - "FR0100G", - "GR0003R", - "GR0101R", - "HR0002R", - "HR0004R", - "IT0002R", - "IT0009R", - "IT0019R", - "IT0020U", - "IT0021U", - "IT0024R", - "KG0001R", - "KG0002U", - "NO0036R", - "NO0039R", - "NO0211R", - "NO0214R", - "NO0225R", - "NO0226R", - "NO0227R", - "NO0229R", - "NO0796R", - "NO0802R", - "NO0907R", - "NO2073R", - "NO2079R", - "NO2085R", - "NO2096R", - "NO2156R", - "NO2210R", - "NO2216R", - "NO2219R", - "NO2233R", - "NO2239R", - "NO2257R", - "NO2263R", - "NO2274R", - "NO2280R", - "NO2288R", - "NO2362R", - "NO2380R", - "NO2397R", - "NO2411R", - "PL0003R", - "PT0005R", - "PT0007R", - "PT0012R", - "RO0002R", - "RO0003R", - "SE0093R", - "SE0094R", - "SI0032R", - "SK0002R", - ] - OBS_GROUNDBASED = { ################## # EBAS @@ -280,7 +180,7 @@ def get_CFG(reportyear, year, model_dir) -> dict: # units, since the colocated StationData objects (from which the # new UngriddedData is computed, will perform AeroCom unit check # and conversion) - "(EBASMC;concpm10/EBASMC;concpm25)" + "(EBASMC;concpm10/EBASMC;concpm25)" }, obs_aux_units={"ratpm10pm25": "1"}, ), @@ -311,7 +211,7 @@ def get_CFG(reportyear, year, model_dir) -> dict: # units, since the colocated StationData objects (from which the # new UngriddedData is computed, will perform AeroCom unit check # and conversion) - "(EBASMC;concpm25/EBASMC;concpm10)" + "(EBASMC;concpm25/EBASMC;concpm10)" }, obs_aux_units={"ratpm25pm10": "1"}, ), From 935f4835722717449b33659a5ff2b7e8ce94c827 Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Wed, 16 Oct 2024 16:37:36 +0200 Subject: [PATCH 09/14] add coverage --- tests/io/mscw_ctm/test_additional_variables.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/io/mscw_ctm/test_additional_variables.py b/tests/io/mscw_ctm/test_additional_variables.py index bccbb057f..7af78c44c 100644 --- a/tests/io/mscw_ctm/test_additional_variables.py +++ b/tests/io/mscw_ctm/test_additional_variables.py @@ -128,3 +128,14 @@ def test_update_EC_units(): assert (concCecpm25 == concCecpm25_from_func).all() assert concCecpm25.units == concCecpm25_from_func.units + + +def test_ratpmconfig(): + from pyaerocom.aeroval.config.pmratios.base_config import get_CFG + reportyear = year = 2019 + CFG = get_CFG( + reportyear=reportyear, + year=year, + model_dir="/lustre/storeB/project/fou/kl/CAMEO/u8_cams0201/", + ) + assert CFG['raise_exceptions'] == False From d5790fbce3fe172bdb77f703689cde3641b863d9 Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Thu, 17 Oct 2024 10:00:34 +0200 Subject: [PATCH 10/14] add seperate example config test --- pyaerocom/aeroval/config/pmratios/base_config.py | 4 ++-- tests/io/mscw_ctm/test_additional_variables.py | 11 ----------- tests/io/mscw_ctm/test_aeroval_configs.py | 13 +++++++++++++ 3 files changed, 15 insertions(+), 13 deletions(-) create mode 100644 tests/io/mscw_ctm/test_aeroval_configs.py diff --git a/pyaerocom/aeroval/config/pmratios/base_config.py b/pyaerocom/aeroval/config/pmratios/base_config.py index 4f0526520..f90eed2ab 100644 --- a/pyaerocom/aeroval/config/pmratios/base_config.py +++ b/pyaerocom/aeroval/config/pmratios/base_config.py @@ -180,7 +180,7 @@ def get_CFG(reportyear, year, model_dir) -> dict: # units, since the colocated StationData objects (from which the # new UngriddedData is computed, will perform AeroCom unit check # and conversion) - "(EBASMC;concpm10/EBASMC;concpm25)" + "(EBASMC;concpm10/EBASMC;concpm25)" }, obs_aux_units={"ratpm10pm25": "1"}, ), @@ -211,7 +211,7 @@ def get_CFG(reportyear, year, model_dir) -> dict: # units, since the colocated StationData objects (from which the # new UngriddedData is computed, will perform AeroCom unit check # and conversion) - "(EBASMC;concpm25/EBASMC;concpm10)" + "(EBASMC;concpm25/EBASMC;concpm10)" }, obs_aux_units={"ratpm25pm10": "1"}, ), diff --git a/tests/io/mscw_ctm/test_additional_variables.py b/tests/io/mscw_ctm/test_additional_variables.py index 7af78c44c..bccbb057f 100644 --- a/tests/io/mscw_ctm/test_additional_variables.py +++ b/tests/io/mscw_ctm/test_additional_variables.py @@ -128,14 +128,3 @@ def test_update_EC_units(): assert (concCecpm25 == concCecpm25_from_func).all() assert concCecpm25.units == concCecpm25_from_func.units - - -def test_ratpmconfig(): - from pyaerocom.aeroval.config.pmratios.base_config import get_CFG - reportyear = year = 2019 - CFG = get_CFG( - reportyear=reportyear, - year=year, - model_dir="/lustre/storeB/project/fou/kl/CAMEO/u8_cams0201/", - ) - assert CFG['raise_exceptions'] == False diff --git a/tests/io/mscw_ctm/test_aeroval_configs.py b/tests/io/mscw_ctm/test_aeroval_configs.py new file mode 100644 index 000000000..c324036be --- /dev/null +++ b/tests/io/mscw_ctm/test_aeroval_configs.py @@ -0,0 +1,13 @@ +from pyaerocom.aeroval.config.pmratios.base_config import get_CFG + + +def test_ratpmconfig(): + """short test if the example configuration for pm ratios is still in the code""" + + reportyear = year = 2019 + CFG = get_CFG( + reportyear=reportyear, + year=year, + model_dir="/lustre/storeB/project/fou/kl/CAMEO/u8_cams0201/", + ) + assert CFG["raise_exceptions"] == False From 6e0ef91a1adb6d656a4fee24ac11aad14da26668 Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Thu, 17 Oct 2024 10:13:50 +0200 Subject: [PATCH 11/14] linting --- docs/_static/aeroval/sample_pm_ratios.py | 59 +++++++++++++---------- tests/io/mscw_ctm/test_aeroval_configs.py | 2 +- 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/docs/_static/aeroval/sample_pm_ratios.py b/docs/_static/aeroval/sample_pm_ratios.py index 044146a7b..5bcad8eca 100644 --- a/docs/_static/aeroval/sample_pm_ratios.py +++ b/docs/_static/aeroval/sample_pm_ratios.py @@ -8,34 +8,41 @@ from pyaerocom.aeroval.config.pmratios.base_config import get_CFG reportyear = year = 2019 -CFG = get_CFG(reportyear=reportyear, year=year, - # model directory - model_dir="/lustre/storeB/project/fou/kl/CAMEO/u8_cams0201/" - ) +CFG = get_CFG( + reportyear=reportyear, + year=year, + # model directory + model_dir="/lustre/storeB/project/fou/kl/CAMEO/u8_cams0201/", +) user = getuser() -CFG.update(dict( - json_basedir=os.path.abspath(f"/home/{user}/data/aeroval-local-web/data"), # always adjust to your environment - coldata_basedir=os.path.abspath(f"/home/{user}/data/aeroval-local-web/coldata"), - # always adjust to your environment - clear_existing_json=True, - add_model_maps=True, - # if True, the analysis will stop whenever an error occurs (else, errors that - # occurred will be written into the logfiles) - raise_exceptions=False, - modelmaps_opts=dict(maps_freq="monthly", maps_res_deg=5), - # Regional filter for analysis - periods=[f"{year}"], - proj_id="RATPM", - exp_id=f"ratpm testing {year}", - exp_name=f"Evaluation of EMEP runs for {year}", - exp_descr=( - f"Evaluation of EMEP runs for {year} for CAMEO. The EMEP model, is compared against observations from EEA and EBAS." - ), - exp_pi="jang@met.no", - public=True, - # directory where colocated data files are supposed to be stored - weighted_stats=True, )) +CFG.update( + dict( + json_basedir=os.path.abspath( + f"/home/{user}/data/aeroval-local-web/data" + ), # always adjust to your environment + coldata_basedir=os.path.abspath(f"/home/{user}/data/aeroval-local-web/coldata"), + # always adjust to your environment + clear_existing_json=True, + add_model_maps=True, + # if True, the analysis will stop whenever an error occurs (else, errors that + # occurred will be written into the logfiles) + raise_exceptions=False, + modelmaps_opts=dict(maps_freq="monthly", maps_res_deg=5), + # Regional filter for analysis + periods=[f"{year}"], + proj_id="RATPM", + exp_id=f"ratpm testing {year}", + exp_name=f"Evaluation of EMEP runs for {year}", + exp_descr=( + f"Evaluation of EMEP runs for {year} for CAMEO. The EMEP model, is compared against observations from EEA and EBAS." + ), + exp_pi="jang@met.no", + public=True, + # directory where colocated data files are supposed to be stored + weighted_stats=True, + ) +) stp = EvalSetup(**CFG) diff --git a/tests/io/mscw_ctm/test_aeroval_configs.py b/tests/io/mscw_ctm/test_aeroval_configs.py index c324036be..cff52fcbb 100644 --- a/tests/io/mscw_ctm/test_aeroval_configs.py +++ b/tests/io/mscw_ctm/test_aeroval_configs.py @@ -10,4 +10,4 @@ def test_ratpmconfig(): year=year, model_dir="/lustre/storeB/project/fou/kl/CAMEO/u8_cams0201/", ) - assert CFG["raise_exceptions"] == False + assert not CFG["raise_exceptions"] From 990e7877ab0b7b182db52c687bf0b7f06518f677 Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Thu, 17 Oct 2024 10:22:28 +0200 Subject: [PATCH 12/14] typo --- docs/aeroval-examples.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/aeroval-examples.rst b/docs/aeroval-examples.rst index b86ac101e..2838933fa 100644 --- a/docs/aeroval-examples.rst +++ b/docs/aeroval-examples.rst @@ -21,7 +21,7 @@ Example IO aux file for model reading .. literalinclude:: _static/aeroval/sample_gridded_io_aux.py -Example for pm ratios compared to EMEP mode data +Example for pm ratios compared to EMEP model data --------------------- .. literalinclude:: _static/aeroval/sample_pm_ratios.py From c630ebd5e8752ca4441f181e85e33d044b42b2ed Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Thu, 17 Oct 2024 11:42:40 +0200 Subject: [PATCH 13/14] adjusted to code review --- pyaerocom/io/mscw_ctm/additional_variables.py | 16 +++++++++++++++- pyaerocom/io/mscw_ctm/reader.py | 4 ---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/pyaerocom/io/mscw_ctm/additional_variables.py b/pyaerocom/io/mscw_ctm/additional_variables.py index 9208141b7..5795b9fb1 100644 --- a/pyaerocom/io/mscw_ctm/additional_variables.py +++ b/pyaerocom/io/mscw_ctm/additional_variables.py @@ -1,9 +1,11 @@ +import logging + import xarray as xr from geonum.atmosphere import T0_STD, p0 - from pyaerocom.aux_var_helpers import concx_to_vmrx from pyaerocom.molmasses import get_molmass +logger = logging.getLogger(__name__) def add_dataarrays(arr0: xr.DataArray, *arrs: xr.DataArray) -> xr.DataArray: """ @@ -401,6 +403,12 @@ def calc_ratpm10pm25(concpm10: xr.DataArray, concpm25: xr.DataArray) -> xr.DataA ratio of concpm10 / concpm25 in units of 1 """ + try: + if concpm10.attrs["units"] != concpm25.attrs["units"]: + logger.warning( + f"concpm10 unit {concpm10.attrs['units']} not equal to concpm25 unit {concpm25.attrs['units']}!") + except KeyError: + pass ratpm10pm25 = concpm10 / concpm25 ratpm10pm25.attrs["units"] = "1" return ratpm10pm25 @@ -423,6 +431,12 @@ def calc_ratpm25pm10(concpm25: xr.DataArray, concpm10: xr.DataArray) -> xr.DataA ratio of concpm25 / concpm10 in units of 1 """ + try: + if concpm10.attrs["units"] != concpm25.attrs["units"]: + logger.warning( + f"concpm10 unit {concpm10.attrs['units']} not equal to concpm25 unit {concpm25.attrs['units']}!") + except KeyError: + pass ratpm25pm10 = concpm25 / concpm10 ratpm25pm10.attrs["units"] = "1" return ratpm25pm10 diff --git a/pyaerocom/io/mscw_ctm/reader.py b/pyaerocom/io/mscw_ctm/reader.py index ed4790081..c717fde3c 100755 --- a/pyaerocom/io/mscw_ctm/reader.py +++ b/pyaerocom/io/mscw_ctm/reader.py @@ -707,10 +707,6 @@ def read_var(self, var_name, ts_type=None, **kwargs): proj_info=proj_info, ) - # !obsolete - # if var.is_deposition: - # implicit_to_explicit_rates(gridded, ts_type) - # At this point a GriddedData object with name gridded should exist gridded.metadata["data_id"] = self._data_id From 61682b8fb58f9ba32070c78c7d18b917dff21f23 Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Thu, 17 Oct 2024 11:56:06 +0200 Subject: [PATCH 14/14] linting --- pyaerocom/io/mscw_ctm/additional_variables.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pyaerocom/io/mscw_ctm/additional_variables.py b/pyaerocom/io/mscw_ctm/additional_variables.py index 5795b9fb1..abdf51e0c 100644 --- a/pyaerocom/io/mscw_ctm/additional_variables.py +++ b/pyaerocom/io/mscw_ctm/additional_variables.py @@ -7,6 +7,7 @@ logger = logging.getLogger(__name__) + def add_dataarrays(arr0: xr.DataArray, *arrs: xr.DataArray) -> xr.DataArray: """ Add a bunch of :class:`xarray.DataArray` instances @@ -406,7 +407,8 @@ def calc_ratpm10pm25(concpm10: xr.DataArray, concpm25: xr.DataArray) -> xr.DataA try: if concpm10.attrs["units"] != concpm25.attrs["units"]: logger.warning( - f"concpm10 unit {concpm10.attrs['units']} not equal to concpm25 unit {concpm25.attrs['units']}!") + f"concpm10 unit {concpm10.attrs['units']} not equal to concpm25 unit {concpm25.attrs['units']}!" + ) except KeyError: pass ratpm10pm25 = concpm10 / concpm25 @@ -434,7 +436,8 @@ def calc_ratpm25pm10(concpm25: xr.DataArray, concpm10: xr.DataArray) -> xr.DataA try: if concpm10.attrs["units"] != concpm25.attrs["units"]: logger.warning( - f"concpm10 unit {concpm10.attrs['units']} not equal to concpm25 unit {concpm25.attrs['units']}!") + f"concpm10 unit {concpm10.attrs['units']} not equal to concpm25 unit {concpm25.attrs['units']}!" + ) except KeyError: pass ratpm25pm10 = concpm25 / concpm10