Merges with main dev

.gitignore

@@ -158,4 +158,7 @@ sorted_out
 *.swp
 
 
-pytest.ini
+pytest.ini
+
+# can store scripts etc. locally for testing and dev purposes
+playground/

docs/api-aeroval.rst

@@ -11,7 +11,7 @@ Tools for AeroVal experiment setup
 High level analysis setup for AeroVal experiment
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-.. automodule:: pyaerocom.aeroval.setupclasses
+.. automodule:: pyaerocom.aeroval.setup_classes
    :members:
 
 Specification of observation datasets

pyaerocom/aeroval/__init__.py

@@ -1,3 +1,3 @@
 # isort:skip_file
-from .setupclasses import EvalSetup
+from .setup_classes import EvalSetup
 from .experiment_processor import ExperimentProcessor

pyaerocom/aeroval/_processing_base.py

@@ -6,7 +6,6 @@
 from pyaerocom.aeroval.experiment_output import ExperimentOutput
 from pyaerocom.colocation.colocation_setup import ColocationSetup
 from pyaerocom.colocation.colocator import Colocator
-from pyaerocom import TsType
 
 logger = logging.getLogger(__name__)
 
@@ -212,3 +211,15 @@ def read_ungridded_obsdata(self, obs_name, var_name):
 
         data = col._read_ungridded(var_name)
         return data
+
+    def read_gridded_obsdata(self, obs_name, var_name):
+        """
+        Import gridded observation data, usually satellite data
+
+        Args:
+            obs_name (str): Name of observation network in :attr:`cfg`
+            var_name (str): Name of variable to be read.
+        """
+        col = self.get_colocator(obs_name=obs_name)
+        data = col._read_gridded(var_name, is_model=False)
+        return data

pyaerocom/aeroval/config/emep/reporting_base.py

@@ -390,12 +390,16 @@ def get_CFG(reportyear, year, model_dir) -> dict:
             "concpm25",
             "concno3pm10",
             "concno3pm25",
+            "concno3pm1",
             "concnh4pm25",
+            "concnh4pm1",
             "concso4pm25",
+            "concso4pm1",
             "concCecpm10",
             "concCecpm25",
             "concCocpm10",  # SURF_ugC_PM_OMCOARSE missing in model-output
             "concCocpm25",
+            "concom1",
             "concsspm10",
             "concsspm25",
             # Depositions
@@ -488,14 +492,18 @@ def get_CFG(reportyear, year, model_dir) -> dict:
         "concNno",
         "concCecpm25",
         "concCocpm25",
+        "concom1",
         "concCecpm10",
         "concCocpm10",
         #        "concnh4pm10", # no output in the model
         "concnh4pm25",
+        "concnh4pm1",
         #        "concso4pm10", # no output in the model
         "concso4pm25",
+        "concso4pm1",
         "concno3pm10",
         "concno3pm25",
+        "concno3pm1",
         "concsspm10",
         "concsspm25",
         "concso4t",
@@ -840,14 +848,18 @@ def get_CFG(reportyear, year, model_dir) -> dict:
             obs_vars=[
                 "concCecpm25",
                 "concCocpm25",
+                "concom1",
                 "concCecpm10",
                 "concCocpm10",
                 #                "concnh4pm10",
                 "concnh4pm25",
+                "concnh4pm1",
                 #                "concso4pm10",
                 "concso4pm25",
+                "concso4pm1",
                 "concno3pm10",
                 "concno3pm25",
+                "concno3pm1",
                 "concsspm10",
                 "concsspm25",
             ],
@@ -863,14 +875,18 @@ def get_CFG(reportyear, year, model_dir) -> dict:
             obs_vars=[
                 "concCecpm25",
                 "concCocpm25",
+                "concom1",
                 "concCecpm10",
                 "concCocpm10",
                 # "concnh4pm10",
                 "concnh4pm25",
+                "concnh4pm1",
                 #                "concso4pm10",
                 "concso4pm25",
+                "concso4pm1",
                 "concno3pm10",
                 "concno3pm25",
+                "concno3pm1",
                 "concsspm10",
                 "concsspm25",
             ],

pyaerocom/aeroval/data/var_web_info.ini

@@ -369,6 +369,11 @@ menu_name = NO3 PM25
 vertical_type = 3D
 category = Particle concentrations
 
+[concNno3pm1]
+menu_name = NO3 PM1
+vertical_type = 3D
+category = Particle concentrations
+
 [concno3pm10]
 menu_name = NO3 PM10
 vertical_type = 3D
@@ -379,6 +384,11 @@ menu_name = NO3 PM25
 vertical_type = 3D
 category = Particle concentrations
 
+[concno3pm1]
+menu_name = NO3 PM1
+vertical_type = 3D
+category = Particle concentrations
+
 [concnh4pm10]
 menu_name = NH4 PM10
 vertical_type = 3D
@@ -389,6 +399,11 @@ menu_name = NH4 PM2.5
 vertical_type = 3D
 category = Particle concentrations
 
+[concnh4pm1]
+menu_name = NH4 PM1
+vertical_type = 3D
+category = Particle concentrations
+
 [concso4t]
 menu_name = SO4 total
 vertical_type = 3D
@@ -409,6 +424,11 @@ menu_name = SO4 PM2.5
 vertical_type = 3D
 category = Particle concentrations
 
+[concso4pm1]
+menu_name = SO4 PM1
+vertical_type = 3D
+category = Particle concentrations
+
 [concss10]
 menu_name = SS PM10
 vertical_type = 3D
@@ -459,6 +479,11 @@ menu_name = OC PM2.5
 vertical_type = 3D
 category = Particle concentrations
 
+[concom1]
+menu_name = OM PM1
+vertical_type = 3D
+category = Particle concentrations
+
 [concom25]
 menu_name = OM PM2.5
 vertical_type = 3D

pyaerocom/aeroval/experiment_output.py

@@ -27,14 +27,14 @@
 )
 from pyaerocom.aeroval.json_utils import round_floats
 from pyaerocom.aeroval.modelentry import ModelEntry
-from pyaerocom.aeroval.setupclasses import EvalSetup
+from pyaerocom.aeroval import EvalSetup
 from pyaerocom.aeroval.varinfo_web import VarinfoWeb
 from pyaerocom.colocation.colocated_data import ColocatedData
 from pyaerocom.exceptions import EntryNotAvailable, VariableDefinitionError
 from pyaerocom.stats.mda8.const import MDA8_OUTPUT_VARS
 from pyaerocom.stats.stats import _init_stats_dummy
 from pyaerocom.utils import recursive_defaultdict
-from pyaerocom.variable_helpers import get_aliases
+from pyaerocom.variable_helpers import get_aliases, get_variable
 
 MapInfo = namedtuple(
     "MapInfo",
@@ -229,8 +229,10 @@ def _sync_heatmaps_with_menu_and_regions(self) -> None:
         with self.avdb.lock():
             menu = self.avdb.get_menu(self.proj_id, self.exp_id, default={})
             all_regions = self.avdb.get_regions(self.proj_id, self.exp_id, default={})
-            for fp in self.avdb.list_glob_stats(self.proj_id, self.exp_id):
-                data = self.avdb.get_by_uri(fp)
+            for uri in self.avdb.list_glob_stats(
+                self.proj_id, self.exp_id, access_type=aerovaldb.AccessType.URI
+            ):
+                data = self.avdb.get_by_uri(uri)
                 hm = {}
                 for vardisp, info in menu.items():
                     obs_dict = info["obs"]
@@ -250,7 +252,7 @@ def _sync_heatmaps_with_menu_and_regions(self) -> None:
                                 hm_data = self._check_hm_all_regions_avail(all_regions, hm_data)
                                 hm[vardisp][obs][vert_code][mod][modvar] = hm_data
 
-                self.avdb.put_by_uri(hm, fp)
+                self.avdb.put_by_uri(hm, uri)
 
     def _check_hm_all_regions_avail(self, all_regions, hm_data) -> dict:
         if all([x in hm_data for x in all_regions]):
@@ -413,7 +415,10 @@ def _check_clean_ts_file(self, fp) -> bool:
 
         with self.avdb.lock():
             try:
-                data = self.avdb.get_by_uri(fp)
+                # TODO: Hack to get uri. Ideally this should be rewritten to use URIs directly further
+                # up.
+                uri = self.avdb._get_uri_for_file(fp)
+                data = self.avdb.get_by_uri(uri)
             except Exception:
                 logger.exception(f"FATAL: detected corrupt json file: {fp}. Removing file...")
                 os.remove(fp)
@@ -433,7 +438,7 @@ def _check_clean_ts_file(self, fp) -> bool:
                     modified = True
                     logger.info(f"Removing data for model {mod_name} from ts file: {fp}")
 
-            self.avdb.put_by_uri(data_new, fp)
+            self.avdb.put_by_uri(data_new, uri)
         return modified
 
     def _clean_modelmap_files(self) -> list[str]:
@@ -534,7 +539,8 @@ def _get_cmap_info(self, var) -> dict[str, str | list[float]]:
             return var_ranges_defaults[var]
         try:
             varinfo = VarinfoWeb(var)
-            info = dict(scale=varinfo.cmap_bins, colmap=varinfo.cmap)
+            # TODO: get unit from pyaerocom/data/variables.ini
+            info = dict(scale=varinfo.cmap_bins, colmap=varinfo.cmap, unit=varinfo.unit)
         except (VariableDefinitionError, AttributeError):
             info = var_ranges_defaults["default"]
             logger.warning(
@@ -554,6 +560,7 @@ def _create_var_ranges_json(self) -> None:
             for var in all_vars:
                 if var not in ranges or ranges[var]["scale"] == []:
                     ranges[var] = self._get_cmap_info(var)
+                ranges[var]["unit"] = get_variable(var).units
             self.avdb.put_ranges(ranges, self.proj_id, self.exp_id)
 
     def _create_statistics_json(self) -> None:

pyaerocom/aeroval/experiment_processor.py

@@ -147,10 +147,15 @@ def run(self, model_name=None, obs_name=None, var_list=None, update_interface=Tr
 
         logger.info("Start processing")
 
-        # compute model maps (completely independent of obs-eval
-        # processing below)
+        # compute model maps (completely independent of obs-eval processing below)
         if self.cfg.webdisp_opts.add_model_maps:
             engine = ModelMapsEngine(self.cfg)
+
+            if isinstance(
+                self.cfg.modelmaps_opts.plot_types, dict
+            ):  # There may be additional obs networks to compute "model" maps for
+                model_list = list(set(model_list) and set(self.cfg.modelmaps_opts.plot_types))
+
             engine.run(model_list=model_list, var_list=var_list)
 
         if not self.cfg.processing_opts.only_model_maps:

pyaerocom/aeroval/fairmode_stats.py

@@ -15,6 +15,7 @@
 SPECIES = dict(
     concno2=dict(UrRV=0.24, RV=200, alpha=0.2),
     conco3=dict(UrRV=0.18, RV=120, alpha=0.79),
+    conco3mda8=dict(UrRV=0.18, RV=120, alpha=0.79),
     concpm10=dict(UrRV=0.28, RV=50, alpha=0.25),
     concpm25=dict(UrRV=0.36, RV=25, alpha=0.5),
 )
@@ -76,7 +77,7 @@ def fairmode_stats(obs_var: str, stats: dict) -> dict:
     assert np.isclose(
         rmsu * beta_mqi,
         np.sqrt((bias) ** 2 + (mod_std - obs_std) ** 2 + (2 * obs_std * mod_std * (1 - R))),
-        rtol=1e-3,
+        rtol=1e-2,
     ), "failed MQI check"
 
     fairmode = dict(

pyaerocom/aeroval/helpers.py

@@ -2,6 +2,8 @@
 import os
 from pathlib import Path
 
+from pydantic import BaseModel
+
 from pyaerocom import const
 from pyaerocom.aeroval.modelentry import ModelEntry
 from pyaerocom.griddeddata import GriddedData
@@ -202,3 +204,10 @@ def delete_dummy_model(model_id: str) -> None:
     for path in renamed.glob("*.nc"):
         print(f"Deleting dummy model {path}")
         path.unlink()
+
+
+class BoundingBox(BaseModel):
+    west: float
+    east: float
+    south: float
+    north: float