merge main-dev to keep current

Merge branch 'main-dev' into earlinet

.github/workflows/CI.yml

@@ -10,7 +10,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     env:
-      PYTHON: 3.8
+      PYTHON: 3.9
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ env.PYTHON }}
@@ -35,9 +35,9 @@ jobs:
       run: tox -e lint
 
   docs:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     env:
-      PYTHON: 3.8
+      PYTHON: 3.9
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ env.PYTHON }}
@@ -62,13 +62,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8, 3.9, '3.10']
+        python-version: [3.9, '3.10', '3.11']
         experimental: [false]
         os: [ubuntu-22.04]
-        include:
-          - python-version: '3.11'
-            experimental: true        
-            os: ubuntu-22.04
+        #include:
+        #  - python-version: '3.12'
+        #    experimental: true        
+        #    os: ubuntu-22.04
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
@@ -105,7 +105,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8, 3.9, '3.10']
+        python-version: [3.9, '3.10', '3.11']
         experimental: [false]
     defaults:
       run:

.readthedocs.yaml

@@ -11,20 +11,20 @@ formats:
   - pdf
 
 build:
-  os: ubuntu-20.04
+  os: ubuntu-22.04
   apt_packages:
     - libudunits2-dev
     - libgeos-dev
     - libproj-dev
     - proj-data
     - proj-bin
   tools:
-    python: "3.8"
+    python: "3.9"
 
 python:
   install:
     - method: pip
       path: .
       extra_requirements:
-        - proj-legacy
+        - proj8
         - docs

docs/intro.rst

@@ -24,7 +24,7 @@ Main features
 	- `EBAS database </>`__.
 	- `EEA Air Quality e-Reporting (AQ e-Reporting) <https://www.eea.europa.eu/data-and-maps/data/aqereporting-9>`__.
 	- `AirNow <https://www.airnow.gov/about-the-data/>`__.
-	- `MarcoPolo <https://www.knmi.nl/kennis-en-datacentrum/project/marcopolo>`__.
+	- `Ministry of Environment Protection (MEP) in China`__.
 	- `GHOST` (Globally Harmonised Observational Surface Treatment) (see e.g., `Petetin et al., 2020 <https://acp.copernicus.org/articles/20/11119/2020/acp-20-11119-2020.html>`_ for more information).
 
 - Reading routines for level 3 gridded satellite observations, such as:

pyaerocom/_warnings.py

@@ -1,12 +1,9 @@
-from __future__ import annotations
-
 import warnings
 from contextlib import contextmanager
-from typing import Type
 
 
 @contextmanager
-def ignore_warnings(category: Type[Warning], *messages: str):
+def ignore_warnings(category: type[Warning], *messages: str):
     """
     Ignore particular warnings with a decorator or context manager
 

pyaerocom/aeroval/experiment_output.py

@@ -617,11 +617,21 @@ def _get_var_name_and_type(self, var_name):
 
     def _init_menu_entry(self, var: str) -> dict:
         name, tp, cat = self._get_var_name_and_type(var)
+        out = {"type": tp, "cat": cat, "name": name, "obs": {}}
         try:
             lname = const.VARS[var].description
         except VariableDefinitionError:
             lname = "UNDEFINED"
-        return {"type": tp, "cat": cat, "name": name, "longname": lname, "obs": {}}
+
+        out["longname"] = lname
+        try:
+            # Comes in as a string. split() here breaks up based on space and returns either just the element in a list or the components of the string in a list
+            only_use_in = const.VARS[var].only_use_in.split(" ")
+            # only return only_use_in if key exists, otherwise do not
+            out["only_use_in"] = only_use_in
+        except AttributeError:
+            pass
+        return out
 
     def _check_ovar_mvar_entry(self, mcfg, mod_var, ocfg, obs_var):
 

pyaerocom/aeroval/experiment_processor.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import logging
 from multiprocessing import dummy

pyaerocom/aeroval/glob_defaults.py

@@ -137,7 +137,6 @@
         "colmap": "bwr",
         "unit": "var",
         "decimals": 1,
-        "forecast": True,
     },
     "mab": {
         "name": "MAB",
@@ -154,7 +153,6 @@
         "colmap": "bwr",
         "unit": "var",
         "decimals": 1,
-        "forecast": True,
     },
     "R": {
         "name": "R",
@@ -323,6 +321,13 @@
     conco3=["O3", "3D", "Gas concentrations"],
     concno310=["NO3_PM10", "3D", "Particle concentration"],
     concno325=["NO3_PM25", "3D", "Particle concentration"],
+    proxyod550bc=["OD (Black Carbon)", "2D", "Optical properties"],
+    proxyod550dust=["OD (Dust)", "2D", "Optical properties"],
+    proxyod550oa=["OD (Organic Matter)", "2D", "Optical properties"],
+    proxyod550so4=["OD (SO4)", "2D", "Optical properties"],
+    proxyod550ss=["OD (Sea Salt)", "2D", "Optical properties"],
+    proxyod550nh4=["OD (NH4)", "2D", "Optical properties"],
+    proxyod550no3=["OD (NO3)", "2D", "Optical properties"],
     # Gases
     concNno=["NO", "3D", "Concentration"],
     concno2=["NO2", "3D", "Gas concentrations"],

pyaerocom/data/variables.ini

@@ -67,6 +67,7 @@ standard_name = atmosphere_optical_thickness_due_to_water_in_ambient_aerosol_par
 use=od550aer
 description = Aerosol optical depth (AOD) at 550 nm due to water
 standard_name = atmosphere_optical_thickness_due_to_water_in_ambient_aerosol_particles
+only_use_in = maps.php
 
 [od550bc]
 use=od550aer
@@ -77,6 +78,7 @@ standard_name = atmosphere_optical_thickness_due_to_elemental_carbon_ambient_aer
 use=od550aer
 description = Aerosol optical depth (AOD) at 550 nm due to elemental carbon
 standard_name = atmosphere_optical_thickness_due_to_elemental_carbon_ambient_aerosol_particles
+only_use_in = maps.php
 
 [od550dust]
 use=od550aer
@@ -87,6 +89,7 @@ standard_name = atmosphere_optical_thickness_due_to_dust_ambient_aerosol_particl
 use=od550aer
 description = Aerosol optical depth (AOD) at 550 nm due to dust
 standard_name = atmosphere_optical_thickness_due_to_dust_ambient_aerosol_particles
+only_use_in = maps.php
 
 [od550nh4]
 use=od550aer
@@ -97,12 +100,19 @@ standard_name = atmosphere_optical_thickness_due_to_ammonium_ambient_aerosol_par
 use=od550aer
 description = Aerosol optical depth (AOD) at 550 nm due to Ammonium
 standard_name = atmosphere_optical_thickness_due_to_ammonium_ambient_aerosol_particles
+only_use_in = maps.php
 
 [od550no3]
 use=od550aer
 description = Aerosol optical depth (AOD) at 550 nm due to nitrate
 standard_name = atmosphere_optical_thickness_due_to_nitrate_ambient_aerosol_particles
 
+[proxyod550no3]
+use=od550aer
+description = Aerosol optical depth (AOD) at 550 nm due to Nitrate
+standard_name = atmosphere_optical_thickness_due_to_nitrate_ambient_aerosol_particles
+only_use_in = maps.php
+
 [od550oa]
 use=od550aer
 description = Aerosol optical depth (AOD) at 550 nm due to organic matter
@@ -112,6 +122,7 @@ standard_name = atmosphere_optical_thickness_due_to_particulate_organic_matter_a
 use=od550aer
 description = Aerosol optical depth (AOD) at 550 nm due to organic matter
 standard_name = atmosphere_optical_thickness_due_to_particulate_organic_matter_ambient_aerosol_particles
+only_use_in = maps.php
 
 [od550pm1]
 use=od550aer
@@ -152,6 +163,7 @@ standard_name = atmosphere_optical_thickness_due_to_sulfate_ambient_aerosol_part
 use=od550aer
 description = Aerosol optical depth (AOD) at 550 nm due to SO4
 standard_name = atmosphere_optical_thickness_due_to_sulfate_ambient_aerosol_particles
+only_use_in = maps.php
 
 [od550ss]
 use=od550aer
@@ -162,6 +174,7 @@ standard_name = atmosphere_optical_thickness_due_to_seasalt_ambient_aerosol_part
 use=od550aer
 description = Aerosol optical depth (AOD) at 550 nm due to sea salt
 standard_name = atmosphere_optical_thickness_due_to_seasalt_ambient_aerosol_particles
+only_use_in  = maps.php
 
 [od550lt1ss]
 use = od550ss

pyaerocom/io/__init__.py

@@ -29,6 +29,5 @@
 from .read_ebas import ReadEbas
 from .read_eea_aqerep import ReadEEAAQEREP
 from .read_eea_aqerep_v2 import ReadEEAAQEREP_V2
-from .read_marcopolo import ReadMarcoPolo
 
 from . import helpers_units

pyaerocom/io/read_aeronet_sunv3.py

@@ -30,7 +30,7 @@ class ReadAeronetSunV3(ReadAeronetBase):
     _FILEMASK = "*.lev*"
 
     #: version log of this class (for caching)
-    __version__ = "0.11_" + ReadAeronetBase.__baseversion__
+    __version__ = "0.12_" + ReadAeronetBase.__baseversion__
 
     #: Name of dataset (OBS_ID)
     DATA_ID = const.AERONET_SUN_V3L2_AOD_DAILY_NAME
@@ -99,6 +99,7 @@ class ReadAeronetSunV3(ReadAeronetBase):
         "proxyod550oa": ["od440aer", "od500aer", "ang4487aer"],
         "proxyod550so4": ["od440aer", "od500aer", "ang4487aer"],
         "proxyod550ss": ["od440aer", "od500aer", "ang4487aer"],
+        "proxyod550no3": ["od440aer", "od500aer", "ang4487aer"],
     }
 
     #: Functions that are used to compute additional variables (i.e. one
@@ -114,6 +115,7 @@ class ReadAeronetSunV3(ReadAeronetBase):
         "proxyod550oa": calc_od550aer,
         "proxyod550so4": calc_od550aer,
         "proxyod550ss": calc_od550aer,
+        "proxyod550no3": calc_od550aer,
     }
 
     #: List of variables that are provided by this dataset (will be extended
@@ -165,10 +167,10 @@ def read_file(self, filename, vars_to_retrieve=None, vars_as_series=False):
             f_out.close()
 
         try:
-            with open(filename, "rt") as in_file:
+            with open(filename) as in_file:
                 lines = in_file.readlines()
         except UnicodeDecodeError:
-            with open(filename, "rt", encoding="ISO-8859-1") as in_file:
+            with open(filename, encoding="ISO-8859-1") as in_file:
                 lines = in_file.readlines()
         except OSError:
             # faulty gzip file, but also the gzip class raises some exceptions