Merge branch 'main-dev' into investigate1291

.git-blame-ignore-revs

@@ -26,3 +26,59 @@ b33da064f6ca28072d93340f8a9fdbcf219bcbd9
 9b7b52365480f5e9205012a849708b7ce7e201a5
 # 2023-09-29: format with black
 81d3cc9262f1746f0480a49ca804c9a2b970c17a
+# 2024-07-29: format with ruff
+d20d663e423b3e55a076a98ed136a4102e05a5df
+# 2024-07-29: Test for membership should be `not in`, E713
+071aac32021c449986d362e409304efb694c0fcf
+# 2024-07-29: Remove assignment to unused variable, F841
+09ada16f33fd9e962008f441cbbbdb7153288e10
+# 2024-07-29: Remove unused import, F401
+85a0b615888b9dca2b03498a4b5de0377e899bf5
+# 2024-07-29: format with ruff
+bf26bc28591df88bcfe738a4e9da935d37fc0b66
+# 2024-07-29:  Remove extraneous `f` prefix, F541
+edb47cfdecc9e0eafdc5d23de85efbdc7dfa38c4
+# 2024-07-29: Convert to `X | Y`, UP007
+93f2a57c2ec3d73eb895c4e4e63fa31c6aaee540
+# 2024-07-30: Convert to f-string
+9a80795c3ec6efad329bec01581f614104291f62
+# 2024-07-30: Replace with `is None`
+4562a88849833335ee158bf58433f93a05dbde98
+# 2024-07-30: replace type comparison
+a82a63067a671931821c1c731277e1e7eb1bbc09
+# 2024-07-30: Do not use bare `except`, E722
+5e1b0580e5b70290c3a858d746a823134fc74e42
+# 2024-07-30: Import from `collections.abc`, UP035
+3f93772e79f12ceb5503f52959ba91b1f3cef2b4
+# 2024-07-30: Replace with `list`, `tuple`, ... UP006
+699b1276bfb2f489551c2745a4cbf0a925bd4fc5
+# 2024-07-30: Replace with str.format & f-string, UP031 & UP032
+20e647ca2764a02286e87cf0209cbd6171e58836
+# 2024-07-30: Redundant open parameters, UP015
+395605b4dbf452b3aae41cdfb6ee2076a82b4bac
+# 2024-07-30: Use `X | Y` in `isinstance` call, UP038
+8276fde021937a673141160d4c0bd38c710f2835
+# 2024-07-30: Unused variable, F841
+44878d9ab6b3fab3b536f46949c108a04b8adcb5
+# 2024-07-30: Import shadowed by loop variable, F402
+44bece181bed06997b5e3bf36cd1d2e355846220
+# 2024-07-30: Remove extraneous parentheses, UP034
+9942c924d890ff9f5e17d40d32d7a39ee923b89f
+# 2024-07-30: Import shadowed by loop variable, F402
+6fecbfdffc64c4a9722af64ff9636db1d1a586b5
+# 2024-07-30: Use is True/False, E712
+333c74fbf86638b543376b3d09179696ea29e8fa
+# 2024-07-30: redundant & unused imports, F811 & F401
+2d1338e64b6eee629419bfdf57ee16534922afbf
+# 2024-07-30: Ambiguous variable name, E741
+e19876ff5653ceddcb047934989ad09ab7ecef30
+# 2024-07-30: Define functions instead of assign lambdas, E731
+nc5a8b4b2b709d9b7adae01e253f638d7d8852ee3
+# 2024-07-30: f-string without any placeholders, F541
+96fae16fbb35ad9f87d93cbe9aaccb3ef05180ec
+# 2024-07-30: Unnecessary `int`/`float` call, UP018
+c23eebb590baa98771a66b9be5147aa137829af1
+# 2024-07-30: Use `super()` instead of `super(__class__, self)`, UP008
+6c6af205832e38de344111861b3ae2b0651bd8a1
+# 2024-07-30: Test for object identity should be `is not`, E714
+5ffd364713f32b72758177cf9a0fdfe020d3dd1d

.github/workflows/CI.yml

@@ -28,9 +28,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         python -m pip install tox
-    - name: Check format
-      run: tox -e format
-    - name: Run linters
+    - name: Check format and Run linter
       run: tox -e lint
     - name: Build docs
       run: tox -e docs

.pre-commit-config.yaml

@@ -1,9 +1,6 @@
 repos:
-- repo: https://github.com/psf/black
-  rev: 23.9.1
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.5.5
   hooks:
-  - id: black
-- repo: https://github.com/pycqa/isort
-  rev: 5.12.0
-  hooks:
-  - id: isort
+    - id: ruff-format
+    - id: ruff # linter

pyaerocom/_concprcp_units_helpers.py

@@ -22,12 +22,8 @@ def translate_rate_units_implicit(unit_implicit, ts_type):
 
     # check if unit is explicitly defined as implicit and if yes add frequency
     # string
-    found = False
-    for imp_unit in DEP_IMPLICIT_UNITS:
-        if unit == imp_unit:
-            unit = f"{imp_unit} {freq_si}-1"
-            found = True
-            break
+    if unit in DEP_IMPLICIT_UNITS:
+        unit = f"{unit} {freq_si}-1"
 
     # Check if frequency in unit corresponds to sampling frequency (e.g.
     # ug m-2 h-1 for hourly data).
@@ -40,7 +36,7 @@ def translate_rate_units_implicit(unit_implicit, ts_type):
 
     # for now, raise NotImplementedError if wdep unit is, e.g. ug m-2 s-1 but
     # ts_type is hourly (later, use units_helpers.implicit_to_explicit_rates)
-    if not freq_si_str in str(unit):
+    if freq_si_str not in str(unit):
         raise NotImplementedError(
             f"Cannot yet handle wdep in {unit} but {freq} sampling frequency"
         )
@@ -87,7 +83,7 @@ def check_pr_units(gridded):  # pragma: no cover
 
     # for now, raise NotImplementedError if wdep unit is, e.g. ug m-2 s-1 but
     # ts_type is hourly (later, use units_helpers.implicit_to_explicit_rates)
-    if not freq_si_str in str(unit):
+    if freq_si_str not in str(unit):
         raise NotImplementedError(
             f"Cannot yet handle wdep in {unit} but {freq} sampling frequency"
         )
@@ -113,7 +109,7 @@ def _check_prlim_units(prlim, prlim_units):  # pragma: no cover
     prlim_units = f"m {spl[1]}"
     prlim_freq = spl[1][:-2]  # it endswith -1
     # convert the freque
-    if not prlim_freq in SI_TO_TS_TYPE:
+    if prlim_freq not in SI_TO_TS_TYPE:
         raise ValueError(
             f"frequency in prlim_units must be either of the "
             f"following values: {list(SI_TO_TS_TYPE)}."

pyaerocom/_logging.py

@@ -72,7 +72,7 @@ def change_verbosity(level: str | int) -> None:
     logger = logging.getLogger("")
     assert logger.handlers, f"{logger.name} logger has not been configured correctly"
     for handler in logger.handlers:
-        if type(handler) == logging.StreamHandler:
+        if type(handler) is logging.StreamHandler:
             handler.setLevel(level)
 
 

pyaerocom/_lowlevel_helpers.py

@@ -293,7 +293,7 @@ def __setitem__(self, key, val) -> None:
         if bool(self.SETTER_CONVERT):
             for fromtp, totp in self.SETTER_CONVERT.items():
                 if isinstance(val, fromtp):
-                    if fromtp == dict:
+                    if fromtp is dict:
                         val = totp(**val)
                     else:
                         val = totp(val)
@@ -378,7 +378,7 @@ def import_from(self, other) -> None:
         None
 
         """
-        if not isinstance(other, (dict, BrowseDict)):
+        if not isinstance(other, dict | BrowseDict):
             raise ValueError("need dict-like object")
         for key, val in other.items():
             if key in self:
@@ -427,7 +427,7 @@ def _invoke_dtype(self, current_tp, val):
 
     def _check_valtype(self, key, val):
         current_tp = type(self[key])
-        if type(val) != current_tp and isinstance(self[key], BrowseDict):
+        if type(val) is not current_tp and isinstance(self[key], BrowseDict):
             val = current_tp(**val)
         return val
 
@@ -438,7 +438,7 @@ def _setitem_checker(self, key, val):
         ----
         Only used in __setitem__ not in __setattr__.
         """
-        if not key in dir(self):
+        if key not in dir(self):
             if self.CRASH_ON_INVALID:
                 raise ValueError(f"Invalid key {key}")
             logger.warning(f"Invalid key {key} in {self._class_name}. Will be ignored.")
@@ -448,7 +448,7 @@ def _setitem_checker(self, key, val):
         val = self._check_valtype(key, val)
         current_tp = type(current)
 
-        if not current is None and not isinstance(val, current_tp):
+        if current is not None and not isinstance(val, current_tp):
             raise ValueError(
                 f"Invalid type {type(val)} for key: {key}. Need {current_tp} "
                 f"(Current value: {current})"
@@ -462,7 +462,7 @@ def _occurs_in(self, key) -> list:
         if key in self:
             objs.append(self)
         for k, v in self.items():
-            if isinstance(v, (dict, BrowseDict)) and key in v:
+            if isinstance(v, dict | BrowseDict) and key in v:
                 objs.append(v)
             if len(objs) > 1:
                 print(key, "is contained in multiple containers ", objs)
@@ -474,7 +474,7 @@ def keys_unnested(self) -> list:
             keys.append(key)
             if isinstance(val, NestedContainer):
                 keys.extend(val.keys_unnested())
-            elif isinstance(val, (ConstrainedContainer, dict)):
+            elif isinstance(val, ConstrainedContainer | dict):
                 for subkey, subval in val.items():
                     keys.append(subkey)
         return keys
@@ -519,7 +519,7 @@ def merge_dicts(dict1, dict2, discard_failing=True):
     for key, val in dict2.items():
         try:
             # entry does not exist in first dict or is None
-            if not key in new or new[key] is None:
+            if key not in new or new[key] is None:
                 new[key] = val
                 continue
             # get value of first input dict
@@ -531,11 +531,11 @@ def merge_dicts(dict1, dict2, discard_failing=True):
             try:
                 if this == val:
                     continue
-            except:
+            except Exception:
                 try:
                     if (this == val).all():
                         continue
-                except:
+                except Exception:
                     pass
 
             # both values are strings, merge with ';' delim
@@ -544,7 +544,7 @@ def merge_dicts(dict1, dict2, discard_failing=True):
 
             elif isinstance(this, list) and isinstance(val, list):
                 for item in val:
-                    if not item in this:
+                    if item not in this:
                         this.append(item)
                 new[key] = this
 
@@ -559,7 +559,7 @@ def merge_dicts(dict1, dict2, discard_failing=True):
                     lst = val
                     check = this  # this is not list
                 for item in lst:
-                    if not type(item) == type(check):
+                    if type(item) is not type(check):
                         raise ValueError(
                             f"Cannot merge key {key} since items in {lst} "
                             f"are of different type, that does not match {check}"
@@ -653,7 +653,7 @@ def sort_dict_by_name(d, pref_list: list = None) -> dict:
         if k in d:
             s[k] = d[k]
     for k in sorted_keys:
-        if not k in pref_list:
+        if k not in pref_list:
             s[k] = d[k]
     return s
 
@@ -689,7 +689,7 @@ def dict_to_str(dictionary, indent=0, ignore_null=False):
     for key, val in dictionary.items():
         if ignore_null and val is None:
             continue
-        elif isinstance(val, (dict, BrowseDict)):
+        elif isinstance(val, dict | BrowseDict):
             val = dict_to_str(val, indent + 2)
         elif isinstance(val, list):
             val = list_to_shortstr(val, indent=indent)

pyaerocom/aeroval/_processing_base.py

@@ -3,11 +3,8 @@
 from pyaerocom._lowlevel_helpers import TypeValidator
 from pyaerocom.aeroval import EvalSetup
 from pyaerocom.aeroval.experiment_output import ExperimentOutput
-from pyaerocom.aeroval.json_utils import round_floats
-from pyaerocom.colocation.colocated_data import ColocatedData
 from pyaerocom.colocation.colocation_setup import ColocationSetup
 from pyaerocom.colocation.colocator import Colocator
-from pyaerocom.utils import recursive_defaultdict
 
 
 class HasConfig:

pyaerocom/aeroval/aux_io_helpers.py

@@ -124,7 +124,7 @@ def import_module(self):
 
         """
         moddir, fname = os.path.split(self.aux_file)
-        if not moddir in sys.path:
+        if moddir not in sys.path:
             sys.path.append(moddir)
         modname = fname.split(".")[0]
         return importlib.import_module(modname)

pyaerocom/aeroval/coldatatojson_engine.py

@@ -1,5 +1,4 @@
 import logging
-import os
 from time import time
 
 from cf_units import Unit
@@ -90,7 +89,6 @@ def process_coldata(self, coldata: ColocatedData):
         annual_stats_constrained = self.cfg.statistics_opts.annual_stats_constrained
 
         out_dirs = self.cfg.path_manager.get_json_output_dirs(True)
-        regions_json = self.exp_output.regions_file
         regions_how = self.cfg.webdisp_opts.regions_how
 
         stats_min_num = self.cfg.statistics_opts.MIN_NUM
@@ -131,10 +129,10 @@ def process_coldata(self, coldata: ColocatedData):
             raise NotImplementedError(
                 "Cannot yet apply country filtering for 4D colocated data instances"
             )
-        elif not main_freq in freqs:
+        elif main_freq not in freqs:
             raise ConfigError(f"main_freq {main_freq} is not in experiment frequencies: {freqs}")
         if self.cfg.statistics_opts.stats_tseries_base_freq is not None:
-            if not self.cfg.statistics_opts.stats_tseries_base_freq in freqs:
+            if self.cfg.statistics_opts.stats_tseries_base_freq not in freqs:
                 raise ConfigError(
                     f"Base frequency for statistics timeseries needs to be "
                     f"specified in experiment frequencies: {freqs}"
@@ -225,21 +223,17 @@ def process_coldata(self, coldata: ColocatedData):
                     regions_how=regions_how,
                     regnames=regnames,
                     meta_glob=meta_glob,
-                    out_dirs=out_dirs,
                 )
         else:
-            logger.info("Processing profile data for vizualization")
+            logger.info("Processing profile data for visualization")
 
-            self._process_profile_data_for_vizualization(
+            self._process_profile_data_for_visualization(
                 data=data,
                 use_country=use_country,
                 region_names=regnames,
                 station_names=coldata.data.station_name.values,
                 periods=periods,
                 seasons=seasons,
-                obs_name=obs_name,
-                var_name_web=var_name_web,
-                out_dirs=out_dirs,
             )
 
         logger.info(
@@ -274,19 +268,16 @@ def _get_vert_code(self, coldata: ColocatedData = None):
             vert_code = coldata.get_meta_item("vert_code")
         return vert_code
 
-    def _process_profile_data_for_vizualization(
+    def _process_profile_data_for_visualization(
         self,
         data: dict[str, ColocatedData] = None,
         use_country: bool = False,
         region_names: dict[str:str] = None,
         station_names: ArrayLike = None,
         periods: tuple[str, ...] = None,
         seasons: tuple[str, ...] = None,
-        obs_name: str = None,
-        var_name_web: str = None,
-        out_dirs: dict = None,
     ):
-        if region_names == None and station_names == None:
+        if region_names is None and station_names is None:
             raise ValueError("Both region_id and station_name can not both be None")
 
         # Loop through regions
@@ -450,14 +441,13 @@ def _process_diurnal_profiles(
         regions_how: str = "default",
         regnames=None,
         meta_glob: dict = None,
-        out_dirs: dict = None,
     ):
-        (ts_objs_weekly, ts_objs_weekly_reg) = _process_sites_weekly_ts(
+        ts_objs_weekly, ts_objs_weekly_reg = _process_sites_weekly_ts(
             coldata, regions_how, regnames, meta_glob
         )
-        outdir = os.path.join(out_dirs["ts/diurnal"])
+
         for ts_data_weekly in ts_objs_weekly:
             self.exp_output.write_station_data(ts_data_weekly)
-        if ts_objs_weekly_reg != None:
+        if ts_objs_weekly_reg is not None:
             for ts_data_weekly_reg in ts_objs_weekly_reg:
                 self.exp_output.write_station_data(ts_data_weekly_reg)