Adds better definition of filters. Adds tqdm to read_pyaro

metno · Sep 5, 2024 · 178b003 · 178b003
1 parent 87d84b3
commit 178b003
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 8 deletions.
diff --git a/pyaerocom/io/pyaro/pyaro_config.py b/pyaerocom/io/pyaro/pyaro_config.py
@@ -29,7 +29,7 @@ class PyaroConfig(BaseModel):
     name: str
     data_id: str
     filename_or_obj_or_url: str
-    filters: dict[str, dict[str, list[str]]]
+    filters: dict[str, dict[str, list[str]] | dict[str, list[tuple]]]
     name_map: dict[str, str] | None = None  # no Unit conversion option
 
     ##########################

diff --git a/pyaerocom/io/pyaro/read_pyaro.py b/pyaerocom/io/pyaro/read_pyaro.py
@@ -4,6 +4,8 @@
 from copy import deepcopy
 from typing import NewType
 
+from tqdm import tqdm
+
 import numpy as np
 from pyaro import list_timeseries_engines, open_timeseries
 from pyaro.timeseries import Data, Reader, Station
@@ -97,7 +99,9 @@ class PyaroToUngriddedData:
     _DATAERRINDEX = 8  # col where errors can be stored
     _DATAFLAGINDEX = 9  # can be used to store flags
     _STOPTIMEINDEX = 10  # can be used to store stop time of acq.
-    _TRASHINDEX = 11  # index where invalid data can be moved to (e.g. when outliers are removed)
+    _TRASHINDEX = (
+        11  # index where invalid data can be moved to (e.g. when outliers are removed)
+    )
 
     # List of keys needed by every station from Pyaro. Used to find extra metadata
     STATION_KEYS = (
@@ -161,9 +165,7 @@ def _convert_to_ungriddeddata(self, pyaro_data: dict[str, Data]) -> UngriddedDat
         metadata_idx = 0
         for var, var_data in pyaro_data.items():
             size = var_size[var]
-            for i in range(
-                0, size
-            ):  # The 1 start is a temp fix for the empty first row of the current Data implementation from pyaro
+            for i in tqdm(range(size), disable=None):
                 data_line = var_data[i]
                 current_station = data_line["stations"]
 
@@ -190,7 +192,9 @@ def _convert_to_ungriddeddata(self, pyaro_data: dict[str, Data]) -> UngriddedDat
 
                 #  Fills meta_idx
                 if station_idx[current_station][ts_type] not in meta_idx:
-                    meta_idx[station_idx[current_station][ts_type]] = {v: [] for v in vars}
+                    meta_idx[station_idx[current_station][ts_type]] = {
+                        v: [] for v in vars
+                    }
 
                 meta_idx[station_idx[current_station][ts_type]][var].append(idx)
 
@@ -200,7 +204,9 @@ def _convert_to_ungriddeddata(self, pyaro_data: dict[str, Data]) -> UngriddedDat
         for station_id in meta_idx:
             new_meta_idx[station_id] = {}
             for var_id in meta_idx[station_id]:
-                new_meta_idx[station_id][var_id] = np.array(meta_idx[station_id][var_id])
+                new_meta_idx[station_id][var_id] = np.array(
+                    meta_idx[station_id][var_id]
+                )
 
         self.data._data = data_array
         self.data.meta_idx = new_meta_idx
@@ -289,7 +295,9 @@ def _add_ts_type_to_metadata(
         for idx in new_metadata:
             station_name = new_metadata[idx]["station_name"]
             ts_type = str(ts_types[station_name])
-            new_metadata[idx]["ts_type"] = ts_type if ts_type is not None else "undefined"
+            new_metadata[idx]["ts_type"] = (
+                ts_type if ts_type is not None else "undefined"
+            )
         return new_metadata
 
     def get_variables(self) -> list[str]: