diff --git a/pyaerocom/io/pyaro/pyaro_config.py b/pyaerocom/io/pyaro/pyaro_config.py index 3ee96e529..4593d9315 100644 --- a/pyaerocom/io/pyaro/pyaro_config.py +++ b/pyaerocom/io/pyaro/pyaro_config.py @@ -29,7 +29,7 @@ class PyaroConfig(BaseModel): name: str data_id: str filename_or_obj_or_url: str - filters: dict[str, dict[str, list[str]]] + filters: dict[str, dict[str, list[str]] | dict[str, list[tuple]]] name_map: dict[str, str] | None = None # no Unit conversion option ########################## diff --git a/pyaerocom/io/pyaro/read_pyaro.py b/pyaerocom/io/pyaro/read_pyaro.py index 41cfff05f..b44f08249 100644 --- a/pyaerocom/io/pyaro/read_pyaro.py +++ b/pyaerocom/io/pyaro/read_pyaro.py @@ -4,6 +4,8 @@ from copy import deepcopy from typing import NewType +from tqdm import tqdm + import numpy as np from pyaro import list_timeseries_engines, open_timeseries from pyaro.timeseries import Data, Reader, Station @@ -97,7 +99,9 @@ class PyaroToUngriddedData: _DATAERRINDEX = 8 # col where errors can be stored _DATAFLAGINDEX = 9 # can be used to store flags _STOPTIMEINDEX = 10 # can be used to store stop time of acq. - _TRASHINDEX = 11 # index where invalid data can be moved to (e.g. when outliers are removed) + _TRASHINDEX = ( + 11 # index where invalid data can be moved to (e.g. when outliers are removed) + ) # List of keys needed by every station from Pyaro. Used to find extra metadata STATION_KEYS = ( @@ -161,9 +165,7 @@ def _convert_to_ungriddeddata(self, pyaro_data: dict[str, Data]) -> UngriddedDat metadata_idx = 0 for var, var_data in pyaro_data.items(): size = var_size[var] - for i in range( - 0, size - ): # The 1 start is a temp fix for the empty first row of the current Data implementation from pyaro + for i in tqdm(range(size), disable=None): data_line = var_data[i] current_station = data_line["stations"] @@ -190,7 +192,9 @@ def _convert_to_ungriddeddata(self, pyaro_data: dict[str, Data]) -> UngriddedDat # Fills meta_idx if station_idx[current_station][ts_type] not in meta_idx: - meta_idx[station_idx[current_station][ts_type]] = {v: [] for v in vars} + meta_idx[station_idx[current_station][ts_type]] = { + v: [] for v in vars + } meta_idx[station_idx[current_station][ts_type]][var].append(idx) @@ -200,7 +204,9 @@ def _convert_to_ungriddeddata(self, pyaro_data: dict[str, Data]) -> UngriddedDat for station_id in meta_idx: new_meta_idx[station_id] = {} for var_id in meta_idx[station_id]: - new_meta_idx[station_id][var_id] = np.array(meta_idx[station_id][var_id]) + new_meta_idx[station_id][var_id] = np.array( + meta_idx[station_id][var_id] + ) self.data._data = data_array self.data.meta_idx = new_meta_idx @@ -289,7 +295,9 @@ def _add_ts_type_to_metadata( for idx in new_metadata: station_name = new_metadata[idx]["station_name"] ts_type = str(ts_types[station_name]) - new_metadata[idx]["ts_type"] = ts_type if ts_type is not None else "undefined" + new_metadata[idx]["ts_type"] = ( + ts_type if ts_type is not None else "undefined" + ) return new_metadata def get_variables(self) -> list[str]: