diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index ba7eafa26..099b05e96 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -26,3 +26,59 @@ b33da064f6ca28072d93340f8a9fdbcf219bcbd9 9b7b52365480f5e9205012a849708b7ce7e201a5 # 2023-09-29: format with black 81d3cc9262f1746f0480a49ca804c9a2b970c17a +# 2024-07-29: format with ruff +d20d663e423b3e55a076a98ed136a4102e05a5df +# 2024-07-29: Test for membership should be `not in`, E713 +071aac32021c449986d362e409304efb694c0fcf +# 2024-07-29: Remove assignment to unused variable, F841 +09ada16f33fd9e962008f441cbbbdb7153288e10 +# 2024-07-29: Remove unused import, F401 +85a0b615888b9dca2b03498a4b5de0377e899bf5 +# 2024-07-29: format with ruff +bf26bc28591df88bcfe738a4e9da935d37fc0b66 +# 2024-07-29: Remove extraneous `f` prefix, F541 +edb47cfdecc9e0eafdc5d23de85efbdc7dfa38c4 +# 2024-07-29: Convert to `X | Y`, UP007 +93f2a57c2ec3d73eb895c4e4e63fa31c6aaee540 +# 2024-07-30: Convert to f-string +9a80795c3ec6efad329bec01581f614104291f62 +# 2024-07-30: Replace with `is None` +4562a88849833335ee158bf58433f93a05dbde98 +# 2024-07-30: replace type comparison +a82a63067a671931821c1c731277e1e7eb1bbc09 +# 2024-07-30: Do not use bare `except`, E722 +5e1b0580e5b70290c3a858d746a823134fc74e42 +# 2024-07-30: Import from `collections.abc`, UP035 +3f93772e79f12ceb5503f52959ba91b1f3cef2b4 +# 2024-07-30: Replace with `list`, `tuple`, ... UP006 +699b1276bfb2f489551c2745a4cbf0a925bd4fc5 +# 2024-07-30: Replace with str.format & f-string, UP031 & UP032 +20e647ca2764a02286e87cf0209cbd6171e58836 +# 2024-07-30: Redundant open parameters, UP015 +395605b4dbf452b3aae41cdfb6ee2076a82b4bac +# 2024-07-30: Use `X | Y` in `isinstance` call, UP038 +8276fde021937a673141160d4c0bd38c710f2835 +# 2024-07-30: Unused variable, F841 +44878d9ab6b3fab3b536f46949c108a04b8adcb5 +# 2024-07-30: Import shadowed by loop variable, F402 +44bece181bed06997b5e3bf36cd1d2e355846220 +# 2024-07-30: Remove extraneous parentheses, UP034 +9942c924d890ff9f5e17d40d32d7a39ee923b89f +# 2024-07-30: Import shadowed by loop variable, F402 +6fecbfdffc64c4a9722af64ff9636db1d1a586b5 +# 2024-07-30: Use is True/False, E712 +333c74fbf86638b543376b3d09179696ea29e8fa +# 2024-07-30: redundant & unused imports, F811 & F401 +2d1338e64b6eee629419bfdf57ee16534922afbf +# 2024-07-30: Ambiguous variable name, E741 +e19876ff5653ceddcb047934989ad09ab7ecef30 +# 2024-07-30: Define functions instead of assign lambdas, E731 +nc5a8b4b2b709d9b7adae01e253f638d7d8852ee3 +# 2024-07-30: f-string without any placeholders, F541 +96fae16fbb35ad9f87d93cbe9aaccb3ef05180ec +# 2024-07-30: Unnecessary `int`/`float` call, UP018 +c23eebb590baa98771a66b9be5147aa137829af1 +# 2024-07-30: Use `super()` instead of `super(__class__, self)`, UP008 +6c6af205832e38de344111861b3ae2b0651bd8a1 +# 2024-07-30: Test for object identity should be `is not`, E714 +5ffd364713f32b72758177cf9a0fdfe020d3dd1d diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d3b0913fb..2ffed779a 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -28,9 +28,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install tox - - name: Check format - run: tox -e format - - name: Run linters + - name: Check format and Run linter run: tox -e lint - name: Build docs run: tox -e docs diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 520bffe59..4ff8ce795 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,9 +1,6 @@ repos: -- repo: https://github.com/psf/black - rev: 23.9.1 +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.5.5 hooks: - - id: black -- repo: https://github.com/pycqa/isort - rev: 5.12.0 - hooks: - - id: isort + - id: ruff-format + - id: ruff # linter diff --git a/pyaerocom/_concprcp_units_helpers.py b/pyaerocom/_concprcp_units_helpers.py index 1621db58d..376d44644 100644 --- a/pyaerocom/_concprcp_units_helpers.py +++ b/pyaerocom/_concprcp_units_helpers.py @@ -22,12 +22,8 @@ def translate_rate_units_implicit(unit_implicit, ts_type): # check if unit is explicitly defined as implicit and if yes add frequency # string - found = False - for imp_unit in DEP_IMPLICIT_UNITS: - if unit == imp_unit: - unit = f"{imp_unit} {freq_si}-1" - found = True - break + if unit in DEP_IMPLICIT_UNITS: + unit = f"{unit} {freq_si}-1" # Check if frequency in unit corresponds to sampling frequency (e.g. # ug m-2 h-1 for hourly data). @@ -40,7 +36,7 @@ def translate_rate_units_implicit(unit_implicit, ts_type): # for now, raise NotImplementedError if wdep unit is, e.g. ug m-2 s-1 but # ts_type is hourly (later, use units_helpers.implicit_to_explicit_rates) - if not freq_si_str in str(unit): + if freq_si_str not in str(unit): raise NotImplementedError( f"Cannot yet handle wdep in {unit} but {freq} sampling frequency" ) @@ -87,7 +83,7 @@ def check_pr_units(gridded): # pragma: no cover # for now, raise NotImplementedError if wdep unit is, e.g. ug m-2 s-1 but # ts_type is hourly (later, use units_helpers.implicit_to_explicit_rates) - if not freq_si_str in str(unit): + if freq_si_str not in str(unit): raise NotImplementedError( f"Cannot yet handle wdep in {unit} but {freq} sampling frequency" ) @@ -113,7 +109,7 @@ def _check_prlim_units(prlim, prlim_units): # pragma: no cover prlim_units = f"m {spl[1]}" prlim_freq = spl[1][:-2] # it endswith -1 # convert the freque - if not prlim_freq in SI_TO_TS_TYPE: + if prlim_freq not in SI_TO_TS_TYPE: raise ValueError( f"frequency in prlim_units must be either of the " f"following values: {list(SI_TO_TS_TYPE)}." diff --git a/pyaerocom/_logging.py b/pyaerocom/_logging.py index b152f1674..bbe63d879 100644 --- a/pyaerocom/_logging.py +++ b/pyaerocom/_logging.py @@ -72,7 +72,7 @@ def change_verbosity(level: str | int) -> None: logger = logging.getLogger("") assert logger.handlers, f"{logger.name} logger has not been configured correctly" for handler in logger.handlers: - if type(handler) == logging.StreamHandler: + if type(handler) is logging.StreamHandler: handler.setLevel(level) diff --git a/pyaerocom/_lowlevel_helpers.py b/pyaerocom/_lowlevel_helpers.py index 0c676e472..18fb48deb 100644 --- a/pyaerocom/_lowlevel_helpers.py +++ b/pyaerocom/_lowlevel_helpers.py @@ -293,7 +293,7 @@ def __setitem__(self, key, val) -> None: if bool(self.SETTER_CONVERT): for fromtp, totp in self.SETTER_CONVERT.items(): if isinstance(val, fromtp): - if fromtp == dict: + if fromtp is dict: val = totp(**val) else: val = totp(val) @@ -378,7 +378,7 @@ def import_from(self, other) -> None: None """ - if not isinstance(other, (dict, BrowseDict)): + if not isinstance(other, dict | BrowseDict): raise ValueError("need dict-like object") for key, val in other.items(): if key in self: @@ -427,7 +427,7 @@ def _invoke_dtype(self, current_tp, val): def _check_valtype(self, key, val): current_tp = type(self[key]) - if type(val) != current_tp and isinstance(self[key], BrowseDict): + if type(val) is not current_tp and isinstance(self[key], BrowseDict): val = current_tp(**val) return val @@ -438,7 +438,7 @@ def _setitem_checker(self, key, val): ---- Only used in __setitem__ not in __setattr__. """ - if not key in dir(self): + if key not in dir(self): if self.CRASH_ON_INVALID: raise ValueError(f"Invalid key {key}") logger.warning(f"Invalid key {key} in {self._class_name}. Will be ignored.") @@ -448,7 +448,7 @@ def _setitem_checker(self, key, val): val = self._check_valtype(key, val) current_tp = type(current) - if not current is None and not isinstance(val, current_tp): + if current is not None and not isinstance(val, current_tp): raise ValueError( f"Invalid type {type(val)} for key: {key}. Need {current_tp} " f"(Current value: {current})" @@ -462,7 +462,7 @@ def _occurs_in(self, key) -> list: if key in self: objs.append(self) for k, v in self.items(): - if isinstance(v, (dict, BrowseDict)) and key in v: + if isinstance(v, dict | BrowseDict) and key in v: objs.append(v) if len(objs) > 1: print(key, "is contained in multiple containers ", objs) @@ -474,7 +474,7 @@ def keys_unnested(self) -> list: keys.append(key) if isinstance(val, NestedContainer): keys.extend(val.keys_unnested()) - elif isinstance(val, (ConstrainedContainer, dict)): + elif isinstance(val, ConstrainedContainer | dict): for subkey, subval in val.items(): keys.append(subkey) return keys @@ -519,7 +519,7 @@ def merge_dicts(dict1, dict2, discard_failing=True): for key, val in dict2.items(): try: # entry does not exist in first dict or is None - if not key in new or new[key] is None: + if key not in new or new[key] is None: new[key] = val continue # get value of first input dict @@ -531,11 +531,11 @@ def merge_dicts(dict1, dict2, discard_failing=True): try: if this == val: continue - except: + except Exception: try: if (this == val).all(): continue - except: + except Exception: pass # both values are strings, merge with ';' delim @@ -544,7 +544,7 @@ def merge_dicts(dict1, dict2, discard_failing=True): elif isinstance(this, list) and isinstance(val, list): for item in val: - if not item in this: + if item not in this: this.append(item) new[key] = this @@ -559,7 +559,7 @@ def merge_dicts(dict1, dict2, discard_failing=True): lst = val check = this # this is not list for item in lst: - if not type(item) == type(check): + if type(item) is not type(check): raise ValueError( f"Cannot merge key {key} since items in {lst} " f"are of different type, that does not match {check}" @@ -653,7 +653,7 @@ def sort_dict_by_name(d, pref_list: list = None) -> dict: if k in d: s[k] = d[k] for k in sorted_keys: - if not k in pref_list: + if k not in pref_list: s[k] = d[k] return s @@ -689,7 +689,7 @@ def dict_to_str(dictionary, indent=0, ignore_null=False): for key, val in dictionary.items(): if ignore_null and val is None: continue - elif isinstance(val, (dict, BrowseDict)): + elif isinstance(val, dict | BrowseDict): val = dict_to_str(val, indent + 2) elif isinstance(val, list): val = list_to_shortstr(val, indent=indent) diff --git a/pyaerocom/aeroval/_processing_base.py b/pyaerocom/aeroval/_processing_base.py index 6257bd3f0..4cc58f7ee 100644 --- a/pyaerocom/aeroval/_processing_base.py +++ b/pyaerocom/aeroval/_processing_base.py @@ -3,11 +3,8 @@ from pyaerocom._lowlevel_helpers import TypeValidator from pyaerocom.aeroval import EvalSetup from pyaerocom.aeroval.experiment_output import ExperimentOutput -from pyaerocom.aeroval.json_utils import round_floats -from pyaerocom.colocation.colocated_data import ColocatedData from pyaerocom.colocation.colocation_setup import ColocationSetup from pyaerocom.colocation.colocator import Colocator -from pyaerocom.utils import recursive_defaultdict class HasConfig: diff --git a/pyaerocom/aeroval/aux_io_helpers.py b/pyaerocom/aeroval/aux_io_helpers.py index da0f0fb64..04552f46b 100644 --- a/pyaerocom/aeroval/aux_io_helpers.py +++ b/pyaerocom/aeroval/aux_io_helpers.py @@ -124,7 +124,7 @@ def import_module(self): """ moddir, fname = os.path.split(self.aux_file) - if not moddir in sys.path: + if moddir not in sys.path: sys.path.append(moddir) modname = fname.split(".")[0] return importlib.import_module(modname) diff --git a/pyaerocom/aeroval/coldatatojson_engine.py b/pyaerocom/aeroval/coldatatojson_engine.py index e736ad245..803020957 100644 --- a/pyaerocom/aeroval/coldatatojson_engine.py +++ b/pyaerocom/aeroval/coldatatojson_engine.py @@ -1,5 +1,4 @@ import logging -import os from time import time from cf_units import Unit @@ -90,7 +89,6 @@ def process_coldata(self, coldata: ColocatedData): annual_stats_constrained = self.cfg.statistics_opts.annual_stats_constrained out_dirs = self.cfg.path_manager.get_json_output_dirs(True) - regions_json = self.exp_output.regions_file regions_how = self.cfg.webdisp_opts.regions_how stats_min_num = self.cfg.statistics_opts.MIN_NUM @@ -131,10 +129,10 @@ def process_coldata(self, coldata: ColocatedData): raise NotImplementedError( "Cannot yet apply country filtering for 4D colocated data instances" ) - elif not main_freq in freqs: + elif main_freq not in freqs: raise ConfigError(f"main_freq {main_freq} is not in experiment frequencies: {freqs}") if self.cfg.statistics_opts.stats_tseries_base_freq is not None: - if not self.cfg.statistics_opts.stats_tseries_base_freq in freqs: + if self.cfg.statistics_opts.stats_tseries_base_freq not in freqs: raise ConfigError( f"Base frequency for statistics timeseries needs to be " f"specified in experiment frequencies: {freqs}" @@ -225,21 +223,17 @@ def process_coldata(self, coldata: ColocatedData): regions_how=regions_how, regnames=regnames, meta_glob=meta_glob, - out_dirs=out_dirs, ) else: - logger.info("Processing profile data for vizualization") + logger.info("Processing profile data for visualization") - self._process_profile_data_for_vizualization( + self._process_profile_data_for_visualization( data=data, use_country=use_country, region_names=regnames, station_names=coldata.data.station_name.values, periods=periods, seasons=seasons, - obs_name=obs_name, - var_name_web=var_name_web, - out_dirs=out_dirs, ) logger.info( @@ -274,7 +268,7 @@ def _get_vert_code(self, coldata: ColocatedData = None): vert_code = coldata.get_meta_item("vert_code") return vert_code - def _process_profile_data_for_vizualization( + def _process_profile_data_for_visualization( self, data: dict[str, ColocatedData] = None, use_country: bool = False, @@ -282,11 +276,8 @@ def _process_profile_data_for_vizualization( station_names: ArrayLike = None, periods: tuple[str, ...] = None, seasons: tuple[str, ...] = None, - obs_name: str = None, - var_name_web: str = None, - out_dirs: dict = None, ): - if region_names == None and station_names == None: + if region_names is None and station_names is None: raise ValueError("Both region_id and station_name can not both be None") # Loop through regions @@ -450,14 +441,13 @@ def _process_diurnal_profiles( regions_how: str = "default", regnames=None, meta_glob: dict = None, - out_dirs: dict = None, ): - (ts_objs_weekly, ts_objs_weekly_reg) = _process_sites_weekly_ts( + ts_objs_weekly, ts_objs_weekly_reg = _process_sites_weekly_ts( coldata, regions_how, regnames, meta_glob ) - outdir = os.path.join(out_dirs["ts/diurnal"]) + for ts_data_weekly in ts_objs_weekly: self.exp_output.write_station_data(ts_data_weekly) - if ts_objs_weekly_reg != None: + if ts_objs_weekly_reg is not None: for ts_data_weekly_reg in ts_objs_weekly_reg: self.exp_output.write_station_data(ts_data_weekly_reg) diff --git a/pyaerocom/aeroval/coldatatojson_helpers.py b/pyaerocom/aeroval/coldatatojson_helpers.py index 2ba669ad2..cb908888a 100644 --- a/pyaerocom/aeroval/coldatatojson_helpers.py +++ b/pyaerocom/aeroval/coldatatojson_helpers.py @@ -3,29 +3,27 @@ """ import logging -import os +from collections.abc import Callable from copy import deepcopy from datetime import datetime, timezone -from typing import Callable, Literal +from typing import Literal import numpy as np import pandas as pd import xarray as xr -from pyaerocom import ColocatedData, TsType +from pyaerocom import ColocatedData from pyaerocom._warnings import ignore_warnings from pyaerocom.aeroval.exceptions import ConfigError, TrendsError from pyaerocom.aeroval.fairmode_stats import fairmode_stats from pyaerocom.aeroval.helpers import _get_min_max_year_periods, _period_str_to_timeslice from pyaerocom.config import ALL_REGION_NAME from pyaerocom.exceptions import DataCoverageError, TemporalResolutionError -from pyaerocom.helpers import start_stop from pyaerocom.region import Region, find_closest_region_coord, get_all_default_region_ids from pyaerocom.region_defs import HTAP_REGIONS_DEFAULT, OLD_AEROCOM_REGIONS from pyaerocom.stats.stats import _init_stats_dummy, calculate_statistics from pyaerocom.trends_engine import TrendsEngine from pyaerocom.trends_helpers import ( - _get_season, _get_season_from_months, _get_unique_seasons, _get_yearly, @@ -981,7 +979,7 @@ def _process_map_and_scat( except (DataCoverageError, TemporalResolutionError): use_dummy = True for i, map_stat in zip(site_indices, map_data): - if not freq in map_stat: + if freq not in map_stat: map_stat[freq] = {} if use_dummy: @@ -1038,7 +1036,7 @@ def _process_map_and_scat( # add only sites to scatter data that have data available # in the lowest of the input resolutions (e.g. yearly) site = map_stat["station_name"] - if not site in scat_data: + if site not in scat_data: scat_data[site] = {} scat_data[site]["latitude"] = map_stat["latitude"] scat_data[site]["longitude"] = map_stat["longitude"] @@ -1135,7 +1133,7 @@ def _apply_annual_constraint(data): """ output = {} - if not "yearly" in data or data["yearly"] is None: + if "yearly" not in data or data["yearly"] is None: raise ConfigError( "Cannot apply annual_stats_constrained option. " 'Please add "yearly" in your setup (see attribute ' @@ -1247,7 +1245,7 @@ def _select_period_season_coldata(coldata, period, season): if len(arr.time) == 0: raise DataCoverageError(f"No data available in period {period}") if season != "all": - if not season in arr.season: + if season not in arr.season: raise DataCoverageError(f"No data available in {season} in period {period}") elif TsType(coldata.ts_type) < "monthly": raise TemporalResolutionError( @@ -1310,7 +1308,7 @@ def _process_heatmap_data( if add_trends and freq != "daily": stats.update(**trend_stats) - except (DataCoverageError, TemporalResolutionError) as e: + except (DataCoverageError, TemporalResolutionError): stats = stats_dummy hm_freq[regname][perstr] = stats @@ -1414,7 +1412,7 @@ def _process_statistics_timeseries( ) output = {} - if not data_freq in data or data[data_freq] is None: + if data_freq not in data or data[data_freq] is None: raise TemporalResolutionError( f"failed to compute statistics timeseries, no co-located data " f"available in specified base resolution {data_freq}" diff --git a/pyaerocom/aeroval/collections.py b/pyaerocom/aeroval/collections.py index e48b7a8fe..f9d4c6fc9 100644 --- a/pyaerocom/aeroval/collections.py +++ b/pyaerocom/aeroval/collections.py @@ -50,7 +50,7 @@ def keylist(self, name_or_pattern: str = None) -> list: matches = [] for key in self.keys(): - if fnmatch(key, name_or_pattern) and not key in matches: + if fnmatch(key, name_or_pattern) and key not in matches: matches.append(key) if len(matches) == 0: raise KeyError(f"No matches could be found that match input {name_or_pattern}") @@ -149,7 +149,7 @@ def get_web_iface_name(self, key): """ entry = self[key] - if not "web_interface_name" in entry: + if "web_interface_name" not in entry: return key return entry["web_interface_name"] diff --git a/pyaerocom/aeroval/config/emep/reporting_base.py b/pyaerocom/aeroval/config/emep/reporting_base.py index 32b8882ca..7e30e5850 100644 --- a/pyaerocom/aeroval/config/emep/reporting_base.py +++ b/pyaerocom/aeroval/config/emep/reporting_base.py @@ -48,12 +48,12 @@ def _get_ignore_stations_from_file(): if os.path.exists("./omit_stations.yaml"): filename = os.path.abspath("./omit_stations.yaml") logger.info(f"reading omit_stations.yaml from {filename}") - with open(filename, "r") as fh: + with open(filename) as fh: stations = yaml.safe_load(fh) else: with resources.path(__package__, "omit_stations.yaml") as filename: logger.info(f"reading omit_stations.yaml from {filename}") - with open(filename, "r") as fh: + with filename.open() as fh: stations = yaml.safe_load(fh) rows = [] @@ -358,7 +358,7 @@ def get_CFG(reportyear, year, model_dir) -> dict: harmonise_units=True, regions_how="country", annual_stats_constrained=True, - proj_id=f"emep", + proj_id="emep", exp_id=f"{reportyear}-reporting", exp_name=f"Evaluation of EMEP runs for {reportyear} EMEP reporting", exp_descr=( diff --git a/pyaerocom/aeroval/experiment_output.py b/pyaerocom/aeroval/experiment_output.py index 665d40707..d00b0db4a 100644 --- a/pyaerocom/aeroval/experiment_output.py +++ b/pyaerocom/aeroval/experiment_output.py @@ -13,7 +13,6 @@ from pyaerocom.aeroval.glob_defaults import ( VariableInfo, extended_statistics, - num_statistics, statistics_defaults, statistics_mean_trend, statistics_median_trend, @@ -147,7 +146,7 @@ def results_available(self) -> bool: """ bool: True if results are available for this experiment, else False """ - if not self.exp_id in os.listdir(self.proj_dir): + if self.exp_id not in os.listdir(self.proj_dir): return False elif not len(self._get_json_output_files("map")) > 0: return False @@ -227,16 +226,16 @@ def _sync_heatmaps_with_menu_and_regions(self) -> None: hm = {} for vardisp, info in menu.items(): obs_dict = info["obs"] - if not vardisp in hm: + if vardisp not in hm: hm[vardisp] = {} for obs, vdict in obs_dict.items(): - if not obs in hm[vardisp]: + if obs not in hm[vardisp]: hm[vardisp][obs] = {} for vert_code, mdict in vdict.items(): - if not vert_code in hm[vardisp][obs]: + if vert_code not in hm[vardisp][obs]: hm[vardisp][obs][vert_code] = {} for mod, minfo in mdict.items(): - if not mod in hm[vardisp][obs][vert_code]: + if mod not in hm[vardisp][obs][vert_code]: hm[vardisp][obs][vert_code][mod] = {} modvar = minfo["model_var"] hm_data = data[vardisp][obs][vert_code][mod][modvar] @@ -252,7 +251,7 @@ def _check_hm_all_regions_avail(self, all_regions, hm_data) -> dict: periods = self.cfg.time_cfg._get_all_period_strings() dummy_stats = _init_stats_dummy() for region in all_regions: - if not region in hm_data: + if region not in hm_data: hm_data[region] = {} for per in periods: hm_data[region][per] = dummy_stats @@ -314,7 +313,6 @@ def _info_from_map_file(filename: str) -> MapInfo: def _results_summary(self) -> dict[str, list[str]]: res = [[], [], [], [], [], []] files = self._get_json_output_files("map") - tab = [] for file in files: map_info = self._info_from_map_file(file) for i, entry in enumerate(map_info): @@ -362,7 +360,7 @@ def clean_json_files(self) -> list[str]: continue if not self._is_part_of_experiment(obs_network, obs_var, mod_name, mod_var): rmmap.append(file_path) - elif not vert_code in vert_codes: + elif vert_code not in vert_codes: rmmap.append(file_path) scatfiles = os.listdir(outdirs["scat"]) @@ -390,7 +388,7 @@ def _check_clean_ts_file(self, fp) -> bool: fname = os.path.basename(fp) spl = fname.split(".json")[0].split("_") vert_code, obsinfo = spl[-1], spl[-2] - if not vert_code in self.cfg.obs_cfg.all_vert_types: + if vert_code not in self.cfg.obs_cfg.all_vert_types: logger.warning( f"Invalid or outdated vert code {vert_code} in ts file {fp}. File will be deleted." ) @@ -546,7 +544,7 @@ def _create_var_ranges_json(self) -> None: avail = self._results_summary() all_vars = list(set(avail["ovar"] + avail["mvar"])) for var in all_vars: - if not var in ranges or ranges[var]["scale"] == []: + if var not in ranges or ranges[var]["scale"] == []: ranges[var] = self._get_cmap_info(var) self.avdb.put_ranges(ranges, self.proj_id, self.exp_id) @@ -760,15 +758,15 @@ def _create_menu_dict(self) -> dict: if self._is_part_of_experiment(obs_name, obs_var, mod_name, mod_var): mcfg = self.cfg.model_cfg.get_entry(mod_name) var = mcfg.get_varname_web(mod_var, obs_var) - if not var in new: + if var not in new: new[var] = self._init_menu_entry(var) - if not obs_name in new[var]["obs"]: + if obs_name not in new[var]["obs"]: new[var]["obs"][obs_name] = {} - if not vert_code in new[var]["obs"][obs_name]: + if vert_code not in new[var]["obs"][obs_name]: new[var]["obs"][obs_name][vert_code] = {} - if not mod_name in new[var]["obs"][obs_name][vert_code]: + if mod_name not in new[var]["obs"][obs_name][vert_code]: new[var]["obs"][obs_name][vert_code][mod_name] = {} model_id = mcfg["model_id"] @@ -1053,7 +1051,7 @@ def add_profile_entry( float(coldata.data.attrs["vertical_layer"]["end"]) + float(coldata.data.attrs["vertical_layer"]["start"]) ) / 2 - if not "z" in current[model_name]: + if "z" not in current[model_name]: current[model_name]["z"] = [midpoint] # initalize with midpoint if ( @@ -1065,9 +1063,9 @@ def add_profile_entry( for season in seasons: perstr = f"{per}-{season}" - if not perstr in current[model_name]["obs"][freq]: + if perstr not in current[model_name]["obs"][freq]: current[model_name]["obs"][freq][perstr] = [] - if not perstr in current[model_name]["mod"][freq]: + if perstr not in current[model_name]["mod"][freq]: current[model_name]["mod"][freq][perstr] = [] current[model_name]["obs"][freq][perstr].append( @@ -1077,7 +1075,7 @@ def add_profile_entry( profile_viz["mod"][freq][perstr] ) - if not "metadata" in current[model_name]: + if "metadata" not in current[model_name]: current[model_name]["metadata"] = { "z_unit": coldata.data.attrs["altitude_units"], "z_description": "Altitude ASL", diff --git a/pyaerocom/aeroval/fairmode_stats.py b/pyaerocom/aeroval/fairmode_stats.py index a8cb51998..bf996c5de 100644 --- a/pyaerocom/aeroval/fairmode_stats.py +++ b/pyaerocom/aeroval/fairmode_stats.py @@ -76,7 +76,7 @@ def fairmode_stats(obs_var: str, stats: dict) -> dict: assert np.isclose( rmsu * beta_mqi, np.sqrt((bias) ** 2 + (mod_std - obs_std) ** 2 + (2 * obs_std * mod_std * (1 - R))), - rtol=1e-4, + rtol=1e-3, ), "failed MQI check" fairmode = dict( diff --git a/pyaerocom/aeroval/glob_defaults.py b/pyaerocom/aeroval/glob_defaults.py index bb643a8b3..64175a7d7 100644 --- a/pyaerocom/aeroval/glob_defaults.py +++ b/pyaerocom/aeroval/glob_defaults.py @@ -56,11 +56,6 @@ def __init__(self, config_file: str = "", **kwargs): class VarWebScaleAndColormap(dict[str, ScaleAndColmap]): - def __init__(self, *args, **kwargs): - # run arguments through pydantic - wvsc = _VarWebScaleAndColormap(scale_colmaps=kwargs) - super().__init__(**{x: y._asdict() for x, y in wvsc.scale_colmaps.items()}) - def __init__(self, config_file="", **kwargs): """This class contains scale and colmap informations and is implemented as dict to allow json serialization. It reads it inital data from data/var_scale_colmap.ini. diff --git a/pyaerocom/aeroval/helpers.py b/pyaerocom/aeroval/helpers.py index cdcd2f903..a2ae9f7eb 100644 --- a/pyaerocom/aeroval/helpers.py +++ b/pyaerocom/aeroval/helpers.py @@ -4,7 +4,6 @@ from pyaerocom import const from pyaerocom.aeroval.modelentry import ModelEntry -from pyaerocom.aeroval.varinfo_web import VarinfoWeb from pyaerocom.griddeddata import GriddedData from pyaerocom.helpers import ( get_highest_resolution, diff --git a/pyaerocom/aeroval/json_utils.py b/pyaerocom/aeroval/json_utils.py index 40955ab57..a1ef642b4 100644 --- a/pyaerocom/aeroval/json_utils.py +++ b/pyaerocom/aeroval/json_utils.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -import os import numpy as np import simplejson @@ -37,14 +36,14 @@ def round_floats(in_data: float | dict | list | tuple) -> float | dict | list | """ - if isinstance(in_data, (float, np.float32, np.float16, np.float128, np.float64)): - # np.float64, is an aliase for the Python float, but is mentioned here for completeness + if isinstance(in_data, float | np.float32 | np.float16 | np.float128 | np.float64): + # np.float64, is an alias for the Python float, but is mentioned here for completeness # note that round and np.round yield different results with the Python round being mathematically correct # details are here: # https://numpy.org/doc/stable/reference/generated/numpy.around.html#numpy.around # use numpy around for now return np.around(in_data, FLOAT_DECIMALS) - elif isinstance(in_data, (list, tuple)): + elif isinstance(in_data, list | tuple): return [round_floats(v) for v in in_data] elif isinstance(in_data, dict): return {k: round_floats(v) for k, v in in_data.items()} diff --git a/pyaerocom/aeroval/modelmaps_engine.py b/pyaerocom/aeroval/modelmaps_engine.py index ee9e5d6fc..cc542e052 100644 --- a/pyaerocom/aeroval/modelmaps_engine.py +++ b/pyaerocom/aeroval/modelmaps_engine.py @@ -31,7 +31,7 @@ def _get_run_kwargs(self, **kwargs): model_list = self.cfg.model_cfg.keylist() try: var_list = kwargs["var_list"] - except: + except Exception: var_list = None return model_list, var_list diff --git a/pyaerocom/aeroval/modelmaps_helpers.py b/pyaerocom/aeroval/modelmaps_helpers.py index 04eb3f424..64ac3ae72 100644 --- a/pyaerocom/aeroval/modelmaps_helpers.py +++ b/pyaerocom/aeroval/modelmaps_helpers.py @@ -47,14 +47,14 @@ def griddeddata_to_jsondict(data, lat_res_deg=5, lon_res_deg=5): try: data.check_dimcoords_tseries() - except: + except Exception: data.reorder_dimensions_tseries() arr = data.to_xarray() latname, lonname = "lat", "lon" try: stacked = arr.stack(station_name=(latname, lonname)) - except: + except Exception: latname, lonname = "latitude", "longitude" stacked = arr.stack(station_name=(latname, lonname)) diff --git a/pyaerocom/aeroval/obsentry.py b/pyaerocom/aeroval/obsentry.py index 05ea15dfb..e7f7e2fd4 100644 --- a/pyaerocom/aeroval/obsentry.py +++ b/pyaerocom/aeroval/obsentry.py @@ -99,7 +99,7 @@ def check_add_obs(self): f"Cannot initialise auxiliary setup for {self.obs_id}. " f"Aux obs reading is so far only possible for ungridded observations." ) - if not self.obs_id in const.OBS_IDS_UNGRIDDED: + if self.obs_id not in const.OBS_IDS_UNGRIDDED: try: const.add_ungridded_post_dataset(**self) except Exception: @@ -140,7 +140,7 @@ def get_vert_code(self, var): val = vc[var] else: raise ValueError(f"invalid value for obs_vert_type: {vc}") - if not val in self.SUPPORTED_VERT_CODES: + if val not in self.SUPPORTED_VERT_CODES: raise ValueError( f"invalid value for obs_vert_type: {val}. Choose from " f"{self.SUPPORTED_VERT_CODES}." @@ -150,7 +150,7 @@ def get_vert_code(self, var): def check_cfg(self): """Check that minimum required attributes are set and okay""" - if not self.is_superobs and not isinstance(self.obs_id, (str, dict)): + if not self.is_superobs and not isinstance(self.obs_id, str | dict): raise ValueError( f"Invalid value for obs_id: {self.obs_id}. Need str or dict " f"or specification of ids and variables via obs_compute_post" @@ -167,17 +167,17 @@ def check_cfg(self): f"It may be specified for all variables (as string) " f"or per variable using a dict" ) - elif isinstance(ovt, str) and not ovt in self.SUPPORTED_VERT_CODES: + elif isinstance(ovt, str) and ovt not in self.SUPPORTED_VERT_CODES: self.obs_vert_type = self._check_ovt(ovt) elif isinstance(self.obs_vert_type, dict): for var_name, val in self.obs_vert_type.items(): - if not val in self.SUPPORTED_VERT_CODES: + if val not in self.SUPPORTED_VERT_CODES: raise ValueError( f"Invalid value for obs_vert_type: {self.obs_vert_type} " f"(variable {var_name}). Supported codes are {self.SUPPORTED_VERT_CODES}." ) ovl = self.instr_vert_loc - if isinstance(ovl, str) and not ovl in self.SUPPORTED_VERT_LOCS: + if isinstance(ovl, str) and ovl not in self.SUPPORTED_VERT_LOCS: raise AttributeError( f"Invalid value for instr_vert_loc: {ovl} for {self.obs_id}. " f"Please choose from: {self.SUPPORTED_VERT_LOCS}" diff --git a/pyaerocom/aeroval/setupclasses.py b/pyaerocom/aeroval/setupclasses.py index fce9c91e8..24db4096e 100644 --- a/pyaerocom/aeroval/setupclasses.py +++ b/pyaerocom/aeroval/setupclasses.py @@ -35,7 +35,7 @@ _get_min_max_year_periods, check_if_year, ) -from pyaerocom.aeroval.json_utils import read_json, set_float_serialization_precision, write_json +from pyaerocom.aeroval.json_utils import read_json, set_float_serialization_precision from pyaerocom.colocation.colocation_setup import ColocationSetup logger = logging.getLogger(__name__) diff --git a/pyaerocom/aeroval/utils.py b/pyaerocom/aeroval/utils.py index d75a0818a..e6e50a027 100644 --- a/pyaerocom/aeroval/utils.py +++ b/pyaerocom/aeroval/utils.py @@ -127,7 +127,7 @@ def compute_model_average_and_diversity( if len(ignore_models) > 0: models = [] for model in model_names: - if not model in ignore_models: + if model not in ignore_models: models.append(model) else: models = model_names diff --git a/pyaerocom/aux_var_helpers.py b/pyaerocom/aux_var_helpers.py index 86f2eecdf..31f3deb3e 100644 --- a/pyaerocom/aux_var_helpers.py +++ b/pyaerocom/aux_var_helpers.py @@ -266,8 +266,8 @@ def _calc_od_helper( if neither ``od_ref`` nor ``od_ref_alt`` are available in data, or if ``use_angstrom_coeff`` is missing """ - if not od_ref in data: - if od_ref_alt is None and not od_ref_alt in data: + if od_ref not in data: + if od_ref_alt is None and od_ref_alt not in data: raise AttributeError(f"No alternative OD found for computation of {var_name}") return compute_od_from_angstromexp( to_lambda=to_lambda, @@ -275,7 +275,7 @@ def _calc_od_helper( lambda_ref=lambda_ref_alt, angstrom_coeff=data[use_angstrom_coeff], ) - elif not use_angstrom_coeff in data: + elif use_angstrom_coeff not in data: raise AttributeError("Angstrom coefficient (440-870 nm) is not available in provided data") result = compute_od_from_angstromexp( to_lambda=to_lambda, @@ -340,7 +340,7 @@ def compute_sc550dryaer(data): vals, rh_mean = _compute_dry_helper( data, data_colname="sc550aer", rh_colname="scrh", rh_max_percent=rh_max ) - if not "sc550dryaer" in data.var_info: + if "sc550dryaer" not in data.var_info: data.var_info["sc550dryaer"] = {} data.var_info["sc550dryaer"]["rh_mean"] = rh_mean @@ -527,7 +527,7 @@ def _compute_wdep_from_concprcp_helper(data, wdep_var, concprcp_var, pr_var): # in units of ts_type, that is, e.g. kg m-2 d freq_str = f" {RATES_FREQ_DEFAULT}-1" wdep_units += freq_str - if not wdep_var in data.var_info: + if wdep_var not in data.var_info: data.var_info[wdep_var] = {} data.var_info[wdep_var]["units"] = wdep_units @@ -775,9 +775,7 @@ def calc_vmro3max(data): o3max = data[var_name] - units = data.var_info[var_name]["units"] - - if not new_var_name in data.var_info: + if new_var_name not in data.var_info: data.var_info[new_var_name] = {} data.var_info[new_var_name] = data.var_info[var_name] @@ -801,10 +799,8 @@ def make_proxy_drydep_from_O3(data): flags = data.data_flagged[var_name] new_var_data = data[var_name] - units = data.var_info[var_name]["units"] - # data.var_info[new_var_name]["units"] = units - if not new_var_name in data.var_info: + if new_var_name not in data.var_info: data.var_info[new_var_name] = {} data.var_info[new_var_name] = data.var_info[var_name] data.var_info[new_var_name]["units"] = "mg m-2 d-1" @@ -823,10 +819,8 @@ def make_proxy_wetdep_from_O3(data): flags = data.data_flagged[var_name] new_var_data = data[var_name] - units = data.var_info[var_name]["units"] - # data.var_info[new_var_name]["units"] = units - if not new_var_name in data.var_info: + if new_var_name not in data.var_info: data.var_info[new_var_name] = {} data.var_info[new_var_name] = data.var_info[var_name] data.var_info[new_var_name]["units"] = "mg m-2 d-1" diff --git a/pyaerocom/colocation/colocated_data.py b/pyaerocom/colocation/colocated_data.py index b0a30ae49..c9ec07c6e 100644 --- a/pyaerocom/colocation/colocated_data.py +++ b/pyaerocom/colocation/colocated_data.py @@ -2,7 +2,6 @@ import logging import os -import warnings from ast import literal_eval from functools import cached_property from pathlib import Path @@ -152,7 +151,7 @@ def validate_data(self): def __init__( self, data: Path | str | xr.DataArray | np.ndarray | None = None, **kwargs ) -> None: - super(ColocatedData, self).__init__(data=data, **kwargs) + super().__init__(data=data, **kwargs) ################################# ## Attributes @@ -907,7 +906,7 @@ def calc_temporal_statistics(self, aggr=None, **kwargs): nc = self.num_coords try: ncd = self.num_coords_with_data - except: + except Exception: ncd = np.nan if self.has_latlon_dims: dim = ("latitude", "longitude") diff --git a/pyaerocom/colocation/colocation_3d.py b/pyaerocom/colocation/colocation_3d.py index bc6e0495b..aa29e0336 100644 --- a/pyaerocom/colocation/colocation_3d.py +++ b/pyaerocom/colocation/colocation_3d.py @@ -67,7 +67,7 @@ def _colocate_vertical_profile_gridded( var_ref_aerocom=None, ) -> list[ColocatedData]: if layer_limits is None: - raise Exception(f"layer limits must be provided") + raise Exception("layer limits must be provided") data_ref_unit = None ts_type_src_ref = None @@ -139,7 +139,7 @@ def _colocate_vertical_profile_gridded( ts_type_src_ref = obs_stat["ts_type_src"] elif obs_stat["ts_type_src"] != ts_type_src_ref: spl = ts_type_src_ref.split(";") - if not obs_stat["ts_type_src"] in spl: + if obs_stat["ts_type_src"] not in spl: spl.append(obs_stat["ts_type_src"]) ts_type_src_ref = ";".join(spl) @@ -348,7 +348,7 @@ def colocate_vertical_profile_gridded( else: var_ref_aerocom = const.VARS[var_ref].var_name_aerocom - if not var_ref in data_ref.contains_vars: + if var_ref not in data_ref.contains_vars: raise VarNotAvailableError( f"Variable {var_ref} is not available in ungridded " f"data (which contains {data_ref.contains_vars})" @@ -371,7 +371,7 @@ def colocate_vertical_profile_gridded( data_ref_meta_idxs_with_var_info = [] for i in range(len(data_ref.metadata)): - if not "altitude" in data_ref.metadata[i]["var_info"]: + if "altitude" not in data_ref.metadata[i]["var_info"]: logger.warning( f"Warning: Station {data_ref.metadata[i]['station_name']} does not have any var_info" ) diff --git a/pyaerocom/colocation/colocation_setup.py b/pyaerocom/colocation/colocation_setup.py index 21c8c1649..ed74838cc 100644 --- a/pyaerocom/colocation/colocation_setup.py +++ b/pyaerocom/colocation/colocation_setup.py @@ -1,9 +1,10 @@ import logging import os import sys +from collections.abc import Callable from functools import cached_property from pathlib import Path -from typing import Callable, Literal +from typing import Literal import pandas as pd from pydantic import ( @@ -459,7 +460,7 @@ def __init__( save_coldata: bool = False, **kwargs, ) -> None: - super(ColocationSetup, self).__init__( + super().__init__( model_id=model_id, obs_config=obs_config, obs_id=obs_id, diff --git a/pyaerocom/colocation/colocation_utils.py b/pyaerocom/colocation/colocation_utils.py index 2906ca650..5ac814506 100644 --- a/pyaerocom/colocation/colocation_utils.py +++ b/pyaerocom/colocation/colocation_utils.py @@ -695,7 +695,7 @@ def colocate_gridded_ungridded( else: var_ref_aerocom = const.VARS[var_ref].var_name_aerocom - if not var_ref in data_ref.contains_vars: + if var_ref not in data_ref.contains_vars: raise VarNotAvailableError( f"Variable {var_ref} is not available in ungridded " f"data (which contains {data_ref.contains_vars})" @@ -830,7 +830,7 @@ def colocate_gridded_ungridded( ts_type_src_ref = obs_stat["ts_type_src"] elif obs_stat["ts_type_src"] != ts_type_src_ref: spl = ts_type_src_ref.split(";") - if not obs_stat["ts_type_src"] in spl: + if obs_stat["ts_type_src"] not in spl: spl.append(obs_stat["ts_type_src"]) ts_type_src_ref = ";".join(spl) diff --git a/pyaerocom/colocation/colocator.py b/pyaerocom/colocation/colocator.py index 8e3163058..61612065f 100644 --- a/pyaerocom/colocation/colocator.py +++ b/pyaerocom/colocation/colocator.py @@ -8,8 +8,9 @@ import traceback import warnings from collections import defaultdict +from collections.abc import Callable from datetime import datetime -from typing import Any, Callable +from typing import Any import pandas as pd from cf_units import Unit @@ -590,7 +591,7 @@ def _filter_var_matches_files_not_exist(self, var_matches, ts_types): def _check_model_add_vars(self): for ovar, mvars in self.colocation_setup.model_add_vars.items(): - if not isinstance(mvars, (list, tuple)): + if not isinstance(mvars, list | tuple): raise ValueError("Values of model_add_vars need to be list or tuple") elif not all([isinstance(x, str) for x in mvars]): raise ValueError("Values of model_add_vars need to be list of strings") @@ -924,7 +925,7 @@ def _infer_start_stop_yr_from_model_reader(self): # 9999 denote climatological data) yrs_avail = self.model_reader.years_avail if self.colocation_setup.model_use_climatology: - if not 9999 in yrs_avail: + if 9999 not in yrs_avail: raise DataCoverageError("No climatology files available") first, last = 9999, None else: diff --git a/pyaerocom/combine_vardata_ungridded.py b/pyaerocom/combine_vardata_ungridded.py index 47481c228..89f171928 100644 --- a/pyaerocom/combine_vardata_ungridded.py +++ b/pyaerocom/combine_vardata_ungridded.py @@ -9,12 +9,12 @@ def _check_input_data_ids_and_vars(data_ids_and_vars): - if not isinstance(data_ids_and_vars, (list, tuple)): + if not isinstance(data_ids_and_vars, list | tuple): raise ValueError("Input data_ids_and_vars must be tuple or list") elif len(data_ids_and_vars) != 2: raise NotImplementedError("Currently, only (and exactly) 2 datasets can be combined...") for item in data_ids_and_vars: - if not isinstance(item, (list, tuple)): + if not isinstance(item, list | tuple): raise ValueError("Each entry in data_ids_and_vars must be tuple or list") elif len(item) != 3: raise ValueError("Each entry in data_ids_and_vars needs to contain exactly 3 items.") @@ -377,12 +377,12 @@ def combine_vardata_ungridded( if data2 is data1 and var2 == var1 and data_id1 == data_id2: raise ValueError("nothing to combine...") - if not data_id1 in data1.contains_datasets: + if data_id1 not in data1.contains_datasets: raise ValueError(f"No such data ID {data_id1} in {data1}") elif len(data1.contains_datasets) > 1: data1 = data1.extract_dataset(data_id1) - if not data_id2 in data2.contains_datasets: + if data_id2 not in data2.contains_datasets: raise ValueError(f"No such data ID {data_id2} in {data2}") elif len(data2.contains_datasets) > 1: data2 = data2.extract_dataset(data_id2) @@ -407,7 +407,7 @@ def combine_vardata_ungridded( match_stats_opts = ["station_name", "closest"] - if not match_stats_how in match_stats_opts: + if match_stats_how not in match_stats_opts: raise ValueError( f"Invalid input for match_stats_how {match_stats_how}, choose from {match_stats_opts}" ) @@ -418,7 +418,7 @@ def combine_vardata_ungridded( # dataset & variable can be provided via this instance prefer = id1 - if not merge_how in merge_how_opts: + if merge_how not in merge_how_opts: raise ValueError(invalid_input_err_str("merge_how", merge_how, merge_how_opts)) elif merge_how == "eval": diff --git a/pyaerocom/config.py b/pyaerocom/config.py index cc4886ec3..3943a9f45 100644 --- a/pyaerocom/config.py +++ b/pyaerocom/config.py @@ -3,7 +3,6 @@ import os from configparser import ConfigParser from pathlib import Path -from typing import Union import numpy as np @@ -145,8 +144,9 @@ class Config: CLIM_RESAMPLE_HOW = "mean" # median, ... # as a function of climatological frequency CLIM_MIN_COUNT = dict( - daily=30, monthly=5 # at least 30 daily measurements in each month over whole period - ) # analogue to daily ... + daily=30, # at least 30 daily measurements in each month over whole period + monthly=5, # analogue to daily ... + ) # names for the satellite data sets SENTINEL5P_NAME = "Sentinel5P" @@ -314,7 +314,7 @@ def infer_basedir_and_config(self): raise FileNotFoundError("Could not establish access to any registered database") def register_custom_variables( - self, vars: Union[dict[str, Variable], dict[str, dict[str, str]]] + self, vars: dict[str, Variable] | dict[str, dict[str, str]] ) -> None: var_dict = {} for key, item in vars.items(): @@ -610,7 +610,7 @@ def add_ungridded_obs(self, obs_id, data_dir, reader=None, check_read=False): reader = get_ungridded_reader(obs_id) - if not obs_id in reader.SUPPORTED_DATASETS: + if obs_id not in reader.SUPPORTED_DATASETS: reader.SUPPORTED_DATASETS.append(obs_id) self.OBSLOCS_UNGRIDDED[obs_id] = data_dir if check_read: @@ -716,7 +716,7 @@ def _check_obsreader(self, obs_id, data_dir, reader): try: check.get_file_list() except DataSourceError: - if not "renamed" in os.listdir(data_dir): + if "renamed" not in os.listdir(data_dir): raise logger.warning( f"Failed to register {obs_id} at {data_dir} using ungridded " @@ -842,7 +842,7 @@ def _add_searchdirs(self, cr, basedir=None): _dir = mcfg["BASEDIR"] if "${HOME}" in _dir: _dir = _dir.replace("${HOME}", os.path.expanduser("~")) - if not _dir in chk_dirs and self._check_access(_dir): + if _dir not in chk_dirs and self._check_access(_dir): chk_dirs.append(_dir) if len(chk_dirs) == 0: return False @@ -862,7 +862,7 @@ def _add_searchdirs(self, cr, basedir=None): continue loc = loc.replace(repl_str, basedir) - if not loc in self._search_dirs: + if loc not in self._search_dirs: self._search_dirs.append(loc) return True @@ -878,7 +878,7 @@ def _add_obsconfig(self, cr, basedir=None): _dir = cfg["BASEDIR"] if "${HOME}" in _dir: _dir = _dir.replace("${HOME}", os.path.expanduser("~")) - if not _dir in chk_dirs and self._check_access(_dir): + if _dir not in chk_dirs and self._check_access(_dir): chk_dirs.append(_dir) if len(chk_dirs) == 0: return False diff --git a/pyaerocom/data/resources.py b/pyaerocom/data/resources.py index 868e98197..cd69baeff 100644 --- a/pyaerocom/data/resources.py +++ b/pyaerocom/data/resources.py @@ -3,8 +3,8 @@ """ import sys +from contextlib import AbstractContextManager from pathlib import Path -from typing import ContextManager if sys.version_info >= (3, 11): from importlib import resources @@ -12,7 +12,7 @@ import importlib_resources as resources -def path(package: str, resource: str) -> ContextManager[Path]: +def path(package: str, resource: str) -> AbstractContextManager[Path]: """A context manager providing a file path object to the resource. If the resource does not already exist on its own on the file system, e.g. it only exists in a zip-file, the resource will be extracted and diff --git a/pyaerocom/extras/satellite_l2/aeolus_l2a.py b/pyaerocom/extras/satellite_l2/aeolus_l2a.py index 3fa56f59a..806a04931 100755 --- a/pyaerocom/extras/satellite_l2/aeolus_l2a.py +++ b/pyaerocom/extras/satellite_l2/aeolus_l2a.py @@ -167,98 +167,98 @@ def __init__(self, data_id=None, index_pointer=0, loglevel=logging.INFO, verbose # test data to be read # self.CODA_READ_PARAMETERS['sca']['metadata']['test_sca_pcd_mid_bins_qc_flag'] = \ # 'sca_pcd/profile_pcd_mid_bins/processing_qc_flag' - self.CODA_READ_PARAMETERS["sca"]["metadata"][ - self._RAYLEIGHALTITUDENAME - ] = "geolocation/measurement_geolocation/rayleigh_geolocation_height_bin/altitude_of_height_bin" - self.CODA_READ_PARAMETERS["sca"]["metadata"][ - self._RAYLEIGHLATNAME - ] = "geolocation/measurement_geolocation/rayleigh_geolocation_height_bin/latitude_of_height_bin" - self.CODA_READ_PARAMETERS["sca"]["metadata"][ - self._RAYLEIGHLONNAME - ] = "geolocation/measurement_geolocation/rayleigh_geolocation_height_bin/longitude_of_height_bin" + self.CODA_READ_PARAMETERS["sca"]["metadata"][self._RAYLEIGHALTITUDENAME] = ( + "geolocation/measurement_geolocation/rayleigh_geolocation_height_bin/altitude_of_height_bin" + ) + self.CODA_READ_PARAMETERS["sca"]["metadata"][self._RAYLEIGHLATNAME] = ( + "geolocation/measurement_geolocation/rayleigh_geolocation_height_bin/latitude_of_height_bin" + ) + self.CODA_READ_PARAMETERS["sca"]["metadata"][self._RAYLEIGHLONNAME] = ( + "geolocation/measurement_geolocation/rayleigh_geolocation_height_bin/longitude_of_height_bin" + ) # /geolocation[?]/measurement_geolocation[?]/rayleigh_geolocation_height_bin[25]/longitude_of_height_bin # /geolocation[?]/measurement_geolocation[?]/rayleigh_geolocation_height_bin[25]/latitude_of_height_bin # /geolocation[?]/measurement_geolocation[?]/rayleigh_geolocation_height_bin[25]/altitude_of_height_bin # self.RETRIEVAL_READ_PARAMETERS['sca']['metadata'][self._QANAME] = 'sca_pcd/qc_flag' - self.CODA_READ_PARAMETERS["sca"]["metadata"][ - self._QANAME - ] = "sca_pcd/profile_pcd_bins/processing_qc_flag" - self.CODA_READ_PARAMETERS["sca"]["metadata"][ - self._LATITUDENAME - ] = "sca_optical_properties/geolocation_middle_bins/latitude" - self.CODA_READ_PARAMETERS["sca"]["metadata"][ - self._LONGITUDENAME - ] = "sca_optical_properties/geolocation_middle_bins/longitude" - self.CODA_READ_PARAMETERS["sca"]["metadata"][ - self._ALTITUDENAME - ] = "sca_optical_properties/geolocation_middle_bins/altitude" - self.CODA_READ_PARAMETERS["sca"]["vars"][ - self._EC355NAME - ] = "sca_optical_properties/sca_optical_properties_mid_bins/extinction" + self.CODA_READ_PARAMETERS["sca"]["metadata"][self._QANAME] = ( + "sca_pcd/profile_pcd_bins/processing_qc_flag" + ) + self.CODA_READ_PARAMETERS["sca"]["metadata"][self._LATITUDENAME] = ( + "sca_optical_properties/geolocation_middle_bins/latitude" + ) + self.CODA_READ_PARAMETERS["sca"]["metadata"][self._LONGITUDENAME] = ( + "sca_optical_properties/geolocation_middle_bins/longitude" + ) + self.CODA_READ_PARAMETERS["sca"]["metadata"][self._ALTITUDENAME] = ( + "sca_optical_properties/geolocation_middle_bins/altitude" + ) + self.CODA_READ_PARAMETERS["sca"]["vars"][self._EC355NAME] = ( + "sca_optical_properties/sca_optical_properties_mid_bins/extinction" + ) # self.CODA_READ_PARAMETERS['sca']['vars'][ # self._EC355NAME] = 'sca_optical_properties/sca_optical_properties/extinction' - self.CODA_READ_PARAMETERS["sca"]["vars"][ - self._BS355NAME - ] = "sca_optical_properties/sca_optical_properties_mid_bins/backscatter" + self.CODA_READ_PARAMETERS["sca"]["vars"][self._BS355NAME] = ( + "sca_optical_properties/sca_optical_properties_mid_bins/backscatter" + ) # self.CODA_READ_PARAMETERS['sca']['vars'][ # self._BS355NAME] = 'sca_optical_properties/sca_optical_properties/backscatter' - self.CODA_READ_PARAMETERS["sca"]["vars"][ - self._LODNAME - ] = "sca_optical_properties/sca_optical_properties/lod" - self.CODA_READ_PARAMETERS["sca"]["vars"][ - self._SRNAME - ] = "sca_optical_properties/sca_optical_properties/sr" + self.CODA_READ_PARAMETERS["sca"]["vars"][self._LODNAME] = ( + "sca_optical_properties/sca_optical_properties/lod" + ) + self.CODA_READ_PARAMETERS["sca"]["vars"][self._SRNAME] = ( + "sca_optical_properties/sca_optical_properties/sr" + ) self.CODA_READ_PARAMETERS["ica"] = {} self.CODA_READ_PARAMETERS["ica"]["metadata"] = {} self.CODA_READ_PARAMETERS["ica"]["vars"] = {} - self.CODA_READ_PARAMETERS["ica"]["metadata"][ - self._TIME_NAME - ] = "ica_optical_properties/starttime" - - self.CODA_READ_PARAMETERS["ica"]["metadata"][ - self._LATITUDENAME - ] = "sca_optical_properties/geolocation_middle_bins/latitude" - self.CODA_READ_PARAMETERS["ica"]["metadata"][ - self._LONGITUDENAME - ] = "sca_optical_properties/geolocation_middle_bins/longitude" - self.CODA_READ_PARAMETERS["ica"]["metadata"][ - self._ALTITUDENAME - ] = "sca_optical_properties/geolocation_middle_bins/altitude" - self.CODA_READ_PARAMETERS["ica"]["vars"][ - self._EC355NAME - ] = "ica_optical_properties/ica_optical_properties/extinction" - self.CODA_READ_PARAMETERS["ica"]["vars"][ - self._BS355NAME - ] = "ica_optical_properties/ica_optical_properties/backscatter" - self.CODA_READ_PARAMETERS["ica"]["vars"][ - self._LODNAME - ] = "ica_optical_properties/ica_optical_properties/lod" + self.CODA_READ_PARAMETERS["ica"]["metadata"][self._TIME_NAME] = ( + "ica_optical_properties/starttime" + ) + + self.CODA_READ_PARAMETERS["ica"]["metadata"][self._LATITUDENAME] = ( + "sca_optical_properties/geolocation_middle_bins/latitude" + ) + self.CODA_READ_PARAMETERS["ica"]["metadata"][self._LONGITUDENAME] = ( + "sca_optical_properties/geolocation_middle_bins/longitude" + ) + self.CODA_READ_PARAMETERS["ica"]["metadata"][self._ALTITUDENAME] = ( + "sca_optical_properties/geolocation_middle_bins/altitude" + ) + self.CODA_READ_PARAMETERS["ica"]["vars"][self._EC355NAME] = ( + "ica_optical_properties/ica_optical_properties/extinction" + ) + self.CODA_READ_PARAMETERS["ica"]["vars"][self._BS355NAME] = ( + "ica_optical_properties/ica_optical_properties/backscatter" + ) + self.CODA_READ_PARAMETERS["ica"]["vars"][self._LODNAME] = ( + "ica_optical_properties/ica_optical_properties/lod" + ) # self.RETRIEVAL_READ_PARAMETERS['ica']['vars'][_CASENAME] = 'ica_optical_properties/ica_optical_properties/case' self.CODA_READ_PARAMETERS["mca"] = {} self.CODA_READ_PARAMETERS["mca"]["metadata"] = {} self.CODA_READ_PARAMETERS["mca"]["vars"] = {} - self.CODA_READ_PARAMETERS["mca"]["metadata"][ - self._TIME_NAME - ] = "mca_optical_properties/starttime" - self.CODA_READ_PARAMETERS["mca"]["metadata"][ - self._LATITUDENAME - ] = "sca_optical_properties/geolocation_middle_bins/latitude" - self.CODA_READ_PARAMETERS["mca"]["metadata"][ - self._LONGITUDENAME - ] = "sca_optical_properties/geolocation_middle_bins/longitude" - self.CODA_READ_PARAMETERS["mca"]["metadata"][ - self._ALTITUDENAME - ] = "sca_optical_properties/geolocation_middle_bins/altitude" - self.CODA_READ_PARAMETERS["mca"]["vars"][ - self._EC355NAME - ] = "mca_optical_properties/mca_optical_properties/extinction" - self.CODA_READ_PARAMETERS["mca"]["vars"][ - self._LODNAME - ] = "mca_optical_properties/mca_optical_properties/lod" + self.CODA_READ_PARAMETERS["mca"]["metadata"][self._TIME_NAME] = ( + "mca_optical_properties/starttime" + ) + self.CODA_READ_PARAMETERS["mca"]["metadata"][self._LATITUDENAME] = ( + "sca_optical_properties/geolocation_middle_bins/latitude" + ) + self.CODA_READ_PARAMETERS["mca"]["metadata"][self._LONGITUDENAME] = ( + "sca_optical_properties/geolocation_middle_bins/longitude" + ) + self.CODA_READ_PARAMETERS["mca"]["metadata"][self._ALTITUDENAME] = ( + "sca_optical_properties/geolocation_middle_bins/altitude" + ) + self.CODA_READ_PARAMETERS["mca"]["vars"][self._EC355NAME] = ( + "mca_optical_properties/mca_optical_properties/extinction" + ) + self.CODA_READ_PARAMETERS["mca"]["vars"][self._LODNAME] = ( + "mca_optical_properties/mca_optical_properties/lod" + ) # DATA_COLNAMES[_MCA_CASENAME] = 'mca_optical_properties/mca_optical_properties/case' GROUP_DELIMITER = "/" @@ -314,31 +314,31 @@ def __init__(self, data_id=None, index_pointer=0, loglevel=logging.INFO, verbose self.NETCDF_VAR_ATTRIBUTES[self._EC355NAME] = {} self.NETCDF_VAR_ATTRIBUTES[self._EC355NAME]["_FillValue"] = np.nan self.NETCDF_VAR_ATTRIBUTES[self._EC355NAME]["long_name"] = "extinction @ 355nm" - self.NETCDF_VAR_ATTRIBUTES[self._EC355NAME][ - "standard_name" - ] = "volume_extinction_coefficient_in_air_due_to_ambient_aerosol_particles" + self.NETCDF_VAR_ATTRIBUTES[self._EC355NAME]["standard_name"] = ( + "volume_extinction_coefficient_in_air_due_to_ambient_aerosol_particles" + ) self.NETCDF_VAR_ATTRIBUTES[self._EC355NAME]["units"] = "1/Mm" self.NETCDF_VAR_ATTRIBUTES[self._EC355NAME + "_mean"] = {} self.NETCDF_VAR_ATTRIBUTES[self._EC355NAME + "_mean"]["_FillValue"] = np.nan self.NETCDF_VAR_ATTRIBUTES[self._EC355NAME + "_mean"]["long_name"] = "extinction @ 355nm" - self.NETCDF_VAR_ATTRIBUTES[self._EC355NAME + "_mean"][ - "standard_name" - ] = "volume_extinction_coefficient_in_air_due_to_ambient_aerosol_particles" + self.NETCDF_VAR_ATTRIBUTES[self._EC355NAME + "_mean"]["standard_name"] = ( + "volume_extinction_coefficient_in_air_due_to_ambient_aerosol_particles" + ) self.NETCDF_VAR_ATTRIBUTES[self._EC355NAME + "_mean"]["units"] = "1/Mm" self.NETCDF_VAR_ATTRIBUTES[self._EC355NAME + "_numobs"] = {} self.NETCDF_VAR_ATTRIBUTES[self._EC355NAME + "_numobs"]["_FillValue"] = np.nan - self.NETCDF_VAR_ATTRIBUTES[self._EC355NAME + "_numobs"][ - "long_name" - ] = "number of observations of extinction @ 355nm" + self.NETCDF_VAR_ATTRIBUTES[self._EC355NAME + "_numobs"]["long_name"] = ( + "number of observations of extinction @ 355nm" + ) self.NETCDF_VAR_ATTRIBUTES[self._EC355NAME + "_numobs"]["units"] = "1" self.NETCDF_VAR_ATTRIBUTES[self._EC355NAME + "_stddev"] = {} self.NETCDF_VAR_ATTRIBUTES[self._EC355NAME + "_stddev"]["_FillValue"] = np.nan - self.NETCDF_VAR_ATTRIBUTES[self._EC355NAME + "_stddev"][ - "long_name" - ] = "standard deviation of extinction @ 355nm" + self.NETCDF_VAR_ATTRIBUTES[self._EC355NAME + "_stddev"]["long_name"] = ( + "standard deviation of extinction @ 355nm" + ) self.NETCDF_VAR_ATTRIBUTES[self._EC355NAME + "_stddev"]["units"] = "1/Mm" self.NETCDF_VAR_ATTRIBUTES[self._QANAME] = {} @@ -926,9 +926,7 @@ def read_file( temp = f"time for single file read [s]: {elapsed_sec:.3f}" self.logger.info(temp) self.logger.info( - "{} points read; {} were valid".format( - self._point_no_found, self._point_no_with_good_quality - ) + f"{self._point_no_found} points read; {self._point_no_with_good_quality} were valid" ) self.RETRIEVAL_READ = read_retrieval self.files_read.append(filename) @@ -1085,9 +1083,7 @@ def calc_dist_in_data( end_time = time.perf_counter() elapsed_sec = end_time - start - temp = "time for single station distance calc using geopy [s]: {:.3f}".format( - elapsed_sec - ) + temp = f"time for single station distance calc using geopy [s]: {elapsed_sec:.3f}" self.logger.info(temp) else: pass @@ -1174,9 +1170,9 @@ def colocate( ) cut_flag = False - ret_data[ - index_counter : index_counter + matching_length, : - ] = ungridded_data_obj._data[matching_indexes, :] + ret_data[index_counter : index_counter + matching_length, :] = ( + ungridded_data_obj._data[matching_indexes, :] + ) index_counter += matching_length elif isinstance(location, tuple): @@ -1457,11 +1453,14 @@ def to_netcdf_simple( ds[lon_dim_name] = (lon_dim_name), _data[lon_dim_name] ds[lev_dim_name] = (lev_dim_name), np.arange(_data[alt_dim_name].shape[1]) ds[alt_dim_name] = ( - time_dim_name, - lev_dim_name, - lat_dim_name, - lon_dim_name, - ), _data[alt_dim_name] + ( + time_dim_name, + lev_dim_name, + lat_dim_name, + lon_dim_name, + ), + _data[alt_dim_name], + ) # ds[self._ALTBOUNDSNAME] = (alt_dim_name, altbnds_dim_name), _data[self._ALTBOUNDSNAME] # for var in vars_to_write_out: @@ -1472,23 +1471,32 @@ def to_netcdf_simple( continue # 3D data ds[var + "_mean"] = ( - time_dim_name, - lev_dim_name, - lat_dim_name, - lon_dim_name, - ), _data[var]["mean"] + ( + time_dim_name, + lev_dim_name, + lat_dim_name, + lon_dim_name, + ), + _data[var]["mean"], + ) ds[var + "_numobs"] = ( - time_dim_name, - lev_dim_name, - lat_dim_name, - lon_dim_name, - ), _data[var]["numobs"] + ( + time_dim_name, + lev_dim_name, + lat_dim_name, + lon_dim_name, + ), + _data[var]["numobs"], + ) ds[var + "_stddev"] = ( - time_dim_name, - lev_dim_name, - lat_dim_name, - lon_dim_name, - ), _data[var]["stddev"] + ( + time_dim_name, + lev_dim_name, + lat_dim_name, + lon_dim_name, + ), + _data[var]["stddev"], + ) # add attributes to variables for var in ds.variables: @@ -1513,7 +1521,7 @@ def to_netcdf_simple( try: for name in global_attributes: ds.attrs[name] = global_attributes[name] - except: + except Exception: pass # temp = 'writing file: {}'.format(netcdf_filename) @@ -2000,8 +2008,6 @@ def to_netcdf_data(self, filename, coda_data, grouping="names", verbose=False): import time start_time = time.perf_counter() - import numpy as np - import pandas as pd import xarray as xr if grouping == "names": @@ -2227,7 +2233,6 @@ def plot_profile_v2( linear_time=False, ): """plot sample profile plot""" - import matplotlib import matplotlib.pyplot as plt from matplotlib.colors import BoundaryNorm from matplotlib.ticker import MaxNLocator @@ -2451,7 +2456,7 @@ def plot_profile_v2( fig.subplots_adjust(hspace=0.3) try: axs = _axs.flatten() - except: + except Exception: axs = [_axs] # levels = MaxNLocator(nbins=15).tick_values(np.nanmin(out_arr), np.nanmax(out_arr)) @@ -2543,7 +2548,6 @@ def plot_profile_v3( linear_time=False, ): """plot sample profile plot""" - import matplotlib import matplotlib.pyplot as plt from matplotlib.colors import BoundaryNorm from matplotlib.ticker import MaxNLocator @@ -2737,7 +2741,7 @@ def plot_profile_v3( fig.subplots_adjust(hspace=0.3) try: axs = _axs.flatten() - except: + except Exception: axs = [_axs] # levels = MaxNLocator(nbins=15).tick_values(np.nanmin(out_arr), np.nanmax(out_arr)) @@ -2901,11 +2905,7 @@ def to_grid( matched_heights += height_match_indexes.size if height_match_indexes.size < self.MIN_VAL_NO_FOR_GRIDDING: continue - print( - "height: {}, matched indexes: {}".format( - grid_height, height_match_indexes.size - ) - ) + print(f"height: {grid_height}, matched indexes: {height_match_indexes.size}") # data_temp = _data[height_match_indexes,:] for lat_idx, grid_lat in enumerate(grid_lats): diff_lat = np.absolute(_data[height_match_indexes, self._LATINDEX] - grid_lat) @@ -2959,12 +2959,12 @@ def to_grid( if less_than_zero_indexes.size == 1: try: - gridded_var_data[var]["mean"][ - lat_idx, lon_idx, height_idx - ] = _data[ - match_indexes[less_than_zero_indexes], - self.INDEX_DICT[var], - ] + gridded_var_data[var]["mean"][lat_idx, lon_idx, height_idx] = ( + _data[ + match_indexes[less_than_zero_indexes], + self.INDEX_DICT[var], + ] + ) gridded_var_data[var]["stddev"][ lat_idx, lon_idx, height_idx ] = 0.0 @@ -2976,13 +2976,13 @@ def to_grid( pass else: try: - gridded_var_data[var]["mean"][ - lat_idx, lon_idx, height_idx - ] = np.nanmean( - _data[ - match_indexes[less_than_zero_indexes], - self.INDEX_DICT[var], - ] + gridded_var_data[var]["mean"][lat_idx, lon_idx, height_idx] = ( + np.nanmean( + _data[ + match_indexes[less_than_zero_indexes], + self.INDEX_DICT[var], + ] + ) ) gridded_var_data[var]["stddev"][ lat_idx, lon_idx, height_idx @@ -3009,20 +3009,16 @@ def to_grid( end_time = time.perf_counter() elapsed_sec = end_time - start_time - temp = "time for global {} gridding with python data types [s]: {:.3f}".format( - gridtype, elapsed_sec - ) + temp = f"time for global {gridtype} gridding with python data types [s]: {elapsed_sec:.3f}" self.logger.info(temp) - temp = "matched {} points out of {} existing points to grid".format( - matching_points, _data.shape[0] + temp = ( + f"matched {matching_points} points out of {_data.shape[0]} existing points to grid" ) self.logger.info(temp) temp = f"{neg_points} points were negative" self.logger.info(temp) - temp = "matched heights: {}; matched latitudes {}; matched longitude {}".format( - matched_heights, matched_latitudes, matched_longitudes - ) + temp = f"matched heights: {matched_heights}; matched latitudes {matched_latitudes}; matched longitude {matched_longitudes}" self.logger.info(temp) if return_data_for_gridding: self.logger.info("returning also data_for_gridding...") @@ -3075,7 +3071,7 @@ def _to_grid_grid_init( try: timeno = len(times) - except: + except Exception: pass if levelno is None or levelno == 1 or levelno == 0: @@ -3108,9 +3104,7 @@ def _to_grid_grid_init( else: # model grid; just dimensions - temp = "starting simple gridding for given grid with dims ({},{},{})...".format( - levelno, latno, lonno - ) + temp = f"starting simple gridding for given grid with dims ({levelno},{latno},{lonno})..." self.logger.info(temp) grid_array_prot = np.full( @@ -3135,9 +3129,7 @@ def _to_grid_grid_init( end_time = time.perf_counter() elapsed_sec = end_time - start_time - temp = "time for global {} gridding with python data types [s] init: {:.3f}".format( - gridtype, elapsed_sec - ) + temp = f"time for global {gridtype} gridding with python data types [s] init: {elapsed_sec:.3f}" self.logger.info(temp) # predefine the output data dict @@ -3150,7 +3142,7 @@ def _to_grid_grid_init( gridded_var_data[self._ALTITUDENAME] = levels try: gridded_var_data[self._TIME_NAME] = times - except: + except Exception: gridded_var_data[self._TIME_NAME] = init_time for var in vars: data_for_gridding[var] = grid_data_prot.copy() @@ -3346,16 +3338,10 @@ def to_model_grid( if lat_verbose_flag: print( - "lon {}, matched {}".format( - grid_lon, _data[lon_match_indexes, self._LONINDEX] - ) + f"lon {grid_lon}, matched {_data[lon_match_indexes, self._LONINDEX]}" ) print( - "lat {}, lon {}, heights {}".format( - _data[lon_match_indexes, self._LATINDEX], - _data[lon_match_indexes, self._LONINDEX], - _data[lon_match_indexes, self._ALTITUDEINDEX], - ) + f"lat {_data[lon_match_indexes, self._LATINDEX]}, lon {_data[lon_match_indexes, self._LONINDEX]}, heights {_data[lon_match_indexes, self._ALTITUDEINDEX]}" ) model_height = model_data[ @@ -3372,15 +3358,13 @@ def to_model_grid( )[0] if lat_verbose_flag: print( - "lowest height distance: {} m".format( - diff_height_to_model[lowest_idx] - ) + f"lowest height distance: {diff_height_to_model[lowest_idx]} m" ) # lowest_idx = height index of model # height_idx = index of satellite data try: mean_idx[lowest_idx[0]].append(lon_match_indexes[match_idx]) - except: + except Exception: mean_idx[lowest_idx[0]] = [lon_match_indexes[match_idx]] # now apply the height index array to the variables and calculate the height means @@ -3458,12 +3442,10 @@ def to_model_grid( end_time = time.perf_counter() elapsed_sec = end_time - start_time - temp = "time for gridding to model grid with python data types [s]: {:.3f}".format( - elapsed_sec - ) + temp = f"time for gridding to model grid with python data types [s]: {elapsed_sec:.3f}" self.logger.info(temp) - temp = "matched {} points out of {} existing points to grid".format( - matching_points, _data.shape[0] + temp = ( + f"matched {matching_points} points out of {_data.shape[0]} existing points to grid" ) self.logger.info(temp) temp = f"{neg_points} points were negative" diff --git a/pyaerocom/extras/satellite_l2/base_reader.py b/pyaerocom/extras/satellite_l2/base_reader.py index 7345145e2..ec95f79c7 100644 --- a/pyaerocom/extras/satellite_l2/base_reader.py +++ b/pyaerocom/extras/satellite_l2/base_reader.py @@ -170,7 +170,7 @@ def __init__(self, data_id=None, index_pointer=0, loglevel=logging.INFO, verbose try: self.LOCAL_TMP_DIR = const.LOCAL_TMP_DIR - except: + except Exception: self.LOCAL_TMP_DIR = const.CACHEDIR # some gridding constants @@ -399,9 +399,9 @@ def read( _key ] elif len(input_shape) == 4: - data_obj._data[_key][ - 0 : file_data[_key].shape[0], :, :, : - ] = file_data[_key] + data_obj._data[_key][0 : file_data[_key].shape[0], :, :, :] = ( + file_data[_key] + ) else: pass @@ -435,9 +435,9 @@ def read( _key ] elif len(current_shape) == 4: - tmp_data[ - 0 : data_obj._data[_key].shape[0], :, :, : - ] = data_obj._data[_key] + tmp_data[0 : data_obj._data[_key].shape[0], :, :, :] = ( + data_obj._data[_key] + ) else: pass @@ -601,13 +601,27 @@ def to_netcdf_simple( continue # 1D data # 3D data - ds[var + "_mean"] = (time_dim_name, lat_dim_name, lon_dim_name), np.reshape( - _data[var]["mean"], - (len(ds[time_dim_name]), len(_data[lat_dim_name]), len(_data[lon_dim_name])), + ds[var + "_mean"] = ( + (time_dim_name, lat_dim_name, lon_dim_name), + np.reshape( + _data[var]["mean"], + ( + len(ds[time_dim_name]), + len(_data[lat_dim_name]), + len(_data[lon_dim_name]), + ), + ), ) - ds[var + "_numobs"] = (time_dim_name, lat_dim_name, lon_dim_name), np.reshape( - _data[var]["numobs"], - (len(ds[time_dim_name]), len(_data[lat_dim_name]), len(_data[lon_dim_name])), + ds[var + "_numobs"] = ( + (time_dim_name, lat_dim_name, lon_dim_name), + np.reshape( + _data[var]["numobs"], + ( + len(ds[time_dim_name]), + len(_data[lat_dim_name]), + len(_data[lon_dim_name]), + ), + ), ) # remove _FillVar attribute for coordinate variables as CF requires it @@ -639,7 +653,7 @@ def to_netcdf_simple( try: for name in global_attributes: ds.attrs[name] = global_attributes[name] - except: + except Exception: pass ds.to_netcdf(netcdf_filename) diff --git a/pyaerocom/extras/satellite_l2/sentinel5p.py b/pyaerocom/extras/satellite_l2/sentinel5p.py index d8e03a478..53df591f2 100755 --- a/pyaerocom/extras/satellite_l2/sentinel5p.py +++ b/pyaerocom/extras/satellite_l2/sentinel5p.py @@ -145,68 +145,68 @@ def __init__( # self.CODA_READ_PARAMETERS[DATASET_NAME]['metadata'][_TIME_NAME] = 'PRODUCT/time_utc' self.CODA_READ_PARAMETERS[self._NO2NAME]["metadata"][self._TIME_NAME] = "PRODUCT/time" - self.CODA_READ_PARAMETERS[self._NO2NAME]["metadata"][ - self._TIME_OFFSET_NAME - ] = "PRODUCT/delta_time" - self.CODA_READ_PARAMETERS[self._NO2NAME]["metadata"][ - self._LATITUDENAME - ] = "PRODUCT/latitude" - self.CODA_READ_PARAMETERS[self._NO2NAME]["metadata"][ - self._LONGITUDENAME - ] = "PRODUCT/longitude" - self.CODA_READ_PARAMETERS[self._NO2NAME]["metadata"][ - self._SCANLINENAME - ] = "PRODUCT/scanline" - self.CODA_READ_PARAMETERS[self._NO2NAME]["metadata"][ - self._GROUNDPIXELNAME - ] = "PRODUCT/ground_pixel" - self.CODA_READ_PARAMETERS[self._NO2NAME]["metadata"][ - self._LONBOUNDSNAME - ] = "PRODUCT/SUPPORT_DATA/GEOLOCATIONS/longitude_bounds" - self.CODA_READ_PARAMETERS[self._NO2NAME]["metadata"][ - self._LATBOUNDSNAME - ] = "PRODUCT/SUPPORT_DATA/GEOLOCATIONS/latitude_bounds" + self.CODA_READ_PARAMETERS[self._NO2NAME]["metadata"][self._TIME_OFFSET_NAME] = ( + "PRODUCT/delta_time" + ) + self.CODA_READ_PARAMETERS[self._NO2NAME]["metadata"][self._LATITUDENAME] = ( + "PRODUCT/latitude" + ) + self.CODA_READ_PARAMETERS[self._NO2NAME]["metadata"][self._LONGITUDENAME] = ( + "PRODUCT/longitude" + ) + self.CODA_READ_PARAMETERS[self._NO2NAME]["metadata"][self._SCANLINENAME] = ( + "PRODUCT/scanline" + ) + self.CODA_READ_PARAMETERS[self._NO2NAME]["metadata"][self._GROUNDPIXELNAME] = ( + "PRODUCT/ground_pixel" + ) + self.CODA_READ_PARAMETERS[self._NO2NAME]["metadata"][self._LONBOUNDSNAME] = ( + "PRODUCT/SUPPORT_DATA/GEOLOCATIONS/longitude_bounds" + ) + self.CODA_READ_PARAMETERS[self._NO2NAME]["metadata"][self._LATBOUNDSNAME] = ( + "PRODUCT/SUPPORT_DATA/GEOLOCATIONS/latitude_bounds" + ) self.CODA_READ_PARAMETERS[self._NO2NAME]["metadata"][self._QANAME] = "PRODUCT/qa_value" - self.CODA_READ_PARAMETERS[self._NO2NAME]["vars"][ - self._NO2NAME - ] = "PRODUCT/nitrogendioxide_tropospheric_column" + self.CODA_READ_PARAMETERS[self._NO2NAME]["vars"][self._NO2NAME] = ( + "PRODUCT/nitrogendioxide_tropospheric_column" + ) self.CODA_READ_PARAMETERS[self._O3NAME]["metadata"][self._TIME_NAME] = "PRODUCT/time" - self.CODA_READ_PARAMETERS[self._O3NAME]["metadata"][ - self._TIME_OFFSET_NAME - ] = "PRODUCT/delta_time" - self.CODA_READ_PARAMETERS[self._O3NAME]["metadata"][ - self._LATITUDENAME - ] = "PRODUCT/latitude" - self.CODA_READ_PARAMETERS[self._O3NAME]["metadata"][ - self._LONGITUDENAME - ] = "PRODUCT/longitude" - self.CODA_READ_PARAMETERS[self._O3NAME]["metadata"][ - self._LONBOUNDSNAME - ] = "PRODUCT/SUPPORT_DATA/GEOLOCATIONS/longitude_bounds" - self.CODA_READ_PARAMETERS[self._O3NAME]["metadata"][ - self._LATBOUNDSNAME - ] = "PRODUCT/SUPPORT_DATA/GEOLOCATIONS/latitude_bounds" - self.CODA_READ_PARAMETERS[self._O3NAME]["metadata"][ - self._SCANLINENAME - ] = "PRODUCT/scanline" - self.CODA_READ_PARAMETERS[self._O3NAME]["metadata"][ - self._GROUNDPIXELNAME - ] = "PRODUCT/ground_pixel" + self.CODA_READ_PARAMETERS[self._O3NAME]["metadata"][self._TIME_OFFSET_NAME] = ( + "PRODUCT/delta_time" + ) + self.CODA_READ_PARAMETERS[self._O3NAME]["metadata"][self._LATITUDENAME] = ( + "PRODUCT/latitude" + ) + self.CODA_READ_PARAMETERS[self._O3NAME]["metadata"][self._LONGITUDENAME] = ( + "PRODUCT/longitude" + ) + self.CODA_READ_PARAMETERS[self._O3NAME]["metadata"][self._LONBOUNDSNAME] = ( + "PRODUCT/SUPPORT_DATA/GEOLOCATIONS/longitude_bounds" + ) + self.CODA_READ_PARAMETERS[self._O3NAME]["metadata"][self._LATBOUNDSNAME] = ( + "PRODUCT/SUPPORT_DATA/GEOLOCATIONS/latitude_bounds" + ) + self.CODA_READ_PARAMETERS[self._O3NAME]["metadata"][self._SCANLINENAME] = ( + "PRODUCT/scanline" + ) + self.CODA_READ_PARAMETERS[self._O3NAME]["metadata"][self._GROUNDPIXELNAME] = ( + "PRODUCT/ground_pixel" + ) self.CODA_READ_PARAMETERS[self._O3NAME]["metadata"][self._QANAME] = "PRODUCT/qa_value" - self.CODA_READ_PARAMETERS[self._O3NAME]["vars"][ - self._O3NAME - ] = "PRODUCT/ozone_total_vertical_column" + self.CODA_READ_PARAMETERS[self._O3NAME]["vars"][self._O3NAME] = ( + "PRODUCT/ozone_total_vertical_column" + ) #################################### self.NETCDF_VAR_ATTRIBUTES[self._NO2NAME] = {} self.NETCDF_VAR_ATTRIBUTES[self._NO2NAME]["_FillValue"] = np.nan - self.NETCDF_VAR_ATTRIBUTES[self._NO2NAME][ - "long_name" - ] = "Tropospheric vertical column of nitrogen dioxide" - self.NETCDF_VAR_ATTRIBUTES[self._NO2NAME][ - "standard_name" - ] = "troposphere_mole_content_of_nitrogen_dioxide" + self.NETCDF_VAR_ATTRIBUTES[self._NO2NAME]["long_name"] = ( + "Tropospheric vertical column of nitrogen dioxide" + ) + self.NETCDF_VAR_ATTRIBUTES[self._NO2NAME]["standard_name"] = ( + "troposphere_mole_content_of_nitrogen_dioxide" + ) # self.NETCDF_VAR_ATTRIBUTES[self._NO2NAME]['units'] = 'mol m-2' self.NETCDF_VAR_ATTRIBUTES[self._NO2NAME]["units"] = "1e15 molecules cm-2" self.NETCDF_VAR_ATTRIBUTES[self._NO2NAME]["coordinates"] = "longitude latitude" @@ -217,9 +217,9 @@ def __init__( self.NETCDF_VAR_ATTRIBUTES[self._NO2NAME + "_numobs"] = {} self.NETCDF_VAR_ATTRIBUTES[self._NO2NAME + "_numobs"]["_FillValue"] = np.nan - self.NETCDF_VAR_ATTRIBUTES[self._NO2NAME + "_numobs"][ - "long_name" - ] = "number of observations" + self.NETCDF_VAR_ATTRIBUTES[self._NO2NAME + "_numobs"]["long_name"] = ( + "number of observations" + ) # self.NETCDF_VAR_ATTRIBUTES[_NO2NAME+'_numobs'][ # 'standard_name'] = 'troposphere_mole_content_of_nitrogen_dioxide' self.NETCDF_VAR_ATTRIBUTES[self._NO2NAME + "_numobs"]["units"] = "1" @@ -227,12 +227,12 @@ def __init__( self.NETCDF_VAR_ATTRIBUTES[self._O3NAME + "_mean"] = {} self.NETCDF_VAR_ATTRIBUTES[self._O3NAME + "_mean"]["_FillValue"] = np.nan - self.NETCDF_VAR_ATTRIBUTES[self._O3NAME + "_mean"][ - "long_name" - ] = "total vertical ozone column" - self.NETCDF_VAR_ATTRIBUTES[self._O3NAME + "_mean"][ - "standard_name" - ] = "atmosphere_mole_content_of_ozone" + self.NETCDF_VAR_ATTRIBUTES[self._O3NAME + "_mean"]["long_name"] = ( + "total vertical ozone column" + ) + self.NETCDF_VAR_ATTRIBUTES[self._O3NAME + "_mean"]["standard_name"] = ( + "atmosphere_mole_content_of_ozone" + ) self.NETCDF_VAR_ATTRIBUTES[self._O3NAME + "_mean"]["units"] = "mol m-2" self.NETCDF_VAR_ATTRIBUTES[self._O3NAME + "_mean"]["coordinates"] = "longitude latitude" @@ -243,9 +243,9 @@ def __init__( self.NETCDF_VAR_ATTRIBUTES[self._O3NAME + "_numobs"] = {} self.NETCDF_VAR_ATTRIBUTES[self._O3NAME + "_numobs"]["_FillValue"] = np.nan - self.NETCDF_VAR_ATTRIBUTES[self._O3NAME + "_numobs"][ - "long_name" - ] = "number of observations" + self.NETCDF_VAR_ATTRIBUTES[self._O3NAME + "_numobs"]["long_name"] = ( + "number of observations" + ) # self.NETCDF_VAR_ATTRIBUTES[_O3NAME+'_numobs'][ # 'standard_name'] = 'troposphere_mole_content_of_nitrogen_dioxide' self.NETCDF_VAR_ATTRIBUTES[self._O3NAME + "_numobs"]["units"] = "1" @@ -254,9 +254,9 @@ def __init__( self.NETCDF_VAR_ATTRIBUTES[self._QANAME] = {} self.NETCDF_VAR_ATTRIBUTES[self._QANAME]["_FillValue"] = np.nan self.NETCDF_VAR_ATTRIBUTES[self._QANAME]["long_name"] = "data quality value" - self.NETCDF_VAR_ATTRIBUTES[self._QANAME][ - "comment" - ] = "A continuous quality descriptor, varying between 0(no data) and 1 (full quality data). Recommend to ignore data with qa_value < 0.5" + self.NETCDF_VAR_ATTRIBUTES[self._QANAME]["comment"] = ( + "A continuous quality descriptor, varying between 0(no data) and 1 (full quality data). Recommend to ignore data with qa_value < 0.5" + ) self.NETCDF_VAR_ATTRIBUTES[self._QANAME]["units"] = "1" self.NETCDF_VAR_ATTRIBUTES[self._QANAME]["coordinates"] = "longitude latitude" @@ -296,9 +296,9 @@ def __init__( self.CODA_READ_PARAMETERS[self._AVERAGINGKERNELNAME]["time_offset"] = np.float_( 24.0 * 60.0 * 60.0 ) - self.CODA_READ_PARAMETERS[self._AVERAGINGKERNELNAME]["metadata"][ - self._TIME_NAME - ] = "PRODUCT/time" + self.CODA_READ_PARAMETERS[self._AVERAGINGKERNELNAME]["metadata"][self._TIME_NAME] = ( + "PRODUCT/time" + ) self.CODA_READ_PARAMETERS[self._AVERAGINGKERNELNAME]["metadata"][ self._TIME_OFFSET_NAME ] = "PRODUCT/delta_time" @@ -320,9 +320,9 @@ def __init__( self.CODA_READ_PARAMETERS[self._AVERAGINGKERNELNAME]["metadata"][ self._LATBOUNDSNAME ] = "PRODUCT/SUPPORT_DATA/GEOLOCATIONS/latitude_bounds" - self.CODA_READ_PARAMETERS[self._AVERAGINGKERNELNAME]["metadata"][ - self._QANAME - ] = "PRODUCT/qa_value" + self.CODA_READ_PARAMETERS[self._AVERAGINGKERNELNAME]["metadata"][self._QANAME] = ( + "PRODUCT/qa_value" + ) self.CODA_READ_PARAMETERS[self._AVERAGINGKERNELNAME]["vars"][ self._AVERAGINGKERNELNAME ] = "PRODUCT/averaging_kernel" @@ -348,9 +348,9 @@ def __init__( self.CODA_READ_PARAMETERS[self._TM5_TROPOPAUSE_LAYER_INDEX_NAME] = {} self.CODA_READ_PARAMETERS[self._TM5_TROPOPAUSE_LAYER_INDEX_NAME]["metadata"] = {} self.CODA_READ_PARAMETERS[self._TM5_TROPOPAUSE_LAYER_INDEX_NAME]["vars"] = {} - self.CODA_READ_PARAMETERS[self._TM5_TROPOPAUSE_LAYER_INDEX_NAME][ - "time_offset" - ] = np.float_(24.0 * 60.0 * 60.0) + self.CODA_READ_PARAMETERS[self._TM5_TROPOPAUSE_LAYER_INDEX_NAME]["time_offset"] = ( + np.float_(24.0 * 60.0 * 60.0) + ) self.CODA_READ_PARAMETERS[self._TM5_TROPOPAUSE_LAYER_INDEX_NAME]["vars"][ self._TM5_TROPOPAUSE_LAYER_INDEX_NAME ] = "PRODUCT/tm5_tropopause_layer_index" @@ -379,38 +379,38 @@ def __init__( self.NETCDF_VAR_ATTRIBUTES[self._AVERAGINGKERNELNAME]["long_name"] = "averaging kernel" # self.NETCDF_VAR_ATTRIBUTES[self._AVERAGINGKERNELNAME]['standard_name'] = 'averaging_kernel' self.NETCDF_VAR_ATTRIBUTES[self._AVERAGINGKERNELNAME]["units"] = "1" - self.NETCDF_VAR_ATTRIBUTES[self._AVERAGINGKERNELNAME][ - "coordinates" - ] = f"longitude latitude {self._LEVELSNAME}" + self.NETCDF_VAR_ATTRIBUTES[self._AVERAGINGKERNELNAME]["coordinates"] = ( + f"longitude latitude {self._LEVELSNAME}" + ) - self.CODA_READ_PARAMETERS[self._NO2NAME]["vars"][ - self._LEVELSNAME - ] = self.CODA_READ_PARAMETERS[self._LEVELSNAME]["vars"][self._LEVELSNAME] - self.CODA_READ_PARAMETERS[self._NO2NAME]["vars"][ - self._GROUNDPRESSURENAME - ] = self.CODA_READ_PARAMETERS[self._GROUNDPRESSURENAME]["vars"][ - self._GROUNDPRESSURENAME - ] + self.CODA_READ_PARAMETERS[self._NO2NAME]["vars"][self._LEVELSNAME] = ( + self.CODA_READ_PARAMETERS[self._LEVELSNAME]["vars"][self._LEVELSNAME] + ) + self.CODA_READ_PARAMETERS[self._NO2NAME]["vars"][self._GROUNDPRESSURENAME] = ( + self.CODA_READ_PARAMETERS[ + self._GROUNDPRESSURENAME + ]["vars"][self._GROUNDPRESSURENAME] + ) self.CODA_READ_PARAMETERS[self._NO2NAME]["vars"][ self._TM5_TROPOPAUSE_LAYER_INDEX_NAME ] = self.CODA_READ_PARAMETERS[self._TM5_TROPOPAUSE_LAYER_INDEX_NAME]["vars"][ self._TM5_TROPOPAUSE_LAYER_INDEX_NAME ] - self.CODA_READ_PARAMETERS[self._NO2NAME]["vars"][ - self._TM5_CONSTANT_A_NAME - ] = self.CODA_READ_PARAMETERS[self._TM5_CONSTANT_A_NAME]["vars"][ - self._TM5_CONSTANT_A_NAME - ] - self.CODA_READ_PARAMETERS[self._NO2NAME]["vars"][ - self._TM5_CONSTANT_B_NAME - ] = self.CODA_READ_PARAMETERS[self._TM5_CONSTANT_B_NAME]["vars"][ - self._TM5_CONSTANT_B_NAME - ] - self.CODA_READ_PARAMETERS[self._NO2NAME]["vars"][ - self._AVERAGINGKERNELNAME - ] = self.CODA_READ_PARAMETERS[self._AVERAGINGKERNELNAME]["vars"][ - self._AVERAGINGKERNELNAME - ] + self.CODA_READ_PARAMETERS[self._NO2NAME]["vars"][self._TM5_CONSTANT_A_NAME] = ( + self.CODA_READ_PARAMETERS[ + self._TM5_CONSTANT_A_NAME + ]["vars"][self._TM5_CONSTANT_A_NAME] + ) + self.CODA_READ_PARAMETERS[self._NO2NAME]["vars"][self._TM5_CONSTANT_B_NAME] = ( + self.CODA_READ_PARAMETERS[ + self._TM5_CONSTANT_B_NAME + ]["vars"][self._TM5_CONSTANT_B_NAME] + ) + self.CODA_READ_PARAMETERS[self._NO2NAME]["vars"][self._AVERAGINGKERNELNAME] = ( + self.CODA_READ_PARAMETERS[ + self._AVERAGINGKERNELNAME + ]["vars"][self._AVERAGINGKERNELNAME] + ) self.STATICFIELDNAMES = [ self._GROUNDPIXELNAME, @@ -611,7 +611,7 @@ def read_file( # time can be a scalar... try: data[index_pointer, self._TIMEINDEX] = _time.astype(np.float_) - except: + except Exception: data[index_pointer, self._TIMEINDEX] = _time[_index].astype(np.float_) # loop over the variables @@ -680,7 +680,7 @@ def to_netcdf_simple( try: for name in global_attributes: ds.attrs[name] = global_attributes[name] - except: + except Exception: pass obj.logger.info(f"writing file {netcdf_filename}...") @@ -715,13 +715,13 @@ def _match_dim_name(self, dim_dict, dim_size=None, data=None): # not all is a ndarray with a shape try: shape = data[var].shape - except: + except Exception: continue for _size in data[var].shape: try: ret_data[var].append(dim_dict[_size]) - except: + except Exception: pass return ret_data else: @@ -802,9 +802,10 @@ def to_xarray(self, data_to_write=None, gridded=False, apply_quality_flag=0.0): elements_to_add = len(keep_indexes[0]) # dim_size_dict[elements_to_add] = point_dim_name ds = xr.Dataset() - ds[self._TIME_NAME] = (point_dim_name), datetimedata.astype("datetime64[ms]")[ - keep_indexes - ] + ds[self._TIME_NAME] = ( + (point_dim_name), + datetimedata.astype("datetime64[ms]")[keep_indexes], + ) ds[point_dim_name] = np.arange(elements_to_add) # point_dim_size = elements_to_add @@ -829,9 +830,10 @@ def to_xarray(self, data_to_write=None, gridded=False, apply_quality_flag=0.0): try: ds[var] = (point_dim_name), _data[var][keep_indexes] except ValueError: - ds[var] = (dim_size_dict[_data[var].shape[0]]), _data[var][ - keep_indexes - ] + ds[var] = ( + (dim_size_dict[_data[var].shape[0]]), + _data[var][keep_indexes], + ) elif len(_data[var].shape) == 2: # var with dimension time and swath (e.g. 3245, 450) @@ -842,9 +844,10 @@ def to_xarray(self, data_to_write=None, gridded=False, apply_quality_flag=0.0): ds[var] = (dim_name_dict[var]), _data[var] else: try: - ds[var] = (point_dim_name), (_data[var].reshape(point_dim_len))[ - keep_indexes - ] + ds[var] = ( + (point_dim_name), + (_data[var].reshape(point_dim_len))[keep_indexes], + ) except ValueError: ds[var] = (dim_name_dict[var]), _data[var] @@ -853,30 +856,38 @@ def to_xarray(self, data_to_write=None, gridded=False, apply_quality_flag=0.0): # store some vars depending on the points dimension as a 2d variable if apply_quality_flag == 0.0: if var == "avg_kernel": - ds[var] = (point_dim_name, level_dim_name), _data[var].reshape( - [point_dim_size, level_dim_size] + ds[var] = ( + (point_dim_name, level_dim_name), + _data[var].reshape([point_dim_size, level_dim_size]), ) elif var == "lat_bnds" or var == "lon_bnds": - ds[var] = (point_dim_name, bounds_dim_name), _data[var].reshape( - [point_dim_size, bounds_dim_size] + ds[var] = ( + (point_dim_name, bounds_dim_name), + _data[var].reshape([point_dim_size, bounds_dim_size]), ) else: ds[var] = (dim_name_dict[var]), _data[var] else: if var == "avg_kernel": # temp = np.squeeze((_data[var].reshape([point_dim_size, level_dim_size]))[keep_indexes,:]) - ds[var] = (point_dim_name, level_dim_name), np.squeeze( - (_data[var].reshape([point_dim_size, level_dim_size]))[ - keep_indexes, : - ] + ds[var] = ( + (point_dim_name, level_dim_name), + np.squeeze( + (_data[var].reshape([point_dim_size, level_dim_size]))[ + keep_indexes, : + ] + ), ) print(f"{var}") elif var == "lat_bnds" or var == "lon_bnds": - ds[var] = (point_dim_name, bounds_dim_name), np.squeeze( - _data[var].reshape([point_dim_size, bounds_dim_size])[ - keep_indexes, : - ] + ds[var] = ( + (point_dim_name, bounds_dim_name), + np.squeeze( + _data[var].reshape([point_dim_size, bounds_dim_size])[ + keep_indexes, : + ] + ), ) else: ds[var] = (dim_name_dict[var]), _data[var] @@ -922,11 +933,17 @@ def to_xarray(self, data_to_write=None, gridded=False, apply_quality_flag=0.0): continue # 1D data # 3D data - ds[var + "_mean"] = (lat_dim_name, lon_dim_name), np.reshape( - _data[var]["mean"], (len(_data[lat_dim_name]), len(_data[lon_dim_name])) + ds[var + "_mean"] = ( + (lat_dim_name, lon_dim_name), + np.reshape( + _data[var]["mean"], (len(_data[lat_dim_name]), len(_data[lon_dim_name])) + ), ) - ds[var + "_numobs"] = (lat_dim_name, lon_dim_name), np.reshape( - _data[var]["numobs"], (len(_data[lat_dim_name]), len(_data[lon_dim_name])) + ds[var + "_numobs"] = ( + (lat_dim_name, lon_dim_name), + np.reshape( + _data[var]["numobs"], (len(_data[lat_dim_name]), len(_data[lon_dim_name])) + ), ) # add attributes to variables diff --git a/pyaerocom/geodesy.py b/pyaerocom/geodesy.py index b91179ef7..9ae4a0216 100644 --- a/pyaerocom/geodesy.py +++ b/pyaerocom/geodesy.py @@ -99,7 +99,7 @@ def get_country_info_coords(coords): """ if isinstance(coords, np.ndarray): coords = list(coords) - if not isinstance(coords, (list, tuple)): + if not isinstance(coords, list | tuple): raise ValueError("Invalid input for coords, need list or tuple or array") geo = Geocoder_Reverse_NE() @@ -343,7 +343,7 @@ def haversine(lat0, lon0, lat1, lon1, earth_radius=6371.0): float horizontal distance between input coordinates in km """ - hav = lambda d_theta: np.sin(d_theta / 2.0) ** 2 + hav = lambda d_theta: np.sin(d_theta / 2.0) ** 2 # noqa: E731 d_lon = np.radians(lon1 - lon0) d_lat = np.radians(lat1 - lat0) diff --git a/pyaerocom/grid_io.py b/pyaerocom/grid_io.py index 13b3faae7..4dd8ce74e 100644 --- a/pyaerocom/grid_io.py +++ b/pyaerocom/grid_io.py @@ -151,7 +151,7 @@ def __setitem__(self, key, value): IOError if key is not a valid setting """ - if not key in self.__dict__: + if key not in self.__dict__: raise OSError("Could not update IO setting: Invalid key") self.__dict__[key] = value @@ -161,7 +161,7 @@ def __getitem__(self, key): GridIO[""] => value """ - if not key in self.__dict__: + if key not in self.__dict__: raise OSError("Invalid attribute") return self.__dict__[key] diff --git a/pyaerocom/griddeddata.py b/pyaerocom/griddeddata.py index f512ae749..76c3c4ba6 100644 --- a/pyaerocom/griddeddata.py +++ b/pyaerocom/griddeddata.py @@ -464,7 +464,7 @@ def grid(self, value): raise TypeError(f"Grid data format {type(value)} is not supported, need Cube") for key, val in self._META_ADD.items(): - if not key in value.attributes: + if key not in value.attributes: value.attributes[key] = val self._grid = value @@ -524,7 +524,7 @@ def shape(self): @property def lon_res(self): - if not "longitude" in self: + if "longitude" not in self: raise AttributeError("Data does not contain longitude information") vals = np.diff(self.longitude.points) val = vals.mean() @@ -534,7 +534,7 @@ def lon_res(self): @property def lat_res(self): - if not "latitude" in self: + if "latitude" not in self: raise AttributeError("Data does not contain longitude information") vals = np.diff(self.latitude.points) val = vals.mean() @@ -661,7 +661,7 @@ def _read_netcdf(self, input, var_name, perform_fmt_checks): from pyaerocom.io.iris_io import load_cube_custom self.grid = load_cube_custom(input, var_name, perform_fmt_checks=perform_fmt_checks) - if not "from_files" in self.metadata: + if "from_files" not in self.metadata: self.metadata["from_files"] = [] elif not isinstance(self.metadata["from_files"], list): self.metadata["from_files"] = [self.metadata["from_files"]] @@ -887,9 +887,8 @@ def years_avail(self): list """ - toyear = lambda x: int(str(x.astype("datetime64[Y]"))) - - return [x for x in set(map(toyear, self.time_stamps()))] + to_year = lambda x: int(str(x.astype("datetime64[Y]"))) # noqa: E731 + return sorted(set(to_year(date) for date in self.time_stamps())) def split_years(self, years=None): """ @@ -946,7 +945,7 @@ def check_dimcoords_tseries(self) -> None: :func:`reorder_dimensions_tseries` may be used to catch the Exception) """ - if not self.ndim in (3, 4): + if self.ndim not in (3, 4): raise DataDimensionError("Time series extraction requires at least 3 dimensions") # list of coordinates needed for timeseries extraction. needed = self.COORDS_ORDER_TSERIES @@ -997,7 +996,7 @@ def reorder_dimensions_tseries(self) -> None: if not len(new_order) == self.ndim: for i in range(self.ndim): - if not i in new_order: + if i not in new_order: new_order.append(i) self.transpose(new_order) self.check_dimcoords_tseries() @@ -1373,7 +1372,7 @@ def _apply_vert_scheme(self, sample_points, vert_scheme): cname = self.dimcoord_names[-1] - if not vert_scheme in self.SUPPORTED_VERT_SCHEMES: + if vert_scheme not in self.SUPPORTED_VERT_SCHEMES: raise ValueError( f"Invalid input for vert_scheme: {vert_scheme}. Supported " f"schemes are: {self.SUPPORTED_VERT_SCHEMES}" @@ -1470,7 +1469,7 @@ def find_closest_index(self, **dimcoord_vals): """Find the closest indices for dimension coordinate values""" idx = {} for dim, val in dimcoord_vals.items(): - if not dim in self.coord_names: + if dim not in self.coord_names: raise DataDimensionError(f"No such dimension {dim}") elif dim == "time": idx[dim] = self._closest_time_idx(val) @@ -1634,7 +1633,7 @@ def _resample_time_iris(self, to_ts_type): if current == to: logger.info(f"Data is already in {to_ts_type} resolution") return self - if not to_ts_type in IRIS_AGGREGATORS: + if to_ts_type not in IRIS_AGGREGATORS: raise TemporalResolutionError(f"Resolution {to_ts_type} cannot converted") elif current < to: # current resolution is smaller than desired raise TemporalResolutionError( @@ -1644,11 +1643,11 @@ def _resample_time_iris(self, to_ts_type): # Create aggregators aggrs = ["yearly"] - if not to_ts_type in aggrs: + if to_ts_type not in aggrs: aggrs.append(to_ts_type) for aggr in aggrs: - if not aggr in [c.name() for c in cube.aux_coords]: + if aggr not in [c.name() for c in cube.aux_coords]: # this adds the corresponding aggregator to the cube IRIS_AGGREGATORS[aggr](cube, "time", name=aggr) # IRIS_AGGREGATORS[to_ts_type](cube, 'time', name=to_ts_type) @@ -1667,9 +1666,7 @@ def _resample_time_xarray(self, to_ts_type, how, min_num_obs): arr_out = rs.resample( to_ts_type, from_ts_type=from_ts_type, how=how, min_num_obs=min_num_obs ) - except ( - ValueError - ): # likely non-standard datetime objects in array (cf https://github.com/pydata/xarray/issues/3426) + except ValueError: # likely non-standard datetime objects in array (cf https://github.com/pydata/xarray/issues/3426) arr["time"] = self.time_stamps() rs = TimeResampler(arr) arr_out = rs.resample( @@ -1696,7 +1693,7 @@ def _resample_time_xarray(self, to_ts_type, how, min_num_obs): ) try: data.check_dimcoords_tseries() - except: + except Exception: data.reorder_dimensions_tseries() return data @@ -1808,7 +1805,7 @@ def filter_region(self, region_id, inplace=False, **kwargs): def apply_region_mask(self, region_id, thresh_coast=0.5, inplace=False): """Apply a masked region filter""" - if not region_id in const.HTAP_REGIONS: + if region_id not in const.HTAP_REGIONS: raise ValueError( f"Invalid input for region_id: {region_id}, choose from: {const.HTAP_REGIONS}" ) @@ -1918,7 +1915,7 @@ def crop(self, lon_range=None, lat_range=None, time_range=None, region=None): return GriddedData(data, **suppl) else: assert len(time_range) == 2 - if all(isinstance(x, (str, np.datetime64)) for x in time_range): + if all(isinstance(x, str | np.datetime64) for x in time_range): time_range = (pd.Timestamp(time_range[0]), pd.Timestamp(time_range[1])) if all(isinstance(x, pd.Timestamp) for x in time_range): logger.info("Cropping along time axis based on Timestamps") @@ -2095,9 +2092,9 @@ def _check_meta_netcdf(self): """Get rid of empty entries and convert bools to int in meta""" meta_out = {} for k, v in self.metadata.items(): - if type(v) == bool: + if isinstance(v, bool): meta_out[k] = int(v) - elif v != None: + elif v is not None: meta_out[k] = v self.cube.attributes = meta_out @@ -2375,9 +2372,9 @@ def quickplot_map( fig matplotlib figure instance containing plot """ - if not "latitude" in self.dimcoord_names: + if "latitude" not in self.dimcoord_names: raise DataDimensionError("Missing latitude dimension...") - elif not "longitude" in self.dimcoord_names: + elif "longitude" not in self.dimcoord_names: raise DataDimensionError("Missing longitude dimension...") tstr = "" if "time" in self.dimcoord_names: diff --git a/pyaerocom/helpers.py b/pyaerocom/helpers.py index 4311021cb..ef663d44c 100644 --- a/pyaerocom/helpers.py +++ b/pyaerocom/helpers.py @@ -152,12 +152,12 @@ def extract_latlon_dataarray( lat_dimname = "lat" if lon_dimname is None: lon_dimname = "lon" - if not lat_dimname in arr.dims and lat_dimname == "lat": + if lat_dimname not in arr.dims and lat_dimname == "lat": for alias in const.COORDINFO["lat"].aliases: if alias in arr.dims: lat_dimname = alias break - if not lon_dimname in arr.dims and lon_dimname == "lon": + if lon_dimname not in arr.dims and lon_dimname == "lon": for alias in const.COORDINFO["lon"].aliases: if alias in arr.dims: lon_dimname = alias @@ -290,7 +290,6 @@ def check_coord_circular(coord_vals, modulus, rtol=1e-5): of the array) """ - from pyaerocom import const if len(coord_vals) < 2: logger.warning( @@ -739,7 +738,7 @@ def isrange(val): bool True, if input value corresponds to a range, else False. """ - if isinstance(val, (list, np.ndarray, tuple)): + if isinstance(val, list | np.ndarray | tuple): if len(val) == 2: return True return False @@ -755,9 +754,9 @@ def _check_stats_merge(statlist, var_name, pref_attr, fill_missing_nan): is_3d = [] stats = [] for stat in statlist: - if not var_name in stat: + if var_name not in stat: raise DataCoverageError(f"All input stations must contain {var_name} data") - elif pref_attr is not None and not pref_attr in stat: + elif pref_attr is not None and pref_attr not in stat: raise MetaDataError( f"Cannot sort station relevance by attribute {pref_attr}. " f"At least one of the input stations does not contain this attribute" @@ -1095,7 +1094,7 @@ def resample_timeseries(ts, freq, how=None, min_num_obs=None): how = "mean" elif "percentile" in how: p = int(how.split("percentile")[0]) - how = lambda x: np.nanpercentile(x, p) + how = lambda x: np.nanpercentile(x, p) # noqa: E731 freq, loffset = _get_pandas_freq_and_loffset(freq) resampler = ts.resample(freq) @@ -1155,7 +1154,7 @@ def resample_time_dataarray(arr, freq, how=None, min_num_obs=None): if not isinstance(arr, xr.DataArray): raise OSError(f"Invalid input for arr: need DataArray, got {type(arr)}") - elif not "time" in arr.dims: + elif "time" not in arr.dims: raise DataDimensionError("Cannot resample time: input DataArray has no time dimension") from pyaerocom.tstype import TsType @@ -1232,10 +1231,10 @@ def str_to_iris(key, **kwargs): corresponding iris analysis object (e.g. Aggregator, method) """ key = key.lower() - if not key in STR_TO_IRIS: + if key not in STR_TO_IRIS: raise KeyError( - "No iris.analysis object available for key %s, please " - "choose from %s" % (key, STR_TO_IRIS.keys()) + f"No iris.analysis object available for key {key}, please " + f"choose from {STR_TO_IRIS.keys()}" ) val = STR_TO_IRIS[key] if callable(val): @@ -1260,7 +1259,7 @@ def to_pandas_timestamp(value): value = str(value) if isinstance(value, pd.Timestamp): return value - elif isinstance(value, (str, np.datetime64, datetime, date)): + elif isinstance(value, str | np.datetime64 | datetime | date): return pd.Timestamp(value) else: try: @@ -1390,8 +1389,6 @@ def start_stop(start, stop=None, stop_sub_sec=True): def datetime2str(time, ts_type=None): - from pyaerocom import const - conv = TS_TYPE_DATETIME_CONV[ts_type] if is_year(time): return str(time) diff --git a/pyaerocom/helpers_landsea_masks.py b/pyaerocom/helpers_landsea_masks.py index e841b7af4..36bc058ea 100644 --- a/pyaerocom/helpers_landsea_masks.py +++ b/pyaerocom/helpers_landsea_masks.py @@ -73,7 +73,7 @@ def download_htap_masks(regions_to_download=None): paths = [] for region in regions_to_download: - if not region in const.HTAP_REGIONS: + if region not in const.HTAP_REGIONS: raise ValueError(f"No such HTAP region {region}") elif region == "EAS": filename = f"{region}htap.nc" @@ -118,7 +118,7 @@ def get_htap_mask_files(*region_ids): raise FileNotFoundError("HTAP mask directory does not exist") out = [] for region in region_ids: - if not region in const.HTAP_REGIONS: + if region not in const.HTAP_REGIONS: raise ValueError(f"No such HTAP region {region}") files = glob.glob(os.path.join(mask_dir, f"{region}*.nc")) if len(files) != 1: diff --git a/pyaerocom/io/aerocom_browser.py b/pyaerocom/io/aerocom_browser.py index 11c823e57..53c113855 100644 --- a/pyaerocom/io/aerocom_browser.py +++ b/pyaerocom/io/aerocom_browser.py @@ -123,7 +123,7 @@ def _browse(self, name_or_pattern, ignorecase=True, return_if_match=True): return _dir else: - _msgs.append("directory %s does not exist\n" % search_dir) + _msgs.append(f"directory {search_dir} does not exist\n") for msg in _msgs: logger.info(msg) diff --git a/pyaerocom/io/aux_components_fun.py b/pyaerocom/io/aux_components_fun.py index 344619aa9..5595ce84b 100644 --- a/pyaerocom/io/aux_components_fun.py +++ b/pyaerocom/io/aux_components_fun.py @@ -1,20 +1,16 @@ import logging import cf_units -import iris import numpy as np from geonum.atmosphere import T0_STD, p0 -from pyaerocom._lowlevel_helpers import merge_dicts -from pyaerocom.helpers import copy_coords_cube from pyaerocom.io.aux_read_cubes import ( CUBE_MATHS, _check_input_iscube, _check_same_units, add_cubes, ) -from pyaerocom.molmasses import get_mmr_to_vmr_fac, get_molmass -from pyaerocom.units_helpers import get_unit_conversion_fac +from pyaerocom.molmasses import get_molmass logger = logging.getLogger(__name__) diff --git a/pyaerocom/io/aux_read_cubes.py b/pyaerocom/io/aux_read_cubes.py index 7c535b8fa..a19dbd012 100644 --- a/pyaerocom/io/aux_read_cubes.py +++ b/pyaerocom/io/aux_read_cubes.py @@ -25,7 +25,7 @@ def _apply_operator_cubes(cube1, cube2, operator_name, allow_coord_merge=True): - if not operator_name in CUBE_MATHS: + if operator_name not in CUBE_MATHS: raise NotImplementedError(f"No such arithmetic cube operator implemented: {operator_name}") fun = CUBE_MATHS[operator_name] try: @@ -98,7 +98,7 @@ def merge_meta_cubes(cube1, cube2): try: if cube1.attributes["ts_type"] == cube2.attributes["ts_type"]: ts_type = cube1.attributes["ts_type"] - except: + except Exception: pass return { diff --git a/pyaerocom/io/cachehandler_ungridded.py b/pyaerocom/io/cachehandler_ungridded.py index a6a84e0f7..7fbc98ed8 100644 --- a/pyaerocom/io/cachehandler_ungridded.py +++ b/pyaerocom/io/cachehandler_ungridded.py @@ -151,7 +151,7 @@ def _check_pkl_head_vs_database(self, in_handle): raise CacheReadError("Invalid cache file") pya_version = "pyaerocom_version" for k, v in head.items(): - if not k in current: + if k not in current: raise CacheReadError(f"Invalid cache header key: {k}") else: if k == pya_version: @@ -325,7 +325,7 @@ def write(self, data, var_or_file_name=None, cache_dir=None): ) var_name = data.contains_vars[0] - elif not var_name in data.contains_vars: + elif var_name not in data.contains_vars: raise CacheWriteError( f"Cannot write cache file: variable {var_name} " f"does not exist in input UngriddedData object" diff --git a/pyaerocom/io/cams2_83/read_obs.py b/pyaerocom/io/cams2_83/read_obs.py index 97bf92913..2682cc79d 100644 --- a/pyaerocom/io/cams2_83/read_obs.py +++ b/pyaerocom/io/cams2_83/read_obs.py @@ -2,9 +2,9 @@ import logging import time +from collections.abc import Iterator from datetime import date, datetime, timedelta from pathlib import Path -from typing import Iterator import numpy as np import pandas as pd @@ -34,7 +34,7 @@ def obs_paths( root_path: Path | str = DATA_FOLDER_PATH, analysis: bool = False, ) -> Iterator[Path]: - for date in dates: + for date in dates: # noqa: F402 if isinstance(date, str): date = datetime.strptime(date, "%Y%m%d").date() if isinstance(date, datetime): @@ -100,7 +100,7 @@ def read( files = files[:last_file] start = time.time() - logger.info(f"Start read obs") + logger.info("Start read obs") # lazy data_iterator returns immediately, unpacked in from_station_data data_iterator = self.__reader(vars_to_retrieve, files) ungriddeddata = UngriddedData.from_station_data(data_iterator) diff --git a/pyaerocom/io/cams2_83/reader.py b/pyaerocom/io/cams2_83/reader.py index d57338263..3fbd06cba 100644 --- a/pyaerocom/io/cams2_83/reader.py +++ b/pyaerocom/io/cams2_83/reader.py @@ -2,22 +2,19 @@ import logging import re +from collections.abc import Iterator from datetime import date, datetime, timedelta from pathlib import Path -from typing import Iterator import numpy as np import pandas as pd import xarray as xr from tqdm import tqdm -from pyaerocom import const from pyaerocom.griddeddata import GriddedData from pyaerocom.io.cams2_83.models import ModelData, ModelName, RunType from pyaerocom.io.gridded_reader import GriddedReader -# from pyaerocom.units_helpers import UALIASES - """ TODO: @@ -88,7 +85,7 @@ def model_paths( root_path: Path | str = DATA_FOLDER_PATH, run: str | RunType = RunType.FC, ) -> Iterator[Path]: - for date in dates: + for date in dates: # noqa: F402 path = __model_path(model, date, run=run, root_path=root_path) if not path.is_file(): logger.warning(f"Could not find {path.name}. Skipping {date}") @@ -316,7 +313,7 @@ def run_type(self): def run_type(self, val): if val is None: raise AttributeError("run_type cannot be set as None") - elif type(val) != RunType: + elif not isinstance(val, RunType): raise AttributeError(f"run_type cannot be set as {type(val)}, but must be a RunType") self._run_type = val @@ -382,7 +379,7 @@ def daterange(self) -> pd.DatetimeIndex: @daterange.setter def daterange(self, dates: pd.DatetimeIndex | list[datetime] | tuple[datetime]): - if not isinstance(dates, (pd.DatetimeIndex, list, tuple)): + if not isinstance(dates, pd.DatetimeIndex | list | tuple): raise TypeError(f"{dates} need to be a pandas DatetimeIndex or 2 datetimes") self._daterange = parse_daterange(dates) diff --git a/pyaerocom/io/ebas_file_index.py b/pyaerocom/io/ebas_file_index.py index a6afce593..ee3154272 100644 --- a/pyaerocom/io/ebas_file_index.py +++ b/pyaerocom/io/ebas_file_index.py @@ -161,14 +161,14 @@ def make_query_str(self, what=None, distinct=True, **kwargs): req += f"station_altitude>{low} and station_altitude<{high}" add_cond += 1 if self.lon_range is not None: - l, r = self.lon_range + west, east = self.lon_range req += " and " if add_cond else " where " - req += f"station_longitude>{l} and station_longitude<{r}" + req += f"station_longitude>{west} and station_longitude<{east}" add_cond += 1 if self.lat_range is not None: - s, n = self.lat_range + south, north = self.lat_range req += " and " if add_cond else " where " - req += f"station_latitude>{s} and station_latitude<{n}" + req += f"station_latitude>{south} and station_latitude<{north}" add_cond += 1 if self.instrument_types is not None: req += " and " if add_cond else " where " diff --git a/pyaerocom/io/ebas_nasa_ames.py b/pyaerocom/io/ebas_nasa_ames.py index 1e08d250f..cbe81e6b7 100644 --- a/pyaerocom/io/ebas_nasa_ames.py +++ b/pyaerocom/io/ebas_nasa_ames.py @@ -66,9 +66,9 @@ def __init__(self, name, is_var, is_flag, unit="1"): def get_wavelength_nm(self): """Try to access wavelength information in nm (as float)""" - if not "wavelength" in self: + if "wavelength" not in self: raise KeyError(f"Column variable {self.name} does not contain wavelength information") - elif not "nm" in self.wavelength: + elif "nm" not in self.wavelength: raise NotImplementedError("Wavelength definition is not in nm") return float(self.wavelength.split("nm")[0].strip()) @@ -126,12 +126,12 @@ class NasaAmesHeader: _HEAD_ROWS_MANDATORY = [0, 5, 8, 9, 10, 11] # conversion methods for first 13 header lines of - CONV_STR = lambda l: str(l.strip()) - CONV_PI = lambda l: "; ".join([x.strip() for x in l.split(";")]) - CONV_MULTIINT = lambda l: [int(x) for x in l.strip().split()] - CONV_MULTIFLOAT = lambda l: [float(x) for x in l.strip().split()] - CONV_INT = lambda l: int(l.strip()) - CONV_FLOAT = lambda l: float(l.strip()) + CONV_STR = lambda l: str(l.strip()) # noqa: E731,E741 + CONV_PI = lambda l: "; ".join([x.strip() for x in l.split(";")]) # noqa: E731,E741 + CONV_MULTIINT = lambda l: [int(x) for x in l.strip().split()] # noqa: E731,E741 + CONV_MULTIFLOAT = lambda l: [float(x) for x in l.strip().split()] # noqa: E731,E741 + CONV_INT = lambda l: int(l.strip()) # noqa: E731,E741 + CONV_FLOAT = lambda l: float(l.strip()) # noqa: E731,E741 _STARTDATE_FMT = "%Y%m%d%H%M%S" # the parameters that are always in the beginning of the file @@ -432,7 +432,7 @@ def col_nums_vars(self): @property def base_date(self): """Base date of data as numpy.datetime64[s]""" - if not "timezone" in self.meta: + if "timezone" not in self.meta: raise AttributeError( "Fatal: could not infer base date. Timezone is not available in file header" ) @@ -513,7 +513,7 @@ def compute_time_stamps(self): ) offs = self.base_date unit = self.time_unit - if not unit in self.TIMEUNIT2SECFAC: + if unit not in self.TIMEUNIT2SECFAC: raise ValueError(f"Invalid unit for temporal resolution: {unit}") mulfac = self.TIMEUNIT2SECFAC[unit] @@ -581,7 +581,7 @@ def _quality_check(self): "Mismatch between variable definitions in header and " "number of data columns in table\n" ) - if not "timezone" in self.meta: + if "timezone" not in self.meta: msgs += "Timezone not defined in metadata" if msgs: raise AttributeError(f"Quality check failed. Messages: {msgs}") @@ -735,7 +735,7 @@ def _read_vardef_line(self, line_from_file): unit = spl[1] data = EbasColDef(name=name, is_flag=True, is_var=False, unit=unit) - if not "numflag" in name: + if "numflag" not in name: data.is_var = True data.is_flag = False for item in spl[2:]: diff --git a/pyaerocom/io/ebas_varinfo.py b/pyaerocom/io/ebas_varinfo.py index fbfb7fc15..a94e14ae5 100644 --- a/pyaerocom/io/ebas_varinfo.py +++ b/pyaerocom/io/ebas_varinfo.py @@ -123,10 +123,10 @@ def parse_from_ini(self, var_name: str, conf_reader: ConfigParser | None = None) if conf_reader is None: conf_reader = self.open_config() - if not var_name in conf_reader: + if var_name not in conf_reader: # this will raise Variable var_name = const.VARS[var_name].var_name_aerocom - if not var_name in conf_reader: + if var_name not in conf_reader: raise VarNotAvailableError( f"Variable {var_name} is not available in EBAS interface" ) diff --git a/pyaerocom/io/fileconventions.py b/pyaerocom/io/fileconventions.py index 5ef5476d0..25258f55e 100644 --- a/pyaerocom/io/fileconventions.py +++ b/pyaerocom/io/fileconventions.py @@ -355,7 +355,7 @@ def import_default(self, name: str): conf_reader = ConfigParser() with resources.path("pyaerocom.data", "file_conventions.ini") as path: conf_reader.read(path) - if not name in conf_reader: + if name not in conf_reader: raise NameError(f"No default available for {name}") self.name = name for key, val in conf_reader[name].items(): diff --git a/pyaerocom/io/gaw/reader.py b/pyaerocom/io/gaw/reader.py index 038638967..2e61e992b 100644 --- a/pyaerocom/io/gaw/reader.py +++ b/pyaerocom/io/gaw/reader.py @@ -97,7 +97,7 @@ def read_file(self, filename, vars_to_retrieve=None, vars_as_series=False): vars_to_retrieve = [vars_to_retrieve] for var in vars_to_retrieve: - if not var in self.PROVIDES_VARIABLES: + if var not in self.PROVIDES_VARIABLES: raise ValueError(f"Invalid input variable {var}") # Iterate over the lines of the file @@ -360,7 +360,7 @@ def read(self, vars_to_retrieve=None, files=None, first_file=None, last_file=Non data_obj._data[start:stop, data_obj._VARINDEX] = var_idx meta_idx[meta_key][var] = np.arange(start, stop) - if not var in data_obj.var_idx: + if var not in data_obj.var_idx: data_obj.var_idx[var] = var_idx idx += totnum diff --git a/pyaerocom/io/ghost/reader.py b/pyaerocom/io/ghost/reader.py index b6eaf6064..e6af6469d 100644 --- a/pyaerocom/io/ghost/reader.py +++ b/pyaerocom/io/ghost/reader.py @@ -287,7 +287,7 @@ def _eval_flags_slice(slc, invalid_flags): def _ts_type_from_data_dir(self): try: freq = str(TsType(os.path.basename(self.data_dir))) - except Exception as e: # pragma: no cover + except Exception: # pragma: no cover freq = "undefined" self.TS_TYPES[self.data_id] = freq return freq @@ -342,7 +342,7 @@ def read_file(self, filename, var_to_read=None, invalidate_flags=None, var_to_wr with xr.open_dataset(filename) as ds: if not {"station", "time"}.issubset(ds.dims): # pragma: no cover raise AttributeError("Missing dimensions") - if not "station_name" in ds: # pragma: no cover + if "station_name" not in ds: # pragma: no cover raise AttributeError("No variable station_name found") stats = [] @@ -453,7 +453,7 @@ def compute_additional_vars(self, statlist_from_file, vars_to_compute): can_compute = True requires = self.AUX_REQUIRES[var] for req in requires: - if not req in first_stat: + if req not in first_stat: can_compute = False if can_compute: # this will add the variable data to each station data in @@ -461,7 +461,7 @@ def compute_additional_vars(self, statlist_from_file, vars_to_compute): statlist_from_file = self.AUX_FUNS[var](statlist_from_file, var, *requires) statlist_from_file = self._add_flags_var_to_compute(statlist_from_file, var) - if not var in vars_added: + if var not in vars_added: vars_added.append(var) return (statlist_from_file, vars_added) @@ -593,7 +593,7 @@ def read( start = idx + j * num_times stop = start + num_times - if not var_to_write in data_obj.var_idx: + if var_to_write not in data_obj.var_idx: var_count_glob += 1 var_idx = var_count_glob data_obj.var_idx[var_to_write] = var_idx diff --git a/pyaerocom/io/gridded_reader.py b/pyaerocom/io/gridded_reader.py index 38d839d58..9ba42cb1f 100644 --- a/pyaerocom/io/gridded_reader.py +++ b/pyaerocom/io/gridded_reader.py @@ -1,5 +1,5 @@ import abc -from typing import Iterator +from collections.abc import Iterator from pyaerocom.griddeddata import GriddedData diff --git a/pyaerocom/io/helpers.py b/pyaerocom/io/helpers.py index ba20d592e..da04a192b 100644 --- a/pyaerocom/io/helpers.py +++ b/pyaerocom/io/helpers.py @@ -209,7 +209,7 @@ def get_standard_name(var_name): VariableDefinitionError if standarad name is not set for variable in *variables.ini* file """ - if not var_name in const.VARS: + if var_name not in const.VARS: raise VarNotAvailableError(f"No such variable {var_name}. Check variables.ini") name = const.VARS[var_name].standard_name if name is None: @@ -256,7 +256,7 @@ def get_obsnetwork_dir(obs_id): IOError if directory does not exist """ - if not obs_id in const.OBSLOCS_UNGRIDDED: + if obs_id not in const.OBSLOCS_UNGRIDDED: raise ValueError(f"Observation network ID {obs_id} does not exist") data_dir = const.OBSLOCS_UNGRIDDED[obs_id] diff --git a/pyaerocom/io/helpers_units.py b/pyaerocom/io/helpers_units.py index e8f1d8952..87ffbbd91 100644 --- a/pyaerocom/io/helpers_units.py +++ b/pyaerocom/io/helpers_units.py @@ -1,4 +1,3 @@ -import numpy as np import pandas as pd from scipy.constants import Avogadro @@ -128,7 +127,6 @@ def unitconv_wet_depo_bck(data, time, ts_type="monthly"): # kg SO4 m-2 s-1 to kg S/ha mm_so4 = 0.001 * 32.065 + 0.001 * 15.999 * 4 # kg/mol mm_s = 32.065 * 0.001 # kg/mol - mm_o = 0.001 * 15.999 * 4 # kg/mol days_in_month = time.dt.daysinmonth monthly_to_sec = days_in_month * 24 * 60 * 60 # Seconds in each diff --git a/pyaerocom/io/icos/reader.py b/pyaerocom/io/icos/reader.py index d8a6d08ae..fb8b3c818 100644 --- a/pyaerocom/io/icos/reader.py +++ b/pyaerocom/io/icos/reader.py @@ -194,7 +194,7 @@ def to_stationdata(cls, ds: xr.Dataset, station_name: str) -> StationData: station["dtime"] = ds["time"].values for var in ds.data_vars: - if not var in cls.PROVIDES_VARIABLES: + if var not in cls.PROVIDES_VARIABLES: continue station[var] = ds[var].to_series() station["var_info"][var] = {"units": ds[var].units} diff --git a/pyaerocom/io/icpforests/metadata.py b/pyaerocom/io/icpforests/metadata.py index 9a5055b67..22b8e24eb 100644 --- a/pyaerocom/io/icpforests/metadata.py +++ b/pyaerocom/io/icpforests/metadata.py @@ -1,7 +1,6 @@ from __future__ import annotations from datetime import datetime, timedelta -from typing import Tuple import numpy as np import pandas as pd @@ -396,8 +395,8 @@ def __init__(self, plot_file: str) -> None: def read_file(self, altitudes: dict[str, int]) -> dict[int, dict[int, dict[int, Plot]]]: plots: dict[int, dict[int, dict[int, Plot]]] = {} - print(f"Starting to read plot metadata") - with open(self.plot_file, "r") as f: + print("Starting to read plot metadata") + with open(self.plot_file) as f: f.readline() for line in f: words = line.split(";") @@ -434,7 +433,7 @@ def read_file(self, altitudes: dict[str, int]) -> dict[int, dict[int, dict[int, plots[country_code][plot_code][sampler_code].add_survey_year( survey_year, start, stop, periods ) - print(f"Done read plot metadata") + print("Done read plot metadata") self.plots = plots return plots @@ -458,7 +457,7 @@ def get_days(self, year: int, country_code: int, plot_code: int, sampler_code: i def get_position( self, year: int, country_code: int, plot_code: int, sampler_code: int - ) -> Tuple[float, float, int]: + ) -> tuple[float, float, int]: lat = self._coord_to_desimal(self.plots[country_code][plot_code][sampler_code].lat) lon = self._coord_to_desimal(self.plots[country_code][plot_code][sampler_code].lon) alt = self.plots[country_code][plot_code][sampler_code].alt @@ -479,7 +478,7 @@ def _coord_to_desimal(self, coord: str) -> float: minute = int(coord[-2:]) coord = coord[:-2] else: - return sign * ((int(coord) / 60.0)) + return sign * (int(coord) / 60.0) degree = int(coord) diff --git a/pyaerocom/io/icpforests/reader.py b/pyaerocom/io/icpforests/reader.py index 51633591c..9a4acf5e2 100644 --- a/pyaerocom/io/icpforests/reader.py +++ b/pyaerocom/io/icpforests/reader.py @@ -4,7 +4,6 @@ import os from datetime import datetime from pathlib import Path -from typing import Tuple import numpy as np from tqdm import tqdm @@ -208,14 +207,14 @@ def read_file(self, filename, vars_to_retrieve=None, last_line=None): stations: dict[str, dict[str, Station]] = {} if self.metadata is None: if self.data_dir is None: - raise ValueError(f"Data Dir is not read yet") + raise ValueError("Data Dir is not read yet") self.metadata = MetadataReader(self.data_dir) if vars_to_retrieve is None: vars_to_retrieve = self.PROVIDES_VARIABLES - with open(filename, "r") as f: + with open(filename) as f: f.readline() for line_nr, line in tqdm(enumerate(f)): @@ -234,8 +233,6 @@ def read_file(self, filename, vars_to_retrieve=None, last_line=None): ): continue - sampler_type = self.metadata.deposition_type[sampler_code] - period = int(words[6]) start = words[4] stop = words[5] @@ -260,7 +257,7 @@ def read_file(self, filename, vars_to_retrieve=None, last_line=None): self.metadata.plots.plots[country_code][plot_code][sampler_code].survey_years[ year ] - except KeyError as e: + except KeyError: logger.warning( f"Year {year} can't be found for {country_code=}, {plot_code=}, {sampler_code=}. Only years found are {self.metadata.plots.plots[country_code][plot_code][sampler_code].survey_years.keys()}" ) @@ -370,7 +367,7 @@ def _get_days_date_ts_type( period: int, start: str | datetime, stop: str | datetime, - ) -> Tuple[float | None, datetime | None, str | None]: + ) -> tuple[float | None, datetime | None, str | None]: if start != "" and stop != "": if isinstance(start, str): start = datetime.strptime(start, "%Y-%m-%d") @@ -383,7 +380,7 @@ def _get_days_date_ts_type( return (stop - start).days, start, self._get_tstype(start, stop) if self.metadata is None: - raise ValueError(f"Metadata is not read yet") + raise ValueError("Metadata is not read yet") try: days = self.metadata.plots.get_days(year, country_code, plot_code, sampler_code) diff --git a/pyaerocom/io/iris_io.py b/pyaerocom/io/iris_io.py index b00b78731..21ead8692 100644 --- a/pyaerocom/io/iris_io.py +++ b/pyaerocom/io/iris_io.py @@ -275,7 +275,7 @@ def check_dim_coords_cube(cube): def _check_cube_unitless(cube): """Make sure unit in Cube is 1 if variable is dimensionless""" var = cube.var_name - if not var in const.VARS: + if var not in const.VARS: raise VariableDefinitionError(f"No such pyaerocom default variable: {cube.var_name}") unit = cf_units.Unit(cube.units) @@ -387,7 +387,7 @@ def check_time_coord(cube, ts_type, year): """ if isinstance(ts_type, str): ts_type = TsType(ts_type) - if not "time" in get_coord_names_cube(cube): + if "time" not in get_coord_names_cube(cube): raise AttributeError("Cube does not contain time dimension") tdim = cube.coord("time") @@ -602,7 +602,7 @@ def concatenate_iris_cubes(cubes, error_on_mismatch=True): try: cubes_concat = iris.cube.CubeList.concatenate_cube(cubes, error_on_mismatch) - except Exception as e: + except Exception: if _check_correct_dtypes_timedim_cube_list(cubes): cubes_concat = iris.cube.CubeList.concatenate_cube(cubes, error_on_mismatch) else: diff --git a/pyaerocom/io/mscw_ctm/reader.py b/pyaerocom/io/mscw_ctm/reader.py index d4da15b04..6fb3c016e 100755 --- a/pyaerocom/io/mscw_ctm/reader.py +++ b/pyaerocom/io/mscw_ctm/reader.py @@ -1,10 +1,8 @@ import functools -import glob import logging import os import re import warnings -from collections import namedtuple import numpy as np import xarray as xr @@ -27,17 +25,13 @@ calc_concNno3pm25, calc_concno3pm10, calc_concno3pm25, - calc_concNtnh, - calc_concpolyol, calc_concso4t, calc_concSso2, calc_concsspm25, calc_conNtnh_emep, - calc_conNtno3, calc_conNtno3_emep, calc_vmrno2, calc_vmro3, - calc_vmrox, calc_vmrox_from_conc, identity, subtract_dataarrays, @@ -345,7 +339,7 @@ def _data_dir(self) -> str | None: Directory containing netcdf files """ if self._private.data_dir is None: - raise AttributeError(f"data_dir needs to be set before accessing") + raise AttributeError("data_dir needs to be set before accessing") return self._private.data_dir @_data_dir.setter @@ -587,7 +581,7 @@ def _filename_from_ts_type(self, ts_type): freq. """ - if not ts_type in self.REVERSE_FREQ_CODES: + if ts_type not in self.REVERSE_FREQ_CODES: raise ValueError(f"unknown ts_type={ts_type}") freq = self.REVERSE_FREQ_CODES[ts_type] return self.FILE_FREQ_TEMPLATE.format(freq=freq) diff --git a/pyaerocom/io/pyaro/pyaro_config.py b/pyaerocom/io/pyaro/pyaro_config.py index 9c66f6660..3ee96e529 100644 --- a/pyaerocom/io/pyaro/pyaro_config.py +++ b/pyaerocom/io/pyaro/pyaro_config.py @@ -3,7 +3,7 @@ import logging from importlib import resources from pathlib import Path -from typing import ClassVar, Optional +from typing import ClassVar import yaml from pydantic import BaseModel, ConfigDict @@ -30,7 +30,7 @@ class PyaroConfig(BaseModel): data_id: str filename_or_obj_or_url: str filters: dict[str, dict[str, list[str]]] - name_map: Optional[dict[str, str]] = None # no Unit conversion option + name_map: dict[str, str] | None = None # no Unit conversion option ########################## # Save and load methods @@ -39,7 +39,7 @@ class PyaroConfig(BaseModel): def json_repr(self): return self.model_dump() - def save(self, path: Optional[Path] = None) -> None: + def save(self, path: Path | None = None) -> None: name = self.name if not path.is_dir(): @@ -60,7 +60,7 @@ def from_dict(cls, data: dict): return PyaroConfig.model_validate(data) @classmethod - def load(cls, name: str, filepath: Optional[Path] = None): + def load(cls, name: str, filepath: Path | None = None): if filepath is not None: if filepath.is_dir(): raise ValueError(f"Filepath {filepath} is a directory not a file") @@ -75,7 +75,7 @@ def load(cls, name: str, filepath: Optional[Path] = None): raise ValueError(f"Config {name} was not found in any catalogs") @classmethod - def list_configs(cls, filepath: Optional[Path] = None) -> list[str]: + def list_configs(cls, filepath: Path | None = None) -> list[str]: data = cls.load_catalog() if filepath is not None: logger.info(f"Updating with private catalog {filepath}") @@ -84,14 +84,14 @@ def list_configs(cls, filepath: Optional[Path] = None) -> list[str]: return list(data.keys()) @classmethod - def load_catalog(cls, filepath: Optional[Path] = None) -> dict: + def load_catalog(cls, filepath: Path | None = None) -> dict: if filepath is None: filepath = cls._DEFAULT_CATALOG if not filepath.exists(): return {} - with open(filepath, "r") as f: + with filepath.open() as f: data = yaml.safe_load(f) return data diff --git a/pyaerocom/io/pyaro/read_pyaro.py b/pyaerocom/io/pyaro/read_pyaro.py index 787c1551e..41cfff05f 100644 --- a/pyaerocom/io/pyaro/read_pyaro.py +++ b/pyaerocom/io/pyaro/read_pyaro.py @@ -2,7 +2,7 @@ import logging from copy import deepcopy -from typing import NewType, Optional, Union +from typing import NewType import numpy as np from pyaro import list_timeseries_engines, open_timeseries @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) -MetadataEntry = NewType("MetadataEntry", dict[str, Union[str, list[str]]]) +MetadataEntry = NewType("MetadataEntry", dict[str, str | list[str]]) Metadata = NewType("Metadata", dict[str, MetadataEntry]) @@ -79,7 +79,7 @@ def read_file(self, filename, vars_to_retrieve=None): def _check_id(self): avail_readers = list_timeseries_engines() - if not self.config.data_id in avail_readers: + if self.config.data_id not in avail_readers: logger.warning( f"Could not find {self.config.data_id} in list of available Pyaro readers: {avail_readers}" ) @@ -147,7 +147,6 @@ def _convert_to_ungriddeddata(self, pyaro_data: dict[str, Data]) -> UngriddedDat vars = list(pyaro_data.keys()) total_size = sum(list(var_size.values())) units = {var: {"units": pyaro_data[var]._units} for var in pyaro_data} - ts_types: dict[str, dict[str, Optional[TsType]]] = {} # Object necessary for ungriddeddata var_idx = {var: i for i, var in enumerate(vars)} @@ -236,7 +235,7 @@ def _get_additional_metadata(self, station: Station) -> list[dict[str, str]]: return station.metadata def _make_single_ungridded_metadata( - self, station: Station, name: str, ts_type: Optional[TsType], units: dict[str, str] + self, station: Station, name: str, ts_type: TsType | None, units: dict[str, str] ) -> MetadataEntry: entry = dict( data_id=self.config.name, @@ -284,7 +283,7 @@ def _calculate_ts_type(self, start: np.datetime64, stop: np.datetime64) -> TsTyp return ts_type def _add_ts_type_to_metadata( - self, metadata: Metadata, ts_types: dict[str, Optional[TsType]] + self, metadata: Metadata, ts_types: dict[str, TsType | None] ) -> Metadata: new_metadata: Metadata = deepcopy(metadata) for idx in new_metadata: diff --git a/pyaerocom/io/read_aasetal.py b/pyaerocom/io/read_aasetal.py index 79ffbb89a..a3f64742b 100644 --- a/pyaerocom/io/read_aasetal.py +++ b/pyaerocom/io/read_aasetal.py @@ -98,7 +98,7 @@ def DEFAULT_VARS(self): return self.PROVIDES_VARIABLES def _get_time_stamps(self, df): - tconv = lambda yr, m: np.datetime64(f"{yr:04d}-{m:02d}-{1:02d}", "s") + tconv = lambda yr, m: np.datetime64(f"{yr:04d}-{m:02d}-{1:02d}", "s") # noqa: E731 dates_alt = [tconv(yr, m) for yr, m in zip(df.year.values, df.month.values)] return np.asarray(dates_alt) @@ -245,7 +245,7 @@ def read(self, vars_to_retrieve=None): for file in files: filename = os.path.basename(file) - if not filename in self.FILES_CONTAIN: + if filename not in self.FILES_CONTAIN: raise OSError(f"Invalid file name {filename}, this should not happen.") var_matches = [var for var in vars_to_retrieve if var in self.FILES_CONTAIN[filename]] if len(var_matches) == 0: @@ -289,7 +289,7 @@ def read(self, vars_to_retrieve=None): start = idx + var_count * num_times stop = start + num_times - if not var in data_obj.var_idx: + if var not in data_obj.var_idx: varindex += 1 data_obj.var_idx[var] = varindex var_idx = varindex diff --git a/pyaerocom/io/read_airnow.py b/pyaerocom/io/read_airnow.py index 5ccc44da5..0eed97344 100644 --- a/pyaerocom/io/read_airnow.py +++ b/pyaerocom/io/read_airnow.py @@ -289,7 +289,7 @@ def _read_file(self, file): on_bad_lines="skip", dtype={2: str, 4: float, 7: float}, ) - except: + except Exception: encoding = self.get_file_encoding(file) df = pd.read_csv( file, @@ -331,8 +331,7 @@ def _read_files(self, files, vars_to_retrieve): """ logger.info("Read AirNow data file(s)") file_vars_to_retrieve = [self.VAR_MAP[x] for x in vars_to_retrieve] - # initialize empty dataframe - varcol = self.FILE_COL_NAMES.index("variable") + arrs = [] unique_stat_ids = None for i in tqdm(range(len(files)), disable=None): @@ -342,13 +341,13 @@ def _read_files(self, files, vars_to_retrieve): arrs.append(filedata[filedata["variable"] == filevar].values) if unique_stat_ids is None: unique_stat_ids = np.unique( - (arrs[-1][:, self.FILE_COL_NAMES.index("station_id")]) + arrs[-1][:, self.FILE_COL_NAMES.index("station_id")] ) else: try: unique_stat_ids = np.union1d( unique_stat_ids, - np.unique((arrs[-1][:, self.FILE_COL_NAMES.index("station_id")])), + np.unique(arrs[-1][:, self.FILE_COL_NAMES.index("station_id")]), ) except (ValueError, TypeError): print(arrs[-1][:, self.FILE_COL_NAMES.index("station_id")]) @@ -418,11 +417,11 @@ def _filedata_to_statlist( # there are stations with a all numeric station ID, but type hints in pd.read_csv made sure # they are read as str... if unique_stat_ids is None: - statlist = np.unique((subset[:, statcol])) + statlist = np.unique(subset[:, statcol]) else: statlist = unique_stat_ids for stat_id in tqdm(statlist, desc=var, disable=None): - if not stat_id in stat_ids: + if stat_id not in stat_ids: continue statmask = subset[:, statcol] == stat_id if statmask.sum() == 0: @@ -485,7 +484,7 @@ def read_file(self, filename, vars_to_retrieve=None): encoding = "cp863" with open(filename, encoding=encoding) as infile: linedata = infile.readlines() - except: + except Exception: logger.info(f"unforeseen encoding in file {filename}! Trying to determine encoding") try: encoding = self.get_file_encoding(filename) @@ -493,9 +492,9 @@ def read_file(self, filename, vars_to_retrieve=None): with open(filename, encoding=encoding) as infile: linedata = infile.readlines() except MemoryError: - logger.info(f"could not determine encoding due to MemoryError: Skipping file...") + logger.info("could not determine encoding due to MemoryError: Skipping file...") raise DataRetrievalError( - f"could not determine encoding due to MemoryError: Skipping file..." + "could not determine encoding due to MemoryError: Skipping file..." ) return ret_data diff --git a/pyaerocom/io/read_earlinet.py b/pyaerocom/io/read_earlinet.py index 54247865e..bba5f4d28 100755 --- a/pyaerocom/io/read_earlinet.py +++ b/pyaerocom/io/read_earlinet.py @@ -211,7 +211,7 @@ def read_file(self, filename, vars_to_retrieve=None, read_err=None, remove_outli # create empty arrays for all variables that are supposed to be read # from file for var in vars_to_read: - if not var in self._var_info: + if var not in self._var_info: self._var_info[var] = Variable(var) var_info = self._var_info @@ -532,7 +532,7 @@ def read( # Is floating point single value time = stat.dtime[0] for var in stat.vars_available: - if not var in data_obj.var_idx: + if var not in data_obj.var_idx: VAR_IDX += 1 data_obj.var_idx[var] = VAR_IDX @@ -586,11 +586,11 @@ def read( if read_err: data_obj._data[idx:stop, col_idx["dataerr"]] = err - if not var in meta_idx[meta_key]: + if var not in meta_idx[meta_key]: meta_idx[meta_key][var] = [] meta_idx[meta_key][var].extend(list(range(idx, stop))) - if not var in metadata[meta_key]["variables"]: + if var not in metadata[meta_key]["variables"]: metadata[meta_key]["variables"].append(var) idx += add @@ -613,7 +613,7 @@ def _get_exclude_filelist(self): # pragma: no cover files = glob.glob(f"{self.data_dir}/EXCLUDE/*.txt") for i, file in enumerate(files): - if not os.path.basename(file) in self.EXCLUDE_CASES: + if os.path.basename(file) not in self.EXCLUDE_CASES: continue count = 0 num = None @@ -661,7 +661,7 @@ def get_file_list(self, vars_to_retrieve=None, pattern=None): logger.info("Fetching EARLINET data files. This might take a while...") patterns = [] for var in vars_to_retrieve: - if not var in self.VAR_PATTERNS_FILE: + if var not in self.VAR_PATTERNS_FILE: from pyaerocom.exceptions import VarNotAvailableError raise VarNotAvailableError(f"Input variable {var} is not supported") @@ -671,7 +671,7 @@ def get_file_list(self, vars_to_retrieve=None, pattern=None): if "." in pattern: raise NotImplementedError("filetype delimiter . not supported") spl = _pattern.split(".") - if not "*" in spl[0]: + if "*" not in spl[0]: raise AttributeError(f"Invalid file pattern: {_pattern}") spl[0] = spl[0].replace("*", pattern) _pattern = ".".join(spl) @@ -684,7 +684,7 @@ def get_file_list(self, vars_to_retrieve=None, pattern=None): for _pattern in patterns: for path in paths: file = os.path.basename(path) - if not _pattern in file: + if _pattern not in file: continue elif file in exclude: self.excluded_files.append(path) diff --git a/pyaerocom/io/read_ebas.py b/pyaerocom/io/read_ebas.py index feb5ac60c..0f07b01cb 100644 --- a/pyaerocom/io/read_ebas.py +++ b/pyaerocom/io/read_ebas.py @@ -689,7 +689,7 @@ def get_file_list(self, vars_to_retrieve, **constraints): files_aux_req[_var] = sorted(paths) for _var in vars_to_retrieve: - if not _var in files_vars: + if _var not in files_vars: files_vars[_var] = self._merge_auxvar_lists(_var, files_aux_req) self._lists_orig = files_vars @@ -726,7 +726,7 @@ def _merge_auxvar_lists(self, aux_var, files_aux_req): return [] remaining = None for _vreq in _required: - if not _vreq in files_aux_req: + if _vreq not in files_aux_req: return [] _lst = files_aux_req[_vreq] if remaining is None: @@ -780,7 +780,7 @@ def _get_var_cols(self, ebas_var_info, data): matrix = col_info["matrix"] else: matrix = data.matrix - if not matrix in ebas_var_info["matrix"]: + if matrix not in ebas_var_info["matrix"]: ok = False if ok and "statistics" in col_info: # ALWAYS ignore columns containing statistics flagged in @@ -788,7 +788,7 @@ def _get_var_cols(self, ebas_var_info, data): if col_info["statistics"] in opts.ignore_statistics: ok = False elif check_stats: - if not col_info["statistics"] in ebas_var_info["statistics"]: + if col_info["statistics"] not in ebas_var_info["statistics"]: ok = False for key in self.IGNORE_COLS_CONTAIN: if key in col_info: @@ -797,13 +797,13 @@ def _get_var_cols(self, ebas_var_info, data): break if ok: col_matches.append(colnum) - if not col_info.name in comps: + if col_info.name not in comps: comps.append(col_info.name) if len(col_matches) == 0: raise NotInFileError(f"Variable {ebas_var_info.var_name} could not be found in file") elif len(comps) > 1 and len(ebas_var_info.component) > 1: for prefcomp in ebas_var_info.component: - if not prefcomp in comps: + if prefcomp not in comps: continue col_matches = [ colnum for colnum in col_matches if prefcomp == data.var_defs[colnum].name @@ -834,7 +834,7 @@ def _resolve_units_cols(self, var_name, result_col, file): """ to_unit = str(self.var_info(var_name).units) - if not to_unit in ("", "1"): + if to_unit not in ("", "1"): _cols = [] for colnum in result_col: try: @@ -1017,7 +1017,7 @@ def _add_meta(self, data_out, file): except KeyError: ival = re.findall(r"\d+", tres_code)[0] code = tres_code.split(ival)[-1] - if not code in self.TS_TYPE_CODES: + if code not in self.TS_TYPE_CODES: raise NotImplementedError(f"Cannot handle EBAS resolution code {tres_code}") ts_type = ival + self.TS_TYPE_CODES[code] self.TS_TYPE_CODES[tres_code] = ts_type @@ -1089,7 +1089,7 @@ def _find_wavelength_matches(self, col_matches, file, var_info): for colnum in col_matches: colinfo = file.var_defs[colnum] - if not "wavelength" in colinfo: + if "wavelength" not in colinfo: logger.warning( f"Ignoring column {colnum}\n{colinfo}\nVar {var_info.var_name}: " f"column misses wavelength specification!" @@ -1123,7 +1123,7 @@ def _find_closest_wavelength_cols(self, col_matches, file, var_info): for colnum in col_matches: colinfo = file.var_defs[colnum] - if not "wavelength" in colinfo: + if "wavelength" not in colinfo: logger.warning( f"Ignoring column {colnum} ({colinfo}) in EBAS file for reading var {var_info}: " f"column misses wavelength specification" @@ -1182,7 +1182,7 @@ def _check_shift_wavelength(self, var, col_info, meta, data): # make sure this variable has wavelength set # vi.ensure_wavelength_avail() if vi.is_wavelength_dependent: - if not "wavelength" in _col: + if "wavelength" not in _col: raise EbasFileError( f"Cannot access column wavelength information for variable {var}" ) @@ -1390,7 +1390,7 @@ def _try_convert_vmr_conc(self, data_out, var, var_info, meta): def get_ebas_var(self, var_name): """Get instance of :class:`EbasVarInfo` for input AeroCom variable""" - if not var_name in self._loaded_ebas_vars: + if var_name not in self._loaded_ebas_vars: self._loaded_ebas_vars[var_name] = EbasVarInfo(var_name) return self._loaded_ebas_vars[var_name] @@ -1458,15 +1458,15 @@ def read_file( meta = file.meta - if not "unit" in _col: # make sure a unit is assigned to data column + if "unit" not in _col: # make sure a unit is assigned to data column _col["unit"] = file.unit data = self._check_shift_wavelength(var, _col, meta, data) # TODO: double-check with NILU if this can be assumed - if not "matrix" in _col: + if "matrix" not in _col: _col["matrix"] = meta["matrix"] - if not "statistics" in _col: + if "statistics" not in _col: stats = None if "statistics" in meta: stats = meta["statistics"] @@ -1558,7 +1558,7 @@ def _flag_incorrect_frequencies(self, filedata): f"{tst} resolution. Minimum requirement for {var} is " f"{opts.freq_min_cov * 100:.2f}%" ) - if not var in filedata.data_flagged: + if var not in filedata.data_flagged: filedata.data_flagged[var] = np.zeros(num).astype(bool) filedata.data_flagged[var][invalid] = True return filedata @@ -1596,7 +1596,7 @@ def compute_additional_vars(self, data, vars_to_compute): """ data = super().compute_additional_vars(data, vars_to_compute) for var in vars_to_compute: - if not var in data: # variable could not be computed -> ignore + if var not in data: # variable could not be computed -> ignore continue data.var_info[var].update(self.get_ebas_var(var)) # self._loaded_ebas_vars[var] @@ -1607,20 +1607,20 @@ def compute_additional_vars(self, data, vars_to_compute): if from_var in data: from_dict = data["var_info"][from_var] for k, v in from_dict.items(): - if not k in to_dict or to_dict[k] is None: + if k not in to_dict or to_dict[k] is None: to_dict[k] = v if from_var in data.data_flagged: data.data_flagged[var] = data.data_flagged[from_var] if from_var in data.data_err: data.data_err[var] = data.data_err[from_var] - if not "units" in data["var_info"][var]: + if "units" not in data["var_info"][var]: data["var_info"][var]["units"] = self.var_info(var)["units"] return data def var_info(self, var_name): """Aerocom variable info for input var_name""" - if not var_name in self._loaded_aerocom_vars: + if var_name not in self._loaded_aerocom_vars: self._loaded_aerocom_vars[var_name] = const.VARS[var_name] return self._loaded_aerocom_vars[var_name] @@ -1648,9 +1648,9 @@ def get_read_opts(self, var_name): options """ - if not var_name in self.VAR_READ_OPTS: + if var_name not in self.VAR_READ_OPTS: return self._opts["default"] - if not var_name in self._opts: + if var_name not in self._opts: vo = ReadEbasOptions(**self.VAR_READ_OPTS[var_name]) self._opts[var_name] = vo return self._opts[var_name] @@ -1896,7 +1896,7 @@ def _read_files(self, files, vars_to_retrieve, files_contain, constraints): start = idx + var_count * num_times stop = start + num_times - if not var in data_obj.var_idx: + if var not in data_obj.var_idx: var_count_glob += 1 var_idx = var_count_glob data_obj.var_idx[var] = var_idx diff --git a/pyaerocom/io/read_eea_aqerep_base.py b/pyaerocom/io/read_eea_aqerep_base.py index 9b3767d29..576eed027 100644 --- a/pyaerocom/io/read_eea_aqerep_base.py +++ b/pyaerocom/io/read_eea_aqerep_base.py @@ -2,6 +2,7 @@ Interface for reading EEA AqERep files (formerly known as Airbase data). """ +import glob import gzip import logging import os @@ -245,7 +246,7 @@ def read_file(self, filename, var_name, vars_as_series=False): Dict-like object containing the results. """ - if not var_name in self.PROVIDES_VARIABLES: + if var_name not in self.PROVIDES_VARIABLES: raise ValueError(f"Invalid input variable {var_name}") # there's only one variable in the file @@ -292,7 +293,7 @@ def read_file(self, filename, var_name, vars_as_series=False): except UnicodeDecodeError: with open(read_filename, encoding="UTF-16") as f: lines = f.readlines() - except: + except Exception: if suffix == ".gz": os.remove(f_out.name) raise EEAv2FileError(f"Found corrupt file {filename}. consider deleteing it") @@ -550,26 +551,14 @@ def get_file_list(self, pattern=None): IOError if no files can be found """ - import glob - import os - - from pyaerocom._lowlevel_helpers import list_to_shortstr if pattern is None: logger.warning("using default pattern *.* for file search") pattern = "*.*" + self.logger.info("Fetching data files. This might take a while...") fp = os.path.join(self.data_dir, pattern) - files = sorted(glob.glob(fp, recursive=True)) - if not len(files) > 0: - all_str = list_to_shortstr(os.listdir(self.data_dir)) - # raise DataSourceError('No files could be detected matching file ' - # 'mask {} in dataset {}, files in folder {}:\n' - # 'Files in folder:{}'.format(pattern, - # self.data_id, - # self.data_dir, - # all_str)) - return files + return sorted(glob.glob(fp, recursive=True)) def get_station_coords(self, meta_key): """ @@ -730,7 +719,7 @@ def read( data_obj._data[start:stop, data_obj._VARINDEX] = var_idx meta_idx[meta_key][var] = np.arange(start, stop) - if not var in data_obj.var_idx: + if var not in data_obj.var_idx: data_obj.var_idx[var] = var_idx idx += num_times diff --git a/pyaerocom/io/readaeronetbase.py b/pyaerocom/io/readaeronetbase.py index a26079151..42a3a2f64 100644 --- a/pyaerocom/io/readaeronetbase.py +++ b/pyaerocom/io/readaeronetbase.py @@ -217,7 +217,7 @@ def _find_vars_name_based(self, mapping, cols): col_index = {} # find meta indices for key, val in self.META_NAMES_FILE.items(): - if not val in mapping: + if val not in mapping: val = self._check_alternative_colnames(val, mapping) col_index[key] = mapping[val] for var, colname in self.VAR_NAMES_FILE.items(): @@ -281,9 +281,9 @@ def _search_var_wavelength_tol(self, var, cols): diff = abs(wvl_col - wvl) if diff < wvl_diff_min: wvl_diff_min = diff - if not var in self._alt_var_cols: + if var not in self._alt_var_cols: self._alt_var_cols[var] = [] - if not col in self._alt_var_cols[var]: + if col not in self._alt_var_cols[var]: self._alt_var_cols[var].append(col) return i raise VariableNotFoundError( @@ -381,7 +381,7 @@ def read( statmeta = station_data.get_meta() except StationCoordinateError as e: stat = station_data.station_name - if isinstance(stat, (list, np.ndarray)): + if isinstance(stat, list | np.ndarray): stat = stat[0] logger.warning(f"\nSkipping station {stat}. Reason: {repr(e)}.\n") skipped += 1 @@ -461,7 +461,7 @@ def read( else: u = self.DEFAULT_UNIT meta["var_info"][var] = dict(units=u) - if not var in data_obj.var_idx: + if var not in data_obj.var_idx: data_obj.var_idx[var] = var_idx idx += totnum diff --git a/pyaerocom/io/readgridded.py b/pyaerocom/io/readgridded.py index 84773f9de..90b62c70c 100755 --- a/pyaerocom/io/readgridded.py +++ b/pyaerocom/io/readgridded.py @@ -404,18 +404,18 @@ def _get_vars_provided(self): for aux_var in self.AUX_REQUIRES: if "*" in aux_var: continue - if not aux_var in _vars and self.check_compute_var(aux_var): + if aux_var not in _vars and self.check_compute_var(aux_var): _vars.append(aux_var) for aux_var in self._aux_requires: if "*" in aux_var: continue - if not aux_var in _vars and self.check_compute_var(aux_var): + if aux_var not in _vars and self.check_compute_var(aux_var): _vars.append(aux_var) # also add standard names of 3D variables if not already in list for var in self._vars_3d: var = var.lower().replace("3d", "") - if not var in _vars: + if var not in _vars: _vars.append(var) return _vars @@ -478,7 +478,7 @@ def _check_var_match_pattern(self, var_name): if fnmatch.fnmatch(var_name, pattern): vars_required = self.AUX_REQUIRES[pattern] for addvar in vars_required: - if not "*" in addvar: + if "*" not in addvar: vars_found.append(addvar) else: _addvar = var_name @@ -537,7 +537,7 @@ def _get_aux_vars_and_fun(self, var_to_compute): callable function that is used to compute input variable """ - if not var_to_compute in self._aux_avail: + if var_to_compute not in self._aux_avail: if not self.check_compute_var(var_to_compute): raise VarNotAvailableError(f"Variable {var_to_compute} cannot be computed") return self._aux_avail[var_to_compute] @@ -1022,7 +1022,7 @@ def filter_query( DataFrame dataframe containing filtered dataset """ - if not var_name in self.file_info.var_name.values: + if var_name not in self.file_info.var_name.values: raise DataCoverageError( f"Variable {var_name} is not available in dataset {self.data_id}" ) @@ -1136,7 +1136,7 @@ def get_var_info_from_files(self) -> dict: for file in self.files: finfo = self.file_convention.get_info_from_file(file) var_name = finfo["var_name"] - if not var_name in result: + if var_name not in result: result[var_name] = var_info = {} for key in finfo: if key != "var_name": @@ -1146,7 +1146,7 @@ def get_var_info_from_files(self) -> dict: for key, val in finfo.items(): if key == "var_name": continue - if val is not None and not val in var_info[key]: + if val is not None and val not in var_info[key]: var_info[key].append(val) # now check auxiliary variables for var_to_compute in self.AUX_REQUIRES: @@ -1186,12 +1186,11 @@ def update(self, **kwargs): for k, v in kwargs.items(): if k in self.__dict__: self.logger.info( - "Updating %s in ModelImportResult for model %s" - "New value: %s" % (k, self.data_id, v) + f"Updating {k} in ModelImportResult for model {self.data_id}. New value: {v}" ) self.__dict__[k] = v else: - self.logger.info("Ignoring key %s in ModelImportResult.update()" % k) + self.logger.info(f"Ignoring key {k} in ModelImportResult.update()") def concatenate_cubes(self, cubes): """Concatenate list of cubes into one cube @@ -1298,7 +1297,7 @@ def compute_var( flex_ts_type=flex_ts_type, ) except DataCoverageError: - if not vert_which in self.VERT_ALT: + if vert_which not in self.VERT_ALT: raise vert_which = self.VERT_ALT[vert_which] @@ -1329,7 +1328,7 @@ def compute_var( if var_name in self.AUX_ADD_ARGS: for key, val in self.AUX_ADD_ARGS[var_name].items(): - if not key in aux_add_args: + if key not in aux_add_args: aux_add_args[key] = val if len(aux_add_args) > 0: @@ -1737,16 +1736,16 @@ def check_constraint_valid(self, constraint): """ if not isinstance(constraint, dict): raise ValueError("Read constraint needs to be dict") - elif not "operator" in constraint: + elif "operator" not in constraint: raise ValueError( f"Constraint requires specification of operator. " f"Valid operators: {self.CONSTRAINT_OPERATORS}" ) - elif not constraint["operator"] in self.CONSTRAINT_OPERATORS: + elif constraint["operator"] not in self.CONSTRAINT_OPERATORS: raise ValueError( f"Invalid constraint operator. Choose from: {self.CONSTRAINT_OPERATORS}" ) - elif not "filter_val" in constraint: + elif "filter_val" not in constraint: raise ValueError("constraint needs specification of filter_val") elif not isnumeric(constraint["filter_val"]): raise ValueError("Need numerical filter value") @@ -2230,7 +2229,7 @@ def __getitem__(self, var_name): ValueError if results for ``var_name`` are not available """ - if not var_name in self.data: + if var_name not in self.data: return self.read_var(var_name) # return self.data[var_name] @@ -2258,4 +2257,5 @@ def name(self): return self.data_id -is_3d = lambda var_name: True if "3d" in var_name.lower() else False +def is_3d(var_name: str) -> bool: + return "3d" in var_name.lower() diff --git a/pyaerocom/io/readungridded.py b/pyaerocom/io/readungridded.py index 8ed6d80ca..97711401e 100755 --- a/pyaerocom/io/readungridded.py +++ b/pyaerocom/io/readungridded.py @@ -3,7 +3,6 @@ import warnings from copy import deepcopy from pathlib import Path -from typing import Optional, Union from pyaerocom import const from pyaerocom.combine_vardata_ungridded import combine_vardata_ungridded @@ -80,7 +79,7 @@ def __init__( data_ids=None, ignore_cache=False, data_dirs=None, - configs: Optional[Union[PyaroConfig, list[PyaroConfig]]] = None, + configs: PyaroConfig | list[PyaroConfig] | None = None, ): # will be assigned in setter method of data_ids self._data_ids = [] @@ -176,7 +175,7 @@ def _check_donotcachefile(self): try: if os.path.exists(os.path.join(const.cache_basedir, self.DONOTCACHE_NAME)): return True - except: + except Exception: pass return False @@ -196,7 +195,7 @@ def data_ids(self): def data_ids(self, val): if isinstance(val, str): val = [val] - elif not isinstance(val, (tuple, list)): + elif not isinstance(val, tuple | list): raise OSError("Invalid input for parameter data_ids") self._data_ids = val @@ -206,13 +205,13 @@ def configs(self): return self._configs @configs.setter - def configs(self, val: Union[PyaroConfig, list[PyaroConfig]]): + def configs(self, val: PyaroConfig | list[PyaroConfig]): if isinstance(val, PyaroConfig): val = [val] - elif not isinstance(val, (tuple, list)): + elif not isinstance(val, tuple | list): raise OSError("Invalid input for parameter data_ids") logger.warning( - f"You are now overwriting the list of configs. This will delete the previous configs, but will leave readeres associated with those configs intact. Use 'add_config' for safer usage!" + "You are now overwriting the list of configs. This will delete the previous configs, but will leave readeres associated with those configs intact. Use 'add_config' for safer usage!" ) for config in val: self._init_pyaro_reader(config=config) @@ -250,7 +249,7 @@ def dataset_provides_variables(self, data_id=None): """List of variables provided by a certain dataset""" if data_id is None: data_id = self.data_id - if not data_id in self._readers: + if data_id not in self._readers: reader = self.get_lowlevel_reader(data_id) else: reader = self._readers[data_id] @@ -354,7 +353,7 @@ def add_config(self, config: PyaroConfig) -> None: """ if not isinstance(config, PyaroConfig): - raise ValueError(f"Given config is not a PyaroConfig") + raise ValueError("Given config is not a PyaroConfig") self._init_pyaro_reader(config=config) self._configs.append(config) @@ -418,7 +417,7 @@ def _init_lowlevel_reader(self, reader, data_id): # data_id = config.name if data_id is None: - raise ValueError(f"Data_id can not be none") + raise ValueError("Data_id can not be none") if data_id in self.config_map: return reader(config=self.config_map[data_id]) @@ -521,7 +520,7 @@ def read_dataset( ) if not only_cached: - vars_to_read = [v for v in vars_available if not v in cache.loaded_data] + vars_to_read = [v for v in vars_available if v not in cache.loaded_data] else: vars_to_read = [] @@ -586,9 +585,9 @@ def _eval_filter_post(self, filter_post, data_id, vars_available): for key, val in filter_post.items(): if key == "ignore_station_names": # for backwards compatibility - if isinstance(val, (str, list)): + if isinstance(val, str | list): filters["station_name"] = val - if not "negate" in filters: + if "negate" not in filters: filters["negate"] = [] filters["negate"].append("station_name") @@ -606,7 +605,7 @@ def _eval_filter_post(self, filter_post, data_id, vars_available): var = vars_available[0] try: filters["station_name"] = val[var] - if not "negate" in filters: + if "negate" not in filters: filters["negate"] = [] filters["negate"].append("station_name") except KeyError: @@ -730,7 +729,7 @@ def read( vars_to_retrieve=None, only_cached=False, filter_post=None, - configs: Optional[Union[PyaroConfig, list[PyaroConfig]]] = None, + configs: PyaroConfig | list[PyaroConfig] | None = None, **kwargs, ): """Read observations @@ -858,7 +857,7 @@ def get_vars_supported(self, obs_id, vars_desired): # , config: Optional[PyaroC postinfo = self.post_compute[obs_id] supported = postinfo["vars_supported"] for var in varlist_aerocom(vars_desired): - if not var in supported: + if var not in supported: try: var = self._check_var_alias(var, supported) except ValueError: diff --git a/pyaerocom/io/readungriddedbase.py b/pyaerocom/io/readungriddedbase.py index 9ccaf7c8d..133a1d12e 100644 --- a/pyaerocom/io/readungriddedbase.py +++ b/pyaerocom/io/readungriddedbase.py @@ -227,7 +227,7 @@ def __init__(self, data_id: str | None = None, data_dir: str | None = None): self._add_aux_variables() if data_id is not None: - if not data_id in self.SUPPORTED_DATASETS: + if data_id not in self.SUPPORTED_DATASETS: raise AttributeError(f"Dataset {data_id} not supported by this interface") self._data_id = data_id @@ -312,12 +312,12 @@ def verbosity_level(self, val): def _add_aux_variables(self): """Helper that makes sure all auxiliary variables can be computed""" for var in self.AUX_REQUIRES: - if not var in self.AUX_FUNS: + if var not in self.AUX_FUNS: raise AttributeError( f"Fatal: no computation method defined for auxiliary variable {var}. " f"Please specify method in class header dictionary AUX_FUNS" ) - if not var in self.PROVIDES_VARIABLES: + if var not in self.PROVIDES_VARIABLES: self.PROVIDES_VARIABLES.append(var) def _add_additional_vars(self, vars_to_retrieve): @@ -455,7 +455,7 @@ def check_vars_to_retrieve(self, vars_to_retrieve): vars_to_compute = [] for var in vars_to_retrieve: - if not var in self.PROVIDES_VARIABLES: + if var not in self.PROVIDES_VARIABLES: raise ValueError(f"Invalid variable {var}") elif var in self.AUX_REQUIRES: vars_to_compute.append(var) @@ -486,13 +486,13 @@ def compute_additional_vars(self, data, vars_to_compute): dict updated data object now containing also computed variables """ - if not "var_info" in data: + if "var_info" not in data: data["var_info"] = {} for var in vars_to_compute: required = self.AUX_REQUIRES[var] missing = [] for req in required: - if not req in data: + if req not in data: missing.append(req) if len(missing) == 0: diff --git a/pyaerocom/mathutils.py b/pyaerocom/mathutils.py index b9079f21b..818f553a2 100644 --- a/pyaerocom/mathutils.py +++ b/pyaerocom/mathutils.py @@ -3,19 +3,18 @@ """ import numpy as np -from scipy.stats import kendalltau, pearsonr, spearmanr +from scipy.stats import pearsonr from pyaerocom._warnings import ignore_warnings -### LAMBDA FUNCTIONS -in_range = lambda x, low, high: low <= x <= high -### OTHER FUNCTIONS +def in_range(x, low, high) -> bool: + return low <= x <= high def is_strictly_monotonic(iter1d) -> bool: """ - Check if 1D iterble is strictly monotonic + Check if 1D iterable is strictly monotonic Parameters ---------- @@ -30,11 +29,8 @@ def is_strictly_monotonic(iter1d) -> bool: return True if np.all(np.diff(iter1d) > 0) else False -def make_binlist(vmin: float, vmax: float, num: int = None) -> list: - """""" - if num is None: - num = 8 - return list(np.linspace(vmin, vmax, num + 1)) +def make_binlist(vmin: float, vmax: float, num: int = 8) -> list[float]: + return np.linspace(vmin, vmax, num + 1).tolist() def weighted_sum(data, weights): diff --git a/pyaerocom/metastandards.py b/pyaerocom/metastandards.py index 574655cad..3e9ca47dc 100644 --- a/pyaerocom/metastandards.py +++ b/pyaerocom/metastandards.py @@ -263,7 +263,7 @@ def to_dict(self): def _values_from_dict(self, meta): vals = [] for key in self.KEYS: - if not key in meta: + if key not in meta: raise KeyError(f"Missing specification of {key} in input meta dict") vals.append(meta[key]) self._data_id = self.from_values(vals) diff --git a/pyaerocom/obs_io.py b/pyaerocom/obs_io.py index 212047461..c7857128e 100644 --- a/pyaerocom/obs_io.py +++ b/pyaerocom/obs_io.py @@ -81,16 +81,16 @@ def check_status(self): self.aux_units = {} for var in self.vars_supported: - if not var in self.aux_requires: + if var not in self.aux_requires: raise ValueError(f"Variable {var} is not defined in attr aux_requires...") - elif not var in self.aux_merge_how: + elif var not in self.aux_merge_how: raise ValueError( f"Missing information about how {var} should be merged (aux_merge_how)" ) merge_how = self.aux_merge_how[var] if merge_how == "eval": - if not var in self.aux_funs: + if var not in self.aux_funs: raise ValueError( f"Specification of computation function is missing for var {var}" ) @@ -110,7 +110,7 @@ def check_status(self): for _var in var_info: obsvar = ObsVarCombi(aux_id, _var) obsvarstr = str(obsvar) - if merge_how == "eval" and not obsvarstr in fun: + if merge_how == "eval" and obsvarstr not in fun: raise ValueError( f"Mismatch between aux_requires and aux_funs for variable {var}. " f"No such obs;var string {obsvarstr} in computation method {fun}" diff --git a/pyaerocom/plot/config.py b/pyaerocom/plot/config.py index e86c366cd..f72fa69f1 100644 --- a/pyaerocom/plot/config.py +++ b/pyaerocom/plot/config.py @@ -63,7 +63,7 @@ def __init__( cmap_map_div=None, cmap_map_div_shifted=True, ): - if not name in _COLOR_THEMES: + if name not in _COLOR_THEMES: warn("Invalid name for color theme, using default theme") name = DEFAULT_THEME @@ -90,9 +90,11 @@ def load_default(self, theme_name="dark"): if ``theme_name`` is not a valid default theme """ - if not theme_name in _COLOR_THEMES: + if theme_name not in _COLOR_THEMES: raise ValueError( - "Default theme with name %s is not available. Choose from %s" % _COLOR_THEMES + "Default theme with name {} is not available. Choose from {}".format( + *_COLOR_THEMES + ) ) self.from_dict(_COLOR_THEMES[theme_name]) @@ -139,6 +141,6 @@ def __str__(self): def get_color_theme(theme_name="dark"): # Settings for colormap (use perceptually uniform colormaps) - if not theme_name in _COLOR_THEMES: - raise ValueError("Invalid input for theme_name, choose from %s" % _COLOR_THEMES) + if theme_name not in _COLOR_THEMES: + raise ValueError(f"Invalid input for theme_name, choose from {_COLOR_THEMES}") return ColorTheme(theme_name) diff --git a/pyaerocom/plot/heatmaps.py b/pyaerocom/plot/heatmaps.py index 951a7856c..f7082e74e 100644 --- a/pyaerocom/plot/heatmaps.py +++ b/pyaerocom/plot/heatmaps.py @@ -246,7 +246,7 @@ def df_to_heatmap( cbar_kws = {} if annot_fontsize is None: annot_fontsize = labelsize - 4 - if not "annot_kws" in kwargs: + if "annot_kws" not in kwargs: kwargs["annot_kws"] = {} kwargs["annot_kws"]["size"] = annot_fontsize df_hm = df diff --git a/pyaerocom/plot/mapping.py b/pyaerocom/plot/mapping.py index 16a15c61d..5fc77d5d5 100644 --- a/pyaerocom/plot/mapping.py +++ b/pyaerocom/plot/mapping.py @@ -8,7 +8,6 @@ from numpy import ceil, linspace, meshgrid from pandas import to_datetime -from pyaerocom import const from pyaerocom._warnings import ignore_warnings from pyaerocom.exceptions import DataDimensionError from pyaerocom.mathutils import exponent @@ -295,7 +294,7 @@ def plot_griddeddata_on_map( if not data.has_latlon_dims: raise DataDimensionError("Input data needs to have latitude and longitude dimension") if not data.ndim == 2: - if not data.ndim == 3 or not "time" in data.dimcoord_names: + if not data.ndim == 3 or "time" not in data.dimcoord_names: raise DataDimensionError( "Input data needs to be 2 dimensional " "or 3D with time being the 3rd " @@ -402,7 +401,7 @@ def plot_griddeddata_on_map( if var_name is not None: var_str = var_name # + VARS.unit_str if unit is not None: - if not str(unit) in ["1", "no_unit"]: + if str(unit) not in ["1", "no_unit"]: var_str += f" [{unit}]" cbar.set_label(var_str) @@ -452,7 +451,7 @@ def plot_map_aerocom(data, region, **kwargs): if not isinstance(data, GriddedData): raise ValueError( "This plotting method needs an instance of pyaerocom " - "GriddedData on input, got: %s" % type(data) + f"GriddedData on input, got: {type(data)}" ) if isinstance(region, str): @@ -589,7 +588,7 @@ def plot_nmb_map_colocateddata( mew = kwargs.pop("mew") except KeyError: mew = 1 - if not coldata.ndim in (3, 4): + if coldata.ndim not in (3, 4): raise DataDimensionError("only 3D or 4D colocated data objects are supported") assert "time" in coldata.dims diff --git a/pyaerocom/plot/plotscatter.py b/pyaerocom/plot/plotscatter.py index bb3e9b3ed..473ab00ab 100644 --- a/pyaerocom/plot/plotscatter.py +++ b/pyaerocom/plot/plotscatter.py @@ -207,7 +207,7 @@ def plot_scatter_aerocom( _ndig = abs(exponent(statistics["refdata_mean"]) - 2) if unit is None: unit = "N/D" - if not str(unit) in ["1", "no_unit"]: + if str(unit) not in ["1", "no_unit"]: var_str += f" [{unit}]" if fontsize_annot is None: diff --git a/pyaerocom/projection_information.py b/pyaerocom/projection_information.py index 612c765a0..48c98ab5f 100644 --- a/pyaerocom/projection_information.py +++ b/pyaerocom/projection_information.py @@ -48,7 +48,7 @@ def from_xarray(ds: xarray.Dataset, var: str): returns None if no projection exists for the variable or ProjectionInformation """ da = ds[var] - if not "grid_mapping" in da.attrs: + if "grid_mapping" not in da.attrs: return None pi = ProjectionInformation() pi._crs = CRS.from_cf(ds[da.grid_mapping].attrs) diff --git a/pyaerocom/region.py b/pyaerocom/region.py index 9ca8074f8..55509635b 100644 --- a/pyaerocom/region.py +++ b/pyaerocom/region.py @@ -241,12 +241,11 @@ def __repr__(self): def __str__(self): s = ( - "pyaeorocom Region\nName: %s\n" - "Longitude range: %s\n" - "Latitude range: %s\n" - "Longitude range (plots): %s\n" - "Latitude range (plots): %s" - % (self.name, self.lon_range, self.lat_range, self.lon_range_plot, self.lat_range_plot) + f"pyaeorocom Region\nName: {self.name}\n" + f"Longitude range: {self.lon_range}\n" + f"Latitude range: {self.lat_range}\n" + f"Longitude range (plots): {self.lon_range_plot}\n" + f"Latitude range (plots): {self.lat_range_plot}" ) return s diff --git a/pyaerocom/scripts/cams2_83/cli.py b/pyaerocom/scripts/cams2_83/cli.py index 42d800317..aa88c9a15 100644 --- a/pyaerocom/scripts/cams2_83/cli.py +++ b/pyaerocom/scripts/cams2_83/cli.py @@ -5,7 +5,7 @@ from copy import deepcopy from datetime import date, datetime, timedelta from pathlib import Path -from typing import List, Optional +from typing import Optional import typer @@ -138,7 +138,7 @@ def main( writable=True, help="where collocated data are stored", ), - model: List[ModelName] = typer.Option( + model: list[ModelName] = typer.Option( [], "--model", "-m", diff --git a/pyaerocom/scripts/cams2_83/engine.py b/pyaerocom/scripts/cams2_83/engine.py index 2b2d538a6..e396a6622 100644 --- a/pyaerocom/scripts/cams2_83/engine.py +++ b/pyaerocom/scripts/cams2_83/engine.py @@ -6,7 +6,6 @@ import warnings from pathlib import Path from reprlib import repr -from typing import Tuple import numpy as np import xarray as xr @@ -33,7 +32,7 @@ def run(self, files: list[list[str | Path]], var_list: list) -> None: # type:ig logger.info(f"Processing Component: {var}") self.process_coldata(coldata[var]) - logger.info(f"Time for wierd plot: {time.time() - start} sec") + logger.info(f"Time for weird plot: {time.time() - start} sec") def process_coldata(self, coldata: list[ColocatedData]) -> None: use_weights = self.cfg.statistics_opts.weighted_stats @@ -116,7 +115,7 @@ def process_coldata(self, coldata: list[ColocatedData]) -> None: for key in stats_list: stats_list[key].append(stats[key]) - out_dirs = self.cfg.path_manager.get_json_output_dirs(True) + out_dirs = self.cfg.path_manager.get_json_output_dirs(True) # noqa: F841 results[f"{regname}"][f"{perstr}"] = stats_list @@ -197,7 +196,7 @@ def _pearson_R_vec(self, x: np.ndarray, y: np.ndarray) -> np.ndarray: def _sort_coldata( self, coldata: list[ColocatedData] - ) -> Tuple[dict[str, list[ColocatedData]], set[str]]: + ) -> tuple[dict[str, list[ColocatedData]], set[str]]: col_dict = dict() var_list = [] @@ -211,8 +210,7 @@ def _sort_coldata( var_list.append(obs_var) for var, cols in col_dict.items(): - l = sorted(cols, key=lambda x: self._get_day(x.model_name)) - col_dict[var] = l + col_dict[var] = sorted(cols, key=lambda x: self._get_day(x.model_name)) return col_dict, set(var_list) diff --git a/pyaerocom/scripts/cams2_83/evaluation.py b/pyaerocom/scripts/cams2_83/evaluation.py index 73e54dcdb..ecf1df814 100644 --- a/pyaerocom/scripts/cams2_83/evaluation.py +++ b/pyaerocom/scripts/cams2_83/evaluation.py @@ -6,7 +6,6 @@ from enum import Enum from pathlib import Path from pprint import pformat -from typing import List from pyaerocom import const from pyaerocom.aeroval import EvalSetup, ExperimentProcessor @@ -102,7 +101,7 @@ def clear_cache(): path.unlink() -def read_observations(specie: str, *, files: List, cache: str | Path | None) -> None: +def read_observations(specie: str, *, files: list, cache: str | Path | None) -> None: logger.info(f"Running {specie}") if cache is not None: diff --git a/pyaerocom/scripts/cli.py b/pyaerocom/scripts/cli.py index a647dba72..647a61aa8 100644 --- a/pyaerocom/scripts/cli.py +++ b/pyaerocom/scripts/cli.py @@ -23,7 +23,7 @@ def version_callback(value: bool): @main.callback() def callback( - version: Optional[bool] = typer.Option(None, "--version", "-V", callback=version_callback) + version: Optional[bool] = typer.Option(None, "--version", "-V", callback=version_callback), ): """🦄 Pyaerocom Command Line Interface""" diff --git a/pyaerocom/scripts/testdata-minimal/create_subset_ebas.py b/pyaerocom/scripts/testdata-minimal/create_subset_ebas.py index a7deed21b..58404e775 100755 --- a/pyaerocom/scripts/testdata-minimal/create_subset_ebas.py +++ b/pyaerocom/scripts/testdata-minimal/create_subset_ebas.py @@ -78,7 +78,7 @@ def check_outdated(filedir): for var, stats in data.items(): for stat, files in stats.items(): for file in files: - if not file in all_files: + if file not in all_files: print("OUTDATED", var, stat) print(file) files_invalid.append(file) @@ -87,7 +87,7 @@ def check_outdated(filedir): files_valid.append(file) for key, file in add_files.items(): - if not file in all_files: + if file not in all_files: print("OUTDATED (add_files)", key) print(file) files_invalid.append(file) @@ -164,9 +164,9 @@ def main(): # directory containing NASA Ames files FILEDIR_SRC = r_lowlev.file_dir - print(f"checking for invalid files in static file list...") + print("checking for invalid files in static file list...") CURRENT_OK, files_valid, files_invalid = check_outdated(FILEDIR_SRC) - print(f"done...") + print("done...") if not CURRENT_OK: print(f"outdated files: {files_invalid}") diff --git a/pyaerocom/scripts/testdata-minimal/create_subsets_aeronet.py b/pyaerocom/scripts/testdata-minimal/create_subsets_aeronet.py index b8215a8d1..0c18d0143 100755 --- a/pyaerocom/scripts/testdata-minimal/create_subsets_aeronet.py +++ b/pyaerocom/scripts/testdata-minimal/create_subsets_aeronet.py @@ -117,8 +117,4 @@ shutil.copy(revision_files[name], outdir.joinpath("Revision.txt")) for name, filelist in files.items(): - print( - "Copied {} files for {} into {}".format( - len(filelist), name, os.path.dirname(filelist[0]) - ) - ) + print(f"Copied {len(filelist)} files for {name} into {os.path.dirname(filelist[0])}") diff --git a/pyaerocom/scripts/testdata-minimal/create_subsets_ghost.py b/pyaerocom/scripts/testdata-minimal/create_subsets_ghost.py index a5a5e8864..e89bb451b 100644 --- a/pyaerocom/scripts/testdata-minimal/create_subsets_ghost.py +++ b/pyaerocom/scripts/testdata-minimal/create_subsets_ghost.py @@ -6,6 +6,7 @@ @author: jonasg """ + import os import matplotlib.pyplot as plt @@ -29,7 +30,10 @@ varis = ["pm10", "sconco3"] datesfiles = ["201810", "201911", "201912"] -filename = lambda var, date: f"{var}_{date}.nc" + +def filename(var: str, date: str) -> str: + return f"{var}_{date}.nc" + files_out = [] for dsname in datasets: diff --git a/pyaerocom/stationdata.py b/pyaerocom/stationdata.py index 1aaea630e..5dde3a083 100644 --- a/pyaerocom/stationdata.py +++ b/pyaerocom/stationdata.py @@ -17,7 +17,6 @@ MetaDataError, StationCoordinateError, TemporalResolutionError, - UnitConversionError, VarNotAvailableError, ) from pyaerocom.helpers import calc_climatology, isnumeric, isrange, to_datetime64 @@ -126,9 +125,9 @@ def has_var(self, var_name): bool True, if variable data is available, else False """ - if not var_name in self: + if var_name not in self: return False - if not var_name in self.var_info: + if var_name not in self.var_info: logger.warning( f"Variable {var_name} exists in data but has no " f"metadata assigned in :attr:`var_info`" @@ -153,7 +152,7 @@ def get_unit(self, var_name): MetaDataError if unit cannot be accessed for variable """ - if not var_name in self.var_info: + if var_name not in self.var_info: raise MetaDataError(f"Could not access variable metadata dict for {var_name}.") try: return str(self.var_info[var_name]["units"]) @@ -352,10 +351,10 @@ def get_station_coords(self, force_single_value=True): output[key] = val else: val = self[key] - if force_single_value and not isinstance(val, (float, np.floating)): - if isinstance(val, (int, np.integer)): + if force_single_value and not isinstance(val, float | np.floating): + if isinstance(val, int | np.integer): val = np.float64(val) - elif isinstance(val, (list, np.ndarray)): + elif isinstance(val, list | np.ndarray): # ToDo: consider tolerance to be specified in input # args. maxdiff = np.max(val) - np.min(val) @@ -423,7 +422,7 @@ def get_meta( keys = [k for k in self.STANDARD_META_KEYS] keys.extend(add_meta_keys) for key in keys: - if not key in self: + if key not in self: logger.warning(f"No such key in StationData: {key}") continue elif key in self.PROTECTED_KEYS: @@ -437,7 +436,7 @@ def get_meta( continue val = self[key] - if force_single_value and isinstance(val, (list, tuple, np.ndarray)): + if force_single_value and isinstance(val, list | tuple | np.ndarray): if quality_check and not all([x == val[0] for x in val]): raise MetaDataError(f"Inconsistencies in meta parameter {key}") val = val[0] @@ -461,7 +460,7 @@ def _check_meta_item(self, key): f"Only 1d numpy arrays are supported..." ) self[key] = list(val) - elif not isinstance(val, (dict, list, str)) and not isnumeric(val): + elif not isinstance(val, dict | list | str) and not isnumeric(val): try: self[key] = to_datetime64(val) except Exception: @@ -493,7 +492,7 @@ def _merge_meta_item(self, key, val): elif isinstance(current_val, str): if not same_type: if isinstance(val, list): - if not current_val in val: + if current_val not in val: newval = val.insert(0, current_val) self[key] = newval else: @@ -504,7 +503,7 @@ def _merge_meta_item(self, key, val): vals_in = [x.strip() for x in val.split(";")] for item in vals_in: - if not item in current_val: + if item not in current_val: current_val += f";{item}" self[key] = current_val @@ -512,7 +511,7 @@ def _merge_meta_item(self, key, val): if not same_type: val = [val] for item in val: - if not item in current_val: + if item not in current_val: current_val.append(item) self[key] = current_val @@ -523,7 +522,7 @@ def _merge_meta_item(self, key, val): self[key] = [current_val, val] elif isinstance(val, list): - if not current_val in val: + if current_val not in val: self[key] = val.insert(0, current_val) elif current_val != val: @@ -541,7 +540,7 @@ def _merge_meta_item(self, key, val): def _append_meta_item(self, key, val): """Add a metadata item""" - if not key in self or self[key] is None: + if key not in self or self[key] is None: self[key] = val else: self._merge_meta_item(key, val) @@ -646,13 +645,13 @@ def merge_varinfo(self, other, var_name): variable name for which info is to be merged (needs to be both available in this object and the provided other object) """ - if not var_name in self.var_info or not var_name in other.var_info: + if var_name not in self.var_info or var_name not in other.var_info: raise MetaDataError(f"No variable meta information available for {var_name}") info_this = self.var_info[var_name] info_other = other.var_info[var_name] for key, val in info_other.items(): - if not key in info_this or info_this[key] == None: + if key not in info_this or info_this[key] is None: info_this[key] = val else: if isinstance(info_this[key], str): @@ -662,23 +661,20 @@ def merge_varinfo(self, other, var_name): vals_in = [x.strip() for x in val.split(";")] for _val in vals_in: - if not _val in vals: + if _val not in vals: info_this[key] = info_this[key] + f";{_val}" else: - if isinstance(val, (list, np.ndarray)): - if len(val) == 0: + if isinstance(val, list | np.ndarray): + if len(val) == 0 or info_this[key] == val: continue - elif type(info_this[key]) == type(val): - if info_this[key] == val: - continue - info_this[key] = [info_this[key], val] + raise ValueError( "Cannot append metadata value that is " "already a list or numpy array due to " "potential ambiguities" ) if isinstance(info_this[key], list): - if not val in info_this[key]: + if val not in info_this[key]: info_this[key].append(val) else: if not info_this[key] == val: @@ -725,8 +721,8 @@ def _update_var_timeinfo(self): self[var] = pd.Series(data, self.dtime) except Exception as e: raise Exception(f"Unexpected error: {repr(e)}.\nPlease debug...") - if not "ts_type" in info or info["ts_type"] is None: - if not self.ts_type in const.GRID_IO.TS_TYPES: + if "ts_type" not in info or info["ts_type"] is None: + if self.ts_type not in const.GRID_IO.TS_TYPES: raise ValueError(f"Cannot identify ts_type for var {var} in {self}") info["ts_type"] = self.ts_type self.ts_type = None @@ -812,20 +808,20 @@ def merge_vardata(self, other, var_name, **kwargs): StationData this object merged with other object """ - if not var_name in self: + if var_name not in self: raise VarNotAvailableError( f"StationData object does not contain data for variable {var_name}" ) - elif not var_name in other: + elif var_name not in other: raise VarNotAvailableError( "Input StationData object does not contain data for variable {var_name}" ) - elif not var_name in self.var_info: + elif var_name not in self.var_info: raise MetaDataError( f"For merging of {var_name} data, variable specific meta " f"data needs to be available in var_info dict" ) - elif not var_name in other.var_info: + elif var_name not in other.var_info: raise MetaDataError( f"For merging of {var_name} data, variable specific meta " f"data needs to be available in var_info dict" @@ -908,7 +904,7 @@ def get_var_ts_type(self, var_name, try_infer=True): cannot be found in :attr:`var_info`) """ # make sure there exists a var_info dict for this variable - if not var_name in self.var_info: + if var_name not in self.var_info: self.var_info[var_name] = {} # use variable specific entry if available @@ -1116,7 +1112,7 @@ def resample_time( outdata = self else: outdata = self.copy() - if not var_name in outdata: + if var_name not in outdata: raise KeyError(f"Variable {var_name} does not exist") to_ts_type = TsType(ts_type) # make sure to use AeroCom ts_type @@ -1133,7 +1129,7 @@ def resample_time( data = outdata[var_name] - if not isinstance(data, (pd.Series, xr.DataArray)): + if not isinstance(data, pd.Series | xr.DataArray): data = outdata.to_timeseries(var_name) resampler = TimeResampler(data) new = resampler.resample( @@ -1298,18 +1294,18 @@ def select_altitude(self, var_name, altitudes): altitudes = slice(altitudes[0], altitudes[1]) result = data.sel(altitude=altitudes) if len(result.altitude) == 0: - raise ValueError(f"no data in specified altitude range") + raise ValueError("no data in specified altitude range") return result raise DataExtractionError("Cannot intepret input for altitude...") elif isinstance(data, pd.Series) or len(self.dtime) == len(data): - if not "altitude" in self: + if "altitude" not in self: raise ValueError("Missing altitude information") if not isinstance(data, pd.Series): data = pd.Series(data, self.dtime) alt = self.altitude - if not isinstance(alt, (list, np.ndarray)): + if not isinstance(alt, list | np.ndarray): raise AttributeError("need 1D altitude array") elif not len(alt) == len(data): raise DataDimensionError( @@ -1317,7 +1313,7 @@ def select_altitude(self, var_name, altitudes): ) mask = np.logical_and(alt >= altitudes[0], alt <= altitudes[1]) if mask.sum() == 0: - raise ValueError(f"no data in specified altitude range") + raise ValueError("no data in specified altitude range") return data[mask] raise DataExtractionError( @@ -1344,7 +1340,7 @@ def to_timeseries(self, var_name, **kwargs): ValueError if length of data array does not equal the length of the time array """ - if not var_name in self: + if var_name not in self: raise KeyError(f"Variable {var_name} does not exist") data = self[var_name] @@ -1356,7 +1352,7 @@ def to_timeseries(self, var_name, **kwargs): "contain 2 dimensions of time " "and altitude" ) - if not "altitude" in kwargs: + if "altitude" not in kwargs: raise ValueError( "please specify altitude range via input " "arg: altitude, e.g. altitude=(100,110)" @@ -1417,7 +1413,7 @@ def plot_timeseries(self, var_name, add_overlaps=False, legend=True, tit=None, * lbl += f" ({ts_type})" except Exception: pass - if not "ax" in kwargs: + if "ax" not in kwargs: if "figsize" in kwargs: fs = kwargs.pop("figsize") else: @@ -1445,7 +1441,7 @@ def plot_timeseries(self, var_name, add_overlaps=False, legend=True, tit=None, * try: if "units" in self.var_info[var_name]: u = self.var_info[var_name]["units"] - if u is not None and not u in [1, "1"]: + if u is not None and u not in [1, "1"]: ylabel += f" [{u}]" except Exception: logger.warning(f"Failed to access unit information for variable {var_name}") diff --git a/pyaerocom/stats/stats.py b/pyaerocom/stats/stats.py index 3c2d518e4..5599a36b9 100644 --- a/pyaerocom/stats/stats.py +++ b/pyaerocom/stats/stats.py @@ -1,9 +1,19 @@ -from typing import Mapping +from collections.abc import Mapping import numpy as np from pyaerocom.stats.data_filters import FilterByLimit, FilterNaN -from pyaerocom.stats.implementations import * +from pyaerocom.stats.implementations import ( + stat_fge, + stat_mab, + stat_mb, + stat_mnmb, + stat_nmb, + stat_R, + stat_R_kendall, + stat_R_spearman, + stat_rms, +) from pyaerocom.stats.stat_filters import FilterDropStats from pyaerocom.stats.types import DataFilter, StatisticsCalculator, StatisticsFilter, StatsDict diff --git a/pyaerocom/stats/types.py b/pyaerocom/stats/types.py index b4bacce7b..f4d26d63f 100644 --- a/pyaerocom/stats/types.py +++ b/pyaerocom/stats/types.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable import numpy as np diff --git a/pyaerocom/time_resampler.py b/pyaerocom/time_resampler.py index 0c8c64d8e..815869256 100644 --- a/pyaerocom/time_resampler.py +++ b/pyaerocom/time_resampler.py @@ -55,19 +55,19 @@ def input_data(self): @input_data.setter def input_data(self, val): - if not isinstance(val, (pd.Series, xarr.DataArray)): + if not isinstance(val, pd.Series | xarr.DataArray): raise ValueError("Invalid input: need Series or DataArray") self._input_data = val @property def fun(self): - """Resamplig method (depends on input data type)""" + """Resampling method (depends on input data type)""" if isinstance(self.input_data, pd.Series): return resample_timeseries return resample_time_dataarray def _get_resample_how(self, fr, to, how): - if not isinstance(how, (str, dict)): + if not isinstance(how, str | dict): val = self.DEFAULT_HOW elif isinstance(how, dict): if to.val in how and fr.val in how[to.val]: diff --git a/pyaerocom/trends_engine.py b/pyaerocom/trends_engine.py index 16611b1fc..ad1e82915 100644 --- a/pyaerocom/trends_engine.py +++ b/pyaerocom/trends_engine.py @@ -56,7 +56,7 @@ def compute_trend( season = "all" if slope_confidence is None: slope_confidence = 0.68 - if not ts_type in ["yearly", "monthly"]: + if ts_type not in ["yearly", "monthly"]: raise ValueError(ts_type) result = _init_trends_result_dict(start_year) @@ -190,7 +190,7 @@ def get_trend_color(self, trend_val): return self.CMAP(self.NORM(trend_val)) def plot(self, season="all", period=None, ax=None): - if not season in self.seasons_avail: + if season not in self.seasons_avail: raise AttributeError(f"No results available for season {season}") if period is None: if len(self.results[season]) > 1: @@ -218,7 +218,7 @@ def plot(self, season="all", period=None, ax=None): ylbl = self.var_name if self.var_name is not None and "units" in self.meta: u = str(self.meta["units"]) - if not u in ["", "1"]: + if u not in ["", "1"]: ylbl += f" [{u}]" ax.set_ylabel(ylbl) tit = "" diff --git a/pyaerocom/trends_helpers.py b/pyaerocom/trends_helpers.py index d06942e4c..4439087d3 100644 --- a/pyaerocom/trends_helpers.py +++ b/pyaerocom/trends_helpers.py @@ -7,7 +7,6 @@ :class:`TrendsEngine` instead. """ -import matplotlib.pyplot as plt import numpy as np import pandas as pd @@ -88,7 +87,7 @@ def _get_unique_seasons(idx): mons = idx.month for mon in mons: seas = _get_season(mon) - if not seas in seasons: + if seas not in seasons: seasons.append(seas) return seasons diff --git a/pyaerocom/tstype.py b/pyaerocom/tstype.py index ee77aa3a5..fd330dccd 100644 --- a/pyaerocom/tstype.py +++ b/pyaerocom/tstype.py @@ -88,7 +88,7 @@ def val(self, val): ivalstr = re.findall(r"\d+", val)[0] val = val.split(ivalstr)[-1] mulfac = int(ivalstr) - if not val in self.VALID: + if val not in self.VALID: try: val = self._from_pandas(val) except TemporalResolutionError: @@ -118,7 +118,7 @@ def timedelta64_str(self): @property def cf_base_unit(self): """Convert ts_type str to CF convention time unit""" - if not self.base in self.TSTR_TO_CF: + if self.base not in self.TSTR_TO_CF: raise NotImplementedError(f"Cannot convert {self.base} to CF str") return self.TSTR_TO_CF[self.base] @@ -187,7 +187,7 @@ def next_lower(self): return tst try: maxmul = self.TS_MAX_VALS[tst.base] - except: + except Exception: maxmul = 10 numsecs = self.num_secs for mulfac in range(1, maxmul + 1): @@ -205,14 +205,14 @@ def valid(val): return False def to_numpy_freq(self): - if not self._val in self.TO_NUMPY: + if self._val not in self.TO_NUMPY: raise TemporalResolutionError(f"numpy frequency not available for {self._val}") freq = self.TO_NUMPY[self._val] return f"{self.mulfac}{freq}" def to_pandas_freq(self): """Convert ts_type to pandas frequency string""" - if not self._val in self.TO_PANDAS: + if self._val not in self.TO_PANDAS: raise TemporalResolutionError(f"pandas frequency not available for {self._val}") freq = self.TO_PANDAS[self._val] if self._mulfac == 1: @@ -222,7 +222,7 @@ def to_pandas_freq(self): def to_si(self): """Convert to SI conform string (e.g. used for unit conversion)""" base = self.base - if not base in self.TO_SI: + if base not in self.TO_SI: raise ValueError(f"Cannot convert ts_type={self} to SI unit string...") si = self.TO_SI[base] return si if self.mulfac == 1 else f"({self.mulfac}{si})" @@ -381,7 +381,7 @@ def from_total_seconds(total_seconds): ) def _from_pandas(self, val): - if not val in self.FROM_PANDAS: + if val not in self.FROM_PANDAS: raise TemporalResolutionError(f"Invalid input: {val}, need pandas frequency string") return self.FROM_PANDAS[val] diff --git a/pyaerocom/ungriddeddata.py b/pyaerocom/ungriddeddata.py index 0b8f50266..834ab0bd7 100644 --- a/pyaerocom/ungriddeddata.py +++ b/pyaerocom/ungriddeddata.py @@ -213,8 +213,8 @@ def _check_index(self): vars_avail = self.var_idx for idx, meta in self.metadata.items(): - if not "var_info" in meta: - if not "variables" in meta: + if "var_info" not in meta: + if "variables" not in meta: raise AttributeError( f"Need either variables (list) or var_info (dict) " f"in meta block {idx}: {meta}" @@ -307,7 +307,7 @@ def from_station_data(stats, add_meta_keys=None): append_vars = list(stat.var_info) for var in append_vars: - if not var in data_obj.var_idx: + if var not in data_obj.var_idx: var_count_glob += 1 var_idx = var_count_glob data_obj.var_idx[var] = var_idx @@ -435,7 +435,7 @@ def _init_index(self, add_cols=None): next_idx = max(idx.values()) + 1 if add_cols is not None: - if not isinstance(add_cols, (list, tuple)): + if not isinstance(add_cols, list | tuple): raise ValueError("Invalid input for add_cols. Need list or tuple") for name in add_cols: if name in idx: @@ -507,7 +507,7 @@ def contains_datasets(self): datasets = [] for info in self.metadata.values(): ds = info["data_id"] - if not ds in datasets: + if ds not in datasets: datasets.append(ds) return datasets @@ -518,7 +518,7 @@ def contains_instruments(self): for info in self.metadata.values(): try: instr = info["instrument_name"] - if instr is not None and not instr in instruments: + if instr is not None and instr not in instruments: instruments.append(instr) except Exception: pass @@ -625,7 +625,7 @@ def available_meta_keys(self): metakeys = [] for meta in self.metadata.values(): for key in meta: - if not key in metakeys: + if key not in metakeys: metakeys.append(key) return metakeys @@ -735,7 +735,7 @@ def _get_stat_coords(self): for idx, meta in self.metadata.items(): try: lat, lon = meta["latitude"], meta["longitude"] - except: + except Exception: logger.warning(f"Could not retrieve lat lon coord at meta index {idx}") continue meta_idx.append(idx) @@ -771,7 +771,7 @@ def check_set_country(self): for i, idx in enumerate(meta_idx): meta = self.metadata[idx] - if not "country" in meta or meta["country"] is None: + if "country" not in meta or meta["country"] is None: country = info[i]["country"] meta["country"] = country meta["country_code"] = info[i]["country_code"] @@ -789,7 +789,7 @@ def countries_available(self): for idx, meta in self.metadata.items(): try: countries.append(meta["country"]) - except: + except Exception: logger.warning("No country information in meta block", idx) if len(countries) == 0: logger.warning( @@ -966,7 +966,7 @@ def to_station_data( stats_ok = [] for stat in stats: for var in vars_to_convert: - if not var in stat: + if var not in stat: continue if freq is not None: stat.resample_time( @@ -1005,7 +1005,7 @@ def _try_infer_stat_merge_pref_attr(self, stats): data_id = None pref_attr = None for stat in stats: - if not "data_id" in stat: + if "data_id" not in stat: return None elif data_id is None: data_id = stat["data_id"] @@ -1054,7 +1054,7 @@ def _metablock_to_stationdata( try: vars_avail = list(meta["var_info"]) except KeyError: - if not "variables" in meta or meta["variables"] in (None, []): + if "variables" not in meta or meta["variables"] in (None, []): raise VarNotAvailableError("Metablock does not contain variable information") vars_avail = meta["variables"] @@ -1183,7 +1183,7 @@ def _generate_station_index(self, by_station_name=True, ignore_index=None): ignore_index = [ignore_index] if not isinstance(ignore_index, list): raise ValueError("Invalid input for ignore_index, need number or list") - return [i for i in range(len(self.metadata)) if not i in ignore_index] + return [i for i in range(len(self.metadata)) if i not in ignore_index] # by station name and ignore certation stations _iter = [] @@ -1320,7 +1320,7 @@ def _check_str_filter_match(self, meta, negate, str_f): # wildcard matching for metakey, filterval in str_f.items(): # key does not exist in this specific meta_block - if not metakey in meta: + if metakey not in meta: return False # check if this key is in negate list (then result will be True # for all that do not match the specified filter input value(s)) @@ -1359,7 +1359,7 @@ def _check_filter_match(self, meta, negate, str_f, list_f, range_f, val_f): return False for metakey, filterval in list_f.items(): - if not metakey in meta: + if metakey not in meta: return False neg = metakey in negate metaval = meta[metakey] @@ -1390,7 +1390,7 @@ def _check_filter_match(self, meta, negate, str_f, list_f, range_f, val_f): return False # range filter for metakey, filterval in range_f.items(): - if not metakey in meta: + if metakey not in meta: return False neg = metakey in negate match = in_range(meta[metakey], filterval[0], filterval[1]) @@ -1398,7 +1398,7 @@ def _check_filter_match(self, meta, negate, str_f, list_f, range_f, val_f): return False for metakey, filterval in val_f.items(): - if not metakey in meta: + if metakey not in meta: return False neg = metakey in negate match = meta[metakey] == filterval @@ -1437,7 +1437,7 @@ def _init_meta_filters(self, **filter_attributes): range_f = {} val_f = {} for key, val in filter_attributes.items(): - if not key in valid_keys: + if key not in valid_keys: raise OSError( f"Invalid input parameter for filtering: {key}. " f"Please choose from {valid_keys}" @@ -1447,7 +1447,7 @@ def _init_meta_filters(self, **filter_attributes): str_f[key] = val elif isnumeric(val): val_f[key] = val - elif isinstance(val, (list, np.ndarray, tuple)): + elif isinstance(val, list | np.ndarray | tuple): if all([isinstance(x, str) for x in val]): list_f[key] = val elif len(val) == 2 and all([isnumeric(x) for x in val]): @@ -1456,7 +1456,7 @@ def _init_meta_filters(self, **filter_attributes): if not low < high: raise ValueError("First entry needs to be smaller than 2nd") range_f[key] = [low, high] - except Exception as e: + except Exception: list_f[key] = val else: list_f[key] = val @@ -1518,7 +1518,7 @@ def check_unit(self, var_name, unit=None): if var_name in meta["var_info"]: try: u = meta["var_info"][var_name]["units"] - if not u in units: + if u not in units: units.append(u) except KeyError: add_str = "" @@ -1689,7 +1689,7 @@ def station_coordinates(self): d = {"station_name": [], "latitude": [], "longitude": [], "altitude": []} for i, meta in self.metadata.items(): - if not "station_name" in meta: + if "station_name" not in meta: logger.debug(f"Skipping meta-block {i}: station_name is not defined") continue elif not all(name in meta for name in const.STANDARD_COORD_NAMES): @@ -1812,7 +1812,7 @@ def apply_region_mask(self, region_id=None): region_id : str or list (of strings) ID of region or IDs of multiple regions to be combined """ - if not region_id in const.HTAP_REGIONS: + if region_id not in const.HTAP_REGIONS: raise ValueError( f"Invalid input for region_id: {region_id}, choose from: {const.HTAP_REGIONS}" ) @@ -1881,7 +1881,7 @@ def apply_filters(self, var_outlier_ranges=None, **filter_attributes): if isinstance(extract_vars, str): extract_vars = [extract_vars] for var in extract_vars: - if not var in data.contains_vars: + if var not in data.contains_vars: raise VarNotAvailableError( f"No such variable {var} in UngriddedData object. " f"Available vars: {self.contains_vars}" @@ -1959,7 +1959,7 @@ def filter_by_projection( ) if len(meta_matches) == len(self.metadata): - logger.info(f"filter_by_projection result in unchanged data object") + logger.info("filter_by_projection result in unchanged data object") return self new = self._new_from_meta_blocks(meta_matches, totnum) return new @@ -2149,7 +2149,7 @@ def extract_var(self, var_name, check_index=True): UngriddedData new data object containing only input variable data """ - if not var_name in self.contains_vars: + if var_name not in self.contains_vars: # try alias _var = const.VARS[var_name].var_name_aerocom if _var in self.contains_vars: @@ -2355,7 +2355,7 @@ def merge_common_meta(self, ignore_keys=None): stop = didx + num new._data[didx:stop, :] = self._data[data_idx] new._data[didx:stop, 0] = i - if not var in _meta_idx_new: + if var not in _meta_idx_new: _meta_idx_new[var] = np.arange(didx, stop) else: _idx = np.append(_meta_idx_new[var], np.arange(didx, stop)) @@ -2559,7 +2559,7 @@ def all_datapoints_var(self, var_name): AttributeError if variable name is not available """ - if not var_name in self.var_idx: + if var_name not in self.var_idx: raise AttributeError(f"Variable {var_name} not available in data") idx = self.var_idx[var_name] mask = np.where(self._data[:, self._VARINDEX] == idx)[0] @@ -2643,7 +2643,7 @@ def find_common_stations( _check_vars = True if isinstance(check_vars_available, str): check_vars_available = [check_vars_available] - elif isinstance(check_vars_available, (tuple, np.ndarray)): + elif isinstance(check_vars_available, tuple | np.ndarray): check_vars_available = list(check_vars_available) if not isinstance(check_vars_available, list): raise ValueError( @@ -2660,7 +2660,7 @@ def find_common_stations( if _check_vars: for var in check_vars_available: try: - if not var in meta["variables"]: + if var not in meta["variables"]: logger.debug(f"No {var} in data of station {name} ({meta['data_id']})") ok = False except Exception: # attribute does not exist or is not iterable @@ -2671,7 +2671,7 @@ def find_common_stations( if _check_vars: for var in check_vars_available: try: - if not var in meta_other["variables"]: + if var not in meta_other["variables"]: logger.debug( f"No {var} in data of station {name} ({meta_other['data_id']})" ) @@ -2878,7 +2878,7 @@ def plot_station_coordinates( else: if not isinstance(var_name, str): raise ValueError("Can only handle single variable (or all -> input var_name=None)") - elif not var_name in subset.contains_vars: + elif var_name not in subset.contains_vars: raise ValueError(f"Input variable {var_name} is not available in dataset ") info_str = var_name @@ -2902,7 +2902,7 @@ def plot_station_coordinates( ) lons = stat_data["longitude"] lats = stat_data["latitude"] - if not "label" in kwargs: + if "label" not in kwargs: kwargs["label"] = info_str ax = plot_coordinates( @@ -3027,13 +3027,7 @@ def __next__(self): return StationData() def __repr__(self): - return "{} list[str]: + return [x.strip() for x in val.split(",")] + + +def str2bool(val: str) -> bool: + return val.lower() in {"true", "1", "t", "yes"} + + class Variable: """Interface that specifies default settings for a variable @@ -69,7 +81,8 @@ class Variable: lower limit of allowed value range upper_limit : float upper limit of allowed value range - obs_wavelength_tol_nm : float + obs_wavelength_tol_nm : float literal_eval_list = lambda val: list(literal_eval(val)) + wavelength tolerance (+/-) for reading of obsdata. Default is 10, i.e. if this variable is defined at 550 nm and obsdata contains measured values of this quantity within interval of 540 - 560, then these data @@ -102,10 +115,6 @@ class Variable: colorbar ticks """ - literal_eval_list = lambda val: list(literal_eval(val)) - str2list = lambda val: [x.strip() for x in val.split(",")] - str2bool = lambda val: val.lower() in ("true", "1", "t", "yes") - _TYPE_CONV = { "wavelength_nm": float, "minimum": float, @@ -350,7 +359,7 @@ def update(self, **kwargs): @property def has_unit(self): """Boolean specifying whether variable has unit""" - return True if not self.units in (1, None) else False + return True if self.units not in (1, None) else False @property def lower_limit(self): @@ -503,7 +512,7 @@ def parse_from_ini(self, var_name=None, cfg=None): cfg = self.read_config() elif not isinstance(cfg, ConfigParser): raise ValueError(f"invalid input for cfg, need config parser got {type(cfg)}") - if not var_name in cfg: + if var_name not in cfg: try: var_name = self._check_aliases(var_name) except VariableDefinitionError: @@ -515,7 +524,7 @@ def parse_from_ini(self, var_name=None, cfg=None): # this variable should import settings from another variable if "use" in var_info: use = var_info["use"] - if not use in cfg: + if use not in cfg: raise VariableDefinitionError( f"Input variable {var_name} depends on {use} " f"which is not available in variables.ini." @@ -531,7 +540,7 @@ def _add(self, key, val): if key in self._TYPE_CONV: try: val = self._TYPE_CONV[key](val) - except: + except Exception: pass elif key == "units" and val == "None": val = "1" diff --git a/pyaerocom/vert_coords.py b/pyaerocom/vert_coords.py index 29fdc31b8..891871bbf 100644 --- a/pyaerocom/vert_coords.py +++ b/pyaerocom/vert_coords.py @@ -13,13 +13,11 @@ import logging import numpy as np -from geonum import atmosphere as atm from pyaerocom import const from pyaerocom.exceptions import ( CoordinateNameError, VariableDefinitionError, - VariableNotFoundError, ) logger = logging.getLogger(__name__) @@ -183,7 +181,7 @@ def lev_increases_with_alt(self) -> bool: ------- True """ - if not self.var_name in self._LEV_INCREASES_WITH_ALT: + if self.var_name not in self._LEV_INCREASES_WITH_ALT: if self.standard_name in self._LEV_INCREASES_WITH_ALT: return self._LEV_INCREASES_WITH_ALT[self.standard_name] raise ValueError( @@ -210,7 +208,7 @@ def calc_pressure(self, lev: np.ndarray, **kwargs) -> np.ndarray: pressure levels in Pa """ - if not self.var_name in self.NAMES_SUPPORTED: + if self.var_name not in self.NAMES_SUPPORTED: raise CoordinateNameError( f"Variable {self.var_name} cannot be converted to pressure levels. " f"Conversion is only possible for supported variables:\n{self.vars_supported_str}" @@ -277,8 +275,8 @@ def __contains__(self, key): @property def coord_list(self): """List of AeroCom coordinate names for altitude access""" - l = self.ADD_FILE_VARS + list(VerticalCoordinate.NAMES_SUPPORTED.values()) - return list(dict.fromkeys(l)) + name = self.ADD_FILE_VARS + list(VerticalCoordinate.NAMES_SUPPORTED.values()) + return sorted(set(name)) def search_aux_coords(self, coord_list) -> bool: """Search and assign coordinates provided by input list @@ -310,7 +308,7 @@ def search_aux_coords(self, coord_list) -> bool: if coord in self: continue try: - coord_info = const.COORDINFO[coord] + const.COORDINFO[coord] except VariableDefinitionError: raise CoordinateNameError(f"Coordinate {coord} is not supported by pyaerocom.") # 1. check if coordinate is assigned in data object directly diff --git a/pyaerocom/vertical_profile.py b/pyaerocom/vertical_profile.py index 4c0715ffb..2e872b7e2 100644 --- a/pyaerocom/vertical_profile.py +++ b/pyaerocom/vertical_profile.py @@ -1,5 +1,3 @@ -from typing import Optional - import matplotlib.pyplot as plt import numpy as np import numpy.typing as npt @@ -16,7 +14,7 @@ def __init__( altitude: npt.ArrayLike, dtime, var_name: str, - data_err: Optional[npt.ArrayLike], + data_err: npt.ArrayLike | None, var_unit: str, altitude_unit: str, ): diff --git a/pyaerocom_env.yml b/pyaerocom_env.yml index b9468019a..c92368886 100644 --- a/pyaerocom_env.yml +++ b/pyaerocom_env.yml @@ -4,7 +4,7 @@ channels: - conda-forge dependencies: - iris >=3.8.1 - - xarray >=2022.10.0 + - xarray >=2022.10.0, <2024.7.0 - cartopy >=0.21.1 - matplotlib-base >=3.7.1 - scipy >=1.10.1 @@ -32,7 +32,7 @@ dependencies: - geojsoncontour - geocoder_reverse_natural_earth >= 0.0.2 - pyaro >= 0.0.10 - - git+https://github.com/metno/aerovaldb.git@v0.0.13 + - git+https://github.com/metno/aerovaldb.git@v0.0.14 ## testing - pytest >=7.4 - pytest-dependency diff --git a/pyproject.toml b/pyproject.toml index 28837ed5f..8dc25f591 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,9 +21,9 @@ classifiers = [ ] requires-python = ">=3.10" dependencies = [ - "aerovaldb@git+https://github.com/metno/aerovaldb.git@v0.0.13", + "aerovaldb@git+https://github.com/metno/aerovaldb.git@v0.0.14", "scitools-iris>=3.8.1", - "xarray>=2022.10.0", + "xarray>=2022.10.0, <2024.7.0", "cartopy>=0.21.1", "matplotlib>=3.7.1", "scipy>=1.10.1", @@ -49,7 +49,7 @@ dependencies = [ "pyproj>=3.0.0", "pyaro>=0.0.10", "pooch>=1.7.0", - "psutil>=5.0.0" + "psutil>=5.0.0", ] [project.readme] @@ -70,12 +70,7 @@ docs = [ "sphinx-argparse", "nbsphinx", ] -test = [ - "pytest>=7.4", - "pytest-dependency", - "pytest-cov", - "packaging", -] +test = ["pytest>=7.4", "pytest-dependency", "pytest-cov", "packaging"] lint = ["mypy>=1.5.1", "types-requests", "types-setuptools", "types-simplejson"] dev = ["pytest-sugar", "pytest-xdist", "pre-commit"] @@ -141,17 +136,25 @@ exclude_lines = [ 'if __name__ == "__main__":', ] -[tool.black] -target-version = ['py310'] -extend-exclude = "pyaerocom-tutorials" +[tool.ruff] +include = ["pyproject.toml", "pyaerocom/**/*.py", "tests/**/*.py"] +target-version = "py310" line-length = 99 +extend-exclude = ["pyaerocom-tutorials", "scripts"] + +[tool.ruff.lint] +select = ["E", "F", "I002", "UP"] +ignore = ["E402", "E501"] -[tool.isort] -py_version = "310" -profile = "black" -src_paths = ["pyaerocom", "tests"] -extend_skip = ["pyaerocom-tutorials"] -line_length = 99 +[tool.ruff.lint.per-file-ignores] +"__init__.py" = ["F401"] +"cli.py" = ["UP007"] +"cli_mos.py" = ["UP007"] +"*/extras/satellite_l2/*.py" = [ + "F402", # https://docs.astral.sh/ruff/rules/import-shadowed-by-loop-var/ + "F821", # https://docs.astral.sh/ruff/rules/undefined-name/ + "F841", # https://docs.astral.sh/ruff/rules/unused-variable/ +] [tool.mypy] python_version = "3.10" @@ -194,7 +197,7 @@ max-line-length = 99 [tool.tox] legacy_tox_ini = """ [tox] -envlist = py310, py311, py312, format, lint, docs, build +envlist = py310, py311, py312, lint, type-check, docs, build skip_missing_interpreters = True isolated_build = True requires = @@ -230,13 +233,14 @@ deps = xarray ==2022.10.0; python_version < "3.11" pandas ==1.5.3; python_version < "3.11" -[testenv:format] +[testenv:lint] +skip_install = True commands = pre-commit run --all-files --show-diff-on-failure deps = pre-commit -[testenv:lint] +[testenv:type-check] ignore_outcome = True commands = mypy pyaerocom/ diff --git a/scripts/aeolus2netcdf.py b/scripts/aeolus2netcdf.py index 2dd30f467..dfb626b6b 100644 --- a/scripts/aeolus2netcdf.py +++ b/scripts/aeolus2netcdf.py @@ -2,6 +2,7 @@ """ read binary ESA L2B files of the ADM Aeolus mission """ + import argparse import glob import logging diff --git a/scripts/read_sentinel5p.py b/scripts/read_sentinel5p.py index f83d766c9..c87b3c548 100644 --- a/scripts/read_sentinel5p.py +++ b/scripts/read_sentinel5p.py @@ -2,6 +2,7 @@ """ small test for the sentinel5p reading """ + import argparse import glob import os diff --git a/tests/aeroval/test_coldatatojson_helpers.py b/tests/aeroval/test_coldatatojson_helpers.py index db04b147e..f04000b4d 100644 --- a/tests/aeroval/test_coldatatojson_helpers.py +++ b/tests/aeroval/test_coldatatojson_helpers.py @@ -1,8 +1,5 @@ from __future__ import annotations -import json -from pathlib import Path - import numpy as np from pytest import mark, param @@ -45,7 +42,7 @@ def test__prepare_regions_json_helper(region_ids: list[str]): for region_name, borders in helper[0].items(): assert region_name in helper[1] - assert type(borders) == dict + assert type(borders) is dict assert "minLat" in borders and -90 <= borders["minLat"] <= 90 assert "maxLat" in borders and -90 <= borders["maxLat"] <= 90 assert "minLon" in borders and -180 <= borders["minLon"] <= 180 diff --git a/tests/aeroval/test_coldatatojson_helpers2.py b/tests/aeroval/test_coldatatojson_helpers2.py index 64c976952..67e8af8dc 100644 --- a/tests/aeroval/test_coldatatojson_helpers2.py +++ b/tests/aeroval/test_coldatatojson_helpers2.py @@ -94,15 +94,19 @@ def test__process_statistics_timeseries( @pytest.mark.parametrize( - "freq,region_ids,data_freq,nmb_avg,drop_stats", + "freq,region_ids,data_freq,drop_stats", [ - ("yearly", {"EUROPE": "Europe"}, "monthly", 0.168, ("mb", "mab")), - ("yearly", {"EUROPE": "Europe"}, None, 0.122, ("nmb")), + ("yearly", {"EUROPE": "Europe"}, "monthly", ("mb", "mab")), + ("yearly", {"EUROPE": "Europe"}, None, ("nmb",)), ], ) @pytest.mark.filterwarnings("ignore:Mean of empty slice:RuntimeWarning") def test__process_statistics_timeseries_drop_stats( - example_coldata, freq: str, region_ids: dict[str, str], data_freq: str, nmb_avg, drop_stats + example_coldata, + freq: str, + region_ids: dict[str, str], + data_freq: str, + drop_stats: tuple[str], ): result = _process_statistics_timeseries( data=example_coldata, @@ -113,7 +117,7 @@ def test__process_statistics_timeseries_drop_stats( use_country=False, data_freq=data_freq, ) - assert all([stat not in result for stat in drop_stats]) + assert result.keys().isdisjoint(drop_stats) @pytest.mark.parametrize( @@ -252,26 +256,29 @@ def test__init_meta_glob(coldata: ColocatedData): no_meta_coldata.data.attrs = {} res = _init_meta_glob(no_meta_coldata) res.pop("processed_utc") - assert np.all(value == "UNDEFINED" for value in res.values()) + assert set(res.values()) == {"UNDEFINED"} + + +@pytest.mark.parametrize("resolution", [("yearly")]) +@pytest.mark.parametrize("coldataset", ["fake_3d_trends"]) +def test__create_diurnal_weekly_data_object(coldata: ColocatedData, resolution: str): + obj = _create_diurnal_weekly_data_object(coldata, resolution) + assert isinstance(obj, xarray.Dataset) @pytest.mark.parametrize( - "resolution", + "resolution,error_message", [ - ("yearly"), - ("seasonal"), - ("cat"), + ("seasonal", "hour must not be empty"), + ("cat", "Invalid resolution"), ], ) @pytest.mark.parametrize("coldataset", ["fake_3d_trends"]) -def test__create_diurnal_weekly_data_object(coldata: ColocatedData, resolution: str): - try: - obj = _create_diurnal_weekly_data_object(coldata, resolution) - assert isinstance(obj, xarray.Dataset) - except: - with pytest.raises(ValueError) as e: - _create_diurnal_weekly_data_object(coldata, resolution) - assert e.type is ValueError +def test__create_diurnal_weekly_data_object__error( + coldata: ColocatedData, resolution: str, error_message: str +): + with pytest.raises(ValueError, match=error_message): + _create_diurnal_weekly_data_object(coldata, resolution) @pytest.mark.parametrize( @@ -330,6 +337,5 @@ def test__remove_less_covered( def test__remove_less_covered_error(): cd = COLDATA["fake_3d_partial_trends_coltime"]() - with pytest.raises(TrendsError) as e: - new_cd = _remove_less_covered(cd, 1000) - assert str(e.value) == "No stations left after removing stations with fewer than 1000 years!" + with pytest.raises(TrendsError, match="No stations left"): + _remove_less_covered(cd, 1000) diff --git a/tests/aeroval/test_json_utils.py b/tests/aeroval/test_json_utils.py index 30243148e..5969e449f 100644 --- a/tests/aeroval/test_json_utils.py +++ b/tests/aeroval/test_json_utils.py @@ -23,7 +23,7 @@ def json_path(tmp_path: Path) -> Path: "raw,precision,rounded", [ pytest.param( - float(1.12344567890), + 1.12344567890, 5, 1.12345, id="single float", @@ -41,7 +41,7 @@ def json_path(tmp_path: Path) -> Path: id="np.float tuple", ), pytest.param( - dict(bla=np.float128(0.1234455667), blubb=int(1), ha="test"), + dict(bla=np.float128(0.1234455667), blubb=1, ha="test"), 5, dict(bla=pytest.approx(0.12345, 1e-5), blubb=1, ha="test"), id="mixed dict", @@ -51,10 +51,10 @@ def json_path(tmp_path: Path) -> Path: def test_round_floats(raw, precision: int, rounded): set_float_serialization_precision(precision) _rounded = round_floats(raw) - if type(raw) in (list, tuple): - assert type(_rounded) == list - if type(raw) == dict: - assert type(_rounded) == dict + if isinstance(raw, list | tuple): + assert type(_rounded) is list + if isinstance(raw, dict): + assert type(_rounded) is dict assert _rounded == rounded diff --git a/tests/aeroval/test_setup.py b/tests/aeroval/test_setup.py index e3dee957c..221eb75b4 100644 --- a/tests/aeroval/test_setup.py +++ b/tests/aeroval/test_setup.py @@ -5,7 +5,6 @@ from pathlib import Path import pytest -from pydantic import ValidationError from pyaerocom.aeroval import EvalSetup from pyaerocom.aeroval._processing_base import DataImporter, HasColocator, HasConfig diff --git a/tests/colocation/test_colocation_setup.py b/tests/colocation/test_colocation_setup.py index 5a99caab7..5a38fe1c4 100644 --- a/tests/colocation/test_colocation_setup.py +++ b/tests/colocation/test_colocation_setup.py @@ -69,12 +69,8 @@ def test_ColocationSetup(stp: ColocationSetup, should_be: dict): def test_ColocationSetup_model_kwargs_validationerror() -> None: - stp_dict = default_setup + with pytest.raises(ValidationError, match="Input should be a valid dictionary"): + ColocationSetup(**default_setup, model_kwargs="not a dict") - with pytest.raises(ValidationError): - stp_dict["model_kwargs"] = "not a dict" - stp = ColocationSetup(**stp_dict) - - with pytest.raises(ValidationError): - stp_dict["model_kwargs"] = {"emep_vars": {}, "ts_type": "daily"} - stp = ColocationSetup(**stp_dict) + with pytest.raises(ValidationError, match="Key ts_type not allowed in model_kwargs "): + ColocationSetup(**default_setup, model_kwargs={"emep_vars": {}, "ts_type": "daily"}) diff --git a/tests/fixtures/cams2_83/config.py b/tests/fixtures/cams2_83/config.py index 37fb717dd..60d2679fc 100644 --- a/tests/fixtures/cams2_83/config.py +++ b/tests/fixtures/cams2_83/config.py @@ -1,6 +1,5 @@ from __future__ import annotations -import os from datetime import date, timedelta from itertools import product from pathlib import Path diff --git a/tests/fixtures/collocated_data.py b/tests/fixtures/collocated_data.py index dccc86750..445271d76 100644 --- a/tests/fixtures/collocated_data.py +++ b/tests/fixtures/collocated_data.py @@ -113,11 +113,6 @@ def _create_fake_trends_coldata_3d(): statnames = [f"FakeStation{c}" for c in range(statnum)] data = np.ones((2, timenum, statnum)) - xrange_modulation = np.linspace(0, np.pi * 40, timenum) - # data[1] += 0.1 #+10% model bias - - # Sets seed to always get same trends - # np.random.seed(13) trend_slops = [1, 2, 50, -3] diff --git a/tests/fixtures/data_access.py b/tests/fixtures/data_access.py index e6356e7d6..cd4d53bfe 100644 --- a/tests/fixtures/data_access.py +++ b/tests/fixtures/data_access.py @@ -9,7 +9,6 @@ from pathlib import Path from typing import NamedTuple -import pooch from pyaerocom import const, io from pyaerocom.io.ghost.reader import ReadGhost diff --git a/tests/fixtures/mscw_ctm.py b/tests/fixtures/mscw_ctm.py index 9a34067cb..339ac8040 100644 --- a/tests/fixtures/mscw_ctm.py +++ b/tests/fixtures/mscw_ctm.py @@ -30,20 +30,15 @@ def create_fake_MSCWCtm_data(year="2019", numval=1, tst=None): tst = "monthly" from pyaerocom import TsType - tbase = TsType(tst).cf_base_unit - _lats_fake = np.linspace(30, 82, 10) _lons_fake = np.linspace(-25, 90, 15) start = datetime.strptime(f"{year}-01-01", "%Y-%m-%d") stop = datetime.strptime(f"{year}-12-31", "%Y-%m-%d") _time_fake = pd.date_range(start, stop, freq=TsType(tst).to_pandas_freq()) - # _time_fake = np.arange(10) - timeattrs = {"units": f"{tbase} since 2000-01-01", "calendar": "gregorian"} sh = (len(_time_fake), len(_lats_fake), len(_lons_fake)) _data_fake = numval * np.ones(sh) - # coords = {"time": ("time", _time_fake, timeattrs), "lat": _lats_fake, "lon": _lons_fake} coords = {"time": _time_fake, "latitude": _lats_fake, "longitude": _lons_fake} dims = ["time", "latitude", "longitude"] diff --git a/tests/fixtures/tm5.py b/tests/fixtures/tm5.py index 7fc2ad457..beb3f8918 100644 --- a/tests/fixtures/tm5.py +++ b/tests/fixtures/tm5.py @@ -36,9 +36,9 @@ def load_coldata_tm5_aeronet_from_scratch(path: Path) -> ColocatedData: info = {} for val in arr.attrs["_min_num_obs"].split(";")[:-1]: to, fr, num = val.split(",") - if not to in info: + if to not in info: info[to] = {} - if not fr in info[to]: + if fr not in info[to]: info[to][fr] = {} info[to][fr] = int(num) arr.attrs["min_num_obs"] = info diff --git a/tests/io/mscw_ctm/test_additional_variables.py b/tests/io/mscw_ctm/test_additional_variables.py index 74ef7d89e..99dccba70 100644 --- a/tests/io/mscw_ctm/test_additional_variables.py +++ b/tests/io/mscw_ctm/test_additional_variables.py @@ -10,16 +10,14 @@ ) from tests.fixtures.mscw_ctm import create_fake_MSCWCtm_data +M_N, M_O, M_H = 14.006, 15.999, 1.007 + def test_calc_concNhno3(): conchno3 = create_fake_MSCWCtm_data() concNhno3_from_func = calc_concNhno3(conchno3) - M_N = 14.006 - M_O = 15.999 - M_H = 1.007 - concNhno3 = conchno3 * (M_N / (M_H + M_N + M_O * 3)) concNhno3.attrs["units"] = "ug N m-3" assert (concNhno3 == concNhno3_from_func).all() @@ -31,9 +29,6 @@ def test_calc_concNno3pm10(): concNno3pm10_from_func = calc_concNno3pm10(concno3f, concno3c) - M_N = 14.006 - M_O = 15.999 - fac = M_N / (M_N + 3 * M_O) concno3pm10 = concno3f + concno3c concNno3pm10 = concno3pm10 * fac @@ -48,9 +43,6 @@ def test_calc_concNno3pm25(): concNno3pm10_from_func = calc_concNno3pm25(concno3f, concno3c) - M_N = 14.006 - M_O = 15.999 - fac = M_N / (M_N + 3 * M_O) concno3pm10 = concno3f + 0.134 * concno3c concNno3pm10 = concno3pm10 * fac @@ -93,10 +85,6 @@ def test_calc_concNnh3(): concNnh3_from_func = calc_concNnh3(concnh3) - M_N = 14.006 - M_O = 15.999 - M_H = 1.007 - concNnh3 = concnh3 * (M_N / (M_H * 3 + M_N)) concNnh3.attrs["units"] = "ug N m-3" assert (concNnh3 == concNnh3_from_func).all() @@ -107,10 +95,6 @@ def test_calc_concNnh4(): concNnh4_from_func = calc_concNnh4(concnh4) - M_N = 14.006 - M_O = 15.999 - M_H = 1.007 - concNnh4 = concnh4 * (M_N / (M_H * 4 + M_N)) concNnh4.attrs["units"] = "ug N m-3" assert (concNnh4 == concNnh4_from_func).all() diff --git a/tests/io/mscw_ctm/test_reader.py b/tests/io/mscw_ctm/test_reader.py index 944409e66..0652f8f54 100644 --- a/tests/io/mscw_ctm/test_reader.py +++ b/tests/io/mscw_ctm/test_reader.py @@ -358,9 +358,8 @@ def test_ReadMscwCtm_has_var(reader, var_name, value): def test_ReadMscwCtm_has_var_error(reader): - with pytest.raises(exc.VariableDefinitionError) as e: + with pytest.raises(exc.VariableDefinitionError, match="input variable blaa is not supported"): reader.has_var("blaa") - assert str(e.value) == "Error (VarCollection): input variable blaa is not supported" def test_ReadMscwCtm__str__(): @@ -372,7 +371,7 @@ def test_ReadMscwCtm__repr__(): def test_ReadEMEP__init__(): - with pytest.warns(DeprecationWarning) as e: + with pytest.warns(DeprecationWarning, match="use ReadMscwCtm instead"): assert isinstance(ReadEMEP(), ReadMscwCtm) @@ -571,7 +570,7 @@ def test_read_emep_LF_tst(tmp_path: Path): reader = ReadMscwCtm(data_dir=str(data_path)) with pytest.raises(ValueError) as e: filepaths = reader._filepaths - wrong_path = Path(filepaths[0]).with_name(f"Base_LF_month.nc") + wrong_path = Path(filepaths[0]).with_name("Base_LF_month.nc") reader._get_tst_from_file(str(wrong_path)) assert str(e.value) == f"The file {wrong_path} is not supported" @@ -581,7 +580,7 @@ def test_read_emep_year_defined_twice(tmp_path: Path): data_path = emep_data_path(tmp_path, "day", vars_and_units={"prmm": "mm"}) reader = ReadMscwCtm(data_dir=str(data_path)) filepaths = reader._filepaths - wrong_path = Path(filepaths[0]).with_name(f"Base_day.nc") + wrong_path = Path(filepaths[0]).with_name("Base_day.nc") filepaths.append(str(wrong_path)) new_yrs = reader._get_yrs_from_filepaths() with pytest.raises(ValueError) as e: diff --git a/tests/io/test_ebas_varinfo.py b/tests/io/test_ebas_varinfo.py index 73b049837..be337d688 100644 --- a/tests/io/test_ebas_varinfo.py +++ b/tests/io/test_ebas_varinfo.py @@ -4,7 +4,6 @@ import pytest -from pyaerocom.io import EbasSQLRequest from pyaerocom.io.ebas_file_index import EbasSQLRequest from pyaerocom.io.ebas_varinfo import EbasVarInfo diff --git a/tests/io/test_readgridded.py b/tests/io/test_readgridded.py index 175e84500..08e6c3d75 100644 --- a/tests/io/test_readgridded.py +++ b/tests/io/test_readgridded.py @@ -69,8 +69,8 @@ def test_read_var(reader_tm5: ReadGridded, input_args: dict, mean_val: float): def test_ReadGridded_class_empty(): r = ReadGridded() - assert r.data_id == None - assert r.data_dir == None + assert r.data_id is None + assert r.data_dir is None from pyaerocom.io.aerocom_browser import AerocomBrowser assert isinstance(r.browser, AerocomBrowser) diff --git a/tests/io/test_readungridded.py b/tests/io/test_readungridded.py index d86a167f4..7ffb7014e 100644 --- a/tests/io/test_readungridded.py +++ b/tests/io/test_readungridded.py @@ -182,8 +182,7 @@ def test_config_name_already_exists(pyaro_testconfig): config.name = data_ids[0] with pytest.raises(NameError, match="cannot have the same name as an included dataset"): - reader = ReadUngridded(configs=pyaro_testconfig) + ReadUngridded(configs=pyaro_testconfig) with pytest.raises(NameError, match="cannot have the same name as an included dataset"): - reader = ReadUngridded(data_ids=data_ids) - data = reader.read(configs=config) + ReadUngridded(data_ids=data_ids).read(configs=config) diff --git a/tests/stats/test_data_filters.py b/tests/stats/test_data_filters.py index 77c66a34f..769f31862 100644 --- a/tests/stats/test_data_filters.py +++ b/tests/stats/test_data_filters.py @@ -1,7 +1,7 @@ import numpy as np import pytest -from pyaerocom.stats.data_filters import * +from pyaerocom.stats.data_filters import FilterByLimit, FilterNaN @pytest.mark.parametrize( diff --git a/tests/stats/test_stats.py b/tests/stats/test_stats.py index 8128607d5..9233521be 100644 --- a/tests/stats/test_stats.py +++ b/tests/stats/test_stats.py @@ -1,7 +1,7 @@ import numpy as np import pytest -from pyaerocom.stats.implementations import * +from pyaerocom.stats.implementations import stat_mb, stat_R, stat_R_kendall from pyaerocom.stats.stats import calculate_statistics diff --git a/tests/test__logging.py b/tests/test__logging.py index cb0db9041..75bab53f1 100644 --- a/tests/test__logging.py +++ b/tests/test__logging.py @@ -14,7 +14,7 @@ def get_level_value(logger: logging.Logger | None) -> int: if not logger.hasHandlers(): return logger.getEffectiveLevel() for handler in logger.handlers: - if type(handler) == logging.StreamHandler: + if type(handler) is logging.StreamHandler: return handler.level if not logger.propagate: return logging.NOTSET diff --git a/tests/test_config.py b/tests/test_config.py index a05551d44..94eb2d100 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -271,7 +271,7 @@ def test_empty_class_header(empty_cfg): assert sd in cfg._DB_SEARCH_SUBDIRS assert cfg._DB_SEARCH_SUBDIRS[sd] == name - assert cfg.DONOTCACHEFILE == None + assert cfg.DONOTCACHEFILE is None assert cfg.ERA5_SURFTEMP_FILENAME == "era5.msl.t2m.201001-201012.nc" diff --git a/tests/test_griddeddata.py b/tests/test_griddeddata.py index 29c6764bf..6376e5d67 100644 --- a/tests/test_griddeddata.py +++ b/tests/test_griddeddata.py @@ -130,7 +130,7 @@ def test_GriddedData_resample_time(data_tm5: GriddedData): def test_GriddedData_interpolate(data_tm5: GriddedData): data = data_tm5.interpolate(latitude=TESTLATS, longitude=TESTLONS) - assert type(data) == GriddedData + assert type(data) is GriddedData assert data.shape == (12, 2, 2) assert_allclose(data.mean(False), 0.13877, rtol=TEST_RTOL) diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 135f040b6..ac69ec283 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -137,7 +137,7 @@ def test_same_meta_dict(): station_name="bla", station_id="blub1", latitude=33, longitude=15, altitude=401, PI="pi2" ) - assert helpers.same_meta_dict(d1, d2) == False + assert helpers.same_meta_dict(d1, d2) is False def test_to_pandas_timestamp(): diff --git a/tests/test_mathutils.py b/tests/test_mathutils.py index 05ee5b7e0..916de56ca 100644 --- a/tests/test_mathutils.py +++ b/tests/test_mathutils.py @@ -1,4 +1,3 @@ -import numpy as np import pytest from pyaerocom.mathutils import estimate_value_range, exponent, is_strictly_monotonic, make_binlist diff --git a/tests/test_obs_io.py b/tests/test_obs_io.py index 4761c1dd5..6d4257a03 100644 --- a/tests/test_obs_io.py +++ b/tests/test_obs_io.py @@ -35,7 +35,7 @@ def test_OBS_WAVELENGTH_TOL_NM(): def test_OBS_ALLOW_ALT_WAVELENGTHS(): - assert OBS_ALLOW_ALT_WAVELENGTHS == True + assert OBS_ALLOW_ALT_WAVELENGTHS is True def test_ObsVarCombi(): diff --git a/tests/test_stationdata.py b/tests/test_stationdata.py index 6fc44be1e..8e0d207a7 100644 --- a/tests/test_stationdata.py +++ b/tests/test_stationdata.py @@ -218,7 +218,7 @@ def test_StationData_get_station_coords( pytest.param( stat4, AttributeError, - f"Invalid value encountered for coord longitude, need float, int, list or ndarray, got ", + "Invalid value encountered for coord longitude, need float, int, list or ndarray, got ", id="wrong station latitude", ), pytest.param( @@ -303,7 +303,7 @@ def test_StationData_merge_meta_same_station( def test_StationData_merge_meta_same_station_error(): - with pytest.raises(CoordinateError, match="differ by more than 0.001 km.") as e: + with pytest.raises(CoordinateError, match="differ by more than 0.001 km."): stat1.merge_meta_same_station( stat2, coord_tol_km=0.001, check_coords=True, inplace=False, raise_on_error=True ) @@ -376,8 +376,8 @@ def test_StationData_remove_variable(): assert var in stat.var_info stat.remove_variable(var) - assert not var in stat - assert not var in stat.var_info + assert var not in stat + assert var not in stat.var_info def test_StationData_remove_variable_error(): diff --git a/tests/test_ungriddeddata.py b/tests/test_ungriddeddata.py index 929ad59e2..6777ecc7a 100644 --- a/tests/test_ungriddeddata.py +++ b/tests/test_ungriddeddata.py @@ -174,7 +174,7 @@ def test_filter_region( ({"station_name": ["Tr*", "Mauna*"]}, ["Trelew", "Mauna_Loa"]), ( {"station_name": ["Tr*", "Mauna*"], "negate": "station_name"}, - [x for x in ALL_SITES if not x in ["Trelew", "Mauna_Loa"]], + [x for x in ALL_SITES if x not in ["Trelew", "Mauna_Loa"]], ), ( {"altitude": [0, 1000], "negate": "altitude"}, @@ -241,15 +241,15 @@ def test_from_single_station_data(): def test_last_meta_idx(aeronetsunv3lev2_subset: UngriddedData): - assert isinstance(aeronetsunv3lev2_subset.last_meta_idx, (np.ndarray, np.generic)) + assert isinstance(aeronetsunv3lev2_subset.last_meta_idx, np.ndarray | np.generic) def test_has_flag_data(aeronetsunv3lev2_subset: UngriddedData): - assert isinstance(aeronetsunv3lev2_subset.has_flag_data, (np.bool_, bool)) + assert isinstance(aeronetsunv3lev2_subset.has_flag_data, np.bool_ | bool) def test_is_filtered(aeronetsunv3lev2_subset: UngriddedData): - assert isinstance(aeronetsunv3lev2_subset.is_filtered, (np.bool_, bool)) + assert isinstance(aeronetsunv3lev2_subset.is_filtered, np.bool_ | bool) def test_available_meta_keys(aeronetsunv3lev2_subset: UngriddedData): diff --git a/tests/test_vert_coords.py b/tests/test_vert_coords.py index b31430f28..bf1bc1ba1 100644 --- a/tests/test_vert_coords.py +++ b/tests/test_vert_coords.py @@ -1,10 +1,14 @@ import numpy as np import pytest +from pyaerocom.exceptions import CoordinateNameError from pyaerocom.griddeddata import GriddedData -from pyaerocom.vert_coords import * -from tests.conftest import lustre_unavail -from tests.fixtures.tm5 import data_tm5 +from pyaerocom.vert_coords import ( + AltitudeAccess, + VerticalCoordinate, + atmosphere_hybrid_sigma_pressure_coordinate_to_pressure, + atmosphere_sigma_coordinate_to_pressure, +) @pytest.fixture @@ -52,7 +56,7 @@ def test_atmosphere_sigma_coordinate_to_pressure_exceptions(): def test_atmosphere_sigma_coordinate_to_pressure(sigma, ps, ptop, expected): result = atmosphere_sigma_coordinate_to_pressure(sigma, ps, ptop) - assert type(result) == type(ptop) + assert type(result) is type(ptop) if isinstance(result, np.ndarray): assert len(result) == len(ptop) @@ -114,10 +118,9 @@ def test_VerticalCoordinate_calc_pressure_exceptions(): vert.calc_pressure(np.ones(5)) -@pytest.mark.parametrize("name,exception", (("gph", ValueError),)) -def test_VerticalCoordinate_lev_increases_with_alt_exception(name: str, exception: Exception): - vert = VerticalCoordinate(name) - with pytest.raises(exception): +def test_VerticalCoordinate_lev_increases_with_alt_exception(): + vert = VerticalCoordinate("gph") + with pytest.raises(ValueError, match="Failed to access information"): vert.lev_increases_with_alt @@ -142,14 +145,14 @@ def test_AltitudeAccess_exceptions(): AltitudeAccess(None) -def test_AltitudeAccess_search_aux_coords(alt: AltitudeAccess): - with pytest.raises(CoordinateNameError): - alt.search_aux_coords(["gkjfdshglk"]) - - def test_AltitudeAccess_search_aux_coords(alt: AltitudeAccess): assert alt.search_aux_coords(["lat"]) assert alt.search_aux_coords("lon") assert not alt.search_aux_coords("z") assert alt.search_aux_coords(["lat", "lon", "time"]) assert not alt.search_aux_coords(["lat", "lon", "time", "z"]) + + +def test_AltitudeAccess_search_aux_coords_exception(alt: AltitudeAccess): + with pytest.raises(CoordinateNameError, match=r"Coordinate .* is not supported"): + alt.search_aux_coords(["gkjfdshglk"]) diff --git a/tests/test_zz_unsorted_highlevel.py b/tests/test_zz_unsorted_highlevel.py index e343c1626..9fff7d21d 100644 --- a/tests/test_zz_unsorted_highlevel.py +++ b/tests/test_zz_unsorted_highlevel.py @@ -38,7 +38,7 @@ def test_od550aer_meanval_stats(aeronetsunv3lev2_subset): mean_vals = [] std_vals = [] for stat in aeronetsunv3lev2_subset.to_station_data_all()["stats"]: - if not "od550aer" in stat: + if "od550aer" not in stat: no_odcount += 1 continue td = stat.od550aer[:100] @@ -58,7 +58,7 @@ def test_ang4487aer_meanval_stats(aeronetsunv3lev2_subset): mean_vals = [] std_vals = [] for stat in aeronetsunv3lev2_subset.to_station_data_all()["stats"]: - if not "ang4487aer" in stat: + if "ang4487aer" not in stat: no_odcount += 1 continue td = stat.ang4487aer[:100]