diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index dcec72425..fdac44168 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -10,7 +10,7 @@ jobs: lint: runs-on: ubuntu-latest env: - PYTHON: 3.8 + PYTHON: 3.9 steps: - uses: actions/checkout@v3 - name: Set up Python ${{ env.PYTHON }} @@ -35,9 +35,9 @@ jobs: run: tox -e lint docs: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest env: - PYTHON: 3.8 + PYTHON: 3.9 steps: - uses: actions/checkout@v3 - name: Set up Python ${{ env.PYTHON }} @@ -62,13 +62,13 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.8, 3.9, '3.10'] + python-version: [3.9, '3.10', '3.11'] experimental: [false] os: [ubuntu-22.04] - include: - - python-version: '3.11' - experimental: true - os: ubuntu-22.04 + #include: + # - python-version: '3.12' + # experimental: true + # os: ubuntu-22.04 steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -105,7 +105,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.8, 3.9, '3.10'] + python-version: [3.9, '3.10', '3.11'] experimental: [false] defaults: run: diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 3abb64d0a..db46cdd0e 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -11,7 +11,7 @@ formats: - pdf build: - os: ubuntu-20.04 + os: ubuntu-22.04 apt_packages: - libudunits2-dev - libgeos-dev @@ -19,12 +19,12 @@ build: - proj-data - proj-bin tools: - python: "3.8" + python: "3.9" python: install: - method: pip path: . extra_requirements: - - proj-legacy + - proj8 - docs diff --git a/docs/intro.rst b/docs/intro.rst index bfb9960bd..a75547a17 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -24,7 +24,7 @@ Main features - `EBAS database `__. - `EEA Air Quality e-Reporting (AQ e-Reporting) `__. - `AirNow `__. - - `MarcoPolo `__. + - `Ministry of Environment Protection (MEP) in China`__. - `GHOST` (Globally Harmonised Observational Surface Treatment) (see e.g., `Petetin et al., 2020 `_ for more information). - Reading routines for level 3 gridded satellite observations, such as: diff --git a/pyaerocom/_warnings.py b/pyaerocom/_warnings.py index 9c61027b5..c0db86579 100644 --- a/pyaerocom/_warnings.py +++ b/pyaerocom/_warnings.py @@ -1,12 +1,9 @@ -from __future__ import annotations - import warnings from contextlib import contextmanager -from typing import Type @contextmanager -def ignore_warnings(category: Type[Warning], *messages: str): +def ignore_warnings(category: type[Warning], *messages: str): """ Ignore particular warnings with a decorator or context manager diff --git a/pyaerocom/aeroval/experiment_output.py b/pyaerocom/aeroval/experiment_output.py index 133804654..9b3da2e68 100644 --- a/pyaerocom/aeroval/experiment_output.py +++ b/pyaerocom/aeroval/experiment_output.py @@ -617,11 +617,21 @@ def _get_var_name_and_type(self, var_name): def _init_menu_entry(self, var: str) -> dict: name, tp, cat = self._get_var_name_and_type(var) + out = {"type": tp, "cat": cat, "name": name, "obs": {}} try: lname = const.VARS[var].description except VariableDefinitionError: lname = "UNDEFINED" - return {"type": tp, "cat": cat, "name": name, "longname": lname, "obs": {}} + + out["longname"] = lname + try: + # Comes in as a string. split() here breaks up based on space and returns either just the element in a list or the components of the string in a list + only_use_in = const.VARS[var].only_use_in.split(" ") + # only return only_use_in if key exists, otherwise do not + out["only_use_in"] = only_use_in + except AttributeError: + pass + return out def _check_ovar_mvar_entry(self, mcfg, mod_var, ocfg, obs_var): diff --git a/pyaerocom/aeroval/experiment_processor.py b/pyaerocom/aeroval/experiment_processor.py index 7e9fdf5db..c71d3c3c3 100644 --- a/pyaerocom/aeroval/experiment_processor.py +++ b/pyaerocom/aeroval/experiment_processor.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- import logging from multiprocessing import dummy diff --git a/pyaerocom/aeroval/glob_defaults.py b/pyaerocom/aeroval/glob_defaults.py index bbee38025..cf7e8897b 100644 --- a/pyaerocom/aeroval/glob_defaults.py +++ b/pyaerocom/aeroval/glob_defaults.py @@ -137,7 +137,6 @@ "colmap": "bwr", "unit": "var", "decimals": 1, - "forecast": True, }, "mab": { "name": "MAB", @@ -154,7 +153,6 @@ "colmap": "bwr", "unit": "var", "decimals": 1, - "forecast": True, }, "R": { "name": "R", @@ -323,6 +321,13 @@ conco3=["O3", "3D", "Gas concentrations"], concno310=["NO3_PM10", "3D", "Particle concentration"], concno325=["NO3_PM25", "3D", "Particle concentration"], + proxyod550bc=["OD (Black Carbon)", "2D", "Optical properties"], + proxyod550dust=["OD (Dust)", "2D", "Optical properties"], + proxyod550oa=["OD (Organic Matter)", "2D", "Optical properties"], + proxyod550so4=["OD (SO4)", "2D", "Optical properties"], + proxyod550ss=["OD (Sea Salt)", "2D", "Optical properties"], + proxyod550nh4=["OD (NH4)", "2D", "Optical properties"], + proxyod550no3=["OD (NO3)", "2D", "Optical properties"], # Gases concNno=["NO", "3D", "Concentration"], concno2=["NO2", "3D", "Gas concentrations"], diff --git a/pyaerocom/data/variables.ini b/pyaerocom/data/variables.ini index 7af225cf0..4d6ac39bc 100644 --- a/pyaerocom/data/variables.ini +++ b/pyaerocom/data/variables.ini @@ -67,6 +67,7 @@ standard_name = atmosphere_optical_thickness_due_to_water_in_ambient_aerosol_par use=od550aer description = Aerosol optical depth (AOD) at 550 nm due to water standard_name = atmosphere_optical_thickness_due_to_water_in_ambient_aerosol_particles +only_use_in = maps.php [od550bc] use=od550aer @@ -77,6 +78,7 @@ standard_name = atmosphere_optical_thickness_due_to_elemental_carbon_ambient_aer use=od550aer description = Aerosol optical depth (AOD) at 550 nm due to elemental carbon standard_name = atmosphere_optical_thickness_due_to_elemental_carbon_ambient_aerosol_particles +only_use_in = maps.php [od550dust] use=od550aer @@ -87,6 +89,7 @@ standard_name = atmosphere_optical_thickness_due_to_dust_ambient_aerosol_particl use=od550aer description = Aerosol optical depth (AOD) at 550 nm due to dust standard_name = atmosphere_optical_thickness_due_to_dust_ambient_aerosol_particles +only_use_in = maps.php [od550nh4] use=od550aer @@ -97,12 +100,19 @@ standard_name = atmosphere_optical_thickness_due_to_ammonium_ambient_aerosol_par use=od550aer description = Aerosol optical depth (AOD) at 550 nm due to Ammonium standard_name = atmosphere_optical_thickness_due_to_ammonium_ambient_aerosol_particles +only_use_in = maps.php [od550no3] use=od550aer description = Aerosol optical depth (AOD) at 550 nm due to nitrate standard_name = atmosphere_optical_thickness_due_to_nitrate_ambient_aerosol_particles +[proxyod550no3] +use=od550aer +description = Aerosol optical depth (AOD) at 550 nm due to Nitrate +standard_name = atmosphere_optical_thickness_due_to_nitrate_ambient_aerosol_particles +only_use_in = maps.php + [od550oa] use=od550aer description = Aerosol optical depth (AOD) at 550 nm due to organic matter @@ -112,6 +122,7 @@ standard_name = atmosphere_optical_thickness_due_to_particulate_organic_matter_a use=od550aer description = Aerosol optical depth (AOD) at 550 nm due to organic matter standard_name = atmosphere_optical_thickness_due_to_particulate_organic_matter_ambient_aerosol_particles +only_use_in = maps.php [od550pm1] use=od550aer @@ -152,6 +163,7 @@ standard_name = atmosphere_optical_thickness_due_to_sulfate_ambient_aerosol_part use=od550aer description = Aerosol optical depth (AOD) at 550 nm due to SO4 standard_name = atmosphere_optical_thickness_due_to_sulfate_ambient_aerosol_particles +only_use_in = maps.php [od550ss] use=od550aer @@ -162,6 +174,7 @@ standard_name = atmosphere_optical_thickness_due_to_seasalt_ambient_aerosol_part use=od550aer description = Aerosol optical depth (AOD) at 550 nm due to sea salt standard_name = atmosphere_optical_thickness_due_to_seasalt_ambient_aerosol_particles +only_use_in = maps.php [od550lt1ss] use = od550ss diff --git a/pyaerocom/io/__init__.py b/pyaerocom/io/__init__.py index 9e2e78494..32ab376b1 100644 --- a/pyaerocom/io/__init__.py +++ b/pyaerocom/io/__init__.py @@ -29,6 +29,5 @@ from .read_ebas import ReadEbas from .read_eea_aqerep import ReadEEAAQEREP from .read_eea_aqerep_v2 import ReadEEAAQEREP_V2 -from .read_marcopolo import ReadMarcoPolo from . import helpers_units diff --git a/pyaerocom/io/read_aeronet_sunv3.py b/pyaerocom/io/read_aeronet_sunv3.py index fed6b4dee..2268e5202 100644 --- a/pyaerocom/io/read_aeronet_sunv3.py +++ b/pyaerocom/io/read_aeronet_sunv3.py @@ -30,7 +30,7 @@ class ReadAeronetSunV3(ReadAeronetBase): _FILEMASK = "*.lev*" #: version log of this class (for caching) - __version__ = "0.11_" + ReadAeronetBase.__baseversion__ + __version__ = "0.12_" + ReadAeronetBase.__baseversion__ #: Name of dataset (OBS_ID) DATA_ID = const.AERONET_SUN_V3L2_AOD_DAILY_NAME @@ -99,6 +99,7 @@ class ReadAeronetSunV3(ReadAeronetBase): "proxyod550oa": ["od440aer", "od500aer", "ang4487aer"], "proxyod550so4": ["od440aer", "od500aer", "ang4487aer"], "proxyod550ss": ["od440aer", "od500aer", "ang4487aer"], + "proxyod550no3": ["od440aer", "od500aer", "ang4487aer"], } #: Functions that are used to compute additional variables (i.e. one @@ -114,6 +115,7 @@ class ReadAeronetSunV3(ReadAeronetBase): "proxyod550oa": calc_od550aer, "proxyod550so4": calc_od550aer, "proxyod550ss": calc_od550aer, + "proxyod550no3": calc_od550aer, } #: List of variables that are provided by this dataset (will be extended @@ -165,10 +167,10 @@ def read_file(self, filename, vars_to_retrieve=None, vars_as_series=False): f_out.close() try: - with open(filename, "rt") as in_file: + with open(filename) as in_file: lines = in_file.readlines() except UnicodeDecodeError: - with open(filename, "rt", encoding="ISO-8859-1") as in_file: + with open(filename, encoding="ISO-8859-1") as in_file: lines = in_file.readlines() except OSError: # faulty gzip file, but also the gzip class raises some exceptions diff --git a/pyaerocom/io/read_marcopolo.py b/pyaerocom/io/read_marcopolo.py deleted file mode 100644 index 9c44ce21d..000000000 --- a/pyaerocom/io/read_marcopolo.py +++ /dev/null @@ -1,348 +0,0 @@ -import os - -import numpy as np -import pandas as pd -from geonum import atmosphere as atm -from tqdm import tqdm - -from pyaerocom import const -from pyaerocom.aux_var_helpers import concx_to_vmrx -from pyaerocom.exceptions import DataRetrievalError -from pyaerocom.helpers import varlist_aerocom -from pyaerocom.io import ReadUngriddedBase -from pyaerocom.molmasses import get_molmass -from pyaerocom.ungriddeddata import UngriddedData - -P_STD = atm.p0 # standard atmosphere pressure -T_STD = atm.T0_STD # standard atmosphere temperature - - -def _conc_to_vmr_marcopolo_stats( - data, to_var, from_var, p_pascal=None, T_kelvin=None, mmol_air=None -): - if p_pascal is None: - p_pascal = P_STD - if T_kelvin is None: - T_kelvin = T_STD - if mmol_air is None: - mmol_air = get_molmass("air_dry") - - for stat in data: - if not from_var in stat: - continue - - concdata = stat[from_var] - - mmol_var = get_molmass(to_var) - from_unit = stat["var_info"][from_var]["units"] - to_unit = const.VARS[to_var].units - vmrvals = concx_to_vmrx( - concdata, - p_pascal=p_pascal, - T_kelvin=T_kelvin, - mmol_var=mmol_var, - mmol_air=mmol_air, - conc_unit=from_unit, - to_unit=to_unit, - ) - stat[to_var] = vmrvals - vi = {} - vi.update(stat["var_info"][from_var]) - vi["computed"] = True - vi["units"] = to_unit - vi["units_info"] = ( - f"The original data is provided as mass conc. in units of ug m-3 and " - f"was converted to ppb assuming a standard atmosphere " - f"(p={p_pascal/100}hPa, T={T_kelvin}K) assuming dry molar mass of air " - f"M_Air_dry={mmol_air}g/mol and total molecular mass of " - f"{from_var}={mmol_var}g/mol" - ) - stat["var_info"][to_var] = vi - - return data - - -class ReadMarcoPolo(ReadUngriddedBase): - """ - Reading routine for Chinese MarcoPolo observations - """ - - #: Data type of files - _FILETYPE = ".csv" - - #: Filemask for glob data file search - _FILEMASK = f"*{_FILETYPE}" - - #: Column delimiter - FILE_COL_DELIM = "," - - #: Version log of this class (for caching) - __version__ = "0.02" - - #: Default data ID - DATA_ID = "MarcoPolo" - - #: List of all datasets supported by this interface - SUPPORTED_DATASETS = [DATA_ID] - - #: Indices of columns in data files - FILE_COLS = {"station_id": 0, "datetime": 1, "var_name": 2, "value": 3} - #: Mapping of columns in station metadata file to pyaerocom standard - STATION_META_MAP = { - "stationId": "station_id", - "stationName": "station_name", - "cityId": "city_id", - "latitude": "latitude", - "longitude": "longitude", - "cityName": "city", - } - - #: conversion functions for metadata dtypes - STATION_META_DTYPES = { - "station_id": str, - "station_name": str, - "city_id": str, - "latitude": float, - "longitude": float, - "city": str, - } - - #: Variable names in data files - VAR_MAP = { - "concco": "co", - "concso2": "so2", - "concno2": "no2", - "conco3": "o3", - "concpm10": "pm10", - "concpm25": "pm2_5", - } - - #: Variable units (not provided in data files) - VAR_UNITS = { - "concco": "unknown", - "concso2": "unknown", - "concno2": "ug m-3", # mail from Henk Eskes, 1 Feb. 2021 - "conco3": "ug m-3", # mail from Henk Eskes, 1 Feb. 2021 - "concpm10": "ug m-3", # mail from Henk Eskes, 1 Feb. 2021 - "concpm25": "ug m-3", # mail from Henk Eskes, 1 Feb. 2021 - } - - #: Variables that are computed (cannot be read directly) - AUX_REQUIRES = {"vmro3": ["conco3"], "vmrno2": ["concno2"]} - - #: functions used to convert variables that are computed - AUX_FUNS = {"vmro3": _conc_to_vmr_marcopolo_stats, "vmrno2": _conc_to_vmr_marcopolo_stats} - - PROVIDES_VARIABLES = list(VAR_MAP) - DEFAULT_VARS = ["concpm10", "concpm25", "concno2", "conco3"] - - TS_TYPE = "hourly" - - STAT_METADATA_FILENAME = "station_info.xlsx" - - def __init__(self, data_id=None, data_dir=None): - super().__init__(data_id=data_id, data_dir=data_dir) - - try: - import openpyxl - except ImportError: - raise ImportError( - "ReadMarcoPolo class needs package openpyxl " - "which is not part of the pyaerocom standard installation. " - "Please install openpyxl via conda or pip." - ) - - def _read_metadata_file(self): - fn = os.path.join(self.data_dir, self.STAT_METADATA_FILENAME) - cfg = pd.read_excel(fn, engine="openpyxl") - return cfg - - def _init_station_metadata(self): - """ - Initiate metadata for all stations - - Returns - ------- - dict - dictionary with metadata dictionaries for all stations - - """ - - cfg = self._read_metadata_file() - - meta_map = self.STATION_META_MAP - dtypes = self.STATION_META_DTYPES - - cols = list(cfg.columns.values) - col_idx = {} - for from_meta, to_meta in meta_map.items(): - col_idx[to_meta] = cols.index(from_meta) - - arr = cfg.values - - stats = {} - for row in arr: - stat = {} - for meta_key, col_num in col_idx.items(): - stat[meta_key] = dtypes[meta_key](row[col_num]) - stat["data_id"] = self.data_id - stat["ts_type"] = self.TS_TYPE - stats[stat["station_id"]] = stat - - return stats - - def _read_file(self, file): - df = pd.read_csv(file, sep=self.FILE_COL_DELIM) - return df - - def _make_station_data( - self, var, var_stats_unique, stat_ids, var_stats, var_dtime, var_values, stat_meta - ): - - units = self.VAR_UNITS - stats = [] - for stat_id in var_stats_unique: - if not stat_id in stat_ids: - continue - - statmask = var_stats == stat_id - - if statmask.sum() == 0: - continue - - timestamps = var_dtime[statmask] - vals = var_values[statmask] - - stat = {} - stat.update(stat_meta[stat_id]) - - stat["dtime"] = timestamps - stat["timezone"] = "unknown" - stat[var] = vals - unit = units[var] - stat["var_info"] = {} - stat["var_info"][var] = dict(units=unit) - stats.append(stat) - return stats - - def _read_files(self, files, vars_to_retrieve): - - filecols = self.FILE_COLS - - stat_meta = self._init_station_metadata() - - # get all station IDs found in the metadata file - stat_ids = list(stat_meta) - - arrs = [] - - varcol = filecols["var_name"] - for i in tqdm(range(len(files))): - filedata = self._read_file(files[i]) - arr = filedata.values - - for i, var in enumerate(vars_to_retrieve): - filevar = self.VAR_MAP[var] - if i == 0: - mask = arr[:, varcol] == filevar - else: - mask = np.logical_or(mask, arr[:, varcol] == filevar) - matches = mask.sum() - if matches: - vardata = arr[mask] - arrs.append(vardata) - if len(arrs) == 0: - raise DataRetrievalError("None of the input variables could be found in input list") - data = np.concatenate(arrs) - - dtime = data[:, filecols["datetime"]].astype("datetime64[s]") - - varis = data[:, varcol].astype(str) - values = data[:, filecols["value"]].astype(np.float64) - statids = data[:, filecols["station_id"]].astype(str) - - stats = [] - for var in vars_to_retrieve: - var_in_file = self.VAR_MAP[var] - - varmask = varis == var_in_file - - var_values = values[varmask] - var_stats = statids[varmask] - var_dtime = dtime[varmask] - - var_stats_unique = np.unique(var_stats) - - var_stats = self._make_station_data( - var, var_stats_unique, stat_ids, var_stats, var_dtime, var_values, stat_meta - ) - stats.extend(var_stats) - - return stats - - def read_file(self): - """ - This method is not implemented (but needs to be declared for template) - - Raises - ------ - NotImplementedError - """ - raise NotImplementedError("Not needed for these data since the format is unsuitable...") - - def compute_additional_vars(self, statlist_from_file, vars_to_compute): - - for var in vars_to_compute: - fun = self.AUX_FUNS[var] - requires = self.AUX_REQUIRES[var] - # this will add the variable data to each station data in - # statlist_from_file - statlist_from_file = fun(statlist_from_file, var, *requires) - - return statlist_from_file - - def read(self, vars_to_retrieve=None, first_file=None, last_file=None): - """ - Read variable data - - Parameters - ---------- - vars_to_retrieve : str or list, optional - List of variables to be retrieved. The default is None. - first_file : int, optional - Index of first file to be read. The default is None, in which case - index 0 in file list is used. - last_file : int, optional - Index of last file to be read. The default is None, in which case - last index in file list is used. - - Returns - ------- - data : UngriddedData - loaded data object. - - """ - if vars_to_retrieve is None: - vars_to_retrieve = self.DEFAULT_VARS - elif isinstance(vars_to_retrieve, str): - vars_to_retrieve = [vars_to_retrieve] - - # make sure to use AeroCom variable names in output data - vars_to_retrieve = varlist_aerocom(vars_to_retrieve) - - vars_to_read, vars_to_compute = self.check_vars_to_retrieve(vars_to_retrieve) - - files = self.get_file_list() - if first_file is None: - first_file = 0 - if last_file is None: - last_file = len(files) - files = files[first_file:last_file] - - stats = self._read_files(files, vars_to_read) - - stats = self.compute_additional_vars(stats, vars_to_compute) - - data = UngriddedData.from_station_data(stats) - - return data diff --git a/pyaerocom/io/readungridded.py b/pyaerocom/io/readungridded.py index 52084d5e0..9c8abc70a 100755 --- a/pyaerocom/io/readungridded.py +++ b/pyaerocom/io/readungridded.py @@ -26,7 +26,6 @@ from pyaerocom.io.read_ebas import ReadEbas from pyaerocom.io.read_eea_aqerep import ReadEEAAQEREP from pyaerocom.io.read_eea_aqerep_v2 import ReadEEAAQEREP_V2 -from pyaerocom.io.read_marcopolo import ReadMarcoPolo from pyaerocom.ungriddeddata import UngriddedData from pyaerocom.variable import get_aliases @@ -58,7 +57,6 @@ class ReadUngridded: ReadEbas, ReadAasEtal, ReadAirNow, - ReadMarcoPolo, ReadEEAAQEREP, ReadEEAAQEREP_V2, ] diff --git a/pyaerocom/plugins/mep/reader.py b/pyaerocom/plugins/mep/reader.py index f81c8bf55..9dc63ed43 100644 --- a/pyaerocom/plugins/mep/reader.py +++ b/pyaerocom/plugins/mep/reader.py @@ -3,9 +3,9 @@ import logging import re from collections import defaultdict +from collections.abc import Iterable from functools import cached_property, lru_cache from pathlib import Path -from typing import Iterable import xarray as xr @@ -115,7 +115,7 @@ def FOUND_FILES(self) -> tuple[Path, ...]: logger.debug(f"found {len(paths)} files") return tuple(paths) - @lru_cache() + @lru_cache def stations(self, files: tuple[str | Path, ...] | None = None) -> dict[str, list[Path]]: if not files: files = self.FOUND_FILES diff --git a/pyaerocom/region.py b/pyaerocom/region.py index d65d49a4b..d40021c45 100644 --- a/pyaerocom/region.py +++ b/pyaerocom/region.py @@ -1,7 +1,7 @@ """ This module contains functionality related to regions in pyaerocom """ -from typing import List, Optional +from __future__ import annotations import numpy as np @@ -48,7 +48,6 @@ class Region(BrowseDict): """ def __init__(self, region_id=None, **kwargs): - if region_id is None: region_id = ALL_REGION_NAME @@ -174,7 +173,6 @@ def get_mask_data(self): return self._mask_data def plot_mask(self, ax, color, alpha=0.2): - mask = self.get_mask_data() # import numpy as np data = mask.data @@ -368,8 +366,8 @@ def get_regions_coord(lat, lon, regions=None): def find_closest_region_coord( lat: float, lon: float, - regions: Optional[dict] = None, -) -> List[str]: + regions: dict | None = None, +) -> list[str]: """Finds list of regions sorted by their center closest to input coordinate Parameters diff --git a/pyaerocom/scripts/testdata-minimal/calc_example_coldata.py b/pyaerocom/scripts/testdata-minimal/calc_example_coldata.py index a5090544a..3a71d46d0 100755 --- a/pyaerocom/scripts/testdata-minimal/calc_example_coldata.py +++ b/pyaerocom/scripts/testdata-minimal/calc_example_coldata.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- import matplotlib.pyplot as plt diff --git a/pyaerocom/scripts/testdata-minimal/create_subset_ebas.py b/pyaerocom/scripts/testdata-minimal/create_subset_ebas.py index 2a99bb6fc..ca0eca422 100755 --- a/pyaerocom/scripts/testdata-minimal/create_subset_ebas.py +++ b/pyaerocom/scripts/testdata-minimal/create_subset_ebas.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """simple script to generate a small enough test data set for the EBAS obs network Works only if the user has access to the standard EBAS data path at Met Norway @@ -73,7 +72,7 @@ def check_outdated(filedir): files_invalid = [] files_valid = [] - with open(JSON_FILE, "r") as f: + with open(JSON_FILE) as f: data = simplejson.load(f, allow_nan=True) @@ -212,7 +211,7 @@ def main(): else: infofile = JSON_FILE if os.path.exists(infofile): - with open(infofile, "r") as f: + with open(infofile) as f: current_files = simplejson.load(f, allow_nan=True) else: current_files = {} diff --git a/pyaerocom/scripts/testdata-minimal/create_subsets_aeronet.py b/pyaerocom/scripts/testdata-minimal/create_subsets_aeronet.py index e3f9ab38c..6c5b400a0 100755 --- a/pyaerocom/scripts/testdata-minimal/create_subsets_aeronet.py +++ b/pyaerocom/scripts/testdata-minimal/create_subsets_aeronet.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Goal """ @@ -87,7 +86,7 @@ stats_ok.append(statname) if len(stats_ok) == 0: raise Exception - print("Found {} common sites for filter {}: {}".format(len(stats_ok), attr, val)) + print(f"Found {len(stats_ok)} common sites for filter {attr}: {val}") use_stats.extend(stats_ok) @@ -98,7 +97,7 @@ outdir = OUTBASE.joinpath(data_id) # make sure to remove old data if outdir.exists(): - print("REMOVING EXISTING DATA FOR {}".format(data_id)) + print(f"REMOVING EXISTING DATA FOR {data_id}") shutil.rmtree(outdir) outdir.mkdir() diff --git a/pyaerocom/scripts/testdata-minimal/create_subsets_ghost.py b/pyaerocom/scripts/testdata-minimal/create_subsets_ghost.py index 46d3e77a1..e70270c49 100644 --- a/pyaerocom/scripts/testdata-minimal/create_subsets_ghost.py +++ b/pyaerocom/scripts/testdata-minimal/create_subsets_ghost.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Create minimal testdataset for GHOST reader diff --git a/pyaerocom_env.yml b/pyaerocom_env.yml index 302999ca9..0e9b3a15a 100644 --- a/pyaerocom_env.yml +++ b/pyaerocom_env.yml @@ -3,7 +3,7 @@ channels: - conda-forge dependencies: - - iris >=3.2.0 + - iris >=3.2.0,!=3.5.0 - xarray >=0.16.0,<2022.11.0 - cartopy >=0.20.0 - matplotlib-base >=3.0.1 diff --git a/pyproject.toml b/pyproject.toml index ed5885796..286f5a1c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,9 +9,9 @@ authors = [{ name = "MET Norway" }] description = "pyaerocom model evaluation software" classifiers = [ "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)", "Operating System :: OS Independent", "Development Status :: 3 - Alpha", @@ -19,7 +19,7 @@ classifiers = [ "Intended Audience :: Education", "Topic :: Scientific/Engineering :: Atmospheric Science", ] -requires-python = ">=3.8" +requires-python = ">=3.9" dependencies = [ 'importlib-metadata>=3.6; python_version < "3.10"', 'importlib-resources>=5.10; python_version < "3.11"', @@ -62,7 +62,7 @@ proj-legacy = [ # cartopy 0.19 incompatible with matplotlib 2.0 "shapely<2.0.0", ] -proj8 = ["cartopy>=0.20", "scitools-iris>=3.2", "matplotlib>=3.6.0"] +proj8 = ["cartopy>=0.20", "scitools-iris>=3.2,!=3.5.0", "matplotlib>=3.6.0"] docs = [ "sphinx>=4.2.0", "sphinxcontrib-napoleon", @@ -147,14 +147,14 @@ extend_skip = ["pyaerocom-tutorials"] line-length = 99 [tool.isort] -py_version = "38" +py_version = "39" profile = "black" src_paths = ["pyaerocom", "tests"] extend_skip = ["pyaerocom-tutorials"] line_length = 99 [tool.mypy] -python_version = "3.8" +python_version = "3.9" warn_unused_configs = true warn_unused_ignores = true warn_no_return = true @@ -195,7 +195,7 @@ max-line-length = 99 [tool.tox] legacy_tox_ini = """ [tox] -envlist = py38, py39, py310, py311, format, lint, docs +envlist = py39, py310, py311, format, lint, docs skip_missing_interpreters = True isolated_build = True requires = @@ -224,13 +224,13 @@ ignore_outcome = True commands = mypy pyaerocom/ extras = - proj-legacy + proj8 lint [testenv:docs] commands = sphinx-build {posargs:-T} docs/ docs/_build/html extras = - proj-legacy + proj8 docs """ diff --git a/tests/aeroval/test_aux_io_helpers.py b/tests/aeroval/test_aux_io_helpers.py index a03c2d1f1..cbe57e85e 100644 --- a/tests/aeroval/test_aux_io_helpers.py +++ b/tests/aeroval/test_aux_io_helpers.py @@ -1,6 +1,5 @@ from pathlib import Path from textwrap import dedent -from typing import List from pytest import mark, param, raises @@ -46,7 +45,7 @@ def test_ReadAuxHandler_empty(tmp_path: Path): param("func", ["last"], {"func": lambda: True}, id="func as last arg"), ], ) -def test_check_aux_info(fun, vars_required: List[str], funcs: dict): +def test_check_aux_info(fun, vars_required: list[str], funcs: dict): aux = check_aux_info(fun, vars_required, funcs) assert aux["fun"] is funcs.get(fun, fun) assert aux["vars_required"] == vars_required @@ -65,7 +64,7 @@ def test_check_aux_info(fun, vars_required: List[str], funcs: dict): ), ], ) -def test_check_aux_info_error(fun, vars_required: List[str], funcs: dict, error: str): +def test_check_aux_info_error(fun, vars_required: list[str], funcs: dict, error: str): with raises(ValueError) as e: check_aux_info(fun, vars_required, funcs) diff --git a/tests/aeroval/test_coldatatojson_helpers2.py b/tests/aeroval/test_coldatatojson_helpers2.py index 79d6b1fa5..563dc0154 100644 --- a/tests/aeroval/test_coldatatojson_helpers2.py +++ b/tests/aeroval/test_coldatatojson_helpers2.py @@ -1,8 +1,6 @@ # ToDo: merge with test_coldatatojson_helpers.py from __future__ import annotations -from typing import Type - import numpy as np import pytest import xarray @@ -145,7 +143,7 @@ def test__process_statistics_timeseries_error( freq: str, region_ids: dict[str, str], data_freq: str, - exception: Type[Exception], + exception: type[Exception], error: str, ): with pytest.raises(exception) as e: @@ -221,7 +219,7 @@ def test__make_trends_error( freq: str, season: str, min_yrs: int, - exception: Type[Exception], + exception: type[Exception], error: str, ): obs_val = coldata.data.data[0, :, 0] diff --git a/tests/aeroval/test_experiment_output.py b/tests/aeroval/test_experiment_output.py index fbc545fde..9cf917611 100644 --- a/tests/aeroval/test_experiment_output.py +++ b/tests/aeroval/test_experiment_output.py @@ -1,7 +1,6 @@ from __future__ import annotations from pathlib import Path -from typing import Type import pytest @@ -45,7 +44,7 @@ def test_ProjectOutput(proj_id: str, json_basedir: str): ), ], ) -def test_ProjectOutput_error(proj_id, json_basedir, exception: Type[Exception], error: str): +def test_ProjectOutput_error(proj_id, json_basedir, exception: type[Exception], error: str): with pytest.raises(exception) as e: ProjectOutput(proj_id, json_basedir) assert str(e.value) == error @@ -310,7 +309,6 @@ def test_Experiment_Output_clean_json_files_CFG1_INVALIDOBS(eval_config: dict): def test_ExperimentOutput_reorder_experiments( dummy_expout: ExperimentOutput, add_names, order, result ): - path = Path(dummy_expout.experiments_file) data = dict().fromkeys(add_names, dict(public=True)) diff --git a/tests/aeroval/test_helpers.py b/tests/aeroval/test_helpers.py index af2bdee45..032648c68 100644 --- a/tests/aeroval/test_helpers.py +++ b/tests/aeroval/test_helpers.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import, annotations +from __future__ import annotations import pytest diff --git a/tests/aeroval/test_setup.py b/tests/aeroval/test_setup.py index 43c386178..5a2846c4d 100644 --- a/tests/aeroval/test_setup.py +++ b/tests/aeroval/test_setup.py @@ -1,8 +1,8 @@ from __future__ import annotations import os +from collections.abc import Iterable from pathlib import Path -from typing import Iterable import pytest diff --git a/tests/fixtures/data_access.py b/tests/fixtures/data_access.py index b7c6df9fe..35ad13816 100644 --- a/tests/fixtures/data_access.py +++ b/tests/fixtures/data_access.py @@ -8,7 +8,7 @@ import logging import tarfile from pathlib import Path -from typing import NamedTuple, Type +from typing import NamedTuple import requests @@ -31,7 +31,7 @@ class DataForTests(NamedTuple): relpath: str - reader: Type[ReadUngriddedBase] | None = None + reader: type[ReadUngriddedBase] | None = None @property def path(self) -> Path: diff --git a/tests/io/test_ebas_nasa_ames.py b/tests/io/test_ebas_nasa_ames.py index 504cba162..4fa3b0ed1 100644 --- a/tests/io/test_ebas_nasa_ames.py +++ b/tests/io/test_ebas_nasa_ames.py @@ -1,3 +1,5 @@ +import sys + import numpy as np import pytest @@ -80,7 +82,10 @@ def test_EbasNasaAmesFile_head_fix(filedata): def test_EbasNasaAmesFile_head_fix_error(filedata): with pytest.raises(AttributeError) as e: filedata.head_fix = "Blaaaaaaaaaaaaaaa" - assert str(e.value).startswith("can't set attribute") + if sys.version_info < (3, 11): + assert str(e.value).startswith("can't set attribute") + else: + assert str(e.value).endswith("object has no setter") def test_EbasNasaAmesFile_data(filedata): diff --git a/tests/io/test_ebas_varinfo.py b/tests/io/test_ebas_varinfo.py index 2845e7c69..b1fe67cce 100644 --- a/tests/io/test_ebas_varinfo.py +++ b/tests/io/test_ebas_varinfo.py @@ -1,7 +1,6 @@ from __future__ import annotations from configparser import ConfigParser -from typing import Type import pytest @@ -261,7 +260,7 @@ def test_make_sql_request(info: EbasVarInfo, constraints: dict): ), ], ) -def test_make_sql_request_error(info: EbasVarInfo, exception: Type[Exception], error: str): +def test_make_sql_request_error(info: EbasVarInfo, exception: type[Exception], error: str): with pytest.raises(exception) as e: info.make_sql_request() assert str(e.value).startswith(error) diff --git a/tests/io/test_iris_io.py b/tests/io/test_iris_io.py index 716a3888d..3c7a6bbb4 100644 --- a/tests/io/test_iris_io.py +++ b/tests/io/test_iris_io.py @@ -1,7 +1,6 @@ from __future__ import annotations from pathlib import Path -from typing import Type import numpy as np import pytest @@ -140,7 +139,7 @@ def test_check_time_coord(cube: Cube): ], ) def test_check_time_coord_error( - cube: Cube, ts_type: str, year: int, exception: Type[Exception], error: str + cube: Cube, ts_type: str, year: int, exception: type[Exception], error: str ): with pytest.raises(exception) as e: iris_io.check_time_coord(cube, ts_type, year) @@ -250,7 +249,7 @@ def test_load_cube_custom(file_path: Path, var_name: str | None): ], ) def test_load_cube_custom_error( - file_path: Path, var_name: str | None, exception: Type[Exception], error: str + file_path: Path, var_name: str | None, exception: type[Exception], error: str ): with pytest.raises(exception) as e: iris_io.load_cube_custom(file_path, var_name) diff --git a/tests/io/test_read_aeronet_invv3.py b/tests/io/test_read_aeronet_invv3.py index f059e7c67..a6eac0e23 100644 --- a/tests/io/test_read_aeronet_invv3.py +++ b/tests/io/test_read_aeronet_invv3.py @@ -11,9 +11,9 @@ def test_load_berlin(): dataset = ReadAeronetInvV3() files = dataset.find_in_file_list("*Berlin*") - assert len(files) == 1 - assert Path(files[0]).name == "19930101_20230325_Berlin_FUB.all" - data = dataset.read_file(files[0], vars_to_retrieve=["abs550aer"]) + assert len(files) == 2 + assert Path(files[1]).name == "19930101_20230506_Berlin_FUB.all" + data = dataset.read_file(files[1], vars_to_retrieve=["abs550aer"]) test_vars = ["abs440aer", "angabs4487aer", "abs550aer"] assert all(x in data for x in test_vars) diff --git a/tests/io/test_read_ebas.py b/tests/io/test_read_ebas.py index 1e7869546..3b07f0cd7 100644 --- a/tests/io/test_read_ebas.py +++ b/tests/io/test_read_ebas.py @@ -1,7 +1,6 @@ from __future__ import annotations from pathlib import Path -from typing import Type import numpy as np import pytest @@ -382,7 +381,7 @@ def test__find_station_matches(reader: ReadEbas): ], ) def test__find_station_matches_error( - reader: ReadEbas, val, exception: Type[Exception], error: str + reader: ReadEbas, val, exception: type[Exception], error: str ): with pytest.raises(exception) as e: reader._find_station_matches(val) @@ -429,7 +428,7 @@ def test_get_ebas_var(reader: ReadEbas): ], ) def test_get_ebas_var_error( - reader: ReadEbas, var_name: str, exception: Type[Exception], error: str + reader: ReadEbas, var_name: str, exception: type[Exception], error: str ): with pytest.raises(exception) as e: reader.get_ebas_var(var_name) @@ -573,7 +572,7 @@ def test_read_file_error( reader: ReadEbas, ebas_issue_files: Path, vars_to_retrieve: str, - exception: Type[Exception], + exception: type[Exception], error: str, ): with pytest.raises(exception) as e: diff --git a/tests/plot/test_mapping.py b/tests/plot/test_mapping.py index 6881133e0..b63b9199d 100644 --- a/tests/plot/test_mapping.py +++ b/tests/plot/test_mapping.py @@ -1,7 +1,5 @@ from __future__ import annotations -from typing import Type - import cartopy.mpl.geoaxes import numpy as np import pytest @@ -81,7 +79,6 @@ def test_init_map(kwargs: dict): ], ) def test_init_map_error(kwargs: dict, error: str): - with pytest.raises(ValueError) as e: init_map(**kwargs) assert str(e.value) == error @@ -199,7 +196,7 @@ class Fake4D(GriddedData): ) @pytest.mark.filterwarnings("ignore:More than 20 figures have been opened:RuntimeWarning") def test_plot_griddeddata_on_map_error( - gridded_data: GriddedData, kwargs: dict, exception: Type[Exception], error: str + gridded_data: GriddedData, kwargs: dict, exception: type[Exception], error: str ): with pytest.raises(exception) as e: plot_griddeddata_on_map(gridded_data, **kwargs) @@ -255,7 +252,7 @@ def test_plot_nmb_map_colocateddata4D(coldata_tm5_tm5: ColocatedData): ], ) def test_plot_nmb_map_colocateddataFAIL( - coldata: ColocatedData, exception: Type[Exception], error: str + coldata: ColocatedData, exception: type[Exception], error: str ): with pytest.raises(exception) as e: plot_nmb_map_colocateddata(coldata) diff --git a/tests/plugins/mscw_ctm/test_reader.py b/tests/plugins/mscw_ctm/test_reader.py index 113ad80e2..3f7158f5c 100644 --- a/tests/plugins/mscw_ctm/test_reader.py +++ b/tests/plugins/mscw_ctm/test_reader.py @@ -2,7 +2,6 @@ import re from pathlib import Path -from typing import Type import cf_units import pytest @@ -220,7 +219,7 @@ def test_ReadMscwCtm_read_var(var_name: str, ts_type: str, data_dir: str): ], ) def test_ReadMscwCtm_read_var_error( - var_name: str, ts_type: str, exception: Type[Exception], error: str, data_dir: str + var_name: str, ts_type: str, exception: type[Exception], error: str, data_dir: str ): reader = ReadMscwCtm(data_dir=data_dir) with pytest.raises(exception) as e: diff --git a/tests/test_colocateddata.py b/tests/test_colocateddata.py index 7fd51761b..e0030630e 100644 --- a/tests/test_colocateddata.py +++ b/tests/test_colocateddata.py @@ -1,7 +1,6 @@ from __future__ import annotations from pathlib import Path -from typing import Type import numpy as np import pytest @@ -30,7 +29,7 @@ def test_ColocatedData__init__(data: Path | str | ArrayLike): ({}, ValueError), ], ) -def test_ColocatedData__init___error(data, exception: Type[Exception]): +def test_ColocatedData__init___error(data, exception: type[Exception]): with pytest.raises(exception): ColocatedData(data=data).data @@ -289,7 +288,7 @@ def test_ColocatedData_calc_temporal_statistics( ], ) def test_ColocatedData_calc_temporal_statistics_error( - coldata: ColocatedData, aggr: str | None, exception: Type[Exception], error: str + coldata: ColocatedData, aggr: str | None, exception: type[Exception], error: str ): with pytest.raises(exception) as e: coldata.calc_temporal_statistics(aggr=aggr) @@ -323,7 +322,7 @@ def test_ColocatedData_calc_spatial_statistics( ], ) def test_ColocatedData_calc_spatial_statistics_error( - coldata: ColocatedData, aggr: str | None, exception: Type[Exception], error: str + coldata: ColocatedData, aggr: str | None, exception: type[Exception], error: str ): with pytest.raises(exception) as e: coldata.calc_spatial_statistics(aggr=aggr) diff --git a/tests/test_config.py b/tests/test_config.py index 491b4c83f..add0e1fcf 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -2,7 +2,6 @@ import getpass from pathlib import Path -from typing import Type import pytest @@ -85,7 +84,7 @@ def test_Config___init__(config_file: str, try_infer_environment: bool): ), ], ) -def test_Config___init___error(config_file: str, exception: Type[Exception], error: str): +def test_Config___init___error(config_file: str, exception: type[Exception], error: str): with pytest.raises(exception) as e: testmod.Config(config_file, False) assert str(e.value) == error diff --git a/tests/test_stationdata.py b/tests/test_stationdata.py index be82a41dd..cc216321f 100644 --- a/tests/test_stationdata.py +++ b/tests/test_stationdata.py @@ -1,7 +1,5 @@ from __future__ import annotations -from typing import Type - import numpy as np import pandas as pd import pytest @@ -138,7 +136,7 @@ def test_StationData_check_var_unit_aerocom(): ], ) def test_StationData_check_var_unit_aerocom_error( - stat: StationData, var_name: str, exception: Type[Exception], error: str + stat: StationData, var_name: str, exception: type[Exception], error: str ): with pytest.raises(exception) as e: stat.check_var_unit_aerocom(var_name) @@ -233,7 +231,7 @@ def test_StationData_get_station_coords( ], ) def test_StationData_get_station_coords_error( - stat: StationData, exception: Type[Exception], error: str + stat: StationData, exception: type[Exception], error: str ): with pytest.raises(exception) as e: stat.get_station_coords(force_single_value=True) @@ -416,7 +414,7 @@ def test_StationData_select_altitude_Series(): ], ) def test_StationData_select_altitude_Series_error( - stat: StationData, altitudes: tuple[int, int], exception: Type[Exception], error: str + stat: StationData, altitudes: tuple[int, int], exception: type[Exception], error: str ): with pytest.raises(exception) as e: stat.select_altitude("od550aer", altitudes) diff --git a/tests/test_vertical_profile.py b/tests/test_vertical_profile.py index 72cbac4cf..07e7a060c 100644 --- a/tests/test_vertical_profile.py +++ b/tests/test_vertical_profile.py @@ -1,7 +1,5 @@ from __future__ import annotations -from typing import Type - import pytest from pyaerocom.vertical_profile import VerticalProfile @@ -45,7 +43,7 @@ def vertical_profile() -> VerticalProfile: ), ], ) -def test_VerticalProfile_error(kwargs: dict, exception: Type[Exception], error: str): +def test_VerticalProfile_error(kwargs: dict, exception: type[Exception], error: str): with pytest.raises(exception) as e: VerticalProfile(**kwargs) assert str(e.value).endswith(error)