From 8f62fb185c78351680f42bd039e73840baddd713 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?thorbjoernl=20=28Thorbj=C3=B8rn=29?= <51087536+thorbjoernl@users.noreply.github.com> Date: Wed, 25 Sep 2024 14:20:54 +0000 Subject: [PATCH 1/3] bump aerovaldb --- pyaerocom_env.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyaerocom_env.yml b/pyaerocom_env.yml index e1915bf86..555aa53f1 100644 --- a/pyaerocom_env.yml +++ b/pyaerocom_env.yml @@ -32,7 +32,7 @@ dependencies: - geojsoncontour - geocoder_reverse_natural_earth >= 0.0.2 - pyaro >= 0.0.10 - - aerovaldb == 0.0.15 + - aerovaldb == 0.1.0rc2 ## testing - pytest >=7.4 - pytest-dependency diff --git a/pyproject.toml b/pyproject.toml index 8dd43c809..0d110a658 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ classifiers = [ ] requires-python = ">=3.10" dependencies = [ - "aerovaldb==0.0.17", + "aerovaldb==0.1.0rc2", "scitools-iris>=3.8.1", "xarray>=2022.12.0", "cartopy>=0.21.1", From e2b0c5e9a00c1df27a70e5e62aed57eeec589196 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?thorbjoernl=20=28Thorbj=C3=B8rn=29?= <51087536+thorbjoernl@users.noreply.github.com> Date: Thu, 26 Sep 2024 07:09:48 +0000 Subject: [PATCH 2/3] Bump aerovaldb --- pyaerocom/aeroval/experiment_output.py | 15 ++++++++++----- pyaerocom_env.yml | 2 +- pyproject.toml | 2 +- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/pyaerocom/aeroval/experiment_output.py b/pyaerocom/aeroval/experiment_output.py index 3ebba44e3..0c611428e 100644 --- a/pyaerocom/aeroval/experiment_output.py +++ b/pyaerocom/aeroval/experiment_output.py @@ -229,8 +229,10 @@ def _sync_heatmaps_with_menu_and_regions(self) -> None: with self.avdb.lock(): menu = self.avdb.get_menu(self.proj_id, self.exp_id, default={}) all_regions = self.avdb.get_regions(self.proj_id, self.exp_id, default={}) - for fp in self.avdb.list_glob_stats(self.proj_id, self.exp_id): - data = self.avdb.get_by_uri(fp) + for uri in self.avdb.list_glob_stats( + self.proj_id, self.exp_id, access_type=aerovaldb.AccessType.URI + ): + data = self.avdb.get_by_uri(uri) hm = {} for vardisp, info in menu.items(): obs_dict = info["obs"] @@ -250,7 +252,7 @@ def _sync_heatmaps_with_menu_and_regions(self) -> None: hm_data = self._check_hm_all_regions_avail(all_regions, hm_data) hm[vardisp][obs][vert_code][mod][modvar] = hm_data - self.avdb.put_by_uri(hm, fp) + self.avdb.put_by_uri(hm, uri) def _check_hm_all_regions_avail(self, all_regions, hm_data) -> dict: if all([x in hm_data for x in all_regions]): @@ -413,7 +415,10 @@ def _check_clean_ts_file(self, fp) -> bool: with self.avdb.lock(): try: - data = self.avdb.get_by_uri(fp) + # TODO: Hack to get uri. Ideally this should be rewritten to use URIs directly further + # up. + uri = self.avdb._get_uri_for_file(fp) + data = self.avdb.get_by_uri(uri) except Exception: logger.exception(f"FATAL: detected corrupt json file: {fp}. Removing file...") os.remove(fp) @@ -433,7 +438,7 @@ def _check_clean_ts_file(self, fp) -> bool: modified = True logger.info(f"Removing data for model {mod_name} from ts file: {fp}") - self.avdb.put_by_uri(data_new, fp) + self.avdb.put_by_uri(data_new, uri) return modified def _clean_modelmap_files(self) -> list[str]: diff --git a/pyaerocom_env.yml b/pyaerocom_env.yml index 555aa53f1..7f810a0a1 100644 --- a/pyaerocom_env.yml +++ b/pyaerocom_env.yml @@ -32,7 +32,7 @@ dependencies: - geojsoncontour - geocoder_reverse_natural_earth >= 0.0.2 - pyaro >= 0.0.10 - - aerovaldb == 0.1.0rc2 + - aerovaldb == 0.1.0 ## testing - pytest >=7.4 - pytest-dependency diff --git a/pyproject.toml b/pyproject.toml index 0d110a658..b2123cee0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ classifiers = [ ] requires-python = ">=3.10" dependencies = [ - "aerovaldb==0.1.0rc2", + "aerovaldb==0.1.0", "scitools-iris>=3.8.1", "xarray>=2022.12.0", "cartopy>=0.21.1", From 1396c0a67a4c051ca2bbe21fe26914818ac32d60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?thorbjoernl=20=28Thorbj=C3=B8rn=29?= <51087536+thorbjoernl@users.noreply.github.com> Date: Thu, 26 Sep 2024 07:12:41 +0000 Subject: [PATCH 3/3] use >= instead of == --- pyaerocom_env.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyaerocom_env.yml b/pyaerocom_env.yml index 7f810a0a1..cf7c645e2 100644 --- a/pyaerocom_env.yml +++ b/pyaerocom_env.yml @@ -32,7 +32,7 @@ dependencies: - geojsoncontour - geocoder_reverse_natural_earth >= 0.0.2 - pyaro >= 0.0.10 - - aerovaldb == 0.1.0 + - aerovaldb >= 0.1.0 ## testing - pytest >=7.4 - pytest-dependency diff --git a/pyproject.toml b/pyproject.toml index b2123cee0..45208f4a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ classifiers = [ ] requires-python = ">=3.10" dependencies = [ - "aerovaldb==0.1.0", + "aerovaldb>=0.1.0", "scitools-iris>=3.8.1", "xarray>=2022.12.0", "cartopy>=0.21.1",