From b5c72e321c851d5eef22c49f2186483b0686c2c4 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Fri, 3 Nov 2023 14:04:16 +0100 Subject: [PATCH 01/21] Add `to_ixmp4()` method --- pyam/core.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pyam/core.py b/pyam/core.py index 7c61e41a0..d4ad85ab9 100755 --- a/pyam/core.py +++ b/pyam/core.py @@ -11,6 +11,8 @@ from pathlib import Path from tempfile import TemporaryDirectory +import ixmp4 + from pyam.slice import IamSlice from pyam.filter import filter_by_time_domain, filter_by_year, filter_by_dt_arg @@ -2333,6 +2335,26 @@ def diff(self, mapping, periods=1, append=False): # append to `self` or return as `IamDataFrame` return self._finalize(_value, append=append) + def to_ixmp4(self, platform: ixmp4.Platform): + """Save all scenarios as new default runs in an ixmp4 platform database instance + + Parameters + ---------- + platform : :class:`ixmp4.Platform` or str + The ixmp4 platform database instance to which the scenario data is saved + """ + if not isinstance(platform, ixmp4.Platform): + platform = ixmp4.Platform(platform) + + for model, scenario in self.index: + _df = self.filter(model=model, scenario=scenario) + + run = platform.Run(model=model, scenario=scenario, version="new") + run.iamc.add(_df.data) + for key, value in dict(_df.meta.iloc[0]).items(): + run.meta[key] = value + run.set_as_default() + def _to_file_format(self, iamc_index): """Return a dataframe suitable for writing to a file""" df = self.timeseries(iamc_index=iamc_index).reset_index() From 53f0da9a8415a7fa4425159fff0c60eafc4ab538 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Wed, 15 Nov 2023 09:08:19 +0100 Subject: [PATCH 02/21] Move ixmp4-methods to own module --- pyam/core.py | 13 ++----------- pyam/ixmp4.py | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 11 deletions(-) create mode 100644 pyam/ixmp4.py diff --git a/pyam/core.py b/pyam/core.py index d4ad85ab9..25b8e992c 100755 --- a/pyam/core.py +++ b/pyam/core.py @@ -13,6 +13,7 @@ import ixmp4 +from pyam.ixmp4 import write_to_ixmp4 from pyam.slice import IamSlice from pyam.filter import filter_by_time_domain, filter_by_year, filter_by_dt_arg @@ -2343,17 +2344,7 @@ def to_ixmp4(self, platform: ixmp4.Platform): platform : :class:`ixmp4.Platform` or str The ixmp4 platform database instance to which the scenario data is saved """ - if not isinstance(platform, ixmp4.Platform): - platform = ixmp4.Platform(platform) - - for model, scenario in self.index: - _df = self.filter(model=model, scenario=scenario) - - run = platform.Run(model=model, scenario=scenario, version="new") - run.iamc.add(_df.data) - for key, value in dict(_df.meta.iloc[0]).items(): - run.meta[key] = value - run.set_as_default() + write_to_ixmp4(self, platform) def _to_file_format(self, iamc_index): """Return a dataframe suitable for writing to a file""" diff --git a/pyam/ixmp4.py b/pyam/ixmp4.py new file mode 100644 index 000000000..ff1252ea5 --- /dev/null +++ b/pyam/ixmp4.py @@ -0,0 +1,24 @@ +import ixmp4 + + +def write_to_ixmp4(df, platform: ixmp4.Platform): + """Save all scenarios as new default runs in an ixmp4 platform database instance + + Parameters + ---------- + df : pyam.IamDataFrame + The IamDataFrame instance with scenario data + platform : :class:`ixmp4.Platform` or str + The ixmp4 platform database instance to which the scenario data is saved + """ + if not isinstance(platform, ixmp4.Platform): + platform = ixmp4.Platform(platform) + + for model, scenario in df.index: + _df = df.filter(model=model, scenario=scenario) + + run = platform.Run(model=model, scenario=scenario, version="new") + run.iamc.add(_df.data) + for key, value in dict(_df.meta.iloc[0]).items(): + run.meta[key] = value + run.set_as_default() \ No newline at end of file From 44db2d5916382e4e9af2f881755d9cfe4ed85b8f Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Wed, 15 Nov 2023 09:25:33 +0100 Subject: [PATCH 03/21] Check if regions and units exist before writing to platform --- pyam/ixmp4.py | 15 ++++++++++++++- tests/test_ixmp4.py | 20 ++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 tests/test_ixmp4.py diff --git a/pyam/ixmp4.py b/pyam/ixmp4.py index ff1252ea5..f286ff67c 100644 --- a/pyam/ixmp4.py +++ b/pyam/ixmp4.py @@ -1,5 +1,6 @@ import ixmp4 - +from ixmp4.core.region import RegionModel +from ixmp4.core.unit import UnitModel def write_to_ixmp4(df, platform: ixmp4.Platform): """Save all scenarios as new default runs in an ixmp4 platform database instance @@ -14,6 +15,18 @@ def write_to_ixmp4(df, platform: ixmp4.Platform): if not isinstance(platform, ixmp4.Platform): platform = ixmp4.Platform(platform) + # TODO: implement a try-except to roll back changes if any error writing to platform + # depends on https://github.com/iiasa/ixmp4/issues/29 + # quickfix: ensure that units and regions exist before writing + + platform_regions = platform.regions.tabulate().name.values + if any([r not in platform_regions for r in df.region]): + raise RegionModel.NotFound() + + platform_units = platform.units.tabulate().name.values + if any([r not in platform_regions for r in df.unit]): + raise UnitModel.NotFound() + for model, scenario in df.index: _df = df.filter(model=model, scenario=scenario) diff --git a/tests/test_ixmp4.py b/tests/test_ixmp4.py new file mode 100644 index 000000000..7d9eeea0e --- /dev/null +++ b/tests/test_ixmp4.py @@ -0,0 +1,20 @@ +import pytest +from ixmp4.core import Platform +from ixmp4.data.backend import SqliteTestBackend +from ixmp4.core.region import RegionModel +from ixmp4.core.unit import UnitModel + + +def test_to_ixmp4_missing_region_raises(test_df_year): + """Writing to platform raises if region not defined""" + platform = Platform(_backend=SqliteTestBackend()) + with pytest.raises(RegionModel.NotFound): + test_df_year.to_ixmp4(platform=platform) + + +def test_to_ixmp4_missing_unit_raises(test_df_year): + """Writing to platform raises if unit not defined""" + platform = Platform(_backend=SqliteTestBackend()) + platform.regions.create(name="World", hierarchy="common") + with pytest.raises(UnitModel.NotFound): + test_df_year.to_ixmp4(platform=platform) From 4bb15dc11628271d0efa8d1cc86de9df6f610a48 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Wed, 15 Nov 2023 10:28:31 +0100 Subject: [PATCH 04/21] Add missing items and and hint to error message --- pyam/ixmp4.py | 14 ++++++++++---- tests/test_ixmp4.py | 4 ++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/pyam/ixmp4.py b/pyam/ixmp4.py index f286ff67c..9617ff7ab 100644 --- a/pyam/ixmp4.py +++ b/pyam/ixmp4.py @@ -20,12 +20,18 @@ def write_to_ixmp4(df, platform: ixmp4.Platform): # quickfix: ensure that units and regions exist before writing platform_regions = platform.regions.tabulate().name.values - if any([r not in platform_regions for r in df.region]): - raise RegionModel.NotFound() + if missing := [r for r in df.region if r not in platform_regions]: + raise RegionModel.NotFound( + ", ".join(missing) + + ". Use `Platform.regions.create()` to add the missing region(s)." + ) platform_units = platform.units.tabulate().name.values - if any([r not in platform_regions for r in df.unit]): - raise UnitModel.NotFound() + if missing := [u for u in df.unit if u not in platform_units]: + raise UnitModel.NotFound( + ", ".join(missing) + + ". Use `Platform.units.create()` to add the missing unit(s)." + ) for model, scenario in df.index: _df = df.filter(model=model, scenario=scenario) diff --git a/tests/test_ixmp4.py b/tests/test_ixmp4.py index 7d9eeea0e..b8e323031 100644 --- a/tests/test_ixmp4.py +++ b/tests/test_ixmp4.py @@ -8,7 +8,7 @@ def test_to_ixmp4_missing_region_raises(test_df_year): """Writing to platform raises if region not defined""" platform = Platform(_backend=SqliteTestBackend()) - with pytest.raises(RegionModel.NotFound): + with pytest.raises(RegionModel.NotFound, match="World. Use `Platform"): test_df_year.to_ixmp4(platform=platform) @@ -16,5 +16,5 @@ def test_to_ixmp4_missing_unit_raises(test_df_year): """Writing to platform raises if unit not defined""" platform = Platform(_backend=SqliteTestBackend()) platform.regions.create(name="World", hierarchy="common") - with pytest.raises(UnitModel.NotFound): + with pytest.raises(UnitModel.NotFound, match="EJ/yr. Use `Platform"): test_df_year.to_ixmp4(platform=platform) From 391aee6cc50517ee824355a4e0b1a2fa58ce988b Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Wed, 15 Nov 2023 10:28:39 +0100 Subject: [PATCH 05/21] Add a smoke test --- tests/test_ixmp4.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_ixmp4.py b/tests/test_ixmp4.py index b8e323031..1719c84f7 100644 --- a/tests/test_ixmp4.py +++ b/tests/test_ixmp4.py @@ -18,3 +18,11 @@ def test_to_ixmp4_missing_unit_raises(test_df_year): platform.regions.create(name="World", hierarchy="common") with pytest.raises(UnitModel.NotFound, match="EJ/yr. Use `Platform"): test_df_year.to_ixmp4(platform=platform) + + +def test_ixmp4_integration(test_df): + """Write an IamDataFrame to the platform""" + platform = Platform(_backend=SqliteTestBackend()) + platform.regions.create(name="World", hierarchy="common") + platform.units.create(name="EJ/yr") + test_df.to_ixmp4(platform=platform) From e4c9ab7b21b00de7f88ff92ac4665d35e2a4660f Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Wed, 15 Nov 2023 10:33:07 +0100 Subject: [PATCH 06/21] Merge validation of dimensions --- pyam/ixmp4.py | 24 ++++++++++-------------- tests/test_ixmp4.py | 4 ++-- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/pyam/ixmp4.py b/pyam/ixmp4.py index 9617ff7ab..9c750b0c3 100644 --- a/pyam/ixmp4.py +++ b/pyam/ixmp4.py @@ -18,20 +18,16 @@ def write_to_ixmp4(df, platform: ixmp4.Platform): # TODO: implement a try-except to roll back changes if any error writing to platform # depends on https://github.com/iiasa/ixmp4/issues/29 # quickfix: ensure that units and regions exist before writing - - platform_regions = platform.regions.tabulate().name.values - if missing := [r for r in df.region if r not in platform_regions]: - raise RegionModel.NotFound( - ", ".join(missing) + - ". Use `Platform.regions.create()` to add the missing region(s)." - ) - - platform_units = platform.units.tabulate().name.values - if missing := [u for u in df.unit if u not in platform_units]: - raise UnitModel.NotFound( - ", ".join(missing) + - ". Use `Platform.units.create()` to add the missing unit(s)." - ) + for dimension, values, model in [ + ("regions", df.region, RegionModel), + ("units", df.unit, UnitModel) + ]: + platform_values = platform.__getattribute__(dimension).tabulate().name.values + if missing := [i for i in values if i not in platform_values]: + raise model.NotFound( + ", ".join(missing) + + f". Use `Platform.{dimension}.create()` to add the missing {dimension}." + ) for model, scenario in df.index: _df = df.filter(model=model, scenario=scenario) diff --git a/tests/test_ixmp4.py b/tests/test_ixmp4.py index 1719c84f7..af9ec6e62 100644 --- a/tests/test_ixmp4.py +++ b/tests/test_ixmp4.py @@ -8,7 +8,7 @@ def test_to_ixmp4_missing_region_raises(test_df_year): """Writing to platform raises if region not defined""" platform = Platform(_backend=SqliteTestBackend()) - with pytest.raises(RegionModel.NotFound, match="World. Use `Platform"): + with pytest.raises(RegionModel.NotFound, match="World. Use `Platform.regions."): test_df_year.to_ixmp4(platform=platform) @@ -16,7 +16,7 @@ def test_to_ixmp4_missing_unit_raises(test_df_year): """Writing to platform raises if unit not defined""" platform = Platform(_backend=SqliteTestBackend()) platform.regions.create(name="World", hierarchy="common") - with pytest.raises(UnitModel.NotFound, match="EJ/yr. Use `Platform"): + with pytest.raises(UnitModel.NotFound, match="EJ/yr. Use `Platform.units."): test_df_year.to_ixmp4(platform=platform) From db8b88e99ccdc2cb3ddc7974a63c2229496beb7e Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Wed, 15 Nov 2023 10:37:06 +0100 Subject: [PATCH 07/21] Quickfix for numpy-int --- pyam/ixmp4.py | 6 +++++- tests/test_ixmp4.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pyam/ixmp4.py b/pyam/ixmp4.py index 9c750b0c3..aeafd6074 100644 --- a/pyam/ixmp4.py +++ b/pyam/ixmp4.py @@ -1,3 +1,4 @@ +import numpy as np import ixmp4 from ixmp4.core.region import RegionModel from ixmp4.core.unit import UnitModel @@ -35,5 +36,8 @@ def write_to_ixmp4(df, platform: ixmp4.Platform): run = platform.Run(model=model, scenario=scenario, version="new") run.iamc.add(_df.data) for key, value in dict(_df.meta.iloc[0]).items(): - run.meta[key] = value + if isinstance(value, np.int64): + run.meta[key] = int(value) + else: + run.meta[key] = value run.set_as_default() \ No newline at end of file diff --git a/tests/test_ixmp4.py b/tests/test_ixmp4.py index af9ec6e62..dc0dbd5f2 100644 --- a/tests/test_ixmp4.py +++ b/tests/test_ixmp4.py @@ -20,9 +20,9 @@ def test_to_ixmp4_missing_unit_raises(test_df_year): test_df_year.to_ixmp4(platform=platform) -def test_ixmp4_integration(test_df): +def test_ixmp4_integration(test_df_year): """Write an IamDataFrame to the platform""" platform = Platform(_backend=SqliteTestBackend()) platform.regions.create(name="World", hierarchy="common") platform.units.create(name="EJ/yr") - test_df.to_ixmp4(platform=platform) + test_df_year.to_ixmp4(platform=platform) From 401207e80102325dcdb452e79a892d734a51684a Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Wed, 15 Nov 2023 11:04:41 +0100 Subject: [PATCH 08/21] Make black --- pyam/ixmp4.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pyam/ixmp4.py b/pyam/ixmp4.py index aeafd6074..bc5ae1099 100644 --- a/pyam/ixmp4.py +++ b/pyam/ixmp4.py @@ -3,6 +3,7 @@ from ixmp4.core.region import RegionModel from ixmp4.core.unit import UnitModel + def write_to_ixmp4(df, platform: ixmp4.Platform): """Save all scenarios as new default runs in an ixmp4 platform database instance @@ -21,13 +22,13 @@ def write_to_ixmp4(df, platform: ixmp4.Platform): # quickfix: ensure that units and regions exist before writing for dimension, values, model in [ ("regions", df.region, RegionModel), - ("units", df.unit, UnitModel) + ("units", df.unit, UnitModel), ]: platform_values = platform.__getattribute__(dimension).tabulate().name.values if missing := [i for i in values if i not in platform_values]: raise model.NotFound( - ", ".join(missing) + - f". Use `Platform.{dimension}.create()` to add the missing {dimension}." + ", ".join(missing) + + f". Use `Platform.{dimension}.create()` to add the missing {dimension}." ) for model, scenario in df.index: @@ -40,4 +41,4 @@ def write_to_ixmp4(df, platform: ixmp4.Platform): run.meta[key] = int(value) else: run.meta[key] = value - run.set_as_default() \ No newline at end of file + run.set_as_default() From 08d9151ca6a29b293064f81fec34d102ca99c7ee Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Wed, 15 Nov 2023 11:06:02 +0100 Subject: [PATCH 09/21] Add to-do for reading from ixmp4 platform --- tests/test_ixmp4.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_ixmp4.py b/tests/test_ixmp4.py index dc0dbd5f2..7bc22f685 100644 --- a/tests/test_ixmp4.py +++ b/tests/test_ixmp4.py @@ -26,3 +26,5 @@ def test_ixmp4_integration(test_df_year): platform.regions.create(name="World", hierarchy="common") platform.units.create(name="EJ/yr") test_df_year.to_ixmp4(platform=platform) + + # TODO add test for reading data from ixmp4 platform From c363e157a7758be523425b670cd8fd31b7313b3b Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Wed, 15 Nov 2023 11:08:47 +0100 Subject: [PATCH 10/21] Add to release notes --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index ddd04e0a7..f5bc620dc 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -10,7 +10,7 @@ - [#804](https://github.com/IAMconsortium/pyam/pull/804) Support filters as direct keyword arguments for `validate()` method - [#801](https://github.com/IAMconsortium/pyam/pull/801) Support initializing with `meta` dataframe in long format -- [#796](https://github.com/IAMconsortium/pyam/pull/796) Raise explicit error message if no connection to IIASA manager service +- [#797](https://github.com/IAMconsortium/pyam/pull/797] Add `to_ixmp4()` method to write to an **ixmp4** platform - [#794](https://github.com/IAMconsortium/pyam/pull/794) Fix wrong color codes for AR6 Illustrative Pathways - [#792](https://github.com/IAMconsortium/pyam/pull/792) Support region-aggregation with weights-index >> data-index From d8b65ac92f8b03cda978a4565b57aa06d7e8551c Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Fri, 17 Nov 2023 16:16:55 +0100 Subject: [PATCH 11/21] Remove "manual" casting to simple types --- pyam/ixmp4.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pyam/ixmp4.py b/pyam/ixmp4.py index bc5ae1099..0eda6e439 100644 --- a/pyam/ixmp4.py +++ b/pyam/ixmp4.py @@ -36,9 +36,5 @@ def write_to_ixmp4(df, platform: ixmp4.Platform): run = platform.Run(model=model, scenario=scenario, version="new") run.iamc.add(_df.data) - for key, value in dict(_df.meta.iloc[0]).items(): - if isinstance(value, np.int64): - run.meta[key] = int(value) - else: - run.meta[key] = value + run.meta = dict(_df.meta.iloc[0]) run.set_as_default() From fa106b8b71aa19fffb76bcd94980df47318fae7a Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Fri, 24 Nov 2023 15:06:31 +0100 Subject: [PATCH 12/21] Clean-up --- RELEASE_NOTES.md | 1 + pyam/utils.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index f5bc620dc..670a0203f 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -11,6 +11,7 @@ - [#804](https://github.com/IAMconsortium/pyam/pull/804) Support filters as direct keyword arguments for `validate()` method - [#801](https://github.com/IAMconsortium/pyam/pull/801) Support initializing with `meta` dataframe in long format - [#797](https://github.com/IAMconsortium/pyam/pull/797] Add `to_ixmp4()` method to write to an **ixmp4** platform +- [#796](https://github.com/IAMconsortium/pyam/pull/796) Raise explicit error message if no connection to IIASA manager service - [#794](https://github.com/IAMconsortium/pyam/pull/794) Fix wrong color codes for AR6 Illustrative Pathways - [#792](https://github.com/IAMconsortium/pyam/pull/792) Support region-aggregation with weights-index >> data-index diff --git a/pyam/utils.py b/pyam/utils.py index 9b3792b7a..6bfefe616 100644 --- a/pyam/utils.py +++ b/pyam/utils.py @@ -228,7 +228,7 @@ def _intuit_column_groups(df, index, include_index=False): elif isinstance(df, pd.DataFrame): existing_cols = existing_cols.union(df.columns) - # check that there is no column in the timeseries data with reserved names + # check that there is no unnamed column in the timeseries data if None in existing_cols: raise ValueError("Unnamed column in timeseries data: None") From 483d7f749a80d7a88f829cb86d4cc5d23121e496 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Tue, 12 Dec 2023 16:57:44 +0100 Subject: [PATCH 13/21] Use new runs API --- RELEASE_NOTES.md | 2 +- pyam/ixmp4.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 670a0203f..b5976da60 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -10,7 +10,7 @@ - [#804](https://github.com/IAMconsortium/pyam/pull/804) Support filters as direct keyword arguments for `validate()` method - [#801](https://github.com/IAMconsortium/pyam/pull/801) Support initializing with `meta` dataframe in long format -- [#797](https://github.com/IAMconsortium/pyam/pull/797] Add `to_ixmp4()` method to write to an **ixmp4** platform +- [#797](https://github.com/IAMconsortium/pyam/pull/797) Add `to_ixmp4()` method to write to an **ixmp4** platform - [#796](https://github.com/IAMconsortium/pyam/pull/796) Raise explicit error message if no connection to IIASA manager service - [#794](https://github.com/IAMconsortium/pyam/pull/794) Fix wrong color codes for AR6 Illustrative Pathways - [#792](https://github.com/IAMconsortium/pyam/pull/792) Support region-aggregation with weights-index >> data-index diff --git a/pyam/ixmp4.py b/pyam/ixmp4.py index 0eda6e439..2a2d2c62d 100644 --- a/pyam/ixmp4.py +++ b/pyam/ixmp4.py @@ -34,7 +34,7 @@ def write_to_ixmp4(df, platform: ixmp4.Platform): for model, scenario in df.index: _df = df.filter(model=model, scenario=scenario) - run = platform.Run(model=model, scenario=scenario, version="new") + run = platform.runs.create(model=model, scenario=scenario) run.iamc.add(_df.data) run.meta = dict(_df.meta.iloc[0]) run.set_as_default() From 846fb3aefd2c65fafe8ba9ed5764b1116ce37555 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Fri, 5 Jan 2024 06:00:27 +0100 Subject: [PATCH 14/21] Move release notes to next-release section --- RELEASE_NOTES.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index b5976da60..ffa618f0f 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,7 @@ +# Next release + +- [#797](https://github.com/IAMconsortium/pyam/pull/797) Add `to_ixmp4()` method to write to an **ixmp4** platform + # Release v2.1.0 ## Highlights @@ -10,7 +14,6 @@ - [#804](https://github.com/IAMconsortium/pyam/pull/804) Support filters as direct keyword arguments for `validate()` method - [#801](https://github.com/IAMconsortium/pyam/pull/801) Support initializing with `meta` dataframe in long format -- [#797](https://github.com/IAMconsortium/pyam/pull/797) Add `to_ixmp4()` method to write to an **ixmp4** platform - [#796](https://github.com/IAMconsortium/pyam/pull/796) Raise explicit error message if no connection to IIASA manager service - [#794](https://github.com/IAMconsortium/pyam/pull/794) Fix wrong color codes for AR6 Illustrative Pathways - [#792](https://github.com/IAMconsortium/pyam/pull/792) Support region-aggregation with weights-index >> data-index From 7c30d978c7dbd27339aac03d5c7b668d16286f53 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Fri, 5 Jan 2024 06:01:39 +0100 Subject: [PATCH 15/21] Bump the dependency (due to the new `runs` API) --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 5c2603f0e..4fd25c099 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,7 +27,7 @@ python_requires = >=3.10, <3.12 # Please also add a section "Dependency changes" to the release notes install_requires = iam-units >= 2020.4.21 - ixmp4 >= 0.4.0 + ixmp4 >= 0.6.0 numpy >= 1.23.0, < 1.24 # requests included via ixmp4 # httpx[http2] included via ixmp4 From 4d40ba8e800ae89ae012b08facee41733e021aa3 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Tue, 9 Jan 2024 14:55:37 +0100 Subject: [PATCH 16/21] Change arg order --- pyam/core.py | 2 +- pyam/ixmp4.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyam/core.py b/pyam/core.py index 25b8e992c..7f1937538 100755 --- a/pyam/core.py +++ b/pyam/core.py @@ -2344,7 +2344,7 @@ def to_ixmp4(self, platform: ixmp4.Platform): platform : :class:`ixmp4.Platform` or str The ixmp4 platform database instance to which the scenario data is saved """ - write_to_ixmp4(self, platform) + write_to_ixmp4(platform, self) def _to_file_format(self, iamc_index): """Return a dataframe suitable for writing to a file""" diff --git a/pyam/ixmp4.py b/pyam/ixmp4.py index 2a2d2c62d..74291a532 100644 --- a/pyam/ixmp4.py +++ b/pyam/ixmp4.py @@ -9,10 +9,10 @@ def write_to_ixmp4(df, platform: ixmp4.Platform): Parameters ---------- - df : pyam.IamDataFrame - The IamDataFrame instance with scenario data platform : :class:`ixmp4.Platform` or str The ixmp4 platform database instance to which the scenario data is saved + df : pyam.IamDataFrame + The IamDataFrame instance with scenario data """ if not isinstance(platform, ixmp4.Platform): platform = ixmp4.Platform(platform) From 271476f0a9503f7e6b434ec312c000dc3222fa4b Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Tue, 9 Jan 2024 15:04:28 +0100 Subject: [PATCH 17/21] Fix the signature --- pyam/ixmp4.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyam/ixmp4.py b/pyam/ixmp4.py index 74291a532..f443f802b 100644 --- a/pyam/ixmp4.py +++ b/pyam/ixmp4.py @@ -1,10 +1,9 @@ -import numpy as np import ixmp4 from ixmp4.core.region import RegionModel from ixmp4.core.unit import UnitModel -def write_to_ixmp4(df, platform: ixmp4.Platform): +def write_to_ixmp4(platform: ixmp4.Platform, df): """Save all scenarios as new default runs in an ixmp4 platform database instance Parameters From 5cfd196c0c13e4e665e49285084f0ec4ec6d2951 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Thu, 22 Feb 2024 11:05:49 +0100 Subject: [PATCH 18/21] Allow string as platform name --- pyam/ixmp4.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyam/ixmp4.py b/pyam/ixmp4.py index f443f802b..d0d50d249 100644 --- a/pyam/ixmp4.py +++ b/pyam/ixmp4.py @@ -3,7 +3,7 @@ from ixmp4.core.unit import UnitModel -def write_to_ixmp4(platform: ixmp4.Platform, df): +def write_to_ixmp4(platform: ixmp4.Platform | str, df): """Save all scenarios as new default runs in an ixmp4 platform database instance Parameters From fae481f2bbfeccbe87ad7fb25d421af6fb03867f Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Thu, 22 Feb 2024 11:23:07 +0100 Subject: [PATCH 19/21] Add explicit `NotImplementedError` for datetime-format --- pyam/ixmp4.py | 3 +++ tests/test_ixmp4.py | 9 +++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pyam/ixmp4.py b/pyam/ixmp4.py index d0d50d249..d27a285fd 100644 --- a/pyam/ixmp4.py +++ b/pyam/ixmp4.py @@ -13,6 +13,9 @@ def write_to_ixmp4(platform: ixmp4.Platform | str, df): df : pyam.IamDataFrame The IamDataFrame instance with scenario data """ + if df.time_domain != "year": + raise NotImplementedError("Only time_domain='year' is supported for now") + if not isinstance(platform, ixmp4.Platform): platform = ixmp4.Platform(platform) diff --git a/tests/test_ixmp4.py b/tests/test_ixmp4.py index 7bc22f685..2367a4182 100644 --- a/tests/test_ixmp4.py +++ b/tests/test_ixmp4.py @@ -20,11 +20,16 @@ def test_to_ixmp4_missing_unit_raises(test_df_year): test_df_year.to_ixmp4(platform=platform) -def test_ixmp4_integration(test_df_year): +def test_ixmp4_integration(test_df): """Write an IamDataFrame to the platform""" platform = Platform(_backend=SqliteTestBackend()) platform.regions.create(name="World", hierarchy="common") platform.units.create(name="EJ/yr") - test_df_year.to_ixmp4(platform=platform) + + if test_df.time_domain == "year": + test_df.to_ixmp4(platform=platform) + else: + with pytest.raises(NotImplementedError): + test_df.to_ixmp4(platform=platform) # TODO add test for reading data from ixmp4 platform From afc357407a5ffe4e19364c9ba67539bd9355b925 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Thu, 22 Feb 2024 13:12:32 +0100 Subject: [PATCH 20/21] Use set notation per suggestion by @phackstock --- pyam/ixmp4.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyam/ixmp4.py b/pyam/ixmp4.py index d27a285fd..b07c46571 100644 --- a/pyam/ixmp4.py +++ b/pyam/ixmp4.py @@ -27,7 +27,7 @@ def write_to_ixmp4(platform: ixmp4.Platform | str, df): ("units", df.unit, UnitModel), ]: platform_values = platform.__getattribute__(dimension).tabulate().name.values - if missing := [i for i in values if i not in platform_values]: + if missing := set(values).difference(platform_values): raise model.NotFound( ", ".join(missing) + f". Use `Platform.{dimension}.create()` to add the missing {dimension}." From 0da2f88ef7c57cfb934fdd5d5d5291ad9fd6a1e3 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Thu, 22 Feb 2024 14:06:54 +0100 Subject: [PATCH 21/21] Use `getattr()` per suggestion by @phackstock --- pyam/ixmp4.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyam/ixmp4.py b/pyam/ixmp4.py index b07c46571..19a3ce8bc 100644 --- a/pyam/ixmp4.py +++ b/pyam/ixmp4.py @@ -26,7 +26,7 @@ def write_to_ixmp4(platform: ixmp4.Platform | str, df): ("regions", df.region, RegionModel), ("units", df.unit, UnitModel), ]: - platform_values = platform.__getattribute__(dimension).tabulate().name.values + platform_values = getattr(platform, dimension).tabulate().name.values if missing := set(values).difference(platform_values): raise model.NotFound( ", ".join(missing)