From c501a0af7ad687f1eb2d51bb2b07494836b41a47 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Thu, 7 Mar 2024 09:54:42 +0100 Subject: [PATCH 01/10] Add a test for `read_ixmp4()` with non-default scenarios --- tests/test_ixmp4.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_ixmp4.py b/tests/test_ixmp4.py index 312d3fa31..60eb77432 100644 --- a/tests/test_ixmp4.py +++ b/tests/test_ixmp4.py @@ -49,6 +49,11 @@ def test_ixmp4_integration(test_df_year): exp.set_meta(1, "version") # add version number added from ixmp4 pyam.assert_iamframe_equal(exp, obs) + # make one scenario a non-default scenario, make sure that it is not included + platform.runs.get("model_a", "scen_b").unset_as_default() + obs = read_ixmp4(platform=platform) + pyam.assert_iamframe_equal(exp.filter(scenario="scen_a"), obs) + # read all scenarios (runs) - version number used as additional index dimension obs = read_ixmp4(platform=platform, default_only=False) data = test_df_year.data From a4df1a106b8f143296fa552e23cb7c4030ce44f7 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Thu, 7 Mar 2024 13:35:21 +0100 Subject: [PATCH 02/10] Drop `version` column when writing to the database --- pyam/ixmp4.py | 16 +++++++++++++++- tests/test_ixmp4.py | 13 +++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/pyam/ixmp4.py b/pyam/ixmp4.py index 751e8106d..19c9aba04 100644 --- a/pyam/ixmp4.py +++ b/pyam/ixmp4.py @@ -1,8 +1,12 @@ +import logging + import ixmp4 import pandas as pd from ixmp4.core.region import RegionModel from ixmp4.core.unit import UnitModel +logger = logging.getLogger(__name__) + def read_ixmp4(platform: ixmp4.Platform | str, default_only: bool = True): """Read scenario runs from an ixmp4 platform database instance @@ -70,10 +74,20 @@ def write_to_ixmp4(platform: ixmp4.Platform | str, df): f"{dimension}." ) + # The "version" meta-indicator should not be written to the database + if "version" in df.meta.columns: + logger.warning( + "The `meta.version` column will be dropped when writing to the ixmp4 database." + ) + meta = df.meta.drop(columns="version") + else: + meta = df.meta.copy() + + # Create runs and add IAMC timeseries data and meta indicators for model, scenario in df.index: _df = df.filter(model=model, scenario=scenario) run = platform.runs.create(model=model, scenario=scenario) run.iamc.add(_df.data) - run.meta = dict(_df.meta.iloc[0]) + run.meta = dict(meta.loc[(model, scenario)]) run.set_as_default() diff --git a/tests/test_ixmp4.py b/tests/test_ixmp4.py index 60eb77432..2558db2b3 100644 --- a/tests/test_ixmp4.py +++ b/tests/test_ixmp4.py @@ -62,3 +62,16 @@ def test_ixmp4_integration(test_df_year): meta["version"] = 1 exp = pyam.IamDataFrame(data, meta=meta, index=["model", "scenario", "version"]) pyam.assert_iamframe_equal(exp, obs) + + +def test_ixmp4_reserved_columns(test_df_year): + """Make sure that the `version` column is not written to the platform""" + platform = Platform(_backend=SqliteTestBackend()) + platform.regions.create(name="World", hierarchy="common") + platform.units.create(name="EJ/yr") + + # test writing to platform + test_df_year.set_meta(1, "version") # add version number added from ixmp4 + test_df_year.to_ixmp4(platform=platform) + + assert "version" not in platform.runs.get("model_a", "scen_a").meta From 437f22090e09740f738cff725b716cad1dbe99c8 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Thu, 7 Mar 2024 14:06:21 +0100 Subject: [PATCH 03/10] Refactor to `test_platform` conftest featue --- pyam/ixmp4.py | 2 +- tests/conftest.py | 10 +++++++++ tests/test_ixmp4.py | 53 +++++++++++++++++---------------------------- 3 files changed, 31 insertions(+), 34 deletions(-) diff --git a/pyam/ixmp4.py b/pyam/ixmp4.py index 19c9aba04..41a4311bf 100644 --- a/pyam/ixmp4.py +++ b/pyam/ixmp4.py @@ -54,7 +54,7 @@ def write_to_ixmp4(platform: ixmp4.Platform | str, df): The IamDataFrame instance with scenario data """ if df.time_domain != "year": - raise NotImplementedError("Only time_domain='year' is supported for now") + raise NotImplementedError("Only time_domain='year' is supported for now.") if not isinstance(platform, ixmp4.Platform): platform = ixmp4.Platform(platform) diff --git a/tests/conftest.py b/tests/conftest.py index 4baf1d1fa..36900d88d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,8 @@ import pandas as pd import pytest from httpx import ConnectError +from ixmp4.core import Platform +from ixmp4.data.backend import SqliteTestBackend from pyam import IamDataFrame, iiasa from pyam.utils import IAMC_IDX, META_IDX @@ -263,6 +265,14 @@ def plot_stackplot_df(): yield df +@pytest.fixture(scope="session") +def test_platform(): + platform = Platform(_backend=SqliteTestBackend()) + platform.regions.create(name="World", hierarchy="common") + platform.units.create(name="EJ/yr") + yield platform + + @pytest.fixture(scope="session") def conn(): if not IIASA_UNAVAILABLE: diff --git a/tests/test_ixmp4.py b/tests/test_ixmp4.py index 2558db2b3..4ad9239ca 100644 --- a/tests/test_ixmp4.py +++ b/tests/test_ixmp4.py @@ -1,61 +1,51 @@ import pytest -from ixmp4.core import Platform from ixmp4.core.region import RegionModel from ixmp4.core.unit import UnitModel -from ixmp4.data.backend import SqliteTestBackend import pyam from pyam import read_ixmp4 -def test_to_ixmp4_missing_region_raises(test_df_year): +def test_to_ixmp4_missing_region_raises(test_platform, test_df_year): """Writing to platform raises if region not defined""" - platform = Platform(_backend=SqliteTestBackend()) - with pytest.raises(RegionModel.NotFound, match="World. Use `Platform.regions."): - test_df_year.to_ixmp4(platform=platform) + test_df_year.rename(region={"World": "foo"}) + with pytest.raises(RegionModel.NotFound, match="foo. Use `Platform.regions."): + test_df_year.to_ixmp4(platform=test_platform) -def test_to_ixmp4_missing_unit_raises(test_df_year): +def test_to_ixmp4_missing_unit_raises(test_platform, test_df_year): """Writing to platform raises if unit not defined""" - platform = Platform(_backend=SqliteTestBackend()) - platform.regions.create(name="World", hierarchy="common") - with pytest.raises(UnitModel.NotFound, match="EJ/yr. Use `Platform.units."): - test_df_year.to_ixmp4(platform=platform) + test_df_year.rename(region={"EJ/yr": "foo"}) + with pytest.raises(UnitModel.NotFound, match="foo. Use `Platform.units."): + test_df_year.to_ixmp4(platform=test_platform) -def test_ixmp4_time_not_implemented(test_df): +def test_ixmp4_time_not_implemented(test_platform, test_df): """Writing an IamDataFrame with datetime-data is not implemented""" - platform = Platform(_backend=SqliteTestBackend()) - platform.regions.create(name="World", hierarchy="common") - platform.units.create(name="EJ/yr") - if test_df.time_domain != "year": with pytest.raises(NotImplementedError): - test_df.to_ixmp4(platform=platform) + test_df.to_ixmp4(platform=test_platform) -def test_ixmp4_integration(test_df_year): +def test_ixmp4_integration(test_platform, test_df_year): """Write an IamDataFrame to the platform""" - platform = Platform(_backend=SqliteTestBackend()) - platform.regions.create(name="World", hierarchy="common") - platform.units.create(name="EJ/yr") # test writing to platform - test_df_year.to_ixmp4(platform=platform) + test_df_year.to_ixmp4(platform=test_platform) # read only default scenarios (runs) - version number added as meta indicator - obs = read_ixmp4(platform=platform) + obs = read_ixmp4(platform=test_platform) exp = test_df_year.copy() exp.set_meta(1, "version") # add version number added from ixmp4 pyam.assert_iamframe_equal(exp, obs) # make one scenario a non-default scenario, make sure that it is not included - platform.runs.get("model_a", "scen_b").unset_as_default() - obs = read_ixmp4(platform=platform) + test_platform.runs.get("model_a", "scen_b").unset_as_default() + obs = read_ixmp4(platform=test_platform) pyam.assert_iamframe_equal(exp.filter(scenario="scen_a"), obs) # read all scenarios (runs) - version number used as additional index dimension - obs = read_ixmp4(platform=platform, default_only=False) + obs = read_ixmp4(platform=test_platform, default_only=False) data = test_df_year.data data["version"] = 1 meta = test_df_year.meta.reset_index() @@ -64,14 +54,11 @@ def test_ixmp4_integration(test_df_year): pyam.assert_iamframe_equal(exp, obs) -def test_ixmp4_reserved_columns(test_df_year): +def test_ixmp4_reserved_columns(test_platform, test_df_year): """Make sure that the `version` column is not written to the platform""" - platform = Platform(_backend=SqliteTestBackend()) - platform.regions.create(name="World", hierarchy="common") - platform.units.create(name="EJ/yr") - # test writing to platform + # test writing to platform with a version-number as meta indicator test_df_year.set_meta(1, "version") # add version number added from ixmp4 - test_df_year.to_ixmp4(platform=platform) + test_df_year.to_ixmp4(platform=test_platform) - assert "version" not in platform.runs.get("model_a", "scen_a").meta + assert "version" not in test_platform.runs.get("model_a", "scen_a").meta From d0ee8178a9b0a286352973f718620c0dc370a0e8 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Thu, 7 Mar 2024 15:04:41 +0100 Subject: [PATCH 04/10] Minor fixes --- tests/conftest.py | 2 +- tests/test_ixmp4.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 36900d88d..750ce5405 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -265,7 +265,7 @@ def plot_stackplot_df(): yield df -@pytest.fixture(scope="session") +@pytest.fixture(scope="function") def test_platform(): platform = Platform(_backend=SqliteTestBackend()) platform.regions.create(name="World", hierarchy="common") diff --git a/tests/test_ixmp4.py b/tests/test_ixmp4.py index 4ad9239ca..bebd0d912 100644 --- a/tests/test_ixmp4.py +++ b/tests/test_ixmp4.py @@ -8,14 +8,14 @@ def test_to_ixmp4_missing_region_raises(test_platform, test_df_year): """Writing to platform raises if region not defined""" - test_df_year.rename(region={"World": "foo"}) + test_df_year.rename(region={"World": "foo"}, inplace=True) with pytest.raises(RegionModel.NotFound, match="foo. Use `Platform.regions."): test_df_year.to_ixmp4(platform=test_platform) def test_to_ixmp4_missing_unit_raises(test_platform, test_df_year): """Writing to platform raises if unit not defined""" - test_df_year.rename(region={"EJ/yr": "foo"}) + test_df_year.rename(region={"EJ/yr": "foo"}, inplace=True) with pytest.raises(UnitModel.NotFound, match="foo. Use `Platform.units."): test_df_year.to_ixmp4(platform=test_platform) @@ -55,7 +55,7 @@ def test_ixmp4_integration(test_platform, test_df_year): def test_ixmp4_reserved_columns(test_platform, test_df_year): - """Make sure that the `version` column is not written to the platform""" + """Make sure that a 'version' column in `meta` is not written to the platform""" # test writing to platform with a version-number as meta indicator test_df_year.set_meta(1, "version") # add version number added from ixmp4 From a520a8d332dcd2c600b7be2eda1006e00d67fb79 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Thu, 7 Mar 2024 15:08:52 +0100 Subject: [PATCH 05/10] Add to release notes --- RELEASE_NOTES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index c3233a147..064a43e71 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -6,6 +6,7 @@ Bumped minimum version of pandas and numpy to fit **ixmp4**'s requirement. ## Individual updates +- [#832](https://github.com/IAMconsortium/pyam/pull/832) Improve the test-suite for the ixmp4 integration - [#827](https://github.com/IAMconsortium/pyam/pull/827) Migrate to poetry for project management - [#830](https://github.com/IAMconsortium/pyam/pull/830) Implement more consistent logging behavior with **ixmp4** - [#829](https://github.com/IAMconsortium/pyam/pull/829) Add a `pyam.iiasa.platforms()` function for a list of available platforms From b838bebcd6e7986f078a1eccab09ce8739965428 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Thu, 7 Mar 2024 15:16:41 +0100 Subject: [PATCH 06/10] Fix failing test --- tests/test_ixmp4.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ixmp4.py b/tests/test_ixmp4.py index bebd0d912..46ee17308 100644 --- a/tests/test_ixmp4.py +++ b/tests/test_ixmp4.py @@ -15,7 +15,7 @@ def test_to_ixmp4_missing_region_raises(test_platform, test_df_year): def test_to_ixmp4_missing_unit_raises(test_platform, test_df_year): """Writing to platform raises if unit not defined""" - test_df_year.rename(region={"EJ/yr": "foo"}, inplace=True) + test_df_year.rename(unit={"EJ/yr": "foo"}, inplace=True) with pytest.raises(UnitModel.NotFound, match="foo. Use `Platform.units."): test_df_year.to_ixmp4(platform=test_platform) From b42d529a0973e0f895797b5d23dfb65fcc6bf0b6 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Thu, 7 Mar 2024 15:19:26 +0100 Subject: [PATCH 07/10] Make ruff --- pyam/ixmp4.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyam/ixmp4.py b/pyam/ixmp4.py index 41a4311bf..898c693b2 100644 --- a/pyam/ixmp4.py +++ b/pyam/ixmp4.py @@ -59,7 +59,7 @@ def write_to_ixmp4(platform: ixmp4.Platform | str, df): if not isinstance(platform, ixmp4.Platform): platform = ixmp4.Platform(platform) - # TODO: implement a try-except to roll back changes if any error writing to platform + # TODO: implement try-except to roll back changes if any error writing to platform # depends on https://github.com/iiasa/ixmp4/issues/29 # quickfix: ensure that units and regions exist before writing for dimension, values, model in [ @@ -74,10 +74,11 @@ def write_to_ixmp4(platform: ixmp4.Platform | str, df): f"{dimension}." ) - # The "version" meta-indicator should not be written to the database + # The "version" meta-indicator, added when reading from an ixmp4 platform, + # should not be written to the platform if "version" in df.meta.columns: logger.warning( - "The `meta.version` column will be dropped when writing to the ixmp4 database." + "The `meta.version` column was dropped when writing to the ixmp4 platform." ) meta = df.meta.drop(columns="version") else: From 3759f87a8e94926a0081f045fb130f19ed033ad8 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Thu, 7 Mar 2024 16:10:27 +0100 Subject: [PATCH 08/10] Add a test per suggestion by @phackstock --- tests/test_ixmp4.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_ixmp4.py b/tests/test_ixmp4.py index 46ee17308..23d3dbbe3 100644 --- a/tests/test_ixmp4.py +++ b/tests/test_ixmp4.py @@ -54,11 +54,18 @@ def test_ixmp4_integration(test_platform, test_df_year): pyam.assert_iamframe_equal(exp, obs) -def test_ixmp4_reserved_columns(test_platform, test_df_year): +@pytest.mark.parametrize("drop_meta", (True, False)) +def test_ixmp4_reserved_columns(test_platform, test_df_year, drop_meta): """Make sure that a 'version' column in `meta` is not written to the platform""" + if drop_meta: + test_df_year = pyam.IamDataFrame(test_df_year.data, index=test_df_year.index) + # test writing to platform with a version-number as meta indicator test_df_year.set_meta(1, "version") # add version number added from ixmp4 test_df_year.to_ixmp4(platform=test_platform) - assert "version" not in test_platform.runs.get("model_a", "scen_a").meta + if drop_meta: + assert test_platform.runs.get("model_a", "scen_a").meta.empty + else: + assert "version" not in test_platform.runs.get("model_a", "scen_a").meta From 10e45bafbbd1f7a9129ecba8a1d9f53bec0b9b12 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Fri, 8 Mar 2024 07:48:44 +0100 Subject: [PATCH 09/10] Minor fixes --- pyam/ixmp4.py | 3 ++- tests/test_ixmp4.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pyam/ixmp4.py b/pyam/ixmp4.py index 898c693b2..174c16fa6 100644 --- a/pyam/ixmp4.py +++ b/pyam/ixmp4.py @@ -90,5 +90,6 @@ def write_to_ixmp4(platform: ixmp4.Platform | str, df): run = platform.runs.create(model=model, scenario=scenario) run.iamc.add(_df.data) - run.meta = dict(meta.loc[(model, scenario)]) + if not meta.empty: + run.meta = dict(meta.loc[(model, scenario)]) run.set_as_default() diff --git a/tests/test_ixmp4.py b/tests/test_ixmp4.py index 23d3dbbe3..33f6bcd91 100644 --- a/tests/test_ixmp4.py +++ b/tests/test_ixmp4.py @@ -59,7 +59,7 @@ def test_ixmp4_reserved_columns(test_platform, test_df_year, drop_meta): """Make sure that a 'version' column in `meta` is not written to the platform""" if drop_meta: - test_df_year = pyam.IamDataFrame(test_df_year.data, index=test_df_year.index) + test_df_year = pyam.IamDataFrame(test_df_year.data) # test writing to platform with a version-number as meta indicator test_df_year.set_meta(1, "version") # add version number added from ixmp4 From 6ed7b56150a706d2a5f10813e52ff2c114e8761e Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Fri, 8 Mar 2024 07:59:27 +0100 Subject: [PATCH 10/10] Another fix --- tests/test_ixmp4.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ixmp4.py b/tests/test_ixmp4.py index 33f6bcd91..1d43a0b41 100644 --- a/tests/test_ixmp4.py +++ b/tests/test_ixmp4.py @@ -66,6 +66,6 @@ def test_ixmp4_reserved_columns(test_platform, test_df_year, drop_meta): test_df_year.to_ixmp4(platform=test_platform) if drop_meta: - assert test_platform.runs.get("model_a", "scen_a").meta.empty + assert len(test_platform.runs.get("model_a", "scen_a").meta) == 0 else: assert "version" not in test_platform.runs.get("model_a", "scen_a").meta