From b59e77c24b3005309c039c7e72c52df2e43b9514 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Fri, 8 Mar 2024 08:15:21 +0100 Subject: [PATCH] Add `test-platform` fixture for ixmp4 tests (#832) --- RELEASE_NOTES.md | 1 + pyam/ixmp4.py | 22 ++++++++++++++--- tests/conftest.py | 10 ++++++++ tests/test_ixmp4.py | 60 +++++++++++++++++++++++++++------------------ 4 files changed, 66 insertions(+), 27 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index c3233a147..064a43e71 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -6,6 +6,7 @@ Bumped minimum version of pandas and numpy to fit **ixmp4**'s requirement. ## Individual updates +- [#832](https://github.com/IAMconsortium/pyam/pull/832) Improve the test-suite for the ixmp4 integration - [#827](https://github.com/IAMconsortium/pyam/pull/827) Migrate to poetry for project management - [#830](https://github.com/IAMconsortium/pyam/pull/830) Implement more consistent logging behavior with **ixmp4** - [#829](https://github.com/IAMconsortium/pyam/pull/829) Add a `pyam.iiasa.platforms()` function for a list of available platforms diff --git a/pyam/ixmp4.py b/pyam/ixmp4.py index 751e8106d..174c16fa6 100644 --- a/pyam/ixmp4.py +++ b/pyam/ixmp4.py @@ -1,8 +1,12 @@ +import logging + import ixmp4 import pandas as pd from ixmp4.core.region import RegionModel from ixmp4.core.unit import UnitModel +logger = logging.getLogger(__name__) + def read_ixmp4(platform: ixmp4.Platform | str, default_only: bool = True): """Read scenario runs from an ixmp4 platform database instance @@ -50,12 +54,12 @@ def write_to_ixmp4(platform: ixmp4.Platform | str, df): The IamDataFrame instance with scenario data """ if df.time_domain != "year": - raise NotImplementedError("Only time_domain='year' is supported for now") + raise NotImplementedError("Only time_domain='year' is supported for now.") if not isinstance(platform, ixmp4.Platform): platform = ixmp4.Platform(platform) - # TODO: implement a try-except to roll back changes if any error writing to platform + # TODO: implement try-except to roll back changes if any error writing to platform # depends on https://github.com/iiasa/ixmp4/issues/29 # quickfix: ensure that units and regions exist before writing for dimension, values, model in [ @@ -70,10 +74,22 @@ def write_to_ixmp4(platform: ixmp4.Platform | str, df): f"{dimension}." ) + # The "version" meta-indicator, added when reading from an ixmp4 platform, + # should not be written to the platform + if "version" in df.meta.columns: + logger.warning( + "The `meta.version` column was dropped when writing to the ixmp4 platform." + ) + meta = df.meta.drop(columns="version") + else: + meta = df.meta.copy() + + # Create runs and add IAMC timeseries data and meta indicators for model, scenario in df.index: _df = df.filter(model=model, scenario=scenario) run = platform.runs.create(model=model, scenario=scenario) run.iamc.add(_df.data) - run.meta = dict(_df.meta.iloc[0]) + if not meta.empty: + run.meta = dict(meta.loc[(model, scenario)]) run.set_as_default() diff --git a/tests/conftest.py b/tests/conftest.py index 4baf1d1fa..750ce5405 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,8 @@ import pandas as pd import pytest from httpx import ConnectError +from ixmp4.core import Platform +from ixmp4.data.backend import SqliteTestBackend from pyam import IamDataFrame, iiasa from pyam.utils import IAMC_IDX, META_IDX @@ -263,6 +265,14 @@ def plot_stackplot_df(): yield df +@pytest.fixture(scope="function") +def test_platform(): + platform = Platform(_backend=SqliteTestBackend()) + platform.regions.create(name="World", hierarchy="common") + platform.units.create(name="EJ/yr") + yield platform + + @pytest.fixture(scope="session") def conn(): if not IIASA_UNAVAILABLE: diff --git a/tests/test_ixmp4.py b/tests/test_ixmp4.py index 312d3fa31..1d43a0b41 100644 --- a/tests/test_ixmp4.py +++ b/tests/test_ixmp4.py @@ -1,59 +1,71 @@ import pytest -from ixmp4.core import Platform from ixmp4.core.region import RegionModel from ixmp4.core.unit import UnitModel -from ixmp4.data.backend import SqliteTestBackend import pyam from pyam import read_ixmp4 -def test_to_ixmp4_missing_region_raises(test_df_year): +def test_to_ixmp4_missing_region_raises(test_platform, test_df_year): """Writing to platform raises if region not defined""" - platform = Platform(_backend=SqliteTestBackend()) - with pytest.raises(RegionModel.NotFound, match="World. Use `Platform.regions."): - test_df_year.to_ixmp4(platform=platform) + test_df_year.rename(region={"World": "foo"}, inplace=True) + with pytest.raises(RegionModel.NotFound, match="foo. Use `Platform.regions."): + test_df_year.to_ixmp4(platform=test_platform) -def test_to_ixmp4_missing_unit_raises(test_df_year): +def test_to_ixmp4_missing_unit_raises(test_platform, test_df_year): """Writing to platform raises if unit not defined""" - platform = Platform(_backend=SqliteTestBackend()) - platform.regions.create(name="World", hierarchy="common") - with pytest.raises(UnitModel.NotFound, match="EJ/yr. Use `Platform.units."): - test_df_year.to_ixmp4(platform=platform) + test_df_year.rename(unit={"EJ/yr": "foo"}, inplace=True) + with pytest.raises(UnitModel.NotFound, match="foo. Use `Platform.units."): + test_df_year.to_ixmp4(platform=test_platform) -def test_ixmp4_time_not_implemented(test_df): +def test_ixmp4_time_not_implemented(test_platform, test_df): """Writing an IamDataFrame with datetime-data is not implemented""" - platform = Platform(_backend=SqliteTestBackend()) - platform.regions.create(name="World", hierarchy="common") - platform.units.create(name="EJ/yr") - if test_df.time_domain != "year": with pytest.raises(NotImplementedError): - test_df.to_ixmp4(platform=platform) + test_df.to_ixmp4(platform=test_platform) -def test_ixmp4_integration(test_df_year): +def test_ixmp4_integration(test_platform, test_df_year): """Write an IamDataFrame to the platform""" - platform = Platform(_backend=SqliteTestBackend()) - platform.regions.create(name="World", hierarchy="common") - platform.units.create(name="EJ/yr") # test writing to platform - test_df_year.to_ixmp4(platform=platform) + test_df_year.to_ixmp4(platform=test_platform) # read only default scenarios (runs) - version number added as meta indicator - obs = read_ixmp4(platform=platform) + obs = read_ixmp4(platform=test_platform) exp = test_df_year.copy() exp.set_meta(1, "version") # add version number added from ixmp4 pyam.assert_iamframe_equal(exp, obs) + # make one scenario a non-default scenario, make sure that it is not included + test_platform.runs.get("model_a", "scen_b").unset_as_default() + obs = read_ixmp4(platform=test_platform) + pyam.assert_iamframe_equal(exp.filter(scenario="scen_a"), obs) + # read all scenarios (runs) - version number used as additional index dimension - obs = read_ixmp4(platform=platform, default_only=False) + obs = read_ixmp4(platform=test_platform, default_only=False) data = test_df_year.data data["version"] = 1 meta = test_df_year.meta.reset_index() meta["version"] = 1 exp = pyam.IamDataFrame(data, meta=meta, index=["model", "scenario", "version"]) pyam.assert_iamframe_equal(exp, obs) + + +@pytest.mark.parametrize("drop_meta", (True, False)) +def test_ixmp4_reserved_columns(test_platform, test_df_year, drop_meta): + """Make sure that a 'version' column in `meta` is not written to the platform""" + + if drop_meta: + test_df_year = pyam.IamDataFrame(test_df_year.data) + + # test writing to platform with a version-number as meta indicator + test_df_year.set_meta(1, "version") # add version number added from ixmp4 + test_df_year.to_ixmp4(platform=test_platform) + + if drop_meta: + assert len(test_platform.runs.get("model_a", "scen_a").meta) == 0 + else: + assert "version" not in test_platform.runs.get("model_a", "scen_a").meta