Add test-platform fixture for ixmp4 tests (#832)

RELEASE_NOTES.md

@@ -6,6 +6,7 @@ Bumped minimum version of pandas and numpy to fit **ixmp4**'s requirement.
 
 ## Individual updates
 
+- [#832](https://github.com/IAMconsortium/pyam/pull/832) Improve the test-suite for the ixmp4 integration
 - [#827](https://github.com/IAMconsortium/pyam/pull/827) Migrate to poetry for project management
 - [#830](https://github.com/IAMconsortium/pyam/pull/830) Implement more consistent logging behavior with **ixmp4**
 - [#829](https://github.com/IAMconsortium/pyam/pull/829) Add a `pyam.iiasa.platforms()` function for a list of available platforms

pyam/ixmp4.py

@@ -1,8 +1,12 @@
+import logging
+
 import ixmp4
 import pandas as pd
 from ixmp4.core.region import RegionModel
 from ixmp4.core.unit import UnitModel
 
+logger = logging.getLogger(__name__)
+
 
 def read_ixmp4(platform: ixmp4.Platform | str, default_only: bool = True):
     """Read scenario runs from an ixmp4 platform database instance
@@ -50,12 +54,12 @@ def write_to_ixmp4(platform: ixmp4.Platform | str, df):
         The IamDataFrame instance with scenario data
     """
     if df.time_domain != "year":
-        raise NotImplementedError("Only time_domain='year' is supported for now")
+        raise NotImplementedError("Only time_domain='year' is supported for now.")
 
     if not isinstance(platform, ixmp4.Platform):
         platform = ixmp4.Platform(platform)
 
-    # TODO: implement a try-except to roll back changes if any error writing to platform
+    # TODO: implement try-except to roll back changes if any error writing to platform
     # depends on https://github.com/iiasa/ixmp4/issues/29
     # quickfix: ensure that units and regions exist before writing
     for dimension, values, model in [
@@ -70,10 +74,22 @@ def write_to_ixmp4(platform: ixmp4.Platform | str, df):
                 f"{dimension}."
             )
 
+    # The "version" meta-indicator, added when reading from an ixmp4 platform,
+    # should not be written to the platform
+    if "version" in df.meta.columns:
+        logger.warning(
+            "The `meta.version` column was dropped when writing to the ixmp4 platform."
+        )
+        meta = df.meta.drop(columns="version")
+    else:
+        meta = df.meta.copy()
+
+    # Create runs and add IAMC timeseries data and meta indicators
     for model, scenario in df.index:
         _df = df.filter(model=model, scenario=scenario)
 
         run = platform.runs.create(model=model, scenario=scenario)
         run.iamc.add(_df.data)
-        run.meta = dict(_df.meta.iloc[0])
+        if not meta.empty:
+            run.meta = dict(meta.loc[(model, scenario)])
         run.set_as_default()

tests/conftest.py

@@ -10,6 +10,8 @@
 import pandas as pd
 import pytest
 from httpx import ConnectError
+from ixmp4.core import Platform
+from ixmp4.data.backend import SqliteTestBackend
 
 from pyam import IamDataFrame, iiasa
 from pyam.utils import IAMC_IDX, META_IDX
@@ -263,6 +265,14 @@ def plot_stackplot_df():
     yield df
 
 
+@pytest.fixture(scope="function")
+def test_platform():
+    platform = Platform(_backend=SqliteTestBackend())
+    platform.regions.create(name="World", hierarchy="common")
+    platform.units.create(name="EJ/yr")
+    yield platform
+
+
 @pytest.fixture(scope="session")
 def conn():
     if not IIASA_UNAVAILABLE:

tests/test_ixmp4.py

@@ -1,59 +1,71 @@
 import pytest
-from ixmp4.core import Platform
 from ixmp4.core.region import RegionModel
 from ixmp4.core.unit import UnitModel
-from ixmp4.data.backend import SqliteTestBackend
 
 import pyam
 from pyam import read_ixmp4
 
 
-def test_to_ixmp4_missing_region_raises(test_df_year):
+def test_to_ixmp4_missing_region_raises(test_platform, test_df_year):
     """Writing to platform raises if region not defined"""
-    platform = Platform(_backend=SqliteTestBackend())
-    with pytest.raises(RegionModel.NotFound, match="World. Use `Platform.regions."):
-        test_df_year.to_ixmp4(platform=platform)
+    test_df_year.rename(region={"World": "foo"}, inplace=True)
+    with pytest.raises(RegionModel.NotFound, match="foo. Use `Platform.regions."):
+        test_df_year.to_ixmp4(platform=test_platform)
 
 
-def test_to_ixmp4_missing_unit_raises(test_df_year):
+def test_to_ixmp4_missing_unit_raises(test_platform, test_df_year):
     """Writing to platform raises if unit not defined"""
-    platform = Platform(_backend=SqliteTestBackend())
-    platform.regions.create(name="World", hierarchy="common")
-    with pytest.raises(UnitModel.NotFound, match="EJ/yr. Use `Platform.units."):
-        test_df_year.to_ixmp4(platform=platform)
+    test_df_year.rename(unit={"EJ/yr": "foo"}, inplace=True)
+    with pytest.raises(UnitModel.NotFound, match="foo. Use `Platform.units."):
+        test_df_year.to_ixmp4(platform=test_platform)
 
 
-def test_ixmp4_time_not_implemented(test_df):
+def test_ixmp4_time_not_implemented(test_platform, test_df):
     """Writing an IamDataFrame with datetime-data is not implemented"""
-    platform = Platform(_backend=SqliteTestBackend())
-    platform.regions.create(name="World", hierarchy="common")
-    platform.units.create(name="EJ/yr")
-
     if test_df.time_domain != "year":
         with pytest.raises(NotImplementedError):
-            test_df.to_ixmp4(platform=platform)
+            test_df.to_ixmp4(platform=test_platform)
 
 
-def test_ixmp4_integration(test_df_year):
+def test_ixmp4_integration(test_platform, test_df_year):
     """Write an IamDataFrame to the platform"""
-    platform = Platform(_backend=SqliteTestBackend())
-    platform.regions.create(name="World", hierarchy="common")
-    platform.units.create(name="EJ/yr")
 
     # test writing to platform
-    test_df_year.to_ixmp4(platform=platform)
+    test_df_year.to_ixmp4(platform=test_platform)
 
     # read only default scenarios (runs) - version number added as meta indicator
-    obs = read_ixmp4(platform=platform)
+    obs = read_ixmp4(platform=test_platform)
     exp = test_df_year.copy()
     exp.set_meta(1, "version")  # add version number added from ixmp4
     pyam.assert_iamframe_equal(exp, obs)
 
+    # make one scenario a non-default scenario, make sure that it is not included
+    test_platform.runs.get("model_a", "scen_b").unset_as_default()
+    obs = read_ixmp4(platform=test_platform)
+    pyam.assert_iamframe_equal(exp.filter(scenario="scen_a"), obs)
+
     # read all scenarios (runs) - version number used as additional index dimension
-    obs = read_ixmp4(platform=platform, default_only=False)
+    obs = read_ixmp4(platform=test_platform, default_only=False)
     data = test_df_year.data
     data["version"] = 1
     meta = test_df_year.meta.reset_index()
     meta["version"] = 1
     exp = pyam.IamDataFrame(data, meta=meta, index=["model", "scenario", "version"])
     pyam.assert_iamframe_equal(exp, obs)
+
+
+@pytest.mark.parametrize("drop_meta", (True, False))
+def test_ixmp4_reserved_columns(test_platform, test_df_year, drop_meta):
+    """Make sure that a 'version' column in `meta` is not written to the platform"""
+
+    if drop_meta:
+        test_df_year = pyam.IamDataFrame(test_df_year.data)
+
+    # test writing to platform with a version-number as meta indicator
+    test_df_year.set_meta(1, "version")  # add version number added from ixmp4
+    test_df_year.to_ixmp4(platform=test_platform)
+
+    if drop_meta:
+        assert len(test_platform.runs.get("model_a", "scen_a").meta) == 0
+    else:
+        assert "version" not in test_platform.runs.get("model_a", "scen_a").meta