86 pytest regressions

changelog/88.improvement.md

@@ -0,0 +1 @@
+Switched to using [pytest-regressions](https://pytest-regressions.readthedocs.io/en/latest) rather than relying on our own solution

pyproject.toml

@@ -68,6 +68,7 @@ models = ["pandas", "ciceroscm", "fair", "pymagicc"]
 
 [tool.poetry.group.tests.dependencies]
 pytest = "^7.3.1"
+pytest-regressions = "^2.5.0"
 
 [tool.poetry.group.docs.dependencies]
 myst-nb = "^0.17.0"

src/openscm_runner/testing.py

@@ -1,40 +1,40 @@
 """
 Miscellaneous testing code
 """
-import json
 from abc import ABC, abstractmethod
 
+import numpy as np
 import numpy.testing as npt
 from scmdata import ScmRun
 
 from openscm_runner import run
 
-try:
-    import pytest
 
-    HAS_PYTEST = True
-
-except ImportError:
-    HAS_PYTEST = False
-
-
-def _check_output(  # pylint: disable=too-many-locals,too-many-branches
-    res, expected_output_file, rtol, update
+def _get_output_dict(  # pylint: disable=too-many-locals,too-many-branches
+    res, outputs_to_get
 ):
-    if not HAS_PYTEST:
-        raise ImportError("pytest not installed, run `pip install pytest`")
+    """
+    Get output dictionary
 
-    with open(expected_output_file, encoding="ascii") as filehandle:
-        expected_output = json.load(filehandle)
+    Returns a dictionary of outputs which can be used with our regression testing.
 
-    updated_output = {}
+    Parameters
+    ----------
+    res
+        Results from which to extract outputs
 
-    for climate_model, checks in expected_output.items():
-        res_cm = res.filter(climate_model=climate_model)
+    outputs_to_get
+        Outputs to get from the results
 
-        updated_output[climate_model] = []
+    Returns
+    -------
+        Dictionary of results which can be understood by our regression tests
+    """
+    output = {}
+    for climate_model, checks in outputs_to_get.items():
+        res_cm = res.filter(climate_model=climate_model)
 
-        for filter_kwargs, expected_val in checks:
+        for filter_kwargs in checks:
             err_msg = f"{filter_kwargs}"
             filter_kwargs_in = {**filter_kwargs}
             check_units = filter_kwargs.pop("unit", None)
@@ -53,25 +53,16 @@ def _check_output(  # pylint: disable=too-many-locals,too-many-branches
                 ).values
             )
 
-            try:
-                npt.assert_allclose(
-                    res_val,
-                    expected_val,
-                    rtol=rtol,
-                    err_msg=err_msg,
-                )
-                new_val = expected_val
-            except AssertionError:
-                new_val = res_val
-                if not update:
-                    raise
+            key = ["climate_model", climate_model]
+            for k, v in filter_kwargs_in.items():
+                key.append(k)
+                key.append(v)
 
-            updated_output[climate_model].append((filter_kwargs_in, new_val))
-    if update:
-        with open(expected_output_file, "w", encoding="ascii") as file_handle:
-            json.dump(updated_output, file_handle, indent=4)
+            # key = tuple(key)
+            key = "_".join([str(v) for v in key])
+            output[key] = np.array([res_val])
 
-        pytest.skip(f"Updated {expected_output_file}")
+    return output
 
 
 class _AdapterTester(ABC):  # nosec
@@ -99,13 +90,13 @@ class _AdapterTester(ABC):  # nosec
         "Effective Radiative Forcing|Aerosols",
         "Effective Radiative Forcing|Aerosols|Direct Effect",
         "Effective Radiative Forcing|Aerosols|Direct Effect|BC",
-        "Effective Radiative Forcing|Aerosols|Direct Effect|BC|MAGICC Fossil and Industrial",
+        "Effective Radiative Forcing|Aerosols|Direct Effect|BC|MAGICC Fossil and Industrial",  # noqa: E501
         "Effective Radiative Forcing|Aerosols|Direct Effect|BC|MAGICC AFOLU",
         "Effective Radiative Forcing|Aerosols|Direct Effect|OC",
-        "Effective Radiative Forcing|Aerosols|Direct Effect|OC|MAGICC Fossil and Industrial",
+        "Effective Radiative Forcing|Aerosols|Direct Effect|OC|MAGICC Fossil and Industrial",  # noqa: E501
         "Effective Radiative Forcing|Aerosols|Direct Effect|OC|MAGICC AFOLU",
         "Effective Radiative Forcing|Aerosols|Direct Effect|SOx",
-        "Effective Radiative Forcing|Aerosols|Direct Effect|SOx|MAGICC Fossil and Industrial",
+        "Effective Radiative Forcing|Aerosols|Direct Effect|SOx|MAGICC Fossil and Industrial",  # noqa: E501
         "Effective Radiative Forcing|Aerosols|Direct Effect|SOx|MAGICC AFOLU",
         "Effective Radiative Forcing|Aerosols|Indirect Effect",
         "Effective Radiative Forcing|Greenhouse Gases",
@@ -143,8 +134,8 @@ def _check_res(self, exp, check_val, raise_error):
 
             print(f"exp: {exp}, check_val: {check_val}")
 
-    def _check_output(self, res, expected_output_file, update):
-        _check_output(res, expected_output_file, self._rtol, update)
+    def _get_output_dict(self, res, outputs_to_get):
+        return _get_output_dict(res, outputs_to_get)
 
     @staticmethod
     def _check_heat_content_heat_uptake_consistency(res):

tests/conftest.py

@@ -5,6 +5,7 @@
 """
 from __future__ import annotations
 
+import subprocess
 from pathlib import Path
 
 import pandas as pd
@@ -26,9 +27,10 @@
     else:
         CORRECT_MAGICC_IS_AVAILABLE = True
 
-except KeyError:
+except (KeyError, subprocess.CalledProcessError) as exc:
     MAGICC_VERSION = None
     CORRECT_MAGICC_IS_AVAILABLE = False
+    MAGICC_LOAD_EXC = exc
 
 
 CICEROSCM_IS_AVAILABLE = False
@@ -45,7 +47,7 @@ def pytest_runtest_setup(item) -> None:
     for mark in item.iter_markers():
         if mark.name == "magicc" and not CORRECT_MAGICC_IS_AVAILABLE:
             if MAGICC_VERSION is None:
-                pytest.skip("MAGICC7 not available")
+                pytest.skip(f"MAGICC7 not available. Exc: {MAGICC_LOAD_EXC}")
             else:
                 pytest.skip(
                     "Wrong MAGICC version for tests ({}), we require {}".format(
@@ -106,17 +108,3 @@ def test_scenario_ssp370_world(test_data_dir: Path) -> scmdata.ScmRun:
     )
 
     return scenario
-
-
-def pytest_addoption(parser):
-    parser.addoption(
-        "--update-expected-values",
-        action="store_true",
-        default=False,
-        help="Overwrite expected values",
-    )
-
-
-@pytest.fixture
-def update_expected_values(request):
-    return request.config.getoption("--update-expected-values")

tests/integration/test_ciceroSCM.py

@@ -20,16 +20,9 @@ class TestCICEROSCMAdapter(_AdapterTester):
     def test_run(
         self,
         test_scenarios,
-        test_data_dir,
-        update_expected_values,
+        num_regression,
         shuffle_column_order,
     ):
-        expected_output_file = os.path.join(
-            test_data_dir,
-            "expected-integration-output",
-            "expected_ciceroscm_test_run_output.json",
-        )
-
         if shuffle_column_order:
             tmp = test_scenarios.long_data()
             cols = tmp.columns.tolist()
@@ -242,14 +235,143 @@ def test_run(
             scenario="ssp245",
             run_id=1,
         )
-        # ch
+
         # check that jump in GHG ERF isn't there
         assert (
             ssp245_ch4_conc_2014.values.squeeze()
             - ssp245_ch4_conc_2015.values.squeeze()
         ) < 0.1
 
-        self._check_output(res, expected_output_file, update_expected_values)
+        outputs_to_get = {
+            "CICERO-SCM": [
+                {
+                    "variable": "Heat Content|Ocean",
+                    "unit": "ZJ",
+                    "region": "World",
+                    "year": 2100,
+                    "scenario": "ssp126",
+                    "quantile": 1,
+                },
+                {
+                    "variable": "Surface Air Temperature Change",
+                    "region": "World",
+                    "year": 2100,
+                    "scenario": "ssp126",
+                    "quantile": 1,
+                },
+                {
+                    "variable": "Surface Air Temperature Change",
+                    "region": "World",
+                    "year": 2100,
+                    "scenario": "ssp126",
+                    "quantile": 0.05,
+                },
+                {
+                    "variable": "Surface Air Temperature Change",
+                    "region": "World",
+                    "year": 2100,
+                    "scenario": "ssp126",
+                    "quantile": 0.95,
+                },
+                {
+                    "variable": "Surface Air Temperature Change",
+                    "region": "World",
+                    "year": 2100,
+                    "scenario": "ssp370",
+                    "quantile": 0.05,
+                },
+                {
+                    "variable": "Surface Air Temperature Change",
+                    "region": "World",
+                    "year": 2100,
+                    "scenario": "ssp370",
+                    "quantile": 0.95,
+                },
+                {
+                    "variable": "Surface Air Ocean Blended Temperature Change",
+                    "region": "World",
+                    "year": 2100,
+                    "scenario": "ssp126",
+                    "quantile": 1,
+                },
+                {
+                    "variable": "Heat Uptake",
+                    "unit": "W/m^2",
+                    "region": "World",
+                    "year": 2100,
+                    "scenario": "ssp126",
+                    "quantile": 1,
+                },
+                {
+                    "variable": "Surface Air Temperature Change",
+                    "region": "World",
+                    "year": 2100,
+                    "scenario": "ssp126",
+                    "quantile": 0,
+                },
+                {
+                    "variable": "Surface Air Temperature Change",
+                    "region": "World",
+                    "year": 2100,
+                    "scenario": "ssp370",
+                    "quantile": 1,
+                },
+                {
+                    "variable": "Surface Air Temperature Change",
+                    "region": "World",
+                    "year": 2100,
+                    "scenario": "ssp370",
+                    "quantile": 0,
+                },
+                {
+                    "variable": "Surface Air Temperature Change",
+                    "region": "World",
+                    "year": 2100,
+                    "scenario": "ssp126",
+                    "quantile": 0.05,
+                },
+                {
+                    "variable": "Surface Air Temperature Change",
+                    "region": "World",
+                    "year": 2100,
+                    "scenario": "ssp126",
+                    "quantile": 0.95,
+                },
+                {
+                    "variable": "Surface Air Temperature Change",
+                    "region": "World",
+                    "year": 2100,
+                    "scenario": "ssp370",
+                    "quantile": 0.05,
+                },
+                {
+                    "variable": "Surface Air Temperature Change",
+                    "region": "World",
+                    "year": 2100,
+                    "scenario": "ssp370",
+                    "quantile": 0.95,
+                },
+                {
+                    "variable": "Atmospheric Concentrations|CO2",
+                    "region": "World",
+                    "year": 2100,
+                    "scenario": "ssp370",
+                    "unit": "ppm",
+                    "quantile": 1,
+                },
+                {
+                    "variable": "Emissions|CO2",
+                    "region": "World",
+                    "year": 2100,
+                    "scenario": "ssp370",
+                    "unit": "PgC / yr",
+                    "quantile": 1,
+                },
+            ]
+        }
+
+        output_dict = self._get_output_dict(res, outputs_to_get)
+        num_regression.check(output_dict, default_tolerance=dict(rtol=self._rtol))
 
     @pytest.mark.ciceroscm
     def test_variable_naming(self, test_scenarios):

tests/integration/test_ciceroSCM/test_run_False_.csv

@@ -0,0 +1,2 @@
+,climate_model_CICERO-SCM_variable_Heat Content|Ocean_unit_ZJ_region_World_year_2100_scenario_ssp126_quantile_1,climate_model_CICERO-SCM_variable_Surface Air Temperature Change_region_World_year_2100_scenario_ssp126_quantile_1,climate_model_CICERO-SCM_variable_Surface Air Temperature Change_region_World_year_2100_scenario_ssp126_quantile_0.05,climate_model_CICERO-SCM_variable_Surface Air Temperature Change_region_World_year_2100_scenario_ssp126_quantile_0.95,climate_model_CICERO-SCM_variable_Surface Air Temperature Change_region_World_year_2100_scenario_ssp370_quantile_0.05,climate_model_CICERO-SCM_variable_Surface Air Temperature Change_region_World_year_2100_scenario_ssp370_quantile_0.95,climate_model_CICERO-SCM_variable_Surface Air Ocean Blended Temperature Change_region_World_year_2100_scenario_ssp126_quantile_1,climate_model_CICERO-SCM_variable_Heat Uptake_unit_W/m^2_region_World_year_2100_scenario_ssp126_quantile_1,climate_model_CICERO-SCM_variable_Surface Air Temperature Change_region_World_year_2100_scenario_ssp126_quantile_0,climate_model_CICERO-SCM_variable_Surface Air Temperature Change_region_World_year_2100_scenario_ssp370_quantile_1,climate_model_CICERO-SCM_variable_Surface Air Temperature Change_region_World_year_2100_scenario_ssp370_quantile_0,climate_model_CICERO-SCM_variable_Atmospheric Concentrations|CO2_region_World_year_2100_scenario_ssp370_unit_ppm_quantile_1,climate_model_CICERO-SCM_variable_Emissions|CO2_region_World_year_2100_scenario_ssp370_unit_PgC / yr_quantile_1
+0,1313.0500000000002,1.50346,1.1448255000000001,1.4845845,2.8616779999999999,3.438542,1.4960599999999999,0.13852100000000001,1.12595,3.4705900000000001,2.8296299999999999,780.28700000000003,22.561599999999999