Skip to content

Commit ef87183

Browse files
znichollsznicholls
authored
Merge pull request #88 from openscm/86-pytest-regressions
86 pytest regressions
2 parents e039378 + b3c8cbf commit ef87183

23 files changed

+648
-860
lines changed

‎changelog/88.improvement.md

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Switched to using [pytest-regressions](https://pytest-regressions.readthedocs.io/en/latest) rather than relying on our own solution

‎poetry.lock

+39-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pyproject.toml

+1
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ models = ["pandas", "ciceroscm", "fair", "pymagicc"]
6868

6969
[tool.poetry.group.tests.dependencies]
7070
pytest = "^7.3.1"
71+
pytest-regressions = "^2.5.0"
7172

7273
[tool.poetry.group.docs.dependencies]
7374
myst-nb = "^0.17.0"

‎src/openscm_runner/testing.py

+33-42
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,40 @@
11
"""
22
Miscellaneous testing code
33
"""
4-
import json
54
from abc import ABC, abstractmethod
65

6+
import numpy as np
77
import numpy.testing as npt
88
from scmdata import ScmRun
99

1010
from openscm_runner import run
1111

12-
try:
13-
import pytest
1412

15-
HAS_PYTEST = True
16-
17-
except ImportError:
18-
HAS_PYTEST = False
19-
20-
21-
def _check_output( # pylint: disable=too-many-locals,too-many-branches
22-
res, expected_output_file, rtol, update
13+
def _get_output_dict( # pylint: disable=too-many-locals,too-many-branches
14+
res, outputs_to_get
2315
):
24-
if not HAS_PYTEST:
25-
raise ImportError("pytest not installed, run `pip install pytest`")
16+
"""
17+
Get output dictionary
2618
27-
with open(expected_output_file, encoding="ascii") as filehandle:
28-
expected_output = json.load(filehandle)
19+
Returns a dictionary of outputs which can be used with our regression testing.
2920
30-
updated_output = {}
21+
Parameters
22+
----------
23+
res
24+
Results from which to extract outputs
3125
32-
for climate_model, checks in expected_output.items():
33-
res_cm = res.filter(climate_model=climate_model)
26+
outputs_to_get
27+
Outputs to get from the results
3428
35-
updated_output[climate_model] = []
29+
Returns
30+
-------
31+
Dictionary of results which can be understood by our regression tests
32+
"""
33+
output = {}
34+
for climate_model, checks in outputs_to_get.items():
35+
res_cm = res.filter(climate_model=climate_model)
3636

37-
for filter_kwargs, expected_val in checks:
37+
for filter_kwargs in checks:
3838
err_msg = f"{filter_kwargs}"
3939
filter_kwargs_in = {**filter_kwargs}
4040
check_units = filter_kwargs.pop("unit", None)
@@ -53,25 +53,16 @@ def _check_output( # pylint: disable=too-many-locals,too-many-branches
5353
).values
5454
)
5555

56-
try:
57-
npt.assert_allclose(
58-
res_val,
59-
expected_val,
60-
rtol=rtol,
61-
err_msg=err_msg,
62-
)
63-
new_val = expected_val
64-
except AssertionError:
65-
new_val = res_val
66-
if not update:
67-
raise
56+
key = ["climate_model", climate_model]
57+
for k, v in filter_kwargs_in.items():
58+
key.append(k)
59+
key.append(v)
6860

69-
updated_output[climate_model].append((filter_kwargs_in, new_val))
70-
if update:
71-
with open(expected_output_file, "w", encoding="ascii") as file_handle:
72-
json.dump(updated_output, file_handle, indent=4)
61+
# key = tuple(key)
62+
key = "_".join([str(v) for v in key])
63+
output[key] = np.array([res_val])
7364

74-
pytest.skip(f"Updated {expected_output_file}")
65+
return output
7566

7667

7768
class _AdapterTester(ABC): # nosec
@@ -99,13 +90,13 @@ class _AdapterTester(ABC): # nosec
9990
"Effective Radiative Forcing|Aerosols",
10091
"Effective Radiative Forcing|Aerosols|Direct Effect",
10192
"Effective Radiative Forcing|Aerosols|Direct Effect|BC",
102-
"Effective Radiative Forcing|Aerosols|Direct Effect|BC|MAGICC Fossil and Industrial",
93+
"Effective Radiative Forcing|Aerosols|Direct Effect|BC|MAGICC Fossil and Industrial", # noqa: E501
10394
"Effective Radiative Forcing|Aerosols|Direct Effect|BC|MAGICC AFOLU",
10495
"Effective Radiative Forcing|Aerosols|Direct Effect|OC",
105-
"Effective Radiative Forcing|Aerosols|Direct Effect|OC|MAGICC Fossil and Industrial",
96+
"Effective Radiative Forcing|Aerosols|Direct Effect|OC|MAGICC Fossil and Industrial", # noqa: E501
10697
"Effective Radiative Forcing|Aerosols|Direct Effect|OC|MAGICC AFOLU",
10798
"Effective Radiative Forcing|Aerosols|Direct Effect|SOx",
108-
"Effective Radiative Forcing|Aerosols|Direct Effect|SOx|MAGICC Fossil and Industrial",
99+
"Effective Radiative Forcing|Aerosols|Direct Effect|SOx|MAGICC Fossil and Industrial", # noqa: E501
109100
"Effective Radiative Forcing|Aerosols|Direct Effect|SOx|MAGICC AFOLU",
110101
"Effective Radiative Forcing|Aerosols|Indirect Effect",
111102
"Effective Radiative Forcing|Greenhouse Gases",
@@ -143,8 +134,8 @@ def _check_res(self, exp, check_val, raise_error):
143134

144135
print(f"exp: {exp}, check_val: {check_val}")
145136

146-
def _check_output(self, res, expected_output_file, update):
147-
_check_output(res, expected_output_file, self._rtol, update)
137+
def _get_output_dict(self, res, outputs_to_get):
138+
return _get_output_dict(res, outputs_to_get)
148139

149140
@staticmethod
150141
def _check_heat_content_heat_uptake_consistency(res):

‎tests/conftest.py

+4-16
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
"""
66
from __future__ import annotations
77

8+
import subprocess
89
from pathlib import Path
910

1011
import pandas as pd
@@ -26,9 +27,10 @@
2627
else:
2728
CORRECT_MAGICC_IS_AVAILABLE = True
2829

29-
except KeyError:
30+
except (KeyError, subprocess.CalledProcessError) as exc:
3031
MAGICC_VERSION = None
3132
CORRECT_MAGICC_IS_AVAILABLE = False
33+
MAGICC_LOAD_EXC = exc
3234

3335

3436
CICEROSCM_IS_AVAILABLE = False
@@ -45,7 +47,7 @@ def pytest_runtest_setup(item) -> None:
4547
for mark in item.iter_markers():
4648
if mark.name == "magicc" and not CORRECT_MAGICC_IS_AVAILABLE:
4749
if MAGICC_VERSION is None:
48-
pytest.skip("MAGICC7 not available")
50+
pytest.skip(f"MAGICC7 not available. Exc: {MAGICC_LOAD_EXC}")
4951
else:
5052
pytest.skip(
5153
"Wrong MAGICC version for tests ({}), we require {}".format(
@@ -106,17 +108,3 @@ def test_scenario_ssp370_world(test_data_dir: Path) -> scmdata.ScmRun:
106108
)
107109

108110
return scenario
109-
110-
111-
def pytest_addoption(parser):
112-
parser.addoption(
113-
"--update-expected-values",
114-
action="store_true",
115-
default=False,
116-
help="Overwrite expected values",
117-
)
118-
119-
120-
@pytest.fixture
121-
def update_expected_values(request):
122-
return request.config.getoption("--update-expected-values")

‎tests/integration/test_ciceroSCM.py

+132-10
Original file line numberDiff line numberDiff line change
@@ -20,16 +20,9 @@ class TestCICEROSCMAdapter(_AdapterTester):
2020
def test_run(
2121
self,
2222
test_scenarios,
23-
test_data_dir,
24-
update_expected_values,
23+
num_regression,
2524
shuffle_column_order,
2625
):
27-
expected_output_file = os.path.join(
28-
test_data_dir,
29-
"expected-integration-output",
30-
"expected_ciceroscm_test_run_output.json",
31-
)
32-
3326
if shuffle_column_order:
3427
tmp = test_scenarios.long_data()
3528
cols = tmp.columns.tolist()
@@ -242,14 +235,143 @@ def test_run(
242235
scenario="ssp245",
243236
run_id=1,
244237
)
245-
# ch
238+
246239
# check that jump in GHG ERF isn't there
247240
assert (
248241
ssp245_ch4_conc_2014.values.squeeze()
249242
- ssp245_ch4_conc_2015.values.squeeze()
250243
) < 0.1
251244

252-
self._check_output(res, expected_output_file, update_expected_values)
245+
outputs_to_get = {
246+
"CICERO-SCM": [
247+
{
248+
"variable": "Heat Content|Ocean",
249+
"unit": "ZJ",
250+
"region": "World",
251+
"year": 2100,
252+
"scenario": "ssp126",
253+
"quantile": 1,
254+
},
255+
{
256+
"variable": "Surface Air Temperature Change",
257+
"region": "World",
258+
"year": 2100,
259+
"scenario": "ssp126",
260+
"quantile": 1,
261+
},
262+
{
263+
"variable": "Surface Air Temperature Change",
264+
"region": "World",
265+
"year": 2100,
266+
"scenario": "ssp126",
267+
"quantile": 0.05,
268+
},
269+
{
270+
"variable": "Surface Air Temperature Change",
271+
"region": "World",
272+
"year": 2100,
273+
"scenario": "ssp126",
274+
"quantile": 0.95,
275+
},
276+
{
277+
"variable": "Surface Air Temperature Change",
278+
"region": "World",
279+
"year": 2100,
280+
"scenario": "ssp370",
281+
"quantile": 0.05,
282+
},
283+
{
284+
"variable": "Surface Air Temperature Change",
285+
"region": "World",
286+
"year": 2100,
287+
"scenario": "ssp370",
288+
"quantile": 0.95,
289+
},
290+
{
291+
"variable": "Surface Air Ocean Blended Temperature Change",
292+
"region": "World",
293+
"year": 2100,
294+
"scenario": "ssp126",
295+
"quantile": 1,
296+
},
297+
{
298+
"variable": "Heat Uptake",
299+
"unit": "W/m^2",
300+
"region": "World",
301+
"year": 2100,
302+
"scenario": "ssp126",
303+
"quantile": 1,
304+
},
305+
{
306+
"variable": "Surface Air Temperature Change",
307+
"region": "World",
308+
"year": 2100,
309+
"scenario": "ssp126",
310+
"quantile": 0,
311+
},
312+
{
313+
"variable": "Surface Air Temperature Change",
314+
"region": "World",
315+
"year": 2100,
316+
"scenario": "ssp370",
317+
"quantile": 1,
318+
},
319+
{
320+
"variable": "Surface Air Temperature Change",
321+
"region": "World",
322+
"year": 2100,
323+
"scenario": "ssp370",
324+
"quantile": 0,
325+
},
326+
{
327+
"variable": "Surface Air Temperature Change",
328+
"region": "World",
329+
"year": 2100,
330+
"scenario": "ssp126",
331+
"quantile": 0.05,
332+
},
333+
{
334+
"variable": "Surface Air Temperature Change",
335+
"region": "World",
336+
"year": 2100,
337+
"scenario": "ssp126",
338+
"quantile": 0.95,
339+
},
340+
{
341+
"variable": "Surface Air Temperature Change",
342+
"region": "World",
343+
"year": 2100,
344+
"scenario": "ssp370",
345+
"quantile": 0.05,
346+
},
347+
{
348+
"variable": "Surface Air Temperature Change",
349+
"region": "World",
350+
"year": 2100,
351+
"scenario": "ssp370",
352+
"quantile": 0.95,
353+
},
354+
{
355+
"variable": "Atmospheric Concentrations|CO2",
356+
"region": "World",
357+
"year": 2100,
358+
"scenario": "ssp370",
359+
"unit": "ppm",
360+
"quantile": 1,
361+
},
362+
{
363+
"variable": "Emissions|CO2",
364+
"region": "World",
365+
"year": 2100,
366+
"scenario": "ssp370",
367+
"unit": "PgC / yr",
368+
"quantile": 1,
369+
},
370+
]
371+
}
372+
373+
output_dict = self._get_output_dict(res, outputs_to_get)
374+
num_regression.check(output_dict, default_tolerance=dict(rtol=self._rtol))
253375

254376
@pytest.mark.ciceroscm
255377
def test_variable_naming(self, test_scenarios):

‎tests/integration/test_ciceroSCM/test_run_False_.csv

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
,climate_model_CICERO-SCM_variable_Heat Content|Ocean_unit_ZJ_region_World_year_2100_scenario_ssp126_quantile_1,climate_model_CICERO-SCM_variable_Surface Air Temperature Change_region_World_year_2100_scenario_ssp126_quantile_1,climate_model_CICERO-SCM_variable_Surface Air Temperature Change_region_World_year_2100_scenario_ssp126_quantile_0.05,climate_model_CICERO-SCM_variable_Surface Air Temperature Change_region_World_year_2100_scenario_ssp126_quantile_0.95,climate_model_CICERO-SCM_variable_Surface Air Temperature Change_region_World_year_2100_scenario_ssp370_quantile_0.05,climate_model_CICERO-SCM_variable_Surface Air Temperature Change_region_World_year_2100_scenario_ssp370_quantile_0.95,climate_model_CICERO-SCM_variable_Surface Air Ocean Blended Temperature Change_region_World_year_2100_scenario_ssp126_quantile_1,climate_model_CICERO-SCM_variable_Heat Uptake_unit_W/m^2_region_World_year_2100_scenario_ssp126_quantile_1,climate_model_CICERO-SCM_variable_Surface Air Temperature Change_region_World_year_2100_scenario_ssp126_quantile_0,climate_model_CICERO-SCM_variable_Surface Air Temperature Change_region_World_year_2100_scenario_ssp370_quantile_1,climate_model_CICERO-SCM_variable_Surface Air Temperature Change_region_World_year_2100_scenario_ssp370_quantile_0,climate_model_CICERO-SCM_variable_Atmospheric Concentrations|CO2_region_World_year_2100_scenario_ssp370_unit_ppm_quantile_1,climate_model_CICERO-SCM_variable_Emissions|CO2_region_World_year_2100_scenario_ssp370_unit_PgC / yr_quantile_1
2+
0,1313.0500000000002,1.50346,1.1448255000000001,1.4845845,2.8616779999999999,3.438542,1.4960599999999999,0.13852100000000001,1.12595,3.4705900000000001,2.8296299999999999,780.28700000000003,22.561599999999999

0 commit comments

Comments
 (0)