From f70523c42b2f646189791b6d755b26d8303d255f Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Tue, 16 Jan 2024 11:53:33 +0100 Subject: [PATCH 1/4] WP5 indiv process testing: code style improvements - avoid in-place dict mutations - add type hints - use more keyword args for call robustness - more clearly mark internal helpers as private - finetune exception handling - finetune pytest asserts (where default value rendering should be good enough) --- .../lib/process_runner/base.py | 4 +- .../lib/process_runner/dask.py | 2 +- .../tests/processes/processing/conftest.py | 1 + .../processes/processing/test_example.py | 228 +++++++++++------- 4 files changed, 143 insertions(+), 92 deletions(-) diff --git a/src/openeo_test_suite/lib/process_runner/base.py b/src/openeo_test_suite/lib/process_runner/base.py index 8af4d8f..51ba392 100644 --- a/src/openeo_test_suite/lib/process_runner/base.py +++ b/src/openeo_test_suite/lib/process_runner/base.py @@ -38,7 +38,7 @@ def encode_labeled_array(self, data: Dict) -> Any: to the internal backend representation openEO process tests. specification -> backend """ - raise Exception("labeled arrays not implemented yet") + raise NotImplementedError("labeled arrays not implemented yet") def encode_datacube(self, data: Dict) -> Any: """ @@ -46,7 +46,7 @@ def encode_datacube(self, data: Dict) -> Any: internal backend representation openEO process tests. specification -> backend """ - raise Exception("datacubes not implemented yet") + raise NotImplementedError("datacubes not implemented yet") def encode_data(self, data: Any) -> Any: """ diff --git a/src/openeo_test_suite/lib/process_runner/dask.py b/src/openeo_test_suite/lib/process_runner/dask.py index a0bfa7f..b107ce9 100644 --- a/src/openeo_test_suite/lib/process_runner/dask.py +++ b/src/openeo_test_suite/lib/process_runner/dask.py @@ -79,7 +79,7 @@ def encode_data(self, data): try: return BoundingBox(**data) except Exception as e: - raise Exception("Failed to parse bounding box: {}".format(str(e))) + raise ValueError("Failed to parse bounding box") from e return data diff --git a/src/openeo_test_suite/tests/processes/processing/conftest.py b/src/openeo_test_suite/tests/processes/processing/conftest.py index e81a21e..8873286 100644 --- a/src/openeo_test_suite/tests/processes/processing/conftest.py +++ b/src/openeo_test_suite/tests/processes/processing/conftest.py @@ -40,6 +40,7 @@ def auto_authenticate() -> bool: def connection( backend_url: str, runner: str, auto_authenticate: bool, pytestconfig ) -> ProcessTestRunner: + # TODO: avoid overriding existing "connection" fixture with a different kind of object (ProcessTestRunner) if runner == "dask": from openeo_test_suite.lib.process_runner.dask import Dask diff --git a/src/openeo_test_suite/tests/processes/processing/test_example.py b/src/openeo_test_suite/tests/processes/processing/test_example.py index 60611dd..e5b5a87 100644 --- a/src/openeo_test_suite/tests/processes/processing/test_example.py +++ b/src/openeo_test_suite/tests/processes/processing/test_example.py @@ -1,19 +1,29 @@ +# TODO rename this module (`test_example.py` was originally just meant as an example) + +import logging import math -import warnings -from pathlib import Path, posixpath +from pathlib import Path +from typing import List, Optional, Tuple, Union import json5 import pytest -import xarray as xr from deepdiff import DeepDiff +import openeo_test_suite +from openeo_test_suite.lib.process_runner.base import ProcessTestRunner from openeo_test_suite.lib.process_runner.util import isostr_to_datetime -# glob path to the test files -examples_path = "assets/processes/tests/*.json5" +_log = logging.getLogger(__name__) + + +DEFAULT_EXAMPLES_ROOT = ( + Path(openeo_test_suite.__file__).parents[2] / "assets/processes/tests" +) -def get_prop(prop, data, test, default=None): +def _get_prop(prop: str, data: dict, test: dict, *, default=None): + """Get property from example data, first trying test data, then full process data, then general fallback value.""" + # TODO make this function more generic (e.g. `multidict_get(key, *dicts, default=None)`) if prop in test: level = test[prop] elif prop in data: @@ -23,32 +33,37 @@ def get_prop(prop, data, test, default=None): return level -def get_examples(): +def get_examples( + root: Union[str, Path] = DEFAULT_EXAMPLES_ROOT +) -> List[Tuple[str, dict, Path, str, bool]]: + """Collect process examples/tests from examples root folder containing JSON5 files.""" + # TODO return a list of NamedTuples? examples = [] - package_root_folder = Path(__file__).parents[5] - files = package_root_folder.glob(examples_path) - for file in files: - id = file.stem + # TODO: it's not recommended use `file` (a built-in) as variable name. `path` would be better. + for file in root.glob("*.json5"): + process_id = file.stem try: with file.open() as f: data = json5.load(f) - for test in data["tests"]: - level = get_prop("level", data, test, "L4") - experimental = get_prop("experimental", data, test, False) - examples.append([id, test, file, level, experimental]) + for test in data["tests"]: + level = _get_prop("level", data, test, default="L4") + experimental = _get_prop("experimental", data, test, default=False) + examples.append((process_id, test, file, level, experimental)) except Exception as e: - warnings.warn("Failed to load {} due to {}".format(file, e)) + _log.warning(f"Failed to load {file}: {e}") return examples -@pytest.mark.parametrize("id,example,file,level, experimental", get_examples()) +@pytest.mark.parametrize( + ["process_id", "example", "file", "level", "experimental"], get_examples() +) def test_process( connection, skip_experimental, process_levels, processes, - id, + process_id, example, file, level, @@ -56,16 +71,16 @@ def test_process( skipper, ): if skip_experimental and experimental: - pytest.skip("Skipping experimental process {}".format(id)) + pytest.skip(f"Skipping experimental process {process_id}") skipper.skip_if_unmatching_process_level(level) - if id not in processes: + if process_id not in processes: pytest.skip( - "Skipping process {id!r} because it is not in the specified processes" + f"Skipping process {process_id!r} because it is not in the specified processes" ) # check whether the process is available - skipper.skip_if_unsupported_process([id]) + skipper.skip_if_unsupported_process([process_id]) # check whether any additionally required processes are available if "required" in example: @@ -74,22 +89,30 @@ def test_process( # prepare the arguments from test JSON encoding to internal backend representations # or skip if not supported by the test runner try: - arguments = prepare_arguments(example["arguments"], id, connection, file) + arguments = _prepare_arguments( + arguments=example["arguments"], + process_id=process_id, + connection=connection, + file=file, + ) except Exception as e: - # TODO: skipping on a generic `Exception` is very liberal and might hide real issues - pytest.skip(str(e)) + # TODO: originally there was a `pytest.skip()` here, but that was too liberal, hiding real issues. + # On what precise conditions should we skip? e.g. NotImplementedError? + # pytest.skip(str(e)) + raise - throws = bool(example["throws"]) if "throws" in example else False + throws = bool(example.get("throws")) returns = "returns" in example # execute the process try: - result = connection.execute(id, arguments) + result = connection.execute(process_id, arguments) except Exception as e: result = e # check the process results / behavior if throws and returns: + # TODO what does it mean if test can both throw and return? if isinstance(result, Exception): check_exception(example, result) else: @@ -102,25 +125,31 @@ def test_process( # TODO: skipping at this point of test is a bit useless. # Instead: skip earlier, or just consider the test as passed? pytest.skip( - "Test for process {} doesn't provide an expected result for arguments: {}".format( - id, example["arguments"] - ) + f"Test for process {process_id} doesn't provide an expected result for arguments: {example['arguments']}" ) -def prepare_arguments(arguments, process_id, connection, file): - for name in arguments: - arguments[name] = prepare_argument( - arguments[name], process_id, name, connection, file +def _prepare_arguments( + arguments: dict, process_id: str, connection: ProcessTestRunner, file: Path +) -> dict: + return { + k: _prepare_argument( + arg=v, process_id=process_id, name=k, connection=connection, file=file ) + for k, v in arguments.items() + } - return arguments - -def prepare_argument(arg, process_id, name, connection, file): +def _prepare_argument( + arg: Union[dict, str, int, float], + process_id: str, + name: str, + connection: ProcessTestRunner, + file: Path, +): # handle external references to files if isinstance(arg, dict) and "$ref" in arg: - arg = load_ref(arg["$ref"], file) + arg = _load_ref(arg["$ref"], file) # handle custom types of data if isinstance(arg, dict): @@ -134,17 +163,36 @@ def prepare_argument(arg, process_id, name, connection, file): # nodata-values elif arg["type"] == "nodata": arg = connection.get_nodata_value() + else: + # TODO: raise NotImplementedError? + _log.warning(f"Unhandled argument type: {arg}") elif "process_graph" in arg: - arg = connection.encode_process_graph(arg, process_id, name) + arg = connection.encode_process_graph( + process=arg, parent_process_id=process_id, parent_parameter=name + ) else: - for key in arg: - arg[key] = prepare_argument( - arg[key], process_id, name, connection, file + arg = { + k: _prepare_argument( + arg=v, + process_id=process_id, + name=name, + connection=connection, + file=file, ) + for k, v in arg.items() + } elif isinstance(arg, list): - for i in range(len(arg)): - arg[i] = prepare_argument(arg[i], process_id, name, connection, file) + arg = [ + _prepare_argument( + arg=a, + process_id=process_id, + name=name, + connection=connection, + file=file, + ) + for a in arg + ] arg = connection.encode_data(arg) @@ -154,48 +202,60 @@ def prepare_argument(arg, process_id, name, connection, file): return arg -def prepare_results(connection, file, example, result=None): +def _prepare_results(connection: ProcessTestRunner, file: Path, example, result=None): # go through the example and result recursively and convert datetimes to iso strings # could be used for more conversions in the future... if isinstance(example, dict): # handle external references to files if isinstance(example, dict) and "$ref" in example: - example = load_ref(example["$ref"], file) + example = _load_ref(example["$ref"], file) if "type" in example: if example["type"] == "datetime": example = isostr_to_datetime(example["value"]) try: result = isostr_to_datetime(result) - except: + except Exception: pass elif example["type"] == "nodata": example = connection.get_nodata_value() else: + # TODO: avoid in-place dict mutation for key in example: if key not in result: - (example[key], _) = prepare_results(connection, file, example[key]) + (example[key], _) = _prepare_results( + connection=connection, file=file, example=example[key] + ) else: - (example[key], result[key]) = prepare_results( - connection, file, example[key], result[key] + (example[key], result[key]) = _prepare_results( + connection=connection, + file=file, + example=example[key], + result=result[key], ) elif isinstance(example, list): + # TODO: avoid in-place list mutation for i in range(len(example)): if i >= len(result): - (example[i], _) = prepare_results(connection, file, example[i]) + (example[i], _) = _prepare_results( + connection=connection, file=file, example=example[i] + ) else: - (example[i], result[i]) = prepare_results( - connection, file, example[i], result[i] + (example[i], result[i]) = _prepare_results( + connection=connection, + file=file, + example=example[i], + result=result[i], ) return (example, result) -def load_ref(ref, file): +def _load_ref(ref: str, file: Path): try: - path = posixpath.join(file.parent, ref) + path = file.parent / ref if ref.endswith(".json") or ref.endswith(".json5") or ref.endswith(".geojson"): with open(path) as f: return json5.load(f) @@ -203,61 +263,56 @@ def load_ref(ref, file): with open(path) as f: return f.read() else: - raise Exception( + raise NotImplementedError( "External references to files with the given extension not implemented yet." ) except Exception as e: - raise Exception("Failed to load external reference {}: {}".format(ref, e)) + # TODO: is this try-except actually useful? + raise Exception(f"Failed to load external reference {ref}") from e def check_non_json_values(value): + # TODO: shouldn't this check be an aspect of Http(ProcessTestRunner)? if isinstance(value, float): if math.isnan(value): - raise Exception("HTTP JSON APIs don't support NaN values") + raise ValueError("HTTP JSON APIs don't support NaN values") elif math.isinf(value): - raise Exception("HTTP JSON APIs don't support infinity values") + raise ValueError("HTTP JSON APIs don't support infinity values") elif isinstance(value, dict): - for key in value: - check_non_json_values(value[key]) + for v in value.values(): + check_non_json_values(v) elif isinstance(value, list): for item in value: check_non_json_values(item) def check_exception(example, result): - assert isinstance(result, Exception), "Excpected an exception, but got {}".format( - result - ) + assert isinstance(result, Exception) if isinstance(example["throws"], str): if result.__class__.__name__ != example["throws"]: - warnings.warn( - "Expected exception {} but got {}".format( - example["throws"], result.__class__.__name__ - ) + # TODO: better way to report this warning? + _log.warning( + f"Expected exception {example['throws']} but got {result.__class__}" ) # todo: we should enable this end remove the two lines above, but right now tooling doesn't really implement this # assert result.__class__.__name__ == example["throws"] def check_return_value(example, result, connection, file): - assert not isinstance(result, Exception), "Unexpected exception: {} ".format( - str(result) - ) + assert not isinstance(result, Exception) # handle custom types of data result = connection.decode_data(result, example["returns"]) # decode special types (currently mostly datetimes and nodata) - (example["returns"], result) = prepare_results( - connection, file, example["returns"], result + (example["returns"], result) = _prepare_results( + connection=connection, file=file, example=example["returns"], result=result ) - delta = example["delta"] if "delta" in example else 0.0000000001 + delta = example.get("delta", 0.0000000001) if isinstance(example["returns"], dict): - assert isinstance(result, dict), "Expected a dict but got {}".format( - type(result) - ) + assert isinstance(result, dict) exclude_regex_paths = [] exclude_paths = [] ignore_order_func = None @@ -282,11 +337,9 @@ def check_return_value(example, result, connection, file): exclude_regex_paths=exclude_regex_paths, ignore_order_func=ignore_order_func, ) - assert {} == diff, "Differences: {}".format(str(diff)) + assert {} == diff, f"Differences: {diff!s}" elif isinstance(example["returns"], list): - assert isinstance(result, list), "Expected a list but got {}".format( - type(result) - ) + assert isinstance(result, list) diff = DeepDiff( example["returns"], result, @@ -294,17 +347,14 @@ def check_return_value(example, result, connection, file): ignore_numeric_type_changes=True, ignore_nan_inequality=True, ) - assert {} == diff, "Differences: {}".format(str(diff)) + assert {} == diff, f"Differences: {diff!s}" elif isinstance(example["returns"], float) and math.isnan(example["returns"]): - assert math.isnan(result), "Got {} instead of NaN".format(result) + assert math.isnan(result), f"Got {result} instead of NaN" elif isinstance(example["returns"], float) or isinstance(example["returns"], int): - msg = "Expected a numerical result but got {} of type {}".format( - result, type(result) - ) + msg = f"Expected a numerical result but got {result} of type {type(result)}" assert isinstance(result, float) or isinstance(result, int), msg assert not math.isnan(result), "Got unexpected NaN as result" # handle numerical data with a delta - assert result == pytest.approx(example["returns"], delta) + assert result == pytest.approx(example["returns"], rel=delta) else: - msg = "Expected {} but got {}".format(example["returns"], result) - assert result == example["returns"], msg + assert result == example["returns"] From b0558a5533f52d25a3632b0f2876d46627085a60 Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Tue, 16 Jan 2024 19:26:31 +0100 Subject: [PATCH 2/4] PR #21 restore some original logic and assert messages --- .../tests/processes/processing/test_example.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/openeo_test_suite/tests/processes/processing/test_example.py b/src/openeo_test_suite/tests/processes/processing/test_example.py index e5b5a87..5d0c740 100644 --- a/src/openeo_test_suite/tests/processes/processing/test_example.py +++ b/src/openeo_test_suite/tests/processes/processing/test_example.py @@ -96,10 +96,9 @@ def test_process( file=file, ) except Exception as e: - # TODO: originally there was a `pytest.skip()` here, but that was too liberal, hiding real issues. + # TODO: this `except: pytest.skip()` is overly liberal, possibly hiding real issues. # On what precise conditions should we skip? e.g. NotImplementedError? - # pytest.skip(str(e)) - raise + pytest.skip(str(e)) throws = bool(example.get("throws")) returns = "returns" in example @@ -287,7 +286,7 @@ def check_non_json_values(value): def check_exception(example, result): - assert isinstance(result, Exception) + assert isinstance(result, Exception), f"Expected an exception, but got {result}" if isinstance(example["throws"], str): if result.__class__.__name__ != example["throws"]: # TODO: better way to report this warning? @@ -299,7 +298,7 @@ def check_exception(example, result): def check_return_value(example, result, connection, file): - assert not isinstance(result, Exception) + assert not isinstance(result, Exception), f"Unexpected exception: {result}" # handle custom types of data result = connection.decode_data(result, example["returns"]) @@ -312,7 +311,7 @@ def check_return_value(example, result, connection, file): delta = example.get("delta", 0.0000000001) if isinstance(example["returns"], dict): - assert isinstance(result, dict) + assert isinstance(result, dict), f"Expected a dict but got {type(result)}" exclude_regex_paths = [] exclude_paths = [] ignore_order_func = None @@ -339,7 +338,7 @@ def check_return_value(example, result, connection, file): ) assert {} == diff, f"Differences: {diff!s}" elif isinstance(example["returns"], list): - assert isinstance(result, list) + assert isinstance(result, list), f"Expected a list but got {type(result)}" diff = DeepDiff( example["returns"], result, @@ -357,4 +356,5 @@ def check_return_value(example, result, connection, file): # handle numerical data with a delta assert result == pytest.approx(example["returns"], rel=delta) else: - assert result == example["returns"] + msg = f"Expected {example['returns']} but got {result}" + assert result == example["returns"], msg From 57792188c87d49f576fa4be117edfa1978d112ef Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Tue, 16 Jan 2024 19:27:27 +0100 Subject: [PATCH 3/4] Improve math.isnan check --- .../tests/processes/processing/test_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openeo_test_suite/tests/processes/processing/test_example.py b/src/openeo_test_suite/tests/processes/processing/test_example.py index 5d0c740..3bb7995 100644 --- a/src/openeo_test_suite/tests/processes/processing/test_example.py +++ b/src/openeo_test_suite/tests/processes/processing/test_example.py @@ -348,7 +348,7 @@ def check_return_value(example, result, connection, file): ) assert {} == diff, f"Differences: {diff!s}" elif isinstance(example["returns"], float) and math.isnan(example["returns"]): - assert math.isnan(result), f"Got {result} instead of NaN" + assert isinstance(result, float) and math.isnan(result), f"Got {result} instead of NaN" elif isinstance(example["returns"], float) or isinstance(example["returns"], int): msg = f"Expected a numerical result but got {result} of type {type(result)}" assert isinstance(result, float) or isinstance(result, int), msg From 750740741888448fd76e36fda3cdaac341a451c7 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Wed, 17 Jan 2024 11:38:12 +0100 Subject: [PATCH 4/4] Fix temporal issues and some other improvements --- assets/processes | 2 +- .../lib/process_runner/util.py | 56 +++++++++++++++---- .../processes/processing/test_example.py | 18 +++--- 3 files changed, 54 insertions(+), 22 deletions(-) diff --git a/assets/processes b/assets/processes index 91ad78a..d364059 160000 --- a/assets/processes +++ b/assets/processes @@ -1 +1 @@ -Subproject commit 91ad78a658346f8dee0b91da6e73ac13eeadf3f1 +Subproject commit d36405932238f78bc5e4bbb43abd5c9c22f18bfb diff --git a/src/openeo_test_suite/lib/process_runner/util.py b/src/openeo_test_suite/lib/process_runner/util.py index b840a00..0c8a021 100644 --- a/src/openeo_test_suite/lib/process_runner/util.py +++ b/src/openeo_test_suite/lib/process_runner/util.py @@ -1,8 +1,13 @@ +import re from datetime import datetime, timezone import numpy as np +import pandas as pd import xarray as xr -from dateutil.parser import parse +from dateutil.parser import isoparse +from pandas._libs.tslibs.timestamps import Timestamp + +ISO8601_REGEX = r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}" def numpy_to_native(data, expected): @@ -27,10 +32,17 @@ def datacube_to_xarray(cube): for name in cube["order"]: dim = cube["dimensions"][name] if dim["type"] == "temporal": - # date replace for older Python versions that don't support ISO parsing (only available since 3.11) values = [ - datetime.fromisoformat(date.replace("Z", "")) for date in dim["values"] + isostr_to_datetime(date, fail_on_error=False) for date in dim["values"] ] + # Verify that the values are all datetimes, otherwise likely the tests are invalid + if all(isinstance(date, datetime) for date in values): + # Ot looks like xarray does not support creating proper time dimensions from datetimes, + # so we convert to np.datetime64 explicitly. + # np.datetime64 doesn't like timezone-aware datetimes, so we remove the timezone. + values = [np.datetime64(dt.replace(tzinfo=None), "ns") for dt in values] + else: + raise Exception("Mixed datetime types in temporal dimension") elif dim["type"] == "spatial": values = dim["values"] if "reference_system" in dim: @@ -60,7 +72,8 @@ def xarray_to_datacube(data): type = "bands" values = [] axis = None - if np.issubdtype(data.coords[c].dtype, np.datetime64): + dtype = data.coords[c].dtype + if np.issubdtype(dtype, np.datetime64) or isinstance(dtype, Timestamp): type = "temporal" values = [datetime_to_isostr(date) for date in data.coords[c].values] else: @@ -71,6 +84,8 @@ def xarray_to_datacube(data): elif c == "y": # todo: non-standardized type = "spatial" axis = "y" + elif c == "t": # todo: non-standardized + type = "temporal" dim = {"type": type, "values": values} if axis is not None: @@ -93,15 +108,36 @@ def xarray_to_datacube(data): return cube -def isostr_to_datetime(dt): - return parse(dt) +def isostr_to_datetime(dt, fail_on_error=True): + if not fail_on_error: + try: + return isostr_to_datetime(dt) + except: + return dt + else: + if re.match(ISO8601_REGEX, dt): + return isoparse(dt) + else: + raise Exception( + "Datetime is not in ISO format (YYYY-MM-DDThh:mm:ss plus timezone))" + ) def datetime_to_isostr(dt): - # Convert numpy.datetime64 to timestamp (in seconds) - timestamp = dt.astype("datetime64[s]").astype(int) - # Create a datetime object from the timestamp - dt_object = datetime.utcfromtimestamp(timestamp).replace(tzinfo=timezone.utc) + if isinstance(dt, Timestamp): + dt_object = dt.to_pydatetime() + elif isinstance(dt, np.datetime64): + # Convert numpy.datetime64 to timestamp (in seconds) + timestamp = dt.astype("datetime64[s]").astype(int) + # Create a datetime object from the timestamp + dt_object = datetime.utcfromtimestamp(timestamp).replace(tzinfo=timezone.utc) + elif isinstance(dt, datetime): + dt_object = dt + elif re.match(ISO8601_REGEX, dt): + return dt + else: + raise NotImplementedError("Unsupported datetime type") + # Convert to ISO format string return dt_object.isoformat().replace("+00:00", "Z") diff --git a/src/openeo_test_suite/tests/processes/processing/test_example.py b/src/openeo_test_suite/tests/processes/processing/test_example.py index 1e16bb3..43490b8 100644 --- a/src/openeo_test_suite/tests/processes/processing/test_example.py +++ b/src/openeo_test_suite/tests/processes/processing/test_example.py @@ -3,7 +3,7 @@ import logging import math from pathlib import Path -from typing import List, Optional, Tuple, Union +from typing import List, Tuple, Union import json5 import pytest @@ -95,9 +95,7 @@ def test_process( connection=connection, file=file, ) - except Exception as e: - # TODO: this `except: pytest.skip()` is overly liberal, possibly hiding real issues. - # On what precise conditions should we skip? e.g. NotImplementedError? + except NotImplementedError as e: pytest.skip(str(e)) throws = bool(example.get("throws")) @@ -111,7 +109,6 @@ def test_process( # check the process results / behavior if throws and returns: - # TODO what does it mean if test can both throw and return? if isinstance(result, Exception): check_exception(example, result) else: @@ -121,8 +118,6 @@ def test_process( elif returns: check_return_value(example, result, connection, file) else: - # TODO: skipping at this point of test is a bit useless. - # Instead: skip earlier, or just consider the test as passed? pytest.skip( f"Test for process {process_id} doesn't provide an expected result for arguments: {example['arguments']}" ) @@ -288,13 +283,12 @@ def check_non_json_values(value): def check_exception(example, result): assert isinstance(result, Exception), f"Expected an exception, but got {result}" if isinstance(example["throws"], str): + # todo: we should assert here and remove the warning, but right now tooling doesn't really implement this + # assert result.__class__.__name__ == example["throws"] if result.__class__.__name__ != example["throws"]: - # TODO: better way to report this warning? _log.warning( f"Expected exception {example['throws']} but got {result.__class__}" ) - # todo: we should enable this end remove the two lines above, but right now tooling doesn't really implement this - # assert result.__class__.__name__ == example["throws"] def check_return_value(example, result, connection, file): @@ -348,7 +342,9 @@ def check_return_value(example, result, connection, file): ) assert {} == diff, f"Differences: {diff!s}" elif isinstance(example["returns"], float) and math.isnan(example["returns"]): - assert isinstance(result, float) and math.isnan(result), f"Got {result} instead of NaN" + assert isinstance(result, float) and math.isnan( + result + ), f"Got {result} instead of NaN" elif isinstance(example["returns"], float) or isinstance(example["returns"], int): msg = f"Expected a numerical result but got {result} of type {type(result)}" assert isinstance(result, float) or isinstance(result, int), msg