From f75dcd4ffd8f985ee1332205c61978cc5261204b Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Mon, 16 Oct 2023 23:08:04 -0700 Subject: [PATCH] More advanced testing parameters --- tests/IVIMmodels/unit_tests/algorithms.json | 151 +++++++++++-------- tests/IVIMmodels/unit_tests/test_ivim_fit.py | 43 ++++-- 2 files changed, 120 insertions(+), 74 deletions(-) diff --git a/tests/IVIMmodels/unit_tests/algorithms.json b/tests/IVIMmodels/unit_tests/algorithms.json index 7ee606d..b77a6d2 100644 --- a/tests/IVIMmodels/unit_tests/algorithms.json +++ b/tests/IVIMmodels/unit_tests/algorithms.json @@ -9,94 +9,125 @@ "IAR_LU_subtracted" ], "IAR_LU_biexp": { - "xfail_names": [ - "Vein", - "Blood RV", - "Blood LV", - "Blood RA" - ], "tolerances": { - "f": 5, - "D": 5, - "Dp": 25 + "rtol": { + "f": 5, + "D": 5, + "Dp": 25 + }, + "atol": { + "f": 1e-2, + "D": 1e-2, + "Dp": 1e-1 + } } }, "IAR_LU_modified_mix": { - "xfail_names": [ - "Blood LV" - ], "tolerances": { - "f": 5, - "D": 5, - "Dp": 25 + "rtol": { + "f": 5, + "D": 5, + "Dp": 25 + }, + "atol": { + "f": 1e-2, + "D": 1e-2, + "Dp": 1e-1 + } } }, "IAR_LU_modified_topopro": { - "xfail_names": [ - "Vein", - "Blood RV", - "Blood LV", - "Blood RA" - ], "tolerances": { - "f": 5, - "D": 5, - "Dp": 25 + "rtol": { + "f": 5, + "D": 5, + "Dp": 25 + }, + "atol": { + "f": 1e-2, + "D": 1e-2, + "Dp": 1e-1 + } } }, "IAR_LU_segmented_2step": { - "xfail_names": [ - "Blood RV", - "Blood LV", - "Blood RA" - ], "tolerances": { - "f": 5, - "D": 5, - "Dp": 25 + "rtol": { + "f": 5, + "D": 5, + "Dp": 25 + }, + "atol": { + "f": 1e-2, + "D": 1e-2, + "Dp": 1e-1 + } } }, "IAR_LU_segmented_3step": { - "xfail_names": [ - "Blood RV", - "Blood LV", - "Blood RA" - ], "tolerances": { - "f": 5, - "D": 5, - "Dp": 25 + "rtol": { + "f": 5, + "D": 5, + "Dp": 25 + }, + "atol": { + "f": 1e-2, + "D": 1e-2, + "Dp": 1e-1 + } } }, "IAR_LU_subtracted": { - "xfail_names": [ - "Blood RV", - "Blood LV", - "Blood RA" - ], "tolerances": { - "f": 5, - "D": 5, - "Dp": 25 + "rtol": { + "f": 5, + "D": 5, + "Dp": 25 + }, + "atol": { + "f": 1e-2, + "D": 1e-2, + "Dp": 1e-1 + } } }, "ETP_SRI_LinearFitting": { - "xfail_names": [ - "Artery", - "Blood RV", - "Blood LV", - "myocardium RV", - "myocardium ra", - "Liver", - "Blood RA" - ], + "xfail_names": { + "Vein": true + }, "options": { "thresholds": [500] }, "tolerances": { - "f": 5, - "D": 5, - "Dp": 25 + "rtol": { + "f": 5, + "D": 5, + "Dp": 25 + }, + "atol": { + "f": 0, + "D": 0, + "Dp": 0 + }, + "dynamic_rtol": { + "offset": 0, + "noise": 0, + "ratio": 0, + "noiseCrossRatio": 10, + "f": 1, + "D": 1, + "Dp": 2 + }, + "dynamic_atol": { + "offset": 5e-2, + "noise": 0, + "ratio": 0, + "noiseCrossRatio": 10, + "f": 1, + "D": 1, + "Dp": 2 + } } } } \ No newline at end of file diff --git a/tests/IVIMmodels/unit_tests/test_ivim_fit.py b/tests/IVIMmodels/unit_tests/test_ivim_fit.py index b0a4198..3bd2b88 100644 --- a/tests/IVIMmodels/unit_tests/test_ivim_fit.py +++ b/tests/IVIMmodels/unit_tests/test_ivim_fit.py @@ -75,29 +75,44 @@ def data_ivim_fit_saved(): for name, data in all_data.items(): for algorithm in algorithms: algorithm_dict = algorithm_information.get(algorithm, {}) - xfail = name in algorithm_dict.get("xfail_names", []) + xfail = {"xfail": name in algorithm_dict.get("xfail_names", {}), + "strict": algorithm_dict.get("xfail_names", {}).get(name, True)} kwargs = algorithm_dict.get("options", {}) - tolerances = algorithm_dict.get("tolerances", None) + tolerances = algorithm_dict.get("tolerances", {}) yield name, bvals, data, algorithm, xfail, kwargs, tolerances @pytest.mark.parametrize("name, bvals, data, algorithm, xfail, kwargs, tolerances", data_ivim_fit_saved()) def test_ivim_fit_saved(name, bvals, data, algorithm, xfail, kwargs, tolerances, request): - if xfail: - mark = pytest.mark.xfail(reason="xfail", strict=True) + if xfail["xfail"]: + mark = pytest.mark.xfail(reason="xfail", strict=xfail["strict"]) request.node.add_marker(mark) fit = OsipiBase(algorithm=algorithm, **kwargs) signal = np.asarray(data['data']) signal = np.abs(signal) signal /= signal[0] + ratio = 1 / signal[0] # has_negatives = np.any(signal<0) - if tolerances is None: - # signal = np.abs(signal) - # ratio = 1 / signal[0] - # signal /= signal[0] - # tolerance = 1e-2 + 25 * data['noise'] * ratio # totally empirical - # tolerance = 1 - tolerances = {"f": 5, "D": 5, "Dp": 25} + if "dynamic_rtol" in tolerances: + dyn_rtol = tolerances["dynamic_rtol"] + scale = dyn_rtol["offset"] + dyn_rtol["ratio"]*ratio + dyn_rtol["noise"]*data["noise"] + dyn_rtol["noiseCrossRatio"]*ratio*data["noise"] + tolerances["rtol"] = {"f": scale*dyn_rtol["f"], "D": scale*dyn_rtol["D"], "Dp": scale*dyn_rtol["Dp"]} + else: + tolerances["rtol"] = tolerances.get("rtol", {"f": 5, "D": 5, "Dp": 25}) + if "dynamic_atol" in tolerances: + dyn_atol = tolerances["dynamic_atol"] + scale = dyn_atol["offset"] + dyn_atol["ratio"]*ratio + dyn_atol["noise"]*data["noise"] + dyn_atol["noiseCrossRatio"]*ratio*data["noise"] + tolerances["atol"] = {"f": scale*dyn_atol["f"], "D": scale*dyn_atol["D"], "Dp": scale*dyn_atol["Dp"]} + else: + tolerances["atol"] = tolerances.get("atol", {"f": 1e-2, "D": 1e-2, "Dp": 1e-1}) + # if tolerances: + # # signal = np.abs(signal) + # # ratio = 1 / signal[0] + # # signal /= signal[0] + # # tolerance = 1e-2 + 25 * data['noise'] * ratio # totally empirical + # # tolerance = 1 + # tolerances = {"rtol": {"f": 5, "D": 5, "Dp": 25}, + # "atol": {"f": 1e-2, "D": 1e-2, "Dp": 1e-1}} #if has_negatives: # this fitting doesn't do well with negatives # tolerance += 1 #if data['f'] == 1.0: @@ -108,6 +123,6 @@ def test_ivim_fit_saved(name, bvals, data, algorithm, xfail, kwargs, tolerances, #npt.assert_allclose([data['f'], data['D']], [f_fit, D_fit], atol=tolerance) #npt.assert_allclose(data['Dp'], Dp_fit, atol=1e-1) # go easy on the perfusion as it's a linear fake [f_fit, Dp_fit, D_fit] = fit.ivim_fit(signal, bvals) - npt.assert_allclose(data['f'], f_fit, rtol=tolerances["f"]) - npt.assert_allclose(data['D'], D_fit, rtol=tolerances["D"]) - npt.assert_allclose(data['Dp'], Dp_fit, rtol=tolerances["Dp"]) # go easy on the perfusion as it's a linear fake + npt.assert_allclose(data['f'], f_fit, rtol=tolerances["rtol"]["f"], atol=tolerances["atol"]["f"]) + npt.assert_allclose(data['D'], D_fit, rtol=tolerances["rtol"]["D"], atol=tolerances["atol"]["D"]) + npt.assert_allclose(data['Dp'], Dp_fit, rtol=tolerances["rtol"]["Dp"], atol=tolerances["atol"]["Dp"])