Skip to content

Commit

Permalink
More advanced testing parameters
Browse files Browse the repository at this point in the history
  • Loading branch information
@etpeterson
etpeterson committed Oct 17, 2023
1 parent 7117bc2 commit f75dcd4
Show file tree
Hide file tree
Showing 2 changed files with 120 additions and 74 deletions.
151 changes: 91 additions & 60 deletions tests/IVIMmodels/unit_tests/algorithms.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,94 +9,125 @@
"IAR_LU_subtracted"
],
"IAR_LU_biexp": {
"xfail_names": [
"Vein",
"Blood RV",
"Blood LV",
"Blood RA"
],
"tolerances": {
"f": 5,
"D": 5,
"Dp": 25
"rtol": {
"f": 5,
"D": 5,
"Dp": 25
},
"atol": {
"f": 1e-2,
"D": 1e-2,
"Dp": 1e-1
}
}
},
"IAR_LU_modified_mix": {
"xfail_names": [
"Blood LV"
],
"tolerances": {
"f": 5,
"D": 5,
"Dp": 25
"rtol": {
"f": 5,
"D": 5,
"Dp": 25
},
"atol": {
"f": 1e-2,
"D": 1e-2,
"Dp": 1e-1
}
}
},
"IAR_LU_modified_topopro": {
"xfail_names": [
"Vein",
"Blood RV",
"Blood LV",
"Blood RA"
],
"tolerances": {
"f": 5,
"D": 5,
"Dp": 25
"rtol": {
"f": 5,
"D": 5,
"Dp": 25
},
"atol": {
"f": 1e-2,
"D": 1e-2,
"Dp": 1e-1
}
}
},
"IAR_LU_segmented_2step": {
"xfail_names": [
"Blood RV",
"Blood LV",
"Blood RA"
],
"tolerances": {
"f": 5,
"D": 5,
"Dp": 25
"rtol": {
"f": 5,
"D": 5,
"Dp": 25
},
"atol": {
"f": 1e-2,
"D": 1e-2,
"Dp": 1e-1
}
}
},
"IAR_LU_segmented_3step": {
"xfail_names": [
"Blood RV",
"Blood LV",
"Blood RA"
],
"tolerances": {
"f": 5,
"D": 5,
"Dp": 25
"rtol": {
"f": 5,
"D": 5,
"Dp": 25
},
"atol": {
"f": 1e-2,
"D": 1e-2,
"Dp": 1e-1
}
}
},
"IAR_LU_subtracted": {
"xfail_names": [
"Blood RV",
"Blood LV",
"Blood RA"
],
"tolerances": {
"f": 5,
"D": 5,
"Dp": 25
"rtol": {
"f": 5,
"D": 5,
"Dp": 25
},
"atol": {
"f": 1e-2,
"D": 1e-2,
"Dp": 1e-1
}
}
},
"ETP_SRI_LinearFitting": {
"xfail_names": [
"Artery",
"Blood RV",
"Blood LV",
"myocardium RV",
"myocardium ra",
"Liver",
"Blood RA"
],
"xfail_names": {
"Vein": true
},
"options": {
"thresholds": [500]
},
"tolerances": {
"f": 5,
"D": 5,
"Dp": 25
"rtol": {
"f": 5,
"D": 5,
"Dp": 25
},
"atol": {
"f": 0,
"D": 0,
"Dp": 0
},
"dynamic_rtol": {
"offset": 0,
"noise": 0,
"ratio": 0,
"noiseCrossRatio": 10,
"f": 1,
"D": 1,
"Dp": 2
},
"dynamic_atol": {
"offset": 5e-2,
"noise": 0,
"ratio": 0,
"noiseCrossRatio": 10,
"f": 1,
"D": 1,
"Dp": 2
}
}
}
}
43 changes: 29 additions & 14 deletions tests/IVIMmodels/unit_tests/test_ivim_fit.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,29 +75,44 @@ def data_ivim_fit_saved():
for name, data in all_data.items():
for algorithm in algorithms:
algorithm_dict = algorithm_information.get(algorithm, {})
xfail = name in algorithm_dict.get("xfail_names", [])
xfail = {"xfail": name in algorithm_dict.get("xfail_names", {}),
"strict": algorithm_dict.get("xfail_names", {}).get(name, True)}
kwargs = algorithm_dict.get("options", {})
tolerances = algorithm_dict.get("tolerances", None)
tolerances = algorithm_dict.get("tolerances", {})
yield name, bvals, data, algorithm, xfail, kwargs, tolerances


@pytest.mark.parametrize("name, bvals, data, algorithm, xfail, kwargs, tolerances", data_ivim_fit_saved())
def test_ivim_fit_saved(name, bvals, data, algorithm, xfail, kwargs, tolerances, request):
if xfail:
mark = pytest.mark.xfail(reason="xfail", strict=True)
if xfail["xfail"]:
mark = pytest.mark.xfail(reason="xfail", strict=xfail["strict"])
request.node.add_marker(mark)
fit = OsipiBase(algorithm=algorithm, **kwargs)
signal = np.asarray(data['data'])
signal = np.abs(signal)
signal /= signal[0]
ratio = 1 / signal[0]
# has_negatives = np.any(signal<0)
if tolerances is None:
# signal = np.abs(signal)
# ratio = 1 / signal[0]
# signal /= signal[0]
# tolerance = 1e-2 + 25 * data['noise'] * ratio # totally empirical
# tolerance = 1
tolerances = {"f": 5, "D": 5, "Dp": 25}
if "dynamic_rtol" in tolerances:
dyn_rtol = tolerances["dynamic_rtol"]
scale = dyn_rtol["offset"] + dyn_rtol["ratio"]*ratio + dyn_rtol["noise"]*data["noise"] + dyn_rtol["noiseCrossRatio"]*ratio*data["noise"]
tolerances["rtol"] = {"f": scale*dyn_rtol["f"], "D": scale*dyn_rtol["D"], "Dp": scale*dyn_rtol["Dp"]}
else:
tolerances["rtol"] = tolerances.get("rtol", {"f": 5, "D": 5, "Dp": 25})
if "dynamic_atol" in tolerances:
dyn_atol = tolerances["dynamic_atol"]
scale = dyn_atol["offset"] + dyn_atol["ratio"]*ratio + dyn_atol["noise"]*data["noise"] + dyn_atol["noiseCrossRatio"]*ratio*data["noise"]
tolerances["atol"] = {"f": scale*dyn_atol["f"], "D": scale*dyn_atol["D"], "Dp": scale*dyn_atol["Dp"]}
else:
tolerances["atol"] = tolerances.get("atol", {"f": 1e-2, "D": 1e-2, "Dp": 1e-1})
# if tolerances:
# # signal = np.abs(signal)
# # ratio = 1 / signal[0]
# # signal /= signal[0]
# # tolerance = 1e-2 + 25 * data['noise'] * ratio # totally empirical
# # tolerance = 1
# tolerances = {"rtol": {"f": 5, "D": 5, "Dp": 25},
# "atol": {"f": 1e-2, "D": 1e-2, "Dp": 1e-1}}
#if has_negatives: # this fitting doesn't do well with negatives
# tolerance += 1
#if data['f'] == 1.0:
Expand All @@ -108,6 +123,6 @@ def test_ivim_fit_saved(name, bvals, data, algorithm, xfail, kwargs, tolerances,
#npt.assert_allclose([data['f'], data['D']], [f_fit, D_fit], atol=tolerance)
#npt.assert_allclose(data['Dp'], Dp_fit, atol=1e-1) # go easy on the perfusion as it's a linear fake
[f_fit, Dp_fit, D_fit] = fit.ivim_fit(signal, bvals)
npt.assert_allclose(data['f'], f_fit, rtol=tolerances["f"])
npt.assert_allclose(data['D'], D_fit, rtol=tolerances["D"])
npt.assert_allclose(data['Dp'], Dp_fit, rtol=tolerances["Dp"]) # go easy on the perfusion as it's a linear fake
npt.assert_allclose(data['f'], f_fit, rtol=tolerances["rtol"]["f"], atol=tolerances["atol"]["f"])
npt.assert_allclose(data['D'], D_fit, rtol=tolerances["rtol"]["D"], atol=tolerances["atol"]["D"])
npt.assert_allclose(data['Dp'], Dp_fit, rtol=tolerances["rtol"]["Dp"], atol=tolerances["atol"]["Dp"])

0 comments on commit f75dcd4

Please sign in to comment.