From 083054b18288ca5cac190442ae92fb3a282f72b8 Mon Sep 17 00:00:00 2001 From: valosekj Date: Fri, 31 May 2024 19:13:13 -0400 Subject: [PATCH 1/2] Add the lesion-wise metrics proposed in https://github.com/ivadomed/MetricsReloaded/pull/3 Resolves: https://github.com/ivadomed/utilities/issues/49 --- compute_metrics/compute_metrics_reloaded.py | 3 ++ tests/test_compute_metrics_reloaded.py | 37 ++++++++++++++++----- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/compute_metrics/compute_metrics_reloaded.py b/compute_metrics/compute_metrics_reloaded.py index 896989f..b29a732 100644 --- a/compute_metrics/compute_metrics_reloaded.py +++ b/compute_metrics/compute_metrics_reloaded.py @@ -52,6 +52,9 @@ 'nsd': 'Normalized surface distance (NSD)', 'vol_diff': 'Volume difference', 'rel_vol_error': 'Relative volume error (RVE)', + 'lesion_ppv': 'Lesion wise positive predictive value (PPV)', + 'lesion_sensitivity': 'Lesion wise sensitivity', + 'lesion_f1_score': 'Lesion wise F1 score', } diff --git a/tests/test_compute_metrics_reloaded.py b/tests/test_compute_metrics_reloaded.py index 4cc96fe..4084e08 100644 --- a/tests/test_compute_metrics_reloaded.py +++ b/tests/test_compute_metrics_reloaded.py @@ -13,7 +13,7 @@ from compute_metrics.compute_metrics_reloaded import compute_metrics_single_subject import tempfile -METRICS = ['dsc', 'fbeta', 'nsd', 'vol_diff', 'rel_vol_error'] +METRICS = ['dsc', 'fbeta', 'nsd', 'vol_diff', 'rel_vol_error', 'lesion_ppv', 'lesion_sensitivity', 'lesion_f1_score'] class TestComputeMetricsReloaded(unittest.TestCase): @@ -59,7 +59,10 @@ def test_empty_ref_and_pred(self): 'fbeta': 1, 'nsd': np.nan, 'rel_vol_error': 0, - 'vol_diff': np.nan}} + 'vol_diff': np.nan, + 'lesion_ppv': 1.0, + 'lesion_sensitivity': 1.0, + 'lesion_f1_score': 1.0}} # Create empty reference self.create_dummy_nii(self.ref_file, np.zeros((10, 10, 10))) @@ -81,7 +84,10 @@ def test_empty_ref(self): 'fbeta': 0, 'nsd': 0.0, 'rel_vol_error': 100, - 'vol_diff': np.inf}} + 'vol_diff': np.inf, + 'lesion_ppv': 0.0, + 'lesion_sensitivity': 1.0, + 'lesion_f1_score': 0.0}} # Create empty reference self.create_dummy_nii(self.ref_file, np.zeros((10, 10, 10))) @@ -105,7 +111,10 @@ def test_empty_pred(self): 'fbeta': 0, 'nsd': 0.0, 'rel_vol_error': -100.0, - 'vol_diff': 1.0}} + 'vol_diff': 1.0, + 'lesion_ppv': 1.0, + 'lesion_sensitivity': 0.0, + 'lesion_f1_score': 0.0}} # Create non-empty reference ref = np.zeros((10, 10, 10)) @@ -129,7 +138,10 @@ def test_non_empty_ref_and_pred(self): 'fbeta': 0.26666667461395266, 'nsd': 0.5373134328358209, 'rel_vol_error': 300.0, - 'vol_diff': 3.0}} + 'vol_diff': 3.0, + 'lesion_ppv': 1.0, + 'lesion_sensitivity': 1.0, + 'lesion_f1_score': 1.0}} # Create non-empty reference ref = np.zeros((10, 10, 10)) @@ -156,14 +168,20 @@ def test_non_empty_ref_and_pred_multi_class(self): 'vol_diff': 2.0, 'rel_vol_error': 200.0, 'EmptyRef': False, - 'EmptyPred': False}, + 'EmptyPred': False, + 'lesion_ppv': 1.0, + 'lesion_sensitivity': 1.0, + 'lesion_f1_score': 1.0}, 2.0: {'dsc': 0.26666666666666666, 'fbeta': 0.26666667461395266, 'nsd': 0.5373134328358209, 'vol_diff': 3.0, 'rel_vol_error': 300.0, 'EmptyRef': False, - 'EmptyPred': False}} + 'EmptyPred': False, + 'lesion_ppv': 1.0, + 'lesion_sensitivity': 1.0, + 'lesion_f1_score': 1.0}} # Create non-empty reference ref = np.zeros((10, 10, 10)) @@ -191,7 +209,10 @@ def test_non_empty_ref_and_pred_with_full_overlap(self): 'fbeta': 1.0, 'nsd': 1.0, 'rel_vol_error': 0.0, - 'vol_diff': 0.0}} + 'vol_diff': 0.0, + 'lesion_ppv': 1.0, + 'lesion_sensitivity': 1.0, + 'lesion_f1_score': 1.0}} # Create non-empty reference ref = np.zeros((10, 10, 10)) From ad23622a8351e663e1bf6255a478beceb82fd629 Mon Sep 17 00:00:00 2001 From: valosekj Date: Tue, 4 Jun 2024 15:16:30 -0400 Subject: [PATCH 2/2] Change lesion PPV to 0.0 in case of non-empty reference and empty prediction Context: the lesion PPV computation for this case was fixed in https://github.com/ivadomed/MetricsReloaded/commit/ce172b53dcf85037e16618ebec1d8b0db9bc47d7 --- tests/test_compute_metrics_reloaded.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_compute_metrics_reloaded.py b/tests/test_compute_metrics_reloaded.py index 4084e08..c02022f 100644 --- a/tests/test_compute_metrics_reloaded.py +++ b/tests/test_compute_metrics_reloaded.py @@ -112,7 +112,7 @@ def test_empty_pred(self): 'nsd': 0.0, 'rel_vol_error': -100.0, 'vol_diff': 1.0, - 'lesion_ppv': 1.0, + 'lesion_ppv': 0.0, 'lesion_sensitivity': 0.0, 'lesion_f1_score': 0.0}}