Save test results to a json file & Display results Visuals on the website

.github/workflows/website.yml

@@ -46,12 +46,20 @@ jobs:
         with:
           name: 'Data'
 
+      - name: Run the test that generates the plots report.
+        run: |
+          pytest tests/IVIMmodels/unit_tests/test_ivim_fit.py --json-report
+          mv .report.json utilities/
+          python utilities/report-summary.py .report.json report-summary.json
+
       - name: 'Filter and compress results file.'
         run: python utilities/reduce_output_size.py test_output.csv test_output.csv.gz
 
       - name: move data to the dashboard folder
         run: |
           mv test_output.csv.gz website/dashboard
+          mv report-summary.json website/dashboard
+
 
       - name: Build documentation
         run: |

requirements.txt

@@ -13,3 +13,4 @@ tqdm
 pandas
 sphinx
 sphinx_rtd_theme
+pytest-json-report

tests/IVIMmodels/unit_tests/test_ivim_fit.py

@@ -3,115 +3,11 @@
 import pytest
 import json
 import pathlib
-import os
 
 from src.wrappers.OsipiBase import OsipiBase
 from utilities.data_simulation.GenerateData import GenerateData
-
 #run using python -m pytest from the root folder
 
-
-# @pytest.fixture
-# def algorithm_fixture()
-# def test_fixtures()
-
-# use a fixture to generate data
-# either read a config file for the test or perhaps hard code a few fixtures and usefixtures in the config?
-# use a fixture to save data
-
-# def algorithm_list():
-#     # Find the algorithms from algorithms.json
-#     file = pathlib.Path(__file__)
-#     algorithm_path = file.with_name('algorithms.json')
-#     with algorithm_path.open() as f:
-#         algorithm_information = json.load(f)
-#     return algorithm_information["algorithms"]
-
-# @pytest.fixture(params=algorithm_list())
-# def algorithm_fixture(request):
-#     # assert request.param == "algorithms"
-#     yield request.param
-
-
-
-# @pytest.fixture(params=SNR)
-# def noise_fixture(request):
-#     return request.config.getoption("--noise")
-
-# @pytest.fixture
-# def noise_fixture(request):
-#     yield request.param
-
-# @pytest.mark.parametrize("S", [SNR])
-# @pytest.mark.parametrize("D, Dp, f, bvals", [[0.0015, 0.1, 0.11000000000000007,[0, 5, 10, 50, 100, 200, 300, 500, 1000]]])
-# def test_generated(ivim_algorithm, ivim_data, SNR):
-#     S0 = 1
-#     gd = GenerateData()
-#     name, bvals, data = ivim_data
-#     D = data["D"]
-#     f = data["f"]
-#     Dp = data["Dp"]
-#     if "data" not in data:
-#         signal = gd.ivim_signal(D, Dp, f, S0, bvals, SNR)
-#     else:
-#         signal = data["data"]
-#     fit = OsipiBase(algorithm=ivim_algorithm)
-#     [f_fit, Dp_fit, D_fit] = fit.osipi_fit(signal, bvals)
-#     npt.assert_allclose([f, D, Dp], [f_fit, D_fit, Dp_fit])
-
-
-
-# test_linear_data = [
-#     pytest.param(0, np.linspace(0, 1000, 11), id='0'),
-#     pytest.param(0.01, np.linspace(0, 1000, 11), id='0.1'),
-#     pytest.param(0.02, np.linspace(0, 1000, 11), id='0.2'),
-#     pytest.param(0.03, np.linspace(0, 1000, 11), id='0.3'),
-#     pytest.param(0.04, np.linspace(0, 1000, 11), id='0.4'),
-#     pytest.param(0.05, np.linspace(0, 1000, 11), id='0.5'),
-#     pytest.param(0.08, np.linspace(0, 1000, 11), id='0.8'),
-#     pytest.param(0.1, np.linspace(0, 1000, 11), id='1'),
-# ]
-
-#@pytest.mark.parametrize("D, bvals", test_linear_data)
-#def test_linear_fit(D, bvals):
-    #gd = GenerateData()
-    #gd_signal = gd.exponential_signal(D, bvals)
-    #print(gd_signal)
-    #fit = LinearFit()
-    #D_fit = fit.linear_fit(bvals, np.log(gd_signal))
-    #npt.assert_allclose([1, D], D_fit)
-
-# test_ivim_data = [
-#     pytest.param(0, 0.01, 0.05, np.linspace(0, 1000, 11), id='0'),
-#     pytest.param(0.1, 0.01, 0.05, np.linspace(0, 1000, 11), id='0.1'),
-#     pytest.param(0.2, 0.01, 0.05, np.linspace(0, 1000, 11), id='0.2'),
-#     pytest.param(0.1, 0.05, 0.1, np.linspace(0, 1000, 11), id='0.3'),
-#     pytest.param(0.4, 0.001, 0.05, np.linspace(0, 1000, 11), id='0.4'),
-#     pytest.param(0.5, 0.001, 0.05, np.linspace(0, 1000, 11), id='0.5'),
-# ]
-
-#@pytest.mark.parametrize("f, D, Dp, bvals", test_ivim_data)
-#def test_ivim_fit(f, D, Dp, bvals):
-    ## We should make a wrapper that runs this for a range of different settings, such as b thresholds, bounds, etc.
-    ## An additional inputs to these functions could perhaps be a "settings" class with attributes that are the settings to the
-    ## algorithms. I.e. bvalues, thresholds, bounds, initial guesses.
-    ## That way, we can write something that defines a range of settings, and then just run them through here.
-
-    #gd = GenerateData()
-    #gd_signal = gd.ivim_signal(D, Dp, f, 1, bvals)
-
-    ##fit = LinearFit()   # This is the old code by ETP
-    #fit = ETP_SRI_LinearFitting() # This is the standardized format by IAR, which every algorithm will be implemented with
-
-    #[f_fit, Dp_fit, D_fit] = fit.ivim_fit(gd_signal, bvals) # Note that I have transposed Dp and D. We should decide on a standard order for these. I usually go with f, Dp, and D ordered after size.
-    #npt.assert_allclose([f, D], [f_fit, D_fit], atol=1e-5)
-    #if not np.allclose(f, 0):
-        #npt.assert_allclose(Dp, Dp_fit, rtol=1e-2, atol=1e-3)
-
-
-# convert the algorithm list and signal list to fixtures that read from the files into params (scope="session")
-# from that helpers can again parse the files?
-
 def signal_helper(signal):
     signal = np.asarray(signal)
     signal = np.abs(signal)
@@ -160,17 +56,36 @@ def data_ivim_fit_saved():
 
 
 @pytest.mark.parametrize("name, bvals, data, algorithm, xfail, kwargs, tolerances", data_ivim_fit_saved())
-def test_ivim_fit_saved(name, bvals, data, algorithm, xfail, kwargs, tolerances, request):
+def test_ivim_fit_saved(name, bvals, data, algorithm, xfail, kwargs, tolerances, request, record_property):
     if xfail["xfail"]:
         mark = pytest.mark.xfail(reason="xfail", strict=xfail["strict"])
         request.node.add_marker(mark)
     fit = OsipiBase(algorithm=algorithm, **kwargs)
     signal, ratio = signal_helper(data["data"])
+
     tolerances = tolerances_helper(tolerances, ratio, data["noise"])
     [f_fit, Dp_fit, D_fit] = fit.osipi_fit(signal, bvals)
+    def to_list_if_needed(value):
+        return value.tolist() if isinstance(value, np.ndarray) else value
+    test_result = {
+        "name": name,
+        "algorithm": algorithm,
+        "f_fit": to_list_if_needed(f_fit),
+        "Dp_fit": to_list_if_needed(Dp_fit),
+        "D_fit": to_list_if_needed(D_fit),
+        "f": to_list_if_needed(data['f']),
+        "Dp": to_list_if_needed(data['Dp']),
+        "D": to_list_if_needed(data['D']),
+        "rtol": tolerances["rtol"],
+        "atol": tolerances["atol"]
+    }
+
+
+    record_property('test_data', test_result)
+
     npt.assert_allclose(data['f'], f_fit, rtol=tolerances["rtol"]["f"], atol=tolerances["atol"]["f"])
+
     if data['f']<0.80: # we need some signal for D to be detected
         npt.assert_allclose(data['D'], D_fit, rtol=tolerances["rtol"]["D"], atol=tolerances["atol"]["D"])
     if data['f']>0.03: #we need some f for D* to be interpretable
         npt.assert_allclose(data['Dp'], Dp_fit, rtol=tolerances["rtol"]["Dp"], atol=tolerances["atol"]["Dp"])
-

utilities/report-summary.py

@@ -0,0 +1,26 @@
+import pathlib
+import json
+import sys
+
+def summarize_test_report(input_file:str, output_file:str):
+    file = pathlib.Path(__file__)
+    report_path = file.with_name(input_file)
+    with report_path.open() as f:
+        report_info = json.load(f)
+    summary = []
+    for test_case in report_info['tests']:
+        values = test_case['user_properties'][0]['test_data']
+        values['status'] = test_case['outcome']
+        summary.append(values)
+
+    with open(output_file, 'w') as f:
+        json.dump(summary, f, indent=4)
+
+if __name__ == '__main__':
+    if len(sys.argv) != 3:
+        print("Usage: python report-summary.py <input_file> <output_file>")
+        sys.exit(1)
+
+    input_file = sys.argv[1]
+    output_file = sys.argv[2]
+    summarize_test_report(input_file, output_file)

website/dashboard/index.html

@@ -10,6 +10,7 @@
   <script src="https://cdnjs.cloudflare.com/ajax/libs/PapaParse/5.3.0/papaparse.min.js"></script>
   <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/pako.min.js"></script>
   <script src="index.js"></script>
+  <script src="test_plots.js"></script>
   <link rel="stylesheet" href="index.css">
 </head>
 <body>
@@ -97,6 +98,12 @@ <h1 class="bar-title">IVIM MRI Algorithm Fitting Dashboard</h1>
       <div class="chart-card" id="regionDiv">
         <!-- New chart will be rendered here -->
       </div>
+      <h1>Validation Data Plots</h1>
+      <div style="display: flex; flex-direction: column; gap: 1rem;">
+      <div class="chart-card" id="plot_f_fit" "></div>
+      <div class="chart-card" id="plot_Dp_fit" "></div>
+      <div class="chart-card" id="plot_D_fit" "></div>
+      </div>
 
 
     </main>

website/dashboard/test_plots.js

@@ -0,0 +1,85 @@
+document.addEventListener('DOMContentLoaded', function() {
+    fetch('report-summary.json')
+        .then(response => response.json())
+        .then(data => {
+            function createPlot(container, parameter) {
+                var reference_values = data.map(d => d[parameter]);
+                var fit_values = data.map(d => d[parameter + '_fit']);
+                var minRefValue = Math.min(...reference_values);
+                var maxRefValue = Math.max(...reference_values);
+                // Create a range for tolerance trace x axis.
+                var xRange = [minRefValue, maxRefValue];
+                // var DefaultTolerance = {
+                //     "rtol": {
+                //         "f": 0.05,
+                //         "D": 2,
+                //         "Dp": 0.5
+                //     },
+                //     "atol": {
+                //         "f": 0.2,
+                //         "D": 0.001,
+                //         "Dp": 0.06
+                //     }
+                // }
+                var DefaultTolerance = data[1] //Majority of the dataset has this tolerance values
+                var tolerance = xRange.map((d) => DefaultTolerance['atol'][parameter] + DefaultTolerance['rtol'][parameter] * d);
+                var negative_tolerance = tolerance.map(t => -t);
+
+                var errors = fit_values.map((d, i) => (d - reference_values[i]));
+
+                // Define colors for each status
+                var statusColors = {
+                    'passed': 'green',
+                    'xfailed': 'blue',
+                    'failed': 'red'
+                };
+
+                // Assign color based on the status
+                var marker_colors = data.map(entry => statusColors[entry.status]);
+
+                var scatter_trace = {
+                    x: reference_values,
+                    y: errors,
+                    mode: 'markers',
+                    type: 'scatter',
+                    name:  `${parameter} fitting values`,
+                    text: data.map(entry => `Algorithm: ${entry.algorithm} Region: ${entry.name}`),
+                    marker: {
+                        color: marker_colors
+                    }
+                };
+
+                var tolerance_trace = {
+                    x: xRange,
+                    y: tolerance,
+                    type: 'scatter',
+                    mode: 'lines',
+                    line: { dash: 'dash', color: 'black' },
+                    name: 'Positive Tolerance'
+                };
+
+                var negative_tolerance_trace = {
+                    x: xRange,
+                    y: negative_tolerance,
+                    type: 'scatter',
+                    mode: 'lines',
+                    line: { dash: 'dash', color: 'black' },
+                    name: 'Negative Tolerance'
+                };
+
+                var layout = {
+                    title: `Error Plot for ${parameter.toUpperCase()}_fit with Tolerance Bands`,
+                    xaxis: { title: `Reference ${parameter.toUpperCase()} Values` },
+                    yaxis: { title: `Error (${parameter}_fit - Reference ${parameter})` }
+                };
+
+                var plot_data = [scatter_trace, tolerance_trace, negative_tolerance_trace];
+
+                Plotly.newPlot(container, plot_data, layout);
+            }
+
+            createPlot('plot_f_fit', 'f');
+            createPlot('plot_Dp_fit', 'Dp');
+            createPlot('plot_D_fit', 'D');
+        });
+});