Merge pull request #184 from scilus/fix/mppca_reproducibility

[Module][FIX] Investiguate MRTRIX randomness

.github/workflows/nf-test_module.yml

@@ -76,16 +76,16 @@ jobs:
             ${{ runner.os }}-pip-pdiff
 
       - uses: actions/github-script@v7
-        id: test-data-cache-key
+        id: test-run-identifier
         with:
           script: |
             identifier='${{ inputs.paths }}'.toLowerCase().replaceAll(/[/. ]+/g, '-').trim('-');
-            core.setOutput('key', identifier);
+            core.setOutput('tag', identifier);
 
       - uses: actions/cache@v4
         with:
           path: ${{ env.NFSCIL_TEST_DATA_HOME }}/nf-scil-test-archives
-          key: nf-scil-test-data-cache-${{ steps.test-data-cache-key.outputs.key }}
+          key: nf-scil-test-data-cache-${{ steps.test-run-identifier.outputs.tag }}
 
       - name: Install Python dependencies
         run: python -m pip install --upgrade pip pdiff cryptography
@@ -112,13 +112,36 @@ jobs:
             --profile=$PROFILE \
             --tap=test.tap \
             --verbose \
+            --debug \
             ${{ inputs.paths }}
 
       - uses: pcolby/tap-summary@v1
         with:
           path: >-
             test.tap
 
+      - name: Tests logs
+        if: failure()
+        run: |
+            shopt -s globstar
+            for f in /home/runner/_work/tests/**/work/**/.command.log
+            do
+
+            echo "$(sed '3q;d' $(dirname $f)/.command.run)"
+            cat $f
+
+            done
+
+      - name: Upload images as artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: image_artifacts-${{ steps.test-run-identifier.outputs.tag }}
+          path: /home/runner/_work/tests/**/work/**/*.nii.gz
+          overwrite: true
+          retention-days: 2
+          compression-level: 9
+
       - name: Clean up
         if: always()
         run: |

modules/nf-scil/denoising/mppca/main.nf

@@ -4,11 +4,11 @@ process DENOISING_MPPCA {
     label 'process_single'
 
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://scil.usherbrooke.ca/containers/scilus_2.0.0.sif':
-        'scilus/scilus:2.0.0' }"
+        'https://scil.usherbrooke.ca/containers/scilus_2.0.2.sif':
+        'scilus/scilus:2.0.2' }"
 
     input:
-    tuple val(meta), path(dwi)
+    tuple val(meta), path(dwi), path(mask)
 
     output:
     tuple val(meta), path("*_dwi_denoised.nii.gz")  , emit: image
@@ -20,19 +20,23 @@ process DENOISING_MPPCA {
     script:
     def prefix = task.ext.prefix ?: "${meta.id}"
     def extent = task.ext.extent ? "-extent " + task.ext.extent : ""
+    def args = ["-nthreads ${task.cpus - 1}"]
+    if (mask) args += ["-mask $mask"]
 
     """
     export ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS=1
     export OMP_NUM_THREADS=1
     export OPENBLAS_NUM_THREADS=1
+    export MRTRIX_RNG_SEED=12345
 
-    dwidenoise $dwi ${prefix}_dwi_denoised.nii.gz $extent -nthreads 1
-    fslmaths ${prefix}_dwi_denoised.nii.gz -thr 0 ${prefix}_dwi_denoised.nii.gz
+    dwidenoise $dwi ${prefix}_dwi_denoised.nii.gz $extent ${args.join(" ")} -debug
+    scil_volume_math.py lower_clip ${prefix}_dwi_denoised.nii.gz 0 \
+        ${prefix}_dwi_denoised.nii.gz -f
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
         mrtrix: \$(mrcalc -version 2>&1 | sed -n 's/== mrcalc \\([0-9.]\\+\\).*/\\1/p')
-        fsl: \$(flirt -version 2>&1 | sed -n 's/FLIRT version \\([0-9.]\\+\\)/\\1/p')
+        scilpy: \$(pip list | grep scilpy | tr -s ' ' | cut -d' ' -f2)
     END_VERSIONS
     """
 
@@ -49,7 +53,7 @@ process DENOISING_MPPCA {
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
         mrtrix: \$(mrcalc -version 2>&1 | sed -n 's/== mrcalc \\([0-9.]\\+\\).*/\\1/p')
-        fsl: \$(flirt -version 2>&1 | sed -n 's/FLIRT version \\([0-9.]\\+\\)/\\1/p')
+        scilpy: \$(pip list | grep scilpy | tr -s ' ' | cut -d' ' -f2)
     END_VERSIONS
     """
 }

modules/nf-scil/denoising/mppca/tests/main.nf.test

@@ -20,7 +20,7 @@ nextflow_process {
                 script "../../../../../subworkflows/nf-scil/load_test_data/main.nf"
                 process {
                     """
-                    input[0] = Channel.from( [ "heavy.zip" ] )
+                    input[0] = Channel.from( [ "raw_DWIss300-dir8.zip", "raw_segmentation.zip" ] )
                     input[1] = "test.load-test-data"
                     """
                 }
@@ -30,12 +30,25 @@ nextflow_process {
         when {
             process {
                 """
-                input[0] = LOAD_DATA.out.test_data_directory.map{
+                ch_split_test_data = LOAD_DATA.out.test_data_directory
+                    .branch{
+                        dwi: it.simpleName == "raw_DWIss300-dir8"
+                        segmentation: it.simpleName == "raw_segmentation"
+                    }
+                ch_dwi = ch_split_test_data.dwi.map{
                     test_data_directory -> [
-                        [ id:'test', single_end:false ], // meta map
-                        file("\${test_data_directory}/dwi/dwi.nii.gz")
+                        [ id:'test' ],
+                        file("\${test_data_directory}/dwi.nii.gz")
                     ]
                 }
+                ch_mask = ch_split_test_data.segmentation.map{
+                    test_data_directory -> [
+                        [ id:'test' ],
+                        file("\${test_data_directory}/brainmask/slices/axial.nii.gz")
+                    ]
+                }
+                input[0] = ch_dwi
+                    .join(ch_mask)
                 """
             }
         }

modules/nf-scil/denoising/mppca/tests/main.nf.test.snap

@@ -5,33 +5,31 @@
                 "0": [
                     [
                         {
-                            "id": "test",
-                            "single_end": false
+                            "id": "test"
                         },
-                        "test_dwi_denoised.nii.gz:md5,c949b91d96513cba506975c271e53596"
+                        "test_dwi_denoised.nii.gz:md5,697aa941c0cb71fe1865662da5b727fa"
                     ]
                 ],
                 "1": [
-                    "versions.yml:md5,fb589ae27eb3edb1cd1a50b0e13d2714"
+                    "versions.yml:md5,c9915fbc1956d7f54bee0748f1ddf920"
                 ],
                 "image": [
                     [
                         {
-                            "id": "test",
-                            "single_end": false
+                            "id": "test"
                         },
-                        "test_dwi_denoised.nii.gz:md5,c949b91d96513cba506975c271e53596"
+                        "test_dwi_denoised.nii.gz:md5,697aa941c0cb71fe1865662da5b727fa"
                     ]
                 ],
                 "versions": [
-                    "versions.yml:md5,fb589ae27eb3edb1cd1a50b0e13d2714"
+                    "versions.yml:md5,c9915fbc1956d7f54bee0748f1ddf920"
                 ]
             }
         ],
         "meta": {
-            "nf-test": "0.8.4",
-            "nextflow": "23.10.1"
+            "nf-test": "0.9.0-rc1",
+            "nextflow": "24.04.4"
         },
-        "timestamp": "2024-04-25T11:07:06.152814"
+        "timestamp": "2024-08-05T15:17:51.718413"
     }
 }

modules/nf-scil/denoising/mppca/tests/nextflow.config

@@ -3,4 +3,5 @@ process {
         publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }
         ext.extent = 3
     }
+    cpus = 1
 }