Merge pull request #66 from RobJY/ci_nf-test

Add our nf-test tests to the GitHub Actions CI

.github/workflows/ci.yml

@@ -11,6 +11,7 @@ on:
 
 env:
   NXF_ANSI_LOG: false
+  NFT_VER: "0.9.0"
   NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
   NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
 
@@ -19,18 +20,42 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  nf-test-changes:
+    name: Check for changes
+    runs-on: ubuntu-latest
+    outputs:
+      nf_test_files: ${{ steps.list.outputs.components }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: List nf-test files
+        id: list
+        uses: adamrtalbot/[email protected]
+        with:
+          head: ${{ github.sha }}
+          base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }}
+
+      - name: print list of nf-test files
+        run: |
+          echo ${{ steps.list.outputs.components }}
+
   test:
-    name: "Run pipeline with test data (${{ matrix.NXF_VER }} | ${{ matrix.test_name }} | ${{ matrix.profile }})"
+    name: "Run tests (${{ matrix.nf_test_files }} ${{ matrix.profile }} NF-${{ matrix.NXF_VER }})"
     # Only run on push if this is the nf-core dev branch (merged PRs)
-    if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/mcmicro') }}"
+    needs: [nf-test-changes]
+    if: "${{ (github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/mcmicro')) && needs.nf-test-changes.outputs.nf_test_files != '[]' }}"
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         NXF_VER:
           - "24.04.2"
           - "latest-everything"
+        nf_test_files: ["${{ fromJson(needs.nf-test-changes.outputs.nf_test_files) }}"]
         profile:
-          - "conda"
           - "docker"
           - "singularity"
         test_name:
@@ -44,6 +69,9 @@ jobs:
           - isMaster: false
             profile: "singularity"
     steps:
+      - name: install curl
+        run: sudo apt update && sudo apt install -y curl
+
       - name: Check out pipeline code
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
 
@@ -62,24 +90,63 @@ jobs:
           mkdir -p $NXF_SINGULARITY_CACHEDIR
           mkdir -p $NXF_SINGULARITY_LIBRARYDIR
 
-      - name: Set up Miniconda
-        if: matrix.profile == 'conda'
-        uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3
+      - name: Install nf-test
+        uses: nf-core/setup-nf-test@v1
+        with:
+          version: ${{ env.NFT_VER }}
+
+      - uses: actions/setup-python@v4
         with:
-          miniconda-version: "latest"
-          auto-update-conda: true
-          conda-solver: libmamba
-          channels: conda-forge,bioconda
+          python-version: "3.11"
+          architecture: "x64"
 
-      - name: Set up Conda
-        if: matrix.profile == 'conda'
+      - name: Install pdiff to see diff between nf-test snapshots
         run: |
-          echo $(realpath $CONDA)/condabin >> $GITHUB_PATH
-          echo $(realpath python) >> $GITHUB_PATH
+          python -m pip install --upgrade pip
+          pip install pdiff
 
       - name: Clean up Disk space
         uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
 
-      - name: "Run pipeline with test data ${{ matrix.NXF_VER }} | ${{ matrix.test_name }} | ${{ matrix.profile }}"
+      - name: Run nf-test
+        run: |
+          nf-test test \
+            --verbose ${{ matrix.nf_test_files }} \
+            --profile "+${{ matrix.profile }},ci" \
+            --junitxml=test.xml \
+            --tap=test.tap
+
+      - uses: pcolby/tap-summary@v1
+        with:
+          path: >-
+            test.tap
+
+      - name: Output log on failure
+        if: failure()
+        run: |
+          sudo apt install bat > /dev/null
+          batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/meta/nextflow.log
+
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v3
+        if: always()
+        with:
+          report_paths: test.xml
+
+  confirm-pass:
+    runs-on: ubuntu-latest
+    needs: [test]
+    if: always()
+    steps:
+      - name: All tests ok
+        if: ${{ !contains(needs.*.result, 'failure') }}
+        run: exit 0
+      - name: One or more tests failed
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
+
+      - name: debug-print
+        if: always()
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.test_name }},${{ matrix.profile }} --outdir ./results
+          echo "toJSON(needs) = ${{ toJSON(needs) }}"
+          echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"

conf/ci.config

@@ -0,0 +1,10 @@
+process {
+    withName: ".*:DEEPCELL_MESMER" {
+        memory = "6.GB"
+        cpus = 2
+    }
+    withName: ".*:CELLPOSE"{
+        memory = "12.GB"
+        cpus   = 4
+    }
+}

conf/modules.config

@@ -28,7 +28,6 @@ process {
     }
 
     withName: BASICPY {
-        containerOptions = '--entrypoint "" --user root'
         publishDir = [
             path: { "${params.outdir}/illumination_correction/basicpy" },
             mode: params.publish_dir_mode,

modules.json

@@ -7,7 +7,7 @@
                 "nf-core": {
                     "ashlar": {
                         "branch": "master",
-                        "git_sha": "a5a287db3590e7aa121e1251bf126c35e1418abf",
+                        "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
                         "installed_by": ["modules"],
                         "patch": "modules/nf-core/ashlar/ashlar.diff"
                     },
@@ -18,7 +18,7 @@
                     },
                     "basicpy": {
                         "branch": "master",
-                        "git_sha": "716ef3019b66772a817b417078edce2f7b337858",
+                        "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
                         "installed_by": ["modules"],
                         "patch": "modules/nf-core/basicpy/basicpy.diff"
                     },
@@ -35,7 +35,7 @@
                     },
                     "deepcell/mesmer": {
                         "branch": "master",
-                        "git_sha": "298406de7fb599b3a7acfed1dddb606ec182d4b6",
+                        "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
                         "installed_by": ["modules"]
                     },
                     "ilastik/multicut": {