Add selective test running (#14)

- Add selective running of tests - Users can use `--run <test name>` to run _only_ a single test from the set. - If left null, all tests are ran. - Can be used to test a single procedure without other tasks (if necessary), which will be useful when a single operation needs to be tested. - Uses string matching syntax and attaches `NF_CANARY:` to task variable. This isn't the best way but it works quickly now. Misc: - Update tests to include output paths - Isolate tests within subworkflow more precise scoping. --------- Co-authored-by: Adam Talbot
seqeralabs · Oct 4, 2023 · 90b263b · 90b263b
1 parent ca5f6e4
commit 90b263b
Show file tree

Hide file tree

Showing 25 changed files with 291 additions and 106 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -40,40 +40,45 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - uses: actions/cache@v3
+      - uses: actions/setup-java@v3
         with:
-          path: /usr/local/bin/nextflow
-          key: ${{ runner.os }}
-          restore-keys: |
-            ${{ runner.os }}-nextflow-
+          distribution: "temurin"
+          java-version: "17"
 
-      - name: Install Nextflow
-        env:
-          CAPSULE_LOG: none
-        run: |
-          wget -qO- get.nextflow.io | bash
-          sudo mv nextflow /usr/local/bin/
-
-      - name: Cache nf-test installation
+      - name: Cache software installation
         id: cache-software
         uses: actions/cache@v3
         with:
           path: |
+            /usr/local/bin/nextflow
             /usr/local/bin/nf-test
             /home/runner/.nf-test/nf-test.jar
-          key: ${{ runner.os }}-nftest
+          key: ${{ runner.os }}-software-${{ github.workspace }}
+
+      - name: Install Nextflow
+        env:
+          CAPSULE_LOG: none
+        if: steps.cache-software.outputs.cache-hit != 'true'
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
 
       - name: Install nf-test
         if: steps.cache-software.outputs.cache-hit != 'true'
         run: |
           wget -qO- https://code.askimed.com/install/nf-test | bash
           sudo mv nf-test /usr/local/bin/
+          nf-test update-plugins
+
+      - name: Run pipeline and cache dependencies
+        run: |
+          nextflow run main.nf
 
         # Test the module
       - name: Run nf-test
         run: |
-          nf-test test \
-              --tap=test.tap
+          java -version
+          nf-test test --tap=test.tap
 
       - uses: pcolby/tap-summary@v1
         with:

diff --git a/.gitignore b/.gitignore
@@ -9,4 +9,5 @@ testing*
 *.pyc
 test-datasets/
 .nf-test/
-test.tap
+test.tap
+.nf-test.log
diff --git a/README.md b/README.md
@@ -34,7 +34,15 @@ nf-canary uses the `--outdir` convention established by [nf-core](https://nf-co.
 
 ### Skipping Tests
 
-Each test can be skipped by name with the parameter `--skip`, e.g. `--skip TEST_INPUT`. Multiple can be specified with comma delimited values, e.g. `--skip TEST_CREATE_FILE,TEST_PASS_FILE`. Note, some tests depend on previous tests running so will be skipped if an upstream test does not run (or fails). All tests which are dependent on previous tests are listed here:
+Each test can be skipped by name with the parameter `--skip`, e.g. `--skip TEST_INPUT`. Multiple test can be specified with comma delimited values, e.g. `--skip TEST_CREATE_FILE,TEST_PASS_FILE`. Case insensitive.
+
+### Selectively running tests
+
+By default, all tests are ran, however you can selectively run a test with `--run`, e.g. `--run TEST_INPUT`. When using this parameter, _only_ the tests selected will be run. Multiple test can be specified with comma delimited values, e.g. `--run TEST_CREATE_FILE,TEST_PASS_FILE`. Case insensitive.
+
+### Note on test dependency
+
+Note, some tests depend on previous tests running so will be skipped if an upstream test does not run (or fails). All tests which are dependent on previous tests are listed here:
 
 ```yaml
 TEST_CREATE_FILE:

diff --git a/main.nf b/main.nf
@@ -3,6 +3,9 @@ process TEST_SUCCESS {
     This process should automatically succeed
     */
 
+    output:
+        stdout
+
     script:
     """
     exit 0
@@ -107,8 +110,7 @@ process TEST_PASS_FOLDER {
         path input
 
     output:
-        path "out", type: 'dir'   , emit: outfolder
-        path "out/*", type: 'file', emit: outfile
+        path "out", type: 'dir', emit: outfolder
 
     """
     cp -rL $input out
@@ -155,6 +157,9 @@ process TEST_IGNORED_FAIL {
     */
     errorStrategy 'ignore'
 
+    output:
+        stdout
+
     """
     exit 1
     """
@@ -180,8 +185,7 @@ process TEST_MV_FOLDER_CONTENTS {
     */
 
     output:
-        path "out", type: 'dir'   , emit: outfolder
-        path "out/*", type: 'file', emit: outfile
+        path "out", type: 'dir', emit: outfolder
 
     """
     mkdir -p test
@@ -191,28 +195,55 @@ process TEST_MV_FOLDER_CONTENTS {
     """
 }
 
-workflow {
-
-    // Create test file on head node
-    Channel
-        .of("alpha", "beta", "gamma")
-        .collectFile(name: 'sample.txt', newLine: true)
-        .set { test_file }
-
-    remote_file = params.remoteFile ? Channel.fromPath(params.remoteFile) : Channel.empty()
-
-    // Run tests
-    TEST_SUCCESS()
-    TEST_CREATE_FILE()
-    TEST_CREATE_FOLDER()
-    TEST_INPUT(test_file)
-    TEST_BIN_SCRIPT()
-    TEST_STAGE_REMOTE(remote_file)
-    TEST_PASS_FILE(TEST_CREATE_FILE.out.outfile)
-    TEST_PASS_FOLDER(TEST_CREATE_FOLDER.out.outfolder)
-    TEST_PUBLISH_FILE()
-    TEST_PUBLISH_FOLDER()
-    TEST_IGNORED_FAIL()
-    TEST_MV_FILE()
-    TEST_MV_FOLDER_CONTENTS()
+workflow NF_CANARY {
+
+    main:
+        // Create test file on head node
+        Channel
+            .of("alpha", "beta", "gamma")
+            .collectFile(name: 'sample.txt', newLine: true)
+            .set { test_file }
+
+        remote_file = params.remoteFile ? Channel.fromPath(params.remoteFile) : Channel.empty()
+
+        // Run tests
+        TEST_SUCCESS()
+        TEST_CREATE_FILE()
+        TEST_CREATE_FOLDER()
+        TEST_INPUT(test_file)
+        TEST_BIN_SCRIPT()
+        TEST_STAGE_REMOTE(remote_file)
+        TEST_PASS_FILE(TEST_CREATE_FILE.out.outfile)
+        TEST_PASS_FOLDER(TEST_CREATE_FOLDER.out.outfolder)
+        TEST_PUBLISH_FILE()
+        TEST_PUBLISH_FOLDER()
+        TEST_IGNORED_FAIL()
+        TEST_MV_FILE()
+        TEST_MV_FOLDER_CONTENTS()
+
+        // POC of emitting the channel
+        Channel.empty()
+            .mix(
+                TEST_SUCCESS.out,
+                TEST_CREATE_FILE.out,
+                TEST_CREATE_FOLDER.out,
+                TEST_INPUT.out,
+                TEST_BIN_SCRIPT.out,
+                TEST_STAGE_REMOTE.out,
+                TEST_PASS_FILE.out,
+                TEST_PASS_FOLDER.out,
+                TEST_PUBLISH_FILE.out,
+                TEST_PUBLISH_FOLDER.out,
+                TEST_IGNORED_FAIL.out,
+                TEST_MV_FILE.out,
+                TEST_MV_FOLDER_CONTENTS.out
+            )
+            .set { ch_out }
+
+    emit:
+        out = ch_out
 }
+
+workflow {
+    NF_CANARY()
+}
diff --git a/nextflow.config b/nextflow.config
@@ -1,11 +1,15 @@
 params {
     skip       = ''
+    run        = null
     outdir     = null
     remoteFile = null
 }
 
 process {
     container     = "docker.io/library/ubuntu:23.10"
     errorStrategy = "finish"
-    when          = { !params.skip.split(',').any{ it.toUpperCase().contains(task.process) } }
+    when          = { 
+        ( params.run ? params.run.split(',').any{ "NF_CANARY:${it.toUpperCase()}".contains(task.process) } : true ) && 
+        (!params.skip.split(',').any{ "NF_CANARY:${it.toUpperCase()}".contains(task.process) } ) 
+    }
 }
diff --git a/nextflow_schema.json b/nextflow_schema.json
@@ -16,6 +16,11 @@
                     "description": "Tests to skip as comma delimited values.",
                     "help_text": "Tests to skip as comma delimited values. E.g. --skip TEST_SUCCESS,TEST_INPUT. Case insensitive."
                 },
+                "run": {
+                    "type": "string",
+                    "description": "Selectively run tests  as comma delimited values",
+                    "help_text": "Tests to run as comma delimited values. E.g. --skip TEST_SUCCESS,TEST_INPUT. Case insensitive. Note this excludes all other tests."
+                },
                 "remoteFile": {
                     "type": "string",
                     "description": "Path to remote file to download and use.",

diff --git a/nf-test.config b/nf-test.config
@@ -2,7 +2,7 @@ config {
 
     testsDir "tests"
     workDir ".nf-test"
-    configFile "tests/nextflow.config"
+    configFile "nextflow.config"
     profile ""
 
 }
diff --git a/tests/main.nf.test b/tests/main.nf.test
@@ -1,14 +1,14 @@
-nextflow_pipeline {
+nextflow_pipeline{
 
-    name "Test Workflow main.nf"
-    tag "full"
+    name "Test nf-canary"
     script "main.nf"
 
     test("Should run without failures") {
 
         when {
             params {
-              remoteFile = "tests/testfile.txt"
+              remoteFile = "${baseDir}/tests/testfile.txt"
+              outdir     = "${launchDir}/output"
             }
         }
 
@@ -17,7 +17,7 @@ nextflow_pipeline {
             assert workflow.trace.tasks().size()     == 13
             assert workflow.trace.succeeded().size() == 12
             assert workflow.trace.failed().size()    == 1
-            assert snapshot(workflow.out).match()
+            assert snapshot(workflow, path(params.outdir).list()).match()
         }
 
     }
@@ -26,8 +26,9 @@ nextflow_pipeline {
 
         when {
             params {
-              skip = "test_success"
-              remoteFile = "tests/testfile.txt"
+              skip       = "test_success"
+              remoteFile = "${baseDir}/tests/testfile.txt"
+              outdir     = "${launchDir}/output"
             }
         }
 
@@ -36,7 +37,28 @@ nextflow_pipeline {
             assert workflow.trace.tasks().size()     == 12
             assert workflow.trace.succeeded().size() == 11
             assert workflow.trace.failed().size()    == 1
-            assert snapshot(workflow.out).match()
+            assert snapshot(workflow, path(params.outdir).list()).match()
+        }
+
+    }
+
+
+    test("Should only run one process") {
+
+        when {
+            params {
+              run        = "test_success"
+              remoteFile = "${baseDir}/tests/testfile.txt"
+              outdir     = "${launchDir}/output"
+            }
+        }
+
+        then {
+            assert workflow.success
+            assert workflow.trace.tasks().size()     == 1
+            assert workflow.trace.succeeded().size() == 1
+            assert workflow.trace.failed().size()    == 0
+            assert snapshot(workflow, path(params.outdir).list()).match()
         }
 
     }
@@ -45,16 +67,17 @@ nextflow_pipeline {
 
         when {
             params {
-              outdir = "results"
+              outdir     = "${launchDir}/output"
+              remoteFile = "${baseDir}/tests/testfile.txt"
             }
         }
 
         then {
             assert workflow.success
-            assert workflow.trace.tasks().size()     == 12
-            assert workflow.trace.succeeded().size() == 11
+            assert workflow.trace.tasks().size()     == 13
+            assert workflow.trace.succeeded().size() == 12
             assert workflow.trace.failed().size()    == 1
-            assert snapshot(workflow.out, path(params.outdir).list()).match()
+            assert snapshot(workflow, path("${launchDir}/output").list()).match()
         }
 
     }