diff --git a/.gitignore b/.gitignore index 13161b0c..41393f50 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ output .nextflow .nextflow.* .settings -params.json test_mock.nf /.nf-test/ site diff --git a/docs/testcases/global_variables.md b/docs/testcases/global_variables.md index 98e39647..06406e66 100644 --- a/docs/testcases/global_variables.md +++ b/docs/testcases/global_variables.md @@ -1,6 +1,63 @@ -# Global Variables +# Params and Global Variables -## outputDir +## Params + +The `params` block is optional and is a simple map that can be used to overwrite Nextflow's input `params`. The `params` block is located in the `when` block of a testcase. You can set params manually: + +```Groovy +when { + params { + outdir = "output" + } +} +``` + +It is also possible to set nested params using the same syntax as in your Nextflow script: + +```Groovy +when { + params { + output { + dir = "output" + } + } +} +``` + +In addition, you can load the `params` from a JSON file: + +```Groovy +when { + params { + load("$baseDir/tests/params.json") + } +} +``` + +or from a YAML file: + +```Groovy +when { + params { + load("$baseDir/tests/params.yaml") + } +} +``` + +nf-test allows to combine both techniques and therefor it is possible to overwrite one or more `params` from the json file: + +```Groovy +when { + params { + load("$baseDir/tests/params.json") + outputDir = "new/output/path" + } +} +``` + +## Global Variables + +### `outputDir` This variable points to the directory within the temporary test directory (`.nf-test/tests//output/`). The variable can be set under params: @@ -10,14 +67,14 @@ params { } ``` -## baseDir +### `baseDir` -This variable points to the directory to locate the base directory of the main nf-test config. The variable can be used e.g. in the process definition: +This variable points to the directory to locate the base directory of the main nf-test config. The variable can be used e.g. in the process definition to build absolute paths for input files: ```Groovy process { """ - f1 = file('$baseDir/tests/input/file123.gz') + file1 = file("$baseDir/tests/input/file123.gz") """ } ``` diff --git a/example/hello.nf.test b/example/hello.nf.test index 5607270d..ce3b95cb 100644 --- a/example/hello.nf.test +++ b/example/hello.nf.test @@ -6,14 +6,9 @@ nextflow_pipeline { test("hello world example should start 4 processes") { expect { - with(workflow){ - assert success - assert trace.tasks().size() == 4 - assert "Ciao world!" in stdout - assert "Bonjour world!" in stdout - assert "Hello world!" in stdout - assert "Hola world!" in stdout - } + assert workflow.success + assert snapshot(workflow).match() + } } diff --git a/mkdocs.yml b/mkdocs.yml index 0ac45511..82b5a008 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -29,7 +29,7 @@ nav: - Workflow Testing: testcases/nextflow_workflow.md - Process Testing: testcases/nextflow_process.md - Function Testing: testcases/nextflow_function.md - - Global Variables: testcases/global_variables.md + - Params and Global Variables: testcases/global_variables.md - Writing Assertions: - Power Assertions: assertions/assertions.md - Files: assertions/files.md diff --git a/src/main/java/com/askimed/nf/test/lang/ParamsMap.java b/src/main/java/com/askimed/nf/test/lang/ParamsMap.java index a80c957a..cb66555c 100644 --- a/src/main/java/com/askimed/nf/test/lang/ParamsMap.java +++ b/src/main/java/com/askimed/nf/test/lang/ParamsMap.java @@ -1,13 +1,27 @@ package com.askimed.nf.test.lang; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; +import java.util.Queue; + +import org.codehaus.groovy.control.CompilationFailedException; + +import groovy.json.JsonSlurper; +import groovy.lang.Closure; +import groovy.lang.Writable; +import groovy.text.SimpleTemplateEngine; +import groovy.yaml.YamlSlurper; public class ParamsMap extends HashMap { private static final long serialVersionUID = 1L; - public String baseDir = "lukas"; - + public String baseDir = ""; + public String outputDir = ""; public void setBaseDir(String baseDir) { @@ -20,13 +34,152 @@ public void setOutputDir(String outputDir) { put("outputDir", outputDir); } - public String getBaseDir() { return baseDir; } - + public String getOutputDir() { return outputDir; } + public void load(String filename) throws CompilationFailedException, ClassNotFoundException, IOException { + load(new File(filename)); + } + + public void load(File file) throws CompilationFailedException, ClassNotFoundException, IOException { + + if (file.getName().endsWith(".json")) { + + loadFromJsonFile(file); + + } else if (file.getName().endsWith(".yaml") || file.getName().endsWith(".yml")) { + + loadFromYamlFile(file); + + } else { + + throw new RuntimeException( + "Could no load params from file '" + file.getAbsolutePath() + "': Unsupported file format"); + + } + } + + public void loadFromJsonFile(File file) throws CompilationFailedException, ClassNotFoundException, IOException { + + JsonSlurper jsonSlurper = new JsonSlurper(); + Map map = (Map) jsonSlurper.parse(file); + map.putAll(this); + + loadFromMap(map); + + } + + public void loadFromYamlFile(File file) throws CompilationFailedException, ClassNotFoundException, IOException { + + YamlSlurper yamlSlurper = new YamlSlurper(); + Map map = (Map) yamlSlurper.parse(new FileReader(file)); + map.putAll(this); + + loadFromMap(map); + + } + + public synchronized void loadFromMap(Map map) + throws CompilationFailedException, ClassNotFoundException, IOException { + + evaluteTemplates(map); + putAll(map); + + } + + protected synchronized void evaluteTemplates(Map map) + throws CompilationFailedException, ClassNotFoundException, IOException { + + SimpleTemplateEngine engine = new SimpleTemplateEngine(); + + Queue> queue = new LinkedList>(); + queue.add(map); + + while (queue.size() > 0) { + + Map item = queue.remove(); // Pop Item + + for (String key : item.keySet()) { + + Object value = item.get(key); + + if (value instanceof String) { + + String template = value.toString(); + Map binding = new HashMap(map); + Writable evaluatedTemplate = engine.createTemplate(template).make(binding); + item.put(key, evaluatedTemplate.toString()); + + } else if (value instanceof Map) { + + Map nestedMap = createNestedMap((Map) value); + queue.add(nestedMap); // Add to queue instead of recurse + item.put(key, nestedMap); + + } else { + + item.put(key, value); + + } + } + + } + + } + + public void evaluateNestedClosures() { + + evaluateNestedClosures(this); + + } + + protected void evaluateNestedClosures(Map map) { + + Queue> queue = new LinkedList>(); + queue.add(map); + + while (queue.size() > 0) { + + Map item = queue.remove(); + + for (String key : item.keySet()) { + Object value = item.get(key); + + if (!(value instanceof Closure)) + continue; + + Map nestedMap = createNestedMap(); + Closure closure = (Closure) value; + closure.setDelegate(nestedMap); + closure.setResolveStrategy(Closure.DELEGATE_FIRST); + closure.call(); + item.put(key, nestedMap); + + queue.add(nestedMap); // Instead of recursion + + } + + } + + } + + protected Map createNestedMap() { + return createNestedMap(null); + } + + protected Map createNestedMap(Map map) { + Map nestedMap = new HashMap(); + nestedMap.put("baseDir", baseDir); + nestedMap.put("outputDir", outputDir); + if (map != null) { + nestedMap.putAll(map); + } + return nestedMap; + } + } diff --git a/src/main/java/com/askimed/nf/test/lang/TestContext.java b/src/main/java/com/askimed/nf/test/lang/TestContext.java index 659d3ba4..f6ced80e 100644 --- a/src/main/java/com/askimed/nf/test/lang/TestContext.java +++ b/src/main/java/com/askimed/nf/test/lang/TestContext.java @@ -1,5 +1,9 @@ package com.askimed.nf.test.lang; +import java.io.IOException; + +import org.codehaus.groovy.control.CompilationFailedException; + import com.askimed.nf.test.core.ITest; import com.askimed.nf.test.lang.extensions.Snapshot; import com.askimed.nf.test.lang.function.Function; @@ -72,12 +76,7 @@ public void params(Closure closure) { this.paramsClosure = closure; } - public void evaluateParamsClosure(String baseDir, String outputDir) { - params.setBaseDir(baseDir); - params.setOutputDir(outputDir); - this.baseDir = baseDir; - this.outputDir = outputDir; - + public void evaluateParamsClosure() { if (paramsClosure == null) { return; } @@ -85,6 +84,7 @@ public void evaluateParamsClosure(String baseDir, String outputDir) { paramsClosure.setResolveStrategy(Closure.DELEGATE_FIRST); paramsClosure.call(); paramsClosure.getMetaClass().getProperties(); + params.evaluateNestedClosures(); } @@ -144,8 +144,14 @@ public Snapshot snapshot(Object ... object ) { } public void init(String baseDir, String outputDir) { + params.setBaseDir(baseDir); + params.setOutputDir(outputDir); this.baseDir = baseDir; this.outputDir = outputDir; } + public void loadParams(String filename) throws CompilationFailedException, ClassNotFoundException, IOException { + params.load(filename); + } + } diff --git a/src/main/java/com/askimed/nf/test/lang/extensions/PathExtension.java b/src/main/java/com/askimed/nf/test/lang/extensions/PathExtension.java index e171698e..e521f3e3 100644 --- a/src/main/java/com/askimed/nf/test/lang/extensions/PathExtension.java +++ b/src/main/java/com/askimed/nf/test/lang/extensions/PathExtension.java @@ -50,8 +50,8 @@ public static Object getJson(Path self) throws FileNotFoundException, IOExceptio /* YAML */ public static Object getYaml(Path self) throws FileNotFoundException, IOException { - YamlSlurper YamlSlurper = new YamlSlurper(); - return YamlSlurper.parse(self); + YamlSlurper yamlSlurper = new YamlSlurper(); + return yamlSlurper.parse(self); } /* File methods */ diff --git a/src/main/java/com/askimed/nf/test/lang/function/FunctionTest.java b/src/main/java/com/askimed/nf/test/lang/function/FunctionTest.java index b313f2b1..fddbd82e 100644 --- a/src/main/java/com/askimed/nf/test/lang/function/FunctionTest.java +++ b/src/main/java/com/askimed/nf/test/lang/function/FunctionTest.java @@ -115,7 +115,7 @@ public void execute() throws Throwable { when.execute(context); } - context.evaluateParamsClosure(baseDir, outputDir.getAbsolutePath()); + context.evaluateParamsClosure(); context.evaluateFunctionClosure(); // Create workflow mock diff --git a/src/main/java/com/askimed/nf/test/lang/pipeline/PipelineTest.java b/src/main/java/com/askimed/nf/test/lang/pipeline/PipelineTest.java index e24e4fcc..2fee7cb5 100644 --- a/src/main/java/com/askimed/nf/test/lang/pipeline/PipelineTest.java +++ b/src/main/java/com/askimed/nf/test/lang/pipeline/PipelineTest.java @@ -90,7 +90,7 @@ public void execute() throws Throwable { when.execute(context); } - context.evaluateParamsClosure(baseDir, outputDir.getAbsolutePath()); + context.evaluateParamsClosure(); context.evaluateProcessClosure(); if (debug) { diff --git a/src/main/java/com/askimed/nf/test/lang/process/ProcessTest.java b/src/main/java/com/askimed/nf/test/lang/process/ProcessTest.java index ac48b8cb..d4f68b9f 100644 --- a/src/main/java/com/askimed/nf/test/lang/process/ProcessTest.java +++ b/src/main/java/com/askimed/nf/test/lang/process/ProcessTest.java @@ -106,7 +106,7 @@ public void execute() throws Throwable { when.execute(context); } - context.evaluateParamsClosure(baseDir, outputDir.getAbsolutePath()); + context.evaluateParamsClosure(); context.evaluateProcessClosure(); // Create workflow mock diff --git a/src/main/java/com/askimed/nf/test/lang/workflow/WorkflowTest.java b/src/main/java/com/askimed/nf/test/lang/workflow/WorkflowTest.java index 0dcc0c8a..5d8e8819 100644 --- a/src/main/java/com/askimed/nf/test/lang/workflow/WorkflowTest.java +++ b/src/main/java/com/askimed/nf/test/lang/workflow/WorkflowTest.java @@ -124,7 +124,7 @@ public void execute() throws Throwable { when.execute(context); } - context.evaluateParamsClosure(baseDir, outputDir.getAbsolutePath()); + context.evaluateParamsClosure(); context.evaluateWorkflowClosure(); // Create workflow mock diff --git a/src/test/java/com/askimed/nf/test/lang/NextflowTestSuiteBuilderTest.java b/src/test/java/com/askimed/nf/test/lang/NextflowTestSuiteBuilderTest.java index e2927980..adb32c7c 100644 --- a/src/test/java/com/askimed/nf/test/lang/NextflowTestSuiteBuilderTest.java +++ b/src/test/java/com/askimed/nf/test/lang/NextflowTestSuiteBuilderTest.java @@ -11,7 +11,7 @@ public class NextflowTestSuiteBuilderTest { @Test public void testParse() throws Exception { File file = new File("test-data/pipeline/dsl1/test1.nf.test"); - assertEquals(2, TestSuiteBuilder.parse(file).getTests().size()); + assertEquals(5, TestSuiteBuilder.parse(file).getTests().size()); } } diff --git a/src/test/java/com/askimed/nf/test/lang/ProcessTest.java b/src/test/java/com/askimed/nf/test/lang/ProcessTest.java index e767cbc2..be4656df 100644 --- a/src/test/java/com/askimed/nf/test/lang/ProcessTest.java +++ b/src/test/java/com/askimed/nf/test/lang/ProcessTest.java @@ -117,4 +117,14 @@ public void testChannelFolder() throws Exception { } + + @Test + public void testNestedParams() throws Exception { + + App app = new App(); + int exitCode = app.run(new String[] { "test", "test-data/process/nested-params/process.nf.test" }); + assertEquals(0, exitCode); + + } + } diff --git a/test-data/pipeline/dsl1/params.json b/test-data/pipeline/dsl1/params.json new file mode 100644 index 00000000..85c201f9 --- /dev/null +++ b/test-data/pipeline/dsl1/params.json @@ -0,0 +1,8 @@ +{ + "lukas": 28, + "forer": "22", + "outdir": "${outputDir}", + "test": { + "lukas": 22 + } +} diff --git a/test-data/pipeline/dsl1/params.yml b/test-data/pipeline/dsl1/params.yml new file mode 100644 index 00000000..a64242dc --- /dev/null +++ b/test-data/pipeline/dsl1/params.yml @@ -0,0 +1,5 @@ +lukas: 28 +forer: "22" +outdir: "${outputDir}" +test: + lukas: 22 diff --git a/test-data/pipeline/dsl1/test1.nf.test b/test-data/pipeline/dsl1/test1.nf.test index f1d88fee..beef6376 100644 --- a/test-data/pipeline/dsl1/test1.nf.test +++ b/test-data/pipeline/dsl1/test1.nf.test @@ -28,6 +28,65 @@ nextflow_pipeline { } + test("Should create 5 output files from params file") { + + when { + loadParams("$baseDir/test-data/pipeline/dsl1/params.json") + } + + then { + assert workflow.success + assert params.lukas == 28 + for (def i in 1..5){ + def file = new File("${params.outdir}/${i}_28.txt"); + assert file.exists() + assert file.text == "lukas forer ${i}\n" + } + } + + } + + test("Should create 5 output files from params file 2") { + + when { + params { + load("$baseDir/test-data/pipeline/dsl1/params.json") + lukas = 44 + } + } + + then { + assert workflow.success + assert params.lukas == 44 + for (def i in 1..5){ + def file = new File("${params.outdir}/${i}_44.txt"); + assert file.exists() + assert file.text == "lukas forer ${i}\n" + } + } + + } + + test("Should create 5 output files from yaml file") { + + when { + params { + load("$baseDir/test-data/pipeline/dsl1/params.yml") + } + } + + then { + assert workflow.success + assert params.lukas == 28 + for (def i in 1..5){ + def file = new File("${params.outdir}/${i}_28.txt"); + assert file.exists() + assert file.text == "lukas forer ${i}\n" + } + } + + } + test("Should fail if outdir is missing") { //debug true diff --git a/test-data/process/nested-params/params.json b/test-data/process/nested-params/params.json new file mode 100644 index 00000000..126125ce --- /dev/null +++ b/test-data/process/nested-params/params.json @@ -0,0 +1,9 @@ +{ + "var1": "var1", + "nested": { + "var2": "var2", + "nested": { + "var3": "$baseDir" + } + } +} diff --git a/test-data/process/nested-params/params.yaml b/test-data/process/nested-params/params.yaml new file mode 100644 index 00000000..962fcb0f --- /dev/null +++ b/test-data/process/nested-params/params.yaml @@ -0,0 +1,5 @@ +var1: "var1" +nested: + var2: "var2" + nested: + var3: "$baseDir" diff --git a/test-data/process/nested-params/process.nf b/test-data/process/nested-params/process.nf new file mode 100644 index 00000000..73a41181 --- /dev/null +++ b/test-data/process/nested-params/process.nf @@ -0,0 +1,35 @@ +process TEST_PROCESS { + + publishDir "${params.outdir}", mode: 'copy' + + input: + val var1 + val var2 + val var3 + + output: + tuple val(var1), val(var2), emit: my_tuples + + script: + + println "var1: ${params.var1}" + println "var2: ${params.nested.var2}" + println "var3: ${params.nested.nested.var3}" + + if (var1 != params.var1){ + error "var1: $var1 vs. ${params.var1}" + } + + if (var2 != params.nested.var2){ + error "var2: $var2 vs. ${params.nested.var2}" + } + + if (var3 != params.nested.nested.var3){ + error "var3: $var3 vs. ${params.nested.nested.var3}" + } + + + """ + """ + +} diff --git a/test-data/process/nested-params/process.nf.test b/test-data/process/nested-params/process.nf.test new file mode 100644 index 00000000..e8f95771 --- /dev/null +++ b/test-data/process/nested-params/process.nf.test @@ -0,0 +1,167 @@ +nextflow_process { + + name "Test process xy" + + script "test-data/process/nested-params/process.nf" + process "TEST_PROCESS" + + test("Test nested params with closure 1") { + + when { + + params { + var1 = "var1" + nested = { + var2 = "var2" + nested = { + var3 = "$baseDir" + } + } + } + + process { + """ + input[0] = "var1" + input[1] = "var2" + input[2] = "$baseDir" + """ + } + } + + then { + assert process.success + } + + } + + + test("Test nested params with closure 2") { + + when { + + params { + var1 = "var1" + nested = { + var2 = "var2" + nested = { + var3 = baseDir + } + } + } + + process { + """ + input[0] = "var1" + input[1] = "var2" + input[2] = "$baseDir" + """ + } + } + + then { + assert process.success + } + + } + + test("Test nested params with groovy syntax 1") { + + when { + + params { + var1 = "var1" + nested = { + var2 = "var2" + nested = [ var3: "$baseDir" ] + } + } + + process { + """ + input[0] = "var1" + input[1] = "var2" + input[2] = "$baseDir" + """ + } + } + + then { + assert process.success + } + + } + + test("Test nested params with groovy syntax 2") { + + when { + + params { + var1 = "var1" + nested = { + var2 = "var2" + nested = [ var3: baseDir ] + } + } + + process { + """ + input[0] = "var1" + input[1] = "var2" + input[2] = "$baseDir" + """ + } + } + + then { + assert process.success + } + + } + + test("Test nested params from json file") { + + when { + + params { + load("$baseDir/test-data/process/nested-params/params.json"); + } + + process { + """ + input[0] = "var1" + input[1] = "var2" + input[2] = "$baseDir" + """ + } + } + + then { + assert process.success + } + + } + + test("Test nested params from yaml file") { + + when { + + params { + load("$baseDir/test-data/process/nested-params/params.yaml"); + } + + process { + """ + input[0] = "var1" + input[1] = "var2" + input[2] = "$baseDir" + """ + } + } + + then { + assert process.success + } + + } + +} \ No newline at end of file