Nested params and json/yaml files (#50)

* Add function to load params from json or yaml file

* Add testcases

* Update broken testcases

* Support nested params

* Switch to iterative approach to handle nested params

* Fix naming of yaml property

* Switch to yamlSlurper

* Add params to documentation

.gitignore

@@ -7,7 +7,6 @@ output
 .nextflow
 .nextflow.*
 .settings
-params.json
 test_mock.nf
 /.nf-test/
 site

docs/testcases/global_variables.md

@@ -1,6 +1,63 @@
-# Global Variables
+# Params and Global Variables
 
-## outputDir
+## Params
+
+The `params` block is optional and is a simple map that can be used to overwrite Nextflow's input `params`. The `params` block is located in the `when` block of a testcase. You can set params manually:
+
+```Groovy
+when {
+    params {
+        outdir = "output"
+    }
+}
+```
+
+It is also possible to set nested params using the same syntax as in your Nextflow script:
+
+```Groovy
+when {
+    params {
+        output {
+          dir = "output"
+        }
+    }
+}
+```
+
+In addition, you can load the `params` from a JSON file:
+
+```Groovy
+when {
+    params {
+        load("$baseDir/tests/params.json")
+    }
+}
+```
+
+or from a YAML file:
+
+```Groovy
+when {
+    params {
+        load("$baseDir/tests/params.yaml")
+    }
+}
+```
+
+nf-test allows to combine both techniques and therefor it is possible to overwrite one or more `params` from the json file:
+
+```Groovy
+when {
+    params {
+        load("$baseDir/tests/params.json")
+        outputDir = "new/output/path"
+    }
+}
+```
+
+## Global Variables
+
+### `outputDir`
 
 This variable points to the directory within the temporary test directory (`.nf-test/tests/<test-dir>/output/`). The variable can be set under params:
 
@@ -10,14 +67,14 @@ params {
 }
 ```
 
-## baseDir
+### `baseDir`
 
-This variable points to the directory to locate the base directory of the main nf-test config. The variable can be used e.g. in the process definition:
+This variable points to the directory to locate the base directory of the main nf-test config. The variable can be used e.g. in the process definition to build absolute paths for input files:
 
 ```Groovy
 process {
     """
-    f1 = file('$baseDir/tests/input/file123.gz')
+    file1 = file("$baseDir/tests/input/file123.gz")
     """
 }
 ```

example/hello.nf.test

@@ -6,14 +6,9 @@ nextflow_pipeline {
   test("hello world example should start 4 processes") {
 
     expect {
-      with(workflow){
-        assert success
-        assert trace.tasks().size() == 4
-        assert "Ciao world!" in stdout
-        assert "Bonjour world!" in stdout
-        assert "Hello world!" in stdout
-        assert "Hola world!" in stdout
-      }
+      assert workflow.success
+		assert snapshot(workflow).match()
+
     }
 
   }

mkdocs.yml

@@ -29,7 +29,7 @@ nav:
     - Workflow Testing: testcases/nextflow_workflow.md
     - Process Testing: testcases/nextflow_process.md
     - Function Testing: testcases/nextflow_function.md
-    - Global Variables: testcases/global_variables.md
+    - Params and Global Variables: testcases/global_variables.md
   - Writing Assertions:
     - Power Assertions: assertions/assertions.md
     - Files: assertions/files.md

src/main/java/com/askimed/nf/test/lang/ParamsMap.java

@@ -1,13 +1,27 @@
 package com.askimed.nf.test.lang;
 
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
 import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Queue;
+
+import org.codehaus.groovy.control.CompilationFailedException;
+
+import groovy.json.JsonSlurper;
+import groovy.lang.Closure;
+import groovy.lang.Writable;
+import groovy.text.SimpleTemplateEngine;
+import groovy.yaml.YamlSlurper;
 
 public class ParamsMap extends HashMap<String, Object> {
 
 	private static final long serialVersionUID = 1L;
 
-	public String baseDir = "lukas";
-	
+	public String baseDir = "";
+
 	public String outputDir = "";
 
 	public void setBaseDir(String baseDir) {
@@ -20,13 +34,152 @@ public void setOutputDir(String outputDir) {
 		put("outputDir", outputDir);
 	}
 
-
 	public String getBaseDir() {
 		return baseDir;
 	}
-	
+
 	public String getOutputDir() {
 		return outputDir;
 	}
 
+	public void load(String filename) throws CompilationFailedException, ClassNotFoundException, IOException {
+		load(new File(filename));
+	}
+
+	public void load(File file) throws CompilationFailedException, ClassNotFoundException, IOException {
+
+		if (file.getName().endsWith(".json")) {
+
+			loadFromJsonFile(file);
+
+		} else if (file.getName().endsWith(".yaml") || file.getName().endsWith(".yml")) {
+
+			loadFromYamlFile(file);
+
+		} else {
+
+			throw new RuntimeException(
+					"Could no load params from file '" + file.getAbsolutePath() + "': Unsupported file format");
+
+		}
+	}
+
+	public void loadFromJsonFile(File file) throws CompilationFailedException, ClassNotFoundException, IOException {
+
+		JsonSlurper jsonSlurper = new JsonSlurper();
+		Map<String, Object> map = (Map<String, Object>) jsonSlurper.parse(file);
+		map.putAll(this);
+
+		loadFromMap(map);
+
+	}
+
+	public void loadFromYamlFile(File file) throws CompilationFailedException, ClassNotFoundException, IOException {
+
+		YamlSlurper yamlSlurper = new YamlSlurper();
+		Map<String, Object> map = (Map<String, Object>) yamlSlurper.parse(new FileReader(file));
+		map.putAll(this);
+
+		loadFromMap(map);
+
+	}
+
+	public synchronized void loadFromMap(Map<String, Object> map)
+			throws CompilationFailedException, ClassNotFoundException, IOException {
+
+		evaluteTemplates(map);
+		putAll(map);
+
+	}
+
+	protected synchronized void evaluteTemplates(Map<String, Object> map)
+			throws CompilationFailedException, ClassNotFoundException, IOException {
+
+		SimpleTemplateEngine engine = new SimpleTemplateEngine();
+
+		Queue<Map<String, Object>> queue = new LinkedList<Map<String, Object>>();
+		queue.add(map);
+
+		while (queue.size() > 0) {
+
+			Map<String, Object> item = queue.remove(); // Pop Item
+
+			for (String key : item.keySet()) {
+
+				Object value = item.get(key);
+
+				if (value instanceof String) {
+
+					String template = value.toString();
+					Map<String, Object> binding = new HashMap<String, Object>(map);
+					Writable evaluatedTemplate = engine.createTemplate(template).make(binding);
+					item.put(key, evaluatedTemplate.toString());
+
+				} else if (value instanceof Map) {
+
+					Map<String, Object> nestedMap = createNestedMap((Map<String, Object>) value);
+					queue.add(nestedMap); // Add to queue instead of recurse
+					item.put(key, nestedMap);
+
+				} else {
+
+					item.put(key, value);
+
+				}
+			}
+
+		}
+
+	}
+
+	public void evaluateNestedClosures() {
+
+		evaluateNestedClosures(this);
+
+	}
+
+	protected void evaluateNestedClosures(Map<String, Object> map) {
+
+		Queue<Map<String, Object>> queue = new LinkedList<Map<String, Object>>();
+		queue.add(map);
+
+		while (queue.size() > 0) {
+
+			Map<String, Object> item = queue.remove();
+
+			for (String key : item.keySet()) {
+				Object value = item.get(key);
+
+				if (!(value instanceof Closure))
+					continue;
+
+				Map<String, Object> nestedMap = createNestedMap();
+				Closure closure = (Closure) value;
+				closure.setDelegate(nestedMap);
+				closure.setResolveStrategy(Closure.DELEGATE_FIRST);
+				closure.call();
+				item.put(key, nestedMap);
+
+				queue.add(nestedMap); // Instead of recursion
+
+			}
+
+		}
+
+	}
+
+	protected Map<String, Object> createNestedMap() {
+		return createNestedMap(null);
+	}
+
+	protected Map<String, Object> createNestedMap(Map<String, Object> map) {
+		Map<String, Object> nestedMap = new HashMap<String, Object>();
+		nestedMap.put("baseDir", baseDir);
+		nestedMap.put("outputDir", outputDir);
+		if (map != null) {
+			nestedMap.putAll(map);
+		}
+		return nestedMap;
+	}
+
 }

src/main/java/com/askimed/nf/test/lang/TestContext.java

@@ -1,5 +1,9 @@
 package com.askimed.nf.test.lang;
 
+import java.io.IOException;
+
+import org.codehaus.groovy.control.CompilationFailedException;
+
 import com.askimed.nf.test.core.ITest;
 import com.askimed.nf.test.lang.extensions.Snapshot;
 import com.askimed.nf.test.lang.function.Function;
@@ -72,19 +76,15 @@ public void params(Closure closure) {
 		this.paramsClosure = closure;
 	}
 
-	public void evaluateParamsClosure(String baseDir, String outputDir) {
-		params.setBaseDir(baseDir);
-		params.setOutputDir(outputDir);
-		this.baseDir = baseDir;
-		this.outputDir = outputDir;
-
+	public void evaluateParamsClosure() {
 		if (paramsClosure == null) {
 			return;
 		}
 		paramsClosure.setDelegate(params);
 		paramsClosure.setResolveStrategy(Closure.DELEGATE_FIRST);
 		paramsClosure.call();
 		paramsClosure.getMetaClass().getProperties();
+		params.evaluateNestedClosures();
 
 	}
 
@@ -144,8 +144,14 @@ public Snapshot snapshot(Object ... object ) {
 	}
 
 	public void init(String baseDir, String outputDir) {
+		params.setBaseDir(baseDir);
+		params.setOutputDir(outputDir);
 		this.baseDir = baseDir;
 		this.outputDir = outputDir;
 	}
 
+	public void loadParams(String filename) throws CompilationFailedException, ClassNotFoundException, IOException {
+		params.load(filename);
+	}
+
 }

src/main/java/com/askimed/nf/test/lang/extensions/PathExtension.java

@@ -50,8 +50,8 @@ public static Object getJson(Path self) throws FileNotFoundException, IOExceptio
 	/* YAML */
 
 	public static Object getYaml(Path self) throws FileNotFoundException, IOException {
-		YamlSlurper YamlSlurper = new YamlSlurper();
-		return YamlSlurper.parse(self);
+		YamlSlurper yamlSlurper = new YamlSlurper();
+		return yamlSlurper.parse(self);
 	}
 
 	/* File methods */

src/main/java/com/askimed/nf/test/lang/function/FunctionTest.java

@@ -115,7 +115,7 @@ public void execute() throws Throwable {
 			when.execute(context);
 		}
 
-		context.evaluateParamsClosure(baseDir, outputDir.getAbsolutePath());
+		context.evaluateParamsClosure();
 		context.evaluateFunctionClosure();
 
 		// Create workflow mock

src/main/java/com/askimed/nf/test/lang/pipeline/PipelineTest.java

@@ -90,7 +90,7 @@ public void execute() throws Throwable {
 			when.execute(context);
 		}
 
-		context.evaluateParamsClosure(baseDir, outputDir.getAbsolutePath());
+		context.evaluateParamsClosure();
 		context.evaluateProcessClosure();
 
 		if (debug) {

src/main/java/com/askimed/nf/test/lang/process/ProcessTest.java

@@ -106,7 +106,7 @@ public void execute() throws Throwable {
 			when.execute(context);
 		}
 
-		context.evaluateParamsClosure(baseDir, outputDir.getAbsolutePath());
+		context.evaluateParamsClosure();
 		context.evaluateProcessClosure();
 
 		// Create workflow mock

src/main/java/com/askimed/nf/test/lang/workflow/WorkflowTest.java

@@ -124,7 +124,7 @@ public void execute() throws Throwable {
 			when.execute(context);
 		}
 
-		context.evaluateParamsClosure(baseDir, outputDir.getAbsolutePath());
+		context.evaluateParamsClosure();
 		context.evaluateWorkflowClosure();
 
 		// Create workflow mock