Merge remote-tracking branch 'origin/features/required-version' into …

…features/required-version

docs/docs/assertions/snapshots.md

@@ -105,6 +105,14 @@ You can also use helper methods to add objects to snapshots. For example, you ca
  assert snapshot(workflow, path(params.outdir).list()).match()
 ```
 
+## Compressed Snapshots
+
+If you add complex objects to snapshots with large content, you could use the `md5()` function to store the hashsum instead of the content in the snapshot file:
+
+```Groovy
+ assert snapshot(hugeObject).md5().match()
+```
+
 ## File Paths
 
 If nf-test detects a path in the snapshot it automatically replace it by a unique *fingerprint* of the file that ensures the file content is the same. The fingerprint is default the md5 sum.

docs/docs/configuration.md

@@ -113,6 +113,10 @@ By understanding this profile evaluation order, you can effectively configure Ne
 
 ## File Staging
 
+!!! warning
+
+    File Staging is obsolete since version >= 0.9.0.
+
 The `stage` section of the `nf-test.config` file is used to define files that are needed by Nextflow in the test environment (`meta` directory). Additionally, the directories `lib`, `bin`, and `assets` are automatically staged.
 
 ### Supported Directives

docs/docs/testcases/setup.md

@@ -38,6 +38,17 @@ run("WorkflowName") {
 }
 ```
 
+If you need to run the same process multiple times, you can set the alias of the process:
+
+```groovy
+run("GENERATE_DATA", alias: "MY_PROCESS") {
+    script "./generate_data.nf"
+    process {
+       ...
+    }
+}
+```
+
 !!! warning
 
     Please keep in mind that changes in procsses or workflows, which are executed in the setup method, can result in a failed test run.
@@ -153,3 +164,61 @@ nextflow_process {
 
 }
 ```
+### 3. Aliasing of Dependencies
+
+In this example, the process `UNTAR` is used multiple times in the setup method:
+
+```groovy
+nextflow_process {
+
+    ...
+
+    setup {
+
+        run("UNTAR", alias: "UNTAR1") {
+            script "modules/nf-core/untar/main.nf"
+            process {
+            """
+            input[0] = Channel.fromList(...)
+            """
+            }
+        }
+
+        run("UNTAR", alias: "UNTAR2") {
+            script "modules/nf-core/untar/main.nf"
+            process {
+            """
+            input[0] = Channel.fromList(...)
+            """
+            }
+        }
+
+        run("UNTAR", alias: "UNTAR3") {
+            script "modules/nf-core/untar/main.nf"
+            process {
+            """
+            input[0] = Channel.fromList(...)
+            """
+            }
+        }
+    }
+
+	test("Test with three different inputs") {
+        when {
+            process {
+                """
+                input[0] = UNTAR1.out.untar.map{ it[1] }
+                input[1] = UNTAR2.out.untar.map{ it[1] }
+                input[2] = UNTAR3.out.untar.map{ it[1] }
+                """
+            }
+        }
+
+        then {
+            ...
+        }
+
+ }
+
+}
+```

docs/resources.md

@@ -9,6 +9,22 @@ This page collects videos and blog posts about nf-test created by the community.
 
 ---
 
+### :material-tooltip-text:{ style="color: #337ab7" } Training Module: Hello nf-test
+
+This training modules introduces nf-test in a simple and easy to follow-along hello-nextflow style aimed for beginners.
+
+[hello nf-test training module](https://training.nextflow.io/hello_nextflow/05_hello_nf-test/)
+
+---
+
+### :material-tooltip-text:{ style="color: #337ab7" } Blog post: Leveraging nf-test for enhanced quality control in nf-core
+
+Reproducibility is an important attribute of all good science. This is specially true in the realm of bioinformatics, where software is hopefully being constantly updated, and pipelines are ideally being maintained. This blog post covers nf-test in the nf-core context.
+
+[Read blog post](https://nextflow.io/blog/2024/nf-test-in-nf-core.html)
+
+---
+
 ### :fontawesome-brands-youtube:{ style="color: #EE0F0F" } nf-core/bytesize: Converting pytest modules to nf-test
 
 **Adam Talbot** & **Sateesh Peri** do a live demo of converting nf-core DSL2 modules pytests to nf-test

pom.xml

@@ -4,7 +4,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>com.askimed</groupId>
 	<artifactId>nf-test</artifactId>
-	<version>0.8.3</version>
+	<version>0.8.4</version>
 	<name>nf-test</name>
 	<description>Simple test framework for Nextflow pipelines</description>
 	<url>https://github.com/askimed/nf-test</url>

src/main/java/com/askimed/nf/test/App.java

@@ -17,7 +17,7 @@ public class App {
 
 	public static final String NAME = "nf-test";
 
-	public static final String VERSION = "0.8.3";
+	public static final String VERSION = "0.8.4";
 
 	public static final String PACKAGE =  "com.askimed.nf.test";
 

src/main/java/com/askimed/nf/test/commands/AbstractCommand.java

@@ -3,6 +3,7 @@
 import java.util.Arrays;
 import java.util.concurrent.Callable;
 
+import com.askimed.nf.test.nextflow.NextflowCommand;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -33,6 +34,7 @@ public Integer call() throws Exception {
 
 		log.info(App.NAME + " " + App.VERSION);
 		log.info("Arguments: " + Arrays.toString(App.args));
+		log.info("Nextflow Version: " + NextflowCommand.getVersion());
 
 		if (!silent) {
 			printHeader();
@@ -48,7 +50,7 @@ private void printHeader() {
 		System.out.println();
 		System.out.println(Emoji.ROCKET + AnsiText.bold(" " + App.NAME + " " + App.VERSION));
 		System.out.println("https://code.askimed.com/nf-test");
-		System.out.println("(c) 2021 - 2023 Lukas Forer and Sebastian Schoenherr");
+		System.out.println("(c) 2021 - 2024 Lukas Forer and Sebastian Schoenherr");
 		System.out.println();
 
 	}

src/main/java/com/askimed/nf/test/core/AbstractTest.java

@@ -2,17 +2,14 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.math.BigInteger;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
 import java.util.List;
 import java.util.Vector;
 
+import com.askimed.nf.test.util.HashUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.askimed.nf.test.config.Config;
-import com.askimed.nf.test.config.FileStaging;
 import com.askimed.nf.test.util.FileUtil;
 
 public abstract class AbstractTest implements ITest {
@@ -57,6 +54,8 @@ public abstract class AbstractTest implements ITest {
 
 	public File projectDir = new File(System.getProperty("user.dir"));
 
+	public File mockFile;
+
 	public boolean skipped = false;
 
 	protected File config = null;
@@ -95,35 +94,27 @@ public File getConfig() {
 		return config;
 	}
 
-	@Override
-	public void setup(Config config, File testDirectory) throws IOException {
+	public void defineDirectories(File testDirectory) throws IOException {
 
 		if (testDirectory == null) {
 			throw new IOException("Testcase setup failed: No home directory set");
 		}
 
-		launchDir = initDirectory("Launch Directory", testDirectory, DIRECTORY_TESTS, getHash());
-		metaDir = initDirectory("Meta Directory", launchDir, DIRECTORY_META);
-		outputDir = initDirectory("Output Directory", launchDir, DIRECTORY_OUTPUT);
-		workDir = initDirectory("Working Directory", launchDir, DIRECTORY_WORK);
-		FileStaging[] sharedDirectories = new FileStaging[]{
-				new FileStaging("bin", config != null ? config.getStageMode() : FileStaging.MODE_COPY),
-				new FileStaging("lib",  config != null ? config.getStageMode() : FileStaging.MODE_COPY),
-				new FileStaging("assets", config != null ? config.getStageMode() : FileStaging.MODE_COPY)
-		};
-		try {
-			// copy bin, assets and lib to metaDir
-			shareDirectories(sharedDirectories, metaDir);
-			if (config != null) {
-				// copy user defined staging directories
-				log.debug("Stage {} user provided files...", config.getStageBuilder().getPaths().size());
-				shareDirectories(config.getStageBuilder().getPaths(), metaDir);
-			}
-			shareDirectories(parent.getStageBuilder().getPaths(), metaDir);
-		} catch (Exception e) {
-			throw new IOException("Testcase setup failed: Directories could not be shared:\n" + e);
-		}
+		launchDir = constructDirectory(testDirectory, DIRECTORY_TESTS, getHash());
+		metaDir = constructDirectory(launchDir, DIRECTORY_META);
+		outputDir = constructDirectory(launchDir, DIRECTORY_OUTPUT);
+		workDir = constructDirectory(launchDir, DIRECTORY_WORK);
 
+		mockFile = new File( ".nf-test-" + getHash() + ".nf");
+		mockFile.deleteOnExit();
+	}
+
+	@Override
+	public void setup(Config config) throws IOException {
+		setupDirectory("Launch Directory", launchDir);
+		setupDirectory("Meta Directory", metaDir);
+		setupDirectory("Output Directory", outputDir);
+		setupDirectory("Working Directory", workDir);
 	}
 
 	@Override
@@ -136,15 +127,17 @@ public void execute() throws Throwable {
 		}
 	}
 
-	public File initDirectory(String name, File root, String... childs) throws IOException {
-
+	private File constructDirectory(File root, String... childs) {
 		String path = FileUtil.path(root.getAbsolutePath(), FileUtil.path(childs));
-
 		File directory = new File(path).getAbsoluteFile();
+		return directory;
+	}
+
+	private void setupDirectory(String name, File directory) throws IOException {
+
 		try {
 			FileUtil.deleteDirectory(directory);
 			FileUtil.createDirectory(directory);
-			return directory;
 		} catch (Exception e) {
 			throw new IOException(name + " '" + directory + "' could not be deleted or created:\n" + e);
 		}
@@ -153,7 +146,7 @@ public File initDirectory(String name, File root, String... childs) throws IOExc
 
 	@Override
 	public void cleanup() {
-		// FileUtil.deleteDirectory(metaDir);
+
 	}
 
 	@Override
@@ -180,23 +173,8 @@ public String getHash() {
 			throw new RuntimeException("Error generating hash");
 		}
 
-		return hash(parent.getFilename() + getName());
-
-	}
+		return HashUtil.getMd5(parent.getFilename() + getName());
 
-	private String hash(String value) {
-
-		MessageDigest md;
-		try {
-			md = MessageDigest.getInstance("MD5");
-			md.update(value.getBytes());
-			byte[] md5sum = md.digest();
-			BigInteger bigInt = new BigInteger(1, md5sum);
-			return bigInt.toString(16);
-		} catch (NoSuchAlgorithmException e) {
-			e.printStackTrace();
-			return "??";
-		}
 	}
 
 	public void tag(String tag) {
@@ -257,20 +235,6 @@ public boolean isWithTrace() {
 		return withTrace;
 	}
 
-	protected void shareDirectories(List<FileStaging> directories, File stageDir) throws IOException {
-		for (FileStaging directory : directories) {
-			String metaDirectory = FileUtil.path(stageDir.getAbsolutePath(), directory.getPath());
-			directory.stage(metaDirectory);
-		}
-	}
-
-	protected void shareDirectories(FileStaging[] directories, File stageDir) throws IOException {
-		for (FileStaging directory : directories) {
-			String metaDirectory = FileUtil.path(stageDir.getAbsolutePath(), directory.getPath());
-			directory.stage(metaDirectory);
-		}
-	}
-
 	@Override
 	public void setUpdateSnapshot(boolean updateSnapshot) {
 		this.updateSnapshot = updateSnapshot;

src/main/java/com/askimed/nf/test/core/AbstractTestSuite.java

@@ -82,14 +82,19 @@ public void evalualteTestClosures() throws Throwable {
 			Closure closure = namedClosure.closure;
 
 			ITest test = getNewTestInstance(testName);
-			test.setup(config, getHomeDirectory());
+			test.defineDirectories(getHomeDirectory());
 			closure.setDelegate(test);
 			closure.setResolveStrategy(Closure.DELEGATE_ONLY);
 			closure.call();
 			addTest(test);
 		}
 	}
 
+	public void setupTest(ITest test) throws Throwable {
+		test.setup(config);
+	}
+
+
 	protected abstract ITest getNewTestInstance(String name);
 
 	public void setScript(String script) {

src/main/java/com/askimed/nf/test/core/ITest.java

@@ -6,7 +6,9 @@
 
 public interface ITest extends ITaggable {
 
-	public void setup(Config config, File homeDirectory) throws Throwable;
+	public void defineDirectories(File testDirectory) throws Throwable;
+
+	public void setup(Config config) throws Throwable;
 
 	public void execute() throws Throwable;
 

src/main/java/com/askimed/nf/test/core/ITestSuite.java

@@ -36,4 +36,6 @@ public interface ITestSuite extends ITaggable {
 
 	public void evalualteTestClosures() throws Throwable;
 
-}
+	public void setupTest(ITest test) throws Throwable;
+
+}

src/main/java/com/askimed/nf/test/core/TestExecutionEngine.java

@@ -199,6 +199,8 @@ public int execute() throws Throwable {
 				log.info("Run test '{}'. type: {}", test, test.getClass().getName());
 				totalTests++;
 
+				testSuite.setupTest(test);
+
 				listener.executionStarted(test);
 				TestExecutionResult result = new TestExecutionResult(test);
 				test.setWithTrace(withTrace);