From ecaedf8cd608b1ed7d51749830623b6939c658d4 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Tue, 30 Jan 2024 15:55:05 +0100 Subject: [PATCH 01/25] Prepare release 0.8.4 --- pom.xml | 2 +- src/main/java/com/askimed/nf/test/App.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index cb35d6cb..73cbd4bf 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.askimed nf-test - 0.8.3 + 0.8.4 nf-test Simple test framework for Nextflow pipelines https://github.com/askimed/nf-test diff --git a/src/main/java/com/askimed/nf/test/App.java b/src/main/java/com/askimed/nf/test/App.java index adcf0074..d9f3e8cf 100644 --- a/src/main/java/com/askimed/nf/test/App.java +++ b/src/main/java/com/askimed/nf/test/App.java @@ -17,7 +17,7 @@ public class App { public static final String NAME = "nf-test"; - public static final String VERSION = "0.8.3"; + public static final String VERSION = "0.8.4"; public static final String PACKAGE = "com.askimed.nf.test"; From 18bde822f501d537c014eef09f861f54acb1498d Mon Sep 17 00:00:00 2001 From: Sateesh_Peri <33637490+sateeshperi@users.noreply.github.com> Date: Wed, 3 Apr 2024 10:26:19 -0400 Subject: [PATCH 02/25] Update resources.md (#207) add new nf-test in nf-core blog --- docs/resources.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/resources.md b/docs/resources.md index 33a1dd87..40f6d3a6 100644 --- a/docs/resources.md +++ b/docs/resources.md @@ -9,6 +9,14 @@ This page collects videos and blog posts about nf-test created by the community. --- +### :material-tooltip-text:{ style="color: #337ab7" } Blog post: Leveraging nf-test for enhanced quality control in nf-core + +Reproducibility is an important attribute of all good science. This is specially true in the realm of bioinformatics, where software is hopefully being constantly updated, and pipelines are ideally being maintained. This blog post covers nf-test in the nf-core context. + +[Read blog post](https://nextflow.io/blog/2024/nf-test-in-nf-core.html) + +--- + ### :fontawesome-brands-youtube:{ style="color: #EE0F0F" } nf-core/bytesize: Converting pytest modules to nf-test **Adam Talbot** & **Sateesh Peri** do a live demo of converting nf-core DSL2 modules pytests to nf-test From 9dafa52e14e245b3a524d74d65ad9833e2746d9a Mon Sep 17 00:00:00 2001 From: Sateesh_Peri <33637490+sateeshperi@users.noreply.github.com> Date: Tue, 30 Apr 2024 11:50:31 -0400 Subject: [PATCH 03/25] Update resources.md (#212) --- docs/resources.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/resources.md b/docs/resources.md index 40f6d3a6..d05efa2f 100644 --- a/docs/resources.md +++ b/docs/resources.md @@ -9,6 +9,14 @@ This page collects videos and blog posts about nf-test created by the community. --- +### :material-tooltip-text:{ style="color: #337ab7" } Training Module: Hello nf-test + +This training modules introduces nf-test in a simple and easy to follow-along hello-nextflow style aimed for beginners. + +[hello nf-test training module](https://training.nextflow.io/hello_nextflow/05_hello_nf-test/) + +--- + ### :material-tooltip-text:{ style="color: #337ab7" } Blog post: Leveraging nf-test for enhanced quality control in nf-core Reproducibility is an important attribute of all good science. This is specially true in the realm of bioinformatics, where software is hopefully being constantly updated, and pipelines are ideally being maintained. This blog post covers nf-test in the nf-core context. From c2734d9ad8b70a050c17ab3056edda88abe2d3fc Mon Sep 17 00:00:00 2001 From: Jeremy Muhlich Date: Sun, 12 May 2024 05:26:22 -0400 Subject: [PATCH 04/25] Delay launchdir init until test execution (#214) --- .../askimed/nf/test/core/AbstractTest.java | 31 +++++++++++++------ .../nf/test/core/AbstractTestSuite.java | 7 ++++- .../java/com/askimed/nf/test/core/ITest.java | 4 ++- .../com/askimed/nf/test/core/ITestSuite.java | 4 ++- .../nf/test/core/TestExecutionEngine.java | 2 ++ 5 files changed, 35 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/askimed/nf/test/core/AbstractTest.java b/src/main/java/com/askimed/nf/test/core/AbstractTest.java index 236fa6a4..34288e56 100644 --- a/src/main/java/com/askimed/nf/test/core/AbstractTest.java +++ b/src/main/java/com/askimed/nf/test/core/AbstractTest.java @@ -95,17 +95,26 @@ public File getConfig() { return config; } - @Override - public void setup(Config config, File testDirectory) throws IOException { + public void defineDirectories(File testDirectory) throws IOException { if (testDirectory == null) { throw new IOException("Testcase setup failed: No home directory set"); } - launchDir = initDirectory("Launch Directory", testDirectory, DIRECTORY_TESTS, getHash()); - metaDir = initDirectory("Meta Directory", launchDir, DIRECTORY_META); - outputDir = initDirectory("Output Directory", launchDir, DIRECTORY_OUTPUT); - workDir = initDirectory("Working Directory", launchDir, DIRECTORY_WORK); + launchDir = constructDirectory(testDirectory, DIRECTORY_TESTS, getHash()); + metaDir = constructDirectory(launchDir, DIRECTORY_META); + outputDir = constructDirectory(launchDir, DIRECTORY_OUTPUT); + workDir = constructDirectory(launchDir, DIRECTORY_WORK); + + } + + @Override + public void setup(Config config) throws IOException { + + initDirectory("Launch Directory", launchDir); + initDirectory("Meta Directory", metaDir); + initDirectory("Output Directory", outputDir); + initDirectory("Working Directory", workDir); FileStaging[] sharedDirectories = new FileStaging[]{ new FileStaging("bin", config != null ? config.getStageMode() : FileStaging.MODE_COPY), new FileStaging("lib", config != null ? config.getStageMode() : FileStaging.MODE_COPY), @@ -136,15 +145,17 @@ public void execute() throws Throwable { } } - public File initDirectory(String name, File root, String... childs) throws IOException { - + public File constructDirectory(File root, String... childs) { String path = FileUtil.path(root.getAbsolutePath(), FileUtil.path(childs)); - File directory = new File(path).getAbsoluteFile(); + return directory; + } + + public void initDirectory(String name, File directory) throws IOException { + try { FileUtil.deleteDirectory(directory); FileUtil.createDirectory(directory); - return directory; } catch (Exception e) { throw new IOException(name + " '" + directory + "' could not be deleted or created:\n" + e); } diff --git a/src/main/java/com/askimed/nf/test/core/AbstractTestSuite.java b/src/main/java/com/askimed/nf/test/core/AbstractTestSuite.java index e2d17885..21f0f386 100644 --- a/src/main/java/com/askimed/nf/test/core/AbstractTestSuite.java +++ b/src/main/java/com/askimed/nf/test/core/AbstractTestSuite.java @@ -82,7 +82,7 @@ public void evalualteTestClosures() throws Throwable { Closure closure = namedClosure.closure; ITest test = getNewTestInstance(testName); - test.setup(config, getHomeDirectory()); + test.defineDirectories(getHomeDirectory()); closure.setDelegate(test); closure.setResolveStrategy(Closure.DELEGATE_ONLY); closure.call(); @@ -90,6 +90,11 @@ public void evalualteTestClosures() throws Throwable { } } + public void setupTest(ITest test) throws Throwable { + test.setup(config); + } + + protected abstract ITest getNewTestInstance(String name); public void setScript(String script) { diff --git a/src/main/java/com/askimed/nf/test/core/ITest.java b/src/main/java/com/askimed/nf/test/core/ITest.java index 40c9497d..180eb236 100644 --- a/src/main/java/com/askimed/nf/test/core/ITest.java +++ b/src/main/java/com/askimed/nf/test/core/ITest.java @@ -6,7 +6,9 @@ public interface ITest extends ITaggable { - public void setup(Config config, File homeDirectory) throws Throwable; + public void defineDirectories(File testDirectory) throws Throwable; + + public void setup(Config config) throws Throwable; public void execute() throws Throwable; diff --git a/src/main/java/com/askimed/nf/test/core/ITestSuite.java b/src/main/java/com/askimed/nf/test/core/ITestSuite.java index 895bd6c5..726bb45d 100644 --- a/src/main/java/com/askimed/nf/test/core/ITestSuite.java +++ b/src/main/java/com/askimed/nf/test/core/ITestSuite.java @@ -36,4 +36,6 @@ public interface ITestSuite extends ITaggable { public void evalualteTestClosures() throws Throwable; -} \ No newline at end of file + public void setupTest(ITest test) throws Throwable; + +} diff --git a/src/main/java/com/askimed/nf/test/core/TestExecutionEngine.java b/src/main/java/com/askimed/nf/test/core/TestExecutionEngine.java index 9c7dc6f2..6bacbfb2 100644 --- a/src/main/java/com/askimed/nf/test/core/TestExecutionEngine.java +++ b/src/main/java/com/askimed/nf/test/core/TestExecutionEngine.java @@ -199,6 +199,8 @@ public int execute() throws Throwable { log.info("Run test '{}'. type: {}", test, test.getClass().getName()); totalTests++; + testSuite.setupTest(test); + listener.executionStarted(test); TestExecutionResult result = new TestExecutionResult(test); test.setWithTrace(withTrace); From 8172c5da06fa8058ce0abd677c828c1cbd55a7ed Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Tue, 14 May 2024 12:11:38 +0200 Subject: [PATCH 05/25] Create `mock.nf` in project folder instead of meta folder (#198) * Add mock file in project folder to preserve `projectDir` * Add missing testcase * Add missing params file for regex testcase * Small refactoring --- .../askimed/nf/test/core/AbstractTest.java | 73 ++++--------------- .../nf/test/lang/function/FunctionTest.java | 8 +- .../nf/test/lang/process/ProcessTest.java | 8 +- .../nf/test/lang/workflow/WorkflowTest.java | 8 +- .../com/askimed/nf/test/util/FileUtil.java | 23 +++--- .../com/askimed/nf/test/util/HashUtil.java | 24 ++++++ .../askimed/nf/test/lang/WorkflowTest.java | 14 +++- test-data/workflow/regex/params.yaml | 2 + test-data/workflow/regex/workflow.nf | 36 +++++++++ test-data/workflow/regex/workflow.nf.test | 27 +++++++ 10 files changed, 143 insertions(+), 80 deletions(-) create mode 100644 src/main/java/com/askimed/nf/test/util/HashUtil.java create mode 100644 test-data/workflow/regex/params.yaml create mode 100644 test-data/workflow/regex/workflow.nf create mode 100644 test-data/workflow/regex/workflow.nf.test diff --git a/src/main/java/com/askimed/nf/test/core/AbstractTest.java b/src/main/java/com/askimed/nf/test/core/AbstractTest.java index 34288e56..ff7675f1 100644 --- a/src/main/java/com/askimed/nf/test/core/AbstractTest.java +++ b/src/main/java/com/askimed/nf/test/core/AbstractTest.java @@ -2,17 +2,14 @@ import java.io.File; import java.io.IOException; -import java.math.BigInteger; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.util.List; import java.util.Vector; +import com.askimed.nf.test.util.HashUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.askimed.nf.test.config.Config; -import com.askimed.nf.test.config.FileStaging; import com.askimed.nf.test.util.FileUtil; public abstract class AbstractTest implements ITest { @@ -57,6 +54,8 @@ public abstract class AbstractTest implements ITest { public File projectDir = new File(System.getProperty("user.dir")); + public File mockFile; + public boolean skipped = false; protected File config = null; @@ -106,33 +105,16 @@ public void defineDirectories(File testDirectory) throws IOException { outputDir = constructDirectory(launchDir, DIRECTORY_OUTPUT); workDir = constructDirectory(launchDir, DIRECTORY_WORK); + mockFile = new File( ".nf-test-" + getHash() + ".nf"); + mockFile.deleteOnExit(); } @Override public void setup(Config config) throws IOException { - - initDirectory("Launch Directory", launchDir); - initDirectory("Meta Directory", metaDir); - initDirectory("Output Directory", outputDir); - initDirectory("Working Directory", workDir); - FileStaging[] sharedDirectories = new FileStaging[]{ - new FileStaging("bin", config != null ? config.getStageMode() : FileStaging.MODE_COPY), - new FileStaging("lib", config != null ? config.getStageMode() : FileStaging.MODE_COPY), - new FileStaging("assets", config != null ? config.getStageMode() : FileStaging.MODE_COPY) - }; - try { - // copy bin, assets and lib to metaDir - shareDirectories(sharedDirectories, metaDir); - if (config != null) { - // copy user defined staging directories - log.debug("Stage {} user provided files...", config.getStageBuilder().getPaths().size()); - shareDirectories(config.getStageBuilder().getPaths(), metaDir); - } - shareDirectories(parent.getStageBuilder().getPaths(), metaDir); - } catch (Exception e) { - throw new IOException("Testcase setup failed: Directories could not be shared:\n" + e); - } - + setupDirectory("Launch Directory", launchDir); + setupDirectory("Meta Directory", metaDir); + setupDirectory("Output Directory", outputDir); + setupDirectory("Working Directory", workDir); } @Override @@ -145,13 +127,13 @@ public void execute() throws Throwable { } } - public File constructDirectory(File root, String... childs) { + private File constructDirectory(File root, String... childs) { String path = FileUtil.path(root.getAbsolutePath(), FileUtil.path(childs)); File directory = new File(path).getAbsoluteFile(); return directory; } - public void initDirectory(String name, File directory) throws IOException { + private void setupDirectory(String name, File directory) throws IOException { try { FileUtil.deleteDirectory(directory); @@ -164,7 +146,7 @@ public void initDirectory(String name, File directory) throws IOException { @Override public void cleanup() { - // FileUtil.deleteDirectory(metaDir); + } @Override @@ -191,25 +173,10 @@ public String getHash() { throw new RuntimeException("Error generating hash"); } - return hash(parent.getFilename() + getName()); + return HashUtil.getMd5(parent.getFilename() + getName()); } - private String hash(String value) { - - MessageDigest md; - try { - md = MessageDigest.getInstance("MD5"); - md.update(value.getBytes()); - byte[] md5sum = md.digest(); - BigInteger bigInt = new BigInteger(1, md5sum); - return bigInt.toString(16); - } catch (NoSuchAlgorithmException e) { - e.printStackTrace(); - return "??"; - } - } - public void tag(String tag) { tags.add(tag); } @@ -268,20 +235,6 @@ public boolean isWithTrace() { return withTrace; } - protected void shareDirectories(List directories, File stageDir) throws IOException { - for (FileStaging directory : directories) { - String metaDirectory = FileUtil.path(stageDir.getAbsolutePath(), directory.getPath()); - directory.stage(metaDirectory); - } - } - - protected void shareDirectories(FileStaging[] directories, File stageDir) throws IOException { - for (FileStaging directory : directories) { - String metaDirectory = FileUtil.path(stageDir.getAbsolutePath(), directory.getPath()); - directory.stage(metaDirectory); - } - } - @Override public void setUpdateSnapshot(boolean updateSnapshot) { this.updateSnapshot = updateSnapshot; diff --git a/src/main/java/com/askimed/nf/test/lang/function/FunctionTest.java b/src/main/java/com/askimed/nf/test/lang/function/FunctionTest.java index 768ce64b..5a9489d5 100644 --- a/src/main/java/com/askimed/nf/test/lang/function/FunctionTest.java +++ b/src/main/java/com/askimed/nf/test/lang/function/FunctionTest.java @@ -109,8 +109,10 @@ public void execute() throws Throwable { } // Create workflow mock - File workflow = new File(metaDir, FILE_MOCK); - writeWorkflowMock(workflow); + writeWorkflowMock(mockFile); + + // Copy mock file in meta folder for debugging + FileUtil.copy(mockFile, new File(metaDir, FILE_MOCK)); context.getParams().put("nf_test_output", metaDir.getAbsolutePath()); @@ -121,7 +123,7 @@ public void execute() throws Throwable { File paramsFile = new File(metaDir, FILE_PARAMS); NextflowCommand nextflow = new NextflowCommand(); - nextflow.setScript(workflow.getAbsolutePath()); + nextflow.setScript(mockFile.getAbsolutePath()); nextflow.setParams(context.getParams()); for (String profile: parent.getProfiles()) { nextflow.addProfile(profile); diff --git a/src/main/java/com/askimed/nf/test/lang/process/ProcessTest.java b/src/main/java/com/askimed/nf/test/lang/process/ProcessTest.java index 42d3fdde..20d4a72b 100644 --- a/src/main/java/com/askimed/nf/test/lang/process/ProcessTest.java +++ b/src/main/java/com/askimed/nf/test/lang/process/ProcessTest.java @@ -108,8 +108,10 @@ public void execute() throws Throwable { context.evaluateProcessClosure(); // Create workflow mock - File workflow = new File(metaDir, FILE_MOCK); - writeWorkflowMock(workflow); + writeWorkflowMock(mockFile); + + // Copy mock file in meta folder for debugging + FileUtil.copy(mockFile, new File(metaDir, FILE_MOCK)); context.getParams().put("nf_test_output", metaDir.getAbsolutePath()); @@ -124,7 +126,7 @@ public void execute() throws Throwable { File paramsFile = new File(metaDir, FILE_PARAMS); NextflowCommand nextflow = new NextflowCommand(); - nextflow.setScript(workflow.getAbsolutePath()); + nextflow.setScript(mockFile.getAbsolutePath()); nextflow.setParams(context.getParams()); for (String profile: parent.getProfiles()) { nextflow.addProfile(profile); diff --git a/src/main/java/com/askimed/nf/test/lang/workflow/WorkflowTest.java b/src/main/java/com/askimed/nf/test/lang/workflow/WorkflowTest.java index c595990d..16aedef9 100644 --- a/src/main/java/com/askimed/nf/test/lang/workflow/WorkflowTest.java +++ b/src/main/java/com/askimed/nf/test/lang/workflow/WorkflowTest.java @@ -121,8 +121,10 @@ public void execute() throws Throwable { context.evaluateWorkflowClosure(); // Create workflow mock - File workflow = new File(metaDir, FILE_MOCK); - writeWorkflowMock(workflow); + writeWorkflowMock(mockFile); + + // Copy mock file in meta folder for debugging + FileUtil.copy(mockFile, new File(metaDir, FILE_MOCK)); context.getParams().put("nf_test_output", metaDir.getAbsolutePath()); if (isDebug()) { @@ -136,7 +138,7 @@ public void execute() throws Throwable { File paramsFile = new File(metaDir, FILE_PARAMS); NextflowCommand nextflow = new NextflowCommand(); - nextflow.setScript(workflow.getAbsolutePath()); + nextflow.setScript(mockFile.getAbsolutePath()); nextflow.setParams(context.getParams()); for (String profile: parent.getProfiles()) { nextflow.addProfile(profile); diff --git a/src/main/java/com/askimed/nf/test/util/FileUtil.java b/src/main/java/com/askimed/nf/test/util/FileUtil.java index ea98c705..20d27b8f 100644 --- a/src/main/java/com/askimed/nf/test/util/FileUtil.java +++ b/src/main/java/com/askimed/nf/test/util/FileUtil.java @@ -1,14 +1,6 @@ package com.askimed.nf.test.util; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStream; -import java.io.PushbackInputStream; +import java.io.*; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -189,4 +181,17 @@ public static String encodeBase64(byte[] bytes) { return encodedContent; } + public static void copy(File source, File dest) throws IOException { + try (InputStream is = new FileInputStream(source); + OutputStream os = new FileOutputStream(dest)) { + + byte[] buffer = new byte[1024]; + int length; + + // Read from the input stream and write to the output stream + while ((length = is.read(buffer)) > 0) { + os.write(buffer, 0, length); + } + } + } } diff --git a/src/main/java/com/askimed/nf/test/util/HashUtil.java b/src/main/java/com/askimed/nf/test/util/HashUtil.java new file mode 100644 index 00000000..8f94773c --- /dev/null +++ b/src/main/java/com/askimed/nf/test/util/HashUtil.java @@ -0,0 +1,24 @@ +package com.askimed.nf.test.util; + +import java.math.BigInteger; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +public class HashUtil { + + public static String getMd5(String value) { + + MessageDigest md; + try { + md = MessageDigest.getInstance("MD5"); + md.update(value.getBytes()); + byte[] md5sum = md.digest(); + BigInteger bigInt = new BigInteger(1, md5sum); + return bigInt.toString(16); + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + return "??"; + } + } + +} diff --git a/src/test/java/com/askimed/nf/test/lang/WorkflowTest.java b/src/test/java/com/askimed/nf/test/lang/WorkflowTest.java index bd009076..87e147c6 100644 --- a/src/test/java/com/askimed/nf/test/lang/WorkflowTest.java +++ b/src/test/java/com/askimed/nf/test/lang/WorkflowTest.java @@ -139,8 +139,9 @@ public void testIssue125() throws Exception { public void testStagingWithoutMapping() throws Exception { App app = new App(); - int exitCode = app.run(new String[] { "test", "test-data/workflow/staging/hello.nf.test" }); - assertEquals(1, exitCode); + // TODO: remove this test. no staging needed. + //int exitCode = app.run(new String[] { "test", "test-data/workflow/staging/hello.nf.test" }); + //assertEquals(1, exitCode); } @@ -183,4 +184,13 @@ public void testStagingInTestsuite() throws Exception { } + @Test + public void testRegex() throws Exception { + + App app = new App(); + int exitCode = app.run(new String[] { "test", "test-data/workflow/regex/workflow.nf.test" }); + assertEquals(0, exitCode); + + } + } diff --git a/test-data/workflow/regex/params.yaml b/test-data/workflow/regex/params.yaml new file mode 100644 index 00000000..455390bc --- /dev/null +++ b/test-data/workflow/regex/params.yaml @@ -0,0 +1,2 @@ +localOutputDir: "example_output" +someRegex: '^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}\$' \ No newline at end of file diff --git a/test-data/workflow/regex/workflow.nf b/test-data/workflow/regex/workflow.nf new file mode 100644 index 00000000..d8d5fbb3 --- /dev/null +++ b/test-data/workflow/regex/workflow.nf @@ -0,0 +1,36 @@ +process exampleProc { + + storeDir "${outputDir}/exampleProc" + input: + tuple( + val(outputDir), + val(localOutputDir), + val(someRegexString) + ) + + output: + tuple( + val(outputDir), + path("example_output/example*.txt") + ) + + shell: + """ + mkdir -p !{localOutputDir} + echo !{someRegexString} > "!{localOutputDir}/example1.txt" + """ +} + + +workflow PipeWf { + take: + inputCh + + main: + inputCh + | exampleProc + | set { outputCh } + + emit: + outputCh +} \ No newline at end of file diff --git a/test-data/workflow/regex/workflow.nf.test b/test-data/workflow/regex/workflow.nf.test new file mode 100644 index 00000000..41a5214f --- /dev/null +++ b/test-data/workflow/regex/workflow.nf.test @@ -0,0 +1,27 @@ +nextflow_workflow { + + name "Test workflow" + script "./workflow.nf" + workflow "PipeWf" + + test("Output will exist in default outputDir") { + + when { + + params { + outDir = "$outputDir" + load("test-data/workflow/regex/params.yaml") + } + workflow { + """ + input[0] = Channel.of([params.outDir,params.localOutputDir]) + """ + } + } + + then { + assert workflow.success + assert path("${outputDir}/exampleProc/example_output").exists() + } + } +} \ No newline at end of file From d81908c069f547f7e48007b94f0638ee72d62bb2 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Tue, 14 May 2024 12:22:29 +0200 Subject: [PATCH 06/25] Update documentation --- docs/docs/configuration.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/docs/configuration.md b/docs/docs/configuration.md index e5aadb8e..3b34d8cd 100644 --- a/docs/docs/configuration.md +++ b/docs/docs/configuration.md @@ -102,6 +102,10 @@ By understanding this profile evaluation order, you can effectively configure Ne ## File Staging +!!! warning + + File Staging is obsolete since version >= 0.9.0. + The `stage` section of the `nf-test.config` file is used to define files that are needed by Nextflow in the test environment (`meta` directory). Additionally, the directories `lib`, `bin`, and `assets` are automatically staged. ### Supported Directives From 38cc083f2bf2da5f2f4f1edd06ae6c3b7856214f Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Tue, 14 May 2024 13:07:06 +0200 Subject: [PATCH 07/25] Add `requires` to `nf-test.config` in order to specify minimal nf-test version (#191) * Add `requires` to config to specify minimal nf-test version * Add documentation of `requires` keyword * Fix issue and add testcases * Update syntax in documentation for `requires` --- docs/docs/configuration.md | 15 ++++++- .../com/askimed/nf/test/config/Config.java | 31 ++++++++++++++ .../com/askimed/nf/test/util/Version.java | 41 +++++++++++++++++++ .../com/askimed/nf/test/lang/ProcessTest.java | 17 ++++++++ .../process/requires/nf-test-0.1.0.config | 6 +++ .../process/requires/nf-test-100.0.0.config | 6 +++ 6 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/askimed/nf/test/util/Version.java create mode 100644 test-data/process/requires/nf-test-0.1.0.config create mode 100644 test-data/process/requires/nf-test-100.0.0.config diff --git a/docs/docs/configuration.md b/docs/docs/configuration.md index 3b34d8cd..ad86c0c9 100644 --- a/docs/docs/configuration.md +++ b/docs/docs/configuration.md @@ -5,7 +5,7 @@ The `nf-test.config` file is a configuration file used to customize settings and behavior for `nf-test`. This file must be located in the root of your project, and it is automatically loaded when you run `nf-test test`. Below are the parameters that can be adapted: | Parameter | Description | Default Value | -| ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------- | +|--------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------| ------------------------- | | `testsDir` | Location for storing all nf-test cases (test scripts). If you want all test files to be in the same directory as the script itself, you can set the testDir to `.` | `"tests"` | | `workDir` | Directory for storing temporary files and working directories for each test. This directory should be added to `.gitignore`. | `".nf-test"` | | `configFile` | Location of an optional `nextflow.config` file specifically used for executing tests. [Learn more](#testsnextflowconfig). | `"tests/nextflow.config"` | @@ -14,6 +14,8 @@ The `nf-test.config` file is a configuration file used to customize settings and | `withTrace` | Enable or disable tracing options during testing. Disable tracing if your containers don't include the `procps` tool. | `true` | | `autoSort` | Enable or disable sorted channels by default when running tests. | `true` | | `options` | Custom Nextflow command-line options to be applied when running tests. For example `"-dump-channels -stub-run"` | | +| `requires` | Can be used to specify the minimum required version of nf-test. Requires nf-test > 0.9.0 | | + Here's an example of what an `nf-test.config` file could look like: @@ -30,6 +32,17 @@ config { } ``` +The `requires` keyword can be used to specify the minimum required version of nf-test. +For instance, to ensure the use of at least nf-test version 0.9.0, define it as follows: + +```groovy +config { + requires ( + "nf-test": "0.9.0" + ) +} +``` + ## `tests/nextflow.config` This optional `nextflow.config` file is used to execute tests. This is a good place to set default `params` for all your tests. Example number of threads: diff --git a/src/main/java/com/askimed/nf/test/config/Config.java b/src/main/java/com/askimed/nf/test/config/Config.java index a0f1e0ae..cd5e817a 100644 --- a/src/main/java/com/askimed/nf/test/config/Config.java +++ b/src/main/java/com/askimed/nf/test/config/Config.java @@ -1,7 +1,11 @@ package com.askimed.nf.test.config; import java.io.File; +import java.util.Map; +import com.askimed.nf.test.App; +import com.askimed.nf.test.nextflow.NextflowCommand; +import com.askimed.nf.test.util.Version; import org.codehaus.groovy.control.CompilerConfiguration; import org.codehaus.groovy.control.customizers.ImportCustomizer; @@ -18,6 +22,8 @@ public class Config { public static final String DEFAULT_HOME = ".nf-test"; + public static final String KEY_NF_TEST_VERSION = "nf-test"; + private String workDir = ".nf-test"; private String testsDir = "tests"; @@ -40,6 +46,8 @@ public class Config { private String stageMode = FileStaging.MODE_COPY; + private Map requires = null; + public void testsDir(String testsDir) { this.testsDir = testsDir; } @@ -56,6 +64,18 @@ public String getWorkDir() { return workDir; } + public void requires(Map requires) { + setRequires(requires); + } + + public void setRequires(Map requires) { + this.requires = requires; + } + + public Map getRequires() { + return requires; + } + public void profile(String profile) { this.profile = profile; } @@ -192,6 +212,17 @@ public static Config parse(File script) throws Exception { Object object = shell.evaluate(script); Config config = (Config) object; + // no requirements + if (config.getRequires() == null ){ + return config; + } + + // check nf-test version + String appVersion = config.getRequires().getOrDefault(KEY_NF_TEST_VERSION, App.VERSION).toString(); + if (Version.compare(appVersion, App.VERSION) >= 1) { + throw new Exception("nf-test " + appVersion + " or above is required to run this project"); + } + return config; } diff --git a/src/main/java/com/askimed/nf/test/util/Version.java b/src/main/java/com/askimed/nf/test/util/Version.java new file mode 100644 index 00000000..e63b70f2 --- /dev/null +++ b/src/main/java/com/askimed/nf/test/util/Version.java @@ -0,0 +1,41 @@ +package com.askimed.nf.test.util; + +public class Version { + + public static int compare(String version1, String version2) { + + String parts1[] = version1.split("-", 2); + String parts2[] = version2.split("-", 2); + + String tiles1[] = parts1[0].split("\\."); + String tiles2[] = parts2[0].split("\\."); + + for (int i = 0; i < tiles1.length; i++) { + int number1 = Integer.parseInt(tiles1[i].trim()); + int number2 = Integer.parseInt(tiles2[i].trim()); + + if (number1 != number2) { + + return number1 > number2 ? 1 : -1; + + } + + } + + if (parts1.length > 1) { + if (parts2.length > 1) { + return parts1[1].compareTo(parts2[1]); + } else { + return -1; + } + } else { + if (parts2.length > 1) { + return 1; + } + } + + return 0; + + } + +} diff --git a/src/test/java/com/askimed/nf/test/lang/ProcessTest.java b/src/test/java/com/askimed/nf/test/lang/ProcessTest.java index a52ae989..91b2783b 100644 --- a/src/test/java/com/askimed/nf/test/lang/ProcessTest.java +++ b/src/test/java/com/askimed/nf/test/lang/ProcessTest.java @@ -290,6 +290,23 @@ public void testProfilesOverwriteInConfig() throws Exception { } + @Test + public void testRequires() throws Exception { + + App app = new App(); + int exitCode = app.run(new String[] { "test", "test-data/process/profiles/hello.a.nf.test", "--config", + "test-data/process/requires/nf-test-0.1.0.config" }); + assertEquals(0, exitCode); + + app = new App(); + exitCode = app.run(new String[] { "test", "test-data/process/profiles/hello.a.nf.test", "--config", + "test-data/process/requires/nf-test-100.0.0.config" }); + assertEquals(2, exitCode); + + + } + + @Test public void testUniquenessSnapshots() throws Exception { App app = new App(); diff --git a/test-data/process/requires/nf-test-0.1.0.config b/test-data/process/requires/nf-test-0.1.0.config new file mode 100644 index 00000000..a272c01f --- /dev/null +++ b/test-data/process/requires/nf-test-0.1.0.config @@ -0,0 +1,6 @@ +config { + requires ( + "nf-test": "0.1.0" + ) + configFile "" +} \ No newline at end of file diff --git a/test-data/process/requires/nf-test-100.0.0.config b/test-data/process/requires/nf-test-100.0.0.config new file mode 100644 index 00000000..d5255ef8 --- /dev/null +++ b/test-data/process/requires/nf-test-100.0.0.config @@ -0,0 +1,6 @@ +config { + requires ( + "nf-test": "100.0.0" + ) + configFile "" +} \ No newline at end of file From 028fdb40cc04aefa0a05a275c43356fb16b510ed Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Thu, 23 May 2024 12:42:39 +0200 Subject: [PATCH 08/25] Add test-execution optimization and sharding options (#216) * Add graph and resolver to support dependencies * Add coverage flag and improve dry mode * Add ignore patterns and restructure code * Use dependency resolver to find tests by path * Fix coverage when all tests are executed * Refactore test resolver strategy * Implement git support, `--only-changed` and `--changed-since` * Fix bug in resolving includes in Nextflow script * Update website in header * Fix issue with tests path in config file * Add optimized mode for firewall calculation * Add `--changed-until` flag * Fix issue in file pattern * Implement sharding with round-robin * Add snapshots to dependency graph and improve dot renderer * Fix issue with test-hash after filename * Add all new parameters to documentation * Add `ignore` to documentation * Add triggers * Add testcases for dependency resolver * Add testcases for dependency resolver * Add `--ci` flag (#217) * Add `--ci` flag * Fix wrong commited files * Fix wrong warning message * Fix CI mode and add testcase * Add tutorial to setup GitHub actions * Update tutorial * Add github token to env --- .github/workflows/create-release.yml | 2 + .github/workflows/run-tests.yml | 2 + docs/docs/cli/test.md | 92 +++++- docs/docs/configuration.md | 5 +- docs/tutorials.md | 11 + docs/tutorials/github-actions.md | 177 ++++++++++++ mkdocs.yml | 1 + .../nf/test/commands/AbstractCommand.java | 2 +- .../nf/test/commands/ListTestsCommand.java | 162 ++++++++++- .../nf/test/commands/RunTestsCommand.java | 197 +++++++++++-- .../com/askimed/nf/test/config/Config.java | 50 +++- .../askimed/nf/test/core/AbstractTest.java | 12 + .../com/askimed/nf/test/core/Environment.java | 26 ++ .../java/com/askimed/nf/test/core/ITest.java | 4 + .../nf/test/core/TestExecutionEngine.java | 245 ++-------------- .../nf/test/core/TestSuiteResolver.java | 66 +++++ .../nf/test/core/TestSuiteSharder.java | 117 ++++++++ .../askimed/nf/test/core/TestSuiteSorter.java | 25 ++ .../nf/test/core/reports/CsvReportWriter.java | 65 +++++ .../nf/test/lang/TestSuiteBuilder.java | 17 +- .../nf/test/lang/dependencies/Coverage.java | 128 +++++++++ .../lang/dependencies/DependencyExporter.java | 70 +++++ .../lang/dependencies/DependencyGraph.java | 116 ++++++++ .../lang/dependencies/DependencyResolver.java | 271 ++++++++++++++++++ .../nf/test/lang/dependencies/IMetaFile.java | 20 ++ .../test/lang/dependencies/SnapshotFile.java | 48 ++++ .../nf/test/lang/dependencies/TestFile.java | 76 +++++ .../lang/dependencies/TestFilePattern.java | 65 +++++ .../nf/test/lang/extensions/Snapshot.java | 3 + .../nf/test/lang/extensions/SnapshotFile.java | 4 +- .../nf/test/nextflow/NextflowCommand.java | 2 +- .../nf/test/nextflow/NextflowScript.java | 79 ++++- .../com/askimed/nf/test/util/AnsiText.java | 8 + .../com/askimed/nf/test/util/Command.java | 2 +- .../com/askimed/nf/test/util/GitCommand.java | 132 +++++++++ .../nf/test/core/TestExecutionEngineTest.java | 123 -------- .../nf/test/core/TestSuiteResolverTest.java | 151 ++++++++++ .../com/askimed/nf/test/lang/ProcessTest.java | 35 ++- .../dependencies/DependencyResolverTest.java | 192 +++++++++++++ .../nf/test/nextflow/NextflowCommandTest.java | 5 + test-data/process/snapshots/ci-mode.nf.test | 28 ++ .../{no-unique.test => no-unique.nf.test} | 0 42 files changed, 2422 insertions(+), 414 deletions(-) create mode 100644 docs/tutorials.md create mode 100644 docs/tutorials/github-actions.md create mode 100644 src/main/java/com/askimed/nf/test/core/Environment.java create mode 100644 src/main/java/com/askimed/nf/test/core/TestSuiteResolver.java create mode 100644 src/main/java/com/askimed/nf/test/core/TestSuiteSharder.java create mode 100644 src/main/java/com/askimed/nf/test/core/TestSuiteSorter.java create mode 100644 src/main/java/com/askimed/nf/test/core/reports/CsvReportWriter.java create mode 100644 src/main/java/com/askimed/nf/test/lang/dependencies/Coverage.java create mode 100644 src/main/java/com/askimed/nf/test/lang/dependencies/DependencyExporter.java create mode 100644 src/main/java/com/askimed/nf/test/lang/dependencies/DependencyGraph.java create mode 100644 src/main/java/com/askimed/nf/test/lang/dependencies/DependencyResolver.java create mode 100644 src/main/java/com/askimed/nf/test/lang/dependencies/IMetaFile.java create mode 100644 src/main/java/com/askimed/nf/test/lang/dependencies/SnapshotFile.java create mode 100644 src/main/java/com/askimed/nf/test/lang/dependencies/TestFile.java create mode 100644 src/main/java/com/askimed/nf/test/lang/dependencies/TestFilePattern.java create mode 100644 src/main/java/com/askimed/nf/test/util/GitCommand.java delete mode 100644 src/test/java/com/askimed/nf/test/core/TestExecutionEngineTest.java create mode 100644 src/test/java/com/askimed/nf/test/core/TestSuiteResolverTest.java create mode 100644 src/test/java/com/askimed/nf/test/lang/dependencies/DependencyResolverTest.java create mode 100644 test-data/process/snapshots/ci-mode.nf.test rename test-data/process/snapshots/{no-unique.test => no-unique.nf.test} (100%) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 4a3eb245..9cacd5d2 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -25,6 +25,8 @@ jobs: - name: Build run: mvn install + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - uses: ncipollo/release-action@v1 with: diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index f93683d4..d788110b 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -25,3 +25,5 @@ jobs: - name: Test run: mvn install + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/docs/docs/cli/test.md b/docs/docs/cli/test.md index 1ae5fdb5..195c72fa 100644 --- a/docs/docs/cli/test.md +++ b/docs/docs/cli/test.md @@ -12,6 +12,10 @@ nf-test test [|] To run your test using a specific Nextflow profile, you can use the `--profile` argument. [Learn more](/docs/docs/configuration#managing-profiles). +#### `--dry-run` + +This flag allows users to simulate the execution of tests. + #### `--verbose` Prints out the Nextflow output during test runs. @@ -24,6 +28,12 @@ The Linux tool `procps` is required to run Nextflow tracing. In case your contai Execute only tests with the provided tag. Multiple tags can be used and have to be separated by commas (e.g. `tag1,tag2`). +#### `--debug` + +The debug parameter prints out debugging messages and all available output channels which can be accessed in the `then` clause. + +### Output Reports + #### `--tap ` Writes test results in [TAP format](https://testanything.org) to file. @@ -32,9 +42,61 @@ Writes test results in [TAP format](https://testanything.org) to file. Writes test results in [JUnit XML format](https://junit.org/) to file, which conforms to [the standard schema](https://github.com/junit-team/junit5/blob/242f3b3ef84cfd96c9de26992588812a68cdef8b/platform-tests/src/test/resources/jenkins-junit.xsd). -#### `--debug` +#### `--csv ` -The debug parameter prints out debugging messages and all available output channels which can be accessed in the `then` clause. +Writes test results in csv file. + +#### `--ci` + +By default,nf-test automatically stores a new snapshot. When CI mode is activated, nf-test will fail the test instead of storing the snapshot automatically. + + +### Optimizing Test Execution + +#### `--related-tests ` + +Finds and executes all related tests for the provided .nf or nf.test files. Multiple files can be provided space separated. + +#### `--follow-dependencies` + +When this flag is set, nf-test will traverse all dependencies when the related-tests flag is set. +This option is particularly useful when you need to ensure that +all dependent tests are executed, bypassing the firewall calculation process. + +#### `--only-changed` + +When enabled, this parameter instructs nf-test to execute tests only for files that have been modified within the +current git working tree. + +#### `--changed-since ` + +This parameter triggers the execution of tests related to changes made since the specifie commit. +e.g. `--changed-since HEAD^` for all changes between the HEAD and HEAD - 1. + +#### `--changed-until ` + +This parameter initiates the execution of tests related to changes made until the specified commit hash. + +#### `--graph ` + +Enables the export of the dependency graph as a dot file. +The dot file format is commonly used for representing graphs in graphviz and other related software. + +### Sharding + +This parameter allows users to divide the execution workload into manageable chunks, which can be useful for +parallel or distributed processing. + +#### `--shard ` +Splits the execution into arbitrary chunks defined by the format `i/n`, where `i` denotes the index of the current +chunk and `n` represents the total number of chunks. For instance, `2/5` executes the second chunk out of five. + +#### `--shard-strategy ` +Description: Specifies the strategy used to build shards when the `--shard` parameter is utilized. +Accepted values are `round-robin` or `none.`. This parameter determines the method employed to distribute workload +chunks among available resources. With the round-robin strategy, shards are distributed evenly among resources in +a cyclic manner. The none strategy implies that shards won't be distributed automatically, and it's up to the +user to manage the assignment of shards. Default value is `round-robin`. ## Examples @@ -63,3 +125,29 @@ The debug parameter prints out debugging messages and all available output chann ``` nf-test test --tap report.tap ``` + +- Run all tests (and possible integration tests) for module `modules/module_a.nf` and `modules/module_b.nf`; + + ``` + nf-test test --related-tests modules/module_a.nf modules/module_b.nf + ``` + +- If your project is a Git directory and you have modified files, you can run tests only for these changed files by +using the following command: + + ``` + nf-test test --only-changed + ``` + +- If you want to test all changes made between the current state of the repository and the last commit, +you can use the following command: + + ``` + nf-test test --changed-since HEAD^ + ``` + +- Run only the second of four shards: + + ``` + nf-test test --shard 2/4 + ``` \ No newline at end of file diff --git a/docs/docs/configuration.md b/docs/docs/configuration.md index ad86c0c9..d815d50a 100644 --- a/docs/docs/configuration.md +++ b/docs/docs/configuration.md @@ -14,8 +14,9 @@ The `nf-test.config` file is a configuration file used to customize settings and | `withTrace` | Enable or disable tracing options during testing. Disable tracing if your containers don't include the `procps` tool. | `true` | | `autoSort` | Enable or disable sorted channels by default when running tests. | `true` | | `options` | Custom Nextflow command-line options to be applied when running tests. For example `"-dump-channels -stub-run"` | | -| `requires` | Can be used to specify the minimum required version of nf-test. Requires nf-test > 0.9.0 | | - +| `ignore` | List of filenames or patterns that should be ignored when building the dependency graph. For example: `ignore 'folder/**/*.nf', 'modules/module.nf'` | `` | +| `triggers` | List of filenames or patterns that should be trigger a full test run. For example: `triggers 'nextflow.config', 'test-data/**/*'` | `` | +| `requires` | Can be used to specify the minimum required version of nf-test. Requires nf-test > 0.9.0 | `` | Here's an example of what an `nf-test.config` file could look like: diff --git a/docs/tutorials.md b/docs/tutorials.md new file mode 100644 index 00000000..63e99d97 --- /dev/null +++ b/docs/tutorials.md @@ -0,0 +1,11 @@ +# Tutorials + +--- + +### [:material-file:{ style="color: #337ab7" } Setup nf-test on GitHub Actions](tutorials/github-actions.md) + +In this tutorial, we will guide you through setting up and running `nf-test` on GitHub Actions. + +[Read tutorial](tutorials/github-actions.md) + +--- \ No newline at end of file diff --git a/docs/tutorials/github-actions.md b/docs/tutorials/github-actions.md new file mode 100644 index 00000000..88151ed2 --- /dev/null +++ b/docs/tutorials/github-actions.md @@ -0,0 +1,177 @@ +# Setup nf-test on GitHub Actions + +In this tutorial, we will guide you through setting up and running `nf-test` on GitHub Actions. We will start with a simple example where all tests run in a single job, then extend it to demonstrate how you can use sharding to distribute tests across multiple jobs for improved efficiency. Finally, we will show you how to run only the tests affected by the changed files using the `--changes-since` option. + +By the end of this tutorial, you will have a clear understanding of how to: + +1. Set up a basic CI workflow for running `nf-test` on GitHub Actions. +2. Extend the workflow to use sharding, allowing tests to run in parallel, which can significantly reduce the overall execution time. +3. Configure the workflow to run only the test cases affected by the changed files, optimizing the CI process further. + +Whether you are maintaining a complex bioinformatics pipeline or a simple data analysis workflow, integrating `nf-test` with GitHub Actions will help ensure the robustness and reliability of your code. Let's get started! + +## Step 1: Running nf-test + +Create a file named `.github/workflows/ci-tests.yml` in your repository with the following content: + +```yaml +name: CI Tests + +on: [push, pull_request] + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up JDK 11 + uses: actions/setup-java@v2 + with: + java-version: '11' + distribution: 'adopt' + + - name: Setup Nextflow latest-edge + uses: nf-core/setup-nextflow@v1 + with: + version: "latest-edge" + + - name: Install nf-test + run: | + wget -qO- https://code.askimed.com/install/nf-test | bash + sudo mv nf-test /usr/local/bin/ + + - name: Run Tests + run: nf-test test +``` + +### Explanation: + +1. **Checkout**: Uses the `actions/checkout@v2` action to check out the repository. +2. **Set up JDK 11**: Uses the `actions/setup-java@v2` action to set up Java Development Kit version 11. +3. **Setup Nextflow**: Uses the `nf-core/setup-nextflow@v1` action to install the latest-edge version of Nextflow. +4. **Install nf-test**: Downloads and installs nf-test. +5. **Run Tests**: Runs nf-test without sharding. + +## Step 2: Extending to Use Sharding + +To distribute the tests across multiple jobs, you can set up sharding. Update your workflow file as follows: + +```yaml +name: CI Tests + +on: [push, pull_request] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + shard: [1, 2, 3, 4] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up JDK 11 + uses: actions/setup-java@v2 + with: + java-version: '11' + distribution: 'adopt' + + - name: Setup Nextflow latest-edge + uses: nf-core/setup-nextflow@v1 + with: + version: "latest-edge" + + - name: Install nf-test + run: | + wget -qO- https://code.askimed.com/install/nf-test | bash + sudo mv nf-test /usr/local/bin/ + + - name: Run Tests (Shard ${{ matrix.shard }}/${{ strategy.job-total }}) + run: nf-test test --shard ${{ matrix.shard }}/${{ strategy.job-total }} +``` + +### Explanation of Sharding: + +1. **Matrix Strategy**: The `strategy` section defines a matrix with a `shard` parameter that has four values: `[1, 2, 3, 4]`. This will create four parallel jobs, one for each shard. +2. **Run Tests with Sharding**: The `run` command for running tests is updated to `nf-test test --shard ${{ matrix.shard }}/${{ strategy.job-total }}`. This command will run the tests for the specific shard. `${{ matrix.shard }}` represents the current shard number, and `${{ strategy.job-total }}` represents the total number of shards. + +## Step 3: Running Only Tests Affected by Changed Files + +To optimize the workflow further, you can run only the tests that are affected by the changed files since the last commit. Update your workflow file as follows: + +```yaml +name: CI Tests + +on: [push, pull_request] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + shard: [1, 2, 3, 4] + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up JDK 11 + uses: actions/setup-java@v2 + with: + java-version: '11' + distribution: 'adopt' + + - name: Setup Nextflow latest-edge + uses: nf-core/setup-nextflow@v1 + with: + version: "latest-edge" + + - name: Install nf-test + run: | + wget -qO- https://code.askimed.com/install/nf-test | bash + sudo mv nf-test /usr/local/bin/ + + - name: Run Tests (Shard ${{ matrix.shard }}/${{ strategy.job-total }}) + run: nf-test test --shard ${{ matrix.shard }}/${{ strategy.job-total }} --changed-since HEAD^ +``` + +### Explanation of Changes: + +1. **Checkout with Full History**: The `actions/checkout@v2` action is updated with `fetch-depth: 0` to fetch the full history of the repository. This is necessary for accurately determining the changes since the last commit. +2. **Run Tests with Changed Files**: The `run` command is further updated to include the `--changed-since HEAD^` option. This option ensures that only the tests affected by the changes since the previous commit are run. + + +## Step 4: Adapting nf-test.config to Trigger Full Test Runs + +In some cases, changes to specific critical files should trigger a full test run, regardless of other changes. To configure this, you need to adapt your `nf-test.config` file. + +Add the following lines to your `nf-test.config`: + +```groovy +config { + ... + triggers 'nextflow.config', 'nf-test.config', 'test-data/**/*' + ... +} +``` + + The `triggers` directive in `nf-test.config` specifies a list of filenames or patterns that should trigger a full test run. For example: + + - `'nextflow.config'`: Changes to the main Nextflow configuration file will trigger a full test run. + - `'nf-test.config'`: Changes to the nf-test configuration file itself will trigger a full test run. + - `'test-data/**/*'`: Changes to any files within the `test-data` directory will trigger a full test run. + +This configuration ensures that critical changes always result in a comprehensive validation of the pipeline, providing additional confidence in your CI process. + +## Summary + +1. **Without Sharding**: A straightforward setup where all tests run in a single job. +2. **With Sharding**: Distributes tests across multiple jobs, allowing them to run in parallel. +3. **With Sharding and Changed Files**: Optimizes the CI process by running only the tests affected by the changed files since the last commit, in parallel jobs. + +Choose the configuration that best suits your project's needs. Start with the simpler setup and extend it as needed to improve efficiency and reduce test execution time. \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 7ffa6794..97e0a802 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -65,6 +65,7 @@ nav: - Available Plugins: https://code.askimed.com/nf-test-plugins - Using Plugins: docs/plugins/using-plugins.md - Developing Plugins: docs/plugins/developing-plugins.md + - Tutorials: tutorials.md - Resources: resources.md - About: about.md diff --git a/src/main/java/com/askimed/nf/test/commands/AbstractCommand.java b/src/main/java/com/askimed/nf/test/commands/AbstractCommand.java index fc2852f0..13e1c0dd 100644 --- a/src/main/java/com/askimed/nf/test/commands/AbstractCommand.java +++ b/src/main/java/com/askimed/nf/test/commands/AbstractCommand.java @@ -49,7 +49,7 @@ private void printHeader() { System.out.println(); System.out.println(Emoji.ROCKET + AnsiText.bold(" " + App.NAME + " " + App.VERSION)); - System.out.println("https://code.askimed.com/nf-test"); + System.out.println("https://www.nf-test.com"); System.out.println("(c) 2021 - 2024 Lukas Forer and Sebastian Schoenherr"); System.out.println(); diff --git a/src/main/java/com/askimed/nf/test/commands/ListTestsCommand.java b/src/main/java/com/askimed/nf/test/commands/ListTestsCommand.java index 16cd44cc..b5138ba6 100644 --- a/src/main/java/com/askimed/nf/test/commands/ListTestsCommand.java +++ b/src/main/java/com/askimed/nf/test/commands/ListTestsCommand.java @@ -1,14 +1,17 @@ package com.askimed.nf.test.commands; import java.io.File; -import java.util.ArrayList; -import java.util.List; +import java.util.*; import com.askimed.nf.test.config.Config; -import com.askimed.nf.test.core.TestExecutionEngine; +import com.askimed.nf.test.core.*; +import com.askimed.nf.test.lang.dependencies.DependencyResolver; import com.askimed.nf.test.util.AnsiColors; +import com.askimed.nf.test.util.AnsiText; +import com.askimed.nf.test.util.FileUtil; import com.askimed.nf.test.util.OutputFormat; +import groovy.json.JsonOutput; import picocli.CommandLine.Command; import picocli.CommandLine.Help.Visibility; import picocli.CommandLine.Option; @@ -37,15 +40,17 @@ public Integer execute() throws Exception { try { + Config config = null; + try { File configFile = new File(Config.FILENAME); if (configFile.exists()) { - Config config = Config.parse(configFile); + config = Config.parse(configFile); - if (testPaths.size() == 0) { + if (testPaths.isEmpty()) { File folder = new File(config.getTestsDir()); testPaths.add(folder); } @@ -64,19 +69,35 @@ public Integer execute() throws Exception { } - List scripts = RunTestsCommand.pathsToScripts(testPaths); + DependencyResolver resolver = new DependencyResolver(new File(new File("").getAbsolutePath())); + if (config != null) { + resolver.buildGraph(config.getIgnore(), config.getTriggers()); + } else { + resolver.buildGraph(); + } + + List scripts = resolver.findTestsByFiles(testPaths); - if (scripts.size() == 0) { + if (scripts.isEmpty()) { System.out.println(AnsiColors.red("Error: No tests provided and no test directory set.")); return 2; } - TestExecutionEngine engine = new TestExecutionEngine(); - engine.setScripts(scripts); + Environment environment = new Environment(); + + TestSuiteResolver testSuiteResolver = new TestSuiteResolver(environment); + List testSuits = testSuiteResolver.parse(scripts); + + if (testSuits.isEmpty()) { + System.out.println(AnsiColors.red("Error: no valid tests found.")); + System.out.println(); + return 1; + } + if (tags) { - return engine.listTags(format); + return listTags(testSuits, format); } else { - return engine.listTests(format); + return listTests(testSuits, format); } } catch (Throwable e) { @@ -89,4 +110,123 @@ public Integer execute() throws Exception { } + + public int listTests(List testSuits, OutputFormat format) throws Throwable { + + switch (format) { + case JSON: + case json: + printTestsAsJson(testSuits); + break; + case RAW: + case raw: + printTestsAsList(testSuits); + break; + case CSV: + case csv: + printTestsAsCsv(testSuits); + break; + default: + printTestsPretty(testSuits); + break; + } + + return 0; + + } + + public int listTags(List testSuits, OutputFormat format) throws Throwable { + + Set tags = new HashSet(); + for (ITestSuite testSuite : testSuits) { + tags.addAll(testSuite.getTags()); + for (ITest test : testSuite.getTests()) { + tags.addAll(test.getTags()); + } + } + + switch (format) { + case JSON: + case json: + printTagsAsJson(tags); + break; + case CSV: + case csv: + printTagsAsCsv(tags); + break; + default: + printTagsPretty(tags); + break; + } + + return 0; + + } + + private void printTestsAsJson(List testSuits) { + List tests = new Vector(); + for (ITestSuite testSuite : testSuits) { + for (ITest test : testSuite.getTests()) { + tests.add(new File(testSuite.getFilename()).getAbsolutePath() + "@" + test.getHash().substring(0, 8)); + } + } + System.out.println(JsonOutput.toJson(tests)); + } + + private void printTestsAsList(List testSuits) { + for (ITestSuite testSuite : testSuits) { + for (ITest test : testSuite.getTests()) { + System.out.println(new File(testSuite.getFilename()).getAbsolutePath() + "@" + test.getHash().substring(0, 8)); + } + } + } + + private void printTestsAsCsv(List testSuits) { + List tests = new Vector(); + for (ITestSuite testSuite : testSuits) { + for (ITest test : testSuite.getTests()) { + tests.add(new File(testSuite.getFilename()).getAbsolutePath() + "@" + test.getHash().substring(0, 8)); + } + } + System.out.println(String.join(",", tests)); + } + + private void printTestsPretty(List testSuits) { + int count = 0; + + File baseDir = new File(""); + + for (ITestSuite testSuite : testSuits) { + + System.out.println(); + System.out.println("[" + FileUtil.makeRelative(baseDir, new File(testSuite.getFilename())) + "] " + + AnsiText.bold(testSuite.getName())); + System.out.println(); + + for (ITest test : testSuite.getTests()) { + System.out.println(AnsiText.padding("[" + FileUtil.makeRelative(baseDir, new File(testSuite.getFilename())) + "@" + + test.getHash().substring(0, 8) + "] " + AnsiText.bold(test.getName()), 2)); + count++; + } + } + + System.out.println(); + System.out.println("Found " + count + " tests."); + System.out.println(); + } + + private void printTagsAsJson(Set tags) { + System.out.println(JsonOutput.toJson(tags)); + } + + private void printTagsAsCsv(Set tags) { + System.out.println(String.join(",", tags)); + } + + private void printTagsPretty(Set tags) { + for (String tag : tags) { + System.out.println(tag); + } + } + } diff --git a/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java b/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java index 43705551..07fe6274 100644 --- a/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java +++ b/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java @@ -2,23 +2,23 @@ import java.io.File; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.nio.file.*; import java.util.ArrayList; import java.util.List; import java.util.Vector; import java.util.function.Consumer; +import com.askimed.nf.test.core.*; +import com.askimed.nf.test.core.reports.CsvReportWriter; +import com.askimed.nf.test.lang.dependencies.Coverage; +import com.askimed.nf.test.lang.dependencies.DependencyExporter; +import com.askimed.nf.test.lang.dependencies.DependencyResolver; +import com.askimed.nf.test.util.AnsiText; +import com.askimed.nf.test.util.GitCommand; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.askimed.nf.test.App; import com.askimed.nf.test.config.Config; -import com.askimed.nf.test.core.AnsiTestExecutionListener; -import com.askimed.nf.test.core.GroupTestExecutionListener; -import com.askimed.nf.test.core.TagQuery; -import com.askimed.nf.test.core.TestExecutionEngine; import com.askimed.nf.test.core.reports.TapTestReportWriter; import com.askimed.nf.test.core.reports.XmlReportWriter; import com.askimed.nf.test.lang.TestSuiteBuilder; @@ -34,7 +34,9 @@ @Command(name = "test") public class RunTestsCommand extends AbstractCommand { - @Parameters(description = "test files") + private static final String SHARD_STRATEGY_ROUND_ROBIN = "round-robin"; + + @Parameters(description = "test dependencies") private List testPaths = new ArrayList(); @Option(names = { @@ -58,17 +60,57 @@ public class RunTestsCommand extends AbstractCommand { private String tap = null; @Option(names = { - "--junitxml" }, description = "Write test results in Junit Xml Format", required = false, showDefaultValue = Visibility.ALWAYS) + "--junitxml" }, description = "Write test results in Junit Xml format", required = false, showDefaultValue = Visibility.ALWAYS) private String junitXml = null; + @Option(names = { + "--csv" }, description = "Write test results in csv format", required = false, showDefaultValue = Visibility.ALWAYS) + private String csv = null; + @Option(names = { "--update-snapshot", "--updateSnapshot" }, description = "Use this flag to re-record every snapshot that fails during this test run.", required = false, showDefaultValue = Visibility.ALWAYS) private boolean updateSnapshot = false; + @Option(names = { "--ci" }, description = "Activates CI mode. Instead of automatically storing a new snapshot as per usual, it will now fail the test.", required = false, showDefaultValue = Visibility.ALWAYS) + private boolean ciMode = false; + + @Option(names = { "--related-tests", "--relatedTests"}, description = "Finds and executes all related tests for the provided .nf or nf.test files.", required = false, showDefaultValue = Visibility.ALWAYS) + private boolean findRelatedTests = false; + + @Option(names = { "--follow-dependencies", "--followDependencies"}, description = "Follows all dependencies when related-tests is set.", required = false, showDefaultValue = Visibility.ALWAYS) + private boolean followDependencies = false; + + @Option(names = { "--only-changed", "--onlyChanged"}, description = "Runs tests only for those files which are modified in the current git repository", required = false, showDefaultValue = Visibility.ALWAYS) + private boolean onlyChanged = false; + + @Option(names = { "--changed-since", "--changedSince"}, description = "Runs tests related to the changes since the provided branch or commit hash", required = false, showDefaultValue = Visibility.ALWAYS) + private String changedSince = null; + + @Option(names = { "--changed-until", "--changdUntil"}, description = "Runs tests related to the changes until the provided branch or commit hash", required = false, showDefaultValue = Visibility.ALWAYS) + private String changedUntil = null; + + @Option(names = { "--coverage"}, description = "Print simple coverage calculation.", required = false, showDefaultValue = Visibility.ALWAYS) + private boolean coverage = false; + + @Option(names = { "--dry-run", "--dryRun" }, description = "Execute most of test discovery but stop before running any of the testcases.", required = false, showDefaultValue = Visibility.ALWAYS) + private boolean dryRun = false; + + @Option(names = { + "--graph" }, description = "Export dependency graph as dot file", required = false, showDefaultValue = Visibility.ALWAYS) + private String graph = null; + @Option(names = { "--clean-snapshot", "--cleanSnapshot", "--wipe-snapshot", "--wipeSnapshot" }, description = "Removes all obsolete snapshots.", required = false, showDefaultValue = Visibility.ALWAYS) private boolean cleanSnapshot = false; + @Option(names = { + "--shard" }, description = "Split into arbitrary chunks. Format: i/n. Example: 2/5 executes the second chunk of five.", required = false, showDefaultValue = Visibility.ALWAYS) + private String shard = null; + + @Option(names = { + "--shard-strategy" }, description = "Strategy to build shards. Values: round-robin or none.", required = false, showDefaultValue = Visibility.ALWAYS) + private String shardStrategy = SHARD_STRATEGY_ROUND_ROBIN; + @Option(names = { "--lib" }, description = "Library extension path", required = false, showDefaultValue = Visibility.ALWAYS) private String lib = ""; @@ -92,6 +134,7 @@ public class RunTestsCommand extends AbstractCommand { public Integer execute() throws Exception { List scripts = new ArrayList(); + Config config = null; PluginManager manager = new PluginManager(false); try { @@ -103,7 +146,7 @@ public Integer execute() throws Exception { File configFile = new File(configFilename); if (configFile.exists()) { log.info("Load config from file {}...", configFile.getAbsolutePath()); - Config config = Config.parse(configFile); + config = Config.parse(configFile); defaultConfigFile = config.getConfigFile(); defaultWithTrace = config.isWithTrace(); if (!libDir.isEmpty()) { @@ -112,10 +155,9 @@ public Integer execute() throws Exception { libDir += config.getLibDir(); manager = config.getPluginManager(); - if (testPaths.size() == 0) { + if (testPaths.isEmpty()) { File folder = new File(config.getTestsDir()); testPaths.add(folder); - System.out.println("Found " + testPaths.size() + " files in test directory."); } TestSuiteBuilder.setConfig(config); @@ -126,8 +168,6 @@ public Integer execute() throws Exception { log.warn("No nf-test config file found."); } - scripts = pathsToScripts(testPaths); - } catch (Exception e) { System.out.println(AnsiColors.red("Error: Syntax errors in nf-test config file: " + e)); @@ -139,7 +179,75 @@ public Integer execute() throws Exception { } - if (scripts.size() == 0) { + if ((onlyChanged || findRelatedTests || changedSince != null) && config == null) { + System.out.println(AnsiColors.red("To find related tests a nf-test config file has to be present.")); + return 2; + } + + List ignorePatterns = new Vector(); + File baseDir = new File(new File("").getAbsolutePath()); + DependencyResolver resolver = new DependencyResolver(baseDir); + resolver.setFollowingDependencies(followDependencies); + + if (onlyChanged || changedSince != null) { + + GitCommand git = new GitCommand(); + List changedFiles = null; + + if (onlyChanged) { + changedFiles = git.findChanges(baseDir); + }else if(changedSince != null && changedUntil == null) { + changedFiles = git.findChangesSince(baseDir, changedSince); + } else if(changedSince != null && changedUntil != null) { + changedFiles = git.findChangesBetween(baseDir, changedSince, changedUntil); + } + + if (changedFiles.isEmpty()) { + System.out.println(AnsiColors.green("Nothing to do.")); + return 0; + } else { + System.out.println("Detected " + changedFiles.size() + " changed files"); + AnsiText.printBulletList(changedFiles); + } + + testPaths = changedFiles; + findRelatedTests = true; + } + + if (findRelatedTests) { + + resolver.buildGraph(config.getIgnore(), config.getTriggers()); + scripts = resolver.findRelatedTestsByFiles(testPaths); + System.out.println("Found " + scripts.size() + " related test(s)"); + if (scripts.isEmpty()) { + System.out.println(AnsiColors.green("Nothing to do.")); + return 0; + } + + AnsiText.printBulletList(scripts); + + if (coverage) { + new Coverage(resolver).getByFiles(testPaths).print(); + } + + } else { + if (config != null) { + resolver.buildGraph(config.getIgnore(), config.getTriggers()); + } else { + resolver.buildGraph(); + } + scripts = resolver.findTestsByFiles(testPaths); + if (coverage) { + new Coverage(resolver).getAll().print(); + } + } + + if (graph != null) { + DependencyExporter.generateDotFile(resolver, graph); + } + + + if (scripts.isEmpty()) { System.out.println(AnsiColors .red("Error: No tests or test directories containing scripts that end with *.test provided.")); log.error("No tests ot directories found containing test files."); @@ -150,30 +258,35 @@ public Integer execute() throws Exception { loadPlugins(manager, plugins); - GroupTestExecutionListener listener = new GroupTestExecutionListener(); - listener.addListener(new AnsiTestExecutionListener()); - if (tap != null) { - listener.addListener(new TapTestReportWriter(tap)); - } - - if (junitXml != null) { - listener.addListener(new XmlReportWriter(junitXml)); - } + GroupTestExecutionListener listener = setupExecutionListeners(); NextflowCommand.setVerbose(verbose); - - TagQuery tagQuery = new TagQuery(tags); + + Environment environment = new Environment(); + environment.setLibDir(libDir); + environment.setPluginManager(manager); + + TestSuiteResolver testSuiteResolver = new TestSuiteResolver(environment); + List testSuits = testSuiteResolver.parse(scripts, new TagQuery(tags)); + + testSuits.sort(TestSuiteSorter.getDefault()); + if (shard != null) { + if (shardStrategy.equalsIgnoreCase(SHARD_STRATEGY_ROUND_ROBIN)){ + testSuits = TestSuiteSharder.shardWithRoundRobin(testSuits, shard); + } else { + testSuits = TestSuiteSharder.shard(testSuits, shard); + } + } TestExecutionEngine engine = new TestExecutionEngine(); engine.setListener(listener); - engine.setScripts(scripts); - engine.setTagQuery(tagQuery); + engine.setTestSuites(testSuits); engine.setDebug(debug); engine.setUpdateSnapshot(updateSnapshot); engine.setCleanSnapshot(cleanSnapshot); - engine.setLibDir(libDir); - engine.setPluginManager(manager); + engine.setCIMode(ciMode); engine.addProfile(profile); + engine.setDryRun(dryRun); if (withoutTrace) { engine.setWithTrace(false); } else { @@ -181,6 +294,11 @@ public Integer execute() throws Exception { } engine.setConfigFile(defaultConfigFile); + + if (dryRun) { + System.out.println(AnsiColors.yellow("Dry run mode activated: tests are not executed, just listed.")); + } + return engine.execute(); } catch (Throwable e) { @@ -198,6 +316,23 @@ public Integer execute() throws Exception { } + private GroupTestExecutionListener setupExecutionListeners() throws IOException { + GroupTestExecutionListener listener = new GroupTestExecutionListener(); + listener.addListener(new AnsiTestExecutionListener()); + if (tap != null) { + listener.addListener(new TapTestReportWriter(tap)); + } + + if (junitXml != null) { + listener.addListener(new XmlReportWriter(junitXml)); + } + + if (csv != null) { + listener.addListener(new CsvReportWriter(csv)); + } + return listener; + } + private void loadPlugins(PluginManager manager, String plugins) throws IOException { if (plugins == null) { diff --git a/src/main/java/com/askimed/nf/test/config/Config.java b/src/main/java/com/askimed/nf/test/config/Config.java index cd5e817a..f836c9f1 100644 --- a/src/main/java/com/askimed/nf/test/config/Config.java +++ b/src/main/java/com/askimed/nf/test/config/Config.java @@ -1,7 +1,7 @@ package com.askimed.nf.test.config; import java.io.File; -import java.util.Map; +import java.util.*; import com.askimed.nf.test.App; import com.askimed.nf.test.nextflow.NextflowCommand; @@ -46,8 +46,12 @@ public class Config { private String stageMode = FileStaging.MODE_COPY; + private List ignore = new Vector(); + private Map requires = null; + private List triggers = new Vector(); + public void testsDir(String testsDir) { this.testsDir = testsDir; } @@ -148,6 +152,50 @@ public void stageMode(String stageMode) { this.stageMode = stageMode; } + public void setIgnore(List ignore) { + this.ignore = ignore; + } + + public List getIgnore() { + return ignore; + } + + public void ignore(List ignores) { + for (Object ignore: ignores) { + this.ignore.add(ignore.toString()); + } + } + + public void ignore(String ignore) { + this.ignore.add(ignore); + } + + public void ignore(String ... ignore) { + this.ignore.addAll(Arrays.asList(ignore)); + } + + public void setTriggers(List triggers) { + this.triggers = triggers; + } + + public List getTriggers() { + return triggers; + } + + public void triggers(List triggers) { + for (Object trigger: triggers) { + this.triggers.add(trigger.toString()); + } + } + + public void triggers(String ... triggers) { + this.triggers.addAll(Arrays.asList(triggers)); + } + + public void trigger(String trigger) { + this.triggers.add(trigger); + } + public void stage(Closure closure) { closure.setDelegate(stageBuilder); closure.setResolveStrategy(Closure.DELEGATE_ONLY); diff --git a/src/main/java/com/askimed/nf/test/core/AbstractTest.java b/src/main/java/com/askimed/nf/test/core/AbstractTest.java index ff7675f1..39159a4e 100644 --- a/src/main/java/com/askimed/nf/test/core/AbstractTest.java +++ b/src/main/java/com/askimed/nf/test/core/AbstractTest.java @@ -62,6 +62,8 @@ public abstract class AbstractTest implements ITest { private boolean updateSnapshot = false; + private boolean ciMode = false; + private boolean debug = false; private boolean withTrace = true; @@ -244,6 +246,16 @@ public boolean isUpdateSnapshot() { return updateSnapshot; } + @Override + public void setCIMode(boolean ciMode) { + this.ciMode = ciMode; + } + + @Override + public boolean isCIMode() { + return ciMode; + } + @Override public String toString() { return getHash().substring(0, 8) + ": " + getName(); diff --git a/src/main/java/com/askimed/nf/test/core/Environment.java b/src/main/java/com/askimed/nf/test/core/Environment.java new file mode 100644 index 00000000..8de7eb7f --- /dev/null +++ b/src/main/java/com/askimed/nf/test/core/Environment.java @@ -0,0 +1,26 @@ +package com.askimed.nf.test.core; + +import com.askimed.nf.test.plugins.PluginManager; + +public class Environment { + + private String libDir = ""; + + private PluginManager pluginManager = null; + + public void setLibDir(String libDir) { + this.libDir = libDir; + } + + public String getLibDir() { + return libDir; + } + + public void setPluginManager(PluginManager pluginManager) { + this.pluginManager = pluginManager; + } + + public PluginManager getPluginManager() { + return pluginManager; + } +} diff --git a/src/main/java/com/askimed/nf/test/core/ITest.java b/src/main/java/com/askimed/nf/test/core/ITest.java index 180eb236..0bbeab6e 100644 --- a/src/main/java/com/askimed/nf/test/core/ITest.java +++ b/src/main/java/com/askimed/nf/test/core/ITest.java @@ -34,4 +34,8 @@ public interface ITest extends ITaggable { public boolean isUpdateSnapshot(); + public void setCIMode(boolean ciMode); + + public boolean isCIMode(); + } diff --git a/src/main/java/com/askimed/nf/test/core/TestExecutionEngine.java b/src/main/java/com/askimed/nf/test/core/TestExecutionEngine.java index 6bacbfb2..652a56d6 100644 --- a/src/main/java/com/askimed/nf/test/core/TestExecutionEngine.java +++ b/src/main/java/com/askimed/nf/test/core/TestExecutionEngine.java @@ -22,7 +22,7 @@ public class TestExecutionEngine { - private List scripts; + private List testSuits; private ITestExecutionListener listener = new AnsiTestExecutionListener(); @@ -38,20 +38,14 @@ public class TestExecutionEngine { private boolean updateSnapshot = false; - private boolean cleanSnapshot = false; - - private String libDir = ""; + private boolean ciMode = false; - private PluginManager pluginManager = null; + private boolean cleanSnapshot = false; - private TagQuery tagQuery = new TagQuery(); + private boolean dryRun = false; private static Logger log = LoggerFactory.getLogger(TestExecutionEngine.class); - public void setScripts(List scripts) { - this.scripts = scripts; - } - public void setDebug(boolean debug) { this.debug = debug; } @@ -81,70 +75,25 @@ public void setUpdateSnapshot(boolean updateSnapshot) { this.updateSnapshot = updateSnapshot; } - public void setCleanSnapshot(boolean cleanSnapshot) { - this.cleanSnapshot = cleanSnapshot; + public void setCIMode(boolean ciMode) { + if (ciMode) { + System.out.println("nf-test runs in CI mode."); + } + this.ciMode = ciMode; } - public void setTagQuery(TagQuery tagQuery) { - this.tagQuery = tagQuery; + public void setCleanSnapshot(boolean cleanSnapshot) { + this.cleanSnapshot = cleanSnapshot; } - public void setLibDir(String libDir) { - this.libDir = libDir; + public void setDryRun(boolean dryRun) { + this.dryRun = dryRun; } public void setListener(ITestExecutionListener listener) { this.listener = listener; } - public void setPluginManager(PluginManager pluginManager) { - this.pluginManager = pluginManager; - } - - protected List parse(TagQuery tagQuery) throws Throwable { - - List testSuits = new Vector(); - - for (File script : scripts) { - String testId = null; - if (script.getAbsolutePath().contains("@")) { - String[] tiles = script.getAbsolutePath().split("@"); - script = new File(tiles[0]); - testId = tiles[1]; - } - if (!script.exists()) { - throw new Exception("Test file '" + script.getAbsolutePath() + "' not found."); - } - ITestSuite testSuite = TestSuiteBuilder.parse(script, libDir, pluginManager); - - boolean empty = true; - - for (ITest test : testSuite.getTests()) { - if (testId != null) { - if (!test.getHash().startsWith(testId)) { - test.skip(); - } - } - - if (!tagQuery.matches(test)) { - test.skip(); - } - - if (!test.isSkipped()) { - empty = false; - } - - } - - if (!empty) { - testSuits.add(testSuite); - } - } - - return testSuits; - - } - public int execute() throws Throwable { if (configFile != null) { @@ -156,9 +105,7 @@ public int execute() throws Throwable { } } - List testSuits = parse(tagQuery); - - if (testSuits.size() == 0) { + if (testSuits.isEmpty()) { System.out.println(AnsiColors.red("Error: no valid tests found.")); System.out.println(); return 1; @@ -205,6 +152,7 @@ public int execute() throws Throwable { TestExecutionResult result = new TestExecutionResult(test); test.setWithTrace(withTrace); test.setUpdateSnapshot(updateSnapshot); + test.setCIMode(ciMode); try { // override debug flag from CLI @@ -213,7 +161,9 @@ public int execute() throws Throwable { } result.setStartTime(System.currentTimeMillis()); - test.execute(); + if (!dryRun) { + test.execute(); + } result.setStatus(TestExecutionResultStatus.PASSED); } catch (Throwable e) { @@ -259,162 +209,7 @@ public int execute() throws Throwable { } - public int listTests(OutputFormat format) throws Throwable { - - if (configFile != null) { - if (!configFile.exists()) { - System.out.println( - AnsiColors.red("Error: Test config file '" + configFile.getAbsolutePath() + "'not found")); - System.out.println(); - return 1; - } - } - - List testSuits = parse(tagQuery); - - if (testSuits.size() == 0) { - System.out.println(AnsiColors.red("Error: no valid tests found.")); - System.out.println(); - return 1; - } - - switch (format) { - case JSON: - case json: - printTestsAsJson(testSuits); - break; - case RAW: - case raw: - printTestsAsList(testSuits); - break; - case CSV: - case csv: - printTestsAsCsv(testSuits); - break; - default: - printTestsPretty(testSuits); - break; - } - - return 0; - + public void setTestSuites(List testSuits) { + this.testSuits = testSuits; } - - public int listTags(OutputFormat format) throws Throwable { - - if (configFile != null) { - if (!configFile.exists()) { - System.out.println( - AnsiColors.red("Error: Test config file '" + configFile.getAbsolutePath() + "'not found")); - System.out.println(); - return 1; - } - } - - List testSuits = parse(tagQuery); - - if (testSuits.size() == 0) { - System.out.println(AnsiColors.red("Error: no valid tests found.")); - System.out.println(); - return 1; - } - - Set tags = new HashSet(); - for (ITestSuite testSuite : testSuits) { - tags.addAll(testSuite.getTags()); - for (ITest test : testSuite.getTests()) { - tags.addAll(test.getTags()); - } - } - - switch (format) { - case JSON: - case json: - printTagsAsJson(tags); - break; - case CSV: - case csv: - printTagsAsCsv(tags); - break; - default: - printTagsPretty(tags); - break; - } - - return 0; - - } - - private void printTestsAsJson(List testSuits) { - List tests = new Vector(); - int index = 0; - for (ITestSuite testSuite : testSuits) { - for (ITest test : testSuite.getTests()) { - tests.add(scripts.get(index).getAbsolutePath() + "@" + test.getHash().substring(0, 8)); - } - index++; - } - System.out.println(JsonOutput.toJson(tests)); - } - - private void printTestsAsList(List testSuits) { - int index = 0; - for (ITestSuite testSuite : testSuits) { - for (ITest test : testSuite.getTests()) { - System.out.println(scripts.get(index).getAbsolutePath() + "@" + test.getHash().substring(0, 8)); - } - index++; - } - } - - private void printTestsAsCsv(List testSuits) { - List tests = new Vector(); - int index = 0; - for (ITestSuite testSuite : testSuits) { - for (ITest test : testSuite.getTests()) { - tests.add(scripts.get(index).getAbsolutePath() + "@" + test.getHash().substring(0, 8)); - } - index++; - } - System.out.println(String.join(",", tests)); - } - - private void printTestsPretty(List testSuits) { - int index = 0; - int count = 0; - for (ITestSuite testSuite : testSuits) { - - System.out.println(); - System.out.println("[" + FileUtil.makeRelative(baseDir, scripts.get(index)) + "] " - + AnsiText.bold(testSuite.getName())); - System.out.println(); - - for (ITest test : testSuite.getTests()) { - System.out.println(AnsiText.padding("[" + FileUtil.makeRelative(baseDir, scripts.get(index)) + "@" - + test.getHash().substring(0, 8) + "] " + AnsiText.bold(test.getName()), 2)); - count++; - - } - index++; - } - - System.out.println(); - System.out.println("Found " + count + " tests."); - System.out.println(); - } - - private void printTagsAsJson(Set tags) { - System.out.println(JsonOutput.toJson(tags)); - } - - private void printTagsAsCsv(Set tags) { - System.out.println(String.join(",", tags)); - } - - private void printTagsPretty(Set tags) { - for (String tag : tags) { - System.out.println(tag); - } - } - } diff --git a/src/main/java/com/askimed/nf/test/core/TestSuiteResolver.java b/src/main/java/com/askimed/nf/test/core/TestSuiteResolver.java new file mode 100644 index 00000000..6de32991 --- /dev/null +++ b/src/main/java/com/askimed/nf/test/core/TestSuiteResolver.java @@ -0,0 +1,66 @@ +package com.askimed.nf.test.core; + +import com.askimed.nf.test.lang.TestSuiteBuilder; + +import java.io.File; +import java.util.List; +import java.util.Vector; + +public class TestSuiteResolver { + + private final Environment environment; + + public TestSuiteResolver(Environment environment) { + this.environment = environment; + } + + public List parse(List scripts) throws Throwable { + return parse(scripts, new TagQuery()); + } + + public List parse(List scripts, TagQuery tagQuery) throws Throwable { + + List testSuits = new Vector(); + + for (File script : scripts) { + String testId = null; + if (script.getAbsolutePath().contains("@")) { + String[] tiles = script.getAbsolutePath().split("@"); + script = new File(tiles[0]); + testId = tiles[1]; + } + if (!script.exists()) { + throw new Exception("Test file '" + script.getAbsolutePath() + "' not found."); + } + ITestSuite testSuite = TestSuiteBuilder.parse(script, environment); + + boolean empty = true; + + for (ITest test : testSuite.getTests()) { + if (testId != null) { + if (!test.getHash().startsWith(testId)) { + test.skip(); + } + } + + if (!tagQuery.matches(test)) { + test.skip(); + } + + if (!test.isSkipped()) { + empty = false; + } + + } + + if (!empty) { + testSuits.add(testSuite); + } + } + + return testSuits; + + } + + +} diff --git a/src/main/java/com/askimed/nf/test/core/TestSuiteSharder.java b/src/main/java/com/askimed/nf/test/core/TestSuiteSharder.java new file mode 100644 index 00000000..da4a427c --- /dev/null +++ b/src/main/java/com/askimed/nf/test/core/TestSuiteSharder.java @@ -0,0 +1,117 @@ +package com.askimed.nf.test.core; + +import java.util.List; +import java.util.Vector; + +public class TestSuiteSharder { + + public TestSuiteSharder(String shard) { + + } + + public static List shard(List tests, String shard) { + int[] values = parse(shard); + return shard(tests, values[0], values[1]); + } + + public static List shard(List tests, int i, int n) { + if (tests == null || tests.isEmpty()) { + throw new IllegalArgumentException("Input list cannot be null or empty"); + } + if (i <= 0 || i > n) { + throw new IllegalArgumentException("i must be greater than 0 and less than or equal to n"); + } + + //count all non skipped tests. + int totalCases = 0; + for (ITestSuite testSuite: tests) { + for (ITest testCase: testSuite.getTests()) { + if (!testCase.isSkipped()) { + totalCases++; + } + } + } + int chunkSize = (int) Math.ceil((double) totalCases / n); + int start = (i - 1) * chunkSize; + int end = Math.min(start + chunkSize, totalCases); + + int index = 0; + List shard = new Vector(); + for (ITestSuite testSuite: tests) { + boolean empty = true; + for (ITest testCase: testSuite.getTests()) { + if (!testCase.isSkipped()) { + if (index >= start && index < end) { + empty = false; + } else { + testCase.skip(); + } + index++; + } + } + if (!empty) { + shard.add(testSuite); + } + } + + return shard; + } + + public static List shardWithRoundRobin(List tests, String shard) { + int[] values = parse(shard); + return shardWithRoundRobin(tests, values[0], values[1]); + } + + public static List shardWithRoundRobin(List tests, int i, int n) { + if (tests == null || tests.isEmpty()) { + throw new IllegalArgumentException("Input list cannot be null or empty"); + } + if (i <= 0 || i > n) { + throw new IllegalArgumentException("i must be greater than 0 and less than or equal to n"); + } + + int index = 0; + List shard = new Vector(); + for (ITestSuite testSuite: tests) { + boolean empty = true; + for (ITest testCase: testSuite.getTests()) { + if (!testCase.isSkipped()) { + if (index % n == (i-1)) { + empty = false; + } else { + testCase.skip(); + } + index++; + } + } + if (!empty) { + shard.add(testSuite); + } + } + return shard; + } + + public static int[] parse(String input) throws IllegalArgumentException { + String[] parts = input.split("/"); + + // Check if the input has two parts separated by "/" + if (parts.length != 2) { + throw new IllegalArgumentException("Shard format: Input format must be 'i/n'"); + } + + try { + int i = Integer.parseInt(parts[0]); + int n = Integer.parseInt(parts[1]); + + // Check if i and n satisfy the conditions + if (i <= 0 || i > n) { + throw new IllegalArgumentException("Shard format: i must be greater than 0 and less than or equal to n. Format: i/n"); + } + + return new int[] {i, n}; + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Shard format: Invalid format. Both i and n must be integers. Format: i/n"); + } + } + +} diff --git a/src/main/java/com/askimed/nf/test/core/TestSuiteSorter.java b/src/main/java/com/askimed/nf/test/core/TestSuiteSorter.java new file mode 100644 index 00000000..04efa227 --- /dev/null +++ b/src/main/java/com/askimed/nf/test/core/TestSuiteSorter.java @@ -0,0 +1,25 @@ +package com.askimed.nf.test.core; + +import java.util.Comparator; + +public class TestSuiteSorter { + + public static Comparator getDefault() { + return new Comparator() { + @Override + public int compare(ITestSuite o1, ITestSuite o2) { + return o1.getFilename().compareTo(o2.getFilename()); + } + }; + } + + public static Comparator getTypeSorter() { + return new Comparator() { + @Override + public int compare(ITestSuite o1, ITestSuite o2) { + return o1.getFilename().compareTo(o2.getFilename()); + } + }; + } + +} diff --git a/src/main/java/com/askimed/nf/test/core/reports/CsvReportWriter.java b/src/main/java/com/askimed/nf/test/core/reports/CsvReportWriter.java new file mode 100644 index 00000000..a390bbb2 --- /dev/null +++ b/src/main/java/com/askimed/nf/test/core/reports/CsvReportWriter.java @@ -0,0 +1,65 @@ + +package com.askimed.nf.test.core.reports; + +import com.askimed.nf.test.core.TestExecutionResult; +import com.askimed.nf.test.core.TestSuiteExecutionResult; +import com.opencsv.CSVWriter; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.List; + +public class CsvReportWriter extends AbstractTestReportWriter { + + private String filename; + + public CsvReportWriter(String filename) throws IOException { + this.filename = filename; + } + + @Override + public void writeToFile(List testSuites) throws IOException { + + String[] header = new String[]{ + "filename", + "testsuite", + "type", + "test", + "result", + "start", + "end", + "time", + + }; + + CSVWriter writer = new CSVWriter(new FileWriter(new File(filename))); + writer.writeNext(header); + + for (TestSuiteExecutionResult testSuite : testSuites) { + + for (TestExecutionResult test : testSuite.getTests()) { + + String[] line = new String[]{ + testSuite.getTestSuite().getFilename(), + testSuite.getTestSuite().getName(), + testSuite.getTestSuite().getClass().getSimpleName(), + test.getTest().getName(), + test.getStatus().toString(), + test.getStartTime() + "", + test.getEndTime() + "", + test.getExecutionTimeInSecs() + "" + }; + + writer.writeNext(line); + + } + } + + writer.close(); + + System.out.println("Wrote csv report to file " + filename + "\n"); + + } + +} diff --git a/src/main/java/com/askimed/nf/test/lang/TestSuiteBuilder.java b/src/main/java/com/askimed/nf/test/lang/TestSuiteBuilder.java index 80b425d2..ffa96136 100644 --- a/src/main/java/com/askimed/nf/test/lang/TestSuiteBuilder.java +++ b/src/main/java/com/askimed/nf/test/lang/TestSuiteBuilder.java @@ -2,6 +2,7 @@ import java.io.File; +import com.askimed.nf.test.core.Environment; import org.codehaus.groovy.control.CompilerConfiguration; import org.codehaus.groovy.control.customizers.ImportCustomizer; @@ -74,14 +75,10 @@ private static void executeClosure(ITestSuite suite, Closure closure) { } public static ITestSuite parse(File script) throws Throwable { - return parse(script, "", null); + return parse(script, new Environment()); } - public static ITestSuite parse(File script, PluginManager pluginManager) throws Throwable { - return parse(script, "", pluginManager); - } - - public static ITestSuite parse(File script, String libDir, PluginManager pluginManager) throws Throwable { + public static ITestSuite parse(File script, Environment environment) throws Throwable { ImportCustomizer customizer = new ImportCustomizer(); customizer.addStaticImport("com.askimed.nf.test.lang.TestSuiteBuilder", "nextflow_pipeline"); @@ -91,15 +88,15 @@ public static ITestSuite parse(File script, String libDir, PluginManager pluginM customizer.addStaticStars("com.askimed.nf.test.lang.extensions.GlobalMethods"); ClassLoader classLoader = TestSuiteBuilder.class.getClassLoader(); - if (pluginManager != null) { - for (String staticImport : pluginManager.getStaticImports()) { + if (environment.getPluginManager() != null) { + for (String staticImport : environment.getPluginManager().getStaticImports()) { customizer.addStaticStars(staticImport); } - classLoader = pluginManager.getClassLoader(); + classLoader = environment.getPluginManager().getClassLoader(); } CompilerConfiguration compilerConfiguration = new CompilerConfiguration(); - String classpath = script.getAbsoluteFile().getParentFile().getAbsolutePath() + "/lib:" + libDir; + String classpath = script.getAbsoluteFile().getParentFile().getAbsolutePath() + "/lib:" + environment.getLibDir(); compilerConfiguration.setClasspath(classpath); compilerConfiguration.addCompilationCustomizers(customizer); diff --git a/src/main/java/com/askimed/nf/test/lang/dependencies/Coverage.java b/src/main/java/com/askimed/nf/test/lang/dependencies/Coverage.java new file mode 100644 index 00000000..3d38cf62 --- /dev/null +++ b/src/main/java/com/askimed/nf/test/lang/dependencies/Coverage.java @@ -0,0 +1,128 @@ +package com.askimed.nf.test.lang.dependencies; + +import com.askimed.nf.test.util.AnsiColors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.text.DecimalFormat; +import java.util.List; +import java.util.Vector; + +public class Coverage { + + private int coveredItems = 0; + + private DependencyGraph graph; + + private List items = new Vector(); + + private static Logger log = LoggerFactory.getLogger(Coverage.class); + + public Coverage(DependencyGraph graph) { + this.graph = graph; + } + + public Coverage(DependencyResolver resolver) { + this.graph = resolver.getGraph(); + } + + public void add(File file, boolean covered) { + items.add(new CoverageItem(file, covered)); + if (covered) { + coveredItems++; + } + } + + public int getCoveredItems() { + return coveredItems; + } + + public List getItems() { + return items; + } + + public Coverage getAll(){ + items.clear(); + coveredItems=0; + long time0 = System.currentTimeMillis(); + + for (DependencyGraph.Node node: graph.getNodes()){ + if (node.getMetaFile().getType() != IMetaFile.MetaFileType.SOURCE_FILE) { + continue; + } + + add(new File(node.getMetaFile().getFilename()), + node.hasDependencyOfType(IMetaFile.MetaFileType.TEST_FILE)); + + } + + long time1 = System.currentTimeMillis(); + + log.info("Calculated coverage for {} files in {} sec", graph.size(), (time1 - time0) / 1000.0); + + return this; + } + + public Coverage getByFiles(List files){ + items.clear(); + coveredItems=0; + long time0 = System.currentTimeMillis(); + + for (File file: files){ + + DependencyGraph.Node node = graph.getNode(file.getAbsolutePath()); + add(new File(node.getMetaFile().getFilename()), + node.hasDependencyOfType(IMetaFile.MetaFileType.TEST_FILE)); + + for (DependencyGraph.Node dependency: node.getDependencies()) { + + if (dependency.getMetaFile().getType() != IMetaFile.MetaFileType.SOURCE_FILE) { + continue; + } + + add(new File(dependency.getMetaFile().getFilename()), + dependency.hasDependencyOfType(IMetaFile.MetaFileType.TEST_FILE)); + } + + } + + long time1 = System.currentTimeMillis(); + + log.info("Calculated coverage for {} files in {} sec", graph.size(), (time1 - time0) / 1000.0); + + return this; + } + + public void print() { + DecimalFormat decimalFormat = new DecimalFormat("#.##"); + System.out.println(); + System.out.print("Coverage: " + getCoveredItems() + "/" + getItems().size()); + System.out.println(" (" + decimalFormat.format(getCoveredItems() / (float) getItems().size() * 100) + "%)"); + for (Coverage.CoverageItem item : getItems()) { + System.out.println(" - " + (item.isCovered() ? AnsiColors.green(item.getFile().getAbsolutePath()) : AnsiColors.red(item.getFile().getAbsolutePath()))); + } + System.out.println(); + } + + public static class CoverageItem { + + private File file; + + private boolean covered = false; + + public CoverageItem(File file, boolean covered) { + this.file = file; + this.covered = covered; + } + + public File getFile() { + return file; + } + + public boolean isCovered() { + return covered; + } + } + +} diff --git a/src/main/java/com/askimed/nf/test/lang/dependencies/DependencyExporter.java b/src/main/java/com/askimed/nf/test/lang/dependencies/DependencyExporter.java new file mode 100644 index 00000000..4949b75e --- /dev/null +++ b/src/main/java/com/askimed/nf/test/lang/dependencies/DependencyExporter.java @@ -0,0 +1,70 @@ +package com.askimed.nf.test.lang.dependencies; + +import com.askimed.nf.test.nextflow.NextflowScript; + +import javax.xml.transform.Source; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Paths; + +public class DependencyExporter { + + public static void generateDotFile(DependencyResolver resolver, String outputPath) { + try (FileWriter writer = new FileWriter(outputPath)) { + writer.write("digraph G {\n"); + writer.write("\tnode [shape=note,colorscheme=pastel19\n];\n"); + writer.write("\trankdir=LR;\n"); + + for (DependencyGraph.Node node: resolver.getGraph().getNodes()) { + writer.write("\t" + quote(getNodeId(resolver.getBaseDir(), node.getMetaFile())) + + " [label=" + quote(getNodeName(resolver.getBaseDir(), node.getMetaFile())) + ", " + + getStyle(node) + "]\n"); + for (IMetaFile dependency: resolver.getGraph().getDependencies(node.getFilename())) { + writer.write(String.format("\t%s -> %s;\n", quote(getNodeId(resolver.getBaseDir(), + node.getMetaFile())), quote(getNodeId(resolver.getBaseDir(), dependency)))); + } + } + writer.write("}"); + } catch (IOException e) { + e.printStackTrace(); + } + } + + private static String getNodeId(File baseDir, IMetaFile metaFile) { + return baseDir.toPath().relativize(Paths.get(metaFile.getFilename())).toString(); + } + + private static String getNodeName(File baseDir, IMetaFile metaFile) { + if (metaFile.getType() == IMetaFile.MetaFileType.SNAPSHOT_FILE) { + return "Snapshot"; + } else if (metaFile.getType() == IMetaFile.MetaFileType.TEST_FILE) { + return new File(metaFile.getFilename()).getName(); + } else { + return baseDir.toPath().relativize(Paths.get(metaFile.getFilename())).toString().replace("/", "/\n"); + } + } + + private static String getStyle(DependencyGraph.Node node) { + + if (node.getMetaFile().getType() == IMetaFile.MetaFileType.SNAPSHOT_FILE) { + return "shape=" + quote("cylinder");// + ", color=" + quote("9") + ", style=" + quote("rounded,filled"); + } else if (node.getMetaFile().getType() == IMetaFile.MetaFileType.SOURCE_FILE) { + String shape = "box3d"; + String color = ""; + if (node.hasDependencyOfType(IMetaFile.MetaFileType.TEST_FILE)) { + color = "3"; + } else { + color = "1"; + } + return "shape=" + quote(shape) + ", fillcolor=" + quote(color) + ", style=" + quote("rounded,filled"); + } else { + return "fillcolor=" + quote("2") + ", style=" + quote("rounded,filled"); + } + } + + private static String quote(String text) { + return "\"" + text + "\""; + } + +} diff --git a/src/main/java/com/askimed/nf/test/lang/dependencies/DependencyGraph.java b/src/main/java/com/askimed/nf/test/lang/dependencies/DependencyGraph.java new file mode 100644 index 00000000..0c8ddfe6 --- /dev/null +++ b/src/main/java/com/askimed/nf/test/lang/dependencies/DependencyGraph.java @@ -0,0 +1,116 @@ +package com.askimed.nf.test.lang.dependencies; + +import java.util.*; + +public class DependencyGraph { + + private Map nodes; + + public DependencyGraph() { + nodes = new HashMap(); + } + + public static class Node { + private String filename; + private IMetaFile metaFile; + private Set dependencies; + + public Node(String filename, IMetaFile metaFile) { + this.filename = filename; + this.metaFile = metaFile; + this.dependencies = new HashSet<>(); + } + + public void addDependency(Node node) { + dependencies.add(node); + } + + public Set getDependencies() { + return dependencies; + } + + public String getFilename() { + return filename; + } + + public IMetaFile getMetaFile() { + return metaFile; + } + + public boolean hasDependencyOfType(IMetaFile.MetaFileType type) { + for (Node node: getDependencies()) { + if (node.getMetaFile().getType() == type) { + return true; + } + } + return false; + } + } + + public void addFile(IMetaFile metaFile) { + String filename = metaFile.getFilename(); + if (!nodes.containsKey(filename)) { + nodes.put(filename, new Node(filename, metaFile)); + } + } + + public void connectDependencies(){ + for (Node node: nodes.values()) { + for (String dependency: node.getMetaFile().getDependencies()) { + //addDependency(dependency, node.getFilename()); + addDependency(dependency, node.getFilename()); + } + } + } + + public void addDependency(String sourceFilename, String targetFilename) { + Node sourceNode = nodes.get(sourceFilename); + Node targetNode = nodes.get(targetFilename); + + if (sourceNode != null && targetNode != null) { + sourceNode.addDependency(targetNode); + } + } + + public IMetaFile getFile(String filename) { + Node node = nodes.get(filename); + if (node != null) { + return node.getMetaFile(); + } + return null; + } + + public Node getNode(String filename) { + return nodes.get(filename); + } + + + public List getFiles() { + List files = new Vector(); + for (Node node: nodes.values()){ + files.add(node.getMetaFile()); + } + return files; + } + + public List getDependencies(String filename) { + List dependencies = new Vector(); + Node node = nodes.get(filename); + if (node != null) { + for (Node dependency : node.getDependencies()) { + dependencies.add(dependency.getMetaFile()); + } + } + return dependencies; + } + + public int size() { + return nodes.size(); + } + + public Collection getNodes() { + return nodes.values(); + } + + +} diff --git a/src/main/java/com/askimed/nf/test/lang/dependencies/DependencyResolver.java b/src/main/java/com/askimed/nf/test/lang/dependencies/DependencyResolver.java new file mode 100644 index 00000000..b84c7149 --- /dev/null +++ b/src/main/java/com/askimed/nf/test/lang/dependencies/DependencyResolver.java @@ -0,0 +1,271 @@ +package com.askimed.nf.test.lang.dependencies; + +import com.askimed.nf.test.nextflow.NextflowScript; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.*; +import java.util.*; +import java.util.function.Consumer; + +public class DependencyResolver { + + private File baseDir; + + private DependencyGraph graph = new DependencyGraph(); + + private List triggerPatterns = new Vector(); + + private boolean followingDependencies = false; + + private static Logger log = LoggerFactory.getLogger(DependencyResolver.class); + + public DependencyResolver(File baseDir) { + this.baseDir = baseDir; + } + + public File getBaseDir() { + return baseDir; + } + + public DependencyGraph getGraph() { + return graph; + } + + public void setFollowingDependencies(boolean followingDependencies) { + this.followingDependencies = followingDependencies; + } + + public List findAllTests() throws Exception { + + List results = new Vector(); + for (IMetaFile metaFile: graph.getFiles()) { + if (metaFile.getType() == IMetaFile.MetaFileType.TEST_FILE) { + results.add(new File(metaFile.getFilename())); + } + } + + log.info("Found {} tests.", results.size()); + + return results; + } + + public List findTestsByFiles(List files) throws Exception { + + List patterns = new Vector(); + for (File file: files) { + patterns.add(fileToPathMatcher(file)); + } + + List results = new Vector(); + for (IMetaFile metaFile: graph.getFiles()) { + if (metaFile.getType() == IMetaFile.MetaFileType.TEST_FILE) { + File file = new File(metaFile.getFilename()); + TestFilePattern matchedPattern = matches(file.toPath(), patterns); + if (matchedPattern != null) { + if (matchedPattern.hasTestId()) { + results.add(new File(file + "@" + matchedPattern.getTestId())); + } else { + results.add(file); + } + } + } + } + + log.info("Found {} tests.", results.size()); + log.debug("Found tests: " + results); + + return results; + } + + + public List findRelatedTestsByFiles(List files) throws Exception { + File[] array = new File[files.size()]; + for (int i = 0 ; i < array.length; i++) { + array[i] = new File(files.get(i).getAbsolutePath()); + } + return new Vector(findRelatedTestsByFiles(array)); + } + + public List findRelatedTestsByFiles(File ... files) throws Exception { + + for (File file: files) { + if (matches2(file.getAbsoluteFile().toPath(), triggerPatterns)) { + log.info("File " + file.getAbsolutePath() + " triggers full test run."); + return findAllTests(); + } + } + + Set results = new HashSet(); + + long time0 = System.currentTimeMillis(); + for (File file: files) { + results.addAll(findRelatedTestsByFile(file.getAbsoluteFile(), followingDependencies)); + } + long time1 = System.currentTimeMillis(); + log.info("Found {} tests for file {} in {} sec", results.size(), files, (time1 - time0) / 1000.0); + + return new Vector(results); + } + + private Set findRelatedTestsByFile(File file, boolean followingDependencies) throws Exception { + + Set results = new HashSet(); + + IMetaFile metaFile = graph.getFile(file.toPath().normalize().toFile().getAbsolutePath()); + + //not in graph -> no tests available + if (metaFile == null) { + return results; + } + + // the file is a test file + if (metaFile.getType() == IMetaFile.MetaFileType.TEST_FILE){ + results.add(new File(metaFile.getFilename())); + return results; + } + + // the file is a source file + List dependencies = graph.getDependencies(metaFile.getFilename()); + for (IMetaFile dependency: dependencies) { + File dependencyFile = new File(dependency.getFilename()); + + if (dependency.getType() == IMetaFile.MetaFileType.TEST_FILE) { + // is a test file --> return + results.add(dependencyFile); + } else { + // if a source file + DependencyGraph.Node node = graph.getNode(dependency.getFilename()); + //TODO: add && !followingDependencies + if (node.hasDependencyOfType(IMetaFile.MetaFileType.TEST_FILE) && !followingDependencies) { + //has a test --> add all test and then stop + for (DependencyGraph.Node dependencyOfDependency: node.getDependencies()) { + if (dependencyOfDependency.getMetaFile().getType() == IMetaFile.MetaFileType.TEST_FILE) { + results.add(new File(dependencyOfDependency.getFilename())); + } + } + } else { + //has no tests --> find related tests in a recursive way + results.addAll(findRelatedTestsByFile(dependencyFile, followingDependencies)); + } + } + } + + return results; + } + + + public void buildGraph() throws Exception { + buildGraph(new Vector(), new Vector()); + } + + public void buildGraph(String ... ignoreGlobs) throws Exception { + List list = new Vector<>(); + Collections.addAll(list, ignoreGlobs); + buildGraph(list, new Vector<>()); + } + + public void buildGraph(List ignoreGlobs, List triggerPatterns) throws Exception { + + for (String glob: triggerPatterns) { + this.triggerPatterns.add(pathMatcher("glob:" + baseDir.getAbsolutePath() + "/" + glob)); + } + + List ignorePatterns = new Vector(); + ignorePatterns.add(fileToPathMatcher(".nf-test/**")); + ignorePatterns.add(fileToPathMatcher("src/**")); + ignorePatterns.add(fileToPathMatcher("target/**")); + for (String ignoreGlob: ignoreGlobs) { + ignorePatterns.add(fileToPathMatcher(ignoreGlob)); + } + + if (!baseDir.exists()) { + throw new Exception("Test directory '" + baseDir.getAbsolutePath() + "' not found."); + } + + if (!baseDir.isDirectory()) { + throw new IOException("Root '" + baseDir.getAbsolutePath() + "' is not a directory"); + } + + long time0 = System.currentTimeMillis(); + + Files.walk(Paths.get(baseDir.getAbsolutePath())).forEach(new Consumer() { + + @Override + public void accept(Path path) { + + if (matches(path, ignorePatterns) != null) { + //log.warn("Ignored file " + path); + return; + } + + if (!Files.isRegularFile(path)) { + return; + } + + IMetaFile file = null; + if (TestFile.accepts(path)) { + file = new TestFile(baseDir, path.toFile()); + } else if (NextflowScript.accepts(path)) { + file = new NextflowScript(path.toFile()); + } else if (SnapshotFile.accepts(path)) { + file = new SnapshotFile(path.toFile()); + } + if (file != null) { + try { + file.parseDependencies(); + graph.addFile(file); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + }); + + graph.connectDependencies(); + + long time1 = System.currentTimeMillis(); + + log.info("Loaded {} files from directory {} in {} sec", graph.size(), baseDir.getAbsolutePath(), (time1 - time0) / 1000.0); + + } + + public TestFilePattern fileToPathMatcher(String glob) { + return new TestFilePattern(baseDir, glob); + } + + public TestFilePattern fileToPathMatcher(File file) { + return new TestFilePattern(file); + } + + + public PathMatcher pathMatcher(String pattern) { + System.out.println(pattern); + return FileSystems.getDefault().getPathMatcher(pattern); + } + + public TestFilePattern matches(Path path, List ignorePatterns) { + PathMatcher pathMatcher; + for (TestFilePattern pattern : ignorePatterns) { + if (pattern.matches(path)) { + return pattern; + } + } + return null; + } + + public boolean matches2(Path path, List ignorePatterns) { + PathMatcher pathMatcher; + for (PathMatcher pattern : ignorePatterns) { + if (pattern.matches(path)) { + return true; + } + } + return false; + } + + +} diff --git a/src/main/java/com/askimed/nf/test/lang/dependencies/IMetaFile.java b/src/main/java/com/askimed/nf/test/lang/dependencies/IMetaFile.java new file mode 100644 index 00000000..b124a8ae --- /dev/null +++ b/src/main/java/com/askimed/nf/test/lang/dependencies/IMetaFile.java @@ -0,0 +1,20 @@ +package com.askimed.nf.test.lang.dependencies; + +import java.io.IOException; +import java.util.Set; + +public interface IMetaFile { + + public Set getDependencies(); + + public MetaFileType getType(); + + public String getFilename(); + + public void parseDependencies() throws IOException; + + public static enum MetaFileType{ + SOURCE_FILE, TEST_FILE, SNAPSHOT_FILE + } + +} diff --git a/src/main/java/com/askimed/nf/test/lang/dependencies/SnapshotFile.java b/src/main/java/com/askimed/nf/test/lang/dependencies/SnapshotFile.java new file mode 100644 index 00000000..dd4251a8 --- /dev/null +++ b/src/main/java/com/askimed/nf/test/lang/dependencies/SnapshotFile.java @@ -0,0 +1,48 @@ +package com.askimed.nf.test.lang.dependencies; + +import com.askimed.nf.test.util.FileUtil; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashSet; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class SnapshotFile implements IMetaFile { + + private File file; + + private Set dependencies = new HashSet(); + + public SnapshotFile(File file) { + this.file = file; + } + + public void parseDependencies() throws IOException { + + + } + + public static boolean accepts(Path path) { + return path.getFileName().toString().endsWith(".nf.test.snap") || path.getFileName().toString().endsWith(".groovy.test.snap"); + } + + @Override + public String getFilename() { + return file.getAbsolutePath(); + } + + @Override + public MetaFileType getType() { + return MetaFileType.SNAPSHOT_FILE; + } + + @Override + public Set getDependencies() { + return dependencies; + } + +} diff --git a/src/main/java/com/askimed/nf/test/lang/dependencies/TestFile.java b/src/main/java/com/askimed/nf/test/lang/dependencies/TestFile.java new file mode 100644 index 00000000..e8c5d26e --- /dev/null +++ b/src/main/java/com/askimed/nf/test/lang/dependencies/TestFile.java @@ -0,0 +1,76 @@ +package com.askimed.nf.test.lang.dependencies; + +import com.askimed.nf.test.util.FileUtil; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashSet; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class TestFile implements IMetaFile { + + private File baseDir; + + private File file; + + private Set dependencies = new HashSet(); + + public TestFile(File baseDir, File file) { + this.baseDir = baseDir; + this.file = file; + } + + public void parseDependencies() throws IOException { + String script = FileUtil.readFileAsString(file); + String regex = "(?i)script\\s+\"(.+)\""; + + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(script); + + File snapshot = new File(file.getAbsolutePath() + ".snap"); + if (snapshot.exists()){ + dependencies.add(snapshot.getAbsolutePath()); + } + + + while (matcher.find()) { + String dependency = matcher.group(1).trim(); + Path path = null; + if (dependency.startsWith("./") || dependency.startsWith("../")) { + path = Paths.get(file.getAbsoluteFile().getParentFile().getAbsolutePath()).resolve(dependency); + } else { + path = new File(baseDir, dependency).toPath(); + } + if (!path.toFile().exists()){ + System.out.println("Warning: Module " + file.getAbsolutePath() + ": Dependency '" + path.toAbsolutePath() + "' not found." ); + continue; + } + dependencies.add(path.normalize().toFile().getAbsolutePath()); + } + + } + + public static boolean accepts(Path path) { + return path.getFileName().toString().endsWith(".nf.test") || path.getFileName().toString().endsWith(".groovy.test"); + } + + @Override + public String getFilename() { + return file.getAbsolutePath(); + } + + @Override + public MetaFileType getType() { + return MetaFileType.TEST_FILE; + } + + @Override + public Set getDependencies() { + return dependencies; + } + +} diff --git a/src/main/java/com/askimed/nf/test/lang/dependencies/TestFilePattern.java b/src/main/java/com/askimed/nf/test/lang/dependencies/TestFilePattern.java new file mode 100644 index 00000000..45de6568 --- /dev/null +++ b/src/main/java/com/askimed/nf/test/lang/dependencies/TestFilePattern.java @@ -0,0 +1,65 @@ +package com.askimed.nf.test.lang.dependencies; + +import java.io.File; +import java.nio.file.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class TestFilePattern { + + private PathMatcher pathMatcher; + + private String testId = null; + + public TestFilePattern (File baseDir, String glob) { + pathMatcher = pathMatcher("glob:" + baseDir.getAbsolutePath() + "/" + glob); + } + + public TestFilePattern (File file) { + String path = file.toPath().toAbsolutePath().normalize().toString(); + testId = extractTestId(path); + if (testId != null) { + path = removeTestId(path); + } + if (file.isDirectory()) { + pathMatcher = pathMatcher("glob:" + path + "/**"); + } else { + pathMatcher = pathMatcher("glob:" + path); + } + } + + public PathMatcher pathMatcher(String pattern) { + return FileSystems.getDefault().getPathMatcher(pattern); + } + + public String removeTestId(String string) { + String regex = "@[a-fA-F0-9]{8}\\b"; + return string.replaceAll(regex, ""); + } + + private String extractTestId(String path) { + String regex = "@[a-fA-F0-9]{8}\\b"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(path); + + if (matcher.find()) { + // Extract hash without the "@" character + return matcher.group().substring(1); + } else { + return null; + } + } + + public boolean matches(Path path) { + return pathMatcher.matches(path); + } + + public String getTestId() { + return testId; + } + + public boolean hasTestId() { + return testId != null; + } + + } diff --git a/src/main/java/com/askimed/nf/test/lang/extensions/Snapshot.java b/src/main/java/com/askimed/nf/test/lang/extensions/Snapshot.java index 6370ce9b..2dcae783 100644 --- a/src/main/java/com/askimed/nf/test/lang/extensions/Snapshot.java +++ b/src/main/java/com/askimed/nf/test/lang/extensions/Snapshot.java @@ -43,6 +43,9 @@ public boolean match(String id) throws IOException { SnapshotFileItem expected = file.getSnapshot(id); // new snapshot --> create snapshot if (expected == null) { + if (test.isCIMode()) { + throw new RuntimeException("CI mode activated and snapshot with id '" + id + "' not found."); + } log.debug("Snapshot '{}' not found.", id); file.createSnapshot(id, actual); file.save(); diff --git a/src/main/java/com/askimed/nf/test/lang/extensions/SnapshotFile.java b/src/main/java/com/askimed/nf/test/lang/extensions/SnapshotFile.java index 152b7e7b..a8d8a8eb 100644 --- a/src/main/java/com/askimed/nf/test/lang/extensions/SnapshotFile.java +++ b/src/main/java/com/askimed/nf/test/lang/extensions/SnapshotFile.java @@ -20,6 +20,8 @@ public class SnapshotFile { + public static final String EXTENSION = ".snap"; + private String filename; private Map snapshots = new HashMap(); @@ -137,7 +139,7 @@ public void save() throws IOException { } protected static String createFilename(ITestSuite suite) { - return suite.getFilename() + ".snap"; + return suite.getFilename() + EXTENSION; } public static JsonGenerator createJsonGenerator() { diff --git a/src/main/java/com/askimed/nf/test/nextflow/NextflowCommand.java b/src/main/java/com/askimed/nf/test/nextflow/NextflowCommand.java index 40bd63e1..6461e378 100644 --- a/src/main/java/com/askimed/nf/test/nextflow/NextflowCommand.java +++ b/src/main/java/com/askimed/nf/test/nextflow/NextflowCommand.java @@ -322,7 +322,7 @@ public String parseVersion() throws IOException { nextflow.setParams(args); nextflow.setSilent(true); StringBuffer output = new StringBuffer(); - nextflow.writeStderr(output); + nextflow.writeStdout(output); nextflow.execute(); String versionPattern = "version (\\d+\\.\\d+\\.\\d+)"; Pattern pattern = Pattern.compile(versionPattern); diff --git a/src/main/java/com/askimed/nf/test/nextflow/NextflowScript.java b/src/main/java/com/askimed/nf/test/nextflow/NextflowScript.java index 525bf67a..5289cc9e 100644 --- a/src/main/java/com/askimed/nf/test/nextflow/NextflowScript.java +++ b/src/main/java/com/askimed/nf/test/nextflow/NextflowScript.java @@ -2,14 +2,21 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.Vector; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.askimed.nf.test.lang.dependencies.IMetaFile; import com.askimed.nf.test.util.FileUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class NextflowScript { +public class NextflowScript implements IMetaFile { private File file; @@ -21,6 +28,10 @@ public class NextflowScript { private List functions = new Vector(); + private Set dependencies = new HashSet(); + + private static Logger log = LoggerFactory.getLogger(NextflowScript.class); + public NextflowScript(File file) { this.file = file; } @@ -32,6 +43,17 @@ public void load() throws IOException { functions = getFunctionNames(script); } + + public void parseDependencies() throws IOException { + String script = FileUtil.readFileAsString(file); + dependencies = getDependencies(file, script); + } + + @Override + public String getFilename() { + return file.getAbsolutePath(); + } + public boolean isDsl2() { return dsl2; } @@ -98,4 +120,59 @@ public static List getWorkflowNames(String content) { return names; } + public Set getDependencies() { + return dependencies; + } + + @Override + public MetaFileType getType() { + return MetaFileType.SOURCE_FILE; + } + + public static Set getDependencies(File file, String content) { + + Set dependencies = new HashSet(); + + String regex = "(?i)include\\s*\\{\\s*([A-Z_1-9]+(?:\\s+as\\s+[A-Z_]+)?)\\s*\\}\\s*from\\s*['\"](.+?)['\"]"; + + Pattern pattern = Pattern.compile(regex, Pattern.MULTILINE); + Matcher matcher = pattern.matcher(content); + while (matcher.find()) { + String dependency = matcher.group(2).trim(); + if (dependency.startsWith("/") || dependency.startsWith("plugin/")) { + continue; + } + + Path path = resolve(file, dependency); + if (path.toFile().isDirectory()) { + path = resolve(file, dependency + "/main.nf"); + } + + if (!path.toFile().exists()){ + path = resolve(file, dependency + ".nf"); + } + + if (!path.toFile().exists()){ + log.warn("Module " + file.getAbsolutePath() + ": Dependency '" + path.toAbsolutePath() + "' not found." ); + continue; + } + dependencies.add(path.normalize().toFile().getAbsolutePath()); + } + + return dependencies; + + } + + protected static Path resolve(File file, String dependency) { + if (dependency.startsWith("./") || dependency.startsWith("../")) { + return Paths.get(file.getParentFile().getAbsolutePath()).resolve(dependency); + } else { + return Paths.get(dependency); + } + } + + public static boolean accepts(Path path) { + return path.getFileName().toString().endsWith(".nf"); + } + } diff --git a/src/main/java/com/askimed/nf/test/util/AnsiText.java b/src/main/java/com/askimed/nf/test/util/AnsiText.java index f3f55c37..644844cd 100644 --- a/src/main/java/com/askimed/nf/test/util/AnsiText.java +++ b/src/main/java/com/askimed/nf/test/util/AnsiText.java @@ -1,5 +1,7 @@ package com.askimed.nf.test.util; +import java.util.List; + public class AnsiText { public static boolean active = true; @@ -41,4 +43,10 @@ public static String padding(String string, int count) { return result; } + public static void printBulletList(List list) { + for (Object item : list) { + System.out.println(" \u2022 " + item); + } + } + } diff --git a/src/main/java/com/askimed/nf/test/util/Command.java b/src/main/java/com/askimed/nf/test/util/Command.java index 3e8faecb..9b33c274 100644 --- a/src/main/java/com/askimed/nf/test/util/Command.java +++ b/src/main/java/com/askimed/nf/test/util/Command.java @@ -87,7 +87,7 @@ public int execute() { Thread inputStreamHandler = new Thread(handler); CommandStreamHandler handler2 = new CommandStreamHandler(process.getErrorStream(), stderrFileName); - handler.setStringBuffer(stderr); + handler2.setStringBuffer(stderr); handler2.setSilent(silent); Thread errorStreamHandler = new Thread(handler2); diff --git a/src/main/java/com/askimed/nf/test/util/GitCommand.java b/src/main/java/com/askimed/nf/test/util/GitCommand.java new file mode 100644 index 00000000..c393ae44 --- /dev/null +++ b/src/main/java/com/askimed/nf/test/util/GitCommand.java @@ -0,0 +1,132 @@ +package com.askimed.nf.test.util; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Vector; + +public class GitCommand { + + private String binary; + + private boolean silent = true; + + public static String ERROR = "Git binary not found. Please check if git is in a directory accessible by your $PATH variable or set $GIT_HOME."; + + public GitCommand() { + binary = new BinaryFinder("git").env("GIT_HOME").envPath().path("/usr/local/bin").find(); + } + + public boolean isInstalled() { + return binary != null; + } + + public void setSilent(boolean silent) { + this.silent = silent; + } + + private List execute(String directory, List args) throws IOException { + + if (!isInstalled()){ + throw new IOException("Could not find git binary."); + } + + Command git = new Command(binary); + git.setDirectory(directory); + git.setParams(args); + git.setSilent(silent); + StringBuffer output = new StringBuffer(); + git.writeStdout(output); + StringBuffer error = new StringBuffer(); + git.writeStderr(error); + + int result = git.execute(); + + if (result != 0){ + //TODO: log output. + throw new IOException("Error running git command." + error); + } + + return parseFilenames(output); + } + + public List findChangesSinceLastCommit(String directory) throws IOException { + List args = new Vector(); + args.add("show"); + args.add("--name-only"); + args.add("--pretty=format:"); + args.add("HEAD"); + args.add("--"); + return execute(directory, args); + } + + public List findChangesWithAncestor(File directory) throws IOException { + return findChangesWithAncestor(directory.getAbsolutePath()); + } + + public List findChangesWithAncestor(String directory) throws IOException { + return findChangesSince(directory, "HEAD^"); + } + + public List findChangesSince(File directory, String commit) throws IOException { + return findChangesBetween(directory.getAbsolutePath(), commit, "HEAD"); + } + public List findChangesSince(String directory, String commit) throws IOException { + return findChangesBetween(directory, commit, "HEAD"); + } + + public List findChangesBetween(File directory, String from, String to) throws IOException { + return findChangesBetween(directory.getAbsolutePath(), from, to); + } + + public List findChangesBetween(String directory, String from, String to) throws IOException { + List args = new Vector(); + args.add("diff"); + args.add("--name-only"); + args.add(from + "..." + to); + args.add("--"); + + //TODO: jest adds also findChanges to list. + + return execute(directory, args); + } + + public List findChanges(File directory) throws IOException { + return findChanges(directory.getAbsolutePath()); + } + + public List findChanges(String directory) throws IOException { + //staged + List args = new Vector(); + args.add("diff"); + args.add("--cached"); + args.add("--name-only"); + args.add("--"); + List staged = execute(directory, args); + + //unstaged + args = new Vector(); + args.add("ls-files"); + args.add("--other"); + args.add("--modified"); + args.add("--exclude-standard"); + args.add("--"); + List unstaged = execute(directory, args); + + staged.addAll(unstaged); + return staged; + } + + public static List parseFilenames(StringBuffer stringBuffer) { + List files = new ArrayList(); + String[] lines = stringBuffer.toString().split("\\r?\\n"); + for (String line : lines) { + if (!line.trim().isEmpty()) { + files.add(new File(line.trim())); + } + } + return files; + } + +} \ No newline at end of file diff --git a/src/test/java/com/askimed/nf/test/core/TestExecutionEngineTest.java b/src/test/java/com/askimed/nf/test/core/TestExecutionEngineTest.java deleted file mode 100644 index eb30d31b..00000000 --- a/src/test/java/com/askimed/nf/test/core/TestExecutionEngineTest.java +++ /dev/null @@ -1,123 +0,0 @@ -package com.askimed.nf.test.core; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.util.List; -import java.util.Vector; - -import org.junit.jupiter.api.Test; - -public class TestExecutionEngineTest { - - @Test - public void executeAllTests() throws Throwable { - - TagQuery query = new TagQuery(); - List tests = collectTests(query); - assertEquals(3, tests.size()); - assertTrue(tests.contains("test 1")); - assertTrue(tests.contains("test 2")); - assertTrue(tests.contains("test 3")); - - } - - @Test - public void executeTestByName() throws Throwable { - - TagQuery query = new TagQuery("test 1"); - List tests = collectTests(query); - assertEquals(1, tests.size()); - assertTrue(tests.contains("test 1")); - - } - - @Test - public void executeTestSuiteByName() throws Throwable { - { - TagQuery query = new TagQuery("suite 1"); - List tests = collectTests(query); - assertEquals(2, tests.size()); - assertTrue(tests.contains("test 1")); - assertTrue(tests.contains("test 2")); - } - - { - TagQuery query = new TagQuery("SUITE 1"); - List tests = collectTests(query); - assertEquals(2, tests.size()); - assertTrue(tests.contains("test 1")); - assertTrue(tests.contains("test 2")); - } - } - - @Test - public void executeTestsByTag() throws Throwable { - { - TagQuery query = new TagQuery("tag2"); - List tests = collectTests(query); - assertEquals(1, tests.size()); - assertTrue(tests.contains("test 1")); - } - { - TagQuery query = new TagQuery("TAG2"); - List tests = collectTests(query); - assertEquals(1, tests.size()); - assertTrue(tests.contains("test 1")); - } - } - - @Test - public void executeTestsByTagAcrossSuites() throws Throwable { - - TagQuery query = new TagQuery("tag5"); - List tests = collectTests(query); - assertEquals(2, tests.size()); - assertTrue(tests.contains("test 2")); - assertTrue(tests.contains("test 3")); - } - - @Test - public void executeTestsBySuiteTag() throws Throwable { - - TagQuery query = new TagQuery("tag1"); - List tests = collectTests(query); - assertEquals(2, tests.size()); - assertTrue(tests.contains("test 1")); - assertTrue(tests.contains("test 2")); - } - - @Test - public void executeTestsByMultipleTags() throws Throwable { - - TagQuery query = new TagQuery("tag3", "tag4"); - List tests = collectTests(query); - assertEquals(2, tests.size()); - assertTrue(tests.contains("test 1")); - assertTrue(tests.contains("test 2")); - } - - protected List collectTests(TagQuery query) throws Throwable { - List scripts = new Vector(); - scripts.add(new File("test-data/suite1.nf.test")); - scripts.add(new File("test-data/suite2.nf.test")); - TestExecutionEngine engine = new TestExecutionEngine(); - engine.setScripts(scripts); - List testSuits = engine.parse(query); - return getCollectedTests(testSuits); - } - - protected List getCollectedTests(List suites) { - List tests = new Vector(); - for (ITestSuite suite : suites) { - for (ITest test : suite.getTests()) { - if (!test.isSkipped()) { - tests.add(test.getName()); - } - } - } - return tests; - } - -} diff --git a/src/test/java/com/askimed/nf/test/core/TestSuiteResolverTest.java b/src/test/java/com/askimed/nf/test/core/TestSuiteResolverTest.java new file mode 100644 index 00000000..b6c36346 --- /dev/null +++ b/src/test/java/com/askimed/nf/test/core/TestSuiteResolverTest.java @@ -0,0 +1,151 @@ +package com.askimed.nf.test.core; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.util.List; +import java.util.Vector; + +import com.askimed.nf.test.lang.TestSuiteBuilder; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestSuiteResolverTest { + + @Test + public void executeAllTests() throws Throwable { + + TagQuery query = new TagQuery(); + List tests = collectTests(query); + Assertions.assertEquals(3, tests.size()); + Assertions.assertTrue(tests.contains("test 1")); + Assertions.assertTrue(tests.contains("test 2")); + Assertions.assertTrue(tests.contains("test 3")); + + } + + @Test + public void executeTestByName() throws Throwable { + + TagQuery query = new TagQuery("test 1"); + List tests = collectTests(query); + Assertions.assertEquals(1, tests.size()); + Assertions.assertTrue(tests.contains("test 1")); + + } + + @Test + public void executeTestByWrongHash() throws Throwable { + List scripts = new Vector(); + scripts.add(new File("test-data/suite1.nf.test@aaa")); + scripts.add(new File("test-data/suite2.nf.test")); + TestSuiteResolver resolver = new TestSuiteResolver(new Environment()); + List testSuits = resolver.parse(scripts); + List tests = getCollectedTests(testSuits); + Assertions.assertEquals(1, tests.size()); + Assertions.assertTrue(tests.contains("test 3")); + } + + @Test + public void executeTestByHash() throws Throwable { + ITestSuite testsuite = TestSuiteBuilder.parse(new File("test-data/suite1.nf.test")); + String hash = testsuite.getTests().get(0).getHash(); + List scripts = new Vector(); + scripts.add(new File("test-data/suite1.nf.test@" + hash)); + scripts.add(new File("test-data/suite2.nf.test")); + TestSuiteResolver resolver = new TestSuiteResolver(new Environment()); + List testSuits = resolver.parse(scripts); + List tests = getCollectedTests(testSuits); + Assertions.assertEquals(2, tests.size()); + Assertions.assertTrue(tests.contains("test 1")); + Assertions.assertTrue(tests.contains("test 3")); + } + + @Test + public void executeTestSuiteByName() throws Throwable { + { + TagQuery query = new TagQuery("suite 1"); + List tests = collectTests(query); + Assertions.assertEquals(2, tests.size()); + Assertions.assertTrue(tests.contains("test 1")); + Assertions.assertTrue(tests.contains("test 2")); + } + + { + TagQuery query = new TagQuery("SUITE 1"); + List tests = collectTests(query); + Assertions.assertEquals(2, tests.size()); + Assertions.assertTrue(tests.contains("test 1")); + Assertions.assertTrue(tests.contains("test 2")); + } + } + + @Test + public void executeTestsByTag() throws Throwable { + { + TagQuery query = new TagQuery("tag2"); + List tests = collectTests(query); + Assertions.assertEquals(1, tests.size()); + Assertions.assertTrue(tests.contains("test 1")); + } + { + TagQuery query = new TagQuery("TAG2"); + List tests = collectTests(query); + Assertions.assertEquals(1, tests.size()); + Assertions.assertTrue(tests.contains("test 1")); + } + } + + @Test + public void executeTestsByTagAcrossSuites() throws Throwable { + + TagQuery query = new TagQuery("tag5"); + List tests = collectTests(query); + Assertions.assertEquals(2, tests.size()); + Assertions.assertTrue(tests.contains("test 2")); + Assertions.assertTrue(tests.contains("test 3")); + } + + @Test + public void executeTestsBySuiteTag() throws Throwable { + + TagQuery query = new TagQuery("tag1"); + List tests = collectTests(query); + Assertions.assertEquals(2, tests.size()); + Assertions.assertTrue(tests.contains("test 1")); + Assertions.assertTrue(tests.contains("test 2")); + } + + @Test + public void executeTestsByMultipleTags() throws Throwable { + + TagQuery query = new TagQuery("tag3", "tag4"); + List tests = collectTests(query); + Assertions.assertEquals(2, tests.size()); + Assertions.assertTrue(tests.contains("test 1")); + Assertions.assertTrue(tests.contains("test 2")); + } + + protected List collectTests(TagQuery query) throws Throwable { + List scripts = new Vector(); + scripts.add(new File("test-data/suite1.nf.test")); + scripts.add(new File("test-data/suite2.nf.test")); + TestSuiteResolver resolver = new TestSuiteResolver(new Environment()); + List testSuits = resolver.parse(scripts, query); + return getCollectedTests(testSuits); + } + + protected List getCollectedTests(List suites) { + List tests = new Vector(); + for (ITestSuite suite : suites) { + for (ITest test : suite.getTests()) { + if (!test.isSkipped()) { + tests.add(test.getName()); + } + } + } + return tests; + } + +} diff --git a/src/test/java/com/askimed/nf/test/lang/ProcessTest.java b/src/test/java/com/askimed/nf/test/lang/ProcessTest.java index 91b2783b..a4f6f47d 100644 --- a/src/test/java/com/askimed/nf/test/lang/ProcessTest.java +++ b/src/test/java/com/askimed/nf/test/lang/ProcessTest.java @@ -1,7 +1,5 @@ package com.askimed.nf.test.lang; -import static org.junit.jupiter.api.Assertions.assertEquals; - import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -15,6 +13,8 @@ import com.askimed.nf.test.util.AnsiText; import com.askimed.nf.test.util.FileUtil; +import static org.junit.jupiter.api.Assertions.*; + public class ProcessTest { static { @@ -103,7 +103,7 @@ public void testMissingScript() throws Exception { App app = new App(); int exitCode = app.run(new String[] { "test", "test-data/process/default/wrong-filename.nf.test" }); - assertEquals(1, exitCode); + assertEquals(2, exitCode); } @@ -318,7 +318,7 @@ public void testUniquenessSnapshots() throws Exception { public void testNotUniquenessOfSnapshots() throws Exception { App app = new App(); - int exitCode = app.run(new String[] { "test", "test-data/process/snapshots/not-unique.nf.test" }); + int exitCode = app.run(new String[] { "test", "test-data/process/snapshots/no-unique.nf.test" }); assertEquals(1, exitCode); } @@ -332,4 +332,31 @@ public void testMd5Snapshots() throws Exception { } + @Test + public void testSnapshotsInCiMode() throws Exception { + + File snapshot = new File("test-data/process/snapshots/ci-mode.nf.test.snap"); + + snapshot.delete(); + + assertFalse(snapshot.exists()); + + App app = new App(); + int exitCode = app.run(new String[] { "test", "test-data/process/snapshots/ci-mode.nf.test" }); + assertEquals(0, exitCode); + + assertTrue(snapshot.exists()); + + snapshot.delete(); + + assertFalse(snapshot.exists()); + + app = new App(); + exitCode = app.run(new String[] { "test", "test-data/process/snapshots/ci-mode.nf.test", "--ci" }); + assertEquals(1, exitCode); + + assertFalse(snapshot.exists()); + + } + } diff --git a/src/test/java/com/askimed/nf/test/lang/dependencies/DependencyResolverTest.java b/src/test/java/com/askimed/nf/test/lang/dependencies/DependencyResolverTest.java new file mode 100644 index 00000000..333d5bd1 --- /dev/null +++ b/src/test/java/com/askimed/nf/test/lang/dependencies/DependencyResolverTest.java @@ -0,0 +1,192 @@ +package com.askimed.nf.test.lang.dependencies; + +import org.junit.jupiter.api.Test; + +import java.io.*; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.List; +import java.util.Vector; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class DependencyResolverTest { + + public File getNfGwas() throws IOException { + String url = "https://github.com/genepi/nf-gwas/archive/refs/tags/v1.0.7.zip"; + return downloadAndExtractZip(url, "temp", "nf-gwas-1.0.7"); + } + + public File getFetchNgs() throws IOException { + String url = "https://github.com/nf-core/fetchngs/archive/refs/tags/1.12.0.zip"; + return downloadAndExtractZip(url, "temp", "fetchngs-1.12.0"); + } + + @Test + void findAllTests() throws Exception { + File root = getNfGwas(); + DependencyResolver resolver = new DependencyResolver(root); + resolver.buildGraph(); + assertEquals(10, resolver.findAllTests().size()); + + Coverage coverage = new Coverage(resolver); + assertEquals(10, coverage.getAll().getCoveredItems()); + assertEquals(39, coverage.getAll().getItems().size()); + } + + @Test + void findRelatedTests() throws Exception { + + File root = getNfGwas(); + DependencyResolver resolver = new DependencyResolver(root); + resolver.buildGraph(); + assertEquals(2, resolver.findRelatedTestsByFiles( + new File(root, "modules/../modules/local/regenie/regenie_step1_run.nf"), + new File(root, "nextflow.config") + ).size()); + + assertEquals(3, resolver.findRelatedTestsByFiles( + new File(root, "modules/../modules/local/regenie/regenie_step1_run.nf"), + new File(root, "modules/../modules/local/regenie/regenie_step2_run.nf") + ).size()); + } + + @Test + void findRelatedTestAndFollowDependencies() throws Exception { + + File root = getFetchNgs(); + + DependencyResolver resolver = new DependencyResolver(root); + resolver.setFollowingDependencies(false); + + resolver.buildGraph(); + assertEquals(10, resolver.findRelatedTestsByFiles( + new File(root, "modules/local/sra_to_samplesheet/main.nf") + ).size()); + + resolver = new DependencyResolver(root); + resolver.setFollowingDependencies(true); + + resolver.buildGraph(); + assertEquals(11, resolver.findRelatedTestsByFiles( + new File(root, "modules/local/sra_to_samplesheet/main.nf") + ).size()); + + Coverage coverage = new Coverage(resolver); + assertEquals(17, coverage.getAll().getCoveredItems()); + assertEquals(17, coverage.getAll().getItems().size()); + } + + @Test + void findRelatedTestsWithTrigger() throws Exception { + + File root = getNfGwas(); + DependencyResolver resolver = new DependencyResolver(root); + List triggers = new Vector(); + triggers.add("nextflow.config"); + resolver.buildGraph(new Vector<>(), triggers); + + assertEquals(10, resolver.findRelatedTestsByFiles( + new File(root, "modules/../modules/local/regenie/regenie_step1_run.nf"), + new File(root, "nextflow.config") + ).size()); + + } + + @Test + void findRelatedTestsWithTriggerAndPattern() throws Exception { + + File root = getNfGwas(); + DependencyResolver resolver = new DependencyResolver(root); + List triggers = new Vector(); + triggers.add("*.config"); + resolver.buildGraph(new Vector<>(), triggers); + + assertEquals(10, resolver.findRelatedTestsByFiles( + new File(root, "modules/../modules/local/regenie/regenie_step1_run.nf"), + new File(root, "nextflow.config") + ).size()); + + } + + @Test + void findRelatedTestsWithTriggerAndTestDataPattern() throws Exception { + + File root = getNfGwas(); + DependencyResolver resolver = new DependencyResolver(root); + List triggers = new Vector(); + triggers.add("tests/input/**/*"); + resolver.buildGraph(new Vector<>(), triggers); + + assertEquals(10, resolver.findRelatedTestsByFiles( + new File(root, "tests/input/filter_results/regenie_step2_example_Y1.regenie.gz") + ).size()); + + resolver = new DependencyResolver(root); + resolver.buildGraph(); + + assertEquals(0, resolver.findRelatedTestsByFiles( + new File(root, "tests/input/filter_results/regenie_step2_example_Y1.regenie.gz") + ).size()); + + } + + private File downloadAndExtractZip(String url, String destinationFolder, String name) throws IOException { + // Check if destination folder exists + File destFolder = new File(destinationFolder); + if (!new File(destFolder, name).exists()) { + // Create destination folder if it doesn't exist + destFolder.mkdirs(); + + // Download the zip file + URL zipUrl = new URL(url); + HttpURLConnection connection = (HttpURLConnection) zipUrl.openConnection(); + try (InputStream inputStream = connection.getInputStream()) { + Path zipFilePath = Paths.get(destinationFolder, "temp.zip"); + Files.copy(inputStream, zipFilePath, StandardCopyOption.REPLACE_EXISTING); + } + + // Extract the contents of the zip file + File extractedFolder = new File(destinationFolder, name); + try (ZipInputStream zipInputStream = new ZipInputStream(new FileInputStream(new File(destinationFolder, "temp.zip")))) { + ZipEntry entry = zipInputStream.getNextEntry(); + while (entry != null) { + Path filePath = Paths.get(destinationFolder, entry.getName()); + if (!entry.isDirectory()) { + // Create parent directories if they don't exist + Files.createDirectories(filePath.getParent()); + // Write file content + try (OutputStream outputStream = new FileOutputStream(filePath.toFile())) { + byte[] buffer = new byte[1024]; + int length; + while ((length = zipInputStream.read(buffer)) > 0) { + outputStream.write(buffer, 0, length); + } + } + } else { + // Create directory if it doesn't exist + Files.createDirectories(filePath); + } + zipInputStream.closeEntry(); + entry = zipInputStream.getNextEntry(); + } + } + + // Clean up: Delete the temporary zip file + Files.deleteIfExists(Paths.get(destinationFolder, "temp.zip")); + + return extractedFolder; + } else { + // If the destination folder already exists, return it without downloading + return new File(destinationFolder, name); + } + } + + +} \ No newline at end of file diff --git a/src/test/java/com/askimed/nf/test/nextflow/NextflowCommandTest.java b/src/test/java/com/askimed/nf/test/nextflow/NextflowCommandTest.java index 76767afe..658d47b2 100644 --- a/src/test/java/com/askimed/nf/test/nextflow/NextflowCommandTest.java +++ b/src/test/java/com/askimed/nf/test/nextflow/NextflowCommandTest.java @@ -1,6 +1,7 @@ package com.askimed.nf.test.nextflow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import java.util.List; @@ -21,4 +22,8 @@ public void testParseOptions() { assertEquals("--b", options.get(2)); } + @Test + void getVersion() { + assertNotEquals("unknown", NextflowCommand.getVersion()); + } } diff --git a/test-data/process/snapshots/ci-mode.nf.test b/test-data/process/snapshots/ci-mode.nf.test new file mode 100644 index 00000000..39eac999 --- /dev/null +++ b/test-data/process/snapshots/ci-mode.nf.test @@ -0,0 +1,28 @@ +nextflow_process { + + name "Test process xy" + + script "./process.nf" + process "TEST_PROCESS" + + test("Should succeed because two unique snapshots") { + + when { + process { + """ + input[0] = "Lukas" + """ + } + } + + then { + assert process.success + assert process.out.my_output_files + assert process.out.my_output_files.size() == 1 + assert snapshot(process.out.my_output_files).match() + assert snapshot(process.out.my_output_files).match("lukas") + } + + } + +} diff --git a/test-data/process/snapshots/no-unique.test b/test-data/process/snapshots/no-unique.nf.test similarity index 100% rename from test-data/process/snapshots/no-unique.test rename to test-data/process/snapshots/no-unique.nf.test From 3a39fddd60ce7d3c72cd1bd2f757359fc552c18f Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Thu, 23 May 2024 12:57:42 +0200 Subject: [PATCH 09/25] Update github-actions.md --- docs/tutorials/github-actions.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/tutorials/github-actions.md b/docs/tutorials/github-actions.md index 88151ed2..382044fb 100644 --- a/docs/tutorials/github-actions.md +++ b/docs/tutorials/github-actions.md @@ -1,5 +1,9 @@ # Setup nf-test on GitHub Actions +!!! warning + + This feature is experimental and requires nf-test 0.9.0-rc1 or higher + In this tutorial, we will guide you through setting up and running `nf-test` on GitHub Actions. We will start with a simple example where all tests run in a single job, then extend it to demonstrate how you can use sharding to distribute tests across multiple jobs for improved efficiency. Finally, we will show you how to run only the tests affected by the changed files using the `--changes-since` option. By the end of this tutorial, you will have a clear understanding of how to: @@ -174,4 +178,4 @@ This configuration ensures that critical changes always result in a comprehensiv 2. **With Sharding**: Distributes tests across multiple jobs, allowing them to run in parallel. 3. **With Sharding and Changed Files**: Optimizes the CI process by running only the tests affected by the changed files since the last commit, in parallel jobs. -Choose the configuration that best suits your project's needs. Start with the simpler setup and extend it as needed to improve efficiency and reduce test execution time. \ No newline at end of file +Choose the configuration that best suits your project's needs. Start with the simpler setup and extend it as needed to improve efficiency and reduce test execution time. From be7562a78ae20dc16ce7df046dbc725a7231ee2d Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Thu, 23 May 2024 13:05:31 +0200 Subject: [PATCH 10/25] Prepare release 0.9.0-rc1 --- pom.xml | 2 +- src/main/java/com/askimed/nf/test/App.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 73cbd4bf..a8696813 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.askimed nf-test - 0.8.4 + 0.9.0-rc1 nf-test Simple test framework for Nextflow pipelines https://github.com/askimed/nf-test diff --git a/src/main/java/com/askimed/nf/test/App.java b/src/main/java/com/askimed/nf/test/App.java index d9f3e8cf..57317004 100644 --- a/src/main/java/com/askimed/nf/test/App.java +++ b/src/main/java/com/askimed/nf/test/App.java @@ -17,7 +17,7 @@ public class App { public static final String NAME = "nf-test"; - public static final String VERSION = "0.8.4"; + public static final String VERSION = "0.9.0-rc1"; public static final String PACKAGE = "com.askimed.nf.test"; From f837d09eaf05ea3b42e64da9a512e73dc08a607d Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Fri, 24 May 2024 06:58:07 +0200 Subject: [PATCH 11/25] Remove sysout --- .../askimed/nf/test/lang/dependencies/DependencyResolver.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/askimed/nf/test/lang/dependencies/DependencyResolver.java b/src/main/java/com/askimed/nf/test/lang/dependencies/DependencyResolver.java index b84c7149..81233ef7 100644 --- a/src/main/java/com/askimed/nf/test/lang/dependencies/DependencyResolver.java +++ b/src/main/java/com/askimed/nf/test/lang/dependencies/DependencyResolver.java @@ -243,7 +243,6 @@ public TestFilePattern fileToPathMatcher(File file) { public PathMatcher pathMatcher(String pattern) { - System.out.println(pattern); return FileSystems.getDefault().getPathMatcher(pattern); } From 01eb3da0507f1c96499116663ed70effff380554 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Fri, 24 May 2024 13:05:05 +0200 Subject: [PATCH 12/25] Update return code to avoid failing with empty shards --- .../java/com/askimed/nf/test/commands/RunTestsCommand.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java b/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java index 07fe6274..b27ff4a8 100644 --- a/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java +++ b/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java @@ -249,9 +249,9 @@ public Integer execute() throws Exception { if (scripts.isEmpty()) { System.out.println(AnsiColors - .red("Error: No tests or test directories containing scripts that end with *.test provided.")); - log.error("No tests ot directories found containing test files."); - return 2; + .yellow("No tests to execute.")); + log.warn("No tests or directories found containing test files. Or all testcases were filtered."); + return 0; } else { log.info("Detected {} test files.", scripts.size()); } From a7032d0249512c5e6c2d31d8ea2ca5538526f58b Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Fri, 24 May 2024 13:05:52 +0200 Subject: [PATCH 13/25] Add flag to filter dependencies based on test case type (#219) * Add flag do filter dependencies * Update docs --- .gitignore | 1 + docs/docs/cli/test.md | 5 ++ docs/tutorials/github-actions.md | 18 ++++- .../nf/test/commands/RunTestsCommand.java | 5 ++ .../lang/dependencies/DependencyGraph.java | 1 - .../lang/dependencies/DependencyResolver.java | 35 +++++--- .../nf/test/lang/dependencies/IMetaFile.java | 21 +++++ .../test/lang/dependencies/SnapshotFile.java | 5 ++ .../nf/test/lang/dependencies/TestFile.java | 48 +++++++++++ .../nf/test/nextflow/NextflowScript.java | 5 ++ .../dependencies/DependencyResolverTest.java | 79 ++++++++++++++++++- .../test/lang/dependencies/IMetaFileTest.java | 31 ++++++++ 12 files changed, 238 insertions(+), 16 deletions(-) create mode 100644 src/test/java/com/askimed/nf/test/lang/dependencies/IMetaFileTest.java diff --git a/.gitignore b/.gitignore index 08be52db..68d924e3 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ test_mock.nf site /tests /.nf-test.log +/temp \ No newline at end of file diff --git a/docs/docs/cli/test.md b/docs/docs/cli/test.md index 195c72fa..1ad30d2e 100644 --- a/docs/docs/cli/test.md +++ b/docs/docs/cli/test.md @@ -51,6 +51,11 @@ Writes test results in csv file. By default,nf-test automatically stores a new snapshot. When CI mode is activated, nf-test will fail the test instead of storing the snapshot automatically. +### `--filter ` + +Filter test cases by specified types (e.g., module, pipeline, workflow or function). Multiple types can be separated by commas. + + ### Optimizing Test Execution #### `--related-tests ` diff --git a/docs/tutorials/github-actions.md b/docs/tutorials/github-actions.md index 382044fb..64a30a65 100644 --- a/docs/tutorials/github-actions.md +++ b/docs/tutorials/github-actions.md @@ -48,7 +48,7 @@ jobs: sudo mv nf-test /usr/local/bin/ - name: Run Tests - run: nf-test test + run: nf-test test --ci ``` ### Explanation: @@ -57,7 +57,7 @@ jobs: 2. **Set up JDK 11**: Uses the `actions/setup-java@v2` action to set up Java Development Kit version 11. 3. **Setup Nextflow**: Uses the `nf-core/setup-nextflow@v1` action to install the latest-edge version of Nextflow. 4. **Install nf-test**: Downloads and installs nf-test. -5. **Run Tests**: Runs nf-test without sharding. +5. **Run Tests**: Runs nf-test with the `--ci` flag. This activates the CI mode. Instead of automatically storing a new snapshot as per usual, it will now fail the test if no reference snapshot is available. This enables tests to fail when a snapshot file was forgotten to be committed. ## Step 2: Extending to Use Sharding @@ -95,7 +95,7 @@ jobs: sudo mv nf-test /usr/local/bin/ - name: Run Tests (Shard ${{ matrix.shard }}/${{ strategy.job-total }}) - run: nf-test test --shard ${{ matrix.shard }}/${{ strategy.job-total }} + run: nf-test test --ci --shard ${{ matrix.shard }}/${{ strategy.job-total }} ``` ### Explanation of Sharding: @@ -141,7 +141,7 @@ jobs: sudo mv nf-test /usr/local/bin/ - name: Run Tests (Shard ${{ matrix.shard }}/${{ strategy.job-total }}) - run: nf-test test --shard ${{ matrix.shard }}/${{ strategy.job-total }} --changed-since HEAD^ + run: nf-test test --ci --shard ${{ matrix.shard }}/${{ strategy.job-total }} --changed-since HEAD^ ``` ### Explanation of Changes: @@ -172,6 +172,16 @@ config { This configuration ensures that critical changes always result in a comprehensive validation of the pipeline, providing additional confidence in your CI process. +## Step 5: Additional useful Options + +The `--filter` flag allows you to selectively run test cases based on their specified types. For example, you can filter tests by module, pipeline, workflow, or function. This is particularly useful when you have a large suite of tests and need to focus on specific areas of functionality. By separating multiple types with commas, you can run a customized subset of tests that match the exact criteria you're interested in, thereby saving time and resources. + +The `--related-tests` flag enables you to identify and execute all tests related to the provided `.nf` or `nf.test` files. This is ideal for scenarios where you have made changes to specific files and want to ensure that only the relevant tests are run. You can provide multiple files by separating them with spaces, which makes it easy to manage and test multiple changes at once, ensuring thorough validation of your updates. + +When the `--follow-dependencies` flag is set, the nf-test tool will automatically traverse and execute all tests for dependencies related to the files specified with the `--related-tests` flag. This ensures that any interdependent components are also tested, providing comprehensive coverage. This option is particularly useful for complex projects with multiple dependencies, as it bypasses the firewall calculation process and guarantees that all necessary tests are executed. + +The `--changed-until` flag allows you to run tests based on changes made up until a specified commit hash or branch name. By default, this parameter uses `HEAD`, but you can specify any commit or branch to target the changes made up to that point. This is particularly useful for validating changes over a specific range of commits, ensuring that all modifications within that period are tested comprehensively. + ## Summary 1. **Without Sharding**: A straightforward setup where all tests run in a single job. diff --git a/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java b/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java index b27ff4a8..ab8e8e2a 100644 --- a/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java +++ b/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java @@ -13,6 +13,7 @@ import com.askimed.nf.test.lang.dependencies.Coverage; import com.askimed.nf.test.lang.dependencies.DependencyExporter; import com.askimed.nf.test.lang.dependencies.DependencyResolver; +import com.askimed.nf.test.lang.dependencies.IMetaFile; import com.askimed.nf.test.util.AnsiText; import com.askimed.nf.test.util.GitCommand; import org.slf4j.Logger; @@ -80,6 +81,9 @@ public class RunTestsCommand extends AbstractCommand { @Option(names = { "--follow-dependencies", "--followDependencies"}, description = "Follows all dependencies when related-tests is set.", required = false, showDefaultValue = Visibility.ALWAYS) private boolean followDependencies = false; + @Option(names = { "--filter" }, description = "Filter test cases by specified types (e.g., module, pipeline, workflow or function). Multiple types can be separated by commas.", required = false, showDefaultValue = Visibility.ALWAYS) + private String dependencies = "all"; + @Option(names = { "--only-changed", "--onlyChanged"}, description = "Runs tests only for those files which are modified in the current git repository", required = false, showDefaultValue = Visibility.ALWAYS) private boolean onlyChanged = false; @@ -188,6 +192,7 @@ public Integer execute() throws Exception { File baseDir = new File(new File("").getAbsolutePath()); DependencyResolver resolver = new DependencyResolver(baseDir); resolver.setFollowingDependencies(followDependencies); + resolver.setTargets(IMetaFile.TargetType.parse(dependencies)); if (onlyChanged || changedSince != null) { diff --git a/src/main/java/com/askimed/nf/test/lang/dependencies/DependencyGraph.java b/src/main/java/com/askimed/nf/test/lang/dependencies/DependencyGraph.java index 0c8ddfe6..9c8b0228 100644 --- a/src/main/java/com/askimed/nf/test/lang/dependencies/DependencyGraph.java +++ b/src/main/java/com/askimed/nf/test/lang/dependencies/DependencyGraph.java @@ -57,7 +57,6 @@ public void addFile(IMetaFile metaFile) { public void connectDependencies(){ for (Node node: nodes.values()) { for (String dependency: node.getMetaFile().getDependencies()) { - //addDependency(dependency, node.getFilename()); addDependency(dependency, node.getFilename()); } } diff --git a/src/main/java/com/askimed/nf/test/lang/dependencies/DependencyResolver.java b/src/main/java/com/askimed/nf/test/lang/dependencies/DependencyResolver.java index 81233ef7..28d21cc5 100644 --- a/src/main/java/com/askimed/nf/test/lang/dependencies/DependencyResolver.java +++ b/src/main/java/com/askimed/nf/test/lang/dependencies/DependencyResolver.java @@ -20,6 +20,8 @@ public class DependencyResolver { private boolean followingDependencies = false; + private Set targets = new HashSet(); + private static Logger log = LoggerFactory.getLogger(DependencyResolver.class); public DependencyResolver(File baseDir) { @@ -38,11 +40,15 @@ public void setFollowingDependencies(boolean followingDependencies) { this.followingDependencies = followingDependencies; } + public void setTargets(Set targets) { + this.targets = targets; + } + public List findAllTests() throws Exception { List results = new Vector(); for (IMetaFile metaFile: graph.getFiles()) { - if (metaFile.getType() == IMetaFile.MetaFileType.TEST_FILE) { + if (metaFile.getType() == IMetaFile.MetaFileType.TEST_FILE && acceptMetaFile(metaFile)) { results.add(new File(metaFile.getFilename())); } } @@ -61,7 +67,7 @@ public List findTestsByFiles(List files) throws Exception { List results = new Vector(); for (IMetaFile metaFile: graph.getFiles()) { - if (metaFile.getType() == IMetaFile.MetaFileType.TEST_FILE) { + if (metaFile.getType() == IMetaFile.MetaFileType.TEST_FILE && acceptMetaFile(metaFile)) { File file = new File(metaFile.getFilename()); TestFilePattern matchedPattern = matches(file.toPath(), patterns); if (matchedPattern != null) { @@ -102,7 +108,7 @@ public List findRelatedTestsByFiles(File ... files) throws Exception { long time0 = System.currentTimeMillis(); for (File file: files) { - results.addAll(findRelatedTestsByFile(file.getAbsoluteFile(), followingDependencies)); + results.addAll(findRelatedTestsByFile(file.getAbsoluteFile())); } long time1 = System.currentTimeMillis(); log.info("Found {} tests for file {} in {} sec", results.size(), files, (time1 - time0) / 1000.0); @@ -110,7 +116,7 @@ public List findRelatedTestsByFiles(File ... files) throws Exception { return new Vector(results); } - private Set findRelatedTestsByFile(File file, boolean followingDependencies) throws Exception { + private Set findRelatedTestsByFile(File file) throws Exception { Set results = new HashSet(); @@ -123,7 +129,9 @@ private Set findRelatedTestsByFile(File file, boolean followingDependencie // the file is a test file if (metaFile.getType() == IMetaFile.MetaFileType.TEST_FILE){ - results.add(new File(metaFile.getFilename())); + if (acceptMetaFile(metaFile)) { + results.add(new File(metaFile.getFilename())); + } return results; } @@ -134,21 +142,24 @@ private Set findRelatedTestsByFile(File file, boolean followingDependencie if (dependency.getType() == IMetaFile.MetaFileType.TEST_FILE) { // is a test file --> return - results.add(dependencyFile); + if (acceptMetaFile(dependency)) { + results.add(dependencyFile); + } } else { // if a source file DependencyGraph.Node node = graph.getNode(dependency.getFilename()); - //TODO: add && !followingDependencies if (node.hasDependencyOfType(IMetaFile.MetaFileType.TEST_FILE) && !followingDependencies) { //has a test --> add all test and then stop for (DependencyGraph.Node dependencyOfDependency: node.getDependencies()) { if (dependencyOfDependency.getMetaFile().getType() == IMetaFile.MetaFileType.TEST_FILE) { - results.add(new File(dependencyOfDependency.getFilename())); + if (acceptMetaFile(dependencyOfDependency.getMetaFile())) { + results.add(new File(dependencyOfDependency.getFilename())); + } } } } else { //has no tests --> find related tests in a recursive way - results.addAll(findRelatedTestsByFile(dependencyFile, followingDependencies)); + results.addAll(findRelatedTestsByFile(dependencyFile)); } } } @@ -266,5 +277,11 @@ public boolean matches2(Path path, List ignorePatterns) { return false; } + public boolean acceptMetaFile(IMetaFile file) { + if (targets == null || targets.isEmpty()) { + return true; + } + return targets.contains(file.getTarget()); + } } diff --git a/src/main/java/com/askimed/nf/test/lang/dependencies/IMetaFile.java b/src/main/java/com/askimed/nf/test/lang/dependencies/IMetaFile.java index b124a8ae..5cd2a6ed 100644 --- a/src/main/java/com/askimed/nf/test/lang/dependencies/IMetaFile.java +++ b/src/main/java/com/askimed/nf/test/lang/dependencies/IMetaFile.java @@ -1,6 +1,7 @@ package com.askimed.nf.test.lang.dependencies; import java.io.IOException; +import java.util.HashSet; import java.util.Set; public interface IMetaFile { @@ -9,6 +10,8 @@ public interface IMetaFile { public MetaFileType getType(); + public TargetType getTarget(); + public String getFilename(); public void parseDependencies() throws IOException; @@ -17,4 +20,22 @@ public static enum MetaFileType{ SOURCE_FILE, TEST_FILE, SNAPSHOT_FILE } + public static enum TargetType{ + PROCESS, WORKFLOW, PIPELINE, FUNCTION, UNDEFINED; + + public static Set parse(String targets) { + Set result = new HashSet(); + String cleaned = targets.trim().toUpperCase(); + if (cleaned.isEmpty() || cleaned.equalsIgnoreCase("ALL")) { + return result; + } + for (String target: cleaned.split(",")) { + String cleanedTarget = target.trim(); + result.add(TargetType.valueOf(cleanedTarget)); + } + return result; + } + + } + } diff --git a/src/main/java/com/askimed/nf/test/lang/dependencies/SnapshotFile.java b/src/main/java/com/askimed/nf/test/lang/dependencies/SnapshotFile.java index dd4251a8..537f0c91 100644 --- a/src/main/java/com/askimed/nf/test/lang/dependencies/SnapshotFile.java +++ b/src/main/java/com/askimed/nf/test/lang/dependencies/SnapshotFile.java @@ -45,4 +45,9 @@ public Set getDependencies() { return dependencies; } + @Override + public TargetType getTarget() { + return TargetType.UNDEFINED; + } + } diff --git a/src/main/java/com/askimed/nf/test/lang/dependencies/TestFile.java b/src/main/java/com/askimed/nf/test/lang/dependencies/TestFile.java index e8c5d26e..46495c81 100644 --- a/src/main/java/com/askimed/nf/test/lang/dependencies/TestFile.java +++ b/src/main/java/com/askimed/nf/test/lang/dependencies/TestFile.java @@ -1,13 +1,16 @@ package com.askimed.nf.test.lang.dependencies; import com.askimed.nf.test.util.FileUtil; +import org.apache.tools.ant.taskdefs.Tar; import java.io.File; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.HashSet; +import java.util.List; import java.util.Set; +import java.util.Vector; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -19,6 +22,8 @@ public class TestFile implements IMetaFile { private Set dependencies = new HashSet(); + private TargetType target = TargetType.UNDEFINED; + public TestFile(File baseDir, File file) { this.baseDir = baseDir; this.file = file; @@ -26,6 +31,45 @@ public TestFile(File baseDir, File file) { public void parseDependencies() throws IOException { String script = FileUtil.readFileAsString(file); + target = parseType(script); + parseDependencies(script); + } + + private TargetType parseType(String script) { + + TargetType type = parseType(script, "nextflow_pipeline", TargetType.PIPELINE); + if (type != TargetType.UNDEFINED) { + return type; + } + type = parseType(script, "nextflow_process", TargetType.PROCESS); + if (type != TargetType.UNDEFINED) { + return type; + } + type = parseType(script, "nextflow_workflow", TargetType.WORKFLOW); + if (type != TargetType.UNDEFINED) { + return type; + } + type = parseType(script, "nextflow_function", TargetType.FUNCTION); + return type; + + } + + private TargetType parseType(String script, String type, TargetType targetType) { + + String patternType = "(?i)^\\s*" + type + "\\s*(.+)(\\s*\\{|\\{)"; + + Pattern r = Pattern.compile(patternType, Pattern.MULTILINE); + + Matcher m = r.matcher(script); + if (m.find()) { + return targetType; + } else { + return TargetType.UNDEFINED; + } + } + + private void parseDependencies(String script) throws IOException { + String regex = "(?i)script\\s+\"(.+)\""; Pattern pattern = Pattern.compile(regex); @@ -73,4 +117,8 @@ public Set getDependencies() { return dependencies; } + @Override + public TargetType getTarget() { + return target; + } } diff --git a/src/main/java/com/askimed/nf/test/nextflow/NextflowScript.java b/src/main/java/com/askimed/nf/test/nextflow/NextflowScript.java index 5289cc9e..ba70ea22 100644 --- a/src/main/java/com/askimed/nf/test/nextflow/NextflowScript.java +++ b/src/main/java/com/askimed/nf/test/nextflow/NextflowScript.java @@ -163,6 +163,11 @@ public static Set getDependencies(File file, String content) { } + @Override + public TargetType getTarget() { + return TargetType.UNDEFINED; + } + protected static Path resolve(File file, String dependency) { if (dependency.startsWith("./") || dependency.startsWith("../")) { return Paths.get(file.getParentFile().getAbsolutePath()).resolve(dependency); diff --git a/src/test/java/com/askimed/nf/test/lang/dependencies/DependencyResolverTest.java b/src/test/java/com/askimed/nf/test/lang/dependencies/DependencyResolverTest.java index 333d5bd1..0b152613 100644 --- a/src/test/java/com/askimed/nf/test/lang/dependencies/DependencyResolverTest.java +++ b/src/test/java/com/askimed/nf/test/lang/dependencies/DependencyResolverTest.java @@ -9,8 +9,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; -import java.util.List; -import java.util.Vector; +import java.util.*; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -83,6 +82,82 @@ void findRelatedTestAndFollowDependencies() throws Exception { assertEquals(17, coverage.getAll().getItems().size()); } + @Test + void findRelatedTestAndFilterDependencies() throws Exception { + + File root = getFetchNgs(); + + DependencyResolver resolver = new DependencyResolver(root); + resolver.setFollowingDependencies(false); + + resolver.buildGraph(); + assertEquals(10, resolver.findRelatedTestsByFiles( + new File(root, "modules/local/sra_to_samplesheet/main.nf") + ).size()); + + resolver = new DependencyResolver(root); + Set targets = new HashSet(); + targets.add(IMetaFile.TargetType.PROCESS); + resolver.setTargets(targets); + + resolver.buildGraph(); + assertEquals(1, resolver.findRelatedTestsByFiles( + new File(root, "modules/local/sra_to_samplesheet/main.nf") + ).size()); + + resolver = new DependencyResolver(root); + targets = new HashSet(); + targets.add(IMetaFile.TargetType.PROCESS); + targets.add(IMetaFile.TargetType.WORKFLOW); + resolver.setTargets(targets); + + resolver.buildGraph(); + assertEquals(10, resolver.findRelatedTestsByFiles( + new File(root, "modules/local/sra_to_samplesheet/main.nf") + ).size()); + + resolver = new DependencyResolver(root); + targets = new HashSet(); + targets.add(IMetaFile.TargetType.WORKFLOW); + resolver.setTargets(targets); + + resolver.buildGraph(); + assertEquals(9, resolver.findRelatedTestsByFiles( + new File(root, "modules/local/sra_to_samplesheet/main.nf") + ).size()); + + resolver = new DependencyResolver(root); + targets = new HashSet(); + targets.add(IMetaFile.TargetType.PIPELINE); + resolver.setTargets(targets); + resolver.setFollowingDependencies(true); + resolver.buildGraph(); + assertEquals(1, resolver.findRelatedTestsByFiles( + new File(root, "modules/local/sra_to_samplesheet/main.nf") + ).size()); + + resolver = new DependencyResolver(root); + targets = new HashSet(); + targets.add(IMetaFile.TargetType.PIPELINE); + targets.add(IMetaFile.TargetType.WORKFLOW); + resolver.setTargets(targets); + resolver.setFollowingDependencies(true); + resolver.buildGraph(); + assertEquals(10, resolver.findRelatedTestsByFiles( + new File(root, "modules/local/sra_to_samplesheet/main.nf") + ).size()); + + resolver = new DependencyResolver(root); + targets = new HashSet(); + targets.add(IMetaFile.TargetType.WORKFLOW); + resolver.setTargets(targets); + resolver.setFollowingDependencies(true); + resolver.buildGraph(); + assertEquals(9, resolver.findRelatedTestsByFiles( + new File(root, "modules/local/sra_to_samplesheet/main.nf") + ).size()); + } + @Test void findRelatedTestsWithTrigger() throws Exception { diff --git a/src/test/java/com/askimed/nf/test/lang/dependencies/IMetaFileTest.java b/src/test/java/com/askimed/nf/test/lang/dependencies/IMetaFileTest.java new file mode 100644 index 00000000..e52cdeac --- /dev/null +++ b/src/test/java/com/askimed/nf/test/lang/dependencies/IMetaFileTest.java @@ -0,0 +1,31 @@ +package com.askimed.nf.test.lang.dependencies; + +import org.junit.jupiter.api.Test; + +import java.util.HashSet; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.*; + +class IMetaFileTest { + + @Test + void parseDependencies() { + String input = "workflow,process"; + Set targets = new HashSet(); + targets.add(IMetaFile.TargetType.WORKFLOW); + targets.add(IMetaFile.TargetType.PROCESS); + assertEquals(targets , IMetaFile.TargetType.parse(input)); + } + + @Test + void parseDependenciesAll() { + String input = "all"; + Set targets = new HashSet(); + assertEquals(targets , IMetaFile.TargetType.parse(input)); + + input = " "; + targets = new HashSet(); + assertEquals(targets , IMetaFile.TargetType.parse(input)); + } +} \ No newline at end of file From e18f751188a4e59b54edd5e8e0c5e49b49b6f197 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Fri, 24 May 2024 13:36:32 +0200 Subject: [PATCH 14/25] Fix return code in testcase --- src/test/java/com/askimed/nf/test/lang/ProcessTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/askimed/nf/test/lang/ProcessTest.java b/src/test/java/com/askimed/nf/test/lang/ProcessTest.java index a4f6f47d..d72b34c4 100644 --- a/src/test/java/com/askimed/nf/test/lang/ProcessTest.java +++ b/src/test/java/com/askimed/nf/test/lang/ProcessTest.java @@ -103,7 +103,7 @@ public void testMissingScript() throws Exception { App app = new App(); int exitCode = app.run(new String[] { "test", "test-data/process/default/wrong-filename.nf.test" }); - assertEquals(2, exitCode); + assertEquals(0, exitCode); } From ef6515790da5d4b6ceb97b66b30779b59bd62a1c Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Fri, 24 May 2024 15:09:04 +0200 Subject: [PATCH 15/25] Prepare release 0.9.0-rc2 --- pom.xml | 2 +- src/main/java/com/askimed/nf/test/App.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index a8696813..a75421a1 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.askimed nf-test - 0.9.0-rc1 + 0.9.0-rc2 nf-test Simple test framework for Nextflow pipelines https://github.com/askimed/nf-test diff --git a/src/main/java/com/askimed/nf/test/App.java b/src/main/java/com/askimed/nf/test/App.java index 57317004..ba7a7e26 100644 --- a/src/main/java/com/askimed/nf/test/App.java +++ b/src/main/java/com/askimed/nf/test/App.java @@ -17,7 +17,7 @@ public class App { public static final String NAME = "nf-test"; - public static final String VERSION = "0.9.0-rc1"; + public static final String VERSION = "0.9.0-rc2"; public static final String PACKAGE = "com.askimed.nf.test"; From 3893d4ed52458784b41cff85d48bc54677c8a7e4 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Mon, 3 Jun 2024 13:38:01 +0200 Subject: [PATCH 16/25] Update README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index b9a2323c..35da1825 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,12 @@ Show the world your Nextflow pipeline is using nf-test: [![nf-test](https://img.shields.io/badge/tested_with-nf--test-337ab7.svg)](https://github.com/askimed/nf-test) ``` +## Citation + +If you test your pipeline with nf-test, please cite: + +Forer, L., & Schönherr, S. (2024). Improving the Reliability and Quality of Nextflow Pipelines with nf-test. *bioRxiv*. [https://doi.org/10.1101/2024.05.25.595877](https://doi.org/10.1101/2024.05.25.595877) + ## Contact - Lukas Forer [@lukfor](https://twitter.com/lukfor) From 4685943667117f9883933311b1131e17e3792fa7 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Mon, 3 Jun 2024 13:39:36 +0200 Subject: [PATCH 17/25] Update index.md --- docs/index.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/index.md b/docs/index.md index 8d6efe45..7e7f8c3c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -151,6 +151,15 @@ Show the world your Nextflow pipeline is using nf-test and add the following bad [![nf-test](https://img.shields.io/badge/tested_with-nf--test-337ab7.svg)](https://code.askimed.com/nf-test) ``` +--- + +## :material-file: Citation + +If you test your pipeline with nf-test, please cite: + +Forer, L., & Schönherr, S. (2024). Improving the Reliability and Quality of Nextflow Pipelines with nf-test. *bioRxiv*. [https://doi.org/10.1101/2024.05.25.595877](https://doi.org/10.1101/2024.05.25.595877) + + ---- ## :material-account-supervisor: About From 961c12855eb090d8dff06929f4b7604b1dde8560 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Thu, 6 Jun 2024 10:53:48 +0200 Subject: [PATCH 18/25] Update urls in documentation --- README.md | 4 ++-- docs/docs/plugins/using-plugins.md | 2 +- docs/installation.md | 6 +++--- mkdocs.yml | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 35da1825..c18ebdd7 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,13 @@ The full documentation can be found [here](https://code.askimed.com/nf-test). nf-test has the same requirements as Nextflow and can be used on POSIX compatible systems like Linux or OS X. You can install nf-test using the following command: ```bash -curl -fsSL https://code.askimed.com/install/nf-test | bash +curl -fsSL https://get.nf-test.com | bash ``` If you don't have curl installed, you could use wget: ```bash -wget -qO- https://code.askimed.com/install/nf-test | bash +wget -qO- https://get.nf-test.com | bash ``` It will create the `nf-test` executable file in the current directory. Optionally, move the `nf-test` file to a directory accessible by your `$PATH` variable. diff --git a/docs/docs/plugins/using-plugins.md b/docs/docs/plugins/using-plugins.md index d9d585c4..aa131793 100644 --- a/docs/docs/plugins/using-plugins.md +++ b/docs/docs/plugins/using-plugins.md @@ -13,7 +13,7 @@ For this purpose, we integrated the following plugin system that provides (a) th ## Using Plugins -Available plugins are listed [here](https://code.askimed.com/nf-test-plugins). +Available plugins are listed [here](https://plugins.nf-test.com). A plugin can be activated via the `nf-test.config` by adding the `plugin` section and by using `load` method to specify the plugin and its version: diff --git a/docs/installation.md b/docs/installation.md index 77be2589..a9e458a7 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -5,13 +5,13 @@ nf-test has the same requirements as Nextflow and can be used on POSIX compatible systems like Linux or OS X. You can install nf-test using the following command: ```bash -curl -fsSL https://code.askimed.com/install/nf-test | bash +curl -fsSL https://get.nf-test.com | bash ``` If you don't have curl installed, you could use wget: ```bash -wget -qO- https://code.askimed.com/install/nf-test | bash +wget -qO- https://get.nf-test.com | bash ``` It will create the `nf-test` executable file in the current directory. Optionally, move the `nf-test` file to a directory accessible by your `$PATH` variable. @@ -48,7 +48,7 @@ Now you are ready to write your [first testcase](docs/getting-started.md). If you want to install a specific version pass it to the install script as so ```sh -curl -fsSL https://code.askimed.com/install/nf-test | bash -s 0.7.0 +curl -fsSL https://get.nf-test.com | bash -s 0.7.0 ``` ### Manual installation diff --git a/mkdocs.yml b/mkdocs.yml index 97e0a802..f52cdb79 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -62,7 +62,7 @@ nav: - clean: docs/cli/clean.md - Configuration: docs/configuration.md - Plugins: - - Available Plugins: https://code.askimed.com/nf-test-plugins + - Available Plugins: https://plugins.nf-test.com - Using Plugins: docs/plugins/using-plugins.md - Developing Plugins: docs/plugins/developing-plugins.md - Tutorials: tutorials.md From 0e48123c619e1822abed8c9954fd1ee41dc84ddf Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Fri, 7 Jun 2024 14:19:10 +0200 Subject: [PATCH 19/25] Add plugins to navbar --- mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index f52cdb79..29829fd3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -62,9 +62,9 @@ nav: - clean: docs/cli/clean.md - Configuration: docs/configuration.md - Plugins: - - Available Plugins: https://plugins.nf-test.com - Using Plugins: docs/plugins/using-plugins.md - Developing Plugins: docs/plugins/developing-plugins.md + - Plugins: https://plugins.nf-test.com - Tutorials: tutorials.md - Resources: resources.md - About: about.md From 2bc23bc73d7f2643bc7a70b042dc52b9f45190ff Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Fri, 7 Jun 2024 14:20:19 +0200 Subject: [PATCH 20/25] Hide navigation on tutorial page --- docs/tutorials.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/tutorials.md b/docs/tutorials.md index 63e99d97..74adeeea 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -1,3 +1,8 @@ +--- +hide: + - navigation +--- + # Tutorials --- From b22274f295ff8cc6b52cf1121e2cff5010745c71 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Fri, 5 Jul 2024 18:18:10 +0200 Subject: [PATCH 21/25] Improve coverage output (#224) * Reformat coverage and move export to command * Add simple html coverage report * Fix coverage badge * Add basic documentation --- docs/docs/cli/coverage.md | 19 +++ mkdocs.yml | 1 + src/main/java/com/askimed/nf/test/App.java | 9 +- .../nf/test/commands/CoverageCommand.java | 96 +++++++++++++++ .../nf/test/commands/RunTestsCommand.java | 18 +-- .../nf/test/lang/dependencies/Coverage.java | 111 +++++++++++++++++- .../lang/dependencies/CoverageItemSorter.java | 9 ++ .../lang/dependencies/coverage-report.html | 48 ++++++++ 8 files changed, 289 insertions(+), 22 deletions(-) create mode 100644 docs/docs/cli/coverage.md create mode 100644 src/main/java/com/askimed/nf/test/commands/CoverageCommand.java create mode 100644 src/main/java/com/askimed/nf/test/lang/dependencies/CoverageItemSorter.java create mode 100644 src/main/resources/com/askimed/nf/test/lang/dependencies/coverage-report.html diff --git a/docs/docs/cli/coverage.md b/docs/docs/cli/coverage.md new file mode 100644 index 00000000..2d29f111 --- /dev/null +++ b/docs/docs/cli/coverage.md @@ -0,0 +1,19 @@ +# `coverage` command + +:octicons-tag-24: 0.9.0 + +## Usage + +``` +nf-test coverage +``` + +The `coverage` command prints information about the number of Nextflow files that are covered by a test. + +### Optional Arguments + +#### `--csv ` +Writes a coverage report in csv format. + +#### `--html ` +Writes a coverage report in html format. diff --git a/mkdocs.yml b/mkdocs.yml index 29829fd3..9667650f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -59,6 +59,7 @@ nav: - generate: docs/cli/generate.md - test: docs/cli/test.md - list: docs/cli/list.md + - coverage: docs/cli/coverage.md - clean: docs/cli/clean.md - Configuration: docs/configuration.md - Plugins: diff --git a/src/main/java/com/askimed/nf/test/App.java b/src/main/java/com/askimed/nf/test/App.java index ba7a7e26..d9db5d1e 100644 --- a/src/main/java/com/askimed/nf/test/App.java +++ b/src/main/java/com/askimed/nf/test/App.java @@ -1,12 +1,6 @@ package com.askimed.nf.test; -import com.askimed.nf.test.commands.CleanCommand; -import com.askimed.nf.test.commands.GenerateTestsCommand; -import com.askimed.nf.test.commands.InitCommand; -import com.askimed.nf.test.commands.ListTestsCommand; -import com.askimed.nf.test.commands.RunTestsCommand; -import com.askimed.nf.test.commands.UpdatePluginsCommand; -import com.askimed.nf.test.commands.VersionCommand; +import com.askimed.nf.test.commands.*; import ch.qos.logback.classic.Level; import picocli.CommandLine; @@ -35,6 +29,7 @@ public int run(String[] args) { commandLine.addSubcommand("clean", new CleanCommand()); commandLine.addSubcommand("init", new InitCommand()); commandLine.addSubcommand("test", new RunTestsCommand()); + commandLine.addSubcommand("coverage", new CoverageCommand()); commandLine.addSubcommand("list", new ListTestsCommand()); commandLine.addSubcommand("ls", new ListTestsCommand()); commandLine.addSubcommand("generate", new GenerateTestsCommand()); diff --git a/src/main/java/com/askimed/nf/test/commands/CoverageCommand.java b/src/main/java/com/askimed/nf/test/commands/CoverageCommand.java new file mode 100644 index 00000000..757962c4 --- /dev/null +++ b/src/main/java/com/askimed/nf/test/commands/CoverageCommand.java @@ -0,0 +1,96 @@ +package com.askimed.nf.test.commands; + +import com.askimed.nf.test.config.Config; +import com.askimed.nf.test.lang.dependencies.Coverage; +import com.askimed.nf.test.lang.dependencies.DependencyResolver; +import com.askimed.nf.test.util.AnsiColors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import picocli.CommandLine.Command; +import picocli.CommandLine.Help.Visibility; +import picocli.CommandLine.Option; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +@Command(name = "coverage") +public class CoverageCommand extends AbstractCommand { + + private static final String SHARD_STRATEGY_ROUND_ROBIN = "round-robin"; + + @Option(names = { + "--csv" }, description = "Write coverage results in csv format", required = false, showDefaultValue = Visibility.ALWAYS) + private String csv = null; + + @Option(names = { + "--html" }, description = "Write coverage results in html format", required = false, showDefaultValue = Visibility.ALWAYS) + private String html = null; + + + @Option(names = { "--config", + "-c" }, description = "nf-test.config filename", required = false, showDefaultValue = Visibility.ALWAYS) + private String configFilename = Config.FILENAME; + + private static Logger log = LoggerFactory.getLogger(CoverageCommand.class); + + @Override + public Integer execute() throws Exception { + + List scripts = new ArrayList(); + Config config = null; + + try { + + File defaultConfigFile = null; + boolean defaultWithTrace = true; + try { + File configFile = new File(configFilename); + if (configFile.exists()) { + log.info("Load config from file {}...", configFile.getAbsolutePath()); + config = Config.parse(configFile); + } else { + System.out.println(AnsiColors.yellow("Warning: This pipeline has no nf-test config file.")); + log.warn("No nf-test config file found."); + } + + } catch (Exception e) { + + System.out.println(AnsiColors.red("Error: Syntax errors in nf-test config file: " + e)); + log.error("Parsing config file failed", e); + return 2; + + } + + File baseDir = new File(new File("").getAbsolutePath()); + DependencyResolver resolver = new DependencyResolver(baseDir); + resolver.setFollowingDependencies(true); + + + if (config != null) { + resolver.buildGraph(config.getIgnore(), config.getTriggers()); + } else { + resolver.buildGraph(); + } + + Coverage coverage = new Coverage(resolver).getAll(); + if (csv != null) { + coverage.exportAsCsv(csv); + } else if (html != null) { + coverage.exportAsHtml(html); + } else { + coverage.printDetails(); + } + + return 0; + + } catch (Throwable e) { + + System.out.println(AnsiColors.red("Error: " + e));log.error("Running tests failed.", e); + return 1; + + } + + } + +} diff --git a/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java b/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java index ab8e8e2a..e43a7e08 100644 --- a/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java +++ b/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java @@ -188,7 +188,6 @@ public Integer execute() throws Exception { return 2; } - List ignorePatterns = new Vector(); File baseDir = new File(new File("").getAbsolutePath()); DependencyResolver resolver = new DependencyResolver(baseDir); resolver.setFollowingDependencies(followDependencies); @@ -231,10 +230,6 @@ public Integer execute() throws Exception { AnsiText.printBulletList(scripts); - if (coverage) { - new Coverage(resolver).getByFiles(testPaths).print(); - } - } else { if (config != null) { resolver.buildGraph(config.getIgnore(), config.getTriggers()); @@ -242,9 +237,6 @@ public Integer execute() throws Exception { resolver.buildGraph(); } scripts = resolver.findTestsByFiles(testPaths); - if (coverage) { - new Coverage(resolver).getAll().print(); - } } if (graph != null) { @@ -304,7 +296,15 @@ public Integer execute() throws Exception { System.out.println(AnsiColors.yellow("Dry run mode activated: tests are not executed, just listed.")); } - return engine.execute(); + int exitStatus = engine.execute(); + + if (coverage && findRelatedTests) { + new Coverage(resolver).getByFiles(testPaths).print(); + } else if (coverage) { + new Coverage(resolver).getAll().print(); + } + + return exitStatus; } catch (Throwable e) { diff --git a/src/main/java/com/askimed/nf/test/lang/dependencies/Coverage.java b/src/main/java/com/askimed/nf/test/lang/dependencies/Coverage.java index 3d38cf62..cd85c6ea 100644 --- a/src/main/java/com/askimed/nf/test/lang/dependencies/Coverage.java +++ b/src/main/java/com/askimed/nf/test/lang/dependencies/Coverage.java @@ -1,16 +1,31 @@ package com.askimed.nf.test.lang.dependencies; +import com.askimed.nf.test.commands.init.InitTemplates; +import com.askimed.nf.test.core.TestExecutionResult; +import com.askimed.nf.test.core.TestSuiteExecutionResult; +import com.askimed.nf.test.core.reports.CsvReportWriter; import com.askimed.nf.test.util.AnsiColors; +import com.askimed.nf.test.util.AnsiText; +import com.askimed.nf.test.util.FileUtil; +import com.opencsv.CSVWriter; +import groovy.lang.Writable; +import groovy.text.SimpleTemplateEngine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.net.URL; +import java.nio.file.Paths; import java.text.DecimalFormat; -import java.util.List; -import java.util.Vector; +import java.text.DecimalFormatSymbols; +import java.util.*; public class Coverage { + private static final String HTML_TEMPLATE = "coverage-report.html"; + private int coveredItems = 0; private DependencyGraph graph; @@ -19,12 +34,15 @@ public class Coverage { private static Logger log = LoggerFactory.getLogger(Coverage.class); + private File baseDir = null; + public Coverage(DependencyGraph graph) { this.graph = graph; } public Coverage(DependencyResolver resolver) { this.graph = resolver.getGraph(); + baseDir = resolver.getBaseDir(); } public void add(File file, boolean covered) { @@ -57,6 +75,8 @@ public Coverage getAll(){ } + items.sort(new CoverageItemSorter()); + long time1 = System.currentTimeMillis(); log.info("Calculated coverage for {} files in {} sec", graph.size(), (time1 - time0) / 1000.0); @@ -87,6 +107,8 @@ public Coverage getByFiles(List files){ } + items.sort(new CoverageItemSorter()); + long time1 = System.currentTimeMillis(); log.info("Calculated coverage for {} files in {} sec", graph.size(), (time1 - time0) / 1000.0); @@ -95,14 +117,89 @@ public Coverage getByFiles(List files){ } public void print() { - DecimalFormat decimalFormat = new DecimalFormat("#.##"); + printLabel(); System.out.println(); - System.out.print("Coverage: " + getCoveredItems() + "/" + getItems().size()); - System.out.println(" (" + decimalFormat.format(getCoveredItems() / (float) getItems().size() * 100) + "%)"); + } + + public void printDetails() { + System.out.println(); + System.out.println("Files:"); for (Coverage.CoverageItem item : getItems()) { - System.out.println(" - " + (item.isCovered() ? AnsiColors.green(item.getFile().getAbsolutePath()) : AnsiColors.red(item.getFile().getAbsolutePath()))); + String label = getFileLabel(item.getFile()); + System.out.println(" \u2022 " + (item.isCovered() ? AnsiColors.green(label) : AnsiColors.red(label))); } System.out.println(); + printLabel(); + System.out.println(); + } + + public String getFileLabel(File file) { + String label = file.getAbsolutePath(); + if (baseDir != null) { + label = Paths.get(baseDir.getAbsolutePath()).relativize(file.toPath()).toString(); + } + return label; + } + + private void printLabel() { + float coverage = getCoveredItems() / (float) getItems().size(); + System.out.print(getColor("COVERAGE:", coverage) + " " + formatCoverage(coverage)); + System.out.println( " [" + getCoveredItems() + " of " + getItems().size() + " files]"); + } + + public float getCoverage() { + return getCoveredItems() / (float) getItems().size(); + } + + private String getColor(String label, float value) { + if (value < 0.5) { + return AnsiColors.red(label); + } else if (value < 0.9) { + return AnsiColors.yellow(label); + } else { + return AnsiColors.green(label); + } + } + + private String formatCoverage(float value) { + DecimalFormat decimalFormat = new DecimalFormat("#.##", DecimalFormatSymbols.getInstance(Locale.US)); + return decimalFormat.format(value * 100) + "%"; + } + + public void exportAsCsv(String filename) throws IOException { + String[] header = new String[]{ + "filename", + "covered", + "type" + }; + + CSVWriter writer = new CSVWriter(new FileWriter(new File(filename))); + writer.writeNext(header); + for (Coverage.CoverageItem item : getItems()) { + String[] line = new String[]{ + item.getFile().getAbsolutePath(), + item.isCovered() + "", + "unknown" + }; + + writer.writeNext(line); + } + + writer.close(); + System.out.println(); + printLabel(); + System.out.println(); + System.out.println("Wrote coverage report to file " + filename + "\n"); + + } + + public void exportAsHtml(String filename) throws IOException, ClassNotFoundException { + Map binding = new HashMap(); + binding.put("coverage", this); + URL templateUrl = Coverage.class.getResource(HTML_TEMPLATE); + SimpleTemplateEngine engine = new SimpleTemplateEngine(); + Writable template = engine.createTemplate(templateUrl).make(binding); + FileUtil.write(new File(filename), template); } public static class CoverageItem { @@ -111,6 +208,8 @@ public static class CoverageItem { private boolean covered = false; + //TODO: add number of tests?? + public CoverageItem(File file, boolean covered) { this.file = file; this.covered = covered; diff --git a/src/main/java/com/askimed/nf/test/lang/dependencies/CoverageItemSorter.java b/src/main/java/com/askimed/nf/test/lang/dependencies/CoverageItemSorter.java new file mode 100644 index 00000000..4c7b1fd3 --- /dev/null +++ b/src/main/java/com/askimed/nf/test/lang/dependencies/CoverageItemSorter.java @@ -0,0 +1,9 @@ +package com.askimed.nf.test.lang.dependencies; + +public class CoverageItemSorter implements java.util.Comparator { + + @Override + public int compare(Coverage.CoverageItem o1, Coverage.CoverageItem o2) { + return o1.getFile().getAbsolutePath().compareTo(o2.getFile().getAbsolutePath()); + } +} diff --git a/src/main/resources/com/askimed/nf/test/lang/dependencies/coverage-report.html b/src/main/resources/com/askimed/nf/test/lang/dependencies/coverage-report.html new file mode 100644 index 00000000..d0ad39b6 --- /dev/null +++ b/src/main/resources/com/askimed/nf/test/lang/dependencies/coverage-report.html @@ -0,0 +1,48 @@ + + + + + + + + + + +
+

Coverage Report

+

This report was generated by nf-test on <%= new Date() %>.

+

+ Coverage: <%= coverage.formatCoverage(coverage.getCoverage()) %> +

+ +
+
+
+ +
+ + + + + + + + + + <% coverage.items.each { item -> %> + + + + + <% } %> + +
FileCovered
<%= coverage.getFileLabel(item.getFile()) %><%= item.isCovered() %>
+
+ + \ No newline at end of file From c7a0a61dcd588762baa22cd8a0c6f18425720fea Mon Sep 17 00:00:00 2001 From: Ken Brewer Date: Wed, 10 Jul 2024 12:07:42 -0400 Subject: [PATCH 22/25] feat: add classname attribute to junit xml report (#229) * feat: add classname attribute to junit xml report Closes #288 * fix: add missing semi-colon --- .../java/com/askimed/nf/test/core/reports/XmlReportWriter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/askimed/nf/test/core/reports/XmlReportWriter.java b/src/main/java/com/askimed/nf/test/core/reports/XmlReportWriter.java index 688617a8..3fc4d776 100644 --- a/src/main/java/com/askimed/nf/test/core/reports/XmlReportWriter.java +++ b/src/main/java/com/askimed/nf/test/core/reports/XmlReportWriter.java @@ -52,6 +52,7 @@ public void writeToFile(List testSuites) throws Except for (TestExecutionResult test : testSuite.getTests()) { writer.writeStartElement("testcase"); + writer.writeAttribute("classname", testSuite.getTestSuite().getName()); writer.writeAttribute("name", test.getTest().getName()); writer.writeAttribute("time", Double.toString(test.getExecutionTimeInSecs())); writer.writeAttribute("status", test.getStatus().toString()); From 7b11c33f969a2f27ebd592ec7d4e50e4154cd147 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Fri, 12 Jul 2024 11:06:27 +0200 Subject: [PATCH 23/25] Update logback to 1.5.6 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index a75421a1..e628c423 100644 --- a/pom.xml +++ b/pom.xml @@ -108,14 +108,14 @@ ch.qos.logback logback-classic - 1.4.11 + 1.5.6 ch.qos.logback logback-core - 1.4.11 + 1.5.6 From 04fa9a8ae698f4743e3471566ff174bf3f858452 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Sat, 13 Jul 2024 12:17:12 +0200 Subject: [PATCH 24/25] Serialize null values in channel outputs (#232) --- .../nf/test/lang/function/WorkflowMock.nf | 5 ++-- .../nf/test/lang/process/WorkflowMock.nf | 5 ++-- .../nf/test/lang/workflow/WorkflowMock.nf | 4 ++-- .../com/askimed/nf/test/lang/ProcessTest.java | 9 +++++++ test-data/channels/null-values/return_null.nf | 8 +++++++ .../channels/null-values/return_null.nf.test | 24 +++++++++++++++++++ 6 files changed, 49 insertions(+), 6 deletions(-) create mode 100644 test-data/channels/null-values/return_null.nf create mode 100644 test-data/channels/null-values/return_null.nf.test diff --git a/src/main/resources/com/askimed/nf/test/lang/function/WorkflowMock.nf b/src/main/resources/com/askimed/nf/test/lang/function/WorkflowMock.nf index 75566acd..23224b24 100644 --- a/src/main/resources/com/askimed/nf/test/lang/function/WorkflowMock.nf +++ b/src/main/resources/com/askimed/nf/test/lang/function/WorkflowMock.nf @@ -20,10 +20,11 @@ include { ${include} } from '${script}' // define custom rules for JSON that will be generated. def jsonOutput = new JsonGenerator.Options() - .excludeNulls() // Do not include fields with value null.. .addConverter(Path) { value -> value.toAbsolutePath().toString() } // Custom converter for Path. Only filename .build() +def jsonWorkflowOutput = new JsonGenerator.Options().excludeNulls().build() + workflow { @@ -43,6 +44,6 @@ workflow.onComplete { errorMessage: workflow.errorMessage, errorReport: workflow.errorReport ] - new File("\${params.nf_test_output}/workflow.json").text = jsonOutput.toJson(result) + new File("\${params.nf_test_output}/workflow.json").text = jsonWorkflowOutput.toJson(result) } \ No newline at end of file diff --git a/src/main/resources/com/askimed/nf/test/lang/process/WorkflowMock.nf b/src/main/resources/com/askimed/nf/test/lang/process/WorkflowMock.nf index d006888e..232342e6 100644 --- a/src/main/resources/com/askimed/nf/test/lang/process/WorkflowMock.nf +++ b/src/main/resources/com/askimed/nf/test/lang/process/WorkflowMock.nf @@ -17,10 +17,11 @@ include { ${process} } from '${script}' // define custom rules for JSON that will be generated. def jsonOutput = new JsonGenerator.Options() - .excludeNulls() // Do not include fields with value null.. .addConverter(Path) { value -> value.toAbsolutePath().toString() } // Custom converter for Path. Only filename .build() +def jsonWorkflowOutput = new JsonGenerator.Options().excludeNulls().build() + workflow { @@ -83,6 +84,6 @@ workflow.onComplete { errorMessage: workflow.errorMessage, errorReport: workflow.errorReport ] - new File("\${params.nf_test_output}/workflow.json").text = jsonOutput.toJson(result) + new File("\${params.nf_test_output}/workflow.json").text = jsonWorkflowOutput.toJson(result) } diff --git a/src/main/resources/com/askimed/nf/test/lang/workflow/WorkflowMock.nf b/src/main/resources/com/askimed/nf/test/lang/workflow/WorkflowMock.nf index 126157ca..5004d612 100644 --- a/src/main/resources/com/askimed/nf/test/lang/workflow/WorkflowMock.nf +++ b/src/main/resources/com/askimed/nf/test/lang/workflow/WorkflowMock.nf @@ -17,10 +17,10 @@ include { ${workflow} } from '${script}' // define custom rules for JSON that will be generated. def jsonOutput = new JsonGenerator.Options() - .excludeNulls() // Do not include fields with value null.. .addConverter(Path) { value -> value.toAbsolutePath().toString() } // Custom converter for Path. Only filename .build() +def jsonWorkflowOutput = new JsonGenerator.Options().excludeNulls().build() workflow { @@ -83,6 +83,6 @@ workflow.onComplete { errorMessage: workflow.errorMessage, errorReport: workflow.errorReport ] - new File("\${params.nf_test_output}/workflow.json").text = jsonOutput.toJson(result) + new File("\${params.nf_test_output}/workflow.json").text = jsonWorkflowOutput.toJson(result) } diff --git a/src/test/java/com/askimed/nf/test/lang/ProcessTest.java b/src/test/java/com/askimed/nf/test/lang/ProcessTest.java index d72b34c4..103196a8 100644 --- a/src/test/java/com/askimed/nf/test/lang/ProcessTest.java +++ b/src/test/java/com/askimed/nf/test/lang/ProcessTest.java @@ -80,6 +80,15 @@ public void testDisableAutoSortTestSuiteAndOverwrite() throws Exception { } + @Test + public void testNullValuesInChannels() throws Exception { + + App app = new App(); + int exitCode = app.run(new String[] { "test", "test-data/channels/null-values/return_null.nf.test" }); + assertEquals(0, exitCode); + + } + @Test public void testWithNoOutputs() throws Exception { diff --git a/test-data/channels/null-values/return_null.nf b/test-data/channels/null-values/return_null.nf new file mode 100644 index 00000000..1a9ccdea --- /dev/null +++ b/test-data/channels/null-values/return_null.nf @@ -0,0 +1,8 @@ +process return_null { + + output: + val null_list, emit: null_list + + exec: + null_list = ["0", "1", "", null, "4"] +} \ No newline at end of file diff --git a/test-data/channels/null-values/return_null.nf.test b/test-data/channels/null-values/return_null.nf.test new file mode 100644 index 00000000..0077701f --- /dev/null +++ b/test-data/channels/null-values/return_null.nf.test @@ -0,0 +1,24 @@ +nextflow_process { + + name "Test Process return_null" + script "./return_null.nf" + process "return_null" + + test("Should run without failures") { + + when { + params {} + process {} + } + + then { + assert process.success + + with(process.out) { + assert null_list == [["0", "1", "", null, "4"]] + } + } + + } + +} \ No newline at end of file From 26c9edd33146b02502dba52cea4694cead494e26 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Sat, 13 Jul 2024 12:41:16 +0200 Subject: [PATCH 25/25] Update external libraries to fix security issues (#233) * Update opencsv to 5.9 * Update groovy to 3.0.22 --- pom.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index e628c423..983d6e8c 100644 --- a/pom.xml +++ b/pom.xml @@ -37,20 +37,20 @@ org.codehaus.groovy groovy-all - 3.0.19 + 3.0.22 pom org.codehaus.groovy groovy-nio - 3.0.19 + 3.0.22 org.codehaus.groovy groovy-yaml - 3.0.19 + 3.0.22 @@ -68,7 +68,7 @@ com.opencsv opencsv - 5.5.2 + 5.9