From 118dbf3f288a2ea94dba2def32c8ad7bfbc456e9 Mon Sep 17 00:00:00 2001 From: chriswyatt1 Date: Sun, 25 Aug 2024 13:27:54 +0000 Subject: [PATCH 01/11] working with orthofinder continue feature --- modules/nf-core/orthofinder/main.nf | 18 +- .../nf-core/orthofinder/tests/main.nf.test | 5 +- .../orthofinder/tests/main.nf.test.snap | 340 +----------------- 3 files changed, 33 insertions(+), 330 deletions(-) diff --git a/modules/nf-core/orthofinder/main.nf b/modules/nf-core/orthofinder/main.nf index 2971fc0acf8..fd4aa2128d0 100644 --- a/modules/nf-core/orthofinder/main.nf +++ b/modules/nf-core/orthofinder/main.nf @@ -9,6 +9,7 @@ process ORTHOFINDER { input: tuple val(meta), path(fastas, stageAs: 'input/') + path(prior_run) output: tuple val(meta), path("$prefix") , emit: orthofinder @@ -18,18 +19,21 @@ process ORTHOFINDER { task.ext.when == null || task.ext.when script: - def args = task.ext.args ?: '' - prefix = task.ext.prefix ?: "${meta.id}" + def args = task.ext.args ?: '' + prefix = task.ext.prefix ?: "${meta.id}" + def include_command = prior_run ? "-b $prior_run" : '' + """ mkdir temp_pickle orthofinder \\ - $args \\ -t $task.cpus \\ -a $task.cpus \\ -p temp_pickle \\ -f input \\ - -n $prefix + -n $prefix \\ + $include_command $args + mv \\ input/OrthoFinder/Results_$prefix \\ @@ -42,8 +46,10 @@ process ORTHOFINDER { """ stub: - def args = task.ext.args ?: '' - prefix = task.ext.prefix ?: "${meta.id}" + def args = task.ext.args ?: '' + prefix = task.ext.prefix ?: "${meta.id}" + def include_command = prior_run ? "-b $prior_run" : '' + """ mkdir -p $prefix/Comparative_Genomics_Statistics mkdir $prefix/Gene_Duplication_Events diff --git a/modules/nf-core/orthofinder/tests/main.nf.test b/modules/nf-core/orthofinder/tests/main.nf.test index 2d648004b06..0be237437eb 100644 --- a/modules/nf-core/orthofinder/tests/main.nf.test +++ b/modules/nf-core/orthofinder/tests/main.nf.test @@ -28,6 +28,7 @@ nextflow_process { [ id:'test', single_end:false ], [ file_a, file_b ] ] + input[1] = [] """ } } @@ -41,7 +42,7 @@ nextflow_process { all_files << file } - def all_file_names = all_files.collect { it.name }.sort(false) + //def all_file_names = all_files.collect { it.name }.sort(false) def stable_file_names = [ 'Statistics_PerSpecies.tsv', @@ -52,7 +53,6 @@ nextflow_process { def stable_files = all_files.findAll { it.name in stable_file_names } assert snapshot( - all_file_names, stable_files, process.out.versions[0] ).match() @@ -80,6 +80,7 @@ nextflow_process { [ id:'test', single_end:false ], [ file_a, file_b ] ] + input[1] = [] """ } } diff --git a/modules/nf-core/orthofinder/tests/main.nf.test.snap b/modules/nf-core/orthofinder/tests/main.nf.test.snap index 8b621c052b5..35dbff39953 100644 --- a/modules/nf-core/orthofinder/tests/main.nf.test.snap +++ b/modules/nf-core/orthofinder/tests/main.nf.test.snap @@ -1,323 +1,4 @@ { - "sarscov2 - candidatus_portiera_aleyrodidarum - proteome": { - "content": [ - [ - "Blast0_0.txt.gz", - "Blast0_1.txt.gz", - "Blast1_0.txt.gz", - "Blast1_1.txt.gz", - "Citation.txt", - "Duplications.tsv", - "Duplications_per_Orthogroup.tsv", - "Duplications_per_Species_Tree_Node.tsv", - "Log.txt", - "N0.tsv", - "OG0000000.fa", - "OG0000000_tree.txt", - "OG0000000_tree.txt", - "OG0000000_tree_id.txt", - "OG0000001.fa", - "OG0000001_tree.txt", - "OG0000001_tree.txt", - "OG0000001_tree_id.txt", - "OG0000002.fa", - "OG0000003.fa", - "OG0000004.fa", - "OG0000005.fa", - "OG0000006.fa", - "OG0000007.fa", - "OG0000008.fa", - "OG0000009.fa", - "OG0000010.fa", - "OG0000011.fa", - "OG0000012.fa", - "OG0000013.fa", - "OG0000014.fa", - "OG0000015.fa", - "OG0000016.fa", - "OG0000017.fa", - "OG0000018.fa", - "OG0000019.fa", - "OG0000020.fa", - "OG0000021.fa", - "OG0000022.fa", - "OG0000023.fa", - "OG0000024.fa", - "OG0000025.fa", - "OG0000026.fa", - "OG0000027.fa", - "OG0000028.fa", - "OG0000029.fa", - "OG0000030.fa", - "OG0000031.fa", - "OG0000032.fa", - "OG0000033.fa", - "OG0000034.fa", - "OG0000035.fa", - "OG0000036.fa", - "OG0000037.fa", - "OG0000038.fa", - "OG0000039.fa", - "OG0000040.fa", - "OG0000041.fa", - "OG0000042.fa", - "OG0000043.fa", - "OG0000044.fa", - "OG0000045.fa", - "OG0000046.fa", - "OG0000047.fa", - "OG0000048.fa", - "OG0000049.fa", - "OG0000050.fa", - "OG0000051.fa", - "OG0000052.fa", - "OG0000053.fa", - "OG0000054.fa", - "OG0000055.fa", - "OG0000056.fa", - "OG0000057.fa", - "OG0000058.fa", - "OG0000059.fa", - "OG0000060.fa", - "OG0000061.fa", - "OG0000062.fa", - "OG0000063.fa", - "OG0000064.fa", - "OG0000065.fa", - "OG0000066.fa", - "OG0000067.fa", - "OG0000068.fa", - "OG0000069.fa", - "OG0000070.fa", - "OG0000071.fa", - "OG0000072.fa", - "OG0000073.fa", - "OG0000074.fa", - "OG0000075.fa", - "OG0000076.fa", - "OG0000077.fa", - "OG0000078.fa", - "OG0000079.fa", - "OG0000080.fa", - "OG0000081.fa", - "OG0000082.fa", - "OG0000083.fa", - "OG0000084.fa", - "OG0000085.fa", - "OG0000086.fa", - "OG0000087.fa", - "OG0000088.fa", - "OG0000089.fa", - "OG0000090.fa", - "OG0000091.fa", - "OG0000092.fa", - "OG0000093.fa", - "OG0000094.fa", - "OG0000095.fa", - "OG0000096.fa", - "OG0000097.fa", - "OG0000098.fa", - "OG0000099.fa", - "OG0000100.fa", - "OG0000101.fa", - "OG0000102.fa", - "OG0000103.fa", - "OG0000104.fa", - "OG0000105.fa", - "OG0000106.fa", - "OG0000107.fa", - "OG0000108.fa", - "OG0000109.fa", - "OG0000110.fa", - "OG0000111.fa", - "OG0000112.fa", - "OG0000113.fa", - "OG0000114.fa", - "OG0000115.fa", - "OG0000116.fa", - "OG0000117.fa", - "OG0000118.fa", - "OG0000119.fa", - "OG0000120.fa", - "OG0000121.fa", - "OG0000122.fa", - "OG0000123.fa", - "OG0000124.fa", - "OG0000125.fa", - "OG0000126.fa", - "OG0000127.fa", - "OG0000128.fa", - "OG0000129.fa", - "OG0000130.fa", - "OG0000131.fa", - "OG0000132.fa", - "OG0000133.fa", - "OG0000134.fa", - "OG0000135.fa", - "OG0000136.fa", - "OG0000137.fa", - "OG0000138.fa", - "OG0000139.fa", - "OG0000140.fa", - "OG0000141.fa", - "OG0000142.fa", - "OG0000143.fa", - "OG0000144.fa", - "OG0000145.fa", - "OG0000146.fa", - "OG0000147.fa", - "OG0000148.fa", - "OG0000149.fa", - "OG0000150.fa", - "OG0000151.fa", - "OG0000152.fa", - "OG0000153.fa", - "OG0000154.fa", - "OG0000155.fa", - "OG0000156.fa", - "OG0000157.fa", - "OG0000158.fa", - "OG0000159.fa", - "OG0000160.fa", - "OG0000161.fa", - "OG0000162.fa", - "OG0000163.fa", - "OG0000164.fa", - "OG0000165.fa", - "OG0000166.fa", - "OG0000167.fa", - "OG0000168.fa", - "OG0000169.fa", - "OG0000170.fa", - "OG0000171.fa", - "OG0000172.fa", - "OG0000173.fa", - "OG0000174.fa", - "OG0000175.fa", - "OG0000176.fa", - "OG0000177.fa", - "OG0000178.fa", - "OG0000179.fa", - "OG0000180.fa", - "OG0000181.fa", - "OG0000182.fa", - "OG0000183.fa", - "OG0000184.fa", - "OG0000185.fa", - "OG0000186.fa", - "OG0000187.fa", - "OG0000188.fa", - "OG0000189.fa", - "OG0000190.fa", - "OG0000191.fa", - "OG0000192.fa", - "OG0000193.fa", - "OG0000194.fa", - "OG0000195.fa", - "OG0000196.fa", - "OG0000197.fa", - "OG0000198.fa", - "OG0000199.fa", - "OG0000200.fa", - "OG0000201.fa", - "OG0000202.fa", - "OG0000203.fa", - "OG0000204.fa", - "OG0000205.fa", - "OG0000206.fa", - "OG0000207.fa", - "OG0000208.fa", - "OG0000209.fa", - "OG0000210.fa", - "OG0000211.fa", - "OG0000212.fa", - "OG0000213.fa", - "OG0000214.fa", - "OG0000215.fa", - "OG0000216.fa", - "OG0000217.fa", - "OG0000218.fa", - "OG0000219.fa", - "OG0000220.fa", - "OG0000221.fa", - "OG0000222.fa", - "OG0000223.fa", - "OG0000224.fa", - "OG0000225.fa", - "OG0000226.fa", - "OG0000227.fa", - "OG0000228.fa", - "OG0000229.fa", - "OG0000230.fa", - "OG0000231.fa", - "OG0000232.fa", - "OG0000233.fa", - "OG0000234.fa", - "OG0000235.fa", - "OG0000236.fa", - "OG0000237.fa", - "OG0000238.fa", - "OG0000239.fa", - "OG0000240.fa", - "OG0000241.fa", - "OG0000242.fa", - "OG0000243.fa", - "OG0000244.fa", - "OG0000245.fa", - "OrthoFinder_graph.txt", - "Orthogroups.GeneCount.tsv", - "Orthogroups.tsv", - "Orthogroups.txt", - "Orthogroups_SingleCopyOrthologues.txt", - "Orthogroups_SpeciesOverlaps.tsv", - "Orthogroups_UnassignedGenes.tsv", - "OrthologuesStats_Totals.tsv", - "OrthologuesStats_many-to-many.tsv", - "OrthologuesStats_many-to-one.tsv", - "OrthologuesStats_one-to-many.tsv", - "OrthologuesStats_one-to-one.tsv", - "SequenceIDs.txt", - "SimpleTest.phy", - "SimpleTest.phy_fastme_stat.txt", - "SimpleTest.tre", - "Species0.fa", - "Species0.fa", - "Species1.fa", - "SpeciesIDs.txt", - "SpeciesTree_Gene_Duplications_0.5_Support.txt", - "SpeciesTree_rooted.txt", - "SpeciesTree_rooted_ids.txt", - "SpeciesTree_rooted_node_labels.txt", - "SpeciesTree_unrooted.txt", - "SpeciesTree_unrooted_ids.txt", - "Statistics_Overall.tsv", - "Statistics_PerSpecies.tsv", - "candidatus_portiera_aleyrodidarum.tsv", - "candidatus_portiera_aleyrodidarum.tsv", - "candidatus_portiera_aleyrodidarum__v__sarscov2.tsv", - "clusters_OrthoFinder_I1.5.txt", - "clusters_OrthoFinder_I1.5.txt_id_pairs.txt", - "diamondDBSpecies0.dmnd", - "diamondDBSpecies0.dmnd", - "diamondDBSpecies1.dmnd", - "sarscov2.tsv", - "sarscov2.tsv", - "sarscov2__v__candidatus_portiera_aleyrodidarum.tsv", - "test_search_results.txt.gz" - ], - [ - "Statistics_PerSpecies.tsv:md5,984b5011a34d54527fe17896bfa36a2d", - "SpeciesTree_Gene_Duplications_0.5_Support.txt:md5,8b7a673e2e8b6d1aeb697f2bb88afa18", - "SpeciesTree_rooted.txt:md5,4d5ea525feebe479fca0c0768271ba81" - ], - "versions.yml:md5,86b472c85626aac1840eec0769016f5c" - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" - }, - "timestamp": "2024-06-07T21:33:50.902329" - }, "sarscov2 - candidatus_portiera_aleyrodidarum - proteome - stub": { "content": [ { @@ -430,9 +111,24 @@ } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-08-25T13:24:52.443958997" + }, + "sarscov2 - candidatus_portiera_aleyrodidarum - proteome": { + "content": [ + [ + "Statistics_PerSpecies.tsv:md5,984b5011a34d54527fe17896bfa36a2d", + "SpeciesTree_rooted.txt:md5,4d5ea525feebe479fca0c0768271ba81", + "SpeciesTree_Gene_Duplications_0.5_Support.txt:md5,8b7a673e2e8b6d1aeb697f2bb88afa18" + ], + "versions.yml:md5,86b472c85626aac1840eec0769016f5c" + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" }, - "timestamp": "2024-06-07T16:00:56.530716" + "timestamp": "2024-08-25T13:23:04.789805577" } } \ No newline at end of file From cac3bf2c792fc1d2fe0e09b777b2bea3ff843810 Mon Sep 17 00:00:00 2001 From: chriswyatt1 Date: Sun, 25 Aug 2024 15:46:08 +0000 Subject: [PATCH 02/11] Working resume test --- .../nf-core/orthofinder/tests/main.nf.test | 47 +++++++++++++++++++ .../orthofinder/tests/main.nf.test.snap | 32 ++++++------- 2 files changed, 63 insertions(+), 16 deletions(-) diff --git a/modules/nf-core/orthofinder/tests/main.nf.test b/modules/nf-core/orthofinder/tests/main.nf.test index 0be237437eb..d94371fc57b 100644 --- a/modules/nf-core/orthofinder/tests/main.nf.test +++ b/modules/nf-core/orthofinder/tests/main.nf.test @@ -60,6 +60,53 @@ nextflow_process { } + + test("sarscov2 - candidatus_portiera_aleyrodidarum - proteome - resume") { + + when { + process { + """ + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/proteome.fasta', checkIfExists: true) + .copyTo("${workDir}/sarscov2.fasta") + + def file_a = file("${workDir}/sarscov2.fasta", checkIfExists:true) + def file_c = file("s3://temporary-nfcore-module-data/test/WorkingDirectory") + input[0] = [ + [ id:'test_2', single_end:false ], + [ file_a ] + ] + input[1] = file_c + """ + } + } + + then { + assert process.success + + def all_files = [] + + file(process.out.orthofinder[0][1]).eachFileRecurse (FileType.FILES) { file -> + all_files << file + } + + //def all_file_names = all_files.collect { it.name }.sort(false) + + def stable_file_names = [ + 'Statistics_PerSpecies.tsv', + 'SpeciesTree_Gene_Duplications_0.5_Support.txt', + 'SpeciesTree_rooted.txt' + ] + + def stable_files = all_files.findAll { it.name in stable_file_names } + + assert snapshot( + stable_files, + process.out.versions[0] + ).match() + } + + } + test("sarscov2 - candidatus_portiera_aleyrodidarum - proteome - stub") { options '-stub' diff --git a/modules/nf-core/orthofinder/tests/main.nf.test.snap b/modules/nf-core/orthofinder/tests/main.nf.test.snap index 35dbff39953..63188ba45bd 100644 --- a/modules/nf-core/orthofinder/tests/main.nf.test.snap +++ b/modules/nf-core/orthofinder/tests/main.nf.test.snap @@ -1,4 +1,19 @@ { + "sarscov2 - candidatus_portiera_aleyrodidarum - proteome": { + "content": [ + [ + "Statistics_PerSpecies.tsv:md5,984b5011a34d54527fe17896bfa36a2d", + "SpeciesTree_rooted.txt:md5,4d5ea525feebe479fca0c0768271ba81", + "SpeciesTree_Gene_Duplications_0.5_Support.txt:md5,8b7a673e2e8b6d1aeb697f2bb88afa18" + ], + "versions.yml:md5,86b472c85626aac1840eec0769016f5c" + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-08-25T15:28:06.023586897" + }, "sarscov2 - candidatus_portiera_aleyrodidarum - proteome - stub": { "content": [ { @@ -114,21 +129,6 @@ "nf-test": "0.9.0", "nextflow": "24.04.4" }, - "timestamp": "2024-08-25T13:24:52.443958997" - }, - "sarscov2 - candidatus_portiera_aleyrodidarum - proteome": { - "content": [ - [ - "Statistics_PerSpecies.tsv:md5,984b5011a34d54527fe17896bfa36a2d", - "SpeciesTree_rooted.txt:md5,4d5ea525feebe479fca0c0768271ba81", - "SpeciesTree_Gene_Duplications_0.5_Support.txt:md5,8b7a673e2e8b6d1aeb697f2bb88afa18" - ], - "versions.yml:md5,86b472c85626aac1840eec0769016f5c" - ], - "meta": { - "nf-test": "0.9.0", - "nextflow": "24.04.4" - }, - "timestamp": "2024-08-25T13:23:04.789805577" + "timestamp": "2024-08-25T15:28:29.054193695" } } \ No newline at end of file From 8b1099534ec574d240df28074e01ffcdfc85383e Mon Sep 17 00:00:00 2001 From: chriswyatt1 Date: Sun, 25 Aug 2024 16:58:12 +0000 Subject: [PATCH 03/11] With existing data from datasets and chose stable expected outputs. --- modules/nf-core/orthofinder/main.nf | 9 ++++++--- .../nf-core/orthofinder/tests/main.nf.test | 10 +++------- .../orthofinder/tests/main.nf.test.snap | 19 +++++++++++++++++-- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/modules/nf-core/orthofinder/main.nf b/modules/nf-core/orthofinder/main.nf index fd4aa2128d0..ab3fd829d7e 100644 --- a/modules/nf-core/orthofinder/main.nf +++ b/modules/nf-core/orthofinder/main.nf @@ -34,10 +34,13 @@ process ORTHOFINDER { -n $prefix \\ $include_command $args + if [ -e input/OrthoFinder/Results_$prefix ]; then + mv input/OrthoFinder/Results_$prefix $prefix + fi - mv \\ - input/OrthoFinder/Results_$prefix \\ - $prefix + if [ -e WorkingDirectory/OrthoFinder/Results_$prefix ]; then + mv WorkingDirectory/OrthoFinder/Results_$prefix $prefix + fi cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/orthofinder/tests/main.nf.test b/modules/nf-core/orthofinder/tests/main.nf.test index d94371fc57b..c39c56984c6 100644 --- a/modules/nf-core/orthofinder/tests/main.nf.test +++ b/modules/nf-core/orthofinder/tests/main.nf.test @@ -42,8 +42,6 @@ nextflow_process { all_files << file } - //def all_file_names = all_files.collect { it.name }.sort(false) - def stable_file_names = [ 'Statistics_PerSpecies.tsv', 'SpeciesTree_Gene_Duplications_0.5_Support.txt', @@ -69,7 +67,7 @@ nextflow_process { file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/proteome.fasta', checkIfExists: true) .copyTo("${workDir}/sarscov2.fasta") - def file_a = file("${workDir}/sarscov2.fasta", checkIfExists:true) + def file_a = file("https://raw.githubusercontent.com/nf-core/test-datasets/proteinfold/testdata/sequences/H1065.fasta") def file_c = file("s3://temporary-nfcore-module-data/test/WorkingDirectory") input[0] = [ [ id:'test_2', single_end:false ], @@ -89,12 +87,10 @@ nextflow_process { all_files << file } - //def all_file_names = all_files.collect { it.name }.sort(false) - def stable_file_names = [ 'Statistics_PerSpecies.tsv', - 'SpeciesTree_Gene_Duplications_0.5_Support.txt', - 'SpeciesTree_rooted.txt' + 'OrthologuesStats_Totals.tsv', + 'Duplications_per_Species_Tree_Node.tsv' ] def stable_files = all_files.findAll { it.name in stable_file_names } diff --git a/modules/nf-core/orthofinder/tests/main.nf.test.snap b/modules/nf-core/orthofinder/tests/main.nf.test.snap index 63188ba45bd..7ed1cf01600 100644 --- a/modules/nf-core/orthofinder/tests/main.nf.test.snap +++ b/modules/nf-core/orthofinder/tests/main.nf.test.snap @@ -12,7 +12,7 @@ "nf-test": "0.9.0", "nextflow": "24.04.4" }, - "timestamp": "2024-08-25T15:28:06.023586897" + "timestamp": "2024-08-25T16:54:47.541031863" }, "sarscov2 - candidatus_portiera_aleyrodidarum - proteome - stub": { "content": [ @@ -129,6 +129,21 @@ "nf-test": "0.9.0", "nextflow": "24.04.4" }, - "timestamp": "2024-08-25T15:28:29.054193695" + "timestamp": "2024-08-25T16:55:44.436567752" + }, + "sarscov2 - candidatus_portiera_aleyrodidarum - proteome - resume": { + "content": [ + [ + "Statistics_PerSpecies.tsv:md5,83174c383b6c6828d1cc9b3be1679890", + "OrthologuesStats_Totals.tsv:md5,20d243abef226051a43cb37e922fc3eb", + "Duplications_per_Species_Tree_Node.tsv:md5,addc6f5ceec40bd82b00038d1872a27c" + ], + "versions.yml:md5,86b472c85626aac1840eec0769016f5c" + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-08-25T16:55:21.180445341" } } \ No newline at end of file From d9d8df3526cb4fd6314496e2d65a5eb7d723a6d9 Mon Sep 17 00:00:00 2001 From: chriswyatt1 Date: Wed, 28 Aug 2024 17:32:06 +0000 Subject: [PATCH 04/11] updated snap shot (sorted) --- modules/nf-core/orthofinder/main.nf | 4 ++-- modules/nf-core/orthofinder/tests/main.nf.test | 4 ++-- .../nf-core/orthofinder/tests/main.nf.test.snap | 14 +++++++------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/modules/nf-core/orthofinder/main.nf b/modules/nf-core/orthofinder/main.nf index ab3fd829d7e..9de92508dad 100644 --- a/modules/nf-core/orthofinder/main.nf +++ b/modules/nf-core/orthofinder/main.nf @@ -38,8 +38,8 @@ process ORTHOFINDER { mv input/OrthoFinder/Results_$prefix $prefix fi - if [ -e WorkingDirectory/OrthoFinder/Results_$prefix ]; then - mv WorkingDirectory/OrthoFinder/Results_$prefix $prefix + if [ -e ${prior_run}/OrthoFinder/Results_$prefix ]; then + mv ${prior_run}/OrthoFinder/Results_$prefix $prefix fi cat <<-END_VERSIONS > versions.yml diff --git a/modules/nf-core/orthofinder/tests/main.nf.test b/modules/nf-core/orthofinder/tests/main.nf.test index c39c56984c6..3f201ee9803 100644 --- a/modules/nf-core/orthofinder/tests/main.nf.test +++ b/modules/nf-core/orthofinder/tests/main.nf.test @@ -51,7 +51,7 @@ nextflow_process { def stable_files = all_files.findAll { it.name in stable_file_names } assert snapshot( - stable_files, + stable_files.toSorted(), process.out.versions[0] ).match() } @@ -96,7 +96,7 @@ nextflow_process { def stable_files = all_files.findAll { it.name in stable_file_names } assert snapshot( - stable_files, + stable_files.toSorted(), process.out.versions[0] ).match() } diff --git a/modules/nf-core/orthofinder/tests/main.nf.test.snap b/modules/nf-core/orthofinder/tests/main.nf.test.snap index 7ed1cf01600..2c5debd80ac 100644 --- a/modules/nf-core/orthofinder/tests/main.nf.test.snap +++ b/modules/nf-core/orthofinder/tests/main.nf.test.snap @@ -3,8 +3,8 @@ "content": [ [ "Statistics_PerSpecies.tsv:md5,984b5011a34d54527fe17896bfa36a2d", - "SpeciesTree_rooted.txt:md5,4d5ea525feebe479fca0c0768271ba81", - "SpeciesTree_Gene_Duplications_0.5_Support.txt:md5,8b7a673e2e8b6d1aeb697f2bb88afa18" + "SpeciesTree_Gene_Duplications_0.5_Support.txt:md5,8b7a673e2e8b6d1aeb697f2bb88afa18", + "SpeciesTree_rooted.txt:md5,4d5ea525feebe479fca0c0768271ba81" ], "versions.yml:md5,86b472c85626aac1840eec0769016f5c" ], @@ -12,7 +12,7 @@ "nf-test": "0.9.0", "nextflow": "24.04.4" }, - "timestamp": "2024-08-25T16:54:47.541031863" + "timestamp": "2024-08-28T17:28:58.249427812" }, "sarscov2 - candidatus_portiera_aleyrodidarum - proteome - stub": { "content": [ @@ -129,14 +129,14 @@ "nf-test": "0.9.0", "nextflow": "24.04.4" }, - "timestamp": "2024-08-25T16:55:44.436567752" + "timestamp": "2024-08-28T17:29:41.511039749" }, "sarscov2 - candidatus_portiera_aleyrodidarum - proteome - resume": { "content": [ [ - "Statistics_PerSpecies.tsv:md5,83174c383b6c6828d1cc9b3be1679890", + "Duplications_per_Species_Tree_Node.tsv:md5,addc6f5ceec40bd82b00038d1872a27c", "OrthologuesStats_Totals.tsv:md5,20d243abef226051a43cb37e922fc3eb", - "Duplications_per_Species_Tree_Node.tsv:md5,addc6f5ceec40bd82b00038d1872a27c" + "Statistics_PerSpecies.tsv:md5,83174c383b6c6828d1cc9b3be1679890" ], "versions.yml:md5,86b472c85626aac1840eec0769016f5c" ], @@ -144,6 +144,6 @@ "nf-test": "0.9.0", "nextflow": "24.04.4" }, - "timestamp": "2024-08-25T16:55:21.180445341" + "timestamp": "2024-08-28T17:29:30.142704901" } } \ No newline at end of file From 4d2d5cee10ddbe80ad386450e4123f914412ca74 Mon Sep 17 00:00:00 2001 From: chriswyatt1 Date: Sat, 31 Aug 2024 10:12:44 +0000 Subject: [PATCH 05/11] fix test to have untar-ed workdir dir input --- modules/nf-core/orthofinder/main.nf | 7 +++-- .../nf-core/orthofinder/tests/main.nf.test | 28 ++++++++++++++++--- .../orthofinder/tests/main.nf.test.snap | 28 +++++++++++++++++-- 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/modules/nf-core/orthofinder/main.nf b/modules/nf-core/orthofinder/main.nf index 9de92508dad..e22125deb81 100644 --- a/modules/nf-core/orthofinder/main.nf +++ b/modules/nf-core/orthofinder/main.nf @@ -9,11 +9,12 @@ process ORTHOFINDER { input: tuple val(meta), path(fastas, stageAs: 'input/') - path(prior_run) + tuple val(meta2), path(prior_run) output: - tuple val(meta), path("$prefix") , emit: orthofinder - path "versions.yml" , emit: versions + tuple val(meta), path("$prefix") , emit: orthofinder + tuple val(meta), path("$prefix/WorkingDirectory") , emit: working + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when diff --git a/modules/nf-core/orthofinder/tests/main.nf.test b/modules/nf-core/orthofinder/tests/main.nf.test index 3f201ee9803..8772c89e461 100644 --- a/modules/nf-core/orthofinder/tests/main.nf.test +++ b/modules/nf-core/orthofinder/tests/main.nf.test @@ -28,7 +28,10 @@ nextflow_process { [ id:'test', single_end:false ], [ file_a, file_b ] ] - input[1] = [] + input[1] = [ + [], + [] + ] """ } } @@ -61,6 +64,20 @@ nextflow_process { test("sarscov2 - candidatus_portiera_aleyrodidarum - proteome - resume") { + + setup { + run("UNTAR") { + script "../../untar/main.nf" + process { + """ + input[0] = [ [ id:'test1' ], // meta map + file("s3://temporary-nfcore-module-data/test/WorkingDirectory.tar.gz") + ] + """ + } + } + } + when { process { """ @@ -68,12 +85,12 @@ nextflow_process { .copyTo("${workDir}/sarscov2.fasta") def file_a = file("https://raw.githubusercontent.com/nf-core/test-datasets/proteinfold/testdata/sequences/H1065.fasta") - def file_c = file("s3://temporary-nfcore-module-data/test/WorkingDirectory") + def file_c = UNTAR.out.untar input[0] = [ [ id:'test_2', single_end:false ], [ file_a ] ] - input[1] = file_c + input[1] = UNTAR.out.untar """ } } @@ -123,7 +140,10 @@ nextflow_process { [ id:'test', single_end:false ], [ file_a, file_b ] ] - input[1] = [] + input[1] = [ + [], + [] + ] """ } } diff --git a/modules/nf-core/orthofinder/tests/main.nf.test.snap b/modules/nf-core/orthofinder/tests/main.nf.test.snap index 2c5debd80ac..cd45b2699d3 100644 --- a/modules/nf-core/orthofinder/tests/main.nf.test.snap +++ b/modules/nf-core/orthofinder/tests/main.nf.test.snap @@ -12,7 +12,7 @@ "nf-test": "0.9.0", "nextflow": "24.04.4" }, - "timestamp": "2024-08-28T17:28:58.249427812" + "timestamp": "2024-08-31T10:03:47.190331668" }, "sarscov2 - candidatus_portiera_aleyrodidarum - proteome - stub": { "content": [ @@ -68,6 +68,17 @@ ] ], "1": [ + [ + { + "id": "test", + "single_end": false + }, + [ + + ] + ] + ], + "2": [ "versions.yml:md5,86b472c85626aac1840eec0769016f5c" ], "orthofinder": [ @@ -122,6 +133,17 @@ ], "versions": [ "versions.yml:md5,86b472c85626aac1840eec0769016f5c" + ], + "working": [ + [ + { + "id": "test", + "single_end": false + }, + [ + + ] + ] ] } ], @@ -129,7 +151,7 @@ "nf-test": "0.9.0", "nextflow": "24.04.4" }, - "timestamp": "2024-08-28T17:29:41.511039749" + "timestamp": "2024-08-31T10:04:30.61014619" }, "sarscov2 - candidatus_portiera_aleyrodidarum - proteome - resume": { "content": [ @@ -144,6 +166,6 @@ "nf-test": "0.9.0", "nextflow": "24.04.4" }, - "timestamp": "2024-08-28T17:29:30.142704901" + "timestamp": "2024-08-31T10:04:20.459213823" } } \ No newline at end of file From e8ccdd7cdb9ce287a63a3163cfb4aaea83e570e6 Mon Sep 17 00:00:00 2001 From: chriswyatt1 Date: Mon, 2 Sep 2024 15:04:26 +0000 Subject: [PATCH 06/11] Fix folder direction --- modules/nf-core/orthofinder/tests/main.nf.test | 2 +- modules/nf-core/orthofinder/tests/main.nf.test.snap | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/nf-core/orthofinder/tests/main.nf.test b/modules/nf-core/orthofinder/tests/main.nf.test index 8772c89e461..836ae8b3606 100644 --- a/modules/nf-core/orthofinder/tests/main.nf.test +++ b/modules/nf-core/orthofinder/tests/main.nf.test @@ -71,7 +71,7 @@ nextflow_process { process { """ input[0] = [ [ id:'test1' ], // meta map - file("s3://temporary-nfcore-module-data/test/WorkingDirectory.tar.gz") + file(params.modules_testdata_base_path + 'delete_me/orthofinder/WorkingDirectory.tar.gz', checkIfExists: true) ] """ } diff --git a/modules/nf-core/orthofinder/tests/main.nf.test.snap b/modules/nf-core/orthofinder/tests/main.nf.test.snap index cd45b2699d3..0902653ed68 100644 --- a/modules/nf-core/orthofinder/tests/main.nf.test.snap +++ b/modules/nf-core/orthofinder/tests/main.nf.test.snap @@ -12,7 +12,7 @@ "nf-test": "0.9.0", "nextflow": "24.04.4" }, - "timestamp": "2024-08-31T10:03:47.190331668" + "timestamp": "2024-09-02T15:01:36.619182617" }, "sarscov2 - candidatus_portiera_aleyrodidarum - proteome - stub": { "content": [ @@ -151,7 +151,7 @@ "nf-test": "0.9.0", "nextflow": "24.04.4" }, - "timestamp": "2024-08-31T10:04:30.61014619" + "timestamp": "2024-09-02T15:02:08.212845312" }, "sarscov2 - candidatus_portiera_aleyrodidarum - proteome - resume": { "content": [ @@ -166,6 +166,6 @@ "nf-test": "0.9.0", "nextflow": "24.04.4" }, - "timestamp": "2024-08-31T10:04:20.459213823" + "timestamp": "2024-09-02T15:01:58.264867297" } } \ No newline at end of file From 2c5e9dc4815db614390ded22944040edac6490af Mon Sep 17 00:00:00 2001 From: chriswyatt1 Date: Mon, 2 Sep 2024 15:21:10 +0000 Subject: [PATCH 07/11] Add new output inputs to yml --- modules/nf-core/orthofinder/meta.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/modules/nf-core/orthofinder/meta.yml b/modules/nf-core/orthofinder/meta.yml index 23cc5320bcb..8c7689b3e86 100644 --- a/modules/nf-core/orthofinder/meta.yml +++ b/modules/nf-core/orthofinder/meta.yml @@ -29,6 +29,15 @@ input: type: list description: Input fasta files pattern: "*.{fa,faa,fasta,fas,pep}" + - meta2: + type: map + description: | + Groovy Map containing a name + e.g. `[ id:'folder1' ]` + - prior_run: + type: directory + description: | + A folder container containing a previous WorkingDirectory from Orthofinder. output: - meta: @@ -43,6 +52,9 @@ output: - orthofinder: type: directory description: Orthofinder output directory + - working: + type: directory + description: Orthofinder output WorkingDirectory (used for the orthofinder resume function) authors: - "@GallVp" From d5ccac2e8098ad689c21f793622f3dbe5b71e18a Mon Sep 17 00:00:00 2001 From: chriswyatt1 Date: Mon, 2 Sep 2024 15:30:38 +0000 Subject: [PATCH 08/11] fix tag --- modules/nf-core/orthofinder/tests/main.nf.test | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/nf-core/orthofinder/tests/main.nf.test b/modules/nf-core/orthofinder/tests/main.nf.test index 836ae8b3606..aa68d1d21ab 100644 --- a/modules/nf-core/orthofinder/tests/main.nf.test +++ b/modules/nf-core/orthofinder/tests/main.nf.test @@ -9,6 +9,7 @@ nextflow_process { tag "modules" tag "modules_nfcore" tag "orthofinder" + tag "untar" test("sarscov2 - candidatus_portiera_aleyrodidarum - proteome") { From fd37df1cceca908451c297a2ba788c4bece3d8dc Mon Sep 17 00:00:00 2001 From: chriswyatt1 Date: Tue, 3 Sep 2024 11:20:40 +0000 Subject: [PATCH 09/11] working conda on gitpod --- modules/nf-core/orthofinder/environment.yml | 6 ++---- modules/nf-core/orthofinder/main.nf | 3 ++- modules/nf-core/orthofinder/tests/main.nf.test.snap | 6 +++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/modules/nf-core/orthofinder/environment.yml b/modules/nf-core/orthofinder/environment.yml index f93b9bb0e88..a0ca0350265 100644 --- a/modules/nf-core/orthofinder/environment.yml +++ b/modules/nf-core/orthofinder/environment.yml @@ -1,9 +1,7 @@ ---- -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json -name: "orthofinder" +name: orthofinder channels: - conda-forge - bioconda - defaults dependencies: - - "bioconda::orthofinder=2.5.5" + - bioconda::orthofinder=2.5.5 diff --git a/modules/nf-core/orthofinder/main.nf b/modules/nf-core/orthofinder/main.nf index e22125deb81..a47c4dea36c 100644 --- a/modules/nf-core/orthofinder/main.nf +++ b/modules/nf-core/orthofinder/main.nf @@ -33,7 +33,8 @@ process ORTHOFINDER { -p temp_pickle \\ -f input \\ -n $prefix \\ - $include_command $args + $include_command \\ + $args if [ -e input/OrthoFinder/Results_$prefix ]; then mv input/OrthoFinder/Results_$prefix $prefix diff --git a/modules/nf-core/orthofinder/tests/main.nf.test.snap b/modules/nf-core/orthofinder/tests/main.nf.test.snap index 0902653ed68..f2c7b916814 100644 --- a/modules/nf-core/orthofinder/tests/main.nf.test.snap +++ b/modules/nf-core/orthofinder/tests/main.nf.test.snap @@ -12,7 +12,7 @@ "nf-test": "0.9.0", "nextflow": "24.04.4" }, - "timestamp": "2024-09-02T15:01:36.619182617" + "timestamp": "2024-09-03T10:59:02.895708598" }, "sarscov2 - candidatus_portiera_aleyrodidarum - proteome - stub": { "content": [ @@ -151,7 +151,7 @@ "nf-test": "0.9.0", "nextflow": "24.04.4" }, - "timestamp": "2024-09-02T15:02:08.212845312" + "timestamp": "2024-09-03T11:07:31.319665056" }, "sarscov2 - candidatus_portiera_aleyrodidarum - proteome - resume": { "content": [ @@ -166,6 +166,6 @@ "nf-test": "0.9.0", "nextflow": "24.04.4" }, - "timestamp": "2024-09-02T15:01:58.264867297" + "timestamp": "2024-09-03T11:04:10.916947006" } } \ No newline at end of file From b5a3cf9e68ebce7a04aac09cbb839f5582e06307 Mon Sep 17 00:00:00 2001 From: chriswyatt1 Date: Thu, 5 Sep 2024 10:42:57 +0000 Subject: [PATCH 10/11] force diamond install to 2.1.9 --- modules/nf-core/orthofinder/environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/nf-core/orthofinder/environment.yml b/modules/nf-core/orthofinder/environment.yml index a0ca0350265..e27de0e8eaf 100644 --- a/modules/nf-core/orthofinder/environment.yml +++ b/modules/nf-core/orthofinder/environment.yml @@ -4,4 +4,5 @@ channels: - bioconda - defaults dependencies: + - bioconda::diamond=2.1.9 - bioconda::orthofinder=2.5.5 From f271711443ad6fe016e661245e22bd163233ee48 Mon Sep 17 00:00:00 2001 From: Usman Rashid Date: Fri, 6 Sep 2024 10:00:03 +1200 Subject: [PATCH 11/11] Removed defaults --- modules/nf-core/orthofinder/environment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/nf-core/orthofinder/environment.yml b/modules/nf-core/orthofinder/environment.yml index e27de0e8eaf..4239c4c1815 100644 --- a/modules/nf-core/orthofinder/environment.yml +++ b/modules/nf-core/orthofinder/environment.yml @@ -2,7 +2,6 @@ name: orthofinder channels: - conda-forge - bioconda - - defaults dependencies: - bioconda::diamond=2.1.9 - bioconda::orthofinder=2.5.5