Merge pull request #182 from nf-core/nft-utils

Adopt nft-utils

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ env:
   NXF_ANSI_LOG: false
   NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
   NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
-  NFT_VER: "0.9.0"
+  NFT_VER: "0.9.2"
   NFT_WORKDIR: "~"
   NFT_DIFF: "pdiff"
   NFT_DIFF_ARGS: "--line-numbers --expand-tabs=2"
@@ -101,13 +101,15 @@ jobs:
       - name: Clean up Disk space
         uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
 
-      - name: Run Tests (Shard ${{ matrix.shard }}/${{ strategy.job-total }})
+      - name: Run Tests (Shard ${{ matrix.shard }}/4)
         run: |
+          NFT_WORKDIR=~ \
           nf-test test \
               --ci \
-              --shard ${{ matrix.shard }}/${{ strategy.job-total }} \
+              --shard ${{ matrix.shard }}/4 \
               --changed-since HEAD^ \
               --profile "+${{ matrix.profile }}" \
+              --verbose \
               --filter pipeline
 
       - name: Publish Test Report

.gitignore

@@ -7,4 +7,4 @@ testing/
 testing*
 *.pyc
 null/
-.nf-test
+.nf-test*

conf/modules.config

@@ -176,7 +176,7 @@ process {
     // Alignment post-processing //
     ///////////////////////////////
 
-    withName: '.*:BAM_SORT_STATS_SAMTOOLS:.*|.*:BAM_SORT_STATS_SAMTOOLS_GENOME:.*' {
+    withName: '.*:BAM_SORT_STATS_SAMTOOLS:.*' {
         ext.prefix = { "${meta.id}.sorted.bam" }
         publishDir = [
             path: { "${params.outdir}/${params.aligner}/samtools_stats" },
@@ -185,7 +185,7 @@ process {
         ]
     }
 
-    withName: '.*:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_.*|.*:BAM_SORT_STATS_SAMTOOLS_GENOME:SAMTOOLS_.*' {
+    withName: '.*:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_.*' {
         ext.prefix = { "${meta.id}.sorted" }
         publishDir = [
             path: { "${params.outdir}/${params.aligner}" },
@@ -194,6 +194,25 @@ process {
         ]
     }
 
+    withName: ".*:FASTQ_ALIGN_STAR:BAM_SORT_STATS_SAMTOOLS_GENOME:.*" {
+        ext.prefix = {"${meta.id}_genome"}
+        publishDir = [
+            path: { "${params.outdir}/${params.aligner}/samtools_stats" },
+            mode: params.publish_dir_mode,
+            pattern: "*.{stats,flagstat,idxstats}"
+        ]
+    }
+
+    withName: ".*:FASTQ_ALIGN_STAR:BAM_SORT_STATS_SAMTOOLS_TRANSCRIPTOME:.*" {
+        ext.prefix = {"${meta.id}_transcriptome"}
+        publishDir = [
+            path: { "${params.outdir}/${params.aligner}/samtools_stats" },
+            mode: params.publish_dir_mode,
+            pattern: "*.{stats,flagstat,idxstats}"
+        ]
+    }
+
+
     if(params.with_umi) {
         withName: '.*:BAM_DEDUP_STATS_SAMTOOLS_UMITOOLS:UMITOOLS_DEDUP' {
             ext.args = { [
@@ -362,7 +381,7 @@ process {
             [
                 path: { "${params.outdir}/transcript_identification/pints" },
                 mode: params.publish_dir_mode,
-                saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
+                saveAs: { filename -> filename.equals('versions.yml') || filename.endsWith('.log') ? null : filename }
             ],
         ]
     }
@@ -439,6 +458,12 @@ process {
         ]
     }
 
+    withName: BED2SAF  {
+        publishDir = [
+            enabled: false
+        ]
+    }
+
     withName: SUBREAD_FEATURECOUNTS_PREDICTED {
         ext.prefix = { "${meta.id}-group_${annotation.baseName}-transcripts" }
         ext.args = '-F "SAF"'

nf-test.config

@@ -5,7 +5,8 @@ config {
     configFile "tests/nextflow.config"
     profile "test"
     plugins {
-        load "[email protected]"
+        load "[email protected]"
+        load "[email protected]"
     }
     triggers "nextflow.config", "nf-test.config", "conf/modules.config", "conf/test.config", "tests/nextflow.config"
 

tests/.nftignore

@@ -0,0 +1,37 @@
+preprocessing/fastqc/*.{html,zip}
+preprocessing/fastp/*.{html,log}
+
+multiqc/multiqc_data/**
+multiqc/multiqc_data/*.{log,json}
+multiqc/multiqc_data/multiqc_general_stats.txt
+multiqc/multiqc_data/fastqc_top_overrepresented_sequences_table.txt
+multiqc/multiqc_data/homer_tagdir.txt
+multiqc/multiqc_data/rseqc_read_dups.txt
+multiqc/multiqc_data/multiqc_software_versions.txt
+multiqc/multiqc_data/multiqc_sources.txt
+multiqc/multiqc_report.html
+multiqc/multiqc_plots/**
+multiqc/multiqc_plots/{pdf,png,svg}/*.{pdf,png,svg}
+pipeline_info/*.{html,json,txt,yml}
+
+*/alignments/logs/*.txt
+star/log/*.Log.{final.out,out,progress.out}
+star/star/Log.out
+hisat2/log/*.hisat2.summary.log
+*/deduplicated/logs/*.txt
+*/{reports,summary}/*.{html,txt}
+**/*.command.log
+**/*.bedGraph.gz
+**/tagInfo.txt
+**/*.{bam,bai}
+**/*.featureCounts.txt
+**/*.featureCounts.txt.summary
+**/*.pdf
+
+transcript_identification/homer/*_tagdir/*
+transcript_identification/filtered/*_filtered.bed
+transcript_identification/intersect/*_intersect.bed
+
+quality_control/bbsplit/*.stats.txt
+quality_control/**
+**/DupRate_plot.pdf

tests/lib/UTILS.groovy

@@ -1,13 +1,4 @@
 class UTILS {
-    // Function to remove Nextflow version from pipeline_software_mqc_versions.yml
-    public static Object removeNextflowVersion(pipeline_software_mqc_versions) {
-        def softwareVersions = path(pipeline_software_mqc_versions).yaml
-        if (softwareVersions.containsKey("Workflow")) {
-            softwareVersions.Workflow.remove("Nextflow")
-        }
-        return softwareVersions
-    }
-
     // Recursively list all files in a directory and its sub-directories, matching a given suffix
     // TODO: use regex pattern instead of suffix?
     public static getAllFilesFromDir(dir, suffix) {

workflows/tests/aligner/bowtie2.nf.test

@@ -15,32 +15,26 @@ nextflow_pipeline {
         }
 
         then {
+            // stable_name: All files + folders in ${params.outdir}/ with a stable name
+            def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}', 'multiqc/multiqc_plots', 'multiqc/multiqc_plots/**'])
+            // stable_path: All files in ${params.outdir}/ with stable content
+            def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore')
+            // bam_files: All bam files
+            def bam_files = getAllFilesFromDir(params.outdir, include: ['**/*.bam'])
             assertAll(
                 { assert workflow.success },
-                { assert snapshot(UTILS.removeNextflowVersion("$outputDir/pipeline_info/nf_core_pipeline_software_mqc_versions.yml")).match("software_versions") },
                 { assert snapshot(
-                    workflow.trace.tasks().size(),
-                    UTILS.getAllFilesFromDir("$outputDir/preprocessing/fastp/", ".json"),
-                    // NOTE Not deterministic
-                    bam("$outputDir/bowtie2/cd4_REP1.sorted.bam").getHeaderMD5(),
-                    bam("$outputDir/bowtie2/cd4_REP2.sorted.bam").getHeaderMD5(),
-                    bam("$outputDir/bowtie2/jurkat.sorted.bam").getHeaderMD5(),
-                    path("$outputDir/bowtie2/log").list(),
-                    path("$outputDir/bowtie2/samtools_stats").list(),
-                    path("$outputDir/quality_control/bbsplit").list(),
-                    UTILS.getAllFilesFromDir("$outputDir/quality_control/preseq", ".txt"),
-                    path("$outputDir/quality_control/rseqc/read_duplication/xls").list(),
-                    path("$outputDir/quality_control/rseqc/read_duplication/rscript").list(),
-                    path("$outputDir/quality_control/rseqc/infer_experiment").list(),
-                    path("$outputDir/quality_control/rseqc/read_distribution").list(),
-                    path("$outputDir/coverage_graphs/").list(),
-                    path("$outputDir/transcript_identification/homer/cd4.bed"),
-                    path("$outputDir/transcript_identification/homer/jurkat.bed"),
-                    path("$outputDir/transcript_identification/intersect/").list(),
-                    path("$outputDir/transcript_identification/filtered/").list(),
-                    path("$outputDir/multiqc/multiqc_report.html").exists(),
-                ).match("output_files")
-                }
+                    // Number of tasks
+                    workflow.trace.succeeded().size(),
+                    // pipeline versions.yml file for multiqc from which Nextflow version is removed because we tests pipelines on multiple Nextflow versions
+                    removeNextflowVersion("$outputDir/pipeline_info/nf_core_pipeline_software_mqc_versions.yml"),
+                    // All stable path name
+                    stable_name,
+                    // All files with stable contents
+                    stable_path,
+                    // All bam files
+                    // FIXME bam_files.collect{ file -> [ file.getName(), bam(file.toString()).getReadsMD5() ] }
+                ).match() }
             )
         }
     }