Merge pull request #5 from SMD-Bioinformatics-Lund/master-html-creati…

…on-with-timestamps

Master html creation with timestamps

CHANGELOG.md

@@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 
 - Provided option to `save_merged_reads`
+- Provided ability to overwrite files in `publishDir` (`params.publish_dir_overwrite`)
 
 ## [v0.1.0]
 

bin/generate_master_html.py

@@ -78,16 +78,14 @@
 
 def find_date_in_string(input_string, date_pattern):
     """Searches for a date within a given string."""
-    date = ""
+    date = "(No date found)"
     match = re.search(date_pattern, input_string)
     if match:
-        date_regex = match.group(1)
-        if len(date_regex) == 8:
-            date = datetime.strptime(date_regex, "%Y%m%d").strftime("%d-%m-%Y")
-        elif len(date_regex) > 8:
-            date = date_regex
-        else:
-            date = "(No date found)"
+        date_matched = match.group(1)
+        if len(date_matched) == 8:
+            date = datetime.strptime(date_matched, "%Y%m%d").strftime("%d-%m-%Y")
+        elif len(date_matched) > 8:
+            date = date_matched
     return date
 
 def get_sample_ids(samplesheet_csv):

conf/cmd.config

@@ -10,9 +10,12 @@
 ----------------------------------------------------------------------------------------
 */
 
+process {
+    executor                    = 'slurm'
+    queue                       = 'low'
+}
+
 params {
-    process.executor            = 'slurm'
-    process.queue               = 'low'
     config_profile_name         = 'cmd profile'
     config_profile_description  = 'CMD High performance profile'
 
@@ -27,4 +30,12 @@ params {
     // Reads
     save_merged_reads           = false
 
+    // Boilerplate
+    publish_dir_overwrite       = true
+}
+
+singularity{
+	enabled = true
+	runOptions = '--bind /fs1/ --bind /fs2/ --bind /local/ --bind /mnt/beegfs/'
+	cacheDir = "/fs1/resources/containers/"
 }

conf/modules.config

@@ -15,13 +15,15 @@ process {
     publishDir = [
         path: { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" },
         mode: params.publish_dir_mode,
+        overwrite: params.publish_dir_overwrite,
         saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
     ]
 
     withName: MERGE_BARCODES {
         publishDir = [
             path: { "${params.outdir}/fastq_pass_merged" },
             mode: params.publish_dir_mode,
+            overwrite: params.publish_dir_overwrite,
             pattern: 'fastq_pass_merged',
             enabled: params.save_merged_reads
         ]
@@ -31,23 +33,26 @@ process {
         publishDir = [
             path: { "${params.outdir}/fastq_pass_merged" },
             mode: params.publish_dir_mode,
+            overwrite: params.publish_dir_overwrite,
             pattern: 'fastq_pass_merged',
             enabled: params.save_merged_reads
         ]
     }
 
     withName: GENERATE_INPUT {
         publishDir = [
-            path: { "${params.outdir}/" },
+            path: { "${params.outdir}" },
             mode: params.publish_dir_mode,
+            overwrite: params.publish_dir_overwrite,
             pattern: 'samplesheet_merged.csv'
         ]
     }
 
     withName: GENERATE_MASTER_HTML {
         publishDir = [
-            path: { "${params.outdir}/" },
+            path: { "${params.outdir}" },
             mode: params.publish_dir_mode,
+            overwrite: params.publish_dir_overwrite,
             pattern: 'master.html'
         ]
     }
@@ -56,6 +61,7 @@ process {
         publishDir = [
             path: { "${params.outdir}/nanoplot" },
             mode: params.publish_dir_mode,
+            overwrite: params.publish_dir_overwrite,
             pattern: '*.{html,log,png,txt,yml}'
         ]
         ext.prefix = { "${meta.id}_nanoplot_unprocessed" }                                  // A closure can be used to access variables defined in the script
@@ -66,40 +72,35 @@ process {
         publishDir = [
             path: { "${params.outdir}/nanoplot_processed_reads" },
             mode: params.publish_dir_mode,
+            overwrite: params.publish_dir_overwrite,
             pattern: '*.{html,log,png,txt,yml}'
         ]
         ext.prefix = { "${meta.id}_nanoplot_processed" }                   // A closure can be used to access variables defined in the script
 
     }
 
-
     withName: PORECHOP_ABI {
         publishDir = [
             path: { "${params.outdir}/porechop_abi" },
             mode: params.publish_dir_mode,
+            overwrite: params.publish_dir_overwrite,
             pattern: '*.{log,yml}'
         ]
     }
 
-
-
-
     withName: SAMPLESHEET_CHECK {
         publishDir = [
             path: { "${params.outdir}/pipeline_info" },
             mode: params.publish_dir_mode,
+            overwrite: params.publish_dir_overwrite,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
 
-
-
-
     withName: FASTQC {
         ext.args = '--quiet'
     }
 
-
     withName: CUTADAPT {
         ext.args = { [
         "--minimum-length 1",
@@ -116,43 +117,48 @@ process {
         publishDir = [
         [   path: { "${params.outdir}/cutadapt" },
             mode: params.publish_dir_mode,
+            overwrite: params.publish_dir_overwrite,
             pattern: "*.log"
         ],
         [   path: { "${params.outdir}/cutadapt/trimmed_reads" },
             mode: params.publish_dir_mode,
+            overwrite: params.publish_dir_overwrite,
             pattern: "*.trim.fastq.gz",
             enabled: params.save_intermediates
         ]
      ]
    }
 
-
-
     withName: CUSTOM_DUMPSOFTWAREVERSIONS {
         publishDir = [
             path: { "${params.outdir}/pipeline_info" },
             mode: params.publish_dir_mode,
+            overwrite: params.publish_dir_overwrite,
             pattern: '*_versions.yml'
         ]
     }
+
     withName: EMU_ABUNDANCE {
          publishDir = [
             [
-                 path: { "${params.outdir}/results" },
-                 mode: params.publish_dir_mode,
-                 pattern: '*{.tsv,txt}'
+                path: { "${params.outdir}/results" },
+                mode: params.publish_dir_mode,
+                overwrite: params.publish_dir_overwrite,
+                pattern: '*{.tsv,txt}'
             ],
             [
-                 path: { "${params.outdir}/results" },
-                 mode: params.publish_dir_mode,
-                 pattern: '*{.sam}',
-                 enabled: params.keep_files,
+                path: { "${params.outdir}/results" },
+                mode: params.publish_dir_mode,
+                overwrite: params.publish_dir_overwrite,
+                pattern: '*{.sam}',
+                enabled: params.keep_files,
             ],
             [
-                 path: { "${params.outdir}/results" },
-                 mode: params.publish_dir_mode,
-                 pattern: '*{.fa}',
-                 enabled: params.output_unclassified
+                path: { "${params.outdir}/results" },
+                mode: params.publish_dir_mode,
+                overwrite: params.publish_dir_overwrite,
+                pattern: '*.{fastq_unclassified_mapped.fasta,fastq_unmapped.fasta}',
+                enabled: params.output_unclassified
             ],
 
         ]
@@ -170,8 +176,8 @@ process {
             params.output_unclassified ? "--output-unclassified" : "",
         ].join(' ')                                                           // Join converts the list here to a string.
         ext.prefix = { "${meta.id}" }                                         // A closure can be used to access variables defined in the script
-
     }
+
     withName: KRONA_KTIMPORTTAXONOMY {
         ext.args = [
         "-i",
@@ -180,17 +186,18 @@ process {
         ].join(' ')                                                           // Join converts the list here to a string.
         ext.prefix = { "${meta.id}_krona" }
         publishDir = [
-            path: { "${params.outdir}/krona/" },
+            path: { "${params.outdir}/krona" },
             mode: params.publish_dir_mode,
+            overwrite: params.publish_dir_overwrite,
             pattern: '*.{html}'
         ]
     }
+
     withName: FILTLONG {
         ext.args = [
             "--min_length ${params.longread_qc_qualityfilter_minlength}",
             "--max_length ${params.longread_qc_qualityfilter_maxlength}",
             "--min_mean_q ${params.longread_qc_qualityfilter_min_mean_q}"
-
         ]
         .join(' ').trim()
         //ext.prefix = { "${meta.id}_${meta.run_accession}_filtered" }
@@ -200,17 +207,16 @@ process {
             [
                 path: { "${params.outdir}/filtlong" },
                 mode: params.publish_dir_mode,
+                overwrite: params.publish_dir_overwrite,
                 pattern: '*.fastq.gz',
                 enabled: params.save_preprocessed_reads
             ],
             [
                 path: { "${params.outdir}/filtlong" },
                 mode: params.publish_dir_mode,
+                overwrite: params.publish_dir_overwrite,
                 pattern: '*.log'
             ]
         ]
     }
-
-
-
 }

modules/local/emu/abundance/main.nf

@@ -18,10 +18,10 @@ process EMU_ABUNDANCE {
 
     //               Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10").
     //               For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems.
-    conda "bioconda::emu=3.4.4"
+    conda "bioconda::emu=3.5.1"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/emu:3.4.4--hdfd78af_1':
-        'quay.io/biocontainers/emu:3.4.4--hdfd78af_1' }"
+        'https://depot.galaxyproject.org/singularity/emu:3.5.1--hdfd78af_0':
+        'quay.io/biocontainers/emu:3.5.1--hdfd78af_0' }"
 
     input:
     //  Where applicable all sample-specific information e.g. "id", "single_end", "read_group"
@@ -37,8 +37,8 @@ process EMU_ABUNDANCE {
     tuple val(meta), path("*read-assignment-distributions.tsv"), emit: assignment_report, optional:true
     path "versions.yml"           , emit: versions
     tuple val(meta), path("*.sam"), emit: samfile, optional:true
-    tuple val(meta), path("*.fa"), emit: unclassified_fa , optional:true
-
+    tuple val(meta), path("*.fastq_unclassified_mapped.fasta"), emit: unclassified_mapped_fa , optional:true
+    tuple val(meta), path("*.fastq_unmapped.fasta"), emit: unclassified_unmapped_fa , optional:true
 
 
     when:

modules/local/emu/abundance/meta.yml

@@ -54,6 +54,17 @@ output:
       type: file
       description: File containing alignment data
       pattern: "*{.sam}
+  //unclassified and mapped fasta file
+  - unclassified_fa
+      type: file
+      description: File containing reads that mapped to the database but was presumed to be not in the sample by EMU´s algorithm. 
+      pattern: "*{.fastq_unclassified_mapped.fasta}
+  //unmapped fasta file
+  - unclassified_unmapped_fa
+      type: file
+      description: File containing reads that was not mapped to the database. 
+      pattern: "*{.fastq_unmapped.fasta}
+
 
   # emu log. Put on ice.
  // - emulog

modules/local/generate_master_html/meta.yml

@@ -1,54 +1,44 @@
-name: "emu_abundance"
-## TODO nf-core: Add a description of the module and list keywords
-description: A taxonomic profiler for metagenomic 16S data optimized for error prone long reads.
+name: "generate_master_html"
+description: A python script that utilises jinja2 to generate a .html file containing links to all of the sample's output html files in the form of a table.
 keywords:
-  - Metagenomics
-  - 16S
-  - Nanopore
+  - Web development
+  - Summary
+  - HTML
 
 tools:
-  - "emu":
-      ## TODO nf-core: Add a description and other details for the software below
-      description: "Emu is a relative abundance estimator for 16s genomic data."
-      homepage: "https://gitlab.com/treangenlab/emu"
-      documentation: "https://gitlab.com/treangenlab/emu"
+  - "generate_master_html.py":
+      description: "Generates master.html file"
+      homepage: "https://github.com/SMD-Bioinformatics-Lund/gms_16S/blob/main/bin/generate_master_html.py"
+      documentation: "None"
       tool_dev_url: "None"
-      doi: "https://doi.org/10.1038/s41592-022-01520-4"
-      licence: "['MIT']"
+      doi: "None"
+      licence: "None"
 
-## TODO nf-core: Add a description of all of the variables used as input
 input:
-  # Only when we have meta
   - meta:
       type: map
       description: |
         Groovy Map containing sample information
         e.g. [ id:'test', single_end:false ]
-  #
-  ## TODO nf-core: Delete / customise this example input
-  - reads:
+  - csv:
       type: file
-      description: fastq.gz file containing metagenomic 16S data
-      pattern: "*.{fastq.gz}"
+      description: csv file containing information regarding 16S metagenomic data
+      pattern: "*.{csv}"
 
-## TODO nf-core: Add a description of all of the variables used as output
 output:
-  #Only when we have meta
   - meta:
       type: map
       description: |
         Groovy Map containing sample information
         e.g. [ id:'test', single_end:false ]
-  #
   - versions:
       type: file
       description: File containing software versions
       pattern: "versions.yml"
-  ## TODO nf-core: Delete / customise this example output
-  - report:
+  - html:
       type: file
-      description: Report (tsv file) over detected species and estimated number of reads and relative abundance
-      pattern: "*{.tsv}"
+      description: HTML file containing table with links to sample outputs
+      pattern: "*{.html}"
 
 authors:
   - "@ryanjameskennedy"

nextflow.config

@@ -69,6 +69,7 @@ params {
     outdir                     = null
     tracedir                   = "${params.outdir}/pipeline_info"
     publish_dir_mode           = 'copy'
+    publish_dir_overwrite      = false
     email                      = null
     email_on_fail              = null
     plaintext_email            = false