Master html creation with timestamps

.github/workflows/changelog_update_reminder.yml

@@ -0,0 +1,14 @@
+name: "Changelog update reminder"
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled]
+
+jobs:
+  changelog:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dangoslen/changelog-enforcer@v3
+        with:
+          changeLogPath: 'CHANGELOG.md'
+          skipLabel: 'Skip-Changelog'

CHANGELOG.md

@@ -4,8 +4,24 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
 ## [Unreleased]
 
+### Added
+
+- Added a `generate_master_html` python script that creates `master.html` file containing a table of samples with corresponding pointers to each html output file
+- Added repective `GENERATE_MASTER_HTML` process
+- Added `cmd.config`
+- Added `params.trace_timestamp` to `nextflow.config`
+- Added `changelog_update_reminder` GA workflow
+- Added optional ability to save merged reads
+
+### Fixed
+
+### Changed
+
+- Provided option to `save_merged_reads`
+
 ## [v0.1.0]
 
 ### Added

assets/master_template.html

@@ -53,9 +53,9 @@ <h2 class="card-title mb-0">Sample Report</h2>
                                 <td><a href="./nanoplot/{{ sample_id }}_T1_nanoplot_unprocessedWeightedHistogramReadlength.html">Weighted Histogram</a></td>
                                 <td><a href="./nanoplot/{{ sample_id }}_T1_nanoplot_unprocessedWeightedLogTransformed_HistogramReadlength.html">Weighted Log-transformed Histogram</a></td>
                                 <td><a href="./nanoplot/{{ sample_id }}_T1_nanoplot_unprocessedYield_By_Length.html">Yield by Length</a></td>
-                                <td><a href="./pipeline_info/execution_report_{{ date_id }}.html">Execution Report</a></td>
-                                <td><a href="./pipeline_info/execution_timeline_{{ date_id }}.html">Execution Timeline</a></td>
-                                <td><a href="./pipeline_info/pipeline_dag_{{ date_id }}.html">Pipeline DAG</a></td>
+                                <td><a href="./pipeline_info/execution_report_{{ timestamp }}.html">Execution Report</a></td>
+                                <td><a href="./pipeline_info/execution_timeline_{{ timestamp }}.html">Execution Timeline</a></td>
+                                <td><a href="./pipeline_info/pipeline_dag_{{ timestamp }}.html">Pipeline DAG</a></td>
                             </tr>
                             {% endfor %}
                         </tbody>

bin/generate_master_html.py

@@ -66,23 +66,16 @@
     dest='html',
     required=True
     )
+parser.add_argument(
+    '-t', '--timestamp',
+    help='pipeline execution timestamp',
+    metavar='PIPELINE_EXECUTION_TIMESTAMP',
+    dest='timestamp',
+    required=True
+    )
 
 args = parser.parse_args()
 
-def get_date_id(samplesheet_csv_fpath):
-    date_ids = []
-    parent_dir = os.path.dirname(samplesheet_csv_fpath)
-    pipeline_info_dir = os.path.join(parent_dir, 'pipeline_info')
-    for filename in os.listdir(pipeline_info_dir):
-        if filename.startswith("execution_report"):
-            execution_report_fpath = os.path.join(pipeline_info_dir, filename)
-            date_id = find_date_in_string(execution_report_fpath, r'(\d{4}-\d{2}-\d{2}[^.]+)')
-            date_ids.append(date_id)
-    date_list = map(find_date_in_string, date_ids, [r'\b(\d{4}-\d{2}-\d{2})']*len(date_ids))
-    date_id_zipped = list(zip(date_ids, date_list))
-    sorted_date_ids = [date_id[0] for date_id in sorted(date_id_zipped, key=lambda date: datetime.strptime(date[1], "%Y-%m-%d"), reverse=True)]    
-    return sorted_date_ids[0]
-
 def find_date_in_string(input_string, date_pattern):
     """Searches for a date within a given string."""
     date = ""
@@ -103,19 +96,18 @@ def get_sample_ids(samplesheet_csv):
     sample_ids = df['sample'].tolist()
     return sample_ids
 
-def generate_master_html(template_html_fpath, sample_ids, seqrun_date, date_id):
+def generate_master_html(template_html_fpath, sample_ids, seqrun_date, timestamp):
     """Read the template from an HTML file."""
     with open(template_html_fpath, "r") as file:
         master_template = file.read()
     template = Template(master_template)
-    rendered_html = template.render(sample_ids=sample_ids, seqrun_date=seqrun_date, date_id=date_id)
+    rendered_html = template.render(sample_ids=sample_ids, seqrun_date=seqrun_date, timestamp=timestamp)
     return rendered_html
 
 def main():
     sample_ids = get_sample_ids(args.csv)
     seqrun_date = find_date_in_string(args.csv, r'/(\d{8})_')
-    date_id = get_date_id(args.csv)
-    rendered_html = generate_master_html(args.html, sample_ids, seqrun_date, date_id)
+    rendered_html = generate_master_html(args.html, sample_ids, seqrun_date, args.timestamp)
     with open("master.html", "w") as fout:
         fout.write(rendered_html)
 

conf/cmd.config

@@ -24,4 +24,7 @@ params {
     max_memory                  = '300.GB'
     max_time                    = '48.h'
 
+    // Reads
+    save_merged_reads           = false
+
 }

conf/modules.config

@@ -22,17 +22,17 @@ process {
         publishDir = [
             path: { "${params.outdir}/fastq_pass_merged" },
             mode: params.publish_dir_mode,
-            pattern: 'fastq_pass_merged'
+            pattern: 'fastq_pass_merged',
+            enabled: params.save_merged_reads
         ]
     }
 
-
     withName: MERGE_BARCODES_SAMPLESHEET {
         publishDir = [
             path: { "${params.outdir}/fastq_pass_merged" },
             mode: params.publish_dir_mode,
-            pattern: 'fastq_pass_merged'
-//            pattern: '*fastq.gz'
+            pattern: 'fastq_pass_merged',
+            enabled: params.save_merged_reads
         ]
     }
 

modules/local/generate_master_html/main.nf

@@ -14,6 +14,6 @@ process GENERATE_MASTER_HTML {
 
     script:
     """
-    generate_master_html.py --csv $csv --html $params.master_template
+    generate_master_html.py --csv ${csv} --html ${params.master_template} --timestamp ${params.trace_timestamp}
     """
 }

nextflow.config

@@ -25,6 +25,15 @@ params {
     // master html
     master_template            = "$projectDir/assets/master_template.html"
 
+    //cutadapt
+    FW_primer                  = null
+    RV_primer                  = null 
+    cutadapt_min_overlap       = 3
+    cutadapt_max_error_rate    = 0.1
+    retain_untrimmed           = false 
+    skip_cutadapt              = false
+    save_intermediates         = false
+
     // porechop_abi
     adapter_trimming           = false
 
@@ -37,6 +46,9 @@ params {
     //Save the trimmed reads
     save_preprocessed_reads    = false
 
+    //Save the merged reads
+    save_merged_reads          = true
+
     // krona
     run_krona                  = true
     krona_taxonomy_tab         = "$projectDir/assets/databases/krona/taxonomy/taxonomy.tab"
@@ -198,8 +210,8 @@ env {
 
 // Capture exit codes from upstream processes when piping
 process.shell = ['/bin/bash', '-euo', 'pipefail']
-
-def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss')
+def trace_timestamp = new java.util.Date().format('yyyy-MM-dd_HH-mm-ss')
+params.trace_timestamp = trace_timestamp
 timeline {
     enabled = true
     file    = "${params.tracedir}/execution_timeline_${trace_timestamp}.html"

workflows/gmsemu.nf

@@ -55,10 +55,6 @@ include { MERGE_BARCODES              } from '../modules/local/merge_barcodes/ma
 include { MERGE_BARCODES_SAMPLESHEET  } from '../modules/local/merge_barcodes_samplesheet/main.nf'
 include { GENERATE_INPUT              } from '../modules/local/generate_input/main.nf'
 include { GENERATE_MASTER_HTML        } from '../modules/local/generate_master_html/main.nf'
-include { NANOPLOT as NANOPLOT1       } from '../modules/nf-core/nanoplot/main.nf'
-include { NANOPLOT  as NANOPLOT2      } from '../modules/nf-core/nanoplot/main.nf'
-include { PORECHOP_ABI                } from '../modules/nf-core/porechop/abi/main.nf'
-include { FILTLONG                    } from '../modules/nf-core/filtlong/main.nf'
 include { EMU_ABUNDANCE               } from '../modules/local/emu/abundance/main.nf'
 include { KRONA_KTIMPORTTAXONOMY      } from '../modules/nf-core/krona/ktimporttaxonomy/main.nf'
 include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/custom/dumpsoftwareversions/main'
@@ -206,6 +202,7 @@ workflow GMSEMU {
     )
     multiqc_report = MULTIQC.out.report.toList()
 
+    GENERATE_MASTER_HTML(GENERATE_INPUT.out.sample_sheet_merged)
 
 }
 /*