Support for running multiple bin QC tools

CHANGELOG.md

@@ -26,12 +26,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### `Added`
 
 - [#873](https://github.com/nf-core/mag/pull/873) - Document usage of `longread_percentidentity` and `shortread_percentidentity` and set the value of `longread_percentidentity` in the `test_full` profile to 85 (by @prototaxites)
+- [#842](https://github.com/nf-core/mag/pull/842) - Add support for running multiple binQC tools in one run using dedicated `--run_busco`, `--run_checkm`, and `--run_checkm2` parameters (by @harper357, with contributions from @dialvarezs, @prototaxites and @jfy133)
 - [#875](https://github.com/nf-core/mag/pull/875) - Add binner COMEBin (by @d4straub)
 
 ### `Changed`
 
 - [#878](https://github.com/nf-core/mag/pull/878) - Refine test_full config with optimised resource usage for AWS release megatests (by @jfy133)
 - [#880](https://github.com/nf-core/mag/pull/880) - Updated to nf-core 3.4.1 `TEMPLATE` (by @jfy133)
+- [#842](https://github.com/nf-core/mag/pull/842) - Change `bin_summary.tsv` format for improved clarity and more comprehensiveness (by @harper357, with contributions from @dialvarezs, @prototaxites and @jfy133)
+  - Now will include columns from all bin QC tools executed in a given run (i.e., all/any of BUSCO, CheckM and CheckM2)
+  - Adds suffixes to all columns (`_<toolname>`) to distinguish which column comes from which tool
 
 ### `Fixed`
 
@@ -48,6 +52,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### `Deprecated`
 
+- [#842](https://github.com/nf-core/mag/pull/842) - Remove `--binqc_tool` (by @harper357, with contributions from @dialvarezs, @prototaxites and @jfy133)
+
 ## 5.0.0 - [2025-09-30]
 
 ### `Added`

bin/combine_tables.py

@@ -19,13 +19,22 @@ def parse_args(args=None):
         metavar="FILE",
         help="Bin depths summary file.",
     )
-    parser.add_argument("-b", "--binqc_summary", metavar="FILE", help="BUSCO summary file.")
-    parser.add_argument("-q", "--quast_summary", metavar="FILE", help="QUAST BINS summary file.")
-    parser.add_argument("-g", "--gtdbtk_summary", metavar="FILE", help="GTDB-Tk summary file.")
-    parser.add_argument("-a", "--cat_summary", metavar="FILE", help="CAT table file.")
     parser.add_argument(
-        "-t", "--binqc_tool", help="Bin QC tool used", choices=["busco", "checkm", "checkm2"]
+        "-q", "--quast_summary", metavar="FILE", help="QUAST BINS summary file."
+    )
+    parser.add_argument(
+        "-g", "--gtdbtk_summary", metavar="FILE", help="GTDB-Tk summary file."
+    )
+    parser.add_argument(
+        "-u", "--busco_summary", metavar="FILE", help="BUSCO summary file."
     )
+    parser.add_argument(
+        "-c", "--checkm_summary", metavar="FILE", help="CheckM summary file."
+    )
+    parser.add_argument(
+        "-e", "--checkm2_summary", metavar="FILE", help="CheckM2 summary file."
+    )
+    parser.add_argument("-a", "--cat_summary", metavar="FILE", help="CAT table file.")
 
     parser.add_argument(
         "-o",
@@ -74,54 +83,81 @@ def parse_cat_table(cat_table):
         header=None,
         skiprows=1,
     )
-    # merge all rank columns into a single column
+    ## merge all rank columns into a single column
     df["CAT_rank"] = (
-        df.filter(regex="rank_\d+").apply(lambda x: ";".join(x.dropna()), axis=1).str.lstrip()
+        df.filter(regex="rank_\d+")
+        .apply(lambda x: ";".join(x.dropna()), axis=1)
+        .str.lstrip()
     )
-    # remove rank_* columns
+    ## remove rank_* columns
     df.drop(df.filter(regex="rank_\d+").columns, axis=1, inplace=True)
+    df = df.add_suffix("_catpack")
 
     return df
 
 
 def main(args=None):
     args = parse_args(args)
 
+    ## INPUT VALIDATION
+
     if (
-        not args.binqc_summary
+        not args.busco_summary
+        and not args.checkm_summary
+        and not args.checkm2_summary
         and not args.quast_summary
         and not args.gtdbtk_summary
     ):
         sys.exit(
             "No summary specified! "
-            "Please specify at least BUSCO, CheckM, CheckM2 or QUAST summary."
+            "Please specify at least one of BUSCO, CheckM, CheckM2 or QUAST summary."
         )
 
-    # GTDB-Tk can only be run in combination with BUSCO, CheckM or CheckM2
-    if args.gtdbtk_summary and not args.binqc_summary:
+    ## GTDB-Tk can only be run in combination with BUSCO, CheckM or CheckM2
+    if (
+        args.gtdbtk_summary
+        and not args.busco_summary
+        and not args.checkm_summary
+        and not args.checkm2_summary
+    ):
         sys.exit(
             "Invalid parameter combination: "
-            "GTDB-TK summary specified, but no BUSCO, CheckM or CheckM2 summary!"
+            "GTDB-TK summary specified, but no BUSCO, CheckM or CheckM2 summary provided!"
         )
 
-    # handle bin depths
+    ## BIN DEPTH PROCESSING
+
+    ## handle bin depths, and extract root bin names
     results = pd.read_csv(args.depths_summary, sep="\t")
-    results.columns = ["Depth " + str(col) if col != "bin" else col for col in results.columns]
+    results.columns = [
+        "Depth " + str(col) if col != "bin" else col for col in results.columns
+    ]
     bins = results["bin"].sort_values().reset_index(drop=True)
 
-    if args.binqc_summary and args.binqc_tool == "busco":
-        busco_results = pd.read_csv(args.binqc_summary, sep="\t")
+    ## BUSCO PROCESSING
+
+    if args.busco_summary:
+        busco_results = pd.read_csv(args.busco_summary, sep="\t")
         busco_bins = set(busco_results["Input_file"])
 
         if set(bins) != busco_bins and len(busco_bins.intersection(set(bins))) > 0:
-            warnings.warn("Bins in BUSCO summary do not match bins in bin depths summary")
+            warnings.warn(
+                "Bins in BUSCO summary do not match bins in bin depths summary"
+            )
         elif len(busco_bins.intersection(set(bins))) == 0:
             sys.exit("Bins in BUSCO summary do not match bins in bin depths summary!")
+        busco_results = busco_results.add_suffix("_busco")
         results = pd.merge(
-            results, busco_results, left_on="bin", right_on="Input_file", how="outer"
+            results,
+            busco_results,
+            left_on="bin",
+            right_on="Input_file_busco",
+            how="outer",
         )  # assuming depths for all bins are given
 
-    if args.binqc_summary and args.binqc_tool == "checkm":
+    ## CHECKM PROCESSING
+
+    if args.checkm_summary:
         use_columns = [
             "Bin Id",
             "Marker lineage",
@@ -141,16 +177,23 @@ def main(args=None):
             "4",
             "5+",
         ]
-        checkm_results = pd.read_csv(args.binqc_summary, usecols=use_columns, sep="\t")
+        checkm_results = pd.read_csv(args.checkm_summary, usecols=use_columns, sep="\t")
         checkm_results["Bin Id"] = checkm_results["Bin Id"] + ".fa"
         if not set(checkm_results["Bin Id"]).issubset(set(bins)):
             sys.exit("Bins in CheckM summary do not match bins in bin depths summary!")
+        checkm_results = checkm_results.add_suffix("_checkm")
         results = pd.merge(
-            results, checkm_results, left_on="bin", right_on="Bin Id", how="outer"
+            results,
+            checkm_results,
+            left_on="bin",
+            right_on="Bin Id_checkm",
+            how="outer",
         )  # assuming depths for all bins are given
-        results["Bin Id"] = results["Bin Id"].str.removesuffix(".fa")
+        results["Bin Id_checkm"] = results["Bin Id_checkm"].str.removesuffix(".fa")
+
+    ## CHECKM2 PROCESSING
 
-    if args.binqc_summary and args.binqc_tool == "checkm2":
+    if args.checkm2_summary:
         use_columns = [
             "Name",
             "Completeness",
@@ -160,40 +203,65 @@ def main(args=None):
             "Translation_Table_Used",
             "Total_Coding_Sequences",
         ]
-        checkm2_results = pd.read_csv(args.binqc_summary, usecols=use_columns, sep="\t")
+        checkm2_results = pd.read_csv(
+            args.checkm2_summary, usecols=use_columns, sep="\t"
+        )
         checkm2_results["Name"] = checkm2_results["Name"] + ".fa"
         if not set(checkm2_results["Name"]).issubset(set(bins)):
             sys.exit("Bins in CheckM2 summary do not match bins in bin depths summary!")
+        checkm2_results = checkm2_results.add_suffix("_checkm2")
         results = pd.merge(
-            results, checkm2_results, left_on="bin", right_on="Name", how="outer"
+            results,
+            checkm2_results,
+            left_on="bin",
+            right_on="Name_checkm2",
+            how="outer",
         )  # assuming depths for all bins are given
-        results["Name"] = results["Name"].str.removesuffix(".fa")
+        results["Name"] = results["Name_checkm2"].str.removesuffix(".fa")
+
+    ## QUAST PROCESSING
 
     if args.quast_summary:
         quast_results = pd.read_csv(args.quast_summary, sep="\t")
-        if not bins.equals(quast_results["Assembly"].sort_values().reset_index(drop=True)):
+        if not bins.equals(
+            quast_results["Assembly"].sort_values().reset_index(drop=True)
+        ):
             sys.exit("Bins in QUAST summary do not match bins in bin depths summary!")
+        quast_results = quast_results.add_suffix("_quast")
         results = pd.merge(
-            results, quast_results, left_on="bin", right_on="Assembly", how="outer"
+            results,
+            quast_results,
+            left_on="bin",
+            right_on="Assembly_quast",
+            how="outer",
         )  # assuming depths for all bins are given
 
+    ## GTDBTK PROCESSING
+
     if args.gtdbtk_summary:
         gtdbtk_results = pd.read_csv(args.gtdbtk_summary, sep="\t")
         if len(set(gtdbtk_results["user_genome"].to_list()).difference(set(bins))) > 0:
             sys.exit("Bins in GTDB-Tk summary do not match bins in bin depths summary!")
+        gtdbtk_results = gtdbtk_results.add_suffix("_gtdbtk")
         results = pd.merge(
-            results, gtdbtk_results, left_on="bin", right_on="user_genome", how="outer"
+            results,
+            gtdbtk_results,
+            left_on="bin",
+            right_on="user_genome_gtdbtk",
+            how="outer",
         )  # assuming depths for all bins are given
 
+    ## CAT_PACK PROCESSING
+
     if args.cat_summary:
         cat_results = parse_cat_table(args.cat_summary)
-        if len(set(cat_results["bin"].to_list()).difference(set(bins))) > 0:
+        if len(set(cat_results["bin_catpack"].to_list()).difference(set(bins))) > 0:
             sys.exit("Bins in CAT summary do not match bins in bin depths summary!")
         results = pd.merge(
             results,
-            cat_results[["bin", "CAT_rank"]],
+            cat_results[["bin_catpack", "CAT_rank_catpack"]],
             left_on="bin",
-            right_on="bin",
+            right_on="bin_catpack",
             how="outer",
         )
 

conf/modules.config

@@ -512,8 +512,8 @@ process {
         ]
     }
 
-    withName: CONCAT_BINQC_TSV {
-        ext.prefix = { "${params.binqc_tool}_summary" }
+    withName: 'CONCAT_BUSCO_TSV|CONCAT_CHECKM_TSV|CONCAT_CHECKM2_TSV' {
+        ext.prefix = { "${meta.id}_summary" }
         publishDir = [
             path: { "${params.outdir}/GenomeBinning/QC" },
             mode: params.publish_dir_mode,
@@ -532,6 +532,7 @@ process {
     }
 
     withName: CHECKM2_PREDICT {
+        tag        = { "${meta.assembler}-${meta.binner}-${meta.domain}-${meta.refinement}-${meta.id}" }
         ext.prefix = { "${meta.assembler}-${meta.binner}-${meta.domain}-${meta.refinement}-${meta.id}" }
         publishDir = [
             path: { "${params.outdir}/GenomeBinning/QC/CheckM2" },
@@ -639,6 +640,7 @@ process {
     }
 
     withName: GTDBTK_CLASSIFYWF {
+        tag        = { "${meta.assembler}-${meta.binner}-${meta.domain}-${meta.refinement}-${meta.id}" }
         ext.args   = [
             "--extension fa",
             "--min_perc_aa ${params.gtdbtk_min_perc_aa}",

conf/test.config

@@ -34,6 +34,7 @@ params {
     busco_clean                 = true
     // Prokka is the slowest step of the tests, so we speed up by turning off CDS/product searching
     prokka_fast_mode            = true
+    // Source: https://data.ace.uq.edu.au/public/gtdb/data/releases/latest/auxillary_files/gtdbtk_package/mockup_db/
     gtdb_db                     = params.pipelines_testdata_base_path + 'mag/databases/gtdbtk/gtdbtk_mockup_20250422.tar.gz'
     cat_db                      = params.pipelines_testdata_base_path + 'mag/databases/cat/minigut_cat.tar.gz'
     cat_no_suggestive_asterisks = true

conf/test_alternatives.config

@@ -27,13 +27,16 @@ params {
     // Input data
     input                      = params.pipelines_testdata_base_path + 'mag/samplesheets/samplesheet.v4.csv'
     clip_tool                  = 'trimmomatic'
-    binqc_tool                 = 'checkm2'
+    run_busco                  = true
+    busco_clean                = true
+    run_checkm2                = true
     bin_domain_classification  = true
     skip_spades                = true
     skip_quast                 = true
     skip_prodigal              = true
     skip_prokka                = true
     skip_gtdbtk                = true
+    gtdbtk_skip_aniscreen      = true
     skip_maxbin2               = true
     skip_concoct               = true
     skip_comebin               = true

conf/test_assembly_input.config

@@ -41,6 +41,7 @@ params {
     skip_prodigal                   = true
     skip_prokka                     = true
     skip_gtdbtk                     = true
+    gtdbtk_skip_aniscreen           = true
     skip_concoct                    = false
     skip_comebin                    = true
 

conf/test_full.config

@@ -40,7 +40,8 @@ params {
     // Skip CONCOCT due to timeout issues
     skip_concoct               = true
 
-    binqc_tool                 = "checkm2"
+    run_checkm2                = true
+    run_busco                  = false
 
     // Set Prokka compliance mode to allow metaSPAdes bins to be annotated
     prokka_with_compliance     = true

conf/test_hybrid.config

@@ -37,6 +37,7 @@ params {
     skip_flye                     = true
     skip_metamdbg                 = true
     skip_gtdbtk                   = true
+    gtdbtk_skip_aniscreen         = true
     skip_concoct                  = true
     skip_comebin                  = true
 

conf/test_longreadonly.config

@@ -36,6 +36,7 @@ params {
     busco_db_lineage           = 'bacteria_odb10'
 
     skip_gtdbtk                = true
+    gtdbtk_skip_aniscreen      = true
     skip_concoct               = true
     skip_comebin               = true
 }

conf/test_longreadonly_alternatives.config

@@ -36,6 +36,7 @@ params {
     skip_prodigal                 = true
     skip_prokka                   = true
     skip_gtdbtk                   = true
+    gtdbtk_skip_aniscreen         = true
     skip_concoct                  = true
     skip_comebin                  = true
     skip_metaeuk                  = true

conf/test_minimal.config

@@ -21,8 +21,8 @@ process {
 }
 
 params {
-    config_profile_name        = 'Test nothing profile'
-    config_profile_description = 'Minimal test dataset to check pipeline function'
+    config_profile_name           = 'Test nothing profile'
+    config_profile_description    = 'Minimal test dataset to check pipeline function'
 
     // Input data
     input                         = params.pipelines_testdata_base_path + 'mag/samplesheets/samplesheet.v4.csv'
@@ -43,8 +43,12 @@ params {
     skip_comebin                  = true
     skip_prokka                   = true
     skip_binqc                    = true
+    run_busco                     = false
+    run_checkm                    = false
+    run_checkm2                   = false
     skip_gtdbtk                   = true
-    skip_ancient_damagecorrection = true
     gtdbtk_min_completeness       = 0.01
+    gtdbtk_skip_aniscreen         = true
     skip_metaeuk                  = true
+    skip_ancient_damagecorrection = true
 }

conf/test_single_end.config

@@ -42,12 +42,16 @@ params {
     skip_comebin                         = true
     min_length_unbinned_contigs          = 1000000
     max_unbinned_contigs                 = 2
-    binqc_tool                           = 'checkm'
+    run_busco                            = false
+    run_checkm                           = true
     run_virus_identification             = true
     genomad_splits                       = 7
     // micro_db not compatible with current genNomad version
-    genomad_db                           = null // 'https://zenodo.org/records/11945948/files/genomad_microdb.tar.gz'
+    genomad_db                           = null
+    // 'https://zenodo.org/records/11945948/files/genomad_microdb.tar.gz'
     gtdb_db                              = params.pipelines_testdata_base_path + 'mag/databases/gtdbtk/gtdbtk_mockup_20250422.tar.gz'
+    gtdbtk_min_completeness              = 3
+    gtdbtk_skip_aniscreen                = true
     cat_db                               = params.pipelines_testdata_base_path + 'mag/databases/cat/minigut_cat.tar.gz'
     cat_no_suggestive_asterisks          = true
 }