Merge branch 'dyn-resource'

FastOMA.nf

@@ -230,12 +230,97 @@ process batch_roothogs{
     """
 }
 
+
+def getMaxFileSize = { Path folderPath ->
+    def maxFileSize = 0L // Initialize maximum file size
+
+    def traverseFolder = { Path currentPath ->
+        def currentFile = currentPath.toFile()
+
+        if (currentFile.isDirectory()) {
+            def files = currentFile.listFiles()
+            if (files) {
+                files.each { file ->
+                    if (file.isDirectory()) {
+                        traverseFolder(file.toPath()) // Recursively traverse subdirectories
+                    } else {
+                        def fileSize = file.length()
+                        if (fileSize > maxFileSize) {
+                            maxFileSize = fileSize // Update maximum file size if larger
+                        }
+                    }
+                }
+            }
+        } else {
+            def fileSize = currentFile.length()
+            if (fileSize > maxFileSize) {
+                maxFileSize = fileSize // Update maximum file size if larger
+            }
+        }
+    }
+
+    traverseFolder(folderPath)
+    return maxFileSize
+}
+
+// function to check maximum resources
+def check_max(obj, type) {
+    if (type == 'memory') {
+        try {
+            if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1)
+                return params.max_memory as nextflow.util.MemoryUnit
+            else
+                return obj
+        } catch (all) {
+            println "   ### ERROR ###   Max memory '${params.max_memory}' is not valid! Using default value: $obj"
+            return obj
+        }
+    } else if (type == 'time') {
+        try {
+            if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1)
+                return params.max_time as nextflow.util.Duration
+            else
+                return obj
+        } catch (all) {
+            println "   ### ERROR ###   Max time '${params.max_time}' is not valid! Using default value: $obj"
+            return obj
+        }
+    } else if (type == 'cpus') {
+        try {
+            return Math.min( obj, params.max_cpus as int )
+        } catch (all) {
+            println "   ### ERROR ###   Max cpus '${params.max_cpus}' is not valid! Using default value: $obj"
+            return obj
+        }
+    }
+}
+
+def mem_cat(filesize, nr_genomes){
+    def fac = 1;
+    if (nr_genomes > 500) fac = 2;
+    if (filesize < 1000000) return 12.GB * fac;
+    else if (filesize < 2000000) return 20.GB * fac;
+    else if (filesize < 5000000) return 32.GB * fac;
+    else return 64.GB * fac;
+}
+
+def time_cat(filesize, nr_genomes){
+    def fac = 1;
+    if (nr_genomes > 500) fac = 2;
+    if (filesize < 1000000) return 4.h * fac;
+    else if (filesize < 20000000) return 24.h * fac;
+    else return 72.h * fac;
+}
+
 process hog_big{
-  label "process_high"
+  cpus { check_max( 4, "cpus") }
+  memory { check_max( mem_cat(getMaxFileSize(rhogsbig), nr_species as int) * task.attempt, "memory") }
+  time { check_max( time_cat(getMaxFileSize(rhogsbig), nr_species as int) * task.attempt, "time") }
 
   input:
     each rhogsbig
     path species_tree
+    val nr_species
   output:
     path "pickle_hogs"
     path "msa/*.fa" , optional: true          // msa         if write True
@@ -329,7 +414,7 @@ workflow {
 
     (rhogs_rest_batches, rhogs_big_batches) = batch_roothogs(omamer_rhogs)
 
-    (pickle_big_rhog, msa_out_big, genetrees_out_rest) = hog_big(rhogs_big_batches.flatten(), species_tree_checked)
+    (pickle_big_rhog, msa_out_big, genetrees_out_rest) = hog_big(rhogs_big_batches.flatten(), species_tree_checked, nr_species)
     (pickle_rest_rhog,  msas_out_rest, genetrees_out_test) = hog_rest(rhogs_rest_batches.flatten(), species_tree_checked)
     channel.empty().concat(pickle_big_rhog, pickle_rest_rhog).set{ all_rhog_pickle }
 

nextflow.config

@@ -48,6 +48,11 @@ profiles {
     process.executor = 'local'
   }
 
+  slurm {
+    process.executor = "slurm"
+    time = 4.h
+  }
+
   conda {
     process.conda = "environment-conda.yml"
     conda.enabled = true