From 43d8cfb7b3e1a3c3cb7b6119f40ab227f1a5e843 Mon Sep 17 00:00:00 2001 From: cgpu <38183826+cgpu@users.noreply.github.com> Date: Wed, 10 Mar 2021 19:33:31 +0000 Subject: [PATCH 01/35] Update nextflow.config --- nextflow.config | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/nextflow.config b/nextflow.config index bdcae79..2d47279 100644 --- a/nextflow.config +++ b/nextflow.config @@ -3,6 +3,9 @@ docker.enabled = true // NOTE: // Initialise the values of the params to the preferred default value or to false params { + processAMachineType = 'n1-standard-16' + processA_memory = '4.GB' + processA_cpus = 1 mainScript = 'main.nf' defaultBranch = 'main' config = 'conf/standard.config' @@ -23,10 +26,9 @@ params { echo = false cpus = 1 - processA_cpus = 1 errorStrategy = 'ignore' - container = 'quay.io/lifebitai/ubuntu:18.10' - maxForks = 200 + container = 'quay.io/lifebitai/ubuntu:18.10' + maxForks = 200 queueSize = 200 executor = 'ignite' @@ -50,6 +52,8 @@ process { withName: processA { cpus = params.processA_cpus + memory = params.processA_memory + machineType = params.processAMachineType } } @@ -63,4 +67,4 @@ manifest { description = 'A stampede of processes at your fingertips' mainScript = params.mainScript defaultBranch = params.defaultBranch -} \ No newline at end of file +} From d207346fee44916f88ae059686a8d599083ccc7f Mon Sep 17 00:00:00 2001 From: cgpu <38183826+cgpu@users.noreply.github.com> Date: Wed, 10 Mar 2021 22:16:35 +0000 Subject: [PATCH 02/35] Rename s3Location to be cloud agnostic --- nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index 2d47279..bc06c84 100644 --- a/nextflow.config +++ b/nextflow.config @@ -9,7 +9,7 @@ params { mainScript = 'main.nf' defaultBranch = 'main' config = 'conf/standard.config' - s3Location = 's3://lifebit-featured-datasets/pipelines/spammer-nf/input_files' + cloudStorageLocation = 's3://lifebit-featured-datasets/pipelines/spammer-nf/input_files' fileSuffix = '' repsProcessA = 10 processAWriteToDiskMb = 10 From 19470891c5726541ed84dc0b2ebe4fcef03742dc Mon Sep 17 00:00:00 2001 From: cgpu <38183826+cgpu@users.noreply.github.com> Date: Thu, 11 Mar 2021 18:53:59 +0000 Subject: [PATCH 03/35] Adds google.config --- conf/google.config | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 conf/google.config diff --git a/conf/google.config b/conf/google.config new file mode 100644 index 0000000..55d42bd --- /dev/null +++ b/conf/google.config @@ -0,0 +1,12 @@ +google { + lifeSciences.bootDiskSize = params.gls_bootDiskSize + lifeSciences.preemptible = params.gls_preemptible + zone = params.zone + network = params.network + subnetwork = params.subnetwork +} + +executor { + name = params.executor +} + From 016346b0aa16350fe9dc92aa1095525a83b86389 Mon Sep 17 00:00:00 2001 From: cgpu <38183826+cgpu@users.noreply.github.com> Date: Thu, 11 Mar 2021 18:56:19 +0000 Subject: [PATCH 04/35] Update nextflow.config --- nextflow.config | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nextflow.config b/nextflow.config index bc06c84..2c07f82 100644 --- a/nextflow.config +++ b/nextflow.config @@ -32,6 +32,13 @@ params { queueSize = 200 executor = 'ignite' + + // gls params + gls_bootDiskSize = false + gls_preemptible = false + zone = false + network = false + subnetwork = false } // Do not update the order because the values set in params scope will not be overwritten From 41d008949ba3c33668cf726fb3f325baf7515fb6 Mon Sep 17 00:00:00 2001 From: cgpu <38183826+cgpu@users.noreply.github.com> Date: Thu, 11 Mar 2021 19:04:54 +0000 Subject: [PATCH 05/35] Adds gls info in main.nf --- main.nf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/main.nf b/main.nf index ec52e87..e8a9c3f 100644 --- a/main.nf +++ b/main.nf @@ -23,6 +23,13 @@ log.info "container : ${params.container}" log.info "maxForks : ${params.maxForks}" log.info "queueSize : ${params.queueSize}" log.info "executor : ${params.executor}" +if(params.executor == 'google-lifesciences') { +log.info "gls_bootDiskSize : ${params.gls_bootDiskSize}" +log.info "gls_preemptible : ${params.gls_preemptible}" +log.info "zone : ${params.zone}" +log.info "network : ${params.network}" +log.info "subnetwork : ${params.subnetwork}" +} log.info "" numberRepetitionsForProcessA = params.repsProcessA From c76c32222a0b195b54a2616cf8451c36b22ab66a Mon Sep 17 00:00:00 2001 From: cgpu <38183826+cgpu@users.noreply.github.com> Date: Thu, 11 Mar 2021 19:23:20 +0000 Subject: [PATCH 06/35] Update google.config --- conf/google.config | 5 ----- 1 file changed, 5 deletions(-) diff --git a/conf/google.config b/conf/google.config index 55d42bd..88bd6c7 100644 --- a/conf/google.config +++ b/conf/google.config @@ -5,8 +5,3 @@ google { network = params.network subnetwork = params.subnetwork } - -executor { - name = params.executor -} - From 0688754c57a8bd101b41a7ee307e53f7d8b70339 Mon Sep 17 00:00:00 2001 From: cgpu Date: Thu, 11 Mar 2021 19:53:52 +0000 Subject: [PATCH 07/35] Inits processAMachineType to false --- nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index 2c07f82..b9396fc 100644 --- a/nextflow.config +++ b/nextflow.config @@ -3,7 +3,7 @@ docker.enabled = true // NOTE: // Initialise the values of the params to the preferred default value or to false params { - processAMachineType = 'n1-standard-16' + processAMachineType = false processA_memory = '4.GB' processA_cpus = 1 mainScript = 'main.nf' From 0586b930db7c3542b1d850d1b1908ab602215d65 Mon Sep 17 00:00:00 2001 From: cgpu Date: Thu, 11 Mar 2021 20:43:22 +0000 Subject: [PATCH 08/35] Removes machineType --- nextflow.config | 2 -- 1 file changed, 2 deletions(-) diff --git a/nextflow.config b/nextflow.config index b9396fc..d51c2d4 100644 --- a/nextflow.config +++ b/nextflow.config @@ -3,7 +3,6 @@ docker.enabled = true // NOTE: // Initialise the values of the params to the preferred default value or to false params { - processAMachineType = false processA_memory = '4.GB' processA_cpus = 1 mainScript = 'main.nf' @@ -60,7 +59,6 @@ process { withName: processA { cpus = params.processA_cpus memory = params.processA_memory - machineType = params.processAMachineType } } From b522e7be0abf28582115c611f9a926a7e6832a71 Mon Sep 17 00:00:00 2001 From: cgpu Date: Thu, 11 Mar 2021 21:11:47 +0000 Subject: [PATCH 09/35] Adds google.lifeSciences.usePrivateAddress --- conf/google.config | 1 + main.nf | 1 + nextflow.config | 1 + 3 files changed, 3 insertions(+) diff --git a/conf/google.config b/conf/google.config index 88bd6c7..8e17269 100644 --- a/conf/google.config +++ b/conf/google.config @@ -4,4 +4,5 @@ google { zone = params.zone network = params.network subnetwork = params.subnetwork + lifeSciences.usePrivateAddress = params.gls_usePrivateAddress } diff --git a/main.nf b/main.nf index e8a9c3f..819aa11 100644 --- a/main.nf +++ b/main.nf @@ -26,6 +26,7 @@ log.info "executor : ${params.executor}" if(params.executor == 'google-lifesciences') { log.info "gls_bootDiskSize : ${params.gls_bootDiskSize}" log.info "gls_preemptible : ${params.gls_preemptible}" +log.info "gls_usePrivateAddress : ${params.gls_usePrivateAddress}" log.info "zone : ${params.zone}" log.info "network : ${params.network}" log.info "subnetwork : ${params.subnetwork}" diff --git a/nextflow.config b/nextflow.config index d51c2d4..9ef3ca1 100644 --- a/nextflow.config +++ b/nextflow.config @@ -35,6 +35,7 @@ params { // gls params gls_bootDiskSize = false gls_preemptible = false + gls_usePrivateAddress = false zone = false network = false subnetwork = false From 19589e10ab19a7f3672f071dca18cb0e7e9033e3 Mon Sep 17 00:00:00 2001 From: cgpu Date: Thu, 11 Mar 2021 21:31:44 +0000 Subject: [PATCH 10/35] Replaces s3Location with cloudStorageLocation --- main.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.nf b/main.nf index 819aa11..7d6d40e 100644 --- a/main.nf +++ b/main.nf @@ -3,7 +3,7 @@ log.info "\nPARAMETERS SUMMARY" log.info "mainScript : ${params.mainScript}" log.info "defaultBranch : ${params.defaultBranch}" log.info "config : ${params.config}" -log.info "s3Location : ${params.s3Location}" +log.info "cloudStorageLocation : ${params.cloudStorageLocation}" log.info "fileSuffix : ${params.fileSuffix}" log.info "repsProcessA : ${params.repsProcessA}" log.info "processAWriteToDiskMb : ${params.processAWriteToDiskMb}" @@ -37,7 +37,7 @@ numberRepetitionsForProcessA = params.repsProcessA numberFilesForProcessA = params.filesProcessA processAWriteToDiskMb = params.processAWriteToDiskMb processAInput = Channel.from([1] * numberRepetitionsForProcessA) -processAS3InputFiles = Channel.fromPath("${params.s3Location}/*${params.fileSuffix}").take( numberRepetitionsForProcessA ) +processAS3InputFiles = Channel.fromPath("${params.cloudStorageLocation}/*${params.fileSuffix}").take( numberRepetitionsForProcessA ) process processA { publishDir "${params.output}/${task.hash}", mode: 'copy' From 41d2be9815dc818734a71e7d33fb94253f921e1f Mon Sep 17 00:00:00 2001 From: cgpu <38183826+cgpu@users.noreply.github.com> Date: Thu, 11 Mar 2021 22:40:49 +0000 Subject: [PATCH 11/35] Update nextflow.config --- nextflow.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nextflow.config b/nextflow.config index 9ef3ca1..84c49b2 100644 --- a/nextflow.config +++ b/nextflow.config @@ -5,6 +5,7 @@ docker.enabled = true params { processA_memory = '4.GB' processA_cpus = 1 + processA_disk = '20000' mainScript = 'main.nf' defaultBranch = 'main' config = 'conf/standard.config' @@ -58,6 +59,7 @@ process { errorStrategy = params.errorStrategy withName: processA { + disk = params.processA_disk cpus = params.processA_cpus memory = params.processA_memory } From f0a00e099d0bdfa7cf99ed4bb654a8d67ab77ad7 Mon Sep 17 00:00:00 2001 From: cgpu <38183826+cgpu@users.noreply.github.com> Date: Thu, 11 Mar 2021 22:41:31 +0000 Subject: [PATCH 12/35] Update nextflow.config --- nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index 84c49b2..416aef6 100644 --- a/nextflow.config +++ b/nextflow.config @@ -5,7 +5,7 @@ docker.enabled = true params { processA_memory = '4.GB' processA_cpus = 1 - processA_disk = '20000' + processA_disk = '20.GB' mainScript = 'main.nf' defaultBranch = 'main' config = 'conf/standard.config' From a186a4253d64e88f78e0de3a81ab9c4ccc005824 Mon Sep 17 00:00:00 2001 From: cgpu Date: Fri, 12 Mar 2021 00:24:49 +0000 Subject: [PATCH 13/35] Updates publishDir; Updates s3 -> agnostic --- main.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.nf b/main.nf index 7d6d40e..1b74e4c 100644 --- a/main.nf +++ b/main.nf @@ -40,8 +40,8 @@ processAInput = Channel.from([1] * numberRepetitionsForProcessA) processAS3InputFiles = Channel.fromPath("${params.cloudStorageLocation}/*${params.fileSuffix}").take( numberRepetitionsForProcessA ) process processA { - publishDir "${params.output}/${task.hash}", mode: 'copy' - tag "cpus: ${task.cpus}, s3 file: ${s3_file}" + publishDir "${params.output}/processA/${task.hash}", mode: 'copy' + tag "cpus: ${task.cpus}, cloud storage file: ${s3_file}" input: val x from processAInput From 015de3676f708db3eee9ff9cc38b853ec042887b Mon Sep 17 00:00:00 2001 From: cgpu Date: Sun, 14 Mar 2021 09:09:11 +0000 Subject: [PATCH 14/35] Adds option to gls_sshDaemon --- conf/google.config | 1 + nextflow.config | 1 + 2 files changed, 2 insertions(+) diff --git a/conf/google.config b/conf/google.config index 8e17269..e237d8a 100644 --- a/conf/google.config +++ b/conf/google.config @@ -5,4 +5,5 @@ google { network = params.network subnetwork = params.subnetwork lifeSciences.usePrivateAddress = params.gls_usePrivateAddress + google.lifeSciences.sshDaemon = params.gls_sshDaemon } diff --git a/nextflow.config b/nextflow.config index 416aef6..85504ef 100644 --- a/nextflow.config +++ b/nextflow.config @@ -37,6 +37,7 @@ params { gls_bootDiskSize = false gls_preemptible = false gls_usePrivateAddress = false + gls_sshDaemon = false zone = false network = false subnetwork = false From bea3be86e7703c4c006563f7c8432b875e944851 Mon Sep 17 00:00:00 2001 From: cgpu <38183826+cgpu@users.noreply.github.com> Date: Mon, 12 Apr 2021 20:09:05 +0100 Subject: [PATCH 15/35] Adds private ip params in log.info --- main.nf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/main.nf b/main.nf index 1a29b67..88cab13 100644 --- a/main.nf +++ b/main.nf @@ -33,7 +33,9 @@ log.info "gls_preemptible : ${params.gls_preemptible}" log.info "gls_usePrivateAddress : ${params.gls_usePrivateAddress}" log.info "zone : ${params.zone}" log.info "network : ${params.network}" -log.info "subnetwork : ${params.subnetwork}" +log.info "subnetwork : ${params.subnetwork}" +log.info "lifeSciences.usePrivateAddress : ${params.gls_usePrivateAddress}" +log.info "google.lifeSciences.sshDaemon : ${params.gls_sshDaemon}" } log.info "" @@ -110,4 +112,3 @@ process processD { sleep \$timeToWait """ } - From 13e1fd0697b735a2c4d5ab19b5ec79a870081341 Mon Sep 17 00:00:00 2001 From: cgpu <38183826+cgpu@users.noreply.github.com> Date: Mon, 12 Apr 2021 20:09:39 +0100 Subject: [PATCH 16/35] Adds empty line From edc502b6af00df7324d3a756f511259ecac89bbc Mon Sep 17 00:00:00 2001 From: cgpu Date: Fri, 23 Apr 2021 09:53:49 +0100 Subject: [PATCH 17/35] Removes defaultBranch --- nextflow.config | 2 -- 1 file changed, 2 deletions(-) diff --git a/nextflow.config b/nextflow.config index 5f9cfae..07cd3b7 100644 --- a/nextflow.config +++ b/nextflow.config @@ -7,7 +7,6 @@ params { processA_cpus = 1 processA_disk = '20.GB' mainScript = 'main.nf' - defaultBranch = 'stage-s3-files' config = 'conf/standard.config' dataLocation = 's3://lifebit-featured-datasets/pipelines/spammer-nf/input_files' fileSuffix = '' @@ -75,5 +74,4 @@ manifest { homePage = 'https://github.com/lifebit-ai/spammer-nf' description = 'A stampede of processes at your fingertips' mainScript = params.mainScript - defaultBranch = params.defaultBranch } From 28f60196c23fac25d5b9d4a973377f5d3066c9ea Mon Sep 17 00:00:00 2001 From: cgpu Date: Wed, 12 May 2021 10:21:48 +0100 Subject: [PATCH 18/35] Removes log.info "defaultBranch" --- main.nf | 1 - 1 file changed, 1 deletion(-) diff --git a/main.nf b/main.nf index 88cab13..5a69d21 100644 --- a/main.nf +++ b/main.nf @@ -4,7 +4,6 @@ fileSystem = params.dataLocation.contains(':') ? params.dataLocation.split(':')[ // Header log info log.info "\nPARAMETERS SUMMARY" log.info "mainScript : ${params.mainScript}" -log.info "defaultBranch : ${params.defaultBranch}" log.info "config : ${params.config}" log.info "fileSystem : ${fileSystem}" log.info "dataLocation : ${params.dataLocation}" From 13c1cc4c183a57041de18ad0f6a927d83f51a3e5 Mon Sep 17 00:00:00 2001 From: cgpu Date: Wed, 12 May 2021 14:47:55 +0100 Subject: [PATCH 19/35] Sets default gls copyImage to gcr.io --- conf/google.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/google.config b/conf/google.config index 7cf4367..852da10 100644 --- a/conf/google.config +++ b/conf/google.config @@ -4,6 +4,7 @@ params { } google { + lifeSciences.copyImage = 'gcr.io/google.com/cloudsdktool/cloud-sdk:alpine' lifeSciences.bootDiskSize = params.gls_bootDiskSize lifeSciences.preemptible = params.gls_preemptible zone = params.zone From 1f7bc34c681bba1efe3d6f3646c0aecc7aa8f1d8 Mon Sep 17 00:00:00 2001 From: cgpu Date: Wed, 12 May 2021 18:59:48 +0100 Subject: [PATCH 20/35] Adds sshImage and copyImage as params --- conf/google.config | 9 ++++++++- nextflow.config | 4 ++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/conf/google.config b/conf/google.config index 852da10..c912002 100644 --- a/conf/google.config +++ b/conf/google.config @@ -1,10 +1,17 @@ params { executor = 'google-lifesciences' dataLocation = 'gs://lifebit-featured-datasets/pipelines/spammer-nf/input_files' + + // Nextflow default values with correction so that all pulled from gcr.io + gls_copyImage = 'gcr.io/google.com/cloudsdktool/cloud-sdk:alpine' + gls_sshImage = 'gcr.io/cloud-genomics-pipelines/tools' } google { - lifeSciences.copyImage = 'gcr.io/google.com/cloudsdktool/cloud-sdk:alpine' + // both images must be hosted in gcr.io of using private ips (gls_usePrivateAddress true) + lifeSciences.copyImage = params.gls_copyImage + lifeSciences.sshImage = params.gls_sshImage + lifeSciences.bootDiskSize = params.gls_bootDiskSize lifeSciences.preemptible = params.gls_preemptible zone = params.zone diff --git a/nextflow.config b/nextflow.config index 07cd3b7..8fe1846 100644 --- a/nextflow.config +++ b/nextflow.config @@ -40,6 +40,10 @@ params { zone = 'us-east1-b' network = 'default' subnetwork = 'default' + + // values defined in conf/google.config + gls_copyImage = false + gls_sshImage = false } // Do not update the order because the values set in params scope will not be overwritten From db87f427bf8112c8ad599bb006a0b6840f2ddb38 Mon Sep 17 00:00:00 2001 From: cgpu <38183826+cgpu@users.noreply.github.com> Date: Fri, 14 May 2021 10:55:00 +0100 Subject: [PATCH 21/35] Updates container for google cloud config --- conf/google.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/google.config b/conf/google.config index c912002..48785b0 100644 --- a/conf/google.config +++ b/conf/google.config @@ -5,6 +5,7 @@ params { // Nextflow default values with correction so that all pulled from gcr.io gls_copyImage = 'gcr.io/google.com/cloudsdktool/cloud-sdk:alpine' gls_sshImage = 'gcr.io/cloud-genomics-pipelines/tools' + container = 'gcr.io/nextflow-250616/ubuntu:latest' } google { From 4479be57b84bd2fcabec36576a0458027ddaf7fa Mon Sep 17 00:00:00 2001 From: cgpu <38183826+cgpu@users.noreply.github.com> Date: Thu, 20 May 2021 18:07:42 +0100 Subject: [PATCH 22/35] Adds awsbatch.config --- conf/awsbatch.config | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 conf/awsbatch.config diff --git a/conf/awsbatch.config b/conf/awsbatch.config new file mode 100644 index 0000000..c8a944a --- /dev/null +++ b/conf/awsbatch.config @@ -0,0 +1,9 @@ +params { + executor = 'awsbatch' + dataLocation = 's3://lifebit-featured-datasets/pipelines/spammer-nf/input_files' +} + +docker { + enabled = true + runOptions = "--ulimit nofile=65535:65535 --ulimit nproc=65535 --user 0" +} From fd4e2c0dd11fe30b42cfbdb2c9e0536ae4d68ad5 Mon Sep 17 00:00:00 2001 From: cgpu <38183826+cgpu@users.noreply.github.com> Date: Thu, 20 May 2021 20:45:23 +0100 Subject: [PATCH 23/35] Parameterises batch options --- conf/awsbatch.config | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/conf/awsbatch.config b/conf/awsbatch.config index c8a944a..3408bfa 100644 --- a/conf/awsbatch.config +++ b/conf/awsbatch.config @@ -1,9 +1,20 @@ params { executor = 'awsbatch' dataLocation = 's3://lifebit-featured-datasets/pipelines/spammer-nf/input_files' + aws_batch_cliPath = '~/miniconda/bin/aws' + aws_batch_fetchInstanceType = true + aws_batch_process_queue = "nf-batch-4" + aws_batch_docker_run_options = "--ulimit nofile=65535:65535 --ulimit nproc=65535 --user 0" } docker { enabled = true - runOptions = "--ulimit nofile=65535:65535 --ulimit nproc=65535 --user 0" + runOptions = params.docker_run_options } + +process { + queue = false +} + +aws.batch.cliPath = params.aws_batch_cliPath +aws.batch.fetchInstanceType = params.aws_batch_fetchInstanceType From 6f1ab7c95c1d14e3f1e09d59b00964e6f4b9c0d3 Mon Sep 17 00:00:00 2001 From: cgpu <38183826+cgpu@users.noreply.github.com> Date: Thu, 20 May 2021 20:48:27 +0100 Subject: [PATCH 24/35] Adds aws batch params in log.info header --- main.nf | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/main.nf b/main.nf index 5a69d21..d7f84ee 100644 --- a/main.nf +++ b/main.nf @@ -26,15 +26,21 @@ log.info "container : ${params.container}" log.info "maxForks : ${params.maxForks}" log.info "queueSize : ${params.queueSize}" log.info "executor : ${params.executor}" +if(params.executor == 'awsbatch') { +log.info "aws_batch_cliPath : ${params.aws_batch_cliPath}" +log.info "aws_batch_fetchInstanceType : ${params.aws_batch_fetchInstanceType}" +log.info "aws_batch_process_queue : ${params.aws_batch_process_queue}" +log.info "aws_batch_docker_run_options : ${params.aws_batch_docker_run_options}" +} if(params.executor == 'google-lifesciences') { log.info "gls_bootDiskSize : ${params.gls_bootDiskSize}" log.info "gls_preemptible : ${params.gls_preemptible}" log.info "gls_usePrivateAddress : ${params.gls_usePrivateAddress}" log.info "zone : ${params.zone}" log.info "network : ${params.network}" -log.info "subnetwork : ${params.subnetwork}" -log.info "lifeSciences.usePrivateAddress : ${params.gls_usePrivateAddress}" -log.info "google.lifeSciences.sshDaemon : ${params.gls_sshDaemon}" +log.info "subnetwork : ${params.subnetwork}" +log.info "lifeSciences.usePrivateAddress : ${params.gls_usePrivateAddress}" +log.info "google.lifeSciences.sshDaemon : ${params.gls_sshDaemon}" } log.info "" From 8308a1fa5b4f6e8d46b5e2d045c1b18944a06572 Mon Sep 17 00:00:00 2001 From: cgpu <38183826+cgpu@users.noreply.github.com> Date: Thu, 20 May 2021 21:23:24 +0100 Subject: [PATCH 25/35] Fix param name typo --- conf/awsbatch.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/awsbatch.config b/conf/awsbatch.config index 3408bfa..de8ed83 100644 --- a/conf/awsbatch.config +++ b/conf/awsbatch.config @@ -9,7 +9,7 @@ params { docker { enabled = true - runOptions = params.docker_run_options + runOptions = params.aws_batch_docker_run_options } process { From b9bafa9ddf156e7dadfd29b91eaf74b58d9dd642 Mon Sep 17 00:00:00 2001 From: cgpu <38183826+cgpu@users.noreply.github.com> Date: Fri, 21 May 2021 10:12:30 +0100 Subject: [PATCH 26/35] Fix params name --- conf/awsbatch.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/awsbatch.config b/conf/awsbatch.config index de8ed83..f69ffd8 100644 --- a/conf/awsbatch.config +++ b/conf/awsbatch.config @@ -13,7 +13,7 @@ docker { } process { - queue = false + queue = params.aws_batch_process_queue } aws.batch.cliPath = params.aws_batch_cliPath From 38a38736d77d657c3043a736f95809c5f6d667b6 Mon Sep 17 00:00:00 2001 From: cgpu <38183826+cgpu@users.noreply.github.com> Date: Thu, 1 Jul 2021 18:37:22 +0100 Subject: [PATCH 27/35] Update google.config --- conf/google.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/google.config b/conf/google.config index 48785b0..19fabb0 100644 --- a/conf/google.config +++ b/conf/google.config @@ -16,8 +16,8 @@ google { lifeSciences.bootDiskSize = params.gls_bootDiskSize lifeSciences.preemptible = params.gls_preemptible zone = params.zone - network = params.network - subnetwork = params.subnetwork + lifeSciences.network = params.network + lifeSciences.subnetwork = params.subnetwork lifeSciences.usePrivateAddress = params.gls_usePrivateAddress google.lifeSciences.sshDaemon = params.gls_sshDaemon } From 1561fbb50609768409d7bf655b09e4cd739b5c82 Mon Sep 17 00:00:00 2001 From: cgpu <38183826+cgpu@users.noreply.github.com> Date: Wed, 18 Aug 2021 15:51:15 +0100 Subject: [PATCH 28/35] Adds aws_ignite.config in conf/ --- conf/aws_ignite.config | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 conf/aws_ignite.config diff --git a/conf/aws_ignite.config b/conf/aws_ignite.config new file mode 100644 index 0000000..01dc365 --- /dev/null +++ b/conf/aws_ignite.config @@ -0,0 +1,13 @@ +params { + cloud_autoscale_enabled = true + cloud_autoscale_max_instances = 2 +} + +docker.enabled = true + +cloud { + autoscale { + enabled = params.cloud_autoscale_enabled + maxInstances = params.cloud_autoscale_max_instances + } +} From e18bf800ff8a3407e50132ca3f1a7da71529c46f Mon Sep 17 00:00:00 2001 From: cgpu <38183826+cgpu@users.noreply.github.com> Date: Wed, 18 Aug 2021 16:04:58 +0100 Subject: [PATCH 29/35] Experimentally add execution vars in print header --- main.nf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/main.nf b/main.nf index d7f84ee..ede254b 100644 --- a/main.nf +++ b/main.nf @@ -32,6 +32,12 @@ log.info "aws_batch_fetchInstanceType : ${params.aws_batch_fetchInstan log.info "aws_batch_process_queue : ${params.aws_batch_process_queue}" log.info "aws_batch_docker_run_options : ${params.aws_batch_docker_run_options}" } +if(params.config == 'conf/aws_ignite.config') { +log.info "cloud_autoscale_enabled : ${params.cloud_autoscale_enabled}" +log.info "cloud.autoscale.enabled : cloud.autoscale.enabled" +log.info "cloud_autoscale_max_instances : ${params.cloud_autoscale_max_instances}" +log.info "cloud.autoscale.maxInstances : cloud.autoscale.maxInstances " +} if(params.executor == 'google-lifesciences') { log.info "gls_bootDiskSize : ${params.gls_bootDiskSize}" log.info "gls_preemptible : ${params.gls_preemptible}" From 909119ad814a46c70ccd7054925ef84b20cf4b02 Mon Sep 17 00:00:00 2001 From: cgpu <38183826+cgpu@users.noreply.github.com> Date: Wed, 18 Aug 2021 16:06:43 +0100 Subject: [PATCH 30/35] Adds defaults for ignite on AWS --- nextflow.config | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nextflow.config b/nextflow.config index 8fe1846..a1a6735 100644 --- a/nextflow.config +++ b/nextflow.config @@ -44,6 +44,10 @@ params { // values defined in conf/google.config gls_copyImage = false gls_sshImage = false + + // ignite on aws + cloud_autoscale_enabled = false + cloud_autoscale_max_instances = 2 } // Do not update the order because the values set in params scope will not be overwritten From 94ff60a7b6117fb53e11adac1f9cee83d5cd1dc5 Mon Sep 17 00:00:00 2001 From: cgpu <38183826+cgpu@users.noreply.github.com> Date: Wed, 18 Aug 2021 16:07:47 +0100 Subject: [PATCH 31/35] Adds defaults for ignite on AWS From d72c3a296115ba11377f9a24512b9ac6724cad33 Mon Sep 17 00:00:00 2001 From: cgpu Date: Tue, 24 Aug 2021 12:59:37 +0100 Subject: [PATCH 32/35] Adds command-logs folder in results (.command*) --- main.nf | 5 ++++- nextflow.config | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/main.nf b/main.nf index ede254b..600af33 100644 --- a/main.nf +++ b/main.nf @@ -57,7 +57,7 @@ processAInput = Channel.from([1] * numberRepetitionsForProcessA) processAInputFiles = Channel.fromPath("${params.dataLocation}/*${params.fileSuffix}").take( numberRepetitionsForProcessA ) process processA { - publishDir "${params.output}/${task.hash}", mode: 'copy' + publishDir "${params.output}/${task.hash}/", mode: 'copy' tag "cpus: ${task.cpus}, cloud storage: ${cloud_storage_file}" input: @@ -69,6 +69,7 @@ process processA { val x into processCInput val x into processDInput file "*.txt" + file("command-logs") optional true script: """ @@ -82,6 +83,8 @@ process processA { done; sleep \$timeToWait echo "task cpus: ${task.cpus}" + + ${params.savescript} """ } diff --git a/nextflow.config b/nextflow.config index a1a6735..2f4eadf 100644 --- a/nextflow.config +++ b/nextflow.config @@ -15,6 +15,7 @@ params { processATimeRange = "1-2" filesProcessA = 1 processATimeBetweenFileCreationInSecs = 0 + savescript = 'mkdir command-logs; cp .command.* command-logs' processBTimeRange = "2-3" processBWriteToDiskMb = 10 From 161df3075b1554c6e146e57b3ef02c0cc479ce17 Mon Sep 17 00:00:00 2001 From: cgpu <38183826+cgpu@users.noreply.github.com> Date: Tue, 28 Sep 2021 15:44:21 +0100 Subject: [PATCH 33/35] Adds qa.config --- conf/qa.config | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 conf/qa.config diff --git a/conf/qa.config b/conf/qa.config new file mode 100644 index 0000000..4ca54e2 --- /dev/null +++ b/conf/qa.config @@ -0,0 +1,8 @@ +params { + dataLocation = false + fileSufix = "cram" + repsProcessA = 2 + processA_cpus = 4 + processATimeRange = "30-120" + errorStrategy = 'terminate' +} From e66ab5481ae20d46832fe0af34d7ad34850ba176 Mon Sep 17 00:00:00 2001 From: cgpu <38183826+cgpu@users.noreply.github.com> Date: Tue, 28 Sep 2021 15:45:07 +0100 Subject: [PATCH 34/35] Update qa.config --- conf/qa.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/qa.config b/conf/qa.config index 4ca54e2..ea9cece 100644 --- a/conf/qa.config +++ b/conf/qa.config @@ -1,7 +1,7 @@ params { dataLocation = false fileSufix = "cram" - repsProcessA = 2 + repsProcessA = 2 processA_cpus = 4 processATimeRange = "30-120" errorStrategy = 'terminate' From 6cc20eae5c658a5ab29a9c45c69e215097a59157 Mon Sep 17 00:00:00 2001 From: cgpu <38183826+cgpu@users.noreply.github.com> Date: Wed, 6 Oct 2021 10:31:35 +0100 Subject: [PATCH 35/35] Updates regex * -> ** [traverse nested folders] --- main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.nf b/main.nf index 600af33..1aab7cf 100644 --- a/main.nf +++ b/main.nf @@ -54,7 +54,7 @@ numberRepetitionsForProcessA = params.repsProcessA numberFilesForProcessA = params.filesProcessA processAWriteToDiskMb = params.processAWriteToDiskMb processAInput = Channel.from([1] * numberRepetitionsForProcessA) -processAInputFiles = Channel.fromPath("${params.dataLocation}/*${params.fileSuffix}").take( numberRepetitionsForProcessA ) +processAInputFiles = Channel.fromPath("${params.dataLocation}/**${params.fileSuffix}").take( numberRepetitionsForProcessA ) process processA { publishDir "${params.output}/${task.hash}/", mode: 'copy'