From 9201a8f7b7fe36dedc02ebb0cde38e52b6c0e55d Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 14 Nov 2023 14:57:12 -0600 Subject: [PATCH 1/6] feat: Update the queues Haven't figured out a clever way to check for Kim queue access --- conf/utd_ganymede.config | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/conf/utd_ganymede.config b/conf/utd_ganymede.config index 872aa9a4d..879f6f991 100644 --- a/conf/utd_ganymede.config +++ b/conf/utd_ganymede.config @@ -18,23 +18,20 @@ singularity { } def select_queue = { memory, cpu -> - // HACK The smallmem queue is clogged and takes forever to submit a job to right now - // if (memory <= 12.GB && cpu <= 8) { - // return 'smallmem' - // } - if (memory <= 28.GB && cpu <= 16) { + if (memory <= 32.GB && cpu <= 16) { return 'genomics' } - if (memory > 22.GB && memory <= 86.GB && cpu <= 8) { - return '96n' - } - //if (memory > 22.GB && memory <= 120.GB && cpu <= 12) { - // return 'Kim' - //} - if (memory > 22.GB && memory <= 120.GB && cpu <= 16) { + // TODO Find a clever way to check if the user has access + // if (memory > 32.GB && memory <= 125.GB && cpu <= 12) { + // return 'Kim' + // } + if (memory > 32.GB && memory <= 125.GB && cpu <= 16) { return '128s' } - if (memory <= 246.GB && cpu <= 16) { + if (memory <= 250.GB && cpu <= 20) { + return '256i' + } + if (memory <= 250.GB && cpu <= 28) { return '256h' } } @@ -64,7 +61,7 @@ process { } params { - max_memory = 240.GB - max_cpus = 16 + max_memory = 250.GB + max_cpus = 28 max_time = 96.h } From 59a8507defe042652a2dd51557d5781f300cacba Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 14 Nov 2023 15:02:11 -0600 Subject: [PATCH 2/6] feat: Be nicer to the slurm DB --- conf/utd_ganymede.config | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/conf/utd_ganymede.config b/conf/utd_ganymede.config index 879f6f991..01dda567d 100644 --- a/conf/utd_ganymede.config +++ b/conf/utd_ganymede.config @@ -36,6 +36,16 @@ def select_queue = { memory, cpu -> } } +// Submit up to 300 concurrent jobs +// pollInterval and queueStatInterval of every 5 minutes +// submitRateLimit of 20 per minute +executor { + queueSize = 300 + pollInterval = '5 min' + queueStatInterval = '5 min' + submitRateLimit = '20 min' +} + process { beforeScript = 'module load singularity/3.2.1' executor = 'slurm' From 7fd71c1345d7c2ad18f88a1fe8d206393007cc2d Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 14 Nov 2023 15:13:28 -0600 Subject: [PATCH 3/6] feat: Add singularity cache --- conf/utd_ganymede.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/utd_ganymede.config b/conf/utd_ganymede.config index 01dda567d..08f87845f 100644 --- a/conf/utd_ganymede.config +++ b/conf/utd_ganymede.config @@ -6,15 +6,15 @@ params { } env { - TMPDIR = '/home/$USER/scratch/tmp' - SINGULARITY_CACHEDIR = '/home/$USER/scratch/tmp' + TMPDIR = "/home/$USER/scratch/tmp" + SINGULARITY_CACHEDIR="/home/$USER/scratch/singularity" } singularity { enabled = true envWhitelist='SINGULARITY_BINDPATH,LD_LIBRARY_PATH' autoMounts = true - // TODO cacheDir = '/home/${USER}/scratch/tmp/singularity' + cacheDir = "/home/$USER/scratch/singularity" } def select_queue = { memory, cpu -> From 9b4e176edca29207d1f47de921dc6f8d89bb162e Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 14 Nov 2023 15:30:32 -0600 Subject: [PATCH 4/6] Add group checks for kim queue --- conf/utd_ganymede.config | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/conf/utd_ganymede.config b/conf/utd_ganymede.config index 08f87845f..f70b6c48d 100644 --- a/conf/utd_ganymede.config +++ b/conf/utd_ganymede.config @@ -18,13 +18,12 @@ singularity { } def select_queue = { memory, cpu -> - if (memory <= 32.GB && cpu <= 16) { + if (memory <= 32.GB && cpu <= 16 && "groups".execute().text.contains('genomics')) { return 'genomics' } - // TODO Find a clever way to check if the user has access - // if (memory > 32.GB && memory <= 125.GB && cpu <= 12) { - // return 'Kim' - // } + if (memory > 32.GB && memory <= 125.GB && cpu <= 12 && "groups".execute().text.contains('kim')) { + return 'Kim' + } if (memory > 32.GB && memory <= 125.GB && cpu <= 16) { return '128s' } From 1b10c2fd901983efd45257095d3ac72b7745b56d Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 14 Nov 2023 15:33:17 -0600 Subject: [PATCH 5/6] Ask for the queue with the shortest line --- conf/utd_ganymede.config | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/conf/utd_ganymede.config b/conf/utd_ganymede.config index f70b6c48d..895f1918d 100644 --- a/conf/utd_ganymede.config +++ b/conf/utd_ganymede.config @@ -19,19 +19,16 @@ singularity { def select_queue = { memory, cpu -> if (memory <= 32.GB && cpu <= 16 && "groups".execute().text.contains('genomics')) { - return 'genomics' + return 'genomics,normal' } if (memory > 32.GB && memory <= 125.GB && cpu <= 12 && "groups".execute().text.contains('kim')) { - return 'Kim' + return 'Kim,128s' } if (memory > 32.GB && memory <= 125.GB && cpu <= 16) { return '128s' } - if (memory <= 250.GB && cpu <= 20) { - return '256i' - } if (memory <= 250.GB && cpu <= 28) { - return '256h' + return '256i,256h' } } From dc4399281d5b7a59a0233ba4ce69d9c97f7c10ac Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 14 Nov 2023 15:34:32 -0600 Subject: [PATCH 6/6] refactor: Make it more readable And I'm afraid it'll execute multiple times --- conf/utd_ganymede.config | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/conf/utd_ganymede.config b/conf/utd_ganymede.config index 895f1918d..17add9e50 100644 --- a/conf/utd_ganymede.config +++ b/conf/utd_ganymede.config @@ -17,11 +17,13 @@ singularity { cacheDir = "/home/$USER/scratch/singularity" } +def membership = "groups".execute().text + def select_queue = { memory, cpu -> - if (memory <= 32.GB && cpu <= 16 && "groups".execute().text.contains('genomics')) { + if (memory <= 32.GB && cpu <= 16 && membership.contains('genomics')) { return 'genomics,normal' } - if (memory > 32.GB && memory <= 125.GB && cpu <= 12 && "groups".execute().text.contains('kim')) { + if (memory > 32.GB && memory <= 125.GB && cpu <= 12 && membership.contains('kim')) { return 'Kim,128s' } if (memory > 32.GB && memory <= 125.GB && cpu <= 16) {