From 1570bd2a0b963839ab8a53c708e0e509dc88eeff Mon Sep 17 00:00:00 2001 From: Andrey Azov Date: Thu, 14 Nov 2024 14:11:42 +0000 Subject: [PATCH 1/8] Update beekeeper scripts for the hive 2.2 -> 2.7 migration --- tools_hive/utils/beekeeper.pl | 27 +++++++++++++++++---------- tools_hive/utils/beekeeper_manager.pl | 14 +++++++++++--- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/tools_hive/utils/beekeeper.pl b/tools_hive/utils/beekeeper.pl index 5a7b44799..c7eca6a8d 100644 --- a/tools_hive/utils/beekeeper.pl +++ b/tools_hive/utils/beekeeper.pl @@ -27,11 +27,6 @@ $config = eval(uri_unescape($config)); die "Could not parse command line arguments:\n$@" if $@; -# save pid to pid file as provided -open(PID, ">$config->{'pid_file'}") or die "Couldn't open $config->{'pid_file'} file: $!"; -print PID $$; -close PID; - # require SiteDefs and LoadPlugins require $config->{'include_script'}; @@ -48,10 +43,22 @@ @INC = grep -d && !$seen{$_} && ($seen{$_} = 1), (reverse(@SiteDefs::ENSEMBL_LIB_DIRS), map("$_/modules", grep /tools/, @{$SiteDefs::ENSEMBL_PLUGINS}), @{$SiteDefs::ENSEMBL_EXTRA_INC}, @INC); $ENV{'PERL5LIB'} = join ':', @INC; -# Finally, run the script with correct paths. This is done with 'do', not 'system' so it's run as a part -# of the same process. We have already saved it's pid in the pid file which can be used to kill the process. -# It also maintains INCs, but that's not of much use, unfortunately (see above). -@ARGV = @{$config->{'command_args'}}; -do $config->{'script'}; +# Fork the process +my $pid = fork(); + +if (defined $pid) { + if ($pid == 0) { + # Child process + exec($config->{'script'}, @{$config->{'command_args'}}) or die "Couldn't exec: $!"; + } else { + # Parent process + # Save the child's PID to pid file using the provided name + open(PID, ">$config->{'pid_file'}") or die "Couldn't open $config->{'pid_file'} file: $!"; + print PID $pid; + close PID; + } +} else { + die "Fork failed: $!"; +} # DONE diff --git a/tools_hive/utils/beekeeper_manager.pl b/tools_hive/utils/beekeeper_manager.pl index 4dfbc9d31..58c462548 100644 --- a/tools_hive/utils/beekeeper_manager.pl +++ b/tools_hive/utils/beekeeper_manager.pl @@ -37,16 +37,24 @@ use Cwd qw(abs_path); use URI::Escape qw(uri_escape); use Time::localtime; +use Getopt::Long; + +# Configure GetOpt to not panic if it sees options that haven't been explicitly declared +Getopt::Long::Configure("pass_through"); $Bin =~ s|\.snapshot?/[^/]+|latest|; #replacing snapshot in path to latest my $path = $Bin; + my $no_cache = grep { $_ eq '--no_cache_config' } @ARGV; my $redirect_out = grep { $_ eq '--redirect_output' } @ARGV; my $kill = grep { $_ eq '--kill' } @ARGV; -my $sleep_time = grep({ $_ =~ /^\-\-sleep/ } @ARGV) ? undef : "0.5"; +my $keep_alive = grep { $_ eq '--keep_alive' } @ARGV; my $include_script = abs_path("$Bin/../../../ensembl-webcode/conf/includeSiteDefs.pl"); my $command_args = []; +my $sleep_time = 0.5; # default sleep time +GetOptions("sleep=f" => \$sleep_time); + for (@ARGV) { $path = $_ and last if !$path; if ($_ =~ /^\-\-path=?(.*)/) { @@ -117,8 +125,8 @@ # --sleep arg push @$command_args, '--sleep', $sleep_time if $sleep_time; -# any other args in the command line -push @$command_args, grep($_ !~ /^\-\-(redirect_output|no_cache_config|kill|path)$/, @ARGV); +# keep beekeeper running +push @$command_args, '--loop_until', 'FOREVER' if $keep_alive; # wrapper command my $command = sprintf q(perl %s/beekeeper.pl '%s' %s), $Bin, uri_escape(Data::Dumper->new([{ From 78a47157dfd502b1a9fdb42a457e68f99fc25f03 Mon Sep 17 00:00:00 2001 From: Jyothish Date: Thu, 14 Nov 2024 15:18:05 +0000 Subject: [PATCH 2/8] update method --- tools_hive/modules/EnsEMBL/Web/JobDispatcher/Hive.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools_hive/modules/EnsEMBL/Web/JobDispatcher/Hive.pm b/tools_hive/modules/EnsEMBL/Web/JobDispatcher/Hive.pm index 96074ed5a..70cecf829 100644 --- a/tools_hive/modules/EnsEMBL/Web/JobDispatcher/Hive.pm +++ b/tools_hive/modules/EnsEMBL/Web/JobDispatcher/Hive.pm @@ -43,7 +43,7 @@ sub dispatch_job { my $hive_dba = $self->_hive_dba; my $job_adaptor = $self->_job_adaptor; - $self->{'_analysis'}{$logic_name} ||= $hive_dba->get_AnalysisAdaptor->fetch_by_logic_name_or_url($logic_name); + $self->{'_analysis'}{$logic_name} ||= $hive_dba->get_AnalysisAdaptor->fetch_by_logic_name($logic_name); # Submit job to hive db my $hive_job = Bio::EnsEMBL::Hive::AnalysisJob->new( @@ -72,7 +72,7 @@ sub delete_jobs { # if (@hive_job_ids) { # $self->_job_adaptor->remove_all(sprintf '`job_id` in (%s)', join(',', @hive_job_ids)); -# $hive_dba->get_Queen->safe_synchronize_AnalysisStats($hive_dba->get_AnalysisAdaptor->fetch_by_logic_name_or_url($logic_name)->stats); +# $hive_dba->get_Queen->safe_synchronize_AnalysisStats($hive_dba->get_AnalysisAdaptor->fetch_by_logic_name($logic_name)->stats); # } } From d529a092bbe5c8d5e7ac3350a49911a34224d2c8 Mon Sep 17 00:00:00 2001 From: Jyothish Date: Thu, 28 Nov 2024 11:13:47 +0000 Subject: [PATCH 3/8] code update to slurm --- tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm index 8a64a15eb..a9fa2381b 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm @@ -52,7 +52,7 @@ sub pipeline_analyses { '-parameters' => {}, '-rc_name' => $class->_resource_class_name, '-analysis_capacity' => $class->analysis_capacity || 500, - '-meadow_type' => $class->is_lsf ? 'LSF' : 'LOCAL', + '-meadow_type' => $class->is_lsf ? 'SLURM' : 'LOCAL', '-max_retry_count' => 0, '-failed_job_tolerance' => 100 }]; @@ -67,11 +67,13 @@ sub _format_resource_class { my $queue = $class->queue_name; my $timeout = $class->lsf_timeout; my $memory = $class->memory_usage; - + $timeout = $timeout ? " -W $timeout" : ''; $memory = $memory ? sprintf(' -M %s -R "rusage[mem=%1$s]"', $memory * 1024) : ''; - return { 'LSF' => "-q $queue$timeout$memory" }; +# 'SLURM' => ' --partition=standard --time=1-00:00:00 --mem=16000m -n 8 -N 1'}, + + return { 'SLURM' => " --partition=standard --time=1-00:00:00 --mem=16000m -n 8 -N 1" }; } sub _resource_class_name { @@ -81,7 +83,8 @@ sub _resource_class_name { my $queue = $class->queue_name || ''; my $timeout = ($class->lsf_timeout || '') =~ s/\:.+$//r; my $memory = $class->memory_usage || ''; - my $str = sprintf('%s %s%s %s%s ', $queue, $timeout ? 'W' : '', $timeout, $memory ? 'M' : '', $memory) =~ s/\W+/-/gr; + # my $str = sprintf('%s %s%s %s%s ', $queue, $timeout ? 'W' : '', $timeout, $memory ? 'M' : '', $memory) =~ s/\W+/-/gr; + my $str = "standard-T1-M16-"; return sprintf '%s%s', $str, substr(md5_hex(join(' ', %$rc)), 0, 4); } From 0ff50d88de774d6fc2a15ca3f8a9a22c5b157e2c Mon Sep 17 00:00:00 2001 From: Jyothish Date: Thu, 28 Nov 2024 17:19:16 +0000 Subject: [PATCH 4/8] remove --partition param --- tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm index a9fa2381b..c46c76925 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm @@ -73,7 +73,7 @@ sub _format_resource_class { # 'SLURM' => ' --partition=standard --time=1-00:00:00 --mem=16000m -n 8 -N 1'}, - return { 'SLURM' => " --partition=standard --time=1-00:00:00 --mem=16000m -n 8 -N 1" }; + return { 'SLURM' => " --time=1-00:00:00 --mem=16000m -n 8 -N 1" }; } sub _resource_class_name { From 9f5db8ef13d90dd7a0f8f037ff93194c3d0c4e8d Mon Sep 17 00:00:00 2001 From: Jyothish Date: Mon, 16 Dec 2024 12:39:13 +0000 Subject: [PATCH 5/8] update PipeConfig to add highpri queue and remove traces of is_lsf --- tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm | 15 ++++++--------- .../Web/ToolsPipeConfig/AlleleFrequency.pm | 2 +- .../Web/ToolsPipeConfig/AssemblyConverter.pm | 2 +- .../modules/EnsEMBL/Web/ToolsPipeConfig/Blast.pm | 2 +- .../modules/EnsEMBL/Web/ToolsPipeConfig/Blat.pm | 2 +- .../EnsEMBL/Web/ToolsPipeConfig/DataSlicer.pm | 2 +- .../EnsEMBL/Web/ToolsPipeConfig/FileChameleon.pm | 2 +- .../EnsEMBL/Web/ToolsPipeConfig/IDMapper.pm | 2 +- .../modules/EnsEMBL/Web/ToolsPipeConfig/LD.pm | 2 +- .../modules/EnsEMBL/Web/ToolsPipeConfig/VEP.pm | 2 +- .../modules/EnsEMBL/Web/ToolsPipeConfig/VR.pm | 2 +- .../Web/ToolsPipeConfig/VariationPattern.pm | 2 +- .../EnsEMBL/Web/ToolsPipeConfig/VcftoPed.pm | 2 +- 13 files changed, 18 insertions(+), 21 deletions(-) diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm index c46c76925..c1994e5a6 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm @@ -30,7 +30,7 @@ use EnsEMBL::Web::Attributes; sub logic_name :Abstract; sub runnable :Abstract; sub queue_name :Abstract; -sub is_lsf :Abstract; +sub is_farm :Abstract; sub lsf_timeout :Abstract; sub memory_usage :Abstract; sub analysis_capacity :Abstract; @@ -52,7 +52,7 @@ sub pipeline_analyses { '-parameters' => {}, '-rc_name' => $class->_resource_class_name, '-analysis_capacity' => $class->analysis_capacity || 500, - '-meadow_type' => $class->is_lsf ? 'SLURM' : 'LOCAL', + '-meadow_type' => $class->is_farm ? 'SLURM' : 'LOCAL', '-max_retry_count' => 0, '-failed_job_tolerance' => 100 }]; @@ -62,18 +62,16 @@ sub _format_resource_class { ## @private my $class = shift; - return { 'LOCAL' => '' } unless $class->is_lsf; + return { 'LOCAL' => '' } unless $class->is_farm; my $queue = $class->queue_name; my $timeout = $class->lsf_timeout; my $memory = $class->memory_usage; $timeout = $timeout ? " -W $timeout" : ''; - $memory = $memory ? sprintf(' -M %s -R "rusage[mem=%1$s]"', $memory * 1024) : ''; + $memory = $memory ? sprintf('%s', $memory * 1024) : '1600'; -# 'SLURM' => ' --partition=standard --time=1-00:00:00 --mem=16000m -n 8 -N 1'}, - - return { 'SLURM' => " --time=1-00:00:00 --mem=16000m -n 8 -N 1" }; + return { 'SLURM' => sprintf(" --time=1-00:00:00 --mem=%s%s -n 8 -N 1", $memory, 'm') }; } sub _resource_class_name { @@ -83,8 +81,7 @@ sub _resource_class_name { my $queue = $class->queue_name || ''; my $timeout = ($class->lsf_timeout || '') =~ s/\:.+$//r; my $memory = $class->memory_usage || ''; - # my $str = sprintf('%s %s%s %s%s ', $queue, $timeout ? 'W' : '', $timeout, $memory ? 'M' : '', $memory) =~ s/\W+/-/gr; - my $str = "standard-T1-M16-"; + my $str = sprintf('%s %s%s %s%s ', $queue, $timeout ? 'T' : '', $timeout, $memory ? 'M' : '', $memory) =~ s/\W+/-/gr; return sprintf '%s%s', $str, substr(md5_hex(join(' ', %$rc)), 0, 4); } diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/AlleleFrequency.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/AlleleFrequency.pm index 3b5c79fff..0af2734fd 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/AlleleFrequency.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/AlleleFrequency.pm @@ -29,7 +29,7 @@ use parent qw(EnsEMBL::Web::ToolsPipeConfig); sub logic_name { 'AlleleFrequency' } sub runnable { 'EnsEMBL::Web::RunnableDB::AlleleFrequency' } sub queue_name { $SiteDefs::ENSEMBL_AF_QUEUE } -sub is_lsf { !$SiteDefs::ENSEMBL_AF_RUN_LOCAL } +sub is_farm { !$SiteDefs::ENSEMBL_AF_RUN_LOCAL } sub lsf_timeout { $SiteDefs::ENSEMBL_AF_LSF_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_AF_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_AF_ANALYSIS_CAPACITY } diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/AssemblyConverter.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/AssemblyConverter.pm index 736dacd29..bd8568ffb 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/AssemblyConverter.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/AssemblyConverter.pm @@ -29,7 +29,7 @@ use parent qw(EnsEMBL::Web::ToolsPipeConfig); sub logic_name { 'AssemblyConverter' } sub runnable { 'EnsEMBL::Web::RunnableDB::AssemblyConverter' } sub queue_name { $SiteDefs::ENSEMBL_AC_QUEUE } -sub is_lsf { !$SiteDefs::ENSEMBL_AC_RUN_LOCAL } +sub is_farm { !$SiteDefs::ENSEMBL_AC_RUN_LOCAL } sub lsf_timeout { $SiteDefs::ENSEMBL_AC_LSF_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_AC_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_AC_ANALYSIS_CAPACITY } diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/Blast.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/Blast.pm index 7c732ab93..347df8449 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/Blast.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/Blast.pm @@ -29,7 +29,7 @@ use parent qw(EnsEMBL::Web::ToolsPipeConfig); sub logic_name { 'Blast' } sub runnable { 'EnsEMBL::Web::RunnableDB::Blast' } sub queue_name { $SiteDefs::ENSEMBL_BLAST_QUEUE } -sub is_lsf { !$SiteDefs::ENSEMBL_BLAST_RUN_LOCAL } +sub is_farm { !$SiteDefs::ENSEMBL_BLAST_RUN_LOCAL } sub lsf_timeout { $SiteDefs::ENSEMBL_BLAST_LSF_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_BLAST_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_BLAST_ANALYSIS_CAPACITY } diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/Blat.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/Blat.pm index 5af7ea368..d7f99c187 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/Blat.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/Blat.pm @@ -29,7 +29,7 @@ use parent qw(EnsEMBL::Web::ToolsPipeConfig); sub logic_name { 'Blat' } sub runnable { 'EnsEMBL::Web::RunnableDB::Blat' } sub queue_name { $SiteDefs::ENSEMBL_BLAT_QUEUE } -sub is_lsf { !$SiteDefs::ENSEMBL_BLAT_RUN_LOCAL } +sub is_farm { !$SiteDefs::ENSEMBL_BLAT_RUN_LOCAL } sub lsf_timeout { $SiteDefs::ENSEMBL_BLAT_LSF_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_BLAT_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_BLAT_ANALYSIS_CAPACITY } diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/DataSlicer.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/DataSlicer.pm index 320712b30..2daf6fd8d 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/DataSlicer.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/DataSlicer.pm @@ -29,7 +29,7 @@ use parent qw(EnsEMBL::Web::ToolsPipeConfig); sub logic_name { 'DataSlicer' } sub runnable { 'EnsEMBL::Web::RunnableDB::DataSlicer' } sub queue_name { $SiteDefs::ENSEMBL_DS_QUEUE } -sub is_lsf { !$SiteDefs::ENSEMBL_DS_RUN_LOCAL } +sub is_farm { !$SiteDefs::ENSEMBL_DS_RUN_LOCAL } sub lsf_timeout { $SiteDefs::ENSEMBL_DS_LSF_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_DS_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_DS_ANALYSIS_CAPACITY } diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/FileChameleon.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/FileChameleon.pm index a995cb899..84ec87aa6 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/FileChameleon.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/FileChameleon.pm @@ -29,7 +29,7 @@ use parent qw(EnsEMBL::Web::ToolsPipeConfig); sub logic_name { 'FileChameleon' } sub runnable { 'EnsEMBL::Web::RunnableDB::FileChameleon' } sub queue_name { $SiteDefs::ENSEMBL_FC_QUEUE } -sub is_lsf { !$SiteDefs::ENSEMBL_FC_RUN_LOCAL } +sub is_farm { !$SiteDefs::ENSEMBL_FC_RUN_LOCAL } sub lsf_timeout { $SiteDefs::ENSEMBL_FC_LSF_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_FC_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_FC_ANALYSIS_CAPACITY } diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/IDMapper.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/IDMapper.pm index aee8f022b..1663a2734 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/IDMapper.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/IDMapper.pm @@ -29,7 +29,7 @@ use parent qw(EnsEMBL::Web::ToolsPipeConfig); sub logic_name { 'IDMapper' } sub runnable { 'EnsEMBL::Web::RunnableDB::IDMapper' } sub queue_name { $SiteDefs::ENSEMBL_IDM_QUEUE } -sub is_lsf { !$SiteDefs::ENSEMBL_IDM_RUN_LOCAL } +sub is_farm { !$SiteDefs::ENSEMBL_IDM_RUN_LOCAL } sub lsf_timeout { $SiteDefs::ENSEMBL_IDM_LSF_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_IDM_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_IDM_ANALYSIS_CAPACITY } diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/LD.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/LD.pm index 1697cecf2..b8d26abf4 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/LD.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/LD.pm @@ -29,7 +29,7 @@ use parent qw(EnsEMBL::Web::ToolsPipeConfig); sub logic_name { 'LD' } sub runnable { 'EnsEMBL::Web::RunnableDB::LD' } sub queue_name { $SiteDefs::ENSEMBL_LD_QUEUE } -sub is_lsf { !$SiteDefs::ENSEMBL_LD_RUN_LOCAL } +sub is_farm { !$SiteDefs::ENSEMBL_LD_RUN_LOCAL } sub lsf_timeout { $SiteDefs::ENSEMBL_LD_LSF_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_LD_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_LD_ANALYSIS_CAPACITY } diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VEP.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VEP.pm index 1081c0add..4a6e0f732 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VEP.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VEP.pm @@ -29,7 +29,7 @@ use parent qw(EnsEMBL::Web::ToolsPipeConfig); sub logic_name { 'VEP' } sub runnable { 'EnsEMBL::Web::RunnableDB::VEP' } sub queue_name { $SiteDefs::ENSEMBL_VEP_QUEUE } -sub is_lsf { !$SiteDefs::ENSEMBL_VEP_RUN_LOCAL } +sub is_farm { !$SiteDefs::ENSEMBL_VEP_RUN_LOCAL } sub lsf_timeout { $SiteDefs::ENSEMBL_VEP_LSF_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_VEP_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_VEP_ANALYSIS_CAPACITY } diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VR.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VR.pm index 98792e5ea..5096d635d 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VR.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VR.pm @@ -26,7 +26,7 @@ use parent qw(EnsEMBL::Web::ToolsPipeConfig); sub logic_name { 'VR' } sub runnable { 'EnsEMBL::Web::RunnableDB::VR' } sub queue_name { $SiteDefs::ENSEMBL_VR_QUEUE } -sub is_lsf { !$SiteDefs::ENSEMBL_VR_RUN_LOCAL } +sub is_farm { !$SiteDefs::ENSEMBL_VR_RUN_LOCAL } sub lsf_timeout { $SiteDefs::ENSEMBL_VR_LSF_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_VR_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_VR_ANALYSIS_CAPACITY } diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VariationPattern.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VariationPattern.pm index 1617ae2c6..0e137e271 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VariationPattern.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VariationPattern.pm @@ -29,7 +29,7 @@ use parent qw(EnsEMBL::Web::ToolsPipeConfig); sub logic_name { 'VariationPattern' } sub runnable { 'EnsEMBL::Web::RunnableDB::VariationPattern' } sub queue_name { $SiteDefs::ENSEMBL_VPF_QUEUE } -sub is_lsf { !$SiteDefs::ENSEMBL_VPF_RUN_LOCAL } +sub is_farm { !$SiteDefs::ENSEMBL_VPF_RUN_LOCAL } sub lsf_timeout { $SiteDefs::ENSEMBL_VPF_LSF_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_VPF_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_VPF_ANALYSIS_CAPACITY } diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VcftoPed.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VcftoPed.pm index a9b35496e..339596ba7 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VcftoPed.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VcftoPed.pm @@ -29,7 +29,7 @@ use parent qw(EnsEMBL::Web::ToolsPipeConfig); sub logic_name { 'VcftoPed' } sub runnable { 'EnsEMBL::Web::RunnableDB::VcftoPed' } sub queue_name { $SiteDefs::ENSEMBL_VP_QUEUE } -sub is_lsf { !$SiteDefs::ENSEMBL_VP_RUN_LOCAL } +sub is_farm { !$SiteDefs::ENSEMBL_VP_RUN_LOCAL } sub lsf_timeout { $SiteDefs::ENSEMBL_VP_LSF_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_VP_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_VP_ANALYSIS_CAPACITY } From 7173686e5595597c3b09b2aea1681bbfa1410373 Mon Sep 17 00:00:00 2001 From: Jyothish Date: Mon, 16 Dec 2024 15:36:35 +0000 Subject: [PATCH 6/8] remove traces of lsf --- tools_hive/conf/SiteDefs.pm | 58 +++++++++---------- .../modules/EnsEMBL/Web/RunnableDB/VEP.pm | 2 +- .../modules/EnsEMBL/Web/ToolsPipeConfig.pm | 6 +- .../Web/ToolsPipeConfig/AlleleFrequency.pm | 2 +- .../Web/ToolsPipeConfig/AssemblyConverter.pm | 2 +- .../EnsEMBL/Web/ToolsPipeConfig/Blast.pm | 2 +- .../EnsEMBL/Web/ToolsPipeConfig/Blat.pm | 2 +- .../EnsEMBL/Web/ToolsPipeConfig/DataSlicer.pm | 2 +- .../Web/ToolsPipeConfig/FileChameleon.pm | 2 +- .../EnsEMBL/Web/ToolsPipeConfig/IDMapper.pm | 2 +- .../modules/EnsEMBL/Web/ToolsPipeConfig/LD.pm | 2 +- .../EnsEMBL/Web/ToolsPipeConfig/VEP.pm | 2 +- .../modules/EnsEMBL/Web/ToolsPipeConfig/VR.pm | 2 +- .../Web/ToolsPipeConfig/VariationPattern.pm | 2 +- .../EnsEMBL/Web/ToolsPipeConfig/VcftoPed.pm | 2 +- 15 files changed, 45 insertions(+), 45 deletions(-) diff --git a/tools_hive/conf/SiteDefs.pm b/tools_hive/conf/SiteDefs.pm index 73611294e..2bcf74fb6 100644 --- a/tools_hive/conf/SiteDefs.pm +++ b/tools_hive/conf/SiteDefs.pm @@ -22,7 +22,7 @@ package EnsEMBL::Tools_hive::SiteDefs; ### tools_hive plugin is a backend for running the ensembl tools jobs (BLAST, BLAT, VEP etc) using ensembl-hive. ### With this plugin, the jobs that are submitted by the website (via the tools plugin) are saved in the ### ENSEMBL_WEB_HIVE db and another process called 'beekeeper' (that could possibly be running on a different -### machine) continually looks at that database, runs the newly submitted jobs on LSF farm or on +### machine) continually looks at that database, runs the newly submitted jobs on the farm or on ### the local machine (LOCAL) where this process itself is running. use strict; @@ -52,7 +52,7 @@ sub update_conf { 'VR' => 'Hive', }; # Overriding tools plugin variable $SiteDefs::ENSEMBL_HIVE_HOSTS = []; # For LOCAL, the machine that runs the beekeeper unless it's same as the web server - # For LSF, list of hosts corresponding to the queues for all jobs plus the machine where + # For farm, list of hosts corresponding to the queues for all jobs plus the machine where # beekeeper is running unless it's same as the web server # Leave it blank if code is located on a shared disk to share between the web server and the machine(s) # running beekeeper @@ -81,30 +81,30 @@ sub update_conf { # BLAST configs $SiteDefs::ENSEMBL_BLAST_RUN_LOCAL = 1; # Flag if on, will run blast jobs on LOCAL meadow - $SiteDefs::ENSEMBL_BLAST_QUEUE = 'highpri'; # LSF or LOCAL queue for blast jobs - $SiteDefs::ENSEMBL_BLAST_LSF_TIMEOUT = undef; # Max timelimit a blast job is allowed to run on LSF + $SiteDefs::ENSEMBL_BLAST_QUEUE = 'highpri'; # farm or LOCAL queue for blast jobs + $SiteDefs::ENSEMBL_BLAST_FARM_TIMEOUT = undef; # Max timelimit a blast job is allowed to run on farm $SiteDefs::ENSEMBL_BLAST_MEMORY_USAGE = 8; # Memory in GBs required for Blast jobs - $SiteDefs::ENSEMBL_BLAST_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the blast queue (LSF or LOCAL) - $SiteDefs::ENSEMBL_NCBIBLAST_BIN_PATH = '/path/to/ncbi-blast/bin'; # path to blast executables on the LSF host (or local machine if job running locally) - $SiteDefs::ENSEMBL_NCBIBLAST_DATA_PATH = "/path/to/genes"; # path for the blast index files (other than DNA) on the LSF host (or local machine if job running locally) - $SiteDefs::ENSEMBL_NCBIBLAST_DATA_PATH_DNA = "/path/to/blast/dna"; # path for the blast DNA index files on the LSF host (or local machine if job running locally) - $SiteDefs::ENSEMBL_REPEATMASK_BIN_PATH = '/path/to/RepeatMasker'; # path to RepeatMasker executable on the LSF host (or local machine if job running locally) + $SiteDefs::ENSEMBL_BLAST_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the blast queue (farm or LOCAL) + $SiteDefs::ENSEMBL_NCBIBLAST_BIN_PATH = '/path/to/ncbi-blast/bin'; # path to blast executables on the farm host (or local machine if job running locally) + $SiteDefs::ENSEMBL_NCBIBLAST_DATA_PATH = "/path/to/genes"; # path for the blast index files (other than DNA) on the farm host (or local machine if job running locally) + $SiteDefs::ENSEMBL_NCBIBLAST_DATA_PATH_DNA = "/path/to/blast/dna"; # path for the blast DNA index files on the farm host (or local machine if job running locally) + $SiteDefs::ENSEMBL_REPEATMASK_BIN_PATH = '/path/to/RepeatMasker'; # path to RepeatMasker executable on the farm host (or local machine if job running locally) # BLAT configs $SiteDefs::ENSEMBL_BLAT_RUN_LOCAL = 1; # Flag if on, will run blat jobs on LOCAL meadow - $SiteDefs::ENSEMBL_BLAT_QUEUE = 'highpri'; # LSF or LOCAL queue for blat jobs - $SiteDefs::ENSEMBL_BLAT_LSF_TIMEOUT = undef; # Max timelimit a blat job is allowed to run on LSF - $SiteDefs::ENSEMBL_BLAT_MEMORY_USAGE = undef; # Memory in GBs required for Blat jobs (undef for default LSF limit) - $SiteDefs::ENSEMBL_BLAT_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the blat queue (LSF or LOCAL) - $SiteDefs::ENSEMBL_BLAT_TWOBIT_DIR = "/path/to/blat/twobit"; # location where blat twobit files are located on LSF node (or local machine if job running locally) + $SiteDefs::ENSEMBL_BLAT_QUEUE = 'highpri'; # farm or LOCAL queue for blat jobs + $SiteDefs::ENSEMBL_BLAT_FARM_TIMEOUT = undef; # Max timelimit a blat job is allowed to run on farm + $SiteDefs::ENSEMBL_BLAT_MEMORY_USAGE = undef; # Memory in GBs required for Blat jobs (undef for default farm limit) + $SiteDefs::ENSEMBL_BLAT_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the blat queue (farm or LOCAL) + $SiteDefs::ENSEMBL_BLAT_TWOBIT_DIR = "/path/to/blat/twobit"; # location where blat twobit files are located on farm node (or local machine if job running locally) $SiteDefs::ENSEMBL_BLAT_QUERY_COMMAND = '/path/to/command [SPECIES].[ASSEMBLY]'; # optional command line that returns server:port for BLAT server for a given species and assembly # VEP configs $SiteDefs::ENSEMBL_VEP_RUN_LOCAL = 1; # Flag if on, will run VEP jobs on LOCAL meadow - $SiteDefs::ENSEMBL_VEP_QUEUE = 'highpri'; # LSF or LOCAL queue for VEP jobs - $SiteDefs::ENSEMBL_VEP_LSF_TIMEOUT = '3:00'; # Max timelimit a VEP job is allowed to run on LSF + $SiteDefs::ENSEMBL_VEP_QUEUE = 'highpri'; # farm or LOCAL queue for VEP jobs + $SiteDefs::ENSEMBL_VEP_FARM_TIMEOUT = '3:00'; # Max timelimit a VEP job is allowed to run on farm $SiteDefs::ENSEMBL_VEP_MEMORY_USAGE = 8; # Memory in GBs required for VEP jobs - $SiteDefs::ENSEMBL_VEP_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the VEP queue (LSF or LOCAL) + $SiteDefs::ENSEMBL_VEP_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the VEP queue (farm or LOCAL) $SiteDefs::ENSEMBL_VEP_CACHE_DIR = "/path/to/vep/cache"; # path to vep cache files $SiteDefs::ENSEMBL_VEP_FASTA_DIR = "/path/to/fasta/files"; # path to bgzipped & indexed FASTA files for use by VEP $SiteDefs::ENSEMBL_VEP_SCRIPT_DEFAULT_OPTIONS = { # Default options for command line vep script (keys with value undef get ignored) @@ -115,21 +115,21 @@ sub update_conf { 'fork' => 4, # Enable forking, using 4 forks }; - $SiteDefs::ENSEMBL_VEP_PLUGIN_DATA_DIR = "/path/to/vep/plugin_data"; # path to vep plugin data files on the LSF host (or local machine if job running locally) + $SiteDefs::ENSEMBL_VEP_PLUGIN_DATA_DIR = "/path/to/vep/plugin_data"; # path to vep plugin data files on the farm host (or local machine if job running locally) $SiteDefs::ENSEMBL_VEP_PLUGIN_DIR = "VEP_plugins"; # path to vep plugin code (if does not start with '/', it's treated relative to ENSEMBL_HIVE_HOSTS_CODE_LOCATION) push @{$SiteDefs::ENSEMBL_VEP_PLUGIN_CONFIG_FILES}, $SiteDefs::ENSEMBL_SERVERROOT.'/public-plugins/tools_hive/conf/vep_plugins_hive_config.txt'; # add extra hive specific configs required to run vep plugins # LD configs $SiteDefs::ENSEMBL_LD_RUN_LOCAL = 1; # Flag if on, will run LD jobs on LOCAL meadow - $SiteDefs::ENSEMBL_LD_QUEUE = 'highpri'; # LSF or LOCAL queue for LD jobs - $SiteDefs::ENSEMBL_LD_LSF_TIMEOUT = undef; # Max timelimit a LD job is allowed to run on LSF - $SiteDefs::ENSEMBL_LD_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the LD queue (LSF or LOCAL) + $SiteDefs::ENSEMBL_LD_QUEUE = 'highpri'; # farm or LOCAL queue for LD jobs + $SiteDefs::ENSEMBL_LD_FARM_TIMEOUT = undef; # Max timelimit a LD job is allowed to run on farm + $SiteDefs::ENSEMBL_LD_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the LD queue (farm or LOCAL) # Variant Recoder configs $SiteDefs::ENSEMBL_VR_RUN_LOCAL = 1; $SiteDefs::ENSEMBL_VR_QUEUE = 'highpri'; - $SiteDefs::ENSEMBL_VR_LSF_TIMEOUT = undef; + $SiteDefs::ENSEMBL_VR_FARM_TIMEOUT = undef; $SiteDefs::ENSEMBL_VR_ANALYSIS_CAPACITY = 500; $SiteDefs::ENSEMBL_VR_SCRIPT_DEFAULT_OPTIONS = { 'host' => undef, @@ -140,9 +140,9 @@ sub update_conf { # Assembly Converter configs $SiteDefs::ENSEMBL_AC_RUN_LOCAL = 1; # Flag if on, will run AC jobs on LOCAL meadow - $SiteDefs::ENSEMBL_AC_QUEUE = 'highpri'; # LSF or LOCAL queue for AC jobs - $SiteDefs::ENSEMBL_AC_LSF_TIMEOUT = undef; # Max timelimit an AC job is allowed to run on LSF - $SiteDefs::ENSEMBL_AC_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the queue (LSF or LOCAL) + $SiteDefs::ENSEMBL_AC_QUEUE = 'highpri'; # farm or LOCAL queue for AC jobs + $SiteDefs::ENSEMBL_AC_FARM_TIMEOUT = undef; # Max timelimit an AC job is allowed to run on farm + $SiteDefs::ENSEMBL_AC_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the queue (farm or LOCAL) $SiteDefs::ENSEMBL_CHAIN_FILE_DIR = '/path/to/assembly_converter/chain_files'; # path to chain files as required by assembly converter $SiteDefs::ASSEMBLY_CONVERTER_BIN_PATH = '/path/to/CrossMap.py'; # path to CrossMap $SiteDefs::WIGTOBIGWIG_BIN_PATH = '/path/to/wigToBigWig'; # path to wigToBigWig (required by CrossMap) @@ -150,17 +150,17 @@ sub update_conf { # ID History converter configs $SiteDefs::ENSEMBL_IDM_RUN_LOCAL = 1; # Flag if on, will run ID mapper jobs on LOCAL meadow - $SiteDefs::ENSEMBL_IDM_QUEUE = 'highpri'; # LSF or LOCAL queue for ID mapper jobs - $SiteDefs::ENSEMBL_IDM_LSF_TIMEOUT = undef; # Max timelimit an ID mapper job is allowed to run on LSF + $SiteDefs::ENSEMBL_IDM_QUEUE = 'highpri'; # farm or LOCAL queue for ID mapper jobs + $SiteDefs::ENSEMBL_IDM_FARM_TIMEOUT = undef; # Max timelimit an ID mapper job is allowed to run on farm $SiteDefs::ENSEMBL_IDM_MEMORY_USAGE = 6; # Memory in GBs required for IDMapper jobs - $SiteDefs::ENSEMBL_IDM_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the queue (LSF or LOCAL) + $SiteDefs::ENSEMBL_IDM_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the queue (farm or LOCAL) $SiteDefs::IDMAPPER_SCRIPT = 'ensembl-tools/scripts/id_history_converter/IDmapper.pl'; # Path to ID History converter script # File Chameleon configs $SiteDefs::ENSEMBL_FC_RUN_LOCAL = 1; $SiteDefs::ENSEMBL_FC_QUEUE = 'highpri'; - $SiteDefs::ENSEMBL_FC_LSF_TIMEOUT = undef; + $SiteDefs::ENSEMBL_FC_FARM_TIMEOUT = undef; $SiteDefs::ENSEMBL_FC_ANALYSIS_CAPACITY = 500; # Allele Frequency configs diff --git a/tools_hive/modules/EnsEMBL/Web/RunnableDB/VEP.pm b/tools_hive/modules/EnsEMBL/Web/RunnableDB/VEP.pm index cf9ad6cc2..6e67e1aaa 100644 --- a/tools_hive/modules/EnsEMBL/Web/RunnableDB/VEP.pm +++ b/tools_hive/modules/EnsEMBL/Web/RunnableDB/VEP.pm @@ -45,7 +45,7 @@ sub run { my $work_dir = $self->param('work_dir'); my $config = $self->param('config'); my $options = $self->param('script_options') || {}; - my $log_file = "$work_dir/lsf_log.txt"; + my $log_file = "$work_dir/farm_log.txt"; # path for VEP_plugins (gets pushed to INC by VEP::Runner) if (my $plugins_path = $self->param('plugins_path')) { diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm index c1994e5a6..7eee12d26 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm @@ -31,7 +31,7 @@ sub logic_name :Abstract; sub runnable :Abstract; sub queue_name :Abstract; sub is_farm :Abstract; -sub lsf_timeout :Abstract; +sub farm_timeout :Abstract; sub memory_usage :Abstract; sub analysis_capacity :Abstract; @@ -65,7 +65,7 @@ sub _format_resource_class { return { 'LOCAL' => '' } unless $class->is_farm; my $queue = $class->queue_name; - my $timeout = $class->lsf_timeout; + my $timeout = $class->farm_timeout; my $memory = $class->memory_usage; $timeout = $timeout ? " -W $timeout" : ''; @@ -79,7 +79,7 @@ sub _resource_class_name { my $class = shift; my $rc = $class->_format_resource_class; my $queue = $class->queue_name || ''; - my $timeout = ($class->lsf_timeout || '') =~ s/\:.+$//r; + my $timeout = ($class->farm_timeout || '') =~ s/\:.+$//r; my $memory = $class->memory_usage || ''; my $str = sprintf('%s %s%s %s%s ', $queue, $timeout ? 'T' : '', $timeout, $memory ? 'M' : '', $memory) =~ s/\W+/-/gr; diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/AlleleFrequency.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/AlleleFrequency.pm index 0af2734fd..8e0f93219 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/AlleleFrequency.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/AlleleFrequency.pm @@ -30,7 +30,7 @@ sub logic_name { 'AlleleFrequency' } sub runnable { 'EnsEMBL::Web::RunnableDB::AlleleFrequency' } sub queue_name { $SiteDefs::ENSEMBL_AF_QUEUE } sub is_farm { !$SiteDefs::ENSEMBL_AF_RUN_LOCAL } -sub lsf_timeout { $SiteDefs::ENSEMBL_AF_LSF_TIMEOUT } +sub farm_timeout { $SiteDefs::ENSEMBL_AF_FARM_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_AF_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_AF_ANALYSIS_CAPACITY } diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/AssemblyConverter.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/AssemblyConverter.pm index bd8568ffb..cb3c35c60 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/AssemblyConverter.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/AssemblyConverter.pm @@ -30,7 +30,7 @@ sub logic_name { 'AssemblyConverter' } sub runnable { 'EnsEMBL::Web::RunnableDB::AssemblyConverter' } sub queue_name { $SiteDefs::ENSEMBL_AC_QUEUE } sub is_farm { !$SiteDefs::ENSEMBL_AC_RUN_LOCAL } -sub lsf_timeout { $SiteDefs::ENSEMBL_AC_LSF_TIMEOUT } +sub farm_timeout { $SiteDefs::ENSEMBL_AC_FARM_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_AC_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_AC_ANALYSIS_CAPACITY } diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/Blast.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/Blast.pm index 347df8449..d2647976d 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/Blast.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/Blast.pm @@ -30,7 +30,7 @@ sub logic_name { 'Blast' } sub runnable { 'EnsEMBL::Web::RunnableDB::Blast' } sub queue_name { $SiteDefs::ENSEMBL_BLAST_QUEUE } sub is_farm { !$SiteDefs::ENSEMBL_BLAST_RUN_LOCAL } -sub lsf_timeout { $SiteDefs::ENSEMBL_BLAST_LSF_TIMEOUT } +sub farm_timeout { $SiteDefs::ENSEMBL_BLAST_FARM_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_BLAST_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_BLAST_ANALYSIS_CAPACITY } diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/Blat.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/Blat.pm index d7f99c187..0a4b94146 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/Blat.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/Blat.pm @@ -30,7 +30,7 @@ sub logic_name { 'Blat' } sub runnable { 'EnsEMBL::Web::RunnableDB::Blat' } sub queue_name { $SiteDefs::ENSEMBL_BLAT_QUEUE } sub is_farm { !$SiteDefs::ENSEMBL_BLAT_RUN_LOCAL } -sub lsf_timeout { $SiteDefs::ENSEMBL_BLAT_LSF_TIMEOUT } +sub farm_timeout { $SiteDefs::ENSEMBL_BLAT_FARM_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_BLAT_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_BLAT_ANALYSIS_CAPACITY } diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/DataSlicer.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/DataSlicer.pm index 2daf6fd8d..fc8017d83 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/DataSlicer.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/DataSlicer.pm @@ -30,7 +30,7 @@ sub logic_name { 'DataSlicer' } sub runnable { 'EnsEMBL::Web::RunnableDB::DataSlicer' } sub queue_name { $SiteDefs::ENSEMBL_DS_QUEUE } sub is_farm { !$SiteDefs::ENSEMBL_DS_RUN_LOCAL } -sub lsf_timeout { $SiteDefs::ENSEMBL_DS_LSF_TIMEOUT } +sub farm_timeout { $SiteDefs::ENSEMBL_DS_FARM_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_DS_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_DS_ANALYSIS_CAPACITY } diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/FileChameleon.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/FileChameleon.pm index 84ec87aa6..62c7a9c20 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/FileChameleon.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/FileChameleon.pm @@ -30,7 +30,7 @@ sub logic_name { 'FileChameleon' } sub runnable { 'EnsEMBL::Web::RunnableDB::FileChameleon' } sub queue_name { $SiteDefs::ENSEMBL_FC_QUEUE } sub is_farm { !$SiteDefs::ENSEMBL_FC_RUN_LOCAL } -sub lsf_timeout { $SiteDefs::ENSEMBL_FC_LSF_TIMEOUT } +sub farm_timeout { $SiteDefs::ENSEMBL_FC_FARM_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_FC_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_FC_ANALYSIS_CAPACITY } diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/IDMapper.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/IDMapper.pm index 1663a2734..8ae792703 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/IDMapper.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/IDMapper.pm @@ -30,7 +30,7 @@ sub logic_name { 'IDMapper' } sub runnable { 'EnsEMBL::Web::RunnableDB::IDMapper' } sub queue_name { $SiteDefs::ENSEMBL_IDM_QUEUE } sub is_farm { !$SiteDefs::ENSEMBL_IDM_RUN_LOCAL } -sub lsf_timeout { $SiteDefs::ENSEMBL_IDM_LSF_TIMEOUT } +sub farm_timeout { $SiteDefs::ENSEMBL_IDM_FARM_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_IDM_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_IDM_ANALYSIS_CAPACITY } diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/LD.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/LD.pm index b8d26abf4..6b9cd5320 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/LD.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/LD.pm @@ -30,7 +30,7 @@ sub logic_name { 'LD' } sub runnable { 'EnsEMBL::Web::RunnableDB::LD' } sub queue_name { $SiteDefs::ENSEMBL_LD_QUEUE } sub is_farm { !$SiteDefs::ENSEMBL_LD_RUN_LOCAL } -sub lsf_timeout { $SiteDefs::ENSEMBL_LD_LSF_TIMEOUT } +sub farm_timeout { $SiteDefs::ENSEMBL_LD_FARM_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_LD_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_LD_ANALYSIS_CAPACITY } diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VEP.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VEP.pm index 4a6e0f732..4bd226f5b 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VEP.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VEP.pm @@ -30,7 +30,7 @@ sub logic_name { 'VEP' } sub runnable { 'EnsEMBL::Web::RunnableDB::VEP' } sub queue_name { $SiteDefs::ENSEMBL_VEP_QUEUE } sub is_farm { !$SiteDefs::ENSEMBL_VEP_RUN_LOCAL } -sub lsf_timeout { $SiteDefs::ENSEMBL_VEP_LSF_TIMEOUT } +sub farm_timeout { $SiteDefs::ENSEMBL_VEP_FARM_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_VEP_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_VEP_ANALYSIS_CAPACITY } diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VR.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VR.pm index 5096d635d..03c724eb7 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VR.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VR.pm @@ -27,7 +27,7 @@ sub logic_name { 'VR' } sub runnable { 'EnsEMBL::Web::RunnableDB::VR' } sub queue_name { $SiteDefs::ENSEMBL_VR_QUEUE } sub is_farm { !$SiteDefs::ENSEMBL_VR_RUN_LOCAL } -sub lsf_timeout { $SiteDefs::ENSEMBL_VR_LSF_TIMEOUT } +sub farm_timeout { $SiteDefs::ENSEMBL_VR_FARM_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_VR_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_VR_ANALYSIS_CAPACITY } diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VariationPattern.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VariationPattern.pm index 0e137e271..44b72fe8b 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VariationPattern.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VariationPattern.pm @@ -30,7 +30,7 @@ sub logic_name { 'VariationPattern' } sub runnable { 'EnsEMBL::Web::RunnableDB::VariationPattern' } sub queue_name { $SiteDefs::ENSEMBL_VPF_QUEUE } sub is_farm { !$SiteDefs::ENSEMBL_VPF_RUN_LOCAL } -sub lsf_timeout { $SiteDefs::ENSEMBL_VPF_LSF_TIMEOUT } +sub farm_timeout { $SiteDefs::ENSEMBL_VPF_FARM_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_VPF_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_VPF_ANALYSIS_CAPACITY } diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VcftoPed.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VcftoPed.pm index 339596ba7..2fcc9f7fd 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VcftoPed.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VcftoPed.pm @@ -30,7 +30,7 @@ sub logic_name { 'VcftoPed' } sub runnable { 'EnsEMBL::Web::RunnableDB::VcftoPed' } sub queue_name { $SiteDefs::ENSEMBL_VP_QUEUE } sub is_farm { !$SiteDefs::ENSEMBL_VP_RUN_LOCAL } -sub lsf_timeout { $SiteDefs::ENSEMBL_VP_LSF_TIMEOUT } +sub farm_timeout { $SiteDefs::ENSEMBL_VP_FARM_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_VP_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_VP_ANALYSIS_CAPACITY } From 276abd591717e6b7a79bf4ffac3ab49acaebd321 Mon Sep 17 00:00:00 2001 From: Jyothish Date: Mon, 16 Dec 2024 15:56:45 +0000 Subject: [PATCH 7/8] update resource class name and description --- tools_hive/conf/SiteDefs.pm | 2 +- tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm | 6 +++--- tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VEP.pm | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tools_hive/conf/SiteDefs.pm b/tools_hive/conf/SiteDefs.pm index 2bcf74fb6..2721008a9 100644 --- a/tools_hive/conf/SiteDefs.pm +++ b/tools_hive/conf/SiteDefs.pm @@ -102,7 +102,7 @@ sub update_conf { # VEP configs $SiteDefs::ENSEMBL_VEP_RUN_LOCAL = 1; # Flag if on, will run VEP jobs on LOCAL meadow $SiteDefs::ENSEMBL_VEP_QUEUE = 'highpri'; # farm or LOCAL queue for VEP jobs - $SiteDefs::ENSEMBL_VEP_FARM_TIMEOUT = '3:00'; # Max timelimit a VEP job is allowed to run on farm + $SiteDefs::ENSEMBL_VEP_FARM_TIMEOUT = '03:00:00'; # Max timelimit a VEP job is allowed to run on farm in HH::MM::SS $SiteDefs::ENSEMBL_VEP_MEMORY_USAGE = 8; # Memory in GBs required for VEP jobs $SiteDefs::ENSEMBL_VEP_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the VEP queue (farm or LOCAL) $SiteDefs::ENSEMBL_VEP_CACHE_DIR = "/path/to/vep/cache"; # path to vep cache files diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm index 7eee12d26..28d1001c1 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm @@ -67,11 +67,11 @@ sub _format_resource_class { my $queue = $class->queue_name; my $timeout = $class->farm_timeout; my $memory = $class->memory_usage; - - $timeout = $timeout ? " -W $timeout" : ''; + my $default_timeout = '1-00:00:00'; # Days-HH::MM::SS + $timeout ||= $default_timeout; $memory = $memory ? sprintf('%s', $memory * 1024) : '1600'; - return { 'SLURM' => sprintf(" --time=1-00:00:00 --mem=%s%s -n 8 -N 1", $memory, 'm') }; + return { 'SLURM' => sprintf(" --time=%s --mem=%s%s -n 8 -N 1", %timeout, $memory, 'm') }; } sub _resource_class_name { diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VEP.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VEP.pm index 4bd226f5b..856a64915 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VEP.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/VEP.pm @@ -30,7 +30,7 @@ sub logic_name { 'VEP' } sub runnable { 'EnsEMBL::Web::RunnableDB::VEP' } sub queue_name { $SiteDefs::ENSEMBL_VEP_QUEUE } sub is_farm { !$SiteDefs::ENSEMBL_VEP_RUN_LOCAL } -sub farm_timeout { $SiteDefs::ENSEMBL_VEP_FARM_TIMEOUT } +sub farm_timeout { $SiteDefs::ENSEMBL_VEP_FARM_TIMEOUT } sub memory_usage { $SiteDefs::ENSEMBL_VEP_MEMORY_USAGE } sub analysis_capacity { $SiteDefs::ENSEMBL_VEP_ANALYSIS_CAPACITY } From 4617a2103a778d64b8fda9c059c02fb6438b9abf Mon Sep 17 00:00:00 2001 From: Jyothish Date: Tue, 7 Jan 2025 12:17:29 +0000 Subject: [PATCH 8/8] update resource class desc --- tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm index 28d1001c1..b45f00481 100644 --- a/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm +++ b/tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm @@ -65,13 +65,14 @@ sub _format_resource_class { return { 'LOCAL' => '' } unless $class->is_farm; my $queue = $class->queue_name; + my $partition = $queue ? "--partition=$queue" : ""; my $timeout = $class->farm_timeout; my $memory = $class->memory_usage; my $default_timeout = '1-00:00:00'; # Days-HH::MM::SS $timeout ||= $default_timeout; $memory = $memory ? sprintf('%s', $memory * 1024) : '1600'; - return { 'SLURM' => sprintf(" --time=%s --mem=%s%s -n 8 -N 1", %timeout, $memory, 'm') }; + return { 'SLURM' => sprintf("$partition --time=%s --mem=%s%s -n 8 -N 1", $timeout, $memory, 'm') }; } sub _resource_class_name {