Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Tools hive: LSF to Slurm migration #842

Draft
wants to merge 7 commits into
base: main
Choose a base branch
from
Draft

Tools hive: LSF to Slurm migration #842

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
1570bd2
Update beekeeper scripts for the hive 2.2 -> 2.7 migration
azangru Nov 14, 2024
78a4715
update method
jyothishnt Nov 14, 2024
d529a09
code update to slurm
jyothishnt Nov 28, 2024
0ff50d8
remove --partition param
jyothishnt Nov 28, 2024
9f5db8e
update PipeConfig to add highpri queue and remove traces of is_lsf
jyothishnt Dec 16, 2024
7173686
remove traces of lsf
jyothishnt Dec 16, 2024
276abd5
update resource class name and description
jyothishnt Dec 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 29 additions & 29 deletions tools_hive/conf/SiteDefs.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ package EnsEMBL::Tools_hive::SiteDefs;
### tools_hive plugin is a backend for running the ensembl tools jobs (BLAST, BLAT, VEP etc) using ensembl-hive.
### With this plugin, the jobs that are submitted by the website (via the tools plugin) are saved in the
### ENSEMBL_WEB_HIVE db and another process called 'beekeeper' (that could possibly be running on a different
### machine) continually looks at that database, runs the newly submitted jobs on LSF farm or on
### machine) continually looks at that database, runs the newly submitted jobs on the farm or on
### the local machine (LOCAL) where this process itself is running.

use strict;
Expand Down Expand Up @@ -52,7 +52,7 @@ sub update_conf {
'VR' => 'Hive',
}; # Overriding tools plugin variable
$SiteDefs::ENSEMBL_HIVE_HOSTS = []; # For LOCAL, the machine that runs the beekeeper unless it's same as the web server
# For LSF, list of hosts corresponding to the queues for all jobs plus the machine where
# For farm, list of hosts corresponding to the queues for all jobs plus the machine where
# beekeeper is running unless it's same as the web server
# Leave it blank if code is located on a shared disk to share between the web server and the machine(s)
# running beekeeper
Expand Down Expand Up @@ -81,30 +81,30 @@ sub update_conf {

# BLAST configs
$SiteDefs::ENSEMBL_BLAST_RUN_LOCAL = 1; # Flag if on, will run blast jobs on LOCAL meadow
$SiteDefs::ENSEMBL_BLAST_QUEUE = 'highpri'; # LSF or LOCAL queue for blast jobs
$SiteDefs::ENSEMBL_BLAST_LSF_TIMEOUT = undef; # Max timelimit a blast job is allowed to run on LSF
$SiteDefs::ENSEMBL_BLAST_QUEUE = 'highpri'; # farm or LOCAL queue for blast jobs
$SiteDefs::ENSEMBL_BLAST_FARM_TIMEOUT = undef; # Max timelimit a blast job is allowed to run on farm
$SiteDefs::ENSEMBL_BLAST_MEMORY_USAGE = 8; # Memory in GBs required for Blast jobs
$SiteDefs::ENSEMBL_BLAST_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the blast queue (LSF or LOCAL)
$SiteDefs::ENSEMBL_NCBIBLAST_BIN_PATH = '/path/to/ncbi-blast/bin'; # path to blast executables on the LSF host (or local machine if job running locally)
$SiteDefs::ENSEMBL_NCBIBLAST_DATA_PATH = "/path/to/genes"; # path for the blast index files (other than DNA) on the LSF host (or local machine if job running locally)
$SiteDefs::ENSEMBL_NCBIBLAST_DATA_PATH_DNA = "/path/to/blast/dna"; # path for the blast DNA index files on the LSF host (or local machine if job running locally)
$SiteDefs::ENSEMBL_REPEATMASK_BIN_PATH = '/path/to/RepeatMasker'; # path to RepeatMasker executable on the LSF host (or local machine if job running locally)
$SiteDefs::ENSEMBL_BLAST_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the blast queue (farm or LOCAL)
$SiteDefs::ENSEMBL_NCBIBLAST_BIN_PATH = '/path/to/ncbi-blast/bin'; # path to blast executables on the farm host (or local machine if job running locally)
$SiteDefs::ENSEMBL_NCBIBLAST_DATA_PATH = "/path/to/genes"; # path for the blast index files (other than DNA) on the farm host (or local machine if job running locally)
$SiteDefs::ENSEMBL_NCBIBLAST_DATA_PATH_DNA = "/path/to/blast/dna"; # path for the blast DNA index files on the farm host (or local machine if job running locally)
$SiteDefs::ENSEMBL_REPEATMASK_BIN_PATH = '/path/to/RepeatMasker'; # path to RepeatMasker executable on the farm host (or local machine if job running locally)

# BLAT configs
$SiteDefs::ENSEMBL_BLAT_RUN_LOCAL = 1; # Flag if on, will run blat jobs on LOCAL meadow
$SiteDefs::ENSEMBL_BLAT_QUEUE = 'highpri'; # LSF or LOCAL queue for blat jobs
$SiteDefs::ENSEMBL_BLAT_LSF_TIMEOUT = undef; # Max timelimit a blat job is allowed to run on LSF
$SiteDefs::ENSEMBL_BLAT_MEMORY_USAGE = undef; # Memory in GBs required for Blat jobs (undef for default LSF limit)
$SiteDefs::ENSEMBL_BLAT_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the blat queue (LSF or LOCAL)
$SiteDefs::ENSEMBL_BLAT_TWOBIT_DIR = "/path/to/blat/twobit"; # location where blat twobit files are located on LSF node (or local machine if job running locally)
$SiteDefs::ENSEMBL_BLAT_QUEUE = 'highpri'; # farm or LOCAL queue for blat jobs
$SiteDefs::ENSEMBL_BLAT_FARM_TIMEOUT = undef; # Max timelimit a blat job is allowed to run on farm
$SiteDefs::ENSEMBL_BLAT_MEMORY_USAGE = undef; # Memory in GBs required for Blat jobs (undef for default farm limit)
$SiteDefs::ENSEMBL_BLAT_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the blat queue (farm or LOCAL)
$SiteDefs::ENSEMBL_BLAT_TWOBIT_DIR = "/path/to/blat/twobit"; # location where blat twobit files are located on farm node (or local machine if job running locally)
$SiteDefs::ENSEMBL_BLAT_QUERY_COMMAND = '/path/to/command [SPECIES].[ASSEMBLY]'; # optional command line that returns server:port for BLAT server for a given species and assembly

# VEP configs
$SiteDefs::ENSEMBL_VEP_RUN_LOCAL = 1; # Flag if on, will run VEP jobs on LOCAL meadow
$SiteDefs::ENSEMBL_VEP_QUEUE = 'highpri'; # LSF or LOCAL queue for VEP jobs
$SiteDefs::ENSEMBL_VEP_LSF_TIMEOUT = '3:00'; # Max timelimit a VEP job is allowed to run on LSF
$SiteDefs::ENSEMBL_VEP_QUEUE = 'highpri'; # farm or LOCAL queue for VEP jobs
$SiteDefs::ENSEMBL_VEP_FARM_TIMEOUT = '03:00:00'; # Max timelimit a VEP job is allowed to run on farm in HH::MM::SS
$SiteDefs::ENSEMBL_VEP_MEMORY_USAGE = 8; # Memory in GBs required for VEP jobs
$SiteDefs::ENSEMBL_VEP_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the VEP queue (LSF or LOCAL)
$SiteDefs::ENSEMBL_VEP_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the VEP queue (farm or LOCAL)
$SiteDefs::ENSEMBL_VEP_CACHE_DIR = "/path/to/vep/cache"; # path to vep cache files
$SiteDefs::ENSEMBL_VEP_FASTA_DIR = "/path/to/fasta/files"; # path to bgzipped & indexed FASTA files for use by VEP
$SiteDefs::ENSEMBL_VEP_SCRIPT_DEFAULT_OPTIONS = { # Default options for command line vep script (keys with value undef get ignored)
Expand All @@ -115,21 +115,21 @@ sub update_conf {
'fork' => 4, # Enable forking, using 4 forks
};

$SiteDefs::ENSEMBL_VEP_PLUGIN_DATA_DIR = "/path/to/vep/plugin_data"; # path to vep plugin data files on the LSF host (or local machine if job running locally)
$SiteDefs::ENSEMBL_VEP_PLUGIN_DATA_DIR = "/path/to/vep/plugin_data"; # path to vep plugin data files on the farm host (or local machine if job running locally)
$SiteDefs::ENSEMBL_VEP_PLUGIN_DIR = "VEP_plugins"; # path to vep plugin code (if does not start with '/', it's treated relative to ENSEMBL_HIVE_HOSTS_CODE_LOCATION)

push @{$SiteDefs::ENSEMBL_VEP_PLUGIN_CONFIG_FILES}, $SiteDefs::ENSEMBL_SERVERROOT.'/public-plugins/tools_hive/conf/vep_plugins_hive_config.txt';
# add extra hive specific configs required to run vep plugins
# LD configs
$SiteDefs::ENSEMBL_LD_RUN_LOCAL = 1; # Flag if on, will run LD jobs on LOCAL meadow
$SiteDefs::ENSEMBL_LD_QUEUE = 'highpri'; # LSF or LOCAL queue for LD jobs
$SiteDefs::ENSEMBL_LD_LSF_TIMEOUT = undef; # Max timelimit a LD job is allowed to run on LSF
$SiteDefs::ENSEMBL_LD_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the LD queue (LSF or LOCAL)
$SiteDefs::ENSEMBL_LD_QUEUE = 'highpri'; # farm or LOCAL queue for LD jobs
$SiteDefs::ENSEMBL_LD_FARM_TIMEOUT = undef; # Max timelimit a LD job is allowed to run on farm
$SiteDefs::ENSEMBL_LD_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the LD queue (farm or LOCAL)

# Variant Recoder configs
$SiteDefs::ENSEMBL_VR_RUN_LOCAL = 1;
$SiteDefs::ENSEMBL_VR_QUEUE = 'highpri';
$SiteDefs::ENSEMBL_VR_LSF_TIMEOUT = undef;
$SiteDefs::ENSEMBL_VR_FARM_TIMEOUT = undef;
$SiteDefs::ENSEMBL_VR_ANALYSIS_CAPACITY = 500;
$SiteDefs::ENSEMBL_VR_SCRIPT_DEFAULT_OPTIONS = {
'host' => undef,
Expand All @@ -140,27 +140,27 @@ sub update_conf {

# Assembly Converter configs
$SiteDefs::ENSEMBL_AC_RUN_LOCAL = 1; # Flag if on, will run AC jobs on LOCAL meadow
$SiteDefs::ENSEMBL_AC_QUEUE = 'highpri'; # LSF or LOCAL queue for AC jobs
$SiteDefs::ENSEMBL_AC_LSF_TIMEOUT = undef; # Max timelimit an AC job is allowed to run on LSF
$SiteDefs::ENSEMBL_AC_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the queue (LSF or LOCAL)
$SiteDefs::ENSEMBL_AC_QUEUE = 'highpri'; # farm or LOCAL queue for AC jobs
$SiteDefs::ENSEMBL_AC_FARM_TIMEOUT = undef; # Max timelimit an AC job is allowed to run on farm
$SiteDefs::ENSEMBL_AC_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the queue (farm or LOCAL)
$SiteDefs::ENSEMBL_CHAIN_FILE_DIR = '/path/to/assembly_converter/chain_files'; # path to chain files as required by assembly converter
$SiteDefs::ASSEMBLY_CONVERTER_BIN_PATH = '/path/to/CrossMap.py'; # path to CrossMap
$SiteDefs::WIGTOBIGWIG_BIN_PATH = '/path/to/wigToBigWig'; # path to wigToBigWig (required by CrossMap)
$SiteDefs::BIGWIGTOWIG_BIN_PATH = '/path/to/bigWigToWig'; # path to bigWigToWig (needed to RunnableDB/AssemblyConverter)

# ID History converter configs
$SiteDefs::ENSEMBL_IDM_RUN_LOCAL = 1; # Flag if on, will run ID mapper jobs on LOCAL meadow
$SiteDefs::ENSEMBL_IDM_QUEUE = 'highpri'; # LSF or LOCAL queue for ID mapper jobs
$SiteDefs::ENSEMBL_IDM_LSF_TIMEOUT = undef; # Max timelimit an ID mapper job is allowed to run on LSF
$SiteDefs::ENSEMBL_IDM_QUEUE = 'highpri'; # farm or LOCAL queue for ID mapper jobs
$SiteDefs::ENSEMBL_IDM_FARM_TIMEOUT = undef; # Max timelimit an ID mapper job is allowed to run on farm
$SiteDefs::ENSEMBL_IDM_MEMORY_USAGE = 6; # Memory in GBs required for IDMapper jobs
$SiteDefs::ENSEMBL_IDM_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the queue (LSF or LOCAL)
$SiteDefs::ENSEMBL_IDM_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the queue (farm or LOCAL)
$SiteDefs::IDMAPPER_SCRIPT = 'ensembl-tools/scripts/id_history_converter/IDmapper.pl';
# Path to ID History converter script

# File Chameleon configs
$SiteDefs::ENSEMBL_FC_RUN_LOCAL = 1;
$SiteDefs::ENSEMBL_FC_QUEUE = 'highpri';
$SiteDefs::ENSEMBL_FC_LSF_TIMEOUT = undef;
$SiteDefs::ENSEMBL_FC_FARM_TIMEOUT = undef;
$SiteDefs::ENSEMBL_FC_ANALYSIS_CAPACITY = 500;

# Allele Frequency configs
Expand Down
4 changes: 2 additions & 2 deletions tools_hive/modules/EnsEMBL/Web/JobDispatcher/Hive.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ sub dispatch_job {
my $hive_dba = $self->_hive_dba;
my $job_adaptor = $self->_job_adaptor;

$self->{'_analysis'}{$logic_name} ||= $hive_dba->get_AnalysisAdaptor->fetch_by_logic_name_or_url($logic_name);
$self->{'_analysis'}{$logic_name} ||= $hive_dba->get_AnalysisAdaptor->fetch_by_logic_name($logic_name);

# Submit job to hive db
my $hive_job = Bio::EnsEMBL::Hive::AnalysisJob->new(
Expand Down Expand Up @@ -72,7 +72,7 @@ sub delete_jobs {

# if (@hive_job_ids) {
# $self->_job_adaptor->remove_all(sprintf '`job_id` in (%s)', join(',', @hive_job_ids));
# $hive_dba->get_Queen->safe_synchronize_AnalysisStats($hive_dba->get_AnalysisAdaptor->fetch_by_logic_name_or_url($logic_name)->stats);
# $hive_dba->get_Queen->safe_synchronize_AnalysisStats($hive_dba->get_AnalysisAdaptor->fetch_by_logic_name($logic_name)->stats);
# }
}

Expand Down
2 changes: 1 addition & 1 deletion tools_hive/modules/EnsEMBL/Web/RunnableDB/VEP.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ sub run {
my $work_dir = $self->param('work_dir');
my $config = $self->param('config');
my $options = $self->param('script_options') || {};
my $log_file = "$work_dir/lsf_log.txt";
my $log_file = "$work_dir/farm_log.txt";

# path for VEP_plugins (gets pushed to INC by VEP::Runner)
if (my $plugins_path = $self->param('plugins_path')) {
Expand Down
22 changes: 11 additions & 11 deletions tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ use EnsEMBL::Web::Attributes;
sub logic_name :Abstract;
sub runnable :Abstract;
sub queue_name :Abstract;
sub is_lsf :Abstract;
sub lsf_timeout :Abstract;
sub is_farm :Abstract;
sub farm_timeout :Abstract;
sub memory_usage :Abstract;
sub analysis_capacity :Abstract;

Expand All @@ -52,7 +52,7 @@ sub pipeline_analyses {
'-parameters' => {},
'-rc_name' => $class->_resource_class_name,
'-analysis_capacity' => $class->analysis_capacity || 500,
'-meadow_type' => $class->is_lsf ? 'LSF' : 'LOCAL',
'-meadow_type' => $class->is_farm ? 'SLURM' : 'LOCAL',
'-max_retry_count' => 0,
'-failed_job_tolerance' => 100
}];
Expand All @@ -62,26 +62,26 @@ sub _format_resource_class {
## @private
my $class = shift;

return { 'LOCAL' => '' } unless $class->is_lsf;
return { 'LOCAL' => '' } unless $class->is_farm;

my $queue = $class->queue_name;
my $timeout = $class->lsf_timeout;
my $timeout = $class->farm_timeout;
my $memory = $class->memory_usage;
my $default_timeout = '1-00:00:00'; # Days-HH::MM::SS
$timeout ||= $default_timeout;
$memory = $memory ? sprintf('%s', $memory * 1024) : '1600';

$timeout = $timeout ? " -W $timeout" : '';
$memory = $memory ? sprintf(' -M %s -R "rusage[mem=%1$s]"', $memory * 1024) : '';

return { 'LSF' => "-q $queue$timeout$memory" };
return { 'SLURM' => sprintf(" --time=%s --mem=%s%s -n 8 -N 1", %timeout, $memory, 'm') };
}

sub _resource_class_name {
## @private
my $class = shift;
my $rc = $class->_format_resource_class;
my $queue = $class->queue_name || '';
my $timeout = ($class->lsf_timeout || '') =~ s/\:.+$//r;
my $timeout = ($class->farm_timeout || '') =~ s/\:.+$//r;
my $memory = $class->memory_usage || '';
my $str = sprintf('%s %s%s %s%s ', $queue, $timeout ? 'W' : '', $timeout, $memory ? 'M' : '', $memory) =~ s/\W+/-/gr;
my $str = sprintf('%s %s%s %s%s ', $queue, $timeout ? 'T' : '', $timeout, $memory ? 'M' : '', $memory) =~ s/\W+/-/gr;

return sprintf '%s%s', $str, substr(md5_hex(join(' ', %$rc)), 0, 4);
}
Expand Down
4 changes: 2 additions & 2 deletions tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/AlleleFrequency.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ use parent qw(EnsEMBL::Web::ToolsPipeConfig);
sub logic_name { 'AlleleFrequency' }
sub runnable { 'EnsEMBL::Web::RunnableDB::AlleleFrequency' }
sub queue_name { $SiteDefs::ENSEMBL_AF_QUEUE }
sub is_lsf { !$SiteDefs::ENSEMBL_AF_RUN_LOCAL }
sub lsf_timeout { $SiteDefs::ENSEMBL_AF_LSF_TIMEOUT }
sub is_farm { !$SiteDefs::ENSEMBL_AF_RUN_LOCAL }
sub farm_timeout { $SiteDefs::ENSEMBL_AF_FARM_TIMEOUT }
sub memory_usage { $SiteDefs::ENSEMBL_AF_MEMORY_USAGE }
sub analysis_capacity { $SiteDefs::ENSEMBL_AF_ANALYSIS_CAPACITY }

Expand Down
4 changes: 2 additions & 2 deletions tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/AssemblyConverter.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ use parent qw(EnsEMBL::Web::ToolsPipeConfig);
sub logic_name { 'AssemblyConverter' }
sub runnable { 'EnsEMBL::Web::RunnableDB::AssemblyConverter' }
sub queue_name { $SiteDefs::ENSEMBL_AC_QUEUE }
sub is_lsf { !$SiteDefs::ENSEMBL_AC_RUN_LOCAL }
sub lsf_timeout { $SiteDefs::ENSEMBL_AC_LSF_TIMEOUT }
sub is_farm { !$SiteDefs::ENSEMBL_AC_RUN_LOCAL }
sub farm_timeout { $SiteDefs::ENSEMBL_AC_FARM_TIMEOUT }
sub memory_usage { $SiteDefs::ENSEMBL_AC_MEMORY_USAGE }
sub analysis_capacity { $SiteDefs::ENSEMBL_AC_ANALYSIS_CAPACITY }

Expand Down
Loading