Skip to content

Commit 5a7a664

Browse files
azangruazangru
authored
Tools hive: LSF to Slurm migration (#842)
1 parent 1762c7d commit 5a7a664

18 files changed

+96
-80
lines changed

‎tools_hive/conf/SiteDefs.pm

+29-29
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ package EnsEMBL::Tools_hive::SiteDefs;
2222
### tools_hive plugin is a backend for running the ensembl tools jobs (BLAST, BLAT, VEP etc) using ensembl-hive.
2323
### With this plugin, the jobs that are submitted by the website (via the tools plugin) are saved in the
2424
### ENSEMBL_WEB_HIVE db and another process called 'beekeeper' (that could possibly be running on a different
25-
### machine) continually looks at that database, runs the newly submitted jobs on LSF farm or on
25+
### machine) continually looks at that database, runs the newly submitted jobs on the farm or on
2626
### the local machine (LOCAL) where this process itself is running.
2727

2828
use strict;
@@ -52,7 +52,7 @@ sub update_conf {
5252
'VR' => 'Hive',
5353
}; # Overriding tools plugin variable
5454
$SiteDefs::ENSEMBL_HIVE_HOSTS = []; # For LOCAL, the machine that runs the beekeeper unless it's same as the web server
55-
# For LSF, list of hosts corresponding to the queues for all jobs plus the machine where
55+
# For farm, list of hosts corresponding to the queues for all jobs plus the machine where
5656
# beekeeper is running unless it's same as the web server
5757
# Leave it blank if code is located on a shared disk to share between the web server and the machine(s)
5858
# running beekeeper
@@ -81,30 +81,30 @@ sub update_conf {
8181

8282
# BLAST configs
8383
$SiteDefs::ENSEMBL_BLAST_RUN_LOCAL = 1; # Flag if on, will run blast jobs on LOCAL meadow
84-
$SiteDefs::ENSEMBL_BLAST_QUEUE = 'highpri'; # LSF or LOCAL queue for blast jobs
85-
$SiteDefs::ENSEMBL_BLAST_LSF_TIMEOUT = undef; # Max timelimit a blast job is allowed to run on LSF
84+
$SiteDefs::ENSEMBL_BLAST_QUEUE = 'highpri'; # farm or LOCAL queue for blast jobs
85+
$SiteDefs::ENSEMBL_BLAST_FARM_TIMEOUT = undef; # Max timelimit a blast job is allowed to run on farm
8686
$SiteDefs::ENSEMBL_BLAST_MEMORY_USAGE = 8; # Memory in GBs required for Blast jobs
87-
$SiteDefs::ENSEMBL_BLAST_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the blast queue (LSF or LOCAL)
88-
$SiteDefs::ENSEMBL_NCBIBLAST_BIN_PATH = '/path/to/ncbi-blast/bin'; # path to blast executables on the LSF host (or local machine if job running locally)
89-
$SiteDefs::ENSEMBL_NCBIBLAST_DATA_PATH = "/path/to/genes"; # path for the blast index files (other than DNA) on the LSF host (or local machine if job running locally)
90-
$SiteDefs::ENSEMBL_NCBIBLAST_DATA_PATH_DNA = "/path/to/blast/dna"; # path for the blast DNA index files on the LSF host (or local machine if job running locally)
91-
$SiteDefs::ENSEMBL_REPEATMASK_BIN_PATH = '/path/to/RepeatMasker'; # path to RepeatMasker executable on the LSF host (or local machine if job running locally)
87+
$SiteDefs::ENSEMBL_BLAST_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the blast queue (farm or LOCAL)
88+
$SiteDefs::ENSEMBL_NCBIBLAST_BIN_PATH = '/path/to/ncbi-blast/bin'; # path to blast executables on the farm host (or local machine if job running locally)
89+
$SiteDefs::ENSEMBL_NCBIBLAST_DATA_PATH = "/path/to/genes"; # path for the blast index files (other than DNA) on the farm host (or local machine if job running locally)
90+
$SiteDefs::ENSEMBL_NCBIBLAST_DATA_PATH_DNA = "/path/to/blast/dna"; # path for the blast DNA index files on the farm host (or local machine if job running locally)
91+
$SiteDefs::ENSEMBL_REPEATMASK_BIN_PATH = '/path/to/RepeatMasker'; # path to RepeatMasker executable on the farm host (or local machine if job running locally)
9292

9393
# BLAT configs
9494
$SiteDefs::ENSEMBL_BLAT_RUN_LOCAL = 1; # Flag if on, will run blat jobs on LOCAL meadow
95-
$SiteDefs::ENSEMBL_BLAT_QUEUE = 'highpri'; # LSF or LOCAL queue for blat jobs
96-
$SiteDefs::ENSEMBL_BLAT_LSF_TIMEOUT = undef; # Max timelimit a blat job is allowed to run on LSF
97-
$SiteDefs::ENSEMBL_BLAT_MEMORY_USAGE = undef; # Memory in GBs required for Blat jobs (undef for default LSF limit)
98-
$SiteDefs::ENSEMBL_BLAT_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the blat queue (LSF or LOCAL)
99-
$SiteDefs::ENSEMBL_BLAT_TWOBIT_DIR = "/path/to/blat/twobit"; # location where blat twobit files are located on LSF node (or local machine if job running locally)
95+
$SiteDefs::ENSEMBL_BLAT_QUEUE = 'highpri'; # farm or LOCAL queue for blat jobs
96+
$SiteDefs::ENSEMBL_BLAT_FARM_TIMEOUT = undef; # Max timelimit a blat job is allowed to run on farm
97+
$SiteDefs::ENSEMBL_BLAT_MEMORY_USAGE = undef; # Memory in GBs required for Blat jobs (undef for default farm limit)
98+
$SiteDefs::ENSEMBL_BLAT_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the blat queue (farm or LOCAL)
99+
$SiteDefs::ENSEMBL_BLAT_TWOBIT_DIR = "/path/to/blat/twobit"; # location where blat twobit files are located on farm node (or local machine if job running locally)
100100
$SiteDefs::ENSEMBL_BLAT_QUERY_COMMAND = '/path/to/command [SPECIES].[ASSEMBLY]'; # optional command line that returns server:port for BLAT server for a given species and assembly
101101

102102
# VEP configs
103103
$SiteDefs::ENSEMBL_VEP_RUN_LOCAL = 1; # Flag if on, will run VEP jobs on LOCAL meadow
104-
$SiteDefs::ENSEMBL_VEP_QUEUE = 'highpri'; # LSF or LOCAL queue for VEP jobs
105-
$SiteDefs::ENSEMBL_VEP_LSF_TIMEOUT = '3:00'; # Max timelimit a VEP job is allowed to run on LSF
104+
$SiteDefs::ENSEMBL_VEP_QUEUE = 'highpri'; # farm or LOCAL queue for VEP jobs
105+
$SiteDefs::ENSEMBL_VEP_FARM_TIMEOUT = '03:00:00'; # Max timelimit a VEP job is allowed to run on farm in HH::MM::SS
106106
$SiteDefs::ENSEMBL_VEP_MEMORY_USAGE = 8; # Memory in GBs required for VEP jobs
107-
$SiteDefs::ENSEMBL_VEP_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the VEP queue (LSF or LOCAL)
107+
$SiteDefs::ENSEMBL_VEP_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the VEP queue (farm or LOCAL)
108108
$SiteDefs::ENSEMBL_VEP_CACHE_DIR = "/path/to/vep/cache"; # path to vep cache files
109109
$SiteDefs::ENSEMBL_VEP_FASTA_DIR = "/path/to/fasta/files"; # path to bgzipped & indexed FASTA files for use by VEP
110110
$SiteDefs::ENSEMBL_VEP_SCRIPT_DEFAULT_OPTIONS = { # Default options for command line vep script (keys with value undef get ignored)
@@ -115,21 +115,21 @@ sub update_conf {
115115
'fork' => 4, # Enable forking, using 4 forks
116116
};
117117

118-
$SiteDefs::ENSEMBL_VEP_PLUGIN_DATA_DIR = "/path/to/vep/plugin_data"; # path to vep plugin data files on the LSF host (or local machine if job running locally)
118+
$SiteDefs::ENSEMBL_VEP_PLUGIN_DATA_DIR = "/path/to/vep/plugin_data"; # path to vep plugin data files on the farm host (or local machine if job running locally)
119119
$SiteDefs::ENSEMBL_VEP_PLUGIN_DIR = "VEP_plugins"; # path to vep plugin code (if does not start with '/', it's treated relative to ENSEMBL_HIVE_HOSTS_CODE_LOCATION)
120120

121121
push @{$SiteDefs::ENSEMBL_VEP_PLUGIN_CONFIG_FILES}, $SiteDefs::ENSEMBL_SERVERROOT.'/public-plugins/tools_hive/conf/vep_plugins_hive_config.txt';
122122
# add extra hive specific configs required to run vep plugins
123123
# LD configs
124124
$SiteDefs::ENSEMBL_LD_RUN_LOCAL = 1; # Flag if on, will run LD jobs on LOCAL meadow
125-
$SiteDefs::ENSEMBL_LD_QUEUE = 'highpri'; # LSF or LOCAL queue for LD jobs
126-
$SiteDefs::ENSEMBL_LD_LSF_TIMEOUT = undef; # Max timelimit a LD job is allowed to run on LSF
127-
$SiteDefs::ENSEMBL_LD_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the LD queue (LSF or LOCAL)
125+
$SiteDefs::ENSEMBL_LD_QUEUE = 'highpri'; # farm or LOCAL queue for LD jobs
126+
$SiteDefs::ENSEMBL_LD_FARM_TIMEOUT = undef; # Max timelimit a LD job is allowed to run on farm
127+
$SiteDefs::ENSEMBL_LD_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the LD queue (farm or LOCAL)
128128

129129
# Variant Recoder configs
130130
$SiteDefs::ENSEMBL_VR_RUN_LOCAL = 1;
131131
$SiteDefs::ENSEMBL_VR_QUEUE = 'highpri';
132-
$SiteDefs::ENSEMBL_VR_LSF_TIMEOUT = undef;
132+
$SiteDefs::ENSEMBL_VR_FARM_TIMEOUT = undef;
133133
$SiteDefs::ENSEMBL_VR_ANALYSIS_CAPACITY = 500;
134134
$SiteDefs::ENSEMBL_VR_SCRIPT_DEFAULT_OPTIONS = {
135135
'host' => undef,
@@ -140,27 +140,27 @@ sub update_conf {
140140

141141
# Assembly Converter configs
142142
$SiteDefs::ENSEMBL_AC_RUN_LOCAL = 1; # Flag if on, will run AC jobs on LOCAL meadow
143-
$SiteDefs::ENSEMBL_AC_QUEUE = 'highpri'; # LSF or LOCAL queue for AC jobs
144-
$SiteDefs::ENSEMBL_AC_LSF_TIMEOUT = undef; # Max timelimit an AC job is allowed to run on LSF
145-
$SiteDefs::ENSEMBL_AC_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the queue (LSF or LOCAL)
143+
$SiteDefs::ENSEMBL_AC_QUEUE = 'highpri'; # farm or LOCAL queue for AC jobs
144+
$SiteDefs::ENSEMBL_AC_FARM_TIMEOUT = undef; # Max timelimit an AC job is allowed to run on farm
145+
$SiteDefs::ENSEMBL_AC_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the queue (farm or LOCAL)
146146
$SiteDefs::ENSEMBL_CHAIN_FILE_DIR = '/path/to/assembly_converter/chain_files'; # path to chain files as required by assembly converter
147147
$SiteDefs::ASSEMBLY_CONVERTER_BIN_PATH = '/path/to/CrossMap.py'; # path to CrossMap
148148
$SiteDefs::WIGTOBIGWIG_BIN_PATH = '/path/to/wigToBigWig'; # path to wigToBigWig (required by CrossMap)
149149
$SiteDefs::BIGWIGTOWIG_BIN_PATH = '/path/to/bigWigToWig'; # path to bigWigToWig (needed to RunnableDB/AssemblyConverter)
150150

151151
# ID History converter configs
152152
$SiteDefs::ENSEMBL_IDM_RUN_LOCAL = 1; # Flag if on, will run ID mapper jobs on LOCAL meadow
153-
$SiteDefs::ENSEMBL_IDM_QUEUE = 'highpri'; # LSF or LOCAL queue for ID mapper jobs
154-
$SiteDefs::ENSEMBL_IDM_LSF_TIMEOUT = undef; # Max timelimit an ID mapper job is allowed to run on LSF
153+
$SiteDefs::ENSEMBL_IDM_QUEUE = 'highpri'; # farm or LOCAL queue for ID mapper jobs
154+
$SiteDefs::ENSEMBL_IDM_FARM_TIMEOUT = undef; # Max timelimit an ID mapper job is allowed to run on farm
155155
$SiteDefs::ENSEMBL_IDM_MEMORY_USAGE = 6; # Memory in GBs required for IDMapper jobs
156-
$SiteDefs::ENSEMBL_IDM_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the queue (LSF or LOCAL)
156+
$SiteDefs::ENSEMBL_IDM_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the queue (farm or LOCAL)
157157
$SiteDefs::IDMAPPER_SCRIPT = 'ensembl-tools/scripts/id_history_converter/IDmapper.pl';
158158
# Path to ID History converter script
159159

160160
# File Chameleon configs
161161
$SiteDefs::ENSEMBL_FC_RUN_LOCAL = 1;
162162
$SiteDefs::ENSEMBL_FC_QUEUE = 'highpri';
163-
$SiteDefs::ENSEMBL_FC_LSF_TIMEOUT = undef;
163+
$SiteDefs::ENSEMBL_FC_FARM_TIMEOUT = undef;
164164
$SiteDefs::ENSEMBL_FC_ANALYSIS_CAPACITY = 500;
165165

166166
# Allele Frequency configs

‎tools_hive/modules/EnsEMBL/Web/JobDispatcher/Hive.pm

+2-2
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ sub dispatch_job {
4343
my $hive_dba = $self->_hive_dba;
4444
my $job_adaptor = $self->_job_adaptor;
4545

46-
$self->{'_analysis'}{$logic_name} ||= $hive_dba->get_AnalysisAdaptor->fetch_by_logic_name_or_url($logic_name);
46+
$self->{'_analysis'}{$logic_name} ||= $hive_dba->get_AnalysisAdaptor->fetch_by_logic_name($logic_name);
4747

4848
# Submit job to hive db
4949
my $hive_job = Bio::EnsEMBL::Hive::AnalysisJob->new(
@@ -72,7 +72,7 @@ sub delete_jobs {
7272

7373
# if (@hive_job_ids) {
7474
# $self->_job_adaptor->remove_all(sprintf '`job_id` in (%s)', join(',', @hive_job_ids));
75-
# $hive_dba->get_Queen->safe_synchronize_AnalysisStats($hive_dba->get_AnalysisAdaptor->fetch_by_logic_name_or_url($logic_name)->stats);
75+
# $hive_dba->get_Queen->safe_synchronize_AnalysisStats($hive_dba->get_AnalysisAdaptor->fetch_by_logic_name($logic_name)->stats);
7676
# }
7777
}
7878

‎tools_hive/modules/EnsEMBL/Web/RunnableDB/VEP.pm

+1-1
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ sub run {
4545
my $work_dir = $self->param('work_dir');
4646
my $config = $self->param('config');
4747
my $options = $self->param('script_options') || {};
48-
my $log_file = "$work_dir/lsf_log.txt";
48+
my $log_file = "$work_dir/farm_log.txt";
4949

5050
# path for VEP_plugins (gets pushed to INC by VEP::Runner)
5151
if (my $plugins_path = $self->param('plugins_path')) {

‎tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm

+12-11
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@ use EnsEMBL::Web::Attributes;
3030
sub logic_name :Abstract;
3131
sub runnable :Abstract;
3232
sub queue_name :Abstract;
33-
sub is_lsf :Abstract;
34-
sub lsf_timeout :Abstract;
33+
sub is_farm :Abstract;
34+
sub farm_timeout :Abstract;
3535
sub memory_usage :Abstract;
3636
sub analysis_capacity :Abstract;
3737

@@ -52,7 +52,7 @@ sub pipeline_analyses {
5252
'-parameters' => {},
5353
'-rc_name' => $class->_resource_class_name,
5454
'-analysis_capacity' => $class->analysis_capacity || 500,
55-
'-meadow_type' => $class->is_lsf ? 'LSF' : 'LOCAL',
55+
'-meadow_type' => $class->is_farm ? 'SLURM' : 'LOCAL',
5656
'-max_retry_count' => 0,
5757
'-failed_job_tolerance' => 100
5858
}];
@@ -62,26 +62,27 @@ sub _format_resource_class {
6262
## @private
6363
my $class = shift;
6464

65-
return { 'LOCAL' => '' } unless $class->is_lsf;
65+
return { 'LOCAL' => '' } unless $class->is_farm;
6666

6767
my $queue = $class->queue_name;
68-
my $timeout = $class->lsf_timeout;
68+
my $partition = $queue ? "--partition=$queue" : "";
69+
my $timeout = $class->farm_timeout;
6970
my $memory = $class->memory_usage;
71+
my $default_timeout = '1-00:00:00'; # Days-HH::MM::SS
72+
$timeout ||= $default_timeout;
73+
$memory = $memory ? sprintf('%s', $memory * 1024) : '1600';
7074

71-
$timeout = $timeout ? " -W $timeout" : '';
72-
$memory = $memory ? sprintf(' -M %s -R "rusage[mem=%1$s]"', $memory * 1024) : '';
73-
74-
return { 'LSF' => "-q $queue$timeout$memory" };
75+
return { 'SLURM' => sprintf("$partition --time=%s --mem=%s%s -n 8 -N 1", $timeout, $memory, 'm') };
7576
}
7677

7778
sub _resource_class_name {
7879
## @private
7980
my $class = shift;
8081
my $rc = $class->_format_resource_class;
8182
my $queue = $class->queue_name || '';
82-
my $timeout = ($class->lsf_timeout || '') =~ s/\:.+$//r;
83+
my $timeout = ($class->farm_timeout || '') =~ s/\:.+$//r;
8384
my $memory = $class->memory_usage || '';
84-
my $str = sprintf('%s %s%s %s%s ', $queue, $timeout ? 'W' : '', $timeout, $memory ? 'M' : '', $memory) =~ s/\W+/-/gr;
85+
my $str = sprintf('%s %s%s %s%s ', $queue, $timeout ? 'T' : '', $timeout, $memory ? 'M' : '', $memory) =~ s/\W+/-/gr;
8586

8687
return sprintf '%s%s', $str, substr(md5_hex(join(' ', %$rc)), 0, 4);
8788
}

‎tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/AlleleFrequency.pm

+2-2
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@ use parent qw(EnsEMBL::Web::ToolsPipeConfig);
2929
sub logic_name { 'AlleleFrequency' }
3030
sub runnable { 'EnsEMBL::Web::RunnableDB::AlleleFrequency' }
3131
sub queue_name { $SiteDefs::ENSEMBL_AF_QUEUE }
32-
sub is_lsf { !$SiteDefs::ENSEMBL_AF_RUN_LOCAL }
33-
sub lsf_timeout { $SiteDefs::ENSEMBL_AF_LSF_TIMEOUT }
32+
sub is_farm { !$SiteDefs::ENSEMBL_AF_RUN_LOCAL }
33+
sub farm_timeout { $SiteDefs::ENSEMBL_AF_FARM_TIMEOUT }
3434
sub memory_usage { $SiteDefs::ENSEMBL_AF_MEMORY_USAGE }
3535
sub analysis_capacity { $SiteDefs::ENSEMBL_AF_ANALYSIS_CAPACITY }
3636

‎tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/AssemblyConverter.pm

+2-2
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@ use parent qw(EnsEMBL::Web::ToolsPipeConfig);
2929
sub logic_name { 'AssemblyConverter' }
3030
sub runnable { 'EnsEMBL::Web::RunnableDB::AssemblyConverter' }
3131
sub queue_name { $SiteDefs::ENSEMBL_AC_QUEUE }
32-
sub is_lsf { !$SiteDefs::ENSEMBL_AC_RUN_LOCAL }
33-
sub lsf_timeout { $SiteDefs::ENSEMBL_AC_LSF_TIMEOUT }
32+
sub is_farm { !$SiteDefs::ENSEMBL_AC_RUN_LOCAL }
33+
sub farm_timeout { $SiteDefs::ENSEMBL_AC_FARM_TIMEOUT }
3434
sub memory_usage { $SiteDefs::ENSEMBL_AC_MEMORY_USAGE }
3535
sub analysis_capacity { $SiteDefs::ENSEMBL_AC_ANALYSIS_CAPACITY }
3636

0 commit comments

Comments
 (0)