diff --git a/bgcval2/parrellel_timeseries.py b/bgcval2/parrellel_timeseries.py index b721443e..30e07bed 100644 --- a/bgcval2/parrellel_timeseries.py +++ b/bgcval2/parrellel_timeseries.py @@ -22,9 +22,9 @@ # ledm@pml.ac.uk # """ -.. module:: analysis_timeseries +.. module:: parrallel_timeseries :platform: Unix - :synopsis: A script to produce analysis for time series. + :synopsis: A script to submit slurm scripts time series. .. moduleauthor:: Lee de Mora @@ -32,6 +32,7 @@ import argparse import subprocess import os +import sys from getpass import getuser @@ -67,34 +68,45 @@ def get_args(): required=False) args = parser.parse_args() - return args + def submits_lotus(compare_yml, config_user, dry_run=False): """ Loads the yaml file and submits individual time series to sbatch. """ - # Below here is analysis + # Load details from yml file details = load_comparison_yml(compare_yml) + + # list of job IDS jobs = details['jobs'] + # username user = getuser() + + # Load current on-going list of this users slurm jobs: out = str(subprocess.check_output(["squeue", "--user="+user])) + # loop over jobs: for job in jobs: - suites = details['suites'][job] - print(suites) - if isinstance(suites, str): - suites = suites.split(' ') - + # Check whether there's already a job running for this jobID if out.find(job) > -1: print("That job exists already: skipping", job) continue + # Get list of suites for each job + suites = details['suites'][job] + + # Make it a list: + if isinstance(suites, str): + suites = suites.split(' ') + + # prepare the command command_txt = ['sbatch', '-J', job, 'lotus_timeseries.sh', job] for suite in suites: command_txt.append(suite) - print(' '.join(command_txt)) + + # Send it! if dry_run: print('Not submitting (dry-run):', ' '.join(command_txt)) else: diff --git a/bgcval2/timeseries/timeseriesAnalysis.py b/bgcval2/timeseries/timeseriesAnalysis.py index 99bd7899..83096752 100644 --- a/bgcval2/timeseries/timeseriesAnalysis.py +++ b/bgcval2/timeseries/timeseriesAnalysis.py @@ -137,7 +137,7 @@ def loadModel(self): if self.debug: print("timeseriesAnalysis:\tloadModel.") #### # load and calculate the model info - if glob.glob(self.shelvefn): + if glob.glob(self.shelvefn+'*'): # shelve files have .bak .dat .dir files now sh = shOpen(self.shelvefn) print('seems fine:', self.shelvefn) sh = shOpen(self.shelvefn) diff --git a/lotus_timeseries.sh b/lotus_timeseries.sh index b6b81c36..7654936d 100644 --- a/lotus_timeseries.sh +++ b/lotus_timeseries.sh @@ -47,12 +47,12 @@ conda activate ${CONDA_ENV} ########################## # Load command line arguments: args=$@ -jobID="${a%% *}" +jobID=$1 suites="${@:2}" echo $suites -echo "analysis timeseries -j $jobID -k $suites" -#analysis_timeseries -j $jobID -k $suites +echo "analysis_timeseries -j $jobID -k $suites" +analysis_timeseries -j ${jobID} -k ${suites} ######################### # Add one or more input_yml files here #cho analysis_timeseries -y $BGCVAL2_SUITE