diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 466a8d4b..894a041b 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -41,6 +41,7 @@ jobs: analysis_level3_sameYear --help analysis_p2p --help analysis_timeseries --help + batch_timeseries --help bgcval2_make_report --help download_from_mass --help - shell: bash -l {0} diff --git a/.gitignore b/.gitignore index de6e55f9..1478f966 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,7 @@ local_test/BGC_data/valeriu mass_scripts CompareReports2 .idea/workspace.xml +*.iml +.idea/inspectionProfiles/profiles_settings.xml +.idea/misc.xml +.idea/vcs.xml diff --git a/README.md b/README.md index 3c9e8aee..d67affd4 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![made-with-python](https://img.shields.io/badge/Made%20with-Python-1f425f.svg)](https://www.python.org/) [![Github Actions Test](https://github.com/valeriupredoi/bgcval2/actions/workflows/run-tests.yml/badge.svg)](https://github.com/valeriupredoi/bgcval2/actions/workflows/run-tests.yml) -![bgcval2logo](https://github.com/valeriupredoi/bgcval2/blob/main/doc/figures/BGCVal2-logo-2.png) +![bgcval2logo](https://github.com/valeriupredoi/bgcval2/blob/main/doc/figures/bgcval2_logo_v_small.png) bgcval2 ======= @@ -115,6 +115,7 @@ Executable name | What it does | Command `bgcval` | runs time series and point to point. | bgcval jobID `bgcval2_make_report` | makes the single model HTML report. | bgcval2_make_report jobID `analysis_compare` | runs comparison of multiple single jobs | analysis_compare +`batch_timeseries` | Submits single job time series analysis to slurm | batch_timeseries ### Checking out development branches @@ -319,6 +320,50 @@ then the report will appear on the [JASMIN public facing page](https://gws-acces which is public facing but password protected. +Batch times series Analysis +=========================== + +The `batch_timeseries` tool can take an `analysis_compare` input yaml file, +and instead of running the time series analysis for each job on +the interactive shell terminal in series, it uses slurm to submit +each job as an independent job. + +On jasmin, users can run up to five jobs simulataneously, +so this can singnificantly boost the speed of the analysis. + +The command to run it is: +``` +batch_timeseries - y comparison_recipe.yml +``` + +This will submit a time-series analysis for each job, using a command which looks like this: +``` +sbatch -J jobID --error=logs/jobID .err --output=logs/jobID .out lotus_timeseries.sh jobID kmf physics bgc +``` +The output and error messages will be in the `logs` directory with the jobID as the file prefix. +The job name on slurm will also be the jobID, so it's easy to tell which jobs are running. +The analysis suites will be appended as a list to the end of the command. +In order to reduce the chance of analysing the same jobID twice, `batch_timeseries` +checks whether a job exists, either currently running or in the queue before submitting. +If a jobID exists, it is not re-submitted. However, this means that +if two versions of the same jobID are submitted one after the other +with different suite lists (`kmf`, `physics`, `bgc`), then only the first +set of suites will be run. + +There is also an optional flag `-d` or `--dry_run` to test `batch_timeseries`, +which outputs the submission command to screen but does not submit the jobs. + +Note that this task does not run the `analysis_compare` suite so it will +not generate the html report. However, the html report can be generated more quickly +with the `-s` argument to skip the `analysis_timeseries` section +described above. + +In addition, note that this will not run the `download_from_mass` +script, so jobs added here will not be included in the automated download. +However, these jobs are added for automated download when `analysis_compare` +is used. + + Downloading data using MASS =========================== diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index e6aa75d0..76e0811b 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -733,7 +733,8 @@ def applyLandMask1e3(nc, keys): gridFile=av[name]['gridFile'], clean=False, ) - + print("analysis_timeseries:\tINFO:\tEnd of the timeseries analysis", jobID, suites) + def get_args(): """Parse command line arguments. """ diff --git a/bgcval2/batch_timeseries.py b/bgcval2/batch_timeseries.py new file mode 100644 index 00000000..7c450e36 --- /dev/null +++ b/bgcval2/batch_timeseries.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python +# +# Copyright 2015, Plymouth Marine Laboratory +# +# This file is part of the bgc-val library. +# +# bgc-val is free software: you can redistribute it and/or modify it +# under the terms of the Revised Berkeley Software Distribution (BSD) 3-clause license. + +# bgc-val is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the revised BSD license for more details. +# You should have received a copy of the revised BSD license along with bgc-val. +# If not, see . +# +# Address: +# Plymouth Marine Laboratory +# Prospect Place, The Hoe +# Plymouth, PL1 3DH, UK +# +# Email: +# ledm@pml.ac.uk +# +""" +.. module:: batch_timeseries + :platform: Unix + :synopsis: A script to submit slurm scripts time series. + +.. moduleauthor:: Lee de Mora + +""" +import argparse +import subprocess +import os +import sys + +from getpass import getuser + +from bgcval2.analysis_compare import load_comparison_yml + + +def get_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + + parser.add_argument('-y', + '--compare_yml', + nargs='+', + type=str, + help='One or more Comparison Analysis configuration file, for examples see bgcval2 input_yml directory.', + required=True, + ) + + parser.add_argument('-c', + '--config-file', + default=os.path.join(os.path.dirname(os.path.realpath(__file__)), + 'default-bgcval2-config.yml'), + help='User configuration file (for paths).', + required=False) + + parser.add_argument('--dry_run', + '-d', + default=False, + help='When True: Do not submit the jobs to lotus.', + action=argparse.BooleanOptionalAction, + required=False) + + args = parser.parse_args() + return args + + +def submits_lotus(compare_yml, config_user, dry_run=False): + """ + Loads the yaml file and submits individual time series to sbatch. + """ + # Load details from yml file + details = load_comparison_yml(compare_yml) + + # list of job IDS + jobs = details['jobs'] + + # username + user = getuser() + + # Load current on-going list of this users slurm jobs: + out = str(subprocess.check_output(["squeue", "--user="+user])) + + # loop over jobs: + for job in jobs: + # Check whether there's already a job running for this jobID + if out.find(job) > -1: + print("That job exists already: skipping", job) + continue + + # Get list of suites for each job + suites = details['suites'][job] + + # Make it a list: + if isinstance(suites, str): + suites = suites.split(' ') + + # prepare the command + command_txt = ['sbatch', + '-J', job, + ''.join(['--error=logs/', job,'.err']), + ''.join(['--output=logs/', job,'.out']), + 'lotus_timeseries.sh', job] + for suite in suites: + command_txt.append(suite) + + # Send it! + if dry_run: + print('Not submitting (dry-run):', ' '.join(command_txt)) + else: + # Submit job: + print('Submitting:', ' '.join(command_txt)) + #command1 = subprocess.Popen(command_txt) + command1 = subprocess.Popen( + command_txt, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + + +def main(): + + """Run the main routine.""" + args = get_args() + + # This has a sensible default value. + config_user=args.config_file + + # This shouldn't fail as it's a required argument. + compare_ymls = args.compare_yml + + for compare_yml in compare_ymls: + print(f"analysis_timeseries: Comparison config file {compare_yml}") + + if not os.path.isfile(compare_yml): + print(f"analysis_timeseries: Could not find comparison config file {compare_yml}") + sys.exit(1) + dry_run = args.dry_run + submits_lotus(compare_yml, config_user, dry_run) + + +if __name__ == "__main__": + from ._version import __version__ + print(f'BGCVal2: {__version__}') + main() + diff --git a/bgcval2/timeseries/timeseriesAnalysis.py b/bgcval2/timeseries/timeseriesAnalysis.py index 99bd7899..83096752 100644 --- a/bgcval2/timeseries/timeseriesAnalysis.py +++ b/bgcval2/timeseries/timeseriesAnalysis.py @@ -137,7 +137,7 @@ def loadModel(self): if self.debug: print("timeseriesAnalysis:\tloadModel.") #### # load and calculate the model info - if glob.glob(self.shelvefn): + if glob.glob(self.shelvefn+'*'): # shelve files have .bak .dat .dir files now sh = shOpen(self.shelvefn) print('seems fine:', self.shelvefn) sh = shOpen(self.shelvefn) diff --git a/doc/figures/bgcval2_logo_v_small.png b/doc/figures/bgcval2_logo_v_small.png new file mode 100644 index 00000000..1aa513ad Binary files /dev/null and b/doc/figures/bgcval2_logo_v_small.png differ diff --git a/input_yml/TerraFIRMA_overshoot_runs.yml b/input_yml/TerraFIRMA_overshoot_runs.yml new file mode 100644 index 00000000..ca5154d5 --- /dev/null +++ b/input_yml/TerraFIRMA_overshoot_runs.yml @@ -0,0 +1,194 @@ +--- +# GC5 N96 ORCA1 spinup analysis +name: TerraFIRMA_overshoot_runs + +# Run the single job analysis +do_analysis_timeseries: True + +# Download from mass: +do_mass_download: False + +# master analysis suite +master_suites: physics bgc kmf #alkalinity physics kmf1 + +# Run without strick check (if True, breaks if job has no years.) +strict_file_check: False + +clean: True + +jobs: + u-cs495: + description: 'PI-Control' + label: 'PIcontrol' + colour: 'blue' + thickness: 1.2 + linestyle: '-' + shifttime: -100. + timerange: [1850, 2300] + suite: kmf physics bgc #alkalinity physics + + + u-cx209: + description: 'E-mode free ice RAMP UP 8GtC/yr #1' + label: 'Ramp-up #1' + colour: 'black' + thickness: 1.2 + linestyle: '-' + shifttime: 0. + #timerange: [1800, 2050] + suite: kmf physics bgc #alkalinity physics + + u-cw988: + description: 'E-mode free ice RAMP UP 8GtC/yr #2' + label: 'Ramp-up #2' + colour: 'black' + thickness: 1.2 + linestyle: '-' + shifttime: 0. + #timerange: [1800, 2050] + suite: kmf physics bgc #alkalinity physics + + u-cw989: + description: 'E-mode free ice RAMP UP 8GtC/yr #3' + label: 'Ramp-up #3' + colour: 'black' + thickness: 1.2 + linestyle: '-' + shifttime: 0. + #timerange: [1800, 2050] + suite: kmf physics bgc #alkalinity physics + + u-cw990: + description: 'E-mode free ice RAMP UP 8GtC/yr #4' + label: 'Ramp-up #4' + colour: 'black' + thickness: 1.2 + linestyle: '-' + shifttime: 0. + #timerange: [1800, 2050] + suite: kmf physics bgc #alkalinity physics + + # 1.5k Stabilisation + u-cy837: + description: '1.5K Stabilisation from u-cx209 (#1)' + label: '1.5K Stable #1' + colour: 'lawngreen' + thickness: 1.2 + linestyle: '-' + shifttime: 0. + #timerange: [1800, 2050] + suite: kmf physics bgc #alkalinity physics + u-cz834: + description: '1.5K Stabilisation from u-cw988 (#2)' + label: '1.5K Stable #2' + colour: 'lawngreen' + thickness: 1.2 + linestyle: '-' + shifttime: 0. + #timerange: [1800, 2050] + suite: kmf physics bgc #alkalinity physics + u-da087: + description: '1.5K Stabilisation from u-cw989 (#3)' + label: '1.5K Stable #3' + colour: 'lawngreen' + thickness: 1.2 + linestyle: '-' + shifttime: 0. + #timerange: [1800, 2050] + suite: kmf physics bgc #alkalinity physics + + # 2k Stabilisation + u-cy838: + description: '2.0K Stabilisation from u-cx209 (#1)' + label: '2K Stable #1' + colour: 'goldenrod' + thickness: 1.2 + linestyle: '-' + shifttime: 0. + #timerange: [1800, 2050] + suite: kmf physics bgc #alkalinity physics + + u-cz855: + description: '2.0K Stabilisation from cw988 (#2)' + label: '2K Stable #2' + colour: 'goldenrod' + thickness: 1.2 + linestyle: '-' + shifttime: 0. + #timerange: [1800, 2050] + suite: kmf physics bgc #alkalinity physics + + + # 2.5k Stabilisation + u-cz374: + description: '2.5K Stabilisation from u-cx209 (#1)' + label: '2.5K Stable #1' + colour: 'orange' + thickness: 1.2 + linestyle: '-' + shifttime: 0. + #timerange: [1800, 2050] + suite: kmf physics bgc #alkalinity physics + u-cz859: + description: '2.5K Stabilisation from cw988 (#2)' + label: '2.5K Stable #2' + colour: 'orange' + thickness: 1.2 + linestyle: '-' + shifttime: 0. + #timerange: [1800, 2050] + suite: kmf physics bgc #alkalinity physics + + + # other Stabilisations + u-cz375: + description: '3.0K Stabilisation from u-cx209 (#1)' + label: '3K Stable #1' + colour: 'red' # + thickness: 1.2 + linestyle: '-' + shifttime: 0. + #timerange: [1800, 2050] + suite: kmf physics bgc #alkalinity physics + u-cz376: + description: '4.0K Stabilisation from u-cx209 (#1)' + label: '4K Stable #1' + colour: 'sienna' + thickness: 1.2 + linestyle: '-' + shifttime: 0. + #timerange: [1800, 2050] + suite: kmf physics bgc #alkalinity physics + u-cz377: + description: '5.0K Stabilisation from u-cx209 (#1)' + label: '5K Stable #1' + colour: 'maroon' + thickness: 1.2 + linestyle: '-' + shifttime: 0. + #timerange: [1800, 2050] + suite: kmf physics bgc #alkalinity physics + + # Ramp downs + u-cz944: + description: 'Ramp down from cy838 (- 2.0K Stabilisation from u-cx209) (#1)' + label: '2K Rampdown #1' + colour: 'goldenrod' + thickness: 1.2 + linestyle: ':' + shifttime: 0. + #timerange: [1800, 2050] + suite: kmf physics bgc #alkalinity physics + + u-da697: + description: '1.5K Rampdown from u-cx209 (#1)' + label: '1.5K Rampdown #1' + colour: 'lawngreen' + thickness: 1.2 + linestyle: ':' + shifttime: 0. + #timerange: [1800, 2050] + suite: kmf physics bgc #alkalinity physics + + + diff --git a/input_yml/TerraFIRMA_overshoot_runs_minimal.yml b/input_yml/TerraFIRMA_overshoot_runs_minimal.yml new file mode 100644 index 00000000..cfa54aac --- /dev/null +++ b/input_yml/TerraFIRMA_overshoot_runs_minimal.yml @@ -0,0 +1,120 @@ +--- +# GC5 N96 ORCA1 spinup analysis +name: TerraFIRMA_overshoot_runs_minimal + +# Run the single job analysis +do_analysis_timeseries: True + +# Download from mass: +do_mass_download: False + +# master analysis suite +master_suites: kmf # physics bgc #alkalinity physics kmf1 + +# Run without strick check (if True, breaks if job has no years.) +strict_file_check: False + +clean: True + +jobs: + u-cs495: + description: 'PI-Control' + label: 'Pi-Control' + colour: 'blue' + thickness: 1.2 + linestyle: '-' + shifttime: -100. + timerange: [1850, 2200] + suite: kmf #physics bgc #alkalinity physics + + + u-cx209: + description: 'E-mode free ice RAMP UP 8GtC/yr #1' + label: 'Ramp-up' + colour: 'black' + thickness: 1.2 + linestyle: '-' + shifttime: 0. + #timerange: [1800, 2050] + suite: kmf #physics bgc #alkalinity physics + + # 1.5k Stabilisation + u-cy837: + description: '1.5K Stabilisation from u-cx209 (#1)' + label: '1.5K Stable' + colour: 'lawngreen' + thickness: 1.2 + linestyle: '-' + shifttime: 0. + #timerange: [1800, 2050] + suite: kmf # physics bgc #alkalinity physics + + + # 2k Stabilisation + u-cy838: + description: '2.0K Stabilisation from u-cx209 (#1)' + label: '2K Stable' + colour: 'goldenrod' + thickness: 1.2 + linestyle: '-' + shifttime: 0. + #timerange: [1800, 2050] + suite: kmf # physics bgc #alkalinity physics + + + # other Stabilisations + u-cz375: + description: '3.0K Stabilisation from u-cx209 (#1)' + label: '3K Stable' + colour: 'red' # + thickness: 1.2 + linestyle: '-' + shifttime: 0. + #timerange: [1800, 2050] + suite: kmf # physics bgc #alkalinity physics + u-cz376: + description: '4.0K Stabilisation from u-cx209 (#1)' + label: '4K Stable' + colour: 'sienna' + thickness: 1.2 + linestyle: '-' + shifttime: 0. + #timerange: [1800, 2050] + suite: kmf # physics bgc #alkalinity physics + u-cz377: + description: '5.0K Stabilisation from u-cx209 (#1)' + label: '5K Stable' + colour: 'maroon' + thickness: 1.2 + linestyle: '-' + shifttime: 0. + #timerange: [1800, 2050] + suite: kmf # physics bgc #alkalinity physics + + # Ramp downs + u-da697: + description: '1.5K Rampdown from u-cx209 (#1)' + label: '1.5K Rampdown' + colour: 'lawngreen' + thickness: 1.2 + linestyle: ':' + shifttime: 0. + #timerange: [1800, 2050] + suite: kmf # physics bgc #alkalinity physics + + u-cz944: + description: 'Ramp down from cy838 (- 2.0K Stabilisation from u-cx209) (#1)' + label: '2K Rampdown' + colour: 'goldenrod' + thickness: 1.2 + linestyle: 'dashed' + shifttime: 0. + #timerange: [1800, 2050] + suite: kmf physics bgc #alkalinity physics + + + + + + + diff --git a/lotus_timeseries.sh b/lotus_timeseries.sh new file mode 100644 index 00000000..bfcd161c --- /dev/null +++ b/lotus_timeseries.sh @@ -0,0 +1,53 @@ +#!/bin/bash +#SBATCH --partition=short-serial +#SBATCH --time 6:00:00 +#SBATCH -o logs/log_bgcval2_ts_%J.out +#SBATCH -e logs/log_bgcval3_ts_%J.err + +# Note that the +# slurm job name should be set in the command line with -J option to the jobID +# and the output and error filers are also set at the command line. +# + +###################### +# This script runs a single time series job. +# +# Runs: +# sbatch -J jobID lotus_timeseries.sh jobID suite1 suite2 etc... +# Note that that batch_timeseries command also +# adds specific out and job scripts, +# and outputs them to ./logs directory.. +######################### + + +######################### +# Change this to your bgcval2 conda environment name +CONDA_ENV=bgcval2 + + +########################## +# Source global definitions +if [ -f ~/.bashrc ]; then + echo 'source ~/.bashrc' + source ~/.bashrc +fi + +########################## +# load your conda env: +#conda activate bgcval2 +echo conda activate $CONDA_ENV +conda activate ${CONDA_ENV} + +########################## +# Load command line arguments: +args=$@ +jobID=$1 +suites="${@:2}" +echo $suites + +########################## +# Run single jog timeseries analysis. +echo "analysis_timeseries -j $jobID -k $suites" +analysis_timeseries -j ${jobID} -k ${suites} + + diff --git a/setup.py b/setup.py index 015512b2..3baa25f6 100755 --- a/setup.py +++ b/setup.py @@ -201,6 +201,7 @@ def read_authors(filename): 'analysis_level3_sameYear = bgcval2.analysis_level3_sameYear:main', 'analysis_p2p = bgcval2.analysis_p2p:run', 'analysis_timeseries = bgcval2.analysis_timeseries:main', + 'batch_timeseries = bgcval2.batch_timeseries:main', 'download_from_mass = bgcval2.download_from_mass:main', 'bgcval2_make_report = bgcval2.bgcval2_make_report:main', ], diff --git a/tests/integration/test_command_line.py b/tests/integration/test_command_line.py index d4114a10..a2315e69 100644 --- a/tests/integration/test_command_line.py +++ b/tests/integration/test_command_line.py @@ -15,7 +15,8 @@ analysis_timeseries, download_from_mass, bgcval2_make_report, - analysis_compare + analysis_compare, + batch_timeseries ) from bgcval2.analysis_timeseries import main as analysis_timeseries_main from bgcval2.download_from_mass import main as download_from_mass_main @@ -23,6 +24,7 @@ # from bgcval2.bgcval import run as bgcval_main from bgcval2.bgcval2_make_report import main as bgcval2_make_report_main from bgcval2.analysis_compare import main as analysis_compare_main +from bgcval2.batch_timeseries import main as bgcval2_batch_timeseries def wrapper(f): @@ -142,3 +144,27 @@ def test_bgcval2_make_report_command(): as (stdout, stderr): bgcval2_make_report_main() assert err in str(stderr.getvalue()) + + +@patch('bgcval2.batch_timeseries.main', new=wrapper(bgcval2_batch_timeseries)) +def test_bgcval2_make_report_command(): + """Test run command.""" + with arguments('batch_timeseries', '--help'): + with pytest.raises(SystemExit) as pytest_wrapped_e: + bgcval2_batch_timeseries() + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 0 + + err = "the following arguments are required: -y/--compare_yml" + with arguments('bgcval2_batch_timeseries'): + with pytest.raises(SystemExit) as cm, capture_sys_output() \ + as (stdout, stderr): + bgcval2_batch_timeseries() + assert err in str(stderr.getvalue()) + + err = "argument -y/--compare_yml: expected at least one argument" + with arguments('bgcval2_batch_timeseries', '--compare_yml'): + with pytest.raises(SystemExit) as cm, capture_sys_output() \ + as (stdout, stderr): + bgcval2_batch_timeseries() + assert err in str(stderr.getvalue())