Batch timeseries analysis in slurm

.gitignore

@@ -21,3 +21,7 @@ local_test/BGC_data/valeriu
 mass_scripts
 CompareReports2
 .idea/workspace.xml
+*.iml
+.idea/inspectionProfiles/profiles_settings.xml
+.idea/misc.xml
+.idea/vcs.xml

bgcval2/analysis_timeseries.py

@@ -733,7 +733,8 @@ def applyLandMask1e3(nc, keys):
                 gridFile=av[name]['gridFile'],
                 clean=False,
             )
-
+    print("analysis_timeseries:\tINFO:\tEnd of the timeseries analysis", jobID, suites)
+
 
 def get_args():
     """Parse command line arguments. """

bgcval2/batch_timeseries.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python
+#
+# Copyright 2015, Plymouth Marine Laboratory
+#
+# This file is part of the bgc-val library.
+#
+# bgc-val is free software: you can redistribute it and/or modify it
+# under the terms of the Revised Berkeley Software Distribution (BSD) 3-clause license.
+
+# bgc-val is distributed in the hope that it will be useful, but
+# without any warranty; without even the implied warranty of merchantability
+# or fitness for a particular purpose. See the revised BSD license for more details.
+# You should have received a copy of the revised BSD license along with bgc-val.
+# If not, see <http://opensource.org/licenses/BSD-3-Clause>.
+#
+# Address:
+# Plymouth Marine Laboratory
+# Prospect Place, The Hoe
+# Plymouth, PL1 3DH, UK
+#
+# Email:
+# [email protected]
+#
+"""
+.. module:: batch_timeseries
+   :platform: Unix
+   :synopsis: A script to submit slurm scripts time series.
+
+.. moduleauthor:: Lee de Mora <[email protected]>
+
+"""
+import argparse
+import subprocess
+import os
+import sys
+
+from getpass import getuser
+
+from bgcval2.analysis_compare import load_comparison_yml
+
+
+def get_args():
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+
+    parser.add_argument('-y',
+                        '--compare_yml',
+                        nargs='+',
+                        type=str,
+                        help='One or more Comparison Analysis configuration file, for examples see bgcval2 input_yml directory.',
+                        required=True,
+                        )
+
+    parser.add_argument('-c',
+                        '--config-file',
+                        default=os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                                             'default-bgcval2-config.yml'),
+                        help='User configuration file (for paths).',
+                        required=False)
+
+    parser.add_argument('--dry_run',
+                        '-d',
+                        default=False,
+                        help='When True: Do not submit the jobs to lotus.',
+                        action=argparse.BooleanOptionalAction,
+                        required=False)
+
+    args = parser.parse_args()
+    return args
+
+
+def submits_lotus(compare_yml, config_user, dry_run=False):
+    """
+     Loads the yaml file and submits individual time series to sbatch.
+     """
+    # Load details from yml file
+    details = load_comparison_yml(compare_yml)
+
+    # list of job IDS
+    jobs = details['jobs']
+
+    # username
+    user = getuser()
+
+    # Load current on-going list of this users slurm jobs:
+    out = str(subprocess.check_output(["squeue", "--user="+user]))
+
+    # loop over jobs:
+    for job in jobs:
+        # Check whether there's already a job running for this jobID
+        if out.find(job) > -1:
+            print("That job exists already: skipping", job)
+            continue
+
+        # Get list of suites for each job
+        suites = details['suites'][job]
+
+        # Make it a list:
+        if isinstance(suites, str):
+            suites = suites.split(' ')
+
+        # prepare the command
+        command_txt = ['sbatch', '-J', job, 'lotus_timeseries.sh', job]
+        for suite in suites:
+            command_txt.append(suite)
+
+        # Send it!
+        if dry_run:
+            print('Not submitting (dry-run):', ' '.join(command_txt))
+        else:
+            # Submit job:
+            print('Submitting:', ' '.join(command_txt))
+            command1 = subprocess.Popen(command_txt)
+
+
+def main():
+
+    """Run the main routine."""
+    args = get_args()
+
+    # This has a sensible default value.
+    config_user=args.config_file
+
+    # This shouldn't fail as it's a required argument.
+    compare_ymls = args.compare_yml
+
+    for compare_yml in compare_ymls:
+        print(f"analysis_timeseries: Comparison config file {compare_yml}")
+
+        if not os.path.isfile(compare_yml):
+            print(f"analysis_timeseries: Could not find comparison config file {compare_yml}")
+            sys.exit(1)
+        dry_run = args.dry_run
+        submits_lotus(compare_yml, config_user, dry_run)
+
+
+if __name__ == "__main__":
+    from ._version import __version__
+    print(f'BGCVal2: {__version__}')
+    main()
+

bgcval2/timeseries/timeseriesAnalysis.py

@@ -137,7 +137,7 @@ def loadModel(self):
         if self.debug: print("timeseriesAnalysis:\tloadModel.")
         ####
         # load and calculate the model info
-        if glob.glob(self.shelvefn):
+        if glob.glob(self.shelvefn+'*'): # shelve files have .bak .dat .dir files now
             sh = shOpen(self.shelvefn)
             print('seems fine:', self.shelvefn)
             sh = shOpen(self.shelvefn)

input_yml/TerraFIRMA_overshoot_runs.yml

@@ -0,0 +1,194 @@
+---
+# GC5 N96 ORCA1 spinup analysis
+name: TerraFIRMA_overshoot_runs
+
+# Run the single job analysis
+do_analysis_timeseries: True  
+
+# Download from mass:
+do_mass_download: False
+
+# master analysis suite
+master_suites: physics bgc kmf #alkalinity physics kmf1
+
+# Run without strick check (if True, breaks if job has no years.)
+strict_file_check: False
+
+clean: True
+
+jobs:
+    u-cs495:
+       description: 'PI-Control'
+       label: 'PIcontrol'
+       colour: 'blue' 
+       thickness: 1.2
+       linestyle: '-'
+       shifttime: -100.
+       timerange: [1850, 2300]
+       suite: kmf physics bgc #alkalinity physics
+
+
+    u-cx209:
+       description: 'E-mode free ice RAMP UP 8GtC/yr #1'
+       label: 'Ramp-up #1'
+       colour: 'black'
+       thickness: 1.2
+       linestyle: '-'
+       shifttime: 0.
+       #timerange: [1800, 2050]
+       suite: kmf physics bgc #alkalinity physics
+
+    u-cw988:
+       description: 'E-mode free ice RAMP UP 8GtC/yr #2'
+       label: 'Ramp-up #2'
+       colour: 'black'
+       thickness: 1.2
+       linestyle: '-'
+       shifttime: 0.
+       #timerange: [1800, 2050]
+       suite: kmf physics bgc #alkalinity physics
+
+    u-cw989:
+       description: 'E-mode free ice RAMP UP 8GtC/yr #3'
+       label: 'Ramp-up #3'
+       colour: 'black'
+       thickness: 1.2
+       linestyle: '-'
+       shifttime: 0.
+       #timerange: [1800, 2050]
+       suite: kmf physics bgc #alkalinity physics
+
+    u-cw990:
+       description: 'E-mode free ice RAMP UP 8GtC/yr #4'
+       label: 'Ramp-up #4'
+       colour: 'black'
+       thickness: 1.2
+       linestyle: '-'
+       shifttime: 0.
+       #timerange: [1800, 2050]
+       suite: kmf physics bgc #alkalinity physics
+
+    # 1.5k Stabilisation
+    u-cy837:
+       description: '1.5K Stabilisation from u-cx209 (#1)'
+       label: '1.5K Stable #1'
+       colour: 'lawngreen'
+       thickness: 1.2
+       linestyle: '-'
+       shifttime: 0.
+       #timerange: [1800, 2050]
+       suite: kmf physics bgc #alkalinity physics   
+    u-cz834:
+       description: '1.5K Stabilisation from u-cw988 (#2)'
+       label: '1.5K Stable #2'
+       colour: 'lawngreen'
+       thickness: 1.2
+       linestyle: '-'
+       shifttime: 0.
+       #timerange: [1800, 2050]
+       suite: kmf physics bgc #alkalinity physics
+    u-da087:
+       description: '1.5K Stabilisation from u-cw989 (#3)'
+       label: '1.5K Stable #3'
+       colour: 'lawngreen' 
+       thickness: 1.2
+       linestyle: '-'
+       shifttime: 0.
+       #timerange: [1800, 2050]
+       suite: kmf physics bgc #alkalinity physics
+
+    # 2k Stabilisation
+    u-cy838:
+       description: '2.0K Stabilisation from u-cx209 (#1)'
+       label: '2K Stable #1'
+       colour: 'goldenrod'
+       thickness: 1.2
+       linestyle: '-'
+       shifttime: 0.
+       #timerange: [1800, 2050]
+       suite: kmf physics bgc #alkalinity physics   
+
+    u-cz855:
+       description: '2.0K Stabilisation from cw988 (#2)'
+       label: '2K Stable #2'
+       colour: 'goldenrod'
+       thickness: 1.2
+       linestyle: '-'
+       shifttime: 0.
+       #timerange: [1800, 2050]
+       suite: kmf physics bgc #alkalinity physics
+
+
+    # 2.5k Stabilisation
+    u-cz374:
+       description: '2.5K Stabilisation from u-cx209 (#1)'
+       label: '2.5K Stable #1'
+       colour: 'orange'
+       thickness: 1.2
+       linestyle: '-'
+       shifttime: 0.
+       #timerange: [1800, 2050]
+       suite: kmf physics bgc #alkalinity physics   
+    u-cz859:
+       description: '2.5K Stabilisation from cw988 (#2)'
+       label: '2.5K Stable #2'
+       colour: 'orange' 
+       thickness: 1.2
+       linestyle: '-'
+       shifttime: 0.
+       #timerange: [1800, 2050]
+       suite: kmf physics bgc #alkalinity physics
+
+
+    # other Stabilisations
+    u-cz375:
+       description: '3.0K Stabilisation from u-cx209 (#1)'
+       label: '3K Stable #1'
+       colour: 'red'     #
+       thickness: 1.2
+       linestyle: '-'
+       shifttime: 0.
+       #timerange: [1800, 2050]
+       suite: kmf physics bgc #alkalinity physics          
+    u-cz376:
+       description: '4.0K Stabilisation from u-cx209 (#1)'
+       label: '4K Stable #1'
+       colour: 'sienna'
+       thickness: 1.2
+       linestyle: '-'
+       shifttime: 0.
+       #timerange: [1800, 2050]
+       suite: kmf physics bgc #alkalinity physics  
+    u-cz377:
+       description: '5.0K Stabilisation from u-cx209 (#1)'
+       label: '5K Stable #1'
+       colour: 'maroon'
+       thickness: 1.2
+       linestyle: '-'
+       shifttime: 0.
+       #timerange: [1800, 2050]
+       suite: kmf physics bgc #alkalinity physics 
+
+    # Ramp downs
+    u-cz944:
+       description: 'Ramp down from cy838 (- 2.0K Stabilisation from u-cx209) (#1)'
+       label: '2K Rampdown #1'
+       colour: 'goldenrod'
+       thickness: 1.2
+       linestyle: ':' 
+       shifttime: 0.
+       #timerange: [1800, 2050]
+       suite: kmf physics bgc #alkalinity physics   
+
+    u-da697:
+       description: '1.5K Rampdown from u-cx209 (#1)'
+       label: '1.5K Rampdown #1'
+       colour: 'lawngreen'
+       thickness: 1.2
+       linestyle: ':'
+       shifttime: 0.
+       #timerange: [1800, 2050]
+       suite: kmf physics bgc #alkalinity physics
+
+
+