working fixes

bgcval2/parrellel_timeseries.py

@@ -22,16 +22,17 @@
 # [email protected]
 #
 """
-.. module:: analysis_timeseries
+.. module:: parrallel_timeseries
    :platform: Unix
-   :synopsis: A script to produce analysis for time series.
+   :synopsis: A script to submit slurm scripts time series.
 
 .. moduleauthor:: Lee de Mora <[email protected]>
 
 """
 import argparse
 import subprocess
 import os
+import sys
 
 from getpass import getuser
 
@@ -67,34 +68,45 @@ def get_args():
                         required=False)
 
     args = parser.parse_args()
-
     return args
 
+
 def submits_lotus(compare_yml, config_user, dry_run=False):
     """
      Loads the yaml file and submits individual time series to sbatch.
      """
-    # Below here is analysis
+    # Load details from yml file
     details = load_comparison_yml(compare_yml)
+
+    # list of job IDS
     jobs = details['jobs']
 
+    # username
     user = getuser()
+
+    # Load current on-going list of this users slurm jobs:
     out = str(subprocess.check_output(["squeue", "--user="+user]))
 
+    # loop over jobs:
     for job in jobs:
-        suites = details['suites'][job]
-        print(suites)
-        if isinstance(suites, str):
-            suites = suites.split(' ')
-
+        # Check whether there's already a job running for this jobID
         if out.find(job) > -1:
             print("That job exists already: skipping", job)
             continue
 
+        # Get list of suites for each job
+        suites = details['suites'][job]
+
+        # Make it a list:
+        if isinstance(suites, str):
+            suites = suites.split(' ')
+
+        # prepare the command
         command_txt = ['sbatch', '-J', job, 'lotus_timeseries.sh', job]
         for suite in suites:
             command_txt.append(suite)
-        print(' '.join(command_txt))
+
+        # Send it!
         if dry_run:
             print('Not submitting (dry-run):', ' '.join(command_txt))
         else:

bgcval2/timeseries/timeseriesAnalysis.py

@@ -137,7 +137,7 @@ def loadModel(self):
         if self.debug: print("timeseriesAnalysis:\tloadModel.")
         ####
         # load and calculate the model info
-        if glob.glob(self.shelvefn):
+        if glob.glob(self.shelvefn+'*'): # shelve files have .bak .dat .dir files now
             sh = shOpen(self.shelvefn)
             print('seems fine:', self.shelvefn)
             sh = shOpen(self.shelvefn)

lotus_timeseries.sh

@@ -47,12 +47,12 @@ conda activate ${CONDA_ENV}
 ##########################
 # Load command line arguments:
 args=$@
-jobID="${a%% *}"
+jobID=$1
 suites="${@:2}"
 echo $suites
 
-echo "analysis timeseries -j $jobID -k $suites"
-#analysis_timeseries -j $jobID -k $suites
+echo "analysis_timeseries -j $jobID -k $suites"
+analysis_timeseries -j ${jobID} -k ${suites}
 #########################
 # Add one or more input_yml files here
 #cho analysis_timeseries -y $BGCVAL2_SUITE