added new files and command revert

revert_shelves.py → bgcval2/revert_shelves.py

@@ -5,6 +5,8 @@
 from shelve import open as shOpen
 import glob
 import os
+import argparse
+
 
 from bgcval2.bgcvaltools import bv2tools as bvt
 from bgcval2._runtime_config import get_run_configuration
@@ -38,6 +40,8 @@ def load_all_datatypes(shelvedir, jobID):
     files = glob.glob(wildcards)
     datatypes = []
     for fn in files:
+        if fn.find('insitu') > -1:
+            continue
         basename = os.path.basename(fn)
         basename = basename.replace(jobID+'_', '')
         basename = basename.replace('.shelve.dat', '')
@@ -64,16 +68,12 @@ def remove_data(jobID,
     if dataTypes == ['all', ]:
         dataTypes = load_all_datatypes(shelvedir, jobID)
 
-    print(dataTypes)
-    return
     for dataType in dataTypes:
         shelvefn = shelvedir + '_'.join([
             jobID,
             dataType,
         ]) + '.shelve'
 
-        print(shelvefn)
-
         if glob.glob(shelvefn+'*'):
             sh = shOpen(shelvefn)
             print('Shelve loads okay:', shelvefn +'*')
@@ -82,7 +82,13 @@ def remove_data(jobID,
             modeldataD      = sh['modeldata']
             sh.close()
 
-        if month: 
+        # generate a time key from year/month.
+        if month:
+            if isinstance(month, str) and len(month) ==1:
+                month = int(month)
+            if isinstance(month, int):
+                month = bvt.mnStr(month)
+
             time_key = year+month
         else:
             time_key = year
@@ -96,7 +102,7 @@ def remove_data(jobID,
 
         if not changes:
             print('Nothing to remove')
-            return
+            continue
 
         # Remove files from list
         for remove_file in remove_files:
@@ -140,15 +146,79 @@ def remove_data(jobID,
            print('Not saving (dry_run):', shelvefn)
 
 
+def get_args():
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument('-c',
+                        '--config-file',
+                        default=os.path.join(os.getcwd(),
+                                             'config-user.yml'),
+                        help='User configuration file',
+                        required=False)
+
+    parser.add_argument('-j',
+                        '--jobids',
+                        default=None,
+                        nargs='+',
+                        type=str,
+                        help='One or more jobIDs (required)',
+                        required=True)
+
+    parser.add_argument('-y',
+                        '--years',
+                        default=None,
+                        nargs='+',
+                        type=str,
+                        help='One or more years (required)',
+                        required=True)
+    parser.add_argument('-m',
+                        '--months',
+                        default=[None, ],
+                        nargs='+',
+                        type=str,
+                        help='One or more months - default is all months',
+                        required=False)
+
+    parser.add_argument('-k',
+                        '--keys',
+                        default=['all',],
+                        nargs='+',
+                        type=str,
+                        help='One or more datasets - default is everything. ',
+                        required=False)
+
+    parser.add_argument('-d',
+                        '--dry-run',
+                        action='store_true',
+                        help='Dry run: Do not edit any files.',
+                        required=False
+                        )
+
+
+    args = parser.parse_args()
+
+    return args
+
+
+
 def main():
-    jobID = 'GFDL_hist'
-    years = ['2007', ]
-    months = ['07', ]
-    dataTypes = ['all', ]
-    dry_run = True 
-    for year in years:
-        for month in months:
-            remove_data(jobID, year, month=month,dataTypes=dataTypes, dry_run=dry_run) #, month)
+    args = get_args()
+    jobIDs = args.jobids
+    years = args.years
+    months = args.months
+    dataTypes = args.keys
+    dry_run = args.dry_run
+
+    for jobID in jobIDs:
+        for year in years:
+            for month in months:
+                print('start: remove_data', jobID, year,month, dataTypes, dry_run)
+                remove_data(jobID, year, month=month,dataTypes=dataTypes, dry_run=dry_run) 
+
+
 
+if __name__ == "__main__":
+    main()
 
-main()

input_yml/mission_atlantic_ALL.yml

@@ -0,0 +1,59 @@
+---
+# GC5 N96 ORCA1 spinup analysis
+name: MissionAtlantic_All
+
+# Run the single job analysis
+do_analysis_timeseries: True 
+
+# Download from mass:
+do_mass_download: False
+
+# Strict file check (fails if True and files are missing)
+strictFileCheck: False
+
+# master analysis suite
+master_suites: missionatlantic
+
+# Auto download jobs (can also be set for individual jobs).
+auto_download: False
+
+# Job ID's suites as named by Rose/Cylc
+jobs:
+  CNRM_hist:
+        description: 'CNRM historical run.'
+        label: 'CNRM-hist'
+        colour: green  
+        thickness: 1.8
+        linestyle: '-'
+        shifttime: 0.
+        suite: missionatlantic missionatlantic_kmf
+
+  GFDL_hist:
+        description: 'GFDL historical run.'
+        label: 'GFDL-hist'
+        colour: black
+        thickness: 1.2
+        linestyle: '-'
+        shifttime: 0.
+        suite: missionatlantic missionatlantic_kmf
+
+
+  CNRM_ssp370:
+        description: 'CNRM SSP3-7.0'
+        label: 'CNRM-ssp370'
+        colour: purple
+        thickness: 1.2
+        linestyle: '-'
+        shifttime: 0.
+        suite: missionatlantic missionatlantic_kmf
+
+  CNRM_ssp126:
+        description: 'CNRM SSP1-2.6'
+        label: 'CNRM-ssp126'
+        colour: dodgerblue
+        thickness: 1.2
+        linestyle: '-'
+        shifttime: 0.
+        suite: missionatlantic missionatlantic_kmf
+
+

key_files/ma_nitrate_kmf.yml

@@ -0,0 +1,18 @@
+---
+name            : MA_Nitrate
+units           : mmol N/m^3
+dimensions      : 1
+model           : NEMO025-ERSEM
+modelgrid       : eORCA025
+modelFiles      : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc
+gridFile        : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc
+model_vars      : N3_n
+model_convert   : NoChange
+  #dataFile        : $BASEDIR_OBS/WOA/annual/woa13_all_n00_01.nc
+dataFile        : $BASEDIR_OBS/WOA/annual/woa18_all_n00_01.nc
+data_vars       : n_an
+data_convert    : NoChange
+data_source     : WOA18
+layers          : Surface 
+regions         : Global
+metrics         : mean 

key_files/ma_sst_kmf.yml

@@ -0,0 +1,20 @@
+---
+name            : MA_SST
+units           : degrees C
+dimensions      : 1
+model           : NEMO025-ERSEM
+modelgrid       : eORCA025
+modelFiles      : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_T*.nc
+gridFile        : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc
+model_vars      : thetao thetao_con votemper
+model_convert   : choose_best_var
+  #dataFile        : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa13_decav_t00_01v2_AMM.nc
+dataFile        : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa18_decav_t00_01.nc
+data_vars       : t_an
+data_convert    : NoChange
+data_source     : WOA18
+layers          : Surface #50m 100m
+regions         : Global 
+metrics         : mean 
+smoothings      : DataOnly both5 
+

setup.py

@@ -203,6 +203,7 @@ def read_authors(filename):
             'analysis_timeseries = bgcval2.analysis_timeseries:main',
             'download_from_mass = bgcval2.download_from_mass:main',   
             'bgcval2_make_report = bgcval2.bgcval2_make_report:main',
+            'revert_shelves = bgcval2.revert_shelves:main',
         ],
     },
     cmdclass={