Added MA fixes and new revert shelves tool.

bgcval2/bgcval2_make_report.py

@@ -1504,8 +1504,8 @@ def newImageLocation(fn):
         'Salinty_Global_Surface',
         'FreshwaterFlux_Global',
         'TotalHeatFlux',
-        'MA_SST',
-        'MA_SSS',
+        'MA_SST_Global_Surface',
+        'MA_SSS_Global_Surface',
         'MA_Drake',
         'MA_AMOC_26N',
         'MA_AEU',
@@ -1546,6 +1546,8 @@ def newImageLocation(fn):
             if found: continue
             sectionTitle = 'Physics Key Metrics'
             if fn.find(key) > -1:
+#               if key in ['MA_SST', 'MA_SSS', 'MA_Nitrate',] and fn.find('Global_Surface') < 0:
+#                   continue
                 try:
                     categories[sectionTitle].append(fn)
                 except:

input_yml/mission_atlantic_CNRM.yml

@@ -25,22 +25,22 @@ jobs:
         thickness: 1.8
         linestyle: '-'
         shifttime: 0.
-        suite: missionatlantic
+        suite: missionatlantic missionatlantic_kmf
 
   CNRM_ssp370:
         description: 'CNRM SSP3-7.0'
         colour: purple
         thickness: 1.8
         linestyle: '-'
         shifttime: 0.
-        suite: missionatlantic
+        suite: missionatlantic missionatlantic_kmf
 
   CNRM_ssp126:
         description: 'CNRM SSP1-2.6'
         colour: dodgerblue
         thickness: 1.8
         linestyle: '-'
         shifttime: 0.
-        suite: missionatlantic
+        suite: missionatlantic missionatlantic_kmf
 
 

key_files/ma_aeu.yml

@@ -14,3 +14,5 @@ model_convert:
   max_depth: 500. 
 layers: layerless
 regions: regionless
+smoothings      : DataOnly both5 
+

key_files/ma_ammonium.yml

@@ -8,6 +8,6 @@ modelFiles      : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc
 gridFile        : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc
 model_vars      : N4_n
 model_convert   : NoChange
-layers          : Surface 500m
+layers          : Surface #00m
 regions         : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean
 metrics         : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max

key_files/ma_amoc_26n.yml

@@ -15,3 +15,5 @@ model_convert:
   grid: eORCA025
 layers: layerless
 regions: regionless
+smoothings      : DataOnly both5 
+

key_files/ma_drake.yml

@@ -13,3 +13,5 @@ model_convert:
   grid: eORCA025
 layers: layerless
 regions: regionless
+smoothings      : DataOnly both5 
+

key_files/ma_iron.yml

@@ -8,6 +8,6 @@ modelFiles      : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc
 gridFile        : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc
 model_vars      : N7_f
 model_convert   : NoChange
-layers          : Surface 500m
+layers          : Surface #500m
 regions         : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean
 metrics         : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max

key_files/ma_nitrate.yml

@@ -13,6 +13,6 @@ dataFile        : $BASEDIR_OBS/WOA/annual/woa18_all_n00_01.nc
 data_vars       : n_an
 data_convert    : NoChange
 data_source     : WOA18
-layers          : Surface 500m
+layers          : Surface #500m
 regions         : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean
 metrics         : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max

key_files/ma_ph.yml

@@ -8,6 +8,6 @@ modelFiles      : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc
 gridFile        : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc
 model_vars      : O3_pH
 model_convert   : NoChange
-layers          : Surface 500m
+layers          : Surface #500m
 regions         : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean
-metrics         : mean, min  #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max
+metrics         : mean #, min  #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max

key_files/ma_phosphate.yml

@@ -13,6 +13,6 @@ dataFile        : $BASEDIR_OBS/WOA/annual/woa18_all_p00_01.nc
 data_vars       : p_an
 data_convert    : NoChange
 data_source     : WOA18
-layers          : Surface 500m
+layers          : Surface #500m
 regions         : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean
 metrics         : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max

key_files/ma_silicate.yml

@@ -13,6 +13,6 @@ dataFile        : $BASEDIR_OBS/WOA/annual/woa18_all_i00_01.nc
 data_vars       : i_an
 data_convert    : NoChange
 data_source     : WOA18
-layers          : Surface 100m 200m 500m 1000m 2000m 4000m 
+layers          : Surface #100m 200m 500m 1000m 2000m 4000m 
 regions         : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean
 metrics         : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max

key_files/ma_sss.yml

@@ -12,6 +12,7 @@ dataFile        : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa18_deca
 data_vars       : s_an
 data_convert    : NoChange
 data_source     : WOA18
-layers          : Surface 50m 100m 200m 500m 1000m 2000m 4000m
-regions         : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean ITCZ
+layers          : Surface #50m 100m 200m 500m 1000m 2000m 
+  #layers          : Surface 50m 100m 200m 500m 1000m 2000m 4000m
+regions         : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean #TCZ
 metrics         : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max

key_files/ma_ta.yml

@@ -8,6 +8,6 @@ modelFiles      : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc
 gridFile        : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc
 model_vars      : O3_TA
 model_convert   : NoChange
-layers          : Surface 500m
+layers          : Surface #500m
 regions         : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean
-metrics         : mean, max #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max
+metrics         : mean #, max #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max

key_lists/missionatlantic_kmf.yml

@@ -5,7 +5,7 @@
 keys:
   # Physics:
   #
-  #  MA_SST: True
+  MA_SST_KMF: True
   #MA_SSS: True
   MA_AEU: True  
   MA_Drake: True
@@ -23,6 +23,7 @@ keys:
   # MA_Totalairseafluxco2: True # Not in test17
   #  MA_TotalIntGPP: True
   MA_TotalIntNPP: True
+  MA_Nitrate_KMF: True
     #MA_Nitrate: True
     # MA_Ammonium: True
     #  MA_Phosphate: True

revert_shelves.py

@@ -0,0 +1,154 @@
+
+"""
+In this script, we remove years of data from a shelve
+"""
+from shelve import open as shOpen
+import glob
+import os
+
+from bgcval2.bgcvaltools import bv2tools as bvt
+from bgcval2._runtime_config import get_run_configuration
+from bgcval2.Paths.paths import paths_setter
+
+
+
+
+def get_paths(
+        config_user=None
+    ):
+    # get runtime configuration
+    if config_user:
+        paths_dict, config_user = get_run_configuration(config_user)
+    else:
+        paths_dict, config_user = get_run_configuration("defaults")
+
+    # filter paths dict into an object that's usable below
+    paths = paths_setter(paths_dict)
+    return paths.shelvedir
+
+
+def load_all_datatypes(shelvedir, jobID):
+    """
+    Returns a 
+    """ 
+    wildcards = shelvedir + '_'.join([
+            jobID,
+            '*',
+        ]) + '.shelve.dat'
+    files = glob.glob(wildcards)
+    datatypes = []
+    for fn in files:
+        basename = os.path.basename(fn)
+        basename = basename.replace(jobID+'_', '')
+        basename = basename.replace('.shelve.dat', '')
+        datatypes.append(basename)
+    return datatypes
+
+
+
+def remove_data(jobID, 
+        year, 
+        month=None,
+        config_user=None,
+        dataTypes= ['MA_AMOC_26N', ],
+        dry_run = True,
+        ):
+    """
+    Remove all data from the year and month from shelve files.
+    """
+
+    path_shelvedir = get_paths(config_user=config_user) 
+
+    shelvedir = bvt.folder([path_shelvedir, "timeseries", jobID])
+
+    if dataTypes == ['all', ]:
+        dataTypes = load_all_datatypes(shelvedir, jobID)
+
+    print(dataTypes)
+    return
+    for dataType in dataTypes:
+        shelvefn = shelvedir + '_'.join([
+            jobID,
+            dataType,
+        ]) + '.shelve'
+
+        print(shelvefn)
+
+        if glob.glob(shelvefn+'*'):
+            sh = shOpen(shelvefn)
+            print('Shelve loads okay:', shelvefn +'*')
+            sh = shOpen(shelvefn)
+            readFiles       = sh['readFiles']
+            modeldataD      = sh['modeldata']
+            sh.close()
+
+        if month: 
+            time_key = year+month
+        else:
+            time_key = year
+
+        changes = 0
+        remove_files = [] 
+        for readFile in readFiles:
+            if readFile.find(time_key) > -1:
+                remove_files.append(readFile)
+                changes +=1
+
+        if not changes:
+            print('Nothing to remove')
+            return
+
+        # Remove files from list
+        for remove_file in remove_files:
+            print('Removing', remove_file)
+            if dry_run:
+                pass
+            else:
+                readFiles.remove(remove_file)
+
+        # remove processed data from file.
+        for (r, l, m), values in modeldataD.items(): 
+            key_removes = []
+            for time in values.keys():
+                if int(time) == int(year): 
+                    # year matches:
+                    if not month: 
+                        # No month (delete all entries from this year
+                        key_removes.append(time)
+                        continue
+                    # search for month:
+                    mn = int((time - int(year)) *12)
+                    if mn == int(month):
+                        # found same month
+                        key_removes.append(time)
+
+            for key_remove in key_removes:
+                print('Removing', key_remove)
+                if dry_run:
+                    pass
+                else:
+                    del values[key_remove]
+
+        # Save file.
+        if not dry_run:
+            print('Saving:', shelvefn)
+            sh = shOpen(shelvefn)
+            sh['readFiles'] = readFiles
+            sh['modeldata'] = modeldataD
+            sh.close()
+        else:
+           print('Not saving (dry_run):', shelvefn)
+
+
+def main():
+    jobID = 'GFDL_hist'
+    years = ['2007', ]
+    months = ['07', ]
+    dataTypes = ['all', ]
+    dry_run = True 
+    for year in years:
+        for month in months:
+            remove_data(jobID, year, month=month,dataTypes=dataTypes, dry_run=dry_run) #, month)
+
+
+main()