From 8ec9ecb1015d212b5662d42b0a127de594f0eacb Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 12 Dec 2023 11:42:18 +0000 Subject: [PATCH] Added MA fixes and new revert shelves tool. --- bgcval2/bgcval2_make_report.py | 6 +- input_yml/mission_atlantic_CNRM.yml | 6 +- key_files/ma_aeu.yml | 2 + key_files/ma_ammonium.yml | 2 +- key_files/ma_amoc_26n.yml | 2 + key_files/ma_drake.yml | 2 + key_files/ma_iron.yml | 2 +- key_files/ma_nitrate.yml | 2 +- key_files/ma_ph.yml | 4 +- key_files/ma_phosphate.yml | 2 +- key_files/ma_silicate.yml | 2 +- key_files/ma_sss.yml | 5 +- key_files/ma_ta.yml | 4 +- key_lists/missionatlantic_kmf.yml | 3 +- revert_shelves.py | 154 ++++++++++++++++++++++++++++ 15 files changed, 181 insertions(+), 17 deletions(-) create mode 100644 revert_shelves.py diff --git a/bgcval2/bgcval2_make_report.py b/bgcval2/bgcval2_make_report.py index a4266042..8c12805c 100755 --- a/bgcval2/bgcval2_make_report.py +++ b/bgcval2/bgcval2_make_report.py @@ -1504,8 +1504,8 @@ def newImageLocation(fn): 'Salinty_Global_Surface', 'FreshwaterFlux_Global', 'TotalHeatFlux', - 'MA_SST', - 'MA_SSS', + 'MA_SST_Global_Surface', + 'MA_SSS_Global_Surface', 'MA_Drake', 'MA_AMOC_26N', 'MA_AEU', @@ -1546,6 +1546,8 @@ def newImageLocation(fn): if found: continue sectionTitle = 'Physics Key Metrics' if fn.find(key) > -1: +# if key in ['MA_SST', 'MA_SSS', 'MA_Nitrate',] and fn.find('Global_Surface') < 0: +# continue try: categories[sectionTitle].append(fn) except: diff --git a/input_yml/mission_atlantic_CNRM.yml b/input_yml/mission_atlantic_CNRM.yml index f2d1546d..163f8802 100644 --- a/input_yml/mission_atlantic_CNRM.yml +++ b/input_yml/mission_atlantic_CNRM.yml @@ -25,7 +25,7 @@ jobs: thickness: 1.8 linestyle: '-' shifttime: 0. - suite: missionatlantic + suite: missionatlantic missionatlantic_kmf CNRM_ssp370: description: 'CNRM SSP3-7.0' @@ -33,7 +33,7 @@ jobs: thickness: 1.8 linestyle: '-' shifttime: 0. - suite: missionatlantic + suite: missionatlantic missionatlantic_kmf CNRM_ssp126: description: 'CNRM SSP1-2.6' @@ -41,6 +41,6 @@ jobs: thickness: 1.8 linestyle: '-' shifttime: 0. - suite: missionatlantic + suite: missionatlantic missionatlantic_kmf diff --git a/key_files/ma_aeu.yml b/key_files/ma_aeu.yml index 3f277cc7..1744824d 100644 --- a/key_files/ma_aeu.yml +++ b/key_files/ma_aeu.yml @@ -14,3 +14,5 @@ model_convert: max_depth: 500. layers: layerless regions: regionless +smoothings : DataOnly both5 + diff --git a/key_files/ma_ammonium.yml b/key_files/ma_ammonium.yml index a86a9843..b7824c04 100644 --- a/key_files/ma_ammonium.yml +++ b/key_files/ma_ammonium.yml @@ -8,6 +8,6 @@ modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N4_n model_convert : NoChange -layers : Surface 500m +layers : Surface #00m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_amoc_26n.yml b/key_files/ma_amoc_26n.yml index cb0a0aca..bfbe5760 100644 --- a/key_files/ma_amoc_26n.yml +++ b/key_files/ma_amoc_26n.yml @@ -15,3 +15,5 @@ model_convert: grid: eORCA025 layers: layerless regions: regionless +smoothings : DataOnly both5 + diff --git a/key_files/ma_drake.yml b/key_files/ma_drake.yml index d4b79187..8b1d33cd 100644 --- a/key_files/ma_drake.yml +++ b/key_files/ma_drake.yml @@ -13,3 +13,5 @@ model_convert: grid: eORCA025 layers: layerless regions: regionless +smoothings : DataOnly both5 + diff --git a/key_files/ma_iron.yml b/key_files/ma_iron.yml index b8e73fb6..05830953 100644 --- a/key_files/ma_iron.yml +++ b/key_files/ma_iron.yml @@ -8,6 +8,6 @@ modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N7_f model_convert : NoChange -layers : Surface 500m +layers : Surface #500m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_nitrate.yml b/key_files/ma_nitrate.yml index 869900a6..e0b6e7b5 100644 --- a/key_files/ma_nitrate.yml +++ b/key_files/ma_nitrate.yml @@ -13,6 +13,6 @@ dataFile : $BASEDIR_OBS/WOA/annual/woa18_all_n00_01.nc data_vars : n_an data_convert : NoChange data_source : WOA18 -layers : Surface 500m +layers : Surface #500m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_ph.yml b/key_files/ma_ph.yml index 9b55f665..e329942c 100644 --- a/key_files/ma_ph.yml +++ b/key_files/ma_ph.yml @@ -8,6 +8,6 @@ modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : O3_pH model_convert : NoChange -layers : Surface 500m +layers : Surface #500m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean -metrics : mean, min #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max +metrics : mean #, min #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_phosphate.yml b/key_files/ma_phosphate.yml index 3cd26121..14f4b90d 100644 --- a/key_files/ma_phosphate.yml +++ b/key_files/ma_phosphate.yml @@ -13,6 +13,6 @@ dataFile : $BASEDIR_OBS/WOA/annual/woa18_all_p00_01.nc data_vars : p_an data_convert : NoChange data_source : WOA18 -layers : Surface 500m +layers : Surface #500m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_silicate.yml b/key_files/ma_silicate.yml index 4e364ff7..53fd8787 100644 --- a/key_files/ma_silicate.yml +++ b/key_files/ma_silicate.yml @@ -13,6 +13,6 @@ dataFile : $BASEDIR_OBS/WOA/annual/woa18_all_i00_01.nc data_vars : i_an data_convert : NoChange data_source : WOA18 -layers : Surface 100m 200m 500m 1000m 2000m 4000m +layers : Surface #100m 200m 500m 1000m 2000m 4000m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_sss.yml b/key_files/ma_sss.yml index e049a85a..2fdc3299 100644 --- a/key_files/ma_sss.yml +++ b/key_files/ma_sss.yml @@ -12,6 +12,7 @@ dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa18_deca data_vars : s_an data_convert : NoChange data_source : WOA18 -layers : Surface 50m 100m 200m 500m 1000m 2000m 4000m -regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean ITCZ +layers : Surface #50m 100m 200m 500m 1000m 2000m + #layers : Surface 50m 100m 200m 500m 1000m 2000m 4000m +regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean #TCZ metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_ta.yml b/key_files/ma_ta.yml index 09f0945a..3973702d 100644 --- a/key_files/ma_ta.yml +++ b/key_files/ma_ta.yml @@ -8,6 +8,6 @@ modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : O3_TA model_convert : NoChange -layers : Surface 500m +layers : Surface #500m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean -metrics : mean, max #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max +metrics : mean #, max #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_lists/missionatlantic_kmf.yml b/key_lists/missionatlantic_kmf.yml index 279b6148..662fa563 100644 --- a/key_lists/missionatlantic_kmf.yml +++ b/key_lists/missionatlantic_kmf.yml @@ -5,7 +5,7 @@ keys: # Physics: # - # MA_SST: True + MA_SST_KMF: True #MA_SSS: True MA_AEU: True MA_Drake: True @@ -23,6 +23,7 @@ keys: # MA_Totalairseafluxco2: True # Not in test17 # MA_TotalIntGPP: True MA_TotalIntNPP: True + MA_Nitrate_KMF: True #MA_Nitrate: True # MA_Ammonium: True # MA_Phosphate: True diff --git a/revert_shelves.py b/revert_shelves.py new file mode 100644 index 00000000..fb7f0963 --- /dev/null +++ b/revert_shelves.py @@ -0,0 +1,154 @@ + +""" +In this script, we remove years of data from a shelve +""" +from shelve import open as shOpen +import glob +import os + +from bgcval2.bgcvaltools import bv2tools as bvt +from bgcval2._runtime_config import get_run_configuration +from bgcval2.Paths.paths import paths_setter + + + + +def get_paths( + config_user=None + ): + # get runtime configuration + if config_user: + paths_dict, config_user = get_run_configuration(config_user) + else: + paths_dict, config_user = get_run_configuration("defaults") + + # filter paths dict into an object that's usable below + paths = paths_setter(paths_dict) + return paths.shelvedir + + +def load_all_datatypes(shelvedir, jobID): + """ + Returns a + """ + wildcards = shelvedir + '_'.join([ + jobID, + '*', + ]) + '.shelve.dat' + files = glob.glob(wildcards) + datatypes = [] + for fn in files: + basename = os.path.basename(fn) + basename = basename.replace(jobID+'_', '') + basename = basename.replace('.shelve.dat', '') + datatypes.append(basename) + return datatypes + + + +def remove_data(jobID, + year, + month=None, + config_user=None, + dataTypes= ['MA_AMOC_26N', ], + dry_run = True, + ): + """ + Remove all data from the year and month from shelve files. + """ + + path_shelvedir = get_paths(config_user=config_user) + + shelvedir = bvt.folder([path_shelvedir, "timeseries", jobID]) + + if dataTypes == ['all', ]: + dataTypes = load_all_datatypes(shelvedir, jobID) + + print(dataTypes) + return + for dataType in dataTypes: + shelvefn = shelvedir + '_'.join([ + jobID, + dataType, + ]) + '.shelve' + + print(shelvefn) + + if glob.glob(shelvefn+'*'): + sh = shOpen(shelvefn) + print('Shelve loads okay:', shelvefn +'*') + sh = shOpen(shelvefn) + readFiles = sh['readFiles'] + modeldataD = sh['modeldata'] + sh.close() + + if month: + time_key = year+month + else: + time_key = year + + changes = 0 + remove_files = [] + for readFile in readFiles: + if readFile.find(time_key) > -1: + remove_files.append(readFile) + changes +=1 + + if not changes: + print('Nothing to remove') + return + + # Remove files from list + for remove_file in remove_files: + print('Removing', remove_file) + if dry_run: + pass + else: + readFiles.remove(remove_file) + + # remove processed data from file. + for (r, l, m), values in modeldataD.items(): + key_removes = [] + for time in values.keys(): + if int(time) == int(year): + # year matches: + if not month: + # No month (delete all entries from this year + key_removes.append(time) + continue + # search for month: + mn = int((time - int(year)) *12) + if mn == int(month): + # found same month + key_removes.append(time) + + for key_remove in key_removes: + print('Removing', key_remove) + if dry_run: + pass + else: + del values[key_remove] + + # Save file. + if not dry_run: + print('Saving:', shelvefn) + sh = shOpen(shelvefn) + sh['readFiles'] = readFiles + sh['modeldata'] = modeldataD + sh.close() + else: + print('Not saving (dry_run):', shelvefn) + + +def main(): + jobID = 'GFDL_hist' + years = ['2007', ] + months = ['07', ] + dataTypes = ['all', ] + dry_run = True + for year in years: + for month in months: + remove_data(jobID, year, month=month,dataTypes=dataTypes, dry_run=dry_run) #, month) + + +main()