diff --git a/revert_shelves.py b/bgcval2/revert_shelves.py similarity index 57% rename from revert_shelves.py rename to bgcval2/revert_shelves.py index fb7f0963..564efa83 100644 --- a/revert_shelves.py +++ b/bgcval2/revert_shelves.py @@ -5,6 +5,8 @@ from shelve import open as shOpen import glob import os +import argparse + from bgcval2.bgcvaltools import bv2tools as bvt from bgcval2._runtime_config import get_run_configuration @@ -38,6 +40,8 @@ def load_all_datatypes(shelvedir, jobID): files = glob.glob(wildcards) datatypes = [] for fn in files: + if fn.find('insitu') > -1: + continue basename = os.path.basename(fn) basename = basename.replace(jobID+'_', '') basename = basename.replace('.shelve.dat', '') @@ -64,16 +68,12 @@ def remove_data(jobID, if dataTypes == ['all', ]: dataTypes = load_all_datatypes(shelvedir, jobID) - print(dataTypes) - return for dataType in dataTypes: shelvefn = shelvedir + '_'.join([ jobID, dataType, ]) + '.shelve' - print(shelvefn) - if glob.glob(shelvefn+'*'): sh = shOpen(shelvefn) print('Shelve loads okay:', shelvefn +'*') @@ -82,7 +82,13 @@ def remove_data(jobID, modeldataD = sh['modeldata'] sh.close() - if month: + # generate a time key from year/month. + if month: + if isinstance(month, str) and len(month) ==1: + month = int(month) + if isinstance(month, int): + month = bvt.mnStr(month) + time_key = year+month else: time_key = year @@ -96,7 +102,7 @@ def remove_data(jobID, if not changes: print('Nothing to remove') - return + continue # Remove files from list for remove_file in remove_files: @@ -140,15 +146,79 @@ def remove_data(jobID, print('Not saving (dry_run):', shelvefn) +def get_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('-c', + '--config-file', + default=os.path.join(os.getcwd(), + 'config-user.yml'), + help='User configuration file', + required=False) + + parser.add_argument('-j', + '--jobids', + default=None, + nargs='+', + type=str, + help='One or more jobIDs (required)', + required=True) + + parser.add_argument('-y', + '--years', + default=None, + nargs='+', + type=str, + help='One or more years (required)', + required=True) + parser.add_argument('-m', + '--months', + default=[None, ], + nargs='+', + type=str, + help='One or more months - default is all months', + required=False) + + parser.add_argument('-k', + '--keys', + default=['all',], + nargs='+', + type=str, + help='One or more datasets - default is everything. ', + required=False) + + parser.add_argument('-d', + '--dry-run', + action='store_true', + help='Dry run: Do not edit any files.', + required=False + ) + + + args = parser.parse_args() + + return args + + + def main(): - jobID = 'GFDL_hist' - years = ['2007', ] - months = ['07', ] - dataTypes = ['all', ] - dry_run = True - for year in years: - for month in months: - remove_data(jobID, year, month=month,dataTypes=dataTypes, dry_run=dry_run) #, month) + args = get_args() + jobIDs = args.jobids + years = args.years + months = args.months + dataTypes = args.keys + dry_run = args.dry_run + + for jobID in jobIDs: + for year in years: + for month in months: + print('start: remove_data', jobID, year,month, dataTypes, dry_run) + remove_data(jobID, year, month=month,dataTypes=dataTypes, dry_run=dry_run) + + +if __name__ == "__main__": + main() -main() diff --git a/input_yml/mission_atlantic_ALL.yml b/input_yml/mission_atlantic_ALL.yml new file mode 100644 index 00000000..b65eadeb --- /dev/null +++ b/input_yml/mission_atlantic_ALL.yml @@ -0,0 +1,59 @@ +--- +# GC5 N96 ORCA1 spinup analysis +name: MissionAtlantic_All + +# Run the single job analysis +do_analysis_timeseries: True + +# Download from mass: +do_mass_download: False + +# Strict file check (fails if True and files are missing) +strictFileCheck: False + +# master analysis suite +master_suites: missionatlantic + +# Auto download jobs (can also be set for individual jobs). +auto_download: False + +# Job ID's suites as named by Rose/Cylc +jobs: + CNRM_hist: + description: 'CNRM historical run.' + label: 'CNRM-hist' + colour: green + thickness: 1.8 + linestyle: '-' + shifttime: 0. + suite: missionatlantic missionatlantic_kmf + + GFDL_hist: + description: 'GFDL historical run.' + label: 'GFDL-hist' + colour: black + thickness: 1.2 + linestyle: '-' + shifttime: 0. + suite: missionatlantic missionatlantic_kmf + + + CNRM_ssp370: + description: 'CNRM SSP3-7.0' + label: 'CNRM-ssp370' + colour: purple + thickness: 1.2 + linestyle: '-' + shifttime: 0. + suite: missionatlantic missionatlantic_kmf + + CNRM_ssp126: + description: 'CNRM SSP1-2.6' + label: 'CNRM-ssp126' + colour: dodgerblue + thickness: 1.2 + linestyle: '-' + shifttime: 0. + suite: missionatlantic missionatlantic_kmf + + diff --git a/key_files/ma_nitrate_kmf.yml b/key_files/ma_nitrate_kmf.yml new file mode 100644 index 00000000..3d74191d --- /dev/null +++ b/key_files/ma_nitrate_kmf.yml @@ -0,0 +1,18 @@ +--- +name : MA_Nitrate +units : mmol N/m^3 +dimensions : 1 +model : NEMO025-ERSEM +modelgrid : eORCA025 +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc +gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc +model_vars : N3_n +model_convert : NoChange + #dataFile : $BASEDIR_OBS/WOA/annual/woa13_all_n00_01.nc +dataFile : $BASEDIR_OBS/WOA/annual/woa18_all_n00_01.nc +data_vars : n_an +data_convert : NoChange +data_source : WOA18 +layers : Surface +regions : Global +metrics : mean diff --git a/key_files/ma_sst_kmf.yml b/key_files/ma_sst_kmf.yml new file mode 100644 index 00000000..131339fa --- /dev/null +++ b/key_files/ma_sst_kmf.yml @@ -0,0 +1,20 @@ +--- +name : MA_SST +units : degrees C +dimensions : 1 +model : NEMO025-ERSEM +modelgrid : eORCA025 +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_T*.nc +gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc +model_vars : thetao thetao_con votemper +model_convert : choose_best_var + #dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa13_decav_t00_01v2_AMM.nc +dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa18_decav_t00_01.nc +data_vars : t_an +data_convert : NoChange +data_source : WOA18 +layers : Surface #50m 100m +regions : Global +metrics : mean +smoothings : DataOnly both5 + diff --git a/setup.py b/setup.py index 015512b2..ac37167e 100755 --- a/setup.py +++ b/setup.py @@ -203,6 +203,7 @@ def read_authors(filename): 'analysis_timeseries = bgcval2.analysis_timeseries:main', 'download_from_mass = bgcval2.download_from_mass:main', 'bgcval2_make_report = bgcval2.bgcval2_make_report:main', + 'revert_shelves = bgcval2.revert_shelves:main', ], }, cmdclass={