diff --git a/README.md b/README.md index d67affd4..693cec07 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,7 @@ Executable name | What it does | Command `bgcval2_make_report` | makes the single model HTML report. | bgcval2_make_report jobID `analysis_compare` | runs comparison of multiple single jobs | analysis_compare `batch_timeseries` | Submits single job time series analysis to slurm | batch_timeseries +`revert_shelves` | Removes specific years or months from processed shelves | revert_shelves -j jobID -y years -m months ### Checking out development branches @@ -607,6 +608,31 @@ The third place that these plots are kept is on the public facing jasmin directo This is where the report is hosted. + + +Fixing mistakes +--------------- + +The `revert_shelves` tool is built to remove erroneous processed data from bgcval2 +shelves (where processed data is stored. +This is useful for processed updated simulations, new runs, post hoc fixes +and for debugging. + +The revert Shelves tool is set up to remove for a given jobID, year, and even month. + +A typical useage would be: +``` +revert_shelves -j jobID1 jobID2 -y 2009 2010 -m 02 11 -k AMOC +``` + +This would remove all AMOC data from February and November from the +years 2009 and 2010. +The default behaviour for the `-k` argument is to run this over all keys. + +Test this command using the dry run argument `-d`. + + + Point to point analysis ----------------------- diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index f1d6b833..6a3bfff1 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -415,7 +415,7 @@ def timeseries_compare(jobs, data=-999, # in situ data distribution title=title, filename=bvt.folder(imageFolder) + - '_'.join([name, region, layer, ts, smoothing + '.png']), + '_'.join([name, region, layer, ts, metric, smoothing + '.png']), units=units, plotStyle=ts, smoothing=smoothing, diff --git a/bgcval2/bgcval.py b/bgcval2/bgcval.py index 02ff2959..894b4bad 100755 --- a/bgcval2/bgcval.py +++ b/bgcval2/bgcval.py @@ -38,13 +38,13 @@ import sys from multiprocessing import Pool -from .download_from_mass import findLastFinishedYear -from .analysis_timeseries import analysis_timeseries, singleTimeSeries, singleTimeSeriesProfile +from bgcval2.download_from_mass import findLastFinishedYear +from bgcval2.analysis_timeseries import analysis_timeseries, singleTimeSeries, singleTimeSeriesProfile # these modules are no more and break the tests # from .analysis_timeseries import level1KeysDict, physKeysDict, bgcKeysDict, keymetricsfirstDict -from .analysis_p2p import analysis_p2p, p2pDict_level2, p2pDict_physics, single_p2p -from .bgcval2_make_report import html5Maker -from .bv2tools import folder +from bgcval2.analysis_p2p import analysis_p2p, p2pDict_level2, p2pDict_physics, single_p2p +from bgcval2.bgcval2_make_report import html5Maker +from bgcval2.bv2tools import folder def timeseriesParrallelL1(index): diff --git a/bgcval2/bgcval2_make_report.py b/bgcval2/bgcval2_make_report.py index 50bd6884..4eae28b2 100755 --- a/bgcval2/bgcval2_make_report.py +++ b/bgcval2/bgcval2_make_report.py @@ -164,6 +164,7 @@ def newImageLocation(fn): Level1Profiles = True level2Horizontal = True level2Physics = False + level2_auto = True summarySections = False level3OMZ = False Level3Salinity = False @@ -953,27 +954,51 @@ def newImageLocation(fn): FileLists=FileLists, FileOrder=FileOrder) - if level2Physics: - l2Fields = [ - 'Temperature', - 'Salinity', - 'MLD', - 'ZonalCurrent', - 'MeridionalCurrent', - 'VerticalCurrent', - ] + + + + + if level2Physics or level2_auto: + if level2Physics: + l2Fields = [ + 'Temperature', + 'Salinity', + 'MLD', + 'ZonalCurrent', + 'MeridionalCurrent', + 'VerticalCurrent', + ] + slices = [ + 'Surface', + '1000m', + 'Transect', + ] + SectionTitle = 'Level 2 - Physics' + region = 'Global' + + if level2_auto: + l2Fields = glob(imagedir + '/' + jobID + '/P2Pplots/*/*') + l2Fields = [os.path.basename(fn) for fn in sorted(l2Fields)] + levels = ['Surface', '4000m', '2000m', '1000m', '750m','500m','200m', '100m', '50m', 'Transect'] + outdict = {} + outlevels = {} + for i, fn in enumerate(l2Fields): + for lv in levels: + if fn.find(lv) > -1: + outlevels[lv] = True + fn = fn.replace(lv, '') + outdict[fn] = True + + l2Fields = [key for key, v in outdict.items()] + slices = [key for key, v in outlevels.items()] + SectionTitle = 'Level 2' + region = '*' + hrefs = [] Titles = {} SidebarTitles = {} Descriptions = {} FileLists = {} - SectionTitle = 'Level 2 - Physics' - region = 'Global' - slices = [ - 'Surface', - '1000m', - 'Transect', - ] FileOrder = {} for key in sorted(l2Fields): @@ -1003,7 +1028,7 @@ def newImageLocation(fn): for s in slices: if s in [ 'Surface', - '1000m', + '*', ]: vfiles.extend( glob(imagedir + '/' + jobID + '/P2Pplots/*/*' + key + @@ -1013,6 +1038,11 @@ def newImageLocation(fn): glob(imagedir + '/' + jobID + '/P2Pplots/*/*' + key + '*/*/*' + s + '*' + region + '*' + key + '*' + year + '*robinquad-cartopy.png')) + vfiles.extend( + glob(imagedir + '/' + jobID + '/P2Pplots/*/*' + key + + '*/*/*' + s + '*' + region + '*' + key + '*' + + year + '*.png')) + if s in [ 'Transect', ]: @@ -1473,8 +1503,8 @@ def newImageLocation(fn): 'Salinty_Global_Surface', 'FreshwaterFlux_Global', 'TotalHeatFlux', - 'MA_SST', - 'MA_SSS', + 'MA_SST_Global_Surface', + 'MA_SSS_Global_Surface', 'MA_Drake', 'MA_AMOC_26N', 'MA_AEU', diff --git a/bgcval2/default-bgcval2-config.yml b/bgcval2/default-bgcval2-config.yml index 2d2cfa58..4571f81d 100644 --- a/bgcval2/default-bgcval2-config.yml +++ b/bgcval2/default-bgcval2-config.yml @@ -70,7 +70,7 @@ standard-paths: p2p_ppDir: "/local1/data/scratch/ledm/bgcval2_postProcessed" imagedir: "images" #ModelFolder_pref: "gle/dare_baseline" #AMM7 - ModelFolder_pref: "ledm/MissionAtlantic/SENEMO" + ModelFolder_pref: "ledm/MissionAtlantic/" orcaGridfn: "/data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc" amm7Gridfn: "/users/modellers/ledm/workspace/coast/AMM7_grid/mesh_mask.nc" ObsFolder: "/data/sthenno1/scratch/ledm/Observations/" diff --git a/bgcval2/functions/TotalIntPP.py b/bgcval2/functions/TotalIntPP.py index 6226fc56..57437b6f 100644 --- a/bgcval2/functions/TotalIntPP.py +++ b/bgcval2/functions/TotalIntPP.py @@ -31,6 +31,8 @@ from bgcval2.bgcvaltools.dataset import dataset from bgcval2.functions.get_kwarg_file import get_kwarg_file from bgcval2.functions.standard_functions import find_best_var +import os + global loadedArea global model_area @@ -84,6 +86,10 @@ def MA_TotalIntPP(nc, keys, **kwargs): # This will only work for NEMO models: thkfn = nc.filename.replace('diag_T', 'grid_T').replace('ptrc_T', 'grid_T') print('MA_TotalIntPP: opening:', thkfn) + if not os.path.exists(thkfn): + print('MA_TotalIntPP: ERROR: does not exist:', thkfn) + return np.ma.masked + thknc = dataset(thkfn, 'r') thick = thknc.variables['thkcello'][:] thknc.close() diff --git a/bgcval2/functions/circulation.py b/bgcval2/functions/circulation.py index 94b6bf3e..7af0d624 100644 --- a/bgcval2/functions/circulation.py +++ b/bgcval2/functions/circulation.py @@ -244,7 +244,6 @@ def TwentySixNorth(nc,keys,**kwargs): assert 0 - zv = np.ma.array(nc.variables[keys[0]][..., latslice26Nnm, :]) # m/s atlmoc = np.array(np.zeros_like(zv[0, :, :, 0])) print('TwentySixNorth:', e3v_AMOC26N.shape, atlmoc.shape, zv.shape) @@ -290,11 +289,28 @@ def twentysixnorth025(nc,keys,**kwargs): depths = np.ma.masked_where(thkcello.mask + np.abs(depths)<500., depths) # masked above 500m depth. e1v = e1v_AMOC26N[:,None, :, :] - flux = vo * depths * e1v_AMOC26N[:,None, :, :]/1.E06 - - moc=np.ma.zeros_like(flux) - np.ma.cumsum(flux[:,::-1], axis=1, out=moc ) # sum floor to surface - return moc.max() +# flux = vo * thkcello * e1v_AMOC26N[:,None, :, :]/1.E06 +# moc=np.ma.zeros_like(flux) +# np.ma.cumsum(flux[:,::-1], axis=1, out=moc ) # sum floor to surface +# return moc.max() + atlmoc = np.array(np.zeros_like(vo.squeeze())) + #print(atlmoc.shape, thkcello.shape, depths.shape, vo.shape, e1v_AMOC26N.shape, e3v_AMOC26N.shape) + # (75, 264) (1, 75, 1, 264) (1, 75, 1, 264) (1, 75, 1, 264) (1, 1, 264) (75, 264) + + #assert 0 + + for (t, z, la, lo), vox in np.ndenumerate(vo): + if not vox: + continue + if not depths[t, z, la, lo] or np.ma.is_masked(depths[t, z, la, lo]): + continue + atlmoc[z, la] = atlmoc[z, la] - e1v[t, 0, la, lo] * thkcello[t, z, la, lo] * vo[t, z, la, lo] / 1.E06 + + for z in range(thkcello.shape[1] -2, 1, -1): # add from the bottom up + atlmoc[z, :] = atlmoc[z+1, :] + atlmoc[z, :] + print('AMOC:', atlmoc.max()) + return atlmoc.max() + def AMOC26N(nc, keys, **kwargs): diff --git a/bgcval2/p2p/matchDataAndModel.py b/bgcval2/p2p/matchDataAndModel.py index 81c86451..f470c46f 100755 --- a/bgcval2/p2p/matchDataAndModel.py +++ b/bgcval2/p2p/matchDataAndModel.py @@ -44,6 +44,7 @@ # local imports import bgcval2.bgcvaltools.bv2tools as bvt from bgcval2.bgcvaltools.pftnames import CMIP5models + ##### # These are availalble in the module: # https://gitlab.ecosystem-modelling.pml.ac.uk/ledm/netcdf_manip @@ -268,8 +269,10 @@ def _convertDataTo1D_(self, ): '100m', '200m', '500m', + '750m', '1000m', '2000m', + '4000m', ]: print( 'matchDataAndModel:\tconvertDataTo1D:\tSlicing along depth direction.' @@ -279,8 +282,10 @@ def _convertDataTo1D_(self, ): if self.depthLevel == '100m': z = 100. if self.depthLevel == '200m': z = 200. if self.depthLevel == '500m': z = 500. + if self.depthLevel == '750m': z = 750. if self.depthLevel == '1000m': z = 1000. if self.depthLevel == '2000m': z = 2000. + if self.depthLevel == '4000m': z = 4000. if nc.variables[self.datacoords['z']].ndim == 1: k = bvt.getORCAdepth( @@ -921,6 +926,7 @@ def loadMesh(self, modelfile=None): if self.loncc.ndim == 1 and self.loncc.shape != self.latcc.shape: self.loncc, self.latcc = np.meshgrid(self.loncc, self.latcc) + # self.depthcc = choose_best_ncvar(ncER, depthNames)[:] # if 'deptht' in list(ncER.variables.keys()): # self.depthcc = ncER.variables['deptht'][:].squeeze() @@ -930,7 +936,13 @@ def loadMesh(self, modelfile=None): # self.depthcc = ncER.variables['lev'][:].squeeze() # else: # print(self.modelcoords['z'], ncER.variables.keys()) - self.depthcc = ncER.variables[self.modelcoords['z']][:] + if self.modelcoords['z'] in ncER.variables.keys(): + self.depthcc = ncER.variables[self.modelcoords['z']][:] + else: + print('load mesh:', self.modelcoords['z'], 'is missing', ncER.variables.keys()) + if not set(ncER.variables.keys()).intersection(set(depthNames)): + print('load mesh: no depth field available in ', modelfile) + self.depthcc = np.array([0]) # self.depthcc = choose_best_ncvar(ncER, depthNames)[:] self.datescc = var_to_datetime(ncER.variables[self.modelcoords['t']]) @@ -1122,6 +1134,8 @@ def var_to_datetime(ncvar): if units in ['months since 0000-01-01 00:00:00', ]: units = 'months since 2000-01-01 00:00:00' return num2date(ncvar[:], 'months since 2000-01-01 00:00:00', calendar='360_day') + elif units.find('months since') > -1: + return num2date(ncvar[:], units, calendar='360_day') return num2date(ncvar[:], ncvar.units, calendar=calendar) diff --git a/bgcval2/p2p/testsuite_p2p.py b/bgcval2/p2p/testsuite_p2p.py index 0b59037a..2d12a840 100644 --- a/bgcval2/p2p/testsuite_p2p.py +++ b/bgcval2/p2p/testsuite_p2p.py @@ -27,6 +27,7 @@ """ #Standard Python modules: +import os from sys import argv, exit from os.path import exists from calendar import month_name @@ -162,6 +163,9 @@ def testsuite_p2p( # Match observations and model. # Does not produce and plots. #annual = True + if not os.path.exists(av[name].get('dataFiles', '')): + print('Data file missing:', name, av[name].get('dataFiles', '')) + assert 0 b = matchDataAndModel(av[name]['dataFiles'], av[name]['modelFiles'], dataType=name, diff --git a/bgcval2/revert_shelves.py b/bgcval2/revert_shelves.py new file mode 100644 index 00000000..b9a2d152 --- /dev/null +++ b/bgcval2/revert_shelves.py @@ -0,0 +1,222 @@ + +""" +In this script, we remove years of data from a shelve +""" +from shelve import open as shOpen +import glob +import os +import argparse + + +from bgcval2.bgcvaltools import bv2tools as bvt +from bgcval2._runtime_config import get_run_configuration +from bgcval2.Paths.paths import paths_setter + + +def get_paths( + config_user=None + ): + # get runtime configuration + if config_user: + paths_dict, config_user = get_run_configuration(config_user) + else: + paths_dict, config_user = get_run_configuration("defaults") + + # filter paths dict into an object that's usable below + paths = paths_setter(paths_dict) + return paths.shelvedir + + +def load_all_datatypes(shelvedir, jobID): + """ + Returns a + """ + wildcards = shelvedir + '_'.join([ + jobID, + '*', + ]) + '.shelve.dat' + files = glob.glob(wildcards) + datatypes = [] + for fn in files: + if fn.find('insitu') > -1: + continue + basename = os.path.basename(fn) + basename = basename.replace(jobID+'_', '') + basename = basename.replace('.shelve.dat', '') + datatypes.append(basename) + return datatypes + + + +def remove_data(jobID, + year, + month=None, + config_user=None, + dataTypes= ['MA_AMOC_26N', ], + dry_run = True, + ): + """ + Remove all data from the year and month from shelve files. + """ + + path_shelvedir = get_paths(config_user=config_user) + + shelvedir = bvt.folder([path_shelvedir, "timeseries", jobID]) + + if dataTypes == ['all', ]: + dataTypes = load_all_datatypes(shelvedir, jobID) + + for dataType in dataTypes: + shelvefn = shelvedir + '_'.join([ + jobID, + dataType, + ]) + '.shelve' + + if glob.glob(shelvefn+'*'): + sh = shOpen(shelvefn) + print('Shelve loads okay:', shelvefn +'*') + sh = shOpen(shelvefn) + readFiles = sh['readFiles'] + modeldataD = sh['modeldata'] + sh.close() + + # generate a time key from year/month. + if month: + if isinstance(month, str) and len(month) == 1: + month = int(month) + if isinstance(month, int): + month = bvt.mnStr(month) + + time_key = year + month + else: + time_key = year + + changes = 0 + remove_files = [] + for readFile in readFiles: + if readFile.find(time_key) > -1: + remove_files.append(readFile) + changes +=1 + + if not changes: + print('Nothing to remove') + continue + + # Remove files from list + for remove_file in remove_files: + print('Removing', remove_file) + if dry_run: + pass + else: + readFiles.remove(remove_file) + + # remove processed data from file. + for (r, l, m), values in modeldataD.items(): + key_removes = [] + for time in values.keys(): + if int(time) == int(year): + # year matches: + if not month: + # No month (delete all entries from this year + key_removes.append(time) + continue + # search for month: + mn = int((time - int(year)) *12) + if mn == int(month): + # found same month + key_removes.append(time) + + for key_remove in key_removes: + print('Removing', key_remove) + if dry_run: + pass + else: + del values[key_remove] + + # Save file. + if not dry_run: + print('Saving:', shelvefn) + sh = shOpen(shelvefn) + sh['readFiles'] = readFiles + sh['modeldata'] = modeldataD + sh.close() + else: + print('Not saving (dry_run):', shelvefn) + + +def get_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('-c', + '--config-file', + default=os.path.join(os.getcwd(), + 'config-user.yml'), + help='User configuration file', + required=False) + + parser.add_argument('-j', + '--jobids', + default=None, + nargs='+', + type=str, + help='One or more jobIDs (required)', + required=True) + + parser.add_argument('-y', + '--years', + default=None, + nargs='+', + type=str, + help='One or more years (required)', + required=True) + parser.add_argument('-m', + '--months', + default=[None, ], + nargs='+', + type=str, + help='One or more months - default is all months', + required=False) + + parser.add_argument('-k', + '--keys', + default=['all',], + nargs='+', + type=str, + help='One or more datasets - default is everything (all available keys). ', + required=False) + + parser.add_argument('-d', + '--dry-run', + action='store_true', + help='Dry run: Do not edit any files.', + required=False + ) + + + args = parser.parse_args() + + return args + + + +def main(): + args = get_args() + jobIDs = args.jobids + years = args.years + months = args.months + dataTypes = args.keys + dry_run = args.dry_run + + for jobID in jobIDs: + for year in years: + for month in months: + print('start: remove_data', jobID, year,month, dataTypes, dry_run) + remove_data(jobID, year, month=month,dataTypes=dataTypes, dry_run=dry_run) + + + +if __name__ == "__main__": + main() + diff --git a/bgcval2/timeseries/timeseriesTools.py b/bgcval2/timeseries/timeseriesTools.py index 38ff1ca2..a890ee46 100644 --- a/bgcval2/timeseries/timeseriesTools.py +++ b/bgcval2/timeseries/timeseriesTools.py @@ -162,22 +162,16 @@ def getHorizontalSlice(nc, coords, details, layer, data=''): '200m', '300m', '500m', + '750m', '1000m', '2000m', '3000m', '4000m', ]: - if layer == 'Surface': z = 0. - if layer == '50m': z = 50. - if layer == '100m': z = 100. - if layer == '200m': z = 200. - if layer == '300m': z = 300. - if layer == '500m': z = 500. - if layer == '1000m': z = 1000. - if layer == '2000m': z = 2000. - if layer == '3000m': z = 3000. - if layer == '4000m': z = 4000. - print(z) + if layer == 'Surface': + z = 0. + else: + z = float(layer.replace('m', '')) k = bvt.getORCAdepth(z, nc.variables[coords['z']][:], debug=False) if isinstance(data, str): data = std_extractData(nc, details) diff --git a/input_yml/mission_atlantic.yml b/input_yml/mission_atlantic.yml deleted file mode 100644 index 2dad7d70..00000000 --- a/input_yml/mission_atlantic.yml +++ /dev/null @@ -1,59 +0,0 @@ ---- -# GC5 N96 ORCA1 spinup analysis -name: MissionAtlantic_PML - -# Run the single job analysis -do_analysis_timeseries: True - -# Download from mass: -do_mass_download: False - -# Strict file check (fails if True and files are missing) -strictFileCheck: False - -# master analysis suite -master_suites: missionatlantic - -# Auto download jobs (can also be set for individual jobs). -auto_download: False - -# Job ID's suites as named by Rose/Cylc -jobs: - test17: - description: 'NEMO-ERSEM eORCA025 - revised rivers ' - colour: green - thickness: 1.8 - linestyle: '-' - shifttime: 0. - suite: missionatlantic - test17a: - description: 'NEMO-ERSEM eORCA025 - with iron deposition x0.1' - colour: red - thickness: 1.7 - linestyle: '-' - shifttime: 0. - suite: missionatlantic - test17aa: - description: 'NEMO-ERSEM eORCA025 - with iron deposition x0.59 spinup year 1' - colour: blue - thickness: 1.7 - linestyle: '-' - shifttime: 0. - suite: missionatlantic - test17b: - description: 'NEMO-ERSEM eORCA025 - with iron deposition x0.59 spinup year 2' - colour: blue - thickness: 1.7 - linestyle: '-' - shifttime: 1. - suite: missionatlantic - test17c: - description: 'NEMO-ERSEM eORCA025 - with iron deposition x0.59 spinup year 3' - colour: blue - thickness: 1.7 - linestyle: '-' - shifttime: 2. - suite: missionatlantic - - - diff --git a/input_yml/mission_atlantic_ALL.yml b/input_yml/mission_atlantic_ALL.yml new file mode 100644 index 00000000..b65eadeb --- /dev/null +++ b/input_yml/mission_atlantic_ALL.yml @@ -0,0 +1,59 @@ +--- +# GC5 N96 ORCA1 spinup analysis +name: MissionAtlantic_All + +# Run the single job analysis +do_analysis_timeseries: True + +# Download from mass: +do_mass_download: False + +# Strict file check (fails if True and files are missing) +strictFileCheck: False + +# master analysis suite +master_suites: missionatlantic + +# Auto download jobs (can also be set for individual jobs). +auto_download: False + +# Job ID's suites as named by Rose/Cylc +jobs: + CNRM_hist: + description: 'CNRM historical run.' + label: 'CNRM-hist' + colour: green + thickness: 1.8 + linestyle: '-' + shifttime: 0. + suite: missionatlantic missionatlantic_kmf + + GFDL_hist: + description: 'GFDL historical run.' + label: 'GFDL-hist' + colour: black + thickness: 1.2 + linestyle: '-' + shifttime: 0. + suite: missionatlantic missionatlantic_kmf + + + CNRM_ssp370: + description: 'CNRM SSP3-7.0' + label: 'CNRM-ssp370' + colour: purple + thickness: 1.2 + linestyle: '-' + shifttime: 0. + suite: missionatlantic missionatlantic_kmf + + CNRM_ssp126: + description: 'CNRM SSP1-2.6' + label: 'CNRM-ssp126' + colour: dodgerblue + thickness: 1.2 + linestyle: '-' + shifttime: 0. + suite: missionatlantic missionatlantic_kmf + + diff --git a/input_yml/mission_atlantic_CNRM.yml b/input_yml/mission_atlantic_CNRM.yml new file mode 100644 index 00000000..163f8802 --- /dev/null +++ b/input_yml/mission_atlantic_CNRM.yml @@ -0,0 +1,46 @@ +--- +# GC5 N96 ORCA1 spinup analysis +name: MissionAtlantic_CNRM + +# Run the single job analysis +do_analysis_timeseries: True + +# Download from mass: +do_mass_download: False + +# Strict file check (fails if True and files are missing) +strictFileCheck: False + +# master analysis suite +master_suites: missionatlantic + +# Auto download jobs (can also be set for individual jobs). +auto_download: False + +# Job ID's suites as named by Rose/Cylc +jobs: + CNRM_hist: + description: 'CNRM historical run.' + colour: green + thickness: 1.8 + linestyle: '-' + shifttime: 0. + suite: missionatlantic missionatlantic_kmf + + CNRM_ssp370: + description: 'CNRM SSP3-7.0' + colour: purple + thickness: 1.8 + linestyle: '-' + shifttime: 0. + suite: missionatlantic missionatlantic_kmf + + CNRM_ssp126: + description: 'CNRM SSP1-2.6' + colour: dodgerblue + thickness: 1.8 + linestyle: '-' + shifttime: 0. + suite: missionatlantic missionatlantic_kmf + + diff --git a/input_yml/mission_atlantic_GS1p0.yml b/input_yml/mission_atlantic_GS1p0.yml new file mode 100644 index 00000000..4e3f150b --- /dev/null +++ b/input_yml/mission_atlantic_GS1p0.yml @@ -0,0 +1,112 @@ +--- +# GC5 N96 ORCA1 spinup analysis +name: MissionAtlantic_GS1p0 + +# Run the single job analysis +do_analysis_timeseries: True + +# Download from mass: +do_mass_download: False + +# Strict file check (fails if True and files are missing) +strictFileCheck: False + +# master analysis suite +master_suites: missionatlantic + +# Auto download jobs (can also be set for individual jobs). +auto_download: False + +# Job ID's suites as named by Rose/Cylc +jobs: + GS1p0_hind: + description: 'NEMO-ERSEM eORCA025 - GS1p0_hind' + colour: green + thickness: 1.8 + linestyle: '-' + shifttime: 0. + suite: missionatlantic + +# test17a: +# description: 'NEMO-ERSEM eORCA025 - with iron deposition x0.1' +# colour: red +# thickness: 1.7 +# linestyle: '-' +# shifttime: 0. +# suite: missionatlantic + +# test17aa: +# description: 'NEMO-ERSEM eORCA025 - with iron deposition x0.59 spinup year 1' +# colour: blue +# thickness: 1.7 +# linestyle: '-' +# shifttime: 0. +# suite: missionatlantic +# test17b: +# description: 'NEMO-ERSEM eORCA025 - with iron deposition x0.59 spinup year 2' +# colour: blue +# thickness: 1.7 +# linestyle: '-' +# shifttime: 1. +# suite: missionatlantic +# test17c: +# description: 'NEMO-ERSEM eORCA025 - with iron deposition x0.59 spinup year 3' +# colour: blue +# thickness: 1.7 +# linestyle: '-' +# shifttime: 2. +# suite: missionatlantic + +# Broken with weird DIC/pH +# test18a: +# description: 'NEMO-ERSEM eORCA025 - new river input - spinup year 1' +# colour: purple +# thickness: 1.7 +# linestyle: '-' +# shifttime: 0. +# suite: missionatlantic +# test18b: +# description: 'NEMO-ERSEM eORCA025 - new river input - spinup year 2' +# colour: purple +# thickness: 1.7 +# linestyle: '-' +# shifttime: 1. +# suite: missionatlantic +# test18c: +# description: 'NEMO-ERSEM eORCA025 - new river input - spinup year 3' +# colour: purple +# thickness: 1.7 +# linestyle: '-' +# shifttime: 2. +# suite: missionatlantic + +# test19_S1: +# description: 'NEMO-ERSEM eORCA025 - same as 18 but better - spinup year 1' +# colour: orange +# thickness: 1.7 +# linestyle: '-' +# shifttime: 0. +# suite: missionatlantic +# test19_S2: +# description: 'NEMO-ERSEM eORCA025 - same as 18 but better - spinup year 2' +# colour: orange +# thickness: 1.7 +# linestyle: '-' +# shifttime: 1. +# suite: missionatlantic +# test19_S3: +# description: 'NEMO-ERSEM eORCA025 - same as 18 but better - spinup year 3' +# colour: orange +# thickness: 1.7 +# linestyle: '-' +# shifttime: 2. +# suite: missionatlantic + + test25: + description: 'NEMO-ERSEM eORCA025 - stable(?!)' + colour: purple + thickness: 1.7 + linestyle: '-' + shifttime: 02. + suite: missionatlantic + diff --git a/input_yml/mission_atlantic_debug.yml b/input_yml/mission_atlantic_debug.yml deleted file mode 100644 index a96d6af1..00000000 --- a/input_yml/mission_atlantic_debug.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -# GC5 N96 ORCA1 spinup analysis -name: MissionAtlantic_PML_debug - -# Run the single job analysis -do_analysis_timeseries: True - -# Download from mass: -do_mass_download: False - -# master analysis suite -master_suites: ma_debug - -# Auto download jobs (can also be set for individual jobs). -auto_download: False - -# Job ID's suites as named by Rose/Cylc -jobs: - test16: - description: 'NEMO-ERSEM eORCA025' - colour: purple - thickness: 1.5 - linestyle: '-' - shifttime: 0. - suite: ma_debug - diff --git a/key_files/ma_aeu.yml b/key_files/ma_aeu.yml index c24219f3..1744824d 100644 --- a/key_files/ma_aeu.yml +++ b/key_files/ma_aeu.yml @@ -3,7 +3,7 @@ name: AEU #AtlanticEquatorialUndercurent units: Sv dimensions: 1 model: NEMO -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*grid_U*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_U*.nc modelgrid : eORCA025 model_vars: uo model_convert: @@ -14,3 +14,5 @@ model_convert: max_depth: 500. layers: layerless regions: regionless +smoothings : DataOnly both5 + diff --git a/key_files/ma_ammonium.yml b/key_files/ma_ammonium.yml index 8106c8d1..b7824c04 100644 --- a/key_files/ma_ammonium.yml +++ b/key_files/ma_ammonium.yml @@ -4,10 +4,10 @@ units : mmol N/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N4_n model_convert : NoChange -layers : Surface 500m +layers : Surface #00m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_amoc_26n.yml b/key_files/ma_amoc_26n.yml index 9388b75a..bfbe5760 100644 --- a/key_files/ma_amoc_26n.yml +++ b/key_files/ma_amoc_26n.yml @@ -4,7 +4,7 @@ name: AMOC_26N units: Sv dimensions: 1 model: NEMO -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*grid_V*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_V*.nc modelgrid : eORCA025 model_vars: vo model_convert: @@ -15,3 +15,5 @@ model_convert: grid: eORCA025 layers: layerless regions: regionless +smoothings : DataOnly both5 + diff --git a/key_files/ma_dic.yml b/key_files/ma_dic.yml index 434ed5b8..e8140b8f 100644 --- a/key_files/ma_dic.yml +++ b/key_files/ma_dic.yml @@ -4,7 +4,7 @@ units : mmol C/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : O3_c model_convert : NoChange diff --git a/key_files/ma_drake.yml b/key_files/ma_drake.yml index 899b8af3..8b1d33cd 100644 --- a/key_files/ma_drake.yml +++ b/key_files/ma_drake.yml @@ -3,7 +3,7 @@ name: DrakePassageTransport units: Sv dimensions: 1 model: NEMO -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*grid_U*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_U*.nc modelgrid : eORCA025 model_vars: uo model_convert: @@ -13,3 +13,5 @@ model_convert: grid: eORCA025 layers: layerless regions: regionless +smoothings : DataOnly both5 + diff --git a/key_files/ma_gpp.yml b/key_files/ma_gpp.yml index abd2bbd9..56058c30 100644 --- a/key_files/ma_gpp.yml +++ b/key_files/ma_gpp.yml @@ -4,7 +4,7 @@ units : "" dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*diag_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : Ptot_GPP_result model_convert : NoChange diff --git a/key_files/ma_iron.yml b/key_files/ma_iron.yml index 015e398c..05830953 100644 --- a/key_files/ma_iron.yml +++ b/key_files/ma_iron.yml @@ -4,10 +4,10 @@ units : umol Fe/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N7_f model_convert : NoChange -layers : Surface 500m +layers : Surface #500m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_mld.yml b/key_files/ma_mld.yml index 47288a3d..bdaadbc9 100644 --- a/key_files/ma_mld.yml +++ b/key_files/ma_mld.yml @@ -4,9 +4,14 @@ units : m dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc -model_vars : somxzint1 +model_convert : choose_best_var +model_vars : somxzint1 mldr10_1 +dataFile : $BASEDIR_OBS/WOA/annual/woa18_A5B7_M0200_01.nc +data_vars : M_an +data_convert : NoChange +data_source : WOA18 layers : layerless regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean ITCZ metrics : mean, max #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_mld_sigma.yml b/key_files/ma_mld_sigma.yml index 8d6987ac..e22254d3 100644 --- a/key_files/ma_mld_sigma.yml +++ b/key_files/ma_mld_sigma.yml @@ -4,7 +4,7 @@ units : m dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : mldr10_1 layers : layerless diff --git a/key_files/ma_nitrate.yml b/key_files/ma_nitrate.yml index 49272554..e0b6e7b5 100644 --- a/key_files/ma_nitrate.yml +++ b/key_files/ma_nitrate.yml @@ -4,14 +4,15 @@ units : mmol N/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N3_n model_convert : NoChange -dataFile : $BASEDIR_OBS/WOA/annual/woa13_all_n00_01.nc + #dataFile : $BASEDIR_OBS/WOA/annual/woa13_all_n00_01.nc +dataFile : $BASEDIR_OBS/WOA/annual/woa18_all_n00_01.nc data_vars : n_an data_convert : NoChange -data_source : WOA13 -layers : Surface 500m +data_source : WOA18 +layers : Surface #500m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_nitrate_kmf.yml b/key_files/ma_nitrate_kmf.yml new file mode 100644 index 00000000..3d74191d --- /dev/null +++ b/key_files/ma_nitrate_kmf.yml @@ -0,0 +1,18 @@ +--- +name : MA_Nitrate +units : mmol N/m^3 +dimensions : 1 +model : NEMO025-ERSEM +modelgrid : eORCA025 +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc +gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc +model_vars : N3_n +model_convert : NoChange + #dataFile : $BASEDIR_OBS/WOA/annual/woa13_all_n00_01.nc +dataFile : $BASEDIR_OBS/WOA/annual/woa18_all_n00_01.nc +data_vars : n_an +data_convert : NoChange +data_source : WOA18 +layers : Surface +regions : Global +metrics : mean diff --git a/key_files/ma_northerntotaliceextent.yml b/key_files/ma_northerntotaliceextent.yml index c24311c6..6ecb3db6 100644 --- a/key_files/ma_northerntotaliceextent.yml +++ b/key_files/ma_northerntotaliceextent.yml @@ -3,7 +3,7 @@ name : NorthernTotalIceExtent units : 1E6 km^2 dimensions : 1 model : CICE -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_T*.nc model_vars : soicecov model_convert: path: bgcval2/functions/ice.py diff --git a/key_files/ma_npp.yml b/key_files/ma_npp.yml index 245859a4..5f94bcb0 100644 --- a/key_files/ma_npp.yml +++ b/key_files/ma_npp.yml @@ -4,7 +4,7 @@ units : "" dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*diag_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : Ptot_NPP_result model_convert : NoChange diff --git a/key_files/ma_o2.yml b/key_files/ma_o2.yml index c511258c..832d2801 100644 --- a/key_files/ma_o2.yml +++ b/key_files/ma_o2.yml @@ -4,14 +4,15 @@ units : mmol O_2/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : O2_o model_convert : NoChange -dataFile : $BASEDIR_OBS/WOA/annual/woa13_all_O00_01.nc -data_vars : O_an + #dataFile : $BASEDIR_OBS/WOA/annual/woa13_all_O00_01.nc +dataFile : $BASEDIR_OBS/WOA/annual/woa18_all_o00_01.nc +data_vars : o_an data_convert : NoChange -data_source : WOA13 -layers : 500m #Surface #50m 100m +data_source : WOA18 +layers : 500m 750m 1000m #Surface #50m 100m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_p1c.yml b/key_files/ma_p1c.yml index f1b9f6f0..826f89de 100644 --- a/key_files/ma_p1c.yml +++ b/key_files/ma_p1c.yml @@ -4,7 +4,7 @@ units : mg C/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : P1_c model_convert : NoChange diff --git a/key_files/ma_ph.yml b/key_files/ma_ph.yml index 85c13500..e329942c 100644 --- a/key_files/ma_ph.yml +++ b/key_files/ma_ph.yml @@ -4,10 +4,10 @@ units : None dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : O3_pH model_convert : NoChange -layers : Surface 500m +layers : Surface #500m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean -metrics : mean, min #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max +metrics : mean #, min #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_phosphate.yml b/key_files/ma_phosphate.yml index 1439526b..14f4b90d 100644 --- a/key_files/ma_phosphate.yml +++ b/key_files/ma_phosphate.yml @@ -4,14 +4,15 @@ units : mmol P/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N1_p model_convert : NoChange -dataFile : $BASEDIR_OBS/WOA/annual/woa13_all_p00_01.nc + #dataFile : $BASEDIR_OBS/WOA/annual/woa13_all_p00_01.nc +dataFile : $BASEDIR_OBS/WOA/annual/woa18_all_p00_01.nc data_vars : p_an data_convert : NoChange -data_source : WOA13 -layers : Surface 500m +data_source : WOA18 +layers : Surface #500m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_silicate.yml b/key_files/ma_silicate.yml index 62e2ffc6..53fd8787 100644 --- a/key_files/ma_silicate.yml +++ b/key_files/ma_silicate.yml @@ -4,14 +4,15 @@ units : mmol Si/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N5_s model_convert : NoChange -dataFile : $BASEDIR_OBS/WOA/annual/woa13_all_i00_01.nc + #ataFile : $BASEDIR_OBS/WOA/annual/woa13_all_i00_01.nc +dataFile : $BASEDIR_OBS/WOA/annual/woa18_all_i00_01.nc data_vars : i_an data_convert : NoChange -data_source : WOA13 -layers : Surface 500m +data_source : WOA18 +layers : Surface #100m 200m 500m 1000m 2000m 4000m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_southerntotaliceextent.yml b/key_files/ma_southerntotaliceextent.yml index 51d0fcae..eee63f1a 100644 --- a/key_files/ma_southerntotaliceextent.yml +++ b/key_files/ma_southerntotaliceextent.yml @@ -3,7 +3,7 @@ name : SouthernTotalIceExtent units : 1E6 km^2 dimensions : 1 model : CICE -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_T*.nc model_vars : soicecov model_convert: path: bgcval2/functions/ice.py diff --git a/key_files/ma_sss.yml b/key_files/ma_sss.yml index 02445cf6..2fdc3299 100644 --- a/key_files/ma_sss.yml +++ b/key_files/ma_sss.yml @@ -4,13 +4,15 @@ units : PSU dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : so_abs so -dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa13_decav_t00_01v2_AMM.nc -data_vars : t_an + # dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa13_decav_s00_01v2_AMM.nc +dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa18_decav_s00_01.nc +data_vars : s_an data_convert : NoChange -data_source : WOA13 -layers : Surface #50m 100m -regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean ITCZ +data_source : WOA18 +layers : Surface #50m 100m 200m 500m 1000m 2000m + #layers : Surface 50m 100m 200m 500m 1000m 2000m 4000m +regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean #TCZ metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_sst.yml b/key_files/ma_sst.yml index 119a88dd..278bf75f 100644 --- a/key_files/ma_sst.yml +++ b/key_files/ma_sst.yml @@ -4,14 +4,15 @@ units : degrees C dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : thetao thetao_con votemper model_convert : choose_best_var -dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa13_decav_t00_01v2_AMM.nc + #dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa13_decav_t00_01v2_AMM.nc +dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa18_decav_t00_01.nc data_vars : t_an data_convert : NoChange -data_source : WOA13 +data_source : WOA18 layers : Surface #50m 100m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean ITCZ metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_sst_kmf.yml b/key_files/ma_sst_kmf.yml new file mode 100644 index 00000000..131339fa --- /dev/null +++ b/key_files/ma_sst_kmf.yml @@ -0,0 +1,20 @@ +--- +name : MA_SST +units : degrees C +dimensions : 1 +model : NEMO025-ERSEM +modelgrid : eORCA025 +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_T*.nc +gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc +model_vars : thetao thetao_con votemper +model_convert : choose_best_var + #dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa13_decav_t00_01v2_AMM.nc +dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa18_decav_t00_01.nc +data_vars : t_an +data_convert : NoChange +data_source : WOA18 +layers : Surface #50m 100m +regions : Global +metrics : mean +smoothings : DataOnly both5 + diff --git a/key_files/ma_ta.yml b/key_files/ma_ta.yml index b962379c..3973702d 100644 --- a/key_files/ma_ta.yml +++ b/key_files/ma_ta.yml @@ -4,10 +4,10 @@ units : umol/kg dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : O3_TA model_convert : NoChange -layers : Surface 500m +layers : Surface #500m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean -metrics : mean, max #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max +metrics : mean #, max #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_totalairseafluxco2.yml b/key_files/ma_totalairseafluxco2.yml index a6fee744..249a7d46 100644 --- a/key_files/ma_totalairseafluxco2.yml +++ b/key_files/ma_totalairseafluxco2.yml @@ -4,7 +4,7 @@ units : Pg C/yr model : NEMO-ERSEM modelgrid : eORCA025 dimensions : 1 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*diag_T*.nc model_vars : O3_fair model_convert : path: bgcval2/functions/AirSeaFluxCO2.py diff --git a/key_files/ma_totalchlorophyll.yml b/key_files/ma_totalchlorophyll.yml index c98711df..b2e00e80 100644 --- a/key_files/ma_totalchlorophyll.yml +++ b/key_files/ma_totalchlorophyll.yml @@ -4,7 +4,7 @@ units : "" dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*diag_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : Ptot_Chl_result model_convert : NoChange diff --git a/key_files/ma_totaliceextent.yml b/key_files/ma_totaliceextent.yml index 1f71a436..a8fbbeff 100644 --- a/key_files/ma_totaliceextent.yml +++ b/key_files/ma_totaliceextent.yml @@ -4,7 +4,7 @@ name : TotalIceExtent units : 1E6 km^2 dimensions : 1 model : NEMO-ERSEM -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_T*.nc model_vars : soicecov model_convert: path: bgcval2/functions/ice.py diff --git a/key_files/ma_totalintgpp.yml b/key_files/ma_totalintgpp.yml index 986759dc..e96d18ad 100644 --- a/key_files/ma_totalintgpp.yml +++ b/key_files/ma_totalintgpp.yml @@ -2,7 +2,7 @@ name : TotalIntGPP units : Gt/yr dimensions : 1 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*diag_T*.nc model_vars : Ptot_NPP_result model_convert : path: bgcval2/functions/TotalIntPP.py diff --git a/key_files/ma_totalintnpp.yml b/key_files/ma_totalintnpp.yml index 6baad60b..0cd3ce7b 100644 --- a/key_files/ma_totalintnpp.yml +++ b/key_files/ma_totalintnpp.yml @@ -2,7 +2,7 @@ name : TotalIntNPP units : Gt/yr dimensions : 1 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*diag_T*.nc model_vars : Ptot_NPP_result model_convert : path: bgcval2/functions/TotalIntPP.py diff --git a/key_files/ma_totalphytoc.yml b/key_files/ma_totalphytoc.yml index f4a97a28..8d29f1b7 100644 --- a/key_files/ma_totalphytoc.yml +++ b/key_files/ma_totalphytoc.yml @@ -4,7 +4,7 @@ units : mg C/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*diag_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : Ptot_c_result model_convert : NoChange diff --git a/key_files/ma_totalzooc.yml b/key_files/ma_totalzooc.yml index 4553066f..1391278b 100644 --- a/key_files/ma_totalzooc.yml +++ b/key_files/ma_totalzooc.yml @@ -4,7 +4,7 @@ units : mg C/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*diag_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : Ztot_c_result model_convert : NoChange diff --git a/key_files/ma_zos.yml b/key_files/ma_zos.yml index 8cbe23cb..3e6d3440 100644 --- a/key_files/ma_zos.yml +++ b/key_files/ma_zos.yml @@ -4,7 +4,7 @@ units : m dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : zos layers : layerless diff --git a/key_lists/ma_debug.yml b/key_lists/ma_debug.yml index e48ec8c2..727ed8cb 100644 --- a/key_lists/ma_debug.yml +++ b/key_lists/ma_debug.yml @@ -4,5 +4,7 @@ # of how a model is working. keys: # BGC - MA_TotalIntGPP: True - MA_TotalIntNPP: True + MA_SST: True + + # MA_TotalIntGPP: True + # MA_TotalIntNPP: True diff --git a/key_lists/missionatlantic_kmf.yml b/key_lists/missionatlantic_kmf.yml new file mode 100644 index 00000000..662fa563 --- /dev/null +++ b/key_lists/missionatlantic_kmf.yml @@ -0,0 +1,45 @@ +--- +# Key metrics first - KMF +# A short list of key metrics to run first to get a very quick idea +# of how a model is working. +keys: + # Physics: + # + MA_SST_KMF: True + #MA_SSS: True + MA_AEU: True + MA_Drake: True + MA_AMOC_26N: True + #MA_ZOS: True + #MA_MLD: True + #MA_MLD_Sigma: True + + # Ice + #MA_SouthernTotalIceExtent: True + #MA_NorthernTotalIceExtent: True + MA_TotalIceExtent: True + + # BGC + # MA_Totalairseafluxco2: True # Not in test17 + # MA_TotalIntGPP: True + MA_TotalIntNPP: True + MA_Nitrate_KMF: True + #MA_Nitrate: True + # MA_Ammonium: True + # MA_Phosphate: True + # MA_Iron: True + # MA_Silicate: True + # MA_TotalChlorophyll: True + # MA_TotalPhytoC: True + # MA_P1c: True + # MA_TotalZooC: True + # MA_NPP: True + # MA_GPP: True + # MA_pH: True + # MA_O2: True + # MA_DIC: True + # MA_TA: True + + +# what do we need here? +# Air Sea Flux of CO2 - need new data. diff --git a/key_lists/missionatlantic_p2p.yml b/key_lists/missionatlantic_p2p.yml new file mode 100644 index 00000000..8b60dcfd --- /dev/null +++ b/key_lists/missionatlantic_p2p.yml @@ -0,0 +1,42 @@ +--- +# Mission Atlantic point to point fields. +keys: + # Physics: + # + MA_SST: True + MA_SSS: True + #MA_AEU: True + #MA_Drake: True + #MA_AMOC_26N: True + #MA_ZOS: True + #MA_MLD: True + #MA_MLD_Sigma: True + + # Ice + #MA_SouthernTotalIceExtent: True + #MA_NorthernTotalIceExtent: True + #MA_TotalIceExtent: True + + # BGC + # MA_Totalairseafluxco2: True # Not in test17 + # MA_TotalIntGPP: True + #MA_TotalIntNPP: True + MA_Nitrate: True + # MA_Ammonium: True + MA_Phosphate: True + #MA_Iron: True + MA_Silicate: True + # MA_TotalChlorophyll: True + # MA_TotalPhytoC: True + # MA_P1c: True + # MA_TotalZooC: True + # MA_NPP: True + # MA_GPP: True + # MA_pH: True + MA_O2: True + # MA_DIC: True + # MA_TA: True + + +# what do we need here? +# Air Sea Flux of CO2 - need new data. diff --git a/setup.py b/setup.py index 3baa25f6..b93aa16e 100755 --- a/setup.py +++ b/setup.py @@ -204,6 +204,7 @@ def read_authors(filename): 'batch_timeseries = bgcval2.batch_timeseries:main', 'download_from_mass = bgcval2.download_from_mass:main', 'bgcval2_make_report = bgcval2.bgcval2_make_report:main', + 'revert_shelves = bgcval2.revert_shelves:main', ], }, cmdclass={