diff --git a/README.md b/README.md index 1a05ee71..3c9e8aee 100644 --- a/README.md +++ b/README.md @@ -229,6 +229,7 @@ strictFileCheck: jobs: : description: + label: 'Job info' colour: red thickness: 0.7 linestyle: '-' @@ -271,6 +272,10 @@ These values are: - The options are: - `description`: - A description of job, which helps people differentiate the jobs in the final report. + - `label`: + - A short description of the job which will appear in the plot legend. + - If you make it too long, it won't fit on the plot. + - Optional: Default behaviour is the jobID. - `colour`: - The colour of this jobs line in the report plots. - Default colour is a randomly generated hex colour. @@ -447,6 +452,8 @@ units: Units dimensions: 1,2 or 3 # The number of dimensions after the calculations are performed layers : Surface regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific +smoothings : DataOnly both5 both30 movingav30years 5and30 30and100 + #paths: modelFiles : $BASEDIR_MODEL/$JOBID/nemo_$JOBIDo_1y_*_grid-T.nc @@ -500,6 +507,27 @@ basic functions such as multiply by or add to, or `noChange`, which can all be called without providing the `path`, and which may have their own key word arguments. +Some of the other options for each analysis include: + - `layers`: + - The z axis layers that you want to use. + - Typically include things like `Surface`, `500m`, `1000m` etc. + - Note that this is a pre-curated list, so you can't use it to select a specific depth (like `327m` or something) + - `regions`: + - A list of regions to investigate. + - This is a pre-curated list, and they are defined in so you can't. + - These regions are defined in the file `bgcval2/bgcvaltools/makeMask.py`. + - `smoothings`: + - This is the smoothing function (if any) to apply to the data before plotting it. + - The smoothing it added before plotting, but after saving the shelve file, so it doesn't impact the data. + - No smoothing is `DataOnly`, which is also the default behaviour. + - If several smoothing options are added here, bgcval2 will generate a comparison plot for each one. + - The smoothings are defined and performed in `bgcval2/timeseries/timeseriesPlots.py` + - Other modes exist: + - `movingav30years`: A 30 year moving average + - `both5`: Both no smoothing and a 5 year moving average + - `both30`: Both no smoothing and a 30 year moving average + - `5and30`: a 5 year moving average and a 30 year moving average. + Clearing the Cache ------------------ diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index e1c3be68..f1d6b833 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -189,6 +189,7 @@ def timeseries_compare(jobs, jobDescriptions={}, lineThicknesses=defaultdict(lambda: 1), linestyles=defaultdict(lambda: '-'), + labels = {}, ensembles={}, config_user=None, ): @@ -382,6 +383,7 @@ def timeseries_compare(jobs, regions = av[name]['regions'] layers = av[name]['layers'] metrics = av[name]['metrics'] + smoothings = av[name]['smoothings'] for region, layer, metric in itertools.product(regions, layers, metrics): timesD = {} arrD = {} @@ -405,20 +407,22 @@ def timeseries_compare(jobs, units = av[name]['modeldetails']['units'] ts = 'Together' - for ls in ['DataOnly', ]: + for smoothing in smoothings: + #or ls in ['DataOnly', ]: tsp.multitimeseries( timesD, # model times (in floats) arrD, # model time series data=-999, # in situ data distribution title=title, filename=bvt.folder(imageFolder) + - '_'.join([name, region, layer, ts, ls + '.png']), + '_'.join([name, region, layer, ts, smoothing + '.png']), units=units, plotStyle=ts, - lineStyle=ls, + smoothing=smoothing, colours=colours, thicknesses=lineThicknesses, linestyles=linestyles, + labels=labels, ) # Generate a list of comparison images: @@ -515,7 +519,8 @@ def load_comparison_yml(master_compare_yml_fn): descriptions = {} shifttimes = {} # number of years to shift time axis. timeranges = {} - + labels = {} + for jobID, job_dict in details['jobs'].items(): if job_dict.get('colour', False): colours[jobID] = job_dict['colour'] @@ -530,6 +535,8 @@ def load_comparison_yml(master_compare_yml_fn): timeranges[jobID] = job_dict.get('timerange', None) suites[jobID] = job_dict.get('suite', default_suite) auto_download_dict[jobID] = job_dict.get('auto_download', auto_download_dict[jobID]) + labels[jobID] = job_dict.get('label', jobID) + details['colours'] = colours details['descriptions'] = descriptions @@ -537,6 +544,7 @@ def load_comparison_yml(master_compare_yml_fn): details['linestyles'] = linestyles details['shifttimes'] = shifttimes details['timeranges'] = timeranges + details['labels'] = labels details['suites'] = suites details['auto_download'] = auto_download_dict return details @@ -567,6 +575,7 @@ def load_yml_and_run(compare_yml, config_user, skip_timeseries): descriptions = details['descriptions'] shifttimes = details['shifttimes'] timeranges = details['timeranges'] + labels = details['labels'] suites = details['suites'] auto_download = details['auto_download'] strictFileCheck = details.get('strictFileCheck', True) @@ -578,6 +587,7 @@ def load_yml_and_run(compare_yml, config_user, skip_timeseries): print(jobID, 'colour:',colours[jobID]) print(jobID, 'line thickness & style:',thicknesses[jobID], linestyles[jobID]) print(jobID, 'Shift time by', shifttimes[jobID]) + print(jobID, 'Label: ', labels[jobID]) print(jobID, 'Time range (None means all):', timeranges.get(jobID, None)) print(jobID, 'suite:', suites[jobID]) print(jobID, 'auto_download', auto_download[jobID]) @@ -620,6 +630,7 @@ def load_yml_and_run(compare_yml, config_user, skip_timeseries): analysisname=analysis_name, lineThicknesses=thicknesses, linestyles=linestyles, + labels=labels, config_user=config_user, ) diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index d063dfa8..e6aa75d0 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -297,6 +297,11 @@ def load_key_file(key, paths, jobID, strictFileCheck=True): output_dict['metrics'] = key_dict.get('metrics', metricList) output_dict['metrics'] = parse_list_from_string(output_dict['metrics']) + # Smoothings: + default_smoothings = ['DataOnly', ] + output_dict['smoothings'] = key_dict.get('smoothings', default_smoothings) + output_dict['smoothings'] = parse_list_from_string(output_dict['smoothings']) + # Load Grid: gridFile = key_dict.get('gridFile', paths.orcaGridfn) output_dict['gridFile'] = list_input_files(gridFile, key_dict, paths)[0] diff --git a/bgcval2/download_from_mass.py b/bgcval2/download_from_mass.py index e2a3cd68..1270c1de 100755 --- a/bgcval2/download_from_mass.py +++ b/bgcval2/download_from_mass.py @@ -503,8 +503,14 @@ def download_from_mass( shared_file_path = os.path.join(paths.shared_mass_scripts, os.path.basename(download_script_path)) print('writing file in shared path', shared_file_path) - # check destination permissions: - i_can_write_this = os.access(shared_file_path, os.W_OK) + # check destination existance and permissions + if os.path.exists(shared_file_path): + # check destination permissions + i_can_write_this = os.access(shared_file_path, os.W_OK) + else: + # File doesn't exist! + i_can_write_this = True + if i_can_write_this: # I created this file and I own it: # make local copy group writable: diff --git a/bgcval2/timeseries/timeseriesPlots.py b/bgcval2/timeseries/timeseriesPlots.py index 543849f4..c34115a2 100644 --- a/bgcval2/timeseries/timeseriesPlots.py +++ b/bgcval2/timeseries/timeseriesPlots.py @@ -702,9 +702,10 @@ def multitimeseries( 'lime', ], plotStyle='Together', - lineStyle='', + smoothing='', thicknesses=defaultdict(lambda: 1), linestyles=defaultdict(lambda: '-'), + labels={}, ): if 0 in [len(timesD), len(list(timesD.keys()))]: return @@ -783,7 +784,8 @@ def multitimeseries( if np.min(arr) < ylims[0]: ylims[0] = np.min(arr) if np.max(arr) > ylims[1]: ylims[1] = np.max(arr) - if lineStyle.lower() in ['spline', 'all']: + label = labels.get(jobID, jobID) + if smoothing.lower() in ['spline', 'all']: tnew = np.linspace(times[0], times[-1], 60) arr_smooth = interpolate.spline(times, arr, tnew) pyplot.plot( @@ -792,10 +794,10 @@ def multitimeseries( c=colours[jobID], ls=linestyles[jobID], lw=thicknesses[jobID], - label=jobID + ' spline', + label=label + ' spline', ) - if lineStyle.lower() in ['movingaverage', 'both', 'all']: + if smoothing.lower() in ['movingaverage', 'both', 'all']: if len(times) > 100.: window = 30 elif len(times) > 30.: window = 15 elif len(times) > 10.: window = 4 @@ -810,10 +812,10 @@ def multitimeseries( c=colours[jobID], ls=linestyles[jobID], lw=thicknesses[jobID], - label=jobID, + label=label, ) - if lineStyle.lower() in [ + if smoothing.lower() in [ 'movingaverage5', ]: window = 5 @@ -827,11 +829,65 @@ def multitimeseries( arr_new, c=colours[jobID], ls=linestyles[jobID], - label=jobID, + label=label, lw=thicknesses[jobID], ) - if lineStyle.lower() in [ + if smoothing.lower() == '5and30': + arr_new5 = movingaverage_DT(arr, + times, + window_len=5., + window_units='years') + + arr_new30 = movingaverage_DT(arr, + times, + window_len=30., + window_units='years') + + pyplot.plot( + times, + arr_new5, + c=colours[jobID], + ls=linestyles[jobID], + lw=thicknesses[jobID]/2., + ) + pyplot.plot( + times, + arr_new30, + c=colours[jobID], + ls=linestyles[jobID], + lw=thicknesses[jobID], + label=label, + ) + if smoothing.lower() == '30and100': + arr_new5 = movingaverage_DT(arr, + times, + window_len=30., + window_units='years') + + arr_new30 = movingaverage_DT(arr, + times, + window_len=100., + window_units='years') + + pyplot.plot( + times, + arr_new5, + c=colours[jobID], + ls=linestyles[jobID], + lw=thicknesses[jobID]/2., + ) + pyplot.plot( + times, + arr_new30, + c=colours[jobID], + ls=linestyles[jobID], + lw=thicknesses[jobID], + label=label, + ) + + + if smoothing.lower() in [ 'movingav1year', ]: arr_new = movingaverage_DT(arr, @@ -844,10 +900,10 @@ def multitimeseries( c=colours[jobID], ls=linestyles[jobID], lw=thicknesses[jobID], - label=jobID, + label=label, ) - if lineStyle.lower() in [ - 'movingav5years', + if smoothing.lower() in [ + 'movingav5years', 'both5' ]: arr_new = movingaverage_DT(arr, times, @@ -859,11 +915,11 @@ def multitimeseries( c=colours[jobID], ls=linestyles[jobID], lw=thicknesses[jobID], - label=jobID, + label=label, ) - if lineStyle.lower() in [ - 'movingav30years', + if smoothing.lower() in [ + 'movingav30years', 'both30' ]: pyplot.plot(times, arr, @@ -881,11 +937,11 @@ def multitimeseries( c=colours[jobID], ls=linestyles[jobID], lw=thicknesses[jobID], - label=jobID, + label=label, ) - if lineStyle.lower() in [ - 'movingav100years', + if smoothing.lower() in [ + 'movingav100years', 'both100', ]: pyplot.plot(times, arr, @@ -903,10 +959,10 @@ def multitimeseries( c=colours[jobID], ls=linestyles[jobID], lw=thicknesses[jobID], - label=jobID, + label=label, ) - if lineStyle.lower() in [ + if smoothing.lower() in [ 'movingaverage12', ]: window = 12 @@ -921,10 +977,10 @@ def multitimeseries( c=colours[jobID], ls=linestyles[jobID], lw=2., - label=jobID, + label=label, ) - if lineStyle.lower() in [ + if smoothing.lower() in [ 'movingaverage60', ]: window = 60 @@ -939,13 +995,14 @@ def multitimeseries( c=colours[jobID], ls=linestyles[jobID], lw=2., - label=jobID, + label=label, ) - if lineStyle.lower() in [ + if smoothing.lower() in [ '', 'both', 'all', + 'both5', 'both30', 'both100', ]: pyplot.plot(times, arr, @@ -953,14 +1010,14 @@ def multitimeseries( ls=linestyles[jobID], lw=0.25) - if lineStyle.lower() in ['dataonly']: + if smoothing.lower() in ['dataonly']: pyplot.plot( times, arr, c=colours[jobID], ls=linestyles[jobID], lw=thicknesses[jobID], - label=jobID, + label=label, ) if type(data) == type(10.): diff --git a/key_files/amoc_26n.yml b/key_files/amoc_26n.yml index 5b77b064..9577ac97 100644 --- a/key_files/amoc_26n.yml +++ b/key_files/amoc_26n.yml @@ -13,3 +13,4 @@ model_convert: altmaskfile: $PATHS_BGCVAL2/bgcval2/data/basinlandmask_eORCA1.nc layers: layerless regions: regionless +smoothings: DataOnly both5 both30 movingav30years 5and30 30and100 diff --git a/key_files/drakepassagetransport.yml b/key_files/drakepassagetransport.yml index 776b572e..124f8b33 100644 --- a/key_files/drakepassagetransport.yml +++ b/key_files/drakepassagetransport.yml @@ -11,3 +11,5 @@ model_convert: areafile: $PATHS_GRIDFILE layers: layerless regions: regionless +smoothings: DataOnly 5and30 30and100 +