From 34b44ba32ff3b85a8c9c155e76fd326e4c655172 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Thu, 2 Nov 2023 14:24:17 +0000 Subject: [PATCH 1/6] fixed missing file case --- bgcval2/download_from_mass.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/bgcval2/download_from_mass.py b/bgcval2/download_from_mass.py index e2a3cd68..1270c1de 100755 --- a/bgcval2/download_from_mass.py +++ b/bgcval2/download_from_mass.py @@ -503,8 +503,14 @@ def download_from_mass( shared_file_path = os.path.join(paths.shared_mass_scripts, os.path.basename(download_script_path)) print('writing file in shared path', shared_file_path) - # check destination permissions: - i_can_write_this = os.access(shared_file_path, os.W_OK) + # check destination existance and permissions + if os.path.exists(shared_file_path): + # check destination permissions + i_can_write_this = os.access(shared_file_path, os.W_OK) + else: + # File doesn't exist! + i_can_write_this = True + if i_can_write_this: # I created this file and I own it: # make local copy group writable: From 841f07369c0cf34a875d9cfa8f73f3598ad3d89c Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Thu, 2 Nov 2023 15:23:09 +0000 Subject: [PATCH 2/6] added capability to choose moving average and labels in the yml files --- bgcval2/analysis_compare.py | 19 ++++++++-- bgcval2/analysis_timeseries.py | 5 +++ bgcval2/timeseries/timeseriesPlots.py | 53 ++++++++++++++------------- key_files/amoc_26n.yml | 1 + 4 files changed, 49 insertions(+), 29 deletions(-) diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index e1c3be68..f1d6b833 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -189,6 +189,7 @@ def timeseries_compare(jobs, jobDescriptions={}, lineThicknesses=defaultdict(lambda: 1), linestyles=defaultdict(lambda: '-'), + labels = {}, ensembles={}, config_user=None, ): @@ -382,6 +383,7 @@ def timeseries_compare(jobs, regions = av[name]['regions'] layers = av[name]['layers'] metrics = av[name]['metrics'] + smoothings = av[name]['smoothings'] for region, layer, metric in itertools.product(regions, layers, metrics): timesD = {} arrD = {} @@ -405,20 +407,22 @@ def timeseries_compare(jobs, units = av[name]['modeldetails']['units'] ts = 'Together' - for ls in ['DataOnly', ]: + for smoothing in smoothings: + #or ls in ['DataOnly', ]: tsp.multitimeseries( timesD, # model times (in floats) arrD, # model time series data=-999, # in situ data distribution title=title, filename=bvt.folder(imageFolder) + - '_'.join([name, region, layer, ts, ls + '.png']), + '_'.join([name, region, layer, ts, smoothing + '.png']), units=units, plotStyle=ts, - lineStyle=ls, + smoothing=smoothing, colours=colours, thicknesses=lineThicknesses, linestyles=linestyles, + labels=labels, ) # Generate a list of comparison images: @@ -515,7 +519,8 @@ def load_comparison_yml(master_compare_yml_fn): descriptions = {} shifttimes = {} # number of years to shift time axis. timeranges = {} - + labels = {} + for jobID, job_dict in details['jobs'].items(): if job_dict.get('colour', False): colours[jobID] = job_dict['colour'] @@ -530,6 +535,8 @@ def load_comparison_yml(master_compare_yml_fn): timeranges[jobID] = job_dict.get('timerange', None) suites[jobID] = job_dict.get('suite', default_suite) auto_download_dict[jobID] = job_dict.get('auto_download', auto_download_dict[jobID]) + labels[jobID] = job_dict.get('label', jobID) + details['colours'] = colours details['descriptions'] = descriptions @@ -537,6 +544,7 @@ def load_comparison_yml(master_compare_yml_fn): details['linestyles'] = linestyles details['shifttimes'] = shifttimes details['timeranges'] = timeranges + details['labels'] = labels details['suites'] = suites details['auto_download'] = auto_download_dict return details @@ -567,6 +575,7 @@ def load_yml_and_run(compare_yml, config_user, skip_timeseries): descriptions = details['descriptions'] shifttimes = details['shifttimes'] timeranges = details['timeranges'] + labels = details['labels'] suites = details['suites'] auto_download = details['auto_download'] strictFileCheck = details.get('strictFileCheck', True) @@ -578,6 +587,7 @@ def load_yml_and_run(compare_yml, config_user, skip_timeseries): print(jobID, 'colour:',colours[jobID]) print(jobID, 'line thickness & style:',thicknesses[jobID], linestyles[jobID]) print(jobID, 'Shift time by', shifttimes[jobID]) + print(jobID, 'Label: ', labels[jobID]) print(jobID, 'Time range (None means all):', timeranges.get(jobID, None)) print(jobID, 'suite:', suites[jobID]) print(jobID, 'auto_download', auto_download[jobID]) @@ -620,6 +630,7 @@ def load_yml_and_run(compare_yml, config_user, skip_timeseries): analysisname=analysis_name, lineThicknesses=thicknesses, linestyles=linestyles, + labels=labels, config_user=config_user, ) diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index d063dfa8..e6aa75d0 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -297,6 +297,11 @@ def load_key_file(key, paths, jobID, strictFileCheck=True): output_dict['metrics'] = key_dict.get('metrics', metricList) output_dict['metrics'] = parse_list_from_string(output_dict['metrics']) + # Smoothings: + default_smoothings = ['DataOnly', ] + output_dict['smoothings'] = key_dict.get('smoothings', default_smoothings) + output_dict['smoothings'] = parse_list_from_string(output_dict['smoothings']) + # Load Grid: gridFile = key_dict.get('gridFile', paths.orcaGridfn) output_dict['gridFile'] = list_input_files(gridFile, key_dict, paths)[0] diff --git a/bgcval2/timeseries/timeseriesPlots.py b/bgcval2/timeseries/timeseriesPlots.py index 543849f4..aecaaca3 100644 --- a/bgcval2/timeseries/timeseriesPlots.py +++ b/bgcval2/timeseries/timeseriesPlots.py @@ -702,9 +702,10 @@ def multitimeseries( 'lime', ], plotStyle='Together', - lineStyle='', + smoothing='', thicknesses=defaultdict(lambda: 1), linestyles=defaultdict(lambda: '-'), + labels={}, ): if 0 in [len(timesD), len(list(timesD.keys()))]: return @@ -783,7 +784,8 @@ def multitimeseries( if np.min(arr) < ylims[0]: ylims[0] = np.min(arr) if np.max(arr) > ylims[1]: ylims[1] = np.max(arr) - if lineStyle.lower() in ['spline', 'all']: + label = labels.get(jobID, jobID) + if smoothing.lower() in ['spline', 'all']: tnew = np.linspace(times[0], times[-1], 60) arr_smooth = interpolate.spline(times, arr, tnew) pyplot.plot( @@ -792,10 +794,10 @@ def multitimeseries( c=colours[jobID], ls=linestyles[jobID], lw=thicknesses[jobID], - label=jobID + ' spline', + label=label + ' spline', ) - if lineStyle.lower() in ['movingaverage', 'both', 'all']: + if smoothing.lower() in ['movingaverage', 'both', 'all']: if len(times) > 100.: window = 30 elif len(times) > 30.: window = 15 elif len(times) > 10.: window = 4 @@ -810,10 +812,10 @@ def multitimeseries( c=colours[jobID], ls=linestyles[jobID], lw=thicknesses[jobID], - label=jobID, + label=label, ) - if lineStyle.lower() in [ + if smoothing.lower() in [ 'movingaverage5', ]: window = 5 @@ -827,11 +829,11 @@ def multitimeseries( arr_new, c=colours[jobID], ls=linestyles[jobID], - label=jobID, + label=label, lw=thicknesses[jobID], ) - if lineStyle.lower() in [ + if smoothing.lower() in [ 'movingav1year', ]: arr_new = movingaverage_DT(arr, @@ -844,10 +846,10 @@ def multitimeseries( c=colours[jobID], ls=linestyles[jobID], lw=thicknesses[jobID], - label=jobID, + label=label, ) - if lineStyle.lower() in [ - 'movingav5years', + if smoothing.lower() in [ + 'movingav5years', 'both5' ]: arr_new = movingaverage_DT(arr, times, @@ -859,11 +861,11 @@ def multitimeseries( c=colours[jobID], ls=linestyles[jobID], lw=thicknesses[jobID], - label=jobID, + label=label, ) - if lineStyle.lower() in [ - 'movingav30years', + if smoothing.lower() in [ + 'movingav30years', 'both30' ]: pyplot.plot(times, arr, @@ -881,11 +883,11 @@ def multitimeseries( c=colours[jobID], ls=linestyles[jobID], lw=thicknesses[jobID], - label=jobID, + label=label, ) - if lineStyle.lower() in [ - 'movingav100years', + if smoothing.lower() in [ + 'movingav100years', 'both100', ]: pyplot.plot(times, arr, @@ -903,10 +905,10 @@ def multitimeseries( c=colours[jobID], ls=linestyles[jobID], lw=thicknesses[jobID], - label=jobID, + label=label, ) - if lineStyle.lower() in [ + if smoothing.lower() in [ 'movingaverage12', ]: window = 12 @@ -921,10 +923,10 @@ def multitimeseries( c=colours[jobID], ls=linestyles[jobID], lw=2., - label=jobID, + label=label, ) - if lineStyle.lower() in [ + if smoothing.lower() in [ 'movingaverage60', ]: window = 60 @@ -939,13 +941,14 @@ def multitimeseries( c=colours[jobID], ls=linestyles[jobID], lw=2., - label=jobID, + label=label, ) - if lineStyle.lower() in [ + if smoothing.lower() in [ '', 'both', 'all', + 'both5', 'both30', 'both100', ]: pyplot.plot(times, arr, @@ -953,14 +956,14 @@ def multitimeseries( ls=linestyles[jobID], lw=0.25) - if lineStyle.lower() in ['dataonly']: + if smoothing.lower() in ['dataonly']: pyplot.plot( times, arr, c=colours[jobID], ls=linestyles[jobID], lw=thicknesses[jobID], - label=jobID, + label=label, ) if type(data) == type(10.): diff --git a/key_files/amoc_26n.yml b/key_files/amoc_26n.yml index 5b77b064..2d99d7a3 100644 --- a/key_files/amoc_26n.yml +++ b/key_files/amoc_26n.yml @@ -13,3 +13,4 @@ model_convert: altmaskfile: $PATHS_BGCVAL2/bgcval2/data/basinlandmask_eORCA1.nc layers: layerless regions: regionless +smoothings: DataOnly both5 both30 movingav30years From f54cb6c8411b4bbf7879dd20306b16e1751287da Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Thu, 2 Nov 2023 15:38:26 +0000 Subject: [PATCH 3/6] added more smoothing options --- bgcval2/timeseries/timeseriesPlots.py | 54 +++++++++++++++++++++++++++ key_files/amoc_26n.yml | 2 +- key_files/drakepassagetransport.yml | 2 + 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/bgcval2/timeseries/timeseriesPlots.py b/bgcval2/timeseries/timeseriesPlots.py index aecaaca3..217b8d21 100644 --- a/bgcval2/timeseries/timeseriesPlots.py +++ b/bgcval2/timeseries/timeseriesPlots.py @@ -833,6 +833,60 @@ def multitimeseries( lw=thicknesses[jobID], ) + if smoothing.lower() == '5and30': + arr_new5 = movingaverage_DT(arr, + times, + window_len=5., + window_units='years') + + arr_new30 = movingaverage_DT(arr, + times, + window_len=30., + window_units='years') + + pyplot.plot( + times, + arr_new5, + c=colours[jobID], + ls=linestyles[jobID], + lw=thicknesses[jobID]/2., + ) + pyplot.plot( + times, + arr_new30, + c=colours[jobID], + ls=linestyles[jobID], + lw=thicknesses[jobID], + label=label, + ) + if smoothing.lower() == '30and100': + arr_new5 = movingaverage_DT(arr, + times, + window_len=30., + window_units='years') + + arr_new30 = movingaverage_DT(arr, + times, + window_len=100., + window_units='years') + + pyplot.plot( + times, + arr_new5, + c=colours[jobID], + ls=linestyles[jobID], + lw=thicknesses[jobID]/2., + ) + pyplot.plot( + times, + arr_new30, + c=colours[jobID], + ls=linestyles[jobID], + lw=thicknesses[jobID], + label=label, + ) + + if smoothing.lower() in [ 'movingav1year', ]: diff --git a/key_files/amoc_26n.yml b/key_files/amoc_26n.yml index 2d99d7a3..9577ac97 100644 --- a/key_files/amoc_26n.yml +++ b/key_files/amoc_26n.yml @@ -13,4 +13,4 @@ model_convert: altmaskfile: $PATHS_BGCVAL2/bgcval2/data/basinlandmask_eORCA1.nc layers: layerless regions: regionless -smoothings: DataOnly both5 both30 movingav30years +smoothings: DataOnly both5 both30 movingav30years 5and30 30and100 diff --git a/key_files/drakepassagetransport.yml b/key_files/drakepassagetransport.yml index 776b572e..124f8b33 100644 --- a/key_files/drakepassagetransport.yml +++ b/key_files/drakepassagetransport.yml @@ -11,3 +11,5 @@ model_convert: areafile: $PATHS_GRIDFILE layers: layerless regions: regionless +smoothings: DataOnly 5and30 30and100 + From c28346d0e5809be9697941f197a97bf1fcbe6dbe Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 6 Nov 2023 08:11:55 +0000 Subject: [PATCH 4/6] Update bgcval2/timeseries/timeseriesPlots.py Co-authored-by: Valeriu Predoi --- bgcval2/timeseries/timeseriesPlots.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgcval2/timeseries/timeseriesPlots.py b/bgcval2/timeseries/timeseriesPlots.py index 217b8d21..c34115a2 100644 --- a/bgcval2/timeseries/timeseriesPlots.py +++ b/bgcval2/timeseries/timeseriesPlots.py @@ -702,7 +702,7 @@ def multitimeseries( 'lime', ], plotStyle='Together', - smoothing='', + smoothing='', thicknesses=defaultdict(lambda: 1), linestyles=defaultdict(lambda: '-'), labels={}, From e9dd4df4fc925d2067239ca8bd235ff2826d2878 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 6 Nov 2023 11:16:41 +0000 Subject: [PATCH 5/6] Added documentation --- README.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/README.md b/README.md index 1a05ee71..5d8c6c7b 100644 --- a/README.md +++ b/README.md @@ -229,6 +229,7 @@ strictFileCheck: jobs: : description: + label: 'Job info' colour: red thickness: 0.7 linestyle: '-' @@ -271,6 +272,10 @@ These values are: - The options are: - `description`: - A description of job, which helps people differentiate the jobs in the final report. + - `label`: + - A short description of the job which will appear in the plot legend. + - If you make it too long, it won't fit on the plot. + - Optional: Default behaviour is the jobID. - `colour`: - The colour of this jobs line in the report plots. - Default colour is a randomly generated hex colour. @@ -447,6 +452,8 @@ units: Units dimensions: 1,2 or 3 # The number of dimensions after the calculations are performed layers : Surface regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific +smoothings : DataOnly both5 both30 movingav30years 5and30 30and100 + #paths: modelFiles : $BASEDIR_MODEL/$JOBID/nemo_$JOBIDo_1y_*_grid-T.nc @@ -500,6 +507,26 @@ basic functions such as multiply by or add to, or `noChange`, which can all be called without providing the `path`, and which may have their own key word arguments. +Some of the other options for each analysis include: + - `layers`: + - The z axis layers that you want to use. + - Typically include things like `Surface`, `500m`, `1000m` etc. + - Note that this is a pre-curated list, so you can't use it to select a specific depth (like `327m` or something) + - `regions`: + - A list of regions to investigate. + - This is a pre-curated list, and they are defined in so you can't. + - These regions are defined in the file `bgcval2/bgcvaltools/makeMask.py`. + - `smoothings`: + - This is the smoothing function (if any) to apply to the data before plotting it. + - The smoothing it added before plotting, but after saving the shelve file, so it doesn't impact the data. + - No smoothing is `DataOnly`, which is also the default behaviour. + - If several smoothing options are added here, bgcval2 will generate a comparison plot for each one. + - Other modes exist: + - `movingav30years`: A 30 year moving average + - `both5`: Both no smoothing and a 5 year moving average + - `both30`: Both no smoothing and a 30 year moving average + - `5and30`: a 5 year moving average and a 30 year moving average. + Clearing the Cache ------------------ From 39661765298f976e04dc549e872e97567d6db9c4 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 6 Nov 2023 11:25:12 +0000 Subject: [PATCH 6/6] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 5d8c6c7b..3c9e8aee 100644 --- a/README.md +++ b/README.md @@ -521,6 +521,7 @@ Some of the other options for each analysis include: - The smoothing it added before plotting, but after saving the shelve file, so it doesn't impact the data. - No smoothing is `DataOnly`, which is also the default behaviour. - If several smoothing options are added here, bgcval2 will generate a comparison plot for each one. + - The smoothings are defined and performed in `bgcval2/timeseries/timeseriesPlots.py` - Other modes exist: - `movingav30years`: A 30 year moving average - `both5`: Both no smoothing and a 5 year moving average