From 947c2ae6bb303e04f4997347e45f21e5ca50babc Mon Sep 17 00:00:00 2001 From: Lee de Mora <ledm@pml.ac.uk> Date: Wed, 27 Mar 2024 11:04:17 +0000 Subject: [PATCH 01/14] Added output as json files option --- bgcval2/analysis_compare.py | 32 +++++++++- bgcval2/timeseries/timeseriesPlots.py | 3 + bgcval2/timeseries/timeseriesTools.py | 68 +++++++++++++++++++++ input_yml/TerraFIRMA_overshoot_recovery.yml | 3 + input_yml/TerraFIRMA_overshoot_runs.yml | 3 + input_yml/TerraFIRMA_overshoot_stables.yml | 3 + 6 files changed, 110 insertions(+), 2 deletions(-) diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index ac7d89f4..f11186a2 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -194,6 +194,7 @@ def timeseries_compare(jobs, config_user=None, dpi=None, savepdf=False, + savecsv=False, ): """ timeseries_compare: @@ -225,6 +226,7 @@ def timeseries_compare(jobs, sys.exit(0) else: imageFolder = paths.imagedir + '/TimeseriesCompare/' + analysisname + csvFolder = paths.imagedir + '/TimeseriesCompare_CSV/' + analysisname annual = True strictFileCheck = False @@ -409,6 +411,25 @@ def timeseries_compare(jobs, units = av[name]['modeldetails']['units'] ts = 'Together' + + if savecsv: + tst.save_csv( + timesD, + arrD, + analysisname, + colours=colours, + linestyles=linestyles, + thicknesses=lineThicknesses, + labels=labels, + name=name, + units=units, + region=region, + layer=layer, + metric=metric, + title=title, + ts=ts, + csvFolder=csvFolder) + for smoothing in smoothings: #or ls in ['DataOnly', ]: tsp.multitimeseries( @@ -417,7 +438,7 @@ def timeseries_compare(jobs, data=-999, # in situ data distribution title=title, filename=bvt.folder(imageFolder) + - '_'.join([name, region, layer, ts, smoothing + '.png']), + '_'.join([name, region, layer, metric, ts, smoothing + '.png']), units=units, plotStyle=ts, smoothing=smoothing, @@ -427,8 +448,10 @@ def timeseries_compare(jobs, labels=labels, dpi=dpi, savepdf=savepdf, + savecsv=savecsv, ) + # Generate a list of comparison images: method_images = 'oswalk' AllImages = [] @@ -512,8 +535,12 @@ def load_comparison_yml(master_compare_yml_fn): # Image output settings: # dpi: pixels per inch (image resolution) # savepdf: also save the image as a pdf. + # savecsv: Save the data that appears in the image. details['dpi'] = input_yml_dict.get('dpi', None) details['savepdf'] = input_yml_dict.get('savepdf', False) + details['savecsv'] = input_yml_dict.get('savecsv', False) + + if details['dpi']: # None is valid! try: @@ -555,7 +582,6 @@ def load_comparison_yml(master_compare_yml_fn): auto_download_dict[jobID] = job_dict.get('auto_download', auto_download_dict[jobID]) labels[jobID] = job_dict.get('label', jobID) - details['colours'] = colours details['descriptions'] = descriptions details['thicknesses'] = thicknesses @@ -599,6 +625,7 @@ def load_yml_and_run(compare_yml, config_user, skip_timeseries): strictFileCheck = details.get('strictFileCheck', True) dpi = details.get('dpi', None) savepdf = details.get('savepdf', False) + savecsv = details.get('savecsv', False) print('---------------------') print('timeseries_compare:', analysis_name) @@ -655,6 +682,7 @@ def load_yml_and_run(compare_yml, config_user, skip_timeseries): config_user=config_user, dpi=dpi, savepdf=savepdf, + savecsv=savecsv, ) diff --git a/bgcval2/timeseries/timeseriesPlots.py b/bgcval2/timeseries/timeseriesPlots.py index 0afa7d67..afee9568 100644 --- a/bgcval2/timeseries/timeseriesPlots.py +++ b/bgcval2/timeseries/timeseriesPlots.py @@ -710,6 +710,7 @@ def multitimeseries( labels={}, dpi=None, savepdf=False, + savecsv=False, ): if 0 in [len(timesD), len(list(timesD.keys()))]: return @@ -751,6 +752,8 @@ def multitimeseries( final_labels = [] + + for i, jobID in enumerate(sorted(timesD.keys())): times = timesD[jobID] arr = arrD[jobID] diff --git a/bgcval2/timeseries/timeseriesTools.py b/bgcval2/timeseries/timeseriesTools.py index 319511b2..b1f47910 100644 --- a/bgcval2/timeseries/timeseriesTools.py +++ b/bgcval2/timeseries/timeseriesTools.py @@ -33,6 +33,9 @@ from bgcval2.bgcvaltools.dataset import dataset from bgcval2.bgcvaltools.makeMask import makeMask from bgcval2.functions.standard_functions import extractData as std_extractData +import json +from jsondiff import diff as jsondiff + """ .. module:: timeseriesTools @@ -254,6 +257,71 @@ def getHorizontalSlice(nc, coords, details, layer, data=''): assert 0 +def save_csv( + timesD, + arrD, + analysisname, + colours={}, + linestyles={}, + thicknesses={}, + labels={}, + name='', + units='', + region='', + layer='', + metric='', + title='', + ts='', + csvFolder='', + csvformat='.json', + ): + """ + Output the data that appears in a plot as a csv. + + """ + if csvformat.lower() not in ['json', '.json']: + print('Format not set up', csvformat ) + return + + filename = bvt.folder(csvFolder) + '_'.join([analysisname, name, region, layer, metric, ts ]) + csvformat + + jsondata = { + 'timesD': timesD, + 'arrD': arrD, + 'analysisname': analysisname, + 'colours':colours, + 'linestyles':linestyles, + 'thicknesses':lineThicknesses, + 'labels':labels, + 'name':name, + 'units':units, + 'region':region, + 'layer':layer, + 'metric':metric, + 'title':title, + 'analysisname': analysisname, + 'ts':ts, + 'filename':filename, + } + + if os.path.exists(filename): + with open(filename) as json_data: + json_old = json.load(json_data) + + diff = jsondiff(jsondata, json_old) + if not len(diff): + # Nothing new to add + return + + with open(filename, 'w', encoding='utf-8') as f: + json.dump(jsondata, f, ensure_ascii=False, indent=4) + + + + + + + class DataLoader: def __init__(self, diff --git a/input_yml/TerraFIRMA_overshoot_recovery.yml b/input_yml/TerraFIRMA_overshoot_recovery.yml index 87aa0a3d..b4168f55 100644 --- a/input_yml/TerraFIRMA_overshoot_recovery.yml +++ b/input_yml/TerraFIRMA_overshoot_recovery.yml @@ -14,6 +14,9 @@ master_suites: physics bgc kmf #alkalinity physics kmf1 # Run without strick check (if True, breaks if job has no years.) strict_file_check: False +# Output the figures as csv json files. +savecsv: True + clean: True jobs: diff --git a/input_yml/TerraFIRMA_overshoot_runs.yml b/input_yml/TerraFIRMA_overshoot_runs.yml index e549b099..1765fb0b 100644 --- a/input_yml/TerraFIRMA_overshoot_runs.yml +++ b/input_yml/TerraFIRMA_overshoot_runs.yml @@ -14,6 +14,9 @@ master_suites: physics bgc kmf #alkalinity physics kmf1 # Run without strick check (if True, breaks if job has no years.) strict_file_check: False +# Output the figures as csv json files. +savecsv: True + clean: True diff --git a/input_yml/TerraFIRMA_overshoot_stables.yml b/input_yml/TerraFIRMA_overshoot_stables.yml index 25975990..53379031 100644 --- a/input_yml/TerraFIRMA_overshoot_stables.yml +++ b/input_yml/TerraFIRMA_overshoot_stables.yml @@ -14,6 +14,9 @@ master_suites: physics bgc kmf #alkalinity physics kmf1 # Run without strick check (if True, breaks if job has no years.) strict_file_check: False +# Output the figures as csv json files. +savecsv: True + clean: True From 21942f05331774a23d8734b3852bad50dc56f7dc Mon Sep 17 00:00:00 2001 From: Lee de Mora <ledm@pml.ac.uk> Date: Wed, 27 Mar 2024 11:31:05 +0000 Subject: [PATCH 02/14] debugging on jasmin --- bgcval2/analysis_compare.py | 2 ++ bgcval2/timeseries/timeseriesTools.py | 10 +++------- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index f11186a2..d459bc46 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -61,6 +61,8 @@ from .timeseries import timeseriesAnalysis from .timeseries import profileAnalysis from .timeseries import timeseriesPlots as tsp +from .timeseries import timeseriesTools as tst + from bgcval2.analysis_timeseries import analysis_timeseries, build_list_of_suite_keys, load_key_file from bgcval2.download_from_mass import download_from_mass diff --git a/bgcval2/timeseries/timeseriesTools.py b/bgcval2/timeseries/timeseriesTools.py index b1f47910..8670d43d 100644 --- a/bgcval2/timeseries/timeseriesTools.py +++ b/bgcval2/timeseries/timeseriesTools.py @@ -291,7 +291,7 @@ def save_csv( 'analysisname': analysisname, 'colours':colours, 'linestyles':linestyles, - 'thicknesses':lineThicknesses, + 'thicknesses':thicknesses, 'labels':labels, 'name':name, 'units':units, @@ -310,16 +310,12 @@ def save_csv( diff = jsondiff(jsondata, json_old) if not len(diff): - # Nothing new to add + print('Nothing new to add', filename) return with open(filename, 'w', encoding='utf-8') as f: json.dump(jsondata, f, ensure_ascii=False, indent=4) - - - - - + print('Data saved in: ', filename) class DataLoader: From 45120829de9f8d2edc4c3a8012c16c8eec0a31be Mon Sep 17 00:00:00 2001 From: Lee de Mora <ledm@pml.ac.uk> Date: Wed, 27 Mar 2024 14:11:05 +0000 Subject: [PATCH 03/14] Update TerraFIRMA_overshoot_historical.yml --- input_yml/TerraFIRMA_overshoot_historical.yml | 38 ++++++++++++++++--- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/input_yml/TerraFIRMA_overshoot_historical.yml b/input_yml/TerraFIRMA_overshoot_historical.yml index 9f1cb965..629ab9c9 100644 --- a/input_yml/TerraFIRMA_overshoot_historical.yml +++ b/input_yml/TerraFIRMA_overshoot_historical.yml @@ -13,14 +13,15 @@ master_suites: physics bgc #alkalinity physics kmf1 clean: True + jobs: u-ca306: description: 'Reference ' - colour: 'black' + colour: 'blue' thickness: 0.6 linestyle: '-' shifttime: 0. - timerange: [1800, 1900] + timerange: [1850, 2020] suite: kmf physics bgc #alkalinity physics u-cy623: @@ -31,9 +32,36 @@ jobs: shifttime: 0. suite: kmf physics bgc #alkalinity physics + + u-da914: + description: 'interactive ice, started from picontrol yr 2197' + colour: red + thickness: 1.7 + linestyle: '-' + shifttime: 0. + suite: kmf physics bgc #alkalinity physics + + u-da916: + description: 'interactive ice, started from picontrol yr 2237' + colour: red + thickness: 1.7 + linestyle: '-' + shifttime: 0. + suite: kmf physics bgc #alkalinity physics + + u-da917: + description: 'interactive ice, started from picontrol yr 2317' + colour: red + thickness: 1.7 + linestyle: '-' + shifttime: 0. + suite: kmf physics bgc #alkalinity physics + + + u-cy690: description: 'static ice sheets, started from picontrol yr 2277' - colour: purple + colour: blue thickness: 1.7 linestyle: '-' shifttime: 0. @@ -49,7 +77,7 @@ jobs: u-cy692: description: ' static ice sheets, started from picontrol yr 2237' - colour: green + colour: blue thickness: 1.7 linestyle: '-' shifttime: 0. @@ -57,7 +85,7 @@ jobs: u-cy693: description: ' static ice sheets, started from picontrol yr 2317' - colour: goldenrod + colour: blue thickness: 1.7 linestyle: '-' shifttime: 0. From 82d9b9c5b362ffbda4ca008905ac090b447f21fd Mon Sep 17 00:00:00 2001 From: Lee de Mora <ledm@pml.ac.uk> Date: Wed, 27 Mar 2024 14:14:27 +0000 Subject: [PATCH 04/14] Update TerraFIRMA_overshoot_historical.yml --- input_yml/TerraFIRMA_overshoot_historical.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/input_yml/TerraFIRMA_overshoot_historical.yml b/input_yml/TerraFIRMA_overshoot_historical.yml index 629ab9c9..802675cd 100644 --- a/input_yml/TerraFIRMA_overshoot_historical.yml +++ b/input_yml/TerraFIRMA_overshoot_historical.yml @@ -13,6 +13,8 @@ master_suites: physics bgc #alkalinity physics kmf1 clean: True +# Output the figures as csv json files. +savecsv: True jobs: u-ca306: From 460e19991d9f5cee4e9db81ce3f7d46e59b53408 Mon Sep 17 00:00:00 2001 From: Lee de Mora <ledm@pml.ac.uk> Date: Wed, 27 Mar 2024 14:17:50 +0000 Subject: [PATCH 05/14] Update TerraFIRMA_overshoot_historical.yml --- input_yml/TerraFIRMA_overshoot_historical.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/input_yml/TerraFIRMA_overshoot_historical.yml b/input_yml/TerraFIRMA_overshoot_historical.yml index 802675cd..da98cd50 100644 --- a/input_yml/TerraFIRMA_overshoot_historical.yml +++ b/input_yml/TerraFIRMA_overshoot_historical.yml @@ -19,6 +19,7 @@ savecsv: True jobs: u-ca306: description: 'Reference ' + label: PiControl colour: 'blue' thickness: 0.6 linestyle: '-' @@ -28,6 +29,7 @@ jobs: u-cy623: description: 'interactive ice, started from picontrol yr 2277' + label: 'hist ice' colour: red thickness: 1.7 linestyle: '-' @@ -37,6 +39,7 @@ jobs: u-da914: description: 'interactive ice, started from picontrol yr 2197' + label: None colour: red thickness: 1.7 linestyle: '-' @@ -45,6 +48,7 @@ jobs: u-da916: description: 'interactive ice, started from picontrol yr 2237' + label: None colour: red thickness: 1.7 linestyle: '-' @@ -53,6 +57,7 @@ jobs: u-da917: description: 'interactive ice, started from picontrol yr 2317' + label: None colour: red thickness: 1.7 linestyle: '-' @@ -64,6 +69,7 @@ jobs: u-cy690: description: 'static ice sheets, started from picontrol yr 2277' colour: blue + label: 'hist static ice' thickness: 1.7 linestyle: '-' shifttime: 0. @@ -71,6 +77,7 @@ jobs: u-cy691: description: 'static ice sheets, started from picontrol yr 2197' + label: None colour: blue thickness: 1.7 linestyle: '-' @@ -79,6 +86,7 @@ jobs: u-cy692: description: ' static ice sheets, started from picontrol yr 2237' + label: None colour: blue thickness: 1.7 linestyle: '-' @@ -87,6 +95,7 @@ jobs: u-cy693: description: ' static ice sheets, started from picontrol yr 2317' + label: None colour: blue thickness: 1.7 linestyle: '-' From 0dc82e0ad5f5ea66fea3ded2f0ba4fdd664ccd52 Mon Sep 17 00:00:00 2001 From: Lee de Mora <ledm@pml.ac.uk> Date: Thu, 28 Mar 2024 10:53:45 +0000 Subject: [PATCH 06/14] new co2 tests run --- bgcval2/timeseries/timeseriesTools.py | 10 +-- input_yml/TerraFIRMA_overshoot_co2tests.yml | 68 +++++++++++++++++++++ 2 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 input_yml/TerraFIRMA_overshoot_co2tests.yml diff --git a/bgcval2/timeseries/timeseriesTools.py b/bgcval2/timeseries/timeseriesTools.py index 8670d43d..7df0bb25 100644 --- a/bgcval2/timeseries/timeseriesTools.py +++ b/bgcval2/timeseries/timeseriesTools.py @@ -286,8 +286,9 @@ def save_csv( filename = bvt.folder(csvFolder) + '_'.join([analysisname, name, region, layer, metric, ts ]) + csvformat jsondata = { - 'timesD': timesD, - 'arrD': arrD, + # json can't save numpy.float32, so we convert to list of floats. + 'timesD': {job:[float(t) for t in times] for job, times in timesD.items()}, + 'arrD': {job:[float(d) for d in data] for job, data in arrD.items()}, 'analysisname': analysisname, 'colours':colours, 'linestyles':linestyles, @@ -305,17 +306,18 @@ def save_csv( } if os.path.exists(filename): + print('Opening old file:', filename) with open(filename) as json_data: json_old = json.load(json_data) diff = jsondiff(jsondata, json_old) if not len(diff): - print('Nothing new to add', filename) + print('Nothing new to add') return + print('Data saved in: ', filename) with open(filename, 'w', encoding='utf-8') as f: json.dump(jsondata, f, ensure_ascii=False, indent=4) - print('Data saved in: ', filename) class DataLoader: diff --git a/input_yml/TerraFIRMA_overshoot_co2tests.yml b/input_yml/TerraFIRMA_overshoot_co2tests.yml new file mode 100644 index 00000000..42887cc0 --- /dev/null +++ b/input_yml/TerraFIRMA_overshoot_co2tests.yml @@ -0,0 +1,68 @@ +--- +name: TerraFIRMA_overshoot_co2tests + +# Run the single job analysis +do_analysis_timeseries: True + +# Download from mass: +do_mass_download: False + +# master analysis suite +master_suites: physics bgc #alkalinity physics kmf1 + +clean: True + +# Output the figures as csv json files. +savecsv: True + +jobs: + u-ca306: + description: 'Reference ' + label: PiControl + colour: 'blue' + thickness: 0.6 + linestyle: '-' + shifttime: 0. + timerange: [1850, 2020] + suite: kmf physics bgc #alkalinity physics + + u-cz014: + description: 'Terrafirma 4x CO2' + label: '4xCO2' + colour: 'green' + thickness: 1.7 + linestyle: '-' + shifttime: 0. + suite: kmf physics bgc #alkalinity physics + + u-cz014: + description: 'Terrafirma 1% CO2' + label: '1%CO2' + colour: 'purple' + thickness: 1.7 + linestyle: '-' + shifttime: 0. + suite: kmf physics bgc #alkalinity physics + + + u-cy623: + description: 'interactive ice, started from picontrol yr 2277' + label: 'hist ice' + colour: 'orange' + thickness: 1.7 + linestyle: 'dashdot' + shifttime: 0. + suite: kmf physics bgc #alkalinity physics + + + u-cy690: + description: 'static ice sheets, started from picontrol yr 2277' + colour: 'red' + label: 'hist static ice' + thickness: 1.7 + linestyle: ':' + shifttime: 0. + suite: kmf physics bgc #alkalinity physics + + + From 2250faf9060a8f0db0cd0ed7c76470705fa9c114 Mon Sep 17 00:00:00 2001 From: Lee de Mora <ledm@pml.ac.uk> Date: Thu, 28 Mar 2024 16:36:13 +0000 Subject: [PATCH 07/14] Added new Atmospheric CO2 field to kmf and elsewhere. --- bgcval2/bgcval2_make_report.py | 1 + bgcval2/bgcvaltools/pftnames.py | 3 +++ input_yml/TerraFIRMA_overshoot_co2tests.yml | 6 +++--- input_yml/TerraFIRMA_overshoot_historical.yml | 2 +- key_files/atmosco2.yml | 12 ++++++++++++ key_lists/atmosco2.yml | 3 +++ key_lists/kmf.yml | 1 + 7 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 key_files/atmosco2.yml create mode 100644 key_lists/atmosco2.yml diff --git a/bgcval2/bgcval2_make_report.py b/bgcval2/bgcval2_make_report.py index 325ed2e8..290c40d1 100755 --- a/bgcval2/bgcval2_make_report.py +++ b/bgcval2/bgcval2_make_report.py @@ -1480,6 +1480,7 @@ def newImageLocation(fn): physicsKM = [ 'AMOC_26N', 'ADRC_26N', + 'AtmosCO2', 'DrakePassage', 'GlobalMeanTemperature', 'GlobalMeanSalinity', diff --git a/bgcval2/bgcvaltools/pftnames.py b/bgcval2/bgcvaltools/pftnames.py index 1e8a3d73..1fa0e6e9 100644 --- a/bgcval2/bgcvaltools/pftnames.py +++ b/bgcval2/bgcvaltools/pftnames.py @@ -270,6 +270,9 @@ def makeLongNameDict(): lnd['PCO2_SW'] = 'pCO2' lnd['pCO2'] = 'pCO2' + lnd['AtmospCO2'] = 'pCO2' + lnd['AtmosCO2'] = 'Atmospheric CO2' + lnd['iron'] = "Iron" lnd['Fe_D_CONC_BOTTLE'] = "Iron (Dissolved)" diff --git a/input_yml/TerraFIRMA_overshoot_co2tests.yml b/input_yml/TerraFIRMA_overshoot_co2tests.yml index 42887cc0..f01c937b 100644 --- a/input_yml/TerraFIRMA_overshoot_co2tests.yml +++ b/input_yml/TerraFIRMA_overshoot_co2tests.yml @@ -16,13 +16,13 @@ clean: True savecsv: True jobs: - u-ca306: + u-cs495: description: 'Reference ' label: PiControl colour: 'blue' thickness: 0.6 linestyle: '-' - shifttime: 0. + shifttime: -427. timerange: [1850, 2020] suite: kmf physics bgc #alkalinity physics @@ -35,7 +35,7 @@ jobs: shifttime: 0. suite: kmf physics bgc #alkalinity physics - u-cz014: + u-cz152: description: 'Terrafirma 1% CO2' label: '1%CO2' colour: 'purple' diff --git a/input_yml/TerraFIRMA_overshoot_historical.yml b/input_yml/TerraFIRMA_overshoot_historical.yml index da98cd50..bc046787 100644 --- a/input_yml/TerraFIRMA_overshoot_historical.yml +++ b/input_yml/TerraFIRMA_overshoot_historical.yml @@ -23,7 +23,7 @@ jobs: colour: 'blue' thickness: 0.6 linestyle: '-' - shifttime: 0. + shifttime: 0 timerange: [1850, 2020] suite: kmf physics bgc #alkalinity physics diff --git a/key_files/atmosco2.yml b/key_files/atmosco2.yml new file mode 100644 index 00000000..9c3e9156 --- /dev/null +++ b/key_files/atmosco2.yml @@ -0,0 +1,12 @@ +--- +name : AtmosCO2 +units : ppm +model : MEDUSA +modelgrid : eORCA1 +dimensions : 2 +modelFiles : $BASEDIR_MODEL/$JOBID/medusa*$JOBIDo_1y_*_diad-T.nc +model_vars : ATM_XCO2 +model_convert : NoChange +layers : layerless +regions : Global #gnoreInlandSeas SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific + diff --git a/key_lists/atmosco2.yml b/key_lists/atmosco2.yml new file mode 100644 index 00000000..64b1150f --- /dev/null +++ b/key_lists/atmosco2.yml @@ -0,0 +1,3 @@ +--- +keys: + atmosco2: True diff --git a/key_lists/kmf.yml b/key_lists/kmf.yml index 08525c8f..b7e2da93 100644 --- a/key_lists/kmf.yml +++ b/key_lists/kmf.yml @@ -7,6 +7,7 @@ keys: SST: True DrakePassageTransport: True AMOC_26N: True + atmosco2: True TotalAirSeaFluxCO2: True # NoCaspianAirSeaFluxCO2: True # IntPP_OSU: True From 25e5bec42a8b8532bf2bf35fbcaec524eeb6dda1 Mon Sep 17 00:00:00 2001 From: Lee de Mora <ledm@pml.ac.uk> Date: Thu, 28 Mar 2024 16:53:05 +0000 Subject: [PATCH 08/14] changed capitals --- key_lists/kmf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/key_lists/kmf.yml b/key_lists/kmf.yml index b7e2da93..6ec82ec4 100644 --- a/key_lists/kmf.yml +++ b/key_lists/kmf.yml @@ -7,7 +7,7 @@ keys: SST: True DrakePassageTransport: True AMOC_26N: True - atmosco2: True + AtmosCO2: True TotalAirSeaFluxCO2: True # NoCaspianAirSeaFluxCO2: True # IntPP_OSU: True From 462dd3607a68d5b6ec22f04fc53a545aea9a31bc Mon Sep 17 00:00:00 2001 From: Lee de Mora <ledm@pml.ac.uk> Date: Fri, 19 Apr 2024 10:36:05 +0100 Subject: [PATCH 09/14] added co2 back and changed line colors --- input_yml/TerraFIRMA_overshoot_co2tests.yml | 2 +- input_yml/TerraFIRMA_overshoot_historical.yml | 24 ++++++++++++------- key_files/mld.yml | 2 +- key_lists/physics.yml | 3 +++ 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/input_yml/TerraFIRMA_overshoot_co2tests.yml b/input_yml/TerraFIRMA_overshoot_co2tests.yml index f01c937b..95d2969a 100644 --- a/input_yml/TerraFIRMA_overshoot_co2tests.yml +++ b/input_yml/TerraFIRMA_overshoot_co2tests.yml @@ -8,7 +8,7 @@ do_analysis_timeseries: True do_mass_download: False # master analysis suite -master_suites: physics bgc #alkalinity physics kmf1 +master_suites: physics bgc kmf #alkalinity physics kmf1 clean: True diff --git a/input_yml/TerraFIRMA_overshoot_historical.yml b/input_yml/TerraFIRMA_overshoot_historical.yml index bc046787..a6edb976 100644 --- a/input_yml/TerraFIRMA_overshoot_historical.yml +++ b/input_yml/TerraFIRMA_overshoot_historical.yml @@ -17,12 +17,22 @@ clean: True savecsv: True jobs: - u-ca306: - description: 'Reference ' + u-cs495: + description: 'PiControl' label: PiControl colour: 'blue' thickness: 0.6 linestyle: '-' + shifttime: -427. + timerange: [1850, 2020] + suite: kmf physics bgc #alkalinity physics + + u-ca306: + description: 'Reference ' + label: Reference + colour: 'black' + thickness: 0.6 + linestyle: '-' shifttime: 0 timerange: [1850, 2020] suite: kmf physics bgc #alkalinity physics @@ -64,11 +74,9 @@ jobs: shifttime: 0. suite: kmf physics bgc #alkalinity physics - - u-cy690: description: 'static ice sheets, started from picontrol yr 2277' - colour: blue + colour: green label: 'hist static ice' thickness: 1.7 linestyle: '-' @@ -78,7 +86,7 @@ jobs: u-cy691: description: 'static ice sheets, started from picontrol yr 2197' label: None - colour: blue + colour: green thickness: 1.7 linestyle: '-' shifttime: 0. @@ -87,7 +95,7 @@ jobs: u-cy692: description: ' static ice sheets, started from picontrol yr 2237' label: None - colour: blue + colour: green thickness: 1.7 linestyle: '-' shifttime: 0. @@ -96,7 +104,7 @@ jobs: u-cy693: description: ' static ice sheets, started from picontrol yr 2317' label: None - colour: blue + colour: green thickness: 1.7 linestyle: '-' shifttime: 0. diff --git a/key_files/mld.yml b/key_files/mld.yml index 4c8c1eef..e6d04417 100644 --- a/key_files/mld.yml +++ b/key_files/mld.yml @@ -30,4 +30,4 @@ data_convert: maskname : mask areafile: $BASEDIR_OBS/IFREMER-MLD/mld_DT02_c1m_reg2.0-annual.nc #layers : Surface -regions : Global OnShelf OffShelf ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthAtlanticOcean SouthAtlanticOcean NorthPacificOcean SouthPacificOcean SPNA +regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthAtlanticOcean SouthAtlanticOcean NorthPacificOcean SouthPacificOcean SPNA diff --git a/key_lists/physics.yml b/key_lists/physics.yml index 34f51bf2..a142d1bb 100644 --- a/key_lists/physics.yml +++ b/key_lists/physics.yml @@ -17,6 +17,9 @@ keys: Salinity: True # WOA Salinity # soga: True + # CO2: + AtmosCO2: True + # Mixed layer depth keys: MLD: True # iFERMER Mixed Layer Depth # MaxMonthlyMLD: True # MLD Monthly max From 6489168d9509a363214ee1910f2cfb9a2115f2e0 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi <valeriu.predoi@gmail.com> Date: Fri, 19 Apr 2024 16:16:18 +0100 Subject: [PATCH 10/14] add jsondiff to pkg requirements --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index 3a45b5a3..3a60e75c 100644 --- a/environment.yml +++ b/environment.yml @@ -7,6 +7,7 @@ channels: dependencies: - basemap >=1.3.6 - cartopy + - jsondiff - matplotlib - nctoolkit >=0.8.7 # use linux64 build - netcdf4 From 3564f3ac2d6fc15f09e94ad48f9c6f4da09b0380 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi <valeriu.predoi@gmail.com> Date: Fri, 19 Apr 2024 16:16:24 +0100 Subject: [PATCH 11/14] add jsondiff to pkg requirements --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 134b294f..377b4674 100755 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ 'install': [ 'basemap>=1.3.6', 'cartopy', + 'jsondiff', 'matplotlib', 'nctoolkit>=0.8.7', # use linux64 build 'netcdf4', From 6f816cdc2653ea1a5ae37702f642af0d25808ca5 Mon Sep 17 00:00:00 2001 From: Lee de Mora <ledm@pml.ac.uk> Date: Wed, 24 Apr 2024 16:01:22 +0100 Subject: [PATCH 12/14] renamed savecsv to more accurate savejson --- bgcval2/analysis_compare.py | 15 +++++++-------- bgcval2/timeseries/timeseriesPlots.py | 1 - bgcval2/timeseries/timeseriesTools.py | 4 ++-- input_yml/TerraFIRMA_overshoot_co2tests.yml | 2 +- input_yml/TerraFIRMA_overshoot_historical.yml | 2 +- input_yml/TerraFIRMA_overshoot_recovery.yml | 2 +- input_yml/TerraFIRMA_overshoot_runs.yml | 2 +- input_yml/TerraFIRMA_overshoot_stables.yml | 2 +- 8 files changed, 14 insertions(+), 16 deletions(-) diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index d459bc46..7f94ac56 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -196,7 +196,7 @@ def timeseries_compare(jobs, config_user=None, dpi=None, savepdf=False, - savecsv=False, + savejson=False, ): """ timeseries_compare: @@ -414,8 +414,8 @@ def timeseries_compare(jobs, ts = 'Together' - if savecsv: - tst.save_csv( + if savejson: + tst.save_json( timesD, arrD, analysisname, @@ -450,7 +450,6 @@ def timeseries_compare(jobs, labels=labels, dpi=dpi, savepdf=savepdf, - savecsv=savecsv, ) @@ -537,10 +536,10 @@ def load_comparison_yml(master_compare_yml_fn): # Image output settings: # dpi: pixels per inch (image resolution) # savepdf: also save the image as a pdf. - # savecsv: Save the data that appears in the image. + # savejson: Save the data that appears in the image. details['dpi'] = input_yml_dict.get('dpi', None) details['savepdf'] = input_yml_dict.get('savepdf', False) - details['savecsv'] = input_yml_dict.get('savecsv', False) + details['savejson'] = input_yml_dict.get('savejson', False) @@ -627,7 +626,7 @@ def load_yml_and_run(compare_yml, config_user, skip_timeseries): strictFileCheck = details.get('strictFileCheck', True) dpi = details.get('dpi', None) savepdf = details.get('savepdf', False) - savecsv = details.get('savecsv', False) + savejson = details.get('savejson', False) print('---------------------') print('timeseries_compare:', analysis_name) @@ -684,7 +683,7 @@ def load_yml_and_run(compare_yml, config_user, skip_timeseries): config_user=config_user, dpi=dpi, savepdf=savepdf, - savecsv=savecsv, + savejson=savejson, ) diff --git a/bgcval2/timeseries/timeseriesPlots.py b/bgcval2/timeseries/timeseriesPlots.py index afee9568..44a68e5d 100644 --- a/bgcval2/timeseries/timeseriesPlots.py +++ b/bgcval2/timeseries/timeseriesPlots.py @@ -710,7 +710,6 @@ def multitimeseries( labels={}, dpi=None, savepdf=False, - savecsv=False, ): if 0 in [len(timesD), len(list(timesD.keys()))]: return diff --git a/bgcval2/timeseries/timeseriesTools.py b/bgcval2/timeseries/timeseriesTools.py index 7df0bb25..59cab227 100644 --- a/bgcval2/timeseries/timeseriesTools.py +++ b/bgcval2/timeseries/timeseriesTools.py @@ -257,7 +257,7 @@ def getHorizontalSlice(nc, coords, details, layer, data=''): assert 0 -def save_csv( +def save_json( timesD, arrD, analysisname, @@ -276,7 +276,7 @@ def save_csv( csvformat='.json', ): """ - Output the data that appears in a plot as a csv. + Output the data that appears in a plot as a json file. """ if csvformat.lower() not in ['json', '.json']: diff --git a/input_yml/TerraFIRMA_overshoot_co2tests.yml b/input_yml/TerraFIRMA_overshoot_co2tests.yml index 95d2969a..62578012 100644 --- a/input_yml/TerraFIRMA_overshoot_co2tests.yml +++ b/input_yml/TerraFIRMA_overshoot_co2tests.yml @@ -13,7 +13,7 @@ master_suites: physics bgc kmf #alkalinity physics kmf1 clean: True # Output the figures as csv json files. -savecsv: True +savejson: True jobs: u-cs495: diff --git a/input_yml/TerraFIRMA_overshoot_historical.yml b/input_yml/TerraFIRMA_overshoot_historical.yml index a6edb976..60cba4e8 100644 --- a/input_yml/TerraFIRMA_overshoot_historical.yml +++ b/input_yml/TerraFIRMA_overshoot_historical.yml @@ -14,7 +14,7 @@ master_suites: physics bgc #alkalinity physics kmf1 clean: True # Output the figures as csv json files. -savecsv: True +savejson: True jobs: u-cs495: diff --git a/input_yml/TerraFIRMA_overshoot_recovery.yml b/input_yml/TerraFIRMA_overshoot_recovery.yml index b4168f55..1c4072b7 100644 --- a/input_yml/TerraFIRMA_overshoot_recovery.yml +++ b/input_yml/TerraFIRMA_overshoot_recovery.yml @@ -15,7 +15,7 @@ master_suites: physics bgc kmf #alkalinity physics kmf1 strict_file_check: False # Output the figures as csv json files. -savecsv: True +savejson: True clean: True diff --git a/input_yml/TerraFIRMA_overshoot_runs.yml b/input_yml/TerraFIRMA_overshoot_runs.yml index 1765fb0b..104c28f7 100644 --- a/input_yml/TerraFIRMA_overshoot_runs.yml +++ b/input_yml/TerraFIRMA_overshoot_runs.yml @@ -15,7 +15,7 @@ master_suites: physics bgc kmf #alkalinity physics kmf1 strict_file_check: False # Output the figures as csv json files. -savecsv: True +savejson: True clean: True diff --git a/input_yml/TerraFIRMA_overshoot_stables.yml b/input_yml/TerraFIRMA_overshoot_stables.yml index 53379031..9b75d417 100644 --- a/input_yml/TerraFIRMA_overshoot_stables.yml +++ b/input_yml/TerraFIRMA_overshoot_stables.yml @@ -15,7 +15,7 @@ master_suites: physics bgc kmf #alkalinity physics kmf1 strict_file_check: False # Output the figures as csv json files. -savecsv: True +savejson: True clean: True From dae17f1ddcd413fcf098d9f552dae7e9d9b6d2ab Mon Sep 17 00:00:00 2001 From: Lee de Mora <ledm@pml.ac.uk> Date: Wed, 24 Apr 2024 16:17:57 +0100 Subject: [PATCH 13/14] Added readme details. --- README.md | 48 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2c6167cf..65d1596f 100644 --- a/README.md +++ b/README.md @@ -228,6 +228,7 @@ auto_download: <bool> strictFileCheck: <bool> dpi: <int> savepdf: <bool> +savejson: <bool> jobs: <jobID1>: @@ -278,7 +279,12 @@ These values are: - Output the image as a pdf in addition to the standard png. - This doesn't replace the image in the report, but saves a pdf version of the image in the images directory. - The pdfs will be web-visible in the report image directory, but will not linked in the html. - - To view a pdf from a report, simply click the image, and replace the `png` extension with `pdf` in the browser path. + - To view a pdf from a report, simply click the image, and replace the `png` extension with `pdf` in the browser path. + - `safejson`: + - Outputs the data and plotting details of each timeseries plot as a json file. + - json is a human readable format that is similar to a python dictionary or a shelve file. + - This file includes all data, time values, units, and line colors, thicknesses and styles. + - `jobs`: - A list of jobIDs, and some options on how they will appear in the final report. - The options are: @@ -331,6 +337,46 @@ then the report will appear on the [JASMIN public facing page](https://gws-acces which is public facing but password protected. +Information about the json output: +---------------------------------- + +The `savejson` flag allows bgcval2 to output all data from the multi-model plots to json. +Json files are effectively python dictionaries that are saved as human readable ASCII files. +This means that they can contain a lot more metadata than a simple csv file. +It also means that it's easier to deal with multiple datasets with different time ranges. + +So, in these files, everything is saved as a series of dictionaries, and the main ones are: + - `timesD`: The time value (in decimal years) + - `arrD`: The data value. + +There's also a lot of info used to make the bgcavl2 plots, including + - `colours`: the colour used by bgcval2 for each jobid + - `linestyle`: the plot line style for each job id + - `label`: the plot lable for each jobid + +Each of these dictionaries uses the jobID as the index, so in python, these can be plotted using the code: + + +``` +import json +from matplotlib import pyplot + +json_file = 'filename.json' +with open(json_file) as json_file: + amoc_data = json.load(json_file) + +for jobID in sorted(amoc_data['timesD'].keys()): + times = amoc_data['timesD'][jobID] + amoc = amoc_data['arrD'][jobID] + colour = amoc_data['colours'][jobID] + pyplot.plot(times, amoc, c=colour, label=jobID) + +pyplot.show() +``` + + + + Batch times series Analysis =========================== From c73780e9406ae6b7e5d4a517df7b4abd44f3850d Mon Sep 17 00:00:00 2001 From: Lee de Mora <ledm@pml.ac.uk> Date: Tue, 30 Apr 2024 15:26:17 +0100 Subject: [PATCH 14/14] replace print with raise --- bgcval2/timeseries/timeseriesTools.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bgcval2/timeseries/timeseriesTools.py b/bgcval2/timeseries/timeseriesTools.py index 59cab227..3a81eb37 100644 --- a/bgcval2/timeseries/timeseriesTools.py +++ b/bgcval2/timeseries/timeseriesTools.py @@ -280,8 +280,7 @@ def save_json( """ if csvformat.lower() not in ['json', '.json']: - print('Format not set up', csvformat ) - return + raise OSError(''.join(['save_json: format not recognised:', csvformat])) filename = bvt.folder(csvFolder) + '_'.join([analysisname, name, region, layer, metric, ts ]) + csvformat