diff --git a/README.md b/README.md index 2c6167cf..65d1596f 100644 --- a/README.md +++ b/README.md @@ -228,6 +228,7 @@ auto_download: strictFileCheck: dpi: savepdf: +savejson: jobs: : @@ -278,7 +279,12 @@ These values are: - Output the image as a pdf in addition to the standard png. - This doesn't replace the image in the report, but saves a pdf version of the image in the images directory. - The pdfs will be web-visible in the report image directory, but will not linked in the html. - - To view a pdf from a report, simply click the image, and replace the `png` extension with `pdf` in the browser path. + - To view a pdf from a report, simply click the image, and replace the `png` extension with `pdf` in the browser path. + - `safejson`: + - Outputs the data and plotting details of each timeseries plot as a json file. + - json is a human readable format that is similar to a python dictionary or a shelve file. + - This file includes all data, time values, units, and line colors, thicknesses and styles. + - `jobs`: - A list of jobIDs, and some options on how they will appear in the final report. - The options are: @@ -331,6 +337,46 @@ then the report will appear on the [JASMIN public facing page](https://gws-acces which is public facing but password protected. +Information about the json output: +---------------------------------- + +The `savejson` flag allows bgcval2 to output all data from the multi-model plots to json. +Json files are effectively python dictionaries that are saved as human readable ASCII files. +This means that they can contain a lot more metadata than a simple csv file. +It also means that it's easier to deal with multiple datasets with different time ranges. + +So, in these files, everything is saved as a series of dictionaries, and the main ones are: + - `timesD`: The time value (in decimal years) + - `arrD`: The data value. + +There's also a lot of info used to make the bgcavl2 plots, including + - `colours`: the colour used by bgcval2 for each jobid + - `linestyle`: the plot line style for each job id + - `label`: the plot lable for each jobid + +Each of these dictionaries uses the jobID as the index, so in python, these can be plotted using the code: + + +``` +import json +from matplotlib import pyplot + +json_file = 'filename.json' +with open(json_file) as json_file: + amoc_data = json.load(json_file) + +for jobID in sorted(amoc_data['timesD'].keys()): + times = amoc_data['timesD'][jobID] + amoc = amoc_data['arrD'][jobID] + colour = amoc_data['colours'][jobID] + pyplot.plot(times, amoc, c=colour, label=jobID) + +pyplot.show() +``` + + + + Batch times series Analysis =========================== diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index ac7d89f4..7f94ac56 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -61,6 +61,8 @@ from .timeseries import timeseriesAnalysis from .timeseries import profileAnalysis from .timeseries import timeseriesPlots as tsp +from .timeseries import timeseriesTools as tst + from bgcval2.analysis_timeseries import analysis_timeseries, build_list_of_suite_keys, load_key_file from bgcval2.download_from_mass import download_from_mass @@ -194,6 +196,7 @@ def timeseries_compare(jobs, config_user=None, dpi=None, savepdf=False, + savejson=False, ): """ timeseries_compare: @@ -225,6 +228,7 @@ def timeseries_compare(jobs, sys.exit(0) else: imageFolder = paths.imagedir + '/TimeseriesCompare/' + analysisname + csvFolder = paths.imagedir + '/TimeseriesCompare_CSV/' + analysisname annual = True strictFileCheck = False @@ -409,6 +413,25 @@ def timeseries_compare(jobs, units = av[name]['modeldetails']['units'] ts = 'Together' + + if savejson: + tst.save_json( + timesD, + arrD, + analysisname, + colours=colours, + linestyles=linestyles, + thicknesses=lineThicknesses, + labels=labels, + name=name, + units=units, + region=region, + layer=layer, + metric=metric, + title=title, + ts=ts, + csvFolder=csvFolder) + for smoothing in smoothings: #or ls in ['DataOnly', ]: tsp.multitimeseries( @@ -417,7 +440,7 @@ def timeseries_compare(jobs, data=-999, # in situ data distribution title=title, filename=bvt.folder(imageFolder) + - '_'.join([name, region, layer, ts, smoothing + '.png']), + '_'.join([name, region, layer, metric, ts, smoothing + '.png']), units=units, plotStyle=ts, smoothing=smoothing, @@ -429,6 +452,7 @@ def timeseries_compare(jobs, savepdf=savepdf, ) + # Generate a list of comparison images: method_images = 'oswalk' AllImages = [] @@ -512,8 +536,12 @@ def load_comparison_yml(master_compare_yml_fn): # Image output settings: # dpi: pixels per inch (image resolution) # savepdf: also save the image as a pdf. + # savejson: Save the data that appears in the image. details['dpi'] = input_yml_dict.get('dpi', None) details['savepdf'] = input_yml_dict.get('savepdf', False) + details['savejson'] = input_yml_dict.get('savejson', False) + + if details['dpi']: # None is valid! try: @@ -555,7 +583,6 @@ def load_comparison_yml(master_compare_yml_fn): auto_download_dict[jobID] = job_dict.get('auto_download', auto_download_dict[jobID]) labels[jobID] = job_dict.get('label', jobID) - details['colours'] = colours details['descriptions'] = descriptions details['thicknesses'] = thicknesses @@ -599,6 +626,7 @@ def load_yml_and_run(compare_yml, config_user, skip_timeseries): strictFileCheck = details.get('strictFileCheck', True) dpi = details.get('dpi', None) savepdf = details.get('savepdf', False) + savejson = details.get('savejson', False) print('---------------------') print('timeseries_compare:', analysis_name) @@ -655,6 +683,7 @@ def load_yml_and_run(compare_yml, config_user, skip_timeseries): config_user=config_user, dpi=dpi, savepdf=savepdf, + savejson=savejson, ) diff --git a/bgcval2/bgcval2_make_report.py b/bgcval2/bgcval2_make_report.py index 325ed2e8..290c40d1 100755 --- a/bgcval2/bgcval2_make_report.py +++ b/bgcval2/bgcval2_make_report.py @@ -1480,6 +1480,7 @@ def newImageLocation(fn): physicsKM = [ 'AMOC_26N', 'ADRC_26N', + 'AtmosCO2', 'DrakePassage', 'GlobalMeanTemperature', 'GlobalMeanSalinity', diff --git a/bgcval2/bgcvaltools/pftnames.py b/bgcval2/bgcvaltools/pftnames.py index 1e8a3d73..1fa0e6e9 100644 --- a/bgcval2/bgcvaltools/pftnames.py +++ b/bgcval2/bgcvaltools/pftnames.py @@ -270,6 +270,9 @@ def makeLongNameDict(): lnd['PCO2_SW'] = 'pCO2' lnd['pCO2'] = 'pCO2' + lnd['AtmospCO2'] = 'pCO2' + lnd['AtmosCO2'] = 'Atmospheric CO2' + lnd['iron'] = "Iron" lnd['Fe_D_CONC_BOTTLE'] = "Iron (Dissolved)" diff --git a/bgcval2/timeseries/timeseriesPlots.py b/bgcval2/timeseries/timeseriesPlots.py index 0afa7d67..44a68e5d 100644 --- a/bgcval2/timeseries/timeseriesPlots.py +++ b/bgcval2/timeseries/timeseriesPlots.py @@ -751,6 +751,8 @@ def multitimeseries( final_labels = [] + + for i, jobID in enumerate(sorted(timesD.keys())): times = timesD[jobID] arr = arrD[jobID] diff --git a/bgcval2/timeseries/timeseriesTools.py b/bgcval2/timeseries/timeseriesTools.py index 319511b2..3a81eb37 100644 --- a/bgcval2/timeseries/timeseriesTools.py +++ b/bgcval2/timeseries/timeseriesTools.py @@ -33,6 +33,9 @@ from bgcval2.bgcvaltools.dataset import dataset from bgcval2.bgcvaltools.makeMask import makeMask from bgcval2.functions.standard_functions import extractData as std_extractData +import json +from jsondiff import diff as jsondiff + """ .. module:: timeseriesTools @@ -254,6 +257,68 @@ def getHorizontalSlice(nc, coords, details, layer, data=''): assert 0 +def save_json( + timesD, + arrD, + analysisname, + colours={}, + linestyles={}, + thicknesses={}, + labels={}, + name='', + units='', + region='', + layer='', + metric='', + title='', + ts='', + csvFolder='', + csvformat='.json', + ): + """ + Output the data that appears in a plot as a json file. + + """ + if csvformat.lower() not in ['json', '.json']: + raise OSError(''.join(['save_json: format not recognised:', csvformat])) + + filename = bvt.folder(csvFolder) + '_'.join([analysisname, name, region, layer, metric, ts ]) + csvformat + + jsondata = { + # json can't save numpy.float32, so we convert to list of floats. + 'timesD': {job:[float(t) for t in times] for job, times in timesD.items()}, + 'arrD': {job:[float(d) for d in data] for job, data in arrD.items()}, + 'analysisname': analysisname, + 'colours':colours, + 'linestyles':linestyles, + 'thicknesses':thicknesses, + 'labels':labels, + 'name':name, + 'units':units, + 'region':region, + 'layer':layer, + 'metric':metric, + 'title':title, + 'analysisname': analysisname, + 'ts':ts, + 'filename':filename, + } + + if os.path.exists(filename): + print('Opening old file:', filename) + with open(filename) as json_data: + json_old = json.load(json_data) + + diff = jsondiff(jsondata, json_old) + if not len(diff): + print('Nothing new to add') + return + + print('Data saved in: ', filename) + with open(filename, 'w', encoding='utf-8') as f: + json.dump(jsondata, f, ensure_ascii=False, indent=4) + + class DataLoader: def __init__(self, diff --git a/environment.yml b/environment.yml index 3a45b5a3..3a60e75c 100644 --- a/environment.yml +++ b/environment.yml @@ -7,6 +7,7 @@ channels: dependencies: - basemap >=1.3.6 - cartopy + - jsondiff - matplotlib - nctoolkit >=0.8.7 # use linux64 build - netcdf4 diff --git a/input_yml/TerraFIRMA_overshoot_co2tests.yml b/input_yml/TerraFIRMA_overshoot_co2tests.yml new file mode 100644 index 00000000..62578012 --- /dev/null +++ b/input_yml/TerraFIRMA_overshoot_co2tests.yml @@ -0,0 +1,68 @@ +--- +name: TerraFIRMA_overshoot_co2tests + +# Run the single job analysis +do_analysis_timeseries: True + +# Download from mass: +do_mass_download: False + +# master analysis suite +master_suites: physics bgc kmf #alkalinity physics kmf1 + +clean: True + +# Output the figures as csv json files. +savejson: True + +jobs: + u-cs495: + description: 'Reference ' + label: PiControl + colour: 'blue' + thickness: 0.6 + linestyle: '-' + shifttime: -427. + timerange: [1850, 2020] + suite: kmf physics bgc #alkalinity physics + + u-cz014: + description: 'Terrafirma 4x CO2' + label: '4xCO2' + colour: 'green' + thickness: 1.7 + linestyle: '-' + shifttime: 0. + suite: kmf physics bgc #alkalinity physics + + u-cz152: + description: 'Terrafirma 1% CO2' + label: '1%CO2' + colour: 'purple' + thickness: 1.7 + linestyle: '-' + shifttime: 0. + suite: kmf physics bgc #alkalinity physics + + + u-cy623: + description: 'interactive ice, started from picontrol yr 2277' + label: 'hist ice' + colour: 'orange' + thickness: 1.7 + linestyle: 'dashdot' + shifttime: 0. + suite: kmf physics bgc #alkalinity physics + + + u-cy690: + description: 'static ice sheets, started from picontrol yr 2277' + colour: 'red' + label: 'hist static ice' + thickness: 1.7 + linestyle: ':' + shifttime: 0. + suite: kmf physics bgc #alkalinity physics + + + diff --git a/input_yml/TerraFIRMA_overshoot_historical.yml b/input_yml/TerraFIRMA_overshoot_historical.yml index 9f1cb965..60cba4e8 100644 --- a/input_yml/TerraFIRMA_overshoot_historical.yml +++ b/input_yml/TerraFIRMA_overshoot_historical.yml @@ -13,18 +13,61 @@ master_suites: physics bgc #alkalinity physics kmf1 clean: True +# Output the figures as csv json files. +savejson: True + jobs: + u-cs495: + description: 'PiControl' + label: PiControl + colour: 'blue' + thickness: 0.6 + linestyle: '-' + shifttime: -427. + timerange: [1850, 2020] + suite: kmf physics bgc #alkalinity physics + u-ca306: description: 'Reference ' + label: Reference colour: 'black' thickness: 0.6 linestyle: '-' - shifttime: 0. - timerange: [1800, 1900] + shifttime: 0 + timerange: [1850, 2020] suite: kmf physics bgc #alkalinity physics u-cy623: description: 'interactive ice, started from picontrol yr 2277' + label: 'hist ice' + colour: red + thickness: 1.7 + linestyle: '-' + shifttime: 0. + suite: kmf physics bgc #alkalinity physics + + + u-da914: + description: 'interactive ice, started from picontrol yr 2197' + label: None + colour: red + thickness: 1.7 + linestyle: '-' + shifttime: 0. + suite: kmf physics bgc #alkalinity physics + + u-da916: + description: 'interactive ice, started from picontrol yr 2237' + label: None + colour: red + thickness: 1.7 + linestyle: '-' + shifttime: 0. + suite: kmf physics bgc #alkalinity physics + + u-da917: + description: 'interactive ice, started from picontrol yr 2317' + label: None colour: red thickness: 1.7 linestyle: '-' @@ -33,7 +76,8 @@ jobs: u-cy690: description: 'static ice sheets, started from picontrol yr 2277' - colour: purple + colour: green + label: 'hist static ice' thickness: 1.7 linestyle: '-' shifttime: 0. @@ -41,7 +85,8 @@ jobs: u-cy691: description: 'static ice sheets, started from picontrol yr 2197' - colour: blue + label: None + colour: green thickness: 1.7 linestyle: '-' shifttime: 0. @@ -49,6 +94,7 @@ jobs: u-cy692: description: ' static ice sheets, started from picontrol yr 2237' + label: None colour: green thickness: 1.7 linestyle: '-' @@ -57,7 +103,8 @@ jobs: u-cy693: description: ' static ice sheets, started from picontrol yr 2317' - colour: goldenrod + label: None + colour: green thickness: 1.7 linestyle: '-' shifttime: 0. diff --git a/input_yml/TerraFIRMA_overshoot_recovery.yml b/input_yml/TerraFIRMA_overshoot_recovery.yml index 87aa0a3d..1c4072b7 100644 --- a/input_yml/TerraFIRMA_overshoot_recovery.yml +++ b/input_yml/TerraFIRMA_overshoot_recovery.yml @@ -14,6 +14,9 @@ master_suites: physics bgc kmf #alkalinity physics kmf1 # Run without strick check (if True, breaks if job has no years.) strict_file_check: False +# Output the figures as csv json files. +savejson: True + clean: True jobs: diff --git a/input_yml/TerraFIRMA_overshoot_runs.yml b/input_yml/TerraFIRMA_overshoot_runs.yml index e549b099..104c28f7 100644 --- a/input_yml/TerraFIRMA_overshoot_runs.yml +++ b/input_yml/TerraFIRMA_overshoot_runs.yml @@ -14,6 +14,9 @@ master_suites: physics bgc kmf #alkalinity physics kmf1 # Run without strick check (if True, breaks if job has no years.) strict_file_check: False +# Output the figures as csv json files. +savejson: True + clean: True diff --git a/input_yml/TerraFIRMA_overshoot_stables.yml b/input_yml/TerraFIRMA_overshoot_stables.yml index 25975990..9b75d417 100644 --- a/input_yml/TerraFIRMA_overshoot_stables.yml +++ b/input_yml/TerraFIRMA_overshoot_stables.yml @@ -14,6 +14,9 @@ master_suites: physics bgc kmf #alkalinity physics kmf1 # Run without strick check (if True, breaks if job has no years.) strict_file_check: False +# Output the figures as csv json files. +savejson: True + clean: True diff --git a/key_files/atmosco2.yml b/key_files/atmosco2.yml new file mode 100644 index 00000000..9c3e9156 --- /dev/null +++ b/key_files/atmosco2.yml @@ -0,0 +1,12 @@ +--- +name : AtmosCO2 +units : ppm +model : MEDUSA +modelgrid : eORCA1 +dimensions : 2 +modelFiles : $BASEDIR_MODEL/$JOBID/medusa*$JOBIDo_1y_*_diad-T.nc +model_vars : ATM_XCO2 +model_convert : NoChange +layers : layerless +regions : Global #gnoreInlandSeas SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific + diff --git a/key_files/mld.yml b/key_files/mld.yml index 4c8c1eef..e6d04417 100644 --- a/key_files/mld.yml +++ b/key_files/mld.yml @@ -30,4 +30,4 @@ data_convert: maskname : mask areafile: $BASEDIR_OBS/IFREMER-MLD/mld_DT02_c1m_reg2.0-annual.nc #layers : Surface -regions : Global OnShelf OffShelf ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthAtlanticOcean SouthAtlanticOcean NorthPacificOcean SouthPacificOcean SPNA +regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthAtlanticOcean SouthAtlanticOcean NorthPacificOcean SouthPacificOcean SPNA diff --git a/key_lists/atmosco2.yml b/key_lists/atmosco2.yml new file mode 100644 index 00000000..64b1150f --- /dev/null +++ b/key_lists/atmosco2.yml @@ -0,0 +1,3 @@ +--- +keys: + atmosco2: True diff --git a/key_lists/kmf.yml b/key_lists/kmf.yml index 08525c8f..6ec82ec4 100644 --- a/key_lists/kmf.yml +++ b/key_lists/kmf.yml @@ -7,6 +7,7 @@ keys: SST: True DrakePassageTransport: True AMOC_26N: True + AtmosCO2: True TotalAirSeaFluxCO2: True # NoCaspianAirSeaFluxCO2: True # IntPP_OSU: True diff --git a/key_lists/physics.yml b/key_lists/physics.yml index 34f51bf2..a142d1bb 100644 --- a/key_lists/physics.yml +++ b/key_lists/physics.yml @@ -17,6 +17,9 @@ keys: Salinity: True # WOA Salinity # soga: True + # CO2: + AtmosCO2: True + # Mixed layer depth keys: MLD: True # iFERMER Mixed Layer Depth # MaxMonthlyMLD: True # MLD Monthly max diff --git a/setup.py b/setup.py index 134b294f..377b4674 100755 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ 'install': [ 'basemap>=1.3.6', 'cartopy', + 'jsondiff', 'matplotlib', 'nctoolkit>=0.8.7', # use linux64 build 'netcdf4',