Outputs comparison suite data to json and AtmosCO2

README.md

@@ -228,6 +228,7 @@ auto_download: <bool>
 strictFileCheck: <bool>
 dpi: <int>
 savepdf: <bool>
+savejson: <bool>
 
 jobs:
    <jobID1>:
@@ -278,7 +279,12 @@ These values are:
    - Output the image as a pdf in addition to the standard png. 
    - This doesn't replace the image in the report, but saves a pdf version of the image in the images directory.
    - The pdfs will be web-visible in the report image directory, but will not linked in the html.
-   - To view a pdf from a report, simply click the image, and replace the `png` extension with `pdf` in the browser path. 
+   - To view a pdf from a report, simply click the image, and replace the `png` extension with `pdf` in the browser path.
+ - `safejson`:
+   - Outputs the data and plotting details of each timeseries plot as a json file.
+   - json is a human readable format that is similar to a python dictionary or a shelve file. 
+   - This file includes all data, time values, units, and line colors, thicknesses and styles.
+
  - `jobs`:
    - A list of jobIDs, and some options on how they will appear in the final report.
    - The options are:
@@ -331,6 +337,46 @@ then the report will appear on the [JASMIN public facing page](https://gws-acces
 which is public facing but password protected.
 
 
+Information about the json output:
+----------------------------------
+
+The `savejson` flag allows bgcval2 to output all data from the multi-model plots to json. 
+Json files are effectively python dictionaries that are saved as human readable ASCII files. 
+This means that they can contain a lot more metadata than a simple csv file. 
+It also means that it's easier to deal with multiple datasets with different time ranges. 
+
+So, in these files, everything is saved as a series of dictionaries, and the main ones are:
+ - `timesD`: The time value (in decimal years)
+ - `arrD`: The data value. 
+
+There's also a lot of info used to make the bgcavl2 plots, including 
+ - `colours`: the colour used by bgcval2 for each jobid
+ - `linestyle`: the plot line style for each job id
+ - `label`: the plot lable for each jobid
+
+Each of these dictionaries uses the jobID as the index, so in python, these can be plotted using the code:
+
+
+```
+import json
+from matplotlib import pyplot
+
+json_file = 'filename.json'
+with open(json_file) as json_file:
+    amoc_data = json.load(json_file)
+
+for jobID in sorted(amoc_data['timesD'].keys()):
+    times = amoc_data['timesD'][jobID]
+    amoc = amoc_data['arrD'][jobID]
+    colour = amoc_data['colours'][jobID]
+    pyplot.plot(times, amoc, c=colour, label=jobID)
+
+pyplot.show()
+```
+
+
+
+
 Batch times series Analysis
 ===========================
 

bgcval2/analysis_compare.py

@@ -61,6 +61,8 @@
 from .timeseries import timeseriesAnalysis
 from .timeseries import profileAnalysis
 from .timeseries import timeseriesPlots as tsp
+from .timeseries import timeseriesTools as tst
+
 from bgcval2.analysis_timeseries import analysis_timeseries, build_list_of_suite_keys, load_key_file
 from bgcval2.download_from_mass import download_from_mass
 
@@ -194,6 +196,7 @@ def timeseries_compare(jobs,
                        config_user=None,
                        dpi=None,
                        savepdf=False,
+                       savejson=False,
     ):
     """
     timeseries_compare:
@@ -225,6 +228,7 @@ def timeseries_compare(jobs,
         sys.exit(0)
     else:
         imageFolder = paths.imagedir + '/TimeseriesCompare/' + analysisname
+        csvFolder = paths.imagedir + '/TimeseriesCompare_CSV/' + analysisname
 
     annual = True
     strictFileCheck = False
@@ -409,6 +413,25 @@ def timeseries_compare(jobs,
             units = av[name]['modeldetails']['units']
 
             ts = 'Together'
+
+            if savejson:
+                tst.save_json(
+                        timesD,
+                        arrD, 
+                        analysisname,
+                        colours=colours,
+                        linestyles=linestyles,
+                        thicknesses=lineThicknesses,
+                        labels=labels,
+                        name=name,
+                        units=units,
+                        region=region,
+                        layer=layer,
+                        metric=metric,
+                        title=title,
+                        ts=ts,
+                        csvFolder=csvFolder)
+
             for smoothing in smoothings:
             #or ls in ['DataOnly', ]:
                 tsp.multitimeseries(
@@ -417,7 +440,7 @@ def timeseries_compare(jobs,
                     data=-999,  # in situ data distribution
                     title=title,
                     filename=bvt.folder(imageFolder) +
-                        '_'.join([name, region, layer, ts, smoothing + '.png']),
+                        '_'.join([name, region, layer, metric, ts, smoothing + '.png']),
                     units=units,
                     plotStyle=ts,
                     smoothing=smoothing,
@@ -429,6 +452,7 @@ def timeseries_compare(jobs,
                     savepdf=savepdf,
                 )
 
+
     # Generate a list of comparison images:
     method_images = 'oswalk'
     AllImages = []
@@ -512,8 +536,12 @@ def load_comparison_yml(master_compare_yml_fn):
     # Image output settings:
     # dpi: pixels per inch (image resolution)
     # savepdf: also save the image as a pdf. 
+    # savejson: Save the data that appears in the image.
     details['dpi'] = input_yml_dict.get('dpi', None)
     details['savepdf'] = input_yml_dict.get('savepdf', False)
+    details['savejson'] = input_yml_dict.get('savejson', False)
+
+
 
     if details['dpi']: # None is valid!
         try: 
@@ -555,7 +583,6 @@ def load_comparison_yml(master_compare_yml_fn):
         auto_download_dict[jobID] = job_dict.get('auto_download', auto_download_dict[jobID]) 
         labels[jobID] = job_dict.get('label', jobID)
 
-
     details['colours'] = colours
     details['descriptions'] = descriptions
     details['thicknesses'] = thicknesses
@@ -599,6 +626,7 @@ def load_yml_and_run(compare_yml, config_user, skip_timeseries):
     strictFileCheck = details.get('strictFileCheck', True)
     dpi = details.get('dpi', None)
     savepdf = details.get('savepdf', False)
+    savejson = details.get('savejson', False)
 
     print('---------------------')
     print('timeseries_compare:',  analysis_name)
@@ -655,6 +683,7 @@ def load_yml_and_run(compare_yml, config_user, skip_timeseries):
         config_user=config_user,
         dpi=dpi,
         savepdf=savepdf,
+        savejson=savejson,
 
     )
 

bgcval2/bgcval2_make_report.py

@@ -1480,6 +1480,7 @@ def newImageLocation(fn):
     physicsKM = [
         'AMOC_26N',
         'ADRC_26N',
+        'AtmosCO2',  
         'DrakePassage',
         'GlobalMeanTemperature',
         'GlobalMeanSalinity',

bgcval2/bgcvaltools/pftnames.py

@@ -270,6 +270,9 @@ def makeLongNameDict():
 
     lnd['PCO2_SW'] = 'pCO2'
     lnd['pCO2'] = 'pCO2'
+    lnd['AtmospCO2'] = 'pCO2'
+    lnd['AtmosCO2'] = 'Atmospheric CO2'
+
 
     lnd['iron'] = "Iron"
     lnd['Fe_D_CONC_BOTTLE'] = "Iron (Dissolved)"

bgcval2/timeseries/timeseriesPlots.py

@@ -751,6 +751,8 @@ def multitimeseries(
 
     final_labels = []
 
+
+
     for i, jobID in enumerate(sorted(timesD.keys())):
         times = timesD[jobID]
         arr = arrD[jobID]

bgcval2/timeseries/timeseriesTools.py

@@ -33,6 +33,9 @@
 from bgcval2.bgcvaltools.dataset import dataset
 from bgcval2.bgcvaltools.makeMask import makeMask
 from bgcval2.functions.standard_functions import extractData as std_extractData
+import json
+from jsondiff import diff as jsondiff
+
 
 """
 .. module:: timeseriesTools
@@ -254,6 +257,68 @@ def getHorizontalSlice(nc, coords, details, layer, data=''):
     assert 0
 
 
+def save_json(
+        timesD,
+        arrD, 
+        analysisname,
+        colours={},
+        linestyles={},
+        thicknesses={},
+        labels={},
+        name='',
+        units='',
+        region='',
+        layer='',
+        metric='',
+        title='',
+        ts='',
+        csvFolder='',
+        csvformat='.json',
+    ):
+    """
+    Output the data that appears in a plot as a json file.
+
+    """
+    if csvformat.lower() not in ['json', '.json']:
+        raise OSError(''.join(['save_json: format not recognised:', csvformat]))  
+
+    filename = bvt.folder(csvFolder) + '_'.join([analysisname, name, region, layer, metric, ts ]) + csvformat
+
+    jsondata = {
+            # json can't save numpy.float32, so we convert to list of floats.
+            'timesD': {job:[float(t) for t in times] for job, times in timesD.items()},
+            'arrD': {job:[float(d) for d in data] for job, data in arrD.items()},
+            'analysisname': analysisname,
+            'colours':colours,
+            'linestyles':linestyles,
+            'thicknesses':thicknesses,
+            'labels':labels,
+            'name':name,
+            'units':units,
+            'region':region,
+            'layer':layer,
+            'metric':metric,
+            'title':title,
+            'analysisname': analysisname,
+            'ts':ts,
+            'filename':filename,
+        }
+
+    if os.path.exists(filename):
+        print('Opening old file:', filename)
+        with open(filename) as json_data:
+            json_old = json.load(json_data)
+
+        diff = jsondiff(jsondata, json_old)
+        if not len(diff):
+            print('Nothing new to add')
+            return
+
+    print('Data saved in: ', filename)
+    with open(filename, 'w', encoding='utf-8') as f:
+        json.dump(jsondata, f, ensure_ascii=False, indent=4)
+
+
 class DataLoader:
 
     def __init__(self,

environment.yml

@@ -7,6 +7,7 @@ channels:
 dependencies:
   - basemap >=1.3.6
   - cartopy
+  - jsondiff
   - matplotlib
   - nctoolkit >=0.8.7  # use linux64 build
   - netcdf4

input_yml/TerraFIRMA_overshoot_co2tests.yml

@@ -0,0 +1,68 @@
+---
+name: TerraFIRMA_overshoot_co2tests
+
+# Run the single job analysis
+do_analysis_timeseries: True 
+
+# Download from mass:
+do_mass_download: False
+
+# master analysis suite
+master_suites: physics bgc kmf #alkalinity physics kmf1
+
+clean: True
+
+# Output the figures as csv json files.
+savejson: True
+
+jobs:
+    u-cs495:
+       description: 'Reference '
+       label: PiControl
+       colour: 'blue'
+       thickness: 0.6
+       linestyle: '-'
+       shifttime: -427.
+       timerange: [1850, 2020]
+       suite: kmf physics bgc #alkalinity physics
+
+    u-cz014:
+       description: 'Terrafirma 4x CO2'
+       label: '4xCO2'
+       colour: 'green'
+       thickness: 1.7
+       linestyle: '-'
+       shifttime: 0.
+       suite: kmf physics bgc #alkalinity physics
+
+    u-cz152:
+       description: 'Terrafirma 1% CO2'
+       label: '1%CO2'
+       colour: 'purple'
+       thickness: 1.7
+       linestyle: '-'
+       shifttime: 0.
+       suite: kmf physics bgc #alkalinity physics
+
+
+    u-cy623:
+       description: 'interactive ice, started from picontrol yr 2277'
+       label: 'hist ice'
+       colour: 'orange'  
+       thickness: 1.7
+       linestyle: 'dashdot'
+       shifttime: 0.
+       suite: kmf physics bgc #alkalinity physics
+
+
+    u-cy690: 
+       description: 'static ice sheets, started from picontrol yr 2277'
+       colour: 'red'
+       label: 'hist static ice'
+       thickness: 1.7
+       linestyle: ':'
+       shifttime: 0.
+       suite: kmf physics bgc #alkalinity physics
+
+
+