From 947c2ae6bb303e04f4997347e45f21e5ca50babc Mon Sep 17 00:00:00 2001
From: Lee de Mora <ledm@pml.ac.uk>
Date: Wed, 27 Mar 2024 11:04:17 +0000
Subject: [PATCH 01/14] Added output as json files option

---
 bgcval2/analysis_compare.py                 | 32 +++++++++-
 bgcval2/timeseries/timeseriesPlots.py       |  3 +
 bgcval2/timeseries/timeseriesTools.py       | 68 +++++++++++++++++++++
 input_yml/TerraFIRMA_overshoot_recovery.yml |  3 +
 input_yml/TerraFIRMA_overshoot_runs.yml     |  3 +
 input_yml/TerraFIRMA_overshoot_stables.yml  |  3 +
 6 files changed, 110 insertions(+), 2 deletions(-)

diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py
index ac7d89f4..f11186a2 100755
--- a/bgcval2/analysis_compare.py
+++ b/bgcval2/analysis_compare.py
@@ -194,6 +194,7 @@ def timeseries_compare(jobs,
                        config_user=None,
                        dpi=None,
                        savepdf=False,
+                       savecsv=False,
     ):
     """
     timeseries_compare:
@@ -225,6 +226,7 @@ def timeseries_compare(jobs,
         sys.exit(0)
     else:
         imageFolder = paths.imagedir + '/TimeseriesCompare/' + analysisname
+        csvFolder = paths.imagedir + '/TimeseriesCompare_CSV/' + analysisname
 
     annual = True
     strictFileCheck = False
@@ -409,6 +411,25 @@ def timeseries_compare(jobs,
             units = av[name]['modeldetails']['units']
 
             ts = 'Together'
+
+            if savecsv:
+                tst.save_csv(
+                        timesD,
+                        arrD, 
+                        analysisname,
+                        colours=colours,
+                        linestyles=linestyles,
+                        thicknesses=lineThicknesses,
+                        labels=labels,
+                        name=name,
+                        units=units,
+                        region=region,
+                        layer=layer,
+                        metric=metric,
+                        title=title,
+                        ts=ts,
+                        csvFolder=csvFolder)
+                    
             for smoothing in smoothings:
             #or ls in ['DataOnly', ]:
                 tsp.multitimeseries(
@@ -417,7 +438,7 @@ def timeseries_compare(jobs,
                     data=-999,  # in situ data distribution
                     title=title,
                     filename=bvt.folder(imageFolder) +
-                        '_'.join([name, region, layer, ts, smoothing + '.png']),
+                        '_'.join([name, region, layer, metric, ts, smoothing + '.png']),
                     units=units,
                     plotStyle=ts,
                     smoothing=smoothing,
@@ -427,8 +448,10 @@ def timeseries_compare(jobs,
                     labels=labels,
                     dpi=dpi,
                     savepdf=savepdf,
+                    savecsv=savecsv,
                 )
 
+
     # Generate a list of comparison images:
     method_images = 'oswalk'
     AllImages = []
@@ -512,8 +535,12 @@ def load_comparison_yml(master_compare_yml_fn):
     # Image output settings:
     # dpi: pixels per inch (image resolution)
     # savepdf: also save the image as a pdf. 
+    # savecsv: Save the data that appears in the image.
     details['dpi'] = input_yml_dict.get('dpi', None)
     details['savepdf'] = input_yml_dict.get('savepdf', False)
+    details['savecsv'] = input_yml_dict.get('savecsv', False)
+
+
 
     if details['dpi']: # None is valid!
         try: 
@@ -555,7 +582,6 @@ def load_comparison_yml(master_compare_yml_fn):
         auto_download_dict[jobID] = job_dict.get('auto_download', auto_download_dict[jobID]) 
         labels[jobID] = job_dict.get('label', jobID)
        
-
     details['colours'] = colours
     details['descriptions'] = descriptions
     details['thicknesses'] = thicknesses
@@ -599,6 +625,7 @@ def load_yml_and_run(compare_yml, config_user, skip_timeseries):
     strictFileCheck = details.get('strictFileCheck', True)
     dpi = details.get('dpi', None)
     savepdf = details.get('savepdf', False)
+    savecsv = details.get('savecsv', False)
 
     print('---------------------')
     print('timeseries_compare:',  analysis_name)
@@ -655,6 +682,7 @@ def load_yml_and_run(compare_yml, config_user, skip_timeseries):
         config_user=config_user,
         dpi=dpi,
         savepdf=savepdf,
+        savecsv=savecsv,
 
     )
 
diff --git a/bgcval2/timeseries/timeseriesPlots.py b/bgcval2/timeseries/timeseriesPlots.py
index 0afa7d67..afee9568 100644
--- a/bgcval2/timeseries/timeseriesPlots.py
+++ b/bgcval2/timeseries/timeseriesPlots.py
@@ -710,6 +710,7 @@ def multitimeseries(
         labels={},
         dpi=None,
         savepdf=False,
+        savecsv=False,
 ):
 
     if 0 in [len(timesD), len(list(timesD.keys()))]: return
@@ -751,6 +752,8 @@ def multitimeseries(
 
     final_labels = []
 
+
+
     for i, jobID in enumerate(sorted(timesD.keys())):
         times = timesD[jobID]
         arr = arrD[jobID]
diff --git a/bgcval2/timeseries/timeseriesTools.py b/bgcval2/timeseries/timeseriesTools.py
index 319511b2..b1f47910 100644
--- a/bgcval2/timeseries/timeseriesTools.py
+++ b/bgcval2/timeseries/timeseriesTools.py
@@ -33,6 +33,9 @@
 from bgcval2.bgcvaltools.dataset import dataset
 from bgcval2.bgcvaltools.makeMask import makeMask
 from bgcval2.functions.standard_functions import extractData as std_extractData
+import json
+from jsondiff import diff as jsondiff
+
 
 """
 .. module:: timeseriesTools
@@ -254,6 +257,71 @@ def getHorizontalSlice(nc, coords, details, layer, data=''):
     assert 0
 
 
+def save_csv(
+        timesD,
+        arrD, 
+        analysisname,
+        colours={},
+        linestyles={},
+        thicknesses={},
+        labels={},
+        name='',
+        units='',
+        region='',
+        layer='',
+        metric='',
+        title='',
+        ts='',
+        csvFolder='',
+        csvformat='.json',
+    ):
+    """
+    Output the data that appears in a plot as a csv.
+
+    """
+    if csvformat.lower() not in ['json', '.json']:
+        print('Format not set up', csvformat )
+        return
+    
+    filename = bvt.folder(csvFolder) + '_'.join([analysisname, name, region, layer, metric, ts ]) + csvformat
+
+    jsondata = {
+            'timesD': timesD,
+            'arrD': arrD, 
+            'analysisname': analysisname,
+            'colours':colours,
+            'linestyles':linestyles,
+            'thicknesses':lineThicknesses,
+            'labels':labels,
+            'name':name,
+            'units':units,
+            'region':region,
+            'layer':layer,
+            'metric':metric,
+            'title':title,
+            'analysisname': analysisname,
+            'ts':ts,
+            'filename':filename,
+        }
+    
+    if os.path.exists(filename):
+        with open(filename) as json_data:
+            json_old = json.load(json_data)
+       
+        diff = jsondiff(jsondata, json_old)
+        if not len(diff):
+            # Nothing new to add
+            return
+
+    with open(filename, 'w', encoding='utf-8') as f:
+        json.dump(jsondata, f, ensure_ascii=False, indent=4)
+
+        
+
+
+
+
+
 class DataLoader:
 
     def __init__(self,
diff --git a/input_yml/TerraFIRMA_overshoot_recovery.yml b/input_yml/TerraFIRMA_overshoot_recovery.yml
index 87aa0a3d..b4168f55 100644
--- a/input_yml/TerraFIRMA_overshoot_recovery.yml
+++ b/input_yml/TerraFIRMA_overshoot_recovery.yml
@@ -14,6 +14,9 @@ master_suites: physics bgc kmf #alkalinity physics kmf1
 # Run without strick check (if True, breaks if job has no years.)
 strict_file_check: False
 
+# Output the figures as csv json files.
+savecsv: True
+
 clean: True
 
 jobs:
diff --git a/input_yml/TerraFIRMA_overshoot_runs.yml b/input_yml/TerraFIRMA_overshoot_runs.yml
index e549b099..1765fb0b 100644
--- a/input_yml/TerraFIRMA_overshoot_runs.yml
+++ b/input_yml/TerraFIRMA_overshoot_runs.yml
@@ -14,6 +14,9 @@ master_suites: physics bgc kmf #alkalinity physics kmf1
 # Run without strick check (if True, breaks if job has no years.)
 strict_file_check: False
 
+# Output the figures as csv json files.
+savecsv: True
+
 clean: True
 
 
diff --git a/input_yml/TerraFIRMA_overshoot_stables.yml b/input_yml/TerraFIRMA_overshoot_stables.yml
index 25975990..53379031 100644
--- a/input_yml/TerraFIRMA_overshoot_stables.yml
+++ b/input_yml/TerraFIRMA_overshoot_stables.yml
@@ -14,6 +14,9 @@ master_suites: physics bgc kmf #alkalinity physics kmf1
 # Run without strick check (if True, breaks if job has no years.)
 strict_file_check: False
 
+# Output the figures as csv json files.
+savecsv: True
+
 clean: True
 
 

From 21942f05331774a23d8734b3852bad50dc56f7dc Mon Sep 17 00:00:00 2001
From: Lee de Mora <ledm@pml.ac.uk>
Date: Wed, 27 Mar 2024 11:31:05 +0000
Subject: [PATCH 02/14] debugging on jasmin

---
 bgcval2/analysis_compare.py           |  2 ++
 bgcval2/timeseries/timeseriesTools.py | 10 +++-------
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py
index f11186a2..d459bc46 100755
--- a/bgcval2/analysis_compare.py
+++ b/bgcval2/analysis_compare.py
@@ -61,6 +61,8 @@
 from .timeseries import timeseriesAnalysis
 from .timeseries import profileAnalysis
 from .timeseries import timeseriesPlots as tsp
+from .timeseries import timeseriesTools as tst
+
 from bgcval2.analysis_timeseries import analysis_timeseries, build_list_of_suite_keys, load_key_file
 from bgcval2.download_from_mass import download_from_mass
 
diff --git a/bgcval2/timeseries/timeseriesTools.py b/bgcval2/timeseries/timeseriesTools.py
index b1f47910..8670d43d 100644
--- a/bgcval2/timeseries/timeseriesTools.py
+++ b/bgcval2/timeseries/timeseriesTools.py
@@ -291,7 +291,7 @@ def save_csv(
             'analysisname': analysisname,
             'colours':colours,
             'linestyles':linestyles,
-            'thicknesses':lineThicknesses,
+            'thicknesses':thicknesses,
             'labels':labels,
             'name':name,
             'units':units,
@@ -310,16 +310,12 @@ def save_csv(
        
         diff = jsondiff(jsondata, json_old)
         if not len(diff):
-            # Nothing new to add
+            print('Nothing new to add', filename)
             return
 
     with open(filename, 'w', encoding='utf-8') as f:
         json.dump(jsondata, f, ensure_ascii=False, indent=4)
-
-        
-
-
-
+    print('Data saved in: ', filename)
 
 
 class DataLoader:

From 45120829de9f8d2edc4c3a8012c16c8eec0a31be Mon Sep 17 00:00:00 2001
From: Lee de Mora <ledm@pml.ac.uk>
Date: Wed, 27 Mar 2024 14:11:05 +0000
Subject: [PATCH 03/14] Update TerraFIRMA_overshoot_historical.yml

---
 input_yml/TerraFIRMA_overshoot_historical.yml | 38 ++++++++++++++++---
 1 file changed, 33 insertions(+), 5 deletions(-)

diff --git a/input_yml/TerraFIRMA_overshoot_historical.yml b/input_yml/TerraFIRMA_overshoot_historical.yml
index 9f1cb965..629ab9c9 100644
--- a/input_yml/TerraFIRMA_overshoot_historical.yml
+++ b/input_yml/TerraFIRMA_overshoot_historical.yml
@@ -13,14 +13,15 @@ master_suites: physics bgc #alkalinity physics kmf1
 
 clean: True
 
+
 jobs:
     u-ca306:
        description: 'Reference '
-       colour: 'black'
+       colour: 'blue'
        thickness: 0.6
        linestyle: '-'
        shifttime: 0.
-       timerange: [1800, 1900]
+       timerange: [1850, 2020]
        suite: kmf physics bgc #alkalinity physics
 
     u-cy623:
@@ -31,9 +32,36 @@ jobs:
        shifttime: 0.
        suite: kmf physics bgc #alkalinity physics
 
+
+    u-da914:
+       description: 'interactive ice, started from picontrol yr 2197'
+       colour: red  
+       thickness: 1.7
+       linestyle: '-'
+       shifttime: 0.
+       suite: kmf physics bgc #alkalinity physics
+
+    u-da916:
+       description: 'interactive ice, started from picontrol yr 2237'
+       colour: red  
+       thickness: 1.7
+       linestyle: '-'
+       shifttime: 0.
+       suite: kmf physics bgc #alkalinity physics
+
+    u-da917:
+       description: 'interactive ice, started from picontrol yr 2317'
+       colour: red  
+       thickness: 1.7
+       linestyle: '-'
+       shifttime: 0.
+       suite: kmf physics bgc #alkalinity physics
+
+
+
     u-cy690: 
        description: 'static ice sheets, started from picontrol yr 2277'
-       colour: purple
+       colour: blue
        thickness: 1.7
        linestyle: '-'
        shifttime: 0.
@@ -49,7 +77,7 @@ jobs:
 
     u-cy692:
        description: ' static ice sheets, started from picontrol yr 2237'
-       colour: green
+       colour: blue
        thickness: 1.7
        linestyle: '-'
        shifttime: 0.
@@ -57,7 +85,7 @@ jobs:
 
     u-cy693:
        description: ' static ice sheets, started from picontrol yr 2317'
-       colour: goldenrod
+       colour: blue
        thickness: 1.7
        linestyle: '-'
        shifttime: 0.

From 82d9b9c5b362ffbda4ca008905ac090b447f21fd Mon Sep 17 00:00:00 2001
From: Lee de Mora <ledm@pml.ac.uk>
Date: Wed, 27 Mar 2024 14:14:27 +0000
Subject: [PATCH 04/14] Update TerraFIRMA_overshoot_historical.yml

---
 input_yml/TerraFIRMA_overshoot_historical.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/input_yml/TerraFIRMA_overshoot_historical.yml b/input_yml/TerraFIRMA_overshoot_historical.yml
index 629ab9c9..802675cd 100644
--- a/input_yml/TerraFIRMA_overshoot_historical.yml
+++ b/input_yml/TerraFIRMA_overshoot_historical.yml
@@ -13,6 +13,8 @@ master_suites: physics bgc #alkalinity physics kmf1
 
 clean: True
 
+# Output the figures as csv json files.
+savecsv: True
 
 jobs:
     u-ca306:

From 460e19991d9f5cee4e9db81ce3f7d46e59b53408 Mon Sep 17 00:00:00 2001
From: Lee de Mora <ledm@pml.ac.uk>
Date: Wed, 27 Mar 2024 14:17:50 +0000
Subject: [PATCH 05/14] Update TerraFIRMA_overshoot_historical.yml

---
 input_yml/TerraFIRMA_overshoot_historical.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/input_yml/TerraFIRMA_overshoot_historical.yml b/input_yml/TerraFIRMA_overshoot_historical.yml
index 802675cd..da98cd50 100644
--- a/input_yml/TerraFIRMA_overshoot_historical.yml
+++ b/input_yml/TerraFIRMA_overshoot_historical.yml
@@ -19,6 +19,7 @@ savecsv: True
 jobs:
     u-ca306:
        description: 'Reference '
+       label: PiControl
        colour: 'blue'
        thickness: 0.6
        linestyle: '-'
@@ -28,6 +29,7 @@ jobs:
 
     u-cy623:
        description: 'interactive ice, started from picontrol yr 2277'
+       label: 'hist ice'
        colour: red  
        thickness: 1.7
        linestyle: '-'
@@ -37,6 +39,7 @@ jobs:
 
     u-da914:
        description: 'interactive ice, started from picontrol yr 2197'
+       label: None
        colour: red  
        thickness: 1.7
        linestyle: '-'
@@ -45,6 +48,7 @@ jobs:
 
     u-da916:
        description: 'interactive ice, started from picontrol yr 2237'
+       label: None
        colour: red  
        thickness: 1.7
        linestyle: '-'
@@ -53,6 +57,7 @@ jobs:
 
     u-da917:
        description: 'interactive ice, started from picontrol yr 2317'
+       label: None
        colour: red  
        thickness: 1.7
        linestyle: '-'
@@ -64,6 +69,7 @@ jobs:
     u-cy690: 
        description: 'static ice sheets, started from picontrol yr 2277'
        colour: blue
+       label: 'hist static ice'
        thickness: 1.7
        linestyle: '-'
        shifttime: 0.
@@ -71,6 +77,7 @@ jobs:
 
     u-cy691:
        description: 'static ice sheets, started from picontrol yr 2197'
+       label: None       
        colour: blue
        thickness: 1.7
        linestyle: '-'
@@ -79,6 +86,7 @@ jobs:
 
     u-cy692:
        description: ' static ice sheets, started from picontrol yr 2237'
+       label: None       
        colour: blue
        thickness: 1.7
        linestyle: '-'
@@ -87,6 +95,7 @@ jobs:
 
     u-cy693:
        description: ' static ice sheets, started from picontrol yr 2317'
+       label: None       
        colour: blue
        thickness: 1.7
        linestyle: '-'

From 0dc82e0ad5f5ea66fea3ded2f0ba4fdd664ccd52 Mon Sep 17 00:00:00 2001
From: Lee de Mora <ledm@pml.ac.uk>
Date: Thu, 28 Mar 2024 10:53:45 +0000
Subject: [PATCH 06/14] new co2 tests run

---
 bgcval2/timeseries/timeseriesTools.py       | 10 +--
 input_yml/TerraFIRMA_overshoot_co2tests.yml | 68 +++++++++++++++++++++
 2 files changed, 74 insertions(+), 4 deletions(-)
 create mode 100644 input_yml/TerraFIRMA_overshoot_co2tests.yml

diff --git a/bgcval2/timeseries/timeseriesTools.py b/bgcval2/timeseries/timeseriesTools.py
index 8670d43d..7df0bb25 100644
--- a/bgcval2/timeseries/timeseriesTools.py
+++ b/bgcval2/timeseries/timeseriesTools.py
@@ -286,8 +286,9 @@ def save_csv(
     filename = bvt.folder(csvFolder) + '_'.join([analysisname, name, region, layer, metric, ts ]) + csvformat
 
     jsondata = {
-            'timesD': timesD,
-            'arrD': arrD, 
+            # json can't save numpy.float32, so we convert to list of floats.
+            'timesD': {job:[float(t) for t in times] for job, times in timesD.items()},
+            'arrD': {job:[float(d) for d in data] for job, data in arrD.items()},
             'analysisname': analysisname,
             'colours':colours,
             'linestyles':linestyles,
@@ -305,17 +306,18 @@ def save_csv(
         }
     
     if os.path.exists(filename):
+        print('Opening old file:', filename)
         with open(filename) as json_data:
             json_old = json.load(json_data)
        
         diff = jsondiff(jsondata, json_old)
         if not len(diff):
-            print('Nothing new to add', filename)
+            print('Nothing new to add')
             return
 
+    print('Data saved in: ', filename)
     with open(filename, 'w', encoding='utf-8') as f:
         json.dump(jsondata, f, ensure_ascii=False, indent=4)
-    print('Data saved in: ', filename)
 
 
 class DataLoader:
diff --git a/input_yml/TerraFIRMA_overshoot_co2tests.yml b/input_yml/TerraFIRMA_overshoot_co2tests.yml
new file mode 100644
index 00000000..42887cc0
--- /dev/null
+++ b/input_yml/TerraFIRMA_overshoot_co2tests.yml
@@ -0,0 +1,68 @@
+---
+name: TerraFIRMA_overshoot_co2tests
+
+# Run the single job analysis
+do_analysis_timeseries: True 
+
+# Download from mass:
+do_mass_download: False
+
+# master analysis suite
+master_suites: physics bgc #alkalinity physics kmf1
+
+clean: True
+
+# Output the figures as csv json files.
+savecsv: True
+
+jobs:
+    u-ca306:
+       description: 'Reference '
+       label: PiControl
+       colour: 'blue'
+       thickness: 0.6
+       linestyle: '-'
+       shifttime: 0.
+       timerange: [1850, 2020]
+       suite: kmf physics bgc #alkalinity physics
+
+    u-cz014:
+       description: 'Terrafirma 4x CO2'
+       label: '4xCO2'
+       colour: 'green'
+       thickness: 1.7
+       linestyle: '-'
+       shifttime: 0.
+       suite: kmf physics bgc #alkalinity physics
+
+    u-cz014:
+       description: 'Terrafirma 1% CO2'
+       label: '1%CO2'
+       colour: 'purple'
+       thickness: 1.7
+       linestyle: '-'
+       shifttime: 0.
+       suite: kmf physics bgc #alkalinity physics
+
+
+    u-cy623:
+       description: 'interactive ice, started from picontrol yr 2277'
+       label: 'hist ice'
+       colour: 'orange'  
+       thickness: 1.7
+       linestyle: 'dashdot'
+       shifttime: 0.
+       suite: kmf physics bgc #alkalinity physics
+
+
+    u-cy690: 
+       description: 'static ice sheets, started from picontrol yr 2277'
+       colour: 'red'
+       label: 'hist static ice'
+       thickness: 1.7
+       linestyle: ':'
+       shifttime: 0.
+       suite: kmf physics bgc #alkalinity physics
+
+
+

From 2250faf9060a8f0db0cd0ed7c76470705fa9c114 Mon Sep 17 00:00:00 2001
From: Lee de Mora <ledm@pml.ac.uk>
Date: Thu, 28 Mar 2024 16:36:13 +0000
Subject: [PATCH 07/14] Added new Atmospheric CO2 field to kmf and elsewhere.

---
 bgcval2/bgcval2_make_report.py                |  1 +
 bgcval2/bgcvaltools/pftnames.py               |  3 +++
 input_yml/TerraFIRMA_overshoot_co2tests.yml   |  6 +++---
 input_yml/TerraFIRMA_overshoot_historical.yml |  2 +-
 key_files/atmosco2.yml                        | 12 ++++++++++++
 key_lists/atmosco2.yml                        |  3 +++
 key_lists/kmf.yml                             |  1 +
 7 files changed, 24 insertions(+), 4 deletions(-)
 create mode 100644 key_files/atmosco2.yml
 create mode 100644 key_lists/atmosco2.yml

diff --git a/bgcval2/bgcval2_make_report.py b/bgcval2/bgcval2_make_report.py
index 325ed2e8..290c40d1 100755
--- a/bgcval2/bgcval2_make_report.py
+++ b/bgcval2/bgcval2_make_report.py
@@ -1480,6 +1480,7 @@ def newImageLocation(fn):
     physicsKM = [
         'AMOC_26N',
         'ADRC_26N',
+        'AtmosCO2',  
         'DrakePassage',
         'GlobalMeanTemperature',
         'GlobalMeanSalinity',
diff --git a/bgcval2/bgcvaltools/pftnames.py b/bgcval2/bgcvaltools/pftnames.py
index 1e8a3d73..1fa0e6e9 100644
--- a/bgcval2/bgcvaltools/pftnames.py
+++ b/bgcval2/bgcvaltools/pftnames.py
@@ -270,6 +270,9 @@ def makeLongNameDict():
 
     lnd['PCO2_SW'] = 'pCO2'
     lnd['pCO2'] = 'pCO2'
+    lnd['AtmospCO2'] = 'pCO2'
+    lnd['AtmosCO2'] = 'Atmospheric CO2'
+
 
     lnd['iron'] = "Iron"
     lnd['Fe_D_CONC_BOTTLE'] = "Iron (Dissolved)"
diff --git a/input_yml/TerraFIRMA_overshoot_co2tests.yml b/input_yml/TerraFIRMA_overshoot_co2tests.yml
index 42887cc0..f01c937b 100644
--- a/input_yml/TerraFIRMA_overshoot_co2tests.yml
+++ b/input_yml/TerraFIRMA_overshoot_co2tests.yml
@@ -16,13 +16,13 @@ clean: True
 savecsv: True
 
 jobs:
-    u-ca306:
+    u-cs495:
        description: 'Reference '
        label: PiControl
        colour: 'blue'
        thickness: 0.6
        linestyle: '-'
-       shifttime: 0.
+       shifttime: -427.
        timerange: [1850, 2020]
        suite: kmf physics bgc #alkalinity physics
 
@@ -35,7 +35,7 @@ jobs:
        shifttime: 0.
        suite: kmf physics bgc #alkalinity physics
 
-    u-cz014:
+    u-cz152:
        description: 'Terrafirma 1% CO2'
        label: '1%CO2'
        colour: 'purple'
diff --git a/input_yml/TerraFIRMA_overshoot_historical.yml b/input_yml/TerraFIRMA_overshoot_historical.yml
index da98cd50..bc046787 100644
--- a/input_yml/TerraFIRMA_overshoot_historical.yml
+++ b/input_yml/TerraFIRMA_overshoot_historical.yml
@@ -23,7 +23,7 @@ jobs:
        colour: 'blue'
        thickness: 0.6
        linestyle: '-'
-       shifttime: 0.
+       shifttime: 0    
        timerange: [1850, 2020]
        suite: kmf physics bgc #alkalinity physics
 
diff --git a/key_files/atmosco2.yml b/key_files/atmosco2.yml
new file mode 100644
index 00000000..9c3e9156
--- /dev/null
+++ b/key_files/atmosco2.yml
@@ -0,0 +1,12 @@
+---
+name            : AtmosCO2
+units           : ppm
+model           : MEDUSA
+modelgrid       : eORCA1
+dimensions      : 2
+modelFiles      : $BASEDIR_MODEL/$JOBID/medusa*$JOBIDo_1y_*_diad-T.nc
+model_vars      : ATM_XCO2
+model_convert   : NoChange
+layers          : layerless
+regions         : Global #gnoreInlandSeas SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific
+
diff --git a/key_lists/atmosco2.yml b/key_lists/atmosco2.yml
new file mode 100644
index 00000000..64b1150f
--- /dev/null
+++ b/key_lists/atmosco2.yml
@@ -0,0 +1,3 @@
+---
+keys:
+    atmosco2: True
diff --git a/key_lists/kmf.yml b/key_lists/kmf.yml
index 08525c8f..b7e2da93 100644
--- a/key_lists/kmf.yml
+++ b/key_lists/kmf.yml
@@ -7,6 +7,7 @@ keys:
     SST: True
     DrakePassageTransport: True
     AMOC_26N: True
+    atmosco2: True
     TotalAirSeaFluxCO2: True
     # NoCaspianAirSeaFluxCO2: True
     # IntPP_OSU: True

From 25e5bec42a8b8532bf2bf35fbcaec524eeb6dda1 Mon Sep 17 00:00:00 2001
From: Lee de Mora <ledm@pml.ac.uk>
Date: Thu, 28 Mar 2024 16:53:05 +0000
Subject: [PATCH 08/14] changed capitals

---
 key_lists/kmf.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/key_lists/kmf.yml b/key_lists/kmf.yml
index b7e2da93..6ec82ec4 100644
--- a/key_lists/kmf.yml
+++ b/key_lists/kmf.yml
@@ -7,7 +7,7 @@ keys:
     SST: True
     DrakePassageTransport: True
     AMOC_26N: True
-    atmosco2: True
+    AtmosCO2: True
     TotalAirSeaFluxCO2: True
     # NoCaspianAirSeaFluxCO2: True
     # IntPP_OSU: True

From 462dd3607a68d5b6ec22f04fc53a545aea9a31bc Mon Sep 17 00:00:00 2001
From: Lee de Mora <ledm@pml.ac.uk>
Date: Fri, 19 Apr 2024 10:36:05 +0100
Subject: [PATCH 09/14] added co2 back and changed line colors

---
 input_yml/TerraFIRMA_overshoot_co2tests.yml   |  2 +-
 input_yml/TerraFIRMA_overshoot_historical.yml | 24 ++++++++++++-------
 key_files/mld.yml                             |  2 +-
 key_lists/physics.yml                         |  3 +++
 4 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/input_yml/TerraFIRMA_overshoot_co2tests.yml b/input_yml/TerraFIRMA_overshoot_co2tests.yml
index f01c937b..95d2969a 100644
--- a/input_yml/TerraFIRMA_overshoot_co2tests.yml
+++ b/input_yml/TerraFIRMA_overshoot_co2tests.yml
@@ -8,7 +8,7 @@ do_analysis_timeseries: True
 do_mass_download: False
 
 # master analysis suite
-master_suites: physics bgc #alkalinity physics kmf1
+master_suites: physics bgc kmf #alkalinity physics kmf1
 
 clean: True
 
diff --git a/input_yml/TerraFIRMA_overshoot_historical.yml b/input_yml/TerraFIRMA_overshoot_historical.yml
index bc046787..a6edb976 100644
--- a/input_yml/TerraFIRMA_overshoot_historical.yml
+++ b/input_yml/TerraFIRMA_overshoot_historical.yml
@@ -17,12 +17,22 @@ clean: True
 savecsv: True
 
 jobs:
-    u-ca306:
-       description: 'Reference '
+    u-cs495:
+       description: 'PiControl'
        label: PiControl
        colour: 'blue'
        thickness: 0.6
        linestyle: '-'
+       shifttime: -427.
+       timerange: [1850, 2020]
+       suite: kmf physics bgc #alkalinity physics
+
+    u-ca306:
+       description: 'Reference '
+       label: Reference
+       colour: 'black'
+       thickness: 0.6
+       linestyle: '-'
        shifttime: 0    
        timerange: [1850, 2020]
        suite: kmf physics bgc #alkalinity physics
@@ -64,11 +74,9 @@ jobs:
        shifttime: 0.
        suite: kmf physics bgc #alkalinity physics
 
-
-
     u-cy690: 
        description: 'static ice sheets, started from picontrol yr 2277'
-       colour: blue
+       colour: green
        label: 'hist static ice'
        thickness: 1.7
        linestyle: '-'
@@ -78,7 +86,7 @@ jobs:
     u-cy691:
        description: 'static ice sheets, started from picontrol yr 2197'
        label: None       
-       colour: blue
+       colour: green
        thickness: 1.7
        linestyle: '-'
        shifttime: 0.
@@ -87,7 +95,7 @@ jobs:
     u-cy692:
        description: ' static ice sheets, started from picontrol yr 2237'
        label: None       
-       colour: blue
+       colour: green
        thickness: 1.7
        linestyle: '-'
        shifttime: 0.
@@ -96,7 +104,7 @@ jobs:
     u-cy693:
        description: ' static ice sheets, started from picontrol yr 2317'
        label: None       
-       colour: blue
+       colour: green
        thickness: 1.7
        linestyle: '-'
        shifttime: 0.
diff --git a/key_files/mld.yml b/key_files/mld.yml
index 4c8c1eef..e6d04417 100644
--- a/key_files/mld.yml
+++ b/key_files/mld.yml
@@ -30,4 +30,4 @@ data_convert:
     maskname : mask
     areafile: $BASEDIR_OBS/IFREMER-MLD/mld_DT02_c1m_reg2.0-annual.nc
 #layers          : Surface
-regions         : Global OnShelf OffShelf ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthAtlanticOcean SouthAtlanticOcean NorthPacificOcean SouthPacificOcean SPNA
+regions         : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthAtlanticOcean SouthAtlanticOcean NorthPacificOcean SouthPacificOcean SPNA
diff --git a/key_lists/physics.yml b/key_lists/physics.yml
index 34f51bf2..a142d1bb 100644
--- a/key_lists/physics.yml
+++ b/key_lists/physics.yml
@@ -17,6 +17,9 @@ keys:
     Salinity: True  # WOA Salinity
 #    soga: True
 
+    # CO2:
+    AtmosCO2: True
+
     # Mixed layer depth keys:
     MLD: True  # iFERMER Mixed Layer Depth
     # MaxMonthlyMLD: True               # MLD Monthly max

From 6489168d9509a363214ee1910f2cfb9a2115f2e0 Mon Sep 17 00:00:00 2001
From: Valeriu Predoi <valeriu.predoi@gmail.com>
Date: Fri, 19 Apr 2024 16:16:18 +0100
Subject: [PATCH 10/14] add jsondiff to pkg requirements

---
 environment.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/environment.yml b/environment.yml
index 3a45b5a3..3a60e75c 100644
--- a/environment.yml
+++ b/environment.yml
@@ -7,6 +7,7 @@ channels:
 dependencies:
   - basemap >=1.3.6
   - cartopy
+  - jsondiff
   - matplotlib
   - nctoolkit >=0.8.7  # use linux64 build
   - netcdf4

From 3564f3ac2d6fc15f09e94ad48f9c6f4da09b0380 Mon Sep 17 00:00:00 2001
From: Valeriu Predoi <valeriu.predoi@gmail.com>
Date: Fri, 19 Apr 2024 16:16:24 +0100
Subject: [PATCH 11/14] add jsondiff to pkg requirements

---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index 134b294f..377b4674 100755
--- a/setup.py
+++ b/setup.py
@@ -26,6 +26,7 @@
     'install': [
         'basemap>=1.3.6',
         'cartopy',
+        'jsondiff',
         'matplotlib',
         'nctoolkit>=0.8.7',  # use linux64 build
         'netcdf4',

From 6f816cdc2653ea1a5ae37702f642af0d25808ca5 Mon Sep 17 00:00:00 2001
From: Lee de Mora <ledm@pml.ac.uk>
Date: Wed, 24 Apr 2024 16:01:22 +0100
Subject: [PATCH 12/14] renamed savecsv to more accurate savejson

---
 bgcval2/analysis_compare.py                   | 15 +++++++--------
 bgcval2/timeseries/timeseriesPlots.py         |  1 -
 bgcval2/timeseries/timeseriesTools.py         |  4 ++--
 input_yml/TerraFIRMA_overshoot_co2tests.yml   |  2 +-
 input_yml/TerraFIRMA_overshoot_historical.yml |  2 +-
 input_yml/TerraFIRMA_overshoot_recovery.yml   |  2 +-
 input_yml/TerraFIRMA_overshoot_runs.yml       |  2 +-
 input_yml/TerraFIRMA_overshoot_stables.yml    |  2 +-
 8 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py
index d459bc46..7f94ac56 100755
--- a/bgcval2/analysis_compare.py
+++ b/bgcval2/analysis_compare.py
@@ -196,7 +196,7 @@ def timeseries_compare(jobs,
                        config_user=None,
                        dpi=None,
                        savepdf=False,
-                       savecsv=False,
+                       savejson=False,
     ):
     """
     timeseries_compare:
@@ -414,8 +414,8 @@ def timeseries_compare(jobs,
 
             ts = 'Together'
 
-            if savecsv:
-                tst.save_csv(
+            if savejson:
+                tst.save_json(
                         timesD,
                         arrD, 
                         analysisname,
@@ -450,7 +450,6 @@ def timeseries_compare(jobs,
                     labels=labels,
                     dpi=dpi,
                     savepdf=savepdf,
-                    savecsv=savecsv,
                 )
 
 
@@ -537,10 +536,10 @@ def load_comparison_yml(master_compare_yml_fn):
     # Image output settings:
     # dpi: pixels per inch (image resolution)
     # savepdf: also save the image as a pdf. 
-    # savecsv: Save the data that appears in the image.
+    # savejson: Save the data that appears in the image.
     details['dpi'] = input_yml_dict.get('dpi', None)
     details['savepdf'] = input_yml_dict.get('savepdf', False)
-    details['savecsv'] = input_yml_dict.get('savecsv', False)
+    details['savejson'] = input_yml_dict.get('savejson', False)
 
 
 
@@ -627,7 +626,7 @@ def load_yml_and_run(compare_yml, config_user, skip_timeseries):
     strictFileCheck = details.get('strictFileCheck', True)
     dpi = details.get('dpi', None)
     savepdf = details.get('savepdf', False)
-    savecsv = details.get('savecsv', False)
+    savejson = details.get('savejson', False)
 
     print('---------------------')
     print('timeseries_compare:',  analysis_name)
@@ -684,7 +683,7 @@ def load_yml_and_run(compare_yml, config_user, skip_timeseries):
         config_user=config_user,
         dpi=dpi,
         savepdf=savepdf,
-        savecsv=savecsv,
+        savejson=savejson,
 
     )
 
diff --git a/bgcval2/timeseries/timeseriesPlots.py b/bgcval2/timeseries/timeseriesPlots.py
index afee9568..44a68e5d 100644
--- a/bgcval2/timeseries/timeseriesPlots.py
+++ b/bgcval2/timeseries/timeseriesPlots.py
@@ -710,7 +710,6 @@ def multitimeseries(
         labels={},
         dpi=None,
         savepdf=False,
-        savecsv=False,
 ):
 
     if 0 in [len(timesD), len(list(timesD.keys()))]: return
diff --git a/bgcval2/timeseries/timeseriesTools.py b/bgcval2/timeseries/timeseriesTools.py
index 7df0bb25..59cab227 100644
--- a/bgcval2/timeseries/timeseriesTools.py
+++ b/bgcval2/timeseries/timeseriesTools.py
@@ -257,7 +257,7 @@ def getHorizontalSlice(nc, coords, details, layer, data=''):
     assert 0
 
 
-def save_csv(
+def save_json(
         timesD,
         arrD, 
         analysisname,
@@ -276,7 +276,7 @@ def save_csv(
         csvformat='.json',
     ):
     """
-    Output the data that appears in a plot as a csv.
+    Output the data that appears in a plot as a json file.
 
     """
     if csvformat.lower() not in ['json', '.json']:
diff --git a/input_yml/TerraFIRMA_overshoot_co2tests.yml b/input_yml/TerraFIRMA_overshoot_co2tests.yml
index 95d2969a..62578012 100644
--- a/input_yml/TerraFIRMA_overshoot_co2tests.yml
+++ b/input_yml/TerraFIRMA_overshoot_co2tests.yml
@@ -13,7 +13,7 @@ master_suites: physics bgc kmf #alkalinity physics kmf1
 clean: True
 
 # Output the figures as csv json files.
-savecsv: True
+savejson: True
 
 jobs:
     u-cs495:
diff --git a/input_yml/TerraFIRMA_overshoot_historical.yml b/input_yml/TerraFIRMA_overshoot_historical.yml
index a6edb976..60cba4e8 100644
--- a/input_yml/TerraFIRMA_overshoot_historical.yml
+++ b/input_yml/TerraFIRMA_overshoot_historical.yml
@@ -14,7 +14,7 @@ master_suites: physics bgc #alkalinity physics kmf1
 clean: True
 
 # Output the figures as csv json files.
-savecsv: True
+savejson: True
 
 jobs:
     u-cs495:
diff --git a/input_yml/TerraFIRMA_overshoot_recovery.yml b/input_yml/TerraFIRMA_overshoot_recovery.yml
index b4168f55..1c4072b7 100644
--- a/input_yml/TerraFIRMA_overshoot_recovery.yml
+++ b/input_yml/TerraFIRMA_overshoot_recovery.yml
@@ -15,7 +15,7 @@ master_suites: physics bgc kmf #alkalinity physics kmf1
 strict_file_check: False
 
 # Output the figures as csv json files.
-savecsv: True
+savejson: True
 
 clean: True
 
diff --git a/input_yml/TerraFIRMA_overshoot_runs.yml b/input_yml/TerraFIRMA_overshoot_runs.yml
index 1765fb0b..104c28f7 100644
--- a/input_yml/TerraFIRMA_overshoot_runs.yml
+++ b/input_yml/TerraFIRMA_overshoot_runs.yml
@@ -15,7 +15,7 @@ master_suites: physics bgc kmf #alkalinity physics kmf1
 strict_file_check: False
 
 # Output the figures as csv json files.
-savecsv: True
+savejson: True
 
 clean: True
 
diff --git a/input_yml/TerraFIRMA_overshoot_stables.yml b/input_yml/TerraFIRMA_overshoot_stables.yml
index 53379031..9b75d417 100644
--- a/input_yml/TerraFIRMA_overshoot_stables.yml
+++ b/input_yml/TerraFIRMA_overshoot_stables.yml
@@ -15,7 +15,7 @@ master_suites: physics bgc kmf #alkalinity physics kmf1
 strict_file_check: False
 
 # Output the figures as csv json files.
-savecsv: True
+savejson: True
 
 clean: True
 

From dae17f1ddcd413fcf098d9f552dae7e9d9b6d2ab Mon Sep 17 00:00:00 2001
From: Lee de Mora <ledm@pml.ac.uk>
Date: Wed, 24 Apr 2024 16:17:57 +0100
Subject: [PATCH 13/14] Added readme details.

---
 README.md | 48 +++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 47 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 2c6167cf..65d1596f 100644
--- a/README.md
+++ b/README.md
@@ -228,6 +228,7 @@ auto_download: <bool>
 strictFileCheck: <bool>
 dpi: <int>
 savepdf: <bool>
+savejson: <bool>
 
 jobs:
    <jobID1>:
@@ -278,7 +279,12 @@ These values are:
    - Output the image as a pdf in addition to the standard png. 
    - This doesn't replace the image in the report, but saves a pdf version of the image in the images directory.
    - The pdfs will be web-visible in the report image directory, but will not linked in the html.
-   - To view a pdf from a report, simply click the image, and replace the `png` extension with `pdf` in the browser path. 
+   - To view a pdf from a report, simply click the image, and replace the `png` extension with `pdf` in the browser path.
+ - `safejson`:
+   - Outputs the data and plotting details of each timeseries plot as a json file.
+   - json is a human readable format that is similar to a python dictionary or a shelve file. 
+   - This file includes all data, time values, units, and line colors, thicknesses and styles.
+
  - `jobs`:
    - A list of jobIDs, and some options on how they will appear in the final report.
    - The options are:
@@ -331,6 +337,46 @@ then the report will appear on the [JASMIN public facing page](https://gws-acces
 which is public facing but password protected.
 
 
+Information about the json output:
+----------------------------------
+
+The `savejson` flag allows bgcval2 to output all data from the multi-model plots to json. 
+Json files are effectively python dictionaries that are saved as human readable ASCII files. 
+This means that they can contain a lot more metadata than a simple csv file. 
+It also means that it's easier to deal with multiple datasets with different time ranges. 
+
+So, in these files, everything is saved as a series of dictionaries, and the main ones are:
+ - `timesD`: The time value (in decimal years)
+ - `arrD`: The data value. 
+
+There's also a lot of info used to make the bgcavl2 plots, including 
+ - `colours`: the colour used by bgcval2 for each jobid
+ - `linestyle`: the plot line style for each job id
+ - `label`: the plot lable for each jobid
+
+Each of these dictionaries uses the jobID as the index, so in python, these can be plotted using the code:
+
+
+```
+import json
+from matplotlib import pyplot
+
+json_file = 'filename.json'
+with open(json_file) as json_file:
+    amoc_data = json.load(json_file)
+
+for jobID in sorted(amoc_data['timesD'].keys()):
+    times = amoc_data['timesD'][jobID]
+    amoc = amoc_data['arrD'][jobID]
+    colour = amoc_data['colours'][jobID]
+    pyplot.plot(times, amoc, c=colour, label=jobID)
+
+pyplot.show()
+```
+
+
+
+
 Batch times series Analysis
 ===========================
 

From c73780e9406ae6b7e5d4a517df7b4abd44f3850d Mon Sep 17 00:00:00 2001
From: Lee de Mora <ledm@pml.ac.uk>
Date: Tue, 30 Apr 2024 15:26:17 +0100
Subject: [PATCH 14/14] replace print with raise

---
 bgcval2/timeseries/timeseriesTools.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/bgcval2/timeseries/timeseriesTools.py b/bgcval2/timeseries/timeseriesTools.py
index 59cab227..3a81eb37 100644
--- a/bgcval2/timeseries/timeseriesTools.py
+++ b/bgcval2/timeseries/timeseriesTools.py
@@ -280,8 +280,7 @@ def save_json(
 
     """
     if csvformat.lower() not in ['json', '.json']:
-        print('Format not set up', csvformat )
-        return
+        raise OSError(''.join(['save_json: format not recognised:', csvformat]))  
     
     filename = bvt.folder(csvFolder) + '_'.join([analysisname, name, region, layer, metric, ts ]) + csvformat