Deleting innecesary data. Check minimum data columns. Fixing small pr…

…oblems in the check config file. Deleting omp in the output variables

Data/Lake1/Data_bubbledissolution.csv

@@ -1,17 +1,17 @@
-Date,Depth,DissPerDepth,Radius,Diss
-2018-06-16,1,1052227.61,109.65,246.68
-2018-06-16,2,332837.84,102.94,107.34
-2018-06-16,3,156383.67,98.05,43.49
-2018-06-16,4,64837.3,92.52,13.03
-2018-06-16,5,40109.38,83.89,6.2
-2018-06-16,6,56648.6,69.7,7.76
-2018-06-16,7,177588.3,47.49,26.49
-2018-06-16,8,315328.51,14.8,234.08
-2018-09-02,1,1052227.61,109.65,246.68
-2018-09-02,2,332837.84,102.94,107.34
-2018-09-02,3,156383.67,98.05,43.49
-2018-09-02,4,64837.3,92.52,13.03
-2018-09-02,5,40109.38,83.89,6.2
-2018-09-02,6,56648.6,69.7,7.76
-2018-09-02,7,177588.3,47.49,26.49
-2018-09-02,8,315328.51,14.8,234.08
+Date,Radius,Diss
+2018-06-16,109.65,246.68
+2018-06-16,102.94,107.34
+2018-06-16,98.05,43.49
+2018-06-16,92.52,13.03
+2018-06-16,83.89,6.2
+2018-06-16,69.7,7.76
+2018-06-16,47.49,26.49
+2018-06-16,14.8,234.08
+2018-09-02,109.65,246.68
+2018-09-02,102.94,107.34
+2018-09-02,98.05,43.49
+2018-09-02,92.52,13.03
+2018-09-02,83.89,6.2
+2018-09-02,69.7,7.76
+2018-09-02,47.49,26.49
+2018-09-02,14.8,234.08

Data/Lake1/Data_transect.csv

@@ -1,21 +1,21 @@
-Sample,Date,Depth,Distance,CH4,dCH4,Tw,pCH4atm,U10,Fa
-T_1,2018-06-16,0.4,8.5,13.377812058339,-54.3928817206795,14,0.1772424,1.34,4.36235162549654
-T_2,2018-06-16,1.9,30.3,7.27441400431991,-52.1528927368675,14,0.1772424,0.894,5.17838241373252
-T_3,2018-06-16,5,51.7,6.30806775271028,-51.9173013476147,15.5,0.1772424,1.437,4.86312752436305
-T_4,2018-06-16,6.7,71.5,6.37851268757941,-51.4627695378245,13,0.1772424,0.423,2.88666660280515
-T_5,2018-06-16,8.9,111.1,5.8214474786461,-51.2851049898339,13,0.1772424,1.195,4.12176201307712
-T_6,2018-06-16,9.4,149.7,6.55034374705138,-53.4789994384007,13,0.1772424,0.628,4.27811736151835
-T_7,2018-06-16,8.6,113.2,5.96880807327061,-52.0512276669877,13.5,0.1772424,0.966,4.1384363647372
-T_8,2018-06-16,8.1,84.7,5.55953980110521,-52.1863277005406,13.5,0.1772424,0.761,9.1963469590645
-T_9,2018-06-16,7,56.5,5.62467320086368,-51.4988047853074,13,0.1772424,0.966,2.84662823584449
-T_10,2018-06-16,1.7,6.59999999999997,5.5069986040029,-51.2270612818495,13.5,0.1772424,0.954,3.63249907662253
-T_1,2018-09-02,0.4,8.5,13.377812058339,-54.3928817206795,14,0.1772424,1.34,4.36235162549654
-T_2,2018-09-02,1.9,30.3,7.27441400431991,-52.1528927368675,14,0.1772424,0.894,5.17838241373252
-T_3,2018-09-02,5,51.7,6.30806775271028,-51.9173013476147,15.5,0.1772424,1.437,4.86312752436305
-T_4,2018-09-02,6.7,71.5,6.37851268757941,-51.4627695378245,13,0.1772424,0.423,2.88666660280515
-T_5,2018-09-02,8.9,111.1,5.8214474786461,-51.2851049898339,13,0.1772424,1.195,4.12176201307712
-T_6,2018-09-02,9.4,149.7,6.55034374705138,-53.4789994384007,13,0.1772424,0.628,4.27811736151835
-T_7,2018-09-02,8.6,113.2,5.96880807327061,-52.0512276669877,13.5,0.1772424,0.966,4.1384363647372
-T_8,2018-09-02,8.1,84.7,5.55953980110521,-52.1863277005406,13.5,0.1772424,0.761,9.1963469590645
-T_9,2018-09-02,7,56.5,5.62467320086368,-51.4988047853074,13,0.1772424,0.966,2.84662823584449
-T_10,2018-09-02,1.7,6.59999999999997,5.5069986040029,-51.2270612818495,13.5,0.1772424,0.954,3.63249907662253
+Sample,Date,Distance,CH4,Tw,pCH4atm,U10,Fa
+T_1,2018-06-16,8.5,13.377812058339,14,0.1772424,1.34,4.36235162549654
+T_2,2018-06-16,30.3,7.27441400431991,14,0.1772424,0.894,5.17838241373252
+T_3,2018-06-16,51.7,6.30806775271028,15.5,0.1772424,1.437,4.86312752436305
+T_4,2018-06-16,71.5,6.37851268757941,13,0.1772424,0.423,2.88666660280515
+T_5,2018-06-16,111.1,5.8214474786461,13,0.1772424,1.195,4.12176201307712
+T_6,2018-06-16,149.7,6.55034374705138,13,0.1772424,0.628,4.27811736151835
+T_7,2018-06-16,113.2,5.96880807327061,13.5,0.1772424,0.966,4.1384363647372
+T_8,2018-06-16,84.7,5.55953980110521,13.5,0.1772424,0.761,9.1963469590645
+T_9,2018-06-16,56.5,5.62467320086368,13,0.1772424,0.966,2.84662823584449
+T_10,2018-06-16,6.59999999999997,5.5069986040029,13.5,0.1772424,0.954,3.63249907662253
+T_1,2018-09-02,8.5,13.377812058339,14,0.1772424,1.34,4.36235162549654
+T_2,2018-09-02,30.3,7.27441400431991,14,0.1772424,0.894,5.17838241373252
+T_3,2018-09-02,51.7,6.30806775271028,15.5,0.1772424,1.437,4.86312752436305
+T_4,2018-09-02,71.5,6.37851268757941,13,0.1772424,0.423,2.88666660280515
+T_5,2018-09-02,111.1,5.8214474786461,13,0.1772424,1.195,4.12176201307712
+T_6,2018-09-02,149.7,6.55034374705138,13,0.1772424,0.628,4.27811736151835
+T_7,2018-09-02,113.2,5.96880807327061,13.5,0.1772424,0.966,4.1384363647372
+T_8,2018-09-02,84.7,5.55953980110521,13.5,0.1772424,0.761,9.1963469590645
+T_9,2018-09-02,56.5,5.62467320086368,13,0.1772424,0.966,2.84662823584449
+T_10,2018-09-02,6.59999999999997,5.5069986040029,13.5,0.1772424,0.954,3.63249907662253

Data/Lake2/Data_bubbledissolution.csv

@@ -1,2 +1,2 @@
-Date,Depth,DissPerDepth,Radius,Diss
-2018-06-18,1,0,0,0
+Date,Radius,Diss
+2018-06-18,0,0

Data/Lake2/Data_transect.csv

@@ -1,12 +1,12 @@
-Sample,Date,Depth,Distance,CH4,dCH4,Tw,pCH4atm,U10,Fa
-T_1,2018-06-18,1,1,0.07,-70,19,0.16,1.792,0.048
-T_2,2018-06-18,3.2,19,0.05,-67.65,19,0.16,1.13,0.082
-T_3,2018-06-18,4.1,33,0.65,-63.63,18,0.16,1.744,0.101
-T_4,2018-06-18,13.6,67,0.04,-64.46,17.5,0.16,1.683,0.102
-T_5,2018-06-18,22.7,110,0.04,-60.22,19,0.16,1.263,0.096
-T_6,2018-06-18,27.5,145,0.05,-61.43,18,0.16,1.335,0.092
-T_7,2018-06-18,30.7,137,0.04,-58.79,19,0.16,2.056,0.12
-T_8,2018-06-18,19,98,0.04,-59.82,19,0.16,1.96,0.126
-T_9,2018-06-18,9.4,55,0.05,-60.4,19,0.16,2.08,0.141
-T_10,2018-06-18,4.6,28,0.04,-60.65,19,0.16,1.623,0.087
-T_11,2018-06-18,1.8,15,0.06,-60.43,19,0.16,1.299,0.087
+Sample,Date,Distance,CH4,Tw,pCH4atm,U10,Fa
+T_1,2018-06-18,1,0.07,19,0.16,1.792,0.048
+T_2,2018-06-18,19,0.05,19,0.16,1.13,0.082
+T_3,2018-06-18,33,0.65,18,0.16,1.744,0.101
+T_4,2018-06-18,67,0.04,17.5,0.16,1.683,0.102
+T_5,2018-06-18,110,0.04,19,0.16,1.263,0.096
+T_6,2018-06-18,145,0.05,18,0.16,1.335,0.092
+T_7,2018-06-18,137,0.04,19,0.16,2.056,0.12
+T_8,2018-06-18,98,0.04,19,0.16,1.96,0.126
+T_9,2018-06-18,55,0.05,19,0.16,2.08,0.141
+T_10,2018-06-18,28,0.04,19,0.16,1.623,0.087
+T_11,2018-06-18,15,0.06,19,0.16,1.299,0.087

Metadata/Metadata_bubbledissolution.csv

@@ -1,7 +1,5 @@
 Column,Units,Description
 ,,
 Date,YYYY-MM-DD,Sample date
-Depth,[m],Depth
 Radius,[m],Distance from center
-DissPerDepth,[umolm-1-d-1],Bubble dissolution per depth
 Diss,[umolm-3-d-1],Bubble dissolution

Metadata/Metadata_transect.csv

@@ -2,10 +2,8 @@ Column,Units,Meaning
 ----------,,
 Sample,,Sample name
 Date,YYYY-MM-DD,Sample date
-Depth,[m],Water depth at sample location
 Distance,[m],Distance from closest shore
 CH4,[mmolm-3],Surface CH4 water concentration
-dCH4,[‰],Carbon CH4 isotopic signature
 Tw,[°C],Surface water temperature at the sample location
 pCH4atm,[Pa],Atmospheric CH4 concentration
 U10,[ms-1],Wind velocity at 10m

scr/functions.py

@@ -4,7 +4,6 @@
 import sys
 import logging
 import numpy as np
-import pandas as pd
 
 def Khmodel(lscale, kind):
     """Horizontal diffusion coefficient (Kh)
@@ -100,7 +99,7 @@ def Hcp(T):
     return Hcp
 
 def param_outputs(cxavg, f_lddata, Rdis, t_lddata, p_lddata, model_conf, opt=None, fa_opt=None, varname_opt=None):
-    OMP = f_lddata.P_avg.values[0]
+    Pnet = f_lddata.P_avg.values[0]
     Fsed = f_lddata.Fs_avg.values[0]
     SurfF = f_lddata.Fa_avg.values[0]
     Fhyp = f_lddata.Fz_avg.values[0]
@@ -113,14 +112,14 @@ def param_outputs(cxavg, f_lddata, Rdis, t_lddata, p_lddata, model_conf, opt=Non
     t_end = model_conf['t_end']
     if model_conf['mode_model']['mode'] == 'OPT' :
         if model_conf['mode_model']['var'] == 'FSED':
-            OMP = 0
-        param = [opt[0], fa_opt, cxavg, Cavg, OMP, Fsed, SurfF, Fhyp, Rdis, zsml, Kh, kch4,
+            Pnet = 0
+        param = [opt[0], fa_opt, cxavg, Cavg, Pnet, Fsed, SurfF, Fhyp, Rdis, zsml, Kh, kch4,
             R, Rs, t_end]
-        nameres = [varname_opt, 'Fa_opt', 'Cmod', 'Cavg', 'OMP', 'Fsed', 'SurfF', 'Fz', 'Rdis',
+        nameres = [varname_opt, 'Fa_opt', 'Cmod', 'Cavg', 'Pnet', 'Fsed', 'SurfF', 'Fz', 'Rdis',
                 'Hsml', 'Kh', 'kch4', 'R', 'Rs', 't_end']
     if model_conf['mode_model']['mode'] == 'EVAL':
-        param = [cxavg, Cavg, OMP, Fsed, SurfF, Fhyp, Rdis, zsml, Kh, kch4, R, Rs, t_end]
-        nameres = ['Cmod', 'Cavg', 'OMP', 'Fsed', 'SurfF', 'Fz', 'Rdis', 'Hsml', 'Kh', 'kch4',
+        param = [cxavg, Cavg, Pnet, Fsed, SurfF, Fhyp, Rdis, zsml, Kh, kch4, R, Rs, t_end]
+        nameres = ['Cmod', 'Cavg', 'Pnet', 'Fsed', 'SurfF', 'Fz', 'Rdis', 'Hsml', 'Kh', 'kch4',
                 'R', 'Rs', 't_end']
     return param, nameres
 

scr/model.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-import sys
 import logging
 import numpy as np
 

scr/read_config.py

@@ -21,7 +21,7 @@ def check_config(conf_run):
     ALLFILES = (TFILE, PFILE, FFILE, DFILE)
     KHMODELS = ('P', 'L')
     K600MODELS = ('VP', 'CC', 'MA-NB', 'MA-PB', 'MA-MB', 'kAVG', '05kAVG', '15kAVG')
-    MODELOPTIONS = ('FSED', 'PNET', 'KH', 'KCH4')
+    MODELOPTIONS = ('FSED', 'PNET', 'KH', 'KCH4','NO-PNET')
 
     check_files(conf_run, ALLFILES)
 
@@ -56,7 +56,7 @@ def check_config(conf_run):
         sys.exit()
 
     if model_conf['mode_model']['mode'] not in ('OPT', 'EVAL'):
-        logging.error('The options for mode are: (OMP, EVAL)')
+        logging.error('The options for mode are: (OPT, EVAL)')
         sys.exit()
 
     if conf_run['Montecarlo']['perform']:

scr/read_inputs.py

@@ -30,19 +30,20 @@ def read_data(conf_run):
 
     logging.info('STEP 1.1: Checking datafiles')
     logging.info('STEP 1.2: Reading input files')
-    tdata = read_inputs(conf_run, allfiles[0])
-    pdata = read_inputs(conf_run, allfiles[1])
-    fdata = read_inputs(conf_run, allfiles[2])
-    ddata = read_inputs(conf_run, allfiles[3])
+    tdata = read_inputs(conf_run, allfiles[0], 'transect')
+    pdata = read_inputs(conf_run, allfiles[1], 'parameters')
+    fdata = read_inputs(conf_run, allfiles[2], 'fluxes')
+    ddata = read_inputs(conf_run, allfiles[3], 'bubble')
     return (ddata, fdata, pdata, tdata)
 
-def read_inputs(conf_run, filename):
+def read_inputs(conf_run, filename, filetype):
     """Reads file define in filename inside conf_run['path']/lake
 
     Parameters
     ----------
     conf_run : dictionary containing the configuration information in config_model.ylm
     filename : string containing the file name to be read it
+    filetype : type of input file (transect, bubble, fluxes, parameters)
 
     Returns
     -------
@@ -55,6 +56,8 @@ def read_inputs(conf_run, filename):
         folder = os.path.join(conf_run['path'], lake)
         rfile = os.path.join(folder, filename)
         datalake = pd.read_csv(rfile, sep=',')
+        datalake.dropna(inplace=True)
+        check_columns(datalake, filetype)
         datalake['Date'] = pd.to_datetime(datalake.Date, format='%Y-%m-%d')
         datalake.insert(0, 'Lake', lake)
         check_dates(conf_run, lake, datalake, filename)
@@ -80,3 +83,20 @@ def select_dates(conf_run, lake, data):
         seldata.append(datadate)
     seldata = pd.concat(seldata, ignore_index=True)
     return seldata
+
+def check_columns(data, filetype):
+
+    mincols = []
+    if filetype == 'transect':
+        mincols = ['Date', 'Distance', 'CH4', 'Tw', 'pCH4atm', 'U10', 'Fa']
+    elif filetype == 'bubble':
+        mincols = ['Date', 'Radius', 'Diss']
+    elif filetype == 'fluxes':
+        mincols = []
+    elif filetype == 'parameters':
+        mincols = []
+
+    if not set(mincols).issubset(set(data.columns)):
+        notin = ','.join(set(mincols).difference(data.columns))
+        logging.error('Columns not found: {}'.format(notin))
+        sys.exit('PLEASE SEE ERROR ABOVE')