Merge branch 'main' into pep8_linting

Conflicts:
	bgcval2/analysis_compare.py

README.md

@@ -229,6 +229,7 @@ strictFileCheck: <bool>
 jobs:
    <jobID1>:
       description: <descrption of the first job>
+      label: 'Job info'
       colour: red
       thickness: 0.7
       linestyle: '-'
@@ -271,6 +272,10 @@ These values are:
    - The options are:
      - `description`:
        - A description of job, which helps people differentiate the jobs in the final report.
+     - `label`:
+       - A short description of the job which will appear in the plot legend.
+       - If you make it too long, it won't fit on the plot.  
+       - Optional: Default behaviour is the jobID.
      - `colour`:
        - The colour of this jobs line in the report plots.
        - Default colour is a randomly generated hex colour.
@@ -447,6 +452,8 @@ units: Units
 dimensions: 1,2 or  3 # The number of dimensions after the calculations are performed
 layers          : Surface
 regions         : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific
+smoothings      : DataOnly both5 both30 movingav30years 5and30 30and100
+
 
 #paths:
 modelFiles      : $BASEDIR_MODEL/$JOBID/nemo_$JOBIDo_1y_*_grid-T.nc
@@ -500,6 +507,27 @@ basic functions such as multiply by or add to, or `noChange`, which can all be
 called without providing the `path`, and which may have their own key word
 arguments.
 
+Some of the other options for each analysis include:
+  - `layers`:
+    - The z axis layers that you want to use.
+    - Typically include things like `Surface`, `500m`, `1000m` etc.
+    - Note that this is a pre-curated list, so you can't use it to select a specific depth (like `327m` or something)
+  - `regions`:
+    - A list of regions to investigate.
+    - This is a pre-curated list, and they are defined in so you can't.
+    - These regions are defined in the file `bgcval2/bgcvaltools/makeMask.py`.
+  -  `smoothings`:
+    - This is the smoothing function (if any) to apply to the data before plotting it.
+    - The smoothing it added before plotting, but after saving the shelve file, so it doesn't impact the data.
+    - No smoothing is `DataOnly`, which is also the default behaviour. 
+    - If several smoothing options are added here, bgcval2 will generate a comparison plot for each one.
+    - The smoothings are defined and performed in `bgcval2/timeseries/timeseriesPlots.py`
+    - Other modes exist:
+      - `movingav30years`: A 30 year moving average
+      - `both5`: Both no smoothing and a 5 year moving average
+      - `both30`: Both no smoothing and a 30 year moving average
+      - `5and30`: a 5 year moving average and a 30 year moving average.
+
 Clearing the Cache
 ------------------
 

bgcval2/analysis_compare.py

@@ -176,6 +176,7 @@ def timeseries_compare(jobs,
                        jobDescriptions={},
                        lineThicknesses=defaultdict(lambda: 1),
                        linestyles=defaultdict(lambda: '-'),
+                       labels = {},
                        ensembles={},
                        config_user=None,
 ):
@@ -364,7 +365,8 @@ def timeseries_compare(jobs,
         regions = av[name]['regions']
         layers = av[name]['layers']
         metrics = av[name]['metrics']
-        for region, layer, metric in itertools.product(regions, layers, metrics):
+        smoothings = av[name]['smoothings']
+        for region, layer, metric  in itertools.product(regions, layers, metrics):
             timesD = {}
             arrD = {}
             title = titleify([region, layer, metric, name])
@@ -387,20 +389,22 @@ def timeseries_compare(jobs,
             units = av[name]['modeldetails']['units']
 
             ts = 'Together'
-            for ls in ['DataOnly', ]:
+            for smoothing in smoothings:
+            #or ls in ['DataOnly', ]:
                 tsp.multitimeseries(
                     timesD,  # model times (in floats)
                     arrD,  # model time series
                     data=-999,  # in situ data distribution
                     title=title,
-                    filename=bvt.folder(image_folder) +
-                        '_'.join([name, region, layer, ts, ls + '.png']),
+                    filename=bvt.folder(imageFolder) +
+                        '_'.join([name, region, layer, ts, smoothing + '.png']),
                     units=units,
                     plotStyle=ts,
-                    lineStyle=ls,
+                    smoothing=smoothing,
                     colours=colours,
                     thicknesses=lineThicknesses,
                     linestyles=linestyles,
+                    labels=labels,
                 )
 
     # Generate a list of comparison images:
@@ -497,7 +501,8 @@ def load_comparison_yml(master_compare_yml_fn):
     descriptions = {}
     shifttimes = {}  # number of years to shift time axis.
     timeranges = {}
-
+    labels = {}
+
     for jobID, job_dict in details['jobs'].items():
         if job_dict.get('colour', False):
             colours[jobID] = job_dict['colour']
@@ -512,13 +517,16 @@ def load_comparison_yml(master_compare_yml_fn):
         timeranges[jobID] = job_dict.get('timerange', None)
         suites[jobID] = job_dict.get('suite', default_suite)
         auto_download_dict[jobID] = job_dict.get('auto_download', auto_download_dict[jobID]) 
+        labels[jobID] = job_dict.get('label', jobID)
+
 
     details['colours'] = colours
     details['descriptions'] = descriptions
     details['thicknesses'] = thicknesses
     details['linestyles'] = linestyles
     details['shifttimes'] = shifttimes
     details['timeranges'] = timeranges
+    details['labels'] = labels
     details['suites'] = suites
     details['auto_download'] = auto_download_dict
     return details
@@ -538,7 +546,7 @@ def load_yml_and_run(compare_yml, config_user, skip_timeseries):
     do_mass_download = details['do_mass_download']
     master_suites = details['master_suites']
 
-    if skip_timeseries == None:
+    if skip_timeseries is None:
         pass
     else:
         do_analysis_timeseries = not skip_timeseries
@@ -549,6 +557,7 @@ def load_yml_and_run(compare_yml, config_user, skip_timeseries):
     descriptions = details['descriptions']
     shifttimes = details['shifttimes']
     timeranges = details['timeranges']
+    labels = details['labels']
     suites = details['suites']
     auto_download = details['auto_download']
     strict_file_check = details.get('strictFileCheck', True)
@@ -561,6 +570,7 @@ def load_yml_and_run(compare_yml, config_user, skip_timeseries):
         print(jobID, 'colour:', colours[jobID])
         print(jobID, 'line thickness & style:', thicknesses[jobID], linestyles[jobID])
         print(jobID, 'Shift time by', shifttimes[jobID])
+        print(jobID, 'Label: ', labels[jobID])
         print(jobID, 'Time range (None means all):', timeranges.get(jobID, None))
         print(jobID, 'suite:', suites[jobID])
         print(jobID, 'auto_download', auto_download[jobID])
@@ -603,6 +613,7 @@ def load_yml_and_run(compare_yml, config_user, skip_timeseries):
         analysisname=analysis_name,
         lineThicknesses=thicknesses,
         linestyles=linestyles,
+        labels=labels,
         config_user=config_user,
     )
 

bgcval2/analysis_timeseries.py

@@ -297,6 +297,11 @@ def load_key_file(key, paths, jobID, strictFileCheck=True):
     output_dict['metrics'] = key_dict.get('metrics', metricList)
     output_dict['metrics'] = parse_list_from_string(output_dict['metrics'])
 
+    # Smoothings:
+    default_smoothings = ['DataOnly', ]
+    output_dict['smoothings'] = key_dict.get('smoothings', default_smoothings)
+    output_dict['smoothings'] = parse_list_from_string(output_dict['smoothings'])
+
     # Load Grid:
     gridFile = key_dict.get('gridFile', paths.orcaGridfn)
     output_dict['gridFile'] = list_input_files(gridFile, key_dict, paths)[0]

bgcval2/download_from_mass.py

@@ -37,6 +37,7 @@
 from socket import gethostname
 import shutil
 import os
+from pwd import getpwuid
 import stat
 from glob import glob
 from re import findall
@@ -407,9 +408,27 @@ def download_from_mass(
 
     paths = get_paths(config_user)
     outputFold = folder([paths.ModelFolder_pref,  jobID,] )
-    # make this folder group writeable. 
     st = os.stat(outputFold)
-    os.chmod(outputFold, st.st_mode | stat.S_IWGRP)
+
+    # Check permissions on the output folder 
+    i_can_write_this = os.access(outputFold, os.W_OK)
+    owner = getpwuid(st.st_uid).pw_name
+    my_username = os.environ.get('USERNAME')
+
+    i_own_this = my_username == owner
+    #group_can_write_this = os.access(outputFold, os.W_OK)
+    if i_can_write_this and i_own_this:
+        # I created this folder and I own it.
+        # make this folder group writeable.:
+        os.chmod(outputFold, st.st_mode | stat.S_IWGRP)
+    elif i_can_write_this and not i_own_this:
+        pass
+    else:
+        # Someone else created it and they own it,
+        # so I can't change permissions.
+        owner = getpwuid(st.st_uid).pw_name
+        print('WARNING: someone else (', owner, ') owns this directory, so you may not be able to download files.')
+        print('WARNING: Ask them politely to run "chmod g+w ',outputFold,'"')
 
     deleteBadLinksAndZeroSize(outputFold, jobID)
 
@@ -483,8 +502,32 @@ def download_from_mass(
     if auto_download:
         shared_file_path = os.path.join(paths.shared_mass_scripts, os.path.basename(download_script_path))
         print('writing file in shared path', shared_file_path)
-        shutil.copy(download_script_path, shared_file_path)
 
+        # check destination existance and permissions
+        if os.path.exists(shared_file_path):
+            # check destination permissions
+            i_can_write_this = os.access(shared_file_path, os.W_OK)
+        else:
+            # File doesn't exist!
+            i_can_write_this = True
+
+        if i_can_write_this:
+            # I created this file and I own it:
+            # make local copy group writable:
+            os.chmod(download_script_path, st.st_mode | stat.S_IWGRP)
+
+            # copy local to shared folder:
+            if not os.path.exists(shared_file_path): 
+                shutil.copy(download_script_path, shared_file_path)
+
+                # make shared copy group writable: (possibly overkill)
+                os.chmod(shared_file_path, st.st_mode | stat.S_IWGRP)
+        else:
+            # I don't have permission to edit this file. Someone else does:
+            uname = getpwuid(os.stat(shared_file_path).st_uid).pw_name
+            print('WARNING: someone else (', shared_file_path, ') owns this directory, so you may not be able to download files.')
+            print('WARNING: Ask them politely to run "chmod g+w ', shared_file_path,'"')
+
     fixFilePaths(outputFold, jobID, debug=False,)
     deleteBadLinksAndZeroSize(outputFold, jobID, debug=False,)
 
@@ -598,13 +641,14 @@ def deleteBadLinksAndZeroSize(outputFold, jobID, debug=True):
     bashCommand1 = "find " + outputFold + "/. -size 0 -print -delete"
     bashCommand2 = "find -L " + outputFold + "/. -type l -delete  -print"
 
-    if debug: print("deleteBadLinksAndZeroSize:\t", bashCommand1)
+    bashCommand1 = bashCommand1.replace('//', '/')
+    bashCommand2 = bashCommand2.replace('//', '/')
 
+    print("deleteBadLinksAndZeroSize:\t", bashCommand1)
     process1 = subprocess.Popen(bashCommand1.split(), stdout=subprocess.PIPE)
     output1 = process1.communicate()[0]
 
-    if debug: print("deleteBadLinksAndZeroSize:\t", bashCommand2)
-
+    print("deleteBadLinksAndZeroSize:\t", bashCommand2)
     process2 = subprocess.Popen(bashCommand2.split(), stdout=subprocess.PIPE)
     output2 = process2.communicate()[0]