From 228090c3c0192c4997250293974f5c584f9fd915 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 8 Aug 2022 13:41:31 +0100 Subject: [PATCH 01/37] add help and a couple checks --- bgcval2/analysis_compare.py | 44 +++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index dd5d2922..3a625ca6 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -28,9 +28,11 @@ :platform: Unix :synopsis: A script to produce an intercomparison of multiple runs the time series analyses. .. moduleauthor:: Lee de Mora +.. moduleauthor:: Valeriu Predoi """ +import argparse import matplotlib # Force matplotlib to not use any Xwindows backend. matplotlib.use('Agg') @@ -4149,6 +4151,11 @@ def load_comparison_yml(master_compare_yml_fn): with open(master_compare_yml_fn, 'r') as openfile: dictionary = yaml.safe_load(openfile) + if not dictionary: + print(f"Configuration file {master_compare_yml_fn} " + "is either empty or corrupt, please check its contents") + sys.exit(1) + details = {} details['name'] = dictionary.get('name', False) details['jobs'] = dictionary.get('jobs', False) @@ -4204,21 +4211,40 @@ def load_comparison_yml(master_compare_yml_fn): details['shifttimes'] = shifttimes details['suites'] = suites return details - + + +def get_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('-c', + '--config-file', + default=os.path.join(os.getcwd(), + 'bgcval2-config-user.yml'), + help='User configuration file') + + args = parser.parse_args() + + return args + def main(): - if "--help" in argv or len(argv) == 1: - print("Running with no arguments. Exiting.") - if "--help" in argv: - print("Read the documentation.") - exit(0) + """Run the main routine.""" + args = get_args() config_user=None - if "bgcval2-config-user.yml" in argv[1:]: - config_user = "bgcval2-config-user.yml" + if args.config_file: + config_user = os.path.join(os.getcwd(), args.config_file) print(f"analysis_timeseries: Using user config file {config_user}") + else: + config_user = os.path.join(os.getcwd(), "bgcval2-config-user.yml") + print(f"analysis_timeseries: Using user default file {config_user}") + if not os.path.isfile(config_user): + print(f"analysis_timeseries: Could not find configuration file {config_user}") + sys.exit(1) - details = load_comparison_yml(argv[1]) + details = load_comparison_yml(config_user) jobs = details['jobs'] analysis_name = details['name'] From 9f1d3ab457d92f5bf73f430b02d9f57bb3505740 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 8 Aug 2022 14:21:08 +0100 Subject: [PATCH 02/37] add help to download from mass --- bgcval2/download_from_mass.py | 49 +++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/bgcval2/download_from_mass.py b/bgcval2/download_from_mass.py index 03f87544..3b0a8667 100755 --- a/bgcval2/download_from_mass.py +++ b/bgcval2/download_from_mass.py @@ -30,7 +30,9 @@ """ ##### # Load Standard Python modules: -from sys import argv, stdout +import argparse + +from sys import stdout import subprocess from socket import gethostname import os @@ -598,29 +600,54 @@ def deleteBadLinksAndZeroSize(outputFold, jobID): process2 = subprocess.Popen(bashCommand2.split(), stdout=subprocess.PIPE) output2 = process2.communicate()[0] + def pop_keys(keys, remove_keys): for k in remove_keys: - keys.remove(k) + if k in keys: + keys.remove(k) + return keys +def get_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('-i', + '--job-id', + default=None, + help='Job ID', + required=True) + parser.add_argument('-k', + '--keys', + default=None, + nargs='+', type=str, + help='Runtime keys', + required=False) + args = parser.parse_args() + + return args + + def main(): - try: - jobID = argv[1] - except: - print("Please provide a jobID") - sys.exit(0) - try: - keys = argv[2:] - except: + """Run the main routine.""" + args = get_args() + jobID = args.job_id + keys = args.keys + if keys is None: keys = [] + if keys: + keys = [str(k) for k in keys] + print(f"Running with job_id {jobID} and keys {keys}") ##### # Default behaviour is to download annual files if 'noMoo' in keys or 'dryrun' in keys or '--dry-run' in keys: doMoo=False dryrun = True - keys = pop_keys(keys, ['noMoo', 'dryrun', '--dry-run']) + if keys: + keys = pop_keys(keys, ['noMoo', 'dryrun', '--dry-run']) else: doMoo=True dryrun=False From 00d0d22b84cd747f38f37880929995104bfed3d2 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 8 Aug 2022 14:38:45 +0100 Subject: [PATCH 03/37] analysis timeseries help stuff --- bgcval2/analysis_timeseries.py | 76 ++++++++++++++++++++++++---------- 1 file changed, 55 insertions(+), 21 deletions(-) diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index 6560e0f6..ff2917b1 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -4927,44 +4927,78 @@ def singleTimeSeries( # print "Error: %s" % sys.exc_info()[0] +def get_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('-c', + '--config-file', + default=os.path.join(os.getcwd(), + 'bgcval2-config-user.yml'), + help='User configuration file') + parser.add_argument('-i', + '--job-id', + default=None, + help='Job ID', + required=True) + parser.add_argument('-k', + '--keys', + default=None, + nargs='+', type=str, + help='Runtime keys', + required=False) + + args = parser.parse_args() + + return args + + def main(): from ._version import __version__ print(f'BGCVal2: {__version__}') - if "--help" in argv or len(argv) == 1: - print("Running with no arguments. Exiting.") - if "--help" in argv: - print("Read the documentation.") - sys.exit(0) - try: - jobID = argv[1] - except: - jobID = "u-ab749" - - if 'debug' in argv[1:]: + args = get_args() + jobID = args.job_id + keys = args.keys + if keys is None: + keys = [] + if keys: + keys = [str(k) for k in keys] + # FIXME this was an exception previously, what do with it? + jobID = "u-ab749" + + if 'debug' in keys: suite = 'debug' #elif 'all' in argv[1:]: suite = 'all' - elif 'spinup' in argv[1:]: + elif 'spinup' in keys: suite = 'spinup' - elif 'salinity' in argv[1:]: + elif 'salinity' in keys: suite = 'salinity' - elif 'level1' in argv[1:]: + elif 'level1' in keys: suite = 'level1' - elif 'fast' in argv[1:]: + elif 'fast' in keys: suite = 'fast' - elif 'level3' in argv[1:]: + elif 'level3' in keys: suite = 'level3' - elif 'physics' in argv[1:]: + elif 'physics' in keys: suite = 'physics' - elif 'bgc' in argv[1:]: + elif 'bgc' in keys: suite = 'bgc' - elif 'kmf' in argv[1:] or 'keymetricsfirst' in argv[1:]: + elif 'kmf' in keys or 'keymetricsfirst' in keys: suite = 'keymetricsfirst' else: suite = 'level1' + config_user = None - if "bgcval2-config-user.yml" in argv[1:]: - config_user = "bgcval2-config-user.yml" + if args.config_file: + config_user = os.path.join(os.getcwd(), args.config_file) print(f"analysis_timeseries: Using user config file {config_user}") + else: + config_user = os.path.join(os.getcwd(), "bgcval2-config-user.yml") + print(f"analysis_timeseries: Using user default file {config_user}") + if not os.path.isfile(config_user): + print(f"analysis_timeseries: Could not find configuration file {config_user}") + sys.exit(1) analysis_timeseries( jobID=jobID, From 01e4f180bf900da6f19ca27715f89dd53270cd23 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 8 Aug 2022 14:39:51 +0100 Subject: [PATCH 04/37] missing module argparse --- bgcval2/analysis_timeseries.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index ff2917b1..467502f0 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -31,6 +31,7 @@ .. moduleauthor:: Lee de Mora """ +import argparse import matplotlib as mpl mpl.use('Agg') From 887b6454ae26b32895c1b90cd32d2d4adea66f38 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 8 Aug 2022 15:35:19 +0100 Subject: [PATCH 05/37] make config user optional --- bgcval2/analysis_timeseries.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index 467502f0..23d5e32f 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -38,7 +38,7 @@ ##### # Load Standard Python modules: -from sys import argv, exit +from sys import exit from os.path import exists from calendar import month_name from socket import gethostname @@ -4990,7 +4990,6 @@ def main(): else: suite = 'level1' - config_user = None if args.config_file: config_user = os.path.join(os.getcwd(), args.config_file) print(f"analysis_timeseries: Using user config file {config_user}") @@ -4998,8 +4997,9 @@ def main(): config_user = os.path.join(os.getcwd(), "bgcval2-config-user.yml") print(f"analysis_timeseries: Using user default file {config_user}") if not os.path.isfile(config_user): - print(f"analysis_timeseries: Could not find configuration file {config_user}") - sys.exit(1) + print(f"analysis_timeseries: Could not find configuration file {config_user}." + "Will proceed with defaults.") + config_user = None analysis_timeseries( jobID=jobID, From 51f4c0db2dff4701a46e173c996cacf23b4630c7 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 8 Aug 2022 15:45:14 +0100 Subject: [PATCH 06/37] make user config optional, comparison config required --- bgcval2/analysis_compare.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index 3a625ca6..b3198c85 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -4221,8 +4221,15 @@ def get_args(): parser.add_argument('-c', '--config-file', default=os.path.join(os.getcwd(), - 'bgcval2-config-user.yml'), - help='User configuration file') + 'config-user.yml'), + help='User configuration file', + required=False) + parser.add_argument('-b', + '--comparison-config', + default=os.path.join(os.getcwd(), + 'bgcval2-comparison-config.yml'), + help='Comparison configuration file', + required=True) args = parser.parse_args() @@ -4233,7 +4240,16 @@ def main(): """Run the main routine.""" args = get_args() - config_user=None + if args.comparison_config: + comp_config = os.path.join(os.getcwd(), args.comparison_config) + print(f"analysis_timeseries: Comparison config file {comp_config}") + else: + comp_config = os.path.join(os.getcwd(), "comparison.yml") + print(f"analysis_timeseries: Using user default file {comp_config}") + if not os.path.isfile(comp_config): + print(f"analysis_timeseries: Could not find comparison config file {comp_config}") + sys.exit(1) + if args.config_file: config_user = os.path.join(os.getcwd(), args.config_file) print(f"analysis_timeseries: Using user config file {config_user}") @@ -4242,7 +4258,7 @@ def main(): print(f"analysis_timeseries: Using user default file {config_user}") if not os.path.isfile(config_user): print(f"analysis_timeseries: Could not find configuration file {config_user}") - sys.exit(1) + config_user = None details = load_comparison_yml(config_user) From 1a3a2951a9dfc5da16124694bbb5b55264ceee8f Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 8 Aug 2022 15:50:25 +0100 Subject: [PATCH 07/37] fixed a bug --- bgcval2/analysis_compare.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index b3198c85..9f91acf4 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -4151,12 +4151,12 @@ def load_comparison_yml(master_compare_yml_fn): with open(master_compare_yml_fn, 'r') as openfile: dictionary = yaml.safe_load(openfile) - if not dictionary: + if not dictionary or not isinstance(dictionary, dict): print(f"Configuration file {master_compare_yml_fn} " "is either empty or corrupt, please check its contents") sys.exit(1) - details = {} + details = {} details['name'] = dictionary.get('name', False) details['jobs'] = dictionary.get('jobs', False) @@ -4260,7 +4260,7 @@ def main(): print(f"analysis_timeseries: Could not find configuration file {config_user}") config_user = None - details = load_comparison_yml(config_user) + details = load_comparison_yml(comp_config) jobs = details['jobs'] analysis_name = details['name'] From a419ef0f19edf7489b0dfee23f8c482092cad576 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 8 Aug 2022 15:59:14 +0100 Subject: [PATCH 08/37] added default for comparison config --- bgcval2/analysis_compare.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index 9f91acf4..6589ba4c 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -4226,10 +4226,10 @@ def get_args(): required=False) parser.add_argument('-b', '--comparison-config', - default=os.path.join(os.getcwd(), - 'bgcval2-comparison-config.yml'), - help='Comparison configuration file', - required=True) + default=os.path.join(os.path.dirname( + os.path.dirname(os.path.abspath(__file__))), + 'input_yml/comparison_analysis_template.yml'), + help='Comparison configuration file') args = parser.parse_args() From 57ee155388cdbcb4b530bb1e44211f59a161984c Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 8 Aug 2022 16:03:04 +0100 Subject: [PATCH 09/37] explaning command line arguments --- bgcval2/analysis_compare.py | 16 ++++++++++++-- bgcval2/analysis_timeseries.py | 38 ++++++++++++++++++++++------------ 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index 3a625ca6..24c3ac92 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -4218,11 +4218,20 @@ def get_args(): parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + + + parser.add_argument('INPUT_FILE', '*', + required=True, + help='Comparison Analysis Configuration file') + parser.add_argument('-c', '--config-file', + required=False, default=os.path.join(os.getcwd(), 'bgcval2-config-user.yml'), - help='User configuration file') + help='User configuration file (paths)') + + args = parser.parse_args() @@ -4232,7 +4241,7 @@ def get_args(): def main(): """Run the main routine.""" args = get_args() - + config_user=None if args.config_file: config_user = os.path.join(os.getcwd(), args.config_file) @@ -4244,6 +4253,9 @@ def main(): print(f"analysis_timeseries: Could not find configuration file {config_user}") sys.exit(1) + + # Below here is analysis + details = load_comparison_yml(config_user) jobs = details['jobs'] diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index 23d5e32f..a42069d7 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -318,7 +318,7 @@ def analysis_timeseries( analysisKeys = [] if analysisSuite.lower() in [ - 'keymetricsfirst', + 'keymetricsfirst', 'kmf', ]: analysisKeys.extend(keymetricsfirstKeys) @@ -4961,13 +4961,24 @@ def main(): args = get_args() jobID = args.job_id keys = args.keys - if keys is None: - keys = [] - if keys: - keys = [str(k) for k in keys] - # FIXME this was an exception previously, what do with it? - jobID = "u-ab749" + accepted_keys = ['kmf', 'physics','bgc', 'debug', 'spinup', 'salinity', 'level1', 'level3', ] + good_keys = True + for key in keys: + if key not in accepted_keys: + print('Key Argument [',key,'] nor recognised. Accepted keys are:', accepted_keys) + good_keys= False + if good_keys: + sys.exit(0) + + #f keys is None: + # keys = [] + #f keys: + # keys = [str(k) for k in keys] + # FIXME this was an exception previously, what do with it? + #jobID = "u-ab749" + + """ if 'debug' in keys: suite = 'debug' #elif 'all' in argv[1:]: suite = 'all' @@ -4989,7 +5000,7 @@ def main(): suite = 'keymetricsfirst' else: suite = 'level1' - + """ if args.config_file: config_user = os.path.join(os.getcwd(), args.config_file) print(f"analysis_timeseries: Using user config file {config_user}") @@ -5001,11 +5012,12 @@ def main(): "Will proceed with defaults.") config_user = None - analysis_timeseries( - jobID=jobID, - analysisSuite=suite, - config_user=config_user - ) #clean=1) + for suite in keys: + analysis_timeseries( + jobID=jobID, + analysisSuite=suite, + config_user=config_user + ) #clean=1) #if suite == 'all': #analysis_timeseries(jobID =jobID,analysisSuite='FullDepth', z_component = 'FullDepth',)#clean=1) From aa6832e19a76e25bf4f5cd63cff73b2d1496252f Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 8 Aug 2022 16:09:31 +0100 Subject: [PATCH 10/37] improve a bit message --- bgcval2/_runtime_config.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bgcval2/_runtime_config.py b/bgcval2/_runtime_config.py index 97ae7c61..075949dc 100644 --- a/bgcval2/_runtime_config.py +++ b/bgcval2/_runtime_config.py @@ -64,9 +64,10 @@ def _establish_hostname(): hostname = "github-actions" # for testing on GA machine else: host = gethostname() - print("Got host name: ", host) - raise ValueError(f"Unidentified hostname {host}" - f"Run at either JASMIN, MONSOON or PML.") + print(f"Got host name: {host}") + raise ValueError(f"Unidentified hostname {host} - " + f"Run at either JASMIN, MONSOON or PML please." + " Exiting now, please log in one of the above clusters.") return hostname From 9fc820e8bb0828a682ad66e3644c4de6a9e01d34 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 8 Aug 2022 16:32:59 +0100 Subject: [PATCH 11/37] make reports helper --- bgcval2/bgcval2_make_report.py | 104 ++++++++++++++++++++++----------- 1 file changed, 69 insertions(+), 35 deletions(-) diff --git a/bgcval2/bgcval2_make_report.py b/bgcval2/bgcval2_make_report.py index 9c94ecd8..5ffd076e 100755 --- a/bgcval2/bgcval2_make_report.py +++ b/bgcval2/bgcval2_make_report.py @@ -30,10 +30,10 @@ ##### # Load Standard Python modules: +import argparse import sys from glob import glob -from sys import argv import os import shutil @@ -1725,21 +1725,65 @@ def newImageLocation(fn): print("-------------\nSuccess\ntest with:\nfirefox", indexhtmlfn) +def get_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('-c', + '--config-file', + default=os.path.join(os.getcwd(), + 'config-user.yml'), + help='User configuration file', + required=False) + parser.add_argument('-i', + '--job-id', + default=None, + help='Job ID', + required=True) + parser.add_argument('-y', + '--year', + default=None, + help='Year', + required=False) + parser.add_argument('-a', + '--clean', + action='store_true', + help='Clean or not', + required=False) + parser.add_argument('-p', + '--physics', + action='store_true', + help='Physics or not', + required=False) + parser.add_argument('-r', + '--report', + default=None, + help='Report repo', + required=False) + + args = parser.parse_args() + + return args + + def main(): """Run the html maker for a single job ID.""" from ._version import __version__ print(f'BGCVal2: {__version__}') - if "--help" in argv or len(argv) == 1: - print("Running with no arguments. Exiting.") - if "--help" in argv: - print("Read the documentation.") - sys.exit(0) - try: - jobID = argv[1] - except: - print("Please provide a jobID next time") - exit() + args = get_args() + jobID = args.job_id + + if args.config_file: + config_user = os.path.join(os.getcwd(), args.config_file) + print(f"analysis_timeseries: Using user config file {config_user}") + else: + config_user = os.path.join(os.getcwd(), "bgcval2-config-user.yml") + print(f"analysis_timeseries: Using user default file {config_user}") + if not os.path.isfile(config_user): + print(f"analysis_timeseries: Could not find configuration file {config_user}") + config_user = None #defaults: clean = False @@ -1747,31 +1791,21 @@ def main(): year = '*' reportdir = folder('reports/' + jobID) - for i, arg in enumerate(argv): - if i <= 1: continue - + if args.year: try: - y = int(arg) - year = y - continue - except: - pass - - if arg == 'clean': - clean = True - continue - - if arg == 'physics': - physicsOnly = True - continue - - reportdir = arg - - # get runtime configuration - config_user = None - if "bgcval2-config-user.yml" in argv[1:]: - config_user = "bgcval2-config-user.yml" - print(f"bgcval2_make_report: Using user config file {config_user}") + year = int(args.year) + except ValueError: + print("analysis_timeseries: Invalid input for year - must be an integer, got {args.year}") + if args.clean: + clean = True + print("analysis_timeseries: Running with Clean option!") + if args.physics: + physicsOnly = True + print("analysis_timeseries: Running with Physics option!") + if args.report: + reportdir = os.path.abspath(args.report) + + # get runtime configuration; not implemented yet if config_user: paths_dict, config_user = get_run_configuration(config_user) else: From b360137c4747d2b580e718c9187e7efd011a289c Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 8 Aug 2022 16:34:48 +0100 Subject: [PATCH 12/37] clearer help messages --- bgcval2/analysis_compare.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index 4720fbfc..aecb8bc5 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -26,7 +26,7 @@ """ .. module:: analysis_compare :platform: Unix - :synopsis: A script to produce an intercomparison of multiple runs the time series analyses. + :synopsis: A tool that generates an intercomparison of multiple UKESM jobs time series analyses. .. moduleauthor:: Lee de Mora .. moduleauthor:: Valeriu Predoi @@ -4219,18 +4219,16 @@ def get_args(): description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('-b', '--comparison-config', - help='Comparison Analysis configuration file', + help='Comparison Analysis configuration file, for examples see bgcval2 input_yml directory.', required=True,) - parser.add_argument('-c', '--config-file', default=os.path.join(os.getcwd(), 'config-user.yml'), - help='User configuration file', + help='User configuration file (for paths)', required=False) args = parser.parse_args() @@ -4251,6 +4249,7 @@ def main(): comp_config = os.path.join(os.getcwd(), "comparison.yml") # This should never happen as this argument is required. print(f"analysis_timeseries: Using user default file {comp_config}") + if not os.path.isfile(comp_config): print(f"analysis_timeseries: Could not find comparison config file {comp_config}") sys.exit(1) From ba028a324a2e30f23721f80d47fdef5406423257 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 8 Aug 2022 16:44:42 +0100 Subject: [PATCH 13/37] Added more complex help --- bgcval2/analysis_timeseries.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index a42069d7..11b3bd14 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -4930,6 +4930,8 @@ def singleTimeSeries( def get_args(): """Parse command line arguments.""" + accepted_keys = ['kmf', 'physics','bgc', 'debug', 'spinup', 'salinity', 'level1', 'level3', ] + parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) @@ -4937,17 +4939,20 @@ def get_args(): '--config-file', default=os.path.join(os.getcwd(), 'bgcval2-config-user.yml'), - help='User configuration file') - parser.add_argument('-i', - '--job-id', + help='User configuration file', + required=False) + parser.add_argument('-j', + '--jobID', default=None, - help='Job ID', + help='Job ID (automatically generated by cycl/rose suite for UKESM jobs).', required=True) parser.add_argument('-k', '--keys', - default=None, - nargs='+', type=str, - help='Runtime keys', + default=['kmf', 'level1',], + nargs='+', + type=str, + help=''.join(['Runtime keys - each key links to a pre-determined list of variables to analyse. ', + 'Keys are: ', ', '.join( accepted_keys)]), required=False) args = parser.parse_args() @@ -4961,6 +4966,8 @@ def main(): args = get_args() jobID = args.job_id keys = args.keys + print('Running analysis_imerseries.\tjobID:', jobID, '\tkeys:', keys) + assert 0 accepted_keys = ['kmf', 'physics','bgc', 'debug', 'spinup', 'salinity', 'level1', 'level3', ] good_keys = True @@ -4971,13 +4978,6 @@ def main(): if good_keys: sys.exit(0) - #f keys is None: - # keys = [] - #f keys: - # keys = [str(k) for k in keys] - # FIXME this was an exception previously, what do with it? - #jobID = "u-ab749" - """ if 'debug' in keys: suite = 'debug' From 68c456c736b583518e5ee275922a03ca300e9310 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 8 Aug 2022 16:47:44 +0100 Subject: [PATCH 14/37] bhgval help --- bgcval2/bgcval.py | 57 +++++++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/bgcval2/bgcval.py b/bgcval2/bgcval.py index e73d671f..0e5cb57a 100755 --- a/bgcval2/bgcval.py +++ b/bgcval2/bgcval.py @@ -31,11 +31,11 @@ .. moduleauthor:: Lee de Mora """ +import argparse import matplotlib # Force matplotlib to not use any Xwindows backend. matplotlib.use('Agg') import sys -from sys import argv, exit from multiprocessing import Pool from .download_from_mass import findLastFinishedYear @@ -245,36 +245,55 @@ def theWholePackage(jobID, year=False, suite='level1'): clean=True, physicsOnly=physicsOnly) + +def get_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('-i', + '--job-id', + default=None, + help='Job ID', + required=True) + parser.add_argument('-y', + '--year', + default=None, + help='Year', + required=False) + parser.add_argument('-p', + '--physics', + action='store_true', + help='Physics or not', + required=False) + + args = parser.parse_args() + + return args + + def run(): from ._version import __version__ print(f'BGCVal2: {__version__}') - if "--help" in argv or len(argv) == 1: - print("Running with no arguments. Exiting.") - if "--help" in argv: - print("Read the documentation.") - sys.exit(0) - try: - jobID = argv[1] - except: - print("Please provide a job ID") - exit() - #if 'ReportOnly' in argv[:]:ReportOnly=True - #else: ReportOnly = False - if 'physics' in argv[:]: + args = get_args() + jobID = args.job_id + + if args.physics: + print("bgcval: Running with Physics option! Number files 6") physicsOnly = True numberfiles = 4 else: + print("bgcval: Running without Physics option! Number files 4") physicsOnly = False numberfiles = 6 year = False - for ar in argv: + if args.year: try: - ar = int(ar) - except: - continue - year = str(ar) + year = str(int(args.year)) + except ValueError: + print("analysis_timeseries: Invalid input for year - must be an integer, got {args.year}") for divby in [100, 50, 25, 10, 5, 1]: print("main", divby, year) if year: continue From 8c29f3b3e4a140eea171fff1eae6722f5f97be67 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 8 Aug 2022 16:55:03 +0100 Subject: [PATCH 15/37] run GA tests --- .github/workflows/run-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 87fb4d2d..ec0fff69 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -5,6 +5,7 @@ on: push: branches: - main + - improve_help_messages schedule: - cron: '0 0 * * *' # nightly From 313bb8a6af07e35d4407a84d8cdb205b7486b88e Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 9 Aug 2022 09:49:25 +0100 Subject: [PATCH 16/37] Moving code to git from jasmin so I can continue working on it at home. --- bgcval2/analysis_compare.py | 20 +++++++++++--------- bgcval2/analysis_timeseries.py | 27 +++++++++++++++------------ 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index aecb8bc5..e225e34c 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -4219,15 +4219,14 @@ def get_args(): description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('-b', - '--comparison-config', + parser.add_argument('compare_yml', help='Comparison Analysis configuration file, for examples see bgcval2 input_yml directory.', - required=True,) + ) parser.add_argument('-c', '--config-file', - default=os.path.join(os.getcwd(), - 'config-user.yml'), + default=os.path.join(paths.bgcval2_repo, + 'bgcval2/default-bgcval2-config.yml'), help='User configuration file (for paths)', required=False) @@ -4240,10 +4239,10 @@ def main(): """Run the main routine.""" args = get_args() - config_user=None + config_user=args.config_file - if args.comparison_config: - comp_config = os.path.join(os.getcwd(), args.comparison_config) + if args.compare_yml: + comp_config = os.path.join(os.getcwd(), args.compare_yml) print(f"analysis_timeseries: Comparison config file {comp_config}") else: comp_config = os.path.join(os.getcwd(), "comparison.yml") @@ -4254,8 +4253,11 @@ def main(): print(f"analysis_timeseries: Could not find comparison config file {comp_config}") sys.exit(1) + print(config_user, comp_config) + assert 0 + + # Below here is analysis - details = load_comparison_yml(config_user) details = load_comparison_yml(comp_config) jobs = details['jobs'] diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index 11b3bd14..d9ad4287 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -4935,17 +4935,12 @@ def get_args(): parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('-c', - '--config-file', - default=os.path.join(os.getcwd(), - 'bgcval2-config-user.yml'), - help='User configuration file', - required=False) - parser.add_argument('-j', - '--jobID', + parser.add_argument('jobID', + #'--jobID', default=None, help='Job ID (automatically generated by cycl/rose suite for UKESM jobs).', - required=True) + ) + parser.add_argument('-k', '--keys', default=['kmf', 'level1',], @@ -4955,6 +4950,14 @@ def get_args(): 'Keys are: ', ', '.join( accepted_keys)]), required=False) + parser.add_argument('-c', + '--config-file', + default=os.path.join(os.getcwd(), + 'bgcval2-config-user.yml'), + help='User configuration file', + required=False) + + args = parser.parse_args() return args @@ -4964,10 +4967,9 @@ def main(): from ._version import __version__ print(f'BGCVal2: {__version__}') args = get_args() - jobID = args.job_id + jobID = args.jobID keys = args.keys print('Running analysis_imerseries.\tjobID:', jobID, '\tkeys:', keys) - assert 0 accepted_keys = ['kmf', 'physics','bgc', 'debug', 'spinup', 'salinity', 'level1', 'level3', ] good_keys = True @@ -4976,7 +4978,7 @@ def main(): print('Key Argument [',key,'] nor recognised. Accepted keys are:', accepted_keys) good_keys= False if good_keys: - sys.exit(0) + sys.exit(1) """ if 'debug' in keys: @@ -5012,6 +5014,7 @@ def main(): "Will proceed with defaults.") config_user = None + assert 0 for suite in keys: analysis_timeseries( jobID=jobID, From 7e8900d2bde26e6c9cad1c48bb060ff4aab63282 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 9 Aug 2022 10:16:43 +0100 Subject: [PATCH 17/37] Added some sensible argument, expanded jobIDs and input yuml to take lists of inputs. --- bgcval2/analysis_compare.py | 159 +++++++++++++++++---------------- bgcval2/analysis_timeseries.py | 66 +++++--------- 2 files changed, 105 insertions(+), 120 deletions(-) diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index e225e34c..ec1145f1 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -135,9 +135,9 @@ def apply_shifttimes(mdata, jobID, shifttimes): for t in sorted(mdata.keys()): t1 = t + float(shifttimes[jobID]) times.append(t1) - datas.append(mdata[t]) - return times, datas - + datas.append(mdata[t]) + return times, datas + def timeseries_compare(jobs, @@ -146,7 +146,7 @@ def timeseries_compare(jobs, bio=False, debug=False, analysisname='', - shifttimes={}, + shifttimes={}, jobDescriptions={}, lineThicknesses=defaultdict(lambda: 1), linestyles=defaultdict(lambda: '-'), @@ -155,16 +155,16 @@ def timeseries_compare(jobs, """ timeseries_compare: Suite of tools to take pre-analyses time series model data - then compile into single plots, then publish it to an html + then compile into single plots, then publish it to an html document. """ ### strategy here is a simple wrapper. # It's a little cheat-y, as I'm copying straight from analysis_timeseries.py - + jobs = sorted(jobs) #jobs = sorted(colours.keys()) - + for ensemble in list(ensembles.keys()): # ensembles names can not be the same as jobIDs jobs.remove(ensemble) @@ -4006,7 +4006,7 @@ def areatotal(nc, keys): for filename in fnmatch.filter(filenames, '*.png'): AllImages.append(os.path.join(root, filename)) print('AllImages:','fors', root, dirnames, filenames, filename) - + if ensembles != {}: jobs = list(ensembles.keys()) @@ -4144,7 +4144,7 @@ def CompareTwoRuns(jobIDA, def load_comparison_yml(master_compare_yml_fn): """ Load the config yaml. - TAkes an file path string + TAkes an file path string Returns: Details dict. """ @@ -4158,7 +4158,7 @@ def load_comparison_yml(master_compare_yml_fn): details = {} details['name'] = dictionary.get('name', False) - details['jobs'] = dictionary.get('jobs', False) + details['jobs'] = dictionary.get('jobs', False) if not details['name']: print('Please provide a name for your analysis. In your yaml, this is:') @@ -4173,129 +4173,90 @@ def load_comparison_yml(master_compare_yml_fn): print(' thickness: 0.7') print(" linestyle: '-'") print(' shifttime: 0.') - sys.exit(0) + sys.exit(0) - details['do_analysis_timeseries'] = dictionary.get('do_analysis_timeseries', False) + details['do_analysis_timeseries'] = dictionary.get('do_analysis_timeseries', False) details['do_mass_download'] = dictionary.get('do_mass_download', False) - + details['master_suites'] = dictionary.get('master_suites', []) - + default_thickness = 0.75 default_linestyle = '-' default_suite = 'kmf' - + thicknesses = {} linestyles = {} colours = {} suites = {} descriptions = {} shifttimes = {} # number of years to shift time axis. - + for jobID, job_dict in details['jobs'].items(): if job_dict.get('colour', False): colours[jobID] = job_dict['colour'] else: - colours[jobID] = ''.join(['#', "%06x" % random.randint(0, 0xFFFFFF)]) + colours[jobID] = ''.join(['#', "%06x" % random.randint(0, 0xFFFFFF)]) print('WARNING: No colour provided, setting to random hex colour:', colours[jobID]) - + descriptions[jobID] = job_dict.get('description', '') thicknesses[jobID] = job_dict.get('thickness', default_thickness) linestyles[jobID] = job_dict.get('linestyle', default_linestyle) shifttimes[jobID] = float(job_dict.get('shifttime', 0.)) suites[jobID] = job_dict.get('suite', default_suite) - + details['colours'] = colours details['descriptions'] = descriptions details['thicknesses'] = thicknesses - details['linestyles'] = linestyles - details['shifttimes'] = shifttimes + details['linestyles'] = linestyles + details['shifttimes'] = shifttimes details['suites'] = suites return details -def get_args(): - """Parse command line arguments.""" - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - - parser.add_argument('compare_yml', - help='Comparison Analysis configuration file, for examples see bgcval2 input_yml directory.', - ) - - parser.add_argument('-c', - '--config-file', - default=os.path.join(paths.bgcval2_repo, - 'bgcval2/default-bgcval2-config.yml'), - help='User configuration file (for paths)', - required=False) - - args = parser.parse_args() - - return args - - -def main(): - """Run the main routine.""" - args = get_args() - - config_user=args.config_file - - if args.compare_yml: - comp_config = os.path.join(os.getcwd(), args.compare_yml) - print(f"analysis_timeseries: Comparison config file {comp_config}") - else: - comp_config = os.path.join(os.getcwd(), "comparison.yml") - # This should never happen as this argument is required. - print(f"analysis_timeseries: Using user default file {comp_config}") - - if not os.path.isfile(comp_config): - print(f"analysis_timeseries: Could not find comparison config file {comp_config}") - sys.exit(1) - - print(config_user, comp_config) - assert 0 - +def load_yml_and_run(compare_yml, config_user): + """ + Loads the comparison yaml file and run compare_yml. + """ # Below here is analysis details = load_comparison_yml(comp_config) - + jobs = details['jobs'] - analysis_name = details['name'] + analysis_name = details['name'] do_analysis_timeseries = details['do_analysis_timeseries'] do_mass_download = details['do_mass_download'] - master_suites = details['master_suites'] - + master_suites = details['master_suites'] + colours = details['colours'] thicknesses = details['thicknesses'] linestyles = details['linestyles'] descriptions = details['descriptions'] shifttimes = details['shifttimes'] - suites = details['suites'] - + suites = details['suites'] + print('---------------------') print('timeseries_compare:', analysis_name) print('job ids:', jobs.keys()) for jobID in jobs: print(jobID, 'description:',descriptions[jobID]) - print(jobID, 'colour:',colours[jobID]) + print(jobID, 'colour:',colours[jobID]) print(jobID, 'line thickness & style:',thicknesses[jobID], linestyles[jobID]) print(jobID, 'Shift time by', shifttimes[jobID]) print(jobID, 'suite:', suites[jobID]) for jobID in jobs: # even if you don't want to download, it's good to run this - # as it clears up the path and ensures recently downloed data is + # as it clears up the path and ensures recently downloed data is # correctly symlinked. download_from_mass(jobID, doMoo=do_mass_download) - + if do_analysis_timeseries: for jobID in jobs: analysis_timeseries( jobID=jobID, analysisSuite=suites[jobID], config_user=config_user - ) + ) # Master suite leys: if not master_suites: @@ -4338,9 +4299,55 @@ def main(): linestyles=linestyles, config_user=config_user ) - + + +def get_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + + parser.add_argument('-y', + '--compare_yml', + nargs='+', + type=str, + help='One or more Comparison Analysis configuration file, for examples see bgcval2 input_yml directory.', + required=True, + ) + + parser.add_argument('-c', + '--config-file', + default=os.path.join(paths.bgcval2_repo, + 'bgcval2/default-bgcval2-config.yml'), + help='User configuration file (for paths).', + required=False) + + args = parser.parse_args() + + return args + + +def main(): + """Run the main routine.""" + args = get_args() + + # This has a sensible default value. + config_user=args.config_file + + # This shouldn't fail as it's a required argument. + compare_ymls = args.compare_yml + for compare_yml in compare_ymls: + comp_config = os.path.join(os.getcwd(), args.compare_yml) + print(f"analysis_timeseries: Comparison config file {comp_config}") + + if not os.path.isfile(comp_config): + print(f"analysis_timeseries: Could not find comparison config file {compare_yml}") + sys.exit(1) + + load_yml_and_run(compare_yml, config_user) + print("Finished... ") -if __name__ == "__main__": +if __name__ == "__main__": main() diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index d9ad4287..5394f299 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -290,7 +290,7 @@ def analysis_timeseries( print('analysisSuite:',analysisSuite) print('regions:', regions) print('clean:', clean, 'annual:',annual, 'strictFileCheck:', strictFileCheck) - print('config_user:', config_user) + print('config_user:', config_user) # get runtime configuration if config_user: @@ -4930,36 +4930,39 @@ def singleTimeSeries( def get_args(): """Parse command line arguments.""" - accepted_keys = ['kmf', 'physics','bgc', 'debug', 'spinup', 'salinity', 'level1', 'level3', ] - + accepted_keys = ['kmf', 'physics','bgc', 'debug', 'spinup', 'salinity', 'fast', 'level1', 'level3', ] + parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('jobID', - #'--jobID', + parser.add_argument('j', + '--jobID', + nargs='+', + type=str, default=None, - help='Job ID (automatically generated by cycl/rose suite for UKESM jobs).', + help='One or more UKESM Job IDs (automatically generated by cycl/rose suite).', + required=True, ) - + parser.add_argument('-k', '--keys', default=['kmf', 'level1',], - nargs='+', + nargs='+', type=str, help=''.join(['Runtime keys - each key links to a pre-determined list of variables to analyse. ', 'Keys are: ', ', '.join( accepted_keys)]), - required=False) + required=False, + ) parser.add_argument('-c', '--config-file', default=os.path.join(os.getcwd(), 'bgcval2-config-user.yml'), help='User configuration file', - required=False) - + required=False, + ) args = parser.parse_args() - return args @@ -4967,42 +4970,19 @@ def main(): from ._version import __version__ print(f'BGCVal2: {__version__}') args = get_args() - jobID = args.jobID + jobIDs = args.jobID keys = args.keys - print('Running analysis_imerseries.\tjobID:', jobID, '\tkeys:', keys) + print('Running analysis_imerseries.\tjobID:', jobIDs, '\tkeys:', keys) - accepted_keys = ['kmf', 'physics','bgc', 'debug', 'spinup', 'salinity', 'level1', 'level3', ] + accepted_keys = ['kmf', 'physics','bgc', 'debug', 'spinup', 'salinity', 'fast', 'level1', 'level3', ] good_keys = True for key in keys: if key not in accepted_keys: print('Key Argument [',key,'] nor recognised. Accepted keys are:', accepted_keys) good_keys= False - if good_keys: + if good_keys: sys.exit(1) - """ - if 'debug' in keys: - suite = 'debug' - #elif 'all' in argv[1:]: suite = 'all' - elif 'spinup' in keys: - suite = 'spinup' - elif 'salinity' in keys: - suite = 'salinity' - elif 'level1' in keys: - suite = 'level1' - elif 'fast' in keys: - suite = 'fast' - elif 'level3' in keys: - suite = 'level3' - elif 'physics' in keys: - suite = 'physics' - elif 'bgc' in keys: - suite = 'bgc' - elif 'kmf' in keys or 'keymetricsfirst' in keys: - suite = 'keymetricsfirst' - else: - suite = 'level1' - """ if args.config_file: config_user = os.path.join(os.getcwd(), args.config_file) print(f"analysis_timeseries: Using user config file {config_user}") @@ -5014,15 +4994,13 @@ def main(): "Will proceed with defaults.") config_user = None - assert 0 - for suite in keys: + for jobID, suite in itertools.product(keys, jobIDs): analysis_timeseries( jobID=jobID, analysisSuite=suite, config_user=config_user - ) #clean=1) - #if suite == 'all': - #analysis_timeseries(jobID =jobID,analysisSuite='FullDepth', z_component = 'FullDepth',)#clean=1) + ) + if __name__ == "__main__": From 3d642aaa4eaca725fa27708d67bb766274892ca0 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 9 Aug 2022 10:35:56 +0100 Subject: [PATCH 18/37] added dryrun key, made jobID a one or many option. --- bgcval2/download_from_mass.py | 123 +++++++++++++++++++--------------- 1 file changed, 68 insertions(+), 55 deletions(-) diff --git a/bgcval2/download_from_mass.py b/bgcval2/download_from_mass.py index 3b0a8667..3e200e48 100755 --- a/bgcval2/download_from_mass.py +++ b/bgcval2/download_from_mass.py @@ -52,7 +52,7 @@ ./download_from_mass.py jobID This tool will only work on machines that have mass enabled. - + """ @@ -75,7 +75,7 @@ def folder(name): def mnStr(month): - """ + """ :param month: An int between 1 and 100. Returns a 2 digit number string with a leading zero, if needed. """ @@ -84,7 +84,7 @@ def mnStr(month): def getYearFromFile(fn): - """ + """ Takes a file anem, and looks for 8 consequetive numbers, then removes those that are months, and returns the year. """ a = findall(r'\d\d\d\d\d\d\d\d', fn) @@ -102,14 +102,14 @@ def getYearFromFile(fn): def rebaseSymlinks(fn, dryrun=True, debug=False): - """ + """ :param fn: A full path to a filename. It should be a symbolic link. :param dryrun: A boolean switch to do a trial run of this function. - This function reduces a chain of symbolic links down to one. It takes a full path, + This function reduces a chain of symbolic links down to one. It takes a full path, checks whether it is a sym link, then checks whether the real path is the target of the link. If not, it replaces the target with the real path. - + """ ##### @@ -142,7 +142,7 @@ def findLastFinishedYear(jobID, dividby=1, numberfiles=6): This tool find the best year to have a close look at the model, by searching through the files and guessing which years are finished. - + """ if jobID == '': return @@ -212,12 +212,12 @@ def downloadField(jobID, :param dryrun: does not download files, just prints. :param extension: Nemo style file extension :param name: Name of the analysis group, used for the folder. - + This tool takes the jobID, the field name, and using the known structure of universally similar MASS and the local filesystem structure from paths.py, downloads the monthly jobID data for the field requested to the local file structure. - + This tool will only work on machines that have mass enabled. - + """ if jobID == '': return @@ -378,12 +378,12 @@ def medusaMonthlyexport(jobID, dryrun=False): def download_from_mass(jobID, doMoo=True): """ :param jobID: The job ID - + This tool takes the jobID, and using the known structure of universally similar MASS and the local filesystem structure from paths.py, downloads the jobID data to the local file structure. - + This tool will only work on machines that have mass enabled. - + """ if jobID == '': return @@ -433,7 +433,7 @@ def download_from_mass(jobID, doMoo=True): header_lines.append('# moo passwd -r # if mass password is expired\n') download_script_txt = ''.join(header_lines) - # moo ls: + # moo ls: bashCommand = "moo ls moose:/crum/" + jobID + "/ony.nc.file/*.nc" download_script_txt = ''.join([download_script_txt, bashCommand, '\n']) @@ -609,49 +609,12 @@ def pop_keys(keys, remove_keys): return keys -def get_args(): - """Parse command line arguments.""" - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('-i', - '--job-id', - default=None, - help='Job ID', - required=True) - parser.add_argument('-k', - '--keys', - default=None, - nargs='+', type=str, - help='Runtime keys', - required=False) - args = parser.parse_args() - - return args - - -def main(): - """Run the main routine.""" - args = get_args() - jobID = args.job_id - keys = args.keys - if keys is None: - keys = [] - if keys: - keys = [str(k) for k in keys] - print(f"Running with job_id {jobID} and keys {keys}") - +def perform_download(jobID, keys, doMoo): + """ + Single model download. + """ ##### # Default behaviour is to download annual files - if 'noMoo' in keys or 'dryrun' in keys or '--dry-run' in keys: - doMoo=False - dryrun = True - if keys: - keys = pop_keys(keys, ['noMoo', 'dryrun', '--dry-run']) - else: - doMoo=True - dryrun=False - if not keys: download_from_mass(jobID, doMoo=doMoo) @@ -696,5 +659,55 @@ def main(): if keys: downloadField(jobID, keys, timeslice='m', dryrun=dryrun) + +def get_args(): + """Parse command line arguments.""" + + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('-i', + '--job-id', + nargs='+', type=str, + help='Job ID to download (one or more)', + required=True) + parser.add_argument('-k', + '--keys', + default=[], + nargs='+', type=str, + help='Runtime keys', + required=False) + parser.add_argument('-d', + '--dry-run', + default=False, + type=bool, + help='Dry run - do not download any files.', + required=False) + + args = parser.parse_args() + + return args + + +def main(): + """Run the main routine.""" + args = get_args() + + jobIDs = args.job_id + keys = args.keys + dryrun = args.dry_run + doMoo = not dryrun + + if keys in [None, '', [],]: + keys = [] + if keys: + keys = [str(k) for k in keys] + + print(f"Running with job_ids: {jobID} and keys {keys}") + + for jobID in jobIDs: + perform_download(jobID, keys, doMoo) + + if __name__ == "__main__": main() From 6fc5c0197e45c47c67cd0ee1b7b17d65228eef98 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 9 Aug 2022 10:56:01 +0100 Subject: [PATCH 19/37] fixed bug, made compare_yml work, fixed config path so it can be run from any path --- bgcval2/analysis_compare.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index ec1145f1..91dc231c 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -4245,7 +4245,7 @@ def load_yml_and_run(compare_yml, config_user): print(jobID, 'suite:', suites[jobID]) for jobID in jobs: - # even if you don't want to download, it's good to run this + # even if you don't want to download, we run this # as it clears up the path and ensures recently downloed data is # correctly symlinked. download_from_mass(jobID, doMoo=do_mass_download) @@ -4317,8 +4317,8 @@ def get_args(): parser.add_argument('-c', '--config-file', - default=os.path.join(paths.bgcval2_repo, - 'bgcval2/default-bgcval2-config.yml'), + default=os.path.join(os.path.dirname(os.path.realpath(__file__)), + 'default-bgcval2-config.yml'), help='User configuration file (for paths).', required=False) @@ -4336,11 +4336,11 @@ def main(): # This shouldn't fail as it's a required argument. compare_ymls = args.compare_yml + for compare_yml in compare_ymls: - comp_config = os.path.join(os.getcwd(), args.compare_yml) - print(f"analysis_timeseries: Comparison config file {comp_config}") + print(f"analysis_timeseries: Comparison config file {compare_yml}") - if not os.path.isfile(comp_config): + if not os.path.isfile(compare_yml): print(f"analysis_timeseries: Could not find comparison config file {compare_yml}") sys.exit(1) From 855f2500bda226192d39acc9dabf00b853313953 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 9 Aug 2022 11:04:21 +0100 Subject: [PATCH 20/37] fixed bug, fixed path to default in get_args --- bgcval2/analysis_timeseries.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index 5394f299..9534d8a1 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -47,6 +47,7 @@ import numpy as np import os, sys from getpass import getuser +import itertools ##### # Load specific local code: @@ -4935,7 +4936,7 @@ def get_args(): parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('j', + parser.add_argument('-j', '--jobID', nargs='+', type=str, @@ -4956,11 +4957,10 @@ def get_args(): parser.add_argument('-c', '--config-file', - default=os.path.join(os.getcwd(), - 'bgcval2-config-user.yml'), - help='User configuration file', - required=False, - ) + default=os.path.join(os.path.dirname(os.path.realpath(__file__)), + 'default-bgcval2-config.yml'), + help='User configuration file (for paths).', + required=False) args = parser.parse_args() return args @@ -4980,16 +4980,13 @@ def main(): if key not in accepted_keys: print('Key Argument [',key,'] nor recognised. Accepted keys are:', accepted_keys) good_keys= False - if good_keys: + if not good_keys: sys.exit(1) - if args.config_file: - config_user = os.path.join(os.getcwd(), args.config_file) + if os.path.isfile(args.config_file): + config_user = args.config_file print(f"analysis_timeseries: Using user config file {config_user}") else: - config_user = os.path.join(os.getcwd(), "bgcval2-config-user.yml") - print(f"analysis_timeseries: Using user default file {config_user}") - if not os.path.isfile(config_user): print(f"analysis_timeseries: Could not find configuration file {config_user}." "Will proceed with defaults.") config_user = None From d63ef71c0cac5c7e3882dfc6992ea6ef3e53a8c0 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 9 Aug 2022 11:47:13 +0100 Subject: [PATCH 21/37] Update bgcval2/analysis_compare.py --- bgcval2/analysis_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index 91dc231c..39c985d9 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -4144,7 +4144,7 @@ def CompareTwoRuns(jobIDA, def load_comparison_yml(master_compare_yml_fn): """ Load the config yaml. - TAkes an file path string + Takes a file path string Returns: Details dict. """ From b589c2eae8058a2a67101162c1ac00ecfe322d12 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 9 Aug 2022 11:59:05 +0100 Subject: [PATCH 22/37] Adding details to README and minor edits. --- README.md | 131 ++++++++++++++++++--- bgcval2/analysis_compare.py | 4 +- input_yml/comparison_analysis_template.yml | 12 +- 3 files changed, 123 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 48917ced..3fee7d0f 100644 --- a/README.md +++ b/README.md @@ -44,22 +44,11 @@ conda activate bgcval2 pip install -e .[develop] ``` -Running the tool -================ - -The tool has a number of executables one can invoke individually, e.g.: - +Test that the tool has been installed correctly with: ``` -analysis_timeseries u-bc179 level1 -analysis_p2p u-bc179 level2 2010 +analysis_compare -h ``` -Once these have completed, a summary HTML page can be generated with the command: - -``` -bgcval2_make_report u-bc179 2010 -``` -This produces an html5 mobile-friendly website which can be opened using your -browser of choice. +which should print the module information and instructions on how to run the tool. ### Available executables @@ -73,6 +62,115 @@ Executable name | What it does | Command `analysis_compare` | runs comparison of multiple single jobs | analysis_compare + +Running the tool to compare multiple jobs +========================================= + +The time developmenmt of several models can be compared +and sumarised in a single comparison report html. +This report can be generated with a single command, based on a simple yml file input: + +``` +analysis_compare --compare_yml comparison_recipe.yml +``` + +Example input yaml files exist in the `input_yml` directory. +However, there are a few key values: + + +In this yml file, the structure is: +``` +--- +name: +do_analysis_timeseries: +do_mass_download: +master_suites: + +jobs: + : + description: + colour: red + thickness: 0.7 + linestyle: '-' + shifttime: 0. + suite: physics + : + description: + ... +``` + +These values are: + - `name`: + - The name of the analysis. + - This will be the path of output report. + - `do_analysis_timeseries`: + - A Boolean value to run or skip the single model timeseries. + - Set to False if the single model analysis has already completed. + - `do_mass_download`: + - A boolean value to run the mass download. + - This is not currently possible as we can only download mass file from mass-cli1 on jasmin. + - See below for details on how to download jobs data. + - `master_suites`: + - A list of the type of analysis report to produce. + - Options are: `physics`, `bio`, `debug`. + - Default is `['physics', 'bio',]`. + - `jobs`: + - A list of jobIDs, and some options on how they will appear in the final report. + - The options are: + - `description`: + - A description of job, which helps people differentiate the jobs in the final report. + - `colour`: + - The colour of this jobs line in the report plots. + - Default colour is a randomly generated hex colour. + - `thickness`: + - The thickness of this jobs line in the report plots. + - default is 0.7 + - `linestyle`: + - The linestyle of this jobs line in the report plots. + - Accepts typical matplotlib line styles: `'solid', 'dashed', 'dotted', 'dashdot', '-'. ';', ', etc` + - default is `-'` + - `shiftime`: + - A number in whole years by which the jobs line is shifted. + - this is useful if jobs start from different initial years in spin up, for instance. + - Default is `0.` ( no time shift ). + - `suite`: + - An analysis suite to run the analysis_timeseries. + - See `analysis_timeseries` for more details. + + +A sample yaml exists in `input_yml/comparison_analysis_template.yml`, +which can be adapted to additional analysis. + + + + +Running the tool for a single job +================================= + +The multi-job analysis described above can only do timeseries analysis. +To run an in-depth analysis of a single job, the following command can be run: + +``` +bgcval2 -j jobID +``` + +This will run a time series analysis, a point to point analysis, and +publish the reports into a single job html5 report. + + +Alternatively, these tasks can be invoked individually, e.g.: + +``` +analysis_timeseries --jobID u-bc179 --keys kmf level1 +analysis_p2p u-bc179 level2 2010 +bgcval2_make_report u-bc179 2010 + +``` +This produces an html5 mobile-friendly website which can be opened using your +browser of choice. + + + Time series analysis -------------------- @@ -80,8 +178,9 @@ This is an analysis that investigates the time development of specific marine physics and Biogeochemistry fields in the given model, and then compares them against historical observations. -The command to run it is `analysis_timeseries jobID key`, where jobID is a mass -job id, such a `u-ab123`, and the key is a pre-defined key, which generates a +The command to run it is `analysis_timeseries --jobID jobID --keys key`, +where jobID is one or more mass jobIDs, such a `u-ab123`. +The key is one or more pre-defined key, which generates a list of variables. Key | What it is | Description diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index 91dc231c..00b5e091 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -4180,8 +4180,8 @@ def load_comparison_yml(master_compare_yml_fn): details['master_suites'] = dictionary.get('master_suites', []) - default_thickness = 0.75 - default_linestyle = '-' + default_thickness = 0.7 + default_linestyle = 'solid' default_suite = 'kmf' thicknesses = {} diff --git a/input_yml/comparison_analysis_template.yml b/input_yml/comparison_analysis_template.yml index 08fce127..16b9c110 100644 --- a/input_yml/comparison_analysis_template.yml +++ b/input_yml/comparison_analysis_template.yml @@ -1,29 +1,29 @@ --- -# GC5 N96 ORCA1 spinup analysis name: Template job # Run the single Job Analysis (analysis_timeseries) do_analysis_timeseries: False # if True, it calls `analysis_timeseries jobID suite` using details provided here. -do_mass_download: True +do_mass_download: False # if True, it calls bgcval2/download_from_mass.py and attempts to download the jobs data. # Job ID's suites as named by Rose/Cylc jobs: - u-aa001: + u-aa001: # Not a real jobID! description: 'Job number 1' colour: red thickness: 1. linestyle: '-' shifttime: 0. - suite: physics - u-aa002: + suite: kmf level1 + + u-aa002: # Not a real Job ID description: 'Job number 2' colour: blue thickness: 1.0 linestyle: ':' shifttime: 0. - suite: physics + suite: kmf level1 From 2d233173b2d3eb8243a596291efadcd4e6502154 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 9 Aug 2022 13:31:13 +0100 Subject: [PATCH 23/37] start fixing some testing --- tests/integration/test_run.py | 58 +++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_run.py b/tests/integration/test_run.py index 70ffca7e..90b520f1 100644 --- a/tests/integration/test_run.py +++ b/tests/integration/test_run.py @@ -10,8 +10,17 @@ import pytest -from bgcval2 import analysis_timeseries, bgcval +from bgcval2 import ( + analysis_timeseries, + bgcval, + download_from_mass, + bgcval2_make_report, + analysis_compare +) from bgcval2.analysis_timeseries import main +from bgcval2.download_from_mass import main +from bgcval2.bgcval2_make_report import main +from bgcval2.analysis_compare import main def wrapper(f): @@ -40,20 +49,63 @@ def test_setargs(): @patch('bgcval2.analysis_timeseries.main', new=wrapper(analysis_timeseries)) -def test_run_analysis_timeseries(): +def test_run_analysis_timeseries_command(): """Test run command.""" with arguments('analysis_timeseries', '--help'): with pytest.raises(SystemExit) as pytest_wrapped_e: main() assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 0 + err = "analysis_timeseries: error: argument \ + -j/--jobID: expected at least one argument" + with arguments('analysis_timeseries', '--jobID', '--keys'): + with pytest.raises(SystemExit) as pytest_wrapped_e: + main() + assert err == pytest_wrapped_e @patch('bgcval2.bgcval.run', new=wrapper(bgcval)) -def test_run_bgcval(): +def test_run_bgcval_command(): """Test run command.""" with arguments('bgcval', '--help'): with pytest.raises(SystemExit) as pytest_wrapped_e: main() assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 0 + err = "bgcval: error: the following arguments are required: -j/--jobID" + with arguments('bgcval', '--job-id'): + with pytest.raises(SystemExit) as pytest_wrapped_e: + main() + assert err == pytest_wrapped_e + + +@patch('bgcval2.download_from_mass.main', new=wrapper(download_from_mass)) +def test_download_from_mass_command(): + """Test run command.""" + with arguments('download_from_mass', '--help'): + with pytest.raises(SystemExit) as pytest_wrapped_e: + main() + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 0 + err = "download_from_mass: error: the following \ + arguments are required: -i/--job-id" + with arguments('bgcval', '--job-id'): + with pytest.raises(SystemExit) as pytest_wrapped_e: + main() + assert err == pytest_wrapped_e + + +@patch('bgcval2.analysis_compare.main', new=wrapper(analysis_compare)) +def test_analysis_compare_command(): + """Test run command.""" + with arguments('analysis_compare', '--help'): + with pytest.raises(SystemExit) as pytest_wrapped_e: + main() + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 0 + err = "download_from_mass: error: the following \ + arguments are required: -i/--job-id" + with arguments('bgcval', '--job-id'): + with pytest.raises(SystemExit) as pytest_wrapped_e: + main() + assert err == pytest_wrapped_e From b59fd83ae67a5b0f94a343f2946955e4f6f5b035 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 9 Aug 2022 13:33:09 +0100 Subject: [PATCH 24/37] bug fix and added annual flag as default - allowing both annual and monthly downloads to happen in same command; --- bgcval2/download_from_mass.py | 102 +++++++++++++++++++--------------- 1 file changed, 57 insertions(+), 45 deletions(-) diff --git a/bgcval2/download_from_mass.py b/bgcval2/download_from_mass.py index 3e200e48..13e16144 100755 --- a/bgcval2/download_from_mass.py +++ b/bgcval2/download_from_mass.py @@ -118,7 +118,8 @@ def rebaseSymlinks(fn, dryrun=True, debug=False): # print "rebaseSymlinks:\tfile does not exist.",fn # return if not os.path.islink(fn): - if debug: print("download_from_mass:\trebaseSymlinks:\tfile is not a symlink.", fn) + if debug: + print("download_from_mass:\trebaseSymlinks:\tfile is not a symlink.", fn) return ##### @@ -128,7 +129,8 @@ def rebaseSymlinks(fn, dryrun=True, debug=False): if realpath == linkpath: return - print("download_from_mass:\trebaseSymlinks:\tdeleting and re-linking ", fn, '-->', realpath) + if debug: + print("download_from_mass:\trebaseSymlinks:\tdeleting and re-linking ", fn, '-->', realpath) if dryrun: return os.remove(fn) os.symlink(realpath, fn) @@ -484,16 +486,17 @@ def download_from_mass(jobID, doMoo=True): outfile.write(download_script_txt) outfile.close() - fixFilePaths(outputFold, jobID) - deleteBadLinksAndZeroSize(outputFold, jobID) + fixFilePaths(outputFold, jobID, debug=False,) + deleteBadLinksAndZeroSize(outputFold, jobID, debug=False,) def fixFilePaths(outputFold, jobID, debug=False): ##### # The coupled model looses the first two characters of the name in the netcdf file. fns = glob(outputFold + "/*" + jobID[2:] + "*.nc") - print("download_from_mass:\tfixFilePaths:\tLooking for", - outputFold + "/" + jobID[2:] + "*.nc") + if debug: + print("download_from_mass:\tfixFilePaths:\tLooking for", + outputFold + "/" + jobID[2:] + "*.nc") fns.extend( glob(outputFold + '/MetOffice*')) # Because ocean assess might use the lisence? @@ -508,22 +511,23 @@ def fixFilePaths(outputFold, jobID, debug=False): correctfn) continue if correctfn == fn: continue - print("download_from_mass:\tfixFilePaths:\tFixing file prefix", fn, - '-->', correctfn) + if debug: + print("download_from_mass:\tfixFilePaths:\tFixing file prefix", fn, + '-->', correctfn) try: os.symlink(fn, correctfn) except: - print("Unable to make link:", correctfn) + if debug: + print("Unable to make link:", correctfn) continue -# print "download_from_mass:\tfixFilePaths:\t", correctfn -##### -# Some runs have nemo/medusa as a preface to the file name. + ##### + # Some runs have nemo/medusa as a preface to the file name. for pref in ['nemo_', 'medusa_']: - #nemo_u-ai886o_1y_26291201-26301201_grid-V.nc fns = glob(outputFold + "/" + pref + jobID + "*.nc") - print("download_from_mass:\tfixFilePaths:\tLooking for new prefix:", - pref, outputFold + "/" + pref + jobID + "*.nc") + if debug: + print("download_from_mass:\tfixFilePaths:\tLooking for new prefix:", + pref, outputFold + "/" + pref + jobID + "*.nc") for fn in sorted(fns): ##### correctfn = os.path.dirname(fn) + '/' + os.path.basename( @@ -534,14 +538,16 @@ def fixFilePaths(outputFold, jobID, debug=False): "download_from_mass:\tfixFilePaths:\tcorrect path exists.", correctfn) continue - print("download_from_mass:\tfixFilePaths:\tFixing file prefix", - pref, - end=' ') + if debug: + print("download_from_mass:\tfixFilePaths:\tFixing file prefix", + pref, + end=' ') os.symlink(fn, correctfn) - print("download_from_mass:\tfixFilePaths:\t", correctfn) + if debug: + print("download_from_mass:\tfixFilePaths:\t", correctfn) -##### -# Some runs have nemo/medusa as a preface to the file name. + ##### + # Some runs have nemo/medusa as a preface to the file name. suffDict = { 'grid-T': 'grid_T', 'grid-U': 'grid_U', @@ -554,8 +560,9 @@ def fixFilePaths(outputFold, jobID, debug=False): for badsuff, suff in list(suffDict.items()): #nemo_u-ai886o_1y_26291201-26301201_grid-V.nc fns = glob(outputFold + "/" + jobID + "*" + badsuff + ".nc") - print("download_from_mass:\tfixFilePaths:\tLooking for new suff:", - badsuff, outputFold + "/" + jobID + "*" + badsuff + ".nc") + if debug: + print("download_from_mass:\tfixFilePaths:\tLooking for new suff:", + badsuff, outputFold + "/" + jobID + "*" + badsuff + ".nc") for fn in sorted(fns): ##### correctfn = os.path.dirname(fn) + '/' + os.path.basename( @@ -566,36 +573,38 @@ def fixFilePaths(outputFold, jobID, debug=False): "download_from_mass:\tfixFilePaths:\tcorrect path exists.", correctfn) continue - print("download_from_mass:\tfixFilePaths:\tFixing file suffix", - badsuff, - '->', - suff, - end=' ') + if debug: + print("download_from_mass:\tfixFilePaths:\tFixing file suffix", + badsuff, + '->', + suff, + end=' ') if correctfn == fn: continue try: os.symlink(fn, correctfn) except: continue - print("download_from_mass:\tfixFilePaths:\t", correctfn) + if debug: + print("download_from_mass:\tfixFilePaths:\t", correctfn) ##### # This code looks at symoblic links and points them at their ultimate source, removing the long link chains. for fn in glob(outputFold + '/*'): - rebaseSymlinks(fn, dryrun=False) + rebaseSymlinks(fn, dryrun=False, debug=False) -def deleteBadLinksAndZeroSize(outputFold, jobID): +def deleteBadLinksAndZeroSize(outputFold, jobID, debug=True): bashCommand1 = "find " + outputFold + "/. -size 0 -print -delete" bashCommand2 = "find -L " + outputFold + "/. -type l -delete -print" - print("deleteBadLinksAndZeroSize:\t", bashCommand1) + if debug: print("deleteBadLinksAndZeroSize:\t", bashCommand1) process1 = subprocess.Popen(bashCommand1.split(), stdout=subprocess.PIPE) output1 = process1.communicate()[0] - print("deleteBadLinksAndZeroSize:\t", bashCommand2) + if debug: print("deleteBadLinksAndZeroSize:\t", bashCommand2) process2 = subprocess.Popen(bashCommand2.split(), stdout=subprocess.PIPE) output2 = process2.communicate()[0] @@ -615,9 +624,11 @@ def perform_download(jobID, keys, doMoo): """ ##### # Default behaviour is to download annual files - if not keys: + if not keys or 'annual' in keys: download_from_mass(jobID, doMoo=doMoo) + keys = pop_keys(keys, ['annual', ]) + dryrun = not doMoo ##### # Monthly Ice files if 'ice' in keys or 'soicecov' in keys: @@ -666,23 +677,24 @@ def get_args(): parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('-i', - '--job-id', + parser.add_argument('-j', + '--jobID', nargs='+', type=str, - help='Job ID to download (one or more)', + help='One or more JobIDs to download', required=True) parser.add_argument('-k', '--keys', - default=[], + default=['annual', ], nargs='+', type=str, - help='Runtime keys', + help='Download keys - default options are: annual (which downloads all the annual files), ' + 'or chl, mld, ice, export, which downkoads monthly files for these fields. ' + 'Note that monthly files download is unstable and slow.', required=False) parser.add_argument('-d', '--dry-run', - default=False, - type=bool, - help='Dry run - do not download any files.', - required=False) + action='store_true', + help='Dry run: Do not download any files.', + ) args = parser.parse_args() @@ -693,7 +705,7 @@ def main(): """Run the main routine.""" args = get_args() - jobIDs = args.job_id + jobIDs = args.jobID keys = args.keys dryrun = args.dry_run doMoo = not dryrun @@ -703,7 +715,7 @@ def main(): if keys: keys = [str(k) for k in keys] - print(f"Running with job_ids: {jobID} and keys {keys}") + print(f"Running with job_ids: {jobIDs} and keys {keys}") for jobID in jobIDs: perform_download(jobID, keys, doMoo) From f9dbeae0df499ece10138a277ae96d1bf9214b4c Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 9 Aug 2022 13:33:23 +0100 Subject: [PATCH 25/37] More text improvements --- README.md | 113 ++++++++++++++++++------------------------------------ 1 file changed, 38 insertions(+), 75 deletions(-) diff --git a/README.md b/README.md index 3fee7d0f..9f9d6b08 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,40 @@ which can be adapted to additional analysis. +Downloading data using MASS +=========================== + +Data can be downloaded and prepared for analysis using the `download_from_mass` bgcval2 tool, +with the command: +``` +download_from_mass -j jobID +``` +where `jobID` is one or more jobIDs. + +This script will only work on jasmin's `mass-cli1` machine, +which is set up to interact with the Met Office Storate System MASS. + +The optional flag `--dry-run` skips the download part of the script, +and generates a script in the `bgcval2/mass_scripts` directory. +From there, users can ssh to `mass-cli1` and execute this script: + +``` +# from login1.jasmin.ac.uk, ssh to the mass machine: +ssh -X mass-cli1 + +# run script with: +source /path/to/bgcval2/is/here/bgcval2/mass_scripts/jobID.sh +``` + +Alternatively, the whole `download_from_mass` tool could be executed on the `mass-cli1` machine. + +Several different keys can be included in the download if monthly data is required. +However, it's not recommended to include monthly data at this stage as that code +both to download, but also to run the monthly analysis is not currently tested. + +This tool downloads the data, but also includes several functions which create symbolic links +in the data's directory in order to accomodate incompatible changes in NEMO's output formatting. + Running the tool for a single job ================================= @@ -199,6 +233,8 @@ Note that there may be some overlap between the contents of these keys. Point to point analysis ----------------------- +WORK IN PROGRESS. + This is an analysis that compares a specific year of the model against several climatological datasets, and produces comparison maps, histograms and scatter plots. @@ -220,6 +256,8 @@ Note that there may be some overlap between the contents of these keys. Single Model report ------------------- +WORK IN PROGRESS. + Once an analysis has run, either time series or point to point, a report can be generated from this output, using the command: ``` @@ -229,81 +267,6 @@ This gnerated an HTML5 mobile-friendlyt report, summarising the output of a single model run. -Multi-model comaprison report ------------------------------ - -Once several models have been analysed using the time series analysis, -their time development can be compared using the `analysis_compare` command: -``` -analysis_compare recipe.yml -``` - -The comparison reports are generated from a user defined yaml recipe. - -In this yml file, the key structure is: -``` -name: -do_analysis_timeseries: -jobs: - : - description: - : - description: -``` -Where the `name` is a short unique string describing the analysis script -and the ultimately the name given here will become part of the path -of the final output comparison report. - -The `do_analysis_timeseries` bool lets `analysis_compare` send jobs to -`analysis_timeseries`, allowing the user to run the entire suite in one -command, instead of individually running the `analysis_timeseries` then -the `analysis_compare` part afterwards. - -The `jobs` is a dict of job IDs, that describe how each job will appear in the -final report. - -The optional arguments for each job are: - - colour: a colour hex, or html recognised string (default is a randomly generated hex colour.) - - thickness: line thickness for matplotlib (default (`0.7`) - - linestyle: line style for matplotlib (default: `'-'`) - - suite: suite to send to `analysis_timeseries` if `do_analysis_timeseries` is true. - -A sample yaml exists in `input_yml/comparison_analysis_template.yml`, -which can be adapted to additional analysis. - -Download data from MASS ------------------------ - -It's straightforward to download data from the Met Office -Storage System, MASS. -The bgcval2 tool `download_data_from mass` sets up the command and -outputs a script which you can run on Jasmin's mass-cli1 machine. - -Note that the only place on CEDA-JASMIN you can download data -is the dedicated virtual machine, mass-cli1.jasmin.ac.uk. - -The recommended process is to generate the download script on an interactive node, -like sci1 with the command: -``` -download_from_mass jobID noMoo -``` -Which will then create a script in the directory `mass_scripts`. -The runtime flag `noMoo` stops the script from attempted to execute the script. - -From there, the user must log into the mass machine, and execute the script: -``` -#from login1.jasmin.ac.uk: -ssh -X mas-cli1 -cd bgcval2 -source mass_script/*.sh -``` - -Note that these scripts can also be automatically generated by -the `analysis_compare` command by including the -``` -do_mass_download: True -``` - Documentation ============= From 559669d4e8bbd4333bac1398f2d23a5e0a03de39 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 9 Aug 2022 15:18:33 +0100 Subject: [PATCH 26/37] fix tests proper --- tests/integration/test_run.py | 101 ++++++++++++++++++++++++---------- 1 file changed, 72 insertions(+), 29 deletions(-) diff --git a/tests/integration/test_run.py b/tests/integration/test_run.py index 90b520f1..97d931a2 100644 --- a/tests/integration/test_run.py +++ b/tests/integration/test_run.py @@ -7,6 +7,7 @@ import functools import sys from unittest.mock import patch +from io import StringIO import pytest @@ -17,10 +18,11 @@ bgcval2_make_report, analysis_compare ) -from bgcval2.analysis_timeseries import main -from bgcval2.download_from_mass import main -from bgcval2.bgcval2_make_report import main -from bgcval2.analysis_compare import main +from bgcval2.analysis_timeseries import main as analysis_timeseries_main +from bgcval2.download_from_mass import main as download_from_mass_main +from bgcval2.bgcval import run as bgcval_main +from bgcval2.bgcval2_make_report import main as bgcval2_make_report_main +from bgcval2.analysis_compare import main as analysis_compare_main def wrapper(f): @@ -48,20 +50,38 @@ def test_setargs(): assert sys.argv == original +@contextlib.contextmanager +def capture_sys_output(): + capture_out, capture_err = StringIO(), StringIO() + current_out, current_err = sys.stdout, sys.stderr + try: + sys.stdout, sys.stderr = capture_out, capture_err + yield capture_out, capture_err + finally: + sys.stdout, sys.stderr = current_out, current_err + + @patch('bgcval2.analysis_timeseries.main', new=wrapper(analysis_timeseries)) def test_run_analysis_timeseries_command(): """Test run command.""" with arguments('analysis_timeseries', '--help'): with pytest.raises(SystemExit) as pytest_wrapped_e: - main() + analysis_timeseries_main() assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 0 - err = "analysis_timeseries: error: argument \ - -j/--jobID: expected at least one argument" + err = "analysis_timeseries: error: the following arguments " \ + "are required: -j/--jobID" + with arguments('analysis_timeseries'): + with pytest.raises(SystemExit) as cm, capture_sys_output() \ + as (stdout, stderr): + analysis_timeseries_main() + assert err in str(stderr.getvalue()) + err = "--jobID: expected at least one argument" with arguments('analysis_timeseries', '--jobID', '--keys'): - with pytest.raises(SystemExit) as pytest_wrapped_e: - main() - assert err == pytest_wrapped_e + with pytest.raises(SystemExit) as cm, capture_sys_output() \ + as (stdout, stderr): + analysis_timeseries_main() + assert err in str(stderr.getvalue()) @patch('bgcval2.bgcval.run', new=wrapper(bgcval)) @@ -69,14 +89,15 @@ def test_run_bgcval_command(): """Test run command.""" with arguments('bgcval', '--help'): with pytest.raises(SystemExit) as pytest_wrapped_e: - main() + bgcval_main() assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 0 - err = "bgcval: error: the following arguments are required: -j/--jobID" - with arguments('bgcval', '--job-id'): - with pytest.raises(SystemExit) as pytest_wrapped_e: - main() - assert err == pytest_wrapped_e + err = "the following arguments are required: -i/--job-id\n" + with arguments('bgcval'): + with pytest.raises(SystemExit) as cm, capture_sys_output() \ + as (stdout, stderr): + bgcval_main() + assert err in str(stderr.getvalue()) @patch('bgcval2.download_from_mass.main', new=wrapper(download_from_mass)) @@ -84,15 +105,15 @@ def test_download_from_mass_command(): """Test run command.""" with arguments('download_from_mass', '--help'): with pytest.raises(SystemExit) as pytest_wrapped_e: - main() + download_from_mass_main() assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 0 - err = "download_from_mass: error: the following \ - arguments are required: -i/--job-id" - with arguments('bgcval', '--job-id'): - with pytest.raises(SystemExit) as pytest_wrapped_e: - main() - assert err == pytest_wrapped_e + err = "the following arguments are required: -i/--job-id" + with arguments('download_from_mass'): + with pytest.raises(SystemExit) as cm, capture_sys_output() \ + as (stdout, stderr): + download_from_mass_main() + assert err in str(stderr.getvalue()) @patch('bgcval2.analysis_compare.main', new=wrapper(analysis_compare)) @@ -100,12 +121,34 @@ def test_analysis_compare_command(): """Test run command.""" with arguments('analysis_compare', '--help'): with pytest.raises(SystemExit) as pytest_wrapped_e: - main() + analysis_compare_main() assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 0 - err = "download_from_mass: error: the following \ - arguments are required: -i/--job-id" - with arguments('bgcval', '--job-id'): + err = "the following arguments are required: -y/--compare_yml" + with arguments('analysis_compare'): + with pytest.raises(SystemExit) as cm, capture_sys_output() \ + as (stdout, stderr): + analysis_compare_main() + assert err in str(stderr.getvalue()) + + +@patch('bgcval2.bgcval2_make_report.main', new=wrapper(bgcval2_make_report)) +def test_bgcval2_make_report_command(): + """Test run command.""" + with arguments('analysis_compare', '--help'): with pytest.raises(SystemExit) as pytest_wrapped_e: - main() - assert err == pytest_wrapped_e + bgcval2_make_report_main() + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 0 + err = "the following arguments are required: -i/--job-id" + with arguments('bgcval2_make_report'): + with pytest.raises(SystemExit) as cm, capture_sys_output() \ + as (stdout, stderr): + bgcval2_make_report_main() + assert err in str(stderr.getvalue()) + err = "argument -r/--report: expected one argument" + with arguments('bgcval2_make_report', '--job-id DUM', '--report'): + with pytest.raises(SystemExit) as cm, capture_sys_output() \ + as (stdout, stderr): + bgcval2_make_report_main() + assert err in str(stderr.getvalue()) From 5f7494127a00be502bbac4f939919a297a240c3e Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 9 Aug 2022 15:20:55 +0100 Subject: [PATCH 27/37] fix test with Lee last changes --- tests/integration/test_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_run.py b/tests/integration/test_run.py index 97d931a2..a5031493 100644 --- a/tests/integration/test_run.py +++ b/tests/integration/test_run.py @@ -108,7 +108,7 @@ def test_download_from_mass_command(): download_from_mass_main() assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 0 - err = "the following arguments are required: -i/--job-id" + err = "the following arguments are required: -j/--jobID" with arguments('download_from_mass'): with pytest.raises(SystemExit) as cm, capture_sys_output() \ as (stdout, stderr): From ce6d463a4ffe30fc6d266e791b56353452caea06 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 9 Aug 2022 15:27:33 +0100 Subject: [PATCH 28/37] change test module name --- tests/integration/test_command_line.py | 154 +++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 tests/integration/test_command_line.py diff --git a/tests/integration/test_command_line.py b/tests/integration/test_command_line.py new file mode 100644 index 00000000..a5031493 --- /dev/null +++ b/tests/integration/test_command_line.py @@ -0,0 +1,154 @@ +"""Tests for BGCVal2 CLI. + +Includes a context manager to temporarily modify sys.argv +""" +import contextlib +import copy +import functools +import sys +from unittest.mock import patch +from io import StringIO + +import pytest + +from bgcval2 import ( + analysis_timeseries, + bgcval, + download_from_mass, + bgcval2_make_report, + analysis_compare +) +from bgcval2.analysis_timeseries import main as analysis_timeseries_main +from bgcval2.download_from_mass import main as download_from_mass_main +from bgcval2.bgcval import run as bgcval_main +from bgcval2.bgcval2_make_report import main as bgcval2_make_report_main +from bgcval2.analysis_compare import main as analysis_compare_main + + +def wrapper(f): + @functools.wraps(f) + def empty(*args, **kwargs): # noqa + if kwargs: + raise ValueError(f'Parameters not supported: {kwargs}') + return True + + return empty + + +@contextlib.contextmanager +def arguments(*args): + backup = sys.argv + sys.argv = list(args) + yield + sys.argv = backup + + +def test_setargs(): + original = copy.deepcopy(sys.argv) + with arguments('testing', 'working', 'with', 'sys.argv'): + assert sys.argv == ['testing', 'working', 'with', 'sys.argv'] + assert sys.argv == original + + +@contextlib.contextmanager +def capture_sys_output(): + capture_out, capture_err = StringIO(), StringIO() + current_out, current_err = sys.stdout, sys.stderr + try: + sys.stdout, sys.stderr = capture_out, capture_err + yield capture_out, capture_err + finally: + sys.stdout, sys.stderr = current_out, current_err + + +@patch('bgcval2.analysis_timeseries.main', new=wrapper(analysis_timeseries)) +def test_run_analysis_timeseries_command(): + """Test run command.""" + with arguments('analysis_timeseries', '--help'): + with pytest.raises(SystemExit) as pytest_wrapped_e: + analysis_timeseries_main() + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 0 + err = "analysis_timeseries: error: the following arguments " \ + "are required: -j/--jobID" + with arguments('analysis_timeseries'): + with pytest.raises(SystemExit) as cm, capture_sys_output() \ + as (stdout, stderr): + analysis_timeseries_main() + assert err in str(stderr.getvalue()) + err = "--jobID: expected at least one argument" + with arguments('analysis_timeseries', '--jobID', '--keys'): + with pytest.raises(SystemExit) as cm, capture_sys_output() \ + as (stdout, stderr): + analysis_timeseries_main() + assert err in str(stderr.getvalue()) + + +@patch('bgcval2.bgcval.run', new=wrapper(bgcval)) +def test_run_bgcval_command(): + """Test run command.""" + with arguments('bgcval', '--help'): + with pytest.raises(SystemExit) as pytest_wrapped_e: + bgcval_main() + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 0 + err = "the following arguments are required: -i/--job-id\n" + with arguments('bgcval'): + with pytest.raises(SystemExit) as cm, capture_sys_output() \ + as (stdout, stderr): + bgcval_main() + assert err in str(stderr.getvalue()) + + +@patch('bgcval2.download_from_mass.main', new=wrapper(download_from_mass)) +def test_download_from_mass_command(): + """Test run command.""" + with arguments('download_from_mass', '--help'): + with pytest.raises(SystemExit) as pytest_wrapped_e: + download_from_mass_main() + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 0 + err = "the following arguments are required: -j/--jobID" + with arguments('download_from_mass'): + with pytest.raises(SystemExit) as cm, capture_sys_output() \ + as (stdout, stderr): + download_from_mass_main() + assert err in str(stderr.getvalue()) + + +@patch('bgcval2.analysis_compare.main', new=wrapper(analysis_compare)) +def test_analysis_compare_command(): + """Test run command.""" + with arguments('analysis_compare', '--help'): + with pytest.raises(SystemExit) as pytest_wrapped_e: + analysis_compare_main() + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 0 + err = "the following arguments are required: -y/--compare_yml" + with arguments('analysis_compare'): + with pytest.raises(SystemExit) as cm, capture_sys_output() \ + as (stdout, stderr): + analysis_compare_main() + assert err in str(stderr.getvalue()) + + +@patch('bgcval2.bgcval2_make_report.main', new=wrapper(bgcval2_make_report)) +def test_bgcval2_make_report_command(): + """Test run command.""" + with arguments('analysis_compare', '--help'): + with pytest.raises(SystemExit) as pytest_wrapped_e: + bgcval2_make_report_main() + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 0 + err = "the following arguments are required: -i/--job-id" + with arguments('bgcval2_make_report'): + with pytest.raises(SystemExit) as cm, capture_sys_output() \ + as (stdout, stderr): + bgcval2_make_report_main() + assert err in str(stderr.getvalue()) + err = "argument -r/--report: expected one argument" + with arguments('bgcval2_make_report', '--job-id DUM', '--report'): + with pytest.raises(SystemExit) as cm, capture_sys_output() \ + as (stdout, stderr): + bgcval2_make_report_main() + assert err in str(stderr.getvalue()) From 4b0f51e655e39e3abac4b5456561fc8436833d4a Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 9 Aug 2022 15:27:43 +0100 Subject: [PATCH 29/37] old name --- tests/integration/test_run.py | 154 ---------------------------------- 1 file changed, 154 deletions(-) delete mode 100644 tests/integration/test_run.py diff --git a/tests/integration/test_run.py b/tests/integration/test_run.py deleted file mode 100644 index a5031493..00000000 --- a/tests/integration/test_run.py +++ /dev/null @@ -1,154 +0,0 @@ -"""Tests for BGCVal2 CLI. - -Includes a context manager to temporarily modify sys.argv -""" -import contextlib -import copy -import functools -import sys -from unittest.mock import patch -from io import StringIO - -import pytest - -from bgcval2 import ( - analysis_timeseries, - bgcval, - download_from_mass, - bgcval2_make_report, - analysis_compare -) -from bgcval2.analysis_timeseries import main as analysis_timeseries_main -from bgcval2.download_from_mass import main as download_from_mass_main -from bgcval2.bgcval import run as bgcval_main -from bgcval2.bgcval2_make_report import main as bgcval2_make_report_main -from bgcval2.analysis_compare import main as analysis_compare_main - - -def wrapper(f): - @functools.wraps(f) - def empty(*args, **kwargs): # noqa - if kwargs: - raise ValueError(f'Parameters not supported: {kwargs}') - return True - - return empty - - -@contextlib.contextmanager -def arguments(*args): - backup = sys.argv - sys.argv = list(args) - yield - sys.argv = backup - - -def test_setargs(): - original = copy.deepcopy(sys.argv) - with arguments('testing', 'working', 'with', 'sys.argv'): - assert sys.argv == ['testing', 'working', 'with', 'sys.argv'] - assert sys.argv == original - - -@contextlib.contextmanager -def capture_sys_output(): - capture_out, capture_err = StringIO(), StringIO() - current_out, current_err = sys.stdout, sys.stderr - try: - sys.stdout, sys.stderr = capture_out, capture_err - yield capture_out, capture_err - finally: - sys.stdout, sys.stderr = current_out, current_err - - -@patch('bgcval2.analysis_timeseries.main', new=wrapper(analysis_timeseries)) -def test_run_analysis_timeseries_command(): - """Test run command.""" - with arguments('analysis_timeseries', '--help'): - with pytest.raises(SystemExit) as pytest_wrapped_e: - analysis_timeseries_main() - assert pytest_wrapped_e.type == SystemExit - assert pytest_wrapped_e.value.code == 0 - err = "analysis_timeseries: error: the following arguments " \ - "are required: -j/--jobID" - with arguments('analysis_timeseries'): - with pytest.raises(SystemExit) as cm, capture_sys_output() \ - as (stdout, stderr): - analysis_timeseries_main() - assert err in str(stderr.getvalue()) - err = "--jobID: expected at least one argument" - with arguments('analysis_timeseries', '--jobID', '--keys'): - with pytest.raises(SystemExit) as cm, capture_sys_output() \ - as (stdout, stderr): - analysis_timeseries_main() - assert err in str(stderr.getvalue()) - - -@patch('bgcval2.bgcval.run', new=wrapper(bgcval)) -def test_run_bgcval_command(): - """Test run command.""" - with arguments('bgcval', '--help'): - with pytest.raises(SystemExit) as pytest_wrapped_e: - bgcval_main() - assert pytest_wrapped_e.type == SystemExit - assert pytest_wrapped_e.value.code == 0 - err = "the following arguments are required: -i/--job-id\n" - with arguments('bgcval'): - with pytest.raises(SystemExit) as cm, capture_sys_output() \ - as (stdout, stderr): - bgcval_main() - assert err in str(stderr.getvalue()) - - -@patch('bgcval2.download_from_mass.main', new=wrapper(download_from_mass)) -def test_download_from_mass_command(): - """Test run command.""" - with arguments('download_from_mass', '--help'): - with pytest.raises(SystemExit) as pytest_wrapped_e: - download_from_mass_main() - assert pytest_wrapped_e.type == SystemExit - assert pytest_wrapped_e.value.code == 0 - err = "the following arguments are required: -j/--jobID" - with arguments('download_from_mass'): - with pytest.raises(SystemExit) as cm, capture_sys_output() \ - as (stdout, stderr): - download_from_mass_main() - assert err in str(stderr.getvalue()) - - -@patch('bgcval2.analysis_compare.main', new=wrapper(analysis_compare)) -def test_analysis_compare_command(): - """Test run command.""" - with arguments('analysis_compare', '--help'): - with pytest.raises(SystemExit) as pytest_wrapped_e: - analysis_compare_main() - assert pytest_wrapped_e.type == SystemExit - assert pytest_wrapped_e.value.code == 0 - err = "the following arguments are required: -y/--compare_yml" - with arguments('analysis_compare'): - with pytest.raises(SystemExit) as cm, capture_sys_output() \ - as (stdout, stderr): - analysis_compare_main() - assert err in str(stderr.getvalue()) - - -@patch('bgcval2.bgcval2_make_report.main', new=wrapper(bgcval2_make_report)) -def test_bgcval2_make_report_command(): - """Test run command.""" - with arguments('analysis_compare', '--help'): - with pytest.raises(SystemExit) as pytest_wrapped_e: - bgcval2_make_report_main() - assert pytest_wrapped_e.type == SystemExit - assert pytest_wrapped_e.value.code == 0 - err = "the following arguments are required: -i/--job-id" - with arguments('bgcval2_make_report'): - with pytest.raises(SystemExit) as cm, capture_sys_output() \ - as (stdout, stderr): - bgcval2_make_report_main() - assert err in str(stderr.getvalue()) - err = "argument -r/--report: expected one argument" - with arguments('bgcval2_make_report', '--job-id DUM', '--report'): - with pytest.raises(SystemExit) as cm, capture_sys_output() \ - as (stdout, stderr): - bgcval2_make_report_main() - assert err in str(stderr.getvalue()) From b0c97c8d6227409f1754bf4f1606c07f99c2c8c5 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 9 Aug 2022 15:34:17 +0100 Subject: [PATCH 30/37] dont run GA on feature branch --- .github/workflows/run-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index ec0fff69..87fb4d2d 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -5,7 +5,6 @@ on: push: branches: - main - - improve_help_messages schedule: - cron: '0 0 * * *' # nightly From f63c666998ba7b29333ca7b76bf7b12bb6a24ac0 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 9 Aug 2022 15:49:34 +0100 Subject: [PATCH 31/37] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9f9d6b08..89c33ec5 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ Running the tool to compare multiple jobs ========================================= The time developmenmt of several models can be compared -and sumarised in a single comparison report html. +and summarized in a single comparison report html. This report can be generated with a single command, based on a simple yml file input: ``` From 717b134e673aefb4bb2f204a7e5a3294aa048710 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 9 Aug 2022 15:49:49 +0100 Subject: [PATCH 32/37] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 89c33ec5..7442f45d 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ These values are: A sample yaml exists in `input_yml/comparison_analysis_template.yml`, -which can be adapted to additional analysis. +which can be adapted to additional analyses. From 8b1287596fe77d1b750fad7f1d3f1b7f76cde5f4 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 9 Aug 2022 15:50:02 +0100 Subject: [PATCH 33/37] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7442f45d..a9c100db 100644 --- a/README.md +++ b/README.md @@ -153,7 +153,7 @@ download_from_mass -j jobID ``` where `jobID` is one or more jobIDs. -This script will only work on jasmin's `mass-cli1` machine, +This script will only work on JASMIN's `mass-cli1` machine, which is set up to interact with the Met Office Storate System MASS. The optional flag `--dry-run` skips the download part of the script, From b9ee0af9dfab6246d8b1033f85a9f11fc7556a4c Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 9 Aug 2022 15:50:17 +0100 Subject: [PATCH 34/37] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a9c100db..63f216fb 100644 --- a/README.md +++ b/README.md @@ -214,7 +214,7 @@ against historical observations. The command to run it is `analysis_timeseries --jobID jobID --keys key`, where jobID is one or more mass jobIDs, such a `u-ab123`. -The key is one or more pre-defined key, which generates a +The key is one or more pre-defined keys, which generates a list of variables. Key | What it is | Description From ad7dae4e49c5e142077b8742b7a0c81235d8e36c Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 9 Aug 2022 15:50:35 +0100 Subject: [PATCH 35/37] Update bgcval2/analysis_timeseries.py --- bgcval2/analysis_timeseries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index 9534d8a1..30ed9261 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -4941,7 +4941,7 @@ def get_args(): nargs='+', type=str, default=None, - help='One or more UKESM Job IDs (automatically generated by cycl/rose suite).', + help='One or more UKESM Job IDs (automatically generated by the cylc/rose suite).', required=True, ) From 678eaf9f4f65d6aa20d4e1dcf23f203b0a7a8339 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 9 Aug 2022 15:50:50 +0100 Subject: [PATCH 36/37] Update bgcval2/download_from_mass.py --- bgcval2/download_from_mass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgcval2/download_from_mass.py b/bgcval2/download_from_mass.py index 13e16144..e45b9574 100755 --- a/bgcval2/download_from_mass.py +++ b/bgcval2/download_from_mass.py @@ -85,7 +85,7 @@ def mnStr(month): def getYearFromFile(fn): """ - Takes a file anem, and looks for 8 consequetive numbers, then removes those that are months, and returns the year. + Takes a file name, and looks for 8 consecutive numbers, then removes those that are months, and returns the year. """ a = findall(r'\d\d\d\d\d\d\d\d', fn) a.reverse() # prefer second year. From f06e230f272727356b086ac1aec52a3ee8e6545b Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 9 Aug 2022 15:51:13 +0100 Subject: [PATCH 37/37] Update bgcval2/download_from_mass.py --- bgcval2/download_from_mass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgcval2/download_from_mass.py b/bgcval2/download_from_mass.py index e45b9574..0d6cf425 100755 --- a/bgcval2/download_from_mass.py +++ b/bgcval2/download_from_mass.py @@ -218,7 +218,7 @@ def downloadField(jobID, This tool takes the jobID, the field name, and using the known structure of universally similar MASS and the local filesystem structure from paths.py, downloads the monthly jobID data for the field requested to the local file structure. - This tool will only work on machines that have mass enabled. + This tool will only work on machines that have connection to MASS enabled. """