From ecb71fa4375d613d47efcea287eb92ecc816fba5 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 15 Aug 2022 12:20:05 +0100 Subject: [PATCH 01/27] Wrote an interface to test new keys list. --- bgcval2/analysis_timeseries.py | 75 ++++++++++++++++++++++++++++------ key_lists/kmf.yml | 12 ++++++ key_lists/physics.yml | 57 ++++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 13 deletions(-) create mode 100644 key_lists/kmf.yml create mode 100644 key_lists/physics.yml diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index 30ed9261..d2cea80a 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -48,6 +48,7 @@ import os, sys from getpass import getuser import itertools +import yaml ##### # Load specific local code: @@ -252,9 +253,52 @@ def listModelDataFiles(jobID, filekey, datafolder, annual): return model_files +def build_list_of_suite_keys(suites, debug=True): + """ + Generate a list of keys from a list of suites. + + """ + + paths_dir = os.path.dirname(os.path.realpath(__file__)) + key_lists_dir = os.path.join(os.path.dirname(paths_dir), 'key_lists') + + analysisKeys = {} + for suite in suites: + # look for a list in keys_list directory: + suite_yml = os.path.join(key_lists_dir, suite.lower(),'.yml') + if debug: + print('build_list_of_suite_keys:\tlooking for suite yaml:', suite_yml) + + if not os.path.exists(suite_yml): + print('build_list_of_suite_keys:\tERROR: suite yaml:', suite_yml, 'does not exist') + sys.exit(1) + + # Open yml file: + with open(master_compare_yml_fn, 'r') as openfile: + dictionary = yaml.safe_load(openfile) + + if not dictionary or not isinstance(dictionary, dict): + print(f"Configuration file {master_compare_yml_fn} " + "is either empty or corrupt, please check its contents") + sys.exit(1) + + keys_dict = dictionary.get('keys', {}) + + for key, keybool in keys_dict: + if key in analysisKeys and keybool != analysisKeys[key]: + print('build_list_of_suite_keys:\tERROR: conflick in input yamls:', key, keybool, '!=', analysisKeys[key]) + sys.exit(1) + + if keybool: + analysisKey[key] = keybool + analysisKeys = [key for key in analysisKeys.keys()] + return analysisKeys + + + def analysis_timeseries( jobID="u-ab671", - analysisSuite='all', + suites=['all', ], regions='all', clean=0, annual=True, @@ -288,9 +332,9 @@ def analysis_timeseries( print('-----------------------') print('Starting analysis_timeseries') print('jobID:', jobID) - print('analysisSuite:',analysisSuite) + print('suites:', suites) print('regions:', regions) - print('clean:', clean, 'annual:',annual, 'strictFileCheck:', strictFileCheck) + print('clean:', clean, 'annual:', annual, 'strictFileCheck:', strictFileCheck) print('config_user:', config_user) # get runtime configuration @@ -309,19 +353,24 @@ def analysis_timeseries( # Switches: # These are some booleans that allow us to choose which analysis to run. # This lets up give a list of keys one at a time, or in parrallel. - if type(analysisSuite) == type(['Its', 'A', 'list!']): - analysisKeys = analysisSuite + #if type(suites) == type(['Its', 'A', 'list!']): + if isinstrance(suites, str): + suites = [suites, ] + + analysisKeys = build_list_of_suite_keys(suites, debug = True) + print('analysisKeys', analysisKeys) + assert 0 + # analysisKeys = analysisSuite ##### # Switches: # These are some preset switches to run in series. - if type(analysisSuite) == type('Its_A_string'): - analysisKeys = [] - if analysisSuite.lower() in [ - 'keymetricsfirst', 'kmf', - ]: - analysisKeys.extend(keymetricsfirstKeys) + + # if analysisSuite.lower() in [ + # 'keymetricsfirst', 'kmf', + # ]: + # analysisKeys.extend(keymetricsfirstKeys) if analysisSuite.lower() in [ 'level1', @@ -4991,10 +5040,10 @@ def main(): "Will proceed with defaults.") config_user = None - for jobID, suite in itertools.product(keys, jobIDs): + for jobID in itertools.product(jobIDs): analysis_timeseries( jobID=jobID, - analysisSuite=suite, + suites=keys, config_user=config_user ) diff --git a/key_lists/kmf.yml b/key_lists/kmf.yml new file mode 100644 index 00000000..2a6b805d --- /dev/null +++ b/key_lists/kmf.yml @@ -0,0 +1,12 @@ +keys: + TotalAirSeaFluxCO2: True + #NoCaspianAirSeaFluxCO2: True + #IntPP_OSU: True + #GlobalExportRatio: True + #TotalIceExtent: True + #NorthernTotalIceExtent: True + #SouthernTotalIceExtent: True + DrakePassageTransport: True + AMOC_26N: True + GlobalMeanTemperature: True + #GlobalMeanSalinity: True diff --git a/key_lists/physics.yml b/key_lists/physics.yml new file mode 100644 index 00000000..9e03578f --- /dev/null +++ b/key_lists/physics.yml @@ -0,0 +1,57 @@ +keys: + # Temperature keys: + Temperature: True # WOA Temperature + GlobalMeanTemperature: True # Global Mean Temperature + GlobalMeanTemperature_700: True + GlobalMeanTemperature_2000: True + VolumeMeanTemperature: True # Global Mean Temperature + #VolWeightedT: True # Volume weighted WOA Temperature + # IcelessMeanSST: True # Global Mean Surface Temperature with no ice + thetaoga: True + + # Salinity keys: + GlobalMeanSalinity: True # Global Mean Salinity + Salinity: True # WOA Salinity + soga: True + + # Mixed layer depth keys: + MLD: True # iFERMER Mixed Layer Depth + # MaxMonthlyMLD: True # MLD Monthly max + # MinMonthlyMLD: True # MLD Monthly min + + # Ice keys: + # TotalIceArea: True # work in progress + # NorthernTotalIceArea: True # work in progress + # SouthernTotalIceArea: True # work in progress + # WeddelTotalIceArea: True + TotalIceExtent: True # work in progress + NorthernTotalIceExtent: True # work in progress + SouthernTotalIceExtent: True # work in progress + # WeddelIceExent: True # work in progress + # NorthernMIZArea: True + # SouthernMIZArea: True + # TotalMIZArea: True + # NorthernMIZfraction: True + # SouthernMIZfraction: True + # TotalMIZfraction: True + + # Circulation keys: + DrakePassageTransport: True # DrakePassageTransport + # AMOC_32S: True # AMOC 32S + AMOC_26N: True # AMOC 26N + # AMOC_26N_nomexico: True # AMOC 26N + # ADRC_26N: True # ADRC 26N + # ZonalCurrent: True # Zonal Veloctity + # MeridionalCurrent: True # Meridional Veloctity + # VerticalCurrent: True # Vertical Veloctity + + # Misc fluxes: + #FreshwaterFlux: True # Freshwater flux + sowaflup: True # Net Upward Water Flux + # soicecov: True # Ice fraction + # HeatFlux: True + # WindStress: True # Wind Stress + # sohefldo: True # Net downward Water Flux + # sofmflup: True # Water flux due to freezing/melting + # sosfldow: True # Downward salt flux + # sossheig: True # Sea surface height From a236d2df165ab39b8076a732929c6892a4bd4e7e Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 15 Aug 2022 13:48:45 +0100 Subject: [PATCH 02/27] Debugged new interface --- bgcval2/analysis_timeseries.py | 25 +++++++++++++++++-------- key_lists/bgc.yml | 0 2 files changed, 17 insertions(+), 8 deletions(-) create mode 100644 key_lists/bgc.yml diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index d2cea80a..01d119bc 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -258,14 +258,16 @@ def build_list_of_suite_keys(suites, debug=True): Generate a list of keys from a list of suites. """ - + print('build_list_of_suite_keys') paths_dir = os.path.dirname(os.path.realpath(__file__)) key_lists_dir = os.path.join(os.path.dirname(paths_dir), 'key_lists') + print('key_lists_dir', key_lists_dir) analysisKeys = {} for suite in suites: + print(suite) # look for a list in keys_list directory: - suite_yml = os.path.join(key_lists_dir, suite.lower(),'.yml') + suite_yml = os.path.join(key_lists_dir, ''.join([suite.lower(),'.yml'])) if debug: print('build_list_of_suite_keys:\tlooking for suite yaml:', suite_yml) @@ -274,23 +276,30 @@ def build_list_of_suite_keys(suites, debug=True): sys.exit(1) # Open yml file: - with open(master_compare_yml_fn, 'r') as openfile: + with open(suite_yml, 'r') as openfile: dictionary = yaml.safe_load(openfile) if not dictionary or not isinstance(dictionary, dict): - print(f"Configuration file {master_compare_yml_fn} " + print(f"Configuration file {suite_yml} " "is either empty or corrupt, please check its contents") sys.exit(1) keys_dict = dictionary.get('keys', {}) - for key, keybool in keys_dict: + for key, keybool in keys_dict.items(): + if debug and key in analysisKeys: + print('build_list_of_suite_keys:\tKey exists in multiple suites:', key) + + if key in analysisKeys and keybool != analysisKeys[key]: print('build_list_of_suite_keys:\tERROR: conflick in input yamls:', key, keybool, '!=', analysisKeys[key]) sys.exit(1) if keybool: - analysisKey[key] = keybool + if debug: + print('build_list_of_suite_keys:\tAdding key:', key) + + analysisKeys[key] = keybool analysisKeys = [key for key in analysisKeys.keys()] return analysisKeys @@ -354,7 +363,7 @@ def analysis_timeseries( # These are some booleans that allow us to choose which analysis to run. # This lets up give a list of keys one at a time, or in parrallel. #if type(suites) == type(['Its', 'A', 'list!']): - if isinstrance(suites, str): + if isinstance(suites, str): suites = [suites, ] analysisKeys = build_list_of_suite_keys(suites, debug = True) @@ -365,7 +374,7 @@ def analysis_timeseries( ##### # Switches: # These are some preset switches to run in series. - + if True: # if analysisSuite.lower() in [ # 'keymetricsfirst', 'kmf', diff --git a/key_lists/bgc.yml b/key_lists/bgc.yml new file mode 100644 index 00000000..e69de29b From 95577db3a723b6f362b1c8f3a5a38bf33e9d2a19 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 15 Aug 2022 14:02:15 +0100 Subject: [PATCH 03/27] Added set of yml files --- bgcval2/analysis_timeseries.py | 402 ++++++++++++++++----------------- key_lists/bgc.yml | 25 ++ key_lists/debug.yml | 14 ++ key_lists/fast.yml | 43 ++++ key_lists/level1.yml | 84 +++++++ key_lists/salinity.yml | 5 + key_lists/spinup.yml | 9 + 7 files changed, 381 insertions(+), 201 deletions(-) create mode 100644 key_lists/debug.yml create mode 100644 key_lists/fast.yml create mode 100644 key_lists/level1.yml create mode 100644 key_lists/salinity.yml create mode 100644 key_lists/spinup.yml diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index 01d119bc..2b073ead 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -69,91 +69,91 @@ ##### # Biogeochemistry keys -bgcKeys = [] -if True: - bgcKeys.append('N') # WOA Nitrate - bgcKeys.append('Si') # WOA Siliate - bgcKeys.append('O2') # WOA Oxygen - bgcKeys.append('Alk') # Glodap Alkalinity - bgcKeys.append('DIC') # Globap tCO2 - bgcKeys.append('AirSeaFluxCO2') # Air Sea Flux - bgcKeys.append('TotalAirSeaFluxCO2') # Total global air sea flux - bgcKeys.append('IntPP_OSU') # OSU Integrated primpary production - bgcKeys.append('PP_OSU') # OSU Integrated primpary production - # bgcKeys.append('LocalExportRatio') # Export ratio (no data) - # bgcKeys.append('GlobalExportRatio') # Export ratio (no data) - bgcKeys.append('TotalOMZVolume') # Total Oxygen Minimum zone Volume - # bgcKeys.append('OMZThickness') # Oxygen Minimum Zone Thickness - # bgcKeys.append('OMZMeanDepth') # Oxygen Minimum Zone mean depth - bgcKeys.append( - 'VolumeMeanOxygen') # Volune Mean regional oxygen concentation - # bgcKeys.append('AOU') # Apparent Oxygen Usage - bgcKeys.append('Iron') # Iron - bgcKeys.append('Dust') # Dust - # bgcKeys.append('TotalDust') # Total Dust - # bgcKeys.append('DiaFrac') # Diatom Fraction - # bgcKeys.append('DTC') # Detrital carbon - bgcKeys.append('CHL') # Total Chlorophyll - # bgcKeys.append('DMS_ARAN') # Total Chlorophyll - bgcKeys.append('pH') - -bgcKeysDict = {i: n for i, n in enumerate(bgcKeys)} +# bgcKeys = [] +# if True: +# bgcKeys.append('N') # WOA Nitrate +# bgcKeys.append('Si') # WOA Siliate +# bgcKeys.append('O2') # WOA Oxygen +# bgcKeys.append('Alk') # Glodap Alkalinity +# bgcKeys.append('DIC') # Globap tCO2 +# bgcKeys.append('AirSeaFluxCO2') # Air Sea Flux +# bgcKeys.append('TotalAirSeaFluxCO2') # Total global air sea flux +# bgcKeys.append('IntPP_OSU') # OSU Integrated primpary production +# bgcKeys.append('PP_OSU') # OSU Integrated primpary production +# # bgcKeys.append('LocalExportRatio') # Export ratio (no data) +# # bgcKeys.append('GlobalExportRatio') # Export ratio (no data) +# bgcKeys.append('TotalOMZVolume') # Total Oxygen Minimum zone Volume +# # bgcKeys.append('OMZThickness') # Oxygen Minimum Zone Thickness +# # bgcKeys.append('OMZMeanDepth') # Oxygen Minimum Zone mean depth +# bgcKeys.append( +# 'VolumeMeanOxygen') # Volune Mean regional oxygen concentation +# # bgcKeys.append('AOU') # Apparent Oxygen Usage +# bgcKeys.append('Iron') # Iron +# bgcKeys.append('Dust') # Dust +# # bgcKeys.append('TotalDust') # Total Dust +# # bgcKeys.append('DiaFrac') # Diatom Fraction +# # bgcKeys.append('DTC') # Detrital carbon +# bgcKeys.append('CHL') # Total Chlorophyll +# # bgcKeys.append('DMS_ARAN') # Total Chlorophyll +# bgcKeys.append('pH') +# +# bgcKeysDict = {i: n for i, n in enumerate(bgcKeys)} ##### # Physical keys -physKeys = [] -if True: - - physKeys.append('Temperature') # WOA Temperature - #physKeys.append('VolWeightedT') # Volume weighted WOA Temperature - physKeys.append('GlobalMeanTemperature') # Global Mean Temperature - physKeys.append('GlobalMeanTemperature_700') - physKeys.append('GlobalMeanTemperature_2000') - - physKeys.append('VolumeMeanTemperature') # Global Mean Temperature - physKeys.append('GlobalMeanSalinity') # Global Mean Salinity - # physKeys.append('IcelessMeanSST') # Global Mean Surface Temperature with no ice - physKeys.append('Salinity') # WOA Salinity - physKeys.append('MLD') # iFERMER Mixed Layer Depth - - # physKeys.append('TotalIceArea') # work in progress - # physKeys.append('NorthernTotalIceArea') # work in progress - # physKeys.append('SouthernTotalIceArea') # work in progress - # physKeys.append('WeddelTotalIceArea') - physKeys.append('TotalIceExtent') # work in progress - physKeys.append('NorthernTotalIceExtent') # work in progress - physKeys.append('SouthernTotalIceExtent') # work in progress - # physKeys.append('WeddelIceExent') # work in progress - #physKeys.append('NorthernMIZArea') - #physKeys.append('SouthernMIZArea') - #physKeys.append('TotalMIZArea') - # physKeys.append('NorthernMIZfraction') - # physKeys.append('SouthernMIZfraction') - # physKeys.append('TotalMIZfraction') - - physKeys.append('DrakePassageTransport') # DrakePassageTransport - # physKeys.append('AMOC_32S') # AMOC 32S - physKeys.append('AMOC_26N') # AMOC 26N - # physKeys.append('AMOC_26N_nomexico') # AMOC 26N - # physKeys.append('ADRC_26N') # ADRC 26N - # physKeys.append('ZonalCurrent') # Zonal Veloctity - # physKeys.append('MeridionalCurrent') # Meridional Veloctity - # physKeys.append('VerticalCurrent') # Vertical Veloctity - - #physKeys.append('FreshwaterFlux') # Freshwater flux - physKeys.append('sowaflup') # Net Upward Water Flux - # physKeys.append('soicecov') # Ice fraction - - ##### - # unused: - - # physKeys.append('MaxMonthlyMLD') # MLD Monthly max - # physKeys.append('MinMonthlyMLD') # MLD Monthly min - # physKeys.append('HeatFlux') - physKeys.append('TotalHeatFlux') - physKeys.append('scvoltot') - physKeys.append('soga') - physKeys.append('thetaoga') +# physKeys = [] +# if True: +# +# physKeys.append('Temperature') # WOA Temperature +# #physKeys.append('VolWeightedT') # Volume weighted WOA Temperature +# physKeys.append('GlobalMeanTemperature') # Global Mean Temperature +# physKeys.append('GlobalMeanTemperature_700') +# physKeys.append('GlobalMeanTemperature_2000') +# +# physKeys.append('VolumeMeanTemperature') # Global Mean Temperature +# physKeys.append('GlobalMeanSalinity') # Global Mean Salinity +# # physKeys.append('IcelessMeanSST') # Global Mean Surface Temperature with no ice +# physKeys.append('Salinity') # WOA Salinity +# physKeys.append('MLD') # iFERMER Mixed Layer Depth +# +# # physKeys.append('TotalIceArea') # work in progress +# # physKeys.append('NorthernTotalIceArea') # work in progress +# # physKeys.append('SouthernTotalIceArea') # work in progress +# # physKeys.append('WeddelTotalIceArea') +# physKeys.append('TotalIceExtent') # work in progress +# physKeys.append('NorthernTotalIceExtent') # work in progress +# physKeys.append('SouthernTotalIceExtent') # work in progress +# # physKeys.append('WeddelIceExent') # work in progress +# #physKeys.append('NorthernMIZArea') +# #physKeys.append('SouthernMIZArea') +# #physKeys.append('TotalMIZArea') +# # physKeys.append('NorthernMIZfraction') +# # physKeys.append('SouthernMIZfraction') +# # physKeys.append('TotalMIZfraction') +# +# physKeys.append('DrakePassageTransport') # DrakePassageTransport +# # physKeys.append('AMOC_32S') # AMOC 32S +# physKeys.append('AMOC_26N') # AMOC 26N +# # physKeys.append('AMOC_26N_nomexico') # AMOC 26N +# # physKeys.append('ADRC_26N') # ADRC 26N +# # physKeys.append('ZonalCurrent') # Zonal Veloctity +# # physKeys.append('MeridionalCurrent') # Meridional Veloctity +# # physKeys.append('VerticalCurrent') # Vertical Veloctity +# +# #physKeys.append('FreshwaterFlux') # Freshwater flux +# physKeys.append('sowaflup') # Net Upward Water Flux +# # physKeys.append('soicecov') # Ice fraction +# +# ##### +# # unused: +# +# # physKeys.append('MaxMonthlyMLD') # MLD Monthly max +# # physKeys.append('MinMonthlyMLD') # MLD Monthly min +# # physKeys.append('HeatFlux') +# physKeys.append('TotalHeatFlux') +# physKeys.append('scvoltot') +# physKeys.append('soga') +# physKeys.append('thetaoga') # physKeys.append('WindStress') # Wind Stress # physKeys.append('sohefldo') # Net downward Water Flux @@ -161,75 +161,75 @@ # physKeys.append('sosfldow') # Downward salt flux # physKeys.append('sossheig') # Sea surface height -physKeysDict = {i: n for i, n in enumerate(physKeys)} - -fastKeys = [] -if True: - fastKeys.append('N') # WOA Nitrate - fastKeys.append('Si') # WOA Siliate - fastKeys.append('O2') # WOA Oxygen - fastKeys.append('Alk') # Glodap Alkalinity - fastKeys.append('DIC') # Globap tCO2 - fastKeys.append('AirSeaFluxCO2') # Air Sea Flux - fastKeys.append('TotalAirSeaFluxCO2') # Total global air sea flux - fastKeys.append('IntPP_OSU') # OSU Integrated primpary production - fastKeys.append('PP_OSU') # OSU Integrated primpary production - # fastKeys.append('TotalOMZVolume') # Total Oxygen Minimum zone Volume - fastKeys.append( - 'VolumeMeanOxygen') # Volune Mean regional oxygen concentation - fastKeys.append('Iron') # Iron - fastKeys.append('Dust') # Dust - fastKeys.append('CHL') # Total Chlorophyll - fastKeys.append('pH') - - fastKeys.append('Temperature') # WOA Temperature - fastKeys.append('GlobalMeanTemperature') # Global Mean Temperature - fastKeys.append('GlobalMeanTemperature_700') - fastKeys.append('GlobalMeanTemperature_2000') - - fastKeys.append('VolumeMeanTemperature') # Global Mean Temperature - fastKeys.append('GlobalMeanSalinity') # Global Mean Salinity - fastKeys.append('Salinity') # WOA Salinity - fastKeys.append('MLD') # iFERMER Mixed Layer Depth - - fastKeys.append('TotalIceExtent') # work in progress - fastKeys.append('NorthernTotalIceExtent') # work in progress - fastKeys.append('SouthernTotalIceExtent') # work in progress - - fastKeys.append('DrakePassageTransport') # DrakePassageTransport - fastKeys.append('AMOC_26N') # AMOC 26N - - fastKeys.append('FreshwaterFlux') # Freshwater flux - fastKeys.append('sowaflup') # Net Upward Water Flux - - fastKeys.append('TotalHeatFlux') - fastKeys.append('scvoltot') - fastKeys.append('soga') - fastKeys.append('thetaoga') +# physKeysDict = {i: n for i, n in enumerate(physKeys)} + +# fastKeys = [] +# if True: +# fastKeys.append('N') # WOA Nitrate +# fastKeys.append('Si') # WOA Siliate +# fastKeys.append('O2') # WOA Oxygen +# fastKeys.append('Alk') # Glodap Alkalinity +# fastKeys.append('DIC') # Globap tCO2 +# fastKeys.append('AirSeaFluxCO2') # Air Sea Flux +# fastKeys.append('TotalAirSeaFluxCO2') # Total global air sea flux +# fastKeys.append('IntPP_OSU') # OSU Integrated primpary production +# fastKeys.append('PP_OSU') # OSU Integrated primpary production +# # fastKeys.append('TotalOMZVolume') # Total Oxygen Minimum zone Volume +# fastKeys.append( +# 'VolumeMeanOxygen') # Volune Mean regional oxygen concentation +# fastKeys.append('Iron') # Iron +# fastKeys.append('Dust') # Dust +# fastKeys.append('CHL') # Total Chlorophyll +# fastKeys.append('pH') +# +# fastKeys.append('Temperature') # WOA Temperature +# fastKeys.append('GlobalMeanTemperature') # Global Mean Temperature +# fastKeys.append('GlobalMeanTemperature_700') +# fastKeys.append('GlobalMeanTemperature_2000') +# +# fastKeys.append('VolumeMeanTemperature') # Global Mean Temperature +# fastKeys.append('GlobalMeanSalinity') # Global Mean Salinity +# fastKeys.append('Salinity') # WOA Salinity +# fastKeys.append('MLD') # iFERMER Mixed Layer Depth +# +# fastKeys.append('TotalIceExtent') # work in progress +# fastKeys.append('NorthernTotalIceExtent') # work in progress +# fastKeys.append('SouthernTotalIceExtent') # work in progress +# +# fastKeys.append('DrakePassageTransport') # DrakePassageTransport +# fastKeys.append('AMOC_26N') # AMOC 26N +# +# fastKeys.append('FreshwaterFlux') # Freshwater flux +# fastKeys.append('sowaflup') # Net Upward Water Flux +# +# fastKeys.append('TotalHeatFlux') +# fastKeys.append('scvoltot') +# fastKeys.append('soga') +# fastKeys.append('thetaoga') ##### # Level 1 keys -level1Keys = [] -level1Keys.extend(physKeys) -level1Keys.extend(bgcKeys) -level1KeysDict = {i: n for i, n in enumerate(level1Keys)} - -##### -# The important keys -keymetricsfirstKeys = [ - 'TotalAirSeaFluxCO2', - # 'NoCaspianAirSeaFluxCO2', - # 'IntPP_OSU', - # 'GlobalExportRatio', - # 'TotalIceExtent', - # 'NorthernTotalIceExtent', - # 'SouthernTotalIceExtent', - 'DrakePassageTransport', - 'AMOC_26N', - 'GlobalMeanTemperature', - #'GlobalMeanSalinity', -] -keymetricsfirstDict = {i: n for i, n in enumerate(keymetricsfirstKeys)} +# level1Keys = [] +# level1Keys.extend(physKeys) +# level1Keys.extend(bgcKeys) +# level1KeysDict = {i: n for i, n in enumerate(level1Keys)} +# +# ##### +# # The important keys +# keymetricsfirstKeys = [ +# 'TotalAirSeaFluxCO2', +# # 'NoCaspianAirSeaFluxCO2', +# # 'IntPP_OSU', +# # 'GlobalExportRatio', +# # 'TotalIceExtent', +# # 'NorthernTotalIceExtent', +# # 'SouthernTotalIceExtent', +# 'DrakePassageTransport', +# 'AMOC_26N', +# 'GlobalMeanTemperature', +# #'GlobalMeanSalinity', +# ] +# keymetricsfirstDict = {i: n for i, n in enumerate(keymetricsfirstKeys)} def listModelDataFiles(jobID, filekey, datafolder, annual): @@ -289,7 +289,7 @@ def build_list_of_suite_keys(suites, debug=True): for key, keybool in keys_dict.items(): if debug and key in analysisKeys: print('build_list_of_suite_keys:\tKey exists in multiple suites:', key) - + if key in analysisKeys and keybool != analysisKeys[key]: print('build_list_of_suite_keys:\tERROR: conflick in input yamls:', key, keybool, '!=', analysisKeys[key]) @@ -374,58 +374,58 @@ def analysis_timeseries( ##### # Switches: # These are some preset switches to run in series. - if True: - - # if analysisSuite.lower() in [ - # 'keymetricsfirst', 'kmf', - # ]: - # analysisKeys.extend(keymetricsfirstKeys) - - if analysisSuite.lower() in [ - 'level1', - ]: - analysisKeys.extend(level1Keys) - - if analysisSuite.lower() in [ - 'fast', - ]: - analysisKeys.extend(fastKeys) - - if analysisSuite.lower() in [ - 'bgc', - ]: - analysisKeys.extend(bgcKeys) - - if analysisSuite.lower() in [ - 'physics', - ]: - analysisKeys.extend(physKeys) - - if analysisSuite.lower() in [ - 'level3', - ]: - analysisKeys.append('DMS_ARAN') # DMS Aranami Tsunogai - - if analysisSuite.lower() in [ - 'spinup', - ]: - analysisKeys.append('O2') # WOA Oxygen - analysisKeys.append('DIC') # work in progress - analysisKeys.append('Alk') # Glodap Alkalinity - analysisKeys.append('Iron') # work in progress - analysisKeys.append('N') # WOA Nitrate - analysisKeys.append('Si') # WOA Nitrate - analysisKeys.append('Temperature') # # WOA Temperature - analysisKeys.append('Salinity') # # WOA Salinity - if analysisSuite.lower() in [ - 'salinity', - ]: - analysisKeys.append('Salinity') # # WOA Salinity - - if analysisSuite.lower() in [ - 'debug', - ]: - analysisKeys.append('AMOC_26N') # AMOC 26N + # if True: + # + # # if analysisSuite.lower() in [ + # # 'keymetricsfirst', 'kmf', + # # ]: + # # analysisKeys.extend(keymetricsfirstKeys) + # + # if analysisSuite.lower() in [ + # 'level1', + # ]: + # analysisKeys.extend(level1Keys) + # + # if analysisSuite.lower() in [ + # 'fast', + # ]: + # analysisKeys.extend(fastKeys) + # + # if analysisSuite.lower() in [ + # 'bgc', + # ]: + # analysisKeys.extend(bgcKeys) + # + # if analysisSuite.lower() in [ + # 'physics', + # ]: + # analysisKeys.extend(physKeys) + # + # if analysisSuite.lower() in [ + # 'level3', + # ]: + # analysisKeys.append('DMS_ARAN') # DMS Aranami Tsunogai + # + # if analysisSuite.lower() in [ + # 'spinup', + # ]: + # analysisKeys.append('O2') # WOA Oxygen + # analysisKeys.append('DIC') # work in progress + # analysisKeys.append('Alk') # Glodap Alkalinity + # analysisKeys.append('Iron') # work in progress + # analysisKeys.append('N') # WOA Nitrate + # analysisKeys.append('Si') # WOA Nitrate + # analysisKeys.append('Temperature') # # WOA Temperature + # analysisKeys.append('Salinity') # # WOA Salinity + # if analysisSuite.lower() in [ + # 'salinity', + # ]: + # analysisKeys.append('Salinity') # # WOA Salinity + # + # if analysisSuite.lower() in [ + # 'debug', + # ]: + # analysisKeys.append('AMOC_26N') # AMOC 26N #analysisKeys.append('AirSeaFlux') # work in progress #analysisKeys.append('TotalAirSeaFluxCO2') # work in progress diff --git a/key_lists/bgc.yml b/key_lists/bgc.yml index e69de29b..da0f9dad 100644 --- a/key_lists/bgc.yml +++ b/key_lists/bgc.yml @@ -0,0 +1,25 @@ +keys: + N: True # WOA Nitrate + Si: True # WOA Siliate + O2: True # WOA Oxygen + Alk: True # Glodap Alkalinity + DIC: True # Globap tCO2 + AirSeaFluxCO2: True # Air Sea Flux + TotalAirSeaFluxCO2: True # Total global air sea flux + IntPP_OSU: True # OSU Integrated primpary production + PP_OSU: True # OSU Integrated primpary production + # LocalExportRatio: True # Export ratio (no data) + # GlobalExportRatio: True # Export ratio (no data) + TotalOMZVolume: True # Total Oxygen Minimum zone Volume + # OMZThickness: True # Oxygen Minimum Zone Thickness + # OMZMeanDepth: True # Oxygen Minimum Zone mean depth + VolumeMeanOxygen: True # Volune Mean regional oxygen concentation + # AOU: True # Apparent Oxygen Usage + Iron: True # Iron + Dust: True # Dust + # TotalDust: True # Total Dust + # DiaFrac: True # Diatom Fraction + # DTC: True # Detrital carbon + CHL: True # Total Chlorophyll + # DMS_ARAN: True # Total Chlorophyll + pH: True diff --git a/key_lists/debug.yml b/key_lists/debug.yml new file mode 100644 index 00000000..646921e3 --- /dev/null +++ b/key_lists/debug.yml @@ -0,0 +1,14 @@ +--- +# Quick shortlist of keys to help debug analysis code. +keys: + #TotalAirSeaFluxCO2: True + #NoCaspianAirSeaFluxCO2: True + #IntPP_OSU: True + #GlobalExportRatio: True + #TotalIceExtent: True + #NorthernTotalIceExtent: True + #SouthernTotalIceExtent: True + #DrakePassageTransport: True + AMOC_26N: True + #GlobalMeanTemperature: True + #GlobalMeanSalinity: True diff --git a/key_lists/fast.yml b/key_lists/fast.yml new file mode 100644 index 00000000..6802e2b6 --- /dev/null +++ b/key_lists/fast.yml @@ -0,0 +1,43 @@ +# Note that fast refers to the UKESM parameterisation, not that this analysis +# will run faster. +keys: + N: True # WOA Nitrate + Si: True # WOA Siliate + O2: True # WOA Oxygen + Alk: True # Glodap Alkalinity + DIC: True # Globap tCO2 + AirSeaFluxCO2: True # Air Sea Flux + TotalAirSeaFluxCO2: True # Total global air sea flux + IntPP_OSU: True # OSU Integrated primpary production + PP_OSU: True # OSU Integrated primpary production + # TotalOMZVolume: True # Total Oxygen Minimum zone Volume + VolumeMeanOxygen: True # Volune Mean regional oxygen concentation + Iron: True # Iron + Dust: True # Dust + CHL: True # Total Chlorophyll + pH: True + + Temperature: True # WOA Temperature + GlobalMeanTemperature: True # Global Mean Temperature + GlobalMeanTemperature_700: True + GlobalMeanTemperature_2000: True + + VolumeMeanTemperature: True # Global Mean Temperature + GlobalMeanSalinity: True # Global Mean Salinity + Salinity: True # WOA Salinity + MLD: True # iFERMER Mixed Layer Depth + + TotalIceExtent: True # work in progress + NorthernTotalIceExtent: True # work in progress + SouthernTotalIceExtent: True # work in progress + + DrakePassageTransport: True # DrakePassageTransport + AMOC_26N: True # AMOC 26N + + FreshwaterFlux: True # Freshwater flux + sowaflup: True # Net Upward Water Flux + + TotalHeatFlux: True + scvoltot: True + soga: True + thetaoga: True diff --git a/key_lists/level1.yml b/key_lists/level1.yml new file mode 100644 index 00000000..2dc905c2 --- /dev/null +++ b/key_lists/level1.yml @@ -0,0 +1,84 @@ +keys: + # BGC keys: + N: True # WOA Nitrate + Si: True # WOA Siliate + O2: True # WOA Oxygen + Alk: True # Glodap Alkalinity + DIC: True # Globap tCO2 + AirSeaFluxCO2: True # Air Sea Flux + TotalAirSeaFluxCO2: True # Total global air sea flux + IntPP_OSU: True # OSU Integrated primpary production + PP_OSU: True # OSU Integrated primpary production + # LocalExportRatio: True # Export ratio (no data) + # GlobalExportRatio: True # Export ratio (no data) + TotalOMZVolume: True # Total Oxygen Minimum zone Volume + # OMZThickness: True # Oxygen Minimum Zone Thickness + # OMZMeanDepth: True # Oxygen Minimum Zone mean depth + VolumeMeanOxygen: True # Volune Mean regional oxygen concentation + # AOU: True # Apparent Oxygen Usage + Iron: True # Iron + Dust: True # Dust + # TotalDust: True # Total Dust + # DiaFrac: True # Diatom Fraction + # DTC: True # Detrital carbon + CHL: True # Total Chlorophyll + # DMS_ARAN: True # Total Chlorophyll + pH: True + + # Physics: + # Temperature keys: + Temperature: True # WOA Temperature + GlobalMeanTemperature: True # Global Mean Temperature + GlobalMeanTemperature_700: True + GlobalMeanTemperature_2000: True + VolumeMeanTemperature: True # Global Mean Temperature + #VolWeightedT: True # Volume weighted WOA Temperature + # IcelessMeanSST: True # Global Mean Surface Temperature with no ice + thetaoga: True + + # Salinity keys: + GlobalMeanSalinity: True # Global Mean Salinity + Salinity: True # WOA Salinity + soga: True + + # Mixed layer depth keys: + MLD: True # iFERMER Mixed Layer Depth + # MaxMonthlyMLD: True # MLD Monthly max + # MinMonthlyMLD: True # MLD Monthly min + + # Ice keys: + # TotalIceArea: True # work in progress + # NorthernTotalIceArea: True # work in progress + # SouthernTotalIceArea: True # work in progress + # WeddelTotalIceArea: True + TotalIceExtent: True # work in progress + NorthernTotalIceExtent: True # work in progress + SouthernTotalIceExtent: True # work in progress + # WeddelIceExent: True # work in progress + # NorthernMIZArea: True + # SouthernMIZArea: True + # TotalMIZArea: True + # NorthernMIZfraction: True + # SouthernMIZfraction: True + # TotalMIZfraction: True + + # Circulation keys: + DrakePassageTransport: True # DrakePassageTransport + # AMOC_32S: True # AMOC 32S + AMOC_26N: True # AMOC 26N + # AMOC_26N_nomexico: True # AMOC 26N + # ADRC_26N: True # ADRC 26N + # ZonalCurrent: True # Zonal Veloctity + # MeridionalCurrent: True # Meridional Veloctity + # VerticalCurrent: True # Vertical Veloctity + + # Misc fluxes: + #FreshwaterFlux: True # Freshwater flux + sowaflup: True # Net Upward Water Flux + # soicecov: True # Ice fraction + # HeatFlux: True + # WindStress: True # Wind Stress + # sohefldo: True # Net downward Water Flux + # sofmflup: True # Water flux due to freezing/melting + # sosfldow: True # Downward salt flux + # sossheig: True # Sea surface height diff --git a/key_lists/salinity.yml b/key_lists/salinity.yml new file mode 100644 index 00000000..85e86f46 --- /dev/null +++ b/key_lists/salinity.yml @@ -0,0 +1,5 @@ +keys: + # Salinity keys: + GlobalMeanSalinity: True + Salinity: True + soga: True diff --git a/key_lists/spinup.yml b/key_lists/spinup.yml new file mode 100644 index 00000000..2ac8900f --- /dev/null +++ b/key_lists/spinup.yml @@ -0,0 +1,9 @@ +keys: + O2: True # WOA Oxygen + DIC: True # work in progress + Alk: True # Glodap Alkalinity + Iron: True # work in progress + N: True # WOA Nitrate + Si: True # WOA Nitrate + Temperature: True # # WOA Temperature + Salinity: True # # WOA Salinity From 7afe4235e3692b98b2b50213cb641586980ff7b4 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 15 Aug 2022 14:14:51 +0100 Subject: [PATCH 04/27] Unified interface in analysis_compare and analysis_timeseries. --- bgcval2/analysis_compare.py | 341 ++++++++++++++++----------------- bgcval2/analysis_timeseries.py | 2 +- 2 files changed, 162 insertions(+), 181 deletions(-) diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index c0f6f165..932bc796 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -60,7 +60,7 @@ from .timeseries import timeseriesAnalysis from .timeseries import profileAnalysis from .timeseries import timeseriesPlots as tsp -from bgcval2.analysis_timeseries import analysis_timeseries +from bgcval2.analysis_timeseries import analysis_timeseries, build_list_of_suite_keys from bgcval2.download_from_mass import download_from_mass @@ -142,9 +142,7 @@ def apply_shifttimes(mdata, jobID, shifttimes): def timeseries_compare(jobs, colours, - physics=True, - bio=False, - debug=False, + suites = [] analysisname='', shifttimes={}, jobDescriptions={}, @@ -183,163 +181,163 @@ def timeseries_compare(jobs, else: imageFolder = paths.imagedir + '/TimeseriesCompare/' + analysisname - if debug: - imageFolder = imageFolder + '_debug' - analysisname = analysisname + '_debug' - annual = True strictFileCheck = False - analysisKeys = [] - - if physics: - # what if these were loaded in advance from a separate file? - analysisKeys.append('DrakePassageTransport') # DrakePassageTransport - analysisKeys.append('TotalIceArea') # TotalIceArea - analysisKeys.append('NorthernTotalIceArea') # North TotalIceArea - analysisKeys.append('SouthernTotalIceArea') # South TotalIceArea - analysisKeys.append('TotalIceExtent') # work in progress - analysisKeys.append('NorthernTotalIceExtent') # work in progress - analysisKeys.append('SouthernTotalIceExtent') # work in progress - analysisKeys.append('WeddelIceExent') # work in progress - #analysisKeys.append('NorthernMIZArea') - #analysisKeys.append('SouthernMIZArea') - #analysisKeys.append('TotalMIZArea') - analysisKeys.append('NorthernMIZfraction') - analysisKeys.append('SouthernMIZfraction') - analysisKeys.append('TotalMIZfraction') - - analysisKeys.append('AMOC_26N') - analysisKeys.append('AMOC_32S') - analysisKeys.append('ADRC_26N') # AMOC 26N - analysisKeys.append('Temperature') # WOA Temperature - analysisKeys.append('Salinity') # WOA Salinity - analysisKeys.append('MLD') # MLD - analysisKeys.append('MaxMonthlyMLD') # MLD Monthly max - analysisKeys.append('MinMonthlyMLD') # MLD Monthly min - - analysisKeys.append('ZonalCurrent') # Zonal Veloctity - analysisKeys.append('MeridionalCurrent') # Meridional Veloctity - analysisKeys.append('VerticalCurrent') # Vertical Veloctity - analysisKeys.append('GlobalMeanTemperature') # Global Mean Temperature - analysisKeys.append('VolumeMeanTemperature') # Global Mean Temperature - analysisKeys.append('GlobalMeanSalinity') # Global Mean Salinity - analysisKeys.append( - 'IcelessMeanSST') # Global Mean Surface Temperature with no ice - - analysisKeys.append('sowaflup') # Net Upward Water Flux - analysisKeys.append('sohefldo') # Net downward Water Flux - analysisKeys.append('sofmflup') # Water flux due to freezing/melting - analysisKeys.append('sosfldow') # Downward salt flux - analysisKeys.append('soicecov') # Ice fraction - analysisKeys.append('sossheig') # SSH - analysisKeys.append('FreshwaterFlux') # Fresh water flux - analysisKeys.append('HeatFlux') - analysisKeys.append('TotalHeatFlux') - analysisKeys.append('scvoltot') - analysisKeys.append('soga') - analysisKeys.append('thetaoga') - analysisKeys.append('scalarHeatContent') - - if bio: - analysisKeys.append('TotalAirSeaFluxCO2') # work in progress - analysisKeys.append('NoCaspianAirSeaFluxCO2') # work in progress - - analysisKeys.append('AirSeaFlux') # work in progress - analysisKeys.append('IntPP_OSU') # OSU Integrated primpary production - analysisKeys.append('GlobalExportRatio') - - analysisKeys.append('N') # WOA Nitrate - analysisKeys.append('Si') # WOA Siliate - analysisKeys.append('O2') # WOA Oxygen - analysisKeys.append('Iron') - analysisKeys.append('Alk') - analysisKeys.append('DIC') - analysisKeys.append('pH') - - analysisKeys.append('CHD') - analysisKeys.append('CHN') - analysisKeys.append('CHL') - analysisKeys.append('DiaFrac') - analysisKeys.append('DMS') - analysisKeys.append('Dust') # Dust - analysisKeys.append('TotalDust') # Total Dust - - # analysisKeys.append('DMS_ARAN') - - analysisKeys.append('DTC') - - analysisKeys.append( - 'TotalOMZVolume') # Total Oxygen Minimum zone Volume - analysisKeys.append('OMZThickness') # Oxygen Minimum Zone Thickness - analysisKeys.append('OMZMeanDepth') # Oxygen Minimum Zone mean depth - analysisKeys.append('VolumeMeanOxygen') # Volume weighted mean Oxygen - - if debug: - #### - # Supercedes other flags. - analysisKeys = [] - # analysisKeys.append('ERSST') - #analysisKeys.append('DrakePassageTransport') # DrakePassageTransport - # analysisKeys.append('VolumeMeanOxygen') # Volume weighted mean Oxygen - analysisKeys.append('AMOC_26N') - # analysisKeys.append('NoCaspianAirSeaFluxCO2') # work in progress - #analysisKeys.append('VolumeMeanTemperature') # Global Mean Temperature - # analysisKeys.append('CHD') - # analysisKeys.append('CHN') - # analysisKeys.append('DiaFrac') - # analysisKeys.append('AMOC_26N') - # analysisKeys.append('MLD') - # analysisKeys.append('Temperature') # WOA Temperature - # analysisKeys.append('Salinity') # WOA Salinity - # analysisKeys.append('DMS') - # analysisKeys.append('N') # WOA Nitrate - # analysisKeys.append('Si') # WOA Siliate - - # analysisKeys.append('TotalAirSeaFluxCO2') # work in progress - # analysisKeys.append('AirSeaFlux') # work in progress - # analysisKeys.append('AirSeaFluxCO2') # work in progress - - # analysisKeys.append('scvoltot') - # analysisKeys.append('soga') - # analysisKeys.append('thetaoga') - # analysisKeys.append('scalarHeatContent') - - #analysisKeys.append('ADRC_26N') # AMOC 26N - # analysisKeys.append('VerticalCurrent') # Vertical Veloctity - # analysisKeys.append('sossheig') # SSH - # analysisKeys.append('NorthernTotalIceArea') # North TotalIceArea - # analysisKeys.append('SouthernTotalIceArea') # South TotalIceArea - # analysisKeys.append('TotalIceArea') # TotalIceArea - # analysisKeys.append('NorthernTotalIceExtent') # work in progress - # analysisKeys.append('SouthernTotalIceExtent') # work in progress - # analysisKeys.append('WeddelIceExent') # work in progress - # analysisKeys.append('TotalIceExtent') # work in progress - # analysisKeys.append('NorthernMIZArea') - # analysisKeys.append('SouthernMIZArea') - # analysisKeys.append('TotalMIZArea') - - # analysisKeys.append('NorthernMIZArea') - # analysisKeys.append('SouthernMIZArea') - # analysisKeys.append('TotalMIZArea') - # analysisKeys.append('NorthernMIZfraction') - # analysisKeys.append('SouthernMIZfraction') - # analysisKeys.append('TotalMIZfraction') - - # analysisKeys.append('FreshwaterFlux') # Fresh water flux - #analysisKeys.append('GlobalMeanTemperature') -# analysisKeys.append('GlobalMeanSalinity') - -# analysisKeys.append('HeatFlux') -# analysisKeys.append('TotalHeatFlux') - -# analysisKeys.append('quickSST') # Area Weighted Mean Surface Temperature -# analysisKeys.append('TotalOMZVolume') # Total Oxygen Minimum zone Volume -# analysisKeys.append('OMZThickness') # Oxygen Minimum Zone Thickness -# analysisKeys.append('OMZMeanDepth') # Oxygen Minimum Zone mean depth -# analysisKeys.append('O2') # WOA Oxygen -# if bio ==False:return -# if physics == True:return + if isinstance(suites, str): + suites = [suites, ] + + analysisKeys = build_list_of_suite_keys(suites, debug = True) + print('analysisKeys', analysisKeys) + +# if physics: +# # what if these were loaded in advance from a separate file? +# analysisKeys.append('DrakePassageTransport') # DrakePassageTransport +# analysisKeys.append('TotalIceArea') # TotalIceArea +# analysisKeys.append('NorthernTotalIceArea') # North TotalIceArea +# analysisKeys.append('SouthernTotalIceArea') # South TotalIceArea +# analysisKeys.append('TotalIceExtent') # work in progress +# analysisKeys.append('NorthernTotalIceExtent') # work in progress +# analysisKeys.append('SouthernTotalIceExtent') # work in progress +# analysisKeys.append('WeddelIceExent') # work in progress +# #analysisKeys.append('NorthernMIZArea') +# #analysisKeys.append('SouthernMIZArea') +# #analysisKeys.append('TotalMIZArea') +# analysisKeys.append('NorthernMIZfraction') +# analysisKeys.append('SouthernMIZfraction') +# analysisKeys.append('TotalMIZfraction') +# +# analysisKeys.append('AMOC_26N') +# analysisKeys.append('AMOC_32S') +# analysisKeys.append('ADRC_26N') # AMOC 26N +# analysisKeys.append('Temperature') # WOA Temperature +# analysisKeys.append('Salinity') # WOA Salinity +# analysisKeys.append('MLD') # MLD +# analysisKeys.append('MaxMonthlyMLD') # MLD Monthly max +# analysisKeys.append('MinMonthlyMLD') # MLD Monthly min +# +# analysisKeys.append('ZonalCurrent') # Zonal Veloctity +# analysisKeys.append('MeridionalCurrent') # Meridional Veloctity +# analysisKeys.append('VerticalCurrent') # Vertical Veloctity +# analysisKeys.append('GlobalMeanTemperature') # Global Mean Temperature +# analysisKeys.append('VolumeMeanTemperature') # Global Mean Temperature +# analysisKeys.append('GlobalMeanSalinity') # Global Mean Salinity +# analysisKeys.append( +# 'IcelessMeanSST') # Global Mean Surface Temperature with no ice +# +# analysisKeys.append('sowaflup') # Net Upward Water Flux +# analysisKeys.append('sohefldo') # Net downward Water Flux +# analysisKeys.append('sofmflup') # Water flux due to freezing/melting +# analysisKeys.append('sosfldow') # Downward salt flux +# analysisKeys.append('soicecov') # Ice fraction +# analysisKeys.append('sossheig') # SSH +# analysisKeys.append('FreshwaterFlux') # Fresh water flux +# analysisKeys.append('HeatFlux') +# analysisKeys.append('TotalHeatFlux') +# analysisKeys.append('scvoltot') +# analysisKeys.append('soga') +# analysisKeys.append('thetaoga') +# analysisKeys.append('scalarHeatContent') +# +# if bgc: +# analysisKeys.append('TotalAirSeaFluxCO2') # work in progress +# analysisKeys.append('NoCaspianAirSeaFluxCO2') # work in progress +# +# analysisKeys.append('AirSeaFlux') # work in progress +# analysisKeys.append('IntPP_OSU') # OSU Integrated primpary production +# analysisKeys.append('GlobalExportRatio') +# +# analysisKeys.append('N') # WOA Nitrate +# analysisKeys.append('Si') # WOA Siliate +# analysisKeys.append('O2') # WOA Oxygen +# analysisKeys.append('Iron') +# analysisKeys.append('Alk') +# analysisKeys.append('DIC') +# analysisKeys.append('pH') +# +# analysisKeys.append('CHD') +# analysisKeys.append('CHN') +# analysisKeys.append('CHL') +# analysisKeys.append('DiaFrac') +# analysisKeys.append('DMS') +# analysisKeys.append('Dust') # Dust +# analysisKeys.append('TotalDust') # Total Dust +# +# # analysisKeys.append('DMS_ARAN') +# +# analysisKeys.append('DTC') +# +# analysisKeys.append( +# 'TotalOMZVolume') # Total Oxygen Minimum zone Volume +# analysisKeys.append('OMZThickness') # Oxygen Minimum Zone Thickness +# analysisKeys.append('OMZMeanDepth') # Oxygen Minimum Zone mean depth +# analysisKeys.append('VolumeMeanOxygen') # Volume weighted mean Oxygen +# +# if debug: +# #### +# # Supercedes other flags. +# analysisKeys = [] +# # analysisKeys.append('ERSST') +# #analysisKeys.append('DrakePassageTransport') # DrakePassageTransport +# # analysisKeys.append('VolumeMeanOxygen') # Volume weighted mean Oxygen +# analysisKeys.append('AMOC_26N') +# # analysisKeys.append('NoCaspianAirSeaFluxCO2') # work in progress +# #analysisKeys.append('VolumeMeanTemperature') # Global Mean Temperature +# # analysisKeys.append('CHD') +# # analysisKeys.append('CHN') +# # analysisKeys.append('DiaFrac') +# # analysisKeys.append('AMOC_26N') +# # analysisKeys.append('MLD') +# # analysisKeys.append('Temperature') # WOA Temperature +# # analysisKeys.append('Salinity') # WOA Salinity +# # analysisKeys.append('DMS') +# # analysisKeys.append('N') # WOA Nitrate +# # analysisKeys.append('Si') # WOA Siliate +# +# # analysisKeys.append('TotalAirSeaFluxCO2') # work in progress +# # analysisKeys.append('AirSeaFlux') # work in progress +# # analysisKeys.append('AirSeaFluxCO2') # work in progress +# +# # analysisKeys.append('scvoltot') +# # analysisKeys.append('soga') +# # analysisKeys.append('thetaoga') +# # analysisKeys.append('scalarHeatContent') +# +# #analysisKeys.append('ADRC_26N') # AMOC 26N +# # analysisKeys.append('VerticalCurrent') # Vertical Veloctity +# # analysisKeys.append('sossheig') # SSH +# # analysisKeys.append('NorthernTotalIceArea') # North TotalIceArea +# # analysisKeys.append('SouthernTotalIceArea') # South TotalIceArea +# # analysisKeys.append('TotalIceArea') # TotalIceArea +# # analysisKeys.append('NorthernTotalIceExtent') # work in progress +# # analysisKeys.append('SouthernTotalIceExtent') # work in progress +# # analysisKeys.append('WeddelIceExent') # work in progress +# # analysisKeys.append('TotalIceExtent') # work in progress +# # analysisKeys.append('NorthernMIZArea') +# # analysisKeys.append('SouthernMIZArea') +# # analysisKeys.append('TotalMIZArea') +# +# # analysisKeys.append('NorthernMIZArea') +# # analysisKeys.append('SouthernMIZArea') +# # analysisKeys.append('TotalMIZArea') +# # analysisKeys.append('NorthernMIZfraction') +# # analysisKeys.append('SouthernMIZfraction') +# # analysisKeys.append('TotalMIZfraction') +# +# # analysisKeys.append('FreshwaterFlux') # Fresh water flux +# #analysisKeys.append('GlobalMeanTemperature') +# # analysisKeys.append('GlobalMeanSalinity') +# +# # analysisKeys.append('HeatFlux') +# # analysisKeys.append('TotalHeatFlux') +# +# # analysisKeys.append('quickSST') # Area Weighted Mean Surface Temperature +# # analysisKeys.append('TotalOMZVolume') # Total Oxygen Minimum zone Volume +# # analysisKeys.append('OMZThickness') # Oxygen Minimum Zone Thickness +# # analysisKeys.append('OMZMeanDepth') # Oxygen Minimum Zone mean depth +# # analysisKeys.append('O2') # WOA Oxygen +# # if bgc ==False:return +# # if physics == True:return layerList = [ 'Surface', @@ -4036,7 +4034,7 @@ def flatten(lats, lons, dataA, dataB): def CompareTwoRuns(jobIDA, jobIDB, physics=True, - bio=True, + bgc=True, yearA='', yearB='', debug=True): @@ -4051,7 +4049,7 @@ def CompareTwoRuns(jobIDA, 'grid_V', 'grid_W', ]) - if bio: + if bgc: filetype.extend([ 'diad_T', 'ptrc_T', @@ -4245,7 +4243,7 @@ def load_yml_and_run(compare_yml, config_user): print(jobID, 'suite:', suites[jobID]) for jobID in jobs: - # even if you don't want to download, we run this + # even if you don't want to download, we run this # as it clears up the path and ensures recently downloed data is # correctly symlinked. download_from_mass(jobID, doMoo=do_mass_download) @@ -4260,7 +4258,7 @@ def load_yml_and_run(compare_yml, config_user): # Master suite leys: if not master_suites: - master_suites=['physics', 'bio'] # Defaults + master_suites=['physics', 'bgc'] # Defaults # make sure its a list: if isinstance(master_suites, list) : @@ -4271,27 +4269,10 @@ def load_yml_and_run(compare_yml, config_user): master_suites = master_suites.replace(split_char, ';') master_suites = master_suites.split(';') - if 'physics' in master_suites: - key_physics = True - else: - key_physics = False - - if 'bio' in master_suites: - key_bio = True - else: - key_bio = False - - if 'debug' in master_suites: - key_debug = True - else: - key_debug = False - timeseries_compare( jobs, colours = colours, - physics=key_physics, - bio=key_bio, - debug=key_debug, + suites = master_suites, shifttimes=shifttimes, jobDescriptions=descriptions, analysisname=analysis_name, diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index 2b073ead..6be515b7 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -368,7 +368,7 @@ def analysis_timeseries( analysisKeys = build_list_of_suite_keys(suites, debug = True) print('analysisKeys', analysisKeys) - assert 0 + # assert 0 # analysisKeys = analysisSuite ##### From 8c3d2b135f2bbc026fb17c2012079270fc51145a Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 15 Aug 2022 15:29:55 +0100 Subject: [PATCH 05/27] improved some other aspects of interface --- bgcval2/UKESMpython.py | 4 +- bgcval2/analysis_compare.py | 4 +- bgcval2/analysis_timeseries.py | 170 ++++++--------------------------- 3 files changed, 33 insertions(+), 145 deletions(-) diff --git a/bgcval2/UKESMpython.py b/bgcval2/UKESMpython.py index f497c517..718d4539 100644 --- a/bgcval2/UKESMpython.py +++ b/bgcval2/UKESMpython.py @@ -71,8 +71,8 @@ def folder(name): """ This snippet takes a string, makes the folder and the string. It also accepts lists of strings. """ - if type(name) == type(['a', 'b', 'c']): - name = join(name, '/') + if isinstance(name, list): + name = os.path.join(*name) # splat! if name[-1] != '/': name = name + '/' if exists(name) is False: diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index 932bc796..0e608cfa 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -142,7 +142,7 @@ def apply_shifttimes(mdata, jobID, shifttimes): def timeseries_compare(jobs, colours, - suites = [] + suites = [], analysisname='', shifttimes={}, jobDescriptions={}, @@ -4217,7 +4217,7 @@ def load_yml_and_run(compare_yml, config_user): """ # Below here is analysis - details = load_comparison_yml(comp_config) + details = load_comparison_yml(compare_yml) jobs = details['jobs'] analysis_name = details['name'] diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index 6be515b7..882351df 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -575,16 +575,11 @@ def analysis_timeseries( 'NorthernHemisphere', ] - if analysisSuite.lower() == 'debug': - regionList = ['Global', 'ArcticOcean'] - - if analysisSuite.lower() in [ - 'spinup', - 'salinity', - ]: + if regions in ['debug', 'spinup']: regionList = [ 'Global', - ] + ] + metricList = [ 'mean', ] @@ -634,83 +629,21 @@ def analysis_timeseries( # So far, this has been run on the following machines: # PML # JASMIN - # Charybdis (Julien's machine at NOCS) # - # Feel free to add other macihines onto this list, if need be. machinelocation = '' + shelvedir = ukp.folder([paths.shelvedir, "timeseries", jobID]) + imagedir = ukp.folder([paths.imagedir, jobID, 'timeseries']) - shelvedir = ukp.folder(paths.shelvedir + "/timeseries/" + jobID) + if annual: WOAFolder = paths.WOAFolder_annual + else: WOAFolder = paths.WOAFolder ##### - # PML hostname = gethostname() + print("analysis-timeseries.py:\tBeing run at ", hostname) - if hostname.find('pmpc') > -1: - print("analysis-timeseries.py:\tBeing run at PML on ", gethostname()) - - imagedir = ukp.folder(paths.imagedir + '/' + jobID + '/timeseries') - - if annual: WOAFolder = paths.WOAFolder_annual - else: WOAFolder = paths.WOAFolder - - #shelvedir = ukp.folder(paths.shelvedir+'/'+jobID+'/timeseries/'+jobID) - #shelvedir = ukp.folder(paths.shelvedir + "/timeseries/" + jobID) ##### - # JASMIN - if hostname.find('ceda.ac.uk') > -1 or hostname.find( - 'jasmin') > -1 or hostname.find('jc.rl.ac.uk') > -1: - print("analysis-timeseries.py:\tBeing run at CEDA on ", hostname) - #machinelocation = 'JASMIN' - - #try: shelvedir = ukp.folder("/group_workspaces/jasmin2/ukesm/BGC_data/"+getuser()+"/shelves/timeseries/"+jobID) - #except: shelvedir = "/group_workspaces/jasmin2/ukesm/BGC_data/"+getuser()+"/shelves/timeseries/"+jobID - #try: - # shelvedir = ukp.folder("/gws/nopw/j04/ukesm/BGC_data/" + - # getuser() + "/shelves/timeseries/" + jobID) - #except: - # shelvedir = "/gws/nopw/j04/ukesm/BGC_data/" + getuser( - # ) + "/shelves/timeseries/" + jobID - - if annual: WOAFolder = paths.WOAFolder_annual - else: WOAFolder = paths.WOAFolder - - try: - imagedir = ukp.folder(paths.imagedir + '/' + jobID + '/timeseries') - except: - imagedir = paths.imagedir + '/' + jobID + '/timeseries' - - if hostname.find('monsoon') > -1: - print("Please set up paths.py") - assert 0 - -#print "analysis-timeseries.py:\tBeing run at the Met Office on ",gethostname() -#machinelocation = 'MONSOON' - -#ObsFolder = "/projects/ukesm/ldmora/BGC-data/" -#ModelFolder = "/projects/ukesm/ldmora/UKESM" -##### -# Location of model files. -#MEDUSAFolder_pref = ukp.folder(ModelFolder) - -##### -# Location of data files. -#if annual: WOAFolder = ukp.folder(ObsFolder+"WOA/annual") -#else: WOAFolder = ukp.folder(ObsFolder+"WOA/") - -#MAREDATFolder = ObsFolder+"/MAREDAT/MAREDAT/" -#GEOTRACESFolder = ObsFolder+"/GEOTRACES/GEOTRACES_PostProccessed/" -#TakahashiFolder = ObsFolder+"/Takahashi2009_pCO2/" -#MLDFolder = ObsFolder+"/IFREMER-MLD/" -#iMarNetFolder = ObsFolder+"/LestersReportData/" -#GlodapDir = ObsFolder+"/GLODAP/" -#GLODAPv2Dir = ObsFolder+"/GLODAPv2/GLODAPv2_Mapped_Climatologies/" -#OSUDir = ObsFolder+"OSU/" -#CCIDir = ObsFolder+"CCI/" -#orcaGridfn = ModelFolder+'/mesh_mask_eORCA1_wrk.nc' - -##### -# Unable to find location of files/data. + # Unable to find location of files/data. if not paths.machinelocation: print( "analysis-timeseries.py:\tFATAL:\tWas unable to determine location of host: ", @@ -722,10 +655,9 @@ def analysis_timeseries( else: assert False -##### -# Because we can never be sure someone won't randomly rename the -# time dimension without saying anything. -# if jobID in ['u-am515','u-am927','u-am064','u-an326',]: + ##### + # Because we can never be sure someone won't randomly rename the + # time dimension without saying anything. try: tmpModelFiles = listModelDataFiles(jobID, 'grid_T', paths.ModelFolder_pref, annual) @@ -773,66 +705,22 @@ def analysis_timeseries( ukesmkeys['w3d'] = 'wo' ukesmkeys['MLD'] = 'ssomxl010' -# else: -# ukesmkeys['time'] = 'time_centered' -# ukesmkeys['temp3d'] = 'thetao' -# ukesmkeys['sst'] = 'tos' -# ukesmkeys['sal3d'] = 'so' -# ukesmkeys['sss'] = 'sos' -# ukesmkeys['v3d'] = 'vo' -# ukesmkeys['u3d'] = 'uo' -# ukesmkeys['e3u'] = 'thkcello' -# ukesmkeys['w3d'] = 'wo' - -# ukesmkeys['time'] = 'time_counter' -# ukesmkeys['temp3d'] = 'votemper' -# ukesmkeys['sst'] = '' -# ukesmkeys['sal3d'] = 'vosaline' -# ukesmkeys['sss'] = '' -# ukesmkeys['v3d'] = 'vomecrty' -# ukesmkeys['u3d'] = 'vozocrtx' -# ukesmkeys['e3u'] = 'e3u' -# ukesmkeys['w3d'] = 'vovecrtz' - -# if jobID > 'u-am514' and jobID not in ['u-an619','u-an629','u-an631','u-an869', 'u-an908', 'u-an911','u-an989',]: -# # There are other changes here too. -# ##### -# # Because we can never be sure someone won't randomly rename the -# # time dimension without saying anything. -# ukesmkeys={} -# ukesmkeys['time'] = 'time_centered' -# ukesmkeys['temp3d'] = 'thetao' -# ukesmkeys['sst'] = 'tos' -# ukesmkeys['sal3d'] = 'so' -# ukesmkeys['sss'] = 'sos' -# ukesmkeys['v3d'] = 'vo' -# ukesmkeys['u3d'] = 'uo' -# ukesmkeys['e3u'] = 'thkcello' -# ukesmkeys['w3d'] = 'wo' -# -# else: -# ukesmkeys={} -# ukesmkeys['time'] = 'time_counter' -# ukesmkeys['temp3d'] = 'votemper' -# ukesmkeys['sst'] = '' -# ukesmkeys['sal3d'] = 'vosaline' -# ukesmkeys['sss'] = '' -# ukesmkeys['v3d'] = 'vomecrty' -# ukesmkeys['u3d'] = 'vozocrtx' -# ukesmkeys['e3u'] = 'e3u' -# ukesmkeys['w3d'] = 'vovecrtz' - -##### -# Coordinate dictionairy -# These are python dictionairies, one for each data source and model. -# This is because each data provider seems to use a different set of standard names for dimensions and time. -# The 'tdict' field is short for "time-dictionary". -# This is a dictionary who's indices are the values on the netcdf time dimension. -# The tdict indices point to a month number in python numbering (ie January = 0) -# An example would be, if a netcdf uses the middle day of the month as it's time value: -# tdict = {15:0, 45:1 ...} - - timekey = ukesmkeys['time'] + ##### + # Coordinate dictionairy + # These are python dictionairies, one for each data source and model. + # This is because each data provider seems to use a different set of standard names for dimensions and time. + # The 'tdict' field is short for "time-dictionary". + # This is a dictionary who's indices are the values on the netcdf time dimension. + # The tdict indices point to a month number in python numbering (ie January = 0) + # An example would be, if a netcdf uses the middle day of the month as it's time value: + # tdict = {15:0, 45:1 ...} + + try: timekey = ukesmkeys['time'] + except KeyError: + print('ERROR: bgcval2 was unable to determine what kind of model data this was.') + print(' Is any data present at:',tmpModelFiles, paths.ModelFolder_pref, jobID) + print(' Otherwise, what name is time given in that data?') + sys.exit(1) medusaCoords = { 't': timekey, 'z': 'deptht', @@ -5049,7 +4937,7 @@ def main(): "Will proceed with defaults.") config_user = None - for jobID in itertools.product(jobIDs): + for jobID in jobIDs: analysis_timeseries( jobID=jobID, suites=keys, From 3e5fafdd4a9eb50755ff2ec8ea3d6f079f4f920c Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 15 Aug 2022 15:42:52 +0100 Subject: [PATCH 06/27] Update bgc.yml --- key_lists/bgc.yml | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/key_lists/bgc.yml b/key_lists/bgc.yml index da0f9dad..f01aaded 100644 --- a/key_lists/bgc.yml +++ b/key_lists/bgc.yml @@ -1,25 +1,38 @@ keys: + # Nutrients N: True # WOA Nitrate Si: True # WOA Siliate - O2: True # WOA Oxygen - Alk: True # Glodap Alkalinity - DIC: True # Globap tCO2 - AirSeaFluxCO2: True # Air Sea Flux - TotalAirSeaFluxCO2: True # Total global air sea flux + Iron: True # Iron + + # Production, Export, Chlorophyll and DMS + CHL: True # Total Chlorophyll + # CHD: True # Diatom Chlorophyll + # CHN: True # Non-diatom Chlorophyll + # DiaFrac: True # Diatom Fraction + IntPP_OSU: True # OSU Integrated primpary production PP_OSU: True # OSU Integrated primpary production + # DMS_ARAN: True + # DMS: True + # DTC: True # Detrital carbon # LocalExportRatio: True # Export ratio (no data) # GlobalExportRatio: True # Export ratio (no data) + + # Carbonate system: + Alk: True # Glodap Alkalinity + DIC: True # Globap tCO2 + AirSeaFluxCO2: True # Air Sea Flux + TotalAirSeaFluxCO2: True # Total global air sea flux + pH: True + + # Oxygen: + O2: True # WOA Oxygen TotalOMZVolume: True # Total Oxygen Minimum zone Volume # OMZThickness: True # Oxygen Minimum Zone Thickness # OMZMeanDepth: True # Oxygen Minimum Zone mean depth VolumeMeanOxygen: True # Volune Mean regional oxygen concentation # AOU: True # Apparent Oxygen Usage - Iron: True # Iron + + #Dust Dust: True # Dust # TotalDust: True # Total Dust - # DiaFrac: True # Diatom Fraction - # DTC: True # Detrital carbon - CHL: True # Total Chlorophyll - # DMS_ARAN: True # Total Chlorophyll - pH: True From 566ba47184c0873fc3e6e18721c6687b5d9e7843 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 15 Aug 2022 15:47:39 +0100 Subject: [PATCH 07/27] Update physics.yml --- key_lists/physics.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/key_lists/physics.yml b/key_lists/physics.yml index 9e03578f..15e51662 100644 --- a/key_lists/physics.yml +++ b/key_lists/physics.yml @@ -55,3 +55,6 @@ keys: # sofmflup: True # Water flux due to freezing/melting # sosfldow: True # Downward salt flux # sossheig: True # Sea surface height + # TotalHeatFlux: True # Total Heat Flux + # scvoltot: True + # scalarHeatContent: True From 25d3682270efdb05105362b5702e78c963e74c76 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 15 Aug 2022 15:51:23 +0100 Subject: [PATCH 08/27] Deleted commented out lines --- bgcval2/analysis_compare.py | 149 ---------- bgcval2/analysis_timeseries.py | 525 ++++++++++----------------------- 2 files changed, 148 insertions(+), 526 deletions(-) diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index 0e608cfa..60088904 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -190,155 +190,6 @@ def timeseries_compare(jobs, analysisKeys = build_list_of_suite_keys(suites, debug = True) print('analysisKeys', analysisKeys) -# if physics: -# # what if these were loaded in advance from a separate file? -# analysisKeys.append('DrakePassageTransport') # DrakePassageTransport -# analysisKeys.append('TotalIceArea') # TotalIceArea -# analysisKeys.append('NorthernTotalIceArea') # North TotalIceArea -# analysisKeys.append('SouthernTotalIceArea') # South TotalIceArea -# analysisKeys.append('TotalIceExtent') # work in progress -# analysisKeys.append('NorthernTotalIceExtent') # work in progress -# analysisKeys.append('SouthernTotalIceExtent') # work in progress -# analysisKeys.append('WeddelIceExent') # work in progress -# #analysisKeys.append('NorthernMIZArea') -# #analysisKeys.append('SouthernMIZArea') -# #analysisKeys.append('TotalMIZArea') -# analysisKeys.append('NorthernMIZfraction') -# analysisKeys.append('SouthernMIZfraction') -# analysisKeys.append('TotalMIZfraction') -# -# analysisKeys.append('AMOC_26N') -# analysisKeys.append('AMOC_32S') -# analysisKeys.append('ADRC_26N') # AMOC 26N -# analysisKeys.append('Temperature') # WOA Temperature -# analysisKeys.append('Salinity') # WOA Salinity -# analysisKeys.append('MLD') # MLD -# analysisKeys.append('MaxMonthlyMLD') # MLD Monthly max -# analysisKeys.append('MinMonthlyMLD') # MLD Monthly min -# -# analysisKeys.append('ZonalCurrent') # Zonal Veloctity -# analysisKeys.append('MeridionalCurrent') # Meridional Veloctity -# analysisKeys.append('VerticalCurrent') # Vertical Veloctity -# analysisKeys.append('GlobalMeanTemperature') # Global Mean Temperature -# analysisKeys.append('VolumeMeanTemperature') # Global Mean Temperature -# analysisKeys.append('GlobalMeanSalinity') # Global Mean Salinity -# analysisKeys.append( -# 'IcelessMeanSST') # Global Mean Surface Temperature with no ice -# -# analysisKeys.append('sowaflup') # Net Upward Water Flux -# analysisKeys.append('sohefldo') # Net downward Water Flux -# analysisKeys.append('sofmflup') # Water flux due to freezing/melting -# analysisKeys.append('sosfldow') # Downward salt flux -# analysisKeys.append('soicecov') # Ice fraction -# analysisKeys.append('sossheig') # SSH -# analysisKeys.append('FreshwaterFlux') # Fresh water flux -# analysisKeys.append('HeatFlux') -# analysisKeys.append('TotalHeatFlux') -# analysisKeys.append('scvoltot') -# analysisKeys.append('soga') -# analysisKeys.append('thetaoga') -# analysisKeys.append('scalarHeatContent') -# -# if bgc: -# analysisKeys.append('TotalAirSeaFluxCO2') # work in progress -# analysisKeys.append('NoCaspianAirSeaFluxCO2') # work in progress -# -# analysisKeys.append('AirSeaFlux') # work in progress -# analysisKeys.append('IntPP_OSU') # OSU Integrated primpary production -# analysisKeys.append('GlobalExportRatio') -# -# analysisKeys.append('N') # WOA Nitrate -# analysisKeys.append('Si') # WOA Siliate -# analysisKeys.append('O2') # WOA Oxygen -# analysisKeys.append('Iron') -# analysisKeys.append('Alk') -# analysisKeys.append('DIC') -# analysisKeys.append('pH') -# -# analysisKeys.append('CHD') -# analysisKeys.append('CHN') -# analysisKeys.append('CHL') -# analysisKeys.append('DiaFrac') -# analysisKeys.append('DMS') -# analysisKeys.append('Dust') # Dust -# analysisKeys.append('TotalDust') # Total Dust -# -# # analysisKeys.append('DMS_ARAN') -# -# analysisKeys.append('DTC') -# -# analysisKeys.append( -# 'TotalOMZVolume') # Total Oxygen Minimum zone Volume -# analysisKeys.append('OMZThickness') # Oxygen Minimum Zone Thickness -# analysisKeys.append('OMZMeanDepth') # Oxygen Minimum Zone mean depth -# analysisKeys.append('VolumeMeanOxygen') # Volume weighted mean Oxygen -# -# if debug: -# #### -# # Supercedes other flags. -# analysisKeys = [] -# # analysisKeys.append('ERSST') -# #analysisKeys.append('DrakePassageTransport') # DrakePassageTransport -# # analysisKeys.append('VolumeMeanOxygen') # Volume weighted mean Oxygen -# analysisKeys.append('AMOC_26N') -# # analysisKeys.append('NoCaspianAirSeaFluxCO2') # work in progress -# #analysisKeys.append('VolumeMeanTemperature') # Global Mean Temperature -# # analysisKeys.append('CHD') -# # analysisKeys.append('CHN') -# # analysisKeys.append('DiaFrac') -# # analysisKeys.append('AMOC_26N') -# # analysisKeys.append('MLD') -# # analysisKeys.append('Temperature') # WOA Temperature -# # analysisKeys.append('Salinity') # WOA Salinity -# # analysisKeys.append('DMS') -# # analysisKeys.append('N') # WOA Nitrate -# # analysisKeys.append('Si') # WOA Siliate -# -# # analysisKeys.append('TotalAirSeaFluxCO2') # work in progress -# # analysisKeys.append('AirSeaFlux') # work in progress -# # analysisKeys.append('AirSeaFluxCO2') # work in progress -# -# # analysisKeys.append('scvoltot') -# # analysisKeys.append('soga') -# # analysisKeys.append('thetaoga') -# # analysisKeys.append('scalarHeatContent') -# -# #analysisKeys.append('ADRC_26N') # AMOC 26N -# # analysisKeys.append('VerticalCurrent') # Vertical Veloctity -# # analysisKeys.append('sossheig') # SSH -# # analysisKeys.append('NorthernTotalIceArea') # North TotalIceArea -# # analysisKeys.append('SouthernTotalIceArea') # South TotalIceArea -# # analysisKeys.append('TotalIceArea') # TotalIceArea -# # analysisKeys.append('NorthernTotalIceExtent') # work in progress -# # analysisKeys.append('SouthernTotalIceExtent') # work in progress -# # analysisKeys.append('WeddelIceExent') # work in progress -# # analysisKeys.append('TotalIceExtent') # work in progress -# # analysisKeys.append('NorthernMIZArea') -# # analysisKeys.append('SouthernMIZArea') -# # analysisKeys.append('TotalMIZArea') -# -# # analysisKeys.append('NorthernMIZArea') -# # analysisKeys.append('SouthernMIZArea') -# # analysisKeys.append('TotalMIZArea') -# # analysisKeys.append('NorthernMIZfraction') -# # analysisKeys.append('SouthernMIZfraction') -# # analysisKeys.append('TotalMIZfraction') -# -# # analysisKeys.append('FreshwaterFlux') # Fresh water flux -# #analysisKeys.append('GlobalMeanTemperature') -# # analysisKeys.append('GlobalMeanSalinity') -# -# # analysisKeys.append('HeatFlux') -# # analysisKeys.append('TotalHeatFlux') -# -# # analysisKeys.append('quickSST') # Area Weighted Mean Surface Temperature -# # analysisKeys.append('TotalOMZVolume') # Total Oxygen Minimum zone Volume -# # analysisKeys.append('OMZThickness') # Oxygen Minimum Zone Thickness -# # analysisKeys.append('OMZMeanDepth') # Oxygen Minimum Zone mean depth -# # analysisKeys.append('O2') # WOA Oxygen -# # if bgc ==False:return -# # if physics == True:return - layerList = [ 'Surface', ] diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index 882351df..af2f898c 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -67,171 +67,6 @@ from .Paths.paths import paths_setter -##### -# Biogeochemistry keys -# bgcKeys = [] -# if True: -# bgcKeys.append('N') # WOA Nitrate -# bgcKeys.append('Si') # WOA Siliate -# bgcKeys.append('O2') # WOA Oxygen -# bgcKeys.append('Alk') # Glodap Alkalinity -# bgcKeys.append('DIC') # Globap tCO2 -# bgcKeys.append('AirSeaFluxCO2') # Air Sea Flux -# bgcKeys.append('TotalAirSeaFluxCO2') # Total global air sea flux -# bgcKeys.append('IntPP_OSU') # OSU Integrated primpary production -# bgcKeys.append('PP_OSU') # OSU Integrated primpary production -# # bgcKeys.append('LocalExportRatio') # Export ratio (no data) -# # bgcKeys.append('GlobalExportRatio') # Export ratio (no data) -# bgcKeys.append('TotalOMZVolume') # Total Oxygen Minimum zone Volume -# # bgcKeys.append('OMZThickness') # Oxygen Minimum Zone Thickness -# # bgcKeys.append('OMZMeanDepth') # Oxygen Minimum Zone mean depth -# bgcKeys.append( -# 'VolumeMeanOxygen') # Volune Mean regional oxygen concentation -# # bgcKeys.append('AOU') # Apparent Oxygen Usage -# bgcKeys.append('Iron') # Iron -# bgcKeys.append('Dust') # Dust -# # bgcKeys.append('TotalDust') # Total Dust -# # bgcKeys.append('DiaFrac') # Diatom Fraction -# # bgcKeys.append('DTC') # Detrital carbon -# bgcKeys.append('CHL') # Total Chlorophyll -# # bgcKeys.append('DMS_ARAN') # Total Chlorophyll -# bgcKeys.append('pH') -# -# bgcKeysDict = {i: n for i, n in enumerate(bgcKeys)} - -##### -# Physical keys -# physKeys = [] -# if True: -# -# physKeys.append('Temperature') # WOA Temperature -# #physKeys.append('VolWeightedT') # Volume weighted WOA Temperature -# physKeys.append('GlobalMeanTemperature') # Global Mean Temperature -# physKeys.append('GlobalMeanTemperature_700') -# physKeys.append('GlobalMeanTemperature_2000') -# -# physKeys.append('VolumeMeanTemperature') # Global Mean Temperature -# physKeys.append('GlobalMeanSalinity') # Global Mean Salinity -# # physKeys.append('IcelessMeanSST') # Global Mean Surface Temperature with no ice -# physKeys.append('Salinity') # WOA Salinity -# physKeys.append('MLD') # iFERMER Mixed Layer Depth -# -# # physKeys.append('TotalIceArea') # work in progress -# # physKeys.append('NorthernTotalIceArea') # work in progress -# # physKeys.append('SouthernTotalIceArea') # work in progress -# # physKeys.append('WeddelTotalIceArea') -# physKeys.append('TotalIceExtent') # work in progress -# physKeys.append('NorthernTotalIceExtent') # work in progress -# physKeys.append('SouthernTotalIceExtent') # work in progress -# # physKeys.append('WeddelIceExent') # work in progress -# #physKeys.append('NorthernMIZArea') -# #physKeys.append('SouthernMIZArea') -# #physKeys.append('TotalMIZArea') -# # physKeys.append('NorthernMIZfraction') -# # physKeys.append('SouthernMIZfraction') -# # physKeys.append('TotalMIZfraction') -# -# physKeys.append('DrakePassageTransport') # DrakePassageTransport -# # physKeys.append('AMOC_32S') # AMOC 32S -# physKeys.append('AMOC_26N') # AMOC 26N -# # physKeys.append('AMOC_26N_nomexico') # AMOC 26N -# # physKeys.append('ADRC_26N') # ADRC 26N -# # physKeys.append('ZonalCurrent') # Zonal Veloctity -# # physKeys.append('MeridionalCurrent') # Meridional Veloctity -# # physKeys.append('VerticalCurrent') # Vertical Veloctity -# -# #physKeys.append('FreshwaterFlux') # Freshwater flux -# physKeys.append('sowaflup') # Net Upward Water Flux -# # physKeys.append('soicecov') # Ice fraction -# -# ##### -# # unused: -# -# # physKeys.append('MaxMonthlyMLD') # MLD Monthly max -# # physKeys.append('MinMonthlyMLD') # MLD Monthly min -# # physKeys.append('HeatFlux') -# physKeys.append('TotalHeatFlux') -# physKeys.append('scvoltot') -# physKeys.append('soga') -# physKeys.append('thetaoga') - -# physKeys.append('WindStress') # Wind Stress -# physKeys.append('sohefldo') # Net downward Water Flux -# physKeys.append('sofmflup') # Water flux due to freezing/melting -# physKeys.append('sosfldow') # Downward salt flux -# physKeys.append('sossheig') # Sea surface height - -# physKeysDict = {i: n for i, n in enumerate(physKeys)} - -# fastKeys = [] -# if True: -# fastKeys.append('N') # WOA Nitrate -# fastKeys.append('Si') # WOA Siliate -# fastKeys.append('O2') # WOA Oxygen -# fastKeys.append('Alk') # Glodap Alkalinity -# fastKeys.append('DIC') # Globap tCO2 -# fastKeys.append('AirSeaFluxCO2') # Air Sea Flux -# fastKeys.append('TotalAirSeaFluxCO2') # Total global air sea flux -# fastKeys.append('IntPP_OSU') # OSU Integrated primpary production -# fastKeys.append('PP_OSU') # OSU Integrated primpary production -# # fastKeys.append('TotalOMZVolume') # Total Oxygen Minimum zone Volume -# fastKeys.append( -# 'VolumeMeanOxygen') # Volune Mean regional oxygen concentation -# fastKeys.append('Iron') # Iron -# fastKeys.append('Dust') # Dust -# fastKeys.append('CHL') # Total Chlorophyll -# fastKeys.append('pH') -# -# fastKeys.append('Temperature') # WOA Temperature -# fastKeys.append('GlobalMeanTemperature') # Global Mean Temperature -# fastKeys.append('GlobalMeanTemperature_700') -# fastKeys.append('GlobalMeanTemperature_2000') -# -# fastKeys.append('VolumeMeanTemperature') # Global Mean Temperature -# fastKeys.append('GlobalMeanSalinity') # Global Mean Salinity -# fastKeys.append('Salinity') # WOA Salinity -# fastKeys.append('MLD') # iFERMER Mixed Layer Depth -# -# fastKeys.append('TotalIceExtent') # work in progress -# fastKeys.append('NorthernTotalIceExtent') # work in progress -# fastKeys.append('SouthernTotalIceExtent') # work in progress -# -# fastKeys.append('DrakePassageTransport') # DrakePassageTransport -# fastKeys.append('AMOC_26N') # AMOC 26N -# -# fastKeys.append('FreshwaterFlux') # Freshwater flux -# fastKeys.append('sowaflup') # Net Upward Water Flux -# -# fastKeys.append('TotalHeatFlux') -# fastKeys.append('scvoltot') -# fastKeys.append('soga') -# fastKeys.append('thetaoga') - -##### -# Level 1 keys -# level1Keys = [] -# level1Keys.extend(physKeys) -# level1Keys.extend(bgcKeys) -# level1KeysDict = {i: n for i, n in enumerate(level1Keys)} -# -# ##### -# # The important keys -# keymetricsfirstKeys = [ -# 'TotalAirSeaFluxCO2', -# # 'NoCaspianAirSeaFluxCO2', -# # 'IntPP_OSU', -# # 'GlobalExportRatio', -# # 'TotalIceExtent', -# # 'NorthernTotalIceExtent', -# # 'SouthernTotalIceExtent', -# 'DrakePassageTransport', -# 'AMOC_26N', -# 'GlobalMeanTemperature', -# #'GlobalMeanSalinity', -# ] -# keymetricsfirstDict = {i: n for i, n in enumerate(keymetricsfirstKeys)} - - def listModelDataFiles(jobID, filekey, datafolder, annual): print("listing model data files:\njobID:\t", jobID, '\nfile key:\t', filekey, '\ndata folder:\t', datafolder, '\nannual flag:\t', annual) @@ -263,7 +98,7 @@ def build_list_of_suite_keys(suites, debug=True): key_lists_dir = os.path.join(os.path.dirname(paths_dir), 'key_lists') print('key_lists_dir', key_lists_dir) - analysisKeys = {} + analysis_keys = {} for suite in suites: print(suite) # look for a list in keys_list directory: @@ -287,21 +122,21 @@ def build_list_of_suite_keys(suites, debug=True): keys_dict = dictionary.get('keys', {}) for key, keybool in keys_dict.items(): - if debug and key in analysisKeys: + if debug and key in analysis_keys: print('build_list_of_suite_keys:\tKey exists in multiple suites:', key) - if key in analysisKeys and keybool != analysisKeys[key]: - print('build_list_of_suite_keys:\tERROR: conflick in input yamls:', key, keybool, '!=', analysisKeys[key]) + if key in analysis_keys and keybool != analysis_keys[key]: + print('build_list_of_suite_keys:\tERROR: conflick in input yamls:', key, keybool, '!=', analysis_keys[key]) sys.exit(1) if keybool: if debug: print('build_list_of_suite_keys:\tAdding key:', key) - analysisKeys[key] = keybool - analysisKeys = [key for key in analysisKeys.keys()] - return analysisKeys + analysis_keys[key] = keybool + analysis_keys = [key for key in analysis_keys.keys()] + return analysis_keys @@ -368,182 +203,6 @@ def analysis_timeseries( analysisKeys = build_list_of_suite_keys(suites, debug = True) print('analysisKeys', analysisKeys) - # assert 0 - # analysisKeys = analysisSuite - - ##### - # Switches: - # These are some preset switches to run in series. - # if True: - # - # # if analysisSuite.lower() in [ - # # 'keymetricsfirst', 'kmf', - # # ]: - # # analysisKeys.extend(keymetricsfirstKeys) - # - # if analysisSuite.lower() in [ - # 'level1', - # ]: - # analysisKeys.extend(level1Keys) - # - # if analysisSuite.lower() in [ - # 'fast', - # ]: - # analysisKeys.extend(fastKeys) - # - # if analysisSuite.lower() in [ - # 'bgc', - # ]: - # analysisKeys.extend(bgcKeys) - # - # if analysisSuite.lower() in [ - # 'physics', - # ]: - # analysisKeys.extend(physKeys) - # - # if analysisSuite.lower() in [ - # 'level3', - # ]: - # analysisKeys.append('DMS_ARAN') # DMS Aranami Tsunogai - # - # if analysisSuite.lower() in [ - # 'spinup', - # ]: - # analysisKeys.append('O2') # WOA Oxygen - # analysisKeys.append('DIC') # work in progress - # analysisKeys.append('Alk') # Glodap Alkalinity - # analysisKeys.append('Iron') # work in progress - # analysisKeys.append('N') # WOA Nitrate - # analysisKeys.append('Si') # WOA Nitrate - # analysisKeys.append('Temperature') # # WOA Temperature - # analysisKeys.append('Salinity') # # WOA Salinity - # if analysisSuite.lower() in [ - # 'salinity', - # ]: - # analysisKeys.append('Salinity') # # WOA Salinity - # - # if analysisSuite.lower() in [ - # 'debug', - # ]: - # analysisKeys.append('AMOC_26N') # AMOC 26N - - #analysisKeys.append('AirSeaFlux') # work in progress - #analysisKeys.append('TotalAirSeaFluxCO2') # work in progress - #analysisKeys.append('NoCaspianAirSeaFluxCO2') # work in progress - #analysisKeys.append('TotalOMZVolume') # work in progress - #analysisKeys.append('TotalOMZVolume50') # work in progress - #analysisKeys.append('OMZMeanDepth') # work in progress - #analysisKeys.append('OMZThickness') # Oxygen Minimum Zone Thickness - #analysisKeys.append('TotalOMZVolume') # work in progress - #analysisKeys.append('O2') # WOA Oxygen - #analysisKeys.append('AOU') # Apparent Oxygen Usage - #analysisKeys.append('WindStress') # Wind Stress - #analysisKeys.append('Dust') # Dust - #analysisKeys.append('TotalDust') # Total Dust - #analysisKeys.append('TotalDust_nomask') - #analysisKeys.append('DIC') # work in progress - #analysisKeys.append('DrakePassageTransport') # DrakePassageTransport - #analysisKeys.append('TotalIceArea') # work in progress - #analysisKeys.append('CHN') - #analysisKeys.append('CHD') - #analysisKeys.append('CHL') - #analysisKeys.append('pH') - #analysisKeys.append('Alk') # Glodap Alkalinity - - #if jobID in ['u-am004','u-am005']: - # analysisKeys.append('DMS_ANDR') # DMS Anderson - #else: analysisKeys.append('DMS_ARAN') # DMS Aranami Tsunogai - - #analysisKeys.append('DiaFrac') # work in progress - #analysisKeys.append('Iron') # work in progress - #analysisKeys.append('DTC') # work in progress - - #analysisKeys.append('Iron') # work in progress - #analysisKeys.append('N') # WOA Nitrate - #analysisKeys.append('Si') # WOA Nitrate - #analysisKeys.append('IntPP_OSU') # OSU Integrated primpary production - #analysisKeys.append('Chl_CCI') - #analysisKeys.append('CHL_MAM') - #analysisKeys.append('CHL_JJA') - #analysisKeys.append('CHL_SON') - #analysisKeys.append('CHL_DJF') - #analysisKeys.append('GC_CHL_MAM') - #analysisKeys.append('GC_CHL_JJA') - #analysisKeys.append('GC_CHL_SON') - #analysisKeys.append('GC_CHL_DJF') - ##### - # Physics switches: - # analysisKeys.append('Temperature') # # WOA Temperature -# analysisKeys.append('HeatFlux') -# analysisKeys.append('TotalHeatFlux') - -# analysisKeys.append('scvoltot') -# analysisKeys.append('soga') -# analysisKeys.append('thetaoga') -# analysisKeys.append('scalarHeatContent') - -#analysisKeys.append('VolumeMeanTemperature')# -# analysisKeys.append('GlobalMeanTemperature_700') -# analysisKeys.append('GlobalMeanTemperature_2000') -# analysisKeys.append('WeddelIceExent') -#analysisKeys.append('Salinity') # WOA Salinity -# analysisKeys.append('MLD') # MLD -#analysisKeys.append('MaxMonthlyMLD') # MLD -#analysisKeys.append('MinMonthlyMLD') - -#analysisKeys.append('NorthernTotalIceArea') # work in progress -#analysisKeys.append('SouthernTotalIceArea') # work in progress -#analysisKeys.append('WeddelTotalIceArea') -# analysisKeys.append('NorthernMIZArea') -# analysisKeys.append('SouthernMIZArea') -# analysisKeys.append('TotalMIZArea') -#analysisKeys.append('NorthernMIZfraction') -#analysisKeys.append('SouthernMIZfraction') -#analysisKeys.append('TotalMIZfraction') - -#analysisKeys.append('TotalIceArea') # work in progress -#analysisKeys.append('TotalIceExtent') # work in progress -#analysisKeys.append('NorthernTotalIceExtent') # work in progress -#analysisKeys.append('SouthernTotalIceExtent') # work in progress -#analysisKeys.append('AMOC_32S') # AMOC 32S -#analysisKeys.append('AMOC_26N') # AMOC 26N -#analysisKeys.append('AMOC_26N_nomexico') -#analysisKeys.append('ADRC_26N') # AMOC 26N - -# analysisKeys.append('ERSST') # Global Surface Mean Temperature -#analysisKeys.append('VolumeMeanOxygen') - -# analysisKeys.append('GlobalMeanTemperature') # Global Mean Temperature# -# analysisKeys.append('GlobalMeanSalinity') # Global Mean Salinity -#analysisKeys.append('IcelessMeanSST') # Global Mean Surface Temperature with no ice -#analysisKeys.append('quickSST') # Area Weighted Mean Surface Temperature - -#analysisKeys.append('ZonalCurrent') # Zonal Veloctity -#analysisKeys.append('MeridionalCurrent') # Meridional Veloctity -#analysisKeys.append('VerticalCurrent') # Vertical Veloctity - -#analysisKeys.append('sowaflup') # Net Upward Water Flux -#analysisKeys.append('sohefldo') # Net downward Water Flux -# analysisKeys.append('sofmflup') # Water flux due to freezing/melting -# analysisKeys.append('sosfldow') # Downward salt flux -# analysisKeys.append('soicecov') # Ice fraction -# analysisKeys.append('sossheig') # Sea surface height -#analysisKeys.append('FreshwaterFlux') # Fresh water flux -#analysisKeys.append('max_soshfldo') # Max short wave radiation. - -##### -# Physics switches: -# if jobID in [ -# 'u-aj588', -# 'u-ak900', -# 'u-ar538', -# 'u-an869', -# 'u-ar977', -# ]: -# try: -# analysisKeys.remove('FreshwaterFlux') -# except: -# pass ##### # Some lists of region. @@ -575,11 +234,16 @@ def analysis_timeseries( 'NorthernHemisphere', ] - if regions in ['debug', 'spinup']: + if analysisSuite.lower() == 'debug': + regionList = ['Global', 'ArcticOcean'] + + if analysisSuite.lower() in [ + 'spinup', + 'salinity', + ]: regionList = [ 'Global', - ] - + ] metricList = [ 'mean', ] @@ -629,21 +293,83 @@ def analysis_timeseries( # So far, this has been run on the following machines: # PML # JASMIN + # Charybdis (Julien's machine at NOCS) # + # Feel free to add other macihines onto this list, if need be. machinelocation = '' - shelvedir = ukp.folder([paths.shelvedir, "timeseries", jobID]) - imagedir = ukp.folder([paths.imagedir, jobID, 'timeseries']) - if annual: WOAFolder = paths.WOAFolder_annual - else: WOAFolder = paths.WOAFolder + shelvedir = ukp.folder(paths.shelvedir + "/timeseries/" + jobID) ##### + # PML hostname = gethostname() - print("analysis-timeseries.py:\tBeing run at ", hostname) + if hostname.find('pmpc') > -1: + print("analysis-timeseries.py:\tBeing run at PML on ", gethostname()) + + imagedir = ukp.folder(paths.imagedir + '/' + jobID + '/timeseries') + + if annual: WOAFolder = paths.WOAFolder_annual + else: WOAFolder = paths.WOAFolder + + #shelvedir = ukp.folder(paths.shelvedir+'/'+jobID+'/timeseries/'+jobID) + #shelvedir = ukp.folder(paths.shelvedir + "/timeseries/" + jobID) ##### - # Unable to find location of files/data. + # JASMIN + if hostname.find('ceda.ac.uk') > -1 or hostname.find( + 'jasmin') > -1 or hostname.find('jc.rl.ac.uk') > -1: + print("analysis-timeseries.py:\tBeing run at CEDA on ", hostname) + #machinelocation = 'JASMIN' + + #try: shelvedir = ukp.folder("/group_workspaces/jasmin2/ukesm/BGC_data/"+getuser()+"/shelves/timeseries/"+jobID) + #except: shelvedir = "/group_workspaces/jasmin2/ukesm/BGC_data/"+getuser()+"/shelves/timeseries/"+jobID + #try: + # shelvedir = ukp.folder("/gws/nopw/j04/ukesm/BGC_data/" + + # getuser() + "/shelves/timeseries/" + jobID) + #except: + # shelvedir = "/gws/nopw/j04/ukesm/BGC_data/" + getuser( + # ) + "/shelves/timeseries/" + jobID + + if annual: WOAFolder = paths.WOAFolder_annual + else: WOAFolder = paths.WOAFolder + + try: + imagedir = ukp.folder(paths.imagedir + '/' + jobID + '/timeseries') + except: + imagedir = paths.imagedir + '/' + jobID + '/timeseries' + + if hostname.find('monsoon') > -1: + print("Please set up paths.py") + assert 0 + +#print "analysis-timeseries.py:\tBeing run at the Met Office on ",gethostname() +#machinelocation = 'MONSOON' + +#ObsFolder = "/projects/ukesm/ldmora/BGC-data/" +#ModelFolder = "/projects/ukesm/ldmora/UKESM" +##### +# Location of model files. +#MEDUSAFolder_pref = ukp.folder(ModelFolder) + +##### +# Location of data files. +#if annual: WOAFolder = ukp.folder(ObsFolder+"WOA/annual") +#else: WOAFolder = ukp.folder(ObsFolder+"WOA/") + +#MAREDATFolder = ObsFolder+"/MAREDAT/MAREDAT/" +#GEOTRACESFolder = ObsFolder+"/GEOTRACES/GEOTRACES_PostProccessed/" +#TakahashiFolder = ObsFolder+"/Takahashi2009_pCO2/" +#MLDFolder = ObsFolder+"/IFREMER-MLD/" +#iMarNetFolder = ObsFolder+"/LestersReportData/" +#GlodapDir = ObsFolder+"/GLODAP/" +#GLODAPv2Dir = ObsFolder+"/GLODAPv2/GLODAPv2_Mapped_Climatologies/" +#OSUDir = ObsFolder+"OSU/" +#CCIDir = ObsFolder+"CCI/" +#orcaGridfn = ModelFolder+'/mesh_mask_eORCA1_wrk.nc' + +##### +# Unable to find location of files/data. if not paths.machinelocation: print( "analysis-timeseries.py:\tFATAL:\tWas unable to determine location of host: ", @@ -655,9 +381,10 @@ def analysis_timeseries( else: assert False - ##### - # Because we can never be sure someone won't randomly rename the - # time dimension without saying anything. +##### +# Because we can never be sure someone won't randomly rename the +# time dimension without saying anything. +# if jobID in ['u-am515','u-am927','u-am064','u-an326',]: try: tmpModelFiles = listModelDataFiles(jobID, 'grid_T', paths.ModelFolder_pref, annual) @@ -705,22 +432,66 @@ def analysis_timeseries( ukesmkeys['w3d'] = 'wo' ukesmkeys['MLD'] = 'ssomxl010' - ##### - # Coordinate dictionairy - # These are python dictionairies, one for each data source and model. - # This is because each data provider seems to use a different set of standard names for dimensions and time. - # The 'tdict' field is short for "time-dictionary". - # This is a dictionary who's indices are the values on the netcdf time dimension. - # The tdict indices point to a month number in python numbering (ie January = 0) - # An example would be, if a netcdf uses the middle day of the month as it's time value: - # tdict = {15:0, 45:1 ...} - - try: timekey = ukesmkeys['time'] - except KeyError: - print('ERROR: bgcval2 was unable to determine what kind of model data this was.') - print(' Is any data present at:',tmpModelFiles, paths.ModelFolder_pref, jobID) - print(' Otherwise, what name is time given in that data?') - sys.exit(1) +# else: +# ukesmkeys['time'] = 'time_centered' +# ukesmkeys['temp3d'] = 'thetao' +# ukesmkeys['sst'] = 'tos' +# ukesmkeys['sal3d'] = 'so' +# ukesmkeys['sss'] = 'sos' +# ukesmkeys['v3d'] = 'vo' +# ukesmkeys['u3d'] = 'uo' +# ukesmkeys['e3u'] = 'thkcello' +# ukesmkeys['w3d'] = 'wo' + +# ukesmkeys['time'] = 'time_counter' +# ukesmkeys['temp3d'] = 'votemper' +# ukesmkeys['sst'] = '' +# ukesmkeys['sal3d'] = 'vosaline' +# ukesmkeys['sss'] = '' +# ukesmkeys['v3d'] = 'vomecrty' +# ukesmkeys['u3d'] = 'vozocrtx' +# ukesmkeys['e3u'] = 'e3u' +# ukesmkeys['w3d'] = 'vovecrtz' + +# if jobID > 'u-am514' and jobID not in ['u-an619','u-an629','u-an631','u-an869', 'u-an908', 'u-an911','u-an989',]: +# # There are other changes here too. +# ##### +# # Because we can never be sure someone won't randomly rename the +# # time dimension without saying anything. +# ukesmkeys={} +# ukesmkeys['time'] = 'time_centered' +# ukesmkeys['temp3d'] = 'thetao' +# ukesmkeys['sst'] = 'tos' +# ukesmkeys['sal3d'] = 'so' +# ukesmkeys['sss'] = 'sos' +# ukesmkeys['v3d'] = 'vo' +# ukesmkeys['u3d'] = 'uo' +# ukesmkeys['e3u'] = 'thkcello' +# ukesmkeys['w3d'] = 'wo' +# +# else: +# ukesmkeys={} +# ukesmkeys['time'] = 'time_counter' +# ukesmkeys['temp3d'] = 'votemper' +# ukesmkeys['sst'] = '' +# ukesmkeys['sal3d'] = 'vosaline' +# ukesmkeys['sss'] = '' +# ukesmkeys['v3d'] = 'vomecrty' +# ukesmkeys['u3d'] = 'vozocrtx' +# ukesmkeys['e3u'] = 'e3u' +# ukesmkeys['w3d'] = 'vovecrtz' + +##### +# Coordinate dictionairy +# These are python dictionairies, one for each data source and model. +# This is because each data provider seems to use a different set of standard names for dimensions and time. +# The 'tdict' field is short for "time-dictionary". +# This is a dictionary who's indices are the values on the netcdf time dimension. +# The tdict indices point to a month number in python numbering (ie January = 0) +# An example would be, if a netcdf uses the middle day of the month as it's time value: +# tdict = {15:0, 45:1 ...} + + timekey = ukesmkeys['time'] medusaCoords = { 't': timekey, 'z': 'deptht', @@ -4937,7 +4708,7 @@ def main(): "Will proceed with defaults.") config_user = None - for jobID in jobIDs: + for jobID in itertools.product(jobIDs): analysis_timeseries( jobID=jobID, suites=keys, From 32de4ea4bfc6840fbb1bf691c92056d3fac9040d Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 15 Aug 2022 16:05:15 +0100 Subject: [PATCH 09/27] deleted unused function. --- bgcval2/analysis_compare.py | 108 ------------------------------------ 1 file changed, 108 deletions(-) diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index 60088904..2f56e166 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -3882,114 +3882,6 @@ def flatten(lats, lons, dataA, dataB): np.ma.masked_where(m, dataB).compressed() -def CompareTwoRuns(jobIDA, - jobIDB, - physics=True, - bgc=True, - yearA='', - yearB='', - debug=True): - # - #spatial maps of specific fields - - filetype = [] - if physics: - filetype.extend([ - 'grid_T', - 'grid_U', - 'grid_V', - 'grid_W', - ]) - if bgc: - filetype.extend([ - 'diad_T', - 'ptrc_T', - ]) - filesA = {} - filesB = {} - - imageFolder = 'images/TimeseriesCompare/' - imageFolder += jobIDA + '-' + yearA + '_and_' + jobIDB + '-' + yearB - - for ft in filetype: - filesA[ft] = listModelDataFiles(jobIDA, - ft, - paths.ModelFolder_pref, - True, - year=yearA)[0] - filesB[ft] = listModelDataFiles(jobIDB, - ft, - paths.ModelFolder_pref, - True, - year=yearB)[0] - print("files A:", filesA[ft]) - print("files B:", filesB[ft]) - - ncA = Dataset(filesA[ft], 'r') - ncB = Dataset(filesB[ft], 'r') - keys = ukp.intersection(list(ncA.variables.keys()), - list(ncB.variables.keys())) - - lats = ncA.variables['nav_lat'][:] - lons = ncA.variables['nav_lon'][:] - - for key in keys: - filename = ukp.folder(imageFolder + '/' + - ft) + ft + '-' + key + '.png' - if os.path.exists(filename): continue - - dataA = 0. - dataB = 0. - if key in alwaysInclude: continue - if key in ['bounds_lon', 'bounds_lat']: continue - if ncA.variables[key].ndim == 4: - dataA = ncA.variables[key][0, 0, :, :] - dataB = ncB.variables[key][0, 0, :, :] - - elif ncA.variables[key].ndim == 3: - dataA = ncA.variables[key][0, :, :] - dataB = ncB.variables[key][0, :, :] - elif ncA.variables[key].ndim == 2: - continue - dataA = ncA.variables[key][:, :] - dataB = ncB.variables[key][:, :] - else: - print("can't plot:", key, ncA.variables[key].ndim) - continue - try: - title = ncA.variables[key].long_name - except: - title = getLongName(key) - - dmin = min([dataA.min(), dataB.min()]) - dmax = max([dataA.max(), dataB.max()]) - - print(key, lats.shape, lons.shape, dataA.shape, dataB.shape) - la, lo, data, datb = flatten(lats, lons, dataA, dataB) - print(key, la.shape, lo.shape, data.shape, datb.shape) - - if 0 in [len(la), len(lo), len(data), len(datb)]: continue - # filename = ukp.folder(imageFolder+'/'+ft)+ft+'-'+key+'.png' - # if os.path.exists(filename):continue - ukp.robinPlotQuad( - lo, - la, - data, - datb, - filename, - titles=[jobIDA + ' ' + yearA, jobIDB + ' ' + yearB], - title=title, - lon0=0., - marble=False, - drawCbar=True, - cbarlabel='', - doLog=False, - vmin=dmin, - vmax=dmax, - maptype='Cartopy', - scatter=False) - - def load_comparison_yml(master_compare_yml_fn): """ Load the config yaml. From 84c1bd1d50f9d6f37d6af9b57f1eb3155cdaf38d Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 15 Aug 2022 16:31:48 +0100 Subject: [PATCH 10/27] Update key_lists/bgc.yml Co-authored-by: Valeriu Predoi --- key_lists/bgc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/key_lists/bgc.yml b/key_lists/bgc.yml index f01aaded..89ec40b7 100644 --- a/key_lists/bgc.yml +++ b/key_lists/bgc.yml @@ -8,7 +8,7 @@ keys: CHL: True # Total Chlorophyll # CHD: True # Diatom Chlorophyll # CHN: True # Non-diatom Chlorophyll - # DiaFrac: True # Diatom Fraction + # DiaFrac: True # Diatom Fraction IntPP_OSU: True # OSU Integrated primpary production PP_OSU: True # OSU Integrated primpary production From 6275338a8d13e58777e0fd4a7e687dbcd83a5895 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 15 Aug 2022 16:31:54 +0100 Subject: [PATCH 11/27] Update bgcval2/analysis_timeseries.py Co-authored-by: Valeriu Predoi --- bgcval2/analysis_timeseries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index af2f898c..d2ba3b0d 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -201,7 +201,7 @@ def analysis_timeseries( if isinstance(suites, str): suites = [suites, ] - analysisKeys = build_list_of_suite_keys(suites, debug = True) + analysisKeys = build_list_of_suite_keys(suites, debug=True) print('analysisKeys', analysisKeys) ##### From 7b8a2e6778d404cb676391aede5ca553e17a6728 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 15 Aug 2022 16:32:00 +0100 Subject: [PATCH 12/27] Update key_lists/bgc.yml Co-authored-by: Valeriu Predoi --- key_lists/bgc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/key_lists/bgc.yml b/key_lists/bgc.yml index 89ec40b7..89016cd6 100644 --- a/key_lists/bgc.yml +++ b/key_lists/bgc.yml @@ -15,7 +15,7 @@ keys: # DMS_ARAN: True # DMS: True # DTC: True # Detrital carbon - # LocalExportRatio: True # Export ratio (no data) + # LocalExportRatio: True # Export ratio (no data) # GlobalExportRatio: True # Export ratio (no data) # Carbonate system: From 88570d59435fbc318b4abebd15679fc09e22ed2a Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 15 Aug 2022 16:32:06 +0100 Subject: [PATCH 13/27] Update key_lists/bgc.yml Co-authored-by: Valeriu Predoi --- key_lists/bgc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/key_lists/bgc.yml b/key_lists/bgc.yml index 89016cd6..fd769c9b 100644 --- a/key_lists/bgc.yml +++ b/key_lists/bgc.yml @@ -16,7 +16,7 @@ keys: # DMS: True # DTC: True # Detrital carbon # LocalExportRatio: True # Export ratio (no data) - # GlobalExportRatio: True # Export ratio (no data) + # GlobalExportRatio: True # Export ratio (no data) # Carbonate system: Alk: True # Glodap Alkalinity From 240c440e128867c78a9010234e8f18ca2fa27b12 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 15 Aug 2022 16:34:47 +0100 Subject: [PATCH 14/27] working on av --- bgcval2/analysis_timeseries.py | 88 +++++++--------------------------- 1 file changed, 17 insertions(+), 71 deletions(-) diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index af2f898c..83d1f6e1 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -234,20 +234,12 @@ def analysis_timeseries( 'NorthernHemisphere', ] - if analysisSuite.lower() == 'debug': - regionList = ['Global', 'ArcticOcean'] + if regions in ['debug', 'Global', 'spinup']: + regionList = ['Global', ] - if analysisSuite.lower() in [ - 'spinup', - 'salinity', - ]: - regionList = [ - 'Global', - ] metricList = [ 'mean', ] - layerList = ['500m', '1000m', '2000m', '4000m'] # Regions from Pierce 1995 - https://doi.org/10.1175/1520-0485(1995)025<2046:CROHAF>2.0.CO;2 PierceRegions = [ @@ -299,7 +291,11 @@ def analysis_timeseries( machinelocation = '' - shelvedir = ukp.folder(paths.shelvedir + "/timeseries/" + jobID) + shelvedir = ukp.folder([paths.shelvedir, "timeseries", jobID]) + imagedir = ukp.folder([paths.imagedir, jobID, 'timeseries']) + + if annual: WOAFolder = paths.WOAFolder_annual + else: WOAFolder = paths.WOAFolder ##### # PML @@ -308,13 +304,6 @@ def analysis_timeseries( if hostname.find('pmpc') > -1: print("analysis-timeseries.py:\tBeing run at PML on ", gethostname()) - imagedir = ukp.folder(paths.imagedir + '/' + jobID + '/timeseries') - - if annual: WOAFolder = paths.WOAFolder_annual - else: WOAFolder = paths.WOAFolder - - #shelvedir = ukp.folder(paths.shelvedir+'/'+jobID+'/timeseries/'+jobID) - #shelvedir = ukp.folder(paths.shelvedir + "/timeseries/" + jobID) ##### # JASMIN if hostname.find('ceda.ac.uk') > -1 or hostname.find( @@ -322,54 +311,8 @@ def analysis_timeseries( print("analysis-timeseries.py:\tBeing run at CEDA on ", hostname) #machinelocation = 'JASMIN' - #try: shelvedir = ukp.folder("/group_workspaces/jasmin2/ukesm/BGC_data/"+getuser()+"/shelves/timeseries/"+jobID) - #except: shelvedir = "/group_workspaces/jasmin2/ukesm/BGC_data/"+getuser()+"/shelves/timeseries/"+jobID - #try: - # shelvedir = ukp.folder("/gws/nopw/j04/ukesm/BGC_data/" + - # getuser() + "/shelves/timeseries/" + jobID) - #except: - # shelvedir = "/gws/nopw/j04/ukesm/BGC_data/" + getuser( - # ) + "/shelves/timeseries/" + jobID - - if annual: WOAFolder = paths.WOAFolder_annual - else: WOAFolder = paths.WOAFolder - - try: - imagedir = ukp.folder(paths.imagedir + '/' + jobID + '/timeseries') - except: - imagedir = paths.imagedir + '/' + jobID + '/timeseries' - - if hostname.find('monsoon') > -1: - print("Please set up paths.py") - assert 0 - -#print "analysis-timeseries.py:\tBeing run at the Met Office on ",gethostname() -#machinelocation = 'MONSOON' - -#ObsFolder = "/projects/ukesm/ldmora/BGC-data/" -#ModelFolder = "/projects/ukesm/ldmora/UKESM" -##### -# Location of model files. -#MEDUSAFolder_pref = ukp.folder(ModelFolder) - -##### -# Location of data files. -#if annual: WOAFolder = ukp.folder(ObsFolder+"WOA/annual") -#else: WOAFolder = ukp.folder(ObsFolder+"WOA/") - -#MAREDATFolder = ObsFolder+"/MAREDAT/MAREDAT/" -#GEOTRACESFolder = ObsFolder+"/GEOTRACES/GEOTRACES_PostProccessed/" -#TakahashiFolder = ObsFolder+"/Takahashi2009_pCO2/" -#MLDFolder = ObsFolder+"/IFREMER-MLD/" -#iMarNetFolder = ObsFolder+"/LestersReportData/" -#GlodapDir = ObsFolder+"/GLODAP/" -#GLODAPv2Dir = ObsFolder+"/GLODAPv2/GLODAPv2_Mapped_Climatologies/" -#OSUDir = ObsFolder+"OSU/" -#CCIDir = ObsFolder+"CCI/" -#orcaGridfn = ModelFolder+'/mesh_mask_eORCA1_wrk.nc' - -##### -# Unable to find location of files/data. + ##### + # Unable to find location of files/data. if not paths.machinelocation: print( "analysis-timeseries.py:\tFATAL:\tWas unable to determine location of host: ", @@ -381,10 +324,9 @@ def analysis_timeseries( else: assert False -##### -# Because we can never be sure someone won't randomly rename the -# time dimension without saying anything. -# if jobID in ['u-am515','u-am927','u-am064','u-an326',]: + ##### + # Because we can never be sure someone won't randomly rename the + # time dimension without saying anything. try: tmpModelFiles = listModelDataFiles(jobID, 'grid_T', paths.ModelFolder_pref, annual) @@ -4518,6 +4460,10 @@ def modeldustsum(nc, keys): # At the moment, this dictioary is not used, but we could for instance open the shelve to highlight specific data, # (ie, andy asked to produce a table showing the final year of data. + for name in list(av.keys()): + print(name, av[name]) + print('ynml dump:\n',yaml.dump(av[name])) + assert 0 shelves = {} shelves_insitu = {} for name in list(av.keys()): @@ -4708,7 +4654,7 @@ def main(): "Will proceed with defaults.") config_user = None - for jobID in itertools.product(jobIDs): + for jobID in jobIDs: analysis_timeseries( jobID=jobID, suites=keys, From ab32b26b2fce23da46ec9caec274881a07fa4652 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 15 Aug 2022 16:35:45 +0100 Subject: [PATCH 15/27] fixed some bugs; --- bgcval2/analysis_timeseries.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index 83d1f6e1..995a0358 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -4451,19 +4451,15 @@ def modeldustsum(nc, keys): av[name]['Dimensions'] = 1 -##### -# Calling timeseriesAnalysis -# This is where the above settings is passed to timeseriesAnalysis, for the actual work to begin. -# We loop over all fiels in the first layer dictionary in the autovificiation, av. -# -# Once the timeseriesAnalysis has completed, we save all the output shelves in a dictionairy. -# At the moment, this dictioary is not used, but we could for instance open the shelve to highlight specific data, -# (ie, andy asked to produce a table showing the final year of data. + ##### + # Calling timeseriesAnalysis + # This is where the above settings is passed to timeseriesAnalysis, for the actual work to begin. + # We loop over all fiels in the first layer dictionary in the autovificiation, av. + # + # Once the timeseriesAnalysis has completed, we save all the output shelves in a dictionairy. + # At the moment, this dictioary is not used, but we could for instance open the shelve to highlight specific data, + # (ie, andy asked to produce a table showing the final year of data. - for name in list(av.keys()): - print(name, av[name]) - print('ynml dump:\n',yaml.dump(av[name])) - assert 0 shelves = {} shelves_insitu = {} for name in list(av.keys()): From 2961c32392d95eb503b0ddf5de2a6a9ce30f95ed Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 16 Aug 2022 05:37:17 +0100 Subject: [PATCH 16/27] Update bgcval2/analysis_compare.py Co-authored-by: Valeriu Predoi --- bgcval2/analysis_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index 2f56e166..7628d037 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -187,7 +187,7 @@ def timeseries_compare(jobs, if isinstance(suites, str): suites = [suites, ] - analysisKeys = build_list_of_suite_keys(suites, debug = True) + analysisKeys = build_list_of_suite_keys(suites, debug=True) print('analysisKeys', analysisKeys) layerList = [ From 00d803b35649d0e8895f99fa1f1ac32584f91677 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 16 Aug 2022 05:37:24 +0100 Subject: [PATCH 17/27] Update bgcval2/analysis_compare.py Co-authored-by: Valeriu Predoi --- bgcval2/analysis_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index 7628d037..6b97d3f4 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -188,7 +188,7 @@ def timeseries_compare(jobs, suites = [suites, ] analysisKeys = build_list_of_suite_keys(suites, debug=True) - print('analysisKeys', analysisKeys) + print(f'Using analysis keys {str(analysisKeys)}') layerList = [ 'Surface', From 367189169791d6796413cf1b4b42477250d63bc0 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 16 Aug 2022 05:37:40 +0100 Subject: [PATCH 18/27] Update bgcval2/analysis_compare.py Co-authored-by: Valeriu Predoi --- bgcval2/analysis_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index 6b97d3f4..12104e7e 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -4001,7 +4001,7 @@ def load_yml_and_run(compare_yml, config_user): # Master suite leys: if not master_suites: - master_suites=['physics', 'bgc'] # Defaults + master_suites=['physics', 'bgc'] # Defaults # make sure its a list: if isinstance(master_suites, list) : From b4d2e273ae123c6e06cbccb993ae9f70f9692fb1 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 16 Aug 2022 05:37:55 +0100 Subject: [PATCH 19/27] Update bgcval2/analysis_timeseries.py Co-authored-by: Valeriu Predoi --- bgcval2/analysis_timeseries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index 3a38eae6..22699d02 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -93,7 +93,7 @@ def build_list_of_suite_keys(suites, debug=True): Generate a list of keys from a list of suites. """ - print('build_list_of_suite_keys') + print('analysis_timeseries: Calling build_list_of_suite_keys to build list of keys') paths_dir = os.path.dirname(os.path.realpath(__file__)) key_lists_dir = os.path.join(os.path.dirname(paths_dir), 'key_lists') From fc9c6a0f0782772d669ba595b6d72b7215ab5fed Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 16 Aug 2022 05:50:01 +0100 Subject: [PATCH 20/27] Made local changes to yml lists to improve readibilty. --- key_lists/bgc.yml | 45 +++++++++++++++++++++--------------------- key_lists/debug.yml | 20 +++++++++---------- key_lists/fast.yml | 10 ++++++---- key_lists/kmf.yml | 22 ++++++++++++--------- key_lists/level1.yml | 3 +++ key_lists/physics.yml | 3 +++ key_lists/salinity.yml | 3 +++ key_lists/spinup.yml | 4 ++++ 8 files changed, 65 insertions(+), 45 deletions(-) diff --git a/key_lists/bgc.yml b/key_lists/bgc.yml index fd769c9b..8ecee1a9 100644 --- a/key_lists/bgc.yml +++ b/key_lists/bgc.yml @@ -1,38 +1,39 @@ +--- +# Biogeochemistry keys for UKESM analysis. keys: # Nutrients - N: True # WOA Nitrate - Si: True # WOA Siliate - Iron: True # Iron + N: True # WOA Nitrate + Si: True # WOA Siliate + Iron: True # Iron - # Production, Export, Chlorophyll and DMS - CHL: True # Total Chlorophyll - # CHD: True # Diatom Chlorophyll - # CHN: True # Non-diatom Chlorophyll - # DiaFrac: True # Diatom Fraction - - IntPP_OSU: True # OSU Integrated primpary production - PP_OSU: True # OSU Integrated primpary production + # Chlorophyll, Production, Export and DMS + CHL: True # Total Chlorophyll + # CHD: True # Diatom Chlorophyll + # CHN: True # Non-diatom Chlorophyll + # DiaFrac: True # Diatom Fraction + IntPP_OSU: True # OSU Integrated primpary production + PP_OSU: True # OSU Integrated primpary production + # LocalExportRatio: True # Export ratio + # GlobalExportRatio: True # Export ratio # DMS_ARAN: True - # DMS: True + # DMS: True # DTC: True # Detrital carbon - # LocalExportRatio: True # Export ratio (no data) - # GlobalExportRatio: True # Export ratio (no data) # Carbonate system: - Alk: True # Glodap Alkalinity - DIC: True # Globap tCO2 - AirSeaFluxCO2: True # Air Sea Flux - TotalAirSeaFluxCO2: True # Total global air sea flux + Alk: True # Glodap Alkalinity + DIC: True # Globap dissolved inorganic carbon + AirSeaFluxCO2: True # Air Sea Flux of CO2 + TotalAirSeaFluxCO2: True # Total global air sea flux pH: True # Oxygen: - O2: True # WOA Oxygen - TotalOMZVolume: True # Total Oxygen Minimum zone Volume + O2: True # WOA Oxygen + TotalOMZVolume: True # Total Oxygen Minimum zone Volume # OMZThickness: True # Oxygen Minimum Zone Thickness # OMZMeanDepth: True # Oxygen Minimum Zone mean depth - VolumeMeanOxygen: True # Volune Mean regional oxygen concentation + VolumeMeanOxygen: True # Volune Mean regional oxygen concentation # AOU: True # Apparent Oxygen Usage - #Dust + # Misc. Dust: True # Dust # TotalDust: True # Total Dust diff --git a/key_lists/debug.yml b/key_lists/debug.yml index 646921e3..041aac8b 100644 --- a/key_lists/debug.yml +++ b/key_lists/debug.yml @@ -1,14 +1,14 @@ --- # Quick shortlist of keys to help debug analysis code. keys: - #TotalAirSeaFluxCO2: True - #NoCaspianAirSeaFluxCO2: True - #IntPP_OSU: True - #GlobalExportRatio: True - #TotalIceExtent: True - #NorthernTotalIceExtent: True - #SouthernTotalIceExtent: True - #DrakePassageTransport: True + # TotalAirSeaFluxCO2: True + # NoCaspianAirSeaFluxCO2: True + # IntPP_OSU: True + # GlobalExportRatio: True + # TotalIceExtent: True + # NorthernTotalIceExtent: True + # SouthernTotalIceExtent: True + # DrakePassageTransport: True AMOC_26N: True - #GlobalMeanTemperature: True - #GlobalMeanSalinity: True + # GlobalMeanTemperature: True + # GlobalMeanSalinity: True diff --git a/key_lists/fast.yml b/key_lists/fast.yml index 6802e2b6..de41cfa4 100644 --- a/key_lists/fast.yml +++ b/key_lists/fast.yml @@ -1,7 +1,9 @@ -# Note that fast refers to the UKESM parameterisation, not that this analysis -# will run faster. +--- +# UKESM-fast analysis. +# Note that `fast` refers to the lighter weight UKESM parameterisation, +# not that this analysis will run faster. keys: - N: True # WOA Nitrate + N: True # WOA Nitrate Si: True # WOA Siliate O2: True # WOA Oxygen Alk: True # Glodap Alkalinity @@ -10,7 +12,7 @@ keys: TotalAirSeaFluxCO2: True # Total global air sea flux IntPP_OSU: True # OSU Integrated primpary production PP_OSU: True # OSU Integrated primpary production - # TotalOMZVolume: True # Total Oxygen Minimum zone Volume + # TotalOMZVolume: True # Total Oxygen Minimum zone Volume VolumeMeanOxygen: True # Volune Mean regional oxygen concentation Iron: True # Iron Dust: True # Dust diff --git a/key_lists/kmf.yml b/key_lists/kmf.yml index 2a6b805d..14f5895b 100644 --- a/key_lists/kmf.yml +++ b/key_lists/kmf.yml @@ -1,12 +1,16 @@ +--- +# Key metrics first - KMF +# A short list of key metrics to run first to get a very quick idea +# of how a model is working. keys: - TotalAirSeaFluxCO2: True - #NoCaspianAirSeaFluxCO2: True - #IntPP_OSU: True - #GlobalExportRatio: True - #TotalIceExtent: True - #NorthernTotalIceExtent: True - #SouthernTotalIceExtent: True + GlobalMeanTemperature: True DrakePassageTransport: True AMOC_26N: True - GlobalMeanTemperature: True - #GlobalMeanSalinity: True + TotalAirSeaFluxCO2: True + # NoCaspianAirSeaFluxCO2: True + # IntPP_OSU: True + # GlobalExportRatio: True + TotalIceExtent: True + # NorthernTotalIceExtent: True + # SouthernTotalIceExtent: True + # GlobalMeanSalinity: True diff --git a/key_lists/level1.yml b/key_lists/level1.yml index 2dc905c2..e34edd2c 100644 --- a/key_lists/level1.yml +++ b/key_lists/level1.yml @@ -1,3 +1,6 @@ +--- +# Level 1 analysis +# A full list of both BGC and Physical keys keys: # BGC keys: N: True # WOA Nitrate diff --git a/key_lists/physics.yml b/key_lists/physics.yml index 15e51662..6ac04ddb 100644 --- a/key_lists/physics.yml +++ b/key_lists/physics.yml @@ -1,3 +1,6 @@ +--- +# Physics: +# A list of physical keys. keys: # Temperature keys: Temperature: True # WOA Temperature diff --git a/key_lists/salinity.yml b/key_lists/salinity.yml index 85e86f46..f77c73d6 100644 --- a/key_lists/salinity.yml +++ b/key_lists/salinity.yml @@ -1,3 +1,6 @@ +--- +# Salinity +# Salinity only related metrics keys: # Salinity keys: GlobalMeanSalinity: True diff --git a/key_lists/spinup.yml b/key_lists/spinup.yml index 2ac8900f..4463d022 100644 --- a/key_lists/spinup.yml +++ b/key_lists/spinup.yml @@ -1,3 +1,7 @@ +--- +# spinup +# A shortened list of keys to watch the spin up progress +# Usually run in addition to KMF.yml. keys: O2: True # WOA Oxygen DIC: True # work in progress From 3ba12e4e732245445c5ec9f09a773aa64b98ccd9 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 16 Aug 2022 05:55:55 +0100 Subject: [PATCH 21/27] Changed requirement that suites is a list. --- bgcval2/analysis_compare.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index 12104e7e..ee434be7 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -184,8 +184,9 @@ def timeseries_compare(jobs, annual = True strictFileCheck = False - if isinstance(suites, str): - suites = [suites, ] + if not isinstance(suites, list): + ValueError(f"Suites need to be a list, got: {suites}") + sys.exit(1) analysisKeys = build_list_of_suite_keys(suites, debug=True) print(f'Using analysis keys {str(analysisKeys)}') From 6e12f88a6c46cd17d7ce43b54c4315a3a709a1cf Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 16 Aug 2022 05:56:16 +0100 Subject: [PATCH 22/27] Update bgcval2/analysis_timeseries.py Co-authored-by: Valeriu Predoi --- bgcval2/analysis_timeseries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index 22699d02..3e01fce4 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -97,7 +97,7 @@ def build_list_of_suite_keys(suites, debug=True): paths_dir = os.path.dirname(os.path.realpath(__file__)) key_lists_dir = os.path.join(os.path.dirname(paths_dir), 'key_lists') - print('key_lists_dir', key_lists_dir) + print(f'analysis_timeseries: Directory where keys are stored: {key_lists_dir}') analysis_keys = {} for suite in suites: print(suite) From bd3012d8a1c772909f0d514e2a8f03352baea597 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 16 Aug 2022 05:56:31 +0100 Subject: [PATCH 23/27] Update bgcval2/analysis_timeseries.py Co-authored-by: Valeriu Predoi --- bgcval2/analysis_timeseries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index 3e01fce4..7d01b2a4 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -107,7 +107,7 @@ def build_list_of_suite_keys(suites, debug=True): print('build_list_of_suite_keys:\tlooking for suite yaml:', suite_yml) if not os.path.exists(suite_yml): - print('build_list_of_suite_keys:\tERROR: suite yaml:', suite_yml, 'does not exist') + print(f'analysis_timeseries: build_list_of_suite_keys:\tERROR: suite yaml: {suite_yml} file does not exist') sys.exit(1) # Open yml file: From 4755b353fcdd4d878e2c08b7406fc203be87fce7 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 16 Aug 2022 05:56:43 +0100 Subject: [PATCH 24/27] Update bgcval2/analysis_timeseries.py Co-authored-by: Valeriu Predoi --- bgcval2/analysis_timeseries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index 7d01b2a4..c397f380 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -123,7 +123,7 @@ def build_list_of_suite_keys(suites, debug=True): for key, keybool in keys_dict.items(): if debug and key in analysis_keys: - print('build_list_of_suite_keys:\tKey exists in multiple suites:', key) + print(f'build_list_of_suite_keys:\tKey {key} exists in multiple suites:') if key in analysis_keys and keybool != analysis_keys[key]: From 66f065a679f68d0a4cc234394c504959c0585ff0 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 16 Aug 2022 05:56:57 +0100 Subject: [PATCH 25/27] Update bgcval2/analysis_timeseries.py Co-authored-by: Valeriu Predoi --- bgcval2/analysis_timeseries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index c397f380..05b6c10b 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -127,7 +127,7 @@ def build_list_of_suite_keys(suites, debug=True): if key in analysis_keys and keybool != analysis_keys[key]: - print('build_list_of_suite_keys:\tERROR: conflick in input yamls:', key, keybool, '!=', analysis_keys[key]) + print(f'build_list_of_suite_keys:\tERROR: conflict in input yamls: {key}, {keybool} != {analysis_keys[key]}') sys.exit(1) if keybool: From f3085e59fbedbb472a2217ec4ffbe8f457e9a52b Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 16 Aug 2022 05:57:05 +0100 Subject: [PATCH 26/27] Update bgcval2/analysis_timeseries.py Co-authored-by: Valeriu Predoi --- bgcval2/analysis_timeseries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgcval2/analysis_timeseries.py b/bgcval2/analysis_timeseries.py index 05b6c10b..ba84b3aa 100755 --- a/bgcval2/analysis_timeseries.py +++ b/bgcval2/analysis_timeseries.py @@ -178,7 +178,7 @@ def analysis_timeseries( print('jobID:', jobID) print('suites:', suites) print('regions:', regions) - print('clean:', clean, 'annual:', annual, 'strictFileCheck:', strictFileCheck) + print(f'clean: {clean}, annual: {annual}, strictFileCheck: {strictFileCheck}') print('config_user:', config_user) # get runtime configuration From bddcc1ebdc2a593292be86a36082fd3c9219938d Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 16 Aug 2022 06:02:46 +0100 Subject: [PATCH 27/27] Added description of keys. also removed end of line spaces. --- README.md | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 63f216fb..4fd3afea 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Test that the tool has been installed correctly with: ``` analysis_compare -h ``` -which should print the module information and instructions on how to run the tool. +which should print the module information and instructions on how to run the tool. ### Available executables @@ -66,8 +66,8 @@ Executable name | What it does | Command Running the tool to compare multiple jobs ========================================= -The time developmenmt of several models can be compared -and summarized in a single comparison report html. +The time developmenmt of several models can be compared +and summarized in a single comparison report html. This report can be generated with a single command, based on a simple yml file input: ``` @@ -100,26 +100,26 @@ jobs: ``` These values are: - - `name`: + - `name`: - The name of the analysis. - This will be the path of output report. - - `do_analysis_timeseries`: - - A Boolean value to run or skip the single model timeseries. + - `do_analysis_timeseries`: + - A Boolean value to run or skip the single model timeseries. - Set to False if the single model analysis has already completed. - - `do_mass_download`: - - A boolean value to run the mass download. + - `do_mass_download`: + - A boolean value to run the mass download. - This is not currently possible as we can only download mass file from mass-cli1 on jasmin. - See below for details on how to download jobs data. - `master_suites`: - - A list of the type of analysis report to produce. + - A list of the type of analysis report to produce. - Options are: `physics`, `bio`, `debug`. - Default is `['physics', 'bio',]`. - `jobs`: - - A list of jobIDs, and some options on how they will appear in the final report. + - A list of jobIDs, and some options on how they will appear in the final report. - The options are: - `description`: - A description of job, which helps people differentiate the jobs in the final report. - - `colour`: + - `colour`: - The colour of this jobs line in the report plots. - Default colour is a randomly generated hex colour. - `thickness`: @@ -134,15 +134,15 @@ These values are: - this is useful if jobs start from different initial years in spin up, for instance. - Default is `0.` ( no time shift ). - `suite`: - - An analysis suite to run the analysis_timeseries. - - See `analysis_timeseries` for more details. + - One or more analysis suites to run the analysis_timeseries. + - Options include: `bgc` (biogeochemistry), `kmf` (key metrics first), `physics`, `level1`. + - See `analysis_timeseries` for more details, below. A sample yaml exists in `input_yml/comparison_analysis_template.yml`, which can be adapted to additional analyses. - Downloading data using MASS =========================== @@ -174,7 +174,7 @@ Several different keys can be included in the download if monthly data is requir However, it's not recommended to include monthly data at this stage as that code both to download, but also to run the monthly analysis is not currently tested. -This tool downloads the data, but also includes several functions which create symbolic links +This tool downloads the data, but also includes several functions which create symbolic links in the data's directory in order to accomodate incompatible changes in NEMO's output formatting. @@ -188,7 +188,7 @@ To run an in-depth analysis of a single job, the following command can be run: bgcval2 -j jobID ``` -This will run a time series analysis, a point to point analysis, and +This will run a time series analysis, a point to point analysis, and publish the reports into a single job html5 report. @@ -212,7 +212,7 @@ This is an analysis that investigates the time development of specific marine physics and Biogeochemistry fields in the given model, and then compares them against historical observations. -The command to run it is `analysis_timeseries --jobID jobID --keys key`, +The command to run it is `analysis_timeseries --jobID jobID --keys key`, where jobID is one or more mass jobIDs, such a `u-ab123`. The key is one or more pre-defined keys, which generates a list of variables. @@ -226,9 +226,12 @@ Key | What it is | Description `debug` | Debug | A very short list of a couple keys to test code changes. `fast` | UKESM1-fast | A list of metrics tailored to the UKESM1-Fast model. - Note that there may be some overlap between the contents of these keys. +These key lists are defined in the bgcval2 directory, `key_lists`. +To add an additional list of keys, the file must be present there +as a yaml and must be named with all-lower-case letters. + Point to point analysis -----------------------