diff --git a/bgcval2/RemoteScripts/downloadOnMassCli.sh b/bgcval2/RemoteScripts/downloadOnMassCli.sh index 8a4d4c21..b3518ece 100755 --- a/bgcval2/RemoteScripts/downloadOnMassCli.sh +++ b/bgcval2/RemoteScripts/downloadOnMassCli.sh @@ -22,13 +22,13 @@ python /home/users/ldemora/workspace/ukesm-validation/RemoteScripts/hello.py $jo #printf "\n\nssh -A -X mass-cli1 'python /home/users/ldemora/workspace/ukesm-validation/RemoteScripts/hello.py'\n" #ssh -X -A mass-cli1 'python /home/users/ldemora/workspace/ukesm-validation/RemoteScripts/hello.py' -ssh -X -A mass-cli.jasmin.ac.uk "python /home/users/ldemora/workspace/ukesm-validation/bgcvaltools/downloadFromMass.py $jobid" +ssh -X -A mass-cli.jasmin.ac.uk "python /home/users/ldemora/workspace/ukesm-validation/download_from_mass.py $jobid" #ssh -X -A mass-cli.jasmin.ac.uk "mkdir -p gws/nopw/j04/ukesm/BGC_data/u/group_workspaces/jasmin2/ukesm/BGC_data/$jobid/1y; ln -s /group_workspaces/jasmin2/ukesm/BGC_data/$jobid/*1y*grid_[UVWT]*.nc /group_workspaces/jasmin2/ukesm/BGC_data/$jobid/1y/.; ls -lhrt /group_workspaces/jasmin2/ukesm/BGC_data/$jobid" ssh -X -A mass-cli.jasmin.ac.uk "mkdir -p /gws/nopw/j04/ukesm/BGC_data/$jobid/1y; ln -s /gws/nopw/j04/ukesm/BGC_data/$jobid/*1y*grid_[UVWT]*.nc /gws/nopw/j04/ukesm/BGC_data/$jobid/1y/.; ls -lhrt /gws/nopw/j04/ukesm/BGC_data/$jobid" ##### # MLD: -#ssh -X -A mass-cli1 "python /home/users/ldemora/workspace/ukesm-validation/bgcvaltools/downloadFromMass.py $jobid mld" +#ssh -X -A mass-cli1 "python /home/users/ldemora/workspace/ukesm-validation/download_from_mass.py $jobid mld" echo "The end of downloadOnMassCli.sh $jobid" diff --git a/bgcval2/RemoteScripts/downloadOnMassCli_MLD.sh b/bgcval2/RemoteScripts/downloadOnMassCli_MLD.sh index fabdfcfc..36dfba79 100755 --- a/bgcval2/RemoteScripts/downloadOnMassCli_MLD.sh +++ b/bgcval2/RemoteScripts/downloadOnMassCli_MLD.sh @@ -22,12 +22,12 @@ python /home/users/ldemora/workspace/ukesm-validation/RemoteScripts/hello.py $jo #printf "\n\nssh -A -X mass-cli1 'python /home/users/ldemora/workspace/ukesm-validation/RemoteScripts/hello.py'\n" #ssh -X -A mass-cli1 'python /home/users/ldemora/workspace/ukesm-validation/RemoteScripts/hello.py' -ssh -X -A mass-cli1 "python /home/users/ldemora/workspace/ukesm-validation/bgcvaltools/downloadFromMass.py $jobid MLD" +ssh -X -A mass-cli1 "python /home/users/ldemora/workspace/ukesm-validation/download_from_mass.py $jobid MLD" ssh -X -A mass-cli1 "mkdir -p /group_workspaces/jasmin2/ukesm/BGC_data/$jobid/1y; ln -s /group_workspaces/jasmin2/ukesm/BGC_data/$jobid/*1y*grid_[UVWT]*.nc /group_workspaces/jasmin2/ukesm/BGC_data/$jobid/1y/.; ls -lhrt /group_workspaces/jasmin2/ukesm/BGC_data/$jobid" ##### # MLD: -#ssh -X -A mass-cli1 "python /home/users/ldemora/workspace/ukesm-validation/bgcvaltools/downloadFromMass.py $jobid mld" +#ssh -X -A mass-cli1 "python /home/users/ldemora/workspace/ukesm-validation/download_from_mass.py $jobid mld" echo "The end of downloadOnMassCli.sh $jobid" diff --git a/bgcval2/bgcval.py b/bgcval2/bgcval.py index 1f195f2b..e73d671f 100755 --- a/bgcval2/bgcval.py +++ b/bgcval2/bgcval.py @@ -38,7 +38,7 @@ from sys import argv, exit from multiprocessing import Pool -from .bgcvaltools.downloadFromMass import downloadMass, findLastFinishedYear +from .download_from_mass import findLastFinishedYear from .analysis_timeseries import analysis_timeseries, singleTimeSeries, singleTimeSeriesProfile from .analysis_timeseries import level1KeysDict, physKeysDict, bgcKeysDict, keymetricsfirstDict from .analysis_p2p import analysis_p2p, p2pDict_level2, p2pDict_physics, single_p2p diff --git a/bgcval2/bgcvaltools/mergeMonthlyFiles.py b/bgcval2/bgcvaltools/mergeMonthlyFiles.py index f8b8e7b1..c12b4d46 100644 --- a/bgcval2/bgcvaltools/mergeMonthlyFiles.py +++ b/bgcval2/bgcvaltools/mergeMonthlyFiles.py @@ -118,7 +118,7 @@ def mergeMonthlyFiles(files, expectedNumberOfFiles=12): ##### # This assuemd that the files have already been split up using the moo filter tool - # done in the the bgcvalTools/downloadFromMass.py + # done in the the download_from_mass.py filesOut = [] years = {} @@ -164,7 +164,7 @@ def meanDJF( ): ##### # This assuemd that the files have already been split up using the moo filter tool - # done in the the bgcvalTools/downloadFromMass.py + # done in download_from_mass.py filesOut = [] years = {} diff --git a/bgcval2/download_from_mass.py b/bgcval2/download_from_mass.py index d4f8e281..03f87544 100755 --- a/bgcval2/download_from_mass.py +++ b/bgcval2/download_from_mass.py @@ -22,7 +22,7 @@ # Email: # ledm@pml.ac.uk """ -.. module:: downloadFromMass +.. module:: download_from_mass :platform: Unix :synopsis: A set of tools to download the UKESM model run data from MASS. .. moduleauthor:: Lee de Mora @@ -47,7 +47,7 @@ When run as a script, the command is:: - ./downloadFromMass.py jobID + ./download_from_mass.py jobID This tool will only work on machines that have mass enabled. @@ -490,7 +490,7 @@ def fixFilePaths(outputFold, jobID, debug=False): ##### # The coupled model looses the first two characters of the name in the netcdf file. fns = glob(outputFold + "/*" + jobID[2:] + "*.nc") - print("downloadFromMass:\tfixFilePaths:\tLooking for", + print("download_from_mass:\tfixFilePaths:\tLooking for", outputFold + "/" + jobID[2:] + "*.nc") fns.extend( glob(outputFold + @@ -502,25 +502,25 @@ def fixFilePaths(outputFold, jobID, debug=False): if os.path.exists(correctfn): if debug: - print("downloadFromMass:\tfixFilePaths:\tcorrect path exists.", + print("download_from_mass:\tfixFilePaths:\tcorrect path exists.", correctfn) continue if correctfn == fn: continue - print("downloadFromMass:\tfixFilePaths:\tFixing file prefix", fn, + print("download_from_mass:\tfixFilePaths:\tFixing file prefix", fn, '-->', correctfn) try: os.symlink(fn, correctfn) except: print("Unable to make link:", correctfn) continue -# print "downloadFromMass:\tfixFilePaths:\t", correctfn +# print "download_from_mass:\tfixFilePaths:\t", correctfn ##### # Some runs have nemo/medusa as a preface to the file name. for pref in ['nemo_', 'medusa_']: #nemo_u-ai886o_1y_26291201-26301201_grid-V.nc fns = glob(outputFold + "/" + pref + jobID + "*.nc") - print("downloadFromMass:\tfixFilePaths:\tLooking for new prefix:", + print("download_from_mass:\tfixFilePaths:\tLooking for new prefix:", pref, outputFold + "/" + pref + jobID + "*.nc") for fn in sorted(fns): ##### @@ -529,14 +529,14 @@ def fixFilePaths(outputFold, jobID, debug=False): if os.path.exists(correctfn): if debug: print( - "downloadFromMass:\tfixFilePaths:\tcorrect path exists.", + "download_from_mass:\tfixFilePaths:\tcorrect path exists.", correctfn) continue - print("downloadFromMass:\tfixFilePaths:\tFixing file prefix", + print("download_from_mass:\tfixFilePaths:\tFixing file prefix", pref, end=' ') os.symlink(fn, correctfn) - print("downloadFromMass:\tfixFilePaths:\t", correctfn) + print("download_from_mass:\tfixFilePaths:\t", correctfn) ##### # Some runs have nemo/medusa as a preface to the file name. @@ -552,7 +552,7 @@ def fixFilePaths(outputFold, jobID, debug=False): for badsuff, suff in list(suffDict.items()): #nemo_u-ai886o_1y_26291201-26301201_grid-V.nc fns = glob(outputFold + "/" + jobID + "*" + badsuff + ".nc") - print("downloadFromMass:\tfixFilePaths:\tLooking for new suff:", + print("download_from_mass:\tfixFilePaths:\tLooking for new suff:", badsuff, outputFold + "/" + jobID + "*" + badsuff + ".nc") for fn in sorted(fns): ##### @@ -561,10 +561,10 @@ def fixFilePaths(outputFold, jobID, debug=False): if os.path.exists(correctfn): if debug: print( - "downloadFromMass:\tfixFilePaths:\tcorrect path exists.", + "download_from_mass:\tfixFilePaths:\tcorrect path exists.", correctfn) continue - print("downloadFromMass:\tfixFilePaths:\tFixing file suffix", + print("download_from_mass:\tfixFilePaths:\tFixing file suffix", badsuff, '->', suff, @@ -575,7 +575,7 @@ def fixFilePaths(outputFold, jobID, debug=False): os.symlink(fn, correctfn) except: continue - print("downloadFromMass:\tfixFilePaths:\t", correctfn) + print("download_from_mass:\tfixFilePaths:\t", correctfn) ##### # This code looks at symoblic links and points them at their ultimate source, removing the long link chains. diff --git a/bgcval2/index.rst b/bgcval2/index.rst index 1cd0629d..80c21f95 100644 --- a/bgcval2/index.rst +++ b/bgcval2/index.rst @@ -118,7 +118,7 @@ The executable top level structure of this module includes the scripts: :makeReport.py: This script creates an html report. :analysis_timeseries.py: This script runs the time series analysis. :analysis_p2p.py: This script runs the point to point analysis. - :downloadFromMass.py: This script is used to download UKESM files from mass. + :download_from_mass.py: This script is used to download UKESM files from mass. These files are set up to run the validation is run for the UK Earth System Model in the eORCA1 grid. They can be used as templates for running other models.