diff --git a/bgcval2/timeseries/profileAnalysis.py b/bgcval2/timeseries/profileAnalysis.py index 4f5361f0..856247f3 100644 --- a/bgcval2/timeseries/profileAnalysis.py +++ b/bgcval2/timeseries/profileAnalysis.py @@ -33,6 +33,7 @@ from netCDF4 import num2date import os import shutil +from glob import glob #Specific local code: from bgcval2.bgcvaltools import bv2tools as bvt @@ -181,25 +182,19 @@ def loadModel(self): # load and calculate the model info self.setmlayers() - try: - if self.clean: - print("profileAnalysis:\tloadModel:\tUser requested clean run. Wiping old data.") - assert 0 - with shOpen(self.shelvefn) as sh: - readFiles = sh['readFiles'] - modeldataD = sh['modeldata'] - print("OprofileAnalysis:\tloadModel:\tpened shelve:", self.shelvefn, '\tread', len( - readFiles)) - except: + if self.clean or not len(glob(self.shelvefn+'*')): + #print("profileAnalysis:\tloadModel:\tUser requested clean run. Wiping old data.") readFiles = [] modeldataD = {} - # self.setmlayers() for r in self.regions: for l in self.mlayers: for m in self.metrics: modeldataD[(r, l, m)] = {} - - print("profileAnalysis:\tloadModel:\tCould not open shelve:", self.shelvefn, '\tread', len( + else: + with shOpen(self.shelvefn) as sh: + readFiles = sh['readFiles'] + modeldataD = sh['modeldata'] + print("ProfileAnalysis:\tloadModel:\tOpened shelve:", self.shelvefn, '\tread', len( readFiles)) ############### @@ -367,17 +362,14 @@ def loadData(self): ############### # load and calculate the real data info - try: - if self.clean: - print("profileAnalysis:\t loadData\tUser requested clean run. Wiping old data.") - assert 0 + if self.clean or not len(glob(self.shelvefn_insitu+'*')): + print("profileAnalysis:\t loadData\tClean run.") + dataD = {} + else: with shOpen(self.shelvefn_insitu) as sh: dataD = sh['dataD'] print("profileAnalysis:\t loadData\tOpened shelve:", self.shelvefn_insitu) self.dataD = dataD - except: - dataD = {} - print("profileAnalysis:\t loadData\tCould not open shelve:", self.shelvefn_insitu) ############### # Test to find out if we need to load the netcdf, or if we can just return the dict as a self.object. @@ -488,26 +480,29 @@ def loadData(self): ############### # Savng shelve print("profileAnalysis:\t loadData.\tSaving shelve:", self.shelvefn_insitu) - try: - with shOpen(self.shelvefn_insitu) as sh: - sh['dataD'] = dataD - print("profileAnalysis:\t loadData.\tSaved shelve:", self.shelvefn_insitu) + with shOpen(self.shelvefn_insitu) as sh: + sh['dataD'] = dataD - except: - print("profileAnalysis:\t WARNING.\tSaving shelve failed, trying again.:", self.shelvefn_insitu) - print("Data is", list(dataD.keys())) + # try: + # with shOpen(self.shelvefn_insitu) as sh: + # sh['dataD'] = dataD + # print("profileAnalysis:\t loadData.\tSaved shelve:", self.shelvefn_insitu) - for key in sorted(dataD.keys()): + # except: + # print("profileAnalysis:\t WARNING.\tSaving shelve failed, trying again.:", self.shelvefn_insitu) + # print("Data is", list(dataD.keys())) - print(key, ':\t', dataD[key]) - with shOpen(bvt.folder('./tmpshelves') + 'tmshelve.shelve') as sh: - sh['dataD'] = dataD[key] - print("saved fine:\t./tmpshelves/tmshelve.shelve") + # for key in sorted(dataD.keys()): - shutil.move(self.shelvefn_insitu, self.shelvefn_insitu + '.broken') + # print(key, ':\t', dataD[key]) + # with shOpen(bvt.folder('./tmpshelves') + 'tmshelve.shelve') as sh: + # sh['dataD'] = dataD[key] + # print("saved fine:\t./tmpshelves/tmshelve.shelve") - with shOpen(self.shelvefn_insitu) as sh: - sh['dataD'] = dataD + # shutil.move(self.shelvefn_insitu, self.shelvefn_insitu + '.broken') + + # with shOpen(self.shelvefn_insitu) as sh: + # sh['dataD'] = dataD # except: # print "profileAnalysis:\t WARNING.\tUnable to Save in situ shelve.\tYou'll have to input it each time.",self.shelvefn_insitu diff --git a/bgcval2/timeseries/timeseriesAnalysis.py b/bgcval2/timeseries/timeseriesAnalysis.py index b3db5bd2..1829292e 100644 --- a/bgcval2/timeseries/timeseriesAnalysis.py +++ b/bgcval2/timeseries/timeseriesAnalysis.py @@ -137,7 +137,7 @@ def loadModel(self): if self.debug: print("timeseriesAnalysis:\tloadModel.") #### # load and calculate the model info - if glob.glob(self.shelvefn+'*'): # shelve files have .bak .dat .dir files now + if len(glob.glob(self.shelvefn+'*')): # shelve files have .bak .dat .dir files now with shOpen(self.shelvefn) as sh: print('Shelf opens fine:', self.shelvefn) print (sh.keys()) @@ -565,10 +565,8 @@ def loadData(self): ############### # load and calculate the real data info - if self.clean: - print( - "timeseriesAnalysis:\t loadData\tUser requested clean run. Wiping old data." - ) + if self.clean or not len(glob(self.shelvefn_insit+'*')): + print("timeseriesAnalysis:\t loadData\tClean run.") dataD = {} else: with shOpen(self.shelvefn_insitu) as sh: