From cf52146aca7e2e363b94aa27cf91e190d58a6e2c Mon Sep 17 00:00:00 2001 From: jrobrien91 Date: Thu, 29 Feb 2024 14:50:27 -0500 Subject: [PATCH 1/2] ENH: Changed default output directory to wolf2, removed processes from dask cluster --- scripts/sail_glue.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/sail_glue.py b/scripts/sail_glue.py index 714f3a1..4c1a4db 100644 --- a/scripts/sail_glue.py +++ b/scripts/sail_glue.py @@ -71,12 +71,9 @@ def fix_times(ds): return ds def granule(Dvolume): - print('in granule') n_tilts = 8 - #data_dir = "/gpfs/wolf/atm124/proj-shared/gucxprecipradarS2.00/nc_files/" + month + "_nc/" - #out_dir = "/gpfs/wolf/atm124/proj-shared/gucxprecipradarS2.00/glue_files/" + month + "_glued/" month = Dvolume[0].split('/')[-2].split('_')[0] - out_dir = Dvolume[0].split('nc_files')[0] + "glue_files/" + month + "_glued/" + out_dir = '/gpfs/wolf2/arm/atm124/proj-shared/gucxprecipradarS2/glue_files/%s_glued/' % month # Read the base scan to determine if it can be read in if len(Dvolume) == 8: @@ -120,7 +117,7 @@ def main(args): # Define directories month = args.month path = '/gpfs/wolf/atm124/proj-shared/gucxprecipradarS2.00/nc_files/%s_nc/*.nc' % month - out_path = '/gpfs/wolf/atm124/proj-shared/gucxprecipradarS2.00/glue_files/%s_glued/' % month + out_path = '/gpfs/wolf2/arm/atm124/proj-shared/gucxprecipradarS2/glue_files/%s_glued/' % month # Define files and determine volumes all_files = sorted(glob.glob(path)) @@ -146,8 +143,13 @@ def main(args): if args.serial is True: granule(volumes[0]) + granule(volumes[1]) + granule(volumes[2]) + print("processing finished: ", time.strftime("%H:%M:%S")) else: - cluster = LocalCluster(n_workers=20, processes=True, threads_per_worker=1) + print("starting dask cluster...") + cluster = LocalCluster(n_workers=8, threads_per_worker=1) + print(cluster) with Client(cluster) as c: results = c.map(granule, volumes) wait(results) @@ -167,7 +169,7 @@ def main(args): default=False, dest='serial', type=bool, - help="Process in Serial" + help="Process in Serial for testing" ) args = parser.parse_args() From 305c8292ca960a55cc538c521a11e87f6edddaca Mon Sep 17 00:00:00 2001 From: jrobrien91 Date: Mon, 22 Apr 2024 13:28:09 -0400 Subject: [PATCH 2/2] ENH: update to python script for glue-processing via slurm --- scripts/sail_glue.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/sail_glue.py b/scripts/sail_glue.py index 4c1a4db..5480683 100644 --- a/scripts/sail_glue.py +++ b/scripts/sail_glue.py @@ -73,7 +73,7 @@ def fix_times(ds): def granule(Dvolume): n_tilts = 8 month = Dvolume[0].split('/')[-2].split('_')[0] - out_dir = '/gpfs/wolf2/arm/atm124/proj-shared/gucxprecipradarS2/glue_files/%s_glued/' % month + out_dir = '/gpfs/wolf2/arm/atm124/proj-shared/gucxprecipradarS2_new/glue_files/%s_glued/' % month # Read the base scan to determine if it can be read in if len(Dvolume) == 8: @@ -117,7 +117,7 @@ def main(args): # Define directories month = args.month path = '/gpfs/wolf/atm124/proj-shared/gucxprecipradarS2.00/nc_files/%s_nc/*.nc' % month - out_path = '/gpfs/wolf2/arm/atm124/proj-shared/gucxprecipradarS2/glue_files/%s_glued/' % month + out_path = '/gpfs/wolf2/arm/atm124/proj-shared/gucxprecipradarS2_new/glue_files/%s_glued/' % month # Define files and determine volumes all_files = sorted(glob.glob(path)) @@ -148,7 +148,7 @@ def main(args): print("processing finished: ", time.strftime("%H:%M:%S")) else: print("starting dask cluster...") - cluster = LocalCluster(n_workers=8, threads_per_worker=1) + cluster = LocalCluster(n_workers=32, threads_per_worker=1) print(cluster) with Client(cluster) as c: results = c.map(granule, volumes)