[Fix] Fix DIST_TIMEOUT

kennymckormick · kennymckormick · commit 0eef2a64883c · 2024-12-27T11:51:44.000Z
diff --git a/run.py b/run.py
@@ -183,7 +183,7 @@ def main():
         torch.cuda.set_device(int(local_rank))
         dist.init_process_group(
             backend='nccl',
-            timeout=datetime.timedelta(seconds=os.environ.get('DIST_TIMEOUT', 3600))
+            timeout=datetime.timedelta(seconds=int(os.environ.get('DIST_TIMEOUT', 3600)))
         )
 
     for _, model_name in enumerate(args.model):

Original file line number	Diff line number	Diff line change
`@@ -183,7 +183,7 @@ def main():`
`183`	`183`	`torch.cuda.set_device(int(local_rank))`
`184`	`184`	`dist.init_process_group(`
`185`	`185`	`backend='nccl',`
`186`		`- timeout=datetime.timedelta(seconds=os.environ.get('DIST_TIMEOUT', 3600))`
	`186`	`+ timeout=datetime.timedelta(seconds=int(os.environ.get('DIST_TIMEOUT', 3600)))`
`187`	`187`	`)`
`188`	`188`
`189`	`189`	`for _, model_name in enumerate(args.model):`