-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathrun.py
54 lines (42 loc) · 1.52 KB
/
run.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
import os
from datetime import datetime
import hydra
import jax
import matplotlib
import wandb
from omegaconf import DictConfig, OmegaConf
from utils.helper import flatten_dict, reset_device_memory
from utils.train_selector import get_train_fn
@hydra.main(version_base=None, config_path="configs", config_name="base_conf")
def main(cfg: DictConfig) -> None:
os.environ['HYDRA_FULL_ERROR'] = '1'
# Load the chosen algorithm-specific configuration dynamically
cfg = hydra.utils.instantiate(cfg)
target = cfg.target.fn
if not cfg.wandb.get("name"):
cfg.wandb.name = f'{cfg.algorithm.name}_{cfg.target.name}_{target.dim}_{datetime.now()}_seed{cfg.seed}'
if not cfg.visualize_samples:
matplotlib.use('agg')
if cfg.use_wandb:
wandb.init(**cfg.wandb,
group=f'{cfg.algorithm.name}',
job_type=f'{cfg.target.name}_{target.dim}D',
config=flatten_dict(OmegaConf.to_container(cfg, resolve=True, throw_on_missing=True)))
train_fn = get_train_fn(cfg.algorithm.name)
try:
if cfg.use_jit:
train_fn(cfg, target)
else:
with jax.disable_jit():
train_fn(cfg, target)
if cfg.use_wandb:
wandb.run.summary["error"] = None
wandb.finish()
except Exception as e:
if cfg.use_wandb:
wandb.run.summary["error"] = str(e)
wandb.finish(exit_code=1)
reset_device_memory()
raise e
if __name__ == "__main__":
main()