update 1b confoigs

PrimeIntellect-ai · Dec 10, 2024 · 8646e14 · 8646e14
1 parent e996e85
commit 8646e14
Showing 1 changed file with 4 additions and 4 deletions.
diff --git a/configs/1B/H100.toml b/configs/1B/H100.toml
@@ -3,13 +3,13 @@ project = "debug_1B_zero_band"
 type_model = "llama2"
 
 [train]
-micro_bs = 16  
+micro_bs = 32
 reshard_after_forward = true
 
 [optim]
-batch_size = 2048
-warmup_steps = 500
+batch_size = 1024
+warmup_steps = 1000
 total_steps = 8192
 
 [optim.optim]
-lr = 4e-4
+lr = 7e-4