update short configs

PrimeIntellect-ai · Dec 5, 2024 · 1492720 · 1492720
1 parent f79125e
commit 1492720
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/configs/150M_short/H100.toml b/configs/150M_short/H100.toml
@@ -4,11 +4,12 @@ type_model = "llama2"
 
 [train]
 micro_bs = 64 # change this base on the gpu
+reshard_after_forward = true
 
 [optim]
 batch_size = 512
 warmup_steps = 1000
-total_steps = 15_000
+total_steps = 8192
 
 [optim.optim]
 lr = 4e-4