add 1B

PrimeIntellect-ai · Sep 21, 2024 · 01d13a6 · 01d13a6
1 parent e5a1938
commit 01d13a6
Showing 1 changed file with 12 additions and 0 deletions.
diff --git a/configs/1B/H100.toml b/configs/1B/H100.toml
@@ -0,0 +1,12 @@
+name_model = "150M"
+project = "debug_150m_zero_band"
+
+[train]
+micro_bs = 16  
+sharding_strategy = "SHARD_GRAD_OP"
+
+[optim]
+batch_size = 512
+warmup_steps = 1000
+total_steps = 88_000
+lr = 4e-4