add 7b

PrimeIntellect-ai · Sep 21, 2024 · 118eb9e · 118eb9e
1 parent 3051acf
commit 118eb9e
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 2 deletions.
diff --git a/configs/1B/H100.toml b/configs/1B/H100.toml
@@ -1,5 +1,5 @@
-name_model = "150M"
-project = "debug_150m_zero_band"
+name_model = "1B"
+project = "debug_1B_zero_band"
 
 [train]
 micro_bs = 16  

diff --git a/configs/7B/H100.toml b/configs/7B/H100.toml
@@ -0,0 +1,12 @@
+name_model = "7B"
+project = "debug_7B_zero_band"
+
+[train]
+micro_bs = 6  
+sharding_strategy = "SHARD_GRAD_OP"
+
+[optim]
+batch_size = 3840
+warmup_steps = 1000
+total_steps = 88_000
+lr = 6e-4