update 7b config (#27)

* align model seq len with data seq len * update 7b config to 2k seq len --------- Co-authored-by: Sami jaghouar <[email protected]>
PrimeIntellect-ai · Oct 1, 2024 · 18db923 · 18db923
1 parent 80d050a
commit 18db923
Showing 1 changed file with 6 additions and 3 deletions.
diff --git a/configs/7B/H100.toml b/configs/7B/H100.toml
@@ -2,11 +2,14 @@ name_model = "7B"
 project = "debug_7B_zero_band"
 
 [train]
-micro_bs = 6  
+micro_bs = 1
 sharding_strategy = "SHARD_GRAD_OP"
 
 [optim]
-batch_size = 3840
+batch_size = 1024 #2M tokens bs
 warmup_steps = 1000
 total_steps = 88_000
-lr = 6e-4
+lr = 3e-4
+
+[data]
+seq_length = 2048