update 7b config to 2k seq len

PrimeIntellect-ai · Sep 30, 2024 · 1607fd4 · 1607fd4
1 parent e03164b
commit 1607fd4
Showing 1 changed file with 6 additions and 3 deletions.
diff --git a/configs/7B/H100.toml b/configs/7B/H100.toml
@@ -2,11 +2,14 @@ name_model = "7B"
 project = "debug_7B_zero_band"
 
 [train]
-micro_bs = 6  
+micro_bs = 1
 sharding_strategy = "SHARD_GRAD_OP"
 
 [optim]
-batch_size = 3840
+batch_size = 1024 #2M tokens bs
 warmup_steps = 1000
 total_steps = 88_000
-lr = 6e-4
+lr = 3e-4
+
+[data]
+seq_length = 2048