update 1b confoigs

PrimeIntellect-ai · Dec 10, 2024 · e996e85 · e996e85
1 parent fb3a721
commit e996e85
Show file tree

Hide file tree

Showing 6 changed files with 3 additions and 117 deletions.
diff --git a/configs/1B/H100.toml b/configs/1B/H100.toml
@@ -4,11 +4,12 @@ type_model = "llama2"
 
 [train]
 micro_bs = 16  
+reshard_after_forward = true
 
 [optim]
 batch_size = 2048
-warmup_steps = 1000
-total_steps = 88_000
+warmup_steps = 500
+total_steps = 8192
 
 [optim.optim]
 lr = 4e-4
diff --git a/configs/1B/H100_c4.toml b/configs/1B/H100_c4.toml
diff --git a/configs/1B/H100_llama2_edu.toml b/configs/1B/H100_llama2_edu.toml
diff --git a/configs/1B/H100_llama2_edu_no_feat.toml b/configs/1B/H100_llama2_edu_no_feat.toml
diff --git a/configs/1B/H100_llama3.toml b/configs/1B/H100_llama3.toml
diff --git a/configs/1B_diloco/H100.toml b/configs/1B_diloco/H100.toml