add short 150m exp

PrimeIntellect-ai · Sep 27, 2024 · 8394e6e · 8394e6e
1 parent d9059e1
commit 8394e6e
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 0 deletions.
diff --git a/configs/150M_short/3090.toml b/configs/150M_short/3090.toml
@@ -0,0 +1,12 @@
+name_model = "150M"
+project = "debug_150m_zero_band"
+
+[train]
+micro_bs = 16 # change this base on the gpu
+sharding_strategy = "SHARD_GRAD_OP"
+
+[optim]
+batch_size = 512
+warmup_steps = 1000
+total_steps = 15_000
+lr = 4e-4
diff --git a/configs/150M_short/A40.toml b/configs/150M_short/A40.toml
@@ -0,0 +1,12 @@
+name_model = "150M"
+project = "debug_150m_zero_band"
+
+[train]
+micro_bs = 32 # change this base on the gpu
+sharding_strategy = "SHARD_GRAD_OP"
+
+[optim]
+batch_size = 512
+warmup_steps = 1000
+total_steps = 15_000
+lr = 4e-4
diff --git a/configs/150M_short/H100.toml b/configs/150M_short/H100.toml
@@ -0,0 +1,12 @@
+name_model = "150M"
+project = "debug_150m_zero_band"
+
+[train]
+micro_bs = 64 # change this base on the gpu
+sharding_strategy = "SHARD_GRAD_OP"
+
+[optim]
+batch_size = 512
+warmup_steps = 1000
+total_steps = 15_000
+lr = 4e-4