From 118eb9e955806d7dfbac2fb482e69b8e3edda328 Mon Sep 17 00:00:00 2001
From: Sami Jaghouar <sami.jaghouar@hotmail.fr>
Date: Sat, 21 Sep 2024 00:57:00 +0000
Subject: [PATCH] add 7b

---
 configs/1B/H100.toml |  4 ++--
 configs/7B/H100.toml | 12 ++++++++++++
 2 files changed, 14 insertions(+), 2 deletions(-)
 create mode 100644 configs/7B/H100.toml

diff --git a/configs/1B/H100.toml b/configs/1B/H100.toml
index cf8111b1..1430dcea 100644
--- a/configs/1B/H100.toml
+++ b/configs/1B/H100.toml
@@ -1,5 +1,5 @@
-name_model = "150M"
-project = "debug_150m_zero_band"
+name_model = "1B"
+project = "debug_1B_zero_band"
 
 [train]
 micro_bs = 16  
diff --git a/configs/7B/H100.toml b/configs/7B/H100.toml
new file mode 100644
index 00000000..c1272c34
--- /dev/null
+++ b/configs/7B/H100.toml
@@ -0,0 +1,12 @@
+name_model = "7B"
+project = "debug_7B_zero_band"
+
+[train]
+micro_bs = 6  
+sharding_strategy = "SHARD_GRAD_OP"
+
+[optim]
+batch_size = 3840
+warmup_steps = 1000
+total_steps = 88_000
+lr = 6e-4
\ No newline at end of file