Add a new tp overlap cfg for tp2 llama2 70B (#435)

* Add a new tp overlap cfg for tp2 llama2 70B Signed-off-by: Guyue Huang <[email protected]> * Fix a name in comment Signed-off-by: Guyue Huang <[email protected]> --------- Signed-off-by: Guyue Huang <[email protected]> Co-authored-by: Guyue Huang <[email protected]>
NVIDIA · Oct 3, 2024 · 65c1b74 · 65c1b74
1 parent 35e4393
commit 65c1b74
Showing 1 changed file with 55 additions and 0 deletions.
diff --git a/launcher_scripts/conf/training/tp_overlap/ub_cfg_h100_h8192_tp2_mbs1_seqlen4096.yaml b/launcher_scripts/conf/training/tp_overlap/ub_cfg_h100_h8192_tp2_mbs1_seqlen4096.yaml
@@ -0,0 +1,55 @@
+# UB communicator configurations
+# Model configs: H100/70B/TP2/MBS1/SeqLen4K/FP8
+
+# Bulk overlap with AllGather / ReduceScatter
+qkv_dgrad:
+  method: bulk
+  num_sm: 8
+  cga_size: 2
+  set_sm_margin: 0
+
+qkv_wgrad:
+  method: bulk
+  num_sm: 32
+  cga_size: 2 
+  set_sm_margin: 0
+
+fc1_dgrad:
+  method: bulk
+  num_sm: 2
+  cga_size: 2
+  set_sm_margin: 0
+
+fc1_wgrad:
+  method: bulk
+  num_sm: 8
+  cga_size: 2
+  set_sm_margin: 0
+
+## Ring-exchange overlap with AllGather
+qkv_fprop:
+  method: ring_exchange
+  aggregate: 0
+
+proj_dgrad:
+  method: ring_exchange
+  aggregate: 0
+
+fc1_fprop:
+  method: ring_exchange
+  aggregate: 0
+
+fc2_dgrad:
+  method: ring_exchange
+  aggregate: 0
+
+proj_fprop:
+  method: ring_exchange
+  num_sm: 1
+  set_sm_margin: 1
+
+fc2_fprop:
+  method: ring_exchange
+  num_sm: 1
+  set_sm_margin: 1
+