Fix dataset type with Pretrain-IdxMap (#70)

* Fix dataset type with Pretrain-IdxMap * Fix dataset type with Pretrain-IdxMap * Fix dataset type with Pretrain-IdxMap --------- Co-authored-by: 同润 <[email protected]>
alibaba · Nov 16, 2023 · 54d0fa9 · 54d0fa9
1 parent 6294df4
commit 54d0fa9
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/examples/llama2/run_pretrain_megatron_llama.sh b/examples/llama2/run_pretrain_megatron_llama.sh
@@ -50,7 +50,7 @@ OUTPUT_BASEPATH=${24}
 
 if [ $MODEL_SIZE = 7B ]; then
 
-NUM_LAYERS=2
+NUM_LAYERS=32
 HIDDEN_SIZE=4096
 NUM_ATTN_HEADS=32
 INTERMEDIATE_SIZE=11008