Enable max_model_len when when initializing VLLM engine

Author: xu-song

configs/models/mistral/vllm_mixtral_8x7b_instruct_v0_1.py

@@ -8,16 +8,18 @@
         dict(role='BOT', begin='', end='</s>', generate=True),
     ],
 )
+max_seq_len = 2048
 
 models = [
     dict(
         type=VLLM,
         abbr='mixtral-8x7b-instruct-v0.1-vllm',
         path='mistralai/Mixtral-8x7B-Instruct-v0.1',
-        model_kwargs=dict(tensor_parallel_size=2),
+        # more vllm model_kwargs: https://github.com/vllm-project/vllm/blob/main/vllm/engine/arg_utils.py
+        model_kwargs=dict(tensor_parallel_size=2, max_model_len=max_seq_len),
         meta_template=_meta_template,
         max_out_len=100,
-        max_seq_len=2048,
+        max_seq_len=max_seq_len,
         batch_size=32,
         generation_kwargs=dict(temperature=0),
         stop_words=['</s>'],

configs/models/qwen/vllm_qwen1_5_14b_chat.py

@@ -7,16 +7,18 @@
         dict(role='BOT', begin='<|im_start|>assistant\n', end='<|im_end|>\n', generate=True),
     ],
 )
+max_seq_len = 2048
 
 models = [
     dict(
         type=VLLM,
         abbr='qwen1.5-14b-chat-vllm',
         path='Qwen/Qwen1.5-14B-Chat',
-        model_kwargs=dict(tensor_parallel_size=2),
+        # more vllm model_kwargs: https://github.com/vllm-project/vllm/blob/main/vllm/engine/arg_utils.py
+        model_kwargs=dict(tensor_parallel_size=2, max_model_len=max_seq_len),
         meta_template=_meta_template,
         max_out_len=100,
-        max_seq_len=2048,
+        max_seq_len=max_seq_len,
         batch_size=32,
         generation_kwargs=dict(temperature=0),
         stop_words=['<|im_end|>'],

configs/models/qwen/vllm_qwen1_5_72b.py

@@ -1,14 +1,16 @@
 from opencompass.models import VLLM
 
+max_seq_len = 2048
 
 models = [
     dict(
         type=VLLM,
         abbr='qwen1.5-72b-vllm',
         path='Qwen/Qwen1.5-72B',
-        model_kwargs=dict(tensor_parallel_size=4),
+        # more vllm model_kwargs: https://github.com/vllm-project/vllm/blob/main/vllm/engine/arg_utils.py
+        model_kwargs=dict(tensor_parallel_size=4, max_model_len=max_seq_len),
         max_out_len=100,
-        max_seq_len=2048,
+        max_seq_len=max_seq_len,
         batch_size=32,
         generation_kwargs=dict(temperature=0),
         run_cfg=dict(num_gpus=4, num_procs=1),

configs/models/qwen/vllm_qwen1_5_72b_chat.py

@@ -7,16 +7,18 @@
         dict(role='BOT', begin='<|im_start|>assistant\n', end='<|im_end|>\n', generate=True),
     ],
 )
+max_seq_len = 2048
 
 models = [
     dict(
         type=VLLM,
         abbr='qwen1.5-72b-chat-vllm',
         path='Qwen/Qwen1.5-72B-Chat',
-        model_kwargs=dict(tensor_parallel_size=4),
+        # more vllm model_kwargs: https://github.com/vllm-project/vllm/blob/main/vllm/engine/arg_utils.py
+        model_kwargs=dict(tensor_parallel_size=4, max_model_len=max_seq_len),
         meta_template=_meta_template,
         max_out_len=100,
-        max_seq_len=2048,
+        max_seq_len=max_seq_len,
         batch_size=32,
         generation_kwargs=dict(temperature=0),
         stop_words=['<|im_end|>'],

configs/models/qwen/vllm_qwen_14b_chat.py

@@ -13,6 +13,7 @@
         type=VLLM,
         abbr='qwen-14b-chat-vllm',
         path='Qwen/Qwen-14B-Chat',
+        # more vllm model_kwargs: https://github.com/vllm-project/vllm/blob/main/vllm/engine/arg_utils.py
         model_kwargs=dict(tensor_parallel_size=4),
         meta_template=_meta_template,
         max_out_len=100,

configs/models/qwen/vllm_qwen_72b_chat.py

@@ -7,16 +7,18 @@
         dict(role='BOT', begin='\n<|im_start|>assistant\n', end='<|im_end|>', generate=True),
     ],
 )
+max_seq_len = 2048
 
 models = [
     dict(
         type=VLLM,
         abbr='qwen-72b-chat-vllm',
         path='Qwen/Qwen-72B-Chat',
-        model_kwargs=dict(tensor_parallel_size=4),
+        # more vllm model_kwargs: https://github.com/vllm-project/vllm/blob/main/vllm/engine/arg_utils.py
+        model_kwargs=dict(tensor_parallel_size=4, max_model_len=max_seq_len),
         meta_template=_meta_template,
         max_out_len=100,
-        max_seq_len=2048,
+        max_seq_len=max_seq_len,
         batch_size=32,
         generation_kwargs=dict(temperature=0),
         stop_words=['<|im_end|>'],

configs/models/wizardlm/vllm_wizardlm_70b_v1_0.py

@@ -7,16 +7,18 @@
         dict(role='BOT', begin='ASSISTANT: ', end='</s>', generate=True),
     ],
 )
+max_seq_len = 2048
 
 models = [
     dict(
         type=VLLM,
         abbr='wizardlm-70b-v1.0-vllm',
         path='WizardLM/WizardLM-70B-V1.0',
-        model_kwargs=dict(tensor_parallel_size=4),
+        # more vllm model_kwargs: https://github.com/vllm-project/vllm/blob/main/vllm/engine/arg_utils.py
+        model_kwargs=dict(tensor_parallel_size=4, max_model_len=max_seq_len),
         meta_template=_meta_template,
         max_out_len=100,
-        max_seq_len=2048,
+        max_seq_len=max_seq_len,
         batch_size=32,
         generation_kwargs=dict(temperature=0),
         stop_words=['</s>'],