Update vLLM parameter max-seq-len-to-capture (opea-project#1809)

Signed-off-by: lvliang-intel <[email protected]>

Author: lvliang-intel

AgentQnA/kubernetes/helm/cpu-values.yaml

@@ -6,7 +6,7 @@ tgi:
 vllm:
   enabled: true
   LLM_MODEL_ID: "meta-llama/Meta-Llama-3-8B-Instruct"
-  extraCmdArgs: ["--max-seq_len-to-capture", "16384", "--enable-auto-tool-choice", "--tool-call-parser", "llama3_json"]
+  extraCmdArgs: ["--max-seq-len-to-capture", "16384", "--enable-auto-tool-choice", "--tool-call-parser", "llama3_json"]
 
 supervisor:
   llm_endpoint_url: http://{{ .Release.Name }}-vllm

AgentQnA/kubernetes/helm/gaudi-values.yaml

@@ -19,7 +19,7 @@ vllm:
   PT_HPU_ENABLE_LAZY_COLLECTIVES: true
   VLLM_SKIP_WARMUP: true
   shmSize: 16Gi
-  extraCmdArgs: ["--tensor-parallel-size", "4", "--max-seq_len-to-capture", "16384", "--enable-auto-tool-choice", "--tool-call-parser", "llama3_json"]
+  extraCmdArgs: ["--tensor-parallel-size", "4", "--max-seq-len-to-capture", "16384", "--enable-auto-tool-choice", "--tool-call-parser", "llama3_json"]
 
 supervisor:
   llm_endpoint_url: http://{{ .Release.Name }}-vllm

AudioQnA/docker_compose/intel/hpu/gaudi/compose.yaml

@@ -62,7 +62,7 @@ services:
     cap_add:
       - SYS_NICE
     ipc: host
-    command: --model ${LLM_MODEL_ID} --tensor-parallel-size ${NUM_CARDS} --host 0.0.0.0 --port 80 --block-size ${BLOCK_SIZE} --max-num-seqs ${MAX_NUM_SEQS} --max-seq_len-to-capture ${MAX_SEQ_LEN_TO_CAPTURE}
+    command: --model ${LLM_MODEL_ID} --tensor-parallel-size ${NUM_CARDS} --host 0.0.0.0 --port 80 --block-size ${BLOCK_SIZE} --max-num-seqs ${MAX_NUM_SEQS} --max-seq-len-to-capture ${MAX_SEQ_LEN_TO_CAPTURE}
   audioqna-gaudi-backend-server:
     image: ${REGISTRY:-opea}/audioqna:${TAG:-latest}
     container_name: audioqna-gaudi-backend-server

AudioQnA/kubernetes/helm/gaudi-values.yaml

@@ -21,7 +21,7 @@ vllm:
     "--tensor-parallel-size", "1",
     "--block-size", "128",
     "--max-num-seqs", "256",
-    "--max-seq_len-to-capture", "2048"
+    "--max-seq-len-to-capture", "2048"
   ]
 
 whisper:

ChatQnA/docker_compose/intel/hpu/gaudi/compose.yaml

@@ -110,7 +110,7 @@ services:
     cap_add:
       - SYS_NICE
     ipc: host
-    command: --model ${LLM_MODEL_ID} --tensor-parallel-size ${NUM_CARDS} --host 0.0.0.0 --port 80 --block-size 128 --max-num-seqs 256 --max-seq_len-to-capture 2048
+    command: --model ${LLM_MODEL_ID} --tensor-parallel-size ${NUM_CARDS} --host 0.0.0.0 --port 80 --block-size 128 --max-num-seqs 256 --max-seq-len-to-capture 2048
   chatqna-gaudi-backend-server:
     image: ${REGISTRY:-opea}/chatqna:${TAG:-latest}
     container_name: chatqna-gaudi-backend-server

ChatQnA/docker_compose/intel/hpu/gaudi/compose_faqgen.yaml

@@ -108,7 +108,7 @@ services:
     cap_add:
       - SYS_NICE
     ipc: host
-    command: --model ${LLM_MODEL_ID} --tensor-parallel-size ${NUM_CARDS} --host 0.0.0.0 --port 80 --block-size 128 --max-num-seqs 256 --max-seq_len-to-capture 2048
+    command: --model ${LLM_MODEL_ID} --tensor-parallel-size ${NUM_CARDS} --host 0.0.0.0 --port 80 --block-size 128 --max-num-seqs 256 --max-seq-len-to-capture 2048
   llm-faqgen:
     image: ${REGISTRY:-opea}/llm-faqgen:${TAG:-latest}
     container_name: llm-faqgen-server

ChatQnA/docker_compose/intel/hpu/gaudi/compose_guardrails.yaml

@@ -139,7 +139,7 @@ services:
     cap_add:
       - SYS_NICE
     ipc: host
-    command: --model ${LLM_MODEL_ID} --tensor-parallel-size ${NUM_CARDS} --host 0.0.0.0 --port 80 --block-size 128 --max-num-seqs 256 --max-seq_len-to-capture 2048
+    command: --model ${LLM_MODEL_ID} --tensor-parallel-size ${NUM_CARDS} --host 0.0.0.0 --port 80 --block-size 128 --max-num-seqs 256 --max-seq-len-to-capture 2048
   chatqna-gaudi-backend-server:
     image: ${REGISTRY:-opea}/chatqna:${TAG:-latest}
     container_name: chatqna-gaudi-guardrails-server

ChatQnA/docker_compose/intel/hpu/gaudi/compose_without_rerank.yaml

@@ -79,7 +79,7 @@ services:
     cap_add:
       - SYS_NICE
     ipc: host
-    command: --model ${LLM_MODEL_ID} --tensor-parallel-size ${NUM_CARDS} --host 0.0.0.0 --port 80 --block-size 128 --max-num-seqs 256 --max-seq_len-to-capture 2048
+    command: --model ${LLM_MODEL_ID} --tensor-parallel-size ${NUM_CARDS} --host 0.0.0.0 --port 80 --block-size 128 --max-num-seqs 256 --max-seq-len-to-capture 2048
   chatqna-gaudi-backend-server:
     image: ${REGISTRY:-opea}/chatqna:${TAG:-latest}
     container_name: chatqna-gaudi-backend-server

ChatQnA/kubernetes/helm/faqgen-gaudi-values.yaml

@@ -30,7 +30,7 @@ vllm:
     "--tensor-parallel-size", "1",
     "--block-size", "128",
     "--max-num-seqs", "256",
-    "--max-seq_len-to-capture", "2048"
+    "--max-seq-len-to-capture", "2048"
   ]
 
 # Reranking: second largest bottleneck when reranking is in use

ChatQnA/kubernetes/helm/gaudi-values.yaml

@@ -25,7 +25,7 @@ vllm:
     "--tensor-parallel-size", "1",
     "--block-size", "128",
     "--max-num-seqs", "256",
-    "--max-seq_len-to-capture", "2048"
+    "--max-seq-len-to-capture", "2048"
   ]
 
 # Reranking: second largest bottleneck when reranking is in use