ChatQnA - Adding files for deploy application on ROCm vLLM and ROCm TGI with Helm

helm-charts/chatqna/README.md

@@ -43,11 +43,19 @@ helm install chatqna chatqna --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --
 
 # To use CPU with vLLM with Qdrant DB
 #helm install chatqna chatqna --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} --set vllm.LLM_MODEL_ID=${MODELNAME} -f chatqna/cpu-qdrant-values.yaml
-# To use CPU with vLLM with Milvus DB
-#helm install chatqna chatqna --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} --set vllm.LLM_MODEL_ID=${MODELNAME} -f chatqna/cpu-milvus-values.yaml
+# To use AMD ROCm device with vLLM
+#helm install chatqna chatqna --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} --set vllm.LLM_MODEL_ID=${MODELNAME} -f chatqna/rocm-values.yaml
+# To use AMD ROCm device with TGI
+#helm install chatqna chatqna --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} --set vllm.LLM_MODEL_ID=${MODELNAME} -f chatqna/rocm-tgi-values.yaml
 
 # To deploy FaqGen
 #helm install faqgen chatqna --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} -f chatqna/faqgen-cpu-values.yaml
+
+# To deploy FaqGen based application on AMD ROCm device with vLLM
+#helm install chatqna chatqna --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} --set vllm.LLM_MODEL_ID=${MODELNAME} -f chatqna/faqgen-rocm-values.yaml
+# To deploy FaqGen based application on AMD ROCm device with TGI
+#helm install chatqna chatqna --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} --set vllm.LLM_MODEL_ID=${MODELNAME} -f chatqna/faqgen-rocm-tgi-values.yaml
+
 ```
 
 ### IMPORTANT NOTE

helm-charts/chatqna/faqgen-rocm-tgi-values.yaml

@@ -0,0 +1,50 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+CHATQNA_TYPE: "CHATQNA_FAQGEN"
+llm-uservice:
+  enabled: true
+  image:
+    repository: opea/llm-faqgen
+  LLM_MODEL_ID: meta-llama/Meta-Llama-3-8B-Instruct
+  FAQGEN_BACKEND: "TGI"
+  service:
+    port: 80
+tgi:
+  enabled: true
+  accelDevice: "rocm"
+  image:
+    repository: ghcr.io/huggingface/text-generation-inference
+    tag: "2.4.1-rocm"
+  MAX_INPUT_LENGTH: "2048"
+  MAX_TOTAL_TOKENS: "4096"
+  USE_FLASH_ATTENTION: "false"
+  FLASH_ATTENTION_RECOMPUTE: "false"
+  HIP_VISIBLE_DEVICES: "0"
+  MAX_BATCH_SIZE: "4"
+  extraCmdArgs: [ "--num-shard","1" ]
+  resources:
+    limits:
+      amd.com/gpu: "1"
+    requests:
+      cpu: 1
+      memory: 16Gi
+  securityContext:
+    readOnlyRootFilesystem: true
+    runAsNonRoot: true
+    runAsUser: 1000
+    capabilities:
+      add:
+        - SYS_PTRACE
+  readinessProbe:
+    initialDelaySeconds: 60
+    periodSeconds: 5
+    timeoutSeconds: 1
+    failureThreshold: 120
+  startupProbe:
+    initialDelaySeconds: 60
+    periodSeconds: 5
+    timeoutSeconds: 1
+    failureThreshold: 120
+vllm:
+  enabled: false

helm-charts/chatqna/faqgen-rocm-values.yaml

@@ -0,0 +1,45 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+CHATQNA_TYPE: "CHATQNA_FAQGEN"
+llm-uservice:
+  enabled: true
+  image:
+    repository: opea/llm-faqgen
+  LLM_MODEL_ID: meta-llama/Meta-Llama-3-8B-Instruct
+  FAQGEN_BACKEND: "vLLM"
+  service:
+    port: 80
+tgi:
+  enabled: false
+vllm:
+  enabled: true
+  accelDevice: "rocm"
+  image:
+    repository: opea/vllm-rocm
+    tag: latest
+  env:
+    HIP_VISIBLE_DEVICES: "0"
+    TENSOR_PARALLEL_SIZE: "1"
+    HF_HUB_DISABLE_PROGRESS_BARS: "1"
+    HF_HUB_ENABLE_HF_TRANSFER: "0"
+    VLLM_USE_TRITON_FLASH_ATTN: "0"
+    VLLM_WORKER_MULTIPROC_METHOD: "spawn"
+    PYTORCH_JIT: "0"
+    HF_HOME: "/data"
+  extraCmd:
+    command: [ "python3", "/workspace/api_server.py" ]
+  extraCmdArgs: [ "--swap-space", "16",
+                  "--disable-log-requests",
+                  "--dtype", "float16",
+                  "--num-scheduler-steps", "1",
+                  "--distributed-executor-backend", "mp" ]
+  resources:
+    limits:
+      amd.com/gpu: "1"
+  startupProbe:
+    failureThreshold: 180
+  securityContext:
+    readOnlyRootFilesystem: false
+    runAsNonRoot: false
+    runAsUser: 0

helm-charts/chatqna/rocm-tgi-values.yaml

@@ -0,0 +1,51 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# Accelerate inferencing in heaviest components to improve performance
+# by overriding their subchart values
+
+tgi:
+  enabled: true
+  accelDevice: "rocm"
+  image:
+    repository: ghcr.io/huggingface/text-generation-inference
+    tag: "2.4.1-rocm"
+  MAX_INPUT_LENGTH: "1024"
+  MAX_TOTAL_TOKENS: "2048"
+  USE_FLASH_ATTENTION: "false"
+  FLASH_ATTENTION_RECOMPUTE: "false"
+  HIP_VISIBLE_DEVICES: "0"
+  MAX_BATCH_SIZE: "4"
+  extraCmdArgs: [ "--num-shard","1" ]
+  resources:
+    limits:
+      amd.com/gpu: "1"
+    requests:
+      cpu: 1
+      memory: 16Gi
+  securityContext:
+    readOnlyRootFilesystem: true
+    allowPrivilegeEscalation: false
+    runAsNonRoot: true
+    runAsUser: 1000
+    capabilities:
+      add:
+        - SYS_PTRACE
+      drop:
+        - ALL
+    seccompProfile:
+      type: RuntimeDefault
+    runAsGroup: 0
+  readinessProbe:
+    initialDelaySeconds: 60
+    periodSeconds: 5
+    timeoutSeconds: 1
+    failureThreshold: 120
+  startupProbe:
+    initialDelaySeconds: 60
+    periodSeconds: 5
+    timeoutSeconds: 1
+    failureThreshold: 120
+
+vllm:
+  enabled: false

helm-charts/chatqna/rocm-values.yaml

@@ -0,0 +1,39 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# Accelerate inferencing in heaviest components to improve performance
+# by overriding their subchart values
+
+tgi:
+  enabled: false
+vllm:
+  enabled: true
+  accelDevice: "rocm"
+  image:
+    repository: opea/vllm-rocm
+    tag: latest
+  env:
+    HIP_VISIBLE_DEVICES: "0"
+    TENSOR_PARALLEL_SIZE: "1"
+    HF_HUB_DISABLE_PROGRESS_BARS: "1"
+    HF_HUB_ENABLE_HF_TRANSFER: "0"
+    VLLM_USE_TRITON_FLASH_ATTN: "0"
+    VLLM_WORKER_MULTIPROC_METHOD: "spawn"
+    PYTORCH_JIT: "0"
+    HF_HOME: "/data"
+  extraCmd:
+    command: [ "python3", "/workspace/api_server.py" ]
+  extraCmdArgs: [ "--swap-space", "16",
+                  "--disable-log-requests",
+                  "--dtype", "float16",
+                  "--num-scheduler-steps", "1",
+                  "--distributed-executor-backend", "mp" ]
+  resources:
+    limits:
+      amd.com/gpu: "1"
+  startupProbe:
+    failureThreshold: 180
+  securityContext:
+    readOnlyRootFilesystem: false
+    runAsNonRoot: false
+    runAsUser: 0

helm-charts/common/tgi/rocm-values.yaml

@@ -2,25 +2,25 @@
 
 accelDevice: "rocm"
 image:
-  repository: ghcr.io/huggingface/text-generation-inference
-  tag: "2.4.1-rocm"
+  repository: opea/tgi-rocm
+  tag: latest
 MAX_INPUT_LENGTH: "1024"
 MAX_TOTAL_TOKENS: "2048"
 USE_FLASH_ATTENTION: "false"
 FLASH_ATTENTION_RECOMPUTE: "false"
 HIP_VISIBLE_DEVICES: "0"
 MAX_BATCH_SIZE: "4"
 extraCmdArgs: ["--num-shard","1"]
+tokenizerMountPath: /usr/src/out
 resources:
   limits:
     amd.com/gpu: "1"
   requests:
     cpu: 1
     memory: 16Gi
 securityContext:
-  readOnlyRootFilesystem: false
-  runAsNonRoot: false
-  runAsUser: 0
+  runAsUser: 2000
+  runAsGroup: 2000
   capabilities:
     add:
       - SYS_PTRACE

helm-charts/common/tgi/templates/deployment.yaml

@@ -98,7 +98,7 @@ spec:
               name: shm
             - mountPath: /tmp
               name: tmp
-            - mountPath: /usr/src/out
+            - mountPath: {{ .Values.tokenizerMountPath | default /usr/src/out }}
               name: tokenizer
           ports:
             - name: http

helm-charts/common/tgi/values.yaml

@@ -107,6 +107,8 @@ startupProbe:
 #   periodSeconds: 5
 #   failureThreshold: 120
 
+tokenizerMountPath: /usr/src/out
+
 nodeSelector: {}
 
 tolerations: []