External LLM endpoint usage improvements + fixes

eero-t · eero-t · commit 21c248c42efc · 2025-07-18T21:26:27.000+03:00
Improve/shorten/fix Helm variable names:
- Consistent Helm variable naming: LLM_SERVICE/LLM_SERVER -&gt; LLM_SERVER
- Drop misleading/redundant _IP suffix: LLM_SERVER_HOST_IP -&gt; LLM_SERVER_HOST

(Helm variables do not need to duplicate naming problems in OPEA applications.)

Add:
- Examples on using OPEA KubeAI installation as external LLM
- LLM_SERVER_PORT setting to everywhere LLM_SERVER_HOST is set
- Comment that NGIX needs to be disabled if UI/data-prep is disabled [1]
- Missing HF_TOKEN settings to external LLM usage examples
- Missing Ollama disabling

[2] Otherwise Helm fails to parse NGINX template file.  Helm would
    fail also on "if .Values.data-prep.enabled" in that file, due to
    '-' in the component name, so best seemed just to add comment
    about the dependency.

Signed-off-by: Eero Tamminen &lt;eero.t.tamminen@intel.com&gt;
diff --git a/helm-charts/chatqna/README.md b/helm-charts/chatqna/README.md
@@ -45,8 +45,10 @@ helm install chatqna chatqna --set global.HF_TOKEN=${HFTOKEN} --set global.model
 # To use AMD ROCm device with TGI
 #helm install chatqna chatqna --set global.HF_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} --set tgi.LLM_MODEL_ID=${MODELNAME} -f chatqna/rocm-tgi-values.yaml
 
-# To use with external OpenAI compatible LLM endpoint
-#helm install chatqna chatqna -f chatqna/variant_external-llm-values.yaml --set global.HF_TOKEN=${HFTOKEN} --set externalLLM.LLM_SERVER_HOST_IP="http://your-llm-server" --set externalLLM.LLM_MODEL="your-model" --set externalLLM.OPENAI_API_KEY="your-api-key"
+# To use with OPEA KubeAI LLM models installed to same cluster
+#helm install chatqna chatqna -f chatqna/variant_external-llm-values.yaml --set global.HF_TOKEN=${HFTOKEN} --set externalLLM.LLM_MODEL="your-model" --set externalLLM.LLM_SERVER_HOST="http://kubeai.kubeai/openai" --set externalLLM.LLM_SERVER_PORT="" --set externalLLM.OPENAI_API_KEY="your-api-key"
+# To use with other external OpenAI compatible LLM endpoints
+#helm install chatqna chatqna -f chatqna/variant_external-llm-values.yaml --set global.HF_TOKEN=${HFTOKEN} --set externalLLM.LLM_MODEL="your-model" --set externalLLM.LLM_SERVER_HOST="http://your-llm-server" --set externalLLM.LLM_SERVER_PORT="80" --set externalLLM.OPENAI_API_KEY="your-api-key"
 
 # To deploy FaqGen
 #helm install faqgen chatqna --set global.HF_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} -f chatqna/faqgen-cpu-values.yaml
diff --git a/helm-charts/chatqna/templates/deployment.yaml b/helm-charts/chatqna/templates/deployment.yaml
@@ -45,7 +45,7 @@ spec:
               {{- else if .Values.tgi.enabled }}
               value: {{ .Release.Name }}-tgi
               {{- else if .Values.externalLLM.enabled }}
-              value: {{ .Values.externalLLM.LLM_SERVER_HOST_IP }}
+              value: {{ .Values.externalLLM.LLM_SERVER_HOST }}
               {{- else }}
               {{- fail "ChatQnA needs a LLM inference backend!" }}
               {{- end }}
diff --git a/helm-charts/chatqna/values.yaml b/helm-charts/chatqna/values.yaml
@@ -75,6 +75,7 @@ vllm:
   shmSize: 128Gi
   VLLM_TORCH_PROFILER_DIR: "/tmp/vllm_profile"
 data-prep:
+  # if set to false, need to set also nginx.enabled=false
   enabled: true
   # the following are for redis-vector-db
   DATAPREP_BACKEND: "REDIS"
@@ -162,6 +163,7 @@ nginx:
 
 # UI configuration
 chatqna-ui:
+  # if set to false, need to set also nginx.enabled=false
   enabled: true
   image:
     repository: opea/chatqna-ui
@@ -174,7 +176,8 @@ dashboard:
 # External LLM configuration
 externalLLM:
   enabled: false
-  LLM_SERVICE_HOST_IP: "http://your-llm-server"
+  LLM_SERVER_HOST: "http://your-llm-server"
+  LLM_SERVER_PORT: "80"
   LLM_MODEL: "your-model"
   OPENAI_API_KEY: "your-api-key"
 
diff --git a/helm-charts/chatqna/variant_external-llm-values.yaml b/helm-charts/chatqna/variant_external-llm-values.yaml
@@ -4,10 +4,10 @@
 # External LLM configuration override
 externalLLM:
   enabled: true  # Enable external LLM service
-  LLM_SERVER_HOST_IP: "http://your-llm-server"  # External LLM service host
+  LLM_SERVER_HOST: "http://your-llm-server"  # External LLM service host
+  LLM_SERVER_PORT: "80"  # Port for the external LLM service
   LLM_MODEL: "your-model"  # LLM model to use
   OPENAI_API_KEY: "your-api-key"  # OpenAI API key for authentication
-  LLM_SERVER_PORT: "80"  # Port for the external LLM service
 
 # Disable internal LLM services when using external LLM
 llm-uservice:
@@ -18,3 +18,6 @@ vllm:
 
 tgi:
   enabled: false
+
+ollama:
+  enabled: false
diff --git a/helm-charts/codegen/README.md b/helm-charts/codegen/README.md
@@ -32,8 +32,11 @@ helm install codegen codegen --set global.HF_TOKEN=${HFTOKEN} --set global.model
 # helm install codegen codegen --set global.HF_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} --set llm-uservcie.LLM_MODEL_ID=${MODELNAME} --set tgi.LLM_MODEL_ID=${MODELNAME} -f codegen/gaudi-tgi-values.yaml
 # To use AMD ROCm device with vLLM
 # helm install codegen codegen --set global.HF_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} --set llm-uservcie.LLM_MODEL_ID=${MODELNAME} --set vllm.LLM_MODEL_ID=${MODELNAME} -f codegen/rocm-values.yaml
-# To use with external OpenAI compatible LLM endpoint
-# helm install codegen codegen -f codegen/variant_external-llm-values.yaml --set externalLLM.LLM_SERVER_HOST_IP="http://your-llm-server" --set externalLLM.LLM_MODEL="your-model" --set externalLLM.OPENAI_API_KEY="your-api-key"
+
+# To use with OPEA KubeAI LLM models installed to same cluster
+# helm install codegen codegen -f codegen/variant_external-llm-values.yaml --set global.HF_TOKEN=${HFTOKEN} --set externalLLM.LLM_MODEL="your-kubeai-model" --set externalLLM.LLM_SERVER_HOST="http://kubeai.kubeai/openai"  --set externalLLM.LLM_SERVER_PORT="" --set externalLLM.OPENAI_API_KEY=""
+# To use with other external OpenAI compatible LLM endpoints
+# helm install codegen codegen -f codegen/variant_external-llm-values.yaml --set global.HF_TOKEN=${HFTOKEN} --set externalLLM.LLM_MODEL="your-model" --set externalLLM.LLM_SERVER_HOST="http://your-llm-server"  --set externalLLM.LLM_SERVER_PORT="80" --set externalLLM.OPENAI_API_KEY="your-api-key"
 ```
 
 ### IMPORTANT NOTE
diff --git a/helm-charts/codegen/templates/deployment.yaml b/helm-charts/codegen/templates/deployment.yaml
@@ -36,13 +36,13 @@ spec:
           env:
             - name: LLM_SERVICE_HOST_IP
               {{- if .Values.externalLLM.enabled }}
-              value: {{ .Values.externalLLM.LLM_SERVICE_HOST_IP }}
+              value: {{ .Values.externalLLM.LLM_SERVER_HOST }}
               {{- else }}
               value: {{ include "llm-uservice.fullname" (index .Subcharts "llm-uservice") | quote }}
               {{- end }}
             - name: LLM_SERVICE_PORT
               {{- if .Values.externalLLM.enabled }}
-              value: {{ .Values.externalLLM.LLM_SERVICE_PORT | default "80" | quote }}
+              value: {{ .Values.externalLLM.LLM_SERVER_PORT | default "80" | quote }}
               {{- else }}
               value: {{ index .Values "llm-uservice" "service" "port" | quote }}
               {{- end }}
diff --git a/helm-charts/codegen/values.yaml b/helm-charts/codegen/values.yaml
@@ -77,6 +77,7 @@ redis-vector-db:
   enabled: true
 
 data-prep:
+  # if set to false, need to set also nginx.enabled=false
   enabled: true
   # the following are for redis-vector-db
   DATAPREP_BACKEND: "REDIS"
@@ -94,6 +95,7 @@ embedding-usvc:
 nginx:
   enabled: false
 codegen-ui:
+  # if set to false, need to set also nginx.enabled=false
   enabled: true
   image:
     repository: opea/codegen-gradio-ui
@@ -119,7 +121,8 @@ codegen-ui:
 # External LLM configuration
 externalLLM:
   enabled: false
-  LLM_SERVICE_HOST_IP: "http://your-llm-server"
+  LLM_SERVER_HOST: "http://your-llm-server"
+  LLM_SERVER_PORT: "80"
   LLM_MODEL_ID: "your-model"
   OPENAI_API_KEY: "your-api-key"
 
diff --git a/helm-charts/codegen/variant_external-llm-values.yaml b/helm-charts/codegen/variant_external-llm-values.yaml
@@ -4,10 +4,10 @@
 # External LLM configuration
 externalLLM:
   enabled: true # Enable external LLM service
-  LLM_SERVICE_HOST_IP: "http://your-llm-server" # External LLM service host
+  LLM_SERVER_HOST: "http://your-llm-server" # External LLM service host
+  LLM_SERVER_PORT: "80"  # Port for the external LLM service
   LLM_MODEL_ID: "your-model" # LLM model to use
   OPENAI_API_KEY: "your-api-key" # OpenAI API key for authentication
-  LLM_SERVER_PORT: "80"  # Port for the external LLM service
 
 # Disable internal LLM services when using external LLM
 tgi:
