External LLM endpoint usage improvements + fixes

helm-charts/chatqna/README.md

@@ -45,8 +45,10 @@ helm install chatqna chatqna --set global.HF_TOKEN=${HFTOKEN} --set global.model
 # To use AMD ROCm device with TGI
 #helm install chatqna chatqna --set global.HF_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} --set tgi.LLM_MODEL_ID=${MODELNAME} -f chatqna/rocm-tgi-values.yaml
 
-# To use with external OpenAI compatible LLM endpoint
-#helm install chatqna chatqna -f chatqna/variant_external-llm-values.yaml --set global.HF_TOKEN=${HFTOKEN} --set externalLLM.LLM_SERVER_HOST_IP="http://your-llm-server" --set externalLLM.LLM_MODEL="your-model" --set externalLLM.OPENAI_API_KEY="your-api-key"
+# To use with OPEA KubeAI LLM models installed to same cluster
+#helm install chatqna chatqna -f chatqna/variant_external-llm-values.yaml --set global.HF_TOKEN=${HFTOKEN} --set externalLLM.LLM_MODEL="your-model" --set externalLLM.LLM_SERVER_HOST="http://kubeai.kubeai/openai" --set externalLLM.LLM_SERVER_PORT="" --set externalLLM.OPENAI_API_KEY="your-api-key"
+# To use with other external OpenAI compatible LLM endpoints
+#helm install chatqna chatqna -f chatqna/variant_external-llm-values.yaml --set global.HF_TOKEN=${HFTOKEN} --set externalLLM.LLM_MODEL="your-model" --set externalLLM.LLM_SERVER_HOST="http://your-llm-server" --set externalLLM.LLM_SERVER_PORT="80" --set externalLLM.OPENAI_API_KEY="your-api-key"
 
 # To deploy FaqGen
 #helm install faqgen chatqna --set global.HF_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} -f chatqna/faqgen-cpu-values.yaml

helm-charts/chatqna/templates/deployment.yaml

@@ -45,7 +45,7 @@ spec:
               {{- else if .Values.tgi.enabled }}
               value: {{ .Release.Name }}-tgi
               {{- else if .Values.externalLLM.enabled }}
-              value: {{ .Values.externalLLM.LLM_SERVER_HOST_IP }}
+              value: {{ .Values.externalLLM.LLM_SERVER_HOST }}
               {{- else }}
               {{- fail "ChatQnA needs a LLM inference backend!" }}
               {{- end }}

helm-charts/chatqna/values.yaml

@@ -75,6 +75,7 @@ vllm:
   shmSize: 128Gi
   VLLM_TORCH_PROFILER_DIR: "/tmp/vllm_profile"
 data-prep:
+  # if set to false, need to set also nginx.enabled=false
   enabled: true
   # the following are for redis-vector-db
   DATAPREP_BACKEND: "REDIS"
@@ -162,6 +163,7 @@ nginx:
 
 # UI configuration
 chatqna-ui:
+  # if set to false, need to set also nginx.enabled=false
   enabled: true
   image:
     repository: opea/chatqna-ui
@@ -174,7 +176,8 @@ dashboard:
 # External LLM configuration
 externalLLM:
   enabled: false
-  LLM_SERVICE_HOST_IP: "http://your-llm-server"
+  LLM_SERVER_HOST: "http://your-llm-server"
+  LLM_SERVER_PORT: "80"
   LLM_MODEL: "your-model"
   OPENAI_API_KEY: "your-api-key"
 

helm-charts/chatqna/variant_external-llm-values.yaml

@@ -4,10 +4,10 @@
 # External LLM configuration override
 externalLLM:
   enabled: true  # Enable external LLM service
-  LLM_SERVER_HOST_IP: "http://your-llm-server"  # External LLM service host
+  LLM_SERVER_HOST: "http://your-llm-server"  # External LLM service host
+  LLM_SERVER_PORT: "80"  # Port for the external LLM service
   LLM_MODEL: "your-model"  # LLM model to use
   OPENAI_API_KEY: "your-api-key"  # OpenAI API key for authentication
-  LLM_SERVER_PORT: "80"  # Port for the external LLM service
 
 # Disable internal LLM services when using external LLM
 llm-uservice:
@@ -18,3 +18,6 @@ vllm:
 
 tgi:
   enabled: false
+
+ollama:
+  enabled: false

helm-charts/codegen/README.md

@@ -32,8 +32,11 @@ helm install codegen codegen --set global.HF_TOKEN=${HFTOKEN} --set global.model
 # helm install codegen codegen --set global.HF_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} --set llm-uservcie.LLM_MODEL_ID=${MODELNAME} --set tgi.LLM_MODEL_ID=${MODELNAME} -f codegen/gaudi-tgi-values.yaml
 # To use AMD ROCm device with vLLM
 # helm install codegen codegen --set global.HF_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} --set llm-uservcie.LLM_MODEL_ID=${MODELNAME} --set vllm.LLM_MODEL_ID=${MODELNAME} -f codegen/rocm-values.yaml
-# To use with external OpenAI compatible LLM endpoint
-# helm install codegen codegen -f codegen/variant_external-llm-values.yaml --set externalLLM.LLM_SERVER_HOST_IP="http://your-llm-server" --set externalLLM.LLM_MODEL="your-model" --set externalLLM.OPENAI_API_KEY="your-api-key"
+
+# To use with OPEA KubeAI LLM models installed to same cluster
+# helm install codegen codegen -f codegen/variant_external-llm-values.yaml --set global.HF_TOKEN=${HFTOKEN} --set externalLLM.LLM_MODEL="your-kubeai-model" --set externalLLM.LLM_SERVER_HOST="http://kubeai.kubeai/openai"  --set externalLLM.LLM_SERVER_PORT="" --set externalLLM.OPENAI_API_KEY=""
+# To use with other external OpenAI compatible LLM endpoints
+# helm install codegen codegen -f codegen/variant_external-llm-values.yaml --set global.HF_TOKEN=${HFTOKEN} --set externalLLM.LLM_MODEL="your-model" --set externalLLM.LLM_SERVER_HOST="http://your-llm-server"  --set externalLLM.LLM_SERVER_PORT="80" --set externalLLM.OPENAI_API_KEY="your-api-key"
 ```
 
 ### IMPORTANT NOTE

helm-charts/codegen/templates/deployment.yaml

@@ -36,13 +36,13 @@ spec:
           env:
             - name: LLM_SERVICE_HOST_IP
               {{- if .Values.externalLLM.enabled }}
-              value: {{ .Values.externalLLM.LLM_SERVICE_HOST_IP }}
+              value: {{ .Values.externalLLM.LLM_SERVER_HOST }}
               {{- else }}
               value: {{ include "llm-uservice.fullname" (index .Subcharts "llm-uservice") | quote }}
               {{- end }}
             - name: LLM_SERVICE_PORT
               {{- if .Values.externalLLM.enabled }}
-              value: {{ .Values.externalLLM.LLM_SERVICE_PORT | default "80" | quote }}
+              value: {{ .Values.externalLLM.LLM_SERVER_PORT | default "80" | quote }}
               {{- else }}
               value: {{ index .Values "llm-uservice" "service" "port" | quote }}
               {{- end }}

helm-charts/codegen/values.yaml

@@ -77,6 +77,7 @@ redis-vector-db:
   enabled: true
 
 data-prep:
+  # if set to false, need to set also nginx.enabled=false
   enabled: true
   # the following are for redis-vector-db
   DATAPREP_BACKEND: "REDIS"
@@ -94,6 +95,7 @@ embedding-usvc:
 nginx:
   enabled: false
 codegen-ui:
+  # if set to false, need to set also nginx.enabled=false
   enabled: true
   image:
     repository: opea/codegen-gradio-ui
@@ -119,7 +121,8 @@ codegen-ui:
 # External LLM configuration
 externalLLM:
   enabled: false
-  LLM_SERVICE_HOST_IP: "http://your-llm-server"
+  LLM_SERVER_HOST: "http://your-llm-server"
+  LLM_SERVER_PORT: "80"
   LLM_MODEL_ID: "your-model"
   OPENAI_API_KEY: "your-api-key"
 

helm-charts/codegen/variant_external-llm-values.yaml

@@ -4,10 +4,10 @@
 # External LLM configuration
 externalLLM:
   enabled: true # Enable external LLM service
-  LLM_SERVICE_HOST_IP: "http://your-llm-server" # External LLM service host
+  LLM_SERVER_HOST: "http://your-llm-server" # External LLM service host
+  LLM_SERVER_PORT: "80"  # Port for the external LLM service
   LLM_MODEL_ID: "your-model" # LLM model to use
   OPENAI_API_KEY: "your-api-key" # OpenAI API key for authentication
-  LLM_SERVER_PORT: "80"  # Port for the external LLM service
 
 # Disable internal LLM services when using external LLM
 tgi: