Generalize ChatQnA external LLM to external inferencing support

helm-charts/chatqna/templates/deployment.yaml

@@ -65,26 +65,47 @@ spec:
               {{- else if .Values.externalLLM.enabled }}
               value: {{ .Values.externalLLM.LLM_MODEL }}
               {{- end }}
-            {{- if .Values.externalLLM.enabled }}
+            {{- if .Values.externalAuthorization.needed }}
             - name: OPENAI_API_KEY
-              value: {{ .Values.externalLLM.OPENAI_API_KEY }}
+              value: {{ .Values.externalAuthorization.OPENAI_API_KEY }}
             {{- end }}
+            {{- if .Values.externalRerank.enabled }}
+            - name: RERANK_SERVER_HOST_IP
+              value: {{ .Values.externalRerank.RERANK_SERVER_HOST }}
+            - name: RERANK_SERVER_PORT
+              value: {{ .Values.externalRerank.RERANK_SERVER_PORT | default "80" | quote }}
+            {{- else }}
             - name: RERANK_SERVER_HOST_IP
               value: {{ .Release.Name }}-teirerank
             - name: RERANK_SERVER_PORT
               value: "80"
+            {{- end }}
             - name: RETRIEVER_SERVICE_HOST_IP
               value: {{ .Release.Name }}-retriever-usvc
             - name: RETRIEVER_SERVICE_PORT
               value: "7000"
+            {{- if .Values.externalEmbed.enabled }}
+            - name: EMBEDDING_SERVER_HOST_IP
+              value: {{ .Values.externalEmbed.EMBEDDING_SERVER_HOST }}
+            - name: EMBEDDING_SERVER_PORT
+              value: {{ .Values.externalEmbed.EMBEDDING_SERVER_PORT | default "80" | quote }}
+            {{- else }}
             - name: EMBEDDING_SERVER_HOST_IP
               value: {{ .Release.Name }}-tei
             - name: EMBEDDING_SERVER_PORT
               value: "80"
+            {{- end }}
+            {{- if .Values.externalGuardrails.enabled }}
+            - name: GUARDRAIL_SERVICE_HOST_IP
+              value: {{ .Values.externalGuardrails.GUARDRAILS_SERVER_HOST }}
+            - name: GUARDRAIL_SERVICE_PORT
+              value: {{ .Values.externalGuardrails.GUARDRAILS_SERVER_PORT | default "80" | quote }}
+            {{- else }}
             - name: GUARDRAIL_SERVICE_HOST_IP
               value: {{ .Release.Name }}-guardrails-usvc
             - name: GUARDRAIL_SERVICE_PORT
               value: "9090"
+            {{- end }}
             - name: LLM_SERVICE_HOST_IP
               value: {{ .Release.Name }}-llm-uservice
             - name: LLM_SERVICE_PORT

helm-charts/chatqna/values.yaml

@@ -173,7 +173,31 @@ chatqna-ui:
 dashboard:
   prefix: "OPEA ChatQnA"
 
-# External LLM configuration
+# External inferencing configurations
+
+externalAuthorization:
+  needed: false
+  # common authorization for all the external inferencing services
+  OPENAI_API_KEY: "your-api-key"
+
+externalEmbed:
+  enabled: false
+  EMBEDDING_SERVER_HOST: "http://your-embed-server"
+  EMBEDDING_SERVER_PORT: "80"
+  EMBEDDING_MODEL_ID: "your-embed-model"
+
+externalRerank:
+  enabled: false
+  RERANK_SERVER_HOST: "http://your-rerank-server"
+  RERANK_SERVER_PORT: "80"
+  RERANK_MODEL_ID: "your-rerank-model"
+
+externalGuardrails:
+  enabled: false
+  GUARDRAIL_SERVER_HOST: "http://your-guardrails-server"
+  GUARDRAIL_SERVER_PORT: "80"
+  GUARDRAIL_MODEL_ID: "your-guardrails-model"
+
 externalLLM:
   enabled: false
   LLM_SERVER_HOST: "http://your-llm-server"

helm-charts/chatqna/variant_external-inferencing-values.yaml

@@ -0,0 +1,73 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+externalAuthorization:
+  needed: true
+  # common authorization key for all the external inferencing services
+  OPENAI_API_KEY: "your-api-key"
+
+
+externalEmbed:
+  enabled: true
+  # External embedding service details
+  EMBEDDING_SERVER_HOST: "http://your-embed-server"
+  EMBEDDING_SERVER_PORT: "80"
+  # TODO: ChatQnA does not support specifying model in embed queries
+  EMBEDDING_MODEL_ID: "your-embed-model"
+
+# Disable internal embedding services when using external embedding
+embedding-usvc:
+  enabled: false
+tei:
+  enabled :false
+
+
+externalRerank:
+  enabled: true
+  # External reranking service details
+  RERANK_SERVER_HOST: "http://your-rerank-server"
+  RERANK_SERVER_PORT: "80"
+  # TODO: ChatQnA does not support specifying model in rerank queries
+  RERANK_MODEL_ID: "your-rerank-model"
+
+# Disable internal reranking services when using external reranking
+reranking-usvc:
+  enabled: false
+teirerank:
+  enabled :false
+
+
+externalGuardrails:
+  enabled: true
+  # External guardrails service details
+  GUARDRAIL_SERVER_HOST: "http://your-guardrails-server"
+  GUARDRAIL_SERVER_PORT: "80"
+  # TODO: ChatQnA does not support specifying model in guardrail queries
+  GUARDRAIL_MODEL_ID: "your-guardrail-model"
+
+# Disable internal guardrail services when using external guardrails
+guardrails-usvc:
+  # TODO: does guardrails work without the OPEA specific wrapper?
+  enabled :false
+vllm-guardrails:
+  enabled :false
+
+
+# External LLM configuration override
+externalLLM:
+  enabled: true
+  # External LLM service details
+  LLM_SERVER_HOST: "http://your-llm-server"
+  LLM_SERVER_PORT: "80"
+  LLM_MODEL: "your-llm-model"
+
+# Disable internal LLM services when using external LLM
+llm-uservice:
+  # TODO: does FaqGen variat work without the OPEA specific wrapper?
+  enabled: false
+vllm:
+  enabled: false
+tgi:
+  enabled: false
+ollama:
+  enabled: false