diff --git a/helm-charts/common/llm-uservice/templates/configmap.yaml b/helm-charts/common/llm-uservice/templates/configmap.yaml index fc5f0ea8f..4e8f9d068 100644 --- a/helm-charts/common/llm-uservice/templates/configmap.yaml +++ b/helm-charts/common/llm-uservice/templates/configmap.yaml @@ -70,6 +70,9 @@ data: {{- if .Values.LLM_MODEL_ID }} LLM_MODEL_ID: {{ .Values.LLM_MODEL_ID | quote }} {{- end }} + {{- if .Values.OPENAI_API_KEY }} + OPENAI_API_KEY: {{ .Values.OPENAI_API_KEY | quote }} + {{- end }} HF_HOME: "/tmp/.cache/huggingface" {{- if not .Values.global.offline }} HF_TOKEN: {{ .Values.global.HUGGINGFACEHUB_API_TOKEN | default .Values.global.HF_TOKEN | quote }} diff --git a/helm-charts/docsum/Chart.yaml b/helm-charts/docsum/Chart.yaml index 4d02035f8..c7527baca 100644 --- a/helm-charts/docsum/Chart.yaml +++ b/helm-charts/docsum/Chart.yaml @@ -21,13 +21,16 @@ dependencies: - name: llm-uservice version: 0-latest repository: "file://../common/llm-uservice" + condition: llm-uservice.enabled - name: whisper version: 0-latest repository: "file://../common/whisper" + condition: whisper.enabled - name: ui version: 0-latest repository: "file://../common/ui" alias: docsum-ui + condition: docsum-ui.enabled - name: nginx version: 0-latest repository: "file://../common/nginx" diff --git a/helm-charts/docsum/README.md b/helm-charts/docsum/README.md index cd03cdd4d..90d9cf908 100644 --- a/helm-charts/docsum/README.md +++ b/helm-charts/docsum/README.md @@ -26,7 +26,12 @@ helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --set global.modelUs # helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --values docsum/rocm-values.yaml # To use AMD ROCm device with TGI # helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --values docsum/rocm-tgi-values.yaml - +# To use with external OpenAI-compatible LLM endpoints (OpenAI, vLLM, TGI, etc.) +# This configures the llm-uservice to connect to external LLM providers while maintaining DocSum compatibility +# For OpenAI: +# helm install docsum docsum --values docsum/variant_external-llm-values.yaml --set llm-uservice.OPENAI_API_KEY="your-api-key" --set llm-uservice.LLM_ENDPOINT="https://api.openai.com" --set llm-uservice.LLM_MODEL_ID="gpt-4-turbo" +# For vLLM/TGI or other OpenAI-compatible endpoints: +# helm install docsum docsum --values docsum/variant_external-llm-values.yaml --set llm-uservice.LLM_ENDPOINT="http://your-server-url" --set llm-uservice.LLM_MODEL_ID="your-model" ``` ## Verify diff --git a/helm-charts/docsum/templates/deployment.yaml b/helm-charts/docsum/templates/deployment.yaml index 486820715..deb1fb4a9 100644 --- a/helm-charts/docsum/templates/deployment.yaml +++ b/helm-charts/docsum/templates/deployment.yaml @@ -38,10 +38,12 @@ spec: value: {{ include "llm-uservice.fullname" (index .Subcharts "llm-uservice") }} - name: LLM_SERVICE_PORT value: {{ index .Values "llm-uservice" "service" "port" | quote }} + {{- if .Values.whisper.enabled }} - name: ASR_SERVICE_HOST_IP value: {{ include "whisper.fullname" (index .Subcharts "whisper") }} - name: ASR_SERVICE_PORT value: {{ index .Values "whisper" "service" "port" | quote }} + {{- end }} securityContext: {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" diff --git a/helm-charts/docsum/values.yaml b/helm-charts/docsum/values.yaml index 9cca28e4c..7a0b9273a 100644 --- a/helm-charts/docsum/values.yaml +++ b/helm-charts/docsum/values.yaml @@ -57,7 +57,14 @@ tolerations: [] affinity: {} # To override values in subchart llm-uservice +# For external LLM configuration, you can override these values here or use variant_external-llm-values.yaml +# Example for external OpenAI-compatible endpoints: +# DOCSUM_BACKEND: "vLLM" # Use vLLM backend for OpenAI-compatible APIs +# LLM_ENDPOINT: "https://api.openai.com" # External endpoint (omit /v1 suffix) +# LLM_MODEL_ID: "gpt-4-turbo" # Model to use +# OPENAI_API_KEY: "your-api-key" # API key for authentication llm-uservice: + enabled: true image: repository: opea/llm-docsum DOCSUM_BACKEND: "vLLM" @@ -79,6 +86,8 @@ vllm: nginx: enabled: false docsum-ui: + # if false, set also nginx.enabled=false + enabled: true image: repository: opea/docsum-gradio-ui tag: "latest" @@ -101,8 +110,12 @@ docsum-ui: # type: ClusterIP dashboard: + enabled: true prefix: "OPEA DocSum" +whisper: + enabled: true + global: http_proxy: "" https_proxy: "" diff --git a/helm-charts/docsum/variant_external-llm-values.yaml b/helm-charts/docsum/variant_external-llm-values.yaml new file mode 100644 index 000000000..34a4f99a3 --- /dev/null +++ b/helm-charts/docsum/variant_external-llm-values.yaml @@ -0,0 +1,17 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# External LLM configuration - configures llm-uservice to use external LLM providers +# This keeps the llm-uservice wrapper (required for /v1/docsum endpoint) but connects it to external LLMs +llm-uservice: + enabled: true # Keep the wrapper service for DocSum compatibility + DOCSUM_BACKEND: "vLLM" # Use vLLM backend for OpenAI-compatible APIs + LLM_ENDPOINT: "https://api.openai.com" # External LLM API endpoint (omit /v1 suffix) + OPENAI_API_KEY: "${OPENAI_API_KEY}" # API key for authentication + LLM_MODEL_ID: "gpt-4-turbo" # Model to use + +# Disable local inference services since we're using external LLMs +vllm: + enabled: false +tgi: + enabled: false