diff --git a/helm-charts/chatqna/README.md b/helm-charts/chatqna/README.md index b23a8ddd4..dfa59074e 100644 --- a/helm-charts/chatqna/README.md +++ b/helm-charts/chatqna/README.md @@ -45,8 +45,10 @@ helm install chatqna chatqna --set global.HF_TOKEN=${HFTOKEN} --set global.model # To use AMD ROCm device with TGI #helm install chatqna chatqna --set global.HF_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} --set tgi.LLM_MODEL_ID=${MODELNAME} -f chatqna/rocm-tgi-values.yaml -# To use with external OpenAI compatible LLM endpoint -#helm install chatqna chatqna -f chatqna/variant_external-llm-values.yaml --set global.HF_TOKEN=${HFTOKEN} --set externalLLM.LLM_SERVER_HOST_IP="http://your-llm-server" --set externalLLM.LLM_MODEL="your-model" --set externalLLM.OPENAI_API_KEY="your-api-key" +# To use with OPEA KubeAI LLM models installed to same cluster +#helm install chatqna chatqna -f chatqna/variant_external-llm-values.yaml --set global.HF_TOKEN=${HFTOKEN} --set externalLLM.LLM_MODEL="your-model" --set externalLLM.LLM_SERVER_HOST="http://kubeai.kubeai/openai" --set externalLLM.LLM_SERVER_PORT="" --set externalLLM.OPENAI_API_KEY="your-api-key" +# To use with other external OpenAI compatible LLM endpoints +#helm install chatqna chatqna -f chatqna/variant_external-llm-values.yaml --set global.HF_TOKEN=${HFTOKEN} --set externalLLM.LLM_MODEL="your-model" --set externalLLM.LLM_SERVER_HOST="http://your-llm-server" --set externalLLM.LLM_SERVER_PORT="80" --set externalLLM.OPENAI_API_KEY="your-api-key" # To deploy FaqGen #helm install faqgen chatqna --set global.HF_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} -f chatqna/faqgen-cpu-values.yaml diff --git a/helm-charts/chatqna/templates/deployment.yaml b/helm-charts/chatqna/templates/deployment.yaml index 97b94904a..a0387788c 100644 --- a/helm-charts/chatqna/templates/deployment.yaml +++ b/helm-charts/chatqna/templates/deployment.yaml @@ -45,7 +45,7 @@ spec: {{- else if .Values.tgi.enabled }} value: {{ .Release.Name }}-tgi {{- else if .Values.externalLLM.enabled }} - value: {{ .Values.externalLLM.LLM_SERVER_HOST_IP }} + value: {{ .Values.externalLLM.LLM_SERVER_HOST }} {{- else }} {{- fail "ChatQnA needs a LLM inference backend!" }} {{- end }} diff --git a/helm-charts/chatqna/values.yaml b/helm-charts/chatqna/values.yaml index bd6fd0fda..5c6086431 100644 --- a/helm-charts/chatqna/values.yaml +++ b/helm-charts/chatqna/values.yaml @@ -75,6 +75,7 @@ vllm: shmSize: 128Gi VLLM_TORCH_PROFILER_DIR: "/tmp/vllm_profile" data-prep: + # if set to false, need to set also nginx.enabled=false enabled: true # the following are for redis-vector-db DATAPREP_BACKEND: "REDIS" @@ -162,6 +163,7 @@ nginx: # UI configuration chatqna-ui: + # if set to false, need to set also nginx.enabled=false enabled: true image: repository: opea/chatqna-ui @@ -174,7 +176,8 @@ dashboard: # External LLM configuration externalLLM: enabled: false - LLM_SERVICE_HOST_IP: "http://your-llm-server" + LLM_SERVER_HOST: "http://your-llm-server" + LLM_SERVER_PORT: "80" LLM_MODEL: "your-model" OPENAI_API_KEY: "your-api-key" diff --git a/helm-charts/chatqna/variant_external-llm-values.yaml b/helm-charts/chatqna/variant_external-llm-values.yaml index 79f996382..ee1be6ed9 100644 --- a/helm-charts/chatqna/variant_external-llm-values.yaml +++ b/helm-charts/chatqna/variant_external-llm-values.yaml @@ -4,10 +4,10 @@ # External LLM configuration override externalLLM: enabled: true # Enable external LLM service - LLM_SERVER_HOST_IP: "http://your-llm-server" # External LLM service host + LLM_SERVER_HOST: "http://your-llm-server" # External LLM service host + LLM_SERVER_PORT: "80" # Port for the external LLM service LLM_MODEL: "your-model" # LLM model to use OPENAI_API_KEY: "your-api-key" # OpenAI API key for authentication - LLM_SERVER_PORT: "80" # Port for the external LLM service # Disable internal LLM services when using external LLM llm-uservice: @@ -18,3 +18,6 @@ vllm: tgi: enabled: false + +ollama: + enabled: false diff --git a/helm-charts/codegen/README.md b/helm-charts/codegen/README.md index 64e5b2f6f..5ec73a013 100644 --- a/helm-charts/codegen/README.md +++ b/helm-charts/codegen/README.md @@ -32,8 +32,11 @@ helm install codegen codegen --set global.HF_TOKEN=${HFTOKEN} --set global.model # helm install codegen codegen --set global.HF_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} --set llm-uservcie.LLM_MODEL_ID=${MODELNAME} --set tgi.LLM_MODEL_ID=${MODELNAME} -f codegen/gaudi-tgi-values.yaml # To use AMD ROCm device with vLLM # helm install codegen codegen --set global.HF_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} --set llm-uservcie.LLM_MODEL_ID=${MODELNAME} --set vllm.LLM_MODEL_ID=${MODELNAME} -f codegen/rocm-values.yaml -# To use with external OpenAI compatible LLM endpoint -# helm install codegen codegen -f codegen/variant_external-llm-values.yaml --set externalLLM.LLM_SERVER_HOST_IP="http://your-llm-server" --set externalLLM.LLM_MODEL="your-model" --set externalLLM.OPENAI_API_KEY="your-api-key" + +# To use with OPEA KubeAI LLM models installed to same cluster +# helm install codegen codegen -f codegen/variant_external-llm-values.yaml --set global.HF_TOKEN=${HFTOKEN} --set externalLLM.LLM_MODEL="your-kubeai-model" --set externalLLM.LLM_SERVER_HOST="http://kubeai.kubeai/openai" --set externalLLM.LLM_SERVER_PORT="" --set externalLLM.OPENAI_API_KEY="" +# To use with other external OpenAI compatible LLM endpoints +# helm install codegen codegen -f codegen/variant_external-llm-values.yaml --set global.HF_TOKEN=${HFTOKEN} --set externalLLM.LLM_MODEL="your-model" --set externalLLM.LLM_SERVER_HOST="http://your-llm-server" --set externalLLM.LLM_SERVER_PORT="80" --set externalLLM.OPENAI_API_KEY="your-api-key" ``` ### IMPORTANT NOTE diff --git a/helm-charts/codegen/templates/deployment.yaml b/helm-charts/codegen/templates/deployment.yaml index 881d632e8..07ad884d0 100644 --- a/helm-charts/codegen/templates/deployment.yaml +++ b/helm-charts/codegen/templates/deployment.yaml @@ -36,13 +36,13 @@ spec: env: - name: LLM_SERVICE_HOST_IP {{- if .Values.externalLLM.enabled }} - value: {{ .Values.externalLLM.LLM_SERVICE_HOST_IP }} + value: {{ .Values.externalLLM.LLM_SERVER_HOST }} {{- else }} value: {{ include "llm-uservice.fullname" (index .Subcharts "llm-uservice") | quote }} {{- end }} - name: LLM_SERVICE_PORT {{- if .Values.externalLLM.enabled }} - value: {{ .Values.externalLLM.LLM_SERVICE_PORT | default "80" | quote }} + value: {{ .Values.externalLLM.LLM_SERVER_PORT | default "80" | quote }} {{- else }} value: {{ index .Values "llm-uservice" "service" "port" | quote }} {{- end }} diff --git a/helm-charts/codegen/values.yaml b/helm-charts/codegen/values.yaml index 75ece38d1..315f59cd8 100644 --- a/helm-charts/codegen/values.yaml +++ b/helm-charts/codegen/values.yaml @@ -77,6 +77,7 @@ redis-vector-db: enabled: true data-prep: + # if set to false, need to set also nginx.enabled=false enabled: true # the following are for redis-vector-db DATAPREP_BACKEND: "REDIS" @@ -94,6 +95,7 @@ embedding-usvc: nginx: enabled: false codegen-ui: + # if set to false, need to set also nginx.enabled=false enabled: true image: repository: opea/codegen-gradio-ui @@ -119,7 +121,8 @@ codegen-ui: # External LLM configuration externalLLM: enabled: false - LLM_SERVICE_HOST_IP: "http://your-llm-server" + LLM_SERVER_HOST: "http://your-llm-server" + LLM_SERVER_PORT: "80" LLM_MODEL_ID: "your-model" OPENAI_API_KEY: "your-api-key" diff --git a/helm-charts/codegen/variant_external-llm-values.yaml b/helm-charts/codegen/variant_external-llm-values.yaml index 190f21e38..904ff002b 100644 --- a/helm-charts/codegen/variant_external-llm-values.yaml +++ b/helm-charts/codegen/variant_external-llm-values.yaml @@ -4,10 +4,10 @@ # External LLM configuration externalLLM: enabled: true # Enable external LLM service - LLM_SERVICE_HOST_IP: "http://your-llm-server" # External LLM service host + LLM_SERVER_HOST: "http://your-llm-server" # External LLM service host + LLM_SERVER_PORT: "80" # Port for the external LLM service LLM_MODEL_ID: "your-model" # LLM model to use OPENAI_API_KEY: "your-api-key" # OpenAI API key for authentication - LLM_SERVER_PORT: "80" # Port for the external LLM service # Disable internal LLM services when using external LLM tgi: