diff --git a/helm-charts/common/speecht5/templates/configmap.yaml b/helm-charts/common/speecht5/templates/configmap.yaml index bf68a1031..2dc305b36 100644 --- a/helm-charts/common/speecht5/templates/configmap.yaml +++ b/helm-charts/common/speecht5/templates/configmap.yaml @@ -18,7 +18,7 @@ data: HF_ENDPOINT: {{ .Values.global.HF_ENDPOINT | quote}} {{- end }} HUGGINGFACE_HUB_CACHE: "/data" - HF_TOKEN: {{ .Values.global.HUGGINGFACEHUB_API_TOKEN | quote}} + HF_TOKEN: {{ .Values.global.HUGGINGFACEHUB_API_TOKEN | default .Values.global.HF_TOKEN | quote }} {{- if .Values.global.offline }} HF_HUB_OFFLINE: "1" {{- end }} diff --git a/helm-charts/common/whisper/README.md b/helm-charts/common/whisper/README.md index 6c1215515..86e79e3a3 100644 --- a/helm-charts/common/whisper/README.md +++ b/helm-charts/common/whisper/README.md @@ -12,6 +12,37 @@ export ASR_MODEL_PATH="openai/whisper-small" helm install whisper whisper --set global.modelUseHostPath=${MODELDIR} --set ASR_MODEL_PATH=${ASR_MODEL_PATH} ``` +### Install the microservice in air gapped (offline) mode + +To run `whisper` microservice in an air gapped environment, users are required to pre-download the model `openai/whisper-small` to a shared storage. + +#### Use node-local directory + +Assuming the model data is shared using node-local directory `/mnt/opea-models`. + +``` +# On every K8s node, run the following command: +export MODEL_DIR=/mnt/opea-models +# Download model, assumes Python huggingface_hub[cli] module is already installed +huggingface-cli download --cache-dir "${MODEL_DIR}" openai/whisper-small + +# On K8s master node, run the following command: +# Install using Helm with the following additional parameters: +helm install ... --set global.offline=true,global.modelUseHostPath=${MODEL_DIR} +``` + +#### Use persistent volume + +Assuming we share the offline data on cluster level using a persistent volume (PV), first we need to create the persistent volume claim (PVC) with name `opea-model-pvc` to store the model data. + +``` +# Download model openai/whisper-small at the root directory of the corresponding PV +# ... +# Install using Helm with the following additional parameters: +# export MODEL_PVC=opea-model-pvc +# helm install ... --set global.offline=true,global.modelUsePVC=${MODEL_PVC} +``` + ## Verify Use port-forward to access it from localhost. @@ -26,7 +57,11 @@ curl http://localhost:1234/v1/asr \ ## Values -| Key | Type | Default | Description | -| ---------------- | ------ | ---------------- | ----------- | -| image.repository | string | `"opea/whisper"` | | -| service.port | string | `"7066"` | | +| Key | Type | Default | Description | +| ----------------------- | ------ | ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| image.repository | string | `"opea/whisper"` | | +| service.port | string | `"7066"` | | +| global.offline | bool | `false` | Whether to run the microservice in air gapped environment | +| global.modelUseHostPath | String | `""` | Cached models directory on Kubernetes node, service will not download if the model is cached here. The host path "modelUseHostPath" will be mounted to the container as /data directory. Setting this to null/empty will force the pod to download the model every time during startup. May not be set if `global.modelUsePVC` is also set. | +| global.modelUsePVC | String | `""` | Name of Persistent Volume Claim to use for model cache. The Persistent Volume will be mounted to the container as /data directory. Setting this to null/empty will force the pod to download the model every time during startup. May not be set if `global.modelUseHostPath` is also set. | +| global.HF_TOKEN | string | `insert-your-huggingface-token-here` | Hugging Face API token | diff --git a/helm-charts/common/whisper/templates/configmap.yaml b/helm-charts/common/whisper/templates/configmap.yaml index 7d87d80cb..ad5c763fa 100644 --- a/helm-charts/common/whisper/templates/configmap.yaml +++ b/helm-charts/common/whisper/templates/configmap.yaml @@ -19,3 +19,6 @@ data: {{- end }} HUGGINGFACE_HUB_CACHE: "/data" HF_TOKEN: {{ .Values.global.HUGGINGFACEHUB_API_TOKEN | default .Values.global.HF_TOKEN | quote }} + {{- if .Values.global.offline }} + HF_HUB_OFFLINE: "1" + {{- end }} diff --git a/helm-charts/common/whisper/templates/deployment.yaml b/helm-charts/common/whisper/templates/deployment.yaml index f8dafbb27..c7d227bad 100644 --- a/helm-charts/common/whisper/templates/deployment.yaml +++ b/helm-charts/common/whisper/templates/deployment.yaml @@ -28,7 +28,7 @@ spec: serviceAccountName: {{ include "whisper.serviceAccountName" . }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} - {{- if not (hasPrefix "/data/" .Values.ASR_MODEL_PATH) }} + {{- if and (not .Values.global.offline) (not (hasPrefix "/data/" .Values.ASR_MODEL_PATH)) }} initContainers: - name: model-downloader envFrom: @@ -116,8 +116,10 @@ spec: hostPath: path: {{ .Values.global.modelUseHostPath }} type: Directory - {{- else }} + {{- else if not .Values.global.offline }} emptyDir: {} + {{- else }} + {{- fail "'global.modelUsePVC' or 'global.modelUseHostPath' must be set in offline mode" }} {{- end }} - name: tmp emptyDir: {} diff --git a/helm-charts/common/whisper/values.yaml b/helm-charts/common/whisper/values.yaml index 5ead6ebf8..65297aa11 100644 --- a/helm-charts/common/whisper/values.yaml +++ b/helm-charts/common/whisper/values.yaml @@ -93,6 +93,9 @@ global: # If set, and serviceAccount.create is false, it will assume this service account is already created by others. sharedSAName: "" + # Running microservice in air gapped (offline) mode + # If offline is enabled, user must set either modelUseHostPath or modelUsePVC and download models `openai/whisper-small`. + offline: false # Choose where to save your downloaded models # Set modelUseHostPath for local directory, this is good for one node test. Example: # modelUseHostPath: /mnt/opea-models