diff --git a/sample-workflows/basic_llm_workflow.json b/sample-workflows/basic_llm_workflow.json index 0f59c7e..d55e4c3 100644 --- a/sample-workflows/basic_llm_workflow.json +++ b/sample-workflows/basic_llm_workflow.json @@ -105,34 +105,8 @@ { "label": "Model Name", "name": "modelName", - "type": "options", + "type": "string", "default": "Intel/neural-chat-7b-v3-3", - "options": [ - { - "name": "Intel/neural-chat-7b-v3-3", - "label": "Intel/neural-chat-7b-v3-3" - }, - { - "name": "Llama-2-7b-chat-hf", - "label": "Llama-2-7b-chat-hf" - }, - { - "name": "Llama-2-70b-chat-hf", - "label": "Llama-2-70b-chat-hf" - }, - { - "name": "Meta-Llama-3-8B-Instruct", - "label": "Meta-Llama-3-8B-Instruct" - }, - { - "name": "Meta-Llama-3-70B-Instruct", - "label": "Meta-Llama-3-70B-Instruct" - }, - { - "name": "Phi-3", - "label": "Phi-3" - } - ], "id": "opea_service@llm_tgi_0-input-modelName-options" }, { diff --git a/sample-workflows/sample_workflow_chatqna.json b/sample-workflows/sample_workflow_chatqna.json index d7b4d74..7247488 100644 --- a/sample-workflows/sample_workflow_chatqna.json +++ b/sample-workflows/sample_workflow_chatqna.json @@ -361,34 +361,8 @@ { "label": "Model Name", "name": "modelName", - "type": "options", + "type": "string", "default": "Intel/neural-chat-7b-v3-3", - "options": [ - { - "name": "Intel/neural-chat-7b-v3-3", - "label": "Intel/neural-chat-7b-v3-3" - }, - { - "name": "Llama-2-7b-chat-hf", - "label": "Llama-2-7b-chat-hf" - }, - { - "name": "Llama-2-70b-chat-hf", - "label": "Llama-2-70b-chat-hf" - }, - { - "name": "Meta-Llama-3-8B-Instruct", - "label": "Meta-Llama-3-8B-Instruct" - }, - { - "name": "Meta-Llama-3-70B-Instruct", - "label": "Meta-Llama-3-70B-Instruct" - }, - { - "name": "Phi-3", - "label": "Phi-3" - } - ], "id": "opea_service@llm_tgi_0-input-modelName-options" }, { diff --git a/studio-backend/app/services/exporter_service.py b/studio-backend/app/services/exporter_service.py index 5d6bf44..1b2c637 100644 --- a/studio-backend/app/services/exporter_service.py +++ b/studio-backend/app/services/exporter_service.py @@ -38,6 +38,11 @@ def convert_proj_info_to_manifest(proj_info_json, output_file=None): if 'app-nginx' in metadata['name'] and output_file is None: continue print(f"Processing {metadata['name']} service") + + # Remove HF_TOKEN if hugging_face_token is 'NA' + if doc.get('kind') == 'ConfigMap' and 'data' in doc: + if 'HF_TOKEN' in doc['data'] and doc['data']['HF_TOKEN'] == 'NA': + del doc['data']['HF_TOKEN'] # Accumulate the YAML document in the manifest string manifest_string += '---\n' @@ -81,6 +86,10 @@ def convert_proj_info_to_compose(proj_info_json, output_file=None): combined_services = {} for doc, service_name in (output_compose): + # Remove HF_TOKEN if hugging_face_token is 'NA' + for _, service_data in doc.items(): + if 'environment' in service_data and 'HF_TOKEN' in service_data['environment'] and service_data['environment']['HF_TOKEN'] == 'NA': + del service_data['environment']['HF_TOKEN'] combined_services.update(doc) services_data = { diff --git a/studio-backend/app/templates/microsvc-composes/tei.yaml b/studio-backend/app/templates/microsvc-composes/tei.yaml index cb239ad..ee3cde0 100644 --- a/studio-backend/app/templates/microsvc-composes/tei.yaml +++ b/studio-backend/app/templates/microsvc-composes/tei.yaml @@ -10,6 +10,7 @@ no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} + HF_TOKEN: "{{huggingFaceToken}}" entrypoint: /bin/sh -c "apt-get update && apt-get install -y curl && text-embeddings-router --json-output --model-id {{modelName}} --auto-truncate" healthcheck: test: ["CMD", "curl", "-f", "http://localhost:80/health"] diff --git a/studio-backend/app/templates/microsvc-composes/tgi.yaml b/studio-backend/app/templates/microsvc-composes/tgi.yaml index 1f31a28..08b176e 100644 --- a/studio-backend/app/templates/microsvc-composes/tgi.yaml +++ b/studio-backend/app/templates/microsvc-composes/tgi.yaml @@ -1,5 +1,5 @@ "{{endpoint}}": - image: ghcr.io/huggingface/text-generation-inference:sha-e4201f4-intel-cpu + image: ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu container_name: "{{endpoint}}" ports: - "{{port}}:80" diff --git a/studio-backend/app/templates/microsvc-manifests/tgi.yaml b/studio-backend/app/templates/microsvc-manifests/tgi.yaml index b0b5b73..9bd3a61 100644 --- a/studio-backend/app/templates/microsvc-manifests/tgi.yaml +++ b/studio-backend/app/templates/microsvc-manifests/tgi.yaml @@ -18,6 +18,8 @@ data: NUMBA_CACHE_DIR: "/tmp" HF_HOME: "/tmp/.cache/huggingface" CUDA_GRAPHS: "0" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + HF_HUB_ENABLE_HF_TRANSFER: "0" --- # Source: tgi/templates/service.yaml # Copyright (C) 2024 Intel Corporation @@ -65,7 +67,7 @@ spec: name: config-{endpoint} securityContext: {} - image: "ghcr.io/huggingface/text-generation-inference:sha-e4201f4-intel-cpu" + image: "ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu" imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /data diff --git a/studio-backend/app/utils/namespace_utils.py b/studio-backend/app/utils/namespace_utils.py index fad3437..275f452 100644 --- a/studio-backend/app/utils/namespace_utils.py +++ b/studio-backend/app/utils/namespace_utils.py @@ -1,7 +1,7 @@ import time from kubernetes.client.rest import ApiException -def wait_for_pod(namespace, pod_name, core_v1_api, timeout=300): +def wait_for_pod(namespace, pod_name, core_v1_api, timeout=600): start_time = time.time() while time.time() - start_time < timeout: try: diff --git a/studio-backend/tests/exporter-groundtruth/gt_app-compose.yaml b/studio-backend/tests/exporter-groundtruth/gt_app-compose.yaml index f74ef89..51c8bd8 100644 --- a/studio-backend/tests/exporter-groundtruth/gt_app-compose.yaml +++ b/studio-backend/tests/exporter-groundtruth/gt_app-compose.yaml @@ -22,7 +22,7 @@ services: https_proxy: ${https_proxy} command: --model-id BAAI/bge-large-en-v1.5 --auto-truncate tgi-0: - image: ghcr.io/huggingface/text-generation-inference:sha-e4201f4-intel-cpu + image: ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu container_name: tgi-0 ports: - 3081:80 diff --git a/studio-backend/tests/exporter-groundtruth/gt_app-manifest-with-nginx.yaml b/studio-backend/tests/exporter-groundtruth/gt_app-manifest-with-nginx.yaml index 1a2b814..609f743 100644 --- a/studio-backend/tests/exporter-groundtruth/gt_app-manifest-with-nginx.yaml +++ b/studio-backend/tests/exporter-groundtruth/gt_app-manifest-with-nginx.yaml @@ -248,7 +248,7 @@ spec: - configMapRef: name: config-tgi-0 securityContext: {} - image: ghcr.io/huggingface/text-generation-inference:sha-e4201f4-intel-cpu + image: ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /data diff --git a/studio-backend/tests/exporter-groundtruth/gt_app-manifest.yaml b/studio-backend/tests/exporter-groundtruth/gt_app-manifest.yaml index 6ba7b1d..6298b31 100644 --- a/studio-backend/tests/exporter-groundtruth/gt_app-manifest.yaml +++ b/studio-backend/tests/exporter-groundtruth/gt_app-manifest.yaml @@ -249,7 +249,7 @@ spec: - configMapRef: name: config-tgi-0 securityContext: {} - image: ghcr.io/huggingface/text-generation-inference:sha-e4201f4-intel-cpu + image: ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /data diff --git a/studio-backend/tests/flowise-pipeline-translator/flowise_pipeline.json b/studio-backend/tests/flowise-pipeline-translator/flowise_pipeline.json index df3f43a..0980fe6 100644 --- a/studio-backend/tests/flowise-pipeline-translator/flowise_pipeline.json +++ b/studio-backend/tests/flowise-pipeline-translator/flowise_pipeline.json @@ -1 +1 @@ -{"id":"ec1105d1-69ad-41de-9c8b-737bc5975397","name":"ChatQnA Flow 4","flowData":{"nodes":[{"name":"chat_input","version":1,"category":"Controls","inMegaservice":false,"inputAnchors":[],"inputParams":[],"inputs":{},"outputs":{},"outputAnchors":[{"id":"chat_input_0-output-chat_input-ChatCompletionRequest","name":"chat_input","label":"ChatCompletionRequest","description":"User Input from Chat Window","type":"ChatCompletionRequest"}],"id":"chat_input_0"},{"name":"doc_input","version":1,"category":"Controls","inMegaservice":false,"inputAnchors":[],"inputParams":[],"inputs":{},"outputs":{},"outputAnchors":[{"id":"doc_input_0-output-doc_input-UploadFile","name":"doc_input","label":"UploadFile","description":"User Input from Document Upload","type":"UploadFile"}],"id":"doc_input_0"},{"name":"opea_service@prepare_doc_redis_prep","version":1,"category":"Data Preparation","inMegaservice":false,"dependent_services":{"tei":{"modelName":"","huggingFaceToken":""}},"inputs":{"doc_input":"{{doc_input_0.data.instance}}","modelName":"BAAI/bge-large-en-v1.5","huggingFaceToken":""},"inputAnchors":[{"label":"Documents","name":"doc_input","type":"UploadFile","id":"opea_service@prepare_doc_redis_prep_0-input-doc_input-UploadFile"}],"inputParams":[{"label":"Model Name","name":"modelName","type":"string","default":"BAAI/bge-large-en-v1.5","id":"opea_service@prepare_doc_redis_prep_0-input-modelName-string"},{"label":"HuggingFace Token","name":"huggingFaceToken","type":"password","optional":true,"id":"opea_service@prepare_doc_redis_prep_0-input-huggingFaceToken-password"}],"outputs":{},"outputAnchors":[{"id":"opea_service@prepare_doc_redis_prep_0-output-opea_service@prepare_doc_redis_prep-EmbedDoc","name":"opea_service@prepare_doc_redis_prep","label":"EmbedDoc","description":"Data Preparation with redis using Langchain","type":"EmbedDoc"}],"id":"opea_service@prepare_doc_redis_prep_0"},{"name":"opea_service@embedding_tei_langchain","version":1,"category":"Embeddings","inMegaservice":true,"dependent_services":{"tei":{"modelName":"","huggingFaceToken":""}},"inputs":{"textToEmbed":"{{chat_input_0.data.instance}}","modelName":"BAAI/bge-large-en-v1.5","huggingFaceToken":""},"inputAnchors":[{"label":"Text To Embed","name":"textToEmbed","type":"TextDoc|EmbeddingRequest|ChatCompletionRequest","id":"opea_service@embedding_tei_langchain_0-input-textToEmbed-TextDoc|EmbeddingRequest|ChatCompletionRequest"}],"inputParams":[{"label":"Model Name","name":"modelName","type":"string","default":"BAAI/bge-large-en-v1.5","id":"opea_service@embedding_tei_langchain_0-input-modelName-string"},{"label":"HuggingFace Token","name":"huggingFaceToken","type":"password","optional":true,"id":"opea_service@embedding_tei_langchain_0-input-huggingFaceToken-password"}],"outputs":{},"outputAnchors":[{"id":"opea_service@embedding_tei_langchain_0-output-opea_service@embedding_tei_langchain-EmbedDoc|EmbeddingResponse|ChatCompletionRequest","name":"opea_service@embedding_tei_langchain","label":"EmbedDoc","description":"Text Embedding Inference using Langchain","type":"EmbedDoc | EmbeddingResponse | ChatCompletionRequest"}],"id":"opea_service@embedding_tei_langchain_0"},{"name":"opea_service@retriever_redis","version":1,"category":"Retreiver","inMegaservice":true,"dependent_services":{"tei":{"modelName":"","huggingFaceToken":""}},"inputs":{"text":"{{opea_service@embedding_tei_langchain_0.data.instance}}","vector_db":"{{redis_vector_store_0.data.instance}}","modelName":"BAAI/bge-base-en-v1.5","huggingFaceToken":"","search_type":"similarity"},"inputAnchors":[{"label":"Search Query","name":"text","type":"EmbedDoc|RetrievalRequest|ChatCompletionRequest","id":"opea_service@retriever_redis_0-input-text-EmbedDoc|RetrievalRequest|ChatCompletionRequest"},{"label":"Redis Vector Store","name":"vector_db","type":"EmbedDoc","id":"opea_service@retriever_redis_0-input-vector_db-EmbedDoc"}],"inputParams":[{"label":"Model Name","name":"modelName","type":"string","default":"BAAI/bge-base-en-v1.5","id":"opea_service@retriever_redis_0-input-modelName-string"},{"label":"HuggingFace Token","name":"huggingFaceToken","type":"password","optional":true,"id":"opea_service@retriever_redis_0-input-huggingFaceToken-password"},{"label":"Search Type","name":"search_type","type":"options","default":"similarity","options":[{"name":"similarity","label":"similarity"},{"name":"similarity_distance_threshold","label":"similarity_distance_threshold"},{"name":"similarity_score_threshold","label":"similarity_score_threshold"},{"name":"mmr","label":"mmr"}],"optional":true,"additionalParams":true,"inferenceParams":true,"id":"opea_service@retriever_redis_0-input-search_type-options"}],"outputs":{},"outputAnchors":[{"id":"opea_service@retriever_redis_0-output-opea_service@retriever_redis-SearchedDoc|RetrievalResponse|ChatCompletionRequest","name":"opea_service@retriever_redis","label":"SearchedDoc","description":"Redis Retreiver with Langchain","type":"SearchedDoc | RetrievalResponse | ChatCompletionRequest"}],"id":"opea_service@retriever_redis_0"},{"name":"redis_vector_store","version":1,"category":"VectorStores","inMegaservice":true,"inputs":{"prepared_doc":"{{opea_service@prepare_doc_redis_prep_0.data.instance}}"},"inputAnchors":[{"label":"Prepared Documents","name":"prepared_doc","type":"EmbedDoc","id":"redis_vector_store_0-input-prepared_doc-EmbedDoc"}],"inputParams":[],"outputs":{},"outputAnchors":[{"id":"redis_vector_store_0-output-redis_vector_store-EmbedDoc","name":"redis_vector_store","label":"EmbedDoc","description":"Redis Vector Store","type":"EmbedDoc"}],"id":"redis_vector_store_0"},{"name":"opea_service@reranking_tei","version":1,"category":"Reranking","inMegaservice":true,"dependent_services":{"tei":{"modelName":"","huggingFaceToken":""}},"inputs":{"retreived_docs":"{{opea_service@retriever_redis_0.data.instance}}","modelName":"BAAI/bge-reranker-base","huggingFaceToken":"","top_n":1},"inputAnchors":[{"label":"Documents","name":"retreived_docs","type":"SearchedDocs|RerankingRequest|ChatCompletionRequest","id":"opea_service@reranking_tei_0-input-retreived_docs-SearchedDocs|RerankingRequest|ChatCompletionRequest"}],"inputParams":[{"label":"Model Name","name":"modelName","type":"string","default":"BAAI/bge-reranker-base","id":"opea_service@reranking_tei_0-input-modelName-string"},{"label":"HuggingFace Token","name":"huggingFaceToken","type":"password","optional":true,"id":"opea_service@reranking_tei_0-input-huggingFaceToken-password"},{"label":"Top N","name":"top_n","type":"number","default":1,"optional":true,"additionalParams":true,"inferenceParams":true,"id":"opea_service@reranking_tei_0-input-top_n-number"}],"outputs":{},"outputAnchors":[{"id":"opea_service@reranking_tei_0-output-opea_service@reranking_tei-LLMParamsDoc|RerankingResponse|ChatCompletionRequest","name":"opea_service@reranking_tei","label":"LLMParamsDoc","description":"TEI Reranking","type":"LLMParamsDoc | RerankingResponse | ChatCompletionRequest"}],"id":"opea_service@reranking_tei_0"},{"name":"opea_service@llm_tgi","version":1,"category":"LLM","inMegaservice":true,"dependent_services":{"tgi":{"modelName":"","huggingFaceToken":"NA"}},"inputs":{"text":"{{opea_service@reranking_tei_0.data.instance}}","modelName":"Intel/neural-chat-7b-v3-3","huggingFaceToken":"","max_tokens":17,"top_k":10,"top_p":0.95,"typical_p":0.95,"temperature":0.01,"presence_penalty":1.03,"frequency_penalty":"","streaming":true,"chat_template":"### You are a helpful, respectful and honest assistant to help the user with questions.\n### Context: {context}\n### Question: {question}\n### Answer:"},"inputAnchors":[{"label":"LLM Params Document","name":"text","type":"LLMParamsDoc|ChatCompletionRequest|SearchedDoc","id":"opea_service@llm_tgi_0-input-text-LLMParamsDoc|ChatCompletionRequest|SearchedDoc"}],"inputParams":[{"label":"Model Name","name":"modelName","type":"options","default":"Intel/neural-chat-7b-v3-3","options":[{"name":"Intel/neural-chat-7b-v3-3","label":"Intel/neural-chat-7b-v3-3"},{"name":"Llama-2-7b-chat-hf","label":"Llama-2-7b-chat-hf"},{"name":"Llama-2-70b-chat-hf","label":"Llama-2-70b-chat-hf"},{"name":"Meta-Llama-3-8B-Instruct","label":"Meta-Llama-3-8B-Instruct"},{"name":"Meta-Llama-3-70B-Instruct","label":"Meta-Llama-3-70B-Instruct"},{"name":"Phi-3","label":"Phi-3"}],"id":"opea_service@llm_tgi_0-input-modelName-options"},{"label":"HuggingFace Token","name":"huggingFaceToken","type":"password","optional":true,"id":"opea_service@llm_tgi_0-input-huggingFaceToken-password"},{"label":"Maximum Tokens","name":"max_tokens","type":"number","default":17,"optional":true,"additionalParams":true,"id":"opea_service@llm_tgi_0-input-max_tokens-number"},{"label":"Top K","name":"top_k","type":"number","default":10,"optional":true,"additionalParams":true,"id":"opea_service@llm_tgi_0-input-top_k-number"},{"label":"Top P","name":"top_p","type":"number","default":0.95,"optional":true,"additionalParams":true,"id":"opea_service@llm_tgi_0-input-top_p-number"},{"label":"Typical P","name":"typical_p","type":"number","default":0.95,"optional":true,"additionalParams":true,"id":"opea_service@llm_tgi_0-input-typical_p-number"},{"label":"Temperature","name":"temperature","type":"number","default":0.01,"optional":true,"additionalParams":true,"id":"opea_service@llm_tgi_0-input-temperature-number"},{"label":"Presence Penalty","name":"presence_penalty","type":"number","default":1.03,"optional":true,"additionalParams":true,"id":"opea_service@llm_tgi_0-input-presence_penalty-number"},{"label":"Frequency Penalty","name":"frequency_penalty","type":"number","default":0,"optional":true,"additionalParams":true,"id":"opea_service@llm_tgi_0-input-frequency_penalty-number"},{"label":"Streaming","name":"streaming","type":"boolean","default":true,"optional":true,"additionalParams":true,"id":"opea_service@llm_tgi_0-input-streaming-boolean"},{"label":"Chat Template","name":"chat_template","type":"string","rows":true,"default":"### You are a helpful, respectful and honest assistant to help the user with questions.\n### Context: {context}\n### Question: {question}\n### Answer:","optional":true,"additionalParams":true,"id":"opea_service@llm_tgi_0-input-chat_template-string"}],"outputs":{},"outputAnchors":[{"id":"opea_service@llm_tgi_0-output-opea_service@llm_tgi-GeneratedDoc|StreamingResponse|ChatCompletion","name":"opea_service@llm_tgi","label":"GeneratedDoc","description":"LLM Text Generation Inference","type":"GeneratedDoc | StreamingResponse | ChatCompletion"}],"id":"opea_service@llm_tgi_0"},{"name":"chat_completion","version":1,"category":"Controls","inMegaservice":false,"inputs":{"llm_response":"{{opea_service@llm_tgi_0.data.instance}}"},"hideOutput":true,"inputAnchors":[{"label":"LLM Response","name":"llm_response","type":"ChatCompletion","id":"chat_completion_0-input-llm_response-ChatCompletion"}],"inputParams":[],"outputs":{},"outputAnchors":[],"id":"chat_completion_0"}]},"deployed":false,"isPublic":false,"apikeyid":null,"chatbotConfig":null,"apiConfig":null,"analytic":null,"speechToText":null,"category":null,"type":"OPEA","sandboxStatus":null,"createdDate":"2024-10-07T16:29:51.000Z","updatedDate":"2024-10-07T16:32:21.000Z"} \ No newline at end of file +{"id":"ec1105d1-69ad-41de-9c8b-737bc5975397","name":"ChatQnA Flow 4","flowData":{"nodes":[{"name":"chat_input","version":1,"category":"Controls","inMegaservice":false,"inputAnchors":[],"inputParams":[],"inputs":{},"outputs":{},"outputAnchors":[{"id":"chat_input_0-output-chat_input-ChatCompletionRequest","name":"chat_input","label":"ChatCompletionRequest","description":"User Input from Chat Window","type":"ChatCompletionRequest"}],"id":"chat_input_0"},{"name":"doc_input","version":1,"category":"Controls","inMegaservice":false,"inputAnchors":[],"inputParams":[],"inputs":{},"outputs":{},"outputAnchors":[{"id":"doc_input_0-output-doc_input-UploadFile","name":"doc_input","label":"UploadFile","description":"User Input from Document Upload","type":"UploadFile"}],"id":"doc_input_0"},{"name":"opea_service@prepare_doc_redis_prep","version":1,"category":"Data Preparation","inMegaservice":false,"dependent_services":{"tei":{"modelName":"","huggingFaceToken":""}},"inputs":{"doc_input":"{{doc_input_0.data.instance}}","modelName":"BAAI/bge-large-en-v1.5","huggingFaceToken":""},"inputAnchors":[{"label":"Documents","name":"doc_input","type":"UploadFile","id":"opea_service@prepare_doc_redis_prep_0-input-doc_input-UploadFile"}],"inputParams":[{"label":"Model Name","name":"modelName","type":"string","default":"BAAI/bge-large-en-v1.5","id":"opea_service@prepare_doc_redis_prep_0-input-modelName-string"},{"label":"HuggingFace Token","name":"huggingFaceToken","type":"password","optional":true,"id":"opea_service@prepare_doc_redis_prep_0-input-huggingFaceToken-password"}],"outputs":{},"outputAnchors":[{"id":"opea_service@prepare_doc_redis_prep_0-output-opea_service@prepare_doc_redis_prep-EmbedDoc","name":"opea_service@prepare_doc_redis_prep","label":"EmbedDoc","description":"Data Preparation with redis using Langchain","type":"EmbedDoc"}],"id":"opea_service@prepare_doc_redis_prep_0"},{"name":"opea_service@embedding_tei_langchain","version":1,"category":"Embeddings","inMegaservice":true,"dependent_services":{"tei":{"modelName":"","huggingFaceToken":""}},"inputs":{"textToEmbed":"{{chat_input_0.data.instance}}","modelName":"BAAI/bge-large-en-v1.5","huggingFaceToken":""},"inputAnchors":[{"label":"Text To Embed","name":"textToEmbed","type":"TextDoc|EmbeddingRequest|ChatCompletionRequest","id":"opea_service@embedding_tei_langchain_0-input-textToEmbed-TextDoc|EmbeddingRequest|ChatCompletionRequest"}],"inputParams":[{"label":"Model Name","name":"modelName","type":"string","default":"BAAI/bge-large-en-v1.5","id":"opea_service@embedding_tei_langchain_0-input-modelName-string"},{"label":"HuggingFace Token","name":"huggingFaceToken","type":"password","optional":true,"id":"opea_service@embedding_tei_langchain_0-input-huggingFaceToken-password"}],"outputs":{},"outputAnchors":[{"id":"opea_service@embedding_tei_langchain_0-output-opea_service@embedding_tei_langchain-EmbedDoc|EmbeddingResponse|ChatCompletionRequest","name":"opea_service@embedding_tei_langchain","label":"EmbedDoc","description":"Text Embedding Inference using Langchain","type":"EmbedDoc | EmbeddingResponse | ChatCompletionRequest"}],"id":"opea_service@embedding_tei_langchain_0"},{"name":"opea_service@retriever_redis","version":1,"category":"Retreiver","inMegaservice":true,"dependent_services":{"tei":{"modelName":"","huggingFaceToken":""}},"inputs":{"text":"{{opea_service@embedding_tei_langchain_0.data.instance}}","vector_db":"{{redis_vector_store_0.data.instance}}","modelName":"BAAI/bge-base-en-v1.5","huggingFaceToken":"","search_type":"similarity"},"inputAnchors":[{"label":"Search Query","name":"text","type":"EmbedDoc|RetrievalRequest|ChatCompletionRequest","id":"opea_service@retriever_redis_0-input-text-EmbedDoc|RetrievalRequest|ChatCompletionRequest"},{"label":"Redis Vector Store","name":"vector_db","type":"EmbedDoc","id":"opea_service@retriever_redis_0-input-vector_db-EmbedDoc"}],"inputParams":[{"label":"Model Name","name":"modelName","type":"string","default":"BAAI/bge-base-en-v1.5","id":"opea_service@retriever_redis_0-input-modelName-string"},{"label":"HuggingFace Token","name":"huggingFaceToken","type":"password","optional":true,"id":"opea_service@retriever_redis_0-input-huggingFaceToken-password"},{"label":"Search Type","name":"search_type","type":"options","default":"similarity","options":[{"name":"similarity","label":"similarity"},{"name":"similarity_distance_threshold","label":"similarity_distance_threshold"},{"name":"similarity_score_threshold","label":"similarity_score_threshold"},{"name":"mmr","label":"mmr"}],"optional":true,"additionalParams":true,"inferenceParams":true,"id":"opea_service@retriever_redis_0-input-search_type-options"}],"outputs":{},"outputAnchors":[{"id":"opea_service@retriever_redis_0-output-opea_service@retriever_redis-SearchedDoc|RetrievalResponse|ChatCompletionRequest","name":"opea_service@retriever_redis","label":"SearchedDoc","description":"Redis Retreiver with Langchain","type":"SearchedDoc | RetrievalResponse | ChatCompletionRequest"}],"id":"opea_service@retriever_redis_0"},{"name":"redis_vector_store","version":1,"category":"VectorStores","inMegaservice":true,"inputs":{"prepared_doc":"{{opea_service@prepare_doc_redis_prep_0.data.instance}}"},"inputAnchors":[{"label":"Prepared Documents","name":"prepared_doc","type":"EmbedDoc","id":"redis_vector_store_0-input-prepared_doc-EmbedDoc"}],"inputParams":[],"outputs":{},"outputAnchors":[{"id":"redis_vector_store_0-output-redis_vector_store-EmbedDoc","name":"redis_vector_store","label":"EmbedDoc","description":"Redis Vector Store","type":"EmbedDoc"}],"id":"redis_vector_store_0"},{"name":"opea_service@reranking_tei","version":1,"category":"Reranking","inMegaservice":true,"dependent_services":{"tei":{"modelName":"","huggingFaceToken":""}},"inputs":{"retreived_docs":"{{opea_service@retriever_redis_0.data.instance}}","modelName":"BAAI/bge-reranker-base","huggingFaceToken":"","top_n":1},"inputAnchors":[{"label":"Documents","name":"retreived_docs","type":"SearchedDocs|RerankingRequest|ChatCompletionRequest","id":"opea_service@reranking_tei_0-input-retreived_docs-SearchedDocs|RerankingRequest|ChatCompletionRequest"}],"inputParams":[{"label":"Model Name","name":"modelName","type":"string","default":"BAAI/bge-reranker-base","id":"opea_service@reranking_tei_0-input-modelName-string"},{"label":"HuggingFace Token","name":"huggingFaceToken","type":"password","optional":true,"id":"opea_service@reranking_tei_0-input-huggingFaceToken-password"},{"label":"Top N","name":"top_n","type":"number","default":1,"optional":true,"additionalParams":true,"inferenceParams":true,"id":"opea_service@reranking_tei_0-input-top_n-number"}],"outputs":{},"outputAnchors":[{"id":"opea_service@reranking_tei_0-output-opea_service@reranking_tei-LLMParamsDoc|RerankingResponse|ChatCompletionRequest","name":"opea_service@reranking_tei","label":"LLMParamsDoc","description":"TEI Reranking","type":"LLMParamsDoc | RerankingResponse | ChatCompletionRequest"}],"id":"opea_service@reranking_tei_0"},{"name":"opea_service@llm_tgi","version":1,"category":"LLM","inMegaservice":true,"dependent_services":{"tgi":{"modelName":"","huggingFaceToken":"NA"}},"inputs":{"text":"{{opea_service@reranking_tei_0.data.instance}}","modelName":"Intel/neural-chat-7b-v3-3","huggingFaceToken":"","max_tokens":17,"top_k":10,"top_p":0.95,"typical_p":0.95,"temperature":0.01,"presence_penalty":1.03,"frequency_penalty":"","streaming":true,"chat_template":"### You are a helpful, respectful and honest assistant to help the user with questions.\n### Context: {context}\n### Question: {question}\n### Answer:"},"inputAnchors":[{"label":"LLM Params Document","name":"text","type":"LLMParamsDoc|ChatCompletionRequest|SearchedDoc","id":"opea_service@llm_tgi_0-input-text-LLMParamsDoc|ChatCompletionRequest|SearchedDoc"}],"inputParams":[{"label":"Model Name","name":"modelName","type":"string","default":"Intel/neural-chat-7b-v3-3","id":"opea_service@llm_tgi_0-input-modelName-options"},{"label":"HuggingFace Token","name":"huggingFaceToken","type":"password","optional":true,"id":"opea_service@llm_tgi_0-input-huggingFaceToken-password"},{"label":"Maximum Tokens","name":"max_tokens","type":"number","default":17,"optional":true,"additionalParams":true,"id":"opea_service@llm_tgi_0-input-max_tokens-number"},{"label":"Top K","name":"top_k","type":"number","default":10,"optional":true,"additionalParams":true,"id":"opea_service@llm_tgi_0-input-top_k-number"},{"label":"Top P","name":"top_p","type":"number","default":0.95,"optional":true,"additionalParams":true,"id":"opea_service@llm_tgi_0-input-top_p-number"},{"label":"Typical P","name":"typical_p","type":"number","default":0.95,"optional":true,"additionalParams":true,"id":"opea_service@llm_tgi_0-input-typical_p-number"},{"label":"Temperature","name":"temperature","type":"number","default":0.01,"optional":true,"additionalParams":true,"id":"opea_service@llm_tgi_0-input-temperature-number"},{"label":"Presence Penalty","name":"presence_penalty","type":"number","default":1.03,"optional":true,"additionalParams":true,"id":"opea_service@llm_tgi_0-input-presence_penalty-number"},{"label":"Frequency Penalty","name":"frequency_penalty","type":"number","default":0,"optional":true,"additionalParams":true,"id":"opea_service@llm_tgi_0-input-frequency_penalty-number"},{"label":"Streaming","name":"streaming","type":"boolean","default":true,"optional":true,"additionalParams":true,"id":"opea_service@llm_tgi_0-input-streaming-boolean"},{"label":"Chat Template","name":"chat_template","type":"string","rows":true,"default":"### You are a helpful, respectful and honest assistant to help the user with questions.\n### Context: {context}\n### Question: {question}\n### Answer:","optional":true,"additionalParams":true,"id":"opea_service@llm_tgi_0-input-chat_template-string"}],"outputs":{},"outputAnchors":[{"id":"opea_service@llm_tgi_0-output-opea_service@llm_tgi-GeneratedDoc|StreamingResponse|ChatCompletion","name":"opea_service@llm_tgi","label":"GeneratedDoc","description":"LLM Text Generation Inference","type":"GeneratedDoc | StreamingResponse | ChatCompletion"}],"id":"opea_service@llm_tgi_0"},{"name":"chat_completion","version":1,"category":"Controls","inMegaservice":false,"inputs":{"llm_response":"{{opea_service@llm_tgi_0.data.instance}}"},"hideOutput":true,"inputAnchors":[{"label":"LLM Response","name":"llm_response","type":"ChatCompletion","id":"chat_completion_0-input-llm_response-ChatCompletion"}],"inputParams":[],"outputs":{},"outputAnchors":[],"id":"chat_completion_0"}]},"deployed":false,"isPublic":false,"apikeyid":null,"chatbotConfig":null,"apiConfig":null,"analytic":null,"speechToText":null,"category":null,"type":"OPEA","sandboxStatus":null,"createdDate":"2024-10-07T16:29:51.000Z","updatedDate":"2024-10-07T16:32:21.000Z"} \ No newline at end of file diff --git a/studio-frontend/packages/server/src/nodes/llm.js b/studio-frontend/packages/server/src/nodes/llm.js index 67b7751..8d9a09d 100644 --- a/studio-frontend/packages/server/src/nodes/llm.js +++ b/studio-frontend/packages/server/src/nodes/llm.js @@ -21,7 +21,7 @@ class OPEA_LLM_TGi { this.dependent_services = { 'tgi': { 'modelName': '', - 'huggingFaceToken': 'NA' + 'huggingFaceToken': '' } } this.inputs = [ @@ -33,34 +33,8 @@ class OPEA_LLM_TGi { { label: 'Model Name', name: 'modelName', - type: 'options', - default: 'Intel/neural-chat-7b-v3-3', - options: [ - { - name: 'Intel/neural-chat-7b-v3-3', - label: 'Intel/neural-chat-7b-v3-3' - }, - { - name: 'Llama-2-7b-chat-hf', - label: 'Llama-2-7b-chat-hf' - }, - { - name: 'Llama-2-70b-chat-hf', - label: 'Llama-2-70b-chat-hf' - }, - { - name: 'Meta-Llama-3-8B-Instruct', - label: 'Meta-Llama-3-8B-Instruct' - }, - { - name: 'Meta-Llama-3-70B-Instruct', - label: 'Meta-Llama-3-70B-Instruct' - }, - { - name: 'Phi-3', - label: 'Phi-3' - } - ] + type: 'string', + default: 'Intel/neural-chat-7b-v3-3' }, { label: 'HuggingFace Token', diff --git a/tests/playwright/studio-e2e/001_test_sandbox_deployment.spec.ts b/tests/playwright/studio-e2e/001_test_sandbox_deployment.spec.ts index 146ba7a..49478a5 100644 --- a/tests/playwright/studio-e2e/001_test_sandbox_deployment.spec.ts +++ b/tests/playwright/studio-e2e/001_test_sandbox_deployment.spec.ts @@ -5,7 +5,7 @@ import path from 'path'; import os from 'os'; test('001_test_sandbox_deployment', async ({ browser, baseURL }) => { - test.setTimeout(600000); + test.setTimeout(1200000); const context = await browser.newContext({ ignoreHTTPSErrors: true }); @@ -38,7 +38,7 @@ test('001_test_sandbox_deployment', async ({ browser, baseURL }) => { // console.log(`Attempt ${i + 1} failed: ${error}`); // } // } - await waitForStatusText(page, 'td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root', 'Ready', 5, 60000); + await waitForStatusText(page, 'td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root', 'Ready', 10, 60000); await page.waitForTimeout(8000); // Open APP-UI diff --git a/tests/playwright/studio-e2e/002_test_sandbox_chatqna.spec.ts b/tests/playwright/studio-e2e/002_test_sandbox_chatqna.spec.ts index 056e3e6..daaefa2 100644 --- a/tests/playwright/studio-e2e/002_test_sandbox_chatqna.spec.ts +++ b/tests/playwright/studio-e2e/002_test_sandbox_chatqna.spec.ts @@ -38,7 +38,7 @@ async function setupResponseListener(page, apiResponse) { } test('002_test_sandbox_chatqna', async ({ browser, baseURL }) => { - test.setTimeout(600000); + test.setTimeout(1200000); let apiResponse = { value: '' }; const context = await browser.newContext({ ignoreHTTPSErrors: true @@ -63,7 +63,7 @@ test('002_test_sandbox_chatqna', async ({ browser, baseURL }) => { await page.goto(IDC_URL); await expect(page.locator('td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root').first()).toHaveText('Not Running', { timeout: 60000 }); await page.getByLabel('a dense table').locator('button').first().click(); - await waitForStatusText(page, 'td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root', 'Ready', 5, 60000); + await waitForStatusText(page, 'td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root', 'Ready', 10, 60000); await page.waitForTimeout(10000); // Open APP-UI diff --git a/tests/playwright/utils.ts b/tests/playwright/utils.ts index a32dea1..1a299d2 100644 --- a/tests/playwright/utils.ts +++ b/tests/playwright/utils.ts @@ -1,6 +1,6 @@ import { expect } from '@playwright/test'; -export async function waitForStatusText(page: any, selector: string, statusText: string, maxAttempts: number = 5, intervalTimeout: number = 60000) { +export async function waitForStatusText(page: any, selector: string, statusText: string, maxAttempts: number = 10, intervalTimeout: number = 60000) { for (let i = 0; i < maxAttempts; i++) { try { const text = await page.locator(selector).first().innerText();