From cdabc5565ef1be8daaac4351a001b9e705004e19 Mon Sep 17 00:00:00 2001 From: Github Actions Date: Mon, 22 Dec 2025 11:48:45 +0000 Subject: [PATCH 01/10] Patch code up to 42722628 Signed-off-by: Github Actions --- .../ChatHistoryList/ChatHistoryList.tsx | 38 ++++++++++++++++--- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/src/ui/apps/chatqna/src/features/chat/components/ChatHistoryList/ChatHistoryList.tsx b/src/ui/apps/chatqna/src/features/chat/components/ChatHistoryList/ChatHistoryList.tsx index 0b339fe8a..2a3a0582e 100644 --- a/src/ui/apps/chatqna/src/features/chat/components/ChatHistoryList/ChatHistoryList.tsx +++ b/src/ui/apps/chatqna/src/features/chat/components/ChatHistoryList/ChatHistoryList.tsx @@ -3,15 +3,35 @@ import "./ChatHistoryList.scss"; -import { LoadingFallback } from "@intel-enterprise-rag-ui/components"; +import { + LoadingFallback, + SearchBar, +} from "@intel-enterprise-rag-ui/components"; import classNames from "classnames"; +import { useMemo, useState } from "react"; import { useGetAllChatsQuery } from "@/features/chat/api/chatHistory"; import ChatHistoryItem from "@/features/chat/components/ChatHistoryItem/ChatHistoryItem"; const ChatHistoryList = () => { const { data, isLoading } = useGetAllChatsQuery(); - const isChatHistoryEmpty = !Array.isArray(data) || data.length === 0; + const [searchFilter, setSearchFilter] = useState(""); + + const filteredData = useMemo(() => { + if (!data || !Array.isArray(data)) return []; + if (!searchFilter.trim()) return data; + + const lowerSearchFilter = searchFilter.toLowerCase(); + return data.filter((item) => + item.name.toLowerCase().includes(lowerSearchFilter), + ); + }, [data, searchFilter]); + + const isChatHistoryEmpty = filteredData.length === 0; + + const emptyStateMessage = searchFilter.trim() + ? "No chat history matches your search." + : "No chat history available."; const chatHistoryListClass = classNames("chat-history-list", { "chat-history-list--empty": isChatHistoryEmpty, @@ -19,15 +39,23 @@ const ChatHistoryList = () => { return ( ); From 8d87f42e1f8da1eecf969318d2ce720378e34e7a Mon Sep 17 00:00:00 2001 From: Github Actions Date: Mon, 22 Dec 2025 11:53:59 +0000 Subject: [PATCH 02/10] Patch code up to 8d9e05df Signed-off-by: Github Actions --- .../components/cards/PromptTemplateCard.tsx | 77 +++++-------------- 1 file changed, 20 insertions(+), 57 deletions(-) diff --git a/src/ui/apps/chatqna/src/features/admin-panel/control-plane/components/cards/PromptTemplateCard.tsx b/src/ui/apps/chatqna/src/features/admin-panel/control-plane/components/cards/PromptTemplateCard.tsx index 331c2c597..e637b7ad9 100644 --- a/src/ui/apps/chatqna/src/features/admin-panel/control-plane/components/cards/PromptTemplateCard.tsx +++ b/src/ui/apps/chatqna/src/features/admin-panel/control-plane/components/cards/PromptTemplateCard.tsx @@ -1,17 +1,10 @@ // Copyright (C) 2024-2025 Intel Corporation // SPDX-License-Identifier: Apache-2.0 -import { Button } from "@intel-enterprise-rag-ui/components"; import { sanitizeString } from "@intel-enterprise-rag-ui/utils"; -import { - ChangeEventHandler, - FormEventHandler, - useEffect, - useState, -} from "react"; +import { ChangeEventHandler, useEffect, useState } from "react"; import { ValidationError } from "yup"; -import { useChangeArgumentsMutation } from "@/features/admin-panel/control-plane/api"; import { ControlPlaneCardProps } from "@/features/admin-panel/control-plane/components/cards"; import SelectedServiceCard from "@/features/admin-panel/control-plane/components/SelectedServiceCard/SelectedServiceCard"; import ServiceArgumentTextArea from "@/features/admin-panel/control-plane/components/ServiceArgumentTextArea/ServiceArgumentTextArea"; @@ -19,23 +12,22 @@ import { PromptTemplateArgs, promptTemplateFormConfig, } from "@/features/admin-panel/control-plane/config/chat-qna-graph/promptTemplate"; -import { ChangeArgumentsRequest } from "@/features/admin-panel/control-plane/types/api"; +import useServiceCard from "@/features/admin-panel/control-plane/hooks/useServiceCard"; import { validatePromptTemplateForm } from "@/features/admin-panel/control-plane/validators/promptTemplateInput"; const PromptTemplateCard = ({ data: { + id, status, displayName, promptTemplateArgs: prevPromptTemplateArguments, }, }: ControlPlaneCardProps) => { - const [changeArguments] = useChangeArgumentsMutation(); - - const [promptTemplateForm, setPromptTemplateForm] = - useState( - (prevPromptTemplateArguments ?? - ({} as PromptTemplateArgs)) as PromptTemplateArgs, - ); + const { + argumentsForm: promptTemplateForm, + onArgumentValueChange, + footerProps, + } = useServiceCard(id, prevPromptTemplateArguments); const [isHydrated, setIsHydrated] = useState( !!prevPromptTemplateArguments, @@ -46,7 +38,6 @@ const PromptTemplateCard = ({ useEffect(() => { if (prevPromptTemplateArguments !== undefined) { - setPromptTemplateForm(prevPromptTemplateArguments); setIsHydrated(true); } else { setIsHydrated(false); @@ -75,40 +66,20 @@ const PromptTemplateCard = ({ const handleChange: ChangeEventHandler = (event) => { const { value, name } = event.target; - setPromptTemplateForm((prevForm) => ({ - ...prevForm, - [name]: sanitizeString(value), - })); - }; - - const handlePromptTemplateArgsSubmit: FormEventHandler = ( - event, - ) => { - event.preventDefault(); - const changeArgumentsRequest: ChangeArgumentsRequest = [ - { - name: "prompt_template", - data: promptTemplateForm, - }, - ]; - - changeArguments(changeArgumentsRequest); + onArgumentValueChange(name, sanitizeString(value)); }; - const changePromptTemplateBtnDisabled = - !isHydrated || - isInvalid || - (promptTemplateForm.user_prompt_template === - prevPromptTemplateArguments?.user_prompt_template && - promptTemplateForm.system_prompt_template === - prevPromptTemplateArguments?.system_prompt_template); - return ( - -
+ +

{error}

-
- +
); }; From 6f9d32d7b669376e078dd708820cf079ca8186c8 Mon Sep 17 00:00:00 2001 From: Github Actions Date: Tue, 23 Dec 2025 10:38:19 +0000 Subject: [PATCH 03/10] Patch code up to e9ec2f4e Signed-off-by: Github Actions --- src/tests/e2e/helpers/k8s_helper.py | 20 ++++++++-- src/tests/e2e/lifecycle/scenarios.yaml | 5 +++ .../lifecycle/test_update_embedding_model.py | 38 +++++++++++++++++++ .../e2e/lifecycle/test_update_llm_model.py | 38 +++++++++++++++++-- .../lifecycle/test_update_reranker_model.py | 27 ++++--------- src/tests/e2e/validation/constants.py | 1 + 6 files changed, 102 insertions(+), 27 deletions(-) create mode 100644 src/tests/e2e/lifecycle/test_update_embedding_model.py diff --git a/src/tests/e2e/helpers/k8s_helper.py b/src/tests/e2e/helpers/k8s_helper.py index 4144feaec..418ac293c 100644 --- a/src/tests/e2e/helpers/k8s_helper.py +++ b/src/tests/e2e/helpers/k8s_helper.py @@ -11,6 +11,10 @@ logger = logging.getLogger(__name__) +class ResourceNotFound(Exception): + pass + + class K8sHelper: def retrieve_admin_password(self, secret_name, namespace): @@ -33,16 +37,24 @@ def list_pods(self, namespace): pods = kr8s.get("pods", namespace=namespace) return [pod.name for pod in pods] - def get_pods_by_label(self, namespace, label_selector): - """Get a list of kr8s Pod objects matching a label selector in a namespace""" + def get_pod_by_label(self, namespace, label_selector): + """Returns first pod matching a label selector in a namespace""" logger.debug(f"Getting pods with label selector '{label_selector}' in namespace '{namespace}'") - return kr8s.get("pods", namespace=namespace, label_selector=label_selector) + pods = kr8s.get("pods", namespace=namespace, label_selector=label_selector) + if len(pods) == 0: + raise ResourceNotFound(f"No running pods found with label '{label_selector}' in namespace '{namespace}'.") + return pods[0] def exec_in_pod(self, pod, command): """Execute a command in a pod's container""" logger.debug(f"Executing command '{command}' in pod '{pod.name}'") try: - return pod.exec(command) + exec_result = pod.exec(command) + stdout = exec_result.stdout + if isinstance(stdout, bytes): + return stdout.decode().strip() + else: + return stdout.strip() except Exception as e: logger.error(f"Failed to execute command in pod '{pod.name}': {e}") raise # Re-raise the exception to fail the test diff --git a/src/tests/e2e/lifecycle/scenarios.yaml b/src/tests/e2e/lifecycle/scenarios.yaml index f7792f4de..2f4e9dca6 100644 --- a/src/tests/e2e/lifecycle/scenarios.yaml +++ b/src/tests/e2e/lifecycle/scenarios.yaml @@ -23,3 +23,8 @@ scenarios: lifecycle: - test_update_reranker_model.py markers: "smoke" + + update_embedding_model: + lifecycle: + - test_update_embedding_model.py + markers: "smoke" diff --git a/src/tests/e2e/lifecycle/test_update_embedding_model.py b/src/tests/e2e/lifecycle/test_update_embedding_model.py new file mode 100644 index 000000000..f87d521a8 --- /dev/null +++ b/src/tests/e2e/lifecycle/test_update_embedding_model.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (C) 2025 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import allure +import logging +import pytest + +from tests.e2e.validation.constants import CHATQA_NAMESPACE +from tests.e2e.validation.buildcfg import cfg + +logger = logging.getLogger(__name__) + +EMBEDDING_POD_LABEL_SELECTOR = "app.kubernetes.io/name=embedding-usvc" + + +@allure.testcase("IEASG-T311") +def test_update_embedding_model(k8s_helper): + """ + Test verifies if reranker pod have the expected EMBEDDING_MODEL_NAME + environment variable set. + """ + expected_model_name = cfg.get("embedding_model_name") + logger.info(f"Expected reranking model name from config: {expected_model_name}") + embedding_pod = k8s_helper.get_pod_by_label(namespace=CHATQA_NAMESPACE, + label_selector=EMBEDDING_POD_LABEL_SELECTOR) + + command = ["sh", "-c", "echo $EMBEDDING_MODEL_NAME"] + logger.info(f"Checking environment variable $EMBEDDING_MODEL_NAME in pod: {embedding_pod.name}") + try: + env_value = k8s_helper.exec_in_pod(embedding_pod, command) + logger.info(f"Pod '{embedding_pod.name}' reports EMBEDDING_MODEL_NAME = '{env_value}'") + except Exception as e: + pytest.fail(f"Test failed while executing command in pod '{embedding_pod.name}'. Error: {e}") + + assert env_value == expected_model_name, \ + f"Pod '{embedding_pod.name}' has an incorrect model name. Expected: '{expected_model_name}', Found: '{env_value}'" diff --git a/src/tests/e2e/lifecycle/test_update_llm_model.py b/src/tests/e2e/lifecycle/test_update_llm_model.py index 9690537f2..5eaae14e7 100644 --- a/src/tests/e2e/lifecycle/test_update_llm_model.py +++ b/src/tests/e2e/lifecycle/test_update_llm_model.py @@ -5,12 +5,42 @@ import allure import logging +import pytest + +from tests.e2e.helpers.k8s_helper import ResourceNotFound +from tests.e2e.validation.buildcfg import cfg +from tests.e2e.validation.constants import CHATQA_NAMESPACE logger = logging.getLogger(__name__) +VLLM_POD_LABEL_SELECTOR = "app.kubernetes.io/name=vllm" +VLLM_GAUDI_POD_LABEL_SELECTOR = "app.kubernetes.io/name=vllm_gaudi" + -# TODO: add allure test case id -@allure.testcase("IEASG-T") +@allure.testcase("IEASG-T288") def test_update_llm_model(k8s_helper): - """ """ - pass + """ + Test verifies if vllm pod have the expected LLM_VLLM_MODEL_NAME env variable set. + It first tries to find the vllm pod, and if not found, it falls back to the vllm-gaudi pod. + Depending on which pod is found, it checks the corresponding model name from the config. + """ + try: + vllm_pod = k8s_helper.get_pod_by_label(namespace=CHATQA_NAMESPACE, label_selector=VLLM_POD_LABEL_SELECTOR) + expected_model_name = cfg.get("llm_model") + except ResourceNotFound as e: + logger.debug(e) + logger.debug("Looking for vllm-gaudi pod as fallback.") + vllm_pod = k8s_helper.get_pod_by_label(namespace=CHATQA_NAMESPACE, label_selector=VLLM_GAUDI_POD_LABEL_SELECTOR) + expected_model_name = cfg.get("llm_model_gaudi") + + logger.info(f"Expected LLM model name from config: {expected_model_name}") + command = ["sh", "-c", "echo $LLM_VLLM_MODEL_NAME"] + logger.info(f"Checking environment variable $LLM_VLLM_MODEL_NAME in pod: {vllm_pod.name}") + try: + env_value = k8s_helper.exec_in_pod(vllm_pod, command) + logger.info(f"Pod '{vllm_pod.name}' reports LLM_VLLM_MODEL_NAME = '{env_value}'") + except Exception as e: + pytest.fail(f"Test failed while executing command in pod '{vllm_pod.name}'. Error: {e}") + + assert env_value == expected_model_name, \ + f"Pod '{vllm_pod.name}' has an incorrect model name. Expected: '{expected_model_name}', Found: '{env_value}'" diff --git a/src/tests/e2e/lifecycle/test_update_reranker_model.py b/src/tests/e2e/lifecycle/test_update_reranker_model.py index 1bdb0411d..25fa2788e 100644 --- a/src/tests/e2e/lifecycle/test_update_reranker_model.py +++ b/src/tests/e2e/lifecycle/test_update_reranker_model.py @@ -7,12 +7,12 @@ import logging import pytest +from tests.e2e.validation.constants import CHATQA_NAMESPACE from tests.e2e.validation.buildcfg import cfg logger = logging.getLogger(__name__) -NAMESPACE = "chatqa" -APP_LABEL_SELECTOR = "app.kubernetes.io/name=reranking-usvc" +RERANKER_POD_LABEL_SELECTOR = "app.kubernetes.io/name=reranking-usvc" @allure.testcase("IEASG-T310") @@ -23,26 +23,15 @@ def test_update_reranker_model(k8s_helper): """ expected_model_name = cfg.get("reranking_model_name") logger.info(f"Expected reranking model name from config: {expected_model_name}") + reranker_pod = k8s_helper.get_pod_by_label(namespace=CHATQA_NAMESPACE, label_selector=RERANKER_POD_LABEL_SELECTOR) - pods = k8s_helper.get_pods_by_label(namespace=NAMESPACE, label_selector=APP_LABEL_SELECTOR) - assert len(pods) > 0, f"No running pods found with label '{APP_LABEL_SELECTOR}' in namespace '{NAMESPACE}'." - reranker_pod = pods[0] - + command = ["sh", "-c", "echo $RERANKING_MODEL_NAME"] logger.info(f"Checking environment variable $RERANKING_MODEL_NAME in pod: {reranker_pod.name}") try: - command = ["sh", "-c", "echo $RERANKING_MODEL_NAME"] - exec_result = k8s_helper.exec_in_pod(reranker_pod, command) - - stdout = exec_result.stdout - if isinstance(stdout, bytes): - env_value = stdout.decode().strip() - else: - env_value = stdout.strip() - + env_value = k8s_helper.exec_in_pod(reranker_pod, command) logger.info(f"Pod '{reranker_pod.name}' reports RERANKING_MODEL_NAME = '{env_value}'") - - assert env_value == expected_model_name, \ - f"Pod '{reranker_pod.name}' has an incorrect model name. Expected: '{expected_model_name}', Found: '{env_value}'" - except Exception as e: pytest.fail(f"Test failed while executing command in pod '{reranker_pod.name}'. Error: {e}") + + assert env_value == expected_model_name, \ + f"Pod '{reranker_pod.name}' has an incorrect model name. Expected: '{expected_model_name}', Found: '{env_value}'" diff --git a/src/tests/e2e/validation/constants.py b/src/tests/e2e/validation/constants.py index 308492d4a..e4d484666 100644 --- a/src/tests/e2e/validation/constants.py +++ b/src/tests/e2e/validation/constants.py @@ -11,3 +11,4 @@ VITE_KEYCLOAK_CLIENT_ID = "EnterpriseRAG-oidc" INGRESS_NGINX_CONTROLLER_NS = "ingress-nginx" INGRESS_NGINX_CONTROLLER_POD_LABEL_SELECTOR = {"app.kubernetes.io/name": "ingress-nginx"} +CHATQA_NAMESPACE = "chatqa" \ No newline at end of file From 365f236090da1f5ba3a28753634bc40d0b6e179a Mon Sep 17 00:00:00 2001 From: Github Actions Date: Tue, 30 Dec 2025 11:42:49 +0000 Subject: [PATCH 04/10] Patch code up to 7f9fa847 Signed-off-by: Github Actions --- .../edp/templates/ingestion/ingestion-configmap.yaml | 1 + deployment/components/edp/values.yaml | 3 +++ deployment/components/gmc/values.yaml | 2 ++ deployment/roles/application/edp/templates/values.yaml.j2 | 4 +++- .../roles/application/pipeline/templates/values.yaml.j2 | 1 + src/comps/vectorstores/utils/connectors/connector_redis.py | 3 ++- 6 files changed, 12 insertions(+), 2 deletions(-) diff --git a/deployment/components/edp/templates/ingestion/ingestion-configmap.yaml b/deployment/components/edp/templates/ingestion/ingestion-configmap.yaml index c6c40299c..4515ba4b5 100644 --- a/deployment/components/edp/templates/ingestion/ingestion-configmap.yaml +++ b/deployment/components/edp/templates/ingestion/ingestion-configmap.yaml @@ -7,6 +7,7 @@ metadata: name: edp-ingestion-configmap namespace: edp data: + EMBEDDING_MODEL_NAME: {{ .Values.ingestion.config.embedding_model_name | quote }} VECTOR_STORE: {{ .Values.ingestion.config.vector_store | quote }} VECTOR_ALGORITHM: {{ .Values.ingestion.config.vector_algorithm | quote }} VECTOR_DIMS: {{ .Values.ingestion.config.vector_dims | quote }} diff --git a/deployment/components/edp/values.yaml b/deployment/components/edp/values.yaml index 203a2d4d8..9f1af7ba8 100644 --- a/deployment/components/edp/values.yaml +++ b/deployment/components/edp/values.yaml @@ -26,6 +26,8 @@ edpOidcConfigUrl: "http://keycloak-http.auth.svc/realms/EnterpriseRAG/.well-know edpOidcClientSecret: "" bucketNameRegexFilter: '.*' presignedUrlCredentialsSystemFallback: "false" +embedding_model_name: &embedding_model_name "BAAI/bge-base-en-v1.5" + minioApiDomain: &minioApiDomain "s3.erag.com" minioBrowserDomain: &minioBrowserDomain "minio.erag.com" @@ -895,6 +897,7 @@ ingestion: tag: latest config: opeaLoggerLevel: "INFO" # "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL" + embedding_model_name: *embedding_model_name # e.g., "BAAI/bge-base-en-v1.5" # Vector Algorithm configuration vector_algorithm: "FLAT" # "FLAT", "HNSW" vector_dims: "768" # Depends on model used in embedding. For example bge-large-en-v1.5=768, bge-large-en-v1.5=1024 diff --git a/deployment/components/gmc/values.yaml b/deployment/components/gmc/values.yaml index 280284c7e..5a8584072 100644 --- a/deployment/components/gmc/values.yaml +++ b/deployment/components/gmc/values.yaml @@ -221,6 +221,8 @@ images: tag: *tag pullPolicy: Always envfile: "src/comps/retrievers/impl/microservice/.env" + envs: + EMBEDDING_MODEL_NAME: *embedding_model_name ingestion-usvc: image: "erag-ingestion" repository: *repo diff --git a/deployment/roles/application/edp/templates/values.yaml.j2 b/deployment/roles/application/edp/templates/values.yaml.j2 index 8bd0c73a9..63eac146a 100644 --- a/deployment/roles/application/edp/templates/values.yaml.j2 +++ b/deployment/roles/application/edp/templates/values.yaml.j2 @@ -12,6 +12,8 @@ proxy: alternateTagging: {{ use_alternate_tagging }} {% endif %} +embedding_model_name: &embedding_model_name {{ embedding_model_name }} + {% set storage = lookup('env', 'edp_storage_type') or edp.storageType if edp.storageType is defined else "minio" %} {% if storage == "minio" %} edpAccessKey: {{ EDP_MINIO_ACCESS_KEY }} @@ -136,10 +138,10 @@ ingestion: tag: {{ tag }} repository: {{ registry }} config: + embedding_model_name: *embedding_model_name {% if edp.hierarchical_indices.enabled is true %} use_hierarchical_indices: "True" {% endif %} - config: vector_dims: {{ vector_databases.vector_dims }} vector_datatype: {{ vector_databases.vector_datatype }} {% if edp.late_chunking.enabled is true %} diff --git a/deployment/roles/application/pipeline/templates/values.yaml.j2 b/deployment/roles/application/pipeline/templates/values.yaml.j2 index ffac6237e..f0b7fccf5 100644 --- a/deployment/roles/application/pipeline/templates/values.yaml.j2 +++ b/deployment/roles/application/pipeline/templates/values.yaml.j2 @@ -148,6 +148,7 @@ images: vector_store: {{ vector_databases.vector_store }} {% endif %} envs: + EMBEDDING_MODEL_NAME: *embedding_model_name {% if edp.hierarchical_indices.enabled is true %} USE_HIERARCHICAL_INDICES: "True" K_SUMMARIES: {{ edp.hierarchical_indices.kSummaries }} diff --git a/src/comps/vectorstores/utils/connectors/connector_redis.py b/src/comps/vectorstores/utils/connectors/connector_redis.py index c1a4b657c..faa138fbd 100644 --- a/src/comps/vectorstores/utils/connectors/connector_redis.py +++ b/src/comps/vectorstores/utils/connectors/connector_redis.py @@ -79,7 +79,8 @@ def _metadata_schema(self): return metadata_schema def _vector_schema(self, schema: dict, metadata_schema: Optional[dict]=None) -> IndexSchema: - index_name = f"{schema['algorithm'].lower()}_{schema['datatype'].lower()}_{schema['distance_metric'].lower()}_index" + model_name = sanitize_env(os.getenv("EMBEDDING_MODEL_NAME", "default")).replace("/", "_").replace("-", "_") + index_name = f"{model_name.lower()}_{schema['algorithm'].lower()}_{schema['datatype'].lower()}_{schema['distance_metric'].lower()}_{schema['dims']}_index" data = { "index": { From 97e08a46656a522bcd056514acf82bad449d4bba Mon Sep 17 00:00:00 2001 From: Github Actions Date: Fri, 2 Jan 2026 11:55:16 +0000 Subject: [PATCH 05/10] Patch code up to 32bd261b Signed-off-by: Github Actions --- .../e2e/evals/evaluation/rag_eval/README.md | 84 ++++++++++++++----- .../evals/evaluation/rag_eval/evaluator.py | 5 ++ .../rag_eval/examples/eval_multihop.py | 42 +++++++++- 3 files changed, 107 insertions(+), 24 deletions(-) diff --git a/src/tests/e2e/evals/evaluation/rag_eval/README.md b/src/tests/e2e/evals/evaluation/rag_eval/README.md index abaebd2ac..af507c129 100644 --- a/src/tests/e2e/evals/evaluation/rag_eval/README.md +++ b/src/tests/e2e/evals/evaluation/rag_eval/README.md @@ -12,9 +12,9 @@ - [MultiHop (English dataset)](#multihop-english-dataset) - [Evaluation](#evaluation) - [Usage Guide](#usage-guide) + - [Tips to Control the Evaluation Scope](#tips-to-control-the-evaluation-scope) - [Acknowledgements](#acknowledgements) - ## Introduction @@ -187,7 +187,7 @@ This evaluation uses [yixuantt/MultiHopRAG](https://huggingface.co/datasets/yixu ### Evaluation -This section explains how to run the evaluation pipeline for Multihop dataset. +This section explains how to run the evaluation pipeline for MultiHop dataset. The evaluation script is located at `examples/eval_multihop.py`. @@ -199,23 +199,42 @@ python eval_multihop.py --help | **Argument** | **Default Value** | **Description** | | ---------------------- |---------------------------------------------------|-------------------------------------------------------------------------------------------------| -| `--output_dir` | `./output` | Directory to save evaluation results | -| `--auth_file` | `deployment/ansible-logs/default_credentials.txt` | Path to credentials file with `KEYCLOAK_ERAG_ADMIN_USERNAME` and `KEYCLOAK_ERAG_ADMIN_PASSWORD` | -| `--cluster_config_file`| `deployment/inventory/sample/config.yaml` | Path to cluster configuration YAML file with deployment settings | -| `--dataset_path` | `multihop_dataset/MultiHopRAG.json` | Path to the evaluation dataset | -| `--docs_path` | `multihop_dataset/corpus.json` | Path to the documents for retrieval | -| `--limits` | `100` | Number of queries to evaluate (0 means evaluate all; default: 100) | -| `--ingest_docs` | *(flag)* | Ingest documents into the vector database (use only on first run) | -| `--generation_metrics` | *(flag)* | Compute text generation metrics (`BLEU`, `ROUGE`) | -| `--retrieval_metrics` | *(flag)* | Compute retrieval metrics (`Hits@K`, `MAP@K`, `MRR@K`) | -| `--skip_normalize` | *(flag)* | Skip 'None' separator normalization for exact 1:1 text matching | -| `--ragas_metrics` | *(flag)* | Compute RAGAS metrics (answer correctness, context precision, etc.) | -| `--resume_checkpoint` | *None* | Path to a checkpoint file to resume evaluation from previous state | -| `--keep_checkpoint` | *(flag)* | Keep the checkpoint file after evaluation (do not delete) | -| `--llm_judge_endpoint` | `http://localhost:8008` | URL of the LLM judge service; only used for RAGAS evaluation | -| `--embedding_endpoint` | `http://localhost:8090/embed` | URL of the embedding service endpoint, only used for RAGAS | -| `--temperature` | Read from RAG system config | Controls text generation randomness; defaults to RAG system setting if omitted. | -| `--max_new_tokens` | Read from RAG system config | Maximum tokens generated; defaults to RAG system setting if omitted. | +| `--output_dir` | `./output` | Directory to save evaluation results +| +| `--auth_file` | `deployment/ansible-logs/default_credentials.txt` | Path to credentials file with `KEYCLOAK_ERAG_ADMIN_USERNAME` and `KEYCLOAK_ERAG_ADMIN_PASSWORD` +| +| `--cluster_config_file`| `deployment/inventory/sample/config.yaml` | Path to cluster configuration YAML file with deployment settings +| +| `--dataset_path` | `multihop_dataset/MultiHopRAG.json` | Path to the evaluation dataset +| +| `--docs_path` | `multihop_dataset/corpus.json` | Path to the documents for retrieval +| +| `--limits` | `100` | Number of queries to evaluate (0 means evaluate all; default: 100) +| +| `--exclude_types` | *None* | Exclude queries by question type. Queries matching these question types will be skipped. Example: --exclude_types comparison_query +| +| `--ingest_docs` | *(flag)* | Ingest documents into the vector database (use only on first run) +| +| `--generation_metrics` | *(flag)* | Compute text generation metrics (`BLEU`, `ROUGE`) +| +| `--retrieval_metrics` | *(flag)* | Compute retrieval metrics (`Hits@K`, `MAP@K`, `MRR@K`) +| +| `--skip_normalize` | *(flag)* | Skip 'None' separator normalization for exact 1:1 text matching +| +| `--ragas_metrics` | *(flag)* | Compute RAGAS metrics (answer correctness, context precision, etc.) +| +| `--resume_checkpoint` | *None* | Path to a checkpoint file to resume evaluation from previous state +| +| `--keep_checkpoint` | *(flag)* | Keep the checkpoint file after evaluation (do not delete) +| +| `--llm_judge_endpoint` | `http://localhost:8008` | URL of the LLM judge service; only used for RAGAS evaluation +| +| `--embedding_endpoint` | `http://localhost:8090/embed` | URL of the embedding service endpoint, only used for RAGAS +| +| `--temperature` | Read from RAG system config | Controls text generation randomness; defaults to RAG system setting if omitted +| +| `--max_new_tokens` | Read from RAG system config | Maximum tokens generated; defaults to RAG system setting if omitted +| > Note: If `--dataset_path` and `--docs_path` are set to their default values and the corresponding files are not found locally, they will be automatically downloaded at runtime from [yixuantt/MultiHopRAG](https://huggingface.co/datasets/yixuantt/MultiHopRAG) and saved to the expected local paths. @@ -224,7 +243,7 @@ python eval_multihop.py --help ### Usage Guide -This section outlines how to run Multihop evaluation of the RAG pipeline using [examples/eval_multihop.py](examples/eval_multihop.py). +This section outlines how to run MultiHop evaluation of the RAG pipeline using [examples/eval_multihop.py](examples/eval_multihop.py). - **Ingest Documents** To ingest the MultiHop dataset into the RAG system, use the flag `--ingest_docs`: @@ -250,7 +269,7 @@ This section outlines how to run Multihop evaluation of the RAG pipeline using [ _Metrics: BLEU, ROUGE, (LLM-score – not implemented yet)_ - To evaluate the quality of RAG generated answers on Multihop queries, run: + To evaluate the quality of RAG generated answers on MultiHop queries, run: ```bash # First-time run (with document ingestion) @@ -374,6 +393,29 @@ The evaluation results are stored in the output/ directory with detailed logs an The query and its corresponding ground_truth_text originate from the yixuantt/MultiHopRAG dataset. +### Tips to Control the Evaluation Scope + +**Controlling the Number of Queries with `--limits`:** + +The `--limits` parameter allows you to control how many queries from the dataset are evaluated. This is particularly useful for quick testing during development to verify that the pipeline works correctly. + +```bash +# Evaluate only the first 2 queries (quick test) +python eval_multihop.py --generation_metrics --limits 2 +``` + +**Excluding Specific Query Types with `--exclude_types`:** + +The dataset specified by `--dataset_path` (default: multihop_dataset/MultiHopRAG.json) contains queries along with their `question_type`. You can selectively exclude specific query types from evaluation using the `--exclude_types` parameter. This is useful when you want to focus on particular aspects of your RAG system, for example, to compute accuracy metrics separately for each query type and identify which types your RAG handles better or worse. + +```bash +# Exclude comparison queries from evaluation +python eval_multihop.py --retrieval_metrics --exclude_types comparison_query + +# Exclude multiple query types (space-separated) +python eval_multihop.py --retrieval_metrics --exclude_types comparison_query inference_query +``` + ## Acknowledgements This example is mostly adapted from [MultiHop-RAG](https://github.com/yixuantt/MultiHop-RAG) repo, we thank the authors for their great work! diff --git a/src/tests/e2e/evals/evaluation/rag_eval/evaluator.py b/src/tests/e2e/evals/evaluation/rag_eval/evaluator.py index e86120073..acc6965d2 100644 --- a/src/tests/e2e/evals/evaluation/rag_eval/evaluator.py +++ b/src/tests/e2e/evals/evaluation/rag_eval/evaluator.py @@ -140,6 +140,9 @@ def get_golden_context(self, data: dict): def get_query(self, data: dict): raise NotImplementedError("Depends on the specific dataset.") + def get_question_type(self, data: dict): + raise NotImplementedError("Depends on the specific dataset.") + def get_document(self, data: dict): raise NotImplementedError("Depends on the specific dataset.") @@ -164,6 +167,7 @@ def scoring(self, data: dict) -> dict: }, "log": { "query": self.get_query(data), + "query_type": self.get_question_type(data), "generated_text": generated_text, "ground_truth_text": ground_truth_text, "evaluateDatetime": str(datetime.now()), @@ -217,6 +221,7 @@ def scoring_retrieval(self, data: dict, normalize: bool = True) -> dict: }, "log": { "query": self.get_query(data), + "query_type": self.get_question_type(data), "golden_context": golden_context, "num_retrieved_documents": len(retrieved_documents), "num_reranked_documents": len(reranked_documents), diff --git a/src/tests/e2e/evals/evaluation/rag_eval/examples/eval_multihop.py b/src/tests/e2e/evals/evaluation/rag_eval/examples/eval_multihop.py index b7b4873d7..df8139f3f 100644 --- a/src/tests/e2e/evals/evaluation/rag_eval/examples/eval_multihop.py +++ b/src/tests/e2e/evals/evaluation/rag_eval/examples/eval_multihop.py @@ -41,6 +41,9 @@ def get_ground_truth_text(self, data: dict): def get_query(self, data: dict): return data["query"] + def get_question_type(self, data: dict): + return data.get("question_type") or "unknown" + def get_template(self): return None @@ -63,7 +66,7 @@ def evaluate(self, all_queries, arguments): generated_text = self.send_request(query, arguments) data["generated_text"] = generated_text - result = {"id": index, "uuid": self.get_uuid(query), **self.scoring(data)} + result = {"id": index, "uuid": self.get_uuid(query), "question_type": self.get_question_type(data), **self.scoring(data)} logger.debug(f"Result for query {index}: {result}") results.append(result) index += 1 @@ -194,6 +197,7 @@ def get_retrieval_metrics(self, all_queries, arguments): def prepare_ragas_record(self, data, arguments): query = self.get_query(data) + question_type = self.get_question_type(data) generated_text = self.send_request(query, arguments) try: @@ -204,6 +208,7 @@ def prepare_ragas_record(self, data, arguments): return { "query": query, + "question_type": question_type, "generated_text": generated_text, "ground_truth": self.get_ground_truth_text(data), "golden_context": self.get_golden_context(data), @@ -293,8 +298,9 @@ def get_ragas_metrics(self, all_queries, arguments): # Store metadata for each query query_metadata.append({ - "query": result["query"], "uuid": self.get_uuid(result["query"]), + "query": result["query"], + "question_type": result["question_type"], "generated_text": result["generated_text"], "ground_truth": result["ground_truth"], "golden_context": result["golden_context"], @@ -328,6 +334,7 @@ def get_ragas_metrics(self, all_queries, arguments): "ragas_metrics": score, "log": { "query": query_metadata[idx]["query"], + "question_type": query_metadata[idx]["question_type"], "generated_text": query_metadata[idx]["generated_text"], "ground_truth": query_metadata[idx]["ground_truth"], "golden_context": query_metadata[idx]["golden_context"], @@ -398,6 +405,7 @@ def args_parser(): parser.add_argument("--ragas_metrics", action="store_true", help="Whether to compute ragas metrics such as answer correctness, relevancy, semantic similarity, context precision, context recall , and faithfulness") parser.add_argument("--skip_normalize", action="store_true", help="Skip normalization of 'None' separators in retrieval metrics. By default, normalization is enabled") parser.add_argument("--limits", type=int, default=100, help="Number of queries to evaluate. Set to 0 to evaluate all provided queries") + parser.add_argument("--exclude_types", type=str, nargs='+', dest='exclude_types', help="Exclude queries by question type. Queries matching these question types will be skipped. Example: --exclude_types comparision_query") parser.add_argument("--resume_checkpoint", type=str, help="Path to a checkpoint file to resume evaluation from previously saved progress") parser.add_argument("--keep_checkpoint", action="store_true", help="Keep the checkpoint file after successful evaluation instead of deleting it") parser.add_argument("--llm_judge_endpoint", type=str, default="http://localhost:8008", help="URL of the LLM judge service. Only used for RAGAS metrics") @@ -475,6 +483,29 @@ def filter_category_null_queries(queries): return [q for q in queries if q.get("question_type") != 'null_query'] + +def filter_queries_by_type(queries, exclude_types=None): + """ + Filter queries by excluding specific question types. + + Args: + queries: List of query dictionaries + exclude_types: List of question types to exclude (if None, exclude none) + + Returns: + Filtered list of queries + """ + if not exclude_types: + return queries + + logger.info(f"Excluding question types: {exclude_types}") + # Normalize exclude_types to lowercase and strip whitespace for case-insensitive comparison + normalized_exclude_types = {qt.lower().strip() for qt in exclude_types} + filtered = [q for q in queries if q.get("question_type", "").lower().strip() not in normalized_exclude_types] + + return filtered + + def main(): args = args_parser() logger.info(f"Running Multihop evaluation with arguments: {args.__dict__}") @@ -536,8 +567,13 @@ def main(): all_queries = filter_category_null_queries(all_queries) logger.info(f"Queries remaining: {len(all_queries)}") + # Filter by question type if specified + if args.exclude_types: + all_queries = filter_queries_by_type(all_queries, args.exclude_types) + logger.info(f"Queries after type filtering: {len(all_queries)}") + except Exception as e: - logger.error(f"Error filtering queries categorized as 'null_query': {e}") + logger.error(f"Error filtering queries: {e}") if not all_queries: logger.error("No queries remain after filtering 'null_query' category. Please check the dataset.") From b3facecf7bff65b30478ae63c9fc37a8ecabbe46 Mon Sep 17 00:00:00 2001 From: Github Actions Date: Wed, 7 Jan 2026 10:31:26 +0000 Subject: [PATCH 06/10] Patch code up to 91af06f1 Signed-off-by: Github Actions --- src/tests/e2e/lifecycle/test_uninstall.py | 2 +- .../e2e/validation/test_api_health_checks.py | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/tests/e2e/lifecycle/test_uninstall.py b/src/tests/e2e/lifecycle/test_uninstall.py index f42ad5fea..4ce10fed0 100644 --- a/src/tests/e2e/lifecycle/test_uninstall.py +++ b/src/tests/e2e/lifecycle/test_uninstall.py @@ -25,7 +25,7 @@ def disable_guards_at_startup(guard_helper, suppress_logging, temporarily_remove @allure.testcase("IEASG-T309") def test_uninstall(k8s_helper): """Verify that after uninstallation, only the allowed namespaces and no unexpected pods exist.""" - allowed_namespaces = ["default", "kube-system", "kube-public", "kube-node-lease", "local-path-storage"] + allowed_namespaces = ["default", "habana-system", "kube-system", "kube-public", "kube-node-lease", "local-path-storage"] current_namespaces = k8s_helper.list_namespaces() logger.debug(f"Namespaces in the cluster: {current_namespaces}") unexpected_ns = [ns for ns in current_namespaces if ns not in allowed_namespaces] diff --git a/src/tests/e2e/validation/test_api_health_checks.py b/src/tests/e2e/validation/test_api_health_checks.py index a69c3e422..a5d3a05d9 100644 --- a/src/tests/e2e/validation/test_api_health_checks.py +++ b/src/tests/e2e/validation/test_api_health_checks.py @@ -52,11 +52,18 @@ def test_api_health_checks(generic_api_helper): try: response = generic_api_helper.call_health_check_api( service['namespace'], service['selector'], service['port'], service['health_path']) - assert response.status_code == 200, \ - f"Got unexpected status code for {service['selector']} health check API call" - except (AssertionError, requests.exceptions.RequestException) as e: - logger.warning(e) + if response.status_code != 200: + error_msg = ( + f"Health check failed for {service['selector']}. " + f"Status Code: {response.status_code}, " + f"Response Body: {response.text}" + ) + logger.error(error_msg) + assert response.status_code == 200, error_msg + + except (AssertionError, requests.exceptions.RequestException) as e: + logger.warning(f"Error during health check for {service['selector']}: {e}") failed_microservices.append(service) - assert failed_microservices == [], "/v1/health_check API call didn't succeed for some microservices" + assert failed_microservices == [], f"Health check failed for services: {failed_microservices}" From 31ffb5e8350f97da6b5aa9922c78cd4234f7c8b8 Mon Sep 17 00:00:00 2001 From: Github Actions Date: Wed, 7 Jan 2026 13:34:59 +0000 Subject: [PATCH 07/10] Patch code up to 838c329e Signed-off-by: Github Actions --- src/comps/reranks/utils/opea_reranking.py | 7 ++----- .../vectorstores/utils/connectors/connector_redis.py | 8 ++++++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/comps/reranks/utils/opea_reranking.py b/src/comps/reranks/utils/opea_reranking.py index 22a93f1c0..ceb487bfd 100644 --- a/src/comps/reranks/utils/opea_reranking.py +++ b/src/comps/reranks/utils/opea_reranking.py @@ -123,7 +123,7 @@ async def run(self, input: SearchedDoc) -> PromptTemplateInput: logger.debug(f"Received response from reranking service: {response_data}") best_response_list = self._filter_top_n(input.top_n, response_data, score_threshold=input.rerank_score_threshold) if len(best_response_list) != len(retrieved_texts): - logger.warning(f"Limiting the number of best responses to {input.top_n} based on {len(retrieved_texts)} retrieved documents using max score of {input.rerank_score_threshold}.") + logger.warning(f"Limiting the number of best responses to {len(best_response_list)} based on {len(retrieved_texts)} retrieved documents using max score of {input.rerank_score_threshold}.") logger.debug(f"Best responses after filtering: {best_response_list}") except TimeoutError as e: @@ -202,13 +202,10 @@ def _combine_sibling_docs(self, reranked_docs: DocList[TextDoc], sibling_docs: D if found: continue - # Copy metadata from the first element - combined_metadata = combined_docs[0].metadata.copy() - # Final result combined_element = TextDoc( text=combined_text, - metadata=combined_metadata + metadata=doc.metadata.copy() # Copy metadata from the original document ) all_combined_docs.append(combined_element) else: diff --git a/src/comps/vectorstores/utils/connectors/connector_redis.py b/src/comps/vectorstores/utils/connectors/connector_redis.py index faa138fbd..dbb50cf44 100644 --- a/src/comps/vectorstores/utils/connectors/connector_redis.py +++ b/src/comps/vectorstores/utils/connectors/connector_redis.py @@ -503,7 +503,9 @@ async def similarity_search_with_siblings(self, input_text: str, embedding: List if before_result.docs: prev_chunk = max(before_result.docs, key=lambda x: int(x.start_index) if hasattr(x, 'start_index') else 0) logger.debug(f"Retrieved previous chunk: {prev_chunk.id, prev_chunk.start_index}") - sibling_docs.append(self._convert_to_text_doc(prev_chunk)) + prev_doc = self._convert_to_text_doc(prev_chunk) + prev_doc.metadata["vector_distance"] = -1 # Siblings do not have a distance score + sibling_docs.append(prev_doc) after_filter = None if filter_expression is None: @@ -516,7 +518,9 @@ async def similarity_search_with_siblings(self, input_text: str, embedding: List if after_result.docs: next_chunk = min(after_result.docs, key=lambda x: int(x.start_index) if hasattr(x, 'start_index') else 0) logger.debug(f"Retrieved next chunk: {next_chunk.id, next_chunk.start_index}") - sibling_docs.append(self._convert_to_text_doc(next_chunk)) + next_doc = self._convert_to_text_doc(next_chunk) + next_doc.metadata["vector_distance"] = -1 # Siblings do not have a distance score + sibling_docs.append(next_doc) all_sibling_docs[doc.id] = sibling_docs From 8a92039917e5928a89259db2a3a5d2ab3f1245bf Mon Sep 17 00:00:00 2001 From: Github Actions Date: Wed, 7 Jan 2026 13:43:56 +0000 Subject: [PATCH 08/10] Patch code up to 10b2bed9 Signed-off-by: Github Actions --- .../dataprep_upload/test_post_backup.txt | 1 + .../files/dataprep_upload/test_pre_backup.txt | 1 + src/tests/e2e/helpers/k8s_helper.py | 16 +++- src/tests/e2e/helpers/keycloak_helper.py | 41 ++++++++++ src/tests/e2e/lifecycle/scenarios.yaml | 6 ++ src/tests/e2e/lifecycle/test_post_backup.py | 66 ++++++++++++++++ src/tests/e2e/lifecycle/test_post_restore.py | 76 +++++++++++++++++++ src/tests/e2e/lifecycle/test_pre_backup.py | 39 ++++++++++ 8 files changed, 244 insertions(+), 2 deletions(-) create mode 100644 src/tests/e2e/files/dataprep_upload/test_post_backup.txt create mode 100644 src/tests/e2e/files/dataprep_upload/test_pre_backup.txt create mode 100644 src/tests/e2e/lifecycle/test_post_backup.py create mode 100644 src/tests/e2e/lifecycle/test_post_restore.py create mode 100644 src/tests/e2e/lifecycle/test_pre_backup.py diff --git a/src/tests/e2e/files/dataprep_upload/test_post_backup.txt b/src/tests/e2e/files/dataprep_upload/test_post_backup.txt new file mode 100644 index 000000000..65d558fff --- /dev/null +++ b/src/tests/e2e/files/dataprep_upload/test_post_backup.txt @@ -0,0 +1 @@ +According to a 2024 report, the mole population in the city of Gdansk exceeded 1.2 million individuals, meaning there were on average four moles for every resident. \ No newline at end of file diff --git a/src/tests/e2e/files/dataprep_upload/test_pre_backup.txt b/src/tests/e2e/files/dataprep_upload/test_pre_backup.txt new file mode 100644 index 000000000..eb9233514 --- /dev/null +++ b/src/tests/e2e/files/dataprep_upload/test_pre_backup.txt @@ -0,0 +1 @@ +Elvira Marqens from the small village of Rulberton crafts a unique cheese called Trindlefoss. diff --git a/src/tests/e2e/helpers/k8s_helper.py b/src/tests/e2e/helpers/k8s_helper.py index 418ac293c..363facb42 100644 --- a/src/tests/e2e/helpers/k8s_helper.py +++ b/src/tests/e2e/helpers/k8s_helper.py @@ -32,10 +32,22 @@ def list_namespaces(self): namespaces = kr8s.get("namespaces") return [ns.name for ns in namespaces] + def get_namespace(self, namespace_name): + """Get a namespace by name""" + namespaces = kr8s.get("namespaces") + for ns in namespaces: + if ns.name == namespace_name: + return ns + return None + + def get_backups(self, namespace): + """Get a list of kr8s CustomResource objects representing backups in a namespace""" + logger.debug(f"Getting backups in namespace '{namespace}'") + return kr8s.get("backups", namespace=namespace) + def list_pods(self, namespace): """List all pods in the specified namespace""" - pods = kr8s.get("pods", namespace=namespace) - return [pod.name for pod in pods] + return kr8s.get("pods", namespace=namespace) def get_pod_by_label(self, namespace, label_selector): """Returns first pod matching a label selector in a namespace""" diff --git a/src/tests/e2e/helpers/keycloak_helper.py b/src/tests/e2e/helpers/keycloak_helper.py index 5a43649bd..f9e1e5031 100644 --- a/src/tests/e2e/helpers/keycloak_helper.py +++ b/src/tests/e2e/helpers/keycloak_helper.py @@ -207,3 +207,44 @@ def set_brute_force_detection(self, enabled: bool): response = requests.put(url, json=realm_settings, headers=headers, verify=False) if response.status_code != 204: raise Exception(f"Failed to update brute force detection (status {response.status_code}): {response.text}") + + def add_user(self, username, password, first_name="", last_name="", email=""): + """Add a new user to Keycloak""" + logger.info(f"Adding new user '{username}' to Keycloak") + url = f"{self.erag_auth_domain}/admin/realms/{VITE_KEYCLOAK_REALM}/users" + headers = { + "Authorization": f"Bearer {self.admin_access_token}", + "Content-Type": "application/json" + } + payload = { + "username": username, + "enabled": True, + "firstName": first_name, + "lastName": last_name, + "email": email, + "credentials": [{ + "type": "password", + "value": password, + "temporary": False + }] + } + response = requests.post(url, json=payload, headers=headers, verify=False) + if response.status_code != 201: + raise Exception(f"Failed to add user '{username}' (status {response.status_code}): {response.text}") + logger.info(f"User '{username}' added successfully") + + def user_exists(self, username): + """Check if a user exists in Keycloak""" + logger.info(f"Checking if user '{username}' exists in Keycloak") + url = f"{self.erag_auth_domain}/admin/realms/{VITE_KEYCLOAK_REALM}/users?username={username}" + headers = { + "Authorization": f"Bearer {self.admin_access_token}", + "Content-Type": "application/json" + } + response = requests.get(url, headers=headers, verify=False) + if response.status_code != 200: + raise Exception(f"Failed to check user existence (status {response.status_code}): {response.text}") + users = response.json() + exists = len(users) > 0 + logger.info(f"User '{username}' exists: {exists}") + return exists diff --git a/src/tests/e2e/lifecycle/scenarios.yaml b/src/tests/e2e/lifecycle/scenarios.yaml index 2f4e9dca6..4dfc6a284 100644 --- a/src/tests/e2e/lifecycle/scenarios.yaml +++ b/src/tests/e2e/lifecycle/scenarios.yaml @@ -28,3 +28,9 @@ scenarios: lifecycle: - test_update_embedding_model.py markers: "smoke" + + backup-restore: + lifecycle: + - test_pre_backup.py + - test_post_backup.py + - test_post_restore.py diff --git a/src/tests/e2e/lifecycle/test_post_backup.py b/src/tests/e2e/lifecycle/test_post_backup.py new file mode 100644 index 000000000..acc6c2ee9 --- /dev/null +++ b/src/tests/e2e/lifecycle/test_post_backup.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (C) 2025 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import allure +import logging +import os + +from tests.e2e.validation.constants import DATAPREP_UPLOAD_DIR + +logger = logging.getLogger(__name__) + + +@allure.testcase("IEASG-T313") +def test_post_backup_backup_created(k8s_helper): + """ + Check backup is properly created. + 1. check if "velero" namespace exists. + 2. check if there is at least one backup in velero namespace. + 3. check if backup is in "Completed" status (check all backups for the sake of simplicity). + 4. check if there are no "Error" or "CrashLoopBackOff" pods in "velero" namespace. + """ + velero_ns = k8s_helper.get_namespace("velero") + assert velero_ns is not None, "Velero namespace does not exist." + + backups = k8s_helper.get_backups(namespace="velero") + assert len(backups) > 0, "No backups found in velero namespace." + + for backup in backups: + logger.debug(f"Backup Name: {backup.name}, Status: {backup.status.phase}") + assert backup.status.phase == "Completed", f"Backup {backup.name} is not completed." + + velero_pods = k8s_helper.list_pods(namespace="velero") + for pod in velero_pods: + pod_status = pod.status.phase + logger.debug(f"Pod Name: {pod.name}, Status: {pod_status}") + assert pod_status not in ["Error", "CrashLoopBackOff"], f"Pod {pod.name} is in {pod_status} status." + + +@allure.testcase("IEASG-T314") +def test_post_backup_data_ingested_after_backup(keycloak_helper, edp_helper, chat_history_helper): + """ + Ingest some data which should not be present in the backup since backup was already created. + 1. Upload a file to the system via EDP. + 2. Create a chat history via Chat History API. + 3. Create a new user in Keycloak. + """ + # Upload a file + file = "test_post_backup.txt" + edp_helper.upload_file_and_wait_for_ingestion(os.path.join(DATAPREP_UPLOAD_DIR, file)) + + # Create chat history + chat_history_helper.save_history([ + { + "question": "How do voles affect the ecosystem near Gdansk Airport?", + "answer": "They serve as a crucial food source for various predators such as owls, " + "foxes, and snakes, helping maintain the balance of the food web.", + "metadata": {} + } + ]) + + # Create a new user + user = "post-backup-user-that-should-not-exist-after-restore" + if not keycloak_helper.user_exists(user): + keycloak_helper.add_user(user, "PostBackupPass123!", "PostBackupUser", "PostBackupSurname", "postbackup@example.com") diff --git a/src/tests/e2e/lifecycle/test_post_restore.py b/src/tests/e2e/lifecycle/test_post_restore.py new file mode 100644 index 000000000..4b60fdab0 --- /dev/null +++ b/src/tests/e2e/lifecycle/test_post_restore.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (C) 2025 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import allure +import logging +import pytest + +logger = logging.getLogger(__name__) + + +@allure.testcase("IEASG-T315") +def test_post_restore_data_exists(keycloak_helper, edp_helper, chat_history_helper, chatqa_api_helper): + """ + 1. Verify that the previously uploaded file is present in the system. + 2. Verify that the chat history is present. + 3. Verify that the previously created user exists. + 4. Ask a question related to the pre-backup data to ensure the system responds correctly. + """ + # Verify uploaded file exists + file_name = "test_pre_backup.txt" + files = edp_helper.list_files() + for item in files.json(): + if file_name in item['object_name']: + logger.debug(f"File found: {item['object_name']}") + break + else: + pytest.fail(f"File {file_name} not found after restore.") + + # Verify chat history exists + response = chat_history_helper.get_all_histories() + all_history_names = [history["history_name"] for history in response.json()] + logger.debug(f"All histories: {all_history_names}") + for history in all_history_names: + if "What is the name of the cheese" in history: + logger.debug(f"Chat history found: {history}") + break + else: + pytest.fail("Chat history not found after restore.") + + # Verify user + user = "backup-test" + assert keycloak_helper.user_exists(user), f"User {user} not found after restore" + + response = chatqa_api_helper.call_chatqa( + "What unique product does Elvira Marqens craft in the village of Rulberton?") + assert response.status_code == 200, "Unexpected status code returned" + response_text = chatqa_api_helper.get_text(response) + logger.info(f"ChatQA response: {response_text}") + assert "cheese" in response_text.lower(), "Unexpected answer from ChatQA after restore" + + +@allure.testcase("IEASG-T316") +def test_post_restore_data_does_not_exists(keycloak_helper, edp_helper, chat_history_helper): + """ + Check if data ingested after backup does not exist after restore. + """ + # Verify uploaded file does not exist + file_name = "test_post_backup.txt" + files = edp_helper.list_files() + for item in files.json(): + if file_name in item['object_name']: + pytest.fail(f"File {file_name} found after restore, but it should not be present.") + + # Verify chat history does not exist + response = chat_history_helper.get_all_histories() + all_history_names = [history["history_name"] for history in response.json()] + logger.debug(f"All histories: {all_history_names}") + for history in all_history_names: + if "How do voles affect" in history: + pytest.fail("Chat history found after restore, but it should not be present.") + + # Verify user does not exist + user = "post-backup-user-that-should-not-exist-after-restore" + assert not keycloak_helper.user_exists(user), f"User {user} found after restore, but it should not be present." diff --git a/src/tests/e2e/lifecycle/test_pre_backup.py b/src/tests/e2e/lifecycle/test_pre_backup.py new file mode 100644 index 000000000..139764db1 --- /dev/null +++ b/src/tests/e2e/lifecycle/test_pre_backup.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (C) 2025 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import allure +import logging +import os + +from tests.e2e.validation.constants import DATAPREP_UPLOAD_DIR + +logger = logging.getLogger(__name__) + + +@allure.testcase("IEASG-T312") +def test_pre_backup(keycloak_helper, edp_helper, chat_history_helper): + """ + Prepare data for backup: + 1. Upload a file to the system via EDP. + 2. Create a chat history via Chat History API. + 3. Create a new user in Keycloak. + """ + # Populate db + file = "test_pre_backup.txt" + edp_helper.upload_file_and_wait_for_ingestion(os.path.join(DATAPREP_UPLOAD_DIR, file)) + + # Populate chat history + chat_history_helper.save_history([ + { + "question": "What is the name of the cheese produced by Elvira Marqens from the village of Rulberton?", + "answer": "Trindlefoss.", + "metadata": {} + } + ]) + + # Add new user if it does not exist + user = "backup-test" + if not keycloak_helper.user_exists(user): + keycloak_helper.add_user(user, "PreBackupPass123!", "BackupUser", "BackupSurname", "backup@example.com") From 17ea8505577f6428e2d3f51d4ad858b9d83492b4 Mon Sep 17 00:00:00 2001 From: Github Actions Date: Wed, 7 Jan 2026 14:16:18 +0000 Subject: [PATCH 09/10] Patch code up to d4fefff8 Signed-off-by: Github Actions --- src/edp/app/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/edp/app/utils.py b/src/edp/app/utils.py index 40250c650..46c9b71cb 100644 --- a/src/edp/app/utils.py +++ b/src/edp/app/utils.py @@ -30,7 +30,8 @@ def get_local_minio_client_using_token_credentials(jwt_token, verify=False): raise ValueError("JWT token does not contain access_token") credentials = WebIdentityProvider( jwt_provider_func=lambda: jwt_token, - sts_endpoint=os.getenv('EDP_STS_ENDPOINT', endpoint) + sts_endpoint=os.getenv('EDP_STS_ENDPOINT', endpoint), + http_client=get_http_client(endpoint, cert_check) ) return get_minio_client(endpoint, region, cert_check, credentials) From 5154b3d5e33746991df44d080443b330e5ad7fd0 Mon Sep 17 00:00:00 2001 From: Github Actions Date: Thu, 8 Jan 2026 13:27:32 +0000 Subject: [PATCH 10/10] Patch code up to e5bac6c4 Signed-off-by: Github Actions --- deployment/inventory/sample/config.yaml | 1 - .../infrastructure/netapp_trident_csi_setup/defaults/main.yaml | 1 - .../netapp_trident_csi_setup/netapp_trident_integration.md | 3 --- .../infrastructure/netapp_trident_csi_setup/tasks/main.yaml | 2 ++ .../netapp_trident_csi_setup/templates/trident-backend.yaml.j2 | 1 - 5 files changed, 2 insertions(+), 6 deletions(-) diff --git a/deployment/inventory/sample/config.yaml b/deployment/inventory/sample/config.yaml index 5be34d858..5d479bb4f 100644 --- a/deployment/inventory/sample/config.yaml +++ b/deployment/inventory/sample/config.yaml @@ -37,7 +37,6 @@ ontap_data_lif: "" # ONTAP data LIF IP address ontap_svm: "" # Storage Virtual Machine (SVM) name ontap_username: "" # ONTAP username with admin privileges ontap_password: "" # ONTAP password -ontap_aggregate: "" # ONTAP aggregate name for volume creation huggingToken: "" # Huggingface token is required for gated/private models diff --git a/deployment/roles/infrastructure/netapp_trident_csi_setup/defaults/main.yaml b/deployment/roles/infrastructure/netapp_trident_csi_setup/defaults/main.yaml index 99c80dcd5..7ba71684a 100644 --- a/deployment/roles/infrastructure/netapp_trident_csi_setup/defaults/main.yaml +++ b/deployment/roles/infrastructure/netapp_trident_csi_setup/defaults/main.yaml @@ -10,4 +10,3 @@ ontap_data_lif: "" ontap_svm: "" ontap_username: "" ontap_password: "" -ontap_aggregate: "" diff --git a/deployment/roles/infrastructure/netapp_trident_csi_setup/netapp_trident_integration.md b/deployment/roles/infrastructure/netapp_trident_csi_setup/netapp_trident_integration.md index 838432dc6..acb77bb70 100644 --- a/deployment/roles/infrastructure/netapp_trident_csi_setup/netapp_trident_integration.md +++ b/deployment/roles/infrastructure/netapp_trident_csi_setup/netapp_trident_integration.md @@ -161,7 +161,6 @@ ontap_data_lif: "" ontap_svm: "" ontap_username: "" ontap_password: "" -ontap_aggregate: "" ``` ## New Ansible Role Created @@ -216,7 +215,6 @@ spec: svm: {{ ontap_svm }} username: {{ ontap_username }} password: {{ ontap_password }} - aggregate: {{ ontap_aggregate }} ``` [**trident-storageclass.yaml.j2**](https://github.com/sushma-m1/Enterprise-RAG/blob/main/deployment/roles/infrastructure/netapp_trident_csi_setup/templates/trident-storageclass.yaml.j2): Kubernetes StorageClass @@ -263,7 +261,6 @@ ontap_data_lif: "192.168.1.101" # ONTAP data interface IP ontap_svm: "svm_ai" # Storage Virtual Machine name ontap_username: "admin" # ONTAP admin username ontap_password: "password123" # ONTAP admin password -ontap_aggregate: "aggr1" # Target aggregate for volumes ``` ## Deployment Process diff --git a/deployment/roles/infrastructure/netapp_trident_csi_setup/tasks/main.yaml b/deployment/roles/infrastructure/netapp_trident_csi_setup/tasks/main.yaml index 434565103..1959c7297 100644 --- a/deployment/roles/infrastructure/netapp_trident_csi_setup/tasks/main.yaml +++ b/deployment/roles/infrastructure/netapp_trident_csi_setup/tasks/main.yaml @@ -27,6 +27,8 @@ release_namespace: "{{ trident_namespace }}" create_namespace: true chart_version: "100.{{ trident_operator_version }}" + update_repo_cache: true + wait: true tags: - install - post-install diff --git a/deployment/roles/infrastructure/netapp_trident_csi_setup/templates/trident-backend.yaml.j2 b/deployment/roles/infrastructure/netapp_trident_csi_setup/templates/trident-backend.yaml.j2 index 0ff69bc22..9fae57e53 100644 --- a/deployment/roles/infrastructure/netapp_trident_csi_setup/templates/trident-backend.yaml.j2 +++ b/deployment/roles/infrastructure/netapp_trident_csi_setup/templates/trident-backend.yaml.j2 @@ -10,6 +10,5 @@ spec: managementLIF: {{ ontap_management_lif }} dataLIF: {{ ontap_data_lif }} svm: {{ ontap_svm }} - aggregate: {{ ontap_aggregate }} credentials: name: {{ trident_backend_name }}-secret