diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 714712322..228a71946 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -18,6 +18,7 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} E2E_OPENAI_MODEL: ${{ vars.E2E_OPENAI_MODEL }} E2E_DEPLOYMENT_MODE: ${{ matrix.mode }} + FAISS_VECTOR_STORE_ID: ${{ vars.FAISS_VECTOR_STORE_ID }} steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/e2e_tests_providers.yaml b/.github/workflows/e2e_tests_providers.yaml index 8209a3b08..82886d6a2 100644 --- a/.github/workflows/e2e_tests_providers.yaml +++ b/.github/workflows/e2e_tests_providers.yaml @@ -24,6 +24,7 @@ jobs: CLIENT_ID: ${{ secrets.CLIENT_ID }} TENANT_ID: ${{ secrets.TENANT_ID }} E2E_DEPLOYMENT_MODE: ${{ matrix.mode }} + FAISS_VECTOR_STORE_ID: ${{ vars.FAISS_VECTOR_STORE_ID }} steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/e2e_tests_rhaiis.yaml b/.github/workflows/e2e_tests_rhaiis.yaml index 9ca10015a..3521d550e 100644 --- a/.github/workflows/e2e_tests_rhaiis.yaml +++ b/.github/workflows/e2e_tests_rhaiis.yaml @@ -18,6 +18,7 @@ jobs: RHAIIS_URL: ${{ secrets.RHAIIS_URL }} RHAIIS_API_KEY: ${{ secrets.RHAIIS_API_KEY }} RHAIIS_MODEL: ${{ vars.RHAIIS_MODEL }} + FAISS_VECTOR_STORE_ID: ${{ vars.FAISS_VECTOR_STORE_ID }} steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/e2e_tests_rhelai.yaml b/.github/workflows/e2e_tests_rhelai.yaml index da6c6d2b9..c9717bf62 100644 --- a/.github/workflows/e2e_tests_rhelai.yaml +++ b/.github/workflows/e2e_tests_rhelai.yaml @@ -18,6 +18,7 @@ jobs: RHEL_AI_API_KEY: ${{ secrets.RHEL_AI_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} RHEL_AI_MODEL: ${{ vars.RHEL_AI_MODEL }} + FAISS_VECTOR_STORE_ID: ${{ vars.FAISS_VECTOR_STORE_ID }} steps: - uses: actions/checkout@v4 diff --git a/docker-compose-library.yaml b/docker-compose-library.yaml index 9c934b89a..0be753152 100644 --- a/docker-compose-library.yaml +++ b/docker-compose-library.yaml @@ -66,6 +66,8 @@ services: - WATSONX_API_KEY=${WATSONX_API_KEY:-} # Enable debug logging if needed - LLAMA_STACK_LOGGING=${LLAMA_STACK_LOGGING:-} + # FAISS test + - FAISS_VECTOR_STORE_ID=${FAISS_VECTOR_STORE_ID:-} healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8080/liveness"] interval: 10s # how often to run the check diff --git a/docker-compose.yaml b/docker-compose.yaml index b1e3f819c..ff3fd7ff5 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -60,6 +60,8 @@ services: - WATSONX_API_KEY=${WATSONX_API_KEY:-} # Enable debug logging if needed - LLAMA_STACK_LOGGING=${LLAMA_STACK_LOGGING:-} + # FAISS test + - FAISS_VECTOR_STORE_ID=${FAISS_VECTOR_STORE_ID:-} networks: - lightspeednet healthcheck: diff --git a/tests/configuration/run.yaml b/tests/configuration/run.yaml index 08ca11ca9..a0668bb90 100644 --- a/tests/configuration/run.yaml +++ b/tests/configuration/run.yaml @@ -98,7 +98,7 @@ providers: - config: persistence: namespace: vector_io::faiss - backend: kv_default + backend: kv_rag provider_id: faiss provider_type: inline::faiss scoring_fns: [] @@ -147,4 +147,4 @@ vector_stores: default_provider_id: faiss default_embedding_model: provider_id: sentence-transformers - model_id: nomic-ai/nomic-embed-text-v1.5 + model_id: all-mpnet-base-v2 diff --git a/tests/e2e-prow/rhoai/configs/run.yaml b/tests/e2e-prow/rhoai/configs/run.yaml index a2e009e0b..20f11f547 100644 --- a/tests/e2e-prow/rhoai/configs/run.yaml +++ b/tests/e2e-prow/rhoai/configs/run.yaml @@ -58,7 +58,7 @@ providers: - config: persistence: namespace: vector_io::faiss - backend: kv_default + backend: kv_rag provider_id: faiss provider_type: inline::faiss agents: @@ -131,6 +131,17 @@ registered_resources: provider_id: vllm model_type: llm provider_model_id: null + - model_id: all-mpnet-base-v2 + model_type: embedding + provider_id: sentence-transformers + provider_model_id: all-mpnet-base-v2 + metadata: + embedding_dimension: 768 + vector_stores: + - embedding_dimension: 768 + embedding_model: sentence-transformers/all-mpnet-base-v2 + provider_id: faiss + vector_store_id: vs_8c94967b-81cc-4028-a294-9cfac6fd9ae2 #TODO: parse this from the rag db shields: - shield_id: llama-guard provider_id: llama-guard @@ -145,6 +156,6 @@ vector_stores: default_provider_id: faiss default_embedding_model: provider_id: sentence-transformers - model_id: nomic-ai/nomic-embed-text-v1.5 + model_id: all-mpnet-base-v2 safety: default_shield_id: llama-guard diff --git a/tests/e2e/configs/run-azure.yaml b/tests/e2e/configs/run-azure.yaml index 25dfe1e22..bca3e4583 100644 --- a/tests/e2e/configs/run-azure.yaml +++ b/tests/e2e/configs/run-azure.yaml @@ -57,7 +57,7 @@ providers: - config: persistence: namespace: vector_io::faiss - backend: kv_default + backend: kv_rag provider_id: faiss provider_type: inline::faiss agents: @@ -105,7 +105,10 @@ storage: backends: kv_default: type: kv_sqlite - db_path: ${env.KV_STORE_PATH:=~/.llama/storage/rag/kv_store.db} + db_path: ${env.KV_STORE_PATH:=~/.llama/storage/kv_store.db} + kv_rag: # Define the storage backend type for RAG + type: kv_sqlite + db_path: ${env.KV_RAG_PATH:=~/.llama/storage/rag/kv_store.db} sql_default: type: sql_sqlite db_path: ${env.SQL_STORE_PATH:=~/.llama/storage/sql_store.db} @@ -130,10 +133,21 @@ registered_resources: provider_id: azure model_type: llm provider_model_id: gpt-4o-mini + - model_id: all-mpnet-base-v2 + model_type: embedding + provider_id: sentence-transformers + provider_model_id: all-mpnet-base-v2 + metadata: + embedding_dimension: 768 shields: - shield_id: llama-guard provider_id: llama-guard provider_shield_id: openai/gpt-4o-mini + vector_stores: + - embedding_dimension: 768 + embedding_model: sentence-transformers/all-mpnet-base-v2 + provider_id: faiss + vector_store_id: ${env.FAISS_VECTOR_STORE_ID} datasets: [] scoring_fns: [] benchmarks: [] @@ -144,6 +158,6 @@ vector_stores: default_provider_id: faiss default_embedding_model: provider_id: sentence-transformers - model_id: nomic-ai/nomic-embed-text-v1.5 + model_id: all-mpnet-base-v2 safety: default_shield_id: llama-guard diff --git a/tests/e2e/configs/run-ci.yaml b/tests/e2e/configs/run-ci.yaml index c5ef1ad87..cc16560f6 100644 --- a/tests/e2e/configs/run-ci.yaml +++ b/tests/e2e/configs/run-ci.yaml @@ -62,7 +62,7 @@ providers: - config: # Define the storage backend for RAG persistence: namespace: vector_io::faiss - backend: kv_default + backend: kv_rag provider_id: faiss provider_type: inline::faiss agents: @@ -108,9 +108,12 @@ server: port: 8321 storage: backends: - kv_default: # Define the storage backend type for RAG, in this case registry and RAG are unified i.e. information on registered resources (e.g. models, vector_stores) are saved together with the RAG chunks + kv_default: + type: kv_sqlite + db_path: ${env.KV_STORE_PATH:=~/.llama/storage/kv_store.db} + kv_rag: # Define the storage backend type for RAG type: kv_sqlite - db_path: ${env.KV_STORE_PATH:=~/.llama/storage/rag/kv_store.db} + db_path: ${env.KV_RAG_PATH:=~/.llama/storage/rag/kv_store.db} sql_default: type: sql_sqlite db_path: ${env.SQL_STORE_PATH:=~/.llama/storage/sql_store.db} @@ -131,22 +134,21 @@ storage: backend: kv_default registered_resources: models: - - model_id: sentence-transformers/all-mpnet-base-v2 + - model_id: all-mpnet-base-v2 model_type: embedding provider_id: sentence-transformers - provider_model_id: sentence-transformers/all-mpnet-base-v2 + provider_model_id: all-mpnet-base-v2 metadata: embedding_dimension: 768 - # Commented out because the vector_store is already registered in the kv_store - # vector_stores: - # - embedding_dimension: 768 - # embedding_model: sentence-transformers/nomic-ai/nomic-embed-text-v1.5 - # provider_id: faiss - # vector_store_id: vs_503a2261-c256-45ff-90aa-580a80de64b8 shields: - shield_id: llama-guard provider_id: llama-guard provider_shield_id: openai/gpt-4o-mini + vector_stores: + - embedding_dimension: 768 + embedding_model: sentence-transformers/all-mpnet-base-v2 + provider_id: faiss + vector_store_id: ${env.FAISS_VECTOR_STORE_ID} datasets: [] scoring_fns: [] benchmarks: [] @@ -157,6 +159,6 @@ vector_stores: default_provider_id: faiss default_embedding_model: # Define the default embedding model for RAG provider_id: sentence-transformers - model_id: nomic-ai/nomic-embed-text-v1.5 + model_id: all-mpnet-base-v2 safety: default_shield_id: llama-guard diff --git a/tests/e2e/configs/run-rhaiis.yaml b/tests/e2e/configs/run-rhaiis.yaml index 02e120e63..5bc7b0ad5 100644 --- a/tests/e2e/configs/run-rhaiis.yaml +++ b/tests/e2e/configs/run-rhaiis.yaml @@ -58,7 +58,7 @@ providers: - config: persistence: namespace: vector_io::faiss - backend: kv_default + backend: kv_rag provider_id: faiss provider_type: inline::faiss agents: @@ -106,7 +106,10 @@ storage: backends: kv_default: type: kv_sqlite - db_path: ${env.KV_STORE_PATH:=~/.llama/storage/rag/kv_store.db} + db_path: ${env.KV_STORE_PATH:=~/.llama/storage/kv_store.db} + kv_rag: # Define the storage backend type for RAG + type: kv_sqlite + db_path: ${env.KV_RAG_PATH:=~/.llama/storage/rag/kv_store.db} sql_default: type: sql_sqlite db_path: ${env.SQL_STORE_PATH:=~/.llama/storage/sql_store.db} @@ -131,10 +134,21 @@ registered_resources: provider_id: vllm model_type: llm provider_model_id: ${env.RHAIIS_MODEL} + - model_id: all-mpnet-base-v2 + model_type: embedding + provider_id: sentence-transformers + provider_model_id: all-mpnet-base-v2 + metadata: + embedding_dimension: 768 shields: - shield_id: llama-guard provider_id: llama-guard provider_shield_id: openai/gpt-4o-mini + vector_stores: + - embedding_dimension: 768 + embedding_model: sentence-transformers/all-mpnet-base-v2 + provider_id: faiss + vector_store_id: ${env.FAISS_VECTOR_STORE_ID} datasets: [] scoring_fns: [] benchmarks: [] @@ -145,6 +159,6 @@ vector_stores: default_provider_id: faiss default_embedding_model: provider_id: sentence-transformers - model_id: nomic-ai/nomic-embed-text-v1.5 + model_id: all-mpnet-base-v2 safety: default_shield_id: llama-guard diff --git a/tests/e2e/configs/run-rhelai.yaml b/tests/e2e/configs/run-rhelai.yaml index c363a8b7b..64d93b841 100644 --- a/tests/e2e/configs/run-rhelai.yaml +++ b/tests/e2e/configs/run-rhelai.yaml @@ -58,7 +58,7 @@ providers: - config: persistence: namespace: vector_io::faiss - backend: kv_default + backend: kv_rag provider_id: faiss provider_type: inline::faiss agents: @@ -106,7 +106,10 @@ storage: backends: kv_default: type: kv_sqlite - db_path: ${env.KV_STORE_PATH:=~/.llama/storage/rag/kv_store.db} + db_path: ${env.KV_STORE_PATH:=~/.llama/storage/kv_store.db} + kv_rag: # Define the storage backend type for RAG + type: kv_sqlite + db_path: ${env.KV_RAG_PATH:=~/.llama/storage/rag/kv_store.db} sql_default: type: sql_sqlite db_path: ${env.SQL_STORE_PATH:=~/.llama/storage/sql_store.db} @@ -131,10 +134,21 @@ registered_resources: provider_id: vllm model_type: llm provider_model_id: ${env.RHEL_AI_MODEL} + - model_id: all-mpnet-base-v2 + model_type: embedding + provider_id: sentence-transformers + provider_model_id: all-mpnet-base-v2 + metadata: + embedding_dimension: 768 shields: - shield_id: llama-guard provider_id: llama-guard provider_shield_id: openai/gpt-4o-mini + vector_stores: + - embedding_dimension: 768 + embedding_model: sentence-transformers/all-mpnet-base-v2 + provider_id: faiss + vector_store_id: ${env.FAISS_VECTOR_STORE_ID} datasets: [] scoring_fns: [] benchmarks: [] @@ -145,6 +159,6 @@ vector_stores: default_provider_id: faiss default_embedding_model: provider_id: sentence-transformers - model_id: nomic-ai/nomic-embed-text-v1.5 + model_id: all-mpnet-base-v2 safety: default_shield_id: llama-guard diff --git a/tests/e2e/configs/run-vertexai.yaml b/tests/e2e/configs/run-vertexai.yaml index 301ee9b46..bfa69b40a 100644 --- a/tests/e2e/configs/run-vertexai.yaml +++ b/tests/e2e/configs/run-vertexai.yaml @@ -57,7 +57,7 @@ providers: - config: persistence: namespace: vector_io::faiss - backend: kv_default + backend: kv_rag provider_id: faiss provider_type: inline::faiss agents: @@ -105,7 +105,10 @@ storage: backends: kv_default: type: kv_sqlite - db_path: ${env.KV_STORE_PATH:=~/.llama/storage/rag/kv_store.db} + db_path: ${env.KV_STORE_PATH:=~/.llama/storage/kv_store.db} + kv_rag: # Define the storage backend type for RAG + type: kv_sqlite + db_path: ${env.KV_RAG_PATH:=~/.llama/storage/rag/kv_store.db} sql_default: type: sql_sqlite db_path: ${env.SQL_STORE_PATH:=~/.llama/storage/sql_store.db} @@ -125,11 +128,22 @@ storage: namespace: prompts backend: kv_default registered_resources: - models: [] + models: + - model_id: all-mpnet-base-v2 + model_type: embedding + provider_id: sentence-transformers + provider_model_id: all-mpnet-base-v2 + metadata: + embedding_dimension: 768 shields: - shield_id: llama-guard provider_id: llama-guard provider_shield_id: openai/gpt-4o-mini + vector_stores: + - embedding_dimension: 768 + embedding_model: sentence-transformers/all-mpnet-base-v2 + provider_id: faiss + vector_store_id: ${env.FAISS_VECTOR_STORE_ID} datasets: [] scoring_fns: [] benchmarks: [] @@ -140,6 +154,6 @@ vector_stores: default_provider_id: faiss default_embedding_model: provider_id: sentence-transformers - model_id: nomic-ai/nomic-embed-text-v1.5 + model_id: all-mpnet-base-v2 safety: default_shield_id: llama-guard diff --git a/tests/e2e/configs/run-watsonx.yaml b/tests/e2e/configs/run-watsonx.yaml index c848e2ce2..4f02853f7 100644 --- a/tests/e2e/configs/run-watsonx.yaml +++ b/tests/e2e/configs/run-watsonx.yaml @@ -65,7 +65,7 @@ providers: - config: # Define the storage backend for RAG persistence: namespace: vector_io::faiss - backend: kv_default + backend: kv_rag provider_id: faiss provider_type: inline::faiss agents: @@ -111,12 +111,15 @@ server: port: 8321 storage: backends: - kv_default: # Define the storage backend type for RAG, in this case registry and RAG are unified i.e. information on registered resources (e.g. models, vector_stores) are saved together with the RAG chunks + kv_default: + type: kv_sqlite + db_path: ${env.KV_STORE_PATH:=~/.llama/storage/kv_store.db} + kv_rag: # Define the storage backend type for RAG type: kv_sqlite - db_path: ${env.SQLITE_STORE_DIR:=~/.llama/storage/rag/kv_store.db} + db_path: ${env.KV_RAG_PATH:=~/.llama/storage/rag/kv_store.db} sql_default: type: sql_sqlite - db_path: ${env.SQLITE_STORE_DIR:=~/.llama/storage/sql_store.db} + db_path: ${env.SQL_STORE_PATH:=~/.llama/storage/sql_store.db} stores: metadata: namespace: registry @@ -138,10 +141,21 @@ registered_resources: provider_id: watsonx model_type: llm provider_model_id: watsonx/meta-llama/llama-3-3-70b-instruct + - model_id: all-mpnet-base-v2 + model_type: embedding + provider_id: sentence-transformers + provider_model_id: all-mpnet-base-v2 + metadata: + embedding_dimension: 768 shields: - shield_id: llama-guard provider_id: llama-guard provider_shield_id: openai/gpt-4o-mini + vector_stores: + - embedding_dimension: 768 + embedding_model: sentence-transformers/all-mpnet-base-v2 + provider_id: faiss + vector_store_id: ${env.FAISS_VECTOR_STORE_ID} vector_stores: [] datasets: [] scoring_fns: [] @@ -153,4 +167,4 @@ vector_stores: default_provider_id: faiss default_embedding_model: # Define the default embedding model for RAG provider_id: sentence-transformers - model_id: nomic-ai/nomic-embed-text-v1.5 + model_id: all-mpnet-base-v2 diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index 3df842f66..3d6a4fdae 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -83,6 +83,8 @@ def before_all(context: Context) -> None: model_override = os.getenv("E2E_DEFAULT_MODEL_OVERRIDE") provider_override = os.getenv("E2E_DEFAULT_PROVIDER_OVERRIDE") + context.faiss_vector_store_id = os.getenv("FAISS_VECTOR_STORE_ID") + # Only override if the variables contain actual values (skip if empty) if model_override and provider_override: context.default_model = model_override diff --git a/tests/e2e/features/faiss.feature b/tests/e2e/features/faiss.feature index 4465e71d2..dd676c5af 100644 --- a/tests/e2e/features/faiss.feature +++ b/tests/e2e/features/faiss.feature @@ -14,7 +14,7 @@ Feature: FAISS support tests """ { "rags": [ - "vs_503a2261-c256-45ff-90aa-580a80de64b8" + "{VECTOR_STORE_ID}" ] } """ diff --git a/tests/e2e/features/steps/common_http.py b/tests/e2e/features/steps/common_http.py index aed1c4336..d35dc6e79 100644 --- a/tests/e2e/features/steps/common_http.py +++ b/tests/e2e/features/steps/common_http.py @@ -383,5 +383,6 @@ def check_response_partially(context: Context) -> None: """ assert context.response is not None, "Request needs to be performed first" body = context.response.json() - expected = json.loads(context.text or "{}") + json_str = replace_placeholders(context, context.text or "{}") + expected = json.loads(json_str) validate_json_partially(body, expected) diff --git a/tests/e2e/rag/kv_store.db b/tests/e2e/rag/kv_store.db index 7ad99e125..187f497e0 100644 Binary files a/tests/e2e/rag/kv_store.db and b/tests/e2e/rag/kv_store.db differ diff --git a/tests/e2e/utils/utils.py b/tests/e2e/utils/utils.py index 5b0b33573..9c11dd9a6 100644 --- a/tests/e2e/utils/utils.py +++ b/tests/e2e/utils/utils.py @@ -4,8 +4,9 @@ import shutil import subprocess import time -import jsonschema from typing import Any + +import jsonschema from behave.runner import Context @@ -254,4 +255,5 @@ def replace_placeholders(context: Context, text: str) -> str: """ result = text.replace("{MODEL}", context.default_model) result = result.replace("{PROVIDER}", context.default_provider) + result = result.replace("{VECTOR_STORE_ID}", context.faiss_vector_store_id) return result