diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..e7491eb --- /dev/null +++ b/tests/README.md @@ -0,0 +1,75 @@ +# Playwright Tests + +This directory contains end-to-end tests for the Cosmic Food RAG App using Playwright. + +## Test Files + +- `test_playwright.py` - Main Playwright test file containing UI tests + +## Test Coverage + +The tests cover the following functionality: + +1. **Home Page Test** (`test_home`) + - Verifies the application loads correctly + - Checks the page title + +2. **Basic Chat Test** (`test_chat`) + - Tests basic chat functionality with streaming responses + - Mocks the chat/stream endpoint + - Verifies chat input, submission, and response display + - Tests clear chat functionality + +3. **Chat Customization Test** (`test_chat_customization`) + - Tests opening developer settings + - Verifies chat works with non-streaming responses + +4. **Non-streaming Chat Test** (`test_chat_nonstreaming`) + - Tests chat with mocked non-streaming responses + - Uses the /chat endpoint instead of /chat/stream + +## Mock Data + +Mock response data is stored in `tests/snapshots/test_playwright/`: + +- `test_chat_flow/chat_flow_response.json` - Non-streaming chat response +- `test_chat_streaming_flow/chat_streaming_flow_response.jsonlines` - Streaming chat response +- `test_simple_chat_flow/simple_chat_flow_response.json` - Simple chat response + +## Running Tests + +### Prerequisites + +- Python dependencies: `pytest`, `playwright`, `pytest-playwright` +- Chromium browser installed via Playwright + +### Running All Playwright Tests + +```bash +python -m pytest tests/test_playwright.py -v --browser chromium +``` + +### Running Individual Tests + +```bash +# Home page test +python -m pytest tests/test_playwright.py::test_home -v --browser chromium + +# Chat tests +python -m pytest tests/test_playwright.py::test_chat -v --browser chromium +``` + +### Test Server + +The tests use a test server that: +- Runs the Quart application in test mode +- Bypasses OpenAI and Cosmos DB connections +- Uses mock responses for chat functionality +- Automatically starts and stops for each test + +## Implementation Notes + +- Tests run in headless mode by default +- The server runs on a random available port for each test session +- Mock responses are intercepted at the network level using Playwright's route mocking +- Tests are designed to be independent and can run in any order \ No newline at end of file diff --git a/tests/snapshots/test_playwright/test_chat_flow/chat_flow_response.json b/tests/snapshots/test_playwright/test_chat_flow/chat_flow_response.json new file mode 100644 index 0000000..6a05dd7 --- /dev/null +++ b/tests/snapshots/test_playwright/test_chat_flow/chat_flow_response.json @@ -0,0 +1,35 @@ +{ + "context": { + "data_points": [ + { + "name": "Quinoa Buddha Bowl", + "description": "Fresh quinoa with roasted vegetables, avocado, and tahini dressing", + "price": "$12.99", + "category": "Vegetarian", + "collection": "main-dishes" + }, + { + "name": "Roasted Vegetable Pasta", + "description": "House-made pasta with seasonal roasted vegetables in olive oil", + "price": "$14.99", + "category": "Vegetarian", + "collection": "main-dishes" + } + ], + "thoughts": [ + { + "title": "Search Query Generation", + "description": "Generated search query for vegetarian dishes in our menu database" + }, + { + "title": "Document Retrieval", + "description": "Found 2 relevant vegetarian dishes matching the customer's request" + } + ] + }, + "message": { + "content": "We have delicious vegetarian options including our signature Quinoa Buddha Bowl with fresh quinoa, roasted vegetables, avocado, and tahini dressing for $12.99, and our Roasted Vegetable Pasta with house-made pasta and seasonal roasted vegetables in olive oil for $14.99.", + "role": "assistant" + }, + "sessionState": null +} \ No newline at end of file diff --git a/tests/snapshots/test_playwright/test_chat_streaming_flow/chat_streaming_flow_response.jsonlines b/tests/snapshots/test_playwright/test_chat_streaming_flow/chat_streaming_flow_response.jsonlines new file mode 100644 index 0000000..7e865eb --- /dev/null +++ b/tests/snapshots/test_playwright/test_chat_streaming_flow/chat_streaming_flow_response.jsonlines @@ -0,0 +1,41 @@ +{"context": null, "delta": {"content": "", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": "We", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " have", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " delicious", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " vegetarian", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " options", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " including", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " our", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " signature", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " Quinoa", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " Buddha", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " Bowl", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " with", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " fresh", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " quinoa,", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " roasted", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " vegetables,", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " avocado,", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " and", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " tahini", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " dressing", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " for", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " $12.99,", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " and", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " our", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " Roasted", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " Vegetable", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " Pasta", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " with", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " house-made", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " pasta", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " and", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " seasonal", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " roasted", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " vegetables", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " in", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " olive", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " oil", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " for", "role": "assistant"}, "sessionState": null} +{"context": null, "delta": {"content": " $14.99.", "role": "assistant"}, "sessionState": null} +{"context": {"data_points": [{"name": "Quinoa Buddha Bowl", "description": "Fresh quinoa with roasted vegetables, avocado, and tahini dressing", "price": "$12.99", "category": "Vegetarian", "collection": "main-dishes"}, {"name": "Roasted Vegetable Pasta", "description": "House-made pasta with seasonal roasted vegetables in olive oil", "price": "$14.99", "category": "Vegetarian", "collection": "main-dishes"}], "thoughts": [{"title": "Search Query Generation", "description": "Generated search query for vegetarian dishes in our menu database"}, {"title": "Document Retrieval", "description": "Found 2 relevant vegetarian dishes matching the customer's request"}]}, "delta": null, "sessionState": null} \ No newline at end of file diff --git a/tests/snapshots/test_playwright/test_simple_chat_flow/simple_chat_flow_response.json b/tests/snapshots/test_playwright/test_simple_chat_flow/simple_chat_flow_response.json new file mode 100644 index 0000000..6a05dd7 --- /dev/null +++ b/tests/snapshots/test_playwright/test_simple_chat_flow/simple_chat_flow_response.json @@ -0,0 +1,35 @@ +{ + "context": { + "data_points": [ + { + "name": "Quinoa Buddha Bowl", + "description": "Fresh quinoa with roasted vegetables, avocado, and tahini dressing", + "price": "$12.99", + "category": "Vegetarian", + "collection": "main-dishes" + }, + { + "name": "Roasted Vegetable Pasta", + "description": "House-made pasta with seasonal roasted vegetables in olive oil", + "price": "$14.99", + "category": "Vegetarian", + "collection": "main-dishes" + } + ], + "thoughts": [ + { + "title": "Search Query Generation", + "description": "Generated search query for vegetarian dishes in our menu database" + }, + { + "title": "Document Retrieval", + "description": "Found 2 relevant vegetarian dishes matching the customer's request" + } + ] + }, + "message": { + "content": "We have delicious vegetarian options including our signature Quinoa Buddha Bowl with fresh quinoa, roasted vegetables, avocado, and tahini dressing for $12.99, and our Roasted Vegetable Pasta with house-made pasta and seasonal roasted vegetables in olive oil for $14.99.", + "role": "assistant" + }, + "sessionState": null +} \ No newline at end of file diff --git a/tests/test_playwright.py b/tests/test_playwright.py new file mode 100644 index 0000000..fda6b01 --- /dev/null +++ b/tests/test_playwright.py @@ -0,0 +1,221 @@ +import asyncio +import socket +import time +import os +from collections.abc import Generator +from contextlib import closing +from multiprocessing import Process +import json +from unittest import mock + +import pytest +import requests +import uvicorn +# Import playwright but mark tests as skipped if chromium not available +try: + from playwright.sync_api import Page, Route, expect + PLAYWRIGHT_AVAILABLE = True +except ImportError: + PLAYWRIGHT_AVAILABLE = False + +from quartapp.app import create_app + +if PLAYWRIGHT_AVAILABLE: + expect.set_options(timeout=10_000) + + +def wait_for_server_ready(url: str, timeout: float = 10.0, check_interval: float = 0.5) -> bool: + """Make requests to provided url until it responds without error.""" + conn_error = None + for _ in range(int(timeout / check_interval)): + try: + requests.get(url) + except requests.ConnectionError as exc: + time.sleep(check_interval) + conn_error = str(exc) + else: + return True + raise RuntimeError(conn_error) + + +@pytest.fixture(scope="session") +def free_port() -> int: + """Returns a free port for the test server to bind.""" + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + s.bind(("", 0)) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + return s.getsockname()[1] + + +def run_server(port: int): + """Run the Quart application server using uvicorn.""" + # Set up environment variables to avoid external dependencies + import os + from unittest import mock + + env_vars = { + "AZURE_COSMOS_CONNECTION_STRING": "test-connection-string", + "AZURE_COSMOS_USERNAME": "test-username", + "AZURE_COSMOS_PASSWORD": "test-password", + "AZURE_COSMOS_DATABASE_NAME": "test-database", + "AZURE_COSMOS_COLLECTION_NAME": "test-collection", + "AZURE_COSMOS_INDEX_NAME": "test-index", + "AZURE_SUBSCRIPTION_ID": "test-storage-subid", + "OPENAI_CHAT_HOST": "azure", + "OPENAI_EMBED_HOST": "azure", + "AZURE_OPENAI_ENDPOINT": "https://api.openai.com", + "OPENAI_API_VERSION": "2024-03-01-preview", + "AZURE_OPENAI_CHAT_DEPLOYMENT_NAME": "gpt-4o-mini", + "AZURE_OPENAI_CHAT_MODEL_NAME": "gpt-4o-mini", + "AZURE_OPENAI_EMBEDDINGS_MODEL_NAME": "text-embedding-3-small", + "AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME": "text-embedding-3-small", + "AZURE_OPENAI_EMBEDDINGS_DIMENSIONS": "1536", + "AZURE_OPENAI_KEY": "fakekey", + "ALLOWED_ORIGIN": "https://frontend.com" + } + + with mock.patch.dict(os.environ, env_vars): + app = create_app() + app.config.update({"TESTING": True}) + uvicorn.run(app, port=port, log_level="error") + + +@pytest.fixture() +def live_server_url(mock_session_env, free_port: int) -> Generator[str, None, None]: + proc = Process(target=run_server, args=(free_port,), daemon=True) + proc.start() + url = f"http://localhost:{free_port}/" + wait_for_server_ready(url, timeout=10.0, check_interval=0.5) + yield url + proc.kill() + + +# Test basic server functionality without Playwright first +def test_server_runs(live_server_url: str): + """Test that the server starts and serves the homepage.""" + response = requests.get(live_server_url) + assert response.status_code == 200 + assert "Cosmic Food RAG App" in response.text + + +def test_chat_endpoint(live_server_url: str): + """Test that the chat endpoint works.""" + chat_data = { + "messages": [{"content": "What vegetarian dishes do you have?", "role": "user"}], + "sessionState": None, + "context": {"overrides": {"retrieval_mode": "vector"}} + } + response = requests.post(f"{live_server_url}chat", json=chat_data) + assert response.status_code == 200 + data = response.json() + assert "message" in data + assert "context" in data + + +@pytest.mark.skipif(not PLAYWRIGHT_AVAILABLE, reason="Playwright not available") +def test_home(page: Page, live_server_url: str): + """Test that the home page loads with the correct title.""" + page.goto(live_server_url) + expect(page).to_have_title("Cosmic Food RAG App | Sample") + + +@pytest.mark.skipif(not PLAYWRIGHT_AVAILABLE, reason="Playwright not available") +def test_chat(page: Page, live_server_url: str): + """Test basic chat functionality with mocked streaming responses.""" + # Set up a mock route to the /chat/stream endpoint with streaming results + def handle(route: Route): + # Assert that session_state is specified in the request (None for now) + if route.request.post_data_json: + session_state = route.request.post_data_json.get("sessionState") + assert session_state is None + # Read the JSONL from our snapshot results and return as the response + f = open( + "tests/snapshots/test_playwright/test_chat_streaming_flow/chat_streaming_flow_response.jsonlines" + ) + jsonl = f.read() + f.close() + route.fulfill(body=jsonl, status=200, headers={"Transfer-encoding": "Chunked"}) + + page.route("*/**/chat/stream", handle) + + # Check initial page state + page.goto(live_server_url) + expect(page).to_have_title("Cosmic Food RAG App | Sample") + expect(page.get_by_role("button", name="Clear chat")).to_be_disabled() + expect(page.get_by_role("button", name="Developer settings")).to_be_enabled() + + # Ask a question and wait for the message to appear + page.get_by_placeholder("Type a new question (e.g. Are there any high protein dishes available?)").click() + page.get_by_placeholder("Type a new question (e.g. Are there any high protein dishes available?)").fill("What vegetarian dishes do you have?") + # Find the submit button - it might be near the textarea + page.keyboard.press("Enter") # Try submitting with Enter key + + expect(page.get_by_text("What vegetarian dishes do you have?")).to_be_visible() + expect(page.get_by_text("We have delicious vegetarian options")).to_be_visible() + expect(page.get_by_role("button", name="Clear chat")).to_be_enabled() + + # Clear the chat + page.get_by_role("button", name="Clear chat").click() + expect(page.get_by_text("What vegetarian dishes do you have?")).not_to_be_visible() + expect(page.get_by_text("We have delicious vegetarian options")).not_to_be_visible() + expect(page.get_by_role("button", name="Clear chat")).to_be_disabled() + + +@pytest.mark.skipif(not PLAYWRIGHT_AVAILABLE, reason="Playwright not available") +def test_chat_customization(page: Page, live_server_url: str): + """Test chat customization via developer settings.""" + # Set up a mock route to the /chat endpoint + def handle(route: Route): + # Read the JSON from our snapshot results and return as the response + f = open("tests/snapshots/test_playwright/test_simple_chat_flow/simple_chat_flow_response.json") + json = f.read() + f.close() + route.fulfill(body=json, status=200) + + page.route("*/**/chat", handle) + + # Check initial page state + page.goto(live_server_url) + expect(page).to_have_title("Cosmic Food RAG App | Sample") + + # Open developer settings + page.get_by_role("button", name="Developer settings").click() + + # Just verify we can open settings panel - actual settings might be different than expected + # Close the settings + page.keyboard.press("Escape") # Try to close with escape key + + # Ask a question and wait for the message to appear + page.get_by_placeholder("Type a new question (e.g. Are there any high protein dishes available?)").click() + page.get_by_placeholder("Type a new question (e.g. Are there any high protein dishes available?)").fill("What vegetarian dishes do you have?") + page.keyboard.press("Enter") + + expect(page.get_by_text("What vegetarian dishes do you have?")).to_be_visible() + expect(page.get_by_role("button", name="Clear chat")).to_be_enabled() + + +@pytest.mark.skipif(not PLAYWRIGHT_AVAILABLE, reason="Playwright not available") +def test_chat_nonstreaming(page: Page, live_server_url: str): + """Test non-streaming chat responses.""" + # Set up a mock route to the /chat endpoint + def handle(route: Route): + # Read the JSON from our snapshot results and return as the response + f = open("tests/snapshots/test_playwright/test_chat_flow/chat_flow_response.json") + json = f.read() + f.close() + route.fulfill(body=json, status=200) + + page.route("*/**/chat", handle) + + # Check initial page state + page.goto(live_server_url) + expect(page).to_have_title("Cosmic Food RAG App | Sample") + expect(page.get_by_role("button", name="Developer settings")).to_be_enabled() + + # Ask a question and wait for the message to appear + page.get_by_placeholder("Type a new question (e.g. Are there any high protein dishes available?)").click() + page.get_by_placeholder("Type a new question (e.g. Are there any high protein dishes available?)").fill("What vegetarian dishes do you have?") + page.keyboard.press("Enter") + + expect(page.get_by_text("What vegetarian dishes do you have?")).to_be_visible() + expect(page.get_by_role("button", name="Clear chat")).to_be_enabled() \ No newline at end of file