Add Playwright end-to-end tests for UI automation

src/quartapp/app.py

@@ -29,19 +29,38 @@ async def format_as_ndjson(r: AsyncGenerator[RetrievalResponseDelta, None]) -> A
 
 
 def create_app(test_config: dict[str, Any] | None = None) -> Quart:
-    app_config = AppConfig()
-
     app = Quart(__name__, static_folder="static")
 
     if test_config:
         # load the test config if passed in
         app.config.from_mapping(test_config)
-
-    available_approaches = {
-        "vector": app_config.run_vector,
-        "rag": app_config.run_rag,
-        "keyword": app_config.run_keyword,
-    }
+        # Create mock async functions for testing
+        async def mock_approach(**kwargs):
+            from quartapp.approaches.schemas import RetrievalResponse, Message, Context, DataPoint, Thought
+            return RetrievalResponse(
+                message=Message(content="We have delicious vegetarian options including our signature Quinoa Buddha Bowl.", role="assistant"),
+                context=Context(
+                    data_points=[DataPoint(name="Quinoa Buddha Bowl", description="Fresh quinoa with roasted vegetables", price="$12.99", category="Vegetarian")],
+                    thoughts=[Thought(title="Search", description="Found vegetarian options")]
+                ),
+                sessionState=None
+            )
+
+        available_approaches = {
+            "vector": mock_approach,
+            "rag": mock_approach,
+            "keyword": mock_approach,
+        }
+
+        # For streaming, we'll handle it separately in the route
+        app_config = None
+    else:
+        app_config = AppConfig()
+        available_approaches = {
+            "vector": app_config.run_vector,
+            "rag": app_config.run_rag,
+            "keyword": app_config.run_keyword,
+        }
 
     @app.route("/")
     async def index() -> Any:
@@ -129,16 +148,26 @@ async def stream_chat() -> Any:
         score_threshold: float = override.get("score_threshold", 0)
 
         if retrieval_mode == "rag":
-            result: AsyncGenerator[RetrievalResponseDelta, None] = app_config.run_rag_stream(
-                session_state=session_state,
-                messages=messages,
-                temperature=temperature,
-                limit=top,
-                score_threshold=score_threshold,
-            )
-            response = await make_response(format_as_ndjson(result))
-            response.mimetype = "application/x-ndjson"
-            return response
+            if app.config.get("TESTING"):
+                # Return a simple mock response for testing
+                async def mock_stream():
+                    from quartapp.approaches.schemas import RetrievalResponseDelta, Message
+                    yield RetrievalResponseDelta(delta=Message(content="Mock", role="assistant"))
+
+                response = await make_response(format_as_ndjson(mock_stream()))
+                response.mimetype = "application/x-ndjson"
+                return response
+            else:
+                result: AsyncGenerator[RetrievalResponseDelta, None] = app_config.run_rag_stream(
+                    session_state=session_state,
+                    messages=messages,
+                    temperature=temperature,
+                    limit=top,
+                    score_threshold=score_threshold,
+                )
+                response = await make_response(format_as_ndjson(result))
+                response.mimetype = "application/x-ndjson"
+                return response
         return jsonify({"error": "Not Implemented!"}), 501
 
     return app

tests/README.md

@@ -0,0 +1,75 @@
+# Playwright Tests
+
+This directory contains end-to-end tests for the Cosmic Food RAG App using Playwright.
+
+## Test Files
+
+- `test_playwright.py` - Main Playwright test file containing UI tests
+
+## Test Coverage
+
+The tests cover the following functionality:
+
+1. **Home Page Test** (`test_home`)
+   - Verifies the application loads correctly
+   - Checks the page title
+
+2. **Basic Chat Test** (`test_chat`)
+   - Tests basic chat functionality with streaming responses
+   - Mocks the chat/stream endpoint
+   - Verifies chat input, submission, and response display
+   - Tests clear chat functionality
+
+3. **Chat Customization Test** (`test_chat_customization`)
+   - Tests opening developer settings
+   - Verifies chat works with non-streaming responses
+
+4. **Non-streaming Chat Test** (`test_chat_nonstreaming`)
+   - Tests chat with mocked non-streaming responses
+   - Uses the /chat endpoint instead of /chat/stream
+
+## Mock Data
+
+Mock response data is stored in `tests/snapshots/test_playwright/`:
+
+- `test_chat_flow/chat_flow_response.json` - Non-streaming chat response
+- `test_chat_streaming_flow/chat_streaming_flow_response.jsonlines` - Streaming chat response
+- `test_simple_chat_flow/simple_chat_flow_response.json` - Simple chat response
+
+## Running Tests
+
+### Prerequisites
+
+- Python dependencies: `pytest`, `playwright`, `pytest-playwright`
+- Chromium browser installed via Playwright
+
+### Running All Playwright Tests
+
+```bash
+python -m pytest tests/test_playwright.py -v --browser chromium
+```
+
+### Running Individual Tests
+
+```bash
+# Home page test
+python -m pytest tests/test_playwright.py::test_home -v --browser chromium
+
+# Chat tests
+python -m pytest tests/test_playwright.py::test_chat -v --browser chromium
+```
+
+### Test Server
+
+The tests use a test server that:
+- Runs the Quart application in test mode
+- Bypasses OpenAI and Cosmos DB connections
+- Uses mock responses for chat functionality
+- Automatically starts and stops for each test
+
+## Implementation Notes
+
+- Tests run in headless mode by default
+- The server runs on a random available port for each test session
+- Mock responses are intercepted at the network level using Playwright's route mocking
+- Tests are designed to be independent and can run in any order

tests/snapshots/test_playwright/test_chat_flow/chat_flow_response.json

@@ -0,0 +1,35 @@
+{
+  "context": {
+    "data_points": [
+      {
+        "name": "Quinoa Buddha Bowl",
+        "description": "Fresh quinoa with roasted vegetables, avocado, and tahini dressing",
+        "price": "$12.99",
+        "category": "Vegetarian",
+        "collection": "main-dishes"
+      },
+      {
+        "name": "Roasted Vegetable Pasta",
+        "description": "House-made pasta with seasonal roasted vegetables in olive oil",
+        "price": "$14.99",
+        "category": "Vegetarian",
+        "collection": "main-dishes"
+      }
+    ],
+    "thoughts": [
+      {
+        "title": "Search Query Generation",
+        "description": "Generated search query for vegetarian dishes in our menu database"
+      },
+      {
+        "title": "Document Retrieval",
+        "description": "Found 2 relevant vegetarian dishes matching the customer's request"
+      }
+    ]
+  },
+  "message": {
+    "content": "We have delicious vegetarian options including our signature Quinoa Buddha Bowl with fresh quinoa, roasted vegetables, avocado, and tahini dressing for $12.99, and our Roasted Vegetable Pasta with house-made pasta and seasonal roasted vegetables in olive oil for $14.99.",
+    "role": "assistant"
+  },
+  "sessionState": null
+}

tests/snapshots/test_playwright/test_chat_streaming_flow/chat_streaming_flow_response.jsonlines

@@ -0,0 +1,41 @@
+{"context": null, "delta": {"content": "", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": "We", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " have", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " delicious", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " vegetarian", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " options", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " including", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " our", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " signature", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " Quinoa", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " Buddha", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " Bowl", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " with", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " fresh", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " quinoa,", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " roasted", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " vegetables,", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " avocado,", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " and", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " tahini", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " dressing", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " for", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " $12.99,", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " and", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " our", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " Roasted", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " Vegetable", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " Pasta", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " with", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " house-made", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " pasta", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " and", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " seasonal", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " roasted", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " vegetables", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " in", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " olive", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " oil", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " for", "role": "assistant"}, "sessionState": null}
+{"context": null, "delta": {"content": " $14.99.", "role": "assistant"}, "sessionState": null}
+{"context": {"data_points": [{"name": "Quinoa Buddha Bowl", "description": "Fresh quinoa with roasted vegetables, avocado, and tahini dressing", "price": "$12.99", "category": "Vegetarian", "collection": "main-dishes"}, {"name": "Roasted Vegetable Pasta", "description": "House-made pasta with seasonal roasted vegetables in olive oil", "price": "$14.99", "category": "Vegetarian", "collection": "main-dishes"}], "thoughts": [{"title": "Search Query Generation", "description": "Generated search query for vegetarian dishes in our menu database"}, {"title": "Document Retrieval", "description": "Found 2 relevant vegetarian dishes matching the customer's request"}]}, "delta": null, "sessionState": null}

tests/snapshots/test_playwright/test_simple_chat_flow/simple_chat_flow_response.json

@@ -0,0 +1,35 @@
+{
+  "context": {
+    "data_points": [
+      {
+        "name": "Quinoa Buddha Bowl",
+        "description": "Fresh quinoa with roasted vegetables, avocado, and tahini dressing",
+        "price": "$12.99",
+        "category": "Vegetarian",
+        "collection": "main-dishes"
+      },
+      {
+        "name": "Roasted Vegetable Pasta",
+        "description": "House-made pasta with seasonal roasted vegetables in olive oil",
+        "price": "$14.99",
+        "category": "Vegetarian",
+        "collection": "main-dishes"
+      }
+    ],
+    "thoughts": [
+      {
+        "title": "Search Query Generation",
+        "description": "Generated search query for vegetarian dishes in our menu database"
+      },
+      {
+        "title": "Document Retrieval",
+        "description": "Found 2 relevant vegetarian dishes matching the customer's request"
+      }
+    ]
+  },
+  "message": {
+    "content": "We have delicious vegetarian options including our signature Quinoa Buddha Bowl with fresh quinoa, roasted vegetables, avocado, and tahini dressing for $12.99, and our Roasted Vegetable Pasta with house-made pasta and seasonal roasted vegetables in olive oil for $14.99.",
+    "role": "assistant"
+  },
+  "sessionState": null
+}