Add a smoke test after deployment

.github/workflows/azure-dev.yaml

@@ -3,8 +3,7 @@ name: Deploy to Azure
 on:
   workflow_dispatch:
   push:
-    # Run when commits are pushed to mainline branch (main or master)
-    # Set this to the mainline branch you are using
+    # Run when commits are pushed to mainline branch
     branches:
       - main
 
@@ -20,6 +19,8 @@ permissions:
 jobs:
   build:
     runs-on: ubuntu-latest
+    outputs:
+      uri: ${{ steps.output.outputs.uri }}
     env:
       AZURE_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }}
       AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }}
@@ -73,3 +74,44 @@ jobs:
           AZURE_ENV_NAME: ${{ vars.AZURE_ENV_NAME }}
           AZURE_LOCATION: ${{ vars.AZURE_LOCATION }}
           AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }}
+
+      - name: Output Deployment URI
+        id: output
+        run: |
+          azd env get-values > .env
+          source .env
+          echo "uri=$SERVICE_ACA_URI" >> "$GITHUB_OUTPUT"
+
+
+  smoketests:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+
+      - name: Basic smoke test (curl)
+        env:
+          URI: ${{needs.build.outputs.uri}}
+        run: |
+          echo "Sleeping 1 minute due to https://github.com/Azure/azure-dev/issues/2669"
+          sleep 60
+          curl -sSf $URI
+
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Set up Python (for Playwright test)
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Install Playwright (Python) and Browsers
+        run: |
+          pip install --no-cache-dir playwright
+          python -m playwright install --with-deps chromium
+
+      - name: End-to-End Chat UI Test (Playwright)
+        shell: bash
+        env:
+          AZURE_ENV_NAME: ${{ vars.AZURE_ENV_NAME }}
+          URI: ${{needs.build.outputs.uri}}
+        run: |
+          python scripts/e2e_chat_playwright.py $URI

scripts/e2e_chat_playwright.py

@@ -0,0 +1,92 @@
+"""Minimal end-to-end browser test using Playwright.
+
+Usage:
+    python scripts/e2e_chat_playwright.py https://your-app.azurecontainerapps.io
+
+Only one argument is accepted: the base URL of the deployed app.
+No environment variables or azd lookups are performed.
+"""
+
+from __future__ import annotations
+
+import re
+import sys
+import time
+
+from playwright.sync_api import Playwright, expect, sync_playwright
+
+
+def run_test(pw: Playwright, base_url: str) -> None:
+    # Internal test configuration (adjust here if needed)
+    message = "Hi"
+    timeout = 60  # seconds
+    headless = True
+    expected_substring = None  # Set to a string to force exact substring match
+    greeting_regex = r"\b(H(i|ello))\b"
+    browser = pw.chromium.launch(headless=headless)
+    context = browser.new_context()
+    page = context.new_page()
+
+    if not base_url.startswith("http"):
+        raise ValueError("Base URL must start with http/https")
+    base_url = base_url.rstrip("/")
+
+    url = base_url
+    if not url.endswith("/"):
+        url += "/"
+    print(f"Navigating to {url}")
+    page.goto(url, wait_until="domcontentloaded")
+
+    textbox = page.get_by_role("textbox", name="Ask ChatGPT")
+    textbox.click()
+    textbox.fill(message)
+    textbox.press("Enter")
+    # Redundant click in case Enter doesn't submit on some platforms
+    page.get_by_role("button", name="Send").click()
+
+    # Wait for the last assistant message content div that is not the typing indicator
+    content_locator = page.locator(".toast-body.message-content").last
+    # Poll until the content no longer contains 'Typing...' and has some text
+    start = time.time()
+    while time.time() - start < timeout:
+        txt = content_locator.inner_text().strip()
+        if txt and "Typing..." not in txt:
+            break
+        time.sleep(0.5)
+    else:
+        raise RuntimeError("Timeout waiting for assistant response")
+
+    txt_final = content_locator.inner_text().strip()
+    if expected_substring:
+        expect(content_locator).to_contain_text(expected_substring)
+    else:
+        if not re.search(greeting_regex, txt_final, flags=re.IGNORECASE):
+            raise RuntimeError(
+                f"Assistant response did not match greeting regex '{greeting_regex}'. Got: {txt_final[:120]}"
+            )
+        if len(txt_final) < 2:
+            raise RuntimeError("Assistant response too short")
+    print("Assistant response snippet:", txt_final[:160])
+
+    # Cleanup
+    context.close()
+    browser.close()
+
+
+def main() -> int:
+    if len(sys.argv) != 2:
+        print("Usage: python scripts/e2e_chat_playwright.py <base_url>", file=sys.stderr)
+        return 1
+    base_url = sys.argv[1]
+    try:
+        with sync_playwright() as pw:
+            run_test(pw, base_url)
+        print("Playwright E2E test succeeded.")
+        return 0
+    except Exception as e:  # broad for CLI convenience
+        print(f"Playwright E2E test failed: {e}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/quartapp/chat.py

@@ -1,8 +1,8 @@
 import json
 import os
 
-import azure.identity.aio
-import openai
+from azure.identity.aio import AzureDeveloperCliCredential, ManagedIdentityCredential, get_bearer_token_provider
+from openai import AsyncOpenAI
 from quart import (
     Blueprint,
     Response,
@@ -17,55 +17,47 @@
 
 @bp.before_app_serving
 async def configure_openai():
+    bp.model_name = os.getenv("OPENAI_MODEL", "gpt-4o")
     openai_host = os.getenv("OPENAI_HOST", "github")
 
     if openai_host == "local":
-        bp.model_name = os.getenv("OPENAI_MODEL", "gpt-4o")
-        current_app.logger.info("Using model %s from local OpenAI-compatible API with no key", bp.model_name)
-        bp.openai_client = openai.AsyncOpenAI(api_key="no-key-required", base_url=os.getenv("LOCAL_OPENAI_ENDPOINT"))
+        bp.openai_client = AsyncOpenAI(api_key="no-key-required", base_url=os.getenv("LOCAL_OPENAI_ENDPOINT"))
+        current_app.logger.info("Using local OpenAI-compatible API service with no key")
     elif openai_host == "github":
-        bp.model_name = os.getenv("OPENAI_MODEL", "openai/gpt-4o")
-        current_app.logger.info("Using model %s from GitHub models with GITHUB_TOKEN as key", bp.model_name)
-        bp.openai_client = openai.AsyncOpenAI(
+        bp.model_name = f"openai/{bp.model_name}"
+        bp.openai_client = AsyncOpenAI(
             api_key=os.environ["GITHUB_TOKEN"],
             base_url="https://models.github.ai/inference",
         )
+        current_app.logger.info("Using GitHub models with GITHUB_TOKEN as key")
+    elif os.getenv("AZURE_OPENAI_KEY_FOR_CHATVISION"):
+        # Authenticate using an Azure OpenAI API key
+        # This is generally discouraged, but is provided for developers
+        # that want to develop locally inside the Docker container.
+        bp.openai_client = AsyncOpenAI(
+            base_url=os.environ["AZURE_OPENAI_ENDPOINT"],
+            api_key=os.getenv("AZURE_OPENAI_KEY_FOR_CHATVISION"),
+        )
+        current_app.logger.info("Using Azure OpenAI with key")
+    elif os.getenv("RUNNING_IN_PRODUCTION"):
+        client_id = os.environ["AZURE_CLIENT_ID"]
+        azure_credential = ManagedIdentityCredential(client_id=client_id)
+        token_provider = get_bearer_token_provider(azure_credential, "https://cognitiveservices.azure.com/.default")
+        bp.openai_client = AsyncOpenAI(
+            base_url=os.environ["AZURE_OPENAI_ENDPOINT"] + "/openai/v1/",
+            api_key=token_provider,
+        )
+        current_app.logger.info("Using Azure OpenAI with managed identity credential for client ID %s", client_id)
     else:
-        # Use an Azure OpenAI endpoint instead,
-        # either with a key or with keyless authentication
-        bp.model_name = os.getenv("OPENAI_MODEL", "gpt-4o")
-        if os.getenv("AZURE_OPENAI_KEY_FOR_CHATVISION"):
-            # Authenticate using an Azure OpenAI API key
-            # This is generally discouraged, but is provided for developers
-            # that want to develop locally inside the Docker container.
-            current_app.logger.info("Using model %s from Azure OpenAI with key", bp.model_name)
-            bp.openai_client = openai.AsyncOpenAI(
-                base_url=os.environ["AZURE_OPENAI_ENDPOINT"],
-                api_key=os.getenv("AZURE_OPENAI_KEY_FOR_CHATVISION"),
-            )
-        elif os.getenv("RUNNING_IN_PRODUCTION"):
-            client_id = os.getenv("AZURE_CLIENT_ID")
-            current_app.logger.info(
-                "Using model %s from Azure OpenAI with managed identity credential for client ID %s",
-                bp.model_name,
-                client_id,
-            )
-            azure_credential = azure.identity.aio.ManagedIdentityCredential(client_id=client_id)
-        else:
-            tenant_id = os.environ["AZURE_TENANT_ID"]
-            current_app.logger.info(
-                "Using model %s from Azure OpenAI with Azure Developer CLI credential for tenant ID: %s",
-                bp.model_name,
-                tenant_id,
-            )
-            azure_credential = azure.identity.aio.AzureDeveloperCliCredential(tenant_id=tenant_id)
-            token_provider = azure.identity.aio.get_bearer_token_provider(
-                azure_credential, "https://cognitiveservices.azure.com/.default"
-            )
-            bp.openai_client = openai.AsyncOpenAI(
-                base_url=os.environ["AZURE_OPENAI_ENDPOINT"] + "/openai/v1/",
-                api_key=token_provider,
-            )
+        tenant_id = os.environ["AZURE_TENANT_ID"]
+        azure_credential = AzureDeveloperCliCredential(tenant_id=tenant_id)
+        token_provider = get_bearer_token_provider(azure_credential, "https://cognitiveservices.azure.com/.default")
+        bp.openai_client = AsyncOpenAI(
+            base_url=os.environ["AZURE_OPENAI_ENDPOINT"] + "/openai/v1/",
+            api_key=token_provider,
+        )
+        current_app.logger.info("Using Azure OpenAI with az CLI credential for tenant ID: %s", tenant_id)
+    current_app.logger.info("Using model %s", bp.model_name)
 
 
 @bp.after_app_serving

src/requirements.txt

@@ -110,7 +110,7 @@ multidict==6.1.0
     # via
     #   aiohttp
     #   yarl
-openai>=1.108.1
+openai==1.108.1
     # via quartapp (pyproject.toml)
 packaging==24.2
     # via gunicorn