Browser smoke test and demo recorder in the repo

.gitignore

@@ -107,3 +107,6 @@ logs/
 .cache/
 .temp/
 .tmp/
+
+# Playwright recordings for the README demo
+Web/e2e/video/

README.md

@@ -367,10 +367,12 @@ npm run migrate:api
 ```sh
 npm run test:api                          # pytest; hermetic, no Ollama needed
 cd Web && npx tsc --noEmit                # type check (the build fails on errors)
-uv run --project API python test_optimization.py   # end-to-end run against a live API
+npm run e2e:install                       # once: downloads Chromium for the browser test
+npm run e2e                               # browser smoke test against a running `npm run dev`
+npm run demo:record                       # re-records docs/demo.gif (needs ffmpeg)
 ```
 
-The backend suite runs without Ollama: model calls use DSPy's `DummyLM` and embeddings a deterministic stand-in. Manual test cases are in [TESTING_GUIDE.md](TESTING_GUIDE.md).
+The backend suite runs without Ollama: model calls use DSPy's `DummyLM` and embeddings a deterministic stand-in; CI runs it on every pull request. The browser smoke test needs Ollama with a model: it imports a dataset, previews and exports it, runs a measured optimization, tries both prompts on an input, checks analytics, deletes the dataset and reloads, and fails on any page error. Manual test cases are in [TESTING_GUIDE.md](TESTING_GUIDE.md).
 
 #### Contributing
 

TESTING_GUIDE.md

@@ -11,6 +11,7 @@ npm run lint:api                # ruff + strict mypy
 cd Web && npx tsc --noEmit      # frontend types; the production build fails on errors
 cd Web && npm run build
 uv run --project API python test_optimization.py   # end-to-end against a running API (npm run dev:api first)
+npm run e2e:install && npm run e2e                  # browser smoke test against a running `npm run dev` (needs Ollama)
 ```
 
 The backend suite replaces model calls with DSPy's `DummyLM` and embeddings with a deterministic stand-in, so it runs in a few seconds anywhere.

Web/e2e/record-demo.mjs

@@ -0,0 +1,93 @@
+// Records the README demo: a GEPA run against the bundled ticket dataset.
+//
+//   node e2e/record-demo.mjs            # writes e2e/video/*.webm
+//   ../scripts/record-demo.sh           # runs this, then converts to ../docs/demo.gif
+//
+// Expects `npm run dev` to be up. Imports the dataset if it is missing. The
+// prompt text is varied per run so DSPy's cache does not replay an earlier
+// identical run instantly (which would leave no progress to show).
+
+import { chromium } from "playwright"
+import fs from "node:fs"
+import path from "node:path"
+
+const WEB = process.env.WEB_URL ?? "http://localhost:3000"
+const API = process.env.API_URL ?? "http://127.0.0.1:8000"
+const OUT = path.resolve("e2e/video")
+const DATASET = "Support ticket priority"
+const PROMPT = `Classify the priority of this support ticket as high, medium, or low. Ticket ${Date.now() % 1000}:`
+const BUDGET = Number(process.env.DEMO_GEPA_BUDGET ?? 60)
+
+fs.rmSync(OUT, { recursive: true, force: true })
+fs.mkdirSync(OUT, { recursive: true })
+
+// Make sure the dataset exists.
+const datasets = await (await fetch(`${API}/api/v1/training/`)).json()
+if (!datasets.some((d) => d.name === DATASET)) {
+  const csv = fs.readFileSync(path.resolve("../docs/examples/support-tickets.csv"), "utf8")
+  await fetch(`${API}/api/v1/training/import`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ name: DATASET, task_type: "classification", file_format: "csv", data: csv }),
+  })
+}
+
+const browser = await chromium.launch()
+const context = await browser.newContext({
+  viewport: { width: 1280, height: 800 },
+  recordVideo: { dir: OUT, size: { width: 1280, height: 800 } },
+  colorScheme: "light",
+})
+const page = await context.newPage()
+const pause = (ms) => page.waitForTimeout(ms)
+const selectOption = async (trigger, name) => {
+  await trigger.click()
+  await pause(500)
+  await page.getByRole("option", { name }).first().click()
+}
+const scrollTo = async (locator) => {
+  await locator.scrollIntoViewIfNeeded()
+  await pause(200)
+}
+
+await page.goto(WEB, { waitUntil: "networkidle" })
+await pause(1200)
+
+const textarea = page.locator("textarea").first()
+await textarea.click()
+await textarea.pressSequentially(PROMPT, { delay: 25 })
+await pause(600)
+
+await selectOption(page.getByRole("combobox").filter({ hasText: /Meta-Prompt/ }).first(), /GEPA/)
+await pause(900)
+await selectOption(page.getByRole("combobox").filter({ hasText: "None" }).first(), new RegExp(DATASET))
+await pause(1200)
+if (BUDGET !== 60) {
+  await page.locator('input[type="range"]').first().fill(String(BUDGET))
+  await pause(400)
+}
+
+await page.getByRole("button", { name: /Start Optimization/ }).click()
+await page.getByText(/Generation \d+/).first().waitFor({ timeout: 90_000 }).catch(() => {})
+await pause(2500)
+await page.getByText("Prompt Evolution").waitFor({ timeout: 600_000 })
+await pause(1500)
+
+await scrollTo(page.getByText("Optimized Prompt").first())
+await pause(1800)
+await scrollTo(page.getByText("Prompt Evolution").first())
+await pause(1500)
+const feedback = page.getByText("Feedback the reflection read before proposing this").first()
+if (await feedback.count()) {
+  await scrollTo(feedback)
+  await pause(2200)
+}
+await scrollTo(page.getByText("Eval Results").first())
+await pause(1500)
+await scrollTo(page.getByText("Held-out samples").first())
+await pause(2500)
+
+await context.close()
+await browser.close()
+const video = fs.readdirSync(OUT).find((f) => f.endsWith(".webm"))
+console.log(path.join(OUT, video))

Web/e2e/smoke.mjs

@@ -0,0 +1,181 @@
+// End-to-end smoke test against a running PromptCraft (web + API + Ollama).
+//
+//   npm run e2e                 # from the repository root, with `npm run dev` up
+//   WEB_URL=... API_URL=... node e2e/smoke.mjs
+//
+// One-time: npx playwright install chromium   (downloads the browser)
+//
+// Imports a dataset, previews and exports it, runs a measured optimization
+// with the default model, checks analytics, deletes the dataset and reloads.
+// Exits non-zero if any step fails or the page logs an error. Needs Ollama
+// with at least one model, so it is a local check, not a CI job.
+
+import { chromium } from "playwright"
+import fs from "node:fs"
+import os from "node:os"
+import path from "node:path"
+
+const WEB = process.env.WEB_URL ?? "http://localhost:3000"
+const API = process.env.API_URL ?? "http://127.0.0.1:8000"
+const OPTIMIZE_TIMEOUT_MS = Number(process.env.E2E_OPTIMIZE_TIMEOUT_MS ?? 240_000)
+const DATASET_NAME = `E2E tickets ${Date.now()}`
+const OUT = process.env.E2E_OUT_DIR ?? fs.mkdtempSync(path.join(os.tmpdir(), "promptcraft-e2e-"))
+
+const CSV = [
+  "input,output",
+  '"Production database is down, all customers affected",high',
+  '"Cannot log in since this morning, blocking my work",high',
+  '"Security alert: suspicious logins from unknown IPs",high',
+  '"Question about how billing cycles work",medium',
+  '"Export to CSV is missing one column",medium',
+  '"Feature request: dark mode for the dashboard",low',
+  '"Thanks for the quick help yesterday!",low',
+  '"Typo on the pricing page footer",low',
+  '"Payment failed and the order is stuck, customer waiting",high',
+  '"Would like to change the email on my account",medium',
+].join("\n")
+
+const log = (...a) => console.log(new Date().toISOString().slice(11, 19), ...a)
+const results = []
+async function step(name, fn) {
+  try {
+    await fn()
+    results.push({ name, ok: true })
+    log("PASS", name)
+  } catch (err) {
+    const message = String(err?.message ?? err).split("\n")[0]
+    results.push({ name, ok: false, error: message })
+    log("FAIL", name, "->", message)
+  }
+}
+
+// Preflight: both servers answer.
+for (const [label, url] of [["web", WEB], ["api", `${API}/health`]]) {
+  try {
+    const r = await fetch(url)
+    if (!r.ok) throw new Error(`HTTP ${r.status}`)
+  } catch (err) {
+    console.error(`${label} not reachable at ${url}: ${err.message}. Start it with: npm run dev`)
+    process.exit(2)
+  }
+}
+
+const browser = await chromium.launch()
+const context = await browser.newContext({ viewport: { width: 1440, height: 1000 }, acceptDownloads: true })
+const page = await context.newPage()
+const consoleMessages = []
+page.on("console", (m) => {
+  if (["error", "warning"].includes(m.type())) consoleMessages.push({ type: m.type(), text: m.text().slice(0, 400) })
+})
+page.on("pageerror", (e) => consoleMessages.push({ type: "pageerror", text: String(e).slice(0, 400) }))
+const shot = (name) => page.screenshot({ path: path.join(OUT, `${name}.png`), fullPage: true })
+const selectOption = async (trigger, name) => {
+  await trigger.click()
+  await page.getByRole("option", { name }).first().click()
+}
+const datasetRow = () => page.locator('[data-sidebar="menu-item"]', { hasText: DATASET_NAME }).first()
+const openRowMenu = async () => {
+  await datasetRow().getByRole("button").last().click()
+  await page.getByRole("menuitem", { name: /Preview samples/ }).waitFor({ timeout: 3000 })
+}
+
+await step("home loads", async () => {
+  await page.goto(WEB, { waitUntil: "networkidle" })
+  await page.getByRole("button", { name: /Start Optimization/ }).waitFor({ timeout: 15000 })
+  await shot("01-home")
+})
+
+await step("import dataset from CSV", async () => {
+  await page.getByRole("tab", { name: /Training Data/ }).click()
+  await page.getByRole("button", { name: /Import Dataset/ }).click()
+  const dialog = page.getByRole("dialog")
+  await dialog.getByPlaceholder("Support ticket triage").fill(DATASET_NAME)
+  await selectOption(dialog.getByRole("combobox").nth(0), "Classification")
+  await selectOption(dialog.getByRole("combobox").nth(1), "CSV")
+  await dialog.locator("textarea").fill(CSV)
+  await dialog.getByRole("button", { name: /^Import$/ }).click()
+  await page.getByText("Dataset imported").first().waitFor({ timeout: 10000 })
+  await datasetRow().waitFor({ timeout: 5000 })
+})
+
+await step("preview samples", async () => {
+  await openRowMenu()
+  await page.getByRole("menuitem", { name: /Preview samples/ }).click()
+  const dialog = page.getByRole("dialog")
+  await dialog.getByText("Expected output").first().waitFor({ timeout: 5000 })
+  await shot("02-preview")
+  await page.keyboard.press("Escape")
+  await dialog.waitFor({ state: "hidden", timeout: 3000 })
+})
+
+await step("export JSON downloads a file", async () => {
+  await openRowMenu()
+  const [download] = await Promise.all([
+    page.waitForEvent("download", { timeout: 10000 }),
+    page.getByRole("menuitem", { name: /Export JSON/ }).click(),
+  ])
+  const target = path.join(OUT, download.suggestedFilename())
+  await download.saveAs(target)
+  const data = JSON.parse(fs.readFileSync(target, "utf8"))
+  if (!Array.isArray(data) || data.length !== 10) throw new Error(`expected 10 rows, got ${data.length}`)
+})
+
+await step("measured optimization", async () => {
+  await page.locator("textarea").first().fill("Classify the priority of this support ticket as high, medium or low.")
+  await selectOption(page.getByRole("combobox").filter({ hasText: "None" }).first(), new RegExp(DATASET_NAME))
+  await page.getByRole("button", { name: /Start Optimization/ }).click()
+  await page.getByText("Eval Results").waitFor({ timeout: OPTIMIZE_TIMEOUT_MS })
+  await page.getByText("Try it").first().waitFor({ timeout: 5000 })
+  await shot("03-results")
+})
+
+await step("try it runs both prompts", async () => {
+  await page.getByPlaceholder(/Paste the input the prompt should handle/).fill("Checkout page shows a 500 error for every customer")
+  await page.getByRole("button", { name: /Run both/ }).click()
+  await page.getByText("Optimized prompt", { exact: true }).first().waitFor({ timeout: OPTIMIZE_TIMEOUT_MS })
+})
+
+await step("analytics tab", async () => {
+  await page.getByRole("tab", { name: /Analytics/ }).click()
+  await page.getByText("Performance Overview").waitFor({ timeout: 5000 })
+  await page.getByRole("button", { name: /View Detailed Analytics/ }).click()
+  await page.getByRole("dialog").waitFor({ timeout: 3000 })
+  await page.keyboard.press("Escape")
+})
+
+await step("delete dataset with confirmation", async () => {
+  await page.getByRole("tab", { name: /Training Data/ }).click()
+  await openRowMenu()
+  await page.getByRole("menuitem", { name: /Delete/ }).click()
+  await page.getByRole("alertdialog").getByRole("button", { name: /^Delete$/ }).click()
+  await page.getByText("Dataset deleted").first().waitFor({ timeout: 10000 })
+})
+
+await step("reload is clean", async () => {
+  consoleMessages.length = 0
+  await page.reload({ waitUntil: "networkidle" })
+  await page.getByRole("button", { name: /Start Optimization/ }).waitFor({ timeout: 15000 })
+})
+
+await browser.close()
+
+// Remove the sessions this run created so the local history stays tidy.
+try {
+  const sessions = await (await fetch(`${API}/api/v1/sessions/?limit=50`)).json()
+  for (const s of sessions) {
+    if (s.original_prompt === "Classify the priority of this support ticket as high, medium or low." && s.name.startsWith("Optimization ")) {
+      const created = new Date(s.created_at.endsWith("Z") ? s.created_at : `${s.created_at}Z`)
+      if (Date.now() - created.getTime() < 30 * 60 * 1000) await fetch(`${API}/api/v1/sessions/${s.id}`, { method: "DELETE" })
+    }
+  }
+} catch {
+  // best effort
+}
+
+const failed = results.filter((r) => !r.ok)
+console.log("\nResults:")
+for (const r of results) console.log(` ${r.ok ? "PASS" : "FAIL"} ${r.name}${r.error ? ` (${r.error})` : ""}`)
+console.log(`\nConsole errors/warnings: ${consoleMessages.length ? "" : "none"}`)
+for (const m of consoleMessages) console.log(` [${m.type}] ${m.text}`)
+console.log(`\nScreenshots: ${OUT}`)
+process.exit(failed.length || consoleMessages.some((m) => m.type !== "warning") ? 1 : 0)

Web/package.json

@@ -68,6 +68,7 @@
     "@types/node": "^22",
     "@types/react": "^19",
     "@types/react-dom": "^19",
+    "playwright": "^1.62.1",
     "postcss": "^8.5",
     "tailwindcss": "^3.4.17",
     "typescript": "^5"

package.json

@@ -19,7 +19,10 @@
     "dev:links": "node scripts/dev-links.mjs",
     "postinstall": "node scripts/setup-api.mjs",
     "lint:api": "node scripts/run-api.mjs ruff check app tests && node scripts/run-api.mjs mypy app",
-    "migrate:api": "node scripts/run-api.mjs alembic upgrade head"
+    "migrate:api": "node scripts/run-api.mjs alembic upgrade head",
+    "e2e": "cd Web && node e2e/smoke.mjs",
+    "e2e:install": "cd Web && npx playwright install chromium",
+    "demo:record": "bash scripts/record-demo.sh"
   },
   "devDependencies": {
     "concurrently": "^8.2.2"