From c0e5d628025cca5dd734baf8a753f0245f7f8cdc Mon Sep 17 00:00:00 2001 From: emmanueltony792-blip Date: Sat, 25 Jul 2026 14:37:35 +0000 Subject: [PATCH] feat: add k6 per-endpoint load tests for #801 #802 #803 #804 - load/pharmacy-compare.js: ramping concurrent GET /pharmacy/compare reads, verifies ranking consistency under db.ts store contention, gates p95/p99 latency and zero 5xx (closes #801) - load/drug-interactions.js: small/large/over-limit drug list scenarios for GET /drug/interactions, confirms MAX_TEXT_LIST_ITEMS=20 cap rejects quickly without running the O(n\xc2\xb2) loop, records interaction-pair counts (closes #802) - load/pharmacy-order.js: concurrent POST /pharmacy/order burst + sustained load, verifies file-lock/atomic-write integrity by comparing 200-count to persisted orders, tracks 503 facilitator throttle rate (closes #803) - load/agent-stream.js: soak + churn scenarios for GET /agent/stream SSE, holds 50 concurrent connections and rapid connect/disconnect cycles to surface broadcastSSE memory leaks and sseClients Set teardown (closes #804) - package.json: add load:pharmacy-compare, load:drug-interactions, load:pharmacy-order, load:agent-stream npm scripts - .gitignore: add coverage/, __snapshots__/, *.snap, test-results/, playwright-report/, .next/, *.tsbuildinfo, OS and editor files --- .gitignore | 30 ++++ load/agent-stream.js | 241 ++++++++++++++++++++++++++++++ load/drug-interactions.js | 302 ++++++++++++++++++++++++++++++++++++++ load/pharmacy-compare.js | 222 ++++++++++++++++++++++++++++ load/pharmacy-order.js | 198 +++++++++++++++++++++++++ package.json | 4 + 6 files changed, 997 insertions(+) create mode 100644 load/agent-stream.js create mode 100644 load/drug-interactions.js create mode 100644 load/pharmacy-compare.js create mode 100644 load/pharmacy-order.js diff --git a/.gitignore b/.gitignore index cd6c1e9..a5ab6f4 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ dist/ .env.local .dev-seed *.log + # data/ is a local working directory — PII-sensitive (medication lists, spending patterns, wallet activity) data/* !data/README.md @@ -12,5 +13,34 @@ data/* !data/.gitkeep data/**/*.json data/**/*.jsonl + +# OS .DS_Store +Thumbs.db + +# Editor / IDE +.vscode/ +.idea/ +*.swp +*.swo fix.md + +# Test artifacts +coverage/ +.nyc_output/ +**/__snapshots__/ +*.snap +test-results/ +playwright-report/ +blob-report/ +playwright/.cache/ + +# Build artifacts +*.tsbuildinfo +.next/ +out/ + +# Temporary files +*.tmp +*.temp +*.bak diff --git a/load/agent-stream.js b/load/agent-stream.js new file mode 100644 index 0000000..fd84381 --- /dev/null +++ b/load/agent-stream.js @@ -0,0 +1,241 @@ +/** + * k6 load test — GET /agent/stream SSE concurrent soak test (Issue #804) + * + * agent/server.ts exposes GET /agent/stream (SSE). The dashboard polls/streams + * aggressively (~333 req/s at 1000 users with 3s polling). This test holds many + * concurrent long-lived SSE connections to verify: + * 1. broadcastSSE does not leak memory or block the event loop + * 2. Connection teardown is clean (sseClients Set is pruned on close) + * 3. Broadcast events reach all connected clients without the event loop stalling + * 4. A circular-reference event payload does not crash broadcast under load + * + * The /agent/stream endpoint: + * - Sets Content-Type: text/event-stream + Connection: keep-alive + * - Immediately sends spending, status, and transactions events on connect + * - Sends ": heartbeat\n\n" every 30s + * - Adds the response to the sseClients Set; removes it on req.close + * + * Usage: + * pnpm load:agent-stream + * # or directly: + * k6 run load/agent-stream.js + * BASE_URL=https://your-app.onrender.com k6 run load/agent-stream.js + * + * Requires: k6 installed (https://k6.io/docs/getting-started/installation/) + * + * Auth note: GET /agent/stream is behind requireApiKey (the /agent/* prefix). + * Provide AGENT_API_KEY via environment: + * AGENT_API_KEY=your-key k6 run load/agent-stream.js + * + * Prerequisites: + * 1. k6 installed + * 2. CareGuard agent server running (node --import tsx agent/server.ts) + * 3. AGENT_API_KEY set (matches the server's AGENT_API_KEY env var) + * Skip auth with a server started without AGENT_API_KEY in non-production mode + */ + +import http from "k6/http"; +import { check, sleep } from "k6"; +import { Counter, Rate, Trend } from "k6/metrics"; + +// --- Metrics --- +const errors5xx = new Counter("errors_5xx"); +const connectionErrors = new Counter("connection_errors"); +const successRate = new Rate("success_rate"); +const connectDuration = new Trend("sse_connect_duration_ms", true); +const initialEventReceived = new Rate("sse_initial_event_received"); + +// --- Config --- +export const options = { + scenarios: { + // Soak: hold N concurrent SSE connections for a sustained period to detect + // memory leaks and event-loop stalls. Each VU opens one connection, reads + // initial events, then holds the connection open. + soak_connections: { + executor: "ramping-vus", + startVUs: 5, + stages: [ + { duration: "30s", target: 30 }, // ramp to 30 concurrent connections + { duration: "2m", target: 50 }, // hold soak at 50 connections + { duration: "30s", target: 0 }, // disconnect all — verify teardown + ], + exec: "holdSseConnection", + }, + // Connection churn: rapid connect/disconnect to verify sseClients Set cleanup + connection_churn: { + executor: "constant-vus", + vus: 10, + duration: "1m", + exec: "churnConnection", + startTime: "30s", + }, + }, + thresholds: { + // Zero 5xx errors on the SSE endpoint + errors_5xx: ["count==0"], + // Zero connection errors (TCP-level failures, not expected HTTP errors) + connection_errors: ["count==0"], + // Connection establishment latency p95 under 500ms + sse_connect_duration_ms: ["p(95)<500"], + // 99%+ of SSE connections established without HTTP error + success_rate: ["rate>0.99"], + // >95% of connections receive at least one initial event + sse_initial_event_received: ["rate>0.95"], + }, +}; + +const BASE_URL = __ENV.BASE_URL || "http://localhost:3004"; +const AGENT_API_KEY = __ENV.AGENT_API_KEY || ""; + +/** + * Build request params with optional API key auth. + * The /agent/* path requires the X-API-Key header when AGENT_API_KEY is set. + */ +function makeRequestParams(extraHeaders) { + const headers = { ...extraHeaders }; + if (AGENT_API_KEY) { + headers["X-API-Key"] = AGENT_API_KEY; + } + return { headers, timeout: "10s" }; +} + +/** + * Scenario: hold a long-lived SSE connection. + * + * k6 does not natively support streaming HTTP responses as an event loop, so this + * uses a bounded read approach: open the connection with a short read timeout to + * capture the initial burst of events (spending + status + transactions sent + * immediately on connect by agent/server.ts), then sleep to simulate a long-held + * connection before disconnecting. + * + * The server-side check is: sseClients.size should return to 0 after all VUs + * disconnect. This is observable via the server logs or a /ready check. + */ +export function holdSseConnection() { + const url = `${BASE_URL}/agent/stream?recipient_id=rosa`; + + // Open the SSE connection and read the initial event burst + // k6 HTTP reads the response body up to the timeout, then returns + const start = Date.now(); + const res = http.get(url, { + ...makeRequestParams({ Accept: "text/event-stream" }), + // Use a short timeout to capture the initial events then release. + // The server sends spending + status + transactions on connect immediately, + // so 3s is enough to receive all three even on a slow connection. + timeout: "3s", + }); + connectDuration.add(Date.now() - start); + + // With a 3s timeout the request will "fail" at the TCP level (timeout), but + // we should have received partial body (the initial events). k6 reports the + // connection as timed out — we check that we at least got a 200 header and + // non-empty body containing the initial event fields. + const gotEvents = check(res, { + "SSE connection established (200 or timeout with partial body)": (r) => + r.status === 200 || (r.error_code === 1050 /* timeout */ && r.body !== null), + "initial events present in body": (r) => { + const body = r.body || ""; + // Server sends three events immediately: spending, status, transactions + return ( + body.includes("event: spending") || + body.includes("event: status") || + body.includes("event: transactions") + ); + }, + "no 5xx response": (r) => r.status < 500, + }); + + if (res.status >= 500) { + errors5xx.add(1); + console.error(`5xx from /agent/stream: ${res.status} — ${(res.body || "").slice(0, 200)}`); + } + + // Count as connection error only on non-timeout transport failures + if (res.error_code && res.error_code !== 1050) { + connectionErrors.add(1); + console.error(`Connection error on /agent/stream: code=${res.error_code} error=${res.error}`); + } + + const hasInitialEvent = + (res.body || "").includes("event: spending") || + (res.body || "").includes("event: status") || + (res.body || "").includes("event: transactions"); + initialEventReceived.add(hasInitialEvent ? 1 : 0); + + successRate.add(gotEvents ? 1 : 0); + + // Simulate client holding the connection open for a while before teardown. + // Randomise to spread disconnect events and avoid simultaneous teardown spike. + sleep(1 + Math.random() * 2); +} + +/** + * Scenario: rapid connect/disconnect churn. + * + * Verifies that the sseClients Set in agent/server.ts is correctly pruned when + * clients disconnect. A leak would cause broadcastSSE to grow its recipient set + * unboundedly and eventually exhaust write buffers or memory. + * + * Each iteration: connect, read initial events (very short timeout), disconnect. + */ +export function churnConnection() { + const url = `${BASE_URL}/agent/stream?recipient_id=rosa`; + + const start = Date.now(); + const res = http.get(url, { + ...makeRequestParams({ Accept: "text/event-stream" }), + timeout: "1s", // very short — just enough for the header + first event + }); + connectDuration.add(Date.now() - start); + + const ok = check(res, { + "churn: connected without 5xx": (r) => + r.status === 200 || r.error_code === 1050 /* timeout after partial read */, + "churn: no 5xx": (r) => r.status < 500, + }); + + if (res.status >= 500) { + errors5xx.add(1); + } + + if (res.error_code && res.error_code !== 1050) { + connectionErrors.add(1); + } + + successRate.add(ok ? 1 : 0); + + // Very short sleep: churn scenario wants rapid reconnects + sleep(0.05 + Math.random() * 0.1); +} + +export function handleSummary(data) { + const p95 = data.metrics.sse_connect_duration_ms?.values?.["p(95)"]?.toFixed(0) ?? "?"; + const totalIter = data.metrics.iterations?.values?.count ?? "?"; + const rate = ((data.metrics.success_rate?.values?.rate ?? 0) * 100).toFixed(1); + const fivexx = data.metrics.errors_5xx?.values?.count ?? 0; + const connErr = data.metrics.connection_errors?.values?.count ?? 0; + const initRate = ((data.metrics.sse_initial_event_received?.values?.rate ?? 0) * 100).toFixed(1); + + return { + stdout: ` +=== CareGuard Agent SSE Stream Load Test Summary (Issue #804) === +Iterations: ${totalIter} +Success rate: ${rate}% +5xx errors: ${fivexx} +Connection errors: ${connErr} +Initial event received rate: ${initRate}% +p95 connection latency: ${p95}ms + +Memory/leak check: + After all VUs disconnect, verify sseClients.size returns to 0. + Check server logs for "sseClients.delete" entries matching your VU count. + If the server's memory grows monotonically after disconnect, broadcastSSE + has a reference leak in the sseClients Set cleanup path. + +Broadcast correctness: + Trigger a /agent/pause or /agent/resume while the soak scenario is running. + All connected clients should receive an SSE 'status' event within one heartbeat + interval (30s). Check k6 stdout for 'event: status' in the body samples above. +`, + }; +} diff --git a/load/drug-interactions.js b/load/drug-interactions.js new file mode 100644 index 0000000..44661f3 --- /dev/null +++ b/load/drug-interactions.js @@ -0,0 +1,302 @@ +/** + * k6 load test — GET /drug/interactions under concurrent load (Issue #802) + * + * Targets services/drug-interaction-api/server.ts's GET /drug/interactions endpoint, + * which runs an O(n²) pair-checking loop over the submitted drug list. The goal is to: + * 1. Confirm the MAX_TEXT_LIST_ITEMS=20 cap rejects over-limit lists quickly (not slowly) + * 2. Verify p95 latency stays bounded under concurrency with large (≤20) drug lists + * 3. Record returned interaction-pair counts across the load run + * + * Usage: + * pnpm load:drug-interactions + * # or directly: + * k6 run load/drug-interactions.js + * BASE_URL=https://your-app.onrender.com k6 run load/drug-interactions.js + * + * Requires: k6 installed (https://k6.io/docs/getting-started/installation/) + * + * IMPORTANT — x402 payment gate: GET /drug/interactions is x402-payment-protected + * ($0.001 per check via OZ Facilitator on Stellar testnet). This script does NOT + * perform real Stellar payments, so it will receive 402 responses against a live + * server with a valid OZ_FACILITATOR_API_KEY. A 402 is treated as an expected + * "payment gate active" response and is NOT a failure. + * + * To load test the actual interaction computation, run the drug interaction server + * with the payment middleware bypassed (remove or mock applyX402Middleware). + * + * CI / manual runs: + * pnpm load:drug-interactions # localhost:3003 + * BASE_URL=http://staging:3003 k6 run load/drug-interactions.js + */ + +import http from "k6/http"; +import { check, sleep } from "k6"; +import { Counter, Rate, Trend } from "k6/metrics"; + +// --- Metrics --- +const errors5xx = new Counter("errors_5xx"); +const successRate = new Rate("success_rate"); +const checkDuration = new Trend("drug_interaction_check_duration_ms", true); +const interactionPairsTotal = new Counter("interaction_pairs_total"); +const overLimitRejections = new Counter("over_limit_rejections"); + +// --- Config --- +export const options = { + scenarios: { + // Small drug lists (2–5 meds): typical real-world use case + small_lists: { + executor: "ramping-vus", + startVUs: 1, + stages: [ + { duration: "30s", target: 10 }, + { duration: "1m", target: 25 }, + { duration: "30s", target: 0 }, + ], + exec: "checkSmallList", + }, + // Large drug lists (18–20 meds, at the cap): stresses the O(n²) path + large_lists: { + executor: "constant-vus", + vus: 5, + duration: "1m", + exec: "checkLargeList", + startTime: "30s", + }, + // Over-limit lists (21+ meds): must be rejected quickly, never processed + over_limit: { + executor: "constant-vus", + vus: 3, + duration: "1m", + exec: "checkOverLimit", + startTime: "30s", + }, + }, + thresholds: { + // Zero 5xx — cap violations must return 400, not 500 + errors_5xx: ["count==0"], + // p95 latency under 600ms (O(n²) on 20 drugs is still fast; if it degrades + // under concurrency that's the signal we want to catch) + drug_interaction_check_duration_ms: ["p(95)<600"], + // 99%+ of requests handled without error + success_rate: ["rate>0.99"], + }, +}; + +const BASE_URL = __ENV.BASE_URL || "http://localhost:3003"; + +// Drugs that exist in the interaction database +// (from services/drug-interaction-api/logic.ts's INTERACTIONS table) +const ALL_KNOWN_DRUGS = [ + "Lisinopril", + "Metformin", + "Atorvastatin", + "Amlodipine", + "Potassium", + "Ibuprofen", + "Omeprazole", + "Alcohol", + "Grapefruit", +]; + +// Extra drug names to pad lists up to the 20-item cap +const EXTRA_DRUGS = [ + "Aspirin", + "Warfarin", + "Clopidogrel", + "Simvastatin", + "Losartan", + "Hydrochlorothiazide", + "Gabapentin", + "Levothyroxine", + "Pantoprazole", + "Sertraline", + "Escitalopram", +]; + +const REQUEST_PARAMS = { + timeout: "10s", +}; + +/** Build a comma-separated drug list string of the requested length. */ +function buildMedsList(count) { + const pool = [...ALL_KNOWN_DRUGS, ...EXTRA_DRUGS]; + const selected = []; + for (let i = 0; i < count; i++) { + selected.push(pool[i % pool.length]); + } + return selected.join(","); +} + +/** + * Scenario: small drug list (2–5 meds). + * Representative of real-world caregiver use: Rosa's 4 medications. + */ +export function checkSmallList() { + const count = 2 + Math.floor(Math.random() * 4); // 2, 3, 4, or 5 + const meds = buildMedsList(count); + const url = `${BASE_URL}/drug/interactions?meds=${encodeURIComponent(meds)}`; + + const start = Date.now(); + const res = http.get(url, REQUEST_PARAMS); + checkDuration.add(Date.now() - start); + + const ok = check(res, { + "status is not 5xx": (r) => r.status < 500, + "status is 200 or 402 (payment gate)": (r) => + r.status === 200 || r.status === 402, + "200 response has interactionCount": (r) => { + if (r.status !== 200) return true; + try { + const body = JSON.parse(r.body); + return typeof body.interactionCount === "number"; + } catch { + return false; + } + }, + "200 response has interactions array": (r) => { + if (r.status !== 200) return true; + try { + const body = JSON.parse(r.body); + return Array.isArray(body.interactions); + } catch { + return false; + } + }, + }); + + if (res.status >= 500) { + errors5xx.add(1); + console.error(`5xx from /drug/interactions (small): ${res.status} — ${res.body.slice(0, 200)}`); + } + + if (res.status === 200) { + try { + const body = JSON.parse(res.body); + interactionPairsTotal.add(body.interactionCount ?? 0); + } catch { + // parse failure already flagged by checks above + } + } + + successRate.add(ok ? 1 : 0); + sleep(0.1); +} + +/** + * Scenario: large drug list (18–20 meds, at the cap boundary). + * Stresses the O(n²) pair-checking loop. p95 must stay within budget even + * under concurrent load. + */ +export function checkLargeList() { + // 18, 19, or 20 — all within MAX_TEXT_LIST_ITEMS=20 + const count = 18 + Math.floor(Math.random() * 3); + const meds = buildMedsList(count); + const url = `${BASE_URL}/drug/interactions?meds=${encodeURIComponent(meds)}`; + + const start = Date.now(); + const res = http.get(url, REQUEST_PARAMS); + checkDuration.add(Date.now() - start); + + const ok = check(res, { + "status is not 5xx": (r) => r.status < 500, + "status is 200 or 402 (payment gate)": (r) => + r.status === 200 || r.status === 402, + "large list 200 has interactionCount": (r) => { + if (r.status !== 200) return true; + try { + const body = JSON.parse(r.body); + return typeof body.interactionCount === "number"; + } catch { + return false; + } + }, + }); + + if (res.status >= 500) { + errors5xx.add(1); + console.error(`5xx from /drug/interactions (large): ${res.status} — ${res.body.slice(0, 200)}`); + } + + if (res.status === 200) { + try { + const body = JSON.parse(res.body); + interactionPairsTotal.add(body.interactionCount ?? 0); + } catch { + // ignore + } + } + + successRate.add(ok ? 1 : 0); + sleep(0.2); +} + +/** + * Scenario: over-limit drug list (21+ meds, exceeds MAX_TEXT_LIST_ITEMS=20). + * These must be rejected QUICKLY (400 validation error), not processed — + * the O(n²) loop must never run on an over-limit list. + * Rejection latency is tracked: if it's high, the cap is not enforcing early. + */ +export function checkOverLimit() { + // 21–25 drugs, clearly over the cap + const count = 21 + Math.floor(Math.random() * 5); + const meds = buildMedsList(count); + const url = `${BASE_URL}/drug/interactions?meds=${encodeURIComponent(meds)}`; + + const start = Date.now(); + const res = http.get(url, REQUEST_PARAMS); + const elapsed = Date.now() - start; + checkDuration.add(elapsed); + + const ok = check(res, { + "over-limit list is rejected with 400 (not 500)": (r) => r.status === 400, + "over-limit rejection is fast (< 200ms)": (_r) => elapsed < 200, + "400 body has error message": (r) => { + if (r.status !== 400) return true; + try { + const body = JSON.parse(r.body); + return typeof body.error === "string" && body.error.length > 0; + } catch { + return false; + } + }, + }); + + if (res.status >= 500) { + errors5xx.add(1); + console.error(`5xx from /drug/interactions (over-limit): ${res.status} — ${res.body.slice(0, 200)}`); + } + + if (res.status === 400) { + overLimitRejections.add(1); + } + + successRate.add(ok ? 1 : 0); + sleep(0.1); +} + +export function handleSummary(data) { + const p95 = data.metrics.drug_interaction_check_duration_ms?.values?.["p(95)"]?.toFixed(0) ?? "?"; + const totalIter = data.metrics.iterations?.values?.count ?? "?"; + const rate = ((data.metrics.success_rate?.values?.rate ?? 0) * 100).toFixed(1); + const fivexx = data.metrics.errors_5xx?.values?.count ?? 0; + const pairs = data.metrics.interaction_pairs_total?.values?.count ?? 0; + const rejections = data.metrics.over_limit_rejections?.values?.count ?? 0; + + return { + stdout: ` +=== CareGuard Drug Interactions Load Test Summary (Issue #802) === +Iterations: ${totalIter} +Success rate: ${rate}% +5xx errors: ${fivexx} +p95 check latency: ${p95}ms +Total interaction pairs seen: ${pairs} +Over-limit rejections (400): ${rejections} + +Note: 402 responses indicate the x402 payment gate is active — see script header +for how to bypass it for load testing the interaction computation path directly. + +Over-limit rejection fast-path: if the rejection latency is >200ms, the Zod +validation cap is not short-circuiting before the O(n²) loop. +`, + }; +} diff --git a/load/pharmacy-compare.js b/load/pharmacy-compare.js new file mode 100644 index 0000000..3204d02 --- /dev/null +++ b/load/pharmacy-compare.js @@ -0,0 +1,222 @@ +/** + * k6 load test — GET /pharmacy/compare under ramping concurrent reads (Issue #801) + * + * Targets services/pharmacy-api/server.ts's GET /pharmacy/compare endpoint, which is + * backed by a db.ts SQLite store. The goal is to surface any cache/store contention + * under concurrent reads, verify ranking consistency, and gate latency/error budgets. + * + * Usage: + * pnpm load:pharmacy-compare + * # or directly: + * k6 run load/pharmacy-compare.js + * BASE_URL=https://your-app.onrender.com k6 run load/pharmacy-compare.js + * + * Requires: k6 installed (https://k6.io/docs/getting-started/installation/) + * + * IMPORTANT — x402 payment gate: GET /pharmacy/compare is x402-payment-protected + * ($0.002 per query via OZ Facilitator on Stellar testnet). This script does NOT + * perform real Stellar payments, so it will receive 402 responses against a live + * server with a valid OZ_FACILITATOR_API_KEY. To load test the actual comparison + * computation, run the server with payments disabled (set ENABLE_PAYMENTS=false or + * pass enablePayments: false to createPharmacyApp). A 402 is treated as an expected + * "payment gate active" response — it is NOT a failure. + * + * Server start with payments disabled (for load testing): + * ENABLE_PAYMENTS=false node --import tsx services/pharmacy-api/server.ts + */ + +import http from "k6/http"; +import { check, sleep } from "k6"; +import { Counter, Rate, Trend } from "k6/metrics"; + +// --- Metrics --- +const errors5xx = new Counter("errors_5xx"); +const successRate = new Rate("success_rate"); +const compareDuration = new Trend("pharmacy_compare_duration_ms", true); +const nonEmptyResults = new Counter("non_empty_compare_results"); +const emptyResults = new Counter("empty_compare_results"); + +// --- Config --- +export const options = { + scenarios: { + // Ramp up concurrent readers to surface db.ts store contention + ramping_reads: { + executor: "ramping-vus", + startVUs: 1, + stages: [ + { duration: "30s", target: 15 }, // ramp to 15 VUs + { duration: "1m", target: 30 }, // hold at 30 VUs (hot path) + { duration: "30s", target: 0 }, // ramp down + ], + exec: "compareKnownDrug", + }, + // Small parallel scenario hitting varied drug+zip combos to test ranking consistency + varied_params: { + executor: "constant-vus", + vus: 5, + duration: "1m", + exec: "compareVariedParams", + startTime: "30s", // overlap with ramp-up phase, not cooldown + }, + }, + thresholds: { + // Zero 5xx errors — payment-gate 402s are expected and OK + errors_5xx: ["count==0"], + // p95 latency under 800ms for the db-backed store read path + pharmacy_compare_duration_ms: ["p(95)<800", "p(99)<1500"], + // Overall success rate (200 or 402 accepted) above 99% + success_rate: ["rate>0.99"], + }, +}; + +const BASE_URL = __ENV.BASE_URL || "http://localhost:3001"; + +// Representative drug/zip combos that exist in the seeded pricing database +// (matches the drugs in shared/pharmacy-pricing.ts and services/pharmacy-api/seed.ts) +const KNOWN_DRUG_PARAMS = [ + { drug: "Lisinopril", zip: "90210" }, + { drug: "Metformin", zip: "10001" }, + { drug: "Atorvastatin", zip: "60601" }, + { drug: "Amlodipine", zip: "77001" }, + { drug: "Omeprazole", zip: "30301" }, +]; + +// Params that are likely not in the DB — should return 404 (not 5xx) +const UNKNOWN_DRUG_PARAMS = [ + { drug: "UnknownDrugXYZ", zip: "90210" }, +]; + +const REQUEST_PARAMS = { + headers: { "Content-Type": "application/json" }, + timeout: "10s", +}; + +/** + * Scenario: compare a known drug — should get 200 (payments disabled) or 402 (payments on). + * Verifies non-empty comparison results are returned under concurrent load. + */ +export function compareKnownDrug() { + const entry = KNOWN_DRUG_PARAMS[Math.floor(Math.random() * KNOWN_DRUG_PARAMS.length)]; + const url = `${BASE_URL}/pharmacy/compare?drug=${encodeURIComponent(entry.drug)}&zip=${entry.zip}`; + + const start = Date.now(); + const res = http.get(url, REQUEST_PARAMS); + compareDuration.add(Date.now() - start); + + const ok = check(res, { + "status is not 5xx": (r) => r.status < 500, + "status is 200, 402, or 404": (r) => + r.status === 200 || r.status === 402 || r.status === 404, + "200 response has prices array": (r) => { + if (r.status !== 200) return true; // payment-gated or not found — skip + try { + const body = JSON.parse(r.body); + return Array.isArray(body.prices); + } catch { + return false; + } + }, + "200 prices array is non-empty": (r) => { + if (r.status !== 200) return true; + try { + const body = JSON.parse(r.body); + return Array.isArray(body.prices) && body.prices.length > 0; + } catch { + return false; + } + }, + "200 ranking is consistent (cheapest first)": (r) => { + if (r.status !== 200) return true; + try { + const body = JSON.parse(r.body); + if (!Array.isArray(body.prices) || body.prices.length < 2) return true; + for (let i = 1; i < body.prices.length; i++) { + if (body.prices[i].price < body.prices[i - 1].price) return false; + } + return true; + } catch { + return false; + } + }, + }); + + if (res.status >= 500) { + errors5xx.add(1); + console.error(`5xx from /pharmacy/compare: ${res.status} — ${res.body.slice(0, 200)}`); + } + + if (res.status === 200) { + try { + const body = JSON.parse(res.body); + if (Array.isArray(body.prices) && body.prices.length > 0) { + nonEmptyResults.add(1); + } else { + emptyResults.add(1); + } + } catch { + emptyResults.add(1); + } + } + + successRate.add(ok ? 1 : 0); + sleep(0.1); +} + +/** + * Scenario: varied drug+zip combos including unknown drugs. + * Confirms unknown drugs return 404 (never 5xx) and store read stays consistent + * across different query parameters under concurrency. + */ +export function compareVariedParams() { + // 80% known, 20% unknown + const useUnknown = Math.random() < 0.2; + const pool = useUnknown ? UNKNOWN_DRUG_PARAMS : KNOWN_DRUG_PARAMS; + const entry = pool[Math.floor(Math.random() * pool.length)]; + const url = `${BASE_URL}/pharmacy/compare?drug=${encodeURIComponent(entry.drug)}&zip=${entry.zip}`; + + const start = Date.now(); + const res = http.get(url, REQUEST_PARAMS); + compareDuration.add(Date.now() - start); + + const ok = check(res, { + "status is not 5xx": (r) => r.status < 500, + "unknown drug gets 404, known gets 200/402": (r) => { + if (useUnknown) return r.status === 404; + return r.status === 200 || r.status === 402; + }, + }); + + if (res.status >= 500) { + errors5xx.add(1); + console.error(`5xx from /pharmacy/compare (varied): ${res.status} — ${res.body.slice(0, 200)}`); + } + + successRate.add(ok ? 1 : 0); + sleep(0.15); +} + +export function handleSummary(data) { + const p95 = data.metrics.pharmacy_compare_duration_ms?.values?.["p(95)"]?.toFixed(0) ?? "?"; + const p99 = data.metrics.pharmacy_compare_duration_ms?.values?.["p(99)"]?.toFixed(0) ?? "?"; + const totalIter = data.metrics.iterations?.values?.count ?? "?"; + const rate = ((data.metrics.success_rate?.values?.rate ?? 0) * 100).toFixed(1); + const fivexx = data.metrics.errors_5xx?.values?.count ?? 0; + const nonEmpty = data.metrics.non_empty_compare_results?.values?.count ?? 0; + const empty = data.metrics.empty_compare_results?.values?.count ?? 0; + + return { + stdout: ` +=== CareGuard Pharmacy Compare Load Test Summary (Issue #801) === +Iterations: ${totalIter} +Success rate: ${rate}% +5xx errors: ${fivexx} +p95 compare latency: ${p95}ms +p99 compare latency: ${p99}ms +Non-empty results: ${nonEmpty} +Empty results: ${empty} + +Note: 402 responses indicate the x402 payment gate is active — see script header +for how to disable payments on the server for load testing the compare path directly. +`, + }; +} diff --git a/load/pharmacy-order.js b/load/pharmacy-order.js new file mode 100644 index 0000000..c4010f7 --- /dev/null +++ b/load/pharmacy-order.js @@ -0,0 +1,198 @@ +/** + * k6 load test — POST /pharmacy/order concurrent burst (Issue #803) + * + * Targets services/pharmacy-payment/server.ts's POST /pharmacy/order endpoint, + * which is protected by the MPP Charge payment flow. The goal is to verify: + * 1. Concurrent orders do not corrupt the orders store (file-lock + atomic write) + * 2. Order count persisted equals successful 2xx responses (no lost/duplicated orders) + * 3. 503 responses (facilitator unavailable) stay within an allowed bound + * 4. p95 latency stays within budget under burst + * + * Usage: + * pnpm load:pharmacy-order + * # or directly: + * k6 run load/pharmacy-order.js + * BASE_URL=https://your-app.onrender.com k6 run load/pharmacy-order.js + * + * Requires: k6 installed (https://k6.io/docs/getting-started/installation/) + * + * IMPORTANT — MPP payment gate: POST /pharmacy/order triggers the MPP Charge flow. + * Without a valid Stellar payment the server returns a 402 challenge. This script + * does NOT sign Stellar transactions, so against a live server it will receive 402 + * responses. A 402 is treated as an expected "payment challenge" response — it is + * NOT a failure, because the server must issue it before processing the order. + * + * To load test the actual order persistence path (past the payment gate), run the + * server with a mock MPP facilitator that always returns "payment verified". There is + * currently no in-repo mock facilitator — see docs/load-testing.md for setup notes. + * + * How to start the server with a mock facilitator for load runs: + * MPP_SECRET_KEY=SXXXXX... PHARMACY_1_PUBLIC_KEY=GXXXXX... \ + * MPP_MOCK=true node --import tsx services/pharmacy-payment/server.ts + * + * Concurrency / corruption verification: + * After the burst scenario completes, handleSummary() fetches GET /pharmacy/orders + * and compares the persisted order count to the number of 200 responses received. + * A mismatch indicates lost writes or duplicate saves from the file-lock path. + */ + +import http from "k6/http"; +import { check, sleep } from "k6"; +import { Counter, Rate, Trend } from "k6/metrics"; + +// --- Metrics --- +const errors5xx = new Counter("errors_5xx"); +const errors503 = new Counter("errors_503"); +const successRate = new Rate("success_rate"); +const orderDuration = new Trend("pharmacy_order_duration_ms", true); +const orders200 = new Counter("orders_200_success"); +const orders402 = new Counter("orders_402_challenge"); + +// --- Config --- +export const options = { + scenarios: { + // Burst scenario: concurrent orders to stress file-lock + atomic write + concurrent_orders: { + executor: "shared-iterations", + vus: 20, + iterations: 60, + maxDuration: "2m", + exec: "placeOrder", + }, + // Sustained load to check for slow degradation of the store + sustained_load: { + executor: "constant-vus", + vus: 5, + duration: "1m", + exec: "placeOrder", + startTime: "30s", + }, + }, + thresholds: { + // Zero 5xx errors (402 challenge and 503 are expected, not 5xx) + errors_5xx: ["count==0"], + // 503 (facilitator throttled) must be rare — at most 5% of total requests + errors_503: ["count<10"], + // p95 order latency under 2s (includes 402 round-trip overhead) + pharmacy_order_duration_ms: ["p(95)<2000"], + // 99%+ handled without 5xx + success_rate: ["rate>0.99"], + }, +}; + +const BASE_URL = __ENV.BASE_URL || "http://localhost:3005"; + +// Representative medication orders matching realistic caregiver use +const MEDICATION_ORDERS = [ + { drug: "Lisinopril", pharmacy: "Costco Pharmacy", amount: "12.00" }, + { drug: "Metformin", pharmacy: "CVS Pharmacy", amount: "8.50" }, + { drug: "Atorvastatin", pharmacy: "Walgreens", amount: "15.75" }, + { drug: "Amlodipine", pharmacy: "Rite Aid", amount: "9.20" }, + { drug: "Omeprazole", pharmacy: "Walmart Pharmacy", amount: "11.30" }, +]; + +const REQUEST_PARAMS = { + headers: { "Content-Type": "application/json" }, + timeout: "15s", +}; + +/** + * Place a single medication order against the MPP-protected endpoint. + * Records 200 (order confirmed), 402 (payment challenge), 503 (facilitator down), + * and any 5xx as separate metrics for post-run analysis. + */ +export function placeOrder() { + const order = MEDICATION_ORDERS[Math.floor(Math.random() * MEDICATION_ORDERS.length)]; + const payload = JSON.stringify(order); + + const start = Date.now(); + const res = http.post(`${BASE_URL}/pharmacy/order`, payload, REQUEST_PARAMS); + orderDuration.add(Date.now() - start); + + const ok = check(res, { + "status is not 5xx": (r) => r.status < 500, + "status is 200, 402, or 503": (r) => + r.status === 200 || r.status === 402 || r.status === 503, + "200 response has order.id": (r) => { + if (r.status !== 200) return true; + try { + const body = JSON.parse(r.body); + return typeof body.order?.id === "string"; + } catch { + return false; + } + }, + "402 response has payment challenge body": (r) => { + if (r.status !== 402) return true; + // 402 challenge body may be text or JSON — just verify it's non-empty + return r.body !== null && r.body.length > 0; + }, + }); + + if (res.status >= 500 && res.status !== 503) { + errors5xx.add(1); + console.error(`5xx from /pharmacy/order: ${res.status} — ${res.body.slice(0, 200)}`); + } else if (res.status === 503) { + errors503.add(1); + } else if (res.status === 200) { + orders200.add(1); + } else if (res.status === 402) { + orders402.add(1); + } + + successRate.add(ok ? 1 : 0); + + // Brief sleep to avoid thundering herd on the file lock + sleep(0.05 + Math.random() * 0.1); +} + +export function handleSummary(data) { + // Fetch persisted order count to verify against 200 responses + let persistenceNote = "Could not fetch /pharmacy/orders for persistence check"; + const ordersRes = http.get(`${BASE_URL}/pharmacy/orders`, { timeout: "5s" }); + if (ordersRes.status === 200) { + try { + const body = JSON.parse(ordersRes.body); + const persistedCount = Array.isArray(body.orders) ? body.orders.length : "unknown"; + const confirmed200 = data.metrics.orders_200_success?.values?.count ?? 0; + + // Note: the test run accumulates on top of any pre-existing orders in the file, + // so we report both and flag if confirmed200 > persistedCount (lost writes). + const lostWriteFlag = + typeof persistedCount === "number" && confirmed200 > persistedCount + ? `⚠ POSSIBLE LOST WRITES: ${confirmed200} 200-responses but only ${persistedCount} persisted orders` + : `✅ Persistence check: ${confirmed200} 200-responses, ${persistedCount} total orders in store`; + + persistenceNote = lostWriteFlag; + } catch { + persistenceNote = "Failed to parse /pharmacy/orders response"; + } + } + + const p95 = data.metrics.pharmacy_order_duration_ms?.values?.["p(95)"]?.toFixed(0) ?? "?"; + const totalIter = data.metrics.iterations?.values?.count ?? "?"; + const rate = ((data.metrics.success_rate?.values?.rate ?? 0) * 100).toFixed(1); + const fivexx = data.metrics.errors_5xx?.values?.count ?? 0; + const fiveohthree = data.metrics.errors_503?.values?.count ?? 0; + const confirmed = data.metrics.orders_200_success?.values?.count ?? 0; + const challenged = data.metrics.orders_402_challenge?.values?.count ?? 0; + + return { + stdout: ` +=== CareGuard Pharmacy Order Load Test Summary (Issue #803) === +Iterations: ${totalIter} +Success rate: ${rate}% +5xx errors: ${fivexx} +503 (facilitator down): ${fiveohthree} +200 (orders confirmed): ${confirmed} +402 (payment challenge): ${challenged} +p95 order latency: ${p95}ms + +${persistenceNote} + +Note: 402 responses mean the MPP payment gate is active — no real Stellar payment +was signed. See script header for how to run with a mock facilitator to test the +order-persistence path under concurrent load. +`, + }; +} diff --git a/package.json b/package.json index e1725f8..8734a45 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,10 @@ "e2e": "cd dashboard && npx playwright test --config ../e2e/playwright.config.ts", "load": "k6 run load/agent-run.js", "load:bill-audit": "k6 run load/bill-audit.js", + "load:pharmacy-compare": "k6 run load/pharmacy-compare.js", + "load:drug-interactions": "k6 run load/drug-interactions.js", + "load:pharmacy-order": "k6 run load/pharmacy-order.js", + "load:agent-stream": "k6 run load/agent-stream.js", "check:env-vars": "tsx scripts/check-env-vars.ts" }, "keywords": [],