diff --git a/README.md b/README.md index c57a827..a2dcac1 100644 --- a/README.md +++ b/README.md @@ -332,7 +332,7 @@ See [Vault payments](docs/vault-payments.md) for both provider flows, safety rul - `webmcp` - List native page tools across every tab and frame in a browser, then synchronously invoke an exact opaque `tool_ref` with structured input. - `exec_command` - Run shell commands inside a browser VM. Returns decoded stdout/stderr. - `search_docs` - Search Kernel platform documentation and guides. -- `submit_feedback` - send product, mcp, or documentation feedback directly to the KERNEL team without interrupting the current task. +- `submit_feedback` - send product, bot-detection, config-registry, mcp, or documentation feedback directly to the KERNEL team without interrupting the current task. Config-registry reports connect the observed site outcome to the browser session, recommendation metadata and evidence, and exact browser and proxy settings applied unchanged; general bot-detection reports remain available for outcomes not tied to a registry recommendation. - `open_auth_login` - Open a secure interactive Managed Auth MCP App after user consent. Registered only for clients that declare MCP Apps support; credentials and MFA never enter MCP/model traffic. ## Resources diff --git a/bun.lock b/bun.lock index ced2fac..d9bdca1 100644 --- a/bun.lock +++ b/bun.lock @@ -29,6 +29,7 @@ "react": "^19.2.1", "react-dom": "^19.2.1", "redis": "^5.6.0", + "tldts": "^7.4.13", "typescript": "^5", }, "devDependencies": { @@ -577,6 +578,10 @@ "tar": ["tar@7.4.3", "", { "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", "minipass": "^7.1.2", "minizlib": "^3.0.1", "mkdirp": "^3.0.1", "yallist": "^5.0.0" } }, "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw=="], + "tldts": ["tldts@7.4.13", "", { "dependencies": { "tldts-core": "^7.4.13" }, "bin": { "tldts": "bin/cli.js" } }, "sha512-iHtaIWWIbMDkCeJdTBzZFGgbluE5J+oHlb2g7+oAz1S1gpuVpabRZdQyd471Vl8UUkcz2vXSL8xZH2kyCe8tfA=="], + + "tldts-core": ["tldts-core@7.4.13", "", {}, "sha512-mbYsrih5FRtGxs3Usvl/PqwJsNpp+jsmrdFviiK02teHDG0/HebBG/pqCylje3kzgXYzuLoHJF/0mz9W53t8Xg=="], + "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="], "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], diff --git a/package.json b/package.json index 222b0f4..c42a705 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ "react": "^19.2.1", "react-dom": "^19.2.1", "redis": "^5.6.0", + "tldts": "^7.4.13", "typescript": "^5" }, "devDependencies": { diff --git a/src/lib/mcp/analytics.test.ts b/src/lib/mcp/analytics.test.ts index 00e5f83..d478fcd 100644 --- a/src/lib/mcp/analytics.test.ts +++ b/src/lib/mcp/analytics.test.ts @@ -613,12 +613,13 @@ describe("captureMcpFeedback", () => { expect(captured).toEqual([ { event: MCP_FEEDBACK_SUBMITTED_EVENT, - properties: { + properties: expect.objectContaining({ $groups: { organization: "org_analytics" }, feedback_summary: "Browser timeout guidance was unclear", feedback_type: "product", feedback_sentiment: "mixed", feedback_product_area: "browsers", + feedback_destination: undefined, feedback_category: undefined, feedback_task_completed: true, feedback_tools_used: ["manage_browsers"], @@ -627,8 +628,162 @@ describe("captureMcpFeedback", () => { "Include a retry interval in the response.", feedback_user_request: undefined, feedback_details: "The error linked to [url] for [email]", + }), + }, + ]); + }); + + test("routes structured bot-detection feedback to config registry prioritization", async () => { + const captured: unknown[] = []; + const analytics = { + capture: async (event: unknown) => { + captured.push(event); + }, + } as McpAnalytics; + + await captureMcpFeedback( + { + summary: "Stealth sessions were consistently blocked", + feedback_type: "bot_detection", + sentiment: "negative", + task_completed: false, + tools_used: ["manage_browsers", "execute_playwright_code"], + bot_detection: { + registrable_domain: "example.com", + observed_outcome: "blocked", + suspected_vendor: "Akamai Bot Manager", + challenge_type: "access_denied", + stealth: "enabled", + proxy_type: "isp", + region: "us-east", + browser_version: "152.0.7977.42", + browser_image_version: "2026.09.14", + reproducibility: "consistent", + browser_session_id: "session_123", + }, + }, + { + authInfo: { + extra: { + connectionContext: { + scope: { organizationId: "org_analytics" }, + }, + }, }, }, + analytics, + ); + + expect(captured).toEqual([ + { + event: MCP_FEEDBACK_SUBMITTED_EVENT, + properties: expect.objectContaining({ + $groups: { organization: "org_analytics" }, + feedback_type: "bot_detection", + feedback_destination: "config_registry_prioritization", + feedback_bot_detection_registrable_domain: "example.com", + feedback_bot_detection_observed_outcome: "blocked", + feedback_bot_detection_suspected_vendor: "Akamai Bot Manager", + feedback_bot_detection_challenge_type: "access_denied", + feedback_bot_detection_stealth: "enabled", + feedback_bot_detection_proxy_type: "isp", + feedback_bot_detection_region: "us-east", + feedback_bot_detection_browser_version: "152.0.7977.42", + feedback_bot_detection_browser_image_version: "2026.09.14", + feedback_bot_detection_reproducibility: "consistent", + feedback_bot_detection_browser_session_id: "session_123", + }), + }, + ]); + }); + + test("attributes config-registry feedback to the applied configuration", async () => { + const captured: unknown[] = []; + const analytics = { + capture: async (event: unknown) => { + captured.push(event); + }, + } as McpAnalytics; + + await captureMcpFeedback( + { + summary: "The recommended configuration remained blocked", + feedback_type: "config_registry", + sentiment: "negative", + task_completed: false, + bot_detection: { + registrable_domain: "example.com", + observed_outcome: "blocked", + challenge_type: "access_denied", + reproducibility: "consistent", + browser_session_id: "session_456", + }, + config_registry: { + request_method: "resolve", + analysis_id: "analysis_123", + recommendation_match_scope: "exact", + recommendation_verification: "verified", + recommendation_evidence: { + sample_size: 5, + success_rate: 1, + last_verified_at: "2026-09-13T12:00:00Z", + }, + applied_browser: { + stealth: true, + headless: false, + gpu: false, + viewport: { width: 1920, height: 1080, refresh_rate: 25 }, + }, + applied_proxy: { + mode: "managed", + type: "residential", + country: "US", + }, + }, + }, + { + authInfo: { + extra: { + connectionContext: { + scope: { organizationId: "org_analytics" }, + }, + }, + }, + }, + analytics, + ); + + expect(captured).toEqual([ + { + event: MCP_FEEDBACK_SUBMITTED_EVENT, + properties: expect.objectContaining({ + $groups: { organization: "org_analytics" }, + feedback_type: "config_registry", + feedback_destination: "config_registry_quality", + feedback_bot_detection_registrable_domain: "example.com", + feedback_bot_detection_observed_outcome: "blocked", + feedback_config_registry_request_method: "resolve", + feedback_config_registry_analysis_id: "analysis_123", + feedback_config_registry_recommendation_match_scope: "exact", + feedback_config_registry_recommendation_verification: "verified", + feedback_config_registry_evidence_sample_size: 5, + feedback_config_registry_evidence_success_rate: 1, + feedback_config_registry_evidence_last_verified_at: + "2026-09-13T12:00:00Z", + feedback_config_registry_applied_config_key: + "stealth-true|headless-false|gpu-false|viewport-1920x1080@25|proxy-managed-residential-US", + feedback_config_registry_browser_stealth: true, + feedback_config_registry_browser_headless: false, + feedback_config_registry_browser_gpu: false, + feedback_config_registry_viewport_width: 1920, + feedback_config_registry_viewport_height: 1080, + feedback_config_registry_viewport_refresh_rate: 25, + feedback_config_registry_proxy_mode: "managed", + feedback_config_registry_proxy_type: "residential", + feedback_config_registry_proxy_country: "US", + feedback_task_completed: false, + }), + }, ]); }); }); @@ -815,11 +970,17 @@ describe("instrumentMcpAnalytics (SDK integration)", () => { name: KERNEL_FEEDBACK_TOOL_NAME, arguments: { context: - "Reporting that browser timeout guidance did not explain when the caller should retry.", - summary: "Browser timeout guidance was unclear", - feedback_type: "product", - sentiment: "mixed", - product_area: "browsers", + "Reporting a repeatable site block so the affected domain can be prioritized for a working browser configuration.", + summary: "Stealth sessions were consistently blocked", + feedback_type: "bot_detection", + sentiment: "negative", + task_completed: false, + bot_detection: { + registrable_domain: "example.com", + observed_outcome: "blocked", + suspected_vendor: "Akamai Bot Manager", + reproducibility: "consistent", + }, }, }); @@ -843,13 +1004,18 @@ describe("instrumentMcpAnalytics (SDK integration)", () => { [PostHogMCPAnalyticsProperty.ProtocolVersion]: "2025-03-26", [PostHogMCPAnalyticsProperty.ServerName]: "test", [PostHogMCPAnalyticsProperty.ServerVersion]: "0.0.0", - feedback_summary: "Browser timeout guidance was unclear", - feedback_type: "product", - feedback_sentiment: "mixed", - feedback_product_area: "browsers", + feedback_summary: "Stealth sessions were consistently blocked", + feedback_type: "bot_detection", + feedback_sentiment: "negative", + feedback_task_completed: false, + feedback_destination: "config_registry_prioritization", + feedback_bot_detection_registrable_domain: "example.com", + feedback_bot_detection_observed_outcome: "blocked", + feedback_bot_detection_suspected_vendor: "Akamai Bot Manager", + feedback_bot_detection_reproducibility: "consistent", }); expect(toolCall.properties[PostHogMCPAnalyticsProperty.Intent]).toBe( - "Reporting that browser timeout guidance did not explain when the caller should retry.", + "Reporting a repeatable site block so the affected domain can be prioritized for a working browser configuration.", ); }); diff --git a/src/lib/mcp/analytics.ts b/src/lib/mcp/analytics.ts index 6a3eaff..6ca8360 100644 --- a/src/lib/mcp/analytics.ts +++ b/src/lib/mcp/analytics.ts @@ -164,6 +164,35 @@ const SENT_PROPERTIES = new Set([ "feedback_type", "feedback_sentiment", "feedback_product_area", + "feedback_destination", + "feedback_bot_detection_registrable_domain", + "feedback_bot_detection_observed_outcome", + "feedback_bot_detection_suspected_vendor", + "feedback_bot_detection_challenge_type", + "feedback_bot_detection_stealth", + "feedback_bot_detection_proxy_type", + "feedback_bot_detection_region", + "feedback_bot_detection_browser_version", + "feedback_bot_detection_browser_image_version", + "feedback_bot_detection_reproducibility", + "feedback_bot_detection_browser_session_id", + "feedback_config_registry_request_method", + "feedback_config_registry_analysis_id", + "feedback_config_registry_recommendation_match_scope", + "feedback_config_registry_recommendation_verification", + "feedback_config_registry_evidence_sample_size", + "feedback_config_registry_evidence_success_rate", + "feedback_config_registry_evidence_last_verified_at", + "feedback_config_registry_applied_config_key", + "feedback_config_registry_browser_stealth", + "feedback_config_registry_browser_headless", + "feedback_config_registry_browser_gpu", + "feedback_config_registry_viewport_width", + "feedback_config_registry_viewport_height", + "feedback_config_registry_viewport_refresh_rate", + "feedback_config_registry_proxy_mode", + "feedback_config_registry_proxy_type", + "feedback_config_registry_proxy_country", "feedback_category", "feedback_task_completed", "feedback_tools_used", @@ -512,11 +541,38 @@ export function captureMcpConnectionScopeFailure( } } +function configRegistryAppliedConfigKey( + configRegistry: NonNullable, +) { + const browser = configRegistry.applied_browser; + const viewport = browser.viewport; + const proxy = configRegistry.applied_proxy; + const proxyKey = + proxy.mode === "direct" + ? "direct" + : `managed-${proxy.type}-${proxy.country ?? "default"}`; + return [ + `stealth-${browser.stealth}`, + `headless-${browser.headless}`, + `gpu-${browser.gpu}`, + `viewport-${viewport.width}x${viewport.height}@${viewport.refresh_rate ?? "default"}`, + `proxy-${proxyKey}`, + ].join("|"); +} + export function captureMcpFeedback( feedback: KernelFeedback, extra: unknown, analytics: McpAnalytics, ) { + const isSiteOutcome = + feedback.feedback_type === "bot_detection" || + feedback.feedback_type === "config_registry"; + const botDetection = isSiteOutcome ? feedback.bot_detection : undefined; + const configRegistry = + feedback.feedback_type === "config_registry" + ? feedback.config_registry + : undefined; return captureMcpCustomEvent(analytics, extra, MCP_FEEDBACK_SUBMITTED_EVENT, { feedback_summary: redactAnalyticsText(feedback.summary), feedback_type: feedback.feedback_type, @@ -524,6 +580,70 @@ export function captureMcpFeedback( feedback_product_area: feedback.product_area ? redactAnalyticsText(feedback.product_area) : undefined, + feedback_destination: configRegistry + ? "config_registry_quality" + : botDetection + ? "config_registry_prioritization" + : undefined, + feedback_bot_detection_registrable_domain: botDetection?.registrable_domain, + feedback_bot_detection_observed_outcome: botDetection?.observed_outcome, + feedback_bot_detection_suspected_vendor: botDetection?.suspected_vendor + ? redactAnalyticsText(botDetection.suspected_vendor) + : undefined, + feedback_bot_detection_challenge_type: botDetection?.challenge_type, + feedback_bot_detection_stealth: botDetection?.stealth, + feedback_bot_detection_proxy_type: botDetection?.proxy_type, + feedback_bot_detection_region: botDetection?.region + ? redactAnalyticsText(botDetection.region) + : undefined, + feedback_bot_detection_browser_version: botDetection?.browser_version + ? redactAnalyticsText(botDetection.browser_version) + : undefined, + feedback_bot_detection_browser_image_version: + botDetection?.browser_image_version + ? redactAnalyticsText(botDetection.browser_image_version) + : undefined, + feedback_bot_detection_reproducibility: botDetection?.reproducibility, + feedback_bot_detection_browser_session_id: botDetection?.browser_session_id + ? redactAnalyticsText(botDetection.browser_session_id) + : undefined, + feedback_config_registry_request_method: configRegistry?.request_method, + feedback_config_registry_analysis_id: configRegistry?.analysis_id + ? redactAnalyticsText(configRegistry.analysis_id) + : undefined, + feedback_config_registry_recommendation_match_scope: + configRegistry?.recommendation_match_scope, + feedback_config_registry_recommendation_verification: + configRegistry?.recommendation_verification, + feedback_config_registry_evidence_sample_size: + configRegistry?.recommendation_evidence.sample_size, + feedback_config_registry_evidence_success_rate: + configRegistry?.recommendation_evidence.success_rate, + feedback_config_registry_evidence_last_verified_at: + configRegistry?.recommendation_evidence.last_verified_at, + feedback_config_registry_applied_config_key: configRegistry + ? configRegistryAppliedConfigKey(configRegistry) + : undefined, + feedback_config_registry_browser_stealth: + configRegistry?.applied_browser.stealth, + feedback_config_registry_browser_headless: + configRegistry?.applied_browser.headless, + feedback_config_registry_browser_gpu: configRegistry?.applied_browser.gpu, + feedback_config_registry_viewport_width: + configRegistry?.applied_browser.viewport.width, + feedback_config_registry_viewport_height: + configRegistry?.applied_browser.viewport.height, + feedback_config_registry_viewport_refresh_rate: + configRegistry?.applied_browser.viewport.refresh_rate, + feedback_config_registry_proxy_mode: configRegistry?.applied_proxy.mode, + feedback_config_registry_proxy_type: + configRegistry?.applied_proxy.mode === "managed" + ? configRegistry.applied_proxy.type + : undefined, + feedback_config_registry_proxy_country: + configRegistry?.applied_proxy.mode === "managed" + ? configRegistry.applied_proxy.country + : undefined, feedback_category: feedback.category, feedback_task_completed: feedback.task_completed, feedback_tools_used: feedback.tools_used?.map(redactAnalyticsText), diff --git a/src/lib/mcp/tools/feedback.test.ts b/src/lib/mcp/tools/feedback.test.ts index fb25afd..06d1125 100644 --- a/src/lib/mcp/tools/feedback.test.ts +++ b/src/lib/mcp/tools/feedback.test.ts @@ -73,6 +73,333 @@ describe("submit_feedback", () => { } }); + test("records structured bot-detection outcomes for config registry prioritization", async () => { + const captured: KernelFeedback[] = []; + const { client, close } = await connectTestMcp( + (server) => + registerFeedbackTool(server, (feedback) => { + captured.push(feedback); + }), + {}, + ); + + try { + const tools = await client.listTools(); + const tool = tools.tools.find( + ({ name }) => name === KERNEL_FEEDBACK_TOOL_NAME, + ); + expect(JSON.stringify(tool?.inputSchema)).toContain('"bot_detection"'); + expect(JSON.stringify(tool?.inputSchema)).toContain( + '"registrable_domain"', + ); + expect(JSON.stringify(tool?.inputSchema)).toContain('"datacenter"'); + expect(JSON.stringify(tool?.inputSchema)).toContain('"config_registry"'); + + const result = await client.callTool({ + name: KERNEL_FEEDBACK_TOOL_NAME, + arguments: { + context: + "Reporting a repeatable site block so the affected domain can be prioritized for a working browser configuration.", + summary: "Stealth sessions were consistently blocked", + feedback_type: "bot_detection", + sentiment: "negative", + task_completed: false, + tools_used: ["manage_browsers", "execute_playwright_code"], + bot_detection: { + registrable_domain: "Example.COM", + observed_outcome: "blocked", + suspected_vendor: "Akamai Bot Manager", + challenge_type: "access_denied", + stealth: "enabled", + proxy_type: "isp", + region: "us-east", + browser_version: "152.0.7977.42", + browser_image_version: "2026.09.14", + reproducibility: "consistent", + browser_session_id: "session_123", + }, + }, + }); + + expect(toolResultJSON(result)).toMatchObject({ + recorded: true, + feedback_type: "bot_detection", + sentiment: "negative", + }); + expect(captured).toEqual([ + { + summary: "Stealth sessions were consistently blocked", + feedback_type: "bot_detection", + sentiment: "negative", + task_completed: false, + tools_used: ["manage_browsers", "execute_playwright_code"], + bot_detection: { + registrable_domain: "example.com", + observed_outcome: "blocked", + suspected_vendor: "Akamai Bot Manager", + challenge_type: "access_denied", + stealth: "enabled", + proxy_type: "isp", + region: "us-east", + browser_version: "152.0.7977.42", + browser_image_version: "2026.09.14", + reproducibility: "consistent", + browser_session_id: "session_123", + }, + }, + ]); + } finally { + await close(); + } + }); + + test("records config-registry outcomes against the applied configuration", async () => { + const captured: KernelFeedback[] = []; + const { client, close } = await connectTestMcp( + (server) => + registerFeedbackTool(server, (feedback) => { + captured.push(feedback); + }), + {}, + ); + + try { + const result = await client.callTool({ + name: KERNEL_FEEDBACK_TOOL_NAME, + arguments: { + context: + "Reporting that an applied config registry recommendation still failed so its quality can be measured.", + summary: "The recommended configuration remained blocked", + feedback_type: "config_registry", + sentiment: "negative", + task_completed: false, + bot_detection: { + registrable_domain: "example.com", + observed_outcome: "blocked", + challenge_type: "access_denied", + reproducibility: "consistent", + browser_session_id: "session_456", + }, + config_registry: { + request_method: "resolve", + analysis_id: "analysis_123", + recommendation_match_scope: "exact", + recommendation_verification: "verified", + recommendation_evidence: { + sample_size: 5, + success_rate: 1, + last_verified_at: "2026-09-13T12:00:00Z", + }, + applied_browser: { + stealth: true, + headless: false, + gpu: false, + viewport: { + width: 1920, + height: 1080, + refresh_rate: 25, + }, + }, + applied_proxy: { + mode: "managed", + type: "residential", + country: "us", + }, + }, + }, + }); + + expect(toolResultJSON(result)).toMatchObject({ + recorded: true, + feedback_type: "config_registry", + sentiment: "negative", + }); + expect(captured).toEqual([ + { + summary: "The recommended configuration remained blocked", + feedback_type: "config_registry", + sentiment: "negative", + task_completed: false, + bot_detection: { + registrable_domain: "example.com", + observed_outcome: "blocked", + challenge_type: "access_denied", + reproducibility: "consistent", + browser_session_id: "session_456", + }, + config_registry: { + request_method: "resolve", + analysis_id: "analysis_123", + recommendation_match_scope: "exact", + recommendation_verification: "verified", + recommendation_evidence: { + sample_size: 5, + success_rate: 1, + last_verified_at: "2026-09-13T12:00:00Z", + }, + applied_browser: { + stealth: true, + headless: false, + gpu: false, + viewport: { + width: 1920, + height: 1080, + refresh_rate: 25, + }, + }, + applied_proxy: { + mode: "managed", + type: "residential", + country: "US", + }, + }, + }, + ]); + } finally { + await close(); + } + }); + + test("requires structured fields for site and config-registry feedback", async () => { + const captured: KernelFeedback[] = []; + const { client, close } = await connectTestMcp( + (server) => + registerFeedbackTool(server, (feedback) => { + captured.push(feedback); + }), + {}, + ); + + try { + const missingReport = await client.callTool({ + name: KERNEL_FEEDBACK_TOOL_NAME, + arguments: { + context: + "Reporting a site-specific browser block without the structured observation required for config registry prioritization.", + summary: "A site blocked the browser", + feedback_type: "bot_detection", + sentiment: "negative", + }, + }); + expect(missingReport.isError).toBe(true); + expect(captured).toEqual([]); + + const missingConfig = await client.callTool({ + name: KERNEL_FEEDBACK_TOOL_NAME, + arguments: { + context: + "Reporting a failed recommendation without the applied settings needed to attribute its outcome.", + summary: "The recommended configuration remained blocked", + feedback_type: "config_registry", + sentiment: "negative", + task_completed: false, + bot_detection: { + registrable_domain: "example.com", + observed_outcome: "blocked", + reproducibility: "single_observation", + }, + }, + }); + expect(missingConfig.isError).toBe(true); + expect(captured).toEqual([]); + + const missingSession = await client.callTool({ + name: KERNEL_FEEDBACK_TOOL_NAME, + arguments: { + context: + "Reporting an applied recommendation without a browser session correlation identifier.", + summary: "The recommended configuration remained blocked", + feedback_type: "config_registry", + sentiment: "negative", + task_completed: false, + bot_detection: { + registrable_domain: "example.com", + observed_outcome: "blocked", + reproducibility: "single_observation", + }, + config_registry: { + request_method: "lookup", + recommendation_evidence: { + sample_size: 3, + success_rate: 1, + last_verified_at: null, + }, + applied_browser: { + stealth: true, + headless: false, + gpu: false, + viewport: { width: 1920, height: 1080 }, + }, + applied_proxy: { mode: "direct" }, + }, + }, + }); + expect(missingSession.isError).toBe(true); + expect(captured).toEqual([]); + + const reportOnProductFeedback = await client.callTool({ + name: KERNEL_FEEDBACK_TOOL_NAME, + arguments: { + context: + "Reporting general browser feedback without routing it into the site-specific bot-detection prioritization queue.", + summary: "Browser startup was clear", + feedback_type: "product", + sentiment: "positive", + bot_detection: { + registrable_domain: "example.com", + observed_outcome: "passed", + reproducibility: "single_observation", + }, + }, + }); + expect(reportOnProductFeedback.isError).toBe(true); + expect(captured).toEqual([]); + } finally { + await close(); + } + }); + + test("rejects URLs, subdomains, and private hosts in domain reports", async () => { + const captured: KernelFeedback[] = []; + const { client, close } = await connectTestMcp( + (server) => + registerFeedbackTool(server, (feedback) => { + captured.push(feedback); + }), + {}, + ); + + try { + for (const registrableDomain of [ + "https://example.com/account?user=1", + "auth.example.com", + "service.local", + ]) { + const result = await client.callTool({ + name: KERNEL_FEEDBACK_TOOL_NAME, + arguments: { + context: + "Reporting a site outcome while ensuring sensitive host details cannot enter the prioritization event.", + summary: "A site blocked the browser", + feedback_type: "bot_detection", + sentiment: "negative", + task_completed: false, + bot_detection: { + registrable_domain: registrableDomain, + observed_outcome: "blocked", + reproducibility: "single_observation", + }, + }, + }); + + expect(result.isError).toBe(true); + } + expect(captured).toEqual([]); + } finally { + await close(); + } + }); + test("keeps analytics failures from failing the tool call", async () => { const { client, close } = await connectTestMcp( (server) => diff --git a/src/lib/mcp/tools/feedback.ts b/src/lib/mcp/tools/feedback.ts index 3b6e0fa..21516c7 100644 --- a/src/lib/mcp/tools/feedback.ts +++ b/src/lib/mcp/tools/feedback.ts @@ -1,10 +1,198 @@ import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { parse as parseDomain } from "tldts"; import { z } from "zod"; import { MCP_INTENT_ARGUMENT_DESCRIPTION } from "@/lib/mcp/analytics-context"; -import { jsonResponse } from "@/lib/mcp/responses"; +import { errorResponse, jsonResponse } from "@/lib/mcp/responses"; export const KERNEL_FEEDBACK_TOOL_NAME = "submit_feedback"; +const configRegistryAppliedBrowserSchema = z.object({ + stealth: z.boolean().describe("the applied browser stealth setting."), + headless: z.boolean().describe("the applied browser headless setting."), + gpu: z.boolean().describe("the applied browser GPU setting."), + viewport: z.object({ + width: z.number().int().positive().describe("the applied viewport width."), + height: z + .number() + .int() + .positive() + .describe("the applied viewport height."), + refresh_rate: z + .number() + .int() + .positive() + .optional() + .describe("the applied viewport refresh rate, if specified."), + }), +}); + +const configRegistryAppliedProxySchema = z.discriminatedUnion("mode", [ + z.object({ mode: z.literal("direct") }), + z.object({ + mode: z.literal("managed"), + type: z.enum(["datacenter", "isp", "residential", "mobile", "custom"]), + country: z + .string() + .trim() + .length(2) + .toUpperCase() + .optional() + .describe( + "the applied two-letter proxy country, if specified. do not include city, state, ZIP code, host, IP, or credentials.", + ), + }), +]); + +const configRegistryFeedbackSchema = z.object({ + request_method: z + .enum(["lookup", "resolve"]) + .describe( + "the config registry operation that returned the recommendation.", + ), + analysis_id: z + .string() + .trim() + .min(1) + .max(100) + .optional() + .describe( + "the config registry analysis ID when the recommendation came from a resolve or known analysis.", + ), + recommendation_match_scope: z + .enum(["exact", "host", "domain"]) + .optional() + .describe("the match_scope returned with the recommendation."), + recommendation_verification: z + .enum(["verified", "inferred"]) + .optional() + .describe("the verification value returned with the recommendation."), + recommendation_evidence: z.object({ + sample_size: z + .number() + .int() + .nonnegative() + .describe("the recommendation evidence sample_size value."), + success_rate: z + .number() + .min(0) + .max(1) + .describe("the recommendation evidence success_rate value."), + last_verified_at: z + .string() + .datetime({ offset: true }) + .nullable() + .optional() + .describe( + "the recommendation evidence last_verified_at timestamp, or null when it has never been verified.", + ), + }), + applied_browser: configRegistryAppliedBrowserSchema.describe( + "the returned browser settings, applied unchanged for the observed outcome.", + ), + applied_proxy: configRegistryAppliedProxySchema.describe( + "the returned proxy settings, applied unchanged for the observed outcome. never include proxy hosts, IPs, or credentials.", + ), +}); + +const botDetectionReportSchema = z.object({ + registrable_domain: z + .string() + .trim() + .toLowerCase() + .refine((value) => { + const parsed = parseDomain(value, { allowPrivateDomains: false }); + return parsed.isIcann && parsed.domain === value; + }, "must be a public registrable domain without a subdomain or URL components") + .describe( + 'the public registrable domain where the result was observed (e.g. "example.com"). include no protocol, path, query, fragment, port, subdomain, account-specific host, or private/internal hostname. public registrable domains are allowed only in this field so reports can prioritize config registry coverage.', + ), + observed_outcome: z + .enum(["passed", "challenged", "blocked", "degraded"]) + .describe( + 'what the site did: "passed" = the intended flow remained usable, "challenged" = an anti-bot step appeared but the flow could continue, "blocked" = the flow could not continue, and "degraded" = content or functionality was restricted.', + ), + suspected_vendor: z + .string() + .trim() + .min(1) + .max(100) + .optional() + .describe( + 'the suspected bot-detection vendor or product, when supported by evidence (e.g. "Akamai Bot Manager"). omit rather than guess.', + ), + challenge_type: z + .enum([ + "captcha", + "javascript_challenge", + "access_denied", + "rate_limited", + "login_block", + "fingerprint_block", + "content_restricted", + "other", + "unknown", + ]) + .optional() + .describe( + "the dominant challenge or block observed. use unknown when the flow failed without a recognizable challenge surface.", + ), + stealth: z + .enum(["enabled", "disabled", "unknown"]) + .optional() + .describe("whether KERNEL stealth mode was enabled for the observation."), + proxy_type: z + .enum([ + "none", + "datacenter", + "isp", + "residential", + "mobile", + "custom", + "unknown", + ]) + .optional() + .describe( + "the egress type used for the observation. never include a proxy URL, credential, provider account, or IP address.", + ), + region: z + .string() + .trim() + .min(1) + .max(50) + .optional() + .describe( + 'the KERNEL browser region used for the observation (e.g. "us-east"). do not include a street address, postal code, or user location.', + ), + browser_version: z + .string() + .trim() + .min(1) + .max(100) + .optional() + .describe("the browser version reported by the KERNEL session, if known."), + browser_image_version: z + .string() + .trim() + .min(1) + .max(100) + .optional() + .describe("the KERNEL browser image version or release tag, if known."), + reproducibility: z + .enum(["single_observation", "intermittent", "consistent", "unknown"]) + .describe( + 'how repeatable the outcome was: "single_observation" = tried once, "intermittent" = outcomes varied, "consistent" = repeated attempts matched, and "unknown" = repetition was not observable.', + ), + browser_session_id: z + .string() + .trim() + .min(1) + .max(100) + .optional() + .describe( + "the KERNEL browser session ID for internal correlation, if available. never substitute a CDP or live-view URL.", + ), +}); + const feedbackFields = { context: z.string().describe(MCP_INTENT_ARGUMENT_DESCRIPTION), summary: z @@ -16,9 +204,16 @@ const feedbackFields = { 'a one-sentence headline capturing the feedback (e.g. "browser creation timed out without recovery guidance", "manage_browsers returned exactly the context needed", or "the proxy docs need a residential example").', ), feedback_type: z - .enum(["product", "mcp", "docs", "other"]) + .enum([ + "product", + "bot_detection", + "config_registry", + "mcp", + "docs", + "other", + ]) .describe( - 'what this feedback is about. "product" = any KERNEL product or feature, such as browsers, apps, profiles, proxies, browser pools, replays, telemetry, managed auth, credentials, extensions, projects, or api keys. "mcp" = this mcp server itself, including a tool, input schema, response format, error, or its instructions. "docs" = KERNEL documentation. "other" = anything that does not fit the other types.', + 'what this feedback is about. "product" = any KERNEL product or feature, such as browsers, apps, profiles, proxies, browser pools, replays, telemetry, managed auth, credentials, extensions, projects, or api keys. "bot_detection" = a site-specific pass, challenge, block, or degraded result not produced by an unchanged config registry recommendation; include `bot_detection`. "config_registry" = the observed result after requesting and applying a config registry recommendation unchanged; include both `bot_detection` and `config_registry` so the outcome is attributed to the settings used. "mcp" = this mcp server itself, including a tool, input schema, response format, error, or its instructions. "docs" = KERNEL documentation. "other" = anything that does not fit the other types.', ), sentiment: z .enum(["positive", "neutral", "negative", "mixed"]) @@ -32,7 +227,17 @@ const feedbackFields = { .max(100) .optional() .describe( - 'the KERNEL product or area this is about, in free text (e.g. "browsers", "apps", "managed auth", "browser pools", "proxies", or "telemetry"). most useful for product feedback; for mcp feedback put the tool name in `details` or `friction_points` instead.', + 'the KERNEL product or area this is about, in free text (e.g. "browsers", "apps", "managed auth", "browser pools", "proxies", or "telemetry"). most useful for product feedback; use `feedback_type: "bot_detection"` instead of putting bot detection here, and for mcp feedback put the tool name in `details` or `friction_points`.', + ), + bot_detection: botDetectionReportSchema + .optional() + .describe( + 'the structured site outcome. required when `feedback_type` is "bot_detection" or "config_registry" and rejected for every other feedback type.', + ), + config_registry: configRegistryFeedbackSchema + .optional() + .describe( + 'the config registry recommendation, evidence snapshot, and exact settings used for the observed site outcome. required when `feedback_type` is "config_registry" and rejected for every other feedback type. `bot_detection.browser_session_id` is also required so KERNEL can investigate without collecting sensitive page details here. use this only after applying the recommendation unchanged, whether it passes or fails; if the settings were changed first, report `bot_detection` instead.', ), category: z .enum([ @@ -54,7 +259,7 @@ const feedbackFields = { .boolean() .optional() .describe( - 'whether the user\'s task was completed. be honest: `false` is useful signal. most relevant when `feedback_type` is "mcp".', + "whether the user's task was completed. be honest: `false` is useful signal. required for bot-detection and config-registry feedback, and also useful for mcp feedback.", ), tools_used: z .array(z.string().trim().min(1).max(100)) @@ -111,7 +316,7 @@ export type KernelFeedbackCapture = ( ) => void | Promise; const TOOL_DESCRIPTION = - "send feedback about anything KERNEL to the KERNEL team. set `feedback_type` to route it: `product` for any KERNEL product or feature, `mcp` for this mcp server, `docs` for KERNEL documentation, or `other`. all sentiments are welcome through `sentiment`: praise and feature requests are useful, not just problems. use this for confusing or broken experiences, papercuts, missing capabilities, unhelpful errors, feature requests, and things that worked especially well. keep `summary` to one sentence and make the detail fields concise and actionable, quoting the product surface, tool name, parameter, or error text when possible. include a concrete `suggested_improvement` when one is clear. never include credentials, tokens, api keys, urls, browser or page content, customer or account names, or personal data. the user can also ask to send feedback directly. submitting feedback is a side report to KERNEL, not a reason to stop: continue and finish the user's task with the other available tools."; + "send feedback about anything KERNEL to the KERNEL team. set `feedback_type` to route it: `product` for any KERNEL product or feature, `bot_detection` for a site-specific pass, challenge, block, or degraded result not tied to an unchanged registry recommendation, `config_registry` for the result after requesting and applying a config registry recommendation unchanged, `mcp` for this mcp server, `docs` for KERNEL documentation, or `other`. for bot detection, fill `bot_detection` with the public registrable domain, outcome, and reproducibility. when a config registry recommendation was requested and applied unchanged, choose `config_registry`; include the request metadata, recommendation evidence, exact browser and proxy settings used, and `bot_detection.browser_session_id`. report both passes and failures so recommendation quality can be measured. if any recommended setting was changed before testing, use `bot_detection` instead so the result is not attributed to the original recommendation. all sentiments are welcome through `sentiment`: praise and feature requests are useful, not just problems. use this for confusing or broken experiences, papercuts, missing capabilities, unhelpful errors, feature requests, and things that worked especially well. keep `summary` to one sentence and make the detail fields concise and actionable, quoting the product surface, tool name, parameter, or error text when possible. include a concrete `suggested_improvement` when one is clear. never include credentials, tokens, api keys, urls, paths, browser or page content, customer or account names, private hosts, IP addresses, or personal data. a public registrable domain is allowed only in `bot_detection.registrable_domain`; never include a subdomain or account-specific host. the user can also ask to send feedback directly. submitting feedback is a side report to KERNEL, not a reason to stop: continue and finish the user's task with the other available tools."; const RESPONSE_MESSAGES = { recorded: @@ -143,6 +348,38 @@ export function registerFeedbackTool( }, }, async ({ context: _context, ...feedback }, extra) => { + const hasSiteOutcome = + feedback.feedback_type === "bot_detection" || + feedback.feedback_type === "config_registry"; + if (hasSiteOutcome) { + if (!feedback.bot_detection || feedback.task_completed === undefined) { + return errorResponse( + "bot_detection and task_completed are required for bot-detection and config-registry feedback.", + ); + } + } else if (feedback.bot_detection) { + return errorResponse( + "bot_detection is only accepted for bot-detection and config-registry feedback.", + ); + } + + if (feedback.feedback_type === "config_registry") { + if (!feedback.config_registry) { + return errorResponse( + "config_registry is required when feedback_type is config_registry.", + ); + } + if (!feedback.bot_detection?.browser_session_id) { + return errorResponse( + "bot_detection.browser_session_id is required when feedback_type is config_registry.", + ); + } + } else if (feedback.config_registry) { + return errorResponse( + "config_registry is only accepted when feedback_type is config_registry.", + ); + } + let status: FeedbackCaptureStatus = "unavailable"; if (capture) { try {