diff --git a/.github/CODENOTIFY b/.github/CODENOTIFY index 3adf8957acda7..61b0a1756ac5d 100644 --- a/.github/CODENOTIFY +++ b/.github/CODENOTIFY @@ -83,6 +83,7 @@ extensions/vscode-api-tests/src/singlefolder-tests/browser*.test.ts @kycutler @j # Testing test/mcp/** @TylerLeonhardt +test/scenario/** @bryanchen-d test/sanity/** @dmitrivMS # Agents Workbench diff --git a/.github/skills/validate-ui-scenario/SKILL.md b/.github/skills/validate-ui-scenario/SKILL.md index 0288ef46749b5..16b002502f7d3 100644 --- a/.github/skills/validate-ui-scenario/SKILL.md +++ b/.github/skills/validate-ui-scenario/SKILL.md @@ -11,7 +11,7 @@ Use this to reproduce a reported bug, to show that a fix works, or to attach a r test-plan item. For deterministic regression coverage that runs on every build, write a smoke test instead (see the `smoke-tests` skill) — this skill is for one-off, issue-derived validation. -A scenario is a small JavaScript file run by `test/mcp/out/runScenario.js`. Nothing else has to be +A scenario is a small JavaScript file run by `test/scenario/out/runScenario.js`. Nothing else has to be configured: the runner launches VS Code, records video and a trace, captures a screenshot at every step boundary, writes the report, and captions the recording with each step and its result. @@ -19,7 +19,7 @@ step boundary, writes the report, and captions the recording with each step and ```bash npm install # once -npm --prefix test/mcp run compile # after any change under test/mcp +npm --prefix test/scenario run compile # after any change under test/scenario ``` Add `ffmpeg` and `ffprobe` to `PATH` to get the caption band on the video. Without them the run still @@ -130,7 +130,7 @@ Each step receives a `context` with `app`, `workbench`, `code`, `page`, and `ski ## Run it ```bash -node test/mcp/out/runScenario.js --build "" +node test/scenario/out/runScenario.js --build "" ``` Exit code `0` means every step passed, `1` means the run failed or was aborted, `2` a usage error. @@ -149,7 +149,7 @@ Evidence is written to `.build/vscode-playwright-mcp/evidence//`: The caption band is added **above** the recorded frame rather than drawn over it, so no recorded pixel is hidden and the recording keeps its original length. Each caption carries the step number and id, its status, the step title, and the validation detail the step reported. Re-render after -editing a manifest with `node test/mcp/out/renderEvidenceChapters.js `. +editing a manifest with `node test/scenario/out/renderEvidenceChapters.js `. ## What makes evidence trustworthy diff --git a/build/npm/dirs.ts b/build/npm/dirs.ts index 289a469754a5b..7bc33635f8681 100644 --- a/build/npm/dirs.ts +++ b/build/npm/dirs.ts @@ -59,6 +59,7 @@ export const dirs = [ 'test/integration/browser', 'test/monaco', 'test/smoke', + 'test/scenario', 'test/mcp', '.vscode/extensions/vscode-selfhost-import-aid', '.vscode/extensions/vscode-selfhost-test-provider', diff --git a/eslint.config.js b/eslint.config.js index 432f521df94c0..b9221655462a0 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -2329,10 +2329,22 @@ export default defineConfig( '*' // node modules ] }, + { + 'target': 'test/scenario/**', + 'restrictions': [ + 'test/automation', + 'test/scenario/**', + '@vscode/*', + '@parcel/*', + '@playwright/*', + '*' // node modules + ] + }, { 'target': 'test/mcp/**', 'restrictions': [ 'test/automation', + 'test/scenario', 'test/mcp/**', '@vscode/*', '@parcel/*', diff --git a/test/mcp/package.json b/test/mcp/package.json index bbddd2ed4fcf0..9938aa648a077 100644 --- a/test/mcp/package.json +++ b/test/mcp/package.json @@ -5,10 +5,10 @@ "main": "./out/main.js", "private": true, "scripts": { - "compile": "cd ../automation && npm run compile && cd ../mcp && node ../../node_modules/typescript/bin/tsc6", - "watch-automation": "cd ../automation && npm run watch", + "compile": "cd ../scenario && npm run compile && cd ../mcp && node ../../node_modules/typescript/bin/tsc6", + "watch-scenario": "cd ../scenario && npm run watch", "watch-mcp": "node ../../node_modules/typescript/bin/tsc6 --watch --preserveWatchOutput", - "watch": "npm-run-all2 -lp watch-automation watch-mcp", + "watch": "npm-run-all2 -lp watch-scenario watch-mcp", "start-stdio": "echo 'Starting vscode-automation-mcp... For customization and troubleshooting, see ./test/mcp/README.md' && npm ci && npm run -s compile && node ./out/stdio.js" }, "dependencies": { diff --git a/test/mcp/src/automation.ts b/test/mcp/src/automation.ts index ef2345e707f4c..97a070b98d321 100644 --- a/test/mcp/src/automation.ts +++ b/test/mcp/src/automation.ts @@ -4,11 +4,11 @@ *--------------------------------------------------------------------------------------------*/ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { ApplicationService } from './application'; +import { ApplicationService, EvidenceService } from '../../scenario'; import { applyAllTools } from './automationTools/index.js'; import type { Server } from '@modelcontextprotocol/sdk/server/index.js'; import { z } from 'zod'; -import { applyEvidenceStartTool, applyEvidenceTools, EvidenceService } from './evidence.js'; +import { applyEvidenceStartTool, applyEvidenceTools } from './evidenceTools.js'; export async function getServer(appService: ApplicationService): Promise { const server = new McpServer({ diff --git a/test/mcp/src/automationTools/activityBar.ts b/test/mcp/src/automationTools/activityBar.ts index 8b06471dda933..e32553eecce26 100644 --- a/test/mcp/src/automationTools/activityBar.ts +++ b/test/mcp/src/automationTools/activityBar.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { ApplicationService } from '../application'; +import { ApplicationService } from '../../../scenario'; /** * Activity Bar Tools diff --git a/test/mcp/src/automationTools/chat.ts b/test/mcp/src/automationTools/chat.ts index 5157bf19ddd88..aeecaf93a3d80 100644 --- a/test/mcp/src/automationTools/chat.ts +++ b/test/mcp/src/automationTools/chat.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { ApplicationService } from '../application'; +import { ApplicationService } from '../../../scenario'; import { z } from 'zod'; /** diff --git a/test/mcp/src/automationTools/core.ts b/test/mcp/src/automationTools/core.ts index d81bc9570c457..dfc523ab2547b 100644 --- a/test/mcp/src/automationTools/core.ts +++ b/test/mcp/src/automationTools/core.ts @@ -5,7 +5,7 @@ import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js'; import { z } from 'zod'; -import { ApplicationService, assertNoProfileOverrides } from '../application'; +import { ApplicationService, assertNoProfileOverrides } from '../../../scenario'; /** * Core Application Management Tools diff --git a/test/mcp/src/automationTools/debug.ts b/test/mcp/src/automationTools/debug.ts index b9dfb2951f3b5..268f209d515e2 100644 --- a/test/mcp/src/automationTools/debug.ts +++ b/test/mcp/src/automationTools/debug.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { ApplicationService } from '../application'; +import { ApplicationService } from '../../../scenario'; import { z } from 'zod'; /** diff --git a/test/mcp/src/automationTools/editor.ts b/test/mcp/src/automationTools/editor.ts index 0af80f1865012..67436bbfd50a5 100644 --- a/test/mcp/src/automationTools/editor.ts +++ b/test/mcp/src/automationTools/editor.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { ApplicationService } from '../application'; +import { ApplicationService } from '../../../scenario'; import { z } from 'zod'; /** diff --git a/test/mcp/src/automationTools/explorer.ts b/test/mcp/src/automationTools/explorer.ts index 777d7e3dfcb71..7f46d8107e62a 100644 --- a/test/mcp/src/automationTools/explorer.ts +++ b/test/mcp/src/automationTools/explorer.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { ApplicationService } from '../application'; +import { ApplicationService } from '../../../scenario'; /** * Explorer and File Management Tools diff --git a/test/mcp/src/automationTools/extensions.ts b/test/mcp/src/automationTools/extensions.ts index e379a7d90e4ee..40a54a2f8f2f4 100644 --- a/test/mcp/src/automationTools/extensions.ts +++ b/test/mcp/src/automationTools/extensions.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { ApplicationService } from '../application'; +import { ApplicationService } from '../../../scenario'; import { z } from 'zod'; /** diff --git a/test/mcp/src/automationTools/index.ts b/test/mcp/src/automationTools/index.ts index e5f595fb47fbd..0cd707451e649 100644 --- a/test/mcp/src/automationTools/index.ts +++ b/test/mcp/src/automationTools/index.ts @@ -26,7 +26,7 @@ import { applyTaskTools } from './task.js'; import { applyProfilerTools } from './profiler.js'; import { applyChatTools } from './chat.js'; import { applyWindowTools } from './windows.js'; -import { ApplicationService } from '../application'; +import { ApplicationService } from '../../../scenario'; /** * Apply all VS Code automation tools to the MCP server diff --git a/test/mcp/src/automationTools/keybindings.ts b/test/mcp/src/automationTools/keybindings.ts index 28908119e0d68..8fec6893862e1 100644 --- a/test/mcp/src/automationTools/keybindings.ts +++ b/test/mcp/src/automationTools/keybindings.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { ApplicationService } from '../application'; +import { ApplicationService } from '../../../scenario'; /** * Keybindings Editor Tools diff --git a/test/mcp/src/automationTools/localization.ts b/test/mcp/src/automationTools/localization.ts index bff17b43e7042..0b468655eb661 100644 --- a/test/mcp/src/automationTools/localization.ts +++ b/test/mcp/src/automationTools/localization.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { ApplicationService } from '../application'; +import { ApplicationService } from '../../../scenario'; /** * Localization Tools diff --git a/test/mcp/src/automationTools/notebook.ts b/test/mcp/src/automationTools/notebook.ts index 68ce82ca6eba5..a157033ac0d9b 100644 --- a/test/mcp/src/automationTools/notebook.ts +++ b/test/mcp/src/automationTools/notebook.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { ApplicationService } from '../application'; +import { ApplicationService } from '../../../scenario'; import { z } from 'zod'; /** diff --git a/test/mcp/src/automationTools/problems.ts b/test/mcp/src/automationTools/problems.ts index 4ff55b05270f8..88e82d184cabf 100644 --- a/test/mcp/src/automationTools/problems.ts +++ b/test/mcp/src/automationTools/problems.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { ApplicationService } from '../application'; +import { ApplicationService } from '../../../scenario'; import { z } from 'zod'; /** diff --git a/test/mcp/src/automationTools/profiler.ts b/test/mcp/src/automationTools/profiler.ts index c47f8bb00cbbd..5852a5438f808 100644 --- a/test/mcp/src/automationTools/profiler.ts +++ b/test/mcp/src/automationTools/profiler.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { ApplicationService } from '../application'; +import { ApplicationService } from '../../../scenario'; /** * Profiler Tools diff --git a/test/mcp/src/automationTools/quickAccess.ts b/test/mcp/src/automationTools/quickAccess.ts index b8f9c500c01ee..1b71ade39ba94 100644 --- a/test/mcp/src/automationTools/quickAccess.ts +++ b/test/mcp/src/automationTools/quickAccess.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { ApplicationService } from '../application'; +import { ApplicationService } from '../../../scenario'; import { z } from 'zod'; /** diff --git a/test/mcp/src/automationTools/scm.ts b/test/mcp/src/automationTools/scm.ts index 55feec02b6902..ee6516cb5e237 100644 --- a/test/mcp/src/automationTools/scm.ts +++ b/test/mcp/src/automationTools/scm.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { ApplicationService } from '../application'; +import { ApplicationService } from '../../../scenario'; import { z } from 'zod'; /** diff --git a/test/mcp/src/automationTools/search.ts b/test/mcp/src/automationTools/search.ts index 040ce94b68b5f..810dbba20af1e 100644 --- a/test/mcp/src/automationTools/search.ts +++ b/test/mcp/src/automationTools/search.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { ApplicationService } from '../application'; +import { ApplicationService } from '../../../scenario'; import { z } from 'zod'; /** diff --git a/test/mcp/src/automationTools/settings.ts b/test/mcp/src/automationTools/settings.ts index 46f91fe8fbf7a..89d3cf8a5b39d 100644 --- a/test/mcp/src/automationTools/settings.ts +++ b/test/mcp/src/automationTools/settings.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { ApplicationService } from '../application'; +import { ApplicationService } from '../../../scenario'; import { z } from 'zod'; /** diff --git a/test/mcp/src/automationTools/statusbar.ts b/test/mcp/src/automationTools/statusbar.ts index 2944d2c9faffc..e5c06f907924d 100644 --- a/test/mcp/src/automationTools/statusbar.ts +++ b/test/mcp/src/automationTools/statusbar.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { ApplicationService } from '../application'; +import { ApplicationService } from '../../../scenario'; /** * Status Bar Tools diff --git a/test/mcp/src/automationTools/task.ts b/test/mcp/src/automationTools/task.ts index 270d596fc1855..82a871ae841ba 100644 --- a/test/mcp/src/automationTools/task.ts +++ b/test/mcp/src/automationTools/task.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { ApplicationService } from '../application'; +import { ApplicationService } from '../../../scenario'; import { z } from 'zod'; /** diff --git a/test/mcp/src/automationTools/terminal.ts b/test/mcp/src/automationTools/terminal.ts index eaab27effdfc3..fd72d2a7ad251 100644 --- a/test/mcp/src/automationTools/terminal.ts +++ b/test/mcp/src/automationTools/terminal.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { ApplicationService } from '../application'; +import { ApplicationService } from '../../../scenario'; import { z } from 'zod'; /** diff --git a/test/mcp/src/automationTools/windows.ts b/test/mcp/src/automationTools/windows.ts index 132cb2841c196..863027529dc74 100644 --- a/test/mcp/src/automationTools/windows.ts +++ b/test/mcp/src/automationTools/windows.ts @@ -5,7 +5,7 @@ import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js'; import { z } from 'zod'; -import { ApplicationService } from '../application'; +import { ApplicationService } from '../../../scenario'; /** * Create a standardized text response for window tools diff --git a/test/mcp/src/evidenceTools.ts b/test/mcp/src/evidenceTools.ts new file mode 100644 index 0000000000000..f5c2caac62228 --- /dev/null +++ b/test/mcp/src/evidenceTools.ts @@ -0,0 +1,85 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; +import { EvidenceService, JSONValue } from '../../scenario'; + +const jsonValueSchema: z.ZodType = z.lazy(() => z.union([ + z.string(), + z.number(), + z.boolean(), + z.null(), + z.array(jsonValueSchema), + z.record(z.string(), jsonValueSchema) +])); + +function isHttpUrl(value: string): boolean { + try { + return ['http:', 'https:'].includes(new URL(value).protocol); + } catch { + return false; + } +} + +export function applyEvidenceStartTool(server: McpServer, evidenceService: EvidenceService): RegisteredTool { + return server.tool( + 'vscode_automation_evidence_start', + 'Start VS Code with video and trace recording for a UI validation scenario', + { + scenarioId: z.string().describe('Stable scenario identifier'), + title: z.string().describe('Human-readable scenario title'), + source: z.string().url().refine(isHttpUrl, 'Source must use HTTP or HTTPS').optional().describe('Source test-plan issue URL'), + scenarioPath: z.string().optional().describe('Path to the Markdown scenario definition'), + workspacePath: z.string().optional().describe('Workspace or folder to open'), + userSettings: z.record(z.string(), jsonValueSchema).optional().describe('User settings to seed before VS Code starts'), + extraArgs: z.array(z.string()).optional().describe('Additional VS Code command-line arguments') + }, + async ({ scenarioId, title, source, scenarioPath, workspacePath, userSettings, extraArgs }) => { + const runPath = await evidenceService.start(scenarioId, title, source, scenarioPath, workspacePath, userSettings, extraArgs); + return { + content: [{ type: 'text' as const, text: `Evidence capture started: ${runPath}` }] + }; + } + ); +} + +export function applyEvidenceTools(server: McpServer, evidenceService: EvidenceService): RegisteredTool[] { + return [ + server.tool( + 'vscode_automation_evidence_step', + 'Mark a scenario step in the video and save a screenshot of the current VS Code window', + { + id: z.string().describe('Stable step identifier from the scenario'), + title: z.string().describe('Human-readable step title'), + status: z.enum(['started', 'passed', 'failed', 'skipped']).describe('Step lifecycle status'), + details: z.string().optional().describe('Validation result or failure details') + }, + async ({ id, title, status, details }) => { + const result = await evidenceService.step(id, title, status, details); + return { + content: [ + { type: 'text' as const, text: `Evidence saved: ${result.screenshotPath}` }, + { type: 'image' as const, data: result.screenshot.toString('base64'), mimeType: 'image/png' } + ] + }; + } + ), + server.tool( + 'vscode_automation_evidence_finish', + 'Finish a UI validation scenario, stop VS Code, and write the evidence report', + { + outcome: z.enum(['passed', 'failed', 'aborted']).describe('Overall scenario outcome'), + notes: z.string().optional().describe('Run summary or blocking condition') + }, + async ({ outcome, notes }) => { + const reportPath = await evidenceService.finish(outcome, notes); + return { + content: [{ type: 'text' as const, text: `Evidence report written: ${reportPath}` }] + }; + } + ) + ]; +} diff --git a/test/mcp/src/stdio.ts b/test/mcp/src/stdio.ts index ea8193baa2f33..e6c9a4bc27cb2 100644 --- a/test/mcp/src/stdio.ts +++ b/test/mcp/src/stdio.ts @@ -4,8 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; import { getServer } from './automation'; -import { ApplicationService } from './application'; -import { opts } from './options'; +import { ApplicationService, opts } from '../../scenario'; const transport: StdioServerTransport = new StdioServerTransport(); (async () => { diff --git a/test/scenario/.gitignore b/test/scenario/.gitignore new file mode 100644 index 0000000000000..e7d563c46ad50 --- /dev/null +++ b/test/scenario/.gitignore @@ -0,0 +1,5 @@ +.DS_Store +npm-debug.log +Thumbs.db +node_modules/ +out/ diff --git a/test/scenario/package-lock.json b/test/scenario/package-lock.json new file mode 100644 index 0000000000000..afafe0902c100 --- /dev/null +++ b/test/scenario/package-lock.json @@ -0,0 +1,247 @@ +{ + "name": "vscode-scenario-runner", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "vscode-scenario-runner", + "version": "0.1.0", + "license": "MIT", + "dependencies": { + "minimist": "^1.2.8" + }, + "devDependencies": { + "npm-run-all2": "^8.0.4" + } + }, + "node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/cross-spawn/node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/cross-spawn/node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/isexe": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-3.1.5.tgz", + "integrity": "sha512-6B3tLtFqtQS4ekarvLVMZ+X+VlvQekbe4taUkf/rhVO3d/h0M2rfARm/pXLcPEsjjMsFgrFgSrhQIxcSVrBz8w==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/json-parse-even-better-errors": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-4.0.0.tgz", + "integrity": "sha512-lR4MXjGNgkJc7tkQ97kb2nuEMnNCyU//XYVH0MKTGcXEiSudQ5MKGKen3C5QubYy0vmq+JGitUg92uuywGEwIA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/memorystream": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/memorystream/-/memorystream-0.3.1.tgz", + "integrity": "sha512-S3UwM3yj5mtUSEfP41UZmt/0SCoVYUcU1rkXv+BQ5Ig8ndL4sPoJNBUJERafdPb5jjHJGuMgytgKvKIf58XNBw==", + "dev": true, + "engines": { + "node": ">= 0.10.0" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/npm-normalize-package-bin": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-4.0.0.tgz", + "integrity": "sha512-TZKxPvItzai9kN9H/TkmCtx/ZN/hvr3vUycjlfmH0ootY9yFBzNOpiXAdIn1Iteqsvk4lQn6B5PTrt+n6h8k/w==", + "dev": true, + "license": "ISC", + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/npm-run-all2": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/npm-run-all2/-/npm-run-all2-8.0.4.tgz", + "integrity": "sha512-wdbB5My48XKp2ZfJUlhnLVihzeuA1hgBnqB2J9ahV77wLS+/YAJAlN8I+X3DIFIPZ3m5L7nplmlbhNiFDmXRDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.1", + "cross-spawn": "^7.0.6", + "memorystream": "^0.3.1", + "picomatch": "^4.0.2", + "pidtree": "^0.6.0", + "read-package-json-fast": "^4.0.0", + "shell-quote": "^1.7.3", + "which": "^5.0.0" + }, + "bin": { + "npm-run-all": "bin/npm-run-all/index.js", + "npm-run-all2": "bin/npm-run-all/index.js", + "run-p": "bin/run-p/index.js", + "run-s": "bin/run-s/index.js" + }, + "engines": { + "node": "^20.5.0 || >=22.0.0", + "npm": ">= 10" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/picomatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pidtree": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/pidtree/-/pidtree-0.6.1.tgz", + "integrity": "sha512-e0F9AOF1JMrCfBsyJOwU9lNvQ0WtXTq0j/4jk0BQ5JSI9VAybPXmDpPRw/2FQ3e5d3ZFN1mLh7jW99m/jjaptw==", + "dev": true, + "license": "MIT", + "bin": { + "pidtree": "bin/pidtree.js" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/read-package-json-fast": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/read-package-json-fast/-/read-package-json-fast-4.0.0.tgz", + "integrity": "sha512-qpt8EwugBWDw2cgE2W+/3oxC+KTez2uSVR8JU9Q36TXPAGCaozfQUs59v4j4GFpWTaw0i6hAZSvOmu1J0uOEUg==", + "dev": true, + "license": "ISC", + "dependencies": { + "json-parse-even-better-errors": "^4.0.0", + "npm-normalize-package-bin": "^4.0.0" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/shell-quote": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.10.0.tgz", + "integrity": "sha512-w1aiOKwKuRgtwAReIIj89puqg+I7GvX4IbLrvmhXbzQsj1+Zwi4VO3+fa6ZF91TWSjIxoEkKnMeHcLEODK5ZXA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/which": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/which/-/which-5.0.0.tgz", + "integrity": "sha512-JEdGzHwwkrbWoGOlIHqQ5gtprKGOenpDHpxE9zVR1bWbOtYRyPPHMe9FaP6x61CmNaTThSkb0DAJte5jD+DmzQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^3.1.1" + }, + "bin": { + "node-which": "bin/which.js" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + } + } +} diff --git a/test/scenario/package.json b/test/scenario/package.json new file mode 100644 index 0000000000000..d6f2c905bb9ab --- /dev/null +++ b/test/scenario/package.json @@ -0,0 +1,20 @@ +{ + "name": "vscode-scenario-runner", + "version": "0.1.0", + "description": "Runs UI validation scenarios against VS Code and records evidence", + "license": "MIT", + "main": "./out/index.js", + "private": true, + "scripts": { + "compile": "cd ../automation && npm run compile && cd ../scenario && node ../../node_modules/typescript/bin/tsc6", + "watch-automation": "cd ../automation && npm run watch", + "watch-scenario": "node ../../node_modules/typescript/bin/tsc6 --watch --preserveWatchOutput", + "watch": "npm-run-all2 -lp watch-automation watch-scenario" + }, + "dependencies": { + "minimist": "^1.2.8" + }, + "devDependencies": { + "npm-run-all2": "^8.0.4" + } +} diff --git a/test/mcp/src/application.ts b/test/scenario/src/application.ts similarity index 99% rename from test/mcp/src/application.ts rename to test/scenario/src/application.ts index 9bfb2e8241c16..dfbe0e00d72de 100644 --- a/test/mcp/src/application.ts +++ b/test/scenario/src/application.ts @@ -238,7 +238,7 @@ async function removeProfileData(userDataPath: string | undefined): Promise | undefined, web: boolean): Promise { if (!userDataDir) { - throw new Error('Cannot pre-seed the MCP test profile without a user data directory.'); + throw new Error('Cannot pre-seed the isolated test profile without a user data directory.'); } const userDir = path.join(userDataDir, ...(web ? ['data', 'User'] : ['User'])); diff --git a/test/mcp/src/evidence.ts b/test/scenario/src/evidence.ts similarity index 78% rename from test/mcp/src/evidence.ts rename to test/scenario/src/evidence.ts index aa124aa0e1f02..d2a3f3c029d68 100644 --- a/test/mcp/src/evidence.ts +++ b/test/scenario/src/evidence.ts @@ -3,11 +3,9 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js'; import type { Page } from '@playwright/test'; import * as fs from 'fs'; import * as path from 'path'; -import { z } from 'zod'; import { ApplicationService, getProductVersion, JSONValue } from './application'; const rootPath = path.join(__dirname, '..', '..', '..'); @@ -16,16 +14,8 @@ const evidenceRootPath = path.join(artifactRootPath, 'evidence'); const logsRootPath = path.join(artifactRootPath, 'logs'); const qualityNames = ['Dev', 'Insiders', 'Stable', 'Exploration', 'OSS']; -type StepStatus = 'started' | 'passed' | 'failed' | 'skipped'; -type RunOutcome = 'passed' | 'failed' | 'aborted'; -const jsonValueSchema: z.ZodType = z.lazy(() => z.union([ - z.string(), - z.number(), - z.boolean(), - z.null(), - z.array(jsonValueSchema), - z.record(z.string(), jsonValueSchema) -])); +export type StepStatus = 'started' | 'passed' | 'failed' | 'skipped'; +export type RunOutcome = 'passed' | 'failed' | 'aborted'; interface EvidenceCapture { status: StepStatus; @@ -162,7 +152,6 @@ export class EvidenceService { app.code.driver.browserContext.on('page', run.pageListener); await app.startTracing(); - await this.showOverlay('Scenario', title, 'started'); await wait(500); await this.capture('00-scenario-started.png'); this.writeManifest(); @@ -233,7 +222,6 @@ export class EvidenceService { let screenshot: Buffer | undefined; for (let attempt = 0; attempt < 2; attempt++) { try { - await this.showOverlay(id, title, status); await wait(status === 'started' ? 500 : 250); const sequence = String(run.steps.indexOf(step) + 1).padStart(2, '0'); screenshotName = `${sequence}-${sanitizePathSegment(id)}-${status}.png`; @@ -307,7 +295,6 @@ export class EvidenceService { if (app) { for (let attempt = 0; attempt < 2; attempt++) { try { - await this.showOverlay('Result', run.title, outcome); await wait(500); await this.capture(`99-result-${outcome}.png`); break; @@ -389,38 +376,11 @@ export class EvidenceService { private requireRun(): EvidenceRun { if (!this.currentRun) { - throw new Error('No evidence run is active. Start one with vscode_automation_evidence_start.'); + throw new Error('No evidence run is active. Start one before recording steps.'); } return this.currentRun; } - private async showOverlay(id: string, title: string, status: string): Promise { - if (process.env.VSCODE_EVIDENCE_CLEAN_CAPTURE === '1') { - // The overlay is appended to the DOM of the product under test, so it can - // shift layout and influence focus or selectors. Callers that annotate the - // recording afterwards opt out to keep the capture faithful. - return; - } - const app = this.appService.application; - if (!app) { - throw new Error('VS Code is not running.'); - } - const values = JSON.stringify({ id, title, status }); - await app.code.driver.evaluateExpression(`(() => { - const values = ${values}; - let overlay = document.getElementById('vscode-ui-evidence-overlay'); - if (!overlay) { - overlay = document.createElement('div'); - overlay.id = 'vscode-ui-evidence-overlay'; - overlay.style.cssText = 'position:fixed;top:12px;left:50%;transform:translateX(-50%);z-index:2147483647;max-width:70vw;padding:10px 16px;border-radius:6px;background:rgba(0,0,0,.88);color:#fff;font:600 14px/1.4 system-ui;box-shadow:0 4px 18px rgba(0,0,0,.35);pointer-events:none;text-align:center'; - document.documentElement.appendChild(overlay); - } - overlay.textContent = values.id + ': ' + values.title + ' [' + values.status.toUpperCase() + ']'; - overlay.dataset.status = values.status; - return overlay.textContent; - })()`); - } - private async capture(name: string): Promise { const run = this.requireRun(); const app = this.appService.application; @@ -513,66 +473,6 @@ export class EvidenceService { } -export function applyEvidenceStartTool(server: McpServer, evidenceService: EvidenceService): RegisteredTool { - return server.tool( - 'vscode_automation_evidence_start', - 'Start VS Code with video and trace recording for a UI validation scenario', - { - scenarioId: z.string().describe('Stable scenario identifier'), - title: z.string().describe('Human-readable scenario title'), - source: z.string().url().refine(isHttpUrl, 'Source must use HTTP or HTTPS').optional().describe('Source test-plan issue URL'), - scenarioPath: z.string().optional().describe('Path to the Markdown scenario definition'), - workspacePath: z.string().optional().describe('Workspace or folder to open'), - userSettings: z.record(z.string(), jsonValueSchema).optional().describe('User settings to seed before VS Code starts'), - extraArgs: z.array(z.string()).optional().describe('Additional VS Code command-line arguments') - }, - async ({ scenarioId, title, source, scenarioPath, workspacePath, userSettings, extraArgs }) => { - const runPath = await evidenceService.start(scenarioId, title, source, scenarioPath, workspacePath, userSettings, extraArgs); - return { - content: [{ type: 'text' as const, text: `Evidence capture started: ${runPath}` }] - }; - } - ); -} - -export function applyEvidenceTools(server: McpServer, evidenceService: EvidenceService): RegisteredTool[] { - return [ - server.tool( - 'vscode_automation_evidence_step', - 'Mark a scenario step in the video and save a screenshot of the current VS Code window', - { - id: z.string().describe('Stable step identifier from the scenario'), - title: z.string().describe('Human-readable step title'), - status: z.enum(['started', 'passed', 'failed', 'skipped']).describe('Step lifecycle status'), - details: z.string().optional().describe('Validation result or failure details') - }, - async ({ id, title, status, details }) => { - const result = await evidenceService.step(id, title, status, details); - return { - content: [ - { type: 'text' as const, text: `Evidence saved: ${result.screenshotPath}` }, - { type: 'image' as const, data: result.screenshot.toString('base64'), mimeType: 'image/png' } - ] - }; - } - ), - server.tool( - 'vscode_automation_evidence_finish', - 'Finish a UI validation scenario, stop VS Code, and write the evidence report', - { - outcome: z.enum(['passed', 'failed', 'aborted']).describe('Overall scenario outcome'), - notes: z.string().optional().describe('Run summary or blocking condition') - }, - async ({ outcome, notes }) => { - const reportPath = await evidenceService.finish(outcome, notes); - return { - content: [{ type: 'text' as const, text: `Evidence report written: ${reportPath}` }] - }; - } - ) - ]; -} - function sanitizePathSegment(value: string): string { const sanitized = value.trim().replace(/[^a-zA-Z0-9._-]+/g, '-').replace(/^-+|-+$/g, ''); return sanitized || 'unnamed'; diff --git a/test/scenario/src/index.ts b/test/scenario/src/index.ts new file mode 100644 index 0000000000000..f8a985856c7ac --- /dev/null +++ b/test/scenario/src/index.ts @@ -0,0 +1,11 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +export * from './application'; +export * from './evidence'; +export * from './options'; +export * from './renderEvidenceChapters'; +export * from './runScenario'; +export * from './utils'; diff --git a/test/mcp/src/options.ts b/test/scenario/src/options.ts similarity index 100% rename from test/mcp/src/options.ts rename to test/scenario/src/options.ts diff --git a/test/mcp/src/renderEvidenceChapters.ts b/test/scenario/src/renderEvidenceChapters.ts similarity index 100% rename from test/mcp/src/renderEvidenceChapters.ts rename to test/scenario/src/renderEvidenceChapters.ts diff --git a/test/mcp/src/runScenario.ts b/test/scenario/src/runScenario.ts similarity index 91% rename from test/mcp/src/runScenario.ts rename to test/scenario/src/runScenario.ts index 697fb9b54c4c5..aa45a1b72173b 100644 --- a/test/mcp/src/runScenario.ts +++ b/test/scenario/src/runScenario.ts @@ -13,12 +13,8 @@ import { renderChapters } from './renderEvidenceChapters'; /** * Runs a UI validation scenario end to end and writes an evidence bundle. * - * This is the same capture pipeline the `vscode_automation_evidence_*` MCP tools - * drive, exposed as a single command so a scenario can be recorded without - * configuring an MCP server: - * * ``` - * node test/mcp/out/runScenario.js [--build ] + * node test/scenario/out/runScenario.js [--build ] * ``` * * The scenario file is not part of this repository, so it can be written next to @@ -99,11 +95,6 @@ function loadScenario(scenarioPath: string): Scenario { } export async function runScenario(scenario: Scenario): Promise<{ runPath: string; outcome: 'passed' | 'failed' | 'aborted' }> { - // The step banner is drawn into the DOM of the product under test, so it can - // shift layout and influence focus. Chapters are rendered onto the finished - // recording instead, which keeps the capture faithful. - process.env.VSCODE_EVIDENCE_CLEAN_CAPTURE ??= '1'; - const appService = new ApplicationService(); const evidence = new EvidenceService(appService); const runPath = await evidence.start( @@ -167,7 +158,7 @@ export async function runScenario(scenario: Scenario): Promise<{ runPath: string if (require.main === module) { const scenarioArgument = process.argv.slice(2).find(argument => !argument.startsWith('--')); if (!scenarioArgument) { - console.error('Usage: node test/mcp/out/runScenario.js [--build ]'); + console.error('Usage: node test/scenario/out/runScenario.js [--build ]'); process.exit(2); } const scenarioPath = path.resolve(scenarioArgument); diff --git a/test/mcp/src/utils.ts b/test/scenario/src/utils.ts similarity index 100% rename from test/mcp/src/utils.ts rename to test/scenario/src/utils.ts diff --git a/test/scenario/tsconfig.json b/test/scenario/tsconfig.json new file mode 100644 index 0000000000000..5341c14f225e7 --- /dev/null +++ b/test/scenario/tsconfig.json @@ -0,0 +1,25 @@ +{ + "compilerOptions": { + "module": "commonjs", + "noImplicitAny": false, + "removeComments": false, + "preserveConstEnums": true, + "target": "es2024", + "strict": true, + "noUnusedParameters": false, + "noUnusedLocals": true, + "rootDir": "./src", + "outDir": "out", + "sourceMap": true, + "declaration": true, + "skipLibCheck": true, + "lib": [ + "esnext", + "dom" + ] + }, + "exclude": [ + "node_modules", + "out" + ] +}