Move the scenario runner out of the MCP server

.github/CODENOTIFY

@@ -83,6 +83,7 @@ extensions/vscode-api-tests/src/singlefolder-tests/browser*.test.ts @kycutler @j
 
 # Testing
 test/mcp/** @TylerLeonhardt
+test/scenario/** @bryanchen-d
 test/sanity/** @dmitrivMS
 
 # Agents Workbench

.github/skills/validate-ui-scenario/SKILL.md

@@ -11,15 +11,15 @@ Use this to reproduce a reported bug, to show that a fix works, or to attach a r
 test-plan item. For deterministic regression coverage that runs on every build, write a smoke test
 instead (see the `smoke-tests` skill) — this skill is for one-off, issue-derived validation.
 
-A scenario is a small JavaScript file run by `test/mcp/out/runScenario.js`. Nothing else has to be
+A scenario is a small JavaScript file run by `test/scenario/out/runScenario.js`. Nothing else has to be
 configured: the runner launches VS Code, records video and a trace, captures a screenshot at every
 step boundary, writes the report, and captions the recording with each step and its result.
 
 ## Prepare
 
 ```bash
 npm install                        # once
-npm --prefix test/mcp run compile  # after any change under test/mcp
+npm --prefix test/scenario run compile  # after any change under test/scenario
 ```
 
 Add `ffmpeg` and `ffprobe` to `PATH` to get the caption band on the video. Without them the run still
@@ -130,7 +130,7 @@ Each step receives a `context` with `app`, `workbench`, `code`, `page`, and `ski
 ## Run it
 
 ```bash
-node test/mcp/out/runScenario.js <scenario.cjs> --build "<app-root>"
+node test/scenario/out/runScenario.js <scenario.cjs> --build "<app-root>"
 ```
 
 Exit code `0` means every step passed, `1` means the run failed or was aborted, `2` a usage error.
@@ -149,7 +149,7 @@ Evidence is written to `.build/vscode-playwright-mcp/evidence/<run-id>/`:
 The caption band is added **above** the recorded frame rather than drawn over it, so no recorded
 pixel is hidden and the recording keeps its original length. Each caption carries the step number
 and id, its status, the step title, and the validation detail the step reported. Re-render after
-editing a manifest with `node test/mcp/out/renderEvidenceChapters.js <run-dir>`.
+editing a manifest with `node test/scenario/out/renderEvidenceChapters.js <run-dir>`.
 
 ## What makes evidence trustworthy
 

build/npm/dirs.ts

@@ -59,6 +59,7 @@ export const dirs = [
 	'test/integration/browser',
 	'test/monaco',
 	'test/smoke',
+	'test/scenario',
 	'test/mcp',
 	'.vscode/extensions/vscode-selfhost-import-aid',
 	'.vscode/extensions/vscode-selfhost-test-provider',

eslint.config.js

@@ -2329,10 +2329,22 @@ export default defineConfig(
 						'*' // node modules
 					]
 				},
+				{
+					'target': 'test/scenario/**',
+					'restrictions': [
+						'test/automation',
+						'test/scenario/**',
+						'@vscode/*',
+						'@parcel/*',
+						'@playwright/*',
+						'*' // node modules
+					]
+				},
 				{
 					'target': 'test/mcp/**',
 					'restrictions': [
 						'test/automation',
+						'test/scenario',
 						'test/mcp/**',
 						'@vscode/*',
 						'@parcel/*',

test/mcp/package.json

@@ -5,10 +5,10 @@
   "main": "./out/main.js",
   "private": true,
   "scripts": {
-    "compile": "cd ../automation && npm run compile && cd ../mcp && node ../../node_modules/typescript/bin/tsc6",
-    "watch-automation": "cd ../automation && npm run watch",
+    "compile": "cd ../scenario && npm run compile && cd ../mcp && node ../../node_modules/typescript/bin/tsc6",
+    "watch-scenario": "cd ../scenario && npm run watch",
     "watch-mcp": "node ../../node_modules/typescript/bin/tsc6 --watch --preserveWatchOutput",
-    "watch": "npm-run-all2 -lp watch-automation watch-mcp",
+    "watch": "npm-run-all2 -lp watch-scenario watch-mcp",
     "start-stdio": "echo 'Starting vscode-automation-mcp... For customization and troubleshooting, see ./test/mcp/README.md' && npm ci && npm run -s compile && node ./out/stdio.js"
   },
   "dependencies": {

test/mcp/src/automation.ts

@@ -4,11 +4,11 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { ApplicationService } from './application';
+import { ApplicationService, EvidenceService } from '../../scenario';
 import { applyAllTools } from './automationTools/index.js';
 import type { Server } from '@modelcontextprotocol/sdk/server/index.js';
 import { z } from 'zod';
-import { applyEvidenceStartTool, applyEvidenceTools, EvidenceService } from './evidence.js';
+import { applyEvidenceStartTool, applyEvidenceTools } from './evidenceTools.js';
 
 export async function getServer(appService: ApplicationService): Promise<Server> {
 	const server = new McpServer({

test/mcp/src/automationTools/activityBar.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { ApplicationService } from '../application';
+import { ApplicationService } from '../../../scenario';
 
 /**
  * Activity Bar Tools

test/mcp/src/automationTools/chat.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { ApplicationService } from '../application';
+import { ApplicationService } from '../../../scenario';
 import { z } from 'zod';
 
 /**

test/mcp/src/automationTools/core.ts

@@ -5,7 +5,7 @@
 
 import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { z } from 'zod';
-import { ApplicationService, assertNoProfileOverrides } from '../application';
+import { ApplicationService, assertNoProfileOverrides } from '../../../scenario';
 
 /**
  * Core Application Management Tools

test/mcp/src/automationTools/debug.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { ApplicationService } from '../application';
+import { ApplicationService } from '../../../scenario';
 import { z } from 'zod';
 
 /**

test/mcp/src/automationTools/editor.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { ApplicationService } from '../application';
+import { ApplicationService } from '../../../scenario';
 import { z } from 'zod';
 
 /**

test/mcp/src/automationTools/explorer.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { ApplicationService } from '../application';
+import { ApplicationService } from '../../../scenario';
 
 /**
  * Explorer and File Management Tools

test/mcp/src/automationTools/extensions.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { ApplicationService } from '../application';
+import { ApplicationService } from '../../../scenario';
 import { z } from 'zod';
 
 /**

test/mcp/src/automationTools/index.ts

@@ -26,7 +26,7 @@ import { applyTaskTools } from './task.js';
 import { applyProfilerTools } from './profiler.js';
 import { applyChatTools } from './chat.js';
 import { applyWindowTools } from './windows.js';
-import { ApplicationService } from '../application';
+import { ApplicationService } from '../../../scenario';
 
 /**
  * Apply all VS Code automation tools to the MCP server

test/mcp/src/automationTools/keybindings.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { ApplicationService } from '../application';
+import { ApplicationService } from '../../../scenario';
 
 /**
  * Keybindings Editor Tools

test/mcp/src/automationTools/localization.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { ApplicationService } from '../application';
+import { ApplicationService } from '../../../scenario';
 
 /**
  * Localization Tools

test/mcp/src/automationTools/notebook.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { ApplicationService } from '../application';
+import { ApplicationService } from '../../../scenario';
 import { z } from 'zod';
 
 /**

test/mcp/src/automationTools/problems.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { ApplicationService } from '../application';
+import { ApplicationService } from '../../../scenario';
 import { z } from 'zod';
 
 /**

test/mcp/src/automationTools/profiler.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { ApplicationService } from '../application';
+import { ApplicationService } from '../../../scenario';
 
 /**
  * Profiler Tools

test/mcp/src/automationTools/quickAccess.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { ApplicationService } from '../application';
+import { ApplicationService } from '../../../scenario';
 import { z } from 'zod';
 
 /**

test/mcp/src/automationTools/scm.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { ApplicationService } from '../application';
+import { ApplicationService } from '../../../scenario';
 import { z } from 'zod';
 
 /**

test/mcp/src/automationTools/search.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { ApplicationService } from '../application';
+import { ApplicationService } from '../../../scenario';
 import { z } from 'zod';
 
 /**

test/mcp/src/automationTools/settings.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { ApplicationService } from '../application';
+import { ApplicationService } from '../../../scenario';
 import { z } from 'zod';
 
 /**

test/mcp/src/automationTools/statusbar.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { ApplicationService } from '../application';
+import { ApplicationService } from '../../../scenario';
 
 /**
  * Status Bar Tools

test/mcp/src/automationTools/task.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { ApplicationService } from '../application';
+import { ApplicationService } from '../../../scenario';
 import { z } from 'zod';
 
 /**

test/mcp/src/automationTools/terminal.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { ApplicationService } from '../application';
+import { ApplicationService } from '../../../scenario';
 import { z } from 'zod';
 
 /**

test/mcp/src/automationTools/windows.ts

@@ -5,7 +5,7 @@
 
 import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { z } from 'zod';
-import { ApplicationService } from '../application';
+import { ApplicationService } from '../../../scenario';
 
 /**
  * Create a standardized text response for window tools

test/mcp/src/evidenceTools.ts

@@ -0,0 +1,85 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { McpServer, RegisteredTool } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { z } from 'zod';
+import { EvidenceService, JSONValue } from '../../scenario';
+
+const jsonValueSchema: z.ZodType<JSONValue> = z.lazy(() => z.union([
+	z.string(),
+	z.number(),
+	z.boolean(),
+	z.null(),
+	z.array(jsonValueSchema),
+	z.record(z.string(), jsonValueSchema)
+]));
+
+function isHttpUrl(value: string): boolean {
+	try {
+		return ['http:', 'https:'].includes(new URL(value).protocol);
+	} catch {
+		return false;
+	}
+}
+
+export function applyEvidenceStartTool(server: McpServer, evidenceService: EvidenceService): RegisteredTool {
+	return server.tool(
+		'vscode_automation_evidence_start',
+		'Start VS Code with video and trace recording for a UI validation scenario',
+		{
+			scenarioId: z.string().describe('Stable scenario identifier'),
+			title: z.string().describe('Human-readable scenario title'),
+			source: z.string().url().refine(isHttpUrl, 'Source must use HTTP or HTTPS').optional().describe('Source test-plan issue URL'),
+			scenarioPath: z.string().optional().describe('Path to the Markdown scenario definition'),
+			workspacePath: z.string().optional().describe('Workspace or folder to open'),
+			userSettings: z.record(z.string(), jsonValueSchema).optional().describe('User settings to seed before VS Code starts'),
+			extraArgs: z.array(z.string()).optional().describe('Additional VS Code command-line arguments')
+		},
+		async ({ scenarioId, title, source, scenarioPath, workspacePath, userSettings, extraArgs }) => {
+			const runPath = await evidenceService.start(scenarioId, title, source, scenarioPath, workspacePath, userSettings, extraArgs);
+			return {
+				content: [{ type: 'text' as const, text: `Evidence capture started: ${runPath}` }]
+			};
+		}
+	);
+}
+
+export function applyEvidenceTools(server: McpServer, evidenceService: EvidenceService): RegisteredTool[] {
+	return [
+		server.tool(
+			'vscode_automation_evidence_step',
+			'Mark a scenario step in the video and save a screenshot of the current VS Code window',
+			{
+				id: z.string().describe('Stable step identifier from the scenario'),
+				title: z.string().describe('Human-readable step title'),
+				status: z.enum(['started', 'passed', 'failed', 'skipped']).describe('Step lifecycle status'),
+				details: z.string().optional().describe('Validation result or failure details')
+			},
+			async ({ id, title, status, details }) => {
+				const result = await evidenceService.step(id, title, status, details);
+				return {
+					content: [
+						{ type: 'text' as const, text: `Evidence saved: ${result.screenshotPath}` },
+						{ type: 'image' as const, data: result.screenshot.toString('base64'), mimeType: 'image/png' }
+					]
+				};
+			}
+		),
+		server.tool(
+			'vscode_automation_evidence_finish',
+			'Finish a UI validation scenario, stop VS Code, and write the evidence report',
+			{
+				outcome: z.enum(['passed', 'failed', 'aborted']).describe('Overall scenario outcome'),
+				notes: z.string().optional().describe('Run summary or blocking condition')
+			},
+			async ({ outcome, notes }) => {
+				const reportPath = await evidenceService.finish(outcome, notes);
+				return {
+					content: [{ type: 'text' as const, text: `Evidence report written: ${reportPath}` }]
+				};
+			}
+		)
+	];
+}

test/mcp/src/stdio.ts

@@ -4,8 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 import { getServer } from './automation';
-import { ApplicationService } from './application';
-import { opts } from './options';
+import { ApplicationService, opts } from '../../scenario';
 
 const transport: StdioServerTransport = new StdioServerTransport();
 (async () => {

test/scenario/.gitignore

@@ -0,0 +1,5 @@
+.DS_Store
+npm-debug.log
+Thumbs.db
+node_modules/
+out/