diff --git a/.github/workflows/pr-darwin-test.yml b/.github/workflows/pr-darwin-test.yml index 3fc4fbea2e978..bd7746bd89b6f 100644 --- a/.github/workflows/pr-darwin-test.yml +++ b/.github/workflows/pr-darwin-test.yml @@ -202,6 +202,7 @@ jobs: timeout-minutes: 20 run: ./scripts/test-integration.sh --tfs "Integration Tests" env: + VSCODE_PARALLEL_NODE_INTEGRATION_TESTS: '1' VSCODE_SKIP_AGENT_HOST_E2E: ${{ steps.agent-host-e2e-changes.outputs.affected == 'false' && '1' || '0' }} VSCODE_SKIP_PRELAUNCH: '1' diff --git a/.github/workflows/pr-linux-test.yml b/.github/workflows/pr-linux-test.yml index 057fb511a3f78..7b433653f0fc4 100644 --- a/.github/workflows/pr-linux-test.yml +++ b/.github/workflows/pr-linux-test.yml @@ -404,6 +404,7 @@ jobs: run: ./scripts/test-integration.sh --tfs "Integration Tests" env: DISPLAY: ":10" + VSCODE_PARALLEL_NODE_INTEGRATION_TESTS: '1' VSCODE_SKIP_AGENT_HOST_E2E: ${{ steps.agent-host-e2e-changes.outputs.affected == 'false' && '1' || '0' }} VSCODE_SKIP_PRELAUNCH: '1' diff --git a/.github/workflows/pr-win32-test.yml b/.github/workflows/pr-win32-test.yml index 8680fae6cf28d..320c0b39330ae 100644 --- a/.github/workflows/pr-win32-test.yml +++ b/.github/workflows/pr-win32-test.yml @@ -227,6 +227,7 @@ jobs: shell: pwsh run: .\scripts\test-integration.bat --tfs "Integration Tests" env: + VSCODE_PARALLEL_NODE_INTEGRATION_TESTS: '1' VSCODE_SKIP_AGENT_HOST_E2E: ${{ steps.agent-host-e2e-changes.outputs.affected == 'false' && '1' || '0' }} VSCODE_SKIP_PRELAUNCH: '1' diff --git a/build/lib/test/agentHostE2ERunner.test.ts b/build/lib/test/agentHostE2ERunner.test.ts new file mode 100644 index 0000000000000..a68de49e63849 --- /dev/null +++ b/build/lib/test/agentHostE2ERunner.test.ts @@ -0,0 +1,339 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { EventEmitter } from 'events'; +import { readFileSync } from 'fs'; +import { stripTypeScriptTypes } from 'module'; +import path from 'path'; +import { PassThrough } from 'stream'; +import { suite, test } from 'node:test'; +import { setImmediate } from 'timers/promises'; +import { runInNewContext } from 'vm'; + +const scriptDirectory = path.resolve(import.meta.dirname, '../../../scripts'); +const runnerSource = stripTypeScriptTypes(readFileSync(path.join(scriptDirectory, 'test-agent-host-e2e.ts'), 'utf8')); +const e2eFiles = [ + 'src/vs/platform/agentHost/test/node/e2e/conformance/agentHostConformance.integrationTest.ts', + 'src/vs/platform/agentHost/test/node/e2e/providers/claudeAgentHostE2E.integrationTest.ts', + 'src/vs/platform/agentHost/test/node/e2e/providers/codexAgentHostE2E.integrationTest.ts', + 'src/vs/platform/agentHost/test/node/e2e/providers/copilotAgentHostE2E.integrationTest.ts', +]; +const nodeArguments = [ + '--runGlob', '**/*.integrationTest.js', + '--excludeRunGlob', '**/agentHost/test/node/e2e/{providers/*AgentHostE2E,conformance/*}.integrationTest.js', +]; + +class TestChildProcess extends EventEmitter { + readonly stdout = new PassThrough(); + readonly stderr = new PassThrough(); + readonly command: string; + readonly args: readonly string[]; + readonly env: NodeJS.ProcessEnv; + readonly testArgs: readonly string[]; + closed = false; + + constructor( + command: string, + args: readonly string[], + env: NodeJS.ProcessEnv, + testArgs: readonly string[], + ) { + super(); + this.command = command; + this.args = args; + this.env = env; + this.testArgs = testArgs; + } + + complete(code = 0, output = ''): void { + assert(!this.closed); + this.closed = true; + this.stdout.end(output); + this.stderr.end(); + this.emit('close', code, null); + } +} + +function runRunner(options: { + args?: readonly string[]; + env?: NodeJS.ProcessEnv; + platform?: NodeJS.Platform; + parallelism?: number; +} = {}) { + const children: TestChildProcess[] = []; + const output: string[] = []; + const files = new Map(); + const platform = options.platform ?? 'linux'; + const state = { + argv: ['node', path.join(scriptDirectory, 'test-agent-host-e2e.ts'), ...options.args ?? []], + env: { VSCODE_SKIP_PRELAUNCH: '1', ...options.env }, + platform, + hrtime: process.hrtime, + exitCode: 0, + stdout: { write: (value: string) => output.push(value) }, + }; + let active = 0; + let maxActive = 0; + + const modules = new Map([ + ['child_process', { + spawn: (command: string, args: readonly string[], options: { env: NodeJS.ProcessEnv }) => { + const testArgs = platform === 'win32' ? args.slice(args.indexOf('-File') + 3) : args; + const child = new TestChildProcess(command, [...args], { ...options.env }, [...testArgs]); + children.push(child); + maxActive = Math.max(maxActive, ++active); + child.once('close', () => active--); + return child; + }, + spawnSync: () => { + throw new Error('Unexpected dependency installation or Electron download'); + }, + }], + ['fs', { + existsSync: (file: string) => file.endsWith('node_modules') || files.has(file), + mkdirSync: () => { }, + rmSync: (file: string) => files.delete(file), + readFileSync: (file: string) => { + const value = files.get(file); + assert.notStrictEqual(value, undefined); + return value; + }, + writeFileSync: (file: string, value: string) => files.set(file, value), + }], + ['os', { + availableParallelism: () => options.parallelism ?? 8, + cpus: () => Array.from({ length: options.parallelism ?? 8 }), + }], + ['path', path], + ]); + + // Isolate the CommonJS CLI's dependencies without patching the test process's globals. + const completion: Promise = runInNewContext(runnerSource, { + __dirname: scriptDirectory, + require: (name: string) => { + assert(modules.has(name), `Unexpected module: ${name}`); + return modules.get(name); + }, + process: state, + console: { + log: (...args: (string | Error)[]) => output.push(args.map(String).join(' ')), + error: (...args: (string | Error)[]) => output.push(args.map(String).join(' ')), + }, + }); + + return { + children, + output, + files, + state, + completion, + get active() { return active; }, + get maxActive() { return maxActive; }, + async finish() { + for (let index = 0; index < children.length; index++) { + if (!children[index].closed) { + children[index].complete(); + } + await setImmediate(); + } + await completion; + }, + }; +} + +suite('Agent Host E2E runner', () => { + for (const platform of ['linux', 'darwin', 'win32'] as const) { + test(`fills the first freed worker with remaining node tests on ${platform}`, async t => { + const runner = runRunner({ + platform, + args: ['--include-node-tests', '--tfs', 'Integration Tests', '--grep', 'test with spaces', '--timeout', '12000'], + env: { ELECTRON_RUN_AS_NODE: '1' }, + }); + t.after(() => runner.finish()); + + const initialFiles = runner.children.map(child => child.testArgs[1]); + await setImmediate(); + assert.strictEqual(runner.children.length, 4); + runner.children[2].complete(); + await setImmediate(); + + assert.deepStrictEqual({ + initialFiles, + active: runner.active, + maxActive: runner.maxActive, + started: runner.children.length, + remainingArgs: runner.children[4].testArgs, + reportNames: runner.children.map(child => child.testArgs[child.testArgs.indexOf('--tfs') + 1]), + childEnvironment: runner.children.map(child => [child.env.VSCODE_SKIP_PRELAUNCH, child.env.ELECTRON_RUN_AS_NODE]), + windowsWrappers: runner.children.every(child => platform !== 'win32' || ( + child.command.endsWith('powershell.exe') + && child.args.includes(path.join(scriptDirectory, 'test-agent-host-e2e-child.ps1')) + && child.args.includes(path.join(scriptDirectory, child.testArgs[0] === '--runGlob' ? 'test.bat' : 'test-integration.bat')) + )), + }, { + initialFiles: e2eFiles, + active: 4, + maxActive: 4, + started: 5, + remainingArgs: [...nodeArguments, '--tfs', 'Integration Tests', '--grep', 'test with spaces', '--timeout', '12000'], + reportNames: ['Integration Tests Conformance', 'Integration Tests Claude', 'Integration Tests Codex', 'Integration Tests Copilot', 'Integration Tests'], + childEnvironment: Array.from({ length: 5 }, () => ['1', undefined]), + windowsWrappers: true, + }); + + await runner.finish(); + assert.strictEqual(runner.state.exitCode, 0); + }); + } + + for (const { args, env, parallelism, workers } of [ + { args: ['--jobs=99'], env: {}, parallelism: 16, workers: 4 }, + { args: [], env: {}, parallelism: 2, workers: 2 }, + { args: ['--jobs', '1'], env: { AGENT_HOST_E2E_JOBS: '3' }, parallelism: 8, workers: 1 }, + { args: [], env: { AGENT_HOST_E2E_JOBS: '3' }, parallelism: 8, workers: 3 }, + ]) { + test(`keeps the requested worker bound (${workers}, ${JSON.stringify(args)})`, async t => { + const runner = runRunner({ args: ['--include-node-tests', ...args], env, parallelism }); + t.after(() => runner.finish()); + const initialCount = runner.children.length; + await runner.finish(); + assert.deepStrictEqual({ + initialCount, + maxActive: runner.maxActive, + selections: runner.children.map(child => child.testArgs.slice(0, 2)), + exitCode: runner.state.exitCode, + }, { + initialCount: workers, + maxActive: workers, + selections: [...e2eFiles.map(file => ['--run', file]), nodeArguments.slice(0, 2)], + exitCode: 0, + }); + }); + } + + test('keeps standalone E2E invocation unchanged', async t => { + const runner = runRunner({ env: { VSCODE_SKIP_AGENT_HOST_E2E: '1' } }); + t.after(() => runner.finish()); + await runner.finish(); + assert.deepStrictEqual({ + files: runner.children.map(child => child.testArgs[1]), + exitCode: runner.state.exitCode, + summary: runner.output.some(line => line.includes('Agent Host E2E suites completed')), + }, { files: e2eFiles, exitCode: 0, summary: true }); + }); + + test('runs only the remaining node tests when E2E tests are unaffected', async t => { + const runner = runRunner({ + args: ['--include-node-tests', '--tfs', 'Integration Tests'], + env: { VSCODE_SKIP_AGENT_HOST_E2E: '1' }, + }); + t.after(() => runner.finish()); + await runner.finish(); + assert.deepStrictEqual({ + args: runner.children.map(child => child.testArgs), + maxActive: runner.maxActive, + exitCode: runner.state.exitCode, + skipped: runner.output.some(line => line.includes('Skipping Agent Host E2E tests')), + }, { + args: [[...nodeArguments, '--tfs', 'Integration Tests']], + maxActive: 1, + exitCode: 0, + skipped: true, + }); + }); + + for (const failingGroup of ['e2e', 'node']) { + test(`waits for all groups and reports a ${failingGroup} failure`, async t => { + const runner = runRunner({ args: ['--include-node-tests'] }); + t.after(() => runner.finish()); + runner.children[0].complete(failingGroup === 'e2e' ? 17 : 0, failingGroup === 'e2e' ? '1 failing\nE2E failure\n' : ''); + await setImmediate(); + runner.children[4].complete(failingGroup === 'node' ? 17 : 0, failingGroup === 'node' ? '1 failing\nNode failure\n' : ''); + await setImmediate(); + const beforeJoining = { active: runner.active, exitCode: runner.state.exitCode }; + await runner.finish(); + assert.deepStrictEqual({ + beforeJoining, + groups: runner.children.length, + exitCode: runner.state.exitCode, + failureSummary: runner.output.some(line => line.includes('failure details:')), + exitStatus: runner.output.some(line => line.includes('failed with code 17')), + }, { + beforeJoining: { active: 3, exitCode: 0 }, + groups: 5, + exitCode: 1, + failureSummary: true, + exitStatus: true, + }); + }); + } + + test('propagates a child spawn failure', async t => { + const runner = runRunner({ args: ['--include-node-tests'] }); + t.after(() => runner.finish()); + runner.children[0].emit('error', new Error('Unable to spawn test process')); + runner.children[0].complete(1); + await runner.finish(); + assert.deepStrictEqual({ + groups: runner.children.length, + exitCode: runner.state.exitCode, + spawnError: runner.output.some(line => line.includes('Unable to spawn test process')), + }, { groups: 5, exitCode: 1, spawnError: true }); + }); + + test('keeps protocol surface output scoped to the E2E entrypoints', async t => { + const combinedOutput = path.join(scriptDirectory, 'observed.json'); + const runner = runRunner({ + args: ['--include-node-tests'], + env: { + AGENT_HOST_RECORD_PROTOCOL_SURFACE: '1', + AGENT_HOST_PROTOCOL_SURFACE_OUT: combinedOutput, + AGENT_HOST_E2E_COVERAGE: '1', + }, + }); + t.after(() => runner.finish()); + for (const [index, child] of runner.children.entries()) { + const output = child.env.AGENT_HOST_PROTOCOL_SURFACE_OUT; + assert(output); + runner.files.set(output, JSON.stringify({ commands: [`command-${index}`], notifications: [], actions: [] })); + } + runner.children[2].complete(); + await setImmediate(); + const remaining = runner.children[4]; + await runner.finish(); + assert.deepStrictEqual({ + remainingSurfaceEnvironment: [remaining.env.AGENT_HOST_RECORD_PROTOCOL_SURFACE, remaining.env.AGENT_HOST_PROTOCOL_SURFACE_OUT], + files: [...runner.files.keys()], + combined: JSON.parse(runner.files.get(combinedOutput)!), + exitCode: runner.state.exitCode, + }, { + remainingSurfaceEnvironment: [undefined, undefined], + files: [combinedOutput], + combined: { commands: ['command-0', 'command-1', 'command-2', 'command-3'], notifications: [], actions: [] }, + exitCode: 0, + }); + }); + + for (const args of [['--jobs', '0'], ['--jobs=-1'], ['--jobs=1.5'], ['--jobs=invalid'], ['--jobs'], ['--run', 'test.ts'], ['--testSplit', '1/2']]) { + test(`rejects invalid scheduler arguments: ${args.join(' ')}`, async () => { + const runner = runRunner({ args: ['--include-node-tests', ...args] }); + await runner.completion; + assert.deepStrictEqual({ children: runner.children.length, exitCode: runner.state.exitCode }, { children: 0, exitCode: 1 }); + }); + } + + for (const flag of ['AGENT_HOST_REPLAY_RECORD', 'AGENT_HOST_UPDATE_AHP_SNAPSHOTS', 'AGENT_HOST_UPDATE_SNAPSHOTS']) { + test(`rejects recording mode: ${flag}`, async () => { + const runner = runRunner({ args: ['--include-node-tests'], env: { [flag]: '1' } }); + await runner.completion; + assert.deepStrictEqual({ + children: runner.children.length, + exitCode: runner.state.exitCode, + diagnostic: runner.output.some(line => line.includes(`unset ${flag}`)), + }, { children: 0, exitCode: 1, diagnostic: true }); + }); + } +}); diff --git a/build/lib/test/integrationTestRunner.test.ts b/build/lib/test/integrationTestRunner.test.ts new file mode 100644 index 0000000000000..f98710fa0c5de --- /dev/null +++ b/build/lib/test/integrationTestRunner.test.ts @@ -0,0 +1,163 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { spawnSync } from 'child_process'; +import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync, chmodSync } from 'fs'; +import { tmpdir } from 'os'; +import path from 'path'; +import { suite, test, type TestContext } from 'node:test'; + +const scriptDirectory = path.resolve(import.meta.dirname, '../../../scripts'); + +interface ITestCall { + readonly phase: 'node' | 'extension'; + readonly args: readonly string[]; +} + +function runIntegrationScript(t: TestContext, options: { + args?: readonly string[]; + parallel?: boolean; + skipE2E?: boolean; + failNode?: boolean; +}) { + const root = mkdtempSync(path.join(tmpdir(), 'vscode-integration-runner-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const scripts = path.join(root, 'scripts'); + const temporaryDirectory = path.join(root, 'tmp'); + const callsDirectory = path.join(root, 'calls'); + for (const directory of [scripts, temporaryDirectory, callsDirectory, path.join(root, 'node_modules')]) { + mkdirSync(directory); + } + for (const file of ['package.json', 'test-agent-host-e2e.ts', 'test-agent-host-e2e-child.ps1', 'test-integration.bat', 'test-integration.sh']) { + copyFileSync(path.join(scriptDirectory, file), path.join(scripts, file)); + } + chmodSync(path.join(scripts, 'test-integration.sh'), 0o755); + writeFileSync(path.join(scripts, 'runner-fixture.cjs'), ` +const fs = require('fs'); +const path = require('path'); +const phase = process.argv[2]; +const args = process.argv.slice(3); +fs.writeFileSync(path.join(__dirname, '..', 'calls', process.pid + '.json'), JSON.stringify({ phase, args })); +process.exit(phase === 'extension' ? 23 : process.env.FAIL_NODE === '1' && args.includes('--runGlob') ? 17 : 0); +`); + for (const [name, phase] of [['test', 'node'], ['code', 'extension']]) { + writeFileSync(path.join(scripts, `${name}.bat`), `@echo off\r\nnode "%~dp0runner-fixture.cjs" ${phase} %*\r\nexit /b %errorlevel%\r\n`); + writeFileSync(path.join(scripts, `${name}.sh`), `#!/usr/bin/env bash\nexec node "$(dirname "$0")/runner-fixture.cjs" ${phase} "$@"\n`, { mode: 0o755 }); + } + const env: NodeJS.ProcessEnv = { + ...process.env, + TEMP: temporaryDirectory, + TMP: temporaryDirectory, + TMPDIR: temporaryDirectory, + VSCODE_SKIP_PRELAUNCH: '1', + VSCODE_PARALLEL_NODE_INTEGRATION_TESTS: options.parallel === false ? '0' : '1', + VSCODE_SKIP_AGENT_HOST_E2E: options.skipE2E ? '1' : '0', + AGENT_HOST_E2E_JOBS: '4', + FAIL_NODE: options.failNode ? '1' : '0', + INTEGRATION_TEST_ELECTRON_PATH: '', + }; + for (const name of [ + 'ELECTRON_RUN_AS_NODE', + 'AGENT_HOST_REPLAY_RECORD', + 'AGENT_HOST_UPDATE_AHP_SNAPSHOTS', + 'AGENT_HOST_UPDATE_SNAPSHOTS', + 'AGENT_HOST_RECORD_PROTOCOL_SURFACE', + 'AGENT_HOST_PROTOCOL_SURFACE_OUT', + 'AGENT_HOST_E2E_COVERAGE', + ]) { + delete env[name]; + } + const args = options.args ?? ['--tfs', 'Integration Tests', '--grep', 'test with spaces']; + const script = path.join(scripts, process.platform === 'win32' ? 'test-integration.bat' : 'test-integration.sh'); + const result = process.platform === 'win32' + ? spawnSync(path.join(process.env.SYSTEMROOT ?? 'C:\\Windows', 'System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe'), [ + '-NoLogo', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', + '-File', path.join(scripts, 'test-agent-host-e2e-child.ps1'), script, ...args, + ], { + cwd: root, env, encoding: 'utf8', timeout: 30_000, + }) + : spawnSync(script, args, { cwd: root, env, encoding: 'utf8', timeout: 30_000 }); + assert.ifError(result.error); + const calls: ITestCall[] = readdirSync(callsDirectory).map(file => JSON.parse(readFileSync(path.join(callsDirectory, file), 'utf8'))); + return { ...result, calls, temporaryDirectories: readdirSync(temporaryDirectory) }; +} + +suite('Integration test entrypoint', () => { + test('runs all five node groups before starting extension host tests', t => { + const result = runIntegrationScript(t, {}); + const nodeCalls = result.calls.filter(call => call.phase === 'node'); + const remaining = nodeCalls.find(call => call.args.includes('--runGlob')); + assert.deepStrictEqual({ + status: result.status, + e2eCount: nodeCalls.filter(call => call.args.includes('--run')).length, + nodeCount: nodeCalls.filter(call => call.args.includes('--runGlob')).length, + extensionCount: result.calls.filter(call => call.phase === 'extension').length, + remainingArgs: remaining?.args, + }, { + status: 23, + e2eCount: 4, + nodeCount: 1, + extensionCount: 1, + remainingArgs: [ + '--runGlob', '**/*.integrationTest.js', + '--excludeRunGlob', '**/agentHost/test/node/e2e/{providers/*AgentHostE2E,conformance/*}.integrationTest.js', + '--tfs', 'Integration Tests', '--grep', 'test with spaces', + ], + }, result.stdout + result.stderr); + }); + + test('does not start extension host tests after the remaining node group fails', t => { + const result = runIntegrationScript(t, { failNode: true }); + assert.deepStrictEqual({ + status: result.status, + nodeCount: result.calls.filter(call => call.phase === 'node').length, + extensionCount: result.calls.filter(call => call.phase === 'extension').length, + cleanedUp: process.platform !== 'win32' || result.temporaryDirectories.length === 0, + }, { status: 1, nodeCount: 5, extensionCount: 0, cleanedUp: true }, result.stdout + result.stderr); + }); + + test('preserves the unaffected-E2E skip without skipping other tests', t => { + const result = runIntegrationScript(t, { skipE2E: true }); + assert.deepStrictEqual({ + status: result.status, + nodeSelections: result.calls.filter(call => call.phase === 'node').map(call => call.args[0]), + extensionCount: result.calls.filter(call => call.phase === 'extension').length, + }, { status: 23, nodeSelections: ['--runGlob'], extensionCount: 1 }, result.stdout + result.stderr); + }); + + test('keeps the serial node phase for callers without the opt-in', t => { + const result = runIntegrationScript(t, { parallel: false }); + assert.deepStrictEqual({ + status: result.status, + e2eCount: result.calls.filter(call => call.args.includes('--run')).length, + nodeCount: result.calls.filter(call => call.args.includes('--runGlob')).length, + extensionCount: result.calls.filter(call => call.phase === 'extension').length, + }, { status: 23, e2eCount: 4, nodeCount: 1, extensionCount: 1 }, result.stdout + result.stderr); + }); + + for (const args of [ + ['--run', 'src/example.integrationTest.ts', '--grep', 'test with spaces'], + ['--runGlob', '**/*.integrationTest.js'], + ['--glob', '**/*.integrationTest.js'], + ['--runGrep', '**/*.integrationTest.js'], + ]) { + test(`preserves file filtering with ${args[0]}`, t => { + const result = runIntegrationScript(t, { args }); + assert.deepStrictEqual({ status: result.status, calls: result.calls }, { + status: 0, + calls: [{ phase: 'node', args }], + }, result.stdout + result.stderr); + }); + } + + test('does not run node tests for an extension suite filter', t => { + const result = runIntegrationScript(t, { args: ['--suite', 'api-folder'] }); + assert.deepStrictEqual({ + status: result.status, + phases: result.calls.map(call => call.phase), + }, { status: 23, phases: ['extension'] }, result.stdout + result.stderr); + }); +}); diff --git a/scripts/test-agent-host-e2e.ts b/scripts/test-agent-host-e2e.ts index 3d5fcd2a96ada..25cfd1dcda643 100644 --- a/scripts/test-agent-host-e2e.ts +++ b/scripts/test-agent-host-e2e.ts @@ -24,7 +24,7 @@ const incompatibleFlags = [ interface ISuite { readonly id: string; readonly label: string; - readonly file: string; + readonly args: readonly string[]; } interface IRunResult { @@ -45,39 +45,58 @@ const suites: readonly ISuite[] = [ { id: 'conformance', label: 'Conformance', - file: 'src/vs/platform/agentHost/test/node/e2e/conformance/agentHostConformance.integrationTest.ts', + args: ['--run', 'src/vs/platform/agentHost/test/node/e2e/conformance/agentHostConformance.integrationTest.ts'], }, { id: 'claude', label: 'Claude', - file: 'src/vs/platform/agentHost/test/node/e2e/providers/claudeAgentHostE2E.integrationTest.ts', + args: ['--run', 'src/vs/platform/agentHost/test/node/e2e/providers/claudeAgentHostE2E.integrationTest.ts'], }, { id: 'codex', label: 'Codex', - file: 'src/vs/platform/agentHost/test/node/e2e/providers/codexAgentHostE2E.integrationTest.ts', + args: ['--run', 'src/vs/platform/agentHost/test/node/e2e/providers/codexAgentHostE2E.integrationTest.ts'], }, { id: 'copilot', label: 'Copilot', - file: 'src/vs/platform/agentHost/test/node/e2e/providers/copilotAgentHostE2E.integrationTest.ts', + args: ['--run', 'src/vs/platform/agentHost/test/node/e2e/providers/copilotAgentHostE2E.integrationTest.ts'], }, ]; +const nodeIntegrationSuite: ISuite = { + id: 'node', + label: 'Node.js integration', + args: [ + '--runGlob', '**/*.integrationTest.js', + '--excludeRunGlob', '**/agentHost/test/node/e2e/{providers/*AgentHostE2E,conformance/*}.integrationTest.js', + ], +}; + async function main(): Promise { validateEnvironment(); - const { jobs, forwardedArgs } = parseArguments(process.argv.slice(2)); + const { jobs, forwardedArgs, includeNodeTests } = parseArguments(process.argv.slice(2)); + const skipAgentHostE2E = includeNodeTests && process.env['VSCODE_SKIP_AGENT_HOST_E2E'] === '1'; + const selectedSuites = skipAgentHostE2E ? [] : [...suites]; + if (includeNodeTests) { + selectedSuites.push(nodeIntegrationSuite); + } + if (skipAgentHostE2E) { + console.log('Skipping Agent Host E2E tests because no relevant files changed.'); + } + const workerCount = Math.min(jobs, selectedSuites.length); + const runLabel = includeNodeTests ? 'Node.js integration' : 'Agent Host E2E'; prepareTestRuntime(); const startedAt = process.hrtime.bigint(); - const surfaceOutputs = prepareSurfaceOutputs(); + const surfaceOutputs = prepareSurfaceOutputs(selectedSuites); const results: IRunResult[] = []; let nextSuite = 0; - const workers = Array.from({ length: jobs }, async () => { - while (nextSuite < suites.length) { + const workers = Array.from({ length: workerCount }, async () => { + while (nextSuite < selectedSuites.length) { const suiteIndex = nextSuite++; - const suite = suites[suiteIndex]; + const suite = selectedSuites[suiteIndex]; results[suiteIndex] = await runSuite(suite, forwardedArgs, surfaceOutputs.get(suite.id)); } }); @@ -89,12 +108,12 @@ async function main(): Promise { } const durationSeconds = elapsedSeconds(startedAt); - console.log(`\nAgent Host E2E suites completed in ${durationSeconds.toFixed(1)}s (${jobs} parallel ${jobs === 1 ? 'worker' : 'workers'}).`); + console.log(`\n${runLabel} suites completed in ${durationSeconds.toFixed(1)}s (${workerCount} parallel ${workerCount === 1 ? 'worker' : 'workers'}).`); for (const result of results) { console.log(` ${result.succeeded ? 'PASS' : 'FAIL'} ${result.suite.label}: ${result.durationSeconds.toFixed(1)}s`); } if (failures.length > 0) { - printFailureDetails(failures); + printFailureDetails(failures, runLabel); process.exitCode = 1; } } @@ -106,9 +125,10 @@ function validateEnvironment(): void { } } -function parseArguments(args: readonly string[]): { jobs: number; forwardedArgs: readonly string[] } { +function parseArguments(args: readonly string[]): { jobs: number; forwardedArgs: readonly string[]; includeNodeTests: boolean } { const forwardedArgs: string[] = []; let requestedJobs: string | undefined = process.env['AGENT_HOST_E2E_JOBS']; + let includeNodeTests = false; for (let index = 0; index < args.length; index++) { const argument = args[index]; @@ -119,6 +139,8 @@ function parseArguments(args: readonly string[]): { jobs: number; forwardedArgs: } } else if (argument.startsWith('--jobs=')) { requestedJobs = argument.slice('--jobs='.length); + } else if (argument === '--include-node-tests') { + includeNodeTests = true; } else { forwardedArgs.push(argument); } @@ -135,7 +157,7 @@ function parseArguments(args: readonly string[]): { jobs: number; forwardedArgs: if (!Number.isInteger(jobs) || jobs < 1) { throw new Error(`Invalid Agent Host E2E worker count: ${requestedJobs}`); } - return { jobs: Math.min(jobs, suites.length), forwardedArgs }; + return { jobs: Math.min(jobs, suites.length), forwardedArgs, includeNodeTests }; } function prepareTestRuntime(): void { @@ -167,17 +189,25 @@ function runSync(command: string, args: readonly string[], environment: NodeJS.P } async function runSuite(suite: ISuite, forwardedArgs: readonly string[], surfaceOutput: string | undefined): Promise { - console.log(`Starting Agent Host E2E — ${suite.label}`); + const suiteLabel = suite === nodeIntegrationSuite ? suite.label : `Agent Host E2E — ${suite.label}`; + console.log(`Starting ${suiteLabel}`); const startedAt = process.hrtime.bigint(); - const environment = { + const environment: NodeJS.ProcessEnv = { ...process.env, VSCODE_SKIP_PRELAUNCH: '1', ...(surfaceOutput ? { AGENT_HOST_PROTOCOL_SURFACE_OUT: surfaceOutput } : {}), }; delete environment.ELECTRON_RUN_AS_NODE; + if (suite === nodeIntegrationSuite) { + delete environment.AGENT_HOST_RECORD_PROTOCOL_SURFACE; + delete environment.AGENT_HOST_PROTOCOL_SURFACE_OUT; + } + const script = suite === nodeIntegrationSuite + ? join(repoRoot, 'scripts', process.platform === 'win32' ? 'test.bat' : 'test.sh') + : testScript; return new Promise(resolveResult => { - const testArguments = ['--run', suite.file, ...suiteArguments(forwardedArgs, suite)]; + const testArguments = [...suite.args, ...suiteArguments(forwardedArgs, suite)]; const child = process.platform === 'win32' ? spawn(join(process.env['SYSTEMROOT'] ?? 'C:\\Windows', 'System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe'), [ '-NoLogo', @@ -185,18 +215,18 @@ async function runSuite(suite: ISuite, forwardedArgs: readonly string[], surface '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-File', windowsTestWrapper, - testScript, + script, ...testArguments, ], { cwd: repoRoot, env: environment, stdio: ['ignore', 'pipe', 'pipe'], }) - : spawn(testScript, testArguments, { - cwd: repoRoot, - env: environment, - stdio: ['ignore', 'pipe', 'pipe'], - }); + : spawn(script, testArguments, { + cwd: repoRoot, + env: environment, + stdio: ['ignore', 'pipe', 'pipe'], + }); let output = ''; child.stdout.setEncoding('utf8'); child.stderr.setEncoding('utf8'); @@ -214,7 +244,7 @@ async function runSuite(suite: ISuite, forwardedArgs: readonly string[], surface child.on('close', (code, signal) => { const succeeded = code === 0; const failure = succeeded ? undefined : signal ? `signal ${signal}` : `code ${code}`; - console.log(`\n===== Agent Host E2E — ${suite.label} =====`); + console.log(`\n===== ${suiteLabel} =====`); process.stdout.write(output); if (!output.endsWith('\n')) { process.stdout.write('\n'); @@ -243,27 +273,28 @@ function extractFailureDetails(output: string): string | undefined { return trimmed.length > 0 ? `${trimmed}\n` : undefined; } -function printFailureDetails(failures: readonly IRunResult[]): void { - console.log('\nAgent Host E2E failure details:'); +function printFailureDetails(failures: readonly IRunResult[], runLabel: string): void { + console.log(`\n${runLabel} failure details:`); for (const result of failures) { - console.log(`\n===== Agent Host E2E — ${result.suite.label} failure =====`); + const suiteLabel = result.suite === nodeIntegrationSuite ? result.suite.label : `Agent Host E2E — ${result.suite.label}`; + console.log(`\n===== ${suiteLabel} failure =====`); if (result.failureDetails) { process.stdout.write(result.failureDetails); } - console.log(`Agent Host E2E — ${result.suite.label} failed with ${result.failure ?? 'an unknown error'}`); + console.log(`${suiteLabel} failed with ${result.failure ?? 'an unknown error'}`); } } function suiteArguments(args: readonly string[], suite: ISuite): readonly string[] { const result = [...args]; const tfsIndex = result.indexOf('--tfs'); - if (tfsIndex >= 0 && result[tfsIndex + 1]) { + if (suite !== nodeIntegrationSuite && tfsIndex >= 0 && result[tfsIndex + 1]) { result[tfsIndex + 1] = `${result[tfsIndex + 1]} ${suite.label}`; } return result; } -function prepareSurfaceOutputs(): ReadonlyMap { +function prepareSurfaceOutputs(selectedSuites: readonly ISuite[]): ReadonlyMap { if (process.env['AGENT_HOST_RECORD_PROTOCOL_SURFACE'] !== '1') { return new Map(); } @@ -273,7 +304,10 @@ function prepareSurfaceOutputs(): ReadonlyMap { const extension = extname(combinedOutput); const stem = basename(combinedOutput, extension); const outputs = new Map(); - for (const suite of suites) { + for (const suite of selectedSuites) { + if (suite === nodeIntegrationSuite) { + continue; + } const output = join(dirname(combinedOutput), `${stem}-${suite.id}${extension}`); rmSync(output, { force: true }); outputs.set(suite.id, output); diff --git a/scripts/test-integration.bat b/scripts/test-integration.bat index 08d17acd68281..4fdbd2ea99289 100644 --- a/scripts/test-integration.bat +++ b/scripts/test-integration.bat @@ -128,6 +128,10 @@ if defined RUN_GLOB ( call .\scripts\test.bat %* ) else if defined RUN_FILE ( call .\scripts\test.bat %* +) else if "%VSCODE_PARALLEL_NODE_INTEGRATION_TESTS%"=="1" ( + call node .\scripts\test-agent-host-e2e.ts --include-node-tests %* + if errorlevel 1 goto :failed + set VSCODE_SKIP_PRELAUNCH=1 ) else ( if "%VSCODE_SKIP_AGENT_HOST_E2E%"=="1" ( echo Skipping Agent Host E2E tests because no relevant files changed. diff --git a/scripts/test-integration.sh b/scripts/test-integration.sh index b9072ca039b0e..1eec95159a5f8 100755 --- a/scripts/test-integration.sh +++ b/scripts/test-integration.sh @@ -173,12 +173,16 @@ if [[ -z "$SUITE_FILTER" ]]; then echo "### node.js integration tests" echo if [[ -z "$RUN_GLOB" && -z "$RUN_FILE" ]]; then - if [[ "$VSCODE_SKIP_AGENT_HOST_E2E" == "1" ]]; then - echo "Skipping Agent Host E2E tests because no relevant files changed." + if [[ "$VSCODE_PARALLEL_NODE_INTEGRATION_TESTS" == "1" ]]; then + node ./scripts/test-agent-host-e2e.ts --include-node-tests "${EXTRA_ARGS[@]}" else - node ./scripts/test-agent-host-e2e.ts "${EXTRA_ARGS[@]}" + if [[ "$VSCODE_SKIP_AGENT_HOST_E2E" == "1" ]]; then + echo "Skipping Agent Host E2E tests because no relevant files changed." + else + node ./scripts/test-agent-host-e2e.ts "${EXTRA_ARGS[@]}" + fi + VSCODE_SKIP_PRELAUNCH=1 ./scripts/test.sh --runGlob "**/*.integrationTest.js" --excludeRunGlob "$AGENT_HOST_E2E_GLOB" "${EXTRA_ARGS[@]}" fi - VSCODE_SKIP_PRELAUNCH=1 ./scripts/test.sh --runGlob "**/*.integrationTest.js" --excludeRunGlob "$AGENT_HOST_E2E_GLOB" "${EXTRA_ARGS[@]}" else ./scripts/test.sh "${EXTRA_ARGS[@]}" fi diff --git a/src/vs/platform/agentHost/test/node/e2e/README.md b/src/vs/platform/agentHost/test/node/e2e/README.md index 6b61b556e784d..e24ae2767a93d 100644 --- a/src/vs/platform/agentHost/test/node/e2e/README.md +++ b/src/vs/platform/agentHost/test/node/e2e/README.md @@ -205,6 +205,8 @@ The complete-suite runner starts one test process per entrypoint and runs up to Pull request Electron jobs run the complete suite only when the changed files can affect the Agent Host, its shared platform dependencies, provider SDK versions, build infrastructure, or the E2E harness. The classification happens inside each already-allocated Electron runner so Linux, macOS, and Windows jobs remain parallel. When no relevant files changed, CI sets `VSCODE_SKIP_AGENT_HOST_E2E=1`; `test-integration.sh` and `test-integration.bat` then skip this suite while continuing with every other integration test. +GitHub pull request Electron jobs also set `VSCODE_PARALLEL_NODE_INTEGRATION_TESTS=1`. For an unfiltered node.js run, the integration scripts invoke the runner with `--include-node-tests`, queuing the remaining node.js integration tests behind the four E2E entrypoints. That group starts in the first available worker slot without increasing the four-worker cap, excludes the E2E entrypoints to avoid duplicate execution, and retains the original integration-test report name. The runner waits for every group and propagates failures before extension host tests begin. When E2E tests are unaffected, only the remaining node.js group runs. Standalone E2E runs, file-filtered invocations, and callers without this opt-in retain their existing behavior. + Provider availability: - **Copilot** (`copilotcli`) — always enabled (the CLI is a dev dependency).