Skip to content

Commit 6e6880c

Browse files
rwollCopilot
andcommitted
evals: attach an existing Agent Host session
Example: code-insiders --attach-to-evaluation-session \ 'remote-example-provider:/session-id' The private argument opens the exact session in an Agents window while the external driver retains turn and tool-approval ownership. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 23f5009 commit 6e6880c

18 files changed

Lines changed: 1188 additions & 16 deletions

File tree

src/vs/code/electron-main/app.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ import { ipcBrowserViewChannelName } from '../../platform/browserView/common/bro
4040
import { ipcBrowserViewGroupChannelName } from '../../platform/browserView/common/browserViewGroup.js';
4141
import { BrowserViewMainService, IBrowserViewMainService } from '../../platform/browserView/electron-main/browserViewMainService.js';
4242
import { BrowserViewGroupMainService, IBrowserViewGroupMainService } from '../../platform/browserView/electron-main/browserViewGroupMainService.js';
43-
import { NativeParsedArgs } from '../../platform/environment/common/argv.js';
43+
import { NativeParsedArgs, shouldOpenAgentsWindow } from '../../platform/environment/common/argv.js';
4444
import { IEnvironmentMainService } from '../../platform/environment/electron-main/environmentMainService.js';
4545
import { isLaunchedFromCli } from '../../platform/environment/node/argvHelper.js';
4646
import { getResolvedShellEnv } from '../../platform/shell/node/shellEnv.js';
@@ -1487,7 +1487,7 @@ export class CodeApplication extends Disposable {
14871487
const args = this.environmentMainService.args;
14881488

14891489
// Handle agents window first based on context
1490-
if (args['agents']) {
1490+
if (shouldOpenAgentsWindow(args)) {
14911491
return windowsMainService.openAgentsWindow({
14921492
context,
14931493
cli: args,

src/vs/platform/agentHost/test/node/agentSideEffects.test.ts

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6774,6 +6774,40 @@ suite('AgentSideEffects', () => {
67746774
assert.deepStrictEqual(sessionInputNeeded(), []);
67756775
});
67766776

6777+
test('approved client tool replaces confirmation with a distinct canonical execution request', () => {
6778+
setupSession();
6779+
startTurn('turn-1');
6780+
stateManager.dispatchServerAction(defaultChatUri, {
6781+
type: ActionType.ChatToolCallStart, turnId: 'turn-1',
6782+
toolCallId: 'tc-client', toolName: 'runTask', displayName: 'Run Task',
6783+
contributor: { kind: ToolCallContributorKind.Client, clientId: 'client-1' },
6784+
});
6785+
stateManager.dispatchServerAction(defaultChatUri, {
6786+
type: ActionType.ChatToolCallReady, turnId: 'turn-1',
6787+
toolCallId: 'tc-client', invocationMessage: 'Run task', confirmationTitle: 'Run task',
6788+
});
6789+
const confirmation = sessionInputNeeded()[0];
6790+
6791+
stateManager.dispatchServerAction(defaultChatUri, {
6792+
type: ActionType.ChatToolCallConfirmed, turnId: 'turn-1',
6793+
toolCallId: 'tc-client', approved: true, confirmed: ToolCallConfirmationReason.UserAction,
6794+
});
6795+
const execution = sessionInputNeeded()[0];
6796+
6797+
assert.deepStrictEqual({
6798+
confirmationId: confirmation?.id,
6799+
executionId: execution?.id,
6800+
kind: execution?.kind,
6801+
status: execution?.kind === SessionInputRequestKind.ToolClientExecution ? execution.toolCall.status : undefined,
6802+
}, {
6803+
confirmationId: `toolConfirmation:${defaultChatUri}:turn-1:tc-client`,
6804+
executionId: `toolClientExecution:${defaultChatUri}:turn-1:tc-client`,
6805+
kind: SessionInputRequestKind.ToolClientExecution,
6806+
status: ToolCallStatus.Running,
6807+
});
6808+
assert.notStrictEqual(execution?.id, confirmation?.id);
6809+
});
6810+
67776811
test('client tool execution is produced while running and removed once complete', () => {
67786812
setupSession();
67796813
startTurn('turn-1');

src/vs/platform/environment/common/argv.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@ export interface NativeParsedArgs {
129129
'file-write'?: boolean;
130130
'file-chmod'?: boolean;
131131
'enable-smoke-test-driver'?: boolean;
132+
'attach-to-evaluation-session'?: string;
132133
'skip-sessions-welcome'?: boolean;
133134
'remote'?: string;
134135
'force'?: boolean;
@@ -180,3 +181,7 @@ export interface NativeParsedArgs {
180181
'trace-startup-duration'?: string;
181182
'xdg-portal-required-version'?: string;
182183
}
184+
185+
export function shouldOpenAgentsWindow(args: NativeParsedArgs): boolean {
186+
return !!(args.agents || args['attach-to-evaluation-session']);
187+
}

src/vs/platform/environment/node/argv.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,7 @@ export const OPTIONS: OptionDescriptions<Required<NativeParsedArgs>> = {
180180
'export-default-keybindings': { type: 'string', allowEmptyValue: true },
181181
'install-source': { type: 'string' },
182182
'enable-smoke-test-driver': { type: 'boolean' },
183+
'attach-to-evaluation-session': { type: 'string' },
183184
'skip-sessions-welcome': { type: 'boolean' },
184185
'logExtensionHostCommunication': { type: 'boolean' },
185186
'skip-release-notes': { type: 'boolean' },
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
* Licensed under the MIT License. See License.txt in the project root for license information.
4+
*--------------------------------------------------------------------------------------------*/
5+
6+
import assert from 'assert';
7+
import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../base/test/common/utils.js';
8+
import { shouldOpenAgentsWindow } from '../../common/argv.js';
9+
10+
suite('Native argv routing', () => {
11+
ensureNoDisposablesAreLeakedInTestSuite();
12+
13+
test('evaluation attachment implies Agents window without changing default routing', () => {
14+
assert.strictEqual(shouldOpenAgentsWindow({ _: [] }), false);
15+
assert.strictEqual(shouldOpenAgentsWindow({ _: [], agents: true }), true);
16+
assert.strictEqual(shouldOpenAgentsWindow({ _: [], 'attach-to-evaluation-session': 'remote-host-copilot:/session' }), true);
17+
});
18+
});

src/vs/platform/launch/electron-main/launchMainService.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import { IProcessEnvironment, isMacintosh } from '../../../base/common/platform.
99
import { URI } from '../../../base/common/uri.js';
1010
import { whenDeleted } from '../../../base/node/pfs.js';
1111
import { IConfigurationService } from '../../configuration/common/configuration.js';
12-
import { NativeParsedArgs } from '../../environment/common/argv.js';
12+
import { NativeParsedArgs, shouldOpenAgentsWindow } from '../../environment/common/argv.js';
1313
import { isLaunchedFromCli } from '../../environment/node/argvHelper.js';
1414
import { createDecorator } from '../../instantiation/common/instantiation.js';
1515
import { ILogService } from '../../log/common/log.js';
@@ -144,7 +144,7 @@ export class LaunchMainService implements ILaunchMainService {
144144
}
145145

146146
// Agents window
147-
else if (args['agents']) {
147+
else if (shouldOpenAgentsWindow(args)) {
148148
usedWindows = await this.windowsMainService.openAgentsWindow(baseConfig);
149149
}
150150

src/vs/sessions/contrib/providers/agentHost/browser/baseAgentHostSessionsProvider.ts

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3031,6 +3031,9 @@ export abstract class BaseAgentHostSessionsProvider extends Disposable implement
30313031
/** Maps a project URI from the session summary to a local URI. Default identity; remote overrides for `file:` paths. */
30323032
protected mapProjectUri(uri: URI): URI { return uri; }
30333033

3034+
/** Optional barrier before publishing the active-client snapshot for a cached session. */
3035+
protected _prepareActiveClientPublication(_cached: AgentHostSessionAdapter, _token: CancellationToken): Promise<boolean> | undefined { return undefined; }
3036+
30343037
// -- Session listing ------------------------------------------------------
30353038

30363039
getSessionTypes(_repositoryUri: URI): ISessionType[] {
@@ -3087,6 +3090,24 @@ export abstract class BaseAgentHostSessionsProvider extends Disposable implement
30873090
return;
30883091
}
30893092

3093+
const preparation = this._prepareActiveClientPublication(cached, token);
3094+
if (preparation !== undefined) {
3095+
if (!await preparation) {
3096+
return;
3097+
}
3098+
const activeSession = this._sessionsService.activeSession.get();
3099+
if (
3100+
token.isCancellationRequested ||
3101+
scope !== this._activeSessionScope.value ||
3102+
this.connection !== connection ||
3103+
this._sessionCache.get(rawId) !== cached ||
3104+
activeSession?.providerId !== this.id ||
3105+
activeSession.sessionId !== activeSessionId
3106+
) {
3107+
return;
3108+
}
3109+
}
3110+
30903111
const activeClient = scope.activeClient(connection.clientId).get();
30913112
const existing = this._lastSessionStates.get(cached.sessionId)?.activeClients.find(client => client.clientId === activeClient.clientId);
30923113
if (equals(existing, activeClient)) {
Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
* Licensed under the MIT License. See License.txt in the project root for license information.
4+
*--------------------------------------------------------------------------------------------*/
5+
6+
import { raceCancellationError } from '../../../../../base/common/async.js';
7+
import { CancellationToken, CancellationTokenSource } from '../../../../../base/common/cancellation.js';
8+
import { CancellationError, isCancellationError } from '../../../../../base/common/errors.js';
9+
import { Event } from '../../../../../base/common/event.js';
10+
import { Disposable, DisposableStore, IDisposable, MutableDisposable, toDisposable } from '../../../../../base/common/lifecycle.js';
11+
import { autorun, IObservable, waitForState } from '../../../../../base/common/observable.js';
12+
import { URI } from '../../../../../base/common/uri.js';
13+
import { IAgentHostConnectionsService } from '../../../../../platform/agentHost/common/agentHostConnectionsService.js';
14+
import { isRemoteAgentHostSessionType } from '../../../../../platform/agentHost/common/agentHostSessionType.js';
15+
import { IEvaluationSessionAttachment, IEvaluationSessionAttachmentService, IEvaluationSessionIdentity } from '../../../../../workbench/contrib/chat/browser/agentSessions/agentHost/evaluationSessionAttachmentService.js';
16+
import { ISessionsService } from '../../../../services/sessions/browser/sessionsService.js';
17+
import { ISession } from '../../../../services/sessions/common/session.js';
18+
import { ISessionsManagementService } from '../../../../services/sessions/common/sessionsManagement.js';
19+
20+
export interface IEvaluationSessionAttachmentStartupServices {
21+
readonly sessionsManagementService: Pick<ISessionsManagementService, 'getSession' | 'onDidChangeSessions'>;
22+
readonly sessionsService: Pick<ISessionsService, 'canOpenSession' | 'openSession'> & { readonly activeSession: IObservable<ISession | undefined>; readonly initialRestoreComplete: IObservable<boolean> };
23+
readonly connectionsService: Pick<IAgentHostConnectionsService, 'connections' | 'resolveSessionResource'>;
24+
readonly attachmentService: IEvaluationSessionAttachmentService;
25+
readonly whenWorkbenchRestored: Promise<void>;
26+
readonly reconcileClientToolSets: () => void;
27+
}
28+
export function parseEvaluationSessionResource(value: string): URI {
29+
const resource = URI.parse(value, true);
30+
const id = resource.path.substring(1);
31+
if (!isRemoteAgentHostSessionType(resource.scheme) || resource.authority || !id
32+
|| resource.path !== `/${id}` || id.includes('/') || resource.query || resource.fragment
33+
|| resource.toString() !== value) {
34+
throw new Error('The evaluation session URI must be a canonical remote session URI.');
35+
}
36+
return resource;
37+
}
38+
export async function startEvaluationSessionAttachment(value: string | undefined, getServices: () => IEvaluationSessionAttachmentStartupServices, token: CancellationToken, onFailure: (error: Error) => void = () => { }): Promise<IDisposable | undefined> {
39+
if (value === undefined) {
40+
return undefined;
41+
}
42+
const resource = parseEvaluationSessionResource(value);
43+
const services = getServices();
44+
let attachment: IEvaluationSessionAttachment | undefined;
45+
try {
46+
await waitForState(services.sessionsService.initialRestoreComplete, complete => complete, undefined, token);
47+
const session = await waitForExactSession(services.sessionsManagementService, resource, token);
48+
const identity = resolveEvaluationSessionIdentity(resource, session, services.connectionsService);
49+
if (!await raceCancellationError(services.sessionsService.canOpenSession(session), token)) {
50+
throw new Error('The evaluation session workspace is not trusted.');
51+
}
52+
attachment = services.attachmentService.attach(identity);
53+
await raceCancellationError(services.sessionsService.openSession(resource), token);
54+
await waitForState(
55+
services.sessionsService.activeSession,
56+
active => active?.resource.toString() === resource.toString(),
57+
active => active && active.resource.toString() !== resource.toString()
58+
? new Error('Opening the evaluation session activated a different session.')
59+
: false,
60+
token,
61+
);
62+
await raceCancellationError(services.whenWorkbenchRestored, token);
63+
if (services.sessionsService.activeSession.get()?.resource.toString() !== resource.toString()) {
64+
throw new Error('The active evaluation session changed before publication was ready.');
65+
}
66+
services.reconcileClientToolSets();
67+
attachment.markActiveClientPublicationReady();
68+
const retained = new DisposableStore();
69+
retained.add(attachment);
70+
attachment = undefined;
71+
retained.add(autorun(reader => {
72+
if (services.sessionsService.activeSession.read(reader)?.resource.toString() !== resource.toString()) {
73+
retained.dispose();
74+
onFailure(new Error('The active evaluation session changed.'));
75+
}
76+
}));
77+
// The SDK supplies the workspace working directory; the driver waits for active-client inventory before sending a turn.
78+
return retained;
79+
} catch (error) {
80+
attachment?.dispose();
81+
throw error;
82+
}
83+
}
84+
async function waitForExactSession(service: Pick<ISessionsManagementService, 'getSession' | 'onDidChangeSessions'>, resource: URI, token: CancellationToken): Promise<ISession> {
85+
for (;;) {
86+
if (token.isCancellationRequested) {
87+
throw new CancellationError();
88+
}
89+
const session = service.getSession(resource);
90+
if (session?.resource.toString() === resource.toString()) {
91+
return session;
92+
}
93+
const change = Event.toPromise(service.onDidChangeSessions);
94+
try {
95+
await raceCancellationError(change, token);
96+
} finally {
97+
change.cancel();
98+
}
99+
}
100+
}
101+
export function resolveEvaluationSessionIdentity(resource: URI, session: ISession, connectionsService: Pick<IAgentHostConnectionsService, 'connections' | 'resolveSessionResource'>): IEvaluationSessionIdentity {
102+
const resolution = connectionsService.resolveSessionResource(resource);
103+
const connection = connectionsService.connections.find(candidate => !candidate.isAmbient && candidate.connection === resolution?.connection);
104+
const backendSession = (session as ISession & { readonly backendUri?: URI }).backendUri;
105+
if (!resolution || !connection || session.resource.toString() !== resource.toString()
106+
|| !URI.isUri(backendSession) || backendSession.authority || backendSession.path !== resource.path
107+
|| backendSession.query || backendSession.fragment) {
108+
throw new Error('The evaluation session is not backed by the exact connected remote agent host.');
109+
}
110+
return { connectionAuthority: connection.authority, backendSession };
111+
}
112+
export class EvaluationSessionAttachmentLifecycle extends Disposable {
113+
private readonly _attachment = this._register(new MutableDisposable<IDisposable>());
114+
constructor(value: string | undefined, getServices: () => IEvaluationSessionAttachmentStartupServices, onFailure: (error: Error) => void) {
115+
super();
116+
if (value === undefined) {
117+
return;
118+
}
119+
const cancellation = new CancellationTokenSource();
120+
this._register(toDisposable(() => cancellation.dispose(true)));
121+
void startEvaluationSessionAttachment(value, getServices, cancellation.token, error => {
122+
this._attachment.clear();
123+
onFailure(error);
124+
}).then(attachment => {
125+
if (cancellation.token.isCancellationRequested) {
126+
attachment?.dispose();
127+
} else {
128+
this._attachment.value = attachment;
129+
}
130+
}).catch(error => {
131+
if (!isCancellationError(error) && !cancellation.token.isCancellationRequested) {
132+
onFailure(error);
133+
}
134+
});
135+
}
136+
}

src/vs/sessions/contrib/providers/remoteAgentHost/browser/remoteAgentHostSessionsProvider.ts

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,17 +30,19 @@ import { ILogService } from '../../../../../platform/log/common/log.js';
3030
import { INotificationService } from '../../../../../platform/notification/common/notification.js';
3131
import { IStorageService } from '../../../../../platform/storage/common/storage.js';
3232
import { IAgentHostActiveClientService } from '../../../../../workbench/contrib/chat/browser/agentSessions/agentHost/agentHostActiveClientService.js';
33+
import { EvaluationSessionActiveClientPublicationState, IEvaluationSessionAttachmentService } from '../../../../../workbench/contrib/chat/browser/agentSessions/agentHost/evaluationSessionAttachmentService.js';
3334
import { IChatWidgetService } from '../../../../../workbench/contrib/chat/browser/chat.js';
3435
import { IChatService } from '../../../../../workbench/contrib/chat/common/chatService/chatService.js';
3536
import { IChatSessionsService } from '../../../../../workbench/contrib/chat/common/chatSessionsService.js';
3637
import { ILanguageModelsService } from '../../../../../workbench/contrib/chat/common/languageModels.js';
38+
import { ILanguageModelToolsService } from '../../../../../workbench/contrib/chat/common/tools/languageModelToolsService.js';
3739
import { ResourceLabelHomeStore } from '../../../../../workbench/services/label/common/resourceLabelHomeStore.js';
3840
import { IAgentHostConnectProgress, IAgentHostGroup } from '../../../../common/agentHostSessionsProvider.js';
3941
import { buildAgentHostSessionWorkspace, readBranchProtectionPatterns } from '../../../../common/agentHostSessionWorkspace.js';
4042
import { IGitHubInfo, ISession, ISessionType, ISessionWorkspace, ISessionWorkspaceBrowseAction, SESSION_WORKSPACE_GROUP_REMOTE } from '../../../../services/sessions/common/session.js';
4143
import { ISessionsService } from '../../../../services/sessions/browser/sessionsService.js';
4244
import { IGitHubService } from '../../../github/browser/githubService.js';
43-
import { BaseAgentHostSessionsProvider } from '../../agentHost/browser/baseAgentHostSessionsProvider.js';
45+
import { AgentHostSessionAdapter, BaseAgentHostSessionsProvider } from '../../agentHost/browser/baseAgentHostSessionsProvider.js';
4446
import { ReconnectableAgentHostAutomationStore } from '../../agentHost/browser/reconnectableAgentHostAutomationStore.js';
4547
import type { ISessionsProviderAutomations } from '../../../../services/sessions/common/sessionsProvider.js';
4648
import { AutomationStore } from '../../../automations/browser/automationService.js';
@@ -210,6 +212,8 @@ export class RemoteAgentHostSessionsProvider extends BaseAgentHostSessionsProvid
210212
@IAgentHostActiveClientService activeClientService: IAgentHostActiveClientService,
211213
@IDialogService dialogService: IDialogService,
212214
@IWorkspaceTrustManagementService workspaceTrustManagementService: IWorkspaceTrustManagementService,
215+
@IEvaluationSessionAttachmentService private readonly _evaluationSessionAttachmentService: IEvaluationSessionAttachmentService,
216+
@ILanguageModelToolsService private readonly _languageModelToolsService: ILanguageModelToolsService,
213217
) {
214218
super(chatSessionsService, chatService, chatWidgetService, languageModelsService, _configurationService, logService, gitHubService, instantiationService, sessionsService, activeClientService, storageService, dialogService, workspaceTrustManagementService);
215219
this._resourceLabelHomes = this._register(instantiationService.createInstance(ResourceLabelHomeStore));
@@ -272,6 +276,22 @@ export class RemoteAgentHostSessionsProvider extends BaseAgentHostSessionsProvid
272276

273277
// -- BaseAgentHostSessionsProvider hooks ---------------------------------
274278

279+
protected override _prepareActiveClientPublication(cached: AgentHostSessionAdapter, token: CancellationToken): Promise<boolean> | undefined {
280+
const identity = { connectionAuthority: this._connectionAuthority, backendSession: cached.backendUri };
281+
const readiness = this._evaluationSessionAttachmentService.waitForActiveClientPublicationReady(identity, token);
282+
if (readiness === undefined) {
283+
return undefined;
284+
}
285+
return readiness.then(ready => {
286+
if (!ready || token.isCancellationRequested
287+
|| this._evaluationSessionAttachmentService.getActiveClientPublicationState(identity) !== EvaluationSessionActiveClientPublicationState.Ready) {
288+
return false;
289+
}
290+
this._languageModelToolsService.flushToolUpdates();
291+
return true;
292+
});
293+
}
294+
275295
protected get connection(): IAgentConnection | undefined { return this._connection; }
276296

277297
protected get authenticationPending(): IObservable<boolean> { return this._authenticationPending; }

0 commit comments

Comments
 (0)