Skip to content

Commit 12dcb6b

Browse files
amungerCopilot
amunger
and
Copilot
authored
agentHost: add assignment context to telemetry (#332602)
* agentHost: add runtime assignment context to telemetry Promote assignment contexts observed on forwarded Copilot runtime telemetry to Agent Host-wide experiment properties so subsequent events carry ExP attribution. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * agentHost: preserve VS Code assignment telemetry Keep the workbench TAS context on forwarded Copilot SDK events under a non-colliding property when the runtime assignment context owns abexp.assignmentcontext. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * agentHost: use workbench assignment context Make the forwarded workbench TAS value the sole source of abexp.assignmentcontext for Agent Host telemetry, while runtime notifications provide only secondary_assignment_context. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 64ae77d commit 12dcb6b

8 files changed

Lines changed: 159 additions & 103 deletions

File tree

‎src/vs/platform/agentHost/node/copilot/copilotAgent.ts‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ import { createCopilotCliEnvironment } from './copilotCliEnvironment.js';
7979
import { ICopilotSessionContext, projectFromCopilotContext } from './copilotGitProject.js';
8080
import { parsedPluginsEqual, toChildCustomizations } from './copilotPluginConverters.js';
8181
import { CopilotGitHubTelemetryForwarder } from './copilotGitHubTelemetryForwarder.js';
82+
import { CopilotSecondaryAssignmentContext } from './copilotSecondaryAssignmentContext.js';
8283
import { CopilotSessionLauncher, ContextSizeConfigKey, ThinkingLevelConfigKey, getCopilotContextTier, isCopilotReasoningEffort, resolveCopilotReasoningEffort, type CopilotSessionLaunchPlan, type IActiveClientSnapshot } from './copilotSessionLauncher.js';
8384
import { CopilotAgentStartupConfig } from './copilotAgentStartupConfig.js';
8485
import { ShellManager } from './copilotShellTools.js';
@@ -875,7 +876,7 @@ export class CopilotAgent extends Disposable implements IAgent {
875876
private readonly _plugins: PluginController;
876877
private readonly _sessionLauncher: CopilotSessionLauncher;
877878
private readonly _gitHubTelemetryForwarder: CopilotGitHubTelemetryForwarder;
878-
private _vscodeAssignmentContext: string | undefined;
879+
private readonly _secondaryAssignmentContext: CopilotSecondaryAssignmentContext;
879880
private readonly _githubTelemetryRouter: AgentHostGitHubTelemetryRouter | undefined;
880881
readonly onDidCustomizationsChange: Event<void>;
881882
/** Per-session active client state for tools + plugin snapshot tracking. */
@@ -916,7 +917,8 @@ export class CopilotAgent extends Disposable implements IAgent {
916917
this._plugins = this._register(this._instantiationService.createInstance(PluginController, () => this._ensureClient()));
917918
this._sessionLauncher = this._instantiationService.createInstance(CopilotSessionLauncher);
918919
this._configurationService.publishRootTransientValues?.({ [CopilotCliVSCodeAssignmentContextKey]: undefined });
919-
this._gitHubTelemetryForwarder = this._instantiationService.createInstance(CopilotGitHubTelemetryForwarder, () => this._restrictedTelemetryEnabled, () => this._vscodeAssignmentContext);
920+
this._gitHubTelemetryForwarder = this._instantiationService.createInstance(CopilotGitHubTelemetryForwarder, () => this._restrictedTelemetryEnabled);
921+
this._secondaryAssignmentContext = this._instantiationService.createInstance(CopilotSecondaryAssignmentContext);
920922
this._register(this._configurationService.onDidRootConfigChange(() => this._updateVSCodeAssignmentContext()));
921923
this._updateVSCodeAssignmentContext();
922924
this._slashCommandProvider = new CopilotSlashCommandProvider(() => this._ensureClient().then(c => c.rpc.commands.list().then(c => c.commands)), this._logService);
@@ -1070,15 +1072,10 @@ export class CopilotAgent extends Disposable implements IAgent {
10701072
);
10711073
}
10721074

1073-
/**
1074-
* A key absent from root config (e.g. dropped by a schema-filtered replace)
1075-
* keeps the last-known context sticky; an explicit empty-string dispatch
1076-
* from the workbench clears it.
1077-
*/
10781075
private _updateVSCodeAssignmentContext(): void {
10791076
const value = this._configurationService.getRootConfigValues?.()[CopilotCliVSCodeAssignmentContextKey];
10801077
if (typeof value === 'string') {
1081-
this._vscodeAssignmentContext = value || undefined;
1078+
this._telemetryService.setExperimentProperty('abexp.assignmentcontext', value);
10821079
}
10831080
}
10841081

@@ -1645,6 +1642,7 @@ export class CopilotAgent extends Disposable implements IAgent {
16451642
}
16461643

16471644
private async _routeGitHubTelemetry(notification: GitHubTelemetryNotification): Promise<void> {
1645+
this._secondaryAssignmentContext.update(notification);
16481646
const additionalProperties = { initiatorClientType: this._clientTypeForTelemetry(notification.sessionId) };
16491647
const router = this._githubTelemetryRouter;
16501648
if (!router?.isTarget(notification)) {

‎src/vs/platform/agentHost/node/copilot/copilotGitHubTelemetryForwarder.ts‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,6 @@ export class CopilotGitHubTelemetryForwarder {
204204

205205
constructor(
206206
private readonly _isRestrictedTelemetryEnabled: () => boolean,
207-
private readonly _getVSCodeAssignmentContext: () => string | undefined,
208207
@ITelemetryService private readonly _telemetryService: ITelemetryService,
209208
) { }
210209

@@ -235,14 +234,6 @@ export class CopilotGitHubTelemetryForwarder {
235234
}
236235
}
237236

238-
// VS Code's TAS assignment context, scoped to forwarded Copilot CLI
239-
// events only — deliberately not a telemetry-service-wide experiment
240-
// property, so Claude/Codex/host events stay unstamped.
241-
const assignmentContext = this._getVSCodeAssignmentContext();
242-
if (assignmentContext) {
243-
data['abexp.assignmentcontext'] = assignmentContext;
244-
}
245-
246237
if (event.features) {
247238
for (const [key, value] of Object.entries(event.features)) {
248239
if (value !== undefined) {

‎src/vs/platform/agentHost/node/copilot/copilotSecondaryAssignmentContext.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
* Licensed under the MIT License. See License.txt in the project root for license information.
4+
*--------------------------------------------------------------------------------------------*/
5+
6+
import type { GitHubTelemetryNotification } from '@github/copilot-sdk';
7+
import { isValidAssignmentContext } from '../../../telemetry/common/assignmentContext.js';
8+
import { ITelemetryService } from '../../../telemetry/common/telemetry.js';
9+
10+
const SECONDARY_ASSIGNMENT_CONTEXT_PROPERTY = 'secondary_assignment_context';
11+
12+
// __GDPR__COMMON__ "secondary_assignment_context" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Secondary experiment assignment context assigned by CAPI during Copilot model calls." }
13+
14+
export class CopilotSecondaryAssignmentContext {
15+
16+
private _value: string | undefined;
17+
18+
constructor(
19+
@ITelemetryService private readonly _telemetryService: ITelemetryService,
20+
) { }
21+
22+
update(notification: GitHubTelemetryNotification): void {
23+
const value = notification.event.properties.secondary_assignment_context;
24+
if (!value || value === this._value || !isValidAssignmentContext(value)) {
25+
return;
26+
}
27+
28+
this._telemetryService.setExperimentProperty(SECONDARY_ASSIGNMENT_CONTEXT_PROPERTY, value);
29+
this._value = value;
30+
}
31+
}

‎src/vs/platform/agentHost/test/node/copilotAgent.test.ts‎

Lines changed: 39 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1118,58 +1118,56 @@ suite('CopilotAgent', () => {
11181118
}
11191119
});
11201120

1121-
test('threads the assignment context from root config into forwarded CLI telemetry, sticky across a wipe', async () => {
1121+
test('promotes the forwarded secondary assignment context to a telemetry-wide property', async () => {
11221122
const client = new TestCopilotClient([]);
1123-
const telemetryService = new class extends RecordingTelemetryService {
1124-
override publicLog(eventName?: string, data?: unknown): void {
1125-
this.events.push({ eventName: eventName ?? '', data });
1126-
}
1127-
}();
1128-
const { agent, configurationService } = createTestAgentContext(disposables, { copilotClient: client, telemetryService });
1123+
const telemetryService = new RecordingTelemetryService();
1124+
const agent = createTestAgent(disposables, { copilotClient: client, telemetryService }) as TestableCopilotAgent;
11291125
try {
11301126
await agent.listChatsToMigrate();
11311127
const forward = getCreatedClientOptions(agent).at(-1)?.onGitHubTelemetry;
11321128
assert.ok(forward);
11331129

1134-
const notification = (sessionId: string): GitHubTelemetryNotification => ({
1135-
sessionId,
1130+
await forward({
1131+
sessionId: 'session',
11361132
restricted: false,
1137-
event: { kind: 'response.success', properties: {}, metrics: {} },
1133+
event: {
1134+
kind: 'response.success',
1135+
properties: { secondary_assignment_context: 'secondary:1' },
1136+
metrics: {},
1137+
exp_assignment_context: 'primary:1',
1138+
},
1139+
});
1140+
1141+
assert.deepStrictEqual(telemetryService.experimentProperties, {
1142+
secondary_assignment_context: 'secondary:1',
11381143
});
1144+
} finally {
1145+
await disposeAgent(agent);
1146+
}
1147+
});
1148+
1149+
test('promotes the VS Code assignment context from root config to telemetry, sticky across a wipe', async () => {
1150+
const client = new TestCopilotClient([]);
1151+
const telemetryService = new class extends RecordingTelemetryService {
1152+
readonly experimentPropertyUpdates: Array<{ name: string; value: string }> = [];
1153+
1154+
override setExperimentProperty(name?: string, value?: string): void {
1155+
super.setExperimentProperty(name, value);
1156+
this.experimentPropertyUpdates.push({ name: name ?? '', value: value ?? '' });
1157+
}
1158+
}();
1159+
const { agent, configurationService } = createTestAgentContext(disposables, { copilotClient: client, telemetryService });
1160+
try {
1161+
await agent.listChatsToMigrate();
1162+
11391163
configurationService.updateRootConfig({ [CopilotCliVSCodeAssignmentContextKey]: 'experiment:1' });
1140-
await forward(notification('set'));
11411164
configurationService.updateRootConfig({}, true);
1142-
await forward(notification('wiped-sticky'));
11431165
configurationService.updateRootConfig({ [CopilotCliVSCodeAssignmentContextKey]: '' });
1144-
await forward(notification('cleared'));
1145-
1146-
const expectedData = (sessionId: string, assignmentContext?: string) => ({
1147-
created_at: undefined,
1148-
model_call_id: undefined,
1149-
exp_assignment_context: undefined,
1150-
session_id: sessionId,
1151-
sdk_session_id: sessionId,
1152-
copilot_tracking_id: undefined,
1153-
kind: 'response.success',
1154-
restricted: false,
1155-
...(assignmentContext ? { 'abexp.assignmentcontext': assignmentContext } : {}),
1156-
});
1157-
const events = telemetryService.events.map(event => {
1158-
if (event.eventName !== 'agentHost.copilotClientStartup') {
1159-
return event;
1160-
}
1161-
const data = event.data as Record<string, unknown>;
1162-
return { ...event, data: { ...data, durationMs: typeof data.durationMs } };
1163-
});
1164-
assert.deepStrictEqual({ events, experimentProperties: telemetryService.experimentProperties }, {
1165-
events: [
1166-
{ eventName: 'agentHost.copilotClientStartup', data: { outcome: 'success', durationMs: 'number', attemptNumber: 1 } },
1167-
{ eventName: 'copilotSdk/response.success', data: expectedData('set', 'experiment:1') },
1168-
{ eventName: 'copilotSdk/response.success', data: expectedData('wiped-sticky', 'experiment:1') },
1169-
{ eventName: 'copilotSdk/response.success', data: expectedData('cleared') },
1170-
],
1171-
experimentProperties: {},
1172-
});
1166+
1167+
assert.deepStrictEqual(telemetryService.experimentPropertyUpdates, [
1168+
{ name: 'abexp.assignmentcontext', value: 'experiment:1' },
1169+
{ name: 'abexp.assignmentcontext', value: '' },
1170+
]);
11731171
} finally {
11741172
await disposeAgent(agent);
11751173
}

‎src/vs/platform/agentHost/test/node/copilotGitHubTelemetryForwarder.test.ts‎

Lines changed: 4 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ suite('CopilotGitHubTelemetryForwarder', () => {
4141

4242
test('forwards a standard event to VS Code telemetry', () => {
4343
const telemetryService = new TestTelemetryService();
44-
const forwarder = new CopilotGitHubTelemetryForwarder(() => false, () => undefined, telemetryService);
44+
const forwarder = new CopilotGitHubTelemetryForwarder(() => false, telemetryService);
4545

4646
forwarder.forward({
4747
sessionId: 'notification-session',
@@ -93,7 +93,7 @@ suite('CopilotGitHubTelemetryForwarder', () => {
9393
test('gates restricted events on the restricted telemetry option', () => {
9494
const telemetryService = new TestTelemetryService();
9595
let restrictedTelemetryEnabled = false;
96-
const forwarder = new CopilotGitHubTelemetryForwarder(() => restrictedTelemetryEnabled, () => undefined, telemetryService);
96+
const forwarder = new CopilotGitHubTelemetryForwarder(() => restrictedTelemetryEnabled, telemetryService);
9797
const notification: GitHubTelemetryNotification = {
9898
sessionId: 'session',
9999
restricted: true,
@@ -123,40 +123,9 @@ suite('CopilotGitHubTelemetryForwarder', () => {
123123
}]);
124124
});
125125

126-
test('stamps VS Code assignment context independently of the runtime context', () => {
127-
const telemetryService = new TestTelemetryService();
128-
const forwarder = new CopilotGitHubTelemetryForwarder(() => false, () => 'experiment:1;experiment:2', telemetryService);
129-
130-
forwarder.forward({
131-
sessionId: 'session',
132-
restricted: false,
133-
event: {
134-
kind: 'response.success',
135-
properties: {},
136-
metrics: {},
137-
exp_assignment_context: 'runtime-context',
138-
},
139-
});
140-
141-
assert.deepStrictEqual(telemetryService.events, [{
142-
eventName: 'copilotSdk/response.success',
143-
data: {
144-
created_at: undefined,
145-
model_call_id: undefined,
146-
exp_assignment_context: 'runtime-context',
147-
session_id: 'session',
148-
sdk_session_id: 'session',
149-
copilot_tracking_id: undefined,
150-
kind: 'response.success',
151-
restricted: false,
152-
'abexp.assignmentcontext': 'experiment:1;experiment:2',
153-
},
154-
}]);
155-
});
156-
157126
test('adds Agent Host turn correlation only to response events', () => {
158127
const telemetryService = new TestTelemetryService();
159-
const forwarder = new CopilotGitHubTelemetryForwarder(() => false, () => undefined, telemetryService);
128+
const forwarder = new CopilotGitHubTelemetryForwarder(() => false, telemetryService);
160129
const notification = (kind: string, properties: Record<string, string> = {}, metrics: Record<string, number> = {}): GitHubTelemetryNotification => ({
161130
sessionId: 'session',
162131
restricted: false,
@@ -185,7 +154,7 @@ suite('CopilotGitHubTelemetryForwarder', () => {
185154

186155
test('forwards tool_call_executed outcome and token-count columns', () => {
187156
const telemetryService = new TestTelemetryService();
188-
const forwarder = new CopilotGitHubTelemetryForwarder(() => false, () => undefined, telemetryService);
157+
const forwarder = new CopilotGitHubTelemetryForwarder(() => false, telemetryService);
189158

190159
forwarder.forward({
191160
sessionId: 'session',

‎src/vs/platform/agentHost/test/node/copilotSecondaryAssignmentContext.test.ts‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
* Licensed under the MIT License. See License.txt in the project root for license information.
4+
*--------------------------------------------------------------------------------------------*/
5+
6+
import type { GitHubTelemetryNotification } from '@github/copilot-sdk';
7+
import assert from 'assert';
8+
import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../base/test/common/utils.js';
9+
import { NullTelemetryServiceShape } from '../../../telemetry/common/telemetryUtils.js';
10+
import { CopilotSecondaryAssignmentContext } from '../../node/copilot/copilotSecondaryAssignmentContext.js';
11+
12+
class RecordingTelemetryService extends NullTelemetryServiceShape {
13+
readonly experimentProperties: Array<{ name: string; value: string }> = [];
14+
15+
override setExperimentProperty(name?: string, value?: string): void {
16+
this.experimentProperties.push({ name: name ?? '', value: value ?? '' });
17+
}
18+
}
19+
20+
suite('CopilotSecondaryAssignmentContext', () => {
21+
ensureNoDisposablesAreLeakedInTestSuite();
22+
23+
const notification = (secondaryAssignmentContext?: string): GitHubTelemetryNotification => ({
24+
sessionId: 'session',
25+
restricted: false,
26+
event: {
27+
kind: 'response.success',
28+
properties: { secondary_assignment_context: secondaryAssignmentContext },
29+
metrics: {},
30+
},
31+
});
32+
33+
test('sets the telemetry-wide secondary assignment context from forwarded notifications', () => {
34+
const telemetryService = new RecordingTelemetryService();
35+
const context = new CopilotSecondaryAssignmentContext(telemetryService);
36+
37+
context.update(notification('secondary:1'));
38+
context.update(notification('secondary:1'));
39+
context.update(notification('secondary:2'));
40+
41+
assert.deepStrictEqual(telemetryService.experimentProperties, [
42+
{ name: 'secondary_assignment_context', value: 'secondary:1' },
43+
{ name: 'secondary_assignment_context', value: 'secondary:2' },
44+
]);
45+
});
46+
47+
test('ignores a malformed secondary assignment context', () => {
48+
const telemetryService = new RecordingTelemetryService();
49+
const context = new CopilotSecondaryAssignmentContext(telemetryService);
50+
51+
context.update(notification('invalid'));
52+
context.update(notification('secondary:1'));
53+
54+
assert.deepStrictEqual(telemetryService.experimentProperties, [
55+
{ name: 'secondary_assignment_context', value: 'secondary:1' },
56+
]);
57+
});
58+
});

‎src/vs/platform/telemetry/common/assignmentContext.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
* Licensed under the MIT License. See License.txt in the project root for license information.
4+
*--------------------------------------------------------------------------------------------*/
5+
6+
const MAX_ASSIGNMENT_CONTEXT_LENGTH = 8 * 1024;
7+
const ASSIGNMENT_CONTEXT_ENTRY_PATTERN = /^[^:;\s\x00-\x1F\x7F]+:[^;\x00-\x1F\x7F]+$/;
8+
9+
/**
10+
* Validates an experiment assignment context before it is trusted onto telemetry events.
11+
*/
12+
export function isValidAssignmentContext(value: string): boolean {
13+
if (value.length === 0 || value.length > MAX_ASSIGNMENT_CONTEXT_LENGTH) {
14+
return false;
15+
}
16+
17+
const entries = value.endsWith(';') ? value.slice(0, -1).split(';') : value.split(';');
18+
return entries.length > 0 && entries.every(entry => ASSIGNMENT_CONTEXT_ENTRY_PATTERN.test(entry));
19+
}

‎src/vs/workbench/api/browser/mainThreadTelemetry.ts‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import { IConfigurationService } from '../../../platform/configuration/common/co
88
import { CommandsRegistry } from '../../../platform/commands/common/commands.js';
99
import { IEnvironmentService } from '../../../platform/environment/common/environment.js';
1010
import { IProductService } from '../../../platform/product/common/productService.js';
11+
import { isValidAssignmentContext } from '../../../platform/telemetry/common/assignmentContext.js';
1112
import { ClassifiedEvent, IGDPRProperty, OmitMetadata, StrictPropertyCheck } from '../../../platform/telemetry/common/gdprTypings.js';
1213
import { ITelemetryService, TelemetryLevel, TELEMETRY_OLD_SETTING_ID, TELEMETRY_SETTING_ID, ITelemetryData } from '../../../platform/telemetry/common/telemetry.js';
1314
import { supportsTelemetry } from '../../../platform/telemetry/common/telemetryUtils.js';
@@ -72,9 +73,6 @@ export const CAPI_ASSIGNMENT_CONTEXT_PROPERTY = 'capi.assignmentcontext';
7273
*/
7374
export const SET_CAPI_ASSIGNMENT_CONTEXT_COMMAND = '_telemetry.setCapiAssignmentContext';
7475

75-
const MAX_CAPI_ASSIGNMENT_CONTEXT_LENGTH = 8 * 1024;
76-
const CAPI_ASSIGNMENT_CONTEXT_ENTRY_PATTERN = /^[^:;\s\x00-\x1F\x7F]+:[^;\x00-\x1F\x7F]+$/;
77-
7876
/**
7977
* Validates a CAPI assignment-context string before it is trusted onto every
8078
* core telemetry event. Because {@link ITelemetryService.setExperimentProperty}
@@ -84,13 +82,7 @@ const CAPI_ASSIGNMENT_CONTEXT_ENTRY_PATTERN = /^[^:;\s\x00-\x1F\x7F]+:[^;\x00-\x
8482
* malformed input is rejected outright.
8583
*/
8684
export function isValidCapiAssignmentContext(value: string): boolean {
87-
if (value.length === 0 || value.length > MAX_CAPI_ASSIGNMENT_CONTEXT_LENGTH) {
88-
return false;
89-
}
90-
91-
// Tolerate a single trailing separator (`a:b;`) but nothing else empty.
92-
const entries = value.endsWith(';') ? value.slice(0, -1).split(';') : value.split(';');
93-
return entries.length > 0 && entries.every(entry => CAPI_ASSIGNMENT_CONTEXT_ENTRY_PATTERN.test(entry));
85+
return isValidAssignmentContext(value);
9486
}
9587

9688
CommandsRegistry.registerCommand(SET_CAPI_ASSIGNMENT_CONTEXT_COMMAND, function (accessor, value: string) {

0 commit comments

Comments
 (0)