diff --git a/extensions/copilot/package.json b/extensions/copilot/package.json index cc394aedb47b1..576eb3fcab982 100644 --- a/extensions/copilot/package.json +++ b/extensions/copilot/package.json @@ -81,6 +81,7 @@ "onStartupFinished", "onLanguageModelChat:copilot", "onUri", + "onCommand:_github.copilot.chat.reportModelFeedbackSurvey", "onFileSystem:ccreq", "onFileSystem:ccsettings" ], diff --git a/extensions/copilot/src/extension/extension/vscode/contributions.ts b/extensions/copilot/src/extension/extension/vscode/contributions.ts index 8bc63ff84e017..4a021168ce635 100644 --- a/extensions/copilot/src/extension/extension/vscode/contributions.ts +++ b/extensions/copilot/src/extension/extension/vscode/contributions.ts @@ -8,6 +8,7 @@ import { asContributionFactory, IExtensionContributionFactory } from '../../comm import * as contextContribution from '../../context/vscode/context.contribution'; import { LifecycleTelemetryContrib } from '../../telemetry/common/lifecycleTelemetryContrib'; import { GithubTelemetryForwardingContrib } from '../../telemetry/vscode/githubTelemetryForwardingContrib'; +import { ChatModelFeedbackSurveyForwardingContrib } from '../../telemetry/vscode/chatModelFeedbackSurveyForwardingContrib'; // ############################################################################### // ### ### @@ -21,6 +22,7 @@ const vscodeContributions: IExtensionContributionFactory[] = [ asContributionFactory(LifecycleTelemetryContrib), asContributionFactory(NesActivationTelemetryContribution), asContributionFactory(GithubTelemetryForwardingContrib), + asContributionFactory(ChatModelFeedbackSurveyForwardingContrib), contextContribution, ]; diff --git a/extensions/copilot/src/extension/telemetry/vscode/chatModelFeedbackSurveyForwardingContrib.ts b/extensions/copilot/src/extension/telemetry/vscode/chatModelFeedbackSurveyForwardingContrib.ts new file mode 100644 index 0000000000000..87dcfc7e06969 --- /dev/null +++ b/extensions/copilot/src/extension/telemetry/vscode/chatModelFeedbackSurveyForwardingContrib.ts @@ -0,0 +1,119 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { commands } from 'vscode'; +import { ITelemetryService, TelemetryEventMeasurements, TelemetryEventProperties } from '../../../platform/telemetry/common/telemetry'; +import { Disposable } from '../../../util/vs/base/common/lifecycle'; +import { IExtensionContribution } from '../../common/contributions'; + +/** + * Command the workbench invokes to report inline model feedback survey results. + * + * Keep in sync with `CHAT_MODEL_FEEDBACK_SURVEY_TELEMETRY_COMMAND_ID` in + * `src/vs/workbench/contrib/chat/common/feedbackSurvey/chatModelFeedbackSurveyTelemetry.ts`. + */ +const REPORT_SURVEY_COMMAND_ID = '_github.copilot.chat.reportModelFeedbackSurvey'; + +const TELEMETRY_EVENT_NAME = 'vscode.chatModelFeedbackSurvey'; + +const KNOWN_EVENT_KINDS: readonly string[] = ['shown', 'opened', 'step', 'submitted', 'dismissed']; + +/** Mirrors the workbench payload in `chatModelFeedbackSurveyTelemetry.ts`. */ +interface IChatModelFeedbackSurveyTelemetryEvent { + readonly kind: 'shown' | 'opened' | 'step' | 'submitted' | 'dismissed'; + readonly surveyId: string; + readonly surveyInstanceId: string; + readonly stepCount: number; + readonly trigger?: 'manual' | 'chance' | 'modelSwitchedAway'; + readonly stepId?: string; + readonly stepIndex?: number; + readonly answerId?: string; + readonly comment?: string; + readonly modelId?: string; + readonly resolvedModelId?: string; + readonly modeId?: string; + readonly harness?: string; + readonly sessionType?: string; + readonly requestId: string; +} + +/** + * Forwards inline model feedback survey results to GitHub restricted telemetry. + * + * The workbench cannot reach that endpoint, so it hands each result over as a command, which + * also activates this extension so early results are not lost. Every event is sent enhanced, + * including the ones carrying no answer, so the whole funnel shares one consent boundary. When + * the user has not opted in to restricted telemetry the send is a no op. + * + * Core already has a sender for the same table in + * `src/vs/platform/agentHost/node/agentHostRestrictedTelemetry.ts`, using the same enhanced + * ingestion key. It is node layer and lives in the agent host process with no channel to the + * renderer, so it cannot serve workbench events today. Exposing it to the renderer would let + * this contribution and `GithubTelemetryForwardingContrib` both go away. + */ +export class ChatModelFeedbackSurveyForwardingContrib extends Disposable implements IExtensionContribution { + + constructor( + @ITelemetryService private readonly _telemetryService: ITelemetryService, + ) { + super(); + + this._register(commands.registerCommand(REPORT_SURVEY_COMMAND_ID, (event: unknown) => { + this._report(event); + })); + } + + private _report(event: unknown): void { + if (!isSurveyEvent(event)) { + return; + } + + const properties: Record = { + kind: event.kind, + surveyId: event.surveyId, + surveyInstanceId: event.surveyInstanceId, + }; + const measurements: Record = { + stepCount: event.stepCount, + }; + + addProperty(properties, 'trigger', event.trigger); + addProperty(properties, 'stepId', event.stepId); + addProperty(properties, 'answerId', event.answerId); + addProperty(properties, 'comment', event.comment); + addProperty(properties, 'modelId', event.modelId); + addProperty(properties, 'resolvedModelId', event.resolvedModelId); + addProperty(properties, 'modeId', event.modeId); + addProperty(properties, 'harness', event.harness); + addProperty(properties, 'sessionType', event.sessionType); + addProperty(properties, 'requestId', event.requestId); + + if (typeof event.stepIndex === 'number') { + measurements.stepIndex = event.stepIndex; + } + + const telemetryProperties: TelemetryEventProperties = properties; + const telemetryMeasurements: TelemetryEventMeasurements = measurements; + this._telemetryService.sendEnhancedGHTelemetryEvent(TELEMETRY_EVENT_NAME, telemetryProperties, telemetryMeasurements); + } +} + +function isSurveyEvent(event: unknown): event is IChatModelFeedbackSurveyTelemetryEvent { + if (typeof event !== 'object' || event === null) { + return false; + } + const candidate = event as IChatModelFeedbackSurveyTelemetryEvent; + return KNOWN_EVENT_KINDS.includes(candidate.kind) + && typeof candidate.surveyId === 'string' + && typeof candidate.surveyInstanceId === 'string' + && typeof candidate.stepCount === 'number' + && typeof candidate.requestId === 'string'; +} + +function addProperty(properties: Record, key: string, value: string | undefined): void { + if (typeof value === 'string' && value.length > 0) { + properties[key] = value; + } +} diff --git a/src/vs/base/browser/ui/toolbar/toolbar.ts b/src/vs/base/browser/ui/toolbar/toolbar.ts index 22430f76741a9..63c37073b00aa 100644 --- a/src/vs/base/browser/ui/toolbar/toolbar.ts +++ b/src/vs/base/browser/ui/toolbar/toolbar.ts @@ -213,8 +213,9 @@ export class ToolBar extends Disposable { return this.element; } - focus(): void { - this.actionBar.focus(); + /** Focuses the item at `index`, or the first item when no index is given. */ + focus(index?: number): void { + this.actionBar.focus(index); } getItemsWidth(): number { diff --git a/src/vs/workbench/contrib/chat/browser/actions/chatModelFeedbackSurveyActions.ts b/src/vs/workbench/contrib/chat/browser/actions/chatModelFeedbackSurveyActions.ts new file mode 100644 index 0000000000000..1263c832e4c38 --- /dev/null +++ b/src/vs/workbench/contrib/chat/browser/actions/chatModelFeedbackSurveyActions.ts @@ -0,0 +1,163 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as dom from '../../../../../base/browser/dom.js'; +import { Codicon } from '../../../../../base/common/codicons.js'; +import { Disposable, markAsSingleton } from '../../../../../base/common/lifecycle.js'; +import { ThemeIcon } from '../../../../../base/common/themables.js'; +import { ServicesAccessor } from '../../../../../editor/browser/editorExtensions.js'; +import { localize2 } from '../../../../../nls.js'; +import { IAction } from '../../../../../base/common/actions.js'; +import { IActionViewItemService } from '../../../../../platform/actions/browser/actionViewItemService.js'; +import { MenuEntryActionViewItem } from '../../../../../platform/actions/browser/menuEntryActionViewItem.js'; +import { Action2, MenuId, MenuItemAction, registerAction2 } from '../../../../../platform/actions/common/actions.js'; +import { ContextKeyExpr } from '../../../../../platform/contextkey/common/contextkey.js'; +import { IInstantiationService } from '../../../../../platform/instantiation/common/instantiation.js'; +import { IWorkbenchContribution } from '../../../../common/contributions.js'; +import { ChatContextKeys } from '../../common/actions/chatContextKeys.js'; +import { isResponseVM } from '../../common/model/chatViewModel.js'; +import { IChatModelFeedbackSurveyService } from '../feedbackSurvey/chatModelFeedbackSurveyService.js'; +import '../feedbackSurvey/media/chatModelFeedbackSurvey.css'; +import { CHAT_CATEGORY } from './chatActions.js'; + +export const ChatModelFeedbackSurveyActionId = 'workbench.action.chat.openModelFeedbackSurvey'; + +const thumbsUpIconClasses = ThemeIcon.asClassNameArray(Codicon.thumbsup); +const thumbsDownIconClasses = ThemeIcon.asClassNameArray(Codicon.thumbsdown); + +/** + * The combined thumbs up and down control that stands in for the helpful and unhelpful actions + * while a survey applies. It is neutral and opens the survey rather than recording a vote. + */ +class ChatModelFeedbackSurveyActionViewItem extends MenuEntryActionViewItem { + + override render(container: HTMLElement): void { + super.render(container); + + if (!this.element || !this.label) { + return; + } + + // The label is the focusable anchor that carries the accessible name, so the styling and + // the icons hang off it rather than off the outer list item. + this.label.classList.add('chat-feedback-survey-pill'); + this.resetLabel(); + + const icons = dom.append(this.label, dom.$('.chat-feedback-survey-pill-icons')); + icons.setAttribute('aria-hidden', 'true'); + for (const iconClasses of [thumbsUpIconClasses, thumbsDownIconClasses]) { + const icon = dom.append(icons, dom.$('.chat-feedback-survey-pill-icon')); + icon.classList.add(...iconClasses); + } + } + + protected override updateClass(): void { + super.updateClass(); + this.resetLabel(); + } + + /** + * The control discloses a panel rather than holding a pressed state, so it reports + * `aria-expanded` instead of the `aria-pressed` the base item would apply. + */ + protected override updateChecked(): void { + super.updateChecked(); + this.label?.removeAttribute('aria-pressed'); + this.label?.setAttribute('aria-expanded', String(!!this.action.checked)); + } + + /** + * The base item paints one icon onto the label, so that has to be cleared before drawing two. + * The label keeps its `aria-label` and only the icons are hidden from screen readers. + */ + private resetLabel(): void { + if (!this.label) { + return; + } + this.label.classList.remove('icon', ...thumbsUpIconClasses, ...thumbsDownIconClasses); + this.label.style.backgroundImage = ''; + this.label.textContent = ''; + } +} + +export class ChatModelFeedbackSurveyActionRendering extends Disposable implements IWorkbenchContribution { + + static readonly ID = 'chat.modelFeedbackSurveyActionRendering'; + + constructor( + @IActionViewItemService actionViewItemService: IActionViewItemService, + @IInstantiationService instantiationService: IInstantiationService, + ) { + super(); + + const disposable = this._register(actionViewItemService.register(MenuId.ChatMessageFooter, ChatModelFeedbackSurveyActionId, (action, options) => { + if (!(action instanceof MenuItemAction)) { + return undefined; + } + return instantiationService.createInstance(ChatModelFeedbackSurveyActionViewItem, action, options); + })); + + markAsSingleton(disposable); + } +} + +/** The part of a toolbar needed to find and focus one of its actions. */ +export interface IFeedbackSurveyToolBar { + getItemsLength(): number; + getItemAction(index: number): IAction | undefined; + focus(index?: number): void; +} + +/** + * Puts focus on the feedback control in `toolbar`, used when the survey panel it opened is torn + * down. Falls back to the toolbar itself when the control is not currently shown. + */ +export function focusChatModelFeedbackSurveyAction(toolbar: IFeedbackSurveyToolBar): void { + for (let i = 0; i < toolbar.getItemsLength(); i++) { + if (toolbar.getItemAction(i)?.id === ChatModelFeedbackSurveyActionId) { + toolbar.focus(i); + return; + } + } + toolbar.focus(); +} + +export function registerChatModelFeedbackSurveyActions(): void { + registerAction2(class OpenModelFeedbackSurveyAction extends Action2 { + constructor() { + super({ + id: ChatModelFeedbackSurveyActionId, + title: localize2('chat.feedbackSurvey.open.label', "Give Feedback"), + f1: false, + category: CHAT_CATEGORY, + icon: Codicon.thumbsup, + toggled: ChatContextKeys.responseFeedbackSurveyOpen, + menu: [{ + id: MenuId.ChatMessageFooter, + group: 'navigation', + order: 2, + // The survey service checks these too, so a shown report is never sent for a + // control that cannot render. This drops the vote actions' + // `lockedToCodingAgent.negate()` because every agent host session is locked to + // its agent, which would make the `harnesses` selector unreachable. + when: ContextKeyExpr.and( + ChatContextKeys.responseHasFeedbackSurvey, + ChatContextKeys.isResponse, + ChatContextKeys.responseHasError.negate(), + ContextKeyExpr.has('config.telemetry.feedback.enabled'), + ), + }], + }); + } + + run(accessor: ServicesAccessor, ...args: unknown[]): void { + const item = args[0]; + if (!isResponseVM(item)) { + return; + } + accessor.get(IChatModelFeedbackSurveyService).toggle(item); + } + }); +} diff --git a/src/vs/workbench/contrib/chat/browser/actions/chatTitleActions.ts b/src/vs/workbench/contrib/chat/browser/actions/chatTitleActions.ts index 215aabfbeed2c..f1b9f230a3997 100644 --- a/src/vs/workbench/contrib/chat/browser/actions/chatTitleActions.ts +++ b/src/vs/workbench/contrib/chat/browser/actions/chatTitleActions.ts @@ -45,7 +45,7 @@ export function registerChatTitleActions() { id: MenuId.ChatMessageFooter, group: 'navigation', order: 2, - when: ContextKeyExpr.and(ChatContextKeys.extensionParticipantRegistered, ChatContextKeys.isResponse, ChatContextKeys.responseHasError.negate(), ContextKeyExpr.has(enableFeedbackConfig), ChatContextKeys.lockedToCodingAgent.negate()) + when: ContextKeyExpr.and(ChatContextKeys.extensionParticipantRegistered, ChatContextKeys.isResponse, ChatContextKeys.responseHasError.negate(), ContextKeyExpr.has(enableFeedbackConfig), ChatContextKeys.lockedToCodingAgent.negate(), ChatContextKeys.responseHasFeedbackSurvey.negate()) }, { id: MENU_INLINE_CHAT_WIDGET_SECONDARY, group: 'navigation', @@ -90,7 +90,7 @@ export function registerChatTitleActions() { id: MenuId.ChatMessageFooter, group: 'navigation', order: 3, - when: ContextKeyExpr.and(ChatContextKeys.extensionParticipantRegistered, ChatContextKeys.isResponse, ContextKeyExpr.has(enableFeedbackConfig), ChatContextKeys.lockedToCodingAgent.negate()) + when: ContextKeyExpr.and(ChatContextKeys.extensionParticipantRegistered, ChatContextKeys.isResponse, ContextKeyExpr.has(enableFeedbackConfig), ChatContextKeys.lockedToCodingAgent.negate(), ChatContextKeys.responseHasFeedbackSurvey.negate()) }, { id: MENU_INLINE_CHAT_WIDGET_SECONDARY, group: 'navigation', diff --git a/src/vs/workbench/contrib/chat/browser/chat.shared.contribution.ts b/src/vs/workbench/contrib/chat/browser/chat.shared.contribution.ts index 50f8e61d9ebbd..75d1998716f01 100644 --- a/src/vs/workbench/contrib/chat/browser/chat.shared.contribution.ts +++ b/src/vs/workbench/contrib/chat/browser/chat.shared.contribution.ts @@ -97,6 +97,9 @@ import { CodeBlockActionRendering, registerChatCodeBlockActions, registerChatCod import { ChatContextContributions } from './actions/chatContext.js'; import { registerChatContextActions } from './actions/chatContextActions.js'; import { ChatCopyActionRendering, registerChatCopyActions } from './actions/chatCopyActions.js'; +import { ChatModelFeedbackSurveyActionRendering, registerChatModelFeedbackSurveyActions } from './actions/chatModelFeedbackSurveyActions.js'; +import { ChatModelFeedbackSurveyService, IChatModelFeedbackSurveyService } from './feedbackSurvey/chatModelFeedbackSurveyService.js'; +import { ChatModelFeedbackSurveyPromptContribution } from './feedbackSurvey/chatModelFeedbackSurveyPromptContribution.js'; import { registerChatDeveloperActions } from './actions/chatDeveloperActions.js'; import { registerChatElicitationActions } from './actions/chatElicitationActions.js'; import { registerChatExecuteActions } from './actions/chatExecuteActions.js'; @@ -3013,6 +3016,8 @@ registerWorkbenchContribution2(ChatPromptFilesExtensionPointHandler.ID, ChatProm registerWorkbenchContribution2(ChatCompatibilityNotifier.ID, ChatCompatibilityNotifier, WorkbenchPhase.Eventually); registerWorkbenchContribution2(CodeBlockActionRendering.ID, CodeBlockActionRendering, WorkbenchPhase.BlockRestore); registerWorkbenchContribution2(ChatCopyActionRendering.ID, ChatCopyActionRendering, WorkbenchPhase.BlockRestore); +registerWorkbenchContribution2(ChatModelFeedbackSurveyActionRendering.ID, ChatModelFeedbackSurveyActionRendering, WorkbenchPhase.BlockRestore); +registerWorkbenchContribution2(ChatModelFeedbackSurveyPromptContribution.ID, ChatModelFeedbackSurveyPromptContribution, WorkbenchPhase.Eventually); registerWorkbenchContribution2(ChatImplicitContextContribution.ID, ChatImplicitContextContribution, WorkbenchPhase.Eventually); registerWorkbenchContribution2(ChatViewsWelcomeHandler.ID, ChatViewsWelcomeHandler, WorkbenchPhase.BlockStartup); registerWorkbenchContribution2(ChatGettingStartedContribution.ID, ChatGettingStartedContribution, WorkbenchPhase.Eventually); @@ -3055,6 +3060,7 @@ registerWorkbenchContribution2(TranscriptContextAttachmentWidgetContribution.ID, registerChatActions(); registerChatAccessibilityActions(); registerChatCopyActions(); +registerChatModelFeedbackSurveyActions(); registerChatOpenAgentDebugPanelAction(); registerChatCodeBlockActions(); registerChatCodeCompareBlockActions(); @@ -3096,6 +3102,7 @@ registerSingleton(IChatWidgetService, ChatWidgetService, InstantiationType.Delay registerSingleton(IChatPasteTargetService, ChatPasteTargetService, InstantiationType.Delayed); registerSingleton(IChatSideChatService, ChatSideChatService, InstantiationType.Delayed); registerSingleton(IChatRequestOriginService, ChatRequestOriginService, InstantiationType.Delayed); +registerSingleton(IChatModelFeedbackSurveyService, ChatModelFeedbackSurveyService, InstantiationType.Delayed); registerSingleton(IChatPetService, ChatPetService, InstantiationType.Delayed); registerSingleton(IQuickChatService, QuickChatService, InstantiationType.Delayed); registerSingleton(IChatAccessibilityService, ChatAccessibilityService, InstantiationType.Delayed); diff --git a/src/vs/workbench/contrib/chat/browser/feedbackSurvey/chatModelFeedbackSurveyPromptContribution.ts b/src/vs/workbench/contrib/chat/browser/feedbackSurvey/chatModelFeedbackSurveyPromptContribution.ts new file mode 100644 index 0000000000000..f0bbf1b0b0993 --- /dev/null +++ b/src/vs/workbench/contrib/chat/browser/feedbackSurvey/chatModelFeedbackSurveyPromptContribution.ts @@ -0,0 +1,72 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Disposable, DisposableMap, DisposableStore } from '../../../../../base/common/lifecycle.js'; +import { autorun, observableSignalFromEvent } from '../../../../../base/common/observable.js'; +import { IWorkbenchContribution } from '../../../../common/contributions.js'; +import { IChatWidget, IChatWidgetService } from '../chat.js'; +import { IChatModelFeedbackSurveyService } from './chatModelFeedbackSurveyService.js'; + +/** + * Watches the model picker of every chat widget and reports switches to the survey service. + * + * It sits outside the picker so model selection knows nothing about surveys, and outside the + * service so the service stays free of widget lifecycle and easy to test. + */ +export class ChatModelFeedbackSurveyPromptContribution extends Disposable implements IWorkbenchContribution { + + static readonly ID = 'chat.modelFeedbackSurveyPrompt'; + + private readonly widgetListeners = this._register(new DisposableMap()); + + constructor( + @IChatWidgetService private readonly chatWidgetService: IChatWidgetService, + @IChatModelFeedbackSurveyService private readonly surveyService: IChatModelFeedbackSurveyService, + ) { + super(); + + for (const widget of this.chatWidgetService.getAllWidgets()) { + this.trackWidget(widget); + } + this._register(this.chatWidgetService.onDidAddWidget(widget => this.trackWidget(widget))); + this._register(this.chatWidgetService.onDidRemoveWidget(widget => this.widgetListeners.deleteAndDispose(widget))); + } + + private trackWidget(widget: IChatWidget): void { + const listeners = new DisposableStore(); + + // The widget loads its session after it registers, and the model resolves around the same + // time. Re-running on that event keeps the pairing below anchored to the right session. + const viewModelChanged = observableSignalFromEvent('chatFeedbackSurveyViewModel', widget.onDidChangeViewModel); + let previous: { readonly modelId: string; readonly session: string } | undefined; + + listeners.add(autorun(reader => { + viewModelChanged.read(reader); + const modelId = widget.input.selectedLanguageModel.read(reader)?.identifier; + const sessionResource = widget.viewModel?.sessionResource; + const session = sessionResource?.toString(); + + const last = previous; + previous = modelId && session ? { modelId, session } : undefined; + + // Loading a different session restores that session's own model, so the next change is + // measured from there rather than from whatever the previous session was using. + if (last && last.session !== session) { + previous = undefined; + return; + } + + // Only a move between two known models counts. The first resolution and any gap while + // models load are not the user rejecting anything. + if (!last || !modelId || !sessionResource || last.modelId === modelId) { + return; + } + + this.surveyService.notifyModelSwitchedAway(sessionResource, last.modelId, modelId); + })); + + this.widgetListeners.set(widget, listeners); + } +} diff --git a/src/vs/workbench/contrib/chat/browser/feedbackSurvey/chatModelFeedbackSurveyService.ts b/src/vs/workbench/contrib/chat/browser/feedbackSurvey/chatModelFeedbackSurveyService.ts new file mode 100644 index 0000000000000..b7d800da4dd5d --- /dev/null +++ b/src/vs/workbench/contrib/chat/browser/feedbackSurvey/chatModelFeedbackSurveyService.ts @@ -0,0 +1,673 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Emitter, Event } from '../../../../../base/common/event.js'; +import { Disposable } from '../../../../../base/common/lifecycle.js'; +import { URI } from '../../../../../base/common/uri.js'; +import { generateUuid } from '../../../../../base/common/uuid.js'; +import { ICommandService } from '../../../../../platform/commands/common/commands.js'; +import { IConfigurationService } from '../../../../../platform/configuration/common/configuration.js'; +import { createDecorator } from '../../../../../platform/instantiation/common/instantiation.js'; +import { ILogService } from '../../../../../platform/log/common/log.js'; +import { IStorageService, StorageScope, StorageTarget } from '../../../../../platform/storage/common/storage.js'; +import { escapeModelIdForTelemetry, ITelemetryService, TelemetryLevel } from '../../../../../platform/telemetry/common/telemetry.js'; +import { IWorkbenchAssignmentService } from '../../../../services/assignment/common/assignmentService.js'; +import { ChatModelFeedbackSurveyStepKind, expandModelMatchCandidates, IChatModelFeedbackSurveyConfig, matchesChatModelFeedbackSurvey, parseChatModelFeedbackSurveyConfig } from '../../common/feedbackSurvey/chatModelFeedbackSurveyConfig.js'; +import { CHAT_MODEL_FEEDBACK_SURVEY_TELEMETRY_COMMAND_ID, ChatModelFeedbackSurveyEventKind, IChatModelFeedbackSurveyTelemetryEvent } from '../../common/feedbackSurvey/chatModelFeedbackSurveyTelemetry.js'; +import { ILanguageModelsService } from '../../common/languageModels.js'; +import { getChatSessionType } from '../../common/model/chatUri.js'; +import { IChatResponseViewModel } from '../../common/model/chatViewModel.js'; +import { IChatSessionsService } from '../../common/chatSessionsService.js'; +import { IChatService } from '../../common/chatService/chatService.js'; + +/** Name of the experiment treatment carrying the survey payload. */ +export const CHAT_MODEL_FEEDBACK_SURVEY_TREATMENT = 'chatModelFeedbackSurvey'; + +/** Gates the in product feedback UI. The survey replaces thumbs up and down, so it obeys this too. */ +const FEEDBACK_ENABLED_CONFIG = 'telemetry.feedback.enabled'; + +const STORAGE_PREFIX = 'chat.modelFeedbackSurvey.'; + +export const enum ChatModelFeedbackSurveyStatus { + /** Available but not showing: only the feedback control is rendered. */ + Collapsed = 'collapsed', + Open = 'open', +} + +/** What caused the survey to open, recorded so the funnel can separate the two paths. */ +export type ChatModelFeedbackSurveyOpenTrigger = 'manual' | 'chance' | 'modelSwitchedAway'; + +export interface IChatModelFeedbackSurveyState { + readonly config: IChatModelFeedbackSurveyConfig; + readonly instanceId: string; + readonly status: ChatModelFeedbackSurveyStatus; + /** Index into `config.steps` of the step currently being shown. */ + readonly stepIndex: number; + /** Answers so far, keyed by step id. Choice steps store an option id, text steps the comment. */ + readonly answers: ReadonlyMap; + /** Uncommitted free text, preserved across the widget being recycled by virtualization. */ + readonly commentDraft: string; + /** Whether this survey was submitted. Reopening it acknowledges rather than asks again. */ + readonly isSubmitted: boolean; + /** What opened the survey, so the UI can take focus only when the user asked for it. */ + readonly openTrigger: ChatModelFeedbackSurveyOpenTrigger | undefined; +} + +/** Identifies the response whose survey changed, without retaining its view model. */ +export interface IChatModelFeedbackSurveyChangeEvent { + readonly sessionResource: URI; + readonly requestId: string; +} + +export const IChatModelFeedbackSurveyService = createDecorator('chatModelFeedbackSurveyService'); + +export interface IChatModelFeedbackSurveyService { + readonly _serviceBrand: undefined; + + /** Fires when a response's survey state changes, so the row can re-render. */ + readonly onDidChangeSurveyState: Event; + + /** + * Fires when the configured survey changes, including when it first resolves. Rows rendered + * before that point carry no control until they re-render, so they listen for this. + */ + readonly onDidChangeConfiguration: Event; + + /** + * The survey attached to a response, or `undefined` when the config does not apply to it. + * + * Presence depends only on the `match` rules and never on the prompting heuristics, so the + * control does not come and go between responses. Repeated calls return the same state, so a + * row scrolling back into view keeps its control and is not asked to prompt twice. + */ + getSurvey(response: IChatResponseViewModel): IChatModelFeedbackSurveyState | undefined; + + /** + * Opens the survey, or closes it when it is already showing. Manual opens are never rate + * limited, because the prompting rules only exist to pace surveys the user did not ask for. + */ + toggle(response: IChatResponseViewModel): void; + + /** + * Reports that the user moved the model picker from one model to another. Leaving a surveyed + * model for an unsurveyed one is the strongest signal that the model was wrong. + */ + notifyModelSwitchedAway(sessionResource: URI, fromModelId: string, toModelId: string): void; + answerChoice(response: IChatResponseViewModel, stepId: string, optionId: string): void; + submit(response: IChatResponseViewModel, comment?: string): void; + dismiss(response: IChatResponseViewModel): void; + /** Records in-progress free text without re-rendering, so it survives row recycling. */ + setCommentDraft(response: IChatResponseViewModel, comment: string): void; +} + +interface IMutableSurveyState { + readonly config: IChatModelFeedbackSurveyConfig; + readonly instanceId: string; + readonly sessionResource: URI; + readonly requestId: string; + status: ChatModelFeedbackSurveyStatus; + stepIndex: number; + readonly answers: Map; + commentDraft: string; + isSubmitted: boolean; + openTrigger: ChatModelFeedbackSurveyOpenTrigger | undefined; + /** Dimensions read from the response when the survey was created. */ + readonly dimensions: IChatModelFeedbackSurveyDimensions; + /** Guards against re-emitting `shown` when a virtualized row is re-rendered. */ + shownReported: boolean; + /** Text already reported, so submitting and dismissing cannot report it twice. */ + reportedComment?: string; +} + +type IChatModelFeedbackSurveyDimensions = Pick; + +/** Whether the user is part way through answering, as opposed to done or not started. */ +function isInProgress(state: IMutableSurveyState): boolean { + return state.status === ChatModelFeedbackSurveyStatus.Open && !state.isSubmitted; +} + +export class ChatModelFeedbackSurveyService extends Disposable implements IChatModelFeedbackSurveyService { + + declare readonly _serviceBrand: undefined; + + private readonly _onDidChangeSurveyState = this._register(new Emitter()); + readonly onDidChangeSurveyState: Event = this._onDidChangeSurveyState.event; + + private readonly _onDidChangeConfiguration = this._register(new Emitter()); + readonly onDidChangeConfiguration: Event = this._onDidChangeConfiguration.event; + + private _config: IChatModelFeedbackSurveyConfig | undefined; + private _configResolved = false; + /** Increments on every treatment refresh so a slow in-flight resolution cannot overwrite a newer one. */ + private _configGeneration = 0; + + private readonly _states = new Map(); + /** How many times the survey has opened itself in each session, keyed by session resource. */ + private readonly _sessionPromptCounts = new Map(); + /** The most recent response carrying a survey in each session, for event-driven prompting. */ + private readonly _lastSurveyedResponse = new Map(); + + constructor( + @IWorkbenchAssignmentService private readonly assignmentService: IWorkbenchAssignmentService, + @IConfigurationService private readonly configurationService: IConfigurationService, + @IStorageService private readonly storageService: IStorageService, + @ITelemetryService private readonly telemetryService: ITelemetryService, + @ICommandService private readonly commandService: ICommandService, + @ILanguageModelsService private readonly languageModelsService: ILanguageModelsService, + @IChatSessionsService private readonly chatSessionsService: IChatSessionsService, + @IChatService private readonly chatService: IChatService, + @ILogService private readonly logService: ILogService, + ) { + super(); + + void this.resolveConfig(); + this._register(this.assignmentService.onDidRefetchAssignments(() => void this.resolveConfig())); + this._register(this.chatService.onDidDisposeSession(e => this.forgetSessions(e.sessionResources))); + } + + private async resolveConfig(): Promise { + const generation = ++this._configGeneration; + let payload: string | undefined; + try { + payload = await this.assignmentService.getTreatment(CHAT_MODEL_FEEDBACK_SURVEY_TREATMENT); + } catch (err) { + this.logService.trace(`[chatModelFeedbackSurvey] failed to resolve treatment: ${err}`); + } + + if (generation !== this._configGeneration) { + return; // a newer resolution won + } + + const previousId = this._config?.id; + if (payload === undefined) { + this._config = undefined; + } else { + const result = parseChatModelFeedbackSurveyConfig(payload); + if (result.error) { + this.logService.warn(`[chatModelFeedbackSurvey] ignoring invalid survey config: ${result.error}`); + this._config = undefined; + } else { + this._config = result.config; + } + } + this._configResolved = true; + + // Only a different survey invalidates state, since reported step indices would no longer + // mean what they did when sent. A treatment that resolves to nothing happens while the + // experimentation client is rebuilt, and must not close an open survey or reset budgets. + if (this._config && this._config.id !== previousId) { + this._states.clear(); + this._sessionPromptCounts.clear(); + this._lastSurveyedResponse.clear(); + } + + if (this._config?.id !== previousId) { + this._onDidChangeConfiguration.fire(); + } + } + + getSurvey(response: IChatResponseViewModel): IChatModelFeedbackSurveyState | undefined { + const key = this.getKey(response); + const existing = this._states.get(key); + if (existing) { + // The config can be retired and feedback can be switched off after a survey was + // offered, so a cached one is only handed back while it still applies. A survey the + // user is part way through is left alone, since a treatment that briefly resolves to + // nothing must not take a form away mid answer. + const stillApplies = this._config?.id === existing.config.id && this.isFeedbackUiEnabled(); + if (!stillApplies && !isInProgress(existing)) { + this._states.delete(key); + return undefined; + } + // A survey the user is part way through stays put, so a form is never pulled away + // when a newer response arrives. Anything else follows the newest response. + if (!response.isLast && !isInProgress(existing)) { + this._states.delete(key); + return undefined; + } + this.reportShownOnce(existing); + return this.toReadonly(existing); + } + + // Only the newest response offers feedback, so history never fills with stale controls. + if (!response.isLast) { + return undefined; + } + + // Runs before the match check so a newer response the survey ignores still supersedes. + this.dropSupersededStates(response.sessionResource, key); + + const config = this.getMatchingConfig(response); + if (!config) { + return undefined; + } + + const state: IMutableSurveyState = { + config, + instanceId: generateUuid(), + sessionResource: response.sessionResource, + requestId: response.requestId, + status: ChatModelFeedbackSurveyStatus.Collapsed, + stepIndex: 0, + answers: new Map(), + commentDraft: '', + isSubmitted: false, + openTrigger: undefined, + dimensions: this.readDimensions(response), + shownReported: false, + }; + this._states.set(key, state); + this._lastSurveyedResponse.set(response.sessionResource.toString(), key); + this.reportShownOnce(state); + + // Rolled once per response so the outcome is stable however often the row re-renders. + // The caller is mid render and reads the state below, so this does not announce a change. + if (this.shouldPromptByChance(state)) { + this.beginPrompt(state, 'chance', false); + } + + return this.toReadonly(state); + } + + toggle(response: IChatResponseViewModel): void { + const state = this._states.get(this.getKey(response)); + if (!state) { + return; + } + if (state.status === ChatModelFeedbackSurveyStatus.Open) { + this.dismiss(response); + } else { + this.openState(state, 'manual'); + } + } + + notifyModelSwitchedAway(sessionResource: URI, fromModelId: string, toModelId: string): void { + const key = this._lastSurveyedResponse.get(sessionResource.toString()); + const state = key ? this._states.get(key) : undefined; + if (!state || state.status === ChatModelFeedbackSurveyStatus.Open || state.isSubmitted) { + return; + } + + // Leaving one surveyed model for another is not abandoning the thing being surveyed, and + // a switch between two unrelated models says nothing about it either. + if (!this.isSurveyedModel(state.config, fromModelId) || this.isSurveyedModel(state.config, toModelId)) { + return; + } + + const trigger = state.config.prompt.triggers.modelSwitchedAway; + if (!trigger.enabled || this.hasSurveyInProgress() || !this.hasPromptBudget(state, trigger.bypassCooldown)) { + return; + } + + this.beginPrompt(state, 'modelSwitchedAway'); + } + + /** Whether a model identifier is one the survey's `selectedModels` selectors name. */ + private isSurveyedModel(config: IChatModelFeedbackSurveyConfig, modelId: string): boolean { + const selectors = config.match.selectedModels; + if (!selectors.length) { + return false; // a survey that does not name a model cannot detect leaving one + } + const candidates = expandModelMatchCandidates(modelId, this.getModelAliases(modelId)); + return selectors.some(selector => candidates.has(selector)); + } + + /** The other identifiers a selector may name a model by. */ + private getModelAliases(modelId: string | undefined): string[] | undefined { + const metadata = modelId ? this.languageModelsService.lookupLanguageModel(modelId) : undefined; + return metadata ? [metadata.id, metadata.family, metadata.name, metadata.vendor] : undefined; + } + + answerChoice(response: IChatResponseViewModel, stepId: string, optionId: string): void { + const state = this._states.get(this.getKey(response)); + if (!state || state.status !== ChatModelFeedbackSurveyStatus.Open || state.isSubmitted) { + return; + } + + const stepIndex = state.config.steps.findIndex(step => step.id === stepId); + const step = state.config.steps[stepIndex]; + if (!step || step.kind !== ChatModelFeedbackSurveyStepKind.Choice) { + return; + } + // Only ids that came from the config may reach telemetry. + if (!step.options.some(option => option.id === optionId)) { + return; + } + + state.answers.set(stepId, optionId); + this.report(state, 'step', { stepId, stepIndex, answerId: optionId }); + + const isLastStep = stepIndex === state.config.steps.length - 1; + if (isLastStep) { + // A survey that ends on a choice has no Submit button, so the final selection is the + // submission. Without this the panel would re-render the same question forever. + this.finish(state, true, 'submitted'); + return; + } + + state.stepIndex = stepIndex + 1; + this._onDidChangeSurveyState.fire(this.toChangeEvent(state)); + } + + submit(response: IChatResponseViewModel, comment?: string): void { + const state = this._states.get(this.getKey(response)); + if (!state || state.status !== ChatModelFeedbackSurveyStatus.Open || state.isSubmitted) { + return; + } + + this.reportCommentOnce(state, comment); + this.finish(state, true, 'submitted'); + } + + dismiss(response: IChatResponseViewModel): void { + const state = this._states.get(this.getKey(response)); + if (!state || state.status !== ChatModelFeedbackSurveyStatus.Open) { + return; + } + + // Closing an acknowledgement is not a dismissal, and reporting one would double count + // against the submission already sent. + if (state.isSubmitted) { + state.status = ChatModelFeedbackSurveyStatus.Collapsed; + this._onDidChangeSurveyState.fire(this.toChangeEvent(state)); + return; + } + + this.reportCommentOnce(state, state.commentDraft); + this.finish(state, false, 'dismissed'); + } + + setCommentDraft(response: IChatResponseViewModel, comment: string): void { + const state = this._states.get(this.getKey(response)); + if (!state || state.status !== ChatModelFeedbackSurveyStatus.Open) { + return; + } + // No change event, because re-rendering on every keystroke would fight the input. The + // draft lives here so recycling the widget cannot discard it. + state.commentDraft = comment; + } + + // --- automatic prompting + + /** + * Decides whether a newly surveyed response should prompt on its own. The odds ramp with + * every response that passed without prompting, so heavier users reach the survey sooner. + */ + private shouldPromptByChance(state: IMutableSurveyState): boolean { + const { chance } = state.config.prompt; + if (chance.initial <= 0 && chance.increment <= 0) { + return false; + } + if (this.hasSurveyInProgress()) { + return false; + } + if (!this.hasPromptBudget(state, false)) { + return false; + } + + const misses = this.readPromptMisses(state.config); + const probability = Math.min(chance.initial + (chance.increment * misses), chance.max); + if (Math.random() < probability) { + return true; + } + + this.writePromptMisses(state.config, misses + 1); + return false; + } + + /** Releases everything held for sessions that have gone away. */ + private forgetSessions(sessionResources: readonly URI[]): void { + for (const sessionResource of sessionResources) { + const session = sessionResource.toString(); + this._sessionPromptCounts.delete(session); + this._lastSurveyedResponse.delete(session); + for (const [key, state] of [...this._states]) { + if (state.sessionResource.toString() === session) { + this._states.delete(key); + } + } + } + } + + /** Whether any survey is part way through, which an unrequested prompt must not displace. */ + private hasSurveyInProgress(): boolean { + for (const state of this._states.values()) { + if (isInProgress(state)) { + return true; + } + } + return false; + } + + private hasPromptBudget(state: IMutableSurveyState, bypassCooldown: boolean): boolean { + const { prompt } = state.config; + const sessionKey = state.sessionResource.toString(); + if ((this._sessionPromptCounts.get(sessionKey) ?? 0) >= prompt.maxPerSession) { + return false; + } + if (bypassCooldown || prompt.cooldownDays <= 0) { + return true; + } + + const lastPromptAt = this.readLastPromptAt(state.config); + if (lastPromptAt === undefined) { + return true; + } + return Date.now() - lastPromptAt >= prompt.cooldownDays * 24 * 60 * 60 * 1000; + } + + /** Opens the survey unprompted and charges it against the pacing budgets. */ + private beginPrompt(state: IMutableSurveyState, trigger: ChatModelFeedbackSurveyOpenTrigger, announce = true): void { + const sessionKey = state.sessionResource.toString(); + this._sessionPromptCounts.set(sessionKey, (this._sessionPromptCounts.get(sessionKey) ?? 0) + 1); + this.writeLastPromptAt(state.config, Date.now()); + this.writePromptMisses(state.config, 0); + this.openState(state, trigger, announce); + } + + private openState(state: IMutableSurveyState, trigger: ChatModelFeedbackSurveyOpenTrigger, announce = true): void { + this.closeOtherOpenSurveys(state); + // A submitted survey reopens read only, so it needs no new instance. + if (!state.isSubmitted && state.stepIndex >= state.config.steps.length) { + state.stepIndex = 0; + } + state.status = ChatModelFeedbackSurveyStatus.Open; + state.openTrigger = trigger; + if (!state.isSubmitted) { + this.report(state, 'opened', { trigger }); + } + if (announce) { + this._onDidChangeSurveyState.fire(this.toChangeEvent(state)); + } + } + + private finish(state: IMutableSurveyState, submitted: boolean, kind: ChatModelFeedbackSurveyEventKind): void { + const stepIndex = state.stepIndex; + state.isSubmitted = submitted; + // A submitted survey stays open to acknowledge. An abandoned one closes and can be + // reopened, since the user never answered it. + state.status = submitted ? ChatModelFeedbackSurveyStatus.Open : ChatModelFeedbackSurveyStatus.Collapsed; + this.report(state, kind, { stepIndex }); + this._onDidChangeSurveyState.fire(this.toChangeEvent(state)); + } + + private toChangeEvent(state: IMutableSurveyState): IChatModelFeedbackSurveyChangeEvent { + return { sessionResource: state.sessionResource, requestId: state.requestId }; + } + + // --- eligibility + + private getMatchingConfig(response: IChatResponseViewModel): IChatModelFeedbackSurveyConfig | undefined { + if (!this._configResolved || !this._config) { + return undefined; + } + if (!this.isFeedbackUiEnabled()) { + return undefined; + } + if (!response.isComplete || response.isCanceled || response.errorDetails) { + return undefined; + } + + const request = response.model.request; + const selectedModelId = request?.modelId; + + return matchesChatModelFeedbackSurvey(this._config, { + selectedModelId, + selectedModelAliases: this.getModelAliases(selectedModelId), + resolvedModelId: this.getResolvedModelId(response), + modeId: request?.modeInfo?.telemetryModeId, + harness: this.getHarness(response.sessionResource), + sessionType: getChatSessionType(response.sessionResource), + }) ? this._config : undefined; + } + + /** Answers that could never be sent must not be collected in the first place. */ + private isFeedbackUiEnabled(): boolean { + return this.configurationService.getValue(FEEDBACK_ENABLED_CONFIG) !== false + && this.telemetryService.telemetryLevel !== TelemetryLevel.NONE; + } + + private getResolvedModelId(response: IChatResponseViewModel): string | undefined { + const resolvedModel = response.result?.metadata?.resolvedModel; + return typeof resolvedModel === 'string' ? resolvedModel : undefined; + } + + /** Normalizes local and remote session types to one provider id, which is what a config targets. */ + private getHarness(sessionResource: URI): string | undefined { + return this.chatSessionsService.getChatSessionContribution(getChatSessionType(sessionResource))?.agentHostProviderId; + } + + // --- prompt pacing storage + + private readPromptMisses(config: IChatModelFeedbackSurveyConfig): number { + const value = this.storageService.getNumber(`${STORAGE_PREFIX}${config.id}.promptMisses`, StorageScope.PROFILE, 0); + return Number.isFinite(value) && value > 0 ? value : 0; + } + + private writePromptMisses(config: IChatModelFeedbackSurveyConfig, misses: number): void { + this.storageService.store(`${STORAGE_PREFIX}${config.id}.promptMisses`, misses, StorageScope.PROFILE, StorageTarget.MACHINE); + } + + private readLastPromptAt(config: IChatModelFeedbackSurveyConfig): number | undefined { + const value = this.storageService.getNumber(`${STORAGE_PREFIX}${config.id}.lastPromptAt`, StorageScope.PROFILE); + return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : undefined; + } + + private writeLastPromptAt(config: IChatModelFeedbackSurveyConfig, timestamp: number): void { + this.storageService.store(`${STORAGE_PREFIX}${config.id}.lastPromptAt`, timestamp, StorageScope.PROFILE, StorageTarget.MACHINE); + } + + // --- state bookkeeping + + private getKey(response: IChatResponseViewModel): string { + return `${response.sessionResource.toString()}\u0000${response.requestId}`; + } + + private toReadonly(state: IMutableSurveyState): IChatModelFeedbackSurveyState { + return { + config: state.config, + instanceId: state.instanceId, + status: state.status, + stepIndex: state.stepIndex, + answers: state.answers, + commentDraft: state.commentDraft, + isSubmitted: state.isSubmitted, + openTrigger: state.openTrigger, + }; + } + + /** + * Closes whichever survey was already showing, since only one is open at a time. This is the + * UI moving on rather than the user rejecting anything, so nothing is reported. + */ + private closeOtherOpenSurveys(keep: IMutableSurveyState): void { + for (const other of [...this._states.values()]) { + if (other === keep || other.status !== ChatModelFeedbackSurveyStatus.Open) { + continue; + } + other.status = ChatModelFeedbackSurveyStatus.Collapsed; + this._onDidChangeSurveyState.fire(this.toChangeEvent(other)); + } + } + + /** + * Within one session, keeps the newest response plus anything the user is part way through. + * Other sessions are separate transcripts and are left alone. + */ + private dropSupersededStates(sessionResource: URI, currentKey: string): void { + const session = sessionResource.toString(); + for (const [key, state] of [...this._states]) { + if (key !== currentKey && state.sessionResource.toString() === session && !isInProgress(state)) { + this._states.delete(key); + } + } + } + + // --- telemetry + + private reportShownOnce(state: IMutableSurveyState): void { + if (state.shownReported) { + return; + } + state.shownReported = true; + this.report(state, 'shown', {}); + } + + private reportCommentOnce(state: IMutableSurveyState, comment: string | undefined): void { + // Validation guarantees at most one text step, and that it is last. + const textStep = state.config.steps.at(-1); + if (textStep?.kind !== ChatModelFeedbackSurveyStepKind.Text) { + return; + } + const trimmed = comment?.trim(); + if (!trimmed || trimmed === state.reportedComment) { + return; + } + + const clamped = trimmed.slice(0, textStep.maxLength); + state.reportedComment = trimmed; + state.answers.set(textStep.id, clamped); + this.report(state, 'step', { + stepId: textStep.id, + stepIndex: state.config.steps.length - 1, + comment: clamped, + }); + } + + private readDimensions(response: IChatResponseViewModel): IChatModelFeedbackSurveyDimensions { + const request = response.model.request; + return { + modelId: escapeModelIdForTelemetry(request?.modelId), + resolvedModelId: escapeModelIdForTelemetry(this.getResolvedModelId(response)), + modeId: request?.modeInfo?.telemetryModeId, + harness: this.getHarness(response.sessionResource), + sessionType: getChatSessionType(response.sessionResource), + }; + } + + private report( + state: IMutableSurveyState, + kind: ChatModelFeedbackSurveyEventKind, + details: Pick, + ): void { + if (!this.isFeedbackUiEnabled()) { + return; // never let survey content cross the process boundary when feedback is off + } + + const event: IChatModelFeedbackSurveyTelemetryEvent = { + kind, + surveyId: state.config.id, + surveyInstanceId: state.instanceId, + stepCount: state.config.steps.length, + ...details, + ...state.dimensions, + requestId: state.requestId, + }; + + // Best effort: the survey must never interfere with the chat session. + this.commandService.executeCommand(CHAT_MODEL_FEEDBACK_SURVEY_TELEMETRY_COMMAND_ID, event) + .catch(err => this.logService.trace(`[chatModelFeedbackSurvey] failed to report '${kind}': ${err}`)); + } +} diff --git a/src/vs/workbench/contrib/chat/browser/feedbackSurvey/chatModelFeedbackSurveyWidget.ts b/src/vs/workbench/contrib/chat/browser/feedbackSurvey/chatModelFeedbackSurveyWidget.ts new file mode 100644 index 0000000000000..4dc275f7f3f41 --- /dev/null +++ b/src/vs/workbench/contrib/chat/browser/feedbackSurvey/chatModelFeedbackSurveyWidget.ts @@ -0,0 +1,291 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as dom from '../../../../../base/browser/dom.js'; +import { StandardKeyboardEvent } from '../../../../../base/browser/keyboardEvent.js'; +import { status } from '../../../../../base/browser/ui/aria/aria.js'; +import { Button } from '../../../../../base/browser/ui/button/button.js'; +import { InputBox } from '../../../../../base/browser/ui/inputbox/inputBox.js'; +import { Codicon } from '../../../../../base/common/codicons.js'; +import { KeyCode } from '../../../../../base/common/keyCodes.js'; +import { Disposable, DisposableStore } from '../../../../../base/common/lifecycle.js'; +import { localize } from '../../../../../nls.js'; +import { IContextKey, IContextKeyService } from '../../../../../platform/contextkey/common/contextkey.js'; +import { IHoverService } from '../../../../../platform/hover/browser/hover.js'; +import { defaultButtonStyles, defaultInputBoxStyles } from '../../../../../platform/theme/browser/defaultStyles.js'; +import { ChatContextKeys } from '../../common/actions/chatContextKeys.js'; +import { ChatModelFeedbackSurveyStepKind, IChatModelFeedbackSurveyTextStep } from '../../common/feedbackSurvey/chatModelFeedbackSurveyConfig.js'; +import { IChatResponseViewModel } from '../../common/model/chatViewModel.js'; +import { ChatModelFeedbackSurveyStatus, IChatModelFeedbackSurveyService, IChatModelFeedbackSurveyState } from './chatModelFeedbackSurveyService.js'; +import './media/chatModelFeedbackSurvey.css'; + +/** Keys the chat list acts on, which the survey has to claim while it is focused. */ +const KEYS_HANDLED_BY_LIST = new Set([ + KeyCode.UpArrow, + KeyCode.DownArrow, + KeyCode.LeftArrow, + KeyCode.RightArrow, + KeyCode.PageUp, + KeyCode.PageDown, + KeyCode.Enter, + KeyCode.Space, + KeyCode.Escape, + KeyCode.Home, + KeyCode.End, +]); + +const TEXT_INPUT_MAX_HEIGHT = 96; + +/** Stands in for a step key once the survey is answered and only acknowledges. */ +const ACKNOWLEDGEMENT_STEP_KEY = 'acknowledged'; + +/** + * The inline survey shown beneath a chat response footer. + * + * It renders below the footer toolbar so opening it never moves the footer icons. All durable + * state lives in the survey service, since chat rows are virtualized and recycle this widget. + */ +export class ChatModelFeedbackSurveyWidget extends Disposable { + + private readonly renderDisposables = this._register(new DisposableStore()); + private response: IChatResponseViewModel | undefined; + private readonly hasSurveyContextKey: IContextKey; + private readonly surveyOpenContextKey: IContextKey; + /** Step focus was last moved to, so an unrelated re-render does not move it again. */ + private lastFocusedStep: string | undefined; + private isPanelOpen = false; + /** Guards against `render` being re-entered when reading state changes that state. */ + private isRendering = false; + + constructor( + private readonly container: HTMLElement, + /** Returns focus to whatever opened the survey once its controls are torn down. */ + private readonly restoreFocus: () => void, + @IChatModelFeedbackSurveyService private readonly surveyService: IChatModelFeedbackSurveyService, + @IHoverService private readonly hoverService: IHoverService, + @IContextKeyService contextKeyService: IContextKeyService, + ) { + super(); + + this.hasSurveyContextKey = ChatContextKeys.responseHasFeedbackSurvey.bindTo(contextKeyService); + this.surveyOpenContextKey = ChatContextKeys.responseFeedbackSurveyOpen.bindTo(contextKeyService); + + this._register(this.surveyService.onDidChangeSurveyState(e => { + if (this.response && e.requestId === this.response.requestId && e.sessionResource.toString() === this.response.sessionResource.toString()) { + this.render(this.response); + } + })); + + this._register(this.surveyService.onDidChangeConfiguration(() => this.render(this.response))); + + // Escape is handled in capture, because Button stops propagation on its own Escape. + this._register(dom.addDisposableListener(this.container, dom.EventType.KEY_DOWN, e => { + const event = new StandardKeyboardEvent(e); + if (event.keyCode === KeyCode.Escape) { + event.stopPropagation(); + event.preventDefault(); + this.dismiss(); + } + }, true)); + + // Claimed on the way back up, after the survey has had its turn, so the chat list does + // not also move its selection. Stopping these in capture would starve the option list. + this._register(dom.addDisposableListener(this.container, dom.EventType.KEY_DOWN, e => { + const event = new StandardKeyboardEvent(e); + if (KEYS_HANDLED_BY_LIST.has(event.keyCode) && !dom.isEditableElement(e.target as HTMLElement)) { + event.stopPropagation(); + } + })); + } + + /** Renders the survey for `response`, or clears the panel when there is nothing to show. */ + render(response: IChatResponseViewModel | undefined): void { + // Reading the survey can open it, which reports a state change back to this widget. The + // call already in flight reads the latest state, so a nested render would only duplicate + // what it is about to draw. + if (this.isRendering) { + return; + } + + this.isRendering = true; + try { + this.doRender(response); + } finally { + this.isRendering = false; + } + } + + private doRender(response: IChatResponseViewModel | undefined): void { + this.response = response; + this.renderDisposables.clear(); + dom.clearNode(this.container); + + const state = response && this.surveyService.getSurvey(response); + this.hasSurveyContextKey.set(!!state); + + const isOpen = !!state && state.status === ChatModelFeedbackSurveyStatus.Open; + const wasOpen = this.isPanelOpen; + this.isPanelOpen = isOpen; + this.surveyOpenContextKey.set(isOpen); + this.container.classList.toggle('hidden', !isOpen); + if (!response || !state || !isOpen) { + this.lastFocusedStep = undefined; + // The focused control has just been removed, so hand focus back to the control that + // opened the survey rather than letting it fall to the document body. + if (wasOpen) { + this.restoreFocus(); + } + return; + } + + this.renderPanel(response, state); + } + + private renderPanel(response: IChatResponseViewModel, state: IChatModelFeedbackSurveyState): void { + const step = state.config.steps[state.stepIndex]; + if (!step) { + return; + } + + const panel = dom.append(this.container, dom.$('.chat-feedback-survey-container')); + const header = dom.append(panel, dom.$('.chat-feedback-survey-header')); + const title = dom.append(header, dom.$('.chat-feedback-survey-title')); + title.textContent = state.isSubmitted + ? localize('chat.feedbackSurvey.acknowledgement', "Thanks, your feedback has been recorded.") + : step.title; + + const closeButton = this.renderCloseButton(header); + + // An answered survey has nothing left to ask, so it only acknowledges. + if (state.isSubmitted) { + if (this.lastFocusedStep !== ACKNOWLEDGEMENT_STEP_KEY) { + this.lastFocusedStep = ACKNOWLEDGEMENT_STEP_KEY; + status(localize('chat.feedbackSurvey.submitted', "Feedback submitted. Thank you.")); + // Submitting removed the control that had focus, so move it to the one left. + closeButton.focus(); + } + return; + } + + const body = dom.append(panel, dom.$('.chat-feedback-survey-body')); + const firstControl = step.kind === ChatModelFeedbackSurveyStepKind.Choice + ? this.renderChoiceStep(response, body, state.instanceId, step.id, step.options, step.title) + : this.renderTextStep(response, state, body, step); + + if (state.config.steps.length > 1) { + const progress = dom.append(panel, dom.$('.chat-feedback-survey-progress')); + progress.textContent = localize('chat.feedbackSurvey.progress', "Step {0} of {1}", state.stepIndex + 1, state.config.steps.length); + } + + const stepKey = `${state.instanceId}:${state.stepIndex}`; + if (this.lastFocusedStep !== stepKey) { + this.lastFocusedStep = stepKey; + status(localize('chat.feedbackSurvey.stepAnnouncement', "{0}. Step {1} of {2}.", step.title, state.stepIndex + 1, state.config.steps.length)); + // A survey the user asked for takes focus, one that appeared on its own does not. + if (state.openTrigger === 'manual' || state.stepIndex > 0) { + firstControl?.focus(); + } + } + } + + private renderCloseButton(header: HTMLElement): Button { + const label = localize('chat.feedbackSurvey.dismiss', "Dismiss Survey"); + const close = this.renderDisposables.add(new Button(header, { ...defaultButtonStyles, secondary: true, supportIcons: true })); + close.label = `$(${Codicon.closeSmall.id})`; + close.element.classList.add('chat-feedback-survey-close'); + close.element.setAttribute('aria-label', label); + this.renderDisposables.add(this.hoverService.setupDelayedHover(close.element, { content: label })); + this.renderDisposables.add(close.onDidClick(() => this.dismiss())); + return close; + } + + /** Renders the options as a single select list, matching the ask question tool. */ + private renderChoiceStep(response: IChatResponseViewModel, body: HTMLElement, instanceId: string, stepId: string, options: readonly { id: string; label: string }[], title: string): HTMLElement { + const list = dom.append(body, dom.$('.chat-feedback-survey-list')); + list.setAttribute('role', 'listbox'); + list.setAttribute('aria-label', title); + list.tabIndex = 0; + + const items: HTMLElement[] = []; + let activeIndex = 0; + + const setActive = (index: number) => { + activeIndex = index; + items.forEach((item, i) => { + const isActive = i === index; + item.classList.toggle('active', isActive); + item.setAttribute('aria-selected', String(isActive)); + }); + list.setAttribute('aria-activedescendant', items[index].id); + }; + + options.forEach((option, index) => { + const item = dom.append(list, dom.$('.chat-feedback-survey-list-item')); + item.id = `chat-feedback-survey-option-${instanceId}-${stepId}-${index}`; + item.setAttribute('role', 'option'); + item.setAttribute('aria-selected', 'false'); + + const label = dom.append(item, dom.$('.chat-feedback-survey-list-label')); + label.textContent = option.label; + + this.renderDisposables.add(dom.addDisposableListener(item, dom.EventType.CLICK, e => { + dom.EventHelper.stop(e, true); + this.surveyService.answerChoice(response, stepId, option.id); + })); + items.push(item); + }); + + setActive(0); + + this.renderDisposables.add(dom.addDisposableListener(list, dom.EventType.KEY_DOWN, e => { + const event = new StandardKeyboardEvent(e); + if (event.keyCode === KeyCode.DownArrow) { + event.preventDefault(); + setActive(activeIndex === items.length - 1 ? 0 : activeIndex + 1); + } else if (event.keyCode === KeyCode.UpArrow) { + event.preventDefault(); + setActive(activeIndex === 0 ? items.length - 1 : activeIndex - 1); + } else if (event.keyCode === KeyCode.Home) { + event.preventDefault(); + setActive(0); + } else if (event.keyCode === KeyCode.End) { + event.preventDefault(); + setActive(items.length - 1); + } else if (event.keyCode === KeyCode.Enter || event.keyCode === KeyCode.Space) { + event.preventDefault(); + this.surveyService.answerChoice(response, stepId, options[activeIndex].id); + } + })); + + return list; + } + + private renderTextStep(response: IChatResponseViewModel, state: IChatModelFeedbackSurveyState, body: HTMLElement, step: IChatModelFeedbackSurveyTextStep): HTMLElement { + const inputBox = this.renderDisposables.add(new InputBox(body, undefined, { + placeholder: step.placeholder, + ariaLabel: step.title, + inputBoxStyles: defaultInputBoxStyles, + flexibleHeight: true, + flexibleMaxHeight: TEXT_INPUT_MAX_HEIGHT, + })); + inputBox.value = state.commentDraft || state.answers.get(step.id) || ''; + inputBox.inputElement.maxLength = step.maxLength; + this.renderDisposables.add(inputBox.onDidChange(value => this.surveyService.setCommentDraft(response, value))); + + const actions = dom.append(body, dom.$('.chat-feedback-survey-actions')); + const submit = this.renderDisposables.add(new Button(actions, { ...defaultButtonStyles })); + submit.label = localize('chat.feedbackSurvey.submit', "Submit"); + this.renderDisposables.add(submit.onDidClick(() => this.surveyService.submit(response, inputBox.value))); + + return inputBox.inputElement; + } + + private dismiss(): void { + if (!this.response || !this.isPanelOpen) { + return; + } + this.surveyService.dismiss(this.response); + } +} diff --git a/src/vs/workbench/contrib/chat/browser/feedbackSurvey/media/chatModelFeedbackSurvey.css b/src/vs/workbench/contrib/chat/browser/feedbackSurvey/media/chatModelFeedbackSurvey.css new file mode 100644 index 0000000000000..11ab45dd98437 --- /dev/null +++ b/src/vs/workbench/contrib/chat/browser/feedbackSurvey/media/chatModelFeedbackSurvey.css @@ -0,0 +1,160 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/* Matches the ask question tool so the two inline surfaces read as one family. */ + +.chat-feedback-survey-widget.hidden { + display: none; +} + +.chat-feedback-survey-container { + display: flex; + flex-direction: column; + margin: 8px 0; + border: var(--vscode-strokeThickness) solid var(--vscode-input-border, var(--vscode-chat-requestBorder)); + border-radius: var(--vscode-cornerRadius-large); + background-color: var(--vscode-panel-background); + overflow: hidden; +} + +.chat-feedback-survey-container:focus-within { + border-color: var(--vscode-focusBorder); +} + +/* In the agents window and the editor the surface is the editor background. */ +.agent-sessions-workbench .chat-feedback-survey-container, +.editor-instance .chat-feedback-survey-container { + background-color: var(--vscode-editor-background); +} + +.chat-feedback-survey-header { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: var(--vscode-spacing-size80); + padding: var(--vscode-spacing-size80) var(--vscode-spacing-size80) var(--vscode-spacing-size80) var(--vscode-spacing-size160); + border-bottom: 1px solid var(--vscode-chat-requestBorder); +} + +.chat-feedback-survey-title { + flex: 1; + min-width: 0; + margin: 0; + font-size: var(--vscode-fontSize-heading3); + font-weight: var(--vscode-fontWeight-semiBold); + line-height: 1.4; + overflow-wrap: anywhere; +} + +/* Chrome free, matching the close button on the ask question tool. */ +.chat-feedback-survey-container .monaco-button.chat-feedback-survey-close { + flex-shrink: 0; + width: 22px; + min-width: 22px; + height: 22px; + padding: 0; + border: none !important; + box-shadow: none !important; + background: transparent !important; + color: var(--vscode-icon-foreground) !important; +} + +.chat-feedback-survey-container .monaco-button.chat-feedback-survey-close:hover:not(.disabled) { + background: var(--vscode-toolbar-hoverBackground) !important; +} + +.chat-feedback-survey-body { + display: flex; + flex-direction: column; + gap: var(--vscode-spacing-size80); + padding: var(--vscode-spacing-size80); +} + +.chat-feedback-survey-list { + display: flex; + flex-direction: column; +} + +/* + * The card border already shows focus, so the list must not draw a second ring. This needs to + * outrank the workbench rule for `[tabindex="0"]:focus` in style.css, hence the extra scoping. + */ +.chat-feedback-survey-widget .chat-feedback-survey-container .chat-feedback-survey-list:focus, +.chat-feedback-survey-widget .chat-feedback-survey-container .chat-feedback-survey-list-item:focus { + outline: none; +} + +.chat-feedback-survey-list-item { + display: flex; + align-items: center; + gap: var(--vscode-spacing-size80); + padding: var(--vscode-spacing-size60) var(--vscode-spacing-size80); + border-radius: var(--vscode-cornerRadius-medium); + cursor: pointer; + user-select: none; +} + +.chat-feedback-survey-list-label { + flex: 1; + font-weight: var(--vscode-fontWeight-semiBold); + line-height: 1.4; + overflow-wrap: break-word; +} + +.chat-feedback-survey-list-item:hover { + background-color: var(--vscode-list-hoverBackground); +} + +.chat-feedback-survey-list-item.active { + background-color: var(--vscode-list-inactiveSelectionBackground, var(--vscode-list-hoverBackground)); + color: var(--vscode-list-inactiveSelectionForeground, var(--vscode-foreground)); +} + +.chat-feedback-survey-list:focus .chat-feedback-survey-list-item.active { + background-color: var(--vscode-list-activeSelectionBackground, var(--vscode-list-hoverBackground)); + color: var(--vscode-list-activeSelectionForeground, var(--vscode-foreground)); +} + +.chat-feedback-survey-actions { + display: flex; + justify-content: flex-end; +} + +.chat-feedback-survey-progress { + padding: 0 var(--vscode-spacing-size80) var(--vscode-spacing-size80); + color: var(--vscode-descriptionForeground); + font-size: var(--vscode-fontSize-label2); +} + +/* + * The combined thumbs control in the response footer. Both glyphs sit in one control so it + * reads as a single way to give feedback rather than two competing verdicts. The class is on + * the action-label anchor, which is the focusable element the toolbar renders. + */ +.action-label.chat-feedback-survey-pill { + width: auto; +} + +.action-label.chat-feedback-survey-pill .chat-feedback-survey-pill-icons { + display: flex; + align-items: center; +} + +.action-label.chat-feedback-survey-pill .chat-feedback-survey-pill-icon::before { + font-size: var(--vscode-codiconFontSize-compact); +} + +/* + * The glyphs overlap and sit on a slight diagonal so the pair reads as one icon on a single + * small button rather than as two separate votes. + */ +.action-label.chat-feedback-survey-pill .chat-feedback-survey-pill-icon:first-child { + transform: translateY(-2px); +} + +.action-label.chat-feedback-survey-pill .chat-feedback-survey-pill-icon:last-child { + margin-left: -5px; + transform: translateY(2px); +} diff --git a/src/vs/workbench/contrib/chat/browser/widget/chatListRenderer.ts b/src/vs/workbench/contrib/chat/browser/widget/chatListRenderer.ts index fab5aca8fb37b..778643d1183e0 100644 --- a/src/vs/workbench/contrib/chat/browser/widget/chatListRenderer.ts +++ b/src/vs/workbench/contrib/chat/browser/widget/chatListRenderer.ts @@ -67,7 +67,9 @@ import { formatChatRequestTimestamp, formatChatResponseDetails, formatChatRespon import { ClickAnimation } from '../../../../../base/browser/ui/animations/animations.js'; import { ForkConversationActionId } from '../actions/chatForkActions.js'; import { MarkHelpfulActionId } from '../actions/chatTitleActions.js'; +import { focusChatModelFeedbackSurveyAction } from '../actions/chatModelFeedbackSurveyActions.js'; import { ChatTreeItem, IChatCodeBlockInfo, IChatFileTreeInfo, IChatListItemRendererOptions, IChatWidgetService } from '../chat.js'; +import { ChatModelFeedbackSurveyWidget } from '../feedbackSurvey/chatModelFeedbackSurveyWidget.js'; import { AgentHostSnapshotController } from '../agentSessions/agentHost/agentHostSnapshotController.js'; import { RestoreCheckpointActionId, StartOverActionId } from '../chatEditing/chatEditingActions.js'; import { ChatForkActionViewItem } from './chatForkActionViewItem.js'; @@ -192,6 +194,8 @@ export interface IChatListItemTemplate { readonly checkpointRestoreToolbar: MenuWorkbenchToolBar; readonly checkpointContainer: HTMLElement; readonly checkpointRestoreContainer: HTMLElement; + /** Inline model feedback survey shown beneath the footer, when an experiment offers one. */ + readonly feedbackSurveyWidget: ChatModelFeedbackSurveyWidget; } function escapeMarkdownLinkLabel(label: string): string { @@ -1040,6 +1044,10 @@ export class ChatListItemRenderer extends Disposable implements ITreeRenderer focusChatModelFeedbackSurveyAction(footerToolbar))); + const checkpointRestoreContainer = dom.append(rowContainer, $('.checkpoint-restore-container')); dom.append(checkpointRestoreContainer, $('.checkpoint-line-left')); const label = dom.append(checkpointRestoreContainer, $('span.checkpoint-label-text')); @@ -1090,7 +1098,7 @@ export class ChatListItemRenderer extends Disposable implements ITreeRenderer { @@ -1250,8 +1258,10 @@ export class ChatListItemRenderer extends Disposable implements ITreeRenderer('chatResponseSupportsIssueReporting', false, { type: 'boolean', description: localize('chatResponseSupportsIssueReporting', "True when the current chat response supports issue reporting.") }); export const responseIsFiltered = new RawContextKey('chatSessionResponseFiltered', false, { type: 'boolean', description: localize('chatResponseFiltered', "True when the chat response was filtered out by the server.") }); export const responseHasError = new RawContextKey('chatSessionResponseError', false, { type: 'boolean', description: localize('chatResponseErrored', "True when the chat response resulted in an error.") }); + export const responseHasFeedbackSurvey = new RawContextKey('chatSessionResponseHasFeedbackSurvey', false, { type: 'boolean', description: localize('chatResponseHasFeedbackSurvey', "True when an inline model feedback survey is offered for the chat response, which replaces the helpful and unhelpful actions.") }); + export const responseFeedbackSurveyOpen = new RawContextKey('chatSessionResponseFeedbackSurveyOpen', false, { type: 'boolean', description: localize('chatResponseFeedbackSurveyOpen', "True when the inline model feedback survey is showing for the chat response.") }); export const requestInProgress = new RawContextKey('chatSessionRequestInProgress', false, { type: 'boolean', description: localize('interactiveSessionRequestInProgress', "True when the current request is still in progress.") }); export const hasActiveRequest = new RawContextKey('chatSessionHasActiveRequest', false, { type: 'boolean', description: localize('chatSessionHasActiveRequest', "True when the current chat response has not completed, regardless of intermediate states like tool calls or elicitations.") }); export const currentlyEditing = new RawContextKey('chatSessionCurrentlyEditing', false, { type: 'boolean', description: localize('interactiveSessionCurrentlyEditing', "True when the current request is being edited.") }); diff --git a/src/vs/workbench/contrib/chat/common/feedbackSurvey/chatModelFeedbackSurveyConfig.ts b/src/vs/workbench/contrib/chat/common/feedbackSurvey/chatModelFeedbackSurveyConfig.ts new file mode 100644 index 0000000000000..acf05a5f6513e --- /dev/null +++ b/src/vs/workbench/contrib/chat/common/feedbackSurvey/chatModelFeedbackSurveyConfig.ts @@ -0,0 +1,568 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** + * Contract for the inline model feedback survey. + * + * A survey is fully described by a versioned JSON payload delivered as an experiment treatment, + * so one can be authored or retired without shipping code. The shapes stay close to the editor + * pane survey in `contrib/surveys/browser/surveyQuestions.ts` so the two can converge later, + * but cannot share code today because that renderer needs telemetry keys known at compile time. + */ + +/** Payload versions this build understands. Bump when making a breaking shape change. */ +export const CHAT_MODEL_FEEDBACK_SURVEY_CONFIG_VERSION = 1; + +/** Sentinel matching sessions that are not backed by an agent host. */ +export const CHAT_MODEL_FEEDBACK_SURVEY_NO_HARNESS = 'none'; + +const MAX_STEPS = 8; +const MIN_OPTIONS = 2; +const MAX_OPTIONS = 8; +const MAX_ID_LENGTH = 64; +const MAX_TITLE_LENGTH = 200; +const MAX_LABEL_LENGTH = 120; +const MAX_PLACEHOLDER_LENGTH = 100; +const MAX_COMMENT_LENGTH = 1000; +const MAX_SELECTORS = 32; + +const MATCH_FIELDS = ['selectedModels', 'resolvedModels', 'modes', 'harnesses', 'sessionTypes'] as const; + +/** Ids appear in telemetry, so they are restricted to a shape that needs no sanitization. */ +const ID_PATTERN = /^[a-z0-9][a-z0-9._-]*$/; + +export const enum ChatModelFeedbackSurveyStepKind { + Choice = 'choice', + Text = 'text', +} + +export interface IChatModelFeedbackSurveyOption { + readonly id: string; + readonly label: string; +} + +interface IChatModelFeedbackSurveyStepBase { + readonly id: string; + readonly title: string; +} + +export interface IChatModelFeedbackSurveyChoiceStep extends IChatModelFeedbackSurveyStepBase { + readonly kind: ChatModelFeedbackSurveyStepKind.Choice; + readonly options: readonly IChatModelFeedbackSurveyOption[]; +} + +export interface IChatModelFeedbackSurveyTextStep extends IChatModelFeedbackSurveyStepBase { + readonly kind: ChatModelFeedbackSurveyStepKind.Text; + readonly placeholder?: string; + readonly maxLength: number; +} + +export type ChatModelFeedbackSurveyStep = IChatModelFeedbackSurveyChoiceStep | IChatModelFeedbackSurveyTextStep; + +/** + * Which responses a survey attaches to. An omitted or empty selector list means any. + * + * Selected and resolved models are matched separately on purpose. A survey about Auto routing + * targets the selected model `auto`, and must not fire just because another request happened to + * be routed to the same model. + */ +export interface IChatModelFeedbackSurveyMatch { + readonly selectedModels: readonly string[]; + readonly resolvedModels: readonly string[]; + readonly modes: readonly string[]; + /** Agent host provider ids (e.g. `copilotcli`), or {@link CHAT_MODEL_FEEDBACK_SURVEY_NO_HARNESS}. */ + readonly harnesses: readonly string[]; + readonly sessionTypes: readonly string[]; +} + +/** + * Rules governing when the survey opens *by itself*. + * + * None of this applies to manual activation: clicking the feedback control is an explicit + * request for the survey and always opens it. These rules exist only to keep unprompted + * surfacing rare enough not to be a nuisance. + */ +export interface IChatModelFeedbackSurveyPrompt { + /** Minimum days between two automatic prompts. `0` disables the cooldown. */ + readonly cooldownDays: number; + /** How many times the survey may open itself within one chat session. */ + readonly maxPerSession: number; + readonly chance: IChatModelFeedbackSurveyChance; + readonly triggers: IChatModelFeedbackSurveyTriggers; +} + +/** + * A probability that ramps with usage. Every eligible response that does not prompt raises the + * odds up to {@link IChatModelFeedbackSurveyChance.max}, so heavier users are asked sooner. The + * odds reset once a prompt is shown. + */ +export interface IChatModelFeedbackSurveyChance { + /** Probability applied to the first eligible response. `0` disables random prompting. */ + readonly initial: number; + /** Added to the probability for each eligible response that did not prompt. */ + readonly increment: number; + /** Ceiling the ramped probability cannot exceed. */ + readonly max: number; +} + +/** Moments that prompt directly, without a probability roll. */ +export interface IChatModelFeedbackSurveyTriggers { + /** Fires when the user switches the picker off a model this survey matches. */ + readonly modelSwitchedAway: IChatModelFeedbackSurveyTrigger; +} + +export interface IChatModelFeedbackSurveyTrigger { + readonly enabled: boolean; + /** Whether the trigger prompts even inside the cooldown window. */ + readonly bypassCooldown: boolean; +} + +export interface IChatModelFeedbackSurveyConfig { + readonly version: number; + readonly id: string; + readonly match: IChatModelFeedbackSurveyMatch; + readonly prompt: IChatModelFeedbackSurveyPrompt; + readonly steps: readonly ChatModelFeedbackSurveyStep[]; +} + +export type ChatModelFeedbackSurveyParseResult = + | { readonly config: IChatModelFeedbackSurveyConfig; readonly error?: undefined } + | { readonly config?: undefined; readonly error: string }; + +/** Describes the response a survey is matched against. The caller resolves any model aliases. */ +export interface IChatModelFeedbackSurveyMatchContext { + /** The model identifier the user selected, as recorded on the request. */ + readonly selectedModelId?: string; + /** Other identifiers for the selected model, such as its id, family, name and vendor. */ + readonly selectedModelAliases?: readonly string[]; + /** The model a routing layer (e.g. Auto) actually resolved to, when different. */ + readonly resolvedModelId?: string; + readonly modeId?: string; + /** Agent host provider id, or `undefined` for sessions with no agent host. */ + readonly harness?: string; + readonly sessionType?: string; +} + +/** + * Parses and validates a survey payload. Never throws, and rejects a bad config whole rather + * than in part, since dropping one malformed step would quietly change what the experiment + * measures. + */ +export function parseChatModelFeedbackSurveyConfig(raw: string | undefined): ChatModelFeedbackSurveyParseResult { + if (typeof raw !== 'string' || !raw.trim()) { + return { error: 'empty payload' }; + } + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (err) { + return { error: `payload is not valid JSON: ${err instanceof Error ? err.message : String(err)}` }; + } + + if (!isObject(parsed)) { + return { error: 'payload is not an object' }; + } + + if (parsed.version !== CHAT_MODEL_FEEDBACK_SURVEY_CONFIG_VERSION) { + return { error: `unsupported version ${JSON.stringify(parsed.version)}, expected ${CHAT_MODEL_FEEDBACK_SURVEY_CONFIG_VERSION}` }; + } + + const id = readId(parsed.id); + if (!id) { + return { error: 'missing or malformed survey id' }; + } + + const match = readMatch(parsed.match); + if (typeof match === 'string') { + return { error: match }; + } + + const prompt = readPrompt(parsed.prompt); + if (typeof prompt === 'string') { + return { error: prompt }; + } + + const steps = readSteps(parsed.steps); + if (typeof steps === 'string') { + return { error: steps }; + } + + return { config: { version: CHAT_MODEL_FEEDBACK_SURVEY_CONFIG_VERSION, id, match, prompt, steps } }; +} + +function readMatch(raw: unknown): IChatModelFeedbackSurveyMatch | string { + if (raw !== undefined && !isObject(raw)) { + return 'match must be an object'; + } + const source = isObject(raw) ? raw : {}; + + const match: Record = {}; + for (const field of MATCH_FIELDS) { + const selectors = readSelectorList(source[field], `match.${field}`); + if (typeof selectors === 'string') { + return selectors; + } + match[field] = selectors; + } + + if (MATCH_FIELDS.every(field => match[field].length === 0)) { + return 'match must narrow at least one dimension'; + } + + return { + selectedModels: match.selectedModels, + resolvedModels: match.resolvedModels, + modes: match.modes, + harnesses: match.harnesses, + sessionTypes: match.sessionTypes, + }; +} + +function readSelectorList(raw: unknown, path: string): string[] | string { + if (raw === undefined) { + return []; + } + if (!Array.isArray(raw)) { + return `${path} must be an array of strings`; + } + if (raw.length > MAX_SELECTORS) { + return `${path} exceeds ${MAX_SELECTORS} entries`; + } + const out: string[] = []; + for (const entry of raw) { + if (typeof entry !== 'string') { + return `${path} must contain only strings`; + } + const normalized = normalizeSelector(entry); + if (!normalized) { + return `${path} must not contain empty strings`; + } + out.push(normalized); + } + return out; +} + +/** + * Reads the automatic prompting rules. An omitted `prompt` block gives a manual only survey, so + * an experiment that forgets to describe its pacing under prompts rather than nags. + */ +function readPrompt(raw: unknown): IChatModelFeedbackSurveyPrompt | string { + if (raw !== undefined && !isObject(raw)) { + return 'prompt must be an object'; + } + const source = isObject(raw) ? raw : {}; + + const cooldownDays = readNonNegativeNumber(source.cooldownDays, 7); + if (cooldownDays === undefined) { + return 'prompt.cooldownDays must be a non-negative number'; + } + + const maxPerSession = readPositiveInteger(source.maxPerSession, 1); + if (maxPerSession === undefined) { + return 'prompt.maxPerSession must be a positive integer'; + } + + const chance = readChance(source.chance); + if (typeof chance === 'string') { + return chance; + } + + const triggers = readTriggers(source.triggers); + if (typeof triggers === 'string') { + return triggers; + } + + return { cooldownDays, maxPerSession, chance, triggers }; +} + +function readChance(raw: unknown): IChatModelFeedbackSurveyChance | string { + if (raw !== undefined && !isObject(raw)) { + return 'prompt.chance must be an object'; + } + const source = isObject(raw) ? raw : {}; + + const initial = readProbability(source.initial, 0); + if (initial === undefined) { + return 'prompt.chance.initial must be a probability between 0 and 1'; + } + const increment = readProbability(source.increment, 0); + if (increment === undefined) { + return 'prompt.chance.increment must be a probability between 0 and 1'; + } + const max = readProbability(source.max, 1); + if (max === undefined) { + return 'prompt.chance.max must be a probability between 0 and 1'; + } + if (max < initial) { + return 'prompt.chance.max must be greater than or equal to prompt.chance.initial'; + } + + return { initial, increment, max }; +} + +function readTriggers(raw: unknown): IChatModelFeedbackSurveyTriggers | string { + if (raw !== undefined && !isObject(raw)) { + return 'prompt.triggers must be an object'; + } + const source = isObject(raw) ? raw : {}; + + const modelSwitchedAway = readTrigger(source.modelSwitchedAway, 'prompt.triggers.modelSwitchedAway'); + if (typeof modelSwitchedAway === 'string') { + return modelSwitchedAway; + } + + return { modelSwitchedAway }; +} + +function readTrigger(raw: unknown, path: string): IChatModelFeedbackSurveyTrigger | string { + if (raw === undefined) { + return { enabled: false, bypassCooldown: false }; + } + // `true` is accepted as shorthand for an enabled trigger that still respects the cooldown. + if (typeof raw === 'boolean') { + return { enabled: raw, bypassCooldown: false }; + } + if (!isObject(raw)) { + return `${path} must be a boolean or an object`; + } + if (raw.enabled !== undefined && typeof raw.enabled !== 'boolean') { + return `${path}.enabled must be a boolean`; + } + if (raw.bypassCooldown !== undefined && typeof raw.bypassCooldown !== 'boolean') { + return `${path}.bypassCooldown must be a boolean`; + } + return { enabled: raw.enabled ?? true, bypassCooldown: raw.bypassCooldown ?? false }; +} + +function readSteps(raw: unknown): ChatModelFeedbackSurveyStep[] | string { + if (!Array.isArray(raw) || raw.length === 0) { + return 'steps must be a non-empty array'; + } + if (raw.length > MAX_STEPS) { + return `steps exceeds ${MAX_STEPS} entries`; + } + + const steps: ChatModelFeedbackSurveyStep[] = []; + const seenIds = new Set(); + + for (let i = 0; i < raw.length; i++) { + const step = readStep(raw[i], i); + if (typeof step === 'string') { + return step; + } + if (seenIds.has(step.id)) { + return `steps[${i}].id "${step.id}" is duplicated`; + } + seenIds.add(step.id); + steps.push(step); + } + + // A text step is terminal because it carries the Submit button, so one in the middle would + // make every later step unreachable. + const textStepIndexes = steps.map((step, index) => step.kind === ChatModelFeedbackSurveyStepKind.Text ? index : -1).filter(index => index >= 0); + if (textStepIndexes.length > 1) { + return 'steps may contain at most one text step'; + } + if (textStepIndexes.length === 1 && textStepIndexes[0] !== steps.length - 1) { + return 'a text step must be the last step'; + } + + return steps; +} + +function readStep(raw: unknown, index: number): ChatModelFeedbackSurveyStep | string { + if (!isObject(raw)) { + return `steps[${index}] must be an object`; + } + + const id = readId(raw.id); + if (!id) { + return `steps[${index}].id is missing or malformed`; + } + + const title = readText(raw.title, MAX_TITLE_LENGTH); + if (!title) { + return `steps[${index}].title is missing or too long`; + } + + if (raw.kind === ChatModelFeedbackSurveyStepKind.Text) { + const placeholder = raw.placeholder === undefined ? undefined : readText(raw.placeholder, MAX_PLACEHOLDER_LENGTH); + if (raw.placeholder !== undefined && !placeholder) { + return `steps[${index}].placeholder is empty or too long`; + } + const requestedMaxLength = readPositiveInteger(raw.maxLength, MAX_COMMENT_LENGTH); + if (requestedMaxLength === undefined) { + return `steps[${index}].maxLength must be a positive integer`; + } + return { + kind: ChatModelFeedbackSurveyStepKind.Text, + id, + title, + placeholder, + maxLength: Math.min(requestedMaxLength, MAX_COMMENT_LENGTH), + }; + } + + if (raw.kind !== ChatModelFeedbackSurveyStepKind.Choice) { + return `steps[${index}].kind must be "${ChatModelFeedbackSurveyStepKind.Choice}" or "${ChatModelFeedbackSurveyStepKind.Text}"`; + } + + if (!Array.isArray(raw.options) || raw.options.length < MIN_OPTIONS || raw.options.length > MAX_OPTIONS) { + return `steps[${index}].options must have between ${MIN_OPTIONS} and ${MAX_OPTIONS} entries`; + } + + const options: IChatModelFeedbackSurveyOption[] = []; + const seenOptionIds = new Set(); + for (let i = 0; i < raw.options.length; i++) { + const option = raw.options[i]; + if (!isObject(option)) { + return `steps[${index}].options[${i}] must be an object`; + } + const optionId = readId(option.id); + if (!optionId) { + return `steps[${index}].options[${i}].id is missing or malformed`; + } + if (seenOptionIds.has(optionId)) { + return `steps[${index}].options[${i}].id "${optionId}" is duplicated`; + } + const label = readText(option.label, MAX_LABEL_LENGTH); + if (!label) { + return `steps[${index}].options[${i}].label is missing or too long`; + } + seenOptionIds.add(optionId); + options.push({ id: optionId, label }); + } + + return { kind: ChatModelFeedbackSurveyStepKind.Choice, id, title, options }; +} + +function readId(raw: unknown): string | undefined { + if (typeof raw !== 'string') { + return undefined; + } + const trimmed = raw.trim().toLowerCase(); + if (!trimmed || trimmed.length > MAX_ID_LENGTH || !ID_PATTERN.test(trimmed)) { + return undefined; + } + return trimmed; +} + +function readText(raw: unknown, maxLength: number): string | undefined { + if (typeof raw !== 'string') { + return undefined; + } + const trimmed = raw.trim(); + if (!trimmed || trimmed.length > maxLength) { + return undefined; + } + return trimmed; +} + +function readPositiveInteger(raw: unknown, fallback: number): number | undefined { + if (raw === undefined) { + return fallback; + } + if (typeof raw !== 'number' || !Number.isInteger(raw) || raw <= 0) { + return undefined; + } + return raw; +} + +function readNonNegativeNumber(raw: unknown, fallback: number): number | undefined { + if (raw === undefined) { + return fallback; + } + if (typeof raw !== 'number' || !Number.isFinite(raw) || raw < 0) { + return undefined; + } + return raw; +} + +function readProbability(raw: unknown, fallback: number): number | undefined { + if (raw === undefined) { + return fallback; + } + if (typeof raw !== 'number' || !Number.isFinite(raw) || raw < 0 || raw > 1) { + return undefined; + } + return raw; +} + +function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** + * Builds every string a selector may match for a model. + * + * Identifiers are qualified differently across harnesses. The language model service uses + * `//` while agent host sessions use `:`, so a selector is + * compared against each segment as well as the whole id. That lets `auto` match both + * `copilot/auto` and `agent-host-copilotcli:auto`. + */ +export function expandModelMatchCandidates(modelId: string | undefined, aliases?: readonly string[]): Set { + const candidates = new Set(); + const add = (value: string | undefined): void => { + const normalized = value === undefined ? '' : normalizeSelector(value); + if (normalized) { + candidates.add(normalized); + } + }; + + if (modelId) { + add(modelId); + for (const segment of modelId.split(/[/:]/)) { + add(segment); + } + } + for (const alias of aliases ?? []) { + add(alias); + } + + return candidates; +} + +/** Whether the response described by `context` should be offered `config`'s survey. */ +export function matchesChatModelFeedbackSurvey(config: IChatModelFeedbackSurveyConfig, context: IChatModelFeedbackSurveyMatchContext): boolean { + const { match } = config; + + if (match.selectedModels.length) { + const candidates = expandModelMatchCandidates(context.selectedModelId, context.selectedModelAliases); + if (!match.selectedModels.some(selector => candidates.has(selector))) { + return false; + } + } + + if (match.resolvedModels.length) { + const candidates = expandModelMatchCandidates(context.resolvedModelId); + if (!match.resolvedModels.some(selector => candidates.has(selector))) { + return false; + } + } + + if (match.modes.length && !matchesScalar(match.modes, context.modeId)) { + return false; + } + + if (match.harnesses.length && !matchesScalar(match.harnesses, context.harness ?? CHAT_MODEL_FEEDBACK_SURVEY_NO_HARNESS)) { + return false; + } + + if (match.sessionTypes.length && !matchesScalar(match.sessionTypes, context.sessionType)) { + return false; + } + + return true; +} + +function matchesScalar(selectors: readonly string[], value: string | undefined): boolean { + const normalized = value === undefined ? undefined : normalizeSelector(value); + return !!normalized && selectors.includes(normalized); +} + +function normalizeSelector(value: string): string { + return value.trim().toLowerCase().replace(/[\s_]+/g, '-'); +} diff --git a/src/vs/workbench/contrib/chat/common/feedbackSurvey/chatModelFeedbackSurveyTelemetry.ts b/src/vs/workbench/contrib/chat/common/feedbackSurvey/chatModelFeedbackSurveyTelemetry.ts new file mode 100644 index 0000000000000..2edeb8ddef31c --- /dev/null +++ b/src/vs/workbench/contrib/chat/common/feedbackSurvey/chatModelFeedbackSurveyTelemetry.ts @@ -0,0 +1,49 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** + * Wire contract between the workbench and the Copilot extension for survey telemetry. + * + * Answers must reach GitHub restricted telemetry, which only the Copilot extension can send to. + * A command is used rather than a data channel because `executeCommand` activates the extension, + * so results produced before activation are not dropped. Command ids are first come first + * served, so this routes the payload rather than securing it. Keep the shape in sync with + * `chatModelFeedbackSurveyForwardingContrib.ts` in the extension. + */ +export const CHAT_MODEL_FEEDBACK_SURVEY_TELEMETRY_COMMAND_ID = '_github.copilot.chat.reportModelFeedbackSurvey'; + +export type ChatModelFeedbackSurveyEventKind = + /** The pill became available on a response. */ + | 'shown' + /** The user opened the survey panel. */ + | 'opened' + /** A step was answered. Sent as it happens so abandoned surveys still report. */ + | 'step' + /** The user submitted on the final step. */ + | 'submitted' + /** The user dismissed the survey without submitting. */ + | 'dismissed'; + +export interface IChatModelFeedbackSurveyTelemetryEvent { + readonly kind: ChatModelFeedbackSurveyEventKind; + readonly surveyId: string; + /** Stitches the events for one survey together. Minted when the survey first applies. */ + readonly surveyInstanceId: string; + readonly stepCount: number; + /** What opened the survey, so asked for and unprompted surveys can be measured apart. */ + readonly trigger?: 'manual' | 'chance' | 'modelSwitchedAway'; + readonly stepId?: string; + readonly stepIndex?: number; + /** The chosen option id for a `choice` step. Always one of the configured option ids. */ + readonly answerId?: string; + /** Free text from a text step. Must only reach GitHub restricted telemetry, never `publicLog2`. */ + readonly comment?: string; + readonly modelId?: string; + readonly resolvedModelId?: string; + readonly modeId?: string; + readonly harness?: string; + readonly sessionType?: string; + readonly requestId: string; +} diff --git a/src/vs/workbench/contrib/chat/common/model/chatViewModel.ts b/src/vs/workbench/contrib/chat/common/model/chatViewModel.ts index a6ed9382dc127..7e094509e441b 100644 --- a/src/vs/workbench/contrib/chat/common/model/chatViewModel.ts +++ b/src/vs/workbench/contrib/chat/common/model/chatViewModel.ts @@ -246,6 +246,8 @@ export interface IChatResponseViewModel { readonly isComplete: boolean; readonly isCanceled: boolean; readonly isStale: boolean; + /** Whether this is the last row in the transcript. */ + readonly isLast: boolean; readonly vote: ChatAgentVoteDirection | undefined; readonly replyFollowups?: IChatFollowup[]; readonly errorDetails?: IChatResponseErrorDetails; diff --git a/src/vs/workbench/contrib/chat/test/browser/feedbackSurvey/chatModelFeedbackSurveyActions.test.ts b/src/vs/workbench/contrib/chat/test/browser/feedbackSurvey/chatModelFeedbackSurveyActions.test.ts new file mode 100644 index 0000000000000..5ffbf289b347d --- /dev/null +++ b/src/vs/workbench/contrib/chat/test/browser/feedbackSurvey/chatModelFeedbackSurveyActions.test.ts @@ -0,0 +1,40 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { IAction } from '../../../../../../base/common/actions.js'; +import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../base/test/common/utils.js'; +import { ChatModelFeedbackSurveyActionId, focusChatModelFeedbackSurveyAction, IFeedbackSurveyToolBar } from '../../../browser/actions/chatModelFeedbackSurveyActions.js'; + +suite('ChatModelFeedbackSurveyActions', () => { + + ensureNoDisposablesAreLeakedInTestSuite(); + + function createToolBar(actionIds: readonly string[]): IFeedbackSurveyToolBar & { focused: number | undefined } { + return { + focused: undefined, + getItemsLength: () => actionIds.length, + getItemAction: (index: number) => ({ id: actionIds[index] } as IAction), + focus(index?: number) { this.focused = index; }, + }; + } + + test('focuses the feedback control rather than whichever action comes first', () => { + // The copy action sits before the survey control in the response footer. + const toolbar = createToolBar(['workbench.action.chat.copyItem', ChatModelFeedbackSurveyActionId, 'workbench.action.chat.reportIssueForBug']); + + focusChatModelFeedbackSurveyAction(toolbar); + + assert.strictEqual(toolbar.focused, 1); + }); + + test('falls back to the toolbar when the control is not shown', () => { + const toolbar = createToolBar(['workbench.action.chat.copyItem']); + + focusChatModelFeedbackSurveyAction(toolbar); + + assert.strictEqual(toolbar.focused, undefined); + }); +}); diff --git a/src/vs/workbench/contrib/chat/test/browser/feedbackSurvey/chatModelFeedbackSurveyPromptContribution.test.ts b/src/vs/workbench/contrib/chat/test/browser/feedbackSurvey/chatModelFeedbackSurveyPromptContribution.test.ts new file mode 100644 index 0000000000000..9764b785d1763 --- /dev/null +++ b/src/vs/workbench/contrib/chat/test/browser/feedbackSurvey/chatModelFeedbackSurveyPromptContribution.test.ts @@ -0,0 +1,106 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { Emitter, Event } from '../../../../../../base/common/event.js'; +import { observableValue } from '../../../../../../base/common/observable.js'; +import { URI } from '../../../../../../base/common/uri.js'; +import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../base/test/common/utils.js'; +import { TestInstantiationService } from '../../../../../../platform/instantiation/test/common/instantiationServiceMock.js'; +import { ChatModelFeedbackSurveyPromptContribution } from '../../../browser/feedbackSurvey/chatModelFeedbackSurveyPromptContribution.js'; +import { IChatModelFeedbackSurveyService } from '../../../browser/feedbackSurvey/chatModelFeedbackSurveyService.js'; +import { IChatWidget, IChatWidgetService } from '../../../browser/chat.js'; +import { ILanguageModelChatMetadataAndIdentifier } from '../../../common/languageModels.js'; +import { MockChatModelFeedbackSurveyService } from './mockChatModelFeedbackSurveyService.js'; + +suite('ChatModelFeedbackSurveyPromptContribution', () => { + + const store = ensureNoDisposablesAreLeakedInTestSuite(); + + const sessionOne = URI.parse('vscode-chat-editor://session-1'); + const sessionTwo = URI.parse('vscode-chat-editor://session-2'); + + function createHarness() { + const switches: { from: string; to: string; session: string }[] = []; + const selectedModel = observableValue('selectedModel', undefined); + const onDidChangeViewModel = store.add(new Emitter()); + let sessionResource: URI | undefined; + + const widget = { + input: { selectedLanguageModel: selectedModel }, + onDidChangeViewModel: onDidChangeViewModel.event, + get viewModel() { return sessionResource ? { sessionResource } : undefined; }, + } as unknown as IChatWidget; + + const surveyService = new MockChatModelFeedbackSurveyService(); + surveyService.notifyModelSwitchedAway = (session, from, to) => { + switches.push({ from, to, session: session.toString() }); + }; + + const instantiationService = store.add(new TestInstantiationService()); + instantiationService.stub(IChatWidgetService, { + getAllWidgets: () => [widget], + onDidAddWidget: Event.None, + onDidRemoveWidget: Event.None, + } as unknown as IChatWidgetService); + instantiationService.stub(IChatModelFeedbackSurveyService, surveyService); + store.add(instantiationService.createInstance(ChatModelFeedbackSurveyPromptContribution)); + + return { + switches, + selectModel: (identifier: string) => selectedModel.set({ identifier } as ILanguageModelChatMetadataAndIdentifier, undefined), + loadSession: (resource: URI | undefined) => { + sessionResource = resource; + onDidChangeViewModel.fire(); + }, + }; + } + + test('reports the user moving from one model to another', () => { + const harness = createHarness(); + harness.loadSession(sessionOne); + + harness.selectModel('copilot/auto'); + harness.selectModel('copilot/gpt-5.2'); + + assert.deepStrictEqual(harness.switches, [{ from: 'copilot/auto', to: 'copilot/gpt-5.2', session: sessionOne.toString() }]); + }); + + test('reports a switch when the model resolved before the session finished loading', () => { + // A widget registers, then resolves its model, then loads the session. The switch that + // follows is still the user rejecting the model. + const harness = createHarness(); + harness.selectModel('copilot/auto'); + harness.loadSession(sessionOne); + + harness.selectModel('copilot/gpt-5.2'); + + assert.deepStrictEqual(harness.switches, [{ from: 'copilot/auto', to: 'copilot/gpt-5.2', session: sessionOne.toString() }]); + }); + + test('ignores the model that comes with a newly loaded session', () => { + const harness = createHarness(); + harness.loadSession(sessionOne); + harness.selectModel('copilot/auto'); + + // Switching sessions restores that session's model, which is not the user rejecting one. + harness.loadSession(sessionTwo); + harness.selectModel('copilot/gpt-5.2'); + + assert.deepStrictEqual(harness.switches, []); + }); + + test('keeps reporting switches made after a session change', () => { + const harness = createHarness(); + harness.loadSession(sessionOne); + harness.selectModel('copilot/auto'); + harness.loadSession(sessionTwo); + harness.selectModel('copilot/auto'); + + harness.selectModel('copilot/gpt-5.2'); + + assert.deepStrictEqual(harness.switches, [{ from: 'copilot/auto', to: 'copilot/gpt-5.2', session: sessionTwo.toString() }]); + }); +}); diff --git a/src/vs/workbench/contrib/chat/test/browser/feedbackSurvey/chatModelFeedbackSurveyService.test.ts b/src/vs/workbench/contrib/chat/test/browser/feedbackSurvey/chatModelFeedbackSurveyService.test.ts new file mode 100644 index 0000000000000..3030c14d9a22b --- /dev/null +++ b/src/vs/workbench/contrib/chat/test/browser/feedbackSurvey/chatModelFeedbackSurveyService.test.ts @@ -0,0 +1,551 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { Emitter, Event } from '../../../../../../base/common/event.js'; +import { URI } from '../../../../../../base/common/uri.js'; +import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../base/test/common/utils.js'; +import { ICommandService } from '../../../../../../platform/commands/common/commands.js'; +import { IConfigurationService } from '../../../../../../platform/configuration/common/configuration.js'; +import { TestConfigurationService } from '../../../../../../platform/configuration/test/common/testConfigurationService.js'; +import { TestInstantiationService } from '../../../../../../platform/instantiation/test/common/instantiationServiceMock.js'; +import { ILogService, NullLogService } from '../../../../../../platform/log/common/log.js'; +import { InMemoryStorageService, IStorageService } from '../../../../../../platform/storage/common/storage.js'; +import { ITelemetryService, TelemetryLevel } from '../../../../../../platform/telemetry/common/telemetry.js'; +import { IAssignmentFilter, IWorkbenchAssignmentService } from '../../../../../services/assignment/common/assignmentService.js'; +import { ChatModelFeedbackSurveyService, ChatModelFeedbackSurveyStatus } from '../../../browser/feedbackSurvey/chatModelFeedbackSurveyService.js'; +import { IChatSessionsService } from '../../../common/chatSessionsService.js'; +import { IChatService } from '../../../common/chatService/chatService.js'; +import { CHAT_MODEL_FEEDBACK_SURVEY_CONFIG_VERSION } from '../../../common/feedbackSurvey/chatModelFeedbackSurveyConfig.js'; +import { IChatModelFeedbackSurveyTelemetryEvent } from '../../../common/feedbackSurvey/chatModelFeedbackSurveyTelemetry.js'; +import { ILanguageModelsService } from '../../../common/languageModels.js'; +import { IChatResponseViewModel } from '../../../common/model/chatViewModel.js'; + +suite('ChatModelFeedbackSurveyService', () => { + + const disposables = ensureNoDisposablesAreLeakedInTestSuite(); + + const steps = [ + { kind: 'choice', id: 'routing', title: 'Right model?', options: [{ id: 'yes', label: 'Yes' }, { id: 'no', label: 'No' }] }, + { kind: 'text', id: 'comments', title: 'Anything else?', maxLength: 200 }, + ]; + + /** Probabilities use their boundaries so no random source needs stubbing. */ + function makePayload(prompt: object = {}): string { + return JSON.stringify({ + version: CHAT_MODEL_FEEDBACK_SURVEY_CONFIG_VERSION, + id: 'auto-routing', + match: { selectedModels: ['auto'] }, + prompt, + steps, + }); + } + + /** Manual-only: the control is available but the survey never surfaces on its own. */ + const payload = makePayload(); + + /** A treatment that resolves but yields no config, as a malformed or retired one would. */ + const UNUSABLE_PAYLOAD = '{}'; + + const defaultSession = URI.parse('vscode-chat-editor://session-1'); + + /** Responses default to being the newest row, which is the only one that offers a survey. */ + function createResponse(requestId: string, options?: { modelId?: string; sessionResource?: URI; isComplete?: boolean; isLast?: boolean }): IChatResponseViewModel { + return { + requestId, + sessionResource: options?.sessionResource ?? defaultSession, + isComplete: options?.isComplete ?? true, + isLast: options?.isLast ?? true, + isCanceled: false, + errorDetails: undefined, + result: undefined, + model: { request: { modelId: options?.modelId ?? 'copilot/auto', modeInfo: { telemetryModeId: 'agent' } } }, + } as unknown as IChatResponseViewModel; + } + + async function createService(options: { + treatment?: string; + feedbackEnabled?: boolean; + onDidRefetchAssignments?: Event; + getTreatment?: () => string | undefined; + } = {}) { + const events: IChatModelFeedbackSurveyTelemetryEvent[] = []; + const instantiationService = disposables.add(new TestInstantiationService()); + + const configurationService = new TestConfigurationService(); + configurationService.setUserConfiguration('telemetry', { feedback: { enabled: options.feedbackEnabled ?? true } }); + + instantiationService.stub(IWorkbenchAssignmentService, { + _serviceBrand: undefined, + onDidRefetchAssignments: options.onDidRefetchAssignments ?? Event.None, + getCurrentExperiments: async () => [], + addTelemetryAssignmentFilter(_filter: IAssignmentFilter): void { }, + getTreatment: async () => (options.getTreatment + ? options.getTreatment() + : options.treatment ?? payload) as T | undefined, + } satisfies IWorkbenchAssignmentService); + instantiationService.stub(IConfigurationService, configurationService); + instantiationService.stub(IStorageService, disposables.add(new InMemoryStorageService())); + instantiationService.stub(ITelemetryService, { telemetryLevel: TelemetryLevel.USAGE } as ITelemetryService); + instantiationService.stub(ICommandService, { + executeCommand: async (_id: string, event: IChatModelFeedbackSurveyTelemetryEvent) => { events.push(event); }, + } as unknown as ICommandService); + instantiationService.stub(ILanguageModelsService, { lookupLanguageModel: () => undefined } as unknown as ILanguageModelsService); + instantiationService.stub(IChatSessionsService, { getChatSessionContribution: () => undefined } as unknown as IChatSessionsService); + const disposeSession = disposables.add(new Emitter<{ readonly sessionResources: readonly URI[]; readonly reason: 'cleared' }>()); + instantiationService.stub(IChatService, { onDidDisposeSession: disposeSession.event } as unknown as IChatService); + instantiationService.stub(ILogService, new NullLogService()); + + const service = disposables.add(instantiationService.createInstance(ChatModelFeedbackSurveyService)); + await new Promise(resolve => setTimeout(resolve, 0)); // let the treatment resolve + return { service, events, disposeSession, configurationService }; + } + + test('offers a matching response one stable survey and reports it as shown once', async () => { + const { service, events } = await createService(); + const response = createResponse('req-1'); + + const first = service.getSurvey(response); + const second = service.getSurvey(response); + + assert.deepStrictEqual({ + offered: !!first, + status: first?.status, + stableInstance: first?.instanceId === second?.instanceId, + reportedKinds: events.map(e => e.kind), + }, { + offered: true, + status: ChatModelFeedbackSurveyStatus.Collapsed, + stableInstance: true, + reportedKinds: ['shown'], + }); + }); + + test('withholds the survey when the response does not qualify', async () => { + const { service } = await createService(); + + assert.deepStrictEqual({ + wrongModel: service.getSurvey(createResponse('req-1', { modelId: 'copilot/gpt-5.2' })), + stillStreaming: service.getSurvey(createResponse('req-2', { isComplete: false })), + }, { + wrongModel: undefined, + stillStreaming: undefined, + }); + }); + + test('reports each step as it is answered so an abandoned survey still yields data', async () => { + const { service, events } = await createService(); + const response = createResponse('req-1'); + service.getSurvey(response); + + service.toggle(response); + service.answerChoice(response, 'routing', 'yes'); + service.setCommentDraft(response, 'some thoughts'); + service.dismiss(response); + + assert.deepStrictEqual(events.map(e => ({ kind: e.kind, stepId: e.stepId, answerId: e.answerId, comment: e.comment })), [ + { kind: 'shown', stepId: undefined, answerId: undefined, comment: undefined }, + { kind: 'opened', stepId: undefined, answerId: undefined, comment: undefined }, + { kind: 'step', stepId: 'routing', answerId: 'yes', comment: undefined }, + { kind: 'step', stepId: 'comments', answerId: undefined, comment: 'some thoughts' }, + { kind: 'dismissed', stepId: undefined, answerId: undefined, comment: undefined }, + ]); + }); + + test('ignores answers that are not configured options', async () => { + const { service, events } = await createService(); + const response = createResponse('req-1'); + service.getSurvey(response); + service.toggle(response); + + service.answerChoice(response, 'routing', 'injected-value'); + + assert.deepStrictEqual(events.map(e => e.kind), ['shown', 'opened']); + }); + + test('acknowledges an answered response instead of re-asking, without removing the control', async () => { + const { service, events } = await createService(); + const response = createResponse('req-1'); + service.getSurvey(response); + service.toggle(response); + service.submit(response, 'done'); + + const afterSubmit = service.getSurvey(response); + service.dismiss(response); // close the acknowledgement + service.toggle(response); // and reopen it + + assert.deepStrictEqual({ + stillAvailable: !!afterSubmit, + isSubmitted: afterSubmit?.isSubmitted, + reopened: service.getSurvey(response)?.isSubmitted, + // Reopening an answered survey must not inflate the funnel. + opens: events.filter(e => e.kind === 'opened').length, + submissions: events.filter(e => e.kind === 'submitted').length, + dismissals: events.filter(e => e.kind === 'dismissed').length, + }, { + stillAvailable: true, + isSubmitted: true, + reopened: true, + opens: 1, + submissions: 1, + dismissals: 0, + }); + }); + + test('offers the survey only on the newest response', async () => { + const { service } = await createService(); + + const older = createResponse('req-1', { isLast: false }); + const newest = createResponse('req-2'); + + assert.deepStrictEqual({ + older: service.getSurvey(older), + newest: !!service.getSurvey(newest), + }, { + older: undefined, + newest: true, + }); + }); + + test('drops the control from a response once a newer one arrives', async () => { + const { service } = await createService(); + const response = createResponse('req-1'); + service.getSurvey(response); + + const superseded = createResponse('req-1', { isLast: false }); + + assert.strictEqual(service.getSurvey(superseded), undefined); + }); + + test('keeps a part answered survey alive after it is superseded', async () => { + const { service } = await createService(); + const response = createResponse('req-1'); + service.getSurvey(response); + service.toggle(response); + service.answerChoice(response, 'routing', 'yes'); + + // The user is mid answer, so a newer response must not pull the form away. + const superseded = createResponse('req-1', { isLast: false }); + + assert.deepStrictEqual(service.getSurvey(superseded)?.status, ChatModelFeedbackSurveyStatus.Open); + }); + + test('opening a survey closes the one already showing', async () => { + const { service, events } = await createService(); + const first = createResponse('req-1'); + service.getSurvey(first); + service.toggle(first); + + const second = createResponse('req-2'); + service.getSurvey(second); + service.toggle(second); + + assert.deepStrictEqual({ + // Closed by the second opening, then dropped because it is no longer the newest row. + first: service.getSurvey(createResponse('req-1', { isLast: false })), + second: service.getSurvey(second)?.status, + // Being superseded is the UI moving on, so it is not reported as a dismissal. + dismissals: events.filter(e => e.kind === 'dismissed').length, + }, { + first: undefined, + second: ChatModelFeedbackSurveyStatus.Open, + dismissals: 0, + }); + }); + + test('leaves other sessions alone when a new response supersedes one', async () => { + const { service, events } = await createService(); + const otherSession = URI.parse('vscode-chat-editor://session-2'); + + const other = createResponse('other-1', { sessionResource: otherSession }); + const otherInstance = service.getSurvey(other)?.instanceId; + + // A new response in the first session must not evict the second session's state. + service.getSurvey(createResponse('req-1')); + service.getSurvey(createResponse('req-2')); + + assert.deepStrictEqual({ + sameInstance: service.getSurvey(other)?.instanceId === otherInstance, + shownForOther: events.filter(e => e.kind === 'shown' && e.requestId === 'other-1').length, + }, { + sameInstance: true, + shownForOther: 1, + }); + }); + + test('does not let an automatic prompt displace a survey being answered', async () => { + const { service } = await createService({ treatment: makePayload({ chance: { initial: 1 }, maxPerSession: 5, cooldownDays: 0 }) }); + + const first = createResponse('req-1'); + service.getSurvey(first); + service.answerChoice(first, 'routing', 'yes'); + + // The next response would normally auto open, but the user is mid answer. + const second = createResponse('req-2'); + + assert.deepStrictEqual({ + first: service.getSurvey(createResponse('req-1', { isLast: false }))?.status, + second: service.getSurvey(second)?.status, + }, { + first: ChatModelFeedbackSurveyStatus.Open, + second: ChatModelFeedbackSurveyStatus.Collapsed, + }); + }); + + test('stops a stale response being prompted once a newer one arrives', async () => { + const { service } = await createService({ treatment: makePayload({ chance: { initial: 0 }, triggers: { modelSwitchedAway: true } }) }); + const surveyed = createResponse('req-1'); + service.getSurvey(surveyed); + + // A newer response the survey does not match still supersedes the old one. + service.getSurvey(createResponse('req-2', { modelId: 'copilot/gpt-5.2' })); + service.notifyModelSwitchedAway(defaultSession, 'copilot/auto', 'copilot/gpt-5.2'); + + assert.strictEqual(service.getSurvey(createResponse('req-1', { isLast: false })), undefined); + }); + + test('toggles the survey closed when the control is pressed again', async () => { + const { service, events } = await createService(); + const response = createResponse('req-1'); + service.getSurvey(response); + + service.toggle(response); + const opened = service.getSurvey(response)?.status; + service.toggle(response); + const closed = service.getSurvey(response)?.status; + service.toggle(response); + + assert.deepStrictEqual({ + opened, + closed, + reopened: service.getSurvey(response)?.status, + kinds: events.map(e => e.kind), + }, { + opened: ChatModelFeedbackSurveyStatus.Open, + closed: ChatModelFeedbackSurveyStatus.Collapsed, + reopened: ChatModelFeedbackSurveyStatus.Open, + // Closing by the control is a dismissal, exactly as the X and Escape are. + kinds: ['shown', 'opened', 'dismissed', 'opened'], + }); + }); + + test('toggling an acknowledgement closed does not report a second dismissal', async () => { + const { service, events } = await createService(); + const response = createResponse('req-1'); + service.getSurvey(response); + service.toggle(response); + service.submit(response, 'done'); + + service.toggle(response); // hide the acknowledgement + const hidden = service.getSurvey(response)?.status; + service.toggle(response); // and show it again + + assert.deepStrictEqual({ + hidden, + shownAgain: service.getSurvey(response)?.isSubmitted, + dismissals: events.filter(e => e.kind === 'dismissed').length, + opens: events.filter(e => e.kind === 'opened').length, + }, { + hidden: ChatModelFeedbackSurveyStatus.Collapsed, + shownAgain: true, + dismissals: 0, + opens: 1, + }); + }); + + test('stops offering a survey once feedback is switched off', async () => { + const { service, configurationService } = await createService(); + const response = createResponse('req-1'); + const offered = !!service.getSurvey(response); + + configurationService.setUserConfiguration('telemetry', { feedback: { enabled: false } }); + + assert.deepStrictEqual({ offered, afterDisabling: service.getSurvey(response) }, { offered: true, afterDisabling: undefined }); + }); + + test('releases what it held for a session once that session goes away', async () => { + const { service, disposeSession } = await createService(); + const response = createResponse('req-1'); + const first = service.getSurvey(response)?.instanceId; + + disposeSession.fire({ sessionResources: [defaultSession], reason: 'cleared' }); + + // A fresh instance means the entry really was released rather than reused. + assert.notStrictEqual(service.getSurvey(createResponse('req-1'))?.instanceId, first); + }); + + test('keeps the control on the newest response even after the prompt budget is spent', async () => { + // Pacing governs unprompted surfacing only, so manual feedback is never rationed. + const { service } = await createService({ treatment: makePayload({ chance: { initial: 1 }, maxPerSession: 1 }) }); + + const first = service.getSurvey(createResponse('req-1')); + const second = service.getSurvey(createResponse('req-2')); + const third = service.getSurvey(createResponse('req-3')); + + assert.deepStrictEqual({ + available: [!!first, !!second, !!third], + autoOpened: [first?.status, second?.status, third?.status], + }, { + available: [true, true, true], + autoOpened: [ + ChatModelFeedbackSurveyStatus.Open, // the one automatic prompt this session allows + ChatModelFeedbackSurveyStatus.Collapsed, + ChatModelFeedbackSurveyStatus.Collapsed, + ], + }); + }); + + test('manual activation always opens, whatever the prompting rules say', async () => { + // One automatic prompt per session and a year long cooldown, both of which the first + // response consumes so the second is left with no automatic budget at all. + const { service, events } = await createService({ treatment: makePayload({ chance: { initial: 1 }, maxPerSession: 1, cooldownDays: 365 }) }); + + const prompted = createResponse('req-1'); + const autoStatus = service.getSurvey(prompted)?.status; + + const second = createResponse('req-2'); + const beforeManual = service.getSurvey(second)?.status; + service.toggle(second); + + assert.deepStrictEqual({ + autoStatus, + beforeManual, + afterManual: service.getSurvey(second)?.status, + triggers: events.filter(e => e.kind === 'opened').map(e => e.trigger), + }, { + autoStatus: ChatModelFeedbackSurveyStatus.Open, + beforeManual: ChatModelFeedbackSurveyStatus.Collapsed, + afterManual: ChatModelFeedbackSurveyStatus.Open, + triggers: ['chance', 'manual'], + }); + }); + + test('ramps the odds with each response that passes without prompting', async () => { + // An increment of 1 makes the ramp observable without stubbing random. The first response + // has probability 0 and the second, after one miss, has probability 1. + const { service } = await createService({ treatment: makePayload({ chance: { initial: 0, increment: 1 }, maxPerSession: 5, cooldownDays: 0 }) }); + + assert.deepStrictEqual([ + service.getSurvey(createResponse('req-1'))?.status, + service.getSurvey(createResponse('req-2'))?.status, + ], [ + ChatModelFeedbackSurveyStatus.Collapsed, + ChatModelFeedbackSurveyStatus.Open, + ]); + }); + + test('prompts on switching away from the surveyed model, and only within the trigger rules', async () => { + const enabled = await createService({ treatment: makePayload({ chance: { initial: 0 }, triggers: { modelSwitchedAway: true } }) }); + const disabled = await createService({ treatment: makePayload({ chance: { initial: 0 } }) }); + + const enabledResponse = createResponse('req-1'); + enabled.service.getSurvey(enabledResponse); + enabled.service.notifyModelSwitchedAway(defaultSession, 'copilot/auto', 'copilot/gpt-5.2'); + + const disabledResponse = createResponse('req-1'); + disabled.service.getSurvey(disabledResponse); + disabled.service.notifyModelSwitchedAway(defaultSession, 'copilot/auto', 'copilot/gpt-5.2'); + + assert.deepStrictEqual({ + enabled: enabled.service.getSurvey(enabledResponse)?.status, + enabledTrigger: enabled.events.filter(e => e.kind === 'opened').map(e => e.trigger), + disabled: disabled.service.getSurvey(disabledResponse)?.status, + }, { + enabled: ChatModelFeedbackSurveyStatus.Open, + enabledTrigger: ['modelSwitchedAway'], + disabled: ChatModelFeedbackSurveyStatus.Collapsed, + }); + }); + + test('ignores model switches that are not away from the surveyed model', async () => { + const { service } = await createService({ treatment: makePayload({ chance: { initial: 0 }, triggers: { modelSwitchedAway: true } }) }); + + const unrelated = createResponse('req-1'); + service.getSurvey(unrelated); + // A switch between two unsurveyed models says nothing about Auto, even though an Auto + // response is still the most recent surveyed one here. + service.notifyModelSwitchedAway(defaultSession, 'copilot/gpt-5.2', 'copilot/claude-sonnet-4.5'); + const afterUnrelated = service.getSurvey(unrelated)?.status; + + // Moving between two surveyed models is not abandoning the thing being surveyed. + service.notifyModelSwitchedAway(defaultSession, 'copilot/auto', 'agent-host-copilotcli:auto'); + + assert.deepStrictEqual({ + afterUnrelated, + afterMatchedToMatched: service.getSurvey(unrelated)?.status, + }, { + afterUnrelated: ChatModelFeedbackSurveyStatus.Collapsed, + afterMatchedToMatched: ChatModelFeedbackSurveyStatus.Collapsed, + }); + }); + + test('completes a survey whose last step is a choice, since it has no submit button', async () => { + const choiceOnly = JSON.stringify({ + version: CHAT_MODEL_FEEDBACK_SURVEY_CONFIG_VERSION, + id: 'choice-only', + match: { selectedModels: ['auto'] }, + steps: [{ kind: 'choice', id: 'routing', title: 'Right model?', options: [{ id: 'yes', label: 'Yes' }, { id: 'no', label: 'No' }] }], + }); + const { service, events } = await createService({ treatment: choiceOnly }); + const response = createResponse('req-1'); + service.getSurvey(response); + service.toggle(response); + + service.answerChoice(response, 'routing', 'yes'); + + assert.deepStrictEqual({ + kinds: events.map(e => e.kind), + // The survey is over, so it acknowledges rather than asking again. + isSubmitted: service.getSurvey(response)?.isSubmitted, + }, { + kinds: ['shown', 'opened', 'step', 'submitted'], + isSubmitted: true, + }); + }); + + test('keeps an open survey and its budget when the treatment stops resolving to a usable config', async () => { + const refetch = new Emitter(); + const treatments: (string | undefined)[] = [payload, UNUSABLE_PAYLOAD]; + const { service } = await createService({ onDidRefetchAssignments: refetch.event, getTreatment: () => treatments.shift() }); + const response = createResponse('req-1'); + service.getSurvey(response); + service.toggle(response); + + refetch.fire(); + await new Promise(resolve => setTimeout(resolve, 0)); + + assert.deepStrictEqual(service.getSurvey(response)?.status, ChatModelFeedbackSurveyStatus.Open); + refetch.dispose(); + }); + + test('preserves an uncommitted comment draft across a widget recycle', async () => { + const { service } = await createService(); + const response = createResponse('req-1'); + service.getSurvey(response); + service.toggle(response); + service.answerChoice(response, 'routing', 'yes'); + + service.setCommentDraft(response, 'half typed'); + + assert.strictEqual(service.getSurvey(response)?.commentDraft, 'half typed'); + }); + + test('collects nothing without a usable experiment or with feedback disabled', async () => { + const unconfigured = await createService({ treatment: UNUSABLE_PAYLOAD }); + const feedbackOff = await createService({ feedbackEnabled: false }); + + assert.deepStrictEqual({ + unconfiguredSurvey: unconfigured.service.getSurvey(createResponse('req-1')), + unconfiguredEvents: unconfigured.events, + feedbackOffSurvey: feedbackOff.service.getSurvey(createResponse('req-1')), + feedbackOffEvents: feedbackOff.events, + }, { + unconfiguredSurvey: undefined, + unconfiguredEvents: [], + feedbackOffSurvey: undefined, + feedbackOffEvents: [], + }); + }); +}); diff --git a/src/vs/workbench/contrib/chat/test/browser/feedbackSurvey/chatModelFeedbackSurveyWidget.test.ts b/src/vs/workbench/contrib/chat/test/browser/feedbackSurvey/chatModelFeedbackSurveyWidget.test.ts new file mode 100644 index 0000000000000..2156c6fe4f100 --- /dev/null +++ b/src/vs/workbench/contrib/chat/test/browser/feedbackSurvey/chatModelFeedbackSurveyWidget.test.ts @@ -0,0 +1,252 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import * as dom from '../../../../../../base/browser/dom.js'; +import { mainWindow } from '../../../../../../base/browser/window.js'; +import { Emitter, Event } from '../../../../../../base/common/event.js'; +import { URI } from '../../../../../../base/common/uri.js'; +import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../base/test/common/utils.js'; +import { workbenchInstantiationService } from '../../../../../test/browser/workbenchTestServices.js'; +import { ChatModelFeedbackSurveyWidget } from '../../../browser/feedbackSurvey/chatModelFeedbackSurveyWidget.js'; +import { ChatModelFeedbackSurveyStatus, IChatModelFeedbackSurveyChangeEvent, IChatModelFeedbackSurveyService, IChatModelFeedbackSurveyState } from '../../../browser/feedbackSurvey/chatModelFeedbackSurveyService.js'; +import { ChatModelFeedbackSurveyStepKind, IChatModelFeedbackSurveyConfig } from '../../../common/feedbackSurvey/chatModelFeedbackSurveyConfig.js'; +import { IChatResponseViewModel } from '../../../common/model/chatViewModel.js'; + +suite('ChatModelFeedbackSurveyWidget', () => { + + const store = ensureNoDisposablesAreLeakedInTestSuite(); + + const config = { + version: 1, + id: 'auto-routing', + match: { selectedModels: ['auto'], resolvedModels: [], modes: [], harnesses: [], sessionTypes: [] }, + prompt: { cooldownDays: 0, maxPerSession: 1, chance: { initial: 0, increment: 0, max: 1 }, triggers: { modelSwitchedAway: { enabled: false, bypassCooldown: false } } }, + steps: [{ + kind: ChatModelFeedbackSurveyStepKind.Choice, + id: 'routing', + title: 'Right model?', + options: [{ id: 'yes', label: 'Yes' }, { id: 'no', label: 'No' }, { id: 'maybe', label: 'Maybe' }], + }], + } as IChatModelFeedbackSurveyConfig; + + const response = { + requestId: 'req-1', + sessionResource: URI.parse('vscode-chat-editor://session-1'), + isLast: true, + } as IChatResponseViewModel; + + interface ISurveyHarness { + readonly container: HTMLElement; + readonly answers: { stepId: string; optionId: string }[]; + readonly dismissals: number; + readonly focusRestores: number; + readonly state: { current: IChatModelFeedbackSurveyState }; + rerender(): void; + } + + function createWidget(options?: { + openTrigger?: 'manual' | 'chance'; + isSubmitted?: boolean; + onGetSurvey?: () => void; + onDidChangeSurveyState?: Event; + }): ISurveyHarness { + const answers: { stepId: string; optionId: string }[] = []; + const counts = { dismissals: 0, focusRestores: 0 }; + const state = { + current: { + config, + instanceId: 'instance-1', + status: ChatModelFeedbackSurveyStatus.Open, + stepIndex: 0, + answers: new Map(), + commentDraft: '', + isSubmitted: options?.isSubmitted ?? false, + openTrigger: options?.openTrigger ?? 'manual', + } satisfies IChatModelFeedbackSurveyState, + }; + + const surveyService: IChatModelFeedbackSurveyService = { + _serviceBrand: undefined, + onDidChangeSurveyState: options?.onDidChangeSurveyState ?? Event.None, + onDidChangeConfiguration: Event.None, + getSurvey: () => { + options?.onGetSurvey?.(); + return state.current; + }, + toggle: () => { }, + notifyModelSwitchedAway: () => { }, + answerChoice: (_response, stepId, optionId) => { answers.push({ stepId, optionId }); }, + submit: () => { }, + dismiss: () => { counts.dismissals++; }, + setCommentDraft: () => { }, + }; + + const instantiationService = workbenchInstantiationService(undefined, store); + instantiationService.stub(IChatModelFeedbackSurveyService, surveyService); + + const container = dom.$('.chat-feedback-survey-widget'); + mainWindow.document.body.appendChild(container); + store.add({ dispose: () => container.remove() }); + + const widget = store.add(instantiationService.createInstance(ChatModelFeedbackSurveyWidget, container, () => { counts.focusRestores++; })); + widget.render(response); + return { + container, + answers, + state, + get dismissals() { return counts.dismissals; }, + get focusRestores() { return counts.focusRestores; }, + rerender: () => widget.render(response), + }; + } + + /** Browser key codes, which is what `StandardKeyboardEvent` reads and maps. */ + const enum BrowserKey { + Enter = 13, + End = 35, + Home = 36, + ArrowUp = 38, + ArrowDown = 40, + Space = 32, + Escape = 27, + } + + function pressKey(target: HTMLElement, keyCode: BrowserKey, key: string): void { + const event = new KeyboardEvent('keydown', { key, bubbles: true, cancelable: true }); + Object.defineProperty(event, 'keyCode', { get: () => keyCode }); + target.dispatchEvent(event); + } + + function activeLabel(container: HTMLElement): string | undefined { + return container.querySelector('.chat-feedback-survey-list-item.active')?.textContent ?? undefined; + } + + test('moves the active option with the arrow keys and answers on Enter', () => { + const { container, answers } = createWidget(); + const list = container.querySelector('.chat-feedback-survey-list')!; + + const initial = activeLabel(container); + pressKey(list, BrowserKey.ArrowDown, 'ArrowDown'); + const afterDown = activeLabel(container); + pressKey(list, BrowserKey.Enter, 'Enter'); + + assert.deepStrictEqual({ initial, afterDown, answers }, { + initial: 'Yes', + afterDown: 'No', + answers: [{ stepId: 'routing', optionId: 'no' }], + }); + }); + + test('wraps around the ends and supports Home and End', () => { + const { container } = createWidget(); + const list = container.querySelector('.chat-feedback-survey-list')!; + + pressKey(list, BrowserKey.ArrowUp, 'ArrowUp'); + const afterUpFromFirst = activeLabel(container); + pressKey(list, BrowserKey.Home, 'Home'); + const afterHome = activeLabel(container); + pressKey(list, BrowserKey.End, 'End'); + + assert.deepStrictEqual({ afterUpFromFirst, afterHome, afterEnd: activeLabel(container) }, { + afterUpFromFirst: 'Maybe', + afterHome: 'Yes', + afterEnd: 'Maybe', + }); + }); + + test('keeps navigation keys away from the chat list once the survey has used them', () => { + const { container } = createWidget(); + const list = container.querySelector('.chat-feedback-survey-list')!; + + let reachedAncestor = false; + store.add(dom.addDisposableListener(container.parentElement!, dom.EventType.KEY_DOWN, () => { reachedAncestor = true; })); + + pressKey(list, BrowserKey.ArrowDown, 'ArrowDown'); + + assert.deepStrictEqual({ reachedAncestor, active: activeLabel(container) }, { reachedAncestor: false, active: 'No' }); + }); + + test('marks the active option as selected for screen readers', () => { + const { container } = createWidget(); + const list = container.querySelector('.chat-feedback-survey-list')!; + + const initial = [...container.querySelectorAll('.chat-feedback-survey-list-item')].map(i => i.getAttribute('aria-selected')); + pressKey(list, BrowserKey.ArrowDown, 'ArrowDown'); + const afterDown = [...container.querySelectorAll('.chat-feedback-survey-list-item')].map(i => i.getAttribute('aria-selected')); + + assert.deepStrictEqual({ initial, afterDown, activeDescendant: list.getAttribute('aria-activedescendant') }, { + initial: ['true', 'false', 'false'], + afterDown: ['false', 'true', 'false'], + activeDescendant: 'chat-feedback-survey-option-instance-1-routing-1', + }); + }); + + test('keeps Space from reaching the chat list, which would toggle the row', () => { + const { container } = createWidget(); + const list = container.querySelector('.chat-feedback-survey-list')!; + + let reachedAncestor = false; + store.add(dom.addDisposableListener(container.parentElement!, dom.EventType.KEY_DOWN, () => { reachedAncestor = true; })); + + pressKey(list, BrowserKey.Space, ' '); + + assert.strictEqual(reachedAncestor, false); + }); + + test('hands focus back when the panel closes', () => { + const harness = createWidget(); + + harness.state.current = { ...harness.state.current, status: ChatModelFeedbackSurveyStatus.Collapsed }; + harness.rerender(); + + assert.deepStrictEqual({ + focusRestores: harness.focusRestores, + panels: harness.container.querySelectorAll('.chat-feedback-survey-container').length, + }, { + focusRestores: 1, + panels: 0, + }); + }); + + test('shows an acknowledgement instead of questions once answered', () => { + const { container } = createWidget({ isSubmitted: true }); + + assert.deepStrictEqual({ + options: container.querySelectorAll('.chat-feedback-survey-list-item').length, + hasCard: container.querySelectorAll('.chat-feedback-survey-container').length, + }, { + options: 0, + hasCard: 1, + }); + }); + + test('escape dismisses the survey', () => { + const harness = createWidget(); + const list = harness.container.querySelector('.chat-feedback-survey-list')!; + + pressKey(list, BrowserKey.Escape, 'Escape'); + + assert.strictEqual(harness.dismissals, 1); + }); + + test('renders one panel when the survey opens itself while the row is rendering', () => { + // Reading the survey can open it, which reports a change back while render is running. + const changeEmitter = store.add(new Emitter()); + let fired = false; + const harness = createWidget({ + openTrigger: 'chance', + onDidChangeSurveyState: changeEmitter.event, + onGetSurvey: () => { + if (!fired) { + fired = true; + changeEmitter.fire({ sessionResource: response.sessionResource, requestId: response.requestId }); + } + }, + }); + + assert.strictEqual(harness.container.querySelectorAll('.chat-feedback-survey-container').length, 1); + }); +}); diff --git a/src/vs/workbench/contrib/chat/test/browser/feedbackSurvey/mockChatModelFeedbackSurveyService.ts b/src/vs/workbench/contrib/chat/test/browser/feedbackSurvey/mockChatModelFeedbackSurveyService.ts new file mode 100644 index 0000000000000..0b9609201fef4 --- /dev/null +++ b/src/vs/workbench/contrib/chat/test/browser/feedbackSurvey/mockChatModelFeedbackSurveyService.ts @@ -0,0 +1,29 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Event } from '../../../../../../base/common/event.js'; +import { URI } from '../../../../../../base/common/uri.js'; +import { IChatModelFeedbackSurveyService, IChatModelFeedbackSurveyState } from '../../../browser/feedbackSurvey/chatModelFeedbackSurveyService.js'; +import { IChatResponseViewModel } from '../../../common/model/chatViewModel.js'; + +/** Never offers a survey, matching any build without the experiment configured. */ +export class MockChatModelFeedbackSurveyService implements IChatModelFeedbackSurveyService { + + declare readonly _serviceBrand: undefined; + + readonly onDidChangeSurveyState = Event.None; + readonly onDidChangeConfiguration = Event.None; + + getSurvey(_response: IChatResponseViewModel): IChatModelFeedbackSurveyState | undefined { + return undefined; + } + + toggle(_response: IChatResponseViewModel): void { } + notifyModelSwitchedAway(_sessionResource: URI, _fromModelId: string, _toModelId: string): void { } + answerChoice(_response: IChatResponseViewModel, _stepId: string, _optionId: string): void { } + submit(_response: IChatResponseViewModel, _comment?: string): void { } + dismiss(_response: IChatResponseViewModel, _comment?: string): void { } + setCommentDraft(_response: IChatResponseViewModel, _comment: string): void { } +} diff --git a/src/vs/workbench/contrib/chat/test/browser/widget/chatListRenderer.test.ts b/src/vs/workbench/contrib/chat/test/browser/widget/chatListRenderer.test.ts index 7ac24846a99e9..0537d740c34f9 100644 --- a/src/vs/workbench/contrib/chat/test/browser/widget/chatListRenderer.test.ts +++ b/src/vs/workbench/contrib/chat/test/browser/widget/chatListRenderer.test.ts @@ -39,6 +39,8 @@ import { ChatEditorOptions } from '../../../browser/widget/chatOptions.js'; import { shouldRenderGeneratedImageResult, shouldRenderSessionCreatedResult } from '../../../browser/widget/chatContentParts/toolInvocationParts/chatToolInvocationPart.js'; import { getGeneratedImageResultParts, getGeneratedImageResultPartsFromContent } from '../../../browser/widget/chatContentParts/toolInvocationParts/chatGeneratedImageResultSubPart.js'; import { MockChatService } from '../../common/chatService/mockChatService.js'; +import { IChatModelFeedbackSurveyService } from '../../../browser/feedbackSurvey/chatModelFeedbackSurveyService.js'; +import { MockChatModelFeedbackSurveyService } from '../feedbackSurvey/mockChatModelFeedbackSurveyService.js'; suite('ChatListRenderer', () => { const store = ensureNoDisposablesAreLeakedInTestSuite(); @@ -627,6 +629,7 @@ suite('ChatListRenderer', () => { configurationService.setUserConfiguration(ChatConfiguration.TurnStatusPills, false); instantiationService.stub(IConfigurationService, configurationService); instantiationService.stub(IChatService, new MockChatService()); + instantiationService.stub(IChatModelFeedbackSurveyService, new MockChatModelFeedbackSurveyService()); instantiationService.stub(IChatAgentService, disposables.add(instantiationService.createInstance(ChatAgentService))); const model = disposables.add(instantiationService.createInstance(ChatModel, undefined, { initialLocation: ChatAgentLocation.Chat, canUseTools: true })); @@ -694,6 +697,7 @@ suite('ChatListRenderer', () => { configurationService.setUserConfiguration(ChatConfiguration.TurnStatusPills, false); instantiationService.stub(IConfigurationService, configurationService); instantiationService.stub(IChatService, new MockChatService()); + instantiationService.stub(IChatModelFeedbackSurveyService, new MockChatModelFeedbackSurveyService()); instantiationService.stub(IChatAgentService, disposables.add(instantiationService.createInstance(ChatAgentService))); const model = disposables.add(instantiationService.createInstance(ChatModel, undefined, { initialLocation: ChatAgentLocation.Chat, canUseTools: true })); @@ -1072,6 +1076,7 @@ suite('ChatListRenderer', () => { configurationService.setUserConfiguration('workbench.reduceMotion', 'on'); instantiationService.stub(IConfigurationService, configurationService); instantiationService.stub(IChatService, new MockChatService()); + instantiationService.stub(IChatModelFeedbackSurveyService, new MockChatModelFeedbackSurveyService()); instantiationService.stub(IChatAgentService, disposables.add(instantiationService.createInstance(ChatAgentService))); const model = disposables.add(instantiationService.createInstance(ChatModel, undefined, { initialLocation: ChatAgentLocation.Chat, canUseTools: true })); @@ -1167,6 +1172,7 @@ suite('ChatListRenderer', () => { configurationService.setUserConfiguration(ChatConfiguration.Verbose, false); instantiationService.stub(IConfigurationService, configurationService); instantiationService.stub(IChatService, new MockChatService()); + instantiationService.stub(IChatModelFeedbackSurveyService, new MockChatModelFeedbackSurveyService()); instantiationService.stub(IChatAgentService, disposables.add(instantiationService.createInstance(ChatAgentService))); instantiationService.stub(IViewDescriptorService, { onDidChangeLocation: Event.None, @@ -1261,6 +1267,7 @@ suite('ChatListRenderer', () => { configurationService.setUserConfiguration(ChatConfiguration.Verbose, false); instantiationService.stub(IConfigurationService, configurationService); instantiationService.stub(IChatService, new MockChatService()); + instantiationService.stub(IChatModelFeedbackSurveyService, new MockChatModelFeedbackSurveyService()); instantiationService.stub(IChatAgentService, disposables.add(instantiationService.createInstance(ChatAgentService))); const model = disposables.add(instantiationService.createInstance(ChatModel, undefined, { initialLocation: ChatAgentLocation.Chat, canUseTools: true })); @@ -1370,6 +1377,7 @@ suite('ChatListRenderer', () => { configurationService.setUserConfiguration(ChatConfiguration.Verbose, false); instantiationService.stub(IConfigurationService, configurationService); instantiationService.stub(IChatService, new MockChatService()); + instantiationService.stub(IChatModelFeedbackSurveyService, new MockChatModelFeedbackSurveyService()); instantiationService.stub(IChatAgentService, disposables.add(instantiationService.createInstance(ChatAgentService))); const model = disposables.add(instantiationService.createInstance(ChatModel, undefined, { initialLocation: ChatAgentLocation.Chat, canUseTools: true })); @@ -1452,6 +1460,7 @@ suite('ChatListRenderer', () => { configurationService.setUserConfiguration(ChatConfiguration.TurnStatusPills, false); instantiationService.stub(IConfigurationService, configurationService); instantiationService.stub(IChatService, new MockChatService()); + instantiationService.stub(IChatModelFeedbackSurveyService, new MockChatModelFeedbackSurveyService()); instantiationService.stub(IChatAgentService, disposables.add(instantiationService.createInstance(ChatAgentService))); const model = disposables.add(instantiationService.createInstance(ChatModel, undefined, { initialLocation: ChatAgentLocation.Chat, canUseTools: true })); @@ -1551,6 +1560,7 @@ suite('ChatListRenderer', () => { const configurationService = new TestConfigurationService(); instantiationService.stub(IConfigurationService, configurationService); instantiationService.stub(IChatService, new MockChatService()); + instantiationService.stub(IChatModelFeedbackSurveyService, new MockChatModelFeedbackSurveyService()); instantiationService.stub(IChatAgentService, disposables.add(instantiationService.createInstance(ChatAgentService))); const model = disposables.add(instantiationService.createInstance(ChatModel, undefined, { initialLocation: ChatAgentLocation.Chat, canUseTools: true })); diff --git a/src/vs/workbench/contrib/chat/test/browser/widget/chatListWidget.test.ts b/src/vs/workbench/contrib/chat/test/browser/widget/chatListWidget.test.ts index 079d86f1ec227..12db45d6c45f3 100644 --- a/src/vs/workbench/contrib/chat/test/browser/widget/chatListWidget.test.ts +++ b/src/vs/workbench/contrib/chat/test/browser/widget/chatListWidget.test.ts @@ -26,6 +26,8 @@ import { ChatAgentService, IChatAgentService } from '../../../common/participant import { ChatRequestTextPart } from '../../../common/requestParser/chatParserTypes.js'; import { ToolDataSource } from '../../../common/tools/languageModelToolsService.js'; import { MockChatService } from '../../common/chatService/mockChatService.js'; +import { IChatModelFeedbackSurveyService } from '../../../browser/feedbackSurvey/chatModelFeedbackSurveyService.js'; +import { MockChatModelFeedbackSurveyService } from '../feedbackSurvey/mockChatModelFeedbackSurveyService.js'; function nextFrame(): Promise { return new Promise(resolve => mainWindow.requestAnimationFrame(() => resolve())); @@ -64,6 +66,7 @@ suite('ChatListWidget', () => { configurationService.setUserConfiguration(ChatConfiguration.Verbose, false); instantiationService.stub(IConfigurationService, configurationService); instantiationService.stub(IChatService, new MockChatService()); + instantiationService.stub(IChatModelFeedbackSurveyService, new MockChatModelFeedbackSurveyService()); instantiationService.stub(IChatAgentService, disposables.add(instantiationService.createInstance(ChatAgentService))); instantiationService.stub(IAccessibleViewService, { getOpenAriaHint: () => '' }); instantiationService.stub(IChatAccessibilityService, { diff --git a/src/vs/workbench/contrib/chat/test/common/feedbackSurvey/chatModelFeedbackSurveyConfig.test.ts b/src/vs/workbench/contrib/chat/test/common/feedbackSurvey/chatModelFeedbackSurveyConfig.test.ts new file mode 100644 index 0000000000000..fed3f0ddc4ecc --- /dev/null +++ b/src/vs/workbench/contrib/chat/test/common/feedbackSurvey/chatModelFeedbackSurveyConfig.test.ts @@ -0,0 +1,184 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../base/test/common/utils.js'; +import { CHAT_MODEL_FEEDBACK_SURVEY_CONFIG_VERSION, IChatModelFeedbackSurveyMatchContext, matchesChatModelFeedbackSurvey, parseChatModelFeedbackSurveyConfig } from '../../../common/feedbackSurvey/chatModelFeedbackSurveyConfig.js'; + +suite('ChatModelFeedbackSurveyConfig', () => { + + ensureNoDisposablesAreLeakedInTestSuite(); + + const validPayload = { + version: CHAT_MODEL_FEEDBACK_SURVEY_CONFIG_VERSION, + id: 'auto-routing-2026-08', + match: { selectedModels: ['auto'], harnesses: ['copilotcli', 'none'] }, + prompt: { cooldownDays: 7, maxPerSession: 1, chance: { initial: 0.1, increment: 0.05, max: 0.5 } }, + steps: [ + { + kind: 'choice', id: 'routing', title: 'Did Auto choose the right model for the job?', + options: [ + { id: 'yes', label: 'Yes' }, + { id: 'too-heavy', label: 'No - too heavy' }, + { id: 'too-light', label: 'No - too light' }, + ], + }, + { kind: 'text', id: 'comments', title: 'Anything else?', placeholder: 'Optional feedback', maxLength: 500 }, + ], + }; + + function parse(payload: unknown): ReturnType { + return parseChatModelFeedbackSurveyConfig(JSON.stringify(payload)); + } + + test('accepts a well formed payload and normalizes selectors', () => { + const result = parse({ ...validPayload, match: { selectedModels: [' AUTO '], modes: ['Agent'] } }); + + assert.deepStrictEqual(result.config, { + version: 1, + id: 'auto-routing-2026-08', + match: { selectedModels: ['auto'], resolvedModels: [], modes: ['agent'], harnesses: [], sessionTypes: [] }, + prompt: { + cooldownDays: 7, + maxPerSession: 1, + chance: { initial: 0.1, increment: 0.05, max: 0.5 }, + triggers: { modelSwitchedAway: { enabled: false, bypassCooldown: false } }, + }, + steps: [ + { + kind: 'choice', id: 'routing', title: 'Did Auto choose the right model for the job?', + options: [ + { id: 'yes', label: 'Yes' }, + { id: 'too-heavy', label: 'No - too heavy' }, + { id: 'too-light', label: 'No - too light' }, + ], + }, + { kind: 'text', id: 'comments', title: 'Anything else?', placeholder: 'Optional feedback', maxLength: 500 }, + ], + }); + }); + + test('rejects malformed payloads whole rather than partially', () => { + const errors = { + empty: parseChatModelFeedbackSurveyConfig('').error, + notJson: parseChatModelFeedbackSurveyConfig('{nope').error?.startsWith('payload is not valid JSON'), + wrongVersion: parse({ ...validPayload, version: 99 }).error, + badId: parse({ ...validPayload, id: 'Has Spaces' }).error, + unnarrowedMatch: parse({ ...validPayload, match: {} }).error, + noSteps: parse({ ...validPayload, steps: [] }).error, + duplicateStepId: parse({ ...validPayload, steps: [validPayload.steps[0], validPayload.steps[0]] }).error, + unknownKind: parse({ ...validPayload, steps: [{ kind: 'slider', id: 'a', title: 'T' }] }).error, + tooFewOptions: parse({ ...validPayload, steps: [{ kind: 'choice', id: 'a', title: 'T', options: [{ id: 'x', label: 'X' }] }] }).error, + badPromptLimit: parse({ ...validPayload, prompt: { maxPerSession: 0 } }).error, + badProbability: parse({ ...validPayload, prompt: { chance: { initial: 2 } } }).error, + invertedChance: parse({ ...validPayload, prompt: { chance: { initial: 0.5, max: 0.1 } } }).error, + badCooldown: parse({ ...validPayload, prompt: { cooldownDays: -1 } }).error, + badTrigger: parse({ ...validPayload, prompt: { triggers: { modelSwitchedAway: 'yes' } } }).error, + }; + + assert.deepStrictEqual(errors, { + empty: 'empty payload', + notJson: true, + wrongVersion: 'unsupported version 99, expected 1', + badId: 'missing or malformed survey id', + unnarrowedMatch: 'match must narrow at least one dimension', + noSteps: 'steps must be a non-empty array', + duplicateStepId: 'steps[1].id "routing" is duplicated', + unknownKind: 'steps[0].kind must be "choice" or "text"', + tooFewOptions: 'steps[0].options must have between 2 and 8 entries', + badPromptLimit: 'prompt.maxPerSession must be a positive integer', + badProbability: 'prompt.chance.initial must be a probability between 0 and 1', + invertedChance: 'prompt.chance.max must be greater than or equal to prompt.chance.initial', + badCooldown: 'prompt.cooldownDays must be a non-negative number', + badTrigger: 'prompt.triggers.modelSwitchedAway must be a boolean or an object', + }); + }); + + test('clamps a text step maxLength to the transport budget', () => { + const result = parse({ ...validPayload, steps: [{ kind: 'text', id: 'c', title: 'T', maxLength: 99999 }] }); + + assert.deepStrictEqual(result.config?.steps, [{ kind: 'text', id: 'c', title: 'T', placeholder: undefined, maxLength: 1000 }]); + }); + + test('rejects text step arrangements that would strand later steps', () => { + const choice = validPayload.steps[0]; + const text = validPayload.steps[1]; + const secondText = { kind: 'text', id: 'more', title: 'More?', maxLength: 100 }; + + assert.deepStrictEqual({ + twoTextSteps: parse({ ...validPayload, steps: [text, secondText] }).error, + textNotLast: parse({ ...validPayload, steps: [text, choice] }).error, + choiceOnly: parse({ ...validPayload, steps: [choice] }).error, + }, { + twoTextSteps: 'steps may contain at most one text step', + textNotLast: 'a text step must be the last step', + choiceOnly: undefined, + }); + }); + + test('defaults an omitted prompt block to manual-only surfacing', () => { + const result = parse({ ...validPayload, prompt: undefined }); + + assert.deepStrictEqual(result.config?.prompt, { + cooldownDays: 7, + maxPerSession: 1, + // Zero probability means the survey never opens unasked, but the control still shows. + chance: { initial: 0, increment: 0, max: 1 }, + triggers: { modelSwitchedAway: { enabled: false, bypassCooldown: false } }, + }); + }); + + suite('matching', () => { + + function match(context: IChatModelFeedbackSurveyMatchContext, payload: unknown = validPayload): boolean { + const config = parse(payload).config; + assert.ok(config, 'expected a valid config'); + return matchesChatModelFeedbackSurvey(config, context); + } + + test('matches a short selector against identifiers from every harness', () => { + // Local model ids are `/` and agent host ids are `:`. + assert.deepStrictEqual({ + bare: match({ selectedModelId: 'auto', harness: undefined }), + vendorQualified: match({ selectedModelId: 'copilot/auto', harness: undefined }), + agentHostQualified: match({ selectedModelId: 'agent-host-copilotcli:auto', harness: 'copilotcli' }), + byAlias: match({ selectedModelId: 'copilot/gpt-5.2', selectedModelAliases: ['auto'], harness: undefined }), + unrelatedModel: match({ selectedModelId: 'copilot/gpt-5.2', harness: undefined }), + }, { + bare: true, + vendorQualified: true, + agentHostQualified: true, + byAlias: true, + unrelatedModel: false, + }); + }); + + test('keeps selected and resolved models as independent dimensions', () => { + const payload = { ...validPayload, match: { selectedModels: ['auto'], resolvedModels: ['gpt-5.2'] } }; + + assert.deepStrictEqual({ + both: match({ selectedModelId: 'copilot/auto', resolvedModelId: 'gpt-5.2' }, payload), + selectedOnly: match({ selectedModelId: 'copilot/auto', resolvedModelId: 'claude-sonnet-4.5' }, payload), + resolvedOnly: match({ selectedModelId: 'copilot/gpt-5.2', resolvedModelId: 'gpt-5.2' }, payload), + }, { + both: true, + selectedOnly: false, + resolvedOnly: false, + }); + }); + + test('treats a session with no agent host as the "none" harness', () => { + assert.deepStrictEqual({ + noHarnessAllowed: match({ selectedModelId: 'auto', harness: undefined }), + harnessAllowed: match({ selectedModelId: 'auto', harness: 'copilotcli' }), + harnessExcluded: match({ selectedModelId: 'auto', harness: 'claude' }), + }, { + noHarnessAllowed: true, + harnessAllowed: true, + harnessExcluded: false, + }); + }); + }); +}); diff --git a/src/vs/workbench/test/browser/componentFixtures/chat/chatFixtureUtils.ts b/src/vs/workbench/test/browser/componentFixtures/chat/chatFixtureUtils.ts index 798a009ff7dfe..c67ff4ba93413 100644 --- a/src/vs/workbench/test/browser/componentFixtures/chat/chatFixtureUtils.ts +++ b/src/vs/workbench/test/browser/componentFixtures/chat/chatFixtureUtils.ts @@ -14,6 +14,8 @@ import { IMenu, IMenuItem, IMenuService, MenuId, MenuItemAction } from '../../.. import { IFileDialogService } from '../../../../../platform/dialogs/common/dialogs.js'; import { IFileService } from '../../../../../platform/files/common/files.js'; import { IListService, ListService } from '../../../../../platform/list/browser/listService.js'; +import { IChatModelFeedbackSurveyService } from '../../../../contrib/chat/browser/feedbackSurvey/chatModelFeedbackSurveyService.js'; +import { MockChatModelFeedbackSurveyService } from '../../../../contrib/chat/test/browser/feedbackSurvey/mockChatModelFeedbackSurveyService.js'; import { IActionWidgetService } from '../../../../../platform/actionWidget/browser/actionWidget.js'; import { ILinkPresentationService } from '../../../../../platform/dataChannel/common/dataChannel.js'; import { IProductService } from '../../../../../platform/product/common/productService.js'; @@ -139,6 +141,7 @@ export function registerChatFixtureServices(reg: ServiceRegistration, options: I reg.define(IMenuService, FixtureMenuService); reg.define(IMarkdownRendererService, MarkdownRendererService); reg.define(IListService, ListService); + reg.defineInstance(IChatModelFeedbackSurveyService, new MockChatModelFeedbackSurveyService()); reg.defineInstance(ILinkPresentationService, new class extends mock() { override getLinkPresentationRule() { return undefined; } override createLinkPresentationWatcher() { return undefined; }