Read chat responses aloud

src/vs/base/browser/markdownRenderer.ts

@@ -757,7 +757,9 @@ function createPlainTextRenderer(): marked.Renderer {
 	const renderer = new marked.Renderer();
 
 	renderer.code = ({ text }: marked.Tokens.Code): string => {
-		return escape(text);
+		// Ends with a line break like every other block, so that the content
+		// following a code block does not run into its last line.
+		return escape(text) + '\n';
 	};
 	renderer.blockquote = ({ text }: marked.Tokens.Blockquote): string => {
 		return text + '\n';

src/vs/base/test/browser/markdownRenderer.test.ts

@@ -480,6 +480,19 @@ suite('MarkdownRenderer', () => {
 			const markdown = { value: '- outer\n    - inner [link](/target)' };
 			assert.strictEqual(renderAsPlaintext(markdown, { omitMarkdownSyntax: true }), 'outer\ninner link');
 		});
+
+		test('separates a code block from the text that follows it', () => {
+			// Every other block renderer ends with a line break; without one the
+			// last line of a code block runs into the next sentence.
+			assert.deepStrictEqual({
+				followedByText: renderAsPlaintext({ value: '```ts\nconst x = 1;\n```\n\nAll tests passed.' }),
+				alone: renderAsPlaintext({ value: '```ts\nconst x = 1;\n```' })
+			}, {
+				followedByText: 'const x = 1;\nAll tests passed.',
+				alone: 'const x = 1;'
+			});
+		});
+
 	});
 
 	suite('supportHtml', () => {

src/vs/sessions/sessions.desktop.main.ts

@@ -242,6 +242,11 @@ import './contrib/providers/remoteAgentHost/browser/cloudSandboxAgentHost.contri
 // Chat
 import './contrib/agentFeedback/browser/agentFeedback.contribution.js';
 import './contrib/chat/electron-browser/chat.contribution.js';
+import '../workbench/contrib/speech/electron-browser/speech.contribution.js';
+
+// Read chat responses aloud (on-device speech synthesis)
+import { registerReadAloudActions } from '../workbench/contrib/chat/electron-browser/actions/readAloudActions.contribution.js';
+registerReadAloudActions();
 
 // Local Agent Host
 import './contrib/providers/agentHost/browser/localAgentHost.contribution.js';

src/vs/workbench/contrib/accessibility/browser/accessibilityConfiguration.ts

@@ -927,6 +927,22 @@ export function registerAccessibilityConfiguration() {
 				tags: ['accessibility'],
 				scope: ConfigurationScope.APPLICATION,
 			},
+			[AccessibilityVoiceSettingId.MaiSpeechEndpoint]: {
+				'markdownDescription': localize('voice.maiSpeechEndpoint', "The endpoint of the speech service used to read text aloud, for example `https://eastus2.tts.speech.microsoft.com`. Run `Speech: Set Up Read Aloud` to configure it together with its key, which is stored securely rather than in your settings. Note that the text being read is sent to this service."),
+				'type': 'string',
+				'default': '',
+				// Application scope so that a workspace cannot point reading aloud
+				// at another server and have the key sent there.
+				'scope': ConfigurationScope.APPLICATION,
+				'tags': ['accessibility', 'usesOnlineServices']
+			},
+			[AccessibilityVoiceSettingId.MaiVoice]: {
+				'markdownDescription': localize('voice.maiVoice', "The voice used to read text aloud, for example `en-US-Harper:MAI-Voice-2`. Leave empty to pick a voice for {0} automatically.", `\`#${AccessibilityVoiceSettingId.SpeechLanguage}#\``),
+				'type': 'string',
+				'default': '',
+				'scope': ConfigurationScope.APPLICATION,
+				'tags': ['accessibility']
+			},
 			[AccessibilityWorkbenchSettingId.HideAccessibleView]: {
 				description: localize('accessibility.hideAccessibleView', "Controls whether the Accessible View is hidden."),
 				type: 'boolean',
@@ -965,8 +981,11 @@ export class DynamicSpeechAccessibilityConfiguration extends Disposable implemen
 	}
 
 	private updateConfiguration(): void {
-		if (!this.speechService.hasSpeechProvider) {
-			return; // these settings require a speech provider
+		// Text to speech is also provided by the built-in engines, which do not
+		// make a speech provider available: gating on that alone would leave
+		// reading aloud with no settings at all, including the language it reads in.
+		if (!this.speechService.hasSpeechProvider && !this.speechService.hasTextToSpeechProvider) {
+			return; // these settings require speech to be available
 		}
 
 		const languages = this.getLanguages();

src/vs/workbench/contrib/chat/browser/actions/chatAccessibilityHelp.ts

@@ -11,6 +11,7 @@ import { AccessibleContentProvider, AccessibleViewProviderId, AccessibleViewType
 import { IAccessibleViewImplementation } from '../../../../../platform/accessibility/browser/accessibleViewRegistry.js';
 import { ContextKeyExpr } from '../../../../../platform/contextkey/common/contextkey.js';
 import { IKeybindingService } from '../../../../../platform/keybinding/common/keybinding.js';
+import { ISpeechService } from '../../../speech/common/speechService.js';
 import { IConfigurationService } from '../../../../../platform/configuration/common/configuration.js';
 import { IWorkbenchEnvironmentService } from '../../../../services/environment/common/environmentService.js';
 import { AccessibilityVerbositySettingId } from '../../../accessibility/browser/accessibilityConfiguration.js';
@@ -63,7 +64,7 @@ export class AgentChatAccessibilityHelp implements IAccessibleViewImplementation
 	}
 }
 
-export function getAccessibilityHelpText(type: 'panelChat' | 'inlineChat' | 'quickChat' | 'editsView' | 'agentView', keybindingService: IKeybindingService, supportsFileReferences: boolean, isSessionsWindow: boolean = false, stickyPromptHeaderShown: boolean = false): string {
+export function getAccessibilityHelpText(type: 'panelChat' | 'inlineChat' | 'quickChat' | 'editsView' | 'agentView', keybindingService: IKeybindingService, supportsFileReferences: boolean, isSessionsWindow: boolean = false, stickyPromptHeaderShown: boolean = false, supportsReadAloud: boolean = false): string {
 	const content = [];
 	if (type === 'panelChat' || type === 'quickChat' || type === 'editsView' || type === 'agentView') {
 		content.push(localize('chat.fileChangesDisclosure', 'File change summaries show the total files, additions, and deletions. Focus the disclosure and press Enter or Space to show or hide the individual files. Focus an additions and deletions label and press Enter or Space to open the changes in a diff editor.'));
@@ -177,6 +178,9 @@ export function getAccessibilityHelpText(type: 'panelChat' | 'inlineChat' | 'qui
 	}
 	content.push(localize('chat.attachments.pastedText', "Long pasted text is stored as an attached text item and replaced in the input with a numbered inline reference."));
 	content.push(localize('chat.paste.asText', "To paste the clipboard as plain text, without converting it to Markdown or storing it as an attachment, invoke Paste as Text{0}.", '<keybinding:editor.action.pasteAsText>'));
+	if (supportsReadAloud) {
+		content.push(localize('chat.readAloud', "A response can be read aloud with Read Aloud{0}, which is also the first button in the response toolbar. Reading stops with Stop Read Aloud{1}, or by starting to read another response.", '<keybinding:workbench.action.chat.readChatResponseAloud>', '<keybinding:workbench.action.chat.stopReadChatItemAloud>'));
+	}
 	content.push(localize('chat.signals', "Accessibility Signals can be changed via settings with a prefix of signals.chat. By default, if a request takes more than 4 seconds, you will hear a sound indicating that progress is still occurring."));
 	return content.join('\n');
 }
@@ -199,7 +203,7 @@ export function getChatAccessibilityHelpProvider(accessor: ServicesAccessor, edi
 
 	const cachedPosition = inputEditor.getPosition();
 	inputEditor.getSupportedActions();
-	const helpText = getAccessibilityHelpText(type, keybindingService, widget.supportsFileReferences, environmentService.isSessionsWindow, isStickyPromptHeaderShown(widget, configurationService));
+	const helpText = getAccessibilityHelpText(type, keybindingService, widget.supportsFileReferences, environmentService.isSessionsWindow, isStickyPromptHeaderShown(widget, configurationService), accessor.get(ISpeechService).hasTextToSpeechProvider);
 	return new AccessibleContentProvider(
 		type === 'panelChat' ? AccessibleViewProviderId.PanelChat : type === 'inlineChat' ? AccessibleViewProviderId.InlineChat : type === 'agentView' ? AccessibleViewProviderId.AgentChat : AccessibleViewProviderId.QuickChat,
 		{ type: AccessibleViewType.Help },

src/vs/workbench/contrib/chat/browser/widget/chatListRenderer.ts

@@ -636,6 +636,20 @@ export class ChatListItemRenderer extends Disposable implements ITreeRenderer<Ch
 	private readonly pendingStickyScrollStateRefresh = this._register(new MutableDisposable<IDisposable>());
 	private readonly templateDataByRow = new WeakMap<HTMLElement, IChatListItemTemplate>();
 
+	private static readonly synthesisInProgressKeys = new Set([ChatContextKeys.synthesisInProgressItemId.key]);
+
+	/**
+	 * Whether `element` is the response that is currently being read aloud, so
+	 * that only its footer offers to stop reading.
+	 */
+	private isSynthesisInProgressFor(element: ChatTreeItem | undefined): boolean {
+		if (!element || !isResponseVM(element)) {
+			return false;
+		}
+
+		return this.contextKeyService.getContextKeyValue<string>(ChatContextKeys.synthesisInProgressItemId.key) === element.id;
+	}
+
 	/** Track pending question carousels by session resource for auto-skip on chat submission */
 	private readonly pendingQuestionCarousels = new ResourceMap<Set<ChatQuestionCarouselPart>>();
 	private readonly _notifiedQuestionCarousels = new Set<string>();
@@ -1169,6 +1183,15 @@ export class ChatListItemRenderer extends Disposable implements ITreeRenderer<Ch
 		const template: IChatListItemTemplate = { header, avatarContainer, requestHover, username, detail, value, requestTimestampContainer, rowContainer, elementDisposables, templateDisposables, contextKeyService, instantiationService: scopedInstantiationService, agentHover, titleToolbar, footerToolbar, footerToolbarContainer, footerDetailsContainer, disabledOverlay, checkpointToolbar, checkpointRestoreToolbar, checkpointContainer, checkpointRestoreContainer, completedResponseDisclosureDisposables, responseTokenStatsHover, feedbackSurveyWidget };
 		this.templateDataByRow.set(rowContainer, template);
 
+		// Only the response that is being read aloud shows a stop button, so keep
+		// this row in sync while it stays rendered.
+		const itemSynthesisInProgress = ChatContextKeys.itemSynthesisInProgress.bindTo(contextKeyService);
+		templateDisposables.add(this.contextKeyService.onDidChangeContext(e => {
+			if (e.affectsSome(ChatListItemRenderer.synthesisInProgressKeys)) {
+				itemSynthesisInProgress.set(this.isSynthesisInProgressFor(template.currentElement));
+			}
+		}));
+
 		templateDisposables.add(this._onDidUpdateViewModel.event(() => {
 			if (!template.currentElement || !this.viewModel?.sessionResource || !isEqual(template.currentElement.sessionResource, this.viewModel.sessionResource)) {
 				this.clearRenderedParts(template);
@@ -1320,6 +1343,7 @@ export class ChatListItemRenderer extends Disposable implements ITreeRenderer<Ch
 
 		ChatContextKeys.isResponse.bindTo(templateData.contextKeyService).set(isResponseVM(element));
 		ChatContextKeys.itemId.bindTo(templateData.contextKeyService).set(element.id);
+		ChatContextKeys.itemSynthesisInProgress.bindTo(templateData.contextKeyService).set(this.isSynthesisInProgressFor(element));
 		ChatContextKeys.isRequest.bindTo(templateData.contextKeyService).set(isRequestVM(element));
 		ChatContextKeys.isFirstRequest.bindTo(templateData.contextKeyService).set(isRequestVM(element) && this.viewModel?.model.getRequests()[0]?.id === element.id);
 		ChatContextKeys.isPendingRequest.bindTo(templateData.contextKeyService).set(isRequestVM(element) && !!element.pendingKind);

src/vs/workbench/contrib/chat/common/actions/chatContextKeys.ts

@@ -38,6 +38,8 @@ export namespace ChatContextKeys {
 	export const isPendingRequest = new RawContextKey<boolean>('chatRequestIsPending', false, { type: 'boolean', description: localize('chatRequestIsPending', "True when the chat request item is pending in the queue.") });
 	export const itemId = new RawContextKey<string>('chatItemId', '', { type: 'string', description: localize('chatItemId', "The id of the chat item.") });
 	export const lastItemId = new RawContextKey<string[]>('chatLastItemId', [], { type: 'string', description: localize('chatLastItemId', "The id of the last chat item.") });
+	export const itemSynthesisInProgress = new RawContextKey<boolean>('chatItemSynthesisInProgress', false, { type: 'boolean', description: localize('chatItemSynthesisInProgress', "True when this specific chat response is being read aloud.") });
+	export const synthesisInProgressItemId = new RawContextKey<string>('chatSynthesisInProgressItemId', '', { type: 'string', description: localize('chatSynthesisInProgressItemId', "The id of the chat response that is currently being read aloud.") });
 
 	export const editApplied = new RawContextKey<boolean>('chatEditApplied', false, { type: 'boolean', description: localize('chatEditApplied', "True when the chat text edits have been applied.") });
 

src/vs/workbench/contrib/chat/electron-browser/actions/readAloudActions.contribution.ts

@@ -0,0 +1,18 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerAction2 } from '../../../../../platform/actions/common/actions.js';
+import { ReadChatResponseAloud, StopReadAloud, StopReadChatItemAloud } from './voiceChatActions.js';
+
+/**
+ * Registers reading chat responses aloud. Kept apart from the voice chat actions
+ * so that windows which do not offer voice chat, such as the Agents window, can
+ * still read responses aloud.
+ */
+export function registerReadAloudActions(): void {
+	registerAction2(ReadChatResponseAloud);
+	registerAction2(StopReadChatItemAloud);
+	registerAction2(StopReadAloud);
+}