From 4e3a1d635deae6337e73e753dede88fce49f765a Mon Sep 17 00:00:00 2001 From: Federico Brancasi Date: Thu, 20 Aug 2026 20:22:37 +0200 Subject: [PATCH 1/4] Read chat responses aloud VS Code has had a "Read Aloud" action in the chat response footer for a while, but it is gated on `HasSpeechProvider` and no speech provider ships in this repository, so the button has never been visible to anyone. Supply the missing engine. Text is cleaned up (emoji and markdown removed, blocks punctuated so the reader pauses between them), split into pieces, and synthesized one piece ahead of playback so that speech starts quickly. Two engines are registered against a new `IBuiltinTextToSpeechEngine` seam and picked by priority: the MAI voice model over the Azure Speech service when it is configured, and the speech synthesizer of the platform for everything else, including languages the model does not speak and machines that are offline. A speech provider from an extension still wins over both. Registering a full `ISpeechProvider` would also have enabled five speech-to-text actions that have no implementation here, which is why text to speech gets its own seam and a `HasTextToSpeechProvider` context key. Authentication is the one open piece: there is no product endpoint for verbatim synthesis yet, so the endpoint is a setting and the key lives in secret storage, never in a settings file. Only `maiSpeechCredentials.ts` has to change once an endpoint authenticated with the identity the user already signed in with exists. --- src/vs/base/browser/markdownRenderer.ts | 4 +- .../test/browser/markdownRenderer.test.ts | 13 + src/vs/sessions/sessions.desktop.main.ts | 4 + .../browser/accessibilityConfiguration.ts | 23 +- .../chat/browser/widget/chatListRenderer.ts | 24 ++ .../chat/common/actions/chatContextKeys.ts | 2 + .../actions/readAloudActions.contribution.ts | 18 + .../actions/voiceChatActions.ts | 108 ++++-- .../electron-browser/chat.contribution.ts | 7 +- .../chat/test/common/voiceChatService.test.ts | 4 +- .../actions/voiceChatActions.test.ts | 24 +- .../speech/browser/builtinTextToSpeech.ts | 170 ++++++++++ .../speech/browser/maiSpeechActions.ts | 105 ++++++ .../speech/browser/maiSpeechCredentials.ts | 145 ++++++++ .../contrib/speech/browser/maiTextToSpeech.ts | 321 ++++++++++++++++++ .../speech/browser/speech.contribution.ts | 10 + .../contrib/speech/browser/speechService.ts | 115 +++++-- .../browser/textToSpeechEngineContribution.ts | 30 ++ .../contrib/speech/common/speechService.ts | 46 +++ .../contrib/speech/common/speechText.ts | 131 +++++++ .../test/browser/builtinTextToSpeech.test.ts | 229 +++++++++++++ .../test/browser/maiTextToSpeech.test.ts | 84 +++++ .../speech/test/common/speechText.test.ts | 179 ++++++++++ src/vs/workbench/workbench.desktop.main.ts | 2 + 24 files changed, 1741 insertions(+), 57 deletions(-) create mode 100644 src/vs/workbench/contrib/chat/electron-browser/actions/readAloudActions.contribution.ts create mode 100644 src/vs/workbench/contrib/speech/browser/builtinTextToSpeech.ts create mode 100644 src/vs/workbench/contrib/speech/browser/maiSpeechActions.ts create mode 100644 src/vs/workbench/contrib/speech/browser/maiSpeechCredentials.ts create mode 100644 src/vs/workbench/contrib/speech/browser/maiTextToSpeech.ts create mode 100644 src/vs/workbench/contrib/speech/browser/textToSpeechEngineContribution.ts create mode 100644 src/vs/workbench/contrib/speech/common/speechText.ts create mode 100644 src/vs/workbench/contrib/speech/test/browser/builtinTextToSpeech.test.ts create mode 100644 src/vs/workbench/contrib/speech/test/browser/maiTextToSpeech.test.ts create mode 100644 src/vs/workbench/contrib/speech/test/common/speechText.test.ts diff --git a/src/vs/base/browser/markdownRenderer.ts b/src/vs/base/browser/markdownRenderer.ts index b106d3fdef11ee..ba9071f9e78f89 100644 --- a/src/vs/base/browser/markdownRenderer.ts +++ b/src/vs/base/browser/markdownRenderer.ts @@ -757,7 +757,9 @@ function createPlainTextRenderer(): marked.Renderer { const renderer = new marked.Renderer(); renderer.code = ({ text }: marked.Tokens.Code): string => { - return escape(text); + // Ends with a line break like every other block, so that the content + // following a code block does not run into its last line. + return escape(text) + '\n'; }; renderer.blockquote = ({ text }: marked.Tokens.Blockquote): string => { return text + '\n'; diff --git a/src/vs/base/test/browser/markdownRenderer.test.ts b/src/vs/base/test/browser/markdownRenderer.test.ts index dad302039724b6..53a7e8d49f8bfd 100644 --- a/src/vs/base/test/browser/markdownRenderer.test.ts +++ b/src/vs/base/test/browser/markdownRenderer.test.ts @@ -480,6 +480,19 @@ suite('MarkdownRenderer', () => { const markdown = { value: '- outer\n - inner [link](/target)' }; assert.strictEqual(renderAsPlaintext(markdown, { omitMarkdownSyntax: true }), 'outer\ninner link'); }); + + test('separates a code block from the text that follows it', () => { + // Every other block renderer ends with a line break; without one the + // last line of a code block runs into the next sentence. + assert.deepStrictEqual({ + followedByText: renderAsPlaintext({ value: '```ts\nconst x = 1;\n```\n\nAll tests passed.' }), + alone: renderAsPlaintext({ value: '```ts\nconst x = 1;\n```' }) + }, { + followedByText: 'const x = 1;\nAll tests passed.', + alone: 'const x = 1;' + }); + }); + }); suite('supportHtml', () => { diff --git a/src/vs/sessions/sessions.desktop.main.ts b/src/vs/sessions/sessions.desktop.main.ts index a667aa0ab864cc..7bd33664172747 100644 --- a/src/vs/sessions/sessions.desktop.main.ts +++ b/src/vs/sessions/sessions.desktop.main.ts @@ -243,6 +243,10 @@ import './contrib/providers/remoteAgentHost/browser/cloudSandboxAgentHost.contri import './contrib/agentFeedback/browser/agentFeedback.contribution.js'; import './contrib/chat/electron-browser/chat.contribution.js'; +// Read chat responses aloud (on-device speech synthesis) +import { registerReadAloudActions } from '../workbench/contrib/chat/electron-browser/actions/readAloudActions.contribution.js'; +registerReadAloudActions(); + // Local Agent Host import './contrib/providers/agentHost/browser/localAgentHost.contribution.js'; import './contrib/providers/agentHost/electron-browser/localAgentHostLifecycle.contribution.js'; diff --git a/src/vs/workbench/contrib/accessibility/browser/accessibilityConfiguration.ts b/src/vs/workbench/contrib/accessibility/browser/accessibilityConfiguration.ts index 4436932b9b2756..5761cb167be1f8 100644 --- a/src/vs/workbench/contrib/accessibility/browser/accessibilityConfiguration.ts +++ b/src/vs/workbench/contrib/accessibility/browser/accessibilityConfiguration.ts @@ -965,8 +965,11 @@ export class DynamicSpeechAccessibilityConfiguration extends Disposable implemen } private updateConfiguration(): void { - if (!this.speechService.hasSpeechProvider) { - return; // these settings require a speech provider + // Text to speech is also provided by the built-in engines, which do not + // make a speech provider available: gating on that alone would leave + // reading aloud with no settings at all, including the language it reads in. + if (!this.speechService.hasSpeechProvider && !this.speechService.hasTextToSpeechProvider) { + return; // these settings require speech to be available } const languages = this.getLanguages(); @@ -1000,6 +1003,22 @@ export class DynamicSpeechAccessibilityConfiguration extends Disposable implemen 'enumDescriptions': languagesSorted.map(key => languages[key].name), 'enumItemLabels': languagesSorted.map(key => languages[key].name) }, + [AccessibilityVoiceSettingId.MaiSpeechEndpoint]: { + 'markdownDescription': localize('voice.maiSpeechEndpoint', "The endpoint of the speech service used to read text aloud, for example `https://eastus2.tts.speech.microsoft.com`. Run `Speech: Set Up Read Aloud` to configure it together with its key, which is stored securely rather than in your settings. Note that the text being read is sent to this service."), + 'type': 'string', + 'default': '', + // Application scope so that a workspace cannot point reading aloud + // at another server and have the key sent there. + 'scope': ConfigurationScope.APPLICATION, + 'tags': ['accessibility', 'usesOnlineServices'] + }, + [AccessibilityVoiceSettingId.MaiVoice]: { + 'markdownDescription': localize('voice.maiVoice', "The voice used to read text aloud, for example `en-US-Harper:MAI-Voice-2`. Leave empty to pick a voice for {0} automatically.", `\`#${AccessibilityVoiceSettingId.SpeechLanguage}#\``), + 'type': 'string', + 'default': '', + 'scope': ConfigurationScope.APPLICATION, + 'tags': ['accessibility'] + }, [AccessibilityVoiceSettingId.AutoSynthesize]: { 'type': 'string', 'enum': ['on', 'off'], diff --git a/src/vs/workbench/contrib/chat/browser/widget/chatListRenderer.ts b/src/vs/workbench/contrib/chat/browser/widget/chatListRenderer.ts index 56c6ed37682642..565228cb0b738c 100644 --- a/src/vs/workbench/contrib/chat/browser/widget/chatListRenderer.ts +++ b/src/vs/workbench/contrib/chat/browser/widget/chatListRenderer.ts @@ -636,6 +636,20 @@ export class ChatListItemRenderer extends Disposable implements ITreeRenderer()); private readonly templateDataByRow = new WeakMap(); + private static readonly synthesisInProgressKeys = new Set([ChatContextKeys.synthesisInProgressItemId.key]); + + /** + * Whether `element` is the response that is currently being read aloud, so + * that only its footer offers to stop reading. + */ + private isSynthesisInProgressFor(element: ChatTreeItem | undefined): boolean { + if (!element || !isResponseVM(element)) { + return false; + } + + return this.contextKeyService.getContextKeyValue(ChatContextKeys.synthesisInProgressItemId.key) === element.id; + } + /** Track pending question carousels by session resource for auto-skip on chat submission */ private readonly pendingQuestionCarousels = new ResourceMap>(); private readonly _notifiedQuestionCarousels = new Set(); @@ -1169,6 +1183,15 @@ export class ChatListItemRenderer extends Disposable implements ITreeRenderer { + if (e.affectsSome(ChatListItemRenderer.synthesisInProgressKeys)) { + itemSynthesisInProgress.set(this.isSynthesisInProgressFor(template.currentElement)); + } + })); + templateDisposables.add(this._onDidUpdateViewModel.event(() => { if (!template.currentElement || !this.viewModel?.sessionResource || !isEqual(template.currentElement.sessionResource, this.viewModel.sessionResource)) { this.clearRenderedParts(template); @@ -1320,6 +1343,7 @@ export class ChatListItemRenderer extends Disposable implements ITreeRenderer('chatRequestIsPending', false, { type: 'boolean', description: localize('chatRequestIsPending', "True when the chat request item is pending in the queue.") }); export const itemId = new RawContextKey('chatItemId', '', { type: 'string', description: localize('chatItemId', "The id of the chat item.") }); export const lastItemId = new RawContextKey('chatLastItemId', [], { type: 'string', description: localize('chatLastItemId', "The id of the last chat item.") }); + export const itemSynthesisInProgress = new RawContextKey('chatItemSynthesisInProgress', false, { type: 'boolean', description: localize('chatItemSynthesisInProgress', "True when this specific chat response is being read aloud.") }); + export const synthesisInProgressItemId = new RawContextKey('chatSynthesisInProgressItemId', '', { type: 'string', description: localize('chatSynthesisInProgressItemId', "The id of the chat response that is currently being read aloud.") }); export const editApplied = new RawContextKey('chatEditApplied', false, { type: 'boolean', description: localize('chatEditApplied', "True when the chat text edits have been applied.") }); diff --git a/src/vs/workbench/contrib/chat/electron-browser/actions/readAloudActions.contribution.ts b/src/vs/workbench/contrib/chat/electron-browser/actions/readAloudActions.contribution.ts new file mode 100644 index 00000000000000..e57d3462010c7d --- /dev/null +++ b/src/vs/workbench/contrib/chat/electron-browser/actions/readAloudActions.contribution.ts @@ -0,0 +1,18 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { registerAction2 } from '../../../../../platform/actions/common/actions.js'; +import { ReadChatResponseAloud, StopReadAloud, StopReadChatItemAloud } from './voiceChatActions.js'; + +/** + * Registers reading chat responses aloud. Kept apart from the voice chat actions + * so that windows which do not offer voice chat, such as the Agents window, can + * still read responses aloud. + */ +export function registerReadAloudActions(): void { + registerAction2(ReadChatResponseAloud); + registerAction2(StopReadChatItemAloud); + registerAction2(StopReadAloud); +} diff --git a/src/vs/workbench/contrib/chat/electron-browser/actions/voiceChatActions.ts b/src/vs/workbench/contrib/chat/electron-browser/actions/voiceChatActions.ts index cfe8acd240a36e..25e01c711c962a 100644 --- a/src/vs/workbench/contrib/chat/electron-browser/actions/voiceChatActions.ts +++ b/src/vs/workbench/contrib/chat/electron-browser/actions/voiceChatActions.ts @@ -42,7 +42,8 @@ import { CTX_INLINE_CHAT_FOCUSED, MENU_INLINE_CHAT_WIDGET_SECONDARY } from '../. import { NOTEBOOK_EDITOR_FOCUSED } from '../../../notebook/common/notebookContextKeys.js'; import { CONTEXT_SETTINGS_EDITOR } from '../../../preferences/common/preferences.js'; import { SearchContext } from '../../../search/common/constants.js'; -import { TextToSpeechInProgress as GlobalTextToSpeechInProgress, HasSpeechProvider, ISpeechService, KeywordRecognitionStatus, SpeechToTextInProgress, SpeechToTextStatus, TextToSpeechStatus } from '../../../speech/common/speechService.js'; +import { TextToSpeechInProgress as GlobalTextToSpeechInProgress, HasSpeechProvider, HasTextToSpeechProvider, ISpeechService, KeywordRecognitionStatus, SpeechToTextInProgress, SpeechToTextStatus, TextToSpeechStatus } from '../../../speech/common/speechService.js'; +import { punctuateLines, stripEmoji } from '../../../speech/common/speechText.js'; import { CHAT_CATEGORY } from '../../browser/actions/chatActions.js'; import { IChatExecuteActionContext } from '../../browser/actions/chatExecuteActions.js'; import { IChatWidget, IChatWidgetService, IQuickChatService } from '../../browser/chat.js'; @@ -63,6 +64,12 @@ const VoiceChatSessionContexts: VoiceChatSessionContext[] = ['view', 'inline', ' // Global Context Keys (set on global context key service) const CanVoiceChat = ContextKeyExpr.and(ChatContextKeys.enabled, HasSpeechProvider); +/** + * Reading responses aloud only needs speech synthesis, which the built-in + * engine also provides. Gating it on {@link CanVoiceChat} would hide it + * whenever no extension registered a full speech provider. + */ +const CanReadAloud = ContextKeyExpr.and(ChatContextKeys.enabled, HasTextToSpeechProvider); const FocusInChatInput = ContextKeyExpr.or(CTX_INLINE_CHAT_FOCUSED, ChatContextKeys.inChatInput); // Scoped Context Keys (set on per-chat-context scoped context key service) @@ -692,9 +699,41 @@ class ChatSynthesizerSessionController { interface IChatSynthesizerContext { readonly ignoreCodeBlocks: boolean; + /** + * Whether to read only the final answer, leaving out the narration of the + * individual steps that led to it. Decided once when reading starts: a + * response that already finished is read as its answer, while one that is + * still being written is followed as it comes in. + */ + readonly finalResponseOnly: boolean; insideCodeBlock: boolean; } +/** + * How much of a response its final answer must cover before only that answer is + * read aloud. Below this the response is read whole, because the answer is then + * a sign-off such as "Done." rather than the substance. + */ +const MIN_FINAL_RESPONSE_RATIO = 0.2; + +/** + * Picks the text to speak for a response. Agent responses interleave the + * narration of each step with the answer, and only the answer is worth listening + * to once the work is done. + * + * The answer is only used when it actually holds most of the response: + * `getFinalResponse()` returns just the trailing run of markdown, which is empty + * when a response ends on a tool invocation and would otherwise leave nothing to + * read at all. + */ +export function selectTextToRead(markdown: string, finalResponse: string, finalResponseOnly: boolean): string { + if (!finalResponseOnly) { + return markdown; + } + + return finalResponse.length >= markdown.length * MIN_FINAL_RESPONSE_RATIO ? finalResponse : markdown; +} + class ChatSynthesizerSessions { private static instance: ChatSynthesizerSessions | undefined = undefined; @@ -711,7 +750,8 @@ class ChatSynthesizerSessions { constructor( @ISpeechService private readonly speechService: ISpeechService, @IConfigurationService private readonly configurationService: IConfigurationService, - @IInstantiationService private readonly instantiationService: IInstantiationService + @IInstantiationService private readonly instantiationService: IInstantiationService, + @IContextKeyService private readonly contextKeyService: IContextKeyService ) { } async start(controller: IChatSynthesizerSessionController): Promise { @@ -736,13 +776,20 @@ class ChatSynthesizerSessions { const scopedChatToSpeechInProgress = ScopedChatSynthesisInProgress.bindTo(controller.contextKeyService); disposables.add(toDisposable(() => scopedChatToSpeechInProgress.reset())); + // Published globally so that only the response being read aloud shows a + // stop button, instead of every response in the chat. + const synthesisInProgressItemId = ChatContextKeys.synthesisInProgressItemId.bindTo(this.contextKeyService); + disposables.add(toDisposable(() => synthesisInProgressItemId.reset())); + disposables.add(session.onDidChange(e => { switch (e.status) { case TextToSpeechStatus.Started: scopedChatToSpeechInProgress.set(true); + synthesisInProgressItemId.set(controller.response.id); break; case TextToSpeechStatus.Stopped: scopedChatToSpeechInProgress.reset(); + synthesisInProgressItemId.reset(); break; } })); @@ -754,18 +801,25 @@ class ChatSynthesizerSessions { await raceCancellation(session.synthesize(chunk), activeSession.token); } + + // The response was read to the end: end the session so that its state + // (and with it the stop button) does not stay around indefinitely. + if (this.activeSession === activeSession) { + this.stop(); + } } private async *nextChatResponseChunk(response: IChatResponseModel, token: CancellationToken): AsyncIterable { const context: IChatSynthesizerContext = { ignoreCodeBlocks: this.configurationService.getValue(AccessibilityVoiceSettingId.IgnoreCodeBlocks), + finalResponseOnly: response.isComplete, insideCodeBlock: false }; let totalOffset = 0; let complete = false; do { - const responseLength = response.response.toString().length; + const responseLength = this.getTextToRead(response, context).length; const { chunk, offset } = this.parseNextChatResponseChunk(response, totalOffset, context); totalOffset = offset; complete = response.isComplete; @@ -778,19 +832,31 @@ class ChatSynthesizerSessions { return; } - if (!complete && responseLength === response.response.toString().length) { + if (!complete && responseLength === this.getTextToRead(response, context).length) { await raceCancellation(Event.toPromise(response.onDidChange), token); // wait for the response to change } } while (!token.isCancellationRequested && !complete); } + /** + * The text of `response` that should be spoken. + */ + private getTextToRead(response: IChatResponseModel, context: IChatSynthesizerContext): string { + // `toString()` would also include tool invocation labels and command + // titles, which read as noise in the middle of a response. + return selectTextToRead(response.response.getMarkdown(), response.response.getFinalResponse(), context.finalResponseOnly); + } + private parseNextChatResponseChunk(response: IChatResponseModel, offset: number, context: IChatSynthesizerContext): { readonly chunk: string | undefined; readonly offset: number } { let chunk: string | undefined = undefined; - const text = response.response.toString(); + const text = this.getTextToRead(response, context); if (response.isComplete) { - chunk = text.substring(offset); + // Guard the offset: the text read from is fixed for the whole session, + // but a response can still shrink (e.g. a tool invocation clearing + // earlier content) while it is being read. + chunk = text.substring(Math.min(offset, text.length)); offset = text.length + 1; } else { const res = parseNextChatResponseChunk(text, offset); @@ -803,7 +869,9 @@ class ChatSynthesizerSessions { } return { - chunk: chunk ? renderAsPlaintext({ value: chunk }) : chunk, // convert markdown to plain text + // Emoji are removed first, while the text is still markdown, so that + // the period added for the pause does not end up after their gap. + chunk: chunk ? punctuateLines(renderAsPlaintext({ value: stripEmoji(chunk) })) : chunk, offset }; } @@ -855,24 +923,24 @@ export class ReadChatResponseAloud extends Action2 { id: 'workbench.action.chat.readChatResponseAloud', title: localize2('workbench.action.chat.readChatResponseAloud', "Read Aloud"), icon: Codicon.unmute, - precondition: CanVoiceChat, + precondition: CanReadAloud, menu: [{ id: MenuId.ChatMessageFooter, when: ContextKeyExpr.and( - CanVoiceChat, - ChatContextKeys.isResponse, // only for responses - ScopedChatSynthesisInProgress.negate(), // but not when already in progress - ChatContextKeys.responseIsFiltered.negate(), // and not when response is filtered + CanReadAloud, + ChatContextKeys.isResponse, // only for responses + ChatContextKeys.itemSynthesisInProgress.negate(), // but not when this response is being read + ChatContextKeys.responseIsFiltered.negate(), // and not when response is filtered ), group: 'navigation', order: -10 // first }, { id: MENU_INLINE_CHAT_WIDGET_SECONDARY, when: ContextKeyExpr.and( - CanVoiceChat, - ChatContextKeys.isResponse, // only for responses - ScopedChatSynthesisInProgress.negate(), // but not when already in progress - ChatContextKeys.responseIsFiltered.negate() // and not when response is filtered + CanReadAloud, + ChatContextKeys.isResponse, // only for responses + ScopedChatSynthesisInProgress.negate(), // but not when already in progress + ChatContextKeys.responseIsFiltered.negate() // and not when response is filtered ), group: 'navigation', order: -10 // first @@ -942,8 +1010,10 @@ export class StopReadAloud extends Action2 { weight: KeybindingWeight.WorkbenchContrib + 100, primary: KeyCode.Escape, when: ScopedChatSynthesisInProgress - }, - menu: primaryVoiceActionMenu(ScopedChatSynthesisInProgress) + } + // No chat input menu: reading a response aloud is stopped from the + // response itself (see `StopReadChatItemAloud`) or with `Escape`. A + // spinner in the input toolbar reads as "the chat is busy" instead. }); } @@ -970,7 +1040,7 @@ export class StopReadChatItemAloud extends Action2 { { id: MenuId.ChatMessageFooter, when: ContextKeyExpr.and( - ScopedChatSynthesisInProgress, // only when in progress + ChatContextKeys.itemSynthesisInProgress, // only on the response being read ChatContextKeys.isResponse, // only for responses ChatContextKeys.responseIsFiltered.negate() // but not when response is filtered ), diff --git a/src/vs/workbench/contrib/chat/electron-browser/chat.contribution.ts b/src/vs/workbench/contrib/chat/electron-browser/chat.contribution.ts index 65d3cb47965a05..0c70067068a804 100644 --- a/src/vs/workbench/contrib/chat/electron-browser/chat.contribution.ts +++ b/src/vs/workbench/contrib/chat/electron-browser/chat.contribution.ts @@ -47,7 +47,8 @@ import { registerChatExportZipAction } from './actions/chatExportZip.js'; import { registerExportAgentTracesDbAction } from './actions/exportAgentTracesDb.js'; import { registerInstallDictationModelAction } from './actions/installDictationModelAction.js'; import { confirmSessionShutdown, getEffectiveSessionShutdownReason, shouldWarnForInFlightSessionShutdown, shouldWarnForSessionShutdown } from './chatLifecycle.js'; -import { HoldToVoiceChatInChatViewAction, InlineVoiceChatAction, KeywordActivationContribution, QuickVoiceChatAction, ReadChatResponseAloud, StartVoiceChatAction, StopListeningAction, StopListeningAndSubmitAction, StopReadAloud, StopReadChatItemAloud, VoiceChatInChatViewAction } from './actions/voiceChatActions.js'; +import { registerReadAloudActions } from './actions/readAloudActions.contribution.js'; +import { HoldToVoiceChatInChatViewAction, InlineVoiceChatAction, KeywordActivationContribution, QuickVoiceChatAction, StartVoiceChatAction, StopListeningAction, StopListeningAndSubmitAction, VoiceChatInChatViewAction } from './actions/voiceChatActions.js'; import { OpenWorkspaceInAgentsWindowAction, OpenWorkspaceInAgentsContribution, OpenAgentsWindowAction, OpenChatSessionInAgentsWindowAction, AgentsHandoffInputTipContribution, ToggleOpenInAgentsWindowTitleBarAction, OpenWorkspaceInAgentsWindowChatTitleAction, OpenWorkspaceInAgentsWindowTitleBarAction } from './agentSessions/agentSessionsActions.js'; import { NativeBuiltinToolsContribution } from './builtInTools/tools.js'; import { NativePluginGitCommandService } from './pluginGitCommandService.js'; @@ -227,9 +228,7 @@ registerAction2(InlineVoiceChatAction); registerAction2(StopListeningAction); registerAction2(StopListeningAndSubmitAction); -registerAction2(ReadChatResponseAloud); -registerAction2(StopReadChatItemAloud); -registerAction2(StopReadAloud); +registerReadAloudActions(); registerChatDeveloperActions(); registerChatExportZipAction(); diff --git a/src/vs/workbench/contrib/chat/test/common/voiceChatService.test.ts b/src/vs/workbench/contrib/chat/test/common/voiceChatService.test.ts index 26335126c145c7..5acafc1db5bd79 100644 --- a/src/vs/workbench/contrib/chat/test/common/voiceChatService.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/voiceChatService.test.ts @@ -11,7 +11,7 @@ import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../base/tes import { ExtensionIdentifier } from '../../../../../platform/extensions/common/extensions.js'; import { MockContextKeyService } from '../../../../../platform/keybinding/test/common/mockKeybindingService.js'; import { nullExtensionDescription } from '../../../../services/extensions/common/extensions.js'; -import { ISpeechProvider, ISpeechService, ISpeechToTextEvent, ISpeechToTextSession, ITextToSpeechSession, KeywordRecognitionStatus, SpeechToTextStatus } from '../../../speech/common/speechService.js'; +import { IBuiltinTextToSpeechEngine, ISpeechProvider, ISpeechService, ISpeechToTextEvent, ISpeechToTextSession, ITextToSpeechSession, KeywordRecognitionStatus, SpeechToTextStatus } from '../../../speech/common/speechService.js'; import { IChatAgent, IChatAgentCommand, IChatAgentCompletionItem, IChatAgentData, IChatAgentHistoryEntry, IChatAgentImplementation, IChatAgentMetadata, IChatAgentRequest, IChatAgentResult, IChatAgentService, IChatParticipantDetectionProvider, UserSelectedTools } from '../../common/participants/chatAgents.js'; import { IChatModel } from '../../common/model/chatModel.js'; import { IChatFollowup, IChatProgress } from '../../common/chatService/chatService.js'; @@ -107,11 +107,13 @@ suite('VoiceChat', () => { onDidChangeHasSpeechProvider = Event.None; readonly hasSpeechProvider = true; + readonly hasTextToSpeechProvider = true; readonly hasActiveSpeechToTextSession = false; readonly hasActiveTextToSpeechSession = false; readonly hasActiveKeywordRecognition = false; registerSpeechProvider(identifier: string, provider: ISpeechProvider): IDisposable { throw new Error('Method not implemented.'); } + registerBuiltinTextToSpeechEngine(engine: IBuiltinTextToSpeechEngine): IDisposable { throw new Error('Method not implemented.'); } onDidStartSpeechToTextSession = Event.None; onDidEndSpeechToTextSession = Event.None; diff --git a/src/vs/workbench/contrib/chat/test/electron-browser/actions/voiceChatActions.test.ts b/src/vs/workbench/contrib/chat/test/electron-browser/actions/voiceChatActions.test.ts index dc58646011c6f4..c943a3c54d9fe0 100644 --- a/src/vs/workbench/contrib/chat/test/electron-browser/actions/voiceChatActions.test.ts +++ b/src/vs/workbench/contrib/chat/test/electron-browser/actions/voiceChatActions.test.ts @@ -5,7 +5,7 @@ import assert from 'assert'; import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../base/test/common/utils.js'; -import { parseNextChatResponseChunk } from '../../../electron-browser/actions/voiceChatActions.js'; +import { parseNextChatResponseChunk, selectTextToRead } from '../../../electron-browser/actions/voiceChatActions.js'; suite('VoiceChatActions', function () { @@ -40,5 +40,27 @@ suite('VoiceChatActions', function () { assertChunk('Hello World.\nHow is your day?\n', 'How is your day?', offset); }); + test('selectTextToRead falls back when there is no final answer to read', function () { + const markdown = 'Looked at the file. Ran the tests. They all passed.'; + + assert.deepStrictEqual({ + // A response ending on a tool call has no trailing markdown, so + // reading only the answer would read nothing at all. + endsOnToolCall: selectTextToRead(markdown, '', true), + // So would one whose answer is just a sign-off. + answerIsOnlyASignOff: selectTextToRead(markdown, 'Done.', true), + answerCarriesTheResponse: selectTextToRead(markdown, 'They all passed.', true), + // While still streaming the whole response is followed as it comes in. + whileStreaming: selectTextToRead(markdown, '', false), + emptyResponse: selectTextToRead('', '', true) + }, { + endsOnToolCall: markdown, + answerIsOnlyASignOff: markdown, + answerCarriesTheResponse: 'They all passed.', + whileStreaming: markdown, + emptyResponse: '' + }); + }); + ensureNoDisposablesAreLeakedInTestSuite(); }); diff --git a/src/vs/workbench/contrib/speech/browser/builtinTextToSpeech.ts b/src/vs/workbench/contrib/speech/browser/builtinTextToSpeech.ts new file mode 100644 index 00000000000000..33284041953642 --- /dev/null +++ b/src/vs/workbench/contrib/speech/browser/builtinTextToSpeech.ts @@ -0,0 +1,170 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { mainWindow } from '../../../../base/browser/window.js'; +import { CancellationToken } from '../../../../base/common/cancellation.js'; +import { Emitter } from '../../../../base/common/event.js'; +import { Disposable, DisposableStore, IDisposable, toDisposable } from '../../../../base/common/lifecycle.js'; +import { ILogService } from '../../../../platform/log/common/log.js'; +import { IBuiltinTextToSpeechEngine, ITextToSpeechEvent, ITextToSpeechSession, ITextToSpeechSessionOptions, TextToSpeechStatus } from '../common/speechService.js'; + +/** + * Picks the voice to read with, preferring an exact match on `language` (a + * BCP-47 tag such as `en-US`) over one on its primary subtag alone. Returns + * `undefined` when nothing matches, leaving the choice to the platform. + * + * Quality is deliberately not ranked: on desktop the on-device engine reads + * English, so this only serves other languages and the web, where the platform + * default is the best available guess. + */ +export function pickVoice(voices: readonly SpeechSynthesisVoice[], language: string | undefined): SpeechSynthesisVoice | undefined { + if (!language) { + return undefined; + } + + const wanted = language.toLowerCase().replace(/_/g, '-'); + const langOf = (voice: SpeechSynthesisVoice) => voice.lang.toLowerCase().replace(/_/g, '-'); + + return voices.find(voice => langOf(voice) === wanted) + ?? voices.find(voice => langOf(voice).split('-')[0] === wanted.split('-')[0]); +} + +/** + * A text-to-speech session backed by the platform synthesizer. Each call to + * {@link synthesize} only resolves once its audio finished playing, because + * callers await it per chunk to keep the spoken response in order. + */ +class BuiltinTextToSpeechSession extends Disposable implements ITextToSpeechSession { + + private readonly _onDidChange = this._register(new Emitter()); + readonly onDidChange = this._onDidChange.event; + + private active = false; + + constructor( + private readonly synthesis: SpeechSynthesis, + private readonly token: CancellationToken, + private readonly options: ITextToSpeechSessionOptions | undefined, + private readonly logService: ILogService + ) { + super(); + + this._register(toDisposable(() => this.stop())); + } + + /** Ties `disposable` to the lifetime of this session. */ + keep(disposable: IDisposable): void { + this._register(disposable); + } + + async synthesize(text: string): Promise { + if (this.token.isCancellationRequested || !text.trim()) { + return; + } + + if (!this.active) { + this.active = true; + this._onDidChange.fire({ status: TextToSpeechStatus.Started }); + } + + try { + await this.speak(text); + } catch (error) { + this.logService.error(`[speech] built-in text to speech failed: ${error}`); + this._onDidChange.fire({ status: TextToSpeechStatus.Error, text: String(error) }); + } + } + + private speak(text: string): Promise { + return new Promise((resolve, reject) => { + const utterance = new SpeechSynthesisUtterance(text); + + const voice = pickVoice(this.synthesis.getVoices(), this.options?.language); + if (voice) { + utterance.voice = voice; + } + if (this.options?.language) { + utterance.lang = this.options.language; + } + + const disposables = new DisposableStore(); + const complete = (error?: Error) => { + disposables.dispose(); + + if (error) { + reject(error); + } else { + resolve(); + } + }; + + disposables.add(toDisposable(() => { + utterance.onend = null; + utterance.onerror = null; + })); + disposables.add(this.token.onCancellationRequested(() => { + this.synthesis.cancel(); + complete(); + })); + + utterance.onend = () => complete(); + utterance.onerror = event => { + // Stopping mid-utterance surfaces as an error, but is expected. + complete(event.error === 'canceled' || event.error === 'interrupted' ? undefined : new Error(event.error)); + }; + + this.synthesis.speak(utterance); + }); + } + + private stop(): void { + if (this.active) { + this.active = false; + this.synthesis.cancel(); + this._onDidChange.fire({ status: TextToSpeechStatus.Stopped }); + } + } +} + +/** + * The text-to-speech engine that ships with VS Code, backed by the platform + * speech synthesizer. It is only consulted when no extension registered a + * speech provider, so installing one keeps its voices in charge. + */ +export class BuiltinTextToSpeechEngine implements IBuiltinTextToSpeechEngine { + + private readonly synthesis: SpeechSynthesis | undefined = mainWindow.speechSynthesis; + + /** Lowest priority: any on-device model should be preferred over this. */ + readonly priority = 0; + + /** + * Voices are populated asynchronously on some platforms, so an empty voice + * list shortly after startup does not mean synthesis is unavailable. A + * platform without any speech service (e.g. Linux without `speech-dispatcher`) + * surfaces that as an error on use instead. + */ + get isSupported(): boolean { + return !!this.synthesis; + } + + constructor( + @ILogService private readonly logService: ILogService + ) { } + + createTextToSpeechSession(token: CancellationToken, options?: ITextToSpeechSessionOptions): ITextToSpeechSession { + const synthesis = this.synthesis; + if (!synthesis) { + throw new Error('The built-in text to speech engine is not supported in this environment.'); + } + + const session = new BuiltinTextToSpeechSession(synthesis, token, options, this.logService); + // Kept by the session, so that it goes away with it rather than living + // on the token until that is disposed. + session.keep(token.onCancellationRequested(() => session.dispose())); + + return session; + } +} diff --git a/src/vs/workbench/contrib/speech/browser/maiSpeechActions.ts b/src/vs/workbench/contrib/speech/browser/maiSpeechActions.ts new file mode 100644 index 00000000000000..ec3002e0763b7d --- /dev/null +++ b/src/vs/workbench/contrib/speech/browser/maiSpeechActions.ts @@ -0,0 +1,105 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { toErrorMessage } from '../../../../base/common/errorMessage.js'; +import { localize, localize2 } from '../../../../nls.js'; +import { Action2, registerAction2 } from '../../../../platform/actions/common/actions.js'; +import { ConfigurationTarget, IConfigurationService } from '../../../../platform/configuration/common/configuration.js'; +import { ContextKeyExpr } from '../../../../platform/contextkey/common/contextkey.js'; +import { ServicesAccessor } from '../../../../platform/instantiation/common/instantiation.js'; +import { INotificationService } from '../../../../platform/notification/common/notification.js'; +import { IQuickInputService } from '../../../../platform/quickinput/common/quickInput.js'; +import { AccessibilityVoiceSettingId } from '../common/speechService.js'; +import { IMaiSpeechCredentialsService } from './maiSpeechCredentials.js'; + +/** + * Reading a response aloud is a chat feature, so setting it up is offered only + * where chat is. Referenced by name because `contrib/chat` already depends on + * this component, and importing it back would close the cycle. + */ +const ChatIsEnabled = ContextKeyExpr.has('chatIsEnabled'); + +/** + * Asks for the endpoint and key of the speech service that reads text aloud. + * + * The key goes to secret storage rather than to a setting, so that it cannot + * reach a settings file that is shared, synchronized or committed. The endpoint + * is asked for here too, because its setting is deliberately not listed in the + * settings editor. + */ +class SetUpReadAloudAction extends Action2 { + + static readonly ID = 'workbench.action.speech.setUpReadAloud'; + + constructor() { + super({ + id: SetUpReadAloudAction.ID, + title: localize2('setUpReadAloud', "Set Up Read Aloud"), + category: localize2('speechCategory', "Speech"), + f1: true, + precondition: ChatIsEnabled + }); + } + + async run(accessor: ServicesAccessor): Promise { + const quickInputService = accessor.get(IQuickInputService); + const credentialsService = accessor.get(IMaiSpeechCredentialsService); + const configurationService = accessor.get(IConfigurationService); + const notificationService = accessor.get(INotificationService); + + const endpoint = await quickInputService.input({ + ignoreFocusLost: true, + value: configurationService.getValue(AccessibilityVoiceSettingId.MaiSpeechEndpoint) ?? '', + title: localize('setUpReadAloud.endpointTitle', "Set Up Read Aloud (1 of 2)"), + prompt: localize('setUpReadAloud.endpointPrompt', "The endpoint of the speech service. The text being read is sent to it."), + placeHolder: localize('setUpReadAloud.endpointPlaceholder', "https://.tts.speech.microsoft.com"), + validateInput: async value => { + const trimmed = value.trim(); + if (!trimmed) { + return undefined; // empty removes the endpoint again + } + + try { + const url = new URL(trimmed); + const isLocal = url.hostname === 'localhost' || url.hostname === '127.0.0.1'; + + return url.protocol === 'https:' || (isLocal && url.protocol === 'http:') + ? undefined + : localize('setUpReadAloud.endpointNotHttps', "The endpoint must use HTTPS, because the key is sent with every request."); + } catch { + return localize('setUpReadAloud.endpointNotAUrl', "Enter a complete URL, for example https://eastus2.tts.speech.microsoft.com."); + } + } + }); + + if (endpoint === undefined) { + return; // cancelled + } + + try { + await configurationService.updateValue(AccessibilityVoiceSettingId.MaiSpeechEndpoint, endpoint.trim() || undefined, ConfigurationTarget.APPLICATION); + } catch (error) { + notificationService.error(localize('setUpReadAloud.endpointFailed', "Could not save the endpoint: {0}", toErrorMessage(error))); + + return; + } + + const key = await quickInputService.input({ + password: true, + ignoreFocusLost: true, + title: localize('setUpReadAloud.keyTitle', "Set Up Read Aloud (2 of 2)"), + prompt: localize('setUpReadAloud.keyPrompt', "The key for that endpoint. It is stored securely and never written to your settings. Leave empty to remove it."), + placeHolder: localize('setUpReadAloud.keyPlaceholder', "Speech service key") + }); + + if (key === undefined) { + return; // cancelled + } + + await credentialsService.setKey(key); + } +} + +registerAction2(SetUpReadAloudAction); diff --git a/src/vs/workbench/contrib/speech/browser/maiSpeechCredentials.ts b/src/vs/workbench/contrib/speech/browser/maiSpeechCredentials.ts new file mode 100644 index 00000000000000..030f40986483fe --- /dev/null +++ b/src/vs/workbench/contrib/speech/browser/maiSpeechCredentials.ts @@ -0,0 +1,145 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Emitter, Event } from '../../../../base/common/event.js'; +import { Disposable } from '../../../../base/common/lifecycle.js'; +import { IConfigurationService } from '../../../../platform/configuration/common/configuration.js'; +import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js'; +import { ILogService } from '../../../../platform/log/common/log.js'; +import { ISecretStorageService } from '../../../../platform/secrets/common/secrets.js'; +import { AccessibilityVoiceSettingId } from '../common/speechService.js'; + +export const IMaiSpeechCredentialsService = createDecorator('maiSpeechCredentialsService'); + +export const MAI_SPEECH_ENDPOINT_SETTING = AccessibilityVoiceSettingId.MaiSpeechEndpoint; + +/** Key of the secret holding the speech service key, in {@link ISecretStorageService}. */ +export const MAI_SPEECH_KEY_SECRET = 'speech.mai.key'; + +export interface IMaiSpeechCredentials { + readonly endpoint: string; + readonly key: string; +} + +/** + * Where the MAI speech service lives and how to authenticate with it. + * + * Deliberately separate from the engine that uses it: the key is supplied by the + * user for now, and is expected to be replaced by an endpoint that authenticates + * with the identity the user already signed in with, the way voice mode does. + * Only this service should need to change for that. + */ +export interface IMaiSpeechCredentialsService { + + readonly _serviceBrand: undefined; + + /** + * Whether an endpoint and a key are both available, and reading aloud can + * therefore use this service. Fires {@link onDidChangeConfigured} when this + * changes, so that the engine can be offered or withdrawn. + */ + readonly isConfigured: boolean; + readonly onDidChangeConfigured: Event; + + /** The credentials to use, or `undefined` when they are not configured. */ + resolve(): Promise; + + /** Stores `key` for the configured endpoint, or forgets it when empty. */ + setKey(key: string | undefined): Promise; +} + +export class MaiSpeechCredentialsService extends Disposable implements IMaiSpeechCredentialsService { + + declare readonly _serviceBrand: undefined; + + private readonly _onDidChangeConfigured = this._register(new Emitter()); + readonly onDidChangeConfigured = this._onDidChangeConfigured.event; + + private hasKey = false; + + get isConfigured(): boolean { + return !!this.endpoint && this.hasKey; + } + + private get endpoint(): string | undefined { + const configured = this.configurationService.getValue(MAI_SPEECH_ENDPOINT_SETTING)?.trim(); + if (!configured) { + return undefined; + } + + // The key travels with every request, so it must never leave the machine + // in the clear. `localhost` is allowed so the service can be run locally. + try { + const url = new URL(configured); + const isLocal = url.hostname === 'localhost' || url.hostname === '127.0.0.1'; + + return url.protocol === 'https:' || (isLocal && url.protocol === 'http:') ? configured : undefined; + } catch { + return undefined; // not a URL at all + } + } + + constructor( + @IConfigurationService private readonly configurationService: IConfigurationService, + @ISecretStorageService private readonly secretStorageService: ISecretStorageService, + @ILogService private readonly logService: ILogService + ) { + super(); + + this._register(this.configurationService.onDidChangeConfiguration(e => { + if (e.affectsConfiguration(MAI_SPEECH_ENDPOINT_SETTING)) { + this._onDidChangeConfigured.fire(); + } + })); + + // The key is shared between windows, so it can be set or removed by one + // of the others while this one is running. + this._register(this.secretStorageService.onDidChangeSecret(key => { + if (key === MAI_SPEECH_KEY_SECRET) { + this.refreshHasKey(); + } + })); + + this.refreshHasKey(); + } + + private async refreshHasKey(): Promise { + const wasConfigured = this.isConfigured; + this.hasKey = !!await this.readKey(); + + if (this.isConfigured !== wasConfigured) { + this._onDidChangeConfigured.fire(); + } + } + + private async readKey(): Promise { + try { + return await this.secretStorageService.get(MAI_SPEECH_KEY_SECRET) || undefined; + } catch (error) { + // Secret storage is unavailable on some platforms; reading aloud then + // falls back to the speech synthesizer of the platform. + this.logService.warn(`[speech] could not read the MAI speech key: ${error}`); + + return undefined; + } + } + + async resolve(): Promise { + const endpoint = this.endpoint; + const key = await this.readKey(); + + return endpoint && key ? { endpoint, key } : undefined; + } + + async setKey(key: string | undefined): Promise { + if (key?.trim()) { + await this.secretStorageService.set(MAI_SPEECH_KEY_SECRET, key.trim()); + } else { + await this.secretStorageService.delete(MAI_SPEECH_KEY_SECRET); + } + + await this.refreshHasKey(); + } +} diff --git a/src/vs/workbench/contrib/speech/browser/maiTextToSpeech.ts b/src/vs/workbench/contrib/speech/browser/maiTextToSpeech.ts new file mode 100644 index 00000000000000..c20b742b178406 --- /dev/null +++ b/src/vs/workbench/contrib/speech/browser/maiTextToSpeech.ts @@ -0,0 +1,321 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { mainWindow } from '../../../../base/browser/window.js'; +import { CancellationToken } from '../../../../base/common/cancellation.js'; +import { Emitter, Event } from '../../../../base/common/event.js'; +import { Disposable, IDisposable, toDisposable } from '../../../../base/common/lifecycle.js'; +import { IConfigurationService } from '../../../../platform/configuration/common/configuration.js'; +import { IInstantiationService } from '../../../../platform/instantiation/common/instantiation.js'; +import { ILogService } from '../../../../platform/log/common/log.js'; +import { AccessibilityVoiceSettingId, IBuiltinTextToSpeechEngine, ITextToSpeechEvent, ITextToSpeechSession, ITextToSpeechSessionOptions, TextToSpeechStatus } from '../common/speechService.js'; +import { splitForSynthesis } from '../common/speechText.js'; +import { BuiltinTextToSpeechEngine } from './builtinTextToSpeech.js'; +import { IMaiSpeechCredentialsService } from './maiSpeechCredentials.js'; + +/** + * Voices of the MAI text-to-speech model, by the language they speak. The model + * offers many more; these are one well rated voice per language so that reading + * works without the user choosing anything. + */ +const VOICES_BY_LANGUAGE = new Map([ + ['en', 'en-US-Harper:MAI-Voice-2'], + ['de', 'de-DE-Mia:MAI-Voice-2'], + ['es', 'es-ES-Marta:MAI-Voice-2'], + ['fr', 'fr-FR-Soleil:MAI-Voice-2'], + ['hi', 'hi-IN-Priya:MAI-Voice-2'], + ['it', 'it-IT-Rosa:MAI-Voice-2'], + ['ja', 'ja-JP-Sakura:MAI-Voice-2-Flash'], + ['ko', 'ko-KR-Haena:MAI-Voice-2'], + ['nl', 'nl-NL-Fleur:MAI-Voice-2'], + ['pt', 'pt-BR-Luana:MAI-Voice-2'], + ['ru', 'ru-RU-Masha:MAI-Voice-2'], + ['th', 'th-TH-Krit:MAI-Voice-2'], + ['tr', 'tr-TR-Elif:MAI-Voice-2'], + ['vi', 'vi-VN-Linh:MAI-Voice-2-Flash'], + ['zh', 'zh-CN-Mei:MAI-Voice-2'], +]); + +const SAMPLE_RATE = 24000; +const OUTPUT_FORMAT = 'riff-24khz-16bit-mono-pcm'; + +export const MAI_VOICE_SETTING = AccessibilityVoiceSettingId.MaiVoice; + +/** + * The shape of a voice identifier, for example `en-US-Harper:MAI-Voice-2`. A + * configured voice is matched against this before it is used: anything else is + * not a voice the service knows, and letting it through would put arbitrary + * text into the attribute of the document below. + */ +const VOICE_PATTERN = /^[a-z]{2,3}(-[A-Za-z0-9]+)*:[A-Za-z0-9-]+$/; + +/** + * Picks the voice to read `language` (a BCP-47 tag such as `en-US`) with, or + * `undefined` when the model has no voice for it and the platform synthesizer + * should read instead. + */ +export function pickMaiVoice(language: string | undefined, configuredVoice?: string): string | undefined { + const configured = configuredVoice?.trim(); + if (configured && VOICE_PATTERN.test(configured)) { + return configured; + } + + const primary = (language ?? 'en').toLowerCase().replace(/_/g, '-').split('-')[0]; + + return VOICES_BY_LANGUAGE.get(primary); +} + +function escapeXml(value: string): string { + return value.replace(/[&<>"']/g, char => { + switch (char) { + case '&': return '&'; + case '<': return '<'; + case '>': return '>'; + case '"': return '"'; + default: return '''; + } + }); +} + +/** + * Wraps `text` in the SSML document the service expects. + * + * Everything interpolated here is escaped, including the attributes: chat + * responses routinely contain `&` and `<`, which would otherwise make the + * document invalid, and text that closed its own element could add markup of + * its own, such as an `