feat(audio): play streamed TTS audio chunks via Web Audio

src/renderer/WebSDK/src/lappmodel.ts

@@ -43,6 +43,7 @@ import { canvas, gl } from "./lappglmanager";
 import { LAppPal } from "./lapppal";
 import { TextureInfo } from "./lapptexturemanager";
 import { LAppWavFileHandler } from "./lappwavfilehandler";
+import { getStreamingLipSyncRms } from "../../src/utils/stream-audio-player";
 import { CubismMoc } from "@framework/model/cubismmoc";
 
 enum LoadStep {
@@ -616,6 +617,9 @@ export class LAppModel extends CubismUserModel {
       let value = 0.0;
       this._wavFileHandler.update(deltaTimeSeconds);
       value = this._wavFileHandler.getRms();
+      // Streamed (chunked PCM) playback bypasses the wav file handler;
+      // take whichever source is currently louder.
+      value = Math.max(value, getStreamingLipSyncRms());
       value = Math.min(1.0, value * 1.5);
 
       const lipSyncWeight = 4.0;

src/renderer/src/hooks/utils/use-audio-task.ts

@@ -8,6 +8,7 @@ import { useSubtitle } from '@/context/subtitle-context';
 import { useChatHistory } from '@/context/chat-history-context';
 import { audioTaskQueue } from '@/utils/task-queue';
 import { audioManager } from '@/utils/audio-manager';
+import { streamAudioPlayer } from '@/utils/stream-audio-player';
 import { toaster } from '@/components/ui/toaster';
 import { useWebSocket } from '@/context/websocket-context';
 import { DisplayText } from '@/services/websocket-service';
@@ -27,6 +28,13 @@ interface AudioTaskOptions {
   forwarded?: boolean
 }
 
+interface StreamAudioTaskOptions {
+  streamId: string
+  displayText?: DisplayText | null
+  expressions?: string[] | number[] | null
+  forwarded?: boolean
+}
+
 /**
  * Custom hook for handling audio playback tasks with Live2D lip sync
  */
@@ -226,6 +234,69 @@ export const useAudioTask = () => {
     }
   });
 
+  /**
+   * Handle streamed audio playback (chunked PCM via streamAudioPlayer)
+   * with the same display/expression handling as file-based playback.
+   * Lip sync comes from the stream player's analyser (see lappmodel.ts).
+   */
+  const handleStreamAudioPlayback = async (options: StreamAudioTaskOptions): Promise<void> => {
+    const {
+      aiState: currentAiState,
+      setSubtitleText: updateSubtitle,
+      appendResponse: appendText,
+      appendAIMessage: appendAI,
+    } = stateRef.current;
+
+    if (currentAiState === 'interrupted') {
+      console.warn('Stream audio playback blocked by interruption state.');
+      streamAudioPlayer.dropStream(options.streamId);
+      return;
+    }
+
+    const { streamId, displayText, expressions, forwarded } = options;
+
+    if (displayText) {
+      appendText(displayText.text);
+      appendAI(displayText.text, displayText.name, displayText.avatar);
+      updateSubtitle(displayText.text);
+      if (!forwarded) {
+        sendMessage({
+          type: "audio-play-start",
+          display_text: displayText,
+          forwarded: true,
+        });
+      }
+    }
+
+    try {
+      const live2dManager = (window as any).getLive2DManager?.();
+      const model = live2dManager?.getModel(0);
+
+      const lappAdapter = (window as any).getLAppAdapter?.();
+      if (lappAdapter && expressions?.[0] !== undefined) {
+        setExpression(
+          expressions[0],
+          lappAdapter,
+          `Set expression to: ${expressions[0]}`,
+        );
+      }
+
+      if (model && LAppDefine && LAppDefine.PriorityNormal) {
+        model.startRandomMotion("Talk", LAppDefine.PriorityNormal);
+      }
+
+      await streamAudioPlayer.play(streamId);
+      console.log('Stream audio playback completed');
+    } catch (error) {
+      console.error('Stream audio playback error:', error);
+      toaster.create({
+        title: `${t('error.audioPlayback')}: ${error}`,
+        type: "error",
+        duration: 2000,
+      });
+    }
+  };
+
   // Handle backend synthesis completion
   useEffect(() => {
     let isMounted = true;
@@ -261,8 +332,26 @@ export const useAudioTask = () => {
     audioTaskQueue.addTask(() => handleAudioPlayback(options));
   };
 
+  /**
+   * Add a streamed audio task to the queue. Rides the same queue as
+   * file-based tasks, so sentence order is preserved across both kinds.
+   */
+  const addStreamAudioTask = async (options: StreamAudioTaskOptions) => {
+    const { aiState: currentState } = stateRef.current;
+
+    if (currentState === 'interrupted') {
+      console.log('Skipping stream audio task due to interrupted state');
+      streamAudioPlayer.dropStream(options.streamId);
+      return;
+    }
+
+    console.log(`Adding stream audio task ${options.displayText?.text} to queue`);
+    audioTaskQueue.addTask(() => handleStreamAudioPlayback(options));
+  };
+
   return {
     addAudioTask,
+    addStreamAudioTask,
     appendResponse,
     stopCurrentAudioAndLipSync,
   };

src/renderer/src/services/websocket-handler.tsx

@@ -10,6 +10,7 @@ import {
 import { ModelInfo, useLive2DConfig } from '@/context/live2d-config-context';
 import { useSubtitle } from '@/context/subtitle-context';
 import { audioTaskQueue } from '@/utils/task-queue';
+import { streamAudioPlayer } from '@/utils/stream-audio-player';
 import { useAudioTask } from '@/components/canvas/live2d';
 import { useBgUrl } from '@/context/bgurl-context';
 import { useConfig } from '@/context/character-config-context';
@@ -31,7 +32,7 @@ function WebSocketHandler({ children }: { children: React.ReactNode }) {
   const { setModelInfo } = useLive2DConfig();
   const { setSubtitleText } = useSubtitle();
   const { clearResponse, setForceNewMessage, appendHumanMessage, appendOrUpdateToolCallMessage } = useChatHistory();
-  const { addAudioTask } = useAudioTask();
+  const { addAudioTask, addStreamAudioTask } = useAudioTask();
   const bgUrlContext = useBgUrl();
   const { confUid, setConfName, setConfUid, setConfigFiles } = useConfig();
   const [pendingModelInfo, setPendingModelInfo] = useState<ModelInfo | undefined>(undefined);
@@ -69,6 +70,7 @@ function WebSocketHandler({ children }: { children: React.ReactNode }) {
       case 'conversation-chain-start':
         setAiState('thinking-speaking');
         audioTaskQueue.clearQueue();
+        streamAudioPlayer.stopAll();
         clearResponse();
         break;
       case 'conversation-chain-end':
@@ -167,6 +169,31 @@ function WebSocketHandler({ children }: { children: React.ReactNode }) {
           });
         }
         break;
+      case 'audio-stream-start':
+        if (!message.stream_id) break;
+        if (aiState === 'interrupted' || aiState === 'listening') {
+          console.log('Stream playback intercepted. Sentence:', message.display_text?.text);
+          streamAudioPlayer.dropStream(message.stream_id);
+        } else {
+          streamAudioPlayer.openStream(message.stream_id, message.sample_rate || 24000);
+          addStreamAudioTask({
+            streamId: message.stream_id,
+            displayText: message.display_text || null,
+            expressions: message.actions?.expressions || null,
+            forwarded: message.forwarded || false,
+          });
+        }
+        break;
+      case 'audio-stream-chunk':
+        if (message.stream_id && message.chunk) {
+          streamAudioPlayer.pushChunk(message.stream_id, message.chunk);
+        }
+        break;
+      case 'audio-stream-end':
+        if (message.stream_id) {
+          streamAudioPlayer.endStream(message.stream_id);
+        }
+        break;
       case 'history-data':
         if (message.messages) {
           setMessages(message.messages);
@@ -289,7 +316,7 @@ function WebSocketHandler({ children }: { children: React.ReactNode }) {
       default:
         console.warn('Unknown message type:', message.type);
     }
-  }, [aiState, addAudioTask, appendHumanMessage, baseUrl, bgUrlContext, setAiState, setConfName, setConfUid, setConfigFiles, setCurrentHistoryUid, setHistoryList, setMessages, setModelInfo, setSubtitleText, startMic, stopMic, setSelfUid, setGroupMembers, setIsOwner, backendSynthComplete, setBackendSynthComplete, clearResponse, handleControlMessage, appendOrUpdateToolCallMessage, interrupt, setBrowserViewData, t]);
+  }, [aiState, addAudioTask, addStreamAudioTask, appendHumanMessage, baseUrl, bgUrlContext, setAiState, setConfName, setConfUid, setConfigFiles, setCurrentHistoryUid, setHistoryList, setMessages, setModelInfo, setSubtitleText, startMic, stopMic, setSelfUid, setGroupMembers, setIsOwner, backendSynthComplete, setBackendSynthComplete, clearResponse, handleControlMessage, appendOrUpdateToolCallMessage, interrupt, setBrowserViewData, t]);
 
   useEffect(() => {
     wsService.connect(wsUrl);

src/renderer/src/services/websocket-service.tsx

@@ -59,6 +59,9 @@ export interface MessageEvent {
   audio?: string;
   volumes?: number[];
   slice_length?: number;
+  stream_id?: string;
+  sample_rate?: number;
+  chunk?: string;
   files?: BackgroundFile[];
   actions?: Actions;
   text?: string;

src/renderer/src/utils/audio-manager.ts

@@ -1,3 +1,5 @@
+import { streamAudioPlayer } from './stream-audio-player';
+
 /**
  * Global audio manager for handling audio playback and interruption
  * This ensures all components share the same audio reference
@@ -18,6 +20,10 @@ class AudioManager {
    * Stop current audio playback and lip sync
    */
   stopCurrentAudioAndLipSync() {
+    // Streamed (chunked PCM) playback is managed separately from the
+    // HTMLAudioElement path — always stop it too.
+    streamAudioPlayer.stopAll();
+
     if (this.currentAudio) {
       console.log('[AudioManager] Stopping current audio and lip sync');
       const audio = this.currentAudio;