diff --git a/.gitignore b/.gitignore index ef8ff67..f82647c 100644 --- a/.gitignore +++ b/.gitignore @@ -142,3 +142,5 @@ vite.config.ts.timestamp-* # macOS Finder metadata .DS_Store + +/apps/web-demo/ diff --git a/AGENT.md b/AGENT.md index 97231c7..1073aef 100644 --- a/AGENT.md +++ b/AGENT.md @@ -36,7 +36,7 @@ pnpm --filter typecheck Exports from `agora-agent-client-toolkit`: - `AgoraVoiceAI` — main singleton class (async `init()`) -- `AgoraVoiceAIConfig`, `RTMConfig` — config interfaces +- `AgoraVoiceAIConfig` — config interface - `AgoraVoiceAIEvents` — event name constants - `CovSubRenderController` — transcript rendering controller - `ChunkedMessageAssembler` — stream message assembly @@ -54,7 +54,7 @@ Exports from `agora-agent-client-toolkit-react`: ## Constraints - **Do not modify `CovSubRenderController`** without explicit task scope. It is battle-tested rendering logic; bugs here are silent and hard to reproduce without real agent traffic. -- **RTM is optional** — never assume `rtmEngine` is present. Use `rtmConfig?.rtmEngine`. +- **RTM is optional** — never assume `rtmEngine` is present. - **`AgoraVoiceAI.init()` is async** — always `await`. - **pnpm only** — no npm or yarn commands. - **`jszip` and `@agora-js/report` are optional deps** — guard all usages. @@ -65,7 +65,7 @@ Exports from `agora-agent-client-toolkit-react`: // Core config interface AgoraVoiceAIConfig { rtcEngine: IAgoraRTCClient; // required - rtmConfig?: { rtmEngine: RTMClient }; // optional + rtmEngine?: RTMClient; // optional renderMode?: TranscriptHelperMode; // TEXT | WORD | AUTO enableLog?: boolean; enableAgoraMetrics?: boolean; diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a8ee5d..e04d2ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,21 @@ Migration notes for each release should link to the matching section in [MIGRATI `AGENT_STATE_CHANGED` remains supported and emitted; React `agentState` APIs remain the aggregate compatibility surface. +## [v2.9.1] - 2026-07-20 + +### Fixed + +- Restored the `ConversationalAIAPI` export and standardized RTM config on top-level `rtmEngine`. +- Fixed WORD-to-TEXT fallback to preserve rendered history and discard pending WORD data. + +### Changed + +- Bumped the core and React npm packages to `2.9.1`. + +### Upgrade notes + +- Migration guide: see [MIGRATION.md#290---291](./MIGRATION.md#290---291). + ## [v2.9.0] - 2026-07-10 Stable release for the 2.9.0 ConvoAI API line. diff --git a/CLAUDE.md b/CLAUDE.md index b596914..f1f0df1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -62,7 +62,7 @@ The rendering controller is the most complex and highest-risk module in the code - `AgoraVoiceAI` is a **singleton** — `init()` creates it, `getInstance()` retrieves it, `destroy()` clears it. - `AgoraVoiceAI.init()` is **async** — always `await` it. -- RTM is **optional** — `rtmConfig?: { rtmEngine }`. Three methods throw if called without it: `sendText`, `sendImage`, `interrupt`. +- RTM is **optional** — use the top-level `rtmEngine` field. RTM-backed methods throw if it is omitted. - `AgoraVoiceAIConfig` is defined in `src/core/config.ts`. `src/core/conversational-ai.ts` re-exports it — don't define it there. ## Package names diff --git a/MIGRATION.md b/MIGRATION.md index 211dae5..c301a99 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -4,11 +4,26 @@ Use this file for all version-to-version upgrade steps. ## Version index +- [2.9.0 -> 2.9.1](#290---291) - [1.2.x -> 2.9.0](#12x---290) - [1.1.x -> 1.2.0](#11x---120) --- +## 2.9.0 -> 2.9.1 + +This release restores the `ConversationalAIAPI` export and its legacy enum/type names. All +initialization paths now use the top-level `rtmEngine` field. Replace +`rtmConfig: { rtmEngine }` with `rtmEngine` when upgrading. + +`enableRenderModeFallback` defaults to `true`. In WORD mode, messages without word timing data +switch rendering to TEXT while preserving text already emitted. Set it to `false` to keep the +previous WORD-only behavior. + +Keep `agora-agent-client-toolkit-react` and `agora-agent-client-toolkit` on the same version. + +--- + ## 1.2.x -> 2.9.0 ### TL;DR diff --git a/README.md b/README.md index ad0fc20..16b09ec 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ pnpm add agora-agent-client-toolkit-react agora-agent-client-toolkit agora-rtc-r ## Migration -Upgrading from an earlier release? See the [migration guide](./MIGRATION.md), including the steps for `1.2.x -> 2.9.0`. +Upgrading from an earlier release? See the [migration guide](./MIGRATION.md), including the steps for `2.9.0 -> 2.9.1`. ### Optional dependencies @@ -58,7 +58,7 @@ rtcClient.on('user-published', async (user, mediaType) => { // --- Add Conversational AI features --- const ai = await AgoraVoiceAI.init({ rtcEngine: rtcClient, - rtmConfig: { rtmEngine: rtmClient }, + rtmEngine: rtmClient, }); ai.on(AgoraVoiceAIEvents.TRANSCRIPT_UPDATED, (transcript) => { @@ -113,7 +113,7 @@ function App() { const config = useMemo( () => ({ channel: 'my-channel', - rtmConfig: { rtmEngine: rtmClient }, + rtmEngine: rtmClient, }), [] ); @@ -155,20 +155,20 @@ function VoiceSession() { | Package | Version | Description | | ---------------------------------------------------------------------------- | ------- | ---------------------------------- | -| [`agora-agent-client-toolkit`](./packages/conversational-ai/README.md) | 2.9.0 | Core SDK — vanilla JS / TypeScript | -| [`agora-agent-client-toolkit-react`](./packages/react/README.md) | 2.9.0 | React hooks | +| [`agora-agent-client-toolkit`](./packages/conversational-ai/README.md) | 2.9.1 | Core SDK — vanilla JS / TypeScript | +| [`agora-agent-client-toolkit-react`](./packages/react/README.md) | 2.9.1 | React hooks | Full API reference, configuration options, and events are in each package's README. ## RTC-only mode (no RTM) -RTM is optional. Transcripts and agent state work without it — just omit `rtmConfig`: +RTM is optional. Transcripts and agent state work without it — just omit `rtmEngine`: ```typescript const ai = await AgoraVoiceAI.init({ rtcEngine: rtcClient }); ``` -RTM-backed methods throw without `rtmConfig`: `sendText`, `sendImage`, `interrupt`, `manualSOS`, and `manualEOS`. +RTM-backed methods throw without `rtmEngine`: `sendText`, `sendImage`, `interrupt`, `manualSOS`, and `manualEOS`. ## Repository layout diff --git a/apps/demo/demo.ts b/apps/demo/demo.ts index 48d7ba4..6a379a7 100644 --- a/apps/demo/demo.ts +++ b/apps/demo/demo.ts @@ -61,7 +61,7 @@ console.log('✓ RTM Client created'); * * Options: * - rtcEngine: Your RTC client instance (required) - * - rtmConfig: Optional RTM config object — omit to run RTC-only + * - rtmEngine: Optional RTM client — omit to run RTC-only * (sendText, sendImage, and interrupt will throw if RTM is absent) * - renderMode: Transcript rendering mode (TEXT, WORD, CHUNK, AUTO, UNKNOWN) * - enableLog: Enable debug logging @@ -70,7 +70,7 @@ console.log('✓ RTM Client created'); async function initVoiceAI() { const voiceAI = await AgoraVoiceAI.init({ rtcEngine: rtcClient, - rtmConfig: { rtmEngine: rtmClient }, + rtmEngine: rtmClient, renderMode: TranscriptHelperMode.TEXT, enableLog: true, }); diff --git a/apps/playground/src/components/SessionProvider.tsx b/apps/playground/src/components/SessionProvider.tsx index 5a851c3..300c420 100644 --- a/apps/playground/src/components/SessionProvider.tsx +++ b/apps/playground/src/components/SessionProvider.tsx @@ -66,7 +66,7 @@ interface Props { export function SessionProvider({ credentials, rtcClient, rtmClient, onDisconnect }: Props) { const config = useMemo( () => ({ - rtmConfig: rtmClient ? { rtmEngine: rtmClient } : undefined, + rtmEngine: rtmClient ?? undefined, renderMode: credentials.renderMode, enableLog: credentials.enableLog, channel: credentials.channelName, diff --git a/package.json b/package.json index 3262ef5..8404312 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "agora-agent-client-toolkit", - "version": "2.9.0", + "version": "2.9.1", "private": true, "description": "Client-side SDK for adding Agora Conversational AI Engine features.", "scripts": { diff --git a/packages/conversational-ai/README.md b/packages/conversational-ai/README.md index 67f378d..6dbac6a 100644 --- a/packages/conversational-ai/README.md +++ b/packages/conversational-ai/README.md @@ -28,7 +28,7 @@ await rtmClient.login({ token: 'YOUR_RTM_TOKEN' }); // 2. Initialize the AI singleton const ai = await AgoraVoiceAI.init({ rtcEngine: rtcClient, - rtmConfig: { rtmEngine: rtmClient }, + rtmEngine: rtmClient, renderMode: TranscriptHelperMode.WORD, }); @@ -95,9 +95,10 @@ The following parameters must be set when starting the AI agent via the Agora RE | Field | Type | Required | Description | |-------|------|----------|-------------| | `rtcEngine` | `RTCEngine` | Yes | Structural RTC client contract (`on/off` for `'audio-pts'` and `'stream-message'`) | -| `rtmConfig` | `{ rtmEngine: RTMEngine }` | No | Structural RTM client contract (`publish`, `addEventListener`, `removeEventListener`) | +| `rtmEngine` | `RTMEngine` | No | Structural RTM client contract (`publish`, `addEventListener`, `removeEventListener`) | | `renderMode` | `TranscriptHelperMode` | No | `TEXT`, `WORD`, `CHUNK`, or `AUTO`. If omitted, defaults to `AUTO` — mode is detected from the first agent message. | | `enableLog` | `boolean` | No | Enable debug logging (default: `false`) | +| `enableRenderModeFallback` | `boolean` | No | Fall back from `WORD` to `TEXT` when word timing data is missing (default: `true`) | | `enableAgoraMetrics` | `boolean` | No | Load `@agora-js/report` for usage metrics (default: `false`) | `RTCEngine` and `RTMEngine` are toolkit-exported structural interfaces. A normal `agora-rtc-sdk-ng` client and `agora-rtm` client satisfy them directly, and no `as unknown as` casts are required. @@ -119,13 +120,13 @@ AgoraVoiceAI.getInstance(): AgoraVoiceAI // throws NotInitializedError if not y ai.subscribeMessage(channel: string): void ai.unsubscribe(): void ai.chat(agentUserId: string, message: ChatMessageText | ChatMessageImage): Promise -ai.sendText(agentUserId: string, message: ChatMessageText): Promise // requires rtmConfig -ai.sendImage(agentUserId: string, message: ChatMessageImage): Promise // requires rtmConfig -ai.interrupt(agentUserId: string): Promise // requires rtmConfig -ai.manualSOS(agentUserId: string, requestId?: string): Promise // requires rtmConfig -ai.manualEOS(agentUserId: string, requestId?: string): Promise // requires rtmConfig +ai.sendText(agentUserId: string, message: ChatMessageText): Promise // requires rtmEngine +ai.sendImage(agentUserId: string, message: ChatMessageImage): Promise // requires rtmEngine +ai.interrupt(agentUserId: string): Promise // requires rtmEngine +ai.manualSOS(agentUserId: string, requestId?: string): Promise // requires rtmEngine +ai.manualEOS(agentUserId: string, requestId?: string): Promise // requires rtmEngine ai.destroy(): void -ai.getCfg(): { rtcEngine, renderMode, channel, enableLog } +ai.getCfg(): { rtcEngine, rtmEngine, renderMode, channel, enableLog, enableRenderModeFallback } ai.on(event, handler): void ai.off(event, handler): void ``` @@ -170,7 +171,7 @@ Word-level timing is at `metadata.words` — not at the top level. --- -#### Agent activity events _(requires `rtmConfig`)_ +#### Agent activity events _(requires `rtmEngine`)_ Use the independent activity events for new integrations. More than one flag may be active at the same time. @@ -191,7 +192,7 @@ ai.on(AgoraVoiceAIEvents.AGENT_SPEAKING_CHANGED, (agentUserId, active) => { --- -#### `AGENT_STATE_CHANGED` _(deprecated, requires `rtmConfig`)_ +#### `AGENT_STATE_CHANGED` _(deprecated, requires `rtmEngine`)_ This event is deprecated but remains supported and continues to be emitted. Existing integrations do not need to migrate. Use the independent activity @@ -255,7 +256,7 @@ ai.on(AgoraVoiceAIEvents.AGENT_ERROR, (agentUserId, error) => { --- -#### `MESSAGE_RECEIPT_UPDATED` _(requires `rtmConfig`)_ +#### `MESSAGE_RECEIPT_UPDATED` _(requires `rtmEngine`)_ Fires when a delivery or read receipt is received for a sent message. @@ -269,7 +270,7 @@ ai.on(AgoraVoiceAIEvents.MESSAGE_RECEIPT_UPDATED, (agentUserId, receipt) => { --- -#### `MESSAGE_ERROR` _(requires `rtmConfig`)_ +#### `MESSAGE_ERROR` _(requires `rtmEngine`)_ Fires when a chat message fails to deliver. @@ -283,7 +284,7 @@ ai.on(AgoraVoiceAIEvents.MESSAGE_ERROR, (agentUserId, error) => { --- -#### `MESSAGE_SAL_STATUS` _(requires `rtmConfig`)_ +#### `MESSAGE_SAL_STATUS` _(requires `rtmEngine`)_ Fires when the Speech Activity Level (SAL) registration status changes. @@ -297,7 +298,7 @@ ai.on(AgoraVoiceAIEvents.MESSAGE_SAL_STATUS, (agentUserId, salStatus) => { --- -#### `USER_MANUAL_SOS` / `USER_MANUAL_EOS` / `AGENT_MANUAL_EOS` _(requires `rtmConfig`)_ +#### `USER_MANUAL_SOS` / `USER_MANUAL_EOS` / `AGENT_MANUAL_EOS` _(requires `rtmEngine`)_ Manual turn control is a two-step flow. `manualSOS()` and `manualEOS()` publish an RTM marker and resolve with the `requestId` used in the payload. That only means RTM publish succeeded; server validation arrives later through events. @@ -368,7 +369,7 @@ Advanced: implement `IMetricsReporter` or use the exported `ConsoleMetricsReport | Error Class | When Thrown | Recovery | |------------|------------|----------| | `NotInitializedError` | `getInstance()` or `getCfg()` called before `init()` | Call `await AgoraVoiceAI.init(config)` first | -| `RTMRequiredError` | `sendText()`, `sendImage()`, `interrupt()`, `manualSOS()`, or `manualEOS()` called without RTM | Pass `rtmConfig: { rtmEngine }` in `init()` config | +| `RTMRequiredError` | `sendText()`, `sendImage()`, `interrupt()`, `manualSOS()`, or `manualEOS()` called without RTM | Pass `rtmEngine` in `init()` config | | `ConversationalAIError` | `chat()` called with unsupported message type | Check `message.messageType` is TEXT or IMAGE | All error classes extend `ConversationalAIError`, which extends `Error`. Use `instanceof` to catch specific error types. @@ -400,7 +401,7 @@ try { }); } catch (e) { if (e instanceof RTMRequiredError) { - console.error('RTM not configured — pass rtmConfig to init()'); + console.error('RTM not configured — pass rtmEngine to init()'); } } @@ -455,11 +456,11 @@ See the [Agora Conversational AI documentation](https://docs.agora.io/en/convers **Cause:** Standalone hooks (`useTranscript`, `useAgentState`, etc.) need access to the `AgoraVoiceAI` instance. Without a `ConversationalAIProvider`, they fall back to a single `getInstance()` attempt which may miss the instance if `init()` hasn't completed yet. -**Fix:** Wrap your component tree in `ConversationalAIProvider` so standalone hooks connect through React context. Pass `rtmConfig` when the child hooks consume RTM-backed state or controls: +**Fix:** Wrap your component tree in `ConversationalAIProvider` so standalone hooks connect through React context. Pass `rtmEngine` when the child hooks consume RTM-backed state or controls: ```tsx {/* useTranscript() connects via context */} {/* useAgentState() connects via context */} diff --git a/packages/conversational-ai/__tests__/event-handlers.test.ts b/packages/conversational-ai/__tests__/event-handlers.test.ts index 9a67391..cb9308a 100644 --- a/packages/conversational-ai/__tests__/event-handlers.test.ts +++ b/packages/conversational-ai/__tests__/event-handlers.test.ts @@ -148,7 +148,7 @@ describe('AgoraVoiceAI event handlers', () => { const rtm = createMockRTMClient(); const ai = await AgoraVoiceAI.init({ rtcEngine: rtc as never, - rtmConfig: { rtmEngine: rtm as never }, + rtmEngine: rtm as never, }); const stateHandler = vi.fn(); const listeningHandler = vi.fn(); @@ -189,7 +189,7 @@ describe('AgoraVoiceAI event handlers', () => { const rtm = createMockRTMClient(); const ai = await AgoraVoiceAI.init({ rtcEngine: rtc as never, - rtmConfig: { rtmEngine: rtm as never }, + rtmEngine: rtm as never, }); const stateHandler = vi.fn(); const listeningHandler = vi.fn(); @@ -215,7 +215,7 @@ describe('AgoraVoiceAI event handlers', () => { const rtm = createMockRTMClient(); const ai = await AgoraVoiceAI.init({ rtcEngine: rtc as never, - rtmConfig: { rtmEngine: rtm as never }, + rtmEngine: rtm as never, }); const stateHandler = vi.fn(); @@ -244,7 +244,7 @@ describe('AgoraVoiceAI event handlers', () => { const rtm = createMockRTMClient(); const ai = await AgoraVoiceAI.init({ rtcEngine: rtc as never, - rtmConfig: { rtmEngine: rtm as never }, + rtmEngine: rtm as never, }); const listeningHandler = vi.fn(); @@ -276,7 +276,7 @@ describe('AgoraVoiceAI event handlers', () => { const rtm = createMockRTMClient(); const ai = await AgoraVoiceAI.init({ rtcEngine: rtc as never, - rtmConfig: { rtmEngine: rtm as never }, + rtmEngine: rtm as never, }); const stateHandler = vi.fn(); const listeningHandler = vi.fn(); @@ -321,7 +321,7 @@ describe('AgoraVoiceAI event handlers', () => { const rtm = createMockRTMClient(); const ai = await AgoraVoiceAI.init({ rtcEngine: rtc as never, - rtmConfig: { rtmEngine: rtm as never }, + rtmEngine: rtm as never, }); const stateHandler = vi.fn(); const listeningHandler = vi.fn(); @@ -364,7 +364,7 @@ describe('AgoraVoiceAI event handlers', () => { const rtm = createMockRTMClient(); const ai = await AgoraVoiceAI.init({ rtcEngine: rtc as never, - rtmConfig: { rtmEngine: rtm as never }, + rtmEngine: rtm as never, }); const handler = vi.fn(); @@ -415,7 +415,7 @@ describe('AgoraVoiceAI event handlers', () => { const rtm = createMockRTMClient(); const ai = await AgoraVoiceAI.init({ rtcEngine: rtc as never, - rtmConfig: { rtmEngine: rtm as never }, + rtmEngine: rtm as never, }); const handler = vi.fn(); ai.on(AgoraVoiceAIEvents.USER_MANUAL_SOS, handler); @@ -452,7 +452,7 @@ describe('AgoraVoiceAI event handlers', () => { const rtm = createMockRTMClient(); const ai = await AgoraVoiceAI.init({ rtcEngine: rtc as never, - rtmConfig: { rtmEngine: rtm as never }, + rtmEngine: rtm as never, }); const handler = vi.fn(); ai.on(AgoraVoiceAIEvents.USER_MANUAL_EOS, handler); @@ -489,7 +489,7 @@ describe('AgoraVoiceAI event handlers', () => { const rtm = createMockRTMClient(); const ai = await AgoraVoiceAI.init({ rtcEngine: rtc as never, - rtmConfig: { rtmEngine: rtm as never }, + rtmEngine: rtm as never, }); const handler = vi.fn(); ai.on(AgoraVoiceAIEvents.AGENT_MANUAL_EOS, handler); @@ -525,7 +525,7 @@ describe('AgoraVoiceAI event handlers', () => { const rtm = createMockRTMClient(); const ai = await AgoraVoiceAI.init({ rtcEngine: rtc as never, - rtmConfig: { rtmEngine: rtm as never }, + rtmEngine: rtm as never, }); const handler = vi.fn(); ai.on(AgoraVoiceAIEvents.AGENT_MANUAL_EOS, handler); diff --git a/packages/conversational-ai/__tests__/lifecycle.test.ts b/packages/conversational-ai/__tests__/lifecycle.test.ts index 38f01bf..a202d6d 100644 --- a/packages/conversational-ai/__tests__/lifecycle.test.ts +++ b/packages/conversational-ai/__tests__/lifecycle.test.ts @@ -51,7 +51,7 @@ describe('AgoraVoiceAI lifecycle', () => { expect(rtcClient.on).toHaveBeenCalled(); }); - it('sendText() without rtmConfig throws with a descriptive message', async () => { + it('sendText() without rtmEngine throws with a descriptive message', async () => { const rtcClient = makeRtcClient(); const ai = await AgoraVoiceAI.init({ rtcEngine: rtcClient as never, @@ -61,7 +61,7 @@ describe('AgoraVoiceAI lifecycle', () => { ).rejects.toThrow('requires RTM'); }); - it('interrupt() without rtmConfig throws with a descriptive message', async () => { + it('interrupt() without rtmEngine throws with a descriptive message', async () => { const rtcClient = makeRtcClient(); const ai = await AgoraVoiceAI.init({ rtcEngine: rtcClient as never, @@ -134,23 +134,19 @@ describe('AgoraVoiceAI lifecycle', () => { await expect( AgoraVoiceAI.init({ rtcEngine: rtcClient as never, - rtmConfig: { - rtmEngine: { - addEventListener: vi.fn(), - removeEventListener: vi.fn(), - } as never, - }, + rtmEngine: { + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + } as never, }) ).rejects.toThrow(ConversationalAIError); await expect( AgoraVoiceAI.init({ rtcEngine: rtcClient as never, - rtmConfig: { - rtmEngine: { - addEventListener: vi.fn(), - removeEventListener: vi.fn(), - } as never, - }, + rtmEngine: { + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + } as never, }) ).rejects.toThrow('rtmEngine.publish(channelName, message, options?)'); }); diff --git a/packages/conversational-ai/__tests__/messaging.test.ts b/packages/conversational-ai/__tests__/messaging.test.ts index 2b6895c..7221de6 100644 --- a/packages/conversational-ai/__tests__/messaging.test.ts +++ b/packages/conversational-ai/__tests__/messaging.test.ts @@ -18,7 +18,7 @@ describe('AgoraVoiceAI messaging', () => { const rtm = createMockRTMClient(); const ai = await AgoraVoiceAI.init({ rtcEngine: rtc as never, - rtmConfig: { rtmEngine: rtm as never }, + rtmEngine: rtm as never, }); await ai.sendText('agent-uid', { @@ -54,7 +54,7 @@ describe('AgoraVoiceAI messaging', () => { const rtm = createMockRTMClient(); const ai = await AgoraVoiceAI.init({ rtcEngine: rtc as never, - rtmConfig: { rtmEngine: rtm as never }, + rtmEngine: rtm as never, }); await ai.sendImage('agent-uid', { @@ -75,7 +75,7 @@ describe('AgoraVoiceAI messaging', () => { const rtm = createMockRTMClient(); const ai = await AgoraVoiceAI.init({ rtcEngine: rtc as never, - rtmConfig: { rtmEngine: rtm as never }, + rtmEngine: rtm as never, }); await ai.interrupt('agent-uid'); @@ -97,7 +97,7 @@ describe('AgoraVoiceAI messaging', () => { const rtm = createMockRTMClient(); const ai = await AgoraVoiceAI.init({ rtcEngine: rtc as never, - rtmConfig: { rtmEngine: rtm as never }, + rtmEngine: rtm as never, }); const requestId = await ai.manualSOS('agent-uid', 'sos-req-001'); @@ -115,7 +115,7 @@ describe('AgoraVoiceAI messaging', () => { const rtm = createMockRTMClient(); const ai = await AgoraVoiceAI.init({ rtcEngine: rtc as never, - rtmConfig: { rtmEngine: rtm as never }, + rtmEngine: rtm as never, }); const requestId = await ai.manualEOS('agent-uid', 'eos-req-001'); @@ -133,7 +133,7 @@ describe('AgoraVoiceAI messaging', () => { const rtm = createMockRTMClient(); const ai = await AgoraVoiceAI.init({ rtcEngine: rtc as never, - rtmConfig: { rtmEngine: rtm as never }, + rtmEngine: rtm as never, }); const requestId = await ai.manualSOS('agent-uid'); @@ -148,7 +148,7 @@ describe('AgoraVoiceAI messaging', () => { const rtm = createMockRTMClient(); const ai = await AgoraVoiceAI.init({ rtcEngine: rtc as never, - rtmConfig: { rtmEngine: rtm as never }, + rtmEngine: rtm as never, }); await expect(ai.manualEOS('agent-uid', '')).rejects.toThrow(ConversationalAIError); @@ -167,7 +167,7 @@ describe('AgoraVoiceAI messaging', () => { const rtm = createMockRTMClient(); const ai = await AgoraVoiceAI.init({ rtcEngine: rtc as never, - rtmConfig: { rtmEngine: rtm as never }, + rtmEngine: rtm as never, }); await ai.chat('agent-uid', { @@ -185,7 +185,7 @@ describe('AgoraVoiceAI messaging', () => { const rtm = createMockRTMClient(); const ai = await AgoraVoiceAI.init({ rtcEngine: rtc as never, - rtmConfig: { rtmEngine: rtm as never }, + rtmEngine: rtm as never, }); await ai.chat('agent-uid', { diff --git a/packages/conversational-ai/__tests__/sub-render.test.ts b/packages/conversational-ai/__tests__/sub-render.test.ts new file mode 100644 index 0000000..b25605a --- /dev/null +++ b/packages/conversational-ai/__tests__/sub-render.test.ts @@ -0,0 +1,152 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { CovSubRenderController } from '../../../src/rendering/sub-render'; +import { + MessageType, + TranscriptHelperMode, + TurnStatus, + type AgentTranscription, +} from '../../../src/core/types'; + +function createAgentTranscription( + turnId: number, + text: string, + words: AgentTranscription['words'] +): AgentTranscription { + return { + object: MessageType.AGENT_TRANSCRIPTION, + text, + start_ms: 0, + duration_ms: 0, + language: 'en-US', + turn_id: turnId, + stream_id: 0, + user_id: 'agent', + words, + quiet: false, + turn_seq_id: turnId, + turn_status: TurnStatus.END, + }; +} + +describe('CovSubRenderController render-mode fallback', () => { + const controllers: CovSubRenderController[] = []; + + afterEach(() => { + controllers.forEach((controller) => controller.cleanup()); + vi.useRealTimers(); + }); + + it('falls back from WORD to TEXT when word timing data is missing', () => { + const onChatHistoryUpdated = vi.fn(); + const controller = new CovSubRenderController({ onChatHistoryUpdated }); + controllers.push(controller); + controller.setMode(TranscriptHelperMode.WORD, { enableRenderModeFallback: true }); + controller.run(); + + controller.handleMessage(createAgentTranscription(1, 'plain text', null), { + publisher: 'agent', + }); + controller.handleMessage( + createAgentTranscription(2, 'timed text', [ + { word: 'timed', start_ms: 1000, duration_ms: 100, stable: true }, + ]), + { publisher: 'agent' } + ); + + expect(controller.chatHistory.map((item) => item.text)).toEqual(['plain text', 'timed text']); + expect(onChatHistoryUpdated).toHaveBeenCalledTimes(2); + }); + + it('enables render-mode fallback by default', () => { + const controller = new CovSubRenderController(); + controllers.push(controller); + controller.setMode(TranscriptHelperMode.WORD); + controller.run(); + + controller.handleMessage(createAgentTranscription(1, 'plain text', null), { + publisher: 'agent', + }); + + expect(controller.chatHistory.map((item) => item.text)).toEqual(['plain text']); + }); + + it('preserves only rendered WORD text when falling back to TEXT', () => { + vi.useFakeTimers(); + const controller = new CovSubRenderController(); + controllers.push(controller); + controller.setMode(TranscriptHelperMode.WORD, { enableRenderModeFallback: true }); + controller.run(); + controller.setPts(200); + + controller.handleMessage( + createAgentTranscription(1, 'hello', [ + { word: 'h', start_ms: 100, duration_ms: 100, stable: true }, + { word: 'e', start_ms: 200, duration_ms: 100, stable: true }, + { word: 'l', start_ms: 300, duration_ms: 100, stable: true }, + { word: 'l', start_ms: 400, duration_ms: 100, stable: true }, + { word: 'o', start_ms: 500, duration_ms: 100, stable: true }, + ]), + { publisher: 'agent' } + ); + vi.advanceTimersByTime(200); + expect(controller.chatHistory[0]?.text).toBe('he'); + + controller.handleMessage(createAgentTranscription(2, 'plain text', null), { + publisher: 'agent', + }); + + expect(controller.chatHistory.map((item) => item.text)).toEqual(['he', 'plain text']); + }); + + it('does not restore unplayed text for an interrupted WORD turn', () => { + vi.useFakeTimers(); + const controller = new CovSubRenderController(); + controllers.push(controller); + controller.setMode(TranscriptHelperMode.WORD, { enableRenderModeFallback: true }); + controller.run(); + controller.setPts(200); + + controller.handleMessage( + createAgentTranscription(1, 'hello', [ + { word: 'h', start_ms: 100, duration_ms: 100, stable: true }, + { word: 'e', start_ms: 200, duration_ms: 100, stable: true }, + { word: 'l', start_ms: 300, duration_ms: 100, stable: true }, + { word: 'l', start_ms: 400, duration_ms: 100, stable: true }, + { word: 'o', start_ms: 500, duration_ms: 100, stable: true }, + ]), + { publisher: 'agent' } + ); + vi.advanceTimersByTime(200); + expect(controller.chatHistory[0]?.text).toBe('he'); + + controller.handleMessage( + { + object: MessageType.MSG_INTERRUPTED, + message_id: 'interrupt-1', + data_type: 'message', + turn_id: 1, + start_ms: 200, + send_ts: 200, + }, + { publisher: 'agent' } + ); + controller.handleMessage(createAgentTranscription(2, 'plain text', null), { + publisher: 'agent', + }); + + expect(controller.chatHistory[0]?.text).toBe('he'); + }); + + it('keeps WORD mode when render-mode fallback is disabled', () => { + const controller = new CovSubRenderController(); + controllers.push(controller); + controller.setMode(TranscriptHelperMode.WORD, { enableRenderModeFallback: false }); + controller.run(); + + controller.handleMessage(createAgentTranscription(1, 'plain text', null), { + publisher: 'agent', + }); + + expect(controller.chatHistory).toEqual([]); + }); +}); diff --git a/packages/conversational-ai/__typetests__/interop.ts b/packages/conversational-ai/__typetests__/interop.ts index d154118..6f7b119 100644 --- a/packages/conversational-ai/__typetests__/interop.ts +++ b/packages/conversational-ai/__typetests__/interop.ts @@ -1,7 +1,51 @@ +import { + ConversationalAIAPI, + EAgentState, + EChatMessagePriority, + EChatMessageType, + EConversationalAIAPIEvents, + ELocalTranscriptStatus, + EMessageSalStatus, + EMessageType, + EModuleType, + ERTCCustomEvents, + ERTCEvents, + ERTMEvents, + ETranscriptHelperMode, + ETurnStatus, + NotFoundError, +} from '../../../src'; import type { AgoraVoiceAIConfig, + IAgentTranscription, + IChatMessageBase, + IChatMessageImage, + IChatMessageText, + IConversationalAIAPIConfig, + IConversationalAIAPIEventHandlers, + IHelperRTCEvents, + ILocalImageTranscription, + ILocalTranscriptionBase, + IMessageError, + IMessageInterrupt, + IMessageMetrics, + IMessageSalStatus, + IPresenceState, + ITranscriptHelperItem, + ITranscriptionBase, + ITurnFinishedMessage, + IUserTracks, + IUserTranscription, RTCEngine, RTMEngine, + TAgentMetric, + TAgentTurnFinished, + TDataChunkMessageWord, + TMessageReceipt, + TModuleError, + TQueueItem, + TStateChangeEvent, + TTranscriptHelperObjectWord, } from '../../../src'; declare const foreignRtcClient: { @@ -42,11 +86,40 @@ const rtmEngine = acceptsRtmEngine(foreignRtmClient); const config: AgoraVoiceAIConfig = { rtcEngine, - rtmConfig: { - rtmEngine, - }, + rtmEngine, + enableRenderModeFallback: true, }; const rtcOnlyConfig: AgoraVoiceAIConfig = { rtcEngine }; +const legacyRtcOnlyConfig: IConversationalAIAPIConfig = rtcOnlyConfig; +const legacyConfig: IConversationalAIAPIConfig = { + rtcEngine, + rtmEngine, + renderMode: ETranscriptHelperMode.WORD, + enableLog: false, + enableRenderModeFallback: true, +}; + +async function initLegacySourceNames() { + const api = await ConversationalAIAPI.init(legacyConfig); + api.on(EConversationalAIAPIEvents.TRANSCRIPT_UPDATED, () => undefined); +} + +const legacyEnumExports = [ + EAgentState, + EChatMessagePriority, + EChatMessageType, + EConversationalAIAPIEvents, + ELocalTranscriptStatus, + EMessageSalStatus, + EMessageType, + EModuleType, + ERTCCustomEvents, + ERTCEvents, + ERTMEvents, + ETranscriptHelperMode, + ETurnStatus, + NotFoundError, +]; declare const strictRtcEngine: RTCEngine; strictRtcEngine.on('audio-pts', (pts) => { @@ -75,3 +148,7 @@ strictRtcEngine.on('stream-message', (pts: number) => { void config; void rtcOnlyConfig; +void legacyRtcOnlyConfig; +void legacyConfig; +void initLegacySourceNames; +void legacyEnumExports; diff --git a/packages/conversational-ai/package.json b/packages/conversational-ai/package.json index ca0a25c..67b5c0b 100644 --- a/packages/conversational-ai/package.json +++ b/packages/conversational-ai/package.json @@ -1,6 +1,6 @@ { "name": "agora-agent-client-toolkit", - "version": "2.9.0", + "version": "2.9.1", "description": "Agora Agent Client Toolkit — real-time voice agent integration for web", "main": "./dist/index.js", "module": "./dist/index.mjs", diff --git a/packages/react/README.md b/packages/react/README.md index 7a16dea..7e8da83 100644 --- a/packages/react/README.md +++ b/packages/react/README.md @@ -7,7 +7,7 @@ For RTC primitives (microphone tracks, camera tracks, remote users, volume level ## Install ```bash -pnpm add agora-agent-client-toolkit-react@2.9.0 agora-agent-client-toolkit@2.9.0 agora-rtc-react agora-rtc-sdk-ng agora-rtm +pnpm add agora-agent-client-toolkit-react@2.9.1 agora-agent-client-toolkit@2.9.1 agora-rtc-react agora-rtc-sdk-ng agora-rtm ``` Keep `agora-agent-client-toolkit-react` and `agora-agent-client-toolkit` on the same version. @@ -32,7 +32,7 @@ await rtmClient.login({ token: 'RTM_TOKEN' }); function App() { const config = useMemo( - () => ({ channel: 'my-channel', rtmConfig: { rtmEngine: rtmClient } }), + () => ({ channel: 'my-channel', rtmEngine: rtmClient }), [] ); @@ -62,7 +62,7 @@ Alternatively, use `useConversationalAI` directly for a batteries-included hook: ```tsx function VoiceAI() { const config = useMemo( - () => ({ channel: 'my-channel', rtmConfig: { rtmEngine: rtmClient } }), + () => ({ channel: 'my-channel', rtmEngine: rtmClient }), [] ); const { transcript, agentState, isConnected, interrupt, manualSOS, manualEOS } = @@ -91,7 +91,7 @@ function VoiceAI() { | `agora-rtc-react` | >= 2.0.0 | | `agora-rtc-sdk-ng` | >= 4.23.4 | | `agora-rtm` | >= 2.0.0 (required for state events and controls) | -| `agora-agent-client-toolkit` | 2.9.0 (same version as this package) | +| `agora-agent-client-toolkit` | 2.9.1 (same version as this package) | ## API Reference @@ -102,7 +102,7 @@ Provider component that manages the `AgoraVoiceAI` lifecycle and exposes the AI ```tsx {/* standalone hooks connect instantly via context */} @@ -140,10 +140,10 @@ const { | `agentState` | `AgentState \| null` | Current agent state (`'idle'`, `'listening'`, `'thinking'`, `'speaking'`, `'silent'`). Null until the first event. | | `isConnected` | `boolean` | `true` after `subscribeMessage` succeeds. | | `error` | `ModuleError \| null` | Most recent error from `AGENT_ERROR`. Null until an error occurs. | -| `interrupt` | `(agentUserId: string) => Promise` | Send an interrupt signal to the agent. Requires `rtmConfig`. | -| `manualSOS` | `(agentUserId: string, requestId?: string) => Promise` | Trigger manual start-of-speech and return the request ID. Requires `rtmConfig`. | -| `manualEOS` | `(agentUserId: string, requestId?: string) => Promise` | Trigger manual end-of-speech and return the request ID. Requires `rtmConfig`. | -| `sendMessage` | `(agentUserId: string, text: string) => Promise` | Send a text message to the agent. Requires `rtmConfig`. | +| `interrupt` | `(agentUserId: string) => Promise` | Send an interrupt signal to the agent. Requires `rtmEngine`. | +| `manualSOS` | `(agentUserId: string, requestId?: string) => Promise` | Trigger manual start-of-speech and return the request ID. Requires `rtmEngine`. | +| `manualEOS` | `(agentUserId: string, requestId?: string) => Promise` | Trigger manual end-of-speech and return the request ID. Requires `rtmEngine`. | +| `sendMessage` | `(agentUserId: string, text: string) => Promise` | Send a text message to the agent. Requires `rtmEngine`. | | `metrics` | `AgentMetric \| null` | Latest metric from `AGENT_METRICS` (module type, name, value, timestamp). | | `messageReceipt` | `MessageReceipt \| null` | Latest delivery receipt from `MESSAGE_RECEIPT_UPDATED`. | diff --git a/packages/react/package.json b/packages/react/package.json index 5ad4c0f..168cebf 100644 --- a/packages/react/package.json +++ b/packages/react/package.json @@ -1,6 +1,6 @@ { "name": "agora-agent-client-toolkit-react", - "version": "2.9.0", + "version": "2.9.1", "description": "React hooks for Agora Agent Client Toolkit", "main": "./dist/index.js", "module": "./dist/index.mjs", diff --git a/packages/react/src/use-conversational-ai.ts b/packages/react/src/use-conversational-ai.ts index 4767de3..f9cdd6d 100644 --- a/packages/react/src/use-conversational-ai.ts +++ b/packages/react/src/use-conversational-ai.ts @@ -39,25 +39,25 @@ export interface UseConversationalAIReturn { error: ModuleError | null; /** * Send an interrupt signal to the agent. - * @remarks Requires `rtmConfig` to be present in the hook config. + * @remarks Requires `rtmEngine` to be present in the hook config. * Throws `[AgoraVoiceAI] This method requires RTM.` when called without RTM. */ interrupt: (agentUserId: string) => Promise; /** * Trigger a manual start-of-speech marker. * @returns The request ID used to correlate the later USER_MANUAL_SOS event. - * @remarks Requires `rtmConfig` to be present in the hook config. + * @remarks Requires `rtmEngine` to be present in the hook config. */ manualSOS: (agentUserId: string, requestId?: string) => Promise; /** * Trigger a manual end-of-speech marker. * @returns The request ID used to correlate the later USER_MANUAL_EOS event. - * @remarks Requires `rtmConfig` to be present in the hook config. + * @remarks Requires `rtmEngine` to be present in the hook config. */ manualEOS: (agentUserId: string, requestId?: string) => Promise; /** * Send a plain-text message to the agent. - * @remarks Requires `rtmConfig` to be present in the hook config. + * @remarks Requires `rtmEngine` to be present in the hook config. * Throws `[AgoraVoiceAI] This method requires RTM.` when called without RTM. */ sendMessage: (agentUserId: string, text: string) => Promise; @@ -278,7 +278,7 @@ function useConversationalAICore(config: UseConversationalAIConfig): UseConversa * function ConversationalApp() { * const config = useMemo(() => ({ * channel: 'my-channel', - * rtmConfig: { rtmEngine: myRtmClient }, + * rtmEngine: myRtmClient, * renderMode: TranscriptHelperMode.WORD, * }), []); * diff --git a/src/core/config.ts b/src/core/config.ts index 02f126f..ad6951a 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -29,28 +29,19 @@ export interface RTMEngine { removeEventListener(eventName: string, listener: (...args: any[]) => void): void; } -export interface RTMConfig { - /** Pre-initialized RTM client. Required if using RTM. */ - rtmEngine: RTMEngine; -} - /** * Configuration for initializing {@link AgoraVoiceAI}. * * Pass your pre-created RTC client in `rtcEngine`. RTM is optional; provide - * `rtmConfig.rtmEngine` only when you need RTM-dependent features such as - * `sendText`, `sendImage`, `interrupt`, and RTM state events. + * `rtmEngine` only when you need RTM-dependent features such as `sendText`, + * `sendImage`, `interrupt`, and RTM state events. */ export interface AgoraVoiceAIConfig { /** Pre-initialized Agora RTC client. Always required. */ rtcEngine: RTCEngine; - /** - * Optional RTM configuration. When absent, the toolkit operates on - * RTC stream-messages only. RTM-dependent features (sendText, sendImage, - * interrupt, agent state events) are unavailable and will throw if called. - */ - rtmConfig?: RTMConfig; + /** Pre-initialized Agora RTM client. Optional for RTC-only integrations. */ + rtmEngine?: RTMEngine; /** * Transcript rendering mode. @@ -63,6 +54,12 @@ export interface AgoraVoiceAIConfig { /** Enable SDK debug logging to console and DEBUG_LOG events. */ enableLog?: boolean; + /** + * Fall back from WORD to TEXT when agent messages omit word timing data. + * Defaults to true. + */ + enableRenderModeFallback?: boolean; + /** * When true, loads `@agora-js/report` dynamically and routes metrics events * through it. Defaults to false (console.debug fallback, zero bundle cost). diff --git a/src/core/conversational-ai.ts b/src/core/conversational-ai.ts index f539c3a..0d89c18 100644 --- a/src/core/conversational-ai.ts +++ b/src/core/conversational-ai.ts @@ -1,10 +1,4 @@ -import type { - AgoraVoiceAIConfig, - RTCEngine, - RTMConfig, - RTMEngine, - RTCStreamMessagePublisher, -} from './config'; +import type { AgoraVoiceAIConfig, RTCEngine, RTMEngine, RTCStreamMessagePublisher } from './config'; import { type AgentState, @@ -55,13 +49,13 @@ import { CovSubRenderController } from '../rendering/sub-render'; import { ChunkedMessageAssembler } from '../messaging/chunked'; const TAG = 'AgoraVoiceAI'; -const VERSION = '2.9.0'; +const VERSION = '2.9.1'; const formatLog = factoryFormatLog({ tag: TAG }); const USER_MANUAL_SOS_CUSTOM_TYPE = 'user.manual_sos'; const USER_MANUAL_EOS_CUSTOM_TYPE = 'user.manual_eos'; -export type { AgoraVoiceAIConfig, RTMConfig }; +export type { AgoraVoiceAIConfig } from './config'; type UnknownRecord = Record; @@ -203,6 +197,7 @@ export class AgoraVoiceAI extends EventHelper { protected rtcEngine: RTCEngine | null = null; protected rtmEngine: RTMEngine | null = null; protected renderMode: TranscriptHelperMode = TranscriptHelperMode.UNKNOWN; + protected enableRenderModeFallback: boolean = true; protected channel: string | null = null; protected covSubRenderController: CovSubRenderController; protected enableLog: boolean = false; @@ -320,12 +315,14 @@ export class AgoraVoiceAI extends EventHelper { renderMode: this.renderMode, channel: this.channel, enableLog: this.enableLog, + rtmEngine: this.rtmEngine, + enableRenderModeFallback: this.enableRenderModeFallback, }; } /** * Requires RTM to be configured. Throws a descriptive error when called - * without rtmConfig. Used internally by sendText, sendImage, and interrupt. + * without rtmEngine. Used internally by sendText, sendImage, and interrupt. */ private requireRTM(method = 'requireRTM'): RTMEngine { if (!this.rtmEngine) { @@ -387,7 +384,7 @@ export class AgoraVoiceAI extends EventHelper { assertFunction('rtcEngine', 'rtcEngine.on(eventName, listener)', rtcEngine?.on); assertFunction('rtcEngine', 'rtcEngine.off(eventName, listener)', rtcEngine?.off); - const rtmEngine = cfg.rtmConfig?.rtmEngine as unknown as + const rtmEngine = cfg.rtmEngine as unknown as | { publish?: unknown; addEventListener?: unknown; @@ -437,8 +434,9 @@ export class AgoraVoiceAI extends EventHelper { } AgoraVoiceAI._instance.rtcEngine = cfg.rtcEngine; - AgoraVoiceAI._instance.rtmEngine = cfg.rtmConfig?.rtmEngine ?? null; + AgoraVoiceAI._instance.rtmEngine = cfg.rtmEngine ?? null; AgoraVoiceAI._instance.renderMode = cfg.renderMode ?? TranscriptHelperMode.UNKNOWN; + AgoraVoiceAI._instance.enableRenderModeFallback = cfg.enableRenderModeFallback ?? true; AgoraVoiceAI._instance.enableLog = cfg.enableLog ?? false; AgoraVoiceAI._instance.setLogLevel(cfg.enableLog ? EventLogLevel.DEBUG : EventLogLevel.NONE); AgoraVoiceAI._instance.metricsReporter = reporter; @@ -467,7 +465,9 @@ export class AgoraVoiceAI extends EventHelper { } this.channel = channel; - this.covSubRenderController.setMode(this.renderMode); + this.covSubRenderController.setMode(this.renderMode, { + enableRenderModeFallback: this.enableRenderModeFallback, + }); this.covSubRenderController.run(); this._startEventTimeoutWarnings(); } @@ -512,6 +512,7 @@ export class AgoraVoiceAI extends EventHelper { instance.unbindRtmEvents(); instance.rtmEngine = null; instance.renderMode = TranscriptHelperMode.UNKNOWN; + instance.enableRenderModeFallback = true; instance.channel = null; instance.removeAllEventListeners(); AgoraVoiceAI._instance = null; @@ -769,7 +770,7 @@ export class AgoraVoiceAI extends EventHelper { * @param agentUserId - The user ID of the agent to send the marker to * @param requestId - Optional non-empty request ID. Generated when omitted. * @returns The request ID used in the RTM payload - * @remarks Requires `rtmConfig` to be present in `init()`. + * @remarks Requires `rtmEngine` to be present in `init()`. */ public async manualSOS(agentUserId: string, requestId?: string): Promise { return this.publishManualTurn( @@ -790,7 +791,7 @@ export class AgoraVoiceAI extends EventHelper { * @param agentUserId - The user ID of the agent to send the marker to * @param requestId - Optional non-empty request ID. Generated when omitted. * @returns The request ID used in the RTM payload - * @remarks Requires `rtmConfig` to be present in `init()`. + * @remarks Requires `rtmEngine` to be present in `init()`. */ public async manualEOS(agentUserId: string, requestId?: string): Promise { return this.publishManualTurn( diff --git a/src/core/events.ts b/src/core/events.ts index f18662b..f6cd4fe 100644 --- a/src/core/events.ts +++ b/src/core/events.ts @@ -53,22 +53,22 @@ export interface AgoraVoiceAIEventHandlers { * Fired when the agent state changes via RTM presence event. * @deprecated This aggregate event remains supported. Use the independent * activity events when multiple flags are needed. - * @remarks Only available when `rtmConfig` is provided to `init()`. + * @remarks Only available when `rtmEngine` is provided to `init()`. */ [AgoraVoiceAIEvents.AGENT_STATE_CHANGED]: (agentUserId: string, event: StateChangeEvent) => void; /** * Fired when the agent listening flag changes via RTM presence event. - * @remarks Only available when `rtmConfig` is provided to `init()`. + * @remarks Only available when `rtmEngine` is provided to `init()`. */ [AgoraVoiceAIEvents.AGENT_LISTENING_CHANGED]: (agentUserId: string, isListening: boolean) => void; /** * Fired when the agent thinking flag changes via RTM presence event. - * @remarks Only available when `rtmConfig` is provided to `init()`. + * @remarks Only available when `rtmEngine` is provided to `init()`. */ [AgoraVoiceAIEvents.AGENT_THINKING_CHANGED]: (agentUserId: string, isThinking: boolean) => void; /** * Fired when the agent speaking flag changes via RTM presence event. - * @remarks Only available when `rtmConfig` is provided to `init()`. + * @remarks Only available when `rtmEngine` is provided to `init()`. */ [AgoraVoiceAIEvents.AGENT_SPEAKING_CHANGED]: (agentUserId: string, isSpeaking: boolean) => void; [AgoraVoiceAIEvents.AGENT_INTERRUPTED]: ( @@ -81,7 +81,7 @@ export interface AgoraVoiceAIEventHandlers { [AgoraVoiceAIEvents.AGENT_METRICS]: (agentUserId: string, metrics: AgentMetric) => void; /** * Fired when the agent reports completed-turn latency metrics. - * @remarks Only available when `rtmConfig` is provided to `init()`. + * @remarks Only available when `rtmEngine` is provided to `init()`. */ [AgoraVoiceAIEvents.AGENT_TURN_FINISHED]: (agentUserId: string, turn: Turn) => void; [AgoraVoiceAIEvents.AGENT_ERROR]: (agentUserId: string, error: ModuleError) => void; @@ -91,7 +91,7 @@ export interface AgoraVoiceAIEventHandlers { [AgoraVoiceAIEvents.DEBUG_LOG]: (message: string) => void; /** * Fired when a message receipt is updated via RTM. - * @remarks Only available when `rtmConfig` is provided to `init()`. + * @remarks Only available when `rtmEngine` is provided to `init()`. */ [AgoraVoiceAIEvents.MESSAGE_RECEIPT_UPDATED]: ( agentUserId: string, @@ -99,7 +99,7 @@ export interface AgoraVoiceAIEventHandlers { ) => void; /** * Fired when a message error is received via RTM. - * @remarks Only available when `rtmConfig` is provided to `init()`. + * @remarks Only available when `rtmEngine` is provided to `init()`. */ [AgoraVoiceAIEvents.MESSAGE_ERROR]: ( agentUserId: string, @@ -112,7 +112,7 @@ export interface AgoraVoiceAIEventHandlers { ) => void; /** * Fired when a SAL status update is received via RTM. - * @remarks Only available when `rtmConfig` is provided to `init()`. + * @remarks Only available when `rtmEngine` is provided to `init()`. */ [AgoraVoiceAIEvents.MESSAGE_SAL_STATUS]: ( agentUserId: string, @@ -120,17 +120,17 @@ export interface AgoraVoiceAIEventHandlers { ) => void; /** * Fired when the server returns the result for a user-triggered manual SOS request. - * @remarks Only available when `rtmConfig` is provided to `init()`. + * @remarks Only available when `rtmEngine` is provided to `init()`. */ [AgoraVoiceAIEvents.USER_MANUAL_SOS]: (agentUserId: string, event: UserManualSosEvent) => void; /** * Fired when the server returns the result for a user-triggered manual EOS request. - * @remarks Only available when `rtmConfig` is provided to `init()`. + * @remarks Only available when `rtmEngine` is provided to `init()`. */ [AgoraVoiceAIEvents.USER_MANUAL_EOS]: (agentUserId: string, event: UserManualEosEvent) => void; /** * Fired when the server reports an automatic EOS in manual mode. - * @remarks Only available when `rtmConfig` is provided to `init()`. + * @remarks Only available when `rtmEngine` is provided to `init()`. */ [AgoraVoiceAIEvents.AGENT_MANUAL_EOS]: (agentUserId: string, event: AgentManualEosEvent) => void; } diff --git a/src/core/types.ts b/src/core/types.ts index c91b230..0ad53b2 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -183,7 +183,7 @@ export class NotInitializedError extends ConversationalAIError { export class RTMRequiredError extends ConversationalAIError { constructor(method: string) { super( - `[AgoraVoiceAI] ${method}() requires RTM. Pass rtmConfig: { rtmEngine } when calling AgoraVoiceAI.init().` + `[AgoraVoiceAI] ${method}() requires RTM. Pass rtmEngine when calling AgoraVoiceAI.init().` ); this.name = 'RTMRequiredError'; } diff --git a/src/index.ts b/src/index.ts index 263bcfc..f81fa55 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,8 +1,8 @@ // Public export surface for agora-convo-ai-toolkit -export { AgoraVoiceAI } from './core/conversational-ai'; +export { AgoraVoiceAI, AgoraVoiceAI as ConversationalAIAPI } from './core/conversational-ai'; export { type AgoraVoiceAIConfig, - type RTMConfig, + type AgoraVoiceAIConfig as IConversationalAIAPIConfig, type RTCEngine, type RTMEngine, type RTCStreamMessagePublisher, @@ -19,14 +19,29 @@ export { export { // Enums AgentState, + AgentState as EAgentState, TurnStatus, + TurnStatus as ETurnStatus, TranscriptHelperMode, + TranscriptHelperMode as ETranscriptHelperMode, MessageSalStatus, + MessageSalStatus as EMessageSalStatus, ModuleType, + ModuleType as EModuleType, ChatMessageType, + ChatMessageType as EChatMessageType, ChatMessagePriority, + ChatMessagePriority as EChatMessagePriority, MessageType, + MessageType as EMessageType, LocalTranscriptStatus, + LocalTranscriptStatus as ELocalTranscriptStatus, + RTMEventType, + RTMEventType as ERTMEvents, + RTCEventType, + RTCEventType as ERTCEvents, + RTCCustomEventType, + RTCCustomEventType as ERTCCustomEvents, // Error classes ConversationalAIError, NotInitializedError, @@ -35,34 +50,68 @@ export { // Type aliases & interfaces type AgoraVoiceAIState, type AgentMetric, + type AgentMetric as TAgentMetric, type ModuleError, + type ModuleError as TModuleError, type StateChangeEvent, + type StateChangeEvent as TStateChangeEvent, type SegmentedLatency, type Turn, + type Turn as TAgentTurnFinished, type MessageReceipt, + type MessageReceipt as TMessageReceipt, type ChatMessageText, + type ChatMessageText as IChatMessageText, type ChatMessageImage, + type ChatMessageImage as IChatMessageImage, type ChatMessageBase, + type ChatMessageBase as IChatMessageBase, type UserTranscription, + type UserTranscription as IUserTranscription, type AgentTranscription, + type AgentTranscription as IAgentTranscription, type TranscriptionBase, + type TranscriptionBase as ITranscriptionBase, type MessageInterrupt, + type MessageInterrupt as IMessageInterrupt, type MessageError, + type MessageError as IMessageError, type MessageMetrics, + type MessageMetrics as IMessageMetrics, type TurnFinishedMessage, + type TurnFinishedMessage as ITurnFinishedMessage, type MessageSalStatusData, + type MessageSalStatusData as IMessageSalStatus, type UserManualEventPayload, type UserManualSosEvent, type UserManualEosEvent, type AgentManualEosPayload, type AgentManualEosEvent, type TranscriptHelperItem, + type TranscriptHelperItem as ITranscriptHelperItem, type TranscriptHelperObjectWord, + type TranscriptHelperObjectWord as TTranscriptHelperObjectWord, type DataChunkMessageWord, + type DataChunkMessageWord as TDataChunkMessageWord, type LocalTranscriptionBase, + type LocalTranscriptionBase as ILocalTranscriptionBase, type LocalImageTranscription, + type LocalImageTranscription as ILocalImageTranscription, type UserTracks, + type UserTracks as IUserTracks, + type PresenceState, + type PresenceState as IPresenceState, + type QueueItem, + type QueueItem as TQueueItem, + type HelperRTCEvents, + type HelperRTCEvents as IHelperRTCEvents, } from './core/types'; // --- Consumer-facing types from core/events --- -export { AgoraVoiceAIEvents, EventLogLevel, type AgoraVoiceAIEventHandlers } from './core/events'; +export { + AgoraVoiceAIEvents, + AgoraVoiceAIEvents as EConversationalAIAPIEvents, + EventLogLevel, + type AgoraVoiceAIEventHandlers, + type AgoraVoiceAIEventHandlers as IConversationalAIAPIEventHandlers, +} from './core/events'; diff --git a/src/rendering/sub-render-queue.ts b/src/rendering/sub-render-queue.ts index bde6781..9f59718 100644 --- a/src/rendering/sub-render-queue.ts +++ b/src/rendering/sub-render-queue.ts @@ -99,6 +99,11 @@ export class SubRenderQueue { } } + public clearPendingItems() { + this.queue = []; + this.lastPoppedQueueItem = null; + } + private _handleTurnObj(queueItem: QueueItem, curPTS: number) { let correspondingChatHistoryItem = this.chatHistory.find( (item) => item.turn_id === queueItem.turn_id && item.stream_id === queueItem.stream_id diff --git a/src/rendering/sub-render.ts b/src/rendering/sub-render.ts index 20103c8..8519750 100644 --- a/src/rendering/sub-render.ts +++ b/src/rendering/sub-render.ts @@ -126,6 +126,7 @@ export class CovSubRenderController { private _enableLog: boolean; private _mode: TranscriptHelperMode = TranscriptHelperMode.UNKNOWN; + private _enableRenderModeFallback: boolean = true; private _agentMessageState: { state: AgentState; turn_id: string | number; @@ -625,7 +626,15 @@ export class CovSubRenderController { * Sets the transcript rendering mode. Can only be called once — subsequent * calls after mode is locked (not UNKNOWN or AUTO) are ignored with a warning. */ - public setMode(mode: TranscriptHelperMode) { + public setMode( + mode: TranscriptHelperMode, + options?: { + enableRenderModeFallback?: boolean; + } + ) { + if (options?.enableRenderModeFallback !== undefined) { + this._enableRenderModeFallback = options.enableRenderModeFallback; + } // Allow setting from UNKNOWN (initial) or AUTO (transitioning to detected mode). // Any other existing mode is considered already locked. if (this._mode !== TranscriptHelperMode.UNKNOWN && this._mode !== TranscriptHelperMode.AUTO) { @@ -650,6 +659,20 @@ export class CovSubRenderController { this._mode = mode; } + private _fallbackToTextMode(reason: string) { + if (this._mode === TranscriptHelperMode.TEXT) { + return; + } + this.callMessagePrint( + ELoggerType.warn, + '[Fallback]', + `Switching render mode to TEXT: ${reason}` + ); + this._pts.teardownInterval(); + this._queue.clearPendingItems(); + this._mode = TranscriptHelperMode.TEXT; + } + public handleMessage( message: T, options: { @@ -692,6 +715,12 @@ export class CovSubRenderController { } if (isAgentMessage && this._mode === TranscriptHelperMode.WORD) { + const hasWordData = Array.isArray(message.words) && message.words.length > 0; + if (this._enableRenderModeFallback && !hasWordData) { + this._fallbackToTextMode('word data missing'); + this.handleTextMessage(options.publisher, message as unknown as AgentTranscription); + return; + } this._pts.setupIntervalForWords({ isForce: false }); this.handleWordAgentMessage(options.publisher, message as unknown as AgentTranscription); return;