diff --git a/CLAUDE.md b/CLAUDE.md index 00af3f2..1adc015 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -50,9 +50,6 @@ useEffect(() => { }, [isReady, joinSuccess]); ``` -**Why not `ConversationalAIProvider` from `agora-agent-client-toolkit-react`?** -The React toolkit has a StrictMode race condition: when any component mounting `ConversationalAIProvider` is double-mounted by StrictMode, two concurrent `AgoraVoiceAI.init()` calls share the same singleton object. The first call's cancellation destroys the instance after the second call has reconfigured it (same object reference), so `subscribeMessage` on the second call fails silently. - **Why `isReady && joinSuccess` works:** - `isReady` is `true` only after the StrictMode fake-unmount cycle completes (via `setTimeout(fn, 0)` pattern). - Once `isReady` is `true`, React does NOT double-invoke the effect for subsequent dependency changes (`joinSuccess` becoming `true`). This means `AgoraVoiceAI.init()` is called exactly once. @@ -72,8 +69,7 @@ Include `INTERRUPTED` turns in `messageList` (filter only `IN_PROGRESS`). If the | Layer | Package | Role | |---|---|---| | Client UI | `agora-rtc-react` | RTC hooks (`useJoin`, `useLocalMicrophoneTrack`, `usePublish`, etc.) | -| Transcripts | `agora-agent-client-toolkit-react` | `ConversationalAIProvider` + `useTranscript()`, `useAgentState()` | -| Toolkit core | `agora-agent-client-toolkit` | `TurnStatus` enum, `TranscriptHelperItem` types | +| Toolkit core | `agora-agent-client-toolkit` | `AgoraVoiceAI`, `TurnStatus` enum, `TranscriptHelperItem` types | | UI components | `agora-agent-uikit` | `AudioVisualizer`, `ConvoTextStream`, `MicButtonWithVisualizer` | | Server SDK | `agora-agent-server-sdk` | Builder pattern — `AgoraClient` → `Agent` → `session.start()` | | Messaging | `agora-rtm` | RTM transport for transcripts | @@ -97,13 +93,12 @@ Tailwind must scan uikit classes: `./node_modules/agora-agent-uikit/dist/**/*.{j ## After Changing Implementation Files -After editing anything in `components/` or `app/api/`, run the `docs-sync` agent to check that GUIDE.md, TEXT_STREAMING_GUIDE.md, and README.md are still accurate. +After editing anything in `components/` or `app/api/`, manually verify that `DOCS/GUIDE.md`, `DOCS/TEXT_STREAMING_GUIDE.md`, `README.md`, and `agents.md` still match the implementation, then run `pnpm lint` and `pnpm build`. ## What NOT To Do - Do not call `client.leave()` manually (breaks `useJoin` cleanup) - Do not call `localMicrophoneTrack.close()` manually (breaks hook ownership) -- Do not use `ConversationalAIProvider` or `useConversationalAI` from `agora-agent-client-toolkit-react` — they have a StrictMode singleton race condition. Use raw `AgoraVoiceAI` gated on `isReady && joinSuccess` instead. - Do not remove the `isReady` guard - Do not set `reactStrictMode: false` - Do not use the deprecated `turnDetection.type: 'agora_vad'` flat API — use `turnDetection.config.start_of_speech` / `end_of_speech` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e52c600..a5a0082 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ Thank you for your interest in contributing! This document provides guidelines f ### Prerequisites -- Node.js 24 or higher +- Node.js 22 or higher (see `engines` in `package.json`) - pnpm 8.x or higher - An Agora account with Conversational AI enabled diff --git a/DOCS/GUIDE.md b/DOCS/GUIDE.md index 4178f21..de38218 100644 --- a/DOCS/GUIDE.md +++ b/DOCS/GUIDE.md @@ -85,15 +85,15 @@ Your project directory should now have a structure like this: The `agora-agent-uikit` package ships pre-built components with Tailwind class names. To ensure those classes are included in your production build, add the uikit's `dist/` folder to Tailwind's `content` array in `tailwind.config.ts`: ```typescript -import type { Config } from "tailwindcss"; +import type { Config } from 'tailwindcss'; const config: Config = { content: [ - "./pages/**/*.{js,ts,jsx,tsx,mdx}", - "./components/**/*.{js,ts,jsx,tsx,mdx}", - "./app/**/*.{js,ts,jsx,tsx,mdx}", + './pages/**/*.{js,ts,jsx,tsx,mdx}', + './components/**/*.{js,ts,jsx,tsx,mdx}', + './app/**/*.{js,ts,jsx,tsx,mdx}', // Scan the uikit's compiled output so its Tailwind classes are included - "./node_modules/agora-agent-uikit/dist/**/*.{js,mjs}", + './node_modules/agora-agent-uikit/dist/**/*.{js,mjs}', ], // ... rest of your config }; @@ -484,11 +484,16 @@ export interface AgoraTokenData { agentId?: string; } +export interface AgoraRenewalTokens { + rtcToken: string; + rtmToken: string; +} + // Props for our conversation component export interface ConversationComponentProps { agoraData: AgoraTokenData; - rtmClient: RTMClient; // RTM client for transcript delivery - onTokenWillExpire: (uid: string) => Promise; + rtmClient: RTMClient; // RTM client for transcript delivery + onTokenWillExpire: (uid: string) => Promise; onEndConversation: () => void; } @@ -759,7 +764,7 @@ export default function ConversationComponent({ token: agoraData.token, uid: parseInt(agoraData.uid, 10) || 0, }, - isReady + isReady, ); const { localMicrophoneTrack } = useLocalMicrophoneTrack(isReady); @@ -772,13 +777,15 @@ export default function ConversationComponent({ } }, [joinSuccess, client]); - // Token renewal — also renews the RTM token since they share the same token + // Token renewal — RTC and RTM now renew with separate tokens const handleTokenWillExpire = useCallback(async () => { if (!onTokenWillExpire || !joinedUID) return; try { - const newToken = await onTokenWillExpire(joinedUID.toString()); - await client?.renewToken(newToken); - await rtmClient.renewToken(newToken); + const { rtcToken, rtmToken } = await onTokenWillExpire( + joinedUID.toString(), + ); + await client?.renewToken(rtcToken); + await rtmClient.renewToken(rtmToken); console.log('Successfully renewed Agora RTC and RTM tokens'); } catch (error) { console.error('Failed to renew Agora token:', error); @@ -865,7 +872,9 @@ If you don't know a specific fact about Agora, say so plainly and suggest checki // First thing the agent says when a user joins the channel. // Set NEXT_AGENT_GREETING in .env.local to override. -const GREETING = process.env.NEXT_AGENT_GREETING ?? `Hi there! I'm Ada, your virtual assistant from Agora. How can I help?`; +const GREETING = + process.env.NEXT_AGENT_GREETING ?? + `Hi there! I'm Ada, your virtual assistant from Agora. How can I help?`; // agentUid identifies the AI in the RTC channel — must match NEXT_PUBLIC_AGENT_UID on the client const agentUid = process.env.NEXT_PUBLIC_AGENT_UID ?? String(DEFAULT_AGENT_UID); @@ -881,7 +890,8 @@ export async function POST(request: NextRequest) { const body: ClientStartRequest = await request.json(); const { requester_id, channel_name } = body; - const appId = process.env.NEXT_PUBLIC_AGORA_APP_ID || requireEnv('NEXT_AGORA_APP_ID'); + const appId = + process.env.NEXT_PUBLIC_AGORA_APP_ID || requireEnv('NEXT_AGORA_APP_ID'); const appCertificate = requireEnv('NEXT_AGORA_APP_CERTIFICATE'); if (!channel_name || !requester_id) { @@ -983,7 +993,7 @@ export async function POST(request: NextRequest) { This quickstart uses **`agora-agent-server-sdk` ^1.3.1** with **DeepgramSTT**, **OpenAI** (`gpt-4o-mini`), and **MiniMaxTTS** through Agora-managed defaults, so the base quickstart only needs Agora credentials. The SDK still supports other STT/LLM/TTS providers if you want to enable the optional BYOK examples. -> **Note:** Only Agora App ID and App Certificate are required for this default agent configuration. `NEXT_PUBLIC_AGENT_UID` is an optional override that defaults to `12345`. See the environment variables reference at the end of this guide. Optional `NEXT_LLM_URL` / `NEXT_LLM_API_KEY` apply only if you enable the optional BYOK block or use the optional `app/api/chat/completions` proxy. +> **Note:** Only Agora App ID and App Certificate are required for this default agent configuration. `NEXT_PUBLIC_AGENT_UID` is an optional override that defaults to `123456`. See the environment variables reference at the end of this guide. Optional `NEXT_LLM_URL` / `NEXT_LLM_API_KEY` apply only if you enable the optional BYOK block or use the optional `app/api/chat/completions` proxy. ### Stop Conversation Route @@ -1451,21 +1461,21 @@ export default function ConversationComponent({ ### Key uikit Components -| Component | Import | Description | -|-----------|--------|-------------| -| `AgentVisualizer` | `agora-agent-uikit` | Agent-state-driven visualizer for not-joined, listening, analyzing, talking, and disconnected states | -| `ConvoTextStream` | `agora-agent-uikit` | Floating chat panel showing live and completed transcript turns | -| `MicButtonWithVisualizer` | `agora-agent-uikit/rtc` | Mic button with built-in Web Audio visualization | -| `ConnectionStatusPanel` | local component | Compact status dot plus expandable RTM/agent issue panel | +| Component | Import | Description | +| ------------------------- | ----------------------- | ---------------------------------------------------------------------------------------------------- | +| `AgentVisualizer` | `agora-agent-uikit` | Agent-state-driven visualizer for not-joined, listening, analyzing, talking, and disconnected states | +| `ConvoTextStream` | `agora-agent-uikit` | Floating chat panel showing live and completed transcript turns | +| `MicButtonWithVisualizer` | `agora-agent-uikit/rtc` | Mic button with built-in Web Audio visualization | +| `ConnectionStatusPanel` | local component | Compact status dot plus expandable RTM/agent issue panel | ### Key toolkit Classes -| Export | Description | -|--------|-------------| -| `AgoraVoiceAI` | Subscribes to RTM transcript events from the AI agent | -| `AgoraVoiceAIEvents.TRANSCRIPT_UPDATED` | Fires whenever a transcript turn is created or updated | -| `TurnStatus` | `IN_PROGRESS`, `END`, `INTERRUPTED` — filters which turns to show as history vs. live | -| `TranscriptHelperMode.TEXT` | Text-only rendering mode | +| Export | Description | +| --------------------------------------- | ------------------------------------------------------------------------------------- | +| `AgoraVoiceAI` | Subscribes to RTM transcript events from the AI agent | +| `AgoraVoiceAIEvents.TRANSCRIPT_UPDATED` | Fires whenever a transcript turn is created or updated | +| `TurnStatus` | `IN_PROGRESS`, `END`, `INTERRUPTED` — filters which turns to show as history vs. live | +| `TranscriptHelperMode.TEXT` | Text-only rendering mode | ## Testing @@ -1525,7 +1535,8 @@ const ADA_PROMPT = `You are a friendly and helpful assistant named Alex. Your pe Update the `greeting` to control the initial message the agent speaks when joining the channel (or set `NEXT_AGENT_GREETING` in `.env.local`): ```typescript -const GREETING = process.env.NEXT_AGENT_GREETING ?? `Hello! How can I assist you today?`; +const GREETING = + process.env.NEXT_AGENT_GREETING ?? `Hello! How can I assist you today?`; ``` ### Customizing the Voice @@ -1566,7 +1577,7 @@ Here's a complete list of environment variables for your `.env.local` file: # Agora Configuration NEXT_PUBLIC_AGORA_APP_ID= NEXT_AGORA_APP_CERTIFICATE= -NEXT_PUBLIC_AGENT_UID=12345 +NEXT_PUBLIC_AGENT_UID=123456 # Optional BYOK examples # NEXT_LLM_URL=https://api.openai.com/v1/chat/completions diff --git a/README.md b/README.md index 714d2d3..0ab815d 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Required environment variables: Optional convenience override: -- `NEXT_PUBLIC_AGENT_UID` defaults to `12345` +- `NEXT_PUBLIC_AGENT_UID` defaults to `123456` The default agent configuration in [`app/api/invite-agent/route.ts`](app/api/invite-agent/route.ts) uses Agora-managed defaults for STT, LLM, and TTS, so no additional vendor API keys are required for the base quickstart. diff --git a/agents.md b/agents.md index ddb8c77..3647320 100644 --- a/agents.md +++ b/agents.md @@ -38,6 +38,10 @@ hooks/ use-mobile.tsx — useIsMobile() — returns true when viewport < 768 px lib/ + agora.ts — DEFAULT_AGENT_UID constant (123456) + conversation.ts — pure helpers: normalizeTranscript, getMessageList, + getCurrentInProgressMessage, mapAgentVisualizerState, + normalizeTimestampMs, toMessageListItem utils.ts — cn() (clsx + tailwind-merge) DOCS/ @@ -51,20 +55,19 @@ DOCS/ ### Client-side -| Package | Role | -|---|---| -| `agora-rtc-react` | RTC hooks: `useJoin`, `useLocalMicrophoneTrack`, `usePublish`, `useRemoteUsers`, `useClientEvent` | -| `agora-rtm` | RTM transport — carries transcript messages from agent to browser | -| `agora-agent-client-toolkit-react` | Installed for compatibility with the UIKit package graph, but not used by the quickstart runtime | -| `agora-agent-client-toolkit` | Core types: `TurnStatus`, `TranscriptHelperItem`, `TranscriptHelperMode` | -| `agora-agent-uikit` | Pre-built components: `AgentVisualizer`, `ConvoTextStream`, `MicButtonWithVisualizer` (from `/rtc`) | +| Package | Role | +| ---------------------------- | ---------------------------------------------------------------------------------------------------- | +| `agora-rtc-react` | RTC hooks: `useJoin`, `useLocalMicrophoneTrack`, `usePublish`, `useRemoteUsers`, `useClientEvent` | +| `agora-rtm` | RTM transport — carries transcript messages from agent to browser | +| `agora-agent-client-toolkit` | `AgoraVoiceAI` runtime plus core types: `TurnStatus`, `TranscriptHelperItem`, `TranscriptHelperMode` | +| `agora-agent-uikit` | Pre-built components: `AgentVisualizer`, `ConvoTextStream`, `MicButtonWithVisualizer` (from `/rtc`) | ### Server-side -| Package | Role | -|---|---| +| Package | Role | +| ------------------------ | ------------------------------------------------------------------------------------------------------------ | | `agora-agent-server-sdk` | `AgoraClient`, `Agent`, `DeepgramSTT`, `OpenAI`, `MiniMaxTTS` — builder pattern for starting/stopping agents | -| `agora-token` | `RtcTokenBuilder.buildTokenWithRtm` — generates RTC+RTM combined token | +| `agora-token` | `RtcTokenBuilder.buildTokenWithRtm` — generates RTC+RTM combined token | --- @@ -72,15 +75,15 @@ DOCS/ All vars live in `.env.local` (gitignored). `env.local.example` is the source of truth. -| Variable | Side | Purpose | -|---|---|---| -| `NEXT_PUBLIC_AGORA_APP_ID` | client+server | Agora project App ID | -| `NEXT_AGORA_APP_CERTIFICATE` | server only | Signs tokens — never expose client-side | -| `NEXT_PUBLIC_AGENT_UID` | client+server, optional | Agent UID override. Defaults to `12345` from `lib/agora.ts`, so the quickstart runs without setting it. | -| `NEXT_LLM_URL` | server only, optional | Any OpenAI-compatible chat completions endpoint for the optional BYOK LLM block | -| `NEXT_LLM_API_KEY` | server only, optional | LLM API key for the optional BYOK LLM block | -| `NEXT_DEEPGRAM_API_KEY` | server only, optional | Deepgram STT API key for the optional BYOK STT block | -| `NEXT_ELEVENLABS_API_KEY` | server only, optional | ElevenLabs TTS API key for the optional BYOK TTS block | +| Variable | Side | Purpose | +| ---------------------------- | ----------------------- | -------------------------------------------------------------------------------------------------------- | +| `NEXT_PUBLIC_AGORA_APP_ID` | client+server | Agora project App ID | +| `NEXT_AGORA_APP_CERTIFICATE` | server only | Signs tokens — never expose client-side | +| `NEXT_PUBLIC_AGENT_UID` | client+server, optional | Agent UID override. Defaults to `123456` from `lib/agora.ts`, so the quickstart runs without setting it. | +| `NEXT_LLM_URL` | server only, optional | Any OpenAI-compatible chat completions endpoint for the optional BYOK LLM block | +| `NEXT_LLM_API_KEY` | server only, optional | LLM API key for the optional BYOK LLM block | +| `NEXT_DEEPGRAM_API_KEY` | server only, optional | Deepgram STT API key for the optional BYOK STT block | +| `NEXT_ELEVENLABS_API_KEY` | server only, optional | ElevenLabs TTS API key for the optional BYOK TTS block | --- @@ -105,6 +108,7 @@ Starts an Agora ConvoAI agent using `agora-agent-server-sdk`. **Input** (`ClientStartRequest`): `{ requester_id, channel_name, input_modalities?, output_modalities? }` **What it does:** + 1. Validates required env vars (throws on startup if missing). 2. Builds the agent: `new AgoraClient(...)` → `new Agent({ instructions, greeting, turnDetection, advancedFeatures })` → `.withStt(DeepgramSTT)` → `.withLlm(OpenAI)` → `.withTts(MiniMaxTTS)`. 3. `agent.createSession(client, { channel, agentUid, remoteUids, idleTimeout, expiresIn })`. @@ -112,6 +116,7 @@ Starts an Agora ConvoAI agent using `agora-agent-server-sdk`. 5. Returns `AgentResponse: { agent_id, create_ts, state }`. **Key configuration (edit in this file):** + - `ADA_PROMPT` / `GREETING` — agent persona and opening line - `turnDetection.config` — VAD sensitivity (`speech_threshold`, `silence_duration_ms`, `interrupt_duration_ms`, `prefix_padding_ms`) - `advancedFeatures: { enable_rtm: true }` — required for RTM transcript delivery @@ -119,6 +124,7 @@ Starts an Agora ConvoAI agent using `agora-agent-server-sdk`. - `voiceId: 'English_captivating_female1'` in `MiniMaxTTS(...)` — default TTS voice **Turn detection** uses the current (non-deprecated) API: + ```ts turnDetection: { config: { @@ -128,6 +134,7 @@ turnDetection: { }, } ``` + Do not use the deprecated `type: 'agora_vad'` flat structure. --- @@ -161,17 +168,20 @@ Core real-time component. Must be inside `AgoraRTCProvider`. **StrictMode guard:** `isReady` state, set via `setTimeout(..., 0)` in a `useEffect`. Both `useJoin(config, isReady)` and `useLocalMicrophoneTrack(isReady)` are gated on it to prevent double-initialization. **Hook ownership:** + - `useJoin` owns `client.leave()` — do not call manually - `useLocalMicrophoneTrack` owns track lifecycle — do not call `.close()` manually - `usePublish` owns publish state — mute via `track.setEnabled()` only -**Transcript + agent state:** Managed with raw `AgoraVoiceAI` from `agora-agent-client-toolkit`. `AgoraVoiceAI.init()` runs in a `useEffect` gated on `isReady && joinSuccess` — this fires exactly once, past the StrictMode double-mount cycle. Transcript and agent state are tracked via `useState` + `ai.on(TRANSCRIPT_UPDATED, ...)` / `ai.on(AGENT_STATE_CHANGED, ...)`. `uid="0"` remapping (local user sentinel → `client.uid`) happens in a `useMemo` over the raw transcript. The React toolkit (`agora-agent-client-toolkit-react`) is NOT used for lifecycle — it has a StrictMode race condition with the `AgoraVoiceAI` singleton. +**Transcript + agent state:** Managed with raw `AgoraVoiceAI` from `agora-agent-client-toolkit`. `AgoraVoiceAI.init()` runs in a `useEffect` gated on `isReady && joinSuccess` — this fires exactly once, past the StrictMode double-mount cycle. Transcript and agent state are tracked via `useState` + `ai.on(TRANSCRIPT_UPDATED, ...)` / `ai.on(AGENT_STATE_CHANGED, ...)`. `uid="0"` remapping (local user sentinel → `client.uid`) happens in a `useMemo` over the raw transcript. **Transcript state:** + - `messageList` — completed + interrupted turns (`status !== IN_PROGRESS`) mapped locally into `IMessageListItem` - `currentInProgressMessage` — the single in-progress turn, if any **UI kit components used:** + - `AgentVisualizer` — agent-state-driven visualizer for lifecycle, listening, thinking, and speaking states - `ConvoTextStream` — floating chat panel with `messageList` + `currentInProgressMessage` + `agentUID` - `MicButtonWithVisualizer` (from `agora-agent-uikit/rtc`) — mic button with Web Audio visualization @@ -223,7 +233,7 @@ User clicks "Try it now!" 1. **`useJoin` owns `client.leave()`** — never call it manually. Causes `AgoraRTCError WS_ABORT: LEAVE`. -2. **StrictMode double-init** — `isReady` + `setTimeout` guard prevents dual mic track creation and double `AgoraVoiceAI` init. Do not remove. The `AgoraVoiceAI.init()` effect is also gated on `isReady && joinSuccess` — by the time `joinSuccess` becomes `true`, the StrictMode cycle is done and the effect runs exactly once. `ConversationalAIProvider` from `agora-agent-client-toolkit-react` is intentionally avoided: it has a singleton race condition in StrictMode where the cancelled first init destroys the singleton after the real second init has reconfigured it. +2. **StrictMode double-init** — `isReady` + `setTimeout` guard prevents dual mic track creation and double `AgoraVoiceAI` init. Do not remove. The `AgoraVoiceAI.init()` effect is also gated on `isReady && joinSuccess` — by the time `joinSuccess` becomes `true`, the StrictMode cycle is done and the effect runs exactly once. 3. **`NEXT_PUBLIC_AGENT_UID` must match exactly** — the component compares `user.uid.toString() === agentUID`. A mismatch means `isAgentConnected` never fires. diff --git a/app/api/generate-agora-token/route.ts b/app/api/generate-agora-token/route.ts index 2372410..95ec760 100644 --- a/app/api/generate-agora-token/route.ts +++ b/app/api/generate-agora-token/route.ts @@ -10,13 +10,11 @@ function generateChannelName(): string { } export async function GET(request: NextRequest) { - console.log('Generating Agora token...'); - + // console.log('Generating Agora token...'); const APP_ID = process.env.NEXT_PUBLIC_AGORA_APP_ID; const APP_CERTIFICATE = process.env.NEXT_AGORA_APP_CERTIFICATE; if (!APP_ID || !APP_CERTIFICATE) { - console.error('Agora credentials are not set'); return NextResponse.json( { error: 'Agora credentials are not set' }, { status: 500 } @@ -33,7 +31,7 @@ export async function GET(request: NextRequest) { Math.floor(Date.now() / 1000) + EXPIRATION_TIME_IN_SECONDS; try { - console.log('Building RTC+RTM token with UID:', uid, 'Channel:', channelName); + // console.log('Building RTC+RTM token: uid =', uid, 'channel =', channelName); const token = RtcTokenBuilder.buildTokenWithRtm( APP_ID, APP_CERTIFICATE, @@ -43,8 +41,8 @@ export async function GET(request: NextRequest) { expirationTime, expirationTime ); + // console.log('Token generated successfully (RTC + RTM)'); - console.log('Token generated successfully (RTC + RTM)'); return NextResponse.json({ token, uid: uid.toString(), diff --git a/app/api/invite-agent/route.ts b/app/api/invite-agent/route.ts index 018430a..2bd76f6 100644 --- a/app/api/invite-agent/route.ts +++ b/app/api/invite-agent/route.ts @@ -62,8 +62,7 @@ export async function POST(request: NextRequest) { // Validate required env vars on first request so misconfiguration surfaces // with a clear error message rather than a silent failure. - const appId = - process.env.NEXT_PUBLIC_AGORA_APP_ID || requireEnv('NEXT_AGORA_APP_ID'); + const appId = requireEnv('NEXT_PUBLIC_AGORA_APP_ID'); const appCertificate = requireEnv('NEXT_AGORA_APP_CERTIFICATE'); if (!channel_name || !requester_id) { diff --git a/app/api/stop-conversation/route.ts b/app/api/stop-conversation/route.ts index 153b57e..34f46a4 100644 --- a/app/api/stop-conversation/route.ts +++ b/app/api/stop-conversation/route.ts @@ -34,8 +34,7 @@ export async function POST(request: Request) { ); } - const appId = - process.env.NEXT_PUBLIC_AGORA_APP_ID || process.env.NEXT_AGORA_APP_ID; + const appId = process.env.NEXT_PUBLIC_AGORA_APP_ID; const appCertificate = process.env.NEXT_AGORA_APP_CERTIFICATE; if (!appId || !appCertificate) { throw new Error( diff --git a/components/ConnectionStatusPanel.tsx b/components/ConnectionStatusPanel.tsx index 25d8a36..25b00a4 100644 --- a/components/ConnectionStatusPanel.tsx +++ b/components/ConnectionStatusPanel.tsx @@ -9,6 +9,8 @@ type ConnectionStatusPanelProps = { onToggle: () => void; }; +// Produces an accessible label from the raw RTC state string, with a special case for +// "Connected (issues detected)" when RTM/agent errors exist while RTC transport is healthy. function getConnectionLabel( connectionState: string, connectionSeverity: 'normal' | 'warning' | 'error' @@ -32,11 +34,12 @@ export function ConnectionStatusPanel({ }: ConnectionStatusPanelProps) { return (
+ {/* Minimal status affordance: color and ping convey RTC health before the user opens details. */} - {getConnectionLabel(connectionState, connectionSeverity)} - + {/* Expandable detail panel: current RTC state plus the captured agent/RTM issues. */}
1e12 ? timestamp : timestamp * 1000; -} - +// Payload shape for signaling-level errors forwarded by the agent over RTM. +// The `module` field identifies which backend subsystem (LLM / ASR / TTS) raised the error. type RtmMessageErrorPayload = { object: 'message.error'; module?: string; @@ -64,16 +55,26 @@ type RtmMessageErrorPayload = { send_ts?: number; }; +// Payload shape for SAL (Session Abstraction Layer) registration status messages. +// VP_REGISTER_FAIL and VP_REGISTER_DUPLICATE indicate RTM channel subscription problems. type RtmSalStatusPayload = { object: 'message.sal_status'; status?: string; timestamp?: number; }; -function isRtmMessageErrorPayload(value: unknown): value is RtmMessageErrorPayload { - return !!value && typeof value === 'object' && (value as { object?: unknown }).object === 'message.error'; +// Type guard for RTM signaling-level error payloads (object: 'message.error'). +function isRtmMessageErrorPayload( + value: unknown, +): value is RtmMessageErrorPayload { + return ( + !!value && + typeof value === 'object' && + (value as { object?: unknown }).object === 'message.error' + ); } +// Type guard for RTM SAL status payloads (object: 'message.sal_status'). function isRtmSalStatusPayload(value: unknown): value is RtmSalStatusPayload { return ( !!value && @@ -82,56 +83,6 @@ function isRtmSalStatusPayload(value: unknown): value is RtmSalStatusPayload { ); } -function mapAgentVisualizerState( - agentState: AgentState | null, - isAgentConnected: boolean, - connectionState: string, -): AgentVisualizerState { - if ( - connectionState === 'DISCONNECTED' || - connectionState === 'DISCONNECTING' - ) { - return 'disconnected'; - } - - if ( - connectionState === 'CONNECTING' || - connectionState === 'RECONNECTING' - ) { - return 'joining'; - } - - if (!isAgentConnected) { - return 'not-joined'; - } - - switch (agentState) { - case 'listening': - return 'listening'; - case 'thinking': - return 'analyzing'; - case 'speaking': - return 'talking'; - case 'idle': - case 'silent': - default: - return 'ambient'; - } -} - -function toMessageListItem( - item: TranscriptHelperItem>, -): IMessageListItem { - return { - turn_id: item.turn_id, - uid: Number(item.uid) || 0, - text: typeof item.text === 'string' ? item.text : '', - status: item.status as unknown as IMessageListItem['status'], - createdAt: - typeof item._time === 'number' ? normalizeTimestampMs(item._time) : undefined, - }; -} - export default function ConversationComponent({ agoraData, rtmClient, @@ -147,7 +98,8 @@ export default function ConversationComponent({ // Tracks granular RTC connection state for the status dot. // Agora states: DISCONNECTED | CONNECTING | CONNECTED | DISCONNECTING | RECONNECTING const [connectionState, setConnectionState] = useState('CONNECTING'); - const agentUID = process.env.NEXT_PUBLIC_AGENT_UID ?? String(DEFAULT_AGENT_UID); + const agentUID = + process.env.NEXT_PUBLIC_AGENT_UID ?? String(DEFAULT_AGENT_UID); const [joinedUID, setJoinedUID] = useState(0); // Transcript + agent state — managed with AgoraVoiceAI (see effect below). @@ -243,9 +195,7 @@ export default function ConversationComponent({ // effect only runs on the real mount (not the discarded fake one). // - Once `isReady` is true, React does NOT double-invoke this effect for // subsequent state changes (`joinSuccess` becoming true). That means - // AgoraVoiceAI.init() is called exactly once, avoiding the singleton - // race condition that occurs when ConversationalAIProvider (React toolkit) - // is double-mounted by StrictMode. + // AgoraVoiceAI.init() is called exactly once. useEffect(() => { if (!isReady || !joinSuccess) return; @@ -263,6 +213,7 @@ export default function ConversationComponent({ if (cancelled) { try { if (AgoraVoiceAI.getInstance() === ai) { + // Tear down only the instance created by this effect run. ai.unsubscribe(); ai.destroy(); } @@ -273,6 +224,7 @@ export default function ConversationComponent({ ai.on(AgoraVoiceAIEvents.TRANSCRIPT_UPDATED, (t) => { setRawTranscript([...t]); }); + // Agent state drives the visualizer, independent of RTC audio presence. ai.on(AgoraVoiceAIEvents.AGENT_STATE_CHANGED, (_, event) => setAgentState(event.state), ); @@ -286,6 +238,7 @@ export default function ConversationComponent({ timestamp: normalizeTimestampMs(error.timestamp), }); }); + // SAL status: capture raw RTM messages so message.sal_status surfaces even if higher-level events don't. ai.on( AgoraVoiceAIEvents.MESSAGE_SAL_STATUS, (agentUserId, salStatus) => { @@ -304,6 +257,7 @@ export default function ConversationComponent({ } }, ); + // Agent error: capture raw RTM messages so message.error surfaces even if higher-level events don't. ai.on(AgoraVoiceAIEvents.AGENT_ERROR, (agentUserId, error) => { addConnectionIssue({ id: `${Date.now()}-${agentUserId}-agent-error-${error.code}`, @@ -314,6 +268,7 @@ export default function ConversationComponent({ timestamp: normalizeTimestampMs(error.timestamp), }); }); + // subscribeMessage binds the toolkit to both RTC stream messages and RTM payloads. ai.subscribeMessage(agoraData.channel); } catch (error) { if (!cancelled) { @@ -335,7 +290,7 @@ export default function ConversationComponent({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [isReady, joinSuccess]); - // Signaling - capture raw RTM messages so message.error surfaces even if higher-level events don't. + // Raw RTM parsing is kept as a fallback for signaling-level errors and SAL status. useEffect(() => { const handleRtmMessage = (event: { message: string | Uint8Array; @@ -390,35 +345,21 @@ export default function ConversationComponent({ }; }, [rtmClient, addConnectionIssue]); - // useTranscript() returns uid="0" for local user speech — remap to actual RTC UID + // The toolkit uses uid="0" for local user speech — remap to actual RTC UID // so ConvoTextStream renders user messages on the correct side. // Also normalize punctuation spacing for display when upstream text arrives compacted. const transcript = useMemo(() => { - const localUID = String(client.uid); - return rawTranscript.map((m) => { - const remappedUID = m.uid === '0' ? localUID : m.uid; - const normalizedText = - typeof m.text === 'string' - ? normalizeTranscriptSpacing(m.text) - : m.text; - return { ...m, uid: remappedUID, text: normalizedText }; - }); + return normalizeTranscript(rawTranscript, String(client.uid)); }, [rawTranscript, client.uid]); // Completed (END + INTERRUPTED) messages shown as history. // INTERRUPTED must be included — if the agent's first turn is cut off, // messageList stays empty and ConvoTextStream never auto-opens. - const messageList = useMemo( - () => - transcript - .filter((m) => m.status !== TurnStatus.IN_PROGRESS) - .map(toMessageListItem), - [transcript], - ); + const messageList = useMemo(() => getMessageList(transcript), [transcript]); const currentInProgressMessage = useMemo(() => { - const m = transcript.find((x) => x.status === TurnStatus.IN_PROGRESS); - return m ? toMessageListItem(m) : null; + // ConvoTextStream renders the live partial turn separately from the history list. + return getCurrentInProgressMessage(transcript); }, [transcript]); // Publish local mic once the track exists; usePublish waits for RTC connection. @@ -445,6 +386,7 @@ export default function ConversationComponent({ }); const connectionSeverity = useMemo<'normal' | 'warning' | 'error'>(() => { + // RTC transport problems take precedence; otherwise derive severity from captured issues. if ( connectionState === 'DISCONNECTED' || connectionState === 'DISCONNECTING' @@ -469,11 +411,7 @@ export default function ConversationComponent({ const visualizerState = useMemo( () => - mapAgentVisualizerState( - agentState, - isAgentConnected, - connectionState, - ), + mapAgentVisualizerState(agentState, isAgentConnected, connectionState), [agentState, isAgentConnected, connectionState], ); @@ -500,6 +438,7 @@ export default function ConversationComponent({ const handleTokenWillExpire = useCallback(async () => { if (!onTokenWillExpire || !joinedUID) return; try { + // RTC and RTM renew independently, but the quickstart fetches both in one request. const { rtcToken, rtmToken } = await onTokenWillExpire( joinedUID.toString(), ); @@ -514,6 +453,7 @@ export default function ConversationComponent({ return (
+ {/* Top-left status affordance: opens transport and agent error details without covering the main controls. */}
+ {/* Top-right destructive action: stops the cloud agent and ends the current session. */}
- {/* Remote users keep RTC audio subscribed. The visualizer itself is driven - by RTM agent state so the user sees lifecycle and speaking status in one place. */} + {/* Center stage: the visualizer shows agent lifecycle/speaking state, while hidden RemoteUser mounts keep agent audio subscribed. */}
- {/* Local controls — pill-framed dock at bottom center */} + {/* Bottom dock: microphone mute/unmute plus input-device switching for the local user. */}
+ {/* Transcript panel: completed turns plus the current partial turn stream into the floating chat UI. */}
Conversation AI Engine Error: {transportCode} diff --git a/components/ErrorBoundary.tsx b/components/ErrorBoundary.tsx index 849f6e5..0ac3736 100644 --- a/components/ErrorBoundary.tsx +++ b/components/ErrorBoundary.tsx @@ -37,6 +37,7 @@ export class ErrorBoundary extends React.Component< } return ( + // Last-resort recovery UI for client-only conversation failures.

@@ -53,6 +54,7 @@ export class ErrorBoundary extends React.Component< ); } + // Happy path: render the wrapped conversation subtree unchanged. return this.props.children; } } diff --git a/components/LandingPage.tsx b/components/LandingPage.tsx index 2526e8f..14c4630 100644 --- a/components/LandingPage.tsx +++ b/components/LandingPage.tsx @@ -1,6 +1,6 @@ 'use client'; -import { useState, useMemo, useRef, Suspense, useEffect } from 'react'; +import { useState, useRef, Suspense, useEffect, useCallback } from 'react'; import dynamic from 'next/dynamic'; import { Loader2 } from 'lucide-react'; import type { RTMClient } from 'agora-rtm'; @@ -24,20 +24,34 @@ const ConversationComponent = dynamic(() => import('./ConversationComponent'), { // the RTC join succeeds, so this wrapper only needs to provide the RTC client. const AgoraProvider = dynamic( async () => { - const { AgoraRTCProvider, default: AgoraRTC } = await import('agora-rtc-react'); + const { AgoraRTCProvider, default: AgoraRTC } = + await import('agora-rtc-react'); return { - default: function AgoraProviders({ children }: { children: React.ReactNode }) { + default: function AgoraProviders({ + children, + }: { + children: React.ReactNode; + }) { // useRef persists across StrictMode's simulated unmount/remount, so only // one RTC client is ever created per session (useMemo creates two in StrictMode). - const clientRef = useRef | null>(null); + const clientRef = useRef | null>(null); if (!clientRef.current) { - clientRef.current = AgoraRTC.createClient({ mode: 'rtc', codec: 'vp8' }); + clientRef.current = AgoraRTC.createClient({ + mode: 'rtc', + codec: 'vp8', + }); } - return {children}; + return ( + + {children} + + ); }, }; }, - { ssr: false } + { ssr: false }, ); export default function LandingPage() { @@ -62,13 +76,15 @@ export default function LandingPage() { try { // 1. Fetch RTC token + channel - console.log('Fetching Agora token...'); + // console.log('Fetching Agora token...'); const agoraResponse = await fetch('/api/generate-agora-token'); const responseData = await agoraResponse.json(); - console.log('Agora API response:', responseData); + // console.log('Agora token response: uid =', responseData.uid, 'channel =', responseData.channel); if (!agoraResponse.ok) { - throw new Error(`Failed to generate Agora token: ${JSON.stringify(responseData)}`); + throw new Error( + `Failed to generate Agora token: ${JSON.stringify(responseData)}`, + ); } // 2. Run agent invite and RTM setup in parallel — both only need the token response. @@ -85,7 +101,10 @@ export default function LandingPage() { } as ClientStartRequest), }) .then(async (res) => { - if (!res.ok) { setAgentJoinError(true); return null; } + if (!res.ok) { + setAgentJoinError(true); + return null; + } return res.json() as Promise; }) .catch((err) => { @@ -103,7 +122,7 @@ export default function LandingPage() { ); await rtm.login({ token: responseData.token }); await rtm.subscribe(responseData.channel); - console.log('RTM ready, channel:', responseData.channel); + // console.log('RTM ready, channel:', responseData.channel); return rtm; })(), ]); @@ -120,41 +139,49 @@ export default function LandingPage() { } }; - const handleTokenWillExpire = async (uid: string): Promise => { - try { - const channel = agoraData?.channel; - if (!channel) { - throw new Error('Missing channel for token renewal'); - } + const handleTokenWillExpire = useCallback( + async (uid: string): Promise => { + try { + const channel = agoraData?.channel; + if (!channel) { + throw new Error('Missing channel for token renewal'); + } - const [rtcResponse, rtmResponse] = await Promise.all([ - fetch(`/api/generate-agora-token?channel=${channel}&uid=${uid}`), - fetch(`/api/generate-agora-token?channel=${channel}&uid=0`), - ]); - const [rtcData, rtmData] = await Promise.all([ - rtcResponse.json(), - rtmResponse.json(), - ]); + // RTC and RTM tokens are renewed independently: + // - RTC uses the browser client's assigned UID (passed in from ConversationComponent). + // - RTM uses uid=0 because buildTokenWithRtm embeds RTM capability by AppID+channel, + // not by a specific RTM userId — so uid=0 is valid for any RTM client instance. + // Both are fetched in parallel to stay within the token-expiry grace-period window. + const [rtcResponse, rtmResponse] = await Promise.all([ + fetch(`/api/generate-agora-token?channel=${channel}&uid=${uid}`), + fetch(`/api/generate-agora-token?channel=${channel}&uid=0`), + ]); + const [rtcData, rtmData] = await Promise.all([ + rtcResponse.json(), + rtmResponse.json(), + ]); - if (!rtcResponse.ok || !rtmResponse.ok) { - throw new Error('Failed to generate renewal tokens'); - } + if (!rtcResponse.ok || !rtmResponse.ok) { + throw new Error('Failed to generate renewal tokens'); + } - return { - rtcToken: rtcData.token, - rtmToken: rtmData.token, - }; - } catch (error) { - console.error('Error renewing token:', error); - throw error; - } - }; + return { + rtcToken: rtcData.token, + rtmToken: rtmData.token, + }; + } catch (error) { + console.error('Error renewing token:', error); + throw error; + } + }, + [agoraData], + ); const handleEndConversation = async () => { // Stop the AI agent if (agoraData?.agentId) { try { - console.log('Stopping agent:', agoraData.agentId); + // console.log('Stopping agent:', agoraData.agentId); const response = await fetch('/api/stop-conversation', { method: 'POST', headers: { 'Content-Type': 'application/json' }, @@ -162,9 +189,8 @@ export default function LandingPage() { }); if (!response.ok) { console.error('Failed to stop agent:', await response.text()); - } else { - console.log('Agent stopped successfully'); } + // else console.log('Agent stopped successfully'); } catch (error) { console.error('Error stopping agent:', error); } @@ -188,6 +214,7 @@ export default function LandingPage() { }} /> + {/* Hero shell: either shows the pre-call CTA or swaps in the live conversation experience. */}

@@ -196,17 +223,23 @@ export default function LandingPage() { {!showConversation && (

- Experience the power of
Agora's Conversational AI Engine. + Experience the power of
+ Agora's Conversational AI Engine.

)} {!showConversation ? ( <> + {/* Entry CTA: starts token fetch, agent invite, and RTM setup for a new session. */} - {error && ( -

{error}

- )} + {error &&

{error}

} ) : agoraData && rtmClient ? ( <> + {/* Non-fatal invite warning: the browser session can still render even if agent start failed. */} {agentJoinError && (
Failed to connect with AI agent. The conversation may not work as expected.
)} + {/* Browser-only conversation mount: RTC provider, error boundary, and lazy-loaded call UI. */} }> @@ -243,14 +276,20 @@ export default function LandingPage() { ) : ( -

Failed to load conversation data.

+ /* Fallback if session bootstrap partially succeeded but required state is missing. */ +

+ Failed to load conversation data. +

)}

+ {/* Persistent attribution footer for the pre-call and in-call views. */}