From 2d5fb76455d7ab372a7d8ad95d963e48a583608c Mon Sep 17 00:00:00 2001 From: digitallysavvy Date: Tue, 7 Apr 2026 10:42:03 -0400 Subject: [PATCH 1/6] toolkit v1.2.0 updates --- app/api/invite-agent/route.ts | 37 ++-- app/api/stop-conversation/route.ts | 30 ++- components/ConnectionStatusPanel.tsx | 99 ++++++++++ components/ConversationComponent.tsx | 281 ++++++++++++++++++++------- components/ConversationErrorCard.tsx | 121 ++++++++++++ components/LandingPage.tsx | 7 +- package.json | 4 +- pnpm-lock.yaml | 35 ++-- 8 files changed, 504 insertions(+), 110 deletions(-) create mode 100644 components/ConnectionStatusPanel.tsx create mode 100644 components/ConversationErrorCard.tsx diff --git a/app/api/invite-agent/route.ts b/app/api/invite-agent/route.ts index 8c762af..2e07db8 100644 --- a/app/api/invite-agent/route.ts +++ b/app/api/invite-agent/route.ts @@ -40,7 +40,9 @@ If you don't know a specific fact about Agora, say so plainly and suggest checki // First thing the agent says when a user joins the channel. // Set NEXT_AGENT_GREETING in .env.local to override. -const GREETING = process.env.NEXT_AGENT_GREETING ?? `Hi there! I'm Ada, your virtual assistant from Agora. How can I help?`; +const GREETING = + process.env.NEXT_AGENT_GREETING ?? + `Hi there! I'm Ada, your virtual assistant from Agora. How can I help?`; // agentUid identifies the AI in the RTC channel — must match NEXT_PUBLIC_AGENT_UID on the client const agentUid = process.env.NEXT_PUBLIC_AGENT_UID || '123456'; @@ -64,7 +66,8 @@ export async function POST(request: NextRequest) { // Validate required env vars on first request so misconfiguration surfaces // with a clear error message rather than a silent failure. - const appId = process.env.NEXT_PUBLIC_AGORA_APP_ID || requireEnv('NEXT_AGORA_APP_ID'); + const appId = + process.env.NEXT_PUBLIC_AGORA_APP_ID || requireEnv('NEXT_AGORA_APP_ID'); const appCertificate = requireEnv('NEXT_AGORA_APP_CERTIFICATE'); if (!channel_name || !requester_id) { @@ -114,6 +117,10 @@ export async function POST(request: NextRequest) { // RTM is required for transcript events in the browser client. // enable_tools is required for MCP tool invocation. advancedFeatures: { enable_rtm: true, enable_tools: true }, + // Required for browser RTM events: + // - data_channel: 'rtm' enables RTM delivery path for state/metrics/errors + // - enable_error_message emits AGENT_ERROR payloads + parameters: { data_channel: 'rtm', enable_error_message: true }, }) .withStt( new DeepgramSTT({ @@ -128,7 +135,21 @@ export async function POST(request: NextRequest) { // }), ) .withLlm( + // new OpenAI({ + // model: 'gpt-4o-mini', + // greetingMessage: GREETING, + // failureMessage: 'Please wait a moment.', + // maxHistory: 15, + // params: { + // max_tokens: 1024, + // temperature: 0.7, + // top_p: 0.95, + // }, + // }), + // BYOK: uncomment the following block and set NEXT_OPENAI_API_KEY and NEXT_OPENAI_ new OpenAI({ + apiKey: requireEnv('NEXT_LLM_API_KEY'), + url: requireEnv('NEXT_LLM_URL'), model: 'gpt-4o-mini', greetingMessage: GREETING, failureMessage: 'Please wait a moment.', @@ -137,18 +158,6 @@ export async function POST(request: NextRequest) { temperature: 0.7, topP: 0.95, }), - // BYOK: uncomment the following block and set NEXT_OPENAI_API_KEY and NEXT_OPENAI_ - // new OpenAI({ - // apiKey: requireEnv('NEXT_OPENAI_API_KEY'), - // url: requireEnv('NEXT_OPENAI_URL'), - // model: 'gpt-4o-mini', - // greetingMessage: GREETING, - // failureMessage: 'Please wait a moment.', - // maxHistory: 15, - // maxTokens: 1024, - // temperature: 0.7, - // topP: 0.95, - // }), ) .withTts( new MiniMaxTTS({ diff --git a/app/api/stop-conversation/route.ts b/app/api/stop-conversation/route.ts index 67c4be9..153b57e 100644 --- a/app/api/stop-conversation/route.ts +++ b/app/api/stop-conversation/route.ts @@ -2,6 +2,26 @@ import { NextResponse } from 'next/server'; import { AgoraClient, Area } from 'agora-agent-server-sdk'; import { StopConversationRequest } from '@/types/conversation'; +function isAgentAlreadyStoppingOrStopped(error: unknown): boolean { + if (!error || typeof error !== 'object') return false; + + const maybeErr = error as { + statusCode?: number; + body?: { detail?: string; reason?: string }; + message?: string; + }; + + const statusCode = maybeErr.statusCode; + const reason = maybeErr.body?.reason?.toLowerCase(); + const detail = maybeErr.body?.detail?.toLowerCase() ?? maybeErr.message?.toLowerCase() ?? ''; + + if (statusCode === 404) return true; + if (reason === 'invalidrequest' && detail.includes('already in the process of shutting down')) { + return true; + } + return false; +} + export async function POST(request: Request) { try { const body: StopConversationRequest = await request.json(); @@ -29,7 +49,15 @@ export async function POST(request: Request) { appId, appCertificate, }); - await client.stopAgent(agent_id); + try { + await client.stopAgent(agent_id); + } catch (error) { + if (isAgentAlreadyStoppingOrStopped(error)) { + // Treat stop as idempotent: agent is already exiting (or gone). + return NextResponse.json({ success: true, state: 'already-stopping' }); + } + throw error; + } return NextResponse.json({ success: true }); } catch (error) { diff --git a/components/ConnectionStatusPanel.tsx b/components/ConnectionStatusPanel.tsx new file mode 100644 index 0000000..2710cea --- /dev/null +++ b/components/ConnectionStatusPanel.tsx @@ -0,0 +1,99 @@ +import React from 'react'; +import { ConversationErrorCard, type ConnectionIssue } from './ConversationErrorCard'; + +type ConnectionStatusPanelProps = { + connectionState: string; + connectionSeverity: 'normal' | 'warning' | 'error'; + connectionIssues: ConnectionIssue[]; + isOpen: boolean; + onToggle: () => void; +}; + +function getConnectionLabel( + connectionState: string, + connectionSeverity: 'normal' | 'warning' | 'error' +): string { + if (connectionSeverity !== 'normal' && connectionState === 'CONNECTED') { + return 'Connected (issues detected)'; + } + if (connectionState === 'CONNECTED') return 'Connected'; + if (connectionState === 'CONNECTING') return 'Connecting...'; + if (connectionState === 'RECONNECTING') return 'Reconnecting...'; + if (connectionState === 'DISCONNECTING') return 'Disconnecting...'; + return 'Disconnected'; +} + +export function ConnectionStatusPanel({ + connectionState, + connectionSeverity, + connectionIssues, + isOpen, + onToggle, +}: ConnectionStatusPanelProps) { + return ( +
+ + + {getConnectionLabel(connectionState, connectionSeverity)} + +
+
+
+ Connection Details +
+
+ RTC {connectionState.toLowerCase()} +
+
+ {connectionIssues.length === 0 ? ( +
No RTM or agent errors reported.
+ ) : ( +
+ {connectionIssues.map((issue) => ( + + ))} +
+ )} +
+
+ ); +} diff --git a/components/ConversationComponent.tsx b/components/ConversationComponent.tsx index 60d258b..bc098f9 100644 --- a/components/ConversationComponent.tsx +++ b/components/ConversationComponent.tsx @@ -2,13 +2,11 @@ import { useState, useEffect, useCallback, useMemo } from 'react'; import { setParameter } from 'agora-rtc-sdk-ng/esm'; -import type { IAgoraRTCClient } from 'agora-rtc-sdk-ng'; import { useRTCClient, useLocalMicrophoneTrack, useRemoteUsers, useClientEvent, - useIsConnected, useJoin, usePublish, RemoteUser, @@ -18,6 +16,7 @@ import { AgoraVoiceAI, AgoraVoiceAIEvents, AgentState, + MessageSalStatus, TurnStatus, TranscriptHelperMode, type TranscriptHelperItem, @@ -32,6 +31,11 @@ import { import { MicButtonWithVisualizer } from 'agora-agent-uikit/rtc'; import { Button } from '@/components/ui/button'; import { MicrophoneSelector } from './MicrophoneSelector'; +import { + getConversationIssueSeverity, + type ConnectionIssue, +} from './ConversationErrorCard'; +import { ConnectionStatusPanel } from './ConnectionStatusPanel'; import type { ConversationComponentProps } from '@/types/conversation'; function normalizeTranscriptSpacing(text: string): string { @@ -49,6 +53,39 @@ const AGENT_AUDIO_VISUALIZER_GRADIENT = [ 'hsl(var(--viz-stop-3))', ]; +const MAX_CONNECTION_ISSUES = 6; + +function normalizeTimestampMs(timestamp: number): number { + // Some payloads are seconds, others are milliseconds. + return timestamp > 1e12 ? timestamp : timestamp * 1000; +} + +type RtmMessageErrorPayload = { + object: 'message.error'; + module?: string; + code?: number; + message?: string; + send_ts?: number; +}; + +type RtmSalStatusPayload = { + object: 'message.sal_status'; + status?: string; + timestamp?: number; +}; + +function isRtmMessageErrorPayload(value: unknown): value is RtmMessageErrorPayload { + return !!value && typeof value === 'object' && (value as { object?: unknown }).object === 'message.error'; +} + +function isRtmSalStatusPayload(value: unknown): value is RtmSalStatusPayload { + return ( + !!value && + typeof value === 'object' && + (value as { object?: unknown }).object === 'message.sal_status' + ); +} + export default function ConversationComponent({ agoraData, rtmClient, @@ -56,10 +93,10 @@ export default function ConversationComponent({ onEndConversation, }: ConversationComponentProps) { const client = useRTCClient(); - const isConnected = useIsConnected(); const remoteUsers = useRemoteUsers(); const [isEnabled, setIsEnabled] = useState(true); const [isAgentConnected, setIsAgentConnected] = useState(false); + const [isConnectionDetailsOpen, setIsConnectionDetailsOpen] = useState(false); // Tracks granular RTC connection state for the status dot. // Agora states: DISCONNECTED | CONNECTING | CONNECTED | DISCONNECTING | RECONNECTING @@ -67,11 +104,34 @@ export default function ConversationComponent({ const agentUID = process.env.NEXT_PUBLIC_AGENT_UID; const [joinedUID, setJoinedUID] = useState(0); - // Transcript + agent state — managed with raw AgoraVoiceAI (see effect below). + // Transcript + agent state — managed with AgoraVoiceAI (see effect below). const [rawTranscript, setRawTranscript] = useState< TranscriptHelperItem>[] >([]); const [agentState, setAgentState] = useState(null); + const [connectionIssues, setConnectionIssues] = useState( + [], + ); + const addConnectionIssue = useCallback((issue: ConnectionIssue) => { + setConnectionIssues((prev) => { + const isDuplicate = prev.some( + (x) => + x.agentUserId === issue.agentUserId && + x.code === issue.code && + x.message === issue.message && + Math.abs(x.timestamp - issue.timestamp) < 1500, + ); + if (isDuplicate) return prev; + return [issue, ...prev].slice(0, MAX_CONNECTION_ISSUES); + }); + }, []); + + // Auto-open details panel as soon as a new issue is recorded. + useEffect(() => { + if (connectionIssues.length > 0) { + setIsConnectionDetailsOpen(true); + } + }, [connectionIssues.length]); // StrictMode guard: delay `useJoin`'s ready flag until after the fake-unmount // cycle completes. React StrictMode fires cleanup synchronously before any @@ -97,7 +157,7 @@ export default function ConversationComponent({ token: agoraData.token, uid: parseInt(agoraData.uid, 10) || 0, }, - isReady + isReady, ); // Create mic track only after the StrictMode fake-unmount cycle completes (isReady). @@ -109,21 +169,12 @@ export default function ConversationComponent({ // graph inside MicButtonWithVisualizer. Mute uses track.setEnabled() only. const { localMicrophoneTrack } = useLocalMicrophoneTrack(isReady); - useEffect(() => { - if (!agentUID) { - console.warn('NEXT_AGENT_UID environment variable is not set'); - } else { - console.log('Agent UID is set to:', agentUID); - } - }, [agentUID]); - // ENABLE_AUDIO_PTS is a module-level SDK parameter (not on the client instance). // It must be set before publishing audio for transcript timing to be accurate. useEffect(() => { if (!client) return; try { setParameter('ENABLE_AUDIO_PTS', true); - console.log('Enabled ENABLE_AUDIO_PTS for timing synchronization'); } catch (error) { console.warn('Could not set ENABLE_AUDIO_PTS:', error); } @@ -133,8 +184,9 @@ export default function ConversationComponent({ useEffect(() => { if (joinSuccess && client) { const uid = client.uid; - setJoinedUID(uid as UID); - console.log('Join successful, using UID:', uid); + if (uid !== null && uid !== undefined) { + setJoinedUID(uid); + } } }, [joinSuccess, client]); @@ -156,7 +208,7 @@ export default function ConversationComponent({ (async () => { try { const ai = await AgoraVoiceAI.init({ - rtcEngine: client as unknown as IAgoraRTCClient, + rtcEngine: client, rtmConfig: { rtmEngine: rtmClient }, renderMode: TranscriptHelperMode.TEXT, enableLog: true, @@ -176,10 +228,47 @@ export default function ConversationComponent({ setRawTranscript([...t]); }); ai.on(AgoraVoiceAIEvents.AGENT_STATE_CHANGED, (_, event) => - setAgentState(event.state) + setAgentState(event.state), + ); + ai.on(AgoraVoiceAIEvents.MESSAGE_ERROR, (agentUserId, error) => { + addConnectionIssue({ + id: `${Date.now()}-${agentUserId}-message-error-${error.code}`, + source: 'rtm', + agentUserId, + code: error.code, + message: error.message, + timestamp: normalizeTimestampMs(error.timestamp), + }); + }); + ai.on( + AgoraVoiceAIEvents.MESSAGE_SAL_STATUS, + (agentUserId, salStatus) => { + if ( + salStatus.status === MessageSalStatus.VP_REGISTER_FAIL || + salStatus.status === MessageSalStatus.VP_REGISTER_DUPLICATE + ) { + addConnectionIssue({ + id: `${Date.now()}-${agentUserId}-sal-${salStatus.status}`, + source: 'rtm', + agentUserId, + code: salStatus.status, + message: `SAL status: ${salStatus.status}`, + timestamp: normalizeTimestampMs(salStatus.timestamp), + }); + } + }, ); + ai.on(AgoraVoiceAIEvents.AGENT_ERROR, (agentUserId, error) => { + addConnectionIssue({ + id: `${Date.now()}-${agentUserId}-agent-error-${error.code}`, + source: 'agent', + agentUserId, + code: error.code, + message: `${error.type}: ${error.message}`, + timestamp: normalizeTimestampMs(error.timestamp), + }); + }); ai.subscribeMessage(agoraData.channel); - console.log('AgoraVoiceAI initialized and subscribed to channel'); } catch (error) { if (!cancelled) { console.error('[AgoraVoiceAI] init failed:', error); @@ -200,6 +289,61 @@ export default function ConversationComponent({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [isReady, joinSuccess]); + // Signaling - capture raw RTM messages so message.error surfaces even if higher-level events don't. + useEffect(() => { + const handleRtmMessage = (event: { + message: string | Uint8Array; + publisher: string; + }) => { + const payloadText = + typeof event.message === 'string' + ? event.message + : new TextDecoder().decode(event.message); + + let parsed: unknown; + try { + parsed = JSON.parse(payloadText); + } catch { + return; + } + + if (isRtmMessageErrorPayload(parsed)) { + const p = parsed; + addConnectionIssue({ + id: `${Date.now()}-${event.publisher}-rtm-msg-error-${p.code ?? 'unknown'}`, + source: 'rtm-signaling', + agentUserId: event.publisher, + code: p.code ?? 'unknown', + message: `${p.module ?? 'unknown'}: ${p.message ?? 'Unknown signaling error'}`, + timestamp: normalizeTimestampMs(p.send_ts ?? Date.now()), + }); + return; + } + + if (isRtmSalStatusPayload(parsed)) { + const p = parsed; + if ( + p.status === 'VP_REGISTER_FAIL' || + p.status === 'VP_REGISTER_DUPLICATE' + ) { + addConnectionIssue({ + id: `${Date.now()}-${event.publisher}-rtm-sal-${p.status}`, + source: 'rtm-signaling', + agentUserId: event.publisher, + code: p.status, + message: `SAL status: ${p.status}`, + timestamp: normalizeTimestampMs(p.timestamp ?? Date.now()), + }); + } + } + }; + + rtmClient.addEventListener('message', handleRtmMessage); + return () => { + rtmClient.removeEventListener('message', handleRtmMessage); + }; + }, [rtmClient, addConnectionIssue]); + // useTranscript() returns uid="0" for local user speech — remap to actual RTC UID // so ConvoTextStream renders user messages on the correct side. // Also normalize punctuation spacing for display when upstream text arrives compacted. @@ -208,7 +352,9 @@ export default function ConversationComponent({ return rawTranscript.map((m) => { const remappedUID = m.uid === '0' ? localUID : m.uid; const normalizedText = - typeof m.text === 'string' ? normalizeTranscriptSpacing(m.text) : m.text; + typeof m.text === 'string' + ? normalizeTranscriptSpacing(m.text) + : m.text; return { ...m, uid: remappedUID, text: normalizedText }; }); }, [rawTranscript, client.uid]); @@ -219,43 +365,62 @@ export default function ConversationComponent({ const messageList = useMemo( () => transcriptToMessageList( - transcript.filter((m) => m.status !== TurnStatus.IN_PROGRESS) + transcript.filter((m) => m.status !== TurnStatus.IN_PROGRESS), ), - [transcript] + [transcript], ); const currentInProgressMessage = useMemo(() => { const m = transcript.find((x) => x.status === TurnStatus.IN_PROGRESS); - return m ? transcriptToMessageList([m])[0] ?? null : null; + return m ? (transcriptToMessageList([m])[0] ?? null) : null; }, [transcript]); // Publish local mic once the track exists; usePublish waits for RTC connection. usePublish([localMicrophoneTrack]); useClientEvent(client, 'user-joined', (user) => { - console.log('Remote user joined:', user.uid); if (user.uid.toString() === agentUID) setIsAgentConnected(true); }); useClientEvent(client, 'user-left', (user) => { - console.log('Remote user left:', user.uid); if (user.uid.toString() === agentUID) setIsAgentConnected(false); }); // Sync isAgentConnected with remoteUsers (covers cases where user-joined/left are missed) useEffect(() => { const isAgentInRemoteUsers = remoteUsers.some( - (user) => user.uid.toString() === agentUID + (user) => user.uid.toString() === agentUID, ); setIsAgentConnected(isAgentInRemoteUsers); }, [remoteUsers, agentUID]); - useClientEvent(client, 'connection-state-change', (curState, prevState) => { - console.log(`Connection state changed from ${prevState} to ${curState}`); - if (curState === 'DISCONNECTED') console.log('Attempting to reconnect...'); + useClientEvent(client, 'connection-state-change', (curState) => { setConnectionState(curState); }); + const connectionSeverity = useMemo<'normal' | 'warning' | 'error'>(() => { + if ( + connectionState === 'DISCONNECTED' || + connectionState === 'DISCONNECTING' + ) { + return 'error'; + } + if ( + connectionState === 'CONNECTING' || + connectionState === 'RECONNECTING' + ) { + return 'warning'; + } + if (connectionIssues.length === 0) { + return 'normal'; + } + return connectionIssues.some( + (issue) => getConversationIssueSeverity(issue) === 'error', + ) + ? 'error' + : 'warning'; + }, [connectionState, connectionIssues]); + /** * Mute/unmute via track.setEnabled() only — usePublish owns publish state. * If we also unpublish in the toggle, usePublish and the button fight each other @@ -282,7 +447,6 @@ export default function ConversationComponent({ const newToken = await onTokenWillExpire(joinedUID.toString()); await client?.renewToken(newToken); await rtmClient.renewToken(newToken); - console.log('Successfully renewed Agora RTC and RTM tokens'); } catch (error) { console.error('Failed to renew Agora token:', error); } @@ -290,52 +454,17 @@ export default function ConversationComponent({ useClientEvent(client, 'token-privilege-will-expire', handleTokenWillExpire); - // Debug: log remote UIDs vs NEXT_PUBLIC_AGENT_UID to catch mismatches - useEffect(() => { - if (remoteUsers.length > 0) { - console.log('Remote users detected:', remoteUsers.map((u) => u.uid)); - console.log('Current NEXT_AGENT_UID:', agentUID); - - const potentialAgents = remoteUsers.map((u) => u.uid.toString()); - if (agentUID && !potentialAgents.includes(agentUID)) { - console.warn('Agent UID mismatch! Expected:', agentUID, 'Available users:', potentialAgents); - console.info(`Consider updating NEXT_AGENT_UID to one of: ${potentialAgents.join(', ')}`); - } - } - }, [remoteUsers, agentUID]); - return (
- {/* Top-right: connection status dot + end call */} + {/* Top-right: connection status + end call */}
- {/* Connection status dot — color reflects RTC state, tooltip on hover */} -
- - {/* Ping ring — shown while connecting or connected (signals activity) */} - {connectionState !== 'DISCONNECTED' && connectionState !== 'DISCONNECTING' && ( - - )} - - - {/* Tooltip label — visible on hover */} - - {connectionState === 'CONNECTED' ? 'Connected' : - connectionState === 'CONNECTING' ? 'Connecting...' : - connectionState === 'RECONNECTING' ? 'Reconnecting...' : - connectionState === 'DISCONNECTING' ? 'Disconnecting...' : - 'Disconnected'} - -
+ setIsConnectionDetailsOpen((open) => !open)} + />
- {/* Remote users (agent audio + RTC subscription). - Framed in a surface card so the visualizer has spatial context. - Fixed h-40 matches AudioVisualizer's height so the layout doesn't - shift when the agent joins or leaves. */} + {/* Remote users keep RTC audio subscribed. The visualizer itself is driven + by RTM agent state so the user sees lifecycle and speaking status in one place. */}
+ {remoteUsers.map((user) => ( -
- +
))} - {remoteUsers.length === 0 && ( -
- Waiting for AI agent to join... -
- )} -
- - {/* Agent state — shown below the visualizer once the agent joins */} -
- {isAgentConnected && agentState ? agentState : null}
{/* Local controls — pill-framed dock at bottom center */} @@ -541,6 +577,7 @@ export default function ConversationComponent({ messageList={messageList} currentInProgressMessage={currentInProgressMessage} agentUID={agentUID} + className="conversation-transcript" />
); diff --git a/components/LandingPage.tsx b/components/LandingPage.tsx index 06cce40..d6d03ac 100644 --- a/components/LandingPage.tsx +++ b/components/LandingPage.tsx @@ -8,6 +8,7 @@ import type { AgoraTokenData, ClientStartRequest, AgentResponse, + AgoraRenewalTokens, } from '../types/conversation'; import { Button } from '@/components/ui/button'; import { ErrorBoundary } from './ErrorBoundary'; @@ -119,14 +120,30 @@ export default function LandingPage() { } }; - const handleTokenWillExpire = async (uid: string) => { + const handleTokenWillExpire = async (uid: string): Promise => { try { - const response = await fetch( - `/api/generate-agora-token?channel=${agoraData?.channel}&uid=${uid}` - ); - const data = await response.json(); - if (!response.ok) throw new Error('Failed to generate new token'); - return data.token; + const channel = agoraData?.channel; + if (!channel) { + throw new Error('Missing channel for token renewal'); + } + + const [rtcResponse, rtmResponse] = await Promise.all([ + fetch(`/api/generate-agora-token?channel=${channel}&uid=${uid}`), + fetch(`/api/generate-agora-token?channel=${channel}&uid=0`), + ]); + const [rtcData, rtmData] = await Promise.all([ + rtcResponse.json(), + rtmResponse.json(), + ]); + + if (!rtcResponse.ok || !rtmResponse.ok) { + throw new Error('Failed to generate renewal tokens'); + } + + return { + rtcToken: rtcData.token, + rtmToken: rtmData.token, + }; } catch (error) { console.error('Error renewing token:', error); throw error; @@ -174,7 +191,7 @@ export default function LandingPage() {

- Voice AI Demo + Voice AI Quickstart

{!showConversation && ( diff --git a/next.config.mjs b/next.config.mjs index 4ce2351..06bc568 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -1,8 +1,16 @@ +import { dirname } from 'node:path' +import { fileURLToPath } from 'node:url' + +const rootDir = dirname(fileURLToPath(import.meta.url)) + /** @type {import('next').NextConfig} */ const nextConfig = { images: { unoptimized: true, }, + turbopack: { + root: rootDir, + }, experimental: { webpackBuildWorker: true, parallelServerBuildTraces: true, diff --git a/package.json b/package.json index e4227a1..83f87b1 100644 --- a/package.json +++ b/package.json @@ -4,10 +4,10 @@ "description": "Next.js quickstart for building conversational AI applications with Agora's Conversational AI Engine", "private": true, "engines": { - "node": ">=24" + "node": ">=22" }, "scripts": { - "dev": "next dev", + "dev": "next dev --webpack", "build": "next build", "start": "next start", "lint": "eslint .", diff --git a/types/conversation.ts b/types/conversation.ts index 673e388..5b752bf 100644 --- a/types/conversation.ts +++ b/types/conversation.ts @@ -21,11 +21,16 @@ export interface AgentResponse { state: string; } +export interface AgoraRenewalTokens { + rtcToken: string; + rtmToken: string; +} + import type { RTMClient } from 'agora-rtm'; export interface ConversationComponentProps { agoraData: AgoraTokenData; rtmClient: RTMClient; - onTokenWillExpire: (uid: string) => Promise; + onTokenWillExpire: (uid: string) => Promise; onEndConversation: () => void; } From cf585454d3f3c1ee4357ecb091e522e036c6f725 Mon Sep 17 00:00:00 2001 From: digitallysavvy Date: Fri, 10 Apr 2026 12:57:54 -0400 Subject: [PATCH 6/6] updated docs and readme --- DOCS/GUIDE.md | 63 ++++++++++++++++++++++++++------------------------- README.md | 4 ++-- agents.md | 20 ++++++++-------- 3 files changed, 44 insertions(+), 43 deletions(-) diff --git a/DOCS/GUIDE.md b/DOCS/GUIDE.md index 4f964df..3f4d4c9 100644 --- a/DOCS/GUIDE.md +++ b/DOCS/GUIDE.md @@ -14,7 +14,7 @@ By the end of this guide, you will have a real-time audio conversation applicati Before starting, for the guide you're going to need to have: -- Node.js (v24 or higher; see `engines` in `package.json`) +- Node.js (v22 or higher; see `engines` in `package.json`) - A basic understanding of React with TypeScript and Next.js. - [An Agora account](https://console.agora.io/signup) - _first 10k minutes each month are free_ - Conversational AI service [activated on your AppID](https://console.agora.io/) @@ -828,16 +828,13 @@ import { AgoraClient, Agent, Area, - AresSTT, + DeepgramSTT, ExpiresIn, + MiniMaxTTS, OpenAI, - OpenAITTS, } from 'agora-agent-server-sdk'; - -// Mirrors `AgentPresets` from the SDK (not re-exported on package entry); used for Agora reseller LLM/TTS. -const BUILTIN_LLM_PRESET = 'openai_gpt_4o_mini' as const; -const BUILTIN_TTS_PRESET = 'openai_tts_1' as const; import { ClientStartRequest, AgentResponse } from '@/types/conversation'; +import { DEFAULT_AGENT_UID } from '@/lib/agora'; // System prompt that defines the agent's personality and behavior. // Swap this out to change what the agent talks about. @@ -871,7 +868,7 @@ If you don't know a specific fact about Agora, say so plainly and suggest checki const GREETING = process.env.NEXT_AGENT_GREETING ?? `Hi there! I'm Ada, your virtual assistant from Agora. How can I help?`; // agentUid identifies the AI in the RTC channel — must match NEXT_PUBLIC_AGENT_UID on the client -const agentUid = process.env.NEXT_PUBLIC_AGENT_UID || 'Agent'; +const agentUid = process.env.NEXT_PUBLIC_AGENT_UID ?? String(DEFAULT_AGENT_UID); function requireEnv(name: string): string { const value = process.env[name]; @@ -900,8 +897,8 @@ export async function POST(request: NextRequest) { appCertificate, }); - // ASR: Agora ARES (no third-party STT API key). - // LLM + TTS: Agora reseller presets (no BYOK); preset IDs must match createSession below. + // Default agent configuration: Agora-managed STT, LLM, and TTS. + // Optional BYOK examples can be added if you want to swap providers. const agent = new Agent({ name: `conversation-${Date.now()}-${Math.random().toString(36).substring(2, 8)}`, instructions: ADA_PROMPT, @@ -928,21 +925,29 @@ export async function POST(request: NextRequest) { }, advancedFeatures: { enable_rtm: true, enable_tools: true }, }) - .withStt(new AresSTT({ language: 'en' })) + .withStt( + new DeepgramSTT({ + model: 'nova-3', + language: 'en', + }), + ) .withLlm( new OpenAI({ model: 'gpt-4o-mini', greetingMessage: GREETING, failureMessage: 'Please wait a moment.', maxHistory: 15, - maxTokens: 1024, - temperature: 0.7, - topP: 0.95, + params: { + max_tokens: 1024, + temperature: 0.7, + top_p: 0.95, + }, }), ) .withTts( - new OpenAITTS({ - voice: 'alloy', + new MiniMaxTTS({ + model: 'speech_2_6_turbo', + voiceId: 'English_captivating_female1', }), ); @@ -952,7 +957,6 @@ export async function POST(request: NextRequest) { remoteUids: [requester_id], idleTimeout: 30, expiresIn: ExpiresIn.hours(1), - preset: [BUILTIN_LLM_PRESET, BUILTIN_TTS_PRESET], }); const agentId = await session.start(); @@ -977,9 +981,9 @@ export async function POST(request: NextRequest) { } ``` -This quickstart uses **`agora-agent-server-sdk` ^1.3.1** with **AresSTT** (Agora ASR), **OpenAI** (`gpt-4o-mini`, no `apiKey` in code — billing via Agora), **OpenAITTS** (`alloy`, no key), and **`createSession` `preset: ['openai_gpt_4o_mini', 'openai_tts_1']`** so LLM/TTS run on Agora’s reseller integration. The SDK still supports other STT/LLM/TTS providers if you bring your own keys and configuration. +This quickstart uses **`agora-agent-server-sdk` ^1.3.1** with **DeepgramSTT**, **OpenAI** (`gpt-4o-mini`), and **MiniMaxTTS** through Agora-managed defaults, so the base quickstart only needs Agora credentials. The SDK still supports other STT/LLM/TTS providers if you want to enable the optional BYOK examples. -> **Note:** Only Agora App ID, App Certificate, and (on the client) `NEXT_PUBLIC_AGENT_UID` are required for this default pipeline. See the environment variables reference at the end of this guide. Optional `NEXT_LLM_URL` / `NEXT_LLM_API_KEY` apply only if you use the optional `app/api/chat/completions` proxy. +> **Note:** Only Agora App ID, App Certificate, and `NEXT_PUBLIC_AGENT_UID` are required for this default agent configuration. See the environment variables reference at the end of this guide. Optional `NEXT_LLM_URL` / `NEXT_LLM_API_KEY` apply only if you enable the optional BYOK block or use the optional `app/api/chat/completions` proxy. ### Stop Conversation Route @@ -1479,7 +1483,7 @@ const GREETING = process.env.NEXT_AGENT_GREETING ?? `Hello! How can I assist you ### Customizing the Voice -The default pipeline uses **OpenAITTS** with `voice: 'alloy'` and the `openai_tts_1` session preset. To use another vendor, replace `.withTts(...)` and the TTS entry in `createSession`’s `preset` array with the matching SDK types and preset IDs from the Agora Conversational AI documentation. +The default agent configuration uses **MiniMaxTTS** with `model: 'speech_2_6_turbo'` and `voiceId: 'English_captivating_female1'`. To use another vendor, replace `.withTts(...)` with the matching SDK type and, if needed, enable the optional BYOK environment variables for that provider. ### Fine-tuning Voice Activity Detection @@ -1515,17 +1519,14 @@ Here's a complete list of environment variables for your `.env.local` file: # Agora Configuration NEXT_PUBLIC_AGORA_APP_ID= NEXT_AGORA_APP_CERTIFICATE= -NEXT_PUBLIC_AGENT_UID=Agent - -# LLM Configuration (OpenAI or compatible) -NEXT_LLM_URL=https://api.openai.com/v1/chat/completions -NEXT_LLM_API_KEY= - -# STT - Deepgram -NEXT_DEEPGRAM_API_KEY= - -# TTS - ElevenLabs -NEXT_ELEVENLABS_API_KEY= +NEXT_PUBLIC_AGENT_UID=12345 + +# Optional BYOK examples +# NEXT_LLM_URL=https://api.openai.com/v1/chat/completions +# NEXT_LLM_API_KEY= +# NEXT_DEEPGRAM_API_KEY= +# NEXT_ELEVENLABS_API_KEY= +# NEXT_ELEVENLABS_VOICE_ID= ``` ## Next Steps diff --git a/README.md b/README.md index d15342d..1e256fb 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Required environment variables: - `NEXT_PUBLIC_AGORA_APP_ID` from your Agora Console project - `NEXT_AGORA_APP_CERTIFICATE` from your Agora Console project -The default agent configuration in [`app/api/invite-agent/route.ts`](app/api/invite-agent/route.ts) uses Agora-managed vendor presets for STT, LLM, and TTS, so no additional vendor API keys are required to run the base quickstart. +The default agent configuration in [`app/api/invite-agent/route.ts`](app/api/invite-agent/route.ts) uses Agora-managed defaults for STT, LLM, and TTS, so no additional vendor API keys are required to run the base quickstart. ## What This Repository Includes @@ -86,7 +86,7 @@ cp env.local.example .env.local You can find `App ID` and `App Certificate` in your project settings in [Agora Console](https://console.agora.io/). -The default agent configuration in [`app/api/invite-agent/route.ts`](app/api/invite-agent/route.ts) uses Agora-managed vendor presets for STT, LLM, and TTS, so no additional vendor API keys are required for the base quickstart. +The default agent configuration in [`app/api/invite-agent/route.ts`](app/api/invite-agent/route.ts) uses Agora-managed defaults for STT, LLM, and TTS, so no additional vendor API keys are required for the base quickstart. 6. Start the development server. diff --git a/agents.md b/agents.md index 1aa635f..1a2e322 100644 --- a/agents.md +++ b/agents.md @@ -6,9 +6,9 @@ ## 1. What This Project Is -A Next.js 16 (App Router) quickstart that lets a browser user speak with an Agora Conversational AI agent. The browser joins an Agora RTC channel for audio; RTM carries real-time transcripts. A server-side call invites an Agora cloud agent into the same channel. The agent runs a full ASR → LLM → TTS pipeline and publishes audio back. +A Next.js 16 (App Router) quickstart that lets a browser user speak with an Agora Conversational AI agent. The browser joins an Agora RTC channel for audio; RTM carries real-time transcripts. A server-side call invites an Agora cloud agent into the same channel. The agent runs a full ASR → LLM → TTS voice experience and publishes audio back. -**Stack:** Next.js 15, React 19, TypeScript, Tailwind, pnpm, `agora-rtc-react`, `agora-rtm`, `agora-token`, `agora-agent-client-toolkit`, `agora-agent-uikit`, `agora-agent-server-sdk`. +**Stack:** Next.js 16, React 19, TypeScript, Tailwind, pnpm, `agora-rtc-react`, `agora-rtm`, `agora-token`, `agora-agent-client-toolkit`, `agora-agent-uikit`, `agora-agent-server-sdk`. --- @@ -76,11 +76,11 @@ All vars live in `.env.local` (gitignored). `env.local.example` is the source of |---|---|---| | `NEXT_PUBLIC_AGORA_APP_ID` | client+server | Agora project App ID | | `NEXT_AGORA_APP_CERTIFICATE` | server only | Signs tokens — never expose client-side | -| `NEXT_PUBLIC_AGENT_UID` | client+server | UID the AI agent joins with (e.g. `"Agent"`). `NEXT_PUBLIC_` prefix required — read in client component. Must match `agentUid` in `invite-agent/route.ts`. | -| `NEXT_LLM_URL` | server only | Any OpenAI-compatible chat completions endpoint | -| `NEXT_LLM_API_KEY` | server only | LLM API key | -| `NEXT_DEEPGRAM_API_KEY` | server only | Deepgram STT API key | -| `NEXT_ELEVENLABS_API_KEY` | server only | ElevenLabs TTS API key | +| `NEXT_PUBLIC_AGENT_UID` | client+server | UID the AI agent joins with (default `12345`). `NEXT_PUBLIC_` prefix required — read in client component. Must match `agentUid` in `invite-agent/route.ts`. | +| `NEXT_LLM_URL` | server only, optional | Any OpenAI-compatible chat completions endpoint for the optional BYOK LLM block | +| `NEXT_LLM_API_KEY` | server only, optional | LLM API key for the optional BYOK LLM block | +| `NEXT_DEEPGRAM_API_KEY` | server only, optional | Deepgram STT API key for the optional BYOK STT block | +| `NEXT_ELEVENLABS_API_KEY` | server only, optional | ElevenLabs TTS API key for the optional BYOK TTS block | --- @@ -106,7 +106,7 @@ Starts an Agora ConvoAI agent using `agora-agent-server-sdk`. **What it does:** 1. Validates required env vars (throws on startup if missing). -2. Builds the agent: `new AgoraClient(...)` → `new Agent({ instructions, greeting, turnDetection, advancedFeatures })` → `.withStt(DeepgramSTT)` → `.withLlm(OpenAI)` → `.withTts(ElevenLabsTTS)`. +2. Builds the agent: `new AgoraClient(...)` → `new Agent({ instructions, greeting, turnDetection, advancedFeatures })` → `.withStt(DeepgramSTT)` → `.withLlm(OpenAI)` → `.withTts(MiniMaxTTS)`. 3. `agent.createSession(client, { channel, agentUid, remoteUids, idleTimeout, expiresIn })`. 4. `await session.start()` → returns agent ID. 5. Returns `AgentResponse: { agent_id, create_ts, state }`. @@ -115,8 +115,8 @@ Starts an Agora ConvoAI agent using `agora-agent-server-sdk`. - `ADA_PROMPT` / `GREETING` — agent persona and opening line - `turnDetection.config` — VAD sensitivity (`speech_threshold`, `silence_duration_ms`, `interrupt_duration_ms`, `prefix_padding_ms`) - `advancedFeatures: { enable_rtm: true }` — required for RTM transcript delivery -- `model: 'gpt-4o'` in `OpenAI(...)` — LLM model -- `ELEVENLABS_VOICE_ID` constant — TTS voice +- `model: 'gpt-4o-mini'` in `OpenAI(...)` — LLM model +- `voiceId: 'English_captivating_female1'` in `MiniMaxTTS(...)` — default TTS voice **Turn detection** uses the current (non-deprecated) API: ```ts