diff --git a/.claude/agents/devex-reviewer.md b/.claude/agents/devex-reviewer.md deleted file mode 100644 index 79e512e..0000000 --- a/.claude/agents/devex-reviewer.md +++ /dev/null @@ -1,76 +0,0 @@ ---- -name: devex-reviewer -description: Reviews the quickstart for developer experience issues — setup friction, confusing patterns, missing documentation, and anything that would trip up a developer trying to use this as a starting point. ---- - -You are a developer experience (devex) reviewer for the agora-convoai-quickstart-nextjs project. - -This is a quickstart repo — developers clone it, read it, and use it as the basis for their own apps. Your job is to find anything that would slow them down, confuse them, or lead them astray. - -## What to review - -### 1. Onboarding friction - -Read through the full setup flow a new developer would follow: - -1. `README.md` — Is the setup clear? Are all prerequisites listed? Does the install command work? Are all env vars explained? -2. `env.local.example` — Does every variable have a comment explaining what it is and where to get the value? -3. `pnpm dev` startup — Are there any error paths a new dev would hit immediately (missing env vars, import errors, TypeScript errors)? - -### 2. Code clarity - -Review the core components for patterns that might confuse a developer reading the code for the first time: - -- `components/LandingPage.tsx` -- `components/ConversationComponent.tsx` -- `app/api/invite-agent/route.ts` - -Look for: -- Comments that explain *why*, not just *what* (especially for non-obvious patterns like the StrictMode guard) -- Missing explanations for surprising behavior -- Code that looks wrong but is intentional (needs a comment) -- TODO/FIXME comments that should be resolved or removed before publishing - -### 3. Documentation coverage - -- Does `DOCS/GUIDE.md` cover all the major concepts a developer needs to understand? -- Is there anything in the implementation that would surprise a developer who followed the guide? -- Are there important extension points (changing the LLM, swapping TTS, adding RAG) that aren't documented? - -### 4. Common gotchas - -Check whether the code adequately warns developers about known failure modes: - -- React StrictMode double-init (is the pattern explained in comments?) -- `useJoin` owning `client.leave()` (is this documented near the hook usage?) -- `NEXT_PUBLIC_AGENT_UID` must match exactly -- RTM token requirement (`buildTokenWithRtm`, not plain RTC token) -- UID remapping for `uid="0"` sentinel - -### 5. Copy-paste safety - -This code gets copied. Check for: -- Hardcoded values that developers will need to change (are they easy to find?) -- Values that look like placeholders but aren't labeled as such -- Constants that should be env vars but are hardcoded -- Anything that would silently break in a different environment - -## Output format - -Produce a prioritized list: - -``` -## High Priority (would block a developer) -1. :
- -## Medium Priority (would confuse or slow a developer) -1. :
- -## Low Priority (polish / nice-to-have) -1. :
- -## Looks Good -- -``` - -Be specific. Quote the relevant code or doc section and explain exactly what a developer would experience. diff --git a/.claude/agents/docs-sync.md b/.claude/agents/docs-sync.md deleted file mode 100644 index e712cc9..0000000 --- a/.claude/agents/docs-sync.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -name: docs-sync -description: Verifies that GUIDE.md, TEXT_STREAMING_GUIDE.md, and README.md are in sync with the actual implementation. Use when docs may have drifted from code. ---- - -You are a documentation accuracy reviewer for the agora-convoai-quickstart-nextjs project. - -Your job is to check whether the documentation matches the current implementation and report every discrepancy. You do not make changes — you produce a structured report. - -## What to check - -### 1. DOCS/GUIDE.md vs implementation - -Compare every code snippet and description in GUIDE.md against the actual source files: - -- `components/LandingPage.tsx` -- `components/ConversationComponent.tsx` -- `components/MicrophoneSelector.tsx` -- `app/api/invite-agent/route.ts` -- `app/api/generate-agora-token/route.ts` -- `app/api/stop-conversation/route.ts` -- `types/conversation.ts` -- `tailwind.config.ts` -- `package.json` (dependency names and presence) - -Flag any snippet in the guide that uses: -- Wrong import paths or package names -- Deprecated APIs (e.g. `turnDetection.type: 'agora_vad'` instead of `turnDetection.config.*`) -- Removed or renamed functions/components -- Wrong prop names or constructor signatures -- Patterns that differ from what's actually in the code - -### 2. DOCS/TEXT_STREAMING_GUIDE.md vs implementation - -Compare against: -- `components/ConversationComponent.tsx` (AgoraVoiceAI init, UID remapping, transcript state) -- `components/LandingPage.tsx` (RTM client setup) -- `app/api/invite-agent/route.ts` (enable_rtm) -- `app/api/generate-agora-token/route.ts` (buildTokenWithRtm) - -### 3. README.md vs implementation - -Check: -- Component names in "Key Components" section match actual files -- Package names are correct -- API endpoint descriptions match route implementations -- Node.js version requirement matches package.json `engines` or next.config -- No references to deleted files or old architecture - -### 4. agents.md vs implementation - -This file is a machine-readable project map. Check that: -- Component list matches actual files in `components/` -- Library descriptions match what's actually imported and used -- API route descriptions are accurate -- Data flow diagram matches actual code paths -- "Known gotchas" are still relevant - -## Output format - -Produce a report with this structure: - -``` -## DOCS/GUIDE.md -- [INACCURACY]
: vs -- [STALE]
: -- [OK]
— accurate - -## DOCS/TEXT_STREAMING_GUIDE.md -... - -## README.md -... - -## agents.md -... - -## Summary -N issues found across M files. -``` - -If a section is fully accurate, a single `[OK]` line is enough. Be specific about inaccuracies — quote the guide and quote the code. diff --git a/DOCS/GUIDE.md b/DOCS/GUIDE.md index 4f964df..3f4d4c9 100644 --- a/DOCS/GUIDE.md +++ b/DOCS/GUIDE.md @@ -14,7 +14,7 @@ By the end of this guide, you will have a real-time audio conversation applicati Before starting, for the guide you're going to need to have: -- Node.js (v24 or higher; see `engines` in `package.json`) +- Node.js (v22 or higher; see `engines` in `package.json`) - A basic understanding of React with TypeScript and Next.js. - [An Agora account](https://console.agora.io/signup) - _first 10k minutes each month are free_ - Conversational AI service [activated on your AppID](https://console.agora.io/) @@ -828,16 +828,13 @@ import { AgoraClient, Agent, Area, - AresSTT, + DeepgramSTT, ExpiresIn, + MiniMaxTTS, OpenAI, - OpenAITTS, } from 'agora-agent-server-sdk'; - -// Mirrors `AgentPresets` from the SDK (not re-exported on package entry); used for Agora reseller LLM/TTS. -const BUILTIN_LLM_PRESET = 'openai_gpt_4o_mini' as const; -const BUILTIN_TTS_PRESET = 'openai_tts_1' as const; import { ClientStartRequest, AgentResponse } from '@/types/conversation'; +import { DEFAULT_AGENT_UID } from '@/lib/agora'; // System prompt that defines the agent's personality and behavior. // Swap this out to change what the agent talks about. @@ -871,7 +868,7 @@ If you don't know a specific fact about Agora, say so plainly and suggest checki const GREETING = process.env.NEXT_AGENT_GREETING ?? `Hi there! I'm Ada, your virtual assistant from Agora. How can I help?`; // agentUid identifies the AI in the RTC channel — must match NEXT_PUBLIC_AGENT_UID on the client -const agentUid = process.env.NEXT_PUBLIC_AGENT_UID || 'Agent'; +const agentUid = process.env.NEXT_PUBLIC_AGENT_UID ?? String(DEFAULT_AGENT_UID); function requireEnv(name: string): string { const value = process.env[name]; @@ -900,8 +897,8 @@ export async function POST(request: NextRequest) { appCertificate, }); - // ASR: Agora ARES (no third-party STT API key). - // LLM + TTS: Agora reseller presets (no BYOK); preset IDs must match createSession below. + // Default agent configuration: Agora-managed STT, LLM, and TTS. + // Optional BYOK examples can be added if you want to swap providers. const agent = new Agent({ name: `conversation-${Date.now()}-${Math.random().toString(36).substring(2, 8)}`, instructions: ADA_PROMPT, @@ -928,21 +925,29 @@ export async function POST(request: NextRequest) { }, advancedFeatures: { enable_rtm: true, enable_tools: true }, }) - .withStt(new AresSTT({ language: 'en' })) + .withStt( + new DeepgramSTT({ + model: 'nova-3', + language: 'en', + }), + ) .withLlm( new OpenAI({ model: 'gpt-4o-mini', greetingMessage: GREETING, failureMessage: 'Please wait a moment.', maxHistory: 15, - maxTokens: 1024, - temperature: 0.7, - topP: 0.95, + params: { + max_tokens: 1024, + temperature: 0.7, + top_p: 0.95, + }, }), ) .withTts( - new OpenAITTS({ - voice: 'alloy', + new MiniMaxTTS({ + model: 'speech_2_6_turbo', + voiceId: 'English_captivating_female1', }), ); @@ -952,7 +957,6 @@ export async function POST(request: NextRequest) { remoteUids: [requester_id], idleTimeout: 30, expiresIn: ExpiresIn.hours(1), - preset: [BUILTIN_LLM_PRESET, BUILTIN_TTS_PRESET], }); const agentId = await session.start(); @@ -977,9 +981,9 @@ export async function POST(request: NextRequest) { } ``` -This quickstart uses **`agora-agent-server-sdk` ^1.3.1** with **AresSTT** (Agora ASR), **OpenAI** (`gpt-4o-mini`, no `apiKey` in code — billing via Agora), **OpenAITTS** (`alloy`, no key), and **`createSession` `preset: ['openai_gpt_4o_mini', 'openai_tts_1']`** so LLM/TTS run on Agora’s reseller integration. The SDK still supports other STT/LLM/TTS providers if you bring your own keys and configuration. +This quickstart uses **`agora-agent-server-sdk` ^1.3.1** with **DeepgramSTT**, **OpenAI** (`gpt-4o-mini`), and **MiniMaxTTS** through Agora-managed defaults, so the base quickstart only needs Agora credentials. The SDK still supports other STT/LLM/TTS providers if you want to enable the optional BYOK examples. -> **Note:** Only Agora App ID, App Certificate, and (on the client) `NEXT_PUBLIC_AGENT_UID` are required for this default pipeline. See the environment variables reference at the end of this guide. Optional `NEXT_LLM_URL` / `NEXT_LLM_API_KEY` apply only if you use the optional `app/api/chat/completions` proxy. +> **Note:** Only Agora App ID, App Certificate, and `NEXT_PUBLIC_AGENT_UID` are required for this default agent configuration. See the environment variables reference at the end of this guide. Optional `NEXT_LLM_URL` / `NEXT_LLM_API_KEY` apply only if you enable the optional BYOK block or use the optional `app/api/chat/completions` proxy. ### Stop Conversation Route @@ -1479,7 +1483,7 @@ const GREETING = process.env.NEXT_AGENT_GREETING ?? `Hello! How can I assist you ### Customizing the Voice -The default pipeline uses **OpenAITTS** with `voice: 'alloy'` and the `openai_tts_1` session preset. To use another vendor, replace `.withTts(...)` and the TTS entry in `createSession`’s `preset` array with the matching SDK types and preset IDs from the Agora Conversational AI documentation. +The default agent configuration uses **MiniMaxTTS** with `model: 'speech_2_6_turbo'` and `voiceId: 'English_captivating_female1'`. To use another vendor, replace `.withTts(...)` with the matching SDK type and, if needed, enable the optional BYOK environment variables for that provider. ### Fine-tuning Voice Activity Detection @@ -1515,17 +1519,14 @@ Here's a complete list of environment variables for your `.env.local` file: # Agora Configuration NEXT_PUBLIC_AGORA_APP_ID= NEXT_AGORA_APP_CERTIFICATE= -NEXT_PUBLIC_AGENT_UID=Agent - -# LLM Configuration (OpenAI or compatible) -NEXT_LLM_URL=https://api.openai.com/v1/chat/completions -NEXT_LLM_API_KEY= - -# STT - Deepgram -NEXT_DEEPGRAM_API_KEY= - -# TTS - ElevenLabs -NEXT_ELEVENLABS_API_KEY= +NEXT_PUBLIC_AGENT_UID=12345 + +# Optional BYOK examples +# NEXT_LLM_URL=https://api.openai.com/v1/chat/completions +# NEXT_LLM_API_KEY= +# NEXT_DEEPGRAM_API_KEY= +# NEXT_ELEVENLABS_API_KEY= +# NEXT_ELEVENLABS_VOICE_ID= ``` ## Next Steps diff --git a/README.md b/README.md index bd9a4cd..1e256fb 100644 --- a/README.md +++ b/README.md @@ -1,169 +1,145 @@ -# Conversational AI: Dev Advocate Agent Demo +# Agora Conversational AI Next.js Quickstart -A Next.js web application demonstrating real-time conversational AI capabilities using Agora's Real-Time Engagement SDKs. This demo showcases voice-first interactions with live transcriptions, multi-device audio input support, and an Agent ready to help you with your Agora build. +This is the official Agora Next.js quickstart for building a browser-based voice AI experience with Agora Conversational AI Engine. -## Overview +## Run It -This application demonstrates how to build a production-ready conversational AI interface with: -- **Real-time voice conversations** with AI agents powered by Agora's Conversational AI Engine -- **RTM-based messaging** for reliable real-time transcriptions and agent state updates -- **Live text transcriptions** with streaming message updates and visual status indicators -- **Advanced audio controls** including device selection and visual feedback -- **Modern UX patterns** like smart auto-scrolling, mobile responsiveness, and accessibility features -- **Flexible backend integration** supporting multiple LLM providers (OpenAI, Anthropic, etc.) and TTS via ElevenLabs -- **Official Agora toolkit** integration for robust conversation management +1. Create or sign in to your Agora account in [Agora Console](https://console.agora.io/). +2. In Agora Console, create a project and copy your `App ID` and `App Certificate`. +3. Clone this repository and install dependencies. +4. Copy `env.local.example` to `.env.local`. +5. Set `NEXT_PUBLIC_AGORA_APP_ID` and `NEXT_AGORA_APP_CERTIFICATE`. +6. Run `pnpm dev`. +7. Open `http://localhost:3000`. + +```bash +git clone https://github.com/AgoraIO-Conversational-AI/agent-quickstart-nextjs.git +cd agent-quickstart-nextjs +pnpm install +cp env.local.example .env.local +pnpm dev +``` + +Required environment variables: + +- `NEXT_PUBLIC_AGORA_APP_ID` from your Agora Console project +- `NEXT_AGORA_APP_CERTIFICATE` from your Agora Console project + +The default agent configuration in [`app/api/invite-agent/route.ts`](app/api/invite-agent/route.ts) uses Agora-managed defaults for STT, LLM, and TTS, so no additional vendor API keys are required to run the base quickstart. + +## What This Repository Includes + +- a Next.js App Router frontend for joining an Agora RTC channel from the browser +- RTM-based live transcripts and agent state updates +- a Next.js backend for token generation and agent lifecycle management +- an Agora cloud agent that joins the same channel and runs the voice experience +- Agora-managed default STT, LLM, and TTS configuration, with optional BYOK examples + +## How It Works + +When a user starts a session, the app: + +1. generates an RTC + RTM token from the Next.js backend +2. invites an Agora Conversational AI agent from the backend +3. joins the browser client to the Agora channel for audio +4. receives live transcript and state events over RTM +5. stops the agent and cleans up the session when the conversation ends ## Guides and Documentation -- [Guide.md](./DOCS/GUIDE.md) - Complete step-by-step guide on how to build this application from scratch. -- [Text Streaming Guide](./DOCS/TEXT_STREAMING_GUIDE.md) - Deep dive into real-time conversation transcriptions using the toolkit and UI kit. +- [Guide.md](./DOCS/GUIDE.md) - step-by-step build guide +- [Text Streaming Guide](./DOCS/TEXT_STREAMING_GUIDE.md) - transcript and RTM flow details ## Prerequisites -Before you begin, ensure you have the following installed: - - [Node.js](https://nodejs.org/) (version 22.x or higher) - [pnpm](https://pnpm.io/) (version 8.x or higher) +- [Agora Console account](https://console.agora.io/) -You must have an Agora account and a project to use this application. - -- [Agora Account](https://console.agora.io/) +## Quickstart -## Installation +1. Create a project in [Agora Console](https://console.agora.io/) and copy the `App ID` and `App Certificate`. -1. Clone the repository: +2. Clone the repository. ```bash git clone https://github.com/AgoraIO-Conversational-AI/agent-quickstart-nextjs.git cd agent-quickstart-nextjs ``` -2. Install dependencies: +3. Install dependencies. ```bash pnpm install ``` -3. Create a `.env.local` file in the root directory and add your environment variables: +4. Create `.env.local`. ```bash cp env.local.example .env.local ``` -The following environment variables are required: - -### Agora +5. Set the required Agora credentials. - `NEXT_PUBLIC_AGORA_APP_ID` - Your Agora App ID - `NEXT_AGORA_APP_CERTIFICATE` - Your Agora App Certificate -- `NEXT_PUBLIC_AGENT_UID` - UID assigned to the AI agent in the RTC channel - -### LLM - -- `NEXT_LLM_URL` - Any OpenAI-compatible endpoint (OpenAI, Azure, Groq, etc.) -- `NEXT_LLM_API_KEY` - LLM API key - -### ASR - -- `NEXT_DEEPGRAM_API_KEY` - Deepgram API key - -### TTS -- `NEXT_ELEVENLABS_API_KEY` - ElevenLabs API key +You can find `App ID` and `App Certificate` in your project settings in [Agora Console](https://console.agora.io/). -Non-sensitive settings (model names, voice ID, language, etc.) are set directly in [`app/api/invite-agent/route.ts`](app/api/invite-agent/route.ts) — edit them there. +The default agent configuration in [`app/api/invite-agent/route.ts`](app/api/invite-agent/route.ts) uses Agora-managed defaults for STT, LLM, and TTS, so no additional vendor API keys are required for the base quickstart. -4. Run the development server: +6. Start the development server. ```bash pnpm dev ``` -5. Open your browser and navigate to `http://localhost:3000` to see the application in action. +7. Open `http://localhost:3000`. + +## Optional BYOK Configuration + +Optional BYOK examples remain commented in [`app/api/invite-agent/route.ts`](app/api/invite-agent/route.ts). Uncomment those only if you want to provide your own vendor credentials such as: + +- `NEXT_LLM_URL` and `NEXT_LLM_API_KEY` +- `NEXT_DEEPGRAM_API_KEY` +- `NEXT_ELEVENLABS_API_KEY` and `NEXT_ELEVENLABS_VOICE_ID` ## Deployment to Vercel -This project is configured for quick deployments to Vercel. +This repository is configured for one-click Vercel deployment. -[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2FAgoraIO-Conversational-AI%2Fagent-quickstart-nextjs&project-name=agent-quickstart-nextjs&repository-name=agent-quickstart-nextjs&env=NEXT_PUBLIC_AGORA_APP_ID,NEXT_AGORA_APP_CERTIFICATE,NEXT_PUBLIC_AGENT_UID,NEXT_LLM_URL,NEXT_LLM_API_KEY,NEXT_DEEPGRAM_API_KEY,NEXT_ELEVENLABS_API_KEY&envDescription=API%20keys%20and%20credentials%20needed%20to%20run%20the%20app&envLink=https%3A%2F%2Fgithub.com%2FAgoraIO-Conversational-AI%2Fagent-quickstart-nextjs%23prerequisites&demo-title=Conversational%20AI%20Demo&demo-description=A%20Next.js-based%20web-app%20for%20conversational%20AI%20agents&demo-image=https%3A%2F%2Fraw.githubusercontent.com%2FAgoraIO-Conversational-AI%2Fagent-quickstart-nextjs%2Fmain%2F.github%2Fassets%2FConversation-Ai-Client.gif&defaultValues=NEXT_LLM_URL=https://api.openai.com/v1/chat/completions) +[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2FAgoraIO-Conversational-AI%2Fagent-quickstart-nextjs&project-name=agent-quickstart-nextjs&repository-name=agent-quickstart-nextjs&env=NEXT_PUBLIC_AGORA_APP_ID,NEXT_AGORA_APP_CERTIFICATE&envDescription=Agora%20credentials%20needed%20to%20run%20the%20app&envLink=https%3A%2F%2Fgithub.com%2FAgoraIO-Conversational-AI%2Fagent-quickstart-nextjs%23prerequisites&demo-title=Agora%20Conversational%20AI%20Next.js%20Quickstart&demo-description=Official%20Next.js%20quickstart%20for%20building%20browser-based%20voice%20AI%20with%20Agora&demo-image=https%3A%2F%2Fraw.githubusercontent.com%2FAgoraIO-Conversational-AI%2Fagent-quickstart-nextjs%2Fmain%2F.github%2Fassets%2FConversation-Ai-Client.gif) This will: 1. Clone the repository to your GitHub account 2. Create a new project on Vercel 3. Prompt you to fill in the required environment variables: - - **Required**: Agora credentials (`NEXT_PUBLIC_AGORA_APP_ID`, `NEXT_AGORA_APP_CERTIFICATE`, `NEXT_PUBLIC_AGENT_UID`) - - **Required**: LLM endpoint and API key (`NEXT_LLM_URL`, `NEXT_LLM_API_KEY`) - - **Required**: Deepgram API key (`NEXT_DEEPGRAM_API_KEY`) and ElevenLabs API key (`NEXT_ELEVENLABS_API_KEY`) + - **Required**: Agora credentials (`NEXT_PUBLIC_AGORA_APP_ID`, `NEXT_AGORA_APP_CERTIFICATE`) 4. Deploy the application automatically -## Features - -### Audio Input Control -- **Microphone Toggle**: Easy-to-use button to enable/disable your microphone -- **Device Selection**: Choose from multiple microphone inputs with the microphone selector dropdown -- **Hot-Swap Support**: Automatically detects when devices are plugged in/unplugged -- **Audio Visualization**: Real-time visual feedback showing microphone input levels - -### Real-Time Text Streaming -- **Live Transcriptions**: See what you say and the AI's responses in real-time as text -- **Message Status Indicators**: Visual feedback for in-progress, completed, and interrupted messages -- **Smart Auto-Scroll**: Automatically scrolls to new messages while preserving scroll position when reviewing history -- **Mobile-Responsive Chat UI**: Collapsible chat window that adapts to different screen sizes -- **Desktop Auto-Open**: Chat window automatically opens on first message (desktop only) -- **Message Persistence**: Full conversation history maintained throughout the session - -### AI Conversation Engine -- **Custom LLM Integration**: Connect your preferred LLM (OpenAI, Anthropic, etc.) -- **ElevenLabs TTS**: High-quality voice synthesis with ElevenLabs -- **Modern Turn Detection**: Advanced turn-taking with configurable interrupt behavior -- **RTM Data Channel**: Reliable message delivery with metrics and error reporting -- **Token Management**: Automatic token renewal for both RTC and RTM to prevent disconnections -- **Agent Lifecycle**: Agent is invited when you click "Try it now!"; End Conversation button stops the agent and closes the session -- **Official Toolkit**: Uses Agora's ConversationalAIAPI for robust conversation management +## Included in This Repo -### User Experience -- **Audio Visualizations**: Animated frequency bars for both user and AI audio -- **Connection Status**: Real-time connection indicators -- **Error Handling**: Graceful error messages and recovery options -- **Accessibility**: ARIA labels and keyboard-friendly controls +### Frontend -## Voice Options +- `app/page.tsx` renders the landing experience +- `components/LandingPage.tsx` starts the session, requests tokens, and logs RTM in +- `components/ConversationComponent.tsx` manages RTC join, mic publishing, agent state, and transcript rendering +- `components/MicrophoneSelector.tsx` handles input device selection -### ElevenLabs +### Backend -Browse and select voices at: https://elevenlabs.io/app/voice-lab - -Set your chosen voice ID in the `ELEVENLABS_VOICE_ID` constant in [`app/api/invite-agent/route.ts`](app/api/invite-agent/route.ts). - -## Key Components - -The application is built with a modular component architecture: - -### Core Components - -- **`LandingPage.tsx`**: Entry point that invites the agent when you click "Try it now!" and manages the conversation lifecycle with proper agent cleanup on end -- **`ConversationComponent.tsx`**: Main conversation container handling RTC and RTM connections, audio/text streaming, and the End Conversation flow -- **`MicrophoneSelector.tsx`**: Dropdown component for selecting audio input devices with hot-swap support +- `app/api/generate-agora-token/route.ts` issues RTC + RTM tokens +- `app/api/invite-agent/route.ts` starts the agent session +- `app/api/stop-conversation/route.ts` stops the active agent session +- `app/api/chat/completions/route.ts` is an optional OpenAI-compatible custom LLM proxy ### Agora Packages -- **`agora-agent-uikit`**: Pre-built conversation UI components used directly in `ConversationComponent`: - - `AudioVisualizer` — animated frequency bars that respond to the agent's audio track - - `ConvoTextStream` — floating chat panel showing live and completed transcript turns - - `MicButtonWithVisualizer` (from `agora-agent-uikit/rtc`) — mic button with built-in Web Audio visualization - - `transcriptToMessageList` — converts toolkit transcript items into UI-ready message objects -- **`agora-agent-client-toolkit`**: `AgoraVoiceAI` class that subscribes to RTM transcript events and normalizes them into a simple message list -- **`agora-agent-server-sdk`**: Server-side SDK used in API routes to start and stop the AI agent - -### Utilities - -- **`lib/utils.ts`**: Helper functions including the shadcn `cn` class merge utility -- **`types/conversation.ts`**: TypeScript type definitions for conversation data structures - -## Contributing - -Contributions are welcome! Please feel free to submit a Pull Request. +- `agora-agent-server-sdk` manages agent lifecycle from the server +- `agora-agent-client-toolkit` handles transcript and agent events in the browser +- `agora-agent-uikit` provides the conversation UI components +- `agora-rtc-react` and `agora-rtm` handle media and messaging transport ## API Endpoints @@ -205,9 +181,9 @@ The application provides the following API endpoints: } ``` -## Technical Implementation Details +## Runtime Details -### Text Streaming Architecture +### Transcript Flow The text streaming feature uses `agora-agent-client-toolkit` with RTM for reliable real-time transcriptions: @@ -216,13 +192,14 @@ The text streaming feature uses `agora-agent-client-toolkit` with RTM for reliab 3. **`ConversationComponent`** handles the event, remaps local user UIDs, and updates React state with separated in-progress and completed turns 4. **`ConvoTextStream`** (from `agora-agent-uikit`) renders the chat panel with smart scrolling and streaming indicators -Key features: -- **Dual RTC + RTM tokens** for secure access to both audio and messaging channels -- **Audio PTS metadata** enabled for accurate transcription timing synchronization -- **Modern turn detection** with configurable interrupt behavior -- **Proper resource cleanup** when conversations end +Key points: -### Microphone Device Management +- dual RTC + RTM tokens are used for secure access to both transports +- audio PTS metadata is enabled for transcript timing accuracy +- turn detection is configured in the invite route +- the app cleans up RTC, RTM, and agent state when the conversation ends + +### Audio Input The MicrophoneSelector component provides: @@ -231,7 +208,7 @@ The MicrophoneSelector component provides: - **Seamless switching** using `localMicrophoneTrack.setDevice(deviceId)` - **Automatic fallback** when the current device is disconnected -### Audio Visualization +### Visualization `AudioVisualizer` and `MicButtonWithVisualizer` (from `agora-agent-uikit`) use the Web Audio API: @@ -241,23 +218,37 @@ The MicrophoneSelector component provides: ## Architecture -This application uses a dual-channel architecture for optimal performance: +This quickstart uses a split media-plane and control-plane architecture: + +```mermaid +flowchart LR + C["Voice AI Client
Next.js browser app"] -->|"RTC audio + RTM transcripts/state"| S["Agora SD-RTN"] + A["Conversational AI Agent
Agora cloud agent session"] -->|"RTC audio + RTM events"| S + C -->|"Token request"| B["Next.js Backend
API routes + server-side agent management"] + B -->|"Generate RTC/RTM token"| C + B -->|"invite / update / list / stop
agent management APIs"| G["Agora Conversational AI service"] + G -->|"Starts and manages agent session"| A +``` + +The browser client and the cloud agent both connect to Agora's SD-RTN for the real-time audio and RTM data path. The Next.js backend stays on the control plane: it generates tokens and manages the agent lifecycle against Agora's service APIs. In this quickstart, the exposed routes cover invite and stop today, and the same backend layer is where update and list operations belong. + +### RTC + RTM -### RTC + RTM Integration - **RTC (Real-Time Communication)**: Handles high-quality audio streaming between users and AI agents - **RTM (Real-Time Messaging)**: Delivers transcriptions, agent state updates, metrics, and error messages - **Dual Token Authentication**: Single token provides secure access to both RTC and RTM services - **Audio PTS Metadata**: Enables precise synchronization between audio playback and transcription display -### Conversation Management -- **`agora-agent-client-toolkit`**: `AgoraVoiceAI` class managing the complete transcript lifecycle -- **Event-Driven Architecture**: Real-time updates for transcripts, agent state changes, and system events -- **Turn Detection**: Modern voice activity detection with configurable interrupt behavior -- **Resource Cleanup**: Automatic cleanup of RTC, RTM, and agent resources when conversations end - -### Benefits -- Reliable message delivery through dedicated RTM channel -- Access to real-time agent metrics and error reporting -- Better timing synchronization for natural conversation flow -- Proper resource management preventing memory leaks -- Modern API patterns following Agora best practices +### Agent Lifecycle + +- **`agora-agent-server-sdk`**: starts and stops the cloud agent session from the Next.js backend +- **Invite Route**: configures the default STT, LLM, TTS, turn detection, and RTM-enabled agent behavior +- **Client Toolkit**: receives transcript, metrics, and state events after the agent joins the channel +- **Resource Cleanup**: the app stops the agent session and tears down RTC and RTM resources when the conversation ends + +### Why It Is Structured This Way + +- audio stays on Agora's real-time network instead of passing through your Next.js server +- the backend only handles token generation and agent control operations +- RTM carries transcripts, metrics, and agent state separately from the audio stream +- the client remains a standard browser app built with Next.js and React diff --git a/agents.md b/agents.md index 1aa635f..1a2e322 100644 --- a/agents.md +++ b/agents.md @@ -6,9 +6,9 @@ ## 1. What This Project Is -A Next.js 16 (App Router) quickstart that lets a browser user speak with an Agora Conversational AI agent. The browser joins an Agora RTC channel for audio; RTM carries real-time transcripts. A server-side call invites an Agora cloud agent into the same channel. The agent runs a full ASR → LLM → TTS pipeline and publishes audio back. +A Next.js 16 (App Router) quickstart that lets a browser user speak with an Agora Conversational AI agent. The browser joins an Agora RTC channel for audio; RTM carries real-time transcripts. A server-side call invites an Agora cloud agent into the same channel. The agent runs a full ASR → LLM → TTS voice experience and publishes audio back. -**Stack:** Next.js 15, React 19, TypeScript, Tailwind, pnpm, `agora-rtc-react`, `agora-rtm`, `agora-token`, `agora-agent-client-toolkit`, `agora-agent-uikit`, `agora-agent-server-sdk`. +**Stack:** Next.js 16, React 19, TypeScript, Tailwind, pnpm, `agora-rtc-react`, `agora-rtm`, `agora-token`, `agora-agent-client-toolkit`, `agora-agent-uikit`, `agora-agent-server-sdk`. --- @@ -76,11 +76,11 @@ All vars live in `.env.local` (gitignored). `env.local.example` is the source of |---|---|---| | `NEXT_PUBLIC_AGORA_APP_ID` | client+server | Agora project App ID | | `NEXT_AGORA_APP_CERTIFICATE` | server only | Signs tokens — never expose client-side | -| `NEXT_PUBLIC_AGENT_UID` | client+server | UID the AI agent joins with (e.g. `"Agent"`). `NEXT_PUBLIC_` prefix required — read in client component. Must match `agentUid` in `invite-agent/route.ts`. | -| `NEXT_LLM_URL` | server only | Any OpenAI-compatible chat completions endpoint | -| `NEXT_LLM_API_KEY` | server only | LLM API key | -| `NEXT_DEEPGRAM_API_KEY` | server only | Deepgram STT API key | -| `NEXT_ELEVENLABS_API_KEY` | server only | ElevenLabs TTS API key | +| `NEXT_PUBLIC_AGENT_UID` | client+server | UID the AI agent joins with (default `12345`). `NEXT_PUBLIC_` prefix required — read in client component. Must match `agentUid` in `invite-agent/route.ts`. | +| `NEXT_LLM_URL` | server only, optional | Any OpenAI-compatible chat completions endpoint for the optional BYOK LLM block | +| `NEXT_LLM_API_KEY` | server only, optional | LLM API key for the optional BYOK LLM block | +| `NEXT_DEEPGRAM_API_KEY` | server only, optional | Deepgram STT API key for the optional BYOK STT block | +| `NEXT_ELEVENLABS_API_KEY` | server only, optional | ElevenLabs TTS API key for the optional BYOK TTS block | --- @@ -106,7 +106,7 @@ Starts an Agora ConvoAI agent using `agora-agent-server-sdk`. **What it does:** 1. Validates required env vars (throws on startup if missing). -2. Builds the agent: `new AgoraClient(...)` → `new Agent({ instructions, greeting, turnDetection, advancedFeatures })` → `.withStt(DeepgramSTT)` → `.withLlm(OpenAI)` → `.withTts(ElevenLabsTTS)`. +2. Builds the agent: `new AgoraClient(...)` → `new Agent({ instructions, greeting, turnDetection, advancedFeatures })` → `.withStt(DeepgramSTT)` → `.withLlm(OpenAI)` → `.withTts(MiniMaxTTS)`. 3. `agent.createSession(client, { channel, agentUid, remoteUids, idleTimeout, expiresIn })`. 4. `await session.start()` → returns agent ID. 5. Returns `AgentResponse: { agent_id, create_ts, state }`. @@ -115,8 +115,8 @@ Starts an Agora ConvoAI agent using `agora-agent-server-sdk`. - `ADA_PROMPT` / `GREETING` — agent persona and opening line - `turnDetection.config` — VAD sensitivity (`speech_threshold`, `silence_duration_ms`, `interrupt_duration_ms`, `prefix_padding_ms`) - `advancedFeatures: { enable_rtm: true }` — required for RTM transcript delivery -- `model: 'gpt-4o'` in `OpenAI(...)` — LLM model -- `ELEVENLABS_VOICE_ID` constant — TTS voice +- `model: 'gpt-4o-mini'` in `OpenAI(...)` — LLM model +- `voiceId: 'English_captivating_female1'` in `MiniMaxTTS(...)` — default TTS voice **Turn detection** uses the current (non-deprecated) API: ```ts diff --git a/app/api/invite-agent/route.ts b/app/api/invite-agent/route.ts index 8c762af..018430a 100644 --- a/app/api/invite-agent/route.ts +++ b/app/api/invite-agent/route.ts @@ -4,16 +4,16 @@ import { Agent, Area, DeepgramSTT, - ElevenLabsTTS, ExpiresIn, MiniMaxTTS, OpenAI, } from 'agora-agent-server-sdk'; import { ClientStartRequest, AgentResponse } from '@/types/conversation'; +import { DEFAULT_AGENT_UID } from '@/lib/agora'; // System prompt that defines the agent's personality and behavior. // Swap this out to change what the agent talks about. -const ADA_PROMPT = `You are **Ada**, a developer advocate AI from **Agora**. You help developers understand and build with Agora's Conversational AI platform. +const ADA_PROMPT = `You are **Ada**, an agentic developer advocate from **Agora**. You help developers understand and build with Agora's Conversational AI platform. # What Agora Actually Is Agora is a real-time communications company. The product you represent is the **Agora Conversational AI Engine** — it lets developers add voice AI agents to any app by connecting ASR, LLM, and TTS into a real-time pipeline over Agora's SD-RTN (Software Defined Real-Time Network). Key facts: @@ -40,14 +40,12 @@ If you don't know a specific fact about Agora, say so plainly and suggest checki // First thing the agent says when a user joins the channel. // Set NEXT_AGENT_GREETING in .env.local to override. -const GREETING = process.env.NEXT_AGENT_GREETING ?? `Hi there! I'm Ada, your virtual assistant from Agora. How can I help?`; +const GREETING = + process.env.NEXT_AGENT_GREETING ?? + `Hi there! I'm Ada, your virtual assistant from Agora. How can I help?`; // agentUid identifies the AI in the RTC channel — must match NEXT_PUBLIC_AGENT_UID on the client -const agentUid = process.env.NEXT_PUBLIC_AGENT_UID || '123456'; - -// ElevenLabs BYOK (`.withTts` commented block). Override with NEXT_ELEVENLABS_VOICE_ID. -const ELEVENLABS_VOICE_ID = - process.env.NEXT_ELEVENLABS_VOICE_ID ?? 'pNInz6obpgDQGcFmaJgB'; +const agentUid = process.env.NEXT_PUBLIC_AGENT_UID ?? String(DEFAULT_AGENT_UID); function requireEnv(name: string): string { const value = process.env[name]; @@ -64,7 +62,8 @@ export async function POST(request: NextRequest) { // Validate required env vars on first request so misconfiguration surfaces // with a clear error message rather than a silent failure. - const appId = process.env.NEXT_PUBLIC_AGORA_APP_ID || requireEnv('NEXT_AGORA_APP_ID'); + const appId = + process.env.NEXT_PUBLIC_AGORA_APP_ID || requireEnv('NEXT_AGORA_APP_ID'); const appCertificate = requireEnv('NEXT_AGORA_APP_CERTIFICATE'); if (!channel_name || !requester_id) { @@ -114,6 +113,10 @@ export async function POST(request: NextRequest) { // RTM is required for transcript events in the browser client. // enable_tools is required for MCP tool invocation. advancedFeatures: { enable_rtm: true, enable_tools: true }, + // Required for browser RTM events: + // - data_channel: 'rtm' enables RTM delivery path for state/metrics/errors + // - enable_error_message emits AGENT_ERROR payloads + parameters: { data_channel: 'rtm', enable_error_message: true }, }) .withStt( new DeepgramSTT({ @@ -133,14 +136,16 @@ export async function POST(request: NextRequest) { greetingMessage: GREETING, failureMessage: 'Please wait a moment.', maxHistory: 15, - maxTokens: 1024, - temperature: 0.7, - topP: 0.95, + params: { + max_tokens: 1024, + temperature: 0.7, + top_p: 0.95, + }, }), - // BYOK: uncomment the following block and set NEXT_OPENAI_API_KEY and NEXT_OPENAI_ + // BYOK: uncomment the following block and set NEXT_LLM_API_KEY and NEXT_LLM_URL // new OpenAI({ - // apiKey: requireEnv('NEXT_OPENAI_API_KEY'), - // url: requireEnv('NEXT_OPENAI_URL'), + // apiKey: requireEnv('NEXT_LLM_API_KEY'), + // url: requireEnv('NEXT_LLM_URL'), // model: 'gpt-4o-mini', // greetingMessage: GREETING, // failureMessage: 'Please wait a moment.', @@ -156,10 +161,10 @@ export async function POST(request: NextRequest) { voiceId: 'English_captivating_female1', }), // BYOK — ElevenLabs (set NEXT_ELEVENLABS_API_KEY; optional NEXT_ELEVENLABS_VOICE_ID) - // new ElevenLabsTTS({ + // new (await import('agora-agent-server-sdk')).ElevenLabsTTS({ // key: requireEnv('NEXT_ELEVENLABS_API_KEY'), // modelId: 'eleven_flash_v2_5', - // voiceId: ELEVENLABS_VOICE_ID || 'pNInz6obpgDQGcFmaJgB', + // voiceId: process.env.NEXT_ELEVENLABS_VOICE_ID ?? 'pNInz6obpgDQGcFmaJgB', // sampleRate: 24000, // }), ); diff --git a/app/api/stop-conversation/route.ts b/app/api/stop-conversation/route.ts index 67c4be9..153b57e 100644 --- a/app/api/stop-conversation/route.ts +++ b/app/api/stop-conversation/route.ts @@ -2,6 +2,26 @@ import { NextResponse } from 'next/server'; import { AgoraClient, Area } from 'agora-agent-server-sdk'; import { StopConversationRequest } from '@/types/conversation'; +function isAgentAlreadyStoppingOrStopped(error: unknown): boolean { + if (!error || typeof error !== 'object') return false; + + const maybeErr = error as { + statusCode?: number; + body?: { detail?: string; reason?: string }; + message?: string; + }; + + const statusCode = maybeErr.statusCode; + const reason = maybeErr.body?.reason?.toLowerCase(); + const detail = maybeErr.body?.detail?.toLowerCase() ?? maybeErr.message?.toLowerCase() ?? ''; + + if (statusCode === 404) return true; + if (reason === 'invalidrequest' && detail.includes('already in the process of shutting down')) { + return true; + } + return false; +} + export async function POST(request: Request) { try { const body: StopConversationRequest = await request.json(); @@ -29,7 +49,15 @@ export async function POST(request: Request) { appId, appCertificate, }); - await client.stopAgent(agent_id); + try { + await client.stopAgent(agent_id); + } catch (error) { + if (isAgentAlreadyStoppingOrStopped(error)) { + // Treat stop as idempotent: agent is already exiting (or gone). + return NextResponse.json({ success: true, state: 'already-stopping' }); + } + throw error; + } return NextResponse.json({ success: true }); } catch (error) { diff --git a/app/globals.css b/app/globals.css index cfc4ee5..e0f11f6 100644 --- a/app/globals.css +++ b/app/globals.css @@ -76,7 +76,7 @@ --viz-stop-3: 285 58% 48%; /* Font */ - --font-sans: 'Instrument Sans', system-ui, sans-serif; + --font-sans: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; } @media (prefers-color-scheme: dark) { @@ -416,3 +416,23 @@ strong { max-height: 3.5rem; } } + +/* Transcript panel sizing: give mobile more breathing room around the top controls. */ +.conversation-transcript .chatbox.expanded { + width: min(24rem, calc(100vw - 2rem)); + min-width: 0; +} + +@media (max-width: 767px) { + .conversation-transcript { + left: 1rem !important; + right: 1rem !important; + bottom: 5.75rem !important; + } + + .conversation-transcript .chatbox.expanded { + width: min(18rem, calc(100vw - 2rem)); + max-width: calc(100vw - 2rem); + max-height: min(20rem, calc(100vh - 14rem)); + } +} diff --git a/app/layout.tsx b/app/layout.tsx index 0cd7e36..ad56330 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -1,14 +1,6 @@ import type { Metadata, Viewport } from 'next'; -import { Instrument_Sans } from 'next/font/google'; import './globals.css'; -const instrumentSans = Instrument_Sans({ - subsets: ['latin'], - weight: ['400', '500', '600', '700'], - variable: '--font-sans', - display: 'swap', -}); - export const viewport: Viewport = { width: 'device-width', initialScale: 1, @@ -47,7 +39,7 @@ export default function RootLayout({ children: React.ReactNode; }>) { return ( - + {children} ); diff --git a/components/ConnectionStatusPanel.tsx b/components/ConnectionStatusPanel.tsx new file mode 100644 index 0000000..25d8a36 --- /dev/null +++ b/components/ConnectionStatusPanel.tsx @@ -0,0 +1,99 @@ +import React from 'react'; +import { ConversationErrorCard, type ConnectionIssue } from './ConversationErrorCard'; + +type ConnectionStatusPanelProps = { + connectionState: string; + connectionSeverity: 'normal' | 'warning' | 'error'; + connectionIssues: ConnectionIssue[]; + isOpen: boolean; + onToggle: () => void; +}; + +function getConnectionLabel( + connectionState: string, + connectionSeverity: 'normal' | 'warning' | 'error' +): string { + if (connectionSeverity !== 'normal' && connectionState === 'CONNECTED') { + return 'Connected (issues detected)'; + } + if (connectionState === 'CONNECTED') return 'Connected'; + if (connectionState === 'CONNECTING') return 'Connecting...'; + if (connectionState === 'RECONNECTING') return 'Reconnecting...'; + if (connectionState === 'DISCONNECTING') return 'Disconnecting...'; + return 'Disconnected'; +} + +export function ConnectionStatusPanel({ + connectionState, + connectionSeverity, + connectionIssues, + isOpen, + onToggle, +}: ConnectionStatusPanelProps) { + return ( +
+ + + {getConnectionLabel(connectionState, connectionSeverity)} + +
+
+
+ Connection Details +
+
+ RTC {connectionState.toLowerCase()} +
+
+ {connectionIssues.length === 0 ? ( +
No RTM or agent errors reported.
+ ) : ( +
+ {connectionIssues.map((issue) => ( + + ))} +
+ )} +
+
+ ); +} diff --git a/components/ConversationComponent.tsx b/components/ConversationComponent.tsx index 60d258b..aa7bd8c 100644 --- a/components/ConversationComponent.tsx +++ b/components/ConversationComponent.tsx @@ -1,14 +1,13 @@ 'use client'; import { useState, useEffect, useCallback, useMemo } from 'react'; +import { X } from 'lucide-react'; import { setParameter } from 'agora-rtc-sdk-ng/esm'; -import type { IAgoraRTCClient } from 'agora-rtc-sdk-ng'; import { useRTCClient, useLocalMicrophoneTrack, useRemoteUsers, useClientEvent, - useIsConnected, useJoin, usePublish, RemoteUser, @@ -18,6 +17,7 @@ import { AgoraVoiceAI, AgoraVoiceAIEvents, AgentState, + MessageSalStatus, TurnStatus, TranscriptHelperMode, type TranscriptHelperItem, @@ -25,13 +25,20 @@ import { type AgentTranscription, } from 'agora-agent-client-toolkit'; import { - AudioVisualizer, + AgentVisualizer, ConvoTextStream, - transcriptToMessageList, + type AgentVisualizerState, + type IMessageListItem, } from 'agora-agent-uikit'; import { MicButtonWithVisualizer } from 'agora-agent-uikit/rtc'; import { Button } from '@/components/ui/button'; +import { DEFAULT_AGENT_UID } from '@/lib/agora'; import { MicrophoneSelector } from './MicrophoneSelector'; +import { + getConversationIssueSeverity, + type ConnectionIssue, +} from './ConversationErrorCard'; +import { ConnectionStatusPanel } from './ConnectionStatusPanel'; import type { ConversationComponentProps } from '@/types/conversation'; function normalizeTranscriptSpacing(text: string): string { @@ -42,12 +49,88 @@ function normalizeTranscriptSpacing(text: string): string { .trim(); } -/** Bar gradient — `--viz-stop-*` swap via `prefers-color-scheme` in globals.css (no JS). */ -const AGENT_AUDIO_VISUALIZER_GRADIENT = [ - 'hsl(var(--viz-stop-1))', - 'hsl(var(--viz-stop-2))', - 'hsl(var(--viz-stop-3))', -]; +const MAX_CONNECTION_ISSUES = 6; + +function normalizeTimestampMs(timestamp: number): number { + // Some payloads are seconds, others are milliseconds. + return timestamp > 1e12 ? timestamp : timestamp * 1000; +} + +type RtmMessageErrorPayload = { + object: 'message.error'; + module?: string; + code?: number; + message?: string; + send_ts?: number; +}; + +type RtmSalStatusPayload = { + object: 'message.sal_status'; + status?: string; + timestamp?: number; +}; + +function isRtmMessageErrorPayload(value: unknown): value is RtmMessageErrorPayload { + return !!value && typeof value === 'object' && (value as { object?: unknown }).object === 'message.error'; +} + +function isRtmSalStatusPayload(value: unknown): value is RtmSalStatusPayload { + return ( + !!value && + typeof value === 'object' && + (value as { object?: unknown }).object === 'message.sal_status' + ); +} + +function mapAgentVisualizerState( + agentState: AgentState | null, + isAgentConnected: boolean, + connectionState: string, +): AgentVisualizerState { + if ( + connectionState === 'DISCONNECTED' || + connectionState === 'DISCONNECTING' + ) { + return 'disconnected'; + } + + if ( + connectionState === 'CONNECTING' || + connectionState === 'RECONNECTING' + ) { + return 'joining'; + } + + if (!isAgentConnected) { + return 'not-joined'; + } + + switch (agentState) { + case 'listening': + return 'listening'; + case 'thinking': + return 'analyzing'; + case 'speaking': + return 'talking'; + case 'idle': + case 'silent': + default: + return 'ambient'; + } +} + +function toMessageListItem( + item: TranscriptHelperItem>, +): IMessageListItem { + return { + turn_id: item.turn_id, + uid: Number(item.uid) || 0, + text: typeof item.text === 'string' ? item.text : '', + status: item.status as unknown as IMessageListItem['status'], + createdAt: + typeof item._time === 'number' ? normalizeTimestampMs(item._time) : undefined, + }; +} export default function ConversationComponent({ agoraData, @@ -56,22 +139,45 @@ export default function ConversationComponent({ onEndConversation, }: ConversationComponentProps) { const client = useRTCClient(); - const isConnected = useIsConnected(); const remoteUsers = useRemoteUsers(); const [isEnabled, setIsEnabled] = useState(true); const [isAgentConnected, setIsAgentConnected] = useState(false); + const [isConnectionDetailsOpen, setIsConnectionDetailsOpen] = useState(false); // Tracks granular RTC connection state for the status dot. // Agora states: DISCONNECTED | CONNECTING | CONNECTED | DISCONNECTING | RECONNECTING const [connectionState, setConnectionState] = useState('CONNECTING'); - const agentUID = process.env.NEXT_PUBLIC_AGENT_UID; + const agentUID = process.env.NEXT_PUBLIC_AGENT_UID ?? String(DEFAULT_AGENT_UID); const [joinedUID, setJoinedUID] = useState(0); - // Transcript + agent state — managed with raw AgoraVoiceAI (see effect below). + // Transcript + agent state — managed with AgoraVoiceAI (see effect below). const [rawTranscript, setRawTranscript] = useState< TranscriptHelperItem>[] >([]); const [agentState, setAgentState] = useState(null); + const [connectionIssues, setConnectionIssues] = useState( + [], + ); + const addConnectionIssue = useCallback((issue: ConnectionIssue) => { + setConnectionIssues((prev) => { + const isDuplicate = prev.some( + (x) => + x.agentUserId === issue.agentUserId && + x.code === issue.code && + x.message === issue.message && + Math.abs(x.timestamp - issue.timestamp) < 1500, + ); + if (isDuplicate) return prev; + return [issue, ...prev].slice(0, MAX_CONNECTION_ISSUES); + }); + }, []); + + // Auto-open details panel as soon as a new issue is recorded. + useEffect(() => { + if (connectionIssues.length > 0) { + setIsConnectionDetailsOpen(true); + } + }, [connectionIssues.length]); // StrictMode guard: delay `useJoin`'s ready flag until after the fake-unmount // cycle completes. React StrictMode fires cleanup synchronously before any @@ -97,7 +203,7 @@ export default function ConversationComponent({ token: agoraData.token, uid: parseInt(agoraData.uid, 10) || 0, }, - isReady + isReady, ); // Create mic track only after the StrictMode fake-unmount cycle completes (isReady). @@ -109,21 +215,12 @@ export default function ConversationComponent({ // graph inside MicButtonWithVisualizer. Mute uses track.setEnabled() only. const { localMicrophoneTrack } = useLocalMicrophoneTrack(isReady); - useEffect(() => { - if (!agentUID) { - console.warn('NEXT_AGENT_UID environment variable is not set'); - } else { - console.log('Agent UID is set to:', agentUID); - } - }, [agentUID]); - // ENABLE_AUDIO_PTS is a module-level SDK parameter (not on the client instance). // It must be set before publishing audio for transcript timing to be accurate. useEffect(() => { if (!client) return; try { setParameter('ENABLE_AUDIO_PTS', true); - console.log('Enabled ENABLE_AUDIO_PTS for timing synchronization'); } catch (error) { console.warn('Could not set ENABLE_AUDIO_PTS:', error); } @@ -133,8 +230,9 @@ export default function ConversationComponent({ useEffect(() => { if (joinSuccess && client) { const uid = client.uid; - setJoinedUID(uid as UID); - console.log('Join successful, using UID:', uid); + if (uid !== null && uid !== undefined) { + setJoinedUID(uid); + } } }, [joinSuccess, client]); @@ -156,7 +254,7 @@ export default function ConversationComponent({ (async () => { try { const ai = await AgoraVoiceAI.init({ - rtcEngine: client as unknown as IAgoraRTCClient, + rtcEngine: client, rtmConfig: { rtmEngine: rtmClient }, renderMode: TranscriptHelperMode.TEXT, enableLog: true, @@ -176,10 +274,47 @@ export default function ConversationComponent({ setRawTranscript([...t]); }); ai.on(AgoraVoiceAIEvents.AGENT_STATE_CHANGED, (_, event) => - setAgentState(event.state) + setAgentState(event.state), + ); + ai.on(AgoraVoiceAIEvents.MESSAGE_ERROR, (agentUserId, error) => { + addConnectionIssue({ + id: `${Date.now()}-${agentUserId}-message-error-${error.code}`, + source: 'rtm', + agentUserId, + code: error.code, + message: error.message, + timestamp: normalizeTimestampMs(error.timestamp), + }); + }); + ai.on( + AgoraVoiceAIEvents.MESSAGE_SAL_STATUS, + (agentUserId, salStatus) => { + if ( + salStatus.status === MessageSalStatus.VP_REGISTER_FAIL || + salStatus.status === MessageSalStatus.VP_REGISTER_DUPLICATE + ) { + addConnectionIssue({ + id: `${Date.now()}-${agentUserId}-sal-${salStatus.status}`, + source: 'rtm', + agentUserId, + code: salStatus.status, + message: `SAL status: ${salStatus.status}`, + timestamp: normalizeTimestampMs(salStatus.timestamp), + }); + } + }, ); + ai.on(AgoraVoiceAIEvents.AGENT_ERROR, (agentUserId, error) => { + addConnectionIssue({ + id: `${Date.now()}-${agentUserId}-agent-error-${error.code}`, + source: 'agent', + agentUserId, + code: error.code, + message: `${error.type}: ${error.message}`, + timestamp: normalizeTimestampMs(error.timestamp), + }); + }); ai.subscribeMessage(agoraData.channel); - console.log('AgoraVoiceAI initialized and subscribed to channel'); } catch (error) { if (!cancelled) { console.error('[AgoraVoiceAI] init failed:', error); @@ -200,6 +335,61 @@ export default function ConversationComponent({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [isReady, joinSuccess]); + // Signaling - capture raw RTM messages so message.error surfaces even if higher-level events don't. + useEffect(() => { + const handleRtmMessage = (event: { + message: string | Uint8Array; + publisher: string; + }) => { + const payloadText = + typeof event.message === 'string' + ? event.message + : new TextDecoder().decode(event.message); + + let parsed: unknown; + try { + parsed = JSON.parse(payloadText); + } catch { + return; + } + + if (isRtmMessageErrorPayload(parsed)) { + const p = parsed; + addConnectionIssue({ + id: `${Date.now()}-${event.publisher}-rtm-msg-error-${p.code ?? 'unknown'}`, + source: 'rtm-signaling', + agentUserId: event.publisher, + code: p.code ?? 'unknown', + message: `${p.module ?? 'unknown'}: ${p.message ?? 'Unknown signaling error'}`, + timestamp: normalizeTimestampMs(p.send_ts ?? Date.now()), + }); + return; + } + + if (isRtmSalStatusPayload(parsed)) { + const p = parsed; + if ( + p.status === 'VP_REGISTER_FAIL' || + p.status === 'VP_REGISTER_DUPLICATE' + ) { + addConnectionIssue({ + id: `${Date.now()}-${event.publisher}-rtm-sal-${p.status}`, + source: 'rtm-signaling', + agentUserId: event.publisher, + code: p.status, + message: `SAL status: ${p.status}`, + timestamp: normalizeTimestampMs(p.timestamp ?? Date.now()), + }); + } + } + }; + + rtmClient.addEventListener('message', handleRtmMessage); + return () => { + rtmClient.removeEventListener('message', handleRtmMessage); + }; + }, [rtmClient, addConnectionIssue]); + // useTranscript() returns uid="0" for local user speech — remap to actual RTC UID // so ConvoTextStream renders user messages on the correct side. // Also normalize punctuation spacing for display when upstream text arrives compacted. @@ -208,7 +398,9 @@ export default function ConversationComponent({ return rawTranscript.map((m) => { const remappedUID = m.uid === '0' ? localUID : m.uid; const normalizedText = - typeof m.text === 'string' ? normalizeTranscriptSpacing(m.text) : m.text; + typeof m.text === 'string' + ? normalizeTranscriptSpacing(m.text) + : m.text; return { ...m, uid: remappedUID, text: normalizedText }; }); }, [rawTranscript, client.uid]); @@ -218,44 +410,73 @@ export default function ConversationComponent({ // messageList stays empty and ConvoTextStream never auto-opens. const messageList = useMemo( () => - transcriptToMessageList( - transcript.filter((m) => m.status !== TurnStatus.IN_PROGRESS) - ), - [transcript] + transcript + .filter((m) => m.status !== TurnStatus.IN_PROGRESS) + .map(toMessageListItem), + [transcript], ); const currentInProgressMessage = useMemo(() => { const m = transcript.find((x) => x.status === TurnStatus.IN_PROGRESS); - return m ? transcriptToMessageList([m])[0] ?? null : null; + return m ? toMessageListItem(m) : null; }, [transcript]); // Publish local mic once the track exists; usePublish waits for RTC connection. usePublish([localMicrophoneTrack]); useClientEvent(client, 'user-joined', (user) => { - console.log('Remote user joined:', user.uid); if (user.uid.toString() === agentUID) setIsAgentConnected(true); }); useClientEvent(client, 'user-left', (user) => { - console.log('Remote user left:', user.uid); if (user.uid.toString() === agentUID) setIsAgentConnected(false); }); // Sync isAgentConnected with remoteUsers (covers cases where user-joined/left are missed) useEffect(() => { const isAgentInRemoteUsers = remoteUsers.some( - (user) => user.uid.toString() === agentUID + (user) => user.uid.toString() === agentUID, ); setIsAgentConnected(isAgentInRemoteUsers); }, [remoteUsers, agentUID]); - useClientEvent(client, 'connection-state-change', (curState, prevState) => { - console.log(`Connection state changed from ${prevState} to ${curState}`); - if (curState === 'DISCONNECTED') console.log('Attempting to reconnect...'); + useClientEvent(client, 'connection-state-change', (curState) => { setConnectionState(curState); }); + const connectionSeverity = useMemo<'normal' | 'warning' | 'error'>(() => { + if ( + connectionState === 'DISCONNECTED' || + connectionState === 'DISCONNECTING' + ) { + return 'error'; + } + if ( + connectionState === 'CONNECTING' || + connectionState === 'RECONNECTING' + ) { + return 'warning'; + } + if (connectionIssues.length === 0) { + return 'normal'; + } + return connectionIssues.some( + (issue) => getConversationIssueSeverity(issue) === 'error', + ) + ? 'error' + : 'warning'; + }, [connectionState, connectionIssues]); + + const visualizerState = useMemo( + () => + mapAgentVisualizerState( + agentState, + isAgentConnected, + connectionState, + ), + [agentState, isAgentConnected, connectionState], + ); + /** * Mute/unmute via track.setEnabled() only — usePublish owns publish state. * If we also unpublish in the toggle, usePublish and the button fight each other @@ -279,10 +500,11 @@ export default function ConversationComponent({ const handleTokenWillExpire = useCallback(async () => { if (!onTokenWillExpire || !joinedUID) return; try { - const newToken = await onTokenWillExpire(joinedUID.toString()); - await client?.renewToken(newToken); - await rtmClient.renewToken(newToken); - console.log('Successfully renewed Agora RTC and RTM tokens'); + const { rtcToken, rtmToken } = await onTokenWillExpire( + joinedUID.toString(), + ); + await client?.renewToken(rtcToken); + await rtmClient.renewToken(rtmToken); } catch (error) { console.error('Failed to renew Agora token:', error); } @@ -290,96 +512,44 @@ export default function ConversationComponent({ useClientEvent(client, 'token-privilege-will-expire', handleTokenWillExpire); - // Debug: log remote UIDs vs NEXT_PUBLIC_AGENT_UID to catch mismatches - useEffect(() => { - if (remoteUsers.length > 0) { - console.log('Remote users detected:', remoteUsers.map((u) => u.uid)); - console.log('Current NEXT_AGENT_UID:', agentUID); - - const potentialAgents = remoteUsers.map((u) => u.uid.toString()); - if (agentUID && !potentialAgents.includes(agentUID)) { - console.warn('Agent UID mismatch! Expected:', agentUID, 'Available users:', potentialAgents); - console.info(`Consider updating NEXT_AGENT_UID to one of: ${potentialAgents.join(', ')}`); - } - } - }, [remoteUsers, agentUID]); - return (
- {/* Top-right: connection status dot + end call */} -
- {/* Connection status dot — color reflects RTC state, tooltip on hover */} -
- - {/* Ping ring — shown while connecting or connected (signals activity) */} - {connectionState !== 'DISCONNECTED' && connectionState !== 'DISCONNECTING' && ( - - )} - - - {/* Tooltip label — visible on hover */} - - {connectionState === 'CONNECTED' ? 'Connected' : - connectionState === 'CONNECTING' ? 'Connecting...' : - connectionState === 'RECONNECTING' ? 'Reconnecting...' : - connectionState === 'DISCONNECTING' ? 'Disconnecting...' : - 'Disconnected'} - -
+
+ setIsConnectionDetailsOpen((open) => !open)} + /> +
+ +
- {/* Remote users (agent audio + RTC subscription). - Framed in a surface card so the visualizer has spatial context. - Fixed h-40 matches AudioVisualizer's height so the layout doesn't - shift when the agent joins or leaves. */} + {/* Remote users keep RTC audio subscribed. The visualizer itself is driven + by RTM agent state so the user sees lifecycle and speaking status in one place. */}
+ {remoteUsers.map((user) => ( -
- +
))} - {remoteUsers.length === 0 && ( -
- Waiting for AI agent to join... -
- )} -
- - {/* Agent state — shown below the visualizer once the agent joins */} -
- {isAgentConnected && agentState ? agentState : null}
{/* Local controls — pill-framed dock at bottom center */} @@ -407,6 +577,7 @@ export default function ConversationComponent({ messageList={messageList} currentInProgressMessage={currentInProgressMessage} agentUID={agentUID} + className="conversation-transcript" />
); diff --git a/components/ConversationErrorCard.tsx b/components/ConversationErrorCard.tsx new file mode 100644 index 0000000..2a78846 --- /dev/null +++ b/components/ConversationErrorCard.tsx @@ -0,0 +1,121 @@ +import React from 'react'; + +export type ConnectionIssue = { + id: string; + source: 'rtm' | 'agent' | 'rtm-signaling'; + agentUserId: string; + code: string | number; + message: string; + timestamp: number; +}; + +export type ConnectionIssueSeverity = 'warning' | 'error'; + +function getEmbeddedIssueCode(issue: ConnectionIssue): string | null { + const msg = issue.message.toLowerCase(); + + const moduleCodeMatch = msg.match(/\b(?:llm|asr|tts)\s*:\s*(\d{3})\b/); + if (moduleCodeMatch?.[1]) return moduleCodeMatch[1]; + + const httpCodeMatch = msg.match(/\bhttp\s*(\d{3})\b/); + if (httpCodeMatch?.[1]) return httpCodeMatch[1]; + + return null; +} + +function getEffectiveIssueCode(issue: ConnectionIssue): string { + return String(getEmbeddedIssueCode(issue) ?? issue.code); +} + +export function getConversationIssueSeverity( + issue: ConnectionIssue +): ConnectionIssueSeverity { + const msg = issue.message.toLowerCase(); + const code = getEffectiveIssueCode(issue).toLowerCase(); + + if ( + code === '429' || + msg.includes('rate limit') || + msg.includes('timeout') || + code === '408' || + code === '409' + ) { + return 'warning'; + } + return 'error'; +} + +function getNormalizedMessage(issue: ConnectionIssue): string { + const lowered = issue.message.toLowerCase(); + const code = getEffectiveIssueCode(issue).toLowerCase(); + + if (code === '401' || lowered.includes('http 401')) { + if (lowered.includes('llm')) return 'LLM authentication failed (401 Unauthorized).'; + if (lowered.includes('asr')) return 'ASR authentication failed (401 Unauthorized).'; + if (lowered.includes('tts')) return 'TTS authentication failed (401 Unauthorized).'; + return 'Authentication failed (401 Unauthorized).'; + } + return issue.message; +} + +function getCta(issue: ConnectionIssue): string | null { + const msg = issue.message.toLowerCase(); + const code = getEffectiveIssueCode(issue).toLowerCase(); + + if (msg.includes('http 401') || code === '401') return 'Verify provider API key.'; + if (msg.includes('http 403') || code === '403') + return 'Verify provider API permissions.'; + if (msg.includes('http 404') || code === '404') + return 'Verify provider endpoint and model ID.'; + if (msg.includes('http 408') || msg.includes('timeout') || code === '408') + return 'Check network stability and retry.'; + if (msg.includes('http 409') || code === '409') + return 'Retry after session state settles.'; + if (msg.includes('http 429') || code === '429' || msg.includes('rate limit')) + return 'Reduce request rate or increase quota.'; + if ( + msg.includes('http 500') || + msg.includes('http 502') || + msg.includes('http 503') || + msg.includes('http 504') + ) { + return 'Retry shortly or switch provider region/model.'; + } + if (msg.includes('websocket') && msg.includes('reject')) { + return 'Verify provider auth, endpoint, and region.'; + } + if (code === 'vp_register_fail' || code === 'vp_register_duplicate') { + return 'Verify RTM configuration and reconnect.'; + } + return null; +} + +type ConversationErrorCardProps = { + issue: ConnectionIssue; +}; + +export function ConversationErrorCard({ issue }: ConversationErrorCardProps) { + const normalizedMessage = getNormalizedMessage(issue); + const showNormalizedMessage = normalizedMessage !== issue.message; + const cta = getCta(issue); + const transportCode = String(issue.code); + const showRaw = issue.message !== normalizedMessage; + + return ( +
+
+ Conversation AI Engine Error: {transportCode} +
+ {showNormalizedMessage &&
{normalizedMessage}
} + {cta &&
{cta}
} + {showRaw && ( +
+ {issue.message} +
+ )} +
+ agent {issue.agentUserId} at {new Date(issue.timestamp).toLocaleTimeString()} +
+
+ ); +} diff --git a/components/LandingPage.tsx b/components/LandingPage.tsx index 1750502..d6d03ac 100644 --- a/components/LandingPage.tsx +++ b/components/LandingPage.tsx @@ -8,6 +8,7 @@ import type { AgoraTokenData, ClientStartRequest, AgentResponse, + AgoraRenewalTokens, } from '../types/conversation'; import { Button } from '@/components/ui/button'; import { ErrorBoundary } from './ErrorBoundary'; @@ -96,11 +97,14 @@ export default function LandingPage() { // 2b. Set up RTM (dynamically imported to keep it client-only) (async () => { const { default: AgoraRTM } = await import('agora-rtm'); - const rtm = new AgoraRTM.RTM(process.env.NEXT_PUBLIC_AGORA_APP_ID!, String(Date.now())); + const rtm: RTMClient = new AgoraRTM.RTM( + process.env.NEXT_PUBLIC_AGORA_APP_ID!, + String(Date.now()), + ); await rtm.login({ token: responseData.token }); await rtm.subscribe(responseData.channel); console.log('RTM ready, channel:', responseData.channel); - return rtm as RTMClient; + return rtm; })(), ]); @@ -116,14 +120,30 @@ export default function LandingPage() { } }; - const handleTokenWillExpire = async (uid: string) => { + const handleTokenWillExpire = async (uid: string): Promise => { try { - const response = await fetch( - `/api/generate-agora-token?channel=${agoraData?.channel}&uid=${uid}` - ); - const data = await response.json(); - if (!response.ok) throw new Error('Failed to generate new token'); - return data.token; + const channel = agoraData?.channel; + if (!channel) { + throw new Error('Missing channel for token renewal'); + } + + const [rtcResponse, rtmResponse] = await Promise.all([ + fetch(`/api/generate-agora-token?channel=${channel}&uid=${uid}`), + fetch(`/api/generate-agora-token?channel=${channel}&uid=0`), + ]); + const [rtcData, rtmData] = await Promise.all([ + rtcResponse.json(), + rtmResponse.json(), + ]); + + if (!rtcResponse.ok || !rtmResponse.ok) { + throw new Error('Failed to generate renewal tokens'); + } + + return { + rtcToken: rtcData.token, + rtmToken: rtmData.token, + }; } catch (error) { console.error('Error renewing token:', error); throw error; @@ -171,7 +191,7 @@ export default function LandingPage() {

- Voice AI Demo + Voice AI Quickstart

{!showConversation && ( diff --git a/env.local.example b/env.local.example index 34b7ed2..f827489 100644 --- a/env.local.example +++ b/env.local.example @@ -1,9 +1,9 @@ # ─── Agora ──────────────────────────────────────────────────────────────────── -# App ID: found in Agora Console → your project → "App ID" +# Required: found in Agora Console → your project → "App ID" NEXT_PUBLIC_AGORA_APP_ID= -# App Certificate: found in Agora Console → your project → "App Certificate" -# Server-side only — never expose this in client code +# Required: found in Agora Console → your project → "App Certificate" +# Server-side only. Never expose this in client code. NEXT_AGORA_APP_CERTIFICATE= # UID the AI agent joins the channel with. @@ -11,21 +11,18 @@ NEXT_AGORA_APP_CERTIFICATE= # Default: 123456 NEXT_PUBLIC_AGENT_UID=123456 -# ─── Conversational AI agent (invite-agent) ─────────────────────────────────── -# Default pipeline: Deepgram nova-3 (STT), OpenAI gpt-4o-mini (LLM), MiniMax speech_2_6_turbo (TTS) -# without vendor API keys — the server SDK infers reseller presets. Requires Agora project/billing support. - -# Optional BYOK (commented blocks in app/api/invite-agent/route.ts — swap preset vs BYOK there): +# Optional BYOK examples: +# Uncomment the matching blocks in app/api/invite-agent/route.ts if you want to bring +# your own vendor credentials instead of using the Agora-managed defaults. # NEXT_DEEPGRAM_API_KEY= -# NEXT_OPENAI_API_KEY= +# NEXT_LLM_URL=https://api.openai.com/v1/chat/completions +# NEXT_LLM_API_KEY= # NEXT_ELEVENLABS_API_KEY= # NEXT_ELEVENLABS_VOICE_ID= # ─── Optional: custom LLM proxy ─────────────────────────────────────────────── -# Only required if you use app/api/chat/completions (e.g. point the agent at your URL). -# NEXT_LLM_URL=https://api.openai.com/v1/chat/completions -# NEXT_LLM_API_KEY= +# Only needed if you use app/api/chat/completions as a custom OpenAI-compatible proxy. # ─── Agent behavior ─────────────────────────────────────────────────────────── -# Opening line the agent speaks when a user joins (optional) +# Optional opening line override # NEXT_AGENT_GREETING=Hi there! I'm Ada, your virtual assistant from Agora. How can I help? diff --git a/lib/agora.ts b/lib/agora.ts new file mode 100644 index 0000000..34c7e6e --- /dev/null +++ b/lib/agora.ts @@ -0,0 +1 @@ +export const DEFAULT_AGENT_UID = 12345; diff --git a/next.config.mjs b/next.config.mjs index 4ce2351..06bc568 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -1,8 +1,16 @@ +import { dirname } from 'node:path' +import { fileURLToPath } from 'node:url' + +const rootDir = dirname(fileURLToPath(import.meta.url)) + /** @type {import('next').NextConfig} */ const nextConfig = { images: { unoptimized: true, }, + turbopack: { + root: rootDir, + }, experimental: { webpackBuildWorker: true, parallelServerBuildTraces: true, diff --git a/package.json b/package.json index 6898291..83f87b1 100644 --- a/package.json +++ b/package.json @@ -4,10 +4,10 @@ "description": "Next.js quickstart for building conversational AI applications with Agora's Conversational AI Engine", "private": true, "engines": { - "node": ">=24" + "node": ">=22" }, "scripts": { - "dev": "next dev", + "dev": "next dev --webpack", "build": "next build", "start": "next start", "lint": "eslint .", @@ -43,10 +43,10 @@ "@radix-ui/react-toggle": "^1.1.1", "@radix-ui/react-toggle-group": "^1.1.1", "@radix-ui/react-tooltip": "^1.1.6", - "agora-agent-client-toolkit": "^1.1.0", - "agora-agent-client-toolkit-react": "^1.1.0", + "agora-agent-client-toolkit": "1.2.0", + "agora-agent-client-toolkit-react": "1.2.0", "agora-agent-server-sdk": "^1.3.1", - "agora-agent-uikit": "^1.0.0", + "agora-agent-uikit": "1.1.0", "agora-rtc-react": "^2.5.1", "agora-rtc-sdk-ng": "^4.24.3", "agora-rtm": "^2.2.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 94bca8e..cb8ec74 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -96,17 +96,17 @@ importers: specifier: ^1.1.6 version: 1.2.8(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) agora-agent-client-toolkit: - specifier: ^1.1.0 - version: 1.1.0(agora-rtc-sdk-ng@4.24.3)(agora-rtm@2.2.3(agora-rtc-sdk-ng@4.24.3)) + specifier: 1.2.0 + version: 1.2.0(agora-rtc-sdk-ng@4.24.3)(agora-rtm@2.2.3(agora-rtc-sdk-ng@4.24.3)) agora-agent-client-toolkit-react: - specifier: ^1.1.0 - version: 1.1.0(agora-agent-client-toolkit@1.1.0(agora-rtc-sdk-ng@4.24.3)(agora-rtm@2.2.3(agora-rtc-sdk-ng@4.24.3)))(agora-rtc-react@2.5.1(react@19.2.4))(react@19.2.4) + specifier: 1.2.0 + version: 1.2.0(agora-agent-client-toolkit@1.2.0(agora-rtc-sdk-ng@4.24.3)(agora-rtm@2.2.3(agora-rtc-sdk-ng@4.24.3)))(agora-rtc-react@2.5.1(react@19.2.4))(react@19.2.4) agora-agent-server-sdk: specifier: ^1.3.1 version: 1.3.1 agora-agent-uikit: - specifier: ^1.0.0 - version: 1.0.0(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(agora-agent-client-toolkit-react@1.1.0(agora-agent-client-toolkit@1.1.0(agora-rtc-sdk-ng@4.24.3)(agora-rtm@2.2.3(agora-rtc-sdk-ng@4.24.3)))(agora-rtc-react@2.5.1(react@19.2.4))(react@19.2.4))(agora-agent-client-toolkit@1.1.0(agora-rtc-sdk-ng@4.24.3)(agora-rtm@2.2.3(agora-rtc-sdk-ng@4.24.3)))(agora-rtc-react@2.5.1(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + specifier: 1.1.0 + version: 1.1.0(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(agora-agent-client-toolkit-react@1.2.0(agora-agent-client-toolkit@1.2.0(agora-rtc-sdk-ng@4.24.3)(agora-rtm@2.2.3(agora-rtc-sdk-ng@4.24.3)))(agora-rtc-react@2.5.1(react@19.2.4))(react@19.2.4))(agora-agent-client-toolkit@1.2.0(agora-rtc-sdk-ng@4.24.3)(agora-rtm@2.2.3(agora-rtc-sdk-ng@4.24.3)))(agora-rtc-react@2.5.1(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4) agora-rtc-react: specifier: ^2.5.1 version: 2.5.1(react@19.2.4) @@ -1586,17 +1586,17 @@ packages: engines: {node: '>=0.4.0'} hasBin: true - agora-agent-client-toolkit-react@1.1.0: - resolution: {integrity: sha512-OgfpSSnpROnwbwtvyxtSbh3t/2V5+G8jRixzbfJbabU2oFSdsYdRt1ANk7GJTuQphW9Ji4PqBApcT67GMelv9w==} - engines: {node: '>=18.0.0'} + agora-agent-client-toolkit-react@1.2.0: + resolution: {integrity: sha512-yZEHzFY2L3TCrMAxmprNwxrUaBvdRxI+355rc+e/+5BIdwqkCaqtHUv4kSSoTAyXhnM3GhZLapas+KL1IKlrrw==} + engines: {node: '>=20.0.0'} peerDependencies: agora-agent-client-toolkit: workspace:* agora-rtc-react: '>=2.0.0' react: '>=18.0.0' - agora-agent-client-toolkit@1.1.0: - resolution: {integrity: sha512-BwPv1GU7ntgeI6KGy/szO6vZ7+Yw3CEnxj2gw9eUlcx7vd6d0j+on+BuufyKOOuA2AJ2DrSp93X1+EOIEN9QyQ==} - engines: {node: '>=18.0.0'} + agora-agent-client-toolkit@1.2.0: + resolution: {integrity: sha512-rbSDfk6veGsvNxSgccPa1zCCJ7QQw0NjoETHiHSXGURkzw90KwOQ2ZIOX6dueMFvZLUBr1UPQodCpOHroHY5bw==} + engines: {node: '>=20.0.0'} peerDependencies: agora-rtc-sdk-ng: '>=4.23.4' agora-rtm: '>=2.0.0' @@ -1608,11 +1608,12 @@ packages: resolution: {integrity: sha512-DBkRg5vx9d65m86j7b4Jjqrxygnq9/KMAJHxlYXe4GJG4B6W8okNwFKBHd6Tj4P8chb07IQ9rz9fgIwsNDsrbA==} engines: {node: '>=18.0.0'} - agora-agent-uikit@1.0.0: - resolution: {integrity: sha512-ASoboLqUj4a0y9L1Mc8G9/5/QML1xKMsbPa3aQhD8uFHT+hTT4fZVlM8A4d4sxXEC4yXTrzvQqwEO0wE5Gd75A==} + agora-agent-uikit@1.1.0: + resolution: {integrity: sha512-gDSleHFYavNEqf/SIUt8xA1Q+VAw4M9q/84MD/+fFG7Z15oSLH6hZnmI+8cr6yzPXBBXHX4dd9tRZod5UjaFyA==} + engines: {node: '>=20.0.0'} peerDependencies: - agora-agent-client-toolkit: ^1.1.0 - agora-agent-client-toolkit-react: ^1.1.0 + agora-agent-client-toolkit: ^1.2.0 + agora-agent-client-toolkit-react: ^1.2.0 agora-rtc-react: '>=2.0.0' agora-rtm-sdk: '>=2.0.0' react: '>=18.0.0' @@ -4852,13 +4853,13 @@ snapshots: acorn@8.16.0: {} - agora-agent-client-toolkit-react@1.1.0(agora-agent-client-toolkit@1.1.0(agora-rtc-sdk-ng@4.24.3)(agora-rtm@2.2.3(agora-rtc-sdk-ng@4.24.3)))(agora-rtc-react@2.5.1(react@19.2.4))(react@19.2.4): + agora-agent-client-toolkit-react@1.2.0(agora-agent-client-toolkit@1.2.0(agora-rtc-sdk-ng@4.24.3)(agora-rtm@2.2.3(agora-rtc-sdk-ng@4.24.3)))(agora-rtc-react@2.5.1(react@19.2.4))(react@19.2.4): dependencies: - agora-agent-client-toolkit: 1.1.0(agora-rtc-sdk-ng@4.24.3)(agora-rtm@2.2.3(agora-rtc-sdk-ng@4.24.3)) + agora-agent-client-toolkit: 1.2.0(agora-rtc-sdk-ng@4.24.3)(agora-rtm@2.2.3(agora-rtc-sdk-ng@4.24.3)) agora-rtc-react: 2.5.1(react@19.2.4) react: 19.2.4 - agora-agent-client-toolkit@1.1.0(agora-rtc-sdk-ng@4.24.3)(agora-rtm@2.2.3(agora-rtc-sdk-ng@4.24.3)): + agora-agent-client-toolkit@1.2.0(agora-rtc-sdk-ng@4.24.3)(agora-rtm@2.2.3(agora-rtc-sdk-ng@4.24.3)): dependencies: agora-rtc-sdk-ng: 4.24.3 optionalDependencies: @@ -4872,7 +4873,7 @@ snapshots: dependencies: agora-token: 2.0.5 - agora-agent-uikit@1.0.0(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(agora-agent-client-toolkit-react@1.1.0(agora-agent-client-toolkit@1.1.0(agora-rtc-sdk-ng@4.24.3)(agora-rtm@2.2.3(agora-rtc-sdk-ng@4.24.3)))(agora-rtc-react@2.5.1(react@19.2.4))(react@19.2.4))(agora-agent-client-toolkit@1.1.0(agora-rtc-sdk-ng@4.24.3)(agora-rtm@2.2.3(agora-rtc-sdk-ng@4.24.3)))(agora-rtc-react@2.5.1(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4): + agora-agent-uikit@1.1.0(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(agora-agent-client-toolkit-react@1.2.0(agora-agent-client-toolkit@1.2.0(agora-rtc-sdk-ng@4.24.3)(agora-rtm@2.2.3(agora-rtc-sdk-ng@4.24.3)))(agora-rtc-react@2.5.1(react@19.2.4))(react@19.2.4))(agora-agent-client-toolkit@1.2.0(agora-rtc-sdk-ng@4.24.3)(agora-rtm@2.2.3(agora-rtc-sdk-ng@4.24.3)))(agora-rtc-react@2.5.1(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4): dependencies: '@lottiefiles/dotlottie-react': 0.17.15(react@19.2.4) '@radix-ui/react-dialog': 1.1.15(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -4890,8 +4891,8 @@ snapshots: react-dom: 19.2.4(react@19.2.4) tailwind-merge: 3.5.0 optionalDependencies: - agora-agent-client-toolkit: 1.1.0(agora-rtc-sdk-ng@4.24.3)(agora-rtm@2.2.3(agora-rtc-sdk-ng@4.24.3)) - agora-agent-client-toolkit-react: 1.1.0(agora-agent-client-toolkit@1.1.0(agora-rtc-sdk-ng@4.24.3)(agora-rtm@2.2.3(agora-rtc-sdk-ng@4.24.3)))(agora-rtc-react@2.5.1(react@19.2.4))(react@19.2.4) + agora-agent-client-toolkit: 1.2.0(agora-rtc-sdk-ng@4.24.3)(agora-rtm@2.2.3(agora-rtc-sdk-ng@4.24.3)) + agora-agent-client-toolkit-react: 1.2.0(agora-agent-client-toolkit@1.2.0(agora-rtc-sdk-ng@4.24.3)(agora-rtm@2.2.3(agora-rtc-sdk-ng@4.24.3)))(agora-rtc-react@2.5.1(react@19.2.4))(react@19.2.4) agora-rtc-react: 2.5.1(react@19.2.4) transitivePeerDependencies: - '@types/react' diff --git a/tsconfig.json b/tsconfig.json index 584d219..4719808 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -25,6 +25,12 @@ "paths": { "@/*": [ "./*" + ], + "agora-agent-uikit": [ + "./node_modules/agora-agent-uikit/dist/index.d.ts" + ], + "agora-agent-uikit/rtc": [ + "./node_modules/agora-agent-uikit/dist/rtc/index.d.ts" ] } }, diff --git a/types/conversation.ts b/types/conversation.ts index 673e388..5b752bf 100644 --- a/types/conversation.ts +++ b/types/conversation.ts @@ -21,11 +21,16 @@ export interface AgentResponse { state: string; } +export interface AgoraRenewalTokens { + rtcToken: string; + rtmToken: string; +} + import type { RTMClient } from 'agora-rtm'; export interface ConversationComponentProps { agoraData: AgoraTokenData; rtmClient: RTMClient; - onTokenWillExpire: (uid: string) => Promise; + onTokenWillExpire: (uid: string) => Promise; onEndConversation: () => void; }