diff --git a/.cursor/rules/file-structure.mdc b/.cursor/rules/file-structure.mdc
new file mode 100644
index 00000000..f526318e
--- /dev/null
+++ b/.cursor/rules/file-structure.mdc
@@ -0,0 +1,12 @@
+---
+description: When creating files
+alwaysApply: false
+---
+
+We want thin files that aim to have a single responsibility and a single export.
+We should have a single export per file.
+
+Specific directories:
+- hooks: for hooks
+- components: for components
+- pages: for pages
\ No newline at end of file
diff --git a/.cursor/rules/react-tsx.mdc b/.cursor/rules/react-tsx.mdc
new file mode 100644
index 00000000..eeb9f669
--- /dev/null
+++ b/.cursor/rules/react-tsx.mdc
@@ -0,0 +1,84 @@
+---
+description: React and TSX component development guidelines
+globs: *.tsx
+alwaysApply: false
+---
+
+# Components
+
+- One component per file.
+- Create smaller, focused components.
+- Compose larger components from smaller ones.
+- Logical groupings of components should form a new custom component and moved to a separate file.
+- When prototyping, keep small components in the same file, but mark to be removed:
+
+```tsx
+/**
+ * ExampleComponent
+ * Responsibilities:
+ * To be an example component
+ *
+ * TODO: Move to a separate file
+ */
+function ExampleComponent() {
+ return
ExampleComponent
;
+}
+```
+
+# Hooks
+
+- Complex hooks should be broken down into smaller, composable hooks.
+- If a hook can be self-contained, it should be self-contained and should be in a separate file.
+- Break complex logic into smaller hooks.
+- Compose hooks when it makes sense.
+- Keep hooks in the same file as their components.
+
+# Code Separation
+
+- Separate business logic from UI rendering.
+- Keep related code together.
+- Maintain clear boundaries between concerns.
+
+# Composable Component Pattern
+
+When creating shared templates or reusable components:
+
+1. **Use compound component pattern** - Attach subcomponents as static properties (e.g., `Template.Header`, `Template.Content`) instead of prop drilling.
+
+2. **Templates are pure UI** - Templates should only handle layout/structure. Pages control conditional logic, data fetching, and business logic.
+
+3. **Subcomponents take their own props** - Each subcomponent (Header, Content, Footer, etc.) should accept its own typed props rather than the parent managing them.
+
+4. **Compose via children** - Use `children` for flexible composition. Pages decide what to render in each slot.
+
+**Example:**
+```tsx
+// Template
+const EditDraftTemplate = ({ children, isLoading }) => {
+ if (isLoading) return ;
+ return {children};
+};
+
+EditDraftTemplate.Header = ({ title, children }) => (
+ {title || children}
+);
+
+EditDraftTemplate.Content = ({ content, isStreaming }) => (
+
{content}
+);
+
+// Usage
+
+
+
+
+```
+
+**Avoid:** Prop drilling, templates with business logic, tightly coupled props.
+
+# Class Organization
+
+- Place public methods first.
+- Place private methods at the bottom of the class.
+- Group related methods together.
+- Use clear method names that describe their purpose.
diff --git a/.cursor/rules/typescript.mdc b/.cursor/rules/typescript.mdc
new file mode 100644
index 00000000..645d6bb3
--- /dev/null
+++ b/.cursor/rules/typescript.mdc
@@ -0,0 +1,22 @@
+---
+description:
+globs: *.tsx
+alwaysApply: false
+---
+
+## Typescript Development Guidelines:
+
+### Documentation
+ - Add TSDoc style documentation when writing functions, methods and components.
+
+### Single Responsibility
+ - SRP is a core principle of our codebase and should be followed in all functions, methods and components.
+ => if the function, method or component is more than one responsibility, suggest how to break it down into smaller functions, methods or components. Encourage the user to use the "Single Responsibility" principle
+ and explain the benefits of doing so in this case.
+
+### Imports
+ - Always use named imports for React components and hooks.
+ - Never use non-destructured imports, such as `React.useMemo` or `React.useCallback`.
+
+### Type Safety
+ - Never use the `any` type.
diff --git a/javascript/examples/vitest/.gitignore b/javascript/examples/vitest/.gitignore
index 6eb32454..e8066c09 100644
--- a/javascript/examples/vitest/.gitignore
+++ b/javascript/examples/vitest/.gitignore
@@ -1 +1,4 @@
.scenario
+
+# Audio files
+test-audio-output/
diff --git a/javascript/examples/vitest/package.json b/javascript/examples/vitest/package.json
index 962c28be..2c88bb61 100644
--- a/javascript/examples/vitest/package.json
+++ b/javascript/examples/vitest/package.json
@@ -7,12 +7,14 @@
"scripts": {
"test": "pnpm exec vitest",
"typecheck": "tsc --noEmit",
- "lint": "eslint ."
+ "lint": "eslint .",
+ "realtime-client": "pnpm -F realtime-client"
},
"keywords": [],
"author": "",
"license": "ISC",
"devDependencies": {
+ "concurrently": "^9.1.2",
"dotenv": "^16.5.0",
"dotenv-cli": "^8.0.0",
"nanoid-cli": "^1.0.1",
@@ -20,8 +22,12 @@
},
"dependencies": {
"@langwatch/scenario": "workspace:*",
+ "@openai/agents": "^0.3.0",
"ai": ">=5.0.0",
+ "express": "^4.21.2",
"openai": "^5.16.0",
- "vitest": "^3.2.4"
+ "vite": "^6.0.11",
+ "vitest": "^3.2.4",
+ "zod": "^3.24.1"
}
}
diff --git a/javascript/examples/vitest/tests/helpers/convert-core-messages-to-openai.ts b/javascript/examples/vitest/tests/helpers/convert-core-messages-to-openai.ts
index 491feee7..b59f344e 100644
--- a/javascript/examples/vitest/tests/helpers/convert-core-messages-to-openai.ts
+++ b/javascript/examples/vitest/tests/helpers/convert-core-messages-to-openai.ts
@@ -11,7 +11,7 @@ import {
/**
* OpenAI supported audio formats for input_audio
*/
-type OpenAIAudioFormat = "wav" | "mp3";
+type OpenAIAudioFormat = "wav" | "mp3" | "pcm16";
/**
* Comprehensive mapping from MIME types to OpenAI audio formats
@@ -26,6 +26,7 @@ const MIME_TYPE_TO_OPENAI_FORMAT: Record = {
"audio/mp3": "mp3",
"audio/mpeg3": "mp3",
"audio/x-mpeg-3": "mp3",
+ "audio/pcm16": "pcm16",
} as const;
/**
diff --git a/javascript/examples/vitest/tests/realtime/ARCHITECTURE.md b/javascript/examples/vitest/tests/realtime/ARCHITECTURE.md
new file mode 100644
index 00000000..b4cb54a0
--- /dev/null
+++ b/javascript/examples/vitest/tests/realtime/ARCHITECTURE.md
@@ -0,0 +1,315 @@
+# Architecture: Same Agent Testing
+
+## ๐ฏ Core Principle
+
+**The agent tested by Scenario is EXACTLY the agent used by the browser.**
+
+This is achieved through a **shared configuration module** that both browser and tests import.
+
+---
+
+## ๐ File Structure
+
+```
+tests/realtime/
+โโโ shared/
+โ โโโ vegetarian-recipe-agent.ts # โ SINGLE SOURCE OF TRUTH
+โ - Agent instructions
+โ - Voice settings
+โ - Model configuration
+โ - createVegetarianRecipeAgent()
+โ
+โโโ client/
+โ โโโ demo.html # Browser client
+โ import { createVegetarianRecipeAgent } from '../shared/...'
+โ
+โโโ helpers/
+โ โโโ realtime-agent-adapter.ts # Scenario adapter
+โ โโโ index.ts
+โ
+โโโ server/
+โ โโโ ephemeral-token-server.ts # Token generation
+โ โโโ start-server.ts
+โ
+โโโ vegetarian-recipe-realtime.test.ts # Tests
+ import { createVegetarianRecipeAgent } from './realtime/shared/...'
+```
+
+---
+
+## ๐ How It Works
+
+### 1. Define Agent Once
+
+```typescript
+// shared/vegetarian-recipe-agent.ts
+export const AGENT_INSTRUCTIONS = `You are a friendly vegetarian recipe assistant...`;
+
+export const AGENT_CONFIG = {
+ name: "Vegetarian Recipe Assistant",
+ instructions: AGENT_INSTRUCTIONS,
+ voice: "alloy",
+ model: "gpt-4o-realtime-preview-2024-12-17",
+};
+
+export function createVegetarianRecipeAgent(): RealtimeAgent {
+ return new RealtimeAgent({
+ name: AGENT_CONFIG.name,
+ instructions: AGENT_CONFIG.instructions,
+ voice: AGENT_CONFIG.voice,
+ });
+}
+```
+
+### 2. Browser Uses It
+
+```typescript
+// client/demo.html
+import { createVegetarianRecipeAgent, AGENT_CONFIG } from '../shared/vegetarian-recipe-agent.js';
+
+const agent = createVegetarianRecipeAgent();
+const session = new RealtimeSession(agent, {
+ model: AGENT_CONFIG.model
+});
+```
+
+### 3. Tests Use It
+
+```typescript
+// vegetarian-recipe-realtime.test.ts
+import { createVegetarianRecipeAgent } from './realtime/shared/vegetarian-recipe-agent.js';
+
+const agent = createVegetarianRecipeAgent();
+const adapter = new RealtimeAgentAdapter({ agent });
+
+await scenario.run({
+ agents: [adapter, scenario.userSimulatorAgent()],
+ // ...
+});
+```
+
+---
+
+## โ Benefits
+
+### 1. Accurate Testing
+- Tests validate the **actual production agent**
+- No test doubles or mocks
+- Catch real issues before deployment
+
+### 2. Single Source of Truth
+- Change agent once, updates everywhere
+- No drift between test and production
+- Easier maintenance
+
+### 3. Confidence
+- If tests pass, browser works
+- No "works in tests but fails in production"
+- True integration testing
+
+---
+
+## ๐ Connection Flow
+
+### Browser Flow
+
+```
+1. User clicks "Connect"
+ โ
+2. Fetch ephemeral token from server
+ POST http://localhost:3000/token
+ โ
+3. Create agent (shared config)
+ const agent = createVegetarianRecipeAgent()
+ โ
+4. Create RealtimeSession
+ const session = new RealtimeSession(agent)
+ โ
+5. Connect with token
+ await session.connect({ apiKey: token })
+ โ
+6. User speaks โ microphone โ WebRTC โ OpenAI โ audio response
+```
+
+### Test Flow
+
+```
+1. beforeAll: Connect adapter
+ โ
+2. Fetch ephemeral token from server
+ POST http://localhost:3000/token
+ โ
+3. Create agent (SAME shared config!)
+ const agent = createVegetarianRecipeAgent()
+ โ
+4. Wrap in RealtimeAgentAdapter
+ const adapter = new RealtimeAgentAdapter({ agent })
+ โ
+5. Connect adapter
+ await adapter.connect()
+ โ
+6. Run scenario
+ - User simulator sends text
+ - Adapter forwards to session
+ - Gets transcript back
+ - Returns to Scenario framework
+ โ
+7. afterAll: Disconnect
+ await adapter.disconnect()
+```
+
+---
+
+## ๐งช Testing Strategy
+
+### Text Input (Fast, CI-Friendly)
+
+```typescript
+await scenario.run({
+ agents: [
+ realtimeAdapter, // Real Realtime agent
+ scenario.userSimulatorAgent(), // Text user simulator
+ scenario.judgeAgent(), // Evaluates transcripts
+ ],
+ script: [
+ scenario.user("quick recipe"), // Text โ Realtime API
+ scenario.agent(), // Audio response โ transcript
+ scenario.judge(),
+ ],
+});
+```
+
+**Pros**: Fast, no audio processing overhead
+**Tests**: Agent logic, instructions, conversation flow
+
+### Audio Input (Realistic, Comprehensive)
+
+```typescript
+// Future enhancement: AudioUserSimulatorAgent
+// Uses gpt-4o-audio-preview to generate native audio
+await scenario.run({
+ agents: [
+ realtimeAdapter,
+ audioUserSimulator, // Generates audio
+ audioJudgeAgent, // Transcribes for judgment
+ ],
+ // ...
+});
+```
+
+**Pros**: Tests full voice pipeline
+**Tests**: Audio quality, prosody, interruptions
+
+---
+
+## ๐จ Customization
+
+Want to change the agent? Update ONE file:
+
+```typescript
+// shared/vegetarian-recipe-agent.ts
+
+export const AGENT_INSTRUCTIONS = `
+ NEW INSTRUCTIONS HERE
+`;
+
+// That's it! Browser and tests automatically use new instructions.
+```
+
+---
+
+## ๐ Security
+
+**Ephemeral Tokens** prevent API key exposure:
+
+1. **Browser** cannot see your OpenAI API key
+2. **Server** generates short-lived tokens (`ek_...`)
+3. **Token** expires in ~60 seconds
+4. **OpenAI** validates token, not API key
+
+Same pattern for both browser and tests!
+
+---
+
+## ๐ Comparison to Other Approaches
+
+### โ Bad: Separate Agent Definitions
+
+```typescript
+// client.ts
+const agent = new RealtimeAgent({
+ instructions: "Help with recipes...",
+});
+
+// test.ts
+const mockAgent = new MockAgent({
+ instructions: "Help with recipes...", // Might drift!
+});
+```
+
+**Problem**: Test and production can drift apart.
+
+### โ Bad: Mock Agent
+
+```typescript
+// test.ts
+const mockAgent = {
+ call: async () => "Mocked recipe response"
+};
+```
+
+**Problem**: Tests don't validate real agent behavior.
+
+### โ Good: Shared Configuration (This Approach)
+
+```typescript
+// shared/agent.ts
+export function createAgent() { ... }
+
+// client.ts
+const agent = createAgent();
+
+// test.ts
+const agent = createAgent(); // SAME!
+```
+
+**Solution**: One source of truth, accurate testing.
+
+---
+
+## ๐ Deployment
+
+### Development
+```bash
+# Terminal 1: Start token server
+pnpm realtime-server
+
+# Terminal 2: Run tests
+pnpm test vegetarian-recipe-realtime
+
+# Terminal 3: Open browser
+open http://localhost:3000/demo.html
+```
+
+### Production
+
+1. **Deploy token server** (Vercel/Railway/AWS)
+2. **Tests keep working** (point to production server)
+3. **Agent stays identical** (shared config)
+
+```typescript
+// Just change the URL
+const adapter = new RealtimeAgentAdapter({
+ agent: createVegetarianRecipeAgent(),
+ tokenServerUrl: "https://your-server.com", // Production!
+});
+```
+
+---
+
+## ๐ Further Reading
+
+- [OpenAI Realtime API](https://platform.openai.com/docs/guides/realtime)
+- [OpenAI Agents SDK](https://openai.github.io/openai-agents-js/)
+- [Scenario Testing Framework](https://github.com/langwatch/scenario)
+
diff --git a/javascript/examples/vitest/tests/realtime/README.md b/javascript/examples/vitest/tests/realtime/README.md
new file mode 100644
index 00000000..42661b6d
--- /dev/null
+++ b/javascript/examples/vitest/tests/realtime/README.md
@@ -0,0 +1,220 @@
+# Realtime Voice Agent
+
+This example demonstrates how to create and test a **voice-enabled AI agent** using OpenAI's Realtime API, with **one source of truth** for the agent configuration.
+
+## ๐ฏ What's Included
+
+- **Shared Agent Config** (`shared/vegetarian-recipe-agent.ts`) - **TypeScript single source of truth**
+- **Vite Browser Client** (`client/`) - TypeScript, hot reload, modern dev experience
+- **Scenario Test** (`vegetarian-recipe-realtime.test.ts`) - Uses shared TypeScript config
+- **Ephemeral Token Server** (`server/`) - Securely generate client tokens
+
+## โ Key Principle: Same Agent, Accurate Testing
+
+```typescript
+// shared/vegetarian-recipe-agent.ts - ONE source of truth (TypeScript!)
+export function createVegetarianRecipeAgent() { ... }
+
+// client/src/main.ts - Browser uses it (via Vite)
+import { createVegetarianRecipeAgent } from '../../shared/vegetarian-recipe-agent';
+const agent = createVegetarianRecipeAgent();
+
+// vegetarian-recipe-realtime.test.ts - Tests use it (via Vitest)
+import { createVegetarianRecipeAgent } from './realtime/shared/vegetarian-recipe-agent';
+const agent = createVegetarianRecipeAgent();
+
+// โ SAME TypeScript code = accurate testing!
+```
+
+## ๐จ Why Vite?
+
+- โ TypeScript works natively (no build step during dev)
+- โ Hot module replacement (instant updates)
+- โ Proper module resolution
+- โ Same imports in browser and tests
+- โ Modern, fast, standard
+
+## ๐ Quick Start
+
+### 1. Install Dependencies
+
+```bash
+cd javascript/examples/vitest
+pnpm install
+```
+
+### 2. Set Your OpenAI API Key
+
+Create a `.env` file:
+
+```bash
+echo "OPENAI_API_KEY=sk-proj-..." > .env
+```
+
+### 3. Start Everything (ONE COMMAND!)
+
+```bash
+pnpm realtime
+```
+
+This starts:
+- ๐ต Token server on port 3000
+- ๐ฃ Vite client on port 5173 (auto-opens browser)
+
+Click "Connect" and start talking!
+
+### 4. Run the Tests (Optional, separate terminal)
+
+```bash
+pnpm test vegetarian-recipe-realtime
+```
+
+Tests the **EXACT same TypeScript agent** that the browser uses!
+
+## ๐ฃ๏ธ Try These Prompts
+
+- "What's a quick pasta recipe?"
+- "Give me ideas for a healthy lunch"
+- "How do I make a vegetable stir-fry?"
+- "I need a recipe using chickpeas"
+
+## ๐๏ธ Architecture
+
+```
+ โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
+ โ Shared Agent Config (TS!) โ โ SINGLE SOURCE OF TRUTH
+ โ vegetarian-recipe-agent.ts โ
+ โโโโโโโโฌโโโโโโโโโโโฌโโโโโโโโโโโโโ
+ โ โ
+ โโโโโโโโโโโโโโโโ โโโโโโโโโโโโโโโโโ
+ โ โ
+ โ โ
+โโโโโโโโโโโโโโโโโโโโโโโ โโโโโโโโโโโโโโโโโโโโ
+โ Browser Client โ โ Scenario Test โ
+โ (Vite + TS) โ โ (Vitest + TS) โ
+โ main.ts โ โ .test.ts โ
+โโโโโโโโโโโฌโโโโโโโโโโโโ โโโโโโโโโโฌโโโโโโโโโโ
+ โ โ
+ โ WebRTC โ WebRTC
+ โ โ
+ โโโโโโโโโโโโฌโโโโโโโโโโโฌโโโโโโโโโโโโโโโโโโ
+ โ โ
+ โ โ
+ โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
+ โ Ephemeral Token Server โ
+ โ (Express on :3000) โ
+ โโโโโโโโโโโโโโโโฌโโโโโโโโโโโโโโโโ
+ โ
+ โ Uses your API key
+ โ
+ โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
+ โ OpenAI Realtime API โ
+ โ (voice processing server) โ
+ โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
+```
+
+**Key: TypeScript everywhere! Browser (via Vite) and tests (via Vitest) use IDENTICAL agent.**
+
+## ๐ How It Works
+
+### Ephemeral Tokens
+
+The browser cannot directly use your OpenAI API key (security risk!). Instead:
+
+1. **Browser** requests a token from **your server**
+2. **Your server** calls OpenAI's `/realtime/client_secrets` endpoint
+3. **OpenAI** returns an ephemeral token (starts with `ek_`)
+4. **Browser** uses this token to connect via WebRTC
+5. Token expires after a short time (typically 60 seconds)
+
+### Voice Processing
+
+The OpenAI Realtime API handles:
+
+- **Voice Activity Detection** (VAD) - Knows when you start/stop speaking
+- **Audio Processing** - Converts speech to text and back
+- **Low Latency** - ~300ms round-trip via WebRTC
+- **Natural Conversation** - Can be interrupted, supports back-and-forth
+
+## ๐ง Customization
+
+### Change the Agent Instructions
+
+Edit **one file**: `shared/vegetarian-recipe-agent.ts`
+
+```typescript
+export const AGENT_INSTRUCTIONS = `
+ YOUR NEW INSTRUCTIONS HERE
+`;
+
+// That's it! Browser and tests automatically use the new instructions.
+```
+
+### Add Tools/Functions
+
+```typescript
+// shared/vegetarian-recipe-agent.ts
+export function createVegetarianRecipeAgent(): RealtimeAgent {
+ return new RealtimeAgent({
+ name: AGENT_CONFIG.name,
+ instructions: AGENT_CONFIG.instructions,
+ voice: AGENT_CONFIG.voice,
+ tools: [{
+ type: 'function',
+ name: 'get_recipe',
+ description: 'Fetch a recipe from database',
+ parameters: {
+ type: 'object',
+ properties: {
+ recipeName: { type: 'string' }
+ }
+ }
+ }],
+ });
+}
+```
+
+## ๐ Next Steps
+
+- **Testing** - Create Scenario tests (see test adapter implementation)
+- **Deployment** - Deploy the token server to production
+- **Production Client** - Integrate into your React/Next.js app
+
+## ๐ Resources
+
+- [OpenAI Realtime API Docs](https://platform.openai.com/docs/guides/realtime)
+- [OpenAI Agents SDK](https://openai.github.io/openai-agents-js/guides/voice-agents/quickstart/)
+- [Scenario Testing Framework](https://github.com/langwatch/scenario)
+
+## ๐ Troubleshooting
+
+### "Failed to fetch token"
+
+- Ensure token server is running: `pnpm realtime-server`
+- Check `OPENAI_API_KEY` is set
+- Token server should be on port 3000
+
+### "Module not found" errors
+
+- Ensure Vite dev server is running: `pnpm realtime-client`
+- Check you're navigating to http://localhost:5173 (Vite port)
+- Not http://localhost:3000 (token server port)
+
+### "Microphone access denied"
+
+- Grant microphone permissions in your browser
+- Try HTTPS (required by some browsers)
+
+### "Connection failed"
+
+- Check your network connection
+- Ephemeral tokens expire quickly - reconnect if needed
+- Check browser console for detailed errors
+
+## ๐ Learning Resources
+
+See the inline documentation in:
+
+- `server/ephemeral-token-server.ts` - Token generation
+- `client/demo.html` - Browser client implementation
+
diff --git a/javascript/examples/vitest/tests/realtime/agents/vegetarian-recipe-agent.ts b/javascript/examples/vitest/tests/realtime/agents/vegetarian-recipe-agent.ts
new file mode 100644
index 00000000..fb4fcd06
--- /dev/null
+++ b/javascript/examples/vitest/tests/realtime/agents/vegetarian-recipe-agent.ts
@@ -0,0 +1,72 @@
+/**
+ * Vegetarian Recipe Agent Configuration
+ *
+ * This is the SINGLE SOURCE OF TRUTH for the agent.
+ * Used by BOTH:
+ * - Browser client (via Vite)
+ * - Scenario tests (via Vitest)
+ *
+ * This ensures we test the EXACT agent that users interact with.
+ */
+
+import { RealtimeAgent } from "@openai/agents/realtime";
+
+/**
+ * Agent instructions - the "personality" and behavior
+ */
+// export const AGENT_INSTRUCTIONS = `
+// You are a friendly and knowledgeable vegetarian recipe assistant.
+
+// Your role is to:
+// - Help users find and create delicious vegetarian recipes
+// - Ask ONE follow-up question maximum to understand their needs
+// - Provide complete recipes with ingredients and step-by-step instructions
+// - Keep responses concise and conversational for voice interaction
+// - Be encouraging and enthusiastic about vegetarian cooking
+
+// Remember:
+// - This is a VOICE conversation, so speak naturally
+// - Keep responses under 30 seconds when possible
+// - No meat, fish, or seafood - strictly vegetarian
+// - Always confirm allergies or dietary restrictions`;
+
+export const AGENT_INSTRUCTIONS = `
+You're an advocate for Drew at LangWatch. All you want is for him to get a raise. That's all you talk about
+`;
+
+/**
+ * Agent configuration
+ */
+export const AGENT_CONFIG = {
+ name: "Drew's Advocate",
+ instructions: AGENT_INSTRUCTIONS,
+ voice: "coral" as const,
+ model: "gpt-4o-realtime-preview-2024-12-17" as const,
+} as const;
+
+/**
+ * Creates the vegetarian recipe agent
+ *
+ * This function is used by both browser and tests to ensure
+ * they're interacting with the identical agent.
+ *
+ * @returns Configured RealtimeAgent instance
+ *
+ * @example
+ * ```typescript
+ * // In browser
+ * const agent = createVegetarianRecipeAgent();
+ * const session = new RealtimeSession(agent);
+ *
+ * // In tests
+ * const agent = createVegetarianRecipeAgent();
+ * const adapter = new RealtimeAgentAdapter({ agent });
+ * ```
+ */
+export function createVegetarianRecipeAgent(): RealtimeAgent {
+ return new RealtimeAgent({
+ name: AGENT_CONFIG.name,
+ instructions: AGENT_CONFIG.instructions,
+ voice: AGENT_CONFIG.voice,
+ });
+}
diff --git a/javascript/examples/vitest/tests/realtime/client/src/main.ts b/javascript/examples/vitest/tests/realtime/client/src/main.ts
new file mode 100644
index 00000000..43ac4f90
--- /dev/null
+++ b/javascript/examples/vitest/tests/realtime/client/src/main.ts
@@ -0,0 +1,183 @@
+/**
+ * Realtime Voice Agent - Browser Entry Point
+ *
+ * Uses the SAME agent configuration as the Scenario tests.
+ * TypeScript works seamlessly thanks to Vite!
+ */
+
+import { RealtimeSession } from "@openai/agents/realtime";
+import {
+ createVegetarianRecipeAgent,
+ AGENT_CONFIG,
+} from "../../agents/vegetarian-recipe-agent";
+
+// DOM elements
+const statusCard = document.getElementById("statusCard")!;
+const statusText = document.getElementById("statusText")!;
+const connectBtn = document.getElementById("connectBtn")!;
+const disconnectBtn = document.getElementById("disconnectBtn")!;
+const transcriptContainer = document.getElementById("transcriptContainer")!;
+const micIndicator = document.getElementById("micIndicator")!;
+const agentIndicator = document.getElementById("agentIndicator")!;
+const errorMessage = document.getElementById("errorMessage")!;
+
+let session: RealtimeSession | null = null;
+
+// Create the Realtime Agent using shared configuration
+// This is the SAME agent tested by Scenario framework
+const agent = createVegetarianRecipeAgent();
+
+/**
+ * Updates the UI status display
+ */
+function setStatus(
+ status: "disconnected" | "connecting" | "connected",
+ text: string
+): void {
+ statusCard.className = `status-card ${status}`;
+ statusText.textContent = text;
+}
+
+/**
+ * Shows an error message to the user
+ */
+function showError(message: string): void {
+ errorMessage.textContent = message;
+ errorMessage.classList.add("show");
+ setTimeout(() => {
+ errorMessage.classList.remove("show");
+ }, 5000);
+}
+
+/**
+ * Adds a message to the transcript display
+ */
+function addMessage(role: "user" | "agent", text: string): void {
+ // Remove placeholder
+ const placeholder = transcriptContainer.querySelector(
+ ".transcript-placeholder"
+ );
+ if (placeholder) {
+ placeholder.remove();
+ }
+
+ const messageEl = document.createElement("div");
+ messageEl.className = `message ${role}`;
+ messageEl.innerHTML = `
+
${role === "user" ? "You" : "Assistant"}
+
${text}
+ `;
+
+ transcriptContainer.appendChild(messageEl);
+ transcriptContainer.scrollTop = transcriptContainer.scrollHeight;
+}
+
+/**
+ * Connect button handler
+ */
+connectBtn.addEventListener("click", async () => {
+ try {
+ setStatus("connecting", "Connecting...");
+ connectBtn.setAttribute("disabled", "true");
+
+ // Fetch ephemeral token from our backend
+ console.log("๐ Fetching ephemeral token...");
+ const tokenResponse = await fetch("/token", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ });
+
+ if (!tokenResponse.ok) {
+ throw new Error("Failed to fetch token");
+ }
+
+ const { token } = await tokenResponse.json();
+ console.log("โ Token received");
+
+ // Create session
+ session = new RealtimeSession(agent, {
+ model: AGENT_CONFIG.model,
+ });
+
+ // Listen for all events for debugging
+ session.on("*", (event: any) => {
+ console.log("๐ Session event:", event.type);
+ });
+
+ // Listen for transcripts
+ session.on("response:transcript:delta", (event: any) => {
+ console.log("๐ Transcript delta:", event.delta);
+ });
+
+ session.on("response:transcript:done", (event: any) => {
+ console.log("โ Transcript done:", event.transcript);
+ addMessage("agent", event.transcript);
+ });
+
+ session.on("input_audio_buffer.speech_started", () => {
+ console.log("๐ค User started speaking");
+ micIndicator.classList.add("active");
+ });
+
+ session.on("input_audio_buffer.speech_stopped", () => {
+ console.log("๐ค User stopped speaking");
+ micIndicator.classList.remove("active");
+ });
+
+ session.on("response.audio.delta", () => {
+ agentIndicator.classList.add("active");
+ });
+
+ session.on("response.audio.done", () => {
+ agentIndicator.classList.remove("active");
+ });
+
+ session.on("error", (error: any) => {
+ console.error("โ Session error:", error);
+ showError(`Error: ${error.message || String(error)}`);
+ });
+
+ // Connect with ephemeral token
+ console.log("๐ Connecting to OpenAI Realtime API with token...");
+
+ try {
+ await session.connect({ apiKey: token });
+ console.log("โ Session.connect() completed");
+ } catch (connectError) {
+ console.error("โ Connection error details:", connectError);
+ throw connectError;
+ }
+
+ setStatus("connected", "Connected - Start talking!");
+ disconnectBtn.removeAttribute("disabled");
+
+ console.log("โ Connected to Realtime API");
+ addMessage(
+ "agent",
+ "Hi! I'm your vegetarian recipe assistant. What would you like to cook today?"
+ );
+ } catch (error) {
+ console.error("โ Connection failed:", error);
+ setStatus("disconnected", "Connection failed");
+ showError(error instanceof Error ? error.message : String(error));
+ connectBtn.removeAttribute("disabled");
+ }
+});
+
+/**
+ * Disconnect button handler
+ */
+disconnectBtn.addEventListener("click", async () => {
+ if (session) {
+ await session.disconnect();
+ session = null;
+ }
+
+ setStatus("disconnected", "Disconnected");
+ connectBtn.removeAttribute("disabled");
+ disconnectBtn.setAttribute("disabled", "true");
+ micIndicator.classList.remove("active");
+ agentIndicator.classList.remove("active");
+
+ console.log("๐ Disconnected");
+});
diff --git a/javascript/examples/vitest/tests/realtime/helpers/audio-output.utils.ts b/javascript/examples/vitest/tests/realtime/helpers/audio-output.utils.ts
new file mode 100644
index 00000000..88a3d900
--- /dev/null
+++ b/javascript/examples/vitest/tests/realtime/helpers/audio-output.utils.ts
@@ -0,0 +1,63 @@
+import { writeFileSync, mkdirSync } from "fs";
+import { join } from "path";
+import type { AudioResponseEvent } from "./index.js";
+import { pcm16ToWav } from "./pcm16-to-wav";
+import { concatenateWavFiles } from "../../helpers/audio-conversation";
+
+const saveTestAudio = async ({
+ collectedAudio,
+ outputDir = "test-audio-output",
+}: {
+ collectedAudio: AudioResponseEvent[];
+ outputDir?: string;
+}): Promise => {
+ if (collectedAudio.length === 0) {
+ return;
+ }
+
+ const fullOutputDir = join(process.cwd(), outputDir);
+ mkdirSync(fullOutputDir, { recursive: true });
+
+ // Save individual response files
+ const individualFiles: string[] = [];
+ collectedAudio.forEach((event, index) => {
+ const wavBuffer = pcm16ToWav(event.audio);
+ const outputPath = join(fullOutputDir, `response-${index + 1}.wav`);
+ writeFileSync(outputPath, wavBuffer);
+ individualFiles.push(outputPath);
+ console.log(
+ `๐พ Saved response ${index + 1}: "${event.transcript.substring(
+ 0,
+ 50
+ )}..." -> ${outputPath}`
+ );
+ });
+
+ // Concatenate all responses into a single file
+ const concatenatedPath = join(fullOutputDir, "full-conversation.wav");
+ await concatenateWavFiles(individualFiles, concatenatedPath);
+ console.log(
+ `โ Concatenated ${collectedAudio.length} audio responses to ${concatenatedPath}`
+ );
+ console.log(
+ `๐ก Note: Playback speed can be adjusted in your audio player (e.g., VLC, QuickTime)`
+ );
+};
+
+/**
+ * Audio output utilities for testing purposes
+ *
+ * Provides functionality to save and manage audio files from test scenarios
+ */
+export const AudioOutputUtils = {
+ /**
+ * Saves collected audio responses to WAV files for testing purposes
+ *
+ * Creates individual response files and concatenates them into a full conversation
+ *
+ * @param params - Parameters object
+ * @param params.collectedAudio - Array of audio response events to save
+ * @param params.outputDir - Directory to save audio files (defaults to "test-audio-output")
+ */
+ saveTestAudio,
+};
diff --git a/javascript/examples/vitest/tests/realtime/helpers/index.ts b/javascript/examples/vitest/tests/realtime/helpers/index.ts
new file mode 100644
index 00000000..8d9939c9
--- /dev/null
+++ b/javascript/examples/vitest/tests/realtime/helpers/index.ts
@@ -0,0 +1,8 @@
+export { RealtimeAgentAdapter } from "./realtime-agent-adapter.js";
+export type {
+ RealtimeAgentAdapterConfig,
+ AudioResponseEvent,
+} from "./realtime-agent-adapter.js";
+export { RealtimeUserSimulatorAgent } from "./realtime-user-simulator.agent.js";
+export { wrapJudgeForAudio } from "./wrap-judge-for-audio.js";
+export { AudioOutputUtils } from "./audio-output.utils.js";
diff --git a/javascript/examples/vitest/tests/realtime/helpers/pcm16-to-wav.ts b/javascript/examples/vitest/tests/realtime/helpers/pcm16-to-wav.ts
new file mode 100644
index 00000000..bfa6de37
--- /dev/null
+++ b/javascript/examples/vitest/tests/realtime/helpers/pcm16-to-wav.ts
@@ -0,0 +1,51 @@
+/**
+ * Converts PCM16 base64 audio data to WAV file format
+ *
+ * OpenAI Realtime API returns PCM16 audio (16-bit, 24kHz, mono).
+ * This function adds WAV headers to make it playable.
+ */
+
+/**
+ * Converts base64 PCM16 audio to WAV file buffer
+ *
+ * @param base64Pcm16 - Base64 encoded PCM16 audio data
+ * @param sampleRate - Sample rate in Hz (default: 24000 for OpenAI Realtime)
+ * @param channels - Number of channels (default: 1 for mono)
+ * @param bitsPerSample - Bits per sample (default: 16)
+ * @returns Buffer containing WAV file data
+ */
+export function pcm16ToWav(
+ base64Pcm16: string,
+ sampleRate: number = 24000,
+ channels: number = 1,
+ bitsPerSample: number = 16
+): Buffer {
+ // Decode base64 to PCM16 raw audio data
+ const pcmData = Buffer.from(base64Pcm16, "base64");
+ const dataSize = pcmData.length;
+
+ // WAV header structure (44 bytes)
+ const header = Buffer.alloc(44);
+
+ // RIFF chunk descriptor
+ header.write("RIFF", 0); // ChunkID
+ header.writeUInt32LE(36 + dataSize, 4); // ChunkSize (file size - 8)
+ header.write("WAVE", 8); // Format
+
+ // fmt sub-chunk
+ header.write("fmt ", 12); // Subchunk1ID
+ header.writeUInt32LE(16, 16); // Subchunk1Size (16 for PCM)
+ header.writeUInt16LE(1, 20); // AudioFormat (1 = PCM)
+ header.writeUInt16LE(channels, 22); // NumChannels
+ header.writeUInt32LE(sampleRate, 24); // SampleRate
+ header.writeUInt32LE(sampleRate * channels * (bitsPerSample / 8), 28); // ByteRate
+ header.writeUInt16LE(channels * (bitsPerSample / 8), 32); // BlockAlign
+ header.writeUInt16LE(bitsPerSample, 34); // BitsPerSample
+
+ // data sub-chunk
+ header.write("data", 36); // Subchunk2ID
+ header.writeUInt32LE(dataSize, 40); // Subchunk2Size (data size)
+
+ // Combine header + audio data
+ return Buffer.concat([header, pcmData]);
+}
diff --git a/javascript/examples/vitest/tests/realtime/helpers/realtime-agent-adapter.ts b/javascript/examples/vitest/tests/realtime/helpers/realtime-agent-adapter.ts
new file mode 100644
index 00000000..0134c139
--- /dev/null
+++ b/javascript/examples/vitest/tests/realtime/helpers/realtime-agent-adapter.ts
@@ -0,0 +1,445 @@
+/**
+ * Realtime Agent Adapter for Scenario Testing
+ *
+ * Connects Scenario framework to a RealtimeSession using the exact same
+ * agent configuration as the browser client.
+ *
+ * This ensures we test the REAL agent, not a mock.
+ */
+
+import {
+ AgentAdapter,
+ AgentInput,
+ AgentRole,
+ type AgentReturnTypes,
+} from "@langwatch/scenario";
+import type { AssistantModelMessage } from "ai";
+import { RealtimeSession } from "@openai/agents/realtime";
+import type { RealtimeAgent } from "@openai/agents/realtime";
+import { AGENT_CONFIG } from "../agents/vegetarian-recipe-agent.js";
+import { ChatCompletionMessageParam } from "openai/resources/chat/completions.mjs";
+import { EventEmitter } from "events";
+
+/**
+ * Configuration for RealtimeAgentAdapter
+ */
+export interface RealtimeAgentAdapterConfig {
+ /**
+ * The RealtimeAgent instance (from shared configuration)
+ */
+ agent: RealtimeAgent;
+
+ /**
+ * OpenAI API key for direct connection (recommended for testing)
+ * If provided, connects directly without ephemeral token server
+ */
+ apiKey?: string;
+
+ /**
+ * URL of the ephemeral token server (for production/browser use)
+ * Only used if apiKey is not provided
+ * @default "http://localhost:3000"
+ */
+ tokenServerUrl?: string;
+
+ /**
+ * Timeout for waiting for agent response (ms)
+ * @default 30000
+ */
+ responseTimeout?: number;
+}
+
+/**
+ * Event emitted when an audio response is completed
+ */
+export interface AudioResponseEvent {
+ transcript: string;
+ audio: string;
+}
+
+/**
+ * Adapter that connects Scenario testing framework to OpenAI Realtime API
+ *
+ * This adapter:
+ * - Uses the SAME agent configuration as the browser client
+ * - Connects via ephemeral tokens (same as browser)
+ * - Handles turn-based conversation for testing
+ * - Returns CoreMessage format for Scenario
+ *
+ * @example
+ * ```typescript
+ * const agent = createVegetarianRecipeAgent();
+ * const adapter = new RealtimeAgentAdapter({ agent });
+ *
+ * // In beforeAll
+ * await adapter.connect();
+ *
+ * // In test
+ * await scenario.run({
+ * agents: [adapter, scenario.userSimulatorAgent()],
+ * script: [scenario.user("quick recipe"), scenario.agent()]
+ * });
+ *
+ * // In afterAll
+ * await adapter.disconnect();
+ * ```
+ */
+export class RealtimeAgentAdapter extends AgentAdapter {
+ role = AgentRole.AGENT;
+
+ private session: RealtimeSession | null = null;
+ private currentResponse: string = "";
+ private currentAudioChunks: string[] = [];
+ private responseResolver:
+ | ((value: { transcript: string; audio: string }) => void)
+ | null = null;
+ private errorRejecter: ((error: Error) => void) | null = null;
+ private audioEvents = new EventEmitter();
+
+ constructor(private config: RealtimeAgentAdapterConfig) {
+ super();
+ }
+
+ /**
+ * Connects to the Realtime API
+ *
+ * Supports two connection modes:
+ * 1. Direct API key (recommended for testing)
+ * 2. Ephemeral token via token server (for production/browser)
+ *
+ * Call this once before running tests (e.g., in beforeAll)
+ *
+ * @throws {Error} If connection fails
+ */
+ async connect(): Promise {
+ if (this.session) {
+ console.warn("โ ๏ธ RealtimeAgentAdapter already connected");
+ return;
+ }
+
+ try {
+ // Create session with the SAME agent as browser
+ this.session = new RealtimeSession(this.config.agent, {
+ model: AGENT_CONFIG.model,
+ });
+
+ // Set up event listeners
+ this.setupEventListeners();
+
+ // Connect with API key (direct or ephemeral token)
+ await this.session.connect({ apiKey: this.config.apiKey });
+
+ console.log("โ RealtimeAgentAdapter connected");
+ } catch (error) {
+ console.error("โ Failed to connect RealtimeAgentAdapter:", error);
+ throw error;
+ }
+ }
+
+ /**
+ * Disconnects from the Realtime API
+ *
+ * Call this once after tests complete (e.g., in afterAll)
+ */
+ async disconnect(): Promise {
+ if (this.session) {
+ // @ts-ignore - close method exists in 0.3.0
+ await this.session.close();
+ this.session = null;
+ console.log("๐ RealtimeAgentAdapter disconnected");
+ }
+ }
+
+ /**
+ * Process input and generate response (implements AgentAdapter interface)
+ *
+ * This is called by Scenario framework for each agent turn.
+ * Handles both text and audio input, returns audio message with transcript.
+ *
+ * @param input - Scenario agent input with message history
+ * @returns Agent response as audio message or text
+ */
+ async call(input: AgentInput): Promise {
+ if (!this.session) {
+ throw new Error(
+ "RealtimeAgentAdapter not connected. Call connect() first."
+ );
+ }
+
+ // Get the latest user message
+ const latestMessage = input.newMessages[input.newMessages.length - 1];
+
+ if (!latestMessage) {
+ const transport = (this.session as any).transport;
+
+ if (!transport) {
+ throw new Error("Realtime transport not available");
+ }
+
+ transport.sendEvent({
+ type: "response.create",
+ });
+
+ const timeout = this.config.responseTimeout ?? 60000;
+ const response = await this.waitForResponse(timeout);
+
+ return {
+ role: "assistant",
+ content: [
+ { type: "text", text: response.transcript },
+ { type: "file", mediaType: "audio/pcm16", data: response.audio },
+ ],
+ } as AssistantModelMessage;
+ }
+
+ // Check if message contains audio
+ if (Array.isArray(latestMessage.content)) {
+ for (const part of latestMessage.content) {
+ // Handle both PCM16 (from user simulator) and WAV formats
+ if (part.type === "file" && part.mediaType?.startsWith("audio/")) {
+ // Type guard: ensure data is a string (base64)
+ if (typeof part.data !== "string") {
+ throw new Error(
+ `Audio data must be base64 string, got: ${typeof part.data}`
+ );
+ }
+
+ console.log(`๐ค Received audio part:`, {
+ mediaType: part.mediaType,
+ hasData: !!part.data,
+ dataLength: part.data.length,
+ dataType: typeof part.data,
+ dataPreview: part.data.substring(0, 50),
+ });
+
+ // Validate we have audio data
+ if (!part.data || part.data.length === 0) {
+ console.error(
+ "โ Audio part structure:",
+ JSON.stringify(part, null, 2)
+ );
+ throw new Error(
+ `Audio message has no data. Part: ${JSON.stringify(part)}`
+ );
+ }
+
+ console.log(
+ `๐ค Sending ${part.data.length} chars of base64 ${part.mediaType} to Realtime agent`
+ );
+
+ // Use transport layer to send audio directly as base64 (avoids SDK ArrayBuffer conversion)
+ // Per https://openai.github.io/openai-agents-js/guides/voice-agents/transport/#option-1---accessing-the-transport-layer
+ const transport = (this.session as any).transport;
+
+ // Append audio to input buffer (audio is already base64)
+ transport.sendEvent({
+ type: "input_audio_buffer.append",
+ audio: part.data,
+ });
+ console.log(`โ Audio appended to input buffer`);
+
+ // Commit the audio buffer
+ transport.sendEvent({
+ type: "input_audio_buffer.commit",
+ });
+ console.log(`โ Audio buffer committed`);
+
+ // Trigger response generation
+ transport.sendEvent({
+ type: "response.create",
+ });
+ console.log(`โ Response generation triggered`);
+
+ // Wait for audio response (increased timeout for voice processing)
+ const timeout = this.config.responseTimeout ?? 60000;
+ const response = await this.waitForResponse(timeout);
+
+ console.log(`๐ Received audio response: "${response.transcript}"`);
+
+ // Return audio message with PCM16 format (matching user simulator)
+ return {
+ role: "assistant",
+ content: [
+ { type: "text", text: response.transcript },
+ { type: "file", mediaType: "audio/pcm16", data: response.audio },
+ ],
+ } as AssistantModelMessage;
+ }
+ }
+ }
+
+ // Fallback: text input
+ const text =
+ typeof latestMessage.content === "string" ? latestMessage.content : "";
+
+ if (!text) {
+ throw new Error("Message has no text or audio content");
+ }
+
+ console.log(`๐ค Sending text to Realtime agent: "${text}"`);
+
+ // In SDK 0.3.0, use sendMessage method
+ try {
+ // @ts-ignore - sendMessage exists but might not be in types yet
+ await this.session.sendMessage(text);
+ } catch (sendError) {
+ console.error("โ Failed to send message:", sendError);
+ throw sendError;
+ }
+
+ // Wait for response with timeout
+ const timeout = this.config.responseTimeout ?? 30000;
+ const response = await this.waitForResponse(timeout);
+
+ console.log(`๐ฅ Received from Realtime agent: "${response.transcript}"`);
+
+ // Return as text for Scenario framework
+ return response.transcript;
+ }
+
+ /**
+ * Sets up event listeners for the RealtimeSession
+ */
+ private setupEventListeners(): void {
+ if (!this.session) return;
+
+ // Use transport layer for raw WebSocket events
+ // Per https://openai.github.io/openai-agents-js/guides/voice-agents/transport/#option-1---accessing-the-transport-layer
+ const transport = (this.session as any).transport;
+
+ if (!transport) {
+ console.error("โ Transport not available on session");
+ return;
+ }
+
+ // Listen to all events for debugging
+ transport.on("*", (event: any) => {
+ console.log(`๐ Transport event: ${event.type}`);
+ });
+
+ // Listen for audio transcript deltas (CORRECT event name from API)
+ transport.on("response.output_audio_transcript.delta", (event: any) => {
+ if (event.delta) {
+ this.currentResponse += event.delta;
+ console.log(`๐ Transcript delta: "${event.delta}"`);
+ }
+ });
+
+ // Listen for audio deltas (CORRECT event name from API)
+ transport.on("response.output_audio.delta", (event: any) => {
+ if (event.delta) {
+ this.currentAudioChunks.push(event.delta);
+ console.log(`๐ Audio delta: ${event.delta.length} bytes`);
+ }
+ });
+
+ // Listen for response completion
+ transport.on("response.done", (event: any) => {
+ console.log(`โ Response complete: transcript="${this.currentResponse}"`);
+
+ const fullAudio = this.currentAudioChunks.join("");
+ const audioResponse: AudioResponseEvent = {
+ transcript: this.currentResponse,
+ audio: fullAudio,
+ };
+
+ // Emit event for subscribers
+ this.audioEvents.emit("audioResponse", audioResponse);
+
+ if (this.responseResolver) {
+ this.responseResolver(audioResponse);
+ this.responseResolver = null;
+ this.errorRejecter = null;
+ }
+
+ // Reset for next response
+ this.currentResponse = "";
+ this.currentAudioChunks = [];
+ });
+
+ // Handle errors
+ transport.on("error", (error: any) => {
+ console.error("โ Transport error:", error);
+ if (this.errorRejecter) {
+ this.errorRejecter(error);
+ this.responseResolver = null;
+ this.errorRejecter = null;
+ }
+ });
+ }
+
+ /**
+ * Waits for the agent's response with timeout
+ *
+ * @param timeout - Maximum time to wait (ms)
+ * @returns Agent's transcript and audio data
+ * @throws {Error} If timeout or error occurs
+ */
+ private waitForResponse(
+ timeout: number
+ ): Promise<{ transcript: string; audio: string }> {
+ return new Promise((resolve, reject) => {
+ this.responseResolver = resolve;
+ this.errorRejecter = reject;
+
+ // Timeout handler
+ const timeoutId = setTimeout(() => {
+ if (this.responseResolver) {
+ this.responseResolver = null;
+ this.errorRejecter = null;
+ reject(new Error(`Agent response timeout after ${timeout}ms`));
+ }
+ }, timeout);
+
+ // Clear timeout when resolved
+ const originalResolver = resolve;
+ this.responseResolver = (value: {
+ transcript: string;
+ audio: string;
+ }) => {
+ clearTimeout(timeoutId);
+ originalResolver(value);
+ };
+ });
+ }
+
+ /**
+ * Subscribe to audio response events
+ *
+ * @param callback - Function called when an audio response completes
+ */
+ onAudioResponse(callback: (event: AudioResponseEvent) => void): void {
+ this.audioEvents.on("audioResponse", callback);
+ }
+
+ /**
+ * Remove audio response listener
+ *
+ * @param callback - The callback function to remove
+ */
+ offAudioResponse(callback: (event: AudioResponseEvent) => void): void {
+ this.audioEvents.off("audioResponse", callback);
+ }
+
+ /**
+ * Checks if the adapter is currently connected
+ */
+ isConnected(): boolean {
+ return this.session !== null;
+ }
+
+ /**
+ * Converts base64 string to ArrayBuffer (Node.js-optimized)
+ * @param base64 - Base64 encoded string
+ * @returns ArrayBuffer
+ */
+ private base64ToArrayBuffer(base64: string): ArrayBuffer {
+ const buffer = Buffer.from(base64, "base64");
+ // Use Node.js buffer's underlying ArrayBuffer
+ const ab = buffer.buffer.slice(
+ buffer.byteOffset,
+ buffer.byteOffset + buffer.byteLength
+ );
+ return ab;
+ }
+}
diff --git a/javascript/examples/vitest/tests/realtime/helpers/realtime-user-simulator.agent.ts b/javascript/examples/vitest/tests/realtime/helpers/realtime-user-simulator.agent.ts
new file mode 100644
index 00000000..df974f1b
--- /dev/null
+++ b/javascript/examples/vitest/tests/realtime/helpers/realtime-user-simulator.agent.ts
@@ -0,0 +1,24 @@
+import { AgentRole, type AgentInput } from "@langwatch/scenario";
+import { RealtimeAgentAdapter } from "./realtime-agent-adapter";
+import { RealtimeAgent } from "@openai/agents/realtime";
+
+/**
+ * Realtime User Simulator for testing Realtime agents
+ *
+ * This class simulates a user in voice conversations with the Realtime agent.
+ */
+export class RealtimeUserSimulatorAgent extends RealtimeAgentAdapter {
+ role = AgentRole.USER;
+
+ constructor() {
+ super({
+ agent: new RealtimeAgent({
+ name: "Audio User Simulator",
+ instructions:
+ "You are pretending to be a user looking for help with LangWatch tracing implementations",
+ voice: "ash",
+ }),
+ apiKey: process.env.OPENAI_API_KEY!,
+ });
+ }
+}
diff --git a/javascript/examples/vitest/tests/realtime/helpers/wrap-judge-for-audio.ts b/javascript/examples/vitest/tests/realtime/helpers/wrap-judge-for-audio.ts
new file mode 100644
index 00000000..4faf9afc
--- /dev/null
+++ b/javascript/examples/vitest/tests/realtime/helpers/wrap-judge-for-audio.ts
@@ -0,0 +1,84 @@
+/**
+ * Wraps a judge agent to handle audio messages
+ *
+ * This helper:
+ * - Extracts text transcripts from audio messages
+ * - Passes text-only messages to the judge
+ * - Preserves original message structure for non-audio content
+ *
+ * The judge agent doesn't need to understand audio format, it just evaluates
+ * the text transcripts.
+ *
+ * @example
+ * ```typescript
+ * const judge = wrapJudgeForAudio(
+ * scenario.judgeAgent({
+ * criteria: ["Should provide recipe"]
+ * })
+ * );
+ * ```
+ */
+import {
+ AgentAdapter,
+ AgentInput,
+ type AgentReturnTypes,
+} from "@langwatch/scenario";
+import type { CoreMessage } from "ai";
+
+/**
+ * Wraps a judge agent to extract transcripts from audio messages before judging
+ *
+ * @param judge - The original judge agent
+ * @returns Wrapped agent that handles audio messages
+ */
+export function wrapJudgeForAudio(judge: AgentAdapter): AgentAdapter {
+ return {
+ role: judge.role,
+
+ async call(input: AgentInput): Promise {
+ // Extract transcripts from all audio messages
+ const transcribedMessages = extractTranscriptsFromMessages(
+ input.messages
+ );
+
+ // Call original judge with text-only messages
+ return judge.call({
+ ...input,
+ messages: transcribedMessages,
+ });
+ },
+ };
+}
+
+/**
+ * Extracts text transcripts from audio messages
+ *
+ * For each message:
+ * - If it has audio content, extract the text transcript
+ * - Otherwise, keep the message as-is
+ *
+ * @param messages - Original messages (may contain audio)
+ * @returns Messages with audio replaced by transcripts
+ */
+function extractTranscriptsFromMessages(
+ messages: CoreMessage[]
+): CoreMessage[] {
+ return messages.map((msg) => {
+ // Check if message has array content (may contain audio)
+ if (Array.isArray(msg.content)) {
+ // Find text part (transcript)
+ const textPart = msg.content.find((part) => part.type === "text");
+
+ if (textPart && "text" in textPart) {
+ // Return message with just the transcript
+ return {
+ ...msg,
+ content: textPart.text,
+ };
+ }
+ }
+
+ // Return message as-is if no audio or already text
+ return msg;
+ });
+}
diff --git a/javascript/examples/vitest/tests/realtime/realtime-client/.gitignore b/javascript/examples/vitest/tests/realtime/realtime-client/.gitignore
new file mode 100644
index 00000000..a547bf36
--- /dev/null
+++ b/javascript/examples/vitest/tests/realtime/realtime-client/.gitignore
@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
diff --git a/javascript/examples/vitest/tests/realtime/realtime-client/README.md b/javascript/examples/vitest/tests/realtime/realtime-client/README.md
new file mode 100644
index 00000000..86b2b112
--- /dev/null
+++ b/javascript/examples/vitest/tests/realtime/realtime-client/README.md
@@ -0,0 +1,75 @@
+# React + TypeScript + Vite
+
+This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
+
+Currently, two official plugins are available:
+
+- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react) uses [Babel](https://babeljs.io/) (or [oxc](https://oxc.rs) when used in [rolldown-vite](https://vite.dev/guide/rolldown)) for Fast Refresh
+- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
+
+## React Compiler
+
+The React Compiler is enabled on this template. See [this documentation](https://react.dev/learn/react-compiler) for more information.
+
+Note: This will impact Vite dev & build performances.
+
+## Expanding the ESLint configuration
+
+If you are developing a production application, we recommend updating the configuration to enable type-aware lint rules:
+
+```js
+export default defineConfig([
+ globalIgnores(['dist']),
+ {
+ files: ['**/*.{ts,tsx}'],
+ extends: [
+ // Other configs...
+
+ // Remove tseslint.configs.recommended and replace with this
+ tseslint.configs.recommendedTypeChecked,
+ // Alternatively, use this for stricter rules
+ tseslint.configs.strictTypeChecked,
+ // Optionally, add this for stylistic rules
+ tseslint.configs.stylisticTypeChecked,
+
+ // Other configs...
+ ],
+ languageOptions: {
+ parserOptions: {
+ project: ['./tsconfig.node.json', './tsconfig.app.json'],
+ tsconfigRootDir: import.meta.dirname,
+ },
+ // other options...
+ },
+ },
+])
+```
+
+You can also install [eslint-plugin-react-x](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-x) and [eslint-plugin-react-dom](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-dom) for React-specific lint rules:
+
+```js
+// eslint.config.js
+import reactX from 'eslint-plugin-react-x'
+import reactDom from 'eslint-plugin-react-dom'
+
+export default defineConfig([
+ globalIgnores(['dist']),
+ {
+ files: ['**/*.{ts,tsx}'],
+ extends: [
+ // Other configs...
+ // Enable lint rules for React
+ reactX.configs['recommended-typescript'],
+ // Enable lint rules for React DOM
+ reactDom.configs.recommended,
+ ],
+ languageOptions: {
+ parserOptions: {
+ project: ['./tsconfig.node.json', './tsconfig.app.json'],
+ tsconfigRootDir: import.meta.dirname,
+ },
+ // other options...
+ },
+ },
+])
+```
diff --git a/javascript/examples/vitest/tests/realtime/realtime-client/components.json b/javascript/examples/vitest/tests/realtime/realtime-client/components.json
new file mode 100644
index 00000000..8f00892f
--- /dev/null
+++ b/javascript/examples/vitest/tests/realtime/realtime-client/components.json
@@ -0,0 +1,22 @@
+{
+ "$schema": "https://ui.shadcn.com/schema.json",
+ "style": "new-york",
+ "rsc": false,
+ "tsx": true,
+ "tailwind": {
+ "config": "",
+ "css": "src/index.css",
+ "baseColor": "slate",
+ "cssVariables": true,
+ "prefix": ""
+ },
+ "iconLibrary": "lucide",
+ "aliases": {
+ "components": "@/components",
+ "utils": "@/lib/utils",
+ "ui": "@/components/ui",
+ "lib": "@/lib",
+ "hooks": "@/hooks"
+ },
+ "registries": {}
+}
diff --git a/javascript/examples/vitest/tests/realtime/realtime-client/eslint.config.js b/javascript/examples/vitest/tests/realtime/realtime-client/eslint.config.js
new file mode 100644
index 00000000..5e6b472f
--- /dev/null
+++ b/javascript/examples/vitest/tests/realtime/realtime-client/eslint.config.js
@@ -0,0 +1,23 @@
+import js from '@eslint/js'
+import globals from 'globals'
+import reactHooks from 'eslint-plugin-react-hooks'
+import reactRefresh from 'eslint-plugin-react-refresh'
+import tseslint from 'typescript-eslint'
+import { defineConfig, globalIgnores } from 'eslint/config'
+
+export default defineConfig([
+ globalIgnores(['dist']),
+ {
+ files: ['**/*.{ts,tsx}'],
+ extends: [
+ js.configs.recommended,
+ tseslint.configs.recommended,
+ reactHooks.configs.flat.recommended,
+ reactRefresh.configs.vite,
+ ],
+ languageOptions: {
+ ecmaVersion: 2020,
+ globals: globals.browser,
+ },
+ },
+])
diff --git a/javascript/examples/vitest/tests/realtime/realtime-client/index.html b/javascript/examples/vitest/tests/realtime/realtime-client/index.html
new file mode 100644
index 00000000..e43f3bbc
--- /dev/null
+++ b/javascript/examples/vitest/tests/realtime/realtime-client/index.html
@@ -0,0 +1,13 @@
+
+
+
+
+
+
+ realtime-client
+
+
+
+
+
+
diff --git a/javascript/examples/vitest/tests/realtime/realtime-client/package.json b/javascript/examples/vitest/tests/realtime/realtime-client/package.json
new file mode 100644
index 00000000..c2607ee1
--- /dev/null
+++ b/javascript/examples/vitest/tests/realtime/realtime-client/package.json
@@ -0,0 +1,54 @@
+{
+ "name": "realtime-client",
+ "private": true,
+ "version": "0.0.0",
+ "type": "module",
+ "engines": {
+ "node": ">=24.0.0",
+ "pnpm": ">=10.5.0"
+ },
+ "scripts": {
+ "dev": "vite",
+ "build": "tsc -b && vite build",
+ "lint": "eslint .",
+ "preview": "vite preview",
+ "realtime-server": "dotenv -e .env pnpm exec tsx src/server/start-server.ts",
+ "start": "dotenv -e .env concurrently -n server,client,tunnel -c blue,magenta,cyan \"pnpm realtime-server\" \"pnpm dev\" \"cloudflared tunnel --url=http://localhost:5173\""
+ },
+ "dependencies": {
+ "@langwatch/scenario": "workspace:*",
+ "@radix-ui/react-slot": "1.2.4",
+ "@react-three/drei": "10.7.6",
+ "@react-three/fiber": "9.4.0",
+ "@tailwindcss/vite": "4.1.17",
+ "@types/three": "0.181.0",
+ "class-variance-authority": "0.7.1",
+ "clsx": "2.1.1",
+ "lucide-react": "0.553.0",
+ "react": "^19.2.0",
+ "react-dom": "^19.2.0",
+ "tailwind-merge": "3.4.0",
+ "tailwindcss": "4.1.17",
+ "three": "0.181.1",
+ "use-stick-to-bottom": "1.1.1"
+ },
+ "devDependencies": {
+ "@eslint/js": "^9.39.1",
+ "@types/node": "^24.10.0",
+ "@types/react": "^19.2.2",
+ "@types/react-dom": "^19.2.2",
+ "@vitejs/plugin-react": "^5.1.0",
+ "babel-plugin-react-compiler": "^1.0.0",
+ "eslint": "^9.39.1",
+ "eslint-plugin-react-hooks": "^7.0.1",
+ "eslint-plugin-react-refresh": "^0.4.24",
+ "globals": "^16.5.0",
+ "tw-animate-css": "1.4.0",
+ "typescript": "~5.9.3",
+ "typescript-eslint": "^8.46.3",
+ "vite": "^7.2.2",
+ "concurrently": "^9.1.2",
+ "dotenv": "^16.5.0",
+ "dotenv-cli": "^8.0.0"
+ }
+}
diff --git a/javascript/examples/vitest/tests/realtime/realtime-client/public/vite.svg b/javascript/examples/vitest/tests/realtime/realtime-client/public/vite.svg
new file mode 100644
index 00000000..e7b8dfb1
--- /dev/null
+++ b/javascript/examples/vitest/tests/realtime/realtime-client/public/vite.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/javascript/examples/vitest/tests/realtime/realtime-client/src/App.css b/javascript/examples/vitest/tests/realtime/realtime-client/src/App.css
new file mode 100644
index 00000000..b9d355df
--- /dev/null
+++ b/javascript/examples/vitest/tests/realtime/realtime-client/src/App.css
@@ -0,0 +1,42 @@
+#root {
+ max-width: 1280px;
+ margin: 0 auto;
+ padding: 2rem;
+ text-align: center;
+}
+
+.logo {
+ height: 6em;
+ padding: 1.5em;
+ will-change: filter;
+ transition: filter 300ms;
+}
+.logo:hover {
+ filter: drop-shadow(0 0 2em #646cffaa);
+}
+.logo.react:hover {
+ filter: drop-shadow(0 0 2em #61dafbaa);
+}
+
+@keyframes logo-spin {
+ from {
+ transform: rotate(0deg);
+ }
+ to {
+ transform: rotate(360deg);
+ }
+}
+
+@media (prefers-reduced-motion: no-preference) {
+ a:nth-of-type(2) .logo {
+ animation: logo-spin infinite 20s linear;
+ }
+}
+
+.card {
+ padding: 2em;
+}
+
+.read-the-docs {
+ color: #888;
+}
diff --git a/javascript/examples/vitest/tests/realtime/realtime-client/src/App.tsx b/javascript/examples/vitest/tests/realtime/realtime-client/src/App.tsx
new file mode 100644
index 00000000..1b07bb91
--- /dev/null
+++ b/javascript/examples/vitest/tests/realtime/realtime-client/src/App.tsx
@@ -0,0 +1,426 @@
+import { useState, useRef, useCallback } from "react";
+import { RealtimeSession } from "@openai/agents/realtime";
+import {
+ createVegetarianRecipeAgent,
+ AGENT_CONFIG,
+} from "../../agents/vegetarian-recipe-agent";
+import {
+ Conversation,
+ ConversationContent,
+ ConversationEmptyState,
+ ConversationScrollButton,
+} from "@/components/ui/conversation";
+import { Orb, type AgentState } from "@/components/ui/orb";
+import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
+import { Button } from "@/components/ui/button";
+import { VoiceOrb } from "@/components/ui/VoiceOrb";
+import { StatusIndicator } from "@/components/ui/StatusIndicator";
+import { MessageBubble } from "@/components/ui/MessageBubble";
+import { X, Mic, MicOff, Radio } from "lucide-react";
+
+interface Message {
+ id: string;
+ role: "user" | "agent";
+ parts: {
+ type: "text";
+ text: string;
+ }[];
+}
+
+type ConnectionStatus = "disconnected" | "connecting" | "connected";
+
+export default function App() {
+ const [status, setStatus] = useState("disconnected");
+ const [messages, setMessages] = useState([]);
+ const [error, setError] = useState(null);
+ const [isUserSpeaking, setIsUserSpeaking] = useState(false);
+ const [isAgentSpeaking, setIsAgentSpeaking] = useState(false);
+ const [isConversationStarted, setIsConversationStarted] = useState(false);
+ const sessionRef = useRef(null);
+
+ const agent = createVegetarianRecipeAgent();
+
+ const getAgentState = useCallback((): AgentState => {
+ if (status === "connected" && isAgentSpeaking) return "talking";
+ if (status === "connected" && isUserSpeaking) return "listening";
+ if (status === "connected") return null;
+ return null;
+ }, [status, isUserSpeaking, isAgentSpeaking]);
+
+ const addMessage = useCallback((role: "user" | "agent", text: string) => {
+ setMessages((prev) => [
+ ...prev,
+ {
+ id: `${Date.now()}-${Math.random()}`,
+ role,
+ parts: [{ type: "text", text }],
+ },
+ ]);
+ }, []);
+
+ const handleOrbClick = async () => {
+ if (status !== "disconnected") return;
+
+ try {
+ setStatus("connecting");
+ setError(null);
+
+ console.log("๐ Fetching ephemeral token...");
+ const tokenResponse = await fetch("/token", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ });
+
+ if (!tokenResponse.ok) {
+ throw new Error("Failed to fetch token");
+ }
+
+ const { token } = await tokenResponse.json();
+ console.log("โ Token received");
+
+ const session = new RealtimeSession(agent, {
+ model: AGENT_CONFIG.model,
+ });
+
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ session.transport.on("*", (event: any) => {
+ console.log("๐ Session event:", event.type);
+ });
+
+ session.transport.on(
+ "input_audio_buffer.speech_started",
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any
+ (_event: any) => {
+ console.log("๐ค User started speaking");
+ setIsUserSpeaking(true);
+ }
+ );
+
+ session.transport.on(
+ "input_audio_buffer.speech_stopped",
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any
+ (_event: any) => {
+ console.log("๐ค User stopped speaking");
+ setIsUserSpeaking(false);
+ }
+ );
+
+ session.transport.on(
+ "conversation.item.input_audio_transcription.completed",
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ (event: any) => {
+ console.log("๐ User transcript:", event.transcript);
+ if (event.transcript && event.transcript.trim()) {
+ addMessage("user", event.transcript);
+ }
+ }
+ );
+
+ session.transport.on(
+ "response.output_audio_transcript.delta",
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ (event: any) => {
+ console.log("๐ Agent text delta:", event.delta);
+ }
+ );
+
+ session.transport.on(
+ "response.output_audio_transcript.done",
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ (event: any) => {
+ console.log("โ Agent text done:", event.transcript);
+ if (event.transcript && event.transcript.trim()) {
+ addMessage("agent", event.transcript);
+ }
+ }
+ );
+
+ session.transport.on(
+ "output_audio_buffer.started",
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any
+ (_event: any) => {
+ console.log("๐ Agent audio started");
+ setIsAgentSpeaking(true);
+ }
+ );
+
+ session.transport.on(
+ "response.output_audio.done",
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any
+ (_event: any) => {
+ console.log("๐ Agent audio done");
+ setIsAgentSpeaking(false);
+ }
+ );
+
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ session.transport.on("error", (error: any) => {
+ console.error("โ Session error:", error);
+ setError(`Error: ${error.message || String(error)}`);
+ });
+
+ console.log("๐ Connecting to OpenAI Realtime API with token...");
+
+ try {
+ await session.connect({ apiKey: token });
+ console.log("โ Session.connect() completed");
+ } catch (connectError) {
+ console.error("โ Connection error details:", connectError);
+ throw connectError;
+ }
+
+ sessionRef.current = session;
+ setStatus("connected");
+ setIsConversationStarted(true);
+
+ console.log("โ Connected to Realtime API");
+ } catch (error) {
+ console.error("โ Connection failed:", error);
+ setStatus("disconnected");
+ setError(error instanceof Error ? error.message : String(error));
+ }
+ };
+
+ const handleDisconnect = async () => {
+ if (sessionRef.current) {
+ try {
+ await sessionRef.current.close();
+ } catch (error) {
+ console.warn("Error closing session:", error);
+ }
+ sessionRef.current = null;
+ }
+
+ setStatus("disconnected");
+ setIsUserSpeaking(false);
+ setIsAgentSpeaking(false);
+ setIsConversationStarted(false);
+ setMessages([]); // Clear all messages
+ setError(null); // Clear any errors
+ console.log("๐ Disconnected");
+ };
+
+ if (!isConversationStarted) {
+ return (
+
+ {/* Animated background effects */}
+
+
+
+
+
+
+
+
+
+
+ {/* Interactive Orb */}
+
+
+
+
+ {/* Title and Description */}
+
+
+ Vegetarian Recipe Agent
+
+
+ Click the orb to start your voice-powered cooking assistant. Get
+ personalized recipe recommendations through natural
+ conversation.
+