diff --git a/README.md b/README.md index 8c4f0d6..a5b8087 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ElevenLabs Monorepo for NPM Package -This repository contains multiple package published on npm under `@elevenlabs` scope. +This repository contains multiple package published on npm under `@elevenlabs` scope. Separate packages can be found in the `packages` folder. ![LOGO](https://github.com/elevenlabs/elevenlabs-python/assets/12028621/21267d89-5e82-4e7e-9c81-caf30b237683) @@ -42,23 +42,24 @@ pnpm link --global pnpm link --global ``` -You can run `pnpm run dev` to automatically apply changes to your project. +You can run `pnpm run dev` to automatically apply changes to your project. Note that many projects don't watch for changes inside of `node_modules` folder to rebuild. You might have to restart the application, or modify you setup to watch for node_modules (possible development performance implications). +Also note that the above won't work with turbopack projects. Instead use webpack to develop locally. Don't forget to run the `unlink` equivalent once you're done, to prevent confusion in the future. ## Creating New Package -You can always just add a new folder with package.json inside of `packages` folder. +You can always just add a new folder with package.json inside of `packages` folder. Alternatively run `pnpm run create --name=[package-name]` in the root of this repository to create a new package from template. ## Publishing -To publish a package from the packages folder, create new GitHub release. +To publish a package from the packages folder, create new GitHub release. Since there are multiple packages contained in this folder, the release name/tag should follow format `@version`. -The release will trigger GitHub action publishing the package, and the tag will be used to publish specific package. +The release will trigger GitHub action publishing the package, and the tag will be used to publish specific package. -The GitHub action will only run the publish command. Make sure you've update the version number manually in `package.json`. +The GitHub action will only run the publish command. Make sure you've update the version number manually in `package.json`. diff --git a/package.json b/package.json index 79816e5..785a0f0 100644 --- a/package.json +++ b/package.json @@ -23,5 +23,6 @@ "esbuild", "msw" ] - } + }, + "packageManager": "pnpm@9.15.4+sha512.b2dc20e2fc72b3e18848459b37359a32064663e5627a51e4c74b2c29dd8e8e0491483c3abb40789cfd578bf362fb6ba8261b05f0387d76792ed6e23ea3b1b6a0" } diff --git a/packages/client/package.json b/packages/client/package.json index 2cd28f6..1d91f89 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -44,5 +44,8 @@ "type": "git", "url": "git+https://github.com/elevenlabs/packages.git", "directory": "packages/client" + }, + "dependencies": { + "livekit-client": "^2.9.5" } } diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index 8da78fe..feaa8e0 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -1,18 +1,17 @@ import { arrayBufferToBase64, base64ToArrayBuffer } from "./utils/audio"; -import { Input, InputConfig } from "./utils/input"; +import { Input } from "./utils/input"; +import type { InputConfig } from "./utils/input"; import { Output } from "./utils/output"; -import { - Connection, - DisconnectionDetails, - OnDisconnectCallback, - SessionConfig, -} from "./utils/connection"; -import { ClientToolCallEvent, IncomingSocketEvent } from "./utils/events"; +import type { ClientToolCallEvent, IncomingSocketEvent } from "./utils/events"; import { isAndroidDevice, isIosDevice } from "./utils/compatibility"; +import { WSSConnection } from "./utils/connection/wssConnection"; +import type { SessionConfig, DisconnectionDetails, OnDisconnectCallback } from "./utils/connection/connection.interface"; +import type { Connection } from "./utils/connection/connection"; + export type { InputConfig } from "./utils/input"; export type { IncomingSocketEvent } from "./utils/events"; -export type { SessionConfig, DisconnectionDetails, Language } from "./utils/connection"; +export type { SessionConfig, DisconnectionDetails, Language, ConnectionType } from "./utils/connection/connection.interface"; export type Role = "user" | "ai"; export type Mode = "speaking" | "listening"; export type Status = @@ -28,16 +27,16 @@ export type ClientToolsConfig = { clientTools: Record< string, ( - parameters: any - ) => Promise | string | number | void + parameters: unknown + ) => Promise | string | number | null >; }; export type Callbacks = { onConnect: (props: { conversationId: string }) => void; // internal debug events, not to be used - onDebug: (props: any) => void; + onDebug: (props: unknown) => void; onDisconnect: OnDisconnectCallback; - onError: (message: string, context?: any) => void; + onError: (message: string, context?: unknown) => void; onMessage: (props: { message: string; source: Role }) => void; onModeChange: (prop: { mode: Mode }) => void; onStatusChange: (prop: { status: Status }) => void; @@ -105,7 +104,8 @@ export class Conversation { await new Promise(resolve => setTimeout(resolve, delay)); } - connection = await Connection.create(options); + connection = await WSSConnection.create(options); + [input, output] = await Promise.all([ Input.create({ ...connection.inputFormat, @@ -114,13 +114,17 @@ export class Conversation { Output.create(connection.outputFormat), ]); - preliminaryInputStream?.getTracks().forEach(track => track.stop()); + for (const track of preliminaryInputStream.getTracks()) { + track.stop(); + } preliminaryInputStream = null; return new Conversation(fullOptions, connection, input, output); } catch (error) { fullOptions.onStatusChange({ status: "disconnected" }); - preliminaryInputStream?.getTracks().forEach(track => track.stop()); + for (const track of preliminaryInputStream?.getTracks() ?? []) { + track.stop(); + } connection?.close(); await input?.close(); await output?.close(); @@ -128,15 +132,15 @@ export class Conversation { } } - private lastInterruptTimestamp: number = 0; + private lastInterruptTimestamp = 0; private mode: Mode = "listening"; private status: Status = "connecting"; private inputFrequencyData?: Uint8Array; private outputFrequencyData?: Uint8Array; - private volume: number = 1; - private currentEventId: number = 1; - private lastFeedbackEventId: number = 1; - private canSendFeedback: boolean = false; + private volume = 1; + private currentEventId = 1; + private lastFeedbackEventId = 1; + private canSendFeedback = false; private constructor( private readonly options: Options, @@ -229,7 +233,8 @@ export class Conversation { case "client_tool_call": { console.info("Received client tool call request", parsedEvent.client_tool_call); if ( - this.options.clientTools.hasOwnProperty( + Object.prototype.hasOwnProperty.call( + this.options.clientTools, parsedEvent.client_tool_call.tool_name ) ) { @@ -250,17 +255,18 @@ export class Conversation { is_error: false, }); } catch (e) { + const errorMessage = e instanceof Error ? e.message : String(e); this.onError( - "Client tool execution failed with following error: " + - (e as Error)?.message, + `Client tool execution failed with following error: ${errorMessage}`, { clientToolName: parsedEvent.client_tool_call.tool_name, } ); + this.connection.sendMessage({ type: "client_tool_result", tool_call_id: parsedEvent.client_tool_call.tool_call_id, - result: "Client tool execution failed: " + (e as Error)?.message, + result: `Client tool execution failed: ${errorMessage}`, is_error: true, }); } @@ -363,7 +369,7 @@ export class Conversation { }, 2000); // Adjust the duration as needed }; - private onError = (message: string, context?: any) => { + private onError = (message: string, context?: unknown) => { console.error(message, context); this.options.onError(message, context); }; diff --git a/packages/client/src/utils/connection.ts b/packages/client/src/utils/connection.ts index e20ba2a..1384cab 100644 --- a/packages/client/src/utils/connection.ts +++ b/packages/client/src/utils/connection.ts @@ -1,10 +1,11 @@ -import { +import type { InitiationClientDataEvent, ConfigEvent, - isValidSocketEvent, OutgoingSocketEvent, IncomingSocketEvent, } from "./events"; +import { isValidSocketEvent } from "./events"; +import { Room, RoomEvent } from "livekit-client"; const MAIN_PROTOCOL = "convai"; @@ -53,13 +54,14 @@ export type SessionConfig = { voiceId?: string; }; }; - customLlmExtraBody?: any; + customLlmExtraBody?: unknown; dynamicVariables?: Record; connectionDelay?: { default: number; android?: number; ios?: number; }; + connectionType?: ConnectionType; } & ( | { signedUrl: string; agentId?: undefined } | { agentId: string; signedUrl?: undefined } @@ -84,11 +86,34 @@ export type DisconnectionDetails = export type OnDisconnectCallback = (details: DisconnectionDetails) => void; export type OnMessageCallback = (event: IncomingSocketEvent) => void; +export enum ConnectionType { + WEBSOCKET = "websocket", + WEBRTC = "webrtc", +} + const WSS_API_ORIGIN = "wss://api.elevenlabs.io"; const WSS_API_PATHNAME = "/v1/convai/conversation?agent_id="; +const WEBRTC_TOKEN_API_ORIGIN = "http://localhost:3000"; +const WEBRTC_TOKEN_PATHNAME = "/api/token"; +const WEBRTC_API_ORIGIN = "wss://livekit.rtc.eleven2.dev" + export class Connection { public static async create(config: SessionConfig): Promise { + return config.connectionType === ConnectionType.WEBSOCKET + ? Connection.createWebSocketConnection(config) + : Connection.createWebRTCConnection(config); + } + + private static async createWebRTCConnection(config: SessionConfig): Promise { + + + + return await Connection.createWebSocketConnection(config); + + } + + private static async createWebSocketConnection(config: SessionConfig): Promise { let socket: WebSocket | null = null; try { @@ -105,46 +130,28 @@ export class Connection { const conversationConfig = await new Promise< ConfigEvent["conversation_initiation_metadata_event"] >((resolve, reject) => { - socket!.addEventListener( + if (!socket) { + reject(new Error("Socket is not initialized")); + return; + } + + socket.addEventListener( "open", () => { - const overridesEvent: InitiationClientDataEvent = { - type: "conversation_initiation_client_data", - }; - - if (config.overrides) { - overridesEvent.conversation_config_override = { - agent: { - prompt: config.overrides.agent?.prompt, - first_message: config.overrides.agent?.firstMessage, - language: config.overrides.agent?.language, - }, - tts: { - voice_id: config.overrides.tts?.voiceId, - }, - }; - } + const overridesEvent = Connection.getOverridesEvent(config); - if (config.customLlmExtraBody) { - overridesEvent.custom_llm_extra_body = config.customLlmExtraBody; - } - - if (config.dynamicVariables) { - overridesEvent.dynamic_variables = config.dynamicVariables; - } - - socket?.send(JSON.stringify(overridesEvent)); + socket!.send(JSON.stringify(overridesEvent)); }, { once: true } ); - socket!.addEventListener("error", event => { + socket.addEventListener("error", event => { // In case the error event is followed by a close event, we want the // latter to be the one that rejects the promise as it contains more // useful information. setTimeout(() => reject(event), 0); }); - socket!.addEventListener("close", reject); - socket!.addEventListener( + socket.addEventListener("close", reject); + socket.addEventListener( "message", (event: MessageEvent) => { const message = JSON.parse(event.data); @@ -181,6 +188,35 @@ export class Connection { } } + private static getOverridesEvent(config: SessionConfig): InitiationClientDataEvent { + const overridesEvent: InitiationClientDataEvent = { + type: "conversation_initiation_client_data", + }; + + if (config.overrides) { + overridesEvent.conversation_config_override = { + agent: { + prompt: config.overrides.agent?.prompt, + first_message: config.overrides.agent?.firstMessage, + language: config.overrides.agent?.language, + }, + tts: { + voice_id: config.overrides.tts?.voiceId, + }, + }; + } + + if (config.customLlmExtraBody) { + overridesEvent.custom_llm_extra_body = config.customLlmExtraBody; + } + + if (config.dynamicVariables) { + overridesEvent.dynamic_variables = config.dynamicVariables; + } + + return overridesEvent; + } + private queue: IncomingSocketEvent[] = []; private disconnectionDetails: DisconnectionDetails | null = null; private onDisconnectCallback: OnDisconnectCallback | null = null; @@ -272,8 +308,8 @@ function parseFormat(format: string): FormatConfig { throw new Error(`Invalid format: ${format}`); } - const sampleRate = parseInt(sampleRatePart); - if (isNaN(sampleRate)) { + const sampleRate = Number.parseInt(sampleRatePart); + if (Number.isNaN(sampleRate)) { throw new Error(`Invalid sample rate: ${sampleRatePart}`); } diff --git a/packages/client/src/utils/connection/connection.interface.ts b/packages/client/src/utils/connection/connection.interface.ts new file mode 100644 index 0000000..b6a3d5d --- /dev/null +++ b/packages/client/src/utils/connection/connection.interface.ts @@ -0,0 +1,94 @@ +import type { IncomingSocketEvent, OutgoingSocketEvent } from "../events"; + +export type Language = + | "en" + | "ja" + | "zh" + | "de" + | "hi" + | "fr" + | "ko" + | "pt" + | "it" + | "es" + | "id" + | "nl" + | "tr" + | "pl" + | "sv" + | "bg" + | "ro" + | "ar" + | "cs" + | "el" + | "fi" + | "ms" + | "da" + | "ta" + | "uk" + | "ru" + | "hu" + | "no" + | "vi"; + +export type OnDisconnectCallback = (details: DisconnectionDetails) => void; +export type OnMessageCallback = (event: IncomingSocketEvent) => void; + +export type DisconnectionDetails = + | { + reason: "error"; + message: string; + context: Event; + } + | { + reason: "agent"; + context: CloseEvent; + } + | { + reason: "user"; + }; + +export type FormatConfig = { + format: "pcm" | "ulaw"; + sampleRate: number; +}; + +export type SessionConfig = { + origin?: string; + authorization?: string; + overrides?: { + agent?: { + prompt?: { + prompt?: string; + }; + firstMessage?: string; + language?: Language; + }; + tts?: { + voiceId?: string; + }; + }; + customLlmExtraBody?: unknown; + dynamicVariables?: Record; + connectionDelay?: { + default: number; + android?: number; + ios?: number; + }; + connectionType?: ConnectionType; +} & ( + | { signedUrl: string; agentId?: undefined } + | { agentId: string; signedUrl?: undefined } +); + +export enum ConnectionType { + WEBSOCKET = "websocket", + WEBRTC = "webrtc", +} + +export interface ConnectionInterface { + close: () => void; + sendMessage: (message: OutgoingSocketEvent) => void; + onMessage: (callback: OnMessageCallback) => void; + onDisconnect: (callback: OnDisconnectCallback) => void; +} \ No newline at end of file diff --git a/packages/client/src/utils/connection/connection.ts b/packages/client/src/utils/connection/connection.ts new file mode 100644 index 0000000..f30234a --- /dev/null +++ b/packages/client/src/utils/connection/connection.ts @@ -0,0 +1,92 @@ +import type { + ConnectionInterface, + SessionConfig, + OnMessageCallback, + OnDisconnectCallback, + FormatConfig, + DisconnectionDetails, +} from "./connection.interface"; + +import type { + InitiationClientDataEvent, + OutgoingSocketEvent, +} from "../events"; +import type { Room } from "livekit-client"; + +export abstract class Connection implements ConnectionInterface { + protected disconnectionDetails: DisconnectionDetails | null = null; + protected onDisconnectCallback: OnDisconnectCallback | null = null; + protected onMessageCallback: OnMessageCallback | null = null; + + constructor( + readonly connection: WebSocket | Room, + readonly conversationId: string, + readonly inputFormat: FormatConfig, + readonly outputFormat: FormatConfig, + ) {} + + abstract create(config: SessionConfig): Promise; + + abstract close(): void; + + abstract sendMessage(message: OutgoingSocketEvent): void; + + abstract onMessage(callback: OnMessageCallback): void; + + abstract onDisconnect(callback: OnDisconnectCallback): void; + + protected disconnect(details: DisconnectionDetails) { + if (!this.disconnectionDetails) { + this.disconnectionDetails = details; + this.onDisconnectCallback?.(details); + } + } + + protected static getOverridesEvent( + config: SessionConfig, + ): InitiationClientDataEvent { + const overridesEvent: InitiationClientDataEvent = { + type: "conversation_initiation_client_data", + }; + + if (config.overrides) { + overridesEvent.conversation_config_override = { + agent: { + prompt: config.overrides.agent?.prompt, + first_message: config.overrides.agent?.firstMessage, + language: config.overrides.agent?.language, + }, + tts: { + voice_id: config.overrides.tts?.voiceId, + }, + }; + } + + if (config.customLlmExtraBody) { + overridesEvent.custom_llm_extra_body = config.customLlmExtraBody; + } + + if (config.dynamicVariables) { + overridesEvent.dynamic_variables = config.dynamicVariables; + } + + return overridesEvent; + } + + protected static parseFormat(format: string): FormatConfig { + const [formatPart, sampleRatePart] = format.split("_"); + if (!["pcm", "ulaw"].includes(formatPart)) { + throw new Error(`Invalid format: ${format}`); + } + + const sampleRate = Number.parseInt(sampleRatePart); + if (Number.isNaN(sampleRate)) { + throw new Error(`Invalid sample rate: ${sampleRatePart}`); + } + + return { + format: formatPart as FormatConfig["format"], + sampleRate, + }; + } +} diff --git a/packages/client/src/utils/connection/webRTConnection.ts b/packages/client/src/utils/connection/webRTConnection.ts new file mode 100644 index 0000000..a5771b0 --- /dev/null +++ b/packages/client/src/utils/connection/webRTConnection.ts @@ -0,0 +1,143 @@ +import { RoomEvent, Track } from "livekit-client"; +import { Room } from "livekit-client"; +import { Connection } from "./connection"; +import type { + ConnectionInterface, + FormatConfig, + OnDisconnectCallback, + OnMessageCallback, + SessionConfig, +} from "./connection.interface"; +import type { ConfigEvent, IncomingSocketEvent, OutgoingSocketEvent } from "../events"; + +export class WebRTCConnection + extends Connection + implements ConnectionInterface +{ + private static readonly WEBRTC_TOKEN_API_ORIGIN = "http://localhost:3000"; + private static readonly WEBRTC_TOKEN_PATHNAME = "/api/token"; + private static readonly WEBRTC_API_ORIGIN = "wss://livekit.rtc.eleven2.dev"; + + public static async create(config: SessionConfig): Promise { + let room: Room; + + try { + // Get token from server + const { token } = await fetch( + `${WebRTCConnection.WEBRTC_TOKEN_API_ORIGIN}/${WebRTCConnection.WEBRTC_TOKEN_PATHNAME}?agent_id=${config.agentId}`, + ).then((res) => res.json()); + room = new Room(); + + // if webrtc becomes the default, use the following on start up + //await room.prepareConnection(WebRTConnection.WEBRTC_API_ORIGIN, token); + + await room.connect(WebRTCConnection.WEBRTC_API_ORIGIN, token, { + autoSubscribe: true, + }); + } catch (error) { + console.error("Error getting token:", error); + throw error; + } + + const overridesEvent = Connection.getOverridesEvent(config); + room.localParticipant.sendText(JSON.stringify(overridesEvent)); + + // When the agent leaves the room + room.on(RoomEvent.ParticipantDisconnected, (participant) => { + console.log("Participant disconnected:", participant); + }); + + room.on(RoomEvent.Disconnected, () => {}); + + const conversationConfig = await new Promise< + ConfigEvent["conversation_initiation_metadata_event"] + >((resolve, reject) => { + room.registerTextStreamHandler( + "conversation_initiation_metadata", + (reader) => { + reader.readAll() + .then(text => { + resolve(JSON.parse(text)); + room.unregisterTextStreamHandler("conversation_initiation_metadata"); + }) + .catch(reject); + }, + ); + + const overridesEvent = Connection.getOverridesEvent(config); + + room.localParticipant.sendText(JSON.stringify(overridesEvent), { + topic: 'conversation_initiation_client_data', + }).catch(reject); + }); + + const { + conversation_id, + agent_output_audio_format, + user_input_audio_format, + } = conversationConfig; + + const inputFormat = Connection.parseFormat( + user_input_audio_format ?? "pcm_16000", + ); + const outputFormat = Connection.parseFormat(agent_output_audio_format); + + return new WebRTCConnection( + room, + conversation_id, + inputFormat, + outputFormat, + ); + } + + private constructor( + public readonly room: Room, + public readonly conversationId: string, + public readonly inputFormat: FormatConfig, + public readonly outputFormat: FormatConfig, + ) { + super(room, conversationId, inputFormat, outputFormat); + + // publish mic track + room.localParticipant.setMicrophoneEnabled(true); + + // set up event listeners + room + .on(RoomEvent.TrackSubscribed, (track, publication, participant) => { + if (track.kind === Track.Kind.Audio) { + + } + + }) + .on(RoomEvent.ParticipantDisconnected, () => { + this.disconnect({ + reason: "agent", + context: new CloseEvent("Agent disconnected"), + }) + }) + } + + async create(config: SessionConfig): Promise { + return WebRTCConnection.create(config); + } + + close() { + this.room.disconnect(); + } + + sendMessage(message: OutgoingSocketEvent) { + this.room.localParticipant.sendText(JSON.stringify(message)); + } + + onMessage(callback: OnMessageCallback) { + this.onMessageCallback = callback; + + } + + onDisconnect(callback: OnDisconnectCallback) { + this.onDisconnectCallback = callback; + if (this.disconnectionDetails) { + callback(this.disconnectionDetails); + } + } +} diff --git a/packages/client/src/utils/connection/wssConnection.ts b/packages/client/src/utils/connection/wssConnection.ts new file mode 100644 index 0000000..dff7dfd --- /dev/null +++ b/packages/client/src/utils/connection/wssConnection.ts @@ -0,0 +1,178 @@ +import { isValidSocketEvent } from "../events"; +import type { + ConnectionInterface, + FormatConfig, + OnDisconnectCallback, + OnMessageCallback, + SessionConfig, +} from "./connection.interface"; +import { Connection } from "./connection"; +import type { ConfigEvent, IncomingSocketEvent, OutgoingSocketEvent } from "../events"; + +export class WSSConnection extends Connection implements ConnectionInterface { + private static readonly WSS_API_ORIGIN = "wss://api.elevenlabs.io"; + private static readonly WSS_API_PATHNAME = "/v1/convai/conversation?agent_id="; + private static readonly MAIN_PROTOCOL = "convai"; + protected queue: IncomingSocketEvent[] = []; + + public static async create(config: SessionConfig): Promise { + let socket: WebSocket | null = null; + + try { + const origin = config.origin ?? WSSConnection.WSS_API_ORIGIN; + const url = config.signedUrl + ? config.signedUrl + : origin + WSSConnection.WSS_API_PATHNAME + config.agentId; + + const protocols = [WSSConnection.MAIN_PROTOCOL]; + if (config.authorization) { + protocols.push(`bearer.${config.authorization}`); + } + socket = new WebSocket(url, protocols); + const conversationConfig = await new Promise< + ConfigEvent["conversation_initiation_metadata_event"] + >((resolve, reject) => { + if (!socket) { + reject(new Error("Socket is not initialized")); + return; + } + + socket.addEventListener( + "open", + () => { + const overridesEvent = Connection.getOverridesEvent(config); + + if (!socket) return; + socket.send(JSON.stringify(overridesEvent)); + }, + { once: true }, + ); + socket.addEventListener("error", (event) => { + // In case the error event is followed by a close event, we want the + // latter to be the one that rejects the promise as it contains more + // useful information. + setTimeout(() => reject(event), 0); + }); + socket.addEventListener("close", reject); + socket.addEventListener( + "message", + (event: MessageEvent) => { + const message = JSON.parse(event.data); + + if (!isValidSocketEvent(message)) { + return; + } + + if (message.type === "conversation_initiation_metadata") { + resolve(message.conversation_initiation_metadata_event); + } else { + console.warn( + "First received message is not conversation metadata.", + ); + } + }, + { once: true }, + ); + }); + + const { + conversation_id, + agent_output_audio_format, + user_input_audio_format, + } = conversationConfig; + + const inputFormat = Connection.parseFormat( + user_input_audio_format ?? "pcm_16000", + ); + const outputFormat = Connection.parseFormat(agent_output_audio_format); + + return new WSSConnection( + socket, + conversation_id, + inputFormat, + outputFormat, + ); + } catch (error) { + socket?.close(); + throw error; + } + } + + private constructor( + public readonly socket: WebSocket, + public readonly conversationId: string, + public readonly inputFormat: FormatConfig, + public readonly outputFormat: FormatConfig + ) { + super(socket, conversationId, inputFormat, outputFormat); + + this.socket.addEventListener("error", event => { + // In case the error event is followed by a close event, we want the + // latter to be the one that disconnects the session as it contains more + // useful information. + setTimeout( + () => + this.disconnect({ + reason: "error", + message: "The connection was closed due to a socket error.", + context: event, + }), + 0 + ); + }); + this.socket.addEventListener("close", event => { + this.disconnect( + event.code === 1000 + ? { + reason: "agent", + context: event, + } + : { + reason: "error", + message: + event.reason || "The connection was closed by the server.", + context: event, + } + ); + }); + this.socket.addEventListener("message", event => { + try { + const parsedEvent = JSON.parse(event.data); + if (!isValidSocketEvent(parsedEvent)) { + return; + } + + if (this.onMessageCallback) { + this.onMessageCallback(parsedEvent); + } else { + this.queue.push(parsedEvent); + } + } catch (_) {} + }); + } + + async create(config: SessionConfig): Promise { + return WSSConnection.create(config); + } + + close() { + this.socket.close(); + } + + sendMessage(message: OutgoingSocketEvent) { + this.socket.send(JSON.stringify(message)); + } + + onMessage(callback: OnMessageCallback) { + this.onMessageCallback = callback; + this.queue.forEach(callback); + this.queue = []; + } + + onDisconnect(callback: OnDisconnectCallback) { + this.onDisconnectCallback = callback; + if (this.disconnectionDetails) { + callback(this.disconnectionDetails); + } + } +} diff --git a/packages/client/src/utils/events.ts b/packages/client/src/utils/events.ts index 45059b8..938c451 100644 --- a/packages/client/src/utils/events.ts +++ b/packages/client/src/utils/events.ts @@ -1,4 +1,4 @@ -import { Language } from "./connection"; +import type { Language } from "./connection"; export type UserTranscriptionEvent = { type: "user_transcript"; @@ -47,7 +47,7 @@ export type ClientToolCallEvent = { client_tool_call: { tool_name: string; tool_call_id: string; - parameters: any; + parameters: unknown; expects_response: boolean; }; }; @@ -78,7 +78,7 @@ export type UserFeedbackEvent = { export type ClientToolResultEvent = { type: "client_tool_result"; tool_call_id: string; - result: any; + result: unknown; is_error: boolean; }; export type InitiationClientDataEvent = { @@ -95,7 +95,7 @@ export type InitiationClientDataEvent = { voice_id?: string; }; }; - custom_llm_extra_body?: any; + custom_llm_extra_body?: unknown; dynamic_variables?: Record; }; export type OutgoingSocketEvent = diff --git a/packages/client/src/utils/output.ts b/packages/client/src/utils/output.ts index 512fd79..dfbcd3d 100644 --- a/packages/client/src/utils/output.ts +++ b/packages/client/src/utils/output.ts @@ -1,5 +1,5 @@ import { audioConcatProcessor } from "./audioConcatProcessor"; -import { FormatConfig } from "./connection"; +import type { FormatConfig } from "./connection/connection.interface"; export class Output { public static async create({ diff --git a/packages/react/package.json b/packages/react/package.json index aa25249..07a7125 100644 --- a/packages/react/package.json +++ b/packages/react/package.json @@ -30,7 +30,7 @@ "author": "ElevenLabs", "license": "MIT", "dependencies": { - "@11labs/client": "workspace:*" + "@11labs/client": "file:../client" }, "peerDependencies": { "react": ">=16.8.0" diff --git a/packages/react/src/index.ts b/packages/react/src/index.ts index 76b7a3d..0b432a8 100644 --- a/packages/react/src/index.ts +++ b/packages/react/src/index.ts @@ -1,13 +1,13 @@ import { useEffect, useRef, useState } from "react"; import { Conversation, - Mode, - SessionConfig, - Callbacks, - Options, - Status, - ClientToolsConfig, - InputConfig, + type Mode, + type SessionConfig, + type Callbacks, + type Options, + type Status, + type ClientToolsConfig, + type InputConfig, } from "@11labs/client"; export type { @@ -45,6 +45,7 @@ export function useConversation(props: const [canSendFeedback, setCanSendFeedback] = useState(false); const [mode, setMode] = useState("listening"); + useEffect(() => { if (micMuted !== undefined) { conversationRef?.current?.setMicMuted(micMuted); @@ -78,13 +79,13 @@ export function useConversation(props: lockRef.current = Conversation.startSession({ ...(defaultOptions ?? {}), ...(options ?? {}), - onModeChange: ({ mode }) => { + onModeChange: ({ mode }: { mode: Mode }) => { setMode(mode); }, - onStatusChange: ({ status }) => { + onStatusChange: ({ status }: { status: Status }) => { setStatus(status); }, - onCanSendFeedbackChange: ({ canSendFeedback }) => { + onCanSendFeedbackChange: ({ canSendFeedback }: { canSendFeedback: boolean }) => { setCanSendFeedback(canSendFeedback); }, } as Options); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c0114cf..48b84a0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -22,6 +22,10 @@ importers: version: 3.3.3 packages/client: + dependencies: + livekit-client: + specifier: ^2.9.5 + version: 2.9.5 devDependencies: '@types/node-wav': specifier: ^0.0.3 @@ -54,7 +58,7 @@ importers: packages/react: dependencies: '@11labs/client': - specifier: workspace:* + specifier: file:../client version: link:../client react: specifier: '>=16.8.0' @@ -731,6 +735,9 @@ packages: '@bcoe/v8-coverage@0.2.3': resolution: {integrity: sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==} + '@bufbuild/protobuf@1.10.0': + resolution: {integrity: sha512-QDdVFLoN93Zjg36NoQPZfsVH9tZew7wKDKyV5qRdj8ntT4wQCOradQjRaTdwMhWUYsgKsvCINKKm87FdEk96Ag==} + '@bundled-es-modules/cookie@2.0.1': resolution: {integrity: sha512-8o+5fRPLNbjbdGRRmJj3h6Hh1AQJf2dk3qQ/5ZFb+PXkRNiSoMGGUKlsgLfrxneb72axVJyIYji64E2+nNfYyw==} @@ -1038,6 +1045,12 @@ packages: '@jridgewell/trace-mapping@0.3.25': resolution: {integrity: sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==} + '@livekit/mutex@1.1.1': + resolution: {integrity: sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==} + + '@livekit/protocol@1.33.0': + resolution: {integrity: sha512-361mBlFgI3nvn8oSQIL38gDUBGbOSwsEOqPgX0c1Jwz75/sD/TTvPeAM4zAz6OrV5Q4vI4Ruswecnyv5SG4oig==} + '@mswjs/interceptors@0.37.6': resolution: {integrity: sha512-wK+5pLK5XFmgtH3aQ2YVvA3HohS3xqV/OxuVOdNx9Wpnz7VE/fnC+e1A7ln6LFYeck7gOJ/dsZV6OLplOtAJ2w==} engines: {node: '>=18'} @@ -1908,6 +1921,10 @@ packages: eventemitter3@4.0.7: resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==} + events@3.3.0: + resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} + engines: {node: '>=0.8.x'} + execa@5.1.1: resolution: {integrity: sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==} engines: {node: '>=10'} @@ -2524,6 +2541,9 @@ packages: lines-and-columns@1.2.4: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} + livekit-client@2.9.5: + resolution: {integrity: sha512-2EJmiB4XItaRjTEmL4XxGzsahLYTer9T5N6lKyhBHQxwH4GrjBWewPySvJEO8zCpD2nvWZCmCQjIJx0+w+y6DA==} + loader-utils@3.3.1: resolution: {integrity: sha512-FMJTLMXfCLMLfJxcX9PFqX5qD88Z5MRGaZCVzfuqeZSPsyiBzs+pahDQjbIWz2QIzPZz0NX9Zy4FX3lmK6YHIg==} engines: {node: '>= 12.13.0'} @@ -2551,6 +2571,10 @@ packages: lodash.uniq@4.5.0: resolution: {integrity: sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==} + loglevel@1.9.2: + resolution: {integrity: sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg==} + engines: {node: '>= 0.6.0'} + loose-envify@1.4.0: resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} hasBin: true @@ -3209,6 +3233,9 @@ packages: run-parallel@1.2.0: resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} + rxjs@7.8.2: + resolution: {integrity: sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==} + sade@1.8.1: resolution: {integrity: sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==} engines: {node: '>=6'} @@ -3227,6 +3254,13 @@ packages: resolution: {integrity: sha512-CdASjNJPvRa7roO6Ra/gLYBTzYzzPyyBXxIMdGW3USQLyjWEls2RgW5UBTXaQVp+OrpeCK3bLem8smtmheoRuw==} engines: {node: '>= 0.4'} + sdp-transform@2.15.0: + resolution: {integrity: sha512-KrOH82c/W+GYQ0LHqtr3caRpM3ITglq3ljGUIb8LTki7ByacJZ9z+piSGiwZDsRyhQbYBOBJgr2k6X4BZXi3Kw==} + hasBin: true + + sdp@3.2.0: + resolution: {integrity: sha512-d7wDPgDV3DDiqulJjKiV2865wKsJ34YI+NDREbm+FySq6WuKOikwyNQcm+doLAZ1O6ltdO0SeKle2xMpN3Brgw==} + semver@6.3.1: resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} hasBin: true @@ -3458,9 +3492,15 @@ packages: resolution: {integrity: sha512-Loo5UUvLD9ScZ6jh8beX1T6sO1w2/MpCRpEP7V280GKMVUQ0Jzar2U3UJPsrdbziLEMMhu3Ujnq//rhiFuIeag==} engines: {node: '>=6'} + ts-debounce@4.0.0: + resolution: {integrity: sha512-+1iDGY6NmOGidq7i7xZGA4cm8DAa6fqdYcvO5Z6yBevH++Bdo9Qt/mN0TzHUgcCcKv1gmh9+W5dHqz8pMWbCbg==} + tslib@2.6.3: resolution: {integrity: sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==} + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} @@ -3493,6 +3533,9 @@ packages: resolution: {integrity: sha512-/OxDN6OtAk5KBpGb28T+HZc2M+ADtvRxXrKKbUwtsLgdoxgX13hyy7ek6bFRl5+aBs2yZzB0c4CnQfAtVypW/g==} engines: {node: '>= 0.4'} + typed-emitter@2.1.0: + resolution: {integrity: sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA==} + typescript@4.9.5: resolution: {integrity: sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==} engines: {node: '>=4.2.0'} @@ -3622,6 +3665,10 @@ packages: walker@1.0.8: resolution: {integrity: sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==} + webrtc-adapter@9.0.1: + resolution: {integrity: sha512-1AQO+d4ElfVSXyzNVTOewgGT/tAomwwztX/6e3totvyyzXPvXIIuUUjAmyZGbKBKbZOXauuJooZm3g6IuFuiNQ==} + engines: {node: '>=6.0.0', npm: '>=3.10.0'} + which-boxed-primitive@1.0.2: resolution: {integrity: sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==} @@ -4549,6 +4596,8 @@ snapshots: '@bcoe/v8-coverage@0.2.3': {} + '@bufbuild/protobuf@1.10.0': {} + '@bundled-es-modules/cookie@2.0.1': dependencies: cookie: 0.7.2 @@ -4888,6 +4937,12 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.0 + '@livekit/mutex@1.1.1': {} + + '@livekit/protocol@1.33.0': + dependencies: + '@bufbuild/protobuf': 1.10.0 + '@mswjs/interceptors@0.37.6': dependencies: '@open-draft/deferred-promise': 2.2.0 @@ -5911,6 +5966,8 @@ snapshots: eventemitter3@4.0.7: {} + events@3.3.0: {} + execa@5.1.1: dependencies: cross-spawn: 7.0.3 @@ -6680,6 +6737,18 @@ snapshots: lines-and-columns@1.2.4: {} + livekit-client@2.9.5: + dependencies: + '@livekit/mutex': 1.1.1 + '@livekit/protocol': 1.33.0 + events: 3.3.0 + loglevel: 1.9.2 + sdp-transform: 2.15.0 + ts-debounce: 4.0.0 + tslib: 2.8.1 + typed-emitter: 2.1.0 + webrtc-adapter: 9.0.1 + loader-utils@3.3.1: {} locate-path@5.0.0: @@ -6700,6 +6769,8 @@ snapshots: lodash.uniq@4.5.0: {} + loglevel@1.9.2: {} + loose-envify@1.4.0: dependencies: js-tokens: 4.0.0 @@ -7387,6 +7458,11 @@ snapshots: dependencies: queue-microtask: 1.2.3 + rxjs@7.8.2: + dependencies: + tslib: 2.8.1 + optional: true + sade@1.8.1: dependencies: mri: 1.2.0 @@ -7408,6 +7484,10 @@ snapshots: es-errors: 1.3.0 is-regex: 1.1.4 + sdp-transform@2.15.0: {} + + sdp@3.2.0: {} + semver@6.3.1: {} semver@7.6.3: {} @@ -7637,8 +7717,12 @@ snapshots: universalify: 0.2.0 url-parse: 1.5.10 + ts-debounce@4.0.0: {} + tslib@2.6.3: {} + tslib@2.8.1: {} + type-check@0.4.0: dependencies: prelude-ls: 1.2.1 @@ -7681,6 +7765,10 @@ snapshots: is-typed-array: 1.1.13 possible-typed-array-names: 1.0.0 + typed-emitter@2.1.0: + optionalDependencies: + rxjs: 7.8.2 + typescript@4.9.5: {} typescript@5.5.4: {} @@ -7803,6 +7891,10 @@ snapshots: dependencies: makeerror: 1.0.12 + webrtc-adapter@9.0.1: + dependencies: + sdp: 3.2.0 + which-boxed-primitive@1.0.2: dependencies: is-bigint: 1.0.4