Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/concepts/vendors.md
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ MLLM (Multimodal LLM) vendors handle audio end-to-end — no separate STT or TTS
| `GeminiLive` | Google Gemini Live API | `apiKey`, `model`, `url?`, `voice?`, `greetingMessage?`, `failureMessage?`, `inputModalities?`, `outputModalities?`, `messages?`, `turnDetection?` |
| `VertexAI` | Vertex AI Gemini Live | `model`, `url?`, `projectId`, `location`, `adcCredentialsString`, `voice?`, `greetingMessage?`, `failureMessage?`, `inputModalities?`, `outputModalities?`, `messages?`, `turnDetection?` |
| `XaiGrok` | xAI Grok (`mllm.vendor`: `xai`) | `apiKey`, `url?`, `voice?`, `language?`, `sampleRate?`, `greetingMessage?`, `failureMessage?`, `inputModalities?`, `outputModalities?`, `messages?`, `turnDetection?` |
| `QwenOmni` | Alibaba Cloud Qwen Omni (CN) | `apiKey`, `model`, `turnDetection`, `url?`, `voice?`, `greetingMessage?`, `failureMessage?` |
| `QwenOmni` | Alibaba Cloud Qwen Omni (CN) | `apiKey`, `model`, `url`, `voice?`, `greetingMessage?`, `failureMessage?`, `turnDetection?` |

`XAiSTT` and `XAiTTS` are the cascading-pipeline xAI wrappers. `XaiGrok` is the realtime MLLM wrapper.

Expand Down
4 changes: 2 additions & 2 deletions docs/guides/mllm-flow.md
Original file line number Diff line number Diff line change
Expand Up @@ -166,10 +166,10 @@ const client = new AgoraClient({

const agent = new Agent({ client }).withMllm(new QwenOmni({
apiKey: 'your-dashscope-key',
model: 'qwen-omni-turbo-realtime',
model: 'qwen3.5-omni-plus-realtime',
url: 'wss://dashscope.aliyuncs.com/api-ws/v1/realtime',
voice: 'Cherry',
greetingMessage: 'Hello, Qwen Omni is ready.',
turnDetection: { mode: 'server_vad' },
}));
```

Expand Down
4 changes: 2 additions & 2 deletions docs/reference/vendors.md
Original file line number Diff line number Diff line change
Expand Up @@ -510,7 +510,7 @@ Chinese mainland Alibaba Cloud Qwen Omni Realtime wrapper. It emits `mllm.vendor
|---|---|---|---|
| `apiKey` | `string` | Yes | Alibaba Cloud DashScope API key |
| `model` | `string` | Yes | Qwen Omni model identifier |
| `url` | `string` | No | Custom Realtime WebSocket URL; the server-side CN endpoint is used when omitted |
| `url` | `string` | Yes | Qwen Omni Realtime WebSocket URL |
| `voice` | `string` | No | Voice identifier |
| `instructions` | `string` | No | System instructions |
| `greetingMessage` | `string` | No | Agent greeting message |
Expand All @@ -519,7 +519,7 @@ Chinese mainland Alibaba Cloud Qwen Omni Realtime wrapper. It emits `mllm.vendor
| `outputModalities` | `string[]` | No | Output modalities |
| `messages` | `Record<string, unknown>[]` | No | Conversation messages |
| `params` | `Record<string, unknown>` | No | Additional Qwen Omni parameters |
| `turnDetection` | `MllmTurnDetectionConfig` | Yes | MLLM turn detection configuration |
| `turnDetection` | `MllmTurnDetectionConfig` | No | MLLM turn detection configuration |

---

Expand Down
10 changes: 6 additions & 4 deletions src/agentkit/__tests__/area-aware-vendors.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,12 +66,14 @@ type _AzureParamsExposeOnlySupportedFields = Assert<
IsExact<keyof AzureOpenAIRealtimeParams, "instructions" | "model" | "voice">
>;
type _AzureTurnDetectionIsRequired = Assert<IsRequired<AzureOpenAIRealtimeOptions, "turnDetection">>;
type _QwenTurnDetectionIsRequired = Assert<IsRequired<QwenOmniOptions, "turnDetection">>;
type _QwenUrlIsRequired = Assert<IsRequired<QwenOmniOptions, "url">>;
type _QwenTurnDetectionIsOptional = Assert<IsExact<IsRequired<QwenOmniOptions, "turnDetection">, false>>;

// @ts-expect-error Qwen Omni is a Chinese mainland MLLM vendor.
const _invalidGlobalMllm: GlobalMllmVendor = new QwenOmni({
apiKey: "key",
model: "qwen-omni-turbo-realtime",
model: "qwen3.5-omni-plus-realtime",
url: "wss://dashscope.aliyuncs.com/api-ws/v1/realtime",
turnDetection: { mode: "server_vad" },
});

Expand Down Expand Up @@ -114,8 +116,8 @@ const _invalidGlobalMllm: GlobalMllmVendor = new QwenOmni({
new Agent({ client }).withTts(cnGenericTts);
const cnMllm: CNMllmVendor = new QwenOmni({
apiKey: "key",
model: "qwen-omni-turbo-realtime",
turnDetection: { mode: "server_vad" },
model: "qwen3.5-omni-plus-realtime",
url: "wss://dashscope.aliyuncs.com/api-ws/v1/realtime",
});
new Agent({ client }).withMllm(cnMllm);
// @ts-expect-error Azure OpenAI Realtime is a global MLLM vendor.
Expand Down
18 changes: 9 additions & 9 deletions src/agentkit/vendors/mllm.ts
Original file line number Diff line number Diff line change
Expand Up @@ -545,8 +545,8 @@ export interface QwenOmniOptions {
apiKey: string;
/** Qwen Omni model identifier */
model: string;
/** Qwen Omni Realtime WebSocket URL. Uses the server-side CN endpoint when omitted. */
url?: string;
/** Qwen Omni Realtime WebSocket URL */
url: string;
/** Voice identifier for audio output */
voice?: string;
/** System instructions that define agent behavior */
Expand All @@ -563,8 +563,8 @@ export interface QwenOmniOptions {
messages?: Record<string, unknown>[];
/** Additional Qwen Omni parameters */
params?: Record<string, unknown>;
/** Required MLLM turn detection configuration. Overrides top-level turn_detection. */
turnDetection: MllmTurnDetectionConfig;
/** MLLM turn detection configuration. Overrides top-level turn_detection. */
turnDetection?: MllmTurnDetectionConfig;
}

/**
Expand All @@ -574,9 +574,9 @@ export interface QwenOmniOptions {
* ```typescript
* const agent = new Agent({ client }).withMllm(new QwenOmni({
* apiKey: process.env.DASHSCOPE_API_KEY,
* model: 'qwen-omni-turbo-realtime',
* model: 'qwen3.5-omni-plus-realtime',
* url: 'wss://dashscope.aliyuncs.com/api-ws/v1/realtime',
* greetingMessage: '你好,有什么可以帮你?',
* turnDetection: { mode: 'server_vad' },
* }));
* ```
*/
Expand All @@ -587,7 +587,7 @@ export class QwenOmni extends BaseCNMLLM {
super();
requireString(options.apiKey, "apiKey", "QwenOmni");
requireString(options.model, "model", "QwenOmni");
requireObject(options.turnDetection, "turnDetection", "QwenOmni");
requireString(options.url, "url", "QwenOmni");
this.options = options;
}

Expand All @@ -610,7 +610,7 @@ export class QwenOmni extends BaseCNMLLM {
return {
vendor: "qwen_omni",
api_key: apiKey,
...(url !== undefined && { url }),
url,
params: {
...params,
model,
Expand All @@ -622,7 +622,7 @@ export class QwenOmni extends BaseCNMLLM {
...(inputModalities && { input_modalities: inputModalities }),
...(outputModalities && { output_modalities: outputModalities }),
...(messages && { messages }),
turn_detection: turnDetection,
...(turnDetection && { turn_detection: turnDetection }),
};
}
}
8 changes: 4 additions & 4 deletions tests/unit/agentkit/cn-vendors.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -127,14 +127,14 @@ describe("CN vendor helpers", () => {
expect(
new QwenOmni({
apiKey: "dashscope-key",
model: "qwen-omni-turbo-realtime",
turnDetection: { mode: "server_vad" },
model: "qwen3.5-omni-plus-realtime",
url: "wss://dashscope.aliyuncs.com/api-ws/v1/realtime",
}).toConfig(),
).toMatchObject({
vendor: "qwen_omni",
api_key: "dashscope-key",
params: { model: "qwen-omni-turbo-realtime" },
turn_detection: { mode: "server_vad" },
url: "wss://dashscope.aliyuncs.com/api-ws/v1/realtime",
params: { model: "qwen3.5-omni-plus-realtime" },
});
});

Expand Down
49 changes: 34 additions & 15 deletions tests/unit/agentkit/request-body.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1420,29 +1420,48 @@ describe("MLLM vendor coverage", () => {
expect((config as Record<string, unknown>)?.params).toMatchObject({ voice: "eve" });
});

test("QwenOmni serializes the CN qwen_omni vendor", () => {
const config = new QwenOmni({
apiKey: "dashscope-key",
model: "qwen-omni-turbo-realtime",
voice: "Cherry",
turnDetection: { mode: "server_vad" },
}).toConfig();
test("QwenOmni serializes the complete CN qwen_omni payload", () => {
const qwenClient = new AgoraClient({
area: Area.CN,
appId: "test-app-id",
appCertificate: "test-app-certificate",
});
const properties = new Agent({ client: qwenClient })
.withMllm(
new QwenOmni({
apiKey: "xxxxxxxx",
model: "qwen3.5-omni-plus-realtime",
url: "wss://dashscope.aliyuncs.com/api-ws/v1/realtime",
}),
)
.toProperties({ ...SESSION_OPTS });

expect(config.vendor).toBe("qwen_omni");
expect(config.api_key).toBe("dashscope-key");
expect(config.params).toMatchObject({
model: "qwen-omni-turbo-realtime",
voice: "Cherry",
expect(properties.mllm).toEqual({
enable: true,
vendor: "qwen_omni",
url: "wss://dashscope.aliyuncs.com/api-ws/v1/realtime",
api_key: "xxxxxxxx",
params: {
model: "qwen3.5-omni-plus-realtime",
},
});
expect(
new QwenOmni({
apiKey: "key",
model: "qwen-omni-turbo-realtime",
model: "qwen3.5-omni-plus-realtime",
url: "wss://dashscope.aliyuncs.com/api-ws/v1/realtime",
turnDetection: { mode: "server_vad" },
}).toConfig().turn_detection,
).toEqual({ mode: "server_vad" });
expect(
new QwenOmni({
apiKey: "key",
model: "qwen3.5-omni-plus-realtime",
url: "wss://dashscope.aliyuncs.com/api-ws/v1/realtime",
}).areaScope,
).toBe("cn");
expect(() => new QwenOmni({ apiKey: "key", model: "qwen-omni-turbo-realtime" } as never)).toThrow(
"QwenOmni requires turnDetection",
expect(() => new QwenOmni({ apiKey: "key", model: "qwen3.5-omni-plus-realtime" } as never)).toThrow(
"QwenOmni requires url",
);
});
});
Expand Down
Loading