From a819e1fa3abb5ac0c9f7958383887b49e1bc1967 Mon Sep 17 00:00:00 2001 From: survivor998 Date: Thu, 11 Jun 2026 06:04:49 +0800 Subject: [PATCH 1/2] fix(llm): sanitize short user_id across transformers to comply with constraints --- llm/transformer/anthropic/outbound_convert.go | 9 ++++- llm/transformer/doubao/outbound.go | 15 ++++++-- llm/transformer/shared/userid.go | 35 +++++++++++++++++++ llm/transformer/zai/outbound.go | 14 ++++++-- 4 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 llm/transformer/shared/userid.go diff --git a/llm/transformer/anthropic/outbound_convert.go b/llm/transformer/anthropic/outbound_convert.go index f10b2aef9..98697d736 100644 --- a/llm/transformer/anthropic/outbound_convert.go +++ b/llm/transformer/anthropic/outbound_convert.go @@ -120,8 +120,15 @@ func buildBaseRequest(chatReq *llm.Request, config *Config) *MessageRequest { MaxTokens: resolveMaxTokens(chatReq), } + var userID string if chatReq.Metadata != nil && chatReq.Metadata["user_id"] != "" { - req.Metadata = &AnthropicMetadata{UserID: chatReq.Metadata["user_id"]} + userID = chatReq.Metadata["user_id"] + } else if chatReq.User != nil && *chatReq.User != "" { + userID = *chatReq.User + } + + if userID != "" { + req.Metadata = &AnthropicMetadata{UserID: shared.SanitizeUserID(userID)} } // DeepSeek Anthropic format supports output_config.effort. When reasoning_effort diff --git a/llm/transformer/doubao/outbound.go b/llm/transformer/doubao/outbound.go index fbd4e7803..eb4535309 100644 --- a/llm/transformer/doubao/outbound.go +++ b/llm/transformer/doubao/outbound.go @@ -17,6 +17,7 @@ import ( "github.com/looplj/axonhub/llm/internal/pkg/xurl" "github.com/looplj/axonhub/llm/transformer" "github.com/looplj/axonhub/llm/transformer/openai" + "github.com/looplj/axonhub/llm/transformer/shared" ) // Config holds all configuration for the Doubao outbound transformer. @@ -142,8 +143,18 @@ func (t *OutboundTransformer) TransformRequest( RequestID: "", } - if llmReq.Metadata != nil { - doubaoReq.UserID = llmReq.Metadata["user_id"] + var userID string + if llmReq.Metadata != nil && llmReq.Metadata["user_id"] != "" { + userID = llmReq.Metadata["user_id"] + } else if llmReq.User != nil && *llmReq.User != "" { + userID = *llmReq.User + } + + if userID != "" { + doubaoReq.UserID = shared.SanitizeUserID(userID) + } + + if llmReq.Metadata != nil && llmReq.Metadata["request_id"] != "" { doubaoReq.RequestID = llmReq.Metadata["request_id"] } diff --git a/llm/transformer/shared/userid.go b/llm/transformer/shared/userid.go new file mode 100644 index 000000000..02eaf4809 --- /dev/null +++ b/llm/transformer/shared/userid.go @@ -0,0 +1,35 @@ +package shared + +import "strings" + +// SanitizeUserID ensures the user ID matches common platform constraints: +// ^[a-zA-Z0-9_-]{6,128}$ +func SanitizeUserID(userID string) string { + if userID == "" { + return "" + } + + var sb strings.Builder + for _, c := range userID { + if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '-' { + sb.WriteRune(c) + } else { + sb.WriteRune('_') + } + } + + sanitized := sb.String() + + if len(sanitized) < 6 { + sanitized = "user_" + sanitized + if len(sanitized) < 6 { + sanitized = sanitized + strings.Repeat("_", 6-len(sanitized)) + } + } + + if len(sanitized) > 128 { + sanitized = sanitized[:128] + } + + return sanitized +} diff --git a/llm/transformer/zai/outbound.go b/llm/transformer/zai/outbound.go index dfc69df8f..af5765f38 100644 --- a/llm/transformer/zai/outbound.go +++ b/llm/transformer/zai/outbound.go @@ -129,8 +129,18 @@ func (t *OutboundTransformer) TransformRequest( RequestID: "", } - if llmReq.Metadata != nil { - zaiReq.UserID = llmReq.Metadata["user_id"] + var userID string + if llmReq.Metadata != nil && llmReq.Metadata["user_id"] != "" { + userID = llmReq.Metadata["user_id"] + } else if llmReq.User != nil && *llmReq.User != "" { + userID = *llmReq.User + } + + if userID != "" { + zaiReq.UserID = shared.SanitizeUserID(userID) + } + + if llmReq.Metadata != nil && llmReq.Metadata["request_id"] != "" { zaiReq.RequestID = llmReq.Metadata["request_id"] } From d9da705e7d5cd8729a04ddb05e92ea4b12a3a73b Mon Sep 17 00:00:00 2001 From: survivor998 Date: Sat, 13 Jun 2026 11:55:00 +0800 Subject: [PATCH 2/2] feat(channels): auto-detect TTS models in channel test button The channel test button previously sent hardcoded chat completion requests that failed for TTS models (e.g. Xiaomi MiMo-TTS). This fix detects TTS models via their model card's modalities.output field and automatically builds the correct audio request format. Changes: - Add ChatCompletionAudioParam to llm.Request (was previously a TODO) - Add Audio field to openai.Request and carry it in RequestFromLLM - Add isTTSModel detection via model card modalities.output lookup - Add buildTestRequest that constructs TTS-shaped requests for audio models - Update response handling to extract audio transcripts from TTS responses - Update streaming handler to accumulate audio transcripts from delta Verified locally: mimo-v2.5-tts tests successfully on Xiaomi channels. Non-TTS models continue to work as before (backward compatible). Co-Authored-By: Claude Opus 4.8 --- internal/server/gql/resolver.go | 2 +- internal/server/orchestrator/tester.go | 199 ++++++++++++++------- llm/model.go | 12 +- llm/transformer/openai/model.go | 11 ++ llm/transformer/openai/outbound_convert.go | 8 + 5 files changed, 169 insertions(+), 63 deletions(-) diff --git a/internal/server/gql/resolver.go b/internal/server/gql/resolver.go index bb404ccd3..8611d3604 100644 --- a/internal/server/gql/resolver.go +++ b/internal/server/gql/resolver.go @@ -117,7 +117,7 @@ func NewSchema( defaultSelector: defaultSelector, candidateSelectorDiagnostics: candidateSelectorDiagnostics, channelLimiterManager: channelLimiterManager, - TestChannelOrchestrator: orchestrator.NewTestChannelOrchestrator(channelService, requestService, systemService, usageLogService, promptProtectionRuleService, httpClient), + TestChannelOrchestrator: orchestrator.NewTestChannelOrchestrator(channelService, requestService, systemService, usageLogService, promptProtectionRuleService, httpClient, modelService), gcWorker: gcWorker, videoWorker: videoWorker, }, diff --git a/internal/server/orchestrator/tester.go b/internal/server/orchestrator/tester.go index 894daf5bb..69dca8126 100644 --- a/internal/server/orchestrator/tester.go +++ b/internal/server/orchestrator/tester.go @@ -12,6 +12,7 @@ import ( "github.com/tidwall/gjson" "golang.org/x/sync/errgroup" + "github.com/looplj/axonhub/internal/ent/model" "github.com/looplj/axonhub/internal/log" "github.com/looplj/axonhub/internal/objects" "github.com/looplj/axonhub/internal/pkg/xjson" @@ -35,6 +36,7 @@ type TestChannelOrchestrator struct { usageLogService *biz.UsageLogService promptProtectionRuleService *biz.PromptProtectionRuleService httpClient *httpclient.HttpClient + modelService *biz.ModelService modelCircuitBreaker *biz.ModelCircuitBreaker modelMapper *ModelMapper loadBalancer *LoadBalancer @@ -49,6 +51,7 @@ func NewTestChannelOrchestrator( usageLogService *biz.UsageLogService, promptProtectionRuleService *biz.PromptProtectionRuleService, httpClient *httpclient.HttpClient, + modelService *biz.ModelService, ) *TestChannelOrchestrator { return &TestChannelOrchestrator{ channelService: channelService, @@ -57,6 +60,7 @@ func NewTestChannelOrchestrator( usageLogService: usageLogService, promptProtectionRuleService: promptProtectionRuleService, httpClient: httpClient, + modelService: modelService, modelCircuitBreaker: biz.NewModelCircuitBreaker(), modelMapper: NewModelMapper(), loadBalancer: NewLoadBalancer(systemService, channelService, NewWeightStrategy()), @@ -78,6 +82,83 @@ type TestChannelResult struct { Error *string } +// isTTSModel checks whether a model outputs audio by looking up its model card. +func (processor *TestChannelOrchestrator) isTTSModel(ctx context.Context, modelID string) bool { + if processor.modelService == nil || modelID == "" { + return false + } + + m, err := processor.modelService.GetModelByModelID(ctx, modelID, model.StatusEnabled) + if err != nil { + return false + } + + if m.ModelCard == nil { + return false + } + + for _, output := range m.ModelCard.Modalities.Output { + if output == "audio" { + return true + } + } + + return false +} + +// buildTestRequest constructs the appropriate test request for a model. +// For TTS models, it builds a request with audio modalities and parameters. +// For regular models, it builds a standard chat completion request. +func (processor *TestChannelOrchestrator) buildTestRequest(ctx context.Context, testModel string, useStream bool) *llm.Request { + if processor.isTTSModel(ctx, testModel) { + return &llm.Request{ + Model: testModel, + Messages: []llm.Message{ + { + Role: "assistant", + Content: llm.MessageContent{ + Content: lo.ToPtr("Hello, this is a TTS test. Please respond with audio."), + }, + }, + }, + Modalities: []string{"text", "audio"}, + Audio: &llm.ChatCompletionAudioParam{}, + MaxCompletionTokens: lo.ToPtr(int64(256)), + Stream: lo.ToPtr(useStream), + } + } + + // Standard chat completion request for non-TTS models. + return &llm.Request{ + Model: testModel, + Messages: []llm.Message{ + { + Role: "system", + Content: llm.MessageContent{ + Content: lo.ToPtr("You are a helpful assistant."), + }, + }, + { + Role: "user", + Content: llm.MessageContent{ + MultipleContent: []llm.MessageContentPart{ + { + Type: "text", + Text: lo.ToPtr("Hello world, I'm AxonHub."), + }, + { + Type: "text", + Text: lo.ToPtr("Please tell me who you are?"), + }, + }, + }, + }, + }, + MaxCompletionTokens: lo.ToPtr(int64(256)), + Stream: lo.ToPtr(useStream), + } +} + // TestChannel tests a specific channel with a simple request. func (processor *TestChannelOrchestrator) TestChannel( ctx context.Context, @@ -122,35 +203,8 @@ func (processor *TestChannelOrchestrator) TestChannel( // Check if the channel requires streaming useStream := channel != nil && channel.Policies.Stream == objects.CapabilityPolicyRequire - // Create a simple test request - llmRequest := &llm.Request{ - Model: testModel, - Messages: []llm.Message{ - { - Role: "system", - Content: llm.MessageContent{ - Content: lo.ToPtr("You are a helpful assistant."), - }, - }, - { - Role: "user", - Content: llm.MessageContent{ - MultipleContent: []llm.MessageContentPart{ - { - Type: "text", - Text: lo.ToPtr("Hello world, I'm AxonHub."), - }, - { - Type: "text", - Text: lo.ToPtr("Please tell me who you are?"), - }, - }, - }, - }, - }, - MaxCompletionTokens: lo.ToPtr(int64(256)), - Stream: lo.ToPtr(useStream), - } + // Build the appropriate test request based on model type + llmRequest := processor.buildTestRequest(ctx, testModel, useStream) body, err := json.Marshal(llmRequest) if err != nil { @@ -205,10 +259,14 @@ func (processor *TestChannelOrchestrator) TestChannel( }, nil } + // Extract message text or audio transcript from the response. + msg := response.Choices[0].Message + messageText := extractMessageText(msg) + return &TestChannelResult{ Latency: latency, Success: true, - Message: response.Choices[0].Message.Content.Content, + Message: messageText, Error: nil, }, nil } @@ -225,6 +283,7 @@ func (processor *TestChannelOrchestrator) handleStreamResponse( // Accumulate stream chunks var accumulatedContent string + var hasAudioResponse bool for stream.Next() { select { @@ -256,8 +315,19 @@ func (processor *TestChannelOrchestrator) handleStreamResponse( } // Accumulate content from the first choice - if len(chunk.Choices) > 0 && chunk.Choices[0].Delta != nil && chunk.Choices[0].Delta.Content.Content != nil { - accumulatedContent += *chunk.Choices[0].Delta.Content.Content + if len(chunk.Choices) > 0 && chunk.Choices[0].Delta != nil { + delta := chunk.Choices[0].Delta + if delta.Content.Content != nil { + accumulatedContent += *delta.Content.Content + } + + // Also accumulate audio transcripts from TTS streaming responses. + if delta.Audio != nil { + hasAudioResponse = true + if delta.Audio.Transcript != "" { + accumulatedContent += delta.Audio.Transcript + } + } } } @@ -283,6 +353,15 @@ func (processor *TestChannelOrchestrator) handleStreamResponse( } if accumulatedContent == "" { + if hasAudioResponse { + return &TestChannelResult{ + Latency: latency, + Success: true, + Message: lo.ToPtr("(audio stream response)"), + Error: nil, + }, nil + } + return &TestChannelResult{ Latency: latency, Success: false, @@ -441,34 +520,7 @@ func (processor *TestChannelOrchestrator) testSingleKey( modelCircuitBreaker: processor.modelCircuitBreaker, } - llmRequest := &llm.Request{ - Model: testModel, - Messages: []llm.Message{ - { - Role: "system", - Content: llm.MessageContent{ - Content: new("You are a helpful assistant."), - }, - }, - { - Role: "user", - Content: llm.MessageContent{ - MultipleContent: []llm.MessageContentPart{ - { - Type: "text", - Text: new("Hello world, I'm AxonHub."), - }, - { - Type: "text", - Text: new("Please tell me who you are?"), - }, - }, - }, - }, - }, - MaxCompletionTokens: new(int64(256)), - Stream: new(useStream), - } + llmRequest := processor.buildTestRequest(ctx, testModel, useStream) body, err := json.Marshal(llmRequest) if err != nil { @@ -554,3 +606,30 @@ func maskAPIKey(key string) string { return key[:4] + "****" + key[len(key)-4:] } + +// extractMessageText extracts displayable text from a message, handling both text +// and audio responses. For TTS models, the response may contain audio data with a +// transcript instead of plain text content. +func extractMessageText(msg *llm.Message) *string { + if msg == nil { + return lo.ToPtr("") + } + + // Prefer text content if available. + if msg.Content.Content != nil && *msg.Content.Content != "" { + return msg.Content.Content + } + + // Fall back to audio transcript for TTS responses. + if msg.Audio != nil { + if msg.Audio.Transcript != "" { + return lo.ToPtr(msg.Audio.Transcript) + } + + if msg.Audio.Data != "" { + return lo.ToPtr("(audio response)") + } + } + + return lo.ToPtr("") +} diff --git a/llm/model.go b/llm/model.go index d92e92be2..99e207bae 100644 --- a/llm/model.go +++ b/llm/model.go @@ -140,8 +140,7 @@ type Request struct { // Parameters for audio output. Required when audio output is requested with // `modalities: ["audio"]`. // [Learn more](https://platform.openai.com/docs/guides/audio). - // TODO - // Audio ChatCompletionAudioParam `json:"audio,omitzero"` + Audio *ChatCompletionAudioParam `json:"audio,omitzero"` // Modify the likelihood of specified tokens appearing in the completion. // @@ -295,6 +294,15 @@ type StreamOptions struct { IncludeUsage bool `json:"include_usage,omitempty"` } +// ChatCompletionAudioParam specifies audio output parameters for chat completions. +// Required when audio output is requested with modalities: ["audio"]. +type ChatCompletionAudioParam struct { + // Format specifies the output audio format (e.g. "mp3", "wav", "opus"). + Format string `json:"format,omitempty"` + // Voice specifies the voice to use for audio generation (e.g. "alloy", "echo", "nova"). + Voice string `json:"voice,omitempty"` +} + type Stop struct { // Stop and MultipleStop are mutually exclusive representations of the same field. // If both are populated, Stop takes precedence during marshaling. diff --git a/llm/transformer/openai/model.go b/llm/transformer/openai/model.go index 2cb48062f..b3be6d371 100644 --- a/llm/transformer/openai/model.go +++ b/llm/transformer/openai/model.go @@ -67,6 +67,9 @@ type Request struct { // Modalities specifies output types (text, audio, image). Modalities []string `json:"modalities,omitempty"` + // Audio specifies audio output parameters. Required when modalities includes "audio". + Audio *ChatCompletionAudioParam `json:"audio,omitempty"` + // ReasoningEffort controls effort on reasoning models. ReasoningEffort string `json:"reasoning_effort,omitempty"` @@ -102,6 +105,14 @@ type StreamOptions struct { IncludeUsage bool `json:"include_usage,omitempty"` } +// ChatCompletionAudioParam specifies audio output parameters for chat completions. +type ChatCompletionAudioParam struct { + // Format specifies the output audio format (e.g. "mp3", "wav", "opus"). + Format string `json:"format,omitempty"` + // Voice specifies the voice to use for audio generation (e.g. "alloy", "echo", "nova"). + Voice string `json:"voice,omitempty"` +} + // Stop represents stop sequences. type Stop struct { // Stop and MultipleStop are mutually exclusive representations of the same field. diff --git a/llm/transformer/openai/outbound_convert.go b/llm/transformer/openai/outbound_convert.go index ce6e285ce..583ec5ffc 100644 --- a/llm/transformer/openai/outbound_convert.go +++ b/llm/transformer/openai/outbound_convert.go @@ -37,6 +37,14 @@ func RequestFromLLM(r *llm.Request, reasoningField ReasoningField) *Request { Verbosity: r.Verbosity, } + // Convert Audio + if r.Audio != nil { + req.Audio = &ChatCompletionAudioParam{ + Format: r.Audio.Format, + Voice: r.Audio.Voice, + } + } + // Convert messages req.Messages = lo.Map(r.Messages, func(m llm.Message, _ int) Message { return MessageFromLLMWithConfig(m, reasoningField)