From 6cebda9607ca18be4aff5f25aa7fc44bb135b95d Mon Sep 17 00:00:00 2001 From: Pratham-Mishra04 Date: Mon, 14 Sep 2026 17:28:15 +0530 Subject: [PATCH 001/118] refactor: move session key stickiness behind a SessionAffinity seam --- core/bifrost.go | 110 ++------ core/bifrost_test.go | 9 +- core/schemas/bifrost.go | 15 +- core/schemas/sessionaffinity.go | 19 ++ core/sessionaffinity.go | 189 +++++++++++++ core/sessionaffinity_test.go | 353 ++++++++++++++++++++++++ core/utils.go | 18 -- docs/quickstart/go-sdk/context-keys.mdx | 2 +- 8 files changed, 593 insertions(+), 122 deletions(-) create mode 100644 core/schemas/sessionaffinity.go create mode 100644 core/sessionaffinity.go create mode 100644 core/sessionaffinity_test.go diff --git a/core/bifrost.go b/core/bifrost.go index 4ff948614e6..89520754014 100644 --- a/core/bifrost.go +++ b/core/bifrost.go @@ -14,7 +14,6 @@ import ( "sync/atomic" "time" - "github.com/bytedance/sonic" "github.com/google/uuid" "github.com/maximhq/bifrost/core/keyselectors" @@ -128,6 +127,7 @@ type Bifrost struct { keySelector schemas.KeySelector // Custom key selector function keyPoolFilter schemas.KeyPoolFilter // optional hook to veto keys before selection (nil = all eligible) kvStore schemas.KVStore // optional KV store for session stickiness (nil = disabled) + sessionAffinity schemas.SessionAffinity // decides which key a session stays on; never nil after Init } // ProviderQueue wraps a provider's request channel with lifecycle management @@ -298,6 +298,12 @@ func Init(ctx context.Context, config schemas.BifrostConfig) (*Bifrost, error) { if bifrost.keySelector == nil { bifrost.keySelector = keyselectors.WeightedRandom } + // Always present, so key selection never checks for it: without a KV store the shipped + // key affinity has nowhere to keep a binding and declines every request. + bifrost.sessionAffinity = config.SessionAffinity + if bifrost.sessionAffinity == nil { + bifrost.sessionAffinity = newKeyAffinity(bifrost.kvStore, bifrost.keySelector, bifrost.logger) + } // Initialize object pools bifrost.channelMessagePool = sync.Pool{ @@ -9157,7 +9163,7 @@ func (bifrost *Bifrost) getKeysForBatchAndFileOps(ctx *schemas.BifrostContext, p // canRotate=false is returned for cases where the caller must always use the same key: // - SkipKeySelection (Claude Code OAuth passthrough to Anthropic; empty slice returned) // - Explicit BifrostContextKeyAPIKeyID / APIKeyName (user pinned a specific key) -// - Session stickiness (key persisted in KV store for the session lifetime) +// - Session affinity (the registered policy named a key for the request's session) // - Single-key pool (only one eligible key — rotation is a no-op, KV write skipped) // // canRotate=true is returned when there are two or more eligible keys and no pinning @@ -9281,105 +9287,23 @@ func (bifrost *Bifrost) selectKeyFromProviderForModelWithPool(ctx *schemas.Bifro return []schemas.Key{supportedKeys[0]}, false, nil } - // Session stickiness: on the first request for a session ID, the randomly selected key is - // persisted in the KV store. Subsequent requests reuse it for the session lifetime. The sticky - // key is intentionally kept fixed across all retry attempts — return it as a single-element - // pool with canRotate=false so rate-limit retries also stay on the same key. - sessionID := "" - if ctx != nil { - if id, ok := ctx.Value(schemas.BifrostContextKeySessionID).(string); ok && id != "" { - sessionID = id - } - } - fallbackIndex := 0 - if ctx != nil { - fallbackIndex, _ = ctx.Value(schemas.BifrostContextKeyFallbackIndex).(int) - } - stickinessActive := sessionID != "" && bifrost.kvStore != nil && fallbackIndex == 0 - - if stickinessActive { - kvKey := buildSessionKey(providerKey, sessionID, model) - ttl, _ := ctx.Value(schemas.BifrostContextKeySessionTTL).(time.Duration) - if ttl <= 0 { - ttl = schemas.DefaultSessionStickyTTL - } - - if cachedKey, found, stale := getCachedKeyFromStore(bifrost.kvStore, kvKey, supportedKeys); found { - if err := bifrost.kvStore.SetWithTTL(kvKey, cachedKey.ID, ttl); err != nil { - bifrost.logger.Warn("error setting session cache for provider=%s key_id=%s: %s", providerKey, cachedKey.ID, err.Error()) - } - return []schemas.Key{cachedKey}, false, nil - } else if stale { - if _, err := bifrost.kvStore.Delete(kvKey); err != nil { - bifrost.logger.Warn("error deleting stale session cache for provider=%s: %s", providerKey, err.Error()) - } - } - - selectedKey, err := bifrost.keySelector(ctx, supportedKeys, providerKey, model) - if err != nil { - return nil, false, err - } - - wasSet, err := bifrost.kvStore.SetNXWithTTL(kvKey, selectedKey.ID, ttl) - if err != nil { - bifrost.logger.Warn("error setting session cache for provider=%s key_id=%s: %s", providerKey, selectedKey.ID, err.Error()) - return []schemas.Key{selectedKey}, false, nil - } - if wasSet { - return []schemas.Key{selectedKey}, false, nil - } - - // Another concurrent request won the race — re-read the persisted key. - if currentKey, found, stale := getCachedKeyFromStore(bifrost.kvStore, kvKey, supportedKeys); found { - return []schemas.Key{currentKey}, false, nil - } else if stale { - if _, err := bifrost.kvStore.Delete(kvKey); err != nil { - bifrost.logger.Warn("error deleting stale session cache for provider=%s: %s", providerKey, err.Error()) + // Session affinity: a session that already has a key here stays on it. The policy decides + // what that means; core honors the key it names, for every attempt of the request, as long + // as that key is one of the eligible ones. A policy naming anything else is ignored, so a + // disabled or model-incompatible key can never reach a request through it. + if key, ok := bifrost.sessionAffinity.ResolveKey(ctx, providerKey, model, supportedKeys); ok { + for _, eligible := range supportedKeys { + if eligible.ID == key.ID { + return []schemas.Key{eligible}, false, nil } - return []schemas.Key{selectedKey}, false, nil } - - return []schemas.Key{selectedKey}, false, nil + bifrost.logger.Warn("session affinity named key %s for provider %s, which is not in the eligible pool; selecting a key normally", key.ID, providerKey) } // Normal case: return the full filtered pool with rotation enabled. return supportedKeys, true, nil } -// getCachedKeyFromStore retrieves a key ID from the KV store and looks it up in supportedKeys. -// Returns the matching Key, found (true if key exists in supportedKeys), and stale (true if -// KV contains an ID but it is not in supportedKeys—caller should delete before SetNXWithTTL). -func getCachedKeyFromStore(kvStore schemas.KVStore, kvKey string, supportedKeys []schemas.Key) (schemas.Key, bool, bool) { - raw, err := kvStore.Get(kvKey) - if err != nil { - return schemas.Key{}, false, false - } - - var cachedKeyID string - switch v := raw.(type) { - case string: - cachedKeyID = v - case []byte: - var s string - if err := sonic.Unmarshal(v, &s); err == nil { - cachedKeyID = s - } else { - cachedKeyID = string(v) - } - } - - if cachedKeyID != "" { - for _, k := range supportedKeys { - if k.ID == cachedKeyID { - return k, true, false - } - } - return schemas.Key{}, false, true - } - - return schemas.Key{}, false, false -} - // Shutdown gracefully stops all workers when triggered. // It closes all request channels and waits for workers to exit. func (bifrost *Bifrost) Shutdown() { diff --git a/core/bifrost_test.go b/core/bifrost_test.go index 9d29e485532..78ac22f17c4 100644 --- a/core/bifrost_test.go +++ b/core/bifrost_test.go @@ -1261,7 +1261,7 @@ func TestSelectKeyFromProviderForModel_SessionStickiness(t *testing.T) { } // Verify kvstore was written - kvKey := buildSessionKey(schemas.OpenAI, "sess-123", "gpt-4") + kvKey := sessionStateKey(bfCtx, SessionStateKindKey, string(schemas.OpenAI), "gpt-4") if raw, err := kvStore.Get(kvKey); err != nil || raw != "key-a" { t.Errorf("kvstore after first call: expected key-a, got %v (err=%v)", raw, err) } @@ -1328,8 +1328,11 @@ func TestSelectKeyFromProviderForModel_NoStickinessWithoutSessionID(t *testing.T t.Errorf("expected 0 keySelector calls from pool building (no session id), got %d", keySelectorCalls) } // KVStore should not have a sticky entry for an empty session id - if _, err := kvStore.Get(buildSessionKey(schemas.OpenAI, "", "gpt-4")); err == nil { - t.Error("kvstore should not have a sticky entry for an empty session id") + kvStore.mu.RLock() + entries := len(kvStore.data) + kvStore.mu.RUnlock() + if entries != 0 { + t.Errorf("kvstore should not have a sticky entry for an empty session id, got %d entries", entries) } } diff --git a/core/schemas/bifrost.go b/core/schemas/bifrost.go index 76fcb0e54b7..1163a2dabc6 100644 --- a/core/schemas/bifrost.go +++ b/core/schemas/bifrost.go @@ -29,13 +29,14 @@ type BifrostConfig struct { OAuth2Provider OAuth2Provider MCPHeadersProvider MCPHeadersProvider // Backend for MCPAuthTypePerUserHeaders credential storage; nil disables per-user-headers auth (resolver errors at use) Logger Logger - Tracer Tracer // Tracer for distributed tracing (nil = NoOpTracer) - InitialPoolSize int // Initial pool size for sync pools in Bifrost. Higher values will reduce memory allocations but will increase memory usage. - DropExcessRequests bool // If true, in cases where the queue is full, requests will not wait for the queue to be empty and will be dropped instead. - MCPConfig *MCPConfig // MCP (Model Context Protocol) configuration for tool integration - KeySelector KeySelector // Custom key selector function - KeyPoolFilter KeyPoolFilter // Optional hook to filter available keys before selection; nil = all keys eligible - KVStore KVStore // shared KV store for clustering/session stickiness; nil = disabled + Tracer Tracer // Tracer for distributed tracing (nil = NoOpTracer) + InitialPoolSize int // Initial pool size for sync pools in Bifrost. Higher values will reduce memory allocations but will increase memory usage. + DropExcessRequests bool // If true, in cases where the queue is full, requests will not wait for the queue to be empty and will be dropped instead. + MCPConfig *MCPConfig // MCP (Model Context Protocol) configuration for tool integration + KeySelector KeySelector // Custom key selector function + KeyPoolFilter KeyPoolFilter // Optional hook to filter available keys before selection; nil = all keys eligible + KVStore KVStore // shared KV store for clustering/session stickiness; nil = disabled + SessionAffinity SessionAffinity // Decides which key a session stays on; nil = Bifrost's own default, which needs KVStore to bind anything ModelCatalog ModelInfoProvider } diff --git a/core/schemas/sessionaffinity.go b/core/schemas/sessionaffinity.go new file mode 100644 index 00000000000..61ce9ba27a8 --- /dev/null +++ b/core/schemas/sessionaffinity.go @@ -0,0 +1,19 @@ +package schemas + +// SessionAffinity keeps a request that carries a session id on what served that session +// before. Core asks it at defined points and applies the answer; the policy behind the +// answer belongs to the implementation. The one Bifrost ships keeps a session on the key it +// was first served by for as long as that key stays eligible, and is installed when nothing +// else is. A deployment that knows more about its keys, such as their health, registers its +// own through BifrostConfig.SessionAffinity. +// +// Every method receives the request context, which carries the session id +// (BifrostContextKeySessionID), the session TTL (BifrostContextKeySessionTTL) and the +// identifiers that scope a session to its caller. +type SessionAffinity interface { + // ResolveKey is asked once the eligible key pool for a provider attempt has been built + // with two or more keys in it. It answers with the key the session's state settles on, + // looking one up or binding one now, or with ok=false to leave the choice to ordinary + // key selection. A returned key is used for every attempt of the request. + ResolveKey(ctx *BifrostContext, provider ModelProvider, model string, eligible []Key) (key Key, ok bool) +} diff --git a/core/sessionaffinity.go b/core/sessionaffinity.go new file mode 100644 index 00000000000..d0f708f0323 --- /dev/null +++ b/core/sessionaffinity.go @@ -0,0 +1,189 @@ +package bifrost + +import ( + "crypto/sha256" + "encoding/hex" + "time" + + "github.com/bytedance/sonic" + "github.com/maximhq/bifrost/core/schemas" +) + +// Session state keeps a request that carries a session id on what served that session before. +// The session id is chosen by the caller, so two callers can send the same one; every piece of +// session state is therefore keyed under who the request's grant identity attributes it to, +// the virtual key and user, or under the deployment as a whole when neither is known. State +// lives in the schemas.KVStore the Bifrost instance was given, so it is shared wherever that +// store is. + +const ( + sessionStateKeyPrefix = "session:v2:" + + // SessionStateKindKey is the state kind of a session's key binding within one provider and + // model. + SessionStateKindKey = "key" +) + +// keyAffinity is the session affinity Bifrost ships, installed when the configuration +// registers none: a session stays on the key it was first served by, per provider and model, +// for as long as that key remains eligible. The binding is written when the key is chosen, +// kept fresh on every reuse, and replaced only once the bound key has left the eligible pool. +// +// Only the primary provider of a request is bound. A fallback provider serves because the +// primary could not, and binding the session to it would keep the session there. +type keyAffinity struct { + kv schemas.KVStore + selector schemas.KeySelector + logger schemas.Logger +} + +// newKeyAffinity builds key affinity over kv, choosing an unbound session's key with selector. +// Neither selector nor logger may be nil. +func newKeyAffinity(kv schemas.KVStore, selector schemas.KeySelector, logger schemas.Logger) *keyAffinity { + return &keyAffinity{kv: kv, selector: selector, logger: logger} +} + +// ResolveKey implements schemas.SessionAffinity. +func (k *keyAffinity) ResolveKey(ctx *schemas.BifrostContext, provider schemas.ModelProvider, model string, eligible []schemas.Key) (schemas.Key, bool) { + if k == nil || k.kv == nil || ctx == nil || len(eligible) < 2 { + return schemas.Key{}, false + } + if sessionIDFromContext(ctx) == "" { + return schemas.Key{}, false + } + if fallbackIndex, _ := ctx.Value(schemas.BifrostContextKeyFallbackIndex).(int); fallbackIndex > 0 { + return schemas.Key{}, false + } + + key := sessionStateKey(ctx, SessionStateKindKey, string(provider), model) + ttl := sessionTTLFromContext(ctx) + + if bound, found := k.lookup(key, provider, eligible); found { + if err := k.kv.SetWithTTL(key, bound.ID, ttl); err != nil { + k.logger.Warn("error refreshing session key binding for provider=%s key_id=%s: %s", provider, bound.ID, err.Error()) + } + return bound, true + } + + selected, err := k.selector(ctx, eligible, provider, model) + if err != nil { + k.logger.Warn("error selecting a key to bind the session to for provider=%s: %s", provider, err.Error()) + return schemas.Key{}, false + } + + wrote, err := k.kv.SetNXWithTTL(key, selected.ID, ttl) + if err != nil { + k.logger.Warn("error binding session to provider=%s key_id=%s: %s", provider, selected.ID, err.Error()) + return selected, true + } + if wrote { + return selected, true + } + + // Another request for the same session bound first. Its key is the session's key. + if bound, found := k.lookup(key, provider, eligible); found { + return bound, true + } + return selected, true +} + +// sessionStateKey builds the store key for one piece of session state: its kind, the request's +// session, who that session belongs to, and whatever further parts tell instances of that kind +// apart (a provider and model for a key binding). The session and its owner come from ctx: the +// session id, and the virtual key and user the grant identity attributes the request to. A +// request nothing governs, or whose identity is not settled, is scoped to the deployment. +// Everything but the kind is hashed, so the key is bounded in size and carries no +// caller-supplied text. +func sessionStateKey(ctx *schemas.BifrostContext, kind string, parts ...string) string { + virtualKeyID, userID := sessionIdentityParts(ctx) + h := sha256.New() + for _, part := range append([]string{virtualKeyID, userID, sessionIDFromContext(ctx)}, parts...) { + _, _ = h.Write([]byte(part)) + _, _ = h.Write([]byte{0}) + } + return sessionStateKeyPrefix + kind + ":" + hex.EncodeToString(h.Sum(nil)) +} + +// sessionIdentityParts is who the request is attributed to, as far as its grant identity has +// settled it: the virtual key it was made with and the user behind it, either possibly empty. +func sessionIdentityParts(ctx *schemas.BifrostContext) (virtualKeyID, userID string) { + if ctx == nil { + return "", "" + } + grant := ctx.Grant() + if grant == nil { + return "", "" + } + identity := grant.Identity() + if identity == nil { + return "", "" + } + if key := identity.VirtualKey(); key != nil { + virtualKeyID = key.ID + } + if user := identity.User(); user != nil { + userID = user.ID + } + return virtualKeyID, userID +} + +// sessionIDFromContext returns the request's session id, or "" when it carries none. +func sessionIDFromContext(ctx *schemas.BifrostContext) string { + if ctx == nil { + return "" + } + sessionID, _ := ctx.Value(schemas.BifrostContextKeySessionID).(string) + return sessionID +} + +// sessionTTLFromContext returns how long session state written for this request stays alive: +// the request's own TTL when it set one, otherwise schemas.DefaultSessionStickyTTL. +func sessionTTLFromContext(ctx *schemas.BifrostContext) time.Duration { + if ctx != nil { + if ttl, ok := ctx.Value(schemas.BifrostContextKeySessionTTL).(time.Duration); ok && ttl > 0 { + return ttl + } + } + return schemas.DefaultSessionStickyTTL +} + +// sessionStateString reads the string under key. A missing key, a read error and an empty +// value all read as not found. Values come back as strings from a local store and as JSON +// bytes from one that replicated them, so both are accepted. +func sessionStateString(kv schemas.KVStore, key string) (string, bool) { + raw, err := kv.Get(key) + if err != nil { + return "", false + } + var value string + switch v := raw.(type) { + case string: + value = v + case []byte: + if err := sonic.Unmarshal(v, &value); err != nil { + value = string(v) + } + } + if value == "" { + return "", false + } + return value, true +} + +// lookup reads the session's binding and resolves it against the eligible pool. A binding to +// a key no longer in the pool is deleted, so the caller binds afresh. +func (k *keyAffinity) lookup(key string, provider schemas.ModelProvider, eligible []schemas.Key) (schemas.Key, bool) { + boundID, found := sessionStateString(k.kv, key) + if !found { + return schemas.Key{}, false + } + for _, candidate := range eligible { + if candidate.ID == boundID { + return candidate, true + } + } + if _, err := k.kv.Delete(key); err != nil { + k.logger.Warn("error deleting stale session key binding for provider=%s: %s", provider, err.Error()) + } + return schemas.Key{}, false +} diff --git a/core/sessionaffinity_test.go b/core/sessionaffinity_test.go new file mode 100644 index 00000000000..96714767943 --- /dev/null +++ b/core/sessionaffinity_test.go @@ -0,0 +1,353 @@ +package bifrost + +import ( + "context" + "errors" + "strings" + "sync" + "testing" + "time" + + "github.com/maximhq/bifrost/core/schemas" +) + +func sessionCtx(sessionID string) *schemas.BifrostContext { + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + if sessionID != "" { + ctx.SetValue(schemas.BifrostContextKeySessionID, sessionID) + } + return ctx +} + +// stubIdentity is the least an installing layer could settle: which key and user the request +// is attributed to. Everything else answers as unknown. +type stubIdentity struct { + virtualKeyID string + userID string +} + +func (i *stubIdentity) Credential() schemas.Credential { return schemas.Credential{} } +func (i *stubIdentity) Presented() bool { return i.virtualKeyID != "" || i.userID != "" } +func (i *stubIdentity) User() *schemas.UserRef { + if i.userID == "" { + return nil + } + return &schemas.UserRef{ID: i.userID} +} +func (i *stubIdentity) VirtualKey() *schemas.EntityRef { + if i.virtualKeyID == "" { + return nil + } + return &schemas.EntityRef{ID: i.virtualKeyID} +} +func (i *stubIdentity) Teams() []schemas.EntityRef { return nil } +func (i *stubIdentity) Customers() []schemas.EntityRef { return nil } +func (i *stubIdentity) BusinessUnits() []schemas.EntityRef { return nil } +func (i *stubIdentity) Project() *schemas.EntityRef { return nil } + +// stubGrant carries only an identity, which is all session state reads. +type stubGrant struct { + identity schemas.Identity +} + +func (g *stubGrant) Identity() schemas.Identity { return g.identity } +func (g *stubGrant) Access() schemas.Access { return nil } +func (g *stubGrant) Limits() schemas.Limits { return nil } +func (g *stubGrant) SetIdentity(i schemas.Identity) bool { g.identity = i; return i != nil } +func (g *stubGrant) SetAccess(schemas.Access) bool { return false } +func (g *stubGrant) SetLimits(schemas.Limits) bool { return false } + +// attributed installs a grant whose identity names the given key and user on ctx. +func attributed(ctx *schemas.BifrostContext, virtualKeyID, userID string) *schemas.BifrostContext { + ctx.SetGrant(&stubGrant{identity: &stubIdentity{virtualKeyID: virtualKeyID, userID: userID}}) + return ctx +} + +var sessionTestPool = []schemas.Key{ + {ID: "key-a", Name: "Key A"}, + {ID: "key-b", Name: "Key B"}, + {ID: "key-c", Name: "Key C"}, +} + +func firstKeySelector(_ *schemas.BifrostContext, keys []schemas.Key, _ schemas.ModelProvider, _ string) (schemas.Key, error) { + return keys[0], nil +} + +func TestSessionStateKeyScopesBySessionAndIdentity(t *testing.T) { + base := sessionStateKey(attributed(sessionCtx("session-1"), "vk-1", ""), SessionStateKindKey, "openai", "gpt-4o") + if !strings.HasPrefix(base, "session:v2:key:") { + t.Fatalf("key %q lacks the kind prefix", base) + } + if strings.Contains(base, "session-1") || strings.Contains(base, "vk-1") || strings.Contains(base, "gpt-4o") { + t.Fatalf("key %q leaks a hashed part", base) + } + if base != sessionStateKey(attributed(sessionCtx("session-1"), "vk-1", ""), SessionStateKindKey, "openai", "gpt-4o") { + t.Fatal("key is not deterministic") + } + + variants := map[string]string{ + "other virtual key": sessionStateKey(attributed(sessionCtx("session-1"), "vk-2", ""), SessionStateKindKey, "openai", "gpt-4o"), + "same id as a user": sessionStateKey(attributed(sessionCtx("session-1"), "", "vk-1"), SessionStateKindKey, "openai", "gpt-4o"), + "key and user": sessionStateKey(attributed(sessionCtx("session-1"), "vk-1", "u-1"), SessionStateKindKey, "openai", "gpt-4o"), + "other session": sessionStateKey(attributed(sessionCtx("session-2"), "vk-1", ""), SessionStateKindKey, "openai", "gpt-4o"), + "other provider": sessionStateKey(attributed(sessionCtx("session-1"), "vk-1", ""), SessionStateKindKey, "azure", "gpt-4o"), + "other model": sessionStateKey(attributed(sessionCtx("session-1"), "vk-1", ""), SessionStateKindKey, "openai", "gpt-4o-mini"), + "other kind": sessionStateKey(attributed(sessionCtx("session-1"), "vk-1", ""), "other", "openai", "gpt-4o"), + } + for name, v := range variants { + if v == base { + t.Fatalf("%s collides with the base key", name) + } + } + + // A request nothing governs, one whose grant has no identity, and one whose identity names + // nothing all scope to the deployment and share a key. + deployment := sessionStateKey(sessionCtx("session-1"), SessionStateKindKey, "openai", "gpt-4o") + unsettled := sessionCtx("session-1") + unsettled.SetGrant(&stubGrant{}) + if got := sessionStateKey(unsettled, SessionStateKindKey, "openai", "gpt-4o"); got != deployment { + t.Fatal("a grant without identity should scope to the deployment") + } + if got := sessionStateKey(attributed(sessionCtx("session-1"), "", ""), SessionStateKindKey, "openai", "gpt-4o"); got != deployment { + t.Fatal("an identity naming no key and no user should scope to the deployment") + } + if deployment == base { + t.Fatal("deployment scope collides with a virtual key scope") + } + if sessionStateKey(nil, SessionStateKindKey, "openai", "gpt-4o") == "" { + t.Fatal("nil context should still produce a key") + } +} + +func TestSessionTTLFromContext(t *testing.T) { + if got := sessionTTLFromContext(nil); got != schemas.DefaultSessionStickyTTL { + t.Fatalf("nil context TTL = %v, want the default", got) + } + if got := sessionTTLFromContext(sessionCtx("s")); got != schemas.DefaultSessionStickyTTL { + t.Fatalf("unset TTL = %v, want the default", got) + } + ctx := sessionCtx("s") + ctx.SetValue(schemas.BifrostContextKeySessionTTL, 3*time.Minute) + if got := sessionTTLFromContext(ctx); got != 3*time.Minute { + t.Fatalf("set TTL = %v, want 3m", got) + } + ctx.SetValue(schemas.BifrostContextKeySessionTTL, time.Duration(0)) + if got := sessionTTLFromContext(ctx); got != schemas.DefaultSessionStickyTTL { + t.Fatalf("zero TTL = %v, want the default", got) + } +} + +func TestKeyAffinityReadsReplicatedBindings(t *testing.T) { + cases := []struct { + name string + value any + want string + }{ + {"plain string", "key-b", "key-b"}, + {"json bytes", []byte(`"key-c"`), "key-c"}, + {"raw bytes", []byte("key-b"), "key-b"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + kv := newMockKVStore() + d := newKeyAffinity(kv, firstKeySelector, NewDefaultLogger(schemas.LogLevelError)) + ctx := sessionCtx("session-1") + _ = kv.SetWithTTL(sessionStateKey(ctx, SessionStateKindKey, "openai", "gpt-4o"), tc.value, time.Minute) + key, ok := d.ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool) + if !ok || key.ID != tc.want { + t.Fatalf("got %q ok=%v, want the stored binding %q", key.ID, ok, tc.want) + } + }) + } + + // An empty stored value is no binding: the session binds afresh. + kv := newMockKVStore() + d := newKeyAffinity(kv, firstKeySelector, NewDefaultLogger(schemas.LogLevelError)) + ctx := sessionCtx("session-1") + _ = kv.SetWithTTL(sessionStateKey(ctx, SessionStateKindKey, "openai", "gpt-4o"), "", time.Minute) + if key, ok := d.ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool); !ok || key.ID != "key-a" { + t.Fatalf("empty binding: got %q ok=%v, want a fresh pick of key-a", key.ID, ok) + } +} + +func TestKeyAffinityBindsThenReuses(t *testing.T) { + kv := newMockKVStore() + calls := 0 + selector := func(ctx *schemas.BifrostContext, keys []schemas.Key, p schemas.ModelProvider, m string) (schemas.Key, error) { + calls++ + return keys[1], nil + } + d := newKeyAffinity(kv, selector, NewDefaultLogger(schemas.LogLevelError)) + ctx := sessionCtx("session-1") + ctx.SetValue(schemas.BifrostContextKeySessionTTL, 5*time.Minute) + + key, ok := d.ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool) + if !ok || key.ID != "key-b" { + t.Fatalf("first request: got %q ok=%v, want the selector's pick key-b", key.ID, ok) + } + stateKey := sessionStateKey(ctx, SessionStateKindKey, "openai", "gpt-4o") + if entry := kv.data[stateKey]; entry.value != "key-b" || entry.ttl != 5*time.Minute { + t.Fatalf("binding after first request: %+v", entry) + } + + // Later requests reuse the binding without asking the selector and refresh its TTL. + ctx.SetValue(schemas.BifrostContextKeySessionTTL, 7*time.Minute) + key, ok = d.ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool) + if !ok || key.ID != "key-b" || calls != 1 { + t.Fatalf("second request: got %q ok=%v selector calls=%d", key.ID, ok, calls) + } + if entry := kv.data[stateKey]; entry.ttl != 7*time.Minute { + t.Fatalf("reuse did not refresh the TTL: %v", entry.ttl) + } + + // Another provider for the same session binds separately. + if key, ok := d.ResolveKey(ctx, schemas.Azure, "gpt-4o", sessionTestPool); !ok || key.ID != "key-b" || calls != 2 { + t.Fatalf("other provider: got %q ok=%v selector calls=%d", key.ID, ok, calls) + } +} + +func TestKeyAffinityRebindsWhenBoundKeyLeavesThePool(t *testing.T) { + kv := newMockKVStore() + d := newKeyAffinity(kv, firstKeySelector, NewDefaultLogger(schemas.LogLevelError)) + ctx := sessionCtx("session-1") + stateKey := sessionStateKey(ctx, SessionStateKindKey, "openai", "gpt-4o") + _ = kv.SetWithTTL(stateKey, "key-gone", time.Minute) + + key, ok := d.ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool) + if !ok || key.ID != "key-a" { + t.Fatalf("got %q ok=%v, want a fresh pick of key-a", key.ID, ok) + } + if entry := kv.data[stateKey]; entry.value != "key-a" { + t.Fatalf("stale binding was not replaced: %v", entry.value) + } +} + +func TestKeyAffinityDeclines(t *testing.T) { + kv := newMockKVStore() + d := newKeyAffinity(kv, firstKeySelector, NewDefaultLogger(schemas.LogLevelError)) + + if _, ok := d.ResolveKey(sessionCtx(""), schemas.OpenAI, "gpt-4o", sessionTestPool); ok { + t.Fatal("request without a session got a preference") + } + if _, ok := d.ResolveKey(nil, schemas.OpenAI, "gpt-4o", sessionTestPool); ok { + t.Fatal("nil context got a preference") + } + if _, ok := d.ResolveKey(sessionCtx("session-1"), schemas.OpenAI, "gpt-4o", sessionTestPool[:1]); ok { + t.Fatal("single-key pool got a preference") + } + fallback := sessionCtx("session-1") + fallback.SetValue(schemas.BifrostContextKeyFallbackIndex, 1) + if _, ok := d.ResolveKey(fallback, schemas.OpenAI, "gpt-4o", sessionTestPool); ok { + t.Fatal("fallback attempt got a preference") + } + if len(kv.data) != 0 { + t.Fatalf("declined requests wrote state: %v", kv.data) + } + + failing := func(*schemas.BifrostContext, []schemas.Key, schemas.ModelProvider, string) (schemas.Key, error) { + return schemas.Key{}, errors.New("no pick") + } + if _, ok := newKeyAffinity(kv, failing, NewDefaultLogger(schemas.LogLevelError)).ResolveKey(sessionCtx("session-1"), schemas.OpenAI, "gpt-4o", sessionTestPool); ok { + t.Fatal("selector error still produced a preference") + } + + var none *keyAffinity + if _, ok := none.ResolveKey(sessionCtx("session-1"), schemas.OpenAI, "gpt-4o", sessionTestPool); ok { + t.Fatal("nil policy got a preference") + } + if _, ok := newKeyAffinity(nil, firstKeySelector, NewDefaultLogger(schemas.LogLevelError)).ResolveKey(sessionCtx("session-1"), schemas.OpenAI, "gpt-4o", sessionTestPool); ok { + t.Fatal("policy without a store got a preference") + } +} + +func TestKeyAffinityConcurrentFirstRequestsConverge(t *testing.T) { + kv := newMockKVStore() + // The selector answers differently per call, as a weighted pick would. + var mu sync.Mutex + next := 0 + selector := func(_ *schemas.BifrostContext, keys []schemas.Key, _ schemas.ModelProvider, _ string) (schemas.Key, error) { + mu.Lock() + defer mu.Unlock() + k := keys[next%len(keys)] + next++ + return k, nil + } + d := newKeyAffinity(kv, selector, NewDefaultLogger(schemas.LogLevelError)) + + results := make([]string, 8) + var wg sync.WaitGroup + for i := range results { + wg.Add(1) + go func(i int) { + defer wg.Done() + key, ok := d.ResolveKey(sessionCtx("session-1"), schemas.OpenAI, "gpt-4o", sessionTestPool) + if ok { + results[i] = key.ID + } + }(i) + } + wg.Wait() + bound, _ := kv.data[sessionStateKey(sessionCtx("session-1"), SessionStateKindKey, "openai", "gpt-4o")].value.(string) + for i, r := range results { + if r != bound { + t.Fatalf("request %d used %q while the session is bound to %q (all: %v)", i, r, bound, results) + } + } +} + +func TestKeyAffinityScopesSessionsByCaller(t *testing.T) { + kv := newMockKVStore() + d := newKeyAffinity(kv, firstKeySelector, NewDefaultLogger(schemas.LogLevelError)) + a := attributed(sessionCtx("shared-session"), "vk-a", "") + b := attributed(sessionCtx("shared-session"), "vk-b", "") + + if _, ok := d.ResolveKey(a, schemas.OpenAI, "gpt-4o", sessionTestPool); !ok { + t.Fatal("caller a got no key") + } + if _, ok := d.ResolveKey(b, schemas.OpenAI, "gpt-4o", sessionTestPool); !ok { + t.Fatal("caller b got no key") + } + if len(kv.data) != 2 { + t.Fatalf("two callers sharing a session id should hold two bindings, got %d", len(kv.data)) + } +} + +// foreignKeyAffinity answers with a key core never offered it, as a misbehaving custom +// SessionAffinity could. +type foreignKeyAffinity struct{} + +func (foreignKeyAffinity) ResolveKey(*schemas.BifrostContext, schemas.ModelProvider, string, []schemas.Key) (schemas.Key, bool) { + return schemas.Key{ID: "key-foreign", Name: "Foreign"}, true +} + +func TestKeyPoolIgnoresAnAffinityKeyOutsideThePool(t *testing.T) { + account := NewMockAccount() + account.AddProvider(schemas.OpenAI, 5, 1000) + account.SetKeysForProvider(schemas.OpenAI, []schemas.Key{ + {ID: "key-a", Name: "Key A", Value: *schemas.NewSecretVar("sk-a"), Models: schemas.WhiteList{"*"}, Weight: 1}, + {ID: "key-b", Name: "Key B", Value: *schemas.NewSecretVar("sk-b"), Models: schemas.WhiteList{"*"}, Weight: 1}, + }) + client, err := Init(context.Background(), schemas.BifrostConfig{ + Account: account, + Logger: NewDefaultLogger(schemas.LogLevelError), + KVStore: newMockKVStore(), + SessionAffinity: foreignKeyAffinity{}, + }) + if err != nil { + t.Fatalf("Init: %v", err) + } + t.Cleanup(client.Shutdown) + + keys, canRotate, err := client.selectKeyFromProviderForModelWithPool(sessionCtx("session-1"), schemas.ChatCompletionRequest, schemas.OpenAI, "gpt-4", schemas.OpenAI) + if err != nil { + t.Fatalf("pool build: %v", err) + } + if !canRotate || len(keys) != 2 { + t.Fatalf("got %d keys canRotate=%v, want the full rotating pool when the affinity names a key outside it", len(keys), canRotate) + } + for _, key := range keys { + if key.ID == "key-foreign" { + t.Fatal("a key the pool never held was handed to the request") + } + } +} diff --git a/core/utils.go b/core/utils.go index 0f4cafdf424..d812e1062a5 100644 --- a/core/utils.go +++ b/core/utils.go @@ -3,8 +3,6 @@ package bifrost import ( "bytes" "context" - "crypto/sha256" - "encoding/hex" "encoding/json" "errors" "fmt" @@ -840,22 +838,6 @@ func IsCodemodeTool(toolName string) bool { return mcp.IsCodeModeTool(toolName) } -// hashSHA256 returns a deterministic hex-encoded SHA-256 hash of the input. -func hashSHA256(value string) string { - h := sha256.Sum256([]byte(value)) - return hex.EncodeToString(h[:]) -} - -func buildSessionKey(providerKey schemas.ModelProvider, sessionID string, model string) string { - // Hash session ID to prevent PII leakage and ensure bounded key size - hashedSessionID := hashSHA256(sessionID) - discriminator := model - if discriminator == "" { - discriminator = "__modelless__" - } - return "session:" + string(providerKey) + ":" + hashedSessionID + ":" + hashSHA256(discriminator) -} - // isPromptOptionalImageEditType returns true for edit task types that do not require a text prompt. // It normalises hyphenated variants (e.g. "erase-object") to underscore form before matching. func isPromptOptionalImageEditType(t *string) bool { diff --git a/docs/quickstart/go-sdk/context-keys.mdx b/docs/quickstart/go-sdk/context-keys.mdx index b47de101539..7f92c429f30 100644 --- a/docs/quickstart/go-sdk/context-keys.mdx +++ b/docs/quickstart/go-sdk/context-keys.mdx @@ -103,7 +103,7 @@ Bind a session to a specific API key so that requests with the same session ID c On the first request for a session ID, Bifrost selects a key (via weighted random) and caches the binding in the KV store. Subsequent requests with the same session ID reuse the cached key as long as it remains valid. If the cached key is no longer in the supported set (disabled, removed, or model support changed), Bifrost re-selects and overwrites the cache. -Session stickiness requires a `KVStore` to be configured in `BifrostConfig`. +The policy that picks and keeps the key is `BifrostConfig.SessionAffinity`. When it is unset, Bifrost's own default applies: it needs a `KVStore` in `BifrostConfig`, binds the session to the key selected for its first request, and scopes bindings to the virtual key and user the request's grant identity is attributed to, so a session id reused by another caller does not share them. A custom `SessionAffinity` may keep its state elsewhere and scope it differently. ```go From 6a5f0643d64507f794b42d12ad8c739e03e78cb6 Mon Sep 17 00:00:00 2001 From: Pratham-Mishra04 Date: Tue, 15 Sep 2026 00:21:58 +0530 Subject: [PATCH 002/118] feat: session affinity at the provider level, bound on outcome --- core/bifrost.go | 48 +- core/bifrost_test.go | 48 +- core/schemas/bifrost.go | 6 + core/schemas/sessionaffinity.go | 64 ++- core/sessionaffinity.go | 265 ++++++++--- core/sessionaffinity_test.go | 714 +++++++++++++++++++++++++---- transports/bifrost-http/lib/ctx.go | 14 + 7 files changed, 963 insertions(+), 196 deletions(-) diff --git a/core/bifrost.go b/core/bifrost.go index 89520754014..179d30867e1 100644 --- a/core/bifrost.go +++ b/core/bifrost.go @@ -302,7 +302,7 @@ func Init(ctx context.Context, config schemas.BifrostConfig) (*Bifrost, error) { // key affinity has nowhere to keep a binding and declines every request. bifrost.sessionAffinity = config.SessionAffinity if bifrost.sessionAffinity == nil { - bifrost.sessionAffinity = newKeyAffinity(bifrost.kvStore, bifrost.keySelector, bifrost.logger) + bifrost.sessionAffinity = newSessionAffinity(bifrost.kvStore, bifrost.logger) } // Initialize object pools @@ -5283,6 +5283,13 @@ func (bifrost *Bifrost) handleRequest(ctx *schemas.BifrostContext, req *schemas. ctx = bifrost.ctx } + // What the caller asked for, before any hook rewrites it: the name under which a session + // remembers where these requests were served. Reported back with how the request ended. + requested := schemas.Route{Provider: provider, Model: model} + var served *schemas.Route + servedFallback := false + defer func() { bifrost.observeSessionOutcome(ctx, requested, served, servedFallback, bifrostErr) }() + // Reset first: bifrost.ctx is shared across every nil-ctx caller. ctx.ResetUpstreamLatency() // Whole-request start for top-down overhead (total minus upstream). On ctx so @@ -5316,6 +5323,8 @@ func (bifrost *Bifrost) handleRequest(ctx *schemas.BifrostContext, req *schemas. preReqPipeline := bifrost.getPluginPipeline() preReqPipeline.RunPreRequestHooks(ctx, req) bifrost.releasePluginPipeline(preReqPipeline) + // The session has the last word on the chain the hooks produced. + bifrost.resolveSessionRoute(ctx, requested, req) bifrost.endCoreSpan(setupSpan) // "miscellaneous" phase: pre-dispatch glue (field re-read + validation) on no other // span. Closed before tryRequest so pipeline-pre stays a separate bucket. @@ -5351,6 +5360,9 @@ func (bifrost *Bifrost) handleRequest(ctx *schemas.BifrostContext, req *schemas. // Check if we should proceed with fallbacks shouldTryFallbacks := bifrost.shouldTryFallbacks(req, primaryErr) if !shouldTryFallbacks { + if primaryErr == nil { + served = &schemas.Route{Provider: provider, Model: model} + } return primaryResult, primaryErr } @@ -5402,6 +5414,8 @@ func (bifrost *Bifrost) handleRequest(ctx *schemas.BifrostContext, req *schemas. bifrost.logger.Debug("successfully used fallback provider %s with model %s", fallback.Provider, fallback.Model) ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelInfo, fmt.Sprintf("Request served by fallback %s/%s (attempt %d/%d)", fallback.Provider, fallback.Model, i+1, len(fallbacks))) tracer.EndSpan(handle, schemas.SpanStatusOk, "") + served = &schemas.Route{Provider: fallback.Provider, Model: fallback.Model} + servedFallback = true return result, nil } @@ -5429,7 +5443,7 @@ func (bifrost *Bifrost) handleRequest(ctx *schemas.BifrostContext, req *schemas. // It handles plugin hooks, request validation, response processing, and fallback providers. // If the primary provider fails, it will try each fallback provider in order until one succeeds. // It is the wrapper for all streaming public API methods. -func (bifrost *Bifrost) handleStreamRequest(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { +func (bifrost *Bifrost) handleStreamRequest(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) (stream chan *schemas.BifrostStreamChunk, bifrostErr *schemas.BifrostError) { defer bifrost.releaseBifrostRequest(req) provider, model, fallbacks := req.GetRequestFields() @@ -5438,6 +5452,13 @@ func (bifrost *Bifrost) handleStreamRequest(ctx *schemas.BifrostContext, req *sc ctx = bifrost.ctx } + // What the caller asked for, before any hook rewrites it: the name under which a session + // remembers where these requests were served. Reported back with how the request ended. + requested := schemas.Route{Provider: provider, Model: model} + var served *schemas.Route + servedFallback := false + defer func() { bifrost.observeSessionOutcome(ctx, requested, served, servedFallback, bifrostErr) }() + ctx.ResetUpstreamLatency() ctx.ResetStreamOverhead() // Whole-request start for overhead on the streaming short-circuit path; the @@ -5459,6 +5480,8 @@ func (bifrost *Bifrost) handleStreamRequest(ctx *schemas.BifrostContext, req *sc preReqPipeline := bifrost.getPluginPipeline() preReqPipeline.RunPreRequestHooks(ctx, req) bifrost.releasePluginPipeline(preReqPipeline) + // The session has the last word on the chain the hooks produced. + bifrost.resolveSessionRoute(ctx, requested, req) // "miscellaneous" phase: pre-dispatch glue (field re-read + validation) on no other // span. Mirrors handleRequest so streaming classifies this cost too. preDispatchSpan := bifrost.startCoreSpan(ctx, "miscellaneous") @@ -5493,6 +5516,9 @@ func (bifrost *Bifrost) handleStreamRequest(ctx *schemas.BifrostContext, req *sc // Check if we should proceed with fallbacks shouldTryFallbacks := bifrost.shouldTryFallbacks(req, primaryErr) if !shouldTryFallbacks { + if primaryErr == nil { + served = &schemas.Route{Provider: provider, Model: model} + } return primaryResult, primaryErr } @@ -5555,6 +5581,8 @@ func (bifrost *Bifrost) handleStreamRequest(ctx *schemas.BifrostContext, req *sc bifrost.logger.Debug("successfully used fallback provider %s with model %s", fallback.Provider, fallback.Model) ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelInfo, fmt.Sprintf("Request served by fallback %s/%s (attempt %d/%d)", fallback.Provider, fallback.Model, i+1, len(fallbacks))) tracer.EndSpan(handle, schemas.SpanStatusOk, "") + served = &schemas.Route{Provider: fallback.Provider, Model: fallback.Model} + servedFallback = true return result, nil } @@ -9287,17 +9315,19 @@ func (bifrost *Bifrost) selectKeyFromProviderForModelWithPool(ctx *schemas.Bifro return []schemas.Key{supportedKeys[0]}, false, nil } - // Session affinity: a session that already has a key here stays on it. The policy decides - // what that means; core honors the key it names, for every attempt of the request, as long + // Session affinity: a request that takes part and already has a key here stays on it. The + // policy decides what that means; core honors the key it names, for every attempt, as long // as that key is one of the eligible ones. A policy naming anything else is ignored, so a // disabled or model-incompatible key can never reach a request through it. - if key, ok := bifrost.sessionAffinity.ResolveKey(ctx, providerKey, model, supportedKeys); ok { - for _, eligible := range supportedKeys { - if eligible.ID == key.ID { - return []schemas.Key{eligible}, false, nil + if schemas.IsSessionAffinityActive(ctx) { + if key, ok := bifrost.sessionAffinity.ResolveKey(ctx, providerKey, model, supportedKeys); ok { + for _, eligible := range supportedKeys { + if eligible.ID == key.ID { + return []schemas.Key{eligible}, false, nil + } } + bifrost.logger.Warn("session affinity named key %s for provider %s, which is not in the eligible pool; selecting a key normally", key.ID, providerKey) } - bifrost.logger.Warn("session affinity named key %s for provider %s, which is not in the eligible pool; selecting a key normally", key.ID, providerKey) } // Normal case: return the full filtered pool with rotation enabled. diff --git a/core/bifrost_test.go b/core/bifrost_test.go index 78ac22f17c4..da002599422 100644 --- a/core/bifrost_test.go +++ b/core/bifrost_test.go @@ -1214,12 +1214,13 @@ func (m *mockKVStore) Delete(key string) (bool, error) { return false, nil } -// Test selectKeyFromProviderForModelWithPool with session stickiness +// Test selectKeyFromProviderForModelWithPool with session stickiness: nothing is bound until a +// request is served, and a bound key then comes back alone with rotation off. func TestSelectKeyFromProviderForModel_SessionStickiness(t *testing.T) { kvStore := newMockKVStore() account := NewMockAccount() account.AddProvider(schemas.OpenAI, 5, 1000) - // Use 2 keys so we hit the keySelector path (single key returns early) + // Use 2 keys so the pool can rotate (single key returns early) account.SetKeysForProvider(schemas.OpenAI, []schemas.Key{ {ID: "key-a", Name: "Key A", Value: *schemas.NewSecretVar("sk-a"), Models: schemas.WhiteList{"*"}, Weight: 1}, {ID: "key-b", Name: "Key B", Value: *schemas.NewSecretVar("sk-b"), Models: schemas.WhiteList{"*"}, Weight: 1}, @@ -1245,40 +1246,44 @@ func TestSelectKeyFromProviderForModel_SessionStickiness(t *testing.T) { bfCtx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) bfCtx.SetValue(schemas.BifrostContextKeySessionID, "sess-123") - // First call: cache miss, keySelector runs, key stored; returns single-element pool (canRotate=false) + // First request: nothing bound, so the whole pool comes back with rotation allowed and the + // pool builder neither selects nor writes. keys1, canRotate1, err := bifrost.selectKeyFromProviderForModelWithPool(bfCtx, schemas.ChatCompletionRequest, schemas.OpenAI, "gpt-4", schemas.OpenAI) if err != nil { t.Fatalf("first selectKeyFromProviderForModelWithPool: %v", err) } - if canRotate1 { - t.Error("first call: canRotate should be false for session-sticky request") + if !canRotate1 || len(keys1) != 2 { + t.Fatalf("first call: got %d keys canRotate=%v, want the full pool with rotation", len(keys1), canRotate1) } - if len(keys1) != 1 || keys1[0].ID != "key-a" { - t.Errorf("first call: expected [key-a], got %v", keys1) + if keySelectorCalls != 0 { + t.Errorf("first call: the pool builder should not select, got %d keySelector calls", keySelectorCalls) } - if keySelectorCalls != 1 { - t.Errorf("first call: expected 1 keySelector call, got %d", keySelectorCalls) + kvKey := sessionStateKey(bfCtx, SessionStateKindKey, string(schemas.OpenAI), "gpt-4") + if _, err := kvStore.Get(kvKey); err == nil { + t.Error("session bound before anything served it") } - // Verify kvstore was written - kvKey := sessionStateKey(bfCtx, SessionStateKindKey, string(schemas.OpenAI), "gpt-4") + // The request is served by key-a, which binds the session to it. + bfCtx.SetValue(schemas.BifrostContextKeySelectedKeyID, "key-a") + route := schemas.Route{Provider: schemas.OpenAI, Model: "gpt-4"} + bifrost.observeSessionOutcome(bfCtx, route, &route, false, nil) if raw, err := kvStore.Get(kvKey); err != nil || raw != "key-a" { - t.Errorf("kvstore after first call: expected key-a, got %v (err=%v)", raw, err) + t.Errorf("kvstore after the request served: expected key-a, got %v (err=%v)", raw, err) } - // Second call: cache hit, same key returned, keySelector NOT called + // Second request: the bound key comes back alone, rotation off, selector not consulted. keys2, canRotate2, err := bifrost.selectKeyFromProviderForModelWithPool(bfCtx, schemas.ChatCompletionRequest, schemas.OpenAI, "gpt-4", schemas.OpenAI) if err != nil { t.Fatalf("second selectKeyFromProviderForModelWithPool: %v", err) } if canRotate2 { - t.Error("second call: canRotate should be false for session-sticky request") + t.Error("second call: canRotate should be false for a session-bound key") } if len(keys2) != 1 || keys2[0].ID != "key-a" { - t.Errorf("second call: expected [key-a] (sticky), got %v", keys2) + t.Errorf("second call: expected [key-a] (bound), got %v", keys2) } - if keySelectorCalls != 1 { - t.Errorf("second call: keySelector should not run (cache hit), got %d calls", keySelectorCalls) + if keySelectorCalls != 0 { + t.Errorf("second call: keySelector should not run, got %d calls", keySelectorCalls) } } @@ -1336,8 +1341,8 @@ func TestSelectKeyFromProviderForModel_NoStickinessWithoutSessionID(t *testing.T } } -// TestSelectKeyFromProviderForModel_SessionStickinessNoRotation verifies that when a session ID -// is present, rate-limit retries reuse the sticky key rather than rotating to another key. +// TestSelectKeyFromProviderForModel_SessionStickinessNoRotation verifies that once a session +// is bound to a key, rate-limit retries reuse that key rather than rotating to another. func TestSelectKeyFromProviderForModel_SessionStickinessNoRotation(t *testing.T) { kvStore := newMockKVStore() account := NewMockAccount() @@ -1366,6 +1371,11 @@ func TestSelectKeyFromProviderForModel_SessionStickinessNoRotation(t *testing.T) bfCtx.SetValue(schemas.BifrostContextKeySessionID, "sess-sticky") bfCtx.SetValue(schemas.BifrostContextKeyTracer, &schemas.NoOpTracer{}) + // An earlier request served by key-a bound the session to it. + bfCtx.SetValue(schemas.BifrostContextKeySelectedKeyID, "key-a") + route := schemas.Route{Provider: schemas.OpenAI, Model: "gpt-4"} + bifrost.observeSessionOutcome(bfCtx, route, &route, false, nil) + config := createTestConfig(3, 0, 0) logger := NewDefaultLogger(schemas.LogLevelError) diff --git a/core/schemas/bifrost.go b/core/schemas/bifrost.go index 1163a2dabc6..c973ea14f55 100644 --- a/core/schemas/bifrost.go +++ b/core/schemas/bifrost.go @@ -412,6 +412,7 @@ const ( BifrostContextKeySSEReaderFactory BifrostContextKey = "bifrost-sse-reader-factory" // *providerUtils.SSEReaderFactory (set by enterprise — replaces default bufio.Scanner SSE readers with streaming readers) BifrostContextKeySessionID BifrostContextKey = "bifrost-session-id" // string session ID for the request (session stickiness) BifrostContextKeySessionTTL BifrostContextKey = "bifrost-session-ttl" // time.Duration session TTL for the request (session stickiness) + BifrostContextKeySessionAffinity BifrostContextKey = "bifrost-session-affinity" // bool: whether the request lets its session decide where it goes (default true) BifrostContextKeyMCPExtraHeaders BifrostContextKey = "bifrost-mcp-extra-headers" // map[string][]string (these headers are forwarded only to the MCP while tool execution if they are in the allowlist of the MCP client) BifrostContextKeyMCPLogID BifrostContextKey = "bifrost-mcp-log-id" // string (unique UUID for each MCP tool log entry - set per goroutine by agent executor - DO NOT SET THIS MANUALLY) BifrostContextKeyMCPHealthCheckRequest BifrostContextKey = "bifrost-mcp-health-check-request" // bool (set by bifrost - DO NOT SET THIS MANUALLY) - true when the MCP connect/ping/list-tools request was generated by bifrost itself (periodic health checks, or the admin connection-verification probe) rather than originating from a caller @@ -470,6 +471,11 @@ const ( RoutingEngineLoadbalancing = "loadbalancing" RoutingEngineModelCatalog = "model-catalog" RoutingEngineCircuitBreaker = "circuit-breaker" + // RoutingEngineSessionAffinity marks decisions a session made for the request: + // staying on the provider or key that served it before, ahead of what the + // routing engines proposed. Recorded so the request's trail says why it went + // where it went. + RoutingEngineSessionAffinity = "session-affinity" // RoutingEngineCore represents the Bifrost core orchestrator's own // routing decisions — primarily fallback transitions. Emitted when the // primary attempt fails and core advances through the fallback chain so diff --git a/core/schemas/sessionaffinity.go b/core/schemas/sessionaffinity.go index 61ce9ba27a8..65d1e204906 100644 --- a/core/schemas/sessionaffinity.go +++ b/core/schemas/sessionaffinity.go @@ -1,19 +1,61 @@ package schemas +// Route is one provider and model a request can be sent to. +type Route struct { + Provider ModelProvider + Model string +} + +// RouteOutcome is how a request ended. +type RouteOutcome struct { + // Served is the route that answered, nil when no route did. + Served *Route + // KeyID is the key Served used, when key selection ran for it. + KeyID string + // Fallback reports that Served was not the first route of the dispatched chain. + Fallback bool + // Err is the failure the request ended in, nil when it was served. + Err *BifrostError +} + +// IsSessionAffinityActive reports whether a request takes part in session affinity: it carries +// a session id and has not switched affinity off (BifrostContextKeySessionAffinity). Core +// consults it before every SessionAffinity call, so an implementation never sees a request +// that does not take part. +func IsSessionAffinityActive(ctx *BifrostContext) bool { + if ctx == nil { + return false + } + if enabled, ok := ctx.Value(BifrostContextKeySessionAffinity).(bool); ok && !enabled { + return false + } + sessionID, _ := ctx.Value(BifrostContextKeySessionID).(string) + return sessionID != "" +} + // SessionAffinity keeps a request that carries a session id on what served that session -// before. Core asks it at defined points and applies the answer; the policy behind the -// answer belongs to the implementation. The one Bifrost ships keeps a session on the key it -// was first served by for as long as that key stays eligible, and is installed when nothing -// else is. A deployment that knows more about its keys, such as their health, registers its -// own through BifrostConfig.SessionAffinity. +// before. Core asks it at three points and applies the answers; the policy behind them +// belongs to the implementation. The one Bifrost ships keeps a session on the provider and +// key that last served it, as far as the request's own outcome shows, and is installed when +// nothing else is. A deployment that knows more, such as the health of its providers and +// keys, registers its own through BifrostConfig.SessionAffinity. // -// Every method receives the request context, which carries the session id -// (BifrostContextKeySessionID), the session TTL (BifrostContextKeySessionTTL) and the -// identifiers that scope a session to its caller. +// Core asks only for requests IsSessionAffinityActive reports as taking part: ones that carry +// a session id and have not switched affinity off. Every method receives the request context, +// which carries the session id (BifrostContextKeySessionID), the session TTL +// (BifrostContextKeySessionTTL) and the identity that scopes the session to its caller. type SessionAffinity interface { + // ResolveRoute is asked once every routing hook has run, with the route the caller asked + // for and the chain the hooks produced: chain[0] is the primary, the rest its fallbacks in + // order. It answers with the chain to dispatch. The chain unchanged, or nil, leaves the + // request as the hooks left it. + ResolveRoute(ctx *BifrostContext, requested Route, chain []Route) []Route // ResolveKey is asked once the eligible key pool for a provider attempt has been built - // with two or more keys in it. It answers with the key the session's state settles on, - // looking one up or binding one now, or with ok=false to leave the choice to ordinary - // key selection. A returned key is used for every attempt of the request. + // with two or more keys in it. It answers with the key the session's state settles on, or + // with ok=false to leave the choice to ordinary key selection. A returned key is used for + // every attempt of the request. ResolveKey(ctx *BifrostContext, provider ModelProvider, model string, eligible []Key) (key Key, ok bool) + // Observe is told how the request ended, once: for a unary request when its response is + // in, for a stream once the upstream has accepted it and started streaming. + Observe(ctx *BifrostContext, requested Route, outcome RouteOutcome) } diff --git a/core/sessionaffinity.go b/core/sessionaffinity.go index d0f708f0323..6f41cfb6510 100644 --- a/core/sessionaffinity.go +++ b/core/sessionaffinity.go @@ -3,6 +3,9 @@ package bifrost import ( "crypto/sha256" "encoding/hex" + "fmt" + "slices" + "strings" "time" "github.com/bytedance/sonic" @@ -19,72 +22,238 @@ import ( const ( sessionStateKeyPrefix = "session:v2:" + // SessionStateKindRoute is the state kind of a session's route binding: the provider and + // model that served the session for what the caller asked for. + SessionStateKindRoute = "route" // SessionStateKindKey is the state kind of a session's key binding within one provider and // model. SessionStateKindKey = "key" + + // sessionAffinityResolvedKey is where the shipped affinity remembers, for the life of one + // request, what it settled at resolve time, so that the outcome can tell a move from a bind + // that lost a race. + sessionAffinityResolvedKey schemas.BifrostContextKey = "bifrost-session-affinity-resolved" ) -// keyAffinity is the session affinity Bifrost ships, installed when the configuration -// registers none: a session stays on the key it was first served by, per provider and model, -// for as long as that key remains eligible. The binding is written when the key is chosen, -// kept fresh on every reuse, and replaced only once the bound key has left the eligible pool. -// -// Only the primary provider of a request is bound. A fallback provider serves because the -// primary could not, and binding the session to it would keep the session there. -type keyAffinity struct { - kv schemas.KVStore - selector schemas.KeySelector - logger schemas.Logger +// sessionResolution is what sessionAffinity settled for a request before it was dispatched. +type sessionResolution struct { + route string // the route binding the request followed, as routeString spells it + keyRoute schemas.Route // the route ResolveKey answered for + keyID string // the key binding the request followed on keyRoute } -// newKeyAffinity builds key affinity over kv, choosing an unbound session's key with selector. -// Neither selector nor logger may be nil. -func newKeyAffinity(kv schemas.KVStore, selector schemas.KeySelector, logger schemas.Logger) *keyAffinity { - return &keyAffinity{kv: kv, selector: selector, logger: logger} +// sessionAffinity is the session affinity Bifrost ships, installed when the configuration +// registers none. It knows nothing beyond what requests show it: a session stays on the +// provider that last served it, when routing still offers that provider, and on the key that +// last served it within a provider, while that key stays eligible. Bindings are written when +// a request is served, never when it fails, so a fallback that served becomes the session's +// new home and a key that only ever failed is never bound. +type sessionAffinity struct { + kv schemas.KVStore + logger schemas.Logger } -// ResolveKey implements schemas.SessionAffinity. -func (k *keyAffinity) ResolveKey(ctx *schemas.BifrostContext, provider schemas.ModelProvider, model string, eligible []schemas.Key) (schemas.Key, bool) { - if k == nil || k.kv == nil || ctx == nil || len(eligible) < 2 { - return schemas.Key{}, false +// newSessionAffinity builds the shipped affinity over kv. Without a store it declines every +// request. The logger may not be nil. +func newSessionAffinity(kv schemas.KVStore, logger schemas.Logger) *sessionAffinity { + return &sessionAffinity{kv: kv, logger: logger} +} + +// ResolveRoute implements schemas.SessionAffinity: for a request that left the provider to +// routing, the provider that last served the session for that model moves to the front of the +// chain when the chain offers it. A request that named its provider is left as it is. +func (a *sessionAffinity) ResolveRoute(ctx *schemas.BifrostContext, requested schemas.Route, chain []schemas.Route) []schemas.Route { + if a == nil || a.kv == nil || ctx == nil { + return chain + } + // A context can carry what an earlier request on it followed, so this request starts clean: + // Observe must never take a stale resolution for its own. + ctx.ClearValue(sessionAffinityResolvedKey) + if len(chain) == 0 { + return chain + } + // A caller that named the provider is asking for it, not for whatever served the session + // last. Such a request is never reordered; only its key within that provider is the + // session's to keep. + if requested.Provider != "" { + return chain + } + key := sessionStateKey(ctx, SessionStateKindRoute, string(requested.Provider), requested.Model) + bound, found := sessionStateString(a.kv, key) + if !found { + return chain + } + boundProvider, _, _ := strings.Cut(bound, "/") + at := slices.IndexFunc(chain, func(route schemas.Route) bool { return string(route.Provider) == boundProvider }) + if at < 0 { + // The chain is what this request may use, so a binding outside it is stale. Dropping it + // lets the next served request bind afresh and the session converge there, instead of + // scattering across routing's picks until the binding expires. + if _, err := a.kv.Delete(key); err != nil { + a.logger.Warn("error dropping session route binding to %s: %s", boundProvider, err.Error()) + } + ctx.AppendRoutingEngineLog(schemas.RoutingEngineSessionAffinity, schemas.LogLevelInfo, fmt.Sprintf("Session was last served by %s for %s, which this request cannot use, so the routing decision stands and the session rebinds on its next served request", boundProvider, requested.Model)) + return chain + } + a.remember(ctx, func(r *sessionResolution) { r.route = routeString(chain[at]) }) + if at == 0 { + return chain } - if sessionIDFromContext(ctx) == "" { + resolved := make([]schemas.Route, 0, len(chain)) + resolved = append(resolved, chain[at]) + resolved = append(resolved, chain[:at]...) + resolved = append(resolved, chain[at+1:]...) + ctx.AppendRoutingEngineLog(schemas.RoutingEngineSessionAffinity, schemas.LogLevelInfo, fmt.Sprintf("Session stays on %s for %s; routing proposed %s", chain[at].Provider, requested.Model, chain[0].Provider)) + schemas.AppendToContextList(ctx, schemas.BifrostContextKeyRoutingEnginesUsed, schemas.RoutingEngineSessionAffinity) + return resolved +} + +// ResolveKey implements schemas.SessionAffinity: the key that last served the session on this +// provider and model is used while it is still eligible. Only the primary provider of a +// request answers from a binding; a fallback provider serves because the primary could not, +// and what it serves with is settled by the outcome. +func (a *sessionAffinity) ResolveKey(ctx *schemas.BifrostContext, provider schemas.ModelProvider, model string, eligible []schemas.Key) (schemas.Key, bool) { + if a == nil || a.kv == nil || ctx == nil || len(eligible) < 2 { return schemas.Key{}, false } if fallbackIndex, _ := ctx.Value(schemas.BifrostContextKeyFallbackIndex).(int); fallbackIndex > 0 { return schemas.Key{}, false } - key := sessionStateKey(ctx, SessionStateKindKey, string(provider), model) + boundID, found := sessionStateString(a.kv, key) + if !found { + return schemas.Key{}, false + } + for _, candidate := range eligible { + if candidate.ID != boundID { + continue + } + a.remember(ctx, func(r *sessionResolution) { + r.keyRoute = schemas.Route{Provider: provider, Model: model} + r.keyID = candidate.ID + }) + ctx.AppendRoutingEngineLog(schemas.RoutingEngineSessionAffinity, schemas.LogLevelInfo, fmt.Sprintf("Session reused key %s for %s/%s", candidate.Name, provider, model)) + schemas.AppendToContextList(ctx, schemas.BifrostContextKeyRoutingEnginesUsed, schemas.RoutingEngineSessionAffinity) + return candidate, true + } + if _, err := a.kv.Delete(key); err != nil { + a.logger.Warn("error deleting stale session key binding for provider=%s: %s", provider, err.Error()) + } + ctx.AppendRoutingEngineLog(schemas.RoutingEngineSessionAffinity, schemas.LogLevelInfo, fmt.Sprintf("The key this session last used for %s/%s is no longer eligible, so one is being picked", provider, model)) + return schemas.Key{}, false +} + +// Observe implements schemas.SessionAffinity: a served request binds the session to the route +// and key that served it. A binding the request followed is refreshed when it served and +// replaced when something else did; a session with no binding takes the first that lands. A +// fallback that served replaces the key binding on its provider, since it picked that key +// without the session. Failures write nothing. +func (a *sessionAffinity) Observe(ctx *schemas.BifrostContext, requested schemas.Route, outcome schemas.RouteOutcome) { + if a == nil || a.kv == nil || ctx == nil || outcome.Served == nil { + return + } ttl := sessionTTLFromContext(ctx) + resolved, _ := ctx.Value(sessionAffinityResolvedKey).(sessionResolution) + served := *outcome.Served + + // Only a request that left the provider to routing binds a route; one that named it has + // nothing to remember at that level, even when a fallback served. + if requested.Provider == "" { + a.settle(sessionStateKey(ctx, SessionStateKindRoute, string(requested.Provider), requested.Model), routeString(served), resolved.route, ttl) + } + if outcome.KeyID != "" { + key := sessionStateKey(ctx, SessionStateKindKey, string(served.Provider), served.Model) + followed := "" + if resolved.keyRoute == served { + followed = resolved.keyID + } + if followed == "" && outcome.Fallback { + // A fallback attempt gets no key from the session, so a key its provider already held + // for the session is older than the one that just served there, and gives way to it. + a.replace(key, outcome.KeyID, ttl) + } else { + a.settle(key, outcome.KeyID, followed, ttl) + } + } +} - if bound, found := k.lookup(key, provider, eligible); found { - if err := k.kv.SetWithTTL(key, bound.ID, ttl); err != nil { - k.logger.Warn("error refreshing session key binding for provider=%s key_id=%s: %s", provider, bound.ID, err.Error()) +// settle writes what served under key. A request that followed no binding binds only if +// nothing has landed first, so parallel first requests of one session converge on whichever +// finished first. A request that followed a binding refreshes it when it served and moves it +// to what served instead. +func (a *sessionAffinity) settle(key, value, followed string, ttl time.Duration) { + if followed == "" { + if _, err := a.kv.SetNXWithTTL(key, value, ttl); err != nil { + a.logger.Warn("error binding session to %s: %s", value, err.Error()) } - return bound, true + return + } + if followed != value { + a.logger.Debug("session moved from %s to %s", followed, value) } + a.replace(key, value, ttl) +} - selected, err := k.selector(ctx, eligible, provider, model) - if err != nil { - k.logger.Warn("error selecting a key to bind the session to for provider=%s: %s", provider, err.Error()) - return schemas.Key{}, false +// replace binds the session to value under key over whatever was there. +func (a *sessionAffinity) replace(key, value string, ttl time.Duration) { + if err := a.kv.SetWithTTL(key, value, ttl); err != nil { + a.logger.Warn("error rebinding session to %s: %s", value, err.Error()) } +} - wrote, err := k.kv.SetNXWithTTL(key, selected.ID, ttl) - if err != nil { - k.logger.Warn("error binding session to provider=%s key_id=%s: %s", provider, selected.ID, err.Error()) - return selected, true +// remember records part of what this request resolved, for Observe. +func (a *sessionAffinity) remember(ctx *schemas.BifrostContext, update func(*sessionResolution)) { + resolution, _ := ctx.Value(sessionAffinityResolvedKey).(sessionResolution) + update(&resolution) + ctx.SetValue(sessionAffinityResolvedKey, resolution) +} + +// routeString spells a route as session state stores it. Provider names carry no slash, so the +// first one separates the two. +func routeString(route schemas.Route) string { + return string(route.Provider) + "/" + route.Model +} + +// resolveSessionRoute asks the session affinity to settle the chain the routing hooks +// produced for req and applies its answer: the first route becomes the primary, the rest its +// fallbacks. A request no hook could route is left for validation to refuse. +func (bifrost *Bifrost) resolveSessionRoute(ctx *schemas.BifrostContext, requested schemas.Route, req *schemas.BifrostRequest) { + if !schemas.IsSessionAffinityActive(ctx) { + return + } + provider, model, fallbacks := req.GetRequestFields() + if provider == "" { + return } - if wrote { - return selected, true + chain := make([]schemas.Route, 0, 1+len(fallbacks)) + chain = append(chain, schemas.Route{Provider: provider, Model: model}) + for _, fallback := range fallbacks { + chain = append(chain, schemas.Route{Provider: fallback.Provider, Model: fallback.Model}) } + resolved := bifrost.sessionAffinity.ResolveRoute(ctx, requested, chain) + if len(resolved) == 0 || slices.Equal(resolved, chain) { + return + } + req.SetProvider(resolved[0].Provider) + req.SetModel(resolved[0].Model) + next := make([]schemas.Fallback, 0, len(resolved)-1) + for _, route := range resolved[1:] { + next = append(next, schemas.Fallback{Provider: route.Provider, Model: route.Model}) + } + req.SetFallbacks(next) +} - // Another request for the same session bound first. Its key is the session's key. - if bound, found := k.lookup(key, provider, eligible); found { - return bound, true +// observeSessionOutcome tells the session affinity how a request ended: the route that served +// it and the key that route used, or nothing served and the error it ended in. +func (bifrost *Bifrost) observeSessionOutcome(ctx *schemas.BifrostContext, requested schemas.Route, served *schemas.Route, fallback bool, err *schemas.BifrostError) { + if !schemas.IsSessionAffinityActive(ctx) { + return + } + outcome := schemas.RouteOutcome{Served: served, Fallback: fallback, Err: err} + if served != nil { + outcome.KeyID, _ = ctx.Value(schemas.BifrostContextKeySelectedKeyID).(string) } - return selected, true + bifrost.sessionAffinity.Observe(ctx, requested, outcome) } // sessionStateKey builds the store key for one piece of session state: its kind, the request's @@ -169,21 +338,3 @@ func sessionStateString(kv schemas.KVStore, key string) (string, bool) { } return value, true } - -// lookup reads the session's binding and resolves it against the eligible pool. A binding to -// a key no longer in the pool is deleted, so the caller binds afresh. -func (k *keyAffinity) lookup(key string, provider schemas.ModelProvider, eligible []schemas.Key) (schemas.Key, bool) { - boundID, found := sessionStateString(k.kv, key) - if !found { - return schemas.Key{}, false - } - for _, candidate := range eligible { - if candidate.ID == boundID { - return candidate, true - } - } - if _, err := k.kv.Delete(key); err != nil { - k.logger.Warn("error deleting stale session key binding for provider=%s: %s", provider, err.Error()) - } - return schemas.Key{}, false -} diff --git a/core/sessionaffinity_test.go b/core/sessionaffinity_test.go index 96714767943..9286bbc9746 100644 --- a/core/sessionaffinity_test.go +++ b/core/sessionaffinity_test.go @@ -2,7 +2,7 @@ package bifrost import ( "context" - "errors" + "slices" "strings" "sync" "testing" @@ -69,8 +69,64 @@ var sessionTestPool = []schemas.Key{ {ID: "key-c", Name: "Key C"}, } -func firstKeySelector(_ *schemas.BifrostContext, keys []schemas.Key, _ schemas.ModelProvider, _ string) (schemas.Key, error) { - return keys[0], nil +func routeOf(provider schemas.ModelProvider, model string) schemas.Route { + return schemas.Route{Provider: provider, Model: model} +} + +func servedBy(route schemas.Route, keyID string, fallback bool) schemas.RouteOutcome { + return schemas.RouteOutcome{Served: &route, KeyID: keyID, Fallback: fallback} +} + +func testAffinity(kv schemas.KVStore) *sessionAffinity { + return newSessionAffinity(kv, NewDefaultLogger(schemas.LogLevelError)) +} + +func enginesUsed(ctx *schemas.BifrostContext) []string { + engines, _ := ctx.Value(schemas.BifrostContextKeyRoutingEnginesUsed).([]string) + return engines +} + +func trailMentions(ctx *schemas.BifrostContext, text string) bool { + for _, entry := range ctx.GetRoutingEngineLogs() { + if entry.Engine == schemas.RoutingEngineSessionAffinity && strings.Contains(entry.Message, text) { + return true + } + } + return false +} + +// recordingAffinity answers what a test tells it and records what core asked. +type recordingAffinity struct { + mu sync.Mutex + routeAnswer func(chain []schemas.Route) []schemas.Route + routeCalls int + keyCalls int + requested []schemas.Route + outcomes []schemas.RouteOutcome +} + +func (r *recordingAffinity) ResolveRoute(_ *schemas.BifrostContext, _ schemas.Route, chain []schemas.Route) []schemas.Route { + r.mu.Lock() + defer r.mu.Unlock() + r.routeCalls++ + if r.routeAnswer != nil { + return r.routeAnswer(chain) + } + return chain +} + +func (r *recordingAffinity) ResolveKey(*schemas.BifrostContext, schemas.ModelProvider, string, []schemas.Key) (schemas.Key, bool) { + r.mu.Lock() + defer r.mu.Unlock() + r.keyCalls++ + return schemas.Key{}, false +} + +func (r *recordingAffinity) Observe(_ *schemas.BifrostContext, requested schemas.Route, outcome schemas.RouteOutcome) { + r.mu.Lock() + defer r.mu.Unlock() + r.requested = append(r.requested, requested) + r.outcomes = append(r.outcomes, outcome) } func TestSessionStateKeyScopesBySessionAndIdentity(t *testing.T) { @@ -92,7 +148,7 @@ func TestSessionStateKeyScopesBySessionAndIdentity(t *testing.T) { "other session": sessionStateKey(attributed(sessionCtx("session-2"), "vk-1", ""), SessionStateKindKey, "openai", "gpt-4o"), "other provider": sessionStateKey(attributed(sessionCtx("session-1"), "vk-1", ""), SessionStateKindKey, "azure", "gpt-4o"), "other model": sessionStateKey(attributed(sessionCtx("session-1"), "vk-1", ""), SessionStateKindKey, "openai", "gpt-4o-mini"), - "other kind": sessionStateKey(attributed(sessionCtx("session-1"), "vk-1", ""), "other", "openai", "gpt-4o"), + "route kind": sessionStateKey(attributed(sessionCtx("session-1"), "vk-1", ""), SessionStateKindRoute, "openai", "gpt-4o"), } for name, v := range variants { if v == base { @@ -137,7 +193,28 @@ func TestSessionTTLFromContext(t *testing.T) { } } -func TestKeyAffinityReadsReplicatedBindings(t *testing.T) { +func TestIsSessionAffinityActive(t *testing.T) { + if schemas.IsSessionAffinityActive(nil) { + t.Fatal("nil context takes part") + } + if schemas.IsSessionAffinityActive(sessionCtx("")) { + t.Fatal("a request without a session takes part") + } + if !schemas.IsSessionAffinityActive(sessionCtx("s")) { + t.Fatal("a request with a session and an unset switch does not take part") + } + ctx := sessionCtx("s") + ctx.SetValue(schemas.BifrostContextKeySessionAffinity, false) + if schemas.IsSessionAffinityActive(ctx) { + t.Fatal("a request that switched affinity off takes part") + } + ctx.SetValue(schemas.BifrostContextKeySessionAffinity, true) + if !schemas.IsSessionAffinityActive(ctx) { + t.Fatal("a request that switched affinity on does not take part") + } +} + +func TestSessionAffinityResolveKeyReadsReplicatedBindings(t *testing.T) { cases := []struct { name string value any @@ -150,165 +227,543 @@ func TestKeyAffinityReadsReplicatedBindings(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { kv := newMockKVStore() - d := newKeyAffinity(kv, firstKeySelector, NewDefaultLogger(schemas.LogLevelError)) ctx := sessionCtx("session-1") _ = kv.SetWithTTL(sessionStateKey(ctx, SessionStateKindKey, "openai", "gpt-4o"), tc.value, time.Minute) - key, ok := d.ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool) + key, ok := testAffinity(kv).ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool) if !ok || key.ID != tc.want { t.Fatalf("got %q ok=%v, want the stored binding %q", key.ID, ok, tc.want) } }) } - // An empty stored value is no binding: the session binds afresh. + // An empty stored value is no binding. kv := newMockKVStore() - d := newKeyAffinity(kv, firstKeySelector, NewDefaultLogger(schemas.LogLevelError)) ctx := sessionCtx("session-1") _ = kv.SetWithTTL(sessionStateKey(ctx, SessionStateKindKey, "openai", "gpt-4o"), "", time.Minute) - if key, ok := d.ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool); !ok || key.ID != "key-a" { - t.Fatalf("empty binding: got %q ok=%v, want a fresh pick of key-a", key.ID, ok) + if _, ok := testAffinity(kv).ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool); ok { + t.Fatal("empty binding produced a key") } } -func TestKeyAffinityBindsThenReuses(t *testing.T) { +func TestSessionAffinityBindsOnOutcomeThenReuses(t *testing.T) { kv := newMockKVStore() - calls := 0 - selector := func(ctx *schemas.BifrostContext, keys []schemas.Key, p schemas.ModelProvider, m string) (schemas.Key, error) { - calls++ - return keys[1], nil - } - d := newKeyAffinity(kv, selector, NewDefaultLogger(schemas.LogLevelError)) + a := testAffinity(kv) + requested := routeOf("", "gpt-4o") + served := routeOf(schemas.OpenAI, "gpt-4o") + + // Nothing bound yet: the pool builder gets no answer and nothing is written. ctx := sessionCtx("session-1") ctx.SetValue(schemas.BifrostContextKeySessionTTL, 5*time.Minute) + if _, ok := a.ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool); ok { + t.Fatal("an unbound session got a key before anything served it") + } + if len(kv.data) != 0 { + t.Fatalf("resolving wrote state: %v", kv.data) + } + + // Served by key-b: the route and the key are bound with the request's TTL. + a.Observe(ctx, requested, servedBy(served, "key-b", false)) + routeKey := sessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o") + keyKey := sessionStateKey(ctx, SessionStateKindKey, "openai", "gpt-4o") + if entry := kv.data[routeKey]; entry.value != "openai/gpt-4o" || entry.ttl != 5*time.Minute { + t.Fatalf("route binding after first request: %+v", entry) + } + if entry := kv.data[keyKey]; entry.value != "key-b" || entry.ttl != 5*time.Minute { + t.Fatalf("key binding after first request: %+v", entry) + } - key, ok := d.ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool) + // The next request reuses key-b and the trail says so. + next := sessionCtx("session-1") + next.SetValue(schemas.BifrostContextKeySessionTTL, 7*time.Minute) + key, ok := a.ResolveKey(next, schemas.OpenAI, "gpt-4o", sessionTestPool) if !ok || key.ID != "key-b" { - t.Fatalf("first request: got %q ok=%v, want the selector's pick key-b", key.ID, ok) + t.Fatalf("second request: got %q ok=%v, want key-b", key.ID, ok) } - stateKey := sessionStateKey(ctx, SessionStateKindKey, "openai", "gpt-4o") - if entry := kv.data[stateKey]; entry.value != "key-b" || entry.ttl != 5*time.Minute { - t.Fatalf("binding after first request: %+v", entry) + if !slices.Contains(enginesUsed(next), schemas.RoutingEngineSessionAffinity) || !trailMentions(next, "reused key Key B") { + t.Fatalf("second request did not record the reuse: engines=%v", enginesUsed(next)) } - // Later requests reuse the binding without asking the selector and refresh its TTL. - ctx.SetValue(schemas.BifrostContextKeySessionTTL, 7*time.Minute) - key, ok = d.ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool) - if !ok || key.ID != "key-b" || calls != 1 { - t.Fatalf("second request: got %q ok=%v selector calls=%d", key.ID, ok, calls) + // Served again by the key it followed: the binding is refreshed, not replaced. + a.Observe(next, requested, servedBy(served, "key-b", false)) + if entry := kv.data[keyKey]; entry.value != "key-b" || entry.ttl != 7*time.Minute { + t.Fatalf("reuse did not refresh the key binding: %+v", entry) } - if entry := kv.data[stateKey]; entry.ttl != 7*time.Minute { - t.Fatalf("reuse did not refresh the TTL: %v", entry.ttl) + + // A fallback provider that served binds its own key, and does not take the route from a + // request that followed no route binding. + a.Observe(next, requested, servedBy(routeOf(schemas.Azure, "gpt-4o"), "az-1", true)) + if entry := kv.data[sessionStateKey(next, SessionStateKindKey, "azure", "gpt-4o")]; entry.value != "az-1" { + t.Fatalf("fallback key binding: %+v", entry) + } + if entry := kv.data[routeKey]; entry.value != "openai/gpt-4o" { + t.Fatalf("route binding overwritten by a request that followed none: %+v", entry) + } +} + +// A fallback picks its key without consulting the session, so a key binding its provider already +// held for the session is older than the key that just served there, and gives way to it. +func TestSessionAffinityFallbackReplacesAnOlderKeyBindingOnItsProvider(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + requested := routeOf("", "gpt-4o") + openai, azure := routeOf(schemas.OpenAI, "gpt-4o"), routeOf(schemas.Azure, "gpt-4o") + + // Earlier turns: openai served with key-a, then the session moved to azure on key-b. + seed := sessionCtx("session-1") + openaiKey := sessionStateKey(seed, SessionStateKindKey, "openai", "gpt-4o") + azureKey := sessionStateKey(seed, SessionStateKindKey, "azure", "gpt-4o") + routeKey := sessionStateKey(seed, SessionStateKindRoute, "", "gpt-4o") + _ = kv.SetWithTTL(openaiKey, "key-a", time.Hour) + _ = kv.SetWithTTL(azureKey, "key-b", time.Hour) + _ = kv.SetWithTTL(routeKey, "azure/gpt-4o", time.Hour) + + // This turn follows azure and its key, azure fails, and the openai fallback serves on key-c, + // picked freely because a fallback attempt gets no key from the session. + ctx := sessionCtx("session-1") + if got := a.ResolveRoute(ctx, requested, []schemas.Route{openai, azure}); got[0] != azure { + t.Fatalf("session did not follow azure: %v", got) + } + if key, ok := a.ResolveKey(ctx, schemas.Azure, "gpt-4o", sessionTestPool); !ok || key.ID != "key-b" { + t.Fatalf("primary attempt: got %q ok=%v, want key-b", key.ID, ok) + } + ctx.SetValue(schemas.BifrostContextKeyFallbackIndex, 1) + if _, ok := a.ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool); ok { + t.Fatal("fallback attempt got a key from the session") } + a.Observe(ctx, requested, servedBy(openai, "key-c", true)) - // Another provider for the same session binds separately. - if key, ok := d.ResolveKey(ctx, schemas.Azure, "gpt-4o", sessionTestPool); !ok || key.ID != "key-b" || calls != 2 { - t.Fatalf("other provider: got %q ok=%v selector calls=%d", key.ID, ok, calls) + if entry := kv.data[routeKey]; entry.value != "openai/gpt-4o" { + t.Fatalf("route binding after the fallback served: %+v, want openai/gpt-4o", entry) + } + if entry := kv.data[openaiKey]; entry.value != "key-c" { + t.Fatalf("openai key binding after its fallback served on key-c: %+v, want key-c", entry) + } + if entry := kv.data[azureKey]; entry.value != "key-b" { + t.Fatalf("azure key binding changed though azure did not serve: %+v", entry) } } -func TestKeyAffinityRebindsWhenBoundKeyLeavesThePool(t *testing.T) { +func TestSessionAffinityRebindsWhenBoundKeyLeavesThePool(t *testing.T) { kv := newMockKVStore() - d := newKeyAffinity(kv, firstKeySelector, NewDefaultLogger(schemas.LogLevelError)) ctx := sessionCtx("session-1") stateKey := sessionStateKey(ctx, SessionStateKindKey, "openai", "gpt-4o") _ = kv.SetWithTTL(stateKey, "key-gone", time.Minute) - key, ok := d.ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool) - if !ok || key.ID != "key-a" { - t.Fatalf("got %q ok=%v, want a fresh pick of key-a", key.ID, ok) + if _, ok := testAffinity(kv).ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool); ok { + t.Fatal("a binding to a key outside the pool produced a key") + } + if _, present := kv.data[stateKey]; present { + t.Fatal("stale binding was not deleted") } - if entry := kv.data[stateKey]; entry.value != "key-a" { - t.Fatalf("stale binding was not replaced: %v", entry.value) + if !trailMentions(ctx, "no longer eligible") { + t.Fatal("stale binding was not explained in the trail") } } -func TestKeyAffinityDeclines(t *testing.T) { +func TestSessionAffinityResolveRoute(t *testing.T) { + chain := []schemas.Route{routeOf(schemas.Groq, "openai/gpt-4o"), routeOf(schemas.OpenAI, "gpt-4o"), routeOf(schemas.Azure, "gpt-4o")} + requested := routeOf("", "gpt-4o") + bind := func(kv *mockKVStore, ctx *schemas.BifrostContext, value string) { + _ = kv.SetWithTTL(sessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o"), value, time.Minute) + } + + t.Run("no binding leaves the chain", func(t *testing.T) { + kv := newMockKVStore() + ctx := sessionCtx("session-1") + if got := testAffinity(kv).ResolveRoute(ctx, requested, chain); !slices.Equal(got, chain) { + t.Fatalf("got %v, want the chain unchanged", got) + } + if _, recorded := ctx.Value(sessionAffinityResolvedKey).(sessionResolution); recorded { + t.Fatal("nothing was followed, nothing should be recorded") + } + }) + + t.Run("bound provider moves to the front and the rest keep their order", func(t *testing.T) { + kv := newMockKVStore() + ctx := sessionCtx("session-1") + bind(kv, ctx, "azure/gpt-4o") + got := testAffinity(kv).ResolveRoute(ctx, requested, chain) + want := []schemas.Route{chain[2], chain[0], chain[1]} + if !slices.Equal(got, want) { + t.Fatalf("got %v, want %v", got, want) + } + if !slices.Contains(enginesUsed(ctx), schemas.RoutingEngineSessionAffinity) || !trailMentions(ctx, "Session stays on azure") { + t.Fatalf("decision not recorded: engines=%v", enginesUsed(ctx)) + } + if res, _ := ctx.Value(sessionAffinityResolvedKey).(sessionResolution); res.route != "azure/gpt-4o" { + t.Fatalf("followed route not recorded: %+v", res) + } + }) + + t.Run("the model comes from the chain, not the binding", func(t *testing.T) { + kv := newMockKVStore() + ctx := sessionCtx("session-1") + bind(kv, ctx, "azure/gpt-4o-old") + got := testAffinity(kv).ResolveRoute(ctx, requested, chain) + if got[0] != chain[2] { + t.Fatalf("got %v, want azure with the chain's model first", got) + } + }) + + t.Run("bound provider already first is left alone and not counted as a decision", func(t *testing.T) { + kv := newMockKVStore() + ctx := sessionCtx("session-1") + bind(kv, ctx, "groq/openai/gpt-4o") + if got := testAffinity(kv).ResolveRoute(ctx, requested, chain); !slices.Equal(got, chain) { + t.Fatalf("got %v, want the chain unchanged", got) + } + if slices.Contains(enginesUsed(ctx), schemas.RoutingEngineSessionAffinity) { + t.Fatal("an unchanged chain was counted as a session decision") + } + if res, _ := ctx.Value(sessionAffinityResolvedKey).(sessionResolution); res.route != "groq/openai/gpt-4o" { + t.Fatalf("followed route not recorded: %+v", res) + } + }) + + t.Run("bound provider the request cannot use leaves the chain and is dropped", func(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + ctx := sessionCtx("session-1") + bind(kv, ctx, "anthropic/claude-sonnet") + if got := a.ResolveRoute(ctx, requested, chain); !slices.Equal(got, chain) { + t.Fatalf("got %v, want the chain unchanged", got) + } + if !trailMentions(ctx, "cannot use") { + t.Fatal("stale binding was not explained in the trail") + } + if _, recorded := ctx.Value(sessionAffinityResolvedKey).(sessionResolution); recorded { + t.Fatal("a binding outside the chain was recorded as followed") + } + routeKey := sessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o") + if _, present := kv.data[routeKey]; present { + t.Fatal("a binding outside the chain was kept") + } + // The request that then serves binds the session afresh, so it converges there. + a.Observe(ctx, requested, servedBy(routeOf(schemas.OpenAI, "gpt-4o"), "oa-1", false)) + if entry := kv.data[routeKey]; entry.value != "openai/gpt-4o" { + t.Fatalf("session did not rebind to what served: %+v", entry) + } + }) + + t.Run("empty chain, nil policy and no store leave the chain", func(t *testing.T) { + kv := newMockKVStore() + bind(kv, sessionCtx("session-1"), "azure/gpt-4o") + if got := testAffinity(kv).ResolveRoute(sessionCtx("session-1"), requested, nil); got != nil { + t.Fatalf("empty chain: got %v", got) + } + var none *sessionAffinity + if got := none.ResolveRoute(sessionCtx("session-1"), requested, chain); !slices.Equal(got, chain) { + t.Fatalf("nil policy: got %v", got) + } + if got := testAffinity(nil).ResolveRoute(sessionCtx("session-1"), requested, chain); !slices.Equal(got, chain) { + t.Fatalf("no store: got %v", got) + } + }) +} + +func TestSessionAffinityObserveRoute(t *testing.T) { + chain := []schemas.Route{routeOf(schemas.OpenAI, "gpt-4o"), routeOf(schemas.Azure, "gpt-4o")} + requested := routeOf("", "gpt-4o") + routeKeyOf := func(ctx *schemas.BifrostContext) string { + return sessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o") + } + + t.Run("a followed route that served is refreshed", func(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + ctx := sessionCtx("session-1") + _ = kv.SetWithTTL(routeKeyOf(ctx), "azure/gpt-4o", time.Minute) + ctx.SetValue(schemas.BifrostContextKeySessionTTL, 9*time.Minute) + a.ResolveRoute(ctx, requested, chain) + a.Observe(ctx, requested, servedBy(routeOf(schemas.Azure, "gpt-4o"), "az-1", false)) + if entry := kv.data[routeKeyOf(ctx)]; entry.value != "azure/gpt-4o" || entry.ttl != 9*time.Minute { + t.Fatalf("route binding after reuse: %+v", entry) + } + }) + + t.Run("a fallback that served moves the route", func(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + ctx := sessionCtx("session-1") + _ = kv.SetWithTTL(routeKeyOf(ctx), "azure/gpt-4o", time.Minute) + a.ResolveRoute(ctx, requested, chain) + a.Observe(ctx, requested, servedBy(routeOf(schemas.OpenAI, "gpt-4o"), "oa-1", true)) + if entry := kv.data[routeKeyOf(ctx)]; entry.value != "openai/gpt-4o" { + t.Fatalf("route binding after a fallback served: %+v", entry) + } + }) + + t.Run("a request that followed no route binding does not overwrite one", func(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + ctx := sessionCtx("session-1") + _ = kv.SetWithTTL(routeKeyOf(ctx), "azure/gpt-4o", time.Minute) + a.Observe(ctx, requested, servedBy(routeOf(schemas.OpenAI, "gpt-4o"), "oa-1", false)) + if entry := kv.data[routeKeyOf(ctx)]; entry.value != "azure/gpt-4o" { + t.Fatalf("route binding overwritten: %+v", entry) + } + }) + + t.Run("failures write nothing", func(t *testing.T) { + kv := newMockKVStore() + ctx := sessionCtx("session-1") + testAffinity(kv).Observe(ctx, requested, schemas.RouteOutcome{Err: &schemas.BifrostError{}}) + if len(kv.data) != 0 { + t.Fatalf("a failure wrote state: %v", kv.data) + } + }) + + t.Run("a served route without a key binds only the route", func(t *testing.T) { + kv := newMockKVStore() + ctx := sessionCtx("session-1") + testAffinity(kv).Observe(ctx, requested, servedBy(routeOf(schemas.OpenAI, "gpt-4o"), "", false)) + if len(kv.data) != 1 { + t.Fatalf("a served route without a key should bind only the route: %v", kv.data) + } + }) +} + +func TestSessionAffinityResolveKeyDeclines(t *testing.T) { kv := newMockKVStore() - d := newKeyAffinity(kv, firstKeySelector, NewDefaultLogger(schemas.LogLevelError)) + a := testAffinity(kv) + bound := sessionCtx("session-1") + _ = kv.SetWithTTL(sessionStateKey(bound, SessionStateKindKey, "openai", "gpt-4o"), "key-a", time.Minute) - if _, ok := d.ResolveKey(sessionCtx(""), schemas.OpenAI, "gpt-4o", sessionTestPool); ok { - t.Fatal("request without a session got a preference") + if _, ok := a.ResolveKey(nil, schemas.OpenAI, "gpt-4o", sessionTestPool); ok { + t.Fatal("nil context got a key") } - if _, ok := d.ResolveKey(nil, schemas.OpenAI, "gpt-4o", sessionTestPool); ok { - t.Fatal("nil context got a preference") - } - if _, ok := d.ResolveKey(sessionCtx("session-1"), schemas.OpenAI, "gpt-4o", sessionTestPool[:1]); ok { - t.Fatal("single-key pool got a preference") + if _, ok := a.ResolveKey(sessionCtx("session-1"), schemas.OpenAI, "gpt-4o", sessionTestPool[:1]); ok { + t.Fatal("single-key pool got a key") } fallback := sessionCtx("session-1") fallback.SetValue(schemas.BifrostContextKeyFallbackIndex, 1) - if _, ok := d.ResolveKey(fallback, schemas.OpenAI, "gpt-4o", sessionTestPool); ok { - t.Fatal("fallback attempt got a preference") - } - if len(kv.data) != 0 { - t.Fatalf("declined requests wrote state: %v", kv.data) + if _, ok := a.ResolveKey(fallback, schemas.OpenAI, "gpt-4o", sessionTestPool); ok { + t.Fatal("fallback attempt got a key") } - - failing := func(*schemas.BifrostContext, []schemas.Key, schemas.ModelProvider, string) (schemas.Key, error) { - return schemas.Key{}, errors.New("no pick") - } - if _, ok := newKeyAffinity(kv, failing, NewDefaultLogger(schemas.LogLevelError)).ResolveKey(sessionCtx("session-1"), schemas.OpenAI, "gpt-4o", sessionTestPool); ok { - t.Fatal("selector error still produced a preference") + if key, ok := a.ResolveKey(sessionCtx("session-1"), schemas.OpenAI, "gpt-4o", sessionTestPool); !ok || key.ID != "key-a" { + t.Fatalf("bound session: got %q ok=%v, want key-a", key.ID, ok) } - - var none *keyAffinity + var none *sessionAffinity if _, ok := none.ResolveKey(sessionCtx("session-1"), schemas.OpenAI, "gpt-4o", sessionTestPool); ok { - t.Fatal("nil policy got a preference") + t.Fatal("nil policy got a key") } - if _, ok := newKeyAffinity(nil, firstKeySelector, NewDefaultLogger(schemas.LogLevelError)).ResolveKey(sessionCtx("session-1"), schemas.OpenAI, "gpt-4o", sessionTestPool); ok { - t.Fatal("policy without a store got a preference") + if _, ok := testAffinity(nil).ResolveKey(sessionCtx("session-1"), schemas.OpenAI, "gpt-4o", sessionTestPool); ok { + t.Fatal("policy without a store got a key") } } -func TestKeyAffinityConcurrentFirstRequestsConverge(t *testing.T) { +func TestSessionAffinityConcurrentFirstRequestsConverge(t *testing.T) { kv := newMockKVStore() - // The selector answers differently per call, as a weighted pick would. - var mu sync.Mutex - next := 0 - selector := func(_ *schemas.BifrostContext, keys []schemas.Key, _ schemas.ModelProvider, _ string) (schemas.Key, error) { - mu.Lock() - defer mu.Unlock() - k := keys[next%len(keys)] - next++ - return k, nil - } - d := newKeyAffinity(kv, selector, NewDefaultLogger(schemas.LogLevelError)) - - results := make([]string, 8) + a := testAffinity(kv) + requested := routeOf("", "gpt-4o") + served := routeOf(schemas.OpenAI, "gpt-4o") + + // Eight first requests of one session resolve before any of them is served: none finds a + // binding, and each is then served by whatever key selection gave it. + contexts := make([]*schemas.BifrostContext, 8) + for i := range contexts { + contexts[i] = sessionCtx("session-1") + if _, ok := a.ResolveKey(contexts[i], schemas.OpenAI, "gpt-4o", sessionTestPool); ok { + t.Fatal("a first request found a binding") + } + } var wg sync.WaitGroup - for i := range results { + for i, ctx := range contexts { wg.Add(1) - go func(i int) { + go func(i int, ctx *schemas.BifrostContext) { defer wg.Done() - key, ok := d.ResolveKey(sessionCtx("session-1"), schemas.OpenAI, "gpt-4o", sessionTestPool) - if ok { - results[i] = key.ID - } - }(i) + a.Observe(ctx, requested, servedBy(served, sessionTestPool[i%len(sessionTestPool)].ID, false)) + }(i, ctx) } wg.Wait() + bound, _ := kv.data[sessionStateKey(sessionCtx("session-1"), SessionStateKindKey, "openai", "gpt-4o")].value.(string) - for i, r := range results { - if r != bound { - t.Fatalf("request %d used %q while the session is bound to %q (all: %v)", i, r, bound, results) - } + if bound == "" { + t.Fatal("no key binding after eight served requests") + } + key, ok := a.ResolveKey(sessionCtx("session-1"), schemas.OpenAI, "gpt-4o", sessionTestPool) + if !ok || key.ID != bound { + t.Fatalf("session resolves to %q ok=%v, want the first bound key %q", key.ID, ok, bound) } } -func TestKeyAffinityScopesSessionsByCaller(t *testing.T) { +func TestSessionAffinityScopesSessionsByCaller(t *testing.T) { kv := newMockKVStore() - d := newKeyAffinity(kv, firstKeySelector, NewDefaultLogger(schemas.LogLevelError)) - a := attributed(sessionCtx("shared-session"), "vk-a", "") - b := attributed(sessionCtx("shared-session"), "vk-b", "") + a := testAffinity(kv) + requested := routeOf("", "gpt-4o") + served := routeOf(schemas.OpenAI, "gpt-4o") + a.Observe(attributed(sessionCtx("shared-session"), "vk-a", ""), requested, servedBy(served, "key-a", false)) + a.Observe(attributed(sessionCtx("shared-session"), "vk-b", ""), requested, servedBy(served, "key-b", false)) + if len(kv.data) != 4 { + t.Fatalf("two callers sharing a session id should hold two route and two key bindings, got %d", len(kv.data)) + } + if key, ok := a.ResolveKey(attributed(sessionCtx("shared-session"), "vk-b", ""), schemas.OpenAI, "gpt-4o", sessionTestPool); !ok || key.ID != "key-b" { + t.Fatalf("caller b resolves to %q ok=%v, want its own key-b", key.ID, ok) + } +} + +func TestResolveSessionRouteAppliesTheAnswer(t *testing.T) { + reverse := func(chain []schemas.Route) []schemas.Route { + out := slices.Clone(chain) + slices.Reverse(out) + return out + } + newRequest := func(provider schemas.ModelProvider) *schemas.BifrostRequest { + req := &schemas.BifrostRequest{ + RequestType: schemas.ChatCompletionRequest, + ChatRequest: &schemas.BifrostChatRequest{Provider: provider, Model: "gpt-4o"}, + } + req.SetFallbacks([]schemas.Fallback{{Provider: schemas.Azure, Model: "gpt-4o"}, {Provider: schemas.Groq, Model: "openai/gpt-4o"}}) + return req + } + setup := func(t *testing.T, answer func([]schemas.Route) []schemas.Route) (*Bifrost, *recordingAffinity) { + t.Helper() + fake := &recordingAffinity{routeAnswer: answer} + client, err := Init(context.Background(), schemas.BifrostConfig{ + Account: NewMockAccount(), + Logger: NewDefaultLogger(schemas.LogLevelError), + SessionAffinity: fake, + }) + if err != nil { + t.Fatalf("Init: %v", err) + } + t.Cleanup(client.Shutdown) + return client, fake + } + + t.Run("the answer becomes primary and fallbacks", func(t *testing.T) { + client, fake := setup(t, reverse) + req := newRequest(schemas.OpenAI) + client.resolveSessionRoute(sessionCtx("s"), routeOf("", "gpt-4o"), req) + provider, model, fallbacks := req.GetRequestFields() + if provider != schemas.Groq || model != "openai/gpt-4o" { + t.Fatalf("primary = %s/%s, want groq/openai/gpt-4o", provider, model) + } + want := []schemas.Fallback{{Provider: schemas.Azure, Model: "gpt-4o"}, {Provider: schemas.OpenAI, Model: "gpt-4o"}} + if !slices.Equal(fallbacks, want) { + t.Fatalf("fallbacks = %v, want %v", fallbacks, want) + } + if fake.routeCalls != 1 { + t.Fatalf("ResolveRoute called %d times, want 1", fake.routeCalls) + } + }) + + t.Run("the same chain, or no answer, leaves the request untouched", func(t *testing.T) { + for name, answer := range map[string]func([]schemas.Route) []schemas.Route{ + "same": nil, + "empty": func([]schemas.Route) []schemas.Route { return nil }, + } { + client, _ := setup(t, answer) + req := newRequest(schemas.OpenAI) + before, beforeModel, beforeFallbacks := req.GetRequestFields() + client.resolveSessionRoute(sessionCtx("s"), routeOf("", "gpt-4o"), req) + provider, model, fallbacks := req.GetRequestFields() + if provider != before || model != beforeModel || !slices.Equal(fallbacks, beforeFallbacks) { + t.Fatalf("%s: request changed to %s/%s %v", name, provider, model, fallbacks) + } + } + }) + + t.Run("a request no hook could route is not offered", func(t *testing.T) { + client, fake := setup(t, reverse) + client.resolveSessionRoute(sessionCtx("s"), routeOf("", "gpt-4o"), newRequest("")) + if fake.routeCalls != 0 { + t.Fatal("ResolveRoute was asked about a request with no provider") + } + }) + + t.Run("a request that takes no part is not offered", func(t *testing.T) { + client, fake := setup(t, reverse) + off := sessionCtx("s") + off.SetValue(schemas.BifrostContextKeySessionAffinity, false) + for _, ctx := range []*schemas.BifrostContext{sessionCtx(""), off} { + req := newRequest(schemas.OpenAI) + client.resolveSessionRoute(ctx, routeOf("", "gpt-4o"), req) + if provider, _, _ := req.GetRequestFields(); provider != schemas.OpenAI { + t.Fatalf("request changed to %s", provider) + } + } + if fake.routeCalls != 0 { + t.Fatal("ResolveRoute was asked about a request that takes no part") + } + }) +} + +func TestObserveSessionOutcomeReportsServedRouteAndKey(t *testing.T) { + fake := &recordingAffinity{} + client, err := Init(context.Background(), schemas.BifrostConfig{ + Account: NewMockAccount(), + Logger: NewDefaultLogger(schemas.LogLevelError), + SessionAffinity: fake, + }) + if err != nil { + t.Fatalf("Init: %v", err) + } + t.Cleanup(client.Shutdown) + + ctx := sessionCtx("s") + ctx.SetValue(schemas.BifrostContextKeySelectedKeyID, "key-a") + requested := routeOf("", "gpt-4o") + client.observeSessionOutcome(ctx, requested, &schemas.Route{Provider: schemas.Azure, Model: "gpt-4o"}, true, nil) + client.observeSessionOutcome(ctx, requested, nil, false, &schemas.BifrostError{}) + + if len(fake.outcomes) != 2 || fake.requested[0] != requested { + t.Fatalf("outcomes recorded: %+v for %v", fake.outcomes, fake.requested) + } + served := fake.outcomes[0] + if served.Served == nil || served.Served.Provider != schemas.Azure || served.KeyID != "key-a" || !served.Fallback || served.Err != nil { + t.Fatalf("served outcome: %+v", served) + } + failed := fake.outcomes[1] + if failed.Served != nil || failed.KeyID != "" || failed.Err == nil { + t.Fatalf("failed outcome: %+v", failed) + } + + // A request that takes no part is not reported. + off := sessionCtx("s") + off.SetValue(schemas.BifrostContextKeySessionAffinity, false) + client.observeSessionOutcome(off, requested, &schemas.Route{Provider: schemas.Azure, Model: "gpt-4o"}, false, nil) + client.observeSessionOutcome(sessionCtx(""), requested, &schemas.Route{Provider: schemas.Azure, Model: "gpt-4o"}, false, nil) + if len(fake.outcomes) != 2 { + t.Fatalf("a request that takes no part was reported: %d outcomes", len(fake.outcomes)) + } +} + +func TestKeyPoolAsksAffinityOnlyForRequestsThatTakePart(t *testing.T) { + account := NewMockAccount() + account.AddProvider(schemas.OpenAI, 5, 1000) + account.SetKeysForProvider(schemas.OpenAI, []schemas.Key{ + {ID: "key-a", Name: "Key A", Value: *schemas.NewSecretVar("sk-a"), Models: schemas.WhiteList{"*"}, Weight: 1}, + {ID: "key-b", Name: "Key B", Value: *schemas.NewSecretVar("sk-b"), Models: schemas.WhiteList{"*"}, Weight: 1}, + }) + fake := &recordingAffinity{} + client, err := Init(context.Background(), schemas.BifrostConfig{ + Account: account, + Logger: NewDefaultLogger(schemas.LogLevelError), + KVStore: newMockKVStore(), + SessionAffinity: fake, + }) + if err != nil { + t.Fatalf("Init: %v", err) + } + t.Cleanup(client.Shutdown) - if _, ok := d.ResolveKey(a, schemas.OpenAI, "gpt-4o", sessionTestPool); !ok { - t.Fatal("caller a got no key") + off := sessionCtx("s") + off.SetValue(schemas.BifrostContextKeySessionAffinity, false) + for name, ctx := range map[string]*schemas.BifrostContext{"no session": sessionCtx(""), "switched off": off} { + keys, canRotate, err := client.selectKeyFromProviderForModelWithPool(ctx, schemas.ChatCompletionRequest, schemas.OpenAI, "gpt-4", schemas.OpenAI) + if err != nil || !canRotate || len(keys) != 2 { + t.Fatalf("%s: got %d keys canRotate=%v err=%v, want the full rotating pool", name, len(keys), canRotate, err) + } + } + if fake.keyCalls != 0 { + t.Fatalf("ResolveKey was asked %d times about requests that take no part", fake.keyCalls) } - if _, ok := d.ResolveKey(b, schemas.OpenAI, "gpt-4o", sessionTestPool); !ok { - t.Fatal("caller b got no key") + if _, _, err := client.selectKeyFromProviderForModelWithPool(sessionCtx("s"), schemas.ChatCompletionRequest, schemas.OpenAI, "gpt-4", schemas.OpenAI); err != nil { + t.Fatalf("pool build: %v", err) } - if len(kv.data) != 2 { - t.Fatalf("two callers sharing a session id should hold two bindings, got %d", len(kv.data)) + if fake.keyCalls != 1 { + t.Fatalf("ResolveKey was asked %d times about a request that takes part, want 1", fake.keyCalls) } } @@ -316,10 +771,16 @@ func TestKeyAffinityScopesSessionsByCaller(t *testing.T) { // SessionAffinity could. type foreignKeyAffinity struct{} +func (foreignKeyAffinity) ResolveRoute(_ *schemas.BifrostContext, _ schemas.Route, chain []schemas.Route) []schemas.Route { + return chain +} + func (foreignKeyAffinity) ResolveKey(*schemas.BifrostContext, schemas.ModelProvider, string, []schemas.Key) (schemas.Key, bool) { return schemas.Key{ID: "key-foreign", Name: "Foreign"}, true } +func (foreignKeyAffinity) Observe(*schemas.BifrostContext, schemas.Route, schemas.RouteOutcome) {} + func TestKeyPoolIgnoresAnAffinityKeyOutsideThePool(t *testing.T) { account := NewMockAccount() account.AddProvider(schemas.OpenAI, 5, 1000) @@ -351,3 +812,56 @@ func TestKeyPoolIgnoresAnAffinityKeyOutsideThePool(t *testing.T) { } } } + +func TestSessionAffinityForgetsWhatAnEarlierRequestFollowed(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + requested := routeOf("", "gpt-4o") + chain := []schemas.Route{routeOf(schemas.OpenAI, "gpt-4o"), routeOf(schemas.Azure, "gpt-4o")} + ctx := sessionCtx("session-1") + routeKey := sessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o") + + // An earlier request on this context followed a binding to azure. + _ = kv.SetWithTTL(routeKey, "azure/gpt-4o", time.Minute) + a.ResolveRoute(ctx, requested, chain) + + // The binding expires. The next request on the same context finds none, and while it is + // in flight another request of the session binds openai first. + _, _ = kv.Delete(routeKey) + a.ResolveRoute(ctx, requested, chain) + _ = kv.SetWithTTL(routeKey, "openai/gpt-4o", time.Minute) + + a.Observe(ctx, requested, servedBy(routeOf(schemas.Azure, "gpt-4o"), "az-1", false)) + if entry := kv.data[routeKey]; entry.value != "openai/gpt-4o" { + t.Fatalf("a request that followed nothing overwrote another request's binding: %+v", entry) + } +} + +func TestSessionAffinityLeavesAnExplicitProviderAlone(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + // The caller named openai; routing kept it first and added azure as a fallback. + requested := routeOf(schemas.OpenAI, "gpt-4o") + chain := []schemas.Route{routeOf(schemas.OpenAI, "gpt-4o"), routeOf(schemas.Azure, "gpt-4o")} + ctx := sessionCtx("session-1") + routeKey := sessionStateKey(ctx, SessionStateKindRoute, "openai", "gpt-4o") + + // A fallback that served does not become the session's home when the caller named the + // provider: no route binding is written, only the key the fallback used. + a.Observe(ctx, requested, servedBy(routeOf(schemas.Azure, "gpt-4o"), "az-1", true)) + if _, present := kv.data[routeKey]; present { + t.Fatal("a route binding was written for a request that named its provider") + } + if entry := kv.data[sessionStateKey(ctx, SessionStateKindKey, "azure", "gpt-4o")]; entry.value != "az-1" { + t.Fatalf("the fallback's key binding was not written: %+v", entry) + } + + // Even with a binding in the store, a request that named its provider is not reordered. + _ = kv.SetWithTTL(routeKey, "azure/gpt-4o", time.Minute) + if got := a.ResolveRoute(ctx, requested, chain); !slices.Equal(got, chain) { + t.Fatalf("a request that named openai was sent to %v", got) + } + if slices.Contains(enginesUsed(ctx), schemas.RoutingEngineSessionAffinity) { + t.Fatal("the session was recorded as deciding a route the caller named") + } +} diff --git a/transports/bifrost-http/lib/ctx.go b/transports/bifrost-http/lib/ctx.go index 27fb35ef86c..1052026e834 100644 --- a/transports/bifrost-http/lib/ctx.go +++ b/transports/bifrost-http/lib/ctx.go @@ -572,6 +572,20 @@ func ConvertToBifrostContext(ctx *fasthttp.RequestCtx, store HandlerStore) (*sch } return true } + // Session affinity: whether this request lets its session decide where it goes. + if keyStr == "x-bf-session-affinity" { + switch strings.ToLower(strings.TrimSpace(string(value))) { + case "on", "true", "1": + bifrostCtx.SetValue(schemas.BifrostContextKeySessionAffinity, true) + case "off", "false", "0": + bifrostCtx.SetValue(schemas.BifrostContextKeySessionAffinity, false) + default: + if logger != nil { + logger.Warn("x-bf-session-affinity is not on or off, ignoring") + } + } + return true + } if labelName, ok := strings.CutPrefix(keyStr, "x-bf-eh-"); ok { // Skip empty header names after prefix removal if labelName == "" { From cd8c35f00e4fc1822baa62ca686d356757cf5c26 Mon Sep 17 00:00:00 2001 From: Pratham-Mishra04 Date: Tue, 15 Sep 2026 00:37:31 +0530 Subject: [PATCH 003/118] ui: session affinity at the provider level, and its engine in the logs UI --- docs/providers/request-options.mdx | 20 +++++++++++++------- docs/quickstart/go-sdk/context-keys.mdx | 19 ++++++++++++++----- ui/lib/constants/icons.tsx | 3 ++- ui/lib/constants/logs.ts | 2 ++ 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/docs/providers/request-options.mdx b/docs/providers/request-options.mdx index 1c505d8b832..8918a420f69 100644 --- a/docs/providers/request-options.mdx +++ b/docs/providers/request-options.mdx @@ -14,8 +14,9 @@ Bifrost provides request options that control behavior, enable features, and pas | `BifrostContextKeyAPIKeyName` | `x-bf-api-key` | `string` | Explicit API key name selection | | `BifrostContextKeyAPIKeyID` | `x-bf-api-key-id` | `string` | Explicit API key ID selection (takes priority over name) | | `BifrostContextKeyDirectKey` | `x-bf-direct-key` (+ `Authorization` / `x-api-key` / `x-goog-api-key`) | `schemas.Key` | Use a caller-supplied raw provider key directly, bypassing the registered key pool. On the gateway, requires `allow_direct_keys` server-side | -| `BifrostContextKeySessionID` | `x-bf-session-id` (falls back to coding-harness session headers) | `string` | Session ID for key stickiness (requires KV store) | -| `BifrostContextKeySessionTTL` | `x-bf-session-ttl` | `time.Duration` | Session-to-key cache TTL (duration string or seconds) | +| `BifrostContextKeySessionID` | `x-bf-session-id` (falls back to coding-harness session headers) | `string` | Session ID for provider and key session affinity (requires KV store) | +| `BifrostContextKeySessionTTL` | `x-bf-session-ttl` | `time.Duration` | Session binding TTL (duration string or seconds) | +| `BifrostContextKeySessionAffinity` | `x-bf-session-affinity` | `bool` | Whether the session decides routing for this request: `on` (default) or `off` | | `BifrostContextKeyRequestID` | `x-request-id` | `string` | Custom request ID for tracking | | `BifrostContextKeySendBackRawRequest` | `x-bf-send-back-raw-request` | `bool` | Include raw provider request in the response | | `BifrostContextKeySendBackRawResponse` | `x-bf-send-back-raw-response` | `bool` | Include raw provider response in the response | @@ -229,13 +230,18 @@ response, err := client.ChatCompletionRequest(ctx, &schemas.BifrostChatRequest{ **Type:** `string` **Required:** No -Bind a session to a specific API key so that requests with the same session ID consistently use the same key. Useful for predictable rate-limit buckets, cost attribution per user, and consistent model routing per session. +Keep a session on what served it before, so a conversation keeps hitting the same provider prompt cache. Affinity works at two levels: -On the first request for a session ID, Bifrost selects a key and caches the binding in the KV store. Subsequent requests with the same session ID reuse the cached key as long as it remains valid. +- **Provider**: when routing offers more than one provider for what the caller asked for (virtual key weights, load balancing, model catalog candidates), the provider that last served the session is tried first, as long as routing still offers it. Routing decides which providers are allowed; the session only decides their order. A request that names its provider (`openai/gpt-4o`) is never reordered: the caller asked for that provider, so only the key level applies to it, even when a fallback served an earlier turn. +- **Key**: within a provider, the key that last served the session is used while it stays eligible. + +Bindings are written when a request is served, never when it fails, and are scoped to the virtual key and user the request is attributed to. A fallback that serves becomes the session's new home. Bindings live in the KV store for the session TTL, refreshed by every served request. **Retry and Fallback Behavior:** -- **Retries**: Session stickiness persists across retry attempts. If a request fails and is retried, the same sticky key is used. -- **Fallbacks**: When falling back to a different provider (e.g., from OpenAI to Anthropic), session stickiness is disabled and a new key is selected for the fallback provider. This ensures availability when the primary provider's keys are exhausted or failing. +- **Retries**: A bound key is used for every retry of a request. A session with no key binding yet rotates keys on rate limits like any other request, and the key that serves is the one that gets bound. +- **Fallbacks**: Fallback providers select keys freely. When a fallback serves, the session is rebound to that provider and its key, so the next request starts there. +- **Switch**: `x-bf-session-affinity: off` turns affinity off for a request, at both levels; `on` is the default. +- **Routing trail**: Decisions the session made appear in the request's routing logs under the `session-affinity` engine, which is also listed among the routing engines used. **Coding Harness Headers:** @@ -294,7 +300,7 @@ Input: messages, **Type:** `time.Duration` (header value: duration string like `"30m"` or `"1h"`, or seconds as integer) **Required:** No -Optional. Controls how long the session-to-key binding is cached. If not set, Bifrost uses 1 hour. The TTL is refreshed on each request so active sessions do not expire. +Optional. Controls how long the session bindings are kept. If not set, Bifrost uses 1 hour. The TTL is refreshed by each served request so active sessions do not expire. Accepts duration strings (`"30s"`, `"5m"`, `"1h"`) or plain numbers (treated as seconds). diff --git a/docs/quickstart/go-sdk/context-keys.mdx b/docs/quickstart/go-sdk/context-keys.mdx index 7f92c429f30..9849db2fce4 100644 --- a/docs/quickstart/go-sdk/context-keys.mdx +++ b/docs/quickstart/go-sdk/context-keys.mdx @@ -98,9 +98,9 @@ A direct key bypasses the **key pool** (no weighted selection, no rotation or fa ### Session Stickiness (Session ID) -Bind a session to a specific API key so that requests with the same session ID consistently use the same key. Useful for predictable rate-limit buckets, cost attribution per user, and consistent model routing per session. +Keep a session on what served it before: the provider that last served it, when routing still offers that provider, and the key that last served it within that provider. A request that names its provider is never reordered; only the key level applies to it. Useful for keeping provider prompt caches warm across a conversation, predictable rate-limit buckets, and cost attribution per user. -On the first request for a session ID, Bifrost selects a key (via weighted random) and caches the binding in the KV store. Subsequent requests with the same session ID reuse the cached key as long as it remains valid. If the cached key is no longer in the supported set (disabled, removed, or model support changed), Bifrost re-selects and overwrites the cache. +Bindings are written when a request is served, never when it fails. A later request with the same session ID puts the bound provider first among the routes the hooks produced and uses the bound key while it remains eligible; a key that is no longer in the supported set (disabled, removed, or model support changed) is dropped and the next served request rebinds. A fallback that serves becomes the session's new home. Decisions the session made are recorded in the request's routing logs under the `session-affinity` engine. The policy that picks and keeps the key is `BifrostConfig.SessionAffinity`. When it is unset, Bifrost's own default applies: it needs a `KVStore` in `BifrostConfig`, binds the session to the key selected for its first request, and scopes bindings to the virtual key and user the request's grant identity is attributed to, so a session id reused by another caller does not share them. A custom `SessionAffinity` may keep its state elsewhere and scope it differently. @@ -112,12 +112,20 @@ bfCtx.SetValue(schemas.BifrostContextKeySessionID, "user-123-session-abc") ### Session TTL -Controls how long the session-to-key binding is cached. If not set, Bifrost uses a default TTL of 1 hour. The TTL is refreshed on each request so active sessions do not expire. +Controls how long the session bindings are kept. If not set, Bifrost uses a default TTL of 1 hour. The TTL is refreshed by each served request so active sessions do not expire. ```go bfCtx.SetValue(schemas.BifrostContextKeySessionTTL, 30*time.Minute) ``` +### Session Affinity Switch + +Whether a request lets its session decide where it goes. On by default; `false` leaves the provider to routing and the key to key selection, and writes no session state for the request. + +```go +bfCtx.SetValue(schemas.BifrostContextKeySessionAffinity, false) +``` + ### Request ID Set a custom request ID for tracking and correlation. @@ -366,8 +374,9 @@ func makeRequest(client *bifrost.Bifrost) { | `BifrostContextKeyRequestID` | `string` | Set | Custom request ID for tracking | | `BifrostContextKeyFallbackRequestID` | `string` | Read | Request ID used for fallback attempt | | `BifrostContextKeySkipKeySelection` | `bool` | Set | Skip key selection entirely | -| `BifrostContextKeySessionID` | `string` | Set | Session ID for key stickiness (requires KV store) | -| `BifrostContextKeySessionTTL` | `time.Duration` | Set | TTL for session-to-key cache (default: 1 hour) | +| `BifrostContextKeySessionID` | `string` | Set | Session ID for provider and key session affinity (requires KV store) | +| `BifrostContextKeySessionTTL` | `time.Duration` | Set | TTL for session bindings (default: 1 hour) | +| `BifrostContextKeySessionAffinity` | `bool` | Set | Whether the session decides routing for the request (default: true) | | `BifrostContextKeyExtraHeaders` | `map[string][]string` | Set | Custom headers forwarded to the provider | | `BifrostContextKeyURLPath` | `string` | Set | Custom URL path appended to provider base URL | | `BifrostContextKeyStreamIdleTimeout` | `time.Duration` | Set | Per-chunk idle timeout for streaming responses | diff --git a/ui/lib/constants/icons.tsx b/ui/lib/constants/icons.tsx index c2661bedf21..4954d98b9e2 100644 --- a/ui/lib/constants/icons.tsx +++ b/ui/lib/constants/icons.tsx @@ -1,4 +1,4 @@ -import { Database, Landmark, Network, Shuffle, Workflow } from "lucide-react"; +import { Database, Landmark, Link2, Network, Shuffle, Workflow } from "lucide-react"; import { useTheme } from "next-themes"; import { cn } from "../utils"; @@ -853,6 +853,7 @@ export const RoutingEngineUsedIcons = { governance: ({ className = "h-5 w-5 text-green-800" }: { className?: string } = {}) => , loadbalancing: ({ className = "h-5 w-5 text-orange-800" }: { className?: string } = {}) => , "model-catalog": ({ className = "h-5 w-5 text-purple-800" }: { className?: string } = {}) => , + "session-affinity": ({ className = "h-5 w-5 text-teal-800" }: { className?: string } = {}) => , core: ({ className = "h-5 w-5 text-sky-800" }: { className?: string } = {}) => , } as const; diff --git a/ui/lib/constants/logs.ts b/ui/lib/constants/logs.ts index 14ca81d502a..cb25708f38d 100644 --- a/ui/lib/constants/logs.ts +++ b/ui/lib/constants/logs.ts @@ -450,6 +450,7 @@ export const RoutingEngineUsedLabels = { governance: "Governance", loadbalancing: "Loadbalancing", "model-catalog": "Model Catalog", + "session-affinity": "Session", core: "Core", } as const; @@ -458,6 +459,7 @@ export const RoutingEngineUsedColors = { governance: "bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-300", loadbalancing: "bg-orange-100 text-orange-800 dark:bg-orange-900 dark:text-orange-300", "model-catalog": "bg-purple-100 text-purple-800 dark:bg-purple-900 dark:text-purple-300", + "session-affinity": "bg-teal-100 text-teal-800 dark:bg-teal-900 dark:text-teal-300", core: "bg-sky-100 text-sky-800 dark:bg-sky-900 dark:text-sky-300", } as const; From 776e70f296b8486e20f94a4c95e87ae9b3bc6fc1 Mon Sep 17 00:00:00 2001 From: Pratham-Mishra04 Date: Tue, 15 Sep 2026 01:09:21 +0530 Subject: [PATCH 004/118] refactor: complexity session state keys through the shared session-state builder --- core/bifrost.go | 2 +- core/bifrost_test.go | 2 +- core/sessionaffinity.go | 65 ++++++++++------ core/sessionaffinity_test.go | 92 +++++++++++++++-------- plugins/routing/complexityrouting.go | 5 +- plugins/routing/complexitysession.go | 35 ++------- plugins/routing/complexitysession_test.go | 51 +++++++++---- plugins/routing/main.go | 2 +- 8 files changed, 151 insertions(+), 103 deletions(-) diff --git a/core/bifrost.go b/core/bifrost.go index 179d30867e1..2bb28294598 100644 --- a/core/bifrost.go +++ b/core/bifrost.go @@ -302,7 +302,7 @@ func Init(ctx context.Context, config schemas.BifrostConfig) (*Bifrost, error) { // key affinity has nowhere to keep a binding and declines every request. bifrost.sessionAffinity = config.SessionAffinity if bifrost.sessionAffinity == nil { - bifrost.sessionAffinity = newSessionAffinity(bifrost.kvStore, bifrost.logger) + bifrost.sessionAffinity = NewSessionAffinity(bifrost.kvStore, bifrost.logger) } // Initialize object pools diff --git a/core/bifrost_test.go b/core/bifrost_test.go index da002599422..aad4f923031 100644 --- a/core/bifrost_test.go +++ b/core/bifrost_test.go @@ -1258,7 +1258,7 @@ func TestSelectKeyFromProviderForModel_SessionStickiness(t *testing.T) { if keySelectorCalls != 0 { t.Errorf("first call: the pool builder should not select, got %d keySelector calls", keySelectorCalls) } - kvKey := sessionStateKey(bfCtx, SessionStateKindKey, string(schemas.OpenAI), "gpt-4") + kvKey := SessionStateKey(bfCtx, SessionStateKindKey, string(schemas.OpenAI), "gpt-4") if _, err := kvStore.Get(kvKey); err == nil { t.Error("session bound before anything served it") } diff --git a/core/sessionaffinity.go b/core/sessionaffinity.go index 6f41cfb6510..3e81703b5b1 100644 --- a/core/sessionaffinity.go +++ b/core/sessionaffinity.go @@ -53,9 +53,11 @@ type sessionAffinity struct { logger schemas.Logger } -// newSessionAffinity builds the shipped affinity over kv. Without a store it declines every -// request. The logger may not be nil. -func newSessionAffinity(kv schemas.KVStore, logger schemas.Logger) *sessionAffinity { +// NewSessionAffinity builds the affinity Bifrost ships over kv, the one Init installs when the +// configuration registers none. It is returned as the interface so a deployment that registers +// its own can embed it and delegate: decide first from what it knows, then let this one keep +// the bindings. Without a store it declines every request. The logger may not be nil. +func NewSessionAffinity(kv schemas.KVStore, logger schemas.Logger) schemas.SessionAffinity { return &sessionAffinity{kv: kv, logger: logger} } @@ -78,24 +80,32 @@ func (a *sessionAffinity) ResolveRoute(ctx *schemas.BifrostContext, requested sc if requested.Provider != "" { return chain } - key := sessionStateKey(ctx, SessionStateKindRoute, string(requested.Provider), requested.Model) - bound, found := sessionStateString(a.kv, key) + key := SessionStateKey(ctx, SessionStateKindRoute, string(requested.Provider), requested.Model) + bound, found := SessionStateString(a.kv, key) if !found { return chain } - boundProvider, _, _ := strings.Cut(bound, "/") - at := slices.IndexFunc(chain, func(route schemas.Route) bool { return string(route.Provider) == boundProvider }) + boundRoute, ok := ParseRouteState(bound) + if !ok { + // Nothing can follow a binding that names no route, and keeping it would refuse the + // next served request's first-writer bind until it expired. + if _, err := a.kv.Delete(key); err != nil { + a.logger.Warn("error dropping unreadable session route binding: %s", err.Error()) + } + return chain + } + at := slices.IndexFunc(chain, func(route schemas.Route) bool { return route.Provider == boundRoute.Provider }) if at < 0 { // The chain is what this request may use, so a binding outside it is stale. Dropping it // lets the next served request bind afresh and the session converge there, instead of // scattering across routing's picks until the binding expires. if _, err := a.kv.Delete(key); err != nil { - a.logger.Warn("error dropping session route binding to %s: %s", boundProvider, err.Error()) + a.logger.Warn("error dropping session route binding to %s: %s", boundRoute.Provider, err.Error()) } - ctx.AppendRoutingEngineLog(schemas.RoutingEngineSessionAffinity, schemas.LogLevelInfo, fmt.Sprintf("Session was last served by %s for %s, which this request cannot use, so the routing decision stands and the session rebinds on its next served request", boundProvider, requested.Model)) + ctx.AppendRoutingEngineLog(schemas.RoutingEngineSessionAffinity, schemas.LogLevelInfo, fmt.Sprintf("Session was last served by %s for %s, which this request cannot use, so the routing decision stands and the session rebinds on its next served request", boundRoute.Provider, requested.Model)) return chain } - a.remember(ctx, func(r *sessionResolution) { r.route = routeString(chain[at]) }) + a.remember(ctx, func(r *sessionResolution) { r.route = RouteStateValue(chain[at]) }) if at == 0 { return chain } @@ -119,8 +129,8 @@ func (a *sessionAffinity) ResolveKey(ctx *schemas.BifrostContext, provider schem if fallbackIndex, _ := ctx.Value(schemas.BifrostContextKeyFallbackIndex).(int); fallbackIndex > 0 { return schemas.Key{}, false } - key := sessionStateKey(ctx, SessionStateKindKey, string(provider), model) - boundID, found := sessionStateString(a.kv, key) + key := SessionStateKey(ctx, SessionStateKindKey, string(provider), model) + boundID, found := SessionStateString(a.kv, key) if !found { return schemas.Key{}, false } @@ -159,10 +169,10 @@ func (a *sessionAffinity) Observe(ctx *schemas.BifrostContext, requested schemas // Only a request that left the provider to routing binds a route; one that named it has // nothing to remember at that level, even when a fallback served. if requested.Provider == "" { - a.settle(sessionStateKey(ctx, SessionStateKindRoute, string(requested.Provider), requested.Model), routeString(served), resolved.route, ttl) + a.settle(SessionStateKey(ctx, SessionStateKindRoute, string(requested.Provider), requested.Model), RouteStateValue(served), resolved.route, ttl) } if outcome.KeyID != "" { - key := sessionStateKey(ctx, SessionStateKindKey, string(served.Provider), served.Model) + key := SessionStateKey(ctx, SessionStateKindKey, string(served.Provider), served.Model) followed := "" if resolved.keyRoute == served { followed = resolved.keyID @@ -208,12 +218,22 @@ func (a *sessionAffinity) remember(ctx *schemas.BifrostContext, update func(*ses ctx.SetValue(sessionAffinityResolvedKey, resolution) } -// routeString spells a route as session state stores it. Provider names carry no slash, so the -// first one separates the two. -func routeString(route schemas.Route) string { +// RouteStateValue spells a route as a route binding stores it. Provider names carry no slash, +// so the first one separates the provider from the model. +func RouteStateValue(route schemas.Route) string { return string(route.Provider) + "/" + route.Model } +// ParseRouteState reads a route back from what RouteStateValue wrote. ok is false when the +// value names no provider. +func ParseRouteState(value string) (schemas.Route, bool) { + provider, model, found := strings.Cut(value, "/") + if !found || provider == "" { + return schemas.Route{}, false + } + return schemas.Route{Provider: schemas.ModelProvider(provider), Model: model}, true +} + // resolveSessionRoute asks the session affinity to settle the chain the routing hooks // produced for req and applies its answer: the first route becomes the primary, the rest its // fallbacks. A request no hook could route is left for validation to refuse. @@ -256,14 +276,15 @@ func (bifrost *Bifrost) observeSessionOutcome(ctx *schemas.BifrostContext, reque bifrost.sessionAffinity.Observe(ctx, requested, outcome) } -// sessionStateKey builds the store key for one piece of session state: its kind, the request's +// SessionStateKey builds the store key for one piece of session state: its kind, the request's // session, who that session belongs to, and whatever further parts tell instances of that kind // apart (a provider and model for a key binding). The session and its owner come from ctx: the // session id, and the virtual key and user the grant identity attributes the request to. A // request nothing governs, or whose identity is not settled, is scoped to the deployment. // Everything but the kind is hashed, so the key is bounded in size and carries no -// caller-supplied text. -func sessionStateKey(ctx *schemas.BifrostContext, kind string, parts ...string) string { +// caller-supplied text. Any component that keeps per-session state keys it this way, so one +// session id means one thing everywhere. +func SessionStateKey(ctx *schemas.BifrostContext, kind string, parts ...string) string { virtualKeyID, userID := sessionIdentityParts(ctx) h := sha256.New() for _, part := range append([]string{virtualKeyID, userID, sessionIDFromContext(ctx)}, parts...) { @@ -316,10 +337,10 @@ func sessionTTLFromContext(ctx *schemas.BifrostContext) time.Duration { return schemas.DefaultSessionStickyTTL } -// sessionStateString reads the string under key. A missing key, a read error and an empty +// SessionStateString reads the string under key. A missing key, a read error and an empty // value all read as not found. Values come back as strings from a local store and as JSON // bytes from one that replicated them, so both are accepted. -func sessionStateString(kv schemas.KVStore, key string) (string, bool) { +func SessionStateString(kv schemas.KVStore, key string) (string, bool) { raw, err := kv.Get(key) if err != nil { return "", false diff --git a/core/sessionaffinity_test.go b/core/sessionaffinity_test.go index 9286bbc9746..28cf14772d7 100644 --- a/core/sessionaffinity_test.go +++ b/core/sessionaffinity_test.go @@ -78,7 +78,7 @@ func servedBy(route schemas.Route, keyID string, fallback bool) schemas.RouteOut } func testAffinity(kv schemas.KVStore) *sessionAffinity { - return newSessionAffinity(kv, NewDefaultLogger(schemas.LogLevelError)) + return NewSessionAffinity(kv, NewDefaultLogger(schemas.LogLevelError)).(*sessionAffinity) } func enginesUsed(ctx *schemas.BifrostContext) []string { @@ -130,25 +130,25 @@ func (r *recordingAffinity) Observe(_ *schemas.BifrostContext, requested schemas } func TestSessionStateKeyScopesBySessionAndIdentity(t *testing.T) { - base := sessionStateKey(attributed(sessionCtx("session-1"), "vk-1", ""), SessionStateKindKey, "openai", "gpt-4o") + base := SessionStateKey(attributed(sessionCtx("session-1"), "vk-1", ""), SessionStateKindKey, "openai", "gpt-4o") if !strings.HasPrefix(base, "session:v2:key:") { t.Fatalf("key %q lacks the kind prefix", base) } if strings.Contains(base, "session-1") || strings.Contains(base, "vk-1") || strings.Contains(base, "gpt-4o") { t.Fatalf("key %q leaks a hashed part", base) } - if base != sessionStateKey(attributed(sessionCtx("session-1"), "vk-1", ""), SessionStateKindKey, "openai", "gpt-4o") { + if base != SessionStateKey(attributed(sessionCtx("session-1"), "vk-1", ""), SessionStateKindKey, "openai", "gpt-4o") { t.Fatal("key is not deterministic") } variants := map[string]string{ - "other virtual key": sessionStateKey(attributed(sessionCtx("session-1"), "vk-2", ""), SessionStateKindKey, "openai", "gpt-4o"), - "same id as a user": sessionStateKey(attributed(sessionCtx("session-1"), "", "vk-1"), SessionStateKindKey, "openai", "gpt-4o"), - "key and user": sessionStateKey(attributed(sessionCtx("session-1"), "vk-1", "u-1"), SessionStateKindKey, "openai", "gpt-4o"), - "other session": sessionStateKey(attributed(sessionCtx("session-2"), "vk-1", ""), SessionStateKindKey, "openai", "gpt-4o"), - "other provider": sessionStateKey(attributed(sessionCtx("session-1"), "vk-1", ""), SessionStateKindKey, "azure", "gpt-4o"), - "other model": sessionStateKey(attributed(sessionCtx("session-1"), "vk-1", ""), SessionStateKindKey, "openai", "gpt-4o-mini"), - "route kind": sessionStateKey(attributed(sessionCtx("session-1"), "vk-1", ""), SessionStateKindRoute, "openai", "gpt-4o"), + "other virtual key": SessionStateKey(attributed(sessionCtx("session-1"), "vk-2", ""), SessionStateKindKey, "openai", "gpt-4o"), + "same id as a user": SessionStateKey(attributed(sessionCtx("session-1"), "", "vk-1"), SessionStateKindKey, "openai", "gpt-4o"), + "key and user": SessionStateKey(attributed(sessionCtx("session-1"), "vk-1", "u-1"), SessionStateKindKey, "openai", "gpt-4o"), + "other session": SessionStateKey(attributed(sessionCtx("session-2"), "vk-1", ""), SessionStateKindKey, "openai", "gpt-4o"), + "other provider": SessionStateKey(attributed(sessionCtx("session-1"), "vk-1", ""), SessionStateKindKey, "azure", "gpt-4o"), + "other model": SessionStateKey(attributed(sessionCtx("session-1"), "vk-1", ""), SessionStateKindKey, "openai", "gpt-4o-mini"), + "route kind": SessionStateKey(attributed(sessionCtx("session-1"), "vk-1", ""), SessionStateKindRoute, "openai", "gpt-4o"), } for name, v := range variants { if v == base { @@ -158,19 +158,19 @@ func TestSessionStateKeyScopesBySessionAndIdentity(t *testing.T) { // A request nothing governs, one whose grant has no identity, and one whose identity names // nothing all scope to the deployment and share a key. - deployment := sessionStateKey(sessionCtx("session-1"), SessionStateKindKey, "openai", "gpt-4o") + deployment := SessionStateKey(sessionCtx("session-1"), SessionStateKindKey, "openai", "gpt-4o") unsettled := sessionCtx("session-1") unsettled.SetGrant(&stubGrant{}) - if got := sessionStateKey(unsettled, SessionStateKindKey, "openai", "gpt-4o"); got != deployment { + if got := SessionStateKey(unsettled, SessionStateKindKey, "openai", "gpt-4o"); got != deployment { t.Fatal("a grant without identity should scope to the deployment") } - if got := sessionStateKey(attributed(sessionCtx("session-1"), "", ""), SessionStateKindKey, "openai", "gpt-4o"); got != deployment { + if got := SessionStateKey(attributed(sessionCtx("session-1"), "", ""), SessionStateKindKey, "openai", "gpt-4o"); got != deployment { t.Fatal("an identity naming no key and no user should scope to the deployment") } if deployment == base { t.Fatal("deployment scope collides with a virtual key scope") } - if sessionStateKey(nil, SessionStateKindKey, "openai", "gpt-4o") == "" { + if SessionStateKey(nil, SessionStateKindKey, "openai", "gpt-4o") == "" { t.Fatal("nil context should still produce a key") } } @@ -228,7 +228,7 @@ func TestSessionAffinityResolveKeyReadsReplicatedBindings(t *testing.T) { t.Run(tc.name, func(t *testing.T) { kv := newMockKVStore() ctx := sessionCtx("session-1") - _ = kv.SetWithTTL(sessionStateKey(ctx, SessionStateKindKey, "openai", "gpt-4o"), tc.value, time.Minute) + _ = kv.SetWithTTL(SessionStateKey(ctx, SessionStateKindKey, "openai", "gpt-4o"), tc.value, time.Minute) key, ok := testAffinity(kv).ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool) if !ok || key.ID != tc.want { t.Fatalf("got %q ok=%v, want the stored binding %q", key.ID, ok, tc.want) @@ -239,7 +239,7 @@ func TestSessionAffinityResolveKeyReadsReplicatedBindings(t *testing.T) { // An empty stored value is no binding. kv := newMockKVStore() ctx := sessionCtx("session-1") - _ = kv.SetWithTTL(sessionStateKey(ctx, SessionStateKindKey, "openai", "gpt-4o"), "", time.Minute) + _ = kv.SetWithTTL(SessionStateKey(ctx, SessionStateKindKey, "openai", "gpt-4o"), "", time.Minute) if _, ok := testAffinity(kv).ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool); ok { t.Fatal("empty binding produced a key") } @@ -263,8 +263,8 @@ func TestSessionAffinityBindsOnOutcomeThenReuses(t *testing.T) { // Served by key-b: the route and the key are bound with the request's TTL. a.Observe(ctx, requested, servedBy(served, "key-b", false)) - routeKey := sessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o") - keyKey := sessionStateKey(ctx, SessionStateKindKey, "openai", "gpt-4o") + routeKey := SessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o") + keyKey := SessionStateKey(ctx, SessionStateKindKey, "openai", "gpt-4o") if entry := kv.data[routeKey]; entry.value != "openai/gpt-4o" || entry.ttl != 5*time.Minute { t.Fatalf("route binding after first request: %+v", entry) } @@ -292,7 +292,7 @@ func TestSessionAffinityBindsOnOutcomeThenReuses(t *testing.T) { // A fallback provider that served binds its own key, and does not take the route from a // request that followed no route binding. a.Observe(next, requested, servedBy(routeOf(schemas.Azure, "gpt-4o"), "az-1", true)) - if entry := kv.data[sessionStateKey(next, SessionStateKindKey, "azure", "gpt-4o")]; entry.value != "az-1" { + if entry := kv.data[SessionStateKey(next, SessionStateKindKey, "azure", "gpt-4o")]; entry.value != "az-1" { t.Fatalf("fallback key binding: %+v", entry) } if entry := kv.data[routeKey]; entry.value != "openai/gpt-4o" { @@ -310,9 +310,9 @@ func TestSessionAffinityFallbackReplacesAnOlderKeyBindingOnItsProvider(t *testin // Earlier turns: openai served with key-a, then the session moved to azure on key-b. seed := sessionCtx("session-1") - openaiKey := sessionStateKey(seed, SessionStateKindKey, "openai", "gpt-4o") - azureKey := sessionStateKey(seed, SessionStateKindKey, "azure", "gpt-4o") - routeKey := sessionStateKey(seed, SessionStateKindRoute, "", "gpt-4o") + openaiKey := SessionStateKey(seed, SessionStateKindKey, "openai", "gpt-4o") + azureKey := SessionStateKey(seed, SessionStateKindKey, "azure", "gpt-4o") + routeKey := SessionStateKey(seed, SessionStateKindRoute, "", "gpt-4o") _ = kv.SetWithTTL(openaiKey, "key-a", time.Hour) _ = kv.SetWithTTL(azureKey, "key-b", time.Hour) _ = kv.SetWithTTL(routeKey, "azure/gpt-4o", time.Hour) @@ -346,7 +346,7 @@ func TestSessionAffinityFallbackReplacesAnOlderKeyBindingOnItsProvider(t *testin func TestSessionAffinityRebindsWhenBoundKeyLeavesThePool(t *testing.T) { kv := newMockKVStore() ctx := sessionCtx("session-1") - stateKey := sessionStateKey(ctx, SessionStateKindKey, "openai", "gpt-4o") + stateKey := SessionStateKey(ctx, SessionStateKindKey, "openai", "gpt-4o") _ = kv.SetWithTTL(stateKey, "key-gone", time.Minute) if _, ok := testAffinity(kv).ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool); ok { @@ -364,7 +364,7 @@ func TestSessionAffinityResolveRoute(t *testing.T) { chain := []schemas.Route{routeOf(schemas.Groq, "openai/gpt-4o"), routeOf(schemas.OpenAI, "gpt-4o"), routeOf(schemas.Azure, "gpt-4o")} requested := routeOf("", "gpt-4o") bind := func(kv *mockKVStore, ctx *schemas.BifrostContext, value string) { - _ = kv.SetWithTTL(sessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o"), value, time.Minute) + _ = kv.SetWithTTL(SessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o"), value, time.Minute) } t.Run("no binding leaves the chain", func(t *testing.T) { @@ -434,7 +434,7 @@ func TestSessionAffinityResolveRoute(t *testing.T) { if _, recorded := ctx.Value(sessionAffinityResolvedKey).(sessionResolution); recorded { t.Fatal("a binding outside the chain was recorded as followed") } - routeKey := sessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o") + routeKey := SessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o") if _, present := kv.data[routeKey]; present { t.Fatal("a binding outside the chain was kept") } @@ -445,6 +445,36 @@ func TestSessionAffinityResolveRoute(t *testing.T) { } }) + t.Run("a binding that cannot be read is dropped so the session can rebind", func(t *testing.T) { + for _, unreadable := range []struct{ name, value string }{ + {"no separator between provider and model", "azure"}, + {"no provider", "/gpt-4o"}, + } { + t.Run(unreadable.name, func(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + ctx := sessionCtx("session-1") + bind(kv, ctx, unreadable.value) + if got := a.ResolveRoute(ctx, requested, chain); !slices.Equal(got, chain) { + t.Fatalf("got %v, want the chain unchanged", got) + } + if _, recorded := ctx.Value(sessionAffinityResolvedKey).(sessionResolution); recorded { + t.Fatal("a binding that names no route was recorded as followed") + } + routeKey := SessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o") + if _, present := kv.data[routeKey]; present { + t.Fatal("a binding that names no route was kept") + } + // Kept, it would refuse the first-writer bind below and strand the session on + // routing's pick until the binding expired. + a.Observe(ctx, requested, servedBy(routeOf(schemas.OpenAI, "gpt-4o"), "oa-1", false)) + if entry := kv.data[routeKey]; entry.value != "openai/gpt-4o" { + t.Fatalf("session did not rebind to what served: %+v", entry) + } + }) + } + }) + t.Run("empty chain, nil policy and no store leave the chain", func(t *testing.T) { kv := newMockKVStore() bind(kv, sessionCtx("session-1"), "azure/gpt-4o") @@ -465,7 +495,7 @@ func TestSessionAffinityObserveRoute(t *testing.T) { chain := []schemas.Route{routeOf(schemas.OpenAI, "gpt-4o"), routeOf(schemas.Azure, "gpt-4o")} requested := routeOf("", "gpt-4o") routeKeyOf := func(ctx *schemas.BifrostContext) string { - return sessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o") + return SessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o") } t.Run("a followed route that served is refreshed", func(t *testing.T) { @@ -527,7 +557,7 @@ func TestSessionAffinityResolveKeyDeclines(t *testing.T) { kv := newMockKVStore() a := testAffinity(kv) bound := sessionCtx("session-1") - _ = kv.SetWithTTL(sessionStateKey(bound, SessionStateKindKey, "openai", "gpt-4o"), "key-a", time.Minute) + _ = kv.SetWithTTL(SessionStateKey(bound, SessionStateKindKey, "openai", "gpt-4o"), "key-a", time.Minute) if _, ok := a.ResolveKey(nil, schemas.OpenAI, "gpt-4o", sessionTestPool); ok { t.Fatal("nil context got a key") @@ -577,7 +607,7 @@ func TestSessionAffinityConcurrentFirstRequestsConverge(t *testing.T) { } wg.Wait() - bound, _ := kv.data[sessionStateKey(sessionCtx("session-1"), SessionStateKindKey, "openai", "gpt-4o")].value.(string) + bound, _ := kv.data[SessionStateKey(sessionCtx("session-1"), SessionStateKindKey, "openai", "gpt-4o")].value.(string) if bound == "" { t.Fatal("no key binding after eight served requests") } @@ -819,7 +849,7 @@ func TestSessionAffinityForgetsWhatAnEarlierRequestFollowed(t *testing.T) { requested := routeOf("", "gpt-4o") chain := []schemas.Route{routeOf(schemas.OpenAI, "gpt-4o"), routeOf(schemas.Azure, "gpt-4o")} ctx := sessionCtx("session-1") - routeKey := sessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o") + routeKey := SessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o") // An earlier request on this context followed a binding to azure. _ = kv.SetWithTTL(routeKey, "azure/gpt-4o", time.Minute) @@ -844,7 +874,7 @@ func TestSessionAffinityLeavesAnExplicitProviderAlone(t *testing.T) { requested := routeOf(schemas.OpenAI, "gpt-4o") chain := []schemas.Route{routeOf(schemas.OpenAI, "gpt-4o"), routeOf(schemas.Azure, "gpt-4o")} ctx := sessionCtx("session-1") - routeKey := sessionStateKey(ctx, SessionStateKindRoute, "openai", "gpt-4o") + routeKey := SessionStateKey(ctx, SessionStateKindRoute, "openai", "gpt-4o") // A fallback that served does not become the session's home when the caller named the // provider: no route binding is written, only the key the fallback used. @@ -852,7 +882,7 @@ func TestSessionAffinityLeavesAnExplicitProviderAlone(t *testing.T) { if _, present := kv.data[routeKey]; present { t.Fatal("a route binding was written for a request that named its provider") } - if entry := kv.data[sessionStateKey(ctx, SessionStateKindKey, "azure", "gpt-4o")]; entry.value != "az-1" { + if entry := kv.data[SessionStateKey(ctx, SessionStateKindKey, "azure", "gpt-4o")]; entry.value != "az-1" { t.Fatalf("the fallback's key binding was not written: %+v", entry) } diff --git a/plugins/routing/complexityrouting.go b/plugins/routing/complexityrouting.go index 21130abebc4..310531fc127 100644 --- a/plugins/routing/complexityrouting.go +++ b/plugins/routing/complexityrouting.go @@ -25,7 +25,6 @@ type complexityProposal struct { func (p *RoutingPlugin) computeComplexity( ctx *schemas.BifrostContext, req *schemas.BifrostRequest, - virtualKeyID string, ) *complexity.ComplexityResult { input, disposition := complexity.BuildInputWithDisposition(ctx, req) sessionID, _ := ctx.Value(schemas.BifrostContextKeySessionID).(string) @@ -33,7 +32,7 @@ func (p *RoutingPlugin) computeComplexity( if disposition == complexity.InputContinuation { if sessionActive { - key := buildComplexitySessionKey(ctx, virtualKeyID, sessionID) + key := complexitySessionKey(ctx) tier, found, err := p.sessionStore.load(key, true) if err != nil { p.logComplexitySessionStoreError("refresh continuation", err) @@ -74,7 +73,7 @@ func (p *RoutingPlugin) computeComplexity( return proposal.Result } - key := buildComplexitySessionKey(ctx, virtualKeyID, sessionID) + key := complexitySessionKey(ctx) priorTier, priorFound, loadErr := p.sessionStore.load(key, false) if loadErr != nil { p.logComplexitySessionStoreError("inspect", loadErr) diff --git a/plugins/routing/complexitysession.go b/plugins/routing/complexitysession.go index 10f6b4ae7a0..6b3ba6db074 100644 --- a/plugins/routing/complexitysession.go +++ b/plugins/routing/complexitysession.go @@ -1,8 +1,6 @@ package routing import ( - "crypto/sha256" - "encoding/hex" "encoding/json" "errors" "fmt" @@ -15,7 +13,8 @@ import ( ) const ( - complexitySessionKeyPrefix = "complexity-session:v1:" + // complexitySessionKind is the session state kind under which the effective tier is kept. + complexitySessionKind = "complexity" complexitySessionInactivityTTL = 24 * time.Hour ) @@ -143,29 +142,9 @@ func decodeStoredComplexityTier(value any) (string, error) { return tier, nil } -// buildComplexitySessionKey isolates equal caller session IDs across virtual -// keys and authenticated users, then hashes the complete tuple so the in-memory -// key has bounded size and reveals no caller-provided identifier. -func buildComplexitySessionKey( - ctx *schemas.BifrostContext, - virtualKeyID string, - sessionID string, -) string { - userID := bifrost.GetStringFromContext(ctx, schemas.BifrostContextKeyUserID) - scopeKind := "deployment" - switch { - case virtualKeyID != "" && userID != "": - scopeKind = "virtual-key-user" - case virtualKeyID != "": - scopeKind = "virtual-key" - case userID != "": - scopeKind = "user" - } - - hash := sha256.New() - for _, part := range []string{scopeKind, virtualKeyID, userID, sessionID} { - _, _ = hash.Write([]byte(part)) - _, _ = hash.Write([]byte{0}) - } - return complexitySessionKeyPrefix + hex.EncodeToString(hash.Sum(nil)) +// complexitySessionKey is where the request's session keeps its effective tier: the shared +// session state key, scoped to the virtual key and user the request is attributed to, under +// the complexity kind. +func complexitySessionKey(ctx *schemas.BifrostContext) string { + return bifrost.SessionStateKey(ctx, complexitySessionKind) } diff --git a/plugins/routing/complexitysession_test.go b/plugins/routing/complexitysession_test.go index 3950efdecd6..754e43fe9e6 100644 --- a/plugins/routing/complexitysession_test.go +++ b/plugins/routing/complexitysession_test.go @@ -10,6 +10,7 @@ import ( "github.com/maximhq/bifrost/core/schemas" "github.com/maximhq/bifrost/framework/configstore" + "github.com/maximhq/bifrost/framework/grant" "github.com/maximhq/bifrost/framework/kvstore" "github.com/maximhq/bifrost/plugins/routing/complexity" "github.com/maximhq/bifrost/plugins/routing/rules" @@ -62,7 +63,7 @@ func (s *recordingSessionKVStore) SetWithTTL(key string, value any, ttl time.Dur func TestComplexitySessionStoreMonotonicLadder(t *testing.T) { sessions, _ := newTestComplexitySessionStore(t, time.Minute) - key := complexitySessionKeyPrefix + "ladder" + key := "complexity-test:ladder" resolution, err := sessions.resolve(key, complexity.TierSimple) require.NoError(t, err) @@ -91,7 +92,7 @@ func TestComplexitySessionStoreMonotonicLadder(t *testing.T) { func TestComplexitySessionStoreEmptyProposalOnlyRefreshesExisting(t *testing.T) { sessions, store := newTestComplexitySessionStore(t, time.Minute) - key := complexitySessionKeyPrefix + "continuation" + key := "complexity-test:continuation" resolution, err := sessions.resolve(key, "") require.NoError(t, err) @@ -114,7 +115,7 @@ func TestComplexitySessionStoreRefreshesTTLWhenTierDoesNotChange(t *testing.T) { const sessionTTL = 45 * time.Minute recordingStore := &recordingSessionKVStore{KVStore: store} sessions := newComplexitySessionStore(recordingStore, sessionTTL) - key := complexitySessionKeyPrefix + "refresh" + key := "complexity-test:refresh" _, err = sessions.resolve(key, complexity.TierMedium) require.NoError(t, err) @@ -131,7 +132,7 @@ func TestComplexitySessionStoreRefreshesTTLWhenTierDoesNotChange(t *testing.T) { func TestComplexitySessionStoreExpiryStartsNewEpoch(t *testing.T) { sessions, _ := newTestComplexitySessionStore(t, 10*time.Millisecond) - key := complexitySessionKeyPrefix + "expiry" + key := "complexity-test:expiry" _, err := sessions.resolve(key, complexity.TierComplex) require.NoError(t, err) @@ -150,7 +151,7 @@ func TestComplexitySessionStoreExpiryStartsNewEpoch(t *testing.T) { func TestComplexitySessionStoreRejectsCorruptTier(t *testing.T) { sessions, store := newTestComplexitySessionStore(t, time.Minute) - key := complexitySessionKeyPrefix + "corrupt" + key := "complexity-test:corrupt" require.NoError(t, store.SetWithTTL(key, "UNKNOWN", time.Minute)) _, _, err := sessions.load(key, false) @@ -161,7 +162,7 @@ func TestComplexitySessionStoreRejectsCorruptTier(t *testing.T) { func TestComplexitySessionStoreDecodesReplicatedString(t *testing.T) { sessions, store := newTestComplexitySessionStore(t, time.Minute) - key := complexitySessionKeyPrefix + "remote" + key := "complexity-test:remote" now := time.Now() require.NoError(t, store.SetRemote(key, []byte(`"MEDIUM"`), now.UnixNano(), now.Add(time.Minute).UnixNano())) @@ -171,22 +172,40 @@ func TestComplexitySessionStoreDecodesReplicatedString(t *testing.T) { require.Equal(t, complexity.TierMedium, tier) } -func TestBuildComplexitySessionKeyScopesAndHidesIdentity(t *testing.T) { - ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) - ctx.SetValue(schemas.BifrostContextKeyUserID, "user-1") +func TestComplexitySessionKeyScopesByIdentityAndHidesIt(t *testing.T) { sessionID := "caller-session-secret" + attributed := func(sessionID, virtualKeyID, userID string) *schemas.BifrostContext { + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + ctx.SetValue(schemas.BifrostContextKeySessionID, sessionID) + var user *schemas.UserRef + if userID != "" { + user = &schemas.UserRef{ID: userID} + } + var virtualKey *schemas.EntityRef + if virtualKeyID != "" { + virtualKey = &schemas.EntityRef{ID: virtualKeyID} + } + g := grant.New() + g.SetIdentity(grant.NewIdentity(schemas.Credential{}, user, virtualKey, nil, nil, nil, nil)) + ctx.SetGrant(g) + return ctx + } - base := buildComplexitySessionKey(ctx, "vk-1", sessionID) - require.True(t, strings.HasPrefix(base, complexitySessionKeyPrefix)) + base := complexitySessionKey(attributed(sessionID, "vk-1", "user-1")) + require.True(t, strings.HasPrefix(base, "session:v2:complexity:")) require.NotContains(t, base, sessionID) require.NotContains(t, base, "user-1") require.NotContains(t, base, "vk-1") - require.Equal(t, base, buildComplexitySessionKey(ctx, "vk-1", sessionID)) + require.Equal(t, base, complexitySessionKey(attributed(sessionID, "vk-1", "user-1"))) + + require.NotEqual(t, base, complexitySessionKey(attributed(sessionID, "vk-2", "user-1"))) + require.NotEqual(t, base, complexitySessionKey(attributed(sessionID, "vk-1", "user-2"))) + require.NotEqual(t, base, complexitySessionKey(attributed("another-session", "vk-1", "user-1"))) - require.NotEqual(t, base, buildComplexitySessionKey(ctx, "vk-2", sessionID)) - ctx.SetValue(schemas.BifrostContextKeyUserID, "user-2") - require.NotEqual(t, base, buildComplexitySessionKey(ctx, "vk-1", sessionID)) - require.NotEqual(t, base, buildComplexitySessionKey(ctx, "vk-1", "another-session")) + // A request nothing governs keys under the deployment, apart from any caller's scope. + ungoverned := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + ungoverned.SetValue(schemas.BifrostContextKeySessionID, sessionID) + require.NotEqual(t, base, complexitySessionKey(ungoverned)) } func TestPublishLocalSessionFallbackPreservesProposalEvidence(t *testing.T) { diff --git a/plugins/routing/main.go b/plugins/routing/main.go index 50489f069ba..c4e9c1be8e2 100644 --- a/plugins/routing/main.go +++ b/plugins/routing/main.go @@ -482,7 +482,7 @@ func (p *RoutingPlugin) applyRoutingRules(ctx *schemas.BifrostContext, req *sche var computeComplexity func() *complexity.ComplexityResult if p.complexityAnalyzer.Load() != nil { computeComplexity = func() *complexity.ComplexityResult { - return p.computeComplexity(ctx, req, scope.VirtualKeyID) + return p.computeComplexity(ctx, req) } } From 87b5d20440dbed87b691c58eca61fe64a98b3912 Mon Sep 17 00:00:00 2001 From: Pratham-Mishra04 Date: Tue, 15 Sep 2026 17:19:23 +0530 Subject: [PATCH 005/118] docs: session affinity guide with routing, agent, and load balancing cross-links --- docs/cli-agents/claude-code.mdx | 4 + docs/cli-agents/codex-cli.mdx | 4 + docs/cli-agents/opencode.mdx | 4 + docs/docs.json | 1 + docs/enterprise/adaptive-load-balancing.mdx | 2 +- .../features/governance/complexity-router.mdx | 2 +- docs/providers/provider-routing.mdx | 9 + docs/providers/request-options.mdx | 2 + docs/providers/session-affinity.mdx | 196 ++++++++++++++++++ 9 files changed, 222 insertions(+), 2 deletions(-) create mode 100644 docs/providers/session-affinity.mdx diff --git a/docs/cli-agents/claude-code.mdx b/docs/cli-agents/claude-code.mdx index d13e243a44e..921245caae8 100644 --- a/docs/cli-agents/claude-code.mdx +++ b/docs/cli-agents/claude-code.mdx @@ -61,6 +61,10 @@ Set `ANTHROPIC_CUSTOM_HEADERS` to `x-bf-vk: your-virtual-key`. This passes the v ## Configuring Claude Code to work with Bifrost + +Claude Code sends `x-claude-code-session-id` on every request, so a session sticks to the provider and key that served it and keeps hitting the same prompt cache, with nothing to configure. See [Session Affinity](/providers/session-affinity). + + **To avoid caching issues in Claude Code**, follow these steps: diff --git a/docs/cli-agents/codex-cli.mdx b/docs/cli-agents/codex-cli.mdx index 75e90e1ed4b..3800f047e63 100644 --- a/docs/cli-agents/codex-cli.mdx +++ b/docs/cli-agents/codex-cli.mdx @@ -18,6 +18,10 @@ npm install -g @openai/codex ## Configuring Codex CLI with Bifrost + +Codex CLI sends a `session-id` header on every request, so a session sticks to the provider and key that served it and keeps hitting the same prompt cache, with nothing to configure. See [Session Affinity](/providers/session-affinity). + + Codex CLI always prefers OAuth over custom API keys. Make sure you run `/logout` before configuring the Bifrost gateway with Codex. diff --git a/docs/cli-agents/opencode.mdx b/docs/cli-agents/opencode.mdx index b6929853409..f13eb6a4a9d 100644 --- a/docs/cli-agents/opencode.mdx +++ b/docs/cli-agents/opencode.mdx @@ -14,6 +14,10 @@ If your Allowed Headers are already set to `*`, you can skip this note. If not a ## Setup + +OpenCode sends `x-session-affinity` and `x-session-id` on every request, so a session sticks to the provider and key that served it and keeps hitting the same prompt cache, with nothing to configure. See [Session Affinity](/providers/session-affinity). + + ### 1. Configure OpenCode to work with Bifrost OpenCode uses a JSON config file (`opencode.json`) to configure providers. Point your provider's `baseURL` to Bifrost. diff --git a/docs/docs.json b/docs/docs.json index 07617d698a9..d915392d2fb 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -173,6 +173,7 @@ }, "providers/routing-rules", "providers/provider-routing", + "providers/session-affinity", "providers/aliasing-models", "providers/reasoning", "providers/performance", diff --git a/docs/enterprise/adaptive-load-balancing.mdx b/docs/enterprise/adaptive-load-balancing.mdx index 7b2f928e48f..5cad1e3fede 100644 --- a/docs/enterprise/adaptive-load-balancing.mdx +++ b/docs/enterprise/adaptive-load-balancing.mdx @@ -230,7 +230,7 @@ Unlike the API, this block is presence-aware: omitted fields keep their current - **Per-node weights**: Each node load-balances on its own observed metrics. The only signal shared across nodes is a rate-limit (TPM) backoff, and only within the same region - there is no global weight consensus or cross-region coordination. This is deliberate: latency and error profiles differ per region, so importing another region's metrics would pollute a node's view of route health. - **~5-second adaptation**: Weight and state changes lag live traffic by up to one recompute cycle. Immediate per-request resilience (key rotation and fallback failover) is handled separately and is not subject to this delay. - **Optimistic cold start**: A brand-new key or provider enters at full weight and competes at roughly fair share before it has been measured, then self-corrects within a cycle or two. -- **Relative, not absolute**: Routes are ranked against their peers, not against a fixed latency or cost target. The system is not cost-, org-, or session-aware, and does not accept manual per-key weights for the adaptive path - those concerns are handled by [governance routing](/providers/provider-routing). +- **Relative, not absolute**: Routes are ranked against their peers, not against a fixed latency or cost target. The system is not cost- or org-aware, and does not accept manual per-key weights for the adaptive path - those concerns are handled by [governance routing](/providers/provider-routing). It is not session-aware either: [Session Affinity](/providers/session-affinity) runs after it and keeps a session on its provider and key only while the load balancer still offers them and has not marked them as failed, and the circuit breaker is not holding that key back. --- diff --git a/docs/features/governance/complexity-router.mdx b/docs/features/governance/complexity-router.mdx index 836c48a09e0..20045a1e3a7 100644 --- a/docs/features/governance/complexity-router.mdx +++ b/docs/features/governance/complexity-router.mdx @@ -70,7 +70,7 @@ Complexity Router does not currently classify Codex requests sent through native -Session-aware routing keeps the **complexity tier** stable; it does not pin a weighted routing target, provider key, or provider prompt-cache entry. Provider cache TTLs remain provider-owned and independent of the 24-hour routing-state lifetime. +Session-aware routing keeps the **complexity tier** stable; it does not pin a weighted routing target, provider key, or provider prompt-cache entry. Provider cache TTLs remain provider-owned and independent of the 24-hour routing-state lifetime. Keeping a session on one provider and key is the job of [Session Affinity](/providers/session-affinity), which uses the same session identity and runs alongside the router. --- diff --git a/docs/providers/provider-routing.mdx b/docs/providers/provider-routing.mdx index c7ff56eef12..543355a38e4 100644 --- a/docs/providers/provider-routing.mdx +++ b/docs/providers/provider-routing.mdx @@ -1288,6 +1288,15 @@ All three routing layers (governance, enterprise LB Level 1, model-catalog-resol *layer* where Level 1 runs has moved from a middleware to PreRequestHook. + + **Session affinity** sits on top of this order. After PreRequestHook it moves the + provider that last served the session to the front of the fallback chain. A request + that names its provider (`openai/gpt-4o`) is never reordered. Inside Level 2 it reuses + the key that last served the session while that key stays in the pool, including for a + request that named its provider. Routing decides which providers and keys are offered; + the session only decides their order. See [Session Affinity](/providers/session-affinity). + + ### Example Scenarios diff --git a/docs/providers/request-options.mdx b/docs/providers/request-options.mdx index 8918a420f69..6d0e0be496b 100644 --- a/docs/providers/request-options.mdx +++ b/docs/providers/request-options.mdx @@ -243,6 +243,8 @@ Bindings are written when a request is served, never when it fails, and are scop - **Switch**: `x-bf-session-affinity: off` turns affinity off for a request, at both levels; `on` is the default. - **Routing trail**: Decisions the session made appear in the request's routing logs under the `session-affinity` engine, which is also listed among the routing engines used. +The [Session Affinity](/providers/session-affinity) guide covers the full lifecycle of a binding, how affinity composes with each routing engine, and the health checks Bifrost Enterprise runs before following one. + **Coding Harness Headers:** Coding-harness session header fallback is available in **Bifrost v2.0.0 and above**. diff --git a/docs/providers/session-affinity.mdx b/docs/providers/session-affinity.mdx new file mode 100644 index 00000000000..03c22fe5c81 --- /dev/null +++ b/docs/providers/session-affinity.mdx @@ -0,0 +1,196 @@ +--- +title: "Session Affinity" +description: "Keep a conversation on the provider and key that served it before, so multi-turn sessions and agent runs keep hitting the same provider prompt cache and rate-limit bucket, whatever routing engine picked the provider." +icon: "link" +--- + +Session affinity, also called session stickiness, keeps a session on what served it before. A session is a conversation, an agent run, or any group of requests that share a session ID. Routing still decides which providers and keys a request may use; the session only decides which of them is tried first. + +It runs for every request that carries a session ID and works with every routing engine: routing rules, virtual key load balancing, the model catalog, and Adaptive Load Balancing. Coding harnesses such as Claude Code, Codex CLI, and OpenCode get it with no configuration, because Bifrost adopts the session header they already send. + +## Why it matters + +- **Provider prompt caches** are per provider, and usually per API key or organization. A conversation that hops between OpenAI and Azure, or between two OpenAI keys, misses the cache it warmed a turn ago. +- **Rate-limit buckets** are per key. A session that stays on one key uses one bucket predictably instead of spreading across several. +- **Consistency** across a run: the same provider serves every turn, so tool behaviour and response style do not drift mid-conversation. + +Affinity does not create or manage provider caches. It makes sure a session keeps landing where its cache lives. + +## What counts as a session + +| Source | How the session ID is set | +| ----------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Explicit header | `x-bf-session-id` on the request. Always wins when present. | +| Coding harness headers | When `x-bf-session-id` is absent, the first non-empty harness session header is adopted. See the [header list](/providers/request-options#session-stickiness-session-id) for the full priority order. | +| Go SDK | `BifrostContextKeySessionID` on the request context. | + +A session ID may be at most 255 runes. An explicit `x-bf-session-id` that is longer gives the request no session at all rather than falling back to a harness header, because grouping requests under a session the caller did not ask for is worse than leaving them ungrouped. + +Bindings are scoped to who the request is attributed to: the virtual key and the user behind it, as resolved by governance, plus the session ID. The same session ID under two different virtual keys is two independent sessions. A request with no virtual key and no user is scoped by its session ID alone. + + +Claude Code subagents send the same session ID as their parent, so an agent run with parallel subagents stays on one provider and one key. + + +## Two levels of affinity + +### Provider level + +When the caller asks for a bare model (`gpt-4o`) and routing offers more than one provider for it, the provider that last served the session is tried first, as long as routing still offers it. Routing engines build the chain of providers; affinity reorders that chain, and nothing else. A provider that routing excluded, for budget, rate limits, model allowance, or health, is never brought back by a session. + +The binding is keyed by what the caller asked for, so a session that asks for `gpt-4o` and later for `gpt-4o-mini` holds one binding per model. + + +A request that names its provider (`openai/gpt-4o`) is never reordered and writes no provider binding. The caller asked for that provider, so only the key level applies, even when a fallback served an earlier turn of the same session. + + +### Key level + +Within the provider that serves the request, the key that last served the session for that provider and model is used again while it stays in the eligible pool. The pool is what key selection would otherwise choose from: the keys enabled for the provider that allow the model. A provider with a single key has nothing to choose, so the key level is skipped. + +A key that leaves the pool, because it was disabled, removed, or stopped allowing the model, loses its binding and a key is picked normally. + +## Lifecycle of a binding + +Bindings are written only when a request is served. A failed request writes nothing, so a session is never bound to a provider or key that did not deliver. + +| Event | Provider binding | Key binding | +| ------------------------------------------------------ | ---------------------------------------------------------------------- | -------------------------------------------------------------------- | +| First served request of the session | Written to the provider that served | Written to the key that served | +| Served request that followed the binding | TTL refreshed | TTL refreshed | +| Served by a fallback provider | Moved to the fallback provider | Moved to the key the fallback used | +| Request failed on every attempt | Untouched | Untouched | +| Bound provider not in the chain routing built | Deleted, the routing decision stands, the next served request rebinds | Unchanged | +| Bound key not in the eligible pool | Unchanged | Deleted, a key is picked normally, the next served request rebinds | +| TTL expires with no served request | Gone | Gone | +| Request sent with `x-bf-session-affinity: off` | Neither read nor written | Neither read nor written | + +The TTL defaults to one hour and can be set per request with `x-bf-session-ttl` (a duration string such as `30m`, or a number of seconds). Every served request that follows a binding refreshes it, so an active session does not expire mid-conversation. + +When several requests of a brand-new session arrive in parallel, they can land on different providers or keys before any of them is served. The first one to be served writes the binding, the rest of that burst is already in flight, and every request after it follows the winner. + +## How it fits with routing + +Affinity runs after every routing engine has had its say, and again inside each provider attempt when a key is chosen: + +1. Governance evaluates routing rules and, for virtual keys with provider configs, load balances across the weighted providers. +2. Adaptive Load Balancing (Enterprise) ranks the eligible providers by measured performance. +3. The model catalog resolver fills in a provider for a bare model that nothing above resolved, with the other catalog providers as fallbacks. +4. **Session affinity, provider level**: the bound provider moves to the front of the chain if it is in it. +5. For each attempt, the key pool is built for the provider and filtered, then **session affinity, key level** reuses the bound key if it is in the pool. +6. Fallbacks run in the resulting order when an attempt fails. + +| Routing engine | What it decides | What the session changes | +| ------------------------------------- | -------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Routing rules | A target and optional fallbacks per matching request | If the rule lists fallbacks, the session's provider is moved first among them. A rule that picks one weighted target and lists no fallbacks leaves a one-provider chain, so there is nothing to reorder and the weights are rolled on every request. | +| Virtual key load balancing | A weighted random provider, with the other weighted providers as fallbacks | After the first served request the session skips the dice and goes to its provider, for as long as budgets, rate limits, and model allowances keep that provider in the chain. | +| Adaptive Load Balancing (Enterprise) | The best-scoring healthy provider, with the healthy rest as fallbacks | The session's provider comes first while the load balancer still offers it. A provider the load balancer marks as failed is dropped, see [Health-aware invalidation](#health-aware-invalidation-enterprise). | +| Model catalog | Any provider whose catalog lists the model, when nothing else picked one | The session's provider comes first among the catalog candidates. | +| Circuit breaker (Enterprise) | Which keys are held back from the pool | A bound key that the circuit breaker removes from the pool loses its binding; a key is picked normally. | +| Complexity Router | The complexity tier a routing rule can match on | Independent. The router keeps the tier stable per session; affinity keeps the provider and key stable. Both use the same session identity and neither reads the other's state. | +| Fallbacks and retries | Which provider to try next, and how often to retry | Fallback attempts pick their keys freely. Retries of one attempt reuse the bound key. Whatever finally serves becomes the session's new binding at both levels. | + +## Health-aware invalidation (Enterprise) + + + **Enterprise Feature**: Health-aware invalidation is part of Bifrost + Enterprise. [Contact us](https://www.getmaxim.ai/bifrost/enterprise) to enable + it. + + +Without health signals, a session follows its binding until the request actually fails. Bifrost Enterprise checks the binding against what this node's Adaptive Load Balancer and circuit breaker know before following it: + +- **Provider level**: if the load balancer marks the bound provider as failed for the requested model on this node (every key in that direction has failed), the binding is dropped and the routing decision stands. The next served request rebinds the session. +- **Key level**: if the load balancer marks the bound key's route as failed on this node, or the circuit breaker holds the key back, the binding is dropped and a key is picked normally. The next served request rebinds the session to the key that served it. + +Bindings live in the shared KV store and replicate across the cluster, while health is measured per node. A node that sees a provider failing drops the binding for every node. A node that has not observed the failure yet still follows the binding until its own load balancer marks the route as failed or the request fails there. + +## Controls + +| Control | Header | Go SDK context key | Values | +| ------------------------ | ------------------------- | ----------------------------------- | ---------------------------------------------------------------------------------- | +| Session ID | `x-bf-session-id` | `BifrostContextKeySessionID` | Any string up to 255 runes. Falls back to coding-harness headers in the gateway. | +| Binding TTL | `x-bf-session-ttl` | `BifrostContextKeySessionTTL` | Duration string (`30m`, `2h`) or seconds. Default one hour. | +| Affinity switch | `x-bf-session-affinity` | `BifrostContextKeySessionAffinity` | `on`, `true`, `1` (default) or `off`, `false`, `0`. Applies to both levels. | + +A request sent with the switch off is routed as if it had no session: it neither follows nor updates any binding. Any other value is ignored with a warning and the default applies. There is no stored default; the switch is per request. + + +Bindings live in Bifrost's KV store. The gateway always creates one, so nothing needs to be configured there. Go SDK users must set `KVStore` on `BifrostConfig`, otherwise affinity has nowhere to keep a binding and every request is routed as if it had no session. In an Enterprise cluster the KV store replicates, so a session can land on any node and still find its binding. + + +```bash Gateway +curl -X POST http://localhost:8080/v1/chat/completions \ + --header 'Content-Type: application/json' \ + --header 'x-bf-session-id: user-123-session-abc' \ + --header 'x-bf-session-ttl: 30m' \ + --data '{"model": "gpt-4o", "messages": [{"role": "user", "content": "Hello"}]}' +``` + +```go Go SDK +ctx = context.WithValue(ctx, schemas.BifrostContextKeySessionID, "user-123-session-abc") +ctx = context.WithValue(ctx, schemas.BifrostContextKeySessionTTL, 30*time.Minute) +``` + +## Seeing what the session did + +Every decision affinity makes is written to the request's routing trail under the `session-affinity` engine, which also appears among the routing engines used for the request. In the logs explorer, filter by session ID to see one session's requests together, and by routing engine to find requests where affinity changed the outcome. + +| Trail entry | Meaning | +| ------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------- | +| `Session stays on azure for gpt-4o; routing proposed openai` | The bound provider was moved ahead of the one routing put first. | +| `Session reused key prod-eu-2 for azure/gpt-4o` | The bound key was used instead of running key selection. | +| `Session was last served by azure for gpt-4o, which this request cannot use, so the routing decision stands and the session rebinds on its next served request` | Routing no longer offers the bound provider, so the binding was deleted. | +| `The key this session last used for azure/gpt-4o is no longer eligible, so one is being picked` | The bound key left the pool, so the binding was deleted. | +| `Session was last served by azure for gpt-4o, which is failing on this node, so the routing decision stands` | Enterprise: the load balancer marks the provider as failed here, so the binding was dropped. | +| `The key this session last used for azure/gpt-4o is held back by the circuit breaker on this node, so one is being picked` | Enterprise: the circuit breaker removed the bound key from the pool, so the binding was dropped. | + +When routing already proposes the bound provider, the provider level writes nothing, because it changed nothing. The key level records every reused key. + +In OpenTelemetry, the trace's root span carries the session ID as the `session.id` attribute, and the `group_traces_by_session` setting puts a whole session into one trace. See [OpenTelemetry](/features/observability/otel#grouping-traces-by-session). + +## Walk-throughs + +### A virtual key that load balances + +A virtual key allows `gpt-4o` on OpenAI with weight 70 and Azure with weight 30. Azure has two keys. A Claude Code session sends its session header on every request. + +1. **Turn 1**: governance rolls the weights and picks Azure, with OpenAI as the fallback. Key selection picks Azure key `prod-eu-2`. The request is served, and the session is bound to Azure for `gpt-4o` and to `prod-eu-2` on Azure. +2. **Turn 2**: governance rolls OpenAI this time and builds the chain `[openai, azure]`. Affinity moves Azure to the front and the trail says `Session stays on azure for gpt-4o; routing proposed openai`. Key selection reuses `prod-eu-2`. The Azure prompt cache warmed by turn 1 is hit. +3. **Turn 3**: the virtual key's Azure budget is exhausted, so governance excludes Azure and offers only OpenAI. The bound provider is not in the chain, so the binding is deleted with `…which this request cannot use…` in the trail. OpenAI serves and the session rebinds to OpenAI and to the OpenAI key that served. + +### A fallback that serves + +The same session is bound to Azure and `prod-eu-2`. + +1. Azure returns a server error on every retry. Retries reuse `prod-eu-2`, because the session is bound to it. +2. The fallback attempt on OpenAI selects a key freely, say `openai-main`, and serves. +3. The session moves: the provider binding now points at OpenAI, and a key binding for OpenAI and `gpt-4o` points at `openai-main`. The stale Azure key binding is left to expire; it is only consulted if Azure serves this session again. +4. The next turn's chain, whatever governance rolls, is reordered to put OpenAI first. + +### A provider that fails on one node (Enterprise) + +A three-node cluster. The session is bound to Azure and `prod-eu-2`. + +1. Azure starts timing out. Node B's Adaptive Load Balancer marks the Azure direction for `gpt-4o` as failed after every key in it has failed. +2. The next request lands on node B. Before following the binding, affinity asks the load balancer and learns the direction is failed. The binding is dropped, replicated to nodes A and C, and the trail says `…which is failing on this node, so the routing decision stands`. The load balancer's own choice, OpenAI, serves, and the session rebinds to OpenAI. +3. Had the request landed on node A before A observed the failure, A would have followed the Azure binding. The request would then fail on Azure, the fallback would serve, and the session would move exactly as in the fallback walk-through. + +## Limits to know + +- **Explicit providers get no provider stickiness.** `openai/gpt-4o` is honoured as written. Use a bare model name and let routing offer several providers if you want the session to choose among them. +- **Weighted routing-rule targets with no fallbacks** are rolled on every request. Add fallbacks to the rule if the session should stick to the target that served it. +- **A bound key is used for every retry.** A session with a key binding does not rotate keys on a rate limit; the retry policy runs on the bound key, and only a fallback provider moves the session. A session with no key binding yet rotates like any other request. +- **A new session's first parallel burst scatters once.** Requests already in flight when the first one is served keep the provider and key they were given. +- **Health checks are node-local (Enterprise).** A binding is refused where a node has seen the failure. A node that has not seen it follows the binding until the request fails or its own load balancer catches up. +- **A node that joins the cluster starts with an empty binding store.** It receives bindings written after it joined. A session that lands on it before then is treated as new, rebinds there, and that binding replicates to the other nodes. +- **Realtime and WebSocket Responses key selection reads but never writes.** These connection paths reuse a key bound by earlier requests of the session, but they do not create or refresh a binding themselves. +- **The switch is per request.** There is no configuration that turns affinity off for a deployment; send `x-bf-session-affinity: off` on the requests that should not take part. + +## Next Steps + +- **[Request Options](/providers/request-options#session-stickiness-session-id)** - Every session header, the coding-harness header list, and the Go SDK context keys +- **[Provider Routing](/providers/provider-routing)** - How governance, Adaptive Load Balancing, and the model catalog build the chain affinity reorders +- **[Adaptive Load Balancing](/enterprise/adaptive-load-balancing)** - The health signals Enterprise affinity checks before following a binding +- **[Circuit Breaker](/enterprise/circuit-breaker)** - Key-level sub-circuits that take a bound key out of the pool +- **[Complexity Router](/features/governance/complexity-router#session-aware-routing)** - Session-stable complexity tiers that compose with affinity From 086726b07e6499fddf615e7c9845abee2465e868 Mon Sep 17 00:00:00 2001 From: Pratham-Mishra04 Date: Fri, 18 Sep 2026 01:54:56 +0530 Subject: [PATCH 006/118] fix: minor routing fixes --- core/sessionaffinity.go | 17 +++++++++++++++- core/sessionaffinity_test.go | 30 +++++++++++++++++++++++++---- docs/providers/request-options.mdx | 2 +- docs/providers/session-affinity.mdx | 15 +++++++++++---- plugins/governance/main.go | 19 +++++++++++++++--- 5 files changed, 70 insertions(+), 13 deletions(-) diff --git a/core/sessionaffinity.go b/core/sessionaffinity.go index 3e81703b5b1..ae027d16eb4 100644 --- a/core/sessionaffinity.go +++ b/core/sessionaffinity.go @@ -103,10 +103,20 @@ func (a *sessionAffinity) ResolveRoute(ctx *schemas.BifrostContext, requested sc a.logger.Warn("error dropping session route binding to %s: %s", boundRoute.Provider, err.Error()) } ctx.AppendRoutingEngineLog(schemas.RoutingEngineSessionAffinity, schemas.LogLevelInfo, fmt.Sprintf("Session was last served by %s for %s, which this request cannot use, so the routing decision stands and the session rebinds on its next served request", boundRoute.Provider, requested.Model)) + // Refusing a stale binding, and dropping it, is as much a decision as following one: the + // engine is listed so a trail entry is never attributed to an engine the request does not + // record. The paths that write no entry, an unreadable binding and no binding at all, stay + // silent at both levels. + schemas.AppendToContextList(ctx, schemas.BifrostContextKeyRoutingEnginesUsed, schemas.RoutingEngineSessionAffinity) return chain } a.remember(ctx, func(r *sessionResolution) { r.route = RouteStateValue(chain[at]) }) + schemas.AppendToContextList(ctx, schemas.BifrostContextKeyRoutingEnginesUsed, schemas.RoutingEngineSessionAffinity) if at == 0 { + // The session and routing agree, so the chain is left as it is. Say so anyway: a trail that + // falls silent here cannot be told apart from one where the session was never consulted, + // and the key level below reports every reuse whether or not it changed anything. + ctx.AppendRoutingEngineLog(schemas.RoutingEngineSessionAffinity, schemas.LogLevelInfo, fmt.Sprintf("Session stays on %s for %s, which routing also proposed", chain[0].Provider, requested.Model)) return chain } resolved := make([]schemas.Route, 0, len(chain)) @@ -114,7 +124,6 @@ func (a *sessionAffinity) ResolveRoute(ctx *schemas.BifrostContext, requested sc resolved = append(resolved, chain[:at]...) resolved = append(resolved, chain[at+1:]...) ctx.AppendRoutingEngineLog(schemas.RoutingEngineSessionAffinity, schemas.LogLevelInfo, fmt.Sprintf("Session stays on %s for %s; routing proposed %s", chain[at].Provider, requested.Model, chain[0].Provider)) - schemas.AppendToContextList(ctx, schemas.BifrostContextKeyRoutingEnginesUsed, schemas.RoutingEngineSessionAffinity) return resolved } @@ -239,6 +248,12 @@ func ParseRouteState(value string) (schemas.Route, bool) { // fallbacks. A request no hook could route is left for validation to refuse. func (bifrost *Bifrost) resolveSessionRoute(ctx *schemas.BifrostContext, requested schemas.Route, req *schemas.BifrostRequest) { if !schemas.IsSessionAffinityActive(ctx) { + // A request that carries a session but switched affinity off is routed as if it had none. + // Say so: the session id is on the log record either way, so a trail that stayed silent + // here reads as though affinity was consulted and had nothing to add. + if sessionIDFromContext(ctx) != "" { + ctx.AppendRoutingEngineLog(schemas.RoutingEngineSessionAffinity, schemas.LogLevelInfo, fmt.Sprintf("Request carries a session but asked not to follow it, so the routing decision stands for %s", requested.Model)) + } return } provider, model, fallbacks := req.GetRequestFields() diff --git a/core/sessionaffinity_test.go b/core/sessionaffinity_test.go index 28cf14772d7..663d64faccf 100644 --- a/core/sessionaffinity_test.go +++ b/core/sessionaffinity_test.go @@ -405,15 +405,17 @@ func TestSessionAffinityResolveRoute(t *testing.T) { } }) - t.Run("bound provider already first is left alone and not counted as a decision", func(t *testing.T) { + t.Run("bound provider already first is left alone but still reported", func(t *testing.T) { kv := newMockKVStore() ctx := sessionCtx("session-1") bind(kv, ctx, "groq/openai/gpt-4o") if got := testAffinity(kv).ResolveRoute(ctx, requested, chain); !slices.Equal(got, chain) { t.Fatalf("got %v, want the chain unchanged", got) } - if slices.Contains(enginesUsed(ctx), schemas.RoutingEngineSessionAffinity) { - t.Fatal("an unchanged chain was counted as a session decision") + // Agreeing with routing is a decision the session made, and a trail that omitted it could + // not be told apart from one where the session was never consulted. + if !slices.Contains(enginesUsed(ctx), schemas.RoutingEngineSessionAffinity) || !trailMentions(ctx, "which routing also proposed") { + t.Fatalf("agreement not recorded: engines=%v", enginesUsed(ctx)) } if res, _ := ctx.Value(sessionAffinityResolvedKey).(sessionResolution); res.route != "groq/openai/gpt-4o" { t.Fatalf("followed route not recorded: %+v", res) @@ -431,6 +433,11 @@ func TestSessionAffinityResolveRoute(t *testing.T) { if !trailMentions(ctx, "cannot use") { t.Fatal("stale binding was not explained in the trail") } + // Refusing a stale binding is a decision, so it is listed among the engines used: a trail + // entry attributed to an engine the request does not record cannot be filtered for. + if !slices.Contains(enginesUsed(ctx), schemas.RoutingEngineSessionAffinity) { + t.Fatalf("refusing a stale binding was not counted as a session decision: engines=%v", enginesUsed(ctx)) + } if _, recorded := ctx.Value(sessionAffinityResolvedKey).(sessionResolution); recorded { t.Fatal("a binding outside the chain was recorded as followed") } @@ -461,6 +468,11 @@ func TestSessionAffinityResolveRoute(t *testing.T) { if _, recorded := ctx.Value(sessionAffinityResolvedKey).(sessionResolution); recorded { t.Fatal("a binding that names no route was recorded as followed") } + // Nothing was decided and nothing is said, at either level: a binding that names no + // route is dropped as corrupt, not refused on the request's behalf. + if len(ctx.GetRoutingEngineLogs()) != 0 || slices.Contains(enginesUsed(ctx), schemas.RoutingEngineSessionAffinity) { + t.Fatalf("an unreadable binding was reported as a decision: engines=%v trail=%v", enginesUsed(ctx), ctx.GetRoutingEngineLogs()) + } routeKey := SessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o") if _, present := kv.data[routeKey]; present { t.Fatal("a binding that names no route was kept") @@ -706,7 +718,8 @@ func TestResolveSessionRouteAppliesTheAnswer(t *testing.T) { client, fake := setup(t, reverse) off := sessionCtx("s") off.SetValue(schemas.BifrostContextKeySessionAffinity, false) - for _, ctx := range []*schemas.BifrostContext{sessionCtx(""), off} { + noSession := sessionCtx("") + for _, ctx := range []*schemas.BifrostContext{noSession, off} { req := newRequest(schemas.OpenAI) client.resolveSessionRoute(ctx, routeOf("", "gpt-4o"), req) if provider, _, _ := req.GetRequestFields(); provider != schemas.OpenAI { @@ -716,6 +729,15 @@ func TestResolveSessionRouteAppliesTheAnswer(t *testing.T) { if fake.routeCalls != 0 { t.Fatal("ResolveRoute was asked about a request that takes no part") } + // A request that carries a session and switched affinity off says so, because its session + // id is on the log record and silence would read as affinity having had nothing to add. + if !trailMentions(off, "asked not to follow") { + t.Fatal("a request that switched affinity off did not say so in the trail") + } + // One with no session at all has nothing to explain. + if len(noSession.GetRoutingEngineLogs()) != 0 { + t.Fatalf("a request with no session wrote a trail: %v", noSession.GetRoutingEngineLogs()) + } }) } diff --git a/docs/providers/request-options.mdx b/docs/providers/request-options.mdx index 6d0e0be496b..51f8600b304 100644 --- a/docs/providers/request-options.mdx +++ b/docs/providers/request-options.mdx @@ -241,7 +241,7 @@ Bindings are written when a request is served, never when it fails, and are scop - **Retries**: A bound key is used for every retry of a request. A session with no key binding yet rotates keys on rate limits like any other request, and the key that serves is the one that gets bound. - **Fallbacks**: Fallback providers select keys freely. When a fallback serves, the session is rebound to that provider and its key, so the next request starts there. - **Switch**: `x-bf-session-affinity: off` turns affinity off for a request, at both levels; `on` is the default. -- **Routing trail**: Decisions the session made appear in the request's routing logs under the `session-affinity` engine, which is also listed among the routing engines used. +- **Routing trail**: Decisions the session made appear in the request's routing logs under the `session-affinity` engine, which is also listed among the routing engines used. The UI labels it **Session**. The [Session Affinity](/providers/session-affinity) guide covers the full lifecycle of a binding, how affinity composes with each routing engine, and the health checks Bifrost Enterprise runs before following one. diff --git a/docs/providers/session-affinity.mdx b/docs/providers/session-affinity.mdx index 03c22fe5c81..e32296dc01a 100644 --- a/docs/providers/session-affinity.mdx +++ b/docs/providers/session-affinity.mdx @@ -60,11 +60,14 @@ Bindings are written only when a request is served. A failed request writes noth | Served request that followed the binding | TTL refreshed | TTL refreshed | | Served by a fallback provider | Moved to the fallback provider | Moved to the key the fallback used | | Request failed on every attempt | Untouched | Untouched | +| Request refused by governance before routing | Untouched | Untouched | | Bound provider not in the chain routing built | Deleted, the routing decision stands, the next served request rebinds | Unchanged | | Bound key not in the eligible pool | Unchanged | Deleted, a key is picked normally, the next served request rebinds | | TTL expires with no served request | Gone | Gone | | Request sent with `x-bf-session-affinity: off` | Neither read nor written | Neither read nor written | +A request refused by governance outright, because its credential is inactive or expired, or because the provider or model it asked for is not allowed, is refused before any routing engine builds a chain. Affinity never sees it, so the session keeps whatever binding it had. Revoking a session's access to the provider it is bound to therefore leaves that binding in the trail until it expires, even though no request can follow it. Narrowing access instead, so routing still offers some other provider, deletes the binding on the next request and rebinds the session to whatever serves it. + The TTL defaults to one hour and can be set per request with `x-bf-session-ttl` (a duration string such as `30m`, or a number of seconds). Every served request that follows a binding refreshes it, so an active session does not expire mid-conversation. When several requests of a brand-new session arrive in parallel, they can land on different providers or keys before any of them is served. The first one to be served writes the binding, the rest of that burst is already in flight, and every request after it follows the winner. @@ -134,18 +137,22 @@ ctx = context.WithValue(ctx, schemas.BifrostContextKeySessionTTL, 30*time.Minute ## Seeing what the session did -Every decision affinity makes is written to the request's routing trail under the `session-affinity` engine, which also appears among the routing engines used for the request. In the logs explorer, filter by session ID to see one session's requests together, and by routing engine to find requests where affinity changed the outcome. +Every decision affinity makes is written to the request's routing trail under the `session-affinity` engine, and every decision also lists that engine among the routing engines used for the request, whether the session followed a binding or refused a stale one. A request that took no part, because it carries no session or asked not to follow one, lists no engine. In the logs explorer, filter by session ID to see one session's requests together, and by routing engine to find requests a session had a say in. The UI labels this engine **Session**; `session-affinity` is the identifier used in the API and in exported logs. | Trail entry | Meaning | | ------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------- | | `Session stays on azure for gpt-4o; routing proposed openai` | The bound provider was moved ahead of the one routing put first. | +| `Session stays on azure for gpt-4o, which routing also proposed` | The bound provider was already first, so the chain was left as it is. | | `Session reused key prod-eu-2 for azure/gpt-4o` | The bound key was used instead of running key selection. | | `Session was last served by azure for gpt-4o, which this request cannot use, so the routing decision stands and the session rebinds on its next served request` | Routing no longer offers the bound provider, so the binding was deleted. | | `The key this session last used for azure/gpt-4o is no longer eligible, so one is being picked` | The bound key left the pool, so the binding was deleted. | -| `Session was last served by azure for gpt-4o, which is failing on this node, so the routing decision stands` | Enterprise: the load balancer marks the provider as failed here, so the binding was dropped. | -| `The key this session last used for azure/gpt-4o is held back by the circuit breaker on this node, so one is being picked` | Enterprise: the circuit breaker removed the bound key from the pool, so the binding was dropped. | +| `Request carries a session but asked not to follow it, so the routing decision stands for gpt-4o` | The request sent `x-bf-session-affinity: off`, so no binding was read or written. | +| `Session was last served by azure for gpt-4o, which is failing on this node, so the routing decision stands` | Enterprise only: the load balancer marks the provider as failed here, so the binding was dropped. | +| `The key this session last used for azure/gpt-4o is held back by the circuit breaker on this node, so one is being picked` | Enterprise only: the circuit breaker removed the bound key from the pool, so the binding was dropped. | + +The last two entries are emitted only by Bifrost Enterprise; an open-source deployment never writes them, because it follows a binding until the request itself fails. -When routing already proposes the bound provider, the provider level writes nothing, because it changed nothing. The key level records every reused key. +Both levels record every binding they follow, whether or not it changed the outcome, so a session that agrees with routing is distinguishable in the trail from one that was never consulted. In OpenTelemetry, the trace's root span carries the session ID as the `session.id` attribute, and the `group_traces_by_session` setting puts a whole session into one trace. See [OpenTelemetry](/features/observability/otel#grouping-traces-by-session). diff --git a/plugins/governance/main.go b/plugins/governance/main.go index e4c9d38b72e..dca07e67dab 100644 --- a/plugins/governance/main.go +++ b/plugins/governance/main.go @@ -458,11 +458,16 @@ func (p *GovernancePlugin) LoadBalanceProvider(ctx *schemas.BifrostContext, req return nil } + // Only weighted candidates can be selected or offered as fallbacks, so a candidate without a + // weight is as excluded as one a budget refused. Name it for the same reason every other + // exclusion above is named: the counts below are otherwise impossible to reconcile from the trail. weighted := make([]schemas.ProviderCandidate, 0, len(eligible)) for _, candidate := range eligible { - if candidate.Weight != nil { - weighted = append(weighted, candidate) + if candidate.Weight == nil { + ctx.AppendRoutingEngineLog(schemas.RoutingEngineGovernance, schemas.LogLevelInfo, fmt.Sprintf("Provider %s excluded: no weight assigned for model %s", candidate.Provider, modelStr)) + continue } + weighted = append(weighted, candidate) } if len(weighted) == 0 { @@ -495,8 +500,16 @@ func (p *GovernancePlugin) LoadBalanceProvider(ctx *schemas.BifrostContext, req selectedProvider = schemas.ModelProvider(weighted[0].Provider) } + var weightedProviders []string + for _, candidate := range weighted { + weightedProviders = append(weightedProviders, candidate.Provider) + } + p.logger.Debug("[governance] Selected provider: %s", selectedProvider) - ctx.AppendRoutingEngineLog(schemas.RoutingEngineGovernance, schemas.LogLevelInfo, fmt.Sprintf("Selected provider %s for model %s (from %d eligible: %v)", selectedProvider, modelStr, len(eligible), eligibleProviders)) + // The pool selection actually ran over, which is `weighted` and not `eligible`: reporting the + // wider set would describe candidates the roll could never have landed on, and would not + // account for the fallbacks added below. + ctx.AppendRoutingEngineLog(schemas.RoutingEngineGovernance, schemas.LogLevelInfo, fmt.Sprintf("Selected provider %s for model %s (from %d weighted: %v)", selectedProvider, modelStr, len(weighted), weightedProviders)) refinedModel := modelStr // Refine the model for the selected provider From c0878dcfc11f1b56e75c583805e4304591e32439 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:21:39 +0000 Subject: [PATCH 007/118] Adds changelog for v2.2.1 --skip-ci --- core/changelog.md | 68 ------- docs/changelogs/v2.2.1.mdx | 220 ++++++++++++++++++++++ docs/docs.json | 196 ++++++++++++++----- framework/changelog.md | 17 -- plugins/compat/changelog.md | 2 - plugins/governance/changelog.md | 8 - plugins/jsonparser/changelog.md | 1 - plugins/logging/changelog.md | 7 - plugins/maxim/changelog.md | 1 - plugins/mocker/changelog.md | 1 - plugins/modelcatalogresolver/changelog.md | 1 - plugins/otel/changelog.md | 1 - plugins/prompts/changelog.md | 1 - plugins/routing/changelog.md | 2 - plugins/semanticcache/changelog.md | 1 - plugins/telemetry/changelog.md | 3 - transports/changelog.md | 43 ----- 17 files changed, 372 insertions(+), 201 deletions(-) create mode 100644 docs/changelogs/v2.2.1.mdx diff --git a/core/changelog.md b/core/changelog.md index 21b5f509c98..e69de29bb2d 100644 --- a/core/changelog.md +++ b/core/changelog.md @@ -1,68 +0,0 @@ -- fix: `POST /v1/realtime/client_secrets` mints a Bifrost-issued ephemeral token instead of returning the provider's token (e.g. OpenAI `ek_...`) verbatim, which let a client connect to the provider directly and bypass governance, metering and attribution; virtual key credentials are validated before a realtime session is admitted, so a nonexistent VK can no longer open an upstream provider session (#7317) -- fix: streaming chat chunks no longer emit empty reasoning/message fields on every content delta, which made clients render repeated empty thinking blocks (#7318) -- fix: an abandoned non-streaming request no longer hangs its caller forever; the worker kept a `ctx.Done()` arm on an already-claimed delivery send, a regression from the #6972 delivery fix (#7313) -- fix: caller-supplied `s3://` file IDs and `s3_bucket` params on the Bedrock provider are validated with a DNS-compatible bucket regex before URL construction, closing an SSRF where a bucket like `127.0.0.1:PORT#` steered the SigV4-signed request to a caller-chosen host (#7315) -- fix: resource IDs (batch, file, video, container, response IDs and cached content names) are validated and escaped through a central `EscapeResourceID` helper before being interpolated into outbound URL paths, so IDs carrying `../`, `?`, `#`, percent-encoded bytes or control characters can no longer redirect requests to unintended upstream endpoints (#7307) -- fix: an internally minted `BillingNonce` is mixed into the billing-idempotency key, which was previously `(RequestID, AttemptNumber)` alone; since `x-request-id` is caller-supplied, two unrelated requests sharing a forged ID collided on the key and the second was never charged (#7303) -- fix: Bedrock service tier forwarding is gated on explicit model capability metadata (failing closed when none exists) on both Converse and Mantle paths, instead of sending whatever tier was requested and getting rejected by models that do not support it (#7266) -- fix: root-level `oneOf`/`anyOf`/`allOf` in a custom tool's `input_schema` are normalized into a flat object schema before reaching Anthropic, which rejects composition keywords at the schema root but accepts them inside properties; unblocks tools like Codex's `automation_update` (#7265) -- fix: speech stream chunks have a GenAI converter (`ToGeminiSpeechStreamResponse`), enabling streaming TTS through the `/genai` integration, and the streaming router returns a clean error instead of panicking when a stream converter is missing (#7248) -- feat: `SupportsPromptCacheBreakpoints` model cap with a name-based fallback for OpenRouter, so `prompt_cache_breakpoint` is only forwarded to models that accept it (#7261, #7262) -- fix: xAI `completion_tokens` folds `reasoning_tokens` into the total so `prompt_tokens + completion_tokens = total_tokens` holds, and streaming cost normalization preserves xAI's authoritative `cost_in_usd_ticks` through to the terminal usage chunk instead of falling back to catalog pricing (#7245, #7250) -- fix: `search_content_types` on `web_search` tools is an OpenAI-specific extension and is now gated behind a per-provider capability check, so Bedrock and other OpenAI-compatible backends receive a clean `web_search` tool without the field (#7244) -- fix: the `diagnostics` field Claude Code sends on every request is stripped for providers that reject it (Bedrock, Vertex, Azure return 400 `diagnostics: Extra inputs are not permitted` on the typed sanitizer path) (#7243) -- fix: AWS Bedrock's flat error shape `{"message": ..., "__type": ...}` seeds the root-level `message` into `BifrostError`, so Bedrock errors served through the shared Anthropic/OpenAI handlers are logged with a human-readable reason (#7222) -- [fix]: the string-form `container` param is carried through the /v1/messages round trip instead of being silently dropped, so container reuse stops provisioning a fresh container (#5829) (thanks [@AdityaPainuli](https://github.com/AdityaPainuli)!) -- fix: the Cohere-compatible route converts fallback responses from non-Cohere providers into the Cohere v2 shape (`message`, `finish_reason`) via `ToCohereChatResponse`, instead of returning the raw Bifrost normalized response the Cohere SDK cannot parse (#7198) -- fix: clearer error message in the Anthropic integration (#7264) -- [fix]: preserve Gemini 3.1 Flash-Lite minimal thinking on Gemini and Vertex [@Javtor](https://github.com/Javtor) -- fix: the Anthropic raw-body request path strips the server-side `thread` field before the wire on both messages and count_tokens (thread state is bound to the creating account and cannot survive Bifrost's per-request key selection, retries, or fallbacks; the HTTP transport refuses continuations outright), `AnthropicMessageErrorStruct` gains an optional `details` object with `error_code` so error envelopes can carry the machine-readable codes clients key their recovery on, and `LargePayloadMetadata` gains `ThreadType` so the refusal also works when large-payload mode skips body parsing (populated by the enterprise metadata extractor) -- [fix]: MCP tool schemas keep one property order across tool syncs [@dougcalobrisi](https://github.com/dougcalobrisi) -- feat: `WhiteList` and `BlackList` entries that start with `regex:` are RE2 patterns. `MatchEntry` compiles them once as a case-insensitive full match, `Validate` refuses a pattern that is empty, `*` or does not compile, and the list-models pipeline never surfaces a pattern as a model. Plain entries keep their exact, case-insensitive match -- fix: fasthttp-level stale-connection retries no longer multiply `max_retries`. `contextTransport.RoundTrip` reports a failure before response headers on a freshly dialed socket with `retry=false`, so `network.StaleConnectionRetryIfErr` only walks past pooled keep-alive sockets the upstream closed while idle; an upstream that closes a fresh connection without answering now costs exactly one attempt of Bifrost's own retry budget instead of up to four. The retry backoff also ends as soon as the request context is cancelled, so a worker is freed immediately instead of after up to `retry_backoff_max` (#7035) -- [fix]: Gemini provider - preserve inline image and audio data in chat completions responses [@Atharva-Kanherkar](https://github.com/Atharva-Kanherkar) -- [fix]: route opencode-zen Responses calls through /v1/chat/completions, which its upstream does not serve on /v1/responses [@miguelchico](https://github.com/miguelchico) -- feat: the namespace-tool names a provider reserves for its own server tools (`web`, `image_gen`, `browser`, `python` on Bedrock and Bedrock Mantle) are now read from the datasheet row `reserved_tool_namespaces` (new `ModelCapabilities` field and `ModelCaps.ReservedToolNamespaces`) for the base provider and canonical model; a non-empty row replaces the hardcoded list, an absent or empty row keeps it, so any OpenAI-wire provider can reserve names through a row without a code change -- fix: Codex >= 0.147 wraps its default tools in a namespace literally named `functions` (openai/codex#37022), which Bedrock Mantle reserves and rejects with `User-defined namespace 'functions' collides with an existing tool namespace`. Core now unwraps that namespace to top-level tools, unprefixed, for every provider before dispatch; Codex treats the bare name and the explicit `functions` namespace as the same tool, so no mapping back is needed; a description on the `functions` namespace is prepended to each hoisted member, as flattening does -- fix: Responses `namespace` tools are now flattened in core for every provider whose wire does not support the type (Anthropic, Gemini, Vertex, Bedrock Converse, DeepSeek, and every OpenAI-compatible third party), with nested functions renamed to `__` so two namespaces that share a function name no longer collide into an upstream `Tool names must be unique` 400; returned `function_call` items are mapped back to the bare `name` plus `namespace` (unary and streaming), prior-turn calls carrying `namespace` and `tool_choice` names are re-aliased to match, and a name that is still duplicated after flattening or a `tool_choice` that matches several namespaces is rejected with a clear 400 before reaching the provider. Bedrock answers namespace support from its own surface resolver via the new optional `ResponsesNamespaceToolProvider` interface. A datasheet row `supports_namespace_tools` (new `ModelCapabilities` field and `ModelCaps.SupportsNamespaceTools`) overrides the per-provider default for a (provider, model) pair; with no row the default applies. A row can only narrow within what the wire can carry: the Anthropic Messages API (Anthropic, Claude on Azure or Bedrock Mantle), the Gemini API (Gemini, Vertex) and Bedrock Converse have no namespace container, so they answer false regardless of the row and always flatten. Flattened names honour the target wire's documented tool-name limit, 64 characters of `[A-Za-z0-9_-]` for OpenAI-compatible wires, Bedrock Converse and Fireworks, 128 for Anthropic, and 128 with `.` and `:` allowed for Gemini and Vertex, overridable per model through the datasheet row `tool_name_max_length` (a row below 10 cannot hold the hashed form and is ignored); a longer alias becomes an 8-hex hash prefix plus the function name, deterministically, so history and `tool_choice` re-alias to the same string. The alias map travels on the prepared request (`BifrostResponsesRequest.NamespaceToolAliases`) and is applied to that attempt's response, unary and streaming; nothing is kept on the request context or in process-wide state (#7048) -- fix: drop namespace tools whose name Amazon Bedrock reserves (`web`, `image_gen`, `browser`, `python`) on the Bedrock and Bedrock Mantle Responses paths instead of forwarding them into a `tools.namespace` collision 400; a dropped Codex `web` namespace becomes the hosted `web_search` tool on Bedrock Mantle -- fix: Bedrock Mantle chat streaming no longer drops the usage-only chunk that arrives after `finish_reason`; `ProviderSendsDoneMarker` now treats `bedrock_mantle` (and the legacy Mantle route under the `bedrock` key) as sending `[DONE]`, so streamed usage and cost are recorded (#7065) -- fix: `default_request_timeout_in_seconds` now bounds the wait for response headers on streaming requests, and cancelling a request now closes the upstream socket. Every fasthttp client is driven through a Bifrost-owned `RoundTripper` (`core/providers/utils/roundtripper.go`) that applies the client's read/write timeouts and the request context to the request write and the header wait, then lifts the socket deadline once headers are parsed so `stream_idle_timeout_in_seconds` remains the only bound on the body. `BuildStreamingClient` and `BuildLargeResponseClient` no longer zero the timeouts. An upstream that accepts the connection and never answers now fails with 504 `RequestTimedOut` after the request timeout and the request's fallbacks are used, instead of pinning the provider worker until the upstream closes the socket; `MakeRequestWithContext` no longer leaves a background `client.Do` running until `ReadTimeout` after its context is cancelled (#7034) -- fix: unary large-response downloads (`FinalizeResponseWithLargeDetection`) now bound every body read with `stream_idle_timeout_in_seconds` and close the upstream socket on request cancellation, mirroring the streamed passthrough path; a stall during the prefetch surfaces as 504 `RequestTimedOut` instead of pinning the provider worker, and a stall while the transport drains the `LargeResponseReader` fails the read with the idle-timeout error instead of blocking the writer indefinitely (#7104) -- fix: gzip-encoded unary responses are now classified for large-response mode by decompressed size; `Content-Length` describes the compressed bytes, so a body that was small on the wire but large once decompressed used to be materialized in full past `large_response_threshold` (#7104) -- fix: an OpenAI-compatible upstream that omits `[DONE]` and then goes silent after `finish_reason` no longer fails the stream when `stream_idle_timeout_in_seconds` fires. The chat and text completion read loops now treat an idle timeout after a terminal signal as a parked upstream, mark the stream parked so the connection is abandoned rather than drained, and synthesize the final chunk with the buffered `finish_reason`; a stall before `finish_reason` still surfaces as the idle-timeout error (#7108) -- fix: a streamed upstream that drops the connection mid-body is again reported as the retryable 502 `provider closed the stream before sending a completion marker` error instead of a generic `Error reading stream: unexpected EOF`. The Bifrost round tripper's chunked decoder surfaced the drop as `io.ErrUnexpectedEOF`, which no provider read loop treats as end of stream; it now reports the plain `io.EOF` fasthttp always did and discards the half-read connection (#7104 follow-up) -- fix: fallbacks for image edit, image variation and video edit requests now reach the configured fallback provider and model. `prepareFallbackRequest` had no arm for those three types, so the shallow request copy kept the primary's sub-request pointer and the "fallback" attempt was routed back to the primary while `RoutingInfo`, the `x-bifrost-routing-info-*` headers, the log row and the `fallback_index` metric label reported it as a fallback. The helper now also verifies the prepared request targets the fallback provider and model and skips the fallback with a warning otherwise, so a future request type added without an arm fails loudly instead of silently re-running the primary (#6966) -- fix: route Bedrock Claude requests that carry a `compact_20260112` edit to InvokeModel / InvokeModelWithResponseStream with the native Anthropic Messages body, so server-side compaction works on `bedrock/` models, including keys that pin an inference-profile ARN. AWS documents compaction as unsupported on the Converse API, which previously received the edit and silently ignored it (#6825) -- feat: serve Anthropic tool search (`tool_search_tool_*`, `defer_loading`) on `bedrock/` Claude models by routing those requests to InvokeModel / InvokeModelWithResponseStream, the only Bedrock API AWS allows it on; CountTokens counts such requests with the same InvokeModel body via the `invokeModel` input -- fix: a `reasoning.context` value the target model does not accept is now dropped on the OpenAI and Azure Responses path, so `all_turns` on the original gpt-5 family incl. gpt-5-pro, gpt-5.1 to gpt-5.3, and the o-series runs under the model's own `current_turn` default instead of failing with `Unsupported value: 'all_turns' is not supported with the 'gpt-5-pro' model`; gpt-5.4, gpt-5.5 and gpt-5.6 keep it. The accepted values come from the datasheet row `supported_reasoning_contexts` (new `ModelCapabilities` field and `ModelCaps.SupportedReasoningContexts`), falling back to a name-based default of `auto` and `current_turn` everywhere plus `all_turns` on gpt-5.4+ -- fix: mid-conversation `role: "system"` messages are now inlined in place as `` user turns for every model family on every converter that has a top-level system field: Bedrock Converse (Responses and Chat Completions), Gemini Chat Completions (`systemInstruction`), and the Anthropic wire shape used by DeepSeek, Fireworks and SGL. Previously only Claude on Bedrock/Anthropic inlined; everything else hoisted each reminder into the top-level system block. Claude Code appends a trailing `` system reminder after every turn, so the hoisted block grew the front of the prompt each turn and prefix-based caches (Bedrock implicit caching for `global.openai.gpt-5.6-luna` on Converse, Gemini implicit caching, DeepSeek context caching) reported a full cache write and zero cache reads on every turn -- [fix]: vLLM key selection resolves the requested alias through `key.Aliases` per key before comparing it with `vllm_key_config.model_name`, so a valid key is no longer rejected when the request used an alias; the same public alias can map to different physical model IDs across vLLM instances, and allow and block checks keep matching the original user-facing alias [@Constantine3](https://github.com/Constantine3) -- [fix]: Gemini Responses requests keep `top_k`, `frequency_penalty`, `presence_penalty`, `stop_sequences` and `media_resolution` across retries and fallbacks. `convertParamsToGenerationConfigResponses` deleted those keys from `ExtraParams` while mapping them into `generationConfig`, and the conversion runs once per attempt on the same request, so every attempt after the first reached the provider without them; the outbound `ExtraParams` is now built through a filtered copy instead of mutating the request [@VictorRequenaMaisa](https://github.com/VictorRequenaMaisa) -- feat: `wait_for_usage` on `custom_provider_config` keeps the OpenAI-compatible chat and text completion read loops open past `finish_reason` until the trailing usage-only frame arrives, so a custom provider that also sets `does_not_send_done_marker` no longer synthesizes a zero-usage terminal chunk and records the request at zero tokens and zero cost. `WaitForUsage` is carried on `BifrostContextKeyWaitForUsage` with the same set-or-clear discipline as `DoesNotSendDoneMarker`, so it never leaks onto a fallback provider that did not declare it, and termination stays bounded by the usage chunk, two consecutive post-finish heartbeat comments, EOF, or `stream_idle_timeout_in_seconds` (#7143) -- fix: `ExtraFields.RawResponse` now carries role-only, finish-only and usage-only SSE frames on the OpenAI-compatible chat stream. Those frames never entered the chunk-forwarding branch, so their bytes were discarded from the reconstructed `raw_response`, which made the captured audit trail irreconcilable against a provider invoice because the usage frame carries the token counts Bifrost bills from. A `pendingRawFrames` buffer accumulates them and drains onto the next forwarded chunk or the synthetic terminal chunk; the Responses-over-Chat fallback no longer stamps one upstream frame onto every derived event, and `delta.refusal` and `delta.annotations` are added to the forwarding predicate so refusals and streamed URL citations are no longer dropped entirely (#7144) -- fix: Anthropic server-side tool search survives the Bedrock-native invoke ingress end to end. `tool_search_tool_*` was dropped during the Converse-shaped intermediate conversion, leaving the egress predicate blind to the tool and routing the request to Converse where server-side tool search is unavailable; the tool is now carried as an ingress-only marker (`json:"-"`, never reaching a Converse body) and the neutral tool is rebuilt in `ToBifrostResponsesRequest` along with `defer_loading`. Results are returned as a `server_tool_use` plus `tool_search_tool_result` pair rather than a client `tool_use`, which Anthropic rejects when the caller echoes a `tool_result` for the `srvtoolu_` ID on the next turn, on both the unary and streaming paths; `BedrockContentBlock.UnmarshalJSON` gained `server_tool_use` and `tool_search_tool_result` cases so replayed turns round-trip unchanged instead of falling through to empty structs; and the Anthropic-native response path carries the same blocks (#7155) -- fix: native Gemini `generateContent` image requests routed through `/genai` no longer return HTTP 200 while silently ignoring part of the request. `isImageEditRequest` only inspected `contents[0].parts[0]`, so a request with the prompt text before the `inlineData` part, the ordering in Google's own REST edit sample, was misclassified as image generation: the image never reached Vertex and the model invented a picture from the prompt alone. Detection now scans every part across every content. `imageConfig.aspectRatio` is preserved as a typed `aspect_ratio` param that both outbound converters prefer, instead of being folded into a `WxH` size string that collapsed any ratio outside `1:1`, `3:4`, `4:3`, `9:16` and `16:9` back to square (#7173) -- fix: Gemini `Part` gained `mediaResolution`, the per-part override for `generationConfig.mediaResolution`. `Part.UnmarshalJSON` decodes into a closed alias, so the key was discarded before any conversion ran and per-part image and PDF tokenization fell back to the model default; an `ULTRA_HIGH` image billed about 21k prompt tokens through `/genai` instead of about 22.1k direct. Both the `mediaResolution` and `media_resolution` spellings are parsed, the value round-trips through `ResponsesMessageContentBlock` and is reconstructed on the outbound part, and it is stripped on the OpenAI wire path where the field is unknown (#7156) -- fix: image blocks inside a Bedrock Converse `toolResult` are hoisted out and re-inserted immediately after the last tool result in the same message, with a placeholder text block left behind so the now image-free result is not rejected for being empty. Some Bedrock-hosted models, notably the OpenAI and Grok families, reject nested tool-result images on Converse even though they accept images in tool output via the Responses API. `BedrockModelSupportsToolResultImages` supplies the name-based default and the datasheet field `supports_converse_tool_result_images` (new `ModelCapabilities` field and `ModelCaps.SupportsConverseToolResultImages`) overrides it per row (#7150) -- feat: `bifrost_error_requests_total` gains a normalized `error_type` label drawn from a closed, prefix-structured vocabulary (`caller_*`, `policy_*`, `provider_*`, `bifrost_*`, `_OTHER`), so a 429 from a governance rate limit is distinguishable from a 429 from an upstream provider, a 403 from a blocked model from one from a bad key, and a 503 Bifrost shed under queue pressure from an upstream overload. `ClassifyErrorType` resolves a declared `ExtraFields.ErrorType` first, then Bifrost's own string markers, then the upstream status code; it deliberately ignores the provider's own `error.type` and `error.code` strings, which disagree across providers for the same condition. Values are prefixed by fault domain so an alarm expression can match a family with a single regex (#7141) -- fix: Azure Foundry deployments of Fireworks-hosted models are no longer silently capped at 4096 output tokens on `/openai/v1/responses`, where Microsoft routes those models through chat completions internally. The Azure provider checks the model's datasheet `supported_endpoints` and, when `/v1/responses` is absent, transparently serves both `Responses` and `ResponsesStream` through `/openai/v1/chat/completions` and converts the result back to the Responses shape; a model with an explicit `/v1/responses` entry or no datasheet row at all is unchanged. Separately, `ToAnthropicResponsesResponse` now follows `StopReason > IncompleteDetails > tool_use inference > end_turn`, so a turn truncated by the output-token cap on any OpenAI-shaped Responses provider reports `stop_reason: max_tokens` instead of hiding the truncation as `end_turn` (#6782) -- fix: DeepSeek chat requests move `max_completion_tokens` into `max_tokens` on the wire. DeepSeek's chat-completions endpoint only recognizes the legacy field and silently ignores `max_completion_tokens`, so the limit had no effect; this matches the behaviour already in place for Opencode and Ollama (#7131) -- feat: guardrail redaction gains an identity-based transformation path for provider-native request and response bodies, so exact provider-managed rewrites such as Model Armor and Bedrock land in the correct native JSON field even when the same text value appears in several fields. `TextTargetID`, `TextTargetIDForIndex` and `TextRewrite` identify each guardrail-visible field by a stable ID rather than by content value; `RawRequestBodyTextTransformer` and `RawResponseTextTransformer` let integrations register handlers on the request context; and `rewriteRawJSONTextTargets` validates target uniqueness, verifies each `Original` value before patching and re-reads the written value after, so stale or mismatched normalized text cannot silently rewrite the wrong field (#7049) -- feat: guardrail redaction covers LLM tool-call arguments, Chat function arguments, Responses function arguments and custom-tool input, across the Anthropic streaming and non-streaming paths. `input_json_delta` events are inspected and rewritten alongside `text_delta`, reading and writing `delta.partial_json`; `collectAnthropicArgumentStringPaths` recursively collects string-valued paths inside a `tool_use` block's `input` without touching tool names, IDs or definitions; `BifrostResponsesStreamResponse` gained `Input` to carry the full custom-tool payload on `custom_tool_call_input.done`; and `DeepCopyResponsesMessage` deep-copies `ResponsesCustomToolCall` so a copied message cannot mutate the original (#6977) -- fix: an abandoned non-streaming request, one whose caller context was already cancelled by the time the upstream finished, is billed and logged deterministically instead of about half the time. The worker's delivery `select` had two simultaneously ready cases, a send into a cap-1 channel and `ctx.Done()`, and Go picks uniformly among ready cases, so the terminal post-hooks that record billing and finalize the log row were skipped roughly 50% of the time. `requestWorker` now checks `req.Context.Err()` before the select on both the error and success paths and calls `billAbandonedTerminal` directly, keeping the 5-second timer guard inside the select for a caller that leaves between the check and the send (#6972) -- fix: provider response headers are filtered by classifier, not only by a fixed map of 28 exact names, so a credential-named header outside that map is no longer re-served to the inference caller in the HTTP response and `extra_fields.provider_response_headers`. The extractors now also consult `schemas.IsSensitiveHeader`, which matches credential names by substring and suffix and already recognized `cf-access-*` and `x-amzn-oidc-*`; a fixed list cannot enumerate the space when `network_config.extra_headers` exists to carry custom authentication headers and some upstreams echo request headers back (#7120) [@Atharva-Kanherkar](https://github.com/Atharva-Kanherkar) -- [fix]: a plugin that returns an incomplete `BifrostError` whose nested `Error` field is nil no longer panics the request worker. The fallback decision helper nil-checks `fallbackErr`, guards access to `Error.Type` and reads the message through the nil-safe `GetErrorString()` helper, continuing to the next fallback when allowed; behaviour for cancelled requests and `AllowFallbacks == false` is unchanged (#6967) [@Constantine3](https://github.com/Constantine3) -- [fix]: `clearCtxForFallback` now clears `BifrostContextKeyProviderResponseHeaders`. Providers set that key from their own HTTP response before the status check so error paths can forward it, and when a fallback attempt failed pre-flight, key selection failing for the fallback provider, a plugin short-circuiting it, or the queue retiring, nothing overwrote the key and the primary's headers survived onto the fallback's error response. A client could receive a response attributed to provider B carrying provider A's `Retry-After` and `x-ratelimit-remaining-*`, and wait according to a limit belonging to a provider that never served the request (#6973) [@Huang-404-Q](https://github.com/Huang-404-Q) -- [fix]: untitled document blocks get unique names on the Bedrock Converse path. Every untitled block was named the literal `document`, and Converse rejects duplicate document names, so any request carrying two or more untitled documents failed unconditionally with `ValidationException: Messages can't contain duplicate document names`. A per-request namer threaded through the content-block conversion disambiguates with numeric suffixes (`document`, `document-2`, `document-3`) and suffixes explicitly titled documents only on an actual collision; the Responses input replay path gets the same treatment scoped to its content-block list (#7003) [@Huang-404-Q](https://github.com/Huang-404-Q) -- [fix]: `BifrostResponsesStreamResponse.Item` is `omitempty`, so Responses stream events that carry no item payload no longer serialize `"item": null`. Strict OpenAI Responses clients reject those as invalid stream frames, which broke streamed `/v1/responses` usage entirely; `response.output_item.added` and `response.output_item.done` still emit the item object [@ReStranger](https://github.com/ReStranger) -- feat: `SecretVar.RedactedIfSecret()` returns a plain clone when the value is a literal and delegates to `Redacted()` when it is env-var or vault-backed, so regions and service URLs (Azure endpoint, Vertex/Bedrock/Bedrock Mantle region, vLLM/Ollama/SGL/Databricks URL, MCP connection string) are readable in the UI instead of being masked as if they were credentials. It always returns a fresh pointer, so the redacted copy never aliases the live config, and it replaces the ad-hoc `IsFromSecret()` guard blocks that tried to replicate this inline (#7085) -- feat: GA realtime transcription is served for OpenAI and Azure over both WebSocket and WebRTC with normal Bifrost authentication, routing, governance, guardrails, logging and transcription-aware pricing. Unlike a normal realtime session, which names its routing model in the connect URL, a GA transcription session carries only `intent=transcription` and delivers the model later in `session.update` under `audio.input.transcription.model` (or in the initial multipart `/v1/realtime/calls` request on WebRTC), so routing now resolves from the nested transcription model while realtime connection and turn semantics are preserved (#7089) -- fix: Claude Code replaying a prior assistant message through `POST /anthropic/v1/messages` at a Bedrock Mantle model is no longer rejected with hundreds of validation errors. The Bedrock-grouped ingress converter tagged user and system input text as `output_text`, though only `input_text` is valid on input messages, and omitted the required `status` on replayed assistant output messages, so Mantle's strict OpenAI-compatible validator refused the whole request. Replayed assistant messages now carry `Status: "completed"` and input text blocks are tagged `input_text`. Bedrock requests with no explicit `max_tokens` also populate it from the model's known capacity instead of truncating silently on large context windows (#7074) -- fix: Anthropic server tools such as `web_search_20250305` are stripped before a request reaches Fireworks, vLLM or SGLang on their Anthropic-compatible endpoints, which return 400 for tools executed by Anthropic-operated infrastructure that does not exist on third-party hosts; clients whose built-in web search is always on, such as Codex, hit this on every request. `ProviderFeatures` entries for `Fireworks`, `VLLM` and `SGL` declare all server-tool flags off so the existing validators strip them, `StripUnsupportedServerToolsFromRawBody` mirrors that on the raw-body passthrough path, the caller's function tools are kept, and the drops are reported on the response's `DroppedUnsupportedTools` instead of failing the call (#7090) -- fix: `guardrailConfig` reaches Bedrock's OpenAI-compatible `chat/completions` and `responses` surfaces, streaming and non-streaming, which apply guardrails through request headers rather than the body field Converse uses, so a configured guardrail was previously ignored there with no error. `withGuardrailHeaders` maps `guardrailIdentifier`, `guardrailVersion` and optional `trace` to `X-Amzn-Bedrock-GuardrailIdentifier`, `X-Amzn-Bedrock-GuardrailVersion` and `X-Amzn-Bedrock-Trace`, then deletes the key from `ExtraParams` so it is not also emitted into the body. A half-formed config with only one of identifier or version is left untouched rather than sent, since both are required upstream, and the base header map is cloned rather than mutated (#7095) -- feat: `use_openai_endpoints` on Bedrock keys and aliases routes chat completions and responses through Bedrock's OpenAI-compatible `/openai/v1` endpoints instead of Converse, for models that support them, mirroring the existing `use_anthropic_endpoints` pattern. It is opt-in by design: Converse carries Bedrock Guardrails, `performanceConfig` and `requestMetadata` that the OpenAI-compatible surface silently ignores, so diverting automatically could stop a guardrail from being enforced with no visible error. `ResolveUseOpenAIEndpoints` gives the alias value precedence over the key, and the narrow `runtimeServesResponses` check is replaced by a general `runtimeServesOpenAIAPI` that takes a `BedrockAPI` discriminator; chat completions, streaming and non-streaming, gained the runtime path that previously existed only for Responses (#7073) -- fix: Bedrock runtime models that serve the Responses API are routed to it through a dedicated surface resolver rather than falling back to Converse (#7071) -- fix: Bedrock Converse document blocks always ship their content base64-encoded through `source.bytes`, never through `source.text`, which Converse rejects unless citations are explicitly enabled. Plain text formats (`text/plain`, `text/markdown`, `text/csv`, `text/html`) previously took the text branch and failed with `DocumentSource must set one of the following keys: bytes, s3Location`; the branch is removed for data URLs, percent-encoded payloads and the non-data-URL `file_data` path alike (#7072) -- fix: the Bedrock Mantle base path is resolved from the model capabilities datasheet instead of hard-coded string matching in two packages. Mantle serves each model on exactly one of `v1` or `openai/v1` and returns a 400 on the other, and the previous matching covered only generations up to GPT-5 and Gemma 4, so GPT-6 and any future closed-generation model silently fell through to the wrong path. `ResolveBedrockMantleBasePath` applies family-name detection as a fallback and defers to the new `BedrockMantleBasePath` field on `ModelCapabilities`, so a new generation needs a datasheet row rather than a code change (#7077) -- fix: `image_generation_call` items where OpenAI emits `action` as a bare JSON string, such as `"generate"`, decode correctly. `ResponsesToolMessageActionStruct.UnmarshalJSON` immediately peeked at a `.type` field, which cannot be read from a JSON string, so the whole decode failed with `failed to peek at type field`; that silently dropped the `response.output_item.done` and `response.completed` events carrying the image, leaving the stream without a terminal event and surfacing as a bogus `provider closed the stream` truncation error. The action is now tried as a bare string before falling back to the object type-peek, and `ResponsesImageGenerationCall` keeps the `background`, `output_format`, `quality`, `revised_prompt` and `size` settings OpenAI echoes back on completed items (#7060) diff --git a/docs/changelogs/v2.2.1.mdx b/docs/changelogs/v2.2.1.mdx new file mode 100644 index 00000000000..3560ee2c778 --- /dev/null +++ b/docs/changelogs/v2.2.1.mdx @@ -0,0 +1,220 @@ +--- +title: "v2.2.1" +description: "v2.2.1 changelog - 2026-09-18" +--- + + + ```bash + npx -y @maximhq/bifrost --transport-version v2.2.1 + ``` + + + ```bash + docker pull maximhq/bifrost:v2.2.1 + docker run -p 8080:8080 maximhq/bifrost:v2.2.1 + ``` + + + + +## ✨ Features + +- **Virtual Key Assignees and Expanded Search** - The virtual key list resolves `assigned_user` for every row via a single batch lookup, and VK search matches team, customer and user names in addition to key fields, so keys can be found by who owns them (#7225, #7226) +- **User Attribution in Prometheus Metrics** - The telemetry plugin's metrics export includes user id and user name labels, so per-user usage and error rates can be graphed and alerted on directly (#7267) +- **Prompt Cache Breakpoints Capability** - A `SupportsPromptCacheBreakpoints` model cap with a name-based fallback for OpenRouter gates `prompt_cache_breakpoint` forwarding, so OpenRouter's non-Claude models stop rejecting requests that carry it (#7261, #7262) +- **MCP Observed Latency** - Native (observed) MCP tool-call latency from `observed_latency_ms` is surfaced in the logs duration column and detail sheet, kept clearly distinct from true execution time and never fabricating a synthetic start timestamp on the timeline (#7271) + +## 🐞 Fixed + +- **Resource ID Path Injection** - Caller-supplied resource IDs (batch, file, video, container, response IDs, cached content names) were interpolated into outbound URL paths unvalidated; a crafted ID with `../`, `?`, `#`, percent-encoded bytes or control characters could redirect the request to an unintended upstream endpoint. A central `EscapeResourceID` helper validates and escapes every such ID (#7307, #7310) +- **Bedrock S3 SSRF** - A caller-supplied `s3://` file ID or `s3_bucket` param could control the upstream host, since S3 virtual-hosted URLs are built as `https://{bucket}.{s3host}/{key}`; a bucket like `127.0.0.1:PORT#` opened a SigV4-signed TLS connection to a caller-chosen host. Bucket names are now validated with a DNS-compatible regex (#7315) +- **Caller-Forged Billing Idempotency** - The billing-idempotency key was `(RequestID, AttemptNumber)`, and `RequestID` may come from the caller's `x-request-id` header, so two unrelated requests sharing a deliberately chosen ID collided and the second was silently never charged. An internally minted `BillingNonce` is mixed into the key, making it unforgeable (#7303) +- **Abandoned Non-Streaming Request Hung Forever** - The worker kept a `ctx.Done()` arm on an already-claimed delivery send, a regression from the #6972 delivery fix, so a non-streaming caller could hang indefinitely (#7313) +- **Repeated Empty Thinking Blocks** - Streaming chat chunks emitted empty reasoning/message fields on every content delta, which clients rendered as repeated empty thinking blocks (#7318) +- **Claude Code Thread Continuations Failed Behind Key Rotation** - Claude Code's server-side conversation threads are bound to the upstream account that creates them, and Bifrost's per-request key selection, retries and fallbacks cannot keep a continuation on that account, so `thread: {"type": "continue"}` requests failed with `thread_not_found` (about half the time on a two-key config). The Anthropic integration now declares itself stateless: continuations are refused before the provider call with a 400 whose `details.error_code` is `thread_unsupported_request`, which makes the client resend the turn in full and stop sending the thread field for the rest of the session, and the provider raw-body path strips `thread` from create requests so no orphaned thread state accumulates upstream. Token counting is never refused (#7274) +- **Gemini Flash-Lite Minimal Thinking Promoted to Low** - Normalized `gemini-3.1-flash-lite` requests silently promoted `minimal` thinking to `low` on both Gemini and Vertex; the text model's four supported levels are now registered, preserving the image variant's separate restrictions (#7288) (thanks [@Javtor](https://github.com/Javtor)!) +- **Config.json Virtual Key Limits Broke Under UI Edits** - VK rate limits and budgets created via the config.json standalone-limits flow and then edited through the UI produced duplicate, conflicting ownership records (standalone budgets owned directly by the VK, plus orphaned UUID rate-limit rows created by the UI). Migration `migrate_vk_standalone_limits_to_model_configs` consolidates ownership into VK-scoped model configs while preserving usage counters, and the write paths stop creating the divergent rows (#7291) +- **Bedrock Service Tier Rejections** - Service tier forwarding for Bedrock (Converse and Mantle paths) sent whatever tier was requested; it is now gated on explicit model capability metadata, failing closed when none exists, so models that do not support the requested tier stop rejecting the request (#7266) +- **Anthropic Root-Level Tool Schema Compositions** - Anthropic rejects `oneOf`/`anyOf`/`allOf` at the root of a tool's `input_schema` but accepts them inside properties; root-level compositions are now rewritten into a flat object schema before dispatch, unblocking tools like Codex's `automation_update` (#7265) +- **GenAI Streaming TTS** - Speech stream chunks routed through the `/genai` integration had no converter registered; `ToGeminiSpeechStreamResponse` now serves streaming TTS, and the streaming router returns a clean error instead of panicking when any stream converter is missing (#7248) +- **xAI Usage and Cost** - xAI reports visible and reasoning completion tokens separately; `completion_tokens` now folds reasoning in so `prompt_tokens + completion_tokens = total_tokens` holds, and streaming cost normalization preserves xAI's authoritative `cost_in_usd_ticks` to the terminal usage chunk instead of falling back to catalog pricing (#7245, #7250) +- **OpenAI-Only `search_content_types` Forwarded Everywhere** - `search_content_types` on `web_search` tools is an OpenAI-specific extension; it is now gated behind a per-provider capability check so Bedrock and other OpenAI-compatible backends receive a clean `web_search` tool without the field (#7244) +- **Claude Code `diagnostics` Field Rejected by Non-Native Providers** - Claude Code sends `diagnostics.previous_message_id` on every request; on the typed-sanitizer path Bedrock, Vertex and Azure returned 400 `diagnostics: Extra inputs are not permitted`. The field is stripped for providers that do not support it (#7243) +- **Unreadable Bedrock Error Logs** - AWS returns errors in a flat `{"message": ..., "__type": ...}` shape the shared Anthropic/OpenAI parsers never looked at, so Bedrock errors were logged with no human-readable reason; the root-level message now seeds `BifrostError`, and the logs UI falls back to showing the raw provider error body when no message could be extracted (#7221, #7222) +- **Anthropic `container` Param Dropped** - The string-form `container` param on `/anthropic/v1/messages` was silently dropped, so container reuse provisioned a fresh container every time; it is now carried through the round trip (#5829) (thanks [@AdityaPainuli](https://github.com/AdityaPainuli)!) +- **Responses API Finish Reason Missing From OTEL Spans** - `gen_ai.response.finish_reason` / `finish_reasons` are now emitted on the `llm.call` span for `/v1/responses` requests, so refusals are visible to OTEL consumers, matching the chat path (#7205) (thanks [@bkfl-notai](https://github.com/bkfl-notai)!) +- **Cohere Fallback Response Shape** - The Cohere-compatible route returned the raw Bifrost normalized response when a fallback served the request from a non-Cohere provider, which the Cohere SDK failed to parse; responses are now converted to the Cohere v2 shape (`message`, `finish_reason`) (#7198) +- **Skill Serving Race** - Fixed a race condition in the skill serving handler +- **Log and Dashboard Label Truncation** - Long model and provider labels in the logs table and dashboard charts truncate from the start so the distinctive suffix stays visible, the logs model column is wider, the search field border and icon spacing are cleaned up, and key picker options are keyed by id so duplicate labels stop highlighting together (#7215, #7216, #7217, #7269, #7297) + +## 🗄️ Database Migrations + +- **migrate_vk_standalone_limits_to_model_configs** - Consolidates virtual key standalone budgets and rate limits into VK-scoped model configs: standalone budgets are re-pointed to the VK's top-level model config (created if missing, usage preserved), orphaned duplicate UUID rate-limit rows created by the UI are deleted with the model config re-pointed to the canonical config.json row, and `vk.rate_limit_id` is cleared. + + +## 🐙 Closed GitHub Issues + +- [#123](https://github.com/maximhq/bifrost/issues/123) - Files API Support +- [#5707](https://github.com/maximhq/bifrost/issues/5707) - string-form `container` param on /anthropic/v1/messages is silently dropped - container reuse provisions a fresh container +- [#7204](https://github.com/maximhq/bifrost/issues/7204) - Responses API path never sets gen_ai.response.finish_reason in OTEL traces (refusals invisible to observability) +- [#7287](https://github.com/maximhq/bifrost/issues/7287) - Gemini 3.1 Flash-Lite minimal thinking is silently promoted to low +- [#7294](https://github.com/maximhq/bifrost/issues/7294) - Streaming chat chunks emit empty reasoning/message fields on every content delta, causing repeated thinking blocks +- [#7308](https://github.com/maximhq/bifrost/issues/7308) - non-streaming caller hangs forever - worker keeps a ctx.Done() arm on a claimed delivery send (regression from #6972) + + + +- fix: `POST /v1/realtime/client_secrets` mints a Bifrost-issued ephemeral token instead of returning the provider's token (e.g. OpenAI `ek_...`) verbatim, which let a client connect to the provider directly and bypass governance, metering and attribution; virtual key credentials are validated before a realtime session is admitted, so a nonexistent VK can no longer open an upstream provider session (#7317) +- fix: streaming chat chunks no longer emit empty reasoning/message fields on every content delta, which made clients render repeated empty thinking blocks (#7318) +- fix: an abandoned non-streaming request no longer hangs its caller forever; the worker kept a `ctx.Done()` arm on an already-claimed delivery send, a regression from the #6972 delivery fix (#7313) +- fix: caller-supplied `s3://` file IDs and `s3_bucket` params on the Bedrock provider are validated with a DNS-compatible bucket regex before URL construction, closing an SSRF where a bucket like `127.0.0.1:PORT#` steered the SigV4-signed request to a caller-chosen host (#7315) +- fix: resource IDs (batch, file, video, container, response IDs and cached content names) are validated and escaped through a central `EscapeResourceID` helper before being interpolated into outbound URL paths, so IDs carrying `../`, `?`, `#`, percent-encoded bytes or control characters can no longer redirect requests to unintended upstream endpoints (#7307) +- fix: an internally minted `BillingNonce` is mixed into the billing-idempotency key, which was previously `(RequestID, AttemptNumber)` alone; since `x-request-id` is caller-supplied, two unrelated requests sharing a forged ID collided on the key and the second was never charged (#7303) +- fix: Bedrock service tier forwarding is gated on explicit model capability metadata (failing closed when none exists) on both Converse and Mantle paths, instead of sending whatever tier was requested and getting rejected by models that do not support it (#7266) +- fix: root-level `oneOf`/`anyOf`/`allOf` in a custom tool's `input_schema` are normalized into a flat object schema before reaching Anthropic, which rejects composition keywords at the schema root but accepts them inside properties; unblocks tools like Codex's `automation_update` (#7265) +- fix: speech stream chunks have a GenAI converter (`ToGeminiSpeechStreamResponse`), enabling streaming TTS through the `/genai` integration, and the streaming router returns a clean error instead of panicking when a stream converter is missing (#7248) +- feat: `SupportsPromptCacheBreakpoints` model cap with a name-based fallback for OpenRouter, so `prompt_cache_breakpoint` is only forwarded to models that accept it (#7261, #7262) +- fix: xAI `completion_tokens` folds `reasoning_tokens` into the total so `prompt_tokens + completion_tokens = total_tokens` holds, and streaming cost normalization preserves xAI's authoritative `cost_in_usd_ticks` through to the terminal usage chunk instead of falling back to catalog pricing (#7245, #7250) +- fix: `search_content_types` on `web_search` tools is an OpenAI-specific extension and is now gated behind a per-provider capability check, so Bedrock and other OpenAI-compatible backends receive a clean `web_search` tool without the field (#7244) +- fix: the `diagnostics` field Claude Code sends on every request is stripped for providers that reject it (Bedrock, Vertex, Azure return 400 `diagnostics: Extra inputs are not permitted` on the typed sanitizer path) (#7243) +- fix: AWS Bedrock's flat error shape `{"message": ..., "__type": ...}` seeds the root-level `message` into `BifrostError`, so Bedrock errors served through the shared Anthropic/OpenAI handlers are logged with a human-readable reason (#7222) +- [fix]: the string-form `container` param is carried through the /v1/messages round trip instead of being silently dropped, so container reuse stops provisioning a fresh container (#5829) (thanks [@AdityaPainuli](https://github.com/AdityaPainuli)!) +- fix: the Cohere-compatible route converts fallback responses from non-Cohere providers into the Cohere v2 shape (`message`, `finish_reason`) via `ToCohereChatResponse`, instead of returning the raw Bifrost normalized response the Cohere SDK cannot parse (#7198) +- fix: clearer error message in the Anthropic integration (#7264) +- [fix]: preserve Gemini 3.1 Flash-Lite minimal thinking on Gemini and Vertex [@Javtor](https://github.com/Javtor) +- fix: the Anthropic raw-body request path strips the server-side `thread` field before the wire on both messages and count_tokens (thread state is bound to the creating account and cannot survive Bifrost's per-request key selection, retries, or fallbacks; the HTTP transport refuses continuations outright), `AnthropicMessageErrorStruct` gains an optional `details` object with `error_code` so error envelopes can carry the machine-readable codes clients key their recovery on, and `LargePayloadMetadata` gains `ThreadType` so the refusal also works when large-payload mode skips body parsing (populated by the enterprise metadata extractor) +- [fix]: MCP tool schemas keep one property order across tool syncs [@dougcalobrisi](https://github.com/dougcalobrisi) +- feat: `WhiteList` and `BlackList` entries that start with `regex:` are RE2 patterns. `MatchEntry` compiles them once as a case-insensitive full match, `Validate` refuses a pattern that is empty, `*` or does not compile, and the list-models pipeline never surfaces a pattern as a model. Plain entries keep their exact, case-insensitive match +- fix: fasthttp-level stale-connection retries no longer multiply `max_retries`. `contextTransport.RoundTrip` reports a failure before response headers on a freshly dialed socket with `retry=false`, so `network.StaleConnectionRetryIfErr` only walks past pooled keep-alive sockets the upstream closed while idle; an upstream that closes a fresh connection without answering now costs exactly one attempt of Bifrost's own retry budget instead of up to four. The retry backoff also ends as soon as the request context is cancelled, so a worker is freed immediately instead of after up to `retry_backoff_max` (#7035) +- [fix]: Gemini provider - preserve inline image and audio data in chat completions responses [@Atharva-Kanherkar](https://github.com/Atharva-Kanherkar) +- [fix]: route opencode-zen Responses calls through /v1/chat/completions, which its upstream does not serve on /v1/responses [@miguelchico](https://github.com/miguelchico) +- feat: the namespace-tool names a provider reserves for its own server tools (`web`, `image_gen`, `browser`, `python` on Bedrock and Bedrock Mantle) are now read from the datasheet row `reserved_tool_namespaces` (new `ModelCapabilities` field and `ModelCaps.ReservedToolNamespaces`) for the base provider and canonical model; a non-empty row replaces the hardcoded list, an absent or empty row keeps it, so any OpenAI-wire provider can reserve names through a row without a code change +- fix: Codex >= 0.147 wraps its default tools in a namespace literally named `functions` (openai/codex#37022), which Bedrock Mantle reserves and rejects with `User-defined namespace 'functions' collides with an existing tool namespace`. Core now unwraps that namespace to top-level tools, unprefixed, for every provider before dispatch; Codex treats the bare name and the explicit `functions` namespace as the same tool, so no mapping back is needed; a description on the `functions` namespace is prepended to each hoisted member, as flattening does +- fix: Responses `namespace` tools are now flattened in core for every provider whose wire does not support the type (Anthropic, Gemini, Vertex, Bedrock Converse, DeepSeek, and every OpenAI-compatible third party), with nested functions renamed to `__` so two namespaces that share a function name no longer collide into an upstream `Tool names must be unique` 400; returned `function_call` items are mapped back to the bare `name` plus `namespace` (unary and streaming), prior-turn calls carrying `namespace` and `tool_choice` names are re-aliased to match, and a name that is still duplicated after flattening or a `tool_choice` that matches several namespaces is rejected with a clear 400 before reaching the provider. Bedrock answers namespace support from its own surface resolver via the new optional `ResponsesNamespaceToolProvider` interface. A datasheet row `supports_namespace_tools` (new `ModelCapabilities` field and `ModelCaps.SupportsNamespaceTools`) overrides the per-provider default for a (provider, model) pair; with no row the default applies. A row can only narrow within what the wire can carry: the Anthropic Messages API (Anthropic, Claude on Azure or Bedrock Mantle), the Gemini API (Gemini, Vertex) and Bedrock Converse have no namespace container, so they answer false regardless of the row and always flatten. Flattened names honour the target wire's documented tool-name limit, 64 characters of `[A-Za-z0-9_-]` for OpenAI-compatible wires, Bedrock Converse and Fireworks, 128 for Anthropic, and 128 with `.` and `:` allowed for Gemini and Vertex, overridable per model through the datasheet row `tool_name_max_length` (a row below 10 cannot hold the hashed form and is ignored); a longer alias becomes an 8-hex hash prefix plus the function name, deterministically, so history and `tool_choice` re-alias to the same string. The alias map travels on the prepared request (`BifrostResponsesRequest.NamespaceToolAliases`) and is applied to that attempt's response, unary and streaming; nothing is kept on the request context or in process-wide state (#7048) +- fix: drop namespace tools whose name Amazon Bedrock reserves (`web`, `image_gen`, `browser`, `python`) on the Bedrock and Bedrock Mantle Responses paths instead of forwarding them into a `tools.namespace` collision 400; a dropped Codex `web` namespace becomes the hosted `web_search` tool on Bedrock Mantle +- fix: Bedrock Mantle chat streaming no longer drops the usage-only chunk that arrives after `finish_reason`; `ProviderSendsDoneMarker` now treats `bedrock_mantle` (and the legacy Mantle route under the `bedrock` key) as sending `[DONE]`, so streamed usage and cost are recorded (#7065) +- fix: `default_request_timeout_in_seconds` now bounds the wait for response headers on streaming requests, and cancelling a request now closes the upstream socket. Every fasthttp client is driven through a Bifrost-owned `RoundTripper` (`core/providers/utils/roundtripper.go`) that applies the client's read/write timeouts and the request context to the request write and the header wait, then lifts the socket deadline once headers are parsed so `stream_idle_timeout_in_seconds` remains the only bound on the body. `BuildStreamingClient` and `BuildLargeResponseClient` no longer zero the timeouts. An upstream that accepts the connection and never answers now fails with 504 `RequestTimedOut` after the request timeout and the request's fallbacks are used, instead of pinning the provider worker until the upstream closes the socket; `MakeRequestWithContext` no longer leaves a background `client.Do` running until `ReadTimeout` after its context is cancelled (#7034) +- fix: unary large-response downloads (`FinalizeResponseWithLargeDetection`) now bound every body read with `stream_idle_timeout_in_seconds` and close the upstream socket on request cancellation, mirroring the streamed passthrough path; a stall during the prefetch surfaces as 504 `RequestTimedOut` instead of pinning the provider worker, and a stall while the transport drains the `LargeResponseReader` fails the read with the idle-timeout error instead of blocking the writer indefinitely (#7104) +- fix: gzip-encoded unary responses are now classified for large-response mode by decompressed size; `Content-Length` describes the compressed bytes, so a body that was small on the wire but large once decompressed used to be materialized in full past `large_response_threshold` (#7104) +- fix: an OpenAI-compatible upstream that omits `[DONE]` and then goes silent after `finish_reason` no longer fails the stream when `stream_idle_timeout_in_seconds` fires. The chat and text completion read loops now treat an idle timeout after a terminal signal as a parked upstream, mark the stream parked so the connection is abandoned rather than drained, and synthesize the final chunk with the buffered `finish_reason`; a stall before `finish_reason` still surfaces as the idle-timeout error (#7108) +- fix: a streamed upstream that drops the connection mid-body is again reported as the retryable 502 `provider closed the stream before sending a completion marker` error instead of a generic `Error reading stream: unexpected EOF`. The Bifrost round tripper's chunked decoder surfaced the drop as `io.ErrUnexpectedEOF`, which no provider read loop treats as end of stream; it now reports the plain `io.EOF` fasthttp always did and discards the half-read connection (#7104 follow-up) +- fix: fallbacks for image edit, image variation and video edit requests now reach the configured fallback provider and model. `prepareFallbackRequest` had no arm for those three types, so the shallow request copy kept the primary's sub-request pointer and the "fallback" attempt was routed back to the primary while `RoutingInfo`, the `x-bifrost-routing-info-*` headers, the log row and the `fallback_index` metric label reported it as a fallback. The helper now also verifies the prepared request targets the fallback provider and model and skips the fallback with a warning otherwise, so a future request type added without an arm fails loudly instead of silently re-running the primary (#6966) +- fix: route Bedrock Claude requests that carry a `compact_20260112` edit to InvokeModel / InvokeModelWithResponseStream with the native Anthropic Messages body, so server-side compaction works on `bedrock/` models, including keys that pin an inference-profile ARN. AWS documents compaction as unsupported on the Converse API, which previously received the edit and silently ignored it (#6825) +- feat: serve Anthropic tool search (`tool_search_tool_*`, `defer_loading`) on `bedrock/` Claude models by routing those requests to InvokeModel / InvokeModelWithResponseStream, the only Bedrock API AWS allows it on; CountTokens counts such requests with the same InvokeModel body via the `invokeModel` input +- fix: a `reasoning.context` value the target model does not accept is now dropped on the OpenAI and Azure Responses path, so `all_turns` on the original gpt-5 family incl. gpt-5-pro, gpt-5.1 to gpt-5.3, and the o-series runs under the model's own `current_turn` default instead of failing with `Unsupported value: 'all_turns' is not supported with the 'gpt-5-pro' model`; gpt-5.4, gpt-5.5 and gpt-5.6 keep it. The accepted values come from the datasheet row `supported_reasoning_contexts` (new `ModelCapabilities` field and `ModelCaps.SupportedReasoningContexts`), falling back to a name-based default of `auto` and `current_turn` everywhere plus `all_turns` on gpt-5.4+ +- fix: mid-conversation `role: "system"` messages are now inlined in place as `` user turns for every model family on every converter that has a top-level system field: Bedrock Converse (Responses and Chat Completions), Gemini Chat Completions (`systemInstruction`), and the Anthropic wire shape used by DeepSeek, Fireworks and SGL. Previously only Claude on Bedrock/Anthropic inlined; everything else hoisted each reminder into the top-level system block. Claude Code appends a trailing `` system reminder after every turn, so the hoisted block grew the front of the prompt each turn and prefix-based caches (Bedrock implicit caching for `global.openai.gpt-5.6-luna` on Converse, Gemini implicit caching, DeepSeek context caching) reported a full cache write and zero cache reads on every turn +- [fix]: vLLM key selection resolves the requested alias through `key.Aliases` per key before comparing it with `vllm_key_config.model_name`, so a valid key is no longer rejected when the request used an alias; the same public alias can map to different physical model IDs across vLLM instances, and allow and block checks keep matching the original user-facing alias [@Constantine3](https://github.com/Constantine3) +- [fix]: Gemini Responses requests keep `top_k`, `frequency_penalty`, `presence_penalty`, `stop_sequences` and `media_resolution` across retries and fallbacks. `convertParamsToGenerationConfigResponses` deleted those keys from `ExtraParams` while mapping them into `generationConfig`, and the conversion runs once per attempt on the same request, so every attempt after the first reached the provider without them; the outbound `ExtraParams` is now built through a filtered copy instead of mutating the request [@VictorRequenaMaisa](https://github.com/VictorRequenaMaisa) +- feat: `wait_for_usage` on `custom_provider_config` keeps the OpenAI-compatible chat and text completion read loops open past `finish_reason` until the trailing usage-only frame arrives, so a custom provider that also sets `does_not_send_done_marker` no longer synthesizes a zero-usage terminal chunk and records the request at zero tokens and zero cost. `WaitForUsage` is carried on `BifrostContextKeyWaitForUsage` with the same set-or-clear discipline as `DoesNotSendDoneMarker`, so it never leaks onto a fallback provider that did not declare it, and termination stays bounded by the usage chunk, two consecutive post-finish heartbeat comments, EOF, or `stream_idle_timeout_in_seconds` (#7143) +- fix: `ExtraFields.RawResponse` now carries role-only, finish-only and usage-only SSE frames on the OpenAI-compatible chat stream. Those frames never entered the chunk-forwarding branch, so their bytes were discarded from the reconstructed `raw_response`, which made the captured audit trail irreconcilable against a provider invoice because the usage frame carries the token counts Bifrost bills from. A `pendingRawFrames` buffer accumulates them and drains onto the next forwarded chunk or the synthetic terminal chunk; the Responses-over-Chat fallback no longer stamps one upstream frame onto every derived event, and `delta.refusal` and `delta.annotations` are added to the forwarding predicate so refusals and streamed URL citations are no longer dropped entirely (#7144) +- fix: Anthropic server-side tool search survives the Bedrock-native invoke ingress end to end. `tool_search_tool_*` was dropped during the Converse-shaped intermediate conversion, leaving the egress predicate blind to the tool and routing the request to Converse where server-side tool search is unavailable; the tool is now carried as an ingress-only marker (`json:"-"`, never reaching a Converse body) and the neutral tool is rebuilt in `ToBifrostResponsesRequest` along with `defer_loading`. Results are returned as a `server_tool_use` plus `tool_search_tool_result` pair rather than a client `tool_use`, which Anthropic rejects when the caller echoes a `tool_result` for the `srvtoolu_` ID on the next turn, on both the unary and streaming paths; `BedrockContentBlock.UnmarshalJSON` gained `server_tool_use` and `tool_search_tool_result` cases so replayed turns round-trip unchanged instead of falling through to empty structs; and the Anthropic-native response path carries the same blocks (#7155) +- fix: native Gemini `generateContent` image requests routed through `/genai` no longer return HTTP 200 while silently ignoring part of the request. `isImageEditRequest` only inspected `contents[0].parts[0]`, so a request with the prompt text before the `inlineData` part, the ordering in Google's own REST edit sample, was misclassified as image generation: the image never reached Vertex and the model invented a picture from the prompt alone. Detection now scans every part across every content. `imageConfig.aspectRatio` is preserved as a typed `aspect_ratio` param that both outbound converters prefer, instead of being folded into a `WxH` size string that collapsed any ratio outside `1:1`, `3:4`, `4:3`, `9:16` and `16:9` back to square (#7173) +- fix: Gemini `Part` gained `mediaResolution`, the per-part override for `generationConfig.mediaResolution`. `Part.UnmarshalJSON` decodes into a closed alias, so the key was discarded before any conversion ran and per-part image and PDF tokenization fell back to the model default; an `ULTRA_HIGH` image billed about 21k prompt tokens through `/genai` instead of about 22.1k direct. Both the `mediaResolution` and `media_resolution` spellings are parsed, the value round-trips through `ResponsesMessageContentBlock` and is reconstructed on the outbound part, and it is stripped on the OpenAI wire path where the field is unknown (#7156) +- fix: image blocks inside a Bedrock Converse `toolResult` are hoisted out and re-inserted immediately after the last tool result in the same message, with a placeholder text block left behind so the now image-free result is not rejected for being empty. Some Bedrock-hosted models, notably the OpenAI and Grok families, reject nested tool-result images on Converse even though they accept images in tool output via the Responses API. `BedrockModelSupportsToolResultImages` supplies the name-based default and the datasheet field `supports_converse_tool_result_images` (new `ModelCapabilities` field and `ModelCaps.SupportsConverseToolResultImages`) overrides it per row (#7150) +- feat: `bifrost_error_requests_total` gains a normalized `error_type` label drawn from a closed, prefix-structured vocabulary (`caller_*`, `policy_*`, `provider_*`, `bifrost_*`, `_OTHER`), so a 429 from a governance rate limit is distinguishable from a 429 from an upstream provider, a 403 from a blocked model from one from a bad key, and a 503 Bifrost shed under queue pressure from an upstream overload. `ClassifyErrorType` resolves a declared `ExtraFields.ErrorType` first, then Bifrost's own string markers, then the upstream status code; it deliberately ignores the provider's own `error.type` and `error.code` strings, which disagree across providers for the same condition. Values are prefixed by fault domain so an alarm expression can match a family with a single regex (#7141) +- fix: Azure Foundry deployments of Fireworks-hosted models are no longer silently capped at 4096 output tokens on `/openai/v1/responses`, where Microsoft routes those models through chat completions internally. The Azure provider checks the model's datasheet `supported_endpoints` and, when `/v1/responses` is absent, transparently serves both `Responses` and `ResponsesStream` through `/openai/v1/chat/completions` and converts the result back to the Responses shape; a model with an explicit `/v1/responses` entry or no datasheet row at all is unchanged. Separately, `ToAnthropicResponsesResponse` now follows `StopReason > IncompleteDetails > tool_use inference > end_turn`, so a turn truncated by the output-token cap on any OpenAI-shaped Responses provider reports `stop_reason: max_tokens` instead of hiding the truncation as `end_turn` (#6782) +- fix: DeepSeek chat requests move `max_completion_tokens` into `max_tokens` on the wire. DeepSeek's chat-completions endpoint only recognizes the legacy field and silently ignores `max_completion_tokens`, so the limit had no effect; this matches the behaviour already in place for Opencode and Ollama (#7131) +- feat: guardrail redaction gains an identity-based transformation path for provider-native request and response bodies, so exact provider-managed rewrites such as Model Armor and Bedrock land in the correct native JSON field even when the same text value appears in several fields. `TextTargetID`, `TextTargetIDForIndex` and `TextRewrite` identify each guardrail-visible field by a stable ID rather than by content value; `RawRequestBodyTextTransformer` and `RawResponseTextTransformer` let integrations register handlers on the request context; and `rewriteRawJSONTextTargets` validates target uniqueness, verifies each `Original` value before patching and re-reads the written value after, so stale or mismatched normalized text cannot silently rewrite the wrong field (#7049) +- feat: guardrail redaction covers LLM tool-call arguments, Chat function arguments, Responses function arguments and custom-tool input, across the Anthropic streaming and non-streaming paths. `input_json_delta` events are inspected and rewritten alongside `text_delta`, reading and writing `delta.partial_json`; `collectAnthropicArgumentStringPaths` recursively collects string-valued paths inside a `tool_use` block's `input` without touching tool names, IDs or definitions; `BifrostResponsesStreamResponse` gained `Input` to carry the full custom-tool payload on `custom_tool_call_input.done`; and `DeepCopyResponsesMessage` deep-copies `ResponsesCustomToolCall` so a copied message cannot mutate the original (#6977) +- fix: an abandoned non-streaming request, one whose caller context was already cancelled by the time the upstream finished, is billed and logged deterministically instead of about half the time. The worker's delivery `select` had two simultaneously ready cases, a send into a cap-1 channel and `ctx.Done()`, and Go picks uniformly among ready cases, so the terminal post-hooks that record billing and finalize the log row were skipped roughly 50% of the time. `requestWorker` now checks `req.Context.Err()` before the select on both the error and success paths and calls `billAbandonedTerminal` directly, keeping the 5-second timer guard inside the select for a caller that leaves between the check and the send (#6972) +- fix: provider response headers are filtered by classifier, not only by a fixed map of 28 exact names, so a credential-named header outside that map is no longer re-served to the inference caller in the HTTP response and `extra_fields.provider_response_headers`. The extractors now also consult `schemas.IsSensitiveHeader`, which matches credential names by substring and suffix and already recognized `cf-access-*` and `x-amzn-oidc-*`; a fixed list cannot enumerate the space when `network_config.extra_headers` exists to carry custom authentication headers and some upstreams echo request headers back (#7120) [@Atharva-Kanherkar](https://github.com/Atharva-Kanherkar) +- [fix]: a plugin that returns an incomplete `BifrostError` whose nested `Error` field is nil no longer panics the request worker. The fallback decision helper nil-checks `fallbackErr`, guards access to `Error.Type` and reads the message through the nil-safe `GetErrorString()` helper, continuing to the next fallback when allowed; behaviour for cancelled requests and `AllowFallbacks == false` is unchanged (#6967) [@Constantine3](https://github.com/Constantine3) +- [fix]: `clearCtxForFallback` now clears `BifrostContextKeyProviderResponseHeaders`. Providers set that key from their own HTTP response before the status check so error paths can forward it, and when a fallback attempt failed pre-flight, key selection failing for the fallback provider, a plugin short-circuiting it, or the queue retiring, nothing overwrote the key and the primary's headers survived onto the fallback's error response. A client could receive a response attributed to provider B carrying provider A's `Retry-After` and `x-ratelimit-remaining-*`, and wait according to a limit belonging to a provider that never served the request (#6973) [@Huang-404-Q](https://github.com/Huang-404-Q) +- [fix]: untitled document blocks get unique names on the Bedrock Converse path. Every untitled block was named the literal `document`, and Converse rejects duplicate document names, so any request carrying two or more untitled documents failed unconditionally with `ValidationException: Messages can't contain duplicate document names`. A per-request namer threaded through the content-block conversion disambiguates with numeric suffixes (`document`, `document-2`, `document-3`) and suffixes explicitly titled documents only on an actual collision; the Responses input replay path gets the same treatment scoped to its content-block list (#7003) [@Huang-404-Q](https://github.com/Huang-404-Q) +- [fix]: `BifrostResponsesStreamResponse.Item` is `omitempty`, so Responses stream events that carry no item payload no longer serialize `"item": null`. Strict OpenAI Responses clients reject those as invalid stream frames, which broke streamed `/v1/responses` usage entirely; `response.output_item.added` and `response.output_item.done` still emit the item object [@ReStranger](https://github.com/ReStranger) +- feat: `SecretVar.RedactedIfSecret()` returns a plain clone when the value is a literal and delegates to `Redacted()` when it is env-var or vault-backed, so regions and service URLs (Azure endpoint, Vertex/Bedrock/Bedrock Mantle region, vLLM/Ollama/SGL/Databricks URL, MCP connection string) are readable in the UI instead of being masked as if they were credentials. It always returns a fresh pointer, so the redacted copy never aliases the live config, and it replaces the ad-hoc `IsFromSecret()` guard blocks that tried to replicate this inline (#7085) +- feat: GA realtime transcription is served for OpenAI and Azure over both WebSocket and WebRTC with normal Bifrost authentication, routing, governance, guardrails, logging and transcription-aware pricing. Unlike a normal realtime session, which names its routing model in the connect URL, a GA transcription session carries only `intent=transcription` and delivers the model later in `session.update` under `audio.input.transcription.model` (or in the initial multipart `/v1/realtime/calls` request on WebRTC), so routing now resolves from the nested transcription model while realtime connection and turn semantics are preserved (#7089) +- fix: Claude Code replaying a prior assistant message through `POST /anthropic/v1/messages` at a Bedrock Mantle model is no longer rejected with hundreds of validation errors. The Bedrock-grouped ingress converter tagged user and system input text as `output_text`, though only `input_text` is valid on input messages, and omitted the required `status` on replayed assistant output messages, so Mantle's strict OpenAI-compatible validator refused the whole request. Replayed assistant messages now carry `Status: "completed"` and input text blocks are tagged `input_text`. Bedrock requests with no explicit `max_tokens` also populate it from the model's known capacity instead of truncating silently on large context windows (#7074) +- fix: Anthropic server tools such as `web_search_20250305` are stripped before a request reaches Fireworks, vLLM or SGLang on their Anthropic-compatible endpoints, which return 400 for tools executed by Anthropic-operated infrastructure that does not exist on third-party hosts; clients whose built-in web search is always on, such as Codex, hit this on every request. `ProviderFeatures` entries for `Fireworks`, `VLLM` and `SGL` declare all server-tool flags off so the existing validators strip them, `StripUnsupportedServerToolsFromRawBody` mirrors that on the raw-body passthrough path, the caller's function tools are kept, and the drops are reported on the response's `DroppedUnsupportedTools` instead of failing the call (#7090) +- fix: `guardrailConfig` reaches Bedrock's OpenAI-compatible `chat/completions` and `responses` surfaces, streaming and non-streaming, which apply guardrails through request headers rather than the body field Converse uses, so a configured guardrail was previously ignored there with no error. `withGuardrailHeaders` maps `guardrailIdentifier`, `guardrailVersion` and optional `trace` to `X-Amzn-Bedrock-GuardrailIdentifier`, `X-Amzn-Bedrock-GuardrailVersion` and `X-Amzn-Bedrock-Trace`, then deletes the key from `ExtraParams` so it is not also emitted into the body. A half-formed config with only one of identifier or version is left untouched rather than sent, since both are required upstream, and the base header map is cloned rather than mutated (#7095) +- feat: `use_openai_endpoints` on Bedrock keys and aliases routes chat completions and responses through Bedrock's OpenAI-compatible `/openai/v1` endpoints instead of Converse, for models that support them, mirroring the existing `use_anthropic_endpoints` pattern. It is opt-in by design: Converse carries Bedrock Guardrails, `performanceConfig` and `requestMetadata` that the OpenAI-compatible surface silently ignores, so diverting automatically could stop a guardrail from being enforced with no visible error. `ResolveUseOpenAIEndpoints` gives the alias value precedence over the key, and the narrow `runtimeServesResponses` check is replaced by a general `runtimeServesOpenAIAPI` that takes a `BedrockAPI` discriminator; chat completions, streaming and non-streaming, gained the runtime path that previously existed only for Responses (#7073) +- fix: Bedrock runtime models that serve the Responses API are routed to it through a dedicated surface resolver rather than falling back to Converse (#7071) +- fix: Bedrock Converse document blocks always ship their content base64-encoded through `source.bytes`, never through `source.text`, which Converse rejects unless citations are explicitly enabled. Plain text formats (`text/plain`, `text/markdown`, `text/csv`, `text/html`) previously took the text branch and failed with `DocumentSource must set one of the following keys: bytes, s3Location`; the branch is removed for data URLs, percent-encoded payloads and the non-data-URL `file_data` path alike (#7072) +- fix: the Bedrock Mantle base path is resolved from the model capabilities datasheet instead of hard-coded string matching in two packages. Mantle serves each model on exactly one of `v1` or `openai/v1` and returns a 400 on the other, and the previous matching covered only generations up to GPT-5 and Gemma 4, so GPT-6 and any future closed-generation model silently fell through to the wrong path. `ResolveBedrockMantleBasePath` applies family-name detection as a fallback and defers to the new `BedrockMantleBasePath` field on `ModelCapabilities`, so a new generation needs a datasheet row rather than a code change (#7077) +- fix: `image_generation_call` items where OpenAI emits `action` as a bare JSON string, such as `"generate"`, decode correctly. `ResponsesToolMessageActionStruct.UnmarshalJSON` immediately peeked at a `.type` field, which cannot be read from a JSON string, so the whole decode failed with `failed to peek at type field`; that silently dropped the `response.output_item.done` and `response.completed` events carrying the image, leaving the stream without a terminal event and surfacing as a bogus `provider closed the stream` truncation error. The action is now tried as a bare string before falling back to the object type-peek, and `ResponsesImageGenerationCall` keeps the `background`, `output_format`, `quality`, `revised_prompt` and `size` settings OpenAI echoes back on completed items (#7060) + + + +- fix: virtual key rate limits and budgets created via the config.json standalone-limits flow and later edited through the UI no longer produce duplicate, conflicting ownership records; migration `migrate_vk_standalone_limits_to_model_configs` consolidates ownership into VK-scoped model configs, preserving usage counters, and the write paths stop creating the divergent rows (#7291) +- feat: virtual key listing batch-resolves assignees (`assigned_user`) instead of resolving per row, and VK search matches team and customer names in addition to key fields (#7225) +- chore: OTEL dependency updates (#7311) +- fix: Responses API requests emit `gen_ai.response.finish_reason` / `gen_ai.response.finish_reasons` on the OTEL `llm.call` span. `PopulateResponsesResponseAttributes` never copied the top-level `stop_reason` into the span attributes, so refusals on `/v1/responses` were invisible to OTEL consumers even though the stop reason reached the Logs DB; it now emits `finish_reasons` like the chat path and the tracer derives the singular key from it (#7204) +- feat: model allow and block lists accept `regex:` entries. The provider-key aggregate keeps an exact name next to a pattern that also covers it, the catalog allow check tries a pattern against the bare name and `provider/model`, and listings skip pattern entries +- fix: ClickHouse log store deletes no longer run as heavyweight `ALTER TABLE ... DELETE` mutations. The retention cleaner issued one such mutation per 100 rows, each rewriting the whole current-month part, and the once-a-minute stale-`processing` sweeps issued one per table unconditionally, filling replica disks in minutes. Every delete on the ClickHouse store (retention sweep, `Flush`/`FlushMCPToolLogs`, UI log deletes, async job and webhook delivery expiry) is now a single lightweight `DELETE FROM ... WHERE` per run, skipped entirely when nothing matches. The table TTL derived from `logs_store.retention_days` is now reconciled on every startup with `MODIFY TTL` (metadata only), so changing the value reaches existing tables; `0` leaves an existing TTL untouched (#7098) +- feat: access profiles and governance projects reference Virtual MCPs through `virtual_mcp_name`, so config files are portable across environments instead of carrying database-assigned integer IDs. Names resolve to stored records on startup and a name that matches nothing is refused; `virtual_mcp_id` is deprecated, still accepted, and wins when both are set. `mcp_configs` (`{ mcp_client_id, tools_to_execute }`) replaces the `mcp_servers` and `mcp_tool_overrides` include-exclude model with a single allowlist, where `["*"]` grants all tools including future ones, `[]` grants none, and a named list grants only those; the old keys are deprecated, still accepted, and folded into the new shape at load time (#7181) +- feat: `MCPToolLog` records governance entity names beside their IDs, so MCP tool logs carry the same attribution shape the `logs` table has instead of rendering raw UUIDs in the dashboard. `user_name`, `team_name`, `customer_name` and `business_unit_name` stop being `gorm:"-"` transients and become storage; the multi-valued `team_ids`/`team_names`, `customer_ids`/`customer_names` and `business_unit_ids`/`business_unit_names` sets are stored as index-aligned JSON arrays; and `budget_ids` and `rate_limit_ids` are recorded id-only, as in `logs`. Names are written from the request context at ingestion through the new `MCPToolLog.ApplyGovernance` in `framework/logstore/governance.go`, with nothing resolved on read. Added by migration `mcp_tool_logs_add_governance_snapshots` (#7154) +- feat: endpoint-attributed MCP observations are carried in the standard logging pipeline, so inspected MCP tool calls are logged with bounded identity (device, app key, server label, tool name, decision) sourced from the gateway rather than from payload-supplied headers. `SetMCPObservation` attaches the attribution to a `BifrostContext` and `applyMCPObservation` writes it onto the `MCPToolLog` from both `PreMCPHook` and `PostMCPHook`, snapshotting the observation so it cannot alias across async log entries (#6959) +- feat: the `error_type` classification vocabulary is threaded through the framework so `bifrost_error_requests_total` can carry a normalized fault-domain label alongside `status_code` (#7141) +- feat: model pricing supports time-of-day peak and off-peak rates. `TableModelPricing` gains `off_peak_cost_multiplier` (a nullable float) and `peak_hours` (a JSON-serialized `PeakHoursSchedule` of recurring weekly windows using IANA timezone names, weekday numbers and half-open `HH:MM` intervals that may wrap past midnight), added by migration `add_time_of_day_pricing_columns` and aliased into the datasheet package so the JSON shape stays self-contained. The cost engine evaluates the schedule against the request start time and scales usage-based charges by the multiplier when the request falls outside every peak window, applied once in `computeCostFromInput` so every modality is covered; flat `CostPerRequest` and `SearchQueriesCost` fees are excluded, as is `AdditionalCost`, which is discounted independently through its own pricing rows. Both fields are exposed on `PricingPatch` in the OpenAPI and governance schemas, with `off_peak_cost_multiplier` bounded to `(0, 1]` (#6574, #6575, #6576) +- feat: `SecretVar.RedactedIfSecret()` is used for non-credential fields in `ProviderConfig.Redacted()`, `Config.GetAllKeys()` and `Config.RedactMCPClientConfig()`, so regions, endpoints, service URLs and MCP connection strings surface as plaintext while anything env-var or vault-backed stays masked (#7085) +- feat: GA realtime transcription sessions are routed, governed, logged and priced through the normal framework pipeline, resolving the routing model from the nested `audio.input.transcription.model` that arrives in `session.update` rather than from the connect URL (#7089) +- feat: a `use_openai_endpoints` column on the provider keys table, added by migration `add_use_openai_endpoints_column`, opts a Bedrock key or alias into Bedrock's OpenAI-compatible endpoints instead of Converse (#7073) +- feat: the reserved tool-namespace list a provider keeps for its own server tools is read from the datasheet row `reserved_tool_namespaces`, so a namespace collision check no longer requires a code change (#7084) +- fix: `exchangeRefreshToken` includes `client_secret` only when the secret is non-empty, matching `exchangeCodeForTokensWithPKCE`. Public OAuth2 clients registered against servers that support only `token_endpoint_auth_method: none` have no secret, and unconditionally setting `client_secret=` sent an empty `client_secret_post` attempt that strict authorization servers answered with `invalid_client`, flipping the token row to `needs_reauth` even though the refresh token was valid (#7042) +- fix: the virtual key `allowed_models: ["*"]` handling for governance added in #6767 is reverted, returning the wildcard-with-empty-synced-catalog case to its previous behaviour (#7053) + + + +- chore: upgraded core to v1.9.1 and framework to v1.7.1 +- fix: removed the namespace-tool flattening that ran under `should_convert_params`; it spliced nested functions into the top-level list without a namespace prefix, so two namespaces sharing a function name produced duplicate tool names and an upstream 400. Flattening now lives in Bifrost core for every provider whose wire lacks the `namespace` type, with unique `__` names and response-side mapping back. `should_convert_params` is still accepted so existing configs load, but it no longer changes any request (#7048) + + + +- fix: the billing-idempotency key mixes in the internally minted `BillingNonce`, so caller-forged `x-request-id` duplicates can no longer suppress charges (#7303) +- fix: virtual key credentials are validated before realtime session admission (#7317) +- chore: upgraded core to v1.9.1 and framework to v1.7.1 +- feat: MCP tool logs record governance entity names beside their IDs, so the plugin stamps `user_name`, `team_name`, `customer_name`, `business_unit_name` and the multi-valued team, customer and business-unit sets onto the log entry at ingestion instead of leaving the dashboard to render raw UUIDs (#7154) +- feat: endpoint-attributed MCP inspections carry bounded identity (device, app key, server label, tool name, decision) from the gateway into the MCP authorization path, so an inspected tool call is attributed without trusting payload-supplied headers (#6959) +- feat: governance errors are classified into the normalized `error_type` vocabulary, so a 429 raised by a governance rate limit is distinguishable from an upstream 429 on `bifrost_error_requests_total` (#7141) +- feat: model allow and block lists accept `regex:` entries. The `*_patterns` fields added earlier in this release window were withdrawn in favour of the prefix form, which needs no new schema fields (#6988, #7133, #7134) +- fix: `UsageTracker.Cleanup()` cancels and waits for the periodic reset worker before taking its final budget and rate-limit snapshots. It previously dumped first, so `trackerCancel()` could cancel an in-flight rate-limit dump and fail with `failed to dump rate limits to database: context canceled`, and the final dump was not guaranteed to be the tracker's last database writer. A queued ticker event can no longer start another reset cycle during shutdown, the current cycle stops when the tracker context is cancelled, and `context.Canceled` is treated as an expected result only when that context was actually cancelled (#7099) [@Constantine3](https://github.com/Constantine3) + + + +- chore: upgraded core to v1.9.1 and framework to v1.7.1 + + + +- chore: upgraded core to v1.9.1 and framework to v1.7.1 +- feat: MCP tool log rows carry governance entity names alongside their IDs, written from the request context at ingestion, so nothing is resolved on read (#7154) +- feat: endpoint-attributed MCP observations are written onto both the pending and the final MCP tool log entry, with the observation snapshotted so it cannot alias across async entries (#6959) +- feat: cost recalculation honours time-of-day peak and off-peak pricing, scaling usage-based charges by `off_peak_cost_multiplier` when a request falls outside every declared peak window (#6575) +- feat: GA realtime transcription sessions are logged and priced through the standard pipeline with transcription-aware pricing (#7089) +- fix: a request whose caller disconnected before the upstream finished is logged and finalized deterministically rather than roughly half the time, so abandoned requests no longer leave a log row stuck in its pending state (#6972) +- docs: clarified that `CountRecalcTargets` reads a materialized view that can lag, so its `Total` is an approximation rather than an exact count (#7078) + + + +- chore: upgraded core to v1.9.1 and framework to v1.7.1 + + + +- chore: upgraded core to v1.9.1 and framework to v1.7.1 + + + +- chore: upgraded core to v1.9.1 and framework to v1.7.1 + + + +- chore: upgraded core to v1.9.1 and framework to v1.7.1 + + + +- chore: upgraded core to v1.9.1 and framework to v1.7.1 + + + +- chore: upgraded core to v1.9.1 and framework to v1.7.1 +- fix: a continuation turn, such as a tool result following a prior user message, is classified instead of skipped when no active session state is found. `BuildInputWithDisposition` now returns the populated `ComplexityInput` including `LastUserText` for trailing-continuation turns in both the chat and responses paths, and `computeComplexity` falls back to classifying that recovered text, so new or recovered sessions get a tier assignment; the skip path applies only when `LastUserText` is also empty (#7122) + + + +- chore: upgraded core to v1.9.1 and framework to v1.7.1 + + + +- feat: metrics export includes user id and user name labels, so per-user attribution is available in Prometheus (#7267) +- chore: upgraded core to v1.9.1 and framework to v1.7.1 +- feat: `bifrost_error_requests_total` carries a normalized `error_type` label alongside `status_code`, drawn from a closed prefix-structured vocabulary (`caller_*`, `policy_*`, `provider_*`, `bifrost_*`, `_OTHER`), so alarm expressions can separate fault domains with a single regex instead of enumerating status codes (#7141) + + diff --git a/docs/docs.json b/docs/docs.json index 790f5aa40ca..c4c8f7e058c 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -60,7 +60,9 @@ { "group": "Overview", "icon": "book-open-cover", - "pages": ["overview"] + "pages": [ + "overview" + ] }, { "group": "Quick Start", @@ -100,22 +102,31 @@ { "group": "Release Cadence", "icon": "calendar", - "pages": ["release-cadence"] + "pages": [ + "release-cadence" + ] }, { "group": "Migration Guides", "icon": "arrow-up-right-dots", - "pages": ["migration-guides/v2.0.0", "migration-guides/v1.5.0"] + "pages": [ + "migration-guides/v2.0.0", + "migration-guides/v1.5.0" + ] }, { "group": "Migrate from other Gateway", "icon": "arrow-up", - "pages": ["migration-guides/litellm"] + "pages": [ + "migration-guides/litellm" + ] }, { "group": "SDK Integrations", "icon": "plug", - "pages": ["integrations/what-is-an-integration"] + "pages": [ + "integrations/what-is-an-integration" + ] }, { "group": "Providers & Guides", @@ -211,7 +222,10 @@ { "group": "Writing Plugins", "icon": "code", - "pages": ["plugins/writing-go-plugin", "plugins/writing-wasm-plugin"] + "pages": [ + "plugins/writing-go-plugin", + "plugins/writing-wasm-plugin" + ] }, "plugins/migration-guide" ] @@ -257,7 +271,10 @@ { "group": "Plugins", "icon": "puzzle-piece", - "pages": ["features/plugins/mocker", "features/plugins/jsonparser"] + "pages": [ + "features/plugins/mocker", + "features/plugins/jsonparser" + ] } ] } @@ -270,7 +287,9 @@ { "group": "Getting Started", "icon": "rocket", - "pages": ["enterprise/overview"] + "pages": [ + "enterprise/overview" + ] }, { "group": "Moving from OSS", @@ -286,12 +305,17 @@ { "group": "Release Cadence", "icon": "calendar", - "pages": ["enterprise/release-cadence"] + "pages": [ + "enterprise/release-cadence" + ] }, { "group": "Migration Guides", "icon": "arrow-up-right-dots", - "pages": ["enterprise/migration-guides/v2.0.0", "enterprise/migration-guides/v1.4.0"] + "pages": [ + "enterprise/migration-guides/v2.0.0", + "enterprise/migration-guides/v1.4.0" + ] }, { "group": "Features", @@ -354,7 +378,9 @@ { "group": "Security", "icon": "shield", - "pages": ["security"] + "pages": [ + "security" + ] } ] }, @@ -364,22 +390,35 @@ "groups": [ { "group": "Overview", - "pages": ["edge/overview", "edge/how-it-works"] + "pages": [ + "edge/overview", + "edge/how-it-works" + ] }, { "group": "Governance & Control", "icon": "shield-halved", - "pages": ["edge/app-governance", "edge/mcp-governance", "edge/security"] + "pages": [ + "edge/app-governance", + "edge/mcp-governance", + "edge/security" + ] }, { "group": "Admin controls", "icon": "sliders", - "pages": ["edge/admin-devices", "edge/admin-approvals", "edge/admin-configurations"] + "pages": [ + "edge/admin-devices", + "edge/admin-approvals", + "edge/admin-configurations" + ] }, { "group": "Deployment", "icon": "cloud-arrow-up", - "pages": ["edge/deployment-mdm"] + "pages": [ + "edge/deployment-mdm" + ] }, { "group": "Onboarding Instructions", @@ -396,7 +435,9 @@ { "group": "Reference", "icon": "list", - "pages": ["edge/supported-applications"] + "pages": [ + "edge/supported-applications" + ] } ] }, @@ -407,7 +448,10 @@ { "group": "Runbooks", "icon": "book-open", - "pages": ["runbooks/claude-code-bedrock", "runbooks/codex-bedrock"] + "pages": [ + "runbooks/claude-code-bedrock", + "runbooks/codex-bedrock" + ] }, { "group": "CLI Agents & Editors", @@ -460,7 +504,9 @@ { "group": "GenAI SDK", "icon": "diamond", - "pages": ["integrations/genai-sdk/overview"] + "pages": [ + "integrations/genai-sdk/overview" + ] }, "integrations/litellm-sdk", "integrations/langchain-sdk", @@ -476,37 +522,50 @@ "group": "Okta", "icon": "/media/okta-icon.svg", "collapsed": true, - "pages": ["enterprise/setting-up-okta/oidc", "enterprise/setting-up-okta/scim"] + "pages": [ + "enterprise/setting-up-okta/oidc", + "enterprise/setting-up-okta/scim" + ] }, { "group": "Microsoft Entra", "icon": "microsoft", "collapsed": true, - "pages": ["enterprise/setting-up-entra/oidc"] + "pages": [ + "enterprise/setting-up-entra/oidc" + ] }, { "group": "Zitadel", "icon": "/media/zitadel-icon.svg", "collapsed": true, - "pages": ["enterprise/setting-up-zitadel/oidc"] + "pages": [ + "enterprise/setting-up-zitadel/oidc" + ] }, { "group": "Keycloak", "icon": "/media/keycloak-icon.svg", "collapsed": true, - "pages": ["enterprise/setting-up-keycloak/oidc"] + "pages": [ + "enterprise/setting-up-keycloak/oidc" + ] }, { "group": "Google Workspace", "icon": "google", "collapsed": true, - "pages": ["enterprise/setting-up-google-workspace"] + "pages": [ + "enterprise/setting-up-google-workspace" + ] }, { "group": "Auth0", "icon": "/media/auth0-icon.svg", "collapsed": true, - "pages": ["enterprise/setting-up-auth0/oidc"] + "pages": [ + "enterprise/setting-up-auth0/oidc" + ] }, { "group": "Generic OIDC", @@ -677,7 +736,9 @@ { "group": "Authentication", "icon": "key", - "pages": ["api/procuring-api-keys"] + "pages": [ + "api/procuring-api-keys" + ] }, { "group": "Inference", @@ -685,17 +746,23 @@ { "group": "Models", "expanded": false, - "pages": ["GET /v1/models"] + "pages": [ + "GET /v1/models" + ] }, { "group": "Chat Completions", "expanded": false, - "pages": ["POST /v1/chat/completions"] + "pages": [ + "POST /v1/chat/completions" + ] }, { "group": "Text Completions", "expanded": false, - "pages": ["POST /v1/completions"] + "pages": [ + "POST /v1/completions" + ] }, { "group": "Responses", @@ -712,22 +779,31 @@ { "group": "OCR", "expanded": false, - "pages": ["POST /v1/ocr"] + "pages": [ + "POST /v1/ocr" + ] }, { "group": "Rerank", "expanded": false, - "pages": ["POST /v1/rerank"] + "pages": [ + "POST /v1/rerank" + ] }, { "group": "Embeddings", "expanded": false, - "pages": ["POST /v1/embeddings"] + "pages": [ + "POST /v1/embeddings" + ] }, { "group": "Audio", "expanded": false, - "pages": ["POST /v1/audio/speech", "POST /v1/audio/transcriptions"] + "pages": [ + "POST /v1/audio/speech", + "POST /v1/audio/transcriptions" + ] }, { "group": "Images", @@ -753,12 +829,16 @@ { "group": "Count Tokens", "expanded": false, - "pages": ["POST /v1/responses/input_tokens"] + "pages": [ + "POST /v1/responses/input_tokens" + ] }, { "group": "Compaction", "expanded": false, - "pages": ["POST /v1/responses/compact"] + "pages": [ + "POST /v1/responses/compact" + ] }, { "group": "Batch", @@ -1078,7 +1158,9 @@ { "group": "Health", "expanded": false, - "pages": ["GET /health"] + "pages": [ + "GET /health" + ] }, { "group": "Configuration", @@ -1293,12 +1375,20 @@ { "group": "Vault", "expanded": false, - "pages": ["POST /api/vault/flush-cache"] + "pages": [ + "POST /api/vault/flush-cache" + ] }, { "group": "Infrastructure", "expanded": false, - "pages": ["GET /ws", "GET /mcp", "POST /mcp", "GET /metrics", "POST /inspect"] + "pages": [ + "GET /ws", + "GET /mcp", + "POST /mcp", + "GET /metrics", + "POST /inspect" + ] }, { "group": "Webhooks", @@ -1319,7 +1409,10 @@ { "group": "Notifications", "expanded": false, - "pages": ["GET /api/notifications", "POST /api/notifications"] + "pages": [ + "GET /api/notifications", + "POST /api/notifications" + ] } ] }, @@ -1490,7 +1583,10 @@ { "group": "Budgets & Rate Limits", "expanded": false, - "pages": ["GET /api/governance/budgets", "GET /api/governance/rate-limits"] + "pages": [ + "GET /api/governance/budgets", + "GET /api/governance/rate-limits" + ] }, { "group": "Model Configs", @@ -1578,7 +1674,9 @@ { "group": "Customers", "expanded": false, - "pages": ["GET /api/customers/{customer_id}/teams"] + "pages": [ + "GET /api/customers/{customer_id}/teams" + ] }, { "group": "RBAC", @@ -1726,6 +1824,7 @@ "item": "Open Source", "icon": "rocket", "pages": [ + "changelogs/v2.2.1", "changelogs/v2.2.0", "changelogs/v2.1.1", "changelogs/v2.1.0", @@ -1935,7 +2034,10 @@ }, { "group": "September 2025", - "pages": ["changelogs/v1.2.22", "changelogs/v1.2.21"] + "pages": [ + "changelogs/v1.2.22", + "changelogs/v1.2.21" + ] } ] }, @@ -2042,7 +2144,10 @@ "changelogs/edge-v0.6.0", { "group": "August 2026", - "pages": ["changelogs/edge-v0.5.6", "changelogs/edge-v0.4.0"] + "pages": [ + "changelogs/edge-v0.5.6", + "changelogs/edge-v0.4.0" + ] }, { "group": "July 2026", @@ -2061,7 +2166,10 @@ "changelogs/cli-v0.10.6", { "group": "May 2026", - "pages": ["changelogs/cli-v0.10.5", "changelogs/cli-v0.10.4"] + "pages": [ + "changelogs/cli-v0.10.5", + "changelogs/cli-v0.10.4" + ] }, { "group": "March 2026", @@ -2400,4 +2508,4 @@ "linkedin": "https://linkedin.com/company/maxim-ai" } } -} \ No newline at end of file +} diff --git a/framework/changelog.md b/framework/changelog.md index d10b09b694d..e69de29bb2d 100644 --- a/framework/changelog.md +++ b/framework/changelog.md @@ -1,17 +0,0 @@ -- fix: virtual key rate limits and budgets created via the config.json standalone-limits flow and later edited through the UI no longer produce duplicate, conflicting ownership records; migration `migrate_vk_standalone_limits_to_model_configs` consolidates ownership into VK-scoped model configs, preserving usage counters, and the write paths stop creating the divergent rows (#7291) -- feat: virtual key listing batch-resolves assignees (`assigned_user`) instead of resolving per row, and VK search matches team and customer names in addition to key fields (#7225) -- chore: OTEL dependency updates (#7311) -- fix: Responses API requests emit `gen_ai.response.finish_reason` / `gen_ai.response.finish_reasons` on the OTEL `llm.call` span. `PopulateResponsesResponseAttributes` never copied the top-level `stop_reason` into the span attributes, so refusals on `/v1/responses` were invisible to OTEL consumers even though the stop reason reached the Logs DB; it now emits `finish_reasons` like the chat path and the tracer derives the singular key from it (#7204) -- feat: model allow and block lists accept `regex:` entries. The provider-key aggregate keeps an exact name next to a pattern that also covers it, the catalog allow check tries a pattern against the bare name and `provider/model`, and listings skip pattern entries -- fix: ClickHouse log store deletes no longer run as heavyweight `ALTER TABLE ... DELETE` mutations. The retention cleaner issued one such mutation per 100 rows, each rewriting the whole current-month part, and the once-a-minute stale-`processing` sweeps issued one per table unconditionally, filling replica disks in minutes. Every delete on the ClickHouse store (retention sweep, `Flush`/`FlushMCPToolLogs`, UI log deletes, async job and webhook delivery expiry) is now a single lightweight `DELETE FROM ... WHERE` per run, skipped entirely when nothing matches. The table TTL derived from `logs_store.retention_days` is now reconciled on every startup with `MODIFY TTL` (metadata only), so changing the value reaches existing tables; `0` leaves an existing TTL untouched (#7098) -- feat: access profiles and governance projects reference Virtual MCPs through `virtual_mcp_name`, so config files are portable across environments instead of carrying database-assigned integer IDs. Names resolve to stored records on startup and a name that matches nothing is refused; `virtual_mcp_id` is deprecated, still accepted, and wins when both are set. `mcp_configs` (`{ mcp_client_id, tools_to_execute }`) replaces the `mcp_servers` and `mcp_tool_overrides` include-exclude model with a single allowlist, where `["*"]` grants all tools including future ones, `[]` grants none, and a named list grants only those; the old keys are deprecated, still accepted, and folded into the new shape at load time (#7181) -- feat: `MCPToolLog` records governance entity names beside their IDs, so MCP tool logs carry the same attribution shape the `logs` table has instead of rendering raw UUIDs in the dashboard. `user_name`, `team_name`, `customer_name` and `business_unit_name` stop being `gorm:"-"` transients and become storage; the multi-valued `team_ids`/`team_names`, `customer_ids`/`customer_names` and `business_unit_ids`/`business_unit_names` sets are stored as index-aligned JSON arrays; and `budget_ids` and `rate_limit_ids` are recorded id-only, as in `logs`. Names are written from the request context at ingestion through the new `MCPToolLog.ApplyGovernance` in `framework/logstore/governance.go`, with nothing resolved on read. Added by migration `mcp_tool_logs_add_governance_snapshots` (#7154) -- feat: endpoint-attributed MCP observations are carried in the standard logging pipeline, so inspected MCP tool calls are logged with bounded identity (device, app key, server label, tool name, decision) sourced from the gateway rather than from payload-supplied headers. `SetMCPObservation` attaches the attribution to a `BifrostContext` and `applyMCPObservation` writes it onto the `MCPToolLog` from both `PreMCPHook` and `PostMCPHook`, snapshotting the observation so it cannot alias across async log entries (#6959) -- feat: the `error_type` classification vocabulary is threaded through the framework so `bifrost_error_requests_total` can carry a normalized fault-domain label alongside `status_code` (#7141) -- feat: model pricing supports time-of-day peak and off-peak rates. `TableModelPricing` gains `off_peak_cost_multiplier` (a nullable float) and `peak_hours` (a JSON-serialized `PeakHoursSchedule` of recurring weekly windows using IANA timezone names, weekday numbers and half-open `HH:MM` intervals that may wrap past midnight), added by migration `add_time_of_day_pricing_columns` and aliased into the datasheet package so the JSON shape stays self-contained. The cost engine evaluates the schedule against the request start time and scales usage-based charges by the multiplier when the request falls outside every peak window, applied once in `computeCostFromInput` so every modality is covered; flat `CostPerRequest` and `SearchQueriesCost` fees are excluded, as is `AdditionalCost`, which is discounted independently through its own pricing rows. Both fields are exposed on `PricingPatch` in the OpenAPI and governance schemas, with `off_peak_cost_multiplier` bounded to `(0, 1]` (#6574, #6575, #6576) -- feat: `SecretVar.RedactedIfSecret()` is used for non-credential fields in `ProviderConfig.Redacted()`, `Config.GetAllKeys()` and `Config.RedactMCPClientConfig()`, so regions, endpoints, service URLs and MCP connection strings surface as plaintext while anything env-var or vault-backed stays masked (#7085) -- feat: GA realtime transcription sessions are routed, governed, logged and priced through the normal framework pipeline, resolving the routing model from the nested `audio.input.transcription.model` that arrives in `session.update` rather than from the connect URL (#7089) -- feat: a `use_openai_endpoints` column on the provider keys table, added by migration `add_use_openai_endpoints_column`, opts a Bedrock key or alias into Bedrock's OpenAI-compatible endpoints instead of Converse (#7073) -- feat: the reserved tool-namespace list a provider keeps for its own server tools is read from the datasheet row `reserved_tool_namespaces`, so a namespace collision check no longer requires a code change (#7084) -- fix: `exchangeRefreshToken` includes `client_secret` only when the secret is non-empty, matching `exchangeCodeForTokensWithPKCE`. Public OAuth2 clients registered against servers that support only `token_endpoint_auth_method: none` have no secret, and unconditionally setting `client_secret=` sent an empty `client_secret_post` attempt that strict authorization servers answered with `invalid_client`, flipping the token row to `needs_reauth` even though the refresh token was valid (#7042) -- fix: the virtual key `allowed_models: ["*"]` handling for governance added in #6767 is reverted, returning the wildcard-with-empty-synced-catalog case to its previous behaviour (#7053) diff --git a/plugins/compat/changelog.md b/plugins/compat/changelog.md index 8562a049d6a..e69de29bb2d 100644 --- a/plugins/compat/changelog.md +++ b/plugins/compat/changelog.md @@ -1,2 +0,0 @@ -- chore: upgraded core to v1.9.1 and framework to v1.7.1 -- fix: removed the namespace-tool flattening that ran under `should_convert_params`; it spliced nested functions into the top-level list without a namespace prefix, so two namespaces sharing a function name produced duplicate tool names and an upstream 400. Flattening now lives in Bifrost core for every provider whose wire lacks the `namespace` type, with unique `__` names and response-side mapping back. `should_convert_params` is still accepted so existing configs load, but it no longer changes any request (#7048) diff --git a/plugins/governance/changelog.md b/plugins/governance/changelog.md index 23cf818167f..e69de29bb2d 100644 --- a/plugins/governance/changelog.md +++ b/plugins/governance/changelog.md @@ -1,8 +0,0 @@ -- fix: the billing-idempotency key mixes in the internally minted `BillingNonce`, so caller-forged `x-request-id` duplicates can no longer suppress charges (#7303) -- fix: virtual key credentials are validated before realtime session admission (#7317) -- chore: upgraded core to v1.9.1 and framework to v1.7.1 -- feat: MCP tool logs record governance entity names beside their IDs, so the plugin stamps `user_name`, `team_name`, `customer_name`, `business_unit_name` and the multi-valued team, customer and business-unit sets onto the log entry at ingestion instead of leaving the dashboard to render raw UUIDs (#7154) -- feat: endpoint-attributed MCP inspections carry bounded identity (device, app key, server label, tool name, decision) from the gateway into the MCP authorization path, so an inspected tool call is attributed without trusting payload-supplied headers (#6959) -- feat: governance errors are classified into the normalized `error_type` vocabulary, so a 429 raised by a governance rate limit is distinguishable from an upstream 429 on `bifrost_error_requests_total` (#7141) -- feat: model allow and block lists accept `regex:` entries. The `*_patterns` fields added earlier in this release window were withdrawn in favour of the prefix form, which needs no new schema fields (#6988, #7133, #7134) -- fix: `UsageTracker.Cleanup()` cancels and waits for the periodic reset worker before taking its final budget and rate-limit snapshots. It previously dumped first, so `trackerCancel()` could cancel an in-flight rate-limit dump and fail with `failed to dump rate limits to database: context canceled`, and the final dump was not guaranteed to be the tracker's last database writer. A queued ticker event can no longer start another reset cycle during shutdown, the current cycle stops when the tracker context is cancelled, and `context.Canceled` is treated as an expected result only when that context was actually cancelled (#7099) [@Constantine3](https://github.com/Constantine3) diff --git a/plugins/jsonparser/changelog.md b/plugins/jsonparser/changelog.md index 967f83f479d..e69de29bb2d 100644 --- a/plugins/jsonparser/changelog.md +++ b/plugins/jsonparser/changelog.md @@ -1 +0,0 @@ -- chore: upgraded core to v1.9.1 and framework to v1.7.1 diff --git a/plugins/logging/changelog.md b/plugins/logging/changelog.md index bb341df26ff..e69de29bb2d 100644 --- a/plugins/logging/changelog.md +++ b/plugins/logging/changelog.md @@ -1,7 +0,0 @@ -- chore: upgraded core to v1.9.1 and framework to v1.7.1 -- feat: MCP tool log rows carry governance entity names alongside their IDs, written from the request context at ingestion, so nothing is resolved on read (#7154) -- feat: endpoint-attributed MCP observations are written onto both the pending and the final MCP tool log entry, with the observation snapshotted so it cannot alias across async entries (#6959) -- feat: cost recalculation honours time-of-day peak and off-peak pricing, scaling usage-based charges by `off_peak_cost_multiplier` when a request falls outside every declared peak window (#6575) -- feat: GA realtime transcription sessions are logged and priced through the standard pipeline with transcription-aware pricing (#7089) -- fix: a request whose caller disconnected before the upstream finished is logged and finalized deterministically rather than roughly half the time, so abandoned requests no longer leave a log row stuck in its pending state (#6972) -- docs: clarified that `CountRecalcTargets` reads a materialized view that can lag, so its `Total` is an approximation rather than an exact count (#7078) diff --git a/plugins/maxim/changelog.md b/plugins/maxim/changelog.md index 967f83f479d..e69de29bb2d 100644 --- a/plugins/maxim/changelog.md +++ b/plugins/maxim/changelog.md @@ -1 +0,0 @@ -- chore: upgraded core to v1.9.1 and framework to v1.7.1 diff --git a/plugins/mocker/changelog.md b/plugins/mocker/changelog.md index 967f83f479d..e69de29bb2d 100644 --- a/plugins/mocker/changelog.md +++ b/plugins/mocker/changelog.md @@ -1 +0,0 @@ -- chore: upgraded core to v1.9.1 and framework to v1.7.1 diff --git a/plugins/modelcatalogresolver/changelog.md b/plugins/modelcatalogresolver/changelog.md index 967f83f479d..e69de29bb2d 100644 --- a/plugins/modelcatalogresolver/changelog.md +++ b/plugins/modelcatalogresolver/changelog.md @@ -1 +0,0 @@ -- chore: upgraded core to v1.9.1 and framework to v1.7.1 diff --git a/plugins/otel/changelog.md b/plugins/otel/changelog.md index 967f83f479d..e69de29bb2d 100644 --- a/plugins/otel/changelog.md +++ b/plugins/otel/changelog.md @@ -1 +0,0 @@ -- chore: upgraded core to v1.9.1 and framework to v1.7.1 diff --git a/plugins/prompts/changelog.md b/plugins/prompts/changelog.md index 967f83f479d..e69de29bb2d 100644 --- a/plugins/prompts/changelog.md +++ b/plugins/prompts/changelog.md @@ -1 +0,0 @@ -- chore: upgraded core to v1.9.1 and framework to v1.7.1 diff --git a/plugins/routing/changelog.md b/plugins/routing/changelog.md index 0ad5e64376c..e69de29bb2d 100644 --- a/plugins/routing/changelog.md +++ b/plugins/routing/changelog.md @@ -1,2 +0,0 @@ -- chore: upgraded core to v1.9.1 and framework to v1.7.1 -- fix: a continuation turn, such as a tool result following a prior user message, is classified instead of skipped when no active session state is found. `BuildInputWithDisposition` now returns the populated `ComplexityInput` including `LastUserText` for trailing-continuation turns in both the chat and responses paths, and `computeComplexity` falls back to classifying that recovered text, so new or recovered sessions get a tier assignment; the skip path applies only when `LastUserText` is also empty (#7122) diff --git a/plugins/semanticcache/changelog.md b/plugins/semanticcache/changelog.md index 967f83f479d..e69de29bb2d 100644 --- a/plugins/semanticcache/changelog.md +++ b/plugins/semanticcache/changelog.md @@ -1 +0,0 @@ -- chore: upgraded core to v1.9.1 and framework to v1.7.1 diff --git a/plugins/telemetry/changelog.md b/plugins/telemetry/changelog.md index b5450fe9eef..e69de29bb2d 100644 --- a/plugins/telemetry/changelog.md +++ b/plugins/telemetry/changelog.md @@ -1,3 +0,0 @@ -- feat: metrics export includes user id and user name labels, so per-user attribution is available in Prometheus (#7267) -- chore: upgraded core to v1.9.1 and framework to v1.7.1 -- feat: `bifrost_error_requests_total` carries a normalized `error_type` label alongside `status_code`, drawn from a closed prefix-structured vocabulary (`caller_*`, `policy_*`, `provider_*`, `bifrost_*`, `_OTHER`), so alarm expressions can separate fault domains with a single regex instead of enumerating status codes (#7141) diff --git a/transports/changelog.md b/transports/changelog.md index 30cb4d1be76..e69de29bb2d 100644 --- a/transports/changelog.md +++ b/transports/changelog.md @@ -1,43 +0,0 @@ -## ✨ Features - -- **Virtual Key Assignees and Expanded Search** - The virtual key list resolves `assigned_user` for every row via a single batch lookup, and VK search matches team, customer and user names in addition to key fields, so keys can be found by who owns them (#7225, #7226) -- **User Attribution in Prometheus Metrics** - The telemetry plugin's metrics export includes user id and user name labels, so per-user usage and error rates can be graphed and alerted on directly (#7267) -- **Prompt Cache Breakpoints Capability** - A `SupportsPromptCacheBreakpoints` model cap with a name-based fallback for OpenRouter gates `prompt_cache_breakpoint` forwarding, so OpenRouter's non-Claude models stop rejecting requests that carry it (#7261, #7262) -- **MCP Observed Latency** - Native (observed) MCP tool-call latency from `observed_latency_ms` is surfaced in the logs duration column and detail sheet, kept clearly distinct from true execution time and never fabricating a synthetic start timestamp on the timeline (#7271) - -## 🐞 Fixed - -- **Resource ID Path Injection** - Caller-supplied resource IDs (batch, file, video, container, response IDs, cached content names) were interpolated into outbound URL paths unvalidated; a crafted ID with `../`, `?`, `#`, percent-encoded bytes or control characters could redirect the request to an unintended upstream endpoint. A central `EscapeResourceID` helper validates and escapes every such ID (#7307, #7310) -- **Bedrock S3 SSRF** - A caller-supplied `s3://` file ID or `s3_bucket` param could control the upstream host, since S3 virtual-hosted URLs are built as `https://{bucket}.{s3host}/{key}`; a bucket like `127.0.0.1:PORT#` opened a SigV4-signed TLS connection to a caller-chosen host. Bucket names are now validated with a DNS-compatible regex (#7315) -- **Caller-Forged Billing Idempotency** - The billing-idempotency key was `(RequestID, AttemptNumber)`, and `RequestID` may come from the caller's `x-request-id` header, so two unrelated requests sharing a deliberately chosen ID collided and the second was silently never charged. An internally minted `BillingNonce` is mixed into the key, making it unforgeable (#7303) -- **Abandoned Non-Streaming Request Hung Forever** - The worker kept a `ctx.Done()` arm on an already-claimed delivery send, a regression from the #6972 delivery fix, so a non-streaming caller could hang indefinitely (#7313) -- **Repeated Empty Thinking Blocks** - Streaming chat chunks emitted empty reasoning/message fields on every content delta, which clients rendered as repeated empty thinking blocks (#7318) -- **Claude Code Thread Continuations Failed Behind Key Rotation** - Claude Code's server-side conversation threads are bound to the upstream account that creates them, and Bifrost's per-request key selection, retries and fallbacks cannot keep a continuation on that account, so `thread: {"type": "continue"}` requests failed with `thread_not_found` (about half the time on a two-key config). The Anthropic integration now declares itself stateless: continuations are refused before the provider call with a 400 whose `details.error_code` is `thread_unsupported_request`, which makes the client resend the turn in full and stop sending the thread field for the rest of the session, and the provider raw-body path strips `thread` from create requests so no orphaned thread state accumulates upstream. Token counting is never refused (#7274) -- **Gemini Flash-Lite Minimal Thinking Promoted to Low** - Normalized `gemini-3.1-flash-lite` requests silently promoted `minimal` thinking to `low` on both Gemini and Vertex; the text model's four supported levels are now registered, preserving the image variant's separate restrictions (#7288) (thanks [@Javtor](https://github.com/Javtor)!) -- **Config.json Virtual Key Limits Broke Under UI Edits** - VK rate limits and budgets created via the config.json standalone-limits flow and then edited through the UI produced duplicate, conflicting ownership records (standalone budgets owned directly by the VK, plus orphaned UUID rate-limit rows created by the UI). Migration `migrate_vk_standalone_limits_to_model_configs` consolidates ownership into VK-scoped model configs while preserving usage counters, and the write paths stop creating the divergent rows (#7291) -- **Bedrock Service Tier Rejections** - Service tier forwarding for Bedrock (Converse and Mantle paths) sent whatever tier was requested; it is now gated on explicit model capability metadata, failing closed when none exists, so models that do not support the requested tier stop rejecting the request (#7266) -- **Anthropic Root-Level Tool Schema Compositions** - Anthropic rejects `oneOf`/`anyOf`/`allOf` at the root of a tool's `input_schema` but accepts them inside properties; root-level compositions are now rewritten into a flat object schema before dispatch, unblocking tools like Codex's `automation_update` (#7265) -- **GenAI Streaming TTS** - Speech stream chunks routed through the `/genai` integration had no converter registered; `ToGeminiSpeechStreamResponse` now serves streaming TTS, and the streaming router returns a clean error instead of panicking when any stream converter is missing (#7248) -- **xAI Usage and Cost** - xAI reports visible and reasoning completion tokens separately; `completion_tokens` now folds reasoning in so `prompt_tokens + completion_tokens = total_tokens` holds, and streaming cost normalization preserves xAI's authoritative `cost_in_usd_ticks` to the terminal usage chunk instead of falling back to catalog pricing (#7245, #7250) -- **OpenAI-Only `search_content_types` Forwarded Everywhere** - `search_content_types` on `web_search` tools is an OpenAI-specific extension; it is now gated behind a per-provider capability check so Bedrock and other OpenAI-compatible backends receive a clean `web_search` tool without the field (#7244) -- **Claude Code `diagnostics` Field Rejected by Non-Native Providers** - Claude Code sends `diagnostics.previous_message_id` on every request; on the typed-sanitizer path Bedrock, Vertex and Azure returned 400 `diagnostics: Extra inputs are not permitted`. The field is stripped for providers that do not support it (#7243) -- **Unreadable Bedrock Error Logs** - AWS returns errors in a flat `{"message": ..., "__type": ...}` shape the shared Anthropic/OpenAI parsers never looked at, so Bedrock errors were logged with no human-readable reason; the root-level message now seeds `BifrostError`, and the logs UI falls back to showing the raw provider error body when no message could be extracted (#7221, #7222) -- **Anthropic `container` Param Dropped** - The string-form `container` param on `/anthropic/v1/messages` was silently dropped, so container reuse provisioned a fresh container every time; it is now carried through the round trip (#5829) (thanks [@AdityaPainuli](https://github.com/AdityaPainuli)!) -- **Responses API Finish Reason Missing From OTEL Spans** - `gen_ai.response.finish_reason` / `finish_reasons` are now emitted on the `llm.call` span for `/v1/responses` requests, so refusals are visible to OTEL consumers, matching the chat path (#7205) (thanks [@bkfl-notai](https://github.com/bkfl-notai)!) -- **Cohere Fallback Response Shape** - The Cohere-compatible route returned the raw Bifrost normalized response when a fallback served the request from a non-Cohere provider, which the Cohere SDK failed to parse; responses are now converted to the Cohere v2 shape (`message`, `finish_reason`) (#7198) -- **Skill Serving Race** - Fixed a race condition in the skill serving handler -- **Log and Dashboard Label Truncation** - Long model and provider labels in the logs table and dashboard charts truncate from the start so the distinctive suffix stays visible, the logs model column is wider, the search field border and icon spacing are cleaned up, and key picker options are keyed by id so duplicate labels stop highlighting together (#7215, #7216, #7217, #7269, #7297) - -## 🗄️ Database Migrations - -- **migrate_vk_standalone_limits_to_model_configs** - Consolidates virtual key standalone budgets and rate limits into VK-scoped model configs: standalone budgets are re-pointed to the VK's top-level model config (created if missing, usage preserved), orphaned duplicate UUID rate-limit rows created by the UI are deleted with the model config re-pointed to the canonical config.json row, and `vk.rate_limit_id` is cleared. - - -## 🐙 Closed GitHub Issues - -- [#123](https://github.com/maximhq/bifrost/issues/123) - Files API Support -- [#5707](https://github.com/maximhq/bifrost/issues/5707) - string-form `container` param on /anthropic/v1/messages is silently dropped - container reuse provisions a fresh container -- [#7204](https://github.com/maximhq/bifrost/issues/7204) - Responses API path never sets gen_ai.response.finish_reason in OTEL traces (refusals invisible to observability) -- [#7287](https://github.com/maximhq/bifrost/issues/7287) - Gemini 3.1 Flash-Lite minimal thinking is silently promoted to low -- [#7294](https://github.com/maximhq/bifrost/issues/7294) - Streaming chat chunks emit empty reasoning/message fields on every content delta, causing repeated thinking blocks -- [#7308](https://github.com/maximhq/bifrost/issues/7308) - non-streaming caller hangs forever - worker keeps a ctx.Done() arm on a claimed delivery send (regression from #6972) \ No newline at end of file From 55d864f75939b8295d11db6a13f513f3e81c9802 Mon Sep 17 00:00:00 2001 From: akshaydeo Date: Sat, 19 Sep 2026 02:16:34 +0530 Subject: [PATCH 008/118] streaming gate fixes --- core/changelog.md | 68 --------- core/version | 2 +- framework/changelog.md | 18 +-- framework/streaming/gate.go | 223 ++++++++++++++++++++++++--- framework/streaming/gate_test.go | 254 ++++++++++++++++++++++++++++++- framework/streaming/types.go | 4 +- framework/version | 2 +- 7 files changed, 453 insertions(+), 118 deletions(-) diff --git a/core/changelog.md b/core/changelog.md index 21b5f509c98..e69de29bb2d 100644 --- a/core/changelog.md +++ b/core/changelog.md @@ -1,68 +0,0 @@ -- fix: `POST /v1/realtime/client_secrets` mints a Bifrost-issued ephemeral token instead of returning the provider's token (e.g. OpenAI `ek_...`) verbatim, which let a client connect to the provider directly and bypass governance, metering and attribution; virtual key credentials are validated before a realtime session is admitted, so a nonexistent VK can no longer open an upstream provider session (#7317) -- fix: streaming chat chunks no longer emit empty reasoning/message fields on every content delta, which made clients render repeated empty thinking blocks (#7318) -- fix: an abandoned non-streaming request no longer hangs its caller forever; the worker kept a `ctx.Done()` arm on an already-claimed delivery send, a regression from the #6972 delivery fix (#7313) -- fix: caller-supplied `s3://` file IDs and `s3_bucket` params on the Bedrock provider are validated with a DNS-compatible bucket regex before URL construction, closing an SSRF where a bucket like `127.0.0.1:PORT#` steered the SigV4-signed request to a caller-chosen host (#7315) -- fix: resource IDs (batch, file, video, container, response IDs and cached content names) are validated and escaped through a central `EscapeResourceID` helper before being interpolated into outbound URL paths, so IDs carrying `../`, `?`, `#`, percent-encoded bytes or control characters can no longer redirect requests to unintended upstream endpoints (#7307) -- fix: an internally minted `BillingNonce` is mixed into the billing-idempotency key, which was previously `(RequestID, AttemptNumber)` alone; since `x-request-id` is caller-supplied, two unrelated requests sharing a forged ID collided on the key and the second was never charged (#7303) -- fix: Bedrock service tier forwarding is gated on explicit model capability metadata (failing closed when none exists) on both Converse and Mantle paths, instead of sending whatever tier was requested and getting rejected by models that do not support it (#7266) -- fix: root-level `oneOf`/`anyOf`/`allOf` in a custom tool's `input_schema` are normalized into a flat object schema before reaching Anthropic, which rejects composition keywords at the schema root but accepts them inside properties; unblocks tools like Codex's `automation_update` (#7265) -- fix: speech stream chunks have a GenAI converter (`ToGeminiSpeechStreamResponse`), enabling streaming TTS through the `/genai` integration, and the streaming router returns a clean error instead of panicking when a stream converter is missing (#7248) -- feat: `SupportsPromptCacheBreakpoints` model cap with a name-based fallback for OpenRouter, so `prompt_cache_breakpoint` is only forwarded to models that accept it (#7261, #7262) -- fix: xAI `completion_tokens` folds `reasoning_tokens` into the total so `prompt_tokens + completion_tokens = total_tokens` holds, and streaming cost normalization preserves xAI's authoritative `cost_in_usd_ticks` through to the terminal usage chunk instead of falling back to catalog pricing (#7245, #7250) -- fix: `search_content_types` on `web_search` tools is an OpenAI-specific extension and is now gated behind a per-provider capability check, so Bedrock and other OpenAI-compatible backends receive a clean `web_search` tool without the field (#7244) -- fix: the `diagnostics` field Claude Code sends on every request is stripped for providers that reject it (Bedrock, Vertex, Azure return 400 `diagnostics: Extra inputs are not permitted` on the typed sanitizer path) (#7243) -- fix: AWS Bedrock's flat error shape `{"message": ..., "__type": ...}` seeds the root-level `message` into `BifrostError`, so Bedrock errors served through the shared Anthropic/OpenAI handlers are logged with a human-readable reason (#7222) -- [fix]: the string-form `container` param is carried through the /v1/messages round trip instead of being silently dropped, so container reuse stops provisioning a fresh container (#5829) (thanks [@AdityaPainuli](https://github.com/AdityaPainuli)!) -- fix: the Cohere-compatible route converts fallback responses from non-Cohere providers into the Cohere v2 shape (`message`, `finish_reason`) via `ToCohereChatResponse`, instead of returning the raw Bifrost normalized response the Cohere SDK cannot parse (#7198) -- fix: clearer error message in the Anthropic integration (#7264) -- [fix]: preserve Gemini 3.1 Flash-Lite minimal thinking on Gemini and Vertex [@Javtor](https://github.com/Javtor) -- fix: the Anthropic raw-body request path strips the server-side `thread` field before the wire on both messages and count_tokens (thread state is bound to the creating account and cannot survive Bifrost's per-request key selection, retries, or fallbacks; the HTTP transport refuses continuations outright), `AnthropicMessageErrorStruct` gains an optional `details` object with `error_code` so error envelopes can carry the machine-readable codes clients key their recovery on, and `LargePayloadMetadata` gains `ThreadType` so the refusal also works when large-payload mode skips body parsing (populated by the enterprise metadata extractor) -- [fix]: MCP tool schemas keep one property order across tool syncs [@dougcalobrisi](https://github.com/dougcalobrisi) -- feat: `WhiteList` and `BlackList` entries that start with `regex:` are RE2 patterns. `MatchEntry` compiles them once as a case-insensitive full match, `Validate` refuses a pattern that is empty, `*` or does not compile, and the list-models pipeline never surfaces a pattern as a model. Plain entries keep their exact, case-insensitive match -- fix: fasthttp-level stale-connection retries no longer multiply `max_retries`. `contextTransport.RoundTrip` reports a failure before response headers on a freshly dialed socket with `retry=false`, so `network.StaleConnectionRetryIfErr` only walks past pooled keep-alive sockets the upstream closed while idle; an upstream that closes a fresh connection without answering now costs exactly one attempt of Bifrost's own retry budget instead of up to four. The retry backoff also ends as soon as the request context is cancelled, so a worker is freed immediately instead of after up to `retry_backoff_max` (#7035) -- [fix]: Gemini provider - preserve inline image and audio data in chat completions responses [@Atharva-Kanherkar](https://github.com/Atharva-Kanherkar) -- [fix]: route opencode-zen Responses calls through /v1/chat/completions, which its upstream does not serve on /v1/responses [@miguelchico](https://github.com/miguelchico) -- feat: the namespace-tool names a provider reserves for its own server tools (`web`, `image_gen`, `browser`, `python` on Bedrock and Bedrock Mantle) are now read from the datasheet row `reserved_tool_namespaces` (new `ModelCapabilities` field and `ModelCaps.ReservedToolNamespaces`) for the base provider and canonical model; a non-empty row replaces the hardcoded list, an absent or empty row keeps it, so any OpenAI-wire provider can reserve names through a row without a code change -- fix: Codex >= 0.147 wraps its default tools in a namespace literally named `functions` (openai/codex#37022), which Bedrock Mantle reserves and rejects with `User-defined namespace 'functions' collides with an existing tool namespace`. Core now unwraps that namespace to top-level tools, unprefixed, for every provider before dispatch; Codex treats the bare name and the explicit `functions` namespace as the same tool, so no mapping back is needed; a description on the `functions` namespace is prepended to each hoisted member, as flattening does -- fix: Responses `namespace` tools are now flattened in core for every provider whose wire does not support the type (Anthropic, Gemini, Vertex, Bedrock Converse, DeepSeek, and every OpenAI-compatible third party), with nested functions renamed to `__` so two namespaces that share a function name no longer collide into an upstream `Tool names must be unique` 400; returned `function_call` items are mapped back to the bare `name` plus `namespace` (unary and streaming), prior-turn calls carrying `namespace` and `tool_choice` names are re-aliased to match, and a name that is still duplicated after flattening or a `tool_choice` that matches several namespaces is rejected with a clear 400 before reaching the provider. Bedrock answers namespace support from its own surface resolver via the new optional `ResponsesNamespaceToolProvider` interface. A datasheet row `supports_namespace_tools` (new `ModelCapabilities` field and `ModelCaps.SupportsNamespaceTools`) overrides the per-provider default for a (provider, model) pair; with no row the default applies. A row can only narrow within what the wire can carry: the Anthropic Messages API (Anthropic, Claude on Azure or Bedrock Mantle), the Gemini API (Gemini, Vertex) and Bedrock Converse have no namespace container, so they answer false regardless of the row and always flatten. Flattened names honour the target wire's documented tool-name limit, 64 characters of `[A-Za-z0-9_-]` for OpenAI-compatible wires, Bedrock Converse and Fireworks, 128 for Anthropic, and 128 with `.` and `:` allowed for Gemini and Vertex, overridable per model through the datasheet row `tool_name_max_length` (a row below 10 cannot hold the hashed form and is ignored); a longer alias becomes an 8-hex hash prefix plus the function name, deterministically, so history and `tool_choice` re-alias to the same string. The alias map travels on the prepared request (`BifrostResponsesRequest.NamespaceToolAliases`) and is applied to that attempt's response, unary and streaming; nothing is kept on the request context or in process-wide state (#7048) -- fix: drop namespace tools whose name Amazon Bedrock reserves (`web`, `image_gen`, `browser`, `python`) on the Bedrock and Bedrock Mantle Responses paths instead of forwarding them into a `tools.namespace` collision 400; a dropped Codex `web` namespace becomes the hosted `web_search` tool on Bedrock Mantle -- fix: Bedrock Mantle chat streaming no longer drops the usage-only chunk that arrives after `finish_reason`; `ProviderSendsDoneMarker` now treats `bedrock_mantle` (and the legacy Mantle route under the `bedrock` key) as sending `[DONE]`, so streamed usage and cost are recorded (#7065) -- fix: `default_request_timeout_in_seconds` now bounds the wait for response headers on streaming requests, and cancelling a request now closes the upstream socket. Every fasthttp client is driven through a Bifrost-owned `RoundTripper` (`core/providers/utils/roundtripper.go`) that applies the client's read/write timeouts and the request context to the request write and the header wait, then lifts the socket deadline once headers are parsed so `stream_idle_timeout_in_seconds` remains the only bound on the body. `BuildStreamingClient` and `BuildLargeResponseClient` no longer zero the timeouts. An upstream that accepts the connection and never answers now fails with 504 `RequestTimedOut` after the request timeout and the request's fallbacks are used, instead of pinning the provider worker until the upstream closes the socket; `MakeRequestWithContext` no longer leaves a background `client.Do` running until `ReadTimeout` after its context is cancelled (#7034) -- fix: unary large-response downloads (`FinalizeResponseWithLargeDetection`) now bound every body read with `stream_idle_timeout_in_seconds` and close the upstream socket on request cancellation, mirroring the streamed passthrough path; a stall during the prefetch surfaces as 504 `RequestTimedOut` instead of pinning the provider worker, and a stall while the transport drains the `LargeResponseReader` fails the read with the idle-timeout error instead of blocking the writer indefinitely (#7104) -- fix: gzip-encoded unary responses are now classified for large-response mode by decompressed size; `Content-Length` describes the compressed bytes, so a body that was small on the wire but large once decompressed used to be materialized in full past `large_response_threshold` (#7104) -- fix: an OpenAI-compatible upstream that omits `[DONE]` and then goes silent after `finish_reason` no longer fails the stream when `stream_idle_timeout_in_seconds` fires. The chat and text completion read loops now treat an idle timeout after a terminal signal as a parked upstream, mark the stream parked so the connection is abandoned rather than drained, and synthesize the final chunk with the buffered `finish_reason`; a stall before `finish_reason` still surfaces as the idle-timeout error (#7108) -- fix: a streamed upstream that drops the connection mid-body is again reported as the retryable 502 `provider closed the stream before sending a completion marker` error instead of a generic `Error reading stream: unexpected EOF`. The Bifrost round tripper's chunked decoder surfaced the drop as `io.ErrUnexpectedEOF`, which no provider read loop treats as end of stream; it now reports the plain `io.EOF` fasthttp always did and discards the half-read connection (#7104 follow-up) -- fix: fallbacks for image edit, image variation and video edit requests now reach the configured fallback provider and model. `prepareFallbackRequest` had no arm for those three types, so the shallow request copy kept the primary's sub-request pointer and the "fallback" attempt was routed back to the primary while `RoutingInfo`, the `x-bifrost-routing-info-*` headers, the log row and the `fallback_index` metric label reported it as a fallback. The helper now also verifies the prepared request targets the fallback provider and model and skips the fallback with a warning otherwise, so a future request type added without an arm fails loudly instead of silently re-running the primary (#6966) -- fix: route Bedrock Claude requests that carry a `compact_20260112` edit to InvokeModel / InvokeModelWithResponseStream with the native Anthropic Messages body, so server-side compaction works on `bedrock/` models, including keys that pin an inference-profile ARN. AWS documents compaction as unsupported on the Converse API, which previously received the edit and silently ignored it (#6825) -- feat: serve Anthropic tool search (`tool_search_tool_*`, `defer_loading`) on `bedrock/` Claude models by routing those requests to InvokeModel / InvokeModelWithResponseStream, the only Bedrock API AWS allows it on; CountTokens counts such requests with the same InvokeModel body via the `invokeModel` input -- fix: a `reasoning.context` value the target model does not accept is now dropped on the OpenAI and Azure Responses path, so `all_turns` on the original gpt-5 family incl. gpt-5-pro, gpt-5.1 to gpt-5.3, and the o-series runs under the model's own `current_turn` default instead of failing with `Unsupported value: 'all_turns' is not supported with the 'gpt-5-pro' model`; gpt-5.4, gpt-5.5 and gpt-5.6 keep it. The accepted values come from the datasheet row `supported_reasoning_contexts` (new `ModelCapabilities` field and `ModelCaps.SupportedReasoningContexts`), falling back to a name-based default of `auto` and `current_turn` everywhere plus `all_turns` on gpt-5.4+ -- fix: mid-conversation `role: "system"` messages are now inlined in place as `` user turns for every model family on every converter that has a top-level system field: Bedrock Converse (Responses and Chat Completions), Gemini Chat Completions (`systemInstruction`), and the Anthropic wire shape used by DeepSeek, Fireworks and SGL. Previously only Claude on Bedrock/Anthropic inlined; everything else hoisted each reminder into the top-level system block. Claude Code appends a trailing `` system reminder after every turn, so the hoisted block grew the front of the prompt each turn and prefix-based caches (Bedrock implicit caching for `global.openai.gpt-5.6-luna` on Converse, Gemini implicit caching, DeepSeek context caching) reported a full cache write and zero cache reads on every turn -- [fix]: vLLM key selection resolves the requested alias through `key.Aliases` per key before comparing it with `vllm_key_config.model_name`, so a valid key is no longer rejected when the request used an alias; the same public alias can map to different physical model IDs across vLLM instances, and allow and block checks keep matching the original user-facing alias [@Constantine3](https://github.com/Constantine3) -- [fix]: Gemini Responses requests keep `top_k`, `frequency_penalty`, `presence_penalty`, `stop_sequences` and `media_resolution` across retries and fallbacks. `convertParamsToGenerationConfigResponses` deleted those keys from `ExtraParams` while mapping them into `generationConfig`, and the conversion runs once per attempt on the same request, so every attempt after the first reached the provider without them; the outbound `ExtraParams` is now built through a filtered copy instead of mutating the request [@VictorRequenaMaisa](https://github.com/VictorRequenaMaisa) -- feat: `wait_for_usage` on `custom_provider_config` keeps the OpenAI-compatible chat and text completion read loops open past `finish_reason` until the trailing usage-only frame arrives, so a custom provider that also sets `does_not_send_done_marker` no longer synthesizes a zero-usage terminal chunk and records the request at zero tokens and zero cost. `WaitForUsage` is carried on `BifrostContextKeyWaitForUsage` with the same set-or-clear discipline as `DoesNotSendDoneMarker`, so it never leaks onto a fallback provider that did not declare it, and termination stays bounded by the usage chunk, two consecutive post-finish heartbeat comments, EOF, or `stream_idle_timeout_in_seconds` (#7143) -- fix: `ExtraFields.RawResponse` now carries role-only, finish-only and usage-only SSE frames on the OpenAI-compatible chat stream. Those frames never entered the chunk-forwarding branch, so their bytes were discarded from the reconstructed `raw_response`, which made the captured audit trail irreconcilable against a provider invoice because the usage frame carries the token counts Bifrost bills from. A `pendingRawFrames` buffer accumulates them and drains onto the next forwarded chunk or the synthetic terminal chunk; the Responses-over-Chat fallback no longer stamps one upstream frame onto every derived event, and `delta.refusal` and `delta.annotations` are added to the forwarding predicate so refusals and streamed URL citations are no longer dropped entirely (#7144) -- fix: Anthropic server-side tool search survives the Bedrock-native invoke ingress end to end. `tool_search_tool_*` was dropped during the Converse-shaped intermediate conversion, leaving the egress predicate blind to the tool and routing the request to Converse where server-side tool search is unavailable; the tool is now carried as an ingress-only marker (`json:"-"`, never reaching a Converse body) and the neutral tool is rebuilt in `ToBifrostResponsesRequest` along with `defer_loading`. Results are returned as a `server_tool_use` plus `tool_search_tool_result` pair rather than a client `tool_use`, which Anthropic rejects when the caller echoes a `tool_result` for the `srvtoolu_` ID on the next turn, on both the unary and streaming paths; `BedrockContentBlock.UnmarshalJSON` gained `server_tool_use` and `tool_search_tool_result` cases so replayed turns round-trip unchanged instead of falling through to empty structs; and the Anthropic-native response path carries the same blocks (#7155) -- fix: native Gemini `generateContent` image requests routed through `/genai` no longer return HTTP 200 while silently ignoring part of the request. `isImageEditRequest` only inspected `contents[0].parts[0]`, so a request with the prompt text before the `inlineData` part, the ordering in Google's own REST edit sample, was misclassified as image generation: the image never reached Vertex and the model invented a picture from the prompt alone. Detection now scans every part across every content. `imageConfig.aspectRatio` is preserved as a typed `aspect_ratio` param that both outbound converters prefer, instead of being folded into a `WxH` size string that collapsed any ratio outside `1:1`, `3:4`, `4:3`, `9:16` and `16:9` back to square (#7173) -- fix: Gemini `Part` gained `mediaResolution`, the per-part override for `generationConfig.mediaResolution`. `Part.UnmarshalJSON` decodes into a closed alias, so the key was discarded before any conversion ran and per-part image and PDF tokenization fell back to the model default; an `ULTRA_HIGH` image billed about 21k prompt tokens through `/genai` instead of about 22.1k direct. Both the `mediaResolution` and `media_resolution` spellings are parsed, the value round-trips through `ResponsesMessageContentBlock` and is reconstructed on the outbound part, and it is stripped on the OpenAI wire path where the field is unknown (#7156) -- fix: image blocks inside a Bedrock Converse `toolResult` are hoisted out and re-inserted immediately after the last tool result in the same message, with a placeholder text block left behind so the now image-free result is not rejected for being empty. Some Bedrock-hosted models, notably the OpenAI and Grok families, reject nested tool-result images on Converse even though they accept images in tool output via the Responses API. `BedrockModelSupportsToolResultImages` supplies the name-based default and the datasheet field `supports_converse_tool_result_images` (new `ModelCapabilities` field and `ModelCaps.SupportsConverseToolResultImages`) overrides it per row (#7150) -- feat: `bifrost_error_requests_total` gains a normalized `error_type` label drawn from a closed, prefix-structured vocabulary (`caller_*`, `policy_*`, `provider_*`, `bifrost_*`, `_OTHER`), so a 429 from a governance rate limit is distinguishable from a 429 from an upstream provider, a 403 from a blocked model from one from a bad key, and a 503 Bifrost shed under queue pressure from an upstream overload. `ClassifyErrorType` resolves a declared `ExtraFields.ErrorType` first, then Bifrost's own string markers, then the upstream status code; it deliberately ignores the provider's own `error.type` and `error.code` strings, which disagree across providers for the same condition. Values are prefixed by fault domain so an alarm expression can match a family with a single regex (#7141) -- fix: Azure Foundry deployments of Fireworks-hosted models are no longer silently capped at 4096 output tokens on `/openai/v1/responses`, where Microsoft routes those models through chat completions internally. The Azure provider checks the model's datasheet `supported_endpoints` and, when `/v1/responses` is absent, transparently serves both `Responses` and `ResponsesStream` through `/openai/v1/chat/completions` and converts the result back to the Responses shape; a model with an explicit `/v1/responses` entry or no datasheet row at all is unchanged. Separately, `ToAnthropicResponsesResponse` now follows `StopReason > IncompleteDetails > tool_use inference > end_turn`, so a turn truncated by the output-token cap on any OpenAI-shaped Responses provider reports `stop_reason: max_tokens` instead of hiding the truncation as `end_turn` (#6782) -- fix: DeepSeek chat requests move `max_completion_tokens` into `max_tokens` on the wire. DeepSeek's chat-completions endpoint only recognizes the legacy field and silently ignores `max_completion_tokens`, so the limit had no effect; this matches the behaviour already in place for Opencode and Ollama (#7131) -- feat: guardrail redaction gains an identity-based transformation path for provider-native request and response bodies, so exact provider-managed rewrites such as Model Armor and Bedrock land in the correct native JSON field even when the same text value appears in several fields. `TextTargetID`, `TextTargetIDForIndex` and `TextRewrite` identify each guardrail-visible field by a stable ID rather than by content value; `RawRequestBodyTextTransformer` and `RawResponseTextTransformer` let integrations register handlers on the request context; and `rewriteRawJSONTextTargets` validates target uniqueness, verifies each `Original` value before patching and re-reads the written value after, so stale or mismatched normalized text cannot silently rewrite the wrong field (#7049) -- feat: guardrail redaction covers LLM tool-call arguments, Chat function arguments, Responses function arguments and custom-tool input, across the Anthropic streaming and non-streaming paths. `input_json_delta` events are inspected and rewritten alongside `text_delta`, reading and writing `delta.partial_json`; `collectAnthropicArgumentStringPaths` recursively collects string-valued paths inside a `tool_use` block's `input` without touching tool names, IDs or definitions; `BifrostResponsesStreamResponse` gained `Input` to carry the full custom-tool payload on `custom_tool_call_input.done`; and `DeepCopyResponsesMessage` deep-copies `ResponsesCustomToolCall` so a copied message cannot mutate the original (#6977) -- fix: an abandoned non-streaming request, one whose caller context was already cancelled by the time the upstream finished, is billed and logged deterministically instead of about half the time. The worker's delivery `select` had two simultaneously ready cases, a send into a cap-1 channel and `ctx.Done()`, and Go picks uniformly among ready cases, so the terminal post-hooks that record billing and finalize the log row were skipped roughly 50% of the time. `requestWorker` now checks `req.Context.Err()` before the select on both the error and success paths and calls `billAbandonedTerminal` directly, keeping the 5-second timer guard inside the select for a caller that leaves between the check and the send (#6972) -- fix: provider response headers are filtered by classifier, not only by a fixed map of 28 exact names, so a credential-named header outside that map is no longer re-served to the inference caller in the HTTP response and `extra_fields.provider_response_headers`. The extractors now also consult `schemas.IsSensitiveHeader`, which matches credential names by substring and suffix and already recognized `cf-access-*` and `x-amzn-oidc-*`; a fixed list cannot enumerate the space when `network_config.extra_headers` exists to carry custom authentication headers and some upstreams echo request headers back (#7120) [@Atharva-Kanherkar](https://github.com/Atharva-Kanherkar) -- [fix]: a plugin that returns an incomplete `BifrostError` whose nested `Error` field is nil no longer panics the request worker. The fallback decision helper nil-checks `fallbackErr`, guards access to `Error.Type` and reads the message through the nil-safe `GetErrorString()` helper, continuing to the next fallback when allowed; behaviour for cancelled requests and `AllowFallbacks == false` is unchanged (#6967) [@Constantine3](https://github.com/Constantine3) -- [fix]: `clearCtxForFallback` now clears `BifrostContextKeyProviderResponseHeaders`. Providers set that key from their own HTTP response before the status check so error paths can forward it, and when a fallback attempt failed pre-flight, key selection failing for the fallback provider, a plugin short-circuiting it, or the queue retiring, nothing overwrote the key and the primary's headers survived onto the fallback's error response. A client could receive a response attributed to provider B carrying provider A's `Retry-After` and `x-ratelimit-remaining-*`, and wait according to a limit belonging to a provider that never served the request (#6973) [@Huang-404-Q](https://github.com/Huang-404-Q) -- [fix]: untitled document blocks get unique names on the Bedrock Converse path. Every untitled block was named the literal `document`, and Converse rejects duplicate document names, so any request carrying two or more untitled documents failed unconditionally with `ValidationException: Messages can't contain duplicate document names`. A per-request namer threaded through the content-block conversion disambiguates with numeric suffixes (`document`, `document-2`, `document-3`) and suffixes explicitly titled documents only on an actual collision; the Responses input replay path gets the same treatment scoped to its content-block list (#7003) [@Huang-404-Q](https://github.com/Huang-404-Q) -- [fix]: `BifrostResponsesStreamResponse.Item` is `omitempty`, so Responses stream events that carry no item payload no longer serialize `"item": null`. Strict OpenAI Responses clients reject those as invalid stream frames, which broke streamed `/v1/responses` usage entirely; `response.output_item.added` and `response.output_item.done` still emit the item object [@ReStranger](https://github.com/ReStranger) -- feat: `SecretVar.RedactedIfSecret()` returns a plain clone when the value is a literal and delegates to `Redacted()` when it is env-var or vault-backed, so regions and service URLs (Azure endpoint, Vertex/Bedrock/Bedrock Mantle region, vLLM/Ollama/SGL/Databricks URL, MCP connection string) are readable in the UI instead of being masked as if they were credentials. It always returns a fresh pointer, so the redacted copy never aliases the live config, and it replaces the ad-hoc `IsFromSecret()` guard blocks that tried to replicate this inline (#7085) -- feat: GA realtime transcription is served for OpenAI and Azure over both WebSocket and WebRTC with normal Bifrost authentication, routing, governance, guardrails, logging and transcription-aware pricing. Unlike a normal realtime session, which names its routing model in the connect URL, a GA transcription session carries only `intent=transcription` and delivers the model later in `session.update` under `audio.input.transcription.model` (or in the initial multipart `/v1/realtime/calls` request on WebRTC), so routing now resolves from the nested transcription model while realtime connection and turn semantics are preserved (#7089) -- fix: Claude Code replaying a prior assistant message through `POST /anthropic/v1/messages` at a Bedrock Mantle model is no longer rejected with hundreds of validation errors. The Bedrock-grouped ingress converter tagged user and system input text as `output_text`, though only `input_text` is valid on input messages, and omitted the required `status` on replayed assistant output messages, so Mantle's strict OpenAI-compatible validator refused the whole request. Replayed assistant messages now carry `Status: "completed"` and input text blocks are tagged `input_text`. Bedrock requests with no explicit `max_tokens` also populate it from the model's known capacity instead of truncating silently on large context windows (#7074) -- fix: Anthropic server tools such as `web_search_20250305` are stripped before a request reaches Fireworks, vLLM or SGLang on their Anthropic-compatible endpoints, which return 400 for tools executed by Anthropic-operated infrastructure that does not exist on third-party hosts; clients whose built-in web search is always on, such as Codex, hit this on every request. `ProviderFeatures` entries for `Fireworks`, `VLLM` and `SGL` declare all server-tool flags off so the existing validators strip them, `StripUnsupportedServerToolsFromRawBody` mirrors that on the raw-body passthrough path, the caller's function tools are kept, and the drops are reported on the response's `DroppedUnsupportedTools` instead of failing the call (#7090) -- fix: `guardrailConfig` reaches Bedrock's OpenAI-compatible `chat/completions` and `responses` surfaces, streaming and non-streaming, which apply guardrails through request headers rather than the body field Converse uses, so a configured guardrail was previously ignored there with no error. `withGuardrailHeaders` maps `guardrailIdentifier`, `guardrailVersion` and optional `trace` to `X-Amzn-Bedrock-GuardrailIdentifier`, `X-Amzn-Bedrock-GuardrailVersion` and `X-Amzn-Bedrock-Trace`, then deletes the key from `ExtraParams` so it is not also emitted into the body. A half-formed config with only one of identifier or version is left untouched rather than sent, since both are required upstream, and the base header map is cloned rather than mutated (#7095) -- feat: `use_openai_endpoints` on Bedrock keys and aliases routes chat completions and responses through Bedrock's OpenAI-compatible `/openai/v1` endpoints instead of Converse, for models that support them, mirroring the existing `use_anthropic_endpoints` pattern. It is opt-in by design: Converse carries Bedrock Guardrails, `performanceConfig` and `requestMetadata` that the OpenAI-compatible surface silently ignores, so diverting automatically could stop a guardrail from being enforced with no visible error. `ResolveUseOpenAIEndpoints` gives the alias value precedence over the key, and the narrow `runtimeServesResponses` check is replaced by a general `runtimeServesOpenAIAPI` that takes a `BedrockAPI` discriminator; chat completions, streaming and non-streaming, gained the runtime path that previously existed only for Responses (#7073) -- fix: Bedrock runtime models that serve the Responses API are routed to it through a dedicated surface resolver rather than falling back to Converse (#7071) -- fix: Bedrock Converse document blocks always ship their content base64-encoded through `source.bytes`, never through `source.text`, which Converse rejects unless citations are explicitly enabled. Plain text formats (`text/plain`, `text/markdown`, `text/csv`, `text/html`) previously took the text branch and failed with `DocumentSource must set one of the following keys: bytes, s3Location`; the branch is removed for data URLs, percent-encoded payloads and the non-data-URL `file_data` path alike (#7072) -- fix: the Bedrock Mantle base path is resolved from the model capabilities datasheet instead of hard-coded string matching in two packages. Mantle serves each model on exactly one of `v1` or `openai/v1` and returns a 400 on the other, and the previous matching covered only generations up to GPT-5 and Gemma 4, so GPT-6 and any future closed-generation model silently fell through to the wrong path. `ResolveBedrockMantleBasePath` applies family-name detection as a fallback and defers to the new `BedrockMantleBasePath` field on `ModelCapabilities`, so a new generation needs a datasheet row rather than a code change (#7077) -- fix: `image_generation_call` items where OpenAI emits `action` as a bare JSON string, such as `"generate"`, decode correctly. `ResponsesToolMessageActionStruct.UnmarshalJSON` immediately peeked at a `.type` field, which cannot be read from a JSON string, so the whole decode failed with `failed to peek at type field`; that silently dropped the `response.output_item.done` and `response.completed` events carrying the image, leaving the stream without a terminal event and surfacing as a bogus `provider closed the stream` truncation error. The action is now tried as a bare string before falling back to the object type-peek, and `ResponsesImageGenerationCall` keeps the `background`, `output_format`, `quality`, `revised_prompt` and `size` settings OpenAI echoes back on completed items (#7060) diff --git a/core/version b/core/version index 9ab8337f396..ee672d89ab3 100644 --- a/core/version +++ b/core/version @@ -1 +1 @@ -1.9.1 +1.9.1 \ No newline at end of file diff --git a/framework/changelog.md b/framework/changelog.md index d10b09b694d..aa2eb3a41de 100644 --- a/framework/changelog.md +++ b/framework/changelog.md @@ -1,17 +1 @@ -- fix: virtual key rate limits and budgets created via the config.json standalone-limits flow and later edited through the UI no longer produce duplicate, conflicting ownership records; migration `migrate_vk_standalone_limits_to_model_configs` consolidates ownership into VK-scoped model configs, preserving usage counters, and the write paths stop creating the divergent rows (#7291) -- feat: virtual key listing batch-resolves assignees (`assigned_user`) instead of resolving per row, and VK search matches team and customer names in addition to key fields (#7225) -- chore: OTEL dependency updates (#7311) -- fix: Responses API requests emit `gen_ai.response.finish_reason` / `gen_ai.response.finish_reasons` on the OTEL `llm.call` span. `PopulateResponsesResponseAttributes` never copied the top-level `stop_reason` into the span attributes, so refusals on `/v1/responses` were invisible to OTEL consumers even though the stop reason reached the Logs DB; it now emits `finish_reasons` like the chat path and the tracer derives the singular key from it (#7204) -- feat: model allow and block lists accept `regex:` entries. The provider-key aggregate keeps an exact name next to a pattern that also covers it, the catalog allow check tries a pattern against the bare name and `provider/model`, and listings skip pattern entries -- fix: ClickHouse log store deletes no longer run as heavyweight `ALTER TABLE ... DELETE` mutations. The retention cleaner issued one such mutation per 100 rows, each rewriting the whole current-month part, and the once-a-minute stale-`processing` sweeps issued one per table unconditionally, filling replica disks in minutes. Every delete on the ClickHouse store (retention sweep, `Flush`/`FlushMCPToolLogs`, UI log deletes, async job and webhook delivery expiry) is now a single lightweight `DELETE FROM ... WHERE` per run, skipped entirely when nothing matches. The table TTL derived from `logs_store.retention_days` is now reconciled on every startup with `MODIFY TTL` (metadata only), so changing the value reaches existing tables; `0` leaves an existing TTL untouched (#7098) -- feat: access profiles and governance projects reference Virtual MCPs through `virtual_mcp_name`, so config files are portable across environments instead of carrying database-assigned integer IDs. Names resolve to stored records on startup and a name that matches nothing is refused; `virtual_mcp_id` is deprecated, still accepted, and wins when both are set. `mcp_configs` (`{ mcp_client_id, tools_to_execute }`) replaces the `mcp_servers` and `mcp_tool_overrides` include-exclude model with a single allowlist, where `["*"]` grants all tools including future ones, `[]` grants none, and a named list grants only those; the old keys are deprecated, still accepted, and folded into the new shape at load time (#7181) -- feat: `MCPToolLog` records governance entity names beside their IDs, so MCP tool logs carry the same attribution shape the `logs` table has instead of rendering raw UUIDs in the dashboard. `user_name`, `team_name`, `customer_name` and `business_unit_name` stop being `gorm:"-"` transients and become storage; the multi-valued `team_ids`/`team_names`, `customer_ids`/`customer_names` and `business_unit_ids`/`business_unit_names` sets are stored as index-aligned JSON arrays; and `budget_ids` and `rate_limit_ids` are recorded id-only, as in `logs`. Names are written from the request context at ingestion through the new `MCPToolLog.ApplyGovernance` in `framework/logstore/governance.go`, with nothing resolved on read. Added by migration `mcp_tool_logs_add_governance_snapshots` (#7154) -- feat: endpoint-attributed MCP observations are carried in the standard logging pipeline, so inspected MCP tool calls are logged with bounded identity (device, app key, server label, tool name, decision) sourced from the gateway rather than from payload-supplied headers. `SetMCPObservation` attaches the attribution to a `BifrostContext` and `applyMCPObservation` writes it onto the `MCPToolLog` from both `PreMCPHook` and `PostMCPHook`, snapshotting the observation so it cannot alias across async log entries (#6959) -- feat: the `error_type` classification vocabulary is threaded through the framework so `bifrost_error_requests_total` can carry a normalized fault-domain label alongside `status_code` (#7141) -- feat: model pricing supports time-of-day peak and off-peak rates. `TableModelPricing` gains `off_peak_cost_multiplier` (a nullable float) and `peak_hours` (a JSON-serialized `PeakHoursSchedule` of recurring weekly windows using IANA timezone names, weekday numbers and half-open `HH:MM` intervals that may wrap past midnight), added by migration `add_time_of_day_pricing_columns` and aliased into the datasheet package so the JSON shape stays self-contained. The cost engine evaluates the schedule against the request start time and scales usage-based charges by the multiplier when the request falls outside every peak window, applied once in `computeCostFromInput` so every modality is covered; flat `CostPerRequest` and `SearchQueriesCost` fees are excluded, as is `AdditionalCost`, which is discounted independently through its own pricing rows. Both fields are exposed on `PricingPatch` in the OpenAPI and governance schemas, with `off_peak_cost_multiplier` bounded to `(0, 1]` (#6574, #6575, #6576) -- feat: `SecretVar.RedactedIfSecret()` is used for non-credential fields in `ProviderConfig.Redacted()`, `Config.GetAllKeys()` and `Config.RedactMCPClientConfig()`, so regions, endpoints, service URLs and MCP connection strings surface as plaintext while anything env-var or vault-backed stays masked (#7085) -- feat: GA realtime transcription sessions are routed, governed, logged and priced through the normal framework pipeline, resolving the routing model from the nested `audio.input.transcription.model` that arrives in `session.update` rather than from the connect URL (#7089) -- feat: a `use_openai_endpoints` column on the provider keys table, added by migration `add_use_openai_endpoints_column`, opts a Bedrock key or alias into Bedrock's OpenAI-compatible endpoints instead of Converse (#7073) -- feat: the reserved tool-namespace list a provider keeps for its own server tools is read from the datasheet row `reserved_tool_namespaces`, so a namespace collision check no longer requires a code change (#7084) -- fix: `exchangeRefreshToken` includes `client_secret` only when the secret is non-empty, matching `exchangeCodeForTokensWithPKCE`. Public OAuth2 clients registered against servers that support only `token_endpoint_auth_method: none` have no secret, and unconditionally setting `client_secret=` sent an empty `client_secret_post` attempt that strict authorization servers answered with `invalid_client`, flipping the token row to `needs_reauth` even though the refresh token was valid (#7042) -- fix: the virtual key `allowed_models: ["*"]` handling for governance added in #6767 is reverted, returning the wildcard-with-empty-synced-catalog case to its previous behaviour (#7053) +- fix: replace streaming gate replay-buffer size accounting with cached zero-marshal estimates (eliminates per-chunk MarshalJSON on the full-hold path) \ No newline at end of file diff --git a/framework/streaming/gate.go b/framework/streaming/gate.go index 47c84c80b91..c431e813829 100644 --- a/framework/streaming/gate.go +++ b/framework/streaming/gate.go @@ -1,9 +1,11 @@ package streaming import ( + "encoding/json" "fmt" "time" + "github.com/bytedance/sonic" "github.com/maximhq/bifrost/core/schemas" ) @@ -12,19 +14,182 @@ import ( // consumer is notified and memory is released. const gateReplayBufMaxBytes int64 = 100 * 1024 * 1024 -// chunkBytes returns the approximate wire size of a BifrostStreamChunk via -// MarshalJSON. Used only on the paused replay path to bound buffer growth. -func chunkBytes(chunk *schemas.BifrostStreamChunk) int64 { - if chunk == nil { +// gateChunkSizeFloor is the per-chunk baseline for replay accounting: struct +// overhead plus the small scalar fields the estimator does not walk. Measured +// heap overhead for a passthrough chunk is ~500B; 512 keeps the floor honest +// without tripping the 100MB cap on large-but-legitimate streams (a 60MB +// stream in 1.4KB events carries ~43k chunks — a 1KB floor alone would have +// added 43MB to the estimate). +const gateChunkSizeFloor int64 = 512 + +// gateReplayEntry pairs a buffered chunk with its size estimate, computed once +// at append so cap accounting never re-serializes held chunks. +// +// Ownership contract: a chunk handed to GateSend belongs to the delivery +// pipeline from that point on. Producers must hand over freshly built, +// right-sized chunks (no shared scratch buffers, no sub-slices of larger +// arrays) and must not retain or mutate them afterward — the same invariant +// the non-gated channel send already requires, since the chunk outlives the +// producer's loop either way. The only sanctioned post-append modification is +// TransformPausedBuffer, whose copy-on-write replacements are re-estimated at +// install. Violating this contract cannot corrupt delivery or bypass guardrail +// inspection; it can only make the cached size (and thus the +// gateReplayBufMaxBytes accounting) under- or over-read. +type gateReplayEntry struct { + chunk *schemas.BifrostStreamChunk + size int64 +} + +// gateRawBytes sizes the raw request/response carriers without serializing: +// they typically hold json.RawMessage, []byte, or string. An atypical carrier +// (e.g. a pre-parsed map) is sonic-sized rather than riding the buffer +// uncounted; that path only exists for providers that deviate from +// compactRawJSON's json.RawMessage contract. +func gateRawBytes(v interface{}) int64 { + switch b := v.(type) { + case nil: + return 0 + case *interface{}: + // Unwrap a pointer-boxed value so its payload hits the fast len() + // cases below instead of the sonic fallback. + if b == nil { + return 0 + } + return gateRawBytes(*b) + case json.RawMessage: + return int64(len(b)) + case []byte: + return int64(len(b)) + case string: + return int64(len(b)) + } + return gateCompositeBytes(v) +} + +func gateStrPtrBytes(s *string) int64 { + if s == nil { return 0 } - b, err := chunk.MarshalJSON() + return int64(len(*s)) +} + +// gateCompositeBytes sizes a composite carrier (full item, part, or terminal +// message copy) with a plain sonic marshal. Composites ride only item-boundary +// events, never per-delta chunks, so this stays off the hot append path. +func gateCompositeBytes(v interface{}) int64 { + b, err := sonic.Marshal(v) if err != nil { return 0 } return int64(len(b)) } +// estimateChunkBytes estimates the heap held by one buffered chunk without +// serializing it. It feeds only the gateReplayBufMaxBytes safety cap, so +// exactness is not required: it counts the carriers that grow with response +// size — passthrough bodies, raw request/response bytes, delta/text strings, +// tool-call arguments, and composite item copies — and covers everything else +// with a fixed per-chunk floor. +func estimateChunkBytes(chunk *schemas.BifrostStreamChunk) int64 { + if chunk == nil { + return 0 + } + size := gateChunkSizeFloor + switch { + case chunk.BifrostPassthroughResponse != nil: + r := chunk.BifrostPassthroughResponse + size += int64(len(r.Body)) + size += gateRawBytes(r.ExtraFields.RawRequest) + gateRawBytes(r.ExtraFields.RawResponse) + case chunk.BifrostChatResponse != nil: + r := chunk.BifrostChatResponse + size += int64(len(r.ID)) + gateChoiceBytes(r.Choices) + size += gateRawBytes(r.ExtraFields.RawRequest) + gateRawBytes(r.ExtraFields.RawResponse) + case chunk.BifrostTextCompletionResponse != nil: + r := chunk.BifrostTextCompletionResponse + size += int64(len(r.ID)) + gateChoiceBytes(r.Choices) + size += gateRawBytes(r.ExtraFields.RawRequest) + gateRawBytes(r.ExtraFields.RawResponse) + case chunk.BifrostResponsesStreamResponse != nil: + r := chunk.BifrostResponsesStreamResponse + size += gateStrPtrBytes(r.Delta) + gateStrPtrBytes(r.Text) + gateStrPtrBytes(r.Arguments) + + gateStrPtrBytes(r.Input) + gateStrPtrBytes(r.Refusal) + gateStrPtrBytes(r.Obfuscation) + + gateStrPtrBytes(r.PartialImageB64) + gateStrPtrBytes(r.Signature) + if r.Item != nil { + size += gateCompositeBytes(r.Item) + } + if r.Part != nil { + size += gateCompositeBytes(r.Part) + } + if r.Response != nil { + size += gateCompositeBytes(r.Response) + } + if len(r.LogProbs) > 0 { + // Same treatment as chat choice logprobs: only present when the + // client requested them, several KB per chunk at high top_logprobs. + size += gateCompositeBytes(r.LogProbs) + } + size += gateRawBytes(r.ExtraFields.RawRequest) + gateRawBytes(r.ExtraFields.RawResponse) + case chunk.BifrostSpeechStreamResponse != nil: + r := chunk.BifrostSpeechStreamResponse + size += int64(len(r.Audio)) + size += gateRawBytes(r.ExtraFields.RawRequest) + gateRawBytes(r.ExtraFields.RawResponse) + case chunk.BifrostTranscriptionStreamResponse != nil: + r := chunk.BifrostTranscriptionStreamResponse + size += gateStrPtrBytes(r.Delta) + int64(len(r.Text)) + size += gateRawBytes(r.ExtraFields.RawRequest) + gateRawBytes(r.ExtraFields.RawResponse) + case chunk.BifrostImageGenerationStreamResponse != nil: + r := chunk.BifrostImageGenerationStreamResponse + size += int64(len(r.B64JSON)) + int64(len(r.URL)) + // The image variant carries direct raw strings separate from ExtraFields. + size += int64(len(r.RawRequest)) + int64(len(r.RawResponse)) + size += gateRawBytes(r.ExtraFields.RawRequest) + gateRawBytes(r.ExtraFields.RawResponse) + case chunk.BifrostError != nil: + e := chunk.BifrostError + if e.Error != nil { + size += int64(len(e.Error.Message)) + } + size += gateRawBytes(e.ExtraFields.RawRequest) + gateRawBytes(e.ExtraFields.RawResponse) + } + return size +} + +// gateChoiceBytes sizes the text carriers of chat/text-completion choices: +// stream deltas (content, reasoning, refusal, tool-call arguments), terminal +// message copies, and text-completion text. +func gateChoiceBytes(choices []schemas.BifrostResponseChoice) int64 { + var size int64 + for i := range choices { + c := &choices[i] + if c.ChatStreamResponseChoice != nil && c.ChatStreamResponseChoice.Delta != nil { + d := c.ChatStreamResponseChoice.Delta + size += gateStrPtrBytes(d.Content) + gateStrPtrBytes(d.Reasoning) + gateStrPtrBytes(d.Refusal) + size += int64(len(d.ExtraContent)) + for j := range d.ToolCalls { + size += int64(len(d.ToolCalls[j].Function.Arguments)) + } + if d.Audio != nil { + size += int64(len(d.Audio.Data)) + int64(len(d.Audio.Transcript)) + } + for j := range d.ReasoningDetails { + rd := &d.ReasoningDetails[j] + size += gateStrPtrBytes(rd.Text) + gateStrPtrBytes(rd.Data) + + gateStrPtrBytes(rd.Signature) + gateStrPtrBytes(rd.Summary) + } + } + if c.LogProbs != nil { + // Only present when the client requested logprobs; several KB per + // chunk at top_logprobs=20, far past the floor. + size += gateCompositeBytes(c.LogProbs) + } + if c.ChatNonStreamResponseChoice != nil && c.ChatNonStreamResponseChoice.Message != nil { + size += gateCompositeBytes(c.ChatNonStreamResponseChoice.Message) + } + if c.TextCompletionResponseChoice != nil { + size += gateStrPtrBytes(c.TextCompletionResponseChoice.Text) + } + } + return size +} + // PauseStream is the Tracer-level entry point for pausing a stream. // Forwards to the per-stream accumulator entry keyed by traceID. func (a *Accumulator) PauseStream(traceID string) { @@ -360,7 +525,12 @@ func (sa *StreamAccumulator) TransformPausedBuffer(transform schemas.PausedStrea sa.mu.Unlock() return fmt.Errorf("paused stream buffer transform scope is invalid") } - original := append([]*schemas.BifrostStreamChunk(nil), sa.gateReplayBuf[start:]...) + original := make([]*schemas.BifrostStreamChunk, len(sa.gateReplayBuf)-start) + var originalBytes int64 + for i, entry := range sa.gateReplayBuf[start:] { + original[i] = entry.chunk + originalBytes += entry.size + } sa.mu.Unlock() result, err := transform(original) @@ -374,10 +544,12 @@ func (sa *StreamAccumulator) TransformPausedBuffer(transform schemas.PausedStrea return fmt.Errorf("paused stream buffer transform release count %d is outside [0,%d]", result.ReleaseCount, len(original)) } - var originalBytes, transformedBytes int64 - for i := range original { - originalBytes += chunkBytes(original[i]) - transformedBytes += chunkBytes(result.Chunks[i]) + // Size the replacements once; cached alongside each installed entry below. + transformedSizes := make([]int64, len(result.Chunks)) + var transformedBytes int64 + for i := range result.Chunks { + transformedSizes[i] = estimateChunkBytes(result.Chunks[i]) + transformedBytes += transformedSizes[i] } sa.mu.Lock() @@ -390,7 +562,7 @@ func (sa *StreamAccumulator) TransformPausedBuffer(transform schemas.PausedStrea return fmt.Errorf("paused stream buffer changed during transformation") } for i := range original { - if sa.gateReplayBuf[currentStart+i] != original[i] { + if sa.gateReplayBuf[currentStart+i].chunk != original[i] { return fmt.Errorf("paused stream buffer changed during transformation") } } @@ -401,7 +573,9 @@ func (sa *StreamAccumulator) TransformPausedBuffer(transform schemas.PausedStrea if nextBytes > gateReplayBufMaxBytes { return fmt.Errorf("transformed paused stream buffer exceeds %d bytes", gateReplayBufMaxBytes) } - copy(sa.gateReplayBuf[currentStart:], result.Chunks) + for i := range result.Chunks { + sa.gateReplayBuf[currentStart+i] = gateReplayEntry{chunk: result.Chunks[i], size: transformedSizes[i]} + } sa.gateReplayBufBytes = nextBytes nextBoundary := currentStart + result.ReleaseCount sa.gateTransformStart = nextBoundary @@ -479,7 +653,7 @@ func (sa *StreamAccumulator) GateSend(chunk *schemas.BifrostStreamChunk, isFinal // or until EndStream is called explicitly. Enforces a 100 MB cap to // prevent unbounded heap growth from a paused-and-forgotten stream. if sa.gateState == StreamStatePaused { - size := chunkBytes(chunk) + size := estimateChunkBytes(chunk) if sa.gateReplayBufBytes+size > gateReplayBufMaxBytes { // Overflow: force-end the gate with a synthetic error so the // consumer is notified and memory is released. Drops this chunk @@ -505,7 +679,7 @@ func (sa *StreamAccumulator) GateSend(chunk *schemas.BifrostStreamChunk, isFinal sa.mu.Unlock() return false } - sa.gateReplayBuf = append(sa.gateReplayBuf, chunk) + sa.gateReplayBuf = append(sa.gateReplayBuf, gateReplayEntry{chunk: chunk, size: size}) sa.gateReplayBufBytes += size if isFinal || isHardErr { sa.gatePendingTerminal = true @@ -530,8 +704,9 @@ func (sa *StreamAccumulator) GateSend(chunk *schemas.BifrostStreamChunk, isFinal // If there's already a flusher or a non-empty buffer, append the final // chunk so it's delivered after pending chunks. Otherwise fast-path send. if sa.gateFlusherOn || len(sa.gateReplayBuf) > 0 { - sa.gateReplayBuf = append(sa.gateReplayBuf, chunk) - sa.gateReplayBufBytes += chunkBytes(chunk) + size := estimateChunkBytes(chunk) + sa.gateReplayBuf = append(sa.gateReplayBuf, gateReplayEntry{chunk: chunk, size: size}) + sa.gateReplayBufBytes += size sa.gateState = StreamStateEnded if sa.gateCond != nil { sa.gateCond.Broadcast() @@ -548,8 +723,9 @@ func (sa *StreamAccumulator) GateSend(chunk *schemas.BifrostStreamChunk, isFinal // so order is preserved between buffered and live chunks. Otherwise pure // passthrough. if len(sa.gateReplayBuf) > 0 { - sa.gateReplayBuf = append(sa.gateReplayBuf, chunk) - sa.gateReplayBufBytes += chunkBytes(chunk) + size := estimateChunkBytes(chunk) + sa.gateReplayBuf = append(sa.gateReplayBuf, gateReplayEntry{chunk: chunk, size: size}) + sa.gateReplayBufBytes += size sa.gateCond.Broadcast() sa.mu.Unlock() return true @@ -564,7 +740,7 @@ func (sa *StreamAccumulator) GateSend(chunk *schemas.BifrostStreamChunk, isFinal // current unevaluated suffix held. It also stops when ctx is done. func (sa *StreamAccumulator) drainBufferLocked() { for len(sa.gateReplayBuf) > 0 && (sa.gateState != StreamStatePaused || sa.gateApprovedPrefix > 0) { - chunk := sa.gateReplayBuf[0] + entry := sa.gateReplayBuf[0] sa.gateReplayBuf = sa.gateReplayBuf[1:] if sa.gateTransformStart > 0 { sa.gateTransformStart-- @@ -573,17 +749,16 @@ func (sa *StreamAccumulator) drainBufferLocked() { sa.gateApprovedPrefix-- } // Decrement bytes per-chunk so the counter stays accurate even if - // the loop exits mid-drain (Pause). Clamp at zero to absorb any - // rounding drift between MarshalJSON sizes captured on append vs. - // re-computed here. - sa.gateReplayBufBytes -= chunkBytes(chunk) + // the loop exits mid-drain (Pause). Sizes are cached at append, so + // this is exact; the clamp is pure defense. + sa.gateReplayBufBytes -= entry.size if sa.gateReplayBufBytes < 0 { sa.gateReplayBufBytes = 0 } ch := sa.gateFlusherCh ctx := sa.gateFlusherCtx sa.mu.Unlock() - ok := sendOrCancel(ctx, ch, chunk) + ok := sendOrCancel(ctx, ch, entry.chunk) sa.mu.Lock() if ok && len(sa.gateReplayBuf) > 0 { ok = sa.waitForReplayIntervalLocked() diff --git a/framework/streaming/gate_test.go b/framework/streaming/gate_test.go index 501ff1ed92f..1131282fbf5 100644 --- a/framework/streaming/gate_test.go +++ b/framework/streaming/gate_test.go @@ -2,6 +2,7 @@ package streaming import ( "context" + "encoding/json" "fmt" "strings" "sync" @@ -32,6 +33,15 @@ func makeChunks(n int) []*schemas.BifrostStreamChunk { return out } +// gateEntries wraps chunks as replay entries with cached sizes, mirroring production appends. +func gateEntries(chunks ...*schemas.BifrostStreamChunk) []gateReplayEntry { + out := make([]gateReplayEntry, len(chunks)) + for i, c := range chunks { + out[i] = gateReplayEntry{chunk: c, size: estimateChunkBytes(c)} + } + return out +} + // recorder consumes chunks off a channel, recording each with its arrival timestamp. type recorder struct { mu sync.Mutex @@ -606,7 +616,7 @@ func TestGate_TransformPausedStreamBufferRejectsConcurrentAppend(t *testing.T) { func TestGate_TransformPausedStreamBufferScopesLatestPause(t *testing.T) { sa := &StreamAccumulator{ gateState: StreamStateActive, - gateReplayBuf: makeChunks(2), + gateReplayBuf: gateEntries(makeChunks(2)...), gateReplayBufBytes: 1, } sa.Pause() @@ -614,7 +624,7 @@ func TestGate_TransformPausedStreamBufferScopesLatestPause(t *testing.T) { t.Fatalf("new pause boundaries = (%d, %d), want transform start 2 and approved prefix 0", sa.gateTransformStart, sa.gateApprovedPrefix) } latest := makeChunks(1)[0] - sa.gateReplayBuf = append(sa.gateReplayBuf, latest) + sa.gateReplayBuf = append(sa.gateReplayBuf, gateEntries(latest)...) seen := 0 err := sa.TransformPausedBuffer(func(buffered []*schemas.BifrostStreamChunk) (schemas.PausedStreamBufferTransformResult, error) { @@ -684,8 +694,8 @@ func TestGate_TransformPausedStreamBufferEnforcesCap(t *testing.T) { sa := &StreamAccumulator{gateState: StreamStateActive} sa.Pause() original := makeChunks(1)[0] - sa.gateReplayBuf = append(sa.gateReplayBuf, original) - originalBytes := chunkBytes(original) + sa.gateReplayBuf = append(sa.gateReplayBuf, gateEntries(original)...) + originalBytes := estimateChunkBytes(original) sa.gateReplayBufBytes = gateReplayBufMaxBytes - 1 err := sa.TransformPausedBuffer(func(buffered []*schemas.BifrostStreamChunk) (schemas.PausedStreamBufferTransformResult, error) { @@ -698,7 +708,7 @@ func TestGate_TransformPausedStreamBufferEnforcesCap(t *testing.T) { if err == nil { t.Fatal("TransformPausedBuffer() error = nil for oversized replacement") } - if sa.gateReplayBuf[0] != original { + if sa.gateReplayBuf[0].chunk != original { t.Fatal("oversized replacement changed the buffered chunk") } } @@ -1775,3 +1785,237 @@ func TestGate_GetAccumulatedResponse_DoesNotEngageGate(t *testing.T) { t.Fatalf("GetAccumulatedResponse must not engage the gate (StreamGated should remain unset)") } } + +// TestGate_EstimateChunkBytesCountsBulkCarriers: the estimator must count the +// fields that grow with response size — passthrough bodies, raw request/response +// bytes, and flat delta/text carriers — without serializing the chunk. Small +// metadata rides the per-chunk floor. +func TestGate_EstimateChunkBytesCountsBulkCarriers(t *testing.T) { + if got := estimateChunkBytes(nil); got != 0 { + t.Fatalf("estimateChunkBytes(nil) = %d, want 0", got) + } + + body := make([]byte, 10*1024) + pt := &schemas.BifrostStreamChunk{ + BifrostPassthroughResponse: &schemas.BifrostPassthroughResponse{Body: body}, + } + if got := estimateChunkBytes(pt); got < int64(len(body)) { + t.Fatalf("passthrough estimate = %d, want >= %d (body length)", got, len(body)) + } + + content := strings.Repeat("c", 8*1024) + args := strings.Repeat("a", 4*1024) + chat := &schemas.BifrostStreamChunk{ + BifrostChatResponse: &schemas.BifrostChatResponse{ + Choices: []schemas.BifrostResponseChoice{{ + ChatStreamResponseChoice: &schemas.ChatStreamResponseChoice{ + Delta: &schemas.ChatStreamResponseChoiceDelta{ + Content: &content, + ToolCalls: []schemas.ChatAssistantMessageToolCall{{ + Function: schemas.ChatAssistantMessageToolCallFunction{Arguments: args}, + }}, + }, + }, + }}, + }, + } + if got := estimateChunkBytes(chat); got < int64(len(content)+len(args)) { + t.Fatalf("chat estimate = %d, want >= %d (delta content + tool args)", got, len(content)+len(args)) + } + + raw := json.RawMessage(make([]byte, 6*1024)) + withRaw := &schemas.BifrostStreamChunk{ + BifrostChatResponse: &schemas.BifrostChatResponse{ + ExtraFields: schemas.BifrostResponseExtraFields{RawResponse: raw}, + }, + } + if got := estimateChunkBytes(withRaw); got < int64(len(raw)) { + t.Fatalf("raw-response estimate = %d, want >= %d", got, len(raw)) + } + + delta := strings.Repeat("d", 5*1024) + resp := &schemas.BifrostStreamChunk{ + BifrostResponsesStreamResponse: &schemas.BifrostResponsesStreamResponse{Delta: &delta}, + } + if got := estimateChunkBytes(resp); got < int64(len(delta)) { + t.Fatalf("responses estimate = %d, want >= %d (delta)", got, len(delta)) + } + + audioData := strings.Repeat("A", 16*1024) + audio := &schemas.BifrostStreamChunk{ + BifrostChatResponse: &schemas.BifrostChatResponse{ + Choices: []schemas.BifrostResponseChoice{{ + ChatStreamResponseChoice: &schemas.ChatStreamResponseChoice{ + Delta: &schemas.ChatStreamResponseChoiceDelta{ + Audio: &schemas.ChatAudioMessageAudio{Data: audioData}, + }, + }, + }}, + }, + } + if got := estimateChunkBytes(audio); got < int64(len(audioData)) { + t.Fatalf("audio-delta estimate = %d, want >= %d (b64 audio data)", got, len(audioData)) + } + + rawImage := strings.Repeat("r", 8*1024) + img := &schemas.BifrostStreamChunk{ + BifrostImageGenerationStreamResponse: &schemas.BifrostImageGenerationStreamResponse{ + RawRequest: rawImage, + RawResponse: rawImage, + }, + } + if got := estimateChunkBytes(img); got < int64(2*len(rawImage)) { + t.Fatalf("image estimate = %d, want >= %d (direct raw request+response strings)", got, 2*len(rawImage)) + } + + lp := make([]schemas.ResponsesOutputMessageContentTextLogProb, 256) + for i := range lp { + lp[i] = schemas.ResponsesOutputMessageContentTextLogProb{Token: strings.Repeat("t", 32)} + } + respLP := &schemas.BifrostStreamChunk{ + BifrostResponsesStreamResponse: &schemas.BifrostResponsesStreamResponse{LogProbs: lp}, + } + if got := estimateChunkBytes(respLP); got < int64(256*32) { + t.Fatalf("responses logprobs estimate = %d, want >= %d", got, 256*32) + } + + structuredRaw := map[string]interface{}{"blob": strings.Repeat("m", 4*1024)} + mapRaw := &schemas.BifrostStreamChunk{ + BifrostChatResponse: &schemas.BifrostChatResponse{ + ExtraFields: schemas.BifrostResponseExtraFields{RawResponse: structuredRaw}, + }, + } + if got := estimateChunkBytes(mapRaw); got < int64(4*1024) { + t.Fatalf("structured raw estimate = %d, want >= %d (sonic fallback for non-bytes carriers)", got, 4*1024) + } + + var boxed interface{} = json.RawMessage(make([]byte, 3*1024)) + boxedRaw := &schemas.BifrostStreamChunk{ + BifrostChatResponse: &schemas.BifrostChatResponse{ + ExtraFields: schemas.BifrostResponseExtraFields{RawResponse: &boxed}, + }, + } + if got := estimateChunkBytes(boxedRaw); got < int64(3*1024) { + t.Fatalf("pointer-boxed raw estimate = %d, want >= %d (unwrapped *interface{})", got, 3*1024) + } + var nilBoxed *interface{} + if got := gateRawBytes(nilBoxed); got != 0 { + t.Fatalf("nil *interface{} raw = %d, want 0", got) + } +} + +// TestGate_PausedBufferAccountingUsesCachedSizes: sizes are computed once at +// append and drained exactly, so the byte counter returns to zero without any +// re-serialization of held chunks. +func TestGate_PausedBufferAccountingUsesCachedSizes(t *testing.T) { + a := newTestAccumulator(t) + traceID := "trace-cached-sizes" + ctx := schemas.NewBifrostContext(context.Background(), time.Time{}) + r := newRecorder(8) + defer r.close() + + chunk := &schemas.BifrostStreamChunk{ + BifrostPassthroughResponse: &schemas.BifrostPassthroughResponse{Body: make([]byte, 2048)}, + } + want := estimateChunkBytes(chunk) + + a.PauseStream(traceID) + if !a.GateSend(traceID, chunk, false, false, r.ch, ctx) { + t.Fatal("paused GateSend returned false") + } + sa := mustGet(t, a, traceID) + sa.mu.Lock() + gotBytes := sa.gateReplayBufBytes + gotCached := sa.gateReplayBuf[0].size + sa.mu.Unlock() + if gotBytes != want || gotCached != want { + t.Fatalf("buffered bytes = %d, cached size = %d, want both %d", gotBytes, gotCached, want) + } + + a.ResumeStream(traceID) + if !r.waitFor(1, time.Second) { + t.Fatal("chunk was not replayed after resume") + } + sa.mu.Lock() + after := sa.gateReplayBufBytes + sa.mu.Unlock() + if after != 0 { + t.Fatalf("buffered bytes after drain = %d, want 0", after) + } + a.EndStream(traceID, nil) + sa.WaitForFlusher() +} + +// benchChunks builds a realistic full-hold workload: passthrough chunks with +// SSE-sized bodies, as buffered on the native Anthropic (claude-cli) path. +func benchChunks(n, bodyBytes int) []*schemas.BifrostStreamChunk { + out := make([]*schemas.BifrostStreamChunk, n) + for i := range out { + out[i] = &schemas.BifrostStreamChunk{ + BifrostPassthroughResponse: &schemas.BifrostPassthroughResponse{Body: make([]byte, bodyBytes)}, + } + } + return out +} + +// BenchmarkChunkSizeEstimator measures the new zero-marshal size accounting. +func BenchmarkChunkSizeEstimator(b *testing.B) { + chunks := benchChunks(1, 1400) + b.ReportAllocs() + for i := 0; i < b.N; i++ { + _ = estimateChunkBytes(chunks[0]) + } +} + +// BenchmarkChunkSizeMarshal measures the pre-change size accounting +// (MarshalJSON per lookup), kept here as the comparison baseline. +func BenchmarkChunkSizeMarshal(b *testing.B) { + chunks := benchChunks(1, 1400) + b.ReportAllocs() + for i := 0; i < b.N; i++ { + data, err := chunks[0].MarshalJSON() + if err != nil { + b.Fatal(err) + } + _ = int64(len(data)) + } +} + +// BenchmarkGatePauseHoldDrain exercises the full production cycle: pause, +// buffer n chunks, resume, drain to a consumer. Allocation count here is what +// a held stream pays end to end under the gate. +func BenchmarkGatePauseHoldDrain(b *testing.B) { + const n, bodyBytes = 500, 1400 + ctx := schemas.NewBifrostContext(context.Background(), time.Time{}) + b.ReportAllocs() + for i := 0; i < b.N; i++ { + b.StopTimer() + a := NewAccumulator(nil, bifrost.NewDefaultLogger(schemas.LogLevelError)) + traceID := fmt.Sprintf("bench-%d", i) + chunks := benchChunks(n, bodyBytes) + ch := make(chan *schemas.BifrostStreamChunk, n+1) + done := make(chan struct{}) + go func() { + for range ch { + } + close(done) + }() + b.StartTimer() + + a.PauseStream(traceID) + for _, c := range chunks { + if !a.GateSend(traceID, c, false, false, ch, ctx) { + b.Fatal("GateSend returned false") + } + } + a.ResumeStream(traceID) + a.EndStream(traceID, nil) + a.WaitForFlusher(traceID) + + b.StopTimer() + close(ch) + <-done + a.Cleanup() + b.StartTimer() + } +} diff --git a/framework/streaming/types.go b/framework/streaming/types.go index b3601e3cc67..723141b5cd6 100644 --- a/framework/streaming/types.go +++ b/framework/streaming/types.go @@ -180,8 +180,8 @@ type StreamAccumulator struct { // buffer and transitions the gate to Ended. gatePendingTerminal bool gateSeq int // monotonic, bumped on every GateSend - gateReplayBuf []*schemas.BifrostStreamChunk // wire-format chunks captured while paused - gateReplayBufBytes int64 // sum of MarshalJSON sizes of chunks in gateReplayBuf; capped by gateReplayBufMaxBytes + gateReplayBuf []gateReplayEntry // wire-format chunks captured while paused, each with its MarshalJSON size cached at append + gateReplayBufBytes int64 // sum of cached sizes in gateReplayBuf; capped by gateReplayBufMaxBytes gateReplayEventInterval time.Duration // delay between buffered events after paced resume is armed gateCond *sync.Cond // wakes flusher on Resume / End / append-while-active gateEndError *schemas.BifrostError // delivered as terminal chunk if EndStream(err) was called with non-nil diff --git a/framework/version b/framework/version index 943f9cbc4ec..0a182f2e3af 100644 --- a/framework/version +++ b/framework/version @@ -1 +1 @@ -1.7.1 +1.7.2 \ No newline at end of file From 76a56019d77f2d85732f5993ad985557a61e5989 Mon Sep 17 00:00:00 2001 From: akshaydeo Date: Sat, 19 Sep 2026 03:30:48 +0530 Subject: [PATCH 009/118] adds more gating test cases --- framework/streaming/gate_test.go | 255 +++++++++++++++++++++++++++++++ 1 file changed, 255 insertions(+) diff --git a/framework/streaming/gate_test.go b/framework/streaming/gate_test.go index 1131282fbf5..7ce94dbbb34 100644 --- a/framework/streaming/gate_test.go +++ b/framework/streaming/gate_test.go @@ -1,6 +1,7 @@ package streaming import ( + "bytes" "context" "encoding/json" "fmt" @@ -2019,3 +2020,257 @@ func BenchmarkGatePauseHoldDrain(b *testing.B) { b.StartTimer() } } + +// ssePassthroughChunk wraps one SSE event as a passthrough stream chunk — the +// shape the native Anthropic path buffers under guardrails full-hold. +func ssePassthroughChunk(event, data string) *schemas.BifrostStreamChunk { + return &schemas.BifrostStreamChunk{ + BifrostPassthroughResponse: &schemas.BifrostPassthroughResponse{ + StatusCode: 200, + Body: []byte("event: " + event + "\ndata: " + data + "\n\n"), + }, + } +} + +// toolCarryingSSEStream builds a realistic tool-carrying Anthropic SSE stream +// whose input_json_delta fragments carry a secret a guardrail would redact. +func toolCarryingSSEStream() []*schemas.BifrostStreamChunk { + return []*schemas.BifrostStreamChunk{ + ssePassthroughChunk("message_start", `{"type":"message_start","message":{"id":"msg_01","role":"assistant"}}`), + ssePassthroughChunk("content_block_start", `{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_01","name":"bash","input":{}}}`), + ssePassthroughChunk("content_block_delta", `{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"command\":\"curl -H "}}`), + ssePassthroughChunk("content_block_delta", `{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"'Authorization: Bearer sk-ant-SECRET123'"}}`), + ssePassthroughChunk("content_block_delta", `{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":" https://api.example.com\"}"}}`), + ssePassthroughChunk("content_block_stop", `{"type":"content_block_stop","index":0}`), + ssePassthroughChunk("message_delta", `{"type":"message_delta","delta":{"stop_reason":"tool_use"}}`), + ssePassthroughChunk("message_stop", `{"type":"message_stop"}`), + } +} + +func passthroughBody(c *schemas.BifrostStreamChunk) []byte { + if c != nil && c.BifrostPassthroughResponse != nil { + return c.BifrostPassthroughResponse.Body + } + return nil +} + +// TestGate_FullHoldRedactReplaySSE walks the guardrails full-hold seam with a +// realistic claude-cli stream: pause before the first chunk, copy-on-write +// redaction via TransformPausedStreamBuffer, then replay. Nothing may escape +// while held, replay must preserve SSE order, the secret must be gone from the +// wire, and the originally buffered chunks must be untouched. +func TestGate_FullHoldRedactReplaySSE(t *testing.T) { + a := newTestAccumulator(t) + traceID := "trace-full-hold-redact" + ctx := schemas.NewBifrostContext(context.Background(), time.Time{}) + r := newRecorder(32) + defer r.close() + chunks := toolCarryingSSEStream() + + a.PauseStream(traceID) + for i, c := range chunks { + if !a.GateSend(traceID, c, i == len(chunks)-1, false, r.ch, ctx) { + t.Fatalf("paused GateSend returned false at %d", i) + } + } + if got, _ := r.snapshot(); len(got) != 0 { + t.Fatalf("%d chunks escaped while held", len(got)) + } + + err := a.TransformPausedStreamBuffer(traceID, func(buffered []*schemas.BifrostStreamChunk) (schemas.PausedStreamBufferTransformResult, error) { + out := append([]*schemas.BifrostStreamChunk(nil), buffered...) + for i, c := range buffered { + body := passthroughBody(c) + if !bytes.Contains(body, []byte("sk-ant-SECRET123")) { + continue + } + cp := *c + pt := *c.BifrostPassthroughResponse + pt.Body = bytes.ReplaceAll(body, []byte("sk-ant-SECRET123"), []byte("[REDACTED:api-key]")) + cp.BifrostPassthroughResponse = &pt + out[i] = &cp + } + return schemas.PausedStreamBufferTransformResult{Chunks: out, ReleaseCount: len(out)}, nil + }) + if err != nil { + t.Fatalf("TransformPausedStreamBuffer() error = %v", err) + } + + a.ResumeStream(traceID) + if !r.waitFor(len(chunks), time.Second) { + got, _ := r.snapshot() + t.Fatalf("replay delivered %d of %d chunks", len(got), len(chunks)) + } + sa := mustGet(t, a, traceID) + sa.WaitForFlusher() + + got, _ := r.snapshot() + var wire []byte + for i, c := range got { + wantEvent := bytes.SplitN(passthroughBody(chunks[i]), []byte("\n"), 2)[0] + if !bytes.HasPrefix(passthroughBody(c), wantEvent) { + t.Fatalf("replay order broken at %d", i) + } + wire = append(wire, passthroughBody(c)...) + } + if bytes.Contains(wire, []byte("sk-ant-SECRET123")) { + t.Fatal("secret leaked to the wire") + } + if !bytes.Contains(wire, []byte("[REDACTED:api-key]")) { + t.Fatal("redaction placeholder missing from the wire") + } + if !bytes.Contains(passthroughBody(chunks[3]), []byte("sk-ant-SECRET123")) { + t.Fatal("copy-on-write violated: original buffered chunk was mutated") + } + if !a.IsStreamEnded(traceID) { + t.Fatal("gate not ended after terminal replay") + } +} + +// TestGate_BlockVerdictClearsBufferAndDeliversTerminal mirrors a guardrail +// block: held content is dropped via ClearPausedStreamBuffer and the client +// receives only the synthetic terminal error; later sends are rejected. +func TestGate_BlockVerdictClearsBufferAndDeliversTerminal(t *testing.T) { + a := newTestAccumulator(t) + traceID := "trace-block-verdict" + ctx := schemas.NewBifrostContext(context.Background(), time.Time{}) + r := newRecorder(32) + defer r.close() + + a.PauseStream(traceID) + for _, c := range toolCarryingSSEStream() { + if !a.GateSend(traceID, c, false, false, r.ch, ctx) { + t.Fatal("paused GateSend returned false") + } + } + if err := a.ClearPausedStreamBuffer(traceID); err != nil { + t.Fatalf("ClearPausedStreamBuffer() error = %v", err) + } + a.EndStream(traceID, &schemas.BifrostError{ + IsBifrostError: true, + Error: &schemas.ErrorField{Message: "blocked by guardrail policy"}, + }) + sa := mustGet(t, a, traceID) + sa.WaitForFlusher() + if !r.waitFor(1, time.Second) { + t.Fatal("block terminal was not delivered") + } + + got, _ := r.snapshot() + if len(got) != 1 || got[0].BifrostError == nil || + !strings.Contains(got[0].BifrostError.Error.Message, "blocked by guardrail policy") { + t.Fatalf("want exactly the block terminal, got %d chunks", len(got)) + } + if a.GateSend(traceID, ssePassthroughChunk("ping", `{}`), false, false, r.ch, ctx) { + t.Fatal("send accepted after block terminal") + } +} + +// TestGate_OverflowCapWithSSEPayloads drives 5MB SSE events into a paused gate +// and asserts the cap trips right at the 100MB boundary, the client is told via +// paused_replay_buffer_overflow, and the gate is terminally closed. +func TestGate_OverflowCapWithSSEPayloads(t *testing.T) { + a := newTestAccumulator(t) + traceID := "trace-overflow-sse" + ctx := schemas.NewBifrostContext(context.Background(), time.Time{}) + r := newRecorder(64) + defer r.close() + + a.PauseStream(traceID) + big := strings.Repeat("x", 5*1024*1024) + refusedAt := -1 + for i := 0; i < 40; i++ { + if !a.GateSend(traceID, ssePassthroughChunk("content_block_delta", big), false, false, r.ch, ctx) { + refusedAt = i + break + } + } + // ~5MB payload + 512B floor per chunk: refusal expected at the 100MB line. + if refusedAt < 18 || refusedAt > 21 { + t.Fatalf("cap refused at chunk %d (~%dMB), want ~20 (100MB)", refusedAt, refusedAt*5) + } + sa := mustGet(t, a, traceID) + sa.WaitForFlusher() + // On overflow the flusher drains the already-buffered chunks first and + // delivers the synthetic terminal last, so poll until it lands rather than + // snapshotting after the first chunk (the recorder appends asynchronously). + sawOverflow := false + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) && !sawOverflow { + got, _ := r.snapshot() + for _, c := range got { + if c.BifrostError != nil && c.BifrostError.Error != nil && c.BifrostError.Error.Type != nil && + *c.BifrostError.Error.Type == "paused_replay_buffer_overflow" { + sawOverflow = true + } + } + if !sawOverflow { + time.Sleep(2 * time.Millisecond) + } + } + if !sawOverflow { + t.Fatal("client never received paused_replay_buffer_overflow") + } + if !a.IsStreamEnded(traceID) { + t.Fatal("gate not ended after overflow") + } + if a.GateSend(traceID, ssePassthroughChunk("ping", `{}`), false, false, r.ch, ctx) { + t.Fatal("send accepted after overflow") + } +} + +// TestGate_TransformInPlaceMutationIsRepriced pins the accounting guarantee +// for a transform flow that modifies held chunks IN PLACE (through the original +// pointers) instead of copy-on-write: because sizes are recomputed from +// result.Chunks after the callback returns, in-place growth inside the +// callback is still re-priced before install and delivered mutated on replay. +func TestGate_TransformInPlaceMutationIsRepriced(t *testing.T) { + a := newTestAccumulator(t) + traceID := "trace-inplace-reprice" + ctx := schemas.NewBifrostContext(context.Background(), time.Time{}) + r := newRecorder(16) + defer r.close() + + a.PauseStream(traceID) + chunk := ssePassthroughChunk("content_block_delta", `{"delta":{"partial_json":"{\"cmd\":\"ls\"}"}}`) + if !a.GateSend(traceID, chunk, false, false, r.ch, ctx) { + t.Fatal("paused GateSend returned false") + } + sa := mustGet(t, a, traceID) + sa.mu.Lock() + before := sa.gateReplayBufBytes + sa.mu.Unlock() + + const grow = 64 * 1024 + err := a.TransformPausedStreamBuffer(traceID, func(buffered []*schemas.BifrostStreamChunk) (schemas.PausedStreamBufferTransformResult, error) { + pt := buffered[0].BifrostPassthroughResponse + pt.Body = append(pt.Body, bytes.Repeat([]byte("z"), grow)...) // in-place, same pointer + return schemas.PausedStreamBufferTransformResult{Chunks: buffered, ReleaseCount: len(buffered)}, nil + }) + if err != nil { + t.Fatalf("in-place transform rejected: %v", err) + } + + sa.mu.Lock() + after := sa.gateReplayBufBytes + entrySize := sa.gateReplayBuf[0].size + entryEstimate := estimateChunkBytes(sa.gateReplayBuf[0].chunk) + sa.mu.Unlock() + if after < before+grow { + t.Fatalf("in-place growth not re-priced: before=%d after=%d grow=%d", before, after, grow) + } + if entrySize != entryEstimate { + t.Fatalf("cached size %d diverges from fresh estimate %d after in-place transform", entrySize, entryEstimate) + } + + a.ResumeStream(traceID) + if !r.waitFor(1, time.Second) { + t.Fatal("mutated chunk was not replayed") + } + got, _ := r.snapshot() + if !bytes.Contains(passthroughBody(got[0]), bytes.Repeat([]byte("z"), 16)) { + t.Fatal("replayed chunk does not carry the in-place mutation") + } + a.EndStream(traceID, nil) + sa.WaitForFlusher() +} From 64854d359af385df19ffc6ae4b091253c1eccee8 Mon Sep 17 00:00:00 2001 From: Akshay Deo Date: Fri, 18 Sep 2026 15:20:00 -0700 Subject: [PATCH 010/118] enterprise docs (#7334) --- docs/changelogs/edge-v0.9.0.mdx | 31 ++ docs/changelogs/ent-v2.2.1.mdx | 497 ++++++++++++++++++++++++++++++++ docs/docs.json | 2 + 3 files changed, 530 insertions(+) create mode 100644 docs/changelogs/edge-v0.9.0.mdx create mode 100644 docs/changelogs/ent-v2.2.1.mdx diff --git a/docs/changelogs/edge-v0.9.0.mdx b/docs/changelogs/edge-v0.9.0.mdx new file mode 100644 index 00000000000..fb8905d4ad9 --- /dev/null +++ b/docs/changelogs/edge-v0.9.0.mdx @@ -0,0 +1,31 @@ +--- +title: "v0.9.0" +description: "Edge v0.9.0 changelog - 2026-09-19" +--- + + + +## Changelog + +v0.9.0 makes pass-through mode a first-class routing state. A single per-application routing policy now governs every request path, and applications left in pass-through are continuously cleaned of stale Edge credentials instead of only once at startup. Cursor gains context-limit recovery through tool-output compaction and rolling conversation summarization. On macOS, administrators get a fleet-level capture kill switch that can be toggled at runtime without restarting anything. Across all supported coding tools, unknown provider event types now pass through as-is instead of failing the request, so new provider features keep working before Edge learns about them. + +## ✨ Features + +- **Per-Application Routing Policy** - A single immutable policy snapshot now decides gateway versus pass-through mode for every request path: HTTP, HTTP/2, WebSocket, model catalog, and Claude Cowork. Application aliases are folded consistently (`codex` resolves to `codex-desktop`, `claude-code-cli` to `claude-code`), an in-flight turn keeps the policy it started with even if the policy changes mid-turn, and the tray and diagnostics now show the applied routing state with its version, last sync time, and any reconciliation error. +- **Continuous Pass-Through Credential Cleanup** - Applications in pass-through mode (Claude Code, Codex, OpenCode, Gemini CLI) are now watched continuously, so a stale Edge credential restored by a client after the policy switch is removed on the next reconcile cycle rather than persisting until the next restart. Cleanup strips only the Edge-planted credential and never claims ownership of, alters, or deletes the user's own settings files. +- **Fleet Capture Kill Switch on macOS** - Administrators can now disable Edge's macOS network capture fleet-wide at runtime. The daemon pushes the policy to the capture extension live, flows are declined and passed through normally while capture is disabled, and re-enabling requires no reinstall, restart, or re-approval. Older agent versions ignore the new field and behave as before. +- **Cursor Context-Limit Recovery** - When a Cursor conversation grows past the model's input limit, Edge now recovers instead of failing the turn: older completed research tool outputs are compacted into explicitly incomplete previews and the request is retried once, and if that is not enough a bounded rolling summary of the conversation is generated through the same governed model. Summaries are checkpointed atomically, so cold resumes stay durable, and accepted streams and partial tool calls are never retried. +- **Gemini CLI Tool Delivery Gate with Incremental Streaming** - Gemini CLI requests now flow through the shared tool delivery gate: text, reasoning, and lifecycle events stream to the client immediately, while function calls are held until they pass policy inspection. A heartbeat keeps the connection alive during upstream and policy waits, and fixed response deadlines across the agent are replaced by request cancellation so long-running turns no longer hit arbitrary cutoffs. +- **Forward Compatibility with Unknown Provider Events** - Unknown streaming event types, content block variants, output item types, and finish reasons are now passed through as opaque data instead of being rejected, across all supported coding tools. Only recognized tool-call shapes enter policy inspection, and an unknown event can never complete or approve a pending tool call. +- **Unknown MCP Calls Continue as Observations** - A tool call for an MCP server that cannot be matched to the device inventory (stale inventory, removed server, unrecognized name) is now logged as an observation with full attribution instead of being hard-denied, and the rest of the batch continues under normal enforcement. Explicit policy denials are still enforced strictly. + +## 🐞 Fixed + +- **Cursor Short Model Aliases Failed to Resolve** - Selecting a short alias from Cursor's model picker could fail model resolution; unresolved names are now matched against the live model catalog's alias lists instead of guessing a provider. +- **Malformed Policy Messages Could Re-Enable Capture** - A capture policy message with a missing or wrongly typed field could unintentionally re-enable capture, and a policy update racing a connection handshake could leave a stale policy state; both paths are now hardened and covered by conformance tests. +- **Unknown Cursor Tool Calls Ended the Conversation** - A tool call Edge could not resolve terminated the Cursor run; it now returns an inline `tool_unavailable` result so the conversation continues, ambiguous tool names are no longer resolved by guessing, and an unconfirmed mutating tool on resume receives an explicit unknown-outcome message instead of failing closed. +- **Cursor Error Messages Buried Under Diagnostics** - Upstream error responses that append large diagnostic payloads after the structured message are now parsed for the message first, so the actual explanation is preserved without draining multi-megabyte bodies. +- **Model Catalog Fetch Timeouts Too Aggressive** - Model catalog fetches now allow 30 seconds instead of 5 to 10, fixing empty model pickers on slow networks. +- **Startup Log Noise** - Removed repetitive daemon log bursts emitted during startup on macOS and Linux. + + diff --git a/docs/changelogs/ent-v2.2.1.mdx b/docs/changelogs/ent-v2.2.1.mdx new file mode 100644 index 00000000000..5be5f268d6d --- /dev/null +++ b/docs/changelogs/ent-v2.2.1.mdx @@ -0,0 +1,497 @@ +--- +title: "v2.2.1" +description: "v2.2.1 changelog - 2026-09-19" +--- + + + +## Changelog + +v2.2.1 moves the enterprise gateway onto OSS transports v2.2.1. API key scopes are now clamped to the caller's own permissions at create and update, closing a privilege escalation where any key-creating user could mint a token carrying every permission in the system. Virtual key search now matches the assigned user's name or email, and the assignee column is resolved in one batched query per page instead of one request per row. The OSS release also carries three security fixes: resource IDs are validated before they enter outbound URL paths, Bedrock S3 bucket names can no longer redirect signed requests to a caller-chosen host, and the billing idempotency key can no longer be forged through the `x-request-id` header. + +## ✨ Features + +- **API Key Scope Ceiling** - Creating or updating an API key now clamps the granted scopes to the permissions the caller holds. Unknown permission ids are rejected with 400, scopes the caller does not hold are rejected with 403 naming the offending permissions, and a key's own scopes win the permission check even when the creator's role is broader. Local-admin and auth-disabled requests keep their existing bypass. +- **Scope Ceiling in the API Key Picker** - The scope picker shows the ceiling before save: permissions above the caller's own are rendered locked with a lock icon and cannot be toggled on, while a locked scope already on the key stays removable so access can always be lowered. If the server still returns a 403 because the permission map went stale, the denied scopes are highlighted inline and the caller's permissions are re-fetched. The picker is built on a shared `PermissionMatrix` component with search and per-resource bulk select, now used by both API key scopes and role permissions. +- **Virtual Key Assignee Search and Batched Resolution** - Virtual key search now matches the assigned user's name or email through the enterprise store, which previously delegated search to the base store and silently dropped the assignee filter. The "Assigned To" column on the list page and in CSV export is resolved with a single batched query per page instead of one request per key. +- **Access-Profile MCP Grants in DAC Scopes** - Own-data and team-data scoped reads now treat access-profile grants as a first-class visibility path for MCP servers and Virtual MCPs. An MCP server granted exclusively through a user's access profile was previously invisible to that user under DAC scoping, because access-profile-managed virtual keys intentionally carry no VK-to-MCP assignment rows. +- **Unknown MCP Calls Recorded as Observations** - When an MCP tool call cannot be matched to any known installation in the device inventory (stale inventory, removed server, unrecognized name), the gateway now records it as an observation with `status=unknown` and full attribution metadata instead of issuing a hard denial, and the rest of the batch continues under normal enforcement. True policy denials and guardrail failures are still enforced strictly. + +## 🌎 Open Source Features + +- **Virtual Key Assignees and Expanded Search** - The virtual key list resolves `assigned_user` for every row via a single batch lookup, and VK search matches team, customer and user names in addition to key fields, so keys can be found by who owns them. +- **User Attribution in Prometheus Metrics** - The telemetry plugin's metrics export includes user id and user name labels, so per-user usage and error rates can be graphed and alerted on directly. +- **Prompt Cache Breakpoints Capability** - A `SupportsPromptCacheBreakpoints` model capability with a name-based fallback for OpenRouter gates `prompt_cache_breakpoint` forwarding, so OpenRouter's non-Claude models stop rejecting requests that carry it. +- **MCP Observed Latency** - Native (observed) MCP tool-call latency from `observed_latency_ms` is surfaced in the logs duration column and detail sheet, kept clearly distinct from true execution time and never fabricating a synthetic start timestamp on the timeline. + +## 🐞 Fixed + +- **Access Profile Role-Detach Revocation Scope** - Detaching a role from an access profile now revokes the profile only from the detached role's users, sparing users whose remaining role still grants the profile. +- **Resource ID Path Injection (OSS)** - Caller-supplied resource IDs (batch, file, video, container, response IDs, cached content names) were interpolated into outbound URL paths unvalidated; a crafted ID could redirect the request to an unintended upstream endpoint. A central `EscapeResourceID` helper validates and escapes every such ID. +- **Bedrock S3 SSRF (OSS)** - A caller-supplied `s3://` file ID or `s3_bucket` param could control the upstream host of a SigV4-signed connection; bucket names are now validated with a DNS-compatible regex. +- **Caller-Forged Billing Idempotency (OSS)** - The billing idempotency key included the caller-controllable `x-request-id`, so two requests sharing a chosen ID collided and the second was never charged. An internally minted `BillingNonce` is mixed into the key, making it unforgeable. +- **Abandoned Non-Streaming Request Hung Forever (OSS)** - The worker kept a `ctx.Done()` arm on an already-claimed delivery send, so a non-streaming caller could hang indefinitely. +- **Repeated Empty Thinking Blocks (OSS)** - Streaming chat chunks emitted empty reasoning/message fields on every content delta, which clients rendered as repeated empty thinking blocks. +- **Claude Code Thread Continuations Behind Key Rotation (OSS)** - Server-side conversation threads are bound to the upstream account that creates them, which per-request key selection cannot guarantee; the Anthropic integration now declares itself stateless, refusing continuations with a 400 `thread_unsupported_request` so the client resends the turn in full, and stripping `thread` from create requests so no orphaned thread state accumulates upstream. +- **Gemini Flash-Lite Minimal Thinking Promoted to Low (OSS)** - `gemini-3.1-flash-lite` requests silently promoted `minimal` thinking to `low` on Gemini and Vertex; the text model's four supported levels are now registered. +- **Config.json Virtual Key Limits Broke Under UI Edits (OSS)** - VK rate limits and budgets created via the config.json standalone-limits flow and then edited through the UI produced duplicate, conflicting ownership records; a migration consolidates ownership into VK-scoped model configs while preserving usage counters. +- **Bedrock Service Tier Rejections (OSS)** - Service tier forwarding for Bedrock is now gated on explicit model capability metadata, failing closed when none exists. +- **Anthropic Root-Level Tool Schema Compositions (OSS)** - Root-level `oneOf`/`anyOf`/`allOf` in a tool's `input_schema` are rewritten into a flat object schema before dispatch, unblocking tools that Anthropic would otherwise reject. +- **GenAI Streaming TTS (OSS)** - Speech stream chunks routed through the `/genai` integration had no converter registered; streaming TTS now works, and a missing stream converter returns a clean error instead of panicking. +- **xAI Usage and Cost (OSS)** - `completion_tokens` now folds reasoning tokens in so token totals add up, and streaming cost normalization preserves xAI's authoritative cost ticks instead of falling back to catalog pricing. +- **OpenAI-Only `search_content_types` Forwarded Everywhere (OSS)** - The field is gated behind a per-provider capability check so Bedrock and other OpenAI-compatible backends receive a clean `web_search` tool. +- **Claude Code `diagnostics` Field Rejected by Non-Native Providers (OSS)** - `diagnostics.previous_message_id` is stripped for providers that do not support it, fixing 400s on Bedrock, Vertex and Azure. +- **Unreadable Bedrock Error Logs (OSS)** - AWS's flat `{"message", "__type"}` error shape now seeds the error message, and the logs UI falls back to the raw provider error body when no message could be extracted. +- **Anthropic `container` Param Dropped (OSS)** - The string-form `container` param on `/anthropic/v1/messages` is now carried through the round trip, so container reuse stops provisioning a fresh container every time. +- **Responses API Finish Reason Missing From OTEL Spans (OSS)** - `gen_ai.response.finish_reason` is now emitted on the `llm.call` span for `/v1/responses` requests, matching the chat path. +- **Cohere Fallback Response Shape (OSS)** - Responses served by a fallback from a non-Cohere provider are converted to the Cohere v2 shape instead of returning the raw normalized response. +- **Skill Serving Race (OSS)** - Fixed a race condition in the skill serving handler. +- **Log and Dashboard Label Truncation (OSS)** - Long model and provider labels truncate from the start so the distinctive suffix stays visible, the logs model column is wider, and key picker options are keyed by id so duplicate labels stop highlighting together. + +## 🗄️ Database Migrations + +- **migrate_vk_standalone_limits_to_model_configs** - Consolidates virtual key standalone budgets and rate limits into VK-scoped model configs: standalone budgets are re-pointed to the VK's top-level model config (created if missing, usage preserved), orphaned duplicate UUID rate-limit rows created by the UI are deleted with the model config re-pointed to the canonical config.json row, and `vk.rate_limit_id` is cleared. + +## 🐙 Closed OSS Issues + +- [#123](https://github.com/maximhq/bifrost/issues/123) - Files API Support +- [#5707](https://github.com/maximhq/bifrost/issues/5707) - string-form `container` param on /anthropic/v1/messages is silently dropped - container reuse provisions a fresh container +- [#7204](https://github.com/maximhq/bifrost/issues/7204) - Responses API path never sets gen_ai.response.finish_reason in OTEL traces (refusals invisible to observability) +- [#7287](https://github.com/maximhq/bifrost/issues/7287) - Gemini 3.1 Flash-Lite minimal thinking is silently promoted to low +- [#7294](https://github.com/maximhq/bifrost/issues/7294) - Streaming chat chunks emit empty reasoning/message fields on every content delta, causing repeated thinking blocks +- [#7308](https://github.com/maximhq/bifrost/issues/7308) - non-streaming caller hangs forever - worker keeps a ctx.Done() arm on a claimed delivery send (regression from #6972) + +## 📀 Base OSS version + +`transports/v2.2.1` (pinned as `github.com/maximhq/bifrost/transports v1.6.12-0.20260918214249-f44b5410fb7b`), with core `v1.9.1`, framework `v1.7.2`, governance `v1.8.1`, and logging `v1.8.1`. + +## 🔌 If you are compiling plugin against this release - use following deps + +```go +module github.com/maximhq/bifrost-enterprise/transports + +go 1.27.0 + +require ( + github.com/bytedance/sonic v1.15.3-0.20260730064818-2a36d6da63e2 + github.com/coreos/go-oidc/v3 v3.18.0 + github.com/fasthttp/router v1.5.4 + github.com/google/cel-go v0.30.0 + github.com/google/uuid v1.6.0 + github.com/maximhq/bifrost-enterprise/core v0.0.0 + github.com/maximhq/bifrost-enterprise/framework v0.0.0 + github.com/maximhq/bifrost-enterprise/plugins v0.0.0 + github.com/maximhq/bifrost/core v1.9.1 + github.com/maximhq/bifrost/framework v1.7.2 + github.com/maximhq/bifrost/plugins/governance v1.8.1 + github.com/maximhq/bifrost/plugins/logging v1.8.1 + github.com/maximhq/bifrost/plugins/routing v1.1.1 + github.com/maximhq/bifrost/plugins/semanticcache v1.6.4 + github.com/maximhq/bifrost/transports v1.6.12-0.20260918214249-f44b5410fb7b + github.com/stretchr/testify v1.11.1 + github.com/valyala/fasthttp v1.74.0 + golang.org/x/sync v0.22.0 + gorm.io/driver/sqlite v1.6.0 + gorm.io/gorm v1.31.1 +) + +require ( + cel.dev/expr v0.25.2 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.20.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/bigquery v1.74.0 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.7.0 // indirect + cloud.google.com/go/monitoring v1.24.3 // indirect + cloud.google.com/go/pubsub/v2 v2.4.0 // indirect + cloud.google.com/go/secretmanager v1.16.0 // indirect + cloud.google.com/go/storage v1.62.1 // indirect + dario.cat/mergo v1.0.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect + github.com/BobuSumisu/aho-corasick v1.0.3 // indirect + github.com/ClickHouse/ch-go v0.65.0 // indirect + github.com/ClickHouse/clickhouse-go/v2 v2.32.0 // indirect + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/obfuscate v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/opentelemetry-mapping-go/otlp/attributes v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/proto v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/template v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/trace v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/trace/log v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/trace/otel v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/trace/stats v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/trace/traceutil v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/log v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/version v0.77.0 // indirect + github.com/DataDog/datadog-go/v5 v5.8.3 // indirect + github.com/DataDog/dd-trace-go/v2 v2.8.2 // indirect + github.com/DataDog/go-libddwaf/v4 v4.9.0 // indirect + github.com/DataDog/go-runtime-metrics-internal v0.0.4-0.20260217080614-b0f4edc38a6d // indirect + github.com/DataDog/go-sqllexer v0.1.13 // indirect + github.com/DataDog/go-tuf v1.1.1-0.5.2 // indirect + github.com/DataDog/sketches-go v1.4.8 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/Masterminds/sprig/v3 v3.3.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/ProtonMail/go-crypto v1.1.6 // indirect + github.com/STARRY-S/zip v0.2.1 // indirect + github.com/andybalholm/brotli v1.2.2 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/apache/arrow/go/v15 v15.0.2 // indirect + github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect + github.com/armon/go-metrics v0.4.1 // indirect + github.com/aws/aws-sdk-go-v2 v1.42.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.14 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect + github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.50.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 // indirect + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.42.3 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect + github.com/aws/smithy-go v1.27.1 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blevesearch/go-porterstemmer v1.0.3 // indirect + github.com/bodgit/plumbing v1.3.0 // indirect + github.com/bodgit/sevenzip v1.6.0 // indirect + github.com/bodgit/windows v1.0.1 // indirect + github.com/buger/jsonparser v1.2.0 // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic/loader v0.5.2 // indirect + github.com/cenkalti/backoff v2.2.1+incompatible // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/charmbracelet/colorprofile v0.3.1 // indirect + github.com/charmbracelet/lipgloss v1.1.0 // indirect + github.com/charmbracelet/x/ansi v0.10.1 // indirect + github.com/charmbracelet/x/cellbuf v0.0.13 // indirect + github.com/charmbracelet/x/term v0.2.1 // indirect + github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect + github.com/cloudflare/circl v1.6.3 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect + github.com/coreos/go-semver v0.3.1 // indirect + github.com/coreos/go-systemd/v22 v22.6.0 // indirect + github.com/cyphar/filepath-securejoin v0.6.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ebitengine/purego v0.10.0 // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect + github.com/fasthttp/websocket v1.5.12 // indirect + github.com/fatih/color v1.18.0 // indirect + github.com/fatih/semgroup v1.2.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/gitleaks/go-gitdiff v0.9.1 // indirect + github.com/go-faster/city v1.0.1 // indirect + github.com/go-faster/errors v0.7.1 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.9.0 // indirect + github.com/go-git/go-git/v5 v5.19.2 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-openapi/analysis v0.24.2 // indirect + github.com/go-openapi/errors v0.22.5 // indirect + github.com/go-openapi/jsonpointer v0.22.4 // indirect + github.com/go-openapi/jsonreference v0.21.4 // indirect + github.com/go-openapi/loads v0.23.2 // indirect + github.com/go-openapi/runtime v0.29.2 // indirect + github.com/go-openapi/spec v0.22.3 // indirect + github.com/go-openapi/strfmt v0.25.0 // indirect + github.com/go-openapi/swag v0.25.4 // indirect + github.com/go-openapi/swag/cmdutils v0.25.4 // indirect + github.com/go-openapi/swag/conv v0.25.4 // indirect + github.com/go-openapi/swag/fileutils v0.25.4 // indirect + github.com/go-openapi/swag/jsonname v0.25.4 // indirect + github.com/go-openapi/swag/jsonutils v0.25.4 // indirect + github.com/go-openapi/swag/loading v0.25.4 // indirect + github.com/go-openapi/swag/mangling v0.25.4 // indirect + github.com/go-openapi/swag/netutils v0.25.4 // indirect + github.com/go-openapi/swag/stringutils v0.25.4 // indirect + github.com/go-openapi/swag/typeutils v0.25.4 // indirect + github.com/go-openapi/swag/yamlutils v0.25.4 // indirect + github.com/go-openapi/validate v0.25.1 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/flatbuffers v23.5.26+incompatible // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.16 // indirect + github.com/googleapis/gax-go/v2 v2.22.0 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect + github.com/grandcat/zeroconf v1.0.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect + github.com/h2non/filetype v1.1.3 // indirect + github.com/hashicorp/consul/api v1.34.3 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-hclog v1.6.3 // indirect + github.com/hashicorp/go-immutable-radix v1.3.1 // indirect + github.com/hashicorp/go-metrics v0.5.4 // indirect + github.com/hashicorp/go-msgpack/v2 v2.1.5 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-retryablehttp v0.7.8 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect + github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect + github.com/hashicorp/go-sockaddr v1.0.7 // indirect + github.com/hashicorp/go-version v1.8.0 // indirect + github.com/hashicorp/golang-lru v1.0.2 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/hashicorp/hcl v1.0.1-vault-7 // indirect + github.com/hashicorp/memberlist v0.5.4 // indirect + github.com/hashicorp/serf v0.10.1 // indirect + github.com/hashicorp/vault/api v1.23.0 // indirect + github.com/huandu/xstrings v1.5.0 // indirect + github.com/invopop/jsonschema v0.13.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.9.2 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/jaswdr/faker/v2 v2.8.0 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/klauspost/pgzip v1.2.6 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/linkdata/deadlock v0.5.5 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 // indirect + github.com/magiconair/properties v1.8.10 // indirect + github.com/mailru/easyjson v0.9.1 // indirect + github.com/mark3labs/mcp-go v0.43.2 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/mattn/go-runewidth v0.0.17 // indirect + github.com/mattn/go-sqlite3 v1.14.32 // indirect + github.com/maximhq/bifrost/plugins/compat v0.3.1 // indirect + github.com/maximhq/bifrost/plugins/maxim v1.7.4 // indirect + github.com/maximhq/bifrost/plugins/mocker v1.6.4 // indirect + github.com/maximhq/bifrost/plugins/modelcatalogresolver v1.1.4 // indirect + github.com/maximhq/bifrost/plugins/otel v1.5.4 // indirect + github.com/maximhq/bifrost/plugins/prompts v1.1.4 // indirect + github.com/maximhq/bifrost/plugins/telemetry v1.8.0 // indirect + github.com/maximhq/maxim-go v0.2.1 // indirect + github.com/mholt/archives v0.1.2 // indirect + github.com/miekg/dns v1.1.68 // indirect + github.com/minio/minlz v1.0.0 // indirect + github.com/minio/simdjson-go v0.4.5 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/molecule-man/go-brrr v1.0.1 // indirect + github.com/muesli/termenv v0.16.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/nakabonne/tstorage v0.3.6 // indirect + github.com/nwaples/rardecode/v2 v2.2.2 // indirect + github.com/oapi-codegen/runtime v1.1.1 // indirect + github.com/oklog/ulid v1.3.1 // indirect + github.com/outcaste-io/ristretto v0.2.3 // indirect + github.com/paulmach/orb v0.11.1 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/philippgille/chromem-go v0.7.0 // indirect + github.com/pierrec/lz4/v4 v4.1.22 // indirect + github.com/pinecone-io/go-pinecone/v5 v5.3.0 // indirect + github.com/pion/datachannel v1.6.0 // indirect + github.com/pion/dtls/v3 v3.1.5 // indirect + github.com/pion/ice/v4 v4.2.1 // indirect + github.com/pion/interceptor v0.1.44 // indirect + github.com/pion/logging v0.2.4 // indirect + github.com/pion/mdns/v2 v2.1.0 // indirect + github.com/pion/randutil v0.1.0 // indirect + github.com/pion/rtcp v1.2.16 // indirect + github.com/pion/rtp v1.10.1 // indirect + github.com/pion/sctp v1.9.2 // indirect + github.com/pion/sdp/v3 v3.0.18 // indirect + github.com/pion/srtp/v3 v3.0.10 // indirect + github.com/pion/stun/v3 v3.1.6 // indirect + github.com/pion/transport/v4 v4.0.2 // indirect + github.com/pion/turn/v4 v4.1.4 // indirect + github.com/pion/webrtc/v4 v4.2.9 // indirect + github.com/pjbgf/sha1cd v0.6.0 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/prometheus/client_golang v1.23.2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.67.5 // indirect + github.com/prometheus/procfs v0.19.2 // indirect + github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect + github.com/qdrant/go-client v1.16.2 // indirect + github.com/redis/go-redis/v9 v9.17.2 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rs/zerolog v1.34.0 // indirect + github.com/ryanuber/go-glob v1.0.0 // indirect + github.com/sagikazarmark/locafero v0.7.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect + github.com/savsgio/gotils v0.0.0-20250408102913-196191ec6287 // indirect + github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect + github.com/secure-systems-lab/go-securesystemslib v0.10.0 // indirect + github.com/segmentio/asm v1.2.0 // indirect + github.com/segmentio/kafka-go v0.4.51 // indirect + github.com/sergi/go-diff v1.4.0 // indirect + github.com/shirou/gopsutil/v4 v4.26.3 // indirect + github.com/shopspring/decimal v1.4.0 // indirect + github.com/skeema/knownhosts v1.3.1 // indirect + github.com/sorairolake/lzip-go v0.3.5 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/spf13/viper v1.19.0 // indirect + github.com/spiffe/go-spiffe/v2 v2.7.0 // indirect + github.com/stretchr/objx v0.5.3 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/tetratelabs/wazero v1.11.0 // indirect + github.com/therootcompany/xz v1.0.1 // indirect + github.com/tidwall/gjson v1.18.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect + github.com/tinylib/msgp v1.6.3 // indirect + github.com/tklauser/go-sysconf v0.3.16 // indirect + github.com/tklauser/numcpus v0.11.0 // indirect + github.com/trailofbits/go-mutexasserts v0.0.0-20250514102930-c1f3d2e37561 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ulikunitz/xz v0.5.15 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/wasilibs/go-re2 v1.9.0 // indirect + github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 // indirect + github.com/weaviate/weaviate v1.38.0 // indirect + github.com/weaviate/weaviate-go-client/v5 v5.7.1 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/wlynxg/anet v0.0.5 // indirect + github.com/x448/float16 v0.8.4 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + github.com/zricethezav/gitleaks/v8 v8.30.1 // indirect + go.etcd.io/etcd/api/v3 v3.6.11 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.6.11 // indirect + go.etcd.io/etcd/client/v3 v3.6.11 // indirect + go.mongodb.org/mongo-driver v1.17.7 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/collector/component v1.51.1-0.20260205185216-81bc641f26c0 // indirect + go.opentelemetry.io/collector/featuregate v1.51.1-0.20260205185216-81bc641f26c0 // indirect + go.opentelemetry.io/collector/pdata v1.51.1-0.20260205185216-81bc641f26c0 // indirect + go.opentelemetry.io/collector/pdata/pprofile v0.145.1-0.20260205185216-81bc641f26c0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect + go.opentelemetry.io/otel v1.45.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/otel/sdk v1.45.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect + go.opentelemetry.io/otel/trace v1.45.0 // indirect + go.opentelemetry.io/proto/otlp v1.10.0 // indirect + go.starlark.net v0.0.0-20260102030733-3fee463870c9 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.1 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + go4.org v0.0.0-20230225012048-214862532bf5 // indirect + golang.org/x/arch v0.23.0 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect + golang.org/x/mod v0.38.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959 // indirect + golang.org/x/term v0.45.0 // indirect + golang.org/x/text v0.41.0 // indirect + golang.org/x/time v0.15.0 // indirect + golang.org/x/tools v0.48.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect + google.golang.org/api v0.282.0 // indirect + google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/grpc v1.83.2 // indirect + google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.67.1 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + gorm.io/driver/clickhouse v0.7.0 // indirect + gorm.io/driver/postgres v1.6.0 // indirect + k8s.io/api v0.36.1 // indirect + k8s.io/apimachinery v0.36.1 // indirect + k8s.io/client-go v0.36.1 // indirect + k8s.io/klog/v2 v2.140.0 // indirect + k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect + k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect +) + +replace github.com/maximhq/bifrost-enterprise/core => ../core + +replace github.com/maximhq/bifrost-enterprise/framework => ../framework + +replace github.com/maximhq/bifrost-enterprise/plugins => ../plugins +``` + + diff --git a/docs/docs.json b/docs/docs.json index c4c8f7e058c..95cf5bb2977 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -2045,6 +2045,7 @@ "item": "Enterprise", "icon": "building", "pages": [ + "changelogs/ent-v2.2.1", "changelogs/ent-v2.2.0", "changelogs/ent-v2.1.1", "changelogs/ent-v2.1.0", @@ -2139,6 +2140,7 @@ "item": "Edge", "icon": "grip", "pages": [ + "changelogs/edge-v0.9.0", "changelogs/edge-v0.8.0", "changelogs/edge-v0.7.0", "changelogs/edge-v0.6.0", From a9937c9d6cf7d82aa252a4977c19cf166b00e1a1 Mon Sep 17 00:00:00 2001 From: akshaydeo Date: Sat, 19 Sep 2026 04:41:55 +0530 Subject: [PATCH 011/118] clean up correct sentence around API key clamping --- docs/changelogs/ent-v2.2.1.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelogs/ent-v2.2.1.mdx b/docs/changelogs/ent-v2.2.1.mdx index 5be5f268d6d..c79ed10d00d 100644 --- a/docs/changelogs/ent-v2.2.1.mdx +++ b/docs/changelogs/ent-v2.2.1.mdx @@ -7,7 +7,7 @@ description: "v2.2.1 changelog - 2026-09-19" ## Changelog -v2.2.1 moves the enterprise gateway onto OSS transports v2.2.1. API key scopes are now clamped to the caller's own permissions at create and update, closing a privilege escalation where any key-creating user could mint a token carrying every permission in the system. Virtual key search now matches the assigned user's name or email, and the assignee column is resolved in one batched query per page instead of one request per row. The OSS release also carries three security fixes: resource IDs are validated before they enter outbound URL paths, Bedrock S3 bucket names can no longer redirect signed requests to a caller-chosen host, and the billing idempotency key can no longer be forged through the `x-request-id` header. +v2.2.1 moves the enterprise gateway onto OSS transports v2.2.1. API key scopes are now clamped to the caller's own permissions at create and update. Virtual key search now matches the assigned user's name or email, and the assignee column is resolved in one batched query per page instead of one request per row. The OSS release also carries three security fixes: resource IDs are validated before they enter outbound URL paths, Bedrock S3 bucket names can no longer redirect signed requests to a caller-chosen host, and the billing idempotency key can no longer be forged through the `x-request-id` header. ## ✨ Features From 217f9d0a7df028a7f9809912648c575e91f8fbd1 Mon Sep 17 00:00:00 2001 From: gyixuan Date: Sun, 20 Sep 2026 21:10:21 +0800 Subject: [PATCH 012/118] [fix]: preserve API web search source names (#7358) OpenAI Responses web search can return specialized API sources ({"type":"api","name":"oai-weather"}) that carry a name and no URL. The typed source schema only modeled type and a required url, so decode dropped name and re-encode fabricated an empty "url":""; the OpenAI request-side source sanitization rebuilt sources without name as well. url is now omitempty and name survives both the schema round-trip and the sanitization path. Affected packages: - core/schemas/responses.go - add Name to web search action sources, make url omitempty - core/providers/openai/types.go - keep Name in the OpenAI source sanitization path - core/schemas/responses_test.go - round-trip regression tests - core/providers/openai/responses_test.go - sanitization regression test - core/changelog.md - changelog entry - tests/e2e/api/collections/provider-harness.json - replay regression case Closes #7356 --- core/changelog.md | 1 + core/providers/openai/responses_test.go | 56 +++++++++++++++++ core/providers/openai/types.go | 4 +- core/schemas/responses.go | 5 +- core/schemas/responses_test.go | 61 +++++++++++++++++++ .../e2e/api/collections/provider-harness.json | 49 +++++++++++++++ 6 files changed, 173 insertions(+), 3 deletions(-) diff --git a/core/changelog.md b/core/changelog.md index e69de29bb2d..556fb73a856 100644 --- a/core/changelog.md +++ b/core/changelog.md @@ -0,0 +1 @@ +- [fix]: web search action sources round-trip their name and no longer fabricate an empty url. OpenAI Responses web search can return specialized API sources ({"type":"api","name":"oai-weather"}) that carry a name and no URL; the typed source schema only modeled type and a required url, so decode dropped name and re-encode emitted "url":"", and the OpenAI request-side source sanitization rebuilt sources without name. url is now omitempty and name survives both the schema round-trip and the sanitization path (#7356) [@g-yixuan](https://github.com/g-yixuan) diff --git a/core/providers/openai/responses_test.go b/core/providers/openai/responses_test.go index 390f1207ba4..4bb8b81b442 100644 --- a/core/providers/openai/responses_test.go +++ b/core/providers/openai/responses_test.go @@ -3334,3 +3334,59 @@ func TestToOpenAIResponsesRequest_MantleGPTOSSReplaysAssistantTextAsInput(t *tes }) } } + +// Web search action sources are sanitized for OpenAI: provider-specific fields +// (title, encrypted_content, page_age) are stripped, while the OpenAI-native +// fields survive. That includes the name of specialized API sources +// ({"type":"api","name":"oai-weather"}), which carry no URL and must not gain a +// fabricated empty one. +func TestToOpenAIResponsesRequest_StripsWebSearchSourceProviderFields(t *testing.T) { + history := `{ + "id": "ws_1", + "type": "web_search_call", + "status": "completed", + "action": { + "type": "search", + "queries": ["weather in paris"], + "sources": [ + {"type": "url", "url": "https://example.com", "title": "Example"}, + {"type": "api", "name": "oai-weather"} + ] + } + }` + var webSearchCall schemas.ResponsesMessage + require.NoError(t, schemas.Unmarshal([]byte(history), &webSearchCall)) + + bifrostReq := &schemas.BifrostResponsesRequest{ + Model: "gpt-4o", + Input: []schemas.ResponsesMessage{webSearchCall}, + } + result := ToOpenAIResponsesRequest(nil, bifrostReq) + require.NotNil(t, result) + + body, err := sonic.Marshal(result) + require.NoError(t, err) + var wire struct { + Input []struct { + Action struct { + Sources []map[string]any `json:"sources"` + } `json:"action"` + } `json:"input"` + } + require.NoError(t, json.Unmarshal(body, &wire)) + require.Len(t, wire.Input, 1, "body: %s", body) + require.Len(t, wire.Input[0].Action.Sources, 2, "body: %s", body) + + urlSource, apiSource := wire.Input[0].Action.Sources[0], wire.Input[0].Action.Sources[1] + + require.NotContains(t, urlSource, "title", "provider-specific title must be stripped: %s", body) + require.Equal(t, "https://example.com", urlSource["url"], "url source keeps its url: %s", body) + + require.Equal(t, "api", apiSource["type"], "api source keeps its type: %s", body) + require.Equal(t, "oai-weather", apiSource["name"], "api source keeps its name: %s", body) + require.NotContains(t, apiSource, "url", "api source must not gain a fabricated empty url: %s", body) + + // The caller's input must not be mutated by the strip. + require.NotNil(t, bifrostReq.Input[0].ResponsesToolMessage.Action.ResponsesWebSearchToolCallAction.Sources[0].Title, + "the caller's input must not be mutated") +} diff --git a/core/providers/openai/types.go b/core/providers/openai/types.go index 3e5b4df2761..cf9c9de582b 100644 --- a/core/providers/openai/types.go +++ b/core/providers/openai/types.go @@ -544,10 +544,12 @@ func (r *OpenAIResponsesRequestInput) MarshalJSON() ([]byte, error) { webSearchActionCopy := *msg.ResponsesToolMessage.Action.ResponsesWebSearchToolCallAction strippedSources := make([]schemas.ResponsesWebSearchToolCallActionSearchSource, len(sources)) for j, source := range sources { - // Only keep Type and URL for OpenAI + // Only keep Type, URL and Name for OpenAI; Name identifies + // specialized API sources (type "api") that carry no URL. strippedSources[j] = schemas.ResponsesWebSearchToolCallActionSearchSource{ Type: source.Type, URL: source.URL, + Name: source.Name, // Title, EncryptedContent, and PageAge are omitted } } diff --git a/core/schemas/responses.go b/core/schemas/responses.go index 1fd0b7cb725..deeb122ae9e 100644 --- a/core/schemas/responses.go +++ b/core/schemas/responses.go @@ -2290,8 +2290,9 @@ type ResponsesWebSearchToolCallAction struct { // ResponsesWebSearchToolCallActionSearchSource represents a web search action search source type ResponsesWebSearchToolCallActionSearchSource struct { - Type string `json:"type"` // always "url" - URL string `json:"url"` + Type string `json:"type"` // "url" for web pages, "api" for specialized API sources + URL string `json:"url,omitempty"` + Name string `json:"name,omitempty"` // Identifies specialized API sources (type "api"), which carry no URL // Anthropic specific fields Title *string `json:"title,omitempty"` diff --git a/core/schemas/responses_test.go b/core/schemas/responses_test.go index b91a97049fb..8b1ded7940f 100644 --- a/core/schemas/responses_test.go +++ b/core/schemas/responses_test.go @@ -945,3 +945,64 @@ func TestDeepCopyResponsesMessagePreservesMediaResolution(t *testing.T) { t.Fatalf("numTokens = %d, want 512", *got.NumTokens) } } + +// TestResponsesWebSearchSourceRoundTrip pins web_search_call action sources +// through a decode -> re-encode cycle. OpenAI's hosted web search can return +// specialized API sources ({"type":"api","name":"oai-weather"}) that carry a +// name and no URL; they must survive the round-trip without losing the name or +// fabricating an empty url. +func TestResponsesWebSearchSourceRoundTrip(t *testing.T) { + roundTripSource := func(t *testing.T, raw string) map[string]any { + t.Helper() + var msg ResponsesMessage + if err := Unmarshal([]byte(raw), &msg); err != nil { + t.Fatalf("unmarshal web_search_call: %v", err) + } + encoded, err := MarshalSorted(msg) + if err != nil { + t.Fatalf("marshal web_search_call: %v", err) + } + var out struct { + Action struct { + Sources []map[string]any `json:"sources"` + } `json:"action"` + } + if err := json.Unmarshal(encoded, &out); err != nil { + t.Fatalf("unmarshal encoded web_search_call: %v", err) + } + if len(out.Action.Sources) != 1 { + t.Fatalf("expected 1 source after round-trip, got %d (encoded: %s)", len(out.Action.Sources), encoded) + } + return out.Action.Sources[0] + } + + t.Run("api source keeps name and gains no url", func(t *testing.T) { + raw := `{"id":"ws_1","type":"web_search_call","status":"completed","action":{"type":"search","queries":["weather in paris"],"sources":[{"type":"api","name":"oai-weather"}]}}` + + source := roundTripSource(t, raw) + if source["type"] != "api" { + t.Fatalf("expected source type %q, got %v", "api", source["type"]) + } + if source["name"] != "oai-weather" { + t.Fatalf("expected source name %q, got %v", "oai-weather", source["name"]) + } + if _, ok := source["url"]; ok { + t.Fatalf("expected no url key on an api source, got %v", source["url"]) + } + }) + + t.Run("url source round-trips unchanged", func(t *testing.T) { + raw := `{"id":"ws_1","type":"web_search_call","status":"completed","action":{"type":"search","queries":["weather in paris"],"sources":[{"type":"url","url":"https://example.com"}]}}` + + source := roundTripSource(t, raw) + if source["type"] != "url" { + t.Fatalf("expected source type %q, got %v", "url", source["type"]) + } + if source["url"] != "https://example.com" { + t.Fatalf("expected source url %q, got %v", "https://example.com", source["url"]) + } + if _, ok := source["name"]; ok { + t.Fatalf("expected no name key on a plain url source, got %v", source["name"]) + } + }) +} diff --git a/tests/e2e/api/collections/provider-harness.json b/tests/e2e/api/collections/provider-harness.json index 07cdde1fa85..3f45fd78224 100644 --- a/tests/e2e/api/collections/provider-harness.json +++ b/tests/e2e/api/collections/provider-harness.json @@ -158143,6 +158143,55 @@ "response": [] } ] + }, + { + "name": "104. Web Search API Source Round-Trip (#7356)", + "description": "OpenAI Responses web search can return specialized API sources in web_search_call.action.sources ({\"type\":\"api\",\"name\":\"oai-weather\"}) that carry a name and no URL; the published OpenAPI schema does not model them yet (openai/openai-openapi#573). Bifrost's typed source schema had no name field and a required url, so a decode/re-encode round-trip dropped name and fabricated an empty url, and the OpenAI request-side source sanitization rebuilt sources without name. The case pins the client-visible path: replaying a completed web_search_call in input through POST /v1/responses must be accepted with the api source intact. Red/green note: pre-fix the replayed api source reached OpenAI as {\"type\":\"api\",\"url\":\"\"}; whether OpenAI rejects that shape depends on server-side validation this case cannot control, so the primary pin is post-fix acceptance of the corrected shape.", + "item": [ + { + "name": "openai/gpt-4o responses api source replay - #7356", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "if ([401, 403, 429, 500, 502, 503, 504].indexOf(pm.response.code) !== -1) { return; }", + "pm.test('web search api source replay accepted - #7356', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "pm.test('response body is valid json - #7356', function () {", + " pm.response.json();", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\"model\":\"openai/gpt-4o\",\"max_output_tokens\":16,\"input\":[{\"type\":\"web_search_call\",\"id\":\"ws_7356\",\"status\":\"completed\",\"action\":{\"type\":\"search\",\"queries\":[\"weather in paris\"],\"sources\":[{\"type\":\"url\",\"url\":\"https://example.com/weather\"},{\"type\":\"api\",\"name\":\"oai-weather\"}]}},{\"role\":\"user\",\"content\":\"Summarize the search results above in one sentence.\"}]}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + } + ] } ] } From 7f2d5ee3697ed6f8cc29b7ed0a158bb1b2a8eca9 Mon Sep 17 00:00:00 2001 From: BASIL K AJI Date: Sun, 20 Sep 2026 18:41:57 +0530 Subject: [PATCH 013/118] fix(anthropic): carry allowed_tools and parallel_tool_calls into the request (#7348) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(anthropic): carry allowed_tools and parallel_tool_calls into the request Two tool restrictions a caller sets were discarded on the way to Anthropic. Both requests still succeeded, so the caller had no way to learn the restriction was not in force. **allowed_tools became "call anything"** `chat.go` mapped the whole subset to a single word: case schemas.ChatToolChoiceTypeAllowedTools: toolChoice.Type = "any" A caller saying "you may call only tool_a" had every declared tool forwarded and the upstream told to call any of them. Mode was lost too, so an "auto" subset was turned into a mandatory call. Anthropic's tool_choice has no subset form (auto, any, tool, none), so the restriction is expressed where it can be: a tool that is not declared cannot be called. The declarations are now filtered to the allowed names, and the mode maps to auto or any rather than always any. An allowed list naming tools that are not in the request keeps the original declarations. Sending none would convert a restriction into a request that cannot call anything, which is a larger change than the caller asked for. **parallel_tool_calls had nowhere to go** `AnthropicToolChoice.DisableParallelToolUse` has existed in types.go all along; nothing ever set it. `ParallelToolCalls` appears nowhere in this provider, so the instruction was dropped and calls stayed parallel. It is now carried across, with the sense inverted. Anthropic accepts the field on tool_choice only, so when a caller disables parallel calls without expressing any preference about which tool runs, an explicit "auto" is emitted to hold it. That is Anthropic's own default, so it adds no restriction that was not already there. Reported in #7344, cases 3 and 6. The Gemini cases in that issue are a separate provider and are not touched here. Six tests, all failing against the previous revision: the subset is enforced, auto mode does not force a call, unknown names keep the declarations, the parallel flag is carried both with and without an existing tool choice, and parallel_tool_calls: true still sends nothing because it is the default. go build, go vet and the full anthropic package pass. The package was green before this change as well, so the delta is only the new cases. * fix(anthropic): an allowed list matching nothing permits nothing Raised in review by CodeRabbit, and it is right. An allowed list naming only tools the request does not declare left an empty intersection, and the filter returned the original declarations. So a caller who restricted the model to a tool that is not present had every tool they excluded forwarded instead - the same failure this conversion exists to stop, reached from the other direction. My reasoning was that a request which cannot call anything is a larger change than the caller asked for. That was wrong: the caller asked for exactly one set, the intersection of that set with the declarations is empty, and "nothing" is the faithful answer rather than "everything". The intersection is now honoured literally, and the tool choice says so. "any" or "auto" with no declarations is a request Anthropic rejects, so an empty result maps to "none". TestToAnthropicChatRequest_AllowedToolsUnknownNamesKeepDeclarations asserted the old behaviour and has been replaced by TestToAnthropicChatRequest_AllowedToolsUnknownNamesPermitNothing, which checks both halves: no tool is forwarded, and the choice is "none". * fix(anthropic): an allowed_tools filter must not strip server tools Found while re-reading my own change. The filter compared every declaration against the allowed names, and anthropicReq.Tools can hold server tools - MCP toolsets, web search, computer use - built by convertServerToolToAnthropic a few lines above. An OpenAI allowed_tools list describes function tools: entries are {type: "function", function: {name}}. A server tool can never appear in one, so matching it on a name it cannot have removed it every time a caller set allowed_tools. Restricting function calling to tool_a silently switched off a web search the caller had enabled elsewhere in the same request. That is the same harm as the bug this PR fixes, inverted: a stated intent quietly discarded. Server tools now pass through the filter untouched; only nameable declarations are subject to the allowed list. The empty-intersection case follows. "None of the allowed tools are declared" is a statement about nameable tools, so the tool_choice now becomes "none" when no function tool survives, rather than when the list is empty - otherwise a request carrying a server tool would keep "any" over nothing the model could be told to call. Two tests, both failing against the previous revision: a server tool survives an allowed_tools filter that excludes every function tool, and a filter that allows nothing nameable still yields "none" while leaving the server tool in place. * test(anthropic): move the tool-choice tests into chat_test.go Requested in review by @akshaydeo: no new test files. The seven cases move into providers/anthropic/chat_test.go, which already holds the TestToAnthropicChatRequest_* tests they belong with, and toolchoicerestrictions_test.go is removed. testify is added to that file's imports; it is already used elsewhere in the package. No test content changed. go vet and the full anthropic package pass. * fix(anthropic): an MCP toolset is a server tool too Raised in review by CodeRabbit, and it is right. My previous commit used Type != nil to recognise a server tool, which is true of web search, web fetch and the computer-use family but not of an MCP toolset. convertServerToolToAnthropic returns one as: return AnthropicTool{MCPToolset: toolset}, true with Type nil and Name empty. The struct says so itself: // MCP toolset (mcp-client-2025-11-20 format) — embedded when Type is nil // and MCPToolset is set MCPToolset *AnthropicMCPToolsetTool `json:"-"` So the filter matched an MCP toolset's empty name against the allowlist and removed it - the same silent capability loss the previous commit set out to fix, still happening for the one server-tool shape I did not check. I had read Type: "mcp_toolset" a few lines up and not noticed it was on the nested type rather than on AnthropicTool. Both shapes are now preserved, and countFunctionTools excludes both, so a request whose nameable tools are all excluded still yields "none" rather than "any" over nothing callable. The test fixture carries both shapes rather than one, which is what let this through: the previous test used Type to stand in for a server tool, so it could not have caught an MCP toolset. Both cases fail against the previous revision. --- core/providers/anthropic/chat.go | 119 ++++++++++++++++- core/providers/anthropic/chat_test.go | 185 ++++++++++++++++++++++++++ 2 files changed, 303 insertions(+), 1 deletion(-) diff --git a/core/providers/anthropic/chat.go b/core/providers/anthropic/chat.go index c78e5d690e4..22a445811d2 100644 --- a/core/providers/anthropic/chat.go +++ b/core/providers/anthropic/chat.go @@ -679,13 +679,48 @@ func ToAnthropicChatRequest(ctx *schemas.BifrostContext, bifrostReq *schemas.Bif toolChoice.Name = bifrostReq.Params.ToolChoice.ChatToolChoiceStruct.Function.Name } case schemas.ChatToolChoiceTypeAllowedTools: - toolChoice.Type = "any" + // Anthropic has no "choose from this subset" form, so the + // restriction is expressed by removing the declarations the + // caller did not allow: the model can only call a tool it was + // given. Mapping straight to "any" kept every tool and also + // forced a call even when the caller's mode was "auto". + allowed := bifrostReq.Params.ToolChoice.ChatToolChoiceStruct.AllowedTools + if allowed != nil { + anthropicReq.Tools = filterToolsByAllowed(anthropicReq.Tools, allowed.Tools) + switch { + case countFunctionTools(anthropicReq.Tools) == 0: + // The allowed list matched nothing declared. "any" or + // "auto" with no tools is a request Anthropic rejects, + // so the restriction is stated as what it means. + toolChoice.Type = "none" + case allowed.Mode == "required": + toolChoice.Type = "any" + default: + toolChoice.Type = "auto" + } + } else { + toolChoice.Type = "any" + } case schemas.ChatToolChoiceTypeCustom: toolChoice.Type = "auto" default: toolChoice.Type = "auto" } } + applyParallelToolUse(toolChoice, bifrostReq.Params.ParallelToolCalls) + anthropicReq.ToolChoice = toolChoice + } + + // A caller can disable parallel tool calls without expressing any + // preference about *which* tool runs. Anthropic carries that setting on + // tool_choice only, so without a tool_choice to attach it to the + // instruction had nowhere to go and was dropped. "auto" is Anthropic's + // default behaviour, so naming it here restores the flag without + // changing which tools may be called. + if anthropicReq.ToolChoice == nil && len(anthropicReq.Tools) > 0 && + bifrostReq.Params.ParallelToolCalls != nil && !*bifrostReq.Params.ParallelToolCalls { + toolChoice := &AnthropicToolChoice{Type: "auto"} + applyParallelToolUse(toolChoice, bifrostReq.Params.ParallelToolCalls) anthropicReq.ToolChoice = toolChoice } @@ -2042,3 +2077,85 @@ func ToAnthropicChatStreamError(bifrostErr *schemas.BifrostError) string { // Format as Anthropic SSE error event return fmt.Sprintf("event: error\ndata: %s\n\n", jsonData) } + +// filterToolsByAllowed narrows a converted Anthropic tool list to the names a +// caller permitted through tool_choice.allowed_tools. +// +// Anthropic's tool_choice has no subset form (auto, any, tool, none), so the +// only place the restriction can be expressed is the declaration list itself: +// a tool that is not declared cannot be called. Callers that pass an empty +// allowed list are left alone rather than being stripped of every tool, since +// sending no declarations changes the request into one that cannot use tools +// at all - a larger change than the caller asked for. +func filterToolsByAllowed(tools []AnthropicTool, allowed []schemas.ChatToolChoiceAllowedToolsTool) []AnthropicTool { + if len(tools) == 0 || len(allowed) == 0 { + return tools + } + permitted := make(map[string]struct{}, len(allowed)) + for _, a := range allowed { + if a.Function.Name != "" { + permitted[a.Function.Name] = struct{}{} + } + } + if len(permitted) == 0 { + return tools + } + kept := make([]AnthropicTool, 0, len(tools)) + for _, t := range tools { + // Server tools are never named in an OpenAI allowed_tools list, which + // describes function tools only. Filtering them on a name they cannot + // have would silently switch off a capability the caller enabled + // elsewhere in the same request - a stated intent quietly discarded, + // which is the harm this function exists to prevent. + // + // Two shapes carry a server tool, and testing only one of them is the + // mistake to avoid here: web search, web fetch and the computer-use + // family set Type, while an MCP toolset leaves Type nil and carries + // everything in MCPToolset with an empty Name (see AnthropicTool, whose + // own comment records this). Checking Type alone therefore matched an + // MCP toolset's empty name against the allowlist and removed it. + if t.Type != nil || t.MCPToolset != nil { + kept = append(kept, t) + continue + } + if _, ok := permitted[t.Name]; ok { + kept = append(kept, t) + } + } + // An empty result means the allowed list named only tools this request does + // not declare. Returning the originals would forward every tool the caller + // just excluded, which is the bug this function exists to prevent, so the + // intersection is honoured literally: nothing may be called. The caller sees + // a request with no tools rather than one that quietly ignored the list. + return kept +} + +// applyParallelToolUse carries an OpenAI-style parallel_tool_calls flag onto an +// Anthropic tool_choice. Anthropic expresses the same instruction as +// disable_parallel_tool_use, so the sense is inverted, and it only accepts the +// field when a tool may actually be called. +func applyParallelToolUse(toolChoice *AnthropicToolChoice, parallel *bool) { + if toolChoice == nil || parallel == nil || *parallel { + return + } + if toolChoice.Type == "none" { + return + } + disable := true + toolChoice.DisableParallelToolUse = &disable +} + +// countFunctionTools reports how many declarations a model could be told to +// call by name. Server tools are excluded in both their shapes - Type set, or +// MCPToolset set with Type nil - because they survive an allowed_tools filter +// untouched. Counting either would hide the case where every nameable tool was +// excluded, and leave the tool choice at "any" over nothing callable. +func countFunctionTools(tools []AnthropicTool) int { + n := 0 + for _, t := range tools { + if t.Type == nil && t.MCPToolset == nil { + n++ + } + } + return n +} diff --git a/core/providers/anthropic/chat_test.go b/core/providers/anthropic/chat_test.go index d3fd5825556..9325e387dfa 100644 --- a/core/providers/anthropic/chat_test.go +++ b/core/providers/anthropic/chat_test.go @@ -9,6 +9,8 @@ import ( "github.com/bytedance/sonic" "github.com/maximhq/bifrost/core/providers/openai" "github.com/maximhq/bifrost/core/schemas" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestToAnthropicChatRequest_PreservesPropertyOrder(t *testing.T) { @@ -2433,3 +2435,186 @@ func TestToAnthropicResponsesRequest_MidConversationSystem_InlinesForNonAnthropi } assertInlinedReminder(t, result, "Mid-conv instruction.") } + +// restrictionRequest builds a two-tool chat request so that a restriction has +// something to remove. The model is a current one so that no capability +// override drops the tool choice before these paths run. +func restrictionRequest(tc *schemas.ChatToolChoice, parallel *bool) *schemas.BifrostChatRequest { + return &schemas.BifrostChatRequest{ + Provider: schemas.Anthropic, + Model: "claude-sonnet-5", + Input: []schemas.ChatMessage{{ + Role: schemas.ChatMessageRoleUser, + Content: &schemas.ChatMessageContent{ContentStr: schemas.Ptr("What is the weather in Tokyo?")}, + }}, + Params: &schemas.ChatParameters{ + ToolChoice: tc, + ParallelToolCalls: parallel, + Tools: []schemas.ChatTool{ + {Type: schemas.ChatToolTypeFunction, Function: &schemas.ChatToolFunction{Name: "tool_a"}}, + {Type: schemas.ChatToolTypeFunction, Function: &schemas.ChatToolFunction{Name: "tool_b"}}, + }, + }, + } +} + +func allowedTools(mode string, names ...string) *schemas.ChatToolChoice { + tools := make([]schemas.ChatToolChoiceAllowedToolsTool, 0, len(names)) + for _, n := range names { + tools = append(tools, schemas.ChatToolChoiceAllowedToolsTool{ + Type: "function", + Function: schemas.ChatToolChoiceFunction{Name: n}, + }) + } + return &schemas.ChatToolChoice{ChatToolChoiceStruct: &schemas.ChatToolChoiceStruct{ + Type: schemas.ChatToolChoiceTypeAllowedTools, + AllowedTools: &schemas.ChatToolChoiceAllowedTools{Mode: mode, Tools: tools}, + }} +} + +func toolNames(req *AnthropicMessageRequest) []string { + names := make([]string, 0, len(req.Tools)) + for _, t := range req.Tools { + names = append(names, t.Name) + } + return names +} + +// A caller that allows only tool_a must not have tool_b forwarded. Anthropic +// has no subset form on tool_choice, so the restriction has to be carried by +// the declaration list; mapping to "any" and forwarding both tools told the +// upstream it could call either one. +func TestToAnthropicChatRequest_AllowedToolsRestrictsDeclarations(t *testing.T) { + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + + req, err := ToAnthropicChatRequest(ctx, restrictionRequest(allowedTools("required", "tool_a"), nil)) + require.NoError(t, err) + + assert.Equal(t, []string{"tool_a"}, toolNames(req), "tool_b was not allowed and must not be forwarded") + require.NotNil(t, req.ToolChoice) + assert.Equal(t, "any", req.ToolChoice.Type, `mode "required" means a tool must be called`) +} + +// mode "auto" means the model may call one of the allowed tools or none at all. +// Forcing "any" changed an optional call into a mandatory one. +func TestToAnthropicChatRequest_AllowedToolsAutoModeDoesNotForceACall(t *testing.T) { + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + + req, err := ToAnthropicChatRequest(ctx, restrictionRequest(allowedTools("auto", "tool_a"), nil)) + require.NoError(t, err) + + require.NotNil(t, req.ToolChoice) + assert.Equal(t, "auto", req.ToolChoice.Type, `mode "auto" must not force a tool call`) + assert.Equal(t, []string{"tool_a"}, toolNames(req)) +} + +// An allowed list naming only tools this request does not declare permits +// nothing. Forwarding the declarations anyway would hand the model every tool +// the caller just excluded - the same failure this conversion exists to stop, +// arrived at from the other direction. +func TestToAnthropicChatRequest_AllowedToolsUnknownNamesPermitNothing(t *testing.T) { + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + + req, err := ToAnthropicChatRequest(ctx, restrictionRequest(allowedTools("required", "not_declared"), nil)) + require.NoError(t, err) + + assert.Empty(t, toolNames(req), "no declared tool was allowed, so none may be forwarded") + require.NotNil(t, req.ToolChoice) + assert.Equal(t, "none", req.ToolChoice.Type, + `"any" with an empty tool list is a request Anthropic rejects`) +} + +// parallel_tool_calls: false has a direct Anthropic equivalent that was never +// populated, so the instruction was dropped and calls stayed parallel. +func TestToAnthropicChatRequest_ParallelToolCallsFalseDisablesParallelUse(t *testing.T) { + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + + t.Run("with an explicit tool choice", func(t *testing.T) { + req, err := ToAnthropicChatRequest(ctx, restrictionRequest( + &schemas.ChatToolChoice{ChatToolChoiceStr: schemas.Ptr("auto")}, schemas.Ptr(false))) + require.NoError(t, err) + require.NotNil(t, req.ToolChoice) + require.NotNil(t, req.ToolChoice.DisableParallelToolUse) + assert.True(t, *req.ToolChoice.DisableParallelToolUse) + }) + + t.Run("with no tool choice of its own", func(t *testing.T) { + req, err := ToAnthropicChatRequest(ctx, restrictionRequest(nil, schemas.Ptr(false))) + require.NoError(t, err) + require.NotNil(t, req.ToolChoice, "the flag has nowhere to live without a tool_choice") + assert.Equal(t, "auto", req.ToolChoice.Type, "auto is Anthropic's default, so this adds no restriction") + require.NotNil(t, req.ToolChoice.DisableParallelToolUse) + assert.True(t, *req.ToolChoice.DisableParallelToolUse) + }) +} + +// parallel_tool_calls: true is Anthropic's default, so nothing should be sent. +func TestToAnthropicChatRequest_ParallelToolCallsTrueSendsNothing(t *testing.T) { + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + + req, err := ToAnthropicChatRequest(ctx, restrictionRequest(nil, schemas.Ptr(true))) + require.NoError(t, err) + if req.ToolChoice != nil { + assert.Nil(t, req.ToolChoice.DisableParallelToolUse) + } +} + +// A server tool - MCP toolset, web search, computer use - cannot be named in +// an OpenAI allowed_tools list, which describes function tools only. Filtering +// it on a name it cannot have would switch off a capability the caller enabled +// elsewhere in the same request. +func TestToAnthropicChatRequest_AllowedToolsLeavesServerToolsAlone(t *testing.T) { + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + + req := restrictionRequest(allowedTools("required", "tool_a"), nil) + // Stand in for a converted server tool: it carries a Type, which a + // converted function tool never does. + converted, err := ToAnthropicChatRequest(ctx, req) + require.NoError(t, err) + + // Two server-tool shapes, because they are filtered by different fields: + // web search sets Type, while an MCP toolset leaves Type nil and carries + // everything in MCPToolset with an empty Name. + webSearch := AnthropicToolType("web_search") + withServer := append([]AnthropicTool{ + {Name: "web-search", Type: &webSearch}, + {MCPToolset: &AnthropicMCPToolsetTool{Type: "mcp_toolset", MCPServerName: "docs"}}, + }, converted.Tools...) + filtered := filterToolsByAllowed(withServer, []schemas.ChatToolChoiceAllowedToolsTool{ + {Type: "function", Function: schemas.ChatToolChoiceFunction{Name: "tool_a"}}, + }) + + names := make([]string, 0, len(filtered)) + for _, tl := range filtered { + names = append(names, tl.Name) + } + assert.Contains(t, names, "web-search", "a Type-carrying server tool must survive") + assert.Contains(t, names, "tool_a") + assert.NotContains(t, names, "tool_b") + + mcpKept := false + for _, tl := range filtered { + if tl.MCPToolset != nil { + mcpKept = true + } + } + assert.True(t, mcpKept, "an MCP toolset has Type nil and an empty Name; it must not be matched against the allowlist") +} + +// With every nameable tool excluded, the choice must still be "none" even +// though server tools remain in the list. +func TestToAnthropicChatRequest_AllowedToolsNoneNameableStillMeansNone(t *testing.T) { + webSearch := AnthropicToolType("web_search") + tools := []AnthropicTool{ + {Name: "web-search", Type: &webSearch}, + {MCPToolset: &AnthropicMCPToolsetTool{Type: "mcp_toolset", MCPServerName: "docs"}}, + {Name: "tool_a"}, + } + assert.Equal(t, 1, countFunctionTools(tools), "only tool_a is nameable; neither server shape counts") + + filtered := filterToolsByAllowed(tools, []schemas.ChatToolChoiceAllowedToolsTool{ + {Type: "function", Function: schemas.ChatToolChoiceFunction{Name: "not_declared"}}, + }) + assert.Equal(t, 0, countFunctionTools(filtered), "no function tool was allowed, so the choice must be none") + assert.Len(t, filtered, 2, "both server tools are still there") +} From 1a17949cd80234de0ceb3018e941854f38b3f7a8 Mon Sep 17 00:00:00 2001 From: BASIL K AJI Date: Sun, 20 Sep 2026 22:36:14 +0530 Subject: [PATCH 014/118] fix(bedrock): keep cache_control when a message's content is a string (#7360) The Anthropic Messages API accepts a message's content either as an array of content blocks or as a plain string, and clients mix the two: a rolling cache breakpoint produces block arrays for the turns carrying cache_control and bare strings for the rest. BedrockMessage.Content is typed []BedrockContentBlock, so one bare string anywhere in messages[] fails the standard unmarshal and diverts the whole request into the AI21 fallback. That branch rebuilt the messages but never called applyMessageContentCacheControl, so every cache_control in the request was dropped, not only the one on the string message. Prompt caching went off silently, with only the system cachePoint surviving on its separate path. The fallback now makes the same translation the standard path does. Fixes #7336 --- core/changelog.md | 1 + core/providers/bedrock/invoke.go | 6 + .../bedrock/invokestringcontentcache_test.go | 168 ++++++++++++++++++ 3 files changed, 175 insertions(+) create mode 100644 core/providers/bedrock/invokestringcontentcache_test.go diff --git a/core/changelog.md b/core/changelog.md index 556fb73a856..d8f79024d02 100644 --- a/core/changelog.md +++ b/core/changelog.md @@ -1 +1,2 @@ +- [fix]: Bedrock InvokeModel keeps cache_control when any message's content is a plain string. BedrockMessage.Content is typed as content blocks, so one bare string anywhere in messages[] failed the standard unmarshal and diverted the whole request into the AI21 string fallback, which rebuilt the messages without calling applyMessageContentCacheControl. Every cache_control in the request was dropped rather than only the one on the string message, so prompt caching went off silently with just the system cachePoint surviving. The fallback now makes the same translation the standard path does (#7336) [@basil-k-aji-dev](https://github.com/basil-k-aji-dev) - [fix]: web search action sources round-trip their name and no longer fabricate an empty url. OpenAI Responses web search can return specialized API sources ({"type":"api","name":"oai-weather"}) that carry a name and no URL; the typed source schema only modeled type and a required url, so decode dropped name and re-encode emitted "url":"", and the OpenAI request-side source sanitization rebuilt sources without name. url is now omitempty and name survives both the schema round-trip and the sanitization path (#7356) [@g-yixuan](https://github.com/g-yixuan) diff --git a/core/providers/bedrock/invoke.go b/core/providers/bedrock/invoke.go index 1f31707618e..d35689a13bc 100644 --- a/core/providers/bedrock/invoke.go +++ b/core/providers/bedrock/invoke.go @@ -113,6 +113,12 @@ func (r *BedrockInvokeRequest) UnmarshalJSON(data []byte) error { } r.Messages = append(r.Messages, msg) } + // The standard path above translates cache_control into cache + // points; this one has to as well. A single message whose + // content is a bare string diverts the whole request here, so + // skipping it dropped cache_control from every other message in + // the same request too. + r.Messages = applyMessageContentCacheControl(r.Messages, aux.Messages) } } } diff --git a/core/providers/bedrock/invokestringcontentcache_test.go b/core/providers/bedrock/invokestringcontentcache_test.go new file mode 100644 index 00000000000..ca0d1cd500e --- /dev/null +++ b/core/providers/bedrock/invokestringcontentcache_test.go @@ -0,0 +1,168 @@ +package bedrock + +import ( + "testing" + + "github.com/bytedance/sonic" +) + +// The Anthropic Messages API accepts a message's content either as an array of +// content blocks or as a plain string, and clients mix the two freely: a +// rolling cache breakpoint tends to produce block arrays for the turns that +// carry cache_control and bare strings for the rest. +// +// BedrockMessage.Content is typed []BedrockContentBlock, so one bare string +// anywhere in messages[] fails the standard unmarshal and diverts the whole +// request into the AI21 fallback. That branch rebuilt the messages but did not +// translate cache_control into cache points, so prompt caching went silently +// off for every message in the request, not only the string one. +// +// Ref: https://github.com/maximhq/bifrost/issues/7336 + +func cachePointCount(msg BedrockMessage) int { + n := 0 + for _, block := range msg.Content { + if block.CachePoint != nil { + n++ + } + } + return n +} + +func unmarshalInvoke(t *testing.T, payload string) *BedrockInvokeRequest { + t.Helper() + req := &BedrockInvokeRequest{} + if err := sonic.Unmarshal([]byte(payload), req); err != nil { + t.Fatalf("UnmarshalJSON: %v", err) + } + return req +} + +// TestInvokeUnmarshal_CacheControlSurvivesAStringContentMessage is the reported +// failure: the string message is what diverts the request, and the cache point +// belongs to a different message entirely. +func TestInvokeUnmarshal_CacheControlSurvivesAStringContentMessage(t *testing.T) { + req := unmarshalInvoke(t, `{ + "anthropic_version": "bedrock-2023-05-31", + "messages": [ + {"role": "user", "content": "an earlier turn, stored as a plain string"}, + {"role": "assistant", "content": [{"type": "text", "text": "reply"}]}, + {"role": "user", "content": [ + {"type": "text", "text": "the turn that carries the breakpoint", + "cache_control": {"type": "ephemeral"}} + ]} + ] + }`) + + if len(req.Messages) != 3 { + t.Fatalf("parsed %d messages, want 3", len(req.Messages)) + } + if got := cachePointCount(req.Messages[2]); got != 1 { + t.Errorf("cache points on the breakpoint message = %d, want 1: cache_control was dropped", got) + } + if got := cachePointCount(req.Messages[0]); got != 0 { + t.Errorf("cache points on the string message = %d, want 0", got) + } + if req.Messages[0].Content[0].Text == nil || + *req.Messages[0].Content[0].Text != "an earlier turn, stored as a plain string" { + t.Errorf("the string message lost its text: %+v", req.Messages[0].Content) + } +} + +// TestInvokeUnmarshal_CacheControlOnAStringOnlyRequest pins that a request with +// no block arrays at all still parses and grows no cache points out of nowhere. +func TestInvokeUnmarshal_CacheControlOnAStringOnlyRequest(t *testing.T) { + req := unmarshalInvoke(t, `{ + "messages": [ + {"role": "user", "content": "one"}, + {"role": "assistant", "content": "two"} + ] + }`) + + if len(req.Messages) != 2 { + t.Fatalf("parsed %d messages, want 2", len(req.Messages)) + } + for i, msg := range req.Messages { + if got := cachePointCount(msg); got != 0 { + t.Errorf("message %d has %d cache points, want 0", i, got) + } + if len(msg.Content) != 1 || msg.Content[0].Text == nil { + t.Errorf("message %d content = %+v, want one text block", i, msg.Content) + } + } +} + +// TestInvokeUnmarshal_CacheControlStillWorksWithoutAStringMessage is the +// control: the standard path was never broken, and must stay that way. +func TestInvokeUnmarshal_CacheControlStillWorksWithoutAStringMessage(t *testing.T) { + req := unmarshalInvoke(t, `{ + "messages": [ + {"role": "user", "content": [ + {"type": "text", "text": "hello", "cache_control": {"type": "ephemeral"}} + ]} + ] + }`) + + if got := cachePointCount(req.Messages[0]); got != 1 { + t.Errorf("cache points = %d, want 1", got) + } +} + +// TestInvokeUnmarshal_EveryBreakpointSurvivesAStringMessage covers the shape a +// rolling breakpoint actually produces: several marked turns, with strings +// interleaved. Losing any one of them costs the cache. +func TestInvokeUnmarshal_EveryBreakpointSurvivesAStringMessage(t *testing.T) { + req := unmarshalInvoke(t, `{ + "messages": [ + {"role": "user", "content": [ + {"type": "text", "text": "first", "cache_control": {"type": "ephemeral"}} + ]}, + {"role": "assistant", "content": "a plain string in the middle"}, + {"role": "user", "content": [ + {"type": "text", "text": "second", "cache_control": {"type": "ephemeral"}} + ]} + ] + }`) + + if len(req.Messages) != 3 { + t.Fatalf("parsed %d messages, want 3", len(req.Messages)) + } + for _, i := range []int{0, 2} { + if got := cachePointCount(req.Messages[i]); got != 1 { + t.Errorf("cache points on message %d = %d, want 1", i, got) + } + } +} + +// TestInvokeUnmarshal_ToolResultCacheControlSurvivesAStringMessage pins the +// nested case, since injectContentBlockCachePoints recurses into tool results +// and the fallback path skipped that recursion along with everything else. +func TestInvokeUnmarshal_ToolResultCacheControlSurvivesAStringMessage(t *testing.T) { + req := unmarshalInvoke(t, `{ + "messages": [ + {"role": "user", "content": "a plain string that diverts the parse"}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "call_1", "content": [ + {"type": "text", "text": "tool output", "cache_control": {"type": "ephemeral"}} + ]} + ]} + ] + }`) + + if len(req.Messages) != 2 { + t.Fatalf("parsed %d messages, want 2", len(req.Messages)) + } + blocks := req.Messages[1].Content + if len(blocks) == 0 || blocks[0].ToolResult == nil { + t.Fatalf("second message lost its tool_result: %+v", blocks) + } + found := 0 + for _, inner := range blocks[0].ToolResult.Content { + if inner.CachePoint != nil { + found++ + } + } + if found != 1 { + t.Errorf("cache points inside the tool result = %d, want 1", found) + } +} From b08d22c3aa4c1ff58804fa6cbb22051bac441c4b Mon Sep 17 00:00:00 2001 From: Akshay Deo Date: Mon, 21 Sep 2026 18:31:33 +0530 Subject: [PATCH 015/118] dependabot cleanups (#7378) ## Summary Extracts repeated label-value splicing logic in the telemetry plugin into a shared `spliceLabelValues` helper, and fixes an over-allocated map in the governance plugin. ## Changes - Introduced `spliceLabelValues(promLabelValues []string, nDefault int, extras ...string) []string` in `plugins/telemetry/utils.go`. This helper inserts metric-specific extra label values (e.g. `is_success`, `status_code`+`error_type`, `cache_type`) between the default labels and custom labels without mutating the input slice, replacing three identical inline `make`/`append` patterns in `PostLLMHook`. - Added `TestSpliceLabelValues` to pin the exact label-value ordering for latency, error, and cache-hit metrics. Because Prometheus metric vectors match values to label names purely by position, any reordering silently attributes values to the wrong labels; the test guards against this. - Fixed `alreadyCharged` map initialization in `reportBatchModelUsage` (governance plugin): the capacity hint `len(usage.BudgetIDs)+len(usage.RateLimitIDs)` was incorrect because `RateLimitIDs` entries are stored under a `"ratelimit:"` prefix and the map is populated in a separate loop, so the hint was misleading. The map is now initialized without a capacity hint. ## Type of change - [ ] Bug fix - [ ] Feature - [x] Refactor - [ ] Documentation - [ ] Chore/CI ## Affected areas - [ ] Core (Go) - [ ] Transports (HTTP) - [ ] Providers/Integrations - [x] Plugins - [ ] UI (React) - [ ] Docs ## How to test ```sh go test ./plugins/telemetry/... ./plugins/governance/... ``` The new `TestSpliceLabelValues` test verifies: - Extra values are inserted at the correct position between default and custom labels. - Multiple extras preserve their relative order. - The original input slice is never mutated. - Edge cases where there are no custom labels or no extras behave correctly. ## Screenshots/Recordings N/A ## Breaking changes - [x] No ## Related issues N/A ## Security considerations None. ## Checklist - [ ] I read `docs/contributing/README.md` and followed the guidelines - [x] I added/updated tests where appropriate - [ ] I updated documentation where needed - [x] I verified builds succeed (Go and UI) - [ ] I verified the CI pipeline passes locally if applicable --- plugins/governance/main.go | 2 +- plugins/telemetry/main.go | 20 ++++------- plugins/telemetry/main_test.go | 62 ++++++++++++++++++++++++++++++++++ plugins/telemetry/utils.go | 9 +++++ 4 files changed, 78 insertions(+), 15 deletions(-) diff --git a/plugins/governance/main.go b/plugins/governance/main.go index 53dbb230aa0..2fc8331f17e 100644 --- a/plugins/governance/main.go +++ b/plugins/governance/main.go @@ -1670,7 +1670,7 @@ func (p *GovernancePlugin) reportBatchModelUsage(ctx context.Context, usage joba if len(usage.ModelUsage) == 0 { return nil } - alreadyCharged := make(map[string]bool, len(usage.BudgetIDs)+len(usage.RateLimitIDs)) + alreadyCharged := make(map[string]bool) for _, id := range usage.BudgetIDs { alreadyCharged["budget:"+id] = true } diff --git a/plugins/telemetry/main.go b/plugins/telemetry/main.go index cd58432b9a1..3711757d8fa 100644 --- a/plugins/telemetry/main.go +++ b/plugins/telemetry/main.go @@ -1315,10 +1315,8 @@ func (p *PrometheusPlugin) PostLLMHook(ctx *schemas.BifrostContext, result *sche // Record latency duration := time.Since(startTime).Seconds() - latencyLabelValues := make([]string, 0, len(promLabelValues)+1) - latencyLabelValues = append(latencyLabelValues, promLabelValues[:len(p.defaultBifrostLabels)]...) // all default labels - latencyLabelValues = append(latencyLabelValues, strconv.FormatBool(bifrostErr == nil)) // is_success - latencyLabelValues = append(latencyLabelValues, promLabelValues[len(p.defaultBifrostLabels):]...) // then custom labels + // Default labels, then is_success, then custom labels. + latencyLabelValues := spliceLabelValues(promLabelValues, len(p.defaultBifrostLabels), strconv.FormatBool(bifrostErr == nil)) p.UpstreamLatencySeconds.WithLabelValues(latencyLabelValues...).Observe(duration) // SDK caller: no transport hooks fire, so this LLM-hook window is all there @@ -1343,11 +1341,8 @@ func (p *PrometheusPlugin) PostLLMHook(ctx *schemas.BifrostContext, result *sche // cannot disagree. Never empty: bifrostErr is non-nil in this branch. errorType := schemas.ClassifyErrorType(bifrostErr, requestType) - errorPromLabelValues := make([]string, 0, len(promLabelValues)+2) - errorPromLabelValues = append(errorPromLabelValues, promLabelValues[:len(p.defaultBifrostLabels)]...) // all default labels - errorPromLabelValues = append(errorPromLabelValues, statusCode) // status_code - errorPromLabelValues = append(errorPromLabelValues, string(errorType)) // error_type - errorPromLabelValues = append(errorPromLabelValues, promLabelValues[len(p.defaultBifrostLabels):]...) // then custom labels + // Default labels, then status_code and error_type, then custom labels. + errorPromLabelValues := spliceLabelValues(promLabelValues, len(p.defaultBifrostLabels), statusCode, string(errorType)) p.ErrorRequestsTotal.WithLabelValues(errorPromLabelValues...).Inc() } else { @@ -1464,11 +1459,8 @@ func (p *PrometheusPlugin) PostLLMHook(ctx *schemas.BifrostContext, result *sche cacheType = *extraFields.CacheDebug.HitType } - // Add cache_type to label values (create new slice to avoid modifying original) - cacheHitLabelValues := make([]string, 0, len(promLabelValues)+1) - cacheHitLabelValues = append(cacheHitLabelValues, promLabelValues[:len(p.defaultBifrostLabels)]...) // all default labels - cacheHitLabelValues = append(cacheHitLabelValues, cacheType) // cache_type - cacheHitLabelValues = append(cacheHitLabelValues, promLabelValues[len(p.defaultBifrostLabels):]...) // then custom labels + // Default labels, then cache_type, then custom labels (clone so the original is untouched). + cacheHitLabelValues := spliceLabelValues(promLabelValues, len(p.defaultBifrostLabels), cacheType) p.CacheHitsTotal.WithLabelValues(cacheHitLabelValues...).Inc() } diff --git a/plugins/telemetry/main_test.go b/plugins/telemetry/main_test.go index f5f0ba4ebd9..06e3f502352 100644 --- a/plugins/telemetry/main_test.go +++ b/plugins/telemetry/main_test.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "net/http/httptest" + "slices" "testing" "time" @@ -604,3 +605,64 @@ func TestApplyCustomLabels(t *testing.T) { }) } } + +// TestSpliceLabelValues pins the final label-value ordering produced for the latency, +// error, and cache-hit metrics: default labels first, then the metric's extra values +// (is_success / status_code+error_type / cache_type), then the custom labels. Metric +// vectors match values to label names purely by position, so any reordering here +// silently attributes values to the wrong labels. +func TestSpliceLabelValues(t *testing.T) { + defaults := []string{"openai", "gpt-4o", "chat"} // provider, model, method + custom := []string{"team-a", "env-prod"} // custom dimension labels + base := append(append([]string{}, defaults...), custom...) + + tests := []struct { + name string + in []string + extras []string + want []string + }{ + { + name: "latency is_success between defaults and custom labels", + in: base, + extras: []string{"true"}, + want: []string{"openai", "gpt-4o", "chat", "true", "team-a", "env-prod"}, + }, + { + name: "error status_code and error_type keep their order", + in: base, + extras: []string{"429", "rate_limit"}, + want: []string{"openai", "gpt-4o", "chat", "429", "rate_limit", "team-a", "env-prod"}, + }, + { + name: "cache_type between defaults and custom labels", + in: base, + extras: []string{"semantic"}, + want: []string{"openai", "gpt-4o", "chat", "semantic", "team-a", "env-prod"}, + }, + { + name: "no custom labels appends extras at the end", + in: defaults, + extras: []string{"true"}, + want: []string{"openai", "gpt-4o", "chat", "true"}, + }, + { + name: "no extras returns the input unchanged", + in: base, + extras: nil, + want: []string{"openai", "gpt-4o", "chat", "team-a", "env-prod"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + before := slices.Clone(tt.in) + got := spliceLabelValues(tt.in, len(defaults), tt.extras...) + if !slices.Equal(got, tt.want) { + t.Errorf("spliceLabelValues() = %v, want %v", got, tt.want) + } + if !slices.Equal(tt.in, before) { + t.Errorf("input mutated: %v, was %v", tt.in, before) + } + }) + } +} diff --git a/plugins/telemetry/utils.go b/plugins/telemetry/utils.go index 6f95ab346a1..12cfd5b9700 100644 --- a/plugins/telemetry/utils.go +++ b/plugins/telemetry/utils.go @@ -6,6 +6,7 @@ package telemetry import ( "log" "math" + "slices" "strings" schemas "github.com/maximhq/bifrost/core/schemas" @@ -62,6 +63,14 @@ func collectPrometheusKeyValues(ctx *fasthttp.RequestCtx) map[string]string { return labelValues } +// spliceLabelValues returns promLabelValues with extras inserted between the default +// labels (the first nDefault values) and the custom labels that follow them, matching +// metric definitions that declare default labels, then extras, then custom labels. +// The input slice is never mutated. +func spliceLabelValues(promLabelValues []string, nDefault int, extras ...string) []string { + return slices.Insert(slices.Clone(promLabelValues), nDefault, extras...) +} + // safeObserve safely records a value in a Prometheus histogram. // It prevents recording invalid values (negative or infinite) that could cause issues. func safeObserve(histogram *prometheus.HistogramVec, value float64, labels ...string) { From 920d743297a2a2d6abcded922df08060560ad755 Mon Sep 17 00:00:00 2001 From: Tejas Ghatte <64637256+TejasGhatte@users.noreply.github.com> Date: Mon, 21 Sep 2026 19:00:01 +0530 Subject: [PATCH 016/118] fix: allow all providers access and api models response (#7375) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary A permit that grants every provider names no provider permits for those providers, so the methods that return a list of providers had nothing to enumerate and would return only the providers that happened to have explicit overrides. This made the listing and routing layers refuse providers that the request path correctly admitted. This PR completes those lists from the deployment's configured provider set when an allow-all permit is in effect. ## Changes - Added a `ProviderLister` function type and `WithConfiguredProviders` option to `Access`, so a resolver can supply the deployment's provider set without affecting resolvers that have nothing to give. - `ProvidersForModel` and `GrantedProvidersForModel` now call `providersGrantedByAllowAll`, which enumerates configured providers not already named by any provider permit and checks each through `IsModelAllowed` — the same gate the request path uses — so the two cannot drift. - A provider that a provider permit explicitly refused is tracked in a `named` set and is not reopened by the allow-all path. - An intersecting scope is not widened: the allow-all completion still passes through `IsModelAllowed`, which applies composition, so a project-scoped intersection that permits only one provider continues to permit only that one. - `GovernancePlugin.ResolveAccess` now passes `configuredProviders()` to `NewAccess`. The lister reads the in-memory store at call time rather than capturing a snapshot, so a provider added after resolution is granted by the same rule. - The `listManagementModels` handler path in the HTTP transport is covered by a new test confirming that a caller permitted every provider sees models from providers it holds no explicit permit for. ## Type of change - [x] Bug fix ## Affected areas - [x] Core (Go) - [x] Transports (HTTP) - [x] Plugins ## How to test ```sh go test ./framework/grant/... ./transports/bifrost-http/handlers/... ./plugins/governance/... ``` The new `TestAllowAllProvidersCompletesTheProviderLists` suite in `framework/grant/access_test.go` covers the four cases: granted providers include one no permit names, candidates include that provider unweighted and unrestricted, a provider its own permit refused is not reopened, and an intersecting scope is not widened. `TestListModels_VKFilterListsProviderAllowedOnlyByAllowAll` covers the HTTP handler path end-to-end. ## Breaking changes - [ ] Yes - [x] No ## Related issues ## Security considerations The allow-all completion path gates every candidate through `IsModelAllowed`, which enforces blacklists and composition mode, so no provider or model is admitted beyond what the existing permission logic already permits. ## Checklist - [ ] I read `docs/contributing/README.md` and followed the guidelines - [x] I added/updated tests where appropriate - [ ] I updated documentation where needed - [x] I verified builds succeed (Go and UI) - [ ] I verified the CI pipeline passes locally if applicable --- plugins/governance/allowbydefault_test.go | 96 +++++++ plugins/governance/main.go | 1 + plugins/governance/store.go | 3 + plugins/governance/utils.go | 35 +++ ...ost-api-management.postman_collection.json | 265 ++++++++++++++++++ .../bifrost-http/handlers/providers_test.go | 57 ++++ transports/bifrost-http/server/server.go | 15 + transports/bifrost-http/server/server_test.go | 43 +++ .../views/virtualKeyDetailsSheet.tsx | 29 +- .../modelAccess/modelAccessBadges.tsx | 19 +- 10 files changed, 557 insertions(+), 6 deletions(-) diff --git a/plugins/governance/allowbydefault_test.go b/plugins/governance/allowbydefault_test.go index 8798b47ac03..dbe81fff302 100644 --- a/plugins/governance/allowbydefault_test.go +++ b/plugins/governance/allowbydefault_test.go @@ -23,6 +23,14 @@ func (m *mockInMemoryStore) GetConfiguredProviders() map[schemas.ModelProvider]c return m.configuredProviders } +func (m *mockInMemoryStore) GetConfiguredProviderNames() []string { + names := make([]string, 0, len(m.configuredProviders)) + for provider := range m.configuredProviders { + names = append(names, string(provider)) + } + return names +} + func (m *mockInMemoryStore) GetMCPClientsAllowedByDefault() map[string]string { return m.allowedByDefaultClients } @@ -196,3 +204,91 @@ func TestAppendMCPPermitsAllowedByDefault(t *testing.T) { assert.Equal(t, own, AppendMCPPermitsAllowedByDefault(own, configured, nil), "nothing allowed by default leaves the list as it was") } + +// A permit that grants every provider names none, so the providers it grants by the flag alone are +// materialised onto it at construction. Every consumer then reads one list and cannot disagree with +// the permit about what it grants — which is what let a listing refuse a provider the request path +// admitted. +func TestAppendAllProviderPermits(t *testing.T) { + configured := []string{"openai", "anthropic", "bedrock", ""} + + t.Run("materialises the providers no config names", func(t *testing.T) { + permits := AppendAllProviderPermits([]schemas.ProviderPermit{ + {Provider: "openai", AllowedModels: schemas.WhiteList{"gpt-4o"}, KeyIDs: schemas.WhiteList{"key-1"}}, + }, configured) + + byProvider := map[string]schemas.ProviderPermit{} + for _, permit := range permits { + byProvider[permit.Provider] = permit + } + if len(byProvider) != 3 { + t.Fatalf("expected openai, anthropic and bedrock, got %v", permits) + } + // The empty name the deployment reported is not a provider and must not become a permit. + if _, ok := byProvider[""]; ok { + t.Error("an empty provider name must not be materialised") + } + // A provider the permit already named keeps its own rules: the flag widens the set, it does + // not relax the overrides. + if got := byProvider["openai"].AllowedModels; len(got) != 1 || got[0] != "gpt-4o" { + t.Errorf("openai override was replaced: %v", got) + } + // One it did not name is granted with nothing narrowed, and no weight, since a weight is a + // routing preference a provider config expresses and this one expresses none. + anthropic := byProvider["anthropic"] + if !anthropic.AllowedModels.IsUnrestricted() { + t.Errorf("expected all models allowed, got %v", anthropic.AllowedModels) + } + if !schemas.WhiteList(anthropic.KeyIDs).IsUnrestricted() { + t.Errorf("expected all keys allowed, got %v", anthropic.KeyIDs) + } + if len(anthropic.BlacklistedModels) != 0 { + t.Errorf("expected nothing blocked, got %v", anthropic.BlacklistedModels) + } + if anthropic.Weight != nil { + t.Errorf("expected no weight, got %v", *anthropic.Weight) + } + }) + + t.Run("a deployment with no providers adds nothing", func(t *testing.T) { + permits := AppendAllProviderPermits(nil, nil) + if len(permits) != 0 { + t.Fatalf("expected no permits, got %v", permits) + } + }) +} + +// The end the fix is for: a key that grants every provider must answer for one it holds no config +// for, in the same list every consumer reads. Asked through the access, because that is what the +// listing routes and the routing allowlist ask. +func TestPermitForVirtualKeyGrantsEveryConfiguredProvider(t *testing.T) { + store := &LocalGovernanceStore{inMemoryStore: &mockInMemoryStore{ + configuredProviders: map[schemas.ModelProvider]configstore.ProviderConfig{ + schemas.OpenAI: {}, schemas.Anthropic: {}, schemas.Bedrock: {}, + }, + }} + // The customer's shape: allow-all, plus one provider config carrying an override. + vk := &configstoreTables.TableVirtualKey{ + ID: "vk-1", Name: "allow-all", AllowAllProviders: true, + ProviderConfigs: []configstoreTables.TableVirtualKeyProviderConfig{{ + Provider: string(schemas.Bedrock), AllowedModels: schemas.WhiteList{"anthropic.claude-haiku-4-5"}, AllowAllKeys: true, + }}, + } + access := grant.NewAccess([]schemas.Permit{store.permitForVirtualKey(emptyCtx(), vk)}, nil, "", nil) + + assert.ElementsMatch(t, []string{"openai", "anthropic", "bedrock"}, access.GrantedProvidersForModel(""), + "a provider the key names no config for is still granted") + // The override survives: the flag widens the provider set, it does not relax what a config says. + assert.True(t, access.IsModelAllowed("bedrock", "anthropic.claude-haiku-4-5")) + assert.False(t, access.IsModelAllowed("bedrock", "amazon.titan-embed-text-v2:0")) + // And a provider granted by the flag alone narrows nothing. + assert.True(t, access.IsModelAllowed("anthropic", "claude-haiku-4-5")) + + t.Run("without the flag only the named providers are granted", func(t *testing.T) { + vk.AllowAllProviders = false + access := grant.NewAccess([]schemas.Permit{store.permitForVirtualKey(emptyCtx(), vk)}, nil, "", nil) + + assert.Equal(t, []string{"bedrock"}, access.GrantedProvidersForModel("")) + assert.False(t, access.IsModelAllowed("anthropic", "claude-haiku-4-5")) + }) +} diff --git a/plugins/governance/main.go b/plugins/governance/main.go index 2fc8331f17e..57999567bd1 100644 --- a/plugins/governance/main.go +++ b/plugins/governance/main.go @@ -40,6 +40,7 @@ type Config struct { type InMemoryStore interface { GetConfiguredProviders() map[schemas.ModelProvider]configstore.ProviderConfig + GetConfiguredProviderNames() []string GetMCPClientsAllowedByDefault() map[string]string // clientID → clientName GetMCPClientNames() map[string]string // clientID → clientName, every client // GetMCPClientBySlug resolves a client by its endpoint slug (for serving one client at /mcp/). diff --git a/plugins/governance/store.go b/plugins/governance/store.go index 6094af6bc13..3127d0e1ff1 100644 --- a/plugins/governance/store.go +++ b/plugins/governance/store.go @@ -1451,6 +1451,9 @@ func (gs *LocalGovernanceStore) permitForVirtualKey(ctx context.Context, vk *con Weight: config.Weight, }) } + if vk.AllowAllProviders && gs.inMemoryStore != nil { + providerPermits = AppendAllProviderPermits(providerPermits, gs.inMemoryStore.GetConfiguredProviderNames()) + } // A key's own MCP configs and its Virtual MCPs build one accumulator, so a Virtual MCP grants // exactly like a config: owns the clients it names, unions per client, blocks allowed-by-default. diff --git a/plugins/governance/utils.go b/plugins/governance/utils.go index 6ba9c04ca07..9f491cf7833 100644 --- a/plugins/governance/utils.go +++ b/plugins/governance/utils.go @@ -258,3 +258,38 @@ func PresentedCredentialResolved(ctx *schemas.BifrostContext) bool { access := g.Access() return access != nil && unusablePermit(access) == nil } + +// AppendAllProviderPermits completes a permit that grants every provider. Such a permit names none, +// so the providers it grants by the flag alone are materialised here, from what the deployment has +// configured, and the permit then carries its whole grant in one readable list. +// +// Doing it at permit construction rather than where a consumer reads the permit is what keeps every +// consumer honest: enumerating provider permits and asking whether the permit allows a provider give +// the same answer, so a listing cannot refuse what the request path admits. Built per request, so a +// provider added after the permit was last written is granted by the same rule. +// +// A provider the permit already names keeps its own entry: those are overrides, and the flag widens +// the set rather than relaxing them. A materialised entry narrows nothing - every model, every key, +// nothing blocked - and carries no weight, because a weight is a routing preference a provider +// config expresses and this one expresses none. +func AppendAllProviderPermits(permits []schemas.ProviderPermit, configured []string) []schemas.ProviderPermit { + named := make(map[string]struct{}, len(permits)) + for i := range permits { + named[permits[i].Provider] = struct{}{} + } + for _, provider := range configured { + if provider == "" { + continue + } + if _, dup := named[provider]; dup { + continue + } + named[provider] = struct{}{} + permits = append(permits, schemas.ProviderPermit{ + Provider: provider, + AllowedModels: schemas.WhiteList{"*"}, + KeyIDs: schemas.WhiteList{"*"}, + }) + } + return permits +} diff --git a/tests/e2e/api/collections/bifrost-api-management.postman_collection.json b/tests/e2e/api/collections/bifrost-api-management.postman_collection.json index 7abdd2e0c5a..6a80c7b38c8 100644 --- a/tests/e2e/api/collections/bifrost-api-management.postman_collection.json +++ b/tests/e2e/api/collections/bifrost-api-management.postman_collection.json @@ -3783,6 +3783,271 @@ } ] }, + { + "name": "Governance - VK Allow All Providers", + "description": "A virtual key with allow_all_providers grants every provider, including ones it holds no provider_config for; the configs it does hold are overrides rather than the whole grant. The listing routes used to answer from the provider_config rows alone, so such a key saw only the providers it happened to override and every other provider vanished from /api/models*, /v1/models and the VK model pickers, while inference on them kept working.", + "item": [ + { + "name": "Baseline - catalog sizes with no VK", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('Baseline listing returns 200', function () { pm.response.to.have.status(200); });", + "// Parse defensively: an SPA fallback, a 401/403 or any non-JSON body makes an unguarded", + "// pm.response.json() throw a script error, which aborts the run instead of failing a", + "// named assertion and strands every request after this one.", + "var body = null;", + "try { body = pm.response.json(); } catch (e) { body = null; }", + "var models = (body && body.models) || [];", + "var counts = {};", + "models.forEach(function (m) { if (m && m.provider) counts[m.provider] = (counts[m.provider] || 0) + 1; });", + "pm.collectionVariables.set('vkall_base_anthropic', String(counts.anthropic || 0));", + "pm.collectionVariables.set('vkall_base_openai', String(counts.openai || 0));", + "// The scenario needs a provider to leave unnamed and one to override; without both", + "// configured there is nothing to tell apart and the assertions below would pass vacuously.", + "pm.test('anthropic is configured on this server', function () { pm.expect(counts.anthropic || 0).to.be.above(0); });", + "pm.test('openai is configured on this server', function () { pm.expect(counts.openai || 0).to.be.above(1); });" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models/details?limit=5000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models", + "details" + ], + "query": [ + { + "key": "limit", + "value": "5000" + } + ] + } + } + }, + { + "name": "Create VK (allow_all_providers, openai overridden)", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var ts = Date.now();", + "var body = {", + " name: 'VK AllowAllProviders ' + ts,", + " is_active: true,", + " allow_all_providers: true,", + " // The only provider_config: openai is narrowed to one model. Every other provider is", + " // granted by the flag alone and names no config at all.", + " provider_configs: [", + " { provider: 'openai', weight: 1, allowed_models: ['gpt-4o-mini'], key_ids: ['*'] }", + " ]", + "};", + "pm.request.body.raw = JSON.stringify(body);" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var code = pm.response.code;", + "pm.test('Create VK returns 2xx', function () { pm.expect(code).to.be.within(200, 299); });", + "if (code >= 200 && code <= 299) {", + " var json = null;", + " try { json = pm.response.json(); } catch (e) { json = null; }", + " var vk = (json && (json.virtual_key || json)) || {};", + " if (vk && vk.id) pm.collectionVariables.set('vkall_id', vk.id);", + " if (vk && vk.value) pm.collectionVariables.set('vkall_value', vk.value);", + " pm.test('allow_all_providers round-trips', function () { pm.expect(vk.allow_all_providers).to.eql(true); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{}" + }, + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys" + ] + } + } + }, + { + "name": "/api/models/details - unnamed provider is granted in full, overridden one stays narrowed", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('GET /api/models/details with VK returns 200', function () { pm.response.to.have.status(200); });", + "// Parse defensively: an SPA fallback, a 401/403 or any non-JSON body makes an unguarded", + "// pm.response.json() throw a script error, which aborts the run instead of failing a", + "// named assertion and strands every request after this one.", + "var body = null;", + "try { body = pm.response.json(); } catch (e) { body = null; }", + "var models = (body && body.models) || [];", + "var counts = {};", + "models.forEach(function (m) { if (m && m.provider) counts[m.provider] = (counts[m.provider] || 0) + 1; });", + "var baseAnthropic = Number(pm.collectionVariables.get('vkall_base_anthropic'));", + "var baseOpenai = Number(pm.collectionVariables.get('vkall_base_openai'));", + "// The regression: anthropic has no provider_config, so it is granted by the flag alone.", + "// Answering from the configs alone dropped it entirely.", + "pm.test('anthropic is listed though the VK names no config for it', function () {", + " pm.expect(counts.anthropic || 0).to.be.above(0);", + "});", + "pm.test('anthropic is granted in full, not narrowed', function () {", + " pm.expect(counts.anthropic).to.eql(baseAnthropic);", + "});", + "// And the other half: a config the VK does hold still narrows its provider, so the fix", + "// widens nothing it should not.", + "pm.test('openai stays narrowed by its own config', function () {", + " pm.expect(counts.openai || 0).to.be.below(baseOpenai);", + "});" + ] + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkall_value}}" + } + ], + "url": { + "raw": "{{base_url}}/api/models/details?limit=5000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models", + "details" + ], + "query": [ + { + "key": "limit", + "value": "5000" + } + ] + } + } + }, + { + "name": "/v1/models - fan-out reaches the unnamed provider", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('GET /v1/models with VK returns 200', function () { pm.response.to.have.status(200); });", + "// Parse defensively: an SPA fallback, a 401/403 or any non-JSON body makes an unguarded", + "// pm.response.json() throw a script error, which aborts the run instead of failing a", + "// named assertion and strands every request after this one.", + "var body = null;", + "try { body = pm.response.json(); } catch (e) { body = null; }", + "var data = (body && body.data) || [];", + "var seen = {};", + "data.forEach(function (m) {", + " var id = (m && m.id) || '';", + " var idx = id.indexOf('/');", + " if (idx > 0) seen[id.substring(0, idx)] = true;", + "});", + "// The fan-out narrows to the same answer, so an allow-all key must not be pinned to the", + "// providers it happens to override.", + "pm.test('/v1/models includes the unnamed provider', function () {", + " pm.expect(Object.keys(seen)).to.include('anthropic');", + "});" + ] + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkall_value}}" + } + ], + "url": { + "raw": "{{base_url}}/v1/models", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "models" + ] + } + } + }, + { + "name": "Teardown - Delete VK", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('Delete VK returns 2xx', function () { pm.expect(pm.response.code).to.be.within(200, 299); });" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys/{{vkall_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys", + "{{vkall_id}}" + ] + } + } + } + ] + }, { "name": "Governance - VK Regex Entries", "description": "regex: entries in the existing model lists (allowed_models / blacklisted_models on a VK provider config, models / blacklisted_models on a provider key): the text after the prefix is an RE2 pattern matched case-insensitively as a full match against the model name, next to exact names in the same list. Sections: write-time validation (400 on an unparsable pattern or regex:* on VK and provider-key endpoints), a base VK exercised on /v1/models plus real completions (family allowed, preview blocked, other family denied), a PUT that mixes an exact name with a regex entry, and a provider-key round-trip.", diff --git a/transports/bifrost-http/handlers/providers_test.go b/transports/bifrost-http/handlers/providers_test.go index 0dda79c0341..03fc65591b9 100644 --- a/transports/bifrost-http/handlers/providers_test.go +++ b/transports/bifrost-http/handlers/providers_test.go @@ -1436,6 +1436,63 @@ func accessForProviderPermits(permits ...schemas.ProviderPermit) schemas.Access return grant.NewAccess([]schemas.Permit{permit}, nil, "", nil) } +// accessAllowingAllProviders builds the access a caller permitted every provider carries, the way +// the governance store builds it: the providers its configs do not name are materialised onto the +// permit, so the permit carries its whole grant and every consumer reads one list. +func accessAllowingAllProviders(configured []string, permits ...schemas.ProviderPermit) schemas.Access { + permit := grant.NewPermit(grant.PermitVirtualKey, "vk-test", "Test VK", true, false, + governanceplugin.AppendAllProviderPermits(permits, configured), nil, + grant.WithAllowAllProviders(true)) + return grant.NewAccess([]schemas.Permit{permit}, nil, "", nil) +} + +// A caller permitted every provider is listed every provider, including ones it holds no provider +// permit for. Narrowing to the permits it happens to hold would make the listing refuse what the +// request path admits. +func TestListModels_VKFilterListsProviderAllowedOnlyByAllowAll(t *testing.T) { + SetLogger(&mockLogger{}) + + h := &ProviderHandler{ + inMemoryStore: &lib.Config{ + ClientConfig: &configstore.ClientConfig{}, + Providers: map[schemas.ModelProvider]configstore.ProviderConfig{ + schemas.OpenAI: {Keys: []schemas.Key{{ID: "key-a"}}}, + schemas.Anthropic: {Keys: []schemas.Key{{ID: "key-b"}}}, + }, + }, + modelsManager: &mockModelsManager{ + filtered: map[schemas.ModelProvider][]string{ + schemas.OpenAI: {"gpt-4o"}, + schemas.Anthropic: {"claude-haiku-4-5"}, + }, + }, + } + + query := modelListQuery{ + Limit: 100, + HasVKFilter: true, + Access: accessAllowingAllProviders( + []string{string(schemas.OpenAI), string(schemas.Anthropic)}, + schemas.ProviderPermit{Provider: "openai", AllowedModels: schemas.WhiteList{"*"}}, + ), + } + if !query.Access.IsProviderAllowed(string(schemas.Anthropic)) { + t.Fatal("control failed: allow-all must permit a provider it holds no permit for") + } + + models, total, err := h.listManagementModels(query) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + names := map[string]bool{} + for _, m := range models { + names[m.Name] = true + } + if total != 2 || !names["gpt-4o"] || !names["claude-haiku-4-5"] { + t.Fatalf("expected both providers listed, got total=%d models=%#v", total, models) + } +} + // A blacklisted model is not listed. The listing answers the same question a request does, so a // model the caller would be refused is not advertised to them as available. func TestListModels_VKFilterHidesBlacklistedModel(t *testing.T) { diff --git a/transports/bifrost-http/server/server.go b/transports/bifrost-http/server/server.go index 4e3cd2c4d4e..a8cb5853f9d 100644 --- a/transports/bifrost-http/server/server.go +++ b/transports/bifrost-http/server/server.go @@ -323,6 +323,21 @@ func (s *GovernanceInMemoryStore) GetConfiguredProviders() map[schemas.ModelProv return s.Config.Providers } +// GetConfiguredProviderNames builds the name slice under the lock, because provider edits write to +// the map GetConfiguredProviders hands back in place: ranging that map after the lock is released +// is a concurrent iteration and write, which is fatal rather than merely stale. +func (s *GovernanceInMemoryStore) GetConfiguredProviderNames() []string { + providers, err := s.Config.GetAllProviders() + if err != nil { + return nil + } + names := make([]string, 0, len(providers)) + for _, provider := range providers { + names = append(names, string(provider)) + } + return names +} + func (s *GovernanceInMemoryStore) GetMCPClientsAllowedByDefault() map[string]string { return s.Config.GetMCPClientsAllowedByDefault() } diff --git a/transports/bifrost-http/server/server_test.go b/transports/bifrost-http/server/server_test.go index a84a6781b42..7e1fb161b4d 100644 --- a/transports/bifrost-http/server/server_test.go +++ b/transports/bifrost-http/server/server_test.go @@ -1125,3 +1125,46 @@ func TestMarshalPluginConfig_WithComplexType(t *testing.T) { t.Errorf("Expected nested name=nested-config, got %s", result.Nested.Name) } } + +// GetConfiguredProviders hands back the live provider map uncopied, which is safe for the callers +// that index it once. The name listing cannot be written that way: provider add, delete and status +// updates all write to that map in place under the write lock, so a caller ranging it after the read +// lock is released hits a concurrent map iteration and write. That is fatal rather than merely +// stale — it takes the process down — so the slice is built under the lock instead. Run with -race. +func TestGetConfiguredProviderNamesIsSafeAgainstConcurrentProviderEdits(t *testing.T) { + config := &lib.Config{ + Providers: map[schemas.ModelProvider]configstore.ProviderConfig{schemas.OpenAI: {}}, + } + store := &GovernanceInMemoryStore{Config: config} + + churn := []schemas.ModelProvider{"churn-a", "churn-b", "churn-c"} + done := make(chan struct{}) + var writers sync.WaitGroup + writers.Add(1) + go func() { + defer writers.Done() + for { + select { + case <-done: + return + default: + } + // The same in-place mutation AddProvider and RemoveProvider make. + for _, provider := range churn { + config.Mu.Lock() + config.Providers[provider] = configstore.ProviderConfig{} + delete(config.Providers, provider) + config.Mu.Unlock() + } + } + }() + + for range 500 { + if names := store.GetConfiguredProviderNames(); len(names) == 0 { + t.Fatal("expected the configured providers to be listed") + } + } + + close(done) + writers.Wait() +} diff --git a/ui/app/workspace/virtual-keys/views/virtualKeyDetailsSheet.tsx b/ui/app/workspace/virtual-keys/views/virtualKeyDetailsSheet.tsx index 4ba166e7cf4..fb5dc6bf87b 100644 --- a/ui/app/workspace/virtual-keys/views/virtualKeyDetailsSheet.tsx +++ b/ui/app/workspace/virtual-keys/views/virtualKeyDetailsSheet.tsx @@ -1,7 +1,7 @@ import { BudgetOverrideDialog } from "@/components/budgetOverrideDialog"; import { BudgetOverrideManagerDialog, type BudgetOverrideSection } from "@/components/budgetOverrideManagerDialog"; import { CopyableId } from "@/components/copyableId"; -import { ModelAccessBadges } from "@/components/modelAccess"; +import { isWildcardList, ModelAccessBadges } from "@/components/modelAccess"; import { SheetNavigationButtons } from "@/components/sheetNavigationButtons"; import { Badge } from "@/components/ui/badge"; import { Label } from "@/components/ui/label"; @@ -235,11 +235,28 @@ export default function VirtualKeyDetailSheet({ {/* Provider Configurations */}
-

Provider Configurations

+
+

Provider Configurations

+ {virtualKey.allow_all_providers && ( + + All providers + + )} +
+ + {/* A key that allows every provider grants ones it holds no entry for, including ones added + later, so the entries below are overrides rather than the whole of what it may reach. */} + {virtualKey.allow_all_providers && ( +

+ Every provider is allowed, including ones added later. Entries below indicate specific provider level configuration. +

+ )}
{!virtualKey.provider_configs || virtualKey.provider_configs.length === 0 ? ( - No providers configured (deny-by-default) + + {virtualKey.allow_all_providers ? "No provider overrides" : "No providers configured (deny-by-default)"} + ) : (
{virtualKey.provider_configs.map((config, index) => ( @@ -279,7 +296,11 @@ export default function VirtualKeyDetailSheet({
Blocked Models
- +
diff --git a/ui/components/modelAccess/modelAccessBadges.tsx b/ui/components/modelAccess/modelAccessBadges.tsx index a462e0a631b..868868828f9 100644 --- a/ui/components/modelAccess/modelAccessBadges.tsx +++ b/ui/components/modelAccess/modelAccessBadges.tsx @@ -15,6 +15,12 @@ interface ModelAccessBadgesProps { * projects carry that as a separate all_models_allowed flag. */ allModels?: boolean; + /** + * For a block list, whether the allow side beside it permits every model. An empty block list + * blocks nothing only when it does: against a restricted allow list, every model outside that + * list is already denied, so "no models blocked" would read as "everything is available". + */ + allowsAllModels?: boolean; /** Rendered when the list is empty; defaults to the mode's standard badge. */ empty?: ReactNode; /** Badge variant for individual entries; defaults per mode. */ @@ -27,7 +33,16 @@ interface ModelAccessBadgesProps { * Read-only rendering of an allow or block side: the "All Models" badge, one * badge per entry (patterns in monospace with an icon), or an empty-state badge. */ -export function ModelAccessBadges({ value, mode, allModels, empty, entryVariant, entryClassName, className }: ModelAccessBadgesProps) { +export function ModelAccessBadges({ + value, + mode, + allModels, + allowsAllModels, + empty, + entryVariant, + entryClassName, + className, +}: ModelAccessBadgesProps) { const entries = (value ?? []).filter((e) => e !== "*"); const isAll = allModels || isWildcardList(value); @@ -51,7 +66,7 @@ export function ModelAccessBadges({ value, mode, allModels, empty, entryVariant, ) : ( - No models blocked + {allowsAllModels === false ? "All models except those allowed" : "No models blocked"} ); } From c686bdf751ff992eb1e4aca1fd56028107b1b52f Mon Sep 17 00:00:00 2001 From: Akshay Deo Date: Mon, 21 Sep 2026 21:28:54 +0530 Subject: [PATCH 017/118] typesafe models support, new decisions endpoint (#7355) ## Summary Adds support for new typesafe models. We are also introducing new /v1/decisions endpoint for systemone class of models ## Changes - Adds new typesafe provider - Adds new /v1/decisions endpoint along with BifrostDecisionRequest and BifrostDecisionResponse - Adds new /typesafe integration ## Type of change - [ ] Bug fix - [x] Feature - [ ] Refactor - [x] Documentation - [x] Chore/CI ## Affected areas - [x] Core (Go) - [x] Transports (HTTP) - [x] Providers/Integrations - [ ] Plugins - [x] UI (React) - [x] Docs ## How to test Describe the steps to validate this change. Include commands and expected outcomes. ```sh # Core/Transports go version go test ./... # UI cd ui pnpm i || npm i pnpm test || npm test pnpm build || npm run build ``` If adding new configs or environment variables, document them here. ## Screenshots/Recordings If UI changes, add before/after screenshots or short clips. ## Breaking changes - [ ] Yes - [x] No If yes, describe impact and migration instructions. ## Related issues NA ## Security considerations NA ## Checklist - [x] I read `docs/contributing/README.md` and followed the guidelines - [x] I added/updated tests where appropriate - [x] I updated documentation where needed - [x] I verified builds succeed (Go and UI) - [x] I verified the CI pipeline passes locally if applicable --- .claude/skills/harness-test-writer/SKILL.md | 7 +- .github/workflows/pr-tests.yml | 1 + .github/workflows/release-pipeline.yml | 10 + .github/workflows/run-core-tests.yml | 1 + AGENTS.md | 2 + Makefile | 12 +- core/bifrost.go | 68 ++ core/internal/llmtests/account.go | 25 + core/internal/llmtests/decision.go | 113 +++ core/internal/llmtests/tests.go | 2 + core/providers/anthropic/anthropic.go | 5 + core/providers/azure/azure.go | 5 + core/providers/bedrock/bedrock.go | 5 + core/providers/bedrockmantle/bedrockmantle.go | 5 + core/providers/cerebras/cerebras.go | 5 + core/providers/cohere/cohere.go | 5 + core/providers/databricks/unsupported.go | 5 + core/providers/deepseek/deepseek.go | 5 + core/providers/elevenlabs/elevenlabs.go | 5 + core/providers/fireworks/fireworks.go | 5 + core/providers/gemini/gemini.go | 5 + core/providers/githubcopilot/githubcopilot.go | 5 + core/providers/groq/groq.go | 5 + core/providers/huggingface/huggingface.go | 5 + core/providers/mistral/mistral.go | 5 + core/providers/nebius/nebius.go | 5 + core/providers/ollama/ollama.go | 5 + core/providers/openai/openai.go | 5 + core/providers/opencode/opencode.go | 5 + core/providers/openrouter/openrouter.go | 5 + core/providers/parasail/parasail.go | 5 + core/providers/perplexity/perplexity.go | 5 + core/providers/replicate/replicate.go | 5 + core/providers/runware/runware.go | 5 + core/providers/runway/runway.go | 5 + core/providers/sarvam/sarvam.go | 5 + core/providers/sgl/sgl.go | 5 + core/providers/typesafe/decision.go | 398 ++++++++ core/providers/typesafe/decision_test.go | 864 ++++++++++++++++++ core/providers/typesafe/errors.go | 72 ++ core/providers/typesafe/types.go | 67 ++ core/providers/typesafe/typesafe.go | 199 ++++ core/providers/typesafe/typesafe_test.go | 40 + core/providers/typesafe/unsupported.go | 229 +++++ core/providers/typesafe/utils.go | 89 ++ core/providers/vertex/vertex.go | 5 + core/providers/vllm/vllm.go | 5 + core/providers/wafer/wafer.go | 5 + core/providers/xai/xai.go | 5 + core/schemas/bifrost.go | 22 + core/schemas/decision.go | 65 ++ core/schemas/errors.go | 1 + core/schemas/errors_test.go | 1 + core/schemas/provider.go | 5 + docs/docs.json | 19 + docs/integrations/typesafe-sdk.mdx | 95 ++ docs/openapi/openapi.json | 539 +++++++++++ docs/openapi/openapi.yaml | 22 + docs/openapi/paths/inference/decisions.yaml | 31 + .../integrations/typesafe/systemone.yaml | 171 ++++ docs/openapi/schemas/inference/decisions.yaml | 114 +++ .../supported-providers/overview.mdx | 65 +- .../supported-providers/typesafe.mdx | 189 ++++ docs/quickstart/gateway/decisions.mdx | 94 ++ docs/quickstart/go-sdk/decisions.mdx | 107 +++ framework/modelcatalog/datasheet/cost.go | 19 + framework/modelcatalog/datasheet/cost_test.go | 74 ++ framework/modelcatalog/datasheet/store.go | 1 + framework/modelcatalog/datasheet/types.go | 2 + plugins/logging/main.go | 2 + plugins/logging/operations.go | 22 + plugins/logging/utils.go | 16 + tests/cmd/e2eseed/go.mod | 2 +- tests/cmd/seed/go.mod | 2 +- tests/cmd/seedvks/go.mod | 2 +- .../e2e/api/collections/provider-harness.json | 261 ++++++ tests/e2e/api/runners/filter-collection.mjs | 1 + tests/integrations/python/config.json | 20 + tests/integrations/python/pyproject.toml | 9 +- .../python/tests/test_typesafe.py | 194 ++++ tests/integrations/python/uv.lock | 17 + tests/integrations/typescript/config.json | 15 + .../integrations/typescript/package-lock.json | 10 + tests/integrations/typescript/package.json | 5 +- .../typescript/tests/test-typesafe.test.ts | 143 +++ transports/bifrost-http/handlers/inference.go | 71 ++ .../bifrost-http/handlers/integrations.go | 1 + .../bifrost-http/integrations/router.go | 29 + .../bifrost-http/integrations/typesafe.go | 97 ++ transports/bifrost-http/integrations/utils.go | 4 + transports/config.schema.json | 10 +- ui/app/__error.tsx | 2 +- .../sheets/customerDetailSheet.tsx | 12 +- .../virtual-mcps/virtualMcpAccessProfiles.tsx | 2 +- ui/app/main.tsx | 2 +- ui/app/workspace/complexity-router/page.tsx | 2 +- .../views/classifierStatusBadge.tsx | 8 +- .../config/views/clientSettingsView.tsx | 2 +- .../config/views/compatibilityView.tsx | 4 +- .../workspace/config/views/securityView.tsx | 11 +- .../overrides/pricingFields.test.ts | 2 +- .../dashboard/components/charts/barShape.tsx | 14 +- ui/app/workspace/dashboard/page.tsx | 2 +- .../workspace/edge-control/devices/page.tsx | 5 +- .../workspace/edge-control/inventory/page.tsx | 5 +- ui/app/workspace/governance/teams/page.tsx | 5 +- .../governance/views/customerSheet.tsx | 4 +- .../workspace/governance/views/teamSheet.tsx | 4 +- ui/app/workspace/logs/page.tsx | 2 +- .../workspace/logs/sheets/logDetailView.tsx | 364 +++++--- .../logs/sheets/logDetailView.utils.test.ts | 14 +- .../workspace/logs/sheets/logDetailsSheet.tsx | 4 +- .../workspace/logs/views/logsHeaderView.tsx | 2 +- .../logs/views/metricStrip.utils.test.ts | 15 +- .../workspace/logs/views/metricStrip.utils.ts | 2 +- .../views/mcpLibraryAddServerSheet.tsx | 6 +- .../library/views/mcpLibrarySettingsSheet.tsx | 11 +- ui/app/workspace/mcp-tool-groups/layout.tsx | 2 +- .../model-limits/views/modelLimitsTable.tsx | 66 +- .../fragments/otelFormFragment.tsx | 112 ++- .../sheets/pluginTracingSheet.tsx | 6 +- .../dialogs/databricksMigrationDialog.tsx | 36 +- .../firstPartyProviderAvailableDialog.tsx | 6 +- .../fragments/apiKeysFormFragment.tsx | 228 ++--- .../fragments/governanceFormFragment.tsx | 3 +- ui/app/workspace/providers/page.tsx | 21 +- .../views/providerKeyForm.utils.test.ts | 2 +- .../providers/views/providerKeyForm.utils.ts | 2 +- .../hooks/useVirtualKeyUsage.utils.test.ts | 2 +- .../hooks/useVirtualKeyUsage.utils.ts | 2 +- .../virtual-keys/views/virtualKeySheet.tsx | 60 +- .../views/virtualKeySheet.utils.test.ts | 2 +- .../views/virtualKeySheet.utils.ts | 2 +- .../virtual-keys/views/virtualKeysTable.tsx | 2 +- .../views/virtualKeysTable.utils.test.ts | 8 +- .../views/virtualKeysTable.utils.ts | 2 +- ui/app/workspace/virtual-mcps/layout.tsx | 2 +- .../virtual-mcps/views/virtualMcpSheet.tsx | 2 +- .../virtual-mcps/views/virtualMcpsTable.tsx | 124 +-- .../deliveries/deliveries.page.utils.test.ts | 2 +- .../deliveries/deliveries.page.utils.ts | 2 +- .../deliveries/views/deliveriesHeaderView.tsx | 2 +- .../workspace/webhooks/views/webhookSheet.tsx | 38 +- ui/components/budgetDisplay.test.ts | 2 +- ui/components/budgetDisplay.tsx | 2 +- ui/components/filters/logsFilterSidebar.tsx | 24 +- ui/components/mcp/mcpClientConfigsEditor.tsx | 5 +- .../notificationCenter.utils.test.ts | 2 +- ui/components/rateLimitDisplay.test.ts | 2 +- ui/components/sidebar.tsx | 120 +-- ui/components/topbar.test.ts | 2 +- ui/components/ui/codeEditor.tsx | 4 +- ui/components/ui/quarterStartSelect.tsx | 2 +- ui/components/ui/sidebar.tsx | 6 +- ui/hooks/useWebSocket.tsx | 2 +- ui/hooks/useWebSocket.utils.test.ts | 2 +- ui/hooks/useWebSocket.utils.ts | 2 +- ui/index.html | 183 +++- ui/lib/constants/config.ts | 2 + ui/lib/constants/icons.tsx | 24 + ui/lib/constants/logs.ts | 5 + .../hooks/useDismissedProviderCollisions.ts | 2 +- ui/lib/schemas/providerForm.copilot.test.ts | 2 +- ui/lib/store/apis/governanceApi.ts | 2 +- ui/lib/store/apis/logsApi.ts | 2 +- ui/lib/store/apis/virtualMcpsApi.ts | 2 +- ui/lib/store/slices/appSlice.test.ts | 7 +- ui/lib/store/slices/appSlice.ts | 2 +- ui/lib/types/complexityRouter.test.ts | 2 +- ui/lib/types/schemas.databricks.test.ts | 2 +- ui/lib/types/virtualMcps.ts | 2 +- ui/lib/utils/databricksMigration.test.ts | 21 +- ui/lib/utils/databricksMigration.ts | 8 +- ui/lib/utils/duration.test.ts | 2 +- ui/lib/utils/duration.ts | 2 +- ui/lib/utils/providerCollision.test.ts | 2 +- ui/lib/utils/providerCollision.ts | 2 +- ui/lib/utils/validation.test.ts | 11 +- ui/lib/utils/validation.ts | 2 +- 179 files changed, 5920 insertions(+), 711 deletions(-) create mode 100644 core/internal/llmtests/decision.go create mode 100644 core/providers/typesafe/decision.go create mode 100644 core/providers/typesafe/decision_test.go create mode 100644 core/providers/typesafe/errors.go create mode 100644 core/providers/typesafe/types.go create mode 100644 core/providers/typesafe/typesafe.go create mode 100644 core/providers/typesafe/typesafe_test.go create mode 100644 core/providers/typesafe/unsupported.go create mode 100644 core/providers/typesafe/utils.go create mode 100644 core/schemas/decision.go create mode 100644 docs/integrations/typesafe-sdk.mdx create mode 100644 docs/openapi/paths/inference/decisions.yaml create mode 100644 docs/openapi/paths/integrations/typesafe/systemone.yaml create mode 100644 docs/openapi/schemas/inference/decisions.yaml create mode 100644 docs/providers/supported-providers/typesafe.mdx create mode 100644 docs/quickstart/gateway/decisions.mdx create mode 100644 docs/quickstart/go-sdk/decisions.mdx create mode 100644 tests/integrations/python/tests/test_typesafe.py create mode 100644 tests/integrations/typescript/tests/test-typesafe.test.ts create mode 100644 transports/bifrost-http/integrations/typesafe.go diff --git a/.claude/skills/harness-test-writer/SKILL.md b/.claude/skills/harness-test-writer/SKILL.md index 3857c67db33..4075c92914d 100644 --- a/.claude/skills/harness-test-writer/SKILL.md +++ b/.claude/skills/harness-test-writer/SKILL.md @@ -104,9 +104,10 @@ Conventions (match the existing collection exactly): pins the same invariant independently (e.g. native `/v1/responses` alongside the converted `/v1/chat/completions` path). - **Test scripts** (Postman `event[].script.exec`, plain ES5 JavaScript): - - Start with an infra guard so auth/rate/server noise skips instead of false-failing: - `if ([401, 403, 429, 500, 502, 503, 504].indexOf(pm.response.code) !== -1) { return; }` - Do NOT guard on 400 when a 400 IS the regression signature - that must fail loudly. + - Never skip any error status code. Do NOT open the script with an early-return + guard like `if ([401, 403, 429, 500, 502, 503, 504].indexOf(pm.response.code) !== -1) { return; }` + - every unexpected status (auth failure, rate limit, 5xx) must fail the case + loudly instead of silently passing it. Assert the exact status the case expects. - Assert the specific failure signature is absent (error substring, param name) AND that the happy path succeeded (status below 400, expected fields present). - Include the response text in failure messages: diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 715d79e3dc3..592f1e7b963 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -132,6 +132,7 @@ jobs: SGL_API_KEY: ${{ secrets.SGL_API_KEY }} CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }} + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WORKSPACE_URL: ${{ secrets.DATABRICKS_WORKSPACE_URL }} GITHUB_COPILOT_APP_ID: ${{ secrets.GITHUB_COPILOT_APP_ID }} diff --git a/.github/workflows/release-pipeline.yml b/.github/workflows/release-pipeline.yml index 078c9ecc59c..be9a1c683d0 100644 --- a/.github/workflows/release-pipeline.yml +++ b/.github/workflows/release-pipeline.yml @@ -440,6 +440,7 @@ jobs: CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }} + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WORKSPACE_URL: ${{ secrets.DATABRICKS_WORKSPACE_URL }} GITHUB_COPILOT_APP_ID: ${{ secrets.GITHUB_COPILOT_APP_ID }} @@ -667,6 +668,7 @@ jobs: CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }} + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WORKSPACE_URL: ${{ secrets.DATABRICKS_WORKSPACE_URL }} GITHUB_COPILOT_APP_ID: ${{ secrets.GITHUB_COPILOT_APP_ID }} @@ -1234,6 +1236,7 @@ jobs: SGL_API_KEY: ${{ secrets.SGL_API_KEY }} CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }} + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WORKSPACE_URL: ${{ secrets.DATABRICKS_WORKSPACE_URL }} COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} @@ -1361,6 +1364,7 @@ jobs: SGL_API_KEY: ${{ secrets.SGL_API_KEY }} CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }} + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WORKSPACE_URL: ${{ secrets.DATABRICKS_WORKSPACE_URL }} COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} @@ -1682,6 +1686,7 @@ jobs: CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }} + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WORKSPACE_URL: ${{ secrets.DATABRICKS_WORKSPACE_URL }} GITHUB_COPILOT_APP_ID: ${{ secrets.GITHUB_COPILOT_APP_ID }} @@ -2032,6 +2037,7 @@ jobs: PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }} CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }} + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WORKSPACE_URL: ${{ secrets.DATABRICKS_WORKSPACE_URL }} GITHUB_COPILOT_APP_ID: ${{ secrets.GITHUB_COPILOT_APP_ID }} @@ -2182,6 +2188,7 @@ jobs: PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }} CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }} + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WORKSPACE_URL: ${{ secrets.DATABRICKS_WORKSPACE_URL }} GITHUB_COPILOT_APP_ID: ${{ secrets.GITHUB_COPILOT_APP_ID }} @@ -2307,6 +2314,7 @@ jobs: SGL_API_KEY: ${{ secrets.SGL_API_KEY }} CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }} + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WORKSPACE_URL: ${{ secrets.DATABRICKS_WORKSPACE_URL }} COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} @@ -2427,6 +2435,7 @@ jobs: SGL_API_KEY: ${{ secrets.SGL_API_KEY }} CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }} + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WORKSPACE_URL: ${{ secrets.DATABRICKS_WORKSPACE_URL }} COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} @@ -2563,6 +2572,7 @@ jobs: SGL_API_KEY: ${{ secrets.SGL_API_KEY }} CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }} + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WORKSPACE_URL: ${{ secrets.DATABRICKS_WORKSPACE_URL }} COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} diff --git a/.github/workflows/run-core-tests.yml b/.github/workflows/run-core-tests.yml index c175908c64a..5f0712dd338 100644 --- a/.github/workflows/run-core-tests.yml +++ b/.github/workflows/run-core-tests.yml @@ -60,6 +60,7 @@ jobs: CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }} + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WORKSPACE_URL: ${{ secrets.DATABRICKS_WORKSPACE_URL }} GITHUB_COPILOT_APP_ID: ${{ secrets.GITHUB_COPILOT_APP_ID }} diff --git a/AGENTS.md b/AGENTS.md index 7a2236b5548..490aa0250bc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -579,6 +579,8 @@ Any change under `core/`, `framework/`, `transports/bifrost-http/`, or `plugins/ These layers all sit on the request path, so any of them can change the bytes a client sees - and that end-to-end behaviour is what the harness exists to pin. A Go unit test proves the function does what you meant; only the harness proves the bytes a real client sends still come back correct through the whole stack. The gap between those two is where regressions live: a fail-soft that fires on one request shape and silently skips a sibling shape passes every unit test it has. +**Never skip any error status code in a harness test script.** Do not open a test with an early-return guard like `if ([401, 403, 429, 500, 502, 503, 504].indexOf(pm.response.code) !== -1) { return; }` — every unexpected status, including auth failures, rate limits, and 5xx, must fail the assertion loudly rather than silently passing the case. Assert the exact status (or bound) the case expects and include `pm.response.text()` in the failure message. + Write the case so it is **red before the change and green after**, and validate it structurally while developing — no live paid run needed: ```bash diff --git a/Makefile b/Makefile index 982407b3d31..6f283ecbcbd 100644 --- a/Makefile +++ b/Makefile @@ -1446,24 +1446,24 @@ test-integrations-py: ## Run Python integration tests (Usage: make test-integrat if [ -n "$(TESTCASE)" ]; then \ $(ECHO) "$(CYAN)Running $(INTEGRATION) integration test: $(TESTCASE)...$(NC)"; \ if [[ "$(TESTCASE)" == *::* ]]; then \ - uv run pytest tests/test_$(INTEGRATION).py::$(TESTCASE) $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \ + uv run --frozen pytest tests/test_$(INTEGRATION).py::$(TESTCASE) $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \ else \ - uv run pytest tests/test_$(INTEGRATION).py -k "$(TESTCASE)" $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \ + uv run --frozen pytest tests/test_$(INTEGRATION).py -k "$(TESTCASE)" $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \ fi; \ elif [ -n "$(PATTERN)" ]; then \ $(ECHO) "$(CYAN)Running $(INTEGRATION) integration tests matching '$(PATTERN)'...$(NC)"; \ - uv run pytest tests/test_$(INTEGRATION).py -k "$(PATTERN)" $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \ + uv run --frozen pytest tests/test_$(INTEGRATION).py -k "$(PATTERN)" $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \ else \ $(ECHO) "$(CYAN)Running $(INTEGRATION) integration tests...$(NC)"; \ - uv run pytest tests/test_$(INTEGRATION).py $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \ + uv run --frozen pytest tests/test_$(INTEGRATION).py $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \ fi; \ else \ if [ -n "$(PATTERN)" ]; then \ $(ECHO) "$(CYAN)Running all integration tests matching '$(PATTERN)'...$(NC)"; \ - uv run pytest -k "$(PATTERN)" $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \ + uv run --frozen pytest -k "$(PATTERN)" $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \ else \ $(ECHO) "$(CYAN)Running all integration tests...$(NC)"; \ - uv run pytest $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \ + uv run --frozen pytest $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \ fi; \ fi; \ fi; \ diff --git a/core/bifrost.go b/core/bifrost.go index 5a72f2124f6..d975b5d62b8 100644 --- a/core/bifrost.go +++ b/core/bifrost.go @@ -47,6 +47,7 @@ import ( "github.com/maximhq/bifrost/core/providers/runway" "github.com/maximhq/bifrost/core/providers/sarvam" "github.com/maximhq/bifrost/core/providers/sgl" + "github.com/maximhq/bifrost/core/providers/typesafe" providerUtils "github.com/maximhq/bifrost/core/providers/utils" "github.com/maximhq/bifrost/core/providers/vertex" "github.com/maximhq/bifrost/core/providers/vllm" @@ -1445,6 +1446,58 @@ func (bifrost *Bifrost) RerankRequest(ctx *schemas.BifrostContext, req *schemas. return response.RerankResponse, nil } +// DecisionRequest sends an decision request to the specified provider. +func (bifrost *Bifrost) DecisionRequest(ctx *schemas.BifrostContext, req *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + if req == nil { + return nil, &schemas.BifrostError{ + IsBifrostError: false, + Error: &schemas.ErrorField{ + Message: "decision request is nil", + }, + ExtraFields: schemas.BifrostErrorExtraFields{ + RequestType: schemas.DecisionRequest, + }, + } + } + if req.State == nil { + return nil, &schemas.BifrostError{ + IsBifrostError: false, + Error: &schemas.ErrorField{ + Message: "state not provided for decision request", + }, + ExtraFields: schemas.BifrostErrorExtraFields{ + RequestType: schemas.DecisionRequest, + Provider: req.Provider, + OriginalModelRequested: req.Model, + ResolvedModelUsed: req.Model, + }, + } + } + if len(req.Questions) == 0 { + return nil, &schemas.BifrostError{ + IsBifrostError: false, + Error: &schemas.ErrorField{ + Message: "questions not provided for decision request", + }, + ExtraFields: schemas.BifrostErrorExtraFields{ + RequestType: schemas.DecisionRequest, + Provider: req.Provider, + OriginalModelRequested: req.Model, + ResolvedModelUsed: req.Model, + }, + } + } + bifrostReq := bifrost.getBifrostRequest() + bifrostReq.RequestType = schemas.DecisionRequest + bifrostReq.DecisionRequest = req + + response, err := bifrost.handleRequest(ctx, bifrostReq) + if err != nil { + return nil, err + } + return response.DecisionResponse, nil +} + // OCRRequest sends an OCR request to the specified provider. func (bifrost *Bifrost) OCRRequest(ctx *schemas.BifrostContext, req *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { if req == nil { @@ -4587,6 +4640,8 @@ func (bifrost *Bifrost) createBaseProvider(providerKey schemas.ModelProvider, co return sarvam.NewSarvamProvider(config, bifrost.logger) case schemas.Databricks: return databricks.NewDatabricksProvider(config, bifrost.logger) + case schemas.Typesafe: + return typesafe.NewTypesafeProvider(config, bifrost.logger) default: return nil, fmt.Errorf("unsupported provider: %s", targetProviderKey) } @@ -5176,6 +5231,12 @@ func (bifrost *Bifrost) prepareFallbackRequest(req *schemas.BifrostRequest, fall tmp.Model = fallback.Model fallbackReq.RerankRequest = &tmp } + if req.DecisionRequest != nil { + tmp := *req.DecisionRequest + tmp.Provider = fallback.Provider + tmp.Model = fallback.Model + fallbackReq.DecisionRequest = &tmp + } if req.OCRRequest != nil { tmp := *req.OCRRequest tmp.Provider = fallback.Provider @@ -7798,6 +7859,12 @@ func (bifrost *Bifrost) handleProviderRequest(provider schemas.Provider, config return nil, bifrostError } response.RerankResponse = rerankResponse + case schemas.DecisionRequest: + decisionResponse, bifrostError := provider.Decision(req.Context, key, req.BifrostRequest.DecisionRequest) + if bifrostError != nil { + return nil, bifrostError + } + response.DecisionResponse = decisionResponse case schemas.OCRRequest: var customProviderConfig *schemas.CustomProviderConfig if config != nil { @@ -8991,6 +9058,7 @@ func resetBifrostRequest(req *schemas.BifrostRequest) { req.CompactionRequest = nil req.EmbeddingRequest = nil req.RerankRequest = nil + req.DecisionRequest = nil req.OCRRequest = nil req.SpeechRequest = nil req.TranscriptionRequest = nil diff --git a/core/internal/llmtests/account.go b/core/internal/llmtests/account.go index 7cebf073f8f..7cf620b9627 100644 --- a/core/internal/llmtests/account.go +++ b/core/internal/llmtests/account.go @@ -91,6 +91,7 @@ type TestScenarios struct { ContainerFileDelete bool // Container File API delete functionality PassThroughExtraParams bool // Pass through extra params functionality Rerank bool // Rerank functionality + Decision bool // Decision functionality (annotated function-tool evaluation) PassthroughAPI bool // Raw HTTP passthrough API (Passthrough + PassthroughStream) WebSocketResponses bool // WebSocket Responses API mode Realtime bool // Realtime API (bidirectional audio/text) @@ -114,6 +115,7 @@ type ComprehensiveTestConfig struct { ReasoningModel string EmbeddingModel string RerankModel string + DecisionModel string TranscriptionModel string SpeechSynthesisModel string ChatAudioModel string @@ -124,6 +126,7 @@ type ComprehensiveTestConfig struct { SpeechSynthesisFallbacks []schemas.Fallback // for speech synthesis tests EmbeddingFallbacks []schemas.Fallback // for embedding tests RerankFallbacks []schemas.Fallback // for rerank tests + DecisionFallbacks []schemas.Fallback // for decision tests SkipReason string // Reason to skip certain tests ImageGenerationModel string // Model for image generation ImageGenerationFallbacks []schemas.Fallback // Fallbacks for image generation @@ -198,6 +201,7 @@ func (account *ComprehensiveTestAccount) GetConfiguredProviders() ([]schemas.Mod schemas.Wafer, schemas.Databricks, schemas.GithubCopilot, + schemas.Typesafe, ProviderOpenAICustom, }, nil } @@ -594,6 +598,14 @@ func (account *ComprehensiveTestAccount) GetKeysForProvider(ctx context.Context, UseForBatchAPI: bifrost.Ptr(true), }, }, nil + case schemas.Typesafe: + return []schemas.Key{ + { + Value: *schemas.NewSecretVar("env.TYPESAFE_API_KEY"), + Models: []string{"*"}, + Weight: 1.0, + }, + }, nil case schemas.Fireworks: return []schemas.Key{ { @@ -1064,6 +1076,19 @@ func (account *ComprehensiveTestAccount) GetConfigForProvider(providerKey schema BufferSize: 10, }, }, nil + case schemas.Typesafe: + return &schemas.ProviderConfig{ + NetworkConfig: schemas.NetworkConfig{ + DefaultRequestTimeoutInSeconds: 300, + MaxRetries: 10, + RetryBackoffInitial: 1 * time.Second, + RetryBackoffMax: 12 * time.Second, + }, + ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ + Concurrency: Concurrency, + BufferSize: 10, + }, + }, nil case schemas.Fireworks: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ diff --git a/core/internal/llmtests/decision.go b/core/internal/llmtests/decision.go new file mode 100644 index 00000000000..76ec3756647 --- /dev/null +++ b/core/internal/llmtests/decision.go @@ -0,0 +1,113 @@ +package llmtests + +import ( + "context" + "os" + "strings" + "testing" + + bifrost "github.com/maximhq/bifrost/core" + "github.com/maximhq/bifrost/core/schemas" +) + +// BasicDecisionExpectations validates common decision invariants for provider +// tests: every requested question produces an answer of its declared kind with +// a well-typed value. +func BasicDecisionExpectations(t *testing.T, response *schemas.BifrostDecisionResponse, request *schemas.BifrostDecisionRequest) { + t.Helper() + + if response == nil { + t.Fatal("❌ Decision response is nil") + } + if len(response.Answers) == 0 { + t.Fatal("❌ Decision response carries no answers") + } + + for name, question := range request.Questions { + answer, ok := response.Answers[name] + if !ok { + t.Fatalf("❌ Question %q produced no answer", name) + } + if answer.Kind != question.Kind { + t.Fatalf("❌ Answer %q has kind %q; expected %q", name, answer.Kind, question.Kind) + } + switch answer.Kind { + case schemas.DecisionKindNoul: + number, ok := answer.Value.(float64) + if !ok || number < 0 || number > 1 { + t.Fatalf("❌ Noul answer %q is not a number in [0,1]: %v", name, answer.Value) + } + case schemas.DecisionKindChoice: + if _, ok := answer.Value.(string); !ok { + t.Fatalf("❌ Choice answer %q is not a string: %v", name, answer.Value) + } + case schemas.DecisionKindScore: + if _, ok := answer.Value.(float64); !ok { + t.Fatalf("❌ Score answer %q is not a number: %v", name, answer.Value) + } + default: + t.Fatalf("❌ Answer %q has unknown kind %q", name, answer.Kind) + } + } +} + +// RunDecisionTest executes the decision test scenario against a live provider. +func RunDecisionTest(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) { + if !testConfig.Scenarios.Decision { + t.Logf("Decision not supported for provider %s", testConfig.Provider) + return + } + + if strings.TrimSpace(testConfig.DecisionModel) == "" { + t.Skipf("Decision enabled but model is not configured for provider %s; skipping", testConfig.Provider) + } + + t.Run("Decision", func(t *testing.T) { + if os.Getenv("SKIP_PARALLEL_TESTS") != "true" { + t.Parallel() + } + + state := "Customer message: I was double charged for my subscription last month and nobody has replied to my two previous emails. I want a refund today or I am cancelling." + + request := &schemas.BifrostDecisionRequest{ + Provider: testConfig.Provider, + Model: testConfig.DecisionModel, + State: state, + Questions: map[string]schemas.DecisionQuestion{ + "is_frustrated": { + Kind: schemas.DecisionKindNoul, + Instructions: "Is the customer frustrated?", + }, + "category": { + Kind: schemas.DecisionKindChoice, + Instructions: "Pick the ticket category", + Criteria: map[string]interface{}{ + "billing": "charges, refunds, invoices", + "bug": "product defects", + "other": "anything else", + }, + }, + "urgency": { + Kind: schemas.DecisionKindScore, + Instructions: "Rate how urgently this ticket needs a human reply", + Criteria: []interface{}{"can wait a week", "should be answered soon", "needs a reply today"}, + }, + }, + Fallbacks: testConfig.DecisionFallbacks, + } + + bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline) + response, bifrostErr := client.DecisionRequest(bfCtx, request) + + if bifrostErr != nil { + t.Fatalf("❌ Decision request failed: %v", GetErrorMessage(bifrostErr)) + } + + BasicDecisionExpectations(t, response, request) + + if response.Usage != nil { + t.Logf("📊 Usage: prompt_tokens=%d, total_tokens=%d", response.Usage.PromptTokens, response.Usage.TotalTokens) + } + t.Logf("✅ Decision test passed: model=%s, answers=%d", response.Model, len(response.Answers)) + }) +} diff --git a/core/internal/llmtests/tests.go b/core/internal/llmtests/tests.go index 30f94e1c5a9..a8f43b760eb 100644 --- a/core/internal/llmtests/tests.go +++ b/core/internal/llmtests/tests.go @@ -58,6 +58,7 @@ func RunAllComprehensiveTests(t *testing.T, client *bifrost.Bifrost, ctx context RunTranscriptionStreamAdvancedTest, RunEmbeddingTest, RunRerankTest, + RunDecisionTest, RunChatCompletionReasoningTest, RunMultiTurnReasoningTest, RunResponsesReasoningTest, @@ -178,6 +179,7 @@ func printTestSummary(t *testing.T, testConfig ComprehensiveTestConfig) { {"TranscriptionStream", testConfig.Scenarios.TranscriptionStream}, {"Embedding", testConfig.Scenarios.Embedding && testConfig.EmbeddingModel != ""}, {"Rerank", testConfig.Scenarios.Rerank && testConfig.RerankModel != ""}, + {"Decision", testConfig.Scenarios.Decision && testConfig.DecisionModel != ""}, {"ChatCompletionReasoning", testConfig.Scenarios.Reasoning && testConfig.ReasoningModel != ""}, {"MultiTurnReasoning", testConfig.Scenarios.Reasoning && testConfig.ReasoningModel != ""}, {"ResponsesReasoning", testConfig.Scenarios.Reasoning && testConfig.ReasoningModel != ""}, diff --git a/core/providers/anthropic/anthropic.go b/core/providers/anthropic/anthropic.go index ab3862f4de0..88198b0d349 100644 --- a/core/providers/anthropic/anthropic.go +++ b/core/providers/anthropic/anthropic.go @@ -2418,6 +2418,11 @@ func (provider *AnthropicProvider) Rerank(ctx *schemas.BifrostContext, key schem return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Anthropic provider. +func (provider *AnthropicProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Anthropic provider. func (provider *AnthropicProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/azure/azure.go b/core/providers/azure/azure.go index 567f6330760..d6f1fab1eda 100644 --- a/core/providers/azure/azure.go +++ b/core/providers/azure/azure.go @@ -734,6 +734,11 @@ func (provider *AzureProvider) Rerank(ctx *schemas.BifrostContext, key schemas.K return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Azure provider. +func (provider *AzureProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Azure provider. func (provider *AzureProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/bedrock/bedrock.go b/core/providers/bedrock/bedrock.go index 39a463b96bb..7760f0e52dd 100644 --- a/core/providers/bedrock/bedrock.go +++ b/core/providers/bedrock/bedrock.go @@ -2415,6 +2415,11 @@ func (provider *BedrockProvider) Rerank(ctx *schemas.BifrostContext, key schemas return bifrostResponse, nil } +// Decision is not supported by the Bedrock provider. +func (provider *BedrockProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Bedrock provider. func (provider *BedrockProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/bedrockmantle/bedrockmantle.go b/core/providers/bedrockmantle/bedrockmantle.go index fe54fe028e9..dacc41948f2 100644 --- a/core/providers/bedrockmantle/bedrockmantle.go +++ b/core/providers/bedrockmantle/bedrockmantle.go @@ -433,6 +433,11 @@ func (provider *BedrockMantleProvider) Rerank(ctx *schemas.BifrostContext, key s return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the BedrockMantle provider. +func (provider *BedrockMantleProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Bedrock Mantle provider. func (provider *BedrockMantleProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/cerebras/cerebras.go b/core/providers/cerebras/cerebras.go index cccbbe530ac..e925b4681b6 100644 --- a/core/providers/cerebras/cerebras.go +++ b/core/providers/cerebras/cerebras.go @@ -209,6 +209,11 @@ func (provider *CerebrasProvider) Rerank(ctx *schemas.BifrostContext, key schema return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Cerebras provider. +func (provider *CerebrasProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Cerebras provider. func (provider *CerebrasProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/cohere/cohere.go b/core/providers/cohere/cohere.go index fd0f73491d5..aa59052a0a3 100644 --- a/core/providers/cohere/cohere.go +++ b/core/providers/cohere/cohere.go @@ -1066,6 +1066,11 @@ func (provider *CohereProvider) Rerank(ctx *schemas.BifrostContext, key schemas. return bifrostResponse, nil } +// Decision is not supported by the Cohere provider. +func (provider *CohereProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Cohere provider. func (provider *CohereProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/databricks/unsupported.go b/core/providers/databricks/unsupported.go index 7f3eff96618..4f7afeed573 100644 --- a/core/providers/databricks/unsupported.go +++ b/core/providers/databricks/unsupported.go @@ -27,6 +27,11 @@ func (provider *DatabricksProvider) Rerank(ctx *schemas.BifrostContext, key sche return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Databricks provider. +func (provider *DatabricksProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + func (provider *DatabricksProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) } diff --git a/core/providers/deepseek/deepseek.go b/core/providers/deepseek/deepseek.go index c9b7ff15d3f..1deaadf47e3 100644 --- a/core/providers/deepseek/deepseek.go +++ b/core/providers/deepseek/deepseek.go @@ -409,6 +409,11 @@ func (provider *DeepSeekProvider) Rerank(ctx *schemas.BifrostContext, key schema return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the DeepSeek provider. +func (provider *DeepSeekProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the DeepSeek provider. func (provider *DeepSeekProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/elevenlabs/elevenlabs.go b/core/providers/elevenlabs/elevenlabs.go index 82806e826a3..229cd93a65b 100644 --- a/core/providers/elevenlabs/elevenlabs.go +++ b/core/providers/elevenlabs/elevenlabs.go @@ -325,6 +325,11 @@ func (provider *ElevenlabsProvider) Rerank(ctx *schemas.BifrostContext, key sche return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Elevenlabs provider. +func (provider *ElevenlabsProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Elevenlabs provider. func (provider *ElevenlabsProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/fireworks/fireworks.go b/core/providers/fireworks/fireworks.go index 2fa4f70a77e..0f1ed235b0e 100644 --- a/core/providers/fireworks/fireworks.go +++ b/core/providers/fireworks/fireworks.go @@ -355,6 +355,11 @@ func (provider *FireworksProvider) Rerank(ctx *schemas.BifrostContext, key schem return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Fireworks provider. +func (provider *FireworksProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Fireworks provider. func (provider *FireworksProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/gemini/gemini.go b/core/providers/gemini/gemini.go index 3e7bdad0625..4a35fc06d59 100644 --- a/core/providers/gemini/gemini.go +++ b/core/providers/gemini/gemini.go @@ -1424,6 +1424,11 @@ func (provider *GeminiProvider) Rerank(ctx *schemas.BifrostContext, key schemas. return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Gemini provider. +func (provider *GeminiProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Gemini provider. func (provider *GeminiProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/githubcopilot/githubcopilot.go b/core/providers/githubcopilot/githubcopilot.go index 9c8372e669b..025f61ab2de 100644 --- a/core/providers/githubcopilot/githubcopilot.go +++ b/core/providers/githubcopilot/githubcopilot.go @@ -243,6 +243,11 @@ func (p *githubCopilotProvider) Rerank(ctx *schemas.BifrostContext, key schemas. return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, p.GetProviderKey()) } +// Decision is not supported by the githubCopilot provider. +func (p *githubCopilotProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, p.GetProviderKey()) +} + // OCR is not supported by GitHub Copilot. func (p *githubCopilotProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, p.GetProviderKey()) diff --git a/core/providers/groq/groq.go b/core/providers/groq/groq.go index bdd6687f2f9..c29a72560aa 100644 --- a/core/providers/groq/groq.go +++ b/core/providers/groq/groq.go @@ -198,6 +198,11 @@ func (provider *GroqProvider) Rerank(ctx *schemas.BifrostContext, key schemas.Ke return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Groq provider. +func (provider *GroqProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Groq provider. func (provider *GroqProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/huggingface/huggingface.go b/core/providers/huggingface/huggingface.go index 767dba04fa7..ebbc26e7386 100644 --- a/core/providers/huggingface/huggingface.go +++ b/core/providers/huggingface/huggingface.go @@ -800,6 +800,11 @@ func (provider *HuggingFaceProvider) Rerank(ctx *schemas.BifrostContext, key sch return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the HuggingFace provider. +func (provider *HuggingFaceProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Huggingface provider. func (provider *HuggingFaceProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/mistral/mistral.go b/core/providers/mistral/mistral.go index b055a2a58f6..a124a8dce46 100644 --- a/core/providers/mistral/mistral.go +++ b/core/providers/mistral/mistral.go @@ -645,6 +645,11 @@ func (provider *MistralProvider) Rerank(ctx *schemas.BifrostContext, key schemas return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Mistral provider. +func (provider *MistralProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR performs an OCR request to the Mistral API. // It sends a JSON request to Mistral's OCR endpoint and returns the extracted content. func (provider *MistralProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { diff --git a/core/providers/nebius/nebius.go b/core/providers/nebius/nebius.go index 280ac1fbfa9..de169957e83 100644 --- a/core/providers/nebius/nebius.go +++ b/core/providers/nebius/nebius.go @@ -237,6 +237,11 @@ func (provider *NebiusProvider) Rerank(ctx *schemas.BifrostContext, key schemas. return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Nebius provider. +func (provider *NebiusProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Nebius provider. func (provider *NebiusProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/ollama/ollama.go b/core/providers/ollama/ollama.go index eb80ca6126d..e55e00ea504 100644 --- a/core/providers/ollama/ollama.go +++ b/core/providers/ollama/ollama.go @@ -284,6 +284,11 @@ func (provider *OllamaProvider) Rerank(ctx *schemas.BifrostContext, key schemas. return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Ollama provider. +func (provider *OllamaProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Ollama provider. func (provider *OllamaProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/openai/openai.go b/core/providers/openai/openai.go index b93578c651b..30b725a2b9b 100644 --- a/core/providers/openai/openai.go +++ b/core/providers/openai/openai.go @@ -4133,6 +4133,11 @@ func (provider *OpenAIProvider) Rerank(ctx *schemas.BifrostContext, key schemas. ) } +// Decision is not supported by the OpenAI provider. +func (provider *OpenAIProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // HandleOpenAIRerankRequest handles rerank requests for custom OpenAI-compatible APIs. func HandleOpenAIRerankRequest( ctx *schemas.BifrostContext, diff --git a/core/providers/opencode/opencode.go b/core/providers/opencode/opencode.go index edfde960547..f35c6ae644f 100644 --- a/core/providers/opencode/opencode.go +++ b/core/providers/opencode/opencode.go @@ -230,6 +230,11 @@ func (p *opencodeProvider) Rerank(ctx *schemas.BifrostContext, key schemas.Key, return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, p.GetProviderKey()) } +// Decision is not supported by the opencode provider. +func (p *opencodeProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, p.GetProviderKey()) +} + // OCR is not supported by Opencode. func (p *opencodeProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, p.GetProviderKey()) diff --git a/core/providers/openrouter/openrouter.go b/core/providers/openrouter/openrouter.go index accfca14a75..99776c68480 100644 --- a/core/providers/openrouter/openrouter.go +++ b/core/providers/openrouter/openrouter.go @@ -484,6 +484,11 @@ func (provider *OpenRouterProvider) Rerank(ctx *schemas.BifrostContext, key sche return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the OpenRouter provider. +func (provider *OpenRouterProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Openrouter provider. func (provider *OpenRouterProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/parasail/parasail.go b/core/providers/parasail/parasail.go index 14e55b2ec46..35641d99fc3 100644 --- a/core/providers/parasail/parasail.go +++ b/core/providers/parasail/parasail.go @@ -194,6 +194,11 @@ func (provider *ParasailProvider) Rerank(ctx *schemas.BifrostContext, key schema return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Parasail provider. +func (provider *ParasailProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Parasail provider. func (provider *ParasailProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/perplexity/perplexity.go b/core/providers/perplexity/perplexity.go index 8e5abe08080..73596e5c935 100644 --- a/core/providers/perplexity/perplexity.go +++ b/core/providers/perplexity/perplexity.go @@ -318,6 +318,11 @@ func (provider *PerplexityProvider) Rerank(ctx *schemas.BifrostContext, key sche return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Perplexity provider. +func (provider *PerplexityProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Perplexity provider. func (provider *PerplexityProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/replicate/replicate.go b/core/providers/replicate/replicate.go index 2fc811624d7..fef20018be7 100644 --- a/core/providers/replicate/replicate.go +++ b/core/providers/replicate/replicate.go @@ -1749,6 +1749,11 @@ func (provider *ReplicateProvider) Rerank(ctx *schemas.BifrostContext, key schem return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Replicate provider. +func (provider *ReplicateProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Replicate provider. func (provider *ReplicateProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/runware/runware.go b/core/providers/runware/runware.go index 55f6aa799c7..8786dce1523 100644 --- a/core/providers/runware/runware.go +++ b/core/providers/runware/runware.go @@ -354,6 +354,11 @@ func (provider *RunwareProvider) Rerank(ctx *schemas.BifrostContext, key schemas return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Runware provider. +func (provider *RunwareProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Runware provider. func (provider *RunwareProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/runway/runway.go b/core/providers/runway/runway.go index 7f023a0fbd3..eed056bc2e4 100644 --- a/core/providers/runway/runway.go +++ b/core/providers/runway/runway.go @@ -130,6 +130,11 @@ func (provider *RunwayProvider) Rerank(ctx *schemas.BifrostContext, key schemas. return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Runway provider. +func (provider *RunwayProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Runway provider. func (provider *RunwayProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/sarvam/sarvam.go b/core/providers/sarvam/sarvam.go index bc43e29cae8..4b68b8efd52 100644 --- a/core/providers/sarvam/sarvam.go +++ b/core/providers/sarvam/sarvam.go @@ -177,6 +177,11 @@ func (provider *SarvamProvider) Rerank(ctx *schemas.BifrostContext, key schemas. return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Sarvam provider. +func (provider *SarvamProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Sarvam provider. func (provider *SarvamProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/sgl/sgl.go b/core/providers/sgl/sgl.go index 66503faa5c7..651321e0b66 100644 --- a/core/providers/sgl/sgl.go +++ b/core/providers/sgl/sgl.go @@ -425,6 +425,11 @@ func (provider *SGLProvider) Rerank(ctx *schemas.BifrostContext, key schemas.Key return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the SGL provider. +func (provider *SGLProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Sgl provider. func (provider *SGLProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/typesafe/decision.go b/core/providers/typesafe/decision.go new file mode 100644 index 00000000000..ada33b78bbf --- /dev/null +++ b/core/providers/typesafe/decision.go @@ -0,0 +1,398 @@ +package typesafe + +import ( + "fmt" + "reflect" + + "github.com/tidwall/gjson" + + providerUtils "github.com/maximhq/bifrost/core/providers/utils" + schemas "github.com/maximhq/bifrost/core/schemas" +) + +// typesafeMaxChoiceOptions is Typesafe's documented cap on choice options. +const typesafeMaxChoiceOptions = 255 + +// typesafeMinScoreLevels and typesafeMaxScoreLevels bound a score criteria array. +const ( + typesafeMinScoreLevels = 2 + typesafeMaxScoreLevels = 10 +) + +// supportedDecisionKinds is the set of kinds Typesafe serves. The Bifrost kind +// vocabulary mirrors the native question types, so the mapping is identity. +var supportedDecisionKinds = map[schemas.DecisionKind]string{ + schemas.DecisionKindNoul: TypesafeQuestionTypeNoul, + schemas.DecisionKindChoice: TypesafeQuestionTypeChoice, + schemas.DecisionKindScore: TypesafeQuestionTypeScore, +} + +// validStructuredValue reports whether a state or instructions value +// serializes to one of the documented JSON shapes: string, object, or array. +// Marshaling the value answers that exactly: pointers dereference, typed +// maps/slices/structs normalize, custom marshalers speak for themselves, and +// unmarshalable values (func, chan) fail closed with a local 400 instead of a +// request-marshal 500 later. Only the value is encoded for inspection; the +// request body is serialized once, unchanged, on the send path. +func validStructuredValue(value interface{}) bool { + data, err := providerUtils.MarshalSorted(value) + if err != nil { + return false + } + for _, c := range data { + switch c { + case ' ', '\t', '\n', '\r': + continue + } + return c == '"' || c == '{' || c == '[' + } + return false +} + +// ToTypesafeDecisionRequest converts a Bifrost decision request into +// Typesafe's native systemone shape. Unsupported kinds and malformed criteria +// are rejected rather than silently approximated. +func ToTypesafeDecisionRequest(request *schemas.BifrostDecisionRequest) (*TypesafeDecisionRequest, error) { + if len(request.Questions) == 0 { + return nil, providerUtils.InvalidRequestErrorf("decision request requires at least one question") + } + if !validStructuredValue(request.State) { + return nil, providerUtils.InvalidRequestErrorf("state must be a string, object, or array") + } + + questions := make(map[string]TypesafeQuestion, len(request.Questions)) + for name, question := range request.Questions { + native, err := toTypesafeQuestion(name, question) + if err != nil { + return nil, err + } + questions[name] = *native + } + + return &TypesafeDecisionRequest{ + State: request.State, + Model: request.Model, + Questions: questions, + }, nil +} + +// toTypesafeQuestion validates one question and maps it to the native shape. +func toTypesafeQuestion(name string, question schemas.DecisionQuestion) (*TypesafeQuestion, error) { + nativeType, ok := supportedDecisionKinds[question.Kind] + if !ok { + return nil, providerUtils.InvalidRequestErrorf("question %q has unsupported kind %q; expected noul, choice, or score", name, question.Kind) + } + if question.Instructions == nil { + return nil, providerUtils.InvalidRequestErrorf("question %q has no instructions", name) + } + if !validStructuredValue(question.Instructions) { + return nil, providerUtils.InvalidRequestErrorf("question %q instructions must be a string, object, or array", name) + } + + native := TypesafeQuestion{Type: nativeType, Instructions: question.Instructions} + + switch question.Kind { + case schemas.DecisionKindNoul: + criteria, err := noulCriteria(name, question.Criteria) + if err != nil { + return nil, err + } + native.Criteria = criteria + case schemas.DecisionKindChoice: + criteria, err := choiceCriteria(name, question.Criteria) + if err != nil { + return nil, err + } + native.Criteria = criteria + case schemas.DecisionKindScore: + criteria, err := scoreCriteria(name, question.Criteria) + if err != nil { + return nil, err + } + native.Criteria = criteria + } + + return &native, nil +} + +// noulCriteria validates optional noul criteria: a map whose only keys are +// "true" and "false", each described by a string, object, or array. Passed +// through losslessly. +func noulCriteria(name string, criteria interface{}) (interface{}, error) { + if criteria == nil { + return nil, nil + } + m, err := criteriaAsMap(name, criteria, false) + if err != nil { + return nil, err + } + for key := range m { + if key != "true" && key != "false" { + return nil, providerUtils.InvalidRequestErrorf("question %q noul criteria allows only \"true\" and \"false\" keys, got %q", name, key) + } + } + return m, nil +} + +// choiceCriteria validates the required option -> description map. A +// description is a string, object, or array, or null when an option needs no +// extra detail. +func choiceCriteria(name string, criteria interface{}) (map[string]any, error) { + if criteria == nil { + return nil, providerUtils.InvalidRequestErrorf("question %q has kind choice and requires criteria options", name) + } + m, err := criteriaAsMap(name, criteria, true) + if err != nil { + return nil, err + } + if len(m) == 0 { + return nil, providerUtils.InvalidRequestErrorf("question %q has kind choice and requires criteria options", name) + } + if len(m) > typesafeMaxChoiceOptions { + return nil, providerUtils.InvalidRequestErrorf("question %q has %d choice options; Typesafe allows at most %d", name, len(m), typesafeMaxChoiceOptions) + } + return m, nil +} + +// scoreCriteria validates the ordered array of 2-10 level descriptions, each +// a string, object, or array, preserved losslessly. Go SDK callers supply +// []string; HTTP JSON decoding supplies []interface{} - both are the same +// ordered array on the wire. +func scoreCriteria(name string, criteria interface{}) (interface{}, error) { + var levels []any + switch typed := criteria.(type) { + case []any: + levels = typed + case []string: + levels = make([]any, len(typed)) + for i, level := range typed { + levels[i] = level + } + default: + // Go SDK callers pass typed slices ([]Rubric, [][]string, named slice + // types). Any slice or array is an ordered list of level descriptions; + // elements are validated by serialized shape below, exactly like the + // untyped forms, and carried losslessly. + value := reflect.ValueOf(criteria) + if !value.IsValid() || (value.Kind() != reflect.Slice && value.Kind() != reflect.Array) { + return nil, providerUtils.InvalidRequestErrorf("question %q has kind score and requires criteria as an ordered array of level descriptions", name) + } + levels = make([]any, value.Len()) + for i := range levels { + levels[i] = value.Index(i).Interface() + } + } + if len(levels) < typesafeMinScoreLevels || len(levels) > typesafeMaxScoreLevels { + return nil, providerUtils.InvalidRequestErrorf("question %q score criteria must have between %d and %d levels, got %d", name, typesafeMinScoreLevels, typesafeMaxScoreLevels, len(levels)) + } + for i, level := range levels { + if !validStructuredValue(level) { + return nil, providerUtils.InvalidRequestErrorf("question %q score criteria level %d must be a string, object, or array", name, i) + } + } + return levels, nil +} + +// criteriaAsMap normalizes a criteria value into map[string]any, rejecting +// non-map shapes. Each description must serialize to a string, object, or +// array; null is additionally allowed when allowNull is set (choice options +// that need no extra detail). Descriptions are carried losslessly. +func criteriaAsMap(name string, criteria interface{}, allowNull bool) (map[string]any, error) { + var m map[string]any + switch typed := criteria.(type) { + case map[string]string: + m = make(map[string]any, len(typed)) + for key, value := range typed { + m[key] = value + } + case map[string]any: + m = typed + default: + // Go SDK callers pass typed maps (map[string]Rubric, map[string][]string, + // ...). Any map with string keys is a map of named descriptions; its + // values are validated by serialized shape below, exactly like the + // untyped forms, and carried losslessly. + value := reflect.ValueOf(criteria) + if !value.IsValid() || value.Kind() != reflect.Map || value.Type().Key().Kind() != reflect.String { + return nil, providerUtils.InvalidRequestErrorf("question %q criteria must be a map of descriptions", name) + } + m = make(map[string]any, value.Len()) + iter := value.MapRange() + for iter.Next() { + m[iter.Key().String()] = iter.Value().Interface() + } + } + for key, value := range m { + if isJSONNull(value) { + if allowNull { + continue + } + return nil, providerUtils.InvalidRequestErrorf("question %q criteria description for %q must be a string, object, or array", name, key) + } + if !validStructuredValue(value) { + return nil, providerUtils.InvalidRequestErrorf("question %q criteria description for %q must be a string, object, or array", name, key) + } + } + return m, nil +} + +// isJSONNull reports whether a description serializes to JSON null. Typed nil +// pointers from Go SDK callers (map[string]*Rubric{"other": nil}) are non-nil +// interfaces wrapping nil pointers - value == nil misses them, but on the wire +// they are null exactly like an untyped nil, so both follow the null rules. +func isJSONNull(value any) bool { + if value == nil { + return true + } + data, err := providerUtils.MarshalSorted(value) + if err != nil { + return false + } + return gjson.ParseBytes(data).Type == gjson.Null +} + +// ToBifrostDecisionResponse converts a native systemone response back into +// the shared decision shape. Every requested question must produce an answer +// of its declared kind. +func ToBifrostDecisionResponse(resp *TypesafeDecisionResponse, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + answers := make(map[string]schemas.DecisionAnswer, len(request.Questions)) + for name, question := range request.Questions { + native, ok := resp.Answers[name] + if !ok { + return nil, providerUtils.NewBifrostOperationError(fmt.Sprintf("typesafe returned no answer for question %q", name), nil) + } + if expected := supportedDecisionKinds[question.Kind]; native.Type != expected { + return nil, providerUtils.NewBifrostOperationError(fmt.Sprintf("typesafe answered question %q as %q; expected %q", name, native.Type, expected), nil) + } + + answer := schemas.DecisionAnswer{ + Kind: question.Kind, + Confidence: native.Confidence, + Probabilities: native.Probabilities, + Legend: native.Legend, + } + switch question.Kind { + case schemas.DecisionKindNoul: + if native.Noul == nil { + return nil, providerUtils.NewBifrostOperationError(fmt.Sprintf("typesafe noul answer for question %q carries no value", name), nil) + } + if *native.Noul < 0 || *native.Noul > 1 { + return nil, providerUtils.NewBifrostOperationError(fmt.Sprintf("typesafe noul answer for question %q is outside [0,1]: %v", name, *native.Noul), nil) + } + answer.Value = *native.Noul + case schemas.DecisionKindChoice: + if native.Choice == nil { + return nil, providerUtils.NewBifrostOperationError(fmt.Sprintf("typesafe choice answer for question %q carries no value", name), nil) + } + answer.Value = *native.Choice + case schemas.DecisionKindScore: + if native.Score == nil { + return nil, providerUtils.NewBifrostOperationError(fmt.Sprintf("typesafe score answer for question %q carries no value", name), nil) + } + answer.Value = *native.Score + } + answers[name] = answer + } + + response := &schemas.BifrostDecisionResponse{ + Model: resp.Model, + Answers: answers, + } + if resp.Usage != nil { + response.Usage = &schemas.BifrostLLMUsage{ + PromptTokens: resp.Usage.InputTokens, + CompletionTokens: resp.Usage.OutputTokens, + TotalTokens: resp.Usage.InputTokens + resp.Usage.OutputTokens, + } + } + return response, nil +} + +// ToBifrostDecisionRequest converts a native systemone request into the +// shared decision shape. Question identifiers become answer field names; the +// native type vocabulary is identical to the Bifrost kind vocabulary. +func (req *TypesafeDecisionRequest) ToBifrostDecisionRequest(ctx *schemas.BifrostContext) (*schemas.BifrostDecisionRequest, error) { + if req == nil { + return nil, providerUtils.InvalidRequestErrorf("request body is required") + } + if req.State == nil { + return nil, providerUtils.InvalidRequestErrorf("state is required") + } + if len(req.Questions) == 0 { + return nil, providerUtils.InvalidRequestErrorf("at least one question is required") + } + + provider, model := schemas.ParseModelString(req.Model, schemas.Typesafe) + + questions := make(map[string]schemas.DecisionQuestion, len(req.Questions)) + for name, question := range req.Questions { + questions[name] = schemas.DecisionQuestion{ + Kind: schemas.DecisionKind(question.Type), + Instructions: question.Instructions, + Criteria: question.Criteria, + } + } + + return &schemas.BifrostDecisionRequest{ + Provider: provider, + Model: model, + State: req.State, + Questions: questions, + }, nil +} + +// ToTypesafeNativeDecisionResponse converts a shared decision response back +// into Typesafe's native model, answers, and usage shape, covering all three +// answer types. +func ToTypesafeNativeDecisionResponse(resp *schemas.BifrostDecisionResponse) (*TypesafeDecisionResponse, error) { + if resp == nil { + return nil, fmt.Errorf("decision response is nil") + } + + answers := make(map[string]TypesafeAnswer, len(resp.Answers)) + for name, answer := range resp.Answers { + native := TypesafeAnswer{ + Confidence: answer.Confidence, + Probabilities: answer.Probabilities, + Legend: answer.Legend, + } + switch answer.Kind { + case schemas.DecisionKindNoul: + number, ok := answer.Value.(float64) + if !ok { + return nil, fmt.Errorf("decision answer %q is not a number", name) + } + native.Type = TypesafeQuestionTypeNoul + native.Noul = &number + case schemas.DecisionKindChoice: + choice, ok := answer.Value.(string) + if !ok { + return nil, fmt.Errorf("decision answer %q is not a string", name) + } + native.Type = TypesafeQuestionTypeChoice + native.Choice = &choice + case schemas.DecisionKindScore: + number, ok := answer.Value.(float64) + if !ok { + return nil, fmt.Errorf("decision answer %q is not a number", name) + } + native.Type = TypesafeQuestionTypeScore + native.Score = &number + default: + return nil, fmt.Errorf("decision answer %q has unknown kind %q", name, answer.Kind) + } + answers[name] = native + } + + native := &TypesafeDecisionResponse{ + Model: resp.Model, + Answers: answers, + } + if resp.Usage != nil { + native.Usage = &TypesafeUsage{ + InputTokens: resp.Usage.PromptTokens, + OutputTokens: resp.Usage.CompletionTokens, + } + } + return native, nil +} diff --git a/core/providers/typesafe/decision_test.go b/core/providers/typesafe/decision_test.go new file mode 100644 index 00000000000..ae4d807601a --- /dev/null +++ b/core/providers/typesafe/decision_test.go @@ -0,0 +1,864 @@ +package typesafe + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/bytedance/sonic" + schemas "github.com/maximhq/bifrost/core/schemas" +) + +// decisionRequest builds a BifrostDecisionRequest around the given questions. +func decisionRequest(state interface{}, questions map[string]schemas.DecisionQuestion) *schemas.BifrostDecisionRequest { + return &schemas.BifrostDecisionRequest{ + Provider: schemas.Typesafe, + Model: "jev-1.13.0", + State: state, + Questions: questions, + } +} + +func TestToTypesafeDecisionRequestMixedKinds(t *testing.T) { + req := decisionRequest("the user asked for a refund", map[string]schemas.DecisionQuestion{ + "is_angry": { + Kind: schemas.DecisionKindNoul, + Instructions: "Is the user angry?", + }, + "category": { + Kind: schemas.DecisionKindChoice, + Instructions: "Pick the ticket category", + Criteria: map[string]interface{}{"billing": "money issues", "bug": "product defects", "other": "anything else"}, + }, + "severity": { + Kind: schemas.DecisionKindScore, + Instructions: "Rate the severity", + Criteria: []interface{}{"cosmetic", "annoying", "blocking"}, + }, + }) + + native, err := ToTypesafeDecisionRequest(req) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if native.Model != "jev-1.13.0" { + t.Errorf("model = %q, want bare jev-1.13.0", native.Model) + } + if native.State != "the user asked for a refund" { + t.Errorf("state not preserved: %v", native.State) + } + if len(native.Questions) != 3 { + t.Fatalf("expected 3 questions, got %d", len(native.Questions)) + } + + angry := native.Questions["is_angry"] + if angry.Type != TypesafeQuestionTypeNoul { + t.Errorf("is_angry type = %q, want noul", angry.Type) + } + if angry.Instructions != "Is the user angry?" { + t.Errorf("is_angry instructions = %v", angry.Instructions) + } + + category := native.Questions["category"] + if category.Type != TypesafeQuestionTypeChoice { + t.Errorf("category type = %q, want choice", category.Type) + } + criteria, ok := category.Criteria.(map[string]any) + if !ok || len(criteria) != 3 || criteria["bug"] != "product defects" { + t.Errorf("category criteria not preserved: %#v", category.Criteria) + } + + severity := native.Questions["severity"] + if severity.Type != TypesafeQuestionTypeScore { + t.Errorf("severity type = %q, want score", severity.Type) + } + levels, ok := severity.Criteria.([]interface{}) + if !ok || len(levels) != 3 || levels[0] != "cosmetic" { + t.Errorf("severity criteria not preserved losslessly: %#v", severity.Criteria) + } +} + +func TestToTypesafeDecisionRequestStructuredStateAndInstructions(t *testing.T) { + state := map[string]interface{}{"ticket": map[string]interface{}{"id": 42, "body": "hello"}} + structured := map[string]interface{}{"goal": "judge tone", "steps": []interface{}{"read", "decide"}} + req := decisionRequest(state, map[string]schemas.DecisionQuestion{ + "tone_ok": {Kind: schemas.DecisionKindNoul, Instructions: structured}, + }) + + native, err := ToTypesafeDecisionRequest(req) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !reflect.DeepEqual(native.Questions["tone_ok"].Instructions, interface{}(structured)) { + t.Errorf("structured instructions not lossless: %#v", native.Questions["tone_ok"].Instructions) + } + if !reflect.DeepEqual(native.State, interface{}(state)) { + t.Errorf("structured state not lossless: %#v", native.State) + } +} + +func TestToTypesafeDecisionRequestStructuredCriteria(t *testing.T) { + // The API spec types criteria descriptions as string | object | array for + // noul and score, and string | object | array | null for choice options - + // structured rubrics must pass through losslessly, not be coerced to strings. + noulCriteria := map[string]any{ + "true": map[string]any{"meaning": "approve", "signals": []any{"polite tone"}}, + "false": []any{"reject", "escalate"}, + } + choiceCriteria := map[string]any{ + "billing": map[string]any{"rubric": "money issues", "examples": []any{"refund request"}}, + "bug": []any{"crash", "defect"}, + "other": nil, // spec: null when an option needs no extra detail + } + scoreCriteria := []any{ + "cosmetic", + map[string]any{"level": "annoying", "examples": []any{"slow load"}}, + []any{"blocking", "data loss"}, + } + req := decisionRequest("state", map[string]schemas.DecisionQuestion{ + "approve": {Kind: schemas.DecisionKindNoul, Instructions: "d", Criteria: noulCriteria}, + "category": {Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: choiceCriteria}, + "severity": {Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: scoreCriteria}, + }) + + native, err := ToTypesafeDecisionRequest(req) + if err != nil { + t.Fatalf("structured criteria rejected: %v", err) + } + if !reflect.DeepEqual(native.Questions["approve"].Criteria, any(noulCriteria)) { + t.Errorf("noul criteria not lossless: %#v", native.Questions["approve"].Criteria) + } + if !reflect.DeepEqual(native.Questions["category"].Criteria, any(choiceCriteria)) { + t.Errorf("choice criteria not lossless: %#v", native.Questions["category"].Criteria) + } + if !reflect.DeepEqual(native.Questions["severity"].Criteria, any(scoreCriteria)) { + t.Errorf("score criteria not lossless: %#v", native.Questions["severity"].Criteria) + } +} + +// TestToTypesafeDecisionRequestCriteriaTypeMatrix exercises every allowed +// value type in every criteria slot: string | object | array for noul keys +// and score levels, plus null for choice options, and the rejected types +// (null where not allowed, numbers, booleans, unmarshalable values) per kind. +func TestToTypesafeDecisionRequestCriteriaTypeMatrix(t *testing.T) { + str := "text description" + obj := map[string]any{"rubric": "detail", "examples": []any{"one"}} + arr := []any{"first", "second"} + allowed := map[string]any{"string": str, "object": obj, "array": arr} + rejectedNoNull := map[string]any{"number": 7, "boolean": true, "float": 3.5, "func": func() {}} + + accept := func(t *testing.T, q schemas.DecisionQuestion) *TypesafeQuestion { + t.Helper() + native, err := ToTypesafeDecisionRequest(decisionRequest("state", map[string]schemas.DecisionQuestion{"q": q})) + if err != nil { + t.Fatalf("expected acceptance, got %v", err) + } + question := native.Questions["q"] + return &question + } + reject := func(t *testing.T, q schemas.DecisionQuestion, wantSub string) { + t.Helper() + _, err := ToTypesafeDecisionRequest(decisionRequest("state", map[string]schemas.DecisionQuestion{"q": q})) + if err == nil || !strings.Contains(err.Error(), wantSub) { + t.Fatalf("expected rejection containing %q, got %v", wantSub, err) + } + } + + t.Run("noul", func(t *testing.T) { + for trueName, trueValue := range allowed { + for falseName, falseValue := range allowed { + t.Run("true="+trueName+"/false="+falseName, func(t *testing.T) { + criteria := map[string]any{"true": trueValue, "false": falseValue} + got := accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul, Instructions: "d", Criteria: criteria}) + if !reflect.DeepEqual(got.Criteria, any(criteria)) { + t.Errorf("criteria not lossless: %#v", got.Criteria) + } + }) + } + t.Run("single key true="+trueName, func(t *testing.T) { + accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul, Instructions: "d", Criteria: map[string]any{"true": trueValue}}) + }) + } + for name, value := range rejectedNoNull { + t.Run("rejects "+name, func(t *testing.T) { + reject(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul, Instructions: "d", Criteria: map[string]any{"true": value}}, "must be a string, object, or array") + }) + } + t.Run("rejects null", func(t *testing.T) { + reject(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul, Instructions: "d", Criteria: map[string]any{"false": nil}}, "must be a string, object, or array") + }) + }) + + t.Run("choice", func(t *testing.T) { + for name, value := range allowed { + t.Run("option="+name, func(t *testing.T) { + criteria := map[string]any{"opt": value, "alt": "plain"} + got := accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: criteria}) + if !reflect.DeepEqual(got.Criteria, any(criteria)) { + t.Errorf("criteria not lossless: %#v", got.Criteria) + } + }) + } + t.Run("option=null", func(t *testing.T) { + criteria := map[string]any{"opt": nil, "alt": "plain"} + got := accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: criteria}) + if !reflect.DeepEqual(got.Criteria, any(criteria)) { + t.Errorf("criteria not lossless: %#v", got.Criteria) + } + }) + t.Run("all four types in one map", func(t *testing.T) { + criteria := map[string]any{"s": str, "o": obj, "a": arr, "n": nil} + got := accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: criteria}) + if !reflect.DeepEqual(got.Criteria, any(criteria)) { + t.Errorf("criteria not lossless: %#v", got.Criteria) + } + }) + for name, value := range rejectedNoNull { + t.Run("rejects "+name, func(t *testing.T) { + reject(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: map[string]any{"opt": value}}, "must be a string, object, or array") + }) + } + }) + + t.Run("score", func(t *testing.T) { + // Each allowed type at each position of a three-level array. + for name, value := range allowed { + for position := 0; position < 3; position++ { + t.Run(fmt.Sprintf("%s at level %d", name, position), func(t *testing.T) { + levels := []any{"base low", "base mid", "base high"} + levels[position] = value + got := accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: levels}) + if !reflect.DeepEqual(got.Criteria, any(levels)) { + t.Errorf("levels not lossless: %#v", got.Criteria) + } + }) + } + } + t.Run("all three types in one array", func(t *testing.T) { + levels := []any{str, obj, arr} + got := accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: levels}) + if !reflect.DeepEqual(got.Criteria, any(levels)) { + t.Errorf("levels not lossless: %#v", got.Criteria) + } + }) + for name, value := range rejectedNoNull { + t.Run("rejects "+name, func(t *testing.T) { + reject(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: []any{"low", value}}, "level 1 must be a string, object, or array") + }) + } + t.Run("rejects null level", func(t *testing.T) { + reject(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: []any{"low", nil}}, "level 1 must be a string, object, or array") + }) + }) + + t.Run("typed Go maps with string keys accepted", func(t *testing.T) { + // Go SDK callers pass typed maps; any map with string keys whose values + // serialize to valid descriptions must be accepted, not only + // map[string]string and map[string]any. + type rubric struct { + Meaning string `json:"meaning"` + } + got := accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: map[string]rubric{ + "billing": {Meaning: "money issues"}, + "bug": {Meaning: "defects"}, + }}) + m, ok := got.Criteria.(map[string]any) + if !ok || len(m) != 2 { + t.Fatalf("typed struct map not normalized: %#v", got.Criteria) + } + if r, ok := m["billing"].(rubric); !ok || r.Meaning != "money issues" { + t.Errorf("typed value not carried losslessly: %#v", m["billing"]) + } + accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: map[string][]string{ + "billing": {"charges", "refunds"}, + "bug": {"crash"}, + }}) + accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul, Instructions: "d", Criteria: map[string][]string{ + "true": {"clearly upset"}, + }}) + // Non-string keys stay rejected: not a map of named descriptions. + reject(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: map[int]string{1: "first"}}, "must be a map of descriptions") + // String-keyed but invalid values still fail per value. + reject(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: map[string]int{"a": 1}}, "must be a string, object, or array") + }) + + t.Run("typed slices accepted for score criteria", func(t *testing.T) { + // Go SDK callers pass typed slices; any slice or array whose elements + // serialize to valid descriptions must be accepted, mirroring the + // typed-map handling for noul and choice. + type rubric struct { + Level string `json:"level"` + } + got := accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: []rubric{ + {Level: "low"}, {Level: "high"}, + }}) + levels, ok := got.Criteria.([]any) + if !ok || len(levels) != 2 { + t.Fatalf("typed struct slice not normalized: %#v", got.Criteria) + } + if r, ok := levels[0].(rubric); !ok || r.Level != "low" { + t.Errorf("typed level not carried losslessly: %#v", levels[0]) + } + accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: [][]string{ + {"can wait"}, {"needs reply", "churn risk"}, + }}) + type namedLevels []string + accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: namedLevels{"low", "high"}}) + // Element validation still applies to typed slices. + reject(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: []int{1, 2}}, "level 0 must be a string, object, or array") + reject(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: []rubric{{Level: "only one"}}}, "between 2 and 10") + }) + + t.Run("typed nil pointers are JSON null descriptions", func(t *testing.T) { + // map[string]*Rubric{"other": nil} wraps a nil pointer in a non-nil + // interface; it serializes to null and must follow the null rules: + // allowed for choice options, rejected for noul. + type rubric struct { + Meaning string `json:"meaning"` + } + got := accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: map[string]*rubric{ + "billing": {Meaning: "money issues"}, + "other": nil, + }}) + m, ok := got.Criteria.(map[string]any) + if !ok || len(m) != 2 { + t.Fatalf("typed pointer map not normalized: %#v", got.Criteria) + } + reject(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul, Instructions: "d", Criteria: map[string]*rubric{ + "true": nil, + }}, "must be a string, object, or array") + }) + + t.Run("typed string maps accepted for noul and choice", func(t *testing.T) { + // Go SDK callers pass map[string]string; both map input branches must work. + accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul, Instructions: "d", Criteria: map[string]string{"true": "yes means this", "false": "no means this"}}) + got := accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: map[string]string{"a": "first", "b": "second"}}) + m, ok := got.Criteria.(map[string]any) + if !ok || m["a"] != "first" || m["b"] != "second" { + t.Errorf("typed string map not normalized losslessly: %#v", got.Criteria) + } + }) +} + +func TestIsJSONNull(t *testing.T) { + type rubric struct { + Meaning string `json:"meaning"` + } + var nilRubric *rubric + var nilMap map[string]string + var nilSlice []string + var nilTypedSlice []rubric + var nilIface any + + cases := []struct { + name string + value any + want bool + }{ + {"untyped nil", nil, true}, + {"nil interface variable", nilIface, true}, + {"typed nil struct pointer", nilRubric, true}, + {"nil map", nilMap, true}, + {"nil slice", nilSlice, true}, + {"nil typed slice", nilTypedSlice, true}, + {"raw json null", json.RawMessage("null"), true}, + {"raw json null with whitespace", json.RawMessage(" \n\tnull"), true}, + {"non-nil pointer", &rubric{Meaning: "x"}, false}, + {"zero struct", rubric{}, false}, + {"empty map", map[string]string{}, false}, + {"empty slice", []string{}, false}, + {"empty string", "", false}, + {"the string null", "null", false}, // serializes to "null" WITH quotes - a string, not null + {"zero int", 0, false}, + {"false", false, false}, + {"unmarshalable func", func() {}, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := isJSONNull(tc.value); got != tc.want { + t.Errorf("isJSONNull(%#v) = %v, want %v", tc.value, got, tc.want) + } + }) + } +} + +func TestToTypesafeDecisionRequestRejections(t *testing.T) { + cases := []struct { + name string + question schemas.DecisionQuestion + wantSub string + }{ + { + name: "unsupported kind", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKind("ranking"), Instructions: "d"}, + wantSub: "unsupported kind", + }, + { + name: "missing instructions", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul}, + wantSub: "no instructions", + }, + { + name: "noul criteria with bad key", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul, Instructions: "d", Criteria: map[string]interface{}{"maybe": "x"}}, + wantSub: `allows only "true" and "false" keys`, + }, + { + name: "choice without criteria", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d"}, + wantSub: "requires criteria options", + }, + { + name: "choice criteria wrong shape", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: []interface{}{"a", "b"}}, + wantSub: "must be a map of descriptions", + }, + { + name: "choice criteria non-string description", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: map[string]interface{}{"a": 1}}, + wantSub: "must be a string", + }, + { + name: "score criteria missing", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d"}, + wantSub: "ordered array", + }, + { + name: "score criteria too short", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: []interface{}{"only one"}}, + wantSub: "between 2 and 10", + }, + { + name: "score criteria non-string level", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: []interface{}{"low", 2, "high"}}, + wantSub: "level 1 must be a string", + }, + { + name: "instructions with unsupported shape", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul, Instructions: 42}, + wantSub: "instructions must be a string, object, or array", + }, + { + // Only choice options may be null; the noul true/false descriptions + // are typed string | object | array. + name: "noul criteria null description", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul, Instructions: "d", Criteria: map[string]any{"true": nil}}, + wantSub: "must be a string, object, or array", + }, + { + name: "noul criteria number description", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul, Instructions: "d", Criteria: map[string]any{"true": 7}}, + wantSub: "must be a string, object, or array", + }, + { + name: "choice criteria unmarshalable description", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: map[string]any{"a": func() {}}}, + wantSub: "must be a string, object, or array", + }, + { + name: "score criteria null level", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: []any{"low", nil, "high"}}, + wantSub: "level 1 must be a string, object, or array", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + req := decisionRequest("state", map[string]schemas.DecisionQuestion{"q": tc.question}) + _, err := ToTypesafeDecisionRequest(req) + if err == nil { + t.Fatalf("expected rejection containing %q, got nil error", tc.wantSub) + } + if !strings.Contains(err.Error(), tc.wantSub) { + t.Errorf("error %q does not contain %q", err.Error(), tc.wantSub) + } + }) + } + + t.Run("state with unsupported shape", func(t *testing.T) { + // Pointers to scalars marshal to JSON scalars; funcs cannot marshal at + // all - both must be caller errors, not upstream 422s or marshal 500s. + for _, bad := range []interface{}{true, 42, 3.14, nil, new(42), func() {}} { + req := decisionRequest(bad, map[string]schemas.DecisionQuestion{ + "q": {Kind: schemas.DecisionKindNoul, Instructions: "d"}, + }) + _, err := ToTypesafeDecisionRequest(req) + if err == nil || !strings.Contains(err.Error(), "state must be a string, object, or array") { + t.Fatalf("expected state shape rejection for %T, got %v", bad, err) + } + } + }) + + t.Run("instructions validated by serialized shape", func(t *testing.T) { + type rubric struct { + Goal string `json:"goal"` + } + for _, good := range []interface{}{rubric{Goal: "judge tone"}, []string{"read", "decide"}, map[string]string{"goal": "judge"}} { + req := decisionRequest("state", map[string]schemas.DecisionQuestion{ + "q": {Kind: schemas.DecisionKindNoul, Instructions: good}, + }) + if _, err := ToTypesafeDecisionRequest(req); err != nil { + t.Fatalf("expected %T instructions to be accepted, got %v", good, err) + } + } + for _, bad := range []interface{}{7, new(3.5), func() {}} { + req := decisionRequest("state", map[string]schemas.DecisionQuestion{ + "q": {Kind: schemas.DecisionKindNoul, Instructions: bad}, + }) + _, err := ToTypesafeDecisionRequest(req) + if err == nil || !strings.Contains(err.Error(), "instructions must be a string, object, or array") { + t.Fatalf("expected instructions shape rejection for %T, got %v", bad, err) + } + } + }) + + t.Run("typed string slice score criteria accepted", func(t *testing.T) { + req := decisionRequest("state", map[string]schemas.DecisionQuestion{ + "q": {Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: []string{"low", "medium", "high"}}, + }) + native, err := ToTypesafeDecisionRequest(req) + if err != nil { + t.Fatalf("expected []string score criteria to be accepted, got %v", err) + } + levels, ok := native.Questions["q"].Criteria.([]interface{}) + if !ok || len(levels) != 3 || levels[0] != "low" { + t.Fatalf("score criteria not normalized to ordered array: %#v", native.Questions["q"].Criteria) + } + }) + + t.Run("typed maps, slices, and structs are valid shapes", func(t *testing.T) { + // Go SDK callers pass typed values; anything serializing to a JSON + // object or array satisfies the contract, not only the interface types + // an HTTP JSON decode produces. + type ticket struct { + ID int `json:"id"` + Body string `json:"body"` + } + for _, good := range []interface{}{ + map[string]string{"key": "value"}, + []string{"a", "b"}, + ticket{ID: 1, Body: "hello"}, + } { + req := decisionRequest(good, map[string]schemas.DecisionQuestion{ + "q": {Kind: schemas.DecisionKindNoul, Instructions: "d"}, + }) + if _, err := ToTypesafeDecisionRequest(req); err != nil { + t.Fatalf("expected %T to be a valid state shape, got %v", good, err) + } + } + }) +} + +func TestToBifrostDecisionResponseAllKindsWithZeroAndFractionalValues(t *testing.T) { + req := decisionRequest("state", map[string]schemas.DecisionQuestion{ + "is_spam": {Kind: schemas.DecisionKindNoul, Instructions: "d"}, + "lang": {Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: map[string]interface{}{"en": "", "de": ""}}, + "quality": {Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: []interface{}{"bad", "ok", "good"}}, + }) + + zero := 0.0 + fractional := 1.75 + confidence := 0.9 + lang := "de" + native := &TypesafeDecisionResponse{ + Model: "jev-1.13.0", + Answers: map[string]TypesafeAnswer{ + // Zero is a legitimate evaluated value and must survive. + "is_spam": {Type: TypesafeQuestionTypeNoul, Noul: &zero, Probabilities: map[string]float64{"true": 0.0, "false": 1.0}}, + "lang": {Type: TypesafeQuestionTypeChoice, Choice: &lang, Probabilities: map[string]float64{"en": 0.1, "de": 0.9}, Confidence: &confidence}, + "quality": {Type: TypesafeQuestionTypeScore, Score: &fractional, Legend: map[string]any{"1": "bad", "2": "ok", "3": "good"}}, + }, + Usage: &TypesafeUsage{InputTokens: 120, OutputTokens: 0}, + } + + resp, bifrostErr := ToBifrostDecisionResponse(native, req) + if bifrostErr != nil { + t.Fatalf("unexpected error: %v", bifrostErr) + } + if resp.Model != "jev-1.13.0" { + t.Errorf("model = %q", resp.Model) + } + + spam := resp.Answers["is_spam"] + if spam.Kind != schemas.DecisionKindNoul || spam.Value != 0.0 { + t.Errorf("zero-valued noul answer lost: %+v", spam) + } + if spam.Probabilities["false"] != 1.0 { + t.Errorf("noul probabilities lost: %+v", spam.Probabilities) + } + + langAnswer := resp.Answers["lang"] + if langAnswer.Value != "de" { + t.Errorf("choice value = %v", langAnswer.Value) + } + if langAnswer.Confidence == nil || *langAnswer.Confidence != 0.9 { + t.Errorf("confidence lost: %+v", langAnswer) + } + + quality := resp.Answers["quality"] + if quality.Value != 1.75 { + t.Errorf("fractional score lost: %v", quality.Value) + } + if quality.Legend["3"] != "good" { + t.Errorf("legend lost: %+v", quality.Legend) + } + + if resp.Usage == nil || resp.Usage.PromptTokens != 120 || resp.Usage.CompletionTokens != 0 || resp.Usage.TotalTokens != 120 { + t.Errorf("usage not normalized: %+v", resp.Usage) + } +} + +func TestToBifrostDecisionResponseStructuredLegendDecodes(t *testing.T) { + // The legend echoes each score level verbatim - the SDKs type it as + // dict[str, str | object | array] (Python) and { [score]: T[score] } (JS) - + // so structured levels come back as objects and arrays, not strings. A + // map[string]string legend fails to decode such a body at all. + raw := []byte(`{ + "model": "jev-1.13.0", + "answers": { + "urgency": { + "type": "score", + "score": 1, + "confidence": 0.8, + "legend": {"0": "low", "1": {"examples": ["outage"], "level": "high"}, "2": ["critical", "churn risk"]} + } + }, + "usage": {"input_tokens": 10, "output_tokens": 0} + }`) + var native TypesafeDecisionResponse + if err := sonic.Unmarshal(raw, &native); err != nil { + t.Fatalf("structured legend failed to decode: %v", err) + } + + req := decisionRequest("state", map[string]schemas.DecisionQuestion{ + "urgency": {Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: []any{ + "low", + map[string]any{"examples": []any{"outage"}, "level": "high"}, + []any{"critical", "churn risk"}, + }}, + }) + resp, bifrostErr := ToBifrostDecisionResponse(&native, req) + if bifrostErr != nil { + t.Fatalf("unexpected error: %v", bifrostErr) + } + legend := resp.Answers["urgency"].Legend + if legend["0"] != "low" { + t.Errorf("string legend level lost: %#v", legend["0"]) + } + if !reflect.DeepEqual(legend["1"], map[string]any{"examples": []any{"outage"}, "level": "high"}) { + t.Errorf("object legend level not lossless: %#v", legend["1"]) + } + if !reflect.DeepEqual(legend["2"], []any{"critical", "churn risk"}) { + t.Errorf("array legend level not lossless: %#v", legend["2"]) + } + + // And straight back out through the native converter without truncation. + back, err := ToTypesafeNativeDecisionResponse(resp) + if err != nil { + t.Fatalf("native round trip failed: %v", err) + } + if !reflect.DeepEqual(back.Answers["urgency"].Legend, legend) { + t.Errorf("legend lost in native round trip: %#v", back.Answers["urgency"].Legend) + } +} + +func TestToBifrostDecisionResponseFailures(t *testing.T) { + req := decisionRequest("state", map[string]schemas.DecisionQuestion{ + "is_spam": {Kind: schemas.DecisionKindNoul, Instructions: "d"}, + }) + + t.Run("missing answer", func(t *testing.T) { + native := &TypesafeDecisionResponse{Model: "jev-1.13.0", Answers: map[string]TypesafeAnswer{}} + if _, err := ToBifrostDecisionResponse(native, req); err == nil { + t.Fatal("expected error for missing answer") + } + }) + + t.Run("answer type mismatch", func(t *testing.T) { + choice := "yes" + native := &TypesafeDecisionResponse{ + Model: "jev-1.13.0", + Answers: map[string]TypesafeAnswer{"is_spam": {Type: TypesafeQuestionTypeChoice, Choice: &choice}}, + } + if _, err := ToBifrostDecisionResponse(native, req); err == nil { + t.Fatal("expected error for mismatched answer type") + } + }) + + t.Run("value missing for type", func(t *testing.T) { + native := &TypesafeDecisionResponse{ + Model: "jev-1.13.0", + Answers: map[string]TypesafeAnswer{"is_spam": {Type: TypesafeQuestionTypeNoul}}, + } + if _, err := ToBifrostDecisionResponse(native, req); err == nil { + t.Fatal("expected error for noul answer without value") + } + }) + + t.Run("noul value outside range", func(t *testing.T) { + // The provider response is untrusted; a noul outside [0,1] violates the + // documented contract and must not surface as a successful answer. + for _, bad := range []float64{-0.2, 1.4} { + value := bad + native := &TypesafeDecisionResponse{ + Model: "jev-1.13.0", + Answers: map[string]TypesafeAnswer{"is_spam": {Type: TypesafeQuestionTypeNoul, Noul: &value}}, + } + if _, err := ToBifrostDecisionResponse(native, req); err == nil { + t.Fatalf("expected error for noul value %v outside [0,1]", bad) + } + } + }) +} + +func TestNativeRoundTrip(t *testing.T) { + native := &TypesafeDecisionRequest{ + State: "some state", + Model: "jev-latest", + Questions: map[string]TypesafeQuestion{ + "approve": {Type: TypesafeQuestionTypeNoul, Instructions: "Approve?", Criteria: map[string]interface{}{"true": "approve it", "false": "reject it"}}, + "bucket": {Type: TypesafeQuestionTypeChoice, Instructions: "Bucket?", Criteria: map[string]interface{}{"a": "first", "b": "second"}}, + "rating": {Type: TypesafeQuestionTypeScore, Instructions: "Rate it", Criteria: []interface{}{"low", "high"}}, + }, + } + + shared, err := native.ToBifrostDecisionRequest(nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if shared.Provider != schemas.Typesafe || shared.Model != "jev-latest" { + t.Errorf("routing = %s/%s", shared.Provider, shared.Model) + } + if len(shared.Questions) != 3 { + t.Fatalf("expected 3 questions, got %d", len(shared.Questions)) + } + + // The shared shape must convert straight back to the native shape. + back, err := ToTypesafeDecisionRequest(shared) + if err != nil { + t.Fatalf("round trip failed: %v", err) + } + for name, question := range native.Questions { + got, ok := back.Questions[name] + if !ok { + t.Fatalf("question %q lost in round trip", name) + } + if got.Type != question.Type { + t.Errorf("question %q type = %q, want %q", name, got.Type, question.Type) + } + } + + // Explicit provider prefix routes the same way. + native.Model = "typesafe/jev-1.13.0" + shared, err = native.ToBifrostDecisionRequest(nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if shared.Provider != schemas.Typesafe || shared.Model != "jev-1.13.0" { + t.Errorf("prefixed routing = %s/%s", shared.Provider, shared.Model) + } +} + +func TestToTypesafeNativeDecisionResponse(t *testing.T) { + resp := &schemas.BifrostDecisionResponse{ + Model: "jev-1.13.0", + Answers: map[string]schemas.DecisionAnswer{ + "approve": {Kind: schemas.DecisionKindNoul, Value: 0.25, Probabilities: map[string]float64{"true": 0.25, "false": 0.75}}, + }, + Usage: &schemas.BifrostLLMUsage{PromptTokens: 10, CompletionTokens: 0, TotalTokens: 10}, + } + + native, err := ToTypesafeNativeDecisionResponse(resp) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + answer := native.Answers["approve"] + if answer.Type != TypesafeQuestionTypeNoul || answer.Noul == nil || *answer.Noul != 0.25 { + t.Errorf("noul answer not rebuilt: %+v", answer) + } + if answer.Probabilities["false"] != 0.75 { + t.Errorf("probabilities lost: %+v", answer.Probabilities) + } + if native.Usage == nil || native.Usage.InputTokens != 10 { + t.Errorf("usage lost: %+v", native.Usage) + } +} + +// noopLogger satisfies schemas.Logger for constructor calls in unit tests. +// schemas cannot provide one and core's DefaultLogger would import-cycle here. +type noopLogger struct{} + +func (noopLogger) Debug(string, ...any) {} +func (noopLogger) Info(string, ...any) {} +func (noopLogger) Warn(string, ...any) {} +func (noopLogger) Error(string, ...any) {} +func (noopLogger) Fatal(string, ...any) {} +func (noopLogger) SetLevel(schemas.LogLevel) {} +func (noopLogger) SetOutputType(schemas.LoggerOutputType) {} +func (noopLogger) LogHTTPRequest(schemas.LogLevel, string) schemas.LogEventBuilder { return nil } + +func TestListModelsEntriesCarryOwnerAndDescription(t *testing.T) { + provider, err := NewTypesafeProvider(&schemas.ProviderConfig{}, noopLogger{}) + if err != nil { + t.Fatalf("constructor failed: %v", err) + } + key := schemas.Key{Models: []string{"*"}} + resp, bifrostErr := provider.listModelsByKey(nil, key, &schemas.BifrostListModelsRequest{}) + if bifrostErr != nil { + t.Fatalf("unexpected error: %v", bifrostErr) + } + if len(resp.Data) != len(typesafeModels) { + t.Fatalf("expected %d models, got %d", len(typesafeModels), len(resp.Data)) + } + for _, model := range resp.Data { + if model.OwnedBy == nil || *model.OwnedBy != "typesafe" { + t.Errorf("model %s missing owned_by: %v", model.ID, model.OwnedBy) + } + if model.Description == nil || *model.Description == "" { + t.Errorf("model %s missing description", model.ID) + } + // Pricing intentionally absent here: it backfills from the datasheet in + // the list-models handler, never from the provider. + if model.Pricing != nil { + t.Errorf("model %s carries hardcoded pricing; pricing must come from the datasheet", model.ID) + } + } +} + +func TestListModelsNativeShape(t *testing.T) { + name := "Jev (latest)" + resp := &schemas.BifrostListModelsResponse{ + Data: []schemas.Model{{ID: "typesafe/jev-latest", Name: &name}}, + } + native := ToTypesafeNativeListModelsResponse(resp) + if len(native.Models) != 1 { + t.Fatalf("expected 1 model, got %d", len(native.Models)) + } + if native.Models[0].Name != "jev-latest" { + t.Errorf("native model name = %q, want bare jev-latest", native.Models[0].Name) + } + if native.Models[0].Description == "" || native.Models[0].ReleaseDate == "" { + t.Errorf("catalog metadata not attached: %+v", native.Models[0]) + } +} + +func TestToTypesafeNativeError(t *testing.T) { + errType := "invalid_request_error" + native := ToTypesafeNativeError(&schemas.BifrostError{ + Error: &schemas.ErrorField{Type: &errType, Message: "question \"q\" has unsupported kind"}, + }) + if native.Detail.ErrorType != "invalid_request_error" { + t.Errorf("error_type = %q", native.Detail.ErrorType) + } + if native.Detail.Message == "" { + t.Error("message lost") + } + + // Missing pieces fall back rather than emitting empty fields. + fallback := ToTypesafeNativeError(nil) + if fallback.Detail.ErrorType != "api_error" || fallback.Detail.Message == "" { + t.Errorf("nil error fallback = %+v", fallback.Detail) + } +} diff --git a/core/providers/typesafe/errors.go b/core/providers/typesafe/errors.go new file mode 100644 index 00000000000..d7b49bac92b --- /dev/null +++ b/core/providers/typesafe/errors.go @@ -0,0 +1,72 @@ +package typesafe + +import ( + providerUtils "github.com/maximhq/bifrost/core/providers/utils" + schemas "github.com/maximhq/bifrost/core/schemas" + "github.com/valyala/fasthttp" +) + +// parseTypesafeError parses a Typesafe error HTTP response into a BifrostError. +// Typesafe returns a JSON body detailing the issue on 401, 422, 429 and 529; +// the upstream status code and validation detail are preserved. +func parseTypesafeError(resp *fasthttp.Response) *schemas.BifrostError { + var errorResp TypesafeError + bifrostErr := providerUtils.HandleProviderAPIError(resp, &errorResp) + + message := errorResp.Message + if message == "" && errorResp.Error != nil { + message = errorResp.Error.Message + } + if message == "" { + if detail, ok := errorResp.Detail.(string); ok { + message = detail + } + } + + if bifrostErr.Error == nil { + bifrostErr.Error = &schemas.ErrorField{} + } + if message != "" { + bifrostErr.Error.Message = message + } else if bifrostErr.Error.Message == "" { + bifrostErr.Error.Message = "Typesafe API request failed" + } + + return bifrostErr +} + +// TypesafeNativeErrorDetail is the payload of Typesafe's native error body. +type TypesafeNativeErrorDetail struct { + ErrorType string `json:"error_type"` + Message string `json:"message"` +} + +// TypesafeNativeError is the error body Typesafe's API returns and its SDKs +// parse: {"detail": {"error_type": ..., "message": ...}} (observed on the live +// API; the docs describe JSON error bodies without pinning the schema). +type TypesafeNativeError struct { + Detail TypesafeNativeErrorDetail `json:"detail"` +} + +// ToTypesafeNativeError converts a Bifrost error into Typesafe's native error +// body so the /typesafe drop-in surface stays parseable by Typesafe's SDKs. +// The HTTP status code rides on the response as usual; this shapes the body. +func ToTypesafeNativeError(bifrostErr *schemas.BifrostError) *TypesafeNativeError { + native := &TypesafeNativeError{ + Detail: TypesafeNativeErrorDetail{ErrorType: "api_error"}, + } + if bifrostErr == nil { + native.Detail.Message = "unknown error" + return native + } + if bifrostErr.Error != nil { + native.Detail.Message = bifrostErr.Error.Message + if bifrostErr.Error.Type != nil && *bifrostErr.Error.Type != "" { + native.Detail.ErrorType = *bifrostErr.Error.Type + } + } + if native.Detail.Message == "" { + native.Detail.Message = "Typesafe API request failed" + } + return native +} diff --git a/core/providers/typesafe/types.go b/core/providers/typesafe/types.go new file mode 100644 index 00000000000..c515471fd42 --- /dev/null +++ b/core/providers/typesafe/types.go @@ -0,0 +1,67 @@ +package typesafe + +// Typesafe question types accepted by POST /v1/systemone. +const ( + TypesafeQuestionTypeNoul = "noul" + TypesafeQuestionTypeChoice = "choice" + TypesafeQuestionTypeScore = "score" +) + +// TypesafeQuestion is one named question in a systemone request. +// Instructions accepts a string, object, or array. Criteria is a map for noul +// (optional "true"/"false" descriptions) and choice (option -> description, +// max 255 options), and an ordered array of 2-10 level descriptions for score. +type TypesafeQuestion struct { + Type string `json:"type"` + Instructions interface{} `json:"instructions,omitempty"` + Criteria interface{} `json:"criteria,omitempty"` +} + +// TypesafeDecisionRequest is the body of POST /v1/systemone. +type TypesafeDecisionRequest struct { + State interface{} `json:"state"` + Model string `json:"model"` + Questions map[string]TypesafeQuestion `json:"questions"` +} + +// GetExtraParams implements providerUtils.RequestBodyWithExtraParams. Typesafe +// takes no passthrough params. +func (r *TypesafeDecisionRequest) GetExtraParams() map[string]interface{} { + return nil +} + +// TypesafeAnswer is one answer in a systemone response. Exactly one of Noul, +// Choice, or Score is set, matching Type. +type TypesafeAnswer struct { + Type string `json:"type"` + Noul *float64 `json:"noul,omitempty"` + Choice *string `json:"choice,omitempty"` + Score *float64 `json:"score,omitempty"` + Probabilities map[string]float64 `json:"probabilities,omitempty"` + // Legend echoes each score level's description verbatim (string, object, + // or array), keyed by level number. + Legend map[string]any `json:"legend,omitempty"` + Confidence *float64 `json:"confidence,omitempty"` +} + +// TypesafeUsage reports token consumption. Typesafe bills input tokens only. +type TypesafeUsage struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` +} + +// TypesafeDecisionResponse is the body of a successful systemone response. +type TypesafeDecisionResponse struct { + Model string `json:"model"` + Answers map[string]TypesafeAnswer `json:"answers"` + Usage *TypesafeUsage `json:"usage,omitempty"` +} + +// TypesafeError is the JSON error body Typesafe returns on 401/422/429/529. +type TypesafeError struct { + Message string `json:"message,omitempty"` + Detail interface{} `json:"detail,omitempty"` + Error *struct { + Message string `json:"message,omitempty"` + } `json:"error,omitempty"` +} diff --git a/core/providers/typesafe/typesafe.go b/core/providers/typesafe/typesafe.go new file mode 100644 index 00000000000..f54f6580b62 --- /dev/null +++ b/core/providers/typesafe/typesafe.go @@ -0,0 +1,199 @@ +// Package typesafe implements the Typesafe provider for Bifrost. +// Typesafe serves judgment models (the jev System One family) through a single +// synchronous evaluation endpoint, POST /v1/systemone. Bifrost exposes it via +// the shared decision operation; every other operation returns unsupported. +package typesafe + +import ( + "net/http" + "strings" + "time" + + providerUtils "github.com/maximhq/bifrost/core/providers/utils" + schemas "github.com/maximhq/bifrost/core/schemas" + "github.com/valyala/fasthttp" +) + +// TypesafeProvider implements the Provider interface for Typesafe's API. +type TypesafeProvider struct { + logger schemas.Logger // Logger for provider operations + client *fasthttp.Client // HTTP client for unary API requests (ReadTimeout bounds overall response) + streamingClient *fasthttp.Client // HTTP client for streaming API requests (no ReadTimeout; unused today, kept per provider pattern) + networkConfig schemas.NetworkConfig // Network configuration including extra headers + sendBackRawRequest bool // Whether to include raw request in BifrostResponse + sendBackRawResponse bool // Whether to include raw response in BifrostResponse +} + +// NewTypesafeProvider creates a new Typesafe provider instance. +func NewTypesafeProvider(config *schemas.ProviderConfig, logger schemas.Logger) (*TypesafeProvider, error) { + config.CheckAndSetDefaults() + + requestTimeout := time.Second * time.Duration(config.NetworkConfig.DefaultRequestTimeoutInSeconds) + client := &fasthttp.Client{ + ReadTimeout: requestTimeout, + WriteTimeout: requestTimeout, + MaxConnsPerHost: config.NetworkConfig.MaxConnsPerHost, + MaxIdleConnDuration: time.Second * time.Duration(config.NetworkConfig.KeepAliveTimeoutInSeconds), + MaxConnWaitTimeout: requestTimeout, + MaxConnDuration: time.Second * time.Duration(schemas.DefaultMaxConnDurationInSeconds), + ConnPoolStrategy: fasthttp.FIFO, + } + + // Configure proxy if provided + client = providerUtils.ConfigureProxy(client, config.ProxyConfig, logger) + client = providerUtils.ConfigureDialer(client, config.NetworkConfig.AllowPrivateNetwork) + client = providerUtils.ConfigureTLS(client, config.NetworkConfig, logger) + streamingClient := providerUtils.BuildStreamingClient(client) + + if config.NetworkConfig.BaseURL == "" { + config.NetworkConfig.BaseURL = typesafeDefaultBaseURL + } + config.NetworkConfig.BaseURL = strings.TrimRight(config.NetworkConfig.BaseURL, "/") + + return &TypesafeProvider{ + logger: logger, + client: client, + streamingClient: streamingClient, + networkConfig: config.NetworkConfig, + sendBackRawRequest: config.SendBackRawRequest, + sendBackRawResponse: config.SendBackRawResponse, + }, nil +} + +// GetProviderKey returns the provider identifier for Typesafe. +func (provider *TypesafeProvider) GetProviderKey() schemas.ModelProvider { + return schemas.Typesafe +} + +// ListModels serves the static jev catalog. Typesafe documents no model-listing +// endpoint, so no upstream call is made; the catalog is pinned in utils.go and +// mirrored in the hosted datasheet. +func (provider *TypesafeProvider) ListModels(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostListModelsRequest) (*schemas.BifrostListModelsResponse, *schemas.BifrostError) { + startTime := time.Now() + + response, err := providerUtils.HandleMultipleListModelsRequests(ctx, keys, request, provider.listModelsByKey) + if err != nil { + return nil, err + } + + response.ExtraFields.Latency = time.Since(startTime).Milliseconds() + return response, nil +} + +// listModelsByKey filters the static catalog through the standard list-models +// pipeline so key whitelists, blacklists, and aliases apply. +func (provider *TypesafeProvider) listModelsByKey(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostListModelsRequest) (*schemas.BifrostListModelsResponse, *schemas.BifrostError) { + response := &schemas.BifrostListModelsResponse{ + Data: make([]schemas.Model, 0, len(typesafeModels)), + } + + pipeline := &providerUtils.ListModelsPipeline{ + AllowedModels: key.Models, + BlacklistedModels: key.BlacklistedModels, + Aliases: key.Aliases, + Unfiltered: request.Unfiltered, + ProviderKey: provider.GetProviderKey(), + MatchFns: providerUtils.DefaultMatchFns(), + } + if pipeline.ShouldEarlyExit() { + return response, nil + } + + included := make(map[string]bool) + for _, model := range typesafeModels { + for _, result := range pipeline.FilterModel(model.ID) { + name := model.Name + description := model.Description + entry := schemas.Model{ + ID: string(provider.GetProviderKey()) + "/" + result.ResolvedID, + Name: &name, + Description: &description, + OwnedBy: new("typesafe"), + } + if result.AliasValue != "" { + alias := result.AliasValue + entry.Alias = &alias + } + response.Data = append(response.Data, entry) + included[strings.ToLower(result.ResolvedID)] = true + } + } + response.Data = append(response.Data, pipeline.BackfillModels(included)...) + + return response, nil +} + +// Decision performs a synchronous evaluation against POST /v1/systemone. +func (provider *TypesafeProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + jsonData, bifrostErr := providerUtils.CheckContextAndGetRequestBody( + ctx, + request, + func() (providerUtils.RequestBodyWithExtraParams, error) { + return ToTypesafeDecisionRequest(request) + }) + if bifrostErr != nil { + return nil, bifrostErr + } + + sendBackRawRequest := providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest) + sendBackRawResponse := providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse) + + req := fasthttp.AcquireRequest() + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseRequest(req) + defer fasthttp.ReleaseResponse(resp) + + providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) + req.SetRequestURI(provider.networkConfig.BaseURL + providerUtils.GetPathFromContext(ctx, typesafeSystemOnePath)) + req.Header.SetMethod(http.MethodPost) + req.Header.SetContentType("application/json") + if key.Value.GetValue() != "" { + req.Header.Set("Authorization", "Bearer "+key.Value.GetValue()) + } + req.SetBody(jsonData) + + latency, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp) + defer wait() + if bifrostErr != nil { + return nil, bifrostErr + } + + if resp.StatusCode() != fasthttp.StatusOK { + return nil, providerUtils.EnrichError(ctx, parseTypesafeError(resp), jsonData, nil, sendBackRawRequest, sendBackRawResponse, latency) + } + + ft, fh := providerUtils.StartPhaseSpan(ctx, "response-finalize") + respBody, err := providerUtils.CheckAndDecodeBody(resp) + if ft != nil { + if err != nil { + ft.EndSpan(fh, schemas.SpanStatusError, err.Error()) + } else { + ft.EndSpan(fh, schemas.SpanStatusOk, "") + } + } + if err != nil { + rawErrBody := append([]byte(nil), resp.Body()...) + return nil, providerUtils.EnrichError(ctx, providerUtils.NewBifrostOperationError(schemas.ErrProviderResponseDecode, err), jsonData, rawErrBody, sendBackRawRequest, sendBackRawResponse, latency) + } + + var typesafeResp TypesafeDecisionResponse + rawRequest, rawResponse, bifrostErr := providerUtils.HandleProviderResponseCtx(ctx, respBody, &typesafeResp, jsonData, sendBackRawRequest, sendBackRawResponse) + if bifrostErr != nil { + return nil, bifrostErr + } + + bifrostResp, bifrostErr := ToBifrostDecisionResponse(&typesafeResp, request) + if bifrostErr != nil { + return nil, providerUtils.EnrichError(ctx, bifrostErr, jsonData, respBody, sendBackRawRequest, sendBackRawResponse, latency) + } + + bifrostResp.ExtraFields.Latency = latency.Milliseconds() + if sendBackRawRequest { + bifrostResp.ExtraFields.RawRequest = rawRequest + } + if sendBackRawResponse { + bifrostResp.ExtraFields.RawResponse = rawResponse + } + + return bifrostResp, nil +} diff --git a/core/providers/typesafe/typesafe_test.go b/core/providers/typesafe/typesafe_test.go new file mode 100644 index 00000000000..278510aa4d7 --- /dev/null +++ b/core/providers/typesafe/typesafe_test.go @@ -0,0 +1,40 @@ +package typesafe_test + +import ( + "os" + "strings" + "testing" + + "github.com/maximhq/bifrost/core/internal/llmtests" + "github.com/maximhq/bifrost/core/schemas" +) + +func TestTypesafe(t *testing.T) { + t.Parallel() + if strings.TrimSpace(os.Getenv("TYPESAFE_API_KEY")) == "" { + t.Skip("Skipping Typesafe tests because TYPESAFE_API_KEY is not set") + } + + client, ctx, cancel, err := llmtests.SetupTest() + if err != nil { + t.Fatalf("Error initializing test setup: %v", err) + } + defer cancel() + defer client.Shutdown() + + testConfig := llmtests.ComprehensiveTestConfig{ + Provider: schemas.Typesafe, + DecisionModel: "jev-1.13.0", + Scenarios: llmtests.TestScenarios{ + // Typesafe serves the decision operation only; every other operation + // returns the standard unsupported-operation error. ListModels is + // served from the static catalog without an upstream call. + Decision: true, + ListModels: true, + }, + } + + t.Run("TypesafeTests", func(t *testing.T) { + llmtests.RunAllComprehensiveTests(t, client, ctx, testConfig) + }) +} diff --git a/core/providers/typesafe/unsupported.go b/core/providers/typesafe/unsupported.go new file mode 100644 index 00000000000..74efab6c37f --- /dev/null +++ b/core/providers/typesafe/unsupported.go @@ -0,0 +1,229 @@ +// Operations the Typesafe evaluation API does not serve. Typesafe exposes a single +// synchronous judgment endpoint (POST /v1/systemone); every other Bifrost operation +// returns the standard unsupported-operation error. +package typesafe + +import ( + "context" + + providerUtils "github.com/maximhq/bifrost/core/providers/utils" + schemas "github.com/maximhq/bifrost/core/schemas" +) + +func (provider *TypesafeProvider) TextCompletion(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostTextCompletionRequest) (*schemas.BifrostTextCompletionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.TextCompletionRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) TextCompletionStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostTextCompletionRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.TextCompletionStreamRequest, provider.GetProviderKey()) +} + +// Speech is not supported by the Typesafe provider. +func (provider *TypesafeProvider) Speech(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostSpeechRequest) (*schemas.BifrostSpeechResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.SpeechRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) Rerank(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostRerankRequest) (*schemas.BifrostRerankResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) SpeechStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostSpeechRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.SpeechStreamRequest, provider.GetProviderKey()) +} + +// Transcription is not supported by the Typesafe provider. +func (provider *TypesafeProvider) Transcription(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostTranscriptionRequest) (*schemas.BifrostTranscriptionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.TranscriptionRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) TranscriptionStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostTranscriptionRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.TranscriptionStreamRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ImageGeneration(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostImageGenerationRequest) (*schemas.BifrostImageGenerationResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ImageGenerationRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ImageGenerationStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostImageGenerationRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ImageGenerationStreamRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ImageEdit(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostImageEditRequest) (*schemas.BifrostImageGenerationResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ImageEditRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ImageEditStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostImageEditRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ImageEditStreamRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ImageVariation(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostImageVariationRequest) (*schemas.BifrostImageGenerationResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ImageVariationRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) VideoGeneration(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostVideoGenerationRequest) (*schemas.BifrostVideoGenerationResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.VideoGenerationRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) VideoRetrieve(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostVideoRetrieveRequest) (*schemas.BifrostVideoGenerationResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.VideoRetrieveRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) VideoDownload(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostVideoDownloadRequest) (*schemas.BifrostVideoDownloadResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.VideoDownloadRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) VideoDelete(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostVideoDeleteRequest) (*schemas.BifrostVideoDeleteResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.VideoDeleteRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) VideoList(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostVideoListRequest) (*schemas.BifrostVideoListResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.VideoListRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) VideoEdit(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostVideoEditRequest) (*schemas.BifrostVideoEditResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.VideoEditRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) VideoRemix(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostVideoRemixRequest) (*schemas.BifrostVideoGenerationResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.VideoRemixRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) BatchCreate(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostBatchCreateRequest) (*schemas.BifrostBatchCreateResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.BatchCreateRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) BatchList(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostBatchListRequest) (*schemas.BifrostBatchListResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.BatchListRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) BatchRetrieve(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostBatchRetrieveRequest) (*schemas.BifrostBatchRetrieveResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.BatchRetrieveRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) BatchCancel(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostBatchCancelRequest) (*schemas.BifrostBatchCancelResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.BatchCancelRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) BatchDelete(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostBatchDeleteRequest) (*schemas.BifrostBatchDeleteResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.BatchDeleteRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) BatchResults(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostBatchResultsRequest) (*schemas.BifrostBatchResultsResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.BatchResultsRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) FileUpload(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostFileUploadRequest) (*schemas.BifrostFileUploadResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.FileUploadRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) FileList(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostFileListRequest) (*schemas.BifrostFileListResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.FileListRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) FileRetrieve(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostFileRetrieveRequest) (*schemas.BifrostFileRetrieveResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.FileRetrieveRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) FileDelete(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostFileDeleteRequest) (*schemas.BifrostFileDeleteResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.FileDeleteRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) FileContent(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostFileContentRequest) (*schemas.BifrostFileContentResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.FileContentRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) CountTokens(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostResponsesRequest) (*schemas.BifrostCountTokensResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.CountTokensRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) Compaction(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostCompactionRequest) (*schemas.BifrostCompactionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.CompactionRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ContainerCreate(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostContainerCreateRequest) (*schemas.BifrostContainerCreateResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerCreateRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ContainerList(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostContainerListRequest) (*schemas.BifrostContainerListResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerListRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ContainerRetrieve(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostContainerRetrieveRequest) (*schemas.BifrostContainerRetrieveResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerRetrieveRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ContainerDelete(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostContainerDeleteRequest) (*schemas.BifrostContainerDeleteResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerDeleteRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ContainerFileCreate(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostContainerFileCreateRequest) (*schemas.BifrostContainerFileCreateResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerFileCreateRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ContainerFileList(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostContainerFileListRequest) (*schemas.BifrostContainerFileListResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerFileListRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ContainerFileRetrieve(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostContainerFileRetrieveRequest) (*schemas.BifrostContainerFileRetrieveResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerFileRetrieveRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ContainerFileContent(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostContainerFileContentRequest) (*schemas.BifrostContainerFileContentResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerFileContentRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ContainerFileDelete(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostContainerFileDeleteRequest) (*schemas.BifrostContainerFileDeleteResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerFileDeleteRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) Passthrough(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostPassthroughRequest) (*schemas.BifrostPassthroughResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.PassthroughRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) PassthroughStream(_ *schemas.BifrostContext, _ schemas.PostHookRunner, _ func(context.Context), _ schemas.Key, _ *schemas.BifrostPassthroughRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.PassthroughStreamRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) CachedContentCreate(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostCachedContentCreateRequest) (*schemas.BifrostCachedContentCreateResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.CachedContentCreateRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) CachedContentDelete(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostCachedContentDeleteRequest) (*schemas.BifrostCachedContentDeleteResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.CachedContentDeleteRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) CachedContentList(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostCachedContentListRequest) (*schemas.BifrostCachedContentListResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.CachedContentListRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) CachedContentRetrieve(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostCachedContentRetrieveRequest) (*schemas.BifrostCachedContentRetrieveResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.CachedContentRetrieveRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) CachedContentUpdate(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostCachedContentUpdateRequest) (*schemas.BifrostCachedContentUpdateResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.CachedContentUpdateRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ChatCompletion(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostChatRequest) (*schemas.BifrostChatResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ChatCompletionRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ChatCompletionStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostChatRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ChatCompletionStreamRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) Embedding(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostEmbeddingRequest) (*schemas.BifrostEmbeddingResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.EmbeddingRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) Responses(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostResponsesRequest) (*schemas.BifrostResponsesResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ResponsesRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ResponsesStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostResponsesRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ResponsesStreamRequest, provider.GetProviderKey()) +} diff --git a/core/providers/typesafe/utils.go b/core/providers/typesafe/utils.go new file mode 100644 index 00000000000..73aa79c1ef0 --- /dev/null +++ b/core/providers/typesafe/utils.go @@ -0,0 +1,89 @@ +package typesafe + +import ( + "strings" + + schemas "github.com/maximhq/bifrost/core/schemas" +) + +// typesafeDefaultBaseURL is the default Typesafe API host. +const typesafeDefaultBaseURL = "https://api.typesafe.ai" + +// typesafeSystemOnePath is the single evaluation endpoint for the System One +// model class; every jev model is served by it, selected via the request's +// model field. +const typesafeSystemOnePath = "/v1/systemone" + +// typesafeModel is one entry of the static model catalog. +type typesafeModel struct { + ID string + Name string + Description string + ReleaseDate string +} + +// typesafeModels is the static catalog served by ListModels. Typesafe documents +// no model-listing endpoint, so the catalog is pinned here and in the hosted +// datasheet. Aliases resolve upstream: jev-latest and jev-preview both point at +// jev-1.13.0 today. +var typesafeModels = []typesafeModel{ + { + ID: "jev-1.13.0", + Name: "Jev 1.13.0", + Description: "TypeSafe's System One Model: Jev, version 1.13.0", + // The SDKs validate release_date on every listed model; 1.13.0 shipped + // at the same instant its aliases were minted. + ReleaseDate: "2026-09-10T18:38:01.391457+00:00", + }, + { + ID: "jev-latest", + Name: "Jev (latest)", + Description: "The latest iteration of TypeSafe's System One Model: Jev", + ReleaseDate: "2026-09-10T18:38:01.391457+00:00", + }, + { + ID: "jev-preview", + Name: "Jev (preview)", + Description: "A preview version of `jev-latest`: should be better in most ways", + ReleaseDate: "2026-09-10T18:39:06.057655+00:00", + }, +} + +// TypesafeNativeModel is one entry of the native model-listing shape. +type TypesafeNativeModel struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + ReleaseDate string `json:"release_date,omitempty"` +} + +// TypesafeNativeListModelsResponse is the native model-listing shape served on +// /typesafe/v1/models. Typesafe documents no upstream listing endpoint, so +// this is synthesized from the static catalog. +type TypesafeNativeListModelsResponse struct { + Models []TypesafeNativeModel `json:"models"` +} + +// ToTypesafeNativeListModelsResponse converts a Bifrost model listing into the +// native shape, restoring bare upstream model names and re-attaching catalog +// descriptions and release dates. +func ToTypesafeNativeListModelsResponse(resp *schemas.BifrostListModelsResponse) *TypesafeNativeListModelsResponse { + native := &TypesafeNativeListModelsResponse{Models: []TypesafeNativeModel{}} + if resp == nil { + return native + } + catalog := make(map[string]typesafeModel, len(typesafeModels)) + for _, model := range typesafeModels { + catalog[model.ID] = model + } + prefix := string(schemas.Typesafe) + "/" + for _, model := range resp.Data { + name := strings.TrimPrefix(model.ID, prefix) + entry := TypesafeNativeModel{Name: name} + if known, ok := catalog[name]; ok { + entry.Description = known.Description + entry.ReleaseDate = known.ReleaseDate + } + native.Models = append(native.Models, entry) + } + return native +} diff --git a/core/providers/vertex/vertex.go b/core/providers/vertex/vertex.go index 2065274dbd6..203aa3dbdd8 100644 --- a/core/providers/vertex/vertex.go +++ b/core/providers/vertex/vertex.go @@ -2002,6 +2002,11 @@ func (provider *VertexProvider) Rerank(ctx *schemas.BifrostContext, key schemas. return bifrostResponse, nil } +// Decision is not supported by the Vertex provider. +func (provider *VertexProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Vertex provider. func (provider *VertexProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/vllm/vllm.go b/core/providers/vllm/vllm.go index d8747359d82..3c1dc3ee8f4 100644 --- a/core/providers/vllm/vllm.go +++ b/core/providers/vllm/vllm.go @@ -542,6 +542,11 @@ func (provider *VLLMProvider) Rerank(ctx *schemas.BifrostContext, key schemas.Ke return bifrostResponse, nil } +// Decision is not supported by the VLLM provider. +func (provider *VLLMProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Vllm provider. func (provider *VLLMProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/wafer/wafer.go b/core/providers/wafer/wafer.go index 3d00197c392..c5979ad5160 100644 --- a/core/providers/wafer/wafer.go +++ b/core/providers/wafer/wafer.go @@ -214,6 +214,11 @@ func (provider *WaferProvider) Rerank(ctx *schemas.BifrostContext, key schemas.K return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Wafer provider. +func (provider *WaferProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Wafer provider. func (provider *WaferProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/xai/xai.go b/core/providers/xai/xai.go index 9942aebc94d..c8a0b16af88 100644 --- a/core/providers/xai/xai.go +++ b/core/providers/xai/xai.go @@ -283,6 +283,11 @@ func (provider *XAIProvider) Rerank(ctx *schemas.BifrostContext, key schemas.Key return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the XAI provider. +func (provider *XAIProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Xai provider. func (provider *XAIProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/schemas/bifrost.go b/core/schemas/bifrost.go index 5d32be9fc5c..8110366ed77 100644 --- a/core/schemas/bifrost.go +++ b/core/schemas/bifrost.go @@ -76,6 +76,7 @@ const ( Wafer ModelProvider = "wafer" GithubCopilot ModelProvider = "github-copilot" Databricks ModelProvider = "databricks" + Typesafe ModelProvider = "typesafe" ) // SupportedBaseProviders is the list of base providers allowed for custom providers. @@ -123,6 +124,7 @@ var StandardProviders = []ModelProvider{ Wafer, GithubCopilot, Databricks, + Typesafe, } // RequestType represents the type of request being made to a provider. @@ -190,6 +192,7 @@ const ( ContainerFileContentRequest RequestType = "container_file_content" ContainerFileDeleteRequest RequestType = "container_file_delete" RerankRequest RequestType = "rerank" + DecisionRequest RequestType = "decisions" OCRRequest RequestType = "ocr" CountTokensRequest RequestType = "count_tokens" CompactionRequest RequestType = "compaction" @@ -590,6 +593,7 @@ type BifrostRequest struct { CompactionRequest *BifrostCompactionRequest EmbeddingRequest *BifrostEmbeddingRequest RerankRequest *BifrostRerankRequest + DecisionRequest *BifrostDecisionRequest OCRRequest *BifrostOCRRequest SpeechRequest *BifrostSpeechRequest TranscriptionRequest *BifrostTranscriptionRequest @@ -658,6 +662,8 @@ func (br *BifrostRequest) GetRequestFields() (provider ModelProvider, model stri return br.EmbeddingRequest.Provider, br.EmbeddingRequest.Model, br.EmbeddingRequest.Fallbacks case br.RerankRequest != nil: return br.RerankRequest.Provider, br.RerankRequest.Model, br.RerankRequest.Fallbacks + case br.DecisionRequest != nil: + return br.DecisionRequest.Provider, br.DecisionRequest.Model, br.DecisionRequest.Fallbacks case br.OCRRequest != nil: return br.OCRRequest.Provider, br.OCRRequest.Model, br.OCRRequest.Fallbacks case br.SpeechRequest != nil: @@ -811,6 +817,8 @@ func (br *BifrostRequest) SetProvider(provider ModelProvider) { br.EmbeddingRequest.Provider = provider case br.RerankRequest != nil: br.RerankRequest.Provider = provider + case br.DecisionRequest != nil: + br.DecisionRequest.Provider = provider case br.OCRRequest != nil: br.OCRRequest.Provider = provider case br.SpeechRequest != nil: @@ -866,6 +874,8 @@ func (br *BifrostRequest) SetModel(model string) { br.EmbeddingRequest.Model = model case br.RerankRequest != nil: br.RerankRequest.Model = model + case br.DecisionRequest != nil: + br.DecisionRequest.Model = model case br.OCRRequest != nil: br.OCRRequest.Model = model case br.SpeechRequest != nil: @@ -923,6 +933,8 @@ func (br *BifrostRequest) SetFallbacks(fallbacks []Fallback) { br.EmbeddingRequest.Fallbacks = fallbacks case br.RerankRequest != nil: br.RerankRequest.Fallbacks = fallbacks + case br.DecisionRequest != nil: + br.DecisionRequest.Fallbacks = fallbacks case br.OCRRequest != nil: br.OCRRequest.Fallbacks = fallbacks case br.SpeechRequest != nil: @@ -966,6 +978,8 @@ func (br *BifrostRequest) SetRawRequestBody(rawRequestBody []byte) { br.EmbeddingRequest.RawRequestBody = rawRequestBody case br.RerankRequest != nil: br.RerankRequest.RawRequestBody = rawRequestBody + case br.DecisionRequest != nil: + br.DecisionRequest.RawRequestBody = rawRequestBody case br.OCRRequest != nil: br.OCRRequest.RawRequestBody = rawRequestBody case br.SpeechRequest != nil: @@ -1141,6 +1155,7 @@ type BifrostResponse struct { CompactionResponse *BifrostCompactionResponse EmbeddingResponse *BifrostEmbeddingResponse RerankResponse *BifrostRerankResponse + DecisionResponse *BifrostDecisionResponse OCRResponse *BifrostOCRResponse SpeechResponse *BifrostSpeechResponse SpeechStreamResponse *BifrostSpeechStreamResponse @@ -1204,6 +1219,8 @@ func (r *BifrostResponse) GetExtraFields() *BifrostResponseExtraFields { return &r.EmbeddingResponse.ExtraFields case r.RerankResponse != nil: return &r.RerankResponse.ExtraFields + case r.DecisionResponse != nil: + return &r.DecisionResponse.ExtraFields case r.OCRResponse != nil: return &r.OCRResponse.ExtraFields case r.SpeechResponse != nil: @@ -1472,6 +1489,11 @@ func (r *BifrostResponse) PopulateExtraFields(requestType RequestType, provider r.RerankResponse.ExtraFields.Provider = provider r.RerankResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.RerankResponse.ExtraFields.ResolvedModelUsed = resolvedModel + case r.DecisionResponse != nil: + r.DecisionResponse.ExtraFields.RequestType = requestType + r.DecisionResponse.ExtraFields.Provider = provider + r.DecisionResponse.ExtraFields.OriginalModelRequested = originalModelRequested + r.DecisionResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.SpeechResponse != nil: r.SpeechResponse.ExtraFields.RequestType = requestType r.SpeechResponse.ExtraFields.Provider = provider diff --git a/core/schemas/decision.go b/core/schemas/decision.go new file mode 100644 index 00000000000..31e78df1794 --- /dev/null +++ b/core/schemas/decision.go @@ -0,0 +1,65 @@ +package schemas + +// DecisionKind identifies how a single question is decided. The vocabulary +// mirrors Typesafe's System One question types. +type DecisionKind string + +const ( + // DecisionKindNoul yields a probability between 0 and 1. + DecisionKindNoul DecisionKind = "noul" + // DecisionKindChoice yields one option from the question's criteria. + DecisionKindChoice DecisionKind = "choice" + // DecisionKindScore yields a numeric rubric score, including fractions. + DecisionKindScore DecisionKind = "score" +) + +// DecisionQuestion is one named question in a decision request. Instructions +// accepts a string, object, or array, carried losslessly. Criteria is a map of +// descriptions for noul (only "true"/"false" keys) and choice (option -> +// description), and an ordered array of level descriptions for score. +type DecisionQuestion struct { + Kind DecisionKind `json:"kind"` + Instructions interface{} `json:"instructions,omitempty"` + Criteria interface{} `json:"criteria,omitempty"` +} + +// BifrostDecisionRequest represents a request to evaluate state against a map +// of named questions. The shape mirrors Typesafe's System One endpoint. +type BifrostDecisionRequest struct { + Provider ModelProvider `json:"provider"` + Model string `json:"model"` + State interface{} `json:"state"` // string, object, or array + Questions map[string]DecisionQuestion `json:"questions"` + Fallbacks []Fallback `json:"fallbacks,omitempty"` + RawRequestBody []byte `json:"-"` +} + +// GetRawRequestBody returns the raw request body for the decision request. +func (r *BifrostDecisionRequest) GetRawRequestBody() []byte { + return r.RawRequestBody +} + +// DecisionAnswer is one evaluated answer. Value carries the decided value for +// the question's kind: a number in [0,1] for noul, an option string for +// choice, a numeric rubric score for score. Confidence, Probabilities, and +// Legend carry per-field metadata when the provider supplies it. Legend echoes +// each score level's description verbatim, so its values are strings, objects, +// or arrays - whatever the criteria carried. +type DecisionAnswer struct { + Kind DecisionKind `json:"kind"` + Value interface{} `json:"value"` + Confidence *float64 `json:"confidence,omitempty"` + Probabilities map[string]float64 `json:"probabilities,omitempty"` + Legend map[string]any `json:"legend,omitempty"` +} + +// BifrostDecisionResponse represents the response from a decision request. +// Answers is keyed by question identifier; every requested question produces +// an answer. +type BifrostDecisionResponse struct { + ID string `json:"id,omitempty"` + Model string `json:"model"` + Answers map[string]DecisionAnswer `json:"answers"` + Usage *BifrostLLMUsage `json:"usage,omitempty"` + ExtraFields BifrostResponseExtraFields `json:"extra_fields"` +} diff --git a/core/schemas/errors.go b/core/schemas/errors.go index 10081e0b192..fe5e1d3e22a 100644 --- a/core/schemas/errors.go +++ b/core/schemas/errors.go @@ -105,6 +105,7 @@ var modelAddressedRequestTypes = map[RequestType]struct{}{ ImageGenerationRequest: {}, ImageGenerationStreamRequest: {}, RerankRequest: {}, + DecisionRequest: {}, CountTokensRequest: {}, } diff --git a/core/schemas/errors_test.go b/core/schemas/errors_test.go index 05290cfa147..ff16b6f7b1f 100644 --- a/core/schemas/errors_test.go +++ b/core/schemas/errors_test.go @@ -412,6 +412,7 @@ var modelAddressingByRequestType = map[string]bool{ "ImageVariationRequest": false, "VideoGenerationRequest": false, "RerankRequest": true, + "DecisionRequest": true, "OCRRequest": false, "CountTokensRequest": true, diff --git a/core/schemas/provider.go b/core/schemas/provider.go index 755fa8009e0..164e0db3ab9 100644 --- a/core/schemas/provider.go +++ b/core/schemas/provider.go @@ -349,6 +349,7 @@ type AllowedRequests struct { Compaction bool `json:"compaction"` Embedding bool `json:"embedding"` Rerank bool `json:"rerank"` + Decision bool `json:"decisions"` OCR bool `json:"ocr"` Speech bool `json:"speech"` SpeechStream bool `json:"speech_stream"` @@ -434,6 +435,8 @@ func (ar *AllowedRequests) IsOperationAllowed(operation RequestType) bool { return ar.Embedding case RerankRequest: return ar.Rerank + case DecisionRequest: + return ar.Decision case OCRRequest: return ar.OCR case SpeechRequest: @@ -706,6 +709,8 @@ type Provider interface { Embedding(ctx *BifrostContext, key Key, request *BifrostEmbeddingRequest) (*BifrostEmbeddingResponse, *BifrostError) // Rerank performs a rerank request to reorder documents by relevance to a query Rerank(ctx *BifrostContext, key Key, request *BifrostRerankRequest) (*BifrostRerankResponse, *BifrostError) + // Decision performs an decision request against an annotated function-tool definition (Typesafe-only; other providers return unsupported) + Decision(ctx *BifrostContext, key Key, request *BifrostDecisionRequest) (*BifrostDecisionResponse, *BifrostError) // OCR performs an optical character recognition request on a document OCR(ctx *BifrostContext, key Key, request *BifrostOCRRequest) (*BifrostOCRResponse, *BifrostError) // Speech performs a text to speech request diff --git a/docs/docs.json b/docs/docs.json index 1d87ccb2de3..2b836d703c7 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -79,6 +79,7 @@ "quickstart/gateway/tool-calling", "quickstart/gateway/multimodal", "quickstart/gateway/reranking", + "quickstart/gateway/decisions", "quickstart/gateway/integrations" ] }, @@ -94,6 +95,7 @@ "quickstart/go-sdk/tool-calling", "quickstart/go-sdk/multimodal", "quickstart/go-sdk/reranking", + "quickstart/go-sdk/decisions", "quickstart/go-sdk/logger" ] } @@ -165,6 +167,7 @@ "providers/supported-providers/runway", "providers/supported-providers/sarvam", "providers/supported-providers/sgl", + "providers/supported-providers/typesafe", "providers/supported-providers/vertex", "providers/supported-providers/vllm", "providers/supported-providers/wafer", @@ -512,6 +515,7 @@ "integrations/litellm-sdk", "integrations/langchain-sdk", "integrations/pydanticai-sdk", + "integrations/typesafe-sdk", "integrations/passthrough" ] }, @@ -791,6 +795,13 @@ "POST /v1/rerank" ] }, + { + "group": "Decisions", + "expanded": false, + "pages": [ + "POST /v1/decisions" + ] + }, { "group": "Embeddings", "expanded": false, @@ -1055,6 +1066,14 @@ "POST /cohere/v1/tokenize" ] }, + { + "group": "Typesafe Integration", + "expanded": false, + "pages": [ + "POST /typesafe/v1/systemone", + "GET /typesafe/v1/models" + ] + }, { "group": "Cursor Integration", "expanded": false, diff --git a/docs/integrations/typesafe-sdk.mdx b/docs/integrations/typesafe-sdk.mdx new file mode 100644 index 00000000000..2636a3318a3 --- /dev/null +++ b/docs/integrations/typesafe-sdk.mdx @@ -0,0 +1,95 @@ +--- +title: "TypeSafe SDK" +description: "Use TypeSafe's official Python and JavaScript SDKs with Bifrost by changing one base URL." +icon: "scale-balanced" +--- + +Bifrost exposes TypeSafe's native API 1:1 under the `/typesafe` prefix, so the official TypeSafe SDKs work by pointing their base URL at Bifrost. You keep the SDK's question builders, retries, and types - and gain Bifrost's key management, governance, logging, and cost tracking. + +## Endpoints + +| Native TypeSafe | Via Bifrost | +|-----------------|-------------| +| `POST https://api.typesafe.ai/v1/systemone` | `POST http://localhost:8080/typesafe/v1/systemone` | +| Model listing (not offered upstream) | `GET http://localhost:8080/typesafe/v1/models` | + +## Python SDK + +Both SDKs read `TYPESAFE_BASE_URL` from the environment, so the zero-code-change setup is: + +```bash +export TYPESAFE_BASE_URL="http://localhost:8080/typesafe" +``` + +Or pass it explicitly (`typesafe-sdk` package): + +```python +from typesafe_sdk import TypeSafeClient, Noul + +with TypeSafeClient(base_url="http://localhost:8080/typesafe") as client: + response = client.system_one( + state={"document": "I was charged twice. Please fix this ASAP."}, + questions={"billing": Noul(instructions="Is this ticket about billing?")}, + ) + print(response.nouls["billing"].noul) +``` + +## JavaScript / TypeScript SDK + +`@typesafe-ai/sdk` takes `baseURL` (falls back to `TYPESAFE_BASE_URL`): + +```ts +import { TypeSafeClient, choice } from "@typesafe-ai/sdk"; + +const client = new TypeSafeClient({ baseURL: "http://localhost:8080/typesafe" }); + +const response = await client.systemOne({ + state: { document: "I was charged twice. Please fix this ASAP." }, + questions: { + category: choice("What is this ticket about?", { + billing: null, + technical: null, + other: null, + }), + }, +}); +``` + +## Authentication + +The SDK's `Authorization: Bearer` header authenticates the request to Bifrost when Bifrost authentication is enabled. Configure the TypeSafe provider with its upstream keys - Bifrost selects and injects the provider key on the upstream call, so clients can use Bifrost virtual keys instead of raw TypeSafe keys. + +**Using a virtual key** - pass it as the SDK's API key; governance (budgets, rate limits, model restrictions) applies per key: + +```python +from typesafe_sdk import TypeSafeClient, Noul + +with TypeSafeClient( + base_url="http://localhost:8080/typesafe", + api_key="sk-bf-v1-your-virtual-key", # Bifrost virtual key, not a TypeSafe key +) as client: + response = client.system_one( + state={"document": "I was charged twice."}, + questions={"billing": Noul(instructions="Is this ticket about billing?")}, + ) +``` + +```ts +import { TypeSafeClient } from "@typesafe-ai/sdk"; + +const client = new TypeSafeClient({ + baseURL: "http://localhost:8080/typesafe", + apiKey: "sk-bf-v1-your-virtual-key", // Bifrost virtual key +}); +``` + +## Behavior Notes + +- **Success responses are shape-compatible** with TypeSafe's own API: native `answers` with `noul`/`choice`/`score` value fields and `input_tokens`/`output_tokens` usage. Bytes are identical only on the raw-response passthrough path; otherwise the response is rebuilt and re-encoded, preserving the shape but not exact JSON byte ordering. +- **Models**: bare IDs (`jev-1.13.0`, `jev-latest`, `jev-preview`) work exactly as upstream; `typesafe/`-prefixed IDs are additionally accepted. +- **Errors**: returned in TypeSafe's native shape (`{"detail": {"error_type", "message"}}`) with upstream status codes (401, 422, 429, 529) preserved, so SDK exception parsing works unchanged. Requests rejected by Bifrost's local validation return `400` where TypeSafe's own validation would return `422`. +- **Retries**: Bifrost retries 429 and 529 with backoff per the provider's `network_config` in addition to any SDK-side retry policy. + + +For provider-routed access with fallbacks and Bifrost's normalized response shape, use [`POST /v1/decisions`](/quickstart/gateway/decisions) instead - same questions, `kind` instead of `type`, and a unified `value` per answer. + diff --git a/docs/openapi/openapi.json b/docs/openapi/openapi.json index 3b6090d8495..afa7224ff69 100644 --- a/docs/openapi/openapi.json +++ b/docs/openapi/openapi.json @@ -61,6 +61,10 @@ "name": "Rerank", "description": "Document reranking by relevance to a query" }, + { + "name": "Decisions", + "description": "Structured decisions evaluated against annotated function-tool definitions" + }, { "name": "Embeddings", "description": "Text embedding generation" @@ -125,6 +129,10 @@ "name": "Cohere Integration", "description": "Cohere compatible API endpoints (/cohere/*)" }, + { + "name": "Typesafe Integration", + "description": "Typesafe compatible API endpoints (/typesafe/*)" + }, { "name": "LiteLLM Integration", "description": "LiteLLM proxy endpoints with multi-provider support (/litellm/*)" @@ -1935,6 +1943,69 @@ ] } }, + "/v1/decisions": { + "post": { + "operationId": "createDecision", + "summary": "Evaluate a decision", + "description": "Evaluates state against a map of named questions and returns one typed\nanswer per question, with per-answer confidence, probabilities, and\nlegend metadata. Served by decision providers such as Typesafe's jev models.\n", + "tags": [ + "Decisions" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DecisionRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DecisionResponse" + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BifrostError" + } + } + } + }, + "500": { + "description": "Internal server error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BifrostError" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + }, + { + "VirtualKeyAuth": [] + }, + { + "ApiKeyAuth": [] + } + ] + } + }, "/v1/embeddings": { "post": { "operationId": "createEmbedding", @@ -27636,6 +27707,300 @@ ] } }, + "/typesafe/v1/systemone": { + "post": { + "operationId": "typesafeSystemOne", + "summary": "Evaluate state (Typesafe format)", + "description": "Evaluates state against a map of named questions using Typesafe's native\nSystem One API. The request body matches Typesafe's native format; question\nidentifiers become result field names.\n", + "tags": [ + "Typesafe Integration" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "state", + "model", + "questions" + ], + "properties": { + "state": { + "description": "A plain string, or structured data (object or array) to evaluate.", + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "additionalProperties": true + }, + { + "type": "array", + "items": {} + } + ] + }, + "model": { + "type": "string", + "description": "Typesafe model identifier (e.g. `jev-1.13.0`, `jev-latest`)." + }, + "questions": { + "type": "object", + "minProperties": 1, + "description": "Map of named questions.", + "additionalProperties": { + "type": "object", + "required": [ + "type", + "instructions" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "noul", + "choice", + "score" + ], + "description": "Question type. `noul` returns a probability from 0 to 1, `choice` a selection with probabilities, `score` a probability-weighted rubric value." + }, + "instructions": { + "description": "Question instructions. A string, or structured data.", + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "additionalProperties": true + }, + { + "type": "array", + "items": {} + } + ] + }, + "criteria": { + "description": "Optional `true`/`false` descriptions for `noul`; a map of option\nto description for `choice` (at most 255 options); an ordered\narray of 2-10 level descriptions for `score`.\n", + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + } + } + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Successful response in Typesafe's native shape", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "model", + "answers" + ], + "properties": { + "model": { + "type": "string", + "description": "Model version that performed the evaluation" + }, + "answers": { + "type": "object", + "description": "Map matching question identifiers to answers", + "additionalProperties": { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "noul", + "choice", + "score" + ] + }, + "noul": { + "type": "number", + "description": "Probability from 0 to 1 (noul answers)" + }, + "choice": { + "type": "string", + "description": "Selected option (choice answers)" + }, + "score": { + "type": "number", + "description": "Probability-weighted value (score answers)" + }, + "probabilities": { + "type": "object", + "additionalProperties": { + "type": "number" + } + }, + "legend": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "confidence": { + "type": "number" + } + } + } + }, + "usage": { + "type": "object", + "properties": { + "input_tokens": { + "type": "integer" + }, + "output_tokens": { + "type": "integer" + } + } + } + } + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BifrostError" + } + } + } + }, + "500": { + "description": "Internal server error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BifrostError" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + }, + { + "VirtualKeyAuth": [] + }, + { + "ApiKeyAuth": [] + } + ] + } + }, + "/typesafe/v1/models": { + "get": { + "operationId": "typesafeListModels", + "summary": "List models (Typesafe format)", + "description": "Lists available Typesafe models in the native shape. Typesafe documents no\nupstream listing endpoint, so the catalog is served from Bifrost's model\ndatasheet.\n", + "tags": [ + "Typesafe Integration" + ], + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "models" + ], + "properties": { + "models": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "example": "jev-latest" + }, + "description": { + "type": "string" + }, + "release_date": { + "type": "string", + "format": "date-time" + } + } + } + } + } + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BifrostError" + } + } + } + }, + "500": { + "description": "Internal server error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BifrostError" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + }, + { + "VirtualKeyAuth": [] + }, + { + "ApiKeyAuth": [] + } + ] + } + }, "/cohere/v1/tokenize": { "post": { "operationId": "cohereTokenize", @@ -101038,6 +101403,180 @@ } } }, + "DecisionRequest": { + "type": "object", + "required": [ + "model", + "state", + "questions" + ], + "properties": { + "model": { + "type": "string", + "description": "Model in provider/model format", + "example": "typesafe/jev-1.13.0" + }, + "state": { + "description": "State to evaluate. A plain string, or structured data (object or array).", + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "additionalProperties": true + }, + { + "type": "array", + "items": {} + } + ] + }, + "questions": { + "type": "object", + "minProperties": 1, + "description": "Map of named questions. Question identifiers become answer field names.", + "additionalProperties": { + "$ref": "#/components/schemas/DecisionQuestion" + } + }, + "fallbacks": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Fallback models in provider/model format" + } + } + }, + "DecisionResponse": { + "type": "object", + "required": [ + "model", + "answers" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier for the decision response" + }, + "model": { + "type": "string", + "description": "Model that performed the decision" + }, + "answers": { + "type": "object", + "description": "Map matching question identifiers to answers. Every requested question produces an answer.", + "additionalProperties": { + "$ref": "#/components/schemas/DecisionAnswer" + } + }, + "usage": { + "$ref": "#/components/schemas/BifrostLLMUsage" + }, + "extra_fields": { + "$ref": "#/components/schemas/BifrostResponseExtraFields" + } + } + }, + "DecisionQuestion": { + "type": "object", + "required": [ + "kind", + "instructions" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "noul", + "choice", + "score" + ], + "description": "How the question is decided. `noul` yields a probability between 0 and 1, `choice` yields one option from the criteria, `score` yields a numeric rubric score including fractions." + }, + "instructions": { + "description": "Instructions for this question. A string, or structured data preserved losslessly.", + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "additionalProperties": true + }, + { + "type": "array", + "items": {} + } + ] + }, + "criteria": { + "description": "Applicable criteria. Optional `true`/`false` descriptions for `noul`; a\nrequired map of option to description for `choice` (at most 255 options);\na required ordered array of 2-10 level descriptions for `score`.\n", + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + } + }, + "DecisionAnswer": { + "type": "object", + "required": [ + "kind", + "value" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "noul", + "choice", + "score" + ], + "description": "The question's decision kind" + }, + "value": { + "description": "The decided value. A number in [0,1] for `noul`, an option string for `choice`, a numeric rubric score for `score`.", + "oneOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + }, + "confidence": { + "type": "number", + "description": "Model confidence for this answer, when supplied" + }, + "probabilities": { + "type": "object", + "additionalProperties": { + "type": "number" + }, + "description": "Probability distribution over options or levels, when supplied" + }, + "legend": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "Legend mapping levels to descriptions, when supplied" + } + } + }, "EmbeddingRequest": { "type": "object", "required": [ diff --git a/docs/openapi/openapi.yaml b/docs/openapi/openapi.yaml index 3e015f4da29..4371d727a6d 100644 --- a/docs/openapi/openapi.yaml +++ b/docs/openapi/openapi.yaml @@ -86,6 +86,8 @@ tags: description: Optical character recognition for documents and images - name: Rerank description: Document reranking by relevance to a query + - name: Decisions + description: Structured decisions evaluated against annotated function-tool definitions - name: Embeddings description: Text embedding generation - name: Images @@ -119,6 +121,8 @@ tags: description: AWS Bedrock compatible API endpoints (/bedrock/*) - name: Cohere Integration description: Cohere compatible API endpoints (/cohere/*) + - name: Typesafe Integration + description: Typesafe compatible API endpoints (/typesafe/*) - name: LiteLLM Integration description: LiteLLM proxy endpoints with multi-provider support (/litellm/*) - name: LangChain Integration @@ -171,6 +175,8 @@ paths: $ref: "./paths/inference/ocr.yaml#/ocr" /v1/rerank: $ref: "./paths/inference/rerank.yaml#/rerank" + /v1/decisions: + $ref: "./paths/inference/decisions.yaml#/decisions" /v1/embeddings: $ref: "./paths/inference/embeddings.yaml#/embeddings" /v1/audio/speech: @@ -519,6 +525,12 @@ paths: /cohere/v2/rerank: $ref: "./paths/integrations/cohere/rerank.yaml#/rerank" + # Typesafe Integration + /typesafe/v1/systemone: + $ref: "./paths/integrations/typesafe/systemone.yaml#/systemone" + /typesafe/v1/models: + $ref: "./paths/integrations/typesafe/systemone.yaml#/models" + # Tokenize (v1) /cohere/v1/tokenize: $ref: "./paths/integrations/cohere/tokenize.yaml#/tokenize" @@ -1515,6 +1527,16 @@ components: RerankResult: $ref: "./schemas/inference/rerank.yaml#/RerankResult" + # ==================== Decisions ==================== + DecisionRequest: + $ref: "./schemas/inference/decisions.yaml#/DecisionRequest" + DecisionResponse: + $ref: "./schemas/inference/decisions.yaml#/DecisionResponse" + DecisionQuestion: + $ref: "./schemas/inference/decisions.yaml#/DecisionQuestion" + DecisionAnswer: + $ref: "./schemas/inference/decisions.yaml#/DecisionAnswer" + # ==================== Embeddings ==================== EmbeddingRequest: $ref: "./schemas/inference/embeddings.yaml#/EmbeddingRequest" diff --git a/docs/openapi/paths/inference/decisions.yaml b/docs/openapi/paths/inference/decisions.yaml new file mode 100644 index 00000000000..fd2c171b13d --- /dev/null +++ b/docs/openapi/paths/inference/decisions.yaml @@ -0,0 +1,31 @@ +decisions: + post: + operationId: createDecision + summary: Evaluate a decision + description: | + Evaluates state against a map of named questions and returns one typed + answer per question, with per-answer confidence, probabilities, and + legend metadata. Served by decision providers such as Typesafe's jev models. + tags: + - Decisions + requestBody: + required: true + content: + application/json: + schema: + $ref: '../../schemas/inference/decisions.yaml#/DecisionRequest' + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '../../schemas/inference/decisions.yaml#/DecisionResponse' + '400': + $ref: '../../openapi.yaml#/components/responses/BadRequest' + '500': + $ref: '../../openapi.yaml#/components/responses/InternalError' + security: + - BearerAuth: [] + - VirtualKeyAuth: [] + - ApiKeyAuth: [] diff --git a/docs/openapi/paths/integrations/typesafe/systemone.yaml b/docs/openapi/paths/integrations/typesafe/systemone.yaml new file mode 100644 index 00000000000..0631196fb14 --- /dev/null +++ b/docs/openapi/paths/integrations/typesafe/systemone.yaml @@ -0,0 +1,171 @@ +# Typesafe Integration - System One Endpoint + +systemone: + post: + operationId: typesafeSystemOne + summary: Evaluate state (Typesafe format) + description: | + Evaluates state against a map of named questions using Typesafe's native + System One API. The request body matches Typesafe's native format; question + identifiers become result field names. + tags: + - Typesafe Integration + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - state + - model + - questions + properties: + state: + description: A plain string, or structured data (object or array) to evaluate. + oneOf: + - type: string + - type: object + additionalProperties: true + - type: array + items: {} + model: + type: string + description: Typesafe model identifier (e.g. `jev-1.13.0`, `jev-latest`). + questions: + type: object + minProperties: 1 + description: Map of named questions. + additionalProperties: + type: object + required: + - type + - instructions + properties: + type: + type: string + enum: [noul, choice, score] + description: Question type. `noul` returns a probability from 0 to 1, `choice` a selection with probabilities, `score` a probability-weighted rubric value. + instructions: + description: Question instructions. A string, or structured data. + oneOf: + - type: string + - type: object + additionalProperties: true + - type: array + items: {} + criteria: + description: | + Optional `true`/`false` descriptions for `noul`; a map of option + to description for `choice` (at most 255 options); an ordered + array of 2-10 level descriptions for `score`. + oneOf: + - type: object + additionalProperties: + type: string + - type: array + items: + type: string + responses: + '200': + description: Successful response in Typesafe's native shape + content: + application/json: + schema: + type: object + required: + - model + - answers + properties: + model: + type: string + description: Model version that performed the evaluation + answers: + type: object + description: Map matching question identifiers to answers + additionalProperties: + type: object + required: + - type + properties: + type: + type: string + enum: [noul, choice, score] + noul: + type: number + description: Probability from 0 to 1 (noul answers) + choice: + type: string + description: Selected option (choice answers) + score: + type: number + description: Probability-weighted value (score answers) + probabilities: + type: object + additionalProperties: + type: number + legend: + type: object + additionalProperties: + type: string + confidence: + type: number + usage: + type: object + properties: + input_tokens: + type: integer + output_tokens: + type: integer + '400': + $ref: '../../../openapi.yaml#/components/responses/BadRequest' + '500': + $ref: '../../../openapi.yaml#/components/responses/InternalError' + security: + - BearerAuth: [] + - VirtualKeyAuth: [] + - ApiKeyAuth: [] + +models: + get: + operationId: typesafeListModels + summary: List models (Typesafe format) + description: | + Lists available Typesafe models in the native shape. Typesafe documents no + upstream listing endpoint, so the catalog is served from Bifrost's model + datasheet. + tags: + - Typesafe Integration + responses: + '200': + description: Successful response + content: + application/json: + schema: + type: object + required: + - models + properties: + models: + type: array + items: + type: object + required: + - name + properties: + name: + type: string + example: jev-latest + description: + type: string + release_date: + type: string + format: date-time + '400': + $ref: '../../../openapi.yaml#/components/responses/BadRequest' + '500': + $ref: '../../../openapi.yaml#/components/responses/InternalError' + security: + - BearerAuth: [] + - VirtualKeyAuth: [] + - ApiKeyAuth: [] diff --git a/docs/openapi/schemas/inference/decisions.yaml b/docs/openapi/schemas/inference/decisions.yaml new file mode 100644 index 00000000000..d8838d7744a --- /dev/null +++ b/docs/openapi/schemas/inference/decisions.yaml @@ -0,0 +1,114 @@ +# Decisions API schemas + +DecisionRequest: + type: object + required: + - model + - state + - questions + properties: + model: + type: string + description: Model in provider/model format + example: typesafe/jev-1.13.0 + state: + description: State to evaluate. A plain string, or structured data (object or array). + oneOf: + - type: string + - type: object + additionalProperties: true + - type: array + items: {} + questions: + type: object + minProperties: 1 + description: Map of named questions. Question identifiers become answer field names. + additionalProperties: + $ref: '#/DecisionQuestion' + fallbacks: + type: array + items: + type: string + description: Fallback models in provider/model format + +DecisionQuestion: + type: object + required: + - kind + - instructions + properties: + kind: + type: string + enum: [noul, choice, score] + description: How the question is decided. `noul` yields a probability between 0 and 1, `choice` yields one option from the criteria, `score` yields a numeric rubric score including fractions. + instructions: + description: Instructions for this question. A string, or structured data preserved losslessly. + oneOf: + - type: string + - type: object + additionalProperties: true + - type: array + items: {} + criteria: + description: | + Applicable criteria. Optional `true`/`false` descriptions for `noul`; a + required map of option to description for `choice` (at most 255 options); + a required ordered array of 2-10 level descriptions for `score`. + oneOf: + - type: object + additionalProperties: + type: string + - type: array + items: + type: string + +DecisionResponse: + type: object + required: + - model + - answers + properties: + id: + type: string + description: Unique identifier for the decision response + model: + type: string + description: Model that performed the decision + answers: + type: object + description: Map matching question identifiers to answers. Every requested question produces an answer. + additionalProperties: + $ref: '#/DecisionAnswer' + usage: + $ref: './usage.yaml#/BifrostLLMUsage' + extra_fields: + $ref: './common.yaml#/BifrostResponseExtraFields' + +DecisionAnswer: + type: object + required: + - kind + - value + properties: + kind: + type: string + enum: [noul, choice, score] + description: The question's decision kind + value: + description: The decided value. A number in [0,1] for `noul`, an option string for `choice`, a numeric rubric score for `score`. + oneOf: + - type: number + - type: string + confidence: + type: number + description: Model confidence for this answer, when supplied + probabilities: + type: object + additionalProperties: + type: number + description: Probability distribution over options or levels, when supplied + legend: + type: object + additionalProperties: + type: string + description: Legend mapping levels to descriptions, when supplied diff --git a/docs/providers/supported-providers/overview.mdx b/docs/providers/supported-providers/overview.mdx index 0271f34dee0..005fde278f7 100644 --- a/docs/providers/supported-providers/overview.mdx +++ b/docs/providers/supported-providers/overview.mdx @@ -14,38 +14,39 @@ Bifrost can also act as a provider-compatible gateway (for example, [Anthropi The following table summarizes which operations are supported by each provider via Bifrost’s unified interface. -| Provider | Models | Text | Text (stream) | Chat | Chat (stream) | Responses | Responses (stream) | Images | Images (stream) | Image Edit | Image Edit (stream) | Image Variation | Embeddings | TTS | TTS (stream) | STT | STT (stream) | Files | Batch | Count tokens | Rerank | OCR | Video | Video Remix | Containers | Passthrough | Passthrough (stream) | -| ------------------------------------ | ------ | ---- | ------------- | ---- | ------------- | --------- | ------------------ | ------ | --------------- | ---------- | ------------------- | --------------- | ---------- | --- | ------------ | --- | ------------ | ----- | ----- | ------------ | ------ | --- | ----- | ----------- | ---------- | ----------- | -------------------- | -| Anthropic (`anthropic/`) | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | -| Azure (`azure/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | -| Bedrock (`bedrock/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Bedrock Mantle (`bedrock_mantle/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Cerebras (`cerebras/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Cohere (`cohere/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Databricks (`databricks/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| DeepSeek (`deepseek/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Elevenlabs (`elevenlabs/`) | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Fireworks (`fireworks/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Gemini (`gemini/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | -| GitHub Copilot (`github-copilot/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Groq (`groq/`) | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Hugging Face (`huggingface/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Mistral (`mistral/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Nebius (`nebius/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Ollama (`ollama/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| OpenAI (`openai/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | -| OpenRouter (`openrouter/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Parasail (`parasail/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Perplexity (`perplexity/`) | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Replicate (`replicate/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | -| Runware (`runware/`) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | -| Runway (`runway/`) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | -| Sarvam AI (`sarvam/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| SGL (`sgl/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Vertex AI (`vertex/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | -| vLLM (`vllm/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Wafer (`wafer/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| xAI (`xai/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Provider | Models | Text | Text (stream) | Chat | Chat (stream) | Responses | Responses (stream) | Images | Images (stream) | Image Edit | Image Edit (stream) | Image Variation | Embeddings | TTS | TTS (stream) | STT | STT (stream) | Files | Batch | Count tokens | Rerank | Decisions | OCR | Video | Video Remix | Containers | Passthrough | Passthrough (stream) | +| ------------------------------------ | ------ | ---- | ------------- | ---- | ------------- | --------- | ------------------ | ------ | --------------- | ---------- | ------------------- | --------------- | ---------- | --- | ------------ | --- | ------------ | ----- | ----- | ------------ | ------ | --------- | --- | ----- | ----------- | ---------- | ----------- | -------------------- | +| Anthropic (`anthropic/`) | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | +| Azure (`azure/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | +| Bedrock (`bedrock/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Bedrock Mantle (`bedrock_mantle/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Cerebras (`cerebras/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Cohere (`cohere/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Databricks (`databricks/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| DeepSeek (`deepseek/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Elevenlabs (`elevenlabs/`) | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Fireworks (`fireworks/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Gemini (`gemini/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | +| GitHub Copilot (`github-copilot/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Groq (`groq/`) | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Hugging Face (`huggingface/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Mistral (`mistral/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Nebius (`nebius/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Ollama (`ollama/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| OpenAI (`openai/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | +| OpenRouter (`openrouter/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Parasail (`parasail/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Perplexity (`perplexity/`) | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Replicate (`replicate/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | +| Runware (`runware/`) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | +| Runway (`runway/`) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | +| Sarvam AI (`sarvam/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| SGL (`sgl/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| TypeSafe (`typesafe/`) | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Vertex AI (`vertex/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | +| vLLM (`vllm/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Wafer (`wafer/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| xAI (`xai/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | - 🟡 Not supported by the downstream provider, but internally implemented by Bifrost as a fallback. - ❌ Not supported by the downstream provider, hence not supported by Bifrost. diff --git a/docs/providers/supported-providers/typesafe.mdx b/docs/providers/supported-providers/typesafe.mdx new file mode 100644 index 00000000000..3eca4940b09 --- /dev/null +++ b/docs/providers/supported-providers/typesafe.mdx @@ -0,0 +1,189 @@ +--- +title: "TypeSafe" +description: "TypeSafe API conversion guide - jev judgment models, decisions endpoint, question types, and model management" +icon: "scale-balanced" +--- + +## Overview + +TypeSafe is a specialized decision provider serving the jev family of System One judgment models. A decision request carries a `state` (what to evaluate) and a map of named `questions`; the response carries one typed answer per question. Bifrost performs conversions including: + +- **Model ID mapping** - Uses provider model identifier directly (`jev-1.13.0`, `jev-latest`, `jev-preview`) +- **Question conversion** - Maps Bifrost decision kinds (`noul`, `choice`, `score`) to TypeSafe's native System One question types 1:1 +- **Answer normalization** - Native `noul`/`choice`/`score` value fields normalize to a unified `value`, with `confidence`, `probabilities`, and `legend` metadata preserved +- **Usage normalization** - `input_tokens`/`output_tokens` map to Bifrost's `prompt_tokens`/`completion_tokens` (TypeSafe bills input tokens only) +- **Strict validation** - Unsupported kinds, missing instructions, and malformed criteria are rejected locally with a 400 rather than silently approximated + +### Supported Operations + +| Operation | Non-Streaming | Streaming | Endpoint | +|-----------|---------------|-----------|----------| +| Decisions | ✅ | - | `/v1/decisions` | +| List Models | ✅ | - | `/v1/models` | +| Chat Completions | ❌ | ❌ | - | +| Responses API | ❌ | ❌ | - | +| Text Completions | ❌ | ❌ | - | +| Embeddings | ❌ | ❌ | - | + + +**Unsupported Operations** (❌): Chat Completions, Responses API, Text Completions, Embeddings, and every other operation return `UnsupportedOperationError` - TypeSafe serves judgment models only. + +**Model listing**: TypeSafe documents no upstream models endpoint. Bifrost serves the jev catalog from its model datasheet, including pricing and context length. + + +## Setup & Configuration + +Configure TypeSafe as a provider with a bearer API key: + +```json +{ + "providers": { + "typesafe": { + "keys": [ + { + "name": "TypeSafe API Key", + "value": "env.TYPESAFE_API_KEY", + "weight": 1, + "models": ["*"] + } + ] + } + } +} +``` + +## Decisions API + +`POST /v1/decisions` evaluates state against named questions. Each question has a `kind`, `instructions` (string or structured data), and kind-specific `criteria`: + +| Kind | Answer value | Criteria | +|------|--------------|----------| +| `noul` | Probability between 0 and 1 | Optional `true`/`false` descriptions | +| `choice` | One option string | Required map of option to description (max 255) | +| `score` | Numeric rubric score, including fractions | Required ordered array of 2-10 level descriptions | + +```bash +curl http://localhost:8080/v1/decisions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "typesafe/jev-1.13.0", + "state": "Customer message: I was double charged and nobody replied. I want a refund today.", + "questions": { + "is_frustrated": { "kind": "noul", "instructions": "Is the customer frustrated?" }, + "category": { + "kind": "choice", + "instructions": "Pick the ticket category", + "criteria": { "billing": "charges and refunds", "bug": "product defects", "other": "anything else" } + }, + "urgency": { + "kind": "score", + "instructions": "Rate how urgently this needs a human reply", + "criteria": ["can wait a week", "should be answered soon", "needs a reply today"] + } + } + }' +``` + +Response: + +```json +{ + "model": "jev-1.13.0", + "answers": { + "is_frustrated": { "kind": "noul", "value": 0.98 }, + "category": { + "kind": "choice", + "value": "billing", + "confidence": 1, + "probabilities": { "billing": 1, "bug": 0, "other": 0 } + }, + "urgency": { + "kind": "score", + "value": 2, + "confidence": 1, + "probabilities": { "0": 0, "1": 0, "2": 1 }, + "legend": { "0": "can wait a week", "1": "should be answered soon", "2": "needs a reply today" } + } + }, + "usage": { "prompt_tokens": 437, "completion_tokens": 72, "total_tokens": 509 } +} +``` + +## Field Mapping Reference + +Complete mapping between Bifrost's `/v1/decisions` contract and TypeSafe's native `/v1/systemone` API. + +### Request + +| Bifrost field | Native field | Accepted shapes | Validation | +|---------------|--------------|-----------------|------------| +| `model` | `model` | `typesafe/` (routing prefix stripped) | Must resolve to the typesafe provider | +| `state` | `state` | string, object, array (lossless) | Any other shape (number, boolean, null) rejected with 400 | +| `questions` | `questions` | Map of 1+ named questions | Empty map rejected with 400 | +| `questions..kind` | `questions..type` | `noul`, `choice`, `score` | Any other kind rejected with 400 | +| `questions..instructions` | `instructions` | string, object, array (lossless) | Required; other shapes rejected with 400 | +| `questions..criteria` | `criteria` | Kind-specific, see below | Kind-specific, see below | +| `fallbacks` | - | `["provider/model", ...]` | Bifrost-only; consumed by fallback routing, never sent upstream | + +### Criteria by kind + +| Kind | Bifrost / native shape (identical) | Validation | +|------|------------------------------------|------------| +| `noul` | Optional map with only `"true"` and `"false"` keys, string descriptions | Other keys or non-string descriptions rejected with 400 | +| `choice` | Required map of option → description (string) | Missing, empty, non-map, non-string descriptions, or more than 255 options rejected with 400 | +| `score` | Required ordered array of level descriptions | Fewer than 2 or more than 10 levels, or any non-string level, rejected with 400 | + +In the converted request, validated criteria values pass through to TypeSafe unchanged - descriptions are never rewritten. The conversion path re-encodes JSON, so whitespace and object-key ordering can differ; raw-request passthrough bypasses conversion and preserves the original request bytes. + +### Response + +| Bifrost field | Built from native | Notes | +|---------------|-------------------|-------| +| `model` | `model` | Aliases resolve upstream: request `jev-latest`, response reports `jev-1.13.0` | +| `answers..kind` | `answers..type` | Identical vocabulary (`noul`/`choice`/`score`) | +| `answers..value` | `answers..noul` \| `.choice` \| `.score` | Unified value field: number in [0,1] for noul (out-of-range provider values rejected), option string for choice, fractional rubric score for score | +| `answers..confidence` | `confidence` | Present on choice/score answers when supplied | +| `answers..probabilities` | `probabilities` | Distribution over options (choice) or level indices (score) | +| `answers..legend` | `legend` | Level index → description map for score answers | +| `usage.prompt_tokens` | `usage.input_tokens` | TypeSafe bills input tokens only | +| `usage.completion_tokens` | `usage.output_tokens` | Output rate is $0 | +| `usage.total_tokens` | - | Computed: input + output | +| `extra_fields` | - | Bifrost-only: request type, routing info, latency, raw request/response capture | + +Every requested question must produce an answer of its declared kind; a missing answer, a kind mismatch, or a missing value fails the request rather than returning partial results. + +### Errors + +| Upstream status | Meaning | Bifrost behavior | +|-----------------|---------|------------------| +| 401 | Invalid or missing API key | Status and message preserved; key marked failed, other keys rotate in | +| 422 | Request validation failed upstream | Status and validation detail preserved | +| 429 | Rate limited | Retried with backoff; keys rotate (per-key limit) | +| 529 | Service overloaded | Retried with backoff on the same key (capacity, not credential) | + +Requests rejected by Bifrost's own validation (see tables above) return `400` with `caller_invalid_request` before any upstream call is made. + +## Native Integration + +Bifrost also exposes TypeSafe's native API 1:1 under the `/typesafe` prefix, for clients written against TypeSafe directly: + +| Native Endpoint | Method | Notes | +|-----------------|--------|-------| +| `/typesafe/v1/systemone` | POST | Native request/response shape; questions use `type` instead of `kind` | +| `/typesafe/v1/models` | GET | Native `{"models": [{"name", "description", "release_date"}]}` shape, served from the datasheet | + +The native route accepts bare model IDs (`jev-1.13.0`) exactly as TypeSafe does, and additionally accepts `typesafe/`-prefixed IDs. + + +Success responses are shape-compatible with TypeSafe's own API - identical byte-for-byte only on the raw-response passthrough path, otherwise rebuilt and re-encoded (native shape preserved, exact JSON byte ordering not guaranteed). Error responses use TypeSafe's native `{"detail": {"error_type", "message"}}` shape with upstream status codes preserved. Requests rejected by Bifrost's local validation return `400` where TypeSafe's own validation would return `422`. + + +## Model Naming + +| Model | Notes | +|-------|-------| +| `jev-1.13.0` | Current versioned release | +| `jev-latest` | Alias, currently resolves to `jev-1.13.0` | +| `jev-preview` | Preview alias, currently resolves to `jev-1.13.0` | + +Responses always report the resolved versioned model. TypeSafe bills input tokens only; the jev pricing, 64k context window, and rate limits are documented at [docs.typesafe.ai/models](https://docs.typesafe.ai/models). diff --git a/docs/quickstart/gateway/decisions.mdx b/docs/quickstart/gateway/decisions.mdx new file mode 100644 index 00000000000..c8b1190dd4c --- /dev/null +++ b/docs/quickstart/gateway/decisions.mdx @@ -0,0 +1,94 @@ +--- +title: "Decisions" +description: "Evaluate state against named questions using /v1/decisions." +icon: "scale-balanced" +--- + +Use decisions to get structured judgments - probabilities, choices, and rubric scores - from judgment models like TypeSafe's jev family. A decision request carries a `state` (what to evaluate) and a map of named `questions`; every question produces one typed answer. + +## Provider Model Examples + +- TypeSafe: `typesafe/jev-1.13.0`, `typesafe/jev-latest`, `typesafe/jev-preview` + +## Basic Request + +```bash +curl --location 'http://localhost:8080/v1/decisions' \ +--header 'Content-Type: application/json' \ +--data '{ + "model": "typesafe/jev-1.13.0", + "state": "Customer message: I was double charged and nobody replied to my two emails. I want a refund today.", + "questions": { + "is_frustrated": { + "kind": "noul", + "instructions": "Is the customer frustrated?" + }, + "category": { + "kind": "choice", + "instructions": "Pick the ticket category", + "criteria": { "billing": "charges and refunds", "bug": "product defects", "other": "anything else" } + }, + "urgency": { + "kind": "score", + "instructions": "Rate how urgently this needs a human reply", + "criteria": ["can wait a week", "should be answered soon", "needs a reply today"] + } + } +}' +``` + +## Question Kinds + +| Kind | Answer value | Criteria | +|------|--------------|----------| +| `noul` | Probability between 0 and 1 | Optional `true`/`false` descriptions | +| `choice` | One option string | Required map of option to description (max 255 options) | +| `score` | Probability-weighted rubric score, including fractions | Required ordered array of 2-10 level descriptions | + +`state` and `instructions` accept a plain string or structured data (object or array), preserved losslessly. + +## Response + +```json +{ + "model": "jev-1.13.0", + "answers": { + "is_frustrated": { "kind": "noul", "value": 0.98 }, + "category": { + "kind": "choice", + "value": "billing", + "confidence": 1, + "probabilities": { "billing": 1, "bug": 0, "other": 0 } + }, + "urgency": { + "kind": "score", + "value": 2, + "confidence": 1, + "probabilities": { "0": 0, "1": 0, "2": 1 }, + "legend": { "0": "can wait a week", "1": "should be answered soon", "2": "needs a reply today" } + } + }, + "usage": { "prompt_tokens": 437, "completion_tokens": 72, "total_tokens": 509 } +} +``` + +Every declared question produces an answer of its declared kind. `confidence`, `probabilities`, and `legend` are included when the provider supplies them. + +## Fallbacks + +Standard fallback syntax applies: + +```json +{ + "model": "typesafe/jev-preview", + "fallbacks": ["typesafe/jev-1.13.0"] +} +``` + +## Validation + +Requests are validated before dispatch and rejected with a `400` rather than silently approximated: unsupported kinds, missing instructions, missing choice criteria, score criteria outside 2-10 levels or with non-string entries, and states or instructions that are neither string, object, nor array. + + +TypeSafe's native API is also available 1:1 at `POST /typesafe/v1/systemone` - see the [TypeSafe provider page](/providers/supported-providers/typesafe) for the mapping between the two. + diff --git a/docs/quickstart/go-sdk/decisions.mdx b/docs/quickstart/go-sdk/decisions.mdx new file mode 100644 index 00000000000..81fadfee681 --- /dev/null +++ b/docs/quickstart/go-sdk/decisions.mdx @@ -0,0 +1,107 @@ +--- +title: "Decisions" +description: "Evaluate state against named questions with Bifrost Go SDK using client.DecisionRequest." +icon: "scale-balanced" +--- + +Use the Go SDK to get structured judgments - probabilities, choices, and rubric scores - from judgment models like TypeSafe's jev family. + +Provider/model examples: +- TypeSafe: `Provider: schemas.Typesafe`, `Model: "jev-1.13.0"` (aliases: `jev-latest`, `jev-preview`) + +## Basic Example + +The account must serve keys for `schemas.Typesafe` - the OpenAI-only account from the setup guide returns "provider not supported" for this example. A minimal TypeSafe-capable account: + +```go +type MyAccount struct{} + +func (a *MyAccount) GetConfiguredProviders() ([]schemas.ModelProvider, error) { + return []schemas.ModelProvider{schemas.Typesafe}, nil +} + +func (a *MyAccount) GetKeysForProvider(ctx context.Context, provider schemas.ModelProvider) ([]schemas.Key, error) { + return []schemas.Key{ + { + Value: *schemas.NewSecretVar("env.TYPESAFE_API_KEY"), + Models: []string{"*"}, + Weight: 1.0, + }, + }, nil +} + +func (a *MyAccount) GetConfigForProvider(provider schemas.ModelProvider) (*schemas.ProviderConfig, error) { + return &schemas.ProviderConfig{}, nil +} +``` + +```go +package main + +import ( + "context" + "fmt" + + bifrost "github.com/maximhq/bifrost/core" + "github.com/maximhq/bifrost/core/schemas" +) + +func main() { + client, err := bifrost.Init(context.Background(), schemas.BifrostConfig{ + Account: &MyAccount{}, + }) + if err != nil { + panic(err) + } + defer client.Shutdown() + + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + + response, bifrostErr := client.DecisionRequest(ctx, &schemas.BifrostDecisionRequest{ + Provider: schemas.Typesafe, + Model: "jev-1.13.0", + State: "Customer message: I was double charged and nobody replied. I want a refund today.", + Questions: map[string]schemas.DecisionQuestion{ + "is_frustrated": { + Kind: schemas.DecisionKindNoul, + Instructions: "Is the customer frustrated?", + }, + "category": { + Kind: schemas.DecisionKindChoice, + Instructions: "Pick the ticket category", + Criteria: map[string]interface{}{ + "billing": "charges and refunds", + "bug": "product defects", + "other": "anything else", + }, + }, + "urgency": { + Kind: schemas.DecisionKindScore, + Instructions: "Rate how urgently this needs a human reply", + Criteria: []interface{}{"can wait a week", "should be answered soon", "needs a reply today"}, + }, + }, + }) + if bifrostErr != nil { + panic(bifrostErr.Error.Message) + } + + for name, answer := range response.Answers { + fmt.Printf("%s (%s): %v\n", name, answer.Kind, answer.Value) + } +} +``` + +## Answers + +Every question produces one `schemas.DecisionAnswer`: + +| Field | Meaning | +|-------|---------| +| `Kind` | The question's kind (`noul`, `choice`, `score`) | +| `Value` | `float64` in [0,1] for noul, option `string` for choice, `float64` rubric score for score | +| `Confidence` | Model confidence, when supplied | +| `Probabilities` | Distribution over options or levels, when supplied | +| `Legend` | Level index to description map for score answers, when supplied | + +`State` and `Instructions` accept a string, `map[string]interface{}`, or `[]interface{}`, preserved losslessly. Fallbacks work like every other request type via the `Fallbacks` field. diff --git a/framework/modelcatalog/datasheet/cost.go b/framework/modelcatalog/datasheet/cost.go index ca23cfa3f80..e3e7ae18a3b 100644 --- a/framework/modelcatalog/datasheet/cost.go +++ b/framework/modelcatalog/datasheet/cost.go @@ -629,6 +629,8 @@ func (s *Store) computeCostFromInput(input costInput, routingInfo schemas.Routin cost = computeEmbeddingCost(pricing, input.usage, input.tier) case schemas.RerankRequest: cost = computeRerankCost(pricing, input.usage, input.tier) + case schemas.DecisionRequest: + cost = computeDecisionCost(pricing, input.usage, input.tier) case schemas.SpeechRequest: cost = computeSpeechCost(pricing, input.usage, input.audioSeconds, input.audioTextInputChars, input.tier) case schemas.TranscriptionRequest: @@ -725,6 +727,9 @@ func extractCostInput(result *schemas.BifrostResponse) costInput { case result.RerankResponse != nil: input.usage = result.RerankResponse.Usage + case result.DecisionResponse != nil && result.DecisionResponse.Usage != nil: + input.usage = result.DecisionResponse.Usage + case result.SpeechResponse != nil && result.SpeechResponse.Usage != nil: input.usage = speechUsageToBifrostUsage(result.SpeechResponse.Usage) input.audioTextInputChars = result.SpeechResponse.Usage.InputChars @@ -1222,6 +1227,20 @@ func computeRerankCost(pricing *configstoreTables.TableModelPricing, usage *sche return newInputOutputCostWithDetails(inputTokensCost, outputTokensCost, inputDetails, outputDetails) } +// computeDecisionCost prices a decision request on token usage. Typesafe bills +// input tokens only, so the output rate is normally zero, but both sides are +// honored from the datasheet so a provider that later charges output stays +// correct without a code change. +func computeDecisionCost(pricing *configstoreTables.TableModelPricing, usage *schemas.BifrostLLMUsage, tier serviceTier) *schemas.BifrostCost { + if usage == nil { + return nil + } + tierTokens := usage.PromptTokens + inputCost := float64(usage.PromptTokens) * tieredInputRate(pricing, tierTokens, tier) + outputCost := float64(usage.CompletionTokens) * tieredOutputRate(pricing, tierTokens, tier) + return newInputOutputCost(inputCost, outputCost) +} + // newInputOutputCost builds a BifrostCost from separate input and output costs, // or returns nil when both are zero (nothing to record). func newInputOutputCost(inputCost, outputCost float64) *schemas.BifrostCost { diff --git a/framework/modelcatalog/datasheet/cost_test.go b/framework/modelcatalog/datasheet/cost_test.go index 8ed338d66f4..a0e047b082a 100644 --- a/framework/modelcatalog/datasheet/cost_test.go +++ b/framework/modelcatalog/datasheet/cost_test.go @@ -255,6 +255,19 @@ func makeRerankResponse(provider schemas.ModelProvider, model string, usage *sch } } +// makeDecisionResponse builds a minimal BifrostResponse for a decision request. +func makeDecisionResponse(provider schemas.ModelProvider, model string, usage *schemas.BifrostLLMUsage) *schemas.BifrostResponse { + return &schemas.BifrostResponse{ + DecisionResponse: &schemas.BifrostDecisionResponse{ + Usage: usage, + ExtraFields: schemas.BifrostResponseExtraFields{ + RequestType: schemas.DecisionRequest, + RoutingInfo: routingInfoFor(provider, model), + }, + }, + } +} + // makeImageResponse builds a minimal BifrostResponse for an image generation request. func makeImageResponse(provider schemas.ModelProvider, model string, usage *schemas.ImageUsage) *schemas.BifrostResponse { return &schemas.BifrostResponse{ @@ -298,6 +311,10 @@ func computeRerankCostTotal(pricing *configstoreTables.TableModelPricing, usage return bcTotal(computeRerankCost(pricing, usage, tier)) } +func computeDecisionCostTotal(pricing *configstoreTables.TableModelPricing, usage *schemas.BifrostLLMUsage, tier serviceTier) float64 { + return bcTotal(computeDecisionCost(pricing, usage, tier)) +} + func computeSpeechCostTotal(pricing *configstoreTables.TableModelPricing, usage *schemas.BifrostLLMUsage, audioSeconds *float64, audioTextInputChars int, tier serviceTier) float64 { return bcTotal(computeSpeechCost(pricing, usage, audioSeconds, audioTextInputChars, tier)) } @@ -5368,6 +5385,63 @@ func TestCalculateCost_RerankPerQuery(t *testing.T) { }) } +// ========================================================================= +// computeDecisionCost — unit tests +// ========================================================================= + +func TestComputeDecisionCost_InputOnlyBilling(t *testing.T) { + // Typesafe's jev pricing shape: input tokens billed, output tokens free. + p := configstoreTables.TableModelPricing{ + InputCostPerToken: bifrost.Ptr(0.000000042), + OutputCostPerToken: bifrost.Ptr(0.0), + } + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 50000, + CompletionTokens: 40, + TotalTokens: 50040, + } + cost := computeDecisionCostTotal(&p, usage, serviceTier{}) + assert.InDelta(t, 50000*0.000000042, cost, 1e-12) +} + +func TestComputeDecisionCost_OutputRateHonored(t *testing.T) { + p := configstoreTables.TableModelPricing{ + InputCostPerToken: bifrost.Ptr(0.000001), + OutputCostPerToken: bifrost.Ptr(0.000002), + } + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 1000, + CompletionTokens: 500, + TotalTokens: 1500, + } + cost := computeDecisionCostTotal(&p, usage, serviceTier{}) + assert.InDelta(t, 1000*0.000001+500*0.000002, cost, 1e-12) +} + +func TestComputeDecisionCost_NilUsage(t *testing.T) { + p := configstoreTables.TableModelPricing{InputCostPerToken: new(0.000000042)} + assert.Equal(t, 0.0, computeDecisionCostTotal(&p, nil, serviceTier{})) +} + +func TestCalculateCost_DecisionTokenBilling(t *testing.T) { + // Pins the dispatch arm: an unhandled request type silently falls into + // CalculateCost's default nil branch and every decision would cost zero. + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ + makeKey("jev-1.13.0", "typesafe", "decisions"): { + Model: "jev-1.13.0", Provider: "typesafe", Mode: "decisions", + InputCostPerToken: bifrost.Ptr(0.000000042), + OutputCostPerToken: bifrost.Ptr(0.0), + }, + }) + + resp := makeDecisionResponse(schemas.Typesafe, "jev-1.13.0", &schemas.BifrostLLMUsage{ + PromptTokens: 50000, + TotalTokens: 50000, + }) + + assert.InDelta(t, 50000*0.000000042, s.CalculateCost(resp, nil), 1e-12) +} + func TestCalculateCost_RerankPerTokenStillWorks(t *testing.T) { // Voyage and Jina style rerankers bill per token and carry no per-query rate; the two // pricing shapes must not interfere. diff --git a/framework/modelcatalog/datasheet/store.go b/framework/modelcatalog/datasheet/store.go index b8a54d5d6c3..2166f04f8ae 100644 --- a/framework/modelcatalog/datasheet/store.go +++ b/framework/modelcatalog/datasheet/store.go @@ -184,6 +184,7 @@ func (s *Store) GetPricingEntryForModel(model string, provider schemas.ModelProv schemas.ResponsesRequest, schemas.EmbeddingRequest, schemas.RerankRequest, + schemas.DecisionRequest, schemas.SpeechRequest, schemas.TranscriptionRequest, schemas.ImageGenerationRequest, diff --git a/framework/modelcatalog/datasheet/types.go b/framework/modelcatalog/datasheet/types.go index 953ac7720b9..39ec08e1970 100644 --- a/framework/modelcatalog/datasheet/types.go +++ b/framework/modelcatalog/datasheet/types.go @@ -445,6 +445,8 @@ func normalizeRequestType(reqType schemas.RequestType) string { return "embedding" case schemas.RerankRequest: return "rerank" + case schemas.DecisionRequest: + return "decisions" case schemas.SpeechRequest, schemas.SpeechStreamRequest: return "audio_speech" case schemas.TranscriptionRequest, schemas.TranscriptionStreamRequest: diff --git a/plugins/logging/main.go b/plugins/logging/main.go index 6886055dca9..456cc09a01b 100644 --- a/plugins/logging/main.go +++ b/plugins/logging/main.go @@ -1642,6 +1642,8 @@ func (p *LoggerPlugin) PreLLMHook(ctx *schemas.BifrostContext, req *schemas.Bifr initialData.Params = req.EmbeddingRequest.Params case schemas.RerankRequest: initialData.Params = req.RerankRequest.Params + case schemas.DecisionRequest: + initialData.Params = req.DecisionRequest.Questions case schemas.OCRRequest: initialData.Params = req.OCRRequest.Params initialData.OCRInput = &req.OCRRequest.Document diff --git a/plugins/logging/operations.go b/plugins/logging/operations.go index e575a7cad87..f3ed5b667fd 100644 --- a/plugins/logging/operations.go +++ b/plugins/logging/operations.go @@ -590,6 +590,10 @@ func (p *LoggerPlugin) applyNonStreamingOutputToEntry(entry *logstore.Log, resul usage = result.CompactionResponse.Usage.ToBifrostLLMUsage() case result.EmbeddingResponse != nil && result.EmbeddingResponse.Usage != nil: usage = result.EmbeddingResponse.Usage + case result.RerankResponse != nil && result.RerankResponse.Usage != nil: + usage = result.RerankResponse.Usage + case result.DecisionResponse != nil && result.DecisionResponse.Usage != nil: + usage = result.DecisionResponse.Usage case result.TranscriptionResponse != nil && result.TranscriptionResponse.Usage != nil: usage = &schemas.BifrostLLMUsage{} if result.TranscriptionResponse.Usage.InputTokens != nil { @@ -704,6 +708,17 @@ func (p *LoggerPlugin) applyNonStreamingOutputToEntry(entry *logstore.Log, resul if result.RerankResponse != nil && len(result.RerankResponse.Results) > 0 { entry.RerankOutputParsed = result.RerankResponse.Results } + if result.DecisionResponse != nil && len(result.DecisionResponse.Answers) > 0 { + if answersJSON, err := sonic.Marshal(result.DecisionResponse.Answers); err == nil { + answers := string(answersJSON) + entry.OutputMessageParsed = &schemas.ChatMessage{ + Role: schemas.ChatMessageRoleAssistant, + Content: &schemas.ChatMessageContent{ + ContentStr: &answers, + }, + } + } + } if result.OCRResponse != nil { entry.OCROutputParsed = result.OCRResponse } @@ -2170,6 +2185,13 @@ func buildResponseForRequestType(requestType schemas.RequestType, usage *schemas ExtraFields: extra, }, } + case schemas.DecisionRequest: + return &schemas.BifrostResponse{ + DecisionResponse: &schemas.BifrostDecisionResponse{ + Usage: usage, + ExtraFields: extra, + }, + } case schemas.OCRRequest: return &schemas.BifrostResponse{ OCRResponse: &schemas.BifrostOCRResponse{ diff --git a/plugins/logging/utils.go b/plugins/logging/utils.go index 6e3e780838f..33f182846db 100644 --- a/plugins/logging/utils.go +++ b/plugins/logging/utils.go @@ -693,6 +693,22 @@ func (p *LoggerPlugin) extractInputHistory(request *schemas.BifrostRequest) ([]s }, }, []schemas.ResponsesMessage{} } + if request.DecisionRequest != nil { + var state string + if s, ok := request.DecisionRequest.State.(string); ok { + state = s + } else if raw, err := sonic.Marshal(request.DecisionRequest.State); err == nil { + state = string(raw) + } + return []schemas.ChatMessage{ + { + Role: schemas.ChatMessageRoleUser, + Content: &schemas.ChatMessageContent{ + ContentStr: &state, + }, + }, + }, []schemas.ResponsesMessage{} + } if request.RerankRequest != nil { query := request.RerankRequest.Query return []schemas.ChatMessage{ diff --git a/tests/cmd/e2eseed/go.mod b/tests/cmd/e2eseed/go.mod index fe6d293cf9f..a4c94f2d46f 100644 --- a/tests/cmd/e2eseed/go.mod +++ b/tests/cmd/e2eseed/go.mod @@ -86,7 +86,7 @@ require ( github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.24 // indirect github.com/mattn/go-sqlite3 v1.14.32 // indirect - github.com/maximhq/bifrost/core v1.8.6 // indirect + github.com/maximhq/bifrost/core v1.9.1 // indirect github.com/maximhq/bifrost/framework v1.3.16 // indirect github.com/molecule-man/go-brrr v1.0.1 // indirect github.com/paulmach/orb v0.11.1 // indirect diff --git a/tests/cmd/seed/go.mod b/tests/cmd/seed/go.mod index 2e9a856d7c3..cd3b00e67b1 100644 --- a/tests/cmd/seed/go.mod +++ b/tests/cmd/seed/go.mod @@ -8,7 +8,7 @@ replace ( ) require ( - github.com/maximhq/bifrost/core v1.8.6 + github.com/maximhq/bifrost/core v1.9.1 github.com/maximhq/bifrost/framework v1.3.16 gorm.io/driver/postgres v1.6.0 gorm.io/driver/sqlite v1.6.0 diff --git a/tests/cmd/seedvks/go.mod b/tests/cmd/seedvks/go.mod index 932006e6c75..1f83dc35153 100644 --- a/tests/cmd/seedvks/go.mod +++ b/tests/cmd/seedvks/go.mod @@ -9,7 +9,7 @@ replace ( require ( github.com/google/uuid v1.6.0 - github.com/maximhq/bifrost/core v1.8.6 + github.com/maximhq/bifrost/core v1.9.1 github.com/maximhq/bifrost/framework v1.3.16 gorm.io/driver/postgres v1.6.0 gorm.io/gorm v1.31.1 diff --git a/tests/e2e/api/collections/provider-harness.json b/tests/e2e/api/collections/provider-harness.json index 3f45fd78224..3a3ec621eef 100644 --- a/tests/e2e/api/collections/provider-harness.json +++ b/tests/e2e/api/collections/provider-harness.json @@ -158192,6 +158192,267 @@ } } ] + }, + { + "name": "105. Typesafe decisions provider (typesafe-decisions)", + "item": [ + { + "name": "/v1/decisions - mixed kinds return typed answers - typesafe/jev-1.13.0", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "pm.test('every question produced a typed answer', function () {", + " pm.expect(body.answers, 'missing answers: ' + pm.response.text()).to.be.an('object');", + " pm.expect(body.answers.is_frustrated.kind).to.eql('noul');", + " pm.expect(body.answers.is_frustrated.value).to.be.a('number');", + " pm.expect(body.answers.is_frustrated.value).to.be.at.least(0);", + " pm.expect(body.answers.is_frustrated.value).to.be.at.most(1);", + " pm.expect(body.answers.category.kind).to.eql('choice');", + " pm.expect(['billing', 'bug', 'other']).to.include(body.answers.category.value);", + " pm.expect(body.answers.urgency.kind).to.eql('score');", + " pm.expect(body.answers.urgency.value).to.be.a('number');", + "});", + "pm.test('usage is normalized', function () {", + " pm.expect(body.usage.prompt_tokens, 'usage: ' + JSON.stringify(body.usage)).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"typesafe/jev-1.13.0\",\n \"state\": \"Customer message: I was double charged last month and nobody replied to my two emails. I want a refund today or I am cancelling.\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges and refunds\",\n \"bug\": \"product defects\",\n \"other\": \"anything else\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"Rate how urgently this needs a human reply\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered soon\",\n \"needs a reply today\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - structured object, array, and null criteria accepted - typesafe/jev-1.13.0", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('every allowed criteria type is accepted, not rejected as non-string', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "pm.test('every question still produced a typed answer', function () {", + " pm.expect(body.answers, 'missing answers: ' + pm.response.text()).to.be.an('object');", + " ['is_frustrated', 'asks_refund', 'threatens_cancel'].forEach(function (name) {", + " pm.expect(body.answers[name].kind, name).to.eql('noul');", + " pm.expect(body.answers[name].value, name).to.be.at.least(0);", + " pm.expect(body.answers[name].value, name).to.be.at.most(1);", + " });", + " pm.expect(body.answers.category.kind).to.eql('choice');", + " pm.expect(['billing', 'bug', 'support', 'other']).to.include(body.answers.category.value);", + " pm.expect(body.answers.category.probabilities, 'choice probabilities are a required response field').to.be.an('object');", + " pm.expect(body.answers.urgency.kind).to.eql('score');", + " pm.expect(body.answers.urgency.value).to.be.a('number');", + " // The legend echoes each level's description verbatim - structured", + " // levels come back as objects/arrays, not stringified.", + " pm.expect(body.answers.urgency.legend['0']).to.eql('can wait a week');", + " pm.expect(body.answers.urgency.legend['1']).to.deep.eql({level: 'should be answered soon'});", + " pm.expect(body.answers.urgency.legend['2']).to.deep.eql(['needs a reply today', 'churn risk']);", + " pm.expect(body.answers.urgency.probabilities, 'score probabilities are a required response field').to.be.an('object');", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"typesafe/jev-1.13.0\",\n \"state\": \"Customer message: I was double charged last month and nobody replied to my two emails. I want a refund today or I am cancelling.\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\",\n \"criteria\": {\n \"true\": {\n \"meaning\": \"clearly upset\",\n \"signals\": [\n \"threats to cancel\",\n \"repeated contact\"\n ]\n },\n \"false\": [\n \"calm\",\n \"neutral tone\"\n ]\n }\n },\n \"asks_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\",\n \"criteria\": {\n \"true\": [\n \"asks for money back\",\n \"mentions refund\"\n ],\n \"false\": {\n \"meaning\": \"no refund language\"\n }\n }\n },\n \"threatens_cancel\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel?\",\n \"criteria\": {\n \"true\": \"cancellation is threatened\",\n \"false\": \"no cancellation language\"\n }\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": {\n \"rubric\": \"charges and refunds\",\n \"examples\": [\n \"double charge\",\n \"refund request\"\n ]\n },\n \"bug\": [\n \"crash\",\n \"product defect\"\n ],\n \"support\": \"service questions\",\n \"other\": null\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"Rate how urgently this needs a human reply\",\n \"criteria\": [\n \"can wait a week\",\n {\n \"level\": \"should be answered soon\"\n },\n [\n \"needs a reply today\",\n \"churn risk\"\n ]\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - unsupported kind rejected with 400 - typesafe/jev-1.13.0", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('unsupported kind is rejected, not silently approximated', function () {", + " pm.expect(pm.response.code, 'expected 400, got: ' + pm.response.text()).to.eql(400);", + " pm.expect(pm.response.text()).to.include('unsupported kind');", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"typesafe/jev-1.13.0\",\n \"state\": \"Customer message: I was double charged last month and nobody replied to my two emails. I want a refund today or I am cancelling.\",\n \"questions\": {\n \"plain\": {\n \"kind\": \"ranking\",\n \"instructions\": \"rank it\"\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/typesafe/v1/systemone - native questions round trip - jev-1.13.0", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('native systemone succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "pm.test('all three native answer types come back', function () {", + " pm.expect(body.answers.is_frustrated.type).to.eql('noul');", + " pm.expect(body.answers.is_frustrated.noul).to.be.a('number');", + " pm.expect(body.answers.category.type).to.eql('choice');", + " pm.expect(['billing', 'bug', 'other']).to.include(body.answers.category.choice);", + " pm.expect(body.answers.urgency.type).to.eql('score');", + " pm.expect(body.answers.urgency.score).to.be.a('number');", + "});", + "pm.test('native model and usage shape retained', function () {", + " pm.expect(body.model, 'model: ' + body.model).to.be.a('string');", + " pm.expect(body.model).to.not.include('typesafe/');", + " pm.expect(body.usage.input_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/typesafe/v1/systemone", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "typesafe", + "v1", + "systemone" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"state\": \"Customer message: I was double charged last month and nobody replied to my two emails. I want a refund today or I am cancelling.\",\n \"model\": \"jev-1.13.0\",\n \"questions\": {\n \"is_frustrated\": {\n \"type\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"category\": {\n \"type\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges and refunds\",\n \"bug\": \"product defects\",\n \"other\": \"anything else\"\n }\n },\n \"urgency\": {\n \"type\": \"score\",\n \"instructions\": \"Rate how urgently this needs a human reply\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered soon\",\n \"needs a reply today\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/typesafe/v1/models - native catalog shape - jev models", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('models list succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "pm.test('native shape lists bare jev model names', function () {", + " pm.expect(body.models, 'body: ' + pm.response.text()).to.be.an('array');", + " var names = body.models.map(function (m) { return m.name; });", + " pm.expect(names).to.include('jev-1.13.0');", + " pm.expect(names).to.include('jev-latest');", + " names.forEach(function (n) { pm.expect(n).to.not.include('typesafe/'); });", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/typesafe/v1/models", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "typesafe", + "v1", + "models" + ] + } + }, + "response": [] + } + ] } ] } diff --git a/tests/e2e/api/runners/filter-collection.mjs b/tests/e2e/api/runners/filter-collection.mjs index 095f48ff7db..a107b6fd5f3 100644 --- a/tests/e2e/api/runners/filter-collection.mjs +++ b/tests/e2e/api/runners/filter-collection.mjs @@ -152,6 +152,7 @@ const PROVIDER_KEYWORDS = { xai: ["xai", "grok"], replicate: ["replicate", "/replicate", "flux", "black-forest-labs"], runware: ["runware", "runware/"], + typesafe: ["typesafe", "/typesafe", "jev-"], }; // Haystack = item JSON + ancestor folder names. Folder names encode the harness diff --git a/tests/integrations/python/config.json b/tests/integrations/python/config.json index 6f00d7a2ddf..e3b9362cb26 100644 --- a/tests/integrations/python/config.json +++ b/tests/integrations/python/config.json @@ -40,6 +40,20 @@ "default_request_timeout_in_seconds": 300 } }, + "typesafe": { + "keys": [ + { + "name": "Typesafe API Key", + "value": "env.TYPESAFE_API_KEY", + "weight": 1, + "models": ["*"], + "use_for_batch_api": false + } + ], + "network_config": { + "default_request_timeout_in_seconds": 300 + } + }, "xai": { "keys": [ { @@ -775,6 +789,12 @@ "key_ids": ["*"], "weight": 1.0 }, + { + "provider": "typesafe", + "allowed_models": ["*"], + "key_ids": ["*"], + "weight": 1.0 + }, { "provider": "elevenlabs", "allowed_models": ["*"], diff --git a/tests/integrations/python/pyproject.toml b/tests/integrations/python/pyproject.toml index 578cde3d1ff..9a09a98c367 100644 --- a/tests/integrations/python/pyproject.toml +++ b/tests/integrations/python/pyproject.toml @@ -51,6 +51,10 @@ dependencies = [ "pytest-xdist>=3.8.0", "pyasn1>=0.6.4", "griffe>=2.3.0", + # typesafe-sdk is newer than exclude-newer above by design (no version + # existed at the cutoff); pinned exact and grafted into uv.lock so no + # other package re-resolves. Its deps are already satisfied by the lock. + "typesafe-sdk==0.7.0", ] [build-system] @@ -146,5 +150,8 @@ exclude_lines = [ # above would otherwise make uv treat it as an editable package, which changes the # root's entry in uv.lock and breaks `uv sync --frozen`. package = false -# Include HTTPX2's security fixes released after the previous cutoff. +# typesafe-sdk (the exception above) is grafted into uv.lock; every other +# package stays at its pre-existing version, so this cutoff keeps the 7-day +# minimum for them. Integration tests run with --frozen so the lock is used +# as-is without re-resolving typesafe past this cutoff. exclude-newer = "2026-09-09T00:00:00Z" diff --git a/tests/integrations/python/tests/test_typesafe.py b/tests/integrations/python/tests/test_typesafe.py new file mode 100644 index 00000000000..04216233c04 --- /dev/null +++ b/tests/integrations/python/tests/test_typesafe.py @@ -0,0 +1,194 @@ +""" +Typesafe Integration Tests - Official SDK against Bifrost + +🌉 SDK DROP-IN TESTING: +This test suite uses the official TypeSafe Python SDK (typesafe-sdk) pointed at +Bifrost's /typesafe prefix via base_url, so a client written against +api.typesafe.ai must work unchanged through Bifrost. Every call in this file +goes through the SDK - no raw HTTP. + +Covered scenarios: +1. system_one with all three question types (Noul, Choice, Score) +2. Structured (JSON) state and per-answer metadata (confidence, probabilities, legend) +3. Model alias resolution (jev-latest resolves to the versioned model) +4. client.models.list() against Bifrost's synthesized native listing +5. SDK exception parsing of Bifrost's native error body (TypeSafeBadRequestError) +""" + +import pytest +from typesafe_sdk import ( + Choice, + Noul, + Score, + TypeSafeBadRequestError, + TypeSafeClient, +) + +from .utils.common import get_bifrost_base_url +from .utils.config_loader import get_config + +STATE = ( + "Customer message: I was double charged last month and nobody replied " + "to my two emails. I want a refund today or I am cancelling." +) + + +@pytest.fixture +def typesafe_client(): + """Official TypeSafe SDK client pointed at Bifrost's /typesafe drop-in. + + Authenticates to Bifrost with the suite's virtual key via the x-bf-vk + header (the cross-provider convention); Bifrost injects the real upstream + key, so the SDK's own api_key never reaches TypeSafe. + """ + config = get_config() + headers = {} + vk = config.get_virtual_key() if config.is_virtual_key_configured() else "" + if vk: + headers["x-bf-vk"] = vk + client = TypeSafeClient( + base_url=f"{get_bifrost_base_url()}/typesafe", + api_key=vk or "dummy-key-bifrost-injects-the-real-one", + headers=headers or None, + ) + yield client + client.close() + + +class TestTypesafeSystemOne: + def test_01_all_question_types(self, typesafe_client): + result = typesafe_client.system_one( + STATE, + { + "is_frustrated": Noul(instructions="Is the customer frustrated?"), + "category": Choice( + instructions="Pick the ticket category", + criteria={"billing": "charges and refunds", "bug": "product defects", "other": "anything else"}, + ), + "urgency": Score( + instructions="Rate how urgently this needs a human reply", + criteria=["can wait a week", "should be answered soon", "needs a reply today"], + ), + }, + model="jev-1.13.0", + ) + + assert result.model == "jev-1.13.0" + assert 0.0 <= result.nouls["is_frustrated"].noul <= 1.0 + assert result.choices["category"].choice in {"billing", "bug", "other"} + assert isinstance(result.scores["urgency"].score, (int, float)) + assert result.usage.input_tokens > 0 + + def test_02_structured_state_and_answer_metadata(self, typesafe_client): + result = typesafe_client.system_one( + {"ticket": {"id": 4211, "body": "The export button crashes the app every time."}, "user_tier": "pro"}, + { + "area": Choice( + instructions="Which product area does the complaint target?", + criteria={"camera": "capture", "stability": "crashes", "support": "service"}, + ), + "priority": Score( + instructions="Bug backlog rank?", + criteria=["backlog", "next sprint", "this sprint", "hotfix now"], + ), + }, + model="jev-1.13.0", + ) + + area = result.choices["area"] + assert area.choice in {"camera", "stability", "support"} + assert area.probabilities is not None and abs(sum(area.probabilities.values()) - 1.0) < 0.05 + priority = result.scores["priority"] + assert priority.legend is not None and len(priority.legend) == 4 + + def test_03_model_alias_resolves(self, typesafe_client): + result = typesafe_client.system_one( + "Reply: Sure, sounds good, see you at 3pm.", + {"is_confirmation": Noul(instructions="Does this reply confirm the meeting?")}, + model="jev-latest", + ) + # Aliases resolve upstream; the response reports the versioned model. + assert result.model.startswith("jev-") + assert result.model != "jev-latest" + + def test_04_structured_criteria_type_matrix(self, typesafe_client): + # The API types criteria descriptions as string | object | array for + # noul keys and score levels, plus null for choice options. One call + # covers every allowed type in every slot; Bifrost must forward all of + # them losslessly instead of rejecting non-string descriptions. + result = typesafe_client.system_one( + STATE, + { + "noul_obj_arr": Noul( + instructions="Is the customer frustrated?", + criteria={ + "true": {"meaning": "clearly upset", "signals": ["threats", "caps"]}, + "false": ["calm", "neutral tone"], + }, + ), + "noul_arr_obj": Noul( + instructions="Does the customer ask for a refund?", + criteria={ + "true": ["asks for money back", "mentions refund"], + "false": {"meaning": "no refund language"}, + }, + ), + "noul_str": Noul( + instructions="Does the customer threaten to cancel?", + criteria={"true": "cancellation is threatened", "false": "no cancellation language"}, + ), + "category": Choice( + instructions="Pick the ticket category", + criteria={ + "billing": {"rubric": "charges and refunds", "examples": ["double charge"]}, + "bug": ["crash", "product defect"], + "support": "service questions", + "other": None, + }, + ), + "urgency": Score( + instructions="Rate how urgently this needs a human reply", + criteria=[ + "can wait a week", + {"level": "should be answered soon"}, + ["needs a reply today", "churn risk"], + ], + ), + }, + model="jev-1.13.0", + ) + + for name in ("noul_obj_arr", "noul_arr_obj", "noul_str"): + assert 0.0 <= result.nouls[name].noul <= 1.0 + assert result.choices["category"].choice in {"billing", "bug", "support", "other"} + urgency = result.scores["urgency"] + assert isinstance(urgency.score, (int, float)) + # The legend echoes each level's description verbatim - structured + # levels come back as objects/arrays, not stringified. + assert urgency.legend is not None and len(urgency.legend) == 3 + assert urgency.legend[0] == "can wait a week" + assert urgency.legend[1] == {"level": "should be answered soon"} + assert urgency.legend[2] == ["needs a reply today", "churn risk"] + + +class TestTypesafeModels: + def test_01_models_list(self, typesafe_client): + listing = typesafe_client.models.list() + names = [m.name for m in listing.models] + assert "jev-1.13.0" in names + assert "jev-latest" in names + assert all("typesafe/" not in name for name in names) + + +class TestTypesafeErrors: + def test_01_bad_request_parses_native_error(self, typesafe_client): + # A choice question with empty criteria is rejected by Bifrost before + # dispatch; the SDK must parse the native {"detail": {...}} body into + # its 400 exception type exactly as it would against api.typesafe.ai. + with pytest.raises(TypeSafeBadRequestError) as excinfo: + typesafe_client.system_one( + STATE, + {"category": Choice(instructions="Pick one", criteria={})}, + model="jev-1.13.0", + ) + assert "criteria" in str(excinfo.value) diff --git a/tests/integrations/python/uv.lock b/tests/integrations/python/uv.lock index ad54b405a58..fe4e1d46e26 100644 --- a/tests/integrations/python/uv.lock +++ b/tests/integrations/python/uv.lock @@ -782,6 +782,7 @@ dependencies = [ { name = "requests" }, { name = "tokenizers" }, { name = "types-requests" }, + { name = "typesafe-sdk" }, { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d2/75/4c346f6e2322e545f8452692304bd4eca15a2a0209ab9af6a0d1a7810b67/cohere-5.21.1.tar.gz", hash = "sha256:e5ade4423b928b01ff2038980e1b62b2a5bb412c8ab83e30882753b810a5509f", size = 191272, upload-time = "2026-03-26T15:09:27.857Z" } @@ -5061,6 +5062,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/e7/010c87f559e216d83f9dc51e939633fd0d0ead3377340181ab0e223cd3b5/types_requests-2.33.0.20260712-py3-none-any.whl", hash = "sha256:de027e28c171d3da529689cbfa023b0b4eab188c8dfa22fd834eebd2cee6e7bb", size = 21392, upload-time = "2026-07-12T05:14:19.616Z" }, ] +[[package]] +name = "typesafe-sdk" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx2" }, + { name = "pydantic" }, + { name = "pydantic-core" }, + { name = "tenacity" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/e2/ac317772d4d5cfabf3cecdeacd83cc838523e240bb4e1cf256837e6005bb/typesafe_sdk-0.7.0.tar.gz", hash = "sha256:930d42fd73cfed6f25bccae488ce0f30f6043eaedd8682ef59734e79ab94a876", size = 22235 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/2a/16f4163e6d8827ef80b2ed7a400c80688e1ebf571c10f143f792bdafc654/typesafe_sdk-0.7.0-py3-none-any.whl", hash = "sha256:c6d2257c4b04b8d4d81cff2b55489b4188795fd44fb40b9cef26ff243b2d1080", size = 35352 }, +] + [[package]] name = "typing-extensions" version = "4.16.0" diff --git a/tests/integrations/typescript/config.json b/tests/integrations/typescript/config.json index 29a13b032da..ac779fbac1f 100644 --- a/tests/integrations/typescript/config.json +++ b/tests/integrations/typescript/config.json @@ -15,6 +15,20 @@ "default_request_timeout_in_seconds": 300 } }, + "typesafe": { + "keys": [ + { + "name": "Typesafe API Key", + "value": "env.TYPESAFE_API_KEY", + "weight": 1, + "models": ["*"], + "use_for_batch_api": false + } + ], + "network_config": { + "default_request_timeout_in_seconds": 300 + } + }, "anthropic": { "keys": [ { @@ -205,6 +219,7 @@ "is_active": true, "provider_configs": [ { "provider": "openai", "allowed_models": ["*"], "key_ids": ["*"], "weight": 1.0 }, + { "provider": "typesafe", "allowed_models": ["*"], "key_ids": ["*"], "weight": 1.0 }, { "provider": "anthropic", "allowed_models": ["*"], "key_ids": ["*"], "weight": 1.0 }, { "provider": "gemini", "allowed_models": ["*"], "key_ids": ["*"], "weight": 1.0 }, { "provider": "vertex", "allowed_models": ["*"], "key_ids": ["*"], "weight": 1.0 }, diff --git a/tests/integrations/typescript/package-lock.json b/tests/integrations/typescript/package-lock.json index 561246239c9..d37ce533d68 100644 --- a/tests/integrations/typescript/package-lock.json +++ b/tests/integrations/typescript/package-lock.json @@ -17,6 +17,7 @@ "@langchain/google-genai": "2.1.26", "@langchain/openai": "1.4.4", "@smithy/node-http-handler": "4.7.4", + "@typesafe-ai/sdk": "^0.6.0", "langsmith": "0.6.0", "openai": "6.15.0", "yaml": "2.9.0", @@ -2440,6 +2441,15 @@ "undici-types": "~6.20.0" } }, + "node_modules/@typesafe-ai/sdk": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@typesafe-ai/sdk/-/sdk-0.6.0.tgz", + "integrity": "sha512-IddX+Q0XM+VagOUZFeP7wZjaO4SHMdvnh2zEBdrZZnXedWI3BNK1lKhMx3ayrkFWvVLbVcUHJy6AVZlY+e6Jaw==", + "license": "MIT", + "engines": { + "node": ">=20" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.0.0.tgz", diff --git a/tests/integrations/typescript/package.json b/tests/integrations/typescript/package.json index 39c53a59471..dd369e9fb1c 100644 --- a/tests/integrations/typescript/package.json +++ b/tests/integrations/typescript/package.json @@ -21,6 +21,7 @@ "@langchain/google-genai": "2.1.26", "@langchain/openai": "1.4.4", "@smithy/node-http-handler": "4.7.4", + "@typesafe-ai/sdk": "0.6.0", "langsmith": "0.6.0", "openai": "6.15.0", "yaml": "2.9.0", @@ -41,14 +42,14 @@ "ast-v8-to-istanbul": "1.0.0", "brace-expansion@1": "1.1.18", "brace-expansion@2": "2.1.4", + "esbuild": "0.28.1", "js-yaml": "4.3.2", "nanoid": "3.3.18", "obug": "2.1.4", "rollup": "4.60.4", - "esbuild": "0.28.1", "vite": "6.4.3" }, "engines": { "node": ">=22.0.0" } -} +} \ No newline at end of file diff --git a/tests/integrations/typescript/tests/test-typesafe.test.ts b/tests/integrations/typescript/tests/test-typesafe.test.ts new file mode 100644 index 00000000000..e2c825cce2a --- /dev/null +++ b/tests/integrations/typescript/tests/test-typesafe.test.ts @@ -0,0 +1,143 @@ +/** + * Typesafe Integration Tests - Official SDK against Bifrost + * + * 🌉 SDK DROP-IN TESTING: + * Uses the official TypeSafe JavaScript SDK (@typesafe-ai/sdk) pointed at + * Bifrost's /typesafe prefix via baseURL, authenticating to Bifrost with the + * suite's virtual key via the x-bf-vk header. Every call goes through the SDK. + */ + +import { BadRequestError, choice, noul, score, TypeSafeClient } from "@typesafe-ai/sdk"; +import { beforeAll, describe, expect, it } from "vitest"; +import { getVirtualKey, isVirtualKeyConfigured } from "../src/utils/config-loader"; + +const STATE = + "Customer message: I was double charged last month and nobody replied to my two emails. I want a refund today or I am cancelling."; + +let client: TypeSafeClient; + +beforeAll(() => { + const baseUrl = process.env.BIFROST_BASE_URL || "http://localhost:8080"; + const vk = isVirtualKeyConfigured() ? getVirtualKey() : ""; + client = new TypeSafeClient({ + baseURL: `${baseUrl}/typesafe`, + apiKey: vk || "dummy-key-bifrost-injects-the-real-one", + defaultHeaders: vk ? { "x-bf-vk": vk } : undefined, + }); +}); + +describe("Typesafe SDK systemOne", () => { + it("answers all three question types", async () => { + const response = await client.systemOne({ + state: STATE, + model: "jev-1.13.0", + questions: { + is_frustrated: noul("Is the customer frustrated?"), + category: choice("Pick the ticket category", { + billing: "charges and refunds", + bug: "product defects", + other: "anything else", + }), + urgency: score("Rate how urgently this needs a human reply", [ + "can wait a week", + "should be answered soon", + "needs a reply today", + ]), + }, + }); + + expect(response.model).toBe("jev-1.13.0"); + expect(response.answers.is_frustrated.noul).toBeGreaterThanOrEqual(0); + expect(response.answers.is_frustrated.noul).toBeLessThanOrEqual(1); + expect(["billing", "bug", "other"]).toContain(response.answers.category.choice); + expect(typeof response.answers.urgency.score).toBe("number"); + expect(response.usage.input_tokens).toBeGreaterThan(0); + }, 60000); + + it("resolves a model alias to its versioned id", async () => { + const response = await client.systemOne({ + state: "Reply: Sure, sounds good, see you at 3pm.", + model: "jev-latest", + questions: { is_confirmation: noul("Does this reply confirm the meeting?") }, + }); + expect(response.model).toMatch(/^jev-/); + expect(response.model).not.toBe("jev-latest"); + }, 60000); + + it("forwards structured criteria of every allowed type", async () => { + // The API types criteria descriptions as string | object | array for + // noul keys and score levels, plus null for choice options. One call + // covers every allowed type in every slot; Bifrost must forward all of + // them losslessly instead of rejecting non-string descriptions. + const response = await client.systemOne({ + state: STATE, + model: "jev-1.13.0", + questions: { + noul_obj_arr: noul("Is the customer frustrated?", { + true: { meaning: "clearly upset", signals: ["threats", "caps"] }, + false: ["calm", "neutral tone"], + }), + noul_arr_obj: noul("Does the customer ask for a refund?", { + true: ["asks for money back", "mentions refund"], + false: { meaning: "no refund language" }, + }), + noul_str: noul("Does the customer threaten to cancel?", { + true: "cancellation is threatened", + false: "no cancellation language", + }), + category: choice("Pick the ticket category", { + billing: { rubric: "charges and refunds", examples: ["double charge"] }, + bug: ["crash", "product defect"], + support: "service questions", + other: null, + }), + urgency: score("Rate how urgently this needs a human reply", [ + "can wait a week", + { level: "should be answered soon" }, + ["needs a reply today", "churn risk"], + ]), + }, + }); + + for (const name of ["noul_obj_arr", "noul_arr_obj", "noul_str"] as const) { + expect(response.answers[name].noul).toBeGreaterThanOrEqual(0); + expect(response.answers[name].noul).toBeLessThanOrEqual(1); + } + expect(["billing", "bug", "support", "other"]).toContain(response.answers.category.choice); + expect(typeof response.answers.urgency.score).toBe("number"); + // The legend echoes each level's description verbatim - structured + // levels come back as objects/arrays, not stringified. + const legend = response.answers.urgency.legend as Record; + expect(Object.keys(legend ?? {})).toHaveLength(3); + expect(legend[0]).toBe("can wait a week"); + expect(legend[1]).toEqual({ level: "should be answered soon" }); + expect(legend[2]).toEqual(["needs a reply today", "churn risk"]); + }, 60000); +}); + +describe("Typesafe SDK models", () => { + it("lists jev models with bare names", async () => { + // The JS SDK returns the model array directly (unlike the Python SDK's + // wrapper object). + const listing = await client.models.list(); + const names = listing.map((m) => m.name); + expect(names).toContain("jev-1.13.0"); + expect(names).toContain("jev-latest"); + for (const name of names) expect(name).not.toContain("typesafe/"); + }, 30000); +}); + +describe("Typesafe SDK errors", () => { + it("parses Bifrost's native error body into a BadRequestError", async () => { + // A choice question with an empty criteria map is rejected before dispatch; + // the SDK must parse the native {"detail": {...}} body into its 400 type, + // exactly as it would against api.typesafe.ai. + await expect( + client.systemOne({ + state: STATE, + model: "jev-1.13.0", + questions: { category: choice("Pick one", {}) }, + }), + ).rejects.toBeInstanceOf(BadRequestError); + }, 30000); +}); diff --git a/transports/bifrost-http/handlers/inference.go b/transports/bifrost-http/handlers/inference.go index 19651a47ac2..6a9e602a0fe 100644 --- a/transports/bifrost-http/handlers/inference.go +++ b/transports/bifrost-http/handlers/inference.go @@ -221,6 +221,13 @@ var rerankParamsKnownFields = map[string]bool{ "next_token": true, } +var decisionParamsKnownFields = map[string]bool{ + "model": true, + "state": true, + "questions": true, + "fallbacks": true, +} + var ocrParamsKnownFields = map[string]bool{ "model": true, "id": true, @@ -573,6 +580,13 @@ type RerankRequest struct { *schemas.RerankParameters } +// DecisionHandlerRequest is a bifrost decision request +type DecisionHandlerRequest struct { + State interface{} `json:"state"` + Questions map[string]schemas.DecisionQuestion `json:"questions"` + BifrostParams +} + // OCRHandlerRequest is a bifrost OCR request type OCRHandlerRequest struct { ID *string `json:"id,omitempty"` @@ -720,6 +734,7 @@ var PathToTypeMapping = map[string]schemas.RequestType{ "/v1/responses": schemas.ResponsesRequest, "/v1/embeddings": schemas.EmbeddingRequest, "/v1/rerank": schemas.RerankRequest, + "/v1/decisions": schemas.DecisionRequest, "/v1/ocr": schemas.OCRRequest, "/v1/audio/speech": schemas.SpeechRequest, "/v1/audio/transcriptions": schemas.TranscriptionRequest, @@ -775,6 +790,7 @@ func (h *CompletionHandler) RegisterRoutes(r *router.Router, middlewares ...sche r.GET("/v1/responses/{response_id}/input_items", lib.ChainMiddlewares(h.responsesInputItems, responsesInputItemsMW...)) r.POST("/v1/embeddings", lib.ChainMiddlewares(h.embeddings, baseMiddlewares...)) r.POST("/v1/rerank", lib.ChainMiddlewares(h.rerank, baseMiddlewares...)) + r.POST("/v1/decisions", lib.ChainMiddlewares(h.evaluation, baseMiddlewares...)) r.POST("/v1/ocr", lib.ChainMiddlewares(h.ocr, baseMiddlewares...)) // ElevenLabs sound-effect models also flow through /v1/audio/speech; the // provider routes them to /v1/sound-generation by model id, keeping SDK and @@ -1293,6 +1309,61 @@ func (h *CompletionHandler) rerank(ctx *fasthttp.RequestCtx) { SendJSON(ctx, resp) } +// prepareDecisionRequest prepares a BifrostDecisionRequest from the HTTP request body +func prepareDecisionRequest(ctx *fasthttp.RequestCtx, config *lib.Config) (*DecisionHandlerRequest, *schemas.BifrostDecisionRequest, error) { + req, base, err := prepareRequest[DecisionHandlerRequest](ctx, config, decisionParamsKnownFields) + if err != nil { + return nil, nil, err + } + if req.State == nil { + return nil, nil, fmt.Errorf("state is required for decision") + } + if len(req.Questions) == 0 { + return nil, nil, fmt.Errorf("questions are required for decision") + } + return req, &schemas.BifrostDecisionRequest{ + Provider: base.Provider, + Model: base.ModelName, + State: req.State, + Questions: req.Questions, + Fallbacks: base.Fallbacks, + }, nil +} + +// evaluation handles POST /v1/decisions - Process decision requests +func (h *CompletionHandler) evaluation(ctx *fasthttp.RequestCtx) { + _, bifrostDecisionReq, err := prepareDecisionRequest(ctx, h.config) + if err != nil { + SendError(ctx, fasthttp.StatusBadRequest, err.Error()) + return + } + + // Convert context + bifrostCtx, cancel := lib.ConvertToBifrostContext(ctx, h.config) + defer cancel() + if bifrostCtx == nil { + SendError(ctx, fasthttp.StatusBadRequest, "Failed to convert context") + return + } + + resp, bifrostErr := h.client.DecisionRequest(bifrostCtx, bifrostDecisionReq) + if bifrostErr != nil { + forwardProviderHeadersFromContext(ctx, bifrostCtx) + SendBifrostError(ctx, bifrostErr) + return + } + + if resp != nil { + lib.ApplyBifrostResponseHeaders(ctx, bifrostCtx, resp.ExtraFields) + } + + if streamLargeResponseIfActive(ctx, bifrostCtx) { + return + } + // Send successful response + SendJSON(ctx, resp) +} + // prepareOCRRequest prepares a BifrostOCRRequest from the HTTP request body func prepareOCRRequest(ctx *fasthttp.RequestCtx, config *lib.Config) (*OCRHandlerRequest, *schemas.BifrostOCRRequest, error) { req, base, err := prepareRequest[OCRHandlerRequest](ctx, config, ocrParamsKnownFields) diff --git a/transports/bifrost-http/handlers/integrations.go b/transports/bifrost-http/handlers/integrations.go index bb4580ab417..bc4768f8c1e 100644 --- a/transports/bifrost-http/handlers/integrations.go +++ b/transports/bifrost-http/handlers/integrations.go @@ -39,6 +39,7 @@ func NewIntegrationHandler(client *bifrost.Bifrost, handlerStore lib.HandlerStor integrations.NewLangChainRouter(client, handlerStore, accessResolver, logger), integrations.NewPydanticAIRouter(client, handlerStore, accessResolver, logger), integrations.NewBedrockRouter(client, handlerStore, accessResolver, logger), + integrations.NewTypesafeRouter(client, handlerStore, accessResolver, logger), // passthrough routers integrations.NewGenAIPassthroughRouter(client, handlerStore, accessResolver, logger), integrations.NewChatGPTPassthroughRouter(client, handlerStore, accessResolver, logger), diff --git a/transports/bifrost-http/integrations/router.go b/transports/bifrost-http/integrations/router.go index 4984ad2634d..095737fd819 100644 --- a/transports/bifrost-http/integrations/router.go +++ b/transports/bifrost-http/integrations/router.go @@ -207,6 +207,10 @@ type EmbeddingResponseConverter func(ctx *schemas.BifrostContext, resp *schemas. // It takes a BifrostRerankResponse and returns the format expected by the specific integration. type RerankResponseConverter func(ctx *schemas.BifrostContext, resp *schemas.BifrostRerankResponse) (interface{}, error) +// DecisionResponseConverter is a function that converts BifrostDecisionResponse to integration-specific format. +// It takes a BifrostDecisionResponse and returns the format expected by the specific integration. +type DecisionResponseConverter func(ctx *schemas.BifrostContext, resp *schemas.BifrostDecisionResponse) (interface{}, error) + // OCRResponseConverter is a function that converts BifrostOCRResponse to integration-specific format. // It takes a BifrostOCRResponse and returns the format expected by the specific integration. type OCRResponseConverter func(ctx *schemas.BifrostContext, resp *schemas.BifrostOCRResponse) (interface{}, error) @@ -457,6 +461,7 @@ const ( RouteConfigTypeGenAI RouteConfigType = "genai" RouteConfigTypeBedrock RouteConfigType = "bedrock" RouteConfigTypeCohere RouteConfigType = "cohere" + RouteConfigTypeTypesafe RouteConfigType = "typesafe" ) // RouteConfig defines the configuration for a single route in an integration. @@ -487,6 +492,7 @@ type RouteConfig struct { AsyncResponsesResponseConverter AsyncResponsesResponseConverter // Function to convert AsyncJobResponse to integration format (SHOULD NOT BE NIL) EmbeddingResponseConverter EmbeddingResponseConverter // Function to convert BifrostEmbeddingResponse to integration format (SHOULD NOT BE NIL) RerankResponseConverter RerankResponseConverter // Function to convert BifrostRerankResponse to integration format + DecisionResponseConverter DecisionResponseConverter // Function to convert BifrostDecisionResponse to integration format OCRResponseConverter OCRResponseConverter // Function to convert BifrostOCRResponse to integration format SpeechResponseConverter SpeechResponseConverter // Function to convert BifrostSpeechResponse to integration format (SHOULD NOT BE NIL) TranscriptionResponseConverter TranscriptionResponseConverter // Function to convert BifrostTranscriptionResponse to integration format (SHOULD NOT BE NIL) @@ -1134,6 +1140,29 @@ func (g *GenericRouter) handleNonStreamingRequest(ctx *fasthttp.RequestCtx, conf response = rerankResponse } + case bifrostReq.DecisionRequest != nil: + decisionResponse, bifrostErr := g.client.DecisionRequest(bifrostCtx, bifrostReq.DecisionRequest) + if bifrostErr != nil { + g.sendError(ctx, bifrostCtx, config.ErrorConverter, bifrostErr) + return + } + if config.PostCallback != nil { + if err := config.PostCallback(ctx, req, decisionResponse); err != nil { + g.sendError(ctx, bifrostCtx, config.ErrorConverter, newBifrostError(err, "failed to execute post-request callback")) + return + } + } + if decisionResponse == nil { + g.sendError(ctx, bifrostCtx, config.ErrorConverter, newBifrostError(nil, "Bifrost response is nil after post-request callback")) + return + } + bifrostExtraFields = decisionResponse.ExtraFields + if config.DecisionResponseConverter != nil { + response, err = config.DecisionResponseConverter(bifrostCtx, decisionResponse) + } else { + response = decisionResponse + } + case bifrostReq.OCRRequest != nil: ocrResponse, bifrostErr := g.client.OCRRequest(bifrostCtx, bifrostReq.OCRRequest) if bifrostErr != nil { diff --git a/transports/bifrost-http/integrations/typesafe.go b/transports/bifrost-http/integrations/typesafe.go new file mode 100644 index 00000000000..96585d93c32 --- /dev/null +++ b/transports/bifrost-http/integrations/typesafe.go @@ -0,0 +1,97 @@ +package integrations + +import ( + "context" + "errors" + + bifrost "github.com/maximhq/bifrost/core" + "github.com/maximhq/bifrost/core/providers/typesafe" + "github.com/maximhq/bifrost/core/schemas" + "github.com/maximhq/bifrost/transports/bifrost-http/lib" + "github.com/valyala/fasthttp" +) + +// TypesafeRouter holds route registrations for Typesafe endpoints: the native +// System One evaluation endpoint and model listing. +type TypesafeRouter struct { + *GenericRouter +} + +// NewTypesafeRouter creates a new TypesafeRouter with the given bifrost client. +func NewTypesafeRouter(client *bifrost.Bifrost, handlerStore lib.HandlerStore, accessResolver AccessResolver, logger schemas.Logger) *TypesafeRouter { + return &TypesafeRouter{ + GenericRouter: NewGenericRouter(client, handlerStore, accessResolver, CreateTypesafeRouteConfigs("/typesafe"), nil, logger), + } +} + +// CreateTypesafeRouteConfigs creates route configurations for Typesafe API endpoints. +func CreateTypesafeRouteConfigs(pathPrefix string) []RouteConfig { + var routes []RouteConfig + + // Decision endpoint (v1/systemone) + routes = append(routes, RouteConfig{ + Type: RouteConfigTypeTypesafe, + Path: pathPrefix + "/v1/systemone", + Method: "POST", + GetHTTPRequestType: func(ctx *fasthttp.RequestCtx) schemas.RequestType { + return schemas.DecisionRequest + }, + GetRequestTypeInstance: func(ctx context.Context) interface{} { + return &typesafe.TypesafeDecisionRequest{} + }, + RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { + if typesafeReq, ok := req.(*typesafe.TypesafeDecisionRequest); ok { + decisionReq, err := typesafeReq.ToBifrostDecisionRequest(ctx) + if err != nil { + return nil, err + } + return &schemas.BifrostRequest{ + DecisionRequest: decisionReq, + }, nil + } + return nil, errors.New("invalid request type") + }, + DecisionResponseConverter: func(ctx *schemas.BifrostContext, resp *schemas.BifrostDecisionResponse) (interface{}, error) { + if resp.ExtraFields.Provider == schemas.Typesafe && resp.ExtraFields.RawResponse != nil { + return resp.ExtraFields.RawResponse, nil + } + return typesafe.ToTypesafeNativeDecisionResponse(resp) + }, + ErrorConverter: func(ctx *schemas.BifrostContext, err *schemas.BifrostError) interface{} { + return typesafe.ToTypesafeNativeError(err) + }, + }) + + // Models endpoint. Typesafe documents no upstream listing API; the response + // is synthesized from the provider's static catalog in native shape. + routes = append(routes, RouteConfig{ + Type: RouteConfigTypeTypesafe, + Path: pathPrefix + "/v1/models", + Method: "GET", + GetHTTPRequestType: func(ctx *fasthttp.RequestCtx) schemas.RequestType { + return schemas.ListModelsRequest + }, + GetRequestTypeInstance: func(ctx context.Context) interface{} { + return &schemas.BifrostListModelsRequest{} + }, + RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { + if listModelsReq, ok := req.(*schemas.BifrostListModelsRequest); ok { + if listModelsReq.Provider == "" { + listModelsReq.Provider = schemas.Typesafe + } + return &schemas.BifrostRequest{ + ListModelsRequest: listModelsReq, + }, nil + } + return nil, errors.New("invalid request type") + }, + ListModelsResponseConverter: func(ctx *schemas.BifrostContext, resp *schemas.BifrostListModelsResponse) (interface{}, error) { + return typesafe.ToTypesafeNativeListModelsResponse(resp), nil + }, + ErrorConverter: func(ctx *schemas.BifrostContext, err *schemas.BifrostError) interface{} { + return typesafe.ToTypesafeNativeError(err) + }, + }) + + return routes +} diff --git a/transports/bifrost-http/integrations/utils.go b/transports/bifrost-http/integrations/utils.go index 84479565525..8b8a09ab50b 100644 --- a/transports/bifrost-http/integrations/utils.go +++ b/transports/bifrost-http/integrations/utils.go @@ -368,6 +368,10 @@ func (g *GenericRouter) extractAndParseFallbacks(ctx *schemas.BifrostContext, re if bifrostReq.RerankRequest != nil { bifrostReq.RerankRequest.Fallbacks = parsedFallbacks } + case schemas.DecisionRequest: + if bifrostReq.DecisionRequest != nil { + bifrostReq.DecisionRequest.Fallbacks = parsedFallbacks + } case schemas.SpeechRequest, schemas.SpeechStreamRequest: if bifrostReq.SpeechRequest != nil { bifrostReq.SpeechRequest.Fallbacks = parsedFallbacks diff --git a/transports/config.schema.json b/transports/config.schema.json index 70ef43d3c77..e02ce73e03c 100644 --- a/transports/config.schema.json +++ b/transports/config.schema.json @@ -463,6 +463,9 @@ "wafer": { "$ref": "#/$defs/provider" }, + "typesafe": { + "$ref": "#/$defs/provider" + }, "vllm": { "$ref": "#/$defs/provider_with_vllm_config" }, @@ -8707,6 +8710,7 @@ "responses", "embedding", "rerank", + "decisions", "speech", "transcription", "image_generation", @@ -8767,7 +8771,8 @@ "runware", "fireworks", "sarvam", - "wafer" + "wafer", + "typesafe" ], "description": "Base provider type to extend" }, @@ -8831,6 +8836,9 @@ "rerank": { "type": "boolean" }, + "decisions": { + "type": "boolean" + }, "ocr": { "type": "boolean" }, diff --git a/ui/app/__error.tsx b/ui/app/__error.tsx index 3a1a9980975..9c2682ed728 100644 --- a/ui/app/__error.tsx +++ b/ui/app/__error.tsx @@ -30,4 +30,4 @@ export function ErrorComponent({ error }: Partial) {
); -} +} \ No newline at end of file diff --git a/ui/app/_fallbacks/enterprise/components/user-groups/sheets/customerDetailSheet.tsx b/ui/app/_fallbacks/enterprise/components/user-groups/sheets/customerDetailSheet.tsx index 888aa9c9d43..9411a70937b 100644 --- a/ui/app/_fallbacks/enterprise/components/user-groups/sheets/customerDetailSheet.tsx +++ b/ui/app/_fallbacks/enterprise/components/user-groups/sheets/customerDetailSheet.tsx @@ -138,12 +138,12 @@ export function CustomerDetailSheet({ customer, open, onOpenChange }: Props) { .sort((a, b) => (b.max_limit || 0) - (a.max_limit || 0)) .map((b) => ( + key={b.id} + current={b.current_usage} + max={b.max_limit} + resetDuration={b.reset_duration} + resetConfig={b.reset_config} + /> ))}
) : ( diff --git a/ui/app/_fallbacks/enterprise/components/virtual-mcps/virtualMcpAccessProfiles.tsx b/ui/app/_fallbacks/enterprise/components/virtual-mcps/virtualMcpAccessProfiles.tsx index 698a86720de..6939bb50e5d 100644 --- a/ui/app/_fallbacks/enterprise/components/virtual-mcps/virtualMcpAccessProfiles.tsx +++ b/ui/app/_fallbacks/enterprise/components/virtual-mcps/virtualMcpAccessProfiles.tsx @@ -2,4 +2,4 @@ // the enterprise build supplies the real "Access Profiles using this Virtual MCP" section. export default function VirtualMcpAccessProfiles(_props: { vmcpId: number; active: boolean }) { return null; -} +} \ No newline at end of file diff --git a/ui/app/main.tsx b/ui/app/main.tsx index 43f82d1955b..5e787cae4e6 100644 --- a/ui/app/main.tsx +++ b/ui/app/main.tsx @@ -90,4 +90,4 @@ createRoot(rootEl).render( , -); +); \ No newline at end of file diff --git a/ui/app/workspace/complexity-router/page.tsx b/ui/app/workspace/complexity-router/page.tsx index 1a131613e40..2c8800279b5 100644 --- a/ui/app/workspace/complexity-router/page.tsx +++ b/ui/app/workspace/complexity-router/page.tsx @@ -797,4 +797,4 @@ export default function ComplexityRouterPage() { ); -} +} \ No newline at end of file diff --git a/ui/app/workspace/complexity-router/views/classifierStatusBadge.tsx b/ui/app/workspace/complexity-router/views/classifierStatusBadge.tsx index 5e98706b3d7..6e077ce981b 100644 --- a/ui/app/workspace/complexity-router/views/classifierStatusBadge.tsx +++ b/ui/app/workspace/complexity-router/views/classifierStatusBadge.tsx @@ -133,7 +133,11 @@ export function ClassifierStatusBadge({ - +

{summary}

{status?.serving_previous && state === "warming" && ( @@ -204,4 +208,4 @@ export function ClassifierStatusBadge({ ); -} +} \ No newline at end of file diff --git a/ui/app/workspace/config/views/clientSettingsView.tsx b/ui/app/workspace/config/views/clientSettingsView.tsx index 782a80934f2..bd8bb75dc02 100644 --- a/ui/app/workspace/config/views/clientSettingsView.tsx +++ b/ui/app/workspace/config/views/clientSettingsView.tsx @@ -589,4 +589,4 @@ export default function ClientSettingsView() {
); -} +} \ No newline at end of file diff --git a/ui/app/workspace/config/views/compatibilityView.tsx b/ui/app/workspace/config/views/compatibilityView.tsx index 551a37d1408..10d7fc8e9d7 100644 --- a/ui/app/workspace/config/views/compatibilityView.tsx +++ b/ui/app/workspace/config/views/compatibilityView.tsx @@ -146,10 +146,10 @@ export default function CompatibilityView() {

- Use Chat Completion APIs for Claude Code, Codex, etc. for Azure Deepseek models. + Use Chat Completion APIs for Claude Code, Codex, etc. for Azure Deepseek models.

diff --git a/ui/app/workspace/custom-pricing/overrides/pricingFields.test.ts b/ui/app/workspace/custom-pricing/overrides/pricingFields.test.ts index de6a7f91c00..711e1e3e822 100644 --- a/ui/app/workspace/custom-pricing/overrides/pricingFields.test.ts +++ b/ui/app/workspace/custom-pricing/overrides/pricingFields.test.ts @@ -121,4 +121,4 @@ describe("pricingFieldError", () => { expect(pricingFieldError("off_peak_cost_multiplier", "-0.5")).toBe("Must be greater than 0 and at most 1"); expect(pricingFieldError("off_peak_cost_multiplier", "1.5")).toBe("Must be greater than 0 and at most 1"); }); -}); +}); \ No newline at end of file diff --git a/ui/app/workspace/dashboard/components/charts/barShape.tsx b/ui/app/workspace/dashboard/components/charts/barShape.tsx index ceebc03db77..78e953dd7b5 100644 --- a/ui/app/workspace/dashboard/components/charts/barShape.tsx +++ b/ui/app/workspace/dashboard/components/charts/barShape.tsx @@ -1,4 +1,4 @@ -import type { ReactNode } from "react"; +import { Children, cloneElement, isValidElement, type ReactElement, type ReactNode } from "react"; import { BarStack, type BarShapeProps, Rectangle, usePlotArea } from "recharts"; // Bars round only the open end (the end that points away from the axis). The @@ -30,10 +30,16 @@ export function rankingBarShape(props: BarShapeProps) { // the stack as one shape instead. // // The cap is sized from the column width Recharts will draw: the plot width -// split across `buckets`, minus the category gap, never wider than `barSize`. +// split across `buckets`, minus a 1px gap, never wider than `barSize`. The +// same width is pushed onto each child : on a numeric X axis a fixed +// barSize wins over barCategoryGap, so dense buckets would otherwise render +// as one fused block with no gap between columns. export function CappedBarStack({ buckets, barSize = 30, children }: { buckets: number; barSize?: number; children: ReactNode }) { const plot = usePlotArea(); - const columnWidth = plot && buckets > 0 ? Math.min(barSize, plot.width / buckets - 2) : barSize; + const columnWidth = plot && buckets > 0 ? Math.max(1, Math.min(barSize, Math.floor(plot.width / buckets) - 1)) : barSize; const r = capRadius(columnWidth); - return {children}; + const sized = Children.map(children, (child) => + isValidElement(child) ? cloneElement(child as ReactElement<{ barSize?: number }>, { barSize: columnWidth }) : child, + ); + return {sized}; } \ No newline at end of file diff --git a/ui/app/workspace/dashboard/page.tsx b/ui/app/workspace/dashboard/page.tsx index d32ff1ff750..5a5e493d759 100644 --- a/ui/app/workspace/dashboard/page.tsx +++ b/ui/app/workspace/dashboard/page.tsx @@ -802,4 +802,4 @@ export default function DashboardPage() {
); -} +} \ No newline at end of file diff --git a/ui/app/workspace/edge-control/devices/page.tsx b/ui/app/workspace/edge-control/devices/page.tsx index 5a2c268626a..a8e3bc34a2c 100644 --- a/ui/app/workspace/edge-control/devices/page.tsx +++ b/ui/app/workspace/edge-control/devices/page.tsx @@ -2,7 +2,10 @@ import DevicesView from "@enterprise/components/edge-control/devicesView"; export default function EdgeDevicesPage() { return ( -
+
); diff --git a/ui/app/workspace/edge-control/inventory/page.tsx b/ui/app/workspace/edge-control/inventory/page.tsx index 831208dc595..7be2afb48c5 100644 --- a/ui/app/workspace/edge-control/inventory/page.tsx +++ b/ui/app/workspace/edge-control/inventory/page.tsx @@ -2,7 +2,10 @@ import InventoryView from "@enterprise/components/edge-control/inventoryView"; export default function EdgeInventoryPage() { return ( -
+
); diff --git a/ui/app/workspace/governance/teams/page.tsx b/ui/app/workspace/governance/teams/page.tsx index 8ca77cae3e3..e5574df5c9e 100644 --- a/ui/app/workspace/governance/teams/page.tsx +++ b/ui/app/workspace/governance/teams/page.tsx @@ -2,7 +2,10 @@ import { TeamsView } from "@enterprise/components/user-groups/teamsView"; export default function GovernanceTeamsPage() { return ( -
+
); diff --git a/ui/app/workspace/governance/views/customerSheet.tsx b/ui/app/workspace/governance/views/customerSheet.tsx index 70bcc3bbc7e..13306e70308 100644 --- a/ui/app/workspace/governance/views/customerSheet.tsx +++ b/ui/app/workspace/governance/views/customerSheet.tsx @@ -370,8 +370,8 @@ export default function CustomerSheet({ open, onOpenChange, customer, onSuccess Align to calendar cycle

- Reset budgets and rate limits at the start of each period (e.g. 1st of month) instead of rolling from creation date. Quarterly budgets always align to fiscal quarter starts. - Applies to durations of a day or longer. + Reset budgets and rate limits at the start of each period (e.g. 1st of month) instead of rolling from creation date. + Quarterly budgets always align to fiscal quarter starts. Applies to durations of a day or longer.

- Reset budgets and rate limits at the start of each period (e.g. 1st of month) instead of rolling from creation date. Quarterly budgets always align to fiscal quarter starts. - Applies to durations of a day or longer. + Reset budgets and rate limits at the start of each period (e.g. 1st of month) instead of rolling from creation date. + Quarterly budgets always align to fiscal quarter starts. Applies to durations of a day or longer.

); -} +} \ No newline at end of file diff --git a/ui/app/workspace/logs/sheets/logDetailView.tsx b/ui/app/workspace/logs/sheets/logDetailView.tsx index d61f7a1c4e6..afca60ebc4e 100644 --- a/ui/app/workspace/logs/sheets/logDetailView.tsx +++ b/ui/app/workspace/logs/sheets/logDetailView.tsx @@ -860,6 +860,15 @@ const messageRoleLabel: Record = { tool: "Tool Result", }; +// Decision logs store the state as the user message and the answers as the +// assistant message; label them by what they actually are. +const decisionRoleLabel = (requestType: string | undefined, role: MessageRole): string | undefined => { + if (requestType !== "decisions") return undefined; + if (role === "user") return "State"; + if (role === "assistant") return "Decision"; + return undefined; +}; + // deriveComplexityRouting returns the complexity tier / classification mechanism / // raw score behind a routing decision. Rows written since the structured columns // exist carry them directly; older rows fall back to parsing the prose routing @@ -976,7 +985,9 @@ function EncryptedReveal({ text, label }: { text: string; label: string }) { function CollapsibleCode({ text, preview = 3, lang, mono = true }: { text: string; preview?: number; lang?: string; mono?: boolean }) { const [open, setOpen] = useState(false); - const lines = text.split("\n"); + // Trailing blank lines would otherwise count as hidden content and render a + // "Show more" that expands to nothing visible. + const lines = text.replace(/\s+$/, "").split("\n"); const shown = open ? lines : lines.slice(0, preview); const hasMore = lines.length > preview; const moreCount = lines.length - preview; @@ -1006,7 +1017,39 @@ function CollapsibleCode({ text, preview = 3, lang, mono = true }: { text: strin ); } -function MessageRow({ role, meta, children, last = false }: { role: MessageRole; meta?: string; children: ReactNode; last?: boolean }) { +// Generated tool identifiers (e.g. Codex-style names with embedded signatures) +// can run to hundreds of characters; truncate the middle and keep the full name +// one hover away. +const TOOL_NAME_MAX = 48; + +function ToolNameLabel({ name }: { name: string }) { + if (name.length <= TOOL_NAME_MAX) return <>{name}; + const truncated = `${name.slice(0, 32)}…${name.slice(-12)}`; + return ( + + + + {truncated} + + + {name} + + ); +} + +function MessageRow({ + role, + meta, + children, + last = false, + label, +}: { + role: MessageRole; + meta?: ReactNode; + children: ReactNode; + last?: boolean; + label?: string; +}) { return (
@@ -1015,7 +1058,7 @@ function MessageRow({ role, meta, children, last = false }: { role: MessageRole;
- {messageRoleLabel[role]} + {label ?? messageRoleLabel[role]} {meta ? {meta} : null}
{children}
@@ -1024,6 +1067,27 @@ function MessageRow({ role, meta, children, last = false }: { role: MessageRole; ); } +// Collapses all but the last two messages of a long input history. The earlier +// turns stay in the DOM order they occurred in; expanding reveals them in place. +function MessageHistoryCollapse({ count, children }: { count: number; children: ReactNode }) { + const [open, setOpen] = useState(false); + return ( + <> + + {open ? children : null} + + ); +} + interface LogDetailViewProps { log: LogEntry | null; resolvedSelectedPromptName?: string; // Current prompt name from prompt-repo when `selected_prompt_id` is set; falls back to stored log name @@ -1628,7 +1692,15 @@ export function LogDetailView({ /> - {(visibleRoles.size < allRoles.length - ? log.input_history?.filter((m) => { - if (!m) return false; - const mainRole = ((m.role as string) || "user") as MessageRole; - const hasReasoning = !!extractChatReasoning(m); - return visibleRoles.has(mainRole) || (hasReasoning && visibleRoles.has("reasoning")); - }) - : log.input_history?.filter(Boolean) - )?.flatMap((message, index) => { - const role = ((message.role as string) || "user") as MessageRole; - const text = extractMessageText(message, activeInputRevealMapping); - const reasoningText = extractChatReasoning(message, activeInputRevealMapping); - const showAll = visibleRoles.size === allRoles.length; - const showMain = showAll || visibleRoles.has(role); - const showReasoning = !!reasoningText && (showAll || visibleRoles.has("reasoning")); - const hasToolCalls = Array.isArray(message.tool_calls) && message.tool_calls.length > 0; - const isOverallLast = - index === (log.input_history?.length ?? 0) - 1 && !log.output_message && !log.error_details?.error.message; - const lineCount = text ? text.split("\n").length : 0; - const approxTokens = text ? Math.max(1, Math.round(text.length / 4)) : 0; - const reasoningTokens = reasoningText ? Math.max(1, Math.round(reasoningText.length / 4)) : 0; - const meta = text - ? role === "system" || role === "tool" - ? `${lineCount} line${lineCount === 1 ? "" : "s"} · ~${approxTokens} tokens` - : `${lineCount} line${lineCount === 1 ? "" : "s"}` - : hasToolCalls - ? `${message.tool_calls!.length} tool call${message.tool_calls!.length === 1 ? "" : "s"}` - : undefined; - const usePlainText = role === "user" || role === "assistant"; - const rows: ReactNode[] = []; - if (showReasoning) { - rows.push( - - - , - ); - } - if (showMain) { - rows.push( - - {text ? ( - usePlainText && isJson(text) ? ( - { - try { - return JSON.stringify(JSON.parse(text), null, 2); - } catch { - return text; - } - })()} - lang="json" - readonly - autoResize - options={{ - collapsibleBlocks: true, - showIndentLines: false, - disableHover: true, - }} - /> - ) : usePlainText ? ( - - ) : ( - - ) - ) : ( - - )} - {text && - Array.isArray(message.content) && - (message.content as ContentBlock[]) - .filter((b) => b.type === "image_url") - .map((b, i) => { - const src = b.image_url?.url; - if (!src) return null; - return Attached image; - })} - {text && - Array.isArray(message.content) && - (message.content as ContentBlock[]) - .filter((b) => b.type === "file" && b.file) - .map((b, i) => ( - { + const historyMessages = + (visibleRoles.size < allRoles.length + ? log.input_history?.filter((m) => { + if (!m) return false; + const mainRole = ((m.role as string) || "user") as MessageRole; + const hasReasoning = !!extractChatReasoning(m); + return visibleRoles.has(mainRole) || (hasReasoning && visibleRoles.has("reasoning")); + }) + : log.input_history?.filter(Boolean)) ?? []; + const messageRows = historyMessages.map((message, index) => { + const role = ((message.role as string) || "user") as MessageRole; + const text = extractMessageText(message, activeInputRevealMapping); + const reasoningText = extractChatReasoning(message, activeInputRevealMapping); + const showAll = visibleRoles.size === allRoles.length; + const showMain = showAll || visibleRoles.has(role); + const showReasoning = !!reasoningText && (showAll || visibleRoles.has("reasoning")); + const hasToolCalls = Array.isArray(message.tool_calls) && message.tool_calls.length > 0; + const isOverallLast = + index === (log.input_history?.length ?? 0) - 1 && !log.output_message && !log.error_details?.error.message; + const lineCount = text ? text.split("\n").length : 0; + const approxTokens = text ? Math.max(1, Math.round(text.length / 4)) : 0; + const reasoningTokens = reasoningText ? Math.max(1, Math.round(reasoningText.length / 4)) : 0; + const meta = text + ? role === "system" || role === "tool" + ? `${lineCount} line${lineCount === 1 ? "" : "s"} · ~${approxTokens} tokens` + : `${lineCount} line${lineCount === 1 ? "" : "s"}` + : hasToolCalls + ? `${message.tool_calls!.length} tool call${message.tool_calls!.length === 1 ? "" : "s"}` + : undefined; + const usePlainText = role === "user" || role === "assistant"; + const rows: ReactNode[] = []; + if (showReasoning) { + rows.push( + + + , + ); + } + if (showMain) { + rows.push( + + {text ? ( + usePlainText && isJson(text) ? ( + { + try { + return JSON.stringify(JSON.parse(text), null, 2); + } catch { + return text; + } + })()} + lang="json" + readonly + autoResize + options={{ + collapsibleBlocks: true, + showIndentLines: false, + disableHover: true, + }} /> - ))} - {hasToolCalls && text ? ( -
- {message - .tool_calls!.map((tc) => tc.function?.name) - .filter(Boolean) - .join(", ") || `${message.tool_calls!.length} tool call${message.tool_calls!.length === 1 ? "" : "s"}`} -
- ) : null} - , - ); - } - return rows; - })} + ) : usePlainText ? ( + + ) : ( + + ) + ) : ( + + )} + {text && + Array.isArray(message.content) && + (message.content as ContentBlock[]) + .filter((b) => b.type === "image_url") + .map((b, i) => { + const src = b.image_url?.url; + if (!src) return null; + return ( + Attached image + ); + })} + {text && + Array.isArray(message.content) && + (message.content as ContentBlock[]) + .filter((b) => b.type === "file" && b.file) + .map((b, i) => ( + + ))} + {hasToolCalls && text ? ( +
+ {message + .tool_calls!.map((tc) => tc.function?.name) + .filter(Boolean) + .join(", ") || `${message.tool_calls!.length} tool call${message.tool_calls!.length === 1 ? "" : "s"}`} +
+ ) : null} + , + ); + } + return rows; + }); + // Show only the last two turns; everything earlier collapses + // behind an expandable history toggle. + const visibleTail = 2; + const splitAt = Math.max(0, messageRows.length - visibleTail); + const earlier = messageRows.slice(0, splitAt); + const tail = messageRows.slice(splitAt); + const earlierCount = earlier.filter((r) => r.length > 0).length; + return ( + <> + {earlierCount > 0 && {earlier}} + {tail} + + ); + })()} {log.output_message && !log.error_details?.error.message && (() => { @@ -3233,7 +3323,7 @@ export function LogDetailView({ ) : null} {showAssistant ? ( - + {showRefusal ? (
@@ -3322,7 +3412,7 @@ export function LogDetailView({ const itemPayload = extractResponsesItemPayload(msg); const lineCount = text ? text.split("\n").length : 0; const approxTokens = text ? Math.max(1, Math.round(text.length / 4)) : 0; - let meta: string | undefined; + let meta: ReactNode | undefined; if (role === "reasoning" && reasoningParts) { const totalLen = reasoningParts.summaries.reduce((acc, s) => acc + s.length, 0) + @@ -3339,24 +3429,33 @@ export function LogDetailView({ ? "encrypted" : undefined; } else { - meta = text - ? role === "system" || role === "tool" - ? msg.name - ? `${msg.name} · ${lineCount} line${lineCount === 1 ? "" : "s"} · ~${approxTokens} tokens` - : `${lineCount} line${lineCount === 1 ? "" : "s"} · ~${approxTokens} tokens` - : `${lineCount} line${lineCount === 1 ? "" : "s"}` - : msg.name - ? msg.name - : msg.type === "function_call_output" && msg.call_id - ? msg.call_id - : Array.isArray(msg.tools) - ? (() => { - const callable = flattenDeclaredTools(msg.tools).length; - return callable !== msg.tools.length - ? `${msg.type} · ${msg.tools.length} declarations · ${callable} callable tools` - : `${msg.type} · ${msg.tools.length} tool${msg.tools.length === 1 ? "" : "s"}`; - })() - : [msg.type, summarizeResponsesToolCall(msg, mapping)].filter(Boolean).join(" · ") || undefined; + meta = text ? ( + role === "system" || role === "tool" ? ( + msg.name ? ( + <> + + {` · ${lineCount} line${lineCount === 1 ? "" : "s"} · ~${approxTokens} tokens`} + + ) : ( + `${lineCount} line${lineCount === 1 ? "" : "s"} · ~${approxTokens} tokens` + ) + ) : ( + `${lineCount} line${lineCount === 1 ? "" : "s"}` + ) + ) : msg.name ? ( + + ) : msg.type === "function_call_output" && msg.call_id ? ( + + ) : Array.isArray(msg.tools) ? ( + (() => { + const callable = flattenDeclaredTools(msg.tools).length; + return callable !== msg.tools.length + ? `${msg.type} · ${msg.tools.length} declarations · ${callable} callable tools` + : `${msg.type} · ${msg.tools.length} tool${msg.tools.length === 1 ? "" : "s"}`; + })() + ) : ( + [msg.type, summarizeResponsesToolCall(msg, mapping)].filter(Boolean).join(" · ") || undefined + ); } const usePlainText = role === "user" || role === "assistant"; return ( @@ -3378,10 +3477,9 @@ export function LogDetailView({
))} {reasoningParts.encrypted ? ( -
-
Encrypted
- -
+ // Ciphertext is noise even at two preview lines; fold it + // entirely until the reader asks for it. + ) : null} {reasoningParts.signatures.length > 0 ? ( { }); it("is storage-disabled when the provider explicitly disables raw storage", () => { - expect( - resolveRawJsonNoticeState({ ...base, providers: [{ name: "openai", store_raw_request_response: false }] }), - ).toBe("storage-disabled"); + expect(resolveRawJsonNoticeState({ ...base, providers: [{ name: "openai", store_raw_request_response: false }] })).toBe( + "storage-disabled", + ); }); it("is unknown when the provider has raw storage enabled", () => { - expect( - resolveRawJsonNoticeState({ ...base, providers: [{ name: "openai", store_raw_request_response: true }] }), - ).toBe("unknown"); + expect(resolveRawJsonNoticeState({ ...base, providers: [{ name: "openai", store_raw_request_response: true }] })).toBe("unknown"); }); it("is unknown when the setting is absent on the provider", () => { @@ -43,9 +41,7 @@ describe("resolveRawJsonNoticeState", () => { }); it("is unknown when this log's provider is not in the list", () => { - expect( - resolveRawJsonNoticeState({ ...base, providers: [{ name: "anthropic", store_raw_request_response: false }] }), - ).toBe("unknown"); + expect(resolveRawJsonNoticeState({ ...base, providers: [{ name: "anthropic", store_raw_request_response: false }] })).toBe("unknown"); }); }); diff --git a/ui/app/workspace/logs/sheets/logDetailsSheet.tsx b/ui/app/workspace/logs/sheets/logDetailsSheet.tsx index 18638a1fb6c..47fb12b5dd2 100644 --- a/ui/app/workspace/logs/sheets/logDetailsSheet.tsx +++ b/ui/app/workspace/logs/sheets/logDetailsSheet.tsx @@ -75,7 +75,7 @@ export function LogDetailSheet({ return ( - + {!isFullDataReady ? (
Loading log details @@ -117,4 +117,4 @@ export function LogDetailSheet({ ); -} +} \ No newline at end of file diff --git a/ui/app/workspace/logs/views/logsHeaderView.tsx b/ui/app/workspace/logs/views/logsHeaderView.tsx index 237aa5f67ad..d521efc621a 100644 --- a/ui/app/workspace/logs/views/logsHeaderView.tsx +++ b/ui/app/workspace/logs/views/logsHeaderView.tsx @@ -367,7 +367,7 @@ export function LogsHeaderView({ {/* Full width while the row wraps, so the search field owns its own line instead of squeezing to its 12rem minimum beside the date picker. */} -
+
{ it("reads a rise as positive by default", () => { @@ -231,4 +242,4 @@ describe("tokenSplit", () => { it("gives the whole bar to output when there is no input", () => { expect(tokenSplit(0, 400)).toEqual({ first: 0, second: 100 }); }); -}); +}); \ No newline at end of file diff --git a/ui/app/workspace/logs/views/metricStrip.utils.ts b/ui/app/workspace/logs/views/metricStrip.utils.ts index 5c0e0c2570b..a1aa2432118 100644 --- a/ui/app/workspace/logs/views/metricStrip.utils.ts +++ b/ui/app/workspace/logs/views/metricStrip.utils.ts @@ -235,4 +235,4 @@ export function tokenSplit(first: number, second: number): TokenSplit | null { if (total <= 0) return null; const share = (first / total) * 100; return { first: share, second: 100 - share }; -} +} \ No newline at end of file diff --git a/ui/app/workspace/mcp-registry/library/views/mcpLibraryAddServerSheet.tsx b/ui/app/workspace/mcp-registry/library/views/mcpLibraryAddServerSheet.tsx index ab663d9798e..8376a8ce6b8 100644 --- a/ui/app/workspace/mcp-registry/library/views/mcpLibraryAddServerSheet.tsx +++ b/ui/app/workspace/mcp-registry/library/views/mcpLibraryAddServerSheet.tsx @@ -172,9 +172,9 @@ export function MCPLibraryAddServerSheet({ open, onClose }: MCPLibraryAddServerS

- This listing stays in your organization's library. It is not published to the global Bifrost MCP catalog. It is also a - catalog entry, not a live MCP connection: credentials are never stored on the listing, each person who installs it - supplies their own. + This listing stays in your organization's library. It is not published to the global Bifrost MCP catalog. It is + also a catalog entry, not a live MCP connection: credentials are never stored on the listing, each person who installs + it supplies their own.

diff --git a/ui/app/workspace/mcp-registry/library/views/mcpLibrarySettingsSheet.tsx b/ui/app/workspace/mcp-registry/library/views/mcpLibrarySettingsSheet.tsx index cb6d0b83800..5cae509f645 100644 --- a/ui/app/workspace/mcp-registry/library/views/mcpLibrarySettingsSheet.tsx +++ b/ui/app/workspace/mcp-registry/library/views/mcpLibrarySettingsSheet.tsx @@ -19,8 +19,7 @@ const mcpLibrarySettingsSchema = z.object({ .string() .trim() .refine( - (value) => - value === "" || value.startsWith("http://") || value.startsWith("https://") || value.startsWith("file://"), + (value) => value === "" || value.startsWith("http://") || value.startsWith("https://") || value.startsWith("file://"), "URL must start with http://, https://, or file://", ), // 0 disables background syncing entirely. Force Sync Now still works. @@ -122,8 +121,8 @@ export function MCPLibrarySettingsSheet({ open, onClose }: MCPLibrarySettingsShe

- URL to a custom MCP server catalog. Leave empty to use the default Bifrost catalog. Use a{" "} - file:// URL to load the catalog from local disk in air-gapped deployments. + URL to a custom MCP server catalog. Leave empty to use the default Bifrost catalog. Use a file:// URL to load + the catalog from local disk in air-gapped deployments.

- How often to sync the MCP server catalog from the source URL. Set to 0 to disable background syncing; - Force Sync Now still works. + How often to sync the MCP server catalog from the source URL. Set to 0 to disable background syncing; Force Sync Now still + works.

{ throw redirect({ to: "/workspace/virtual-mcps" }); }, -}); +}); \ No newline at end of file diff --git a/ui/app/workspace/model-limits/views/modelLimitsTable.tsx b/ui/app/workspace/model-limits/views/modelLimitsTable.tsx index adc4109d280..ea37264fb73 100644 --- a/ui/app/workspace/model-limits/views/modelLimitsTable.tsx +++ b/ui/app/workspace/model-limits/views/modelLimitsTable.tsx @@ -423,39 +423,39 @@ export default function ModelLimitsTable({
- {config.scope !== "global" && config.scope_id && config.scope_name ? ( - - - - { - if (!config.scope_id) return; - const target = getModelLimitScope(config.scope ?? "global")?.buildDeepLink?.(config.scope_id); - if (target) navigate(target as never); - }} - > - {config.scope_name} - {getModelLimitScope(config.scope ?? "global")?.buildDeepLink && ( - - )} - - - {config.scope_name} - - - ) : ( - - - )} - {(() => { - const ManagedBy = getModelLimitScope(config.scope ?? "global")?.ManagedByComponent; - return ManagedBy ? : null; - })()} + {config.scope !== "global" && config.scope_id && config.scope_name ? ( + + + + { + if (!config.scope_id) return; + const target = getModelLimitScope(config.scope ?? "global")?.buildDeepLink?.(config.scope_id); + if (target) navigate(target as never); + }} + > + {config.scope_name} + {getModelLimitScope(config.scope ?? "global")?.buildDeepLink && ( + + )} + + + {config.scope_name} + + + ) : ( + - + )} + {(() => { + const ManagedBy = getModelLimitScope(config.scope ?? "global")?.ManagedByComponent; + return ManagedBy ? : null; + })()}
diff --git a/ui/app/workspace/observability/fragments/otelFormFragment.tsx b/ui/app/workspace/observability/fragments/otelFormFragment.tsx index 5ca420779f0..2095b465931 100644 --- a/ui/app/workspace/observability/fragments/otelFormFragment.tsx +++ b/ui/app/workspace/observability/fragments/otelFormFragment.tsx @@ -609,60 +609,60 @@ function OtelProfileSection({ form, control, index, hasOtelAccess, canRemove, op )} />
- ( - - Format - + + + + + + + {traceTypeOptions.map((option) => ( + + {option.label} + + ))} + + + + + )} + /> + ( + + Export Timeout (seconds) - - - + field.onChange(e.target.value === "" ? null : Number(e.target.value))} + /> - - {traceTypeOptions.map((option) => ( - - {option.label} - - ))} - - - - - )} - /> - ( - - Export Timeout (seconds) - - field.onChange(e.target.value === "" ? null : Number(e.target.value))} - /> - - - Maximum time for a single trace export (1-60 seconds). Traces are dropped rather than retried past this limit, so - an unreachable collector cannot slow down request handling. - - - - )} - /> + + Maximum time for a single trace export (1-60 seconds). Traces are dropped rather than retried past this limit, + so an unreachable collector cannot slow down request handling. + + + + )} + />

Overhead breakdown

-

- Export per-component Bifrost overhead latency as a histogram. -

+

Export per-component Bifrost overhead latency as a histogram.

- +
diff --git a/ui/app/workspace/providers/dialogs/databricksMigrationDialog.tsx b/ui/app/workspace/providers/dialogs/databricksMigrationDialog.tsx index fde99193e9e..17a3d47630d 100644 --- a/ui/app/workspace/providers/dialogs/databricksMigrationDialog.tsx +++ b/ui/app/workspace/providers/dialogs/databricksMigrationDialog.tsx @@ -223,24 +223,30 @@ export default function DatabricksMigrationDialog({ show, provider, onDeferred, - {stage === "finished" && result ? (result.ok ? "Migration complete" : "Migration failed") : "Databricks is now a first-party provider"} + {stage === "finished" && result + ? result.ok + ? "Migration complete" + : "Migration failed" + : "Databricks is now a first-party provider"}
{(stage === "intro" || stage === "loading") && ( <>

- {provider.name} is a custom provider pointing at a Databricks workspace. - Bifrost now supports Databricks natively, with personal access tokens, OAuth service principals, Model Serving and AI Gateway - routing. Your existing configuration needs to be migrated to the official provider. + {provider.name} is a custom provider pointing at a Databricks + workspace. Bifrost now supports Databricks natively, with personal access tokens, OAuth service principals, Model + Serving and AI Gateway routing. Your existing configuration needs to be migrated to the official provider.

- Nothing changes until you confirm. You will see exactly what will be copied before the migration runs, and the custom provider is - only removed after the new one is verified. + Nothing changes until you confirm. You will see exactly what will be copied before the migration runs, and the custom + provider is only removed after the new one is verified.

)} - {stage === "preview" &&

Review what will be migrated. Secrets are masked; anything that could not be read must be entered below.

} + {stage === "preview" && ( +

Review what will be migrated. Secrets are masked; anything that could not be read must be entered below.

+ )} {stage === "running" &&

Migrating. Keep this window open until it finishes.

} {stage === "finished" && result &&

{result.message}

}
@@ -273,7 +279,11 @@ export default function DatabricksMigrationDialog({ show, provider, onDeferred, Not now - @@ -286,7 +296,9 @@ export default function DatabricksMigrationDialog({ show, provider, onDeferred, Back Not now - + @@ -485,7 +497,9 @@ function MigrationProgress({ steps }: { steps: MigrationStep[] }) {
{step.label}
{step.detail && ( -
+
{step.detail}
)} @@ -522,4 +536,4 @@ function Stat({ label, value }: { label: string; value: string }) {
{value}
); -} +} \ No newline at end of file diff --git a/ui/app/workspace/providers/dialogs/firstPartyProviderAvailableDialog.tsx b/ui/app/workspace/providers/dialogs/firstPartyProviderAvailableDialog.tsx index 3c1438d99be..c2f8497d0f8 100644 --- a/ui/app/workspace/providers/dialogs/firstPartyProviderAvailableDialog.tsx +++ b/ui/app/workspace/providers/dialogs/firstPartyProviderAvailableDialog.tsx @@ -32,8 +32,8 @@ export default function FirstPartyProviderAvailableDialog({ show, customProvider Official {label} integration available - We noticed you have a custom provider named {customProviderName}. - Bifrost now supports {label} natively. You can delete the custom provider and add the official {label} integration from{" "} + We noticed you have a custom provider named {customProviderName}. Bifrost + now supports {label} natively. You can delete the custom provider and add the official {label} integration from{" "} Add Provider, or keep using your custom provider as is. @@ -44,4 +44,4 @@ export default function FirstPartyProviderAvailableDialog({ show, customProvider ); -} +} \ No newline at end of file diff --git a/ui/app/workspace/providers/fragments/apiKeysFormFragment.tsx b/ui/app/workspace/providers/fragments/apiKeysFormFragment.tsx index c2b6b31e298..feb58cabe46 100644 --- a/ui/app/workspace/providers/fragments/apiKeysFormFragment.tsx +++ b/ui/app/workspace/providers/fragments/apiKeysFormFragment.tsx @@ -370,121 +370,121 @@ export function ApiKeyFormFragment({ control, providerName, baseProviderType, fo )} /> )} - <> - ( - - - - Allowed Models - - - - - - - - -

- Select specific models this key applies to, or choose "Allow All Models" to allow all. Leave empty to deny - all. Aliases must be added by their alias name - listing only the underlying model does not allow the alias - (an alias best-model → gpt-4o requires "best-model" here, not just "gpt-4o"). -

- - - - - } - /> - - - - )} - /> - ( - - - - Blocked Models - - - - - - - - -

- Models this key must never serve. The denylist always wins - if a model appears in both Allowed Models and - here, it is blocked. Select "All Models" to block every model on this key. Aliases are matched by their - alias name - blocking only the underlying model does not block aliases that point to it. -

- - - - - } - /> - - - - )} - /> - ( - - Deployments (Optional) - - Map a request model name to the provider's identifier (deployment name, inference profile ID, etc.). Expand a row for - canonical name, model family, and provider overrides - these drive cost logs and family-based routing. - {isReplicate && ( + <> + ( + + + - {" "} - Replicate deployments are listed only while "Use Deployments Endpoint" is on - otherwise type the owner/name - and press Enter. + Allowed Models + + + + + + + + +

+ Select specific models this key applies to, or choose "Allow All Models" to allow all. Leave empty to deny + all. Aliases must be added by their alias name - listing only the underlying model does not allow the alias + (an alias best-model → gpt-4o requires "best-model" here, not just "gpt-4o"). +

+ + + - )} - - -
- { - form.clearErrors("key.aliases"); - field.onChange(Object.keys(next).length > 0 ? next : {}); - }} - /> -
- - - - )} - /> - + } + /> + + + + )} + /> + ( + + + + Blocked Models + + + + + + + + +

+ Models this key must never serve. The denylist always wins - if a model appears in both Allowed Models and + here, it is blocked. Select "All Models" to block every model on this key. Aliases are matched by their alias + name - blocking only the underlying model does not block aliases that point to it. +

+ + + + + } + /> + + + + )} + /> + ( + + Deployments (Optional) + + Map a request model name to the provider's identifier (deployment name, inference profile ID, etc.). Expand a row for + canonical name, model family, and provider overrides - these drive cost logs and family-based routing. + {isReplicate && ( + <> + {" "} + Replicate deployments are listed only while "Use Deployments Endpoint" is on - otherwise type the owner/name + and press Enter. + + )} + + +
+ { + form.clearErrors("key.aliases"); + field.onChange(Object.keys(next).length > 0 ? next : {}); + }} + /> +
+ + + + )} + /> + {supportsBatchAPI && !isBedrock && !isAzure && !isVertex && } {isAzure && (
@@ -1687,4 +1687,4 @@ export function ApiKeyFormFragment({ control, providerName, baseProviderType, fo )}
); -} +} \ No newline at end of file diff --git a/ui/app/workspace/providers/fragments/governanceFormFragment.tsx b/ui/app/workspace/providers/fragments/governanceFormFragment.tsx index 3e0e0021db3..e3492a9e66a 100644 --- a/ui/app/workspace/providers/fragments/governanceFormFragment.tsx +++ b/ui/app/workspace/providers/fragments/governanceFormFragment.tsx @@ -224,7 +224,8 @@ export function GovernanceFormFragment({ provider }: GovernanceFormFragmentProps Align to calendar cycle

- Reset budgets at the start of each period (e.g. 1st of month) instead of rolling from creation date. Quarterly budgets always align to fiscal quarter starts. + Reset budgets at the start of each period (e.g. 1st of month) instead of rolling from creation date. Quarterly budgets + always align to fiscal quarter starts.