diff --git a/apps/docs/en/manual/model-prices.md b/apps/docs/en/manual/model-prices.md index da89030a..03cde56d 100644 --- a/apps/docs/en/manual/model-prices.md +++ b/apps/docs/en/manual/model-prices.md @@ -17,7 +17,7 @@ Open the [Model Prices Demo](https://seakee.github.io/CPA-Manager-Plus/#/demo/mo Synchronization only occurs when the user triggers it and may use the current Manager Server proxy configuration. -The same models.dev model ID may be offered by multiple providers, and a real model ID may itself contain `/`. CPAMP compares source identities separately from original model IDs and only matches automatically when the complete pricing metadata, including tiers and experimental-mode prices, is consistent. Identity or price conflicts remain in the candidate-confirmation flow; a same-named LiteLLM or OpenRouter entry cannot bypass that conflict. +Automatic matching runs strictly in models.dev, LiteLLM, then OpenRouter order. CPAMP uses the canonical model metadata in the models.dev catalog to prefer the first-party official entry. A source is saved automatically only when it has one clear, strong identity match; fuzzy similarities are never auto-confirmed. An ambiguous source falls through to the next source. If none of the three sources yields a unique match, the confirmation list keeps candidates from each source separately, even when they share the same original model ID. The current sync maps models.dev `cost.input`, `cost.output`, `cost.cache_read`, and `cost.cache_write`, converts valid `cost.tiers` context tiers into CPAMP billing rules, and maps `experimental.modes.fast.cost` to short-context Fast/Priority prices. The complete model object remains available in raw metadata; reasoning prices, unknown experimental modes, unknown tier types, and rules that cannot be validated safely do not activate automatic billing. @@ -25,7 +25,7 @@ The current sync maps models.dev `cost.input`, `cost.output`, `cost.cache_read`, - When models.dev is temporarily unavailable, CPAMP continues with LiteLLM and OpenRouter. - A transient models.dev failure cannot automatically replace a stored models.dev price with a lower-priority source; fallback sources may still fill models that have no local price. -- A fallback source may replace a model normally when models.dev responds successfully but does not contain that model. +- When models.dev responds successfully but has no official entry or remains ambiguous, fallback sources are tried in order; only a unique strong identity match may replace the model. - If every source fails, synchronization stops before any database write and existing prices remain unchanged. - A synchronized price remains the last-known-good value until a later successful sync or a manual edit; `syncedAtMs` indicates its freshness. diff --git a/apps/docs/manual/model-prices.md b/apps/docs/manual/model-prices.md index 84ed8465..00ec3a14 100644 --- a/apps/docs/manual/model-prices.md +++ b/apps/docs/manual/model-prices.md @@ -17,7 +17,7 @@ description: 配置 CPA Manager Plus 模型价格、service tier、长上下文 同步只在用户主动触发时发生，可能使用当前 Manager Server 代理设置。 -models.dev 中同一个模型 ID 可能由多个 Provider 提供，而且真实模型 ID 本身也可能包含 `/`。CPAMP 会分别比较来源身份和原始模型 ID；只有完整价格元数据（包括阶梯和实验模式价格）明确一致时才自动匹配。任何身份或价格冲突都会进入候选确认流程，LiteLLM 或 OpenRouter 的同名条目不会绕过该冲突。 +自动匹配会严格按 models.dev、LiteLLM、OpenRouter 的顺序进行。CPAMP 使用 models.dev catalog 的规范模型元数据优先识别第一方官方条目；每个来源都只有唯一、明确的模型身份匹配才会自动保存，模糊相似项不会自动确认。某个来源存在歧义时会继续尝试下一来源；三个来源都无法唯一确认时，待确认列表会分别保留各来源的候选，即使它们的原始模型 ID 相同也不会互相覆盖。 当前同步会映射 models.dev 的 `cost.input`、`cost.output`、`cost.cache_read` 和 `cost.cache_write`，将有效的 `cost.tiers` 上下文阶梯转换为 CPAMP 计费规则，并将 `experimental.modes.fast.cost` 映射为 Fast/Priority 短上下文价格。完整模型对象仍保存在原始元数据中；reasoning、未知实验模式、未知阶梯类型或无法安全验证的规则不会激活自动计费。 @@ -25,7 +25,7 @@ models.dev 中同一个模型 ID 可能由多个 Provider 提供，而且真实 - models.dev 暂时不可用时，CPAMP 会继续尝试 LiteLLM 和 OpenRouter。 - 已保存的 models.dev 价格不会因为本次网络失败而被低优先级来源自动覆盖；回退来源仍可补充本地没有价格的模型。 -- 只有 models.dev 成功响应但明确不包含某个模型时，才允许回退来源正常替换该模型。 +- models.dev 成功响应但缺少官方条目或匹配存在歧义时，会按顺序尝试回退来源；只有唯一的强身份匹配才会替换该模型。 - 如果所有来源都失败，同步在写入数据库前终止，现有价格保持不变。 - 同步价格会一直作为最后有效数据使用，直到后续成功同步或用户手动修改；`syncedAtMs` 可用于判断数据新鲜度。 diff --git a/apps/manager-server/internal/httpapi/server.go b/apps/manager-server/internal/httpapi/server.go index 3b74a703..fd4dce49 100644 --- a/apps/manager-server/internal/httpapi/server.go +++ b/apps/manager-server/internal/httpapi/server.go @@ -17,7 +17,7 @@ var embeddedPanel embed.FS const serviceID = "cpa-manager-plus" -var modelsDevModelPriceSyncURL = "https://models.dev/api.json" +var modelsDevModelPriceSyncURL = "https://models.dev/catalog.json" var modelPriceSyncURL = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json" var openRouterModelPriceSyncURL = "https://openrouter.ai/api/v1/models" diff --git a/apps/manager-server/internal/httpapi/server_test.go b/apps/manager-server/internal/httpapi/server_test.go index 5de8c6e0..ea6492b4 100644 --- a/apps/manager-server/internal/httpapi/server_test.go +++ b/apps/manager-server/internal/httpapi/server_test.go @@ -746,20 +746,25 @@ func TestModelPricesSyncFromLiteLLMFormat(t *testing.T) { } } -func TestModelPricesSyncPrefersModelsDevProviderScopedPrices(t *testing.T) { +func TestModelPricesSyncUsesModelsDevOfficialAndOrderedFallback(t *testing.T) { modelsDevSource := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{ - "openai": {"models": { - "gpt-test": {"cost":{"input":9,"output":10,"cache_read":1}}, - "ambiguous": {"cost":{"input":3,"output":4}} - }}, - "azure": {"models": { - "ambiguous": {"cost":{"input":5,"output":6}} - }}, - "crossmodel": {"models": { - "openai/gpt-test": {"cost":{"input":11,"output":12}} - }} + "models": { + "openai/gpt-test": {"id":"openai/gpt-test","name":"GPT Test"} + }, + "providers": { + "openai": {"models": { + "gpt-test": {"cost":{"input":9,"output":10,"cache_read":1}}, + "ambiguous": {"cost":{"input":3,"output":4}} + }}, + "azure": {"models": { + "ambiguous": {"cost":{"input":5,"output":6}} + }}, + "crossmodel": {"models": { + "openai/gpt-test": {"cost":{"input":11,"output":12}} + }} + } }`)) })) t.Cleanup(modelsDevSource.Close) @@ -809,22 +814,9 @@ func TestModelPricesSyncPrefersModelsDevProviderScopedPrices(t *testing.T) { if len(response.Sources) != 2 || response.Sources[0] != "models.dev" || response.Sources[1] != "litellm" { t.Fatalf("source order = %#v", response.Sources) } - if response.Imported != 4 || len(response.Candidates) != 2 { + if response.Imported != 6 || len(response.Candidates) != 0 { t.Fatalf("sync selection = %#v", response) } - candidateSources := map[string]map[string]bool{} - for _, set := range response.Candidates { - candidateSources[set.Model] = map[string]bool{} - for _, candidate := range set.Candidates { - candidateSources[set.Model][candidate.SourceModelID] = true - } - } - if !candidateSources["ambiguous"]["openai/ambiguous"] || - !candidateSources["ambiguous"]["azure/ambiguous"] || - !candidateSources["openai/gpt-test"]["openai/gpt-test"] || - !candidateSources["openai/gpt-test"]["crossmodel/openai/gpt-test"] { - t.Fatalf("candidate sources = %#v", candidateSources) - } price, ok := response.Prices["gpt-test"] if !ok || !closeFloat(price.Prompt, 9) || !closeFloat(price.Completion, 10) || price.Source != "models.dev" || price.SourceModelID != "openai/gpt-test" { t.Fatalf("models.dev alias price = %#v", price) @@ -837,8 +829,13 @@ func TestModelPricesSyncPrefersModelsDevProviderScopedPrices(t *testing.T) { if !ok || !closeFloat(crossmodel.Prompt, 11) || crossmodel.SourceModelID != "crossmodel/openai/gpt-test" { t.Fatalf("nested provider-scoped price = %#v", crossmodel) } - if _, ok := response.Prices["openai/gpt-test"]; ok { - t.Fatalf("colliding model was imported without confirmation: %#v", response.Prices["openai/gpt-test"]) + officialScoped, ok := response.Prices["openai/gpt-test"] + if !ok || !closeFloat(officialScoped.Prompt, 9) || officialScoped.Source != "models.dev" || officialScoped.SourceModelID != "openai/gpt-test" { + t.Fatalf("official scoped price = %#v", officialScoped) + } + ambiguous, ok := response.Prices["ambiguous"] + if !ok || !closeFloat(ambiguous.Prompt, 7) || ambiguous.Source != "litellm" || ambiguous.SourceModelID != "ambiguous" { + t.Fatalf("ordered ambiguity fallback = %#v", ambiguous) } fallback, ok := response.Prices["fallback-only"] if !ok || !closeFloat(fallback.Prompt, 1) || fallback.Source != "litellm" || fallback.SourceModelID != "fallback-only" { @@ -858,7 +855,10 @@ func TestModelPricesSyncCachesModelsDevAndSkipsCoveredFallbacks(t *testing.T) { } w.Header().Set("Content-Type", "application/json") w.Header().Set("ETag", etag) - _, _ = w.Write([]byte(`{"openai":{"models":{"gpt-test":{"cost":{"input":9,"output":10}}}}}`)) + _, _ = w.Write([]byte(`{ + "models":{"openai/gpt-test":{"id":"openai/gpt-test"}}, + "providers":{"openai":{"models":{"gpt-test":{"cost":{"input":9,"output":10}}}}} + }`)) return } if received := r.Header.Get("If-None-Match"); received != etag { diff --git a/apps/manager-server/internal/service/modelprice/service.go b/apps/manager-server/internal/service/modelprice/service.go index 87c8523a..8eadcf80 100644 --- a/apps/manager-server/internal/service/modelprice/service.go +++ b/apps/manager-server/internal/service/modelprice/service.go @@ -92,7 +92,28 @@ type Service struct { pricesChangedNotifier func() } -type fetchModelPricesFunc func(context.Context, string, *http.Client) (map[string]store.ModelPrice, int, error) +type modelPriceMatchMetadata struct { + modelsDevCanonicalByIdentity map[string]string + modelsDevOfficialSourceModelIDs map[string]struct{} +} + +type fetchedModelPriceSource struct { + Prices map[string]store.ModelPrice + Metadata modelPriceMatchMetadata +} + +type modelPriceSourceEntry struct { + Key string + Price store.ModelPrice +} + +type modelPriceCollection struct { + Entries []modelPriceSourceEntry + Metadata modelPriceMatchMetadata +} + +type fetchModelPricesFunc func(context.Context, string, *http.Client) (fetchedModelPriceSource, int, error) +type fetchModelPriceMapFunc func(context.Context, string, *http.Client) (map[string]store.ModelPrice, int, error) type priceSyncSource struct { Source string @@ -100,6 +121,89 @@ type priceSyncSource struct { Fetch fetchModelPricesFunc } +func wrapModelPriceMapFetcher(fetch fetchModelPriceMapFunc) fetchModelPricesFunc { + return func(ctx context.Context, syncURL string, client *http.Client) (fetchedModelPriceSource, int, error) { + prices, skipped, err := fetch(ctx, syncURL, client) + return fetchedModelPriceSource{Prices: prices}, skipped, err + } +} + +func normalizeModelPriceIdentity(identity string) string { + return strings.ToLower(strings.TrimSpace(identity)) +} + +func (metadata *modelPriceMatchMetadata) merge(other modelPriceMatchMetadata) { + if len(other.modelsDevCanonicalByIdentity) > 0 { + if metadata.modelsDevCanonicalByIdentity == nil { + metadata.modelsDevCanonicalByIdentity = make(map[string]string, len(other.modelsDevCanonicalByIdentity)) + } + for identity, canonicalID := range other.modelsDevCanonicalByIdentity { + if existing, ok := metadata.modelsDevCanonicalByIdentity[identity]; ok && !strings.EqualFold(existing, canonicalID) { + delete(metadata.modelsDevCanonicalByIdentity, identity) + continue + } + metadata.modelsDevCanonicalByIdentity[identity] = canonicalID + } + } + if len(other.modelsDevOfficialSourceModelIDs) > 0 { + if metadata.modelsDevOfficialSourceModelIDs == nil { + metadata.modelsDevOfficialSourceModelIDs = make(map[string]struct{}, len(other.modelsDevOfficialSourceModelIDs)) + } + for sourceModelID := range other.modelsDevOfficialSourceModelIDs { + metadata.modelsDevOfficialSourceModelIDs[sourceModelID] = struct{}{} + } + } +} + +func newModelsDevMatchMetadata(models map[string]json.RawMessage) modelPriceMatchMetadata { + if len(models) == 0 { + return modelPriceMatchMetadata{} + } + metadata := modelPriceMatchMetadata{ + modelsDevCanonicalByIdentity: make(map[string]string, len(models)*2), + } + tailMatches := make(map[string]string, len(models)) + for rawModelID := range models { + modelID := strings.TrimSpace(rawModelID) + if modelID == "" { + continue + } + normalizedModelID := normalizeModelPriceIdentity(modelID) + metadata.modelsDevCanonicalByIdentity[normalizedModelID] = modelID + _, tail, ok := strings.Cut(modelID, "/") + tail = strings.TrimSpace(tail) + if !ok || tail == "" { + continue + } + normalizedTail := normalizeModelPriceIdentity(tail) + if existing, exists := tailMatches[normalizedTail]; exists && !strings.EqualFold(existing, modelID) { + tailMatches[normalizedTail] = "" + continue + } + tailMatches[normalizedTail] = modelID + } + for tail, modelID := range tailMatches { + if modelID != "" { + metadata.modelsDevCanonicalByIdentity[tail] = modelID + } + } + return metadata +} + +func (metadata modelPriceMatchMetadata) modelsDevCanonicalModelID(modelID string) (string, bool) { + modelID = strings.TrimSpace(modelID) + if modelID == "" || len(metadata.modelsDevCanonicalByIdentity) == 0 { + return "", false + } + canonicalID, ok := metadata.modelsDevCanonicalByIdentity[normalizeModelPriceIdentity(modelID)] + return canonicalID, ok +} + +func (metadata modelPriceMatchMetadata) isModelsDevOfficialSourceModelID(sourceModelID string) bool { + _, ok := metadata.modelsDevOfficialSourceModelIDs[normalizeModelPriceIdentity(sourceModelID)] + return ok +} + func New(store *store.Store, syncURL *string, setupResolver ...SetupResolver) *Service { return NewMultiSource(store, syncURL, nil, setupResolver...) } @@ -138,19 +242,19 @@ func newMultiSource( sources = append(sources, priceSyncSource{ Source: SyncSourceModelsDev, URL: modelsDevSyncURL, - Fetch: modelsDevCache.fetch, + Fetch: modelsDevCache.fetchSource, }) } sources = append(sources, priceSyncSource{ Source: SyncSourceLiteLLM, URL: liteLLMSyncURL, - Fetch: fetchLiteLLMModelPrices, + Fetch: wrapModelPriceMapFetcher(fetchLiteLLMModelPrices), }) if openRouterSyncURL != nil { sources = append(sources, priceSyncSource{ Source: SyncSourceOpenRouter, URL: openRouterSyncURL, - Fetch: fetchOpenRouterModelPrices, + Fetch: wrapModelPriceMapFetcher(fetchOpenRouterModelPrices), }) } return &Service{ @@ -205,7 +309,7 @@ func (s *Service) Sync(ctx context.Context, req SyncRequest) (SyncResult, error) if err != nil { return SyncResult{}, err } - selection := selectModelPrices(remotePrices, req.Models) + selection := selectModelPriceCollection(remotePrices, req.Models) preserved := []string(nil) if hasFailedSyncSource(sourceResults) { existingPrices, err := s.store.LoadModelPrices(ctx) @@ -244,18 +348,15 @@ func (s *Service) SyncFromLiteLLM(ctx context.Context, req SyncRequest) (SyncRes return s.Sync(ctx, req) } -func (s *Service) fetchAllModelPrices(ctx context.Context, client *http.Client, models []string) (map[string]store.ModelPrice, int, []string, []SyncSourceResult, error) { - remotePrices := map[string]store.ModelPrice{} - selectedPriorities := map[string]int{} - selectedNormalizedPriorities := map[string]int{} - modelsDevModelIDs := map[string]struct{}{} +func (s *Service) fetchAllModelPrices(ctx context.Context, client *http.Client, models []string) (modelPriceCollection, int, []string, []SyncSourceResult, error) { + remotePrices := modelPriceCollection{} requestedModels := normalizedRequestedModels(models) sources := make([]string, 0, len(s.syncSources)) sourceResults := make([]SyncSourceResult, 0, len(s.syncSources)) failures := []string{} totalSkipped := 0 - for priority, source := range s.syncSources { + for _, source := range s.syncSources { syncURL := source.currentURL() result := SyncSourceResult{Source: source.Source} if syncURL == "" { @@ -269,7 +370,7 @@ func (s *Service) fetchAllModelPrices(ctx context.Context, client *http.Client, if s.syncSourceTimeout > 0 { sourceCtx, cancel = context.WithTimeout(ctx, s.syncSourceTimeout) } - prices, skipped, err := source.Fetch(sourceCtx, syncURL, client) + fetched, skipped, err := source.Fetch(sourceCtx, syncURL, client) cancel() result.Skipped = skipped if err != nil { @@ -278,38 +379,26 @@ func (s *Service) fetchAllModelPrices(ctx context.Context, client *http.Client, failures = append(failures, source.Source+": "+err.Error()) continue } - result.Models = len(prices) + result.Models = len(fetched.Prices) sourceResults = append(sourceResults, result) sources = append(sources, source.Source) totalSkipped += skipped - if source.Source == SyncSourceModelsDev { - modelsDevModelIDs = collectModelsDevModelIDs(prices) - } + remotePrices.Metadata.merge(fetched.Metadata) - for modelID, price := range prices { - normalizedModelID := strings.ToLower(strings.TrimSpace(modelID)) - if source.Source != SyncSourceModelsDev { - if _, blocked := modelsDevModelIDs[normalizedModelID]; blocked { - continue - } - if selectedPriority, exists := selectedNormalizedPriorities[normalizedModelID]; exists && selectedPriority < priority { - continue - } - } + for _, modelID := range sortedPriceKeys(fetched.Prices) { + price := fetched.Prices[modelID] if price.Source == "" { price.Source = source.Source } if price.SourceModelID == "" { price.SourceModelID = modelID } - if _, exists := remotePrices[modelID]; exists && selectedPriorities[modelID] <= priority { - continue - } - remotePrices[modelID] = price - selectedPriorities[modelID] = priority - selectedNormalizedPriorities[normalizedModelID] = priority + remotePrices.Entries = append(remotePrices.Entries, modelPriceSourceEntry{ + Key: modelID, + Price: price, + }) } - if len(requestedModels) > 0 && modelPricesCoverRequested(remotePrices, requestedModels) { + if len(requestedModels) > 0 && modelPriceCollectionCoversRequested(remotePrices, requestedModels) { break } } @@ -318,7 +407,7 @@ func (s *Service) fetchAllModelPrices(ctx context.Context, client *http.Client, if len(failures) == 0 { failures = append(failures, "no price sync sources configured") } - return nil, 0, nil, sourceResults, errors.New("model price sync failed; existing prices were not changed: " + strings.Join(failures, "; ")) + return modelPriceCollection{}, 0, nil, sourceResults, errors.New("model price sync failed; existing prices were not changed: " + strings.Join(failures, "; ")) } return remotePrices, totalSkipped, sources, sourceResults, nil } @@ -447,7 +536,7 @@ type modelsDevPriceCache struct { mu sync.Mutex url string etag string - prices map[string]store.ModelPrice + fetched fetchedModelPriceSource skipped int } @@ -455,51 +544,61 @@ type modelsDevPriceCache struct { // models.dev prices are already expressed as USD per 1M tokens, unlike the // token-level values published by LiteLLM and OpenRouter. func fetchModelsDevModelPrices(ctx context.Context, syncURL string, client *http.Client) (map[string]store.ModelPrice, int, error) { + fetched, skipped, err := fetchModelsDevPriceSource(ctx, syncURL, client) + return fetched.Prices, skipped, err +} + +func fetchModelsDevPriceSource(ctx context.Context, syncURL string, client *http.Client) (fetchedModelPriceSource, int, error) { res, err := fetchModelsDevResponse(ctx, syncURL, client, "") if err != nil { - return nil, 0, err + return fetchedModelPriceSource{}, 0, err } defer res.Body.Close() if res.StatusCode == http.StatusNotModified { - return nil, 0, errors.New("model price sync failed: unexpected 304 Not Modified") + return fetchedModelPriceSource{}, 0, errors.New("model price sync failed: unexpected 304 Not Modified") } - return decodeModelsDevModelPrices(res.Body) + return decodeModelsDevPriceSource(res.Body) } func (cache *modelsDevPriceCache) fetch(ctx context.Context, syncURL string, client *http.Client) (map[string]store.ModelPrice, int, error) { + fetched, skipped, err := cache.fetchSource(ctx, syncURL, client) + return fetched.Prices, skipped, err +} + +func (cache *modelsDevPriceCache) fetchSource(ctx context.Context, syncURL string, client *http.Client) (fetchedModelPriceSource, int, error) { cache.mu.Lock() defer cache.mu.Unlock() if cache.url != syncURL { cache.url = syncURL cache.etag = "" - cache.prices = nil + cache.fetched = fetchedModelPriceSource{} cache.skipped = 0 } res, err := fetchModelsDevResponse(ctx, syncURL, client, cache.etag) if err != nil { - return nil, 0, err + return fetchedModelPriceSource{}, 0, err } defer res.Body.Close() if res.StatusCode == http.StatusNotModified { - if cache.prices == nil { - return nil, 0, errors.New("model price sync failed: models.dev returned 304 without cached prices") + if cache.fetched.Prices == nil { + return fetchedModelPriceSource{}, 0, errors.New("model price sync failed: models.dev returned 304 without cached prices") } if etag := strings.TrimSpace(res.Header.Get("ETag")); etag != "" { cache.etag = etag } - return cache.prices, cache.skipped, nil + return cache.fetched, cache.skipped, nil } - prices, skipped, err := decodeModelsDevModelPrices(res.Body) + fetched, skipped, err := decodeModelsDevPriceSource(res.Body) if err != nil { - return nil, skipped, err + return fetchedModelPriceSource{}, skipped, err } cache.etag = strings.TrimSpace(res.Header.Get("ETag")) - cache.prices = prices + cache.fetched = fetched cache.skipped = skipped - return prices, skipped, nil + return fetched, skipped, nil } func fetchModelsDevResponse(ctx context.Context, syncURL string, client *http.Client, etag string) (*http.Response, error) { @@ -527,18 +626,55 @@ func fetchModelsDevResponse(ctx context.Context, syncURL string, client *http.Cl return res, nil } +type modelsDevRawProvider struct { + Models map[string]json.RawMessage `json:"models"` +} + func decodeModelsDevModelPrices(reader io.Reader) (map[string]store.ModelPrice, int, error) { - var raw map[string]struct { - Models map[string]json.RawMessage `json:"models"` - } + fetched, skipped, err := decodeModelsDevPriceSource(reader) + return fetched.Prices, skipped, err +} + +func decodeModelsDevPriceSource(reader io.Reader) (fetchedModelPriceSource, int, error) { + var root map[string]json.RawMessage decoder := json.NewDecoder(reader) decoder.UseNumber() - if err := decoder.Decode(&raw); err != nil { - return nil, 0, err + if err := decoder.Decode(&root); err != nil { + return fetchedModelPriceSource{}, 0, err + } + + providerMessages := root + canonicalModels := map[string]json.RawMessage(nil) + if rawProviders, hasProviders := root["providers"]; hasProviders { + rawModels, hasModels := root["models"] + if !hasModels { + return fetchedModelPriceSource{}, 0, errors.New("model price sync failed: models.dev catalog contained no canonical models") + } + var catalogProviders map[string]json.RawMessage + if err := json.Unmarshal(rawProviders, &catalogProviders); err != nil { + return fetchedModelPriceSource{}, 0, err + } + if err := json.Unmarshal(rawModels, &canonicalModels); err != nil { + return fetchedModelPriceSource{}, 0, err + } + if len(canonicalModels) == 0 { + return fetchedModelPriceSource{}, 0, errors.New("model price sync failed: models.dev catalog contained no canonical models") + } + providerMessages = catalogProviders + } + + raw := make(map[string]modelsDevRawProvider, len(providerMessages)) + for providerID, message := range providerMessages { + var provider modelsDevRawProvider + if err := json.Unmarshal(message, &provider); err != nil { + return fetchedModelPriceSource{}, 0, err + } + raw[providerID] = provider } now := time.Now().UnixMilli() prices := map[string]store.ModelPrice{} + metadata := newModelsDevMatchMetadata(canonicalModels) skipped := 0 providerIDs := make([]string, 0, len(raw)) for providerID := range raw { @@ -600,13 +736,19 @@ func decodeModelsDevModelPrices(reader io.Reader) (map[string]store.ModelPrice, SyncedAtMS: &now, } prices[sourceModelID] = price + if canonicalID, ok := metadata.modelsDevCanonicalByIdentity[normalizeModelPriceIdentity(sourceModelID)]; ok && strings.EqualFold(canonicalID, sourceModelID) { + if metadata.modelsDevOfficialSourceModelIDs == nil { + metadata.modelsDevOfficialSourceModelIDs = map[string]struct{}{} + } + metadata.modelsDevOfficialSourceModelIDs[normalizeModelPriceIdentity(sourceModelID)] = struct{}{} + } } } if len(prices) == 0 { - return nil, skipped, errors.New("model price sync failed: models.dev catalog contained no usable prices") + return fetchedModelPriceSource{}, skipped, errors.New("model price sync failed: models.dev catalog contained no usable prices") } - return prices, skipped, nil + return fetchedModelPriceSource{Prices: prices, Metadata: metadata}, skipped, nil } func readModelsDevContextTiers(cost map[string]any) []store.ModelPriceContextTier { @@ -700,91 +842,12 @@ func readModelsDevServiceTiers(entry map[string]any) []store.ModelPriceServiceTi return tiers } -type basePriceRule struct { - Prompt float64 `json:"prompt"` - Completion float64 `json:"completion"` - Cache float64 `json:"cache"` - CacheRead float64 `json:"cacheRead"` - CacheCreation float64 `json:"cacheCreation"` - PromptConfigured bool `json:"promptConfigured"` - CompletionConfigured bool `json:"completionConfigured"` - CacheReadConfigured bool `json:"cacheReadConfigured"` - CacheCreationConfigured bool `json:"cacheCreationConfigured"` -} - -func basePriceRuleOf(price store.ModelPrice) basePriceRule { - return basePriceRule{ - Prompt: price.Prompt, - Completion: price.Completion, - Cache: price.Cache, - CacheRead: price.CacheRead, - CacheCreation: price.CacheCreation, - PromptConfigured: price.PromptConfigured, - CompletionConfigured: price.CompletionConfigured, - CacheReadConfigured: price.CacheReadConfigured, - CacheCreationConfigured: price.CacheCreationConfigured, - } -} - -func modelPriceRuleSignature(price store.ModelPrice) string { - if price.Source == SyncSourceModelsDev && price.RawJSON != "" { - if signature, ok := modelsDevPriceRuleSignature(price.RawJSON); ok { - return "models.dev:" + signature - } - } - rule, _ := json.Marshal(basePriceRuleOf(price)) - return "base:" + string(rule) -} - -func modelsDevPriceRuleSignature(rawJSON string) (string, bool) { - var entry map[string]any - if err := json.Unmarshal([]byte(rawJSON), &entry); err != nil { - return "", false - } - signature := map[string]any{} - if cost, ok := entry["cost"]; ok { - signature["cost"] = cost - } - if experimental, ok := entry["experimental"].(map[string]any); ok { - if modes, ok := experimental["modes"].(map[string]any); ok { - modeCosts := map[string]any{} - for modeName, rawMode := range modes { - mode, ok := rawMode.(map[string]any) - if !ok { - continue - } - if cost, ok := mode["cost"]; ok { - modeCosts[modeName] = cost - } - } - if len(modeCosts) > 0 { - signature["experimentalModeCosts"] = modeCosts - } - } - } - encoded, err := json.Marshal(signature) - return string(encoded), err == nil -} - func modelsDevModelID(sourceModelID string) (string, bool) { _, modelID, ok := strings.Cut(strings.TrimSpace(sourceModelID), "/") modelID = strings.TrimSpace(modelID) return modelID, ok && modelID != "" } -func collectModelsDevModelIDs(prices map[string]store.ModelPrice) map[string]struct{} { - modelIDs := map[string]struct{}{} - for _, price := range prices { - if price.Source != SyncSourceModelsDev { - continue - } - if modelID, ok := modelsDevModelID(price.SourceModelID); ok { - modelIDs[strings.ToLower(modelID)] = struct{}{} - } - } - return modelIDs -} - func fetchLiteLLMModelPrices(ctx context.Context, syncURL string, client *http.Client) (map[string]store.ModelPrice, int, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, syncURL, nil) if err != nil { @@ -908,51 +971,88 @@ type priceSelectionResult struct { Unmatched []string } -type modelPriceMatch struct { - Key string - Price store.ModelPrice -} - type modelPriceEntry struct { - key string - price store.ModelPrice - identities []string - signature string - signatureReady bool + key string + price store.ModelPrice + directIdentities []string + aliasIdentities []string } type modelPriceMatcher struct { - entries []modelPriceEntry - exact map[string][]int - caseFold map[string][]int - tail map[string][]int - canonical map[string][]int + entries []modelPriceEntry + exact map[string][]int + caseFold map[string][]int + aliasExact map[string][]int + aliasFold map[string][]int + tail map[string][]int + canonical map[string][]int + sourceEntry map[string][]int + sources []string + metadata modelPriceMatchMetadata } func newModelPriceMatcher(prices map[string]store.ModelPrice) *modelPriceMatcher { - matcher := &modelPriceMatcher{ - entries: make([]modelPriceEntry, 0, len(prices)), - exact: make(map[string][]int, len(prices)), - caseFold: make(map[string][]int, len(prices)), - tail: make(map[string][]int, len(prices)), - canonical: make(map[string][]int, len(prices)), + collection := modelPriceCollection{ + Entries: make([]modelPriceSourceEntry, 0, len(prices)), } for _, key := range sortedPriceKeys(prices) { - price := prices[key] - identities := modelPriceIdentities(key, price) + collection.Entries = append(collection.Entries, modelPriceSourceEntry{ + Key: key, + Price: prices[key], + }) + } + return newModelPriceCollectionMatcher(collection) +} + +func newModelPriceCollectionMatcher(collection modelPriceCollection) *modelPriceMatcher { + matcher := &modelPriceMatcher{ + entries: make([]modelPriceEntry, 0, len(collection.Entries)), + exact: make(map[string][]int, len(collection.Entries)), + caseFold: make(map[string][]int, len(collection.Entries)), + aliasExact: make(map[string][]int, len(collection.Entries)), + aliasFold: make(map[string][]int, len(collection.Entries)), + tail: make(map[string][]int, len(collection.Entries)), + canonical: make(map[string][]int, len(collection.Entries)), + sourceEntry: make(map[string][]int, 4), + metadata: collection.Metadata, + } + for _, sourceEntry := range collection.Entries { + key := sourceEntry.Key + price := sourceEntry.Price + directIdentities, aliasIdentities := modelPriceEntryIdentities(key, price) entryIndex := len(matcher.entries) matcher.entries = append(matcher.entries, modelPriceEntry{ - key: key, - price: price, - identities: identities, + key: key, + price: price, + directIdentities: directIdentities, + aliasIdentities: aliasIdentities, }) - for _, identity := range identities { + source := strings.TrimSpace(price.Source) + if _, exists := matcher.sourceEntry[source]; !exists { + matcher.sources = append(matcher.sources, source) + } + matcher.sourceEntry[source] = append(matcher.sourceEntry[source], entryIndex) + for _, identity := range directIdentities { appendModelPriceIndex(matcher.exact, identity, entryIndex) appendModelPriceIndex(matcher.caseFold, strings.ToLower(identity), entryIndex) appendModelPriceIndex(matcher.tail, canonicalModelTail(identity), entryIndex) appendModelPriceIndex(matcher.canonical, canonicalModelID(identity), entryIndex) } + for _, identity := range aliasIdentities { + appendModelPriceIndex(matcher.aliasExact, identity, entryIndex) + appendModelPriceIndex(matcher.aliasFold, strings.ToLower(identity), entryIndex) + appendModelPriceIndex(matcher.tail, canonicalModelTail(identity), entryIndex) + appendModelPriceIndex(matcher.canonical, canonicalModelID(identity), entryIndex) + } } + sort.SliceStable(matcher.sources, func(i, j int) bool { + leftPriority := modelPriceSourcePriority(matcher.sources[i]) + rightPriority := modelPriceSourcePriority(matcher.sources[j]) + if leftPriority != rightPriority { + return leftPriority < rightPriority + } + return matcher.sources[i] < matcher.sources[j] + }) return matcher } @@ -968,11 +1068,24 @@ func appendModelPriceIndex(index map[string][]int, identity string, entryIndex i } func selectModelPrices(prices map[string]store.ModelPrice, models []string) priceSelectionResult { + collection := modelPriceCollection{ + Entries: make([]modelPriceSourceEntry, 0, len(prices)), + } + for _, key := range sortedPriceKeys(prices) { + collection.Entries = append(collection.Entries, modelPriceSourceEntry{ + Key: key, + Price: prices[key], + }) + } + return selectModelPriceCollection(collection, models) +} + +func selectModelPriceCollection(collection modelPriceCollection, models []string) priceSelectionResult { result := priceSelectionResult{ Prices: map[string]store.ModelPrice{}, Matched: map[string]store.ModelPrice{}, } - matcher := newModelPriceMatcher(prices) + matcher := newModelPriceCollectionMatcher(collection) if len(models) == 0 { return matcher.selectAllUnambiguousModelPrices() } @@ -983,18 +1096,13 @@ func selectModelPrices(prices map[string]store.ModelPrice, models []string) pric continue } seen[normalized] = true - price, _, ok, indexedMatches := matcher.findAutomaticModelPrice(normalized) + price, _, ok, _ := matcher.findAutomaticModelPrice(normalized) if ok { result.Prices[normalized] = price result.Matched[normalized] = price continue } - var candidates []SyncCandidate - if len(indexedMatches) > 0 { - candidates = matcher.candidateModelPricesForIndexes(normalized, indexedMatches) - } else { - candidates = matcher.findCandidateModelPrices(normalized) - } + candidates := matcher.findCandidateModelPrices(normalized) if len(candidates) > 0 { result.Candidates = append(result.Candidates, SyncCandidateSet{ Model: normalized, @@ -1033,7 +1141,7 @@ func (matcher *modelPriceMatcher) selectAllUnambiguousModelPrices() priceSelecti } sort.Strings(orderedModelIDs) for _, modelID := range orderedModelIDs { - price, ok := matcher.selectUnambiguousModelPrice(matcher.exact[modelID]) + price, _, ok, _ := matcher.findAutomaticModelPrice(modelID) if !ok { continue } @@ -1049,130 +1157,118 @@ func findAutomaticModelPrice(prices map[string]store.ModelPrice, modelID string) } func (matcher *modelPriceMatcher) findAutomaticModelPrice(modelID string) (store.ModelPrice, string, bool, []int) { - matches, reason := matcher.indexedModelPriceMatches(modelID) - if len(matches) == 0 { + modelID = strings.TrimSpace(modelID) + if modelID == "" { return store.ModelPrice{}, "", false, nil } - price, ok := matcher.selectUnambiguousModelPrice(matches) - return price, reason, ok, matches + allMatches := make([]int, 0) + for _, source := range matcher.sources { + matches, reason := matcher.indexedModelPriceMatchesForSource(modelID, source) + allMatches = appendUniqueModelPriceIndexes(allMatches, matches...) + if source == SyncSourceModelsDev && len(matcher.metadata.modelsDevCanonicalByIdentity) > 0 { + if officialMatches := matcher.modelsDevOfficialMatches(modelID); len(officialMatches) == 1 { + entry := matcher.entries[officialMatches[0]] + return entry.price, "models.dev-official", true, allMatches + } + if strings.Contains(modelID, "/") { + directMatches, directReason := matcher.directModelPriceMatchesForSource(modelID, source) + allMatches = appendUniqueModelPriceIndexes(allMatches, directMatches...) + if len(directMatches) == 1 { + return matcher.entries[directMatches[0]].price, directReason, true, allMatches + } + } + continue + } + if len(matches) == 1 { + return matcher.entries[matches[0]].price, reason, true, allMatches + } + } + return store.ModelPrice{}, "", false, allMatches } -func (matcher *modelPriceMatcher) indexedModelPriceMatches(modelID string) ([]int, string) { +func (matcher *modelPriceMatcher) indexedModelPriceMatchesForSource(modelID string, source string) ([]int, string) { modelID = strings.TrimSpace(modelID) if modelID == "" { return nil, "" } - if matches := matcher.exact[modelID]; len(matches) > 0 { + if matches := matcher.filterIndexesBySource(matcher.exact[modelID], source); len(matches) > 0 { return matches, "exact" } - if matches := matcher.caseFold[strings.ToLower(modelID)]; len(matches) > 0 { + if matches := matcher.filterIndexesBySource(matcher.caseFold[strings.ToLower(modelID)], source); len(matches) > 0 { return matches, "case-insensitive" } + if matches := matcher.filterIndexesBySource(matcher.aliasExact[modelID], source); len(matches) > 0 { + return matches, "source-model-id" + } + if matches := matcher.filterIndexesBySource(matcher.aliasFold[strings.ToLower(modelID)], source); len(matches) > 0 { + return matches, "case-insensitive-source-model-id" + } modelTail := canonicalModelTail(modelID) if modelTail != "" { - if matches := matcher.tail[modelTail]; len(matches) > 0 { + if matches := matcher.filterIndexesBySource(matcher.tail[modelTail], source); len(matches) > 0 { return matches, "provider-prefix" } } modelCanonical := canonicalModelID(modelID) if modelCanonical != "" { - if matches := matcher.canonical[modelCanonical]; len(matches) > 0 { + if matches := matcher.filterIndexesBySource(matcher.canonical[modelCanonical], source); len(matches) > 0 { return matches, "normalized" } } return nil, "" } -func findModelPriceMatches(prices map[string]store.ModelPrice, matchIdentity func(string) bool) []modelPriceMatch { - matches := []modelPriceMatch{} - for _, key := range sortedPriceKeys(prices) { - price := prices[key] - for _, identity := range modelPriceIdentities(key, price) { - if !matchIdentity(identity) { - continue - } - matches = append(matches, modelPriceMatch{Key: key, Price: price}) - break - } - } - return matches -} - -func selectUnambiguousModelPrice(matches []modelPriceMatch) (store.ModelPrice, bool) { - if len(matches) == 0 { - return store.ModelPrice{}, false - } - if len(matches) == 1 { - return matches[0].Price, true +func (matcher *modelPriceMatcher) directModelPriceMatchesForSource(modelID string, source string) ([]int, string) { + if matches := matcher.filterIndexesBySource(matcher.exact[modelID], source); len(matches) > 0 { + return matches, "exact" } - signature := modelPriceRuleSignature(matches[0].Price) - selected := matches[0] - for _, match := range matches[1:] { - if modelPriceRuleSignature(match.Price) != signature { - return store.ModelPrice{}, false - } - if modelPriceMatchLess(match, selected) { - selected = match - } + if matches := matcher.filterIndexesBySource(matcher.caseFold[strings.ToLower(modelID)], source); len(matches) > 0 { + return matches, "case-insensitive" } - return selected.Price, true + return nil, "" } -func modelPriceMatchLess(left modelPriceMatch, right modelPriceMatch) bool { - leftPriority := modelPriceSourcePriority(left.Price.Source) - rightPriority := modelPriceSourcePriority(right.Price.Source) - if leftPriority != rightPriority { - return leftPriority < rightPriority - } - leftID := strings.TrimSpace(left.Price.SourceModelID) - rightID := strings.TrimSpace(right.Price.SourceModelID) - if leftID != rightID { - return leftID < rightID +func (matcher *modelPriceMatcher) filterIndexesBySource(indexes []int, source string) []int { + filtered := make([]int, 0, len(indexes)) + for _, entryIndex := range indexes { + if strings.TrimSpace(matcher.entries[entryIndex].price.Source) == source { + filtered = appendUniqueModelPriceIndexes(filtered, entryIndex) + } } - return left.Key < right.Key + return filtered } -func (matcher *modelPriceMatcher) selectUnambiguousModelPrice(matches []int) (store.ModelPrice, bool) { - if len(matches) == 0 { - return store.ModelPrice{}, false - } - if len(matches) == 1 { - return matcher.entries[matches[0]].price, true - } - signature := matcher.modelPriceRuleSignature(matches[0]) - selected := matches[0] - for _, entryIndex := range matches[1:] { - if matcher.modelPriceRuleSignature(entryIndex) != signature { - return store.ModelPrice{}, false +func appendUniqueModelPriceIndexes(indexes []int, additions ...int) []int { + for _, addition := range additions { + exists := false + for _, existing := range indexes { + if existing == addition { + exists = true + break + } } - if modelPriceEntryLess(&matcher.entries[entryIndex], &matcher.entries[selected]) { - selected = entryIndex + if !exists { + indexes = append(indexes, addition) } } - return matcher.entries[selected].price, true -} - -func (matcher *modelPriceMatcher) modelPriceRuleSignature(entryIndex int) string { - entry := &matcher.entries[entryIndex] - if !entry.signatureReady { - entry.signature = modelPriceRuleSignature(entry.price) - entry.signatureReady = true - } - return entry.signature + return indexes } -func modelPriceEntryLess(left *modelPriceEntry, right *modelPriceEntry) bool { - leftPriority := modelPriceSourcePriority(left.price.Source) - rightPriority := modelPriceSourcePriority(right.price.Source) - if leftPriority != rightPriority { - return leftPriority < rightPriority +func (matcher *modelPriceMatcher) modelsDevOfficialMatches(modelID string) []int { + canonicalID, ok := matcher.metadata.modelsDevCanonicalModelID(modelID) + if !ok { + return nil } - leftID := strings.TrimSpace(left.price.SourceModelID) - rightID := strings.TrimSpace(right.price.SourceModelID) - if leftID != rightID { - return leftID < rightID + matches := make([]int, 0, 1) + for _, entryIndex := range matcher.sourceEntry[SyncSourceModelsDev] { + entry := &matcher.entries[entryIndex] + sourceModelID := strings.TrimSpace(entry.price.SourceModelID) + if !matcher.metadata.isModelsDevOfficialSourceModelID(sourceModelID) || !strings.EqualFold(sourceModelID, canonicalID) { + continue + } + matches = append(matches, entryIndex) } - return left.key < right.key + return matches } func modelPriceSourcePriority(source string) int { @@ -1188,28 +1284,29 @@ func modelPriceSourcePriority(source string) int { } } -func modelPriceIdentities(key string, price store.ModelPrice) []string { - identities := make([]string, 0, 3) - add := func(identity string) { +func modelPriceEntryIdentities(key string, price store.ModelPrice) ([]string, []string) { + direct := make([]string, 0, 2) + aliases := make([]string, 0, 1) + add := func(target *[]string, identity string) { identity = strings.TrimSpace(identity) if identity == "" { return } - for _, existing := range identities { + for _, existing := range *target { if existing == identity { return } } - identities = append(identities, identity) + *target = append(*target, identity) } - add(key) - add(price.SourceModelID) + add(&direct, key) + add(&direct, price.SourceModelID) if price.Source == SyncSourceModelsDev { if modelID, ok := modelsDevModelID(price.SourceModelID); ok { - add(modelID) + add(&aliases, modelID) } } - return identities + return direct, aliases } func findCandidateModelPrices(prices map[string]store.ModelPrice, modelID string) []SyncCandidate { @@ -1217,11 +1314,16 @@ func findCandidateModelPrices(prices map[string]store.ModelPrice, modelID string } func (matcher *modelPriceMatcher) findCandidateModelPrices(modelID string) []SyncCandidate { - candidates := make([]SyncCandidate, 0, maxSyncCandidates) - for entryIndex := range matcher.entries { - candidates = appendModelPriceCandidate(candidates, modelID, &matcher.entries[entryIndex]) + candidates := make([]SyncCandidate, 0, maxSyncCandidates*len(matcher.sources)) + for _, source := range matcher.sources { + indexes, _ := matcher.indexedModelPriceMatchesForSource(modelID, source) + if len(indexes) == 0 { + indexes = matcher.sourceEntry[source] + } + sourceCandidates := matcher.candidateModelPricesForIndexes(modelID, indexes) + candidates = append(candidates, sourceCandidates...) } - return sortAndLimitModelPriceCandidates(candidates) + return candidates } func (matcher *modelPriceMatcher) candidateModelPricesForIndexes(modelID string, indexes []int) []SyncCandidate { @@ -1229,19 +1331,27 @@ func (matcher *modelPriceMatcher) candidateModelPricesForIndexes(modelID string, for _, entryIndex := range indexes { candidates = appendModelPriceCandidate(candidates, modelID, &matcher.entries[entryIndex]) } - return sortAndLimitModelPriceCandidates(candidates) + return matcher.sortAndLimitModelPriceCandidates(candidates) } func appendModelPriceCandidate(candidates []SyncCandidate, modelID string, entry *modelPriceEntry) []SyncCandidate { score := 0.0 reason := "" - for _, identity := range entry.identities { + for _, identity := range entry.directIdentities { candidateScore, candidateReason := modelIdentitySimilarity(modelID, identity) if candidateScore > score { score = candidateScore reason = candidateReason } } + for _, identity := range entry.aliasIdentities { + candidateScore, _ := modelIdentitySimilarity(modelID, identity) + candidateScore = math.Min(candidateScore, 0.94) + if candidateScore > score { + score = candidateScore + reason = "same-model-with-provider-prefix" + } + } if score < minCandidateScore && !(score >= minWeakCandidateScore && isWeakRecallReason(reason)) { return candidates } @@ -1257,8 +1367,13 @@ func appendModelPriceCandidate(candidates []SyncCandidate, modelID string, entry }) } -func sortAndLimitModelPriceCandidates(candidates []SyncCandidate) []SyncCandidate { +func (matcher *modelPriceMatcher) sortAndLimitModelPriceCandidates(candidates []SyncCandidate) []SyncCandidate { sort.SliceStable(candidates, func(i, j int) bool { + leftOfficial := candidates[i].Price.Source == SyncSourceModelsDev && matcher.metadata.isModelsDevOfficialSourceModelID(candidates[i].SourceModelID) + rightOfficial := candidates[j].Price.Source == SyncSourceModelsDev && matcher.metadata.isModelsDevOfficialSourceModelID(candidates[j].SourceModelID) + if leftOfficial != rightOfficial { + return leftOfficial + } if candidates[i].Score == candidates[j].Score { return candidates[i].SourceModelID < candidates[j].SourceModelID } @@ -1287,14 +1402,13 @@ func normalizedRequestedModels(models []string) []string { return normalized } -func modelPricesCoverRequested(prices map[string]store.ModelPrice, models []string) bool { +func modelPriceCollectionCoversRequested(collection modelPriceCollection, models []string) bool { if len(models) == 0 { return false } - matcher := newModelPriceMatcher(prices) + matcher := newModelPriceCollectionMatcher(collection) for _, modelID := range models { - matches, _ := matcher.indexedModelPriceMatches(modelID) - if len(matches) == 0 { + if _, _, ok, _ := matcher.findAutomaticModelPrice(modelID); !ok { return false } } diff --git a/apps/manager-server/internal/service/modelprice/service_test.go b/apps/manager-server/internal/service/modelprice/service_test.go index 30a73cff..9a3b3a27 100644 --- a/apps/manager-server/internal/service/modelprice/service_test.go +++ b/apps/manager-server/internal/service/modelprice/service_test.go @@ -2,6 +2,8 @@ package modelprice import ( "context" + "encoding/json" + "fmt" "net/http" "net/http/httptest" "strings" @@ -87,15 +89,51 @@ func TestFetchModelsDevModelPrices(t *testing.T) { if !ok || unique.SourceModelID != "provider-a/unique-model" { t.Fatalf("unique alias = %#v", unique) } - sameRule, ok := selection.Prices["same-rule"] - if !ok || sameRule.SourceModelID != "provider-b/same-rule" { - t.Fatalf("same-rule alias = %#v", sameRule) + if _, ok := selection.Prices["same-rule"]; ok || + !hasCandidate(selection, "same-rule", "provider-b/same-rule") || + !hasCandidate(selection, "same-rule", "provider-c/same-rule") { + t.Fatalf("same-rule ambiguity = %#v", selection) } - if len(selection.Candidates) != 0 || len(selection.Unmatched) != 0 { + if len(selection.Candidates) != 1 || len(selection.Unmatched) != 0 { t.Fatalf("unexpected selection result = %#v", selection) } } +func TestDecodeModelsDevCatalogSelectsCanonicalOfficialPrice(t *testing.T) { + fetched, skipped, err := decodeModelsDevPriceSource(strings.NewReader(`{ + "models": { + "openai/gpt-5.5": {"name":"GPT 5.5"} + }, + "providers": { + "abacus": {"models": {"gpt-5.5": {"cost":{"input":9,"output":18}}}}, + "openai": {"models": {"gpt-5.5": {"cost":{"input":1,"output":2}}}}, + "third-party": {"models": {"gpt-5.5": {"cost":{"input":7,"output":14}}}} + } + }`)) + if err != nil { + t.Fatalf("decode models.dev catalog: %v", err) + } + if skipped != 0 { + t.Fatalf("skipped = %d", skipped) + } + + collection := collectionFromFetchedSource(fetched) + selection := selectModelPriceCollection(collection, []string{"gpt-5.5"}) + price, ok := selection.Prices["gpt-5.5"] + if !ok || price.Source != SyncSourceModelsDev || price.SourceModelID != "openai/gpt-5.5" || price.Prompt != 1 { + t.Fatalf("official selection = %#v", selection) + } + if len(selection.Candidates) != 0 || len(selection.Unmatched) != 0 { + t.Fatalf("official selection required confirmation: %#v", selection) + } + + scoped := selectModelPriceCollection(collection, []string{"abacus/gpt-5.5"}) + price, ok = scoped.Prices["abacus/gpt-5.5"] + if !ok || price.SourceModelID != "abacus/gpt-5.5" || price.Prompt != 9 { + t.Fatalf("explicit provider selection = %#v", scoped) + } +} + func TestDecodeModelsDevContextTiersPreservesConfiguredZerosAndIgnoresUnsafeRules(t *testing.T) { prices, skipped, err := decodeModelsDevModelPrices(strings.NewReader(`{ "provider-a":{"models":{ @@ -164,6 +202,18 @@ func TestDecodeModelsDevRejectsCatalogWithoutUsablePrices(t *testing.T) { } } +func TestDecodeModelsDevRejectsCatalogWithoutCanonicalModels(t *testing.T) { + for _, payload := range []string{ + `{"providers":{"abacus":{"models":{"gpt-test":{"cost":{"input":1}}}}}}`, + `{"models":null,"providers":{"abacus":{"models":{"gpt-test":{"cost":{"input":1}}}}}}`, + } { + fetched, _, err := decodeModelsDevPriceSource(strings.NewReader(payload)) + if err == nil || !strings.Contains(err.Error(), "no canonical models") { + t.Fatalf("decode error = %v, fetched = %#v", err, fetched) + } + } +} + func TestPriceMutationsNotifyPricingRollup(t *testing.T) { ctx := context.Background() st := testutil.NewStore(t, testutil.NewConfig(t)) @@ -394,7 +444,8 @@ func TestModelsDevCacheFailureFallsBackWithoutStalePrices(t *testing.T) { if err != nil { t.Fatalf("prime models.dev source: %v", err) } - if len(sources) != 1 || sources[0] != SyncSourceModelsDev || prices["openai/gpt-test"].Prompt != 9 { + price, ok := collectionPrice(prices, SyncSourceModelsDev, "openai/gpt-test") + if len(sources) != 1 || sources[0] != SyncSourceModelsDev || !ok || price.Prompt != 9 { t.Fatalf("primed sources = %#v, prices = %#v", sources, prices) } if got := liteLLMRequests.Load(); got != 0 { @@ -412,11 +463,11 @@ func TestModelsDevCacheFailureFallsBackWithoutStalePrices(t *testing.T) { if len(sourceResults) != 2 || sourceResults[0].Source != SyncSourceModelsDev || sourceResults[0].Error == "" || sourceResults[1].Source != SyncSourceLiteLLM { t.Fatalf("fallback source results = %#v", sourceResults) } - price := prices["gpt-test"] - if price.Source != SyncSourceLiteLLM || price.Prompt != 1 { + price, ok = collectionPrice(prices, SyncSourceLiteLLM, "gpt-test") + if !ok || price.Source != SyncSourceLiteLLM || price.Prompt != 1 { t.Fatalf("fallback price = %#v", price) } - if _, exists := prices["openai/gpt-test"]; exists { + if _, exists := collectionPrice(prices, SyncSourceModelsDev, "openai/gpt-test"); exists { t.Fatalf("stale models.dev price was reused: %#v", prices) } } @@ -456,7 +507,7 @@ func TestFetchAllModelPricesFallsBackWhenPreferredSourceHangs(t *testing.T) { sourceResults[0].Error == "" || sourceResults[1].Source != SyncSourceLiteLLM { t.Fatalf("fallback source results = %#v", sourceResults) } - if price := prices["gpt-test"]; price.Source != SyncSourceLiteLLM || price.Prompt != 1 { + if price, ok := collectionPrice(prices, SyncSourceLiteLLM, "gpt-test"); !ok || price.Source != SyncSourceLiteLLM || price.Prompt != 1 { t.Fatalf("fallback price = %#v", price) } } @@ -671,7 +722,7 @@ func TestPreserveFailedSourcePricesReportsOnlyRequestedModels(t *testing.T) { } } -func TestSelectModelPricesRequiresConfirmationForScopedIdentityCollision(t *testing.T) { +func TestSelectModelPricesPrefersDirectScopedIdentity(t *testing.T) { prices := map[string]store.ModelPrice{ "openai/gpt-test": { Prompt: 1, @@ -692,13 +743,9 @@ func TestSelectModelPricesRequiresConfirmationForScopedIdentityCollision(t *test } selection := selectModelPrices(prices, []string{"openai/gpt-test"}) - if len(selection.Prices) != 0 || len(selection.Candidates) != 1 { + if selection.Prices["openai/gpt-test"].SourceModelID != "openai/gpt-test" || len(selection.Candidates) != 0 { t.Fatalf("collision selection = %#v", selection) } - if !hasCandidate(selection, "openai/gpt-test", "openai/gpt-test") || - !hasCandidate(selection, "openai/gpt-test", "crossmodel/openai/gpt-test") { - t.Fatalf("collision candidates = %#v", selection.Candidates) - } scoped := selectModelPrices(prices, []string{"crossmodel/openai/gpt-test"}) if scoped.Prices["crossmodel/openai/gpt-test"].SourceModelID != "crossmodel/openai/gpt-test" { @@ -706,8 +753,8 @@ func TestSelectModelPricesRequiresConfirmationForScopedIdentityCollision(t *test } all := selectModelPrices(prices, nil) - if _, ok := all.Prices["openai/gpt-test"]; ok { - t.Fatalf("unsafe colliding alias imported by empty sync: %#v", all.Prices) + if all.Prices["openai/gpt-test"].SourceModelID != "openai/gpt-test" { + t.Fatalf("direct scoped identity missing from empty sync: %#v", all.Prices) } if all.Prices["gpt-test"].SourceModelID != "openai/gpt-test" { t.Fatalf("safe direct alias missing from empty sync: %#v", all.Prices) @@ -744,7 +791,7 @@ func TestSelectModelPricesTreatsAdvancedPricingDifferencesAsAmbiguous(t *testing } } -func TestModelsDevAmbiguityBlocksLowerPriorityBareFallback(t *testing.T) { +func TestModelsDevAmbiguityContinuesToLowerPriorityFallback(t *testing.T) { modelsDev := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{ @@ -769,14 +816,119 @@ func TestModelsDevAmbiguityBlocksLowerPriorityBareFallback(t *testing.T) { if len(sources) != 2 || sources[0] != SyncSourceModelsDev || sources[1] != SyncSourceLiteLLM { t.Fatalf("sources = %#v", sources) } - if _, ok := prices["shared"]; ok { - t.Fatalf("lower-priority bare fallback bypassed ambiguity protection: %#v", prices["shared"]) + selection := selectModelPriceCollection(prices, []string{"shared"}) + if price, ok := selection.Prices["shared"]; !ok || price.Source != SyncSourceLiteLLM || price.Prompt != 9 { + t.Fatalf("lower-priority fallback selection = %#v", selection) } - if _, ok := prices["fallback-only"]; !ok { + if _, ok := collectionPrice(prices, SyncSourceLiteLLM, "fallback-only"); !ok { t.Fatalf("unrelated fallback model missing: %#v", prices) } } +func TestLiteLLMAmbiguityContinuesToOpenRouter(t *testing.T) { + syncURL := "https://example.test/prices" + service := &Service{syncSources: []priceSyncSource{ + { + Source: SyncSourceModelsDev, + URL: &syncURL, + Fetch: wrapModelPriceMapFetcher(func(context.Context, string, *http.Client) (map[string]store.ModelPrice, int, error) { + return map[string]store.ModelPrice{}, 0, nil + }), + }, + { + Source: SyncSourceLiteLLM, + URL: &syncURL, + Fetch: wrapModelPriceMapFetcher(func(context.Context, string, *http.Client) (map[string]store.ModelPrice, int, error) { + return map[string]store.ModelPrice{ + "provider-a/shared": {Prompt: 1, SourceModelID: "provider-a/shared"}, + "provider-b/shared": {Prompt: 2, SourceModelID: "provider-b/shared"}, + }, 0, nil + }), + }, + { + Source: SyncSourceOpenRouter, + URL: &syncURL, + Fetch: wrapModelPriceMapFetcher(func(context.Context, string, *http.Client) (map[string]store.ModelPrice, int, error) { + return map[string]store.ModelPrice{ + "shared": {Prompt: 3, SourceModelID: "shared"}, + }, 0, nil + }), + }, + }} + + prices, _, sources, _, err := service.fetchAllModelPrices(context.Background(), nil, []string{"shared"}) + if err != nil { + t.Fatalf("fetch model prices: %v", err) + } + if got := strings.Join(sources, ","); got != "models.dev,litellm,openrouter" { + t.Fatalf("sources = %q", got) + } + selection := selectModelPriceCollection(prices, []string{"shared"}) + price, ok := selection.Prices["shared"] + if !ok || price.Source != SyncSourceOpenRouter || price.SourceModelID != "shared" || price.Prompt != 3 { + t.Fatalf("OpenRouter fallback selection = %#v", selection) + } +} + +func TestSelectModelPriceCollectionKeepsCandidatesFromEverySource(t *testing.T) { + metadata := newModelsDevMatchMetadata(map[string]json.RawMessage{ + "openai/gpt-5.5": json.RawMessage(`{}`), + }) + metadata.modelsDevOfficialSourceModelIDs = map[string]struct{}{ + normalizeModelPriceIdentity("openai/gpt-5.5"): {}, + } + collection := modelPriceCollection{ + Metadata: metadata, + Entries: []modelPriceSourceEntry{ + { + Key: "openai/gpt-5.5", + Price: store.ModelPrice{ + Prompt: 2, Source: SyncSourceLiteLLM, SourceModelID: "openai/gpt-5.5", + }, + }, + { + Key: "openai/gpt-5.5", + Price: store.ModelPrice{ + Prompt: 3, Source: SyncSourceOpenRouter, SourceModelID: "openai/gpt-5.5", + }, + }, + }, + } + for index := range 10 { + sourceModelID := fmt.Sprintf("provider-%02d/gpt-5.5", index) + collection.Entries = append(collection.Entries, modelPriceSourceEntry{ + Key: sourceModelID, + Price: store.ModelPrice{ + Prompt: float64(index + 10), Source: SyncSourceModelsDev, SourceModelID: sourceModelID, + }, + }) + } + collection.Entries = append(collection.Entries, modelPriceSourceEntry{ + Key: "openai/gpt-5.5", + Price: store.ModelPrice{ + Prompt: 1, Source: SyncSourceModelsDev, SourceModelID: "openai/gpt-5.5", + }, + }) + + selection := selectModelPriceCollection(collection, []string{"gpt-5.5-latest"}) + if len(selection.Prices) != 0 || len(selection.Candidates) != 1 || len(selection.Unmatched) != 0 { + t.Fatalf("fuzzy selection = %#v", selection) + } + candidates := selection.Candidates[0].Candidates + if len(candidates) != 10 { + t.Fatalf("candidates = %#v", candidates) + } + if candidates[0].Price.Source != SyncSourceModelsDev || candidates[0].SourceModelID != "openai/gpt-5.5" { + t.Fatalf("official models.dev candidate was not prioritized: %#v", candidates) + } + if got := candidateSources(candidates, "openai/gpt-5.5"); strings.Join(got, ",") != "models.dev,litellm,openrouter" { + t.Fatalf("cross-source candidate identities = %#v", candidates) + } + if count := candidateSourceCount(candidates, SyncSourceModelsDev); count != maxSyncCandidates { + t.Fatalf("models.dev candidate count = %d, want %d: %#v", count, maxSyncCandidates, candidates) + } +} + func TestFetchAllModelPricesStopsAfterRequestedModelsAreCovered(t *testing.T) { modelsDevPrices := map[string]store.ModelPrice{ "provider-a/primary": { @@ -824,7 +976,7 @@ func TestFetchAllModelPricesStopsAfterRequestedModelsAreCovered(t *testing.T) { wantCalls [3]int32 }{ {name: "models.dev coverage", models: []string{"primary"}, wantSources: SyncSourceModelsDev, wantCalls: [3]int32{1, 0, 0}}, - {name: "models.dev ambiguity", models: []string{"shared"}, wantSources: SyncSourceModelsDev, wantCalls: [3]int32{1, 0, 0}}, + {name: "models.dev ambiguity", models: []string{"shared"}, wantSources: SyncSourceModelsDev + "," + SyncSourceLiteLLM + "," + SyncSourceOpenRouter, wantCalls: [3]int32{1, 1, 1}}, {name: "LiteLLM completes coverage", models: []string{"primary", "lite-only"}, wantSources: SyncSourceModelsDev + "," + SyncSourceLiteLLM, wantCalls: [3]int32{1, 1, 0}}, {name: "OpenRouter still required", models: []string{"router-only"}, wantSources: SyncSourceModelsDev + "," + SyncSourceLiteLLM + "," + SyncSourceOpenRouter, wantCalls: [3]int32{1, 1, 1}}, {name: "empty request fetches all", models: nil, wantSources: SyncSourceModelsDev + "," + SyncSourceLiteLLM + "," + SyncSourceOpenRouter, wantCalls: [3]int32{1, 1, 1}}, @@ -838,26 +990,26 @@ func TestFetchAllModelPricesStopsAfterRequestedModelsAreCovered(t *testing.T) { { Source: SyncSourceModelsDev, URL: &syncURL, - Fetch: func(context.Context, string, *http.Client) (map[string]store.ModelPrice, int, error) { + Fetch: wrapModelPriceMapFetcher(func(context.Context, string, *http.Client) (map[string]store.ModelPrice, int, error) { calls[0].Add(1) return modelsDevPrices, 0, nil - }, + }), }, { Source: SyncSourceLiteLLM, URL: &syncURL, - Fetch: func(context.Context, string, *http.Client) (map[string]store.ModelPrice, int, error) { + Fetch: wrapModelPriceMapFetcher(func(context.Context, string, *http.Client) (map[string]store.ModelPrice, int, error) { calls[1].Add(1) return liteLLMPrices, 0, nil - }, + }), }, { Source: SyncSourceOpenRouter, URL: &syncURL, - Fetch: func(context.Context, string, *http.Client) (map[string]store.ModelPrice, int, error) { + Fetch: wrapModelPriceMapFetcher(func(context.Context, string, *http.Client) (map[string]store.ModelPrice, int, error) { calls[2].Add(1) return openRouterPrices, 0, nil - }, + }), }, }} @@ -877,7 +1029,7 @@ func TestFetchAllModelPricesStopsAfterRequestedModelsAreCovered(t *testing.T) { } } if test.name == "models.dev ambiguity" { - selection := selectModelPrices(prices, test.models) + selection := selectModelPriceCollection(prices, test.models) if len(selection.Prices) != 0 || len(selection.Candidates) != 1 { t.Fatalf("ambiguity selection = %#v", selection) } @@ -1078,6 +1230,46 @@ func hasCandidate(selection priceSelectionResult, model string, sourceModelID st return false } +func collectionPrice(collection modelPriceCollection, source string, sourceModelID string) (store.ModelPrice, bool) { + for _, entry := range collection.Entries { + if entry.Price.Source == source && entry.Price.SourceModelID == sourceModelID { + return entry.Price, true + } + } + return store.ModelPrice{}, false +} + +func collectionFromFetchedSource(fetched fetchedModelPriceSource) modelPriceCollection { + collection := modelPriceCollection{Metadata: fetched.Metadata} + for _, key := range sortedPriceKeys(fetched.Prices) { + collection.Entries = append(collection.Entries, modelPriceSourceEntry{ + Key: key, + Price: fetched.Prices[key], + }) + } + return collection +} + +func candidateSources(candidates []SyncCandidate, sourceModelID string) []string { + sources := make([]string, 0) + for _, candidate := range candidates { + if candidate.SourceModelID == sourceModelID { + sources = append(sources, candidate.Price.Source) + } + } + return sources +} + +func candidateSourceCount(candidates []SyncCandidate, source string) int { + count := 0 + for _, candidate := range candidates { + if candidate.Price.Source == source { + count++ + } + } + return count +} + func closePrice(left float64, right float64) bool { if left > right { return left-right < 0.0000001 diff --git a/apps/web/src/features/monitoring/ModelPricesPage.tsx b/apps/web/src/features/monitoring/ModelPricesPage.tsx index aa6d43d6..01641ccf 100644 --- a/apps/web/src/features/monitoring/ModelPricesPage.tsx +++ b/apps/web/src/features/monitoring/ModelPricesPage.tsx @@ -25,6 +25,8 @@ import { formatContextThreshold, formatPriceUnit, formatServiceTierRule, + getModelPriceCandidateIdentity, + groupModelPriceCandidatesBySource, resolveContextTierDisplayPrice, resolveServiceTierDisplayPrice, type ModelPriceFilter, @@ -405,11 +407,17 @@ export function ModelPricesPage() { const candidates = candidateSets.find((candidateSet) => candidateSet.model === row.model) ?.candidates ?? []; - const selectedSource = - selectedCandidates[row.model] || candidates[0]?.sourceModelId || ''; + const candidateGroups = groupModelPriceCandidatesBySource(candidates); + const requestedCandidateIdentity = selectedCandidates[row.model] || ''; const selectedCandidate = - candidates.find((candidate) => candidate.sourceModelId === selectedSource) ?? + candidates.find( + (candidate) => + getModelPriceCandidateIdentity(candidate) === requestedCandidateIdentity + ) ?? candidates[0]; + const selectedCandidateIdentity = selectedCandidate + ? getModelPriceCandidateIdentity(selectedCandidate) + : ''; const contextTiers = row.price?.contextTiers ?? []; const serviceTiers = row.price?.serviceTiers ?? []; @@ -528,7 +536,7 @@ export function ModelPricesPage() { ) : candidates.length > 0 && selectedCandidate ? (