From f424eedb596c8abca64c95cd8170775956b68a2e Mon Sep 17 00:00:00 2001 From: seakee Date: Thu, 30 Jul 2026 12:09:54 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=90=9B=20fix(manager-server):=20prior?= =?UTF-8?q?itize=20canonical=20model=20prices?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read models.dev catalog metadata and select canonical first-party prices before ordered LiteLLM and OpenRouter fallbacks. Keep ambiguous and fuzzy matches out of automatic persistence while preserving candidates from every source. Reject incomplete catalogs and retain cache metadata so third-party entries cannot silently become official prices. --- .../internal/httpapi/server_test.go | 56 +- .../internal/service/modelprice/service.go | 662 ++++++++++-------- .../service/modelprice/service_test.go | 250 ++++++- 3 files changed, 637 insertions(+), 331 deletions(-) diff --git a/apps/manager-server/internal/httpapi/server_test.go b/apps/manager-server/internal/httpapi/server_test.go index 5de8c6e0..ea6492b4 100644 --- a/apps/manager-server/internal/httpapi/server_test.go +++ b/apps/manager-server/internal/httpapi/server_test.go @@ -746,20 +746,25 @@ func TestModelPricesSyncFromLiteLLMFormat(t *testing.T) { } } -func TestModelPricesSyncPrefersModelsDevProviderScopedPrices(t *testing.T) { +func TestModelPricesSyncUsesModelsDevOfficialAndOrderedFallback(t *testing.T) { modelsDevSource := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{ - "openai": {"models": { - "gpt-test": {"cost":{"input":9,"output":10,"cache_read":1}}, - "ambiguous": {"cost":{"input":3,"output":4}} - }}, - "azure": {"models": { - "ambiguous": {"cost":{"input":5,"output":6}} - }}, - "crossmodel": {"models": { - "openai/gpt-test": {"cost":{"input":11,"output":12}} - }} + "models": { + "openai/gpt-test": {"id":"openai/gpt-test","name":"GPT Test"} + }, + "providers": { + "openai": {"models": { + "gpt-test": {"cost":{"input":9,"output":10,"cache_read":1}}, + "ambiguous": {"cost":{"input":3,"output":4}} + }}, + "azure": {"models": { + "ambiguous": {"cost":{"input":5,"output":6}} + }}, + "crossmodel": {"models": { + "openai/gpt-test": {"cost":{"input":11,"output":12}} + }} + } }`)) })) t.Cleanup(modelsDevSource.Close) @@ -809,22 +814,9 @@ func TestModelPricesSyncPrefersModelsDevProviderScopedPrices(t *testing.T) { if len(response.Sources) != 2 || response.Sources[0] != "models.dev" || response.Sources[1] != "litellm" { t.Fatalf("source order = %#v", response.Sources) } - if response.Imported != 4 || len(response.Candidates) != 2 { + if response.Imported != 6 || len(response.Candidates) != 0 { t.Fatalf("sync selection = %#v", response) } - candidateSources := map[string]map[string]bool{} - for _, set := range response.Candidates { - candidateSources[set.Model] = map[string]bool{} - for _, candidate := range set.Candidates { - candidateSources[set.Model][candidate.SourceModelID] = true - } - } - if !candidateSources["ambiguous"]["openai/ambiguous"] || - !candidateSources["ambiguous"]["azure/ambiguous"] || - !candidateSources["openai/gpt-test"]["openai/gpt-test"] || - !candidateSources["openai/gpt-test"]["crossmodel/openai/gpt-test"] { - t.Fatalf("candidate sources = %#v", candidateSources) - } price, ok := response.Prices["gpt-test"] if !ok || !closeFloat(price.Prompt, 9) || !closeFloat(price.Completion, 10) || price.Source != "models.dev" || price.SourceModelID != "openai/gpt-test" { t.Fatalf("models.dev alias price = %#v", price) @@ -837,8 +829,13 @@ func TestModelPricesSyncPrefersModelsDevProviderScopedPrices(t *testing.T) { if !ok || !closeFloat(crossmodel.Prompt, 11) || crossmodel.SourceModelID != "crossmodel/openai/gpt-test" { t.Fatalf("nested provider-scoped price = %#v", crossmodel) } - if _, ok := response.Prices["openai/gpt-test"]; ok { - t.Fatalf("colliding model was imported without confirmation: %#v", response.Prices["openai/gpt-test"]) + officialScoped, ok := response.Prices["openai/gpt-test"] + if !ok || !closeFloat(officialScoped.Prompt, 9) || officialScoped.Source != "models.dev" || officialScoped.SourceModelID != "openai/gpt-test" { + t.Fatalf("official scoped price = %#v", officialScoped) + } + ambiguous, ok := response.Prices["ambiguous"] + if !ok || !closeFloat(ambiguous.Prompt, 7) || ambiguous.Source != "litellm" || ambiguous.SourceModelID != "ambiguous" { + t.Fatalf("ordered ambiguity fallback = %#v", ambiguous) } fallback, ok := response.Prices["fallback-only"] if !ok || !closeFloat(fallback.Prompt, 1) || fallback.Source != "litellm" || fallback.SourceModelID != "fallback-only" { @@ -858,7 +855,10 @@ func TestModelPricesSyncCachesModelsDevAndSkipsCoveredFallbacks(t *testing.T) { } w.Header().Set("Content-Type", "application/json") w.Header().Set("ETag", etag) - _, _ = w.Write([]byte(`{"openai":{"models":{"gpt-test":{"cost":{"input":9,"output":10}}}}}`)) + _, _ = w.Write([]byte(`{ + "models":{"openai/gpt-test":{"id":"openai/gpt-test"}}, + "providers":{"openai":{"models":{"gpt-test":{"cost":{"input":9,"output":10}}}}} + }`)) return } if received := r.Header.Get("If-None-Match"); received != etag { diff --git a/apps/manager-server/internal/service/modelprice/service.go b/apps/manager-server/internal/service/modelprice/service.go index 87c8523a..8eadcf80 100644 --- a/apps/manager-server/internal/service/modelprice/service.go +++ b/apps/manager-server/internal/service/modelprice/service.go @@ -92,7 +92,28 @@ type Service struct { pricesChangedNotifier func() } -type fetchModelPricesFunc func(context.Context, string, *http.Client) (map[string]store.ModelPrice, int, error) +type modelPriceMatchMetadata struct { + modelsDevCanonicalByIdentity map[string]string + modelsDevOfficialSourceModelIDs map[string]struct{} +} + +type fetchedModelPriceSource struct { + Prices map[string]store.ModelPrice + Metadata modelPriceMatchMetadata +} + +type modelPriceSourceEntry struct { + Key string + Price store.ModelPrice +} + +type modelPriceCollection struct { + Entries []modelPriceSourceEntry + Metadata modelPriceMatchMetadata +} + +type fetchModelPricesFunc func(context.Context, string, *http.Client) (fetchedModelPriceSource, int, error) +type fetchModelPriceMapFunc func(context.Context, string, *http.Client) (map[string]store.ModelPrice, int, error) type priceSyncSource struct { Source string @@ -100,6 +121,89 @@ type priceSyncSource struct { Fetch fetchModelPricesFunc } +func wrapModelPriceMapFetcher(fetch fetchModelPriceMapFunc) fetchModelPricesFunc { + return func(ctx context.Context, syncURL string, client *http.Client) (fetchedModelPriceSource, int, error) { + prices, skipped, err := fetch(ctx, syncURL, client) + return fetchedModelPriceSource{Prices: prices}, skipped, err + } +} + +func normalizeModelPriceIdentity(identity string) string { + return strings.ToLower(strings.TrimSpace(identity)) +} + +func (metadata *modelPriceMatchMetadata) merge(other modelPriceMatchMetadata) { + if len(other.modelsDevCanonicalByIdentity) > 0 { + if metadata.modelsDevCanonicalByIdentity == nil { + metadata.modelsDevCanonicalByIdentity = make(map[string]string, len(other.modelsDevCanonicalByIdentity)) + } + for identity, canonicalID := range other.modelsDevCanonicalByIdentity { + if existing, ok := metadata.modelsDevCanonicalByIdentity[identity]; ok && !strings.EqualFold(existing, canonicalID) { + delete(metadata.modelsDevCanonicalByIdentity, identity) + continue + } + metadata.modelsDevCanonicalByIdentity[identity] = canonicalID + } + } + if len(other.modelsDevOfficialSourceModelIDs) > 0 { + if metadata.modelsDevOfficialSourceModelIDs == nil { + metadata.modelsDevOfficialSourceModelIDs = make(map[string]struct{}, len(other.modelsDevOfficialSourceModelIDs)) + } + for sourceModelID := range other.modelsDevOfficialSourceModelIDs { + metadata.modelsDevOfficialSourceModelIDs[sourceModelID] = struct{}{} + } + } +} + +func newModelsDevMatchMetadata(models map[string]json.RawMessage) modelPriceMatchMetadata { + if len(models) == 0 { + return modelPriceMatchMetadata{} + } + metadata := modelPriceMatchMetadata{ + modelsDevCanonicalByIdentity: make(map[string]string, len(models)*2), + } + tailMatches := make(map[string]string, len(models)) + for rawModelID := range models { + modelID := strings.TrimSpace(rawModelID) + if modelID == "" { + continue + } + normalizedModelID := normalizeModelPriceIdentity(modelID) + metadata.modelsDevCanonicalByIdentity[normalizedModelID] = modelID + _, tail, ok := strings.Cut(modelID, "/") + tail = strings.TrimSpace(tail) + if !ok || tail == "" { + continue + } + normalizedTail := normalizeModelPriceIdentity(tail) + if existing, exists := tailMatches[normalizedTail]; exists && !strings.EqualFold(existing, modelID) { + tailMatches[normalizedTail] = "" + continue + } + tailMatches[normalizedTail] = modelID + } + for tail, modelID := range tailMatches { + if modelID != "" { + metadata.modelsDevCanonicalByIdentity[tail] = modelID + } + } + return metadata +} + +func (metadata modelPriceMatchMetadata) modelsDevCanonicalModelID(modelID string) (string, bool) { + modelID = strings.TrimSpace(modelID) + if modelID == "" || len(metadata.modelsDevCanonicalByIdentity) == 0 { + return "", false + } + canonicalID, ok := metadata.modelsDevCanonicalByIdentity[normalizeModelPriceIdentity(modelID)] + return canonicalID, ok +} + +func (metadata modelPriceMatchMetadata) isModelsDevOfficialSourceModelID(sourceModelID string) bool { + _, ok := metadata.modelsDevOfficialSourceModelIDs[normalizeModelPriceIdentity(sourceModelID)] + return ok +} + func New(store *store.Store, syncURL *string, setupResolver ...SetupResolver) *Service { return NewMultiSource(store, syncURL, nil, setupResolver...) } @@ -138,19 +242,19 @@ func newMultiSource( sources = append(sources, priceSyncSource{ Source: SyncSourceModelsDev, URL: modelsDevSyncURL, - Fetch: modelsDevCache.fetch, + Fetch: modelsDevCache.fetchSource, }) } sources = append(sources, priceSyncSource{ Source: SyncSourceLiteLLM, URL: liteLLMSyncURL, - Fetch: fetchLiteLLMModelPrices, + Fetch: wrapModelPriceMapFetcher(fetchLiteLLMModelPrices), }) if openRouterSyncURL != nil { sources = append(sources, priceSyncSource{ Source: SyncSourceOpenRouter, URL: openRouterSyncURL, - Fetch: fetchOpenRouterModelPrices, + Fetch: wrapModelPriceMapFetcher(fetchOpenRouterModelPrices), }) } return &Service{ @@ -205,7 +309,7 @@ func (s *Service) Sync(ctx context.Context, req SyncRequest) (SyncResult, error) if err != nil { return SyncResult{}, err } - selection := selectModelPrices(remotePrices, req.Models) + selection := selectModelPriceCollection(remotePrices, req.Models) preserved := []string(nil) if hasFailedSyncSource(sourceResults) { existingPrices, err := s.store.LoadModelPrices(ctx) @@ -244,18 +348,15 @@ func (s *Service) SyncFromLiteLLM(ctx context.Context, req SyncRequest) (SyncRes return s.Sync(ctx, req) } -func (s *Service) fetchAllModelPrices(ctx context.Context, client *http.Client, models []string) (map[string]store.ModelPrice, int, []string, []SyncSourceResult, error) { - remotePrices := map[string]store.ModelPrice{} - selectedPriorities := map[string]int{} - selectedNormalizedPriorities := map[string]int{} - modelsDevModelIDs := map[string]struct{}{} +func (s *Service) fetchAllModelPrices(ctx context.Context, client *http.Client, models []string) (modelPriceCollection, int, []string, []SyncSourceResult, error) { + remotePrices := modelPriceCollection{} requestedModels := normalizedRequestedModels(models) sources := make([]string, 0, len(s.syncSources)) sourceResults := make([]SyncSourceResult, 0, len(s.syncSources)) failures := []string{} totalSkipped := 0 - for priority, source := range s.syncSources { + for _, source := range s.syncSources { syncURL := source.currentURL() result := SyncSourceResult{Source: source.Source} if syncURL == "" { @@ -269,7 +370,7 @@ func (s *Service) fetchAllModelPrices(ctx context.Context, client *http.Client, if s.syncSourceTimeout > 0 { sourceCtx, cancel = context.WithTimeout(ctx, s.syncSourceTimeout) } - prices, skipped, err := source.Fetch(sourceCtx, syncURL, client) + fetched, skipped, err := source.Fetch(sourceCtx, syncURL, client) cancel() result.Skipped = skipped if err != nil { @@ -278,38 +379,26 @@ func (s *Service) fetchAllModelPrices(ctx context.Context, client *http.Client, failures = append(failures, source.Source+": "+err.Error()) continue } - result.Models = len(prices) + result.Models = len(fetched.Prices) sourceResults = append(sourceResults, result) sources = append(sources, source.Source) totalSkipped += skipped - if source.Source == SyncSourceModelsDev { - modelsDevModelIDs = collectModelsDevModelIDs(prices) - } + remotePrices.Metadata.merge(fetched.Metadata) - for modelID, price := range prices { - normalizedModelID := strings.ToLower(strings.TrimSpace(modelID)) - if source.Source != SyncSourceModelsDev { - if _, blocked := modelsDevModelIDs[normalizedModelID]; blocked { - continue - } - if selectedPriority, exists := selectedNormalizedPriorities[normalizedModelID]; exists && selectedPriority < priority { - continue - } - } + for _, modelID := range sortedPriceKeys(fetched.Prices) { + price := fetched.Prices[modelID] if price.Source == "" { price.Source = source.Source } if price.SourceModelID == "" { price.SourceModelID = modelID } - if _, exists := remotePrices[modelID]; exists && selectedPriorities[modelID] <= priority { - continue - } - remotePrices[modelID] = price - selectedPriorities[modelID] = priority - selectedNormalizedPriorities[normalizedModelID] = priority + remotePrices.Entries = append(remotePrices.Entries, modelPriceSourceEntry{ + Key: modelID, + Price: price, + }) } - if len(requestedModels) > 0 && modelPricesCoverRequested(remotePrices, requestedModels) { + if len(requestedModels) > 0 && modelPriceCollectionCoversRequested(remotePrices, requestedModels) { break } } @@ -318,7 +407,7 @@ func (s *Service) fetchAllModelPrices(ctx context.Context, client *http.Client, if len(failures) == 0 { failures = append(failures, "no price sync sources configured") } - return nil, 0, nil, sourceResults, errors.New("model price sync failed; existing prices were not changed: " + strings.Join(failures, "; ")) + return modelPriceCollection{}, 0, nil, sourceResults, errors.New("model price sync failed; existing prices were not changed: " + strings.Join(failures, "; ")) } return remotePrices, totalSkipped, sources, sourceResults, nil } @@ -447,7 +536,7 @@ type modelsDevPriceCache struct { mu sync.Mutex url string etag string - prices map[string]store.ModelPrice + fetched fetchedModelPriceSource skipped int } @@ -455,51 +544,61 @@ type modelsDevPriceCache struct { // models.dev prices are already expressed as USD per 1M tokens, unlike the // token-level values published by LiteLLM and OpenRouter. func fetchModelsDevModelPrices(ctx context.Context, syncURL string, client *http.Client) (map[string]store.ModelPrice, int, error) { + fetched, skipped, err := fetchModelsDevPriceSource(ctx, syncURL, client) + return fetched.Prices, skipped, err +} + +func fetchModelsDevPriceSource(ctx context.Context, syncURL string, client *http.Client) (fetchedModelPriceSource, int, error) { res, err := fetchModelsDevResponse(ctx, syncURL, client, "") if err != nil { - return nil, 0, err + return fetchedModelPriceSource{}, 0, err } defer res.Body.Close() if res.StatusCode == http.StatusNotModified { - return nil, 0, errors.New("model price sync failed: unexpected 304 Not Modified") + return fetchedModelPriceSource{}, 0, errors.New("model price sync failed: unexpected 304 Not Modified") } - return decodeModelsDevModelPrices(res.Body) + return decodeModelsDevPriceSource(res.Body) } func (cache *modelsDevPriceCache) fetch(ctx context.Context, syncURL string, client *http.Client) (map[string]store.ModelPrice, int, error) { + fetched, skipped, err := cache.fetchSource(ctx, syncURL, client) + return fetched.Prices, skipped, err +} + +func (cache *modelsDevPriceCache) fetchSource(ctx context.Context, syncURL string, client *http.Client) (fetchedModelPriceSource, int, error) { cache.mu.Lock() defer cache.mu.Unlock() if cache.url != syncURL { cache.url = syncURL cache.etag = "" - cache.prices = nil + cache.fetched = fetchedModelPriceSource{} cache.skipped = 0 } res, err := fetchModelsDevResponse(ctx, syncURL, client, cache.etag) if err != nil { - return nil, 0, err + return fetchedModelPriceSource{}, 0, err } defer res.Body.Close() if res.StatusCode == http.StatusNotModified { - if cache.prices == nil { - return nil, 0, errors.New("model price sync failed: models.dev returned 304 without cached prices") + if cache.fetched.Prices == nil { + return fetchedModelPriceSource{}, 0, errors.New("model price sync failed: models.dev returned 304 without cached prices") } if etag := strings.TrimSpace(res.Header.Get("ETag")); etag != "" { cache.etag = etag } - return cache.prices, cache.skipped, nil + return cache.fetched, cache.skipped, nil } - prices, skipped, err := decodeModelsDevModelPrices(res.Body) + fetched, skipped, err := decodeModelsDevPriceSource(res.Body) if err != nil { - return nil, skipped, err + return fetchedModelPriceSource{}, skipped, err } cache.etag = strings.TrimSpace(res.Header.Get("ETag")) - cache.prices = prices + cache.fetched = fetched cache.skipped = skipped - return prices, skipped, nil + return fetched, skipped, nil } func fetchModelsDevResponse(ctx context.Context, syncURL string, client *http.Client, etag string) (*http.Response, error) { @@ -527,18 +626,55 @@ func fetchModelsDevResponse(ctx context.Context, syncURL string, client *http.Cl return res, nil } +type modelsDevRawProvider struct { + Models map[string]json.RawMessage `json:"models"` +} + func decodeModelsDevModelPrices(reader io.Reader) (map[string]store.ModelPrice, int, error) { - var raw map[string]struct { - Models map[string]json.RawMessage `json:"models"` - } + fetched, skipped, err := decodeModelsDevPriceSource(reader) + return fetched.Prices, skipped, err +} + +func decodeModelsDevPriceSource(reader io.Reader) (fetchedModelPriceSource, int, error) { + var root map[string]json.RawMessage decoder := json.NewDecoder(reader) decoder.UseNumber() - if err := decoder.Decode(&raw); err != nil { - return nil, 0, err + if err := decoder.Decode(&root); err != nil { + return fetchedModelPriceSource{}, 0, err + } + + providerMessages := root + canonicalModels := map[string]json.RawMessage(nil) + if rawProviders, hasProviders := root["providers"]; hasProviders { + rawModels, hasModels := root["models"] + if !hasModels { + return fetchedModelPriceSource{}, 0, errors.New("model price sync failed: models.dev catalog contained no canonical models") + } + var catalogProviders map[string]json.RawMessage + if err := json.Unmarshal(rawProviders, &catalogProviders); err != nil { + return fetchedModelPriceSource{}, 0, err + } + if err := json.Unmarshal(rawModels, &canonicalModels); err != nil { + return fetchedModelPriceSource{}, 0, err + } + if len(canonicalModels) == 0 { + return fetchedModelPriceSource{}, 0, errors.New("model price sync failed: models.dev catalog contained no canonical models") + } + providerMessages = catalogProviders + } + + raw := make(map[string]modelsDevRawProvider, len(providerMessages)) + for providerID, message := range providerMessages { + var provider modelsDevRawProvider + if err := json.Unmarshal(message, &provider); err != nil { + return fetchedModelPriceSource{}, 0, err + } + raw[providerID] = provider } now := time.Now().UnixMilli() prices := map[string]store.ModelPrice{} + metadata := newModelsDevMatchMetadata(canonicalModels) skipped := 0 providerIDs := make([]string, 0, len(raw)) for providerID := range raw { @@ -600,13 +736,19 @@ func decodeModelsDevModelPrices(reader io.Reader) (map[string]store.ModelPrice, SyncedAtMS: &now, } prices[sourceModelID] = price + if canonicalID, ok := metadata.modelsDevCanonicalByIdentity[normalizeModelPriceIdentity(sourceModelID)]; ok && strings.EqualFold(canonicalID, sourceModelID) { + if metadata.modelsDevOfficialSourceModelIDs == nil { + metadata.modelsDevOfficialSourceModelIDs = map[string]struct{}{} + } + metadata.modelsDevOfficialSourceModelIDs[normalizeModelPriceIdentity(sourceModelID)] = struct{}{} + } } } if len(prices) == 0 { - return nil, skipped, errors.New("model price sync failed: models.dev catalog contained no usable prices") + return fetchedModelPriceSource{}, skipped, errors.New("model price sync failed: models.dev catalog contained no usable prices") } - return prices, skipped, nil + return fetchedModelPriceSource{Prices: prices, Metadata: metadata}, skipped, nil } func readModelsDevContextTiers(cost map[string]any) []store.ModelPriceContextTier { @@ -700,91 +842,12 @@ func readModelsDevServiceTiers(entry map[string]any) []store.ModelPriceServiceTi return tiers } -type basePriceRule struct { - Prompt float64 `json:"prompt"` - Completion float64 `json:"completion"` - Cache float64 `json:"cache"` - CacheRead float64 `json:"cacheRead"` - CacheCreation float64 `json:"cacheCreation"` - PromptConfigured bool `json:"promptConfigured"` - CompletionConfigured bool `json:"completionConfigured"` - CacheReadConfigured bool `json:"cacheReadConfigured"` - CacheCreationConfigured bool `json:"cacheCreationConfigured"` -} - -func basePriceRuleOf(price store.ModelPrice) basePriceRule { - return basePriceRule{ - Prompt: price.Prompt, - Completion: price.Completion, - Cache: price.Cache, - CacheRead: price.CacheRead, - CacheCreation: price.CacheCreation, - PromptConfigured: price.PromptConfigured, - CompletionConfigured: price.CompletionConfigured, - CacheReadConfigured: price.CacheReadConfigured, - CacheCreationConfigured: price.CacheCreationConfigured, - } -} - -func modelPriceRuleSignature(price store.ModelPrice) string { - if price.Source == SyncSourceModelsDev && price.RawJSON != "" { - if signature, ok := modelsDevPriceRuleSignature(price.RawJSON); ok { - return "models.dev:" + signature - } - } - rule, _ := json.Marshal(basePriceRuleOf(price)) - return "base:" + string(rule) -} - -func modelsDevPriceRuleSignature(rawJSON string) (string, bool) { - var entry map[string]any - if err := json.Unmarshal([]byte(rawJSON), &entry); err != nil { - return "", false - } - signature := map[string]any{} - if cost, ok := entry["cost"]; ok { - signature["cost"] = cost - } - if experimental, ok := entry["experimental"].(map[string]any); ok { - if modes, ok := experimental["modes"].(map[string]any); ok { - modeCosts := map[string]any{} - for modeName, rawMode := range modes { - mode, ok := rawMode.(map[string]any) - if !ok { - continue - } - if cost, ok := mode["cost"]; ok { - modeCosts[modeName] = cost - } - } - if len(modeCosts) > 0 { - signature["experimentalModeCosts"] = modeCosts - } - } - } - encoded, err := json.Marshal(signature) - return string(encoded), err == nil -} - func modelsDevModelID(sourceModelID string) (string, bool) { _, modelID, ok := strings.Cut(strings.TrimSpace(sourceModelID), "/") modelID = strings.TrimSpace(modelID) return modelID, ok && modelID != "" } -func collectModelsDevModelIDs(prices map[string]store.ModelPrice) map[string]struct{} { - modelIDs := map[string]struct{}{} - for _, price := range prices { - if price.Source != SyncSourceModelsDev { - continue - } - if modelID, ok := modelsDevModelID(price.SourceModelID); ok { - modelIDs[strings.ToLower(modelID)] = struct{}{} - } - } - return modelIDs -} - func fetchLiteLLMModelPrices(ctx context.Context, syncURL string, client *http.Client) (map[string]store.ModelPrice, int, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, syncURL, nil) if err != nil { @@ -908,51 +971,88 @@ type priceSelectionResult struct { Unmatched []string } -type modelPriceMatch struct { - Key string - Price store.ModelPrice -} - type modelPriceEntry struct { - key string - price store.ModelPrice - identities []string - signature string - signatureReady bool + key string + price store.ModelPrice + directIdentities []string + aliasIdentities []string } type modelPriceMatcher struct { - entries []modelPriceEntry - exact map[string][]int - caseFold map[string][]int - tail map[string][]int - canonical map[string][]int + entries []modelPriceEntry + exact map[string][]int + caseFold map[string][]int + aliasExact map[string][]int + aliasFold map[string][]int + tail map[string][]int + canonical map[string][]int + sourceEntry map[string][]int + sources []string + metadata modelPriceMatchMetadata } func newModelPriceMatcher(prices map[string]store.ModelPrice) *modelPriceMatcher { - matcher := &modelPriceMatcher{ - entries: make([]modelPriceEntry, 0, len(prices)), - exact: make(map[string][]int, len(prices)), - caseFold: make(map[string][]int, len(prices)), - tail: make(map[string][]int, len(prices)), - canonical: make(map[string][]int, len(prices)), + collection := modelPriceCollection{ + Entries: make([]modelPriceSourceEntry, 0, len(prices)), } for _, key := range sortedPriceKeys(prices) { - price := prices[key] - identities := modelPriceIdentities(key, price) + collection.Entries = append(collection.Entries, modelPriceSourceEntry{ + Key: key, + Price: prices[key], + }) + } + return newModelPriceCollectionMatcher(collection) +} + +func newModelPriceCollectionMatcher(collection modelPriceCollection) *modelPriceMatcher { + matcher := &modelPriceMatcher{ + entries: make([]modelPriceEntry, 0, len(collection.Entries)), + exact: make(map[string][]int, len(collection.Entries)), + caseFold: make(map[string][]int, len(collection.Entries)), + aliasExact: make(map[string][]int, len(collection.Entries)), + aliasFold: make(map[string][]int, len(collection.Entries)), + tail: make(map[string][]int, len(collection.Entries)), + canonical: make(map[string][]int, len(collection.Entries)), + sourceEntry: make(map[string][]int, 4), + metadata: collection.Metadata, + } + for _, sourceEntry := range collection.Entries { + key := sourceEntry.Key + price := sourceEntry.Price + directIdentities, aliasIdentities := modelPriceEntryIdentities(key, price) entryIndex := len(matcher.entries) matcher.entries = append(matcher.entries, modelPriceEntry{ - key: key, - price: price, - identities: identities, + key: key, + price: price, + directIdentities: directIdentities, + aliasIdentities: aliasIdentities, }) - for _, identity := range identities { + source := strings.TrimSpace(price.Source) + if _, exists := matcher.sourceEntry[source]; !exists { + matcher.sources = append(matcher.sources, source) + } + matcher.sourceEntry[source] = append(matcher.sourceEntry[source], entryIndex) + for _, identity := range directIdentities { appendModelPriceIndex(matcher.exact, identity, entryIndex) appendModelPriceIndex(matcher.caseFold, strings.ToLower(identity), entryIndex) appendModelPriceIndex(matcher.tail, canonicalModelTail(identity), entryIndex) appendModelPriceIndex(matcher.canonical, canonicalModelID(identity), entryIndex) } + for _, identity := range aliasIdentities { + appendModelPriceIndex(matcher.aliasExact, identity, entryIndex) + appendModelPriceIndex(matcher.aliasFold, strings.ToLower(identity), entryIndex) + appendModelPriceIndex(matcher.tail, canonicalModelTail(identity), entryIndex) + appendModelPriceIndex(matcher.canonical, canonicalModelID(identity), entryIndex) + } } + sort.SliceStable(matcher.sources, func(i, j int) bool { + leftPriority := modelPriceSourcePriority(matcher.sources[i]) + rightPriority := modelPriceSourcePriority(matcher.sources[j]) + if leftPriority != rightPriority { + return leftPriority < rightPriority + } + return matcher.sources[i] < matcher.sources[j] + }) return matcher } @@ -968,11 +1068,24 @@ func appendModelPriceIndex(index map[string][]int, identity string, entryIndex i } func selectModelPrices(prices map[string]store.ModelPrice, models []string) priceSelectionResult { + collection := modelPriceCollection{ + Entries: make([]modelPriceSourceEntry, 0, len(prices)), + } + for _, key := range sortedPriceKeys(prices) { + collection.Entries = append(collection.Entries, modelPriceSourceEntry{ + Key: key, + Price: prices[key], + }) + } + return selectModelPriceCollection(collection, models) +} + +func selectModelPriceCollection(collection modelPriceCollection, models []string) priceSelectionResult { result := priceSelectionResult{ Prices: map[string]store.ModelPrice{}, Matched: map[string]store.ModelPrice{}, } - matcher := newModelPriceMatcher(prices) + matcher := newModelPriceCollectionMatcher(collection) if len(models) == 0 { return matcher.selectAllUnambiguousModelPrices() } @@ -983,18 +1096,13 @@ func selectModelPrices(prices map[string]store.ModelPrice, models []string) pric continue } seen[normalized] = true - price, _, ok, indexedMatches := matcher.findAutomaticModelPrice(normalized) + price, _, ok, _ := matcher.findAutomaticModelPrice(normalized) if ok { result.Prices[normalized] = price result.Matched[normalized] = price continue } - var candidates []SyncCandidate - if len(indexedMatches) > 0 { - candidates = matcher.candidateModelPricesForIndexes(normalized, indexedMatches) - } else { - candidates = matcher.findCandidateModelPrices(normalized) - } + candidates := matcher.findCandidateModelPrices(normalized) if len(candidates) > 0 { result.Candidates = append(result.Candidates, SyncCandidateSet{ Model: normalized, @@ -1033,7 +1141,7 @@ func (matcher *modelPriceMatcher) selectAllUnambiguousModelPrices() priceSelecti } sort.Strings(orderedModelIDs) for _, modelID := range orderedModelIDs { - price, ok := matcher.selectUnambiguousModelPrice(matcher.exact[modelID]) + price, _, ok, _ := matcher.findAutomaticModelPrice(modelID) if !ok { continue } @@ -1049,130 +1157,118 @@ func findAutomaticModelPrice(prices map[string]store.ModelPrice, modelID string) } func (matcher *modelPriceMatcher) findAutomaticModelPrice(modelID string) (store.ModelPrice, string, bool, []int) { - matches, reason := matcher.indexedModelPriceMatches(modelID) - if len(matches) == 0 { + modelID = strings.TrimSpace(modelID) + if modelID == "" { return store.ModelPrice{}, "", false, nil } - price, ok := matcher.selectUnambiguousModelPrice(matches) - return price, reason, ok, matches + allMatches := make([]int, 0) + for _, source := range matcher.sources { + matches, reason := matcher.indexedModelPriceMatchesForSource(modelID, source) + allMatches = appendUniqueModelPriceIndexes(allMatches, matches...) + if source == SyncSourceModelsDev && len(matcher.metadata.modelsDevCanonicalByIdentity) > 0 { + if officialMatches := matcher.modelsDevOfficialMatches(modelID); len(officialMatches) == 1 { + entry := matcher.entries[officialMatches[0]] + return entry.price, "models.dev-official", true, allMatches + } + if strings.Contains(modelID, "/") { + directMatches, directReason := matcher.directModelPriceMatchesForSource(modelID, source) + allMatches = appendUniqueModelPriceIndexes(allMatches, directMatches...) + if len(directMatches) == 1 { + return matcher.entries[directMatches[0]].price, directReason, true, allMatches + } + } + continue + } + if len(matches) == 1 { + return matcher.entries[matches[0]].price, reason, true, allMatches + } + } + return store.ModelPrice{}, "", false, allMatches } -func (matcher *modelPriceMatcher) indexedModelPriceMatches(modelID string) ([]int, string) { +func (matcher *modelPriceMatcher) indexedModelPriceMatchesForSource(modelID string, source string) ([]int, string) { modelID = strings.TrimSpace(modelID) if modelID == "" { return nil, "" } - if matches := matcher.exact[modelID]; len(matches) > 0 { + if matches := matcher.filterIndexesBySource(matcher.exact[modelID], source); len(matches) > 0 { return matches, "exact" } - if matches := matcher.caseFold[strings.ToLower(modelID)]; len(matches) > 0 { + if matches := matcher.filterIndexesBySource(matcher.caseFold[strings.ToLower(modelID)], source); len(matches) > 0 { return matches, "case-insensitive" } + if matches := matcher.filterIndexesBySource(matcher.aliasExact[modelID], source); len(matches) > 0 { + return matches, "source-model-id" + } + if matches := matcher.filterIndexesBySource(matcher.aliasFold[strings.ToLower(modelID)], source); len(matches) > 0 { + return matches, "case-insensitive-source-model-id" + } modelTail := canonicalModelTail(modelID) if modelTail != "" { - if matches := matcher.tail[modelTail]; len(matches) > 0 { + if matches := matcher.filterIndexesBySource(matcher.tail[modelTail], source); len(matches) > 0 { return matches, "provider-prefix" } } modelCanonical := canonicalModelID(modelID) if modelCanonical != "" { - if matches := matcher.canonical[modelCanonical]; len(matches) > 0 { + if matches := matcher.filterIndexesBySource(matcher.canonical[modelCanonical], source); len(matches) > 0 { return matches, "normalized" } } return nil, "" } -func findModelPriceMatches(prices map[string]store.ModelPrice, matchIdentity func(string) bool) []modelPriceMatch { - matches := []modelPriceMatch{} - for _, key := range sortedPriceKeys(prices) { - price := prices[key] - for _, identity := range modelPriceIdentities(key, price) { - if !matchIdentity(identity) { - continue - } - matches = append(matches, modelPriceMatch{Key: key, Price: price}) - break - } - } - return matches -} - -func selectUnambiguousModelPrice(matches []modelPriceMatch) (store.ModelPrice, bool) { - if len(matches) == 0 { - return store.ModelPrice{}, false - } - if len(matches) == 1 { - return matches[0].Price, true +func (matcher *modelPriceMatcher) directModelPriceMatchesForSource(modelID string, source string) ([]int, string) { + if matches := matcher.filterIndexesBySource(matcher.exact[modelID], source); len(matches) > 0 { + return matches, "exact" } - signature := modelPriceRuleSignature(matches[0].Price) - selected := matches[0] - for _, match := range matches[1:] { - if modelPriceRuleSignature(match.Price) != signature { - return store.ModelPrice{}, false - } - if modelPriceMatchLess(match, selected) { - selected = match - } + if matches := matcher.filterIndexesBySource(matcher.caseFold[strings.ToLower(modelID)], source); len(matches) > 0 { + return matches, "case-insensitive" } - return selected.Price, true + return nil, "" } -func modelPriceMatchLess(left modelPriceMatch, right modelPriceMatch) bool { - leftPriority := modelPriceSourcePriority(left.Price.Source) - rightPriority := modelPriceSourcePriority(right.Price.Source) - if leftPriority != rightPriority { - return leftPriority < rightPriority - } - leftID := strings.TrimSpace(left.Price.SourceModelID) - rightID := strings.TrimSpace(right.Price.SourceModelID) - if leftID != rightID { - return leftID < rightID +func (matcher *modelPriceMatcher) filterIndexesBySource(indexes []int, source string) []int { + filtered := make([]int, 0, len(indexes)) + for _, entryIndex := range indexes { + if strings.TrimSpace(matcher.entries[entryIndex].price.Source) == source { + filtered = appendUniqueModelPriceIndexes(filtered, entryIndex) + } } - return left.Key < right.Key + return filtered } -func (matcher *modelPriceMatcher) selectUnambiguousModelPrice(matches []int) (store.ModelPrice, bool) { - if len(matches) == 0 { - return store.ModelPrice{}, false - } - if len(matches) == 1 { - return matcher.entries[matches[0]].price, true - } - signature := matcher.modelPriceRuleSignature(matches[0]) - selected := matches[0] - for _, entryIndex := range matches[1:] { - if matcher.modelPriceRuleSignature(entryIndex) != signature { - return store.ModelPrice{}, false +func appendUniqueModelPriceIndexes(indexes []int, additions ...int) []int { + for _, addition := range additions { + exists := false + for _, existing := range indexes { + if existing == addition { + exists = true + break + } } - if modelPriceEntryLess(&matcher.entries[entryIndex], &matcher.entries[selected]) { - selected = entryIndex + if !exists { + indexes = append(indexes, addition) } } - return matcher.entries[selected].price, true -} - -func (matcher *modelPriceMatcher) modelPriceRuleSignature(entryIndex int) string { - entry := &matcher.entries[entryIndex] - if !entry.signatureReady { - entry.signature = modelPriceRuleSignature(entry.price) - entry.signatureReady = true - } - return entry.signature + return indexes } -func modelPriceEntryLess(left *modelPriceEntry, right *modelPriceEntry) bool { - leftPriority := modelPriceSourcePriority(left.price.Source) - rightPriority := modelPriceSourcePriority(right.price.Source) - if leftPriority != rightPriority { - return leftPriority < rightPriority +func (matcher *modelPriceMatcher) modelsDevOfficialMatches(modelID string) []int { + canonicalID, ok := matcher.metadata.modelsDevCanonicalModelID(modelID) + if !ok { + return nil } - leftID := strings.TrimSpace(left.price.SourceModelID) - rightID := strings.TrimSpace(right.price.SourceModelID) - if leftID != rightID { - return leftID < rightID + matches := make([]int, 0, 1) + for _, entryIndex := range matcher.sourceEntry[SyncSourceModelsDev] { + entry := &matcher.entries[entryIndex] + sourceModelID := strings.TrimSpace(entry.price.SourceModelID) + if !matcher.metadata.isModelsDevOfficialSourceModelID(sourceModelID) || !strings.EqualFold(sourceModelID, canonicalID) { + continue + } + matches = append(matches, entryIndex) } - return left.key < right.key + return matches } func modelPriceSourcePriority(source string) int { @@ -1188,28 +1284,29 @@ func modelPriceSourcePriority(source string) int { } } -func modelPriceIdentities(key string, price store.ModelPrice) []string { - identities := make([]string, 0, 3) - add := func(identity string) { +func modelPriceEntryIdentities(key string, price store.ModelPrice) ([]string, []string) { + direct := make([]string, 0, 2) + aliases := make([]string, 0, 1) + add := func(target *[]string, identity string) { identity = strings.TrimSpace(identity) if identity == "" { return } - for _, existing := range identities { + for _, existing := range *target { if existing == identity { return } } - identities = append(identities, identity) + *target = append(*target, identity) } - add(key) - add(price.SourceModelID) + add(&direct, key) + add(&direct, price.SourceModelID) if price.Source == SyncSourceModelsDev { if modelID, ok := modelsDevModelID(price.SourceModelID); ok { - add(modelID) + add(&aliases, modelID) } } - return identities + return direct, aliases } func findCandidateModelPrices(prices map[string]store.ModelPrice, modelID string) []SyncCandidate { @@ -1217,11 +1314,16 @@ func findCandidateModelPrices(prices map[string]store.ModelPrice, modelID string } func (matcher *modelPriceMatcher) findCandidateModelPrices(modelID string) []SyncCandidate { - candidates := make([]SyncCandidate, 0, maxSyncCandidates) - for entryIndex := range matcher.entries { - candidates = appendModelPriceCandidate(candidates, modelID, &matcher.entries[entryIndex]) + candidates := make([]SyncCandidate, 0, maxSyncCandidates*len(matcher.sources)) + for _, source := range matcher.sources { + indexes, _ := matcher.indexedModelPriceMatchesForSource(modelID, source) + if len(indexes) == 0 { + indexes = matcher.sourceEntry[source] + } + sourceCandidates := matcher.candidateModelPricesForIndexes(modelID, indexes) + candidates = append(candidates, sourceCandidates...) } - return sortAndLimitModelPriceCandidates(candidates) + return candidates } func (matcher *modelPriceMatcher) candidateModelPricesForIndexes(modelID string, indexes []int) []SyncCandidate { @@ -1229,19 +1331,27 @@ func (matcher *modelPriceMatcher) candidateModelPricesForIndexes(modelID string, for _, entryIndex := range indexes { candidates = appendModelPriceCandidate(candidates, modelID, &matcher.entries[entryIndex]) } - return sortAndLimitModelPriceCandidates(candidates) + return matcher.sortAndLimitModelPriceCandidates(candidates) } func appendModelPriceCandidate(candidates []SyncCandidate, modelID string, entry *modelPriceEntry) []SyncCandidate { score := 0.0 reason := "" - for _, identity := range entry.identities { + for _, identity := range entry.directIdentities { candidateScore, candidateReason := modelIdentitySimilarity(modelID, identity) if candidateScore > score { score = candidateScore reason = candidateReason } } + for _, identity := range entry.aliasIdentities { + candidateScore, _ := modelIdentitySimilarity(modelID, identity) + candidateScore = math.Min(candidateScore, 0.94) + if candidateScore > score { + score = candidateScore + reason = "same-model-with-provider-prefix" + } + } if score < minCandidateScore && !(score >= minWeakCandidateScore && isWeakRecallReason(reason)) { return candidates } @@ -1257,8 +1367,13 @@ func appendModelPriceCandidate(candidates []SyncCandidate, modelID string, entry }) } -func sortAndLimitModelPriceCandidates(candidates []SyncCandidate) []SyncCandidate { +func (matcher *modelPriceMatcher) sortAndLimitModelPriceCandidates(candidates []SyncCandidate) []SyncCandidate { sort.SliceStable(candidates, func(i, j int) bool { + leftOfficial := candidates[i].Price.Source == SyncSourceModelsDev && matcher.metadata.isModelsDevOfficialSourceModelID(candidates[i].SourceModelID) + rightOfficial := candidates[j].Price.Source == SyncSourceModelsDev && matcher.metadata.isModelsDevOfficialSourceModelID(candidates[j].SourceModelID) + if leftOfficial != rightOfficial { + return leftOfficial + } if candidates[i].Score == candidates[j].Score { return candidates[i].SourceModelID < candidates[j].SourceModelID } @@ -1287,14 +1402,13 @@ func normalizedRequestedModels(models []string) []string { return normalized } -func modelPricesCoverRequested(prices map[string]store.ModelPrice, models []string) bool { +func modelPriceCollectionCoversRequested(collection modelPriceCollection, models []string) bool { if len(models) == 0 { return false } - matcher := newModelPriceMatcher(prices) + matcher := newModelPriceCollectionMatcher(collection) for _, modelID := range models { - matches, _ := matcher.indexedModelPriceMatches(modelID) - if len(matches) == 0 { + if _, _, ok, _ := matcher.findAutomaticModelPrice(modelID); !ok { return false } } diff --git a/apps/manager-server/internal/service/modelprice/service_test.go b/apps/manager-server/internal/service/modelprice/service_test.go index 30a73cff..9a3b3a27 100644 --- a/apps/manager-server/internal/service/modelprice/service_test.go +++ b/apps/manager-server/internal/service/modelprice/service_test.go @@ -2,6 +2,8 @@ package modelprice import ( "context" + "encoding/json" + "fmt" "net/http" "net/http/httptest" "strings" @@ -87,15 +89,51 @@ func TestFetchModelsDevModelPrices(t *testing.T) { if !ok || unique.SourceModelID != "provider-a/unique-model" { t.Fatalf("unique alias = %#v", unique) } - sameRule, ok := selection.Prices["same-rule"] - if !ok || sameRule.SourceModelID != "provider-b/same-rule" { - t.Fatalf("same-rule alias = %#v", sameRule) + if _, ok := selection.Prices["same-rule"]; ok || + !hasCandidate(selection, "same-rule", "provider-b/same-rule") || + !hasCandidate(selection, "same-rule", "provider-c/same-rule") { + t.Fatalf("same-rule ambiguity = %#v", selection) } - if len(selection.Candidates) != 0 || len(selection.Unmatched) != 0 { + if len(selection.Candidates) != 1 || len(selection.Unmatched) != 0 { t.Fatalf("unexpected selection result = %#v", selection) } } +func TestDecodeModelsDevCatalogSelectsCanonicalOfficialPrice(t *testing.T) { + fetched, skipped, err := decodeModelsDevPriceSource(strings.NewReader(`{ + "models": { + "openai/gpt-5.5": {"name":"GPT 5.5"} + }, + "providers": { + "abacus": {"models": {"gpt-5.5": {"cost":{"input":9,"output":18}}}}, + "openai": {"models": {"gpt-5.5": {"cost":{"input":1,"output":2}}}}, + "third-party": {"models": {"gpt-5.5": {"cost":{"input":7,"output":14}}}} + } + }`)) + if err != nil { + t.Fatalf("decode models.dev catalog: %v", err) + } + if skipped != 0 { + t.Fatalf("skipped = %d", skipped) + } + + collection := collectionFromFetchedSource(fetched) + selection := selectModelPriceCollection(collection, []string{"gpt-5.5"}) + price, ok := selection.Prices["gpt-5.5"] + if !ok || price.Source != SyncSourceModelsDev || price.SourceModelID != "openai/gpt-5.5" || price.Prompt != 1 { + t.Fatalf("official selection = %#v", selection) + } + if len(selection.Candidates) != 0 || len(selection.Unmatched) != 0 { + t.Fatalf("official selection required confirmation: %#v", selection) + } + + scoped := selectModelPriceCollection(collection, []string{"abacus/gpt-5.5"}) + price, ok = scoped.Prices["abacus/gpt-5.5"] + if !ok || price.SourceModelID != "abacus/gpt-5.5" || price.Prompt != 9 { + t.Fatalf("explicit provider selection = %#v", scoped) + } +} + func TestDecodeModelsDevContextTiersPreservesConfiguredZerosAndIgnoresUnsafeRules(t *testing.T) { prices, skipped, err := decodeModelsDevModelPrices(strings.NewReader(`{ "provider-a":{"models":{ @@ -164,6 +202,18 @@ func TestDecodeModelsDevRejectsCatalogWithoutUsablePrices(t *testing.T) { } } +func TestDecodeModelsDevRejectsCatalogWithoutCanonicalModels(t *testing.T) { + for _, payload := range []string{ + `{"providers":{"abacus":{"models":{"gpt-test":{"cost":{"input":1}}}}}}`, + `{"models":null,"providers":{"abacus":{"models":{"gpt-test":{"cost":{"input":1}}}}}}`, + } { + fetched, _, err := decodeModelsDevPriceSource(strings.NewReader(payload)) + if err == nil || !strings.Contains(err.Error(), "no canonical models") { + t.Fatalf("decode error = %v, fetched = %#v", err, fetched) + } + } +} + func TestPriceMutationsNotifyPricingRollup(t *testing.T) { ctx := context.Background() st := testutil.NewStore(t, testutil.NewConfig(t)) @@ -394,7 +444,8 @@ func TestModelsDevCacheFailureFallsBackWithoutStalePrices(t *testing.T) { if err != nil { t.Fatalf("prime models.dev source: %v", err) } - if len(sources) != 1 || sources[0] != SyncSourceModelsDev || prices["openai/gpt-test"].Prompt != 9 { + price, ok := collectionPrice(prices, SyncSourceModelsDev, "openai/gpt-test") + if len(sources) != 1 || sources[0] != SyncSourceModelsDev || !ok || price.Prompt != 9 { t.Fatalf("primed sources = %#v, prices = %#v", sources, prices) } if got := liteLLMRequests.Load(); got != 0 { @@ -412,11 +463,11 @@ func TestModelsDevCacheFailureFallsBackWithoutStalePrices(t *testing.T) { if len(sourceResults) != 2 || sourceResults[0].Source != SyncSourceModelsDev || sourceResults[0].Error == "" || sourceResults[1].Source != SyncSourceLiteLLM { t.Fatalf("fallback source results = %#v", sourceResults) } - price := prices["gpt-test"] - if price.Source != SyncSourceLiteLLM || price.Prompt != 1 { + price, ok = collectionPrice(prices, SyncSourceLiteLLM, "gpt-test") + if !ok || price.Source != SyncSourceLiteLLM || price.Prompt != 1 { t.Fatalf("fallback price = %#v", price) } - if _, exists := prices["openai/gpt-test"]; exists { + if _, exists := collectionPrice(prices, SyncSourceModelsDev, "openai/gpt-test"); exists { t.Fatalf("stale models.dev price was reused: %#v", prices) } } @@ -456,7 +507,7 @@ func TestFetchAllModelPricesFallsBackWhenPreferredSourceHangs(t *testing.T) { sourceResults[0].Error == "" || sourceResults[1].Source != SyncSourceLiteLLM { t.Fatalf("fallback source results = %#v", sourceResults) } - if price := prices["gpt-test"]; price.Source != SyncSourceLiteLLM || price.Prompt != 1 { + if price, ok := collectionPrice(prices, SyncSourceLiteLLM, "gpt-test"); !ok || price.Source != SyncSourceLiteLLM || price.Prompt != 1 { t.Fatalf("fallback price = %#v", price) } } @@ -671,7 +722,7 @@ func TestPreserveFailedSourcePricesReportsOnlyRequestedModels(t *testing.T) { } } -func TestSelectModelPricesRequiresConfirmationForScopedIdentityCollision(t *testing.T) { +func TestSelectModelPricesPrefersDirectScopedIdentity(t *testing.T) { prices := map[string]store.ModelPrice{ "openai/gpt-test": { Prompt: 1, @@ -692,13 +743,9 @@ func TestSelectModelPricesRequiresConfirmationForScopedIdentityCollision(t *test } selection := selectModelPrices(prices, []string{"openai/gpt-test"}) - if len(selection.Prices) != 0 || len(selection.Candidates) != 1 { + if selection.Prices["openai/gpt-test"].SourceModelID != "openai/gpt-test" || len(selection.Candidates) != 0 { t.Fatalf("collision selection = %#v", selection) } - if !hasCandidate(selection, "openai/gpt-test", "openai/gpt-test") || - !hasCandidate(selection, "openai/gpt-test", "crossmodel/openai/gpt-test") { - t.Fatalf("collision candidates = %#v", selection.Candidates) - } scoped := selectModelPrices(prices, []string{"crossmodel/openai/gpt-test"}) if scoped.Prices["crossmodel/openai/gpt-test"].SourceModelID != "crossmodel/openai/gpt-test" { @@ -706,8 +753,8 @@ func TestSelectModelPricesRequiresConfirmationForScopedIdentityCollision(t *test } all := selectModelPrices(prices, nil) - if _, ok := all.Prices["openai/gpt-test"]; ok { - t.Fatalf("unsafe colliding alias imported by empty sync: %#v", all.Prices) + if all.Prices["openai/gpt-test"].SourceModelID != "openai/gpt-test" { + t.Fatalf("direct scoped identity missing from empty sync: %#v", all.Prices) } if all.Prices["gpt-test"].SourceModelID != "openai/gpt-test" { t.Fatalf("safe direct alias missing from empty sync: %#v", all.Prices) @@ -744,7 +791,7 @@ func TestSelectModelPricesTreatsAdvancedPricingDifferencesAsAmbiguous(t *testing } } -func TestModelsDevAmbiguityBlocksLowerPriorityBareFallback(t *testing.T) { +func TestModelsDevAmbiguityContinuesToLowerPriorityFallback(t *testing.T) { modelsDev := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{ @@ -769,14 +816,119 @@ func TestModelsDevAmbiguityBlocksLowerPriorityBareFallback(t *testing.T) { if len(sources) != 2 || sources[0] != SyncSourceModelsDev || sources[1] != SyncSourceLiteLLM { t.Fatalf("sources = %#v", sources) } - if _, ok := prices["shared"]; ok { - t.Fatalf("lower-priority bare fallback bypassed ambiguity protection: %#v", prices["shared"]) + selection := selectModelPriceCollection(prices, []string{"shared"}) + if price, ok := selection.Prices["shared"]; !ok || price.Source != SyncSourceLiteLLM || price.Prompt != 9 { + t.Fatalf("lower-priority fallback selection = %#v", selection) } - if _, ok := prices["fallback-only"]; !ok { + if _, ok := collectionPrice(prices, SyncSourceLiteLLM, "fallback-only"); !ok { t.Fatalf("unrelated fallback model missing: %#v", prices) } } +func TestLiteLLMAmbiguityContinuesToOpenRouter(t *testing.T) { + syncURL := "https://example.test/prices" + service := &Service{syncSources: []priceSyncSource{ + { + Source: SyncSourceModelsDev, + URL: &syncURL, + Fetch: wrapModelPriceMapFetcher(func(context.Context, string, *http.Client) (map[string]store.ModelPrice, int, error) { + return map[string]store.ModelPrice{}, 0, nil + }), + }, + { + Source: SyncSourceLiteLLM, + URL: &syncURL, + Fetch: wrapModelPriceMapFetcher(func(context.Context, string, *http.Client) (map[string]store.ModelPrice, int, error) { + return map[string]store.ModelPrice{ + "provider-a/shared": {Prompt: 1, SourceModelID: "provider-a/shared"}, + "provider-b/shared": {Prompt: 2, SourceModelID: "provider-b/shared"}, + }, 0, nil + }), + }, + { + Source: SyncSourceOpenRouter, + URL: &syncURL, + Fetch: wrapModelPriceMapFetcher(func(context.Context, string, *http.Client) (map[string]store.ModelPrice, int, error) { + return map[string]store.ModelPrice{ + "shared": {Prompt: 3, SourceModelID: "shared"}, + }, 0, nil + }), + }, + }} + + prices, _, sources, _, err := service.fetchAllModelPrices(context.Background(), nil, []string{"shared"}) + if err != nil { + t.Fatalf("fetch model prices: %v", err) + } + if got := strings.Join(sources, ","); got != "models.dev,litellm,openrouter" { + t.Fatalf("sources = %q", got) + } + selection := selectModelPriceCollection(prices, []string{"shared"}) + price, ok := selection.Prices["shared"] + if !ok || price.Source != SyncSourceOpenRouter || price.SourceModelID != "shared" || price.Prompt != 3 { + t.Fatalf("OpenRouter fallback selection = %#v", selection) + } +} + +func TestSelectModelPriceCollectionKeepsCandidatesFromEverySource(t *testing.T) { + metadata := newModelsDevMatchMetadata(map[string]json.RawMessage{ + "openai/gpt-5.5": json.RawMessage(`{}`), + }) + metadata.modelsDevOfficialSourceModelIDs = map[string]struct{}{ + normalizeModelPriceIdentity("openai/gpt-5.5"): {}, + } + collection := modelPriceCollection{ + Metadata: metadata, + Entries: []modelPriceSourceEntry{ + { + Key: "openai/gpt-5.5", + Price: store.ModelPrice{ + Prompt: 2, Source: SyncSourceLiteLLM, SourceModelID: "openai/gpt-5.5", + }, + }, + { + Key: "openai/gpt-5.5", + Price: store.ModelPrice{ + Prompt: 3, Source: SyncSourceOpenRouter, SourceModelID: "openai/gpt-5.5", + }, + }, + }, + } + for index := range 10 { + sourceModelID := fmt.Sprintf("provider-%02d/gpt-5.5", index) + collection.Entries = append(collection.Entries, modelPriceSourceEntry{ + Key: sourceModelID, + Price: store.ModelPrice{ + Prompt: float64(index + 10), Source: SyncSourceModelsDev, SourceModelID: sourceModelID, + }, + }) + } + collection.Entries = append(collection.Entries, modelPriceSourceEntry{ + Key: "openai/gpt-5.5", + Price: store.ModelPrice{ + Prompt: 1, Source: SyncSourceModelsDev, SourceModelID: "openai/gpt-5.5", + }, + }) + + selection := selectModelPriceCollection(collection, []string{"gpt-5.5-latest"}) + if len(selection.Prices) != 0 || len(selection.Candidates) != 1 || len(selection.Unmatched) != 0 { + t.Fatalf("fuzzy selection = %#v", selection) + } + candidates := selection.Candidates[0].Candidates + if len(candidates) != 10 { + t.Fatalf("candidates = %#v", candidates) + } + if candidates[0].Price.Source != SyncSourceModelsDev || candidates[0].SourceModelID != "openai/gpt-5.5" { + t.Fatalf("official models.dev candidate was not prioritized: %#v", candidates) + } + if got := candidateSources(candidates, "openai/gpt-5.5"); strings.Join(got, ",") != "models.dev,litellm,openrouter" { + t.Fatalf("cross-source candidate identities = %#v", candidates) + } + if count := candidateSourceCount(candidates, SyncSourceModelsDev); count != maxSyncCandidates { + t.Fatalf("models.dev candidate count = %d, want %d: %#v", count, maxSyncCandidates, candidates) + } +} + func TestFetchAllModelPricesStopsAfterRequestedModelsAreCovered(t *testing.T) { modelsDevPrices := map[string]store.ModelPrice{ "provider-a/primary": { @@ -824,7 +976,7 @@ func TestFetchAllModelPricesStopsAfterRequestedModelsAreCovered(t *testing.T) { wantCalls [3]int32 }{ {name: "models.dev coverage", models: []string{"primary"}, wantSources: SyncSourceModelsDev, wantCalls: [3]int32{1, 0, 0}}, - {name: "models.dev ambiguity", models: []string{"shared"}, wantSources: SyncSourceModelsDev, wantCalls: [3]int32{1, 0, 0}}, + {name: "models.dev ambiguity", models: []string{"shared"}, wantSources: SyncSourceModelsDev + "," + SyncSourceLiteLLM + "," + SyncSourceOpenRouter, wantCalls: [3]int32{1, 1, 1}}, {name: "LiteLLM completes coverage", models: []string{"primary", "lite-only"}, wantSources: SyncSourceModelsDev + "," + SyncSourceLiteLLM, wantCalls: [3]int32{1, 1, 0}}, {name: "OpenRouter still required", models: []string{"router-only"}, wantSources: SyncSourceModelsDev + "," + SyncSourceLiteLLM + "," + SyncSourceOpenRouter, wantCalls: [3]int32{1, 1, 1}}, {name: "empty request fetches all", models: nil, wantSources: SyncSourceModelsDev + "," + SyncSourceLiteLLM + "," + SyncSourceOpenRouter, wantCalls: [3]int32{1, 1, 1}}, @@ -838,26 +990,26 @@ func TestFetchAllModelPricesStopsAfterRequestedModelsAreCovered(t *testing.T) { { Source: SyncSourceModelsDev, URL: &syncURL, - Fetch: func(context.Context, string, *http.Client) (map[string]store.ModelPrice, int, error) { + Fetch: wrapModelPriceMapFetcher(func(context.Context, string, *http.Client) (map[string]store.ModelPrice, int, error) { calls[0].Add(1) return modelsDevPrices, 0, nil - }, + }), }, { Source: SyncSourceLiteLLM, URL: &syncURL, - Fetch: func(context.Context, string, *http.Client) (map[string]store.ModelPrice, int, error) { + Fetch: wrapModelPriceMapFetcher(func(context.Context, string, *http.Client) (map[string]store.ModelPrice, int, error) { calls[1].Add(1) return liteLLMPrices, 0, nil - }, + }), }, { Source: SyncSourceOpenRouter, URL: &syncURL, - Fetch: func(context.Context, string, *http.Client) (map[string]store.ModelPrice, int, error) { + Fetch: wrapModelPriceMapFetcher(func(context.Context, string, *http.Client) (map[string]store.ModelPrice, int, error) { calls[2].Add(1) return openRouterPrices, 0, nil - }, + }), }, }} @@ -877,7 +1029,7 @@ func TestFetchAllModelPricesStopsAfterRequestedModelsAreCovered(t *testing.T) { } } if test.name == "models.dev ambiguity" { - selection := selectModelPrices(prices, test.models) + selection := selectModelPriceCollection(prices, test.models) if len(selection.Prices) != 0 || len(selection.Candidates) != 1 { t.Fatalf("ambiguity selection = %#v", selection) } @@ -1078,6 +1230,46 @@ func hasCandidate(selection priceSelectionResult, model string, sourceModelID st return false } +func collectionPrice(collection modelPriceCollection, source string, sourceModelID string) (store.ModelPrice, bool) { + for _, entry := range collection.Entries { + if entry.Price.Source == source && entry.Price.SourceModelID == sourceModelID { + return entry.Price, true + } + } + return store.ModelPrice{}, false +} + +func collectionFromFetchedSource(fetched fetchedModelPriceSource) modelPriceCollection { + collection := modelPriceCollection{Metadata: fetched.Metadata} + for _, key := range sortedPriceKeys(fetched.Prices) { + collection.Entries = append(collection.Entries, modelPriceSourceEntry{ + Key: key, + Price: fetched.Prices[key], + }) + } + return collection +} + +func candidateSources(candidates []SyncCandidate, sourceModelID string) []string { + sources := make([]string, 0) + for _, candidate := range candidates { + if candidate.SourceModelID == sourceModelID { + sources = append(sources, candidate.Price.Source) + } + } + return sources +} + +func candidateSourceCount(candidates []SyncCandidate, source string) int { + count := 0 + for _, candidate := range candidates { + if candidate.Price.Source == source { + count++ + } + } + return count +} + func closePrice(left float64, right float64) bool { if left > right { return left-right < 0.0000001 From bb9cfcbc8bb0d3571ea920fa22f916ff99529c72 Mon Sep 17 00:00:00 2001 From: seakee Date: Thu, 30 Jul 2026 12:10:07 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=94=A7=20fix(manager-server):=20use?= =?UTF-8?q?=20the=20models.dev=20catalog=20endpoint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch production model-price synchronization from the legacy api.json endpoint to catalog.json. The catalog exposes canonical model identities required for official-price selection. Legacy payload decoding remains supported for compatibility. --- apps/manager-server/internal/httpapi/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/manager-server/internal/httpapi/server.go b/apps/manager-server/internal/httpapi/server.go index 3b74a703..fd4dce49 100644 --- a/apps/manager-server/internal/httpapi/server.go +++ b/apps/manager-server/internal/httpapi/server.go @@ -17,7 +17,7 @@ var embeddedPanel embed.FS const serviceID = "cpa-manager-plus" -var modelsDevModelPriceSyncURL = "https://models.dev/api.json" +var modelsDevModelPriceSyncURL = "https://models.dev/catalog.json" var modelPriceSyncURL = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json" var openRouterModelPriceSyncURL = "https://openrouter.ai/api/v1/models" From e073df257d2f79f1f5b615e9491f0b1b05400588 Mon Sep 17 00:00:00 2001 From: seakee Date: Thu, 30 Jul 2026 12:10:26 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=90=9B=20fix(web):=20preserve=20model?= =?UTF-8?q?=20price=20candidate=20sources?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Key candidate selections by source and model ID instead of model ID alone. Group confirmation options by provider so identical IDs from multiple sources remain selectable. This only changes unresolved-price confirmation behavior. --- .../features/monitoring/ModelPricesPage.tsx | 34 +++++++++++++------ .../model/modelPricesPageModel.test.ts | 27 +++++++++++++++ .../monitoring/model/modelPricesPageModel.ts | 30 ++++++++++++++++ 3 files changed, 80 insertions(+), 11 deletions(-) diff --git a/apps/web/src/features/monitoring/ModelPricesPage.tsx b/apps/web/src/features/monitoring/ModelPricesPage.tsx index aa6d43d6..01641ccf 100644 --- a/apps/web/src/features/monitoring/ModelPricesPage.tsx +++ b/apps/web/src/features/monitoring/ModelPricesPage.tsx @@ -25,6 +25,8 @@ import { formatContextThreshold, formatPriceUnit, formatServiceTierRule, + getModelPriceCandidateIdentity, + groupModelPriceCandidatesBySource, resolveContextTierDisplayPrice, resolveServiceTierDisplayPrice, type ModelPriceFilter, @@ -405,11 +407,17 @@ export function ModelPricesPage() { const candidates = candidateSets.find((candidateSet) => candidateSet.model === row.model) ?.candidates ?? []; - const selectedSource = - selectedCandidates[row.model] || candidates[0]?.sourceModelId || ''; + const candidateGroups = groupModelPriceCandidatesBySource(candidates); + const requestedCandidateIdentity = selectedCandidates[row.model] || ''; const selectedCandidate = - candidates.find((candidate) => candidate.sourceModelId === selectedSource) ?? + candidates.find( + (candidate) => + getModelPriceCandidateIdentity(candidate) === requestedCandidateIdentity + ) ?? candidates[0]; + const selectedCandidateIdentity = selectedCandidate + ? getModelPriceCandidateIdentity(selectedCandidate) + : ''; const contextTiers = row.price?.contextTiers ?? []; const serviceTiers = row.price?.serviceTiers ?? []; @@ -528,7 +536,7 @@ export function ModelPricesPage() { ) : candidates.length > 0 && selectedCandidate ? (