diff --git a/docs/configuration.md b/docs/configuration.md index d06cc67..52a52f6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -428,14 +428,19 @@ appears for the Pilot strategy. When onboarding switches to another LLM provider, AgentOS saves a profile for the provider being left and restores it when you return. A profile contains the -active model, router mode and settings (including text/image tiers, Smart -Routing judge model and endpoint, and Pilot settings), plus non-secret +active model; provider-specific router settings (text/image-tier overrides, +default tier, and Smart Routing judge target/endpoint); and non-secret connection settings such as `base_url`, `proxy`, `api_key_env`, and provider -routing preferences. Profiles are persisted in `config.toml`. +routing preferences. Router strategy and tuning (including Pilot settings) are +global preferences and remain unchanged while switching providers. Shipped tier +tables are re-derived on restore, so new recommended model IDs reach existing +installs instead of being frozen in `config.toml`. Literal `api_key` values and local `judge_api_key` values are not copied into a -profile. Prefer environment-variable references for credentials you need to -survive a provider switch. +profile. Returning to a provider originally configured with a literal API key +requires entering that key again; the setup flow reports this explicitly. +Prefer environment-variable references for credentials you need to survive a +provider switch. #### Upgrading from v4_phase3 diff --git a/src/agentos/gateway/config.py b/src/agentos/gateway/config.py index e940f36..6b09659 100644 --- a/src/agentos/gateway/config.py +++ b/src/agentos/gateway/config.py @@ -17,6 +17,7 @@ Field, PrivateAttr, SerializeAsAny, + ValidationError, field_validator, model_validator, ) @@ -796,6 +797,28 @@ def _merge_tier_dicts(defaults: dict, overrides: object) -> dict: return merged +def _router_tier_overrides(tiers: dict[str, Any], profile: str) -> dict[str, Any]: + """Return only tier values that differ from a shipped provider profile. + + Provider-switch profiles must not serialize a fully-expanded copy of a + shipped tier table: keeping the defaults implicit allows later model-table + updates to reach an existing installation. This helper intentionally keeps + an entire tier when it is not mapping-shaped, so malformed-but-loadable + custom values are not silently discarded while the caller validates them. + """ + defaults = _router_tier_profile_defaults(profile) + overrides: dict[str, Any] = {} + for tier_name, tier in tiers.items(): + default_tier = defaults.get(tier_name) + if isinstance(tier, dict) and isinstance(default_tier, dict): + changed = {key: value for key, value in tier.items() if default_tier.get(key) != value} + if changed: + overrides[tier_name] = changed + elif tier != default_tier: + overrides[tier_name] = tier + return overrides + + def _router_tier_profile_defaults(profile: str | None) -> dict: normalized = (profile or "openrouter").strip().lower() if normalized not in ROUTER_TIER_PROFILE_IDS: @@ -1218,6 +1241,43 @@ def _resolve_tier_profile_defaults(cls, values: Any) -> Any: AgentOSRouterConfig.model_rebuild() +class ProviderRouterProfileConfig(BaseModel): + """The router settings that are specific to one LLM provider. + + Router strategy and tuning are gateway-wide preferences. Only the model + selection table, default tier, and explicit LLM-judge target change when + returning to another provider. + """ + + model_config = ConfigDict(extra="ignore") + + tier_profile: str | None = None + tiers: dict[str, Any] = Field(default_factory=dict) + default_tier: str = DEFAULT_TEXT_TIER + judge_model: str | None = None + judge_provider: str | None = None + judge_base_url: str | None = None + + @field_validator("tier_profile") + @classmethod + def _validate_tier_profile(cls, value: str | None) -> str | None: + if value is None: + return None + normalized = str(value).strip().lower() + if normalized not in ROUTER_TIER_PROFILE_IDS: + allowed = ", ".join(sorted(ROUTER_TIER_PROFILE_IDS)) + raise ValueError( + f"unknown provider-profile tier_profile {value!r}; expected one of {allowed}" + ) + return normalized + + @model_validator(mode="after") + def _compact_shipped_tier_profile(self) -> ProviderRouterProfileConfig: + if self.tier_profile: + self.tiers = _router_tier_overrides(self.tiers, self.tier_profile) + return self + + class ProviderProfileConfig(BaseModel): """Restorable non-secret LLM and router settings for one provider. @@ -1226,14 +1286,14 @@ class ProviderProfileConfig(BaseModel): use the active provider configuration and existing secret-handling paths. """ - model: str + model: str = "" api_key_env: str = "" base_url: str = "" proxy: str = "" max_tokens: int = 0 thinking: str | None = None provider_routing: dict[str, str] = Field(default_factory=dict) - agentos_router: AgentOSRouterConfig + agentos_router: ProviderRouterProfileConfig = Field(default_factory=ProviderRouterProfileConfig) class AgentTokenSavingConfig(BaseSettings): @@ -1729,6 +1789,40 @@ class GatewayConfig(BaseSettings): control_ui: ControlUiConfig = Field(default_factory=ControlUiConfig) diagnostics_enabled: bool = False + @model_validator(mode="before") + @classmethod + def _discard_invalid_provider_profiles(cls, values: Any) -> Any: + """Ignore malformed provider-switch cache entries instead of blocking boot.""" + if not isinstance(values, dict): + return values + raw_profiles = values.get("provider_profiles") + if raw_profiles is None: + return values + values = dict(values) + if not isinstance(raw_profiles, dict): + import logging + + logging.getLogger(__name__).warning( + "provider_profiles_invalid_ignored reason=not_a_mapping" + ) + values.pop("provider_profiles", None) + return values + profiles: dict[str, ProviderProfileConfig] = {} + for provider, raw_profile in raw_profiles.items(): + provider_id = str(provider).strip().lower() + if not provider_id: + continue + try: + profiles[provider_id] = ProviderProfileConfig.model_validate(raw_profile) + except ValidationError: + import logging + + logging.getLogger(__name__).warning( + "provider_profile_invalid_ignored provider=%s", provider_id + ) + values["provider_profiles"] = profiles + return values + @model_validator(mode="after") def _default_agentos_router_profile_for_direct_provider(self) -> GatewayConfig: router = self.agentos_router @@ -2040,6 +2134,27 @@ def to_toml_dict(self) -> dict[str, Any]: defaults = None if defaults is not None and router.get("tiers") == defaults: router.pop("tiers", None) + provider_profiles = data.get("provider_profiles") + if isinstance(provider_profiles, dict): + for provider, profile in provider_profiles.items(): + if not isinstance(profile, dict): + continue + profile_router = profile.get("agentos_router") + if not isinstance(profile_router, dict): + continue + profile_tiers = profile_router.get("tiers") + if not profile_tiers: + profile_router.pop("tiers", None) + continue + profile_id = str(profile_router.get("tier_profile") or provider).strip().lower() + if profile_id not in ROUTER_TIER_PROFILE_IDS: + continue + if isinstance(profile_tiers, dict): + compact_tiers = _router_tier_overrides(profile_tiers, profile_id) + if compact_tiers: + profile_router["tiers"] = compact_tiers + else: + profile_router.pop("tiers", None) for path in sorted(self._runtime_secret_paths): _delete_path(data, path) return data diff --git a/src/agentos/onboarding/mutations.py b/src/agentos/onboarding/mutations.py index 208d478..d57e42b 100644 --- a/src/agentos/onboarding/mutations.py +++ b/src/agentos/onboarding/mutations.py @@ -17,9 +17,8 @@ LlmProviderConfig, MemoryEmbeddingConfig, ProviderProfileConfig, - _bankr_tiers, - _opencap_tiers, - _openrouter_tiers, + ProviderRouterProfileConfig, + _router_tier_overrides, _router_tier_profile_defaults, ) from agentos.onboarding.audio_specs import get_audio_provider_setup_spec @@ -73,11 +72,46 @@ def _clone(cfg: GatewayConfig) -> GatewayConfig: return new_cfg -def _provider_router_snapshot(router: AgentOSRouterConfig) -> AgentOSRouterConfig: - """Copy router settings for a provider switch without duplicating secrets.""" - payload = router.model_dump(mode="python") - payload.pop("judge_api_key", None) - return AgentOSRouterConfig(**payload) +_SHIPPED_ROUTER_TIER_DEFAULTS = tuple( + _router_tier_profile_defaults(profile) for profile in ROUTER_TIER_PROFILE_IDS +) + + +def _provider_router_snapshot( + router: AgentOSRouterConfig, + *, + provider_id: str, + model: str, +) -> ProviderRouterProfileConfig: + """Capture only router state that is specific to one provider. + + Router strategy and tuning remain global. Shipped tier profiles are reduced + to operator-authored overrides so a later release can update their model + identifiers. A local provider's generated all-tier pin is also derived on + restore instead of persisting another copy of the active model. + """ + tier_profile = str(router.tier_profile or "").strip().lower() or None + if tier_profile: + tiers = _router_tier_overrides(router.tiers, tier_profile) + elif is_local_provider(provider_id) and _tiers_are_machine_written_defaults( + router.tiers, provider_id, model + ): + tiers = {} + elif provider_id in ROUTER_TIER_PROFILE_IDS: + tiers = _router_tier_overrides(router.tiers, provider_id) + else: + tiers = { + name: dict(tier) if isinstance(tier, dict) else tier + for name, tier in router.tiers.items() + } + return ProviderRouterProfileConfig( + tier_profile=tier_profile, + tiers=tiers, + default_tier=router.default_tier, + judge_model=router.judge_model, + judge_provider=router.judge_provider, + judge_base_url=router.judge_base_url, + ) def _clean_optional_str(value: str | None) -> str: @@ -142,12 +176,7 @@ def _tiers_are_machine_written_defaults( Anything else is treated as a custom, operator-authored tier set and left untouched. """ - if tiers in ( - _openrouter_tiers(), - _bankr_tiers(), - _opencap_tiers(), - *(_router_tier_profile_defaults(profile) for profile in ROUTER_TIER_PROFILE_IDS), - ): + if tiers in _SHIPPED_ROUTER_TIER_DEFAULTS: return True old_provider = str(old_provider or "").strip().lower() old_model = str(old_model or "").strip() @@ -185,7 +214,6 @@ def _reconcile_router_profile_for_provider( # the persisted config is self-consistent (the runtime degrade guard then # becomes a no-op). router_payload = cfg.agentos_router.model_dump(mode="python") - router_payload["enabled"] = True router_payload["tier_profile"] = None if _tiers_are_machine_written_defaults(cfg.agentos_router.tiers, old_provider, old_model): router_payload["tiers"] = _local_provider_tiers( @@ -230,6 +258,99 @@ def _reconcile_router_profile_for_provider( return warnings +def _restore_provider_router_profile( + cfg: GatewayConfig, + profile: ProviderProfileConfig, + *, + provider_id: str, + model: str, +) -> list[str]: + """Restore provider-specific router state without reverting global tuning.""" + saved_router = profile.agentos_router + router_payload = cfg.agentos_router.model_dump(mode="python") + router_payload["default_tier"] = saved_router.default_tier + + if is_local_provider(provider_id): + router_payload["tier_profile"] = None + router_payload["tiers"] = ( + _merge_router_tiers(saved_router.tiers, None) + if saved_router.tiers + else _local_provider_tiers( + _router_tier_profile_defaults("openrouter"), provider_id, model + ) + ) + elif provider_id in ROUTER_TIER_PROFILE_IDS: + # OpenRouter's recommended mix intentionally has no explicit profile; + # direct providers always restore their own current shipped profile. + router_payload["tier_profile"] = ( + saved_router.tier_profile if provider_id == "openrouter" else provider_id + ) + router_payload["tiers"] = _merge_router_tiers( + _router_tier_profile_defaults(provider_id), saved_router.tiers + ) + else: + return _reconcile_router_profile_for_provider( + cfg, + provider_id, + model=model, + old_provider="", + old_model="", + ) + + # A profile intentionally has no literal judge credential. Clear any value + # carried by the provider being left before restoring its target fields. + router_payload["judge_api_key"] = None + if router_payload.get("enabled"): + warnings = _apply_router_judge_fields( + router_payload, + llm_provider=provider_id, + judge_model=saved_router.judge_model or "", + judge_provider=saved_router.judge_provider, + judge_base_url=saved_router.judge_base_url, + # A profile never carries a literal judge key. Clear the previous + # provider's value when restoring a saved local endpoint. + judge_api_key=None, + ) + else: + warnings = [] + router_payload["judge_model"] = saved_router.judge_model + router_payload["judge_provider"] = saved_router.judge_provider + router_payload["judge_base_url"] = saved_router.judge_base_url + router_payload["judge_api_key"] = None + cfg.agentos_router = AgentOSRouterConfig(**router_payload) + return warnings + + +def _should_snapshot_active_provider( + config: GatewayConfig, + *, + provider_id: str, + model: str, +) -> bool: + """Avoid inventing a profile for GatewayConfig's unconfigured defaults.""" + defaults = LlmProviderConfig() + if getattr(config, "config_path", None): + return True + if provider_id != defaults.provider or model != defaults.model: + return True + llm = config.llm + if any( + ( + llm.api_key, + llm.api_key_env, + llm.base_url != defaults.base_url, + llm.proxy, + llm.max_tokens, + llm.thinking, + llm.provider_routing, + ) + ): + return True + return config.agentos_router.model_dump(mode="python") != AgentOSRouterConfig().model_dump( + mode="python" + ) + + def _default_text_tier(default_tier: str | None) -> str: tier = normalize_text_tier(default_tier or DEFAULT_TEXT_TIER) return tier if tier in _TEXT_ROUTER_TIERS else DEFAULT_TEXT_TIER @@ -366,6 +487,7 @@ def upsert_llm_provider( proxy: str = "", provider_routing: dict[str, str] | None = None, ) -> MutationResult: + provider_id = str(provider_id).strip().lower() spec = get_provider_setup_spec(provider_id) if not spec.runtime_supported: raise ValueError( @@ -407,6 +529,11 @@ def upsert_llm_provider( ): effective_api_key = config.llm.api_key if spec.requires_api_key and not effective_api_key and not effective_api_key_env: + if saved_profile is not None and active_provider != provider_id: + raise ValueError( + f"provider {provider_id!r} has a saved profile but no stored credential; " + "re-enter the API key" + ) raise ValueError(f"provider {provider_id!r} requires an api_key") saved_base_url = ( saved_profile.base_url @@ -448,8 +575,17 @@ def upsert_llm_provider( for provider, profile in config.provider_profiles.items() if str(provider).strip() } - if old_provider and old_model: - provider_profiles[old_provider.strip().lower()] = ProviderProfileConfig( + if ( + old_provider + and old_model + and active_provider != provider_id + and _should_snapshot_active_provider( + config, + provider_id=active_provider, + model=old_model.strip(), + ) + ): + provider_profiles[active_provider] = ProviderProfileConfig( model=old_model.strip(), api_key_env=str(config.llm.api_key_env or "").strip(), base_url=str(config.llm.base_url or "").strip(), @@ -457,9 +593,15 @@ def upsert_llm_provider( max_tokens=config.llm.max_tokens, thinking=config.llm.thinking, provider_routing=dict(config.llm.provider_routing), - agentos_router=_provider_router_snapshot(config.agentos_router), + agentos_router=_provider_router_snapshot( + config.agentos_router, + provider_id=active_provider, + model=old_model.strip(), + ), ) - restored_profile = provider_profiles.get(provider_id) if provider_id != old_provider else None + restored_profile = ( + provider_profiles.get(provider_id) if provider_id != active_provider else None + ) new_cfg = _clone(config) new_cfg.provider_profiles = provider_profiles new_cfg.llm = LlmProviderConfig( @@ -474,8 +616,12 @@ def upsert_llm_provider( provider_routing=effective_provider_routing, ) if restored_profile is not None: - new_cfg.agentos_router = restored_profile.agentos_router.model_copy(deep=True) - reconcile_warnings: list[str] = [] + reconcile_warnings = _restore_provider_router_profile( + new_cfg, + restored_profile, + provider_id=provider_id, + model=model_clean, + ) else: reconcile_warnings = _reconcile_router_profile_for_provider( new_cfg, diff --git a/tests/test_onboarding/test_mutations.py b/tests/test_onboarding/test_mutations.py index e7254c0..5b5eda3 100644 --- a/tests/test_onboarding/test_mutations.py +++ b/tests/test_onboarding/test_mutations.py @@ -602,9 +602,14 @@ def test_ollama_opencap_ollama_switch_restores_provider_model_and_router_profile assert restored_ollama.provider_profiles["opencap"].provider_routing == { "glm-5.2": "surplus" } - assert restored_ollama.provider_profiles["opencap"].agentos_router.model_dump( - mode="python" - ) == opencap.agentos_router.model_dump(mode="python") + opencap_profile_router = restored_ollama.provider_profiles["opencap"].agentos_router + assert opencap_profile_router.tier_profile == "opencap" + assert opencap_profile_router.tiers == {} + assert opencap_profile_router.default_tier == "c2" + assert ( + "tiers" + not in restored_ollama.to_toml_dict()["provider_profiles"]["opencap"]["agentos_router"] + ) assert ( restored_ollama.to_toml_dict()["provider_profiles"]["ollama"]["model"] == configured_ollama_model @@ -668,6 +673,102 @@ def test_provider_switch_restores_smart_routing_settings_without_judge_secret(): assert router.judge_api_key is None +def test_provider_profile_keeps_global_router_tuning_and_compacts_shipped_tiers(): + ollama = upsert_llm_provider(GatewayConfig(), provider_id="ollama", model="qwen3.5:9b").config + deepseek = upsert_llm_provider( + ollama, + provider_id="deepseek", + model="deepseek-chat", + api_key_env="DEEPSEEK_API_KEY", + ).config + + ollama_profile = deepseek.provider_profiles["ollama"].agentos_router + assert ollama_profile.tiers == {} + assert "tiers" not in deepseek.to_toml_dict()["provider_profiles"]["ollama"]["agentos_router"] + + deepseek.agentos_router.strategy = "llm_judge" + deepseek.agentos_router.pilot.safety_net_threshold = 0.9 + restored = upsert_llm_provider(deepseek, provider_id="ollama").config + + assert restored.agentos_router.strategy == "llm_judge" + assert restored.agentos_router.pilot.safety_net_threshold == 0.9 + assert all(tier["model"] == "qwen3.5:9b" for tier in restored.agentos_router.tiers.values()) + + +def test_restored_local_profile_honors_an_explicit_model(): + ollama = upsert_llm_provider(GatewayConfig(), provider_id="ollama", model="qwen3.5:9b").config + deepseek = upsert_llm_provider( + ollama, + provider_id="deepseek", + model="deepseek-chat", + api_key_env="DEEPSEEK_API_KEY", + ).config + + restored = upsert_llm_provider(deepseek, provider_id="ollama", model="llama4:70b").config + + assert restored.llm.model == "llama4:70b" + assert all(tier["provider"] == "ollama" for tier in restored.agentos_router.tiers.values()) + assert all(tier["model"] == "llama4:70b" for tier in restored.agentos_router.tiers.values()) + + +def test_provider_switch_normalizes_active_provider_before_restoring_profile(): + cfg = GatewayConfig( + llm={ + "provider": "OpenCap", + "model": "glm-5.2", + "api_key_env": "OPENCAP_API_KEY", + } + ) + + result = upsert_llm_provider(cfg, provider_id="opencap", model="glm-5.3-new") + + assert result.config.llm.model == "glm-5.3-new" + assert result.config.agentos_router.tier_profile == "opencap" + assert result.config.agentos_router.tiers["c1"]["provider"] == "opencap" + assert result.config.provider_profiles == {} + + +def test_first_provider_switch_does_not_snapshot_unconfigured_defaults(): + result = upsert_llm_provider(GatewayConfig(), provider_id="ollama", model="qwen3.5:9b") + + assert result.config.provider_profiles == {} + + +def test_saved_literal_key_profile_requires_reentry_with_actionable_error(): + deepseek = upsert_llm_provider( + GatewayConfig(), + provider_id="deepseek", + model="deepseek-chat", + api_key="sk-literal-secret", + ).config + ollama = upsert_llm_provider(deepseek, provider_id="ollama", model="qwen3.5:9b").config + + with pytest.raises( + ValueError, + match="has a saved profile but no stored credential; re-enter the API key", + ): + upsert_llm_provider(ollama, provider_id="deepseek") + + assert "sk-literal-secret" not in str(ollama.to_toml_dict()) + + +def test_provider_profiles_accept_partial_entries_and_drop_invalid_ones( + caplog: pytest.LogCaptureFixture, +): + with caplog.at_level("WARNING"): + cfg = GatewayConfig( + provider_profiles={ + "ollama": {"model": "qwen3.5:9b"}, + "bad": {"agentos_router": {"tiers": []}}, + } + ) + + assert cfg.provider_profiles["ollama"].model == "qwen3.5:9b" + assert cfg.provider_profiles["ollama"].agentos_router.tiers == {} + assert "bad" not in cfg.provider_profiles + assert "provider_profile_invalid_ignored provider=bad" in caplog.text + + def test_upsert_router_recommended_writes_profile_without_expanded_tiers(): cfg = GatewayConfig(llm={"provider": "deepseek", "model": "deepseek-chat"})