From 0e307d71cb24bb8f8a1f8f40a26de38bda637ac5 Mon Sep 17 00:00:00 2001 From: yonle Date: Thu, 6 Aug 2026 23:41:19 +0700 Subject: [PATCH 1/2] fix(onboarding): restore provider profiles on switch --- docs/configuration.md | 13 ++ src/agentos/gateway/config.py | 19 +++ src/agentos/onboarding/mutations.py | 133 ++++++++++++++++----- tests/test_onboarding/test_mutations.py | 152 ++++++++++++++++++++++++ 4 files changed, 287 insertions(+), 30 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index fe27b661..d06cc678 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -424,6 +424,19 @@ English-optimized (Pilot)** (`pilot-v1`, the default), **Smart routing only appears for the LLM-based strategy; the "Pilot safety net" field only appears for the Pilot strategy. +#### Provider-switch profiles + +When onboarding switches to another LLM provider, AgentOS saves a profile for +the provider being left and restores it when you return. A profile contains the +active model, router mode and settings (including text/image tiers, Smart +Routing judge model and endpoint, and Pilot settings), plus non-secret +connection settings such as `base_url`, `proxy`, `api_key_env`, and provider +routing preferences. Profiles are persisted in `config.toml`. + +Literal `api_key` values and local `judge_api_key` values are not copied into a +profile. Prefer environment-variable references for credentials you need to +survive a provider switch. + #### Upgrading from v4_phase3 Historical onboarding persisted `strategy = "v4_phase3"` explicitly in diff --git a/src/agentos/gateway/config.py b/src/agentos/gateway/config.py index 693a518a..e940f367 100644 --- a/src/agentos/gateway/config.py +++ b/src/agentos/gateway/config.py @@ -1218,6 +1218,24 @@ def _resolve_tier_profile_defaults(cls, values: Any) -> Any: AgentOSRouterConfig.model_rebuild() +class ProviderProfileConfig(BaseModel): + """Restorable non-secret LLM and router settings for one provider. + + Literal API credentials deliberately remain outside this snapshot. An + ``api_key_env`` reference is safe to preserve; direct API keys continue to + use the active provider configuration and existing secret-handling paths. + """ + + model: str + api_key_env: str = "" + base_url: str = "" + proxy: str = "" + max_tokens: int = 0 + thinking: str | None = None + provider_routing: dict[str, str] = Field(default_factory=dict) + agentos_router: AgentOSRouterConfig + + class AgentTokenSavingConfig(BaseSettings): model_config = SettingsConfigDict(env_prefix="AGENTOS_AGENT_TOKEN_SAVING_") @@ -1691,6 +1709,7 @@ class GatewayConfig(BaseSettings): prompt: PromptConfig = Field(default_factory=PromptConfig) memory: MemoryConfig = Field(default_factory=MemoryConfig) agentos_router: AgentOSRouterConfig = Field(default_factory=AgentOSRouterConfig) + provider_profiles: dict[str, ProviderProfileConfig] = Field(default_factory=dict) agent_token_saving: AgentTokenSavingConfig = Field(default_factory=AgentTokenSavingConfig) compaction: CompactionLlmConfig = Field(default_factory=CompactionLlmConfig) auxiliary: AuxiliaryConfig = Field(default_factory=AuxiliaryConfig) diff --git a/src/agentos/onboarding/mutations.py b/src/agentos/onboarding/mutations.py index ffe51175..208d478d 100644 --- a/src/agentos/onboarding/mutations.py +++ b/src/agentos/onboarding/mutations.py @@ -16,6 +16,7 @@ GatewayConfig, LlmProviderConfig, MemoryEmbeddingConfig, + ProviderProfileConfig, _bankr_tiers, _opencap_tiers, _openrouter_tiers, @@ -72,6 +73,13 @@ def _clone(cfg: GatewayConfig) -> GatewayConfig: return new_cfg +def _provider_router_snapshot(router: AgentOSRouterConfig) -> AgentOSRouterConfig: + """Copy router settings for a provider switch without duplicating secrets.""" + payload = router.model_dump(mode="python") + payload.pop("judge_api_key", None) + return AgentOSRouterConfig(**payload) + + def _clean_optional_str(value: str | None) -> str: if value is None: return "" @@ -125,8 +133,8 @@ def _tiers_are_machine_written_defaults( """True when ``tiers`` are safe to rewrite (not operator-customised). Two shapes count as machine-written: - * the shipped default tier sets (openrouter or bankr), matched exactly, - exactly as :meth:`GatewayConfig._default_agentos_router_profile_for_direct_provider` + * the shipped tier profiles, matched exactly, exactly as + :meth:`GatewayConfig._default_agentos_router_profile_for_direct_provider` detects "not custom"; and * tiers this reconcile already local-pinned — every entry's provider equals the OLD llm provider AND every entry's model equals the OLD llm model @@ -134,7 +142,12 @@ def _tiers_are_machine_written_defaults( Anything else is treated as a custom, operator-authored tier set and left untouched. """ - if tiers in (_openrouter_tiers(), _bankr_tiers(), _opencap_tiers()): + if tiers in ( + _openrouter_tiers(), + _bankr_tiers(), + _opencap_tiers(), + *(_router_tier_profile_defaults(profile) for profile in ROUTER_TIER_PROFILE_IDS), + ): return True old_provider = str(old_provider or "").strip().lower() old_model = str(old_model or "").strip() @@ -165,26 +178,24 @@ def _reconcile_router_profile_for_provider( return [] if current_profile and str(current_profile).strip().lower() == provider_id: return [] - if is_local_provider(provider_id) and not current_profile: + if is_local_provider(provider_id): # Local providers have no tier profile and build no per-tier client. # When the current tiers are the untouched shipped defaults OR tiers a # previous reconcile local-pinned, rewrite them to this provider+model so # the persisted config is self-consistent (the runtime degrade guard then # becomes a no-op). - if _tiers_are_machine_written_defaults( - cfg.agentos_router.tiers, old_provider, old_model - ): - router_payload = cfg.agentos_router.model_dump(mode="python") - router_payload["tier_profile"] = None + router_payload = cfg.agentos_router.model_dump(mode="python") + router_payload["enabled"] = True + router_payload["tier_profile"] = None + if _tiers_are_machine_written_defaults(cfg.agentos_router.tiers, old_provider, old_model): router_payload["tiers"] = _local_provider_tiers( cfg.agentos_router.tiers, provider_id, model ) - cfg.agentos_router = AgentOSRouterConfig(**router_payload) - return [] - # Operator-customised tiers: leave the router exactly as the operator - # authored it (enabled + custom tiers). The runtime degrade guard pins - # any mismatched-provider tier to llm.model per turn, so custom local - # tiers stay safe without being clobbered here. + # Operator-customised tiers are preserved, but a local provider cannot + # retain a cloud tier profile. The runtime degrade guard pins any + # mismatched-provider tier to llm.model per turn, so custom local tiers + # stay safe without being clobbered here. + cfg.agentos_router = AgentOSRouterConfig(**router_payload) return [] if ( not current_profile @@ -360,7 +371,13 @@ def upsert_llm_provider( raise ValueError( f"provider {provider_id!r} is not runtime-supported and cannot be configured" ) + saved_profile = config.provider_profiles.get(provider_id) + active_provider = str(config.llm.provider or "").strip().lower() model_clean = _clean_optional_str(model) + if not model_clean and saved_profile is not None: + model_clean = _clean_optional_str(saved_profile.model) + if not model_clean and active_provider == provider_id: + model_clean = _clean_optional_str(config.llm.model) if not model_clean: model_clean = _router_default_model_for_provider( provider_id, @@ -376,41 +393,97 @@ def upsert_llm_provider( if api_key and api_key_env.strip(): raise ValueError("configure either api_key or api_key_env, not both") effective_api_key_env = "" if api_key else api_key_env.strip() - if not api_key and not effective_api_key_env and config.llm.provider == provider_id: - effective_api_key_env = getattr(config.llm, "api_key_env", "").strip() + if not api_key and not effective_api_key_env: + if active_provider == provider_id: + effective_api_key_env = getattr(config.llm, "api_key_env", "").strip() + elif saved_profile is not None: + effective_api_key_env = saved_profile.api_key_env if ( not effective_api_key and spec.requires_api_key and not api_key_env - and config.llm.provider == provider_id + and active_provider == provider_id and config.llm.api_key ): effective_api_key = config.llm.api_key if spec.requires_api_key and not effective_api_key and not effective_api_key_env: raise ValueError(f"provider {provider_id!r} requires an api_key") - effective_base_url = base_url or spec.default_base_url + saved_base_url = ( + saved_profile.base_url + if saved_profile is not None + else (config.llm.base_url if active_provider == provider_id else "") + ) + effective_base_url = base_url or saved_base_url or spec.default_base_url if spec.requires_base_url and not effective_base_url: raise ValueError(f"provider {provider_id!r} requires a base_url") + saved_proxy = ( + saved_profile.proxy + if saved_profile is not None + else (config.llm.proxy if active_provider == provider_id else "") + ) + effective_proxy = proxy or saved_proxy + saved_provider_routing = ( + saved_profile.provider_routing + if saved_profile is not None + else (config.llm.provider_routing if active_provider == provider_id else {}) + ) + effective_provider_routing = ( + dict(provider_routing) if provider_routing is not None else dict(saved_provider_routing) + ) + saved_max_tokens = ( + saved_profile.max_tokens + if saved_profile is not None + else (config.llm.max_tokens if active_provider == provider_id else 0) + ) + saved_thinking = ( + saved_profile.thinking + if saved_profile is not None + else (config.llm.thinking if active_provider == provider_id else None) + ) old_provider = str(config.llm.provider or "") old_model = str(config.llm.model or "") + provider_profiles = { + str(provider).strip().lower(): profile.model_copy(deep=True) + for provider, profile in config.provider_profiles.items() + if str(provider).strip() + } + if old_provider and old_model: + provider_profiles[old_provider.strip().lower()] = ProviderProfileConfig( + model=old_model.strip(), + api_key_env=str(config.llm.api_key_env or "").strip(), + base_url=str(config.llm.base_url or "").strip(), + proxy=str(config.llm.proxy or "").strip(), + max_tokens=config.llm.max_tokens, + thinking=config.llm.thinking, + provider_routing=dict(config.llm.provider_routing), + agentos_router=_provider_router_snapshot(config.agentos_router), + ) + restored_profile = provider_profiles.get(provider_id) if provider_id != old_provider else None new_cfg = _clone(config) + new_cfg.provider_profiles = provider_profiles new_cfg.llm = LlmProviderConfig( provider=provider_id, model=model_clean, api_key=effective_api_key, api_key_env=effective_api_key_env, base_url=effective_base_url, - proxy=proxy, - provider_routing=dict(provider_routing or {}), - ) - reconcile_warnings = _reconcile_router_profile_for_provider( - new_cfg, - provider_id, - model=model_clean, - old_provider=old_provider, - old_model=old_model, + proxy=effective_proxy, + max_tokens=saved_max_tokens, + thinking=saved_thinking, + provider_routing=effective_provider_routing, ) + if restored_profile is not None: + new_cfg.agentos_router = restored_profile.agentos_router.model_copy(deep=True) + reconcile_warnings: list[str] = [] + else: + reconcile_warnings = _reconcile_router_profile_for_provider( + new_cfg, + provider_id, + model=model_clean, + old_provider=old_provider, + old_model=old_model, + ) if api_key: new_cfg.clear_runtime_secret("llm.api_key") @@ -423,8 +496,8 @@ def upsert_llm_provider( "explicit" if effective_api_key else ("env" if effective_api_key_env else "none") ), "base_url": effective_base_url, - "proxy": proxy, - "provider_routing": dict(provider_routing or {}), + "proxy": effective_proxy, + "provider_routing": effective_provider_routing, } return MutationResult( config=new_cfg, diff --git a/tests/test_onboarding/test_mutations.py b/tests/test_onboarding/test_mutations.py index 44022028..e7254c02 100644 --- a/tests/test_onboarding/test_mutations.py +++ b/tests/test_onboarding/test_mutations.py @@ -516,6 +516,158 @@ def test_local_to_cloud_switch_restores_tier_profile(): assert router.tiers["c0"]["provider"] == "deepseek" +def test_cloud_to_local_switch_reenables_router_and_repins_profile_tiers(): + # A cloud profile creates machine-written provider tiers. Switching back to + # a local provider must clear that profile and pin every tier to the local + # model, just like a first-time local setup. + ollama = upsert_llm_provider(GatewayConfig(), provider_id="ollama", model="qwen3.5:9b").config + deepseek = upsert_llm_provider( + ollama, + provider_id="deepseek", + model="deepseek-chat", + api_key_env="DEEPSEEK_API_KEY", + ).config + + res = upsert_llm_provider(deepseek, provider_id="ollama", model="qwen3.5:9b") + router = res.config.agentos_router + + assert router.enabled is True + assert router.tier_profile is None + for tier in router.tiers.values(): + assert tier["provider"] == "ollama" + assert tier["model"] == "qwen3.5:9b" + + +def test_ollama_opencap_ollama_switch_restores_provider_model_and_router_profile(): + import tomllib + + import tomli_w + + ollama_model = "qwen3.5:9b" + opencap_model = "glm-5.2" + ollama = upsert_llm_provider( + GatewayConfig(), + provider_id="ollama", + model=ollama_model, + base_url="http://ollama.example:11434", + proxy="http://ollama-proxy.example:8080", + provider_routing={"qwen3.5:14b": "local-fast"}, + ).config + ollama.llm.max_tokens = 8_192 + ollama.llm.thinking = "high" + ollama_router = upsert_router( + ollama, + mode="recommended", + strategy="pilot-v1", + default_tier="c2", + safety_net_threshold=0.65, + tiers={ + "c0": {"provider": "ollama", "model": "qwen3.5:2b", "thinking_level": "off"}, + "c1": {"provider": "ollama", "model": ollama_model, "thinking_level": "low"}, + "c2": {"provider": "ollama", "model": "qwen3.5:14b", "thinking_level": "high"}, + "c3": {"provider": "ollama", "model": "qwen3.5:32b", "thinking_level": "xhigh"}, + "image_model": { + "provider": "ollama", + "model": "qwen2.5-vl:7b", + "supports_image": True, + "image_only": True, + }, + }, + ).config + configured_ollama_model = ollama_router.llm.model + opencap = upsert_llm_provider( + ollama_router, + provider_id="opencap", + model=opencap_model, + api_key_env="OPENCAP_API_KEY", + base_url="https://opencap.example/v1", + proxy="http://opencap-proxy.example:8080", + provider_routing={"glm-5.2": "surplus"}, + ).config + + restored_ollama = upsert_llm_provider(opencap, provider_id="ollama").config + + assert restored_ollama.llm.provider == "ollama" + assert restored_ollama.llm.model == configured_ollama_model + assert restored_ollama.llm.base_url == "http://ollama.example:11434" + assert restored_ollama.llm.proxy == "http://ollama-proxy.example:8080" + assert restored_ollama.llm.max_tokens == 8_192 + assert restored_ollama.llm.thinking == "high" + assert restored_ollama.llm.provider_routing == {"qwen3.5:14b": "local-fast"} + assert restored_ollama.provider_profiles["ollama"].model == configured_ollama_model + assert restored_ollama.provider_profiles["opencap"].model == opencap_model + assert restored_ollama.provider_profiles["opencap"].api_key_env == "OPENCAP_API_KEY" + assert restored_ollama.provider_profiles["opencap"].base_url == "https://opencap.example/v1" + assert restored_ollama.provider_profiles["opencap"].proxy == "http://opencap-proxy.example:8080" + assert restored_ollama.provider_profiles["opencap"].provider_routing == { + "glm-5.2": "surplus" + } + assert restored_ollama.provider_profiles["opencap"].agentos_router.model_dump( + mode="python" + ) == opencap.agentos_router.model_dump(mode="python") + assert ( + restored_ollama.to_toml_dict()["provider_profiles"]["ollama"]["model"] + == configured_ollama_model + ) + assert restored_ollama.agentos_router.model_dump(mode="python") == ( + ollama_router.agentos_router.model_dump(mode="python") + ) + assert restored_ollama.agentos_router.default_tier == "c2" + assert restored_ollama.agentos_router.pilot.safety_net_threshold == 0.65 + assert restored_ollama.agentos_router.tiers["c0"]["model"] == "qwen3.5:2b" + assert restored_ollama.agentos_router.tiers["c1"]["model"] == ollama_model + assert restored_ollama.agentos_router.tiers["c2"]["model"] == "qwen3.5:14b" + assert restored_ollama.agentos_router.tiers["c3"]["model"] == "qwen3.5:32b" + assert restored_ollama.agentos_router.tiers["image_model"]["model"] == "qwen2.5-vl:7b" + + reloaded = GatewayConfig(**tomllib.loads(tomli_w.dumps(restored_ollama.to_toml_dict()))) + assert reloaded.provider_profiles["ollama"].agentos_router.tiers == ( + ollama_router.agentos_router.tiers + ) + + restored_opencap = upsert_llm_provider(reloaded, provider_id="opencap").config + assert restored_opencap.llm.model == opencap_model + assert restored_opencap.llm.api_key_env == "OPENCAP_API_KEY" + assert restored_opencap.llm.base_url == "https://opencap.example/v1" + assert restored_opencap.llm.proxy == "http://opencap-proxy.example:8080" + assert restored_opencap.llm.provider_routing == {"glm-5.2": "surplus"} + assert restored_opencap.agentos_router.model_dump(mode="python") == ( + opencap.agentos_router.model_dump(mode="python") + ) + + +def test_provider_switch_restores_smart_routing_settings_without_judge_secret(): + ollama = upsert_llm_provider( + GatewayConfig(), provider_id="ollama", model="qwen3.5:9b" + ).config + smart_routing = upsert_router( + ollama, + mode="recommended", + strategy="llm_judge", + default_tier="c2", + judge_model="qwen3.5:2b", + judge_base_url="http://ollama.example:11434/v1", + judge_api_key="sk-local-judge-secret", + ).config + opencap = upsert_llm_provider( + smart_routing, + provider_id="opencap", + model="glm-5.2", + api_key_env="OPENCAP_API_KEY", + ).config + + restored = upsert_llm_provider(opencap, provider_id="ollama").config + router = restored.agentos_router + + assert router.enabled is True + assert router.strategy == "llm_judge" + assert router.default_tier == "c2" + assert router.judge_model == "qwen3.5:2b" + assert router.judge_base_url == "http://ollama.example:11434/v1" + assert router.judge_provider is None + assert router.judge_api_key is None + + def test_upsert_router_recommended_writes_profile_without_expanded_tiers(): cfg = GatewayConfig(llm={"provider": "deepseek", "model": "deepseek-chat"}) From 0cdcd011c104044c248f918f9cea40bc24be6b5f Mon Sep 17 00:00:00 2001 From: yonle Date: Fri, 7 Aug 2026 16:10:47 +0700 Subject: [PATCH 2/2] chore(ci): trigger fresh pull request checks