diff --git a/docs/reasoning-levels.md b/docs/reasoning-levels.md new file mode 100644 index 000000000..d84f6600c --- /dev/null +++ b/docs/reasoning-levels.md @@ -0,0 +1,123 @@ +# Reasoning levels + +UnifiedLLM exposes the choices a configured route supports. Model names do not +determine those choices: registry YAML maps each label to exact request parameters. +Selecting a label applies `params.update(level_settings)` before dispatch. + +```python +from pathlib import Path + +from nooa.unifiedllm import get_llm_client +from nooa.unifiedllm.registry import reload_registry + +# From the repository root; this example registry is not loaded automatically. +reload_registry(Path("examples/reasoning_levels/llm_config.yaml")) +client = get_llm_client("gpt-5.6-sol") +print(client.reasoning_levels) # tuple of labels; None = unknown, () = unsupported +print(client.reasoning_default) # documented default, or None when unknown +response = await client.acall(messages, reasoning_level="high") +``` + +The constructor also accepts `reasoning_level` as a persistent selection. A +per-call selection overrides it; explicit `reasoning_level=None` uses the raw +base configuration for that call. `reasoning_default` is metadata only: declaring +it does not add parameters, change costs or override existing provider settings. + +## Registry declarations + +```yaml +models: + my-route: + model_name: openai/my-model + reasoning: {effort: medium, context: all_turns} + reasoning_default: medium + reasoning_levels: + low: {reasoning: {effort: low, context: all_turns}} + medium: {reasoning: {effort: medium, context: all_turns}} + high: {reasoning: {effort: high, context: all_turns}} +``` + +Write the complete nested block for each level. There is no deep merge or +inheritance: selecting a level replaces the base value at each key it sets. +The declarations are trusted configuration, just like the rest of the registry; +they are not restricted to a list of provider fields maintained by NOOA. +Client routing and framework controls are reserved: `model`, `api_base`, +`base_url`, `api_key`, `custom_llm_provider`, `client`, `messages`, `input`, `extra_body`, +and the three `reasoning_*` configuration fields. They cannot appear inside a +level's settings. A level changes effort, not the endpoint, credentials or history. + +Omit `reasoning_levels` (or use null) when support is unknown. An empty mapping +explicitly declares selection unsupported. Unknown support, unsupported selection +and invalid labels produce distinct errors; an invalid label lists valid choices. +Without a managed selection, raw provider parameters continue working as before. + +A selected level cannot be combined with a per-call setting of the same key, +including inside `extra_body`. Choose the label or the raw settings, not both. +Constructor defaults in `extra_body` are replaced by the selected settings, +just like top-level defaults. Unrelated defaults are preserved. +Declarations belong on the constructor, not per-call kwargs or `extra_body`. +Changing model or endpoint while using a managed level requires a new client: +one route's declared choices must not be applied to another route. +Supplying an SDK client per call is also rejected with a managed level because +that client can choose a different endpoint. When overriding a registry alias's +route or client type, inherited levels, default and selection are cleared; +declare replacement levels explicitly, or leave support unknown. + +## Where the data comes from + +The [example registry](../examples/reasoning_levels/llm_config.yaml) illustrates +three request shapes, based on the [GPT-5.6 Sol model documentation](https://developers.openai.com/api/docs/models/gpt-5.6-sol), +[Claude effort documentation](https://platform.claude.com/docs/en/build-with-claude/effort) +and [Gemini's OpenAI-compatible API](https://ai.google.dev/gemini-api/docs/openai). +Its endpoints are placeholders. Replace them, the model IDs and the declared +choices with settings for your own route before making calls. It is not +auto-loaded. Load your configured file with `reload_registry(Path(...))`. +Private endpoint settings and credentials belong in private configuration, not +this public example. + +Provider documentation describes provider APIs, not all gateway routes. Mocked +HTTP tests check that the example's settings survive the installed transport; +the opt-in live test checks your configured route's acceptance, not reasoning quality or every +level's behavior. Do not infer support merely from a successful HTTP response +if a gateway silently ignores parameters. + +Do not enable LiteLLM's global `drop_params` when verifying a declaration: it can +discard fields for gateway IDs it does not recognize, even with per-call +`drop_params=False`. The HTTP tests explicitly disable that global flag and check +the serialized fields. NOOA does not change process-global SDK configuration. + +LangChain/Pi data can inform maintenance, but neither is a runtime dependency or +an automatic build input. Updating a route means reviewing its small declaration +and request tests, rather than importing hundreds of profiles. + +## Scope and architecture + +- `unifiedllm/reasoning.py` validates declarations and applies the chosen settings. + The same function serves synchronous and asynchronous Chat and Responses calls. +- Registry fields are passed into UnifiedLLM and consumed before provider dispatch. + Renderers, events and middleware do not translate reasoning levels. +- No selection changes existing behavior. Stored reasoning, compatibility gates, + session archives and replay remain unchanged. This is not a retention toggle. +- Effort labels are provider-local, not comparable units of intelligence or cost. + Changing effort can invalidate a provider's cached prefix. This PR does not add + cache-preserving mid-turn steering, TUI controls, or selection persistence. + +## Tests + +Run `uv run pytest tests/unifiedllm/test_reasoning_levels.py tests/unifiedllm/test_reasoning_levels_wire.py`. +The tests check configuration ownership, invalid selections, route changes, +unchanged defaults and the serialized HTTP requests for the example routes. + +For paid probes, configure registry aliases with a `low` level and credentials +through their normal `api_key_env` settings. Then opt in and name the aliases: + +```sh +NOOA_RUN_REASONING_LEVELS_LIVE=1 NOOA_REASONING_TEST_MODELS=my-route uv run pytest \ + tests/integration/test_reasoning_levels_live.py -m integration -q -s +``` + +The alias list is comma-separated. Missing aliases are skipped. Each configured +alias gets one request capped at 256 output tokens, with retries disabled. +The test checks outgoing settings and route acceptance, not that every effort +label changes model behavior. Provider-specific results belong with the +configuration used to run them. diff --git a/examples/reasoning_levels/llm_config.yaml b/examples/reasoning_levels/llm_config.yaml new file mode 100644 index 000000000..5fe49c6d1 --- /dev/null +++ b/examples/reasoning_levels/llm_config.yaml @@ -0,0 +1,44 @@ +# Illustrative request shapes, not a built-in or verified model catalog. +# Replace the placeholder endpoint, model IDs and levels with your route's settings. +models: + gpt-5.6-sol: + model_name: openai/gpt-5.6-sol + client_type: responses + api_base: https://gateway.example.com/v1 + api_key_env: MODEL_API_KEY + store: false + include: [reasoning.encrypted_content] + reasoning: {effort: medium} + # https://developers.openai.com/api/docs/models/gpt-5.6-sol + reasoning_default: medium + reasoning_levels: + none: {reasoning: {effort: none}} + low: {reasoning: {effort: low}} + medium: {reasoning: {effort: medium}} + high: {reasoning: {effort: high}} + xhigh: {reasoning: {effort: xhigh}} + max: {reasoning: {effort: max}} + + claude-sonnet-5: + model_name: anthropic/claude-sonnet-5 + api_base: https://gateway.example.com + api_key_env: MODEL_API_KEY + # https://platform.claude.com/docs/en/build-with-claude/effort + reasoning_default: high + reasoning_levels: + low: {thinking: {type: adaptive}, output_config: {effort: low}} + medium: {thinking: {type: adaptive}, output_config: {effort: medium}} + high: {thinking: {type: adaptive}, output_config: {effort: high}} + max: {thinking: {type: adaptive}, output_config: {effort: max}} + + gemini-3.1-pro-preview: + model_name: openai/gemini-3.1-pro-preview + api_base: https://gateway.example.com/v1 + api_key_env: MODEL_API_KEY + allowed_openai_params: [reasoning_effort] + # https://ai.google.dev/gemini-api/docs/openai#thinking + reasoning_default: high + reasoning_levels: + low: {reasoning_effort: low} + medium: {reasoning_effort: medium} + high: {reasoning_effort: high} diff --git a/skills/nooa-agent-authoring/SKILL.md b/skills/nooa-agent-authoring/SKILL.md index 1a4021e45..deaae9021 100644 --- a/skills/nooa-agent-authoring/SKILL.md +++ b/skills/nooa-agent-authoring/SKILL.md @@ -1,6 +1,6 @@ --- name: nooa-agent-authoring -description: Author agents with NVIDIA-labs Object Oriented Agents (NOOA). Use when writing or modifying an Agent subclass, agentic methods (ellipsis bodies), docstring prompts, structured output contracts, strategy selection (CodeAct/Predict), visibility control, orchestrators, or subagent composition. +description: Author agents with NVIDIA-labs Object Oriented Agents (NOOA). Use when configuring model registry entries or reasoning levels, writing or modifying an Agent subclass, agentic methods (ellipsis bodies), docstring prompts, structured output contracts, strategy selection (CodeAct/Predict), visibility control, orchestrators, or subagent composition. compatibility: 'Python >= 3.12, uv, nooa package (CLI: nooa)' --- @@ -80,6 +80,66 @@ Without a rendered boundary, direct callers cache only leading instructions. [stable-prefix caching](../../docs/stable-prefix-caching.md) for migration and direct-call examples. +### Configuring reasoning levels + +Declare choices for the **configured route**, not a model family inferred from +its name. Each `reasoning_levels` label maps to exact request parameters; +selection replaces whole top-level values, with no nested merge. Repeat any +nested settings that must survive a level change. + +For a Responses route that accepts this request shape, adapt this +`llm_config.yaml` example (replace the placeholder model and configure its key): + +```yaml +models: + my-route: + model_name: openai/your-model + client_type: responses + reasoning: {effort: medium, summary: auto} + reasoning_default: medium + reasoning_levels: + low: {reasoning: {effort: low, summary: auto}} + medium: {reasoning: {effort: medium, summary: auto}} + high: {reasoning: {effort: high, summary: auto}} +``` + +```python +from pathlib import Path +from nooa.unifiedllm import get_llm_client +from nooa.unifiedllm.registry import reload_registry + +reload_registry(Path("llm_config.yaml")) +llm = get_llm_client("my-route", reasoning_level="high") # persistent selection +levels = llm.reasoning_levels +default = llm.reasoning_default +# Pass llm to an Agent through the normal cascade below. +# For a direct UnifiedLLM call, override only this request: +reply = await llm.acall(messages, reasoning_level="low") +``` + +- Omit `reasoning_levels` (or use null) for unknown support; `{}` declares + selection unsupported. The public property returns `None`, `()`, or a tuple + of valid labels respectively. Invalid selections raise, naming the choices. +- `reasoning_default` is metadata, not a selection: if supplied it must name + a declared level and describe the base configuration. `reasoning_level=None` + on a call bypasses a persistent selection and uses those base parameters. +- Declare levels/defaults on the registry entry or client constructor, never + per-call or in `extra_body`. Do not combine a selected level with a per-call + raw setting of the same key (including inside `extra_body`); that raises. + Level blocks cannot change routing, credentials, messages or framework controls. + Construct a new client when changing routes rather than reusing its level map. +- Use provider documentation or catalogues as candidates, then check the actual + route's outgoing HTTP fields: SDKs and gateways may reject or silently drop + settings. HTTP acceptance alone does not prove a level was honored. Leave + unverified choices unknown rather than inventing mappings. +- This controls effort, not retention of stored reasoning. Changing effort may + invalidate the cached prefix; labels are not comparable across providers. + +See [reasoning-level configuration](../../docs/reasoning-levels.md) for reserved +keys, transport caveats, and bounded validation commands. The +`examples/reasoning_levels/llm_config.yaml` entries require explicit loading; +they are not automatically available registry aliases. + **Resolution cascade** for which LLM a method uses — first match wins: 1. `await agent.method(..., llm=special_llm)` — call override diff --git a/src/nooa/config/model_config.py b/src/nooa/config/model_config.py index c081837e8..a4bc3e0bd 100644 --- a/src/nooa/config/model_config.py +++ b/src/nooa/config/model_config.py @@ -43,6 +43,9 @@ class ModelConfig(BaseModel): max_tokens: int | None = None temperature: float | None = None top_p: float | None = None + reasoning_levels: dict[str, dict[str, Any]] | None = None + reasoning_default: str | None = None + reasoning_level: str | None = None @classmethod def from_registry(cls, name: str, raw: dict[str, Any]) -> ModelConfig: diff --git a/src/nooa/unifiedllm/fake.py b/src/nooa/unifiedllm/fake.py index 28bcb3d0c..4c6f48a22 100644 --- a/src/nooa/unifiedllm/fake.py +++ b/src/nooa/unifiedllm/fake.py @@ -84,6 +84,7 @@ async def acall( Thread-safe: uses asyncio.Lock to ensure concurrent calls get responses in order. """ async with self._lock: + self._prepare_call_config(kwargs) self.call_count += 1 # A non-provider test client must never observe private replay state. self.last_messages, _, _ = apply_cache_policy( @@ -114,6 +115,7 @@ def call( ) -> LLMResponse: """Synchronous version of acall for UnifiedLLM compatibility.""" # For sync call, we don't need locking since tests are usually single-threaded + self._prepare_call_config(kwargs) self.call_count += 1 self.last_messages, _, _ = apply_cache_policy( prepare_chat_messages(messages, None), None, responses=False diff --git a/src/nooa/unifiedllm/reasoning.py b/src/nooa/unifiedllm/reasoning.py new file mode 100644 index 000000000..b2cf1f10c --- /dev/null +++ b/src/nooa/unifiedllm/reasoning.py @@ -0,0 +1,120 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Declared effort choices, independent of model names and provider discovery.""" + +from collections.abc import Mapping +from copy import deepcopy +from typing import Any + +from pydantic import BaseModel, ConfigDict, model_validator + +_DECLARATIONS = {"reasoning_levels", "reasoning_default"} +# These select the client/request itself, not a provider's effort behavior. +_RESERVED = _DECLARATIONS | { + "reasoning_level", + "model", + "api_base", + "base_url", + "api_key", + "custom_llm_provider", + "messages", + "input", + "extra_body", + "client", +} + + +class ReasoningConfig(BaseModel): + """Map public level names to request settings for one configured route. + + None means unknown support; an empty mapping means unsupported. The default + documents the route's default, not a request to send it on every call. + Declarations live in registry YAML, not a model-name table in this module. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + levels: dict[str, dict[str, Any]] | None = None + default: str | None = None + + @model_validator(mode="after") + def validate_declaration(self): + """Reject malformed choices and request-control fields at construction.""" + if self.default is not None and self.default not in (self.levels or {}): + raise ValueError("reasoning_default must name a declared reasoning level") + for level, settings in (self.levels or {}).items(): + if not level.strip() or not settings: + raise ValueError("reasoning_levels must have non-empty names and request settings") + if conflict := _RESERVED & settings.keys(): + raise ValueError( + f"reasoning level {level!r} contains reserved fields: {sorted(conflict)}" + ) + return self + + def settings(self, level: str) -> dict[str, Any]: + """Validate a selection and detach its settings from the stored declaration.""" + if self.levels is None: + raise ValueError( + "Reasoning levels are unknown for this route; declare reasoning_levels" + ) + if not self.levels: + raise ValueError("Reasoning-level selection is not supported for this route") + if not isinstance(level, str) or level not in self.levels: + raise ValueError( + f"Invalid reasoning level {level!r}; allowed: {', '.join(self.levels)}" + ) + # Frozen Pydantic attributes do not freeze nested dictionaries. Reuse the + # declaration checks so later edits cannot introduce routing controls. + self.validate_declaration() + # Only the small chosen configuration is copied, never conversation data. + return deepcopy(self.levels[level]) + + +def apply_reasoning_level( + declaration: ReasoningConfig, + model: str, + defaults: dict[str, Any], + overrides: dict[str, Any], + default_selection: str | None, +) -> dict[str, Any]: + """Resolve effort once before either client dispatches. + + No selection leaves existing provider settings untouched. An explicit level + replaces constructor defaults; mixing it with per-call native controls is an + error. Route changes cannot inherit a declaration for a different endpoint. + This affects requested effort, never stored reasoning or replay compatibility. + """ + if _DECLARATIONS & overrides.keys(): + raise ValueError("reasoning_levels and reasoning_default belong on the client constructor") + params = {**defaults, **overrides} + extra = params.get("extra_body") + if isinstance(extra, Mapping) and (set(extra) & (_DECLARATIONS | {"reasoning_level"})): + raise ValueError("Reasoning configuration cannot be passed through extra_body") + level = params.pop("reasoning_level", default_selection) + if level is None: + return params + patch = declaration.settings(level) + if ( + any( + key in overrides and overrides[key] != defaults.get(key) + for key in ("api_base", "base_url", "custom_llm_provider") + ) + or overrides.get("model", model) != model + or overrides.get("client") is not None + ): + raise ValueError("Reasoning levels are route-specific; create a client for the new route") + explicit_extra = overrides.get("extra_body") + if conflict := patch.keys() & ( + overrides.keys() | (explicit_extra.keys() if isinstance(explicit_extra, Mapping) else set()) + ): + raise ValueError( + f"reasoning_level conflicts with explicit request field(s): {sorted(conflict)}" + ) + # Whole top-level values replace defaults. Authors write complete nested + # blocks in YAML; no provider-specific merge or inheritance rules live here. + # Remove replaced defaults from extra_body too: SDKs otherwise merge those + # back over the selected top-level values when assembling the HTTP body. + if isinstance(extra, Mapping) and patch.keys() & extra.keys(): + params["extra_body"] = {key: value for key, value in extra.items() if key not in patch} + params.update(patch) + return params diff --git a/src/nooa/unifiedllm/registry.py b/src/nooa/unifiedllm/registry.py index ddbfc90a8..7b25277f9 100644 --- a/src/nooa/unifiedllm/registry.py +++ b/src/nooa/unifiedllm/registry.py @@ -386,6 +386,9 @@ def get_llm_client(name: str, *, client_type: str | None = None, **overrides) -> "max_tokens", "reasoning", "reasoning_effort", + "reasoning_levels", + "reasoning_default", + "reasoning_level", "allowed_openai_params", "additional_drop_params", "extra_body", @@ -413,6 +416,27 @@ def get_llm_client(name: str, *, client_type: str | None = None, **overrides) -> type(retry_config).__name__, ) + # An alias's declared levels describe its route, not any replacement client. + # Discard inherited selections/defaults as well; explicit declarations below + # belong to the replacement route and are validated by its constructor. + if config and ( + overrides.get("model", model) != model + or any( + key in overrides and overrides[key] != config.get(key) + for key in ("api_base", "base_url", "custom_llm_provider") + ) + or (client_type is not None and client_type != config.get("client_type", "completion")) + or overrides.get("client") is not None + ): + for key in ("reasoning_levels", "reasoning_default", "reasoning_level"): + params.pop(key, None) + if "reasoning_levels" in config: + logger.warning( + "Route overridden for %r; inherited reasoning choices were cleared. " + "Declare reasoning_levels explicitly for the replacement route.", + name, + ) + params.update(overrides) # Select client class: explicit param > YAML config > default diff --git a/src/nooa/unifiedllm/unifiedllm.py b/src/nooa/unifiedllm/unifiedllm.py index 96e183911..9bf278bd2 100644 --- a/src/nooa/unifiedllm/unifiedllm.py +++ b/src/nooa/unifiedllm/unifiedllm.py @@ -36,6 +36,7 @@ from . import replay_state, response_parts from .errors import EmptyContentError from .http_config import HttpConfig +from .reasoning import ReasoningConfig, apply_reasoning_level from .retry import sync_retry, with_retry from .retry_config import RetryConfig @@ -1151,8 +1152,24 @@ class UnifiedLLM(ABC): _registry_config: dict[str, Any] | None cache_breakpoint: Literal["auto", "openai", "anthropic"] | None - def __init__(self, model: str, **config): + def __init__( + self, + model: str, + *, + reasoning_levels: dict[str, dict[str, Any]] | None = None, + reasoning_default: str | None = None, + reasoning_level: str | None = None, + **config, + ): reject_legacy_cache_config(config) + # Freeze prevents field assignment, not mutations inside nested Any + # settings. Detach this small configuration once, never the history. + self._reasoning_config = ReasoningConfig( + levels=reasoning_levels, default=reasoning_default + ).model_copy(deep=True) + if reasoning_level is not None: + self._reasoning_config.settings(reasoning_level) + self.reasoning_level = reasoning_level self.model = model self.config = config self._registry_config = None @@ -1161,6 +1178,22 @@ def __init__(self, model: str, **config): # concrete subclasses; guarded here so base helpers stay safe. self._http: _ClientHttp | None = None + @property + def reasoning_levels(self) -> tuple[str, ...] | None: + """Selectable levels; None means unknown, () means unsupported.""" + levels = self._reasoning_config.levels + return None if levels is None else tuple(levels) + + @property + def reasoning_default(self) -> str | None: + """Documented route default; not a request override.""" + return self._reasoning_config.default + + def _prepare_call_config(self, overrides: dict[str, Any]) -> dict[str, Any]: + return apply_reasoning_level( + self._reasoning_config, self.model, self.config, overrides, self.reasoning_level + ) + def _effective_model(self, call_config: dict[str, Any]) -> str: """Return the model this individual request will actually dispatch.""" model = call_config.get("model", self.model) @@ -1758,7 +1791,7 @@ def call( If retry_config.retry_on_empty_content is True, will retry when the model returns empty content but has reasoning_content (common with some reasoning models). """ - call_config = {**self.config, **kwargs} + call_config = self._prepare_call_config(kwargs) self._validate_request_config("messages", call_config) effective_model = self._effective_model(call_config) self._validate_cache_breakpoint_model(effective_model) @@ -1772,8 +1805,7 @@ def call( api_params = { "model": self.model, - **self.config, - **kwargs, + **call_config, "messages": prepared_messages, } @@ -1847,7 +1879,7 @@ async def acall( If retry_config.retry_on_empty_content is True, will retry when the model returns empty content but has reasoning_content (common with some reasoning models). """ - call_config = {**self.config, **kwargs} + call_config = self._prepare_call_config(kwargs) self._validate_request_config("messages", call_config) effective_model = self._effective_model(call_config) self._validate_cache_breakpoint_model(effective_model) @@ -1861,8 +1893,7 @@ async def acall( api_params = { "model": self.model, - **self.config, - **kwargs, + **call_config, "messages": prepared_messages, } @@ -2111,7 +2142,7 @@ def call( Accepts public message dictionaries and LLMResponse objects. Stored turns are projected here; only leading system messages become `instructions`. """ - call_config = {**self.config, **kwargs} + call_config = self._prepare_call_config(kwargs) self._validate_request_config("input", call_config) effective_model = self._effective_model(call_config) self._validate_cache_breakpoint_model(effective_model) @@ -2123,8 +2154,7 @@ def call( api_params = { "model": self.model, "truncation": "disabled", - **self.config, - **kwargs, + **call_config, "input": input_messages, } if openai_explicit: @@ -2187,7 +2217,7 @@ async def acall( Accepts public message dictionaries and LLMResponse objects. Stored turns are projected here; only leading system messages become `instructions`. """ - call_config = {**self.config, **kwargs} + call_config = self._prepare_call_config(kwargs) self._validate_request_config("input", call_config) effective_model = self._effective_model(call_config) self._validate_cache_breakpoint_model(effective_model) @@ -2199,8 +2229,7 @@ async def acall( api_params = { "model": self.model, "truncation": "disabled", - **self.config, - **kwargs, + **call_config, "input": input_messages, } if openai_explicit: diff --git a/tests/integration/test_reasoning_levels_live.py b/tests/integration/test_reasoning_levels_live.py new file mode 100644 index 000000000..b9b840e2a --- /dev/null +++ b/tests/integration/test_reasoning_levels_live.py @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Opt-in registry acceptance probes, capped at 256 output tokens per alias.""" + +import json +import os + +import httpx +import litellm +import pytest + +from nooa.unifiedllm import RetryConfig, get_llm_client + +pytestmark = [ + pytest.mark.integration, + pytest.mark.skipif( + os.getenv("NOOA_RUN_REASONING_LEVELS_LIVE") != "1", reason="opt-in paid provider test" + ), +] +ALIASES = [ + alias.strip() + for alias in os.getenv("NOOA_REASONING_TEST_MODELS", "").split(",") + if alias.strip() +] + + +@pytest.mark.parametrize("alias", ALIASES) +async def test_low_effort_on_configured_route(alias, monkeypatch): + from nooa.secrets import load_secrets_into_env + from nooa.unifiedllm import registry + + load_secrets_into_env() + config = registry.get_registry_config(alias) + if not config: + pytest.skip(f"Registry alias {alias!r} is not configured") + monkeypatch.setattr(litellm, "drop_params", False) + settings = config["reasoning_levels"]["low"] + sent = [] + original_send = httpx.AsyncClient.send + + async def send(client, request, **kwargs): + if request.method == "POST": + body = json.loads(request.content) + sent.append({field: body.get(field) for field in settings}) + return await original_send(client, request, **kwargs) + + monkeypatch.setattr(httpx.AsyncClient, "send", send) + cap = ( + {"max_output_tokens": 256} + if config.get("client_type") == "responses" + else {"max_tokens": 256} + ) + async with get_llm_client( + alias, + drop_params=False, + num_retries=0, + retry_config=RetryConfig(max_retries=0, rate_limit_extra_retries=0), + **cap, + ) as client: + response = await client.acall( + [{"role": "user", "content": "Compute 17 times 19. Reply with only the number."}], + reasoning_level="low", + ) + assert sent == [settings] # Exactly one attempt, with the declared wire fields. + assert response.usage.input_tokens > 0 + assert response.finish_reason != "error" + print( + json.dumps( + { + "model": alias, + "level": "low", + "usage": response.usage.model_dump(), + "finish_reason": response.finish_reason, + } + ) + ) diff --git a/tests/unifiedllm/test_reasoning_levels.py b/tests/unifiedllm/test_reasoning_levels.py new file mode 100644 index 000000000..743efd820 --- /dev/null +++ b/tests/unifiedllm/test_reasoning_levels.py @@ -0,0 +1,368 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Effort declarations are data; consumers only select and inspect labels.""" + +import ast +import re +from pathlib import Path +from types import MappingProxyType, SimpleNamespace +from unittest.mock import AsyncMock, patch + +import httpx +import litellm +import pytest +import yaml +from pydantic import ValidationError + +from nooa.config.model_config import ModelConfig +from nooa.unifiedllm import CompletionClient, FakeLLMClient, ResponsesClient, get_llm_client +from nooa.unifiedllm.reasoning import ReasoningConfig, apply_reasoning_level + +LEVELS = { + "low": {"reasoning": {"effort": "low", "context": "all_turns"}}, + "high": {"reasoning": {"effort": "high", "context": "all_turns"}}, +} + + +def _response(responses): + if not responses: + return litellm.ModelResponse(choices=[{"message": {"content": "ok"}}]) + return SimpleNamespace( + output=[{"type": "message", "content": [{"type": "output_text", "text": "ok"}]}], + output_text="ok", + status="completed", + usage=None, + ) + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("client_type", [CompletionClient, ResponsesClient]) +async def test_selected_level_reaches_both_dispatch_paths(client_type, asynchronous): + responses = client_type is ResponsesClient + method = ("a" if asynchronous else "") + ("responses" if responses else "completion") + with client_type("openai/test", reasoning_levels=LEVELS, reasoning_default="low") as client: + assert client.reasoning_levels == ("low", "high") + assert client.reasoning_default == "low" + with patch( + f"litellm.{method}", new_callable=AsyncMock if asynchronous else None + ) as transport: + transport.return_value = _response(responses) + messages = [{"role": "user", "content": "hello"}] + if asynchronous: + result = await client.acall(messages, reasoning_level="high") + else: + result = client.call(messages, reasoning_level="high") + assert result.content == "ok" + params = transport.call_args.kwargs + assert params["reasoning"] == LEVELS["high"]["reasoning"] + assert not {"reasoning_levels", "reasoning_default", "reasoning_level"} & params.keys() + assert client.config == {} + + +@pytest.mark.parametrize( + "levels,expected", [(None, "unknown"), ({}, "not supported"), (LEVELS, "allowed: low, high")] +) +def test_unknown_unsupported_and_invalid_are_distinct(levels, expected): + with CompletionClient("openai/test", reasoning_levels=levels) as client: + assert client.reasoning_levels == (None if levels is None else tuple(levels)) + with pytest.raises(ValueError, match=expected): + client.call([], reasoning_level="max") + + +@pytest.mark.parametrize( + "declaration", + [ + {"levels": {"": {"reasoning_effort": "low"}}}, + {"levels": {"low": {}}}, + {"levels": {"low": "low"}}, + {"levels": {"low": {"reasoning_effort": "low"}}, "default": "high"}, + {"default": "high"}, + ], +) +def test_malformed_declarations_fail_early(declaration): + with pytest.raises(ValidationError): + ReasoningConfig(**declaration) + + +@pytest.mark.parametrize( + "field", + [ + "model", + "api_base", + "base_url", + "api_key", + "custom_llm_provider", + "messages", + "input", + "extra_body", + "reasoning_levels", + "reasoning_default", + "reasoning_level", + "client", + ], +) +def test_level_settings_cannot_replace_framework_or_routing_fields(field): + with pytest.raises(ValidationError, match="reserved.*" + field): + CompletionClient("openai/test", reasoning_levels={"low": {field: "value"}}) + + +def test_level_settings_allow_new_provider_fields_without_an_allowlist(): + config = ReasoningConfig(levels={"low": {"future_provider_control": {"budget": 12}}}) + assert config.settings("low") == {"future_provider_control": {"budget": 12}} + + +@pytest.mark.parametrize("field", ["model", "api_base", "extra_body", "reasoning_level"]) +def test_mutating_a_declared_level_cannot_bypass_reserved_fields(field): + config = ReasoningConfig(levels={"low": {"reasoning_effort": "low"}}) + config.levels["low"][field] = "injected" + with pytest.raises(ValueError, match="reserved.*" + field): + apply_reasoning_level(config, "openai/test", {}, {}, "low") + + +def test_no_selection_preserves_raw_controls_and_default_is_only_metadata(): + raw = {"reasoning": {"effort": "medium", "summary": "auto"}} + with CompletionClient( + "openai/test", reasoning_levels=LEVELS, reasoning_default="low", **raw + ) as client: + assert client._prepare_call_config({}) == raw + assert client._prepare_call_config({"reasoning_effort": "future"}) == { + **raw, + "reasoning_effort": "future", + } + + +def test_selection_replaces_whole_blocks_and_does_not_mutate_configuration(): + levels = {"low": {"reasoning": {"effort": "low", "context": "all_turns"}}} + raw = {"reasoning": {"effort": "high", "summary": "auto"}} + with ResponsesClient( + "openai/test", reasoning_levels=levels, reasoning_level="low", **raw + ) as client: + levels["low"]["reasoning"]["effort"] = "changed by caller" + first = client._prepare_call_config({}) + assert first == LEVELS["low"] # No implicit merge of summary from raw defaults. + first["reasoning"]["effort"] = "changed by SDK" + assert client._prepare_call_config({}) == LEVELS["low"] + assert client._prepare_call_config({"reasoning_level": None}) == raw + assert client.config == raw + + +@pytest.mark.parametrize( + "overrides", + [ + {"reasoning": {"effort": "high"}}, + {"extra_body": {"reasoning": {"effort": "high"}}}, + {"extra_body": MappingProxyType({"reasoning": {"effort": "high"}})}, + ], +) +def test_competing_request_settings_raise(overrides): + with CompletionClient("openai/test", reasoning_levels=LEVELS) as client: + with pytest.raises(ValueError, match="conflicts.*reasoning"): + client.call([], reasoning_level="low", **overrides) + + +@pytest.mark.parametrize("client_type", [CompletionClient, ResponsesClient]) +def test_selected_level_replaces_inherited_extra_body_without_mutating_it(client_type): + inherited = {"reasoning": {"effort": "high"}, "other": {"enabled": True}} + with client_type("openai/test", reasoning_levels=LEVELS, extra_body=inherited) as client: + selected = client._prepare_call_config({"reasoning_level": "low"}) + assert selected["reasoning"] == LEVELS["low"]["reasoning"] + assert selected["extra_body"] == {"other": {"enabled": True}} + assert inherited["reasoning"] == {"effort": "high"} + assert client._prepare_call_config({})["extra_body"] == inherited + with pytest.raises(ValueError, match="conflicts.*reasoning"): + client._prepare_call_config({"reasoning_level": "low", "extra_body": inherited}) + + +@pytest.mark.parametrize("client_type", [CompletionClient, ResponsesClient]) +def test_managed_level_rejects_sdk_client_override(client_type, monkeypatch): + from openai import OpenAI + + responses = client_type is ResponsesClient + monkeypatch.setattr( + litellm, "responses" if responses else "completion", lambda **_: _response(responses) + ) + with OpenAI(api_key="test", base_url="https://different-route.test/v1") as sdk: + with client_type("openai/test", reasoning_levels=LEVELS) as client: + with pytest.raises(ValueError, match="route-specific"): + client.call([], reasoning_level="low", client=sdk) + assert client._prepare_call_config({"client": sdk})["client"] is sdk + + +@pytest.mark.parametrize( + "route", + [ + {"model": "openai/other"}, + {"api_base": "https://other.test"}, + {"base_url": "https://other.test"}, + {"custom_llm_provider": "anthropic"}, + ], +) +def test_managed_level_cannot_follow_a_route_override(route): + with CompletionClient("openai/test", reasoning_levels=LEVELS, reasoning_level="low") as client: + with pytest.raises(ValueError, match="route-specific"): + client.call([], **route) + # An explicitly unmanaged call is still the existing raw-transport API. + assert client._prepare_call_config({**route, "reasoning_level": None}) == route + + +@pytest.mark.parametrize( + "overrides", + [ + {"reasoning_levels": LEVELS}, + {"reasoning_default": "low"}, + {"extra_body": {"reasoning_level": "low"}}, + {"extra_body": {"reasoning_levels": LEVELS}}, + {"extra_body": MappingProxyType({"reasoning_level": "low"})}, + ], +) +def test_framework_settings_cannot_leak_through_call_kwargs(overrides): + with CompletionClient("openai/test") as client: + with pytest.raises(ValueError, match="constructor|extra_body"): + client.call([], **overrides) + + +def test_registry_declarations_reach_the_client(monkeypatch): + from nooa.unifiedllm import registry + + config = { + "model_name": "openai/test", + "reasoning_levels": LEVELS, + "reasoning_default": "low", + "reasoning_level": "high", + } + monkeypatch.setattr(registry, "ensure_loaded", lambda: None) + monkeypatch.setattr(registry, "MODELS", {"alias": config}) + assert ModelConfig.from_registry("alias", config).reasoning_default == "low" + with get_llm_client("alias") as client: + assert client.reasoning_levels == ("low", "high") + assert client.reasoning_default == "low" + assert client._prepare_call_config({})["reasoning"] == LEVELS["high"]["reasoning"] + + +@pytest.mark.parametrize( + "route", + [ + {"api_base": "https://different-route.test/v1"}, + {"base_url": "https://different-route.test/v1"}, + {"model": "openai/other"}, + {"custom_llm_provider": "openai"}, + {"client_type": "responses"}, + {"client": object()}, + ], +) +def test_registry_route_override_drops_inherited_reasoning(monkeypatch, route): + from nooa.unifiedllm import registry + + config = { + "model_name": "openai/test", + "api_base": "https://original-route.test/v1", + "reasoning_levels": LEVELS, + "reasoning_default": "high", + "reasoning_level": "high", + } + monkeypatch.setattr(registry, "ensure_loaded", lambda: None) + monkeypatch.setattr(registry, "MODELS", {"alias": config}) + with get_llm_client("alias", api_key="test", **route) as client: + assert client.reasoning_levels is None + assert client.reasoning_default is None + assert client.reasoning_level is None + with pytest.raises(ValueError, match="unknown"): + client._prepare_call_config({"reasoning_level": "low"}) + replacement = {"custom": {"reasoning_effort": "medium"}} + with get_llm_client("alias", api_key="test", reasoning_levels=replacement, **route) as client: + assert client.reasoning_levels == ("custom",) + assert client.reasoning_default is None + assert client.reasoning_level is None + assert ( + client._prepare_call_config({"reasoning_level": "custom"})["reasoning_effort"] + == "medium" + ) + assert config["reasoning_levels"] is LEVELS + + +def test_registry_same_route_override_preserves_reasoning(monkeypatch): + from nooa.unifiedllm import registry + + config = { + "model_name": "openai/test", + "api_base": "https://original-route.test/v1", + "reasoning_levels": LEVELS, + "reasoning_default": "low", + } + monkeypatch.setattr(registry, "ensure_loaded", lambda: None) + monkeypatch.setattr(registry, "MODELS", {"alias": config}) + with get_llm_client( + "alias", + model=config["model_name"], + api_base=config["api_base"], + client_type="completion", + api_key="replacement-key", + ) as client: + assert client.reasoning_levels == ("low", "high") + assert client.reasoning_default == "low" + + +@pytest.mark.parametrize("asynchronous", [False, True]) +async def test_fake_does_not_report_success_for_unknown_reasoning(asynchronous): + client = FakeLLMClient() + with pytest.raises(ValueError, match="unknown"): + if asynchronous: + await client.acall([], reasoning_level="low") + else: + client.call([], reasoning_level="low") + assert client.call_count == 0 + + +async def test_agent_authoring_skill_reasoning_example(tmp_path, monkeypatch): + """Execute the shipped skill's registry and selection example without inference.""" + from nooa.skill import _parse_skill_md + from nooa.unifiedllm import registry + + def unexpected_request(*args, **kwargs): + pytest.fail("The skill example must not make live HTTP requests") + + monkeypatch.setattr(httpx.Client, "send", unexpected_request) + monkeypatch.setattr(httpx.AsyncClient, "send", unexpected_request) + name, _, skill = _parse_skill_md( + Path(__file__).resolve().parents[2] / "skills/nooa-agent-authoring" + ) + assert name == "nooa-agent-authoring" + declarations = re.findall(r"```yaml\n(.*?)```", skill, re.DOTALL) + assert len(declarations) == 1, "Provide one executable reasoning registry example" + config = yaml.safe_load(declarations[0])["models"]["my-route"] + (tmp_path / "llm_config.yaml").write_text(declarations[0]) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(registry, "MODELS", {}) + monkeypatch.setattr(registry, "_loaded", False) + snippets = [ + block + for block in re.findall(r"```python\n(.*?)```", skill, re.DOTALL) + if 'reload_registry(Path("llm_config.yaml"))' in block + ] + assert len(snippets) == 1, "Load the example registry before selecting its alias" + scope = {"messages": [{"role": "user", "content": "hello"}]} + with patch("litellm.aresponses", new_callable=AsyncMock) as transport: + transport.return_value = _response(True) + try: + await eval( + compile(snippets[0], "SKILL.md", "exec", ast.PyCF_ALLOW_TOP_LEVEL_AWAIT), scope + ) + client = scope["llm"] + assert client.reasoning_levels == ("low", "medium", "high") + assert client.reasoning_default == "medium" + assert ( + client._prepare_call_config({})["reasoning"] + == config["reasoning_levels"]["high"]["reasoning"] + ) + assert ( + client._prepare_call_config({"reasoning_level": None})["reasoning"] + == config["reasoning"] + ) + assert ( + transport.call_args.kwargs["reasoning"] + == config["reasoning_levels"]["low"]["reasoning"] + ) + assert scope["reply"].content == "ok" + finally: + if "llm" in scope: + await scope["llm"].aclose() diff --git a/tests/unifiedllm/test_reasoning_levels_wire.py b/tests/unifiedllm/test_reasoning_levels_wire.py new file mode 100644 index 000000000..6b6e6bbcf --- /dev/null +++ b/tests/unifiedllm/test_reasoning_levels_wire.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Exercise the installed SDK, not just parameters passed to LiteLLM.""" + +import json +import runpy +from pathlib import Path + +import httpx +import litellm +import pytest +import yaml + +from nooa.unifiedllm import RetryConfig, get_llm_client + +CONFIG_PATH = Path(__file__).resolve().parents[2] / "examples/reasoning_levels/llm_config.yaml" +MODELS = yaml.safe_load(CONFIG_PATH.read_text())["models"] + + +@pytest.mark.parametrize("alias", MODELS) +async def test_live_probe_uses_registry_configuration_without_route_assumptions(alias, monkeypatch): + from nooa import secrets + from nooa.unifiedllm import registry + + monkeypatch.setattr(secrets, "load_secrets_into_env", lambda: None) + monkeypatch.setenv("MODEL_API_KEY", "test") + monkeypatch.setattr(registry, "ensure_loaded", lambda: None) + monkeypatch.setattr(registry, "MODELS", MODELS) + requests = [] + + async def send(http_client, request, **kwargs): + requests.append(request) + return httpx.Response(200, json=_reply(alias), request=request) + + monkeypatch.setattr(httpx.AsyncClient, "send", send) + probe = runpy.run_path( + str(Path(__file__).resolve().parents[1] / "integration/test_reasoning_levels_live.py") + ) + await probe["test_low_effort_on_configured_route"](alias, monkeypatch) + assert len(requests) == 1 + assert requests[0].url.host == "gateway.example.com" + + +def _reply(alias): + if alias == "gpt-5.6-sol": + return { + "id": "resp_test", + "object": "response", + "created_at": 0, + "status": "completed", + "model": "gpt-5.6-sol", + "parallel_tool_calls": False, + "store": False, + "tools": [], + "output": [ + { + "id": "msg_test", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + "usage": {"input_tokens": 5, "output_tokens": 1, "total_tokens": 6}, + } + if alias == "claude-sonnet-5": + return { + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": alias, + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 5, "output_tokens": 1}, + } + return { + "id": "chat_test", + "object": "chat.completion", + "created": 0, + "model": alias, + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6}, + } + + +@pytest.mark.parametrize( + "alias,level", + [(alias, level) for alias, model in MODELS.items() for level in model["reasoning_levels"]], +) +async def test_declared_settings_survive_the_sdk(alias, level, monkeypatch): + from nooa.unifiedllm import registry + + monkeypatch.setattr(registry, "ensure_loaded", lambda: None) + monkeypatch.setattr(registry, "MODELS", MODELS) + # Other suites enable LiteLLM's process-global parameter dropping. It + # overrides even per-call False; this test verifies an unsuppressed request. + monkeypatch.setattr(litellm, "drop_params", False) + bodies = [] + + async def send(http_client, request, **kwargs): + bodies.append(json.loads(request.content)) + return httpx.Response(200, json=_reply(alias), request=request) + + monkeypatch.setattr(httpx.AsyncClient, "send", send) + async with get_llm_client( + alias, + api_key="test", + drop_params=False, + num_retries=0, + retry_config=RetryConfig(max_retries=0, rate_limit_extra_retries=0), + ) as client: + result = await client.acall([{"role": "user", "content": "hello"}], reasoning_level=level) + assert result.content == "ok" + assert len(bodies) == 1 + body = bodies[0] + for key, value in MODELS[alias]["reasoning_levels"][level].items(): + assert body[key] == value + assert not {"reasoning_levels", "reasoning_default", "reasoning_level"} & body.keys() + + +async def test_selected_level_replaces_extra_body_default_on_wire(monkeypatch): + from nooa.unifiedllm import registry + + monkeypatch.setattr(registry, "ensure_loaded", lambda: None) + monkeypatch.setattr(registry, "MODELS", MODELS) + monkeypatch.setattr(litellm, "drop_params", False) + bodies = [] + + async def send(http_client, request, **kwargs): + bodies.append(json.loads(request.content)) + return httpx.Response(200, json=_reply("gpt-5.6-sol"), request=request) + + monkeypatch.setattr(httpx.AsyncClient, "send", send) + inherited = {"reasoning": {"effort": "high"}, "metadata": {"test": "kept"}} + async with get_llm_client( + "gpt-5.6-sol", + api_key="test", + extra_body=inherited, + retry_config=RetryConfig(max_retries=0, rate_limit_extra_retries=0), + ) as client: + await client.acall([{"role": "user", "content": "hello"}], reasoning_level="low") + assert len(bodies) == 1 + assert bodies[0]["reasoning"] == {"effort": "low"} + assert bodies[0]["metadata"] == {"test": "kept"} + assert inherited["reasoning"] == {"effort": "high"}