diff --git a/examples/advanced/codeact_event_sequence.py b/examples/advanced/codeact_event_sequence.py index c9198474b..57806e4fd 100644 --- a/examples/advanced/codeact_event_sequence.py +++ b/examples/advanced/codeact_event_sequence.py @@ -9,7 +9,7 @@ 1. Task - The initial task/prompt 2. ToolCallEvent - The LLM's request to call execute_python 3. ToolResultEvent - The result of code execution -4. LLMOutput - The final structured output +4. LLMResponse - The final structured output Run with: uv run python examples/advanced/codeact_event_sequence.py diff --git a/examples/arc_agi_3/tests/test_no_action_cap.py b/examples/arc_agi_3/tests/test_no_action_cap.py index d3592cd7b..26d3cb640 100644 --- a/examples/arc_agi_3/tests/test_no_action_cap.py +++ b/examples/arc_agi_3/tests/test_no_action_cap.py @@ -149,17 +149,6 @@ def _codeact_response(code: str): ToolCall(id=f"call_{abs(hash(code)) & 0xFFFF}", name="execute_python", arguments=args) ], finish_reason="tool_calls", - assistant_message={ - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": f"call_{abs(hash(code)) & 0xFFFF}", - "type": "function", - "function": {"name": "execute_python", "arguments": args}, - } - ], - }, reasoning=None, usage={"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, ) diff --git a/examples/arc_agi_3/tests/test_wait_guard_and_helpers.py b/examples/arc_agi_3/tests/test_wait_guard_and_helpers.py index a96dc1409..18f267b28 100644 --- a/examples/arc_agi_3/tests/test_wait_guard_and_helpers.py +++ b/examples/arc_agi_3/tests/test_wait_guard_and_helpers.py @@ -37,17 +37,6 @@ def _tool_response(name: str, args: dict): content="", tool_calls=[ToolCall(id=call_id, name=name, arguments=args_json)], finish_reason="tool_calls", - assistant_message={ - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": call_id, - "type": "function", - "function": {"name": name, "arguments": args_json}, - } - ], - }, reasoning=None, usage={"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, ) diff --git a/examples/nooa-slides-prototype.ipynb b/examples/nooa-slides-prototype.ipynb index 9f17bad21..832d05a18 100644 --- a/examples/nooa-slides-prototype.ipynb +++ b/examples/nooa-slides-prototype.ipynb @@ -702,9 +702,9 @@ " │ MESSAGES │\n", " │ ┌────────────────────────────────┐ │\n", " │ │ [1] Task(\"analyze this data\") │ │\n", - " │ │ [2] LLMOutput(code=\"...\") │ │\n", + " │ │ [2] LLMResponse(code=\"...\") │ │\n", " │ │ [3] PythonOutput(stdout=\"...\") │ │\n", - " │ │ [4] LLMOutput(code=\"...\") │ │\n", + " │ │ [4] LLMResponse(code=\"...\") │ │\n", " │ │ [5] Message(\"Here are results\")│ │\n", " │ └────────────────────────────────┘ │\n", " │ self.events API: │\n", diff --git a/packages/nooa-acp/src/nooa_acp/event_bridge.py b/packages/nooa-acp/src/nooa_acp/event_bridge.py index 531adfdc0..7c9231f29 100644 --- a/packages/nooa-acp/src/nooa_acp/event_bridge.py +++ b/packages/nooa-acp/src/nooa_acp/event_bridge.py @@ -32,7 +32,7 @@ from nooa.agentdoc import pformat from nooa.context_blocks.events import EventBase, ResultStatus, ToolCallEvent -from nooa.events import LLMComplete, PythonOutput +from nooa.events import LLMResponse, PythonOutput from nooa.interactive import AgentMessage # ACP owns stdout for JSON-RPC; diagnostics belong on stderr, which is where @@ -85,7 +85,7 @@ def __init__(self, agent: CodingAgent, client: Client, session_id: str) -> None: agent.event_manager.on("AgentMessage", self._on_agent_message), agent.event_manager.on("ToolCallEvent", self._on_tool_call), agent.event_manager.on("PythonOutput", self._on_python_output), - agent.event_manager.on("LLMComplete", self._on_llm_complete), + agent.event_manager.on("LLMResponse", self._on_llm_response), agent.event_manager.on("FileEdit", self._on_file_edit), agent.event_manager.on("TerminalCommandStarted", self._on_terminal_started), agent.event_manager.on("TerminalCommandOutput", self._on_terminal_output), @@ -254,18 +254,21 @@ def _on_terminal_finished(self, event: EventBase) -> None: ) ) - def _on_llm_complete(self, event: EventBase) -> None: - if not isinstance(event, LLMComplete): + def _on_llm_response(self, event: EventBase) -> None: + if not isinstance(event, LLMResponse): return - self._cost_usd += event.cost_usd + usage = event.usage + if usage is None: + return + self._cost_usd += usage.cost_usd context_window = getattr(self.agent.llm, "context_window", None) if context_window is None: return self._enqueue( UsageUpdate( session_update="usage_update", - used=event.prompt_tokens, - size=max(context_window, event.prompt_tokens), + used=usage.input_tokens, + size=max(context_window, usage.input_tokens), cost=Cost(amount=self._cost_usd, currency="USD"), ) ) diff --git a/packages/nooa-acp/tests/test_event_bridge.py b/packages/nooa-acp/tests/test_event_bridge.py index d23d3c7d7..796a19e62 100644 --- a/packages/nooa-acp/tests/test_event_bridge.py +++ b/packages/nooa-acp/tests/test_event_bridge.py @@ -25,8 +25,9 @@ ) from nooa.context_blocks.events import ResultStatus, ToolCallEvent -from nooa.events import LLMComplete, PythonOutput +from nooa.events import LLMResponse, PythonOutput from nooa.interactive import AgentMessage +from nooa.llm_types import LLMUsage from nooa.unifiedllm import FakeLLMClient @@ -80,7 +81,9 @@ async def test_bridge_preserves_message_tool_and_usage_order(tmp_path): stdout="hello\n", ) ) - agent.event_manager.add(LLMComplete(prompt_tokens=40, completion_tokens=10, cost_usd=0.25)) + agent.event_manager.add( + LLMResponse(usage=LLMUsage(input_tokens=40, output_tokens=10, cost_usd=0.25)) + ) await bridge.flush() updates = [update for _, update in client.updates] @@ -203,7 +206,9 @@ async def test_bridge_omits_usage_when_context_window_is_unknown(tmp_path): bridge = ACPEventBridge(agent, client, "session-1") # type: ignore[arg-type] agent.event_manager.add(AgentMessage(content="alive")) - agent.event_manager.add(LLMComplete(prompt_tokens=40, completion_tokens=10, cost_usd=0.25)) + agent.event_manager.add( + LLMResponse(usage=LLMUsage(input_tokens=40, output_tokens=10, cost_usd=0.25)) + ) await bridge.flush() # Positive control: prove the bridge is actually forwarding before asserting @@ -217,13 +222,15 @@ async def test_bridge_omits_usage_when_context_window_is_unknown(tmp_path): await agent.close() # Paired positive: the same event with a known context window must emit a - # UsageUpdate. Without this, `return` at the top of _on_llm_complete passes + # UsageUpdate. Without this, `return` at the top of _on_llm_response passes # both halves — an AgentMessageChunk control comes from a different handler # and cannot tell "the guard works" from "usage never fires". sized = CodingAgent(llm=FakeLLMClient(), cwd=tmp_path) sized_client = _RecordingClient() sized_bridge = ACPEventBridge(sized, sized_client, "session-2") # type: ignore[arg-type] - sized.event_manager.add(LLMComplete(prompt_tokens=40, completion_tokens=10, cost_usd=0.25)) + sized.event_manager.add( + LLMResponse(usage=LLMUsage(input_tokens=40, output_tokens=10, cost_usd=0.25)) + ) await sized_bridge.flush() assert any(isinstance(update, UsageUpdate) for _, update in sized_client.updates) await sized_bridge.close() @@ -259,6 +266,7 @@ async def test_bridge_emits_structured_file_edit(tmp_path): assert update.locations is not None assert update.locations[0].path == path assert update.locations[0].line == 2 + assert update.content is not None content = cast(FileEditToolCallContent, update.content[0]) assert content.path == path assert content.old_text == "old\n" @@ -293,6 +301,7 @@ async def test_bridge_emits_terminal_lifecycle(tmp_path): assert started.kind == "execute" assert started.title == "$ pytest -q" progress = cast(ToolCallProgress, updates[1]) + assert progress.content is not None content = cast(ContentToolCallContent, progress.content[0]) assert content.content.text == "2 passed\n" finished = cast(ToolCallProgress, updates[2]) diff --git a/packages/nooa-bench/src/nooa_bench/runner.py b/packages/nooa-bench/src/nooa_bench/runner.py index 9da58afcf..7218e5074 100644 --- a/packages/nooa-bench/src/nooa_bench/runner.py +++ b/packages/nooa-bench/src/nooa_bench/runner.py @@ -156,7 +156,9 @@ def _write_trajectory(agent: Any) -> None: { "event_id": event_id, "event_type": type(event).__name__, - **event.model_dump(mode="json"), + # Opaque provider replay state belongs only in the durable event + # backend and compatible provider requests, never debug exports. + **event.model_dump(mode="json", exclude={"llm_state"}), } for event_id, event in manager.items() ] diff --git a/packages/nooa-bench/tests/test_bench_agent.py b/packages/nooa-bench/tests/test_bench_agent.py index 350d6e765..156385091 100644 --- a/packages/nooa-bench/tests/test_bench_agent.py +++ b/packages/nooa-bench/tests/test_bench_agent.py @@ -6,10 +6,11 @@ import pytest from nooa_bench import bench_agent as bench_agent_module +from nooa_bench import runner from nooa_bench.bench_agent import BenchAgent, TaskResult from nooa.agentdoc import doc -from nooa.unifiedllm import FakeLLMClient +from nooa.unifiedllm import FakeLLMClient, LLMResponse class _FakeShell: @@ -34,6 +35,22 @@ def __init__(self, root: str, session: object | None = None) -> None: self.session = session +def test_trajectory_excludes_opaque_provider_state(monkeypatch, tmp_path): + response = LLMResponse( + content="public answer", + llm_state={"encrypted_content": "provider-secret"}, + ) + agent = type("Agent", (), {"event_manager": {response.id: response}})() + monkeypatch.setattr(runner, "LOGS_DIR", tmp_path) + + runner._write_trajectory(agent) + + payload = (tmp_path / "trajectory.json").read_text() + assert "public answer" in payload + assert "provider-secret" not in payload + assert "llm_state" not in payload + + def test_task_result_model(): """TaskResult validates required fields with solution_description.""" r = TaskResult( diff --git a/packages/nooa-memory/src/nooa_memory/README.md b/packages/nooa-memory/src/nooa_memory/README.md index 74021e1a8..64a95158f 100644 --- a/packages/nooa-memory/src/nooa_memory/README.md +++ b/packages/nooa-memory/src/nooa_memory/README.md @@ -198,7 +198,7 @@ specific query; spontaneous recall covers the passive "what's relevant right now |---|---| | `last_message` *(default)* | the most recent user-text message | | `recent_events` | the last `recent_events_n` events, concatenated | -| `working_state` | recent `PythonOutput`/`LLMOutput` (the agent's scratch state) | +| `working_state` | recent `PythonOutput`/`LLMResponse` (the agent's scratch state) | | `distilled` | LLM-distilled query — falls back to `recent_events` with no LLM | If no anchor can be derived (e.g. empty turn) **nothing is injected** — it never diff --git a/packages/nooa-memory/src/nooa_memory/retrieval.py b/packages/nooa-memory/src/nooa_memory/retrieval.py index 3bb43829b..d14666a19 100644 --- a/packages/nooa-memory/src/nooa_memory/retrieval.py +++ b/packages/nooa-memory/src/nooa_memory/retrieval.py @@ -339,7 +339,7 @@ def _strategy_recent_events(agent: object, cfg) -> list[str]: def _strategy_working_state(agent: object, cfg) -> list[str]: texts = [] for ev in _recent_events(agent, cfg.recent_events_n * 2): - if type(ev).__name__ in ("PythonOutput", "LLMOutput"): + if type(ev).__name__ in ("PythonOutput", "LLMResponse"): txt = _event_text(ev) or getattr(ev, "output", "") if isinstance(txt, str) and txt: texts.append(txt) diff --git a/packages/nooa-memory/tests/memory/test_memory_generative.py b/packages/nooa-memory/tests/memory/test_memory_generative.py index ab805750f..b618503e3 100644 --- a/packages/nooa-memory/tests/memory/test_memory_generative.py +++ b/packages/nooa-memory/tests/memory/test_memory_generative.py @@ -138,7 +138,7 @@ def test_distinct_facts_survive_reconciliation(): assert mgr.store.count() == 3 # the model said "distinct" -> nothing archived -def test_malformed_llm_output_is_contained(): +def test_malformed_llm_response_is_contained(): agent = MemAgent() llm = ScriptedLLM(lambda prompt: "sorry, I can't produce JSON today") mgr = _install(agent, reconciler=llm_reconciler(lambda: llm)) diff --git a/skills/nooa-codeact-advanced/SKILL.md b/skills/nooa-codeact-advanced/SKILL.md index 76b702c85..ac536c49b 100644 --- a/skills/nooa-codeact-advanced/SKILL.md +++ b/skills/nooa-codeact-advanced/SKILL.md @@ -171,7 +171,7 @@ Single LLM turn, no tools, no code. The prompt is the docstring plus each parame | `max_retries` | `10` | Validation retries. Each failure adds an `Error` event with the formatted validation error plus the raw output truncated to `max_error_chars` (1000). Exhaustion raises `GenerationError` (not ValidationError). | | `max_param_chars` | `200_000` | **Hard pre-flight guard, not a truncator**: any parameter whose `repr` exceeds it raises `ValueError` — Predict is single-shot, so a silently-truncated input would mean silently-wrong output. Chunk the input or raise/disable (`None`) the limit. | | `max_tokens` / `temperature` / `top_p` | `None` | Forwarded to the LLM call when set. | -| `output_serialization` | `"event"` | How the result is recorded in history: `"event"` keeps the LLMOutput (replayed as a plain assistant message); `"tool_call"` replaces it with a synthetic `return_result` ToolCallEvent — prefer it when a downstream tool-using model reads this history. | +| `output_serialization` | `"event"` | How the result is recorded in history: `"event"` keeps the LLMResponse (replayed as a plain assistant message); `"tool_call"` also appends a synthetic `return_result` ToolCallEvent — prefer it when a downstream tool-using model reads this history. | - Return-type handling: `Optional[X]` unwraps; `dict[K,V]` uses a root-object schema; bare `list`/scalars are wrapped in a hidden `{"value": ...}` schema (Responses-API rejects array-rooted schemas) and unwrapped after validation; models with hidden fields get a public-subset schema and are rehydrated. Non-JSON-serializable **return types** (DataFrame, ndarray) are rejected up front with a pointer to CodeAct (parameters are only size-checked via `max_param_chars`). - Reasoning models: the JSON must land in `content`; `reasoning` is only used as a fallback when content is empty. Prose-in-content + JSON-in-reasoning fails and retries. diff --git a/skills/nooa-context-and-state/SKILL.md b/skills/nooa-context-and-state/SKILL.md index 1d78477dd..5364d7e4e 100644 --- a/skills/nooa-context-and-state/SKILL.md +++ b/skills/nooa-context-and-state/SKILL.md @@ -85,7 +85,7 @@ async def solve(self, problem: str) -> str: ## Events -Event history is what fills the LLM's conversation window. Key model-visible event types (names have no "Event" suffix): `Task`, `Message`, `Reasoning`, `Error`, `Feedback`, `LLMOutput`, `PythonOutput`, `Summary`, `Notification`. Runtime-only events (never shown to the LLM) include `BeforeAgentCall`/`AfterAgentCall`, `LLMCallStart`/`LLMCallEnd`, `LLMComplete` (token/cost metrics). +Event history is what fills the LLM's conversation window. Key model-visible event types (names have no "Event" suffix): `Task`, `Message`, `Reasoning`, `Error`, `Feedback`, `LLMResponse`, `PythonOutput`, `Summary`, `Notification`. `LLMResponse` is both the canonical assistant turn and the home of its token/cost metadata; renderers expose only its conversational fields. Runtime-only events (never shown to the LLM) include `BeforeAgentCall`/`AfterAgentCall` and `LLMCallStart`/`LLMCallEnd`. ```python # Query (AND semantics; chronological; limit keeps most recent) diff --git a/skills/nooa-middleware-hooks/SKILL.md b/skills/nooa-middleware-hooks/SKILL.md index decd95159..de4c3c61e 100644 --- a/skills/nooa-middleware-hooks/SKILL.md +++ b/skills/nooa-middleware-hooks/SKILL.md @@ -1,6 +1,6 @@ --- name: nooa-middleware-hooks -description: Intercept and observe NOOA execution — middleware via event_manager.intercept() (guardrails, input/output transforms, blocking), event observers via event_manager.on() (react to Task/Error/LLMComplete/turn events), and the InstrumentationHooks protocol for observability backends. Use when adding guardrails, redacting or rewriting prompts, blocking or faking an LLM call or code execution, rate-limiting agent methods, subscribing to lifecycle events, or wiring custom telemetry. +description: Intercept and observe NOOA execution — middleware via event_manager.intercept() (guardrails, input/output transforms, blocking), event observers via event_manager.on() (react to Task/Error/LLMResponse/turn events), and the InstrumentationHooks protocol for observability backends. Use when adding guardrails, redacting or rewriting prompts, blocking or faking an LLM call or code execution, rate-limiting agent methods, subscribing to lifecycle events, or wiring custom telemetry. compatibility: nooa package --- @@ -44,7 +44,7 @@ unsubscribe() # intercept() returns a remo Verified semantics: - **Order**: registration order = execution order; first registered is outermost. Nesting across kinds: `agent_call` → per-turn `llm_call` → per-cell `execute_python`. -- **Short-circuiting** (don't call `nxt`) is allowed for guardrails/caching, but you MUST set the output slot (`ctx.result` / `ctx.response`) — the runtime raises `RuntimeError` if middleware returns without it. To fake an LLM turn, construct an `LLMResponse` (`content`, `tool_calls=[]`, `finish_reason="stop"`, `assistant_message={...}`, `raw_response=None`). +- **Short-circuiting** (don't call `nxt`) is allowed for guardrails/caching, but you MUST set the output slot (`ctx.result` / `ctx.response`) — the runtime raises `RuntimeError` if middleware returns without it. To fake an LLM turn, return a fresh `LLMResponse(content=..., tool_calls=[], finish_reason="stop")`; response instances cannot be reused across turns because runtime correlation data is stamped onto the same object that is recorded. - **Blocking**: raise from the middleware — the exception propagates to the caller exactly like a failure of the wrapped operation (for `llm_call`, CodeAct counts it against its session error budget). - **Exceptions are NOT swallowed** — middleware is control flow, unlike hooks. - Per-agent: registered on that agent's `EventManager`; subagents have their own. @@ -63,8 +63,8 @@ unsub = agent.event_manager.on("Error", lambda e: log.warning("agent error: %s", agent.event_manager.on("*", audit) # wildcard: every event ``` -- Useful runtime-only events (never rendered to the model): `BeforeTurn` / `AfterTurn` (per generation turn; `AfterTurn.is_final` marks method completion) and `LLMComplete` (tokens, cost, model_name, tool_calls, reasoning metadata per round-trip — emitted precisely so you don't need `intercept("llm_call")` just to read LLM metrics). -- Model-visible events (`Task`, `Message`, `Error`, `PythonOutput`, ...) are observable the same way — see `nooa-context-and-state` for the full list. +- Useful runtime-only events (never rendered to the model): `BeforeTurn` / `AfterTurn` (per generation turn; `AfterTurn.is_final` marks method completion) and `LLMCallStart` / `LLMCallEnd`. +- `LLMResponse` is the canonical, model-visible assistant turn. It also carries hidden token, cost, model, and reasoning metadata, so observers can read LLM metrics without wrapping `intercept("llm_call")`. Other model-visible events (`Task`, `Message`, `Error`, `PythonOutput`, ...) are observable the same way — see `nooa-context-and-state` for the full list. - The summarizers are the house pattern: subscribe to `AfterTurn` to *schedule* work, apply it at the next `BeforeTurn` (`agents/summarization.py:159-160`). ## InstrumentationHooks (`set_hooks`) @@ -90,7 +90,7 @@ set_hooks(TimingHooks()) # set_hooks(None) removes ## Pitfalls -- Don't use hooks for app logic (they're swallowed-exception observational); don't use middleware for metrics you can get from `LLMComplete` (you'd pay complexity for nothing). +- Don't use hooks for app logic (they're swallowed-exception observational); don't use middleware for metrics you can get from `LLMResponse` (you'd pay complexity for nothing). - `AgentCallContext.result` uses a not-set sentinel — a short-circuiting `agent_call` middleware that "returns None" on purpose must still assign `ctx.result = None`. - Middleware lives on the instance's event manager: install in `__init__` (after `super().__init__()`) or on the constructed agent, not on the class. - Keep `llm_call` middleware fast — it's on the critical path of every turn, and runs again on context-window retries. diff --git a/src/nooa/_llm_state.py b/src/nooa/_llm_state.py new file mode 100644 index 000000000..4bc22571a --- /dev/null +++ b/src/nooa/_llm_state.py @@ -0,0 +1,150 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Private in-memory transport for reasoning replay metadata. + +The mapping stays provider-wire-safe: JSON serializers see only its ordinary +message fields. UnifiedLLM consumes the private attributes before dispatch, +demotes portable text reasoning, and drops opaque state unless a later +provider-specific layer explicitly recognizes it. +""" + +from __future__ import annotations + +import copy +from typing import Any +from uuid import uuid4 + +LLM_STATE_KEY = "_nooa_llm_state" + + +class ReplayCarryingMessage(dict[str, Any]): + """A public wire message with replay metadata outside its mapping.""" + + __slots__ = ("llm_state", "reasoning", "replay_batch_id", "replay_batch_size") + + def __init__( + self, + message: dict[str, Any], + llm_state: dict[str, Any] | None = None, + reasoning: str | None = None, + *, + replay_batch_id: str | None = None, + replay_batch_size: int = 0, + ): + super().__init__(message) + self.llm_state = copy.deepcopy(llm_state) + self.reasoning = reasoning + self.replay_batch_id = replay_batch_id + self.replay_batch_size = replay_batch_size + + +def carried_state(message: Any) -> dict[str, Any] | None: + """Read opaque state attached outside a rendered message mapping.""" + state = getattr(message, "llm_state", None) + return state if isinstance(state, dict) else None + + +def carried_reasoning(message: Any) -> str | None: + """Read provider-exposed text reasoning from a rendered message.""" + reasoning = getattr(message, "reasoning", None) + return reasoning if isinstance(reasoning, str) and reasoning else None + + +def carry_replay_batch( + messages: list[dict[str, Any]], + llm_state: dict[str, Any] | None, + reasoning: str | None, +) -> list[dict[str, Any]]: + """Attach replay metadata and one identity to a Responses item batch.""" + batch_id = uuid4().hex + size = len(messages) + return [ + ReplayCarryingMessage( + message, + llm_state if index == 0 else None, + reasoning if index == 0 else None, + replay_batch_id=batch_id, + replay_batch_size=size, + ) + for index, message in enumerate(messages) + ] + + +def carried_replay_batch(message: Any) -> tuple[str, int] | None: + """Return a rendered Responses batch identity stored outside its mapping.""" + batch_id = getattr(message, "replay_batch_id", None) + batch_size = getattr(message, "replay_batch_size", 0) + if isinstance(batch_id, str) and isinstance(batch_size, int) and batch_size > 0: + return batch_id, batch_size + return None + + +def demote_reasoning_text(message: dict[str, Any], reasoning: str | None) -> None: + """Demote portable reasoning onto an assistant message without duplication.""" + if not reasoning or message.get("role") != "assistant": + return + content = message.get("content") + if isinstance(content, str): + if content.strip() == reasoning.strip(): + return + message["content"] = f"{reasoning}\n\n{content}" if content else reasoning + elif isinstance(content, list): + message["content"] = [{"type": "text", "text": reasoning}, *content] + elif content is None: + message["content"] = reasoning + + +def demote_chat_reasoning(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Build public Chat messages, withholding opaque state by default.""" + prepared: list[dict[str, Any]] = [] + for original in messages: + state = carried_state(original) + reasoning = carried_reasoning(original) + message = copy.deepcopy(dict(original)) + message.pop(LLM_STATE_KEY, None) + demote_reasoning_text(message, reasoning) + if ( + state is not None + and not reasoning + and message.get("role") == "assistant" + and not message.get("content") + and not message.get("tool_calls") + ): + continue + prepared.append(message) + return prepared + + +def demote_responses_batch( + batch: list[dict[str, Any]], + llm_state: dict[str, Any] | None, + reasoning: str | None, +) -> list[dict[str, Any]]: + """Build public Responses items, withholding opaque state by default.""" + clean = [copy.deepcopy(dict(item)) for item in batch] + for item in clean: + item.pop(LLM_STATE_KEY, None) + + if not reasoning: + if ( + llm_state is not None + and len(clean) == 1 + and clean[0].get("role") == "assistant" + and not clean[0].get("content") + and not clean[0].get("tool_calls") + ): + return [] + return clean + + message = next((item for item in clean if item.get("role") == "assistant"), None) + if message is None: + return [{"role": "assistant", "content": reasoning}, *clean] + if isinstance(message.get("content"), list): + index = clean.index(message) + return [ + *clean[:index], + {"role": "assistant", "content": reasoning}, + *clean[index:], + ] + demote_reasoning_text(message, reasoning) + return clean diff --git a/src/nooa/agentdoc/tests/test_event_max_string_override.py b/src/nooa/agentdoc/tests/test_event_max_string_override.py index d64411653..8b314f47b 100644 --- a/src/nooa/agentdoc/tests/test_event_max_string_override.py +++ b/src/nooa/agentdoc/tests/test_event_max_string_override.py @@ -8,7 +8,7 @@ """ from nooa.agentdoc import pformat -from nooa.events import LLMOutput, PythonOutput, ResultStatus, Summary, Task +from nooa.events import LLMResponse, PythonOutput, ResultStatus, Summary, Task LONG_STRING = "x" * 20_000 MAX_STRING = 100 # Aggressively low to verify the override works @@ -33,9 +33,9 @@ def test_task_prompt_not_truncated(self): rendered = pformat(event, max_string=MAX_STRING) assert LONG_STRING in rendered - def test_llm_output_content_not_truncated(self): - """LLMOutput.content is bounded by token limits — truncating hides previous code.""" - event = LLMOutput(content=LONG_STRING) + def test_llm_response_content_not_truncated(self): + """LLMResponse.content is bounded by token limits — truncating hides previous code.""" + event = LLMResponse(content=LONG_STRING) rendered = pformat(event, max_string=MAX_STRING) assert LONG_STRING in rendered diff --git a/src/nooa/atif/exporter.py b/src/nooa/atif/exporter.py index 01419e81c..4041899c4 100644 --- a/src/nooa/atif/exporter.py +++ b/src/nooa/atif/exporter.py @@ -3,7 +3,7 @@ """ATIF v1.7 exporter — event-driven state machine. Consumes the framework's own event stream (Task, BeforeTurn, -SystemPrompt, LLMComplete, LLMOutput, ToolCallEvent, PythonOutput, +SystemPrompt, LLMResponse, ToolCallEvent, PythonOutput, AfterTurn, etc.) and assembles an in-memory :class:`Trajectory` Pydantic model that is serialized to disk atomically. Handles single-trajectory runs as well as nesting, concurrency, compaction, @@ -48,8 +48,7 @@ AfterTurn, BeforeTurn, Error, - LLMComplete, - LLMOutput, + LLMResponse, Notification, PythonOutput, Reasoning, @@ -111,7 +110,7 @@ def __init__( self.reasoning_content: str | None = None self.message: str = "" self.extra: dict[str, Any] = {} - self.llm_call_count: int = 0 # bumped by LLMComplete + self.llm_call_count: int = 0 # bumped by LLMResponse class _DispatchStep(NamedTuple): @@ -139,8 +138,7 @@ class _DispatchStep(NamedTuple): "AfterAgentCall": "on_after_agent_call", "BeforeTurn": "on_before_turn", "SystemPrompt": "on_system_prompt", - "LLMComplete": "on_llm_complete", - "LLMOutput": "on_llm_output", + "LLMResponse": "on_llm_response", "Reasoning": "on_reasoning", "ToolCallEvent": "on_tool_call_event", "PythonOutput": "on_python_output", @@ -229,12 +227,11 @@ class AtifExporter: * Trajectory root is initialised in ``__init__``. * ``BeforeTurn`` opens a ``_PendingStep`` keyed by ``generation_id``. - * ``LLMComplete`` fills metrics, model_name, tool_calls, - reasoning_content on the matching pending step. - * ``LLMOutput`` sets the pending step's assistant message text. + * ``LLMResponse`` fills the matching step's assistant message, metrics, + model name, tool calls, and reasoning. * ``ToolCallEvent`` (creation) registers the tool_call_id ⇆ Python event-reference mapping; tool_calls[i] entries are - already in place from ``LLMComplete``. + already in place from ``LLMResponse``. * ``PythonOutput`` registers observation content for an ``execute_python`` call (the dominant case). * ``AfterTurn`` closes the pending step: builds @@ -355,7 +352,7 @@ def _dispatch_event(self, event: EventBase) -> None: """Route every event off the EventManager's wildcard subscription. Specific event types listed in :data:`_HANDLER_DISPATCH` route to - their dedicated handlers (``on_task``, ``on_llm_complete``, etc.). + their dedicated handlers (``on_task``, ``on_llm_response``, etc.). Custom :class:`EventBase` subclasses defined outside the framework (or added in future releases) fall through to @@ -484,7 +481,7 @@ def on_system_prompt(self, event: SystemPrompt) -> None: (the LLM saw the same system prompt — no information added). - **Subsequent occurrences with different content**: stash the new content in ``_pending_system_drift``; - :meth:`on_llm_complete` will annotate the next agent step + :meth:`on_llm_response` will annotate the next agent step with ``extra.system_prompt_changed = True`` and ``extra.system_prompt = ``. @@ -619,40 +616,42 @@ def on_before_turn(self, event: BeforeTurn) -> None: ): self._run_token = _atif_exporter_var.set(self) - def on_llm_complete(self, event: LLMComplete) -> None: - """``LLMComplete`` ⇒ fill metrics / tool_calls / model_name / reasoning.""" + def on_llm_response(self, event: LLMResponse) -> None: + """Populate a pending step from the one canonical response event.""" with self._lock: ps = self._pending.get(event.generation_id) if ps is None: logger.debug( - "atif: LLMComplete for unknown generation_id=%s (ignored)", + "atif: LLMResponse for unknown generation_id=%s (ignored)", event.generation_id, ) return ps.model_name = event.model_name or ps.model_name + usage = event.usage ps.metrics = MetricsSchema( - prompt_tokens=event.prompt_tokens, - completion_tokens=event.completion_tokens, - cached_tokens=event.cached_tokens, - cost_usd=event.cost_usd, - extra={"reasoning_tokens": event.reasoning_tokens} - if event.reasoning_tokens + prompt_tokens=usage.input_tokens if usage else 0, + completion_tokens=usage.output_tokens if usage else 0, + cached_tokens=usage.cached_input_tokens if usage else 0, + cost_usd=usage.cost_usd if usage else 0.0, + extra={"reasoning_tokens": usage.reasoning_tokens} + if usage and usage.reasoning_tokens else None, ) ps.tool_calls = [ ToolCallSchema( - tool_call_id=tc["tool_call_id"], - function_name=tc["function_name"], - arguments=_parse_arguments(tc.get("arguments")), + tool_call_id=tc.id, + function_name=tc.name, + arguments=_parse_arguments(tc.arguments), ) for tc in event.tool_calls ] - if event.reasoning_content: + if event.reasoning: # Accumulate alongside any Reasoning events that fired pre-LLM. if ps.reasoning_content: - ps.reasoning_content += "\n" + event.reasoning_content + ps.reasoning_content += "\n" + event.reasoning else: - ps.reasoning_content = event.reasoning_content + ps.reasoning_content = event.reasoning + ps.message = event.replay_content ps.llm_call_count += 1 # Per-turn dynamic context envelope (re-rendered every LLM call # from current agent state — see runtime/actor.py @@ -673,20 +672,6 @@ def on_llm_complete(self, event: LLMComplete) -> None: self._system_content_hash = hash(self._pending_system_drift) self._pending_system_drift = None - def on_llm_output(self, event: LLMOutput) -> None: - """``LLMOutput`` ⇒ set the assistant message text on the current pending step. - - We use the most-recently-opened pending step. If multiple are - active (concurrent generation), keyed dispatch handles it; this - fallback covers the simple linear case. - """ - with self._lock: - ps = self._most_recent_pending() - if ps is None: - logger.debug("atif: LLMOutput with no pending step (ignored)") - return - ps.message = event.content - def on_tool_call_event(self, event: ToolCallEvent) -> None: """``ToolCallEvent`` ⇒ index by tool_call_id for later result lookup. @@ -700,7 +685,7 @@ def on_tool_call_event(self, event: ToolCallEvent) -> None: _emit_synthetic_inline_return``), append it to the current pending step's ``tool_calls``. This makes framework-emitted completion markers visible in the trajectory even though - ``LLMComplete.tool_calls`` did not include them. + ``LLMResponse.tool_calls`` did not include them. """ with self._lock: self._tool_call_events[event.tool_call_id] = event diff --git a/src/nooa/config/strategy_config.py b/src/nooa/config/strategy_config.py index d63766e0b..3d57522ae 100644 --- a/src/nooa/config/strategy_config.py +++ b/src/nooa/config/strategy_config.py @@ -162,11 +162,11 @@ def _normalize_conditions(cls, value: Any, info: Any) -> Sequence[Any]: # ``None`` = unconstrained (parameter-size guard disabled). max_param_chars: int | None = 200_000 # How the Predict output is serialized back into the conversation history: - # - "event": The LLMOutput event stays; it replays as a plain assistant + # - "event": The LLMResponse event stays; it replays as a plain assistant # message (raw JSON content, no wrapper). - # - "tool_call": Replace with a synthetic return_result() ToolCallEvent that - # renders natively through the provider formatter — clearer for downstream - # tool-using models that read this history. + # - "tool_call": Keep the LLMResponse and append a synthetic return_result() + # ToolCallEvent that renders natively through the provider formatter — + # clearer for downstream tool-using models that read this history. output_serialization: Literal["event", "tool_call"] = "event" def merge_with(self, other: "PredictConfig") -> "PredictConfig": diff --git a/src/nooa/context_blocks/events.py b/src/nooa/context_blocks/events.py index 5d3722764..ed872347d 100644 --- a/src/nooa/context_blocks/events.py +++ b/src/nooa/context_blocks/events.py @@ -66,6 +66,7 @@ class EventStatus(StrEnum): class ResultStatus(StrEnum): """Status of a tool result.""" + RUNNING = "running" COMPLETE = "complete" ERROR = "error" @@ -209,9 +210,17 @@ class ToolResult(BaseModel): """ tool_call_id: Annotated[str, Field(description="ID of the tool call this is a result for")] - content: Annotated[str, Field(description="Result content from the tool")] + content: Annotated[ + str, + Field( + description=( + "Provider-visible result text; immutable after an LLM generation observes it" + ) + ), + ] result_status: ResultStatus = Field( - default=ResultStatus.COMPLETE, description="Execution status" + default=ResultStatus.COMPLETE, + description="Execution lifecycle status; not part of provider-visible result content", ) @@ -232,13 +241,10 @@ class ToolCallEvent(EventBase): tool_call_id: Annotated[str, Field(description="Unique identifier for this tool call")] name: Annotated[str, Field(description="Name of the tool being called")] arguments: Annotated[dict[str, Any], Field(description="Arguments passed to the tool")] - reasoning_items: list[dict[str, Any]] | None = Field( + llm_response_id: str | None = Field( default=None, repr=False, - description=( - "Opaque provider reasoning state that must accompany this assistant " - "tool call when conversation history is replayed" - ), + description="Canonical LLMResponse event that emitted this tool call", ) # Nested result (filled after execution via EventManager.update()) diff --git a/src/nooa/context_blocks/formatter.py b/src/nooa/context_blocks/formatter.py index f2574a367..56878dcdb 100644 --- a/src/nooa/context_blocks/formatter.py +++ b/src/nooa/context_blocks/formatter.py @@ -23,10 +23,13 @@ from abc import ABC, abstractmethod from collections.abc import Callable from enum import StrEnum -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, TypeGuard + +from nooa._llm_state import ReplayCarryingMessage, carry_replay_batch if TYPE_CHECKING: from nooa.config.truncation_config import FormatConfig + from nooa.llm_types import LLMResponse from nooa.agentdoc import pformat from nooa.context_blocks.events import EventBase, ToolCallEvent @@ -40,6 +43,13 @@ logger = logging.getLogger(__name__) +def _is_llm_response(event: EventBase | None) -> TypeGuard["LLMResponse"]: + """Recognize canonical turns without a module-initialization cycle.""" + from nooa.llm_types import LLMResponse + + return isinstance(event, LLMResponse) + + class FormatType(StrEnum): """Format type identifier used for block-level truncation.""" @@ -102,7 +112,7 @@ def format_event( string. ToolCallEvents are handled structurally by :meth:`format` — they are not routed through ``format_event``. - ASSISTANT-role events (``Message``, ``Reasoning``, ``LLMOutput``) with + ASSISTANT-role events (``Message``, ``Reasoning``, ``LLMResponse``) with string content replay verbatim — no ``EventName(...)`` repr — so the model's past turns look exactly like what it naturally emits. Non-string content (e.g. structured Pydantic models from Predict) @@ -113,6 +123,8 @@ def format_event( No OOM-safety cap is applied here — that belongs to L2 (stdout/stderr capture). """ + if _is_llm_response(event): + return event.replay_content if getattr(event, "_role", None) is Role.ASSISTANT: content = getattr(event, "content", None) if isinstance(content, str): @@ -158,7 +170,7 @@ def _xml_message_content(block: ResolvedBlock) -> str: class name is already inside the rendered content (``PythonOutput(...)``, ``Task(...)``) — nothing is lost, and the tag stays uniform across event types. - * ASSISTANT-role events (``Message``, ``Reasoning``, ``LLMOutput``) + * ASSISTANT-role events (``Message``, ``Reasoning``, ``LLMResponse``) are the LLM's own outputs: they pass through **unwrapped** so the model's past turns replay exactly as it produced them. Wrapping them (the old ```` tag) taught models to emit @@ -198,6 +210,38 @@ def _markdown_message_content(block: ResolvedBlock) -> str: return f"### {role_label}{inline_meta}\n\n{block.content}" +def _tool_result_message(event: ToolCallEvent) -> RenderedMessage: + """Project one execution event into a provider-neutral tool result.""" + if event.result is not None: + content = event.result.content + else: + logger.warning( + "ToolCallEvent %s has result=None — emitting placeholder tool_result " + "to prevent context corruption.", + event.tool_call_id, + ) + content = "(no result recorded)" + return RenderedMessage( + role=Role.TOOL, + content=content, + tool_call_id=event.tool_call_id, + ) + + +def _is_replayable_tool_call_turn(event: Any) -> bool: + """Return whether a captured turn is safe to project as provider tool calls.""" + if event.finish_reason != "tool_calls" or not event.tool_calls: + return False + for call in event.tool_calls: + try: + arguments = json.loads(call.arguments) + except json.JSONDecodeError: + return False + if not isinstance(arguments, dict): + return False + return True + + def _event_block_to_messages( block: ResolvedBlock, *, @@ -215,47 +259,36 @@ def _event_block_to_messages( """ from nooa.context_blocks.models import BlockPart + if _is_llm_response(block.event) and not _is_replayable_tool_call_turn(block.event): + # Replay metadata gets an internal carrier even when the public turn has + # no text. UnifiedLLM drops opaque state and demotes plain reasoning. + if block.event.llm_state or block.event.reasoning: + return [ + RenderedMessage( + role=Role.ASSISTANT, + content=block.event.replay_content or None, + llm_state=block.event.llm_state, + reasoning=block.event.reasoning, + ) + ] + if not block.event.replay_content: + return [] + if isinstance(block.event, ToolCallEvent): event = block.event - messages: list[RenderedMessage] = [ + return [ RenderedMessage( role=Role.ASSISTANT, - tool_call=ToolCallInfo( - id=event.tool_call_id, - name=event.name, - arguments=event.arguments, + tool_calls=( + ToolCallInfo( + id=event.tool_call_id, + name=event.name, + arguments=event.arguments, + ), ), - reasoning_items=event.reasoning_items, - ) + ), + _tool_result_message(event), ] - if event.result is not None: - messages.append( - RenderedMessage( - role=Role.TOOL, - content=event.result.content, - tool_call_id=event.tool_call_id, - ) - ) - else: - # Defensive: ToolCallEvent with result=None should not happen in - # normal flow, but if it does (e.g. a GenerationError was raised - # before the caller could update the event), emit a minimal - # tool_result to avoid producing tool_use without a matching - # tool_result — which corrupts the conversation for the next - # session. - logger.warning( - "ToolCallEvent %s has result=None — emitting placeholder tool_result " - "to prevent context corruption.", - event.tool_call_id, - ) - messages.append( - RenderedMessage( - role=Role.TOOL, - content="(no result recorded)", - tool_call_id=event.tool_call_id, - ) - ) - return messages # Non-tool event content = ( @@ -274,6 +307,84 @@ def _event_block_to_messages( ] +def _event_blocks_to_messages( + blocks: list[ResolvedBlock], + *, + wrap_content: "Callable[[ResolvedBlock], str] | None", +) -> list[RenderedMessage]: + """Project the public event-list IR into neutral conversation turns. + + ``LLMResponse`` is the canonical assistant turn. Linked ``ToolCallEvent`` + objects describe executions of calls from that turn; they are not separate + assistant turns. Grouping here preserves the provider's call batch and + prevents call/result interleaving from changing its meaning. + + Legacy and synthetic ``ToolCallEvent`` objects without a response link + remain independently renderable. Linked executions are only projected as + part of a complete canonical response/result batch. + """ + replayable_turn_ids = { + block.event.id + for block in blocks + if _is_llm_response(block.event) and _is_replayable_tool_call_turn(block.event) + } + executions: dict[str, dict[str, ToolCallEvent]] = {} + for block in blocks: + event = block.event + if ( + isinstance(event, ToolCallEvent) + and event.llm_response_id is not None + and event.llm_response_id in replayable_turn_ids + ): + executions.setdefault(event.llm_response_id, {})[event.tool_call_id] = event + + messages: list[RenderedMessage] = [] + for block in blocks: + event = block.event + if _is_llm_response(event) and _is_replayable_tool_call_turn(event): + by_call_id = executions.get(event.id, {}) + if any( + call.id not in by_call_id or by_call_id[call.id].result is None + for call in event.tool_calls + ): + logger.warning( + "LLMResponse %s has an incomplete visible execution batch — " + "omitting the assistant tool-call turn from replay.", + event.id, + ) + continue + messages.append( + RenderedMessage( + role=Role.ASSISTANT, + # Event projection stores the raw object on a contentless + # block; the canonical provider text lives on LLMResponse. + content=event.replay_content or None, + tool_calls=tuple( + ToolCallInfo( + id=call.id, + name=call.name, + arguments=call.arguments, + ) + for call in event.tool_calls + ), + llm_state=event.llm_state, + reasoning=event.reasoning, + ) + ) + for call in event.tool_calls: + messages.append(_tool_result_message(by_call_id[call.id])) + continue + + # A linked execution is not an independent assistant turn. If its + # source response was filtered out (or its batch is incomplete), fail + # closed instead of fabricating provider history from the sidecar. + if isinstance(event, ToolCallEvent) and event.llm_response_id is not None: + continue + + messages.extend(_event_block_to_messages(block, wrap_content=wrap_content)) + return messages + + def _build_messages( blocks: list[ResolvedBlock], *, @@ -314,8 +425,7 @@ def _build_messages( parts=system_parts if system_parts else None, ) ] - for block in event_blocks: - messages.extend(_event_block_to_messages(block, wrap_content=wrap_message)) + messages.extend(_event_blocks_to_messages(event_blocks, wrap_content=wrap_message)) return messages @@ -419,30 +529,56 @@ def _append_openai_image_message(out: list[dict], msg: RenderedMessage) -> None: out.append({"role": msg.role.value, "content": content_parts}) +def _arguments_json(arguments: dict[str, Any] | str) -> str: + """Return provider-standard JSON text without rewriting captured arguments.""" + return json.dumps(arguments) if isinstance(arguments, dict) else arguments + + +def _arguments_object(arguments: dict[str, Any] | str) -> dict[str, Any]: + """Return Anthropic's object-shaped tool input with a useful failure.""" + if isinstance(arguments, dict): + return arguments + try: + parsed = json.loads(arguments) + except json.JSONDecodeError as exc: + raise ValueError("Anthropic tool arguments must be a JSON object") from exc + if not isinstance(parsed, dict): + raise ValueError("Anthropic tool arguments must decode to a JSON object") + return parsed + + +def _with_replay_data( + message: dict[str, Any], + state: dict[str, Any] | None, + reasoning: str | None, +) -> dict[str, Any]: + """Carry replay metadata outside the provider-visible mapping.""" + return ReplayCarryingMessage(message, state, reasoning) if state or reasoning else message + + class OpenAIProviderFormatter(ProviderFormatter): """Emit OpenAI-compatible messages (``list[dict]``).""" def format(self, messages: list[RenderedMessage]) -> list[dict]: out: list[dict] = [] for msg in messages: - if msg.tool_call is not None: + if msg.tool_calls: assistant_message = { "role": "assistant", - "content": None, + "content": msg.content, "tool_calls": [ { - "id": msg.tool_call.id, + "id": call.id, "type": "function", "function": { - "name": msg.tool_call.name, - "arguments": json.dumps(msg.tool_call.arguments), + "name": call.name, + "arguments": _arguments_json(call.arguments), }, } + for call in msg.tool_calls ], } - if msg.reasoning_items: - assistant_message["reasoning_items"] = msg.reasoning_items - out.append(assistant_message) + out.append(_with_replay_data(assistant_message, msg.llm_state, msg.reasoning)) elif msg.tool_call_id is not None: out.append( { @@ -456,7 +592,13 @@ def format(self, messages: list[RenderedMessage]) -> list[dict]: else: if msg.role in (Role.RUNTIME_EVENT, Role.METADATA): continue - out.append({"role": msg.role.value, "content": msg.content or ""}) + out.append( + _with_replay_data( + {"role": msg.role.value, "content": msg.content or ""}, + msg.llm_state, + msg.reasoning, + ) + ) return out @@ -472,19 +614,28 @@ def format(self, messages: list[RenderedMessage]) -> dict: system_parts.append(msg.content) continue - if msg.tool_call is not None: - out.append( + if msg.tool_calls: + content: list[dict[str, Any]] = [] + if msg.content: + content.append({"type": "text", "text": msg.content}) + content.extend( { - "role": "assistant", - "content": [ - { - "type": "tool_use", - "id": msg.tool_call.id, - "name": msg.tool_call.name, - "input": msg.tool_call.arguments, - } - ], + "type": "tool_use", + "id": call.id, + "name": call.name, + "input": _arguments_object(call.arguments), } + for call in msg.tool_calls + ) + out.append( + _with_replay_data( + { + "role": "assistant", + "content": content, + }, + msg.llm_state, + msg.reasoning, + ) ) elif msg.tool_call_id is not None: out.append( @@ -509,7 +660,13 @@ def format(self, messages: list[RenderedMessage]) -> dict: if msg.role in (Role.RUNTIME_EVENT, Role.METADATA): continue role = msg.role if msg.role in (Role.USER, Role.ASSISTANT) else Role.USER - out.append({"role": role.value, "content": msg.content or ""}) + out.append( + _with_replay_data( + {"role": role.value, "content": msg.content or ""}, + msg.llm_state, + msg.reasoning, + ) + ) return {"system": "\n\n".join(system_parts), "messages": out} @@ -533,22 +690,24 @@ def format(self, messages: list[RenderedMessage]) -> list[dict]: out.append({"role": "system", "content": msg.content or ""}) continue - if msg.tool_call is not None: + if msg.tool_calls: # Preserve assistant text that precedes the tool call + batch: list[dict[str, Any]] = [] if msg.content and msg.role == Role.ASSISTANT: - out.append({"role": "assistant", "content": msg.content}) - if msg.reasoning_items: - out.extend(msg.reasoning_items) - out.append( - { - "type": "function_call", - "call_id": msg.tool_call.id, - "name": msg.tool_call.name, - "arguments": json.dumps(msg.tool_call.arguments) - if isinstance(msg.tool_call.arguments, dict) - else msg.tool_call.arguments, - } - ) + batch.append({"role": "assistant", "content": msg.content}) + for call in msg.tool_calls: + batch.append( + { + "type": "function_call", + "call_id": call.id, + "name": call.name, + "arguments": _arguments_json(call.arguments), + } + ) + if msg.llm_state or msg.reasoning: + out.extend(carry_replay_batch(batch, msg.llm_state, msg.reasoning)) + else: + out.extend(batch) elif msg.tool_call_id is not None: out.append( { @@ -591,5 +750,9 @@ def format(self, messages: list[RenderedMessage]) -> list[dict]: if msg.role in (Role.RUNTIME_EVENT, Role.METADATA): continue role = msg.role if msg.role in (Role.USER, Role.ASSISTANT) else Role.USER - out.append({"role": role.value, "content": msg.content or ""}) + message = {"role": role.value, "content": msg.content or ""} + if (msg.llm_state or msg.reasoning) and role == Role.ASSISTANT: + out.extend(carry_replay_batch([message], msg.llm_state, msg.reasoning)) + else: + out.append(message) return out diff --git a/src/nooa/context_blocks/models.py b/src/nooa/context_blocks/models.py index 57782ad28..17a34d8c5 100644 --- a/src/nooa/context_blocks/models.py +++ b/src/nooa/context_blocks/models.py @@ -215,7 +215,10 @@ class ToolCallInfo(BaseModel): id: Annotated[str, Field(description="Tool call id (matches the result's tool_call_id)")] name: Annotated[str, Field(description="Tool name")] - arguments: Annotated[dict[str, Any], Field(description="Tool arguments as a plain dict")] + arguments: Annotated[ + dict[str, Any] | str, + Field(description="Tool arguments as a plain dict or their original JSON string"), + ] class TextPart(BaseModel): @@ -263,8 +266,8 @@ class RenderedMessage(BaseModel): Fields are optional and combine based on message kind: * A plain text message sets ``role`` and ``content``. - * An assistant tool call sets ``role=ASSISTANT`` and ``tool_call`` (and - leaves ``content=None``). + * An assistant tool-call turn sets ``role=ASSISTANT`` and the complete, + ordered ``tool_calls`` batch. It may also carry assistant ``content``. * A tool result sets ``role=TOOL``, ``tool_call_id`` to the matching call id, and ``content`` to the result text. * A multimodal message sets ``content`` to the text and ``images`` to a @@ -278,7 +281,7 @@ class RenderedMessage(BaseModel): formatters continue to read ``content`` and are unaware of parts. """ - model_config = ConfigDict(frozen=True) + model_config = ConfigDict(frozen=True, extra="forbid") role: Role = Field(description="Message role (SYSTEM / USER / ASSISTANT / TOOL)") content: str | None = Field( @@ -293,12 +296,19 @@ class RenderedMessage(BaseModel): "messages where no blocks are involved." ), ) - tool_call: ToolCallInfo | None = Field( - default=None, description="Assistant tool-call payload, if any" + tool_calls: tuple[ToolCallInfo, ...] = Field( + default_factory=tuple, + description="Complete ordered tool-call batch on an assistant turn", + ) + llm_state: dict[str, Any] | None = Field( + default=None, + repr=False, + description="Opaque state carried only to the UnifiedLLM replay boundary", ) - reasoning_items: list[dict[str, Any]] | None = Field( + reasoning: str | None = Field( default=None, - description="Opaque provider reasoning state associated with an assistant tool call", + repr=False, + description="Plain reasoning carried to UnifiedLLM for replay as assistant text", ) tool_call_id: str | None = Field( default=None, description="Tool-call id this message is a result for" diff --git a/src/nooa/context_blocks/renderers/cached.py b/src/nooa/context_blocks/renderers/cached.py index 4f4866ea6..63cce578d 100644 --- a/src/nooa/context_blocks/renderers/cached.py +++ b/src/nooa/context_blocks/renderers/cached.py @@ -26,7 +26,7 @@ FORMAT_XML, BlockFormatter, FormatType, - _event_block_to_messages, + _event_blocks_to_messages, _xml_system_block, ) from nooa.context_blocks.models import ( @@ -114,15 +114,12 @@ def format(self, blocks: list[ResolvedBlock]) -> list[RenderedMessage]: content, parts = _concat_parts(static_blocks) messages.append(RenderedMessage(role=Role.SYSTEM, content=content, parts=parts)) - # Event messages (wrap like XMLBlockFormatter does, except ToolCallEvents - # still fan out into tool_call + tool_result messages). - for block in message_blocks: - messages.extend( - _event_block_to_messages( - block, - wrap_content=_xml_message_content_shim, - ) + messages.extend( + _event_blocks_to_messages( + message_blocks, + wrap_content=_xml_message_content_shim, ) + ) if dynamic_blocks: dynamic_rendered = [_xml_system_block(b) for b in dynamic_blocks] diff --git a/src/nooa/events.py b/src/nooa/events.py index ca371e688..3954e1ff0 100644 --- a/src/nooa/events.py +++ b/src/nooa/events.py @@ -28,6 +28,7 @@ from nooa.context_blocks import EventBase as EventBase from nooa.context_blocks import ResultStatus as ResultStatus from nooa.context_blocks.models import Role +from nooa.llm_types import LLMResponse as LLMResponse # Sentinel value to distinguish "no return" from "return None" _NO_RETURN = object() @@ -150,16 +151,6 @@ class TextOnlyReply(EventBase): # type: ignore[misc] ] = 0 -class LLMOutput(EventBase): # type: ignore[misc] - """Raw LLM output - code (PURE_PYTHON), JSON (STRUCTURED_OUTPUT), or tool calls (CODEACT).""" - - _role: ClassVar[Role] = Role.ASSISTANT - - content: Annotated[ - str, spec(max_string=None), Field(description="LLM response content (code or JSON)") - ] - - class PythonOutput(EventBase): # type: ignore[misc] """Output from execute_python - appears as user message in events. @@ -499,87 +490,6 @@ class Summary(EventBase): # type: ignore[misc] doc: str = Field(default="", description="Usage hint for accessing collapsed events") -class LLMComplete(EventBase): # type: ignore[misc] - """Emitted after each LLM round-trip completes. - - Carries the LLMResponse metadata (tokens, cost, model_name, - structured tool_calls list, reasoning_content) as a structured - event so downstream consumers don't need ``intercept('llm_call')`` - just to observe LLM metrics. The ATIF exporter relies on this - event for ``step.metrics`` and ``step.tool_calls``. - - Uses ``Role.RUNTIME_EVENT`` to keep it out of LLM context — this - is observability metadata, never rendered to the model. - """ - - _role: ClassVar[Role] = Role.RUNTIME_EVENT - - model_name: Annotated[ - str, - Field(description="Model identifier the LLM call was routed to"), - ] = "" - prompt_tokens: Annotated[ - int, - Field(description="Total input tokens (incl. cached) sent to the model"), - ] = 0 - completion_tokens: Annotated[ - int, - Field(description="Total tokens generated by the model"), - ] = 0 - cached_tokens: Annotated[ - int, - Field(description="Subset of prompt_tokens served from prompt cache"), - ] = 0 - reasoning_tokens: Annotated[ - int, - Field(description="Reasoning tokens (subset of completion_tokens for thinking models)"), - ] = 0 - cost_usd: Annotated[ - float, - Field(description="Estimated monetary cost of this LLM call in USD"), - ] = 0.0 - tool_calls: Annotated[ - list[dict[str, Any]], - Field( - description=( - "Structured tool_calls list from LLMResponse.tool_calls. " - "Each entry: {tool_call_id, function_name, arguments}. " - "Canonical runtime ids (call_* for OpenAI Responses)." - ) - ), - ] = [] # noqa: RUF012 # default_factory not needed; events are immutable post-construction - reasoning_content: Annotated[ - str, - Field(description="Chain-of-thought string from the response, when provided"), - ] = "" - generation_id: Annotated[ - str, - Field( - description=( - "Matches the surrounding BeforeTurn/AfterTurn generation_id; " - "used by consumers (e.g. ATIF exporter) to pair the metrics " - "with the right turn under concurrent generation." - ) - ), - ] = "" - dynamic_context: Annotated[ - str, - Field( - description=( - "Rendered content of the trailing ``…`` " - "envelope that wraps dynamic SYSTEM-role blocks at the end " - "of the messages list (``CachedBlockFormatter`` design). " - "Re-rendered every LLM call from current agent state; " - "captured here so downstream consumers (e.g. ATIF exporter " - "writing it onto ``step.extra.dynamic_context``) can " - "reconstruct what the LLM saw on each turn without having " - "to keep the full per-turn messages list. Empty string when " - "no dynamic blocks are present." - ) - ), - ] = "" - - class SystemPrompt(EventBase): # type: ignore[misc] """Snapshot of the static SYSTEM-role content for an LLM call. @@ -587,7 +497,7 @@ class SystemPrompt(EventBase): # type: ignore[misc] built. Carries the rendered ``messages[0].content`` (the static system prompt: strategy prompt + ``doc(self)`` + static context blocks). The trailing dynamic-context envelope is captured - separately on :class:`LLMComplete.dynamic_context`. + separately on :class:`LLMResponse.dynamic_context`. Uses ``Role.RUNTIME_EVENT`` to keep it out of LLM context — this is pure observability metadata. ATIF consumers emit a single @@ -631,7 +541,7 @@ class SystemPrompt(EventBase): # type: ignore[misc] | Reasoning | Error | Feedback - | LLMOutput + | LLMResponse | PythonOutput | Notification | Summary @@ -639,7 +549,6 @@ class SystemPrompt(EventBase): # type: ignore[misc] | AfterTurn | TuiSessionResumed | TuiSessionCleared - | LLMComplete | SystemPrompt | TextOnlyReply ) diff --git a/src/nooa/llm_types.py b/src/nooa/llm_types.py new file mode 100644 index 000000000..aeccecd45 --- /dev/null +++ b/src/nooa/llm_types.py @@ -0,0 +1,212 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Provider-independent values returned by :mod:`nooa.unifiedllm`.""" + +from __future__ import annotations + +from typing import Annotated, Any, ClassVar, Literal + +from pydantic import BaseModel, Field, field_validator, model_validator + +from nooa.agentdoc import spec +from nooa.context_blocks.events import EventBase +from nooa.context_blocks.roles import Role + + +class ToolCall(BaseModel): + """Provider-independent tool call exactly as emitted by the model.""" + + id: str = Field(description="Provider-assigned identifier used to match the tool result") + name: str = Field(description="Name of the tool requested by the model") + arguments: str = Field(description="Exact JSON argument string emitted by the model") + + +class LLMUsage(BaseModel): + """Normalized usage reported for one successful LLM response.""" + + input_tokens: int = Field(default=0, description="Total input tokens reported by the provider") + output_tokens: int = Field( + default=0, description="Total output tokens reported by the provider" + ) + cached_input_tokens: int = Field( + default=0, description="Input tokens read from the provider's prompt cache" + ) + cache_write_input_tokens: int = Field( + default=0, description="Input tokens written to the provider's prompt cache" + ) + reasoning_tokens: int = Field( + default=0, description="Output tokens attributed to reasoning by the provider" + ) + total_tokens: int = Field( + default=0, description="Total input and output tokens reported by the provider" + ) + cost_usd: float = Field( + default=0.0, description="Estimated call cost in US dollars, when available" + ) + + @classmethod + def from_provider(cls, value: Any) -> LLMUsage | None: + """Normalize common provider and LiteLLM usage shapes once.""" + if value is None: + return None + if isinstance(value, cls): + return value + if hasattr(value, "_asdict"): + value = value._asdict() + elif hasattr(value, "model_dump"): + value = value.model_dump() + + def get(source: Any, key: str, default: Any = None) -> Any: + if isinstance(source, dict): + return source.get(key, default) + return getattr(source, key, default) + + def first(source: Any, *keys: str) -> Any: + for key in keys: + result = get(source, key) + if result is not None: + return result + return None + + prompt_details = first(value, "prompt_tokens_details", "input_tokens_details") + completion_details = first(value, "completion_tokens_details", "output_tokens_details") + input_tokens = int(first(value, "input_tokens", "prompt_tokens") or 0) + output_tokens = int(first(value, "output_tokens", "completion_tokens") or 0) + return cls( + input_tokens=input_tokens, + output_tokens=output_tokens, + cached_input_tokens=int( + first(value, "cached_input_tokens", "cached_tokens", "cache_read_input_tokens") + or first(prompt_details, "cached_tokens", "cache_read_input_tokens") + or 0 + ), + cache_write_input_tokens=int( + first(value, "cache_write_input_tokens", "cache_creation_input_tokens") or 0 + ), + reasoning_tokens=int( + first(value, "reasoning_tokens") + or first(completion_details, "reasoning_tokens") + or 0 + ), + total_tokens=int(first(value, "total_tokens") or input_tokens + output_tokens), + cost_usd=float(first(value, "cost_usd", "cost") or 0.0), + ) + + +class LLMResponse(EventBase): + """Canonical response produced by UnifiedLLM and persisted by NOOA. + + UnifiedLLM creates a fresh object for every call. The runtime records that + same object; renderers project its conversational fields while telemetry + consumers read its model and usage metadata. + """ + + _role: ClassVar[Role] = Role.ASSISTANT + + raw_response: Any = Field( + default=None, + exclude=True, + repr=False, + description=( + "Live provider SDK response; excluded from persistence because it is " + "provider-specific, may not be serializable, and duplicates normalized fields" + ), + ) + content: Annotated[ + str, + spec(max_string=None), + Field(description="Exact normalized assistant text used for replay"), + ] = "" + parsed: Any = Field( + default=None, + exclude=True, + repr=False, + description=( + "Live typed return value; excluded from persistence because arbitrary Python " + "objects are not a durable wire format (the source JSON remains in content " + "or provider-exposed reasoning)" + ), + ) + tool_calls: list[ToolCall] = Field( + default_factory=list, + repr=False, + description="Ordered public tool calls emitted on this assistant turn", + ) + finish_reason: Literal["stop", "tool_calls", "length", "error"] = Field( + default="stop", + repr=False, + description=( + "NOOA-normalized outcome: stop, tool_calls, length, or error; provider-specific " + "finish reasons are deliberately collapsed into these four portable values" + ), + ) + reasoning: str | None = Field( + default=None, + repr=False, + description="Provider-exposed plain reasoning returned with this assistant turn", + ) + llm_state: dict[str, Any] | None = Field( + default=None, + repr=False, + description="Opaque state returned by UnifiedLLM for exact provider replay", + ) + usage: LLMUsage | None = Field( + default=None, + repr=False, + description="Normalized token, cache, reasoning, and cost usage", + ) + model_name: str = Field( + default="", repr=False, description="Model identifier used for this response" + ) + generation_id: str = Field( + default="", + repr=False, + description="Generation turn that produced this response", + ) + dynamic_context: str = Field( + default="", + repr=False, + description=( + "Snapshot of the trailing dynamic context block included in the request that " + "produced this response, retained for session export and debugging" + ), + ) + + @field_validator("usage", mode="before") + @classmethod + def _normalize_usage(cls, value: Any) -> LLMUsage | None: + return LLMUsage.from_provider(value) + + @model_validator(mode="before") + @classmethod + def _separate_parsed_content(cls, value: Any) -> Any: + if not isinstance(value, dict): + return value + content = value.get("content", "") + if content is None: + value = dict(value) + value["content"] = "" + content = "" + if isinstance(content, BaseModel): + value = dict(value) + value.setdefault("parsed", content) + value["content"] = content.model_dump_json() + elif not isinstance(content, str): + value = dict(value) + value["content"] = str(content) + if not value.get("model_name"): + raw_response = value.get("raw_response") + raw_model = ( + raw_response.get("model") + if isinstance(raw_response, dict) + else getattr(raw_response, "model", None) + ) + if isinstance(raw_model, str): + value = dict(value) + value["model_name"] = raw_model + return value + + @property + def replay_content(self) -> str: + """Return the serializable assistant text used for event replay.""" + return self.content diff --git a/src/nooa/nemo_relay_middleware.py b/src/nooa/nemo_relay_middleware.py index 834185556..f3847574c 100644 --- a/src/nooa/nemo_relay_middleware.py +++ b/src/nooa/nemo_relay_middleware.py @@ -39,6 +39,7 @@ from contextvars import ContextVar from typing import TYPE_CHECKING, Any +from nooa.llm_types import LLMResponse from nooa.runtime.middleware import ( MIDDLEWARE_AGENT_CALL, MIDDLEWARE_EXECUTE_PYTHON, @@ -47,6 +48,59 @@ _logger = logging.getLogger(__name__) + +def _relay_response(response: LLMResponse) -> dict[str, Any]: + """Project the canonical response only when NeMo Relay needs wire JSON.""" + result: dict[str, Any] = {"finish_reason": response.finish_reason} + if response.content or response.tool_calls or response.reasoning: + message: dict[str, Any] = {"role": "assistant", "content": response.content} + if response.tool_calls: + message["tool_calls"] = [ + { + "id": call.id, + "type": "function", + "function": {"name": call.name, "arguments": call.arguments}, + } + for call in response.tool_calls + ] + if response.reasoning: + message["reasoning_content"] = response.reasoning + result["message"] = message + if response.usage is not None: + result["usage"] = { + "prompt_tokens": response.usage.input_tokens, + "completion_tokens": response.usage.output_tokens, + "total_tokens": response.usage.total_tokens, + "cached_tokens": response.usage.cached_input_tokens, + "cache_creation_input_tokens": response.usage.cache_write_input_tokens, + "reasoning_tokens": response.usage.reasoning_tokens, + "cost_usd": response.usage.cost_usd, + } + return result + + +def _response_for_relay(response: Any) -> dict[str, Any]: + """Return observable Relay JSON without exposing canonical opaque state.""" + # Canonical responses may retain a raw SDK object containing encrypted + # provider state. Always project their public fields before considering raw + # compatibility fallbacks. + if isinstance(response, LLMResponse): + return _relay_response(response) + raw = getattr(response, "raw_response", None) + if raw is not None and hasattr(raw, "model_dump"): + return raw.model_dump(mode="json") + if hasattr(response, "model_dump"): + return response.model_dump(mode="json") + if hasattr(response, "assistant_message"): + result: dict[str, Any] = {"message": response.assistant_message} + if response.usage: + result["usage"] = response.usage + if response.finish_reason: + result["finish_reason"] = response.finish_reason + return result + return {} + + if TYPE_CHECKING: from nooa.runtime.event_manager import EventManager from nooa.runtime.middleware import ( @@ -182,24 +236,7 @@ async def _wrapper(req: Any) -> Any: resp = captured_ctx.response if resp is None: return {} - # Prefer the raw litellm ModelResponse (Pydantic) — gives NeMo Relay the - # full OpenAI-style structure matching what the old hooks-based - # integration returned via captured_response.model_dump(mode="json"). - raw = getattr(resp, "raw_response", None) - if raw is not None and hasattr(raw, "model_dump"): - return raw.model_dump(mode="json") - # Pydantic response (e.g. passed directly) - if hasattr(resp, "model_dump"): - return resp.model_dump(mode="json") # type: ignore[union-attr] - # Fallback: manual serialization from unifiedllm.LLMResponse dataclass. - if hasattr(resp, "assistant_message"): - result: dict[str, Any] = {"message": resp.assistant_message} - if resp.usage: - result["usage"] = resp.usage - if resp.finish_reason: - result["finish_reason"] = resp.finish_reason - return result - return {} + return _response_for_relay(resp) # Note: nemo_relay.llm.execute() returns the pre-guardrail response. # Sanitize-response guardrails transform data for NeMo Relay internals diff --git a/src/nooa/runtime/actor.py b/src/nooa/runtime/actor.py index bf7325085..c1f036266 100644 --- a/src/nooa/runtime/actor.py +++ b/src/nooa/runtime/actor.py @@ -20,8 +20,6 @@ from typing import TYPE_CHECKING, Any, cast, get_type_hints from uuid import uuid4 -from pydantic import BaseModel - from nooa.agentdoc import FileBackedTruncatingStringIO, TruncatingStringIO from nooa.agentdoc.introspect import methods, variables from nooa.context_blocks import ( @@ -42,10 +40,9 @@ ExecutionSignal, LLMCallEnd, LLMCallStart, - LLMComplete, - LLMOutput, SystemPrompt, ) +from nooa.llm_types import LLMUsage from nooa.runtime.context_vars import ( _current_event_format_var, _in_exec_middleware, @@ -103,10 +100,11 @@ def _make_llm_metrics_bridge(hm: "HarnessMetrics") -> Callable[[str, Any], None] from nooa.runtime.token_usage import accumulate_tokens def _handle_token_usage(usage: Any) -> None: - if isinstance(usage, dict): + normalized = LLMUsage.from_provider(usage) + if normalized is not None: accumulate_tokens( - input_tokens=usage.get("prompt_tokens", 0) or 0, - output_tokens=usage.get("completion_tokens", 0) or 0, + input_tokens=normalized.input_tokens, + output_tokens=normalized.output_tokens, ) _dispatch: dict[str, Callable[[Any], None]] = { @@ -180,7 +178,7 @@ def _snapshot_llm_request( and returns the trailing ``…`` envelope from the same list. Called right after ``_build_messages`` so both reflect the exact bytes about to be sent to the LLM; the returned envelope is - stamped onto the matching :class:`LLMComplete`. ``record=False`` keeps + stamped onto the matching :class:`LLMResponse`. ``record=False`` keeps the snapshot out of the LLM-visible event timeline. """ if messages and isinstance(messages[0], dict) and messages[0].get("role") == "system": @@ -601,7 +599,7 @@ def __init__(self, agent: Any): For callbacks on LLM output, use: agent.event_manager.on("Message", handler) - agent.event_manager.on("LLMOutput", handler) + agent.event_manager.on("LLMResponse", handler) """ # Agent instance self.agent: Any = agent @@ -876,7 +874,7 @@ async def generate( - System message: context blocks + strategy.strategy_prompt - Events: conversation events - Creates an LLMOutput event and adds it to event manager. + Creates an LLMResponse event and adds it to event manager. Args: tools: Optional list of tool definitions. @@ -886,7 +884,7 @@ async def generate( Returns: Tuple of (LLMResponse, event_id) where: - LLMResponse from unifiedllm with content, reasoning, usage - - event_id can be used for event_manager.update() or event_manager.get() + - event_id identifies the canonical provider turn for inspection or linking """ if self._current_method is None: raise RuntimeError("generate() called with no current method context") @@ -1126,59 +1124,20 @@ async def _core_llm(ctx: LLMCallContext) -> LLMCallContext: raise _emit_llm_end(success=True) - # Emit LLMComplete BEFORE LLMOutput so subscribers that build per-turn - # records (e.g. the ATIF exporter) populate metrics/tool_calls before - # the assistant-message content arrives. record=False keeps this off - # the LLM-visible event timeline (Role.RUNTIME_EVENT) but on() - # subscribers still receive it. - _model_name = getattr(llm_client, "model", "") or "" - usage = getattr(response, "usage", None) - # Normalize usage to a dict regardless of whether the provider returned - # a dict, a Pydantic model with attributes, or nothing at all. The - # token-calibration logic above already grovels through both shapes; - # mirror that here so LLMComplete metrics don't silently zero out. - _usage_raw = usage if usage is not None else getattr(response, "usage", None) - _usage_dict: dict[str, Any] = {} - if isinstance(_usage_raw, dict): - _usage_dict = _usage_raw - elif _usage_raw is not None: - for _key in ( - "prompt_tokens", - "input_tokens", - "completion_tokens", - "output_tokens", - "cached_tokens", - "cache_read_input_tokens", - "reasoning_tokens", - "cost", - "cost_usd", - ): - _val = getattr(_usage_raw, _key, None) - if _val is not None: - _usage_dict[_key] = _val - _prompt_details = getattr(_usage_raw, "prompt_tokens_details", None) - if _prompt_details is not None: - _usage_dict["prompt_tokens_details"] = ( - _prompt_details - if isinstance(_prompt_details, dict) - else { - "cached_tokens": getattr(_prompt_details, "cached_tokens", None), - } - ) - _completion_details = getattr(_usage_raw, "completion_tokens_details", None) - if _completion_details is not None: - _usage_dict["completion_tokens_details"] = ( - _completion_details - if isinstance(_completion_details, dict) - else { - "reasoning_tokens": getattr(_completion_details, "reasoning_tokens", None), - } - ) - _prompt_tokens = int( - _usage_dict.get("prompt_tokens") or _usage_dict.get("input_tokens") or 0 - ) - if _prompt_tokens > 0: - self._last_prompt_tokens_actual = _prompt_tokens + # UnifiedLLM creates the canonical response. Enrich that same object + # with runtime correlation data and persist it once as the assistant + # turn; there is no second output or completion event to synchronize. + if response.tag is not None: + raise RuntimeError( + "LLM middleware returned an already-recorded LLMResponse; " + "each call must return a fresh response object" + ) + response.model_name = getattr(llm_client, "model", "") or response.model_name + response.generation_id = current_generation_id or "" + response.dynamic_context = _dynamic_context + usage = response.usage + if usage is not None and usage.input_tokens > 0: + self._last_prompt_tokens_actual = usage.input_tokens if self._last_context_stats is not None: # The provider's exact prompt-token count is the single source of # truth for ctx% display, summarization triggers, and archive @@ -1186,7 +1145,7 @@ async def _core_llm(ctx: LLMCallContext) -> LLMCallContext: # estimate); we write the authoritative value back here. stats = self._last_context_stats self._last_context_stats = stats.model_copy( - update={"prompt_tokens": _prompt_tokens} + update={"prompt_tokens": usage.input_tokens} ) # Recalibrate the chars→tokens ratio from this real response: # tokens_per_char = prompt_tokens / total_chars. The next @@ -1194,56 +1153,8 @@ async def _core_llm(ctx: LLMCallContext) -> LLMCallContext: # or the litellm tokenizer. total_chars = stats.context_blocks_chars + stats.events_chars if total_chars > 0: - self._tokens_per_char = _prompt_tokens / total_chars - _completion_tokens = int( - _usage_dict.get("completion_tokens") or _usage_dict.get("output_tokens") or 0 - ) - _cached_tokens = int( - _usage_dict.get("cached_tokens") - or _usage_dict.get("cache_read_input_tokens") - or (_usage_dict.get("prompt_tokens_details") or {}).get("cached_tokens") - or 0 - ) - _reasoning_tokens = int( - (_usage_dict.get("completion_tokens_details") or {}).get("reasoning_tokens") - or _usage_dict.get("reasoning_tokens") - or 0 - ) - _cost_usd = float(_usage_dict.get("cost") or _usage_dict.get("cost_usd") or 0.0) - _tool_calls_payload = [ - {"tool_call_id": tc.id, "function_name": tc.name, "arguments": tc.arguments} - for tc in (getattr(response, "tool_calls", None) or []) - ] - # _dynamic_context was captured at render time alongside the - # SystemPrompt snapshot (see _snapshot_llm_request), so it reflects - # the exact messages sent to the LLM even across context-window retry. - self.event_manager.add( - LLMComplete( - model_name=_model_name, - prompt_tokens=_prompt_tokens, - completion_tokens=_completion_tokens, - cached_tokens=_cached_tokens, - reasoning_tokens=_reasoning_tokens, - cost_usd=_cost_usd, - tool_calls=_tool_calls_payload, - reasoning_content=getattr(response, "reasoning", None) or "", - generation_id=current_generation_id or "", - dynamic_context=_dynamic_context, - ), - record=False, - ) - - # Create and record LLMOutput - # Serialize Pydantic models to JSON for proper event storage - content = response.content or "" - if isinstance(content, BaseModel): - # Pydantic model - serialize to JSON string - content = content.model_dump_json() - elif not isinstance(content, str): - # Other non-string types - convert to string representation - content = str(content) - event = LLMOutput(content=content) - event_id = self.event_manager.add(event) + self._tokens_per_char = usage.input_tokens / total_chars + event_id = self.event_manager.add(response) return response, event_id diff --git a/src/nooa/runtime/context_builder.py b/src/nooa/runtime/context_builder.py index a5f9d68fd..ab09e7230 100644 --- a/src/nooa/runtime/context_builder.py +++ b/src/nooa/runtime/context_builder.py @@ -30,7 +30,7 @@ ResolvedBlock, Role, ) -from nooa.events import LLMOutput +from nooa.events import LLMResponse if TYPE_CHECKING: from nooa.config.truncation_config import FormatConfig @@ -461,8 +461,9 @@ def _phase_events( # event, so removing only the provider-visible block preserves the # append-only history without producing an invalid message. if ( - isinstance(event, LLMOutput) - and not event.content + isinstance(event, LLMResponse) + and not event.replay_content.strip() + and not event.tool_calls and not getattr(event, "llm_state", None) and not getattr(event, "reasoning", None) ): diff --git a/src/nooa/runtime/event_manager.py b/src/nooa/runtime/event_manager.py index c0cd0ebf5..061050589 100644 --- a/src/nooa/runtime/event_manager.py +++ b/src/nooa/runtime/event_manager.py @@ -48,6 +48,27 @@ # Monotonic counter for stable EventManager identity (middleware re-entry guard). _em_id_counter = itertools.count(1) +# Old rows are migrated at the persistence boundary, but subscriptions are +# executable application code and should be updated instead of silently going +# dead after an event rename. +_REMOVED_EVENT_TYPES = frozenset({"LLMOutput", "LLMComplete"}) + + +def _validate_event_type_name(event_type: str, *, action: str) -> None: + """Reject event APIs folded into the canonical assistant-turn event.""" + if event_type not in _REMOVED_EVENT_TYPES: + return + archive_note = ( + " Stored LLMOutput rows are migrated to LLMResponse automatically when a session is loaded." + if event_type == "LLMOutput" + else " LLMComplete was a non-persisted runtime event, so no stored rows require migration." + ) + raise ValueError( + f"Cannot {action} removed event type {event_type!r}. Use 'LLMResponse' instead. " + "LLMResponse is the canonical assistant-turn event and includes content, reasoning, " + f"tool calls, usage, and replay state.{archive_note}" + ) + def _make_next( mw_fn: Callable[..., Awaitable[Any]], @@ -188,13 +209,20 @@ def on(self, event_type: str, handler: EventHandler) -> Callable[[], None]: """Subscribe to events of a specific type. Args: - event_type: Event type (e.g., "Task", "LLMOutput", "Error") + event_type: Event type (e.g., "Task", "LLMResponse", "Error") or "*" for all events. handler: Callback function receiving Event. Returns: Unsubscribe function - call to remove handler. + + Raises: + ValueError: If *event_type* names a removed event API. The error + identifies its replacement; persisted legacy rows remain + readable through storage migration. """ + _validate_event_type_name(event_type, action="subscribe to") + self._handlers[event_type].append(handler) def unsubscribe() -> None: @@ -371,6 +399,9 @@ def filter( Returns: List of matching events. """ + if type is not None: + _validate_event_type_name(type, action="query") + events = list(self._backend.all_events()) # Apply type filter @@ -409,8 +440,8 @@ def _get_searchable_text(self, event: EventBase) -> str: """Extract searchable text from an event's public fields.""" parts: list[str] = [] - # Get all public fields from model_dump (excludes private fields) - for _field_name, value in event.model_dump().items(): + # Opaque replay state is durable but not a conversational/search field. + for value in event.model_dump(exclude={"llm_state"}).values(): if value is not None: if isinstance(value, list): parts.append(" ".join(str(item) for item in value)) diff --git a/src/nooa/runtime/event_query.py b/src/nooa/runtime/event_query.py index a65ecbb35..077734588 100644 --- a/src/nooa/runtime/event_query.py +++ b/src/nooa/runtime/event_query.py @@ -10,6 +10,7 @@ from typing import Self from nooa.context_blocks import EventBase as EventBase +from nooa.runtime.event_manager import _validate_event_type_name @dataclass(frozen=True) @@ -49,6 +50,10 @@ class EventQuery: regex: bool = False limit: int | None = None + def __post_init__(self) -> None: + if self.type is not None: + _validate_event_type_name(self.type, action="query") + @classmethod def current_call(cls, limit: int | None = None) -> Self: """Filter to events from the current method call only. diff --git a/src/nooa/runtime/harness_metrics.py b/src/nooa/runtime/harness_metrics.py index fa547b977..2c202bf99 100644 --- a/src/nooa/runtime/harness_metrics.py +++ b/src/nooa/runtime/harness_metrics.py @@ -122,7 +122,6 @@ class HarnessMetrics(BaseModel): stop_to_return_result_count: int = 0 stop_to_return_result_previews: list[str] = Field(default_factory=list) text_only_loop_aborts_count: int = 0 - content_prepended_as_comment_count: int = 0 empty_response_count: int = 0 gpt4o_double_quote_fix_count: int = 0 gpt4o_double_quote_fix_previews: list[str] = Field(default_factory=list) @@ -240,9 +239,6 @@ def stop_to_return_result(self, content: str | None) -> None: def text_only_loop_abort(self) -> None: self.text_only_loop_aborts_count += 1 - def content_prepended_as_comment(self) -> None: - self.content_prepended_as_comment_count += 1 - def empty_response(self) -> None: self.empty_response_count += 1 @@ -594,12 +590,6 @@ def _timing_schema_entries( "Response Format Fixups", lambda m: m.text_only_loop_aborts_count, ), - SchemaEntry( - "harness.content_prepended_as_comment.count", - "Content prepended as comment", - "Response Format Fixups", - lambda m: m.content_prepended_as_comment_count, - ), SchemaEntry( "harness.empty_response.count", "Empty responses", diff --git a/src/nooa/runtime/middleware.py b/src/nooa/runtime/middleware.py index 9427f0344..ea7b35a15 100644 --- a/src/nooa/runtime/middleware.py +++ b/src/nooa/runtime/middleware.py @@ -19,7 +19,7 @@ **intercept() vs on()** — both live on ``EventManager``: - ``intercept("llm_call", fn)`` wraps a live operation (can transform / block) -- ``on("LLMOutput", fn)`` observes a recorded event (fire-and-forget, after +- ``on("LLMResponse", fn)`` observes a recorded event (fire-and-forget, after the operation completes and the result is recorded) """ diff --git a/src/nooa/storage/sqlite.py b/src/nooa/storage/sqlite.py index ba6d5db75..744951903 100644 --- a/src/nooa/storage/sqlite.py +++ b/src/nooa/storage/sqlite.py @@ -31,7 +31,6 @@ BeforeTurn, Error, Feedback, - LLMOutput, Message, PythonOutput, Reasoning, @@ -40,6 +39,7 @@ TuiSessionCleared, TuiSessionResumed, ) +from nooa.llm_types import LLMResponse from nooa.storage.json_snapshot import snapshot_from_dict, snapshot_to_dict from nooa.storage.snapshot import AgentSnapshot @@ -82,7 +82,7 @@ def _registry_key(cls: type[EventBase]) -> str: Reasoning, Error, Feedback, - LLMOutput, + LLMResponse, PythonOutput, Summary, BeforeTurn, @@ -293,6 +293,12 @@ def _deserialize(self, data: str) -> EventBase: if not isinstance(raw, dict): raise TypeError("event data JSON root must be an object") event_type = raw.get("event_type", "") + # LLMResponse replaced LLMOutput as the durable assistant-turn event. + # Preserve assistant text when an existing session is resumed; all new + # fields default safely and opaque state was never present on LLMOutput. + if event_type == "LLMOutput": + raw["event_type"] = "LLMResponse" + return LLMResponse.model_validate(raw) cls = self._registry.get(event_type) if cls is None: # Fall back to the global auto-registration registry diff --git a/src/nooa/strategies/base.py b/src/nooa/strategies/base.py index cf5f77e86..acd92a6c8 100644 --- a/src/nooa/strategies/base.py +++ b/src/nooa/strategies/base.py @@ -67,7 +67,7 @@ async def generate( - System message: context blocks + strategy.strategy_prompt - Events: conversation events - Creates an LLMOutput event and adds it to event manager. + Creates an LLMResponse event and adds it to event manager. Args: tools: Optional list of tool definitions. @@ -77,7 +77,7 @@ async def generate( Returns: Tuple of (LLMResponse, event_id) where: - LLMResponse has content, reasoning, usage - - event_id can be used for event_manager.update() (e.g., strip reasoning) + - event_id identifies the append-only provider turn for inspection or linking """ ... diff --git a/src/nooa/strategies/codeact.py b/src/nooa/strategies/codeact.py index 7459aa509..5f70d9928 100644 --- a/src/nooa/strategies/codeact.py +++ b/src/nooa/strategies/codeact.py @@ -23,7 +23,7 @@ import types from collections.abc import AsyncIterator, Awaitable, Callable, Iterator from contextlib import asynccontextmanager -from dataclasses import dataclass, field, replace +from dataclasses import dataclass, field from typing import ( TYPE_CHECKING, Annotated, @@ -79,6 +79,8 @@ logger = logging.getLogger(__name__) +_EXECUTE_PYTHON_RECEIPT = "status: accepted" + @dataclass(frozen=True) class TextOnlyResponseContext: @@ -221,36 +223,6 @@ def __init__(self, result: dict[str, Any]): super().__init__("return_result() called") -def _as_comment(text: str) -> str: - """Render *text* as Python comment lines (one ``#`` prefix per line).""" - return "\n".join(f"# {line}" if line else "#" for line in text.splitlines()) - - -def _prepend_comment(tool_calls: list[ToolCall], text: str) -> list[ToolCall]: - """Return a copy of *tool_calls* with *text* prepended as a comment to the - first execute_python code block. Other tool calls are left unchanged. - """ - result: list[ToolCall] = [] - prepended = False - preview = _as_comment(text) - for tc in tool_calls: - if not prepended and tc.name == "execute_python": - try: - args = json.loads(tc.arguments) - original_code = args.get("code", "") - args["code"] = f"{preview}\n{original_code}" - tc = replace(tc, arguments=json.dumps(args)) - prepended = True - get_harness_metrics().content_prepended_as_comment() - except json.JSONDecodeError: - logger.debug( - "[CODEACT] _prepend_comment: skipping execute_python with unparseable arguments (tool_call_id=%s)", - tc.id, - ) - result.append(tc) - return result - - @dataclass class _ToolCallsResult: """Result of processing tool calls in a single turn.""" @@ -990,14 +962,13 @@ async def _run_generation( continue # Output-limit responses are incomplete even when they carry - # partial text. Preserve non-empty text in its LLMOutput for - # diagnostics, but never let a text-only handler accept it as - # a successful result. + # partial text. Preserve the exact LLMResponse for diagnostics, + # but never let a text-only handler accept it as a successful + # result. Empty turns are filtered from provider projection. if response.finish_reason == "length": session.record_error() if not response.content and not response.tool_calls: get_harness_metrics().empty_response() - runtime.event_manager.remove(event_id) runtime.event_manager.add( DebugTrace( content=f"Truncated response: {_response_debug_details(response)}" @@ -1017,7 +988,6 @@ async def _run_generation( session.record_error() if not response.content and not response.tool_calls: get_harness_metrics().empty_response() - runtime.event_manager.remove(event_id) runtime.event_manager.add( DebugTrace(content=f"Failed response: {_response_debug_details(response)}") ) @@ -1026,30 +996,10 @@ async def _run_generation( # ── Post-response cleanup (CodeAct) ────────────────────── # Intercept point: strategy-specific response transforms. - # Handles text-only→synthetic, comment prepend, tool call - # translation. Consider making extensible in the future. + # Handles text-only recovery and tool-call translation. + # Consider making extensible in the future. if response.finish_reason == "tool_calls" and response.tool_calls: tool_calls = response.tool_calls - assistant_message = getattr(response, "assistant_message", None) - reasoning_items = ( - assistant_message.get("reasoning_items") - if isinstance(assistant_message, dict) - else None - ) - if not isinstance(reasoning_items, list): - reasoning_items = None - # If the LLM also emitted message content alongside the tool - # call(s), preserve it by prepending it as a comment at the - # top of the first execute_python code block. - if response.content: - content = response.content - text = ( - content.model_dump_json() - if isinstance(content, BaseModel) - else str(content) - ) - if text.strip(): - tool_calls = _prepend_comment(tool_calls, text) # A real tool call counts as progress: reset the consecutive # text-only guard (issue 185) before executing, so a single # exec mid-stream rescues the run from accidental drift. @@ -1062,7 +1012,6 @@ async def _run_generation( call, return_type, event_id or "", - reasoning_items=reasoning_items, ) if result.completed: turn_state.success = True @@ -1072,14 +1021,7 @@ async def _run_generation( continue # ── Text-only response (no tool call) ────────────────────── - _raw_content = response.content - _text = ( - _raw_content.model_dump_json() - if isinstance(_raw_content, BaseModel) - else str(_raw_content) - if _raw_content - else "" - ) + _text = response.content _has_text = bool(_text.strip()) if _has_text or response.finish_reason == "stop": @@ -1099,7 +1041,7 @@ async def _run_generation( ) # Capture the drift faithfully for /bug reports. The original - # LLMOutput remains the assistant turn; the handler may only + # LLMResponse remains the assistant turn; the handler may only # append recovery events after it. runtime.event_manager.add( TextOnlyReply( @@ -1146,7 +1088,7 @@ async def _run_generation( call, return_type, event_id or "", - preserve_llm_output=True, + preserve_llm_response=True, ) if result.completed: turn_state.success = True @@ -1175,12 +1117,12 @@ async def _run_generation( # Empty response - error get_harness_metrics().empty_response() session.record_error() - # Capture raw LLM response for debugging before removing the event + # Keep the canonical provider turn and append recovery feedback. + # Context projection omits empty assistant messages from the + # next request without mutating the event journal. runtime.event_manager.add( DebugTrace(content=f"Empty response: {_response_debug_details(response)}") ) - # Remove the empty assistant event - APIs reject empty content - runtime.event_manager.remove(event_id) feedback = await self._tool_use_reminder(runtime, reason="Empty response received.") runtime.event_manager.add(Error(content=feedback)) @@ -1265,17 +1207,16 @@ async def _process_tool_calls( call: "CurrentCall", return_type: Any, event_id: str, - reasoning_items: list[dict[str, Any]] | None = None, - preserve_llm_output: bool = False, + preserve_llm_response: bool = False, ) -> _ToolCallsResult: """Process tool calls from a single LLM turn. Executes tool calls sequentially, stopping at the first error. Returns a _ToolCallsResult indicating whether the task completed. - ``preserve_llm_output`` is used only for tool calls synthesized by a - text-only response handler. Real provider tool calls replace the empty - LLMOutput with their ToolCallEvent representation as before. + ``preserve_llm_response`` distinguishes calls synthesized by a text-only + response handler from calls already recorded on the provider's + canonical LLMResponse event. """ # Handle tool calls - process ALL tool calls sequentially # Some LLMs return multiple tool calls in one response even when @@ -1283,19 +1224,17 @@ async def _process_tool_calls( # cell's output available to subsequent cells via session_locals. session.record_iteration() - if not preserve_llm_output: - # Replace the empty LLMOutput created by runtime.generate() with the - # provider's actual ToolCallEvent representation. - runtime.event_manager.remove(event_id) - num_tool_calls = len(tool_calls) if num_tool_calls > 1: logger.debug(f"[CODEACT] Processing {num_tool_calls} tool calls sequentially") + llm_response = runtime.event_manager.get(event_id) if not preserve_llm_response else None + llm_response_id = getattr(llm_response, "id", None) + # Process each tool call in order, stopping at the first error. # If one cell fails, subsequent cells likely depend on its output # and would cascade into confusing errors. - for tool_call_index, tool_call in enumerate(tool_calls): + for tool_call in tool_calls: # Parse arguments try: args = json.loads(tool_call.arguments) @@ -1313,7 +1252,7 @@ async def _process_tool_calls( tool_call_id=tool_call.id, name=tool_call.name, arguments=args, - reasoning_items=(reasoning_items if tool_call_index == 0 else None), + llm_response_id=llm_response_id, result=None, # Will be updated after execution ) ) @@ -1502,14 +1441,19 @@ async def _handle_execute_python( ) -> Any | None: """Handle execute_python tool call with deferred output pattern. - The deferred output pattern ensures tool result is nested in ToolCallEvent - even when nested agent calls occur during execution: + The deferred output pattern ensures a protocol-valid tool result is + nested in ToolCallEvent even when nested agent calls occur during + execution: - 1. Update ToolCallEvent.result with "status: executing" immediately + 1. Add a stable "status: accepted" receipt immediately 2. Execute code (nested agent events may be added here) - 3. Update ToolCallEvent.result status to "complete" or "error" + 3. Record final success/error without changing the receipt text 4. Add PythonOutput with actual output content + The receipt text must not change after a nested generation has seen it. + Rewriting it from "executing" to "complete" would invalidate the + provider's cached prompt prefix containing the nested trajectory. + Returns the execution result, a tuple ("TASK_COMPLETE", result) if return_result() was called inline, or None if an error occurred. """ @@ -1551,14 +1495,15 @@ async def _handle_execute_python( ) return None - # Update ToolCallEvent with executing status immediately - BEFORE code execution - # This ensures result is nested even if nested agents add events + # Install a stable protocol receipt BEFORE code execution. Nested agent + # generations can observe this message, so its provider-visible content + # must remain byte-identical after execution completes. runtime.event_manager.update( tool_call_event_id, result=ToolResult( tool_call_id=tool_call.id, - content="status: executing", - result_status=ResultStatus.COMPLETE, # Will update to error if needed + content=_EXECUTE_PYTHON_RECEIPT, + result_status=ResultStatus.RUNNING, ), ) @@ -1583,12 +1528,13 @@ async def _handle_execute_python( ) hm.exec_error(error_type, str(result.error)[:500], session.iteration, code[:200]) - # Update ToolCallEvent with final status + # Preserve the already-rendered receipt and update only lifecycle status. + # PythonOutput below appends the actual outcome to the conversation. runtime.event_manager.update( tool_call_event_id, result=ToolResult( tool_call_id=tool_call.id, - content=f"status: {final_status.value}", + content=_EXECUTE_PYTHON_RECEIPT, result_status=final_status, ), ) @@ -1679,7 +1625,7 @@ async def _handle_execute_python( # Emit a synthetic return_result ToolCallEvent so the final # answer appears in the trajectory (otherwise the inline # path leaves no trace of the value). Mirrors PredictStrategy's - # _replace_with_tool_call pattern in predict.py. + # append-only synthetic tool-call pattern. self._emit_synthetic_inline_return(runtime, validated) logger.info("[CODEACT] Task completed successfully via inline return_result()") return ("TASK_COMPLETE", validated) @@ -1976,8 +1922,7 @@ def _emit_synthetic_inline_return(self, runtime: RuntimeServices, value: Any) -> observability, we emit a synthetic ``ToolCallEvent`` with the captured value. - Mirrors :meth:`PredictStrategy._replace_with_tool_call` in - ``predict.py``. The event carries + Mirrors :meth:`PredictStrategy._append_tool_call`. The event carries ``metadata.synthetic = True`` and ``metadata.synthetic_type = "codeact_inline_return"`` so downstream consumers can distinguish framework-emitted markers @@ -2656,13 +2601,14 @@ async def _execute_prefill_step( ) ) - # Update with executing status immediately (deferred output pattern) + # Keep the provider-facing receipt stable if this prefill recursively + # triggers a generation before it completes. runtime.event_manager.update( prefill_event_id, result=ToolResult( tool_call_id=prefill_id, - content="status: executing", - result_status=ResultStatus.COMPLETE, # Will update to error if needed + content=_EXECUTE_PYTHON_RECEIPT, + result_status=ResultStatus.RUNNING, ), ) @@ -2679,13 +2625,13 @@ async def _execute_prefill_step( f"{list(result.captured_locals.keys())}" ) - # Update ToolCallEvent with final status + # Preserve the receipt text; only observability status changes. final_status = ResultStatus.ERROR if result.error else ResultStatus.COMPLETE runtime.event_manager.update( prefill_event_id, result=ToolResult( tool_call_id=prefill_id, - content=f"status: {final_status.value}", + content=_EXECUTE_PYTHON_RECEIPT, result_status=final_status, ), ) diff --git a/src/nooa/strategies/codeact_lite.py b/src/nooa/strategies/codeact_lite.py index 7cb51e28f..c11d8fbac 100644 --- a/src/nooa/strategies/codeact_lite.py +++ b/src/nooa/strategies/codeact_lite.py @@ -22,17 +22,15 @@ from nooa.context_blocks import ( RenderedMessage, ResolvedBlock, - ToolCallEvent, - ToolCallInfo, ) -from nooa.context_blocks.formatter import XMLBlockFormatter +from nooa.context_blocks.formatter import XMLBlockFormatter, _event_blocks_to_messages from nooa.context_blocks.models import Role from nooa.context_blocks.scoped import ScopedContext from nooa.context_blocks.utils import truncating_pformat from nooa.events import ( Error, Feedback, - LLMOutput, + LLMResponse, Message, PythonOutput, Reasoning, @@ -97,8 +95,8 @@ def plain_event_content( parts.append(f"Out[{event.execution_count}]: {value_str}") return "\n".join(parts) if parts else "(no output)" - # Error, Message, Reasoning, LLMOutput, Feedback — use content directly - if isinstance(event, (Error, Message, Reasoning, LLMOutput, Feedback)): + # Error, Message, Reasoning, LLMResponse, Feedback — use content directly + if isinstance(event, (Error, Message, Reasoning, LLMResponse, Feedback)): return event.content # Fallback @@ -169,48 +167,23 @@ def format(self, blocks: list[ResolvedBlock]) -> list[RenderedMessage]: if isinstance(block.event, PythonOutput): python_outputs[block.event.tool_call_id] = block - for block in message_blocks: - if block.role == Role.RUNTIME_EVENT: - continue - - if isinstance(block.event, ToolCallEvent): - event = block.event - messages.append( - RenderedMessage( - role=Role.ASSISTANT, - tool_call=ToolCallInfo( - id=event.tool_call_id, - name=event.name, - arguments=event.arguments, - ), - ) - ) - # Tool result: merge PythonOutput content if available. - py_out_block = python_outputs.get(event.tool_call_id) - if py_out_block and py_out_block.event: - content = self._content_for_block(py_out_block) - elif event.result is not None: - content = event.result.content - else: - content = "" - messages.append( - RenderedMessage( - role=Role.TOOL, - content=content, - tool_call_id=event.tool_call_id, - ) - ) - - elif isinstance(block.event, PythonOutput): - # Already merged into tool result above. - assert block.event.tool_call_id in python_outputs, ( - "PythonOutput not in python_outputs index — indexing loop above should capture all" + event_messages = _event_blocks_to_messages( + [ + block + for block in message_blocks + if block.role != Role.RUNTIME_EVENT and not isinstance(block.event, PythonOutput) + ], + wrap_content=self._content_for_block, + ) + for message in event_messages: + py_out_block = ( + python_outputs.get(message.tool_call_id) if message.tool_call_id else None + ) + if py_out_block is not None: + message = message.model_copy( + update={"content": self._content_for_block(py_out_block)} ) - continue - - else: - content = self._content_for_block(block) - messages.append(RenderedMessage(role=block.role, content=content)) + messages.append(message) return messages diff --git a/src/nooa/strategies/predict.py b/src/nooa/strategies/predict.py index b46f43281..0b6aa3640 100644 --- a/src/nooa/strategies/predict.py +++ b/src/nooa/strategies/predict.py @@ -290,7 +290,7 @@ async def _execute_inner(self, runtime: RuntimeServices, call: "CurrentCall") -> logger.debug(f"[PREDICT attempt={attempt}] Validation successful") if self.config.output_serialization == "tool_call": - self._replace_with_tool_call(runtime, _event_id, validated_data) + self._append_tool_call(runtime, validated_data) return validated_data @@ -451,9 +451,8 @@ def _jsonable(self, value: Any) -> Any: except TypeError: return str(value) - def _replace_with_tool_call(self, runtime: RuntimeServices, event_id: str, result: Any) -> None: - """Replace Predict's LLMOutput with a synthetic return_result tool call.""" - runtime.event_manager.remove(event_id) + def _append_tool_call(self, runtime: RuntimeServices, result: Any) -> None: + """Append a synthetic return_result without replacing the provider turn.""" tool_call_id = f"predict_{uuid4().hex[:8]}" runtime.event_manager.add( ToolCallEvent( @@ -666,10 +665,13 @@ def _parse_llm_response(self, llm_response: Any, method_name: str) -> dict[str, json.JSONDecodeError: If string content cannot be parsed as JSON GenerationError: If response type is unexpected """ - # For structured output, the JSON result should be in the content field. - # The reasoning field contains the model's thinking process (not JSON). - # Try content first, then fall back to reasoning only if content is empty. - if llm_response.content: + # UnifiedLLM keeps the validated object in ``parsed`` while ``content`` + # remains the durable provider text. Plain reasoning is only a fallback + # for providers that place their requested JSON there. + if llm_response.parsed is not None: + content_to_parse = llm_response.parsed + source = "parsed content" + elif llm_response.content: content_to_parse = llm_response.content source = "content" elif llm_response.reasoning: @@ -683,8 +685,7 @@ def _parse_llm_response(self, llm_response: Any, method_name: str) -> dict[str, logger.debug(f"[PREDICT] Parsing: using={source}, type={type(content_to_parse).__name__}") - # Validated Pydantic model directly in response.content - # Convert to dict for our validation layer + # Convert UnifiedLLM's validated Pydantic value to the validation layer's dict. if isinstance(content_to_parse, BaseModel): return content_to_parse.model_dump() diff --git a/src/nooa/strategies/pure_python.py b/src/nooa/strategies/pure_python.py index 12089eca7..eccdb5af3 100644 --- a/src/nooa/strategies/pure_python.py +++ b/src/nooa/strategies/pure_python.py @@ -376,20 +376,17 @@ async def execute(self, runtime: RuntimeServices, call: "CurrentCall") -> Any: if not code: session.record_error() - # Remove the empty assistant event — some APIs reject empty content if generate_event_id is not None: - # Preserve LLM output for trace visibility before removing _evt = runtime.event_manager.get(generate_event_id) _raw = getattr(_evt, "content", "") if _evt else "" runtime.event_manager.add( DebugTrace( content=( - "Removed LLM output (empty code extraction): " + "Empty code extraction from retained LLM output: " f"raw response({len(_raw)} chars)={_raw!r}" ) ) ) - runtime.event_manager.remove(generate_event_id) await self._send_empty_response_error(runtime, call.method_name) continue @@ -516,7 +513,7 @@ async def _run_prefill( Executes prefill code as a synthetic first turn through the normal execution path. Results persist in session_locals for subsequent turns. """ - from nooa.events import LLMOutput + from nooa.context_blocks.events import AssistantEvent if not self.prefill: return @@ -529,10 +526,9 @@ async def _run_prefill( logger.debug(f"[PURE_PYTHON] Running prefill for {call.method_name}") - # Add as assistant message (as if LLM output this code) - # Mark with metadata so it's identifiable in traces + # This is an assistant-role prompt artifact, not a provider response. runtime.event_manager.add( - LLMOutput( + AssistantEvent( content=code, metadata={"prefill": True, "prefill_type": "inspect_inputs"}, ) @@ -588,8 +584,7 @@ async def _generate_code( Returns: (code, event_id): code ready for execution (without fences/XML), - and the event_id of the LLMOutput event so the caller can remove - it if empty (some APIs reject empty assistant messages). + and the event_id of the exact provider LLMResponse event. """ logger.debug( f"[PURE_PYTHON] Loop iteration: iter={session.iteration}/{session.max_iterations}, " @@ -604,26 +599,17 @@ async def _generate_code( try: code = self._strip_wrappers(raw_code) except XMLFormatError as e: - # Preserve LLM output for trace visibility before removing runtime.event_manager.add( DebugTrace( content=( - f"Removed LLM output (XML format error): " + f"Retained LLM output with XML format error: " f"raw_code({len(raw_code)} chars)={raw_code!r}" ) ) ) - # Remove the malformed LLMOutput — some APIs reject empty/malformed content - runtime.event_manager.remove(event_id) runtime.event_manager.add(Error(content=f"**Format Error**: {e}")) raise - # Store the unwrapped code in events so LLM learns to output plain Python. - # Note: legacy reasoning() calls are NOT rewritten — the builtin was - # removed, so they raise NameError and the model corrects itself from - # the error feedback. - runtime.event_manager.update(event_id, content=code) - # Debug breadcrumbs: we keep these fairly high-signal so log output stays useful. logger.debug( "[PURE_PYTHON] Generated code (raw_len=%s, code_len=%s): %s", @@ -733,15 +719,6 @@ async def _execute_code( ) if was_extracted: - # Update events to show unpacked code so LLM learns from example - # Find the most recent generated code event and update it - recent_events = runtime.event_manager.filter(limit=20) - for event in reversed(recent_events): - if event.event_type == "LLMOutput": - runtime.event_manager.update(event.id, content=extracted_code) - logger.debug(f"[PURE_PYTHON] Updated event {event.id} with unpacked code") - break - code = extracted_code # 1) Validate REPL policy (no classes, await async methods) diff --git a/src/nooa/strategies/reflexion.py b/src/nooa/strategies/reflexion.py index abd046871..3dedb751d 100644 --- a/src/nooa/strategies/reflexion.py +++ b/src/nooa/strategies/reflexion.py @@ -272,10 +272,10 @@ async def _reflect( ) # Parse response - if isinstance(response.content, ReflectionOutput): - return response.content - elif isinstance(response.content, dict): - return ReflectionOutput(**response.content) + if isinstance(response.parsed, ReflectionOutput): + return response.parsed + elif isinstance(response.parsed, dict): + return ReflectionOutput(**response.parsed) else: # Fallback: assume NOT satisfactory to trigger retry logger.warning( diff --git a/src/nooa/strategies/tests/test_predict_output_serialization.py b/src/nooa/strategies/tests/test_predict_output_serialization.py index b90c78227..f25aea721 100644 --- a/src/nooa/strategies/tests/test_predict_output_serialization.py +++ b/src/nooa/strategies/tests/test_predict_output_serialization.py @@ -3,13 +3,15 @@ """Tests for PredictStrategy output serialization modes.""" from types import SimpleNamespace +from typing import cast from pydantic import BaseModel from nooa.config.strategy_config import PredictConfig from nooa.context_blocks import ResultStatus, ToolCallEvent -from nooa.events import LLMOutput +from nooa.events import LLMResponse from nooa.runtime.event_manager import EventManager +from nooa.strategies.base import RuntimeServices from nooa.strategies.predict import PredictStrategy @@ -17,22 +19,24 @@ class Payload(BaseModel): value: str -def test_tool_call_mode_replaces_llm_output_with_predict_return_result(): - """Verify tool_call mode replaces LLMOutput with synthetic return_result.""" +def test_tool_call_mode_retains_llm_response_and_appends_predict_return_result(): + """The provider turn remains canonical when a synthetic result is appended.""" strategy = PredictStrategy(PredictConfig(output_serialization="tool_call")) event_manager = EventManager() - event_id = event_manager.add(LLMOutput(content='{"value":"hello"}')) + output = LLMResponse(content='{"value":"hello"}') + event_manager.add(output) - strategy._replace_with_tool_call( - SimpleNamespace(event_manager=event_manager), - event_id, + strategy._append_tool_call( + cast(RuntimeServices, SimpleNamespace(event_manager=event_manager)), Payload(value="hello"), ) events = event_manager.values() - assert len(events) == 1 + assert len(events) == 2 + assert events[0] is output + assert output.content == '{"value":"hello"}' - event = events[0] + event = events[1] assert isinstance(event, ToolCallEvent) assert event.name == "return_result" assert event.tool_call_id.startswith("predict_") @@ -61,6 +65,6 @@ def test_jsonable_sorts_sets_for_deterministic_tool_arguments(): def test_default_output_serialization_is_existing_event_behavior(): - """Verify Predict keeps existing LLMOutput event serialization by default.""" + """Verify Predict keeps existing LLMResponse event serialization by default.""" assert PredictConfig().output_serialization == "event" assert PredictStrategy().config.output_serialization == "event" diff --git a/src/nooa/tracing/_journal_builder.py b/src/nooa/tracing/_journal_builder.py index a470a9b13..63c887626 100644 --- a/src/nooa/tracing/_journal_builder.py +++ b/src/nooa/tracing/_journal_builder.py @@ -88,18 +88,19 @@ def build_journal_payload(messages: list[Any]) -> JournalPayload: blocks[h] = content_s entry["parts"] = [{"block_hash": h}] - if msg.tool_call is not None: - tc = msg.tool_call - args = tc.arguments if isinstance(tc.arguments, str) else json.dumps(tc.arguments) - ah = _hash(args) - blocks[ah] = args - entry["tool_calls"] = [ - { - "id": tc.id, - "type": "function", - "function": {"name": tc.name, "arguments_hash": ah}, - } - ] + if msg.tool_calls: + entry["tool_calls"] = [] + for tc in msg.tool_calls: + args = tc.arguments if isinstance(tc.arguments, str) else json.dumps(tc.arguments) + ah = _hash(args) + blocks[ah] = args + entry["tool_calls"].append( + { + "id": tc.id, + "type": "function", + "function": {"name": tc.name, "arguments_hash": ah}, + } + ) if msg.tool_call_id is not None: entry["tool_call_id"] = msg.tool_call_id if msg.images: @@ -120,7 +121,7 @@ def set_journal_payload_from_messages(messages: list[Any]) -> None: """Public entry point for the runtime: build + publish in one call. Pass an iterable of ``RenderedMessage``-shaped objects (anything with - ``role``, ``parts`` / ``content``, ``tool_call``, ``tool_call_id``, + ``role``, ``parts`` / ``content``, ``tool_calls``, ``tool_call_id``, ``images``). The resulting :class:`JournalPayload` is written into the tracing sideband ``ContextVar``; the journal callback consumes it on the next ``log_pre_api_call``. diff --git a/src/nooa/unifiedllm/__init__.py b/src/nooa/unifiedllm/__init__.py index 321742c0e..591218d68 100644 --- a/src/nooa/unifiedllm/__init__.py +++ b/src/nooa/unifiedllm/__init__.py @@ -1,5 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from nooa.llm_types import LLMResponse, LLMUsage, ToolCall from nooa.unifiedllm.fake import FakeLLMClient from nooa.unifiedllm.http_config import HttpConfig from nooa.unifiedllm.registry import ( @@ -19,11 +20,9 @@ from nooa.unifiedllm.retry_config import RetryConfig from nooa.unifiedllm.unifiedllm import ( CompletionClient, - LLMResponse, ReasoningCompletionClient, ResponsesClient, Tool, - ToolCall, UnifiedLLM, create_tool_from_callable, extract_and_parse_json, @@ -48,6 +47,7 @@ "create_tool_from_callable", # Response types "LLMResponse", + "LLMUsage", # HTTP config "HttpConfig", # Retry utilities diff --git a/src/nooa/unifiedllm/fake.py b/src/nooa/unifiedllm/fake.py index 8a23e7ffe..1b242d6f3 100644 --- a/src/nooa/unifiedllm/fake.py +++ b/src/nooa/unifiedllm/fake.py @@ -5,11 +5,13 @@ import asyncio import json from collections import deque +from datetime import datetime from typing import Any +from uuid import uuid4 from pydantic import BaseModel -from nooa.unifiedllm.unifiedllm import LLMResponse, Tool, ToolCall, UnifiedLLM +from nooa.unifiedllm.unifiedllm import LLMResponse, LLMUsage, Tool, ToolCall, UnifiedLLM class FakeLLMClient(UnifiedLLM): @@ -31,7 +33,24 @@ def __init__( scripted_responses: Pre-defined responses to return (in order). """ super().__init__(model="fake-model") - self._response_queue = deque(scripted_responses or []) + # Each provider call owns one canonical response/event. Tests often use + # ``[response] * n`` as shorthand; materialize those aliases as distinct + # event objects while preserving the first response by identity. + responses: list[LLMResponse] = [] + seen: set[int] = set() + for response in scripted_responses or []: + if id(response) in seen: + response = response.model_copy( + update={ + "id": str(uuid4()), + "metadata": dict(response.metadata), + "tag": None, + "timestamp": datetime.now(), + } + ) + seen.add(id(response)) + responses.append(response) + self._response_queue = deque(responses) self._lock = asyncio.Lock() self.call_count = 0 self.last_messages: list[dict[str, Any]] = [] @@ -75,7 +94,6 @@ async def acall( content="", tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": ""}, reasoning=None, usage=None, ) @@ -101,7 +119,6 @@ def call( content="", tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": ""}, reasoning=None, usage=None, ) @@ -133,13 +150,12 @@ def with_code_responses(cls, code_strings: list[str]) -> "FakeLLMClient": content=code, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": code}, reasoning=None, - usage={ - "prompt_tokens": 10, - "completion_tokens": len(code.split()), - "total_tokens": 10 + len(code.split()), - }, + usage=LLMUsage( + input_tokens=10, + output_tokens=len(code.split()), + total_tokens=10 + len(code.split()), + ), ) ) return cls(scripted_responses=responses) @@ -163,13 +179,12 @@ def simple_message(cls, message: str) -> "FakeLLMClient": content=message, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": message}, reasoning=None, - usage={ - "prompt_tokens": 10, - "completion_tokens": len(words), - "total_tokens": 10 + len(words), - }, + usage=LLMUsage( + input_tokens=10, + output_tokens=len(words), + total_tokens=10 + len(words), + ), ) ] ) @@ -205,22 +220,8 @@ def with_tool_call( ) ], finish_reason="tool_calls", - assistant_message={ - "role": "assistant", - "content": message, - "tool_calls": [ - { - "id": "call_fake_123", - "type": "function", - "function": { - "name": tool_name, - "arguments": json.dumps(tool_args), - }, - } - ], - }, reasoning=None, - usage={"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + usage=LLMUsage(input_tokens=10, output_tokens=5, total_tokens=15), ) ] ) @@ -248,9 +249,8 @@ def with_reasoning( content=message, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": message}, reasoning=reasoning, - usage={"prompt_tokens": 15, "completion_tokens": 10, "total_tokens": 25}, + usage=LLMUsage(input_tokens=15, output_tokens=10, total_tokens=25), ) ] ) diff --git a/src/nooa/unifiedllm/unifiedllm.py b/src/nooa/unifiedllm/unifiedllm.py index 7251d1b81..bc958b7a2 100644 --- a/src/nooa/unifiedllm/unifiedllm.py +++ b/src/nooa/unifiedllm/unifiedllm.py @@ -17,6 +17,16 @@ import litellm from pydantic import BaseModel, RootModel +from nooa._llm_state import ( + LLM_STATE_KEY, + carried_reasoning, + carried_replay_batch, + carried_state, + demote_chat_reasoning, + demote_responses_batch, +) +from nooa.llm_types import LLMResponse, LLMUsage, ToolCall + from .http_config import HttpConfig from .retry import EmptyContentError, sync_retry, with_retry from .retry_config import RetryConfig @@ -661,33 +671,6 @@ def create_tool_from_callable(tool_callable: Callable) -> Tool: ) -@dataclass -class ToolCall: - """Standardized tool call representation across all LLM APIs""" - - id: str - name: str - arguments: str - - -@dataclass -class LLMResponse: - """Standardized response from any LLM API""" - - raw_response: Any - content: str | BaseModel - tool_calls: list[ToolCall] - finish_reason: Literal["stop", "tool_calls", "length", "error"] - assistant_message: dict[str, Any] - reasoning: str | None = None # o1-style or DeepSeek/QwQ reasoning - usage: dict[str, int] | None = None # Token usage stats - - @property - def message(self) -> str | BaseModel | None: - """Backward-compatible alias for content.""" - return self.content - - # --- Bedrock JSON schema sanitization (gl-134) --- # Bedrock Claude rejects schemas with certain JSON schema keywords. # We strip/fix these for Bedrock models and rely on Pydantic's client-side @@ -1095,7 +1078,7 @@ def __repr__(self) -> str: def _update_token_calibration( model: str, messages: list[dict[str, Any]], - usage: dict[str, int], + usage: LLMUsage, tools: list[dict[str, Any]] | None = None, ) -> None: """Update token calibration from an API response's usage data. @@ -1114,7 +1097,7 @@ def _update_token_calibration( and inflated the ratio (observed ~2.7x), which then scaled every displayed/triggering token count up by that bogus factor. """ - actual = usage.get("prompt_tokens") or usage.get("input_tokens") or 0 + actual = usage.input_tokens if actual <= 0: return # Calibration is best-effort: it must NEVER raise out of the (already paid) @@ -1540,10 +1523,10 @@ def _finish_reason_for_tool_calls( return "tool_calls" -def _extract_reasoning_and_usage(raw_response: Any) -> tuple[str | None, dict[str, int] | None]: +def _extract_reasoning_and_usage(raw_response: Any) -> tuple[str | None, LLMUsage | None]: """Extract reasoning and usage from raw LLM response.""" reasoning = None - usage = None + usage: LLMUsage | None = None # Extract reasoning (o1-style or DeepSeek/QwQ) if hasattr(raw_response, "choices") and raw_response.choices: @@ -1552,65 +1535,54 @@ def _extract_reasoning_and_usage(raw_response: Any) -> tuple[str | None, dict[st # Extract usage if hasattr(raw_response, "usage") and raw_response.usage: - usage_obj = raw_response.usage - if hasattr(usage_obj, "_asdict"): - usage = usage_obj._asdict() - elif hasattr(usage_obj, "model_dump"): - usage = usage_obj.model_dump() - elif isinstance(usage_obj, dict): - usage = usage_obj - else: - # Try to extract common fields - usage = { - "prompt_tokens": getattr(usage_obj, "prompt_tokens", 0), - "completion_tokens": getattr(usage_obj, "completion_tokens", 0), - "total_tokens": getattr(usage_obj, "total_tokens", 0), - } + usage = LLMUsage.from_provider(raw_response.usage) return reasoning, usage -def _completion_assistant_message( - message: Any, - *, - tool_calls: list[Any] | None = None, -) -> dict[str, Any]: - """Build a replayable Chat-Completions assistant message. +def _completion_llm_state(message: Any) -> dict[str, Any] | None: + """Capture opaque Chat-Completions reasoning without duplicating the turn. LiteLLM 1.97 bridges GPT-5.4+ function-tool requests to the Responses API and returns the encrypted reasoning state as ``reasoning_items`` on the - chat-shaped message. That state must be replayed with the assistant tool - call on the next turn; dropping it makes multi-turn reasoning tool calls - lose their provider state. + chat-shaped message. Public content and tool calls already have canonical + fields on :class:`LLMResponse`; only the opaque provider state belongs here. """ - assistant_message: dict[str, Any] = { - "role": "assistant", - "content": getattr(message, "content", None) or "", - } - - if tool_calls: - assistant_message["tool_calls"] = [ - { - "id": tool_call.id, - "type": "function", - "function": { - "name": tool_call.function.name, - "arguments": tool_call.function.arguments, - }, - } - for tool_call in tool_calls - ] - reasoning_items = getattr(message, "reasoning_items", None) - if reasoning_items: - assistant_message["reasoning_items"] = [ - item.model_dump(exclude_none=True) - if hasattr(item, "model_dump") - else copy.deepcopy(item) - for item in reasoning_items - ] - - return assistant_message + if not reasoning_items: + return None + return {"reasoning_items": [_opaque_item(item) for item in reasoning_items]} + + +def _opaque_item(item: Any) -> Any: + """Detach one provider-owned item for durable storage.""" + if hasattr(item, "model_dump"): + return item.model_dump(exclude_none=True) + return copy.deepcopy(item) + + +def _item_field(item: Any, name: str) -> Any: + return item.get(name) if isinstance(item, dict) else getattr(item, name, None) + + +def _responses_llm_state(output: list[Any]) -> dict[str, Any] | None: + """Capture opaque Responses items plus lightweight ordering anchors.""" + reasoning_items: list[Any] = [] + order: list[dict[str, Any]] = [] + for item in output: + item_type = _item_field(item, "type") + if item_type == "reasoning": + order.append({"type": "reasoning", "index": len(reasoning_items)}) + reasoning_items.append(_opaque_item(item)) + elif item_type == "function_call": + call_id = _item_field(item, "call_id") + if isinstance(call_id, str): + order.append({"type": "function_call", "call_id": call_id}) + elif item_type == "message": + order.append({"type": "message"}) + if not reasoning_items: + return None + return {"items": reasoning_items, "order": order} def _extract_xml_tool_calls(content: str) -> list["ToolCall"]: @@ -1812,6 +1784,8 @@ def call( If retry_config.retry_on_empty_content is True, will retry when the model returns empty content but has reasoning_content (common with some reasoning models). """ + messages = demote_chat_reasoning(messages) + # Inject cache_control at the message level for prompt caching cache_points = ( self.cache_control_injection_points @@ -1896,16 +1870,14 @@ def _make_call(): return LLMResponse( raw_response=raw_response, - content="", + content=response_message.content or "", tool_calls=tool_calls, finish_reason=_finish_reason_for_tool_calls( _map_completion_finish_reason(raw_response) ), - assistant_message=_completion_assistant_message( - response_message, tool_calls=raw_tool_calls - ), reasoning=reasoning, usage=usage, + llm_state=_completion_llm_state(response_message), ) text_content = raw_response.choices[0].message.content or "" # type: ignore[union-attr] @@ -1922,20 +1894,9 @@ def _make_call(): finish_reason=_finish_reason_for_tool_calls( _map_completion_finish_reason(raw_response) ), - assistant_message={ - "role": "assistant", - "content": text_content, - "tool_calls": [ - { - "id": tc.id, - "type": "function", - "function": {"name": tc.name, "arguments": tc.arguments}, - } - for tc in xml_tool_calls - ], - }, reasoning=reasoning, usage=usage, + llm_state=_completion_llm_state(raw_response.choices[0].message), ) if output_model: @@ -1951,12 +1912,13 @@ def _make_call(): return LLMResponse( raw_response=raw_response, - content=parsed_content, + content=text_content, + parsed=parsed_content, tool_calls=[], finish_reason=_map_completion_finish_reason(raw_response), - assistant_message=_completion_assistant_message(raw_response.choices[0].message), - reasoning=reasoning if text_content else None, + reasoning=reasoning, usage=usage, + llm_state=_completion_llm_state(raw_response.choices[0].message), ) return LLMResponse( @@ -1964,9 +1926,9 @@ def _make_call(): content=text_content, tool_calls=[], finish_reason=_map_completion_finish_reason(raw_response), - assistant_message=_completion_assistant_message(raw_response.choices[0].message), reasoning=reasoning, usage=usage, + llm_state=_completion_llm_state(raw_response.choices[0].message), ) async def acall( @@ -1984,6 +1946,8 @@ async def acall( If retry_config.retry_on_empty_content is True, will retry when the model returns empty content but has reasoning_content (common with some reasoning models). """ + messages = demote_chat_reasoning(messages) + # Inject cache_control at the message level for prompt caching cache_points = ( self.cache_control_injection_points @@ -2070,16 +2034,14 @@ async def _make_call(): return LLMResponse( raw_response=raw_response, - content="", + content=response_message.content or "", tool_calls=tool_calls, finish_reason=_finish_reason_for_tool_calls( _map_completion_finish_reason(raw_response) ), - assistant_message=_completion_assistant_message( - response_message, tool_calls=raw_tool_calls - ), reasoning=reasoning, usage=usage, + llm_state=_completion_llm_state(response_message), ) text_content = raw_response.choices[0].message.content or "" # type: ignore[union-attr] @@ -2096,20 +2058,9 @@ async def _make_call(): finish_reason=_finish_reason_for_tool_calls( _map_completion_finish_reason(raw_response) ), - assistant_message={ - "role": "assistant", - "content": text_content, - "tool_calls": [ - { - "id": tc.id, - "type": "function", - "function": {"name": tc.name, "arguments": tc.arguments}, - } - for tc in xml_tool_calls - ], - }, reasoning=reasoning, usage=usage, + llm_state=_completion_llm_state(raw_response.choices[0].message), ) if output_model: @@ -2125,12 +2076,13 @@ async def _make_call(): return LLMResponse( raw_response=raw_response, - content=parsed_content, + content=text_content, + parsed=parsed_content, tool_calls=[], finish_reason=_map_completion_finish_reason(raw_response), - assistant_message=_completion_assistant_message(raw_response.choices[0].message), - reasoning=reasoning if text_content else None, + reasoning=reasoning, usage=usage, + llm_state=_completion_llm_state(raw_response.choices[0].message), ) return LLMResponse( @@ -2138,9 +2090,9 @@ async def _make_call(): content=text_content, tool_calls=[], finish_reason=_map_completion_finish_reason(raw_response), - assistant_message=_completion_assistant_message(raw_response.choices[0].message), reasoning=reasoning, usage=usage, + llm_state=_completion_llm_state(raw_response.choices[0].message), ) @@ -2197,18 +2149,8 @@ def call( else think_reasoning ) - return LLMResponse( - raw_response=response.raw_response, - content=cleaned_content, - tool_calls=response.tool_calls, - finish_reason=response.finish_reason, - assistant_message={ - "role": "assistant", - "content": cleaned_content, - }, - reasoning=combined_reasoning, - usage=response.usage, - ) + response.content = cleaned_content + response.reasoning = combined_reasoning return response @@ -2238,18 +2180,8 @@ async def acall( else think_reasoning ) - return LLMResponse( - raw_response=response.raw_response, - content=cleaned_content, - tool_calls=response.tool_calls, - finish_reason=response.finish_reason, - assistant_message={ - "role": "assistant", - "content": cleaned_content, - }, - reasoning=combined_reasoning, - usage=response.usage, - ) + response.content = cleaned_content + response.reasoning = combined_reasoning return response @@ -2398,14 +2330,7 @@ def _make_call(): else _make_call() ) - # Extract usage if available (Responses API may have different structure) - usage = None - if hasattr(raw_response, "usage") and raw_response.usage: - usage_obj = raw_response.usage - if hasattr(usage_obj, "model_dump"): - usage = usage_obj.model_dump() - elif isinstance(usage_obj, dict): - usage = usage_obj + usage = LLMUsage.from_provider(getattr(raw_response, "usage", None)) if usage: _update_token_calibration(self.model, messages, usage, tools=api_params.get("tools")) @@ -2418,23 +2343,16 @@ def _make_call(): for tc in raw_tool_calls ] - assistant_messages = [] - for item in output: - if hasattr(item, "model_dump"): - assistant_messages.append(item.model_dump()) - else: - assistant_messages.append(item) - return LLMResponse( raw_response=raw_response, - content="", + content=self._extract_text_from_output(raw_response), tool_calls=tool_calls, finish_reason=_finish_reason_for_tool_calls( _map_responses_finish_reason(raw_response) ), - assistant_message={"_batch": assistant_messages}, reasoning=None, # Responses API doesn't have reasoning usage=usage, + llm_state=_responses_llm_state(output), ) text_content = self._extract_text_from_output(raw_response) @@ -2445,12 +2363,13 @@ def _make_call(): return LLMResponse( raw_response=raw_response, - content=parsed_content, + content=text_content, + parsed=parsed_content, tool_calls=[], finish_reason=_map_responses_finish_reason(raw_response), - assistant_message={"role": "assistant", "content": text_content}, reasoning=None, usage=usage, + llm_state=_responses_llm_state(output), ) return LLMResponse( @@ -2458,9 +2377,9 @@ def _make_call(): content=text_content, tool_calls=[], finish_reason=_map_responses_finish_reason(raw_response), - assistant_message={"role": "assistant", "content": text_content}, reasoning=None, usage=usage, + llm_state=_responses_llm_state(output), ) async def acall( @@ -2531,14 +2450,7 @@ async def _make_call(): else await _make_call() ) - # Extract usage if available (Responses API may have different structure) - usage = None - if hasattr(raw_response, "usage") and raw_response.usage: - usage_obj = raw_response.usage - if hasattr(usage_obj, "model_dump"): - usage = usage_obj.model_dump() - elif isinstance(usage_obj, dict): - usage = usage_obj + usage = LLMUsage.from_provider(getattr(raw_response, "usage", None)) if usage: _update_token_calibration(self.model, messages, usage, tools=api_params.get("tools")) @@ -2551,23 +2463,16 @@ async def _make_call(): for tc in raw_tool_calls ] - assistant_messages = [] - for item in output: - if hasattr(item, "model_dump"): - assistant_messages.append(item.model_dump()) - else: - assistant_messages.append(item) - return LLMResponse( raw_response=raw_response, - content="", + content=self._extract_text_from_output(raw_response), tool_calls=tool_calls, finish_reason=_finish_reason_for_tool_calls( _map_responses_finish_reason(raw_response) ), - assistant_message={"_batch": assistant_messages}, reasoning=None, usage=usage, + llm_state=_responses_llm_state(output), ) text_content = self._extract_text_from_output(raw_response) @@ -2578,12 +2483,13 @@ async def _make_call(): return LLMResponse( raw_response=raw_response, - content=parsed_content, + content=text_content, + parsed=parsed_content, tool_calls=[], finish_reason=_map_responses_finish_reason(raw_response), - assistant_message={"role": "assistant", "content": text_content}, reasoning=None, usage=usage, + llm_state=_responses_llm_state(output), ) return LLMResponse( @@ -2591,9 +2497,9 @@ async def _make_call(): content=text_content, tool_calls=[], finish_reason=_map_responses_finish_reason(raw_response), - assistant_message={"role": "assistant", "content": text_content}, reasoning=None, usage=usage, + llm_state=_responses_llm_state(output), ) def _transform_messages( @@ -2614,7 +2520,32 @@ def _transform_messages( instructions_parts: list[str] = [] transformed: list[dict[str, Any]] = [] - for msg in messages: + skip_batch_items = 0 + for index, original in enumerate(messages): + if skip_batch_items: + skip_batch_items -= 1 + continue + state = copy.deepcopy(carried_state(original)) + reasoning = carried_reasoning(original) + msg = copy.deepcopy(dict(original)) + msg.pop(LLM_STATE_KEY, None) + + batch_info = carried_replay_batch(original) + if batch_info is not None and (state is not None or reasoning is not None): + batch_id, batch_size = batch_info + candidates = messages[index : index + batch_size] + if len(candidates) == batch_size and all( + carried_replay_batch(item) == (batch_id, batch_size) for item in candidates + ): + batch = [copy.deepcopy(dict(item)) for item in candidates] + transformed.extend(demote_responses_batch(batch, state, reasoning)) + skip_batch_items = batch_size - 1 + continue + # Middleware changed the batch. Keep its public items, but do + # not attach private reasoning or state to different neighbors. + state = None + reasoning = None + # System messages → extract to instructions if msg.get("role") == "system": content = msg.get("content", "") @@ -2624,7 +2555,10 @@ def _transform_messages( # Already in native Responses format (from ResponsesProviderFormatter) if "type" in msg: - transformed.append(msg) + if state is not None or reasoning is not None: + transformed.extend(demote_responses_batch([msg], state, reasoning)) + else: + transformed.append(msg) continue # Legacy OpenAI format: tool result messages @@ -2662,11 +2596,12 @@ def _transform_messages( # Legacy OpenAI format: assistant messages with tool_calls if msg.get("role") == "assistant" and msg.get("tool_calls"): # Preserve assistant text that precedes tool calls (matches native formatter) + batch: list[dict[str, Any]] = [] if msg.get("content"): - transformed.append({"role": "assistant", "content": msg["content"]}) + batch.append({"role": "assistant", "content": msg["content"]}) for tc in msg["tool_calls"]: fn = tc.get("function", {}) - transformed.append( + batch.append( { "type": "function_call", "call_id": tc["id"], @@ -2674,6 +2609,7 @@ def _transform_messages( "arguments": fn.get("arguments", ""), } ) + transformed.extend(demote_responses_batch(batch, state, reasoning)) continue # User/Assistant text messages → passthrough with cache_control preservation @@ -2684,7 +2620,10 @@ def _transform_messages( item = {"role": msg["role"], "content": content} if "cache_control" in msg: item["cache_control"] = msg["cache_control"] - transformed.append(item) + if (state is not None or reasoning is not None) and msg.get("role") == "assistant": + transformed.extend(demote_responses_batch([item], state, reasoning)) + else: + transformed.append(item) continue # Unknown format → passthrough diff --git a/tests/agents/test_method_summarizer_call_id.py b/tests/agents/test_method_summarizer_call_id.py index 64aeb53d5..14a7ce6cc 100644 --- a/tests/agents/test_method_summarizer_call_id.py +++ b/tests/agents/test_method_summarizer_call_id.py @@ -40,7 +40,6 @@ def _resp(content: str, tool_calls: list | None = None) -> LLMResponse: content=content, tool_calls=tool_calls or [], finish_reason=finish_reason, - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/agents/test_summarization_agents.py b/tests/agents/test_summarization_agents.py index b35c29b24..07211d5d4 100644 --- a/tests/agents/test_summarization_agents.py +++ b/tests/agents/test_summarization_agents.py @@ -27,7 +27,6 @@ def _resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/atif/test_custom_event_types.py b/tests/atif/test_custom_event_types.py index 75673313b..d41277aee 100644 --- a/tests/atif/test_custom_event_types.py +++ b/tests/atif/test_custom_event_types.py @@ -22,8 +22,7 @@ from nooa.events import ( AfterTurn, BeforeTurn, - LLMComplete, - LLMOutput, + LLMResponse, SystemPrompt, Task, ) @@ -148,7 +147,7 @@ def test_metadata_role_event_skipped(self, tmp_path: Path) -> None: def test_known_event_types_still_route_to_specific_handlers(self, tmp_path: Path) -> None: """The wildcard dispatcher must not duplicate or skip framework events. - Pin that Task, BeforeTurn, LLMComplete, AfterTurn still produce + Pin that Task, BeforeTurn, LLMResponse, AfterTurn still produce the same trajectory shape as before — specific handlers ran, no generic-event fallback triggered. """ @@ -164,14 +163,13 @@ def test_known_event_types_still_route_to_specific_handlers(self, tmp_path: Path ) ) exporter._dispatch_event( - LLMComplete( + LLMResponse( model_name="fake", - prompt_tokens=10, - completion_tokens=2, + usage={"prompt_tokens": 10, "completion_tokens": 2}, generation_id="gen-1", + content="answered", ) ) - exporter._dispatch_event(LLMOutput(content="answered")) exporter._dispatch_event( AfterTurn( method_name="run", @@ -188,7 +186,7 @@ def test_known_event_types_still_route_to_specific_handlers(self, tmp_path: Path sources = [s.source for s in traj.steps] assert sources == ["system", "user", "agent"] agent_step = traj.steps[2] - # Specific LLMComplete handler ran ⇒ metrics populated, llm_call_count=1. + # Specific LLMResponse handler ran ⇒ metrics populated, llm_call_count=1. assert agent_step.metrics is not None assert agent_step.metrics.prompt_tokens == 10 assert agent_step.llm_call_count == 1 @@ -234,7 +232,6 @@ def _resp(tool_calls: list[ToolCall] | None = None) -> LLMResponse: content="", tool_calls=tool_calls or [], finish_reason="tool_calls" if tool_calls else "stop", - assistant_message={"role": "assistant", "content": ""}, usage={"prompt_tokens": 5, "completion_tokens": 1}, ) diff --git a/tests/atif/test_enable_atif_isolation.py b/tests/atif/test_enable_atif_isolation.py index 4a0353b86..b064ec1f1 100644 --- a/tests/atif/test_enable_atif_isolation.py +++ b/tests/atif/test_enable_atif_isolation.py @@ -53,7 +53,6 @@ def _resp(tool_calls: list[ToolCall] | None = None, content: str = "") -> LLMRes content=content, tool_calls=tool_calls or [], finish_reason="tool_calls" if tool_calls else "stop", - assistant_message={"role": "assistant", "content": content}, usage={"prompt_tokens": 5, "completion_tokens": 1}, ) diff --git a/tests/atif/test_end_to_end_codeact.py b/tests/atif/test_end_to_end_codeact.py index 9ba967457..b2617e446 100644 --- a/tests/atif/test_end_to_end_codeact.py +++ b/tests/atif/test_end_to_end_codeact.py @@ -44,7 +44,6 @@ def _resp( content=content, tool_calls=tool_calls or [], finish_reason=finish_reason, - assistant_message={"role": "assistant", "content": content}, usage=usage or {"prompt_tokens": 50, "completion_tokens": 10}, ) @@ -198,7 +197,7 @@ class _Boom(RuntimeError): async def test_observation_paired_end_to_end(tmp_path: Path) -> None: """End-to-end pin: every tool_call has its matching observation result (the joinability invariant). The fc_*/call_* bridge is unnecessary; - LLMComplete + PythonOutput route everything by canonical call_* id. + LLMResponse + PythonOutput route everything by canonical call_* id. """ llm = FakeLLMClient( scripted_responses=[ @@ -275,7 +274,6 @@ async def test_canonical_call_id_used_when_both_ids_present(tmp_path: Path) -> N ) ], finish_reason="tool_calls", - assistant_message={"role": "assistant", "content": ""}, usage={"prompt_tokens": 5, "completion_tokens": 2}, ) llm = FakeLLMClient(scripted_responses=[response]) diff --git a/tests/atif/test_exporter_state_machine.py b/tests/atif/test_exporter_state_machine.py index 45b645077..d19807044 100644 --- a/tests/atif/test_exporter_state_machine.py +++ b/tests/atif/test_exporter_state_machine.py @@ -22,8 +22,7 @@ AfterTurn, BeforeTurn, Error, - LLMComplete, - LLMOutput, + LLMResponse, Notification, PythonOutput, Reasoning, @@ -31,6 +30,7 @@ SystemPrompt, Task, ) +from nooa.unifiedllm import ToolCall from tests.atif.normative import assert_atif_normative # Minimal system-prompt content used by synthetic tests. In a real run the @@ -83,7 +83,7 @@ def _drive_basic_codeact_turn( is_final_after: bool = True, fire_system_prompt: bool = True, ) -> None: - """Push a complete BeforeTurn → LLMComplete → ToolCallEvent → PythonOutput → AfterTurn sequence. + """Push a complete BeforeTurn → LLMResponse → ToolCallEvent → PythonOutput → AfterTurn sequence. By default also fires a SystemPrompt before BeforeTurn (matching the real runtime order: ``_build_messages → SystemPrompt → LLM call``). @@ -100,25 +100,26 @@ def _drive_basic_codeact_turn( turn_number=1, ) ) - exp.on_llm_complete( - LLMComplete( + exp.on_llm_response( + LLMResponse( model_name="fake-model", - prompt_tokens=100, - completion_tokens=20, - cached_tokens=10, - cost_usd=0.001, + usage={ + "prompt_tokens": 100, + "completion_tokens": 20, + "cached_tokens": 10, + "cost_usd": 0.001, + }, tool_calls=[ - { - "tool_call_id": "call_alpha", - "function_name": "execute_python", - "arguments": json.dumps({"code": code}), - } + ToolCall( + id="call_alpha", + name="execute_python", + arguments=json.dumps({"code": code}), + ) ], - reasoning_content="thinking...", + reasoning="thinking...", generation_id=generation_id, ) ) - exp.on_llm_output(LLMOutput(content="")) exp.on_tool_call_event( ToolCallEvent( tool_call_id="call_alpha", @@ -195,7 +196,7 @@ def test_writes_file_atomically(self, exporter: AtifExporter, tmp_path: Path) -> class TestJoinabilityByConstruction: def test_observation_paired_with_tool_call(self, exporter: AtifExporter) -> None: - """The fc_*/call_* bridge is unnecessary: both come from LLMComplete + PythonOutput.""" + """The fc_*/call_* bridge is unnecessary: both come from LLMResponse + PythonOutput.""" exporter.on_task(Task(prompt="run")) _drive_basic_codeact_turn(exporter) @@ -226,18 +227,20 @@ def test_return_result_observation_from_tool_call_event(self, exporter: AtifExpo turn_number=1, ) ) - exporter.on_llm_complete( - LLMComplete( + exporter.on_llm_response( + LLMResponse( model_name="fake-model", - prompt_tokens=50, - completion_tokens=5, - cost_usd=0.0001, + usage={ + "prompt_tokens": 50, + "completion_tokens": 5, + "cost_usd": 0.0001, + }, tool_calls=[ - { - "tool_call_id": "call_ret", - "function_name": "return_result", - "arguments": json.dumps({"result": 42}), - } + ToolCall( + id="call_ret", + name="return_result", + arguments=json.dumps({"result": 42}), + ) ], generation_id="gen-1", ) @@ -291,17 +294,16 @@ def test_tool_call_event_result_mutated_after_capture(self, exporter: AtifExport turn_number=1, ) ) - exporter.on_llm_complete( - LLMComplete( + exporter.on_llm_response( + LLMResponse( model_name="fake-model", - prompt_tokens=1, - completion_tokens=1, + usage={"prompt_tokens": 1, "completion_tokens": 1}, tool_calls=[ - { - "tool_call_id": "call_mut", - "function_name": "return_result", - "arguments": "{}", - } + ToolCall( + id="call_mut", + name="return_result", + arguments="{}", + ) ], generation_id="gen-1", ) @@ -505,16 +507,15 @@ def test_reasoning_event_attaches_to_pending_step(self, exporter: AtifExporter) ) # Reasoning fires inside the turn (mid execute_python). exporter.on_reasoning(Reasoning(content="Step A. ")) - exporter.on_llm_complete( - LLMComplete( + exporter.on_llm_response( + LLMResponse( model_name="fake-model", - prompt_tokens=1, - completion_tokens=1, - reasoning_content="Initial CoT.", + usage={"prompt_tokens": 1, "completion_tokens": 1}, + reasoning="Initial CoT.", generation_id="gen-1", ) ) - # Reasoning appended even though LLMComplete already set reasoning_content. + # Reasoning appended even though LLMResponse already set reasoning_content. exporter.on_reasoning(Reasoning(content="Step B.")) exporter.on_after_turn( AfterTurn( @@ -591,7 +592,7 @@ def test_system_prompt_drift_annotates_next_agent_step(self, exporter: AtifExpor exporter.on_task(Task(prompt="hi")) # New LLM call sees a different system prompt (e.g. a dynamic static # block mutated). The runtime fires SystemPrompt again with the new - # content right before LLMComplete. + # content right before LLMResponse. _seed_system_prompt(exporter, content="Drifted system prompt") _drive_basic_codeact_turn(exporter, fire_system_prompt=False) diff --git a/tests/atif/test_multi_agent_embedding.py b/tests/atif/test_multi_agent_embedding.py index 762b8754c..92d8f10cf 100644 --- a/tests/atif/test_multi_agent_embedding.py +++ b/tests/atif/test_multi_agent_embedding.py @@ -32,7 +32,6 @@ def _resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, usage={"prompt_tokens": 5, "completion_tokens": 1}, ) diff --git a/tests/atif/test_phase4_nesting.py b/tests/atif/test_phase4_nesting.py index 68957d073..686dbc658 100644 --- a/tests/atif/test_phase4_nesting.py +++ b/tests/atif/test_phase4_nesting.py @@ -21,8 +21,6 @@ from nooa.events import ( AfterTurn, BeforeTurn, - LLMComplete, - LLMOutput, SystemPrompt, Task, ) @@ -52,7 +50,6 @@ def _resp( content=content, tool_calls=tool_calls or [], finish_reason=finish_reason, - assistant_message={"role": "assistant", "content": content}, usage=usage or {"prompt_tokens": 50, "completion_tokens": 10}, ) @@ -304,15 +301,14 @@ def test_multimodal_task_image_rendered_as_content_parts(tmp_path: Path) -> None turn_number=1, ) ) - exporter.on_llm_complete( - LLMComplete( + exporter.on_llm_response( + LLMResponse( model_name="fake-model", - prompt_tokens=1, - completion_tokens=1, + usage={"prompt_tokens": 1, "completion_tokens": 1}, generation_id="gen-1", + content="A 1x1 transparent pixel.", ) ) - exporter.on_llm_output(LLMOutput(content="A 1x1 transparent pixel.")) exporter.on_after_turn( AfterTurn( method_name="run", diff --git a/tests/atif/test_standalone_entrypoint_cascade.py b/tests/atif/test_standalone_entrypoint_cascade.py index 7d87d3599..e418ca47d 100644 --- a/tests/atif/test_standalone_entrypoint_cascade.py +++ b/tests/atif/test_standalone_entrypoint_cascade.py @@ -55,7 +55,6 @@ def _resp(tool_calls: list[ToolCall] | None = None, content: str = "") -> LLMRes content=content, tool_calls=tool_calls or [], finish_reason="tool_calls" if tool_calls else "stop", - assistant_message={"role": "assistant", "content": content}, usage={"prompt_tokens": 5, "completion_tokens": 1}, ) diff --git a/tests/atif/test_structural_scenarios.py b/tests/atif/test_structural_scenarios.py index 239cf95fb..d31b11623 100644 --- a/tests/atif/test_structural_scenarios.py +++ b/tests/atif/test_structural_scenarios.py @@ -33,8 +33,6 @@ from nooa.events import ( AfterTurn, BeforeTurn, - LLMComplete, - LLMOutput, PythonOutput, Summary, SystemPrompt, @@ -65,7 +63,6 @@ def _resp( content=content, tool_calls=tool_calls or [], finish_reason="tool_calls" if tool_calls else "stop", - assistant_message={"role": "assistant", "content": content}, usage=usage or {"prompt_tokens": 50, "completion_tokens": 10}, ) @@ -467,12 +464,14 @@ def test_multimodal_input(tmp_path: Path) -> None: turn_number=1, ) ) - exporter.on_llm_complete( - LLMComplete( - model_name="fake-model", prompt_tokens=10, completion_tokens=4, generation_id="gen-1" + exporter.on_llm_response( + LLMResponse( + model_name="fake-model", + usage={"prompt_tokens": 10, "completion_tokens": 4}, + generation_id="gen-1", + content="A 1x1 transparent pixel.", ) ) - exporter.on_llm_output(LLMOutput(content="A 1x1 transparent pixel.")) exporter.on_after_turn( AfterTurn( method_name="run", @@ -516,23 +515,24 @@ def test_compaction_boundary(tmp_path: Path) -> None: turn_number=1, ) ) - exporter.on_llm_complete( - LLMComplete( + exporter.on_llm_response( + LLMResponse( model_name="fake-model", - prompt_tokens=100, - completion_tokens=10, - cost_usd=0.0001, + usage={ + "prompt_tokens": 100, + "completion_tokens": 10, + "cost_usd": 0.0001, + }, tool_calls=[ - { - "tool_call_id": "call_a", - "function_name": "execute_python", - "arguments": json.dumps({"code": "x = 1"}), - } + ToolCall( + id="call_a", + name="execute_python", + arguments=json.dumps({"code": "x = 1"}), + ) ], generation_id="gen-1", ) ) - exporter.on_llm_output(LLMOutput(content="")) exporter.on_python_output( PythonOutput( tool_call_id="call_a", @@ -570,23 +570,24 @@ def test_compaction_boundary(tmp_path: Path) -> None: turn_number=2, ) ) - exporter.on_llm_complete( - LLMComplete( + exporter.on_llm_response( + LLMResponse( model_name="fake-model", - prompt_tokens=50, - completion_tokens=5, - cost_usd=0.00005, + usage={ + "prompt_tokens": 50, + "completion_tokens": 5, + "cost_usd": 0.00005, + }, tool_calls=[ - { - "tool_call_id": "call_b", - "function_name": "return_result", - "arguments": json.dumps({"result": 1}), - } + ToolCall( + id="call_b", + name="return_result", + arguments=json.dumps({"result": 1}), + ) ], generation_id="gen-2", ) ) - exporter.on_llm_output(LLMOutput(content="")) exporter.on_after_turn( AfterTurn( method_name="run", diff --git a/tests/capability/test_class_method_replacement_bug.py b/tests/capability/test_class_method_replacement_bug.py index f12999581..cbd7309a9 100644 --- a/tests/capability/test_class_method_replacement_bug.py +++ b/tests/capability/test_class_method_replacement_bug.py @@ -34,7 +34,6 @@ def make_fake_llm() -> FakeLLMClient: content="test", tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": "test"}, ) ] ) diff --git a/tests/capability/test_router_repeated_runs.py b/tests/capability/test_router_repeated_runs.py index 997dad10a..b5034b30a 100644 --- a/tests/capability/test_router_repeated_runs.py +++ b/tests/capability/test_router_repeated_runs.py @@ -35,12 +35,12 @@ import asyncio import json import linecache -from dataclasses import dataclass from typing import TypedDict import pytest from nooa import Agent +from nooa.unifiedllm import LLMResponse, ToolCall class SimpleResult(TypedDict): @@ -63,20 +63,6 @@ async def compute(self, x: int, y: int) -> SimpleResult: ... -@dataclass -class FakeToolCall: - id: str - name: str - arguments: str - - -@dataclass -class FakeLLMResponse: - finish_reason: str - tool_calls: list[FakeToolCall] - content: str | None = None - - class FakeLLM: """Fake LLM that returns deterministic code.""" @@ -95,10 +81,10 @@ async def acall(self, messages, **kwargs): else: code = """return_result(computed=True, value=x + y)""" - return FakeLLMResponse( + return LLMResponse( finish_reason="tool_calls", tool_calls=[ - FakeToolCall( + ToolCall( id=f"call_{self.call_count}", name="execute_python", arguments=json.dumps({"code": code}), @@ -354,10 +340,10 @@ async def acall(self, messages, **kwargs): return_result(agents_called=["Validator"], results={"Validator": result}) """ - return FakeLLMResponse( + return LLMResponse( finish_reason="tool_calls", tool_calls=[ - FakeToolCall( + ToolCall( id=f"call_{self.call_count}", name="execute_python", arguments=json.dumps({"code": code}), diff --git a/tests/context_blocks/test_formatters.py b/tests/context_blocks/test_formatters.py index eccaf3222..ece137510 100644 --- a/tests/context_blocks/test_formatters.py +++ b/tests/context_blocks/test_formatters.py @@ -8,6 +8,7 @@ """ import pytest +from pydantic import ValidationError from nooa.context_blocks.events import ToolCallEvent, ToolResult from nooa.context_blocks.formatter import ( @@ -24,6 +25,8 @@ Role, ToolCallInfo, ) +from nooa.events import LLMResponse +from nooa.unifiedllm import ToolCall def _tool_call_block( @@ -33,7 +36,7 @@ def _tool_call_block( name: str, arguments: dict, result_content: str | None = None, - reasoning_items: list[dict] | None = None, + llm_response_id: str | None = None, ) -> ResolvedBlock: """Helper: ResolvedBlock carrying a ToolCallEvent.""" result = ( @@ -45,7 +48,7 @@ def _tool_call_block( tool_call_id=tool_call_id, name=name, arguments=arguments, - reasoning_items=reasoning_items, + llm_response_id=llm_response_id, result=result, ) return ResolvedBlock(key=key, content="", role=Role.ASSISTANT, event=event) @@ -136,6 +139,209 @@ def test_with_metadata_expr_renders_only_for_dynamic(self): def test_format_type(self): assert XMLBlockFormatter().format_type == "xml" + def test_groups_linked_executions_under_original_assistant_turn(self): + turn = LLMResponse( + content="I will run both.", + tool_calls=( + ToolCall( + id="call_1", + name="execute_python", + arguments='{"code":"first()"}', + ), + ToolCall( + id="call_2", + name="execute_python", + arguments='{"code":"second()"}', + ), + ), + finish_reason="tool_calls", + ) + call_1 = _tool_call_block( + key="call_1", + tool_call_id="call_1", + name="execute_python", + arguments={"code": "first()"}, + result_content="status: complete", + llm_response_id=turn.id, + ) + call_2 = _tool_call_block( + key="call_2", + tool_call_id="call_2", + name="execute_python", + arguments={"code": "second()"}, + result_content="status: complete", + llm_response_id=turn.id, + ) + + messages = XMLBlockFormatter().format( + [ + # Runtime event projection carries the object on an otherwise + # contentless block; assistant text comes from the canonical turn. + ResolvedBlock(key="turn", content="", role=Role.ASSISTANT, event=turn), + call_1, + ResolvedBlock(key="output_1", content="first output", role=Role.USER), + call_2, + ResolvedBlock(key="output_2", content="second output", role=Role.USER), + ] + ) + + assert [message.role for message in messages] == [ + Role.SYSTEM, + Role.ASSISTANT, + Role.TOOL, + Role.TOOL, + Role.USER, + Role.USER, + ] + assert [call.id for call in messages[1].tool_calls] == ["call_1", "call_2"] + assert [call.arguments for call in messages[1].tool_calls] == [ + '{"code":"first()"}', + '{"code":"second()"}', + ] + assert messages[1].content == "I will run both." + assert [message.tool_call_id for message in messages[2:4]] == ["call_1", "call_2"] + + def test_incomplete_linked_call_batch_is_omitted(self): + turn = LLMResponse( + content="", + tool_calls=( + ToolCall(id="call_1", name="one", arguments="{}"), + ToolCall(id="call_2", name="two", arguments="{}"), + ), + finish_reason="tool_calls", + ) + call_1 = _tool_call_block( + tool_call_id="call_1", + name="one", + arguments={}, + result_content="failed", + llm_response_id=turn.id, + ) + + messages = XMLBlockFormatter().format( + [ + ResolvedBlock(key="turn", content="", role=Role.ASSISTANT, event=turn), + call_1, + ] + ) + + assert [message.role for message in messages] == [Role.SYSTEM] + + def test_linked_execution_without_source_turn_is_not_rendered(self): + call = _tool_call_block( + tool_call_id="call_1", + name="one", + arguments={}, + result_content="complete", + llm_response_id="filtered-response", + ) + + messages = XMLBlockFormatter().format([call]) + + assert [message.role for message in messages] == [Role.SYSTEM] + + @pytest.mark.parametrize( + ("finish_reason", "arguments", "content"), + [ + ("length", '{"code":"partial()"}', ""), + ("tool_calls", '{"code":', "partial response"), + ("tool_calls", "[]", "non-object arguments"), + ], + ) + def test_incomplete_or_malformed_tool_batch_is_not_replayed( + self, finish_reason, arguments, content + ): + turn = LLMResponse( + content=content, + tool_calls=( + ToolCall( + id="partial", + name="execute_python", + arguments=arguments, + ), + ), + finish_reason=finish_reason, + ) + + messages = XMLBlockFormatter().format( + [ + ResolvedBlock( + key="turn", + content=content, + role=Role.ASSISTANT, + event=turn, + ) + ] + ) + + assert all(not message.tool_calls for message in messages) + assert AnthropicProviderFormatter().format(messages) == { + "system": "", + "messages": ([{"role": "assistant", "content": content}] if content else []), + } + + @pytest.mark.parametrize("field", ["reasoning", "llm_state"]) + def test_replay_only_response_creates_private_carrier(self, field): + value = "private thought" if field == "reasoning" else {"opaque": "state"} + response = LLMResponse(content="", **{field: value}) + messages = XMLBlockFormatter().format( + [ResolvedBlock(key="turn", content="", role=Role.ASSISTANT, event=response)] + ) + + carrier = next(message for message in messages if message.role is Role.ASSISTANT) + assert carrier.content is None + assert getattr(carrier, field) == value + + def test_event_type_spoof_does_not_impersonate_an_llm_response(self): + from nooa.events import Message + + event = Message(content="ordinary message", event_type="LLMResponse") + + assert XMLBlockFormatter().format_event(event) == "ordinary message" + + def test_llm_response_subclass_keeps_canonical_semantics(self): + class CustomLLMResponse(LLMResponse): + pass + + response = CustomLLMResponse(content="", llm_state={"opaque": "state"}) + + messages = XMLBlockFormatter().format( + [ResolvedBlock(key="turn", content="", role=Role.ASSISTANT, event=response)] + ) + + carrier = next(message for message in messages if message.role is Role.ASSISTANT) + assert carrier.content is None + assert carrier.llm_state == {"opaque": "state"} + + def test_linked_execution_is_omitted_when_carrier_is_rejected(self): + turn = LLMResponse( + content="", + tool_calls=( + ToolCall( + id="partial", + name="execute_python", + arguments='{"code":"partial()"}', + ), + ), + finish_reason="length", + ) + execution = _tool_call_block( + tool_call_id="partial", + name="execute_python", + arguments={"code": "completed()"}, + result_content="status: complete", + llm_response_id=turn.id, + ) + + messages = XMLBlockFormatter().format( + [ + ResolvedBlock(key="turn", content="", role=Role.ASSISTANT, event=turn), + execution, + ] + ) + + assert [message.role for message in messages] == [Role.SYSTEM] + class TestMarkdownBlockFormatter: def test_single_block(self): @@ -234,8 +440,8 @@ def test_tool_call_message(self): RenderedMessage(role=Role.SYSTEM, content="System"), RenderedMessage( role=Role.ASSISTANT, - tool_call=ToolCallInfo( - id="call_abc", name="get_weather", arguments={"location": "SF"} + tool_calls=( + ToolCallInfo(id="call_abc", name="get_weather", arguments={"location": "SF"}), ), ), ] @@ -251,8 +457,8 @@ def test_tool_call_with_result(self): RenderedMessage(role=Role.SYSTEM, content="System"), RenderedMessage( role=Role.ASSISTANT, - tool_call=ToolCallInfo( - id="call_abc", name="get_weather", arguments={"location": "SF"} + tool_calls=( + ToolCallInfo(id="call_abc", name="get_weather", arguments={"location": "SF"}), ), ), RenderedMessage(role=Role.TOOL, content="Sunny", tool_call_id="call_abc"), @@ -262,6 +468,31 @@ def test_tool_call_with_result(self): assert result[1]["role"] == "assistant" and "tool_calls" in result[1] assert result[2] == {"role": "tool", "tool_call_id": "call_abc", "content": "Sunny"} + def test_tool_call_batch_preserves_order_and_raw_arguments(self): + messages = [ + RenderedMessage( + role=Role.ASSISTANT, + content="Calling both", + tool_calls=( + ToolCallInfo(id="a", name="one", arguments='{"x":1}'), + ToolCallInfo(id="b", name="two", arguments='{"y":2}'), + ), + ) + ] + + result = OpenAIProviderFormatter().format(messages) + + assert result[0]["content"] == "Calling both" + assert [call["id"] for call in result[0]["tool_calls"]] == ["a", "b"] + assert result[0]["tool_calls"][0]["function"]["arguments"] == '{"x":1}' + + def test_removed_singular_tool_call_fails_loudly(self): + with pytest.raises(ValidationError, match="tool_call"): + RenderedMessage( + role=Role.ASSISTANT, + tool_call=ToolCallInfo(id="old", name="old", arguments={}), + ) + def test_runtime_event_skipped(self): messages = [ RenderedMessage(role=Role.USER, content="Hello"), @@ -308,7 +539,7 @@ def test_tool_call_message(self): RenderedMessage(role=Role.SYSTEM, content="System"), RenderedMessage( role=Role.ASSISTANT, - tool_call=ToolCallInfo(id="tc_1", name="search", arguments={"q": "test"}), + tool_calls=(ToolCallInfo(id="tc_1", name="search", arguments={"q": "test"}),), ), ] result = AnthropicProviderFormatter().format(messages) @@ -321,7 +552,7 @@ def test_tool_call_with_result(self): RenderedMessage(role=Role.SYSTEM, content="System"), RenderedMessage( role=Role.ASSISTANT, - tool_call=ToolCallInfo(id="tc_1", name="search", arguments={"q": "test"}), + tool_calls=(ToolCallInfo(id="tc_1", name="search", arguments={"q": "test"}),), ), RenderedMessage(role=Role.TOOL, content="Result", tool_call_id="tc_1"), ] @@ -375,32 +606,39 @@ def test_markdown_with_anthropic(self): assert "# Persona" in result["system"] assert result["messages"][0]["content"] == "Hello" - def test_reasoning_items_survive_tool_call_pipeline(self): + def test_legacy_reasoning_items_fail_closed_without_provider_gate(self): + """Removed opaque legacy fields are ignored and cannot be emitted.""" reasoning_item = { "id": "rs_123", "type": "reasoning", "encrypted_content": "encrypted-state", "summary": [], } - blocks = [ - _tool_call_block( - tool_call_id="call_123", - name="search", - arguments={"query": "weather"}, - result_content="sunny", - reasoning_items=[reasoning_item], - ) - ] + legacy_event = ToolCallEvent.model_validate( + { + "tool_call_id": "call_123", + "name": "search", + "arguments": {"query": "weather"}, + "result": {"tool_call_id": "call_123", "content": "sunny"}, + "reasoning_items": [reasoning_item], + } + ) + assert "reasoning_items" not in type(legacy_event).model_fields + blocks = [ResolvedBlock(key="tc", content="", role=Role.ASSISTANT, event=legacy_event)] messages = XMLBlockFormatter().format(blocks) openai_input = OpenAIProviderFormatter().format(messages) responses_input = ResponsesProviderFormatter().format(messages) openai_tool_call = next(message for message in openai_input if "tool_calls" in message) - assert openai_tool_call["reasoning_items"] == [reasoning_item] - reasoning_index = responses_input.index(reasoning_item) - assert responses_input[reasoning_index + 1]["type"] == "function_call" - assert responses_input[reasoning_index + 2]["type"] == "function_call_output" + assert "reasoning_items" not in openai_tool_call + assert reasoning_item not in responses_input + function_call_index = next( + index + for index, item in enumerate(responses_input) + if item.get("type") == "function_call" + ) + assert responses_input[function_call_index + 1]["type"] == "function_call_output" class TestBlockFormatterFormatEvent: diff --git a/tests/core_runtime/test_code_caching.py b/tests/core_runtime/test_code_caching.py index c0aadf64b..4b7952873 100644 --- a/tests/core_runtime/test_code_caching.py +++ b/tests/core_runtime/test_code_caching.py @@ -21,7 +21,6 @@ def _resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/core_runtime/test_execution.py b/tests/core_runtime/test_execution.py index a7182e818..105e47075 100644 --- a/tests/core_runtime/test_execution.py +++ b/tests/core_runtime/test_execution.py @@ -24,7 +24,6 @@ def _resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/core_runtime/test_implemented_plan.py b/tests/core_runtime/test_implemented_plan.py index 5d3d773ca..f1f250446 100644 --- a/tests/core_runtime/test_implemented_plan.py +++ b/tests/core_runtime/test_implemented_plan.py @@ -23,7 +23,6 @@ def _resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/core_runtime/test_llm_client_reuse.py b/tests/core_runtime/test_llm_client_reuse.py index a6bb40145..cb0e7a8ed 100644 --- a/tests/core_runtime/test_llm_client_reuse.py +++ b/tests/core_runtime/test_llm_client_reuse.py @@ -16,7 +16,6 @@ def _resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/core_runtime/test_task_queuing_edge_cases.py b/tests/core_runtime/test_task_queuing_edge_cases.py index 4603db461..b4908eeb6 100644 --- a/tests/core_runtime/test_task_queuing_edge_cases.py +++ b/tests/core_runtime/test_task_queuing_edge_cases.py @@ -25,7 +25,6 @@ def _resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/edge_cases/test_builtin_shadowing.py b/tests/edge_cases/test_builtin_shadowing.py index 4d9df5afb..09d7a3dda 100644 --- a/tests/edge_cases/test_builtin_shadowing.py +++ b/tests/edge_cases/test_builtin_shadowing.py @@ -27,7 +27,6 @@ def _resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/edge_cases/test_child_agent_edge_cases.py b/tests/edge_cases/test_child_agent_edge_cases.py index 402769eb3..6b6b42651 100644 --- a/tests/edge_cases/test_child_agent_edge_cases.py +++ b/tests/edge_cases/test_child_agent_edge_cases.py @@ -24,7 +24,6 @@ def _resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/edge_cases/test_generation_lock_edge_cases.py b/tests/edge_cases/test_generation_lock_edge_cases.py index 097ec87cd..c51ae1576 100644 --- a/tests/edge_cases/test_generation_lock_edge_cases.py +++ b/tests/edge_cases/test_generation_lock_edge_cases.py @@ -25,7 +25,6 @@ def _resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/edge_cases/test_missing_await.py b/tests/edge_cases/test_missing_await.py index 355d18227..2e8b61402 100644 --- a/tests/edge_cases/test_missing_await.py +++ b/tests/edge_cases/test_missing_await.py @@ -23,7 +23,6 @@ def _resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/edge_cases/test_nested_generation_edge_cases.py b/tests/edge_cases/test_nested_generation_edge_cases.py index 27edd8eca..690b89fa1 100644 --- a/tests/edge_cases/test_nested_generation_edge_cases.py +++ b/tests/edge_cases/test_nested_generation_edge_cases.py @@ -24,7 +24,6 @@ def _resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/edge_cases/test_sandbox_edge_cases.py b/tests/edge_cases/test_sandbox_edge_cases.py index 61ea76e30..8be5b490c 100644 --- a/tests/edge_cases/test_sandbox_edge_cases.py +++ b/tests/edge_cases/test_sandbox_edge_cases.py @@ -32,7 +32,6 @@ def _resp(content: str = "", tool_calls: list | None = None) -> LLMResponse: content=content, tool_calls=tool_calls or [], finish_reason="tool_calls" if tool_calls else "stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/edge_cases/test_signal_edge_cases.py b/tests/edge_cases/test_signal_edge_cases.py index c02ee3b29..698e15f5a 100644 --- a/tests/edge_cases/test_signal_edge_cases.py +++ b/tests/edge_cases/test_signal_edge_cases.py @@ -31,7 +31,6 @@ def _resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/external/test_notebook_scenarios.py b/tests/external/test_notebook_scenarios.py index 66ae3bb63..9016f092c 100644 --- a/tests/external/test_notebook_scenarios.py +++ b/tests/external/test_notebook_scenarios.py @@ -22,7 +22,6 @@ def _resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/integration/test_codeact_dynamic_structured_output.py b/tests/integration/test_codeact_dynamic_structured_output.py index d41009817..fb87cb3cf 100644 --- a/tests/integration/test_codeact_dynamic_structured_output.py +++ b/tests/integration/test_codeact_dynamic_structured_output.py @@ -42,7 +42,6 @@ def _resp(content: str = "", tool_calls: list | None = None) -> LLMResponse: content=content, tool_calls=tool_calls or [], finish_reason=finish_reason, - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/integration/test_codeact_nested_structured_output.py b/tests/integration/test_codeact_nested_structured_output.py index d41401f6f..e431df021 100644 --- a/tests/integration/test_codeact_nested_structured_output.py +++ b/tests/integration/test_codeact_nested_structured_output.py @@ -55,7 +55,6 @@ def _resp(content: str = "", tool_calls: list | None = None) -> LLMResponse: content=content, tool_calls=tool_calls or [], finish_reason=finish_reason, - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/integration/test_concurrent_traces.py b/tests/integration/test_concurrent_traces.py index 74bb9a4dd..5fb68dd85 100644 --- a/tests/integration/test_concurrent_traces.py +++ b/tests/integration/test_concurrent_traces.py @@ -62,7 +62,6 @@ async def acall(self, messages, tools=None, **kwargs): ) ], finish_reason="tool_calls", - assistant_message={}, ) diff --git a/tests/integration/test_nested_agent_history_bug.py b/tests/integration/test_nested_agent_history_bug.py index 8565299d8..51484b968 100644 --- a/tests/integration/test_nested_agent_history_bug.py +++ b/tests/integration/test_nested_agent_history_bug.py @@ -12,6 +12,8 @@ """ import json +from copy import deepcopy +from typing import Any import pytest @@ -29,7 +31,6 @@ def _resp(content: str = "", tool_calls: list | None = None) -> LLMResponse: content=content, tool_calls=tool_calls or [], finish_reason=finish_reason, - assistant_message={"role": "assistant", "content": content}, ) @@ -55,6 +56,18 @@ def _return_result(result, call_id: str = "call_return") -> ToolCall: _TEST_LLM = FakeLLMClient() +class _RecordingFakeLLM(FakeLLMClient): + """Fake client that retains each rendered provider prompt.""" + + def __init__(self, scripted_responses: list[LLMResponse]): + super().__init__(scripted_responses) + self.message_history: list[list[dict[str, Any]]] = [] + + async def acall(self, messages, tools=None, output_model=None, **kwargs): + self.message_history.append(deepcopy(messages)) + return await super().acall(messages, tools, output_model, **kwargs) + + class TestNestedAgentHistoryBug: """Tests demonstrating the nested agent history ordering bug.""" @@ -100,7 +113,7 @@ async def inner_method(self) -> str: # 1. outer_method turn 1: execute_python calling inner_method, then print result # 2. inner_method turn 1: return_result with "inner_done" # 3. outer_method turn 2: return_result with combined result (SHOULD FAIL due to bug) - fake_llm = FakeLLMClient( + fake_llm = _RecordingFakeLLM( scripted_responses=[ # outer_method turn 1: call inner_method and capture result _resp( @@ -137,6 +150,39 @@ async def inner_method(self) -> str: assert result == "outer_with_inner_done" + # The inner generation sees the outer execute_python call while that + # cell is still active. The next outer generation must retain that + # exact provider-visible prefix instead of rewriting its tool result + # from "executing" to "complete" and invalidating the prompt cache. + inner_prompt = fake_llm.message_history[1] + outer_followup_prompt = fake_llm.message_history[2] + + def from_outer_call(messages): + index = next( + i + for i, message in enumerate(messages) + if message.get("role") == "assistant" + and any( + call.get("id") == "call_outer_exec_1" for call in message.get("tool_calls", []) + ) + ) + return messages[index:] + + inner_prefix = from_outer_call(inner_prompt) + outer_suffix = from_outer_call(outer_followup_prompt) + # Dynamic context is deliberately a recomputed trailing suffix, so + # compare only the stable history before it. + if inner_prefix[-1].get("content", "").startswith(""): + inner_prefix = inner_prefix[:-1] + assert outer_suffix[: len(inner_prefix)] == inner_prefix + + receipt = next( + message + for message in inner_prefix + if message.get("tool_call_id") == "call_outer_exec_1" + ) + assert receipt["content"] == "status: accepted" + @pytest.mark.asyncio async def test_single_method_no_nesting_works(self): """Baseline: single method without nesting should work fine.""" diff --git a/tests/integration/test_nested_generation.py b/tests/integration/test_nested_generation.py index 8318567cf..8ce3c3a38 100644 --- a/tests/integration/test_nested_generation.py +++ b/tests/integration/test_nested_generation.py @@ -19,7 +19,6 @@ def _resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/performance/test_client_creation_overhead.py b/tests/performance/test_client_creation_overhead.py index 45af66f5d..3c587d6fe 100644 --- a/tests/performance/test_client_creation_overhead.py +++ b/tests/performance/test_client_creation_overhead.py @@ -21,7 +21,6 @@ def _resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/runtime/sandbox/test_codeact_sandbox.py b/tests/runtime/sandbox/test_codeact_sandbox.py index 2c5643dd5..80a3bf307 100644 --- a/tests/runtime/sandbox/test_codeact_sandbox.py +++ b/tests/runtime/sandbox/test_codeact_sandbox.py @@ -34,7 +34,6 @@ def _resp(content: str, tool_calls: list | None = None) -> LLMResponse: content=content, tool_calls=tool_calls or [], finish_reason="tool_calls" if tool_calls else "stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/runtime/test_agent_call_events.py b/tests/runtime/test_agent_call_events.py index 483996e06..794e20536 100644 --- a/tests/runtime/test_agent_call_events.py +++ b/tests/runtime/test_agent_call_events.py @@ -33,7 +33,6 @@ def _predict_resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, usage={"prompt_tokens": 5, "completion_tokens": 1}, ) diff --git a/tests/runtime/test_codeexec_method_parenting.py b/tests/runtime/test_codeexec_method_parenting.py index 3b78eed8a..41e002c05 100644 --- a/tests/runtime/test_codeexec_method_parenting.py +++ b/tests/runtime/test_codeexec_method_parenting.py @@ -39,7 +39,6 @@ async def acall(self, messages, tools=None, **kwargs): ), ], finish_reason="tool_calls", - assistant_message={}, ) @@ -78,7 +77,6 @@ async def acall(self, messages, tools=None, **kwargs): ), ], finish_reason="tool_calls", - assistant_message={}, ) diff --git a/tests/runtime/test_context_builder.py b/tests/runtime/test_context_builder.py index 958838994..768600ab3 100644 --- a/tests/runtime/test_context_builder.py +++ b/tests/runtime/test_context_builder.py @@ -557,12 +557,12 @@ def test_non_tool_event_carries_original_event(self): assert result[0].event is event assert result[0].content == "" # Deferred — serialized at render time - def test_empty_llm_output_is_persisted_but_not_provider_visible(self): - from nooa.events import LLMOutput + def test_empty_llm_response_is_persisted_but_not_provider_visible(self): + from nooa.events import LLMResponse from nooa.runtime.context_builder import _phase_events - empty = LLMOutput(content="", tag="1") - visible = LLMOutput(content="answer", tag="2") + empty = LLMResponse(content="", tag="1") + visible = LLMResponse(content="answer", tag="2") events = [empty, visible] em = _make_event_manager(events) @@ -571,6 +571,38 @@ def test_empty_llm_output_is_persisted_but_not_provider_visible(self): assert [block.event for block in result] == [visible] assert em.values() == events + def test_tool_turn_and_linked_executions_are_public_context_ir(self): + from nooa.events import LLMResponse + from nooa.runtime.context_builder import _phase_events + from nooa.unifiedllm import ToolCall + + turn = LLMResponse( + content="", + tag="1", + tool_calls=( + ToolCall( + id="call-1", + name="execute_python", + arguments='{"code":"print(1)"}', + ), + ), + finish_reason="tool_calls", + ) + call = ToolCallEvent( + tool_call_id="call-1", + name="execute_python", + arguments={"code": "print(1)"}, + llm_response_id=turn.id, + result=ToolResult(tool_call_id="call-1", content="status: complete"), + tag="2", + ) + em = _make_event_manager([turn, call]) + + result = _phase_events([], em) + + assert [block.event for block in result] == [turn, call] + assert em.values() == [turn, call] + def test_current_call_query_keeps_task_event(self): """EventQuery.current_call() must keep the task so LLM gets system + task. diff --git a/tests/runtime/test_context_integration.py b/tests/runtime/test_context_integration.py index a813c953d..1eaad5c34 100644 --- a/tests/runtime/test_context_integration.py +++ b/tests/runtime/test_context_integration.py @@ -586,7 +586,6 @@ async def test_llm_receives_task_prompt_with_scoped_current_call(self): ) ], finish_reason="tool_calls", - assistant_message={"role": "assistant", "content": ""}, ) ] ) diff --git a/tests/runtime/test_event_manager_events.py b/tests/runtime/test_event_manager_events.py index 7fd8c3826..b24e728a3 100644 --- a/tests/runtime/test_event_manager_events.py +++ b/tests/runtime/test_event_manager_events.py @@ -4,9 +4,11 @@ from unittest.mock import MagicMock +import pytest + from nooa import Agent from nooa.context_blocks import ResultStatus -from nooa.events import LLMOutput, PythonOutput, Task +from nooa.events import LLMResponse, PythonOutput, Task from nooa.runtime.event_manager import EventManager from nooa.runtime.events import EventsApi from nooa.unifiedllm import FakeLLMClient @@ -108,6 +110,29 @@ def test_add_with_metadata(self): class TestEventManagerOn: """Tests for EventManager.on() method.""" + @pytest.mark.parametrize("event_type", ["LLMOutput", "LLMComplete"]) + def test_removed_llm_event_subscription_explains_migration(self, event_type): + """Executable consumers fail loudly; stored legacy rows still migrate.""" + manager = EventManager() + + with pytest.raises(ValueError) as exc_info: + manager.on(event_type, lambda _event: None) + + message = str(exc_info.value) + assert f"removed event type {event_type!r}" in message + assert "Use 'LLMResponse' instead" in message + if event_type == "LLMOutput": + assert "Stored LLMOutput rows are migrated" in message + else: + assert "non-persisted runtime event" in message + assert event_type not in manager._handlers + + def test_removed_llm_output_query_fails_instead_of_returning_empty(self): + manager = EventManager() + + with pytest.raises(ValueError, match="Use 'LLMResponse' instead"): + manager.filter(type="LLMOutput") + def test_on_registers_handler(self): """on() should register handler for event type.""" manager = EventManager() @@ -135,17 +160,17 @@ def test_on_different_event_types(self): """on() should dispatch to correct handlers based on event_type.""" manager = EventManager() task_handler = MagicMock() - llm_output_handler = MagicMock() + llm_response_handler = MagicMock() # Register handlers by event_type manager.on("Task", task_handler) - manager.on("LLMOutput", llm_output_handler) + manager.on("LLMResponse", llm_response_handler) manager.add(Task(prompt="Question")) - manager.add(LLMOutput(content="Answer")) # Uses LLMOutput via alias + manager.add(LLMResponse(content="Answer")) # Uses LLMResponse via alias task_handler.assert_called_once() - llm_output_handler.assert_called_once() + llm_response_handler.assert_called_once() def test_on_returns_unsubscribe_function(self): """on() should return function to unsubscribe.""" @@ -166,7 +191,7 @@ def test_on_wildcard_receives_all_events(self): manager.on("*", handler) manager.add(Task(prompt="Task")) - manager.add(LLMOutput(content="Response")) + manager.add(LLMResponse(content="Response")) assert handler.call_count == 2 @@ -270,7 +295,7 @@ def test_filter_by_call_returns_events_with_call_id(self): event1.metadata["call_id"] = "call_1" manager.add(event1) - event2 = LLMOutput(content="Response 1") + event2 = LLMResponse(content="Response 1") event2.metadata["call_id"] = "call_1" manager.add(event2) @@ -418,7 +443,7 @@ def test_format_events_basic(self): """format_events() converts events to OpenAI format.""" manager = EventManager() manager.add(Task(prompt="Hello")) - manager.add(LLMOutput(content="Hi there")) + manager.add(LLMResponse(content="Hi there")) messages = _format_events_for_test(manager.values()) assert len(messages) == 2 diff --git a/tests/runtime/test_event_query.py b/tests/runtime/test_event_query.py index 129e50601..37f36fd3b 100644 --- a/tests/runtime/test_event_query.py +++ b/tests/runtime/test_event_query.py @@ -2,6 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 """Tests for EventQuery - event filtering configuration.""" +import pytest + from nooa.events import Task from nooa.runtime.event_query import EventQuery @@ -76,3 +78,8 @@ def test_current_call_keeps_task_message(self): assert len(result) >= 1, "must keep at least the task for this call" task_prompts = [e.prompt for e in result if isinstance(e, Task)] assert "Classify the sentiment" in task_prompts[0] + + +def test_removed_llm_output_type_explains_replacement(): + with pytest.raises(ValueError, match="Use 'LLMResponse' instead"): + EventQuery(type="LLMOutput") diff --git a/tests/runtime/test_journal_payload.py b/tests/runtime/test_journal_payload.py index 67e82fe73..007cf9109 100644 --- a/tests/runtime/test_journal_payload.py +++ b/tests/runtime/test_journal_payload.py @@ -35,7 +35,7 @@ def _text_msg(role: Role, content: str) -> RenderedMessage: def _assistant_tool_call(call_id: str, name: str, arguments: dict) -> RenderedMessage: return RenderedMessage( role=Role.ASSISTANT, - tool_call=ToolCallInfo(id=call_id, name=name, arguments=arguments), + tool_calls=(ToolCallInfo(id=call_id, name=name, arguments=arguments),), ) diff --git a/tests/runtime/test_llm_complete_event.py b/tests/runtime/test_llm_response_event.py similarity index 71% rename from tests/runtime/test_llm_complete_event.py rename to tests/runtime/test_llm_response_event.py index 47ccc132d..50851ee65 100644 --- a/tests/runtime/test_llm_complete_event.py +++ b/tests/runtime/test_llm_response_event.py @@ -1,8 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""LLMComplete event lifecycle. +"""LLMResponse event lifecycle. -Pins that ``runtime.generate()`` emits exactly one ``LLMComplete`` event +Pins that ``runtime.generate()`` emits exactly one ``LLMResponse`` event per LLM round-trip, with the correct payload assembled from ``LLMResponse.usage`` / ``.tool_calls`` / ``.reasoning`` and the surrounding generation_id context. @@ -16,7 +16,7 @@ import pytest from nooa import strategy -from nooa.events import LLMComplete +from nooa.runtime.event_manager import EventManager from nooa.strategies import PredictStrategy from nooa.unifiedllm import FakeLLMClient, LLMResponse, ToolCall @@ -34,30 +34,65 @@ def _resp( content=content, tool_calls=tool_calls or [], finish_reason=finish_reason, - assistant_message={"role": "assistant", "content": content}, usage=usage, reasoning=reasoning, ) -class TestLLMCompleteEvent: - """LLMComplete fires once per runtime.generate() with the right payload.""" +def test_durable_response_excludes_ephemeral_views() -> None: + """Persistence keeps replay/telemetry data, not provider or parsed objects.""" + response = LLMResponse( + raw_response=object(), + content='{"value":42}', + parsed={"value": 42}, + reasoning="plain reasoning", + llm_state={"opaque": "provider-state"}, + usage={"input_tokens": 12, "cached_input_tokens": 8}, + ) + + durable = response.model_dump() + + assert "raw_response" not in durable + assert "parsed" not in durable + assert "assistant_message" not in type(response).model_fields + assert durable["content"] == '{"value":42}' + assert durable["reasoning"] == "plain reasoning" + assert durable["llm_state"] == {"opaque": "provider-state"} + assert durable["usage"]["cached_input_tokens"] == 8 + + +def test_none_content_normalizes_to_empty_public_text() -> None: + """Provider null content never becomes a synthetic ``"None"`` message.""" + assert LLMResponse(content=None).content == "" + + +def test_opaque_state_is_not_searchable() -> None: + manager = EventManager() + response = LLMResponse( + content="public answer", + llm_state={"encrypted_content": "provider-secret"}, + ) + manager.add(response) + + assert manager.filter(query="public answer") == [response] + assert manager.filter(query="provider-secret") == [] + + +class TestLLMResponseEvent: + """LLMResponse fires once per runtime.generate() with the right payload.""" @pytest.mark.asyncio async def test_fires_exactly_once_per_generate(self) -> None: - """Single LLM round-trip emits exactly one LLMComplete event.""" - recorded: list[LLMComplete] = [] + """Runtime records the exact response object returned by UnifiedLLM once.""" + recorded: list[LLMResponse] = [] + response = _resp( + content='{"answer":"hi"}', + usage={"prompt_tokens": 42, "completion_tokens": 7}, + ) @strategy( PredictStrategy(), - llm=FakeLLMClient( - scripted_responses=[ - _resp( - content='{"answer":"hi"}', - usage={"prompt_tokens": 42, "completion_tokens": 7}, - ) - ] - ), + llm=FakeLLMClient(scripted_responses=[response]), ) async def predict_fn(prompt: str) -> dict: """{prompt}""" @@ -70,7 +105,7 @@ async def predict_fn(prompt: str) -> dict: class _Capture: def _attach_child(self, em, child_agent_name: str = "") -> None: - em.on("LLMComplete", lambda e: recorded.append(e)) + em.on("LLMResponse", lambda e: recorded.append(e)) def _detach_child(self, em) -> None: pass @@ -81,12 +116,13 @@ def _detach_child(self, em) -> None: finally: _atif_exporter_var.reset(token) - assert len(recorded) == 1, f"expected exactly 1 LLMComplete, got {len(recorded)}" + assert len(recorded) == 1, f"expected exactly 1 LLMResponse, got {len(recorded)}" + assert recorded[0] is response @pytest.mark.asyncio async def test_payload_carries_model_tokens_cost_generation_id(self) -> None: - """LLMComplete payload reflects LLMResponse.usage and llm_client.model.""" - recorded: list[LLMComplete] = [] + """LLMResponse payload reflects LLMResponse.usage and llm_client.model.""" + recorded: list[LLMResponse] = [] @strategy( PredictStrategy(), @@ -113,7 +149,7 @@ async def predict_fn(prompt: str) -> dict: class _Capture: def _attach_child(self, em, child_agent_name: str = "") -> None: - em.on("LLMComplete", lambda e: recorded.append(e)) + em.on("LLMResponse", lambda e: recorded.append(e)) def _detach_child(self, em) -> None: pass @@ -127,11 +163,12 @@ def _detach_child(self, em) -> None: assert len(recorded) == 1 ev = recorded[0] assert ev.model_name == "fake-model" - assert ev.prompt_tokens == 100 - assert ev.completion_tokens == 20 - assert ev.cached_tokens == 30 - assert ev.cost_usd == pytest.approx(0.0042) - assert ev.reasoning_content == "I think..." + assert ev.usage is not None + assert ev.usage.input_tokens == 100 + assert ev.usage.output_tokens == 20 + assert ev.usage.cached_input_tokens == 30 + assert ev.usage.cost_usd == pytest.approx(0.0042) + assert ev.reasoning == "I think..." assert ev.tool_calls == [] # generation_id is non-empty (set by the strategy that wrapped this turn). assert ev.generation_id != "" @@ -139,7 +176,7 @@ def _detach_child(self, em) -> None: @pytest.mark.asyncio async def test_carries_structured_tool_calls(self) -> None: """tool_calls list mirrors LLMResponse.tool_calls (canonical ids).""" - recorded: list[LLMComplete] = [] + recorded: list[LLMResponse] = [] tcs = [ ToolCall( id="call_alpha", name="execute_python", arguments=json.dumps({"code": "print(1)"}) @@ -167,7 +204,7 @@ async def predict_fn() -> dict: class _Capture: def _attach_child(self, em, child_agent_name: str = "") -> None: - em.on("LLMComplete", lambda e: recorded.append(e)) + em.on("LLMResponse", lambda e: recorded.append(e)) def _detach_child(self, em) -> None: pass @@ -178,21 +215,21 @@ def _detach_child(self, em) -> None: await predict_fn() except Exception: # PredictStrategy may surface a validation error on tool_calls; - # we only care that LLMComplete fired before that. + # we only care that LLMResponse fired before that. pass finally: _atif_exporter_var.reset(token) assert len(recorded) >= 1 ev = recorded[0] - assert [tc["tool_call_id"] for tc in ev.tool_calls] == ["call_alpha", "call_beta"] - assert ev.tool_calls[0]["function_name"] == "execute_python" - assert json.loads(ev.tool_calls[0]["arguments"]) == {"code": "print(1)"} + assert [tc.id for tc in ev.tool_calls] == ["call_alpha", "call_beta"] + assert ev.tool_calls[0].name == "execute_python" + assert json.loads(ev.tool_calls[0].arguments) == {"code": "print(1)"} @pytest.mark.asyncio async def test_zero_usage_renders_zero_tokens(self) -> None: - """When the provider returns no usage block, LLMComplete carries zeros.""" - recorded: list[LLMComplete] = [] + """When the provider returns no usage block, LLMResponse carries zeros.""" + recorded: list[LLMResponse] = [] @strategy( PredictStrategy(), @@ -206,7 +243,7 @@ async def predict_fn() -> dict: class _Capture: def _attach_child(self, em, child_agent_name: str = "") -> None: - em.on("LLMComplete", lambda e: recorded.append(e)) + em.on("LLMResponse", lambda e: recorded.append(e)) def _detach_child(self, em) -> None: pass @@ -219,10 +256,7 @@ def _detach_child(self, em) -> None: assert len(recorded) == 1 ev = recorded[0] - assert ev.prompt_tokens == 0 - assert ev.completion_tokens == 0 - assert ev.cached_tokens == 0 - assert ev.cost_usd == 0.0 + assert ev.usage is None class TestAtifExporterContextVar: @@ -244,7 +278,7 @@ async def test_set_value_visible_to_standalone_wrapper(self) -> None: class _Capture: def _attach_child(self, em, child_agent_name: str = "") -> None: seen_event_managers.append(em) - em.on("LLMComplete", lambda e: None) + em.on("LLMResponse", lambda e: None) def _detach_child(self, em) -> None: pass diff --git a/tests/runtime/test_llm_routing_observability.py b/tests/runtime/test_llm_routing_observability.py index 918658654..667081f93 100644 --- a/tests/runtime/test_llm_routing_observability.py +++ b/tests/runtime/test_llm_routing_observability.py @@ -21,7 +21,6 @@ def _resp(value: str) -> LLMResponse: content=f'{{"value": "{value}"}}', tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": f'{{"value": "{value}"}}'}, ) diff --git a/tests/runtime/test_pure_python_executor.py b/tests/runtime/test_pure_python_executor.py index 649008400..3a656da07 100644 --- a/tests/runtime/test_pure_python_executor.py +++ b/tests/runtime/test_pure_python_executor.py @@ -17,7 +17,6 @@ def _resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/runtime/test_span_parent_relationship.py b/tests/runtime/test_span_parent_relationship.py index a9f9a200f..ae07add38 100644 --- a/tests/runtime/test_span_parent_relationship.py +++ b/tests/runtime/test_span_parent_relationship.py @@ -25,7 +25,6 @@ def _resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/runtime/test_structured_output_executor.py b/tests/runtime/test_structured_output_executor.py index bb41f2c26..3d87d1fa8 100644 --- a/tests/runtime/test_structured_output_executor.py +++ b/tests/runtime/test_structured_output_executor.py @@ -183,7 +183,6 @@ class Person(BaseModel): content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, reasoning=None, usage=None, ), @@ -217,7 +216,6 @@ async def test_basic_type_generation(self): content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, reasoning=None, usage=None, ), @@ -256,7 +254,6 @@ class Person(BaseModel): content=content1, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content1}, reasoning=None, usage=None, ), @@ -265,7 +262,6 @@ class Person(BaseModel): content=content2, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content2}, reasoning=None, usage=None, ), @@ -303,7 +299,6 @@ class Person(BaseModel): content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, reasoning=None, usage=None, ) @@ -341,7 +336,6 @@ class Greeting(BaseModel): content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, reasoning=None, usage=None, ), @@ -397,7 +391,6 @@ class Person(BaseModel): content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, reasoning=None, usage=None, ), diff --git a/tests/runtime/test_token_calibration.py b/tests/runtime/test_token_calibration.py index 80f0d67b9..db50213d7 100644 --- a/tests/runtime/test_token_calibration.py +++ b/tests/runtime/test_token_calibration.py @@ -56,7 +56,6 @@ async def test_context_stats_populated_after_llm_call(self): content="ok", tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": "ok"}, usage={"prompt_tokens": 500, "completion_tokens": 7}, ) ] @@ -106,7 +105,6 @@ async def test_headline_is_raw_provider_total_no_ratio(self): content="ok", tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": "ok"}, usage={"prompt_tokens": 150_000, "completion_tokens": 9}, ) ] @@ -146,7 +144,6 @@ async def test_provider_response_recalibrates_tokens_per_char(self): content="ok", tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": "ok"}, usage={"prompt_tokens": 5_000, "completion_tokens": 4}, ) ] @@ -221,7 +218,6 @@ async def test_response_usage_overwrites_context_stats_total_tokens(self): content="ok", tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": "ok"}, usage={"prompt_tokens": 12_345, "completion_tokens": 7}, ) ] @@ -256,7 +252,6 @@ async def test_missing_usage_leaves_total_tokens_none(self): content="ok", tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": "ok"}, usage=None, ) ] diff --git a/tests/runtime/test_truncation_config_integration.py b/tests/runtime/test_truncation_config_integration.py index 824552130..02cf7f091 100644 --- a/tests/runtime/test_truncation_config_integration.py +++ b/tests/runtime/test_truncation_config_integration.py @@ -317,7 +317,6 @@ async def acall(self, messages, tools=None, output_model=None, **kwargs): ) ], finish_reason="tool_calls", - assistant_message={"role": "assistant", "content": ""}, reasoning=None, usage={"prompt_tokens": 11, "completion_tokens": 3}, ) diff --git a/tests/strategies/test_codeact_future_annotations.py b/tests/strategies/test_codeact_future_annotations.py index c0eab46d3..9aba4d1cc 100644 --- a/tests/strategies/test_codeact_future_annotations.py +++ b/tests/strategies/test_codeact_future_annotations.py @@ -211,7 +211,6 @@ def _resp(content: str = "", tool_calls: list | None = None) -> LLMResponse: content=content, tool_calls=tool_calls or [], finish_reason=finish_reason, - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/strategies/test_codeact_max_tokens_error.py b/tests/strategies/test_codeact_max_tokens_error.py index b55675bdd..fd07c6e19 100644 --- a/tests/strategies/test_codeact_max_tokens_error.py +++ b/tests/strategies/test_codeact_max_tokens_error.py @@ -4,13 +4,14 @@ import json from types import SimpleNamespace +from typing import Literal import pytest from nooa import Agent, return_text_as_result, strategy from nooa.config import CodeActConfig from nooa.errors import GenerationError -from nooa.events import DebugTrace +from nooa.events import DebugTrace, Error from nooa.strategies.codeact import CodeActStrategy from nooa.unifiedllm import FakeLLMClient, LLMResponse, ToolCall @@ -18,7 +19,9 @@ def _resp( - content: str, tool_calls: list | None = None, finish_reason: str | None = None + content: str, + tool_calls: list[ToolCall] | None = None, + finish_reason: Literal["stop", "tool_calls", "length", "error"] | None = None, ) -> LLMResponse: """Create a test LLM response.""" if finish_reason is None: @@ -28,7 +31,6 @@ def _resp( content=content, tool_calls=tool_calls or [], finish_reason=finish_reason, - assistant_message={"role": "assistant", "content": content}, ) @@ -59,8 +61,8 @@ async def my_task(self) -> str: # Verify a DebugTrace was emitted (not an Error event in LLM context) all_events = agent_instance.event_manager.values() - debug_events = [e for e in all_events if e.event_type == "DebugTrace"] - error_events = [e for e in all_events if e.event_type == "Error"] + debug_events = [e for e in all_events if isinstance(e, DebugTrace)] + error_events = [e for e in all_events if isinstance(e, Error)] assert any("finish_reason='length'" in e.content for e in debug_events), ( f"Expected DebugTrace with finish_reason, got: {[e.content for e in debug_events]}" ) @@ -69,6 +71,7 @@ async def my_task(self) -> str: assert len(max_tokens_errors) == 0, ( f"max_tokens error should not be an Error event (LLM-visible), got: {max_tokens_errors}" ) + assert [event.content for event in all_events if isinstance(event, LLMResponse)] == [""] @pytest.mark.asyncio async def test_finish_reason_length_does_not_return_partial_text(self): @@ -92,7 +95,7 @@ async def my_task(self) -> str: await agent_instance.my_task() events = agent_instance.event_manager.values() - assert [event.content for event in events if event.event_type == "LLMOutput"] == [ + assert [event.content for event in events if isinstance(event, LLMResponse)] == [ "truncated partial" ] assert not any(event.event_type == "TextOnlyReply" for event in events) @@ -140,7 +143,7 @@ async def my_task(self) -> str: """A task.""" ... - def _ret(val, cid="c2"): + def _ret(val: str, cid: str = "c2") -> ToolCall: return ToolCall(id=cid, name="return_result", arguments=json.dumps({"result": val})) fake_llm = FakeLLMClient( @@ -153,3 +156,15 @@ def _ret(val, cid="c2"): agent_instance = TestAgent(llm=fake_llm) result = await agent_instance.my_task() assert result == "hello" + assert [ + event.content + for event in agent_instance.event_manager.values() + if isinstance(event, LLMResponse) + ] == ["", ""] + assert not any( + message.get("role") == "assistant" + and isinstance(message.get("content"), str) + and not message["content"].strip() + and not message.get("tool_calls") + for message in fake_llm.last_messages + ) diff --git a/tests/strategies/test_codeact_pure_python_coverage.py b/tests/strategies/test_codeact_pure_python_coverage.py index 2957e1aaf..0dbeb5a74 100644 --- a/tests/strategies/test_codeact_pure_python_coverage.py +++ b/tests/strategies/test_codeact_pure_python_coverage.py @@ -51,7 +51,6 @@ def _resp(content: str, tool_calls: list | None = None) -> LLMResponse: content=content, tool_calls=tool_calls or [], finish_reason=finish_reason, - assistant_message={"role": "assistant", "content": content}, ) @@ -1320,8 +1319,8 @@ async def compute(self) -> int: assert result == 99 @pytest.mark.asyncio - async def test_xml_format_error_removes_malformed_event(self): - """XMLFormatError should remove the malformed LLMOutput event.""" + async def test_xml_format_error_retains_malformed_provider_turn(self): + """XML recovery appends feedback without deleting the provider turn.""" class TestAgent(Agent, llm=_TEST_LLM): @strategy(PurePythonStrategy(max_iterations=10, max_retries=3)) @@ -1338,20 +1337,16 @@ async def compute(self) -> int: ) agent = TestAgent(llm=fake_llm) - # Spy on event_manager.remove to verify it's called - removed_ids = [] - original_remove = agent.runtime.event_manager.remove - - def spy_remove(key): - removed_ids.append(key) - return original_remove(key) - - with patch.object(agent.runtime.event_manager, "remove", side_effect=spy_remove): - result = await agent.compute() + result = await agent.compute() assert result == 99 - # At least one event was removed (the malformed XML response) - assert len(removed_ids) >= 1 + outputs = [ + event for event in agent.event_manager.values() if event.event_type == "LLMResponse" + ] + assert [event.content for event in outputs] == [ + "return 42", + "return 99", + ] @pytest.mark.asyncio async def test_empty_code_response_records_error(self): @@ -1372,10 +1367,49 @@ async def compute(self) -> int: agent = TestAgent(llm=fake_llm) result = await agent.compute() assert result == 42 + outputs = [ + event.content + for event in agent.event_manager.values() + if event.event_type == "LLMResponse" + ] + assert outputs == ["", "return 42"] + + @pytest.mark.asyncio + async def test_whitespace_response_is_retained_but_not_replayed(self): + """Whitespace stays in the journal without becoming an assistant message.""" + + class TestAgent(Agent, llm=_TEST_LLM): + @strategy(PurePythonStrategy(max_iterations=10, max_retries=3)) + async def compute(self) -> int: + """Compute.""" + ... + + whitespace = " \n" + fake_llm = FakeLLMClient( + scripted_responses=[ + _resp(whitespace), + _resp("return 42"), + ] + ) + agent = TestAgent(llm=fake_llm) + + assert await agent.compute() == 42 + outputs = [ + event.content + for event in agent.event_manager.values() + if event.event_type == "LLMResponse" + ] + assert outputs == [whitespace, "return 42"] + assert not any( + message.get("role") == "assistant" + and isinstance(message.get("content"), str) + and not message["content"].strip() + for message in fake_llm.last_messages + ) @pytest.mark.asyncio - async def test_empty_response_removes_event(self): - """Empty response should call event_manager.remove() to clean up the LLMOutput.""" + async def test_empty_response_does_not_remove_provider_turn(self): + """Empty recovery relies on context projection instead of mutating history.""" class TestAgent(Agent, llm=_TEST_LLM): @strategy(PurePythonStrategy(max_iterations=10, max_retries=3)) @@ -1395,7 +1429,6 @@ async def patched_generate_code(self_strat, runtime, session): fake_llm = FakeLLMClient(scripted_responses=[_resp("return 42")]) agent = TestAgent(llm=fake_llm) - # Spy on event_manager.remove to track calls removed_ids = [] original_remove = agent.runtime.event_manager.remove @@ -1410,7 +1443,7 @@ def spy_remove(key): result = await agent.compute() assert result == 42 - assert "empty_evt_id" in removed_ids + assert "empty_evt_id" not in removed_ids @pytest.mark.asyncio async def test_api_error_exhausts_retries(self): @@ -1548,7 +1581,8 @@ def get_code(self, call, config=None): await strat._run_prefill(rt, call, builtins, session) event_types = [type(e).__name__ for e in added_events] - assert "LLMOutput" in event_types + assert "AssistantEvent" in event_types + assert "LLMResponse" not in event_types assert "PythonOutput" in event_types assert "Feedback" not in event_types diff --git a/tests/strategies/test_codeact_strategy.py b/tests/strategies/test_codeact_strategy.py index 89c732cbe..5bdf1e413 100644 --- a/tests/strategies/test_codeact_strategy.py +++ b/tests/strategies/test_codeact_strategy.py @@ -37,7 +37,6 @@ def _resp(content: str, tool_calls: list | None = None) -> LLMResponse: content=content, tool_calls=tool_calls or [], finish_reason=finish_reason, - assistant_message={"role": "assistant", "content": content}, ) @@ -216,8 +215,8 @@ async def compute_sum(self) -> int: assert result == 15 @pytest.mark.asyncio - async def test_reasoning_items_replayed_with_tool_call_history(self): - """Opaque reasoning state is retained for the next CodeAct turn.""" + async def test_reasoning_items_are_stored_once_and_fail_closed_on_replay(self): + """The IR retains opaque state but does not replay it before #310's gate.""" class TestAgent(Agent, llm=_TEST_LLM): async def compute(self) -> int: @@ -233,7 +232,7 @@ async def compute(self) -> int: first_response = _resp( "", tool_calls=[_tool_call("value = 42\nprint(value)", call_id="call_reasoning")] ) - first_response.assistant_message["reasoning_items"] = [reasoning_item] + first_response.llm_state = {"reasoning_items": [reasoning_item]} fake_llm = FakeLLMClient( scripted_responses=[ first_response, @@ -245,18 +244,27 @@ async def compute(self) -> int: result = await agent_instance.compute() assert result == 42 + output_event = next( + event + for event in agent_instance.event_manager.values() + if event.event_type == "LLMResponse" + and event.tool_calls + and event.tool_calls[0].id == "call_reasoning" + ) tool_call_event = next( event for event in agent_instance.event_manager.values() if event.event_type == "ToolCallEvent" and event.tool_call_id == "call_reasoning" ) - assert tool_call_event.reasoning_items == [reasoning_item] + assert "reasoning_items" not in type(tool_call_event).model_fields + assert tool_call_event.llm_response_id == output_event.id + assert output_event.llm_state == {"reasoning_items": [reasoning_item]} replayed_tool_call = next( message for message in fake_llm.last_messages if message.get("role") == "assistant" and message.get("tool_calls") ) - assert replayed_tool_call["reasoning_items"] == [reasoning_item] + assert "reasoning_items" not in replayed_tool_call @pytest.mark.asyncio async def test_multiple_tool_calls_then_result(self): @@ -1052,7 +1060,7 @@ async def test_tool_call_creates_correct_event_sequence(self): The architecture nests ToolResult inside ToolCallEvent.result, so there are no separate tool_result events. - Sequence: task -> tool_call (with nested result) -> execute_python -> tool_call (with nested result) + Each provider turn remains as LLMResponse before its execution events. """ class TestAgent(Agent, llm=_TEST_LLM): @@ -1082,18 +1090,23 @@ async def get_value(self) -> int: events = agent_instance.event_manager.values() event_types = [e.event_type for e in events] - # Architecture: ToolResult is nested in ToolCallEvent.result (no separate tool_result events) - # Sequence: Task -> ToolCallEvent -> PythonOutput -> ToolCallEvent + # LLMResponse is the canonical provider turn. ToolResult remains nested + # in ToolCallEvent, with no separate tool-result event. assert event_types == [ "Task", + "LLMResponse", "ToolCallEvent", "PythonOutput", + "LLMResponse", "ToolCallEvent", - ], f"Expected ['Task', 'ToolCallEvent', 'PythonOutput', 'ToolCallEvent'], got {event_types}" + ] # Verify first ToolCallEvent has correct data (execute_python) - tool_call_event = events[1] + first_output = events[1] + tool_call_event = events[2] + assert first_output.tool_calls[0].id == "call_abc123" assert tool_call_event.event_type == "ToolCallEvent" + assert tool_call_event.llm_response_id == first_output.id assert tool_call_event.tool_call_id == "call_abc123" assert tool_call_event.name == "execute_python" assert "code" in tool_call_event.arguments @@ -1103,23 +1116,21 @@ async def get_value(self) -> int: assert tool_call_event.result.tool_call_id == "call_abc123" # Verify execute_python event contains the deferred output - exec_output_event = events[2] + exec_output_event = events[3] assert exec_output_event.event_type == "PythonOutput" assert exec_output_event.tool_call_id == "call_abc123" # Verify second ToolCallEvent is return_result - return_call_event = events[3] + return_output = events[4] + return_call_event = events[5] assert return_call_event.event_type == "ToolCallEvent" + assert return_call_event.llm_response_id == return_output.id assert return_call_event.name == "return_result" assert return_call_event.result is not None @pytest.mark.asyncio - async def test_no_empty_assistant_event_before_tool_call(self): - """Empty LLMOutputs should be removed before ToolCallEvents. - - When the LLM returns a tool call with no content, the CodeAct strategy - removes the empty LLMOutput to keep history clean. - """ + async def test_tool_turn_is_retained_but_not_rendered_twice(self): + """The canonical LLMResponse persists while ToolCallEvent drives replay.""" class TestAgent(Agent, llm=_TEST_LLM): @strategy(CodeActStrategy(config=CodeActConfig())) @@ -1139,19 +1150,20 @@ async def simple_task(self) -> str: assert result == "done" - # Check that we don't have empty LLMOutputs before ToolCallEvents events = agent_instance.event_manager.values() - event_types = [e.event_type for e in events] - - # Verify no empty LLMOutput events remain (they should be removed when tool calls are made) - for i in range(len(event_types) - 1): - if event_types[i] == "LLMOutput" and event_types[i + 1] == "ToolCallEvent": - # Check if it's an empty assistant event - if not events[i].content: - pytest.fail( - f"Found empty LLMOutput before ToolCallEvent at index {i}. " - f"Event sequence: {event_types}" - ) + outputs = [event for event in events if event.event_type == "LLMResponse"] + calls = [event for event in events if event.event_type == "ToolCallEvent"] + assert len(outputs) == len(calls) == 2 + assert [output.tool_calls[0].id for output in outputs] == [ + call.tool_call_id for call in calls + ] + assert [call.llm_response_id for call in calls] == [output.id for output in outputs] + assert not any( + message.get("role") == "assistant" + and not message.get("tool_calls") + and not message.get("content") + for message in fake_llm.last_messages + ) @pytest.mark.asyncio async def test_text_only_stop_response_routes_through_return_result(self): @@ -1192,8 +1204,8 @@ async def think_and_answer(self) -> str: # The original assistant turn is preserved; no synthetic provider tool # exchange is added to history. - llm_outputs = [e for e in events if e.event_type == "LLMOutput"] - assert [event.content for event in llm_outputs] == ["The answer is 42."] + llm_responses = [e for e in events if e.event_type == "LLMResponse" and not e.tool_calls] + assert [event.content for event in llm_responses] == ["The answer is 42."] tool_call_events = [e for e in events if e.event_type == "ToolCallEvent"] assert tool_call_events == [] @@ -1229,7 +1241,6 @@ async def think_and_answer(self) -> str: content=ThoughtModel(thought="I need to reason carefully here."), tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": ""}, ) fake_llm = FakeLLMClient( scripted_responses=[ @@ -1244,9 +1255,9 @@ async def think_and_answer(self) -> str: assert "I need to reason carefully here." in result events = agent_instance.event_manager.values() - llm_outputs = [e for e in events if e.event_type == "LLMOutput"] - assert len(llm_outputs) == 1 - assert "I need to reason carefully here." in llm_outputs[0].content + llm_responses = [e for e in events if e.event_type == "LLMResponse" and not e.tool_calls] + assert len(llm_responses) == 1 + assert "I need to reason carefully here." in llm_responses[0].content tool_call_events = [e for e in events if e.event_type == "ToolCallEvent"] assert tool_call_events == [] @@ -1289,8 +1300,8 @@ async def compute_stats(self) -> dict: events = agent_instance.event_manager.values() # The text-only assistant turn is retained, followed by a user correction # and the model's real return_result call. - llm_outputs = [e for e in events if e.event_type == "LLMOutput"] - assert [event.content for event in llm_outputs] == [ + llm_responses = [e for e in events if e.event_type == "LLMResponse" and not e.tool_calls] + assert [event.content for event in llm_responses] == [ "I have successfully completed the computation!" ] corrections = [e for e in events if e.event_type == "Error"] @@ -1332,8 +1343,8 @@ async def do_side_effects(self) -> None: assert result is None events = agent_instance.event_manager.values() - llm_outputs = [e for e in events if e.event_type == "LLMOutput"] - assert [event.content for event in llm_outputs] == [""] + llm_responses = [e for e in events if e.event_type == "LLMResponse"] + assert [event.content for event in llm_responses] == [""] tool_call_events = [e for e in events if e.event_type == "ToolCallEvent"] assert tool_call_events == [] @@ -1371,11 +1382,8 @@ async def whitespace_task(self) -> str: ) @pytest.mark.asyncio - async def test_text_only_basemodel_response_with_tool_calls_prepends_comment(self): - """BaseModel content alongside execute_python tool calls is prepended as a comment. - - Exercises the model_dump_json() branch in the content+tool_calls path. - """ + async def test_basemodel_content_with_tool_calls_preserves_turn_and_arguments(self): + """BaseModel prose is serialized on the turn without mutating execution.""" from pydantic import BaseModel as PydanticBaseModel class ThoughtModel(PydanticBaseModel): @@ -1392,7 +1400,6 @@ async def think_and_answer(self) -> str: content=ThoughtModel(thought="I should calculate this."), tool_calls=[_tool_call("x = 6 * 7", call_id="c1")], finish_reason="tool_calls", - assistant_message={"role": "assistant", "content": ""}, ) fake_llm = FakeLLMClient( scripted_responses=[ @@ -1410,10 +1417,10 @@ async def think_and_answer(self) -> str: e for e in events if e.event_type == "ToolCallEvent" and e.name == "execute_python" ] assert len(exec_calls) == 1 - code = exec_calls[0].arguments["code"] - assert code.startswith("# "), f"Expected comment prepended, got: {code!r}" - assert "I should calculate this." in code, f"BaseModel JSON should appear in code: {code!r}" - assert "x = 6 * 7" in code, f"Original code should follow: {code!r}" + assert exec_calls[0].arguments["code"] == "x = 6 * 7" + output = next(e for e in events if e.event_type == "LLMResponse" and e.content) + assert output.content == '{"thought":"I should calculate this."}' + assert output.tool_calls[0].arguments == json.dumps({"code": "x = 6 * 7"}) @pytest.mark.asyncio async def test_empty_stop_response_routes_through_return_result(self): @@ -1445,8 +1452,10 @@ async def empty_task(self) -> str: # The empty assistant turn is retained and the only tool event is the # model's successful self-correction. all_events = agent_instance.event_manager.values() - llm_outputs = [e for e in all_events if e.event_type == "LLMOutput"] - assert [event.content for event in llm_outputs] == [""] + llm_responses = [ + e for e in all_events if e.event_type == "LLMResponse" and not e.tool_calls + ] + assert [event.content for event in llm_responses] == [""] tool_call_events = [e for e in all_events if e.event_type == "ToolCallEvent"] assert len(tool_call_events) == 1 assert tool_call_events[0].name == "return_result" @@ -1481,14 +1490,16 @@ async def multi_step(self) -> int: events = agent_instance.event_manager.values() event_types = [e.event_type for e in events] - # Architecture: ToolResult is nested in ToolCallEvent.result (no separate tool_result events) - # Sequence: Task -> (ToolCallEvent -> PythonOutput) x2 -> ToolCallEvent + # Each provider response is retained immediately before its execution projection. assert event_types == [ "Task", + "LLMResponse", "ToolCallEvent", "PythonOutput", + "LLMResponse", "ToolCallEvent", "PythonOutput", + "LLMResponse", "ToolCallEvent", ], f"Expected correct sequence with nested results, got {event_types}" @@ -1500,13 +1511,8 @@ async def multi_step(self) -> int: ) @pytest.mark.asyncio - async def test_content_plus_tool_calls_prepends_comment(self): - """When LLM returns both content and execute_python tool calls, the content - is prepended as a comment at the top of the first execute_python code. - - This preserves any explanatory text the LLM produced alongside its tool - call without creating a separate synthetic event. - """ + async def test_content_plus_tool_calls_preserves_turn_and_arguments(self): + """Assistant prose remains on LLMResponse and execution stays exact.""" class TestAgent(Agent, llm=_TEST_LLM): @strategy(CodeActStrategy(config=CodeActConfig())) @@ -1531,20 +1537,15 @@ async def think_and_answer(self) -> str: assert result == "done" events = agent_instance.event_manager.values() - tool_calls = [e for e in events if e.event_type == "ToolCallEvent"] - - # First tool call should have the content prepended as a comment - first_tc = tool_calls[0] - code = first_tc.arguments["code"] - assert code.startswith("# "), f"Expected comment prepended, got: {code!r}" - assert "Let me work through this step by step." in code, ( - f"Original content should appear in the comment, got: {code!r}" - ) - assert "x = 42" in code, f"Original code should follow the comment, got: {code!r}" + tool_call = next(e for e in events if e.event_type == "ToolCallEvent") + assert tool_call.arguments["code"] == "x = 42" + output = next(e for e in events if e.event_type == "LLMResponse" and e.content) + assert output.content == "Let me work through this step by step." + assert output.tool_calls[0].arguments == json.dumps({"code": "x = 42"}) @pytest.mark.asyncio - async def test_content_plus_tool_calls_empty_content_not_prepended(self): - """Whitespace-only content alongside tool calls is ignored (not prepended).""" + async def test_content_plus_tool_calls_empty_content_does_not_mutate_call(self): + """Whitespace-only assistant content does not mutate execution arguments.""" class TestAgent(Agent, llm=_TEST_LLM): @strategy(CodeActStrategy(config=CodeActConfig())) @@ -1563,14 +1564,10 @@ async def compute(self) -> int: assert result == 7 - # The return_result tool call should have no comment prepended events = agent_instance.event_manager.values() tool_calls = [e for e in events if e.event_type == "ToolCallEvent"] final_tc = tool_calls[0] - code = final_tc.arguments.get("code", "") - assert not code.startswith("# "), ( - f"Whitespace-only content should not be prepended, got: {code!r}" - ) + assert final_tc.arguments == {"result": 7} @pytest.mark.asyncio async def test_text_only_loop_aborts_after_threshold(self): @@ -1617,8 +1614,8 @@ async def stuck_task(self) -> dict: # The three assistant turns remain in history without fabricated tool calls. events = agent_instance.event_manager.values() - llm_outputs = [e for e in events if e.event_type == "LLMOutput"] - assert len(llm_outputs) == 3 + llm_responses = [e for e in events if e.event_type == "LLMResponse"] + assert len(llm_responses) == 3 return_result_calls = [ e for e in events if e.event_type == "ToolCallEvent" and e.name == "return_result" ] @@ -1721,8 +1718,8 @@ async def stuck(self) -> str: assert last.exception_type == "GenerationError" @pytest.mark.asyncio - async def test_content_plus_tool_calls_prepends_first_execute_python_only(self): - """The comment is prepended to the first execute_python; later ones are untouched.""" + async def test_content_plus_tool_calls_preserves_turn_and_execution_arguments(self): + """Assistant prose and raw tool calls are retained without mutating execution.""" class TestAgent(Agent, llm=_TEST_LLM): @strategy(CodeActStrategy(config=CodeActConfig())) @@ -1751,43 +1748,13 @@ async def compute(self) -> int: exec_calls = [ e for e in events if e.event_type == "ToolCallEvent" and e.name == "execute_python" ] - # First execute_python should have the comment prepended - assert exec_calls[0].arguments["code"].startswith("# "), ( - f"First execute_python should have the comment prepended, got: {exec_calls[0].arguments['code']!r}" - ) - # Second execute_python should be unchanged - assert exec_calls[1].arguments["code"] == "y = 2", ( - f"Second execute_python should be unchanged, got: {exec_calls[1].arguments['code']!r}" - ) - - def test_prepend_comment_skips_to_next_on_invalid_json(self): - """If the first execute_python has invalid JSON arguments, skip it and prepend to next.""" - from nooa.strategies.codeact import _prepend_comment - from nooa.unifiedllm import ToolCall - - bad_tc = ToolCall(id="bad", name="execute_python", arguments="NOT VALID JSON") - good_tc = ToolCall(id="c2", name="execute_python", arguments=json.dumps({"code": "x = 42"})) - result = _prepend_comment([bad_tc, good_tc], "Thinking aloud.") - - # First tool call unchanged (bad JSON) - assert result[0].arguments == "NOT VALID JSON" - # Second tool call should have the comment prepended - args = json.loads(result[1].arguments) - assert args["code"].startswith("# "), ( - f"Second execute_python should have the comment prepended, got: {args['code']!r}" - ) - assert "x = 42" in args["code"] - - def test_prepend_comment_no_execute_python_unchanged(self): - """If there's no execute_python in the list, all tool calls are returned unchanged.""" - from nooa.strategies.codeact import _prepend_comment - from nooa.unifiedllm import ToolCall - - rr = ToolCall(id="ret", name="return_result", arguments=json.dumps({"result": 7})) - result = _prepend_comment([rr], "some content") - - assert len(result) == 1 - assert result[0].arguments == rr.arguments + assert [call.arguments["code"] for call in exec_calls] == ["x = 1", "y = 2"] + output = next(event for event in events if event.event_type == "LLMResponse") + assert output.content == "Thinking aloud." + assert [call.arguments for call in output.tool_calls] == [ + json.dumps({"code": "x = 1"}), + json.dumps({"code": "y = 2"}), + ] class TestCodeActStrategyPersistentState: @@ -2896,6 +2863,23 @@ async def compute(self) -> int: if isinstance(event, PythonOutput) ] assert [event.execution_count for event in outputs] == [1, 2, 3] + events = agent_instance.event_manager.values() + multi_turn = next( + event + for event in events + if event.event_type == "LLMResponse" and len(event.tool_calls) == 3 + ) + projected_calls = [ + event + for event in events + if event.event_type == "ToolCallEvent" and event.name == "execute_python" + ] + assert [call.id for call in multi_turn.tool_calls] == [ + "call_1", + "call_2", + "call_3", + ] + assert {call.llm_response_id for call in projected_calls} == {multi_turn.id} @pytest.mark.asyncio async def test_multi_tool_calls_stop_on_first_error(self): diff --git a/tests/strategies/test_codeact_text_only_reply.py b/tests/strategies/test_codeact_text_only_reply.py index 609102afa..90126a8ed 100644 --- a/tests/strategies/test_codeact_text_only_reply.py +++ b/tests/strategies/test_codeact_text_only_reply.py @@ -16,7 +16,7 @@ from nooa.config import CodeActConfig from nooa.context_blocks import ToolCallEvent from nooa.errors import GenerationError -from nooa.events import LLMOutput, PythonOutput, TextOnlyReply +from nooa.events import PythonOutput, TextOnlyReply from nooa.runtime.event_manager import EventManager from nooa.runtime.harness_metrics import HarnessMetrics from nooa.storage import SQLiteStorageManager @@ -25,7 +25,7 @@ _TEST_LLM = FakeLLMClient() -def _resp(content="", tool_calls=None, finish_reason=None): +def _resp(content="", tool_calls=None, finish_reason=None, reasoning=None): if finish_reason is None: finish_reason = "tool_calls" if tool_calls else "stop" return LLMResponse( @@ -33,7 +33,7 @@ def _resp(content="", tool_calls=None, finish_reason=None): content=content, tool_calls=tool_calls or [], finish_reason=finish_reason, - assistant_message={"role": "assistant", "content": content}, + reasoning=reasoning, ) @@ -49,6 +49,10 @@ def _events(agent, event_type): return [event for event in agent.event_manager.values() if isinstance(event, event_type)] +def _text_outputs(agent): + return [event for event in _events(agent, LLMResponse) if not event.tool_calls] + + @pytest.mark.asyncio async def test_default_preserves_text_adds_error_and_retries(): class TestAgent(Agent, llm=_TEST_LLM): @@ -59,7 +63,7 @@ async def my_task(self) -> dict: fake_llm = FakeLLMClient( scripted_responses=[ - _resp("I think the answer is ready."), + _resp("I think the answer is ready.", reasoning="I checked the evidence."), _resp(tool_calls=[_ret({"ok": True})]), ] ) @@ -67,8 +71,9 @@ async def my_task(self) -> dict: assert await agent.my_task() == {"ok": True} - outputs = _events(agent, LLMOutput) + outputs = _text_outputs(agent) assert [event.content for event in outputs] == ["I think the answer is ready."] + assert outputs[0].reasoning == "I checked the evidence." diagnostics = _events(agent, TextOnlyReply) assert len(diagnostics) == 1 @@ -106,7 +111,7 @@ async def my_task(self) -> str: agent = TestAgent(llm=FakeLLMClient(scripted_responses=[_resp("done")])) assert await agent.my_task() == "done" - assert [event.content for event in _events(agent, LLMOutput)] == ["done"] + assert [event.content for event in _events(agent, LLMResponse)] == ["done"] assert _events(agent, ToolCallEvent) == [] diagnostic = _events(agent, TextOnlyReply)[0] assert diagnostic.handler == "return_text_as_result" @@ -135,10 +140,27 @@ async def my_task(self) -> str: await agent.my_task() assert calls == [] - assert [event.content for event in _events(agent, LLMOutput)] == ["partial"] + assert [event.content for event in _events(agent, LLMResponse)] == ["partial"] assert _events(agent, TextOnlyReply) == [] +@pytest.mark.asyncio +async def test_empty_error_response_remains_durable(): + class TestAgent(Agent, llm=_TEST_LLM): + @strategy(CodeActStrategy()) + async def my_task(self) -> str: + """Return a string.""" + ... + + agent = TestAgent(llm=FakeLLMClient(scripted_responses=[_resp("", finish_reason="error")])) + + with pytest.raises(GenerationError, match="incomplete response"): + await agent.my_task() + + responses = _events(agent, LLMResponse) + assert [(event.content, event.finish_reason) for event in responses] == [("", "error")] + + @pytest.mark.asyncio async def test_default_does_not_treat_valid_string_as_result(): class TestAgent(Agent, llm=_TEST_LLM): @@ -151,7 +173,7 @@ async def my_task(self) -> str: agent = TestAgent(llm=fake_llm) assert await agent.my_task() == "done" - assert [event.content for event in _events(agent, LLMOutput)] == ["prose"] + assert [event.content for event in _text_outputs(agent)] == ["prose"] assert any(event.event_type == "Error" for event in agent.event_manager.values()) @@ -167,7 +189,7 @@ async def my_task(self) -> dict: agent = TestAgent(llm=fake_llm) assert await agent.my_task() == {"ok": True} - assert [event.content for event in _events(agent, LLMOutput)] == [""] + assert [event.content for event in _text_outputs(agent)] == [""] assert [event.content for event in _events(agent, TextOnlyReply)] == [""] assert not any( message.get("role") == "assistant" and not message.get("content") @@ -196,7 +218,7 @@ async def my_task(self) -> str: agent = TestAgent(llm=fake_llm) assert await agent.my_task() == "done" - assert [event.content for event in _events(agent, LLMOutput)] == ["hello"] + assert [event.content for event in _text_outputs(agent)] == ["hello"] assert [event.tool_call_id for event in _events(agent, ToolCallEvent)] == [ "synthetic_cell", "c_ret", @@ -301,9 +323,11 @@ async def my_task(self) -> dict: reopened = SQLiteStorageManager(db_path) try: resumed = EventManager(backend=reopened.event_backend).values() - assert [event.content for event in resumed if isinstance(event, LLMOutput)] == [ - "I should have used a tool." - ] + assert [ + event.content + for event in resumed + if isinstance(event, LLMResponse) and not event.tool_calls + ] == ["I should have used a tool."] diagnostics = [event for event in resumed if isinstance(event, TextOnlyReply)] assert len(diagnostics) == 1 assert diagnostics[0].action == "retry" diff --git a/tests/strategies/test_error_recovery_gl106.py b/tests/strategies/test_error_recovery_gl106.py index a2c95a574..4bc3309a1 100644 --- a/tests/strategies/test_error_recovery_gl106.py +++ b/tests/strategies/test_error_recovery_gl106.py @@ -56,7 +56,6 @@ def _resp(content: str = "", tool_calls: list | None = None) -> LLMResponse: content=content, tool_calls=tool_calls or [], finish_reason=finish_reason, - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/strategies/test_non_pydantic_return_types.py b/tests/strategies/test_non_pydantic_return_types.py index e98918e68..700c23a0b 100644 --- a/tests/strategies/test_non_pydantic_return_types.py +++ b/tests/strategies/test_non_pydantic_return_types.py @@ -65,7 +65,6 @@ def _resp(content: str, tool_calls: list | None = None) -> LLMResponse: content=content, tool_calls=tool_calls or [], finish_reason=finish_reason, - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/strategies/test_predict_list_root_schema.py b/tests/strategies/test_predict_list_root_schema.py index 2cf836cb9..97d1d130f 100644 --- a/tests/strategies/test_predict_list_root_schema.py +++ b/tests/strategies/test_predict_list_root_schema.py @@ -42,7 +42,6 @@ def _llm_resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/strategies/test_pure_python_nested_structured_output.py b/tests/strategies/test_pure_python_nested_structured_output.py index 01e46c7bd..f3948578b 100644 --- a/tests/strategies/test_pure_python_nested_structured_output.py +++ b/tests/strategies/test_pure_python_nested_structured_output.py @@ -45,7 +45,6 @@ async def _summarize_doc(self, doc: str) -> str: return summaries''', tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": "code"}, ), # PredictStrategy for basic types wraps in {"value": ...} LLMResponse( @@ -53,7 +52,6 @@ async def _summarize_doc(self, doc: str) -> str: content='{"value": "Document summary"}', tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": '{"value": "Document summary"}'}, ), ] @@ -97,7 +95,6 @@ async def _process_item(self, item: str) -> str: return results''', tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": "code"}, ), # Three PredictStrategy responses for the three loop iterations LLMResponse( @@ -105,21 +102,18 @@ async def _process_item(self, item: str) -> str: content='{"value": "processed_a"}', tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": '{"value": "processed_a"}'}, ), LLMResponse( raw_response=None, content='{"value": "processed_b"}', tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": '{"value": "processed_b"}'}, ), LLMResponse( raw_response=None, content='{"value": "processed_c"}', tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": '{"value": "processed_c"}'}, ), ] diff --git a/tests/strategies/test_pure_python_strategy.py b/tests/strategies/test_pure_python_strategy.py index 113d39dd7..56624b5c3 100644 --- a/tests/strategies/test_pure_python_strategy.py +++ b/tests/strategies/test_pure_python_strategy.py @@ -20,7 +20,6 @@ def _resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) @@ -213,11 +212,7 @@ async def solve_task(self, x: int) -> int: # Get LLM-generated events (exclude synthetic prefill events) history_events = agent_instance.event_manager.values() - assistant_events = [ - e - for e in history_events - if e.event_type == "LLMOutput" and not (e.metadata or {}).get("prefill") - ] + assistant_events = [e for e in history_events if e.event_type == "LLMResponse"] # Should have 2 assistant events (one per LLM call) assert len(assistant_events) >= 2 @@ -330,21 +325,12 @@ async def compute(self, x: int) -> int: assert result == 6 - # Check history - the assistant message should have CLEAN code (no fences) + # The canonical assistant event keeps the exact provider response. history_events = agent_instance.event_manager.values() - assistant_events = [ - e - for e in history_events - if e.event_type == "LLMOutput" and not (e.metadata or {}).get("prefill") - ] + assistant_events = [e for e in history_events if e.event_type == "LLMResponse"] assert len(assistant_events) >= 1 - stored_code = assistant_events[0].content - - # Should NOT contain fence markers - assert "```" not in stored_code, f"History should not contain fences: {stored_code}" - # Should contain the actual code - assert "return x + 1" in stored_code + assert assistant_events[0].content == fenced_code class TestPurePythonMalformedOutputs: @@ -549,8 +535,8 @@ async def calculate_single(self, a: int, b: int, calculation: str) -> int: assert fake_llm.call_count == 1 @pytest.mark.asyncio - async def test_history_stores_clean_code_after_xml_stripping(self): - """History should store the clean code (without XML wrapper) for LLM learning.""" + async def test_history_retains_provider_turn_after_xml_stripping(self): + """Execution strips XML without rewriting canonical history.""" from nooa.strategies.pure_python import PurePythonStrategy class TestAgent(Agent, llm=_TEST_LLM): @@ -571,22 +557,12 @@ async def compute(self, x: int) -> int: assert result == 6 - # Check history - should have clean code without XML tags + # The canonical assistant event keeps the exact provider response. history_events = agent_instance.event_manager.values() - assistant_events = [ - e - for e in history_events - if e.event_type == "LLMOutput" and not (e.metadata or {}).get("prefill") - ] + assistant_events = [e for e in history_events if e.event_type == "LLMResponse"] assert len(assistant_events) >= 1 - stored_code = assistant_events[0].content - - # Should NOT contain XML tags - assert "" not in stored_code - assert "" not in stored_code - # Should contain the actual code - assert "return x + 1" in stored_code + assert assistant_events[0].content == wrapped_response @pytest.fixture diff --git a/tests/strategies/test_reflexion_strategy.py b/tests/strategies/test_reflexion_strategy.py index cd4e74bac..d230fa91f 100644 --- a/tests/strategies/test_reflexion_strategy.py +++ b/tests/strategies/test_reflexion_strategy.py @@ -184,7 +184,7 @@ async def test_execute_returns_on_satisfactory(self, mock_runtime): mock_runtime.generate = AsyncMock( return_value=( MagicMock( - content=ReflectionOutput( + parsed=ReflectionOutput( is_satisfactory=True, reasoning="Result looks good", ), @@ -220,7 +220,7 @@ async def test_execute_retries_on_unsatisfactory(self, mock_runtime): # First reflection: not satisfactory ( MagicMock( - content=ReflectionOutput( + parsed=ReflectionOutput( is_satisfactory=False, issues=["Needs improvement"], suggestions=["Try harder"], @@ -231,7 +231,7 @@ async def test_execute_retries_on_unsatisfactory(self, mock_runtime): # Second reflection: satisfactory ( MagicMock( - content=ReflectionOutput( + parsed=ReflectionOutput( is_satisfactory=True, reasoning="Now it's good", ), @@ -265,7 +265,7 @@ async def test_execute_respects_max_reflections(self, mock_runtime): mock_runtime.generate = AsyncMock( return_value=( MagicMock( - content=ReflectionOutput( + parsed=ReflectionOutput( is_satisfactory=False, issues=["Still not good"], ), @@ -306,7 +306,7 @@ async def test_execute_handles_base_strategy_error(self, mock_runtime): mock_runtime.generate = AsyncMock( return_value=( MagicMock( - content=ReflectionOutput(is_satisfactory=True), + parsed=ReflectionOutput(is_satisfactory=True), ), "event_123", ) @@ -364,7 +364,7 @@ async def test_execute_adds_feedback_on_error(self, mock_runtime): mock_runtime.generate = AsyncMock( return_value=( MagicMock( - content=ReflectionOutput(is_satisfactory=True), + parsed=ReflectionOutput(is_satisfactory=True), ), "event_123", ) @@ -403,7 +403,7 @@ async def test_execute_handles_dict_response(self, mock_runtime): mock_runtime.generate = AsyncMock( return_value=( MagicMock( - content={ + parsed={ "is_satisfactory": True, "issues": [], "suggestions": [], @@ -571,7 +571,7 @@ async def test_execute_adds_feedback_between_iterations(self, mock_runtime): # First reflection: not satisfactory ( MagicMock( - content=ReflectionOutput( + parsed=ReflectionOutput( is_satisfactory=False, issues=["Not good enough"], suggestions=["Do better"], @@ -582,7 +582,7 @@ async def test_execute_adds_feedback_between_iterations(self, mock_runtime): # Second reflection: satisfactory ( MagicMock( - content=ReflectionOutput(is_satisfactory=True), + parsed=ReflectionOutput(is_satisfactory=True), ), "event_2", ), @@ -619,7 +619,7 @@ async def test_execute_with_complex_result(self, mock_runtime): mock_runtime.generate = AsyncMock( return_value=( MagicMock( - content=ReflectionOutput( + parsed=ReflectionOutput( is_satisfactory=True, reasoning="Analysis looks complete", ), diff --git a/tests/strategies/test_return_type_mismatch.py b/tests/strategies/test_return_type_mismatch.py index 6401c0ee9..e4afd9cae 100644 --- a/tests/strategies/test_return_type_mismatch.py +++ b/tests/strategies/test_return_type_mismatch.py @@ -27,7 +27,6 @@ def _resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/strategies/test_session_locals_injection.py b/tests/strategies/test_session_locals_injection.py index 64126de1e..6ee452a6b 100644 --- a/tests/strategies/test_session_locals_injection.py +++ b/tests/strategies/test_session_locals_injection.py @@ -26,7 +26,6 @@ def _resp(content: str, tool_calls: list | None = None) -> LLMResponse: content=content, tool_calls=tool_calls or [], finish_reason=finish_reason, - assistant_message={"role": "assistant", "content": content}, ) @@ -173,7 +172,6 @@ def _pure_resp(code: str) -> LLMResponse: content=code, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": code}, ) diff --git a/tests/strategies/test_strategies_coverage.py b/tests/strategies/test_strategies_coverage.py index ba221cd10..f590b3533 100644 --- a/tests/strategies/test_strategies_coverage.py +++ b/tests/strategies/test_strategies_coverage.py @@ -65,7 +65,6 @@ def _resp(content: str, tool_calls: list | None = None) -> LLMResponse: content=content, tool_calls=tool_calls or [], finish_reason=finish_reason, - assistant_message={"role": "assistant", "content": content}, ) @@ -92,7 +91,6 @@ def _llm_resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) @@ -1291,54 +1289,42 @@ class M(BaseModel): value: int s = PredictStrategy() - mock = MagicMock() - mock.content = M(value=42) - mock.reasoning = None - result = s._parse_llm_response(mock, "test") + response = LLMResponse(content='{"value":42}', parsed=M(value=42)) + result = s._parse_llm_response(response, "test") assert result == {"value": 42} def test_dict_content(self): s = PredictStrategy() - mock = MagicMock() - mock.content = {"key": "val"} - mock.reasoning = None - result = s._parse_llm_response(mock, "test") + response = LLMResponse(content='{"key":"val"}', parsed={"key": "val"}) + result = s._parse_llm_response(response, "test") assert result == {"key": "val"} def test_string_json_content(self): s = PredictStrategy() - mock = MagicMock() - mock.content = '{"value": 123}' - mock.reasoning = None - result = s._parse_llm_response(mock, "test") + response = LLMResponse(content='{"value": 123}') + result = s._parse_llm_response(response, "test") assert result == {"value": 123} def test_non_dict_json_wrapped(self): """Non-dict JSON (e.g., list) is wrapped in {"value": ...}.""" s = PredictStrategy() - mock = MagicMock() - mock.content = "[1, 2, 3]" - mock.reasoning = None - result = s._parse_llm_response(mock, "test") + response = LLMResponse(content="[1, 2, 3]") + result = s._parse_llm_response(response, "test") assert result == {"value": [1, 2, 3]} def test_reasoning_fallback(self): """When content is empty, falls back to reasoning field.""" s = PredictStrategy() - mock = MagicMock() - mock.content = None - mock.reasoning = '{"value": 99}' - result = s._parse_llm_response(mock, "test") + response = LLMResponse(content="", reasoning='{"value": 99}') + result = s._parse_llm_response(response, "test") assert result == {"value": 99} def test_empty_content_raises_json_error(self): """Empty content/reasoning causes JSONDecodeError.""" s = PredictStrategy() - mock = MagicMock() - mock.content = None - mock.reasoning = None + response = LLMResponse(content="") with pytest.raises(json.JSONDecodeError): - s._parse_llm_response(mock, "test") + s._parse_llm_response(response, "test") class TestPredictStrategyExtractRaw: @@ -1714,7 +1700,7 @@ def test_tool_call_event_with_no_result_and_no_python_output(self): messages = formatter.format([block]) tool_msgs = [m for m in messages if m.role == Role.TOOL] assert len(tool_msgs) == 1 - assert tool_msgs[0].content == "" + assert tool_msgs[0].content == "(no result recorded)" def test_block_with_no_event_uses_content(self): from nooa.context_blocks import ResolvedBlock diff --git a/tests/strategies/test_strategy_validators.py b/tests/strategies/test_strategy_validators.py index b2e811aac..cb648560e 100644 --- a/tests/strategies/test_strategy_validators.py +++ b/tests/strategies/test_strategy_validators.py @@ -33,7 +33,6 @@ def _resp(tool_calls: list) -> LLMResponse: content="", tool_calls=tool_calls, finish_reason="tool_calls", - assistant_message={"role": "assistant", "content": ""}, ) diff --git a/tests/strategies/test_toolcall_result_none_regression.py b/tests/strategies/test_toolcall_result_none_regression.py index 87bc46886..2e2de8f5f 100644 --- a/tests/strategies/test_toolcall_result_none_regression.py +++ b/tests/strategies/test_toolcall_result_none_regression.py @@ -50,7 +50,6 @@ def _resp(content: str, tool_calls: list | None = None) -> LLMResponse: content=content, tool_calls=tool_calls or [], finish_reason=finish_reason, - assistant_message={"role": "assistant", "content": content}, ) @@ -225,8 +224,7 @@ def test_none_result_produces_placeholder_tool_result(self): tool_call_msg = messages[0] assert tool_call_msg.role == Role.ASSISTANT - assert tool_call_msg.tool_call is not None - assert tool_call_msg.tool_call.id == "tc_orphan" + assert [call.id for call in tool_call_msg.tool_calls] == ["tc_orphan"] result_msg = messages[1] assert result_msg.role == Role.TOOL @@ -331,8 +329,7 @@ async def get_number(self) -> int: ) messages = _event_block_to_messages(block, wrap_content=None) for msg in messages: - if msg.tool_call is not None: - tool_use_ids.add(msg.tool_call.id) + tool_use_ids.update(call.id for call in msg.tool_calls) if msg.role == Role.TOOL and msg.tool_call_id: tool_result_ids.add(msg.tool_call_id) @@ -479,7 +476,6 @@ async def compute(self) -> int: ) ], finish_reason="tool_calls", - assistant_message={"role": "assistant", "content": ""}, ), # Then return the result _resp("", tool_calls=[_return_result(call_id="call_ret", result=3)]), @@ -526,7 +522,6 @@ async def compute(self) -> int: ) ], finish_reason="tool_calls", - assistant_message={"role": "assistant", "content": ""}, ), # Then return valid result _resp("", tool_calls=[_return_result(call_id="call_ok", result=42)]), @@ -590,8 +585,8 @@ async def get_number(self) -> int: await agent_instance.get_number() events = agent_instance.event_manager.values() - llm_outputs = [e for e in events if e.event_type == "LLMOutput"] - assert [e.content for e in llm_outputs] == ["hello world"] + llm_responses = [e for e in events if e.event_type == "LLMResponse"] + assert [e.content for e in llm_responses] == ["hello world"] tool_call_events = [e for e in events if e.event_type == "ToolCallEvent"] assert tool_call_events == [] @@ -618,7 +613,6 @@ async def do_something(self) -> None: content="", tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": ""}, ), ] ) @@ -628,8 +622,8 @@ async def do_something(self) -> None: assert result is None events = agent_instance.event_manager.values() - llm_outputs = [e for e in events if e.event_type == "LLMOutput"] - assert [e.content for e in llm_outputs] == [""] + llm_responses = [e for e in events if e.event_type == "LLMResponse"] + assert [e.content for e in llm_responses] == [""] tool_call_events = [e for e in events if e.event_type == "ToolCallEvent"] assert tool_call_events == [] @@ -686,7 +680,7 @@ def test_xml_formatter_mixed_none_and_normal_results(self): formatter = XMLBlockFormatter() messages = formatter.format([normal_block, none_block]) - tool_use_ids = {m.tool_call.id for m in messages if m.tool_call is not None} + tool_use_ids = {call.id for message in messages for call in message.tool_calls} tool_result_ids = { m.tool_call_id for m in messages if m.role == Role.TOOL and m.tool_call_id } diff --git a/tests/test_archived_events_excluded_from_context.py b/tests/test_archived_events_excluded_from_context.py index 4c2bbef75..87905af47 100644 --- a/tests/test_archived_events_excluded_from_context.py +++ b/tests/test_archived_events_excluded_from_context.py @@ -63,11 +63,11 @@ def test_archived_context_blocks_events_not_in_context(event_manager): def test_archived_nemo_events_not_in_context(event_manager): """nooa events collapsed into a Summary must not appear in context.""" - from nooa.events import LLMOutput, Task + from nooa.events import LLMResponse, Task em = event_manager em.add(Task(prompt="do the thing")) # tag "1" - em.add(LLMOutput(content="done")) # tag "2" + em.add(LLMResponse(content="done")) # tag "2" em.add(Task(prompt="do another thing")) # tag "3" em.collapse("1", "2", summary_text="summarized first exchange") diff --git a/tests/test_cross_module_inheritance.py b/tests/test_cross_module_inheritance.py index 485065371..a43d0295a 100644 --- a/tests/test_cross_module_inheritance.py +++ b/tests/test_cross_module_inheritance.py @@ -32,7 +32,6 @@ def _exec_python_resp(code: str) -> LLMResponse: content="", tool_calls=[ToolCall(id="c1", name="execute_python", arguments=json.dumps({"code": code}))], finish_reason="tool_calls", - assistant_message={"role": "assistant", "content": ""}, ) diff --git a/tests/test_cross_session_tool_call_visibility.py b/tests/test_cross_session_tool_call_visibility.py index 6a5286181..7cb1d7c4c 100644 --- a/tests/test_cross_session_tool_call_visibility.py +++ b/tests/test_cross_session_tool_call_visibility.py @@ -48,7 +48,6 @@ def _resp(content: str = "", tool_calls: list[ToolCall] | None = None) -> LLMRes content=content, tool_calls=tool_calls or [], finish_reason=finish_reason, - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/test_event_auto_registration.py b/tests/test_event_auto_registration.py index 2521d3d23..ca201bc27 100644 --- a/tests/test_event_auto_registration.py +++ b/tests/test_event_auto_registration.py @@ -228,7 +228,7 @@ def test_nemo_events(self): BeforeTurn, Error, Feedback, - LLMOutput, + LLMResponse, Message, PythonOutput, Reasoning, @@ -241,7 +241,7 @@ def test_nemo_events(self): assert Reasoning(content="x").event_type == "Reasoning" assert Error(content="x").event_type == "Error" assert Feedback(content="x").event_type == "Feedback" - assert LLMOutput(content="x").event_type == "LLMOutput" + assert LLMResponse(content="x").event_type == "LLMResponse" assert ( PythonOutput( tool_call_id="t", execution_status="complete", execution_count=1 @@ -272,7 +272,7 @@ def test_core_events_in_registry(self): BeforeTurn, Error, Feedback, - LLMOutput, + LLMResponse, Message, PythonOutput, Reasoning, @@ -285,7 +285,7 @@ def test_core_events_in_registry(self): assert _EVENT_REGISTRY.get("Reasoning") is Reasoning assert _EVENT_REGISTRY.get("Error") is Error assert _EVENT_REGISTRY.get("Feedback") is Feedback - assert _EVENT_REGISTRY.get("LLMOutput") is LLMOutput + assert _EVENT_REGISTRY.get("LLMResponse") is LLMResponse assert _EVENT_REGISTRY.get("PythonOutput") is PythonOutput assert _EVENT_REGISTRY.get("Summary") is Summary assert _EVENT_REGISTRY.get("BeforeTurn") is BeforeTurn diff --git a/tests/test_event_backend_protocol.py b/tests/test_event_backend_protocol.py index e6fe613be..0ef62dab7 100644 --- a/tests/test_event_backend_protocol.py +++ b/tests/test_event_backend_protocol.py @@ -29,7 +29,7 @@ UserEvent, ) from nooa.context_blocks.models import Role -from nooa.events import LLMOutput, Task +from nooa.events import LLMResponse, Task from nooa.runtime.event_backend import InMemoryBackend from nooa.storage.sqlite import SQLiteEventBackend @@ -371,12 +371,12 @@ def test_tool_call_with_result_type_preserved(backend): def test_nemo_event_type_preserved(backend): - """nooa event types (Task, LLMOutput, etc.) must also round-trip.""" + """nooa event types (Task, LLMResponse, etc.) must also round-trip.""" backend.store("1", Task(prompt="do the thing")) - backend.store("2", LLMOutput(content="done")) + backend.store("2", LLMResponse(content="done")) events = list(backend.all_events()) assert type(events[0]) is Task - assert type(events[1]) is LLMOutput + assert type(events[1]) is LLMResponse def test_context_blocks_roles_correct_after_roundtrip(backend): diff --git a/tests/test_event_backend_roundtrip.py b/tests/test_event_backend_roundtrip.py index b1445e18f..222bddf96 100644 --- a/tests/test_event_backend_roundtrip.py +++ b/tests/test_event_backend_roundtrip.py @@ -25,13 +25,14 @@ from nooa.context_blocks import ResultStatus, ToolCallEvent, ToolResult from nooa.context_blocks.events import AssistantEvent, UserEvent -from nooa.context_blocks.models import Role +from nooa.context_blocks.formatter import XMLBlockFormatter +from nooa.context_blocks.models import ResolvedBlock, Role from nooa.events import ( AfterTurn, BeforeTurn, Error, Feedback, - LLMOutput, + LLMResponse, Message, PythonOutput, Reasoning, @@ -40,6 +41,7 @@ ) from nooa.runtime.event_backend import InMemoryBackend from nooa.storage.sqlite import SQLiteEventBackend +from nooa.unifiedllm import ToolCall # --------------------------------------------------------------------------- # Fixtures @@ -117,8 +119,8 @@ def backend(request, sqlite_conn): ), ( "9", - LLMOutput(content="result = compute()"), - LLMOutput, + LLMResponse(content="result = compute()"), + LLMResponse, Role.ASSISTANT, ), ( @@ -228,6 +230,106 @@ def test_event_roundtrip_via_all_events(backend, tag, event, expected_type, expe ) +@pytest.mark.parametrize( + "event", + [ + LLMResponse( + content="done", + tool_calls=( + ToolCall( + id="call-state", + name="execute_python", + arguments='{"code":"print(1)"}', + ), + ), + finish_reason="tool_calls", + reasoning="Check the inputs before running the tool.", + llm_state={"opaque": {"provider": "state"}}, + usage={ + "input_tokens": 100, + "output_tokens": 20, + "cached_input_tokens": 75, + }, + model_name="provider/model", + generation_id="generation-1", + ), + ToolCallEvent( + tool_call_id="tc-state", + name="execute_python", + arguments={"code": "print(1)"}, + llm_response_id="assistant-turn-id", + ), + ], + ids=["llm-output", "tool-call"], +) +def test_assistant_turn_ir_survives_backend_roundtrip(backend, event): + """Canonical turns and execution links survive session persistence.""" + backend.store("state", event) + + restored = backend.get("state") + + assert restored is not None + if isinstance(event, LLMResponse): + assert restored.tool_calls == event.tool_calls + assert restored.finish_reason == event.finish_reason + assert restored.reasoning == event.reasoning + assert restored.llm_state == event.llm_state + assert restored.usage == event.usage + assert restored.model_name == event.model_name + assert restored.generation_id == event.generation_id + else: + assert restored.llm_response_id == event.llm_response_id + + +def test_linked_assistant_turn_renders_after_backend_roundtrip(backend): + """A persisted canonical turn and its execution retain their relationship.""" + turn = LLMResponse( + content="I will run it.", + tool_calls=( + ToolCall( + id="call-roundtrip", + name="execute_python", + arguments='{"code":"print(1)"}', + ), + ), + finish_reason="tool_calls", + ) + execution = ToolCallEvent( + tool_call_id="call-roundtrip", + name="execute_python", + arguments={"code": "print(1)"}, + llm_response_id=turn.id, + result=ToolResult(tool_call_id="call-roundtrip", content="status: complete"), + ) + backend.store("turn", turn) + backend.store("execution", execution) + + restored_turn, restored_execution = backend.all_events() + messages = XMLBlockFormatter().format( + [ + ResolvedBlock( + key="turn", + content=restored_turn.content, + role=Role.ASSISTANT, + event=restored_turn, + ), + ResolvedBlock( + key="execution", + content="", + role=Role.ASSISTANT, + event=restored_execution, + ), + ] + ) + + assistant = next(message for message in messages if message.role == Role.ASSISTANT) + assert assistant.content == "I will run it." + assert [call.id for call in assistant.tool_calls] == ["call-roundtrip"] + result = next(message for message in messages if message.role == Role.TOOL) + assert result.tool_call_id == "call-roundtrip" + assert result.content == "status: complete" + + def test_tool_call_event_result_preserved_after_update(backend): """ToolCallEvent.result (added via update) survives a round-trip. diff --git a/tests/test_event_filtering.py b/tests/test_event_filtering.py index 14fe7e69a..0319e047c 100644 --- a/tests/test_event_filtering.py +++ b/tests/test_event_filtering.py @@ -149,7 +149,7 @@ def test_filter_by_call_id(): def test_filter_by_call_id_and_type(): """Test that call_id and type filters are ANDed together.""" - from nooa.events import LLMOutput, Task + from nooa.events import LLMResponse, Task events = EventManager() @@ -157,7 +157,7 @@ def test_filter_by_call_id_and_type(): t1.metadata["call_id"] = "call-1" events.add(t1) - llm1 = LLMOutput(content="LLM for call-1") + llm1 = LLMResponse(content="LLM for call-1") llm1.metadata["call_id"] = "call-1" events.add(llm1) @@ -171,7 +171,7 @@ def test_filter_by_call_id_and_type(): assert result[0].prompt == "Task for call-1" # LLM output for call-1 - result = events.filter(call_id="call-1", type="LLMOutput") + result = events.filter(call_id="call-1", type="LLMResponse") assert len(result) == 1 assert result[0].content == "LLM for call-1" diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index 304ba843e..df336cda2 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -4,7 +4,7 @@ import pytest -from nooa.events import Error, Feedback, LLMOutput, Task +from nooa.events import Error, Feedback, LLMResponse, Task from nooa.runtime.event_backend import InMemoryBackend, _tag_max_num from nooa.runtime.event_manager import EventManager @@ -49,7 +49,7 @@ def test_basic_conversation_flow(): hm.add(Task(prompt="Write a function to add two numbers")) # Add LLM response - hm.add(LLMOutput(content="I'll write that for you")) + hm.add(LLMResponse(content="I'll write that for you")) # Convert to OpenAI format via formatter messages = _format_events_for_test(hm.values()) @@ -69,13 +69,13 @@ def test_error_feedback_flow(): hm.add(Task(prompt="Generate code")) # Assistant response (with error) - hm.add(LLMOutput(content="def foo(): syntax error")) + hm.add(LLMResponse(content="def foo(): syntax error")) # Error feedback hm.add(Error(content="SyntaxError: invalid syntax")) # Retry response - hm.add(LLMOutput(content="def foo(): pass")) + hm.add(LLMResponse(content="def foo(): pass")) messages = _format_events_for_test(hm.values()) assert len(messages) == 4 @@ -91,7 +91,7 @@ def test_execution_feedback_flow(): hm.add(Task(prompt="Solve the problem")) # Assistant code - hm.add(LLMOutput(content="print('exploring')")) + hm.add(LLMResponse(content="print('exploring')")) # Execution feedback hm.add(Feedback(content="Output:\n```\nexploring\n```\nDefine `solve` to complete.")) @@ -138,7 +138,7 @@ def test_len(): hm.add(Task(prompt="One")) assert len(hm) == 1 - hm.add(LLMOutput(content="Two")) + hm.add(LLMResponse(content="Two")) assert len(hm) == 2 @@ -279,7 +279,7 @@ def test_filter_by_query_basic(): hm = EventManager() hm.add(Task(prompt="Find the database schema")) hm.add(Task(prompt="Query the user table")) - hm.add(LLMOutput(content="Here is the schema information")) + hm.add(LLMResponse(content="Here is the schema information")) # Filter for "schema" should return 2 events results = hm.filter(query="schema") diff --git a/tests/test_event_manager_integration.py b/tests/test_event_manager_integration.py index 31e965edb..bf452c1e5 100644 --- a/tests/test_event_manager_integration.py +++ b/tests/test_event_manager_integration.py @@ -3,7 +3,7 @@ """Integration test for event management with Event-based API.""" from nooa import Agent, strategy -from nooa.events import LLMOutput, Task +from nooa.events import LLMResponse, Task from nooa.strategies.pure_python import PurePythonStrategy from nooa.unifiedllm import FakeLLMClient @@ -75,7 +75,7 @@ class SimpleAgent(Agent, llm=_TEST_LLM): hm.add(Task(prompt="Test task")) assert len(hm) == 1 - hm.add(LLMOutput(content="Test response")) + hm.add(LLMResponse(content="Test response")) assert len(hm) == 2 # Convert to OpenAI format via formatter diff --git a/tests/test_event_types.py b/tests/test_event_types.py index 56dbea50a..8ae13a83b 100644 --- a/tests/test_event_types.py +++ b/tests/test_event_types.py @@ -5,7 +5,7 @@ from nooa.events import ( Error, Feedback, - LLMOutput, + LLMResponse, Message, Reasoning, Task, @@ -46,10 +46,10 @@ def test_feedback_event(self): assert event.event_type == "Feedback" assert event.content == "Code executed. Output: 42" - def test_llm_output_event(self): - """LLMOutput for LLM responses.""" - event = LLMOutput(content="def foo(): pass") - assert event.event_type == "LLMOutput" + def test_llm_response_event(self): + """LLMResponse for LLM responses.""" + event = LLMResponse(content="def foo(): pass") + assert event.event_type == "LLMResponse" assert event.content == "def foo(): pass" def test_event_serialization(self): @@ -99,8 +99,8 @@ def test_feedback_event_alias(self): assert event.event_type == "Feedback" def test_assistant_event_alias(self): - event = LLMOutput(content="test") - assert event.event_type == "LLMOutput" + event = LLMResponse(content="test") + assert event.event_type == "LLMResponse" class TestEventManagerEventAPI: @@ -117,16 +117,16 @@ def test_add_task_event(self): assert em.values()[0].prompt == "Do something" assert em.values()[0].event_type == "Task" - def test_add_llm_output_event(self): - """add() accepts LLMOutput.""" + def test_add_llm_response_event(self): + """add() accepts LLMResponse.""" em = EventManager() - event = LLMOutput(content="def foo(): pass") + event = LLMResponse(content="def foo(): pass") em.add(event) assert len(em) == 1 assert em.values()[0].content == "def foo(): pass" - assert em.values()[0].event_type == "LLMOutput" + assert em.values()[0].event_type == "LLMResponse" def test_add_error_event(self): """add() accepts Error.""" diff --git a/tests/test_events.py b/tests/test_events.py index fb26e0eb5..360b5b862 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -72,16 +72,16 @@ def test_feedback_event(self): event2 = Feedback(content="Test") assert event2.event_type == "Feedback" - def test_llm_output_event(self): - """LLMOutput for LLM responses.""" - from nooa.events import LLMOutput + def test_llm_response_event(self): + """LLMResponse for LLM responses.""" + from nooa.events import LLMResponse - event = LLMOutput(content="def foo(): pass") - assert event.event_type == "LLMOutput" + event = LLMResponse(content="def foo(): pass") + assert event.event_type == "LLMResponse" assert event.content == "def foo(): pass" # Verify event_type - event2 = LLMOutput(content="Test") - assert event2.event_type == "LLMOutput" + event2 = LLMResponse(content="Test") + assert event2.event_type == "LLMResponse" def test_tag_property_returns_event_position(self): """tag property returns event position (set by EventManager).""" diff --git a/tests/test_generate_tools_forwarding.py b/tests/test_generate_tools_forwarding.py index 80a7868ca..d2a327b14 100644 --- a/tests/test_generate_tools_forwarding.py +++ b/tests/test_generate_tools_forwarding.py @@ -29,7 +29,6 @@ def _resp(content: Any) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": str(content)}, ) diff --git a/tests/test_harness_metrics.py b/tests/test_harness_metrics.py index 7f40d7070..1816155a1 100644 --- a/tests/test_harness_metrics.py +++ b/tests/test_harness_metrics.py @@ -82,11 +82,6 @@ def test_text_to_synthetic(self): m.text_to_synthetic() assert m.text_to_synthetic_count == 2 - def test_content_prepended_as_comment(self): - m = HarnessMetrics() - m.content_prepended_as_comment() - assert m.content_prepended_as_comment_count == 1 - def test_empty_response(self): m = HarnessMetrics() m.empty_response() @@ -373,7 +368,6 @@ def _populate_all_fields(m: HarnessMetrics) -> None: m.import_stripped("import x") m.blocked_module_removed("os") m.text_to_synthetic() - m.content_prepended_as_comment() m.empty_response() m.gpt4o_double_quote_fix('"x\\n"') m.variable_ref_resolved("x") diff --git a/tests/test_initial_llm_messages_no_errors.py b/tests/test_initial_llm_messages_no_errors.py index 2fe8e7a73..fab730c00 100644 --- a/tests/test_initial_llm_messages_no_errors.py +++ b/tests/test_initial_llm_messages_no_errors.py @@ -96,7 +96,6 @@ async def test_first_llm_call_messages_contain_no_errors(self): content="", tool_calls=[_return_result_tool(42)], finish_reason="tool_calls", - assistant_message={"role": "assistant", "content": ""}, ), ] ) diff --git a/tests/test_llm.py b/tests/test_llm.py index 234f48d33..093ad5d0d 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -92,14 +92,12 @@ async def test_fake_llm_custom_responses(): content="First response", tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": "First response"}, ), LLMResponse( raw_response=None, content="Second response", tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": "Second response"}, ), ] @@ -126,21 +124,18 @@ async def test_fake_llm_multiple_calls(): content="Response 1", tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": "Response 1"}, ), LLMResponse( raw_response=None, content="Response 2", tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": "Response 2"}, ), LLMResponse( raw_response=None, content="Response 3", tool_calls=[ToolCall(id="1", name="test_tool", arguments=json.dumps({}))], finish_reason="tool_calls", - assistant_message={"role": "assistant", "content": "Response 3"}, ), ] ) diff --git a/tests/test_metaclass.py b/tests/test_metaclass.py index 520b151e9..c4639d0ee 100644 --- a/tests/test_metaclass.py +++ b/tests/test_metaclass.py @@ -909,7 +909,6 @@ def _resp(content): content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) mock_hooks = MagicMock(spec=InstrumentationHooks) @@ -947,7 +946,6 @@ def _resp(content): content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) mock_hooks = MagicMock(spec=InstrumentationHooks) @@ -985,7 +983,6 @@ def _resp(content): content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) mock_hooks = MagicMock(spec=InstrumentationHooks) diff --git a/tests/test_method_call_permutations.py b/tests/test_method_call_permutations.py index d62dd53c1..92f46e6b9 100644 --- a/tests/test_method_call_permutations.py +++ b/tests/test_method_call_permutations.py @@ -24,7 +24,6 @@ def _resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/test_method_llm_callable.py b/tests/test_method_llm_callable.py index 42652ad66..e16e411bc 100644 --- a/tests/test_method_llm_callable.py +++ b/tests/test_method_llm_callable.py @@ -25,7 +25,6 @@ def _response(answer: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/test_middleware_integration.py b/tests/test_middleware_integration.py index a35972293..821bff0b1 100644 --- a/tests/test_middleware_integration.py +++ b/tests/test_middleware_integration.py @@ -122,7 +122,6 @@ async def fake_response(ctx, nxt): content="faked", tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": "faked"}, reasoning=None, usage=None, ) @@ -607,7 +606,6 @@ def _resp(tc_list): content="", tool_calls=tc_list, finish_reason="tool_calls", - assistant_message={"role": "assistant", "content": ""}, ) # outer: exec code that calls inner, then return @@ -709,7 +707,6 @@ async def mw(ctx: AgentCallContext, nxt): ) ], finish_reason="tool_calls", - assistant_message={"role": "assistant", "content": ""}, ), ] ) @@ -752,7 +749,6 @@ async def spy(ctx: AgentCallContext, nxt): ) ], finish_reason="tool_calls", - assistant_message={"role": "assistant", "content": ""}, ), ] ) @@ -798,7 +794,6 @@ async def spy(ctx: AgentCallContext, nxt): ) ], finish_reason="tool_calls", - assistant_message={"role": "assistant", "content": ""}, ), ] ) @@ -895,7 +890,6 @@ async def exec_mw(ctx, nxt): ) ], finish_reason="tool_calls", - assistant_message={"role": "assistant", "content": ""}, ), LLMResponse( raw_response=None, @@ -908,7 +902,6 @@ async def exec_mw(ctx, nxt): ) ], finish_reason="tool_calls", - assistant_message={"role": "assistant", "content": ""}, ), ] ) diff --git a/tests/test_module_imports.py b/tests/test_module_imports.py index 05aa9fed0..424f8e1ba 100644 --- a/tests/test_module_imports.py +++ b/tests/test_module_imports.py @@ -20,7 +20,6 @@ def _resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/test_nemo_relay_middleware_no_dep.py b/tests/test_nemo_relay_middleware_no_dep.py index b73ab93c7..da4f0d3a7 100644 --- a/tests/test_nemo_relay_middleware_no_dep.py +++ b/tests/test_nemo_relay_middleware_no_dep.py @@ -12,6 +12,62 @@ import pytest import nooa.nemo_relay_middleware as nm +from nooa.unifiedllm import LLMResponse, LLMUsage, ToolCall + + +def test_canonical_response_projects_to_relay_shape_without_private_state(): + response = LLMResponse( + raw_response=object(), + content="hello", + parsed={"value": 42}, + tool_calls=[ToolCall(id="call-1", name="search", arguments='{"q":"x"}')], + finish_reason="tool_calls", + reasoning="plain reasoning", + llm_state={"opaque": "provider-only"}, + usage=LLMUsage(input_tokens=12, output_tokens=3, cached_input_tokens=8), + ) + + payload = nm._relay_response(response) + + assert payload["message"] == { + "role": "assistant", + "content": "hello", + "reasoning_content": "plain reasoning", + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": {"name": "search", "arguments": '{"q":"x"}'}, + } + ], + } + assert payload["usage"]["cached_tokens"] == 8 + assert "llm_state" not in payload + assert "raw_response" not in payload + assert "parsed" not in payload + + +def test_canonical_response_never_exposes_private_raw_response_to_relay(): + raw = MagicMock() + raw.model_dump.return_value = {"encrypted_content": "provider-secret"} + response = LLMResponse( + raw_response=raw, + content="public answer", + llm_state={"encrypted_content": "provider-secret"}, + ) + + payload = nm._response_for_relay(response) + + assert payload["message"]["content"] == "public answer" + assert "provider-secret" not in repr(payload) + raw.model_dump.assert_not_called() + + +def test_state_only_response_does_not_create_a_relay_message(): + payload = nm._relay_response(LLMResponse(llm_state={"opaque": "provider-only"})) + + assert "message" not in payload + assert "provider-only" not in repr(payload) @pytest.fixture() diff --git a/tests/test_nested_debug.py b/tests/test_nested_debug.py index ccba85e6c..ba82213f1 100644 --- a/tests/test_nested_debug.py +++ b/tests/test_nested_debug.py @@ -18,7 +18,6 @@ def _resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/test_pure_python_method_rejection.py b/tests/test_pure_python_method_rejection.py index 7e0c435f1..8f03ace16 100644 --- a/tests/test_pure_python_method_rejection.py +++ b/tests/test_pure_python_method_rejection.py @@ -138,14 +138,14 @@ async def process_data(self, items: list[str]) -> list[str]: @pytest.mark.asyncio async def test_function_body_extraction_when_wrapped_in_function_definition(): """Test that when LLM returns code wrapped in function definition matching target method, - we extract the body and execute it, updating history to show unpacked code. + we extract the body for execution while retaining the exact provider turn. Scenario: 1. LLM returns code wrapped in function definition: `async def process(self): ...` 2. Function name matches target method name (`process`) 3. No other top-level code exists 4. System extracts function body and executes it - 5. History is updated to show unpacked code so LLM learns from example + 5. History preserves the exact provider response """ class TestAgent(Agent): @@ -167,17 +167,15 @@ async def process(self) -> str: # Should successfully execute the extracted body assert result == "Hello, world!" - # Verify history was updated with unpacked code + # The canonical assistant turn remains byte-for-byte provider output. history_events = agent_instance.event_manager.values() - assistant_events = [e for e in history_events if e.event_type == "LLMOutput"] + assistant_events = [e for e in history_events if e.event_type == "LLMResponse"] # Should have at least one assistant event assert len(assistant_events) >= 1 - # The unpacked code should be in the history (without function definition wrapper) last_assistant_msg = assistant_events[-1].content - assert "async def process" not in last_assistant_msg - assert "return" in last_assistant_msg and "Hello, world!" in last_assistant_msg + assert last_assistant_msg == wrapped_code @pytest.mark.asyncio @@ -227,19 +225,15 @@ async def find_negative_sentiment(self) -> str: # helpers are plain callables, not attached to the agent. assert not hasattr(agent_instance, "is_negative") - # Verify history was updated with unpacked code + # The canonical assistant turn remains byte-for-byte provider output. history_events = agent_instance.event_manager.values() - assistant_events = [e for e in history_events if e.event_type == "LLMOutput"] + assistant_events = [e for e in history_events if e.event_type == "LLMResponse"] # Should have at least one assistant event assert len(assistant_events) >= 1 - # The unpacked code should be in the history (target method body unwrapped) last_assistant_msg = assistant_events[-1].content - assert "async def find_negative_sentiment" not in last_assistant_msg - assert "for sentence in" in last_assistant_msg or "return" in last_assistant_msg - # Helper method definition should still be there - assert "async def is_negative" in last_assistant_msg + assert last_assistant_msg == wrapped_code @pytest.mark.asyncio diff --git a/tests/test_sqlite_specific.py b/tests/test_sqlite_specific.py index 680f19fed..a264d7c0e 100644 --- a/tests/test_sqlite_specific.py +++ b/tests/test_sqlite_specific.py @@ -9,12 +9,14 @@ - register_event_type() overwrite warning """ +import json import logging from typing import Literal from nooa.context_blocks import EventBase, Metadata from nooa.context_blocks.events import AssistantEvent, ToolCallEvent, UserEvent from nooa.storage.sqlite import _CONTEXT_BLOCKS_TYPES, SQLiteEventBackend +from nooa.unifiedllm import LLMResponse # --------------------------------------------------------------------------- # _CONTEXT_BLOCKS_TYPES sanity @@ -44,10 +46,62 @@ def test_context_blocks_types_all_are_event_base_subclasses(): # --------------------------------------------------------------------------- +def test_deserialize_legacy_llm_output_as_canonical_response(sqlite_conn): + """Existing sessions retain assistant text after the event migration.""" + backend = SQLiteEventBackend(sqlite_conn) + legacy = { + "event_type": "LLMOutput", + "id": "legacy-response", + "metadata": {}, + "status": "active", + "tag": "1", + "timestamp": "2025-01-01T00:00:00", + "content": "saved assistant text", + } + + response = backend._deserialize(json.dumps(legacy)) + + assert isinstance(response, LLMResponse) + assert response.event_type == "LLMResponse" + assert response.content == "saved assistant text" + + +def test_archived_legacy_llm_output_migrates_without_becoming_active(sqlite_conn): + """Archived session history is upgraded on read without changing its status.""" + backend = SQLiteEventBackend(sqlite_conn) + legacy = { + "event_type": "LLMOutput", + "id": "archived-legacy-response", + "metadata": {}, + "status": "archived", + "tag": "4", + "timestamp": "2025-01-01T00:00:00", + "content": "archived assistant text", + } + sqlite_conn.execute( + "INSERT INTO events (tag, event_id, event_type, status, data, insertion_order) " + "VALUES (?, ?, ?, ?, ?, ?)", + ( + "4", + "archived-legacy-response", + "LLMOutput", + "archived", + json.dumps(legacy), + 0, + ), + ) + sqlite_conn.commit() + + response = backend.get("4") + + assert isinstance(response, LLMResponse) + assert response.content == "archived assistant text" + assert response.status.value == "archived" + assert backend.active_tags() == [] + + def test_deserialize_unknown_event_type_falls_back_to_metadata(sqlite_conn): """An event_type not in the registry must deserialize as Metadata, not raise.""" - import json - backend = SQLiteEventBackend(sqlite_conn) # Insert a row with an unrecognised event_type directly @@ -77,8 +131,6 @@ def test_deserialize_unknown_event_type_falls_back_to_metadata(sqlite_conn): def test_deserialize_unknown_type_logs_warning(sqlite_conn, caplog): """_deserialize() must log a warning when falling back to Metadata.""" - import json - backend = SQLiteEventBackend(sqlite_conn) unknown_json = json.dumps( diff --git a/tests/test_standalone.py b/tests/test_standalone.py index 675aef22a..abb9e85b4 100644 --- a/tests/test_standalone.py +++ b/tests/test_standalone.py @@ -85,7 +85,6 @@ def _resp(content: str = "", tool_calls: list | None = None) -> LLMResponse: content=content, tool_calls=tool_calls or [], finish_reason=finish_reason, - assistant_message={"role": "assistant", "content": content}, ) diff --git a/tests/test_token_calibration.py b/tests/test_token_calibration.py index c434126f7..f47013949 100644 --- a/tests/test_token_calibration.py +++ b/tests/test_token_calibration.py @@ -4,6 +4,7 @@ import pytest +from nooa.llm_types import LLMUsage from nooa.unifiedllm.unifiedllm import ( TokenCalibration, _token_calibration, @@ -103,9 +104,7 @@ def test_updates_from_messages(self): {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Hello, how are you?"}, ] - _update_token_calibration( - "gpt-4o", messages, {"prompt_tokens": 50, "completion_tokens": 10} - ) + _update_token_calibration("gpt-4o", messages, LLMUsage(input_tokens=50, output_tokens=10)) ratio = _token_calibration.ratio("gpt-4o") # API reports 50 prompt tokens; litellm raw estimate is ~9 tokens for this text # Ratio should be significantly > 1.0 (50/9 ≈ 5.6) @@ -114,7 +113,7 @@ def test_updates_from_messages(self): def test_skips_empty_usage(self): messages = [{"role": "user", "content": "hello"}] - _update_token_calibration("gpt-4o", messages, {"prompt_tokens": 0}) + _update_token_calibration("gpt-4o", messages, LLMUsage(input_tokens=0)) assert _token_calibration.ratio("gpt-4o") == 1.0 # unchanged def test_handles_multipart_content(self): @@ -127,6 +126,6 @@ def test_handles_multipart_content(self): ], } ] - _update_token_calibration("gpt-4o", messages, {"prompt_tokens": 30, "completion_tokens": 5}) + _update_token_calibration("gpt-4o", messages, LLMUsage(input_tokens=30, output_tokens=5)) ratio = _token_calibration.ratio("gpt-4o") assert ratio > 0 # didn't crash on multipart diff --git a/tests/tracing/test_openinference_conformance.py b/tests/tracing/test_openinference_conformance.py index 938b416f1..5dffc1a7e 100644 --- a/tests/tracing/test_openinference_conformance.py +++ b/tests/tracing/test_openinference_conformance.py @@ -73,7 +73,6 @@ async def acall(self, messages, tools=None, **kwargs): ), ], finish_reason="tool_calls", - assistant_message={}, ) diff --git a/tests/unifiedllm/test_empty_content_retry.py b/tests/unifiedllm/test_empty_content_retry.py index a8e6638c6..a60fa2328 100644 --- a/tests/unifiedllm/test_empty_content_retry.py +++ b/tests/unifiedllm/test_empty_content_retry.py @@ -251,10 +251,8 @@ async def test_tool_calls_bypass_retry(self, client_with_retry): assert mock_acompletion.call_count == 1 # No retry for tool calls @pytest.mark.asyncio - async def test_tool_calls_with_none_content_stores_empty_string_in_assistant_message( - self, client_with_retry - ): - """When API returns tool_calls with message.content None, assistant_message['content'] must be ''.""" + async def test_tool_calls_with_none_content_normalizes_public_content(self, client_with_retry): + """Tool-call-only responses normalize provider null content to empty text.""" mock_tool_call = make_tool_call( id="call_123", name="test_function", arguments='{"arg": "value"}' ) @@ -267,7 +265,6 @@ async def test_tool_calls_with_none_content_stores_empty_string_in_assistant_mes response = await client_with_retry.acall([{"role": "user", "content": "Hi"}]) assert response.finish_reason == "tool_calls" - assert response.assistant_message["content"] == "" assert response.content == "" @@ -329,10 +326,14 @@ async def test_async_output_model_falls_back_to_reasoning(self, client): output_model=SentimentResponse, ) - assert isinstance(response.content, SentimentResponse) - assert response.content.value == "positive" - # When reasoning was consumed as content, it should be cleared - assert response.reasoning is None + assert response.content == "" + assert isinstance(response.parsed, SentimentResponse) + assert response.parsed.value == "positive" + assert response.reasoning == '{"value": "positive"}' + restored = type(response).model_validate_json(response.model_dump_json()) + assert restored.content == "" + assert restored.parsed is None + assert restored.reasoning == '{"value": "positive"}' assert mock_acompletion.call_count == 1 def test_sync_output_model_falls_back_to_reasoning(self, client): @@ -347,9 +348,10 @@ def test_sync_output_model_falls_back_to_reasoning(self, client): output_model=SentimentResponse, ) - assert isinstance(response.content, SentimentResponse) - assert response.content.value == "negative" - assert response.reasoning is None + assert response.content == "" + assert isinstance(response.parsed, SentimentResponse) + assert response.parsed.value == "negative" + assert response.reasoning == '{"value": "negative"}' @pytest.mark.asyncio async def test_output_model_prefers_content_over_reasoning(self, client): @@ -367,7 +369,8 @@ async def test_output_model_prefers_content_over_reasoning(self, client): output_model=SentimentResponse, ) - assert response.content.value == "from_content" + assert isinstance(response.parsed, SentimentResponse) + assert response.parsed.value == "from_content" # Reasoning is preserved when content was used assert response.reasoning == '{"value": "from_reasoning"}' @@ -403,7 +406,8 @@ async def test_output_model_reasoning_with_whitespace_json(self, client): output_model=SentimentResponse, ) - assert response.content.value == "neutral" + assert isinstance(response.parsed, SentimentResponse) + assert response.parsed.value == "neutral" @pytest.mark.asyncio async def test_non_output_model_does_not_use_reasoning(self, client): diff --git a/tests/unifiedllm/test_finish_reason_propagation.py b/tests/unifiedllm/test_finish_reason_propagation.py index 785cc1960..5c1b4a3be 100644 --- a/tests/unifiedllm/test_finish_reason_propagation.py +++ b/tests/unifiedllm/test_finish_reason_propagation.py @@ -190,6 +190,21 @@ async def test_async_length_takes_precedence_over_xml_tool_fallback(self, client assert out.finish_reason == "length" assert len(out.tool_calls) == 1 + def test_sync_tool_call_preserves_accompanying_text(self, client): + tc = make_tool_call("call_1", "do_thing", "{}") + resp = make_mock_response(content="I will use the tool.", tool_calls=[tc]) + with patch("litellm.completion", return_value=resp): + out = client.call([{"role": "user", "content": "Hi"}]) + assert out.content == "I will use the tool." + + @pytest.mark.asyncio + async def test_async_tool_call_preserves_accompanying_text(self, client): + tc = make_tool_call("call_1", "do_thing", "{}") + resp = make_mock_response(content="I will use the tool.", tool_calls=[tc]) + with patch("litellm.acompletion", new_callable=AsyncMock, return_value=resp): + out = await client.acall([{"role": "user", "content": "Hi"}]) + assert out.content == "I will use the tool." + def _make_responses_api_response(status: str, reason: str | None = None): """Fake a litellm Responses-API response with a given status/reason.""" @@ -218,6 +233,23 @@ def _make_incomplete_responses_tool_response(): ) +def _make_responses_tool_response(text: str): + return SimpleNamespace( + output=[ + SimpleNamespace(type="message", content=[SimpleNamespace(text=text)]), + SimpleNamespace( + type="function_call", + call_id="call_1", + name="do_thing", + arguments="{}", + ), + ], + usage=None, + status="completed", + incomplete_details=None, + ) + + class TestResponsesClientPropagation: """The real ResponsesClient return path surfaces the derived finish_reason.""" @@ -265,6 +297,19 @@ async def test_async_length_takes_precedence_over_parsed_tool_calls(self, client assert out.finish_reason == "length" assert len(out.tool_calls) == 1 + def test_sync_tool_call_preserves_accompanying_text(self, client): + resp = _make_responses_tool_response("I will use the tool.") + with patch("litellm.responses", return_value=resp): + out = client.call([{"role": "user", "content": "Hi"}]) + assert out.content == "I will use the tool." + + @pytest.mark.asyncio + async def test_async_tool_call_preserves_accompanying_text(self, client): + resp = _make_responses_tool_response("I will use the tool.") + with patch("litellm.aresponses", new_callable=AsyncMock, return_value=resp): + out = await client.acall([{"role": "user", "content": "Hi"}]) + assert out.content == "I will use the tool." + class TestCodeActAbortOnRealLengthPath: """End-to-end: a real CompletionClient truncation triggers CodeAct's abort.""" @@ -321,3 +366,38 @@ async def my_task(self) -> str: assert all( event.event_type != "ToolCallEvent" for event in agent_instance.event_manager.values() ) + + @pytest.mark.asyncio + async def test_real_client_preserves_tool_turn_without_mutating_execution(self): + first_response = make_mock_response( + content="I will calculate this.", + tool_calls=[make_tool_call("call_1", "execute_python", '{"code":"x = 42"}')], + ) + final_response = make_mock_response( + tool_calls=[make_tool_call("call_2", "return_result", '{"result":"done"}')] + ) + real_llm = CompletionClient(model="test-model") + + class TestAgent(Agent, llm=real_llm): + @strategy(CodeActStrategy(config=CodeActConfig(max_retries=3, max_iterations=10))) + async def my_task(self) -> str: + """A task.""" + ... + + agent_instance = TestAgent(llm=real_llm) + with patch( + "litellm.acompletion", + new_callable=AsyncMock, + side_effect=[first_response, final_response], + ): + assert await agent_instance.my_task() == "done" + + events = agent_instance.event_manager.values() + first_output = next(event for event in events if event.event_type == "LLMResponse") + execution = next( + event + for event in events + if event.event_type == "ToolCallEvent" and event.name == "execute_python" + ) + assert first_output.content == "I will calculate this." + assert execution.arguments == {"code": "x = 42"} diff --git a/tests/unifiedllm/test_litellm_responses_bridge.py b/tests/unifiedllm/test_litellm_responses_bridge.py index 596034b4e..fb1a32109 100644 --- a/tests/unifiedllm/test_litellm_responses_bridge.py +++ b/tests/unifiedllm/test_litellm_responses_bridge.py @@ -95,7 +95,7 @@ async def test_default_reasoning_tool_call_uses_responses_bridge(model: str) -> responses.assert_called_once() assert result.finish_reason == "tool_calls" - assert result.assistant_message["reasoning_items"] == [REASONING_ITEM] + assert result.llm_state == {"reasoning_items": [REASONING_ITEM]} finally: await client.aclose() @@ -136,48 +136,3 @@ def test_custom_api_base_keeps_chat_completions_by_default() -> None: assert result.content == "done" finally: client.close() - - -def test_reasoning_items_round_trip_into_next_responses_request() -> None: - client = _client() - try: - with ( - patch( - "litellm.responses", - side_effect=[_responses_tool_call(), _responses_tool_call()], - ) as responses, - patch( - "litellm.main._complete_custom_openai", - side_effect=AssertionError("chat endpoint should not be used"), - ), - ): - first = client.call( - messages=[{"role": "user", "content": "Run Python."}], - tools=[TOOL], - ) - client.call( - messages=[ - {"role": "user", "content": "Run Python."}, - first.assistant_message, - { - "role": "tool", - "tool_call_id": first.tool_calls[0].id, - "content": "1", - }, - ], - tools=[TOOL], - ) - - second_input = responses.call_args_list[1].kwargs["input"] - reasoning_index = second_input.index(REASONING_ITEM) - function_call_index = next( - index for index, item in enumerate(second_input) if item.get("type") == "function_call" - ) - output_index = next( - index - for index, item in enumerate(second_input) - if item.get("type") == "function_call_output" - ) - assert reasoning_index < function_call_index < output_index - finally: - client.close() diff --git a/tests/unifiedllm/test_llm_types.py b/tests/unifiedllm/test_llm_types.py new file mode 100644 index 000000000..589213b04 --- /dev/null +++ b/tests/unifiedllm/test_llm_types.py @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Canonical UnifiedLLM response and usage contracts.""" + +import pytest + +from nooa.unifiedllm import FakeLLMClient, LLMResponse, LLMUsage + + +@pytest.mark.parametrize( + ("provider_usage", "expected"), + [ + ( + { + "prompt_tokens": 100, + "completion_tokens": 20, + "prompt_tokens_details": {"cached_tokens": 75}, + "completion_tokens_details": {"reasoning_tokens": 8}, + "total_tokens": 120, + }, + LLMUsage( + input_tokens=100, + output_tokens=20, + cached_input_tokens=75, + reasoning_tokens=8, + total_tokens=120, + ), + ), + ( + { + "input_tokens": 90, + "output_tokens": 10, + "cache_read_input_tokens": 60, + "cache_creation_input_tokens": 15, + }, + LLMUsage( + input_tokens=90, + output_tokens=10, + cached_input_tokens=60, + cache_write_input_tokens=15, + total_tokens=100, + ), + ), + ], +) +def test_usage_normalizes_provider_cache_and_reasoning_fields(provider_usage, expected): + assert LLMUsage.from_provider(provider_usage) == expected + + +@pytest.mark.asyncio +async def test_fake_materializes_repeated_script_aliases_as_distinct_events(): + response = LLMResponse(content="same response") + fake = FakeLLMClient(scripted_responses=[response, response, response]) + + first = await fake.acall([]) + second = await fake.acall([]) + third = await fake.acall([]) + + assert first is response + assert second is not response + assert third is not response + assert len({first.id, second.id, third.id}) == 3 + assert [first.content, second.content, third.content] == ["same response"] * 3 diff --git a/tests/unifiedllm/test_plain_reasoning_replay.py b/tests/unifiedllm/test_plain_reasoning_replay.py new file mode 100644 index 000000000..dec13465c --- /dev/null +++ b/tests/unifiedllm/test_plain_reasoning_replay.py @@ -0,0 +1,89 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Provider-neutral replay of persisted plain reasoning.""" + +import json +from unittest.mock import patch + +from litellm.types.utils import Choices, Message, ModelResponse + +from nooa.context_blocks.formatter import ( + OpenAIProviderFormatter, + ResponsesProviderFormatter, + XMLBlockFormatter, +) +from nooa.context_blocks.models import ResolvedBlock, Role +from nooa.unifiedllm import CompletionClient, LLMResponse, ResponsesClient + + +def _render(response: LLMResponse, *, responses: bool = False) -> list[dict]: + neutral = XMLBlockFormatter().format( + [ResolvedBlock(key="turn", content="", role=Role.ASSISTANT, event=response)] + ) + formatter = ResponsesProviderFormatter() if responses else OpenAIProviderFormatter() + return formatter.format(neutral) + + +def _chat_response() -> ModelResponse: + return ModelResponse( + model="test-model", + choices=[Choices(message=Message(role="assistant", content="done"), finish_reason="stop")], + ) + + +def test_reasoning_backed_structured_output_replays_after_persistence() -> None: + response = LLMResponse( + content="", + parsed={"value": "positive"}, + reasoning='{"value":"positive"}', + ) + restored = LLMResponse.model_validate_json(response.model_dump_json()) + rendered = _render(restored) + + # Replay metadata is an in-memory sidecar, never a provider-visible key. + assert '{"value":"positive"}' not in json.dumps(rendered) + + client = CompletionClient(model="openai/gpt-4o") + try: + with patch("litellm.completion", return_value=_chat_response()) as completion: + client.call(rendered) + + assert restored.parsed is None + assert restored.content == "" + assert restored.reasoning == '{"value":"positive"}' + assistant = next( + message + for message in completion.call_args.kwargs["messages"] + if message.get("role") == "assistant" + ) + assert assistant == {"role": "assistant", "content": '{"value":"positive"}'} + finally: + client.close() + + +def test_reasoning_only_response_demotes_for_responses_api() -> None: + response = LLMResponse(content="", reasoning="portable thought") + client = ResponsesClient(model="openai/gpt-5") + try: + transformed, instructions = client._transform_messages(_render(response, responses=True)) + finally: + client.close() + + assert instructions is None + assert transformed == [{"role": "assistant", "content": "portable thought"}] + + +def test_opaque_only_response_is_withheld_without_a_provider_gate() -> None: + response = LLMResponse(content="", llm_state={"opaque": "provider state"}) + client = CompletionClient(model="openai/gpt-4o") + try: + with patch("litellm.completion", return_value=_chat_response()) as completion: + client.call(_render(response)) + + assert all( + message.get("role") != "assistant" + for message in completion.call_args.kwargs["messages"] + ) + assert "provider state" not in repr(completion.call_args.kwargs["messages"]) + finally: + client.close() diff --git a/tests/unifiedllm/test_reasoning_completion_identity.py b/tests/unifiedllm/test_reasoning_completion_identity.py new file mode 100644 index 000000000..13db54b95 --- /dev/null +++ b/tests/unifiedllm/test_reasoning_completion_identity.py @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Reasoning cleanup must not replace the canonical response object.""" + +from unittest.mock import AsyncMock, patch + +import pytest + +from nooa.unifiedllm import CompletionClient, LLMResponse, ReasoningCompletionClient + + +def _response() -> LLMResponse: + return LLMResponse( + content="new thoughtpublic answer", + parsed={"answer": 42}, + reasoning="provider thought", + llm_state={"opaque": "state"}, + usage={"input_tokens": 10, "output_tokens": 5}, + ) + + +def _assert_cleaned_in_place(response: LLMResponse, returned: LLMResponse) -> None: + assert returned is response + assert returned.content == "public answer" + assert returned.reasoning == "provider thought\n\nnew thought" + assert returned.parsed == {"answer": 42} + assert returned.llm_state == {"opaque": "state"} + assert returned.usage == response.usage + + +def test_sync_cleanup_mutates_only_reasoning_views() -> None: + response = _response() + identity = (response.id, response.timestamp) + client = ReasoningCompletionClient(model="test-model") + + with patch.object(CompletionClient, "call", return_value=response): + returned = client.call([{"role": "user", "content": "hi"}]) + + _assert_cleaned_in_place(response, returned) + assert (returned.id, returned.timestamp) == identity + client.close() + + +@pytest.mark.asyncio +async def test_async_cleanup_mutates_only_reasoning_views() -> None: + response = _response() + identity = (response.id, response.timestamp) + client = ReasoningCompletionClient(model="test-model") + + with patch.object(CompletionClient, "acall", new=AsyncMock(return_value=response)): + returned = await client.acall([{"role": "user", "content": "hi"}]) + + _assert_cleaned_in_place(response, returned) + assert (returned.id, returned.timestamp) == identity + await client.aclose() diff --git a/tests/unifiedllm/test_responses_client.py b/tests/unifiedllm/test_responses_client.py index 5d25031e0..4cbbd7969 100644 --- a/tests/unifiedllm/test_responses_client.py +++ b/tests/unifiedllm/test_responses_client.py @@ -82,9 +82,10 @@ def test_structured_output(self, client): ) assert isinstance(response, LLMResponse) - assert isinstance(response.content, SimpleResponse) - assert "Paris" in response.content.answer or "paris" in response.content.answer.lower() - assert 0 <= response.content.confidence <= 1 + assert isinstance(response.content, str) + assert isinstance(response.parsed, SimpleResponse) + assert "Paris" in response.parsed.answer or "paris" in response.parsed.answer.lower() + assert 0 <= response.parsed.confidence <= 1 def test_tool_calling(self, client): """Test that the model can call tools.""" @@ -143,8 +144,9 @@ async def test_async_structured_output(self, client): ) assert isinstance(response, LLMResponse) - assert isinstance(response.content, SimpleResponse) + assert isinstance(response.content, str) + assert isinstance(response.parsed, SimpleResponse) # Check that we got a valid structured response with non-empty answer # (LLM responses can be flaky, so we just verify structure, not exact content) - assert response.content.answer is not None - assert len(response.content.answer) > 0 + assert response.parsed.answer is not None + assert len(response.parsed.answer) > 0 diff --git a/tests/unifiedllm/test_responses_client_retry.py b/tests/unifiedllm/test_responses_client_retry.py index df9bf9cf5..3c0cc4514 100644 --- a/tests/unifiedllm/test_responses_client_retry.py +++ b/tests/unifiedllm/test_responses_client_retry.py @@ -101,6 +101,40 @@ def test_non_retryable_not_retried(self): client.call(messages=[{"role": "user", "content": "hi"}]) assert mock_responses.call_count == 1 + def test_reasoning_state_retains_interleaving_without_copying_public_calls(self): + """The canonical state has enough anchors for exact ordered replay.""" + reasoning_1 = MagicMock(type="reasoning") + reasoning_1.model_dump.return_value = {"type": "reasoning", "encrypted": "one"} + call_1 = MagicMock(type="function_call", call_id="call-1", arguments="{}") + call_1.name = "one" + reasoning_2 = MagicMock(type="reasoning") + reasoning_2.model_dump.return_value = {"type": "reasoning", "encrypted": "two"} + call_2 = MagicMock(type="function_call", call_id="call-2", arguments="{}") + call_2.name = "two" + raw_response = MagicMock( + output=[reasoning_1, call_1, reasoning_2, call_2], + output_text="", + usage=None, + ) + client = ResponsesClient(model="test-model", retry_config=NO_RETRY) + + with patch("litellm.responses", return_value=raw_response): + response = client.call(messages=[{"role": "user", "content": "hi"}]) + + assert [call.id for call in response.tool_calls] == ["call-1", "call-2"] + assert response.llm_state == { + "items": [ + {"type": "reasoning", "encrypted": "one"}, + {"type": "reasoning", "encrypted": "two"}, + ], + "order": [ + {"type": "reasoning", "index": 0}, + {"type": "function_call", "call_id": "call-1"}, + {"type": "reasoning", "index": 1}, + {"type": "function_call", "call_id": "call-2"}, + ], + } + class TestResponsesClientAsyncRetry: """Async acall() retry behaviour.""" diff --git a/tests/unifiedllm/test_responses_formatter.py b/tests/unifiedllm/test_responses_formatter.py index 8da04271b..d59bfced8 100644 --- a/tests/unifiedllm/test_responses_formatter.py +++ b/tests/unifiedllm/test_responses_formatter.py @@ -45,8 +45,10 @@ def test_tool_call_format(self): messages = [ RenderedMessage( role=Role.ASSISTANT, - tool_call=ToolCallInfo( - id="call_123", name="execute_python", arguments={"code": "print(1)"} + tool_calls=( + ToolCallInfo( + id="call_123", name="execute_python", arguments={"code": "print(1)"} + ), ), ), ] @@ -85,7 +87,9 @@ def test_multi_turn_tool_calling(self): RenderedMessage(role=Role.USER, content="Add 2+2"), RenderedMessage( role=Role.ASSISTANT, - tool_call=ToolCallInfo(id="tc_1", name="execute_python", arguments={"code": "2+2"}), + tool_calls=( + ToolCallInfo(id="tc_1", name="execute_python", arguments={"code": "2+2"}), + ), ), RenderedMessage(role=Role.TOOL, content="4", tool_call_id="tc_1"), RenderedMessage(role=Role.USER, content="Now multiply by 3"), diff --git a/tests/unit/test_actor_full_coverage.py b/tests/unit/test_actor_full_coverage.py index 24ff66efa..628b458da 100644 --- a/tests/unit/test_actor_full_coverage.py +++ b/tests/unit/test_actor_full_coverage.py @@ -43,7 +43,6 @@ def _resp(content: str, tool_calls: list | None = None) -> LLMResponse: content=content, tool_calls=tool_calls or [], finish_reason=finish_reason, - assistant_message={"role": "assistant", "content": content}, ) @@ -145,7 +144,6 @@ async def task(self) -> int: content=12345, # numeric, not str tool_calls=[_return_result(result=99)], finish_reason="tool_calls", - assistant_message={"role": "assistant", "content": "12345"}, ) fake_llm = FakeLLMClient(scripted_responses=[numeric_resp]) agent = _Agent(llm=fake_llm) diff --git a/tests/unit/test_context_vars_subagent_concurrency.py b/tests/unit/test_context_vars_subagent_concurrency.py index 095e5680d..56274437e 100644 --- a/tests/unit/test_context_vars_subagent_concurrency.py +++ b/tests/unit/test_context_vars_subagent_concurrency.py @@ -47,7 +47,6 @@ def _resp(content: str) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, ) @@ -671,7 +670,6 @@ def _codeact_resp(code: str) -> LLMResponse: content="", tool_calls=[_tool_call(code)], finish_reason="tool_calls", - assistant_message={"role": "assistant", "content": ""}, ) def _codeact_return(val) -> LLMResponse: @@ -680,7 +678,6 @@ def _codeact_return(val) -> LLMResponse: content="", tool_calls=[_return(val)], finish_reason="tool_calls", - assistant_message={"role": "assistant", "content": ""}, ) class _InnerAgent(Agent): diff --git a/tests/unit/test_coverage_gaps_v2.py b/tests/unit/test_coverage_gaps_v2.py index 9ab173da8..0bdfd0cdf 100644 --- a/tests/unit/test_coverage_gaps_v2.py +++ b/tests/unit/test_coverage_gaps_v2.py @@ -668,11 +668,11 @@ class TestPlainFormatterNoOutput: """format_event() returns '(no output)' when all fields are empty (line 50).""" def test_empty_event_returns_no_output(self): - from nooa.events import LLMOutput + from nooa.events import LLMResponse from nooa.plain_formatter import PlainBlockFormatter - # LLMOutput with empty content and no reasoning → all repr fields are empty/None - event = LLMOutput(content="", reasoning=None) + # LLMResponse with empty content and no reasoning → all repr fields are empty/None + event = LLMResponse(content="", reasoning=None) formatter = PlainBlockFormatter() result = formatter.format_event(event) assert result == "(no output)" diff --git a/tests/unit/test_strategy_full_coverage.py b/tests/unit/test_strategy_full_coverage.py index d7f2cee53..a4f19a37e 100644 --- a/tests/unit/test_strategy_full_coverage.py +++ b/tests/unit/test_strategy_full_coverage.py @@ -42,7 +42,6 @@ def _resp(content: str, tool_calls: list | None = None) -> LLMResponse: content=content, tool_calls=tool_calls or [], finish_reason=finish_reason, - assistant_message={"role": "assistant", "content": content}, ) @@ -69,7 +68,6 @@ def _llm_resp(content: str, reasoning: str | None = None) -> LLMResponse: content=content, tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": content}, reasoning=reasoning, ) @@ -731,7 +729,6 @@ async def mock_call_llm_raw(self_strat, runtime, response_model): content="not json at all!!!", tool_calls=[], finish_reason="stop", - assistant_message={"role": "assistant", "content": "not json"}, ) return resp, "evt_1" # Second call: correct response