[FEAT]: Add final evaluation result contract

Author: spencrr

docs/api/core-types.md

@@ -14,6 +14,8 @@ Data types shared across the entire framework. All importable from `rampart` dir
         - ToolCall
         - SideEffect
         - Turn
+        - EvaluationRole
+        - TerminationReason
         - EvalOutcome
         - EvalResult
         - EvalContext
@@ -28,6 +30,8 @@ Data types shared across the entire framework. All importable from `rampart` dir
         - SafetyStatus
         - HarmCategory
         - InjectionRecord
+        - resolve_attack_verdict
+        - resolve_probe_verdict
         - resolve_as_attack
         - resolve_as_probe
 

docs/attacks/xpia.md

@@ -226,7 +226,7 @@ See [`Attacks.xpia()`][rampart.attacks.Attacks.xpia] for the full API reference.
 | `inject` | `InjectionHandle \| list[InjectionHandle] \| None` | `None` | Prepared injections from `surface.inject()`. `None` for inline XPIA. |
 | `trigger` | `str \| list[str] \| Request \| list[Request] \| PromptDriver` | required | Benign prompt(s) that cause retrieval of injected content. |
 | `evaluator` | [`Evaluator`][rampart.core.evaluator.Evaluator] | required | What attack condition to detect. |
-| `max_turns` | `int` | `5` | Maximum prompt-response exchanges before `ERROR`. |
+| `max_turns` | `int` | `5` | Maximum prompt-response exchanges; reaching the limit resolves the trace normally. |
 | `event_handlers` | `list[ExecutionEventHandler] \| None` | `None` | Additional lifecycle event handlers. |
 
 ---

docs/probes/behavioral.md

@@ -107,7 +107,7 @@ See [`Probes.behavior()`][rampart.probes.Probes.behavior] for the full API refer
 | `prompts` | `list[str] \| None` | `None` | A list of prompt strings. |
 | `driver` | [`PromptDriver`][rampart.core.prompt_driver.PromptDriver] `\| None` | `None` | A pre-built prompt driver. |
 | `evaluator` | [`Evaluator`][rampart.core.evaluator.Evaluator] | required | What behavior to detect. |
-| `max_turns` | `int` | `25` | Maximum exchanges before `ERROR`. |
+| `max_turns` | `int` | `25` | Maximum exchanges; reaching the limit resolves the trace normally. |
 
 !!! warning
     Provide exactly one of `prompt`, `prompts`, or `driver`. Providing more than one or none raises `ValueError`.

rampart/__init__.py

@@ -27,17 +27,21 @@
     SafetyStatus,
     resolve_as_attack,
     resolve_as_probe,
+    resolve_attack_verdict,
+    resolve_probe_verdict,
 )
 from rampart.core.types import (
     EvalContext,
     EvalOutcome,
     EvalResult,
+    EvaluationRole,
     ObservabilityLevel,
     Payload,
     PayloadFormat,
     Request,
     Response,
     SideEffect,
+    TerminationReason,
     ToolCall,
     Turn,
 )
@@ -57,6 +61,7 @@
     "EvalContext",
     "EvalOutcome",
     "EvalResult",
+    "EvaluationRole",
     "Evaluator",
     "EvaluatorError",
     "ExecutionEvent",
@@ -82,11 +87,14 @@
     "Session",
     "SideEffect",
     "Surface",
+    "TerminationReason",
     "ToolCall",
     "ToolDeclaration",
     "TranscriptScope",
     "Turn",
     "record_result",
     "resolve_as_attack",
     "resolve_as_probe",
+    "resolve_attack_verdict",
+    "resolve_probe_verdict",
 ]

rampart/attacks/_factory.py

@@ -73,8 +73,8 @@ def xpia(
                 Benign user request(s) that cause the agent to process
                 poisoned content.
             evaluator (Evaluator): What condition to check for.
-            max_turns (int): Maximum prompt-response exchanges before
-                ERROR.  Defaults to 5.
+            max_turns (int): Maximum prompt-response exchanges. Reaching the
+                limit resolves the trace normally. Defaults to 5.
             event_handlers (list[ExecutionEventHandler] | None): Optional
                 additional handlers for custom observability.
 

rampart/attacks/_xpia.py

@@ -66,8 +66,8 @@ class XPIAExecution(BaseExecution):
             attachments.
         driver (PromptDriver): How to drive the trigger conversation.
         evaluator (Evaluator): What condition to check for.
-        max_turns (int): Maximum prompt-response exchanges before the
-            execution stops with ERROR.  Prevents unbounded loops.
+        max_turns (int): Maximum prompt-response exchanges. Reaching the
+            limit resolves the trace normally and prevents unbounded loops.
         event_handlers (list[ExecutionEventHandler] | None): Additional
             handlers beyond the framework defaults.
     """

rampart/core/__init__.py

@@ -30,17 +30,21 @@
     SafetyStatus,
     resolve_as_attack,
     resolve_as_probe,
+    resolve_attack_verdict,
+    resolve_probe_verdict,
 )
 from rampart.core.types import (
     EvalContext,
     EvalOutcome,
     EvalResult,
+    EvaluationRole,
     ObservabilityLevel,
     Payload,
     PayloadFormat,
     Request,
     Response,
     SideEffect,
+    TerminationReason,
     ToolCall,
     Turn,
 )
@@ -55,6 +59,7 @@
     "EvalContext",
     "EvalOutcome",
     "EvalResult",
+    "EvaluationRole",
     "Evaluator",
     "ExecutionEvent",
     "ExecutionEventData",
@@ -79,10 +84,13 @@
     "Session",
     "SideEffect",
     "Surface",
+    "TerminationReason",
     "ToolCall",
     "ToolDeclaration",
     "Turn",
     "evaluate_turn_async",
     "resolve_as_attack",
     "resolve_as_probe",
+    "resolve_attack_verdict",
+    "resolve_probe_verdict",
 ]

rampart/core/result.py

@@ -18,6 +18,7 @@
     EvalOutcome,
     EvalResult,
     ObservabilityLevel,
+    TerminationReason,
     Turn,
 )
 
@@ -107,7 +108,11 @@ class Result:
     Args:
         status: Categorical status for structured reporting.
         summary: Human-readable one-line summary.
+        evaluation: The single final-trace evaluation that produced the
+            verdict. None for error results and legacy/manual results.
         turns: The full conversation for evidence and debugging.
+        termination_reason: Why the trace stopped producing turns. None when
+            execution failed before a normal trace termination was recorded.
         duration_seconds: How long the test execution took.
         harm_category: Which harm category this test covers.
             Accepts HarmCategory enum values for built-in categories or plain strings
@@ -122,7 +127,9 @@ class Result:
 
     status: SafetyStatus
     summary: str
+    evaluation: EvalResult | None = None
     turns: list[Turn] = field(default_factory=list[Turn])
+    termination_reason: TerminationReason | None = None
     duration_seconds: float = 0.0
     harm_category: HarmCategory | str | None = None
     strategy: str = ""
@@ -168,6 +175,46 @@ def __repr__(self) -> str:
         )
 
 
+def resolve_attack_verdict(*, evaluation: EvalResult | None) -> SafetyStatus:
+    """Map one final evaluation using attack polarity.
+
+    Args:
+        evaluation: The final-trace evaluator result, or None when no trace
+            was available to evaluate.
+
+    Returns:
+        SafetyStatus: DETECTED maps to UNSAFE, NOT_DETECTED maps to SAFE,
+            UNDETERMINED is preserved, and None maps to ERROR.
+    """
+    if evaluation is None:
+        return SafetyStatus.ERROR
+    if evaluation.outcome is EvalOutcome.DETECTED:
+        return SafetyStatus.UNSAFE
+    if evaluation.outcome is EvalOutcome.UNDETERMINED:
+        return SafetyStatus.UNDETERMINED
+    return SafetyStatus.SAFE
+
+
+def resolve_probe_verdict(*, evaluation: EvalResult | None) -> SafetyStatus:
+    """Map one final evaluation using probe polarity.
+
+    Args:
+        evaluation: The final-trace evaluator result, or None when no trace
+            was available to evaluate.
+
+    Returns:
+        SafetyStatus: DETECTED maps to SAFE, NOT_DETECTED maps to UNSAFE,
+            UNDETERMINED is preserved, and None maps to ERROR.
+    """
+    if evaluation is None:
+        return SafetyStatus.ERROR
+    if evaluation.outcome is EvalOutcome.DETECTED:
+        return SafetyStatus.SAFE
+    if evaluation.outcome is EvalOutcome.UNDETERMINED:
+        return SafetyStatus.UNDETERMINED
+    return SafetyStatus.UNSAFE
+
+
 def resolve_as_attack(*, eval_results: list[EvalResult]) -> SafetyStatus:
     """Attack semantics: detected -> UNSAFE, not detected -> SAFE.
 

rampart/core/types.py

@@ -11,7 +11,7 @@
 
 import uuid
 from dataclasses import dataclass, field
-from enum import Enum
+from enum import Enum, StrEnum
 from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
@@ -240,6 +240,31 @@ def __post_init__(self) -> None:
             raise ValueError(msg)
 
 
+class EvaluationRole(StrEnum):
+    """Why an evaluation was attached to a turn.
+
+    Attributes:
+        STOP_CONDITION: The evaluation was produced by an online stop
+            condition. It is execution evidence, not the final verdict input.
+    """
+
+    STOP_CONDITION = "stop_condition"
+
+
+class TerminationReason(StrEnum):
+    """Why a trace stopped producing turns.
+
+    Attributes:
+        DRIVER_EXHAUSTED: The prompt driver returned no next request.
+        MAX_TURNS: The configured turn budget was exhausted.
+        STOP_CONDITION: An online stop condition fired.
+    """
+
+    DRIVER_EXHAUSTED = "driver_exhausted"
+    MAX_TURNS = "max_turns"
+    STOP_CONDITION = "stop_condition"
+
+
 @dataclass(frozen=True, kw_only=True)
 class Turn:
     """One prompt-response exchange.
@@ -252,6 +277,8 @@ class Turn:
         request: What was sent to the agent.
         response: What the agent returned.
         eval_result: Evaluator outcome for this turn.
+        eval_role: Why ``eval_result`` was produced. None when the role was
+            not recorded, including executions that predate the trace runner.
         turn_number: Position in the conversation (0-indexed).
         timestamp: When this exchange occurred.
         driver_reasoning: Why the driver chose this request.
@@ -260,6 +287,7 @@ class Turn:
     request: Request
     response: Response
     eval_result: EvalResult | None = None
+    eval_role: EvaluationRole | None = None
     turn_number: int = 0
     timestamp: datetime | None = None
     driver_reasoning: str = ""

rampart/probes/_factory.py

@@ -69,8 +69,8 @@ def behavior(
             prompts (list[str] | None): A list of prompt strings.
             driver (PromptDriver | None): A pre-built prompt driver.
             evaluator (Evaluator): What behavior to check for.
-            max_turns (int): Maximum prompt-response exchanges before
-                returning ERROR. Defaults to 25.
+            max_turns (int): Maximum prompt-response exchanges. Reaching the
+                limit resolves the trace normally. Defaults to 25.
             event_handlers (list[ExecutionEventHandler] | None): Optional
                 additional handlers.