Skip to content

Commit 01e7f2c

Browse files
linj-glitchclaude
andauthored
fix(libsy): anchor every pre-reply user message for the escalation judge and log verdicts (#639)
* Log escalation judge verdicts at debug The escalation judge already returns a reason with every verdict, but the router discarded it after reading the boolean, so an operator tuning the judge prompt could not see why sessions were held on the efficient tier. Keep the reason on the verdict and emit one debug event per verdict under the util::escalation target; it is silent unless that target is enabled. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> (cherry picked from commit 5e6cf24ed4a86b223c2aba4940f43c655049a58e) Signed-off-by: Lin Jia <linj@nvidia.com> * Anchor every pre-reply user message as task framing for the escalation judge The trajectory summary handed to the escalation judge pinned only the first user message as the task statement. Codex sends an environment context block as its first user message and the task as the second, so the judge saw shell and cwd boilerplate as the task while the real task sat in the rolling window, truncated to the per-message cap, and scrolled out after about thirty messages. From then on every task-aware pattern in the rubric (drift, unverified completion, violated constraints) had nothing to compare against, and only friction patterns could fire. Treat every user message that precedes the first assistant reply as task framing and anchor them all, with a wider per-message budget so a multi-thousand-character feature specification survives intact. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> (cherry picked from commit fc61f715ed165e73e0664e4a866ec3b99aa38742) Signed-off-by: Lin Jia <linj@nvidia.com> --------- Signed-off-by: Lin Jia <linj@nvidia.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
1 parent a9729f6 commit 01e7f2c

1 file changed

Lines changed: 75 additions & 13 deletions

File tree

‎crates/libsy/src/algorithms/util/escalation.rs‎

Lines changed: 75 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,12 @@ const TRUNCATION_SUFFIX: &str = "...<truncated>";
3434
/// which coding-agent harnesses make very large.
3535
const SYSTEM_CHARS: usize = 1_000;
3636

37-
/// Cap for the first user message — the task statement, so it gets the widest anchor budget.
38-
const FIRST_USER_CHARS: usize = 2_000;
37+
/// Per-message cap for task-framing user messages — every user message that precedes the first
38+
/// assistant reply. Coding-agent harnesses often send environment boilerplate as the first user
39+
/// message and the task itself as the second, so anchoring only the first would pin the
40+
/// boilerplate and let the task scroll out of the window. Feature specifications run to several
41+
/// thousand characters, so this gets the widest anchor budget.
42+
const TASK_CHARS: usize = 4_000;
3943

4044
/// Backstop on the assembled transcript; the per-message caps normally bind first.
4145
const MAX_REQUEST_CHARS: usize = 18_000;
@@ -89,11 +93,14 @@ impl Default for EscalationJudgeConfig {
8993
}
9094

9195
/// The judge's verdict. The schema also requires a `reason`, which makes the judge state its
92-
/// case and measurably sharpens the verdict — routing reads only the boolean, so it is
93-
/// deserialized away rather than carried.
96+
/// case and measurably sharpens the verdict. Routing reads only the boolean; the reason is
97+
/// kept solely so an operator can see why the judge held or escalated when the
98+
/// `switchyard_libsy::algorithms::util::escalation` target is enabled at `debug`.
9499
#[derive(Deserialize)]
95100
pub(crate) struct EscalationVerdict {
96101
escalate: bool,
102+
#[serde(default)]
103+
reason: String,
97104
}
98105

99106
/// Builds the condensed trajectory presented to the escalation judge.
@@ -126,6 +133,13 @@ impl JudgePolicy for EscalationPolicy {
126133
type Verdict = EscalationVerdict;
127134

128135
fn to_classification(&self, verdict: Option<&EscalationVerdict>) -> Classification {
136+
if let Some(verdict) = verdict {
137+
tracing::debug!(
138+
escalate = verdict.escalate,
139+
reason = %verdict.reason,
140+
"escalation judge verdict"
141+
);
142+
}
129143
match verdict {
130144
Some(verdict) if verdict.escalate => Classification::Scores(vec![Score {
131145
target: self.capable.clone(),
@@ -262,7 +276,7 @@ fn summarize_for_judge(
262276
) -> String {
263277
let mut anchors: Vec<String> = Vec::new();
264278
let mut window: Vec<String> = Vec::new();
265-
let mut first_user_seen = false;
279+
let mut assistant_seen = false;
266280

267281
for message in messages {
268282
let text = message_text(message);
@@ -272,18 +286,23 @@ fn summarize_for_judge(
272286
role_label(message.role),
273287
truncate_middle(&text, SYSTEM_CHARS)
274288
)),
275-
Role::User if !first_user_seen => {
276-
first_user_seen = true;
289+
// Everything the user said before the agent first replied is task framing.
290+
Role::User if !assistant_seen => {
277291
anchors.push(format!(
278292
"[user (task)] {}",
279-
truncate_middle(&text, FIRST_USER_CHARS)
293+
truncate_middle(&text, TASK_CHARS)
294+
));
295+
}
296+
role => {
297+
if role == Role::Assistant {
298+
assistant_seen = true;
299+
}
300+
window.push(format!(
301+
"[{}] {}",
302+
role_label(role),
303+
truncate_middle(&text, config.window_message_chars)
280304
));
281305
}
282-
role => window.push(format!(
283-
"[{}] {}",
284-
role_label(role),
285-
truncate_middle(&text, config.window_message_chars)
286-
)),
287306
}
288307
}
289308

@@ -508,6 +527,49 @@ mod tests {
508527
assert!(!summary.contains("step 6"), "{summary}");
509528
}
510529

530+
#[test]
531+
fn summary_anchors_every_user_message_before_the_first_reply() {
532+
// Codex sends environment boilerplate as the first user message and the task as the
533+
// second. Both are framing; the task must stay visible after the window has moved on.
534+
let mut messages = vec![
535+
Message::text(
536+
Role::Developer,
537+
"<skills_instructions>...</skills_instructions>",
538+
),
539+
Message::text(
540+
Role::User,
541+
"<environment_context><cwd>/app</cwd></environment_context>",
542+
),
543+
Message::text(Role::User, "Implement RFC 5545 timezone interop in rrule."),
544+
];
545+
for i in 0..40 {
546+
messages.push(Message::text(Role::Assistant, format!("step {i}")));
547+
messages.push(Message::text(Role::User, format!("later user note {i}")));
548+
}
549+
let config = EscalationJudgeConfig {
550+
recent_turn_window: 3,
551+
..EscalationJudgeConfig::default()
552+
};
553+
554+
let summary = summarize_for_judge(&messages, 40, &config);
555+
556+
assert!(
557+
summary.contains("[user (task)] <environment_context>"),
558+
"{summary}"
559+
);
560+
assert!(
561+
summary.contains("[user (task)] Implement RFC 5545 timezone interop in rrule."),
562+
"{summary}"
563+
);
564+
// User messages after the first reply are ordinary window entries, not anchors.
565+
assert!(
566+
!summary.contains("[user (task)] later user note"),
567+
"{summary}"
568+
);
569+
assert!(summary.contains("[user] later user note 39"), "{summary}");
570+
assert!(!summary.contains("later user note 0\n"), "{summary}");
571+
}
572+
511573
#[test]
512574
fn summary_drops_oldest_window_lines_under_the_char_cap() {
513575
// MAX_REQUEST_CHARS is a backstop, not a dial: at default settings the window caps

0 commit comments

Comments
 (0)