Skip to content

Commit c5cbfc9

Browse files
linj-glitchclaude
andcommitted
Anchor every pre-reply user message as task framing for the escalation judge
The trajectory summary handed to the escalation judge pinned only the first user message as the task statement. Codex sends an environment context block as its first user message and the task as the second, so the judge saw shell and cwd boilerplate as the task while the real task sat in the rolling window, truncated to the per-message cap, and scrolled out after about thirty messages. From then on every task-aware pattern in the rubric (drift, unverified completion, violated constraints) had nothing to compare against, and only friction patterns could fire. Treat every user message that precedes the first assistant reply as task framing and anchor them all, with a wider per-message budget so a multi-thousand-character feature specification survives intact. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> (cherry picked from commit fc61f715ed165e73e0664e4a866ec3b99aa38742) Signed-off-by: Lin Jia <linj@nvidia.com>
1 parent b04238f commit c5cbfc9

1 file changed

Lines changed: 63 additions & 11 deletions

File tree

‎crates/libsy/src/algorithms/util/escalation.rs‎

Lines changed: 63 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,12 @@ const TRUNCATION_SUFFIX: &str = "...<truncated>";
3333
/// which coding-agent harnesses make very large.
3434
const SYSTEM_CHARS: usize = 1_000;
3535

36-
/// Cap for the first user message — the task statement, so it gets the widest anchor budget.
37-
const FIRST_USER_CHARS: usize = 2_000;
36+
/// Per-message cap for task-framing user messages — every user message that precedes the first
37+
/// assistant reply. Coding-agent harnesses often send environment boilerplate as the first user
38+
/// message and the task itself as the second, so anchoring only the first would pin the
39+
/// boilerplate and let the task scroll out of the window. Feature specifications run to several
40+
/// thousand characters, so this gets the widest anchor budget.
41+
const TASK_CHARS: usize = 4_000;
3842

3943
/// Backstop on the assembled transcript; the per-message caps normally bind first.
4044
const MAX_REQUEST_CHARS: usize = 18_000;
@@ -257,7 +261,7 @@ fn summarize_for_judge(
257261
) -> String {
258262
let mut anchors: Vec<String> = Vec::new();
259263
let mut window: Vec<String> = Vec::new();
260-
let mut first_user_seen = false;
264+
let mut assistant_seen = false;
261265

262266
for message in messages {
263267
let text = message_text(message);
@@ -267,18 +271,23 @@ fn summarize_for_judge(
267271
role_label(message.role),
268272
truncate_middle(&text, SYSTEM_CHARS)
269273
)),
270-
Role::User if !first_user_seen => {
271-
first_user_seen = true;
274+
// Everything the user said before the agent first replied is task framing.
275+
Role::User if !assistant_seen => {
272276
anchors.push(format!(
273277
"[user (task)] {}",
274-
truncate_middle(&text, FIRST_USER_CHARS)
278+
truncate_middle(&text, TASK_CHARS)
279+
));
280+
}
281+
role => {
282+
if role == Role::Assistant {
283+
assistant_seen = true;
284+
}
285+
window.push(format!(
286+
"[{}] {}",
287+
role_label(role),
288+
truncate_middle(&text, config.window_message_chars)
275289
));
276290
}
277-
role => window.push(format!(
278-
"[{}] {}",
279-
role_label(role),
280-
truncate_middle(&text, config.window_message_chars)
281-
)),
282291
}
283292
}
284293

@@ -503,6 +512,49 @@ mod tests {
503512
assert!(!summary.contains("step 6"), "{summary}");
504513
}
505514

515+
#[test]
516+
fn summary_anchors_every_user_message_before_the_first_reply() {
517+
// Codex sends environment boilerplate as the first user message and the task as the
518+
// second. Both are framing; the task must stay visible after the window has moved on.
519+
let mut messages = vec![
520+
Message::text(
521+
Role::Developer,
522+
"<skills_instructions>...</skills_instructions>",
523+
),
524+
Message::text(
525+
Role::User,
526+
"<environment_context><cwd>/app</cwd></environment_context>",
527+
),
528+
Message::text(Role::User, "Implement RFC 5545 timezone interop in rrule."),
529+
];
530+
for i in 0..40 {
531+
messages.push(Message::text(Role::Assistant, format!("step {i}")));
532+
messages.push(Message::text(Role::User, format!("later user note {i}")));
533+
}
534+
let config = EscalationJudgeConfig {
535+
recent_turn_window: 3,
536+
..EscalationJudgeConfig::default()
537+
};
538+
539+
let summary = summarize_for_judge(&messages, 40, &config);
540+
541+
assert!(
542+
summary.contains("[user (task)] <environment_context>"),
543+
"{summary}"
544+
);
545+
assert!(
546+
summary.contains("[user (task)] Implement RFC 5545 timezone interop in rrule."),
547+
"{summary}"
548+
);
549+
// User messages after the first reply are ordinary window entries, not anchors.
550+
assert!(
551+
!summary.contains("[user (task)] later user note"),
552+
"{summary}"
553+
);
554+
assert!(summary.contains("[user] later user note 39"), "{summary}");
555+
assert!(!summary.contains("later user note 0\n"), "{summary}");
556+
}
557+
506558
#[test]
507559
fn summary_drops_oldest_window_lines_under_the_char_cap() {
508560
// MAX_REQUEST_CHARS is a backstop, not a dial: at default settings the window caps

0 commit comments

Comments
 (0)