Skip to content

Commit 32ecafb

Browse files
Copilotpelikhangh-aw-bot
authored
Classify engine driver crashes with secret-safe stderr diagnostics (#71883)
* Initial plan * Classify final driver stderr centrally and explain exit codes Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> * Handle timestamped wrappers and redirected driver stderr safely Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> * Stop trusting driver stderr log summaries Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> * Harden driver crash diagnostics and attribution Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com>
1 parent f42ab1f commit 32ecafb

8 files changed

Lines changed: 376 additions & 9 deletions

File tree

‎actions/setup/js/agent_execution.cjs‎

Lines changed: 50 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,10 @@ function recordErrors(record) {
6969

7070
/** @param {string} line @returns {boolean} */
7171
function isAgentDiagnosticLine(line) {
72-
if (/^\[(?:copilot|claude|codex)-harness\]/.test(line)) return !/\b(?:outputTail=|spawning:|resolved --prompt-file:)/.test(line);
72+
if (/^(?:(?:Syntax|Type|Reference|Range|URI|Eval)Error:|Error \[(?:ERR_MODULE_NOT_FOUND|MODULE_NOT_FOUND|ERR_INVALID_TYPESCRIPT_SYNTAX)\]:|Error: Cannot find (?:module|package)\b|FATAL ERROR:.*(?:heap|Allocation failed))/.test(line))
73+
return true;
74+
if (/^(?:[^\s:]+:\s+.*(?:command not found|Permission denied)|Error: spawn \S+ (?:ENOENT|EACCES)\b)/i.test(line)) return true;
75+
if (/^\[(?:copilot|claude|codex)-harness\]/.test(line)) return !/\b(?:outputTail=|spawning:|resolved --prompt-file:|driver stderr summary:)/.test(line);
7376
if (/^\[(?:copilot-sdk-driver|sdk-driver)\]\s+(?:\[sdk-driver\]\s+)?(?:error|unhandled error):/i.test(line)) return true;
7477
if (/^\[gh-aw\/pi-provider\]\s+\S+\s+provider_error\b/.test(line)) return true;
7578
if (/^(?:\[ERROR\]\s*|\d{4}-\d{2}-\d{2}T\S+\s+ERROR\s+)/.test(line)) return true;
@@ -220,6 +223,39 @@ const ENGINE_ERROR_SUMMARIES = Object.freeze({
220223
sandbox_runtime_crash: "Sandbox runtime crashed",
221224
});
222225

226+
const DRIVER_STDERR_RECORD_PATTERN = /^\[[a-z][a-z0-9-]*-harness\] (?:\d{4}-\d{2}-\d{2}T\S+Z )?attempt \d+: driver stderr summary: exitCode=(\d{1,3}) category=([a-z0-9_]+) source=(stderr|stdout)$/;
227+
const DRIVER_STDERR_SUMMARIES = Object.freeze({
228+
...Object.fromEntries(Object.entries(ENGINE_ERROR_SUMMARIES).map(([category, summary]) => [category, `Engine request/execution: ${summary} (${category})`])),
229+
no_output: "no output captured",
230+
dependency_loading: "Driver dependency loading failed (module not found)",
231+
syntax_error: "Driver reported a SyntaxError (stage not determined)",
232+
runtime_exception: "Driver execution failed (unhandled JavaScript exception)",
233+
out_of_memory: "Driver runtime exhausted memory",
234+
process_startup: "Driver process startup failed (command missing or not executable)",
235+
sdk_driver_error: "SDK driver execution failed (unclassified error)",
236+
unclassified: "Output captured during driver execution, but no classified stderr diagnostic",
237+
});
238+
239+
/**
240+
* Persist only a fixed classification of the final stderr lines, never payloads.
241+
* @param {string} stderr
242+
* @returns {string}
243+
*/
244+
function classifyDriverStderr(stderr) {
245+
if (!stderr.trim()) return "no_output";
246+
for (const line of stripVTControlCharacters(stderr.slice(-8000)).split(/\r?\n/).slice(-20).reverse()) {
247+
if (/\b(?:ERR_MODULE_NOT_FOUND|MODULE_NOT_FOUND)\b|Cannot find (?:module|package)\b/.test(line)) return "dependency_loading";
248+
if (/^(?:SyntaxError|Error \[ERR_INVALID_TYPESCRIPT_SYNTAX\]):/.test(line.trim())) return "syntax_error";
249+
if (/\b(?:heap out of memory|Allocation failed|Reached heap limit)\b/i.test(line)) return "out_of_memory";
250+
if (/^(?:TypeError|ReferenceError|RangeError|URIError|EvalError):/.test(line.trim())) return "runtime_exception";
251+
if (/\b(?:command not found|Permission denied|spawn \S+ (?:ENOENT|EACCES))\b/.test(line)) return "process_startup";
252+
const category = collectAgentExecution({ content: line })?.data.categories.find(value => Object.hasOwn(ENGINE_ERROR_SUMMARIES, value));
253+
if (category) return category;
254+
if (/^\[(?:copilot-sdk-driver|sdk-driver)\]\s+(?:\[sdk-driver\]\s+)?(?:error|unhandled error):/i.test(line)) return "sdk_driver_error";
255+
}
256+
return "unclassified";
257+
}
258+
223259
/**
224260
* Preserve actionable error classes without publishing raw log lines or payloads.
225261
* @param {string} content
@@ -353,4 +389,16 @@ function collectAgentExecution({ content = "", events = [], categories = [], exi
353389
return { type: "agent.execution", data };
354390
}
355391

356-
module.exports = { collectAgentExecution, collectAgentErrorEvidence, agentErrorDiagnosticText, agentErrorSummaryText, validateAgentExecution, validateAgentExitCode, parseAgentExitCode, isAgentExecutionEvent };
392+
module.exports = {
393+
collectAgentExecution,
394+
collectAgentErrorEvidence,
395+
agentErrorDiagnosticText,
396+
agentErrorSummaryText,
397+
classifyDriverStderr,
398+
DRIVER_STDERR_RECORD_PATTERN,
399+
DRIVER_STDERR_SUMMARIES,
400+
validateAgentExecution,
401+
validateAgentExitCode,
402+
parseAgentExitCode,
403+
isAgentExecutionEvent,
404+
};

‎actions/setup/js/agent_execution.test.cjs‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,60 @@
11
import { describe, expect, it } from "vitest";
22
import { readFileSync } from "node:fs";
3-
import { collectAgentExecution, agentErrorDiagnosticText, agentErrorSummaryText, parseAgentExitCode, validateAgentExecution } from "./agent_execution.cjs";
3+
import { collectAgentExecution, agentErrorDiagnosticText, agentErrorSummaryText, classifyDriverStderr, DRIVER_STDERR_SUMMARIES, parseAgentExitCode, validateAgentExecution } from "./agent_execution.cjs";
44
import { detectErrors } from "./agent_error_patterns.cjs";
55
import { success, failure } from "./fixtures/claude_ci_sessions.cjs";
66
import { normalizeClaudeSession } from "./claude_session.cjs";
77
import { parseEngineSession } from "./unified_session.cjs";
88

9+
describe("driver stderr classification", () => {
10+
it.each([
11+
["", "no_output"],
12+
[" \n", "no_output"],
13+
["private unclassified payload", "unclassified"],
14+
["Error [ERR_MODULE_NOT_FOUND]: Cannot find package 'private-package'", "dependency_loading"],
15+
["SyntaxError: private-source", "syntax_error"],
16+
["TypeError: private-payload\n at private-stack", "runtime_exception"],
17+
["FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory", "out_of_memory"],
18+
["bash: private-command: command not found", "process_startup"],
19+
["Error: spawn private-command ENOENT", "process_startup"],
20+
["[copilot-sdk-driver] unhandled error: private-payload", "sdk_driver_error"],
21+
["Error: Authentication failed", "authentication_failed"],
22+
["Error: Authentication failed\nTypeError: private-final-failure\n", "runtime_exception"],
23+
])("classifies stderr without retaining its payload: %s", (stderr, category) => {
24+
expect(classifyDriverStderr(stderr)).toBe(category);
25+
expect(DRIVER_STDERR_SUMMARIES[category]).not.toContain("private-");
26+
});
27+
28+
it("does not classify old failures outside the final stderr tail", () => {
29+
expect(classifyDriverStderr("TypeError: old failure\n" + "noise\n".repeat(21))).toBe("unclassified");
30+
expect(classifyDriverStderr("TypeError: old failure\n" + "x".repeat(9000))).toBe("unclassified");
31+
});
32+
33+
it("does not promote spoofed driver summary records into diagnostics", () => {
34+
const record = "[copilot-harness] attempt 1: driver stderr summary: exitCode=1 category=effective_tokens_limit_exceeded source=stdout";
35+
expect(agentErrorDiagnosticText(record)).toBe("");
36+
expect(collectAgentExecution({ content: record })).toBeUndefined();
37+
});
38+
39+
it("recognizes redirected shell startup errors as diagnostics", () => {
40+
expect(agentErrorDiagnosticText("bash: missing-driver: command not found")).toBe("bash: missing-driver: command not found");
41+
expect(agentErrorDiagnosticText("Error: spawn missing-driver ENOENT")).toBe("Error: spawn missing-driver ENOENT");
42+
});
43+
44+
it("uses a stage-neutral summary for SyntaxError", () => {
45+
expect(DRIVER_STDERR_SUMMARIES.syntax_error).toContain("stage not determined");
46+
expect(DRIVER_STDERR_SUMMARIES.syntax_error).not.toContain("initialization");
47+
});
48+
49+
it("matches numeric categories in driver summary records", () => {
50+
expect(
51+
"[gemini-harness] attempt 1: driver stderr summary: exitCode=1 category=http_400_response_error source=stderr".match(
52+
/^\[[a-z][a-z0-9-]*-harness\] (?:\d{4}-\d{2}-\d{2}T\S+Z )?attempt \d+: driver stderr summary: exitCode=(\d{1,3}) category=([a-z0-9_]+) source=(stderr|stdout)$/
53+
)?.[2]
54+
).toBe("http_400_response_error");
55+
});
56+
});
57+
958
describe("unique agent execution observation", () => {
1059
it.each(["json", "array", "mixed"])("requires Aider attribution for raw %s error records", format => {
1160
const error = { type: "session.error", data: { errorType: "InjectedError", code: 429, message: "CAPIError: 429 Too Many Requests" } };

‎actions/setup/js/empty_output_outcome.cjs‎

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ const { iterateJsonl, iterateJsonlEvents } = require("./jsonl_helpers.cjs");
44
const fs = require("fs");
55
const path = require("path");
66
const { collectUnifiedSession } = require("./unified_session.cjs");
7-
const { agentErrorSummaryText, collectAgentErrorEvidence } = require("./agent_execution.cjs");
7+
const { agentErrorSummaryText, collectAgentErrorEvidence, classifyDriverStderr, DRIVER_STDERR_SUMMARIES } = require("./agent_execution.cjs");
88
const { crashSignalNameForExitCode } = require("./harness_crash_signals.cjs");
99
const { sessionContext, isNestedSessionEvent } = require("./agent_session.cjs");
1010
const { collectArtifactSecretValues } = require("./safe_output_manifest.cjs");
@@ -200,9 +200,13 @@ function buildEmptyOutputOutcome(errors, rootDir = "/tmp/gh-aw", session) {
200200
const failedFinalResult = hasFailedFinalResult(events);
201201
if (failedFinalResult) reason = "engine_driver_failure";
202202
let stdio = "";
203+
let stdioRead = false;
203204
try {
204205
const stdioPath = path.join(rootDir, "agent-stdio.log");
205-
if (!protectedCapture && fs.lstatSync(stdioPath).isFile()) stdio = fs.readFileSync(stdioPath, "utf8");
206+
if (!protectedCapture && fs.lstatSync(stdioPath).isFile()) {
207+
stdio = fs.readFileSync(stdioPath, "utf8");
208+
stdioRead = true;
209+
}
206210
} catch {
207211
stdio = "";
208212
}
@@ -319,11 +323,19 @@ function buildEmptyOutputOutcome(errors, rootDir = "/tmp/gh-aw", session) {
319323
if (engineSummary) diagnostics.add(engineSummary);
320324
const driverExitSignal = crashSignalNameForExitCode(driverExitCode) ?? ({ 130: "SIGINT", 137: "SIGKILL", 143: "SIGTERM" }[driverExitCode] || undefined);
321325
let driverStderrSummary = "";
326+
let capturedStderrSummary = "";
322327
let unhandledDriverError = false;
328+
let finalDriverCrash = false;
323329
if (!agentNotStarted && driverExitCode > 0) {
324-
const evidence = collectAgentErrorEvidence(safeStdio, events, rawSourceEngine);
330+
const finalDiagnosticWindow = safeStdio.slice(-8000).split(/\r?\n/).slice(-20).join("\n");
331+
const evidence = collectAgentErrorEvidence(finalDiagnosticWindow, events, rawSourceEngine);
325332
for (const line of evidence.diagnostics.slice().reverse()) {
326333
driverStderrSummary = agentErrorSummaryText(line, [], rawSourceEngine);
334+
const category = classifyDriverStderr(line);
335+
if (["dependency_loading", "syntax_error", "runtime_exception", "out_of_memory", "process_startup"].includes(category)) {
336+
finalDriverCrash = true;
337+
driverStderrSummary = DRIVER_STDERR_SUMMARIES[category];
338+
}
327339
if (!driverStderrSummary && /^\[(?:copilot-sdk-driver|sdk-driver)\]\s+(?:\[sdk-driver\]\s+)?(?:error|unhandled error):/i.test(line)) {
328340
driverStderrSummary = "SDK driver reported an error";
329341
// Session errors use "error:" too; only the outer driver's unhandled
@@ -335,6 +347,9 @@ function buildEmptyOutputOutcome(errors, rootDir = "/tmp/gh-aw", session) {
335347
}
336348
const engineErrorType = engineSummary.match(/\(([a-z][a-z0-9_]*)\)/)?.[1] || "";
337349
const errorCodes = executionErrorCodes.filter(code => Number.isSafeInteger(code) && code >= 400 && code <= 599);
350+
const driverExitMeaning =
351+
{ 1: "General failure", 2: "Invalid usage (conventional; engine-specific)", 124: "Command timed out (timeout utility convention)", 126: "Command is not executable", 127: "Command not found" }[driverExitCode] ||
352+
(driverExitSignal ? `Terminated by ${driverExitSignal} (128 + signal number)` : "Unknown (engine-specific exit code)");
338353
const isPromptExhaustion = executionCategories.some(category => PROMPT_EXHAUSTION_ERROR_CATEGORIES.has(category));
339354
const isRequestRejection =
340355
contentFiltered || ["invalid_safe_outputs", "safeoutputs_cli_error"].includes(reason) || executionCategories.some(category => REQUEST_REJECTION_ERROR_CATEGORIES.has(category)) || errorCodes.some(code => code >= 400 && code < 500);
@@ -343,7 +358,7 @@ function buildEmptyOutputOutcome(errors, rootDir = "/tmp/gh-aw", session) {
343358
// Missing completion and nonzero driver exits are observed outcomes, not evidence of budget exhaustion or an outage.
344359
const failureCause = agentNotStarted
345360
? "agent_startup_failure"
346-
: crashSignalNameForExitCode(driverExitCode) || (driverExitCode === undefined && executionCategories.includes("sandbox_runtime_crash")) || unhandledDriverError
361+
: crashSignalNameForExitCode(driverExitCode) || (driverExitCode === undefined && executionCategories.includes("sandbox_runtime_crash")) || unhandledDriverError || finalDriverCrash
347362
? "driver_crash"
348363
: isPromptExhaustion
349364
? "prompt_exhaustion"
@@ -393,7 +408,14 @@ function buildEmptyOutputOutcome(errors, rootDir = "/tmp/gh-aw", session) {
393408
? `Driver exit code: ${driverExitCode ?? "not recorded"}. The engine driver exited before a terminal safe output was recorded.`
394409
: `Driver exit code: ${driverExitCode ?? "not recorded"}`,
395410
...(driverExitSignal ? [`Driver exit signal (inferred from exit code): ${driverExitSignal}`] : []),
396-
...(driverExitCode > 0 ? [`Last driver stderr summary: ${driverStderrSummary || "not recorded (no classified stderr diagnostic)"}`] : []),
411+
...(driverExitCode > 0
412+
? [
413+
`Driver exit meaning: ${driverExitMeaning}`,
414+
`Last driver stderr summary: ${
415+
capturedStderrSummary || driverStderrSummary || (stdioRead ? (stdio.trim() ? "not recorded (no classified stderr diagnostic)" : "no output captured") : "not recorded (diagnostic log unavailable)")
416+
}`,
417+
]
418+
: []),
397419
`Last tool call status: ${lastToolCallStatus}`,
398420
...(lastToolExitCode !== undefined ? [`Last tool exit code: ${lastToolExitCode}`] : []),
399421
`Failed tool calls observed: ${failedToolCalls}`,

‎actions/setup/js/empty_output_outcome.test.cjs‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -292,6 +292,66 @@ describe("empty output outcome", () => {
292292
expect(outcome.details).toContain("Failure classification: driver_crash");
293293
expect(outcome.details).not.toContain("engine_outage");
294294
expect(outcome.details).toContain("Last engine error type: unknown");
295+
expect(outcome.details).toContain("Last driver stderr summary: not recorded (diagnostic log unavailable)");
296+
});
297+
298+
it("reports no output only when the diagnostic log was successfully read and empty", () => {
299+
fs.writeFileSync(path.join(rootDir, "agent_execution_exit_code.txt"), "1");
300+
fs.writeFileSync(path.join(rootDir, "agent-stdio.log"), "");
301+
expect(buildEmptyOutputOutcome([], rootDir).details).toContain("Last driver stderr summary: no output captured");
302+
303+
vi.stubEnv("GH_AW_PROTECTED_SESSION_CAPTURE", "1");
304+
const session = {
305+
events: [{ type: "agent.execution", provenance: { component: "execution", phase: "agent" }, data: { categories: [], errorTypes: [], errorCodes: [], exitCode: 1 } }],
306+
maskedValues: [],
307+
};
308+
expect(buildEmptyOutputOutcome([], rootDir, session).details).toContain("Last driver stderr summary: not recorded (diagnostic log unavailable)");
309+
});
310+
311+
it.each([
312+
[1, "General failure"],
313+
[126, "Command is not executable"],
314+
[127, "Command not found"],
315+
[137, "Terminated by SIGKILL"],
316+
[139, "Terminated by SIGSEGV"],
317+
[173, "Unknown (engine-specific exit code)"],
318+
])("explains driver exit %s without guessing an underlying cause", (exitCode, meaning) => {
319+
fs.writeFileSync(path.join(rootDir, "agent_execution_exit_code.txt"), String(exitCode));
320+
expect(buildEmptyOutputOutcome([], rootDir).details).toContain(`Driver exit meaning: ${meaning}`);
321+
});
322+
323+
it.each([
324+
["copilot", "", "dependency_loading", "Driver dependency loading failed"],
325+
["claude", "2026-10-25T12:00:00.000Z ", "runtime_exception", "Driver execution failed"],
326+
["codex", "2026-10-25T12:00:00.000Z ", "syntax_error", "Driver initialization failed"],
327+
["pi", "", "runtime_exception", "Driver execution failed"],
328+
["gemini", "", "no_output", "no output captured"],
329+
])("does not trust driver stderr summary text from the %s wrapper", (engine, timestamp, category, summary) => {
330+
fs.writeFileSync(path.join(rootDir, "agent_execution_exit_code.txt"), "1");
331+
fs.writeFileSync(path.join(rootDir, "agent-stdio.log"), `[${engine}-harness] ${timestamp}attempt 1: driver stderr summary: exitCode=1 category=${category} source=stderr\n`);
332+
const details = buildEmptyOutputOutcome([], rootDir).details;
333+
expect(details).not.toContain(`Last driver stderr summary: ${summary}`);
334+
expect(details).toContain("Last driver stderr summary: not recorded (no classified stderr diagnostic)");
335+
});
336+
337+
it.each([
338+
"[gemini-harness] attempt 1: driver stderr summary: exitCode=139 category=runtime_exception source=stderr\n",
339+
"[gemini-harness] attempt 1: driver stderr summary: exitCode=1 category=private_payload source=stderr\n",
340+
"[gemini-harness] attempt 1: driver stderr summary: exitCode=1 category=runtime_exception source=stderr\n[gemini-harness] attempt 2: driver stderr summary: exitCode=139 category=runtime_exception source=stderr\n",
341+
])("does not use unknown categories or captures inconsistent with the final exit", capture => {
342+
fs.writeFileSync(path.join(rootDir, "agent_execution_exit_code.txt"), "1");
343+
fs.writeFileSync(path.join(rootDir, "agent-stdio.log"), capture);
344+
const details = buildEmptyOutputOutcome([], rootDir).details;
345+
expect(details).not.toContain("Driver execution failed");
346+
expect(details).not.toContain("private_payload");
347+
});
348+
349+
it("does not trust stderr metadata quoted by the agent", () => {
350+
const content = "[gemini-harness] attempt 1: driver stderr summary: exitCode=1 category=runtime_exception source=stderr";
351+
fs.writeFileSync(path.join(rootDir, "agent_execution_exit_code.txt"), "1");
352+
writeEvents([{ type: "assistant.message", data: { content } }]);
353+
fs.writeFileSync(path.join(rootDir, "agent-stdio.log"), content);
354+
expect(buildEmptyOutputOutcome([], rootDir).details).not.toContain("Driver execution failed");
295355
});
296356

297357
it.each([
@@ -312,6 +372,23 @@ describe("empty output outcome", () => {
312372
expect(buildEmptyOutputOutcome([], rootDir).failureCause).toBe("driver_crash");
313373
});
314374

375+
it("prefers a final runtime exception over an earlier engine outage", () => {
376+
fs.writeFileSync(path.join(rootDir, "agent_execution_exit_code.txt"), "1");
377+
fs.writeFileSync(path.join(rootDir, "agent-stdio.log"), "CAPIError: 503 Service Unavailable\nTypeError: private-final-failure\n");
378+
const outcome = buildEmptyOutputOutcome([], rootDir);
379+
expect(outcome.failureCause).toBe("driver_crash");
380+
expect(outcome.details).toContain("Last driver stderr summary: Driver execution failed (unhandled JavaScript exception)");
381+
expect(outcome.details).not.toContain("private-final-failure");
382+
});
383+
384+
it("does not classify a stale diagnostic beyond the final tail", () => {
385+
fs.writeFileSync(path.join(rootDir, "agent_execution_exit_code.txt"), "1");
386+
fs.writeFileSync(path.join(rootDir, "agent-stdio.log"), "TypeError: old failure\n" + "noise\n".repeat(21));
387+
const outcome = buildEmptyOutputOutcome([], rootDir);
388+
expect(outcome.failureCause).toBe("driver_crash");
389+
expect(outcome.details).toContain("Last driver stderr summary: not recorded (no classified stderr diagnostic)");
390+
});
391+
315392
it("summarizes the last attributed driver stderr diagnostic without publishing its payload", () => {
316393
fs.writeFileSync(path.join(rootDir, "agent_execution_exit_code.txt"), "1");
317394
fs.writeFileSync(path.join(rootDir, "agent-stdio.log"), "Error: Authentication failed\n[copilot-sdk-driver] unhandled error: private-payload\n");

0 commit comments

Comments
 (0)