Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/resources/extensions/gsd/auto-post-unit.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1277,7 +1277,7 @@ export async function postUnitPreVerification(pctx: PostUnitContext, opts?: PreV
s.verificationRetryCount.set(retryKey, attempt);
s.pendingVerificationRetry = {
unitId: s.currentUnit.id,
failureContext: `${failureDetails} (attempt ${attempt}/${MAX_ARTIFACT_VERIFICATION_RETRIES}).`,
failureContext: `${failureDetails}.`,
attempt,
};
debugLog("postUnit", { phase: "artifact-verify-retry", unitType: s.currentUnit.type, unitId: s.currentUnit.id, attempt });
Expand Down
22 changes: 20 additions & 2 deletions src/resources/extensions/gsd/auto/detect-stuck.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,11 @@
*/
export function detectStuck(
window: readonly WindowEntry[],
options?: {
suppressSameUnitKey?: string;
suppressSameUnitUntilAttempt?: number;
currentAttempt?: number;
},
): { stuck: true; reason: string } | null {
if (window.length < 2) return null;

Expand All @@ -80,9 +85,22 @@

// Rule 2: Same unit 3+ consecutive times — suppressed if unit_dispatches
// says we're inside the retry-backoff window (codex MEDIUM B3).
const currentAttempt = options?.currentAttempt;
const suppressUntilAttempt = options?.suppressSameUnitUntilAttempt;
const suppressSameUnitStuck = Boolean(
options?.suppressSameUnitKey
&& options.suppressSameUnitKey === last.key
&& Number.isFinite(currentAttempt)
&& Number.isFinite(suppressUntilAttempt)
&& currentAttempt <= suppressUntilAttempt,

Check failure on line 95 in src/resources/extensions/gsd/auto/detect-stuck.ts

View workflow job for this annotation

GitHub Actions / build

'suppressUntilAttempt' is possibly 'undefined'.

Check failure on line 95 in src/resources/extensions/gsd/auto/detect-stuck.ts

View workflow job for this annotation

GitHub Actions / build

'currentAttempt' is possibly 'undefined'.
);
if (window.length >= 3) {
const lastThree = window.slice(-3);
if (lastThree.every((u) => u.key === last.key) && !retryBudgetSuppresses(last.key)) {
if (
lastThree.every((u) => u.key === last.key)
&& !suppressSameUnitStuck
&& !retryBudgetSuppresses(last.key)
) {
return {
stuck: true,
reason: `${last.key} derived 3 consecutive times without progress${suffix}`,
Expand All @@ -93,7 +111,7 @@
// Rule 2b: Same unit key 3+ times anywhere in the active window — same
// retry-budget suppression as Rule 2.
const countInWindow = window.filter((entry) => entry.key === last.key).length;
if (countInWindow >= 3 && !retryBudgetSuppresses(last.key)) {
if (countInWindow >= 3 && !suppressSameUnitStuck && !retryBudgetSuppresses(last.key)) {
return {
stuck: true,
reason: `${last.key} derived ${countInWindow} times in last ${window.length} attempts without progress${suffix}`,
Expand Down
16 changes: 15 additions & 1 deletion src/resources/extensions/gsd/auto/phases.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import type { LoopDeps } from "./loop-deps.js";
import {
USER_DRIVEN_DEEP_UNITS,
isAwaitingUserInput,
MAX_ARTIFACT_VERIFICATION_RETRIES,
type PostUnitContext,
type PreVerificationOpts,
} from "../auto-post-unit.js";
Expand Down Expand Up @@ -1384,7 +1385,20 @@ export async function runDispatch(
loopState.recentUnits.push({ key: derivedKey });
if (loopState.recentUnits.length > STUCK_WINDOW_SIZE) loopState.recentUnits.shift();

const stuckSignal = detectStuck(loopState.recentUnits);
const unitVerificationRetryKey = `${unitType}:${unitId}`;
const activeVerificationRetryAttempt = s.verificationRetryCount.get(unitVerificationRetryKey);
const isAttemptNumber = typeof activeVerificationRetryAttempt === "number"
&& Number.isFinite(activeVerificationRetryAttempt);
const hasActiveVerificationRetry = s.pendingVerificationRetry?.unitId === unitId
&& isAttemptNumber
&& activeVerificationRetryAttempt >= 1;
const stuckSignal = detectStuck(loopState.recentUnits, hasActiveVerificationRetry
? {
suppressSameUnitKey: derivedKey,
currentAttempt: activeVerificationRetryAttempt,
suppressSameUnitUntilAttempt: MAX_ARTIFACT_VERIFICATION_RETRIES,
}
: undefined);
if (stuckSignal) {
debugLog("autoLoop", {
phase: "stuck-check",
Expand Down
6 changes: 3 additions & 3 deletions src/resources/extensions/gsd/tests/artifact-retry-cap.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,13 @@ test("#2007 bug 1: retry state can represent each allowed attempt before exhaust
s.verificationRetryCount.set(retryKey, attempt);
s.pendingVerificationRetry = {
unitId: "M001/S01/T01",
failureContext: `Missing expected artifact (attempt ${attempt}/${MAX_ARTIFACT_VERIFICATION_RETRIES}).`,
failureContext: "Missing expected artifact.",
attempt,
};

assert.equal(s.verificationRetryCount.get(retryKey), attempt);
assert.equal(s.pendingVerificationRetry.attempt, attempt);
assert.match(s.pendingVerificationRetry.failureContext, /attempt \d\/3/);
assert.equal(s.pendingVerificationRetry.failureContext, "Missing expected artifact.");
}
});

Expand All @@ -66,7 +66,7 @@ test("#2007 bug 2: pendingVerificationRetry state is available for dispatch regr
const s = new AutoSession();
s.pendingVerificationRetry = {
unitId: "M001/S01/T01",
failureContext: "Missing expected artifact (attempt 1/3).",
failureContext: "Missing expected artifact.",
attempt: 1,
};

Expand Down
30 changes: 11 additions & 19 deletions src/resources/extensions/gsd/tests/auto-loop.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2299,7 +2299,7 @@ test("stuck detection: window resets recovery when deriveState returns a differe
);
});

test("stuck detection: verification retries remain visible to the sliding window", async () => {
test("stuck detection: verification retries are not preempted while retry budget is active", async () => {
_resetPendingResolve();
mock.timers.enable({ apis: ["Date", "setTimeout"], now: 20_000 });

Expand All @@ -2313,13 +2313,12 @@ test("stuck detection: verification retries remain visible to the sliding window
let verifyCallCount = 0;
let stopReason = "";

// Pre-queued responses: 3 retries then a continue (exit). Failure
// contexts differ so this test exercises stuck-window behavior without
// tripping duplicate-failure suppression.
// Pre-queued responses: 3 retries then continue. While retry budget is
// active, same-unit stuck detection should stay suppressed.
const verifyActions: Array<() => "retry" | "continue"> = [
() => { s.pendingVerificationRetry = { unitId: "M001/S01/T01", failureContext: "test failed: 1", attempt: 1 }; return "retry"; },
() => { s.pendingVerificationRetry = { unitId: "M001/S01/T01", failureContext: "test failed: 2", attempt: 2 }; return "retry"; },
() => { s.pendingVerificationRetry = { unitId: "M001/S01/T01", failureContext: "test failed: 3", attempt: 3 }; return "retry"; },
() => { s.verificationRetryCount.set("execute-task:M001/S01/T01", 1); s.pendingVerificationRetry = { unitId: "M001/S01/T01", failureContext: "missing artifact: summary", attempt: 1 }; return "retry"; },
() => { s.verificationRetryCount.set("execute-task:M001/S01/T01", 2); s.pendingVerificationRetry = { unitId: "M001/S01/T01", failureContext: "missing artifact: checklist", attempt: 2 }; return "retry"; },
() => { s.verificationRetryCount.set("execute-task:M001/S01/T01", 3); s.pendingVerificationRetry = { unitId: "M001/S01/T01", failureContext: "missing artifact: notes", attempt: 3 }; return "retry"; },
() => { s.active = false; return "continue"; },
];

Expand Down Expand Up @@ -2354,9 +2353,9 @@ test("stuck detection: verification retries remain visible to the sliding window

const loopPromise = autoLoop(ctx, pi, s, deps);

// Resolve agent_end for 3 attempts. The 4th iteration should stop before
// dispatch because retry dispatches stay visible to stuck detection.
for (let i = 1; i <= 3; i++) {
// Resolve agent_end for all 4 iterations. We should not stop early as
// stuck while retries are still in policy range.
for (let i = 1; i <= 4; i++) {
await waitForMicrotasks(() => pi.calls.length === i, `dispatch ${i}`);
resolveAgentEnd(makeEvent());
await drainMicrotasks(100);
Expand All @@ -2365,15 +2364,8 @@ test("stuck detection: verification retries remain visible to the sliding window

await loopPromise;

assert.ok(
stopReason.includes("Stuck"),
`stuck detection should fire during repeated verification retries, got: ${stopReason}`,
);
assert.equal(
verifyCallCount,
3,
"verification should stop before a 4th repeated retry dispatch",
);
assert.equal(stopReason, "", `stuck detection should not preempt bounded verification retries, got: ${stopReason}`);
assert.ok(verifyCallCount >= 4, "verification retries should continue through the bounded retry window");
} finally {
mock.timers.reset();
}
Expand Down
Loading