Suppress spurious report_incomplete failures after successful Copilot task output (#45599)

* Initial plan

* Suppress spurious incomplete failure reports

Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>

* Tighten spurious incomplete suppression heuristic

Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>
Co-authored-by: Peli de Halleux <pelikhan@users.noreply.github.com>

Author: Copilot

actions/setup/js/handle_agent_failure.cjs

@@ -1518,6 +1518,42 @@ function loadReportIncompleteMessages(items) {
   }
 }
 
+const DIAGNOSTIC_AGENT_OUTPUT_TYPES = new Set(["noop", "missing_tool", "missing_data", "report_incomplete"]);
+const TASK_COMPLETE_REGISTRATION_SIGNALS = ["not registering", "not registered", "not recognizing", "not recognized", "recognition issue", "not yet marked", "haven't marked", "tool calls are not registering"];
+const TASK_COMPLETE_COMPLETION_SIGNALS = ["completed successfully", "safe-output", "safe output", "no remaining work"];
+const TASK_COMPLETE_COMPLETION_REGEXPS = [/\banalysis steps? completed\b/, /\ball steps completed\b/];
+
+/**
+ * Determine whether agent output contains at least one real task-level item.
+ * @param {Array<any> | undefined} items
+ * @returns {boolean}
+ */
+function hasTaskLevelAgentOutput(items) {
+  if (!Array.isArray(items)) {
+    return false;
+  }
+  return items.some(item => item && typeof item.type === "string" && !DIAGNOSTIC_AGENT_OUTPUT_TYPES.has(item.type));
+}
+
+/**
+ * Detect a spurious report_incomplete caused only by task_complete registration/recognition
+ * trouble after the agent already finished the real task work.
+ * @param {{reason?: string, details?: string} | undefined} item
+ * @returns {boolean}
+ */
+function isTaskCompleteRegistrationIssue(item) {
+  if (!item) {
+    return false;
+  }
+  const text = `${item.reason || ""}\n${item.details || ""}`.toLowerCase();
+  if (!text.includes("task_complete")) {
+    return false;
+  }
+  const hasRegistrationSignal = TASK_COMPLETE_REGISTRATION_SIGNALS.some(signal => text.includes(signal));
+  const hasCompletionSignal = TASK_COMPLETE_COMPLETION_SIGNALS.some(signal => text.includes(signal)) || TASK_COMPLETE_COMPLETION_REGEXPS.some(regexp => regexp.test(text));
+  return hasRegistrationSignal && hasCompletionSignal;
+}
+
 /**
  * Build report_incomplete context string for display in failure issues/comments.
  * This surfaces the agent's structured incompletion signal so maintainers can
@@ -2964,6 +3000,13 @@ async function main() {
     let hasCompletedDespiteJobFailure = false;
     const { loadAgentOutput } = require("./load_agent_output.cjs");
     const agentOutputResult = loadAgentOutput();
+    const taskCompleteRegistrationIssueOnlyReportIncomplete =
+      agentOutputResult.success &&
+      agentOutputResult.items &&
+      agentOutputResult.items.some(item => item.type === "report_incomplete") &&
+      hasAgentTerminalReasonCompleted() &&
+      hasTaskLevelAgentOutput(agentOutputResult.items) &&
+      agentOutputResult.items.filter(item => item.type === "report_incomplete").every(isTaskCompleteRegistrationIssue);
 
     if (agentConclusion === "success") {
       if (!agentOutputResult.success || !agentOutputResult.items || agentOutputResult.items.length === 0) {
@@ -2987,7 +3030,7 @@ async function main() {
         if (nonNoopItems.length === 0) {
           hasOnlyNoopOutputs = true;
           core.info("Agent failed with exit code 1 but produced only noop outputs - treating as successful no-action (transient AI model error)");
-        } else if (!nonNoopItems.some(item => item.type === "report_incomplete")) {
+        } else if (!nonNoopItems.some(item => item.type === "report_incomplete" && !isTaskCompleteRegistrationIssue(item))) {
           // The agent produced valid non-noop safe outputs (e.g. create_discussion) but the
           // job exit code is non-zero. If terminal_reason: completed is present in the log,
           // the failure was a transient error after the agent finished its task — do not report
@@ -3008,10 +3051,14 @@ async function main() {
     if (agentOutputResult.success && agentOutputResult.items && agentOutputResult.items.length > 0) {
       const reportIncompleteItems = agentOutputResult.items.filter(item => item.type === "report_incomplete");
       if (reportIncompleteItems.length > 0) {
-        hasReportIncomplete = true;
-        core.info(`Agent emitted ${reportIncompleteItems.length} report_incomplete signal(s) - activating failure handling`);
-        for (const item of reportIncompleteItems) {
-          core.info(`  report_incomplete reason: ${item.reason}`);
+        if (taskCompleteRegistrationIssueOnlyReportIncomplete) {
+          core.info("Ignoring report_incomplete signal(s) caused only by task_complete registration trouble after successful task-level outputs");
+        } else {
+          hasReportIncomplete = true;
+          core.info(`Agent emitted ${reportIncompleteItems.length} report_incomplete signal(s) - activating failure handling`);
+          for (const item of reportIncompleteItems) {
+            core.info(`  report_incomplete reason: ${item.reason}`);
+          }
         }
       }
     }

actions/setup/js/handle_agent_failure.test.cjs

@@ -4093,6 +4093,47 @@ describe("handle_agent_failure", () => {
       // report_incomplete overrides the hasCompletedDespiteJobFailure exemption
       expect(createIssueMock).toHaveBeenCalled();
     });
+
+    it("ignores report_incomplete when it only describes task_complete registration trouble after successful outputs", async () => {
+      fs.writeFileSync(
+        path.join(tmpDir, "agent_output.json"),
+        JSON.stringify({
+          items: [
+            { type: "add_comment", body: "done" },
+            { type: "submit_pull_request_review", event: "APPROVE", body: "done" },
+            {
+              type: "report_incomplete",
+              reason: "Test Quality Sentinel analysis completed successfully. However, task_complete tool calls are not registering with the system despite repeated attempts and no remaining work identified.",
+            },
+          ],
+        })
+      );
+      fs.writeFileSync(path.join(tmpDir, "agent-stdio.log"), '{"type":"result","subtype":"success","terminal_reason":"completed","num_turns":10}\n');
+
+      const createIssueMock = vi.fn();
+      const createCommentMock = vi.fn();
+
+      global.github = {
+        rest: {
+          search: {
+            issuesAndPullRequests: vi.fn(async () => ({ data: { total_count: 0, items: [] } })),
+          },
+          issues: {
+            create: createIssueMock,
+            createComment: createCommentMock,
+          },
+          pulls: { get: vi.fn() },
+        },
+        graphql: vi.fn(),
+      };
+
+      vi.resetModules();
+      const { main: mainFn } = require("./handle_agent_failure.cjs");
+      await mainFn();
+
+      expect(createIssueMock).not.toHaveBeenCalled();
+      expect(createCommentMock).not.toHaveBeenCalled();
+    });
   });
 
   describe("parseFirewallAuthErrors", () => {