Codex harness: detect unsupported-model tool-schema failures with a dedicated message (#57256)

Author: Copilot

actions/setup/js/codex_harness.cjs

@@ -93,6 +93,61 @@ const SERVER_ERROR_PATTERN = /InternalServerError|ServiceUnavailableError|500 In
 // an identical rejection: retrying only re-bills the turns that succeeded before the failure point.
 const INVALID_REQUEST_ERROR_PATTERN = /invalid_request_error/i;
 
+// Codex's `turn.failed` event nests the actual provider error as a JSON string inside
+// `error.message` (sometimes doubly-nested, e.g. `error.message` -> `{"error": {...}}`).
+// This is a specific, common form of "unsupported model" failure: the configured model does
+// not support the `custom` tool type Codex uses for its `apply_patch`/freeform tool schema.
+// The provider rejects the whole request before any work happens, surfacing as:
+//   {"error": {"message": "Invalid value: 'custom'", "type": "invalid_request_error",
+//              "param": "tools", "code": "unknown_parameter"}}
+// This is a model-capability mismatch, not a malformed request, so it warrants a dedicated,
+// more actionable message than the generic invalid_request_error handling below.
+
+/**
+ * Unwraps up to a few levels of Codex's nested provider error payload to find the
+ * innermost object that carries string `param`/`code` fields.
+ * @param {unknown} error
+ * @returns {{ param?: string, code?: string } | null}
+ */
+function extractNestedProviderErrorDetails(error) {
+  const candidates = [error];
+  for (let visited = 0; visited < 8 && candidates.length > 0; visited++) {
+    const current = candidates.shift();
+    if (!current || typeof current !== "object") continue;
+    /** @type {{ param?: unknown, code?: unknown, error?: unknown, message?: unknown, metadata?: unknown }} */
+    const candidate = current;
+    if (typeof candidate.param === "string" && typeof candidate.code === "string") {
+      return { param: candidate.param, code: candidate.code };
+    }
+    if (candidate.error && typeof candidate.error === "object") candidates.push(candidate.error);
+    if (typeof candidate.message === "string") {
+      const parsed = parseJsonOrUndefined(candidate.message);
+      if (parsed !== undefined) candidates.push(parsed);
+    }
+    if (candidate.metadata && typeof candidate.metadata === "object") {
+      /** @type {{ raw?: unknown }} */
+      const metadata = candidate.metadata;
+      if (typeof metadata.raw === "string") {
+        const parsed = parseJsonOrUndefined(metadata.raw);
+        if (parsed !== undefined) candidates.push(parsed);
+      }
+    }
+  }
+  return null;
+}
+
+/**
+ * @param {string} value
+ * @returns {unknown}
+ */
+function parseJsonOrUndefined(value) {
+  try {
+    return JSON.parse(value);
+  } catch {
+    return undefined;
+  }
+}
+
 // Post-result watchdog: once the agent writes a terminal safe-output the harness
 // arms a watchdog timer and kills the Codex process if it is still running after
 // POST_RESULT_WATCHDOG_IDLE_TIMEOUT_MS of inactivity.  This prevents the step from
@@ -203,6 +258,28 @@ function isInvalidRequestError(output) {
   });
 }
 
+/**
+ * Determines if Codex emitted a `turn.failed` provider event indicating the configured model
+ * does not support Codex's required `custom` tool-calling schema (the provider rejects the
+ * `tools` request parameter with code `unknown_parameter`). This is a model-capability mismatch
+ * — the model itself is valid but incompatible with Codex — so it is surfaced as a dedicated,
+ * non-retryable condition with actionable guidance rather than the generic invalid-request message.
+ * @param {string} output - Collected stdout+stderr from the process
+ * @returns {boolean}
+ */
+function isUnsupportedModelToolsError(output) {
+  return output.split(/\r?\n/).some(line => {
+    try {
+      const event = JSON.parse(line);
+      if (event?.type !== "turn.failed" || !event.error) return false;
+      const details = extractNestedProviderErrorDetails(event.error);
+      return !!details && details.param === "tools" && details.code === "unknown_parameter";
+    } catch {
+      return false;
+    }
+  });
+}
+
 /**
  * Determines if the collected output shows that Codex's internal stream-reconnect
  * retries are exhausted (i.e., the output contains "Reconnecting... N/N" where both
@@ -778,6 +855,7 @@ async function main() {
       const isMissingApiKey = isMissingApiKeyError(result.output);
       const isServer = isServerError(result.output);
       const isInvalidModel = isInvalidModelError(result.output);
+      const isUnsupportedModelTools = isUnsupportedModelToolsError(result.output);
       const isInvalidRequest = isInvalidRequestError(result.output);
       const permissionDeniedCount = countPermissionDeniedIssues(result.output);
       const hasNumerousPermissionDenied = hasNumerousPermissionDeniedIssues(result.output);
@@ -792,6 +870,7 @@ async function main() {
           ` isMissingApiKeyError=${isMissingApiKey}` +
           ` isServerError=${isServer}` +
           ` isInvalidModelError=${isInvalidModel}` +
+          ` isUnsupportedModelToolsError=${isUnsupportedModelTools}` +
           ` isInvalidRequestError=${isInvalidRequest}` +
           ` permissionDeniedCount=${permissionDeniedCount}` +
           ` hasNumerousPermissionDenied=${hasNumerousPermissionDenied}` +
@@ -849,6 +928,15 @@ async function main() {
         return { action: "stop" };
       }
 
+      if (isUnsupportedModelTools) {
+        log(
+          `attempt ${attempt + 1}: configured model does not support Codex's required tool-calling schema` +
+            ` ("tools" param rejected with code "unknown_parameter") — not retrying` +
+            ` (pick a model documented as compatible with Codex CLI, or remove the \`model:\` override in workflow frontmatter to use the engine default)`
+        );
+        return { action: "stop" };
+      }
+
       if (isInvalidRequest) {
         log(`attempt ${attempt + 1}: invalid_request_error (HTTP 400) — not retrying (the provider rejected the request payload; an identical fresh run would fail the same way)`);
         return { action: "stop" };
@@ -913,6 +1001,7 @@ if (typeof module !== "undefined" && module.exports) {
     isMissingApiKeyError,
     isServerError,
     isInvalidModelError,
+    isUnsupportedModelToolsError,
     isInvalidRequestError,
     isReconnectExhaustedError,
     countPermissionDeniedIssues,

actions/setup/js/codex_harness.test.cjs

@@ -15,6 +15,7 @@ const {
   isMissingApiKeyError,
   isServerError,
   isInvalidModelError,
+  isUnsupportedModelToolsError,
   isInvalidRequestError,
   isReconnectExhaustedError,
   countPermissionDeniedIssues,
@@ -609,6 +610,39 @@ env_key = "OPENAI_API_KEY"
     });
   });
 
+  describe("isUnsupportedModelToolsError", () => {
+    it("returns true for the observed 'tools' unknown_parameter turn.failed event", () => {
+      const output =
+        '{"type":"thread.started","thread_id":"01a0545e-6060-7472-9d50-d4a643611434"}\n' +
+        '{"type":"turn.started"}\n' +
+        String.raw`{"type":"error","message":"{\n  \"error\": {\n    \"message\": \"Invalid value: 'custom'\",\n    \"type\": \"invalid_request_error\",\n    \"param\": \"tools\",\n    \"code\": \"unknown_parameter\"\n  }\n}"}` +
+        "\n" +
+        String.raw`{"type":"turn.failed","error":{"message":"{\n  \"error\": {\n    \"message\": \"Invalid value: 'custom'\",\n    \"type\": \"invalid_request_error\",\n    \"param\": \"tools\",\n    \"code\": \"unknown_parameter\"\n  }\n}"}}`;
+      expect(isUnsupportedModelToolsError(output)).toBe(true);
+    });
+
+    it("returns true regardless of the order of param/code fields", () => {
+      const output = '{"type":"turn.failed","error":{"code":"unknown_parameter","param":"tools"}}';
+      expect(isUnsupportedModelToolsError(output)).toBe(true);
+    });
+
+    it("returns true for a provider metadata.raw envelope", () => {
+      const output = String.raw`{"type":"turn.failed","error":{"message":"{\"error\":{\"message\":\"Provider returned error\",\"code\":400,\"metadata\":{\"raw\":\"{\\\"type\\\": \\\"invalid_request_error\\\",\\n    \\\"param\\\": \\\"tools\\\",\\n    \\\"code\\\": \\\"unknown_parameter\\\"}\"}}}"}}`;
+      expect(isUnsupportedModelToolsError(output)).toBe(true);
+    });
+
+    it("returns false for an unrelated invalid_request_error (e.g. empty message array)", () => {
+      const output = String.raw`{"type":"turn.failed","error":{"message":"{\"error\":{\"message\":\"Provider returned error\",\"code\":400,\"metadata\":{\"raw\":\"{\\\"type\\\": \\\"invalid_request_error\\\",\\n    \\\"param\\\": \\\"messages[4].content\\\",\\n    \\\"code\\\": \\\"empty_array\\\"}\"}}}"}}`;
+      expect(isUnsupportedModelToolsError(output)).toBe(false);
+    });
+
+    it("returns false for unrelated errors and empty output", () => {
+      expect(isUnsupportedModelToolsError("rate_limit_exceeded")).toBe(false);
+      expect(isUnsupportedModelToolsError('{"type":"item.completed","item":{"type":"tool_call_output","output":"unknown_parameter tools"}}')).toBe(false);
+      expect(isUnsupportedModelToolsError("")).toBe(false);
+    });
+  });
+
   describe("permission-denied classification helpers", () => {
     it("counts repeated permission-denied signals", () => {
       const output = "permission denied\npermissions denied\nEACCES: permission denied";
@@ -747,6 +781,45 @@ process.exit(1);`,
       expect(result.stderr).toContain("invalid_request_error (HTTP 400) — not retrying");
     });
 
+    it("does not retry the observed unsupported-model 'tools' unknown_parameter failure", () => {
+      const tempDir = makeHarnessTempDir("codex-unsupported-model-tools-");
+      const stubPath = path.join(tempDir, "stub.cjs");
+      const promptPath = path.join(tempDir, "prompt.txt");
+      const callsPath = path.join(tempDir, "calls.jsonl");
+      fs.writeFileSync(
+        stubPath,
+        `const fs = require("fs");
+fs.appendFileSync(process.env.CODEX_HARNESS_STUB_CALLS, "called\\n");
+process.stderr.write(JSON.stringify({ type: "thread.started", thread_id: "01a0545e-6060-7472-9d50-d4a643611434" }) + "\\n");
+process.stderr.write(JSON.stringify({ type: "turn.started" }) + "\\n");
+const errorMessage = JSON.stringify({ error: { message: "Invalid value: 'custom'", type: "invalid_request_error", param: "tools", code: "unknown_parameter" } });
+process.stderr.write(JSON.stringify({ type: "error", message: errorMessage }) + "\\n");
+process.stderr.write(JSON.stringify({ type: "turn.failed", error: { message: errorMessage } }) + "\\n");
+process.exit(1);`,
+        "utf8"
+      );
+      fs.writeFileSync(promptPath, "fix the bug", "utf8");
+
+      const result = spawnSync(process.execPath, ["codex_harness.cjs", process.execPath, stubPath, "exec", "--prompt-file", promptPath], {
+        cwd: path.dirname(require.resolve("./codex_harness.cjs")),
+        env: {
+          ...process.env,
+          CODEX_HARNESS_STUB_CALLS: callsPath,
+          CODEX_API_KEY: "fake-key-for-test",
+          GH_AW_HARNESS_MAX_RETRIES: "1",
+          GH_AW_HARNESS_INITIAL_DELAY_MS: "1",
+        },
+        encoding: "utf8",
+        timeout: 10000,
+      });
+
+      expect(fs.readFileSync(callsPath, "utf8").trim().split("\n")).toHaveLength(1);
+      expect(result.status).toBe(1);
+      expect(result.stderr).toContain("isUnsupportedModelToolsError=true");
+      expect(result.stderr).toContain("configured model does not support Codex's required tool-calling schema");
+      expect(result.stderr).toContain("not retrying");
+    });
+
     it("exits 0 when the AWF API proxy returns HTTP 403 max-AI-credits as an authentication failure", () => {
       // Same proxy signature as the claude_harness.test.cjs regression, replayed against codex.
       // CODEX_API_KEY is set so the `!isMissingApiKey` guard cannot suppress the budget path.