Add tool output consumption rate trajectory grader (#57252)

Author: Copilot

.github/workflows/shared/graders/README.md

@@ -45,7 +45,7 @@ to `Implemented` in the same PR that adds `shared/graders/<id>.md`.
 | 2 | `skill-constraint-coverage` | Precompiled constraints | Implemented |
 | 3 | `exploration-error` | State/task model | Implemented |
 | 4 | `exploitation-error` | State/task model | Implemented |
-| 12 | `tool-output-consumption-rate` | Provenance/reference IDs | Not started |
+| 12 | `tool-output-consumption-rate` | Provenance/reference IDs | Implemented |
 | 13 | `end-to-end-lineage-completeness` | Provenance graph | Not started |
 | 14 | `action-provenance-coverage` | Provenance graph | Not started |
 | 15 | `premature-termination-gap` | Completion predicates | Not started |

.github/workflows/shared/graders/tool-output-consumption-rate.md

@@ -0,0 +1,92 @@
+---
+graders:
+  # Measures the fraction of tool-originated observations that were actually
+  # referenced by a later action, per observations[].consumedByActionIds in
+  # the canonical Trajectory IR. A low rate indicates the agent frequently
+  # called tools and then ignored their outputs (wasted or speculative tool
+  # calls); a high rate indicates most tool outputs fed into subsequent
+  # decisions. Higher is better.
+  tool-output-consumption-rate:
+    name: Tool Output Consumption Rate
+    unit: ratio
+    direction: higher_is_better
+    min: 0.0
+    max: 1.0
+    script: |
+      const isRecord = value => value !== null && typeof value === "object" && !Array.isArray(value);
+      const candidates = [
+        trace,
+        trace.trajectoryIR,
+        trace.trajectoryIr,
+        trace.ir,
+        isRecord(trace.agentOutput) ? trace.agentOutput.trajectoryIR : null,
+        isRecord(trace.agentOutput) ? trace.agentOutput.trajectoryIr : null,
+        isRecord(trace.agentOutput) ? trace.agentOutput.trajectory : null,
+        isRecord(trace.agentOutput) ? trace.agentOutput : null,
+      ].filter(isRecord);
+
+      const candidate =
+        candidates.find(value =>
+          Array.isArray(value.observations) &&
+          value.observations.some(isRecord) &&
+          Array.isArray(value.toolCalls) &&
+          value.toolCalls.some(isRecord)
+        ) ??
+        // Preserve observations-only traces so missing toolCalls is reported
+        // as no tool-originated observations rather than no observations.
+        candidates.find(value => Array.isArray(value.observations) && value.observations.some(isRecord)) ??
+        candidates.find(value => Array.isArray(value.observations)) ??
+        null;
+      const observations = (candidate && Array.isArray(candidate.observations) ? candidate.observations : []).filter(isRecord);
+
+      if (observations.length === 0) {
+        return { value: null, unit: "ratio", passed: null, message: "not applicable: no observations in the trace" };
+      }
+
+      const toolCalls = (candidate && Array.isArray(candidate.toolCalls) ? candidate.toolCalls : []).filter(isRecord);
+      const toolCallIds = new Set(
+        toolCalls
+          .map(toolCall => toolCall.id)
+          .filter(id => typeof id === "string" && id !== "")
+      );
+      const toolObservations = observations.filter(
+        observation =>
+          typeof observation.sourceToolCallId === "string" &&
+          observation.sourceToolCallId !== "" &&
+          toolCallIds.has(observation.sourceToolCallId)
+      );
+
+      if (toolObservations.length === 0) {
+        return { value: null, unit: "ratio", passed: null, message: "not applicable: no tool-originated observations in the trace" };
+      }
+
+      const isConsumed = observation =>
+        Array.isArray(observation.consumedByActionIds) &&
+        observation.consumedByActionIds.some(actionId => typeof actionId === "string" && actionId !== "");
+
+      const consumed = toolObservations.filter(isConsumed);
+      const unconsumedIds = toolObservations
+        .filter(observation => !isConsumed(observation))
+        .slice(0, 5)
+        .map(observation =>
+          typeof observation.id === "string" && observation.id !== "" ? observation.id : observation.sourceToolCallId
+        );
+
+      return {
+        value: helpers.ratio(consumed.length, toolObservations.length),
+        unit: "ratio",
+        details: `toolObservations=${toolObservations.length} consumed=${consumed.length}${unconsumedIds.length === 0 ? "" : `; unconsumed: ${unconsumedIds.join(", ")}`}`,
+      };
+---
+
+<!--
+tool-output-consumption-rate computes the fraction of tool-originated
+observations whose consumedByActionIds array is non-empty in the canonical
+Trajectory IR, i.e. the fraction of tool outputs that a later action
+actually referenced. Depends on observations[].sourceToolCallId,
+observations[].consumedByActionIds, and toolCalls[].id from the IR. Reports
+not-applicable (passed: null) when the trace has no observations, or no
+observations that originate from a matching tool call, rather than fabricating
+a value. Complements the built-in tool-success-rate grader (which measures
+whether tool calls succeeded, not whether their outputs were used).
+-->

actions/setup/js/trace_graders.test.cjs

@@ -141,6 +141,24 @@ function runSkillConstraintCoverage(trace, config) {
   });
 }
 
+const toolOutputConsumptionRateScriptMatch = fs.readFileSync(path.join(__dirname, "../../../.github/workflows/shared/graders/tool-output-consumption-rate.md"), "utf8").match(/script: \|\n([\s\S]*?)\n^---\s*$/m);
+if (!toolOutputConsumptionRateScriptMatch?.[1]) {
+  throw new Error("unable to extract tool-output-consumption-rate grader script");
+}
+const toolOutputConsumptionRateScript = toolOutputConsumptionRateScriptMatch[1]
+  .split("\n")
+  .map(line => line.slice(6))
+  .join("\n");
+
+function runToolOutputConsumptionRate(trace) {
+  return runCustomGrader("tool-output-consumption-rate", toolOutputConsumptionRateScript, makeTrace(trace), {
+    name: "Tool Output Consumption Rate",
+    unit: "ratio",
+    direction: "higher_is_better",
+    source: "inline",
+  });
+}
+
 describe("trace_graders", () => {
   describe("buildGradersSummaryBody", () => {
     it("renders all computed grader values without emojis", () => {
@@ -1008,6 +1026,98 @@ describe("trace_graders", () => {
     });
   });
 
+  describe("tool-output-consumption-rate custom grader", () => {
+    it("scores the fraction of matching tool observations that were consumed", () => {
+      const result = runToolOutputConsumptionRate({
+        trajectoryIR: {
+          toolCalls: [{ id: "tc-1" }, { id: "tc-2" }],
+          observations: [
+            { id: "obs-1", sourceToolCallId: "tc-1", consumedByActionIds: ["act-1"] },
+            { id: "obs-2", sourceToolCallId: "tc-2", consumedByActionIds: [] },
+            { id: "obs-3", sourceToolCallId: null, consumedByActionIds: ["act-2"] },
+            { id: "obs-4", sourceToolCallId: "unknown", consumedByActionIds: ["act-3"] },
+          ],
+        },
+      });
+
+      expect(result.value).toBeCloseTo(0.5);
+      expect(result.details).toContain("toolObservations=2 consumed=1");
+      expect(result.details).toContain("unconsumed: obs-2");
+    });
+
+    it("treats malformed consumption metadata as unconsumed", () => {
+      const result = runToolOutputConsumptionRate({
+        trajectoryIR: {
+          toolCalls: [{ id: "tc-1" }],
+          observations: [{ id: "obs-1", sourceToolCallId: "tc-1", consumedByActionIds: "act-1" }],
+        },
+      });
+
+      expect(result.value).toBe(0);
+      expect(result.details).toContain("toolObservations=1 consumed=0");
+    });
+
+    it("treats consumedByActionIds arrays containing only non-string/empty entries as unconsumed", () => {
+      const result = runToolOutputConsumptionRate({
+        trajectoryIR: {
+          toolCalls: [{ id: "tc-1" }, { id: "tc-2" }],
+          observations: [
+            { id: "obs-1", sourceToolCallId: "tc-1", consumedByActionIds: [null] },
+            { id: "obs-2", sourceToolCallId: "tc-2", consumedByActionIds: [42, ""] },
+          ],
+        },
+      });
+
+      expect(result.value).toBe(0);
+      expect(result.details).toContain("toolObservations=2 consumed=0");
+      expect(result.details).toContain("unconsumed: obs-1, obs-2");
+    });
+
+    it("reads observations/toolCalls from the root trace object", () => {
+      const result = runToolOutputConsumptionRate({
+        toolCalls: [{ id: "tc-1" }],
+        observations: [{ id: "obs-1", sourceToolCallId: "tc-1", consumedByActionIds: ["act-1"] }],
+      });
+
+      expect(result.value).toBe(1);
+    });
+
+    it("reads a complete IR nested in agentOutput", () => {
+      const result = runToolOutputConsumptionRate({
+        agentOutput: {
+          trajectoryIR: {
+            toolCalls: [{ id: "tc-1" }],
+            observations: [{ id: "obs-1", sourceToolCallId: "tc-1", consumedByActionIds: ["act-1"] }],
+          },
+        },
+      });
+
+      expect(result.value).toBe(1);
+    });
+
+    it.each([
+      ["no observations", { trajectoryIR: { toolCalls: [{ id: "tc-1" }] } }, "no observations"],
+      ["no tool calls", { trajectoryIR: { observations: [{ id: "obs-1", sourceToolCallId: "tc-1", consumedByActionIds: ["act-1"] }] } }, "no tool-originated observations"],
+      [
+        "no matching tool call",
+        {
+          trajectoryIR: {
+            toolCalls: [{ id: "tc-1" }],
+            observations: [{ id: "obs-1", sourceToolCallId: "unknown", consumedByActionIds: ["act-1"] }],
+          },
+        },
+        "no tool-originated observations",
+      ],
+    ])("normalizes %s as unavailable", (_name, trace, message) => {
+      const result = runToolOutputConsumptionRate(trace);
+
+      expect(result.value).toBeNull();
+      expect(result.passed).toBeNull();
+      expect(result.status).toBe("unavailable");
+      expect(result.message).toContain(message);
+    });
+  });
+
   // --- Hostile data ---
   describe("hostile data handling", () => {
     it("handles hostile strings in tool call names", () => {