Emit grader results in OpenTelemetry spans (#57015)

Author: Copilot

.changeset/insert-grader-results-otel.md

@@ -708,6 +708,52 @@ function buildExperimentAttributes(assignments) {
   return attrs;
 }
 
+/**
+ * Build summary attributes and per-result events from valid deterministic grader output.
+ * Free-form grader messages, details, and errors are intentionally excluded because
+ * custom graders may derive them from trace content containing sensitive values.
+ *
+ * @param {any} graderOutput
+ * @param {number} eventTimeMs
+ * @returns {{attributes: Array<{key: string, value: object}>, events: Array<{timeUnixNano: string, name: string, attributes: Array<{key: string, value: object}>}>}}
+ */
+function buildGraderTelemetry(graderOutput, eventTimeMs) {
+  if (!graderOutput || typeof graderOutput !== "object" || !Array.isArray(graderOutput.results) || graderOutput.results.length === 0) {
+    return { attributes: [], events: [] };
+  }
+
+  const results = graderOutput.results.filter(result => result && typeof result === "object" && typeof result.id === "string" && result.id);
+  if (results.length === 0) {
+    return { attributes: [], events: [] };
+  }
+
+  const countByStatus = status => results.filter(result => result.status === status).length;
+  const attributes = [
+    buildAttr("gh-aw.graders.count", results.length),
+    buildAttr("gh-aw.graders.passed", countByStatus("pass")),
+    buildAttr("gh-aw.graders.failed", countByStatus("fail")),
+    buildAttr("gh-aw.graders.errors", countByStatus("error")),
+    buildAttr("gh-aw.graders.unavailable", countByStatus("unavailable")),
+    buildAttr("gh-aw.graders.other", results.length - countByStatus("pass") - countByStatus("fail") - countByStatus("error") - countByStatus("unavailable")),
+  ];
+  const timeUnixNano = toNanoString(eventTimeMs);
+  const events = results.map(result => {
+    const resultAttributes = [buildAttr("gh-aw.grader.id", result.id)];
+    if (typeof result.name === "string" && result.name) resultAttributes.push(buildAttr("gh-aw.grader.name", result.name));
+    if (typeof result.status === "string" && result.status) resultAttributes.push(buildAttr("gh-aw.grader.status", result.status));
+    if (typeof result.source === "string" && result.source) resultAttributes.push(buildAttr("gh-aw.grader.source", result.source));
+    if (typeof result.unit === "string" && result.unit) resultAttributes.push(buildAttr("gh-aw.grader.unit", result.unit));
+    if (typeof result.value === "number" && Number.isFinite(result.value)) resultAttributes.push(buildDoubleAttr("gh-aw.grader.value", result.value));
+    if (typeof result.passed === "boolean") resultAttributes.push(buildAttr("gh-aw.grader.passed", result.passed));
+    if (typeof result.severity === "string" && result.severity) resultAttributes.push(buildAttr("gh-aw.grader.severity", result.severity));
+    if (typeof result.baselineValue === "number" && Number.isFinite(result.baselineValue)) resultAttributes.push(buildDoubleAttr("gh-aw.grader.baseline_value", result.baselineValue));
+    if (typeof result.deltaFromBaseline === "number" && Number.isFinite(result.deltaFromBaseline)) resultAttributes.push(buildDoubleAttr("gh-aw.grader.delta_from_baseline", result.deltaFromBaseline));
+    return { timeUnixNano, name: "grader.result", attributes: resultAttributes };
+  });
+
+  return { attributes, events };
+}
+
 // ---------------------------------------------------------------------------
 // Custom OTLP attributes (GH_AW_OTLP_ATTRIBUTES)
 // ---------------------------------------------------------------------------
@@ -1991,6 +2037,8 @@ function readAgentRuntimeMetrics() {
  * - `/tmp/gh-aw/agent_usage.json` – per-type token breakdown written by parse_token_usage.cjs;
  *                                    provides `input_tokens`, `output_tokens`,
  *                                    `cache_read_tokens`, and `cache_write_tokens` counters
+ * - `/tmp/gh-aw/agent/graders/grader_results.json` – deterministic grader
+ *                                    summary attributes and per-result span events
  *
  * @param {string} spanName - OTLP span name (e.g. `"gh-aw.job.conclusion"`)
  * @param {{ startMs?: number }} [options]
@@ -2327,6 +2375,8 @@ async function sendJobConclusionSpan(spanName, options = {}) {
     }
   }
 
+  const graderTelemetry = jobName === "agent" ? buildGraderTelemetry(readJSONIfExists("/tmp/gh-aw/agent/graders/grader_results.json"), endMs) : { attributes: [], events: [] };
+
   const resourceAttributes = buildGitHubActionsResourceAttributes({
     repository,
     runId,
@@ -2385,7 +2435,7 @@ async function sendJobConclusionSpan(spanName, options = {}) {
       });
   };
 
-  const spanEvents = buildSpanEvents(endMs);
+  const spanEvents = [...buildSpanEvents(endMs), ...graderTelemetry.events];
 
   // Prefer the timestamp written at the very beginning of the Execute Agent CLI step
   // (captures true step start on the host, before the AWF container launches) so the
@@ -2480,6 +2530,11 @@ async function sendJobConclusionSpan(spanName, options = {}) {
     }
   }
 
+  // Grader results are run-level outcomes. They belong only on the agent job's
+  // conclusion span, rather than its dedicated child span or downstream jobs
+  // which may have downloaded the agent artifact.
+  attributes.push(...graderTelemetry.attributes);
+
   // Only attach token-usage attributes to jobs that actually executed model usage.
   // Most downstream jobs (conclusion, safe_outputs) may have agent_usage.json on
   // disk via artifact download but must NOT emit token data — otherwise every
@@ -2564,6 +2619,7 @@ module.exports = {
   OTEL_JSONL_PATH,
   appendToOTLPJSONL,
   buildExperimentAttributes,
+  buildGraderTelemetry,
   parseOTLPCustomAttributes,
   buildCustomOTLPAttributes,
 };

actions/setup/js/send_otlp_span.cjs

@@ -35,6 +35,7 @@ const {
   buildCurrentWorkflowCallId,
   buildEpisodeAttributesFromContext,
   buildExperimentAttributes,
+  buildGraderTelemetry,
   hasProxyConfigured,
   resolveEngineId,
   parseOTLPCustomAttributes,
@@ -2644,6 +2645,30 @@ describe("sendJobConclusionSpan", () => {
     expect(span.spanId).toMatch(/^[0-9a-f]{16}$/);
   });
 
+  it("emits graders only on the agent job conclusion span", async () => {
+    const mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" });
+    vi.stubGlobal("fetch", mockFetch);
+    process.env.GH_AW_OTLP_ENDPOINTS = JSON.stringify([{ url: "https://traces.example.com" }]);
+    process.env.INPUT_JOB_NAME = "agent";
+    const readFileSpy = vi.spyOn(fs, "readFileSync").mockImplementation(filePath => {
+      if (filePath === "/tmp/gh-aw/agent/graders/grader_results.json") {
+        return JSON.stringify({ results: [{ id: "quality", status: "pass", value: 0.9 }] });
+      }
+      throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
+    });
+
+    await sendJobConclusionSpan("gh-aw.agent.conclusion");
+    process.env.INPUT_JOB_NAME = "conclusion";
+    await sendJobConclusionSpan("gh-aw.conclusion.conclusion");
+    readFileSpy.mockRestore();
+
+    const spans = mockFetch.mock.calls.map(([, request]) => JSON.parse(request.body).resourceSpans[0].scopeSpans[0].spans[0]);
+    expect(spans[0].attributes).toContainEqual(buildAttr("gh-aw.graders.count", 1));
+    expect(spans[0].events).toContainEqual(expect.objectContaining({ name: "grader.result" }));
+    expect(spans[1].attributes.map(attribute => attribute.key)).not.toContain("gh-aw.graders.count");
+    expect((spans[1].events ?? []).map(event => event.name)).not.toContain("grader.result");
+  });
+
   it("emits live episode attributes on conclusion spans from aw_info workflow_call context", async () => {
     const mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" });
     vi.stubGlobal("fetch", mockFetch);
@@ -2692,6 +2717,9 @@ describe("sendJobConclusionSpan", () => {
       if (filePath === "/tmp/gh-aw/agent_output.json") {
         return JSON.stringify({ items: [{ type: "issue" }, { type: "pull_request" }] });
       }
+      if (filePath === "/tmp/gh-aw/agent/graders/grader_results.json") {
+        return JSON.stringify({ results: [{ id: "quality", status: "pass" }] });
+      }
       throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
     });
 
@@ -2716,6 +2744,9 @@ describe("sendJobConclusionSpan", () => {
     expect(conclusionSpan.parentSpanId).toBe("abcdef1234567890");
     expect(agentSpan.attributes).toContainEqual({ key: "gh-aw.output.item_count", value: { intValue: 2 } });
     expect(conclusionSpan.attributes).toContainEqual({ key: "gh-aw.output.item_count", value: { intValue: 2 } });
+    expect(agentSpan.attributes.map(attribute => attribute.key)).not.toContain("gh-aw.graders.count");
+    expect(conclusionSpan.attributes).toContainEqual(buildAttr("gh-aw.graders.count", 1));
+    expect(conclusionSpan.events).toContainEqual(expect.objectContaining({ name: "grader.result" }));
     const agentKeys = agentSpan.attributes.map(a => a.key);
     const conclusionKeys = conclusionSpan.attributes.map(a => a.key);
     expect(agentKeys).not.toContain("gh-aw.max_ai_credits");
@@ -6449,6 +6480,94 @@ describe("sendJobConclusionSpan", () => {
   });
 });
 
+// ---------------------------------------------------------------------------
+// buildGraderTelemetry
+// ---------------------------------------------------------------------------
+
+describe("buildGraderTelemetry", () => {
+  it("builds summary attributes and one event per grader result", () => {
+    const telemetry = buildGraderTelemetry(
+      {
+        results: [
+          {
+            id: "quality",
+            name: "Quality",
+            value: 0.75,
+            unit: "ratio",
+            passed: true,
+            status: "pass",
+            source: "builtin",
+            severity: "info",
+            baselineValue: 0.5,
+            deltaFromBaseline: 0.25,
+          },
+          { id: "reliability", name: "Reliability", value: null, passed: false, status: "fail", source: "inline" },
+          { id: "broken", status: "error", source: "inline" },
+          { id: "missing", status: "unavailable", source: "builtin" },
+        ],
+      },
+      1700000000000
+    );
+
+    expect(telemetry.attributes).toEqual([
+      buildAttr("gh-aw.graders.count", 4),
+      buildAttr("gh-aw.graders.passed", 1),
+      buildAttr("gh-aw.graders.failed", 1),
+      buildAttr("gh-aw.graders.errors", 1),
+      buildAttr("gh-aw.graders.unavailable", 1),
+      buildAttr("gh-aw.graders.other", 0),
+    ]);
+    expect(telemetry.events).toHaveLength(4);
+    expect(telemetry.events[0]).toEqual({
+      timeUnixNano: toNanoString(1700000000000),
+      name: "grader.result",
+      attributes: [
+        buildAttr("gh-aw.grader.id", "quality"),
+        buildAttr("gh-aw.grader.name", "Quality"),
+        buildAttr("gh-aw.grader.status", "pass"),
+        buildAttr("gh-aw.grader.source", "builtin"),
+        buildAttr("gh-aw.grader.unit", "ratio"),
+        buildDoubleAttr("gh-aw.grader.value", 0.75),
+        buildAttr("gh-aw.grader.passed", true),
+        buildAttr("gh-aw.grader.severity", "info"),
+        buildDoubleAttr("gh-aw.grader.baseline_value", 0.5),
+        buildDoubleAttr("gh-aw.grader.delta_from_baseline", 0.25),
+      ],
+    });
+  });
+
+  it("omits free-form and non-finite grader values", () => {
+    const telemetry = buildGraderTelemetry(
+      {
+        results: [
+          {
+            id: "custom",
+            status: "error",
+            value: Number.NaN,
+            message: "sensitive message",
+            details: "sensitive details",
+            error: "sensitive error",
+          },
+        ],
+      },
+      1
+    );
+
+    const eventKeys = telemetry.events[0].attributes.map(attribute => attribute.key);
+    expect(eventKeys).toEqual(["gh-aw.grader.id", "gh-aw.grader.status"]);
+    expect(JSON.stringify(telemetry)).not.toContain("sensitive");
+  });
+
+  it.each([null, {}, { results: [] }, { results: [null, {}, { id: "" }] }])("returns empty telemetry for output without valid results", output => {
+    expect(buildGraderTelemetry(output, 1)).toEqual({ attributes: [], events: [] });
+  });
+
+  it("counts unrecognized statuses as other", () => {
+    const telemetry = buildGraderTelemetry({ results: [{ id: "skipped", status: "skipped" }] }, 1);
+    expect(telemetry.attributes).toContainEqual(buildAttr("gh-aw.graders.other", 1));
+  });
+});
+
 // ---------------------------------------------------------------------------
 // parseOTLPEndpoints
 // ---------------------------------------------------------------------------

actions/setup/js/send_otlp_span.test.cjs

@@ -524,8 +524,7 @@ function normalizeResult(id, rawResult, meta) {
   if (typeof rawResult === "object" && rawResult !== null && !Array.isArray(rawResult)) {
     // Object result from custom script
     value = rawResult.value;
-    if (rawResult.unit) base.unit = String(rawResult.unit);
-    if (rawResult.severity) base.severity = String(rawResult.severity);
+    if (typeof rawResult.severity === "string" && ["error", "warning", "info", "note"].includes(rawResult.severity)) base.severity = rawResult.severity;
     if (rawResult.details) base.details = String(rawResult.details);
     if (rawResult.message) base.message = String(rawResult.message);
     if (typeof rawResult.passed === "boolean") base.passed = rawResult.passed;

actions/setup/js/trace_graders.cjs

@@ -415,14 +415,19 @@ describe("trace_graders", () => {
       expect(r.status).toBe("fail");
     });
 
-    it("handles object results from custom scripts", () => {
+    it("uses manifest metadata for object results from custom scripts", () => {
       const r = normalizeResult("test", { value: 42, unit: "ms", severity: "warning", details: "too slow" }, { ...meta, source: "inline" });
       expect(r.value).toBe(42);
-      expect(r.unit).toBe("ms");
+      expect(r.unit).toBe("count");
       expect(r.severity).toBe("warning");
       expect(r.details).toBe("too slow");
     });
 
+    it("omits an unrecognized custom severity", () => {
+      const r = normalizeResult("test", { value: 42, severity: "sensitive trace content" }, { ...meta, source: "inline" });
+      expect(r.severity).toBeUndefined();
+    });
+
     it("handles null result as unavailable", () => {
       const r = normalizeResult("test", null, meta);
       expect(r.status).toBe("unavailable");
@@ -572,7 +577,7 @@ describe("trace_graders", () => {
         }
       `;
       const trace = makeTrace({ toolCalls: [{ name: "a" }, { name: "b" }] });
-      const result = runCustomGrader("test", script, trace, meta);
+      const result = runCustomGrader("test", script, trace, { ...meta, unit: "count" });
       expect(result.value).toBe(2);
       expect(result.unit).toBe("count");
     });