Report eval results in OpenTelemetry spans (#57166)

* Plan eval telemetry reporting

Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>

* Report eval results in OTLP spans

Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>

* Finalize eval OTLP reporting

Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>

* Normalize whitespace in eval telemetry answers

Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>
Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com>

Author: 3 people

.changeset/report-evals-otel.md

@@ -754,6 +754,41 @@ function buildGraderTelemetry(graderOutput, eventTimeMs) {
   return { attributes, events };
 }
 
+/**
+ * Build summary attributes and per-result events from BinEval JSONL records.
+ * Only bounded, structured fields are included; free-form questions are excluded.
+ *
+ * @param {any[]} evalResults
+ * @param {number} eventTimeMs
+ * @returns {{attributes: Array<{key: string, value: object}>, events: Array<{timeUnixNano: string, name: string, attributes: Array<{key: string, value: object}>}>}}
+ */
+function buildEvalTelemetry(evalResults, eventTimeMs) {
+  if (!Array.isArray(evalResults)) {
+    return { attributes: [], events: [] };
+  }
+  const results = evalResults.filter(result => result && typeof result === "object" && typeof result.id === "string" && result.id);
+  if (results.length === 0) {
+    return { attributes: [], events: [] };
+  }
+
+  const normalizedAnswers = results.map(result => {
+    const answer = typeof result.answer === "string" ? result.answer.trim().toUpperCase() : "UNKNOWN";
+    return answer === "YES" || answer === "NO" ? answer : "UNKNOWN";
+  });
+  const countAnswer = answer => normalizedAnswers.filter(value => value === answer).length;
+  const attributes = [buildAttr("gh-aw.evals.count", results.length), buildAttr("gh-aw.evals.yes", countAnswer("YES")), buildAttr("gh-aw.evals.no", countAnswer("NO")), buildAttr("gh-aw.evals.unknown", countAnswer("UNKNOWN"))];
+  const timeUnixNano = toNanoString(eventTimeMs);
+  const events = results.map((result, index) => {
+    const resultAttributes = [buildAttr("gh-aw.eval.id", result.id), buildAttr("gh-aw.eval.answer", normalizedAnswers[index])];
+    if (typeof result.model === "string" && result.model) {
+      resultAttributes.push(buildAttr("gh-aw.eval.model", result.model));
+    }
+    return { timeUnixNano, name: "eval.result", attributes: resultAttributes };
+  });
+
+  return { attributes, events };
+}
+
 // ---------------------------------------------------------------------------
 // Custom OTLP attributes (GH_AW_OTLP_ATTRIBUTES)
 // ---------------------------------------------------------------------------
@@ -1496,6 +1531,20 @@ function readJSONIfExists(filePath) {
   }
 }
 
+/**
+ * Safely read and parse a JSONL file. Returns an empty array on any error.
+ *
+ * @param {string} filePath - Absolute path to the JSONL file
+ * @returns {any[]}
+ */
+function readJSONLIfExists(filePath) {
+  try {
+    return parseJsonlContent(fs.readFileSync(filePath, "utf8"));
+  } catch {
+    return [];
+  }
+}
+
 /**
  * Path to the GitHub rate-limit JSONL log file.
  * Mirrors GITHUB_RATE_LIMITS_JSONL_PATH from constants.cjs without introducing
@@ -2376,6 +2425,7 @@ async function sendJobConclusionSpan(spanName, options = {}) {
   }
 
   const graderTelemetry = jobName === "agent" ? buildGraderTelemetry(readJSONIfExists("/tmp/gh-aw/agent/graders/grader_results.json"), endMs) : { attributes: [], events: [] };
+  const evalTelemetry = jobName === "evals" ? buildEvalTelemetry(readJSONLIfExists("/tmp/gh-aw/evals.jsonl"), endMs) : { attributes: [], events: [] };
 
   const resourceAttributes = buildGitHubActionsResourceAttributes({
     repository,
@@ -2435,7 +2485,7 @@ async function sendJobConclusionSpan(spanName, options = {}) {
       });
   };
 
-  const spanEvents = [...buildSpanEvents(endMs), ...graderTelemetry.events];
+  const spanEvents = [...buildSpanEvents(endMs), ...graderTelemetry.events, ...evalTelemetry.events];
 
   // Prefer the timestamp written at the very beginning of the Execute Agent CLI step
   // (captures true step start on the host, before the AWF container launches) so the
@@ -2534,6 +2584,7 @@ async function sendJobConclusionSpan(spanName, options = {}) {
   // conclusion span, rather than its dedicated child span or downstream jobs
   // which may have downloaded the agent artifact.
   attributes.push(...graderTelemetry.attributes);
+  attributes.push(...evalTelemetry.attributes);
 
   // Only attach token-usage attributes to jobs that actually executed model usage.
   // Most downstream jobs (conclusion, safe_outputs) may have agent_usage.json on
@@ -2620,6 +2671,7 @@ module.exports = {
   appendToOTLPJSONL,
   buildExperimentAttributes,
   buildGraderTelemetry,
+  buildEvalTelemetry,
   parseOTLPCustomAttributes,
   buildCustomOTLPAttributes,
 };

actions/setup/js/send_otlp_span.cjs

@@ -36,6 +36,7 @@ const {
   buildEpisodeAttributesFromContext,
   buildExperimentAttributes,
   buildGraderTelemetry,
+  buildEvalTelemetry,
   hasProxyConfigured,
   resolveEngineId,
   parseOTLPCustomAttributes,
@@ -2669,6 +2670,30 @@ describe("sendJobConclusionSpan", () => {
     expect((spans[1].events ?? []).map(event => event.name)).not.toContain("grader.result");
   });
 
+  it("emits eval results only on the evals job conclusion span", async () => {
+    const mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" });
+    vi.stubGlobal("fetch", mockFetch);
+    process.env.GH_AW_OTLP_ENDPOINTS = JSON.stringify([{ url: "https://traces.example.com" }]);
+    process.env.INPUT_JOB_NAME = "evals";
+    const readFileSpy = vi.spyOn(fs, "readFileSync").mockImplementation(filePath => {
+      if (filePath === "/tmp/gh-aw/evals.jsonl") {
+        return `${JSON.stringify({ id: "quality", answer: "YES", model: "gpt-5" })}\n`;
+      }
+      throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
+    });
+
+    await sendJobConclusionSpan("gh-aw.evals.conclusion");
+    process.env.INPUT_JOB_NAME = "conclusion";
+    await sendJobConclusionSpan("gh-aw.conclusion.conclusion");
+    readFileSpy.mockRestore();
+
+    const spans = mockFetch.mock.calls.map(([, request]) => JSON.parse(request.body).resourceSpans[0].scopeSpans[0].spans[0]);
+    expect(spans[0].attributes).toContainEqual(buildAttr("gh-aw.evals.count", 1));
+    expect(spans[0].events).toContainEqual(expect.objectContaining({ name: "eval.result" }));
+    expect(spans[1].attributes.map(attribute => attribute.key)).not.toContain("gh-aw.evals.count");
+    expect((spans[1].events ?? []).map(event => event.name)).not.toContain("eval.result");
+  });
+
   it("emits live episode attributes on conclusion spans from aw_info workflow_call context", async () => {
     const mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" });
     vi.stubGlobal("fetch", mockFetch);
@@ -6568,6 +6593,35 @@ describe("buildGraderTelemetry", () => {
   });
 });
 
+// ---------------------------------------------------------------------------
+// buildEvalTelemetry
+// ---------------------------------------------------------------------------
+
+describe("buildEvalTelemetry", () => {
+  it("builds summary attributes and one event per eval result", () => {
+    const telemetry = buildEvalTelemetry(
+      [
+        { id: "quality", question: "Sensitive free-form content", answer: " yes ", model: "gpt-5" },
+        { id: "tests", answer: "NO" },
+        { id: "unknown", answer: "MAYBE" },
+      ],
+      1700000000000
+    );
+
+    expect(telemetry.attributes).toEqual([buildAttr("gh-aw.evals.count", 3), buildAttr("gh-aw.evals.yes", 1), buildAttr("gh-aw.evals.no", 1), buildAttr("gh-aw.evals.unknown", 1)]);
+    expect(telemetry.events[0]).toEqual({
+      timeUnixNano: toNanoString(1700000000000),
+      name: "eval.result",
+      attributes: [buildAttr("gh-aw.eval.id", "quality"), buildAttr("gh-aw.eval.answer", "YES"), buildAttr("gh-aw.eval.model", "gpt-5")],
+    });
+    expect(JSON.stringify(telemetry)).not.toContain("Sensitive free-form content");
+  });
+
+  it.each([null, undefined, [], [null, {}, { id: "" }]])("returns empty telemetry without valid results", results => {
+    expect(buildEvalTelemetry(results, 1)).toEqual({ attributes: [], events: [] });
+  });
+});
+
 // ---------------------------------------------------------------------------
 // parseOTLPEndpoints
 // ---------------------------------------------------------------------------

actions/setup/js/send_otlp_span.test.cjs

@@ -288,16 +288,16 @@
         "statuses": {
           "$ref": "#/definitions/permissions-level"
         },
+        "vulnerability-alerts": {
+          "type": "string",
+          "enum": ["read", "none"]
+        },
         "copilot-requests": {
           "type": "string",
           "enum": ["write", "none"]
         },
         "drives": {
           "$ref": "#/definitions/permissions-level"
-        },
-        "vulnerability-alerts": {
-          "type": "string",
-          "enum": ["read", "none"]
         }
       }
     },