Report eval results in OTLP spans

Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>

Author: Copilot

.changeset/report-evals-otel.md

@@ -754,6 +754,41 @@ function buildGraderTelemetry(graderOutput, eventTimeMs) {
   return { attributes, events };
 }
 
+/**
+ * Build summary attributes and per-result events from BinEval JSONL records.
+ * Only bounded, structured fields are included; free-form questions are excluded.
+ *
+ * @param {unknown[]} evalResults
+ * @param {number} eventTimeMs
+ * @returns {{attributes: Array<{key: string, value: object}>, events: Array<{timeUnixNano: string, name: string, attributes: Array<{key: string, value: object}>}>}}
+ */
+function buildEvalTelemetry(evalResults, eventTimeMs) {
+  if (!Array.isArray(evalResults)) {
+    return { attributes: [], events: [] };
+  }
+  const results = evalResults.filter(result => result && typeof result === "object" && typeof result.id === "string" && result.id);
+  if (results.length === 0) {
+    return { attributes: [], events: [] };
+  }
+
+  const normalizedAnswers = results.map(result => {
+    const answer = typeof result.answer === "string" ? result.answer.toUpperCase() : "UNKNOWN";
+    return answer === "YES" || answer === "NO" ? answer : "UNKNOWN";
+  });
+  const countAnswer = answer => normalizedAnswers.filter(value => value === answer).length;
+  const attributes = [buildAttr("gh-aw.evals.count", results.length), buildAttr("gh-aw.evals.yes", countAnswer("YES")), buildAttr("gh-aw.evals.no", countAnswer("NO")), buildAttr("gh-aw.evals.unknown", countAnswer("UNKNOWN"))];
+  const timeUnixNano = toNanoString(eventTimeMs);
+  const events = results.map((result, index) => {
+    const resultAttributes = [buildAttr("gh-aw.eval.id", result.id), buildAttr("gh-aw.eval.answer", normalizedAnswers[index])];
+    if (typeof result.model === "string" && result.model) {
+      resultAttributes.push(buildAttr("gh-aw.eval.model", result.model));
+    }
+    return { timeUnixNano, name: "eval.result", attributes: resultAttributes };
+  });
+
+  return { attributes, events };
+}
+
 // ---------------------------------------------------------------------------
 // Custom OTLP attributes (GH_AW_OTLP_ATTRIBUTES)
 // ---------------------------------------------------------------------------
@@ -2376,6 +2411,19 @@ async function sendJobConclusionSpan(spanName, options = {}) {
   }
 
   const graderTelemetry = jobName === "agent" ? buildGraderTelemetry(readJSONIfExists("/tmp/gh-aw/agent/graders/grader_results.json"), endMs) : { attributes: [], events: [] };
+  const evalTelemetry =
+    jobName === "evals"
+      ? buildEvalTelemetry(
+          (() => {
+            try {
+              return parseJsonlContent(fs.readFileSync("/tmp/gh-aw/evals.jsonl", "utf8"));
+            } catch {
+              return [];
+            }
+          })(),
+          endMs
+        )
+      : { attributes: [], events: [] };
 
   const resourceAttributes = buildGitHubActionsResourceAttributes({
     repository,
@@ -2435,7 +2483,7 @@ async function sendJobConclusionSpan(spanName, options = {}) {
       });
   };
 
-  const spanEvents = [...buildSpanEvents(endMs), ...graderTelemetry.events];
+  const spanEvents = [...buildSpanEvents(endMs), ...graderTelemetry.events, ...evalTelemetry.events];
 
   // Prefer the timestamp written at the very beginning of the Execute Agent CLI step
   // (captures true step start on the host, before the AWF container launches) so the
@@ -2534,6 +2582,7 @@ async function sendJobConclusionSpan(spanName, options = {}) {
   // conclusion span, rather than its dedicated child span or downstream jobs
   // which may have downloaded the agent artifact.
   attributes.push(...graderTelemetry.attributes);
+  attributes.push(...evalTelemetry.attributes);
 
   // Only attach token-usage attributes to jobs that actually executed model usage.
   // Most downstream jobs (conclusion, safe_outputs) may have agent_usage.json on
@@ -2620,6 +2669,7 @@ module.exports = {
   appendToOTLPJSONL,
   buildExperimentAttributes,
   buildGraderTelemetry,
+  buildEvalTelemetry,
   parseOTLPCustomAttributes,
   buildCustomOTLPAttributes,
 };

actions/setup/js/send_otlp_span.cjs

@@ -36,6 +36,7 @@ const {
   buildEpisodeAttributesFromContext,
   buildExperimentAttributes,
   buildGraderTelemetry,
+  buildEvalTelemetry,
   hasProxyConfigured,
   resolveEngineId,
   parseOTLPCustomAttributes,
@@ -2669,6 +2670,30 @@ describe("sendJobConclusionSpan", () => {
     expect((spans[1].events ?? []).map(event => event.name)).not.toContain("grader.result");
   });
 
+  it("emits eval results only on the evals job conclusion span", async () => {
+    const mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" });
+    vi.stubGlobal("fetch", mockFetch);
+    process.env.GH_AW_OTLP_ENDPOINTS = JSON.stringify([{ url: "https://traces.example.com" }]);
+    process.env.INPUT_JOB_NAME = "evals";
+    const readFileSpy = vi.spyOn(fs, "readFileSync").mockImplementation(filePath => {
+      if (filePath === "/tmp/gh-aw/evals.jsonl") {
+        return `${JSON.stringify({ id: "quality", answer: "YES", model: "gpt-5" })}\n`;
+      }
+      throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
+    });
+
+    await sendJobConclusionSpan("gh-aw.evals.conclusion");
+    process.env.INPUT_JOB_NAME = "conclusion";
+    await sendJobConclusionSpan("gh-aw.conclusion.conclusion");
+    readFileSpy.mockRestore();
+
+    const spans = mockFetch.mock.calls.map(([, request]) => JSON.parse(request.body).resourceSpans[0].scopeSpans[0].spans[0]);
+    expect(spans[0].attributes).toContainEqual(buildAttr("gh-aw.evals.count", 1));
+    expect(spans[0].events).toContainEqual(expect.objectContaining({ name: "eval.result" }));
+    expect(spans[1].attributes.map(attribute => attribute.key)).not.toContain("gh-aw.evals.count");
+    expect((spans[1].events ?? []).map(event => event.name)).not.toContain("eval.result");
+  });
+
   it("emits live episode attributes on conclusion spans from aw_info workflow_call context", async () => {
     const mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" });
     vi.stubGlobal("fetch", mockFetch);
@@ -6568,6 +6593,35 @@ describe("buildGraderTelemetry", () => {
   });
 });
 
+// ---------------------------------------------------------------------------
+// buildEvalTelemetry
+// ---------------------------------------------------------------------------
+
+describe("buildEvalTelemetry", () => {
+  it("builds summary attributes and one event per eval result", () => {
+    const telemetry = buildEvalTelemetry(
+      [
+        { id: "quality", question: "Sensitive free-form content", answer: "yes", model: "gpt-5" },
+        { id: "tests", answer: "NO" },
+        { id: "unknown", answer: "MAYBE" },
+      ],
+      1700000000000
+    );
+
+    expect(telemetry.attributes).toEqual([buildAttr("gh-aw.evals.count", 3), buildAttr("gh-aw.evals.yes", 1), buildAttr("gh-aw.evals.no", 1), buildAttr("gh-aw.evals.unknown", 1)]);
+    expect(telemetry.events[0]).toEqual({
+      timeUnixNano: toNanoString(1700000000000),
+      name: "eval.result",
+      attributes: [buildAttr("gh-aw.eval.id", "quality"), buildAttr("gh-aw.eval.answer", "YES"), buildAttr("gh-aw.eval.model", "gpt-5")],
+    });
+    expect(JSON.stringify(telemetry)).not.toContain("Sensitive free-form content");
+  });
+
+  it.each([null, undefined, [], [null, {}, { id: "" }]])("returns empty telemetry without valid results", results => {
+    expect(buildEvalTelemetry(results, 1)).toEqual({ attributes: [], events: [] });
+  });
+});
+
 // ---------------------------------------------------------------------------
 // parseOTLPEndpoints
 // ---------------------------------------------------------------------------