Surface AWF steering counters in compact audit usage data (#63664)

Author: Copilot

.changeset/minor-audit-gateway-steering.md

@@ -7,13 +7,15 @@
 //   session: aggregate Copilot session event counters
 //   gateway: tool-call counts, sizes, durations, and per-server/tool breakdowns
 //   integrity: aggregate DIFC filtering counts from gateway/RPC logs
+//   steering: aggregate AWF steering-event counts by event type
 //   safe_outputs: total item count and per-type breakdown from safe-output-items manifest
 //   experiments: A/B experiment variant assignments for the current run
 //   working_set: cumulative input-token traffic relative to peak invocation input
 
 const fs = require("fs");
 const path = require("path");
 const { readExperimentAssignments } = require("./experiment_helpers.cjs");
+const { countSteeringEventsByTypeInApiProxyJsonl } = require("./steering_helpers.cjs");
 const { calculateWorkingSetFromJSONL } = require("./working_set_metrics.cjs");
 
 require("./shim.cjs");
@@ -29,6 +31,14 @@ const PLACEHOLDER_DEST_KEY = "-:-";
 const ERROR_DOMAIN_PREFIX = "error:";
 const AGENT_TOKEN_USAGE_PATH = "/tmp/gh-aw/usage/agent/token_usage.jsonl";
 const RPC_EVENT_TO_TYPE = { rpc_request: "REQUEST", rpc_response: "RESPONSE", difc_filtered: "DIFC_FILTERED" };
+const API_PROXY_EVENT_LOG_PATHS = [
+  "/tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/event-logs.jsonl",
+  "/tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/events.jsonl",
+  "/tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/event-logs.jsonl",
+  "/tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/events.jsonl",
+  "/tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/event-logs.jsonl",
+  "/tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/events.jsonl",
+];
 
 function findFiles(rootDir, shouldIncludeFile, maxDepth = Number.POSITIVE_INFINITY, currentDepth = 0) {
   if (!fs.existsSync(rootDir)) {
@@ -335,6 +345,31 @@ function parseSessionLogs(sessionLogDirs = ["/tmp/gh-aw/sandbox/agent/logs/copil
   return session.total_events > 0 ? session : null;
 }
 
+/**
+ * Parse the first AWF API proxy event log with steering events.
+ *
+ * @param {string[]} eventLogPaths
+ * @returns {{ total_events: number, event_counts: Record<string, number> } | null}
+ */
+function parseSteeringEvents(eventLogPaths = API_PROXY_EVENT_LOG_PATHS) {
+  for (const eventLogPath of eventLogPaths) {
+    try {
+      const stat = fs.statSync(eventLogPath);
+      if (!stat || stat.size <= 0) {
+        continue;
+      }
+      const eventCounts = countSteeringEventsByTypeInApiProxyJsonl(fs.readFileSync(eventLogPath, "utf-8"));
+      const totalEvents = Object.values(eventCounts).reduce((total, count) => total + count, 0);
+      if (totalEvents > 0) {
+        return { total_events: totalEvents, event_counts: eventCounts };
+      }
+    } catch {
+      // Ignore missing or unreadable candidate files and try the next layout.
+    }
+  }
+  return null;
+}
+
 /**
  * @param {unknown} value
  * @returns {number}
@@ -809,6 +844,11 @@ function main() {
     summary.integrity = gatewayActivity.integrity;
   }
 
+  const steering = parseSteeringEvents();
+  if (steering) {
+    summary.steering = steering;
+  }
+
   // Parse safe outputs manifest.
   // parseSafeOutputsManifest() has three distinct outcomes that drive the three
   // states downstream consumers need to distinguish:
@@ -870,6 +910,7 @@ if (require.main === module) {
 module.exports = {
   parseFirewallLogs,
   parseSessionLogs,
+  parseSteeringEvents,
   parseGatewayLogs,
   parseGatewayActivity,
   parseSafeOutputsManifest,

actions/setup/js/generate_usage_activity_summary.cjs

@@ -9,7 +9,8 @@ const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
 const req = createRequire(import.meta.url);
-const { parseFirewallLogs, parseSessionLogs, parseGatewayActivity, parseSafeOutputsManifest, parseExperimentsData, calculateWorkingSetFromJSONL, parseWorkingSetMetrics, MANIFEST_FILE_PATH } = req("./generate_usage_activity_summary.cjs");
+const { parseFirewallLogs, parseSessionLogs, parseSteeringEvents, parseGatewayActivity, parseSafeOutputsManifest, parseExperimentsData, calculateWorkingSetFromJSONL, parseWorkingSetMetrics, MANIFEST_FILE_PATH } =
+  req("./generate_usage_activity_summary.cjs");
 
 describe("generate_usage_activity_summary.cjs", () => {
   /** Unique directory for each test to avoid cross-test interference */
@@ -116,6 +117,47 @@ describe("generate_usage_activity_summary.cjs", () => {
     });
   });
 
+  describe("parseSteeringEvents", () => {
+    it("aggregates steering counters from the first available AWF event log", () => {
+      const root = fs.mkdtempSync(path.join(os.tmpdir(), "steering-events-test-"));
+      const missingPath = path.join(root, "missing.jsonl");
+      const eventsPath = path.join(root, "events.jsonl");
+      fs.writeFileSync(eventsPath, ['{"event":"token_steering"}', '{"type":"TOKEN_STEERING"}', '{"event_name":"timeout_steering"}', '{"eventName":"model_steering"}', '{"event":"request"}'].join("\n"));
+
+      try {
+        expect(parseSteeringEvents([missingPath, eventsPath])).toEqual({
+          total_events: 4,
+          event_counts: {
+            model_steering: 1,
+            timeout_steering: 1,
+            token_steering: 2,
+          },
+        });
+      } finally {
+        fs.rmSync(root, { recursive: true, force: true });
+      }
+    });
+
+    it("falls back past empty and steering-free event logs", () => {
+      const root = fs.mkdtempSync(path.join(os.tmpdir(), "steering-events-fallback-test-"));
+      const emptyPath = path.join(root, "event-logs.jsonl");
+      const unrelatedPath = path.join(root, "unrelated.jsonl");
+      const eventsPath = path.join(root, "events.jsonl");
+      fs.writeFileSync(emptyPath, "");
+      fs.writeFileSync(unrelatedPath, '{"event":"request"}\n');
+      fs.writeFileSync(eventsPath, '{"event":"token_steering"}\n');
+
+      try {
+        expect(parseSteeringEvents([emptyPath, unrelatedPath, eventsPath])).toEqual({
+          total_events: 1,
+          event_counts: { token_steering: 1 },
+        });
+      } finally {
+        fs.rmSync(root, { recursive: true, force: true });
+      }
+    });
+  });
+
   describe("parseGatewayActivity", () => {
     it("aggregates RPC v2 tool calls, payload sizes, durations, failures, and integrity filtering", () => {
       const root = fs.mkdtempSync(path.join(os.tmpdir(), "gateway-activity-test-"));

actions/setup/js/generate_usage_activity_summary.test.cjs

@@ -13,9 +13,10 @@ const STEERING_EVENT_PATTERN = /steering/i;
 /**
  * Resolve an event name from a firewall proxy event entry.
  *
- * Supports three log schema variants:
+ * Supports log schema variants used across AWF versions:
  *   - Top-level `event` field: `{ event: "token_steering", ... }`
  *   - Top-level `type` field:  `{ type: "model_steering", ... }`
+ *   - Top-level `event_name` or `eventName` field
  *   - Nested payload:          `{ payload: { event: "steering" } }`
  *
  * @param {unknown} entry
@@ -31,6 +32,12 @@ function getApiProxyEventName(entry) {
   if ("type" in entry && typeof entry.type === "string") {
     return entry.type;
   }
+  if ("event_name" in entry && typeof entry.event_name === "string") {
+    return entry.event_name;
+  }
+  if ("eventName" in entry && typeof entry.eventName === "string") {
+    return entry.eventName;
+  }
   if ("payload" in entry) {
     const payload = entry.payload;
     if (payload && typeof payload === "object" && !Array.isArray(payload)) {
@@ -46,28 +53,39 @@ function getApiProxyEventName(entry) {
 }
 
 /**
- * Count steering events in proxy event-log JSONL content.
+ * Count steering events by normalized event name in proxy event-log JSONL content.
  *
  * Known steering event names: "steering", "token_steering", "model_steering".
  * Any event whose name is exactly "steering" or ends with "_steering" is counted.
  *
  * @param {string} jsonlContent
- * @returns {number}
+ * @returns {Record<string, number>}
  */
-function countSteeringEventsInApiProxyJsonl(jsonlContent) {
-  let count = 0;
+function countSteeringEventsByTypeInApiProxyJsonl(jsonlContent) {
+  /** @type {Record<string, number>} */
+  const counts = {};
   for (const parsed of parseJsonlContent(jsonlContent, line => STEERING_EVENT_PATTERN.test(line))) {
     const eventName = getApiProxyEventName(parsed).toLowerCase();
-    // Known steering events: "steering", "token_steering", "model_steering".
     if (eventName === "steering" || eventName.endsWith("_steering")) {
-      count += 1;
+      counts[eventName] = (counts[eventName] || 0) + 1;
     }
   }
-  return count;
+  return Object.fromEntries(Object.entries(counts).sort(([left], [right]) => left.localeCompare(right)));
+}
+
+/**
+ * Count all steering events in proxy event-log JSONL content.
+ *
+ * @param {string} jsonlContent
+ * @returns {number}
+ */
+function countSteeringEventsInApiProxyJsonl(jsonlContent) {
+  return Object.values(countSteeringEventsByTypeInApiProxyJsonl(jsonlContent)).reduce((total, count) => total + count, 0);
 }
 
 module.exports = {
   STEERING_EVENT_PATTERN,
   getApiProxyEventName,
+  countSteeringEventsByTypeInApiProxyJsonl,
   countSteeringEventsInApiProxyJsonl,
 };

actions/setup/js/steering_helpers.cjs

@@ -1,5 +1,5 @@
 import { describe, it, expect } from "vitest";
-import { getApiProxyEventName, countSteeringEventsInApiProxyJsonl } from "./steering_helpers.cjs";
+import { getApiProxyEventName, countSteeringEventsByTypeInApiProxyJsonl, countSteeringEventsInApiProxyJsonl } from "./steering_helpers.cjs";
 
 describe("steering_helpers", () => {
   describe("getApiProxyEventName", () => {
@@ -19,6 +19,11 @@ describe("steering_helpers", () => {
       expect(getApiProxyEventName({ type: "model_steering", request_id: "r2" })).toBe("model_steering");
     });
 
+    it("returns snake-case and camel-case event name fields", () => {
+      expect(getApiProxyEventName({ event_name: "timeout_steering" })).toBe("timeout_steering");
+      expect(getApiProxyEventName({ eventName: "model_steering" })).toBe("model_steering");
+    });
+
     it("returns payload.event when top-level fields are absent", () => {
       expect(getApiProxyEventName({ payload: { event: "steering" }, request_id: "r3" })).toBe("steering");
     });
@@ -39,6 +44,15 @@ describe("steering_helpers", () => {
   });
 
   describe("countSteeringEventsInApiProxyJsonl", () => {
+    it("aggregates counters for each normalized steering event", () => {
+      const content = ['{"event":"TOKEN_STEERING"}', '{"type":"token_steering"}', '{"event_name":"timeout_steering"}', '{"eventName":"model_steering"}'].join("\n");
+      expect(countSteeringEventsByTypeInApiProxyJsonl(content)).toEqual({
+        model_steering: 1,
+        timeout_steering: 1,
+        token_steering: 2,
+      });
+    });
+
     it("counts events with exact 'steering' name", () => {
       const content = '{"event":"steering","request_id":"r1"}\n';
       expect(countSteeringEventsInApiProxyJsonl(content)).toBe(1);

actions/setup/js/steering_helpers.test.cjs

@@ -0,0 +1,46 @@
+# ADR-63664: Surface AWF steering counters in compact usage data
+
+**Date**: 2026-09-26
+**Status**: Draft
+**Deciders**: gh-aw maintainers
+
+---
+
+### Context
+
+The PR adds support for reading AWF API proxy steering events from multiple log file layouts and schema variants, then backfills those counts into compact usage artifacts and audit output. Today, detailed steering behavior requires raw firewall logs, while compact usage summaries and usage-only audits do not expose per-event steering totals. The changed files span JavaScript artifact generation, Go audit backfill logic, schema updates, tests, and documentation, which indicates a cross-cutting product decision rather than a localized bug fix. The decision is how gh-aw should represent steering behavior when only compact usage artifacts are available.
+
+### Decision
+
+We will aggregate AWF steering events by normalized event name inside the compact usage activity summary and propagate those counters into audit token-usage output when detailed firewall analysis is unavailable. We will support the log filename and schema variants visible in the PR evidence, including `events.jsonl`, `event-logs.jsonl`, and top-level or nested event-name fields. We chose this because it preserves steering visibility for `gh aw audit --artifacts usage` and related usage-only flows without requiring raw firewall log downloads.
+
+### Alternatives Considered
+
+#### Alternative 1: Keep steering analysis only in raw firewall log processing
+
+This was a realistic option because gh-aw already derives detailed steering information from firewall artifacts when those logs are present. It was not chosen because the PR evidence explicitly adds steering data to `usage/activity/summary.json` and backfills audit results from compact usage artifacts, showing that raw-log-only visibility is insufficient for the intended audit workflows.
+
+#### Alternative 2: Publish only a single total steering-event count
+
+This was considered because earlier code paths already tracked `total_steering_events`, and a single aggregate is simpler to compute and document. It was not chosen because the PR updates both JS and Go paths to preserve per-event counters such as `token_steering` and `timeout_steering`, which provides more actionable inspection of AWF behavior than a single total.
+
+### Consequences
+
+#### Positive
+- Usage-only audit flows can report steering behavior even when raw firewall logs are unavailable.
+- Operators gain per-event steering counters, which makes it easier to distinguish token, timeout, model, or other steering causes.
+- The implementation becomes more robust across AWF versions by recognizing multiple log filenames and event-name field variants.
+
+#### Negative
+- Steering parsing logic now exists in both JavaScript artifact-generation code and Go audit-analysis code, which increases maintenance overhead.
+- Supporting multiple historical file layouts and schema variants adds complexity and ongoing compatibility expectations.
+- Compact usage artifacts and schemas become broader, which increases documentation and regression-test surface area.
+
+#### Neutral
+- Audit consumers now need to understand both `total_steering_events` and `steering_event_counts` fields in token-usage output.
+- The change does not replace detailed firewall analysis; it supplements it when only usage artifacts are available.
+- Additional tests are required to keep the JS and Go aggregation paths aligned as AWF logging evolves.
+
+---
+
+*ADR created by [adr-writer agent]. Review and finalize before changing status from Draft to Accepted.*

docs/adr/63664-surface-awf-steering-counters-in-compact-usage-data.md

@@ -259,6 +259,13 @@ Its `activity/summary.json` file uses the `usage-activity-summary/v1` schema. Th
     "filtered_tool_counts": { "issue_read": 2 },
     "filtered_reason_counts": { "integrity": 2 }
   },
+  "steering": {
+    "total_events": 3,
+    "event_counts": {
+      "token_steering": 2,
+      "timeout_steering": 1
+    }
+  },
   "working_set": {
     "measurement_state": "measured",
     "rebuild_factor": 3.9017857142857144,
@@ -325,6 +332,8 @@ label name, and GitHub database or node ID when returned by the API.
 
 The conclusion job derives `gateway` and `integrity` from MCP gateway logs, falling back to `rpc-messages.jsonl` when `gateway.jsonl` is unavailable. These compact aggregates let `gh aw logs --artifacts usage` report MCP call, payload-size, duration, failure, and integrity-filter metrics without downloading raw logs. Cross-run reports include `runs_with_filtered_events`; the existing logs report summary remains the source for the total number of runs.
 
+The `steering` section aggregates AWF API proxy events by normalized event name. `gh aw audit --artifacts usage` exposes these counters in `firewall_token_usage.steering_event_counts`, so steering behavior can be inspected without downloading raw firewall logs.
+
 `rebuild_factor` is `cumulative_input_tokens / peak_input_tokens`, where each invocation contributes the canonical `input_tokens` value from the agent `token_usage.jsonl` record. Cache-read and cache-write fields are not added because provider normalization has already produced that logical input count. The factor is omitted when `measurement_state` is `unavailable`; `partial` means usable records were measured but malformed or unsupported records were ignored.
 
 Working-Set Rebuild Factor measures cumulative context reconstruction relative to peak invocation context. It is an efficiency/trajectory metric, not a measurement of semantic coherence debt and not a predictor of task success. It cannot identify missing task facts or classify outcome quality. The metric is conceptually inspired by [“The Working Set of a Coding Agent: Coherence Debt in Repository-Scale Tasks”](https://arxiv.org/abs/2608.16630), while deliberately limiting the implementation to observable token traffic.

docs/src/content/docs/reference/artifacts.md

@@ -40,6 +40,7 @@ func collectAuditAnalysisResults(ctx context.Context, run WorkflowRun, runOutput
 	}
 	if usageSummary != nil {
 		results.workingSet = usageSummary.WorkingSet
+		applyUsageActivitySteeringSummary(usageSummary.Steering, &results.tokenUsageSummary)
 	}
 	return results, nil
 }