Skip to content

Commit 764c9dd

Browse files
authored
Use authoritative AWF AI-credit totals in usage reporting (#56975)
1 parent aa70602 commit 764c9dd

24 files changed

Lines changed: 2046 additions & 345 deletions

.changeset/patch-report-awf-ai-credits.md

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

.github/workflows/pr-code-quality-reviewer.lock.yml

Lines changed: 175 additions & 94 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

.github/workflows/pr-code-quality-reviewer.md

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,8 @@ on:
1616
name: review
1717
events: [pull_request_comment, pull_request_review_comment]
1818
engine:
19-
id: pi
20-
model-provider: openai
21-
model: openai/gpt-5.4
19+
id: copilot
20+
model: copilot/gpt-5.4
2221
permissions:
2322
contents: read
2423
issues: read
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{"_schema":"token-usage/v0.28.7","timestamp":"2026-08-28T08:59:54.002Z","event":"token_usage","request_id":"5c6141a6-7331-400f-94c8-d38defdcbc7e","provider":"copilot","model":"gpt-4o-mini-2024-07-18","path":"/chat/completions","status":200,"streaming":true,"input_tokens":19288,"output_tokens":35,"cache_read_tokens":0,"cache_write_tokens":0,"duration_ms":2242,"response_bytes":8642,"x_initiator":"user","ai_credits_this_response":0.29142,"ai_credits_total":0.29142,"ai_credits_pricing_source":"models.dev","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false}
2+
{"_schema":"token-usage/v0.28.7","timestamp":"2026-08-28T08:59:56.563Z","event":"token_usage","request_id":"cc30e0e7-0491-484e-aaa9-6b020a2707f4","provider":"copilot","model":"gpt-4o-mini-2024-07-18","path":"/chat/completions","status":200,"streaming":true,"input_tokens":19380,"output_tokens":35,"cache_read_tokens":0,"cache_write_tokens":0,"duration_ms":2469,"response_bytes":8642,"x_initiator":"agent","ai_credits_this_response":0.2928,"ai_credits_total":0.58422,"ai_credits_pricing_source":"models.dev","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false}
3+
{"_schema":"token-usage/v0.28.7","timestamp":"2026-08-28T08:59:58.490Z","event":"token_usage","request_id":"bf3f5fcd-173c-4da9-90d9-680dbc836352","provider":"copilot","model":"gpt-4o-mini-2024-07-18","path":"/chat/completions","status":200,"streaming":true,"input_tokens":272,"output_tokens":35,"cache_read_tokens":19200,"cache_write_tokens":0,"duration_ms":1799,"response_bytes":8648,"x_initiator":"agent","ai_credits_this_response":0.15018,"ai_credits_total":0.7344,"ai_credits_pricing_source":"models.dev","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false}
4+
{"_schema":"token-usage/v0.28.7","timestamp":"2026-08-28T09:00:00.426Z","event":"token_usage","request_id":"776b23fd-a45c-46d4-9e74-95d5f0707402","provider":"copilot","model":"gpt-4o-mini-2024-07-18","path":"/chat/completions","status":200,"streaming":true,"input_tokens":236,"output_tokens":35,"cache_read_tokens":19328,"cache_write_tokens":0,"duration_ms":1861,"response_bytes":8648,"x_initiator":"agent","ai_credits_this_response":0.1506,"ai_credits_total":0.885,"ai_credits_pricing_source":"models.dev","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false}
5+
{"_schema":"token-usage/v0.28.7","timestamp":"2026-08-28T09:00:02.132Z","event":"token_usage","request_id":"59db8b91-cfc2-4cb9-9315-dbcb5e7bcfe7","provider":"copilot","model":"gpt-4o-mini-2024-07-18","path":"/chat/completions","status":200,"streaming":true,"input_tokens":200,"output_tokens":35,"cache_read_tokens":19456,"cache_write_tokens":0,"duration_ms":1552,"response_bytes":8648,"x_initiator":"agent","ai_credits_this_response":0.15102,"ai_credits_total":1.03602,"ai_credits_pricing_source":"models.dev","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false}

actions/setup/js/model_costs.cjs

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -190,9 +190,10 @@ function usdToAIC(usd) {
190190
* @param {number} params.cacheReadTokens
191191
* @param {number} params.cacheWriteTokens
192192
* @param {number} [params.reasoningTokens]
193+
* @param {boolean} [params.inputTokensIncludeCache]
193194
* @returns {number}
194195
*/
195-
function computeInferenceCostUSD({ provider, model, inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, reasoningTokens = 0 }) {
196+
function computeInferenceCostUSD({ provider, model, inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, reasoningTokens = 0, inputTokensIncludeCache }) {
196197
const pricing = findModelPricing(provider, model);
197198
if (!pricing) return 0;
198199

@@ -201,7 +202,12 @@ function computeInferenceCostUSD({ provider, model, inputTokens, outputTokens, c
201202
const cacheRead = cacheReadTokens || 0;
202203
const cacheWrite = cacheWriteTokens || 0;
203204
const reasoning = reasoningTokens || 0;
204-
const effectiveInput = providerIncludesCacheReadsInInput(provider) ? Math.max(input - cacheRead, 0) : input;
205+
let effectiveInput = input;
206+
if (inputTokensIncludeCache === true) {
207+
effectiveInput = Math.max(input - cacheRead - cacheWrite, 0);
208+
} else if (typeof inputTokensIncludeCache !== "boolean" && providerIncludesCacheReadsInInput(provider)) {
209+
effectiveInput = Math.max(input - cacheRead, 0);
210+
}
205211

206212
const promptPrice = pricing.input || 0;
207213
const completionPrice = pricing.output || 0;
@@ -221,6 +227,7 @@ function computeInferenceCostUSD({ provider, model, inputTokens, outputTokens, c
221227
* @param {number} params.cacheReadTokens
222228
* @param {number} params.cacheWriteTokens
223229
* @param {number} [params.reasoningTokens]
230+
* @param {boolean} [params.inputTokensIncludeCache]
224231
* @returns {number}
225232
*/
226233
function computeInferenceAIC(params) {

actions/setup/js/model_costs.test.cjs

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,64 @@ describe("model_costs.cjs", () => {
207207
expect(aic).toBeCloseTo(0.336, 6);
208208
});
209209

210+
it("uses explicit inclusive cache semantics when present", async () => {
211+
writeModelsFixture({
212+
"github-copilot": {
213+
models: {
214+
"gpt-4o-mini": {
215+
cost: {
216+
input: "0.00000015",
217+
output: "0.0000006",
218+
cache_read: "0.000000075",
219+
},
220+
},
221+
},
222+
},
223+
});
224+
225+
const { computeInferenceAIC } = await import("./model_costs.cjs");
226+
const aic = computeInferenceAIC({
227+
provider: "copilot",
228+
model: "gpt-4o-mini",
229+
inputTokens: 1000,
230+
outputTokens: 100,
231+
cacheReadTokens: 400,
232+
cacheWriteTokens: 100,
233+
inputTokensIncludeCache: true,
234+
});
235+
236+
expect(aic).toBeCloseTo(0.018, 6);
237+
});
238+
239+
it("uses explicit additive cache semantics when present", async () => {
240+
writeModelsFixture({
241+
"github-copilot": {
242+
models: {
243+
"gpt-4o-mini": {
244+
cost: {
245+
input: "0.00000015",
246+
output: "0.0000006",
247+
cache_read: "0.000000075",
248+
},
249+
},
250+
},
251+
},
252+
});
253+
254+
const { computeInferenceAIC } = await import("./model_costs.cjs");
255+
const aic = computeInferenceAIC({
256+
provider: "copilot",
257+
model: "gpt-4o-mini",
258+
inputTokens: 1000,
259+
outputTokens: 100,
260+
cacheReadTokens: 400,
261+
cacheWriteTokens: 100,
262+
inputTokensIncludeCache: false,
263+
});
264+
265+
expect(aic).toBeCloseTo(0.0255, 6);
266+
});
267+
210268
it("falls back to bundled models.json when GH_AW_MODELS_JSON_PATH points to a non-existent file", async () => {
211269
// Simulate the detection/evals job scenario: GH_AW_MODELS_JSON_PATH is set to
212270
// /tmp/gh-aw/models.json but that file was never downloaded from the activation artifact.

0 commit comments

Comments
 (0)