Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/patch-report-awf-ai-credits.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions actions/setup/js/fixtures/awf-v0.28.7-aic-token-usage.jsonl
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{"_schema":"token-usage/v0.28.7","timestamp":"2026-08-28T08:59:54.002Z","event":"token_usage","request_id":"5c6141a6-7331-400f-94c8-d38defdcbc7e","provider":"copilot","model":"gpt-4o-mini-2024-07-18","path":"/chat/completions","status":200,"streaming":true,"input_tokens":19288,"output_tokens":35,"cache_read_tokens":0,"cache_write_tokens":0,"duration_ms":2242,"response_bytes":8642,"x_initiator":"user","ai_credits_this_response":0.29142,"ai_credits_total":0.29142,"ai_credits_pricing_source":"models.dev","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false}
{"_schema":"token-usage/v0.28.7","timestamp":"2026-08-28T08:59:56.563Z","event":"token_usage","request_id":"cc30e0e7-0491-484e-aaa9-6b020a2707f4","provider":"copilot","model":"gpt-4o-mini-2024-07-18","path":"/chat/completions","status":200,"streaming":true,"input_tokens":19380,"output_tokens":35,"cache_read_tokens":0,"cache_write_tokens":0,"duration_ms":2469,"response_bytes":8642,"x_initiator":"agent","ai_credits_this_response":0.2928,"ai_credits_total":0.58422,"ai_credits_pricing_source":"models.dev","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false}
{"_schema":"token-usage/v0.28.7","timestamp":"2026-08-28T08:59:58.490Z","event":"token_usage","request_id":"bf3f5fcd-173c-4da9-90d9-680dbc836352","provider":"copilot","model":"gpt-4o-mini-2024-07-18","path":"/chat/completions","status":200,"streaming":true,"input_tokens":272,"output_tokens":35,"cache_read_tokens":19200,"cache_write_tokens":0,"duration_ms":1799,"response_bytes":8648,"x_initiator":"agent","ai_credits_this_response":0.15018,"ai_credits_total":0.7344,"ai_credits_pricing_source":"models.dev","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false}
{"_schema":"token-usage/v0.28.7","timestamp":"2026-08-28T09:00:00.426Z","event":"token_usage","request_id":"776b23fd-a45c-46d4-9e74-95d5f0707402","provider":"copilot","model":"gpt-4o-mini-2024-07-18","path":"/chat/completions","status":200,"streaming":true,"input_tokens":236,"output_tokens":35,"cache_read_tokens":19328,"cache_write_tokens":0,"duration_ms":1861,"response_bytes":8648,"x_initiator":"agent","ai_credits_this_response":0.1506,"ai_credits_total":0.885,"ai_credits_pricing_source":"models.dev","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false}
{"_schema":"token-usage/v0.28.7","timestamp":"2026-08-28T09:00:02.132Z","event":"token_usage","request_id":"59db8b91-cfc2-4cb9-9315-dbcb5e7bcfe7","provider":"copilot","model":"gpt-4o-mini-2024-07-18","path":"/chat/completions","status":200,"streaming":true,"input_tokens":200,"output_tokens":35,"cache_read_tokens":19456,"cache_write_tokens":0,"duration_ms":1552,"response_bytes":8648,"x_initiator":"agent","ai_credits_this_response":0.15102,"ai_credits_total":1.03602,"ai_credits_pricing_source":"models.dev","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false}
11 changes: 9 additions & 2 deletions actions/setup/js/model_costs.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -190,9 +190,10 @@ function usdToAIC(usd) {
* @param {number} params.cacheReadTokens
* @param {number} params.cacheWriteTokens
* @param {number} [params.reasoningTokens]
* @param {boolean} [params.inputTokensIncludeCache]
* @returns {number}
*/
function computeInferenceCostUSD({ provider, model, inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, reasoningTokens = 0 }) {
function computeInferenceCostUSD({ provider, model, inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, reasoningTokens = 0, inputTokensIncludeCache }) {
const pricing = findModelPricing(provider, model);
if (!pricing) return 0;

Expand All @@ -201,7 +202,12 @@ function computeInferenceCostUSD({ provider, model, inputTokens, outputTokens, c
const cacheRead = cacheReadTokens || 0;
const cacheWrite = cacheWriteTokens || 0;
const reasoning = reasoningTokens || 0;
const effectiveInput = providerIncludesCacheReadsInInput(provider) ? Math.max(input - cacheRead, 0) : input;
let effectiveInput = input;
if (inputTokensIncludeCache === true) {
effectiveInput = Math.max(input - cacheRead - cacheWrite, 0);
} else if (typeof inputTokensIncludeCache !== "boolean" && providerIncludesCacheReadsInInput(provider)) {
effectiveInput = Math.max(input - cacheRead, 0);
}

const promptPrice = pricing.input || 0;
const completionPrice = pricing.output || 0;
Expand All @@ -221,6 +227,7 @@ function computeInferenceCostUSD({ provider, model, inputTokens, outputTokens, c
* @param {number} params.cacheReadTokens
* @param {number} params.cacheWriteTokens
* @param {number} [params.reasoningTokens]
* @param {boolean} [params.inputTokensIncludeCache]
* @returns {number}
*/
function computeInferenceAIC(params) {
Expand Down
58 changes: 58 additions & 0 deletions actions/setup/js/model_costs.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,64 @@ describe("model_costs.cjs", () => {
expect(aic).toBeCloseTo(0.336, 6);
});

it("uses explicit inclusive cache semantics when present", async () => {
writeModelsFixture({
"github-copilot": {
models: {
"gpt-4o-mini": {
cost: {
input: "0.00000015",
output: "0.0000006",
cache_read: "0.000000075",
},
},
},
},
});

const { computeInferenceAIC } = await import("./model_costs.cjs");
const aic = computeInferenceAIC({
provider: "copilot",
model: "gpt-4o-mini",
inputTokens: 1000,
outputTokens: 100,
cacheReadTokens: 400,
cacheWriteTokens: 100,
inputTokensIncludeCache: true,
});

expect(aic).toBeCloseTo(0.018, 6);
});

it("uses explicit additive cache semantics when present", async () => {
writeModelsFixture({
"github-copilot": {
models: {
"gpt-4o-mini": {
cost: {
input: "0.00000015",
output: "0.0000006",
cache_read: "0.000000075",
},
},
},
},
});

const { computeInferenceAIC } = await import("./model_costs.cjs");
const aic = computeInferenceAIC({
provider: "copilot",
model: "gpt-4o-mini",
inputTokens: 1000,
outputTokens: 100,
cacheReadTokens: 400,
cacheWriteTokens: 100,
inputTokensIncludeCache: false,
});

expect(aic).toBeCloseTo(0.0255, 6);
});

it("falls back to bundled models.json when GH_AW_MODELS_JSON_PATH points to a non-existent file", async () => {
// Simulate the detection/evals job scenario: GH_AW_MODELS_JSON_PATH is set to
// /tmp/gh-aw/models.json but that file was never downloaded from the activation artifact.
Expand Down
Loading
Loading