Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/patch-report-awf-ai-credits.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

269 changes: 175 additions & 94 deletions .github/workflows/pr-code-quality-reviewer.lock.yml

Large diffs are not rendered by default.

5 changes: 2 additions & 3 deletions .github/workflows/pr-code-quality-reviewer.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,8 @@ on:
name: review
events: [pull_request_comment, pull_request_review_comment]
engine:
id: pi
model-provider: openai
model: openai/gpt-5.4
id: copilot
model: copilot/gpt-5.4
permissions:
contents: read
issues: read
Expand Down
5 changes: 5 additions & 0 deletions actions/setup/js/fixtures/awf-v0.28.7-aic-token-usage.jsonl
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{"_schema":"token-usage/v0.28.7","timestamp":"2026-08-28T08:59:54.002Z","event":"token_usage","request_id":"5c6141a6-7331-400f-94c8-d38defdcbc7e","provider":"copilot","model":"gpt-4o-mini-2024-07-18","path":"/chat/completions","status":200,"streaming":true,"input_tokens":19288,"output_tokens":35,"cache_read_tokens":0,"cache_write_tokens":0,"duration_ms":2242,"response_bytes":8642,"x_initiator":"user","ai_credits_this_response":0.29142,"ai_credits_total":0.29142,"ai_credits_pricing_source":"models.dev","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false}
{"_schema":"token-usage/v0.28.7","timestamp":"2026-08-28T08:59:56.563Z","event":"token_usage","request_id":"cc30e0e7-0491-484e-aaa9-6b020a2707f4","provider":"copilot","model":"gpt-4o-mini-2024-07-18","path":"/chat/completions","status":200,"streaming":true,"input_tokens":19380,"output_tokens":35,"cache_read_tokens":0,"cache_write_tokens":0,"duration_ms":2469,"response_bytes":8642,"x_initiator":"agent","ai_credits_this_response":0.2928,"ai_credits_total":0.58422,"ai_credits_pricing_source":"models.dev","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false}
{"_schema":"token-usage/v0.28.7","timestamp":"2026-08-28T08:59:58.490Z","event":"token_usage","request_id":"bf3f5fcd-173c-4da9-90d9-680dbc836352","provider":"copilot","model":"gpt-4o-mini-2024-07-18","path":"/chat/completions","status":200,"streaming":true,"input_tokens":272,"output_tokens":35,"cache_read_tokens":19200,"cache_write_tokens":0,"duration_ms":1799,"response_bytes":8648,"x_initiator":"agent","ai_credits_this_response":0.15018,"ai_credits_total":0.7344,"ai_credits_pricing_source":"models.dev","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false}
{"_schema":"token-usage/v0.28.7","timestamp":"2026-08-28T09:00:00.426Z","event":"token_usage","request_id":"776b23fd-a45c-46d4-9e74-95d5f0707402","provider":"copilot","model":"gpt-4o-mini-2024-07-18","path":"/chat/completions","status":200,"streaming":true,"input_tokens":236,"output_tokens":35,"cache_read_tokens":19328,"cache_write_tokens":0,"duration_ms":1861,"response_bytes":8648,"x_initiator":"agent","ai_credits_this_response":0.1506,"ai_credits_total":0.885,"ai_credits_pricing_source":"models.dev","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false}
{"_schema":"token-usage/v0.28.7","timestamp":"2026-08-28T09:00:02.132Z","event":"token_usage","request_id":"59db8b91-cfc2-4cb9-9315-dbcb5e7bcfe7","provider":"copilot","model":"gpt-4o-mini-2024-07-18","path":"/chat/completions","status":200,"streaming":true,"input_tokens":200,"output_tokens":35,"cache_read_tokens":19456,"cache_write_tokens":0,"duration_ms":1552,"response_bytes":8648,"x_initiator":"agent","ai_credits_this_response":0.15102,"ai_credits_total":1.03602,"ai_credits_pricing_source":"models.dev","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false}
11 changes: 9 additions & 2 deletions actions/setup/js/model_costs.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -190,9 +190,10 @@ function usdToAIC(usd) {
* @param {number} params.cacheReadTokens
* @param {number} params.cacheWriteTokens
* @param {number} [params.reasoningTokens]
* @param {boolean} [params.inputTokensIncludeCache]
* @returns {number}
*/
function computeInferenceCostUSD({ provider, model, inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, reasoningTokens = 0 }) {
function computeInferenceCostUSD({ provider, model, inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, reasoningTokens = 0, inputTokensIncludeCache }) {
const pricing = findModelPricing(provider, model);
if (!pricing) return 0;

Expand All @@ -201,7 +202,12 @@ function computeInferenceCostUSD({ provider, model, inputTokens, outputTokens, c
const cacheRead = cacheReadTokens || 0;
const cacheWrite = cacheWriteTokens || 0;
const reasoning = reasoningTokens || 0;
const effectiveInput = providerIncludesCacheReadsInInput(provider) ? Math.max(input - cacheRead, 0) : input;
let effectiveInput = input;
if (inputTokensIncludeCache === true) {
effectiveInput = Math.max(input - cacheRead - cacheWrite, 0);
} else if (typeof inputTokensIncludeCache !== "boolean" && providerIncludesCacheReadsInInput(provider)) {
effectiveInput = Math.max(input - cacheRead, 0);
}

const promptPrice = pricing.input || 0;
const completionPrice = pricing.output || 0;
Expand All @@ -221,6 +227,7 @@ function computeInferenceCostUSD({ provider, model, inputTokens, outputTokens, c
* @param {number} params.cacheReadTokens
* @param {number} params.cacheWriteTokens
* @param {number} [params.reasoningTokens]
* @param {boolean} [params.inputTokensIncludeCache]
* @returns {number}
*/
function computeInferenceAIC(params) {
Expand Down
58 changes: 58 additions & 0 deletions actions/setup/js/model_costs.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,64 @@ describe("model_costs.cjs", () => {
expect(aic).toBeCloseTo(0.336, 6);
});

it("uses explicit inclusive cache semantics when present", async () => {
writeModelsFixture({
"github-copilot": {
models: {
"gpt-4o-mini": {
cost: {
input: "0.00000015",
output: "0.0000006",
cache_read: "0.000000075",
},
},
},
},
});

const { computeInferenceAIC } = await import("./model_costs.cjs");
const aic = computeInferenceAIC({
provider: "copilot",
model: "gpt-4o-mini",
inputTokens: 1000,
outputTokens: 100,
cacheReadTokens: 400,
cacheWriteTokens: 100,
inputTokensIncludeCache: true,
});

expect(aic).toBeCloseTo(0.018, 6);
});

it("uses explicit additive cache semantics when present", async () => {
writeModelsFixture({
"github-copilot": {
models: {
"gpt-4o-mini": {
cost: {
input: "0.00000015",
output: "0.0000006",
cache_read: "0.000000075",
},
},
},
},
});

const { computeInferenceAIC } = await import("./model_costs.cjs");
const aic = computeInferenceAIC({
provider: "copilot",
model: "gpt-4o-mini",
inputTokens: 1000,
outputTokens: 100,
cacheReadTokens: 400,
cacheWriteTokens: 100,
inputTokensIncludeCache: false,
});

expect(aic).toBeCloseTo(0.0255, 6);
});

it("falls back to bundled models.json when GH_AW_MODELS_JSON_PATH points to a non-existent file", async () => {
// Simulate the detection/evals job scenario: GH_AW_MODELS_JSON_PATH is set to
// /tmp/gh-aw/models.json but that file was never downloaded from the activation artifact.
Expand Down
Loading
Loading