diff --git a/.github/workflows/actions-fleet-monitor.yml b/.github/workflows/actions-fleet-monitor.yml index 1b1ce37dd..f6fbdb0c9 100644 --- a/.github/workflows/actions-fleet-monitor.yml +++ b/.github/workflows/actions-fleet-monitor.yml @@ -74,90 +74,6 @@ jobs: GH_TOKEN: ${{ secrets.GH_PAT_WORKFLOWS }} run: bash scripts/fleet_monitor.sh - - name: Summarize token usage by workflow - env: - GH_TOKEN: ${{ secrets.GH_PAT_WORKFLOWS }} - REPO: petry-projects/.github-private - LOOKBACK_DAYS: ${{ env.LOOKBACK_DAYS }} - run: | - set -euo pipefail - - workdir="$(mktemp -d)" - trap 'rm -rf "$workdir"' EXIT - jsonl_dir="$workdir/jsonl" - artifacts_file="$workdir/token-artifacts.base64" - mkdir -p "$jsonl_dir" - - cutoff_epoch="$(date -u -d "-${LOOKBACK_DAYS} days" +%s)" - artifact_count=0 - - gh api "repos/$REPO/actions/artifacts" --paginate \ - --jq '.artifacts[] | select(.name | startswith("token-usage-")) | @base64' \ - > "$artifacts_file" - - while IFS= read -r encoded; do - [ -n "$encoded" ] || continue - item="$(printf '%s' "$encoded" | base64 -d)" - id="$(printf '%s' "$item" | jq -r '.id')" - created_at="$(printf '%s' "$item" | jq -r '.created_at')" - expired="$(printf '%s' "$item" | jq -r '.expired')" - created_epoch="$(date -u -d "$created_at" +%s)" - - if [ "$expired" = "true" ] || [ "$created_epoch" -lt "$cutoff_epoch" ]; then - continue - fi - - zip_file="$workdir/artifact-$id.zip" - unzip_dir="$workdir/unzip-$id" - mkdir -p "$unzip_dir" - gh api "repos/$REPO/actions/artifacts/$id/zip" > "$zip_file" - unzip -q -o "$zip_file" -d "$unzip_dir" - - found_any=false - while IFS= read -r f; do - found_any=true - cp "$f" "$jsonl_dir/${id}-$(basename "$f")" - done < <(find "$unzip_dir" -type f -name '*.jsonl' -print) - - if [ "$found_any" = true ]; then - artifact_count=$((artifact_count + 1)) - fi - done < "$artifacts_file" - - if ! find "$jsonl_dir" -type f -name '*.jsonl' | grep -q .; then - { - echo "## Token usage (${LOOKBACK_DAYS} day lookback)" - echo - echo "No token-usage artifacts found in the lookback window." - } >> "$GITHUB_STEP_SUMMARY" - exit 0 - fi - - aggregate="$workdir/token-usage-summary.json" - jq -s ' - sort_by(.workflow) | - group_by(.workflow) | - map({ - workflow: (.[0].workflow // "unknown"), - calls: length, - input_tokens: (map(.input_tokens // 0) | add), - cache_read_tokens: (map(.cache_read_tokens // 0) | add), - output_tokens: (map(.output_tokens // 0) | add), - et: (map(.et // 0) | add) - }) | - sort_by(.et) | reverse - ' "$jsonl_dir"/*.jsonl > "$aggregate" - - { - echo "## Token usage (${LOOKBACK_DAYS} day lookback)" - echo - echo "Processed ${artifact_count} token-usage artifact(s)." - echo - echo "| Workflow | Calls | Input Tokens | Cache Read Tokens | Output Tokens | Total ET |" - echo "|---|---:|---:|---:|---:|---:|" - jq -r '.[] | "| \(.workflow) | \(.calls) | \(.input_tokens) | \(.cache_read_tokens) | \(.output_tokens) | \(.et | tostring) |"' "$aggregate" - } >> "$GITHUB_STEP_SUMMARY" - - name: Create issue with findings if: env.HAS_FAILURES == 'true' && hashFiles('fleet_monitor_report.md') != '' uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 @@ -367,6 +283,17 @@ jobs: {"event_type":"dev-lead-ci-failure","client_payload":{"source":"fleet-monitor","count":"${{ env.HIGH_FAILURE_COUNT }}"}} EOF + - name: Summarize token usage (org-wide) + # Runs after all alerting steps so a token-report failure or timeout does + # not prevent monitoring findings from being created or dispatched. + # continue-on-error keeps this informational step non-blocking. + continue-on-error: true + env: + GH_TOKEN: ${{ secrets.GH_PAT_WORKFLOWS }} + ORG: ${{ env.ORG }} + LOOKBACK_DAYS: ${{ env.LOOKBACK_DAYS }} + run: bash scripts/token_report.sh >> "$GITHUB_STEP_SUMMARY" + - name: Annotate clean run if: env.HAS_FAILURES != 'true' run: | diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6bca6d757..6078c0967 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -22,7 +22,7 @@ jobs: - name: shellcheck # --severity=warning: SC2016 (info) fires on printf '`%s`' where backticks # are literal markdown, not shell expressions — suppress info-level noise. - run: shellcheck --severity=warning scripts/fleet_monitor.sh scripts/fleet_report.sh + run: shellcheck --severity=warning scripts/fleet_monitor.sh scripts/fleet_report.sh scripts/token_report.sh scripts/lib/model-pricing.sh bats: runs-on: ubuntu-latest @@ -34,7 +34,7 @@ jobs: run: sudo apt-get install -y bats - name: Run tests - run: bats tests/fleet_report.bats + run: bats tests/fleet_report.bats tests/token_report.bats tests/model_pricing.bats validate-agent-profiles: runs-on: ubuntu-latest diff --git a/.github/workflows/token-report.yml b/.github/workflows/token-report.yml new file mode 100644 index 000000000..087f7e794 --- /dev/null +++ b/.github/workflows/token-report.yml @@ -0,0 +1,119 @@ +# ───────────────────────────────────────────────────────────────────────────── +# SOURCE OF TRUTH: petry-projects/.github-private/.github/workflows/token-report.yml +# +# AGENTS — READ BEFORE EDITING: +# • Weekly Token Cost Observatory report. The org-wide collection + aggregation +# logic lives in scripts/token_report.sh (unit-tested in tests/token_report.bats). +# • The script reads token-usage artifacts from EVERY non-archived repo in the org, +# so it needs GH_PAT_WORKFLOWS (cross-org actions:read) — github.token cannot read +# other repos' artifacts. +# • The report is posted as a comment on a single pinned tracking issue +# (label: token-report) in .github-private, so github.token suffices for the +# comment step. +# • Do NOT change checkout ref from 'main'. +# ───────────────────────────────────────────────────────────────────────────── +name: Token Cost Report + +on: + schedule: + # Mondays at 08:00 UTC — weekly cost rollup before the work week + - cron: '0 8 * * 1' + workflow_dispatch: + inputs: + org: + description: "GitHub org to scan" + required: false + default: "petry-projects" + type: string + lookback_days: + description: "Days of token-usage history to include" + required: false + default: "7" + type: string + +permissions: + contents: read + issues: write + +concurrency: + group: token-report-${{ inputs.org || 'petry-projects' }} + cancel-in-progress: true + +jobs: + report: + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + ORG: ${{ inputs.org || 'petry-projects' }} + LOOKBACK_DAYS: ${{ inputs.lookback_days || '7' }} + + steps: + - name: Checkout agent repo + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + repository: petry-projects/.github-private + ref: main + token: ${{ secrets.GH_PAT_WORKFLOWS }} + + - name: Generate org-wide token report + env: + GH_TOKEN: ${{ secrets.GH_PAT_WORKFLOWS }} + TOKEN_REPORT_OUT: token_report.md + run: | + bash scripts/token_report.sh >> "$GITHUB_STEP_SUMMARY" + + - name: Post report to tracking issue + if: hashFiles('token_report.md') != '' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + github-token: ${{ github.token }} + script: | + const fs = require('fs'); + const MAX = 65000; + let report = fs.readFileSync('token_report.md', 'utf8'); + if (!report.trim()) { core.setFailed('Report file is empty'); return; } + + const runUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`; + if (report.length > MAX) { + report = report.slice(0, MAX) + + `\n\n---\n_Report truncated — full output in the [workflow run summary](${runUrl})._`; + } + + const LABEL = 'token-report'; + const TITLE = '📊 Token Cost Observatory — Weekly Report'; + + // Ensure the tracking label exists. + try { + await github.rest.issues.getLabel({ owner: context.repo.owner, repo: context.repo.repo, name: LABEL }); + } catch (e) { + await github.rest.issues.createLabel({ + owner: context.repo.owner, repo: context.repo.repo, + name: LABEL, color: '5319e7', + description: 'Weekly Token Cost Observatory report', + }); + } + + // Find-or-create the single pinned tracking issue. + const existing = await github.paginate(github.rest.issues.listForRepo, { + owner: context.repo.owner, repo: context.repo.repo, + labels: LABEL, state: 'open', per_page: 100, + }); + + let issue = existing.find(i => i.title === TITLE); + if (!issue) { + const created = await github.rest.issues.create({ + owner: context.repo.owner, repo: context.repo.repo, + title: TITLE, labels: [LABEL], + body: 'Tracking issue for the weekly Token Cost Observatory report. ' + + 'Each run posts a new comment below with org-wide Effective Tokens (ET) usage.\n\n' + + 'Source: `scripts/token_report.sh` · `.github/workflows/token-report.yml`', + }); + issue = created.data; + core.notice(`Created tracking issue #${issue.number}: ${issue.html_url}`); + } + + const comment = await github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, + issue_number: issue.number, body: report, + }); + core.notice(`Posted weekly token report: ${comment.data.html_url}`); diff --git a/AGENTS.md b/AGENTS.md index bb94086c1..571cc2abb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,6 +30,10 @@ This is the `.github-private` org infrastructure repo for `petry-projects`. It c agentic workflow compilation. It is not covered by the org template and must not be removed by template syncs. If the org template gains a `gh-aw-compile` equivalent, remove this exception and defer to the template instead. +- **Exception:** `token-report.yml` (weekly Token Cost Observatory report) is a documented repo-specific + workflow with no corresponding org template in `standards/workflows/`. It must not be removed by template + syncs. If the org template gains a token-report equivalent, remove this exception and defer to the + template instead. ### Agent Profiles (`agents/*.md`) diff --git a/docs/actions-fleet-monitor.md b/docs/actions-fleet-monitor.md index 984fe0789..8116e54c3 100644 --- a/docs/actions-fleet-monitor.md +++ b/docs/actions-fleet-monitor.md @@ -19,6 +19,7 @@ The fleet summary table is sorted by severity so the worst performers appear fir - **Step Summary** — fleet table written on every run; visible in the Actions UI without leaving GitHub. - **GitHub Issue** — opened when any workflow has failed runs; see [Issue destination](#issue-destination) below. +- **Token usage (org-wide)** — each run also appends an Effective Tokens (ET) rollup to the Step Summary via `scripts/token_report.sh`, scanning token-usage artifacts across every repo. The weekly digest is delivered as a tracking-issue comment by `token-report.yml` — see [docs/token-report.md](token-report.md). ## Usage diff --git a/docs/token-report.md b/docs/token-report.md new file mode 100644 index 000000000..dca50af46 --- /dev/null +++ b/docs/token-report.md @@ -0,0 +1,159 @@ +# Token Cost Observatory — Weekly Report + +Org-wide visibility into LLM token spend for the agentic workflows (pr-review and +dev-lead). Builds on the per-call JSONL logging added by the +[Token Cost Observatory](https://github.com/petry-projects/.github-private/discussions/332) +work — this layer aggregates that data across **every repo in the org** and +delivers it where humans actually look. + +## The problem it solves + +The agents emit a token-usage record per LLM call (`scripts/lib/token-metrics.sh`) +and upload it as a `token-usage-` artifact. But because pr-review and +dev-lead run as **reusable workflows in each caller repo**, those artifacts land in +the *caller's* repo — not in `.github-private`. The original inline summary only +scanned `.github-private`, so it saw a few percent of real org spend. This report +scans all non-archived repos. + +## Cost (USD) + +The report shows **estimated USD cost** alongside tokens, everywhere tokens are +surfaced (totals, by workflow/tier/model, by repository, and a most-expensive-PRs +rollup). Cost is the headline metric; ET is kept as a normalised comparator. + +### Single source of truth: `scripts/lib/model-pricing.tsv` + +All prices live in one git-versioned table — prices are **data, not code**, so every +change is a reviewed PR with full history. `scripts/lib/model-pricing.sh` is the only +place token counts become dollars (`cost_usd`) or ET multipliers (`et_multiplier_for`), +so cost and ET can never drift apart. + +``` +cost = (input × P_in + cache_read × P_cache + output × P_out) / 1,000,000 +``` + +### Effective-dated pricing + +Each table row carries an `effective_from` date. A record is priced at **the rate in +effect on its own timestamp** — not today's rate. For model `M` at time `T`, the lookup +picks the row whose glob matches `M` and whose `effective_from ≤ T`, choosing the most +specific glob (ties → latest date). Selection is deterministic, so a given record always +resolves to the same price. + +**To change a price:** never edit an existing row — **append** a new row for the same +glob with a later `effective_from`. Records before that date keep their historical rate; +later records pick up the new rate automatically. Example (Sonnet drops on 2026-09-01): + +``` +claude-sonnet-4-* 2025-01-01 3.00 0.30 3.75 15.00 +claude-sonnet-4-* 2026-09-01 2.50 0.25 3.125 12.50 # ← append, don't edit +``` + +(Columns: `input cache_read cache_write output`, USD/MTok.) + +### Real token usage (incl. cache) + +Cost is computed from **actual API usage**, not estimates. When `TOKEN_LOG_FILE` is set, +`scripts/engine.sh` runs Claude/Gemini with `--output-format json`, extracts the model's +text for downstream consumers, and records the real `input` / `cache_read` / +`cache_write` (cache-creation) / `output` token counts (`scripts/lib/token-metrics.sh: +parse_engine_usage`). This replaced the earlier `char_count / 4` estimate, which always +reported zero cache. + +- **Safety switch:** set `ENGINE_USAGE_JSON=0` to instantly revert to text output + the + estimate path (no code change). JSON capture only engages when `TOKEN_LOG_FILE` is set, + and falls back to the raw output if extraction yields nothing — so a parse hiccup never + breaks a review. +- **Copilot:** the GitHub Copilot CLI exposes no machine-readable token usage, so copilot + runs remain estimate-based (cache = 0). This is a CLI limitation, not a gap in the report. +- **Cache-write** (cache-creation, billed at 1.25× input by Anthropic) is now both captured + and priced. One-shot agent calls typically show more cache-write than cache-read. + +### Reliability guarantees + +- **Unknown model → never silent $0.** Calls with no matching price row are excluded + from cost totals and flagged (`*` + a ⚠️ note), so missing prices are visible. +- **Scope:** cost covers metered per-token engines (Claude, Gemini, o4-mini). It does + **not** cover Copilot code review, which bills GitHub Actions minutes / AI credits + from 2026-06-01 — a different unit. Cache-write, batch (−50%), fast-mode and + server-tool surcharges are not modelled (records only carry input/cache-read/output). +- Claude rates are verified against [Anthropic's pricing](https://platform.claude.com/docs/en/docs/about-claude/pricing); + non-Claude rows are best-effort and marked `VERIFY` in the table. + +## Effective Tokens (ET) + +A normalised cost comparator, kept alongside USD: + +``` +ET = m × (1.0 × input + 0.1 × cache_read + 4.0 × output) +``` + +`m` is **derived from the price table** (`m = input_price(model) ÷ input_price(haiku)`), +so it stays in sync with real prices: + +| Model | Multiplier | +|---|---| +| haiku | 1× | +| sonnet | 3× | +| o4-mini, gemini-pro | 2× | +| gemini-flash | 0.5× | +| opus (4.5+) | 5× | + +Output is weighted 4× and cache-read 0.1×, so ET tracks *cost*, not raw token count. +(ET and USD can rank workflows differently when output ratios differ — trust USD for +budgeting; ET is a quick comparator.) + +## Delivery + +| Channel | Cadence | Where | +|---|---|---| +| **Tracking issue comment** | Weekly (Mon 08:00 UTC) | A single pinned issue in `.github-private` labelled `token-report`; each run adds a comment. | +| **Step Summary** | Weekly + on every fleet-monitor run | Actions run summary of `token-report.yml` and `actions-fleet-monitor.yml`. | + +## Usage + +Runs automatically every Monday. To trigger manually: + +```bash +gh workflow run token-report.yml \ + --repo petry-projects/.github-private \ + --field org=petry-projects \ + --field lookback_days=7 +``` + +To generate the report locally (requires a token with org-wide `actions:read`): + +```bash +ORG=petry-projects LOOKBACK_DAYS=7 GH_TOKEN="$(gh auth token)" \ + bash scripts/token_report.sh +``` + +## Architecture + +| File | Role | +|---|---| +| `scripts/lib/model-pricing.tsv` | Single source of truth for prices — effective-dated rows. | +| `scripts/lib/model-pricing.sh` | `price_for` / `cost_usd` / `et_multiplier_for` (glob + date lookup). | +| `scripts/token_report.sh` | Org-wide collection (`main`/`collect_org_jsonl`) + pure rendering (`annotate_records`, `render_token_report`). | +| `tests/token_report.bats`, `tests/model_pricing.bats` | Unit tests for rendering and dated pricing. | +| `.github/workflows/token-report.yml` | Weekly cron → tracking-issue comment. | +| `.github/workflows/actions-fleet-monitor.yml` | Daily Step-Summary rollup (reuses the same script). | + +## Environment variables (script) + +| Variable | Description | +|---|---| +| `ORG` | Org to scan (default `petry-projects`). All non-archived repos discovered automatically. | +| `LOOKBACK_DAYS` | Rolling window of artifact history to include (default `7`). | +| `GH_TOKEN` | PAT with `actions:read` across the org — token-usage artifacts are per-repo. Uses `GH_PAT_WORKFLOWS` in CI. | +| `TOKEN_REPORT_OUT` | Optional path to also write the report to (used by the workflow to post the issue comment). | + +## Known limitations + +- **Artifact retention**: token-usage artifacts use GitHub's default retention. A + lookback longer than the retention window will silently miss expired artifacts. +- **Cache tokens**: captured from real API usage for Claude/Gemini. Copilot exposes no + usage, so copilot calls fall back to estimates with 0 cache. Records written before this + capture landed have 0 cache (the old estimate path). +- **Per-repo artifact API**: one `actions/artifacts` listing per repo per run. For + very large orgs this is linear in repo count. diff --git a/scripts/engine.sh b/scripts/engine.sh index 07c9290ea..77b302f4b 100644 --- a/scripts/engine.sh +++ b/scripts/engine.sh @@ -249,6 +249,50 @@ copilot_chat() { -s "$@" < /dev/null } +# _gemini_invoke [extra_args...] +# Runs the gemini CLI and emits the model's text to stdout (stderr passes through +# so callers that merge it for rate-limit detection still work). When token +# logging is on (and ENGINE_USAGE_JSON != 0) it runs with --output-format json, +# captures real usage into the LAST_* globals, and emits the extracted text; +# otherwise it uses --output-format text (estimate path). Robust fallback: if the +# JSON result text is empty or the call fails, the raw payload is emitted. +_gemini_invoke() { + local prompt_file="$1" timeout_sec="$2" model="$3" + shift 3 + local extra_args=("$@") + + declare -f reset_engine_usage >/dev/null 2>&1 && reset_engine_usage + + local _usage_json=0 + if [ -n "${TOKEN_LOG_FILE:-}" ] && [ "${ENGINE_USAGE_JSON:-1}" != "0" ] \ + && declare -f parse_engine_usage >/dev/null 2>&1; then + _usage_json=1 + fi + + if [ "$_usage_json" -eq 1 ]; then + local _json_tmp; _json_tmp="$(mktemp 2>/dev/null || true)" + if [ -n "$_json_tmp" ]; then + local rc=0 + # stderr intentionally NOT redirected — flows to caller for rate-limit checks. + timeout "$timeout_sec" gemini --prompt "" --model "$model" "${extra_args[@]}" \ + --output-format json < "$prompt_file" > "$_json_tmp" || rc=$? + if [ "$rc" -eq 0 ]; then + parse_engine_usage gemini "$_json_tmp" || true + local _txt; _txt="$(extract_engine_text gemini "$_json_tmp")" + if [ -n "$_txt" ]; then printf '%s\n' "$_txt"; else cat "$_json_tmp"; fi + else + cat "$_json_tmp" # emit raw payload so downstream/rate-limit logic sees it + fi + rm -f "$_json_tmp" + return "$rc" + fi + fi + + # Estimate path (logging off, or mktemp failed): plain text output. + timeout "$timeout_sec" gemini --prompt "" --model "$model" "${extra_args[@]}" \ + --output-format text < "$prompt_file" +} + # _claude_chain_invoke [extra_args...] # Walks a comma-separated list of Claude models, invoking `claude --print --model X` # with the given extra arguments. The first model whose run does NOT trigger @@ -272,6 +316,18 @@ _claude_chain_invoke() { return 1 fi + # When token logging is on, capture real usage by running claude with + # --output-format json and emitting the extracted .result text to the caller. + # Gated by ENGINE_USAGE_JSON (default on) so it can be disabled without a code + # change. Falls back to raw output if extraction yields nothing. + declare -f reset_engine_usage >/dev/null 2>&1 && reset_engine_usage + local _usage_json=0 fmt_args=() + if [ -n "${TOKEN_LOG_FILE:-}" ] && [ "${ENGINE_USAGE_JSON:-1}" != "0" ] \ + && declare -f parse_engine_usage >/dev/null 2>&1; then + _usage_json=1 + fmt_args=(--output-format json) + fi + local saved_ifs="$IFS" IFS=',' read -ra models <<< "$chain_csv" IFS="$saved_ifs" @@ -293,7 +349,7 @@ _claude_chain_invoke() { rc=0 if [ -n "$stdout_tmp" ] && [ -n "$stderr_tmp" ]; then - timeout "$timeout_sec" claude --print --model "$model" "${extra_args[@]}" \ + timeout "$timeout_sec" claude --print --model "$model" "${fmt_args[@]}" "${extra_args[@]}" \ < "$prompt_file" > "$stdout_tmp" 2> "$stderr_tmp" || rc=$? else # mktemp failure (one or both) — clean up the partial tmp before degrading @@ -350,9 +406,23 @@ _claude_chain_invoke() { final_rc=2 fi - # Emit the final attempt's captured output to the caller. + # Emit the final attempt's captured output to the caller. In JSON-usage mode, + # parse the usage block and emit the extracted .result text (so consumers still + # receive plain text); fall back to the raw payload if extraction is empty or + # the call failed (preserves error/rate-limit text for downstream detection). if [ -n "$final_stdout" ]; then - cat "$final_stdout" + if [ "$_usage_json" -eq 1 ] && [ "$final_rc" -eq 0 ]; then + parse_engine_usage claude "$final_stdout" || true + local _txt + _txt="$(extract_engine_text claude "$final_stdout")" + if [ -n "$_txt" ]; then + printf '%s\n' "$_txt" + else + cat "$final_stdout" + fi + else + cat "$final_stdout" + fi rm -f "$final_stdout" fi if [ -n "$final_stderr" ]; then @@ -374,13 +444,41 @@ _record_engine_tokens() { declare -f emit_token_record >/dev/null 2>&1 || return 0 local tier="$1" engine="$2" model="$3" prompt_file="$4" output_file="${5:-}" - local workflow="${TOKEN_WORKFLOW:-unknown}" - local input_tokens output_tokens context - input_tokens=$(estimate_tokens_from_file "$prompt_file") - output_tokens=$(estimate_tokens_from_file "$output_file") - context="${PR_URL:-}" + local workflow="${TOKEN_WORKFLOW:-unknown}" context="${PR_URL:-}" + local input_tokens cache_read_tokens cache_write_tokens output_tokens + local _have_usage=0 + + # Prefer real usage captured by the engine. It is read from the sidecar file + # (written even when the engine ran inside a `cmd | tee` subshell); the LAST_* + # globals are a secondary source for non-piped callers. + local _uf="" + declare -f _engine_usage_sidecar >/dev/null 2>&1 && _uf="$(_engine_usage_sidecar)" + if [ -n "$_uf" ] && [ -s "$_uf" ]; then + IFS=$'\t' read -r input_tokens cache_read_tokens cache_write_tokens output_tokens < "$_uf" + rm -f "$_uf" 2>/dev/null || true + case "${input_tokens}${cache_read_tokens}${cache_write_tokens}${output_tokens}" in + ''|*[!0-9]*) _have_usage=0 ;; + *) _have_usage=1 ;; + esac + elif [ "${LAST_USAGE_OK:-0}" = "1" ]; then + input_tokens="${LAST_INPUT_TOKENS:-0}" + cache_read_tokens="${LAST_CACHE_READ_TOKENS:-0}" + cache_write_tokens="${LAST_CACHE_WRITE_TOKENS:-0}" + output_tokens="${LAST_OUTPUT_TOKENS:-0}" + _have_usage=1 + fi + + if [ "$_have_usage" -ne 1 ]; then + # Fallback: estimate from byte counts (engine reported no usage, e.g. copilot + # or text-mode). Cache figures are unknown → 0. + input_tokens=$(estimate_tokens_from_file "$prompt_file") + output_tokens=$(estimate_tokens_from_file "$output_file") + cache_read_tokens=0 + cache_write_tokens=0 + fi + emit_token_record "$workflow" "$tier" "$engine" "$model" \ - "$input_tokens" 0 "$output_tokens" "$context" || true + "$input_tokens" "$cache_read_tokens" "$output_tokens" "$context" "$cache_write_tokens" || true } # run_triage @@ -418,17 +516,11 @@ run_triage() { ;; gemini) if [ -n "$_tok_tmp" ]; then - timeout "$TRIAGE_TIMEOUT_SEC" gemini --prompt "" \ - --model "$ENGINE_TRIAGE_MODEL" \ - --approval-mode auto_edit \ - --output-format text \ - < "$prompt_file" | tee "$_tok_tmp" || rc=${PIPESTATUS[0]} + _gemini_invoke "$prompt_file" "$TRIAGE_TIMEOUT_SEC" "$ENGINE_TRIAGE_MODEL" \ + --approval-mode auto_edit | tee "$_tok_tmp" || rc=${PIPESTATUS[0]} else - timeout "$TRIAGE_TIMEOUT_SEC" gemini --prompt "" \ - --model "$ENGINE_TRIAGE_MODEL" \ - --approval-mode auto_edit \ - --output-format text \ - < "$prompt_file" || rc=$? + _gemini_invoke "$prompt_file" "$TRIAGE_TIMEOUT_SEC" "$ENGINE_TRIAGE_MODEL" \ + --approval-mode auto_edit || rc=$? fi ;; copilot) @@ -515,17 +607,11 @@ run_agentic() { ;; gemini) if [ -n "$_tok_tmp" ]; then - timeout "$DEEP_TIMEOUT_SEC" gemini --prompt "" \ - --model "$model" \ - --approval-mode auto_edit \ - --output-format text \ - < "$prompt_file" | tee "$_tok_tmp" || rc=${PIPESTATUS[0]} + _gemini_invoke "$prompt_file" "$DEEP_TIMEOUT_SEC" "$model" \ + --approval-mode auto_edit | tee "$_tok_tmp" || rc=${PIPESTATUS[0]} else - timeout "$DEEP_TIMEOUT_SEC" gemini --prompt "" \ - --model "$model" \ - --approval-mode auto_edit \ - --output-format text \ - < "$prompt_file" || rc=$? + _gemini_invoke "$prompt_file" "$DEEP_TIMEOUT_SEC" "$model" \ + --approval-mode auto_edit || rc=$? fi ;; copilot) @@ -608,37 +694,31 @@ run_duck() { claude) unset COPILOT_GITHUB_TOKEN 2>/dev/null || true unset GOOGLE_API_KEY 2>/dev/null || true + # Route through the chain helper so real token usage (incl. cache) is captured + # when logging is on; a single-model "chain" preserves prior behaviour. if [ -n "$_tok_tmp" ]; then - timeout "$DUCK_TIMEOUT_SEC" claude --print \ - --model "$model" \ + _claude_chain_invoke "$model" "$prompt_file" "$DUCK_TIMEOUT_SEC" \ --permission-mode acceptEdits \ --allowed-tools "Bash,Read,Grep,Glob" \ --max-turns 25 \ - < "$prompt_file" | tee "$_tok_tmp" || rc=${PIPESTATUS[0]} + | tee "$_tok_tmp" || rc=${PIPESTATUS[0]} else - timeout "$DUCK_TIMEOUT_SEC" claude --print \ - --model "$model" \ + _claude_chain_invoke "$model" "$prompt_file" "$DUCK_TIMEOUT_SEC" \ --permission-mode acceptEdits \ --allowed-tools "Bash,Read,Grep,Glob" \ --max-turns 25 \ - < "$prompt_file" || rc=$? + || rc=$? fi ;; gemini) unset CLAUDE_CODE_OAUTH_TOKEN 2>/dev/null || true unset COPILOT_GITHUB_TOKEN 2>/dev/null || true if [ -n "$_tok_tmp" ]; then - timeout "$DUCK_TIMEOUT_SEC" gemini --prompt "" \ - --model "$model" \ - --approval-mode auto_edit \ - --output-format text \ - < "$prompt_file" | tee "$_tok_tmp" || rc=${PIPESTATUS[0]} + _gemini_invoke "$prompt_file" "$DUCK_TIMEOUT_SEC" "$model" \ + --approval-mode auto_edit | tee "$_tok_tmp" || rc=${PIPESTATUS[0]} else - timeout "$DUCK_TIMEOUT_SEC" gemini --prompt "" \ - --model "$model" \ - --approval-mode auto_edit \ - --output-format text \ - < "$prompt_file" || rc=$? + _gemini_invoke "$prompt_file" "$DUCK_TIMEOUT_SEC" "$model" \ + --approval-mode auto_edit || rc=$? fi ;; copilot) @@ -787,17 +867,11 @@ run_writer() { ;; gemini) if [ -n "$_tmp" ]; then - timeout "$ACTION_TIMEOUT_SEC" gemini --prompt "" \ - --model "$model" \ - --approval-mode auto_edit \ - --output-format text \ - < "$prompt_file" 2>&1 | tee "$_tmp" || rc=${PIPESTATUS[0]} + _gemini_invoke "$prompt_file" "$ACTION_TIMEOUT_SEC" "$model" \ + --approval-mode auto_edit 2>&1 | tee "$_tmp" || rc=${PIPESTATUS[0]} else - timeout "$ACTION_TIMEOUT_SEC" gemini --prompt "" \ - --model "$model" \ - --approval-mode auto_edit \ - --output-format text \ - < "$prompt_file" || rc=$? + _gemini_invoke "$prompt_file" "$ACTION_TIMEOUT_SEC" "$model" \ + --approval-mode auto_edit || rc=$? fi ;; copilot) diff --git a/scripts/lib/model-pricing.sh b/scripts/lib/model-pricing.sh new file mode 100644 index 000000000..2d77cbab3 --- /dev/null +++ b/scripts/lib/model-pricing.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# model-pricing.sh — effective-dated LLM price lookup + cost/ET derivation. +# +# Single source of truth for prices is model-pricing.tsv (same directory). This +# library is the ONLY place token counts are turned into dollars or ET multipliers, +# so every surface that shows cost stays consistent. +# +# Functions (all pure — deterministic, no network): +# price_for [iso_date] → "input cache_read output" USD/MTok, or "" if unknown +# cost_usd [iso_date] → USD cost (6dp), or "" if price unknown +# et_multiplier_for [iso_date] → ET multiplier m = input(model)/input(baseline), default 1.0 +# +# Pricing is effective-dated: a record is priced at the rate in effect on its own +# date (see model-pricing.tsv for the selection rule). iso_date defaults to today. +# +# Environment: +# PRICING_TABLE — path to the TSV (default: alongside this script) +# ET_BASELINE_MODEL — model whose input price is ET's "1.0" (default claude-haiku-4-5) + +# Resolve the table path once, relative to this file (works when sourced from anywhere). +if [ -z "${PRICING_TABLE:-}" ]; then + PRICING_TABLE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/model-pricing.tsv" +fi +: "${ET_BASELINE_MODEL:=claude-haiku-4-5}" + +# price_for [iso_date] +# Echoes "input cache_read cache_write output" (USD/MTok) for the rate in effect at +# iso_date, selecting the most-specific matching glob (ties → latest effective_from). +# Echoes nothing when no row matches (caller treats as unknown price). +price_for() { + local model="$1" date="${2:-}" + [ -n "$date" ] || date="$(date -u +%Y-%m-%d 2>/dev/null || echo '9999-12-31')" + [ -f "$PRICING_TABLE" ] || return 0 + + awk -F'\t' -v model="$model" -v d="$date" ' + function glob2re(g, re) { + re = g + gsub(/[.[\]()^$+{}|\\]/, "\\\\&", re) # escape regex metachars + gsub(/\*/, ".*", re) # glob * → regex .* + gsub(/\?/, ".", re) # glob ? → regex . + return "^" re "$" + } + /^[[:space:]]*#/ { next } # comments + NF < 6 { next } # blanks / malformed + { + glob = $1; eff = $2 + if (model ~ glob2re(glob) && eff <= d) { + lit = glob; gsub(/[*?]/, "", lit); spec = length(lit) + if (spec > best_spec || (spec == best_spec && eff > best_eff)) { + best_spec = spec; best_eff = eff; bi = $3; bc = $4; bw = $5; bo = $6; found = 1 + } + } + } + END { if (found) printf "%s %s %s %s", bi, bc, bw, bo } + ' "$PRICING_TABLE" +} + +# cost_usd [iso_date] [cache_write] +# Echoes the USD cost (6 decimals) using the date-accurate rate. Echoes nothing when +# the model's price is unknown — callers MUST surface that (never treat as $0). +# cache_write defaults to 0 so existing 4-arg callers are unaffected. +cost_usd() { + local model="$1" in="${2:-0}" cr="${3:-0}" out="${4:-0}" date="${5:-}" cw="${6:-0}" + local p; p="$(price_for "$model" "$date")" + [ -n "$p" ] || return 0 + local pin pcr pcw pout; read -r pin pcr pcw pout <<< "$p" + awk -v i="$in" -v c="$cr" -v w="$cw" -v o="$out" \ + -v pin="$pin" -v pcr="$pcr" -v pcw="$pcw" -v pout="$pout" \ + 'BEGIN { printf "%.6f", (i * pin + c * pcr + w * pcw + o * pout) / 1000000 }' +} + +# et_multiplier_for [iso_date] +# Returns m = input_price(model, date) / input_price(baseline, date). Date-accurate, +# so ET can never drift from the price table. Falls back to 1.0 when either price is +# unknown (safe default — never fails). +et_multiplier_for() { + local model="$1" date="${2:-}" + local mp bp + mp="$(price_for "$model" "$date")" + bp="$(price_for "$ET_BASELINE_MODEL" "$date")" + if [ -z "$mp" ] || [ -z "$bp" ]; then echo "1.0"; return 0; fi + # %.1f keeps the canonical "5.0"/"0.5" form used elsewhere; all current multipliers + # (0.5, 1, 2, 3, 5) are exact at one decimal. + awk -v m="${mp%% *}" -v b="${bp%% *}" \ + 'BEGIN { printf "%.1f", (b > 0 ? m / b : 1.0) }' +} diff --git a/scripts/lib/model-pricing.tsv b/scripts/lib/model-pricing.tsv new file mode 100644 index 000000000..b7f76ec17 --- /dev/null +++ b/scripts/lib/model-pricing.tsv @@ -0,0 +1,43 @@ +# model-pricing.tsv — single source of truth for LLM token prices (USD per 1M tokens). +# +# Consumed by scripts/lib/model-pricing.sh (price_for / cost_usd / et_multiplier_for). +# Every price change is a reviewed PR — prices are DATA, not code. +# +# Columns (TAB-separated): +# 1 model_glob shell glob matched against the record's model string ('*' wildcard) +# 2 effective_from ISO date (UTC) this rate took effect; rows are date-versioned +# 3 input USD / MTok for base (uncached) input tokens +# 4 cache_read USD / MTok for cache-read (hit) tokens +# 5 cache_write USD / MTok for cache-creation (write) tokens +# 6 output USD / MTok for output tokens +# +# SELECTION (deterministic): for a record with model M and timestamp T, pick the row +# whose glob matches M AND whose effective_from <= T, choosing the MOST SPECIFIC glob +# (most literal characters); ties break to the LATEST effective_from. So a call is +# always priced at the rate in effect on the day it ran. +# +# PRICE CHANGES: never edit an existing row — APPEND a new row for the same glob with a +# later effective_from. Old records keep their historical rate; new ones pick up the new +# rate automatically. Example (Sonnet drops on 2026-09-01): +# claude-sonnet-4-* 2026-09-01 2.50 0.25 3.125 12.50 +# +# cache_write is Anthropic's 5-minute cache-write rate (1.25x base input). Batch (-50%), +# fast-mode, data-residency (1.1x) and server-tool surcharges are NOT modelled. +# +# Claude rates verified against https://platform.claude.com/docs/en/docs/about-claude/pricing +# (2026-06-06). Non-Claude rows are best-effort — verify before trusting (marked VERIFY). +# +# model_glob effective_from input cache_read cache_write output + +# ── Anthropic Claude (authoritative) ────────────────────────────────────────── +claude-opus-4-1* 2025-01-01 15.00 1.50 18.75 75.00 +claude-opus-4-0* 2025-01-01 15.00 1.50 18.75 75.00 +claude-opus-4-* 2025-11-01 5.00 0.50 6.25 25.00 +claude-sonnet-4-* 2025-01-01 3.00 0.30 3.75 15.00 +claude-haiku-4-5* 2025-10-01 1.00 0.10 1.25 5.00 +claude-haiku-3-5* 2024-01-01 0.80 0.08 1.00 4.00 + +# ── Cross-engine (VERIFY — provisional; cache_write placeholder = input) ─────── +o4-mini* 2025-01-01 2.00 0.50 2.00 8.00 +*gemini*flash* 2025-01-01 0.50 0.05 0.50 2.00 +*gemini*pro* 2025-01-01 2.00 0.20 2.00 8.00 diff --git a/scripts/lib/token-metrics.sh b/scripts/lib/token-metrics.sh index 2795c5350..ba22b9f1e 100644 --- a/scripts/lib/token-metrics.sh +++ b/scripts/lib/token-metrics.sh @@ -19,15 +19,28 @@ set -euo pipefail # # All functions are safe to call even when TOKEN_LOG_FILE is unset or unwritable. +# Load the dated price table so ET multipliers derive from the same source as USD cost +# (and can never drift apart). Best-effort: if the lib is absent, model_multiplier_for +# falls back to a static table so this file stays self-contained. +if [ -f "$(dirname "${BASH_SOURCE[0]}")/model-pricing.sh" ]; then + # shellcheck source=scripts/lib/model-pricing.sh + source "$(dirname "${BASH_SOURCE[0]}")/model-pricing.sh" +fi + # model_multiplier_for -# Returns a float cost multiplier relative to claude-haiku-4-5 = 1.0. -# Prints 1.0 for unknown models (safe default — never fails). +# Returns a float cost multiplier relative to claude-haiku-4-5 = 1.0, derived from +# model-pricing.tsv at today's rate. Prints 1.0 for unknown models (safe default). model_multiplier_for() { local model="$1" + if declare -f et_multiplier_for >/dev/null 2>&1; then + et_multiplier_for "$model" + return + fi + # Fallback static table (used only if model-pricing.sh failed to load). case "$model" in *haiku*) echo "1.0" ;; *sonnet*) echo "3.0" ;; - *opus*) echo "15.0" ;; + *opus*) echo "5.0" ;; o4-mini | *o4-mini*) echo "2.0" ;; *gemini*flash*) echo "0.5" ;; *gemini*pro*) echo "2.0" ;; @@ -54,14 +67,18 @@ estimate_tokens_from_file() { } # emit_token_record -# +# +# [cache_write_tokens] # Appends one JSONL record to TOKEN_LOG_FILE. No-op when TOKEN_LOG_FILE is unset. # Silently swallows I/O errors so token logging never aborts a workflow run. +# cache_write_tokens (cache-creation) is optional and defaults to 0 for callers +# that do not capture it (backward compatible). emit_token_record() { [ -n "${TOKEN_LOG_FILE:-}" ] || return 0 local workflow="$1" tier="$2" engine="$3" model="$4" local input="${5:-0}" cache="${6:-0}" output="${7:-0}" context="${8:-}" + local cache_write="${9:-0}" local mult et ts run_id record mult=$(model_multiplier_for "$model") @@ -77,6 +94,7 @@ emit_token_record() { --arg model "$model" \ --arg input "$input" \ --arg cache "$cache" \ + --arg cache_write "$cache_write" \ --arg output "$output" \ --arg et "$et" \ --arg run_id "$run_id" \ @@ -89,6 +107,7 @@ emit_token_record() { model: $model, input_tokens: ($input | tonumber? // 0), cache_read_tokens: ($cache | tonumber? // 0), + cache_creation_tokens: ($cache_write | tonumber? // 0), output_tokens: ($output | tonumber? // 0), et: ($et | tonumber? // 0), run_id: $run_id, @@ -97,3 +116,133 @@ emit_token_record() { printf '%s\n' "$record" >> "$TOKEN_LOG_FILE" 2>/dev/null || true } + +# ───────────────────────────────────────────────────────────────────────────── +# Real-usage capture (replaces the char/4 estimate when an engine reports usage) +# +# Engines that emit a machine-readable usage block (claude/gemini --output-format +# json) let us record actual input / cache-read / cache-write / output tokens +# instead of estimating. Parsers set these globals; engine.sh reads them: +# LAST_USAGE_OK 1 when real usage was parsed, else 0 (→ estimate) +# LAST_INPUT_TOKENS non-cached input tokens +# LAST_CACHE_READ_TOKENS cache-hit (read) tokens +# LAST_CACHE_WRITE_TOKENS cache-creation (write) tokens +# LAST_OUTPUT_TOKENS output tokens +# ───────────────────────────────────────────────────────────────────────────── + +# _engine_usage_sidecar +# Prints the path of the per-process usage sidecar (empty when TOKEN_LOG_FILE is +# unset). Engine invocations often run inside a `cmd | tee` pipeline — i.e. a +# SUBSHELL — so the LAST_* globals they set never reach the parent. Writing usage +# to this file (the filesystem is shared) lets _record_engine_tokens read it back. +# $$ (shell PID) is constant across pipeline subshells: both the writing left-hand +# side and the reading parent see the same value. Separate background processes +# (e.g. `run_agentic &` forking a new process vs `(run_duck) &` running as a +# subshell) each have a distinct $$, so concurrent tier-2 calls never share a +# sidecar file. +_engine_usage_sidecar() { + [ -n "${TOKEN_LOG_FILE:-}" ] || return 0 + printf '%s' "${TOKEN_LOG_FILE}.last-usage.$$" +} + +# reset_engine_usage +# Clears the LAST_* usage globals and the sidecar file. Call before each engine +# invocation so a parse failure falls back to estimation rather than reusing a +# prior call's numbers. +reset_engine_usage() { + LAST_USAGE_OK=0 + LAST_INPUT_TOKENS=0 + LAST_CACHE_READ_TOKENS=0 + LAST_CACHE_WRITE_TOKENS=0 + LAST_OUTPUT_TOKENS=0 + local f; f="$(_engine_usage_sidecar)" + [ -n "$f" ] && rm -f "$f" 2>/dev/null || true +} + +# _usage_all_numeric +# True when every argument is a non-negative integer (guards against null/garbage +# from a malformed usage block). +_usage_all_numeric() { + local v + for v in "$@"; do + case "$v" in + ''|*[!0-9]*) return 1 ;; + esac + done + return 0 +} + +# parse_engine_usage +# Parses a usage block into the LAST_* globals. Returns 0 and sets LAST_USAGE_OK=1 +# on success; returns 1 (LAST_USAGE_OK stays 0) when usage is absent/malformed so +# the caller estimates instead. Never aborts (safe under set -e via the `||` guards). +parse_engine_usage() { + local engine="$1" file="$2" + reset_engine_usage + [ -f "$file" ] || return 1 + command -v jq >/dev/null 2>&1 || return 1 + + local i cr cw o + case "$engine" in + claude) + # claude --print --output-format json → top-level .usage block. + IFS=$'\t' read -r i cr cw o < <(jq -r ' + (.usage // {}) | + [ (.input_tokens // 0), (.cache_read_input_tokens // 0), + (.cache_creation_input_tokens // 0), (.output_tokens // 0) ] | @tsv + ' "$file" 2>/dev/null) || return 1 + ;; + gemini) + # gemini --output-format json → .stats.models..tokens. + # prompt = total input (includes cached); cached = cache-read; gemini bills + # no separate cache-write, so cache_write = 0 and input = prompt - cached. + local prompt cached + IFS=$'\t' read -r prompt cached o < <(jq -r ' + ([.. | objects | select(has("tokens")) | .tokens] | first // {}) as $t | + [ ($t.prompt // 0), ($t.cached // 0), (($t.candidates // $t.output // 0) + ($t.thoughts // 0)) ] | @tsv + ' "$file" 2>/dev/null) || return 1 + _usage_all_numeric "$prompt" "$cached" "$o" || return 1 + cr="$cached"; cw=0 + i=$(( prompt > cached ? prompt - cached : prompt )) + ;; + *) + return 1 ;; + esac + + _usage_all_numeric "$i" "$cr" "$cw" "$o" || return 1 + # All-zero usually means the block was missing rather than a real empty call. + [ "$i" -eq 0 ] && [ "$cr" -eq 0 ] && [ "$cw" -eq 0 ] && [ "$o" -eq 0 ] && return 1 + + # These globals are consumed by engine.sh (which sources this lib); shellcheck + # can't see the cross-file read. + # shellcheck disable=SC2034 + LAST_INPUT_TOKENS="$i" + # shellcheck disable=SC2034 + LAST_CACHE_READ_TOKENS="$cr" + # shellcheck disable=SC2034 + LAST_CACHE_WRITE_TOKENS="$cw" + # shellcheck disable=SC2034 + LAST_OUTPUT_TOKENS="$o" + # shellcheck disable=SC2034 + LAST_USAGE_OK=1 + + # Persist to the sidecar so a parent shell can read usage captured inside a + # `cmd | tee` subshell. + local f; f="$(_engine_usage_sidecar)" + [ -n "$f" ] && printf '%s\t%s\t%s\t%s\n' "$i" "$cr" "$cw" "$o" > "$f" 2>/dev/null || true + return 0 +} + +# extract_engine_text +# Prints the human-readable result text from an engine's JSON envelope to stdout. +# Prints nothing when the field is absent (caller decides how to fall back). +extract_engine_text() { + local engine="$1" file="$2" + [ -f "$file" ] || return 0 + command -v jq >/dev/null 2>&1 || { cat "$file" 2>/dev/null; return 0; } + case "$engine" in + claude) jq -j '.result // empty' "$file" 2>/dev/null ;; + gemini) jq -j '(.response // .text // empty)' "$file" 2>/dev/null ;; + *) cat "$file" 2>/dev/null ;; + esac +} diff --git a/scripts/token_report.sh b/scripts/token_report.sh new file mode 100755 index 000000000..30ca90fff --- /dev/null +++ b/scripts/token_report.sh @@ -0,0 +1,350 @@ +#!/usr/bin/env bash +# token_report.sh — org-wide Token Cost Observatory report. +# +# Collects per-call token-usage JSONL artifacts (emitted by scripts/lib/token-metrics.sh +# via the pr-review and dev-lead agents) from EVERY non-archived repo in the org, then +# renders a Markdown report aggregated by workflow/tier/model and by repository. +# +# Why org-wide: the agents run as reusable workflows in each *caller* repo, so their +# token-usage artifacts land in those repos — not in .github-private. A single-repo scan +# (the original fleet-monitor inline step) saw only a few percent of real org spend. +# +# Layout: +# * The render_* functions are PURE — they read a directory of JSONL files and write +# Markdown to stdout. They are unit-tested in tests/token_report.bats (no network). +# * main() does the network I/O: repo discovery, artifact download, extraction. +# +# Usage: +# ORG=petry-projects LOOKBACK_DAYS=7 GH_TOKEN= bash scripts/token_report.sh > report.md +# +# Environment: +# ORG — GitHub org to scan (default: petry-projects) +# LOOKBACK_DAYS — rolling window of artifact history to include (default: 7) +# GH_TOKEN — PAT with actions:read across the org (artifacts are per-repo) +# TOKEN_REPORT_OUT — optional path; when set, the report is written there in addition +# to stdout. +# +# ET formula (GitHub's framework): ET = m × (1.0×I + 0.1×C + 4.0×O) +# The `et` field is precomputed per call by token-metrics.sh; this script only sums it. + +set -euo pipefail + +ORG="${ORG:-petry-projects}" +LOOKBACK_DAYS="${LOOKBACK_DAYS:-7}" + +# --------------------------------------------------------------------------- +# Pricing — load the dated price table so cost + ET share one source of truth. +# --------------------------------------------------------------------------- +if [ -f "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/model-pricing.sh" ]; then + # shellcheck source=scripts/lib/model-pricing.sh + source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/model-pricing.sh" +fi + +# --------------------------------------------------------------------------- +# Pure rendering helpers (unit-tested) +# --------------------------------------------------------------------------- + +# _fmt_int +# Renders an integer with thousands separators (e.g. 1234567 → 1,234,567). +_fmt_int() { + awk -v n="${1:-0}" 'BEGIN { + n = int(n + 0.5); s = ""; if (n < 0) { neg = 1; n = -n } + if (n == 0) s = "0" + while (n > 0) { r = n % 1000; n = int(n / 1000); + s = (n > 0) ? sprintf("%03d", r) (s == "" ? "" : "," s) : r (s == "" ? "" : "," s) } + printf "%s%s", (neg ? "-" : ""), s + }' +} + +# _fmt_usd +# Renders a USD amount to 4 decimals (small per-window costs stay legible), e.g. $1.0548. +_fmt_usd() { + awk -v v="${1:-0}" 'BEGIN { printf "$%.4f", v }' +} + +# annotate_records +# Joins every record against the dated price table and emits enriched TSV (one row per +# call), pricing each call at the rate in effect on its OWN timestamp. Columns: +# 1 repo 2 workflow 3 tier 4 model 5 input 6 cache 7 output +# 8 cost_usd (-1 when the model price is unknown) 9 et 10 known(1/0) 11 context +# ET is recomputed here from the table (m = input(model)/input(haiku)), so it can never +# drift from the dollar figure. +annotate_records() { + local dir="$1" + local files=("$dir"/*.jsonl) + [ -e "${files[0]}" ] || return 0 # no JSONL files → no rows + jq -r 'select(type == "object") | [ + (.repo // "unknown"), (.workflow // "unknown"), (.tier // "-"), (.model // "-"), + (.input_tokens // 0), (.cache_read_tokens // 0), (.output_tokens // 0), + (.ts // "-"), (.context // ""), (.cache_creation_tokens // 0) + ] | @tsv' "${files[@]}" 2>/dev/null \ + | awk -F'\t' -v table="${PRICING_TABLE:-}" -v baseline="${ET_BASELINE_MODEL:-claude-haiku-4-5}" ' + function glob2re(g, re) { + re = g + gsub(/[.[\]()^$+{}|\\]/, "\\\\&", re); gsub(/\*/, ".*", re); gsub(/\?/, ".", re) + return "^" re "$" + } + function best_idx(model, d, i, bs, be, bi) { + bs = -1; be = ""; bi = 0 + for (i = 1; i <= nr; i++) + if (model ~ gre[i] && eff[i] <= d) + if (spec[i] > bs || (spec[i] == bs && eff[i] > be)) { bs = spec[i]; be = eff[i]; bi = i } + return bi + } + BEGIN { + nr = 0 + if (table != "") + while ((getline line < table) > 0) { + if (line ~ /^[[:space:]]*#/) continue + n = split(line, f, "\t"); if (n < 6) continue + nr++; gre[nr] = glob2re(f[1]); eff[nr] = f[2] + tin[nr] = f[3]; tcr[nr] = f[4]; tcw[nr] = f[5]; tout[nr] = f[6] + lit = f[1]; gsub(/[*?]/, "", lit); spec[nr] = length(lit) + } + } + { + repo = $1; wf = $2; tier = $3; model = $4 + inp = $5 + 0; ca = $6 + 0; out = $7 + 0; d = substr($8, 1, 10); ctx = $9 + cw = $10 + 0 + mi = best_idx(model, d) + bi = best_idx(baseline, d) + bpin = (bi > 0) ? tin[bi] : 0 + if (mi > 0) { + known = 1 + cost = (inp * tin[mi] + ca * tcr[mi] + cw * tcw[mi] + out * tout[mi]) / 1000000 + m = (bpin > 0) ? tin[mi] / bpin : 1.0 + } else { known = 0; cost = -1; m = 1.0 } + et = m * (1.0 * inp + 0.1 * ca + 4.0 * out) + # Enriched cols 1-11 unchanged for downstream; cache_write appended as col 12. + printf "%s\t%s\t%s\t%s\t%d\t%d\t%d\t%.6f\t%.4f\t%d\t%s\t%d\n", + repo, wf, tier, model, inp, ca, out, cost, et, known, ctx, cw + }' +} + +# render_token_report [generated_at] +# Writes the full Markdown report (with USD cost) to stdout. Pure: no network. +render_token_report() { + local dir="$1" lookback="$2" repo_count="$3" artifact_count="$4" generated_at="${5:-}" + + local enriched; enriched="$(mktemp)" + annotate_records "$dir" > "$enriched" + + local total_calls + total_calls="$(wc -l < "$enriched" | tr -d ' ')" + + printf '# 📊 Token Cost Observatory — %s-day Report\n\n' "$lookback" + if [ -n "$generated_at" ]; then + printf '_Generated %s · org `%s` · %s repos scanned · %s agent runs · %s LLM calls_\n\n' \ + "$generated_at" "$ORG" "$repo_count" "$artifact_count" "$(_fmt_int "$total_calls")" + else + printf '_org `%s` · %s repos scanned · %s agent runs · %s LLM calls_\n\n' \ + "$ORG" "$repo_count" "$artifact_count" "$(_fmt_int "$total_calls")" + fi + + if [ "$total_calls" -eq 0 ]; then + rm -f "$enriched" + printf 'No token-usage records found in the last %s days.\n' "$lookback" + return 0 + fi + + # Totals (cost over priced calls; ET over all): et, cost, input, cache_read, cache_write, output, unknown + local totals total_et total_cost total_in total_cr total_cw total_out unpriced + totals="$(awk -F'\t' '{ et += $9; inp += $5; cr += $6; cw += $12; out += $7; + if ($10 == 1) cost += $8; else unk++ } + END { printf "%.4f\t%.6f\t%d\t%d\t%d\t%d\t%d", et, cost, inp, cr, cw, out, unk }' "$enriched")" + IFS=$'\t' read -r total_et total_cost total_in total_cr total_cw total_out unpriced <<< "$totals" + + printf 'Estimated USD cost, priced per `scripts/lib/model-pricing.tsv` at each call'"'"'s date ' + printf '(input + cache-read + cache-write + output). ' + printf 'Effective Tokens (ET) `= m × (1.0·input + 0.1·cache + 4.0·output)`, ' + printf 'where `m` = model input price ÷ haiku input price (haiku 1× · sonnet 3× · opus 5×).\n\n' + + printf '## Totals\n\n' + printf -- '- **Total cost:** %s\n' "$(_fmt_usd "$total_cost")" + printf -- '- **Total ET:** %s\n' "$(_fmt_int "$total_et")" + printf -- '- **LLM calls:** %s · **Input tokens:** %s · **Output tokens:** %s\n' \ + "$(_fmt_int "$total_calls")" "$(_fmt_int "$total_in")" "$(_fmt_int "$total_out")" + printf -- '- **Cache tokens:** %s read · %s write\n' \ + "$(_fmt_int "$total_cr")" "$(_fmt_int "$total_cw")" + if [ "${unpriced:-0}" -gt 0 ]; then + printf -- '- ⚠️ **%s call(s) had no price** in the table and are excluded from cost (marked `*`).\n' \ + "$(_fmt_int "$unpriced")" + fi + printf '\n' + + printf '## Top cost drivers (workflow / tier / model)\n\n' + printf '| Workflow | Tier | Model | Calls | Input | Cache | Output | Cost | %% of $ | ET |\n' + printf '|---|---|---|---:|---:|---:|---:|---:|---:|---:|\n' + awk -F'\t' '{ k = $2"\t"$3"\t"$4 + calls[k]++; inp[k] += $5; ca[k] += $6; out[k] += $7; et[k] += $9 + if ($10 == 1) cost[k] += $8; else unk[k]++ } + END { for (k in calls) + printf "%.6f\t%.4f\t%d\t%d\t%d\t%d\t%d\t%s\n", + cost[k], et[k], calls[k], inp[k], ca[k], out[k], unk[k], k }' "$enriched" \ + | sort -t$'\t' -k1,1rn \ + | while IFS=$'\t' read -r cost et calls input cache output unk wf tier model; do + local pct mark=""; [ "$unk" -gt 0 ] && mark="*" + pct="$(awk -v c="$cost" -v t="$total_cost" 'BEGIN { printf "%d%%", (t > 0 ? c / t * 100 : 0) }')" + printf '| `%s` | %s | `%s` | %s | %s | %s | %s | %s%s | %s | %s |\n' \ + "$wf" "$tier" "$model" "$(_fmt_int "$calls")" "$(_fmt_int "$input")" \ + "$(_fmt_int "$cache")" "$(_fmt_int "$output")" "$(_fmt_usd "$cost")" "$mark" "$pct" "$(_fmt_int "$et")" + done + printf '\n' + + printf '## By repository\n\n' + printf '| Repository | Calls | Cost | %% of $ | ET |\n' + printf '|---|---:|---:|---:|---:|\n' + awk -F'\t' '{ k = $1 + calls[k]++; et[k] += $9; if ($10 == 1) cost[k] += $8; else unk[k]++ } + END { for (k in calls) + printf "%.6f\t%.4f\t%d\t%d\t%s\n", cost[k], et[k], calls[k], unk[k], k }' "$enriched" \ + | sort -t$'\t' -k1,1rn \ + | while IFS=$'\t' read -r cost et calls unk repo; do + local pct mark=""; [ "$unk" -gt 0 ] && mark="*" + pct="$(awk -v c="$cost" -v t="$total_cost" 'BEGIN { printf "%d%%", (t > 0 ? c / t * 100 : 0) }')" + printf '| `%s` | %s | %s%s | %s | %s |\n' \ + "$repo" "$(_fmt_int "$calls")" "$(_fmt_usd "$cost")" "$mark" "$pct" "$(_fmt_int "$et")" + done + printf '\n' + + # Cost-per-PR — each record carries its PR URL in context; surface the priciest PRs. + # Limit with `awk 'NR<=10'` rather than `head -10`: awk consumes the whole stream, so + # `sort` never receives SIGPIPE and the command substitution can't fail under pipefail + # (which would otherwise abort the report — see PR #456 review). + local pr_rows + pr_rows="$(awk -F'\t' '$11 ~ /\/pull\// { k = $11 + calls[k]++; if ($10 == 1) cost[k] += $8 } + END { for (k in calls) printf "%.6f\t%d\t%s\n", cost[k], calls[k], k }' "$enriched" \ + | sort -t$'\t' -k1,1rn | awk 'NR <= 10')" + if [ -n "$pr_rows" ]; then + printf '## Most expensive PRs (top 10)\n\n' + printf '| PR | Calls | Cost |\n|---|---:|---:|\n' + while IFS=$'\t' read -r cost calls pr; do + [ -n "$pr" ] || continue + printf '| %s | %s | %s |\n' "$pr" "$(_fmt_int "$calls")" "$(_fmt_usd "$cost")" + done <<< "$pr_rows" + printf '\n' + fi + + rm -f "$enriched" +} + +# --------------------------------------------------------------------------- +# Network I/O (main) +# --------------------------------------------------------------------------- + +# _extract_zip +# Extracts a zip using unzip when available, else a python3 fallback (CI runners +# have unzip; the fallback keeps the script runnable in minimal environments). +_extract_zip() { + local zip="$1" dest="$2" + if command -v unzip >/dev/null 2>&1; then + unzip -q -o "$zip" -d "$dest" + else + python3 - "$zip" "$dest" <<'PY' +import sys, zipfile +with zipfile.ZipFile(sys.argv[1]) as z: + z.extractall(sys.argv[2]) +PY + fi +} + +# collect_org_jsonl (stdout: " ") +# Discovers non-archived repos, downloads each token-usage artifact inside the +# lookback window, extracts its JSONL, and tags every record with its source repo. +collect_org_jsonl() { + local jsonl_dir="$1" + mkdir -p "$jsonl_dir" + + local repos_raw + if ! repos_raw="$(gh api "orgs/${ORG}/repos?per_page=100&type=all" --paginate \ + --jq '.[] | select(.archived == false) | .full_name')"; then + echo "ERROR: org repo discovery for '${ORG}' failed — verify GH_TOKEN has org read access." >&2 + return 1 + fi + [ -n "$repos_raw" ] || { echo "0 0"; return 0; } + + local cutoff repo_count=0 artifact_count=0 + cutoff="$(date -u -d "${LOOKBACK_DAYS} days ago" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null \ + || date -u -v-"${LOOKBACK_DAYS}"d +%Y-%m-%dT%H:%M:%SZ)" + + local workdir; workdir="$(mktemp -d)" + # shellcheck disable=SC2064 + trap "rm -rf '$workdir'" RETURN + + local repo + while IFS= read -r repo; do + [ -n "$repo" ] || continue + repo_count=$((repo_count + 1)) + + local arts + arts="$(gh api "repos/${repo}/actions/artifacts" --paginate 2>/dev/null \ + | jq -r --arg cutoff "$cutoff" \ + '.artifacts[] | select(.name | startswith("token-usage-")) | select(.expired == false) | select(.created_at >= $cutoff) | .id | tostring')" || { + echo "WARN: could not fetch artifacts for ${repo} — skipping (check actions:read permission)" >&2 + continue + } + [ -n "$arts" ] || continue + + local id + while IFS= read -r id; do + [ -n "$id" ] || continue + + local zip="$workdir/a-$id.zip" ex="$workdir/x-$id" + mkdir -p "$ex" + if ! gh api "repos/${repo}/actions/artifacts/${id}/zip" > "$zip" 2>/dev/null; then + echo "WARN: artifact ${id} from ${repo} — download failed; skipping (report may be incomplete)" >&2 + continue + fi + if ! _extract_zip "$zip" "$ex" 2>/dev/null; then + echo "WARN: artifact ${id} from ${repo} — extraction failed; skipping (report may be incomplete)" >&2 + continue + fi + + local found=false f dest + while IFS= read -r f; do + # Tag each record with its source repo for the by-repo rollup. + dest="$jsonl_dir/${id}-$(basename "$f")" + if jq -c --arg repo "$repo" 'select(type=="object") | . + {repo: $repo}' \ + "$f" > "$dest" 2>/dev/null; then + found=true + else + # jq creates the destination before it can fail on malformed input; + # remove it so annotate_records never sees a corrupt *.jsonl file. + rm -f "$dest" 2>/dev/null || true + fi + done < <(find "$ex" -type f -name '*.jsonl' -print) + [ "$found" = true ] && artifact_count=$((artifact_count + 1)) + + rm -rf "$zip" "$ex" + done <<< "$arts" + done <<< "$repos_raw" + + echo "${repo_count} ${artifact_count}" +} + +main() { + local jsonl_dir counts repo_count artifact_count generated_at report + jsonl_dir="$(mktemp -d)" + # shellcheck disable=SC2064 + trap "rm -rf '$jsonl_dir'" EXIT + + echo "Collecting token-usage artifacts across ${ORG} (last ${LOOKBACK_DAYS} days)..." >&2 + counts="$(collect_org_jsonl "$jsonl_dir")" + repo_count="${counts%% *}" + artifact_count="${counts##* }" + generated_at="$(date -u +%Y-%m-%d 2>/dev/null || echo '')" + + report="$(render_token_report "$jsonl_dir" "$LOOKBACK_DAYS" \ + "$repo_count" "$artifact_count" "$generated_at")" + + printf '%s\n' "$report" + if [ -n "${TOKEN_REPORT_OUT:-}" ]; then + printf '%s\n' "$report" > "$TOKEN_REPORT_OUT" + fi +} + +# Only run main when executed directly (not when sourced by tests). +if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + main "$@" +fi diff --git a/tests/dev-lead/fixtures/engines/stub-claude b/tests/dev-lead/fixtures/engines/stub-claude index 017f89edb..60d6dcb34 100755 --- a/tests/dev-lead/fixtures/engines/stub-claude +++ b/tests/dev-lead/fixtures/engines/stub-claude @@ -9,13 +9,19 @@ # same lookup as EXIT_BY_MODEL. # STUB_ENGINE_RECORD_MODELS — if set to a writable path, every --model value # seen is appended as a new line for inspection. +# STUB_CLAUDE_USAGE — "input cache_read cache_write output" emitted as the +# .usage block when --output-format json is requested. +# Default "100 20 30 40". model="" +output_format="text" while [[ $# -gt 0 ]]; do case "$1" in --print) shift ;; --permission-mode|--allowed-tools|--disallowed-tools|--max-turns) shift; shift ;; + --output-format) + output_format="$2"; shift; shift ;; --model) model="$2"; shift; shift ;; *) shift ;; @@ -38,7 +44,24 @@ if [ -n "${STUB_ENGINE_RESPONSE_BY_MODEL:-}" ] && [ -n "$model" ]; then esac done fi -printf '%s\n' "$response" + +if [ "$output_format" = "json" ]; then + # Mirror `claude --print --output-format json`: a result envelope with usage. + read -r _in _cr _cw _out <<< "${STUB_CLAUDE_USAGE:-100 20 30 40}" + if command -v jq >/dev/null 2>&1; then + jq -cn --arg result "$response" \ + --argjson i "${_in:-0}" --argjson cr "${_cr:-0}" \ + --argjson cw "${_cw:-0}" --argjson o "${_out:-0}" \ + '{type:"result", subtype:"success", is_error:false, result:$result, + usage:{input_tokens:$i, cache_read_input_tokens:$cr, + cache_creation_input_tokens:$cw, output_tokens:$o}}' + else + printf '{"type":"result","result":"%s","usage":{"input_tokens":%s,"cache_read_input_tokens":%s,"cache_creation_input_tokens":%s,"output_tokens":%s}}\n' \ + "$response" "${_in:-0}" "${_cr:-0}" "${_cw:-0}" "${_out:-0}" + fi +else + printf '%s\n' "$response" +fi # Resolve exit code: per-model override → default. exit_code="${STUB_ENGINE_EXIT:-0}" diff --git a/tests/dev-lead/fixtures/engines/stub-gemini b/tests/dev-lead/fixtures/engines/stub-gemini index efff98bd8..aec46d83e 100755 --- a/tests/dev-lead/fixtures/engines/stub-gemini +++ b/tests/dev-lead/fixtures/engines/stub-gemini @@ -1,12 +1,46 @@ #!/usr/bin/env bash -# Configurable via: STUB_ENGINE_RESPONSE, STUB_ENGINE_EXIT, STUB_ENGINE_DELAY +# Configurable via: +# STUB_ENGINE_RESPONSE, STUB_ENGINE_EXIT, STUB_ENGINE_DELAY +# STUB_GEMINI_USAGE — "prompt cached candidates" emitted under +# .stats.models..tokens when --output-format json +# is requested. Default "120 20 40". +# STUB_GEMINI_THOUGHTS — thinking token count added to .tokens.thoughts. +# Default 0 (omitted when 0). +model="stub-gemini-model" +output_format="text" while [[ $# -gt 0 ]]; do case "$1" in - --prompt|--model|--approval-mode|--output-format) + --model) + model="$2"; shift; shift ;; + --output-format) + output_format="$2"; shift; shift ;; + --prompt|--approval-mode) shift; shift ;; *) shift ;; esac done sleep "${STUB_ENGINE_DELAY:-0}" -printf '%s\n' "${STUB_ENGINE_RESPONSE:-stub engine response}" + +response="${STUB_ENGINE_RESPONSE:-stub engine response}" + +if [ "$output_format" = "json" ]; then + read -r _p _c _cand <<< "${STUB_GEMINI_USAGE:-120 20 40}" + _th="${STUB_GEMINI_THOUGHTS:-0}" + if command -v jq >/dev/null 2>&1; then + jq -cn --arg response "$response" --arg model "$model" \ + --argjson p "${_p:-0}" --argjson c "${_c:-0}" --argjson cand "${_cand:-0}" \ + --argjson th "${_th:-0}" \ + '{response:$response, + stats:{models:{($model):{tokens:( + {prompt:$p, cached:$c, candidates:$cand, total:($p + $cand)} + | if $th > 0 then . + {thoughts:$th} else . end + )}}}}' + else + printf '{"response":"%s","stats":{"models":{"%s":{"tokens":{"prompt":%s,"cached":%s,"candidates":%s}}}}}\n' \ + "$response" "$model" "${_p:-0}" "${_c:-0}" "${_cand:-0}" + fi +else + printf '%s\n' "$response" +fi + exit "${STUB_ENGINE_EXIT:-0}" diff --git a/tests/dev-lead/unit/test_engine_writer.bats b/tests/dev-lead/unit/test_engine_writer.bats index b36c91607..76a5e33d5 100644 --- a/tests/dev-lead/unit/test_engine_writer.bats +++ b/tests/dev-lead/unit/test_engine_writer.bats @@ -415,6 +415,77 @@ STUB [ "$status" -eq 0 ] } +# ── real-usage capture (cache) tests ────────────────────────────────────────── + +@test "usage: claude run captures real cache-read and cache-write from JSON" { + _source_engine "claude" + export STUB_ENGINE_EXIT=0 + export STUB_ENGINE_RESPONSE="verdict text" + export STUB_CLAUDE_USAGE="2000 800 600 150" # input cache_read cache_write output + local log; log=$(mktemp) + export TOKEN_LOG_FILE="$log" + export TEST_OWNED_TOKEN_LOG="$log" + + run run_triage "$TEST_PROMPT" + + [ "$status" -eq 0 ] + [ "$(jq -r '.input_tokens' < "$log")" = "2000" ] + [ "$(jq -r '.cache_read_tokens' < "$log")" = "800" ] + [ "$(jq -r '.cache_creation_tokens' < "$log")" = "600" ] + [ "$(jq -r '.output_tokens' < "$log")" = "150" ] +} + +@test "usage: claude JSON mode still returns plain text to the caller (not JSON)" { + _source_engine "claude" + export STUB_ENGINE_EXIT=0 + export STUB_ENGINE_RESPONSE="just the verdict" + export STUB_CLAUDE_USAGE="10 0 0 5" + local log; log=$(mktemp) + export TOKEN_LOG_FILE="$log" + export TEST_OWNED_TOKEN_LOG="$log" + + run run_triage "$TEST_PROMPT" + + [ "$status" -eq 0 ] + [ "$output" = "just the verdict" ] # extracted .result, not the JSON envelope +} + +@test "usage: ENGINE_USAGE_JSON=0 disables JSON capture (falls back to estimate)" { + _source_engine "claude" + export STUB_ENGINE_EXIT=0 + export STUB_ENGINE_RESPONSE="verdict" + export STUB_CLAUDE_USAGE="2000 800 600 150" + export ENGINE_USAGE_JSON=0 + local log; log=$(mktemp) + export TOKEN_LOG_FILE="$log" + export TEST_OWNED_TOKEN_LOG="$log" + + run run_triage "$TEST_PROMPT" + + [ "$status" -eq 0 ] + # Estimate path → cache figures unknown → 0 (not the stub's 800/600). + [ "$(jq -r '.cache_read_tokens' < "$log")" = "0" ] + [ "$(jq -r '.cache_creation_tokens' < "$log")" = "0" ] +} + +@test "usage: gemini run captures usage (input = prompt - cached)" { + _source_engine "gemini" + export STUB_ENGINE_EXIT=0 + export STUB_ENGINE_RESPONSE="gem verdict" + export STUB_GEMINI_USAGE="500 100 60" # prompt cached candidates + local log; log=$(mktemp) + export TOKEN_LOG_FILE="$log" + export TEST_OWNED_TOKEN_LOG="$log" + + run run_triage "$TEST_PROMPT" + + [ "$status" -eq 0 ] + [ "$(jq -r '.engine' < "$log")" = "gemini" ] + [ "$(jq -r '.input_tokens' < "$log")" = "400" ] + [ "$(jq -r '.cache_read_tokens' < "$log")" = "100" ] + [ "$(jq -r '.output_tokens' < "$log")" = "60" ] +} + @test "token: run_writer_with_fallback logs the fallback engine (not rate-limited primary)" { _source_engine "claude" export DEV_LEAD_DRY_RUN=false diff --git a/tests/dev-lead/unit/test_token_metrics.bats b/tests/dev-lead/unit/test_token_metrics.bats index 4276a53a6..485f2ff07 100644 --- a/tests/dev-lead/unit/test_token_metrics.bats +++ b/tests/dev-lead/unit/test_token_metrics.bats @@ -30,10 +30,10 @@ teardown() { [ "$output" = "3.0" ] } -@test "model_multiplier_for: opus 4.7 returns 15.0" { +@test "model_multiplier_for: opus 4.7 returns 5.0 (table-derived: \$5 input / \$1 haiku)" { run model_multiplier_for "claude-opus-4-7" [ "$status" -eq 0 ] - [ "$output" = "15.0" ] + [ "$output" = "5.0" ] } @test "model_multiplier_for: o4-mini returns 2.0" { @@ -180,11 +180,96 @@ teardown() { } @test "emit_token_record: et value matches formula for opus" { - # m=15.0, I=1000, C=500, O=100 → ET = 15.0*(1000+50+400) = 21750.00 + # m=5.0 (Opus 4.5+ = \$5 input), I=1000, C=500, O=100 → ET = 5.0*(1000+50+400) = 7250.00 emit_token_record "pr-review" "audit" "claude" "claude-opus-4-7" 1000 500 100 "" local et et=$(jq -r '.et' < "$TOKEN_LOG_FILE") - [ "$et" = "21750.00" ] + [ "$et" = "7250.00" ] +} + +@test "emit_token_record: records cache_creation_tokens from the 9th arg" { + emit_token_record "pr-review" "deep" "claude" "claude-sonnet-4-6" 1000 200 100 "" 350 + local cw + cw=$(jq -r '.cache_creation_tokens' < "$TOKEN_LOG_FILE") + [ "$cw" = "350" ] +} + +@test "emit_token_record: cache_creation_tokens defaults to 0 when omitted" { + emit_token_record "pr-review" "triage" "claude" "claude-haiku-4-5-20251001" 1000 200 100 "" + local cw + cw=$(jq -r '.cache_creation_tokens' < "$TOKEN_LOG_FILE") + [ "$cw" = "0" ] +} + +# ── parse_engine_usage / extract_engine_text ────────────────────────────────── + +@test "parse_engine_usage: claude JSON sets input/cache-read/cache-write/output" { + local f; f=$(mktemp) + printf '%s\n' '{"result":"hi","usage":{"input_tokens":1234,"cache_read_input_tokens":500,"cache_creation_input_tokens":300,"output_tokens":90}}' > "$f" + run parse_engine_usage claude "$f" + rm -f "$f" + [ "$status" -eq 0 ] +} + +@test "parse_engine_usage: claude globals carry the parsed values" { + local f; f=$(mktemp) + printf '%s\n' '{"result":"hi","usage":{"input_tokens":1234,"cache_read_input_tokens":500,"cache_creation_input_tokens":300,"output_tokens":90}}' > "$f" + parse_engine_usage claude "$f" + rm -f "$f" + [ "$LAST_USAGE_OK" = "1" ] + [ "$LAST_INPUT_TOKENS" = "1234" ] + [ "$LAST_CACHE_READ_TOKENS" = "500" ] + [ "$LAST_CACHE_WRITE_TOKENS" = "300" ] + [ "$LAST_OUTPUT_TOKENS" = "90" ] +} + +@test "parse_engine_usage: gemini computes non-cached input = prompt - cached" { + local f; f=$(mktemp) + printf '%s\n' '{"response":"hi","stats":{"models":{"g":{"tokens":{"prompt":500,"cached":100,"candidates":60}}}}}' > "$f" + parse_engine_usage gemini "$f" + rm -f "$f" + [ "$LAST_USAGE_OK" = "1" ] + [ "$LAST_INPUT_TOKENS" = "400" ] + [ "$LAST_CACHE_READ_TOKENS" = "100" ] + [ "$LAST_CACHE_WRITE_TOKENS" = "0" ] + [ "$LAST_OUTPUT_TOKENS" = "60" ] +} + +@test "parse_engine_usage: gemini adds thinking tokens to output count" { + local f; f=$(mktemp) + printf '%s\n' '{"response":"hi","stats":{"models":{"g":{"tokens":{"prompt":500,"cached":100,"candidates":60,"thoughts":25}}}}}' > "$f" + parse_engine_usage gemini "$f" + rm -f "$f" + [ "$LAST_USAGE_OK" = "1" ] + [ "$LAST_INPUT_TOKENS" = "400" ] + [ "$LAST_CACHE_READ_TOKENS" = "100" ] + [ "$LAST_OUTPUT_TOKENS" = "85" ] # 60 candidates + 25 thoughts +} + +@test "_engine_usage_sidecar: path includes shell PID to isolate parallel invocations" { + # $$ is constant across pipeline subshells (left-hand side inherits parent PID) + # so both the writing subshell and the reading parent agree on the same path, + # while separate background processes each get their own distinct $$. + local sidecar + sidecar="$(_engine_usage_sidecar)" + [ -n "$sidecar" ] + [[ "$sidecar" == "${TOKEN_LOG_FILE}.last-usage.$$" ]] +} + +@test "parse_engine_usage: missing usage block returns non-zero (→ estimate)" { + local f; f=$(mktemp) + printf '%s\n' '{"result":"no usage here"}' > "$f" + run parse_engine_usage claude "$f" + rm -f "$f" + [ "$status" -ne 0 ] +} + +@test "extract_engine_text: claude returns the .result text" { + local f; f=$(mktemp) + printf '%s\n' '{"result":"hello world","usage":{}}' > "$f" + run extract_engine_text claude "$f" + rm -f "$f" + [ "$output" = "hello world" ] } @test "emit_token_record: is a no-op when TOKEN_LOG_FILE is unset" { diff --git a/tests/fixtures/token_jsonl/run-a.jsonl b/tests/fixtures/token_jsonl/run-a.jsonl new file mode 100644 index 000000000..3822acb47 --- /dev/null +++ b/tests/fixtures/token_jsonl/run-a.jsonl @@ -0,0 +1,2 @@ +{"ts":"2026-06-01T10:00:00Z","workflow":"pr-review","tier":"single","engine":"claude","model":"claude-opus-4-7","input_tokens":1000,"cache_read_tokens":500,"output_tokens":200,"et":80250,"run_id":"a","context":"https://github.com/petry-projects/markets/pull/1","repo":"petry-projects/markets"} +{"ts":"2026-06-01T11:00:00Z","workflow":"dev-lead","tier":"writer","engine":"claude","model":"claude-sonnet-4-6","input_tokens":2000,"cache_read_tokens":0,"output_tokens":300,"et":9600,"run_id":"a","context":"https://github.com/petry-projects/markets/pull/2","repo":"petry-projects/markets"} diff --git a/tests/fixtures/token_jsonl/run-b.jsonl b/tests/fixtures/token_jsonl/run-b.jsonl new file mode 100644 index 000000000..e6b4b40d4 --- /dev/null +++ b/tests/fixtures/token_jsonl/run-b.jsonl @@ -0,0 +1,2 @@ +{"ts":"2026-06-02T10:05:00Z","workflow":"pr-review","tier":"triage","engine":"claude","model":"claude-haiku-4-5-20251001","input_tokens":500,"cache_read_tokens":1000,"output_tokens":50,"et":800,"run_id":"b","context":"https://github.com/petry-projects/broodly/pull/3","repo":"petry-projects/broodly"} +{"ts":"2026-06-02T10:00:00Z","workflow":"dev-lead","tier":"writer","engine":"claude","model":"claude-sonnet-4-6","input_tokens":1000,"cache_read_tokens":0,"output_tokens":100,"et":4200,"run_id":"b","context":"https://github.com/petry-projects/broodly/pull/3","repo":"petry-projects/broodly"} diff --git a/tests/model_pricing.bats b/tests/model_pricing.bats new file mode 100644 index 000000000..cab0e3137 --- /dev/null +++ b/tests/model_pricing.bats @@ -0,0 +1,104 @@ +#!/usr/bin/env bats +# Tests for scripts/lib/model-pricing.sh — effective-dated price lookup, cost, ET. +# Run locally: bats tests/model_pricing.bats + +setup() { + # shellcheck source=scripts/lib/model-pricing.sh + source "${BATS_TEST_DIRNAME}/../scripts/lib/model-pricing.sh" +} + +# --------------------------------------------------------------------------- +# price_for — against the production table +# --------------------------------------------------------------------------- + +# Output is "input cache_read cache_write output". +@test "price_for: opus 4.7 at a 2026 date → \$5 / \$0.50 / \$6.25 / \$25" { + run price_for "claude-opus-4-7" "2026-06-07" + [ "$output" = "5.00 0.50 6.25 25.00" ] +} + +@test "price_for: sonnet 4.6 → \$3 / \$0.30 / \$3.75 / \$15" { + run price_for "claude-sonnet-4-6" "2026-06-07" + [ "$output" = "3.00 0.30 3.75 15.00" ] +} + +@test "price_for: more-specific glob wins (opus-4-1 legacy = \$15, not \$5)" { + run price_for "claude-opus-4-1-20250805" "2026-06-07" + [ "$output" = "15.00 1.50 18.75 75.00" ] +} + +@test "price_for: unknown model → empty" { + run price_for "mystery-model-v9" "2026-06-07" + [ -z "$output" ] +} + +# --------------------------------------------------------------------------- +# Effective-date selection (dedicated fixture table with two dated rows) +# --------------------------------------------------------------------------- + +@test "price_for: selects the rate in effect on the record's date" { + PRICING_TABLE="$(mktemp)" + { + printf '# glob\teff\tin\tcr\tcw\tout\n' + printf 'test-model*\t2025-01-01\t10.00\t1.00\t12.50\t40.00\n' + printf 'test-model*\t2026-03-01\t8.00\t0.80\t10.00\t32.00\n' + } > "$PRICING_TABLE" + before="$(price_for "test-model-x" "2026-02-15")" # before the price drop + after="$(price_for "test-model-x" "2026-03-02")" # after the price drop + rm -f "$PRICING_TABLE" + [ "$before" = "10.00 1.00 12.50 40.00" ] + [ "$after" = "8.00 0.80 10.00 32.00" ] +} + +@test "price_for: date before the earliest effective_from → empty (no retroactive guess)" { + PRICING_TABLE="$(mktemp)" + printf 'test-model*\t2026-03-01\t8.00\t0.80\t10.00\t32.00\n' > "$PRICING_TABLE" + run price_for "test-model-x" "2025-01-01" + rm -f "$PRICING_TABLE" + [ -z "$output" ] +} + +# --------------------------------------------------------------------------- +# cost_usd +# --------------------------------------------------------------------------- + +@test "cost_usd: opus 21952/0/14665 ≈ \$0.476385 (no cache)" { + run cost_usd "claude-opus-4-7" 21952 0 14665 "2026-06-07" + [ "$output" = "0.476385" ] +} + +@test "cost_usd: includes cache-read and cache-write terms" { + # opus: input 0, cache_read 1000 (@\$0.50), output 0, cache_write 1000 (@\$6.25) + # = (1000*0.5 + 1000*6.25)/1e6 = 6750/1e6 = 0.006750 + run cost_usd "claude-opus-4-7" 0 1000 0 "2026-06-07" 1000 + [ "$output" = "0.006750" ] +} + +@test "cost_usd: unknown model → empty (never silently \$0)" { + run cost_usd "mystery-model-v9" 1000 0 500 "2026-06-07" + [ -z "$output" ] +} + +# --------------------------------------------------------------------------- +# et_multiplier_for — derived from the same table (fixes stale opus=15) +# --------------------------------------------------------------------------- + +@test "et_multiplier_for: haiku baseline = 1.0" { + run et_multiplier_for "claude-haiku-4-5-20251001" "2026-06-07" + [ "$output" = "1.0" ] +} + +@test "et_multiplier_for: sonnet = 3.0" { + run et_multiplier_for "claude-sonnet-4-6" "2026-06-07" + [ "$output" = "3.0" ] +} + +@test "et_multiplier_for: opus 4.7 = 5.0 (was hardcoded 15)" { + run et_multiplier_for "claude-opus-4-7" "2026-06-07" + [ "$output" = "5.0" ] +} + +@test "et_multiplier_for: unknown model → safe default 1.0" { + run et_multiplier_for "mystery-model-v9" "2026-06-07" + [ "$output" = "1.0" ] +} diff --git a/tests/token_report.bats b/tests/token_report.bats new file mode 100644 index 000000000..117447fb1 --- /dev/null +++ b/tests/token_report.bats @@ -0,0 +1,206 @@ +#!/usr/bin/env bats +# Tests for scripts/token_report.sh — pure annotation + Markdown rendering with USD cost. +# Network I/O (collect_org_jsonl / main) is not exercised here. +# Run locally: bats tests/token_report.bats + +FIXTURES="${BATS_TEST_DIRNAME}/fixtures/token_jsonl" + +setup() { + # shellcheck source=scripts/token_report.sh + source "${BATS_TEST_DIRNAME}/../scripts/token_report.sh" +} + +# --------------------------------------------------------------------------- +# _fmt_int / _fmt_usd +# --------------------------------------------------------------------------- + +@test "_fmt_int: adds thousands separators" { + run _fmt_int 1234567 + [ "$output" = "1,234,567" ] +} + +@test "_fmt_int: handles zero" { + run _fmt_int 0 + [ "$output" = "0" ] +} + +@test "_fmt_usd: renders dollars to 4 decimals" { + run _fmt_usd 1.0548 + [ "$output" = "\$1.0548" ] +} + +# --------------------------------------------------------------------------- +# annotate_records — date-accurate cost + table-derived ET +# --------------------------------------------------------------------------- + +@test "annotate_records: one enriched row per call" { + run bash -c "source '${BATS_TEST_DIRNAME}/../scripts/token_report.sh'; annotate_records '$FIXTURES' | wc -l" + [ "$output" -eq 4 ] +} + +@test "annotate_records: prices opus at \$5 input / \$25 output (cost & known flag)" { + # opus-4-7 1000/500/200 → (1000*5 + 500*0.5 + 200*25)/1e6 = 0.010250 + result="$(annotate_records "$FIXTURES" | awk -F'\t' '$4=="claude-opus-4-7"{print $8, $10}')" + [ "$result" = "0.010250 1" ] +} + +@test "annotate_records: recomputes ET from table (opus m=5, ignores stored et=80250)" { + # ET = 5 * (1000 + 0.1*500 + 4*200) = 5*1850 = 9250 + result="$(annotate_records "$FIXTURES" | awk -F'\t' '$4=="claude-opus-4-7"{print $9}')" + [ "$result" = "9250.0000" ] +} + +@test "annotate_records: unknown model → cost sentinel -1 and known=0" { + tmp="$(mktemp -d)" + printf '%s\n' '{"ts":"2026-06-01T00:00:00Z","workflow":"x","tier":"y","model":"mystery-model-v9","input_tokens":100,"output_tokens":50,"repo":"r","context":""}' > "$tmp/u.jsonl" + result="$(annotate_records "$tmp" | awk -F'\t' '{print $8, $10}')" + rm -rf "$tmp" + [ "$result" = "-1.000000 0" ] +} + +# --------------------------------------------------------------------------- +# render_token_report +# --------------------------------------------------------------------------- + +@test "render_token_report: total cost sums priced calls" { + # 0.010250 + 0.010500 + 0.004500 + 0.000850 = 0.026100 + run render_token_report "$FIXTURES" 7 2 2 2026-06-07 + [ "$status" -eq 0 ] + [[ "$output" == *"Total cost:** \$0.0261"* ]] +} + +@test "render_token_report: has cost columns and a cost-per-PR section" { + run render_token_report "$FIXTURES" 7 2 2 2026-06-07 + [[ "$output" == *"| Cost | % of \$ | ET |"* ]] + [[ "$output" == *"Most expensive PRs"* ]] +} + +@test "render_token_report: by-repo sorted by cost (markets first)" { + run render_token_report "$FIXTURES" 7 2 2 2026-06-07 + repos="$(printf '%s\n' "$output" | grep -oE 'petry-projects/(markets|broodly)' | head -2 | tr '\n' ' ')" + [ "$repos" = "petry-projects/markets petry-projects/broodly " ] +} + +@test "render_token_report: flags unpriced calls" { + tmp="$(mktemp -d)" + printf '%s\n' '{"ts":"2026-06-01T00:00:00Z","workflow":"x","tier":"y","model":"mystery-model-v9","input_tokens":100,"output_tokens":50,"repo":"r","context":""}' > "$tmp/u.jsonl" + run render_token_report "$tmp" 7 1 1 2026-06-07 + rm -rf "$tmp" + [[ "$output" == *"had no price"* ]] +} + +@test "render_token_report: empty dir yields a no-data message" { + empty="$(mktemp -d)" + run render_token_report "$empty" 7 0 0 2026-06-07 + rmdir "$empty" + [ "$status" -eq 0 ] + [[ "$output" == *"No token-usage records found"* ]] +} + +@test "render_token_report: caps PR list at 10 without aborting on many PRs (SIGPIPE guard)" { + # >10 distinct PR contexts — the top-10 limit must not abort under pipefail. + tmp="$(mktemp -d)" + for i in $(seq 1 25); do + printf '{"ts":"2026-06-01T00:00:00Z","workflow":"pr-review","tier":"single","model":"claude-haiku-4-5-20251001","input_tokens":%d,"cache_read_tokens":0,"output_tokens":10,"repo":"petry-projects/r","context":"https://github.com/petry-projects/r/pull/%d"}\n' "$((i * 100))" "$i" >> "$tmp/many.jsonl" + done + run render_token_report "$tmp" 7 1 1 2026-06-07 + rm -rf "$tmp" + [ "$status" -eq 0 ] + [[ "$output" == *"Most expensive PRs"* ]] + # Exactly 10 PR rows rendered (lines containing a /pull/ link in the table). + local pr_lines + pr_lines="$(printf '%s\n' "$output" | grep -c '/pull/')" + [ "$pr_lines" -eq 10 ] + # Highest-cost PR (#25, most input tokens) must be first in the list. + [[ "$output" == *"/pull/25 |"* ]] +} + +# --------------------------------------------------------------------------- +# collect_org_jsonl — error-handling (gh is stubbed; no network) +# --------------------------------------------------------------------------- + +@test "collect_org_jsonl: emits ERROR and returns non-zero when org repo listing fails" { + local tmpdir + tmpdir="$(mktemp -d)" + run bash -c " + gh() { return 1; } + export -f gh + source '${BATS_TEST_DIRNAME}/../scripts/token_report.sh' + export ORG=petry-projects LOOKBACK_DAYS=7 + collect_org_jsonl '${tmpdir}' 2>&1 + " + rm -rf "$tmpdir" + [ "$status" -ne 0 ] + [[ "$output" == *"ERROR"* ]] +} + +@test "collect_org_jsonl: emits WARN and continues when per-repo artifact listing fails" { + local tmpdir + tmpdir="$(mktemp -d)" + run bash -c " + gh() { + case \"\$2\" in + orgs/*/repos*) echo 'petry-projects/test-repo'; return 0;; + repos/*/actions/artifacts*) return 1;; + *) return 1;; + esac + } + export -f gh + source '${BATS_TEST_DIRNAME}/../scripts/token_report.sh' + export ORG=petry-projects LOOKBACK_DAYS=7 + collect_org_jsonl '${tmpdir}' 2>&1 + " + rm -rf "$tmpdir" + [ "$status" -eq 0 ] + [[ "$output" == *"WARN"* ]] +} + +@test "collect_org_jsonl: removes malformed dest JSONL when jq conversion fails" { + local tmpdir jsonl_dir + tmpdir="$(mktemp -d)" + jsonl_dir="$(mktemp -d)" + run bash -c " + set -euo pipefail + source '${BATS_TEST_DIRNAME}/../scripts/token_report.sh' + # Simulate the inner conversion loop for a binary-corrupted source file. + f='$tmpdir/bad.jsonl' + printf '\x00\x01\x02\x03' > \"\$f\" + dest='$jsonl_dir/999-bad.jsonl' + if jq -c --arg repo 'r' 'select(type==\"object\") | . + {repo: \$repo}' \ + \"\$f\" > \"\$dest\" 2>/dev/null; then + true + else + rm -f \"\$dest\" 2>/dev/null || true + fi + # No lingering dest file should remain. + [ -z \"\$(ls '$jsonl_dir')\" ] && echo 'clean' || echo 'dirty' + " + rm -rf "$tmpdir" "$jsonl_dir" + [ "$status" -eq 0 ] + [ "$output" = "clean" ] +} + +@test "collect_org_jsonl: emits WARN and continues when artifact zip download fails" { + local tmpdir + tmpdir="$(mktemp -d)" + run bash -c " + gh() { + case \"\$2\" in + orgs/*/repos*) echo 'petry-projects/test-repo'; return 0;; + repos/*/actions/artifacts/*/zip) return 1;; + repos/*/actions/artifacts*) + printf '{\"artifacts\":[{\"name\":\"token-usage-x\",\"id\":999,\"expired\":false,\"created_at\":\"2099-01-01T00:00:00Z\"}]}' + return 0;; + *) return 0;; + esac + } + export -f gh + source '${BATS_TEST_DIRNAME}/../scripts/token_report.sh' + export ORG=petry-projects LOOKBACK_DAYS=7 + collect_org_jsonl '${tmpdir}' 2>&1 + " + rm -rf "$tmpdir" + [ "$status" -eq 0 ] + [[ "$output" == *"WARN"* ]] + [[ "$output" == *"download failed"* ]] +}