From 2dea8c611a20f409176b2d814f990c1e64558381 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sat, 4 Jul 2026 17:34:16 -0700 Subject: [PATCH 1/5] fix(observability): report actual orb ai token usage on the dashboards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Codex usage dashboard's durable SQL panels filtered on model LIKE '%codex%' and only surfaced estimated_neurons, even though migration 0109 added real provider/effort/input_tokens/output_tokens/ total_tokens/cost_usd columns to ai_usage_events months ago: - Rewrite every durable panel in codex-usage.json to filter on provider = 'codex' instead of pattern-matching the model string. - Replace the "Estimated neuron usage by day" panel with a real "Durable token usage by day" panel (input/output tokens), add a new "Durable cost by day" panel, and add effort/token/cost columns to the recent-events table (keeping estimated_neurons alongside for reference, not as the only metric). - Add grafana/dashboards/orb-ai-usage.json: a provider-neutral dashboard over ai_usage_events showing every configured provider (Codex, Claude Code, OpenAI-compatible, Ollama, Anthropic BYOK) broken down by provider/model/effort/repo/PR, plus an "events missing real usage" counter to track which features still haven't threaded real usage through (see #3270). - Clarify claude-usage.json's description: it's Claude Code's own OTEL/session telemetry, not the durable ai_usage_events DB attribution — cross-references the new orb-ai-usage dashboard for that. All three dashboard JSON files are auto-provisioned from grafana/dashboards/ (no config wiring needed for the new file) and validated by scripts/validate-observability-configs.mjs. --- grafana/dashboards/claude-usage.json | 4 +- grafana/dashboards/codex-usage.json | 46 +++++--- grafana/dashboards/orb-ai-usage.json | 169 +++++++++++++++++++++++++++ 3 files changed, 204 insertions(+), 15 deletions(-) create mode 100644 grafana/dashboards/orb-ai-usage.json diff --git a/grafana/dashboards/claude-usage.json b/grafana/dashboards/claude-usage.json index 5cab598cf3..b2933c4bcc 100644 --- a/grafana/dashboards/claude-usage.json +++ b/grafana/dashboards/claude-usage.json @@ -4,10 +4,10 @@ "tags": ["gittensory", "claude", "ai"], "timezone": "browser", "schemaVersion": 39, - "version": 3, + "version": 4, "refresh": "30s", "time": { "from": "now-7d", "to": "now" }, - "description": "Claude Code review-CLI usage via OpenTelemetry. Each headless review is a session-scoped cumulative counter, so totals use last_over_time (sum of each session's final value) — increase()/rate() would read flat because a session exports its total in one shot.", + "description": "Claude Code CLI/session telemetry via OpenTelemetry — NOT the durable ORB ai_usage_events DB attribution (see the 'Gittensory - ORB AI usage' dashboard for that, filterable by provider/model/repo/PR). Each headless review is a session-scoped cumulative counter, so totals use last_over_time (sum of each session's final value) — increase()/rate() would read flat because a session exports its total in one shot.", "templating": { "list": [ { diff --git a/grafana/dashboards/codex-usage.json b/grafana/dashboards/codex-usage.json index c314967c8e..8ab40226c7 100644 --- a/grafana/dashboards/codex-usage.json +++ b/grafana/dashboards/codex-usage.json @@ -4,10 +4,10 @@ "tags": ["gittensory", "codex", "ai"], "timezone": "browser", "schemaVersion": 39, - "version": 1, + "version": 2, "refresh": "30s", "time": { "from": "now-7d", "to": "now" }, - "description": "Codex review observability for the self-hosted stack. Prometheus panels are subscription-CLI attempt counters for the selected range; durable review records come from ai_usage_events and must be read with their status. USD cost appears only when the Codex CLI emits a cost field.", + "description": "Codex review observability for the self-hosted stack. Prometheus panels are subscription-CLI attempt counters for the selected range; durable review records come from ai_usage_events, filtered by provider='codex' (migration 0109), and must be read with their status. Token/cost columns are populated whenever the Codex CLI reports real usage; estimated_neurons is a heuristic budget proxy, not billing-grade accounting.", "panels": [ { "id": 1, @@ -61,8 +61,8 @@ { "refId": "A", "queryType": "table", - "queryText": "SELECT count(*) AS reviews FROM ai_usage_events WHERE feature = 'ai_review_pr' AND status = 'ok' AND (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%')", - "rawQueryText": "SELECT count(*) AS reviews FROM ai_usage_events WHERE feature = 'ai_review_pr' AND status = 'ok' AND (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%')" + "queryText": "SELECT count(*) AS reviews FROM ai_usage_events WHERE feature = 'ai_review_pr' AND status = 'ok' AND provider = 'codex'", + "rawQueryText": "SELECT count(*) AS reviews FROM ai_usage_events WHERE feature = 'ai_review_pr' AND status = 'ok' AND provider = 'codex'" } ] }, @@ -118,18 +118,19 @@ { "id": 10, "type": "timeseries", - "title": "Estimated neuron usage by day", + "title": "Durable token usage by day", + "description": "Real input/output tokens from ai_usage_events (migration 0109), populated when the Codex CLI reports usage. Replaces the old estimated-neuron-only proxy for this panel.", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, - "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "fillOpacity": 70, "lineWidth": 1 } } }, + "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "fillOpacity": 70, "lineWidth": 1, "stacking": { "mode": "normal" } } } }, "options": { "legend": { "showLegend": true, "placement": "bottom" }, "tooltip": { "mode": "multi" } }, "targets": [ { "refId": "A", "queryType": "time series", "timeColumns": ["time"], - "queryText": "SELECT date(created_at) AS time, sum(estimated_neurons) AS estimated_neurons FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%') GROUP BY date(created_at) ORDER BY time", - "rawQueryText": "SELECT date(created_at) AS time, sum(estimated_neurons) AS estimated_neurons FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%') GROUP BY date(created_at) ORDER BY time" + "queryText": "SELECT date(created_at) AS time, sum(input_tokens) AS input_tokens, sum(output_tokens) AS output_tokens FROM ai_usage_events WHERE feature = 'ai_review_pr' AND provider = 'codex' GROUP BY date(created_at) ORDER BY time", + "rawQueryText": "SELECT date(created_at) AS time, sum(input_tokens) AS input_tokens, sum(output_tokens) AS output_tokens FROM ai_usage_events WHERE feature = 'ai_review_pr' AND provider = 'codex' GROUP BY date(created_at) ORDER BY time" } ] }, @@ -145,8 +146,27 @@ { "refId": "A", "queryType": "table", - "queryText": "SELECT status, count(*) AS count FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%') GROUP BY status ORDER BY count DESC", - "rawQueryText": "SELECT status, count(*) AS count FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%') GROUP BY status ORDER BY count DESC" + "queryText": "SELECT status, count(*) AS count FROM ai_usage_events WHERE feature = 'ai_review_pr' AND provider = 'codex' GROUP BY status ORDER BY count DESC", + "rawQueryText": "SELECT status, count(*) AS count FROM ai_usage_events WHERE feature = 'ai_review_pr' AND provider = 'codex' GROUP BY status ORDER BY count DESC" + } + ] + }, + { + "id": 13, + "type": "timeseries", + "title": "Durable cost by day", + "description": "Real USD cost from ai_usage_events (migration 0109), populated when the Codex CLI reports a cost field.", + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 24 }, + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "fieldConfig": { "defaults": { "unit": "currencyUSD", "decimals": 4, "custom": { "drawStyle": "bars", "fillOpacity": 70, "lineWidth": 1 } } }, + "options": { "legend": { "showLegend": true, "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [ + { + "refId": "A", + "queryType": "time series", + "timeColumns": ["time"], + "queryText": "SELECT date(created_at) AS time, sum(cost_usd) AS cost_usd FROM ai_usage_events WHERE feature = 'ai_review_pr' AND provider = 'codex' GROUP BY date(created_at) ORDER BY time", + "rawQueryText": "SELECT date(created_at) AS time, sum(cost_usd) AS cost_usd FROM ai_usage_events WHERE feature = 'ai_review_pr' AND provider = 'codex' GROUP BY date(created_at) ORDER BY time" } ] }, @@ -154,7 +174,7 @@ "id": 12, "type": "table", "title": "Recent Codex-attributed review events", - "gridPos": { "h": 10, "w": 24, "x": 0, "y": 24 }, + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 32 }, "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, "options": { "showHeader": true, "cellHeight": "sm", "sortBy": [{ "displayName": "created_at", "desc": true }] }, "fieldConfig": { "defaults": { "custom": { "filterable": true } } }, @@ -162,8 +182,8 @@ { "refId": "A", "queryType": "table", - "queryText": "SELECT created_at, model, status, estimated_neurons, detail, json_extract(metadata_json, '$.repoFullName') AS repo, json_extract(metadata_json, '$.pullNumber') AS pr FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%') ORDER BY created_at DESC LIMIT 100", - "rawQueryText": "SELECT created_at, model, status, estimated_neurons, detail, json_extract(metadata_json, '$.repoFullName') AS repo, json_extract(metadata_json, '$.pullNumber') AS pr FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%') ORDER BY created_at DESC LIMIT 100" + "queryText": "SELECT created_at, model, effort, status, input_tokens, output_tokens, total_tokens, cost_usd, estimated_neurons, detail, json_extract(metadata_json, '$.repoFullName') AS repo, json_extract(metadata_json, '$.pullNumber') AS pr FROM ai_usage_events WHERE feature = 'ai_review_pr' AND provider = 'codex' ORDER BY created_at DESC LIMIT 100", + "rawQueryText": "SELECT created_at, model, effort, status, input_tokens, output_tokens, total_tokens, cost_usd, estimated_neurons, detail, json_extract(metadata_json, '$.repoFullName') AS repo, json_extract(metadata_json, '$.pullNumber') AS pr FROM ai_usage_events WHERE feature = 'ai_review_pr' AND provider = 'codex' ORDER BY created_at DESC LIMIT 100" } ] } diff --git a/grafana/dashboards/orb-ai-usage.json b/grafana/dashboards/orb-ai-usage.json new file mode 100644 index 0000000000..3007607e75 --- /dev/null +++ b/grafana/dashboards/orb-ai-usage.json @@ -0,0 +1,169 @@ +{ + "uid": "gittensory-orb-ai-usage", + "title": "Gittensory - ORB AI usage", + "tags": ["gittensory", "ai", "orb"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { "from": "now-7d", "to": "now" }, + "description": "Durable, provider-neutral AI usage from ai_usage_events (migration 0109) — every configured provider (Codex, Claude Code, OpenAI-compatible, Ollama, Anthropic BYOK), not just Codex. Populated whenever the configured provider reports real usage; falls back to estimated_neurons-only rows for features that don't yet thread real usage through. See 'Gittensory - Codex usage (self-host)' for a Codex-focused view and 'Gittensory — Claude usage (OTEL)' for Claude Code's own CLI/session telemetry (a separate, non-durable data source).", + "panels": [ + { + "id": 1, + "type": "row", + "title": "Summary (all providers)", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 } + }, + { + "id": 2, + "type": "stat", + "title": "Total AI events", + "description": "Every ai_usage_events row with a non-null provider (i.e. a real, attributed AI call), across all features.", + "gridPos": { "h": 5, "w": 6, "x": 0, "y": 1 }, + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "blue" } } }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none", "justifyMode": "center" }, + "targets": [ + { + "refId": "A", + "queryType": "table", + "queryText": "SELECT count(*) AS events FROM ai_usage_events WHERE provider IS NOT NULL", + "rawQueryText": "SELECT count(*) AS events FROM ai_usage_events WHERE provider IS NOT NULL" + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "Total tokens", + "gridPos": { "h": 5, "w": 6, "x": 6, "y": 1 }, + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "purple" } } }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none", "justifyMode": "center" }, + "targets": [ + { + "refId": "A", + "queryType": "table", + "queryText": "SELECT coalesce(sum(total_tokens), 0) AS total_tokens FROM ai_usage_events WHERE provider IS NOT NULL", + "rawQueryText": "SELECT coalesce(sum(total_tokens), 0) AS total_tokens FROM ai_usage_events WHERE provider IS NOT NULL" + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Total cost", + "gridPos": { "h": 5, "w": 6, "x": 12, "y": 1 }, + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "fieldConfig": { "defaults": { "unit": "currencyUSD", "decimals": 4, "color": { "mode": "fixed", "fixedColor": "green" } } }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none", "justifyMode": "center" }, + "targets": [ + { + "refId": "A", + "queryType": "table", + "queryText": "SELECT coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL", + "rawQueryText": "SELECT coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL" + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "Events missing real usage", + "description": "Rows with a feature/status implying a real AI call but no provider attribution yet (estimated_neurons-only). Should shrink over time as more features thread real usage through recordAiUsageEvent.", + "gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 }, + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "orange" } } }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none", "justifyMode": "center" }, + "targets": [ + { + "refId": "A", + "queryType": "table", + "queryText": "SELECT count(*) AS unattributed FROM ai_usage_events WHERE provider IS NULL AND feature != 'ai_key_change'", + "rawQueryText": "SELECT count(*) AS unattributed FROM ai_usage_events WHERE provider IS NULL AND feature != 'ai_key_change'" + } + ] + }, + { + "id": 6, + "type": "row", + "title": "By provider / model / effort", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 } + }, + { + "id": 7, + "type": "table", + "title": "Usage by provider", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 }, + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "options": { "showHeader": true, "cellHeight": "sm", "sortBy": [{ "displayName": "events", "desc": true }] }, + "fieldConfig": { "defaults": { "custom": { "filterable": true } } }, + "targets": [ + { + "refId": "A", + "queryType": "table", + "queryText": "SELECT provider, count(*) AS events, sum(input_tokens) AS input_tokens, sum(output_tokens) AS output_tokens, sum(cost_usd) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL GROUP BY provider ORDER BY events DESC", + "rawQueryText": "SELECT provider, count(*) AS events, sum(input_tokens) AS input_tokens, sum(output_tokens) AS output_tokens, sum(cost_usd) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL GROUP BY provider ORDER BY events DESC" + } + ] + }, + { + "id": 8, + "type": "table", + "title": "Usage by provider + model + effort", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 }, + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "options": { "showHeader": true, "cellHeight": "sm", "sortBy": [{ "displayName": "events", "desc": true }] }, + "fieldConfig": { "defaults": { "custom": { "filterable": true } } }, + "targets": [ + { + "refId": "A", + "queryType": "table", + "queryText": "SELECT provider, model, effort, count(*) AS events, sum(total_tokens) AS total_tokens, sum(cost_usd) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL GROUP BY provider, model, effort ORDER BY events DESC LIMIT 50", + "rawQueryText": "SELECT provider, model, effort, count(*) AS events, sum(total_tokens) AS total_tokens, sum(cost_usd) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL GROUP BY provider, model, effort ORDER BY events DESC LIMIT 50" + } + ] + }, + { + "id": 9, + "type": "row", + "title": "By repo / PR", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 } + }, + { + "id": 10, + "type": "table", + "title": "Top repos by AI usage", + "gridPos": { "h": 10, "w": 12, "x": 0, "y": 16 }, + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "options": { "showHeader": true, "cellHeight": "sm", "sortBy": [{ "displayName": "events", "desc": true }] }, + "fieldConfig": { "defaults": { "custom": { "filterable": true } } }, + "targets": [ + { + "refId": "A", + "queryType": "table", + "queryText": "SELECT json_extract(metadata_json, '$.repoFullName') AS repo, provider, count(*) AS events, sum(total_tokens) AS total_tokens, sum(cost_usd) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL AND json_extract(metadata_json, '$.repoFullName') IS NOT NULL GROUP BY repo, provider ORDER BY events DESC LIMIT 50", + "rawQueryText": "SELECT json_extract(metadata_json, '$.repoFullName') AS repo, provider, count(*) AS events, sum(total_tokens) AS total_tokens, sum(cost_usd) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL AND json_extract(metadata_json, '$.repoFullName') IS NOT NULL GROUP BY repo, provider ORDER BY events DESC LIMIT 50" + } + ] + }, + { + "id": 11, + "type": "table", + "title": "Recent AI events (all providers)", + "gridPos": { "h": 10, "w": 12, "x": 12, "y": 16 }, + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "options": { "showHeader": true, "cellHeight": "sm", "sortBy": [{ "displayName": "created_at", "desc": true }] }, + "fieldConfig": { "defaults": { "custom": { "filterable": true } } }, + "targets": [ + { + "refId": "A", + "queryType": "table", + "queryText": "SELECT created_at, feature, provider, model, effort, status, input_tokens, output_tokens, total_tokens, cost_usd, json_extract(metadata_json, '$.repoFullName') AS repo, json_extract(metadata_json, '$.pullNumber') AS pr FROM ai_usage_events WHERE provider IS NOT NULL ORDER BY created_at DESC LIMIT 100", + "rawQueryText": "SELECT created_at, feature, provider, model, effort, status, input_tokens, output_tokens, total_tokens, cost_usd, json_extract(metadata_json, '$.repoFullName') AS repo, json_extract(metadata_json, '$.pullNumber') AS pr FROM ai_usage_events WHERE provider IS NOT NULL ORDER BY created_at DESC LIMIT 100" + } + ] + } + ] +} From 45d9f6f6cc2328ab86908467b037c0cc2810f3dd Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sat, 4 Jul 2026 17:59:13 -0700 Subject: [PATCH 2/5] fix(observability): coalesce empty aggregates and tighten the missing-usage filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Wrap every sum(input_tokens|output_tokens|total_tokens|cost_usd) in coalesce(..., 0) across codex-usage.json's and orb-ai-usage.json's SQL panels, matching the existing coalesce(sum(estimated_neurons), 0) convention already used in sumAiEstimatedNeuronsSince — an empty result set now renders 0, not a blank/missing data point. - Tighten orb-ai-usage.json's "Events missing real usage" panel to filter status = 'ok' instead of a bare feature exclusion. Verified against the actual status literals ever written to ai_usage_events (quota_exceeded/disabled/unavailable never reach a DB write at all; ai_key_change audit rows use "set"/"replace"/"delete") that the old filter was overcounting: quota_exceeded and BYOK key-lifecycle rows never carry provider/usage data by design, not because a feature is missing usage tracking. Reworded the panel + dashboard descriptions to match what the tightened query actually measures. - Verified all four rewritten SQL queries against a real sqlite3 instance seeded with rows matching each status/provider combination. --- grafana/dashboards/codex-usage.json | 10 +++++----- grafana/dashboards/orb-ai-usage.json | 22 +++++++++++----------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/grafana/dashboards/codex-usage.json b/grafana/dashboards/codex-usage.json index 8ab40226c7..f1a4393f39 100644 --- a/grafana/dashboards/codex-usage.json +++ b/grafana/dashboards/codex-usage.json @@ -4,7 +4,7 @@ "tags": ["gittensory", "codex", "ai"], "timezone": "browser", "schemaVersion": 39, - "version": 2, + "version": 3, "refresh": "30s", "time": { "from": "now-7d", "to": "now" }, "description": "Codex review observability for the self-hosted stack. Prometheus panels are subscription-CLI attempt counters for the selected range; durable review records come from ai_usage_events, filtered by provider='codex' (migration 0109), and must be read with their status. Token/cost columns are populated whenever the Codex CLI reports real usage; estimated_neurons is a heuristic budget proxy, not billing-grade accounting.", @@ -129,8 +129,8 @@ "refId": "A", "queryType": "time series", "timeColumns": ["time"], - "queryText": "SELECT date(created_at) AS time, sum(input_tokens) AS input_tokens, sum(output_tokens) AS output_tokens FROM ai_usage_events WHERE feature = 'ai_review_pr' AND provider = 'codex' GROUP BY date(created_at) ORDER BY time", - "rawQueryText": "SELECT date(created_at) AS time, sum(input_tokens) AS input_tokens, sum(output_tokens) AS output_tokens FROM ai_usage_events WHERE feature = 'ai_review_pr' AND provider = 'codex' GROUP BY date(created_at) ORDER BY time" + "queryText": "SELECT date(created_at) AS time, coalesce(sum(input_tokens), 0) AS input_tokens, coalesce(sum(output_tokens), 0) AS output_tokens FROM ai_usage_events WHERE feature = 'ai_review_pr' AND provider = 'codex' GROUP BY date(created_at) ORDER BY time", + "rawQueryText": "SELECT date(created_at) AS time, coalesce(sum(input_tokens), 0) AS input_tokens, coalesce(sum(output_tokens), 0) AS output_tokens FROM ai_usage_events WHERE feature = 'ai_review_pr' AND provider = 'codex' GROUP BY date(created_at) ORDER BY time" } ] }, @@ -165,8 +165,8 @@ "refId": "A", "queryType": "time series", "timeColumns": ["time"], - "queryText": "SELECT date(created_at) AS time, sum(cost_usd) AS cost_usd FROM ai_usage_events WHERE feature = 'ai_review_pr' AND provider = 'codex' GROUP BY date(created_at) ORDER BY time", - "rawQueryText": "SELECT date(created_at) AS time, sum(cost_usd) AS cost_usd FROM ai_usage_events WHERE feature = 'ai_review_pr' AND provider = 'codex' GROUP BY date(created_at) ORDER BY time" + "queryText": "SELECT date(created_at) AS time, coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE feature = 'ai_review_pr' AND provider = 'codex' GROUP BY date(created_at) ORDER BY time", + "rawQueryText": "SELECT date(created_at) AS time, coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE feature = 'ai_review_pr' AND provider = 'codex' GROUP BY date(created_at) ORDER BY time" } ] }, diff --git a/grafana/dashboards/orb-ai-usage.json b/grafana/dashboards/orb-ai-usage.json index 3007607e75..00fcd6d3e6 100644 --- a/grafana/dashboards/orb-ai-usage.json +++ b/grafana/dashboards/orb-ai-usage.json @@ -4,10 +4,10 @@ "tags": ["gittensory", "ai", "orb"], "timezone": "browser", "schemaVersion": 39, - "version": 1, + "version": 2, "refresh": "30s", "time": { "from": "now-7d", "to": "now" }, - "description": "Durable, provider-neutral AI usage from ai_usage_events (migration 0109) — every configured provider (Codex, Claude Code, OpenAI-compatible, Ollama, Anthropic BYOK), not just Codex. Populated whenever the configured provider reports real usage; falls back to estimated_neurons-only rows for features that don't yet thread real usage through. See 'Gittensory - Codex usage (self-host)' for a Codex-focused view and 'Gittensory — Claude usage (OTEL)' for Claude Code's own CLI/session telemetry (a separate, non-durable data source).", + "description": "Durable, provider-neutral AI usage from ai_usage_events (migration 0109) — every configured provider (Codex, Claude Code, OpenAI-compatible, Ollama, Anthropic BYOK), not just Codex. Every AI feature (ai_review_pr, agent_private_summary, agent_public_summary, pr_intelligence_comment, ai_slop_pr, issue_plan) threads real usage through recordAiUsageEvent on a completed call; a row still lands as estimated_neurons-only when the call never happened (quota_exceeded/disabled/unavailable), is a BYOK call (not yet usage-tracked), or the provider omitted a parseable usage field — see the 'Events missing real usage' panel. See 'Gittensory - Codex usage (self-host)' for a Codex-focused view and 'Gittensory — Claude usage (OTEL)' for Claude Code's own CLI/session telemetry (a separate, non-durable data source).", "panels": [ { "id": 1, @@ -71,7 +71,7 @@ "id": 5, "type": "stat", "title": "Events missing real usage", - "description": "Rows with a feature/status implying a real AI call but no provider attribution yet (estimated_neurons-only). Should shrink over time as more features thread real usage through recordAiUsageEvent.", + "description": "Completed AI calls (status='ok', a response was actually returned) with no provider attribution. Excludes quota_exceeded/disabled/unavailable rows (no call was attempted, so no usage was ever possible) and ai_key_change audit rows (not an AI call at all) — those never carry provider/usage data by design, not by gap. A nonzero count here means either a BYOK call (callAiProvider does not yet capture usage) or a provider response that omitted a parseable usage field.", "gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 }, "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "orange" } } }, @@ -80,8 +80,8 @@ { "refId": "A", "queryType": "table", - "queryText": "SELECT count(*) AS unattributed FROM ai_usage_events WHERE provider IS NULL AND feature != 'ai_key_change'", - "rawQueryText": "SELECT count(*) AS unattributed FROM ai_usage_events WHERE provider IS NULL AND feature != 'ai_key_change'" + "queryText": "SELECT count(*) AS unattributed FROM ai_usage_events WHERE provider IS NULL AND status = 'ok'", + "rawQueryText": "SELECT count(*) AS unattributed FROM ai_usage_events WHERE provider IS NULL AND status = 'ok'" } ] }, @@ -103,8 +103,8 @@ { "refId": "A", "queryType": "table", - "queryText": "SELECT provider, count(*) AS events, sum(input_tokens) AS input_tokens, sum(output_tokens) AS output_tokens, sum(cost_usd) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL GROUP BY provider ORDER BY events DESC", - "rawQueryText": "SELECT provider, count(*) AS events, sum(input_tokens) AS input_tokens, sum(output_tokens) AS output_tokens, sum(cost_usd) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL GROUP BY provider ORDER BY events DESC" + "queryText": "SELECT provider, count(*) AS events, coalesce(sum(input_tokens), 0) AS input_tokens, coalesce(sum(output_tokens), 0) AS output_tokens, coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL GROUP BY provider ORDER BY events DESC", + "rawQueryText": "SELECT provider, count(*) AS events, coalesce(sum(input_tokens), 0) AS input_tokens, coalesce(sum(output_tokens), 0) AS output_tokens, coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL GROUP BY provider ORDER BY events DESC" } ] }, @@ -120,8 +120,8 @@ { "refId": "A", "queryType": "table", - "queryText": "SELECT provider, model, effort, count(*) AS events, sum(total_tokens) AS total_tokens, sum(cost_usd) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL GROUP BY provider, model, effort ORDER BY events DESC LIMIT 50", - "rawQueryText": "SELECT provider, model, effort, count(*) AS events, sum(total_tokens) AS total_tokens, sum(cost_usd) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL GROUP BY provider, model, effort ORDER BY events DESC LIMIT 50" + "queryText": "SELECT provider, model, effort, count(*) AS events, coalesce(sum(total_tokens), 0) AS total_tokens, coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL GROUP BY provider, model, effort ORDER BY events DESC LIMIT 50", + "rawQueryText": "SELECT provider, model, effort, count(*) AS events, coalesce(sum(total_tokens), 0) AS total_tokens, coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL GROUP BY provider, model, effort ORDER BY events DESC LIMIT 50" } ] }, @@ -143,8 +143,8 @@ { "refId": "A", "queryType": "table", - "queryText": "SELECT json_extract(metadata_json, '$.repoFullName') AS repo, provider, count(*) AS events, sum(total_tokens) AS total_tokens, sum(cost_usd) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL AND json_extract(metadata_json, '$.repoFullName') IS NOT NULL GROUP BY repo, provider ORDER BY events DESC LIMIT 50", - "rawQueryText": "SELECT json_extract(metadata_json, '$.repoFullName') AS repo, provider, count(*) AS events, sum(total_tokens) AS total_tokens, sum(cost_usd) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL AND json_extract(metadata_json, '$.repoFullName') IS NOT NULL GROUP BY repo, provider ORDER BY events DESC LIMIT 50" + "queryText": "SELECT json_extract(metadata_json, '$.repoFullName') AS repo, provider, count(*) AS events, coalesce(sum(total_tokens), 0) AS total_tokens, coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL AND json_extract(metadata_json, '$.repoFullName') IS NOT NULL GROUP BY repo, provider ORDER BY events DESC LIMIT 50", + "rawQueryText": "SELECT json_extract(metadata_json, '$.repoFullName') AS repo, provider, count(*) AS events, coalesce(sum(total_tokens), 0) AS total_tokens, coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL AND json_extract(metadata_json, '$.repoFullName') IS NOT NULL GROUP BY repo, provider ORDER BY events DESC LIMIT 50" } ] }, From 842e7284f1599245a458035fb01ee7782e3a92f1 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sat, 4 Jul 2026 19:45:50 -0700 Subject: [PATCH 3/5] fix(observability): exclude ai_key_change rows from the missing-usage counter explicitly The "Events missing real usage" panel's query relied on an implicit invariant in a different file (recordAiKeyChange's status param is typed "set"|"replace"|"delete", never "ok") to keep ai_key_change audit rows out of the count, even though the panel's own description already promised that exclusion explicitly. Add the feature <> 'ai_key_change' filter directly to the query so the panel is self-contained and doesn't silently over-count if that invariant ever changes elsewhere. Verified behavior-neutral against a seeded sqlite3 table: identical result (1) for realistic data with and without the new filter, and the new filter correctly excludes a row that violates the current type invariant (an ai_key_change row with status='ok'), which the old query would have wrongly counted. --- grafana/dashboards/orb-ai-usage.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/grafana/dashboards/orb-ai-usage.json b/grafana/dashboards/orb-ai-usage.json index 00fcd6d3e6..710216081f 100644 --- a/grafana/dashboards/orb-ai-usage.json +++ b/grafana/dashboards/orb-ai-usage.json @@ -4,7 +4,7 @@ "tags": ["gittensory", "ai", "orb"], "timezone": "browser", "schemaVersion": 39, - "version": 2, + "version": 3, "refresh": "30s", "time": { "from": "now-7d", "to": "now" }, "description": "Durable, provider-neutral AI usage from ai_usage_events (migration 0109) — every configured provider (Codex, Claude Code, OpenAI-compatible, Ollama, Anthropic BYOK), not just Codex. Every AI feature (ai_review_pr, agent_private_summary, agent_public_summary, pr_intelligence_comment, ai_slop_pr, issue_plan) threads real usage through recordAiUsageEvent on a completed call; a row still lands as estimated_neurons-only when the call never happened (quota_exceeded/disabled/unavailable), is a BYOK call (not yet usage-tracked), or the provider omitted a parseable usage field — see the 'Events missing real usage' panel. See 'Gittensory - Codex usage (self-host)' for a Codex-focused view and 'Gittensory — Claude usage (OTEL)' for Claude Code's own CLI/session telemetry (a separate, non-durable data source).", @@ -71,7 +71,7 @@ "id": 5, "type": "stat", "title": "Events missing real usage", - "description": "Completed AI calls (status='ok', a response was actually returned) with no provider attribution. Excludes quota_exceeded/disabled/unavailable rows (no call was attempted, so no usage was ever possible) and ai_key_change audit rows (not an AI call at all) — those never carry provider/usage data by design, not by gap. A nonzero count here means either a BYOK call (callAiProvider does not yet capture usage) or a provider response that omitted a parseable usage field.", + "description": "Completed AI calls (status='ok', a response was actually returned) with no provider attribution. Excludes quota_exceeded/disabled/unavailable rows (no call was attempted, so no usage was ever possible) and ai_key_change audit rows (not an AI call at all — recordAiKeyChange's status is always \"set\"/\"replace\"/\"delete\", never \"ok\", but the feature is excluded explicitly here too so this panel doesn't rely on that invariant holding in another file) — those never carry provider/usage data by design, not by gap. A nonzero count here means either a BYOK call (callAiProvider does not yet capture usage) or a provider response that omitted a parseable usage field.", "gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 }, "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "orange" } } }, @@ -80,8 +80,8 @@ { "refId": "A", "queryType": "table", - "queryText": "SELECT count(*) AS unattributed FROM ai_usage_events WHERE provider IS NULL AND status = 'ok'", - "rawQueryText": "SELECT count(*) AS unattributed FROM ai_usage_events WHERE provider IS NULL AND status = 'ok'" + "queryText": "SELECT count(*) AS unattributed FROM ai_usage_events WHERE provider IS NULL AND status = 'ok' AND feature <> 'ai_key_change'", + "rawQueryText": "SELECT count(*) AS unattributed FROM ai_usage_events WHERE provider IS NULL AND status = 'ok' AND feature <> 'ai_key_change'" } ] }, From c743eed2c56409b81fd495ce6929d871863a25a2 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sat, 4 Jul 2026 21:18:44 -0700 Subject: [PATCH 4/5] fix(observability): catch provider-attributed rows with zero real usage too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "Events missing real usage" panel only checked provider IS NULL, missing a second reachable gap: coerceAiUsage (ai-review.ts) extracts provider independently of input_tokens/output_tokens/total_tokens/ cost_usd, so a raw provider response like {"usage": {"provider": "codex"}} with no parseable token counts yields a stored row with provider='codex' but all four numeric columns at their default. The AI reviewer's literal suggested fix (checking those four columns IS NULL) is dead code against this schema — migration 0109 declares them NOT NULL DEFAULT 0, and recordAiUsageEvent's finiteNumber() coercion means they can never be SQL NULL, only 0. Verified this with an independent two-agent analysis (both traced the same schema/ coercion/extraction code) plus two adversarial agents that built a scratch sqlite3 table from the actual migration SQL and ran the fixed predicate against 16 seeded rows total across both verification passes, confirming: the original provider-IS-NULL case still counts, the new provider-set-all-zero case now counts, a provider-set-with- real-nonzero-usage case does NOT count (no false positive), and ai_key_ change/quota_exceeded rows stay excluded. Also empirically confirmed a literal NULL insert into these columns is rejected by the schema itself. Query is now: status = 'ok' AND feature <> 'ai_key_change' AND (provider IS NULL OR (input_tokens = 0 AND output_tokens = 0 AND total_tokens = 0 AND cost_usd = 0)). Description updated to explain why a zero-value check, not an IS NULL check, is the correct "no data" signal for this schema. --- grafana/dashboards/orb-ai-usage.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/grafana/dashboards/orb-ai-usage.json b/grafana/dashboards/orb-ai-usage.json index 710216081f..823163314b 100644 --- a/grafana/dashboards/orb-ai-usage.json +++ b/grafana/dashboards/orb-ai-usage.json @@ -4,7 +4,7 @@ "tags": ["gittensory", "ai", "orb"], "timezone": "browser", "schemaVersion": 39, - "version": 3, + "version": 4, "refresh": "30s", "time": { "from": "now-7d", "to": "now" }, "description": "Durable, provider-neutral AI usage from ai_usage_events (migration 0109) — every configured provider (Codex, Claude Code, OpenAI-compatible, Ollama, Anthropic BYOK), not just Codex. Every AI feature (ai_review_pr, agent_private_summary, agent_public_summary, pr_intelligence_comment, ai_slop_pr, issue_plan) threads real usage through recordAiUsageEvent on a completed call; a row still lands as estimated_neurons-only when the call never happened (quota_exceeded/disabled/unavailable), is a BYOK call (not yet usage-tracked), or the provider omitted a parseable usage field — see the 'Events missing real usage' panel. See 'Gittensory - Codex usage (self-host)' for a Codex-focused view and 'Gittensory — Claude usage (OTEL)' for Claude Code's own CLI/session telemetry (a separate, non-durable data source).", @@ -71,7 +71,7 @@ "id": 5, "type": "stat", "title": "Events missing real usage", - "description": "Completed AI calls (status='ok', a response was actually returned) with no provider attribution. Excludes quota_exceeded/disabled/unavailable rows (no call was attempted, so no usage was ever possible) and ai_key_change audit rows (not an AI call at all — recordAiKeyChange's status is always \"set\"/\"replace\"/\"delete\", never \"ok\", but the feature is excluded explicitly here too so this panel doesn't rely on that invariant holding in another file) — those never carry provider/usage data by design, not by gap. A nonzero count here means either a BYOK call (callAiProvider does not yet capture usage) or a provider response that omitted a parseable usage field.", + "description": "Completed AI calls (status='ok', a response was actually returned) with no real usage captured — either no provider attribution at all, or a provider attributed with input_tokens/output_tokens/total_tokens/cost_usd all still at their zero default. (input_tokens/output_tokens/total_tokens/cost_usd are NOT NULL DEFAULT 0 per migration 0109 — recordAiUsageEvent's finiteNumber() coercion means they can never be SQL NULL, so a zero-value check, not an IS NULL check, is the real \"no data\" signal here; coerceAiUsage extracts provider independently of the numeric fields, so a provider name with no parseable token counts is a real, reachable row shape.) Excludes quota_exceeded/disabled/unavailable rows (no call was attempted, so no usage was ever possible) and ai_key_change audit rows (not an AI call at all — recordAiKeyChange's status is always \"set\"/\"replace\"/\"delete\", never \"ok\", but the feature is excluded explicitly here too so this panel doesn't rely on that invariant holding in another file). A nonzero count here means either a BYOK call (callAiProvider does not yet capture usage) or a provider response that omitted a parseable usage field.", "gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 }, "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "orange" } } }, @@ -80,8 +80,8 @@ { "refId": "A", "queryType": "table", - "queryText": "SELECT count(*) AS unattributed FROM ai_usage_events WHERE provider IS NULL AND status = 'ok' AND feature <> 'ai_key_change'", - "rawQueryText": "SELECT count(*) AS unattributed FROM ai_usage_events WHERE provider IS NULL AND status = 'ok' AND feature <> 'ai_key_change'" + "queryText": "SELECT count(*) AS unattributed FROM ai_usage_events WHERE status = 'ok' AND feature <> 'ai_key_change' AND (provider IS NULL OR (input_tokens = 0 AND output_tokens = 0 AND total_tokens = 0 AND cost_usd = 0))", + "rawQueryText": "SELECT count(*) AS unattributed FROM ai_usage_events WHERE status = 'ok' AND feature <> 'ai_key_change' AND (provider IS NULL OR (input_tokens = 0 AND output_tokens = 0 AND total_tokens = 0 AND cost_usd = 0))" } ] }, From 910267b83e1124a5ee19dc954cf33deb67e2896c Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sat, 4 Jul 2026 21:39:36 -0700 Subject: [PATCH 5/5] fix(observability): recover quota-blocked codex reviews in the status/recent-events panels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The status-breakdown and recent-events panels filtered strictly on provider = 'codex', but a quota-blocked ai_review_pr call never gets provider attribution — record() short-circuits before any provider ever runs, so `actualUsage` (and therefore `provider`) is never passed. That call's `model` field, however, is computed unconditionally via reviewerModelLabel(env, input) inside record() regardless of status, so it always reflects the configured/intended reviewer (e.g. "codex", "codex:o4-mini", "codex+claude-code") even when the call never executed. Filtering by provider alone therefore made every quota-blocked codex review invisible to these panels — a real regression, not the "disabled"/"unavailable" rows the review comment also named (those two statuses never reach a DB write at all, confirmed by tracing every return path in runGittensoryAiReview, so no filter change could have dropped them). Fixed the "Successful review records" (for consistency with the panel below), "Review record status", and "Recent Codex-attributed review events" panels to match 'codex' as a reviewer-name token in EITHER provider OR the "+"-joined model label — reusing the exact matching pattern already established (and covered by test/unit/selfhost-grafana-reporting.test.ts) for the pre-existing model-string-based filter this PR replaced. Left the token/cost sum panels (10, 13) on the strict provider filter, since an unattributed row contributes 0 to a sum regardless of which filter is used, and documented why in both the dashboard and panel descriptions to preempt an "inconsistent filters" flag on a future pass. Verified via two independent agents tracing the exact code paths (early-return statements, record()'s model/provider computation) plus two adversarial agents running the corrected predicate against 9-10 seeded sqlite3 rows each, covering: the flagged quota_exceeded/codex case, cross-provider exclusion, dual-reviewer-join and provider-model override label formats, substring false-positive safety, cross-feature scoping, and no double-counting. Final end-to-end check against all three fixed panels' exact query text confirms the expected row/count results. --- grafana/dashboards/codex-usage.json | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/grafana/dashboards/codex-usage.json b/grafana/dashboards/codex-usage.json index f1a4393f39..72e67ac105 100644 --- a/grafana/dashboards/codex-usage.json +++ b/grafana/dashboards/codex-usage.json @@ -4,10 +4,10 @@ "tags": ["gittensory", "codex", "ai"], "timezone": "browser", "schemaVersion": 39, - "version": 3, + "version": 4, "refresh": "30s", "time": { "from": "now-7d", "to": "now" }, - "description": "Codex review observability for the self-hosted stack. Prometheus panels are subscription-CLI attempt counters for the selected range; durable review records come from ai_usage_events, filtered by provider='codex' (migration 0109), and must be read with their status. Token/cost columns are populated whenever the Codex CLI reports real usage; estimated_neurons is a heuristic budget proxy, not billing-grade accounting.", + "description": "Codex review observability for the self-hosted stack. Prometheus panels are subscription-CLI attempt counters for the selected range. Durable review records come from ai_usage_events (migration 0109) and must be read with their status. Two different filters are used deliberately: the token/cost panels (real usage sums) filter strictly on provider='codex', since an unattributed row always contributes 0 to a sum either way. The record-count/status/recent-events panels instead match 'codex' as a reviewer-name token in either provider OR the model label (\"+\"-joined reviewer names, e.g. 'codex', 'codex:o4-mini', 'codex+claude-code') — a quota-blocked call never gets provider attribution (no call ever ran) but IS attributable via its configured model label, so filtering by provider alone silently drops every quota_exceeded row for a codex-configured repo. estimated_neurons is a heuristic budget proxy, not billing-grade accounting.", "panels": [ { "id": 1, @@ -52,7 +52,7 @@ "id": 5, "type": "stat", "title": "Successful review records", - "description": "Durable Codex-attributed ai_review_pr records with status=ok.", + "description": "Durable ai_review_pr records with status=ok where codex was the reviewer — matched by provider attribution OR the configured model label (see dashboard description), so this stays consistent with the status-breakdown panel below rather than under-counting successes where the CLI's response reported no usage envelope at all.", "gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 }, "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "green" } } }, @@ -61,8 +61,8 @@ { "refId": "A", "queryType": "table", - "queryText": "SELECT count(*) AS reviews FROM ai_usage_events WHERE feature = 'ai_review_pr' AND status = 'ok' AND provider = 'codex'", - "rawQueryText": "SELECT count(*) AS reviews FROM ai_usage_events WHERE feature = 'ai_review_pr' AND status = 'ok' AND provider = 'codex'" + "queryText": "SELECT count(*) AS reviews FROM ai_usage_events WHERE feature = 'ai_review_pr' AND status = 'ok' AND (provider = 'codex' OR (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%'))", + "rawQueryText": "SELECT count(*) AS reviews FROM ai_usage_events WHERE feature = 'ai_review_pr' AND status = 'ok' AND (provider = 'codex' OR (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%'))" } ] }, @@ -119,7 +119,7 @@ "id": 10, "type": "timeseries", "title": "Durable token usage by day", - "description": "Real input/output tokens from ai_usage_events (migration 0109), populated when the Codex CLI reports usage. Replaces the old estimated-neuron-only proxy for this panel.", + "description": "Real input/output tokens from ai_usage_events (migration 0109), populated when the Codex CLI reports usage. Replaces the old estimated-neuron-only proxy for this panel. Filters strictly on provider='codex' (not the broader model-label match used by the status/recent-events panels below): a quota-blocked or otherwise unattributed row contributes 0 tokens regardless of which filter is used, so the stricter filter is equivalent here and simpler.", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "fillOpacity": 70, "lineWidth": 1, "stacking": { "mode": "normal" } } } }, @@ -138,6 +138,7 @@ "id": 11, "type": "piechart", "title": "Review record status", + "description": "Includes quota_exceeded rows for a codex-configured repo: those rows never get provider attribution (the call never ran), but the model label already reflects the configured reviewer at write time (record() computes it via reviewerModelLabel() regardless of status), so matching 'codex' as a reviewer-name token there recovers them. A plain provider='codex' filter would silently make every quota-blocked codex review invisible here.", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, "fieldConfig": { "defaults": { "unit": "short" } }, @@ -146,8 +147,8 @@ { "refId": "A", "queryType": "table", - "queryText": "SELECT status, count(*) AS count FROM ai_usage_events WHERE feature = 'ai_review_pr' AND provider = 'codex' GROUP BY status ORDER BY count DESC", - "rawQueryText": "SELECT status, count(*) AS count FROM ai_usage_events WHERE feature = 'ai_review_pr' AND provider = 'codex' GROUP BY status ORDER BY count DESC" + "queryText": "SELECT status, count(*) AS count FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (provider = 'codex' OR (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%')) GROUP BY status ORDER BY count DESC", + "rawQueryText": "SELECT status, count(*) AS count FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (provider = 'codex' OR (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%')) GROUP BY status ORDER BY count DESC" } ] }, @@ -155,7 +156,7 @@ "id": 13, "type": "timeseries", "title": "Durable cost by day", - "description": "Real USD cost from ai_usage_events (migration 0109), populated when the Codex CLI reports a cost field.", + "description": "Real USD cost from ai_usage_events (migration 0109), populated when the Codex CLI reports a cost field. Filters strictly on provider='codex' for the same reason as the token-usage panel above — an unattributed row contributes $0 regardless of filter.", "gridPos": { "h": 8, "w": 24, "x": 0, "y": 24 }, "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, "fieldConfig": { "defaults": { "unit": "currencyUSD", "decimals": 4, "custom": { "drawStyle": "bars", "fillOpacity": 70, "lineWidth": 1 } } }, @@ -174,6 +175,7 @@ "id": 12, "type": "table", "title": "Recent Codex-attributed review events", + "description": "Includes quota-blocked codex-configured rows (matched via the model label, not just provider attribution — see the status-breakdown panel's description) so an operator can see recent failures/blocks, not only rows with a completed usage response. The provider column itself is not displayed here; the model column already reflects the configured reviewer for every row, including blocked ones.", "gridPos": { "h": 10, "w": 24, "x": 0, "y": 32 }, "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, "options": { "showHeader": true, "cellHeight": "sm", "sortBy": [{ "displayName": "created_at", "desc": true }] }, @@ -182,8 +184,8 @@ { "refId": "A", "queryType": "table", - "queryText": "SELECT created_at, model, effort, status, input_tokens, output_tokens, total_tokens, cost_usd, estimated_neurons, detail, json_extract(metadata_json, '$.repoFullName') AS repo, json_extract(metadata_json, '$.pullNumber') AS pr FROM ai_usage_events WHERE feature = 'ai_review_pr' AND provider = 'codex' ORDER BY created_at DESC LIMIT 100", - "rawQueryText": "SELECT created_at, model, effort, status, input_tokens, output_tokens, total_tokens, cost_usd, estimated_neurons, detail, json_extract(metadata_json, '$.repoFullName') AS repo, json_extract(metadata_json, '$.pullNumber') AS pr FROM ai_usage_events WHERE feature = 'ai_review_pr' AND provider = 'codex' ORDER BY created_at DESC LIMIT 100" + "queryText": "SELECT created_at, model, effort, status, input_tokens, output_tokens, total_tokens, cost_usd, estimated_neurons, detail, json_extract(metadata_json, '$.repoFullName') AS repo, json_extract(metadata_json, '$.pullNumber') AS pr FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (provider = 'codex' OR (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%')) ORDER BY created_at DESC LIMIT 100", + "rawQueryText": "SELECT created_at, model, effort, status, input_tokens, output_tokens, total_tokens, cost_usd, estimated_neurons, detail, json_extract(metadata_json, '$.repoFullName') AS repo, json_extract(metadata_json, '$.pullNumber') AS pr FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (provider = 'codex' OR (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%')) ORDER BY created_at DESC LIMIT 100" } ] }