From 7fc4bc74d58757fb14c8abe48a700aa70e142ee6 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sat, 4 Jul 2026 17:09:09 -0700 Subject: [PATCH] refactor(ai): remove legacy workers-ai framing from live review features MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Workers AI has no live binding anywhere today (hosted or self-host — see CONVERGENCE_RUNBOOK.md); Codex/Claude Code via the self-host provider adapter is what actually serves every AI feature now. Update the four live AI call sites (ai-review.ts, ai-summaries.ts, ai-slop.ts, planner.ts) so their user/log-facing text and defaults reflect that: - Replace the "Workers AI binding is not configured" / "workers_ai_failed" strings with provider-neutral wording. - Drop the non-functional @cf/... default model in ai-summaries.ts in favor of the configured provider's own default. - Thread real per-call usage (provider/effort/tokens/cost) into recordAiUsageEvent for ai-summaries.ts, ai-slop.ts, and planner.ts via the shared coerceAiUsage helper (already exported from ai-review.ts), matching the tracking ai-review.ts's own review path already has. - Reword stale comments/docstrings that framed Workers AI as the live default reviewer instead of the legacy last-resort fallback. --- src/review/planner.ts | 49 ++++++++++++++++++++-------- src/services/ai-review.ts | 51 ++++++++++++++++++----------- src/services/ai-slop.ts | 59 +++++++++++++++++++++++----------- src/services/ai-summaries.ts | 59 +++++++++++++++++++++++++++------- test/unit/ai-slop.test.ts | 11 +++++++ test/unit/ai-summaries.test.ts | 14 ++++---- 6 files changed, 172 insertions(+), 71 deletions(-) diff --git a/src/review/planner.ts b/src/review/planner.ts index a8f0671a82..34bbba7008 100644 --- a/src/review/planner.ts +++ b/src/review/planner.ts @@ -6,10 +6,11 @@ // • flag-OFF (default) → isPlannerEnabled is false, the handler short-circuits BEFORE parsing, and the worker // is byte-identical to today (`@gittensory plan` falls through to the existing mention path → help card). // • flag-ON → only a MAINTAINER can trigger it; the model sees only the (already-public) issue title + body; -// shared AI budget accounting runs before Workers AI; the output is public-safe-sanitized before posting; -// any model/error degrades to a no-plan no-op. +// shared AI budget accounting runs before the configured reviewer (self-host Codex/Claude Code/etc, or the +// legacy Workers-AI pair); the output is public-safe-sanitized before posting; any model/error degrades to +// a no-plan no-op. -import { BEST_REVIEW_MODELS, clampNumber, coerceAiText, estimateNeurons, RELIABLE_FALLBACK_MODELS, utcDayStartIso } from "../services/ai-review"; +import { type AiReviewActualUsage, BEST_REVIEW_MODELS, clampNumber, coerceAiText, coerceAiUsage, estimateNeurons, RELIABLE_FALLBACK_MODELS, utcDayStartIso } from "../services/ai-review"; import { recordAiUsageEvent, sumAiEstimatedNeuronsSince } from "../db/repositories"; import { sanitizePublicComment } from "../github/commands"; import { AGENT_COMMAND_COMMENT_MARKER } from "../github/comments"; @@ -72,7 +73,18 @@ function plannerDailyBudget(env: Env): number { return clampNumber(env.AI_DAILY_NEURON_BUDGET && Number.isFinite(raw) ? raw : 10_000_000, 0, 10_000_000); } -async function recordPlannerUsage(env: Env, args: { actor?: string | null | undefined; repoFullName?: string | null | undefined; issueNumber?: number | null | undefined; status: string; estimatedNeurons: number; detail: string }): Promise { +async function recordPlannerUsage( + env: Env, + args: { + actor?: string | null | undefined; + repoFullName?: string | null | undefined; + issueNumber?: number | null | undefined; + status: string; + estimatedNeurons: number; + detail: string; + usage?: AiReviewActualUsage | undefined; + }, +): Promise { await recordAiUsageEvent(env, { feature: "issue_plan", actor: args.actor ?? null, @@ -80,16 +92,25 @@ async function recordPlannerUsage(env: Env, args: { actor?: string | null | unde model: [BEST_REVIEW_MODELS[0], RELIABLE_FALLBACK_MODELS[0]].join("+"), status: args.status, estimatedNeurons: args.estimatedNeurons, + provider: args.usage?.provider, + effort: args.usage?.effort, + inputTokens: args.usage?.inputTokens, + outputTokens: args.usage?.outputTokens, + totalTokens: args.usage?.totalTokens, + costUsd: args.usage?.costUsd, detail: args.detail, metadata: { repoFullName: args.repoFullName ?? null, issueNumber: args.issueNumber ?? null }, }); } -/** One Workers-AI text completion for the planner: primary model, one reliable fallback, a single retry each. +type PlannerModelResult = { text: string | null; usage?: AiReviewActualUsage | undefined }; + +/** One reviewer text completion for the planner (whichever provider `env.AI` resolves to — self-host Codex/ + * Claude Code/etc, or the legacy Workers-AI pair): primary model, one reliable fallback, a single retry each. * Fail-safe — any error or empty output returns null. Mirrors runWorkersOpinion's routing (AI Gateway when set). */ -async function runPlannerModel(env: Env, system: string, user: string): Promise { +async function runPlannerModel(env: Env, system: string, user: string): Promise { const ai = env.AI as unknown as { run?: (model: string, options: Record, extra?: unknown) => Promise } | undefined; - if (!ai || typeof ai.run !== "function") return null; + if (!ai || typeof ai.run !== "function") return { text: null }; const gatewayId = env.AI_GATEWAY_ID?.trim(); const extra = gatewayId ? { gateway: { id: gatewayId } } : undefined; for (const model of [BEST_REVIEW_MODELS[0], RELIABLE_FALLBACK_MODELS[0]]) { @@ -97,18 +118,18 @@ async function runPlannerModel(env: Env, system: string, user: string): Promise< try { const result = await ai.run(model, { max_tokens: PLANNER_MAX_TOKENS, temperature: 0.2, messages: [{ role: "system", content: system }, { role: "user", content: user }] }, extra); const text = coerceAiText(result).trim(); - if (text) return text; + if (text) return { text, usage: coerceAiUsage(result) }; } catch { /* retry, then fall through to the fallback model */ } } } - return null; + return { text: null }; } -/** Generate an implementation plan (markdown) from an issue's title + body via Workers AI. Returns null when AI - * is unavailable or returns nothing (the caller then posts no plan). The returned text is bounded; the caller - * still sanitizes it before posting. */ +/** Generate an implementation plan (markdown) from an issue's title + body via the configured reviewer. Returns + * null when AI is unavailable or returns nothing (the caller then posts no plan). The returned text is bounded; + * the caller still sanitizes it before posting. */ export async function generateIssuePlan( env: Env, issue: { title?: string | null | undefined; body?: string | null | undefined }, @@ -124,8 +145,8 @@ export async function generateIssuePlan( await recordPlannerUsage(env, { ...accounting, status: "quota_exceeded", estimatedNeurons: 0, detail: `estimated ${estimatedNeurons} neurons exceeds remaining ${remainingBudget}` }); return null; } - const plan = await runPlannerModel(env, PLANNER_SYSTEM_PROMPT, user); - await recordPlannerUsage(env, { ...accounting, status: plan ? "ok" : "no_output", estimatedNeurons: plan ? estimatedNeurons : 0, detail: plan ? "issue plan generated" : "no usable output" }); + const { text: plan, usage } = await runPlannerModel(env, PLANNER_SYSTEM_PROMPT, user); + await recordPlannerUsage(env, { ...accounting, status: plan ? "ok" : "no_output", estimatedNeurons: plan ? estimatedNeurons : 0, detail: plan ? "issue plan generated" : "no usable output", usage }); if (!plan) return null; return plan.slice(0, MAX_PLAN_CHARS); } diff --git a/src/services/ai-review.ts b/src/services/ai-review.ts index 05f46ca4da..b203cbcd7b 100644 --- a/src/services/ai-review.ts +++ b/src/services/ai-review.ts @@ -5,17 +5,20 @@ // // • Advisory notes — a concise maintainer-style write-up (assessment + suggestions + risks). When the // repo has BYOK configured, the maintainer's own frontier model (Anthropic/OpenAI) writes it; -// otherwise free Cloudflare Workers AI does. Advisory only — never blocks. -// • Consensus defect — a conservative gate signal. The free Workers-AI model PAIR each independently +// otherwise the configured free/default reviewer does (self-host: the AI_PROVIDER chain — Codex +// primary, Claude Code fallback, etc; unconfigured/hosted: the legacy Workers-AI pair below). +// Advisory only — never blocks. +// • Consensus defect — a conservative gate signal. The configured reviewer PAIR each independently // reviews the diff; a defect is reported ONLY when BOTH models flag a high-confidence critical defect // (bug / security / data-loss / build break). BYOK never changes this path, so it never changes who // can be blocked. The resulting finding is honored by the gate only in `block` mode AND only for // confirmed Gittensor contributors (the gate enforces that downstream). // // Every public string (notes + defect title/detail) is forced through `sanitizePublicComment`; anything -// that trips the public/private boundary is dropped, not published. Free Workers-AI calls are metered against -// the shared daily neuron budget; maintainer-paid BYOK calls have a separate repo/day cap. All calls -// are audited via `recordAiUsageEvent`. +// that trips the public/private boundary is dropped, not published. Free/default-reviewer calls are metered +// against the shared daily neuron budget; maintainer-paid BYOK calls have a separate repo/day cap. All calls +// are audited via `recordAiUsageEvent` (with real provider/token/cost usage when the configured provider +// reports it, per migration 0109 — see `coerceAiUsage`/`aggregateActualUsage`). import { countByokAiEventsForRepoSince, recordAiUsageEvent, @@ -33,15 +36,18 @@ import { isTestPath } from "../signals/test-evidence"; import type { CombineStrategy, OnMerge } from "../types"; /** - * The best free Workers-AI model pair for review accuracy — two different families for independence, - * both probe-verified in reviewbot to emit clean JSON. The consensus blocker always uses this pair. + * The legacy free Workers-AI model pair — used ONLY when neither a self-host `AI_REVIEW_PLAN` reviewer + * pair nor any configured provider (`AI_PROVIDER`) is present (see `reviewerModelLabel`). No `ai` binding + * exists in the deployed Worker today (Workers AI is fully retired — see CONVERGENCE_RUNBOOK.md), so this + * pair is inert in every current deployment; it stays only as the last-resort default these model ids + * were originally probe-verified against (both families independently clean-JSON in reviewbot). */ export const BEST_REVIEW_MODELS: readonly [string, string] = [ "@cf/openai/gpt-oss-120b", "@cf/nvidia/nemotron-3-120b-a12b", ]; -/** Reliable per-slot fallbacks (non-reasoning, clean JSON) so a slot never comes back empty. */ +/** Reliable per-slot fallbacks for the legacy pair above (non-reasoning, clean JSON) so a slot never comes back empty. */ export const RELIABLE_FALLBACK_MODELS: readonly [string, string] = [ "@cf/meta/llama-3.3-70b-instruct-fp8-fast", "@cf/mistralai/mistral-small-3.1-24b-instruct", @@ -475,7 +481,11 @@ function stringField(value: unknown): string | undefined { return typeof value === "string" && value.trim() ? value.trim() : undefined; } -function coerceAiUsage(result: unknown): AiReviewActualUsage | undefined { +/** Extract a provider's real usage (tokens/cost/effort) from an `env.AI.run()` result, when the configured + * provider reports one (self-host CLI/HTTP providers do; the legacy Workers-AI binding never did). Shared + * by every AI feature's `recordAiUsageEvent` call so migration 0109's columns get real data, not just the + * estimated-neurons proxy, whenever it's available. */ +export function coerceAiUsage(result: unknown): AiReviewActualUsage | undefined { if (!result || typeof result !== "object") return undefined; const usage = (result as Record).usage; if (!usage || typeof usage !== "object" || Array.isArray(usage)) return undefined; @@ -616,8 +626,9 @@ function buildUserPrompt(input: GittensoryAiReviewInput): string { : "Description: (none)", "", "Unified diff (truncated if large):", - // Widened 60k→120k so a large multi-file PR is actually reviewed in full (the 120B Workers-AI models have a - // 128k context window; pairing this with the higher output ceiling gives a thorough review). (#extensive-reviews) + // Widened 60k→120k so a large multi-file PR is actually reviewed in full (tuned against the legacy 120B + // Workers-AI pair's 128k context window; pairing this with the higher output ceiling gives a thorough + // review — self-host reviewers are configured with at least as much room). (#extensive-reviews) input.diff.slice(0, 120000), ]; // Convergence (grounding): append the FINISHED CI status + FULL file content when the caller supplied them @@ -708,7 +719,8 @@ function buildRepoInstructionsSystemAppend(repoInstructions: string | null | und : ""; } -/** One Workers-AI opinion with a per-slot reliable fallback and a 3× retry on the primary. */ +/** One reviewer opinion (whichever provider `env.AI` resolves to — self-host Codex/Claude Code/etc, or the + * legacy Workers-AI pair) with a per-slot reliable fallback and a 3× retry on the primary. */ async function runWorkersOpinion( env: Env, primary: string, @@ -1192,12 +1204,13 @@ export async function runGittensoryAiReview( if (!env.AI) return { status: "unavailable", - reason: "Workers AI binding is not configured.", + reason: "AI provider is not configured.", }; // Output ceiling for the review. The old 1024 cap forced a shallow "no blockers" scorecard across large diffs; - // a thorough finding-by-finding review needs real room. Default 4096, max 8192 (the free Workers-AI 120B models - // support it); an explicit env value still wins, clamped. (#extensive-reviews) + // a thorough finding-by-finding review needs real room. Default 4096, max 8192 (the configured reviewer — + // self-host Codex/Claude Code or the legacy free Workers-AI 120B pair — supports it); an explicit env value + // still wins, clamped. (#extensive-reviews) const maxTokens = clampNumber( Number(env.AI_MAX_OUTPUT_TOKENS) || 4096, 512, @@ -1223,11 +1236,11 @@ export async function runGittensoryAiReview( // unchanged (byte-identical). Computed from `promptInput` so it travels with the (possibly defanged) input. const system = buildSystemPrompt(promptInput); const repoInstructionsSystemAppend = buildRepoInstructionsSystemAppend(promptInput.repoInstructions); - // The daily neuron budget governs FREE Workers-AI spend only. BYOK advisory calls bill the maintainer's + // The daily neuron budget governs FREE/default-reviewer spend only. BYOK advisory calls bill the maintainer's // own provider account, so they are not counted here (and a BYOK advisory still runs when the free - // budget is exhausted). Free calls = the consensus pair in block mode (always Workers AI), plus the - // advisory leg only when it is NOT BYOK. - // Reviewers + combine strategy (#dual-ai-combiner). DEFAULT = the free Workers-AI pair (per-slot fallbacks) + // budget is exhausted). Free calls = the consensus pair in block mode (the configured self-host reviewers, + // or the legacy Workers-AI pair when none is configured), plus the advisory leg only when it is NOT BYOK. + // Reviewers + combine strategy (#dual-ai-combiner). DEFAULT = the legacy Workers-AI pair (per-slot fallbacks) // combined by `consensus` — byte-identical to today. The self-host boot plan (`env.AI_REVIEW_PLAN`) supplies // named providers (e.g. claude-code + codex) and a strategy; an explicit `input` field overrides it. `single` // (or a single configured reviewer) runs ONE opinion; consensus/synthesis run two. diff --git a/src/services/ai-slop.ts b/src/services/ai-slop.ts index f2f3035dca..d779ebed17 100644 --- a/src/services/ai-slop.ts +++ b/src/services/ai-slop.ts @@ -12,14 +12,16 @@ // • Fail-safe on every path: AI off / no binding / over-budget / unparseable / unsafe text → no finding. // • Opt-in: only runs when the repo set `gate.slop.aiAdvisory: true` on top of `gate.slop.mode != off`. // -// Free Cloudflare Workers AI only (bounded retry/fallback attempts, metered against the shared daily neuron -// budget). BYOK is a possible later enhancement; slop assessment does not need a frontier model. Every -// public string is forced through `toPublicSafe`; anything tripping the public/private boundary is dropped, -// not published. +// Free/default-reviewer only (bounded retry/fallback attempts, metered against the shared daily neuron +// budget) — the configured self-host provider (Codex/Claude Code/etc via `env.AI`), or the legacy Workers-AI +// pair when none is configured (Workers AI has no live binding anywhere today, see CONVERGENCE_RUNBOOK.md). +// BYOK is a possible later enhancement; slop assessment does not need a frontier model. Every public string +// is forced through `toPublicSafe`; anything tripping the public/private boundary is dropped, not published. import type { SignalFinding } from "../signals/engine"; import type { SlopBand } from "../signals/slop"; import { countByokAiEventsForRepoSince, recordAiUsageEvent, sumAiEstimatedNeuronsSince } from "../db/repositories"; import { + type AiReviewActualUsage, type AiReviewProviderKey, BEST_REVIEW_MODELS, DEFAULT_BYOK_DAILY_REPO_LIMIT, @@ -27,6 +29,7 @@ import { callAiProvider, clampNumber, coerceAiText, + coerceAiUsage, estimateNeurons, isEnabled, toPublicSafe, @@ -64,8 +67,8 @@ export type AiSlopInput = { * or temper it. Never used to override the model's own judgement. */ deterministicBand?: SlopBand | undefined; /** Optional BYOK: when present, the maintainer's frontier model writes the advisory (billed to their - * account, counted against the shared per-repo/day BYOK cap) instead of free Workers AI. Advisory-only - * either way — BYOK never changes whether this can block (it can't). */ + * account, counted against the shared per-repo/day BYOK cap) instead of the free/default reviewer. + * Advisory-only either way — BYOK never changes whether this can block (it can't). */ providerKey?: AiReviewProviderKey | null | undefined; }; @@ -134,10 +137,13 @@ export function slopFindingFromOpinion(opinion: SlopOpinion): SignalFinding | nu }; } -/** Free Workers-AI slop opinion with bounded retry/fallback attempts, all pre-budgeted. */ -async function runWorkersSlopOpinion(env: Env, system: string, user: string, maxTokens: number): Promise { +type WorkersSlopOpinionResult = { opinion: SlopOpinion | null; usage?: AiReviewActualUsage | undefined }; + +/** One free/default-reviewer slop opinion (whichever provider `env.AI` resolves to — self-host Codex/Claude + * Code/etc, or the legacy Workers-AI pair) with bounded retry/fallback attempts, all pre-budgeted. */ +async function runWorkersSlopOpinion(env: Env, system: string, user: string, maxTokens: number): Promise { const ai = env.AI as unknown as AiRunner | undefined; - if (!ai || typeof ai.run !== "function") return null; + if (!ai || typeof ai.run !== "function") return { opinion: null }; const gatewayId = env.AI_GATEWAY_ID?.trim(); const extra: AiGatewayOptions | undefined = gatewayId ? { gateway: { id: gatewayId } } : undefined; // Primary then a reliable per-slot fallback (distinct model families), 3× retry each before giving up. @@ -150,13 +156,13 @@ async function runWorkersSlopOpinion(env: Env, system: string, user: string, max extra, ); const parsed = parseSlopOpinion(coerceAiText(result)); - if (parsed) return parsed; + if (parsed) return { opinion: parsed, usage: coerceAiUsage(result) }; } catch { /* retry / fall through to fallback */ } } } - return null; + return { opinion: null }; } function buildUserPrompt(input: AiSlopInput): string { @@ -180,14 +186,14 @@ function buildUserPrompt(input: AiSlopInput): string { export async function runGittensoryAiSlopAdvisory(env: Env, input: AiSlopInput): Promise { if (!isEnabled(env.AI_SUMMARIES_ENABLED)) return { status: "disabled", reason: "AI summaries are disabled." }; if (!isEnabled(env.AI_PUBLIC_COMMENTS_ENABLED)) return { status: "disabled", reason: "Public AI comments are disabled." }; - if (!env.AI) return { status: "unavailable", reason: "Workers AI binding is not configured." }; + if (!env.AI) return { status: "unavailable", reason: "AI provider is not configured." }; const maxTokens = clampNumber(Number(env.AI_MAX_OUTPUT_TOKENS || 256), 256, 1024); const user = buildUserPrompt(input); // BYOK bills the maintainer's own account, so it does NOT draw on the free neuron budget — it has a - // separate per-repo/day cap shared with the AI review path. Free Workers-AI retry/fallback attempts are - // pre-budgeted at their worst case so malformed output or transient failures cannot amplify spend beyond - // the daily neuron budget. + // separate per-repo/day cap shared with the AI review path. Free/default-reviewer retry/fallback attempts + // are pre-budgeted at their worst case so malformed output or transient failures cannot amplify spend + // beyond the daily neuron budget. const freeCalls = input.providerKey ? 0 : WORKERS_SLOP_MAX_CALLS; const estimatedNeurons = freeCalls === 0 ? 0 : estimateNeurons(SLOP_SYSTEM_PROMPT.length + user.length, maxTokens, freeCalls); // Resolve the shared daily neuron budget IDENTICALLY to the AI review path (ai-review.ts): default HIGH @@ -210,24 +216,33 @@ export async function runGittensoryAiSlopAdvisory(env: Env, input: AiSlopInput): } } - // BYOK frontier model if configured, else the free Workers-AI primary (with fallback). Both fail-safe to null. + // BYOK frontier model if configured, else the free/default-reviewer primary (with fallback). Both fail-safe to null. let opinion: SlopOpinion | null; + let usage: AiReviewActualUsage | undefined; if (input.providerKey) { const { text } = await callAiProvider(input.providerKey, SLOP_SYSTEM_PROMPT, user, maxTokens); opinion = text ? parseSlopOpinion(text) : null; } else { - opinion = await runWorkersSlopOpinion(env, SLOP_SYSTEM_PROMPT, user, maxTokens); + ({ opinion, usage } = await runWorkersSlopOpinion(env, SLOP_SYSTEM_PROMPT, user, maxTokens)); } const finding = opinion ? slopFindingFromOpinion(opinion) : null; await record(env, input, "ok", estimatedNeurons, finding ? `advisory finding (${opinion?.band})` : opinion ? `clean/no-op (${opinion.band})` : "no usable output", { band: opinion?.band ?? null, surfaced: Boolean(finding), byok: Boolean(input.providerKey), - }); + }, usage); return { status: "ok", finding, band: opinion?.band ?? null, estimatedNeurons }; } -async function record(env: Env, input: AiSlopInput, status: string, estimatedNeurons: number, detail: string, metadata?: Record): Promise { +async function record( + env: Env, + input: AiSlopInput, + status: string, + estimatedNeurons: number, + detail: string, + metadata?: Record, + usage?: AiReviewActualUsage | undefined, +): Promise { await recordAiUsageEvent(env, { feature: "ai_slop_pr", actor: input.actor ?? null, @@ -236,6 +251,12 @@ async function record(env: Env, input: AiSlopInput, status: string, estimatedNeu model: input.providerKey ? `byok:${input.providerKey.provider}` : WORKERS_SLOP_MODELS.join("+"), status, estimatedNeurons, + provider: usage?.provider, + effort: usage?.effort, + inputTokens: usage?.inputTokens, + outputTokens: usage?.outputTokens, + totalTokens: usage?.totalTokens, + costUsd: usage?.costUsd, detail, metadata: { repoFullName: input.repoFullName, pullNumber: input.prNumber, ...(metadata ?? {}) }, }); diff --git a/src/services/ai-summaries.ts b/src/services/ai-summaries.ts index 7386f95f8b..af31ed97e0 100644 --- a/src/services/ai-summaries.ts +++ b/src/services/ai-summaries.ts @@ -2,6 +2,7 @@ import { recordAiUsageEvent, recordAuditEvent, sumAiEstimatedNeuronsSince } from import { sanitizePublicComment } from "../queue-intelligence"; import type { JsonValue } from "../types"; import type { AgentRunBundle } from "./agent-orchestrator"; +import { coerceAiUsage, type AiReviewActualUsage } from "./ai-review"; const PR_INTELLIGENCE_MARKER = ""; @@ -26,15 +27,17 @@ export async function summarizeAgentBundleWithAi(env: Env, bundle: AgentRunBundl const publicEnabled = isEnabled(env.AI_PUBLIC_COMMENTS_ENABLED); if (!privateEnabled) return { status: "disabled", reason: "AI summaries are disabled." }; if (visibility === "public" && !publicEnabled) return { status: "disabled", reason: "Public AI summaries are disabled." }; - if (!env.AI) return { status: "unavailable", reason: "Workers AI binding is not configured." }; + if (!env.AI) return { status: "unavailable", reason: "AI provider is not configured." }; - const model = env.WORKERS_AI_SUMMARY_MODEL || "@cf/meta/llama-3.1-8b-instruct-fp8-fast"; + // Empty string (not a Workers-AI `@cf/...` id — Workers AI has no live binding anywhere today, see + // CONVERGENCE_RUNBOOK.md): resolveModel's own per-provider default wins when no override is set. + const model = env.WORKERS_AI_SUMMARY_MODEL || ""; const maxOutputTokens = clampNumber(Number(env.AI_MAX_OUTPUT_TOKENS || 256), 64, 512); const signalBundle = compactAgentSignalBundle(bundle, visibility); const prompt = buildPrompt(signalBundle, visibility); const estimatedNeurons = estimateNeurons(prompt, maxOutputTokens); // Resolve the SHARED daily neuron budget exactly like ai-review.ts / ai-slop.ts (#1369): all three - // Workers-AI features sum into one `sumAiEstimatedNeuronsSince` counter, so the old `|| 10000` default + + // AI features sum into one `sumAiEstimatedNeuronsSince` counter, so the old `|| 10000` default + // 1M ceiling here starved summaries into quota_exceeded once shared usage crossed 10k — well under the // real 10M shared budget — and capped a configured budget at 1M. Default HIGH (10M) and clamp to 10M. const rawNeuronBudget = Number(env.AI_DAILY_NEURON_BUDGET); @@ -69,6 +72,7 @@ export async function summarizeAgentBundleWithAi(env: Env, bundle: AgentRunBundl temperature: 0.1, }); const rawText = extractAiText(response); + const usage = coerceAiUsage(response); if (!rawText) throw new Error("empty_ai_summary"); if (visibility === "public" && containsPublicForbiddenText(rawText)) { await recordAi(env, bundle, { @@ -77,6 +81,7 @@ export async function summarizeAgentBundleWithAi(env: Env, bundle: AgentRunBundl status: "unsafe", estimatedNeurons, detail: "public summary failed sanitizer", + usage, }); return { status: "unsafe", model, estimatedNeurons, reason: "public summary failed sanitizer" }; } @@ -88,10 +93,11 @@ export async function summarizeAgentBundleWithAi(env: Env, bundle: AgentRunBundl estimatedNeurons, detail: "summary generated", metadata: { visibility }, + usage, }); return { status: "ok", model, estimatedNeurons, text }; } catch (error) { - const reason = error instanceof Error ? error.message : "workers_ai_failed"; + const reason = error instanceof Error ? error.message : "ai_summary_failed"; await recordAi(env, bundle, { feature: `agent_${visibility}_summary`, model, @@ -216,12 +222,24 @@ async function recordAi( estimatedNeurons: number; detail?: string; metadata?: Record; + /** Real per-call usage from the configured provider (see `coerceAiUsage`), when available. */ + usage?: AiReviewActualUsage | undefined; }, ): Promise { await recordAiUsageEvent(env, { - ...event, + feature: event.feature, + model: event.model, + status: event.status, + estimatedNeurons: event.estimatedNeurons, + detail: event.detail, actor: bundle.run.actorLogin, route: bundle.run.surface, + provider: event.usage?.provider, + effort: event.usage?.effort, + inputTokens: event.usage?.inputTokens, + outputTokens: event.usage?.outputTokens, + totalTokens: event.usage?.totalTokens, + costUsd: event.usage?.costUsd, metadata: { runId: bundle.run.id, ...(event.metadata ?? {}) }, }); await recordAuditEvent(env, { @@ -273,14 +291,16 @@ export async function rewriteSignalBundleWithAi(env: Env, req: AiRewriteRequest) const publicEnabled = isEnabled(env.AI_PUBLIC_COMMENTS_ENABLED); if (!privateEnabled) return { status: "disabled", text: req.fallbackText, reason: "AI summaries are disabled." }; if (req.visibility === "public" && !publicEnabled) return { status: "disabled", text: req.fallbackText, reason: "Public AI summaries are disabled." }; - if (!env.AI) return { status: "unavailable", text: req.fallbackText, reason: "Workers AI binding is not configured." }; + if (!env.AI) return { status: "unavailable", text: req.fallbackText, reason: "AI provider is not configured." }; - const model = env.WORKERS_AI_SUMMARY_MODEL || "@cf/meta/llama-3.1-8b-instruct-fp8-fast"; + // Empty string (not a Workers-AI `@cf/...` id — Workers AI has no live binding anywhere today, see + // CONVERGENCE_RUNBOOK.md): resolveModel's own per-provider default wins when no override is set. + const model = env.WORKERS_AI_SUMMARY_MODEL || ""; const maxOutputTokens = clampNumber(Number(env.AI_MAX_OUTPUT_TOKENS || 256), 64, 512); const prompt = buildBundlePrompt(req.bundle, req.visibility); const estimatedNeurons = estimateNeurons(prompt, maxOutputTokens); // Resolve the SHARED daily neuron budget exactly like ai-review.ts / ai-slop.ts (#1369): all three - // Workers-AI features sum into one `sumAiEstimatedNeuronsSince` counter, so the old `|| 10000` default + + // AI features sum into one `sumAiEstimatedNeuronsSince` counter, so the old `|| 10000` default + // 1M ceiling here starved summaries into quota_exceeded once shared usage crossed 10k — well under the // real 10M shared budget — and capped a configured budget at 1M. Default HIGH (10M) and clamp to 10M. const rawNeuronBudget = Number(env.AI_DAILY_NEURON_BUDGET); @@ -308,16 +328,17 @@ export async function rewriteSignalBundleWithAi(env: Env, req: AiRewriteRequest) temperature: 0.1, }); const rawText = extractAiText(response); + const usage = coerceAiUsage(response); if (!rawText) throw new Error("empty_ai_summary"); if (req.visibility === "public" && containsPublicForbiddenText(rawText)) { - await recordGenericAi(env, req, { model, status: "unsafe", estimatedNeurons, detail: "public summary failed sanitizer" }); + await recordGenericAi(env, req, { model, status: "unsafe", estimatedNeurons, detail: "public summary failed sanitizer", usage }); return { status: "unsafe", text: req.fallbackText, model, estimatedNeurons, reason: "public summary failed sanitizer" }; } const text = sanitizeAiText(rawText, req.visibility); - await recordGenericAi(env, req, { model, status: "ok", estimatedNeurons, detail: "summary generated", metadata: { visibility: req.visibility } }); + await recordGenericAi(env, req, { model, status: "ok", estimatedNeurons, detail: "summary generated", metadata: { visibility: req.visibility }, usage }); return { status: "ok", text, model, estimatedNeurons }; } catch (error) { - const reason = error instanceof Error ? error.message : "workers_ai_failed"; + const reason = error instanceof Error ? error.message : "ai_summary_failed"; await recordGenericAi(env, req, { model, status: "error", estimatedNeurons: 0, detail: reason }); return { status: "error", text: req.fallbackText, model, estimatedNeurons, reason }; } @@ -366,7 +387,15 @@ function buildBundlePrompt(signalBundle: Record, visibility: async function recordGenericAi( env: Env, req: AiRewriteRequest, - event: { model: string; status: string; estimatedNeurons: number; detail?: string; metadata?: Record }, + event: { + model: string; + status: string; + estimatedNeurons: number; + detail?: string; + metadata?: Record; + /** Real per-call usage from the configured provider (see `coerceAiUsage`), when available. */ + usage?: AiReviewActualUsage | undefined; + }, ): Promise { await recordAiUsageEvent(env, { feature: req.feature, @@ -376,6 +405,12 @@ async function recordGenericAi( status: event.status, estimatedNeurons: event.estimatedNeurons, detail: event.detail, + provider: event.usage?.provider, + effort: event.usage?.effort, + inputTokens: event.usage?.inputTokens, + outputTokens: event.usage?.outputTokens, + totalTokens: event.usage?.totalTokens, + costUsd: event.usage?.costUsd, metadata: { ...(req.metadata ?? {}), ...(event.metadata ?? {}) }, }); await recordAuditEvent(env, { diff --git a/test/unit/ai-slop.test.ts b/test/unit/ai-slop.test.ts index 19bd074eee..6eee39fed0 100644 --- a/test/unit/ai-slop.test.ts +++ b/test/unit/ai-slop.test.ts @@ -205,6 +205,17 @@ describe("runGittensoryAiSlopAdvisory gating + fail-safe", () => { expect(result.estimatedNeurons).toBeGreaterThanOrEqual(6); }); + it("degrades to no finding when env.AI is present but not a valid runner (no .run function)", async () => { + const env = createTestEnv({ + AI: {} as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + }); + const result = await runGittensoryAiSlopAdvisory(env, baseInput); + expect(result).toMatchObject({ status: "ok", finding: null, band: null }); + }); + it("returns an advisory finding when the model flags an elevated band", async () => { const run = vi.fn(async () => ({ response: slopJson({ band: "elevated" }) })); const result = await runGittensoryAiSlopAdvisory(enabledEnv(run), baseInput); diff --git a/test/unit/ai-summaries.test.ts b/test/unit/ai-summaries.test.ts index 060c12549b..73577f4b8f 100644 --- a/test/unit/ai-summaries.test.ts +++ b/test/unit/ai-summaries.test.ts @@ -40,12 +40,12 @@ describe("Workers AI summaries", () => { expect(run).not.toHaveBeenCalled(); }); - it("reports unavailable Workers AI bindings when summaries are enabled", async () => { + it("reports unavailable AI provider when summaries are enabled", async () => { const env = createTestEnv({ AI_SUMMARIES_ENABLED: "true" }); await expect(summarizeAgentBundleWithAi(env, bundleFixture(), "private")).resolves.toEqual({ status: "unavailable", - reason: "Workers AI binding is not configured.", + reason: "AI provider is not configured.", }); }); @@ -63,7 +63,7 @@ describe("Workers AI summaries", () => { expect(result).toMatchObject({ status: "ok" }); expect(result.status === "ok" ? result.text : "").not.toMatch(/wallet|payout/i); expect(run).toHaveBeenCalledWith( - "@cf/meta/llama-3.1-8b-instruct-fp8-fast", + "", expect.objectContaining({ messages: expect.arrayContaining([expect.objectContaining({ role: "user", content: expect.not.stringContaining("source code") })]), }), @@ -249,7 +249,7 @@ describe("Workers AI summaries", () => { }); await expect(summarizeAgentBundleWithAi(thrown, bundleFixture(), "private")).resolves.toMatchObject({ status: "error", - reason: "workers_ai_failed", + reason: "ai_summary_failed", }); }); @@ -336,8 +336,8 @@ describe("optional deterministic-summary rewrite layer", () => { const run = vi.fn(async () => ({ response: "Default-config summary." })); const env = createTestEnv({ AI: { run } as unknown as Ai, AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true" }); const result = await rewriteSignalBundleWithAi(env, rewriteReq()); - expect(result).toMatchObject({ status: "ok", model: "@cf/meta/llama-3.1-8b-instruct-fp8-fast" }); - expect(run).toHaveBeenCalledWith("@cf/meta/llama-3.1-8b-instruct-fp8-fast", expect.objectContaining({ max_tokens: 256 })); + expect(result).toMatchObject({ status: "ok", model: "" }); + expect(run).toHaveBeenCalledWith("", expect.objectContaining({ max_tokens: 256 })); }); it("resolves the rewrite path's SHARED neuron budget like ai-review/ai-slop: default 10M, ceiling 10M, invalid → default (#1369)", async () => { @@ -367,7 +367,7 @@ describe("optional deterministic-summary rewrite layer", () => { await expect(rewriteSignalBundleWithAi(publicEnv({}, throwingRun), rewriteReq())).resolves.toMatchObject({ status: "error", text: DETERMINISTIC_BODY, - reason: "workers_ai_failed", + reason: "ai_summary_failed", }); });