Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 35 additions & 14 deletions src/review/planner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,11 @@
// • flag-OFF (default) → isPlannerEnabled is false, the handler short-circuits BEFORE parsing, and the worker
// is byte-identical to today (`@gittensory plan` falls through to the existing mention path → help card).
// • flag-ON → only a MAINTAINER can trigger it; the model sees only the (already-public) issue title + body;
// shared AI budget accounting runs before Workers AI; the output is public-safe-sanitized before posting;
// any model/error degrades to a no-plan no-op.
// shared AI budget accounting runs before the configured reviewer (self-host Codex/Claude Code/etc, or the
// legacy Workers-AI pair); the output is public-safe-sanitized before posting; any model/error degrades to
// a no-plan no-op.

import { BEST_REVIEW_MODELS, clampNumber, coerceAiText, estimateNeurons, RELIABLE_FALLBACK_MODELS, utcDayStartIso } from "../services/ai-review";
import { type AiReviewActualUsage, BEST_REVIEW_MODELS, clampNumber, coerceAiText, coerceAiUsage, estimateNeurons, RELIABLE_FALLBACK_MODELS, utcDayStartIso } from "../services/ai-review";
import { recordAiUsageEvent, sumAiEstimatedNeuronsSince } from "../db/repositories";
import { sanitizePublicComment } from "../github/commands";
import { AGENT_COMMAND_COMMENT_MARKER } from "../github/comments";
Expand Down Expand Up @@ -72,43 +73,63 @@ function plannerDailyBudget(env: Env): number {
return clampNumber(env.AI_DAILY_NEURON_BUDGET && Number.isFinite(raw) ? raw : 10_000_000, 0, 10_000_000);
}

async function recordPlannerUsage(env: Env, args: { actor?: string | null | undefined; repoFullName?: string | null | undefined; issueNumber?: number | null | undefined; status: string; estimatedNeurons: number; detail: string }): Promise<void> {
async function recordPlannerUsage(
env: Env,
args: {
actor?: string | null | undefined;
repoFullName?: string | null | undefined;
issueNumber?: number | null | undefined;
status: string;
estimatedNeurons: number;
detail: string;
usage?: AiReviewActualUsage | undefined;
},
): Promise<void> {
await recordAiUsageEvent(env, {
feature: "issue_plan",
actor: args.actor ?? null,
route: "github_app.issue_plan",
model: [BEST_REVIEW_MODELS[0], RELIABLE_FALLBACK_MODELS[0]].join("+"),
status: args.status,
estimatedNeurons: args.estimatedNeurons,
provider: args.usage?.provider,
effort: args.usage?.effort,
inputTokens: args.usage?.inputTokens,
outputTokens: args.usage?.outputTokens,
totalTokens: args.usage?.totalTokens,
costUsd: args.usage?.costUsd,
detail: args.detail,
metadata: { repoFullName: args.repoFullName ?? null, issueNumber: args.issueNumber ?? null },
});
}

/** One Workers-AI text completion for the planner: primary model, one reliable fallback, a single retry each.
type PlannerModelResult = { text: string | null; usage?: AiReviewActualUsage | undefined };

/** One reviewer text completion for the planner (whichever provider `env.AI` resolves to — self-host Codex/
* Claude Code/etc, or the legacy Workers-AI pair): primary model, one reliable fallback, a single retry each.
* Fail-safe — any error or empty output returns null. Mirrors runWorkersOpinion's routing (AI Gateway when set). */
async function runPlannerModel(env: Env, system: string, user: string): Promise<string | null> {
async function runPlannerModel(env: Env, system: string, user: string): Promise<PlannerModelResult> {
const ai = env.AI as unknown as { run?: (model: string, options: Record<string, unknown>, extra?: unknown) => Promise<unknown> } | undefined;
if (!ai || typeof ai.run !== "function") return null;
if (!ai || typeof ai.run !== "function") return { text: null };
const gatewayId = env.AI_GATEWAY_ID?.trim();
const extra = gatewayId ? { gateway: { id: gatewayId } } : undefined;
for (const model of [BEST_REVIEW_MODELS[0], RELIABLE_FALLBACK_MODELS[0]]) {
for (let attempt = 0; attempt < 2; attempt += 1) {
try {
const result = await ai.run(model, { max_tokens: PLANNER_MAX_TOKENS, temperature: 0.2, messages: [{ role: "system", content: system }, { role: "user", content: user }] }, extra);
const text = coerceAiText(result).trim();
if (text) return text;
if (text) return { text, usage: coerceAiUsage(result) };
} catch {
/* retry, then fall through to the fallback model */
}
}
}
return null;
return { text: null };
}

/** Generate an implementation plan (markdown) from an issue's title + body via Workers AI. Returns null when AI
* is unavailable or returns nothing (the caller then posts no plan). The returned text is bounded; the caller
* still sanitizes it before posting. */
/** Generate an implementation plan (markdown) from an issue's title + body via the configured reviewer. Returns
* null when AI is unavailable or returns nothing (the caller then posts no plan). The returned text is bounded;
* the caller still sanitizes it before posting. */
export async function generateIssuePlan(
env: Env,
issue: { title?: string | null | undefined; body?: string | null | undefined },
Expand All @@ -124,8 +145,8 @@ export async function generateIssuePlan(
await recordPlannerUsage(env, { ...accounting, status: "quota_exceeded", estimatedNeurons: 0, detail: `estimated ${estimatedNeurons} neurons exceeds remaining ${remainingBudget}` });
return null;
}
const plan = await runPlannerModel(env, PLANNER_SYSTEM_PROMPT, user);
await recordPlannerUsage(env, { ...accounting, status: plan ? "ok" : "no_output", estimatedNeurons: plan ? estimatedNeurons : 0, detail: plan ? "issue plan generated" : "no usable output" });
const { text: plan, usage } = await runPlannerModel(env, PLANNER_SYSTEM_PROMPT, user);
await recordPlannerUsage(env, { ...accounting, status: plan ? "ok" : "no_output", estimatedNeurons: plan ? estimatedNeurons : 0, detail: plan ? "issue plan generated" : "no usable output", usage });
if (!plan) return null;
return plan.slice(0, MAX_PLAN_CHARS);
}
Expand Down
51 changes: 32 additions & 19 deletions src/services/ai-review.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,20 @@
//
// • Advisory notes — a concise maintainer-style write-up (assessment + suggestions + risks). When the
// repo has BYOK configured, the maintainer's own frontier model (Anthropic/OpenAI) writes it;
// otherwise free Cloudflare Workers AI does. Advisory only — never blocks.
// • Consensus defect — a conservative gate signal. The free Workers-AI model PAIR each independently
// otherwise the configured free/default reviewer does (self-host: the AI_PROVIDER chain — Codex
// primary, Claude Code fallback, etc; unconfigured/hosted: the legacy Workers-AI pair below).
// Advisory only — never blocks.
// • Consensus defect — a conservative gate signal. The configured reviewer PAIR each independently
// reviews the diff; a defect is reported ONLY when BOTH models flag a high-confidence critical defect
// (bug / security / data-loss / build break). BYOK never changes this path, so it never changes who
// can be blocked. The resulting finding is honored by the gate only in `block` mode AND only for
// confirmed Gittensor contributors (the gate enforces that downstream).
//
// Every public string (notes + defect title/detail) is forced through `sanitizePublicComment`; anything
// that trips the public/private boundary is dropped, not published. Free Workers-AI calls are metered against
// the shared daily neuron budget; maintainer-paid BYOK calls have a separate repo/day cap. All calls
// are audited via `recordAiUsageEvent`.
// that trips the public/private boundary is dropped, not published. Free/default-reviewer calls are metered
// against the shared daily neuron budget; maintainer-paid BYOK calls have a separate repo/day cap. All calls
// are audited via `recordAiUsageEvent` (with real provider/token/cost usage when the configured provider
// reports it, per migration 0109 — see `coerceAiUsage`/`aggregateActualUsage`).
import {
countByokAiEventsForRepoSince,
recordAiUsageEvent,
Expand All @@ -33,15 +36,18 @@ import { isTestPath } from "../signals/test-evidence";
import type { CombineStrategy, OnMerge } from "../types";

/**
* The best free Workers-AI model pair for review accuracy — two different families for independence,
* both probe-verified in reviewbot to emit clean JSON. The consensus blocker always uses this pair.
* The legacy free Workers-AI model pair — used ONLY when neither a self-host `AI_REVIEW_PLAN` reviewer
* pair nor any configured provider (`AI_PROVIDER`) is present (see `reviewerModelLabel`). No `ai` binding
* exists in the deployed Worker today (Workers AI is fully retired — see CONVERGENCE_RUNBOOK.md), so this
* pair is inert in every current deployment; it stays only as the last-resort default these model ids
* were originally probe-verified against (both families independently clean-JSON in reviewbot).
*/
export const BEST_REVIEW_MODELS: readonly [string, string] = [
"@cf/openai/gpt-oss-120b",
"@cf/nvidia/nemotron-3-120b-a12b",
];

/** Reliable per-slot fallbacks (non-reasoning, clean JSON) so a slot never comes back empty. */
/** Reliable per-slot fallbacks for the legacy pair above (non-reasoning, clean JSON) so a slot never comes back empty. */
export const RELIABLE_FALLBACK_MODELS: readonly [string, string] = [
"@cf/meta/llama-3.3-70b-instruct-fp8-fast",
"@cf/mistralai/mistral-small-3.1-24b-instruct",
Expand Down Expand Up @@ -475,7 +481,11 @@ function stringField(value: unknown): string | undefined {
return typeof value === "string" && value.trim() ? value.trim() : undefined;
}

function coerceAiUsage(result: unknown): AiReviewActualUsage | undefined {
/** Extract a provider's real usage (tokens/cost/effort) from an `env.AI.run()` result, when the configured
* provider reports one (self-host CLI/HTTP providers do; the legacy Workers-AI binding never did). Shared
* by every AI feature's `recordAiUsageEvent` call so migration 0109's columns get real data, not just the
* estimated-neurons proxy, whenever it's available. */
export function coerceAiUsage(result: unknown): AiReviewActualUsage | undefined {
if (!result || typeof result !== "object") return undefined;
const usage = (result as Record<string, unknown>).usage;
if (!usage || typeof usage !== "object" || Array.isArray(usage)) return undefined;
Expand Down Expand Up @@ -616,8 +626,9 @@ function buildUserPrompt(input: GittensoryAiReviewInput): string {
: "Description: (none)",
"",
"Unified diff (truncated if large):",
// Widened 60k→120k so a large multi-file PR is actually reviewed in full (the 120B Workers-AI models have a
// 128k context window; pairing this with the higher output ceiling gives a thorough review). (#extensive-reviews)
// Widened 60k→120k so a large multi-file PR is actually reviewed in full (tuned against the legacy 120B
// Workers-AI pair's 128k context window; pairing this with the higher output ceiling gives a thorough
// review — self-host reviewers are configured with at least as much room). (#extensive-reviews)
input.diff.slice(0, 120000),
];
// Convergence (grounding): append the FINISHED CI status + FULL file content when the caller supplied them
Expand Down Expand Up @@ -708,7 +719,8 @@ function buildRepoInstructionsSystemAppend(repoInstructions: string | null | und
: "";
}

/** One Workers-AI opinion with a per-slot reliable fallback and a 3× retry on the primary. */
/** One reviewer opinion (whichever provider `env.AI` resolves to — self-host Codex/Claude Code/etc, or the
* legacy Workers-AI pair) with a per-slot reliable fallback and a 3× retry on the primary. */
async function runWorkersOpinion(
env: Env,
primary: string,
Expand Down Expand Up @@ -1192,12 +1204,13 @@ export async function runGittensoryAiReview(
if (!env.AI)
return {
status: "unavailable",
reason: "Workers AI binding is not configured.",
reason: "AI provider is not configured.",
};

// Output ceiling for the review. The old 1024 cap forced a shallow "no blockers" scorecard across large diffs;
// a thorough finding-by-finding review needs real room. Default 4096, max 8192 (the free Workers-AI 120B models
// support it); an explicit env value still wins, clamped. (#extensive-reviews)
// a thorough finding-by-finding review needs real room. Default 4096, max 8192 (the configured reviewer —
// self-host Codex/Claude Code or the legacy free Workers-AI 120B pair — supports it); an explicit env value
// still wins, clamped. (#extensive-reviews)
const maxTokens = clampNumber(
Number(env.AI_MAX_OUTPUT_TOKENS) || 4096,
512,
Expand All @@ -1223,11 +1236,11 @@ export async function runGittensoryAiReview(
// unchanged (byte-identical). Computed from `promptInput` so it travels with the (possibly defanged) input.
const system = buildSystemPrompt(promptInput);
const repoInstructionsSystemAppend = buildRepoInstructionsSystemAppend(promptInput.repoInstructions);
// The daily neuron budget governs FREE Workers-AI spend only. BYOK advisory calls bill the maintainer's
// The daily neuron budget governs FREE/default-reviewer spend only. BYOK advisory calls bill the maintainer's
// own provider account, so they are not counted here (and a BYOK advisory still runs when the free
// budget is exhausted). Free calls = the consensus pair in block mode (always Workers AI), plus the
// advisory leg only when it is NOT BYOK.
// Reviewers + combine strategy (#dual-ai-combiner). DEFAULT = the free Workers-AI pair (per-slot fallbacks)
// budget is exhausted). Free calls = the consensus pair in block mode (the configured self-host reviewers,
// or the legacy Workers-AI pair when none is configured), plus the advisory leg only when it is NOT BYOK.
// Reviewers + combine strategy (#dual-ai-combiner). DEFAULT = the legacy Workers-AI pair (per-slot fallbacks)
// combined by `consensus` — byte-identical to today. The self-host boot plan (`env.AI_REVIEW_PLAN`) supplies
// named providers (e.g. claude-code + codex) and a strategy; an explicit `input` field overrides it. `single`
// (or a single configured reviewer) runs ONE opinion; consensus/synthesis run two.
Expand Down
Loading
Loading