From d4effd87095304a086cc299a6ac58c28c744e347 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 14 Jun 2026 08:08:16 -0700 Subject: [PATCH] feat(slop): AI-assisted advisory slop layer (advisory-only, never blocks) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the slop feature (#716) with an opt-in AI second opinion that augments the deterministic detector with the SEMANTIC slop it cannot quantify: generated boilerplate that does not match intent, comments that merely restate code, cosmetic churn dressed up as substantive, a description that does not correspond to the diff. Hard guarantees (AI assistance never changes who can be blocked — only the deterministic core can block): - the new `ai_slop_advisory` finding code is NOT recognised by isConfiguredGateBlocker, so it can never be a blocker; - severity is at most `warning`, never `critical`; - it never feeds slopRisk or the gate policy; - fail-safe on every path (AI off / no binding / over budget / unparseable / unsafe text -> no finding, never throws). - src/services/ai-slop.ts: runGittensoryAiSlopAdvisory — one free Workers-AI call (reuses the ai-review budget window + public-safe sanitizer), returns at most one advisory SignalFinding. - opt-in `slopAiAdvisory` setting wired through types, schema (migration 0034), repositories, focus-manifest (.gittensory.yml gate.slop.aiAdvisory) and the RepositorySettings OpenAPI schema. Runs only when slopGateMode != off AND slopAiAdvisory. - processors: runAiSlopForAdvisory appends the advisory finding, reusing the already-fetched changed files; deliberately does NOT touch slopRisk. Also fixes a latent bug: slop_gate_mode / slop_gate_min_score were absent from the settings upsert's onConflictDoUpdate SET clause, so slop settings silently did not persist on update of an existing row. --- apps/gittensory-ui/public/openapi.json | 4 + migrations/0034_slop_ai_advisory.sql | 5 + src/db/repositories.ts | 9 + src/db/schema.ts | 1 + src/openapi/schemas.ts | 1 + src/queue/processors.ts | 42 ++- src/services/ai-review.ts | 10 +- src/services/ai-slop.ts | 207 +++++++++++++ src/signals/focus-manifest.ts | 8 +- src/types.ts | 5 + test/unit/ai-slop.test.ts | 293 ++++++++++++++++++ test/unit/data-spine.test.ts | 11 +- test/unit/focus-manifest.test.ts | 20 +- test/unit/maintainer-activation.test.ts | 1 + test/unit/policy-sanitizer.test.ts | 1 + test/unit/registration-readiness.test.ts | 1 + test/unit/repo-policy-readiness.test.ts | 1 + .../self-dogfood-registration-pack.test.ts | 1 + test/unit/settings-preview.test.ts | 1 + test/unit/signals-coverage.test.ts | 1 + test/unit/signals-v2.test.ts | 1 + test/unit/signals.test.ts | 5 + 22 files changed, 620 insertions(+), 9 deletions(-) create mode 100644 migrations/0034_slop_ai_advisory.sql create mode 100644 src/services/ai-slop.ts create mode 100644 test/unit/ai-slop.test.ts diff --git a/apps/gittensory-ui/public/openapi.json b/apps/gittensory-ui/public/openapi.json index e05fa8303b..8e830d0326 100644 --- a/apps/gittensory-ui/public/openapi.json +++ b/apps/gittensory-ui/public/openapi.json @@ -7975,6 +7975,9 @@ "slopGateMinScore": { "type": "number", "nullable": true + }, + "slopAiAdvisory": { + "type": "boolean" } }, "required": [ @@ -7990,6 +7993,7 @@ "duplicatePrGateMode", "qualityGateMode", "slopGateMode", + "slopAiAdvisory", "autoLabelEnabled", "gittensorLabel", "createMissingLabel", diff --git a/migrations/0034_slop_ai_advisory.sql b/migrations/0034_slop_ai_advisory.sql new file mode 100644 index 0000000000..7a8b1ddd48 --- /dev/null +++ b/migrations/0034_slop_ai_advisory.sql @@ -0,0 +1,5 @@ +-- Opt-in AI-assisted slop advisory (the `slopAiAdvisory` capability). When 1 AND slop_gate_mode != 'off', a +-- free Workers-AI pass adds an ADVISORY-only `ai_slop_advisory` finding for semantic slop the deterministic +-- detector cannot quantify. It NEVER feeds slopRisk or the gate (only the deterministic core can block). +-- Default 0 (off) preserves existing behavior for every current repo; opt-in via `.gittensory.yml`. +ALTER TABLE repository_settings ADD COLUMN slop_ai_advisory INTEGER NOT NULL DEFAULT 0; diff --git a/src/db/repositories.ts b/src/db/repositories.ts index 9685b11278..b010a4551c 100644 --- a/src/db/repositories.ts +++ b/src/db/repositories.ts @@ -400,6 +400,7 @@ export async function getRepositorySettings(env: Env, fullName: string): Promise qualityGateMinScore: null, slopGateMode: "off", slopGateMinScore: null, + slopAiAdvisory: false, aiReviewMode: "off", aiReviewByok: false, aiReviewProvider: null, @@ -430,6 +431,7 @@ export async function getRepositorySettings(env: Env, fullName: string): Promise qualityGateMinScore: normalizeQualityGateMinScore(row.qualityGateMinScore), slopGateMode: parseGateRuleMode(row.slopGateMode), slopGateMinScore: normalizeQualityGateMinScore(row.slopGateMinScore), + slopAiAdvisory: row.slopAiAdvisory, aiReviewMode: parseGateRuleMode(row.aiReviewMode), aiReviewByok: row.aiReviewByok, aiReviewProvider: normalizeAiReviewProvider(row.aiReviewProvider), @@ -464,6 +466,7 @@ export async function upsertRepositorySettings(env: Env, settings: Partial>; + repoFullName: string; + pr: { number: number; title: string; body?: string | null | undefined }; + author: string | null; + files: Awaited>; + deterministicBand: SlopBand; + }, +): Promise { + if (!args.advisory.headSha) return; + try { + const result = await runGittensoryAiSlopAdvisory(env, { + repoFullName: args.repoFullName, + prNumber: args.pr.number, + title: args.pr.title, + body: args.pr.body ?? undefined, + diff: buildAiReviewDiff(args.files), + actor: args.author, + deterministicBand: args.deterministicBand, + }); + if (result.status === "ok" && result.finding) args.advisory.findings.push(result.finding); + } catch (error) { + console.error(JSON.stringify({ level: "warn", event: "ai_slop_failed", repository: args.repoFullName, pullNumber: args.pr.number, error: errorMessage(error) })); + } +} + function linkedIssueDuplicatePullRequestsForGate(pr: PullRequestRecord, pullRequests: PullRequestRecord[]): number[] { const linkedIssues = new Set(pr.linkedIssues); if (linkedIssues.size === 0) return []; @@ -1117,6 +1152,11 @@ async function maybePublishPrPublicSurface( }); slopRisk = slop.slopRisk; advisory.findings.push(...slop.findings); + // AI-assisted slop advisory (#533, opt-in). Reuses the already-fetched files; appends at most one + // advisory-only finding. Deliberately does NOT update slopRisk — only the deterministic core blocks. + if (settings.slopAiAdvisory) { + await runAiSlopForAdvisory(env, { advisory, repoFullName, pr, author, files: slopFiles, deterministicBand: slop.band }); + } } if (gateEnabled && author && !publicSurfaceSkipped && !official) { diff --git a/src/services/ai-review.ts b/src/services/ai-review.ts index 2ae2edd083..2e7b15c8ab 100644 --- a/src/services/ai-review.ts +++ b/src/services/ai-review.ts @@ -87,21 +87,23 @@ type ModelReview = { type AiGatewayOptions = { gateway?: { id: string } }; type AiRunner = { run?: (model: string, options: Record, extra?: AiGatewayOptions) => Promise }; -function isEnabled(value: string | undefined): boolean { +// Exported so the sibling AI-advisory features (e.g. the slop advisory in `./ai-slop`) share ONE budget +// window + neuron estimator and never drift from the review path's accounting. +export function isEnabled(value: string | undefined): boolean { return /^(1|true|yes|on)$/i.test(value ?? ""); } -function clampNumber(value: number, min: number, max: number): number { +export function clampNumber(value: number, min: number, max: number): number { if (!Number.isFinite(value)) return min; return Math.min(max, Math.max(min, Math.floor(value))); } -function utcDayStartIso(): string { +export function utcDayStartIso(): string { const now = new Date(); return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())).toISOString(); } -function estimateNeurons(promptChars: number, maxOutputTokens: number, calls: number): number { +export function estimateNeurons(promptChars: number, maxOutputTokens: number, calls: number): number { const inputTokens = Math.ceil(promptChars / 4); return Math.max(1, Math.ceil((inputTokens + maxOutputTokens) * 0.035) * Math.max(1, calls)); } diff --git a/src/services/ai-slop.ts b/src/services/ai-slop.ts new file mode 100644 index 0000000000..1f69cf9f9f --- /dev/null +++ b/src/services/ai-slop.ts @@ -0,0 +1,207 @@ +// Gittensory AI-assisted slop advisory (the `slopAiAdvisory` capability). +// +// An ADVISORY-ONLY second opinion that augments the deterministic slop detector (src/signals/slop.ts). +// It exists to catch the SEMANTIC slop the deterministic rules cannot quantify — generated boilerplate +// that does not match the stated intent, comments that merely restate code, cosmetic churn dressed up as +// substantive work, a description that does not correspond to the diff. +// +// Hard guarantees (so AI assistance never changes who can be blocked — only the deterministic core blocks): +// • It NEVER feeds `slopRisk` or the gate. Its output is a single advisory `SignalFinding` with the code +// `ai_slop_advisory`, which `isConfiguredGateBlocker` does not recognise, so it can never be a blocker. +// • Severity is at most `warning` (never `critical`), so it cannot be mistaken for a consensus defect. +// • Fail-safe on every path: AI off / no binding / over-budget / unparseable / unsafe text → no finding. +// • Opt-in: only runs when the repo set `gate.slop.aiAdvisory: true` on top of `gate.slop.mode != off`. +// +// Free Cloudflare Workers AI only (one call, metered against the shared daily neuron budget). BYOK is a +// possible later enhancement; slop assessment does not need a frontier model. Every public string is +// forced through `toPublicSafe`; anything tripping the public/private boundary is dropped, not published. +import type { SignalFinding } from "../signals/engine"; +import type { SlopBand } from "../signals/slop"; +import { recordAiUsageEvent, sumAiEstimatedNeuronsSince } from "../db/repositories"; +import { + BEST_REVIEW_MODELS, + RELIABLE_FALLBACK_MODELS, + clampNumber, + coerceAiText, + estimateNeurons, + isEnabled, + toPublicSafe, + utcDayStartIso, +} from "./ai-review"; + +/** The finding code carried by the AI slop advisory. Deliberately NOT recognised by the gate's + * `isConfiguredGateBlocker`, which is what guarantees this advisory can never block. */ +export const AI_SLOP_FINDING_CODE = "ai_slop_advisory"; + +const SLOP_SYSTEM_PROMPT = [ + "You are a senior open-source maintainer giving a SECOND OPINION on whether a pull request shows signs of", + "low-effort, automated, or padding-style contribution ('slop'). Deterministic checks already ran; you add", + "judgement they cannot, focusing on semantics.", + "Judge ONLY the diff and context provided. Be conservative and fair — most pull requests are genuine.", + "Reserve 'elevated' or 'high' for clear, evidence-backed cases: generated boilerplate that does not match", + "the stated intent, comments that merely restate the code, no-op or cosmetic churn presented as", + "substantive, or a description that does not correspond to the diff. When in doubt, choose 'clean' or 'low'.", + "Never accuse; describe the observable characteristics constructively so the maintainer can decide.", + "Never mention rewards, rankings, payouts, wallets, hotkeys, coldkeys, trust scores, scoreability,", + "reviewability, or farming.", + "Respond with ONLY a JSON object of this exact shape (no prose, no code fence):", + '{"band": "clean"|"low"|"elevated"|"high", "rationale": string, "signals": string[]}', +].join(" "); + +export type AiSlopInput = { + repoFullName: string; + prNumber: number; + title: string; + body?: string | null | undefined; + /** A bounded unified-diff-ish string (filenames + patches), built by the caller. */ + diff: string; + actor?: string | null | undefined; + /** The deterministic band already computed for this PR — passed as context so the model can corroborate + * or temper it. Never used to override the model's own judgement. */ + deterministicBand?: SlopBand | undefined; +}; + +export type AiSlopResult = + | { status: "disabled"; reason: string } + | { status: "unavailable"; reason: string } + | { status: "quota_exceeded"; estimatedNeurons: number; remainingBudget: number } + | { status: "ok"; finding: SignalFinding | null; band: SlopBand | null; estimatedNeurons: number }; + +type SlopOpinion = { band: SlopBand; rationale: string; signals: string[] }; + +const SLOP_BANDS: readonly SlopBand[] = ["clean", "low", "elevated", "high"]; + +function isSlopBand(value: unknown): value is SlopBand { + return typeof value === "string" && (SLOP_BANDS as readonly string[]).includes(value); +} + +type AiGatewayOptions = { gateway?: { id: string } }; +type AiRunner = { run?: (model: string, options: Record, extra?: AiGatewayOptions) => Promise }; + +/** Parse a model's JSON slop opinion into a normalized {@link SlopOpinion}, or null when unusable. */ +export function parseSlopOpinion(text: string): SlopOpinion | null { + const match = text + .replace(/^```(?:json)?\s*/i, "") + .replace(/```$/i, "") + .match(/\{[\s\S]*\}/); + if (!match) return null; + try { + const obj = JSON.parse(match[0]) as Record; + if (!isSlopBand(obj.band)) return null; + const rationale = typeof obj.rationale === "string" ? obj.rationale.trim().slice(0, 400) : ""; + const signals = Array.isArray(obj.signals) + ? obj.signals.filter((x): x is string => typeof x === "string").map((x) => x.trim()).filter(Boolean).slice(0, 4) + : []; + if (!rationale && signals.length === 0) return null; + return { band: obj.band, rationale, signals }; + } catch { + return null; + } +} + +/** + * Convert a parsed opinion into a public-safe advisory finding, or null to add nothing. Returns null for a + * `clean` band (no noise) and whenever the public text does not survive the public/private sanitizer. + */ +export function slopFindingFromOpinion(opinion: SlopOpinion): SignalFinding | null { + if (opinion.band === "clean") return null; + const safeRationale = toPublicSafe(opinion.rationale); + const safeSignals = opinion.signals.map((s) => toPublicSafe(s)).filter((s): s is string => Boolean(s)); + // Nothing publishable survived sanitization → drop the advisory entirely (fail-safe, never publish). + if (!safeRationale && safeSignals.length === 0) return null; + const detailBody = safeRationale ?? "An AI maintainer-assist pass flagged possible low-effort patterns in this change."; + const detail = safeSignals.length > 0 ? `${detailBody} Observations: ${safeSignals.join("; ")}.` : detailBody; + const publicText = `AI maintainer-assist (advisory): ${detail}`; + return { + code: AI_SLOP_FINDING_CODE, + title: `AI maintainer-assist flagged possible low-effort patterns (${opinion.band})`, + // `elevated`/`high` read as a warning; `low` as an informational note. NEVER `critical` (never a blocker). + severity: opinion.band === "elevated" || opinion.band === "high" ? "warning" : "info", + detail, + action: "Advisory only — review the noted patterns; this AI assist never blocks the gate.", + publicText, + }; +} + +/** One free Workers-AI slop opinion with a reliable per-slot fallback and a 3× retry on the primary. */ +async function runWorkersSlopOpinion(env: Env, system: string, user: string, maxTokens: number): Promise { + const ai = env.AI as unknown as AiRunner | undefined; + if (!ai || typeof ai.run !== "function") return null; + const gatewayId = env.AI_GATEWAY_ID?.trim(); + const extra: AiGatewayOptions | undefined = gatewayId ? { gateway: { id: gatewayId } } : undefined; + // Primary then a reliable per-slot fallback (distinct model families), 3× retry each before giving up. + for (const model of [BEST_REVIEW_MODELS[0], RELIABLE_FALLBACK_MODELS[0]]) { + for (let attempt = 0; attempt < 3; attempt += 1) { + try { + const result = await ai.run( + model, + { max_tokens: maxTokens, temperature: 0, messages: [{ role: "system", content: system }, { role: "user", content: user }] }, + extra, + ); + const parsed = parseSlopOpinion(coerceAiText(result)); + if (parsed) return parsed; + } catch { + /* retry / fall through to fallback */ + } + } + } + return null; +} + +function buildUserPrompt(input: AiSlopInput): string { + return [ + `Repository: ${input.repoFullName}`, + `Pull request #${input.prNumber}: ${input.title}`, + input.body ? `Description:\n${input.body.slice(0, 2000)}` : "Description: (none)", + input.deterministicBand ? `Deterministic slop band (for reference): ${input.deterministicBand}` : "", + "", + "Unified diff (truncated if large):", + input.diff.slice(0, 60000), + ] + .filter(Boolean) + .join("\n"); +} + +/** + * Run the AI slop advisory. Returns a single advisory finding (or null) plus the model's band. Fail-safe on + * every path: no finding and no thrown error ever reaches the caller. + */ +export async function runGittensoryAiSlopAdvisory(env: Env, input: AiSlopInput): Promise { + if (!isEnabled(env.AI_SUMMARIES_ENABLED)) return { status: "disabled", reason: "AI summaries are disabled." }; + if (!isEnabled(env.AI_PUBLIC_COMMENTS_ENABLED)) return { status: "disabled", reason: "Public AI comments are disabled." }; + if (!env.AI) return { status: "unavailable", reason: "Workers AI binding is not configured." }; + + const maxTokens = clampNumber(Number(env.AI_MAX_OUTPUT_TOKENS || 256), 256, 1024); + const user = buildUserPrompt(input); + const estimatedNeurons = estimateNeurons(SLOP_SYSTEM_PROMPT.length + user.length, maxTokens, 1); + const budget = clampNumber(Number(env.AI_DAILY_NEURON_BUDGET || 10000), 0, 1_000_000); + const used = await sumAiEstimatedNeuronsSince(env, utcDayStartIso()); + const remainingBudget = Math.max(0, budget - used); + if (estimatedNeurons > remainingBudget) { + await record(env, input, "quota_exceeded", 0, `estimated ${estimatedNeurons} neurons exceeds remaining ${remainingBudget}`); + return { status: "quota_exceeded", estimatedNeurons, remainingBudget }; + } + + const opinion = await runWorkersSlopOpinion(env, SLOP_SYSTEM_PROMPT, user, maxTokens); + const finding = opinion ? slopFindingFromOpinion(opinion) : null; + await record(env, input, "ok", estimatedNeurons, finding ? `advisory finding (${opinion?.band})` : opinion ? `clean/no-op (${opinion.band})` : "no usable output", { + band: opinion?.band ?? null, + surfaced: Boolean(finding), + }); + return { status: "ok", finding, band: opinion?.band ?? null, estimatedNeurons }; +} + +async function record(env: Env, input: AiSlopInput, status: string, estimatedNeurons: number, detail: string, metadata?: Record): Promise { + await recordAiUsageEvent(env, { + feature: "ai_slop_pr", + actor: input.actor ?? null, + route: "github_app.ai_slop", + model: BEST_REVIEW_MODELS.join("+"), + status, + estimatedNeurons, + detail, + metadata: { repoFullName: input.repoFullName, pullNumber: input.prNumber, ...(metadata ?? {}) }, + }); +} + +export const __aiSlopInternals = { parseSlopOpinion, slopFindingFromOpinion, buildUserPrompt }; diff --git a/src/signals/focus-manifest.ts b/src/signals/focus-manifest.ts index 727173140a..2596e3412b 100644 --- a/src/signals/focus-manifest.ts +++ b/src/signals/focus-manifest.ts @@ -24,6 +24,7 @@ export type FocusManifestGateConfig = { readinessMinScore: number | null; slopMode: GateRuleMode | null; slopMinScore: number | null; + slopAiAdvisory: boolean | null; aiReviewMode: GateRuleMode | null; aiReviewByok: boolean | null; aiReviewProvider: "anthropic" | "openai" | null; @@ -151,6 +152,7 @@ const EMPTY_GATE_CONFIG: FocusManifestGateConfig = { readinessMinScore: null, slopMode: null, slopMinScore: null, + slopAiAdvisory: null, aiReviewMode: null, aiReviewByok: null, aiReviewProvider: null, @@ -290,6 +292,7 @@ function parseGateConfig(value: JsonValue | undefined, warnings: string[]): Focu readinessMinScore: normalizeOptionalScore(readinessRecord?.minScore, "gate.readiness.minScore", warnings), slopMode: normalizeOptionalGateMode(slopRecord?.mode, "gate.slop.mode", warnings), slopMinScore: normalizeOptionalScore(slopRecord?.minScore, "gate.slop.minScore", warnings), + slopAiAdvisory: normalizeOptionalBoolean(slopRecord?.aiAdvisory, "gate.slop.aiAdvisory", warnings), aiReviewMode: normalizeOptionalGateMode(aiReviewRecord?.mode, "gate.aiReview.mode", warnings), aiReviewByok: normalizeOptionalBoolean(aiReviewRecord?.byok, "gate.aiReview.byok", warnings), aiReviewProvider: normalizeOptionalEnum(aiReviewRecord?.provider, "gate.aiReview.provider", ["anthropic", "openai"] as const, warnings), @@ -304,6 +307,7 @@ function parseGateConfig(value: JsonValue | undefined, warnings: string[]): Focu gate.readinessMinScore !== null || gate.slopMode !== null || gate.slopMinScore !== null || + gate.slopAiAdvisory !== null || gate.aiReviewMode !== null || gate.aiReviewByok !== null || gate.aiReviewProvider !== null || @@ -328,10 +332,11 @@ export function gateConfigToJson(gate: FocusManifestGateConfig): JsonValue { if (gate.readinessMinScore !== null) readiness.minScore = gate.readinessMinScore; out.readiness = readiness; } - if (gate.slopMode !== null || gate.slopMinScore !== null) { + if (gate.slopMode !== null || gate.slopMinScore !== null || gate.slopAiAdvisory !== null) { const slop: Record = {}; if (gate.slopMode !== null) slop.mode = gate.slopMode; if (gate.slopMinScore !== null) slop.minScore = gate.slopMinScore; + if (gate.slopAiAdvisory !== null) slop.aiAdvisory = gate.slopAiAdvisory; out.slop = slop; } if (gate.aiReviewMode !== null || gate.aiReviewByok !== null || gate.aiReviewProvider !== null || gate.aiReviewModel !== null) { @@ -482,6 +487,7 @@ export function resolveEffectiveSettings(dbSettings: RepositorySettings, manifes if (gate.readinessMinScore !== null) effective.qualityGateMinScore = gate.readinessMinScore; if (gate.slopMode !== null) effective.slopGateMode = gate.slopMode; if (gate.slopMinScore !== null) effective.slopGateMinScore = gate.slopMinScore; + if (gate.slopAiAdvisory !== null) effective.slopAiAdvisory = gate.slopAiAdvisory; if (gate.aiReviewMode !== null) effective.aiReviewMode = gate.aiReviewMode; if (gate.aiReviewByok !== null) effective.aiReviewByok = gate.aiReviewByok; if (gate.aiReviewProvider !== null) effective.aiReviewProvider = gate.aiReviewProvider; diff --git a/src/types.ts b/src/types.ts index a354f111e3..439e1e79fb 100644 --- a/src/types.ts +++ b/src/types.ts @@ -397,6 +397,11 @@ export type RepositorySettings = { slopGateMode: GateRuleMode; /** Slop-risk threshold (0-100) at/above which `slopGateMode: block` blocks. Default 60 (the `high` band). */ slopGateMinScore?: number | null | undefined; + /** AI-assisted slop advisory (the `slopAiAdvisory` capability). When true AND `slopGateMode != off`, a + * free Workers-AI pass adds an ADVISORY-only `ai_slop_advisory` finding for semantic slop the + * deterministic detector cannot quantify. It NEVER feeds slopRisk or the gate (only the deterministic + * core blocks). Default false — opt-in via `.gittensory.yml gate.slop.aiAdvisory`. */ + slopAiAdvisory: boolean; /** AI maintainer review. `off` = no AI; `advisory` = post AI review notes only; `block` = ALSO let a * dual-model high-confidence consensus defect become a gate blocker (confirmed-contributors only, * like every other blocker). Default `off` — AI is opt-in. */ diff --git a/test/unit/ai-slop.test.ts b/test/unit/ai-slop.test.ts new file mode 100644 index 0000000000..784e0c9dc5 --- /dev/null +++ b/test/unit/ai-slop.test.ts @@ -0,0 +1,293 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + AI_SLOP_FINDING_CODE, + __aiSlopInternals, + runGittensoryAiSlopAdvisory, + type AiSlopInput, +} from "../../src/services/ai-slop"; +import { evaluateGateCheck } from "../../src/rules/advisory"; +import { runAiSlopForAdvisory } from "../../src/queue/processors"; +import type { Advisory, PullRequestFileRecord } from "../../src/types"; +import { createTestEnv } from "../helpers/d1"; + +const { parseSlopOpinion, slopFindingFromOpinion, buildUserPrompt } = __aiSlopInternals; + +function slopJson(over: Partial<{ band: string; rationale: string; signals: string[] }> = {}): string { + return JSON.stringify({ + band: over.band ?? "elevated", + rationale: over.rationale ?? "The diff is large but adds little substantive logic.", + signals: over.signals ?? ["Most lines are reformatting", "Comments restate the code"], + }); +} + +const baseInput: AiSlopInput = { + repoFullName: "acme/widgets", + prNumber: 7, + title: "Tidy things up", + body: "General cleanup", + diff: "### src/a.ts (modified) +80/-2\n@@\n+// set x to one\n+const x = 1;", + actor: "alice", + deterministicBand: "elevated", +}; + +const enabledEnv = (run: unknown) => + createTestEnv({ + AI: { run } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + }); + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("parseSlopOpinion", () => { + it("parses a well-formed opinion and caps the signals list", () => { + const parsed = parseSlopOpinion(slopJson({ signals: ["a", "b", "c", "d", "e", "f"] })); + expect(parsed).toMatchObject({ band: "elevated" }); + expect(parsed?.signals).toHaveLength(4); // capped at 4 + }); + + it("strips a ```json code fence before parsing", () => { + expect(parseSlopOpinion("```json\n" + slopJson({ band: "high" }) + "\n```")?.band).toBe("high"); + }); + + it("rejects an invalid band", () => { + expect(parseSlopOpinion(JSON.stringify({ band: "toxic", rationale: "x", signals: [] }))).toBeNull(); + }); + + it("rejects when there is neither a rationale nor any signal", () => { + expect(parseSlopOpinion(JSON.stringify({ band: "low", rationale: "", signals: [] }))).toBeNull(); + }); + + it("returns null on non-JSON text", () => { + expect(parseSlopOpinion("the model refused to answer")).toBeNull(); + }); + + it("returns null when a brace-shaped blob is not valid JSON (parse throws)", () => { + // Matches the {…} regex but JSON.parse throws on the unquoted keys → caught → null. + expect(parseSlopOpinion("{ band: high, rationale: nope }")).toBeNull(); + }); + + it("filters non-string signals", () => { + const parsed = parseSlopOpinion(JSON.stringify({ band: "low", rationale: "ok", signals: ["real", 5, null, "two"] })); + expect(parsed?.signals).toEqual(["real", "two"]); + }); +}); + +describe("slopFindingFromOpinion", () => { + it("returns null for a clean band (no advisory noise)", () => { + expect(slopFindingFromOpinion({ band: "clean", rationale: "looks genuine", signals: [] })).toBeNull(); + }); + + it("maps low → info and elevated/high → warning, with the advisory code", () => { + expect(slopFindingFromOpinion({ band: "low", rationale: "minor", signals: [] })).toMatchObject({ + code: AI_SLOP_FINDING_CODE, + severity: "info", + }); + expect(slopFindingFromOpinion({ band: "elevated", rationale: "padding", signals: ["x"] })?.severity).toBe("warning"); + expect(slopFindingFromOpinion({ band: "high", rationale: "generated", signals: ["x"] })?.severity).toBe("warning"); + }); + + it("never emits a critical severity (so it can never look like a consensus defect)", () => { + for (const band of ["low", "elevated", "high"] as const) { + expect(slopFindingFromOpinion({ band, rationale: "r", signals: [] })?.severity).not.toBe("critical"); + } + }); + + it("composes signals into the public-safe detail", () => { + const finding = slopFindingFromOpinion({ band: "elevated", rationale: "Large but shallow.", signals: ["reformatting", "restated comments"] }); + expect(finding?.detail).toContain("Large but shallow."); + expect(finding?.detail).toContain("reformatting"); + expect(finding?.publicText).toContain("AI maintainer-assist"); + }); + + it("drops the finding when nothing survives public-safe sanitization", () => { + // 'reward' / 'farming' are forbidden public terms → sanitizer strips them; with no safe content left, drop. + const finding = slopFindingFromOpinion({ band: "high", rationale: "reward farming payout", signals: ["reward", "payout"] }); + // Either dropped entirely, or the public text never leaks a forbidden term. + if (finding) expect(finding.publicText ?? "").not.toMatch(/reward|farming|payout/i); + }); + + it("falls back to a generic detail body when only signals (no rationale) survive", () => { + const finding = slopFindingFromOpinion({ band: "elevated", rationale: "", signals: ["mostly reformatting"] }); + expect(finding?.detail).toContain("An AI maintainer-assist pass flagged"); + expect(finding?.detail).toContain("mostly reformatting"); + }); +}); + +describe("buildUserPrompt", () => { + it("omits the description and band lines when they are absent", () => { + const prompt = buildUserPrompt({ repoFullName: "a/b", prNumber: 1, title: "t", diff: "d" }); + expect(prompt).toContain("Description: (none)"); + expect(prompt).not.toContain("Deterministic slop band"); + }); + + it("includes the description and band when provided", () => { + const prompt = buildUserPrompt({ repoFullName: "a/b", prNumber: 1, title: "t", diff: "d", body: "the body", deterministicBand: "high" }); + expect(prompt).toContain("the body"); + expect(prompt).toContain("Deterministic slop band (for reference): high"); + }); +}); + +describe("runGittensoryAiSlopAdvisory gating + fail-safe", () => { + it("is disabled until both AI flags are on, and never calls the model", async () => { + const run = vi.fn(); + const env = createTestEnv({ AI: { run } as unknown as Ai, AI_SUMMARIES_ENABLED: "true" }); + await expect(runGittensoryAiSlopAdvisory(env, baseInput)).resolves.toMatchObject({ status: "disabled" }); + expect(run).not.toHaveBeenCalled(); + }); + + it("reports unavailable when the Workers AI binding is missing", async () => { + const env = createTestEnv({ AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true" }); + await expect(runGittensoryAiSlopAdvisory(env, baseInput)).resolves.toMatchObject({ status: "unavailable" }); + }); + + it("enforces the shared daily neuron budget before calling the model", async () => { + const run = vi.fn(); + const env = createTestEnv({ AI: { run } as unknown as Ai, AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true", AI_DAILY_NEURON_BUDGET: "1" }); + await expect(runGittensoryAiSlopAdvisory(env, baseInput)).resolves.toMatchObject({ status: "quota_exceeded" }); + expect(run).not.toHaveBeenCalled(); + }); + + it("returns an advisory finding when the model flags an elevated band", async () => { + const run = vi.fn(async () => ({ response: slopJson({ band: "elevated" }) })); + const result = await runGittensoryAiSlopAdvisory(enabledEnv(run), baseInput); + expect(result.status).toBe("ok"); + if (result.status !== "ok") throw new Error("unreachable"); + expect(result.band).toBe("elevated"); + expect(result.finding).toMatchObject({ code: AI_SLOP_FINDING_CODE, severity: "warning" }); + }); + + it("returns no finding when the model judges the change clean", async () => { + const run = vi.fn(async () => ({ response: slopJson({ band: "clean", rationale: "genuine effort", signals: [] }) })); + const result = await runGittensoryAiSlopAdvisory(enabledEnv(run), baseInput); + expect(result.status).toBe("ok"); + if (result.status !== "ok") throw new Error("unreachable"); + expect(result.band).toBe("clean"); + expect(result.finding).toBeNull(); + }); + + it("is fail-safe: a throwing model yields ok with no finding (never throws, never blocks)", async () => { + const run = vi.fn(async () => { + throw new Error("model exploded"); + }); + const result = await runGittensoryAiSlopAdvisory(enabledEnv(run), baseInput); + expect(result.status).toBe("ok"); + if (result.status !== "ok") throw new Error("unreachable"); + expect(result.finding).toBeNull(); + expect(run).toHaveBeenCalled(); // it tried (3× primary + fallback) and gave up cleanly + }); + + it("falls back to the reliable model when the primary keeps returning garbage", async () => { + const run = vi.fn(async (model: string) => ({ response: model.includes("gpt-oss") ? "not json" : slopJson({ band: "low" }) })); + const result = await runGittensoryAiSlopAdvisory(enabledEnv(run), baseInput); + expect(result.status).toBe("ok"); + if (result.status !== "ok") throw new Error("unreachable"); + expect(result.band).toBe("low"); + }); +}); + +describe("the AI slop advisory can never become a gate blocker", () => { + function advisoryWithAiSlop(): Advisory { + return { + id: "advisory-aislop", + targetType: "pull_request", + targetKey: "owner/repo#7", + repoFullName: "owner/repo", + pullNumber: 7, + headSha: "sha7", + conclusion: "neutral", + severity: "warning", + title: "Gittensory advisory available", + summary: "1 advisory finding generated.", + findings: [slopFindingFromOpinion({ band: "high", rationale: "looks low effort", signals: ["padding"] })!], + generatedAt: "2026-06-14T00:00:00.000Z", + }; + } + + it("is not a configured blocker even for a confirmed contributor with every gate mode on", () => { + const gate = evaluateGateCheck(advisoryWithAiSlop(), { + confirmedContributor: true, + linkedIssueGateMode: "block", + duplicatePrGateMode: "block", + qualityGateMode: "block", + aiReviewGateMode: "block", + // slop block mode but a sub-threshold risk → the deterministic slop blocker does NOT fire either. + slopGateMode: "block", + slopRisk: 10, + slopGateMinScore: 60, + }); + expect(gate.conclusion).toBe("success"); + expect(gate.blockers).toHaveLength(0); + // It still surfaces as an advisory warning. + expect(gate.warnings.some((w) => w.code === AI_SLOP_FINDING_CODE)).toBe(true); + }); +}); + +describe("runAiSlopForAdvisory (processor wiring)", () => { + function advisory(over: Partial = {}): Advisory { + return { + id: "adv-slop", + targetType: "pull_request", + targetKey: "acme/widgets#3", + repoFullName: "acme/widgets", + pullNumber: 3, + headSha: "sha3", + conclusion: "neutral", + severity: "info", + title: "Gittensory advisory available", + summary: "ok", + findings: [], + generatedAt: "2026-06-14T00:00:00.000Z", + ...over, + }; + } + const files: PullRequestFileRecord[] = [ + { repoFullName: "acme/widgets", pullNumber: 3, path: "src/a.ts", status: "modified", additions: 80, deletions: 2, changes: 82, payload: { patch: "@@\n+// set x\n+const x = 1;" } }, + ]; + const pr = { number: 3, title: "Tidy", body: "cleanup" }; + + it("appends a single ai_slop_advisory finding when the model flags slop", async () => { + const adv = advisory(); + await runAiSlopForAdvisory(enabledEnv(async () => ({ response: slopJson({ band: "high" }) })), { + advisory: adv, + repoFullName: "acme/widgets", + pr, + author: "alice", + files, + deterministicBand: "elevated", + }); + expect(adv.findings.map((f) => f.code)).toEqual([AI_SLOP_FINDING_CODE]); + }); + + it("no-ops when the advisory has no head SHA", async () => { + const noSha = advisory(); + delete (noSha as Partial).headSha; + const run = vi.fn(); + await runAiSlopForAdvisory(enabledEnv(run), { advisory: noSha, repoFullName: "acme/widgets", pr, author: "alice", files, deterministicBand: "low" }); + expect(noSha.findings).toEqual([]); + expect(run).not.toHaveBeenCalled(); + }); + + it("adds nothing when the model judges the change clean", async () => { + const adv = advisory(); + await runAiSlopForAdvisory(enabledEnv(async () => ({ response: slopJson({ band: "clean", rationale: "genuine", signals: [] }) })), { + advisory: adv, + repoFullName: "acme/widgets", + pr, + author: "alice", + files, + deterministicBand: "clean", + }); + expect(adv.findings).toEqual([]); + }); + + it("is fail-safe: a thrown error (broken DB) yields no finding and never throws", async () => { + const adv = advisory(); + const env = { ...enabledEnv(async () => ({ response: slopJson() })), DB: undefined } as unknown as Env; + await expect(runAiSlopForAdvisory(env, { advisory: adv, repoFullName: "acme/widgets", pr, author: "alice", files, deterministicBand: "high" })).resolves.toBeUndefined(); + expect(adv.findings).toEqual([]); + }); +}); diff --git a/test/unit/data-spine.test.ts b/test/unit/data-spine.test.ts index 550d7c99e9..3f2d8a961d 100644 --- a/test/unit/data-spine.test.ts +++ b/test/unit/data-spine.test.ts @@ -246,11 +246,20 @@ describe("data spine repositories", () => { await upsertRepositorySettings(env, { repoFullName: "owner/defaultpack" }); expect((await getRepositorySettings(env, "owner/defaultpack")).gatePack).toBe("gittensor"); // slop gate (#530/#532) round-trips and defaults to off. - await upsertRepositorySettings(env, { repoFullName: "owner/sloprepo", slopGateMode: "block", slopGateMinScore: 55 }); + await upsertRepositorySettings(env, { repoFullName: "owner/sloprepo", slopGateMode: "block", slopGateMinScore: 55, slopAiAdvisory: true }); const slopSettings = await getRepositorySettings(env, "owner/sloprepo"); expect(slopSettings.slopGateMode).toBe("block"); expect(slopSettings.slopGateMinScore).toBe(55); + expect(slopSettings.slopAiAdvisory).toBe(true); // AI advisory opt-in round-trips expect((await getRepositorySettings(env, "owner/defaultpack")).slopGateMode).toBe("off"); + expect((await getRepositorySettings(env, "owner/defaultpack")).slopAiAdvisory).toBe(false); // defaults off + // Persist-on-UPDATE: re-upserting an existing row must persist slop_* (these were previously missing + // from the onConflictDoUpdate SET clause, so updates silently dropped them). + await upsertRepositorySettings(env, { repoFullName: "owner/sloprepo", slopGateMode: "advisory", slopGateMinScore: 40, slopAiAdvisory: false }); + const updated = await getRepositorySettings(env, "owner/sloprepo"); + expect(updated.slopGateMode).toBe("advisory"); + expect(updated.slopGateMinScore).toBe(40); + expect(updated.slopAiAdvisory).toBe(false); expect(await getRepoSyncState(env, "missing/repo")).toBeNull(); expect(await getPullRequest(env, "owner/repo", 404)).toBeNull(); expect(await getIssue(env, "owner/repo", 404)).toBeNull(); diff --git a/test/unit/focus-manifest.test.ts b/test/unit/focus-manifest.test.ts index 65366b59fb..3f9d28f5b1 100644 --- a/test/unit/focus-manifest.test.ts +++ b/test/unit/focus-manifest.test.ts @@ -400,7 +400,7 @@ describe("compileFocusManifestPolicy", () => { issueDiscoveryPolicy: "neutral", maintainerNotes: [], publicNotes: ["Keep PRs focused.", "Maximize your reward payout"], - gate: { present: false, enabled: null, pack: null, linkedIssue: null, duplicates: null, readinessMode: null, readinessMinScore: null, slopMode: null, slopMinScore: null, aiReviewMode: null, aiReviewByok: null, aiReviewProvider: null, aiReviewModel: null }, + gate: { present: false, enabled: null, pack: null, linkedIssue: null, duplicates: null, readinessMode: null, readinessMinScore: null, slopMode: null, slopMinScore: null, slopAiAdvisory: null, aiReviewMode: null, aiReviewByok: null, aiReviewProvider: null, aiReviewModel: null }, settings: {}, review: { present: false, footerText: null, note: null, fields: {} }, warnings: [], @@ -688,7 +688,7 @@ describe("parseFocusManifest gate config", () => { it("parses a full gate section including the readiness block", () => { const m = parseFocusManifest({ gate: { linkedIssue: "block", duplicates: "advisory", readiness: { mode: "block", minScore: 70 } } }); expect(m.present).toBe(true); - expect(m.gate).toEqual({ present: true, enabled: null, pack: null, linkedIssue: "block", duplicates: "advisory", readinessMode: "block", readinessMinScore: 70, slopMode: null, slopMinScore: null, aiReviewMode: null, aiReviewByok: null, aiReviewProvider: null, aiReviewModel: null }); + expect(m.gate).toEqual({ present: true, enabled: null, pack: null, linkedIssue: "block", duplicates: "advisory", readinessMode: "block", readinessMinScore: 70, slopMode: null, slopMinScore: null, slopAiAdvisory: null, aiReviewMode: null, aiReviewByok: null, aiReviewProvider: null, aiReviewModel: null }); }); it("parses the gate.slop block, round-trips it, and warns on a non-mapping (#530/#532)", () => { @@ -703,6 +703,22 @@ describe("parseFocusManifest gate config", () => { expect(bad.warnings.some((w) => /gate\.slop/.test(w))).toBe(true); }); + it("parses gate.slop.aiAdvisory, round-trips it, resolves it, and warns on a non-boolean", () => { + const m = parseFocusManifest({ gate: { slop: { mode: "advisory", aiAdvisory: true } } }); + expect(m.gate.slopMode).toBe("advisory"); + expect(m.gate.slopAiAdvisory).toBe(true); + expect(gateConfigToJson(m.gate)).toMatchObject({ slop: { mode: "advisory", aiAdvisory: true } }); + + // aiAdvisory layers onto the effective settings (off by default in the DB row). + const eff = resolveEffectiveSettings({ slopGateMode: "off", slopAiAdvisory: false } as RepositorySettings, m); + expect(eff.slopGateMode).toBe("advisory"); + expect(eff.slopAiAdvisory).toBe(true); + + const bad = parseFocusManifest({ gate: { slop: { aiAdvisory: "yes please" } } }); + expect(bad.gate.slopAiAdvisory).toBeNull(); + expect(bad.warnings.some((w) => /gate\.slop\.aiAdvisory/.test(w))).toBe(true); + }); + it("parses gate.pack and ignores an unknown pack with a warning (#692)", () => { expect(parseFocusManifest({ gate: { pack: "oss-anti-slop" } }).gate.pack).toBe("oss-anti-slop"); expect(parseFocusManifest({ gate: { pack: "gittensor" } }).gate.pack).toBe("gittensor"); diff --git a/test/unit/maintainer-activation.test.ts b/test/unit/maintainer-activation.test.ts index 7533a00511..7db0851938 100644 --- a/test/unit/maintainer-activation.test.ts +++ b/test/unit/maintainer-activation.test.ts @@ -33,6 +33,7 @@ function settings(overrides: Partial = {}): RepositorySettin duplicatePrGateMode: "advisory", qualityGateMode: "advisory", slopGateMode: "off", + slopAiAdvisory: false, qualityGateMinScore: null, autoLabelEnabled: true, gittensorLabel: "gittensor", diff --git a/test/unit/policy-sanitizer.test.ts b/test/unit/policy-sanitizer.test.ts index ad68478ee9..65010737f9 100644 --- a/test/unit/policy-sanitizer.test.ts +++ b/test/unit/policy-sanitizer.test.ts @@ -67,6 +67,7 @@ function settingsFor(repoFullName: string, overrides: Partial = {}): RepositorySettin duplicatePrGateMode: "advisory", qualityGateMode: "advisory", slopGateMode: "off", + slopAiAdvisory: false, qualityGateMinScore: null, autoLabelEnabled: true, gittensorLabel: "gittensor", diff --git a/test/unit/self-dogfood-registration-pack.test.ts b/test/unit/self-dogfood-registration-pack.test.ts index 4eeaabfc64..d889044863 100644 --- a/test/unit/self-dogfood-registration-pack.test.ts +++ b/test/unit/self-dogfood-registration-pack.test.ts @@ -60,6 +60,7 @@ function settingsFor(repoFullName: string, overrides: Partial = {}): RepositorySettin duplicatePrGateMode: "advisory", qualityGateMode: "advisory", slopGateMode: "off", + slopAiAdvisory: false, qualityGateMinScore: null, autoLabelEnabled: true, gittensorLabel: "gittensor", diff --git a/test/unit/signals-coverage.test.ts b/test/unit/signals-coverage.test.ts index 9ee2543891..57d5c06e12 100644 --- a/test/unit/signals-coverage.test.ts +++ b/test/unit/signals-coverage.test.ts @@ -1530,6 +1530,7 @@ function repoSettings(repoFullName: string): RepositorySettings { duplicatePrGateMode: "advisory", qualityGateMode: "advisory", slopGateMode: "off", + slopAiAdvisory: false, qualityGateMinScore: null, autoLabelEnabled: true, gittensorLabel: "gittensor", diff --git a/test/unit/signals-v2.test.ts b/test/unit/signals-v2.test.ts index 7afbfaf0f4..b51ac601b8 100644 --- a/test/unit/signals-v2.test.ts +++ b/test/unit/signals-v2.test.ts @@ -1618,6 +1618,7 @@ describe("v2 signal builders", () => { duplicatePrGateMode: "advisory", qualityGateMode: "advisory", slopGateMode: "off", + slopAiAdvisory: false, qualityGateMinScore: null, autoLabelEnabled: true, gittensorLabel: "gittensor", diff --git a/test/unit/signals.test.ts b/test/unit/signals.test.ts index 49457efc9d..ffc108d31a 100644 --- a/test/unit/signals.test.ts +++ b/test/unit/signals.test.ts @@ -390,6 +390,7 @@ describe("world-class backend signals", () => { duplicatePrGateMode: "advisory" as const, qualityGateMode: "advisory" as const, slopGateMode: "off" as const, + slopAiAdvisory: false, qualityGateMinScore: null, autoLabelEnabled: true, gittensorLabel: "gittensor", @@ -437,6 +438,7 @@ describe("world-class backend signals", () => { duplicatePrGateMode: "advisory" as const, qualityGateMode: "advisory" as const, slopGateMode: "off" as const, + slopAiAdvisory: false, qualityGateMinScore: null, autoLabelEnabled: true, gittensorLabel: "gittensor", @@ -504,6 +506,7 @@ describe("world-class backend signals", () => { duplicatePrGateMode: "advisory", qualityGateMode: "advisory", slopGateMode: "off", + slopAiAdvisory: false, qualityGateMinScore: null, autoLabelEnabled: true, gittensorLabel: "gittensor", @@ -592,6 +595,7 @@ describe("world-class backend signals", () => { duplicatePrGateMode: "advisory", qualityGateMode: "advisory", slopGateMode: "off", + slopAiAdvisory: false, qualityGateMinScore: null, autoLabelEnabled: true, gittensorLabel: "gittensor", @@ -655,6 +659,7 @@ describe("world-class backend signals", () => { duplicatePrGateMode: "advisory", qualityGateMode: "advisory", slopGateMode: "off", + slopAiAdvisory: false, qualityGateMinScore: null, autoLabelEnabled: true, gittensorLabel: "gittensor",