diff --git a/migrations/0119_ai_slop_cache.sql b/migrations/0119_ai_slop_cache.sql new file mode 100644 index 0000000000..622b6405e9 --- /dev/null +++ b/migrations/0119_ai_slop_cache.sql @@ -0,0 +1,24 @@ +-- AI slop advisory cache (mirrors ai_review_cache, #74/#98/#112): runGittensoryAiSlopAdvisory makes a real +-- LLM call (up to 6 free-tier attempts, or one BYOK call) with NO caching, so every scheduled re-gate sweep +-- tick re-spends it for every open PR with slopAiAdvisory on, even at an unchanged head SHA -- confirmed in +-- production: 1,469 ai_slop_pr calls in 24h across 3 repos, 110 of them on a single PR. Unlike ai_review_cache, +-- the slop advisory has no dynamic-context dimension (no RAG/grounding/enrichment/reputation feed into it -- +-- see ai-slop.ts's AiSlopInput) and nothing analogous to a "published" GitHub artifact to protect against +-- replaying: its output is folded into the SAME advisory pass that (re)computes it, never stamped separately. +-- So this cache is unconditionally durable for a given (repo, pull, head SHA) -- no cacheable/published_at +-- cooldown columns needed, deliberately simpler than ai_review_cache. +CREATE TABLE IF NOT EXISTS ai_slop_cache ( + repo_full_name TEXT NOT NULL, + pull_number INTEGER NOT NULL, + head_sha TEXT NOT NULL, + -- Fingerprints the one input that can change independently of the head SHA: which provider produced the + -- opinion (free/default reviewer vs. a maintainer's BYOK key/model). Title/body/diff/deterministicBand are + -- all already pinned to the head SHA (see getReviewFiles/buildAiReviewDiff), so they need no fingerprinting. + input_fingerprint TEXT NOT NULL, + status TEXT NOT NULL, + band TEXT, + finding_json TEXT, + estimated_neurons INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (repo_full_name, pull_number, head_sha) +); diff --git a/src/db/repositories.ts b/src/db/repositories.ts index 53ae77a5c9..e301394a73 100644 --- a/src/db/repositories.ts +++ b/src/db/repositories.ts @@ -4371,6 +4371,58 @@ export async function markAiReviewPublished( .run(); } +/** #ai-slop-cache: the stored AI slop advisory result for (repo, pull, head SHA), or null on a miss. Mirrors + * getCachedAiReview but deliberately simpler -- see ai_slop_cache's migration doc comment for why no + * cacheable/allowNonCacheable/maxAgeMs dimension is needed here: every stored row is unconditionally durable. + * A nullish head SHA is always a miss (nothing to key on). `expectedInputFingerprint` mismatching (e.g. the + * repo turned BYOK on/off, or changed its BYOK provider/model, since this row was written) is also a miss so a + * config change can't silently replay an opinion produced under a different reviewer. */ +export async function getCachedAiSlopAdvisory( + env: Env, + repoFullName: string, + pullNumber: number, + headSha: string | null | undefined, + expectedInputFingerprint: string, +): Promise<{ status: string; band: string | null; finding: AdvisoryFinding | null; estimatedNeurons: number } | null> { + if (!headSha) return null; + const row = await env.DB + .prepare("SELECT status, band, finding_json AS findingJson, estimated_neurons AS estimatedNeurons, input_fingerprint AS inputFingerprint FROM ai_slop_cache WHERE repo_full_name = ? AND pull_number = ? AND head_sha = ?") + .bind(repoFullName, pullNumber, headSha) + .first<{ status: string; band: string | null; findingJson: string | null; estimatedNeurons: number; inputFingerprint: string }>(); + if (!row || row.inputFingerprint !== expectedInputFingerprint) return null; + return { + status: row.status, + band: row.band, + finding: parseJson(row.findingJson, null), + estimatedNeurons: row.estimatedNeurons, + }; +} + +/** #ai-slop-cache: upsert the AI slop advisory result for (repo, pull, head SHA). A nullish head SHA is a + * no-op (mirrors putCachedAiReview). Only call this for a result that actually spent the LLM call/attempts + * (status "ok") -- the caller is responsible for not caching a pre-call short-circuit (disabled/unavailable/ + * quota_exceeded), since those return before any provider call and caching them would suppress a legitimate + * retry once quota resets without having saved anything. */ +export async function putCachedAiSlopAdvisory( + env: Env, + repoFullName: string, + pullNumber: number, + headSha: string | null | undefined, + inputFingerprint: string, + result: { status: string; band: string | null; finding: AdvisoryFinding | null; estimatedNeurons: number }, +): Promise { + if (!headSha) return; + await env.DB + .prepare( + `INSERT INTO ai_slop_cache (repo_full_name, pull_number, head_sha, input_fingerprint, status, band, finding_json, estimated_neurons, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(repo_full_name, pull_number, head_sha) DO UPDATE SET + input_fingerprint = excluded.input_fingerprint, status = excluded.status, band = excluded.band, finding_json = excluded.finding_json, estimated_neurons = excluded.estimated_neurons, created_at = excluded.created_at`, + ) + .bind(repoFullName, pullNumber, headSha, inputFingerprint, result.status, result.band, jsonString(result.finding), result.estimatedNeurons, nowIso()) + .run(); +} + export async function replaceCollisionEdges(env: Env, repoFullName: string, edges: CollisionEdgeRecord[]): Promise { const db = getDb(env.DB); await env.DB.prepare("DELETE FROM collision_edges WHERE repo_full_name = ?").bind(repoFullName).run(); diff --git a/src/db/schema.ts b/src/db/schema.ts index 9da832425d..d0c5d07bef 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -1358,3 +1358,27 @@ export const aiReviewCache = sqliteTable( primary: primaryKey({ columns: [table.repoFullName, table.pullNumber, table.headSha] }), }), ); + +// AI slop advisory cache (#ai-slop-cache): mirrors aiReviewCache above but deliberately simpler -- the slop +// advisory has no dynamic-context dimension (no RAG/grounding/enrichment feed into it, see ai-slop.ts) and +// nothing analogous to a published GitHub artifact to protect against replaying, so a hit here is always +// unconditionally durable for a given (repo, pull, head SHA) -- no cacheable/published_at columns needed. +export const aiSlopCache = sqliteTable( + "ai_slop_cache", + { + repoFullName: text("repo_full_name").notNull(), + pullNumber: integer("pull_number").notNull(), + headSha: text("head_sha").notNull(), + // Fingerprints the one input that can change independently of the head SHA: which provider produced the + // opinion (free/default reviewer vs. a maintainer's BYOK key/model) -- see ai-slop-cache-input.ts. + inputFingerprint: text("input_fingerprint").notNull(), + status: text("status").notNull(), + band: text("band"), + findingJson: text("finding_json"), + estimatedNeurons: integer("estimated_neurons").notNull().default(0), + createdAt: text("created_at").notNull().$defaultFn(() => nowIso()), + }, + (table) => ({ + primary: primaryKey({ columns: [table.repoFullName, table.pullNumber, table.headSha] }), + }), +); diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 22306fa55e..5fb1d17967 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -49,6 +49,8 @@ import { countPublishedAiReviewHeads, putCachedAiReview, markAiReviewPublished, + getCachedAiSlopAdvisory, + putCachedAiSlopAdvisory, markPullRequestsRegated, markPullRequestReviewsInvalidated, markPullRequestSurfacePublished, @@ -270,6 +272,7 @@ import { queueSnapshotFromBinding, } from "../selfhost/queue-common"; import { aiReviewCacheInputFingerprint } from "../review/ai-review-cache-input"; +import { aiSlopCacheInputFingerprint } from "../review/ai-slop-cache-input"; import { AGENT_LABEL_NEEDS_REVIEW, DEFAULT_REVIEW_EVASION_LABEL, @@ -7502,16 +7505,75 @@ export async function runAiSlopForAdvisory( model: args.settings.aiReviewModel ?? storedKey.model, } : null; - const result = await runGittensoryAiSlopAdvisory(env, { - repoFullName: args.repoFullName, - prNumber: args.pr.number, - title: args.pr.title, - body: args.pr.body ?? undefined, - diff: buildAiReviewDiff(args.files), - actor: args.author, - deterministicBand: args.deterministicBand, - providerKey, + // #ai-slop-cache: the slop advisory's LLM call is fully deterministic given the same head SHA (no RAG/ + // grounding/enrichment feeds into it, unlike ai review — see ai_slop_cache's migration doc comment), so a + // repeated scheduled sweep pass at an unchanged head reuses the stored result instead of re-spending up to + // 6 free-tier attempts (or a BYOK call) on every tick — confirmed in production: 110 ai_slop_pr calls on a + // single PR in 24h at an unchanged head. The fingerprint only needs to cover which provider would answer + // (free vs. this repo's BYOK key/model); everything else the model sees is already pinned to the head SHA. + const inputFingerprint = await aiSlopCacheInputFingerprint({ + byok: Boolean(providerKey), + provider: providerKey?.provider, + model: providerKey?.model, }); + const cachedSlop = await getCachedAiSlopAdvisory(env, args.repoFullName, args.pr.number, args.advisory.headSha, inputFingerprint).catch(() => null); + let result: Awaited>; + if (cachedSlop) { + result = { status: "ok", finding: cachedSlop.finding, band: cachedSlop.band as SlopBand | null, estimatedNeurons: cachedSlop.estimatedNeurons }; + incr("gittensory_ai_slop_cache_hit_total"); + await recordAuditEvent(env, { + eventType: "github_app.ai_slop_cache_hit", + actor: args.author, + targetKey: `${args.repoFullName}#${args.pr.number}`, + outcome: "completed", + detail: "reused a stored AI slop advisory instead of re-spending an LLM call", + /* v8 ignore next -- reached only past this function's own `!args.advisory.headSha` early return, so headSha is always truthy here; the `?? null` is a type-level fallback for an unreachable branch. */ + metadata: { repoFullName: args.repoFullName, headSha: args.advisory.headSha ?? null }, + }).catch(() => undefined); + } else { + incr("gittensory_ai_slop_cache_miss_total"); + await recordAuditEvent(env, { + eventType: "github_app.ai_slop_cache_miss", + actor: args.author, + targetKey: `${args.repoFullName}#${args.pr.number}`, + outcome: "completed", + detail: "no reusable stored AI slop advisory for this head+fingerprint; running a fresh advisory", + /* v8 ignore next -- reached only past this function's own `!args.advisory.headSha` early return, so headSha is always truthy here; the `?? null` is a type-level fallback for an unreachable branch. */ + metadata: { repoFullName: args.repoFullName, headSha: args.advisory.headSha ?? null }, + }).catch(() => undefined); + result = await runGittensoryAiSlopAdvisory(env, { + repoFullName: args.repoFullName, + prNumber: args.pr.number, + title: args.pr.title, + body: args.pr.body ?? undefined, + diff: buildAiReviewDiff(args.files), + actor: args.author, + deterministicBand: args.deterministicBand, + providerKey, + }); + // Only "ok" actually spent the LLM call (free-tier attempts or a BYOK call) — disabled/unavailable/ + // quota_exceeded all short-circuit BEFORE any provider call, so caching them would suppress a legitimate + // retry once the condition clears without having saved anything. + if (result.status === "ok") { + await putCachedAiSlopAdvisory(env, args.repoFullName, args.pr.number, args.advisory.headSha, inputFingerprint, { + status: result.status, + band: result.band, + finding: result.finding, + estimatedNeurons: result.estimatedNeurons, + }).catch((error) => { + incr("gittensory_ai_slop_cache_write_error_total"); + return recordAuditEvent(env, { + eventType: "github_app.ai_slop_cache_write_error", + actor: args.author, + targetKey: `${args.repoFullName}#${args.pr.number}`, + outcome: "error", + detail: errorMessage(error), + /* v8 ignore next -- reached only past this function's own `!args.advisory.headSha` early return, so headSha is always truthy here; the `?? null` is a type-level fallback for an unreachable branch. */ + metadata: { repoFullName: args.repoFullName, headSha: args.advisory.headSha ?? null }, + }).catch(() => undefined); + }); + } + } if (result.status === "ok" && result.finding) args.advisory.findings.push(result.finding); } catch (error) { diff --git a/src/review/ai-slop-cache-input.ts b/src/review/ai-slop-cache-input.ts new file mode 100644 index 0000000000..7f8368f4bb --- /dev/null +++ b/src/review/ai-slop-cache-input.ts @@ -0,0 +1,27 @@ +import { sha256Hex } from "../utils/crypto"; + +// #ai-slop-cache: unlike ai-review-cache-input.ts (whose fingerprint spans a large, independently-mutable +// prompt-shaping surface -- reviewer plan, model overrides, path instructions, feature toggles, ...), the slop +// advisory's ONLY input that can change independently of the PR's head SHA is which provider writes the +// opinion: the free/default reviewer vs. a maintainer's BYOK key/model (see AiSlopInput in ../services/ai-slop). +// Title/body/diff/deterministicBand are all already pinned to the head SHA -- the same commit always produces +// the same diff and the same deterministic band, so none of them need fingerprinting. A repo flipping BYOK on +// or changing its BYOK provider/model must miss the cache rather than replay an opinion written under a +// different reviewer. +export const AI_SLOP_CACHE_INPUT_VERSION = "ai-slop-input:v1"; + +export type AiSlopCacheInput = { + byok: boolean; + provider: string | null | undefined; + model: string | null | undefined; +}; + +export async function aiSlopCacheInputFingerprint(input: AiSlopCacheInput): Promise { + const payload = [ + AI_SLOP_CACHE_INPUT_VERSION, + input.byok ? "1" : "0", + input.provider ?? "", + input.model ?? "", + ].join("|"); + return `${AI_SLOP_CACHE_INPUT_VERSION}:${await sha256Hex(payload)}`; +} diff --git a/src/selfhost/metrics.ts b/src/selfhost/metrics.ts index 3d63d04c05..2c46fbbebb 100644 --- a/src/selfhost/metrics.ts +++ b/src/selfhost/metrics.ts @@ -114,6 +114,9 @@ const DEFAULT_METRIC_META: readonly (readonly [string, MetricMeta])[] = [ ["gittensory_ai_review_cache_hit_total", { help: "AI review cache hits.", type: "counter" }], ["gittensory_ai_review_cache_miss_total", { help: "AI review cache misses.", type: "counter" }], ["gittensory_ai_review_cache_write_error_total", { help: "AI review cache write errors.", type: "counter" }], + ["gittensory_ai_slop_cache_hit_total", { help: "AI slop advisory cache hits.", type: "counter" }], + ["gittensory_ai_slop_cache_miss_total", { help: "AI slop advisory cache misses.", type: "counter" }], + ["gittensory_ai_slop_cache_write_error_total", { help: "AI slop advisory cache write errors.", type: "counter" }], ["gittensory_ai_review_non_cacheable_total", { help: "AI reviews skipped by cacheability rules.", type: "counter" }], ["gittensory_ai_review_force_bypass_total", { help: "AI review cache force-bypass events.", type: "counter" }], ["gittensory_ai_review_inconclusive_total", { help: "AI review inconclusive outcomes.", type: "counter" }], diff --git a/test/unit/ai-slop-cache.test.ts b/test/unit/ai-slop-cache.test.ts new file mode 100644 index 0000000000..f1153dd649 --- /dev/null +++ b/test/unit/ai-slop-cache.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it, vi } from "vitest"; +import { getCachedAiSlopAdvisory, putCachedAiSlopAdvisory } from "../../src/db/repositories"; +import { aiSlopCacheInputFingerprint } from "../../src/review/ai-slop-cache-input"; +import { createTestEnv } from "../helpers/d1"; + +const fp = () => aiSlopCacheInputFingerprint({ byok: false, provider: null, model: null }); + +describe("AI slop advisory cache (#ai-slop-cache)", () => { + it("misses on a nullish head SHA (read returns null; write is a no-op)", async () => { + const env = createTestEnv(); + const fingerprint = await fp(); + expect(await getCachedAiSlopAdvisory(env, "o/r", 1, null, fingerprint)).toBeNull(); + expect(await getCachedAiSlopAdvisory(env, "o/r", 1, undefined, fingerprint)).toBeNull(); + await putCachedAiSlopAdvisory(env, "o/r", 1, null, fingerprint, { status: "ok", band: null, finding: null, estimatedNeurons: 5 }); // no-op, no throw + expect(await getCachedAiSlopAdvisory(env, "o/r", 1, "sha", fingerprint)).toBeNull(); // nothing was stored + }); + + it("reuses a stored advisory ONLY on the same (repo, pull, head SHA)", async () => { + const env = createTestEnv(); + const fingerprint = await fp(); + await putCachedAiSlopAdvisory(env, "o/r", 7, "sha1", fingerprint, { status: "ok", band: "elevated", finding: null, estimatedNeurons: 12 }); + expect(await getCachedAiSlopAdvisory(env, "o/r", 7, "sha1", fingerprint)).toEqual({ status: "ok", band: "elevated", finding: null, estimatedNeurons: 12 }); + expect(await getCachedAiSlopAdvisory(env, "o/r", 7, "sha2", fingerprint)).toBeNull(); // new head SHA → miss + expect(await getCachedAiSlopAdvisory(env, "o/r", 8, "sha1", fingerprint)).toBeNull(); // different PR → miss + expect(await getCachedAiSlopAdvisory(env, "o/r2", 7, "sha1", fingerprint)).toBeNull(); // different repo → miss + }); + + it("misses when the input fingerprint does not match (e.g. BYOK toggled on/off since the row was written)", async () => { + const env = createTestEnv(); + const freeFingerprint = await aiSlopCacheInputFingerprint({ byok: false, provider: null, model: null }); + const byokFingerprint = await aiSlopCacheInputFingerprint({ byok: true, provider: "anthropic", model: "claude-sonnet-5" }); + expect(freeFingerprint).not.toBe(byokFingerprint); + + await putCachedAiSlopAdvisory(env, "o/r", 9, "sha1", freeFingerprint, { status: "ok", band: "low", finding: null, estimatedNeurons: 6 }); + expect(await getCachedAiSlopAdvisory(env, "o/r", 9, "sha1", byokFingerprint)).toBeNull(); + expect(await getCachedAiSlopAdvisory(env, "o/r", 9, "sha1", freeFingerprint)).toEqual({ status: "ok", band: "low", finding: null, estimatedNeurons: 6 }); + }); + + it("upserts — a re-run at the same key replaces the stored advisory", async () => { + const env = createTestEnv(); + const fingerprint = await fp(); + await putCachedAiSlopAdvisory(env, "o/r", 10, "sha1", fingerprint, { status: "ok", band: "clean", finding: null, estimatedNeurons: 3 }); + await putCachedAiSlopAdvisory(env, "o/r", 10, "sha1", fingerprint, { + status: "ok", + band: "high", + finding: { code: "ai_slop_advisory", title: "t", severity: "warning", detail: "d" }, + estimatedNeurons: 9, + }); + expect(await getCachedAiSlopAdvisory(env, "o/r", 10, "sha1", fingerprint)).toEqual({ + status: "ok", + band: "high", + finding: { code: "ai_slop_advisory", title: "t", severity: "warning", detail: "d" }, + estimatedNeurons: 9, + }); + }); + + it("round-trips a null band and a null finding (a clean-band advisory with no surfaced finding)", async () => { + const env = createTestEnv(); + const fingerprint = await fp(); + await putCachedAiSlopAdvisory(env, "o/r", 11, "sha1", fingerprint, { status: "ok", band: "clean", finding: null, estimatedNeurons: 6 }); + expect(await getCachedAiSlopAdvisory(env, "o/r", 11, "sha1", fingerprint)).toEqual({ status: "ok", band: "clean", finding: null, estimatedNeurons: 6 }); + }); + + it("stores an ISO created_at value on insert and conflict update", async () => { + const env = createTestEnv(); + const fingerprint = await fp(); + + vi.useFakeTimers(); + try { + vi.setSystemTime(new Date("2026-07-06T09:00:00.123Z")); + await putCachedAiSlopAdvisory(env, "o/r", 12, "sha1", fingerprint, { status: "ok", band: "low", finding: null, estimatedNeurons: 6 }); + const inserted = await env.DB.prepare("SELECT created_at AS createdAt FROM ai_slop_cache WHERE repo_full_name = ? AND pull_number = ? AND head_sha = ?") + .bind("o/r", 12, "sha1") + .first<{ createdAt: string }>(); + expect(inserted?.createdAt).toBe("2026-07-06T09:00:00.123Z"); + + vi.setSystemTime(new Date("2026-07-06T09:05:00.456Z")); + await putCachedAiSlopAdvisory(env, "o/r", 12, "sha1", fingerprint, { status: "ok", band: "high", finding: null, estimatedNeurons: 9 }); + const updated = await env.DB.prepare("SELECT created_at AS createdAt FROM ai_slop_cache WHERE repo_full_name = ? AND pull_number = ? AND head_sha = ?") + .bind("o/r", 12, "sha1") + .first<{ createdAt: string }>(); + expect(updated?.createdAt).toBe("2026-07-06T09:05:00.456Z"); + } finally { + vi.useRealTimers(); + } + }); +}); + +describe("aiSlopCacheInputFingerprint", () => { + it("is stable for the same input", async () => { + const a = await aiSlopCacheInputFingerprint({ byok: false, provider: null, model: null }); + const b = await aiSlopCacheInputFingerprint({ byok: false, provider: null, model: null }); + expect(a).toBe(b); + }); + + it("differs when byok flips", async () => { + const free = await aiSlopCacheInputFingerprint({ byok: false, provider: null, model: null }); + const byok = await aiSlopCacheInputFingerprint({ byok: true, provider: null, model: null }); + expect(free).not.toBe(byok); + }); + + it("differs when the BYOK provider changes", async () => { + const anthropic = await aiSlopCacheInputFingerprint({ byok: true, provider: "anthropic", model: null }); + const openai = await aiSlopCacheInputFingerprint({ byok: true, provider: "openai", model: null }); + expect(anthropic).not.toBe(openai); + }); + + it("differs when the BYOK model changes", async () => { + const sonnet = await aiSlopCacheInputFingerprint({ byok: true, provider: "anthropic", model: "claude-sonnet-5" }); + const opus = await aiSlopCacheInputFingerprint({ byok: true, provider: "anthropic", model: "claude-opus-5" }); + expect(sonnet).not.toBe(opus); + }); + + it("treats a nullish provider/model the same as an absent one", async () => { + const withUndefined = await aiSlopCacheInputFingerprint({ byok: false, provider: undefined, model: undefined }); + const withNull = await aiSlopCacheInputFingerprint({ byok: false, provider: null, model: null }); + expect(withUndefined).toBe(withNull); + }); +}); diff --git a/test/unit/ai-slop.test.ts b/test/unit/ai-slop.test.ts index 3c94be157d..3a5ebdf75e 100644 --- a/test/unit/ai-slop.test.ts +++ b/test/unit/ai-slop.test.ts @@ -7,7 +7,8 @@ import { } from "../../src/services/ai-slop"; import { evaluateGateCheck } from "../../src/rules/advisory"; import { runAiSlopForAdvisory } from "../../src/queue/processors"; -import { recordAiUsageEvent, upsertRepositoryAiKey } from "../../src/db/repositories"; +import { getCachedAiSlopAdvisory, putCachedAiSlopAdvisory, recordAiUsageEvent, upsertRepositoryAiKey } from "../../src/db/repositories"; +import { aiSlopCacheInputFingerprint } from "../../src/review/ai-slop-cache-input"; import type { Advisory, PullRequestFileRecord, RepositorySettings } from "../../src/types"; import { createTestEnv } from "../helpers/d1"; @@ -474,4 +475,176 @@ describe("runAiSlopForAdvisory (processor wiring)", () => { expect(fetchMock).not.toHaveBeenCalled(); expect(run).not.toHaveBeenCalled(); }); + + describe("AI slop advisory cache wiring (#ai-slop-cache)", () => { + it("reuses a stored advisory for an unchanged head SHA instead of calling the model again", async () => { + const run = vi.fn(async () => ({ response: slopJson({ band: "high" }) })); + const env = enabledEnv(run); + const fingerprint = await aiSlopCacheInputFingerprint({ byok: false, provider: null, model: null }); + await putCachedAiSlopAdvisory(env, "acme/widgets", 3, "sha3", fingerprint, { + status: "ok", + band: "high", + finding: { code: AI_SLOP_FINDING_CODE, title: "cached finding", severity: "warning", detail: "from cache" }, + estimatedNeurons: 42, + }); + const adv = advisory(); + await runAiSlopForAdvisory(env, { settings: noByok, advisory: adv, repoFullName: "acme/widgets", pr, author: "alice", files, deterministicBand: "high", confirmedContributor: true }); + + expect(run).not.toHaveBeenCalled(); // no LLM call spent on the cache hit + expect(adv.findings).toEqual([{ code: AI_SLOP_FINDING_CODE, title: "cached finding", severity: "warning", detail: "from cache" }]); + }); + + it("swallows a throwing audit-event write on a cache hit (fail-safe, the finding still reaches the advisory)", async () => { + const run = vi.fn(); + const env = enabledEnv(run); + const fingerprint = await aiSlopCacheInputFingerprint({ byok: false, provider: null, model: null }); + await putCachedAiSlopAdvisory(env, "acme/widgets", 3, "sha3", fingerprint, { + status: "ok", + band: "high", + finding: { code: AI_SLOP_FINDING_CODE, title: "cached finding", severity: "warning", detail: "from cache" }, + estimatedNeurons: 42, + }); + const repositoriesModule = await import("../../src/db/repositories"); + const auditSpy = vi.spyOn(repositoriesModule, "recordAuditEvent").mockRejectedValueOnce(new Error("D1 audit write error")); + const adv = advisory(); + await runAiSlopForAdvisory(env, { settings: noByok, advisory: adv, repoFullName: "acme/widgets", pr, author: "alice", files, deterministicBand: "high", confirmedContributor: true }); + + expect(run).not.toHaveBeenCalled(); // still a cache hit despite the audit-write failure + expect(adv.findings).toEqual([{ code: AI_SLOP_FINDING_CODE, title: "cached finding", severity: "warning", detail: "from cache" }]); + auditSpy.mockRestore(); + }); + + it("passes a nullish PR body through as undefined (not null) to the fresh advisory call on a cache miss", async () => { + const run = vi.fn(async (_model: string, options: { messages: { content: string }[] }) => { + expect(options.messages[1]?.content).toContain("Description: (none)"); // buildUserPrompt's no-body branch + return { response: slopJson({ band: "elevated" }) }; + }); + const env = enabledEnv(run); + const adv = advisory(); + await runAiSlopForAdvisory(env, { + settings: noByok, + advisory: adv, + repoFullName: "acme/widgets", + pr: { number: 3, title: "Tidy" }, // no `body` key at all — exercises the `?? undefined` nullish arm + author: "alice", + files, + deterministicBand: "elevated", + confirmedContributor: true, + }); + expect(run).toHaveBeenCalledTimes(1); + expect(adv.findings.map((f) => f.code)).toEqual([AI_SLOP_FINDING_CODE]); + }); + + it("misses the cache and writes back a fresh 'ok' result so the NEXT call at this head is a hit", async () => { + const run = vi.fn(async () => ({ response: slopJson({ band: "elevated" }) })); + const env = enabledEnv(run); + const adv = advisory(); + await runAiSlopForAdvisory(env, { settings: noByok, advisory: adv, repoFullName: "acme/widgets", pr, author: "alice", files, deterministicBand: "elevated", confirmedContributor: true }); + expect(run).toHaveBeenCalledTimes(1); // fresh call on the miss + + const fingerprint = await aiSlopCacheInputFingerprint({ byok: false, provider: null, model: null }); + const cached = await getCachedAiSlopAdvisory(env, "acme/widgets", 3, "sha3", fingerprint); + expect(cached).toMatchObject({ status: "ok", band: "elevated" }); + + // A second call for the SAME head must now reuse the cache, not spend another LLM call. + const adv2 = advisory(); + await runAiSlopForAdvisory(env, { settings: noByok, advisory: adv2, repoFullName: "acme/widgets", pr, author: "alice", files, deterministicBand: "elevated", confirmedContributor: true }); + expect(run).toHaveBeenCalledTimes(1); // still 1 — the second pass was a cache hit + expect(adv2.findings).toEqual(adv.findings); + }); + + it("does not cache a quota_exceeded short-circuit — a later call still tries the model once quota allows", async () => { + const run = vi.fn(async () => ({ response: slopJson({ band: "elevated" }) })); + const budgetedEnv = createTestEnv({ AI: { run } as unknown as Ai, AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true", AI_DAILY_NEURON_BUDGET: "1" }); + const adv = advisory(); + await runAiSlopForAdvisory(budgetedEnv, { settings: noByok, advisory: adv, repoFullName: "acme/widgets", pr, author: "alice", files, deterministicBand: "elevated", confirmedContributor: true }); + expect(run).not.toHaveBeenCalled(); // quota_exceeded short-circuits before any model call + expect(adv.findings).toEqual([]); + + const fingerprint = await aiSlopCacheInputFingerprint({ byok: false, provider: null, model: null }); + expect(await getCachedAiSlopAdvisory(budgetedEnv, "acme/widgets", 3, "sha3", fingerprint)).toBeNull(); // nothing was persisted + + // Same head, budget now available — must still attempt the model instead of replaying a quota miss. + const richEnv = createTestEnv({ AI: { run } as unknown as Ai, AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true", AI_DAILY_NEURON_BUDGET: "100000" }); + const adv2 = advisory(); + await runAiSlopForAdvisory(richEnv, { settings: noByok, advisory: adv2, repoFullName: "acme/widgets", pr, author: "alice", files, deterministicBand: "elevated", confirmedContributor: true }); + expect(run).toHaveBeenCalledTimes(1); + }); + + it("misses the cache when BYOK is toggled on since the row was written (fresh reviewer → fresh call)", async () => { + const run = vi.fn(async () => ({ response: slopJson({ band: "clean", rationale: "genuine", signals: [] }) })); // Workers AI must not be used once BYOK is on + const env = createTestEnv({ + AI: { run } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + TOKEN_ENCRYPTION_SECRET: "ai-slop-byok-cache-test-encryption-secret-32", + }); + const freeFingerprint = await aiSlopCacheInputFingerprint({ byok: false, provider: null, model: null }); + await putCachedAiSlopAdvisory(env, "acme/widgets", 3, "sha3", freeFingerprint, { + status: "ok", + band: "high", + finding: { code: AI_SLOP_FINDING_CODE, title: "stale free-tier finding", severity: "warning", detail: "d" }, + estimatedNeurons: 12, + }); + await upsertRepositoryAiKey(env, { repoFullName: "acme/widgets", provider: "anthropic", key: "sk-ant-byok-cache-9999", model: null }); + const fetchMock = vi.fn(async () => new Response(JSON.stringify({ content: [{ type: "text", text: slopJson({ band: "high" }) }] }), { status: 200 })); + vi.stubGlobal("fetch", fetchMock); + + const adv = advisory(); + await runAiSlopForAdvisory(env, { settings: { aiReviewByok: true } as RepositorySettings, advisory: adv, repoFullName: "acme/widgets", pr, author: "alice", files, deterministicBand: "elevated", confirmedContributor: true }); + + // A fresh BYOK call was made (not the stale free-tier cache row, and not Workers AI). + expect(fetchMock).toHaveBeenCalled(); + expect(run).not.toHaveBeenCalled(); + expect(adv.findings.map((f) => f.title)).not.toContain("stale free-tier finding"); + }); + + it("is fail-safe when the cache READ throws — falls through to a fresh model call, never blocks the advisory", async () => { + const run = vi.fn(async () => ({ response: slopJson({ band: "elevated" }) })); + const env = enabledEnv(run); + const repositoriesModule = await import("../../src/db/repositories"); + const readSpy = vi.spyOn(repositoriesModule, "getCachedAiSlopAdvisory").mockRejectedValueOnce(new Error("D1 read error")); + const adv = advisory(); + await runAiSlopForAdvisory(env, { settings: noByok, advisory: adv, repoFullName: "acme/widgets", pr, author: "alice", files, deterministicBand: "elevated", confirmedContributor: true }); + expect(run).toHaveBeenCalledTimes(1); // degraded to a miss instead of throwing + expect(adv.findings.map((f) => f.code)).toEqual([AI_SLOP_FINDING_CODE]); + readSpy.mockRestore(); + }); + + it("is fail-safe when the cache WRITE throws — the fresh finding still reaches the advisory, and the failure is audited", async () => { + const run = vi.fn(async () => ({ response: slopJson({ band: "elevated" }) })); + const env = enabledEnv(run); + const repositoriesModule = await import("../../src/db/repositories"); + const writeSpy = vi.spyOn(repositoriesModule, "putCachedAiSlopAdvisory").mockRejectedValueOnce(new Error("D1 write error")); + const adv = advisory(); + await runAiSlopForAdvisory(env, { settings: noByok, advisory: adv, repoFullName: "acme/widgets", pr, author: "alice", files, deterministicBand: "elevated", confirmedContributor: true }); + expect(adv.findings.map((f) => f.code)).toEqual([AI_SLOP_FINDING_CODE]); // swallowed, not thrown + writeSpy.mockRestore(); + + const audit = await env.DB.prepare("select outcome, detail from audit_events where event_type = ? and target_key = ?") + .bind("github_app.ai_slop_cache_write_error", "acme/widgets#3") + .first<{ outcome: string; detail: string }>(); + expect(audit?.outcome).toBe("error"); + expect(audit?.detail).toContain("D1 write error"); + }); + + it("is fail-safe when BOTH the cache WRITE and its own error-audit write throw (doubly-nested fail-safe)", async () => { + const run = vi.fn(async () => ({ response: slopJson({ band: "elevated" }) })); + const env = enabledEnv(run); + const repositoriesModule = await import("../../src/db/repositories"); + const writeSpy = vi.spyOn(repositoriesModule, "putCachedAiSlopAdvisory").mockRejectedValueOnce(new Error("D1 write error")); + const auditSpy = vi.spyOn(repositoriesModule, "recordAuditEvent").mockImplementation(async (_env, event) => { + if (event.eventType === "github_app.ai_slop_cache_write_error") throw new Error("D1 audit write error"); + return undefined; + }); + const adv = advisory(); + await expect( + runAiSlopForAdvisory(env, { settings: noByok, advisory: adv, repoFullName: "acme/widgets", pr, author: "alice", files, deterministicBand: "elevated", confirmedContributor: true }), + ).resolves.toBeUndefined(); // never throws, even with both the cache write AND its own audit write failing + expect(adv.findings.map((f) => f.code)).toEqual([AI_SLOP_FINDING_CODE]); + writeSpy.mockRestore(); + auditSpy.mockRestore(); + }); + }); }); diff --git a/test/unit/schema-timestamp-defaults.test.ts b/test/unit/schema-timestamp-defaults.test.ts index aeec02d8b6..efa3a11e9a 100644 --- a/test/unit/schema-timestamp-defaults.test.ts +++ b/test/unit/schema-timestamp-defaults.test.ts @@ -1,7 +1,7 @@ import { eq } from "drizzle-orm"; import { describe, expect, it } from "vitest"; import { getDb } from "../../src/db/client"; -import { aiReviewCache, orbRelayPending, repositorySettings, webhookEvents } from "../../src/db/schema"; +import { aiReviewCache, aiSlopCache, orbRelayPending, repositorySettings, webhookEvents } from "../../src/db/schema"; import { createTestEnv } from "../helpers/d1"; const ISO = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/; @@ -77,4 +77,19 @@ describe("timestamp column defaults", () => { expect(row?.createdAt).toMatch(ISO); expect(row?.createdAt).not.toBe("CURRENT_TIMESTAMP"); }); + + it("applies the AI slop advisory cache createdAt default on omit (#ai-slop-cache)", async () => { + const env = createTestEnv(); + const db = getDb(env.DB); + await db.insert(aiSlopCache).values({ + repoFullName: "acme/widgets", + pullNumber: 2, + headSha: "sha", + inputFingerprint: "fp-v1", + status: "ok", + }); + const [row] = await db.select().from(aiSlopCache).where(eq(aiSlopCache.repoFullName, "acme/widgets")).limit(1); + expect(row?.createdAt).toMatch(ISO); + expect(row?.createdAt).not.toBe("CURRENT_TIMESTAMP"); + }); });