diff --git a/migrations/0041_gate_outcomes.sql b/migrations/0041_gate_outcomes.sql new file mode 100644 index 0000000000..851ba6615b --- /dev/null +++ b/migrations/0041_gate_outcomes.sql @@ -0,0 +1,20 @@ +-- #554 gate false-positive telemetry: one latest gate-block row per (repo, PR). MEASUREMENT only — it lets a +-- maintainer compute a per-gate-type false-positive rate (blocked-then-merged / blocked) as the evidence +-- needed before promoting a gate from advisory to block. Privacy: repo full name + PR number + blocker +-- codes + timestamps ONLY — no actor logins, no trust/reward internals. Mirrors agent_recommendation_outcomes. +CREATE TABLE IF NOT EXISTS gate_outcomes ( + id TEXT PRIMARY KEY NOT NULL, + repo_full_name TEXT NOT NULL, + pull_number INTEGER NOT NULL, + head_sha TEXT, + blocker_codes_json TEXT NOT NULL DEFAULT '[]', + overridden INTEGER NOT NULL DEFAULT 0, + blocked_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE UNIQUE INDEX IF NOT EXISTS gate_outcomes_pr_unique + ON gate_outcomes(repo_full_name, pull_number); + +CREATE INDEX IF NOT EXISTS gate_outcomes_repo_updated_idx + ON gate_outcomes(repo_full_name, updated_at); diff --git a/src/api/routes.ts b/src/api/routes.ts index c3ced94615..2f275487dd 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -214,6 +214,7 @@ import { buildSlopAssessment, buildIssueSlopAssessment, SLOP_RUBRIC_MARKDOWN, IS import { buildPredictedGateVerdict } from "../rules/predicted-gate"; import { buildMaintainerActivationPreview, recommendedAdvisoryActivationSettings } from "../services/maintainer-activation"; import { buildRepoOutcomeCalibration } from "../services/outcome-calibration"; +import { loadGatePrecisionReport } from "../services/gate-precision"; import { buildMaintainerQualityDashboard, isMaintainerQualityDataStale } from "../services/maintainer-quality-dashboard"; import { MAX_LOCAL_SCORER_WARNING_CHARS, MAX_LOCAL_SCORER_WARNING_COUNT } from "../signals/local-scorer-diagnostics"; import { compileFocusManifestPolicy } from "../signals/focus-manifest"; @@ -1958,6 +1959,19 @@ export function createApp() { return c.json(await buildRepoOutcomeCalibration(c.env, fullName, windowDays)); }); + // #554 gate false-positive telemetry: is the gate PRECISE? Read-only measurement of blocked-then-merged + // (and overridden) per gate type — the evidence a maintainer needs before promoting a gate to block. NEVER + // adjusts a gate. Maintainer-authenticated, repo-scoped; no public route. Optional ?windowDays bounds the + // block ledger window. + app.get("/v1/repos/:owner/:repo/gate-precision", async (c) => { + const fullName = `${c.req.param("owner")}/${c.req.param("repo")}`; + const gate = await requireRepoMaintainer(c, fullName); + if (gate instanceof Response) return gate; + const windowDaysRaw = Number(c.req.query("windowDays")); + const windowDays = windowDaysRaw > 0 ? windowDaysRaw : undefined; + return c.json(await loadGatePrecisionReport(c.env, fullName, windowDays !== undefined ? { windowDays } : {})); + }); + // One-click "enable advisory mode" — turns on the gate + deterministic rules in advisory (non-blocking) // mode. Merges onto current settings so unrelated fields are preserved. app.post("/v1/repos/:owner/:repo/activation", async (c) => { @@ -4271,6 +4285,7 @@ function canSessionAccessPath(env: Env, identity: Extract { + const repoFullName = boundedString(input.repoFullName, 200); + const values = { + id: `gate:${repoFullName}#${input.pullNumber}`, + repoFullName, + pullNumber: input.pullNumber, + headSha: input.headSha ?? null, + blockerCodesJson: jsonString(input.blockerCodes), + overridden: false, + // blockedAt + updatedAt default to nowIso() via the schema `$defaultFn` on a fresh insert. + }; + await getDb(env.DB) + .insert(gateOutcomes) + .values(values) + .onConflictDoUpdate({ + target: [gateOutcomes.repoFullName, gateOutcomes.pullNumber], + // Refresh the codes/head/timestamp on a re-block; `overridden` is deliberately omitted so a true value + // is preserved. + set: { headSha: values.headSha, blockerCodesJson: values.blockerCodesJson, updatedAt: nowIso() }, + }); +} + +// Flag a gate-block row as maintainer-overridden (#538). No-op when no row exists (an override without a +// recorded block — e.g. a pre-#554 PR — has nothing to flag). +export async function markGateOutcomeOverridden(env: Env, repoFullName: string, pullNumber: number): Promise { + await getDb(env.DB) + .update(gateOutcomes) + .set({ overridden: true, updatedAt: nowIso() }) + .where(and(eq(gateOutcomes.repoFullName, boundedString(repoFullName, 200)), eq(gateOutcomes.pullNumber, pullNumber))); +} + +export async function listGateOutcomes( + env: Env, + options: { repoFullName?: string; windowDays?: number; now?: string; limit?: number } = {}, +): Promise { + const limit = clampInteger(options.limit ?? 500, 1, 5000); + const conditions = []; + if (options.repoFullName) conditions.push(eq(gateOutcomes.repoFullName, options.repoFullName)); + if (options.windowDays !== undefined) { + const windowDays = clampInteger(options.windowDays, 1, 365); + const now = options.now ?? nowIso(); + conditions.push(gte(gateOutcomes.updatedAt, new Date(Date.parse(now) - windowDays * 24 * 60 * 60 * 1000).toISOString())); + } + const rows = await getDb(env.DB) + .select() + .from(gateOutcomes) + .where(conditions.length === 0 ? undefined : and(...conditions)) + .orderBy(desc(gateOutcomes.updatedAt), gateOutcomes.id) + .limit(limit); + return rows.map(toGateOutcomeRecord); +} + export async function getAgentRecommendationOutcomeSummary( env: Env, actorLogin: string, @@ -3956,6 +4016,19 @@ function toAgentRecommendationOutcomeRecord(row: typeof agentRecommendationOutco }; } +function toGateOutcomeRecord(row: typeof gateOutcomes.$inferSelect): GateOutcomeRecord { + return { + id: row.id, + repoFullName: row.repoFullName, + pullNumber: row.pullNumber, + headSha: row.headSha, + blockerCodes: parseJson(row.blockerCodesJson, []), + overridden: row.overridden, + blockedAt: row.blockedAt, + updatedAt: row.updatedAt, + }; +} + function toInstallationHealthRecord(row: typeof installationHealth.$inferSelect): InstallationHealthRecord { return { installationId: row.installationId, diff --git a/src/db/schema.ts b/src/db/schema.ts index 86e0e355a4..4013b3599b 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -557,6 +557,30 @@ export const agentRecommendationOutcomes = sqliteTable( }), ); +// #554 gate false-positive telemetry: one latest gate-block row per (repo, PR). MEASUREMENT only — it lets a +// maintainer compute a per-gate-type false-positive rate (blocked-then-merged / blocked) before promoting a +// gate from advisory to block. Privacy: repo full name + PR number + blocker codes + timestamps ONLY — no +// actor logins, no trust/reward internals. Mirrors agentRecommendationOutcomes (dedicated ledger + upsert). +export const gateOutcomes = sqliteTable( + "gate_outcomes", + { + id: text("id").primaryKey(), + repoFullName: text("repo_full_name").notNull(), + pullNumber: integer("pull_number").notNull(), + headSha: text("head_sha"), + // JSON array of the blocker `code`s that fired (e.g. ["missing_linked_issue","slop_risk"]). + blockerCodesJson: text("blocker_codes_json").notNull().default("[]"), + // Set true when a maintainer overrides the block via #538 — the strongest false-positive signal. + overridden: integer("overridden", { mode: "boolean" }).notNull().default(false), + blockedAt: text("blocked_at").notNull().$defaultFn(() => nowIso()), + updatedAt: text("updated_at").notNull().$defaultFn(() => nowIso()), + }, + (table) => ({ + pr: uniqueIndex("gate_outcomes_pr_unique").on(table.repoFullName, table.pullNumber), + repoUpdated: index("gate_outcomes_repo_updated_idx").on(table.repoFullName, table.updatedAt), + }), +); + export const installationHealth = sqliteTable("installation_health", { installationId: integer("installation_id").primaryKey(), accountLogin: text("account_login").notNull(), diff --git a/src/queue/processors.ts b/src/queue/processors.ts index ac88aac781..58463c94e9 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -38,6 +38,8 @@ import { persistAdvisory, recordAgentCommandFeedback, recordAuditEvent, + recordGateBlockOutcome, + markGateOutcomeOverridden, recordProductUsageEvent, persistSignalSnapshot, recordWebhookEvent, @@ -1291,6 +1293,20 @@ async function maybePublishPrPublicSurface( const gatePolicy = gateCheckPolicy(settings, readiness.total, confirmedContributor, slopRisk, authorHistory); gateEvaluation = gateEnabled ? evaluateGateCheck(advisory, gatePolicy) : undefined; + // #554 gate false-positive telemetry: when the gate BLOCKS, record the block (one latest row per PR) so a + // maintainer can later compute a per-gate-type false-positive rate (blocked-then-merged / blocked). + // MEASUREMENT only — never adjusts the gate. Best-effort: a write failure must NOT abort finalization + // (mirrors the slop-assessment persist above). Privacy: codes + PR number only, no actor/trust fields. + if (gateEvaluation?.conclusion === "failure") { + const blockerCodes = gateEvaluation.blockers.map((blocker) => blocker.code); + await recordGateBlockOutcome(env, { repoFullName, pullNumber: pr.number, headSha: pr.headSha, blockerCodes }).catch(() => undefined); + await recordGithubProductUsage(env, "gate_blocked", { + repoFullName, + targetKey: `${repoFullName}#${pr.number}`, + outcome: "completed", + metadata: { blockerCodes }, + }); + } if (gateEnabled) { const gateCheckResult = await createOrUpdateGateCheckRun( env, @@ -1591,6 +1607,10 @@ async function maybeProcessGateOverrideCommand(env: Env, deliveryId: string, pay outcome: "completed", metadata: { actorKind: authorization.actorKind, headSha: advisory.headSha ?? null }, }); + // #554 gate false-positive telemetry: flag the gate-block row as maintainer-overridden — the strongest + // false-positive signal (a human explicitly judged the block wrong). Best-effort + no-op if no block was + // recorded; never affects the override outcome above. + await markGateOutcomeOverridden(env, repoFullName, pr.number).catch(() => undefined); return true; } diff --git a/src/services/gate-precision.ts b/src/services/gate-precision.ts new file mode 100644 index 0000000000..ee96f47c4e --- /dev/null +++ b/src/services/gate-precision.ts @@ -0,0 +1,144 @@ +// #554 gate false-positive telemetry: is the gate PRECISE? This is the evidence a maintainer needs before +// promoting a gate from advisory to block. +// +// MEASUREMENT only — like the #543 outcome-calibration service it NEVER auto-adjusts a gate or score (that +// would change what blocks live PRs; an owner-review decision). It only records + aggregates. +// +// A gate-block is a FALSE POSITIVE when the gate blocked a PR that turned out to be mergeable: the PR was +// blocked and later MERGED anyway. Per gate type (each blocker `code` that fired) we report blocked count, +// blocked-then-merged count, the count maintainers OVERRODE (the strongest false-positive signal — a human +// explicitly judged the block wrong), and a false-positive rate (blocked-then-merged / blocked), null below a +// min sample so a noisy rate is never reported. Inputs already exist: the gate_outcomes ledger (#554, this +// PR) records each block, and closed/merged PRs are retained on the PR row, so terminalOutcome resolves the +// same way outcome-calibration's does. +// +// Privacy: the report carries repo full name + PR-derived counts + gate-type codes ONLY — no actor logins, no +// trust/reward/credibility numbers. Internal/maintainer-authenticated; never publicly exposed. +import { listGateOutcomes, listPullRequests } from "../db/repositories"; +import type { GateOutcomeRecord, PullRequestRecord } from "../types"; +import { nowIso } from "../utils/json"; + +// Below this per-gate-type blocked sample the false-positive rate is too noisy to judge. +const MIN_SAMPLE = 5; + +export type GatePrecisionPerType = { + gateType: string; + blocked: number; + blockedThenMerged: number; + overridden: number; + falsePositiveRate: number | null; +}; + +export type GatePrecisionReport = { + repoFullName: string; + generatedAt: string; + windowDays: number | null; + perGateType: GatePrecisionPerType[]; + overall: { blocked: number; blockedThenMerged: number; falsePositiveRate: number | null }; + signals: string[]; +}; + +function round(value: number): number { + return Math.round(value * 1000) / 1000; +} + +// A PR's terminal outcome: merged if it has a merge timestamp; closed (unmerged) if its state is closed +// without one; otherwise still open (no outcome yet). Same logic as outcome-calibration. +function terminalOutcome(pr: PullRequestRecord): "merged" | "closed" | null { + if (pr.mergedAt) return "merged"; + if (pr.state === "closed") return "closed"; + return null; +} + +function sameRepo(a: string | null | undefined, b: string): boolean { + return (a ?? "").toLowerCase() === b.toLowerCase(); +} + +/** + * Per-gate-type false-positive measurement over recorded gate blocks. Pure. For each block row we look up the + * PR's terminal outcome; a blocked PR that later MERGED is a false positive. Each blocker `code` on the row + * contributes to that code's bucket (a block citing two codes counts toward both). Overridden-then-merged is + * the strongest signal — `overridden` is counted separately per type. When `options.repoFullName` is given, + * only blocks for that repo are counted. The rate is null below MIN_SAMPLE. + */ +export function buildGatePrecisionReport( + outcomes: GateOutcomeRecord[], + pullRequests: PullRequestRecord[], + options: { repoFullName?: string } = {}, +): Omit { + const repoFullName = options.repoFullName; + // Index PRs by number for an O(1) terminal-outcome lookup, scoped to the repo when one is given. + const prByNumber = new Map(); + for (const pr of pullRequests) { + if (repoFullName && !sameRepo(pr.repoFullName, repoFullName)) continue; + prByNumber.set(pr.number, pr); + } + const scoped = repoFullName ? outcomes.filter((o) => sameRepo(o.repoFullName, repoFullName)) : outcomes; + + const perType = new Map(); + let overallBlocked = 0; + let overallMerged = 0; + for (const outcome of scoped) { + const pr = prByNumber.get(outcome.pullNumber); + // A blocked PR that later MERGED is a false positive; closed/open are not (the block held or is unresolved). + const merged = pr ? terminalOutcome(pr) === "merged" : false; + overallBlocked += 1; + if (merged) overallMerged += 1; + for (const code of outcome.blockerCodes) { + const entry = perType.get(code) ?? { blocked: 0, blockedThenMerged: 0, overridden: 0 }; + entry.blocked += 1; + if (merged) entry.blockedThenMerged += 1; + if (outcome.overridden) entry.overridden += 1; + perType.set(code, entry); + } + } + + const perGateType: GatePrecisionPerType[] = [...perType.entries()] + .map(([gateType, entry]) => ({ + gateType, + blocked: entry.blocked, + blockedThenMerged: entry.blockedThenMerged, + overridden: entry.overridden, + // Null below the min sample — a 1-of-1 "false positive" is noise, not a precision signal. + falsePositiveRate: entry.blocked >= MIN_SAMPLE ? round(entry.blockedThenMerged / entry.blocked) : null, + })) + .sort((a, b) => b.blocked - a.blocked || a.gateType.localeCompare(b.gateType)); + + return { + perGateType, + overall: { + blocked: overallBlocked, + blockedThenMerged: overallMerged, + falsePositiveRate: overallBlocked >= MIN_SAMPLE ? round(overallMerged / overallBlocked) : null, + }, + signals: buildGatePrecisionSignals(perGateType, overallBlocked, overallMerged), + }; +} + +export function buildGatePrecisionSignals(perGateType: GatePrecisionPerType[], overallBlocked: number, overallMerged: number): string[] { + const signals: string[] = []; + if (overallBlocked < MIN_SAMPLE) { + signals.push(`Not enough recorded gate blocks to judge precision yet (${overallBlocked} blocked).`); + return signals; + } + signals.push(`${overallMerged} of ${overallBlocked} blocked PRs later merged (${Math.round((overallMerged / overallBlocked) * 100)}% overall false-positive rate).`); + // Surface the worst per-type rate that cleared the sample bar — the gate a maintainer should hesitate to promote to block. + const judged = perGateType.filter((type) => type.falsePositiveRate !== null); + const worst = judged.reduce((acc, type) => (acc === null || type.falsePositiveRate! > acc.falsePositiveRate! ? type : acc), null); + if (worst && worst.falsePositiveRate! > 0) { + signals.push(`Highest false-positive gate: \`${worst.gateType}\` — ${Math.round(worst.falsePositiveRate! * 100)}% of its ${worst.blocked} blocks merged anyway (${worst.overridden} overridden). Keep it advisory until this drops.`); + } else { + signals.push(`No gate type with enough sample is producing false positives — blocked PRs are staying blocked.`); + } + return signals; +} + +/** Load a repo's gate-block ledger + PRs and assemble the precision report. */ +export async function loadGatePrecisionReport(env: Env, repoFullName: string, options: { windowDays?: number } = {}): Promise { + const [pullRequests, outcomes] = await Promise.all([ + listPullRequests(env, repoFullName), + listGateOutcomes(env, { repoFullName, ...(options.windowDays !== undefined ? { windowDays: options.windowDays } : {}) }), + ]); + const report = buildGatePrecisionReport(outcomes, pullRequests, { repoFullName }); + return { repoFullName, generatedAt: nowIso(), windowDays: options.windowDays ?? null, ...report }; +} diff --git a/src/types.ts b/src/types.ts index ae9fe67abd..f904f8ddc5 100644 --- a/src/types.ts +++ b/src/types.ts @@ -856,6 +856,19 @@ export type AgentRecommendationOutcomeRepoSummary = { signal: "positive" | "negative" | "mixed" | "neutral"; }; +// #554 gate false-positive telemetry. One latest gate-block row per (repo, PR). Privacy: repo + PR number + +// blocker codes + timestamps ONLY — deliberately no actor login, no trust/reward fields. +export type GateOutcomeRecord = { + id?: string | undefined; + repoFullName: string; + pullNumber: number; + headSha?: string | null | undefined; + blockerCodes: string[]; + overridden: boolean; + blockedAt?: string | null | undefined; + updatedAt?: string | null | undefined; +}; + export type AgentRecommendationOutcomeSummary = { login: string; generatedAt: string; diff --git a/test/integration/api.test.ts b/test/integration/api.test.ts index 1ad9fc2363..ba78020c51 100644 --- a/test/integration/api.test.ts +++ b/test/integration/api.test.ts @@ -664,6 +664,22 @@ describe("api routes", () => { const calibrationNoWindow = await app.request("/v1/repos/entrius/allways-ui/outcome-calibration", { headers: apiHeaders(env) }, env); await expect(calibrationNoWindow.json()).resolves.toMatchObject({ windowDays: null }); + // #554 gate false-positive telemetry: maintainer-scoped, read-only. + const gatePrecisionUnauthenticated = await app.request("/v1/repos/entrius/allways-ui/gate-precision", {}, env); + expect(gatePrecisionUnauthenticated.status).toBe(401); + const gatePrecision = await app.request("/v1/repos/entrius/allways-ui/gate-precision?windowDays=30", { headers: apiHeaders(env) }, env); + expect(gatePrecision.status).toBe(200); + await expect(gatePrecision.json()).resolves.toMatchObject({ + repoFullName: "entrius/allways-ui", + windowDays: 30, + perGateType: expect.any(Array), + overall: { blocked: expect.any(Number), blockedThenMerged: expect.any(Number) }, + signals: expect.any(Array), + }); + // No windowDays → full window (covers the param-absent path). + const gatePrecisionNoWindow = await app.request("/v1/repos/entrius/allways-ui/gate-precision", { headers: apiHeaders(env) }, env); + await expect(gatePrecisionNoWindow.json()).resolves.toMatchObject({ windowDays: null }); + const settingsPreviewUnauthenticated = await app.request("/v1/repos/entrius/allways-ui/settings-preview", { method: "POST", body: "{}" }, env); expect(settingsPreviewUnauthenticated.status).toBe(401); diff --git a/test/unit/gate-precision.test.ts b/test/unit/gate-precision.test.ts new file mode 100644 index 0000000000..9006450d1b --- /dev/null +++ b/test/unit/gate-precision.test.ts @@ -0,0 +1,183 @@ +import { describe, expect, it } from "vitest"; +import { buildGatePrecisionReport, buildGatePrecisionSignals, loadGatePrecisionReport } from "../../src/services/gate-precision"; +import type { GatePrecisionPerType } from "../../src/services/gate-precision"; +import { + recordGateBlockOutcome, + markGateOutcomeOverridden, + listGateOutcomes, + upsertPullRequestFromGitHub, +} from "../../src/db/repositories"; +import type { GateOutcomeRecord, PullRequestRecord } from "../../src/types"; +import { createTestEnv } from "../helpers/d1"; + +// A recorded gate block for one PR, citing the given blocker codes. +function block(pullNumber: number, blockerCodes: string[], overridden = false): GateOutcomeRecord { + return { repoFullName: "owner/repo", pullNumber, blockerCodes, overridden }; +} + +// A resolved PR: `merged` → has a merge timestamp (a false positive when it was also blocked); otherwise +// closed-unmerged (the block held). `open` PRs have no terminal outcome yet. +function pr(number: number, outcome: "merged" | "closed" | "open"): PullRequestRecord { + return { + repoFullName: "owner/repo", + number, + title: `PR ${number}`, + state: outcome === "open" ? "open" : "closed", + mergedAt: outcome === "merged" ? "2026-06-01T00:00:00.000Z" : null, + labels: [], + linkedIssues: [], + }; +} + +// n blocks citing one code, `merged` of them on PRs that later merged (false positives); plus the matching PRs. +function scenario(code: string, n: number, merged: number, base: number): { blocks: GateOutcomeRecord[]; prs: PullRequestRecord[] } { + const blocks: GateOutcomeRecord[] = []; + const prs: PullRequestRecord[] = []; + for (let i = 0; i < n; i += 1) { + const num = base + i; + blocks.push(block(num, [code])); + prs.push(pr(num, i < merged ? "merged" : "closed")); + } + return { blocks, prs }; +} + +describe("buildGatePrecisionReport", () => { + it("counts a blocked-then-merged PR as a per-gate-type false positive and computes the rate", () => { + // missing_linked_issue: 6 blocks, 2 merged anyway → 2/6 false positive. slop_risk: 5 blocks, 0 merged → 0. + const a = scenario("missing_linked_issue", 6, 2, 0); + const b = scenario("slop_risk", 5, 0, 100); + const report = buildGatePrecisionReport([...a.blocks, ...b.blocks], [...a.prs, ...b.prs]); + const byType = Object.fromEntries(report.perGateType.map((t) => [t.gateType, t])); + expect(byType.missing_linked_issue).toMatchObject({ blocked: 6, blockedThenMerged: 2, falsePositiveRate: 0.333 }); + expect(byType.slop_risk).toMatchObject({ blocked: 5, blockedThenMerged: 0, falsePositiveRate: 0 }); + expect(report.overall).toMatchObject({ blocked: 11, blockedThenMerged: 2, falsePositiveRate: 0.182 }); + }); + + it("attributes a multi-code block to every cited gate type", () => { + // One block citing two codes on a merged PR → both codes get a false positive. + const report = buildGatePrecisionReport([block(1, ["missing_linked_issue", "slop_risk"])], [pr(1, "merged")]); + const byType = Object.fromEntries(report.perGateType.map((t) => [t.gateType, t])); + expect(byType.missing_linked_issue).toMatchObject({ blocked: 1, blockedThenMerged: 1 }); + expect(byType.slop_risk).toMatchObject({ blocked: 1, blockedThenMerged: 1 }); + expect(report.overall.blocked).toBe(1); // a multi-code block is ONE blocked PR overall + }); + + it("returns a null rate per type below the min sample", () => { + const report = buildGatePrecisionReport([block(1, ["x"]), block(2, ["x"])], [pr(1, "merged"), pr(2, "merged")]); + expect(report.perGateType[0]).toMatchObject({ gateType: "x", blocked: 2, blockedThenMerged: 2, falsePositiveRate: null }); + expect(report.overall.falsePositiveRate).toBeNull(); + }); + + it("excludes still-open PRs and blocks with no matching PR from the false-positive count", () => { + const blocks = [block(1, ["x"]), block(2, ["x"]), block(3, ["x"]), block(4, ["x"]), block(5, ["x"])]; + // pr 1 merged (false positive); pr 2 closed (held); pr 3 open (no outcome); pr 4 missing entirely; pr 5 merged. + const prs = [pr(1, "merged"), pr(2, "closed"), pr(3, "open"), pr(5, "merged")]; + const report = buildGatePrecisionReport(blocks, prs); + expect(report.overall).toMatchObject({ blocked: 5, blockedThenMerged: 2 }); // only the two merged ones + }); + + it("tracks overridden blocks separately as the strongest false-positive signal", () => { + const report = buildGatePrecisionReport([block(1, ["x"], true), block(2, ["x"], false)], [pr(1, "merged"), pr(2, "merged")]); + expect(report.perGateType[0]).toMatchObject({ blocked: 2, overridden: 1 }); + }); + + it("carries no actor login or trust/reward fields (privacy)", () => { + const report = buildGatePrecisionReport([block(1, ["x"])], [pr(1, "merged")]); + expect(JSON.stringify(report)).not.toMatch(/login|actor|reward|payout|trust|wallet|hotkey|credibility/i); + }); + + it("scopes to options.repoFullName — ignores blocks and PRs from other repos", () => { + const own = { repoFullName: "owner/repo", pullNumber: 1, blockerCodes: ["x"], overridden: false }; + const other = { repoFullName: "other/repo", pullNumber: 2, blockerCodes: ["x"], overridden: false }; + const otherPr: PullRequestRecord = { repoFullName: "other/repo", number: 2, title: "PR 2", state: "closed", mergedAt: "2026-06-01T00:00:00.000Z", labels: [], linkedIssues: [] }; + // A PR with a null repoFullName exercises sameRepo's nullish-coalesce guard. + const nullRepoPr: PullRequestRecord = { repoFullName: null as unknown as string, number: 3, title: "PR 3", state: "closed", mergedAt: null, labels: [], linkedIssues: [] }; + const report = buildGatePrecisionReport([own, other], [pr(1, "merged"), otherPr, nullRepoPr], { repoFullName: "owner/repo" }); + // Only owner/repo's single block counts; other/repo's block + merged PR (and the null-repo PR) are filtered out. + expect(report.overall).toMatchObject({ blocked: 1, blockedThenMerged: 1 }); + }); +}); + +describe("buildGatePrecisionSignals", () => { + const type = (gateType: string, blocked: number, blockedThenMerged: number, falsePositiveRate: number | null, overridden = 0): GatePrecisionPerType => ({ + gateType, + blocked, + blockedThenMerged, + overridden, + falsePositiveRate, + }); + + it("notes insufficient data below the min blocked sample", () => { + expect(buildGatePrecisionSignals([], 2, 1).join(" ")).toMatch(/Not enough recorded gate blocks/i); + }); + + it("reports the overall rate and the worst false-positive gate to keep advisory", () => { + const out = buildGatePrecisionSignals( + [type("missing_linked_issue", 6, 3, 0.5), type("slop_risk", 5, 0, 0)], + 11, + 3, + ).join(" "); + expect(out).toMatch(/false-positive rate/i); + expect(out).toMatch(/missing_linked_issue/); + expect(out).toMatch(/Keep it advisory/i); + }); + + it("says no gate is producing false positives when every sampled rate is zero", () => { + expect(buildGatePrecisionSignals([type("x", 6, 0, 0)], 6, 0).join(" ")).toMatch(/staying blocked/i); + }); +}); + +describe("loadGatePrecisionReport (env loader)", () => { + it("loads a repo's gate-block ledger + PRs and assembles the report; upsert/override flow round-trips", async () => { + const env = createTestEnv(); + // A blocked PR that later merged → false positive. Re-block the SAME PR: upsert keeps ONE row. + await recordGateBlockOutcome(env, { repoFullName: "owner/repo", pullNumber: 1, headSha: "sha1", blockerCodes: ["slop_risk"] }); + await recordGateBlockOutcome(env, { repoFullName: "owner/repo", pullNumber: 1, headSha: "sha2", blockerCodes: ["missing_linked_issue", "slop_risk"] }); + await markGateOutcomeOverridden(env, "owner/repo", 1); + await upsertPullRequestFromGitHub(env, "owner/repo", { number: 1, title: "merged", state: "closed", user: { login: "alice" }, merged_at: "2026-06-01T00:00:00.000Z" }); + // A blocked PR that stayed closed → not a false positive. + await recordGateBlockOutcome(env, { repoFullName: "owner/repo", pullNumber: 2, headSha: "sha3", blockerCodes: ["slop_risk"] }); + await upsertPullRequestFromGitHub(env, "owner/repo", { number: 2, title: "closed", state: "closed", user: { login: "bob" } }); + + const rows = await listGateOutcomes(env, { repoFullName: "owner/repo" }); + expect(rows).toHaveLength(2); // PR 1 upserted to one row, not duplicated + const pr1 = rows.find((row) => row.pullNumber === 1)!; + expect(pr1).toMatchObject({ headSha: "sha2", overridden: true }); + expect(pr1.blockerCodes).toEqual(["missing_linked_issue", "slop_risk"]); // latest codes won + + const report = await loadGatePrecisionReport(env, "owner/repo"); + expect(report.repoFullName).toBe("owner/repo"); + const byType = Object.fromEntries(report.perGateType.map((t) => [t.gateType, t])); + expect(byType.slop_risk).toMatchObject({ blocked: 2, blockedThenMerged: 1, overridden: 1 }); // PR1 (merged+overridden) + PR2 (closed) + expect(byType.missing_linked_issue).toMatchObject({ blocked: 1, blockedThenMerged: 1, overridden: 1 }); + expect(report.overall).toMatchObject({ blocked: 2, blockedThenMerged: 1 }); + expect(report.signals.length).toBeGreaterThan(0); + expect(JSON.stringify(report)).not.toMatch(/reward|payout|trust score|wallet|hotkey|login|actor/i); + }); + + it("preserves overridden across a later re-block (a re-block must not clear a maintainer override)", async () => { + const env = createTestEnv(); + await recordGateBlockOutcome(env, { repoFullName: "owner/repo", pullNumber: 5, blockerCodes: ["x"] }); + await markGateOutcomeOverridden(env, "owner/repo", 5); + await recordGateBlockOutcome(env, { repoFullName: "owner/repo", pullNumber: 5, blockerCodes: ["x", "y"] }); + const [row] = await listGateOutcomes(env, { repoFullName: "owner/repo" }); + expect(row).toMatchObject({ overridden: true }); + expect(row!.blockerCodes).toEqual(["x", "y"]); + }); + + it("markGateOutcomeOverridden is a no-op when no block was recorded", async () => { + const env = createTestEnv(); + await markGateOutcomeOverridden(env, "owner/repo", 99); // must not throw + expect(await listGateOutcomes(env, { repoFullName: "owner/repo" })).toHaveLength(0); + }); + + it("listGateOutcomes honors the windowDays/now/limit options and an unscoped (no-repo) listing", async () => { + const env = createTestEnv(); + await recordGateBlockOutcome(env, { repoFullName: "owner/repo", pullNumber: 7, blockerCodes: ["x"] }); + // windowDays + an explicit `now` (exercises the provided-now branch) + an explicit limit. + const windowed = await listGateOutcomes(env, { repoFullName: "owner/repo", windowDays: 7, now: "2026-06-17T00:00:00.000Z", limit: 10 }); + expect(windowed).toHaveLength(1); + // No repoFullName → unscoped listing (exercises the absent-repo branch + the empty-conditions path). + expect(await listGateOutcomes(env, {})).toHaveLength(1); + }); +});