feat(telemetry): gate false-positive measurement loop (#554)

migrations/0041_gate_outcomes.sql

@@ -0,0 +1,20 @@
+-- #554 gate false-positive telemetry: one latest gate-block row per (repo, PR). MEASUREMENT only — it lets a
+-- maintainer compute a per-gate-type false-positive rate (blocked-then-merged / blocked) as the evidence
+-- needed before promoting a gate from advisory to block. Privacy: repo full name + PR number + blocker
+-- codes + timestamps ONLY — no actor logins, no trust/reward internals. Mirrors agent_recommendation_outcomes.
+CREATE TABLE IF NOT EXISTS gate_outcomes (
+  id TEXT PRIMARY KEY NOT NULL,
+  repo_full_name TEXT NOT NULL,
+  pull_number INTEGER NOT NULL,
+  head_sha TEXT,
+  blocker_codes_json TEXT NOT NULL DEFAULT '[]',
+  overridden INTEGER NOT NULL DEFAULT 0,
+  blocked_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS gate_outcomes_pr_unique
+  ON gate_outcomes(repo_full_name, pull_number);
+
+CREATE INDEX IF NOT EXISTS gate_outcomes_repo_updated_idx
+  ON gate_outcomes(repo_full_name, updated_at);

src/api/routes.ts

@@ -1,4 +1,4 @@
 import { Hono, type Context } from "hono";
 import { z } from "zod";
 import { analyzePRQueue, type AuthorRole, type ChecksStatus } from "../queue-intelligence";
 import { completeGitHubWebOAuth, createSessionFromGitHubToken, pollGitHubDeviceFlow, startGitHubDeviceFlow, startGitHubWebOAuth } from "../auth/github-oauth";
@@ -214,6 +214,7 @@
 import { buildPredictedGateVerdict } from "../rules/predicted-gate";
 import { buildMaintainerActivationPreview, recommendedAdvisoryActivationSettings } from "../services/maintainer-activation";
 import { buildRepoOutcomeCalibration } from "../services/outcome-calibration";
+import { loadGatePrecisionReport } from "../services/gate-precision";
 import { buildMaintainerQualityDashboard, isMaintainerQualityDataStale } from "../services/maintainer-quality-dashboard";
 import { MAX_LOCAL_SCORER_WARNING_CHARS, MAX_LOCAL_SCORER_WARNING_COUNT } from "../signals/local-scorer-diagnostics";
 import { compileFocusManifestPolicy } from "../signals/focus-manifest";
@@ -1958,6 +1959,19 @@
     return c.json(await buildRepoOutcomeCalibration(c.env, fullName, windowDays));
   });
 
+  // #554 gate false-positive telemetry: is the gate PRECISE? Read-only measurement of blocked-then-merged
+  // (and overridden) per gate type — the evidence a maintainer needs before promoting a gate to block. NEVER
+  // adjusts a gate. Maintainer-authenticated, repo-scoped; no public route. Optional ?windowDays bounds the
+  // block ledger window.
+  app.get("/v1/repos/:owner/:repo/gate-precision", async (c) => {
+    const fullName = `${c.req.param("owner")}/${c.req.param("repo")}`;
+    const gate = await requireRepoMaintainer(c, fullName);
+    if (gate instanceof Response) return gate;
+    const windowDaysRaw = Number(c.req.query("windowDays"));
+    const windowDays = windowDaysRaw > 0 ? windowDaysRaw : undefined;
+    return c.json(await loadGatePrecisionReport(c.env, fullName, windowDays !== undefined ? { windowDays } : {}));
+  });
+
   // One-click "enable advisory mode" — turns on the gate + deterministic rules in advisory (non-blocking)
   // mode. Merges onto current settings so unrelated fields are preserved.
   app.post("/v1/repos/:owner/:repo/activation", async (c) => {
@@ -4271,6 +4285,7 @@
   if (isRepoSettingsPath(path)) return true;
   if (isRepoActivationPath(path)) return true;
   if (isRepoOutcomeCalibrationPath(path)) return true;
+  if (isRepoGatePrecisionPath(path)) return true;
   if (isRepoSettingsPreviewPath(path)) return true;
   if (isRepoOnboardingPackPreviewPath(path)) return true;
   if (isRepoFocusManifestPath(path)) return true;
@@ -4298,6 +4313,10 @@
   return /^\/v1\/repos\/[^/]+\/[^/]+\/outcome-calibration$/.test(path);
 }
 
+function isRepoGatePrecisionPath(path: string): boolean {
+  return /^\/v1\/repos\/[^/]+\/[^/]+\/gate-precision$/.test(path);
+}
+
 function isRepoSettingsPreviewPath(path: string): boolean {
   return /^\/v1\/repos\/[^/]+\/[^/]+\/settings-preview$/.test(path);
 }

src/db/repositories.ts

@@ -1,4 +1,4 @@
 import { and, desc, eq, gte, inArray, not, or, sql, type SQL } from "drizzle-orm";
 import { getDb } from "./client";
 import {
   advisories,
@@ -19,6 +19,7 @@
   contributorScoringProfiles,
   contributors,
   digestSubscriptions,
+  gateOutcomes,
   githubAgentCommandAnswers,
   githubAgentCommandFeedback,
   installationHealth,
@@ -69,6 +70,7 @@
   AgentRecommendationOutcomeState,
   AgentRecommendationOutcomeSummary,
   AgentRecommendationOutcomeTargetType,
+  GateOutcomeRecord,
   AgentMode,
   AgentRunRecord,
   AgentRunStatus,
@@ -3194,6 +3196,64 @@
   return rows.map(toAgentRecommendationOutcomeRecord);
 }
 
+// #554 gate false-positive telemetry. Upsert the latest gate-block row for a (repo, PR): one row per PR so a
+// re-evaluation overwrites the prior block. Preserves `overridden` once set true (a later block must not
+// clear a maintainer's override). Privacy: never stores actor or trust/reward fields.
+export async function recordGateBlockOutcome(
+  env: Env,
+  input: { repoFullName: string; pullNumber: number; headSha?: string | null | undefined; blockerCodes: string[] },
+): Promise<void> {
+  const repoFullName = boundedString(input.repoFullName, 200);
+  const values = {
+    id: `gate:${repoFullName}#${input.pullNumber}`,
+    repoFullName,
+    pullNumber: input.pullNumber,
+    headSha: input.headSha ?? null,
+    blockerCodesJson: jsonString(input.blockerCodes),
+    overridden: false,
+    // blockedAt + updatedAt default to nowIso() via the schema `$defaultFn` on a fresh insert.
+  };
+  await getDb(env.DB)
+    .insert(gateOutcomes)
+    .values(values)
+    .onConflictDoUpdate({
+      target: [gateOutcomes.repoFullName, gateOutcomes.pullNumber],
+      // Refresh the codes/head/timestamp on a re-block; `overridden` is deliberately omitted so a true value
+      // is preserved.
+      set: { headSha: values.headSha, blockerCodesJson: values.blockerCodesJson, updatedAt: nowIso() },
+    });
+}
+
+// Flag a gate-block row as maintainer-overridden (#538). No-op when no row exists (an override without a
+// recorded block — e.g. a pre-#554 PR — has nothing to flag).
+export async function markGateOutcomeOverridden(env: Env, repoFullName: string, pullNumber: number): Promise<void> {
+  await getDb(env.DB)
+    .update(gateOutcomes)
+    .set({ overridden: true, updatedAt: nowIso() })
+    .where(and(eq(gateOutcomes.repoFullName, boundedString(repoFullName, 200)), eq(gateOutcomes.pullNumber, pullNumber)));
+}
+
+export async function listGateOutcomes(
+  env: Env,
+  options: { repoFullName?: string; windowDays?: number; now?: string; limit?: number } = {},
+): Promise<GateOutcomeRecord[]> {
+  const limit = clampInteger(options.limit ?? 500, 1, 5000);
+  const conditions = [];
+  if (options.repoFullName) conditions.push(eq(gateOutcomes.repoFullName, options.repoFullName));
+  if (options.windowDays !== undefined) {
+    const windowDays = clampInteger(options.windowDays, 1, 365);
+    const now = options.now ?? nowIso();
+    conditions.push(gte(gateOutcomes.updatedAt, new Date(Date.parse(now) - windowDays * 24 * 60 * 60 * 1000).toISOString()));
+  }
+  const rows = await getDb(env.DB)
+    .select()
+    .from(gateOutcomes)
+    .where(conditions.length === 0 ? undefined : and(...conditions))
+    .orderBy(desc(gateOutcomes.updatedAt), gateOutcomes.id)
+    .limit(limit);
+  return rows.map(toGateOutcomeRecord);
+}
+
 export async function getAgentRecommendationOutcomeSummary(
   env: Env,
   actorLogin: string,
@@ -3956,6 +4016,19 @@
   };
 }
 
+function toGateOutcomeRecord(row: typeof gateOutcomes.$inferSelect): GateOutcomeRecord {
+  return {
+    id: row.id,
+    repoFullName: row.repoFullName,
+    pullNumber: row.pullNumber,
+    headSha: row.headSha,
+    blockerCodes: parseJson<string[]>(row.blockerCodesJson, []),
+    overridden: row.overridden,
+    blockedAt: row.blockedAt,
+    updatedAt: row.updatedAt,
+  };
+}
+
 function toInstallationHealthRecord(row: typeof installationHealth.$inferSelect): InstallationHealthRecord {
   return {
     installationId: row.installationId,

src/db/schema.ts

@@ -1,4 +1,4 @@
 import { index, integer, real, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core";
 // Timestamp columns use a drizzle $defaultFn so an insert that omits the column gets a real ISO-8601
 // timestamp. A static `.default("CURRENT_TIMESTAMP")` would make drizzle inject the literal STRING
 // "CURRENT_TIMESTAMP" (it applies static defaults client-side, never reaching SQLite's CURRENT_TIMESTAMP),
@@ -557,6 +557,30 @@
   }),
 );
 
+// #554 gate false-positive telemetry: one latest gate-block row per (repo, PR). MEASUREMENT only — it lets a
+// maintainer compute a per-gate-type false-positive rate (blocked-then-merged / blocked) before promoting a
+// gate from advisory to block. Privacy: repo full name + PR number + blocker codes + timestamps ONLY — no
+// actor logins, no trust/reward internals. Mirrors agentRecommendationOutcomes (dedicated ledger + upsert).
+export const gateOutcomes = sqliteTable(
+  "gate_outcomes",
+  {
+    id: text("id").primaryKey(),
+    repoFullName: text("repo_full_name").notNull(),
+    pullNumber: integer("pull_number").notNull(),
+    headSha: text("head_sha"),
+    // JSON array of the blocker `code`s that fired (e.g. ["missing_linked_issue","slop_risk"]).
+    blockerCodesJson: text("blocker_codes_json").notNull().default("[]"),
+    // Set true when a maintainer overrides the block via #538 — the strongest false-positive signal.
+    overridden: integer("overridden", { mode: "boolean" }).notNull().default(false),
+    blockedAt: text("blocked_at").notNull().$defaultFn(() => nowIso()),
+    updatedAt: text("updated_at").notNull().$defaultFn(() => nowIso()),
+  },
+  (table) => ({
+    pr: uniqueIndex("gate_outcomes_pr_unique").on(table.repoFullName, table.pullNumber),
+    repoUpdated: index("gate_outcomes_repo_updated_idx").on(table.repoFullName, table.updatedAt),
+  }),
+);
+
 export const installationHealth = sqliteTable("installation_health", {
   installationId: integer("installation_id").primaryKey(),
   accountLogin: text("account_login").notNull(),

src/queue/processors.ts

@@ -1,4 +1,4 @@
 import {
   countOpenIssues,
   countOpenPullRequests,
   getAgentCommandAnswer,
@@ -38,6 +38,8 @@
   persistAdvisory,
   recordAgentCommandFeedback,
   recordAuditEvent,
+  recordGateBlockOutcome,
+  markGateOutcomeOverridden,
   recordProductUsageEvent,
   persistSignalSnapshot,
   recordWebhookEvent,
@@ -1291,6 +1293,20 @@
 
     const gatePolicy = gateCheckPolicy(settings, readiness.total, confirmedContributor, slopRisk, authorHistory);
     gateEvaluation = gateEnabled ? evaluateGateCheck(advisory, gatePolicy) : undefined;
+    // #554 gate false-positive telemetry: when the gate BLOCKS, record the block (one latest row per PR) so a
+    // maintainer can later compute a per-gate-type false-positive rate (blocked-then-merged / blocked).
+    // MEASUREMENT only — never adjusts the gate. Best-effort: a write failure must NOT abort finalization
+    // (mirrors the slop-assessment persist above). Privacy: codes + PR number only, no actor/trust fields.
+    if (gateEvaluation?.conclusion === "failure") {
+      const blockerCodes = gateEvaluation.blockers.map((blocker) => blocker.code);
+      await recordGateBlockOutcome(env, { repoFullName, pullNumber: pr.number, headSha: pr.headSha, blockerCodes }).catch(() => undefined);
+      await recordGithubProductUsage(env, "gate_blocked", {
+        repoFullName,
+        targetKey: `${repoFullName}#${pr.number}`,
+        outcome: "completed",
+        metadata: { blockerCodes },
+      });
+    }
     if (gateEnabled) {
       const gateCheckResult = await createOrUpdateGateCheckRun(
         env,
@@ -1591,6 +1607,10 @@
     outcome: "completed",
     metadata: { actorKind: authorization.actorKind, headSha: advisory.headSha ?? null },
   });
+  // #554 gate false-positive telemetry: flag the gate-block row as maintainer-overridden — the strongest
+  // false-positive signal (a human explicitly judged the block wrong). Best-effort + no-op if no block was
+  // recorded; never affects the override outcome above.
+  await markGateOutcomeOverridden(env, repoFullName, pr.number).catch(() => undefined);
   return true;
 }
 

src/services/gate-precision.ts

@@ -0,0 +1,144 @@
+// #554 gate false-positive telemetry: is the gate PRECISE? This is the evidence a maintainer needs before
+// promoting a gate from advisory to block.
+//
+// MEASUREMENT only — like the #543 outcome-calibration service it NEVER auto-adjusts a gate or score (that
+// would change what blocks live PRs; an owner-review decision). It only records + aggregates.
+//
+// A gate-block is a FALSE POSITIVE when the gate blocked a PR that turned out to be mergeable: the PR was
+// blocked and later MERGED anyway. Per gate type (each blocker `code` that fired) we report blocked count,
+// blocked-then-merged count, the count maintainers OVERRODE (the strongest false-positive signal — a human
+// explicitly judged the block wrong), and a false-positive rate (blocked-then-merged / blocked), null below a
+// min sample so a noisy rate is never reported. Inputs already exist: the gate_outcomes ledger (#554, this
+// PR) records each block, and closed/merged PRs are retained on the PR row, so terminalOutcome resolves the
+// same way outcome-calibration's does.
+//
+// Privacy: the report carries repo full name + PR-derived counts + gate-type codes ONLY — no actor logins, no
+// trust/reward/credibility numbers. Internal/maintainer-authenticated; never publicly exposed.
+import { listGateOutcomes, listPullRequests } from "../db/repositories";
+import type { GateOutcomeRecord, PullRequestRecord } from "../types";
+import { nowIso } from "../utils/json";
+
+// Below this per-gate-type blocked sample the false-positive rate is too noisy to judge.
+const MIN_SAMPLE = 5;
+
+export type GatePrecisionPerType = {
+  gateType: string;
+  blocked: number;
+  blockedThenMerged: number;
+  overridden: number;
+  falsePositiveRate: number | null;
+};
+
+export type GatePrecisionReport = {
+  repoFullName: string;
+  generatedAt: string;
+  windowDays: number | null;
+  perGateType: GatePrecisionPerType[];
+  overall: { blocked: number; blockedThenMerged: number; falsePositiveRate: number | null };
+  signals: string[];
+};
+
+function round(value: number): number {
+  return Math.round(value * 1000) / 1000;
+}
+
+// A PR's terminal outcome: merged if it has a merge timestamp; closed (unmerged) if its state is closed
+// without one; otherwise still open (no outcome yet). Same logic as outcome-calibration.
+function terminalOutcome(pr: PullRequestRecord): "merged" | "closed" | null {
+  if (pr.mergedAt) return "merged";
+  if (pr.state === "closed") return "closed";
+  return null;
+}
+
+function sameRepo(a: string | null | undefined, b: string): boolean {
+  return (a ?? "").toLowerCase() === b.toLowerCase();
+}
+
+/**
+ * Per-gate-type false-positive measurement over recorded gate blocks. Pure. For each block row we look up the
+ * PR's terminal outcome; a blocked PR that later MERGED is a false positive. Each blocker `code` on the row
+ * contributes to that code's bucket (a block citing two codes counts toward both). Overridden-then-merged is
+ * the strongest signal — `overridden` is counted separately per type. When `options.repoFullName` is given,
+ * only blocks for that repo are counted. The rate is null below MIN_SAMPLE.
+ */
+export function buildGatePrecisionReport(
+  outcomes: GateOutcomeRecord[],
+  pullRequests: PullRequestRecord[],
+  options: { repoFullName?: string } = {},
+): Omit<GatePrecisionReport, "repoFullName" | "generatedAt" | "windowDays"> {
+  const repoFullName = options.repoFullName;
+  // Index PRs by number for an O(1) terminal-outcome lookup, scoped to the repo when one is given.
+  const prByNumber = new Map<number, PullRequestRecord>();
+  for (const pr of pullRequests) {
+    if (repoFullName && !sameRepo(pr.repoFullName, repoFullName)) continue;
+    prByNumber.set(pr.number, pr);
+  }
+  const scoped = repoFullName ? outcomes.filter((o) => sameRepo(o.repoFullName, repoFullName)) : outcomes;
+
+  const perType = new Map<string, { blocked: number; blockedThenMerged: number; overridden: number }>();
+  let overallBlocked = 0;
+  let overallMerged = 0;
+  for (const outcome of scoped) {
+    const pr = prByNumber.get(outcome.pullNumber);
+    // A blocked PR that later MERGED is a false positive; closed/open are not (the block held or is unresolved).
+    const merged = pr ? terminalOutcome(pr) === "merged" : false;
+    overallBlocked += 1;
+    if (merged) overallMerged += 1;
+    for (const code of outcome.blockerCodes) {
+      const entry = perType.get(code) ?? { blocked: 0, blockedThenMerged: 0, overridden: 0 };
+      entry.blocked += 1;
+      if (merged) entry.blockedThenMerged += 1;
+      if (outcome.overridden) entry.overridden += 1;
+      perType.set(code, entry);
+    }
+  }
+
+  const perGateType: GatePrecisionPerType[] = [...perType.entries()]
+    .map(([gateType, entry]) => ({
+      gateType,
+      blocked: entry.blocked,
+      blockedThenMerged: entry.blockedThenMerged,
+      overridden: entry.overridden,
+      // Null below the min sample — a 1-of-1 "false positive" is noise, not a precision signal.
+      falsePositiveRate: entry.blocked >= MIN_SAMPLE ? round(entry.blockedThenMerged / entry.blocked) : null,
+    }))
+    .sort((a, b) => b.blocked - a.blocked || a.gateType.localeCompare(b.gateType));
+
+  return {
+    perGateType,
+    overall: {
+      blocked: overallBlocked,
+      blockedThenMerged: overallMerged,
+      falsePositiveRate: overallBlocked >= MIN_SAMPLE ? round(overallMerged / overallBlocked) : null,
+    },
+    signals: buildGatePrecisionSignals(perGateType, overallBlocked, overallMerged),
+  };
+}
+
+export function buildGatePrecisionSignals(perGateType: GatePrecisionPerType[], overallBlocked: number, overallMerged: number): string[] {
+  const signals: string[] = [];
+  if (overallBlocked < MIN_SAMPLE) {
+    signals.push(`Not enough recorded gate blocks to judge precision yet (${overallBlocked} blocked).`);
+    return signals;
+  }
+  signals.push(`${overallMerged} of ${overallBlocked} blocked PRs later merged (${Math.round((overallMerged / overallBlocked) * 100)}% overall false-positive rate).`);
+  // Surface the worst per-type rate that cleared the sample bar — the gate a maintainer should hesitate to promote to block.
+  const judged = perGateType.filter((type) => type.falsePositiveRate !== null);
+  const worst = judged.reduce<GatePrecisionPerType | null>((acc, type) => (acc === null || type.falsePositiveRate! > acc.falsePositiveRate! ? type : acc), null);
+  if (worst && worst.falsePositiveRate! > 0) {
+    signals.push(`Highest false-positive gate: \`${worst.gateType}\` — ${Math.round(worst.falsePositiveRate! * 100)}% of its ${worst.blocked} blocks merged anyway (${worst.overridden} overridden). Keep it advisory until this drops.`);
+  } else {
+    signals.push(`No gate type with enough sample is producing false positives — blocked PRs are staying blocked.`);
+  }
+  return signals;
+}
+
+/** Load a repo's gate-block ledger + PRs and assemble the precision report. */
+export async function loadGatePrecisionReport(env: Env, repoFullName: string, options: { windowDays?: number } = {}): Promise<GatePrecisionReport> {
+  const [pullRequests, outcomes] = await Promise.all([
+    listPullRequests(env, repoFullName),
+    listGateOutcomes(env, { repoFullName, ...(options.windowDays !== undefined ? { windowDays: options.windowDays } : {}) }),
+  ]);
+  const report = buildGatePrecisionReport(outcomes, pullRequests, { repoFullName });
+  return { repoFullName, generatedAt: nowIso(), windowDays: options.windowDays ?? null, ...report };
+}