diff --git a/migrations/0134_predicted_gate_calls.sql b/migrations/0134_predicted_gate_calls.sql new file mode 100644 index 0000000000..f44b3c5156 --- /dev/null +++ b/migrations/0134_predicted_gate_calls.sql @@ -0,0 +1,43 @@ +-- #predicted-live-gate-agreement (maintainer review-stack x AMS integration audit, 2026-07-09): the data +-- substrate for measuring how often the MCP `gittensory_predict_gate`/`gittensory_explain_gate_disposition` +-- verdict agrees with the REAL gate decision the same contributor's PR later receives. +-- +-- WHY A NEW TABLE, NOT A `review_audit` ROW: `predictGateShape` has no PR-number field (it is an explicit +-- pre-PR-existence dry run), so a predicted call cannot be keyed `project#pr` the way recordNativeGateDecision +-- keys a real gate_decision -- there is no PR yet to key against. The only reliable correlation key available +-- at predict-time is (project, login, timestamp), which means CORRELATING a predicted call to its eventual +-- real PR requires a login-keyed join. `review_audit` (migrations/0049) is DELIBERATELY actor-login-free +-- ("No actor logins... ONLY") specifically because it feeds the anonymized cross-instance orb-collector export +-- (src/selfhost/orb-collector.ts) -- exactly the same reason migrations/0126 (contributor_gate_history) is its +-- own separate, local-only table rather than a review_audit column. This table follows that identical +-- precedent: a SEPARATE, LOCAL-ONLY, login-keyed substrate, never wired into exportOrbBatch or any other +-- cross-instance/public export path. See src/review/predicted-gate-calls.ts for the writer and +-- src/review/predicted-gate-agreement.ts for the reader, which joins this table against the ALREADY-EXISTING +-- contributor_gate_history (0126) -- the login-keyed real-decision data that table already records -- rather +-- than duplicating the real side of the comparison into a second copy. +-- +-- Privacy: this table is per-login by design (see migrations/0126's identical rationale for why login, not a +-- hash, is fine for a LOCAL-ONLY table). Any output DERIVED from it (the agreement-rate metric) must remain +-- aggregated -- never render which login contributed which paired row on any public/contributor-facing surface. +CREATE TABLE IF NOT EXISTS predicted_gate_calls ( + id TEXT PRIMARY KEY NOT NULL, + -- The GitHub login the prediction was requested for (the miner's own `login` input to predict_gate). + login TEXT NOT NULL, + -- Which repo the prediction is for. + project TEXT NOT NULL, + -- The predicted gate action: 'merge' | 'hold' (nativeGateActionFromConclusion's mapping -- the predicted-gate + -- engine never predicts 'close', mirroring the live gate: it is a CHECK that passes or blocks, never closes). + predicted_action TEXT NOT NULL, + -- The raw predicted verdict conclusion (success/failure/action_required/neutral), kept alongside the + -- collapsed predicted_action for observability -- e.g. distinguishing a hard blocker from an inconclusive hold. + conclusion TEXT NOT NULL, + -- Bounded reason-class code for the predicted verdict (mirrors review_audit.summary / neutralHoldReasonCode), + -- never a raw finding title/detail (which can embed contributor- or per-repo-controlled text). + reason_code TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +-- The read side (computePredictedGateAgreement) scans "this project's predicted calls in a recency window, +-- grouped by login" to pair each against contributor_gate_history's real decisions. +CREATE INDEX IF NOT EXISTS predicted_gate_calls_project_login_idx + ON predicted_gate_calls(project, login, created_at); diff --git a/scripts/check-schema-drift.mjs b/scripts/check-schema-drift.mjs index 7798a139a2..1e88549a51 100755 --- a/scripts/check-schema-drift.mjs +++ b/scripts/check-schema-drift.mjs @@ -50,6 +50,7 @@ export const RAW_SQL_ONLY_TABLES = new Set([ "orb_signals", "orb_webhook_events", "override_audit", + "predicted_gate_calls", "repo_chunks", "review_audit", "review_targets", diff --git a/src/api/routes.ts b/src/api/routes.ts index a7296171f6..ab9d16b2b1 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -255,6 +255,7 @@ import { buildRepoOutcomeCalibration } from "../services/outcome-calibration"; import { loadGatePrecisionReport } from "../services/gate-precision"; import { computeOpsStats, isOpsEnabled } from "../review/ops-wire"; import { computeParityReadiness, isParityAuditEnabled } from "../review/parity-wire"; +import { computePredictedGateAgreement } from "../review/predicted-gate-agreement"; import { isRagEnabled } from "../review/rag-wire"; import { getPublicStats, isPublicStatsEnabled } from "../review/public-stats"; import { buildMaintainerQualityDashboard, isMaintainerQualityDataStale } from "../services/maintainer-quality-dashboard"; @@ -3498,6 +3499,18 @@ export function createApp() { return c.json(await computeParityReadiness(c.env)); }); + // #predicted-live-gate-agreement (maintainer review-stack x AMS integration audit, 2026-07-09): how often the + // MCP predict_gate/explain_gate_disposition verdict agrees with the REAL gate decision a contributor's PR + // later receives -- a DIFFERENT question than /v1/internal/parity's reviewbot-vs-gittensory migration parity + // (see src/review/predicted-gate-agreement.ts's module header). Same gate/auth contract as /v1/internal/parity: + // bearer-gated by the `/v1/internal/*` middleware, 404 when GITTENSORY_REVIEW_PARITY_AUDIT is off so the + // endpoint does not exist on a deploy not running this telemetry family. Aggregate counts only — no PR + // content / actor logins (see that module's privacy note on why a per-login breakdown never belongs here). + app.get("/v1/internal/predicted-agreement", async (c) => { + if (!isParityAuditEnabled(c.env)) return c.json({ error: "not_found" }, 404); + return c.json(await computePredictedGateAgreement(c.env, { days: 90, nowMs: Date.now() })); + }); + app.post("/v1/internal/jobs/refresh-registry", async (c) => { const message: JobMessage = { type: "refresh-registry", requestedBy: "api" }; await c.env.JOBS.send(message); diff --git a/src/mcp/server.ts b/src/mcp/server.ts index f325111f7e..8cd91b49cd 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -162,6 +162,7 @@ import { loadUpstreamStatus } from "../upstream/ruleset"; import { simulateOpenPrPressure, type OpenPrPressureInput } from "../services/open-pr-pressure-scenarios"; import { buildFindingTaxonomyDocument, FINDING_TAXONOMY_URI } from "../review/finding-taxonomy"; import { buildEnrichmentAnalyzersTaxonomyDocument, ENRICHMENT_ANALYZERS_URI } from "../review/enrichment-analyzers-taxonomy"; +import { recordPredictedGateCall } from "../review/predicted-gate-calls"; type AppContext = Context<{ Bindings: Env }>; type ToolPayload = { @@ -2957,6 +2958,13 @@ export class GittensoryMcp { confirmedContributor, ...(input.changedPaths === undefined ? {} : { changedPaths: input.changedPaths }), }); + // #predicted-live-gate-agreement: record this call so a later real gate decision for the same + // (repo, login) can be paired against it (src/review/predicted-gate-agreement.ts). Shared by BOTH + // predictGate and explainGateDisposition (this function backs both tools) -- a caller that invokes both + // for what is really one logical check records two rows, a small, acceptable volume over-count rather + // than threading a request-scoped dedup key through a read-only prediction path. Best-effort; never + // blocks or fails the tool response. + await recordPredictedGateCall(this.env, { login: input.login, project: repoFullName, verdict }); return { repoFullName, verdict }; } diff --git a/src/review/predicted-gate-agreement.ts b/src/review/predicted-gate-agreement.ts new file mode 100644 index 0000000000..dbdd31f2c7 Binary files /dev/null and b/src/review/predicted-gate-agreement.ts differ diff --git a/src/review/predicted-gate-calls.ts b/src/review/predicted-gate-calls.ts new file mode 100644 index 0000000000..9d8344cf7f --- /dev/null +++ b/src/review/predicted-gate-calls.ts @@ -0,0 +1,75 @@ +// Predicted-gate call history (#predicted-live-gate-agreement, maintainer review-stack x AMS integration +// audit 2026-07-09) -- records EVERY MCP `gittensory_predict_gate`/`gittensory_explain_gate_disposition` call, +// so a later real gate decision for the same (project, login) can be paired against it (see +// src/review/predicted-gate-agreement.ts for the read/join side). Structurally a sibling of +// src/review/contributor-calibration.ts: `review_audit` (migrations/0049) is DELIBERATELY actor-login-free +// (feeds the anonymized orb-collector export), so this is its own separate, LOCAL-ONLY table +// (migrations/0132) -- never wired into exportOrbBatch or any other cross-instance/public export path. +// +// UNLIKE contributor-calibration.ts's per-commit dedup (a re-run at the same head_sha replaces its prior row), +// every predict_gate call gets its OWN row here: there is no commit to dedup against pre-submission, and a +// miner iterating on the same repo (tweaking a title, retrying after a blocker) makes a genuinely new inquiry +// each time -- collapsing them would undercount how often the tool was actually consulted. + +import { isParityAuditEnabled, nativeGateActionFromConclusion } from "./parity-wire"; +import type { GateCheckConclusion } from "../rules/advisory"; +import { isSelfHostedReviewRuntime } from "../selfhost/review-runtime"; +import { errorMessage, nowIso } from "../utils/json"; + +/** The minimal env shape the recorder needs -- mirrors parity-wire.ts's ParityRecorderEnv / contributor- + * calibration.ts's ContributorCalibrationEnv exactly, since this records under the identical self-hosted/ + * parity-flag gate (one flag controls the whole gate-accuracy telemetry family). */ +type PredictedGateCallEnv = { + DB: D1Database; + GITTENSORY_REVIEW_PARITY_AUDIT?: string | undefined; + SELFHOST_TRANSIENT_CACHE?: NonNullable; +}; + +/** The minimal verdict shape this recorder needs -- structurally compatible with PredictedGateVerdict + * (packages/gittensory-engine), whose `blockers` entries are the public-safe shape (no `severity`), unlike + * the real gate's AdvisoryFinding -- so this reads only `.code`, never reusing neutralHoldReasonCode's + * stricter AdvisoryFinding-typed signature (see the reasonCode comment below for why that's an acceptable, + * deliberately coarser fallback on the predicted side). */ +type RecordablePredictedVerdict = { + conclusion: GateCheckConclusion; + blockers: Array<{ code: string }>; +}; + +/** + * Record one MCP predict_gate/explain_gate_disposition call into `predicted_gate_calls`, keyed by the + * requested contributor's login. Gated identically to {@link recordNativeGateDecision} in parity-wire.ts (same + * self-hosted-always-records / cloud-flag-gated contract) -- this is additive telemetry alongside the same + * gate-accuracy measurement family, not a separate feature with its own on/off knob. + * + * Best-effort: a write failure is swallowed (telemetry must never break the MCP tool response). A missing/ + * empty login records nothing -- there is no meaningful per-actor row to write without one. + */ +export async function recordPredictedGateCall( + env: PredictedGateCallEnv, + input: { login: string | null | undefined; project: string; verdict: RecordablePredictedVerdict }, +): Promise { + if (!isSelfHostedReviewRuntime(env) && !isParityAuditEnabled(env)) return; + const login = input.login?.trim(); + if (!login) return; + const action = nativeGateActionFromConclusion(input.verdict.conclusion); + if (action === null) return; // "skipped" -- not a comparable prediction (mirrors recordNativeGateDecision) + const project = input.project.slice(0, 200); + // Coarser than the real gate_decision's summary (which recovers a specific neutral-hold sub-code via + // neutralHoldReasonCode): the predicted-gate engine's public verdict shape carries no `severity` on its + // findings, so it isn't AdvisoryFinding-shaped and can't reuse that stricter-typed helper. reason_code here + // is an observability aid only (not read by computePredictedGateAgreement's core comparison), so the bare + // conclusion string is an acceptable fallback for every non-failure case. + const reasonCode = input.verdict.conclusion === "failure" ? (input.verdict.blockers[0]?.code ?? input.verdict.conclusion) : input.verdict.conclusion; + try { + // Every call gets its own row (no dedup key) -- see the module header for why, unlike + // recordContributorGateDecision's per-commit replace. + await env.DB.prepare( + `INSERT INTO predicted_gate_calls (id, login, project, predicted_action, conclusion, reason_code, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + ) + .bind(`predicted:${login}:${project}:${nowIso()}:${Math.random().toString(36).slice(2, 8)}`, login, project, action, input.verdict.conclusion, reasonCode.slice(0, 200), nowIso()) + .run(); + } catch (error) { + console.warn(JSON.stringify({ event: "predicted_gate_calls_record_error", project, message: errorMessage(error).slice(0, 200) })); + } +} diff --git a/test/unit/mcp-predict-gate.test.ts b/test/unit/mcp-predict-gate.test.ts index dcb51f7fc0..5ec61f9812 100644 --- a/test/unit/mcp-predict-gate.test.ts +++ b/test/unit/mcp-predict-gate.test.ts @@ -228,4 +228,50 @@ testExpectations: expect(result.isError).toBe(true); expect(JSON.stringify(result.content)).toContain("authenticated GitHub login"); }); + + describe("records the call for predicted-vs-live agreement measurement (#predicted-live-gate-agreement)", () => { + async function rawAll(env: Env, sql: string): Promise[]> { + const res = await (env.DB as unknown as { prepare: (s: string) => { all: () => Promise<{ results: T[] }> } }).prepare(sql).all>(); + return res.results; + } + + it("SELF-HOSTED instances record a predicted_gate_calls row on a successful predict_gate call", async () => { + const env = createTestEnv(); // SELFHOST_TRANSIENT_CACHE present by default → self-hosted + const client = await connect(env); + + await client.callTool({ + name: "gittensory_predict_gate", + arguments: { login: "miner1", owner: "acme", repo: "widgets", title: "Add retry to upload client" }, + }); + + const rows = await rawAll(env, "SELECT * FROM predicted_gate_calls"); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ login: "miner1", project: "acme/widgets" }); + }); + + it("also records from gittensory_explain_gate_disposition (both tools share computePredictedGateVerdict)", async () => { + const env = createTestEnv(); + const client = await connect(env); + + await client.callTool({ + name: "gittensory_explain_gate_disposition", + arguments: { login: "miner1", owner: "acme", repo: "widgets", title: "Add retry to upload client" }, + }); + + expect(await rawAll(env, "SELECT * FROM predicted_gate_calls")).toHaveLength(1); + }); + + it("records NOTHING on the CLOUD WORKER when GITTENSORY_REVIEW_PARITY_AUDIT is unset (byte-identical default)", async () => { + const env = createTestEnv(); + delete env.SELFHOST_TRANSIENT_CACHE; // simulate the cloud worker + const client = await connect(env); + + await client.callTool({ + name: "gittensory_predict_gate", + arguments: { login: "miner1", owner: "acme", repo: "widgets", title: "Add retry to upload client" }, + }); + + expect(await rawAll(env, "SELECT * FROM predicted_gate_calls")).toHaveLength(0); + }); + }); }); diff --git a/test/unit/predicted-gate-agreement.test.ts b/test/unit/predicted-gate-agreement.test.ts new file mode 100644 index 0000000000..c272751306 --- /dev/null +++ b/test/unit/predicted-gate-agreement.test.ts @@ -0,0 +1,252 @@ +import { describe, expect, it, vi } from "vitest"; +import { createApp } from "../../src/api/routes"; +import { computePredictedGateAgreement } from "../../src/review/predicted-gate-agreement"; +import { createTestEnv } from "../helpers/d1"; + +async function seedPredicted(env: Env, opts: { login: string; project: string; action: "merge" | "hold" | string; createdAt: string }) { + await env.DB.prepare(`INSERT INTO predicted_gate_calls (id, login, project, predicted_action, conclusion, reason_code, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)`) + .bind(crypto.randomUUID(), opts.login, opts.project, opts.action, opts.action === "merge" ? "success" : "failure", null, opts.createdAt) + .run(); +} + +async function seedReal(env: Env, opts: { login: string; project: string; decision: "merge" | "hold" | "close" | string; createdAt: string; pullNumber?: number }) { + const pr = opts.pullNumber ?? 1; + await env.DB.prepare(`INSERT INTO contributor_gate_history (id, login, source, project, target_id, decision, head_sha, created_at) VALUES (?, ?, 'gittensory-native', ?, ?, ?, 'sha', ?)`) + .bind(crypto.randomUUID(), opts.login, opts.project, `${opts.project}#${pr}`, opts.decision, opts.createdAt) + .run(); +} + +function hoursAfter(iso: string, hours: number): string { + return new Date(new Date(iso).getTime() + hours * 60 * 60 * 1000).toISOString(); +} + +const T0 = "2026-05-01T00:00:00.000Z"; +const NOW = new Date("2026-06-01T00:00:00.000Z").getTime(); + +describe("computePredictedGateAgreement — predicted-vs-live gate agreement (#predicted-live-gate-agreement)", () => { + it("pairs a predicted 'merge' with a real 'merge' as bothMerge, full agreement", async () => { + const env = createTestEnv(); + await seedPredicted(env, { login: "octocat", project: "owner/repo", action: "merge", createdAt: T0 }); + await seedReal(env, { login: "octocat", project: "owner/repo", decision: "merge", createdAt: hoursAfter(T0, 2) }); + + const report = await computePredictedGateAgreement(env, { days: 90, nowMs: NOW }); + const row = report.rows.find((r) => r.project === "owner/repo"); + expect(row).toMatchObject({ pairedSamples: 1, bothMerge: 1, bothHold: 0, disagree: 0, unsafeDisagreements: 0, agreementRate: 1 }); + }); + + it("pairs a predicted 'hold' with a real 'hold' as bothHold, full agreement", async () => { + const env = createTestEnv(); + await seedPredicted(env, { login: "octocat", project: "owner/repo", action: "hold", createdAt: T0 }); + await seedReal(env, { login: "octocat", project: "owner/repo", decision: "hold", createdAt: hoursAfter(T0, 2) }); + + const report = await computePredictedGateAgreement(env, { days: 90, nowMs: NOW }); + const row = report.rows.find((r) => r.project === "owner/repo"); + expect(row).toMatchObject({ pairedSamples: 1, bothMerge: 0, bothHold: 1, agreementRate: 1 }); + }); + + it("flags predicted 'merge' vs real 'hold' as an UNSAFE disagreement (the misleading direction)", async () => { + const env = createTestEnv(); + await seedPredicted(env, { login: "octocat", project: "owner/repo", action: "merge", createdAt: T0 }); + await seedReal(env, { login: "octocat", project: "owner/repo", decision: "hold", createdAt: hoursAfter(T0, 2) }); + + const report = await computePredictedGateAgreement(env, { days: 90, nowMs: NOW }); + const row = report.rows.find((r) => r.project === "owner/repo"); + expect(row).toMatchObject({ pairedSamples: 1, disagree: 1, unsafeDisagreements: 1, agreementRate: 0 }); + }); + + it("does NOT flag predicted 'hold' vs real 'merge' as unsafe (a wasted double-check, not a false all-clear)", async () => { + const env = createTestEnv(); + await seedPredicted(env, { login: "octocat", project: "owner/repo", action: "hold", createdAt: T0 }); + await seedReal(env, { login: "octocat", project: "owner/repo", decision: "merge", createdAt: hoursAfter(T0, 2) }); + + const report = await computePredictedGateAgreement(env, { days: 90, nowMs: NOW }); + const row = report.rows.find((r) => r.project === "owner/repo"); + expect(row).toMatchObject({ pairedSamples: 1, disagree: 1, unsafeDisagreements: 0 }); + }); + + it("pairs MULTIPLE predicted calls (a contributor iterating) against the SAME eventual real decision", async () => { + const env = createTestEnv(); + await seedPredicted(env, { login: "octocat", project: "owner/repo", action: "hold", createdAt: T0 }); + await seedPredicted(env, { login: "octocat", project: "owner/repo", action: "merge", createdAt: hoursAfter(T0, 1) }); + await seedReal(env, { login: "octocat", project: "owner/repo", decision: "merge", createdAt: hoursAfter(T0, 3) }); + + const report = await computePredictedGateAgreement(env, { days: 90, nowMs: NOW }); + const row = report.rows.find((r) => r.project === "owner/repo"); + // Both predicted calls pair to the one real decision: the first (hold) disagrees, the second (merge) agrees. + expect(row).toMatchObject({ pairedSamples: 2, bothMerge: 1, disagree: 1 }); + }); + + it("does not pair a predicted call with no real decision at all (project absent from the report)", async () => { + const env = createTestEnv(); + await seedPredicted(env, { login: "octocat", project: "owner/repo", action: "merge", createdAt: T0 }); + + const report = await computePredictedGateAgreement(env, { days: 90, nowMs: NOW }); + expect(report.rows.find((r) => r.project === "owner/repo")).toBeUndefined(); + }); + + it("does not pair across DIFFERENT logins in the same repo", async () => { + const env = createTestEnv(); + await seedPredicted(env, { login: "octocat", project: "owner/repo", action: "merge", createdAt: T0 }); + await seedReal(env, { login: "someone-else", project: "owner/repo", decision: "merge", createdAt: hoursAfter(T0, 2) }); + + const report = await computePredictedGateAgreement(env, { days: 90, nowMs: NOW }); + expect(report.rows.find((r) => r.project === "owner/repo")).toBeUndefined(); + }); + + it("does not pair across DIFFERENT projects for the same login", async () => { + const env = createTestEnv(); + await seedPredicted(env, { login: "octocat", project: "owner/repo", action: "merge", createdAt: T0 }); + await seedReal(env, { login: "octocat", project: "owner/other-repo", decision: "merge", createdAt: hoursAfter(T0, 2) }); + + const report = await computePredictedGateAgreement(env, { days: 90, nowMs: NOW }); + expect(report.rows).toHaveLength(0); + }); + + it("skips a non-binary real decision (e.g. an autonomous 'close') and pairs the NEXT binary one in the window", async () => { + const env = createTestEnv(); + await seedPredicted(env, { login: "octocat", project: "owner/repo", action: "merge", createdAt: T0 }); + // An unrelated earlier PR from the same contributor auto-closed (e.g. CI failure) -- not a comparable + // gate verdict, so pairing must skip past it rather than giving up on the whole window. + await seedReal(env, { login: "octocat", project: "owner/repo", decision: "close", createdAt: hoursAfter(T0, 1), pullNumber: 1 }); + await seedReal(env, { login: "octocat", project: "owner/repo", decision: "merge", createdAt: hoursAfter(T0, 2), pullNumber: 2 }); + + const report = await computePredictedGateAgreement(env, { days: 90, nowMs: NOW }); + const row = report.rows.find((r) => r.project === "owner/repo"); + expect(row).toMatchObject({ pairedSamples: 1, bothMerge: 1 }); + }); + + it("respects a custom correlationWindowMs — pairs exactly AT the boundary, excludes just past it", async () => { + const env = createTestEnv(); + const oneHourMs = 60 * 60 * 1000; + await seedPredicted(env, { login: "at-edge", project: "owner/repo", action: "merge", createdAt: T0 }); + await seedReal(env, { login: "at-edge", project: "owner/repo", decision: "merge", createdAt: hoursAfter(T0, 1) }); // exactly at the edge + + await seedPredicted(env, { login: "past-edge", project: "owner/repo", action: "merge", createdAt: T0 }); + await seedReal(env, { login: "past-edge", project: "owner/repo", decision: "merge", createdAt: new Date(new Date(T0).getTime() + oneHourMs + 1).toISOString() }); // 1ms past + + const report = await computePredictedGateAgreement(env, { days: 90, nowMs: NOW, correlationWindowMs: oneHourMs }); + const row = report.rows.find((r) => r.project === "owner/repo"); + // Only the at-edge pair counts; the past-edge pair is excluded. + expect(row?.pairedSamples).toBe(1); + }); + + it("scopes to ONE project when opts.project is supplied, even with other projects' data present", async () => { + const env = createTestEnv(); + await seedPredicted(env, { login: "octocat", project: "owner/repo-a", action: "merge", createdAt: T0 }); + await seedReal(env, { login: "octocat", project: "owner/repo-a", decision: "merge", createdAt: hoursAfter(T0, 1) }); + await seedPredicted(env, { login: "octocat", project: "owner/repo-b", action: "merge", createdAt: T0 }); + await seedReal(env, { login: "octocat", project: "owner/repo-b", decision: "merge", createdAt: hoursAfter(T0, 1) }); + + const report = await computePredictedGateAgreement(env, { days: 90, nowMs: NOW, project: "owner/repo-a" }); + expect(report.rows.map((r) => r.project)).toEqual(["owner/repo-a"]); + }); + + it("aggregates multiple projects independently, sorted by project name", async () => { + const env = createTestEnv(); + await seedPredicted(env, { login: "octocat", project: "owner/zzz", action: "merge", createdAt: T0 }); + await seedReal(env, { login: "octocat", project: "owner/zzz", decision: "merge", createdAt: hoursAfter(T0, 1) }); + await seedPredicted(env, { login: "octocat", project: "owner/aaa", action: "hold", createdAt: T0 }); + await seedReal(env, { login: "octocat", project: "owner/aaa", decision: "merge", createdAt: hoursAfter(T0, 1) }); + + const report = await computePredictedGateAgreement(env, { days: 90, nowMs: NOW }); + expect(report.rows.map((r) => r.project)).toEqual(["owner/aaa", "owner/zzz"]); + expect(report.rows.find((r) => r.project === "owner/zzz")).toMatchObject({ bothMerge: 1 }); + expect(report.rows.find((r) => r.project === "owner/aaa")).toMatchObject({ disagree: 1 }); + }); + + it("ignores a non-binary predicted_action defensively (never written in practice, but the read must not crash)", async () => { + const env = createTestEnv(); + await seedPredicted(env, { login: "octocat", project: "owner/repo", action: "bogus", createdAt: T0 }); + await seedReal(env, { login: "octocat", project: "owner/repo", decision: "merge", createdAt: hoursAfter(T0, 1) }); + + const report = await computePredictedGateAgreement(env, { days: 90, nowMs: NOW }); + expect(report.rows.find((r) => r.project === "owner/repo")).toBeUndefined(); + }); + + it("hasSignal flips true once a project reaches 30 paired samples, false below it", async () => { + const env = createTestEnv(); + for (let i = 0; i < 29; i++) { + await seedPredicted(env, { login: `c${i}`, project: "owner/repo", action: "merge", createdAt: T0 }); + await seedReal(env, { login: `c${i}`, project: "owner/repo", decision: "merge", createdAt: hoursAfter(T0, 1) }); + } + const below = await computePredictedGateAgreement(env, { days: 90, nowMs: NOW }); + expect(below.rows.find((r) => r.project === "owner/repo")?.pairedSamples).toBe(29); + expect(below.hasSignal).toBe(false); + + await seedPredicted(env, { login: "c29", project: "owner/repo", action: "merge", createdAt: T0 }); + await seedReal(env, { login: "c29", project: "owner/repo", decision: "merge", createdAt: hoursAfter(T0, 1) }); + const atThreshold = await computePredictedGateAgreement(env, { days: 90, nowMs: NOW }); + expect(atThreshold.rows.find((r) => r.project === "owner/repo")?.pairedSamples).toBe(30); + expect(atThreshold.hasSignal).toBe(true); + }); + + it("fails safe (empty report, never throws) when the predicted_gate_calls read errors", async () => { + const env = createTestEnv(); + const realPrepare = env.DB.prepare.bind(env.DB); + env.DB.prepare = ((sql: string) => { + if (/predicted_gate_calls/i.test(sql)) throw new Error("d1 down"); + return realPrepare(sql); + }) as typeof env.DB.prepare; + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + + await expect(computePredictedGateAgreement(env, { days: 90, nowMs: NOW })).resolves.toEqual({ rows: [], hasSignal: false }); + expect(warn.mock.calls.map((c) => String(c[0])).some((line) => line.includes("predicted_gate_agreement_read_error"))).toBe(true); + warn.mockRestore(); + }); + + it("fails safe (empty report, never throws) when the contributor_gate_history read errors", async () => { + const env = createTestEnv(); + const realPrepare = env.DB.prepare.bind(env.DB); + env.DB.prepare = ((sql: string) => { + if (/contributor_gate_history/i.test(sql)) throw new Error("d1 down"); + return realPrepare(sql); + }) as typeof env.DB.prepare; + + await expect(computePredictedGateAgreement(env, { days: 90, nowMs: NOW })).resolves.toEqual({ rows: [], hasSignal: false }); + }); + + it("defaults `days` to 90 when invalid (0/negative/non-finite), mirroring parity.ts's own convention", async () => { + const env = createTestEnv(); + await seedPredicted(env, { login: "octocat", project: "owner/repo", action: "merge", createdAt: hoursAfter(T0, 24 * 20) }); + await seedReal(env, { login: "octocat", project: "owner/repo", decision: "merge", createdAt: hoursAfter(T0, 24 * 20 + 1) }); + + const report = await computePredictedGateAgreement(env, { days: 0, nowMs: NOW }); + expect(report.rows.find((r) => r.project === "owner/repo")?.pairedSamples).toBe(1); + }); +}); + +describe("GET /v1/internal/predicted-agreement — bearer-gated, flag-gated endpoint", () => { + const bearer = (env: Env) => ({ authorization: `Bearer ${env.INTERNAL_JOB_TOKEN}` }); + + it("401s without the internal token", async () => { + const app = createApp(); + const env = createTestEnv({ GITTENSORY_REVIEW_PARITY_AUDIT: "true" }); + expect((await app.request("/v1/internal/predicted-agreement", {}, env)).status).toBe(401); + }); + + it("404s when GITTENSORY_REVIEW_PARITY_AUDIT is OFF — the endpoint does not exist", async () => { + const app = createApp(); + const env = createTestEnv(); // flag unset → OFF + const res = await app.request("/v1/internal/predicted-agreement", { headers: bearer(env) }, env); + expect(res.status).toBe(404); + expect(((await res.json()) as { error: string }).error).toBe("not_found"); + }); + + it("200s with the predicted-agreement report when ON and authorized", async () => { + const app = createApp(); + const env = createTestEnv({ GITTENSORY_REVIEW_PARITY_AUDIT: "true" }); + // The route hardcodes nowMs: Date.now() (no query-param override yet), so seed data relative to the + // ACTUAL current time rather than a fixed calendar date -- a fixed T0 would silently fall outside the + // 90-day window once enough real time has passed since this test was written. + const nowIso = new Date().toISOString(); + await seedPredicted(env, { login: "octocat", project: "owner/repo", action: "merge", createdAt: nowIso }); + await seedReal(env, { login: "octocat", project: "owner/repo", decision: "merge", createdAt: hoursAfter(nowIso, 1) }); + + const res = await app.request("/v1/internal/predicted-agreement", { headers: bearer(env) }, env); + expect(res.status).toBe(200); + const body = (await res.json()) as { hasSignal: boolean; rows: Array<{ project: string; pairedSamples: number }> }; + expect(body.rows.find((r) => r.project === "owner/repo")?.pairedSamples).toBe(1); + // Privacy: aggregate only — never actor logins / trust internals. + expect(JSON.stringify(body)).not.toMatch(/octocat|login|actor|reward|payout|trust|wallet|hotkey/i); + }); +}); diff --git a/test/unit/predicted-gate-calls.test.ts b/test/unit/predicted-gate-calls.test.ts new file mode 100644 index 0000000000..538235b4d8 --- /dev/null +++ b/test/unit/predicted-gate-calls.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it, vi } from "vitest"; +import { recordPredictedGateCall } from "../../src/review/predicted-gate-calls"; +import { createTestEnv } from "../helpers/d1"; + +// ── Direct D1 helpers over the real migrated schema (0132 predicted_gate_calls) ─────────────────────────────── + +async function rawAll(env: Env, sql: string, ...binds: unknown[]): Promise[]> { + const res = await (env.DB as unknown as { prepare: (s: string) => { bind: (...v: unknown[]) => { all: () => Promise<{ results: T[] }> } } }) + .prepare(sql) + .bind(...binds) + .all>(); + return res.results; +} + +function verdict(overrides: Partial<{ conclusion: string; blockers: Array<{ code: string }> }> = {}) { + return { conclusion: "success", blockers: [], ...overrides } as { conclusion: "success" | "failure" | "action_required" | "neutral" | "skipped"; blockers: Array<{ code: string }> }; +} + +describe("recordPredictedGateCall — write-only predicted-gate call history (0132, #predicted-live-gate-agreement)", () => { + it("SELF-HOSTED instances record ONE row keyed by login (createTestEnv's default self-host signal)", async () => { + const env = createTestEnv(); // flag unset → OFF, but SELFHOST_TRANSIENT_CACHE present → self-hosted + await recordPredictedGateCall(env, { login: "octocat", project: "owner/repo", verdict: verdict() }); + + const rows = await rawAll(env, "SELECT * FROM predicted_gate_calls"); + expect(rows.length).toBe(1); + expect(rows[0]).toMatchObject({ login: "octocat", project: "owner/repo", predicted_action: "merge", conclusion: "success", reason_code: "success" }); + expect(typeof rows[0]!.created_at).toBe("string"); + }); + + it("EVERY call gets its own row — no dedup, unlike recordContributorGateDecision's per-commit replace", async () => { + const env = createTestEnv(); + await recordPredictedGateCall(env, { login: "octocat", project: "owner/repo", verdict: verdict() }); + await recordPredictedGateCall(env, { login: "octocat", project: "owner/repo", verdict: verdict() }); + expect((await rawAll(env, "SELECT * FROM predicted_gate_calls")).length).toBe(2); + }); + + it("maps 'failure' to predicted_action 'hold' and uses the first blocker's code as reason_code", async () => { + const env = createTestEnv(); + await recordPredictedGateCall(env, { login: "octocat", project: "owner/repo", verdict: verdict({ conclusion: "failure", blockers: [{ code: "missing_linked_issue" }, { code: "oversized_pr" }] }) }); + const rows = await rawAll(env, "SELECT * FROM predicted_gate_calls"); + expect(rows[0]).toMatchObject({ predicted_action: "hold", conclusion: "failure", reason_code: "missing_linked_issue" }); + }); + + it("falls back to the bare conclusion string as reason_code when 'failure' has no blockers", async () => { + const env = createTestEnv(); + await recordPredictedGateCall(env, { login: "octocat", project: "owner/repo", verdict: verdict({ conclusion: "failure", blockers: [] }) }); + const rows = await rawAll(env, "SELECT * FROM predicted_gate_calls"); + expect(rows[0]).toMatchObject({ predicted_action: "hold", reason_code: "failure" }); + }); + + it("maps 'action_required' and 'neutral' to predicted_action 'hold', reason_code the bare conclusion", async () => { + const env = createTestEnv(); + await recordPredictedGateCall(env, { login: "octocat", project: "owner/repo", verdict: verdict({ conclusion: "action_required" }) }); + await recordPredictedGateCall(env, { login: "octocat", project: "owner/repo", verdict: verdict({ conclusion: "neutral" }) }); + const rows = await rawAll(env, "SELECT * FROM predicted_gate_calls ORDER BY rowid ASC"); + expect(rows[0]).toMatchObject({ predicted_action: "hold", conclusion: "action_required", reason_code: "action_required" }); + expect(rows[1]).toMatchObject({ predicted_action: "hold", conclusion: "neutral", reason_code: "neutral" }); + }); + + it("does NOT record a 'skipped' conclusion — not a comparable prediction (mirrors recordNativeGateDecision)", async () => { + const env = createTestEnv(); + await recordPredictedGateCall(env, { login: "octocat", project: "owner/repo", verdict: verdict({ conclusion: "skipped" }) }); + expect((await rawAll(env, "SELECT * FROM predicted_gate_calls")).length).toBe(0); + }); + + it("does NOT record when the login is missing, null, or blank", async () => { + const env = createTestEnv(); + await recordPredictedGateCall(env, { login: undefined, project: "owner/repo", verdict: verdict() }); + await recordPredictedGateCall(env, { login: null, project: "owner/repo", verdict: verdict() }); + await recordPredictedGateCall(env, { login: " ", project: "owner/repo", verdict: verdict() }); + expect((await rawAll(env, "SELECT * FROM predicted_gate_calls")).length).toBe(0); + }); + + it("flag-OFF records NOTHING on the CLOUD WORKER — no D1 write (byte-identical, same gate family as recordNativeGateDecision)", async () => { + const env = createTestEnv(); + delete env.SELFHOST_TRANSIENT_CACHE; // simulate the cloud worker (no self-host binding) + await recordPredictedGateCall(env, { login: "octocat", project: "owner/repo", verdict: verdict() }); + expect((await rawAll(env, "SELECT * FROM predicted_gate_calls")).length).toBe(0); + + const envFalse = createTestEnv({ GITTENSORY_REVIEW_PARITY_AUDIT: "false" }); + delete envFalse.SELFHOST_TRANSIENT_CACHE; + await recordPredictedGateCall(envFalse, { login: "octocat", project: "owner/repo", verdict: verdict() }); + expect((await rawAll(envFalse, "SELECT * FROM predicted_gate_calls")).length).toBe(0); + }); + + it("the cloud worker records when GITTENSORY_REVIEW_PARITY_AUDIT is explicitly ON", async () => { + const env = createTestEnv({ GITTENSORY_REVIEW_PARITY_AUDIT: "true" }); + delete env.SELFHOST_TRANSIENT_CACHE; + await recordPredictedGateCall(env, { login: "octocat", project: "owner/repo", verdict: verdict() }); + expect((await rawAll(env, "SELECT * FROM predicted_gate_calls")).length).toBe(1); + }); + + it("fails safe: a D1 write error is swallowed + logged (telemetry never breaks the MCP tool response)", async () => { + const env = createTestEnv(); + const realPrepare = env.DB.prepare.bind(env.DB); + env.DB.prepare = ((sql: string) => { + if (/predicted_gate_calls/i.test(sql)) throw new Error("poisoned write"); + return realPrepare(sql); + }) as typeof env.DB.prepare; + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + + await expect(recordPredictedGateCall(env, { login: "octocat", project: "owner/repo", verdict: verdict() })).resolves.toBeUndefined(); + + expect(warn.mock.calls.map((c) => String(c[0])).some((line) => line.includes("predicted_gate_calls_record_error"))).toBe(true); + warn.mockRestore(); + }); +});