Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .release-please-manifest.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"packages/loopover-mcp": "3.14.1",
"packages/loopover-engine": "3.14.1",
"packages/loopover-engine": "3.15.0",
"packages/loopover-miner": "3.14.1",
"packages/loopover-ui-kit": "1.2.0"
}
14 changes: 14 additions & 0 deletions migrations/0181_decision_replay_inputs.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
-- #8838 (epic #8828, Phase 4 — trust surface): private replay inputs for decision records.
--
-- One row per decision record holding the EXACT inputs the deterministic gate pipeline consumed — the input
-- advisory findings, the resolved GateCheckPolicy, the policy-close kind, and the evaluation snapshot
-- (conclusion + ordered blocker codes) — so any decision can be re-derived bit-exactly by the replay
-- harness (scripts/replay-decision.ts). DELIBERATELY A SIBLING TABLE, not a decision_records column: the
-- public record's contract is digests-only for config (a contributor sees the COMMITMENT, never the
-- resolved private policy values), and the input findings carry contributor content. Replay inputs are
-- operator-private; the record's record_digest still publicly commits the decision they explain.
CREATE TABLE IF NOT EXISTS decision_replay_inputs (
record_id TEXT PRIMARY KEY, -- decision_records.id (record:<owner/repo>#<pr>@<head sha>)
replay_json TEXT NOT NULL, -- {findings, policy, policyCloseKind, evaluated:{conclusion, blockerCodes}}
created_at TEXT NOT NULL
);
1 change: 1 addition & 0 deletions scripts/check-schema-drift.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ export const RAW_SQL_ONLY_TABLES: Set<string> = new Set([
"decision_audit_labels",
"decision_ledger",
"decision_records",
"decision_replay_inputs",
"global_agent_controls",
"global_contributor_blacklist",
"global_moderation_config",
Expand Down
59 changes: 59 additions & 0 deletions scripts/replay-decision.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
#!/usr/bin/env node
// Decision replay CLI (#8838) — re-derive a gate decision from its persisted record + replay input and
// prove it bit-exactly, or exit non-zero with the first divergent stage.
//
// node --experimental-strip-types scripts/replay-decision.ts <bundle.json>
// ... | node --experimental-strip-types scripts/replay-decision.ts -
//
// The bundle is one JSON object: { record: {...decision_records row}, replayInput: {...replay_json} }.
// EXTRACT (operator, against the instance DB):
// SELECT json_build_object('record', to_jsonb(dr), 'replayInput', dri.replay_json::jsonb)
// FROM decision_records dr JOIN decision_replay_inputs dri ON dri.record_id = dr.id
// WHERE dr.id = 'record:<owner/repo>#<pr>@<head sha>';
//
// Exit codes: 0 = replayed, same verdict ("here is the clause"); 1 = DIVERGENCE — a bug by definition,
// file it with the printed stage diff; 2 = unusable input. Replay mode cannot re-query the model or touch
// any network/DB by construction: replayDecision is a pure function of the two JSON values.
import { readFileSync } from "node:fs";
import { replayDecision, type DecisionReplayInput, type ReplayableRecord } from "../src/review/decision-replay";

/** Parse + normalize a bundle (snake_case SQL rows accepted) and replay it. Exported for tests. */
export function runReplayBundle(raw: string): { outcome: ReturnType<typeof replayDecision> | null; error?: string } {
let bundle: { record?: Record<string, unknown>; replayInput?: unknown };
try {
bundle = JSON.parse(raw) as never;
} catch (error) {
return { outcome: null, error: `unparseable bundle JSON: ${error instanceof Error ? error.message : String(error)}` };
}
const rawRecord = bundle.record;
const replayInput = bundle.replayInput as DecisionReplayInput | undefined;
const record: ReplayableRecord | null =
rawRecord && typeof rawRecord.id === "string"
? {
id: rawRecord.id,
reasonCode: String(rawRecord.reasonCode ?? rawRecord.reason_code ?? ""),
action: String(rawRecord.action ?? ""),
}
: null;
if (!record || !replayInput || !Array.isArray(replayInput.findings) || typeof replayInput.evaluated !== "object") {
return { outcome: null, error: "bundle must carry {record: {id, reason_code|reasonCode, action}, replayInput: {findings, policy, evaluated}}" };
}
return { outcome: replayDecision(record, replayInput) };
}

const invokedDirectly = process.argv[1]?.endsWith("replay-decision.ts") === true;
if (invokedDirectly) {
const source = process.argv[2];
if (!source) {
console.error("usage: replay-decision.ts <bundle.json | ->");
process.exit(2);
}
const raw = source === "-" ? readFileSync(0, "utf8") : readFileSync(source, "utf8");
const { outcome, error } = runReplayBundle(raw);
if (!outcome) {
console.error(`replay-decision: ${error}`);
process.exit(2);
}
console.log(JSON.stringify(outcome, null, 2));
process.exit(outcome.verdict === "match" ? 0 : 1);
}
14 changes: 8 additions & 6 deletions src/queue/processors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -647,6 +647,7 @@ import {
} from "../review/outcomes-wire";
import { AI_JUDGMENT_BLOCKER_CODES } from "../rules/advisory";
import { computeSalvageabilityForTarget } from "../review/salvageability-wire";
import { deriveDecisionReasonCode, persistDecisionReplayInputForGate } from "../review/decision-replay";
import { REVIEW_PROMPT_VERSION, REVIEW_SYSTEM_PROMPT } from "../services/ai-review";
import { resolveAutomaticCloseConfidence } from "../review/risk-control-wire";
import { maybeApplyCloseAuditHoldout } from "../review/close-audit-holdout";
Expand Down Expand Up @@ -3353,12 +3354,9 @@ async function runAgentMaintenancePlanAndExecute(
headSha: pr.headSha ?? "unknown",
baseSha: null,
action: disposition.actionClass,
reasonCode:
disposition.blockerClass !== "none"
? disposition.blockerClass
: policyCloseKind !== undefined
? `policy_close:${policyCloseKind}`
: gate.conclusion,
// #8838: the shared derivation — replayDecision recomputes reasonCode through this SAME function, so
// the live mapping and the replay mapping can never drift apart.
reasonCode: deriveDecisionReasonCode(disposition.blockerClass, policyCloseKind ?? null, gate.conclusion),
configDigest: await contentDigest(settings),
gatePack: settings.gatePack,
ciState: null,
Expand All @@ -3368,6 +3366,10 @@ async function runAgentMaintenancePlanAndExecute(
salvageability,
});
await persistDecisionRecord(env, record, recordDigest);
// #8838: persist the evaluation's own exact inputs beside the record (PRIVATE sibling, migration 0181)
// so the replay harness can re-derive this decision bit-exactly. Best-effort, like the record itself;
// the no-replay no-op (synthetic content-lane/bridge evaluations) lives inside the helper.
await persistDecisionReplayInputForGate(env, `record:${record.repoFullName}#${record.pullNumber}@${record.headSha}`.slice(0, 250), gate, policyCloseKind ?? null);
}
// #2349 (PR 1): additive per-contributor calibration data, gated identically to recordNativeGateDecision
// above -- see src/review/contributor-calibration.ts's doc comment. Currently write-only; nothing reads
Expand Down
126 changes: 126 additions & 0 deletions src/review/decision-replay.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
// Deterministic decision replay (#8838, epic #8828 Phase 4) — re-derive a decision from its recorded
// inputs and prove it, or find the first divergent stage.
//
// The replay contract mirrors the golden corpus (#8832): `evaluateGateCheck(advisory, policy)` is the pure
// pipeline, and the persisted replay input carries its EXACT inputs (the advisory findings + the resolved
// policy) plus the decision-time evaluation snapshot. Replay is STRUCTURALLY incapable of re-querying the
// model or touching the network: `replayDecision` is a pure function of two JSON values — the no-requery
// guarantee is by construction, not by flag.
//
// Stages, compared in pipeline order (the first mismatch wins — everything after it is downstream noise):
// 1. `conclusion` — re-evaluated gate conclusion vs the decision-time snapshot.
// 2. `blocker_codes` — ordered blocker code list (order IS meaning: blockerClass is the first code).
// 3. `reason_code` — re-derived exactly as the finalize site derives it (blockerClass →
// policy_close:<kind> → conclusion) vs the PUBLIC record's reason_code.
// `action` is reported as PINNED, never re-derived: it depends on plan state (autonomy, holds, approvals)
// outside the recorded pipeline — documented v1 scope on #8838.
//
// A divergence is a bug BY DEFINITION (the pipeline is supposed to be deterministic): callers exit non-zero
// and file it; there is no "close enough" outcome.
import { evaluateGateCheck, type GateCheckPolicy } from "../rules/advisory";
import { neutralHoldReasonCode } from "./parity-wire";
import type { Advisory, AdvisoryFinding } from "../types";
import { errorMessage, nowIso } from "../utils/json";

/** What decision_replay_inputs.replay_json holds — the pipeline's exact inputs + the decision-time snapshot. */
export type DecisionReplayInput = {
findings: AdvisoryFinding[];
policy: GateCheckPolicy;
/** The policy-close kind the finalize used in its reasonCode derivation, when one applied. */
policyCloseKind?: string | null | undefined;
/** Decision-time evaluation snapshot: what the live pipeline produced from these same inputs. */
evaluated: { conclusion: string; blockerCodes: string[] };
};

/** The slice of the PUBLIC decision record replay verifies against. */
export type ReplayableRecord = {
id: string;
reasonCode: string;
action: string;
};

export type ReplayOutcome =
| { verdict: "match"; recordId: string; conclusion: string; blockerCodes: string[]; reasonCode: string; pinnedAction: string }
| {
verdict: "divergence";
recordId: string;
/** The FIRST divergent stage — later stages are downstream of it and not reported. */
stage: "conclusion" | "blocker_codes" | "reason_code";
expected: string;
actual: string;
};

/** The finalize site's reasonCode derivation, extracted verbatim so replay and live can never disagree
* about the mapping itself (single source of truth — processors.ts finalize imports this too). */
export function deriveDecisionReasonCode(blockerClass: string, policyCloseKind: string | null | undefined, conclusion: string): string {
return blockerClass !== "none" ? blockerClass : policyCloseKind != null ? `policy_close:${policyCloseKind}` : conclusion;
}

/** blockerClass exactly as agentDispositionLabels derives it from an evaluation: first blocker code, else
* the neutral-hold reason, else "none". */
export function deriveBlockerClass(evaluation: { blockers: Array<{ code: string }>; conclusion: string; warnings: AdvisoryFinding[] }): string {
return evaluation.blockers[0]?.code ?? neutralHoldReasonCode(evaluation as never) ?? "none";
}

/** PURE bit-exact replay. See the module doc for the stage contract. */
export function replayDecision(record: ReplayableRecord, input: DecisionReplayInput): ReplayOutcome {
const advisory: Advisory = {
id: `replay-${record.id}`,
targetType: "pull_request",
targetKey: record.id,
repoFullName: "replay/harness",
pullNumber: 0,
conclusion: "neutral",
severity: "info",
title: "replay",
summary: "decision replay",
findings: input.findings,
generatedAt: "2026-01-01T00:00:00.000Z",
} as Advisory;
const evaluation = evaluateGateCheck(advisory, input.policy);
if (evaluation.conclusion !== input.evaluated.conclusion) {
return { verdict: "divergence", recordId: record.id, stage: "conclusion", expected: input.evaluated.conclusion, actual: evaluation.conclusion };
}
const blockerCodes = evaluation.blockers.map((blocker) => blocker.code);
if (blockerCodes.join(",") !== input.evaluated.blockerCodes.join(",")) {
return { verdict: "divergence", recordId: record.id, stage: "blocker_codes", expected: input.evaluated.blockerCodes.join(","), actual: blockerCodes.join(",") };
}
const reasonCode = deriveDecisionReasonCode(deriveBlockerClass(evaluation), input.policyCloseKind, evaluation.conclusion);
if (reasonCode !== record.reasonCode) {
return { verdict: "divergence", recordId: record.id, stage: "reason_code", expected: record.reasonCode, actual: reasonCode };
}
return { verdict: "match", recordId: record.id, conclusion: evaluation.conclusion, blockerCodes, reasonCode, pinnedAction: record.action };
}

/** Persist the replay input beside its record (PRIVATE sibling — see migration 0181). Best-effort: replay
* legibility must never break finalization, mirroring persistDecisionRecord's posture. Accepts the gate
* EVALUATION and owns the no-replay no-op: content-lane/bridge evaluations are synthetic (their verdicts
* come from their own deterministic pipelines, not the advisory evaluator) and carry no replay input —
* documented v1 scope on #8838. */
export async function persistDecisionReplayInputForGate(
env: Env,
recordId: string,
gate: { replay?: { findings: AdvisoryFinding[]; policy: GateCheckPolicy } | undefined; conclusion: string; blockers: Array<{ code: string }> },
policyCloseKind: string | null,
): Promise<void> {
if (!gate.replay) return;
await persistDecisionReplayInput(env, recordId, {
findings: gate.replay.findings,
policy: gate.replay.policy,
policyCloseKind,
evaluated: { conclusion: gate.conclusion, blockerCodes: gate.blockers.map((blocker) => blocker.code) },
});
}

export async function persistDecisionReplayInput(env: Env, recordId: string, input: DecisionReplayInput): Promise<void> {
try {
await env.DB.prepare(
`INSERT INTO decision_replay_inputs (record_id, replay_json, created_at) VALUES (?, ?, ?)
ON CONFLICT(record_id) DO UPDATE SET replay_json = excluded.replay_json, created_at = excluded.created_at`,
)
.bind(recordId.slice(0, 250), JSON.stringify(input), nowIso())
.run();
} catch (error) {
console.warn(JSON.stringify({ event: "decision_replay_persist_error", recordId: recordId.slice(0, 120), message: errorMessage(error).slice(0, 160) }));
}
}
13 changes: 11 additions & 2 deletions src/rules/advisory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,10 @@ export type GateCheckEvaluation = {
summary: string;
blockers: AdvisoryFinding[];
warnings: AdvisoryFinding[];
/** #8838: the evaluation's own exact inputs, attached by evaluateGateCheck. OPTIONAL by design: the
* content-lane and bridge paths construct synthetic evaluations whose verdicts come from their own
* deterministic pipelines, not this evaluator — those decisions have no advisory replay input. */
replay?: { findings: AdvisoryFinding[]; policy: GateCheckPolicy } | undefined;
};

// AI-JUDGMENT blocker codes. Kept distinct from deterministic blockers for telemetry and regression tests; the old
Expand Down Expand Up @@ -679,10 +683,15 @@ function promoteAdvisoryToBlock(policy: GateCheckPolicy): GateCheckPolicy {
* the core eval with advisory sub-gates promoted to block and attaches that as `displayConclusion` — the would-be
* merge/close/manual verdict — while the POSTED `conclusion` stays the real, non-enforcing one. */
export function evaluateGateCheck(advisoryResult: Advisory, policy: GateCheckPolicy = {}): GateCheckEvaluation {
// #8838: every evaluation carries its own EXACT inputs so the finalize site can persist them for the
// deterministic replay harness — captured here, at the single choke point every call site goes through,
// instead of threading (advisory, policy) across callers. In-memory only; the finalize site decides what
// is persisted (decision_replay_inputs — a PRIVATE sibling of the public record).
const replay = { findings: advisoryResult.findings, policy };
const result = evaluateGateCheckCore(advisoryResult, policy);
if (!policy.dryRun) return result;
if (!policy.dryRun) return { ...result, replay };
const wouldBe = evaluateGateCheckCore(advisoryResult, promoteAdvisoryToBlock(policy));
return { ...result, displayConclusion: wouldBe.conclusion };
return { ...result, displayConclusion: wouldBe.conclusion, replay };
}

function evaluateGateCheckCore(advisoryResult: Advisory, policy: GateCheckPolicy = {}): GateCheckEvaluation {
Expand Down
29 changes: 29 additions & 0 deletions test/fixtures/decision-replay/ai-consensus-close.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
{
"record": {
"id": "record:fx/repo#1@ai-consensus-close",
"reason_code": "ai_consensus_defect",
"action": "close"
},
"replayInput": {
"findings": [
{
"code": "ai_consensus_defect",
"title": "Consensus defect",
"severity": "critical",
"detail": "unused import join is dead code",
"confidence": 0.95
}
],
"policy": {
"aiReviewGateMode": "block",
"aiReviewCloseConfidence": 0.93
},
"policyCloseKind": null,
"evaluated": {
"conclusion": "failure",
"blockerCodes": [
"ai_consensus_defect"
]
}
}
}
16 changes: 16 additions & 0 deletions test/fixtures/decision-replay/clean-merge.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"record": {
"id": "record:fx/repo#1@clean-merge",
"reason_code": "success",
"action": "merge"
},
"replayInput": {
"findings": [],
"policy": {},
"policyCloseKind": null,
"evaluated": {
"conclusion": "success",
"blockerCodes": []
}
}
}
16 changes: 16 additions & 0 deletions test/fixtures/decision-replay/policy-close.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"record": {
"id": "record:fx/repo#1@policy-close",
"reason_code": "policy_close:stale_superseded",
"action": "close"
},
"replayInput": {
"findings": [],
"policy": {},
"policyCloseKind": "stale_superseded",
"evaluated": {
"conclusion": "success",
"blockerCodes": []
}
}
}
25 changes: 25 additions & 0 deletions test/fixtures/decision-replay/secret-leak-close.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
{
"record": {
"id": "record:fx/repo#1@secret-leak-close",
"reason_code": "secret_leak",
"action": "close"
},
"replayInput": {
"findings": [
{
"code": "secret_leak",
"title": "Committed secret",
"severity": "critical",
"detail": "API key in config"
}
],
"policy": {},
"policyCloseKind": null,
"evaluated": {
"conclusion": "failure",
"blockerCodes": [
"secret_leak"
]
}
}
}
Loading
Loading