Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions src/queue/processors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,7 @@ import {
buildIssueAdvisory,
buildPullRequestAdvisory,
evaluateGateCheck,
recordConfiguredGateBlockerSignals,
resolveAiReviewLowConfidenceHold,
} from "../rules/advisory";
import { hasValidationNote, isTestPath } from "../signals/test-evidence";
Expand Down Expand Up @@ -10298,6 +10299,11 @@ async function maybePublishPrPublicSurface(
let evaluation = shouldEvaluateGate
? evaluateGateCheck(advisory, gatePolicy)
: undefined;
// #8104: record RuleFiredEvent for every configured gate blocker except linked_issue_scope_mismatch
// (#8101). Same advisory+policy as evaluateGateCheck above so the filter stays in lock-step.
if (evaluation) {
await recordConfiguredGateBlockerSignals(env, advisory, gatePolicy, repoFullName, pr.number);
}
// Deterministic content/registry surface lane (#1255) — flag-gated + per-repo allowlist, byte-identical when
// off (evaluateWithSurfaceLane returns the generic evaluation unchanged and resolves no files). A metagraphed
// registry-submission PR's surface verdict OVERRIDES the generic gate; the helper preserves a generic HARD
Expand Down
33 changes: 33 additions & 0 deletions src/review/outcomes-wire.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,10 @@ import { tryEnqueueDecisionPackRebuild } from "../services/decision-pack";
import { incr } from "../selfhost/metrics";
import { loadRepoFocusManifest } from "../signals/focus-manifest-loader";
import type { GitHubWebhookPayload } from "../types";
import {
CONFIGURED_GATE_BLOCKER_SIGNAL_CODES,
CONFIGURED_GATE_BLOCKER_SIGNAL_LOOKBACK_MS,
} from "../rules/advisory";
import { errorMessage, nowIso } from "../utils/json";
import {
applyAutoTune,
Expand Down Expand Up @@ -448,6 +452,33 @@ async function hasRecentOwnerReopenPendingReversal(env: Env, targetKey: string,
}
}

// #8104: when a reversal is recorded for a target that any configured-gate-blocker rule (except
// linked_issue_scope_mismatch — #8101 owns that one) previously fired against, the human undoing of the bot
// action IS the human judgment on those findings. Fixed 30-day lookback; candidate codes come from
// CONFIGURED_GATE_BLOCKER_SIGNAL_CODES so the list cannot silently drift from isConfiguredGateBlocker.
// Callers attach `.catch(() => undefined)`: a SignalStore failure (including a queryRuleHistory read error,
// which deliberately propagates) must never affect whether the underlying reversal itself is recorded.
async function recordConfiguredGateBlockerOverrides(env: Env, targetId: string): Promise<void> {
const store = createSignalStore(env);
const sinceMs = Date.now() - CONFIGURED_GATE_BLOCKER_SIGNAL_LOOKBACK_MS;
await Promise.all(
CONFIGURED_GATE_BLOCKER_SIGNAL_CODES.map(async (ruleId) => {
try {
const history = await store.queryRuleHistory(ruleId, sinceMs);
if (!history.fired.some((event) => event.targetKey === targetId)) return;
await store.recordHumanOverride({
ruleId,
targetKey: targetId,
verdict: "reversed",
occurredAt: nowIso(),
});
} catch {
// Fail-open per code: one SignalStore reject must not skip the rest of the candidate list.
}
}),
);
}

// #8101: when a reversal is recorded for a target that a `linked_issue_scope_mismatch` finding fired
// against (fixed 30-day lookback), the human undoing of the bot action IS the human judgment on that
// finding — record a "reversed" HumanOverrideEvent in the shared calibration module (#7982) so the
Expand Down Expand Up @@ -537,6 +568,7 @@ export async function recordReversalSignals(
detail: `Bot-closed PR #${pr.number} reopened by a contributor.`,
metadata: { repoFullName, pullNumber: pr.number },
}).catch(() => undefined);
await recordConfiguredGateBlockerOverrides(env, targetId).catch(() => undefined); // #8104
await recordLinkedIssueScopeMismatchOverride(env, targetId).catch(() => undefined); // #8101
return;
}
Expand All @@ -561,6 +593,7 @@ export async function recordReversalSignals(
detail: `Bot-closed PR #${pr.number} reopened and merged by the repo owner.`,
metadata: { repoFullName, pullNumber: pr.number },
}).catch(() => undefined);
await recordConfiguredGateBlockerOverrides(env, targetId).catch(() => undefined); // #8104
await recordLinkedIssueScopeMismatchOverride(env, targetId).catch(() => undefined); // #8101
}
const reverted = parseRevertedPrNumber(pr.body);
Expand Down
63 changes: 63 additions & 0 deletions src/rules/advisory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import { nowIso } from "../utils/json";
import { LOOPOVER_GATE_CHECK_NAME } from "../review/check-names";
import { CLA_CHECK_UNRESOLVED_CODE, CLA_CONSENT_MISSING_CODE } from "../review/cla-check";
import { REVIEW_THREAD_BLOCKER_CODE } from "../review/review-thread-findings";
import { createSignalStore } from "../review/signal-tracking-wire";
import { labelMatchesPattern } from "../scoring/preview";

export type GateCheckConclusion = "success" | "failure" | "action_required" | "neutral" | "skipped";
Expand Down Expand Up @@ -164,6 +165,29 @@ export type GateCheckEvaluation = {
// `ai_review_inconclusive` is deliberately EXCLUDED — that is a "could not review" HOLD, not a false defect.
export const AI_JUDGMENT_BLOCKER_CODES = new Set<string>(["ai_consensus_defect", "ai_review_split"]);

/**
* Every finding code `isConfiguredGateBlocker` can return true for, EXCEPT `linked_issue_scope_mismatch`
* (#8104). That one code is wired by #8101 at its own upstream push / reversal sites — including it here
* would double-count fired/reversed history. Keep this list in sync with `isConfiguredGateBlocker`'s body.
*/
export const CONFIGURED_GATE_BLOCKER_SIGNAL_CODES: readonly string[] = Object.freeze([
"missing_linked_issue",
"duplicate_pr_risk",
...AI_JUDGMENT_BLOCKER_CODES,
REVIEW_THREAD_BLOCKER_CODE,
"secret_leak",
"pre_merge_check_required",
"manifest_missing_tests",
"manifest_linked_issue_required",
"self_authored_linked_issue",
"content_lane_deliverable_missing",
"lockfile_tamper_risk",
CLA_CONSENT_MISSING_CODE,
]);

/** Fixed lookback for reversal→HumanOverrideEvent pairing (#8104) — 30 days in milliseconds. */
export const CONFIGURED_GATE_BLOCKER_SIGNAL_LOOKBACK_MS = 30 * 24 * 60 * 60 * 1000;

/** True when the gate FAILED *solely* because of AI-judgment blockers (every blocker is an AI-judgment code).
* An empty blocker list is NOT an AI-judgment-only failure. PURE. */
export function isAiJudgmentOnlyFailure(evaluation: GateCheckEvaluation): boolean {
Expand Down Expand Up @@ -612,6 +636,10 @@ function evaluateGateCheckCore(advisoryResult: Advisory, policy: GateCheckPolicy
// pass/fail. Readiness/quality stays advisory-only.
const effective = applyMergeReadinessGate(policy);
const configuredBlockers = advisoryResult.findings.filter((finding) => isConfiguredGateBlocker(finding, effective));
// #8104: every configured blocker except linked_issue_scope_mismatch (#8101) records a RuleFiredEvent in
// the shared calibration module. evaluateGateCheckCore stays sync/pure (engine parity twin); the env-bearing
// caller awaits {@link recordConfiguredGateBlockerSignals} with the same advisory+policy so this filter and
// the recording loop stay in lock-step.
const qualityWarning = buildQualityGateWarning(effective);
const slopBlocker = buildSlopGateBlocker(effective);
const blockers = [...configuredBlockers, ...(slopBlocker ? [slopBlocker] : [])];
Expand Down Expand Up @@ -1025,6 +1053,41 @@ function isConfiguredGateBlocker(finding: AdvisoryFinding, policy: GateCheckPoli
return false;
}

/**
* Record a {@link RuleFiredEvent} for every finding that `isConfiguredGateBlocker` would put into
* `configuredBlockers`, excluding `linked_issue_scope_mismatch` (#8104 / complements #8101). Call from the
* env-bearing gate path immediately after {@link evaluateGateCheck} with the SAME advisory + policy so the
* filter matches `evaluateGateCheckCore`'s own. Best-effort: a SignalStore failure never throws and never
* affects the gate verdict.
*/
export async function recordConfiguredGateBlockerSignals(
env: Env,
advisoryResult: Advisory,
policy: GateCheckPolicy,
repoFullName: string,
prNumber: number,
): Promise<void> {
const effective = applyMergeReadinessGate(policy);
const configuredBlockers = advisoryResult.findings.filter((finding) => isConfiguredGateBlocker(finding, effective));
const store = createSignalStore(env);
const targetKey = `${repoFullName}#${prNumber}`;
const occurredAt = nowIso();
await Promise.all(
configuredBlockers.map((finding) => {
if (finding.code === "linked_issue_scope_mismatch") return Promise.resolve();
return store
.recordRuleFired({
ruleId: finding.code,
targetKey,
outcome: finding.severity ?? "blocker",
occurredAt,
...(finding.confidence !== undefined ? { metadata: { confidence: finding.confidence } } : {}),
})
.catch(() => undefined);
}),
);
}

function buildQualityGateWarning(policy: GateCheckPolicy): AdvisoryFinding | null {
if (gateMode(policy.qualityGateMode) === "off") return null;
const score = normalizeScore(policy.readinessScore);
Expand Down
167 changes: 167 additions & 0 deletions test/unit/configured-gate-blocker-signals.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import {
recordConfiguredGateBlockerSignals,
type GateCheckPolicy,
} from "../../src/rules/advisory";
import * as signalTrackingWire from "../../src/review/signal-tracking-wire";
import { createSignalStore } from "../../src/review/signal-tracking-wire";
import type { Advisory, AdvisoryFinding } from "../../src/types";
import { createTestEnv } from "../helpers/d1";

function finding(over: Partial<AdvisoryFinding> & Pick<AdvisoryFinding, "code">): AdvisoryFinding {
return {
title: over.title ?? over.code,
severity: over.severity ?? "warning",
detail: over.detail ?? `${over.code} detail`,
action: over.action ?? "fix it",
...over,
};
}

function advisory(findings: AdvisoryFinding[]): Advisory {
return {
id: "advisory-8104",
targetType: "pull_request",
targetKey: "owner/repo#7",
repoFullName: "owner/repo",
pullNumber: 7,
headSha: "abc",
conclusion: "neutral",
severity: "warning",
title: "advisory",
summary: `${findings.length} finding(s)`,
findings,
generatedAt: "2026-07-22T00:00:00.000Z",
};
}

const blockAi: GateCheckPolicy = { aiReviewGateMode: "block" };
const blockLinked: GateCheckPolicy = { linkedIssueGateMode: "block" };
const blockSatisfaction: GateCheckPolicy = { linkedIssueSatisfactionGateMode: "block" };

describe("recordConfiguredGateBlockerSignals (#8104)", () => {
afterEach(() => {
vi.restoreAllMocks();
});

it("records a fired signal for ai_consensus_defect when it is a configured gate blocker", async () => {
const env = createTestEnv();
await recordConfiguredGateBlockerSignals(
env,
advisory([finding({ code: "ai_consensus_defect", confidence: 0.95 })]),
blockAi,
"owner/repo",
7,
);
const history = await createSignalStore(env).queryRuleHistory("ai_consensus_defect", 0);
expect(history.fired).toHaveLength(1);
expect(history.fired[0]).toMatchObject({
ruleId: "ai_consensus_defect",
targetKey: "owner/repo#7",
outcome: "warning",
metadata: { confidence: 0.95 },
});
});

it("records a fired signal for ai_review_split when it is a configured gate blocker", async () => {
const env = createTestEnv();
await recordConfiguredGateBlockerSignals(
env,
advisory([finding({ code: "ai_review_split", severity: "critical" })]),
blockAi,
"owner/repo",
7,
);
const history = await createSignalStore(env).queryRuleHistory("ai_review_split", 0);
expect(history.fired).toHaveLength(1);
expect(history.fired[0]).toMatchObject({
ruleId: "ai_review_split",
targetKey: "owner/repo#7",
outcome: "critical",
});
expect(history.fired[0]?.metadata).toBeUndefined();
});

it("records a fired signal for a deterministic code (secret_leak)", async () => {
const env = createTestEnv();
await recordConfiguredGateBlockerSignals(
env,
advisory([finding({ code: "secret_leak", severity: "critical" })]),
{},
"owner/repo",
7,
);
const history = await createSignalStore(env).queryRuleHistory("secret_leak", 0);
expect(history.fired).toHaveLength(1);
expect(history.fired[0]).toMatchObject({
ruleId: "secret_leak",
targetKey: "owner/repo#7",
outcome: "critical",
});
});

it("records a fired signal for missing_linked_issue when linkedIssueGateMode is block", async () => {
const env = createTestEnv();
await recordConfiguredGateBlockerSignals(
env,
advisory([finding({ code: "missing_linked_issue" })]),
blockLinked,
"owner/repo",
7,
);
expect((await createSignalStore(env).queryRuleHistory("missing_linked_issue", 0)).fired).toHaveLength(1);
});

it("records NO fired signal for linked_issue_scope_mismatch even when it is a configured blocker (#8101 owns it)", async () => {
const env = createTestEnv();
await recordConfiguredGateBlockerSignals(
env,
advisory([finding({ code: "linked_issue_scope_mismatch" }), finding({ code: "secret_leak", severity: "critical" })]),
blockSatisfaction,
"owner/repo",
7,
);
expect((await createSignalStore(env).queryRuleHistory("linked_issue_scope_mismatch", 0)).fired).toEqual([]);
expect((await createSignalStore(env).queryRuleHistory("secret_leak", 0)).fired).toHaveLength(1);
});

it("records NO fired signal when isConfiguredGateBlocker returns false", async () => {
const env = createTestEnv();
// missing_linked_issue defaults to advisory — not a configured blocker.
await recordConfiguredGateBlockerSignals(
env,
advisory([finding({ code: "missing_linked_issue" })]),
{ linkedIssueGateMode: "advisory" },
"owner/repo",
7,
);
expect((await createSignalStore(env).queryRuleHistory("missing_linked_issue", 0)).fired).toEqual([]);
});

it("uses outcome 'blocker' when finding.severity is missing (nullish coalescing arm)", async () => {
const env = createTestEnv();
const noSeverity = finding({ code: "secret_leak" });
delete (noSeverity as { severity?: AdvisoryFinding["severity"] }).severity;
await recordConfiguredGateBlockerSignals(env, advisory([noSeverity]), {}, "owner/repo", 7);
expect((await createSignalStore(env).queryRuleHistory("secret_leak", 0)).fired[0]?.outcome).toBe("blocker");
});

it("degrades silently when the SignalStore write rejects: nothing throws", async () => {
vi.spyOn(signalTrackingWire, "createSignalStore").mockReturnValue({
recordRuleFired: async () => {
throw new Error("signal store down");
},
recordHumanOverride: async () => undefined,
queryRuleHistory: async () => ({ fired: [], overrides: [] }),
});
await expect(
recordConfiguredGateBlockerSignals(
createTestEnv(),
advisory([finding({ code: "secret_leak", severity: "critical" })]),
{},
"owner/repo",
7,
),
).resolves.toBeUndefined();
});
});
Loading