diff --git a/docs/review-configuration.md b/docs/review-configuration.md index 7e23f92be5..940ee2f95e 100644 --- a/docs/review-configuration.md +++ b/docs/review-configuration.md @@ -111,6 +111,7 @@ already-enabled gate. | AI review BYOK | `gate.aiReview.byok` | `aiReviewByok` | bool | `false` | When `true` and a provider key is configured, the *advisory* write-up uses the maintainer's frontier model. The consensus blocker always uses the free Workers-AI pair, so BYOK never changes who can be blocked. | | AI review provider | `gate.aiReview.provider` | `aiReviewProvider` | `anthropic` / `openai` / `null` | `null` | `null` = use the stored key's own provider. Must match the stored key's provider or BYOK is skipped (Workers-AI fallback). The key itself is only in the encrypted key store. | | AI review model | `gate.aiReview.model` | `aiReviewModel` | string / `null` | `null` | Model override for the BYOK advisory write-up (e.g. `claude-3-5-sonnet-latest`). `null` = the key record's model, else a conservative per-provider default. | +| AI close confidence | `gate.aiReview.closeConfidence` | `aiReviewCloseConfidence` | number 0–1 (nullable) | `null` (engine uses `0.9`) | Minimum **calibrated** AI-reviewer confidence for a consensus defect / split to **block** under `aiReview.mode: block`. Below-threshold AI defects stay advisory (visible, never close). Each reviewer rates its own confidence; consensus carries the weaker reviewer's. Config-as-code only (no dashboard/DB column). | ### Guardrails and scope (focus manifest) diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 8b672af480..4e62e4173b 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -3356,6 +3356,9 @@ export function gateCheckPolicy( qualityGateMode: settings.qualityGateMode, qualityGateMinScore: settings.qualityGateMinScore ?? null, aiReviewGateMode: settings.aiReviewMode, + // Calibrated AI close-confidence floor (#7) — config-as-code via `.gittensory.yml gate.aiReview.closeConfidence`, + // resolved into settings upstream. `null`/undefined ⇒ advisory.ts applies the 0.9 default. + aiReviewCloseConfidence: settings.aiReviewCloseConfidence ?? null, readinessScore: readinessScore ?? null, slopGateMode: settings.slopGateMode, mergeReadinessGateMode: settings.mergeReadinessGateMode, @@ -3762,6 +3765,8 @@ export async function runAiReviewForAdvisory( detail: result.consensusDefect.detail, action: "Resolve the flagged defect, or override if the AI reviewers are mistaken, then re-run the gate.", + // Calibrated confidence (#8): the gate blocks this defect only when it clears aiReviewCloseConfidence. + confidence: result.consensusDefect.confidence, }); } else if (result.split) { // The reviewers DISAGREED — exactly one flagged a blocking defect. reviewbot's quorum: ANY reviewer @@ -3775,6 +3780,14 @@ export async function runAiReviewForAdvisory( "One AI reviewer independently flagged a concrete must-fix defect in this change (the other did not). Under the quorum rule, a single rejection closes the PR; see the review notes for specifics.", action: "Resolve the flagged defect and open a new pull request, or override if the reviewers are mistaken.", + // Calibrated confidence (#8) of the lone flagging reviewer; the gate blocks only when it clears + // aiReviewCloseConfidence. A consensus split ALWAYS carries this (combineReviews sets it whenever split is + // true), so the spread is effectively unconditional; the guard is a defensive belt-and-braces — an absent + // value would degrade to 1.0 in the threshold check (advisory.ts `?? 1`), matching today's always-block. + /* v8 ignore next 3 -- a split always carries splitConfidence; the absent arm is an unreachable guard. */ + ...(result.splitConfidence !== undefined + ? { confidence: result.splitConfidence } + : {}), }); } else if (result.inconclusive) { // Fail-CLOSED (#ai-fail-closed): block-mode AI could not return a usable verdict. Hold the PR for a human diff --git a/src/rules/advisory.ts b/src/rules/advisory.ts index 2cc2dfccb8..b169b2012c 100644 --- a/src/rules/advisory.ts +++ b/src/rules/advisory.ts @@ -23,6 +23,12 @@ export type GateCheckPolicy = { /** When `block`, a dual-model AI consensus defect (`ai_consensus_defect` finding) becomes a hard * blocker. Defaults to advisory — AI never blocks unless the maintainer opts in. */ aiReviewGateMode?: GateRuleMode | undefined; + /** Minimum calibrated confidence (0-1) for an AI-judgment defect (`ai_consensus_defect` / `ai_review_split`) to + * BLOCK under `aiReviewGateMode: block` (#7). The finding blocks only when its `confidence >= this`; below-threshold + * AI defects stay advisory (visible, never block). `null`/undefined ⇒ the 0.9 default. A finding with no + * confidence (deterministic, or a graceful-fallback AI defect) is treated as 1.0 and always clears the floor — + * matching the historical always-block behavior. */ + aiReviewCloseConfidence?: number | null | undefined; readinessScore?: number | null | undefined; /** When `block`, the deterministic slop score becomes a hard blocker once `slopRisk >= slopGateMinScore` * (default threshold 60, the `high` band). Defaults to off/advisory — slop never blocks unless opted in. */ @@ -477,7 +483,7 @@ function evaluateGateCheckCore(advisoryResult: Advisory, policy: GateCheckPolicy // Merge-readiness composite (#551): when set, escalate every sub-gate to its mode so they roll into one // pass/fail. When off, this is a no-op and each sub-gate keeps its own mode. const effective = applyMergeReadinessGate(policy); - const configuredBlockers = advisoryResult.findings.filter((finding) => isConfiguredGateBlocker(finding.code, effective)); + const configuredBlockers = advisoryResult.findings.filter((finding) => isConfiguredGateBlocker(finding, effective)); const qualityBlocker = buildQualityGateBlocker(effective); const slopBlocker = buildSlopGateBlocker(effective); const blockers = [...configuredBlockers, ...(qualityBlocker ? [qualityBlocker] : []), ...(slopBlocker ? [slopBlocker] : [])]; @@ -810,17 +816,29 @@ function isEvaluationBlocker(code: string): boolean { return code === "repo_not_registered" || code === "repo_not_seen" || code === "pr_not_cached" || code === "pre_merge_check_unresolved"; } -function isConfiguredGateBlocker(code: string, policy: GateCheckPolicy): boolean { +// Default minimum calibrated confidence for an AI defect to BLOCK (#7) — used when the repo set `aiReview: block` +// without a `closeConfidence`. 0.9 = block only on a high-confidence AI defect; below that stays advisory. +const DEFAULT_AI_REVIEW_CLOSE_CONFIDENCE = 0.9; + +function isConfiguredGateBlocker(finding: AdvisoryFinding, policy: GateCheckPolicy): boolean { + const code = finding.code; // Missing linked issue defaults to ADVISORY — issues aren't always available, so it only blocks when a // repo explicitly opts in with linkedIssueGateMode: "block". Duplicates still default to blocking. if (code === "missing_linked_issue") return gateMode(policy.linkedIssueGateMode ?? "advisory") === "block"; if (code === "duplicate_pr_risk") return gateMode(policy.duplicatePrGateMode ?? "block") === "block"; // A dual-model AI consensus defect blocks ONLY when the maintainer opted into aiReview: block. It is the - // most conservative AI signal (two independent models, high confidence) but still confirmed-contributor - // gated by evaluateGateCheck, and advisory by default. + // most conservative AI signal (two independent models) but still confirmed-contributor gated by + // evaluateGateCheck, and advisory by default. // A consensus defect (both reviewers) OR a SPLIT (one reviewer flagged a blocker the other did not) both block - // when aiReviewGateMode is `block` — reviewbot's quorum: ANY reviewer rejection closes the PR. (#ai-review-split) - if (code === "ai_consensus_defect" || code === "ai_review_split") return gateMode(policy.aiReviewGateMode ?? "advisory") === "block"; + // when aiReviewGateMode is `block` AND the finding's CALIBRATED confidence clears the close-confidence floor (#7): + // the historical hardcoded `confidence: 1` always blocked, so a below-floor AI defect now stays advisory (visible, + // never closes) instead of false-closing. A finding with no confidence (graceful fallback) is treated as 1.0 and + // always clears the floor — byte-identical to today's behavior. (#ai-review-split) + if (code === "ai_consensus_defect" || code === "ai_review_split") { + if (gateMode(policy.aiReviewGateMode ?? "advisory") !== "block") return false; + const confidence = finding.confidence ?? 1; + return confidence >= (policy.aiReviewCloseConfidence ?? DEFAULT_AI_REVIEW_CLOSE_CONFIDENCE); + } // A leaked-secret finding (`secret_leak`) ALWAYS hard-blocks: a committed credential must be removed and // rotated before merge, with no opt-in. This finding is produced ONLY by the flag-gated safety scan // (GITTENSORY_REVIEW_SAFETY); when the flag is off the finding never exists, so this branch is unreachable and the diff --git a/src/rules/predicted-gate.ts b/src/rules/predicted-gate.ts index de0d4767fc..90fce89be4 100644 --- a/src/rules/predicted-gate.ts +++ b/src/rules/predicted-gate.ts @@ -224,6 +224,7 @@ export function buildPredictedGateVerdict(args: { qualityGateMode: gate.readinessMode ?? undefined, qualityGateMinScore: gate.readinessMinScore ?? null, aiReviewGateMode: gate.aiReviewMode ?? undefined, + aiReviewCloseConfidence: gate.aiReviewCloseConfidence ?? null, mergeReadinessGateMode: gate.mergeReadiness ?? undefined, // #12: only meaningful when changed paths were supplied (the policy findings are pushed above only then); // absent paths ⇒ no manifest finding exists, so this mode has nothing to act on (byte-identical). diff --git a/src/services/ai-review.ts b/src/services/ai-review.ts index 3be8fc5146..032f052f05 100644 --- a/src/services/ai-review.ts +++ b/src/services/ai-review.ts @@ -49,9 +49,10 @@ const REVIEW_SYSTEM_PROMPT = [ "You are a senior open-source maintainer giving a FOCUSED, high-signal code review of a single pull request diff.", "Read each meaningful hunk and review like a careful human; judge ONLY the diff and the context provided.", "Respond with ONLY a JSON object of this exact shape (no prose, no code fence):", - '{"assessment": string, "blockers": string[], "nits": string[], "suggestions": string[]}', + '{"assessment": string, "blockers": string[], "nits": string[], "suggestions": string[], "confidence": number}', "- assessment: a substantive but CONCISE summary (2-4 sentences) — what the change does, whether it is correct, and the most notable detail. Specific to THIS diff; never a generic one-liner and never hedging ('appears to', 'seems to').", "- blockers: each ONE sentence naming a defect that WILL break the code as written — a missing import/symbol (ReferenceError), a logic error that produces wrong output, a security hole, data loss, a build/test breakage, or an API/contract break. Reference the file (and function/line). Empty [] if there are genuinely none.", + "- confidence: a single number in [0,1] — your CALIBRATED probability that the blockers above are REAL, must-fix defects (not false positives). Use 1.0 only when you are certain the diff itself breaks; use 0.5 for a genuine coin-flip; lower it when you cannot fully see the breaking code or the defect is speculative. When blockers is empty, set confidence to 1.0.", "- nits: each ONE sentence — a NON-blocking point: style, naming, a missing doc, or DEFENSIVE hardening ('should handle the empty case', 'consider catching errors', 'add validation'). File-reference where you can.", "- suggestions: a few concrete, file-referenced improvements (may overlap nits).", "BE SELECTIVE — report only the findings that genuinely matter. List at MOST ~3 blockers and ~5 nits, keeping only the most important; prefer signal over volume and do NOT pad the lists.", @@ -193,6 +194,9 @@ export type GittensoryAiReviewResult = advisoryNotes: string | null; consensusDefect: AiConsensusDefect | null; split: boolean; + /** Calibrated confidence of the lone reviewer whose blocker caused a SPLIT (#8), so the `ai_review_split` + * finding carries the same confidence as a consensus defect would. Present only when `split` is true. */ + splitConfidence?: number; inconclusive: boolean; estimatedNeurons: number; reviewerCount: number; @@ -216,6 +220,11 @@ export type ModelReview = { blockers: string[]; nits: string[]; suggestions: string[]; + // Calibrated confidence in [0,1] (#8): the reviewer's own probability that its blocker(s) are a REAL defect. Drives + // the gate's `aiReviewCloseConfidence` floor (an AI defect blocks only when confidence clears it). parseModelReview + // sets it from the model's `confidence` field; an absent/unparseable/out-of-range value degrades to 1.0 (FALLBACK), + // so behavior matches the historical hardcoded `confidence: 1` until a calibrated value is actually present. + confidence: number; // Line-anchored findings for inline PR review comments (#inline-comments). ALWAYS present (parseModelReview // sets []); populated only when the caller asked for them (input.inlineFindings) AND the model emitted any. inlineFindings: InlineFinding[]; @@ -352,6 +361,20 @@ export function extractLastJsonObject(text: string): string | null { return last; } +/** Default reviewer confidence when the model omits a usable `confidence` (#8) — 1.0, so an absent/garbage value + * degrades to EXACTLY the historical hardcoded `confidence: 1` (a defect always cleared the floor). Shared by the + * parser and the combiners so the fallback is identical everywhere. */ +export const DEFAULT_REVIEW_CONFIDENCE = 1; + +/** Coerce a model's `confidence` field to a calibrated value in [0,1] (#8). A finite number is clamped into range; + * anything else (absent, NaN/±Infinity — which JSON can't even encode — string, etc.) falls back to 1.0 so the gate + * degrades to today's always-block behavior rather than silently un-blocking a real defect. PURE. */ +export function parseReviewConfidence(value: unknown): number { + if (typeof value !== "number" || !Number.isFinite(value)) + return DEFAULT_REVIEW_CONFIDENCE; + return Math.min(1, Math.max(0, value)); +} + /** Parse a model's JSON review into a normalized {@link ModelReview}, or null when unparseable. */ export function parseModelReview(text: string): ModelReview | null { const jsonText = extractLastJsonObject(text); @@ -393,6 +416,9 @@ export function parseModelReview(text: string): ModelReview | null { const nits = toList(obj.nits); const suggestions = toList(obj.suggestions); const inlineFindings = toInlineFindings(obj.inlineFindings); + // Calibrated reviewer confidence (#8): clamp the model's `confidence` to [0,1]; an absent/garbage value falls + // back to 1.0 (parseReviewConfidence) so the gate degrades to the historical always-block behavior. + const confidence = parseReviewConfidence(obj.confidence); if (assessment === INCOHERENT_DIFF_ASSESSMENT) return null; if ( !assessment && @@ -401,7 +427,7 @@ export function parseModelReview(text: string): ModelReview | null { suggestions.length === 0 ) return null; - return { assessment, blockers, nits, suggestions, inlineFindings }; + return { assessment, blockers, nits, suggestions, inlineFindings, confidence }; } catch { return null; } @@ -692,8 +718,8 @@ export function composeInlineFindings(reviews: ModelReview[]): InlineFinding[] { /** A CONSENSUS defect = BOTH reviews independently name at least one concrete blocker (the severity-disciplined * reviewbot model: a lone blocker in a dual review is a split, not a hard block). Requiring two independent - * models to AGREE is itself the precision mechanism — the free Workers-AI models emit no calibrated confidence - * score, so there is no numeric floor to enforce; agreement is the signal. */ + * models to AGREE is itself the precision mechanism; the calibrated confidence (#8) ADDS a numeric floor on top — + * a consensus is only as strong as its WEAKER reviewer, so the defect carries `min(a.confidence, b.confidence)`. */ export function consensusDefectOf( a: ModelReview, b: ModelReview, @@ -710,23 +736,30 @@ export function consensusDefectOf( const detail = toPublicSafe(a.blockers[0] || b.blockers[0] || "") ?? "Both AI reviewers independently flagged a concrete must-fix defect in this change."; - return { title, detail, confidence: 1 }; + // The consensus is only as strong as the WEAKER reviewer: take the minimum of the two confidences (#8). + return { title, detail, confidence: Math.min(a.confidence, b.confidence) }; } /** Deterministic SYNTHESIS of one public-safe defect from the reviews that named a blocker — same public-safe * discipline as `consensusDefectOf` (cite the primary blocker; an unsafe title drops the whole block, fail-safe). - * Used by the `synthesis` and `single` combine strategies. */ + * Used by the `synthesis` and `single` combine strategies. The defect carries the CONFIDENCE of the reviewer that + * supplied the cited primary blocker (#8) — for `single` that is that one reviewer's confidence. */ function synthesizeDefect( reviews: ReadonlyArray, ): AiConsensusDefect | null { - const primary = reviews - .flatMap((r) => r.blockers) + // Find the FIRST reviewer with a non-blank blocker so the cited title + the carried confidence come from the + // SAME reviewer (a flat-map would divorce the blocker text from its reviewer's confidence). + const source = reviews.find((r) => + r.blockers.some((b) => b.trim().length > 0), + ); + const primary = source?.blockers .map((b) => b.trim()) .find((b) => b.length > 0); - if (!primary) return null; + if (!source || !primary) return null; const title = toPublicSafe(primary); if (!title) return null; // unsafe title → drop the block entirely (fail-safe) - return { title, detail: title, confidence: 1 }; // cite the primary blocker as both title + detail + // cite the primary blocker as both title + detail; confidence = the flagging reviewer's calibrated confidence. + return { title, detail: title, confidence: source.confidence }; } /** Combine the independent reviewer opinions into ONE gate decision per the configured strategy (#dual-ai-combiner). @@ -738,7 +771,13 @@ function synthesizeDefect( export function combineReviews( reviews: ReadonlyArray, opts: { strategy: CombineStrategy; onMerge?: OnMerge | null | undefined }, -): { defect: AiConsensusDefect | null; split: boolean; inconclusive: boolean } { +): { + defect: AiConsensusDefect | null; + split: boolean; + inconclusive: boolean; + /** The lone-flagging reviewer's calibrated confidence when `split` is true (#8); absent otherwise. */ + splitConfidence?: number; +} { const present = reviews.filter((r): r is ModelReview => Boolean(r)); const missing = reviews.length - present.length; @@ -779,12 +818,21 @@ export function combineReviews( return { defect: null, split: false, inconclusive: missing > 0 }; } - // `consensus` (default) — BYTE-IDENTICAL to the historical block-mode pair logic. + // `consensus` (default) — the historical block-mode pair logic, now ALSO surfacing the split's confidence (#8). const [a, b] = reviews; if (a && b) { const defect = consensusDefectOf(a, b); const split = !defect && a.blockers.length > 0 !== b.blockers.length > 0; - return { defect, split, inconclusive: false }; + // On a split, exactly one reviewer flagged a blocker — carry THAT reviewer's confidence so the + // `ai_review_split` finding gates on the same calibrated floor a consensus defect would. + return split + ? { + defect, + split, + inconclusive: false, + splitConfidence: a.blockers.length > 0 ? a.confidence : b.confidence, + } + : { defect, split, inconclusive: false }; } return { defect: null, split: false, inconclusive: true }; } @@ -948,6 +996,7 @@ export async function runGittensoryAiReview( let consensusDefect: AiConsensusDefect | null = null; let secondReview: ModelReview | null = null; let aiReviewSplit = false; + let splitConfidence: number | undefined; let inconclusive = false; if (input.mode === "block") { if (dual) { @@ -981,6 +1030,7 @@ export async function runGittensoryAiReview( const combined = combineReviews([a, b], { strategy: combine, onMerge }); consensusDefect = combined.defect; aiReviewSplit = combined.split; + splitConfidence = combined.splitConfidence; inconclusive = combined.inconclusive; } else { // Single reviewer: its verdict IS the decision. Reuse the advisory leg (non-BYOK) or run the one reviewer. @@ -1039,6 +1089,9 @@ export async function runGittensoryAiReview( advisoryNotes, consensusDefect, split: aiReviewSplit, + // Carry the split's calibrated confidence (#8) so the caller can gate `ai_review_split` on the same floor as a + // consensus defect. Only present on a split (combineReviews leaves it undefined otherwise). + ...(splitConfidence !== undefined ? { splitConfidence } : {}), inconclusive, estimatedNeurons, reviewerCount: reviewsForNotes.length, @@ -1084,6 +1137,7 @@ async function record( export const __aiReviewInternals = { parseModelReview, + parseReviewConfidence, coerceAiText, composeAdvisoryNotes, composeInlineFindings, diff --git a/src/signals/focus-manifest.ts b/src/signals/focus-manifest.ts index 43f3728882..b3de51be64 100644 --- a/src/signals/focus-manifest.ts +++ b/src/signals/focus-manifest.ts @@ -33,6 +33,9 @@ export type FocusManifestGateConfig = { aiReviewProvider: "anthropic" | "openai" | null; aiReviewModel: string | null; aiReviewAllAuthors: boolean | null; + /** `gate.aiReview.closeConfidence` (#7): minimum calibrated AI-reviewer confidence (0-1) for an AI defect to BLOCK + * under `aiReview.mode: block`. null (unset) ⇒ the gate's 0.9 default. Clamped to [0,1] at parse time. */ + aiReviewCloseConfidence: number | null; mergeReadiness: GateRuleMode | null; manifestPolicy: GateRuleMode | null; selfAuthoredLinkedIssue: GateRuleMode | null; @@ -247,6 +250,7 @@ const EMPTY_GATE_CONFIG: FocusManifestGateConfig = { aiReviewProvider: null, aiReviewModel: null, aiReviewAllAuthors: null, + aiReviewCloseConfidence: null, mergeReadiness: null, manifestPolicy: null, selfAuthoredLinkedIssue: null, @@ -360,6 +364,18 @@ function normalizeOptionalScore(value: JsonValue | undefined, field: string, war return Math.max(0, Math.min(100, Math.round(value))); } +/** Normalize an optional confidence threshold in [0,1] (#7) — a fractional value (NOT a 0-100 score), so it is + * clamped into range WITHOUT rounding. Absent/null ⇒ null (the resolver leaves the gate's 0.9 default in place); + * a non-finite/non-number value is ignored with a warning. */ +function normalizeOptionalConfidence(value: JsonValue | undefined, field: string, warnings: string[]): number | null { + if (value === undefined || value === null) return null; + if (typeof value !== "number" || !Number.isFinite(value)) { + warnings.push(`Manifest gate field "${field}" must be a number between 0 and 1; ignoring it.`); + return null; + } + return Math.max(0, Math.min(1, value)); +} + /** * Parse the optional `gate:` mapping. Every field stays `null` when unset so the resolver can layer * this OVER DB settings without clobbering. A nested `readiness: { mode, minScore }` block is accepted. @@ -408,6 +424,7 @@ function parseGateConfig(value: JsonValue | undefined, warnings: string[]): Focu aiReviewProvider: normalizeOptionalEnum(aiReviewRecord?.provider, "gate.aiReview.provider", ["anthropic", "openai"] as const, warnings), aiReviewModel: normalizeOptionalString(aiReviewRecord?.model, "gate.aiReview.model", warnings), aiReviewAllAuthors: normalizeOptionalBoolean(aiReviewRecord?.allAuthors, "gate.aiReview.allAuthors", warnings), + aiReviewCloseConfidence: normalizeOptionalConfidence(aiReviewRecord?.closeConfidence, "gate.aiReview.closeConfidence", warnings), mergeReadiness: normalizeOptionalGateMode(record.mergeReadiness, "gate.mergeReadiness", warnings), manifestPolicy: normalizeOptionalGateMode(record.manifestPolicy, "gate.manifestPolicy", warnings), selfAuthoredLinkedIssue: normalizeOptionalGateMode(record.selfAuthoredLinkedIssue, "gate.selfAuthoredLinkedIssue", warnings), @@ -430,6 +447,7 @@ function parseGateConfig(value: JsonValue | undefined, warnings: string[]): Focu gate.aiReviewProvider !== null || gate.aiReviewModel !== null || gate.aiReviewAllAuthors !== null || + gate.aiReviewCloseConfidence !== null || gate.mergeReadiness !== null || gate.manifestPolicy !== null || gate.selfAuthoredLinkedIssue !== null || @@ -463,13 +481,14 @@ export function gateConfigToJson(gate: FocusManifestGateConfig): JsonValue { if (gate.slopAiAdvisory !== null) slop.aiAdvisory = gate.slopAiAdvisory; out.slop = slop; } - if (gate.aiReviewMode !== null || gate.aiReviewByok !== null || gate.aiReviewProvider !== null || gate.aiReviewModel !== null || gate.aiReviewAllAuthors !== null) { + if (gate.aiReviewMode !== null || gate.aiReviewByok !== null || gate.aiReviewProvider !== null || gate.aiReviewModel !== null || gate.aiReviewAllAuthors !== null || gate.aiReviewCloseConfidence !== null) { const aiReview: Record = {}; if (gate.aiReviewMode !== null) aiReview.mode = gate.aiReviewMode; if (gate.aiReviewByok !== null) aiReview.byok = gate.aiReviewByok; if (gate.aiReviewProvider !== null) aiReview.provider = gate.aiReviewProvider; if (gate.aiReviewModel !== null) aiReview.model = gate.aiReviewModel; if (gate.aiReviewAllAuthors !== null) aiReview.allAuthors = gate.aiReviewAllAuthors; + if (gate.aiReviewCloseConfidence !== null) aiReview.closeConfidence = gate.aiReviewCloseConfidence; out.aiReview = aiReview; } if (gate.mergeReadiness !== null) out.mergeReadiness = gate.mergeReadiness; @@ -940,6 +959,7 @@ export function resolveEffectiveSettings(dbSettings: RepositorySettings, manifes if (gate.aiReviewProvider !== null) effective.aiReviewProvider = gate.aiReviewProvider; if (gate.aiReviewModel !== null) effective.aiReviewModel = gate.aiReviewModel; if (gate.aiReviewAllAuthors !== null) effective.aiReviewAllAuthors = gate.aiReviewAllAuthors; + if (gate.aiReviewCloseConfidence !== null) effective.aiReviewCloseConfidence = gate.aiReviewCloseConfidence; if (gate.mergeReadiness !== null) effective.mergeReadinessGateMode = gate.mergeReadiness; if (gate.manifestPolicy !== null) effective.manifestPolicyGateMode = gate.manifestPolicy; if (gate.selfAuthoredLinkedIssue !== null) effective.selfAuthoredLinkedIssueGateMode = gate.selfAuthoredLinkedIssue; diff --git a/src/types.ts b/src/types.ts index d59a2fc179..d5728731c0 100644 --- a/src/types.ts +++ b/src/types.ts @@ -376,6 +376,12 @@ export type AdvisoryFinding = { detail: string; action?: string; publicText?: string; + /** Calibrated confidence in [0,1] for an AI-judgment finding (`ai_consensus_defect` / `ai_review_split`) — the + * reviewer's own probability that the flagged blocker is a real defect (#8). The gate's `aiReviewCloseConfidence` + * threshold uses it: an AI defect blocks ONLY when this clears the floor. Absent for deterministic findings (they + * carry no model confidence); an absent/unparseable reviewer confidence degrades to 1.0 upstream, so omitting it + * here behaves exactly like today. */ + confidence?: number; }; export type Advisory = { @@ -556,6 +562,12 @@ export type RepositorySettings = { * AI themselves. Default false — opt-in via `.gittensory.yml gate.aiReview.allAuthors`. Independent of * `aiReviewMode`: `off` still means no AI; this only widens WHO an enabled review covers. */ aiReviewAllAuthors: boolean; + /** Minimum calibrated AI-reviewer confidence (0-1) for an AI defect to BLOCK under `aiReviewMode: block` (#7). + * A dual-model consensus defect / split blocks only when its finding `confidence >= aiReviewCloseConfidence`; + * below-threshold AI defects stay advisory (visible, never block). Config-as-code only — set via + * `.gittensory.yml gate.aiReview.closeConfidence` (no dashboard/DB column); unset ⇒ the gate uses the 0.9 + * default. Clamped to [0,1] at parse time. */ + aiReviewCloseConfidence?: number | null | undefined; /** When TRUE, the repo OWNER's (and maintainer's) own PRs are eligible for auto-CLOSE like a contributor's * (still subject to the `close` autonomy class + the same adverse-signal conditions). Default FALSE — owner * PRs are exempt from auto-close (merge or manual-hold only). Per-repo configurable so maintainers choose diff --git a/test/unit/ai-review-advisory.test.ts b/test/unit/ai-review-advisory.test.ts index 996bfc7ce7..de596bf3a7 100644 --- a/test/unit/ai-review-advisory.test.ts +++ b/test/unit/ai-review-advisory.test.ts @@ -212,6 +212,23 @@ describe("runAiReviewForAdvisory", () => { expect(result?.notes).toContain("Likely crash."); }); + it("threads the calibrated MIN consensus confidence onto the ai_consensus_defect finding (#8)", async () => { + const adv = advisory(); + // Two reviewers agree on a blocker but with DIFFERENT confidences (0.95 vs 0.6) → the finding carries the min. + const json = (confidence: number) => JSON.stringify({ assessment: "Likely crash.", blockers: ["Null dereference of a possibly-null value in src/a.ts."], nits: [], suggestions: [], confidence }); + const run = (async (model: string) => ({ response: model === BEST_REVIEW_MODELS[0] ? json(0.95) : json(0.6) })) as unknown as () => Promise; + await runAiReviewForAdvisory(aiEnv(run), { + settings: { aiReviewMode: "block" } as RepositorySettings, + advisory: adv, + repoFullName: "acme/widgets", + pr, + author: "alice", + confirmedContributor: true, + }); + expect(adv.findings[0]?.code).toBe("ai_consensus_defect"); + expect(adv.findings[0]?.confidence).toBe(0.6); // weaker reviewer governs the gate floor + }); + it("appends an ai_review_inconclusive finding (fail-closed hold) when block-mode AI lacks a second opinion, surfacing it to Sentry as an error", async () => { const adv = advisory(); // The first slot parses; the second slot's primary AND its reliable fallback fail → no consensus possible. @@ -279,6 +296,23 @@ describe("runAiReviewForAdvisory", () => { expect(result?.notes).toBeDefined(); }); + it("threads the lone flagging reviewer's calibrated confidence onto the ai_review_split finding (#8)", async () => { + const adv = advisory(); + // Only the FIRST reviewer flags a blocker, at confidence 0.45 → split, and the finding carries 0.45. + const flagged = JSON.stringify({ assessment: "Likely crash.", blockers: ["Null deref in src/a.ts."], nits: [], suggestions: [], confidence: 0.45 }); + const run = (async (model: string) => ({ response: model === BEST_REVIEW_MODELS[0] ? flagged : notesOnlyJson() })) as unknown as () => Promise; + await runAiReviewForAdvisory(aiEnv(run), { + settings: { aiReviewMode: "block" } as RepositorySettings, + advisory: adv, + repoFullName: "acme/widgets", + pr, + author: "alice", + confirmedContributor: true, + }); + expect(adv.findings[0]?.code).toBe("ai_review_split"); + expect(adv.findings[0]?.confidence).toBe(0.45); + }); + it("uses the caller's pre-resolved files (FIX B) instead of the stored read, so the model sees the real diff", async () => { // FIX B: the processor passes `files` (its resolvePullRequestFilesForReview output). With no rows ever // written to the test DB, a stored read would yield an EMPTY diff; passing files proves the model gets the diff --git a/test/unit/ai-review.test.ts b/test/unit/ai-review.test.ts index 61265b3bf3..19b964d982 100644 --- a/test/unit/ai-review.test.ts +++ b/test/unit/ai-review.test.ts @@ -9,11 +9,13 @@ import { createTestEnv } from "../helpers/d1"; const { parseModelReview, + parseReviewConfidence, coerceAiText, composeAdvisoryNotes, composeInlineFindings, consensusDefectOf, combineReviews, + synthesizeDefect, toPublicSafe, runWorkersOpinion, } = __aiReviewInternals; @@ -30,6 +32,7 @@ type ModelReviewShape = { nits: string[]; suggestions: string[]; inlineFindings: InlineFinding[]; + confidence: number; }; const reviewWithFindings = ( inlineFindings: InlineFinding[], @@ -39,6 +42,7 @@ const reviewWithFindings = ( nits: [], suggestions: [], inlineFindings, + confidence: 1, }); function reviewJson( @@ -902,13 +906,39 @@ describe("pure helpers", () => { expect(parsed?.blockers).toEqual(["X in src/a.ts"]); }); + it("parseReviewConfidence uses a present value, falls back to 1.0 when absent/garbage, and clamps to [0,1] (#8)", () => { + expect(parseReviewConfidence(0.75)).toBe(0.75); // present, in range → used verbatim + expect(parseReviewConfidence(0)).toBe(0); // explicit zero is honored (not treated as falsy/absent) + expect(parseReviewConfidence(undefined)).toBe(1); // absent → fallback 1.0 + expect(parseReviewConfidence("0.5")).toBe(1); // non-number → fallback 1.0 + expect(parseReviewConfidence(Number.NaN)).toBe(1); // non-finite → fallback 1.0 + expect(parseReviewConfidence(1.7)).toBe(1); // above range → clamped to 1 + expect(parseReviewConfidence(-0.3)).toBe(0); // below range → clamped to 0 + }); + + it("parseModelReview threads a calibrated confidence and defaults it to 1.0 when absent/unparseable (#8)", () => { + const withConfidence = parseModelReview( + '{"assessment":"leak in b.ts","blockers":["Unclosed handle in src/b.ts"],"nits":[],"suggestions":[],"confidence":0.4}', + ); + expect(withConfidence?.confidence).toBe(0.4); // present value used + const noConfidence = parseModelReview( + reviewJson({ present: true, title: "Null deref in src/a.ts" }), + ); + expect(noConfidence?.confidence).toBe(1); // absent → fallback 1.0 + const garbageConfidence = parseModelReview( + '{"assessment":"ok","blockers":["X in src/a.ts"],"nits":[],"suggestions":[],"confidence":"high"}', + ); + expect(garbageConfidence?.confidence).toBe(1); // unparseable → fallback 1.0 + }); + describe("combineReviews (#dual-ai-combiner)", () => { - const r = (blockers: string[]) => ({ + const r = (blockers: string[], confidence = 1) => ({ assessment: "", suggestions: [], nits: [], blockers, inlineFindings: [], + confidence, }); const clean = r([]); const blocked = r(["Null deref in src/a.ts"]); @@ -1017,6 +1047,47 @@ describe("pure helpers", () => { }), ).toEqual({ defect: null, split: false, inconclusive: false }); // unsafe title dropped }); + + it("a consensus defect carries the MIN of the two reviewers' confidences (#8)", () => { + const defect = combineReviews( + [r(["Null deref in src/a.ts"], 0.95), r(["Null deref in src/a.ts"], 0.6)], + { strategy: "consensus" }, + ).defect; + expect(defect?.confidence).toBe(0.6); // weaker reviewer governs + }); + + it("single: the synthesized defect carries that one reviewer's confidence (#8)", () => { + const defect = combineReviews([r(["Null deref in src/a.ts"], 0.42)], { + strategy: "single", + }).defect; + expect(defect?.confidence).toBe(0.42); + }); + + it("a SPLIT carries the lone flagging reviewer's confidence — from whichever slot flagged (#8)", () => { + // reviewer A flags → splitConfidence = A's confidence + const aFlags = combineReviews( + [r(["Null deref in src/a.ts"], 0.55), clean], + { strategy: "consensus" }, + ); + expect(aFlags.split).toBe(true); + expect(aFlags.splitConfidence).toBe(0.55); + // reviewer B flags → splitConfidence = B's confidence (exercises the other side of the ternary) + const bFlags = combineReviews( + [clean, r(["Off-by-one in src/b.ts"], 0.3)], + { strategy: "consensus" }, + ); + expect(bFlags.split).toBe(true); + expect(bFlags.splitConfidence).toBe(0.3); + // no split → splitConfidence is absent (consensus + both-clean cases) + expect( + combineReviews([blocked, blocked], { strategy: "consensus" }) + .splitConfidence, + ).toBeUndefined(); + expect( + combineReviews([clean, clean], { strategy: "consensus" }) + .splitConfidence, + ).toBeUndefined(); + }); }); it("consensusDefectOf requires a concrete blocker in BOTH reviews and drops unsafe titles", () => { @@ -1026,6 +1097,7 @@ describe("pure helpers", () => { nits: [], blockers, inlineFindings: [], + confidence: 1, }); expect( consensusDefectOf( @@ -1050,6 +1122,7 @@ describe("pure helpers", () => { nits: [], blockers: [""], inlineFindings: [], + confidence: 1, }; const b = { assessment: "", @@ -1057,6 +1130,7 @@ describe("pure helpers", () => { nits: [], blockers: ["Race condition in src/x.ts"], inlineFindings: [], + confidence: 1, }; expect(consensusDefectOf(a, b)?.title).toBe("Race condition in src/x.ts"); }); @@ -1068,12 +1142,33 @@ describe("pure helpers", () => { nits: [], blockers: [""], inlineFindings: [], + confidence: 1, }; const out = consensusDefectOf(blank, { ...blank, blockers: [""] }); expect(out?.title).toContain("AI reviewers agree"); // both blockers[0] falsy → default title expect(out?.detail).toContain("independently flagged"); // joined detail empty → default detail }); + it("synthesizeDefect cites the FLAGGING reviewer's blocker + confidence, skipping an earlier clean reviewer (#8)", () => { + const review = (blockers: string[], confidence: number) => ({ + assessment: "", + suggestions: [], + nits: [], + blockers, + inlineFindings: [], + confidence, + }); + // first reviewer is clean → the title + confidence must come from the SECOND (flagging) reviewer. + const out = synthesizeDefect([ + review([], 0.99), + review(["Off-by-one in src/b.ts"], 0.35), + ]); + expect(out?.title).toBe("Off-by-one in src/b.ts"); + expect(out?.confidence).toBe(0.35); + // no reviewer with a non-blank blocker → null (fail-safe). + expect(synthesizeDefect([review([""], 0.5)])).toBeNull(); + }); + it("runWorkersOpinion returns null without a binding and handles a single-model (no distinct fallback) list", async () => { expect( await runWorkersOpinion(createTestEnv({}), "m", "f", "sys", "user", 256), @@ -1113,6 +1208,7 @@ describe("pure helpers", () => { nits: ["reward"], blockers: [], inlineFindings: [], + confidence: 1, }, ]), ).toBeNull(); @@ -1311,6 +1407,7 @@ describe("pure helpers", () => { nits: over.nits ?? [], blockers: over.blockers ?? [], inlineFindings: [], + confidence: 1, }); const assessmentOnly = composeAdvisoryNotes([ review({ assessment: "Looks good." }), @@ -1334,6 +1431,7 @@ describe("pure helpers", () => { nits: ["Rename x."], blockers: ["Null deref in src/a.ts."], inlineFindings: [], + confidence: 1, }; const b = { assessment: "Second look.", @@ -1341,6 +1439,7 @@ describe("pure helpers", () => { nits: ["Rename x.", "Tighten the type."], blockers: ["Null deref in src/a.ts.", "Off-by-one in the loop bound."], inlineFindings: [], + confidence: 1, }; const out = composeAdvisoryNotes([a, b]) ?? ""; expect(out).toContain("Solid change."); // first reviewer's assessment wins diff --git a/test/unit/focus-manifest.test.ts b/test/unit/focus-manifest.test.ts index be5821f80d..e051b86623 100644 --- a/test/unit/focus-manifest.test.ts +++ b/test/unit/focus-manifest.test.ts @@ -477,7 +477,7 @@ describe("compileFocusManifestPolicy", () => { issueDiscoveryPolicy: "neutral", maintainerNotes: [], publicNotes: ["Keep PRs focused.", "Maximize your reward payout"], - gate: { present: false, enabled: null, pack: null, linkedIssue: null, duplicates: null, readinessMode: null, readinessMinScore: null, slopMode: null, slopMinScore: null, slopAiAdvisory: null, sizeMode: null, aiReviewMode: null, aiReviewByok: null, aiReviewProvider: null, aiReviewModel: null, aiReviewAllAuthors: null, mergeReadiness: null, selfAuthoredLinkedIssue: null, manifestPolicy: null, dryRun: null, firstTimeContributorGrace: null }, + gate: { present: false, enabled: null, pack: null, linkedIssue: null, duplicates: null, readinessMode: null, readinessMinScore: null, slopMode: null, slopMinScore: null, slopAiAdvisory: null, sizeMode: null, aiReviewMode: null, aiReviewByok: null, aiReviewProvider: null, aiReviewModel: null, aiReviewAllAuthors: null, aiReviewCloseConfidence: null, mergeReadiness: null, selfAuthoredLinkedIssue: null, manifestPolicy: null, dryRun: null, firstTimeContributorGrace: null }, settings: {}, review: { present: false, footerText: null, note: null, fields: {}, profile: null, inlineComments: null, pathInstructions: [], instructions: null, excludePaths: [], preMergeChecks: [] }, features: { present: false, rag: null, reputation: null, unifiedComment: null, safety: null }, @@ -766,7 +766,7 @@ describe("parseFocusManifest gate config", () => { it("parses a full gate section including the readiness block", () => { const m = parseFocusManifest({ gate: { linkedIssue: "block", duplicates: "advisory", readiness: { mode: "block", minScore: 70 } } }); expect(m.present).toBe(true); - expect(m.gate).toEqual({ present: true, enabled: null, pack: null, linkedIssue: "block", duplicates: "advisory", readinessMode: "block", readinessMinScore: 70, slopMode: null, slopMinScore: null, slopAiAdvisory: null, sizeMode: null, aiReviewMode: null, aiReviewByok: null, aiReviewProvider: null, aiReviewModel: null, aiReviewAllAuthors: null, mergeReadiness: null, selfAuthoredLinkedIssue: null, manifestPolicy: null, dryRun: null, firstTimeContributorGrace: null }); + expect(m.gate).toEqual({ present: true, enabled: null, pack: null, linkedIssue: "block", duplicates: "advisory", readinessMode: "block", readinessMinScore: 70, slopMode: null, slopMinScore: null, slopAiAdvisory: null, sizeMode: null, aiReviewMode: null, aiReviewByok: null, aiReviewProvider: null, aiReviewModel: null, aiReviewAllAuthors: null, aiReviewCloseConfidence: null, mergeReadiness: null, selfAuthoredLinkedIssue: null, manifestPolicy: null, dryRun: null, firstTimeContributorGrace: null }); }); it("parses gate.mergeReadiness + gate.firstTimeContributorGrace, round-trips them, and warns on bad values (#822)", () => { @@ -933,6 +933,29 @@ describe("parseFocusManifest gate config", () => { expect(resolveEffectiveSettings({ aiReviewAllAuthors: true , closeOwnerAuthors: false} as unknown as RepositorySettings, noFlag).aiReviewAllAuthors).toBe(true); }); + it("parses gate.aiReview.closeConfidence, clamps to [0,1], makes the gate present, round-trips + resolves it, and warns on a bad value (#7)", () => { + // closeConfidence alone makes the gate present, serializes back under gate.aiReview.closeConfidence, and the + // gate alias projects it onto effective settings. + const m = parseFocusManifest({ gate: { aiReview: { closeConfidence: 0.75 } } }); + expect(m.gate.present).toBe(true); + expect(m.gate.aiReviewCloseConfidence).toBe(0.75); + expect((gateConfigToJson(m.gate) as { aiReview: { closeConfidence: number } }).aiReview.closeConfidence).toBe(0.75); + expect(parseFocusManifest({ gate: gateConfigToJson(m.gate) }).gate).toEqual(m.gate); // round-trips + // Clamped to [0,1] WITHOUT rounding (a fractional confidence, not a 0-100 score). + expect(parseFocusManifest({ gate: { aiReview: { closeConfidence: 1.5 } } }).gate.aiReviewCloseConfidence).toBe(1); + expect(parseFocusManifest({ gate: { aiReview: { closeConfidence: -0.2 } } }).gate.aiReviewCloseConfidence).toBe(0); + expect(parseFocusManifest({ gate: { aiReview: { closeConfidence: 0.333 } } }).gate.aiReviewCloseConfidence).toBe(0.333); // not rounded + // A non-number value warns and is dropped (stays null). + expect(parseFocusManifest({ gate: { aiReview: { closeConfidence: "high" } } }).warnings.some((w) => /gate\.aiReview\.closeConfidence/.test(w))).toBe(true); + expect(parseFocusManifest({ gate: { aiReview: { closeConfidence: "high" } } }).gate.aiReviewCloseConfidence).toBeNull(); + // The gate alias projects it onto effective settings; absent ⇒ null ⇒ the DB value (here undefined) is untouched. + const eff = resolveEffectiveSettings({ aiReviewCloseConfidence: undefined } as unknown as RepositorySettings, m); + expect(eff.aiReviewCloseConfidence).toBe(0.75); + const noFlag = parseFocusManifest({ gate: { aiReview: { mode: "advisory" } } }); + expect(noFlag.gate.aiReviewCloseConfidence).toBeNull(); + expect(resolveEffectiveSettings({ aiReviewCloseConfidence: 0.6 } as unknown as RepositorySettings, noFlag).aiReviewCloseConfidence).toBe(0.6); + }); + it("parses the features: block (per-repo converged-feature toggles), round-trips it, and makes the manifest present", () => { const m = parseFocusManifest({ features: { rag: true, reputation: false, unifiedComment: true } }); expect(m.present).toBe(true); diff --git a/test/unit/gate-check-policy.test.ts b/test/unit/gate-check-policy.test.ts index ed119093c9..5b1dce004a 100644 --- a/test/unit/gate-check-policy.test.ts +++ b/test/unit/gate-check-policy.test.ts @@ -213,6 +213,68 @@ describe("AI consensus defect gate blocker", () => { }); }); +describe("AI close-confidence threshold gate (#7)", () => { + const aiDefectWith = (confidence: number | undefined): Advisory => ({ + ...missingIssueAdvisory(), + findings: [{ code: "ai_consensus_defect", title: "AI reviewers agree on a likely critical defect", severity: "critical", detail: "Both models flagged a null deref.", action: "Resolve it.", ...(confidence !== undefined ? { confidence } : {}) }], + }); + const splitDefectWith = (confidence: number): Advisory => ({ + ...missingIssueAdvisory(), + findings: [{ code: "ai_review_split", title: "An AI reviewer flagged a likely blocking defect", severity: "critical", detail: "One reviewer flagged it.", action: "Resolve it.", confidence }], + }); + + it("blocks when mode=block AND confidence >= the default 0.9 floor", () => { + const out = evaluateGateCheck(aiDefectWith(0.95), gateCheckPolicy(settings({ aiReviewMode: "block" }), null, true)); + expect(out.conclusion).toBe("failure"); + expect(out.blockers.map((f) => f.code)).toEqual(["ai_consensus_defect"]); + }); + + it("does NOT block when mode=block but confidence < the floor — the AI defect stays advisory (#7)", () => { + const out = evaluateGateCheck(aiDefectWith(0.5), gateCheckPolicy(settings({ aiReviewMode: "block" }), null, true)); + expect(out.conclusion).toBe("success"); // below 0.9 → not a blocker, never closes + expect(out.blockers).toEqual([]); + }); + + it("blocks exactly AT the floor (>= boundary) and not just below it", () => { + // policy.aiReviewCloseConfidence is threaded onto the policy from settings. + const policy = gateCheckPolicy(settings({ aiReviewMode: "block", aiReviewCloseConfidence: 0.7 }), null, true); + expect(evaluateGateCheck(aiDefectWith(0.7), policy).conclusion).toBe("failure"); // == threshold → blocks + expect(evaluateGateCheck(aiDefectWith(0.69), policy).conclusion).toBe("success"); // just below → advisory + }); + + it("honors a custom aiReviewCloseConfidence (the `?? 0.9` default is NOT used when set) (#7)", () => { + // A high custom floor of 0.99 keeps a 0.95 defect advisory (the 0.9 default would have blocked it). + const strict = gateCheckPolicy(settings({ aiReviewMode: "block", aiReviewCloseConfidence: 0.99 }), null, true); + expect(evaluateGateCheck(aiDefectWith(0.95), strict).conclusion).toBe("success"); + // A low custom floor of 0.3 blocks a 0.5 defect that the 0.9 default would have left advisory. + const lenient = gateCheckPolicy(settings({ aiReviewMode: "block", aiReviewCloseConfidence: 0.3 }), null, true); + expect(evaluateGateCheck(aiDefectWith(0.5), lenient).conclusion).toBe("failure"); + }); + + it("a finding WITHOUT a confidence degrades to 1.0 and blocks under the default floor (graceful fallback) (#7)", () => { + const out = evaluateGateCheck(aiDefectWith(undefined), gateCheckPolicy(settings({ aiReviewMode: "block" }), null, true)); + expect(out.conclusion).toBe("failure"); // no confidence → treated as 1.0 → always clears 0.9 + }); + + it("never blocks when mode=advisory, regardless of a high confidence (#7)", () => { + expect(evaluateGateCheck(aiDefectWith(1), gateCheckPolicy(settings({ aiReviewMode: "advisory" }), null, true)).conclusion).toBe("success"); + }); + + it("applies the same confidence floor to an ai_review_split finding (#7)", () => { + const policy = gateCheckPolicy(settings({ aiReviewMode: "block" }), null, true); + expect(evaluateGateCheck(splitDefectWith(0.95), policy).conclusion).toBe("failure"); // clears 0.9 → blocks + expect(evaluateGateCheck(splitDefectWith(0.5), policy).conclusion).toBe("success"); // below 0.9 → advisory + }); + + it("resolveEffectiveSettings maps gate.aiReview.closeConfidence (clamped) into the policy floor (#7)", () => { + const eff = resolveEffectiveSettings(settings({ aiReviewMode: "off" }), parseFocusManifest({ gate: { aiReview: { mode: "block", closeConfidence: 0.4 } } })); + expect(eff.aiReviewCloseConfidence).toBe(0.4); + expect(eff.aiReviewMode).toBe("block"); + // a 0.5 defect clears the configured 0.4 floor → blocks (it would NOT under the 0.9 default). + expect(evaluateGateCheck(aiDefectWith(0.5), gateCheckPolicy(eff, null, true)).conclusion).toBe("failure"); + }); +}); + describe("slop gate (#530/#532)", () => { function cleanAdvisory(): Advisory { return { ...missingIssueAdvisory(), findings: [] }; diff --git a/test/unit/predicted-gate.test.ts b/test/unit/predicted-gate.test.ts index dc10a9fb67..b195a0ccd4 100644 --- a/test/unit/predicted-gate.test.ts +++ b/test/unit/predicted-gate.test.ts @@ -50,6 +50,14 @@ describe("buildPredictedGateVerdict", () => { expect(result.note).toContain("public .gittensory.yml"); }); + it("threads gate.aiReview.closeConfidence into the policy without disturbing the public-config verdict (#7)", () => { + // The predictor builds the advisory from PUBLIC metadata only (no AI finding exists), so a closeConfidence + // floor has nothing to act on — the verdict stays a clean pass. This exercises the truthy `?? null` branch. + const result = verdict({ gate: { duplicates: "block", linkedIssue: "advisory", aiReview: { mode: "block", closeConfidence: 0.4 } } }); + expect(result.conclusion).toBe("success"); + expect(result.blockers).toHaveLength(0); + }); + it("predicts a BLOCK when a duplicate PR exists and duplicates:block (the default)", () => { // Another open PR already targets the same linked issue → duplicate_pr_risk. const result = verdict({ gate: { duplicates: "block" }, pullRequests: [openPr(42, "Retry uploads on 5xx responses", [7])] });