diff --git a/packages/gittensory-engine/README.md b/packages/gittensory-engine/README.md index 1ba796205c..3469aff75e 100644 --- a/packages/gittensory-engine/README.md +++ b/packages/gittensory-engine/README.md @@ -442,6 +442,75 @@ repo was not opted in, had invalid ids, or exposed no recognized non-empty tiers with component scores, effective weights, contributing repos, per-tier tables, rejected rows, and a contributing-repo summary. All caller-supplied ids and repo names are Markdown-escaped and newline-collapsed before rendering. +## Structured reviewer-consensus calibration + +`resolveReviewerConsensusCalibrationConfig()`, `ingestReviewerConsensusCalibrationSignals()`, and +`computeReviewerConsensusCompositeCalibrationScore()` provide the pure engine contract for the opt-in +reviewer-consensus calibration signal. When a review runs more than one independent reviewer (multiple models, or the +same model sampled multiple times), each reviewer casts a per-dimension verdict; this signal measures how much they +**agree**. It is a companion to the pairwise judge (which measures order-stability of a single judge) at the level of +independent reviewers, and — like the rest of the family — the engine contract is deliberately default-off and safe to +call at ingestion time. + +The preferred config-as-code surface is: + +```yaml +miner: + calibration: + shareStructuredReviewerConsensus: true + structuredReviewerConsensusWeight: 0.2 +``` + +Only `shareStructuredReviewerConsensus: true` enables ingestion. Missing, malformed, or falsey values all fail closed to +no sharing. `calibration.shareStructuredReviewerConsensus` is accepted as a narrow top-level alias. The optional weight +is non-negative and finite; malformed values fall back to the default. + +The accepted signal is intentionally narrow: repo/run ids plus, per dimension (`correctness`, `tests`, `security`, +`maintainability`, `scope`, `freshness`, `ci`, `policy`), the set of independent reviewer votes (`pass`/`warn`/`fail`). +It has no fields for raw review text, secrets, trust scores, reward values, private rankings, or maintainer evidence. + +```ts +import { + computeReviewerConsensusCompositeCalibrationScore, + ingestReviewerConsensusCalibrationSignals, +} from "@jsonbored/gittensory-engine"; + +const reviewerConsensus = ingestReviewerConsensusCalibrationSignals([ + { + repoFullName: "jsonbored/gittensory", + replayRunId: "replay-2026-07-05", + reviewRunId: "review-123", + optedIn: true, + dimensions: [ + { dimension: "correctness", votes: ["pass", "pass", "pass"] }, + { dimension: "security", votes: ["fail", "warn", "fail"] }, + ], + }, +]); + +const score = computeReviewerConsensusCompositeCalibrationScore({ + objectiveAnchor: 0.65, + pairwise: 0.8, + reviewerConsensus, +}); +``` + +Per dimension, unrecognized and abstention votes are dropped, the remaining votes are tallied, the plurality outcome is +chosen (ties broken toward the more severe outcome so a genuine split never rounds a real `fail`/`warn` down to `pass`), +and the **agreement** fraction is the plurality's share of the definite votes. The per-PR score is the +**vote-count-weighted** mean of the per-dimension agreements, so a dimension reviewed by more reviewers carries more +weight than one seen by a single reviewer. + +The composite scorer renormalizes weights when a signal is absent: if a repo opts out or no dimension carries a definite +vote, the structured reviewer-consensus weight drops to zero and the objective/pairwise signals are renormalized. The +returned audit trail records which opted-in repos contributed and which rows were rejected because the repo was not +opted in, had invalid ids, or exposed no definite per-dimension votes. + +`renderReviewerConsensusCalibrationAuditMarkdown(result)` turns the composite result into a deterministic local artifact +with component scores, effective weights, contributing repos, per-dimension agreement tables, rejected rows, and a +contributing-repo summary. All caller-supplied ids and repo names are Markdown-escaped and newline-collapsed before +rendering. + ## Plan templates `plan-templates.ts` exports one builder per miner lifecycle stage (`analyze`, `plan`, `prepare`, `create`, `manage`). diff --git a/packages/gittensory-engine/src/index.ts b/packages/gittensory-engine/src/index.ts index 26af2c2973..d9158fa49f 100644 --- a/packages/gittensory-engine/src/index.ts +++ b/packages/gittensory-engine/src/index.ts @@ -92,6 +92,23 @@ export { type FindingSeverityTierInput, type FindingSeverityTierSignal, } from "./finding-severity-calibration.js"; +export { + computeReviewerConsensusCompositeCalibrationScore, + ingestReviewerConsensusCalibrationSignals, + renderReviewerConsensusCalibrationAuditMarkdown, + resolveReviewerConsensusCalibrationConfig, + type ReviewerConsensusCalibrationConfig, + type ReviewerConsensusCalibrationIngestion, + type ReviewerConsensusCalibrationManifest, + type ReviewerConsensusCalibrationSignal, + type ReviewerConsensusCalibrationSignalInput, + type ReviewerConsensusCalibrationWeights, + type ReviewerConsensusCompositeCalibrationScore, + type ReviewerConsensusDimension, + type ReviewerConsensusDimensionInput, + type ReviewerConsensusDimensionSignal, + type ReviewerConsensusVote, +} from "./reviewer-consensus-calibration.js"; export { computeTrackRecordSummary, renderTrackRecordSummaryMarkdown, diff --git a/packages/gittensory-engine/src/reviewer-consensus-calibration.ts b/packages/gittensory-engine/src/reviewer-consensus-calibration.ts new file mode 100644 index 0000000000..049af1d498 --- /dev/null +++ b/packages/gittensory-engine/src/reviewer-consensus-calibration.ts @@ -0,0 +1,578 @@ +// Opt-in structured reviewer-consensus calibration signal (#1955 calibration family). +// +// This module is the pure engine half of reviewer-consensus calibration. When a review runs more than one +// independent reviewer (multiple models, or the same model sampled multiple times), each reviewer casts a per-dimension +// verdict. This signal measures how much those reviewers AGREE per dimension: a high-agreement verdict is reliable, +// while a split verdict is unstable and the replay harness should weight it less. It is a companion to the pairwise +// judge (which measures order-stability of a single judge) at the level of independent reviewers. +// +// The hosted review stack decides whether a repo is currently opted in from its resolved `.gittensory.yml`/private +// config; the miner replay harness can then ingest only the structured per-dimension vote fields exposed here. No raw +// review text, secrets, trust values, rewards, rankings, or maintainer evidence is represented in this type surface. + +import type { ObjectiveAnchorScore } from "./objective-anchor.js"; +import type { PairwiseCalibrationScore } from "./pairwise-calibration.js"; + +export type ReviewerConsensusDimension = + | "correctness" + | "tests" + | "security" + | "maintainability" + | "scope" + | "freshness" + | "ci" + | "policy"; + +export type ReviewerConsensusVote = "pass" | "warn" | "fail"; + +export type ReviewerConsensusCalibrationManifest = { + miner?: { + calibration?: { + /** Explicit maintainer opt-in. Default false. */ + shareStructuredReviewerConsensus?: unknown; + /** Optional weight for the structured reviewer-consensus signal when composed into a replay score. */ + structuredReviewerConsensusWeight?: unknown; + } | null; + } | null; + calibration?: { + /** Back-compat/future-friendly alias, still explicit and default-off. */ + shareStructuredReviewerConsensus?: unknown; + structuredReviewerConsensusWeight?: unknown; + } | null; +}; + +export type ReviewerConsensusCalibrationConfig = { + shareStructuredReviewerConsensus: boolean; + structuredReviewerConsensusWeight: number; + warnings: string[]; +}; + +export type ReviewerConsensusDimensionInput = { + dimension: ReviewerConsensusDimension | string; + /** One verdict per independent reviewer. Unrecognized / abstention votes are dropped before agreement is measured. */ + votes: readonly (ReviewerConsensusVote | string)[]; +}; + +export type ReviewerConsensusCalibrationSignalInput = { + repoFullName: string; + replayRunId: string; + reviewRunId: string; + optedIn: boolean; + observedAt?: string | undefined; + dimensions: readonly ReviewerConsensusDimensionInput[]; +}; + +export type ReviewerConsensusDimensionSignal = { + dimension: ReviewerConsensusDimension; + voteCount: number; + majorityOutcome: ReviewerConsensusVote; + agreement: number; + score: number; +}; + +export type ReviewerConsensusCalibrationSignal = { + repoFullName: string; + replayRunId: string; + reviewRunId: string; + observedAt: string | null; + dimensions: ReviewerConsensusDimensionSignal[]; + score: number; +}; + +export type ReviewerConsensusCalibrationIngestion = { + accepted: ReviewerConsensusCalibrationSignal[]; + rejected: Array<{ + repoFullName: string; + replayRunId: string; + reviewRunId: string; + reason: "not_opted_in" | "empty_dimensions" | "invalid_repo" | "invalid_run_id"; + }>; +}; + +export type ReviewerConsensusCalibrationWeights = { + objectiveAnchor?: number | undefined; + pairwiseJudge?: number | undefined; + structuredReviewerConsensus?: number | undefined; +}; + +export type ReviewerConsensusCompositeCalibrationScore = { + compositeScore: number; + objectiveAnchorScore: number; + pairwiseJudgeScore: number | null; + structuredReviewerConsensusScore: number | null; + weights: { + objectiveAnchor: number; + pairwiseJudge: number; + structuredReviewerConsensus: number; + }; + audit: { + contributingRepos: Array<{ + repoFullName: string; + replayRunId: string; + reviewRunId: string; + observedAt: string | null; + score: number; + dimensions: ReviewerConsensusDimensionSignal[]; + }>; + rejected: ReviewerConsensusCalibrationIngestion["rejected"]; + }; +}; + +const DIMENSION_ORDER: ReviewerConsensusDimension[] = [ + "correctness", + "tests", + "security", + "maintainability", + "scope", + "freshness", + "ci", + "policy", +]; + +// Tie-break order when two outcomes draw the plurality: prefer the more severe outcome, so a genuine split never +// rounds a real `fail`/`warn` signal down to `pass`. +const VOTE_SEVERITY: Record = { + fail: 2, + warn: 1, + pass: 0, +}; + +const DEFAULT_STRUCTURED_REVIEWER_CONSENSUS_WEIGHT = 0.2; +const DEFAULT_COMPOSITE_WEIGHTS = { + objectiveAnchor: 0.45, + pairwiseJudge: 0.35, + structuredReviewerConsensus: 0.2, +}; + +function isRecord(value: unknown): value is Record { + return Boolean(value && typeof value === "object" && !Array.isArray(value)); +} + +function finiteNonNegative(value: number | undefined, fallback: number): number { + if (value === undefined) return fallback; + if (!Number.isFinite(value) || value < 0) return 0; + return value; +} + +function roundScore(value: number): number { + return Math.round(Math.min(1, Math.max(0, value)) * 1_000_000) / 1_000_000; +} + +function normalizeRepoFullName(value: string): string | null { + const trimmed = value.trim().toLowerCase(); + if (!/^[a-z0-9_.-]+\/[a-z0-9_.-]+$/u.test(trimmed)) return null; + return trimmed; +} + +function normalizeId(value: string): string | null { + const trimmed = value.trim(); + if (!trimmed || trimmed.length > 160 || /[\r\n\0]/u.test(trimmed)) return null; + return trimmed; +} + +function normalizeObservedAt(value: string | undefined): string | null { + if (!value) return null; + const ms = Date.parse(value); + if (!Number.isFinite(ms)) return null; + return new Date(ms).toISOString(); +} + +function normalizeBoolean(value: unknown): boolean | undefined { + if (typeof value === "boolean") return value; + if (typeof value !== "string") return undefined; + const normalized = value.trim().toLowerCase(); + if (["true", "1", "yes", "on"].includes(normalized)) return true; + if (["false", "0", "no", "off"].includes(normalized)) return false; + return undefined; +} + +function normalizeOptionalWeight(value: unknown): number | undefined { + if (value === undefined || value === null) return undefined; + const number = typeof value === "number" ? value : typeof value === "string" ? Number(value.trim()) : Number.NaN; + if (!Number.isFinite(number) || number < 0) return undefined; + return number; +} + +function normalizeDimension(value: string): ReviewerConsensusDimension | null { + const normalized = value.trim().toLowerCase().replace(/[_\s-]+/gu, "_"); + if (normalized === "quality" || normalized === "code_quality") return "correctness"; + if (normalized === "test" || normalized === "coverage") return "tests"; + if (normalized === "maintenance") return "maintainability"; + if (normalized === "size" || normalized === "blast_radius") return "scope"; + if (normalized === "rebase" || normalized === "up_to_date") return "freshness"; + if (normalized === "workflow" || normalized === "checks") return "ci"; + if ((DIMENSION_ORDER as string[]).includes(normalized)) return normalized as ReviewerConsensusDimension; + return null; +} + +function normalizeVote(value: string): ReviewerConsensusVote | null { + const normalized = value.trim().toLowerCase().replace(/[_\s-]+/gu, "_"); + if (normalized === "ok" || normalized === "success" || normalized === "passed" || normalized === "approve") { + return "pass"; + } + if (normalized === "warning" || normalized === "advisory" || normalized === "hold" || normalized === "comment") { + return "warn"; + } + if (normalized === "block" || normalized === "blocked" || normalized === "failed" || normalized === "reject") { + return "fail"; + } + if ((["pass", "warn", "fail"] as string[]).includes(normalized)) return normalized as ReviewerConsensusVote; + return null; +} + +/** + * Reduce a dimension's independent votes to a consensus signal: drop unrecognized/abstention votes, tally the rest, + * pick the plurality outcome (ties broken toward the more severe outcome), and derive the agreement fraction. A + * dimension left with no definite votes is dropped. + */ +function summarizeDimensionVotes( + votes: readonly (ReviewerConsensusVote | string)[], +): { majorityOutcome: ReviewerConsensusVote; voteCount: number; agreement: number } | null { + const counts: Record = { pass: 0, warn: 0, fail: 0 }; + let voteCount = 0; + for (const raw of votes) { + const vote = normalizeVote(raw); + if (!vote) continue; + counts[vote] += 1; + voteCount += 1; + } + if (voteCount === 0) return null; + let majorityOutcome: ReviewerConsensusVote = "pass"; + let best = -1; + for (const vote of ["fail", "warn", "pass"] as ReviewerConsensusVote[]) { + const count = counts[vote]; + // Strictly greater wins; on a tie the earlier (more severe, per the iteration order) outcome is kept. + if (count > best || (count === best && VOTE_SEVERITY[vote] > VOTE_SEVERITY[majorityOutcome])) { + best = count; + majorityOutcome = vote; + } + } + return { majorityOutcome, voteCount, agreement: roundScore(best / voteCount) }; +} + +function normalizeDimensions( + dimensions: readonly ReviewerConsensusDimensionInput[], +): ReviewerConsensusDimensionSignal[] { + const byDimension = new Map(); + for (const item of dimensions) { + const dimension = normalizeDimension(item.dimension); + if (!dimension) continue; + const existing = byDimension.get(dimension); + if (existing) { + existing.push(...item.votes); + } else { + byDimension.set(dimension, [...item.votes]); + } + } + return DIMENSION_ORDER.flatMap((dimension) => { + const votes = byDimension.get(dimension); + if (!votes) return []; + const summary = summarizeDimensionVotes(votes); + if (!summary) return []; + return [ + { + dimension, + voteCount: summary.voteCount, + majorityOutcome: summary.majorityOutcome, + agreement: summary.agreement, + score: summary.agreement, + }, + ]; + }); +} + +/** + * The per-PR consensus score: the vote-count-weighted mean of the per-dimension agreement fractions, so a dimension + * with more reviewers carries more weight than one with a single reviewer. Returns null when no dimension carries a + * definite vote (already rejected upstream). + */ +function scoreDimensions(dimensions: readonly ReviewerConsensusDimensionSignal[]): number | null { + let weightedAgreement = 0; + let voteSum = 0; + for (const dimension of dimensions) { + weightedAgreement += dimension.voteCount * dimension.agreement; + voteSum += dimension.voteCount; + } + if (voteSum <= 0) return null; + return roundScore(weightedAgreement / voteSum); +} + +function averageSignals(signals: readonly ReviewerConsensusCalibrationSignal[]): number | null { + if (signals.length === 0) return null; + return roundScore(signals.reduce((sum, signal) => sum + signal.score, 0) / signals.length); +} + +function isReviewerConsensusCalibrationIngestion(value: unknown): value is ReviewerConsensusCalibrationIngestion { + return isRecord(value) && Array.isArray(value.accepted) && Array.isArray(value.rejected); +} + +function normalizeCompositeWeights(weights: ReviewerConsensusCalibrationWeights | undefined): { + objectiveAnchor: number; + pairwiseJudge: number; + structuredReviewerConsensus: number; +} { + const raw = { + objectiveAnchor: finiteNonNegative(weights?.objectiveAnchor, DEFAULT_COMPOSITE_WEIGHTS.objectiveAnchor), + pairwiseJudge: finiteNonNegative(weights?.pairwiseJudge, DEFAULT_COMPOSITE_WEIGHTS.pairwiseJudge), + structuredReviewerConsensus: finiteNonNegative( + weights?.structuredReviewerConsensus, + DEFAULT_COMPOSITE_WEIGHTS.structuredReviewerConsensus, + ), + }; + const total = raw.objectiveAnchor + raw.pairwiseJudge + raw.structuredReviewerConsensus; + // Preserve explicitly-zeroed weights rather than substituting the defaults: a caller that zeroes every component + // must reach the objective-only fallback in the composite scorer, not silently get the default 45/35/20 blend. + if (total <= 0) return { objectiveAnchor: 0, pairwiseJudge: 0, structuredReviewerConsensus: 0 }; + return { + objectiveAnchor: raw.objectiveAnchor / total, + pairwiseJudge: raw.pairwiseJudge / total, + structuredReviewerConsensus: raw.structuredReviewerConsensus / total, + }; +} + +function markdownSafe(value: string): string { + return value.replace(/[\r\n]+/gu, " ").replace(/[\\`*_[\]<>|]/gu, "\\$&"); +} + +function markdownList(values: readonly string[]): string { + if (values.length === 0) return "- none"; + return values.map((value) => `- ${markdownSafe(value)}`).join("\n"); +} + +function renderDimensionRows(dimensions: readonly ReviewerConsensusDimensionSignal[]): string { + if (dimensions.length === 0) return "| Dimension | Votes | Majority | Agreement |\n| --- | ---: | --- | ---: |\n"; + return [ + "| Dimension | Votes | Majority | Agreement |", + "| --- | ---: | --- | ---: |", + ...dimensions.map( + (dimension) => + `| ${markdownSafe(dimension.dimension)} | ${dimension.voteCount} | ${markdownSafe( + dimension.majorityOutcome, + )} | ${dimension.agreement.toFixed(6)} |`, + ), + ].join("\n"); +} + +function renderContributingRepo( + signal: ReviewerConsensusCompositeCalibrationScore["audit"]["contributingRepos"][number], +): string { + return [ + `### ${markdownSafe(signal.repoFullName)}`, + "", + `- replayRunId: ${markdownSafe(signal.replayRunId)}`, + `- reviewRunId: ${markdownSafe(signal.reviewRunId)}`, + `- observedAt: ${signal.observedAt ? markdownSafe(signal.observedAt) : "n/a"}`, + `- score: ${signal.score.toFixed(6)}`, + "", + renderDimensionRows(signal.dimensions), + ].join("\n"); +} + +function renderRejectedRow(row: ReviewerConsensusCalibrationIngestion["rejected"][number]): string { + return `| ${markdownSafe(row.repoFullName)} | ${markdownSafe(row.replayRunId)} | ${markdownSafe( + row.reviewRunId, + )} | ${markdownSafe(row.reason)} |`; +} + +/** + * Resolve the explicit per-repo opt-in from a parsed `.gittensory.yml`-style object. Default is opted out. The + * preferred path is `miner.calibration.shareStructuredReviewerConsensus`; + * `calibration.shareStructuredReviewerConsensus` is accepted as a narrow alias so private-config surfaces can place + * the field at top level if needed. + */ +export function resolveReviewerConsensusCalibrationConfig( + manifest: ReviewerConsensusCalibrationManifest | Record | null | undefined, +): ReviewerConsensusCalibrationConfig { + const warnings: string[] = []; + const root = isRecord(manifest) ? manifest : {}; + const miner = isRecord(root.miner) ? root.miner : {}; + const minerCalibration = isRecord(miner.calibration) ? miner.calibration : {}; + const topCalibration = isRecord(root.calibration) ? root.calibration : {}; + const optInRaw = + minerCalibration.shareStructuredReviewerConsensus ?? topCalibration.shareStructuredReviewerConsensus ?? undefined; + const optIn = normalizeBoolean(optInRaw); + if (optInRaw !== undefined && optIn === undefined) { + warnings.push( + "miner.calibration.shareStructuredReviewerConsensus must be a boolean-like value; defaulting to false.", + ); + } + const weightRaw = + minerCalibration.structuredReviewerConsensusWeight ?? topCalibration.structuredReviewerConsensusWeight; + const weight = normalizeOptionalWeight(weightRaw); + if (weightRaw !== undefined && weight === undefined) { + warnings.push( + "miner.calibration.structuredReviewerConsensusWeight must be a non-negative finite number; using default.", + ); + } + return { + shareStructuredReviewerConsensus: optIn === true, + structuredReviewerConsensusWeight: weight ?? DEFAULT_STRUCTURED_REVIEWER_CONSENSUS_WEIGHT, + warnings, + }; +} + +/** + * Ingest only currently opted-in structured reviewer-consensus signals. The opt-in check happens at ingestion time, so + * a maintainer opt-out immediately prevents additional calibration rows from contributing even if older collected data + * exists elsewhere. + */ +export function ingestReviewerConsensusCalibrationSignals( + signals: readonly ReviewerConsensusCalibrationSignalInput[], +): ReviewerConsensusCalibrationIngestion { + const accepted: ReviewerConsensusCalibrationSignal[] = []; + const rejected: ReviewerConsensusCalibrationIngestion["rejected"] = []; + for (const signal of signals) { + const repoFullName = normalizeRepoFullName(signal.repoFullName); + const replayRunId = normalizeId(signal.replayRunId); + const reviewRunId = normalizeId(signal.reviewRunId); + if (!repoFullName) { + rejected.push({ + repoFullName: signal.repoFullName, + replayRunId: signal.replayRunId, + reviewRunId: signal.reviewRunId, + reason: "invalid_repo", + }); + continue; + } + if (!replayRunId || !reviewRunId) { + rejected.push({ + repoFullName, + replayRunId: signal.replayRunId, + reviewRunId: signal.reviewRunId, + reason: "invalid_run_id", + }); + continue; + } + if (!signal.optedIn) { + rejected.push({ repoFullName, replayRunId, reviewRunId, reason: "not_opted_in" }); + continue; + } + const dimensions = normalizeDimensions(signal.dimensions); + const score = scoreDimensions(dimensions); + if (dimensions.length === 0 || score === null) { + rejected.push({ repoFullName, replayRunId, reviewRunId, reason: "empty_dimensions" }); + continue; + } + accepted.push({ + repoFullName, + replayRunId, + reviewRunId, + observedAt: normalizeObservedAt(signal.observedAt), + dimensions, + score, + }); + } + return { accepted, rejected }; +} + +export function computeReviewerConsensusCompositeCalibrationScore(input: { + objectiveAnchor: number | ObjectiveAnchorScore; + pairwise: number | PairwiseCalibrationScore | null; + reviewerConsensus: ReviewerConsensusCalibrationIngestion | readonly ReviewerConsensusCalibrationSignalInput[]; + weights?: ReviewerConsensusCalibrationWeights | undefined; +}): ReviewerConsensusCompositeCalibrationScore { + const ingestion = isReviewerConsensusCalibrationIngestion(input.reviewerConsensus) + ? input.reviewerConsensus + : ingestReviewerConsensusCalibrationSignals(input.reviewerConsensus); + const objectiveAnchorScore = + typeof input.objectiveAnchor === "number" ? roundScore(input.objectiveAnchor) : input.objectiveAnchor.score; + const pairwiseJudgeScore = + input.pairwise === null + ? null + : typeof input.pairwise === "number" + ? roundScore(input.pairwise) + : input.pairwise.pairwiseJudgeScore; + const structuredReviewerConsensusScore = averageSignals(ingestion.accepted); + const rawWeights = normalizeCompositeWeights(input.weights); + const usableWeights = { + objectiveAnchor: rawWeights.objectiveAnchor, + pairwiseJudge: pairwiseJudgeScore === null ? 0 : rawWeights.pairwiseJudge, + structuredReviewerConsensus: + structuredReviewerConsensusScore === null ? 0 : rawWeights.structuredReviewerConsensus, + }; + const total = + usableWeights.objectiveAnchor + usableWeights.pairwiseJudge + usableWeights.structuredReviewerConsensus; + const weights = + total <= 0 + ? { objectiveAnchor: 1, pairwiseJudge: 0, structuredReviewerConsensus: 0 } + : { + objectiveAnchor: usableWeights.objectiveAnchor / total, + pairwiseJudge: usableWeights.pairwiseJudge / total, + structuredReviewerConsensus: usableWeights.structuredReviewerConsensus / total, + }; + const compositeScore = roundScore( + objectiveAnchorScore * weights.objectiveAnchor + + (pairwiseJudgeScore ?? 0) * weights.pairwiseJudge + + (structuredReviewerConsensusScore ?? 0) * weights.structuredReviewerConsensus, + ); + return { + compositeScore, + objectiveAnchorScore, + pairwiseJudgeScore, + structuredReviewerConsensusScore, + weights, + audit: { + contributingRepos: ingestion.accepted.map((signal) => ({ + repoFullName: signal.repoFullName, + replayRunId: signal.replayRunId, + reviewRunId: signal.reviewRunId, + observedAt: signal.observedAt, + score: signal.score, + dimensions: signal.dimensions, + })), + rejected: ingestion.rejected, + }, + }; +} + +/** + * Render a deterministic, public-safe Markdown report for a structured reviewer-consensus calibration result. The + * report is local-run evidence: it includes aggregate scores, normalized weights, opted-in contributors, and rejected + * rows, but never accepts or emits raw review text or private scoring fields. + */ +export function renderReviewerConsensusCalibrationAuditMarkdown( + result: ReviewerConsensusCompositeCalibrationScore, +): string { + const lines = [ + "# Structured Reviewer-Consensus Calibration", + "", + `Composite score: ${result.compositeScore.toFixed(6)}`, + "", + "## Component Scores", + "", + `- objectiveAnchor: ${result.objectiveAnchorScore.toFixed(6)}`, + `- pairwiseJudge: ${result.pairwiseJudgeScore === null ? "n/a" : result.pairwiseJudgeScore.toFixed(6)}`, + `- structuredReviewerConsensus: ${ + result.structuredReviewerConsensusScore === null ? "n/a" : result.structuredReviewerConsensusScore.toFixed(6) + }`, + "", + "## Effective Weights", + "", + `- objectiveAnchor: ${result.weights.objectiveAnchor.toFixed(6)}`, + `- pairwiseJudge: ${result.weights.pairwiseJudge.toFixed(6)}`, + `- structuredReviewerConsensus: ${result.weights.structuredReviewerConsensus.toFixed(6)}`, + "", + "## Contributing Repos", + "", + result.audit.contributingRepos.length === 0 + ? "_No opted-in structured reviewer-consensus signals contributed._" + : result.audit.contributingRepos.map(renderContributingRepo).join("\n\n"), + "", + "## Rejected Rows", + "", + ]; + + if (result.audit.rejected.length === 0) { + lines.push("- none"); + } else { + lines.push( + "| Repo | Replay run | Review run | Reason |", + "| --- | --- | --- | --- |", + ...result.audit.rejected.map(renderRejectedRow), + ); + } + + const contributingRepos = result.audit.contributingRepos.map((repo) => repo.repoFullName); + lines.push("", "## Contributing Repo Summary", "", markdownList(contributingRepos)); + return `${lines.join("\n")}\n`; +} diff --git a/packages/gittensory-engine/test/reviewer-consensus-calibration.test.ts b/packages/gittensory-engine/test/reviewer-consensus-calibration.test.ts new file mode 100644 index 0000000000..08689843dc --- /dev/null +++ b/packages/gittensory-engine/test/reviewer-consensus-calibration.test.ts @@ -0,0 +1,356 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { + computeReviewerConsensusCompositeCalibrationScore, + ingestReviewerConsensusCalibrationSignals, + renderReviewerConsensusCalibrationAuditMarkdown, + resolveReviewerConsensusCalibrationConfig, + type ObjectiveAnchorScore, + type PairwiseCalibrationScore, + type ReviewerConsensusCalibrationSignalInput, +} from "../dist/index.js"; + +function signal( + overrides: Partial = {}, +): ReviewerConsensusCalibrationSignalInput { + return { + repoFullName: "acme/widgets", + replayRunId: "replay-1", + reviewRunId: "review-1", + optedIn: true, + dimensions: [{ dimension: "correctness", votes: ["pass", "pass", "pass"] }], + ...overrides, + }; +} + +test("barrel: exports structured reviewer-consensus calibration APIs", () => { + assert.equal(typeof resolveReviewerConsensusCalibrationConfig, "function"); + assert.equal(typeof ingestReviewerConsensusCalibrationSignals, "function"); + assert.equal(typeof computeReviewerConsensusCompositeCalibrationScore, "function"); + assert.equal(typeof renderReviewerConsensusCalibrationAuditMarkdown, "function"); +}); + +test("resolveReviewerConsensusCalibrationConfig defaults to opted out with the default structured weight", () => { + for (const manifest of [undefined, null, "nope" as unknown as Record]) { + assert.deepEqual(resolveReviewerConsensusCalibrationConfig(manifest), { + shareStructuredReviewerConsensus: false, + structuredReviewerConsensusWeight: 0.2, + warnings: [], + }); + } +}); + +test("resolveReviewerConsensusCalibrationConfig reads the preferred path and the top-level alias with precedence", () => { + const preferred = resolveReviewerConsensusCalibrationConfig({ + miner: { calibration: { shareStructuredReviewerConsensus: true, structuredReviewerConsensusWeight: 0.5 } }, + }); + assert.equal(preferred.shareStructuredReviewerConsensus, true); + assert.equal(preferred.structuredReviewerConsensusWeight, 0.5); + assert.deepEqual(preferred.warnings, []); + + const alias = resolveReviewerConsensusCalibrationConfig({ + calibration: { shareStructuredReviewerConsensus: "yes" }, + }); + assert.equal(alias.shareStructuredReviewerConsensus, true); + + const both = resolveReviewerConsensusCalibrationConfig({ + miner: { calibration: { shareStructuredReviewerConsensus: "off" } }, + calibration: { shareStructuredReviewerConsensus: "on" }, + }); + assert.equal(both.shareStructuredReviewerConsensus, false); +}); + +test("resolveReviewerConsensusCalibrationConfig warns on non-boolean opt-in and invalid weight, failing closed", () => { + const config = resolveReviewerConsensusCalibrationConfig({ + miner: { + calibration: { shareStructuredReviewerConsensus: "maybe", structuredReviewerConsensusWeight: "heavy" }, + }, + }); + assert.equal(config.shareStructuredReviewerConsensus, false); + assert.equal(config.structuredReviewerConsensusWeight, 0.2); + assert.equal(config.warnings.length, 2); + + const negative = resolveReviewerConsensusCalibrationConfig({ + calibration: { structuredReviewerConsensusWeight: -3 }, + }); + assert.equal(negative.structuredReviewerConsensusWeight, 0.2); + assert.equal(negative.warnings.length, 1); + const zero = resolveReviewerConsensusCalibrationConfig({ + calibration: { structuredReviewerConsensusWeight: 0 }, + }); + assert.equal(zero.structuredReviewerConsensusWeight, 0); + assert.deepEqual(zero.warnings, []); +}); + +test("ingest scores a unanimous verdict as full agreement and a split verdict below it", () => { + const unanimous = ingestReviewerConsensusCalibrationSignals([signal()]); + assert.equal(unanimous.accepted.length, 1); + assert.equal(unanimous.accepted[0]!.score, 1); + assert.deepEqual(unanimous.accepted[0]!.dimensions, [ + { dimension: "correctness", voteCount: 3, majorityOutcome: "pass", agreement: 1, score: 1 }, + ]); + + const split = ingestReviewerConsensusCalibrationSignals([ + signal({ dimensions: [{ dimension: "correctness", votes: ["pass", "pass", "fail"] }] }), + ]).accepted[0]!; + assert.equal(split.dimensions[0]!.majorityOutcome, "pass"); + assert.equal(split.dimensions[0]!.agreement, Math.round((2 / 3) * 1_000_000) / 1_000_000); + assert.ok(split.score < unanimous.accepted[0]!.score, "a split verdict must calibrate below a unanimous one"); +}); + +test("ingest breaks a plurality tie toward the more severe outcome", () => { + const tie = ingestReviewerConsensusCalibrationSignals([ + signal({ dimensions: [{ dimension: "security", votes: ["pass", "fail"] }] }), + ]).accepted[0]!; + assert.equal(tie.dimensions[0]!.majorityOutcome, "fail"); + assert.equal(tie.dimensions[0]!.agreement, 0.5); + + const warnVsPass = ingestReviewerConsensusCalibrationSignals([ + signal({ dimensions: [{ dimension: "security", votes: ["warn", "pass"] }] }), + ]).accepted[0]!; + assert.equal(warnVsPass.dimensions[0]!.majorityOutcome, "warn"); +}); + +test("ingest normalizes vote aliases and drops unrecognized/abstention votes", () => { + const aliased = ingestReviewerConsensusCalibrationSignals([ + signal({ dimensions: [{ dimension: "ci", votes: ["success", "approve", "reject"] }] }), + ]).accepted[0]!; + // success/approve -> pass (2), reject -> fail (1) : majority pass, agreement 2/3 + assert.equal(aliased.dimensions[0]!.majorityOutcome, "pass"); + assert.equal(aliased.dimensions[0]!.voteCount, 3); + + const withAbstentions = ingestReviewerConsensusCalibrationSignals([ + signal({ dimensions: [{ dimension: "ci", votes: ["pass", "unknown", "???", "pass"] }] }), + ]).accepted[0]!; + // Only the two definite "pass" votes count. + assert.equal(withAbstentions.dimensions[0]!.voteCount, 2); + assert.equal(withAbstentions.dimensions[0]!.agreement, 1); +}); + +test("ingest aggregates repeated dimensions, normalizes dimension aliases, and preserves order", () => { + const aggregated = ingestReviewerConsensusCalibrationSignals([ + signal({ + dimensions: [ + { dimension: "coverage", votes: ["pass"] }, // alias -> tests + { dimension: "tests", votes: ["fail"] }, + { dimension: "correctness", votes: ["pass", "pass"] }, + ], + }), + ]).accepted[0]!; + assert.deepEqual( + aggregated.dimensions.map((dimension) => dimension.dimension), + ["correctness", "tests"], + ); + const tests = aggregated.dimensions.find((dimension) => dimension.dimension === "tests")!; + assert.equal(tests.voteCount, 2); + assert.equal(tests.agreement, 0.5); // one pass + one fail +}); + +test("ingest weights per-dimension agreement by vote count", () => { + const result = ingestReviewerConsensusCalibrationSignals([ + signal({ + dimensions: [ + { dimension: "correctness", votes: ["pass", "pass", "pass", "fail"] }, // 4 votes, agreement 3/4 + { dimension: "tests", votes: ["pass", "fail"] }, // 2 votes, agreement 1/2 + ], + }), + ]).accepted[0]!; + // weighted = (4 * 3/4 + 2 * 1/2) / 6 = (3 + 1) / 6 = 2/3 + assert.equal(result.score, Math.round((2 / 3) * 1_000_000) / 1_000_000); +}); + +test("ingest handles a three-way split and a single-reviewer dimension", () => { + // A fully three-way split (pass/warn/fail) has a plurality of 1 out of 3 definite votes → agreement 1/3, with the + // tie broken toward the most severe outcome. + const threeWay = ingestReviewerConsensusCalibrationSignals([ + signal({ dimensions: [{ dimension: "correctness", votes: ["pass", "warn", "fail"] }] }), + ]).accepted[0]!; + assert.equal(threeWay.dimensions[0]!.voteCount, 3); + assert.equal(threeWay.dimensions[0]!.majorityOutcome, "fail"); + assert.equal(threeWay.dimensions[0]!.agreement, Math.round((1 / 3) * 1_000_000) / 1_000_000); + + // A single reviewer trivially agrees with itself: one definite vote → agreement 1. + const single = ingestReviewerConsensusCalibrationSignals([ + signal({ dimensions: [{ dimension: "policy", votes: ["warn"] }] }), + ]).accepted[0]!; + assert.deepEqual(single.dimensions, [ + { dimension: "policy", voteCount: 1, majorityOutcome: "warn", agreement: 1, score: 1 }, + ]); + assert.equal(single.score, 1); +}); + +test("ingest drops dimensions with no definite votes, rejecting a signal left empty", () => { + const mixed = ingestReviewerConsensusCalibrationSignals([ + signal({ + dimensions: [ + { dimension: "correctness", votes: ["unknown", "???"] }, // dropped (no definite votes) + { dimension: "nonsense", votes: ["pass"] }, // dropped (unknown dimension) + { dimension: "security", votes: ["fail", "fail"] }, + ], + }), + ]); + assert.equal(mixed.accepted.length, 1); + assert.deepEqual( + mixed.accepted[0]!.dimensions.map((dimension) => dimension.dimension), + ["security"], + ); + + const empty = ingestReviewerConsensusCalibrationSignals([ + signal({ dimensions: [{ dimension: "correctness", votes: ["abstain"] }] }), + ]); + assert.equal(empty.accepted.length, 0); + assert.equal(empty.rejected[0]!.reason, "empty_dimensions"); +}); + +test("ingest rejects invalid repos, run ids, and non-opted-in signals with specific reasons", () => { + const result = ingestReviewerConsensusCalibrationSignals([ + signal({ repoFullName: "not-a-repo" }), + signal({ replayRunId: " " }), + signal({ reviewRunId: "bad\nid" }), + signal({ optedIn: false }), + signal(), + ]); + assert.equal(result.accepted.length, 1); + assert.deepEqual( + result.rejected.map((row) => row.reason), + ["invalid_repo", "invalid_run_id", "invalid_run_id", "not_opted_in"], + ); + assert.equal(result.rejected[0]!.repoFullName, "not-a-repo"); +}); + +test("ingest normalizes repo casing and observedAt to ISO, or null for an unparseable timestamp", () => { + const result = ingestReviewerConsensusCalibrationSignals([ + signal({ repoFullName: "ACME/Widgets", observedAt: "2026-07-04T00:00:00Z" }), + signal({ observedAt: "not-a-date" }), + ]); + assert.equal(result.accepted[0]!.repoFullName, "acme/widgets"); + assert.equal(result.accepted[0]!.observedAt, "2026-07-04T00:00:00.000Z"); + assert.equal(result.accepted[1]!.observedAt, null); +}); + +test("composite blends objective-anchor, pairwise, and reviewer-consensus, accepting numbers or score objects", () => { + const ingestion = ingestReviewerConsensusCalibrationSignals([signal()]); // structured score 1 + const withNumbers = computeReviewerConsensusCompositeCalibrationScore({ + objectiveAnchor: 0.8, + pairwise: 0.6, + reviewerConsensus: ingestion, + }); + const expected = Math.round((0.8 * 0.45 + 0.6 * 0.35 + 1 * 0.2) * 1_000_000) / 1_000_000; + assert.equal(withNumbers.compositeScore, expected); + assert.equal(withNumbers.structuredReviewerConsensusScore, 1); + assert.equal(withNumbers.audit.contributingRepos.length, 1); + + const inline = computeReviewerConsensusCompositeCalibrationScore({ + objectiveAnchor: 0.8, + pairwise: 0.6, + reviewerConsensus: [signal()], + }); + assert.equal(inline.compositeScore, withNumbers.compositeScore); + + const anchor = { score: 0.7 } as unknown as ObjectiveAnchorScore; + const pairwise = { pairwiseJudgeScore: 0.4 } as unknown as PairwiseCalibrationScore; + const withObjects = computeReviewerConsensusCompositeCalibrationScore({ + objectiveAnchor: anchor, + pairwise, + reviewerConsensus: ingestion, + }); + assert.equal(withObjects.objectiveAnchorScore, 0.7); + assert.equal(withObjects.pairwiseJudgeScore, 0.4); +}); + +test("composite drops the pairwise weight when pairwise is null and redistributes it", () => { + const ingestion = ingestReviewerConsensusCalibrationSignals([signal()]); + const result = computeReviewerConsensusCompositeCalibrationScore({ + objectiveAnchor: 0.8, + pairwise: null, + reviewerConsensus: ingestion, + }); + assert.equal(result.pairwiseJudgeScore, null); + assert.equal(result.weights.pairwiseJudge, 0); + const sum = + result.weights.objectiveAnchor + result.weights.pairwiseJudge + result.weights.structuredReviewerConsensus; + assert.ok(Math.abs(sum - 1) < 1e-9); + const expected = Math.round((0.8 * (0.45 / 0.65) + 1 * (0.2 / 0.65)) * 1_000_000) / 1_000_000; + assert.equal(result.compositeScore, expected); +}); + +test("composite drops the structured weight when no signal contributes", () => { + const result = computeReviewerConsensusCompositeCalibrationScore({ + objectiveAnchor: 0.5, + pairwise: 0.9, + reviewerConsensus: [signal({ optedIn: false })], + }); + assert.equal(result.structuredReviewerConsensusScore, null); + assert.equal(result.weights.structuredReviewerConsensus, 0); + const expected = Math.round((0.5 * (0.45 / 0.8) + 0.9 * (0.35 / 0.8)) * 1_000_000) / 1_000_000; + assert.equal(result.compositeScore, expected); + assert.equal(result.audit.rejected.length, 1); +}); + +test("composite honors custom weights and falls back to objective-only when all weights are zero", () => { + const ingestion = ingestReviewerConsensusCalibrationSignals([signal()]); + const weighted = computeReviewerConsensusCompositeCalibrationScore({ + objectiveAnchor: 0.4, + pairwise: 0.4, + reviewerConsensus: ingestion, + weights: { objectiveAnchor: 0, pairwiseJudge: 0, structuredReviewerConsensus: 1 }, + }); + assert.equal(weighted.compositeScore, 1); + + const allZero = computeReviewerConsensusCompositeCalibrationScore({ + objectiveAnchor: 0.4, + pairwise: 0.4, + reviewerConsensus: ingestion, + weights: { objectiveAnchor: 0, pairwiseJudge: 0, structuredReviewerConsensus: 0 }, + }); + // Explicitly zeroing every component falls back to objective-only — NOT the default 45/35/20 blend. + assert.deepEqual(allZero.weights, { objectiveAnchor: 1, pairwiseJudge: 0, structuredReviewerConsensus: 0 }); + assert.equal(allZero.compositeScore, 0.4); +}); + +test("renderAuditMarkdown is deterministic, public-safe, and reports contributors and rejections", () => { + const ingestion = ingestReviewerConsensusCalibrationSignals([ + signal({ repoFullName: "acme/widgets", observedAt: "2026-07-04T00:00:00Z" }), + signal({ repoFullName: "bad", replayRunId: "r2", reviewRunId: "v2" }), + ]); + const result = computeReviewerConsensusCompositeCalibrationScore({ + objectiveAnchor: 0.8, + pairwise: null, + reviewerConsensus: ingestion, + }); + const markdown = renderReviewerConsensusCalibrationAuditMarkdown(result); + assert.equal(markdown, renderReviewerConsensusCalibrationAuditMarkdown(result), "render must be deterministic"); + assert.ok(markdown.startsWith("# Structured Reviewer-Consensus Calibration\n")); + assert.ok(markdown.includes("### acme/widgets")); + assert.ok(markdown.includes("| correctness | 3 | pass |")); + assert.ok(markdown.includes("- pairwiseJudge: n/a")); + assert.ok(markdown.includes("invalid\\_repo")); + assert.ok(markdown.endsWith("\n")); +}); + +test("renderAuditMarkdown escapes markdown metacharacters in identifiers and handles the empty case", () => { + const ingestion = ingestReviewerConsensusCalibrationSignals([ + signal({ repoFullName: "acme/widgets", replayRunId: "run|with*meta_", reviewRunId: "v1" }), + ]); + const escaped = renderReviewerConsensusCalibrationAuditMarkdown( + computeReviewerConsensusCompositeCalibrationScore({ + objectiveAnchor: 0.5, + pairwise: 0.5, + reviewerConsensus: ingestion, + }), + ); + assert.ok(escaped.includes("run\\|with\\*meta\\_")); + assert.ok(!escaped.includes("run|with*meta_")); + + const empty = renderReviewerConsensusCalibrationAuditMarkdown( + computeReviewerConsensusCompositeCalibrationScore({ + objectiveAnchor: 0.5, + pairwise: null, + reviewerConsensus: [], + }), + ); + assert.ok(empty.includes("_No opted-in structured reviewer-consensus signals contributed._")); + assert.ok(empty.includes("## Rejected Rows\n\n- none")); + assert.ok(empty.includes("## Contributing Repo Summary\n\n- none")); +});