From 8cb3b9bc5b1f5abd50e5280d7ccbdf5361f0dc58 Mon Sep 17 00:00:00 2001 From: joaovictor91123 Date: Sat, 4 Jul 2026 21:12:59 +0400 Subject: [PATCH] feat(miner): add pairwise calibration scoring --- packages/gittensory-engine/README.md | 127 ++++ packages/gittensory-engine/src/index.ts | 27 + .../gittensory-engine/src/objective-anchor.ts | 555 ++++++++++++++++++ .../src/pairwise-calibration.ts | 159 +++++ .../test/objective-anchor.test.ts | 385 ++++++++++++ .../test/pairwise-calibration.test.ts | 155 +++++ 6 files changed, 1408 insertions(+) create mode 100644 packages/gittensory-engine/src/objective-anchor.ts create mode 100644 packages/gittensory-engine/src/pairwise-calibration.ts create mode 100644 packages/gittensory-engine/test/objective-anchor.test.ts create mode 100644 packages/gittensory-engine/test/pairwise-calibration.test.ts diff --git a/packages/gittensory-engine/README.md b/packages/gittensory-engine/README.md index 82d71a3053..ed3085a1ba 100644 --- a/packages/gittensory-engine/README.md +++ b/packages/gittensory-engine/README.md @@ -58,6 +58,133 @@ rankOpportunities(candidates); // sorted by descending score, each annotated wit `rankOpportunities` is a stable sort with an explicit index tie-break: candidates with an equal score keep their input order. +## Objective-anchor calibration + +`scoreObjectiveAnchor()` provides the deterministic half of historical replay calibration. It compares the structural +features of a miner replay against the revealed post-snapshot history without any model call, network call, wall-clock +read, or random input. + +The score is intended for replay harnesses that need an auditable floor before a pairwise judge runs. Callers pass +the replayed plan or PR target data and the revealed history target data: + +```ts +import { scoreObjectiveAnchor } from "@jsonbored/gittensory-engine"; + +const result = scoreObjectiveAnchor({ + replayed: { + paths: ["packages/gittensory-engine/src/opportunity-ranker.ts"], + labels: ["feature"], + titles: ["feat(miner): add deterministic opportunity ranking"], + }, + revealed: { + paths: ["packages/gittensory-engine/src/objective-anchor.ts"], + labels: ["feature"], + titles: ["feat(miner): add objective-anchor calibration scoring"], + }, +}); +``` + +The returned object includes: + +- `score`: a composite value in `[0, 1]`. +- `dimensions.paths`: exact/tight path overlap. +- `dimensions.modules`: coarser module overlap, so a replay that targets the right package but the wrong file receives + visible partial credit. +- `dimensions.changeKinds`: overlap between caller-supplied or inferred change classes. +- `audit`: normalized replayed/revealed feature sets, intersections, misses, and normalized weights. + +The default weight split is path-heavy but still gives module-level and kind-level signal: + +```ts +{ + paths: 0.45, + modules: 0.4, + changeKinds: 0.15 +} +``` + +Custom weights are normalized to sum to `1`. Negative, non-finite, or otherwise invalid weights are treated as `0`; +if every provided weight is unusable, the defaults are restored. + +Feature extraction is intentionally conservative: + +- Paths are normalized to lowercase slash paths, deduplicated, and sorted. +- Modules are derived only from paths, never guessed from free text. +- Change kinds can come from explicit `changeKinds`, issue/PR labels, titles, notes, and path conventions. +- If no change-kind signal exists, the kind is `unknown` so an opaque replay and opaque revealed history can still be + compared deterministically. + +Given the same inputs, `JSON.stringify(scoreObjectiveAnchor(input))` is byte-stable across runs. + +Replay harnesses that already represent the two sides as arrays of plans, PRs, or commits can use the history +helpers instead: + +```ts +import { scoreObjectiveAnchorHistory } from "@jsonbored/gittensory-engine"; + +const result = scoreObjectiveAnchorHistory({ + replayed: [ + { + id: "plan:objective-anchor", + source: "plan", + paths: ["packages/gittensory-engine/src/objective-anchor.ts"], + labels: ["feature"], + }, + ], + revealed: [ + { + id: "pr:3142", + source: "pull_request", + paths: ["packages/gittensory-engine/src/objective-anchor.ts"], + labels: ["feature"], + }, + ], +}); +``` + +`result.history.replayed.items` and `result.history.revealed.items` preserve the per-record normalized features, while +`result.audit` shows the aggregate intersections and misses used for the score. Empty histories remain valid inputs: +they produce empty path/module sets and an `unknown` change kind rather than throwing, so a replay batch can record a +low-information calibration row without special casing. + +For local replay artifacts, `renderObjectiveAnchorAuditMarkdown(result)` turns either score shape into a deterministic +Markdown report. It includes dimensions, weights, normalized feature sets, intersections, misses, and per-item history +evidence when present. Report values are escaped and collapsed to one line so caller-supplied ids or paths cannot +reshape the artifact. + +## Pairwise calibration + +`computePairwiseCalibrationScore()` is the deterministic half of the order-swapped pairwise judge layer. The miner +runtime owns the model calls; the engine package owns the stable post-processing contract: + +- run a judge attempt in both presentation orders, +- accept only outcomes that agree after inverting the swapped-order verdict, +- discard `incomparable` and order-flipping attempts, +- cap retries, +- track order-instability rate, +- combine the surviving pairwise average with the objective-anchor score. + +```ts +import { computePairwiseCalibrationScore } from "@jsonbored/gittensory-engine"; + +const result = computePairwiseCalibrationScore({ + objectiveAnchor: 0.55, + samples: [ + { + attempts: [ + { + replayFirst: "replay_better", + revealedFirst: "revealed_better", + }, + ], + }, + ], +}); +``` + +If every pairwise sample is unstable, the composite falls back to the objective-anchor score and records the failed +samples in `metrics` rather than averaging noise into the calibration signal. + ## Plan templates `plan-templates.ts` exports one builder per miner lifecycle stage (`analyze`, `plan`, `prepare`, `create`, `manage`). diff --git a/packages/gittensory-engine/src/index.ts b/packages/gittensory-engine/src/index.ts index 43b5f0cf32..d013b0e0ec 100644 --- a/packages/gittensory-engine/src/index.ts +++ b/packages/gittensory-engine/src/index.ts @@ -9,6 +9,33 @@ export { rankOpportunities, type OpportunityRankInput, } from "./opportunity-ranker.js"; +export { + extractObjectiveAnchorHistory, + extractObjectiveAnchorFeatures, + scoreObjectiveAnchor, + scoreObjectiveAnchorHistory, + renderObjectiveAnchorAuditMarkdown, + type ObjectiveAnchorAudit, + type ObjectiveAnchorChangeKind, + type ObjectiveAnchorDimensionScores, + type ObjectiveAnchorFeatures, + type ObjectiveAnchorHistoryExtraction, + type ObjectiveAnchorHistoryItem, + type ObjectiveAnchorHistoryItemAudit, + type ObjectiveAnchorHistoryScore, + type ObjectiveAnchorInput, + type ObjectiveAnchorScore, + type ObjectiveAnchorWeights, +} from "./objective-anchor.js"; +export { + computePairwiseCalibrationScore, + resolvePairwiseCalibrationSample, + type PairwiseCalibrationAttempt, + type PairwiseCalibrationResolvedSample, + type PairwiseCalibrationScore, + type PairwiseCalibrationVerdict, + type PairwiseCalibrationWeights, +} from "./pairwise-calibration.js"; export * from "./governor/rate-limit.js"; export { GOVERNOR_LEDGER_EVENT_TYPES, diff --git a/packages/gittensory-engine/src/objective-anchor.ts b/packages/gittensory-engine/src/objective-anchor.ts new file mode 100644 index 0000000000..86d7bb72d1 --- /dev/null +++ b/packages/gittensory-engine/src/objective-anchor.ts @@ -0,0 +1,555 @@ +// Deterministic objective-anchor scoring for historical replay calibration (#3012). +// +// The replay harness needs a stable, auditable score before any LLM judge is involved: compare what the miner +// planned or changed against what the revealed post-T history actually changed. This module is deliberately pure: +// no IO, no Date, no random, no model calls. Given the same replayed and revealed records, it returns the same +// normalized features, dimension scores, composite score, and audit payload byte-for-byte. + +export type ObjectiveAnchorChangeKind = + | "feature" + | "fix" + | "test" + | "docs" + | "refactor" + | "config" + | "ci" + | "security" + | "dependency" + | "unknown"; + +export type ObjectiveAnchorInput = { + /** Paths touched or explicitly targeted by a replayed plan/PR or by revealed history. */ + paths?: readonly string[] | undefined; + /** Labels from an issue, PR, or local candidate record. Used only for change-kind extraction. */ + labels?: readonly string[] | undefined; + /** Titles or short subjects. Used only for change-kind extraction. */ + titles?: readonly string[] | undefined; + /** Longer plan/review/commit notes. Used only for change-kind extraction. */ + notes?: readonly string[] | undefined; + /** Optional already-classified kinds from an upstream caller. */ + changeKinds?: readonly ObjectiveAnchorChangeKind[] | undefined; +}; + +export type ObjectiveAnchorHistoryItem = ObjectiveAnchorInput & { + /** Stable caller-side identifier, e.g. `plan:abc`, `pr:123`, or `commit:deadbeef`. */ + id?: string | undefined; + /** Human-readable source bucket for audit reports. */ + source?: "plan" | "pull_request" | "commit" | "issue" | "manual" | "unknown" | undefined; +}; + +export type ObjectiveAnchorFeatures = { + /** Stable, normalized file paths with duplicates removed. */ + paths: string[]; + /** Coarse module buckets derived from paths, e.g. `src/review`, `packages/gittensory-engine`, `docs`. */ + modules: string[]; + /** Inferred or caller-supplied change kinds with duplicates removed. */ + changeKinds: ObjectiveAnchorChangeKind[]; +}; + +export type ObjectiveAnchorHistoryItemAudit = { + id: string; + source: NonNullable; + features: ObjectiveAnchorFeatures; +}; + +export type ObjectiveAnchorHistoryExtraction = { + features: ObjectiveAnchorFeatures; + items: ObjectiveAnchorHistoryItemAudit[]; +}; + +export type ObjectiveAnchorWeights = { + /** Weight for exact/tight path overlap. Default: 0.45. */ + paths?: number | undefined; + /** Weight for coarser module overlap. Default: 0.4. */ + modules?: number | undefined; + /** Weight for change-kind overlap. Default: 0.15. */ + changeKinds?: number | undefined; +}; + +type NormalizedObjectiveAnchorWeights = { + paths: number; + modules: number; + changeKinds: number; +}; + +export type ObjectiveAnchorDimensionScores = { + paths: number; + modules: number; + changeKinds: number; +}; + +export type ObjectiveAnchorAudit = { + replayed: ObjectiveAnchorFeatures; + revealed: ObjectiveAnchorFeatures; + weights: NormalizedObjectiveAnchorWeights; + dimensions: ObjectiveAnchorDimensionScores; + intersections: { + paths: string[]; + modules: string[]; + changeKinds: ObjectiveAnchorChangeKind[]; + }; + misses: { + replayedOnlyPaths: string[]; + revealedOnlyPaths: string[]; + replayedOnlyModules: string[]; + revealedOnlyModules: string[]; + replayedOnlyChangeKinds: ObjectiveAnchorChangeKind[]; + revealedOnlyChangeKinds: ObjectiveAnchorChangeKind[]; + }; +}; + +export type ObjectiveAnchorScore = { + /** Composite score in [0, 1]. */ + score: number; + dimensions: ObjectiveAnchorDimensionScores; + audit: ObjectiveAnchorAudit; +}; + +export type ObjectiveAnchorHistoryScore = ObjectiveAnchorScore & { + history: { + replayed: ObjectiveAnchorHistoryExtraction; + revealed: ObjectiveAnchorHistoryExtraction; + }; +}; + +const DEFAULT_WEIGHTS: NormalizedObjectiveAnchorWeights = { + paths: 0.45, + modules: 0.4, + changeKinds: 0.15, +}; + +const CHANGE_KIND_ORDER: ObjectiveAnchorChangeKind[] = [ + "feature", + "fix", + "test", + "docs", + "refactor", + "config", + "ci", + "security", + "dependency", + "unknown", +]; + +const KIND_SYNONYMS: Array<[ObjectiveAnchorChangeKind, RegExp]> = [ + ["feature", /\b(feat|feature|enhancement|add|adds|introduce|support|capability)\b/iu], + ["fix", /\b(fix|bug|bugfix|regression|repair|broken|incorrect|failure|fails?)\b/iu], + ["test", /\b(test|tests|coverage|regression-test|vitest|unit|integration)\b/iu], + ["docs", /\b(doc|docs|readme|documentation|guide|quickstart|manual)\b/iu], + ["refactor", /\b(refactor|cleanup|simplify|extract|rename|restructure)\b/iu], + ["config", /\b(config|configuration|settings|env|schema|yaml|jsonc|wrangler|toml)\b/iu], + ["ci", /\b(ci|workflow|github-actions|actionlint|codecov|pipeline|build)\b/iu], + ["security", /\b(security|secret|token|credential|auth|permission|vulnerability|cve)\b/iu], + ["dependency", /\b(dependency|dependencies|deps|package-lock|npm|pnpm|yarn|version|upgrade|pin)\b/iu], +]; + +const DOC_EXTENSIONS = new Set([".md", ".mdx", ".rst", ".adoc", ".txt"]); +const TEST_SEGMENTS = new Set(["test", "tests", "__tests__", "spec", "specs"]); +const CI_SEGMENTS = new Set([".github", "workflows"]); +const CONFIG_FILENAMES = new Set([ + ".env", + ".env.example", + ".env.selfhost.example", + ".gittensory.yml", + "package.json", + "package-lock.json", + "tsconfig.json", + "tsconfig.test.json", + "wrangler.jsonc", + "vite.config.ts", + "vitest.config.ts", +]); + +function normalizePath(path: string): string | undefined { + const normalized = path.trim().replace(/\\/g, "/").replace(/\/+/g, "/").replace(/^\.\//u, ""); + if (!normalized || normalized === "." || normalized.includes("\0")) return undefined; + return normalized.toLowerCase(); +} + +function extensionOf(path: string): string { + const last = path.split("/").pop() ?? ""; + const dot = last.lastIndexOf("."); + return dot <= 0 ? "" : last.slice(dot); +} + +function uniqueSorted(values: Iterable): string[] { + return [...new Set(values)].sort((a, b) => a.localeCompare(b)); +} + +function uniqueKinds(kinds: Iterable): ObjectiveAnchorChangeKind[] { + const seen = new Set(kinds); + return CHANGE_KIND_ORDER.filter((kind) => seen.has(kind)); +} + +function combineFeatures(features: readonly ObjectiveAnchorFeatures[]): ObjectiveAnchorFeatures { + return { + paths: uniqueSorted(features.flatMap((feature) => feature.paths)), + modules: uniqueSorted(features.flatMap((feature) => feature.modules)), + changeKinds: uniqueKinds(features.flatMap((feature) => feature.changeKinds)), + }; +} + +function isKnownKind(kind: string): kind is ObjectiveAnchorChangeKind { + return (CHANGE_KIND_ORDER as string[]).includes(kind); +} + +function normalizeKind(value: string): ObjectiveAnchorChangeKind | undefined { + const normalized = value.trim().toLowerCase().replace(/[_\s]+/gu, "-"); + if (isKnownKind(normalized)) return normalized; + if (normalized === "feat" || normalized === "enhancement") return "feature"; + if (normalized === "bug" || normalized === "bugfix" || normalized === "regression") return "fix"; + if (normalized === "documentation" || normalized === "readme") return "docs"; + if (normalized === "build" || normalized === "workflow") return "ci"; + if (normalized === "deps" || normalized === "package") return "dependency"; + return undefined; +} + +function pathModule(path: string): string { + const segments = path.split("/").filter(Boolean); + if (segments.length === 0) return "root"; + const [first, second] = segments; + if (first === "packages" || first === "apps") { + return second ? `${first}/${second}` : first; + } + if (first === "src" || first === "test" || first === "tests") { + return second ? `${first}/${second}` : first; + } + if (first === ".github") return segments[1] === "workflows" ? ".github/workflows" : ".github"; + return first!; +} + +function kindsFromPath(path: string): ObjectiveAnchorChangeKind[] { + const segments = path.split("/"); + const filename = segments[segments.length - 1] ?? path; + const kinds: ObjectiveAnchorChangeKind[] = []; + if (segments.some((segment) => TEST_SEGMENTS.has(segment)) || /\.test\.|\.spec\./u.test(filename)) { + kinds.push("test"); + } + if (DOC_EXTENSIONS.has(extensionOf(path)) || segments.includes("docs") || filename.toLowerCase() === "readme.md") { + kinds.push("docs"); + } + if (segments.some((segment) => CI_SEGMENTS.has(segment)) || filename.endsWith(".yml") || filename.endsWith(".yaml")) { + kinds.push("ci"); + } + if (CONFIG_FILENAMES.has(filename) || filename.endsWith(".jsonc") || filename.endsWith(".toml")) { + kinds.push("config"); + } + if (/package(?:-lock)?\.json$/u.test(filename)) { + kinds.push("dependency"); + } + return kinds; +} + +function kindsFromText(values: readonly string[] | undefined): ObjectiveAnchorChangeKind[] { + if (!values) return []; + const kinds: ObjectiveAnchorChangeKind[] = []; + for (const value of values) { + for (const explicit of value.split(/[,\s/()[\]{}:;]+/u)) { + const normalized = normalizeKind(explicit); + if (normalized) kinds.push(normalized); + } + for (const [kind, pattern] of KIND_SYNONYMS) { + if (pattern.test(value)) kinds.push(kind); + } + } + return kinds; +} + +function normalizeWeights(weights: ObjectiveAnchorWeights | undefined): NormalizedObjectiveAnchorWeights { + const raw = { + paths: finiteNonNegative(weights?.paths, DEFAULT_WEIGHTS.paths), + modules: finiteNonNegative(weights?.modules, DEFAULT_WEIGHTS.modules), + changeKinds: finiteNonNegative(weights?.changeKinds, DEFAULT_WEIGHTS.changeKinds), + }; + const total = raw.paths + raw.modules + raw.changeKinds; + if (total <= 0) return DEFAULT_WEIGHTS; + return { + paths: raw.paths / total, + modules: raw.modules / total, + changeKinds: raw.changeKinds / total, + }; +} + +function finiteNonNegative(value: number | undefined, fallback: number): number { + if (value === undefined) return fallback; + if (!Number.isFinite(value) || value < 0) return 0; + return value; +} + +function diceOverlap(left: readonly string[], right: readonly string[]): number { + if (left.length === 0 && right.length === 0) return 1; + if (left.length === 0 || right.length === 0) return 0; + const rightSet = new Set(right); + const intersection = left.filter((value) => rightSet.has(value)).length; + return (2 * intersection) / (left.length + right.length); +} + +function intersectStrings(left: readonly string[], right: readonly string[]): string[] { + const rightSet = new Set(right); + return left.filter((value) => rightSet.has(value)); +} + +function differenceStrings(left: readonly string[], right: readonly string[]): string[] { + const rightSet = new Set(right); + return left.filter((value) => !rightSet.has(value)); +} + +function intersectKinds( + left: readonly ObjectiveAnchorChangeKind[], + right: readonly ObjectiveAnchorChangeKind[], +): ObjectiveAnchorChangeKind[] { + const rightSet = new Set(right); + return CHANGE_KIND_ORDER.filter((kind) => left.includes(kind) && rightSet.has(kind)); +} + +function differenceKinds( + left: readonly ObjectiveAnchorChangeKind[], + right: readonly ObjectiveAnchorChangeKind[], +): ObjectiveAnchorChangeKind[] { + const rightSet = new Set(right); + return CHANGE_KIND_ORDER.filter((kind) => left.includes(kind) && !rightSet.has(kind)); +} + +function roundScore(value: number): number { + return Math.round(Math.min(1, Math.max(0, value)) * 1_000_000) / 1_000_000; +} + +function auditItemId(item: ObjectiveAnchorHistoryItem, index: number): string { + const trimmed = item.id?.trim(); + return trimmed ? trimmed : `item:${index + 1}`; +} + +function auditItemSource(item: ObjectiveAnchorHistoryItem): NonNullable { + return item.source ?? "unknown"; +} + +function markdownSafe(value: string): string { + return value.replace(/[\r\n]+/gu, " ").replace(/[\\`*_[\]<>|]/gu, "\\$&"); +} + +function markdownList(values: readonly string[]): string { + if (values.length === 0) return "- none"; + return values.map((value) => `- ${markdownSafe(value)}`).join("\n"); +} + +function markdownKindList(values: readonly ObjectiveAnchorChangeKind[]): string { + return markdownList(values); +} + +function markdownFeatureBlock(features: ObjectiveAnchorFeatures): string { + return [ + "Paths:", + markdownList(features.paths), + "", + "Modules:", + markdownList(features.modules), + "", + "Change kinds:", + markdownKindList(features.changeKinds), + ].join("\n"); +} + +function markdownHistoryBlock(extraction: ObjectiveAnchorHistoryExtraction): string { + if (extraction.items.length === 0) return "_No history items._"; + return extraction.items + .map((item) => + [ + `### ${markdownSafe(item.id)} (${markdownSafe(item.source)})`, + "", + markdownFeatureBlock(item.features), + ].join("\n"), + ) + .join("\n\n"); +} + +/** + * Extract normalized structural features from replayed or revealed history input. The extractor is intentionally + * conservative: paths determine modules, and labels/titles/notes only classify change kind. It never guesses a + * module from free text, which keeps the path/module score auditable and reproducible. + */ +export function extractObjectiveAnchorFeatures(input: ObjectiveAnchorInput): ObjectiveAnchorFeatures { + const paths = uniqueSorted((input.paths ?? []).map(normalizePath).filter((path): path is string => Boolean(path))); + const modules = uniqueSorted(paths.map(pathModule)); + const directKinds = (input.changeKinds ?? []).filter((kind): kind is ObjectiveAnchorChangeKind => isKnownKind(kind)); + const textKinds = kindsFromText([...(input.labels ?? []), ...(input.titles ?? []), ...(input.notes ?? [])]); + const pathKinds = paths.flatMap(kindsFromPath); + const changeKinds = uniqueKinds([...directKinds, ...textKinds, ...pathKinds]); + return { + paths, + modules, + changeKinds: changeKinds.length > 0 ? changeKinds : ["unknown"], + }; +} + +/** + * Extract and aggregate structural features from a replay/revealed history list. Each item keeps its own normalized + * feature set in `items` for auditability, while `features` is the deduplicated union used for scoring. Empty history + * is valid and produces empty path/module sets with an `unknown` change kind, matching single-input extraction. + */ +export function extractObjectiveAnchorHistory(items: readonly ObjectiveAnchorHistoryItem[]): ObjectiveAnchorHistoryExtraction { + const itemAudits = items.map((item, index) => ({ + id: auditItemId(item, index), + source: auditItemSource(item), + features: extractObjectiveAnchorFeatures(item), + })); + const features = itemAudits.length > 0 ? combineFeatures(itemAudits.map((item) => item.features)) : extractObjectiveAnchorFeatures({}); + return { features, items: itemAudits }; +} + +/** + * Score replayed structural features against revealed history. Path and module dimensions use Dice overlap so a + * partial module match gets visible credit without pretending it is exact. Change-kind overlap is the same metric + * over inferred/caller-supplied kinds. The revealed side may have zero overlapping modules; that is a valid low + * score, never an error, and the misses section explains what diverged. + */ +export function scoreObjectiveAnchor(input: { + replayed: ObjectiveAnchorInput | ObjectiveAnchorFeatures; + revealed: ObjectiveAnchorInput | ObjectiveAnchorFeatures; + weights?: ObjectiveAnchorWeights | undefined; +}): ObjectiveAnchorScore { + const replayed = isFeatures(input.replayed) ? input.replayed : extractObjectiveAnchorFeatures(input.replayed); + const revealed = isFeatures(input.revealed) ? input.revealed : extractObjectiveAnchorFeatures(input.revealed); + const weights = normalizeWeights(input.weights); + const dimensions: ObjectiveAnchorDimensionScores = { + paths: roundScore(diceOverlap(replayed.paths, revealed.paths)), + modules: roundScore(diceOverlap(replayed.modules, revealed.modules)), + changeKinds: roundScore(diceOverlap(replayed.changeKinds, revealed.changeKinds)), + }; + const score = roundScore( + dimensions.paths * weights.paths + dimensions.modules * weights.modules + dimensions.changeKinds * weights.changeKinds, + ); + + return { + score, + dimensions, + audit: { + replayed, + revealed, + weights, + dimensions, + intersections: { + paths: intersectStrings(replayed.paths, revealed.paths), + modules: intersectStrings(replayed.modules, revealed.modules), + changeKinds: intersectKinds(replayed.changeKinds, revealed.changeKinds), + }, + misses: { + replayedOnlyPaths: differenceStrings(replayed.paths, revealed.paths), + revealedOnlyPaths: differenceStrings(revealed.paths, replayed.paths), + replayedOnlyModules: differenceStrings(replayed.modules, revealed.modules), + revealedOnlyModules: differenceStrings(revealed.modules, replayed.modules), + replayedOnlyChangeKinds: differenceKinds(replayed.changeKinds, revealed.changeKinds), + revealedOnlyChangeKinds: differenceKinds(revealed.changeKinds, replayed.changeKinds), + }, + }, + }; +} + +/** + * Score arrays of replayed and revealed history records, preserving the per-record extraction evidence alongside the + * normal score/audit payload. This is the ergonomic entrypoint for replay harnesses that compare a generated plan/PR + * bundle with multiple revealed commits or merged PRs after the snapshot timestamp. + */ +export function scoreObjectiveAnchorHistory(input: { + replayed: readonly ObjectiveAnchorHistoryItem[]; + revealed: readonly ObjectiveAnchorHistoryItem[]; + weights?: ObjectiveAnchorWeights | undefined; +}): ObjectiveAnchorHistoryScore { + const replayed = extractObjectiveAnchorHistory(input.replayed); + const revealed = extractObjectiveAnchorHistory(input.revealed); + const score = scoreObjectiveAnchor({ + replayed: replayed.features, + revealed: revealed.features, + weights: input.weights, + }); + return { + ...score, + history: { replayed, revealed }, + }; +} + +/** + * Render the score audit as deterministic Markdown for local replay artifacts. The renderer escapes Markdown control + * characters and collapses newlines in untrusted ids/paths so a caller can persist the output next to a replay run + * without letting a path or caller-supplied id reshape the report. + */ +export function renderObjectiveAnchorAuditMarkdown(result: ObjectiveAnchorScore | ObjectiveAnchorHistoryScore): string { + const lines = [ + "# Objective-Anchor Score", + "", + `Score: ${result.score.toFixed(6)}`, + "", + "## Dimensions", + "", + `- paths: ${result.dimensions.paths.toFixed(6)}`, + `- modules: ${result.dimensions.modules.toFixed(6)}`, + `- changeKinds: ${result.dimensions.changeKinds.toFixed(6)}`, + "", + "## Weights", + "", + `- paths: ${result.audit.weights.paths.toFixed(6)}`, + `- modules: ${result.audit.weights.modules.toFixed(6)}`, + `- changeKinds: ${result.audit.weights.changeKinds.toFixed(6)}`, + "", + "## Replayed Features", + "", + markdownFeatureBlock(result.audit.replayed), + "", + "## Revealed Features", + "", + markdownFeatureBlock(result.audit.revealed), + "", + "## Intersections", + "", + "Paths:", + markdownList(result.audit.intersections.paths), + "", + "Modules:", + markdownList(result.audit.intersections.modules), + "", + "Change kinds:", + markdownKindList(result.audit.intersections.changeKinds), + "", + "## Misses", + "", + "Replayed-only paths:", + markdownList(result.audit.misses.replayedOnlyPaths), + "", + "Revealed-only paths:", + markdownList(result.audit.misses.revealedOnlyPaths), + "", + "Replayed-only modules:", + markdownList(result.audit.misses.replayedOnlyModules), + "", + "Revealed-only modules:", + markdownList(result.audit.misses.revealedOnlyModules), + "", + "Replayed-only change kinds:", + markdownKindList(result.audit.misses.replayedOnlyChangeKinds), + "", + "Revealed-only change kinds:", + markdownKindList(result.audit.misses.revealedOnlyChangeKinds), + ]; + + if ("history" in result) { + lines.push( + "", + "## Replayed History Items", + "", + markdownHistoryBlock(result.history.replayed), + "", + "## Revealed History Items", + "", + markdownHistoryBlock(result.history.revealed), + ); + } + + return `${lines.join("\n")}\n`; +} + +function isFeatures(value: ObjectiveAnchorInput | ObjectiveAnchorFeatures): value is ObjectiveAnchorFeatures { + return ( + Array.isArray((value as ObjectiveAnchorFeatures).paths) && + Array.isArray((value as ObjectiveAnchorFeatures).modules) && + Array.isArray((value as ObjectiveAnchorFeatures).changeKinds) + ); +} diff --git a/packages/gittensory-engine/src/pairwise-calibration.ts b/packages/gittensory-engine/src/pairwise-calibration.ts new file mode 100644 index 0000000000..e85f4b91cf --- /dev/null +++ b/packages/gittensory-engine/src/pairwise-calibration.ts @@ -0,0 +1,159 @@ +// Deterministic pairwise-judge calibration combiner (#3013). +// +// The model invocation itself belongs to the miner runtime. This engine module owns the pure part: interpret the +// two order-swapped judge outputs, discard unstable pairs, cap retries, expose instability metrics, and combine the +// surviving judge score with the objective-anchor score. + +import type { ObjectiveAnchorScore } from "./objective-anchor.js"; + +export type PairwiseCalibrationVerdict = "replay_better" | "revealed_better" | "tie" | "incomparable"; + +export type PairwiseCalibrationAttempt = { + /** Judge result when replayed output is shown first and revealed history second. */ + replayFirst: PairwiseCalibrationVerdict; + /** Judge result when revealed history is shown first and replayed output second. */ + revealedFirst: PairwiseCalibrationVerdict; +}; + +export type PairwiseCalibrationWeights = { + objectiveAnchor?: number | undefined; + pairwiseJudge?: number | undefined; +}; + +export type PairwiseCalibrationResolvedSample = { + stable: boolean; + exhausted: boolean; + attemptsUsed: number; + maxAttempts: number; + verdict: PairwiseCalibrationVerdict | "unstable"; + pairwiseScore: number | null; +}; + +export type PairwiseCalibrationScore = { + compositeScore: number; + objectiveAnchorScore: number; + pairwiseJudgeScore: number | null; + weights: { objectiveAnchor: number; pairwiseJudge: number }; + samples: PairwiseCalibrationResolvedSample[]; + metrics: { + totalSamples: number; + stableSamples: number; + unstableSamples: number; + exhaustedSamples: number; + orderInstabilityRate: number; + }; +}; + +const DEFAULT_PAIRWISE_WEIGHTS = { + objectiveAnchor: 0.5, + pairwiseJudge: 0.5, +}; + +function finiteNonNegative(value: number | undefined, fallback: number): number { + if (value === undefined) return fallback; + if (!Number.isFinite(value) || value < 0) return 0; + return value; +} + +function normalizePairwiseWeights(weights: PairwiseCalibrationWeights | undefined): { + objectiveAnchor: number; + pairwiseJudge: number; +} { + const raw = { + objectiveAnchor: finiteNonNegative(weights?.objectiveAnchor, DEFAULT_PAIRWISE_WEIGHTS.objectiveAnchor), + pairwiseJudge: finiteNonNegative(weights?.pairwiseJudge, DEFAULT_PAIRWISE_WEIGHTS.pairwiseJudge), + }; + const total = raw.objectiveAnchor + raw.pairwiseJudge; + if (total <= 0) return DEFAULT_PAIRWISE_WEIGHTS; + return { + objectiveAnchor: raw.objectiveAnchor / total, + pairwiseJudge: raw.pairwiseJudge / total, + }; +} + +function invertedVerdict(verdict: PairwiseCalibrationVerdict): PairwiseCalibrationVerdict { + if (verdict === "replay_better") return "revealed_better"; + if (verdict === "revealed_better") return "replay_better"; + return verdict; +} + +function verdictScore(verdict: PairwiseCalibrationVerdict): number | null { + if (verdict === "replay_better") return 1; + if (verdict === "tie") return 0.5; + if (verdict === "revealed_better") return 0; + return null; +} + +function roundScore(value: number): number { + return Math.round(Math.min(1, Math.max(0, value)) * 1_000_000) / 1_000_000; +} + +export function resolvePairwiseCalibrationSample(input: { + attempts: readonly PairwiseCalibrationAttempt[]; + maxAttempts?: number | undefined; +}): PairwiseCalibrationResolvedSample { + const requestedMaxAttempts = input.maxAttempts ?? input.attempts.length; + const maxAttempts = Math.max(1, Math.floor(requestedMaxAttempts || 1)); + const attempts = input.attempts.slice(0, maxAttempts); + for (let index = 0; index < attempts.length; index += 1) { + const attempt = attempts[index]!; + const stable = attempt.replayFirst === invertedVerdict(attempt.revealedFirst); + if (stable) { + const score = verdictScore(attempt.replayFirst); + if (score !== null) { + return { + stable: true, + exhausted: false, + attemptsUsed: index + 1, + maxAttempts, + verdict: attempt.replayFirst, + pairwiseScore: score, + }; + } + } + } + return { + stable: false, + exhausted: attempts.length >= maxAttempts, + attemptsUsed: attempts.length, + maxAttempts, + verdict: "unstable", + pairwiseScore: null, + }; +} + +export function computePairwiseCalibrationScore(input: { + objectiveAnchor: number | ObjectiveAnchorScore; + samples: readonly { attempts: readonly PairwiseCalibrationAttempt[]; maxAttempts?: number | undefined }[]; + weights?: PairwiseCalibrationWeights | undefined; +}): PairwiseCalibrationScore { + const objectiveAnchorScore = + typeof input.objectiveAnchor === "number" ? roundScore(input.objectiveAnchor) : input.objectiveAnchor.score; + const samples = input.samples.map(resolvePairwiseCalibrationSample); + const stableScores = samples + .map((sample) => sample.pairwiseScore) + .filter((score): score is number => score !== null); + const pairwiseJudgeScore = + stableScores.length === 0 ? null : roundScore(stableScores.reduce((sum, score) => sum + score, 0) / stableScores.length); + const weights = normalizePairwiseWeights(input.weights); + const compositeScore = + pairwiseJudgeScore === null + ? objectiveAnchorScore + : roundScore(objectiveAnchorScore * weights.objectiveAnchor + pairwiseJudgeScore * weights.pairwiseJudge); + const unstableSamples = samples.filter((sample) => !sample.stable).length; + const exhaustedSamples = samples.filter((sample) => sample.exhausted).length; + return { + compositeScore, + objectiveAnchorScore, + pairwiseJudgeScore, + weights, + samples, + metrics: { + totalSamples: samples.length, + stableSamples: stableScores.length, + unstableSamples, + exhaustedSamples, + orderInstabilityRate: samples.length === 0 ? 0 : roundScore(unstableSamples / samples.length), + }, + }; +} diff --git a/packages/gittensory-engine/test/objective-anchor.test.ts b/packages/gittensory-engine/test/objective-anchor.test.ts new file mode 100644 index 0000000000..f6a92000f9 --- /dev/null +++ b/packages/gittensory-engine/test/objective-anchor.test.ts @@ -0,0 +1,385 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { + extractObjectiveAnchorHistory, + extractObjectiveAnchorFeatures, + renderObjectiveAnchorAuditMarkdown, + scoreObjectiveAnchor, + scoreObjectiveAnchorHistory, + type ObjectiveAnchorInput, +} from "../dist/index.js"; + +function replay(overrides: ObjectiveAnchorInput = {}): ObjectiveAnchorInput { + return { + paths: ["packages/gittensory-engine/src/opportunity-ranker.ts", "packages/gittensory-engine/test/opportunity-ranker.test.ts"], + labels: ["gittensor:feature"], + titles: ["feat(miner): add deterministic ranking"], + ...overrides, + }; +} + +function revealed(overrides: ObjectiveAnchorInput = {}): ObjectiveAnchorInput { + return { + paths: ["packages/gittensory-engine/src/opportunity-ranker.ts", "packages/gittensory-engine/test/opportunity-ranker.test.ts"], + labels: ["gittensor:feature"], + titles: ["feat(miner): add deterministic ranking"], + ...overrides, + }; +} + +test("barrel: the public entrypoint re-exports objective-anchor APIs (#3012)", () => { + assert.equal(typeof extractObjectiveAnchorFeatures, "function"); + assert.equal(typeof extractObjectiveAnchorHistory, "function"); + assert.equal(typeof scoreObjectiveAnchor, "function"); + assert.equal(typeof scoreObjectiveAnchorHistory, "function"); + assert.equal(typeof renderObjectiveAnchorAuditMarkdown, "function"); +}); + +test("extractObjectiveAnchorFeatures normalizes paths, derives modules, and classifies change kind", () => { + const features = extractObjectiveAnchorFeatures({ + paths: [ + ".\\Packages\\Gittensory-Engine\\src\\Objective-Anchor.ts", + "./packages/gittensory-engine/src/objective-anchor.ts", + "README.md", + ".github/workflows/ci.yml", + "package.json", + "", + ], + labels: ["enhancement", "security"], + titles: ["refactor scoring support"], + notes: ["adds tests and docs"], + }); + + assert.deepEqual(features.paths, [ + ".github/workflows/ci.yml", + "package.json", + "packages/gittensory-engine/src/objective-anchor.ts", + "readme.md", + ]); + assert.deepEqual(features.modules, [".github/workflows", "package.json", "packages/gittensory-engine", "readme.md"]); + assert.deepEqual(features.changeKinds, [ + "feature", + "test", + "docs", + "refactor", + "config", + "ci", + "security", + "dependency", + ]); +}); + +test("scoreObjectiveAnchor returns 1 for full structural overlap", () => { + const result = scoreObjectiveAnchor({ replayed: replay(), revealed: revealed() }); + + assert.equal(result.score, 1); + assert.deepEqual(result.dimensions, { paths: 1, modules: 1, changeKinds: 1 }); + assert.deepEqual(result.audit.intersections.paths, [ + "packages/gittensory-engine/src/opportunity-ranker.ts", + "packages/gittensory-engine/test/opportunity-ranker.test.ts", + ]); + assert.deepEqual(result.audit.misses.revealedOnlyModules, []); +}); + +test("scoreObjectiveAnchor gives a low floor, not an error, when revealed history touches zero overlapping modules", () => { + const result = scoreObjectiveAnchor({ + replayed: replay({ + paths: ["packages/gittensory-engine/src/opportunity-ranker.ts"], + labels: ["feature"], + }), + revealed: revealed({ + paths: ["apps/gittensory-ui/src/routes/index.tsx"], + labels: ["bug"], + titles: ["fix(site): repair the homepage"], + }), + }); + + assert.equal(result.dimensions.paths, 0); + assert.equal(result.dimensions.modules, 0); + assert.equal(result.dimensions.changeKinds, 0); + assert.equal(result.score, 0); + assert.deepEqual(result.audit.misses.replayedOnlyModules, ["packages/gittensory-engine"]); + assert.deepEqual(result.audit.misses.revealedOnlyModules, ["apps/gittensory-ui"]); +}); + +test("scoreObjectiveAnchor grants partial module credit when paths differ inside the same module", () => { + const result = scoreObjectiveAnchor({ + replayed: replay({ + paths: ["packages/gittensory-engine/src/opportunity-ranker.ts"], + labels: ["feature"], + }), + revealed: revealed({ + paths: ["packages/gittensory-engine/src/objective-anchor.ts"], + labels: ["feature"], + }), + }); + + assert.equal(result.dimensions.paths, 0); + assert.equal(result.dimensions.modules, 1); + assert.equal(result.dimensions.changeKinds, 1); + assert.equal(result.score, 0.55); + assert.deepEqual(result.audit.intersections.modules, ["packages/gittensory-engine"]); +}); + +test("scoreObjectiveAnchor exposes deterministic intermediate features for audit without rerunning extraction", () => { + const result = scoreObjectiveAnchor({ + replayed: replay({ + paths: ["src/review/enrichment-wire.ts", "test/unit/enrichment-wire.test.ts"], + labels: ["bug"], + titles: [], + notes: ["regression test for an auth failure"], + }), + revealed: revealed({ + paths: ["src/review/enrichment-wire.ts", "src/review/enrichment-config.ts"], + labels: ["fix"], + titles: [], + notes: ["auth configuration bug fixed"], + }), + }); + + assert.deepEqual(result.audit.replayed.modules, ["src/review", "test/unit"]); + assert.deepEqual(result.audit.revealed.modules, ["src/review"]); + assert.deepEqual(result.audit.intersections.paths, ["src/review/enrichment-wire.ts"]); + assert.deepEqual(result.audit.intersections.changeKinds, ["fix", "security"]); + assert.deepEqual(result.audit.misses.replayedOnlyPaths, ["test/unit/enrichment-wire.test.ts"]); + assert.deepEqual(result.audit.misses.revealedOnlyPaths, ["src/review/enrichment-config.ts"]); +}); + +test("scoreObjectiveAnchor is byte-stable for the same inputs and normalized weights", () => { + const input = { + replayed: replay({ + paths: ["src/mcp/server.ts", "test/unit/mcp-discovery.test.ts"], + labels: ["feature"], + }), + revealed: revealed({ + paths: ["src/mcp/server.ts", "src/mcp/schema.ts", "test/unit/mcp-find-opportunities.test.ts"], + labels: ["feature"], + }), + weights: { paths: 9, modules: 8, changeKinds: 3 }, + }; + + const first = JSON.stringify(scoreObjectiveAnchor(input)); + const second = JSON.stringify(scoreObjectiveAnchor(input)); + + assert.equal(first, second); +}); + +test("scoreObjectiveAnchor normalizes caller weights and ignores invalid weight values", () => { + const result = scoreObjectiveAnchor({ + replayed: replay({ paths: ["src/review/a.ts"], labels: ["feature"] }), + revealed: revealed({ paths: ["src/review/b.ts"], labels: ["feature"] }), + weights: { paths: Number.NaN, modules: 8, changeKinds: -1 }, + }); + + assert.deepEqual(result.audit.weights, { paths: 0, modules: 1, changeKinds: 0 }); + assert.equal(result.score, 1); +}); + +test("scoreObjectiveAnchor accepts pre-extracted features from a caller-side cache", () => { + const replayed = extractObjectiveAnchorFeatures(replay({ paths: ["src/rules/predicted-gate.ts"], labels: ["fix"] })); + const revealedFeatures = extractObjectiveAnchorFeatures( + revealed({ paths: ["src/rules/predicted-gate.ts"], labels: ["fix"] }), + ); + + const result = scoreObjectiveAnchor({ replayed, revealed: revealedFeatures }); + + assert.equal(result.score, 1); + assert.deepEqual(result.audit.replayed, replayed); + assert.deepEqual(result.audit.revealed, revealedFeatures); +}); + +test("scoreObjectiveAnchor falls back to unknown change kind when no kind signal exists", () => { + const result = scoreObjectiveAnchor({ + replayed: { paths: ["src/opaque.ts"] }, + revealed: { paths: ["src/other.ts"] }, + }); + + assert.deepEqual(result.audit.replayed.changeKinds, ["unknown"]); + assert.deepEqual(result.audit.revealed.changeKinds, ["unknown"]); + assert.equal(result.dimensions.changeKinds, 1); +}); + +test("extractObjectiveAnchorHistory keeps per-item extraction evidence and aggregates the union", () => { + const extraction = extractObjectiveAnchorHistory([ + { + id: "plan:1", + source: "plan", + paths: ["packages/gittensory-engine/src/objective-anchor.ts"], + labels: ["feature"], + }, + { + id: "commit:abc", + source: "commit", + paths: ["packages/gittensory-engine/test/objective-anchor.test.ts", "README.md"], + titles: ["test(engine): cover objective-anchor scoring"], + }, + ]); + + assert.deepEqual(extraction.features.paths, [ + "packages/gittensory-engine/src/objective-anchor.ts", + "packages/gittensory-engine/test/objective-anchor.test.ts", + "readme.md", + ]); + assert.deepEqual(extraction.features.modules, ["packages/gittensory-engine", "readme.md"]); + assert.deepEqual(extraction.features.changeKinds, ["feature", "test", "docs"]); + assert.deepEqual( + extraction.items.map((item) => [item.id, item.source, item.features.modules]), + [ + ["plan:1", "plan", ["packages/gittensory-engine"]], + ["commit:abc", "commit", ["packages/gittensory-engine", "readme.md"]], + ], + ); +}); + +test("extractObjectiveAnchorHistory supplies deterministic ids and unknown source for sparse history items", () => { + const extraction = extractObjectiveAnchorHistory([ + { paths: ["src/a.ts"] }, + { id: " ", source: undefined, paths: ["src/b.ts"] }, + ]); + + assert.deepEqual( + extraction.items.map((item) => [item.id, item.source]), + [ + ["item:1", "unknown"], + ["item:2", "unknown"], + ], + ); +}); + +test("extractObjectiveAnchorHistory handles an empty history without throwing", () => { + const extraction = extractObjectiveAnchorHistory([]); + + assert.deepEqual(extraction.features, { paths: [], modules: [], changeKinds: ["unknown"] }); + assert.deepEqual(extraction.items, []); +}); + +test("scoreObjectiveAnchorHistory scores aggregate replayed and revealed records while retaining item audits", () => { + const result = scoreObjectiveAnchorHistory({ + replayed: [ + { + id: "plan:objective-anchor", + source: "plan", + paths: ["packages/gittensory-engine/src/objective-anchor.ts"], + labels: ["feature"], + }, + { + id: "plan:tests", + source: "plan", + paths: ["packages/gittensory-engine/test/objective-anchor.test.ts"], + labels: ["test"], + }, + ], + revealed: [ + { + id: "pr:3142", + source: "pull_request", + paths: ["packages/gittensory-engine/src/objective-anchor.ts"], + labels: ["feature"], + }, + { + id: "commit:test", + source: "commit", + paths: ["packages/gittensory-engine/test/objective-anchor.test.ts"], + labels: ["test"], + }, + ], + }); + + assert.equal(result.score, 1); + assert.deepEqual( + result.history.replayed.items.map((item) => item.id), + ["plan:objective-anchor", "plan:tests"], + ); + assert.deepEqual( + result.history.revealed.items.map((item) => item.source), + ["pull_request", "commit"], + ); +}); + +test("scoreObjectiveAnchorHistory matches scoreObjectiveAnchor on aggregated features", () => { + const historyResult = scoreObjectiveAnchorHistory({ + replayed: [ + { paths: ["src/mcp/server.ts"], labels: ["feature"] }, + { paths: ["test/unit/mcp.test.ts"], labels: ["test"] }, + ], + revealed: [ + { paths: ["src/mcp/server.ts"], labels: ["feature"] }, + { paths: ["src/mcp/schema.ts"], labels: ["feature"] }, + ], + weights: { paths: 2, modules: 2, changeKinds: 1 }, + }); + const directResult = scoreObjectiveAnchor({ + replayed: historyResult.history.replayed.features, + revealed: historyResult.history.revealed.features, + weights: { paths: 2, modules: 2, changeKinds: 1 }, + }); + + assert.deepEqual(historyResult.score, directResult.score); + assert.deepEqual(historyResult.dimensions, directResult.dimensions); + assert.deepEqual(historyResult.audit, directResult.audit); +}); + +test("renderObjectiveAnchorAuditMarkdown renders a deterministic single-score audit", () => { + const result = scoreObjectiveAnchor({ + replayed: replay({ paths: ["src/review/a.ts"], labels: ["feature"] }), + revealed: revealed({ paths: ["src/review/b.ts"], labels: ["feature"] }), + }); + const markdown = renderObjectiveAnchorAuditMarkdown(result); + + assert.ok(markdown.startsWith("# Objective-Anchor Score\n\nScore: 0.550000\n")); + assert.match(markdown, /## Dimensions\n\n- paths: 0\.000000\n- modules: 1\.000000\n- changeKinds: 1\.000000/u); + assert.match(markdown, /## Intersections[\s\S]*Modules:\n- src\/review/u); + assert.match(markdown, /Replayed-only paths:\n- src\/review\/a\.ts/u); + assert.match(markdown, /Revealed-only paths:\n- src\/review\/b\.ts/u); +}); + +test("renderObjectiveAnchorAuditMarkdown includes per-item history evidence for history scores", () => { + const result = scoreObjectiveAnchorHistory({ + replayed: [{ id: "plan:one", source: "plan", paths: ["src/review/a.ts"], labels: ["feature"] }], + revealed: [{ id: "pr:two", source: "pull_request", paths: ["src/review/a.ts"], labels: ["feature"] }], + }); + const markdown = renderObjectiveAnchorAuditMarkdown(result); + + assert.match(markdown, /## Replayed History Items\n\n### plan:one \(plan\)/u); + assert.match(markdown, /## Revealed History Items\n\n### pr:two \(pull\\_request\)/u); + assert.match(markdown, /Paths:\n- src\/review\/a\.ts/u); +}); + +test("renderObjectiveAnchorAuditMarkdown reports none for empty miss lists", () => { + const result = scoreObjectiveAnchor({ + replayed: replay({ paths: ["src/review/a.ts"], labels: ["feature"] }), + revealed: revealed({ paths: ["src/review/a.ts"], labels: ["feature"] }), + }); + const markdown = renderObjectiveAnchorAuditMarkdown(result); + + assert.match(markdown, /Replayed-only paths:\n- none/u); + assert.match(markdown, /Revealed-only modules:\n- none/u); +}); + +test("renderObjectiveAnchorAuditMarkdown escapes markdown controls and collapses newlines from audit values", () => { + const result = scoreObjectiveAnchorHistory({ + replayed: [ + { + id: "plan:*bold*\nnext", + source: "manual", + paths: ["src/review/[unsafe].ts"], + labels: ["feature"], + }, + ], + revealed: [ + { + id: "pr:`code`", + source: "pull_request", + paths: ["src/review/.ts"], + labels: ["feature"], + }, + ], + }); + const markdown = renderObjectiveAnchorAuditMarkdown(result); + + assert.ok(markdown.includes("### plan:\\*bold\\* next (manual)")); + assert.ok(markdown.includes("### pr:\\`code\\` (pull\\_request)")); + assert.ok(markdown.includes("- src/review/\\[unsafe\\].ts")); + assert.ok(markdown.includes("- src/review/\\.ts")); +}); diff --git a/packages/gittensory-engine/test/pairwise-calibration.test.ts b/packages/gittensory-engine/test/pairwise-calibration.test.ts new file mode 100644 index 0000000000..f1c3d09612 --- /dev/null +++ b/packages/gittensory-engine/test/pairwise-calibration.test.ts @@ -0,0 +1,155 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { + computePairwiseCalibrationScore, + resolvePairwiseCalibrationSample, + scoreObjectiveAnchor, +} from "../dist/index.js"; + +test("barrel: exports the pairwise calibration APIs (#3013)", () => { + assert.equal(typeof resolvePairwiseCalibrationSample, "function"); + assert.equal(typeof computePairwiseCalibrationScore, "function"); +}); + +test("resolvePairwiseCalibrationSample accepts a stable replay-better order swap", () => { + const result = resolvePairwiseCalibrationSample({ + attempts: [{ replayFirst: "replay_better", revealedFirst: "revealed_better" }], + }); + + assert.deepEqual(result, { + stable: true, + exhausted: false, + attemptsUsed: 1, + maxAttempts: 1, + verdict: "replay_better", + pairwiseScore: 1, + }); +}); + +test("resolvePairwiseCalibrationSample accepts stable tie and revealed-better outcomes", () => { + assert.equal( + resolvePairwiseCalibrationSample({ + attempts: [{ replayFirst: "tie", revealedFirst: "tie" }], + }).pairwiseScore, + 0.5, + ); + assert.equal( + resolvePairwiseCalibrationSample({ + attempts: [{ replayFirst: "revealed_better", revealedFirst: "replay_better" }], + }).pairwiseScore, + 0, + ); +}); + +test("resolvePairwiseCalibrationSample discards order-flipping judgments", () => { + const result = resolvePairwiseCalibrationSample({ + attempts: [{ replayFirst: "replay_better", revealedFirst: "replay_better" }], + }); + + assert.equal(result.stable, false); + assert.equal(result.exhausted, true); + assert.equal(result.verdict, "unstable"); + assert.equal(result.pairwiseScore, null); +}); + +test("resolvePairwiseCalibrationSample retries until the first stable non-incomparable verdict", () => { + const result = resolvePairwiseCalibrationSample({ + attempts: [ + { replayFirst: "replay_better", revealedFirst: "replay_better" }, + { replayFirst: "incomparable", revealedFirst: "incomparable" }, + { replayFirst: "tie", revealedFirst: "tie" }, + ], + maxAttempts: 3, + }); + + assert.equal(result.stable, true); + assert.equal(result.exhausted, false); + assert.equal(result.attemptsUsed, 3); + assert.equal(result.verdict, "tie"); + assert.equal(result.pairwiseScore, 0.5); +}); + +test("resolvePairwiseCalibrationSample respects the retry cap boundary", () => { + const result = resolvePairwiseCalibrationSample({ + attempts: [ + { replayFirst: "replay_better", revealedFirst: "replay_better" }, + { replayFirst: "tie", revealedFirst: "tie" }, + ], + maxAttempts: 1, + }); + + assert.equal(result.stable, false); + assert.equal(result.exhausted, true); + assert.equal(result.attemptsUsed, 1); + assert.equal(result.pairwiseScore, null); +}); + +test("computePairwiseCalibrationScore combines objective-anchor and stable pairwise scores", () => { + const objectiveAnchor = scoreObjectiveAnchor({ + replayed: { paths: ["src/review/a.ts"], labels: ["feature"] }, + revealed: { paths: ["src/review/b.ts"], labels: ["feature"] }, + }); + const result = computePairwiseCalibrationScore({ + objectiveAnchor, + samples: [ + { attempts: [{ replayFirst: "replay_better", revealedFirst: "revealed_better" }] }, + { attempts: [{ replayFirst: "tie", revealedFirst: "tie" }] }, + ], + weights: { objectiveAnchor: 1, pairwiseJudge: 3 }, + }); + + assert.equal(objectiveAnchor.score, 0.55); + assert.equal(result.pairwiseJudgeScore, 0.75); + assert.deepEqual(result.weights, { objectiveAnchor: 0.25, pairwiseJudge: 0.75 }); + assert.equal(result.compositeScore, 0.7); + assert.deepEqual(result.metrics, { + totalSamples: 2, + stableSamples: 2, + unstableSamples: 0, + exhaustedSamples: 0, + orderInstabilityRate: 0, + }); +}); + +test("computePairwiseCalibrationScore tracks order-instability rate and excludes unstable samples", () => { + const result = computePairwiseCalibrationScore({ + objectiveAnchor: 0.25, + samples: [ + { attempts: [{ replayFirst: "replay_better", revealedFirst: "revealed_better" }] }, + { attempts: [{ replayFirst: "replay_better", revealedFirst: "replay_better" }] }, + { attempts: [{ replayFirst: "revealed_better", revealedFirst: "replay_better" }] }, + ], + }); + + assert.equal(result.pairwiseJudgeScore, 0.5); + assert.equal(result.compositeScore, 0.375); + assert.deepEqual(result.metrics, { + totalSamples: 3, + stableSamples: 2, + unstableSamples: 1, + exhaustedSamples: 1, + orderInstabilityRate: 0.333333, + }); +}); + +test("computePairwiseCalibrationScore falls back to objective-anchor when every pairwise sample is unstable", () => { + const result = computePairwiseCalibrationScore({ + objectiveAnchor: 0.42, + samples: [{ attempts: [{ replayFirst: "incomparable", revealedFirst: "incomparable" }] }], + }); + + assert.equal(result.pairwiseJudgeScore, null); + assert.equal(result.compositeScore, 0.42); +}); + +test("computePairwiseCalibrationScore normalizes invalid weights without producing NaN", () => { + const result = computePairwiseCalibrationScore({ + objectiveAnchor: 1, + samples: [{ attempts: [{ replayFirst: "revealed_better", revealedFirst: "replay_better" }] }], + weights: { objectiveAnchor: Number.NaN, pairwiseJudge: -1 }, + }); + + assert.deepEqual(result.weights, { objectiveAnchor: 0.5, pairwiseJudge: 0.5 }); + assert.equal(result.compositeScore, 0.5); +});