From 0fa6ac52ac25c125b577a45b96a3626a738aa3fb Mon Sep 17 00:00:00 2001 From: RealDiligent Date: Mon, 20 Jul 2026 19:53:06 +0800 Subject: [PATCH] fix(engine): preserve explicit zero pairwise calibration weights Closes #7443 Co-authored-by: Cursor --- .../src/pairwise-calibration.ts | 40 ++++++++++-- .../test/pairwise-calibration.test.ts | 14 +++++ .../engine-calibration-convergence.test.ts | 61 +++++++++++++++++++ 3 files changed, 109 insertions(+), 6 deletions(-) diff --git a/packages/loopover-engine/src/pairwise-calibration.ts b/packages/loopover-engine/src/pairwise-calibration.ts index e85f4b91cf..bde2e40c4c 100644 --- a/packages/loopover-engine/src/pairwise-calibration.ts +++ b/packages/loopover-engine/src/pairwise-calibration.ts @@ -55,6 +55,10 @@ function finiteNonNegative(value: number | undefined, fallback: number): number return value; } +function isInvalidWeight(value: number | undefined): boolean { + return value !== undefined && (!Number.isFinite(value) || value < 0); +} + function normalizePairwiseWeights(weights: PairwiseCalibrationWeights | undefined): { objectiveAnchor: number; pairwiseJudge: number; @@ -64,7 +68,17 @@ function normalizePairwiseWeights(weights: PairwiseCalibrationWeights | undefine pairwiseJudge: finiteNonNegative(weights?.pairwiseJudge, DEFAULT_PAIRWISE_WEIGHTS.pairwiseJudge), }; const total = raw.objectiveAnchor + raw.pairwiseJudge; - if (total <= 0) return DEFAULT_PAIRWISE_WEIGHTS; + // Preserve explicitly-zeroed weights rather than substituting the defaults: a caller that zeroes every + // component must reach the objective-only fallback in computePairwiseCalibrationScore, not silently get + // the default 50/50 blend (converges with reviewer-consensus-calibration.ts / #6170; #7443). + // NaN/negative inputs still recover to DEFAULT_PAIRWISE_WEIGHTS when the clamped total is empty — same as + // pre-#7443 — so the invalid-weight suite keeps asserting the 50/50 default. + if (total <= 0) { + if (isInvalidWeight(weights?.objectiveAnchor) || isInvalidWeight(weights?.pairwiseJudge)) { + return DEFAULT_PAIRWISE_WEIGHTS; + } + return { objectiveAnchor: 0, pairwiseJudge: 0 }; + } return { objectiveAnchor: raw.objectiveAnchor / total, pairwiseJudge: raw.pairwiseJudge / total, @@ -135,11 +149,25 @@ export function computePairwiseCalibrationScore(input: { .filter((score): score is number => score !== null); const pairwiseJudgeScore = stableScores.length === 0 ? null : roundScore(stableScores.reduce((sum, score) => sum + score, 0) / stableScores.length); - const weights = normalizePairwiseWeights(input.weights); - const compositeScore = - pairwiseJudgeScore === null - ? objectiveAnchorScore - : roundScore(objectiveAnchorScore * weights.objectiveAnchor + pairwiseJudgeScore * weights.pairwiseJudge); + const rawWeights = normalizePairwiseWeights(input.weights); + // Second-stage usable-weights pass mirrors reviewer-consensus-calibration.ts (#6170 / #7443): zero out any + // component whose own signal is unavailable, then fall back to objective-only when that usable total is empty + // (covers explicit all-zero weights even when pairwiseJudgeScore is present). + const usableWeights = { + objectiveAnchor: rawWeights.objectiveAnchor, + pairwiseJudge: pairwiseJudgeScore === null ? 0 : rawWeights.pairwiseJudge, + }; + const usableTotal = usableWeights.objectiveAnchor + usableWeights.pairwiseJudge; + const weights = + usableTotal <= 0 + ? { objectiveAnchor: 1, pairwiseJudge: 0 } + : { + objectiveAnchor: usableWeights.objectiveAnchor / usableTotal, + pairwiseJudge: usableWeights.pairwiseJudge / usableTotal, + }; + const compositeScore = roundScore( + objectiveAnchorScore * weights.objectiveAnchor + (pairwiseJudgeScore ?? 0) * weights.pairwiseJudge, + ); const unstableSamples = samples.filter((sample) => !sample.stable).length; const exhaustedSamples = samples.filter((sample) => sample.exhausted).length; return { diff --git a/packages/loopover-engine/test/pairwise-calibration.test.ts b/packages/loopover-engine/test/pairwise-calibration.test.ts index f1c3d09612..a75394f176 100644 --- a/packages/loopover-engine/test/pairwise-calibration.test.ts +++ b/packages/loopover-engine/test/pairwise-calibration.test.ts @@ -153,3 +153,17 @@ test("computePairwiseCalibrationScore normalizes invalid weights without produci assert.deepEqual(result.weights, { objectiveAnchor: 0.5, pairwiseJudge: 0.5 }); assert.equal(result.compositeScore, 0.5); }); + +test("computePairwiseCalibrationScore falls back to objective-only when all weights are explicitly zero (#7443)", () => { + const result = computePairwiseCalibrationScore({ + objectiveAnchor: 0.42, + samples: [{ attempts: [{ replayFirst: "replay_better", revealedFirst: "revealed_better" }] }], + weights: { objectiveAnchor: 0, pairwiseJudge: 0 }, + }); + + // Explicit zeros must not silently restore the 50/50 default, and must not collapse compositeScore to 0 + // when a real pairwiseJudgeScore is present — fall back to objective-anchor only (#6170 sibling pattern). + assert.equal(result.pairwiseJudgeScore, 1); + assert.deepEqual(result.weights, { objectiveAnchor: 1, pairwiseJudge: 0 }); + assert.equal(result.compositeScore, 0.42); +}); diff --git a/test/unit/engine-calibration-convergence.test.ts b/test/unit/engine-calibration-convergence.test.ts index 3f4518bf9c..e2dd190520 100644 --- a/test/unit/engine-calibration-convergence.test.ts +++ b/test/unit/engine-calibration-convergence.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { computeGateVerdictCompositeCalibrationScore, computeFindingSeverityCompositeCalibrationScore, + computePairwiseCalibrationScore, } from "../../packages/loopover-engine/src/index"; // Converges gate-verdict + finding-severity calibration with reviewer-consensus-calibration.ts's already-correct @@ -75,3 +76,63 @@ describe("gate-verdict/finding-severity calibration convergence (#6170)", () => ]); }); }); + +// Extends the #6170 all-zero-weight pattern to pairwise-calibration.ts (#7443). Vitest coverage is what +// Codecov grades; the engine package's node:test suite mirrors the same assertions. +describe("pairwise calibration zero-weight convergence (#7443)", () => { + it("explicit all-zero weights fall back to objective-only even when pairwiseJudgeScore is present", () => { + const result = computePairwiseCalibrationScore({ + objectiveAnchor: 0.42, + samples: [{ attempts: [{ replayFirst: "replay_better", revealedFirst: "revealed_better" }] }], + weights: { objectiveAnchor: 0, pairwiseJudge: 0 }, + }); + expect(result.pairwiseJudgeScore).toBe(1); + expect(result.weights).toEqual({ objectiveAnchor: 1, pairwiseJudge: 0 }); + expect(result.compositeScore).toBe(0.42); + }); + + it("NaN/negative weights still recover to the 50/50 default (not the objective-only fallback)", () => { + const result = computePairwiseCalibrationScore({ + objectiveAnchor: 1, + samples: [{ attempts: [{ replayFirst: "revealed_better", revealedFirst: "replay_better" }] }], + weights: { objectiveAnchor: Number.NaN, pairwiseJudge: -1 }, + }); + expect(result.weights).toEqual({ objectiveAnchor: 0.5, pairwiseJudge: 0.5 }); + expect(result.compositeScore).toBe(0.5); + }); + + it("non-zero weights take the normalized usable path (covers usableTotal > 0)", () => { + const result = computePairwiseCalibrationScore({ + objectiveAnchor: 0.55, + samples: [ + { attempts: [{ replayFirst: "replay_better", revealedFirst: "revealed_better" }] }, + { attempts: [{ replayFirst: "tie", revealedFirst: "tie" }] }, + ], + weights: { objectiveAnchor: 1, pairwiseJudge: 3 }, + }); + expect(result.weights).toEqual({ objectiveAnchor: 0.25, pairwiseJudge: 0.75 }); + expect(result.compositeScore).toBe(0.7); + }); + + it("missing pairwise signal zeros that component then falls back to objective-only when usable total is empty", () => { + const result = computePairwiseCalibrationScore({ + objectiveAnchor: 0.42, + samples: [{ attempts: [{ replayFirst: "incomparable", revealedFirst: "incomparable" }] }], + weights: { objectiveAnchor: 0, pairwiseJudge: 0 }, + }); + expect(result.pairwiseJudgeScore).toBeNull(); + expect(result.weights).toEqual({ objectiveAnchor: 1, pairwiseJudge: 0 }); + expect(result.compositeScore).toBe(0.42); + }); + + it("missing pairwise signal with non-zero weights renormalizes to objective-only (covers usableTotal > 0 + null pairwise)", () => { + const result = computePairwiseCalibrationScore({ + objectiveAnchor: 0.42, + samples: [{ attempts: [{ replayFirst: "incomparable", revealedFirst: "incomparable" }] }], + weights: { objectiveAnchor: 1, pairwiseJudge: 1 }, + }); + expect(result.pairwiseJudgeScore).toBeNull(); + expect(result.weights).toEqual({ objectiveAnchor: 1, pairwiseJudge: 0 }); + expect(result.compositeScore).toBe(0.42); + }); +});