Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 34 additions & 6 deletions packages/loopover-engine/src/pairwise-calibration.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,10 @@ function finiteNonNegative(value: number | undefined, fallback: number): number
return value;
}

function isInvalidWeight(value: number | undefined): boolean {
return value !== undefined && (!Number.isFinite(value) || value < 0);
}

function normalizePairwiseWeights(weights: PairwiseCalibrationWeights | undefined): {
objectiveAnchor: number;
pairwiseJudge: number;
Expand All @@ -64,7 +68,17 @@ function normalizePairwiseWeights(weights: PairwiseCalibrationWeights | undefine
pairwiseJudge: finiteNonNegative(weights?.pairwiseJudge, DEFAULT_PAIRWISE_WEIGHTS.pairwiseJudge),
};
const total = raw.objectiveAnchor + raw.pairwiseJudge;
if (total <= 0) return DEFAULT_PAIRWISE_WEIGHTS;
// Preserve explicitly-zeroed weights rather than substituting the defaults: a caller that zeroes every
// component must reach the objective-only fallback in computePairwiseCalibrationScore, not silently get
// the default 50/50 blend (converges with reviewer-consensus-calibration.ts / #6170; #7443).
// NaN/negative inputs still recover to DEFAULT_PAIRWISE_WEIGHTS when the clamped total is empty — same as
// pre-#7443 — so the invalid-weight suite keeps asserting the 50/50 default.
if (total <= 0) {
if (isInvalidWeight(weights?.objectiveAnchor) || isInvalidWeight(weights?.pairwiseJudge)) {
return DEFAULT_PAIRWISE_WEIGHTS;
}
return { objectiveAnchor: 0, pairwiseJudge: 0 };
}
return {
objectiveAnchor: raw.objectiveAnchor / total,
pairwiseJudge: raw.pairwiseJudge / total,
Expand Down Expand Up @@ -135,11 +149,25 @@ export function computePairwiseCalibrationScore(input: {
.filter((score): score is number => score !== null);
const pairwiseJudgeScore =
stableScores.length === 0 ? null : roundScore(stableScores.reduce((sum, score) => sum + score, 0) / stableScores.length);
const weights = normalizePairwiseWeights(input.weights);
const compositeScore =
pairwiseJudgeScore === null
? objectiveAnchorScore
: roundScore(objectiveAnchorScore * weights.objectiveAnchor + pairwiseJudgeScore * weights.pairwiseJudge);
const rawWeights = normalizePairwiseWeights(input.weights);
// Second-stage usable-weights pass mirrors reviewer-consensus-calibration.ts (#6170 / #7443): zero out any
// component whose own signal is unavailable, then fall back to objective-only when that usable total is empty
// (covers explicit all-zero weights even when pairwiseJudgeScore is present).
const usableWeights = {
objectiveAnchor: rawWeights.objectiveAnchor,
pairwiseJudge: pairwiseJudgeScore === null ? 0 : rawWeights.pairwiseJudge,
};
const usableTotal = usableWeights.objectiveAnchor + usableWeights.pairwiseJudge;
const weights =
usableTotal <= 0
? { objectiveAnchor: 1, pairwiseJudge: 0 }
: {
objectiveAnchor: usableWeights.objectiveAnchor / usableTotal,
pairwiseJudge: usableWeights.pairwiseJudge / usableTotal,
};
const compositeScore = roundScore(
objectiveAnchorScore * weights.objectiveAnchor + (pairwiseJudgeScore ?? 0) * weights.pairwiseJudge,
);
const unstableSamples = samples.filter((sample) => !sample.stable).length;
const exhaustedSamples = samples.filter((sample) => sample.exhausted).length;
return {
Expand Down
14 changes: 14 additions & 0 deletions packages/loopover-engine/test/pairwise-calibration.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -153,3 +153,17 @@ test("computePairwiseCalibrationScore normalizes invalid weights without produci
assert.deepEqual(result.weights, { objectiveAnchor: 0.5, pairwiseJudge: 0.5 });
assert.equal(result.compositeScore, 0.5);
});

test("computePairwiseCalibrationScore falls back to objective-only when all weights are explicitly zero (#7443)", () => {
const result = computePairwiseCalibrationScore({
objectiveAnchor: 0.42,
samples: [{ attempts: [{ replayFirst: "replay_better", revealedFirst: "revealed_better" }] }],
weights: { objectiveAnchor: 0, pairwiseJudge: 0 },
});

// Explicit zeros must not silently restore the 50/50 default, and must not collapse compositeScore to 0
// when a real pairwiseJudgeScore is present — fall back to objective-anchor only (#6170 sibling pattern).
assert.equal(result.pairwiseJudgeScore, 1);
assert.deepEqual(result.weights, { objectiveAnchor: 1, pairwiseJudge: 0 });
assert.equal(result.compositeScore, 0.42);
});
61 changes: 61 additions & 0 deletions test/unit/engine-calibration-convergence.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
import {
computeGateVerdictCompositeCalibrationScore,
computeFindingSeverityCompositeCalibrationScore,
computePairwiseCalibrationScore,
} from "../../packages/loopover-engine/src/index";

// Converges gate-verdict + finding-severity calibration with reviewer-consensus-calibration.ts's already-correct
Expand Down Expand Up @@ -75,3 +76,63 @@ describe("gate-verdict/finding-severity calibration convergence (#6170)", () =>
]);
});
});

// Extends the #6170 all-zero-weight pattern to pairwise-calibration.ts (#7443). Vitest coverage is what
// Codecov grades; the engine package's node:test suite mirrors the same assertions.
describe("pairwise calibration zero-weight convergence (#7443)", () => {
it("explicit all-zero weights fall back to objective-only even when pairwiseJudgeScore is present", () => {
const result = computePairwiseCalibrationScore({
objectiveAnchor: 0.42,
samples: [{ attempts: [{ replayFirst: "replay_better", revealedFirst: "revealed_better" }] }],
weights: { objectiveAnchor: 0, pairwiseJudge: 0 },
});
expect(result.pairwiseJudgeScore).toBe(1);
expect(result.weights).toEqual({ objectiveAnchor: 1, pairwiseJudge: 0 });
expect(result.compositeScore).toBe(0.42);
});

it("NaN/negative weights still recover to the 50/50 default (not the objective-only fallback)", () => {
const result = computePairwiseCalibrationScore({
objectiveAnchor: 1,
samples: [{ attempts: [{ replayFirst: "revealed_better", revealedFirst: "replay_better" }] }],
weights: { objectiveAnchor: Number.NaN, pairwiseJudge: -1 },
});
expect(result.weights).toEqual({ objectiveAnchor: 0.5, pairwiseJudge: 0.5 });
expect(result.compositeScore).toBe(0.5);
});

it("non-zero weights take the normalized usable path (covers usableTotal > 0)", () => {
const result = computePairwiseCalibrationScore({
objectiveAnchor: 0.55,
samples: [
{ attempts: [{ replayFirst: "replay_better", revealedFirst: "revealed_better" }] },
{ attempts: [{ replayFirst: "tie", revealedFirst: "tie" }] },
],
weights: { objectiveAnchor: 1, pairwiseJudge: 3 },
});
expect(result.weights).toEqual({ objectiveAnchor: 0.25, pairwiseJudge: 0.75 });
expect(result.compositeScore).toBe(0.7);
});

it("missing pairwise signal zeros that component then falls back to objective-only when usable total is empty", () => {
const result = computePairwiseCalibrationScore({
objectiveAnchor: 0.42,
samples: [{ attempts: [{ replayFirst: "incomparable", revealedFirst: "incomparable" }] }],
weights: { objectiveAnchor: 0, pairwiseJudge: 0 },
});
expect(result.pairwiseJudgeScore).toBeNull();
expect(result.weights).toEqual({ objectiveAnchor: 1, pairwiseJudge: 0 });
expect(result.compositeScore).toBe(0.42);
});

it("missing pairwise signal with non-zero weights renormalizes to objective-only (covers usableTotal > 0 + null pairwise)", () => {
const result = computePairwiseCalibrationScore({
objectiveAnchor: 0.42,
samples: [{ attempts: [{ replayFirst: "incomparable", revealedFirst: "incomparable" }] }],
weights: { objectiveAnchor: 1, pairwiseJudge: 1 },
});
expect(result.pairwiseJudgeScore).toBeNull();
expect(result.weights).toEqual({ objectiveAnchor: 1, pairwiseJudge: 0 });
expect(result.compositeScore).toBe(0.42);
});
});