From f6e4b9143aeca47f54bf7d2d7ac64a72579d060d Mon Sep 17 00:00:00 2001 From: davion-knight <298846663+davion-knight@users.noreply.github.com> Date: Fri, 10 Jul 2026 03:10:56 -0500 Subject: [PATCH] feat(miner-selfimprove): calibration accuracy-trend view over a snapshot series Extend the single-snapshot calibration dashboard (#4261/#4504) into an accuracy *trend* over accumulated snapshots (#4268): direction (improving/degrading/flat), a normalized sparkline, per-point delta vs baseline, and latest/change-over-window. Read-only and pure. A brand-new install has no snapshots and warming-up snapshots carry a null accuracy, so empty/single-point history renders an explicit insufficient-history state instead of a broken chart. Closes #4268 --- .../src/calibration-trend.ts | 119 ++++++++++++++ packages/gittensory-engine/src/index.ts | 8 + test/unit/calibration-trend.test.ts | 149 ++++++++++++++++++ 3 files changed, 276 insertions(+) create mode 100644 packages/gittensory-engine/src/calibration-trend.ts create mode 100644 test/unit/calibration-trend.test.ts diff --git a/packages/gittensory-engine/src/calibration-trend.ts b/packages/gittensory-engine/src/calibration-trend.ts new file mode 100644 index 0000000000..ede9f6d632 --- /dev/null +++ b/packages/gittensory-engine/src/calibration-trend.ts @@ -0,0 +1,119 @@ +import { DOCUMENTED_CALIBRATION_BASELINE, type Phase7CalibrationLoopResult } from "./phase7-calibration-loop.js"; + +// Calibration accuracy-trend view (#4268). A read-only projection of a SERIES of accumulated calibration +// snapshots (each a point-in-time computePhase7CalibrationLoop result) into a trend over a rolling window — +// the multi-snapshot counterpart to the single-snapshot calibration-dashboard.ts (#4261/#4504). Pure: it +// re-shapes an already-accumulated series and adds NO new calibration computation. Public-safe: only +// accuracies, the documented baseline, and observation timestamps are surfaced (no scores/rewards). A +// brand-new install has zero snapshots, so empty/single-point history renders an explicit +// "insufficient history" state rather than a broken/empty chart. + +/** One accumulated point in the calibration history: when it was computed and its combined accuracy. */ +export type CalibrationTrendSnapshot = { + observedAt: string; + combinedAccuracy: number | null; + baselineAccuracy: number; +}; + +export type CalibrationTrendDirection = "improving" | "degrading" | "flat" | "insufficient"; + +/** One rendered point on the trend line: a data-bearing snapshot with its delta vs the baseline. */ +export type CalibrationTrendPoint = { + observedAt: string; + combinedAccuracy: number; + /** Whole percentage-point delta vs that snapshot's baseline (e.g. +6 / -4). */ + deltaFromBaseline: number; + aboveBaseline: boolean; +}; + +/** The read-only trend projection of a calibration-snapshot series. */ +export type CalibrationTrendView = { + direction: CalibrationTrendDirection; + headline: string; + /** Unicode sparkline of the data points' combined accuracy, normalized across the window. */ + sparkline: string; + points: readonly CalibrationTrendPoint[]; + latestAccuracy: number | null; + /** Latest minus earliest combined accuracy, in whole percentage points; null with < 2 data points. */ + changeOverWindow: number | null; + /** Snapshots that carry a combined accuracy (an install still warming up contributes none). */ + sampleCount: number; + baselineAccuracy: number; +}; + +const SPARK_TICKS = "▁▂▃▄▅▆▇█"; + +/** Derive a trend snapshot from a computed calibration-loop result observed at a given time. */ +export function calibrationSnapshotFromResult( + result: Phase7CalibrationLoopResult, + observedAt: string, +): CalibrationTrendSnapshot { + return { observedAt, combinedAccuracy: result.combinedAccuracy, baselineAccuracy: result.baselineAccuracy }; +} + +function percentPoints(value: number): number { + return Math.round(value * 100); +} + +function formatPercent(value: number): string { + return `${percentPoints(value)}%`; +} + +function formatDeltaPoints(points: number): string { + return `${points >= 0 ? "+" : ""}${points}pts`; +} + +/** Map data-point accuracies onto sparkline ticks, normalized across the window's own min..max. */ +function sparkline(values: readonly number[]): string { + if (values.length === 0) return ""; + const min = Math.min(...values); + const span = Math.max(...values) - min; + return values + .map((v) => SPARK_TICKS[span === 0 ? 0 : Math.round(((v - min) / span) * (SPARK_TICKS.length - 1))]) + .join(""); +} + +/** + * Project an accumulated series of calibration snapshots into a read-only trend view. Pure and + * deterministic. Snapshots without a combined accuracy (an install still warming up) are dropped from the + * trend line; if fewer than two data points remain, the view reports an explicit "insufficient" state + * instead of a misleading flat line. `baselineAccuracy` is taken from the most recent snapshot (or the + * documented default when there is no history yet). + */ +export function buildCalibrationTrendView(snapshots: readonly CalibrationTrendSnapshot[]): CalibrationTrendView { + const baselineAccuracy = + snapshots.length === 0 ? DOCUMENTED_CALIBRATION_BASELINE : snapshots[snapshots.length - 1]!.baselineAccuracy; + + const points: CalibrationTrendPoint[] = snapshots + .filter((s): s is CalibrationTrendSnapshot & { combinedAccuracy: number } => s.combinedAccuracy !== null) + .map((s) => ({ + observedAt: s.observedAt, + combinedAccuracy: s.combinedAccuracy, + deltaFromBaseline: percentPoints(s.combinedAccuracy - s.baselineAccuracy), + aboveBaseline: s.combinedAccuracy >= s.baselineAccuracy, + })); + + const sampleCount = points.length; + const spark = sparkline(points.map((p) => p.combinedAccuracy)); + const latestAccuracy = sampleCount === 0 ? null : points[sampleCount - 1]!.combinedAccuracy; + + if (sampleCount < 2) { + const headline = + sampleCount === 0 + ? "No calibration history yet" + : `Insufficient history: 1 snapshot (${formatPercent(points[0]!.combinedAccuracy)})`; + return { direction: "insufficient", headline, sparkline: spark, points, latestAccuracy, changeOverWindow: null, sampleCount, baselineAccuracy }; + } + + const earliest = points[0]!.combinedAccuracy; + const latest = points[sampleCount - 1]!.combinedAccuracy; + const changeOverWindow = percentPoints(latest - earliest); + const direction: CalibrationTrendDirection = + changeOverWindow > 0 ? "improving" : changeOverWindow < 0 ? "degrading" : "flat"; + const headline = + direction === "flat" + ? `Flat at ${formatPercent(latest)} over ${sampleCount} snapshots` + : `${direction === "improving" ? "Improving" : "Degrading"}: ${formatPercent(earliest)} → ${formatPercent(latest)} over ${sampleCount} snapshots (${formatDeltaPoints(changeOverWindow)})`; + + return { direction, headline, sparkline: spark, points, latestAccuracy, changeOverWindow, sampleCount, baselineAccuracy }; +} diff --git a/packages/gittensory-engine/src/index.ts b/packages/gittensory-engine/src/index.ts index d11729453c..f06c9a7867 100644 --- a/packages/gittensory-engine/src/index.ts +++ b/packages/gittensory-engine/src/index.ts @@ -84,6 +84,14 @@ export { type CalibrationDashboardStatus, type CalibrationDashboardView, } from "./calibration-dashboard.js"; +export { + buildCalibrationTrendView, + calibrationSnapshotFromResult, + type CalibrationTrendDirection, + type CalibrationTrendPoint, + type CalibrationTrendSnapshot, + type CalibrationTrendView, +} from "./calibration-trend.js"; export { computeFindingSeverityCompositeCalibrationScore, ingestFindingSeverityCalibrationSignals, diff --git a/test/unit/calibration-trend.test.ts b/test/unit/calibration-trend.test.ts new file mode 100644 index 0000000000..dca8a61d84 --- /dev/null +++ b/test/unit/calibration-trend.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, it } from "vitest"; +import { + buildCalibrationTrendView, + calibrationSnapshotFromResult, +} from "../../packages/gittensory-engine/src/index"; +import type { + CalibrationTrendSnapshot, + Phase7CalibrationLoopResult, +} from "../../packages/gittensory-engine/src/index"; + +function snap(observedAt: string, combinedAccuracy: number | null, baselineAccuracy = 0.62): CalibrationTrendSnapshot { + return { observedAt, combinedAccuracy, baselineAccuracy }; +} + +function makeResult(over: Partial = {}): Phase7CalibrationLoopResult { + const metric = { source: "pr_outcome" as const, accuracy: 0.66, sampleSize: 12, observedAt: null, fresh: true }; + return { + enabled: true, + baselineAccuracy: 0.62, + combinedAccuracy: 0.68, + deltaFromBaseline: 0.06, + weights: { historicalReplay: 0.5, prOutcome: 0.5 }, + bySource: { historical_replay: { ...metric, source: "historical_replay" }, pr_outcome: metric }, + replayHarnessHold: false, + replayHarnessStatus: "healthy", + autonomyIncreasePermitted: true, + holdReasons: [], + replayRunDue: false, + audit: { contributingSources: ["pr_outcome"], rejectedSources: [] }, + ...over, + }; +} + +describe("buildCalibrationTrendView (#4268)", () => { + it("reports an explicit empty state for a brand-new install with no history", () => { + const view = buildCalibrationTrendView([]); + expect(view.direction).toBe("insufficient"); + expect(view.headline).toBe("No calibration history yet"); + expect(view.sparkline).toBe(""); + expect(view.points).toEqual([]); + expect(view.latestAccuracy).toBeNull(); + expect(view.changeOverWindow).toBeNull(); + expect(view.sampleCount).toBe(0); + expect(view.baselineAccuracy).toBe(0.62); + }); + + it("reports insufficient history with a single data point (can't trend one)", () => { + const view = buildCalibrationTrendView([snap("2026-01-01T00:00:00Z", 0.66)]); + expect(view.direction).toBe("insufficient"); + expect(view.headline).toBe("Insufficient history: 1 snapshot (66%)"); + expect(view.sparkline).toBe("▁"); + expect(view.latestAccuracy).toBe(0.66); + expect(view.changeOverWindow).toBeNull(); + expect(view.sampleCount).toBe(1); + expect(view.points[0]).toEqual({ + observedAt: "2026-01-01T00:00:00Z", + combinedAccuracy: 0.66, + deltaFromBaseline: 4, + aboveBaseline: true, + }); + }); + + it("projects an improving multi-point series with a rising sparkline", () => { + const view = buildCalibrationTrendView([ + snap("2026-01-01T00:00:00Z", 0.58), + snap("2026-01-02T00:00:00Z", 0.63), + snap("2026-01-03T00:00:00Z", 0.71), + ]); + expect(view.direction).toBe("improving"); + expect(view.headline).toBe("Improving: 58% → 71% over 3 snapshots (+13pts)"); + expect(view.sparkline).toBe("▁▄█"); + expect(view.latestAccuracy).toBe(0.71); + expect(view.changeOverWindow).toBe(13); + expect(view.sampleCount).toBe(3); + // crosses the baseline: first point below, last point above + expect(view.points[0]?.aboveBaseline).toBe(false); + expect(view.points[0]?.deltaFromBaseline).toBe(-4); + expect(view.points[2]?.aboveBaseline).toBe(true); + expect(view.points[2]?.deltaFromBaseline).toBe(9); + }); + + it("projects a degrading multi-point series with a falling sparkline", () => { + const view = buildCalibrationTrendView([ + snap("2026-01-01T00:00:00Z", 0.7), + snap("2026-01-02T00:00:00Z", 0.6), + snap("2026-01-03T00:00:00Z", 0.52), + ]); + expect(view.direction).toBe("degrading"); + expect(view.headline).toBe("Degrading: 70% → 52% over 3 snapshots (-18pts)"); + expect(view.sparkline).toBe("█▄▁"); + expect(view.changeOverWindow).toBe(-18); + }); + + it("reports a flat series (zero net change) without dividing by a zero span", () => { + const view = buildCalibrationTrendView([ + snap("2026-01-01T00:00:00Z", 0.64), + snap("2026-01-02T00:00:00Z", 0.64), + snap("2026-01-03T00:00:00Z", 0.64), + ]); + expect(view.direction).toBe("flat"); + expect(view.headline).toBe("Flat at 64% over 3 snapshots"); + expect(view.sparkline).toBe("▁▁▁"); + expect(view.changeOverWindow).toBe(0); + }); + + it("drops warming-up snapshots (null combined accuracy) from the trend line", () => { + const view = buildCalibrationTrendView([ + snap("2026-01-01T00:00:00Z", null), + snap("2026-01-02T00:00:00Z", 0.66), + snap("2026-01-03T00:00:00Z", 0.7), + ]); + expect(view.sampleCount).toBe(2); + expect(view.direction).toBe("improving"); + expect(view.headline).toBe("Improving: 66% → 70% over 2 snapshots (+4pts)"); + expect(view.sparkline).toBe("▁█"); + }); + + it("takes the baseline from the most recent snapshot, not the documented default", () => { + const view = buildCalibrationTrendView([snap("2026-01-01T00:00:00Z", 0.6, 0.55), snap("2026-01-02T00:00:00Z", 0.62, 0.55)]); + expect(view.baselineAccuracy).toBe(0.55); + expect(view.points[1]?.aboveBaseline).toBe(true); + }); + + it("treats accuracy exactly at the baseline as above-baseline (>= boundary) with a zero delta", () => { + const view = buildCalibrationTrendView([snap("2026-01-01T00:00:00Z", 0.62, 0.62), snap("2026-01-02T00:00:00Z", 0.62, 0.62)]); + expect(view.direction).toBe("flat"); + expect(view.points[0]?.aboveBaseline).toBe(true); + expect(view.points[0]?.deltaFromBaseline).toBe(0); + }); +}); + +describe("calibrationSnapshotFromResult", () => { + it("bridges a computed loop result into a trend snapshot", () => { + const snapshot = calibrationSnapshotFromResult(makeResult({ combinedAccuracy: 0.71 }), "2026-01-03T00:00:00Z"); + expect(snapshot).toEqual({ observedAt: "2026-01-03T00:00:00Z", combinedAccuracy: 0.71, baselineAccuracy: 0.62 }); + }); + + it("carries a null accuracy through so the trend can filter it as warming-up", () => { + const series = [ + calibrationSnapshotFromResult(makeResult({ combinedAccuracy: null, deltaFromBaseline: null }), "2026-01-01T00:00:00Z"), + calibrationSnapshotFromResult(makeResult({ combinedAccuracy: 0.65 }), "2026-01-02T00:00:00Z"), + calibrationSnapshotFromResult(makeResult({ combinedAccuracy: 0.73 }), "2026-01-03T00:00:00Z"), + ]; + const view = buildCalibrationTrendView(series); + expect(view.sampleCount).toBe(2); + expect(view.direction).toBe("improving"); + expect(view.latestAccuracy).toBe(0.73); + }); +});