diff --git a/apps/gittensory-ui/src/components/site/calibration-card-model.ts b/apps/gittensory-ui/src/components/site/calibration-card-model.ts new file mode 100644 index 0000000000..4e4c3bea14 --- /dev/null +++ b/apps/gittensory-ui/src/components/site/calibration-card-model.ts @@ -0,0 +1,80 @@ +import type { Status } from "@/components/site/control-primitives"; + +// UI-side mirror of FleetOutcomeCalibration (src/services/outcome-calibration.ts), delivered on the +// operator-dashboard payload's `calibration` field (#2192, part of #1967). "Bins" here are slop-severity +// bands (clean/low/elevated/high) — the deterministic PREDICTED-risk signal gittensory actually computes +// — each carrying its REALIZED merge rate (the "kept rate"). This intentionally does not mirror #2192's +// own reference (src/review/ops.ts's `Calibration`/`computeCalibration`): that module is ported-but-unwired +// "reviewbot" code whose `review_targets`/`review_audit` tables are never populated in gittensory (see +// src/review/ops-wire.ts's header comment) — nothing constructs its config anywhere in this codebase, and +// its handlers are registered on no route. This mirrors the live native equivalent instead. +export type SlopBand = "clean" | "low" | "elevated" | "high"; + +export type SlopBandCalibration = { + band: SlopBand; + sampleSize: number; + merged: number; + closed: number; + mergeRate: number; +}; + +export type SlopOutcomeCalibration = { + totalResolved: number; + bands: SlopBandCalibration[]; + overallMergeRate: number | null; + discriminates: boolean | null; +}; + +export type RecommendationOutcomeCalibration = { + total: number; + positive: number; + negative: number; + pending: number; + positiveRate: number | null; +}; + +export type FleetOutcomeCalibration = { + generatedAt: string; + slop: SlopOutcomeCalibration; + recommendations: RecommendationOutcomeCalibration; + signals: string[]; +}; + +export const SLOP_BAND_LABEL: Record = { + clean: "Clean", + low: "Low", + elevated: "Elevated", + high: "High", +}; + +export type CalibrationBandRow = { + band: SlopBand; + label: string; + sampleSize: number; + /** 0-100, or null when there's no sample to show a bar for (distinct from a real 0% merge rate). */ + mergeRatePercent: number | null; +}; + +/** Pure: shape each slop band into a display-ready row. */ +export function calibrationBandRows(slop: SlopOutcomeCalibration): CalibrationBandRow[] { + return slop.bands.map((entry) => ({ + band: entry.band, + label: SLOP_BAND_LABEL[entry.band], + sampleSize: entry.sampleSize, + mergeRatePercent: entry.sampleSize > 0 ? Math.round(entry.mergeRate * 100) : null, + })); +} + +/** Pure: pill tone for the discrimination verdict. */ +export function calibrationVerdictTone(discriminates: boolean | null): Status { + if (discriminates === true) return "ready"; + if (discriminates === false) return "blocked"; + return "warn"; +} + +/** Pure: human label for the discrimination verdict. */ +export function calibrationVerdictLabel(discriminates: boolean | null): string { + if (discriminates === true) return "Predictive"; + if (discriminates === false) return "Not discriminating"; + return "Insufficient data"; +} diff --git a/apps/gittensory-ui/src/components/site/calibration-card.test.tsx b/apps/gittensory-ui/src/components/site/calibration-card.test.tsx new file mode 100644 index 0000000000..e61b5b5e06 --- /dev/null +++ b/apps/gittensory-ui/src/components/site/calibration-card.test.tsx @@ -0,0 +1,164 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; + +import { CalibrationCard } from "@/components/site/calibration-card"; +import { + calibrationBandRows, + calibrationVerdictLabel, + calibrationVerdictTone, + type FleetOutcomeCalibration, + type SlopBandCalibration, +} from "@/components/site/calibration-card-model"; + +function band(overrides: Partial = {}): SlopBandCalibration { + return { band: "clean", sampleSize: 0, merged: 0, closed: 0, mergeRate: 0, ...overrides }; +} + +function calibration(overrides: Partial = {}): FleetOutcomeCalibration { + return { + generatedAt: "2026-07-10T00:00:00.000Z", + slop: { + totalResolved: 0, + bands: [ + band({ band: "clean" }), + band({ band: "low" }), + band({ band: "elevated" }), + band({ band: "high" }), + ], + overallMergeRate: null, + discriminates: null, + }, + recommendations: { total: 0, positive: 0, negative: 0, pending: 0, positiveRate: null }, + signals: [], + ...overrides, + }; +} + +describe("calibrationBandRows", () => { + it("marks a zero-sample band's merge rate as null (no bar), not a real 0%", () => { + const rows = calibrationBandRows({ + totalResolved: 0, + bands: [band({ band: "clean", sampleSize: 0 })], + overallMergeRate: null, + discriminates: null, + }); + expect(rows).toEqual([ + { band: "clean", label: "Clean", sampleSize: 0, mergeRatePercent: null }, + ]); + }); + + it("converts a populated band's merge rate to a rounded 0-100 percent", () => { + const rows = calibrationBandRows({ + totalResolved: 6, + bands: [band({ band: "high", sampleSize: 6, merged: 1, closed: 5, mergeRate: 0.167 })], + overallMergeRate: 0.167, + discriminates: null, + }); + expect(rows).toEqual([{ band: "high", label: "High", sampleSize: 6, mergeRatePercent: 17 }]); + }); +}); + +describe("calibrationVerdictTone / calibrationVerdictLabel", () => { + it("reports a ready/predictive verdict when the score discriminates", () => { + expect(calibrationVerdictTone(true)).toBe("ready"); + expect(calibrationVerdictLabel(true)).toBe("Predictive"); + }); + it("reports a blocked/not-discriminating verdict when the score inverts", () => { + expect(calibrationVerdictTone(false)).toBe("blocked"); + expect(calibrationVerdictLabel(false)).toBe("Not discriminating"); + }); + it("reports a warn/insufficient-data verdict when there isn't enough signal to judge", () => { + expect(calibrationVerdictTone(null)).toBe("warn"); + expect(calibrationVerdictLabel(null)).toBe("Insufficient data"); + }); +}); + +describe("CalibrationCard", () => { + it("renders nothing when there is no resolved-PR or recommendation signal at all (empty bins)", () => { + const { container } = render(); + expect(container.firstChild).toBeNull(); + }); + + it("renders a single populated band alongside the rest as 'no data'", () => { + render( + , + ); + expect(screen.getByText("Confidence calibration")).toBeTruthy(); + expect(screen.getByText("83% · n=6")).toBeTruthy(); + expect(screen.getAllByText("no data")).toHaveLength(3); + expect(screen.getByText("Insufficient data")).toBeTruthy(); + expect(screen.getByText(/Not enough resolved PRs per band/)).toBeTruthy(); + }); + + it("renders the full curve across every band plus the recommendation-outcome split", () => { + render( + , + ); + expect(screen.getByText("Predictive")).toBeTruthy(); + expect(screen.getByText("24 resolved")).toBeTruthy(); + expect(screen.getByText("83% · n=6")).toBeTruthy(); + expect(screen.getByText("50% · n=6")).toBeTruthy(); + expect(screen.getByText("33% · n=6")).toBeTruthy(); + expect(screen.getByText("17% · n=6")).toBeTruthy(); + expect(screen.queryByText("no data")).toBeNull(); + expect(screen.getByText("75%")).toBeTruthy(); + expect(screen.getByText(/3\/4 positive/)).toBeTruthy(); + expect(screen.getByText(/1 pending/)).toBeTruthy(); + }); + + it("shows a '—' recommendation rate when nothing is resolved yet, even with slop signal present", () => { + render( + , + ); + expect(screen.getByText("—")).toBeTruthy(); + expect(screen.getByText(/0\/0 positive/)).toBeTruthy(); + }); +}); diff --git a/apps/gittensory-ui/src/components/site/calibration-card.tsx b/apps/gittensory-ui/src/components/site/calibration-card.tsx new file mode 100644 index 0000000000..393895750f --- /dev/null +++ b/apps/gittensory-ui/src/components/site/calibration-card.tsx @@ -0,0 +1,91 @@ +import { BoundaryBadge, Stat, StatusPill } from "@/components/site/control-primitives"; +import { + calibrationBandRows, + calibrationVerdictLabel, + calibrationVerdictTone, + type FleetOutcomeCalibration, +} from "@/components/site/calibration-card-model"; + +/** + * Confidence-calibration card (#2192, part of #1967): predicted slop-severity band vs. realized merge + * rate per bucket, plus the recommendation-outcome split, over the existing FleetOutcomeCalibration + * payload. Renders nothing when there's no resolved-PR or recommendation signal yet (keeps the analytics + * page clean until calibration data exists), matching GatePrecisionCard's convention on this same page. + */ +export function CalibrationCard({ calibration }: { calibration: FleetOutcomeCalibration }) { + if (calibration.slop.totalResolved === 0 && calibration.recommendations.total === 0) return null; + const rows = calibrationBandRows(calibration.slop); + const resolvedRecommendations = + calibration.recommendations.positive + calibration.recommendations.negative; + + return ( +
+
+
+

Confidence calibration

+

+ Predicted slop severity vs. realized merge rate per band — public-safe counts only. +

+
+ +
+ +
+ + {calibration.slop.totalResolved} resolved + + } + /> + + {calibration.recommendations.positive}/{resolvedRecommendations} positive ·{" "} + {calibration.recommendations.pending} pending + + } + /> +
+ +
+ {rows.map((row) => ( +
+ + {row.label} + +
+ {row.mergeRatePercent !== null ? ( +
+ ) : null} +
+ + {row.mergeRatePercent !== null + ? `${row.mergeRatePercent}% · n=${row.sampleSize}` + : "no data"} + +
+ ))} +
+ + {calibration.signals.length > 0 ? ( +
    + {calibration.signals.map((signal) => ( +
  • {signal}
    • + ))} +
+ ) : null} +
+ ); +} diff --git a/apps/gittensory-ui/src/routes/app.analytics.tsx b/apps/gittensory-ui/src/routes/app.analytics.tsx index 68650d5e1f..a35651129f 100644 --- a/apps/gittensory-ui/src/routes/app.analytics.tsx +++ b/apps/gittensory-ui/src/routes/app.analytics.tsx @@ -12,6 +12,8 @@ import { } from "@/components/site/usage-analytics-panels"; import { GatePrecisionCard } from "@/components/site/app-panels/gate-precision-card"; import type { GateEvalReport } from "@/components/site/app-panels/gate-precision-card-model"; +import { CalibrationCard } from "@/components/site/calibration-card"; +import type { FleetOutcomeCalibration } from "@/components/site/calibration-card-model"; import { CycleTimeCard } from "@/components/site/app-panels/cycle-time-card"; import type { CycleTimeAggregate } from "@/components/site/app-panels/cycle-time-card-model"; import { useApiResource } from "@/lib/api/use-api-resource"; @@ -104,6 +106,7 @@ type OperatorDashboard = { }; upstreamDrift?: { status?: string; openReportCount?: number } | null; gateEval?: GateEvalReport; + calibration?: FleetOutcomeCalibration; cycleTime?: CycleTimeAggregate; }; @@ -188,6 +191,8 @@ function ProductAnalytics() { {data.gateEval ? : null} + {data.calibration ? : null} + {data.cycleTime ? : null} {data.usageSummary ? ( diff --git a/src/services/operator-dashboard.ts b/src/services/operator-dashboard.ts index dcaa0a2630..b448ce8c6f 100644 --- a/src/services/operator-dashboard.ts +++ b/src/services/operator-dashboard.ts @@ -30,6 +30,7 @@ import { computeGateEval, type GateEvalReport } from "../review/parity"; import { computeCycleTimeAggregate, type CycleTimeAggregate } from "../review/stats"; import { loadUpstreamStatus, type UpstreamStatus } from "../upstream/ruleset"; import { nowIso } from "../utils/json"; +import { buildFleetOutcomeCalibration, type FleetOutcomeCalibration } from "./outcome-calibration"; import { buildRecommendationQualityReport, type RecommendationQualityReport } from "./recommendation-quality-report"; import { buildWeeklyValueReport } from "./weekly-value-report"; @@ -64,6 +65,10 @@ export type OperatorDashboardPayload = { // Gate-precision eval (#2191): the per-project confusion matrix + precisions from computeGateEval, surfaced // read-only for the maintainer analytics card. Fail-safe empty report when there is no review_audit signal. gateEval: GateEvalReport; + // Fleet-wide confidence calibration (#2192, part of #1967): slop-band merge-rate curve + recommendation + // outcome split from buildFleetOutcomeCalibration, surfaced read-only for the analytics card. Fails safe + // to an all-zero/null-discriminates report when there is no resolved-PR signal yet. + calibration: FleetOutcomeCalibration; // PR review cycle-time percentiles (#2194): gate decision → outcome from review_audit; fail-safe empty aggregate. cycleTime: CycleTimeAggregate; }; @@ -90,6 +95,7 @@ export async function buildOperatorDashboardPayload(env: Env): Promise { + const [pullRequests, outcomes] = await Promise.all([ + listAllPullRequests(env), + listAgentRecommendationOutcomes(env, { limit: 5000 }), + ]); + const slop = buildSlopOutcomeCalibration(pullRequests); + const recommendations = buildRecommendationOutcomeCalibration(outcomes); + return { generatedAt: nowIso(), slop, recommendations, signals: buildOutcomeCalibrationSignals(slop, recommendations) }; +} diff --git a/test/unit/operator-dashboard.test.ts b/test/unit/operator-dashboard.test.ts index 35bd38c070..82abca9fb3 100644 --- a/test/unit/operator-dashboard.test.ts +++ b/test/unit/operator-dashboard.test.ts @@ -26,6 +26,10 @@ describe("operator dashboard payload", () => { // #2191: gate-eval report is surfaced read-only; with no review_audit signal it fails safe to an empty // report (no rows, no signal) rather than being absent. expect(payload.gateEval).toEqual({ rows: [], hasSignal: false }); + // #2192: fleet-wide calibration fails safe to an all-zero/null-discriminates report, never absent. + expect(payload.calibration.slop).toMatchObject({ totalResolved: 0, overallMergeRate: null, discriminates: null }); + expect(payload.calibration.recommendations).toMatchObject({ total: 0, positiveRate: null }); + expect(payload.calibration.signals.length).toBeGreaterThan(0); expect(payload.cycleTime).toEqual({ p50Ms: null, p90Ms: null, diff --git a/test/unit/outcome-calibration.test.ts b/test/unit/outcome-calibration.test.ts index 512f86643f..6193b971e0 100644 --- a/test/unit/outcome-calibration.test.ts +++ b/test/unit/outcome-calibration.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { + buildFleetOutcomeCalibration, buildOutcomeCalibrationSignals, buildRecommendationOutcomeCalibration, buildRepoOutcomeCalibration, @@ -191,6 +192,48 @@ describe("buildRepoOutcomeCalibration (env loader)", () => { }); }); +describe("buildFleetOutcomeCalibration (env loader)", () => { + it("folds resolved PRs + recommendation outcomes across EVERY repo, not just one", async () => { + const env = createTestEnv(); + await upsertPullRequestFromGitHub(env, "owner/repo-a", { number: 1, title: "merged clean", state: "closed", user: { login: "alice" }, merged_at: "2026-06-01T00:00:00.000Z" }); + await updatePullRequestSlopAssessment(env, "owner/repo-a", 1, { slopRisk: 0, slopBand: "clean" }); + await upsertPullRequestFromGitHub(env, "owner/repo-b", { number: 1, title: "closed high", state: "closed", user: { login: "bob" } }); + await updatePullRequestSlopAssessment(env, "owner/repo-b", 1, { slopRisk: 70, slopBand: "high" }); + await createAgentRun(env, runRecord("r-fleet", "miner", "2026-06-01T00:00:00.000Z")); + await replaceAgentActions(env, "r-fleet", [actionRecord("fleet-merged", "r-fleet")]); + await upsertAgentRecommendationOutcome(env, { + actionId: "fleet-merged", + runId: "r-fleet", + actorLogin: "miner", + actionType: "choose_next_work", + targetRepoFullName: "owner/repo-a", + source: "explicit", + outcomeState: "merged", + outcomeTargetType: "pull_request", + maintainerLane: false, + confidence: "high", + reason: "fleet-wide signal", + metadata: {}, + }); + + const report = await buildFleetOutcomeCalibration(env); + expect(report.slop.totalResolved).toBe(2); // both repos counted, not just one + expect(report.slop.bands.find((b) => b.band === "clean")).toMatchObject({ merged: 1, closed: 0 }); + expect(report.slop.bands.find((b) => b.band === "high")).toMatchObject({ merged: 0, closed: 1 }); + expect(report.recommendations).toMatchObject({ total: 1, positive: 1, positiveRate: 1 }); + expect(report.signals.length).toBeGreaterThan(0); + expect(JSON.stringify(report)).not.toMatch(/reward|payout|trust score|wallet|hotkey/i); + }); + + it("fails safe to an all-empty report when there is no resolved-PR signal yet", async () => { + const env = createTestEnv(); + const report = await buildFleetOutcomeCalibration(env); + expect(report.slop).toMatchObject({ totalResolved: 0, overallMergeRate: null, discriminates: null }); + expect(report.recommendations).toMatchObject({ total: 0, positiveRate: null }); + expect(report.signals.length).toBeGreaterThan(0); // still narrates the "not enough data" state + }); +}); + function runRecord(id: string, actorLogin: string, createdAt: string): AgentRunRecord { return { id,