From bcbd3b5e5216181b2d4b5f001e8c9e928c4cb2e3 Mon Sep 17 00:00:00 2001 From: peter-minion Date: Thu, 9 Jul 2026 02:53:50 +0800 Subject: [PATCH] feat(ui): add gate-precision analytics card (#2191) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #2191. Adds a self-host maintainer analytics card that renders gate merge-precision and the TP/FP/FN/TN confusion matrix already computed by computeGateEval, read-only on the operator analytics page. - Surface the eval on the existing operator-dashboard fetch: add a `gateEval` field to OperatorDashboardPayload populated by the existing computeGateEval (no new compute; fails safe to an empty report when there is no review_audit signal). The response schema is an untyped record, so OpenAPI is unaffected. - GatePrecisionCard reuses Stat + StatusPill from control-primitives, aggregates the per-project GateEvalReport rows into one confusion matrix via a pure helper, shows merge precision as a percentage, and flags below-floor sample size via the StatusPill. Rendered on app.analytics.tsx alongside the weekly-value section. - Tests: pure aggregation (populated + empty/null-precision arms) and the card (populated, null-precision, below-floor, and empty→renders nothing) states; the backend payload test asserts the fail-safe empty report. Public-safe counts only — no actor, PR content, or scoring internals. --- .../app-panels/gate-precision-card-model.ts | 58 ++++++++++ .../app-panels/gate-precision-card.test.tsx | 104 ++++++++++++++++++ .../site/app-panels/gate-precision-card.tsx | 93 ++++++++++++++++ .../src/routes/app.analytics.tsx | 5 + src/services/operator-dashboard.ts | 8 ++ test/unit/operator-dashboard.test.ts | 3 + 6 files changed, 271 insertions(+) create mode 100644 apps/gittensory-ui/src/components/site/app-panels/gate-precision-card-model.ts create mode 100644 apps/gittensory-ui/src/components/site/app-panels/gate-precision-card.test.tsx create mode 100644 apps/gittensory-ui/src/components/site/app-panels/gate-precision-card.tsx diff --git a/apps/gittensory-ui/src/components/site/app-panels/gate-precision-card-model.ts b/apps/gittensory-ui/src/components/site/app-panels/gate-precision-card-model.ts new file mode 100644 index 0000000000..fd10bd198c --- /dev/null +++ b/apps/gittensory-ui/src/components/site/app-panels/gate-precision-card-model.ts @@ -0,0 +1,58 @@ +// Gate-precision analytics card model (#2191). UI-side mirror of the GateEvalReport/GateEvalRow shape produced +// by computeGateEval (src/review/parity.ts) and surfaced on the operator-dashboard payload — plus the pure fold +// that turns the per-project rows into a single 2x2 confusion matrix + merge precision for the card. Types + +// pure helper live here (not in the .tsx) so the component file exports only components +// (react-refresh/only-export-components). + +/** One project's gate confusion matrix + precisions (mirror of src/review/parity.ts GateEvalRow). */ +export interface GateEvalRow { + project: string; + wouldMerge: number; + mergeConfirmed: number; // predicted merge AND human merged (true positive) + mergeFalse: number; // predicted merge BUT human closed (false positive) + wouldClose: number; + closeConfirmed: number; // predicted close AND human closed (true negative) + closeFalse: number; // predicted close BUT human merged (false negative) + hold: number; + decided: number; + mergePrecision: number | null; + closePrecision: number | null; +} + +/** The gate-eval report as delivered on the operator-dashboard payload (mirror of parity.ts GateEvalReport). */ +export interface GateEvalReport { + rows: GateEvalRow[]; + /** True once at least one project has enough decided samples to read meaningfully (parity.ts's floor). */ + hasSignal: boolean; +} + +/** Aggregated 2x2 confusion matrix across every project row, with overall merge precision. */ +export interface GateConfusionMatrix { + truePositive: number; // merge predicted → merged + falsePositive: number; // merge predicted → closed + falseNegative: number; // close predicted → merged + trueNegative: number; // close predicted → closed + decided: number; + /** TP / (TP + FP); null when the gate made no merge predictions (empty denominator). */ + mergePrecision: number | null; +} + +/** Fold the per-project rows into one confusion matrix. Pure; an empty report yields all-zero counts and a + * null precision (no merge predictions ⇒ nothing to be precise about). */ +export function aggregateGateEval(report: GateEvalReport): GateConfusionMatrix { + const totals = report.rows.reduce( + (acc, row) => ({ + truePositive: acc.truePositive + row.mergeConfirmed, + falsePositive: acc.falsePositive + row.mergeFalse, + falseNegative: acc.falseNegative + row.closeFalse, + trueNegative: acc.trueNegative + row.closeConfirmed, + decided: acc.decided + row.decided, + }), + { truePositive: 0, falsePositive: 0, falseNegative: 0, trueNegative: 0, decided: 0 }, + ); + const mergePredictions = totals.truePositive + totals.falsePositive; + return { + ...totals, + mergePrecision: mergePredictions > 0 ? totals.truePositive / mergePredictions : null, + }; +} diff --git a/apps/gittensory-ui/src/components/site/app-panels/gate-precision-card.test.tsx b/apps/gittensory-ui/src/components/site/app-panels/gate-precision-card.test.tsx new file mode 100644 index 0000000000..274c59d0e1 --- /dev/null +++ b/apps/gittensory-ui/src/components/site/app-panels/gate-precision-card.test.tsx @@ -0,0 +1,104 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; + +import { GatePrecisionCard } from "@/components/site/app-panels/gate-precision-card"; +import { + aggregateGateEval, + type GateEvalReport, +} from "@/components/site/app-panels/gate-precision-card-model"; + +function row(overrides: Partial = {}) { + return { + project: "acme/widgets", + wouldMerge: 0, + mergeConfirmed: 0, + mergeFalse: 0, + wouldClose: 0, + closeConfirmed: 0, + closeFalse: 0, + hold: 0, + decided: 0, + mergePrecision: null, + closePrecision: null, + ...overrides, + }; +} + +describe("aggregateGateEval", () => { + it("folds multiple project rows into one confusion matrix + merge precision", () => { + const report: GateEvalReport = { + hasSignal: true, + rows: [ + row({ mergeConfirmed: 6, mergeFalse: 2, closeConfirmed: 3, closeFalse: 1, decided: 12 }), + row({ + project: "acme/other", + mergeConfirmed: 2, + mergeFalse: 0, + closeConfirmed: 1, + closeFalse: 0, + decided: 3, + }), + ], + }; + expect(aggregateGateEval(report)).toEqual({ + truePositive: 8, + falsePositive: 2, + falseNegative: 1, + trueNegative: 4, + decided: 15, + mergePrecision: 8 / 10, + }); + }); + + it("returns all-zero counts and null precision for an empty report (no merge predictions ⇒ empty denominator)", () => { + expect(aggregateGateEval({ hasSignal: false, rows: [] })).toEqual({ + truePositive: 0, + falsePositive: 0, + falseNegative: 0, + trueNegative: 0, + decided: 0, + mergePrecision: null, + }); + }); +}); + +describe("GatePrecisionCard", () => { + it("renders precision, decided count, and the 2x2 confusion matrix for a populated report", () => { + const report: GateEvalReport = { + hasSignal: true, + rows: [ + row({ mergeConfirmed: 6, mergeFalse: 2, closeConfirmed: 3, closeFalse: 1, decided: 12 }), + ], + }; + render(); + expect(screen.getByText("Gate precision")).toBeTruthy(); + // 6/(6+2) = 75% + expect(screen.getByText("75%")).toBeTruthy(); + expect(screen.getByText("12 decided")).toBeTruthy(); + expect(screen.getByText("True positive")).toBeTruthy(); + expect(screen.getByText("False negative")).toBeTruthy(); + }); + + it("renders '—' for precision when the gate made no merge predictions (null-precision arm)", () => { + const report: GateEvalReport = { + hasSignal: true, + rows: [row({ closeConfirmed: 4, decided: 4 })], // only close predictions → merge denominator is 0 + }; + render(); + expect(screen.getByText("—")).toBeTruthy(); + }); + + it("flags the below-floor state via the StatusPill when hasSignal is false", () => { + const report: GateEvalReport = { + hasSignal: false, + rows: [row({ mergeConfirmed: 1, decided: 1 })], + }; + render(); + expect(screen.getByText("below 10-sample floor")).toBeTruthy(); + }); + + it("renders nothing when there are no evaluated project rows at all", () => { + const { container } = render(); + expect(container.firstChild).toBeNull(); + }); +}); diff --git a/apps/gittensory-ui/src/components/site/app-panels/gate-precision-card.tsx b/apps/gittensory-ui/src/components/site/app-panels/gate-precision-card.tsx new file mode 100644 index 0000000000..f05e3d4886 --- /dev/null +++ b/apps/gittensory-ui/src/components/site/app-panels/gate-precision-card.tsx @@ -0,0 +1,93 @@ +import { cn } from "@/lib/utils"; +import { Stat, StatusPill } from "@/components/site/control-primitives"; +import { aggregateGateEval, type GateEvalReport } from "./gate-precision-card-model"; + +// Documented sample-size floor: below this many decided predictions the confusion matrix is too noisy to read, +// so the card flags it via the StatusPill. Mirrors parity.ts's MIN_DECIDED_FOR_SIGNAL (which sets `hasSignal`). +const MIN_DECIDED_FLOOR = 10; + +/** Self-host maintainer analytics card (#2191): gate merge-precision + the TP/FP/FN/TN confusion matrix from + * computeGateEval, read-only over the operator-dashboard payload. Renders nothing when there are no evaluated + * projects at all (keeps the analytics page clean until the gate has produced eval rows). */ +export function GatePrecisionCard({ report }: { report: GateEvalReport }) { + if (report.rows.length === 0) return null; + const matrix = aggregateGateEval(report); + return ( +
+
+
+

Gate precision

+

+ The gate's merge/close predictions scored against realized PR outcomes. Public-safe + counts only. +

+
+ + {report.hasSignal + ? `${matrix.decided} decided` + : `below ${MIN_DECIDED_FLOOR}-sample floor`} + +
+
+ P(merged | gate predicted merge)} + /> + predictions with a known outcome} + /> +
+
+ + + + +
+
+ ); +} + +function ConfusionCell({ + label, + detail, + value, + tone, +}: { + label: string; + detail: string; + value: number; + tone: string; +}) { + return ( +
+
{value}
+
{label}
+
{detail}
+
+ ); +} diff --git a/apps/gittensory-ui/src/routes/app.analytics.tsx b/apps/gittensory-ui/src/routes/app.analytics.tsx index b708c9a6c4..d8d49371c3 100644 --- a/apps/gittensory-ui/src/routes/app.analytics.tsx +++ b/apps/gittensory-ui/src/routes/app.analytics.tsx @@ -9,6 +9,8 @@ import { ProductUsageBreakdownPanel, WeeklyValueMetricsPanel, } from "@/components/site/usage-analytics-panels"; +import { GatePrecisionCard } from "@/components/site/app-panels/gate-precision-card"; +import type { GateEvalReport } from "@/components/site/app-panels/gate-precision-card-model"; import { useApiResource } from "@/lib/api/use-api-resource"; export const Route = createFileRoute("/app/analytics")({ @@ -98,6 +100,7 @@ type OperatorDashboard = { }>; }; upstreamDrift?: { status?: string; openReportCount?: number } | null; + gateEval?: GateEvalReport; }; function ProductAnalytics() { @@ -178,6 +181,8 @@ function ProductAnalytics() { /> ) : null} + {data.gateEval ? : null} + {data.usageSummary ? ( { expect(payload.usageSummary).toMatchObject({ totalEvents: expect.any(Number), activeActors: expect.any(Number) }); expect(payload.commandUsefulness.totals).toMatchObject({ feedbackCount: expect.any(Number) }); expect(serialized).not.toMatch(FORBIDDEN_EXPORT_TERMS); + // #2191: gate-eval report is surfaced read-only; with no review_audit signal it fails safe to an empty + // report (no rows, no signal) rather than being absent. + expect(payload.gateEval).toEqual({ rows: [], hasSignal: false }); // Empty fleet → instanceCount 0, null precision card ("—"), no-outlier delta. expect(payload.fleetMetrics.instanceCount).toBe(0); expect(payload.metrics).toEqual(