diff --git a/apps/gittensory-ui/src/components/site/app-panels/reversal-health-card-model.ts b/apps/gittensory-ui/src/components/site/app-panels/reversal-health-card-model.ts new file mode 100644 index 0000000000..31f2c25d2c --- /dev/null +++ b/apps/gittensory-ui/src/components/site/app-panels/reversal-health-card-model.ts @@ -0,0 +1,45 @@ +// Reversal-health analytics card model (#2193). UI-side mirror of the AgentHealth fields from +// src/review/ops.ts surfaced on the operator-dashboard payload — plus status helpers for the card. + +/** A bot auto-action a human overrode (revert of a bot-merge / reopen of a bot-close). */ +export type ReversedTarget = { + number: number; + repo: string; + status: string; + eventType: string; +}; + +/** AgentHealth subset used by ReversalHealthCard (ops.ts:42-55). */ +export type ReversalHealth = { + reversals: number; + reversalRate: number; + manualRate: number; + recentAutoActions: number; + reversedTargets?: ReversedTarget[]; +}; + +/** alerts.ts:204 — any human reversal of a bot auto-action is the calibration-regression signal. */ +export const REVERSAL_ALERT_MIN = 1; + +export function formatRatePct(rate: number): string { + return `${Math.round(rate * 100)}%`; +} + +export function formatReversalEventType(eventType: string): string { + if (eventType === "reversal_reverted") return "merge reverted"; + if (eventType === "reversal_reopened") return "close reopened"; + return eventType.replaceAll("_", " "); +} + +export function reversalHealthStatus(health: ReversalHealth): { + tone: "ready" | "warn" | "info"; + label: string; +} { + if (health.recentAutoActions === 0) { + return { tone: "info", label: "no auto-actions in window" }; + } + if (health.reversals >= REVERSAL_ALERT_MIN) { + return { tone: "warn", label: `${health.reversals} reversal(s)` }; + } + return { tone: "ready", label: "0 reversals" }; +} diff --git a/apps/gittensory-ui/src/components/site/app-panels/reversal-health-card.test.tsx b/apps/gittensory-ui/src/components/site/app-panels/reversal-health-card.test.tsx new file mode 100644 index 0000000000..4a5043957f --- /dev/null +++ b/apps/gittensory-ui/src/components/site/app-panels/reversal-health-card.test.tsx @@ -0,0 +1,89 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; + +import { ReversalHealthCard } from "@/components/site/app-panels/reversal-health-card"; +import { + formatRatePct, + reversalHealthStatus, + type ReversalHealth, +} from "@/components/site/app-panels/reversal-health-card-model"; + +function health(overrides: Partial = {}): ReversalHealth { + return { + reversals: 0, + reversalRate: 0, + manualRate: 0.1, + recentAutoActions: 20, + reversedTargets: [], + ...overrides, + }; +} + +describe("reversalHealthStatus", () => { + it("returns ready when there are auto-actions but zero reversals", () => { + expect(reversalHealthStatus(health())).toEqual({ tone: "ready", label: "0 reversals" }); + }); + + it("returns warn when reversals meet the documented alert minimum (above-threshold arm)", () => { + expect(reversalHealthStatus(health({ reversals: 2, reversalRate: 0.1 }))).toEqual({ + tone: "warn", + label: "2 reversal(s)", + }); + }); + + it("returns info when recentAutoActions is 0 (empty-denominator arm keeps reversalRate at 0)", () => { + expect(reversalHealthStatus(health({ recentAutoActions: 0, reversalRate: 0 }))).toEqual({ + tone: "info", + label: "no auto-actions in window", + }); + }); +}); + +describe("ReversalHealthCard", () => { + it("renders zero-reversals stats and the empty reversed-targets state", () => { + render(); + expect(screen.getByText("Reversal health")).toBeTruthy(); + expect(screen.getByText("0%")).toBeTruthy(); + expect(screen.getByText("0 reversals")).toBeTruthy(); + expect(screen.getByText("No reversals in window")).toBeTruthy(); + expect(formatRatePct(0.1)).toBe("10%"); + expect(screen.getByText("10%")).toBeTruthy(); + expect(screen.getByText("20")).toBeTruthy(); + }); + + it("renders above-threshold reversal stats and lists reversed targets", () => { + render( + , + ); + expect(screen.getByText("25%")).toBeTruthy(); + expect(screen.getByText("1 reversal(s)")).toBeTruthy(); + expect(screen.getByText("acme/widgets#42")).toBeTruthy(); + expect(screen.getByText(/close reopened · merged/)).toBeTruthy(); + expect(screen.queryByText("No reversals in window")).toBeNull(); + }); + + it("shows 0% reversal rate when recentAutoActions is 0 (empty-denominator branch)", () => { + render( + , + ); + expect(screen.getByText("no auto-actions in window")).toBeTruthy(); + expect(screen.getAllByText("0%").length).toBeGreaterThanOrEqual(1); + expect(screen.getByText("No reversals in window")).toBeTruthy(); + }); +}); diff --git a/apps/gittensory-ui/src/components/site/app-panels/reversal-health-card.tsx b/apps/gittensory-ui/src/components/site/app-panels/reversal-health-card.tsx new file mode 100644 index 0000000000..fd711f190e --- /dev/null +++ b/apps/gittensory-ui/src/components/site/app-panels/reversal-health-card.tsx @@ -0,0 +1,86 @@ +import { Stat, StatusPill } from "@/components/site/control-primitives"; +import { EmptyState } from "@/components/site/state-views"; +import { + formatRatePct, + formatReversalEventType, + reversalHealthStatus, + type ReversalHealth, +} from "@/components/site/app-panels/reversal-health-card-model"; + +/** Analytics card (#2193): reversal rate and recent auto-action health from computeAgentHealth — read-only + * over the operator-dashboard payload. Lists reversed targets when present; EmptyState when none. */ +export function ReversalHealthCard({ health }: { health: ReversalHealth }) { + const status = reversalHealthStatus(health); + const reversedTargets = health.reversedTargets ?? []; + + return ( +
+
+
+

Reversal health

+

+ How often humans reopened or reverted a bot auto-action in the last 7 days. Public-safe + counts only. +

+
+ {status.label} +
+ +
+ + reversals / recent auto-actions (7d window) + + } + /> + human overrides in window} + /> + merged + closed in window} + /> + lifetime terminal decisions punted} + /> +
+ + {reversedTargets.length > 0 ? ( + + ) : ( + + )} +
+ ); +} diff --git a/apps/gittensory-ui/src/routes/app.analytics.tsx b/apps/gittensory-ui/src/routes/app.analytics.tsx index 094868e8e6..42be38ef76 100644 --- a/apps/gittensory-ui/src/routes/app.analytics.tsx +++ b/apps/gittensory-ui/src/routes/app.analytics.tsx @@ -15,6 +15,8 @@ import { GatePrecisionCard } from "@/components/site/app-panels/gate-precision-c import type { GateEvalReport } from "@/components/site/app-panels/gate-precision-card-model"; import { CycleTimeCard } from "@/components/site/app-panels/cycle-time-card"; import type { CycleTimeAggregate } from "@/components/site/app-panels/cycle-time-card-model"; +import { ReversalHealthCard } from "@/components/site/app-panels/reversal-health-card"; +import type { ReversalHealth } from "@/components/site/app-panels/reversal-health-card-model"; import { AnalyticsCardShell } from "@/components/site/app-panels/analytics-card-shell"; import { AcceptanceRateCard, @@ -112,6 +114,7 @@ type OperatorDashboard = { upstreamDrift?: { status?: string; openReportCount?: number } | null; gateEval?: GateEvalReport; cycleTime?: CycleTimeAggregate; + agentHealth?: ReversalHealth; acceptance?: FindingAcceptance; }; @@ -214,6 +217,7 @@ function ProductAnalytics() { /> )} + {data.agentHealth ? : null} {data.usageSummary ? ( diff --git a/src/review/ops.ts b/src/review/ops.ts index f29103af77..7b20db1225 100644 --- a/src/review/ops.ts +++ b/src/review/ops.ts @@ -51,6 +51,8 @@ export interface AgentHealth { dlqTargets?: FailedTarget[]; reversals: number; reversalRate: number; + /** Merged + closed auto-actions in the 7d anomaly window — the reversalRate denominator. */ + recentAutoActions: number; failedTargets?: FailedTarget[]; reversedTargets?: ReversedTarget[]; configIssues: string[]; @@ -229,6 +231,7 @@ export async function computeAgentHealth(env: Env, config: OpsAgentConfig, deps: dlqTargets, reversals, reversalRate: recentAutoActions ? Number((reversals / recentAutoActions).toFixed(3)) : 0, + recentAutoActions, failedTargets, reversedTargets, configIssues: deps.validateAgentConfig(config), diff --git a/src/services/operator-dashboard.ts b/src/services/operator-dashboard.ts index dcaa0a2630..f689429c81 100644 --- a/src/services/operator-dashboard.ts +++ b/src/services/operator-dashboard.ts @@ -26,6 +26,7 @@ import type { WeeklyValueReport, } from "../types"; import { computeFleetAnalytics, type FleetAnalytics } from "../orb/analytics"; +import { computeAgentHealth, type AgentHealth } from "../review/ops"; import { computeGateEval, type GateEvalReport } from "../review/parity"; import { computeCycleTimeAggregate, type CycleTimeAggregate } from "../review/stats"; import { loadUpstreamStatus, type UpstreamStatus } from "../upstream/ruleset"; @@ -66,6 +67,8 @@ export type OperatorDashboardPayload = { gateEval: GateEvalReport; // PR review cycle-time percentiles (#2194): gate decision → outcome from review_audit; fail-safe empty aggregate. cycleTime: CycleTimeAggregate; + // Agent reversal health (#2193): how often humans reopened/reverted bot auto-actions (ops.ts AgentHealth). + agentHealth: AgentHealth; }; const USAGE_WINDOW_DAYS = 7; @@ -91,6 +94,7 @@ export async function buildOperatorDashboardPayload(env: Env): Promise } { + const slug = + typeof env.GITHUB_APP_SLUG === "string" && env.GITHUB_APP_SLUG.trim() + ? env.GITHUB_APP_SLUG.trim() + : "gittensory"; + return { slug, secrets: {} }; +} + +export const __operatorDashboardInternals = { operatorAgentConfig }; + export function latestUsageRollup(rollups: ProductUsageDailyRollupRecord[]): ProductUsageDailyRollupRecord | null { if (rollups.length === 0) return null; return [...rollups].sort((a, b) => b.day.localeCompare(a.day))[0]!; diff --git a/test/unit/operator-dashboard.test.ts b/test/unit/operator-dashboard.test.ts index 35bd38c070..79898f0932 100644 --- a/test/unit/operator-dashboard.test.ts +++ b/test/unit/operator-dashboard.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { buildOperatorDashboardPayload, latestUsageRollup } from "../../src/services/operator-dashboard"; +import { buildOperatorDashboardPayload, latestUsageRollup, __operatorDashboardInternals } from "../../src/services/operator-dashboard"; import type { ProductUsageDailyRollupRecord } from "../../src/types"; import { createTestEnv } from "../helpers/d1"; @@ -33,6 +33,13 @@ describe("operator dashboard payload", () => { distribution: [], sampleSize: 0, }); + expect(payload.agentHealth).toMatchObject({ + reversals: 0, + reversalRate: 0, + manualRate: 0, + recentAutoActions: 0, + reversedTargets: [], + }); // Empty fleet → instanceCount 0, null precision card ("—"), no-outlier delta. expect(payload.fleetMetrics.instanceCount).toBe(0); expect(payload.metrics).toEqual( @@ -43,6 +50,49 @@ describe("operator dashboard payload", () => { ); }); + it("falls back to the default agent slug when GITHUB_APP_SLUG is unset or blank", async () => { + const env = createTestEnv({ PRODUCT_USAGE_HASH_SALT: "operator-dashboard-test-salt" }); + delete (env as Partial).GITHUB_APP_SLUG; + const missingSlug = await buildOperatorDashboardPayload(env); + expect(missingSlug.agentHealth).toMatchObject({ + reversals: 0, + reversalRate: 0, + manualRate: 0, + recentAutoActions: 0, + reversedTargets: [], + }); + + const blankSlug = await buildOperatorDashboardPayload( + createTestEnv({ PRODUCT_USAGE_HASH_SALT: "operator-dashboard-test-salt", GITHUB_APP_SLUG: " " }), + ); + expect(blankSlug.agentHealth).toMatchObject({ + reversals: 0, + reversalRate: 0, + manualRate: 0, + recentAutoActions: 0, + reversedTargets: [], + }); + }); + + it("operatorAgentConfig trims a configured slug and falls back when absent", () => { + const { operatorAgentConfig } = __operatorDashboardInternals; + expect(operatorAgentConfig(createTestEnv({ GITHUB_APP_SLUG: " custom-app " }))).toEqual({ + slug: "custom-app", + secrets: {}, + }); + expect(operatorAgentConfig(createTestEnv({ GITHUB_APP_SLUG: "gittensory" }))).toEqual({ + slug: "gittensory", + secrets: {}, + }); + const unset = createTestEnv(); + delete (unset as Partial).GITHUB_APP_SLUG; + expect(operatorAgentConfig(unset)).toEqual({ slug: "gittensory", secrets: {} }); + expect(operatorAgentConfig(createTestEnv({ GITHUB_APP_SLUG: "" }))).toEqual({ + slug: "gittensory", + secrets: {}, + }); + }); + it("surfaces populated fleet metrics + outliers from orb_signals", async () => { const env = createTestEnv(); let n = 0; diff --git a/test/unit/ops.test.ts b/test/unit/ops.test.ts index 3c4bb2c6cf..9e007c994c 100644 --- a/test/unit/ops.test.ts +++ b/test/unit/ops.test.ts @@ -251,6 +251,7 @@ describe("computeAgentHealth (native D1, default gate deps)", () => { expect(h.manualRate).toBe(0.2); expect(h.reversals).toBe(1); expect(h.reversalRate).toBe(0.5); // 1 reversal / 2 recent auto-actions + expect(h.recentAutoActions).toBe(2); expect(h.configIssues).toEqual([]); expect(h.frozen).toBe(false); expect(h.holdOnly).toBe(false); @@ -547,6 +548,7 @@ describe("computeAgentHealth empty-ledger fallbacks", () => { expect(h.dlqTargets).toEqual([]); expect(h.reversals).toBe(0); expect(h.reversalRate).toBe(0); // recentAutoActions 0 → ternary false branch + expect(h.recentAutoActions).toBe(0); }); it("computes manualRate with a present terminalCount but no manual rows (byStatus.manual ?? 0 fallback)", async () => {