feat(review): fleet-wide gaming-pattern detector for the self-host orb fleet (#2350)

src/mcp/server.ts

@@ -2859,7 +2859,7 @@ export class GittensoryMcp {
     const report = await computeFleetAnalytics(this.env, input.windowDays !== undefined ? { windowDays: input.windowDays } : {});
     const merge = report.fleet.mergePrecision !== null ? `${Math.round(report.fleet.mergePrecision * 100)}%` : "n/a";
     return {
-      summary: `Fleet calibration over ${report.windowDays}d: ${report.instanceCount} instance(s), median merge precision ${merge}, ${report.outliers.length} outlier(s).`,
+      summary: `Fleet calibration over ${report.windowDays}d: ${report.instanceCount} instance(s), median merge precision ${merge}, ${report.outliers.length} outlier(s), ${report.gamingPatternFlags.length} gaming-pattern flag(s).`,
       data: report as unknown as Record<string, unknown>,
     };
   }

src/orb/analytics.ts

@@ -1,9 +1,36 @@
 // Gittensory Orb (#1255) — fleet calibration ANALYTICS. Reads the anonymized orb_signals collected from
 // self-hosted instances and derives gate-accuracy metrics across the fleet. Aggregation is median/percentile
 // (never mean) so a single instance contributing fabricated data cannot move the fleet numbers.
+//
+// ANTI-FARMING DETECTION (#2350): gamingPatternFlags below extends the existing outlier check with a more targeted,
+// ONE-SIDED signal for the specific "gaming" pattern the issue describes -- an instance mass-submitting only
+// trivially-safe PRs to inflate its own merge-precision. mergePrecision alone can't distinguish "gamed" from
+// "genuinely excellent" (a careful team also has high precision); combining it with UNUSUALLY HIGH volume and
+// UNUSUALLY LOW reversal-rate, all three simultaneously, is the actual farming signature: lots of easy merges,
+// nothing risky enough to ever get reverted. Detection only — never an automatic action.
+//
+// SCOPE (explicit non-goals, read before extending): this flags a self-hosted INSTANCE, never an individual
+// miner. The fleet pipeline (orb_signals, review_audit's export) carries NO per-actor identity by deliberate,
+// repeatedly-stated design (review_audit has no login column; predicted_gate_calibration_ledger is explicitly
+// documented as never-exported, citing THIS issue as the reason) -- a genuine per-miner detector would require
+// adding a new anonymized per-actor signal to the export pipeline, which is a privacy-sensitive design
+// decision deserving its own focused issue/PR, not a rushed addition here. This module never deanonymizes,
+// never auto-bans, and never touches the live gate — instanceId here is the SAME opaque, HMAC-derived handle
+// already used everywhere else in this pipeline (see selfhost/orb-collector.ts), nothing more identifying.
+//
+// OUT OF SCOPE: "duplicate-claim-election win-rate skew" (isDuplicateClusterWinnerByClaim,
+// src/signals/duplicate-winner.ts) is NOT implemented here. Its outcome is never persisted anywhere in this
+// pipeline — only the LOSING side of a duplicate cluster produces a finding (duplicate_pr_risk), bucketed as
+// gate_reasoncode_bucket="duplicate_risk" on export with no cluster id and no actor linkage. There is no
+// winner marker to measure a win-rate FROM, and a per-instance duplicate_risk rate would measure something
+// different (how often THIS instance's own PRs lose a local collision) than "identities farming wins," so no
+// proxy for it is implemented — a misleading proxy would be worse than none.
 
 const MIN_DECIDED = 5; // an instance needs at least this many decided PRs to count toward the fleet median
 const OUTLIER_BAND = 0.25; // |instance precision − fleet median| beyond this flags the instance
+const GAMING_VOLUME_MULTIPLIER = 2; // an instance's decided count more than this many times the fleet median
+const GAMING_PRECISION_BAND = OUTLIER_BAND; // mergePrecision this far ABOVE the fleet median (one-sided)
+const GAMING_REVERSAL_RATIO = 0.5; // reversalRate below this fraction of the fleet median
 
 /** Per-instance confusion-matrix cell as stored. */
 interface Cell {
@@ -29,6 +56,20 @@ export interface InstanceMetrics {
   reversalRate: number; // share of decided PRs a human reversed
 }
 
+/** #2350: one self-hosted instance whose combined volume/precision/reversal-rate pattern looks like it is
+ *  gaming the fleet-aggregate accuracy signal (see the module doc comment for the exact signature and its
+ *  scope). Detection only — a human reads this, nothing here takes any action automatically. `instanceId` is
+ *  the same opaque, HMAC-derived handle used throughout this pipeline; nothing more identifying is included. */
+export interface GamingPatternFlag {
+  instanceId: string;
+  decided: number;
+  mergePrecision: number;
+  reversalRate: number;
+  fleetMedianDecided: number;
+  fleetMergePrecision: number;
+  fleetReversalRate: number;
+}
+
 export interface FleetAnalytics {
   windowDays: number;
   instanceCount: number; // instances meeting MIN_DECIDED
@@ -42,6 +83,7 @@ export interface FleetAnalytics {
   };
   instances: InstanceMetrics[];
   outliers: Array<{ instanceId: string; metric: string; value: number; fleetMedian: number }>;
+  gamingPatternFlags: GamingPatternFlag[];
 }
 
 function median(xs: number[]): number | null {
@@ -127,7 +169,7 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe
     const reg = await env.DB.prepare(`SELECT instance_id FROM orb_instances WHERE registered = 1`).all<{ instance_id: string }>();
     registered = new Set((reg.results ?? []).map((r) => r.instance_id));
   } catch {
-    return { windowDays, instanceCount: 0, fleet: { mergePrecision: null, closePrecision: null, fpRate: null, reversalRate: null, cycleP50Ms: null, cycleP95Ms: null }, instances: [], outliers: [] };
+    return { windowDays, instanceCount: 0, fleet: { mergePrecision: null, closePrecision: null, fpRate: null, reversalRate: null, cycleP50Ms: null, cycleP95Ms: null }, instances: [], outliers: [], gamingPatternFlags: [] };
   }
 
   // Group cells by instance, fold each.
@@ -157,6 +199,31 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe
     }
   }
 
+  // #2350: gamingPatternFlags. Gated on fleetMergeP !== null (at least one eligible instance made a comparable
+  // merge verdict) — decided/reversalRate are never null per-instance, so once `eligible` is known non-empty
+  // (implied by fleetMergeP being resolvable), both medians below are guaranteed non-null too.
+  const gamingPatternFlags: FleetAnalytics["gamingPatternFlags"] = [];
+  if (fleetMergeP !== null) {
+    const fleetMedianDecided = median(eligible.map((i) => i.decided))!;
+    const fleetReversalRate = median(eligible.map((i) => i.reversalRate))!;
+    for (const i of eligible) {
+      const highVolume = i.decided > fleetMedianDecided * GAMING_VOLUME_MULTIPLIER;
+      const highPrecision = i.mergePrecision !== null && i.mergePrecision - fleetMergeP > GAMING_PRECISION_BAND;
+      const lowReversal = i.reversalRate < fleetReversalRate * GAMING_REVERSAL_RATIO;
+      if (highVolume && highPrecision && lowReversal) {
+        gamingPatternFlags.push({
+          instanceId: i.instanceId,
+          decided: i.decided,
+          mergePrecision: i.mergePrecision!,
+          reversalRate: i.reversalRate,
+          fleetMedianDecided,
+          fleetMergePrecision: fleetMergeP,
+          fleetReversalRate,
+        });
+      }
+    }
+  }
+
   return {
     windowDays,
     instanceCount: eligible.length,
@@ -170,5 +237,6 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe
     },
     instances,
     outliers,
+    gamingPatternFlags,
   };
 }

src/services/operator-dashboard.ts

@@ -181,6 +181,12 @@ export async function buildOperatorDashboardPayload(env: Env): Promise<OperatorD
         value: fleetMetrics.fleet.mergePrecision !== null ? `${Math.round(fleetMetrics.fleet.mergePrecision * 100)}%` : "—",
         delta: "median across the fleet",
       },
+      {
+        // #2350: human-facing detection signal only — no automatic action reads this value.
+        label: "Fleet gaming-pattern flags",
+        value: String(fleetMetrics.gamingPatternFlags.length),
+        delta: fleetMetrics.gamingPatternFlags.length > 0 ? `${fleetMetrics.gamingPatternFlags.map((f) => f.instanceId).join(", ")}` : "no gaming pattern detected",
+      },
     ],
     noiseReduction: [
       {

test/unit/operator-dashboard.test.ts

@@ -127,10 +127,37 @@ describe("operator dashboard payload", () => {
       expect.arrayContaining([
         expect.objectContaining({ label: "Fleet instances", value: "3", delta: "1 outlier(s)" }),
         expect.objectContaining({ label: "Fleet merge precision", value: "100%" }),
+        expect.objectContaining({ label: "Fleet gaming-pattern flags", value: "0", delta: "no gaming pattern detected" }),
       ]),
     );
   });
 
+  it("surfaces a fleet farming flag (#2350) as a dedicated dashboard tile naming the flagged instance", async () => {
+    const env = createTestEnv();
+    let n = 0;
+    const seed = async (instance: string, count: number, opts: { reversal?: string } = {}): Promise<void> => {
+      for (let i = 0; i < count; i++) {
+        await env.DB
+          .prepare(`INSERT INTO orb_signals (instance_id, repo_hash, pr_hash, gate_verdict, outcome, reversal_flag) VALUES (?, ?, ?, 'merge', 'merged', ?)`)
+          .bind(instance, `r${n}`, `p${n++}`, opts.reversal ?? "none")
+          .run();
+      }
+    };
+    // Two normal instances: decided 10, precision 0.7, reversalRate 0.3 (7 confirmed + 3 reverted).
+    for (const id of ["normal1", "normal2"]) {
+      await seed(id, 7);
+      await seed(id, 3, { reversal: "reverted" });
+    }
+    // Farmer: decided 30 (> 2x the fleet median volume of 10), precision 1.0 (> 0.7 + 0.25), reversalRate 0.
+    await seed("farmer", 30);
+    for (const id of ["normal1", "normal2", "farmer"]) {
+      await env.DB.prepare(`INSERT INTO orb_instances (instance_id, registered) VALUES (?, 1)`).bind(id).run();
+    }
+    const payload = await buildOperatorDashboardPayload(env);
+    expect(payload.fleetMetrics.gamingPatternFlags.map((f) => f.instanceId)).toEqual(["farmer"]);
+    expect(payload.metrics).toEqual(expect.arrayContaining([expect.objectContaining({ label: "Fleet gaming-pattern flags", value: "1", delta: "farmer" })]));
+  });
+
   it("picks the newest rollup day for adoption insights", () => {
     const rollups: ProductUsageDailyRollupRecord[] = [
       rollup("2026-05-28"),

test/unit/orb-analytics.test.ts

@@ -159,3 +159,154 @@ describe("computeFleetAnalytics()", () => {
     expect(a.fleet.cycleP95Ms).toBe(3000);
   });
 });
+
+describe("gamingPatternFlags — anti-farming detection (#2350)", () => {
+  /** 7 confirmed merges + 3 reverted merges: precision 0.7, reversalRate 0.3. */
+  async function normalInstance(env: Env, id: string): Promise<void> {
+    await signals(env, id, 7, { verdict: "merge", outcome: "merged", reversal: "none" });
+    await signals(env, id, 3, { verdict: "merge", outcome: "merged", reversal: "reverted" });
+  }
+
+  it("a normal-distribution fleet (similar volume/precision/reversal everywhere) produces no flag", async () => {
+    const env = createTestEnv();
+    await normalInstance(env, "a");
+    await normalInstance(env, "b");
+    await normalInstance(env, "c");
+    await register(env, "a", "b", "c");
+    const result = await computeFleetAnalytics(env);
+    expect(result.instanceCount).toBe(3);
+    expect(result.gamingPatternFlags).toEqual([]);
+  });
+
+  it("an inflated-trivial-volume instance (high volume + high precision + low reversal, all three) flags", async () => {
+    const env = createTestEnv();
+    await normalInstance(env, "normal1"); // decided 10, precision 0.7, reversalRate 0.3
+    await normalInstance(env, "normal2");
+    await normalInstance(env, "normal3");
+    // 30 decided (> 2x the fleet median of 10), precision 1.0 (> 0.7 + 0.25), reversalRate 0 (< 0.5 * 0.3).
+    await signals(env, "farmer", 30, { verdict: "merge", outcome: "merged", reversal: "none" });
+    await register(env, "normal1", "normal2", "normal3", "farmer");
+
+    const result = await computeFleetAnalytics(env);
+    expect(result.gamingPatternFlags).toHaveLength(1);
+    const flag = result.gamingPatternFlags[0]!;
+    expect(flag.instanceId).toBe("farmer");
+    expect(flag.decided).toBe(30);
+    expect(flag.mergePrecision).toBe(1);
+    expect(flag.reversalRate).toBe(0);
+    expect(flag.fleetMedianDecided).toBe(10);
+    expect(flag.fleetMergePrecision).toBeCloseTo(0.7);
+    expect(flag.fleetReversalRate).toBeCloseTo(0.3);
+    // No identity beyond the same opaque instance handle used everywhere else in the pipeline.
+    expect(Object.keys(flag).sort()).toEqual(["decided", "fleetMedianDecided", "fleetMergePrecision", "fleetReversalRate", "instanceId", "mergePrecision", "reversalRate"]);
+  });
+
+  it("high precision WITHOUT elevated volume does not flag (precision alone is not the signature)", async () => {
+    const env = createTestEnv();
+    await normalInstance(env, "normal1");
+    await normalInstance(env, "normal2");
+    // Same volume (10) as the normals, but perfect precision and zero reversals.
+    await signals(env, "precise", 10, { verdict: "merge", outcome: "merged", reversal: "none" });
+    await register(env, "normal1", "normal2", "precise");
+
+    const result = await computeFleetAnalytics(env);
+    expect(result.gamingPatternFlags).toEqual([]);
+    // It IS still caught by the existing, broader outlier check — this test isolates gamingPatternFlags specifically.
+    expect(result.outliers.map((o) => o.instanceId)).toContain("precise");
+  });
+
+  it("elevated volume WITHOUT elevated precision does not flag (volume alone is not the signature)", async () => {
+    const env = createTestEnv();
+    await normalInstance(env, "normal1");
+    await normalInstance(env, "normal2");
+    // 30 decided (high volume) but the SAME precision/reversal profile as everyone else.
+    await signals(env, "busy", 21, { verdict: "merge", outcome: "merged", reversal: "none" });
+    await signals(env, "busy", 9, { verdict: "merge", outcome: "merged", reversal: "reverted" });
+
+    await register(env, "normal1", "normal2", "busy");
+
+    const result = await computeFleetAnalytics(env);
+    expect(result.gamingPatternFlags).toEqual([]);
+  });
+
+  it("elevated volume + elevated precision but NOT a suspiciously low reversal rate does not flag", async () => {
+    const env = createTestEnv();
+    await normalInstance(env, "normal1"); // decided 10, precision 0.7, reversalRate 0.3
+    await normalInstance(env, "normal2");
+    // 30 decided (high volume). mergePrecision is 25/25 = 1.0 (elevated) -- ONLY the merge-verdict rows count
+    // toward it. reversalRate is 5/30 ≈ 0.167, from separate close-verdict reopens -- above the 0.5x-fleet-
+    // median floor (0.15), i.e. NOT suspiciously low, so this must not read as "farming".
+    await signals(env, "risky", 25, { verdict: "merge", outcome: "merged", reversal: "none" });
+    await signals(env, "risky", 5, { verdict: "close", outcome: "closed", reversal: "reopened" });
+    await register(env, "normal1", "normal2", "risky");
+
+    const result = await computeFleetAnalytics(env);
+    const risky = result.instances.find((i) => i.instanceId === "risky")!;
+    expect(risky.mergePrecision).toBe(1);
+    expect(risky.reversalRate).toBeCloseTo(5 / 30);
+    expect(result.gamingPatternFlags).toEqual([]);
+  });
+
+  it("an instance with no merge verdicts at all (null mergePrecision) never flags, even alongside a farmer", async () => {
+    const env = createTestEnv();
+    await normalInstance(env, "normal1"); // decided 10
+    await normalInstance(env, "normal2"); // decided 10
+    await signals(env, "farmer", 30, { verdict: "merge", outcome: "merged", reversal: "none" });
+    // Every verdict is "close" — mergePrecision is null, so it cannot be flagged on precision regardless of
+    // volume. Kept at MIN_DECIDED so it counts toward the fleet without shifting the volume median the farmer
+    // is measured against.
+    await signals(env, "close-only", 5, { verdict: "close", outcome: "closed", reversal: "none" });
+    await register(env, "normal1", "normal2", "farmer", "close-only");
+
+    const result = await computeFleetAnalytics(env);
+    expect(result.gamingPatternFlags.map((f) => f.instanceId)).toEqual(["farmer"]);
+  });
+
+  it("no flags when no eligible instance has any merge verdict (fleetMergeP unresolvable)", async () => {
+    const env = createTestEnv();
+    await signals(env, "a", 10, { verdict: "close", outcome: "closed" });
+    await signals(env, "b", 10, { verdict: "close", outcome: "closed" });
+    await register(env, "a", "b");
+
+    const result = await computeFleetAnalytics(env);
+    expect(result.fleet.mergePrecision).toBeNull();
+    expect(result.gamingPatternFlags).toEqual([]);
+  });
+
+  it("an unregistered instance never flags, even with an extreme farming-shaped pattern", async () => {
+    const env = createTestEnv();
+    await normalInstance(env, "normal1");
+    await normalInstance(env, "normal2");
+    await signals(env, "unregistered-farmer", 30, { verdict: "merge", outcome: "merged", reversal: "none" });
+    await register(env, "normal1", "normal2"); // deliberately NOT registering the farmer
+
+    const result = await computeFleetAnalytics(env);
+    expect(result.gamingPatternFlags).toEqual([]);
+    // Still visible per-instance for the operator, same precedent as outliers.
+    expect(result.instances.map((i) => i.instanceId)).toContain("unregistered-farmer");
+  });
+
+  it("a below-MIN_DECIDED instance never flags, even if registered with an extreme farming-shaped pattern", async () => {
+    const env = createTestEnv();
+    await normalInstance(env, "normal1");
+    await normalInstance(env, "normal2");
+    // Only 3 decided (< MIN_DECIDED = 5) — excluded from `eligible` regardless of registration.
+    await signals(env, "tiny-farmer", 3, { verdict: "merge", outcome: "merged", reversal: "none" });
+    await register(env, "normal1", "normal2", "tiny-farmer");
+
+    const result = await computeFleetAnalytics(env);
+    expect(result.gamingPatternFlags).toEqual([]);
+  });
+
+  it("empty store -> empty gamingPatternFlags (not undefined)", async () => {
+    const env = createTestEnv();
+    const result = await computeFleetAnalytics(env);
+    expect(result.gamingPatternFlags).toEqual([]);
+  });
+
+  it("fail-safe on a DB error -> empty gamingPatternFlags", async () => {
+    const broken = { DB: { prepare: () => ({ bind: () => ({ all: () => Promise.reject(new Error("boom")) }) }) } } as unknown as Env;
+    const result = await computeFleetAnalytics(broken);
+    expect(result.gamingPatternFlags).toEqual([]);
+  });
+});