JSONbored · JSONbored · Jun 14, 2026 · Jun 14, 2026
@@ -200,6 +200,7 @@ import { buildPullRequestReviewability, type PullRequestReviewability } from "..
 import { buildLocalBranchAnalysis, findCurrentBranchPullRequest } from "../signals/local-branch";
 import { buildPredictedGateVerdict } from "../rules/predicted-gate";
 import { buildMaintainerActivationPreview, recommendedAdvisoryActivationSettings } from "../services/maintainer-activation";
+import { buildRepoOutcomeCalibration } from "../services/outcome-calibration";
 import { buildMaintainerQualityDashboard, isMaintainerQualityDataStale } from "../services/maintainer-quality-dashboard";
 import { MAX_LOCAL_SCORER_WARNING_CHARS, MAX_LOCAL_SCORER_WARNING_COUNT } from "../signals/local-scorer-diagnostics";
 import { compileFocusManifestPolicy } from "../signals/focus-manifest";
@@ -1850,6 +1851,20 @@ export function createApp() {
     return c.json(buildMaintainerActivationPreview({ repoFullName: fullName, repo, settings, pullRequests, generatedAt: nowIso() }));
   });
 
+  // #543 outcome-learning loop: is the slop score predictive, and are recommendations panning out? Read-only
+  // measurement over resolved PRs (slop band -> merge/close) + the recommendation-outcome ledger. Optional
+  // ?windowDays bounds the recommendation window.
+  app.get("/v1/repos/:owner/:repo/outcome-calibration", async (c) => {
+    const fullName = `${c.req.param("owner")}/${c.req.param("repo")}`;
+    const gate = await requireRepoMaintainer(c, fullName);
+    if (gate instanceof Response) return gate;
+    // A positive number opts into a bounded recommendation window; anything else (absent/0/NaN) → full
+    // history. The repository layer clamps + floors the value, so one comparison covers every input.
+    const windowDaysRaw = Number(c.req.query("windowDays"));
+    const windowDays = windowDaysRaw > 0 ? windowDaysRaw : undefined;
+    return c.json(await buildRepoOutcomeCalibration(c.env, fullName, windowDays));
+  });
+
   // One-click "enable advisory mode" — turns on the gate + deterministic rules in advisory (non-blocking)
   // mode. Merges onto current settings so unrelated fields are preserved.
   app.post("/v1/repos/:owner/:repo/activation", async (c) => {
@@ -3990,6 +4005,7 @@ function canSessionAccessPath(env: Env, identity: Extract<AuthIdentity, { kind:
   if (isIssueQualityPath(path)) return true;
   if (isRepoSettingsPath(path)) return true;
   if (isRepoActivationPath(path)) return true;
+  if (isRepoOutcomeCalibrationPath(path)) return true;
   if (isRepoSettingsPreviewPath(path)) return true;
   if (isRepoOnboardingPackPreviewPath(path)) return true;
   if (isRepoFocusManifestPath(path)) return true;
@@ -4008,6 +4024,10 @@ function isRepoActivationPath(path: string): boolean {
   return /^\/v1\/repos\/[^/]+\/[^/]+\/activation(?:-preview)?$/.test(path);
 }
 
+function isRepoOutcomeCalibrationPath(path: string): boolean {
+  return /^\/v1\/repos\/[^/]+\/[^/]+\/outcome-calibration$/.test(path);
+}
+
 function isRepoSettingsPreviewPath(path: string): boolean {
   return /^\/v1\/repos\/[^/]+\/[^/]+\/settings-preview$/.test(path);
 }

@@ -0,0 +1,142 @@
+// #543 outcome-learning loop: calibrate gittensory's predictions against real merge/close outcomes.
+//
+// MEASUREMENT only — it never auto-adjusts a score (that would move live rankings; like time-decay it would
+// need owner review). It answers two questions a maintainer/operator can act on:
+//   • Is the deterministic slop score PREDICTIVE? For resolved PRs that carry a persisted slop band, do
+//     higher-slop bands actually merge less often? (`discriminates`).
+//   • Are gittensory's recommendations panning out? The positive vs negative outcome split from the agent
+//     recommendation-outcome ledger.
+// All inputs already exist: slop_band persists on the PR row (#726) + closed PRs are retained, and the
+// agent_recommendation_outcomes ledger (#543's recommendation half) is populated by evaluateRecommendationOutcomes.
+import { listAgentRecommendationOutcomes, listPullRequests } from "../db/repositories";
+import type { SlopBand } from "../signals/slop";
+import type { AgentRecommendationOutcomeRecord, PullRequestRecord } from "../types";
+import { nowIso } from "../utils/json";
+
+// Severity order — calibration checks that merge rate is non-increasing along it.
+const SLOP_BAND_ORDER: readonly SlopBand[] = ["clean", "low", "elevated", "high"];
+// Below this per-band sample the merge rate is too noisy to judge discrimination.
+const MIN_BAND_SAMPLE = 5;
+
+export type SlopBandCalibration = { band: SlopBand; sampleSize: number; merged: number; closed: number; mergeRate: number };
+
+export type SlopOutcomeCalibration = {
+  totalResolved: number;
+  bands: SlopBandCalibration[];
+  overallMergeRate: number | null;
+  /** True iff the score discriminates (merge rate non-increasing as band severity rises) given enough
+   *  per-band sample; false iff it inverts; null iff there isn't enough resolved data to judge. */
+  discriminates: boolean | null;
+};
+
+export type RecommendationOutcomeCalibration = { total: number; positive: number; negative: number; pending: number; positiveRate: number | null };
+
+export type OutcomeCalibration = {
+  repoFullName: string;
+  generatedAt: string;
+  windowDays: number | null;
+  slop: SlopOutcomeCalibration;
+  recommendations: RecommendationOutcomeCalibration;
+  signals: string[];
+};
+
+function round(value: number): number {
+  return Math.round(value * 1000) / 1000;
+}
+
+// A PR's terminal outcome for calibration: merged if it has a merge timestamp; closed (unmerged) if its
+// state is closed without one; otherwise still open (excluded — no outcome yet).
+function terminalOutcome(pr: PullRequestRecord): "merged" | "closed" | null {
+  if (pr.mergedAt) return "merged";
+  if (pr.state === "closed") return "closed";
+  return null;
+}
+
+/** Per-slop-band merge/close calibration over the resolved PRs that carry a slop assessment. Pure. */
+export function buildSlopOutcomeCalibration(pullRequests: PullRequestRecord[]): SlopOutcomeCalibration {
+  const counts = new Map<SlopBand, { merged: number; closed: number }>();
+  let totalMerged = 0;
+  let totalResolved = 0;
+  for (const pr of pullRequests) {
+    if (typeof pr.slopRisk !== "number" || !pr.slopBand) continue; // never assessed
+    const band = pr.slopBand as SlopBand;
+    if (!SLOP_BAND_ORDER.includes(band)) continue;
+    const outcome = terminalOutcome(pr);
+    if (!outcome) continue; // still open
+    const entry = counts.get(band) ?? { merged: 0, closed: 0 };
+    if (outcome === "merged") {
+      entry.merged += 1;
+      totalMerged += 1;
+    } else {
+      entry.closed += 1;
+    }
+    counts.set(band, entry);
+    totalResolved += 1;
+  }
+  const bands: SlopBandCalibration[] = SLOP_BAND_ORDER.map((band) => {
+    const { merged, closed } = counts.get(band) ?? { merged: 0, closed: 0 };
+    const sampleSize = merged + closed;
+    return { band, sampleSize, merged, closed, mergeRate: sampleSize > 0 ? round(merged / sampleSize) : 0 };
+  });
+  return {
+    totalResolved,
+    bands,
+    overallMergeRate: totalResolved > 0 ? round(totalMerged / totalResolved) : null,
+    discriminates: computeDiscriminates(bands),
+  };
+}
+
+function computeDiscriminates(bands: SlopBandCalibration[]): boolean | null {
+  const sampled = bands.filter((band) => band.sampleSize >= MIN_BAND_SAMPLE); // already in severity order
+  if (sampled.length < 2) return null; // not enough signal to judge
+  for (let index = 1; index < sampled.length; index += 1) {
+    // A later (higher-severity) band merging MORE than an earlier one means the score is not discriminating.
+    if (sampled[index]!.mergeRate > sampled[index - 1]!.mergeRate + 0.001) return false;
+  }
+  return true;
+}
+
+/**
+ * Positive (accepted/merged/improved) vs negative (rejected/closed) vs pending (stale/ignored) split. Pure.
+ * When `repoFullName` is given, only outcomes targeting that repo are counted (by outcome/target repo).
+ */
+export function buildRecommendationOutcomeCalibration(outcomes: AgentRecommendationOutcomeRecord[], repoFullName?: string): RecommendationOutcomeCalibration {
+  const scoped = repoFullName ? outcomes.filter((o) => sameRepo(o.outcomeRepoFullName ?? o.targetRepoFullName, repoFullName)) : outcomes;
+  const positive = scoped.filter((o) => o.outcomeState === "accepted" || o.outcomeState === "merged" || o.outcomeState === "improved").length;
+  const negative = scoped.filter((o) => o.outcomeState === "rejected" || o.outcomeState === "closed").length;
+  const pending = scoped.filter((o) => o.outcomeState === "stale" || o.outcomeState === "ignored").length;
+  const resolved = positive + negative;
+  return { total: scoped.length, positive, negative, pending, positiveRate: resolved > 0 ? round(positive / resolved) : null };
+}
+
+export function buildOutcomeCalibrationSignals(slop: SlopOutcomeCalibration, recommendations: RecommendationOutcomeCalibration): string[] {
+  const signals: string[] = [];
+  if (slop.discriminates === true) {
+    signals.push(`Slop score is predictive: merge rate falls as the band rises (${slop.totalResolved} resolved PRs).`);
+  } else if (slop.discriminates === false) {
+    signals.push(`Slop score is NOT discriminating on current data — a higher band merged more often than a lower one. Consider recalibration.`);
+  } else {
+    signals.push(`Not enough resolved PRs per band to judge slop calibration yet (${slop.totalResolved} resolved).`);
+  }
+  if (recommendations.positiveRate !== null) {
+    signals.push(`Recommendations: ${Math.round(recommendations.positiveRate * 100)}% positive outcomes across ${recommendations.positive + recommendations.negative} resolved (${recommendations.pending} still pending).`);
+  } else {
+    signals.push(`No resolved recommendation outcomes yet to calibrate against.`);
+  }
+  return signals;
+}
+
+function sameRepo(a: string | null | undefined, b: string): boolean {
+  return (a ?? "").toLowerCase() === b.toLowerCase();
+}
+
+/** Load a repo's PRs + recommendation outcomes and assemble the calibration report. */
+export async function buildRepoOutcomeCalibration(env: Env, repoFullName: string, windowDays?: number): Promise<OutcomeCalibration> {
+  const [pullRequests, outcomes] = await Promise.all([
+    listPullRequests(env, repoFullName),
+    listAgentRecommendationOutcomes(env, windowDays !== undefined ? { windowDays } : {}),
+  ]);
+  const slop = buildSlopOutcomeCalibration(pullRequests);
+  const recommendations = buildRecommendationOutcomeCalibration(outcomes, repoFullName);
+  return { repoFullName, generatedAt: nowIso(), windowDays: windowDays ?? null, slop, recommendations, signals: buildOutcomeCalibrationSignals(slop, recommendations) };
+}
@@ -592,6 +592,22 @@ describe("api routes", () => {
       dataQuality: expect.any(Object),
     });
 
+    // #543 outcome-learning calibration: maintainer-scoped, read-only.
+    const calibrationUnauthenticated = await app.request("/v1/repos/entrius/allways-ui/outcome-calibration", {}, env);
+    expect(calibrationUnauthenticated.status).toBe(401);
+    const calibration = await app.request("/v1/repos/entrius/allways-ui/outcome-calibration?windowDays=30", { headers: apiHeaders(env) }, env);
+    expect(calibration.status).toBe(200);
+    await expect(calibration.json()).resolves.toMatchObject({
+      repoFullName: "entrius/allways-ui",
+      windowDays: 30,
+      slop: { totalResolved: expect.any(Number), bands: expect.any(Array), discriminates: null },
+      recommendations: { total: expect.any(Number) },
+      signals: expect.any(Array),
+    });
+    // No windowDays → defaults to the full window (covers the param-absent path).
+    const calibrationNoWindow = await app.request("/v1/repos/entrius/allways-ui/outcome-calibration", { headers: apiHeaders(env) }, env);
+    await expect(calibrationNoWindow.json()).resolves.toMatchObject({ windowDays: null });
+
     const settingsPreviewUnauthenticated = await app.request("/v1/repos/entrius/allways-ui/settings-preview", { method: "POST", body: "{}" }, env);
     expect(settingsPreviewUnauthenticated.status).toBe(401);
 

@@ -0,0 +1,122 @@
+import { describe, expect, it } from "vitest";
+import {
+  buildOutcomeCalibrationSignals,
+  buildRecommendationOutcomeCalibration,
+  buildRepoOutcomeCalibration,
+  buildSlopOutcomeCalibration,
+} from "../../src/services/outcome-calibration";
+import { updatePullRequestSlopAssessment, upsertPullRequestFromGitHub } from "../../src/db/repositories";
+import type { SlopBand } from "../../src/signals/slop";
+import type { AgentRecommendationOutcomeRecord, AgentRecommendationOutcomeState, PullRequestRecord } from "../../src/types";
+import { createTestEnv } from "../helpers/d1";
+
+// A resolved PR carrying a slop assessment. `merged` → has a merge timestamp; otherwise closed-unmerged.
+function pr(band: SlopBand, merged: boolean, number: number): PullRequestRecord {
+  return {
+    repoFullName: "owner/repo",
+    number,
+    title: `PR ${number}`,
+    state: "closed",
+    mergedAt: merged ? "2026-06-01T00:00:00.000Z" : null,
+    labels: [],
+    linkedIssues: [],
+    slopRisk: band === "clean" ? 0 : band === "low" ? 10 : band === "elevated" ? 40 : 70,
+    slopBand: band,
+  };
+}
+
+// n PRs in a band, `merged` of them merged (the rest closed-unmerged).
+function band(b: SlopBand, n: number, merged: number, base: number): PullRequestRecord[] {
+  return Array.from({ length: n }, (_, i) => pr(b, i < merged, base + i));
+}
+
+describe("buildSlopOutcomeCalibration", () => {
+  it("computes per-band merge rates and reports discrimination when higher bands merge less", () => {
+    const result = buildSlopOutcomeCalibration([...band("clean", 6, 5, 0), ...band("high", 6, 1, 100)]);
+    expect(result.totalResolved).toBe(12);
+    const byBand = Object.fromEntries(result.bands.map((b) => [b.band, b]));
+    expect(byBand.clean).toMatchObject({ sampleSize: 6, merged: 5, mergeRate: 0.833 });
+    expect(byBand.high).toMatchObject({ sampleSize: 6, merged: 1, mergeRate: 0.167 });
+    expect(result.overallMergeRate).toBe(0.5);
+    expect(result.discriminates).toBe(true); // clean merges more than high → predictive
+  });
+
+  it("flags a non-discriminating score when a higher band merges MORE", () => {
+    const result = buildSlopOutcomeCalibration([...band("clean", 6, 1, 0), ...band("high", 6, 5, 100)]);
+    expect(result.discriminates).toBe(false);
+  });
+
+  it("returns null discrimination when there isn't enough per-band sample", () => {
+    const result = buildSlopOutcomeCalibration([...band("clean", 2, 2, 0), ...band("high", 2, 0, 100)]);
+    expect(result.discriminates).toBeNull(); // each band below the min sample
+    expect(result.totalResolved).toBe(4);
+  });
+
+  it("excludes open PRs and PRs with no slop assessment", () => {
+    const open: PullRequestRecord = { repoFullName: "owner/repo", number: 9, title: "open", state: "open", labels: [], linkedIssues: [], slopRisk: 70, slopBand: "high" };
+    const unassessed: PullRequestRecord = { repoFullName: "owner/repo", number: 10, title: "no slop", state: "closed", mergedAt: "2026-06-01T00:00:00.000Z", labels: [], linkedIssues: [] };
+    const result = buildSlopOutcomeCalibration([open, unassessed, ...band("clean", 1, 1, 0)]);
+    expect(result.totalResolved).toBe(1); // only the one assessed+resolved PR
+  });
+});
+
+describe("buildRecommendationOutcomeCalibration", () => {
+  function outcome(state: AgentRecommendationOutcomeState): AgentRecommendationOutcomeRecord {
+    return { actionId: `a-${state}`, runId: "r", actorLogin: "miner", actionType: "choose_next_work", source: "explicit", outcomeState: state, outcomeTargetType: "pull_request", maintainerLane: false, confidence: "high", reason: "x", metadata: {} };
+  }
+  it("splits positive / negative / pending and computes a positive rate over resolved", () => {
+    const result = buildRecommendationOutcomeCalibration([outcome("merged"), outcome("improved"), outcome("accepted"), outcome("closed"), outcome("stale"), outcome("ignored")]);
+    expect(result).toMatchObject({ total: 6, positive: 3, negative: 1, pending: 2, positiveRate: 0.75 }); // 3 of 4 resolved
+  });
+  it("reports a null rate when nothing is resolved", () => {
+    expect(buildRecommendationOutcomeCalibration([outcome("stale")]).positiveRate).toBeNull();
+    expect(buildRecommendationOutcomeCalibration([]).positiveRate).toBeNull();
+  });
+  it("scopes to a repo (case-insensitive, by outcome repo then target repo) when repoFullName is given", () => {
+    const outcomes: AgentRecommendationOutcomeRecord[] = [
+      { ...outcome("merged"), outcomeRepoFullName: "Owner/Repo" }, // in scope (case-insensitive on outcome repo)
+      { ...outcome("closed"), outcomeRepoFullName: null, targetRepoFullName: "owner/repo" }, // in scope via target-repo fallback
+      { ...outcome("merged"), outcomeRepoFullName: "other/repo" }, // out of scope
+      { ...outcome("accepted") }, // no repo at all → excluded by scope
+    ];
+    expect(buildRecommendationOutcomeCalibration(outcomes, "owner/repo")).toMatchObject({ total: 2, positive: 1, negative: 1, positiveRate: 0.5 });
+  });
+});
+
+describe("buildOutcomeCalibrationSignals", () => {
+  const slop = (discriminates: boolean | null) => ({ totalResolved: 12, bands: [], overallMergeRate: 0.5, discriminates });
+  const recs = (positiveRate: number | null) => ({ total: 4, positive: 3, negative: 1, pending: 0, positiveRate });
+
+  it("describes a predictive score + a recommendation positive rate", () => {
+    const out = buildOutcomeCalibrationSignals(slop(true), recs(0.75)).join(" ");
+    expect(out).toMatch(/predictive/i);
+    expect(out).toMatch(/75% positive/);
+  });
+  it("warns when the score is NOT discriminating", () => {
+    expect(buildOutcomeCalibrationSignals(slop(false), recs(0.5)).join(" ")).toMatch(/NOT discriminating/i);
+  });
+  it("notes insufficient data when discrimination is unknown and no recommendations are resolved", () => {
+    const out = buildOutcomeCalibrationSignals(slop(null), recs(null)).join(" ");
+    expect(out).toMatch(/Not enough resolved PRs/i);
+    expect(out).toMatch(/No resolved recommendation/i);
+  });
+});
+
+describe("buildRepoOutcomeCalibration (env loader)", () => {
+  it("loads a repo's resolved PRs + slop bands and assembles the report", async () => {
+    const env = createTestEnv();
+    await upsertPullRequestFromGitHub(env, "owner/repo", { number: 1, title: "merged clean", state: "closed", user: { login: "alice" }, merged_at: "2026-06-01T00:00:00.000Z" });
+    await updatePullRequestSlopAssessment(env, "owner/repo", 1, { slopRisk: 0, slopBand: "clean" });
+    await upsertPullRequestFromGitHub(env, "owner/repo", { number: 2, title: "closed high", state: "closed", user: { login: "bob" } });
+    await updatePullRequestSlopAssessment(env, "owner/repo", 2, { slopRisk: 70, slopBand: "high" });
+
+    const report = await buildRepoOutcomeCalibration(env, "owner/repo");
+    expect(report.repoFullName).toBe("owner/repo");
+    expect(report.slop.totalResolved).toBe(2);
+    expect(report.slop.bands.find((b) => b.band === "clean")).toMatchObject({ merged: 1, closed: 0 });
+    expect(report.slop.bands.find((b) => b.band === "high")).toMatchObject({ merged: 0, closed: 1 });
+    expect(report.recommendations).toMatchObject({ total: 0, positiveRate: null }); // none seeded for this repo
+    expect(report.signals.length).toBeGreaterThan(0);
+    expect(JSON.stringify(report)).not.toMatch(/reward|payout|trust score|wallet|hotkey/i);
+  });
+});