diff --git a/src/api/routes.ts b/src/api/routes.ts index c505898468..2fd081a411 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -200,6 +200,7 @@ import { buildPullRequestReviewability, type PullRequestReviewability } from ".. import { buildLocalBranchAnalysis, findCurrentBranchPullRequest } from "../signals/local-branch"; import { buildPredictedGateVerdict } from "../rules/predicted-gate"; import { buildMaintainerActivationPreview, recommendedAdvisoryActivationSettings } from "../services/maintainer-activation"; +import { buildRepoOutcomeCalibration } from "../services/outcome-calibration"; import { buildMaintainerQualityDashboard, isMaintainerQualityDataStale } from "../services/maintainer-quality-dashboard"; import { MAX_LOCAL_SCORER_WARNING_CHARS, MAX_LOCAL_SCORER_WARNING_COUNT } from "../signals/local-scorer-diagnostics"; import { compileFocusManifestPolicy } from "../signals/focus-manifest"; @@ -1850,6 +1851,20 @@ export function createApp() { return c.json(buildMaintainerActivationPreview({ repoFullName: fullName, repo, settings, pullRequests, generatedAt: nowIso() })); }); + // #543 outcome-learning loop: is the slop score predictive, and are recommendations panning out? Read-only + // measurement over resolved PRs (slop band -> merge/close) + the recommendation-outcome ledger. Optional + // ?windowDays bounds the recommendation window. + app.get("/v1/repos/:owner/:repo/outcome-calibration", async (c) => { + const fullName = `${c.req.param("owner")}/${c.req.param("repo")}`; + const gate = await requireRepoMaintainer(c, fullName); + if (gate instanceof Response) return gate; + // A positive number opts into a bounded recommendation window; anything else (absent/0/NaN) → full + // history. The repository layer clamps + floors the value, so one comparison covers every input. + const windowDaysRaw = Number(c.req.query("windowDays")); + const windowDays = windowDaysRaw > 0 ? windowDaysRaw : undefined; + return c.json(await buildRepoOutcomeCalibration(c.env, fullName, windowDays)); + }); + // One-click "enable advisory mode" — turns on the gate + deterministic rules in advisory (non-blocking) // mode. Merges onto current settings so unrelated fields are preserved. app.post("/v1/repos/:owner/:repo/activation", async (c) => { @@ -3990,6 +4005,7 @@ function canSessionAccessPath(env: Env, identity: Extract(); + let totalMerged = 0; + let totalResolved = 0; + for (const pr of pullRequests) { + if (typeof pr.slopRisk !== "number" || !pr.slopBand) continue; // never assessed + const band = pr.slopBand as SlopBand; + if (!SLOP_BAND_ORDER.includes(band)) continue; + const outcome = terminalOutcome(pr); + if (!outcome) continue; // still open + const entry = counts.get(band) ?? { merged: 0, closed: 0 }; + if (outcome === "merged") { + entry.merged += 1; + totalMerged += 1; + } else { + entry.closed += 1; + } + counts.set(band, entry); + totalResolved += 1; + } + const bands: SlopBandCalibration[] = SLOP_BAND_ORDER.map((band) => { + const { merged, closed } = counts.get(band) ?? { merged: 0, closed: 0 }; + const sampleSize = merged + closed; + return { band, sampleSize, merged, closed, mergeRate: sampleSize > 0 ? round(merged / sampleSize) : 0 }; + }); + return { + totalResolved, + bands, + overallMergeRate: totalResolved > 0 ? round(totalMerged / totalResolved) : null, + discriminates: computeDiscriminates(bands), + }; +} + +function computeDiscriminates(bands: SlopBandCalibration[]): boolean | null { + const sampled = bands.filter((band) => band.sampleSize >= MIN_BAND_SAMPLE); // already in severity order + if (sampled.length < 2) return null; // not enough signal to judge + for (let index = 1; index < sampled.length; index += 1) { + // A later (higher-severity) band merging MORE than an earlier one means the score is not discriminating. + if (sampled[index]!.mergeRate > sampled[index - 1]!.mergeRate + 0.001) return false; + } + return true; +} + +/** + * Positive (accepted/merged/improved) vs negative (rejected/closed) vs pending (stale/ignored) split. Pure. + * When `repoFullName` is given, only outcomes targeting that repo are counted (by outcome/target repo). + */ +export function buildRecommendationOutcomeCalibration(outcomes: AgentRecommendationOutcomeRecord[], repoFullName?: string): RecommendationOutcomeCalibration { + const scoped = repoFullName ? outcomes.filter((o) => sameRepo(o.outcomeRepoFullName ?? o.targetRepoFullName, repoFullName)) : outcomes; + const positive = scoped.filter((o) => o.outcomeState === "accepted" || o.outcomeState === "merged" || o.outcomeState === "improved").length; + const negative = scoped.filter((o) => o.outcomeState === "rejected" || o.outcomeState === "closed").length; + const pending = scoped.filter((o) => o.outcomeState === "stale" || o.outcomeState === "ignored").length; + const resolved = positive + negative; + return { total: scoped.length, positive, negative, pending, positiveRate: resolved > 0 ? round(positive / resolved) : null }; +} + +export function buildOutcomeCalibrationSignals(slop: SlopOutcomeCalibration, recommendations: RecommendationOutcomeCalibration): string[] { + const signals: string[] = []; + if (slop.discriminates === true) { + signals.push(`Slop score is predictive: merge rate falls as the band rises (${slop.totalResolved} resolved PRs).`); + } else if (slop.discriminates === false) { + signals.push(`Slop score is NOT discriminating on current data — a higher band merged more often than a lower one. Consider recalibration.`); + } else { + signals.push(`Not enough resolved PRs per band to judge slop calibration yet (${slop.totalResolved} resolved).`); + } + if (recommendations.positiveRate !== null) { + signals.push(`Recommendations: ${Math.round(recommendations.positiveRate * 100)}% positive outcomes across ${recommendations.positive + recommendations.negative} resolved (${recommendations.pending} still pending).`); + } else { + signals.push(`No resolved recommendation outcomes yet to calibrate against.`); + } + return signals; +} + +function sameRepo(a: string | null | undefined, b: string): boolean { + return (a ?? "").toLowerCase() === b.toLowerCase(); +} + +/** Load a repo's PRs + recommendation outcomes and assemble the calibration report. */ +export async function buildRepoOutcomeCalibration(env: Env, repoFullName: string, windowDays?: number): Promise { + const [pullRequests, outcomes] = await Promise.all([ + listPullRequests(env, repoFullName), + listAgentRecommendationOutcomes(env, windowDays !== undefined ? { windowDays } : {}), + ]); + const slop = buildSlopOutcomeCalibration(pullRequests); + const recommendations = buildRecommendationOutcomeCalibration(outcomes, repoFullName); + return { repoFullName, generatedAt: nowIso(), windowDays: windowDays ?? null, slop, recommendations, signals: buildOutcomeCalibrationSignals(slop, recommendations) }; +} diff --git a/test/integration/api.test.ts b/test/integration/api.test.ts index 928dde59e3..c7b4e7fbef 100644 --- a/test/integration/api.test.ts +++ b/test/integration/api.test.ts @@ -592,6 +592,22 @@ describe("api routes", () => { dataQuality: expect.any(Object), }); + // #543 outcome-learning calibration: maintainer-scoped, read-only. + const calibrationUnauthenticated = await app.request("/v1/repos/entrius/allways-ui/outcome-calibration", {}, env); + expect(calibrationUnauthenticated.status).toBe(401); + const calibration = await app.request("/v1/repos/entrius/allways-ui/outcome-calibration?windowDays=30", { headers: apiHeaders(env) }, env); + expect(calibration.status).toBe(200); + await expect(calibration.json()).resolves.toMatchObject({ + repoFullName: "entrius/allways-ui", + windowDays: 30, + slop: { totalResolved: expect.any(Number), bands: expect.any(Array), discriminates: null }, + recommendations: { total: expect.any(Number) }, + signals: expect.any(Array), + }); + // No windowDays → defaults to the full window (covers the param-absent path). + const calibrationNoWindow = await app.request("/v1/repos/entrius/allways-ui/outcome-calibration", { headers: apiHeaders(env) }, env); + await expect(calibrationNoWindow.json()).resolves.toMatchObject({ windowDays: null }); + const settingsPreviewUnauthenticated = await app.request("/v1/repos/entrius/allways-ui/settings-preview", { method: "POST", body: "{}" }, env); expect(settingsPreviewUnauthenticated.status).toBe(401); diff --git a/test/unit/outcome-calibration.test.ts b/test/unit/outcome-calibration.test.ts new file mode 100644 index 0000000000..b57007e9d1 --- /dev/null +++ b/test/unit/outcome-calibration.test.ts @@ -0,0 +1,122 @@ +import { describe, expect, it } from "vitest"; +import { + buildOutcomeCalibrationSignals, + buildRecommendationOutcomeCalibration, + buildRepoOutcomeCalibration, + buildSlopOutcomeCalibration, +} from "../../src/services/outcome-calibration"; +import { updatePullRequestSlopAssessment, upsertPullRequestFromGitHub } from "../../src/db/repositories"; +import type { SlopBand } from "../../src/signals/slop"; +import type { AgentRecommendationOutcomeRecord, AgentRecommendationOutcomeState, PullRequestRecord } from "../../src/types"; +import { createTestEnv } from "../helpers/d1"; + +// A resolved PR carrying a slop assessment. `merged` → has a merge timestamp; otherwise closed-unmerged. +function pr(band: SlopBand, merged: boolean, number: number): PullRequestRecord { + return { + repoFullName: "owner/repo", + number, + title: `PR ${number}`, + state: "closed", + mergedAt: merged ? "2026-06-01T00:00:00.000Z" : null, + labels: [], + linkedIssues: [], + slopRisk: band === "clean" ? 0 : band === "low" ? 10 : band === "elevated" ? 40 : 70, + slopBand: band, + }; +} + +// n PRs in a band, `merged` of them merged (the rest closed-unmerged). +function band(b: SlopBand, n: number, merged: number, base: number): PullRequestRecord[] { + return Array.from({ length: n }, (_, i) => pr(b, i < merged, base + i)); +} + +describe("buildSlopOutcomeCalibration", () => { + it("computes per-band merge rates and reports discrimination when higher bands merge less", () => { + const result = buildSlopOutcomeCalibration([...band("clean", 6, 5, 0), ...band("high", 6, 1, 100)]); + expect(result.totalResolved).toBe(12); + const byBand = Object.fromEntries(result.bands.map((b) => [b.band, b])); + expect(byBand.clean).toMatchObject({ sampleSize: 6, merged: 5, mergeRate: 0.833 }); + expect(byBand.high).toMatchObject({ sampleSize: 6, merged: 1, mergeRate: 0.167 }); + expect(result.overallMergeRate).toBe(0.5); + expect(result.discriminates).toBe(true); // clean merges more than high → predictive + }); + + it("flags a non-discriminating score when a higher band merges MORE", () => { + const result = buildSlopOutcomeCalibration([...band("clean", 6, 1, 0), ...band("high", 6, 5, 100)]); + expect(result.discriminates).toBe(false); + }); + + it("returns null discrimination when there isn't enough per-band sample", () => { + const result = buildSlopOutcomeCalibration([...band("clean", 2, 2, 0), ...band("high", 2, 0, 100)]); + expect(result.discriminates).toBeNull(); // each band below the min sample + expect(result.totalResolved).toBe(4); + }); + + it("excludes open PRs and PRs with no slop assessment", () => { + const open: PullRequestRecord = { repoFullName: "owner/repo", number: 9, title: "open", state: "open", labels: [], linkedIssues: [], slopRisk: 70, slopBand: "high" }; + const unassessed: PullRequestRecord = { repoFullName: "owner/repo", number: 10, title: "no slop", state: "closed", mergedAt: "2026-06-01T00:00:00.000Z", labels: [], linkedIssues: [] }; + const result = buildSlopOutcomeCalibration([open, unassessed, ...band("clean", 1, 1, 0)]); + expect(result.totalResolved).toBe(1); // only the one assessed+resolved PR + }); +}); + +describe("buildRecommendationOutcomeCalibration", () => { + function outcome(state: AgentRecommendationOutcomeState): AgentRecommendationOutcomeRecord { + return { actionId: `a-${state}`, runId: "r", actorLogin: "miner", actionType: "choose_next_work", source: "explicit", outcomeState: state, outcomeTargetType: "pull_request", maintainerLane: false, confidence: "high", reason: "x", metadata: {} }; + } + it("splits positive / negative / pending and computes a positive rate over resolved", () => { + const result = buildRecommendationOutcomeCalibration([outcome("merged"), outcome("improved"), outcome("accepted"), outcome("closed"), outcome("stale"), outcome("ignored")]); + expect(result).toMatchObject({ total: 6, positive: 3, negative: 1, pending: 2, positiveRate: 0.75 }); // 3 of 4 resolved + }); + it("reports a null rate when nothing is resolved", () => { + expect(buildRecommendationOutcomeCalibration([outcome("stale")]).positiveRate).toBeNull(); + expect(buildRecommendationOutcomeCalibration([]).positiveRate).toBeNull(); + }); + it("scopes to a repo (case-insensitive, by outcome repo then target repo) when repoFullName is given", () => { + const outcomes: AgentRecommendationOutcomeRecord[] = [ + { ...outcome("merged"), outcomeRepoFullName: "Owner/Repo" }, // in scope (case-insensitive on outcome repo) + { ...outcome("closed"), outcomeRepoFullName: null, targetRepoFullName: "owner/repo" }, // in scope via target-repo fallback + { ...outcome("merged"), outcomeRepoFullName: "other/repo" }, // out of scope + { ...outcome("accepted") }, // no repo at all → excluded by scope + ]; + expect(buildRecommendationOutcomeCalibration(outcomes, "owner/repo")).toMatchObject({ total: 2, positive: 1, negative: 1, positiveRate: 0.5 }); + }); +}); + +describe("buildOutcomeCalibrationSignals", () => { + const slop = (discriminates: boolean | null) => ({ totalResolved: 12, bands: [], overallMergeRate: 0.5, discriminates }); + const recs = (positiveRate: number | null) => ({ total: 4, positive: 3, negative: 1, pending: 0, positiveRate }); + + it("describes a predictive score + a recommendation positive rate", () => { + const out = buildOutcomeCalibrationSignals(slop(true), recs(0.75)).join(" "); + expect(out).toMatch(/predictive/i); + expect(out).toMatch(/75% positive/); + }); + it("warns when the score is NOT discriminating", () => { + expect(buildOutcomeCalibrationSignals(slop(false), recs(0.5)).join(" ")).toMatch(/NOT discriminating/i); + }); + it("notes insufficient data when discrimination is unknown and no recommendations are resolved", () => { + const out = buildOutcomeCalibrationSignals(slop(null), recs(null)).join(" "); + expect(out).toMatch(/Not enough resolved PRs/i); + expect(out).toMatch(/No resolved recommendation/i); + }); +}); + +describe("buildRepoOutcomeCalibration (env loader)", () => { + it("loads a repo's resolved PRs + slop bands and assembles the report", async () => { + const env = createTestEnv(); + await upsertPullRequestFromGitHub(env, "owner/repo", { number: 1, title: "merged clean", state: "closed", user: { login: "alice" }, merged_at: "2026-06-01T00:00:00.000Z" }); + await updatePullRequestSlopAssessment(env, "owner/repo", 1, { slopRisk: 0, slopBand: "clean" }); + await upsertPullRequestFromGitHub(env, "owner/repo", { number: 2, title: "closed high", state: "closed", user: { login: "bob" } }); + await updatePullRequestSlopAssessment(env, "owner/repo", 2, { slopRisk: 70, slopBand: "high" }); + + const report = await buildRepoOutcomeCalibration(env, "owner/repo"); + expect(report.repoFullName).toBe("owner/repo"); + expect(report.slop.totalResolved).toBe(2); + expect(report.slop.bands.find((b) => b.band === "clean")).toMatchObject({ merged: 1, closed: 0 }); + expect(report.slop.bands.find((b) => b.band === "high")).toMatchObject({ merged: 0, closed: 1 }); + expect(report.recommendations).toMatchObject({ total: 0, positiveRate: null }); // none seeded for this repo + expect(report.signals.length).toBeGreaterThan(0); + expect(JSON.stringify(report)).not.toMatch(/reward|payout|trust score|wallet|hotkey/i); + }); +});