diff --git a/src/mcp/server.ts b/src/mcp/server.ts index b57b117e83..bddf7263de 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -68,6 +68,7 @@ import { loadOrComputeIssueQualityResponse } from "../services/issue-quality"; import { loadOrComputeBurdenForecastResponse } from "../services/burden-forecast"; import { buildMcpClientTelemetry } from "../services/client-telemetry"; import { loadOrComputeRepoOutcomePatternsResponse } from "../services/repo-outcome-patterns"; +import { buildRepoOutcomeCalibration, outcomeCalibrationSummary } from "../services/outcome-calibration"; import { buildUnavailableQueueTrendReport } from "../services/queue-trends"; import { applyMcpPlanningChoices, @@ -137,6 +138,12 @@ const ownerRepoShape = { repo: z.string().min(1), }; +const ownerRepoWindowShape = { + owner: z.string().min(1), + repo: z.string().min(1), + windowDays: z.number().int().positive().optional(), +}; + const loginShape = { login: z.string().min(1), }; @@ -587,6 +594,16 @@ const freshnessResponseOutputSchema = { report: z.unknown().optional(), }; +const maintainerMeasurementReportOutputSchema = { + repoFullName: z.string().optional(), + generatedAt: z.string().optional(), + windowDays: z.number().nullable().optional(), + slop: z.unknown().optional(), + recommendations: z.unknown().optional(), + signals: z.array(z.string()).optional(), + status: z.string().optional(), +}; + const contributorProfileOutputSchema = { login: z.string().optional(), github: z.unknown().optional(), @@ -1026,6 +1043,17 @@ export class GittensoryMcp { async (input) => this.toolResult(await this.getRepoOutcomePatterns(input)), ); + server.registerTool( + "gittensory_get_outcome_calibration", + { + description: + "Return slop-band and recommendation outcome calibration for a repo: whether higher-slop bands merge less often and how agent recommendations are panning out. Maintainer-authenticated; measurement only.", + inputSchema: ownerRepoWindowShape, + outputSchema: maintainerMeasurementReportOutputSchema, + }, + async (input) => this.toolResult(await this.getOutcomeCalibration(input)), + ); + server.registerTool( "gittensory_get_contributor_profile", { @@ -1884,6 +1912,16 @@ export class GittensoryMcp { }; } + private async getOutcomeCalibration(input: { owner: string; repo: string; windowDays?: number | undefined }): Promise { + const fullName = `${input.owner}/${input.repo}`; + await this.requireRepoAccess(fullName); + const report = await buildRepoOutcomeCalibration(this.env, fullName, input.windowDays); + return { + summary: outcomeCalibrationSummary(fullName, report.slop), + data: report as unknown as Record, + }; + } + private async loadOpenQueueCounts(fullName: string): Promise<{ openIssues: number; openPullRequests: number }> { const [totals, openIssues, openPullRequests] = await Promise.all([ getLatestRepoGithubTotalsSnapshot(this.env, fullName), diff --git a/src/services/outcome-calibration.ts b/src/services/outcome-calibration.ts index 7681c0f803..1f13ea0a42 100644 --- a/src/services/outcome-calibration.ts +++ b/src/services/outcome-calibration.ts @@ -130,6 +130,13 @@ function sameRepo(a: string | null | undefined, b: string): boolean { return (a ?? "").toLowerCase() === b.toLowerCase(); } +/** One-line human summary of a repo's slop-band calibration verdict (mirrors the discriminates signal). Pure. */ +export function outcomeCalibrationSummary(fullName: string, slop: SlopOutcomeCalibration): string { + if (slop.discriminates === true) return `Outcome calibration for ${fullName}: slop bands are predictive across ${slop.totalResolved} resolved PRs.`; + if (slop.discriminates === false) return `Outcome calibration for ${fullName}: slop bands are NOT discriminating on current data (${slop.totalResolved} resolved PRs).`; + return `Outcome calibration for ${fullName}: not enough resolved PR data to judge slop calibration yet.`; +} + /** Load a repo's PRs + recommendation outcomes and assemble the calibration report. */ export async function buildRepoOutcomeCalibration(env: Env, repoFullName: string, windowDays?: number): Promise { const [pullRequests, outcomes] = await Promise.all([ diff --git a/test/integration/api.test.ts b/test/integration/api.test.ts index 7f2bf062a6..f91cad1263 100644 --- a/test/integration/api.test.ts +++ b/test/integration/api.test.ts @@ -4872,6 +4872,7 @@ describe("api routes", () => { expect(toolNames).toContain("gittensory_preflight_local_diff"); expect(toolNames).toContain("gittensory_preview_local_pr_score"); expect(toolNames).toContain("gittensory_explain_score_breakdown"); + expect(toolNames).toContain("gittensory_get_outcome_calibration"); expect(toolNames).toContain("gittensory_get_registry_changes"); expect(toolNames).toContain("gittensory_get_upstream_drift"); expect(toolNames).toContain("gittensory_explain_review_risk"); diff --git a/test/unit/mcp-output-schemas.test.ts b/test/unit/mcp-output-schemas.test.ts index 9c505bbd84..02eb3c9626 100644 --- a/test/unit/mcp-output-schemas.test.ts +++ b/test/unit/mcp-output-schemas.test.ts @@ -1,7 +1,7 @@ import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; import { describe, expect, it } from "vitest"; -import { persistSignalSnapshot, upsertBounty, upsertIssueFromGitHub, upsertRepositoryFromGitHub } from "../../src/db/repositories"; +import { persistSignalSnapshot, upsertBounty, upsertIssueFromGitHub, upsertPullRequestFromGitHub, upsertRepositoryFromGitHub, updatePullRequestSlopAssessment } from "../../src/db/repositories"; import { GittensoryMcp } from "../../src/mcp/server"; import { normalizeRegistryPayload } from "../../src/registry/normalize"; import { persistRegistrySnapshot } from "../../src/registry/sync"; @@ -13,6 +13,7 @@ const TOOLS_WITH_OUTPUT_SCHEMA = [ "gittensory_get_repo_context", "gittensory_get_burden_forecast", "gittensory_get_repo_outcome_patterns", + "gittensory_get_outcome_calibration", "gittensory_get_contributor_profile", "gittensory_get_decision_pack", "gittensory_monitor_open_prs", @@ -380,6 +381,31 @@ describe("MCP tool calls return schema-valid structured content", () => { expect(cached.isError).toBeFalsy(); expect(cached.structuredContent).toMatchObject({ status: "ready", source: "snapshot", freshness: "fresh", repoFullName: "owner/cached" }); }); + + it("gittensory_get_outcome_calibration returns structured slop calibration for a repo", async () => { + const env = createTestEnv(); + await upsertRepositoryFromGitHub(env, { name: "demo", full_name: "octo/demo", private: false, owner: { login: "octo" }, default_branch: "main" }); + await upsertPullRequestFromGitHub(env, "octo/demo", { + number: 1, + title: "merged clean", + state: "closed", + user: { login: "alice" }, + merged_at: "2026-06-01T00:00:00.000Z", + }); + await updatePullRequestSlopAssessment(env, "octo/demo", 1, { slopRisk: 0, slopBand: "clean" }); + const { client } = await connectTestClient(env); + const result = await client.callTool({ + name: "gittensory_get_outcome_calibration", + arguments: { owner: "octo", repo: "demo", windowDays: 30 }, + }); + expect(result.isError).toBeFalsy(); + const data = result.structuredContent as Record; + expect(data.repoFullName).toBe("octo/demo"); + expect(data.windowDays).toBe(30); + expect(data.slop).toBeTruthy(); + expect(data.recommendations).toBeTruthy(); + expect(Array.isArray(data.signals)).toBe(true); + }); }); // ── Public/private safety ───────────────────────────────────────────────────── diff --git a/test/unit/outcome-calibration.test.ts b/test/unit/outcome-calibration.test.ts index c901439a43..bebcb41d72 100644 --- a/test/unit/outcome-calibration.test.ts +++ b/test/unit/outcome-calibration.test.ts @@ -4,6 +4,8 @@ import { buildRecommendationOutcomeCalibration, buildRepoOutcomeCalibration, buildSlopOutcomeCalibration, + outcomeCalibrationSummary, + type SlopOutcomeCalibration, } from "../../src/services/outcome-calibration"; import { createAgentRun, replaceAgentActions, updatePullRequestSlopAssessment, upsertAgentRecommendationOutcome, upsertPullRequestFromGitHub } from "../../src/db/repositories"; import type { SlopBand } from "../../src/signals/slop"; @@ -216,3 +218,30 @@ function actionRecord(id: string, runId: string): AgentActionRecord { createdAt: "2026-06-01T00:00:00.000Z", }; } + +describe("outcomeCalibrationSummary", () => { + const slop = (discriminates: boolean | null, totalResolved: number): SlopOutcomeCalibration => ({ + totalResolved, + bands: [], + overallMergeRate: null, + discriminates, + }); + + it("reports a predictive verdict when bands discriminate", () => { + expect(outcomeCalibrationSummary("octo/demo", slop(true, 12))).toBe( + "Outcome calibration for octo/demo: slop bands are predictive across 12 resolved PRs.", + ); + }); + + it("reports a non-discriminating verdict when bands invert", () => { + expect(outcomeCalibrationSummary("octo/demo", slop(false, 11))).toBe( + "Outcome calibration for octo/demo: slop bands are NOT discriminating on current data (11 resolved PRs).", + ); + }); + + it("reports an insufficient-data verdict when discrimination cannot be judged", () => { + expect(outcomeCalibrationSummary("octo/demo", slop(null, 2))).toBe( + "Outcome calibration for octo/demo: not enough resolved PR data to judge slop calibration yet.", + ); + }); +});