Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions src/mcp/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ import { loadOrComputeIssueQualityResponse } from "../services/issue-quality";
import { loadOrComputeBurdenForecastResponse } from "../services/burden-forecast";
import { buildMcpClientTelemetry } from "../services/client-telemetry";
import { loadOrComputeRepoOutcomePatternsResponse } from "../services/repo-outcome-patterns";
import { buildRepoOutcomeCalibration, outcomeCalibrationSummary } from "../services/outcome-calibration";
import { buildUnavailableQueueTrendReport } from "../services/queue-trends";
import {
applyMcpPlanningChoices,
Expand Down Expand Up @@ -137,6 +138,12 @@ const ownerRepoShape = {
repo: z.string().min(1),
};

const ownerRepoWindowShape = {
owner: z.string().min(1),
repo: z.string().min(1),
windowDays: z.number().int().positive().optional(),
};

const loginShape = {
login: z.string().min(1),
};
Expand Down Expand Up @@ -587,6 +594,16 @@ const freshnessResponseOutputSchema = {
report: z.unknown().optional(),
};

const maintainerMeasurementReportOutputSchema = {
repoFullName: z.string().optional(),
generatedAt: z.string().optional(),
windowDays: z.number().nullable().optional(),
slop: z.unknown().optional(),
recommendations: z.unknown().optional(),
signals: z.array(z.string()).optional(),
status: z.string().optional(),
};

const contributorProfileOutputSchema = {
login: z.string().optional(),
github: z.unknown().optional(),
Expand Down Expand Up @@ -1026,6 +1043,17 @@ export class GittensoryMcp {
async (input) => this.toolResult(await this.getRepoOutcomePatterns(input)),
);

server.registerTool(
"gittensory_get_outcome_calibration",
{
description:
"Return slop-band and recommendation outcome calibration for a repo: whether higher-slop bands merge less often and how agent recommendations are panning out. Maintainer-authenticated; measurement only.",
inputSchema: ownerRepoWindowShape,
outputSchema: maintainerMeasurementReportOutputSchema,
},
async (input) => this.toolResult(await this.getOutcomeCalibration(input)),
);

server.registerTool(
"gittensory_get_contributor_profile",
{
Expand Down Expand Up @@ -1884,6 +1912,16 @@ export class GittensoryMcp {
};
}

private async getOutcomeCalibration(input: { owner: string; repo: string; windowDays?: number | undefined }): Promise<ToolPayload> {
const fullName = `${input.owner}/${input.repo}`;
await this.requireRepoAccess(fullName);
const report = await buildRepoOutcomeCalibration(this.env, fullName, input.windowDays);
return {
summary: outcomeCalibrationSummary(fullName, report.slop),
data: report as unknown as Record<string, unknown>,
};
}

private async loadOpenQueueCounts(fullName: string): Promise<{ openIssues: number; openPullRequests: number }> {
const [totals, openIssues, openPullRequests] = await Promise.all([
getLatestRepoGithubTotalsSnapshot(this.env, fullName),
Expand Down
7 changes: 7 additions & 0 deletions src/services/outcome-calibration.ts
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,13 @@ function sameRepo(a: string | null | undefined, b: string): boolean {
return (a ?? "").toLowerCase() === b.toLowerCase();
}

/** One-line human summary of a repo's slop-band calibration verdict (mirrors the discriminates signal). Pure. */
export function outcomeCalibrationSummary(fullName: string, slop: SlopOutcomeCalibration): string {
if (slop.discriminates === true) return `Outcome calibration for ${fullName}: slop bands are predictive across ${slop.totalResolved} resolved PRs.`;
if (slop.discriminates === false) return `Outcome calibration for ${fullName}: slop bands are NOT discriminating on current data (${slop.totalResolved} resolved PRs).`;
return `Outcome calibration for ${fullName}: not enough resolved PR data to judge slop calibration yet.`;
}

/** Load a repo's PRs + recommendation outcomes and assemble the calibration report. */
export async function buildRepoOutcomeCalibration(env: Env, repoFullName: string, windowDays?: number): Promise<OutcomeCalibration> {
const [pullRequests, outcomes] = await Promise.all([
Expand Down
1 change: 1 addition & 0 deletions test/integration/api.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4872,6 +4872,7 @@ describe("api routes", () => {
expect(toolNames).toContain("gittensory_preflight_local_diff");
expect(toolNames).toContain("gittensory_preview_local_pr_score");
expect(toolNames).toContain("gittensory_explain_score_breakdown");
expect(toolNames).toContain("gittensory_get_outcome_calibration");
expect(toolNames).toContain("gittensory_get_registry_changes");
expect(toolNames).toContain("gittensory_get_upstream_drift");
expect(toolNames).toContain("gittensory_explain_review_risk");
Expand Down
28 changes: 27 additions & 1 deletion test/unit/mcp-output-schemas.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
import { describe, expect, it } from "vitest";
import { persistSignalSnapshot, upsertBounty, upsertIssueFromGitHub, upsertRepositoryFromGitHub } from "../../src/db/repositories";
import { persistSignalSnapshot, upsertBounty, upsertIssueFromGitHub, upsertPullRequestFromGitHub, upsertRepositoryFromGitHub, updatePullRequestSlopAssessment } from "../../src/db/repositories";
import { GittensoryMcp } from "../../src/mcp/server";
import { normalizeRegistryPayload } from "../../src/registry/normalize";
import { persistRegistrySnapshot } from "../../src/registry/sync";
Expand All @@ -13,6 +13,7 @@ const TOOLS_WITH_OUTPUT_SCHEMA = [
"gittensory_get_repo_context",
"gittensory_get_burden_forecast",
"gittensory_get_repo_outcome_patterns",
"gittensory_get_outcome_calibration",
"gittensory_get_contributor_profile",
"gittensory_get_decision_pack",
"gittensory_monitor_open_prs",
Expand Down Expand Up @@ -380,6 +381,31 @@ describe("MCP tool calls return schema-valid structured content", () => {
expect(cached.isError).toBeFalsy();
expect(cached.structuredContent).toMatchObject({ status: "ready", source: "snapshot", freshness: "fresh", repoFullName: "owner/cached" });
});

it("gittensory_get_outcome_calibration returns structured slop calibration for a repo", async () => {
const env = createTestEnv();
await upsertRepositoryFromGitHub(env, { name: "demo", full_name: "octo/demo", private: false, owner: { login: "octo" }, default_branch: "main" });
await upsertPullRequestFromGitHub(env, "octo/demo", {
number: 1,
title: "merged clean",
state: "closed",
user: { login: "alice" },
merged_at: "2026-06-01T00:00:00.000Z",
});
await updatePullRequestSlopAssessment(env, "octo/demo", 1, { slopRisk: 0, slopBand: "clean" });
const { client } = await connectTestClient(env);
const result = await client.callTool({
name: "gittensory_get_outcome_calibration",
arguments: { owner: "octo", repo: "demo", windowDays: 30 },
});
expect(result.isError).toBeFalsy();
const data = result.structuredContent as Record<string, unknown>;
expect(data.repoFullName).toBe("octo/demo");
expect(data.windowDays).toBe(30);
expect(data.slop).toBeTruthy();
expect(data.recommendations).toBeTruthy();
expect(Array.isArray(data.signals)).toBe(true);
});
});

// ── Public/private safety ─────────────────────────────────────────────────────
Expand Down
29 changes: 29 additions & 0 deletions test/unit/outcome-calibration.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import {
buildRecommendationOutcomeCalibration,
buildRepoOutcomeCalibration,
buildSlopOutcomeCalibration,
outcomeCalibrationSummary,
type SlopOutcomeCalibration,
} from "../../src/services/outcome-calibration";
import { createAgentRun, replaceAgentActions, updatePullRequestSlopAssessment, upsertAgentRecommendationOutcome, upsertPullRequestFromGitHub } from "../../src/db/repositories";
import type { SlopBand } from "../../src/signals/slop";
Expand Down Expand Up @@ -216,3 +218,30 @@ function actionRecord(id: string, runId: string): AgentActionRecord {
createdAt: "2026-06-01T00:00:00.000Z",
};
}

describe("outcomeCalibrationSummary", () => {
const slop = (discriminates: boolean | null, totalResolved: number): SlopOutcomeCalibration => ({
totalResolved,
bands: [],
overallMergeRate: null,
discriminates,
});

it("reports a predictive verdict when bands discriminate", () => {
expect(outcomeCalibrationSummary("octo/demo", slop(true, 12))).toBe(
"Outcome calibration for octo/demo: slop bands are predictive across 12 resolved PRs.",
);
});

it("reports a non-discriminating verdict when bands invert", () => {
expect(outcomeCalibrationSummary("octo/demo", slop(false, 11))).toBe(
"Outcome calibration for octo/demo: slop bands are NOT discriminating on current data (11 resolved PRs).",
);
});

it("reports an insufficient-data verdict when discrimination cannot be judged", () => {
expect(outcomeCalibrationSummary("octo/demo", slop(null, 2))).toBe(
"Outcome calibration for octo/demo: not enough resolved PR data to judge slop calibration yet.",
);
});
});
Loading