From 39bac431cfa88d9babb8677ba7cc5bb4d433523e Mon Sep 17 00:00:00 2001 From: kiannidev <156195510+kiannidev@users.noreply.github.com> Date: Mon, 6 Jul 2026 10:13:48 +0200 Subject: [PATCH] feat(mcp): add gittensory_check_test_evidence tool (#2235) Expose the deterministic changed-path test-evidence classifier as a metadata-only MCP tool with coverage-gap guidance. Co-authored-by: Cursor --- src/mcp/check-test-evidence.ts | 62 ++++++++++++ src/mcp/server.ts | 35 +++++++ test/unit/mcp-check-test-evidence.test.ts | 109 ++++++++++++++++++++++ test/unit/mcp-output-schemas.test.ts | 1 + 4 files changed, 207 insertions(+) create mode 100644 src/mcp/check-test-evidence.ts create mode 100644 test/unit/mcp-check-test-evidence.test.ts diff --git a/src/mcp/check-test-evidence.ts b/src/mcp/check-test-evidence.ts new file mode 100644 index 0000000000..32290618cb --- /dev/null +++ b/src/mcp/check-test-evidence.ts @@ -0,0 +1,62 @@ +import { classifyTestCoverage, isTestPath, type TestCoverageClassification } from "../signals/test-evidence"; +import { isCodeFile } from "../signals/path-matchers"; + +export type CheckTestEvidenceInput = { + changedPaths: string[]; + testPaths?: string[] | undefined; +}; + +export type CheckTestEvidenceReport = { + classification: TestCoverageClassification; + codeFileCount: number; + testFileCount: number; + docsOnly: boolean; + guidance: string[]; + generatedAt: string; +}; + +const CLASSIFICATION_GUIDANCE: Record = { + absent: "Add focused regression tests for the changed code paths, or explain why existing coverage is sufficient.", + weak: "Some test files are present, but coverage is proportionally light — add more focused tests for the code you changed.", + adequate: "Test changes are proportionally adequate for the number of code files changed.", + strong: "Test changes are proportionally strong for the number of code files changed.", +}; + +function uniquePaths(paths: readonly string[]): string[] { + const seen = new Set(); + const normalized: string[] = []; + for (const path of paths) { + const trimmed = path.trim(); + if (!trimmed || seen.has(trimmed)) continue; + seen.add(trimmed); + normalized.push(trimmed); + } + return normalized; +} + +/** Deterministic coverage-gap report for MCP `gittensory_check_test_evidence` (#2235). Pure — paths only. */ +export function buildCheckTestEvidenceReport(input: CheckTestEvidenceInput): CheckTestEvidenceReport { + const changedPaths = uniquePaths(input.changedPaths ?? []); + const extraTestPaths = uniquePaths(input.testPaths ?? []); + const pathsForClassification = uniquePaths([...changedPaths, ...extraTestPaths]); + const codeFileCount = changedPaths.filter(isCodeFile).length; + const testFileCount = pathsForClassification.filter(isTestPath).length; + const docsOnly = codeFileCount === 0; + const classification: TestCoverageClassification = docsOnly ? "absent" : classifyTestCoverage(pathsForClassification); + + const guidance: string[] = []; + if (docsOnly) { + guidance.push("No code files changed — dedicated test evidence is not required for docs-only churn."); + } else { + guidance.push(CLASSIFICATION_GUIDANCE[classification]); + } + + return { + classification, + codeFileCount, + testFileCount, + docsOnly, + guidance, + generatedAt: new Date().toISOString(), + }; +} diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 149ef27761..12c9424880 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -124,6 +124,7 @@ import { buildPredictedGateVerdict } from "../rules/predicted-gate"; import { buildIssueSlopAssessment, buildSlopAssessment } from "../signals/slop"; import { buildRepoDataQuality } from "../signals/data-quality"; import { PREFLIGHT_LIMITS } from "../signals/preflight-limits"; +import { buildCheckTestEvidenceReport } from "./check-test-evidence"; import { SCENARIO_MAX_BRANCH_REF_CHARS, SCENARIO_MAX_LINKED_ISSUE_NUMBERS, SCENARIO_MAX_REPO_FULL_NAME_CHARS } from "../scenarios/input-model"; import { loadUpstreamStatus } from "../upstream/ruleset"; @@ -768,6 +769,20 @@ const checkIssueSlopShape = { const checkIssueSlopOutputSchema = checkSlopRiskOutputSchema; +const checkTestEvidenceShape = { + changedPaths: z.array(z.string().min(1).max(PREFLIGHT_LIMITS.changedFileChars)).max(PREFLIGHT_LIMITS.changedFiles), + testPaths: z.array(z.string().min(1).max(PREFLIGHT_LIMITS.changedFileChars)).max(PREFLIGHT_LIMITS.changedFiles).optional(), +}; + +const checkTestEvidenceOutputSchema = { + classification: z.enum(["strong", "adequate", "weak", "absent"]).optional(), + codeFileCount: z.number().int().min(0).optional(), + testFileCount: z.number().int().min(0).optional(), + docsOnly: z.boolean().optional(), + guidance: z.array(z.string()).optional(), + generatedAt: z.string().optional(), +}; + const predictGateOutputSchema = { predicted: z.boolean().optional(), basis: z.string().optional(), @@ -1234,6 +1249,17 @@ export class GittensoryMcp { async (input) => this.toolResult(await this.checkIssueSlop(input)), ); + server.registerTool( + "gittensory_check_test_evidence", + { + description: + "Classify whether a planned change carries enough accompanying test evidence from changed paths and optional test-file paths alone — metadata-only, no source upload, no GitHub writes. Returns a coverage-gap band and actionable guidance.", + inputSchema: checkTestEvidenceShape, + outputSchema: checkTestEvidenceOutputSchema, + }, + async (input) => this.toolResult(await this.checkTestEvidence(input)), + ); + server.registerTool( "gittensory_pr_outcome", { @@ -2223,6 +2249,15 @@ export class GittensoryMcp { }; } + private async checkTestEvidence(input: z.infer>): Promise { + await this.enforceToolRateLimit("gittensory_check_test_evidence"); + const report = buildCheckTestEvidenceReport(input); + return { + summary: `Test-evidence classification: ${report.classification}.`, + data: report as unknown as Record, + }; + } + private async predictGate(input: z.infer>): Promise { this.requireContributorAccess(input.login); const repoFullName = `${input.owner}/${input.repo}`; diff --git a/test/unit/mcp-check-test-evidence.test.ts b/test/unit/mcp-check-test-evidence.test.ts new file mode 100644 index 0000000000..041dccae02 --- /dev/null +++ b/test/unit/mcp-check-test-evidence.test.ts @@ -0,0 +1,109 @@ +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import { describe, expect, it } from "vitest"; +import { buildCheckTestEvidenceReport } from "../../src/mcp/check-test-evidence"; +import { GittensoryMcp } from "../../src/mcp/server"; +import { createTestEnv } from "../helpers/d1"; + +async function connect() { + const server = new GittensoryMcp(createTestEnv()).createServer(); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + const client = new Client({ name: "gittensory-test-evidence-test", version: "0.1.0" }, { capabilities: {} }); + await client.connect(clientTransport); + return client; +} + +describe("buildCheckTestEvidenceReport (#2235)", () => { + it("classifies code-only changes without tests as absent", () => { + const report = buildCheckTestEvidenceReport({ changedPaths: ["src/auth.ts", "src/utils.ts"] }); + expect(report.classification).toBe("absent"); + expect(report.codeFileCount).toBe(2); + expect(report.testFileCount).toBe(0); + expect(report.docsOnly).toBe(false); + expect(report.guidance[0]).toMatch(/Add focused regression tests/); + }); + + it("classifies code plus proportionally strong tests as strong", () => { + const report = buildCheckTestEvidenceReport({ + changedPaths: ["src/a.ts", "src/b.ts", "test/a.test.ts", "test/b.test.ts"], + }); + expect(report.classification).toBe("strong"); + expect(report.testFileCount).toBe(2); + }); + + it("classifies adequate and weak threshold bands", () => { + const adequate = buildCheckTestEvidenceReport({ + changedPaths: ["src/a.ts", "src/b.ts", "src/c.ts", "test/a.test.ts"], + }); + expect(adequate.classification).toBe("adequate"); + + const weak = buildCheckTestEvidenceReport({ + changedPaths: [...Array.from({ length: 9 }, (_, i) => `src/file${i}.ts`), "test/single.test.ts"], + }); + expect(weak.classification).toBe("weak"); + }); + + it("treats docs-only churn as not requiring dedicated test evidence", () => { + const report = buildCheckTestEvidenceReport({ changedPaths: ["README.md", "docs/guide.md"] }); + expect(report.docsOnly).toBe(true); + expect(report.codeFileCount).toBe(0); + expect(report.classification).toBe("absent"); + expect(report.guidance[0]).toMatch(/docs-only churn/); + }); + + it("counts optional testPaths supplied separately from changedPaths", () => { + const report = buildCheckTestEvidenceReport({ + changedPaths: ["src/a.ts", "src/b.ts", "src/c.ts"], + testPaths: ["test/a.test.ts"], + }); + expect(report.classification).toBe("adequate"); + expect(report.testFileCount).toBe(1); + }); +}); + +describe("MCP gittensory_check_test_evidence (#2235)", () => { + it("registers the tool and returns a public-safe classification for code-only paths", async () => { + const client = await connect(); + const { tools } = await client.listTools(); + expect(tools.map((tool) => tool.name)).toContain("gittensory_check_test_evidence"); + + const result = await client.callTool({ + name: "gittensory_check_test_evidence", + arguments: { changedPaths: ["src/api/routes.ts"] }, + }); + expect(result.isError).toBeFalsy(); + const data = result.structuredContent as { + classification: string; + guidance: string[]; + docsOnly: boolean; + }; + expect(data.classification).toBe("absent"); + expect(data.docsOnly).toBe(false); + expect(data.guidance.length).toBeGreaterThan(0); + expect(JSON.stringify(data)).not.toMatch(/wallet|hotkey|reward|payout|trust score/i); + }); + + it("returns strong classification when code and tests are supplied together", async () => { + const client = await connect(); + const result = await client.callTool({ + name: "gittensory_check_test_evidence", + arguments: { + changedPaths: ["src/a.ts", "src/b.ts", "test/a.test.ts", "test/b.test.ts"], + }, + }); + const data = result.structuredContent as { classification: string }; + expect(data.classification).toBe("strong"); + }); + + it("returns docs-only guidance without requiring tests", async () => { + const client = await connect(); + const result = await client.callTool({ + name: "gittensory_check_test_evidence", + arguments: { changedPaths: ["docs/miner-goal-spec.md"] }, + }); + const data = result.structuredContent as { docsOnly: boolean; guidance: string[] }; + expect(data.docsOnly).toBe(true); + expect(data.guidance[0]).toMatch(/docs-only churn/); + }); +}); diff --git a/test/unit/mcp-output-schemas.test.ts b/test/unit/mcp-output-schemas.test.ts index b8895672bb..51b75af823 100644 --- a/test/unit/mcp-output-schemas.test.ts +++ b/test/unit/mcp-output-schemas.test.ts @@ -26,6 +26,7 @@ const TOOLS_WITH_OUTPUT_SCHEMA = [ "gittensory_validate_linked_issue", "gittensory_check_before_start", "gittensory_lint_pr_text", + "gittensory_check_test_evidence", "gittensory_get_registry_changes", "gittensory_get_upstream_drift", "gittensory_local_status",