From a0a3d9b3f357a1ce43783b9a2f4cae97e31b9433 Mon Sep 17 00:00:00 2001 From: Jeff <158072326+jeffrey701@users.noreply.github.com> Date: Fri, 10 Jul 2026 09:53:16 -0400 Subject: [PATCH] feat(agent): auto-apply high-confidence project/milestone matches Wire the "auto" branch of autoProjectMilestoneMatch (the tri-state config already shipped and parsed by #3183/#3184) to actually attach the matched milestone/project instead of only commenting (#3185). maybeSuggestMilestoneMatchForPr now routes mode "auto" to a new maybeAutoApplyProjectOrMilestoneMatch, which resolves matches against the repo's configured backend (GitHub by default, Linear when opted in) and attaches whichever milestone/project clears a confidence threshold, via the very same adapters suggest mode uses. Auto-apply is deliberately more conservative than suggest mode (DEFAULT_AUTO_APPLY_MIN_SCORE 0.85 vs the 0.65 suggest floor): a wrong auto-attach silently mislabels a PR, whereas a wrong suggestion is only an advisory comment a maintainer can ignore. A "native" confirmed link (score 1, e.g. Linear's GitHub integration) always clears it; a below-threshold match is left untouched. The threshold is a parameter (defaulting to the constant) so it can be tuned to a repo's observed suggest-mode false-positive rate. Best-effort: an attach failure is logged (milestone_auto_apply_failed) and swallowed by the webhook entry point, never blocking the maintenance step -- the same fail-open contract suggest mode already uses. Attaching is idempotent, so a repeated sweep never double-applies. Covered by test/unit/project-tracker-adapter.test.ts: attach on a threshold-clearing match, no-attach below the default threshold, a lowered-threshold override attaching the same match, a Projects v2 GraphQL attach, an inert Linear backend, the "auto" route attaching instead of commenting, and a failing attach logged-and-swallowed. Closes #3185 --- src/integrations/project-tracker-adapter.ts | 66 +++++++- test/unit/project-tracker-adapter.test.ts | 170 ++++++++++++++++++++ 2 files changed, 235 insertions(+), 1 deletion(-) diff --git a/src/integrations/project-tracker-adapter.ts b/src/integrations/project-tracker-adapter.ts index 5bf9fb6f86..3d6f2a6e23 100644 --- a/src/integrations/project-tracker-adapter.ts +++ b/src/integrations/project-tracker-adapter.ts @@ -255,6 +255,13 @@ export class GitHubProjectsAdapter implements ProjectTrackerAdapter { const TRACKER_MATCH_MIN_SCORE = 0.65; const TRACKER_MATCH_MIN_SHARED = 3; +// Auto-apply (#3185) uses a deliberately higher confidence bar than the suggest-mode floor above: a wrong +// auto-attach silently mislabels a PR, whereas a wrong suggestion is only an advisory comment a maintainer can +// ignore. Only a match at or above this title/body term-overlap score is attached automatically; a "native" +// confirmed link (score 1, e.g. Linear's own GitHub integration) always clears it. A repo can tighten this per +// its observed suggest-mode false-positive rate via the `threshold` argument. +export const DEFAULT_AUTO_APPLY_MIN_SCORE = 0.85; + export type ProjectTrackerMatch = { item: ProjectTrackerRef; // "native" (#3186): a CONFIRMED link (e.g. Linear's own GitHub integration already linked this PR), not a @@ -405,6 +412,45 @@ export async function maybeSuggestProjectOrMilestoneMatch( return { suggested: true }; } +export type ProjectMilestoneAutoApplyResult = { + attachedMilestone: boolean; + attachedProject: boolean; +}; + +/** + * Auto-apply mode (#3185): resolve matches against the repo's configured backend and ACTUALLY attach whichever + * milestone/project clears `threshold` (default {@link DEFAULT_AUTO_APPLY_MIN_SCORE}) -- via the very same + * adapters suggest mode uses -- instead of only commenting. Attaching is idempotent (re-PATCHing the same + * milestone / re-adding the same Projects v2 item is a no-op), so a repeated maintenance/webhook sweep never + * double-applies or spams. A below-threshold match is deliberately left untouched -- guessing wrong in auto mode + * silently mislabels a PR, whereas a wrong suggestion is only an advisory comment. This can THROW on a tracker + * API error; the webhook entry point {@link maybeSuggestMilestoneMatchForPr} runs it best-effort so an attach + * failure is logged and swallowed rather than breaking the maintenance step. + */ +export async function maybeAutoApplyProjectOrMilestoneMatch( + ctx: ProjectTrackerContext, + pullNumber: number, + prTitle: string, + prBody: string | null | undefined, + backend: ProjectMilestoneMatchBackendInput, + prUrl: string, + threshold: number = DEFAULT_AUTO_APPLY_MIN_SCORE, +): Promise { + const matches = await resolveTrackerMatches(ctx, backend, prTitle, prBody, prUrl); + const isLinear = backend === "linear"; + const milestoneAdapter: ProjectTrackerAdapter = isLinear ? new LinearAdapter() : new GitHubMilestonesAdapter(); + const projectAdapter: ProjectTrackerAdapter = isLinear ? new LinearAdapter() : new GitHubProjectsAdapter(); + let attachedMilestone = false; + let attachedProject = false; + if (matches.milestone && matches.milestone.score >= threshold) { + attachedMilestone = (await milestoneAdapter.attachToMilestone(ctx, pullNumber, matches.milestone.item.id)).attached; + } + if (matches.project && matches.project.score >= threshold) { + attachedProject = (await projectAdapter.attachToProject(ctx, pullNumber, matches.project.item.id)).attached; + } + return { attachedMilestone, attachedProject }; +} + /** * Webhook-level entry point (#3183): folds the "should this even run" gating (installed app, PR still open, * feature opted in, and a PR lifecycle/title-body webhook) AND the best-effort error logging into one call, so @@ -432,8 +478,26 @@ export async function maybeSuggestMilestoneMatchForPr(args: { if (!args.installationId) return; if (args.prState !== "open") return; if (!args.mode || args.mode === "off") return; + const ctx = { env: args.env, installationId: args.installationId, repoFullName: args.repoFullName }; + if (args.mode === "auto") { + // "auto": actually attach the high-confidence match(es) instead of only commenting (#3185). Best-effort -- + // an attach failure is logged and swallowed, never blocking the maintenance step, same as suggest mode. + await maybeAutoApplyProjectOrMilestoneMatch(ctx, args.pullNumber, args.prTitle, args.prBody, args.backend, args.prUrl ?? "").catch((error) => { + console.error( + JSON.stringify({ + level: "warn", + event: "milestone_auto_apply_failed", + deliveryId: args.deliveryId, + repoFullName: args.repoFullName, + pullNumber: args.pullNumber, + error: errorMessage(error), + }), + ); + }); + return; + } await maybeSuggestProjectOrMilestoneMatch( - { env: args.env, installationId: args.installationId, repoFullName: args.repoFullName }, + ctx, args.pullNumber, args.prTitle, args.prBody, diff --git a/test/unit/project-tracker-adapter.test.ts b/test/unit/project-tracker-adapter.test.ts index dda2006352..30acd2d9af 100644 --- a/test/unit/project-tracker-adapter.test.ts +++ b/test/unit/project-tracker-adapter.test.ts @@ -1,9 +1,11 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { generateKeyPairSync } from "node:crypto"; import { + DEFAULT_AUTO_APPLY_MIN_SCORE, GitHubMilestonesAdapter, GitHubProjectsAdapter, PROJECT_TRACKER_SUGGEST_COMMENT_MARKER, + maybeAutoApplyProjectOrMilestoneMatch, maybeSuggestMilestoneMatchForPr, maybeSuggestProjectOrMilestoneMatch, matchOpenTrackerItems, @@ -883,3 +885,171 @@ describe("maybeSuggestMilestoneMatchForPr (#3183 webhook-level gating)", () => { consoleError.mockRestore(); }); }); + +describe("maybeAutoApplyProjectOrMilestoneMatch (#3185)", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + const MILESTONE_TITLE = "database migration rollback safety checklist"; + const STRONG_TITLE = "database migration rollback safety"; // scores 1.0 against MILESTONE_TITLE + const WEAK_TITLE = "database migration rollback tooling"; // scores 0.75: clears the 0.65 suggest floor, below the 0.85 auto default + const PR_URL = "https://github.com/JSONbored/gittensory/pull/4"; + + const ctx = () => ({ + env: createTestEnv({ GITHUB_APP_PRIVATE_KEY: generateRsaPrivateKeyPem(), GITHUB_APP_SLUG: "gittensory" }), + installationId: 123, + repoFullName: "JSONbored/gittensory", + }); + + type MilestoneAttachRecord = { patchedMilestone?: number | undefined; patchCalled: boolean }; + function milestoneAttachFetch(record: MilestoneAttachRecord, opts: { patchStatus?: number } = {}) { + return async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/milestones")) return Response.json([{ number: 20, title: MILESTONE_TITLE }]); + if (url.endsWith("/graphql")) return Response.json(noOpenProjectsGraphQlBody()); + if (url.includes("/issues/4") && method === "PATCH") { + record.patchCalled = true; + if (opts.patchStatus && opts.patchStatus >= 400) return new Response("boom", { status: opts.patchStatus }); + record.patchedMilestone = (JSON.parse(String(init?.body ?? "{}")) as { milestone?: number }).milestone; + return Response.json({ number: 4, milestone: { number: 20 } }); + } + return new Response("unexpected", { status: 500 }); + }; + } + + it("attaches a milestone that clears the default confidence threshold", async () => { + const record: MilestoneAttachRecord = { patchCalled: false }; + vi.stubGlobal("fetch", milestoneAttachFetch(record)); + const result = await maybeAutoApplyProjectOrMilestoneMatch(ctx(), 4, STRONG_TITLE, null, "github", PR_URL); + expect(result).toEqual({ attachedMilestone: true, attachedProject: false }); + expect(record.patchedMilestone).toBe(20); + }); + + it("does NOT attach a match below the default threshold (a 0.75 fuzzy match is suggest-worthy, not auto-apply-worthy)", async () => { + const record: MilestoneAttachRecord = { patchCalled: false }; + vi.stubGlobal("fetch", milestoneAttachFetch(record)); + const result = await maybeAutoApplyProjectOrMilestoneMatch(ctx(), 4, WEAK_TITLE, null, "github", PR_URL); + expect(result).toEqual({ attachedMilestone: false, attachedProject: false }); + expect(record.patchCalled).toBe(false); + }); + + it("honors a lowered threshold override: the same 0.75 match attaches once the bar drops below its score", async () => { + const record: MilestoneAttachRecord = { patchCalled: false }; + vi.stubGlobal("fetch", milestoneAttachFetch(record)); + const result = await maybeAutoApplyProjectOrMilestoneMatch(ctx(), 4, WEAK_TITLE, null, "github", PR_URL, 0.7); + expect(result).toEqual({ attachedMilestone: true, attachedProject: false }); + expect(record.patchedMilestone).toBe(20); + }); + + it("attaches a matching Projects v2 item via GraphQL when it clears the threshold", async () => { + let mutationVariables: unknown; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/milestones")) return Response.json([]); + if (url.includes("/pulls/4") && method === "GET") return Response.json({ number: 4, node_id: "PR_kwABC" }); + if (url.endsWith("/graphql")) { + const body = JSON.parse(String(init?.body ?? "{}")) as { query?: string; variables?: unknown }; + if (body.query?.includes("addProjectV2ItemById")) { + mutationVariables = body.variables; + return Response.json({ data: { addProjectV2ItemById: { item: { id: "PVTI_x" } } } }); + } + return Response.json({ + data: { repositoryOwner: { __typename: "Organization", projectsV2: { nodes: [{ id: "PVT_1", title: MILESTONE_TITLE, closed: false, public: true }], pageInfo: { hasNextPage: false, endCursor: null } } } }, + }); + } + return new Response("unexpected", { status: 500 }); + }); + const result = await maybeAutoApplyProjectOrMilestoneMatch(ctx(), 4, STRONG_TITLE, null, "github", PR_URL); + expect(result).toEqual({ attachedMilestone: false, attachedProject: true }); + expect(mutationVariables).toEqual({ projectId: "PVT_1", contentId: "PR_kwABC" }); + }); + + it("attaches nothing for a Linear backend, whose attach is inert (best-effort, no throw)", async () => { + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.endsWith("/graphql")) return Response.json({ data: { viewer: { organization: null }, organization: null } }); + return new Response("[]", { status: 200 }); + }); + const result = await maybeAutoApplyProjectOrMilestoneMatch(ctx(), 4, STRONG_TITLE, null, "linear", PR_URL); + expect(result).toEqual({ attachedMilestone: false, attachedProject: false }); + }); + + it('routes mode "auto" through maybeSuggestMilestoneMatchForPr to an attach, never a suggestion comment', async () => { + let patchedMilestone: number | undefined; + let commentPosted = false; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/milestones")) return Response.json([{ number: 20, title: MILESTONE_TITLE }]); + if (url.endsWith("/graphql")) return Response.json(noOpenProjectsGraphQlBody()); + if (url.includes("/issues/4/comments") && method === "POST") { + commentPosted = true; + return Response.json({ id: 1 }); + } + if (url.includes("/issues/4/comments") && method === "GET") return Response.json([]); + if (url.includes("/issues/4") && method === "PATCH") { + patchedMilestone = (JSON.parse(String(init?.body ?? "{}")) as { milestone?: number }).milestone; + return Response.json({ number: 4 }); + } + return new Response("unexpected", { status: 500 }); + }); + await expect( + maybeSuggestMilestoneMatchForPr({ + env: createTestEnv({ GITHUB_APP_PRIVATE_KEY: generateRsaPrivateKeyPem(), GITHUB_APP_SLUG: "gittensory" }), + installationId: 123, + repoFullName: "JSONbored/gittensory", + pullNumber: 4, + prState: "open", + prTitle: STRONG_TITLE, + prBody: null, + prUrl: null, // GitHub backend ignores prUrl (only Linear's native-link path uses it); also covers the prUrl ?? "" fallback + mode: "auto", + backend: "github", + deliveryId: "d1", + eventName: "pull_request", + action: "opened", + }), + ).resolves.toBeUndefined(); + expect(patchedMilestone).toBe(20); + expect(commentPosted).toBe(false); + }); + + it('is best-effort in "auto" mode: a failing attach is logged and swallowed, never blocking the maintenance step', async () => { + const record: MilestoneAttachRecord = { patchCalled: false }; + vi.stubGlobal("fetch", milestoneAttachFetch(record, { patchStatus: 500 })); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => undefined); + await expect( + maybeSuggestMilestoneMatchForPr({ + env: createTestEnv({ GITHUB_APP_PRIVATE_KEY: generateRsaPrivateKeyPem(), GITHUB_APP_SLUG: "gittensory" }), + installationId: 123, + repoFullName: "JSONbored/gittensory", + pullNumber: 4, + prState: "open", + prTitle: STRONG_TITLE, + prBody: null, + prUrl: PR_URL, + mode: "auto", + backend: "github", + deliveryId: "delivery-99", + eventName: "pull_request", + action: "opened", + }), + ).resolves.toBeUndefined(); + expect(record.patchCalled).toBe(true); + expect(consoleError).toHaveBeenCalledTimes(1); + expect(JSON.parse(String(consoleError.mock.calls[0]?.[0]))).toMatchObject({ event: "milestone_auto_apply_failed", deliveryId: "delivery-99" }); + consoleError.mockRestore(); + }); + + it("keeps the auto-apply confidence bar above the suggest-mode floor and within [0, 1]", () => { + expect(DEFAULT_AUTO_APPLY_MIN_SCORE).toBeGreaterThan(0.65); + expect(DEFAULT_AUTO_APPLY_MIN_SCORE).toBeLessThanOrEqual(1); + }); +});