From b747f680f7fa6d4f3c8fb24f1dfe6dae244e2609 Mon Sep 17 00:00:00 2001 From: bittoby <218712309+bittoby@users.noreply.github.com> Date: Fri, 29 May 2026 04:32:46 +0000 Subject: [PATCH] feat(maintainers): add settings preview diagnostics --- CHANGELOG.md | 2 + src/api/routes.ts | 52 ++++++ src/openapi/schemas.ts | 65 +++++++ src/openapi/spec.ts | 10 + src/queue/processors.ts | 56 +++--- src/signals/settings-preview.ts | 285 +++++++++++++++++++++++++++++ test/integration/api.test.ts | 74 +++++++- test/unit/openapi.test.ts | 2 + test/unit/settings-preview.test.ts | 203 ++++++++++++++++++++ 9 files changed, 718 insertions(+), 31 deletions(-) create mode 100644 src/signals/settings-preview.ts create mode 100644 test/unit/settings-preview.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index d2e2755b98..eace4e4b8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,8 @@ - Add deterministic base-agent orchestrator (#14) +- Add settings preview diagnostics + ### Fixes diff --git a/src/api/routes.ts b/src/api/routes.ts index c2a5f9cadf..de6eeb27c7 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -101,6 +101,7 @@ import { import { attachDataQuality, buildCoreSignalFidelity, buildRepoDataQuality, buildSignalFidelity } from "../signals/data-quality"; import { buildPullRequestReviewability } from "../signals/reward-risk"; import { buildLocalBranchAnalysis } from "../signals/local-branch"; +import { buildRepoSettingsPreview } from "../signals/settings-preview"; import type { ContributorEvidenceRecord, JobMessage, JsonValue, RepoSyncSegmentRecord } from "../types"; import { errorMessage, nowIso } from "../utils/json"; @@ -254,6 +255,21 @@ const repositorySettingsSchema = z.object({ privateTrustEnabled: z.boolean().default(true), }); +const settingsPreviewSchema = z.object({ + sample: z + .object({ + authorLogin: z.string().trim().min(1).max(100).optional(), + authorType: z.enum(["User", "Bot"]).optional(), + authorAssociation: z.enum(["OWNER", "MEMBER", "COLLABORATOR", "CONTRIBUTOR", "FIRST_TIMER", "FIRST_TIME_CONTRIBUTOR", "MANNEQUIN", "NONE"]).optional(), + minerStatus: z.enum(["confirmed", "not_found", "unavailable"]).optional(), + title: z.string().max(300).optional(), + body: z.string().max(10000).nullable().optional(), + labels: z.array(z.string().max(100)).max(50).optional(), + linkedIssues: z.array(z.number().int().positive()).max(50).optional(), + }) + .optional(), +}); + export function createApp() { const app = new Hono(); app.use( @@ -558,6 +574,42 @@ export function createApp() { return c.json(await getRepositorySettings(c.env, fullName)); }); + app.post("/v1/repos/:owner/:repo/settings-preview", async (c) => { + const fullName = `${c.req.param("owner")}/${c.req.param("repo")}`; + const body = (await c.req.json().catch(() => null)) ?? {}; + const parsed = settingsPreviewSchema.safeParse(body); + if (!parsed.success) return c.json({ error: "invalid_settings_preview_request", issues: parsed.error.issues }, 400); + const [repo, settings, issues, pullRequests] = await Promise.all([ + getRepository(c.env, fullName), + getRepositorySettings(c.env, fullName), + listIssues(c.env, fullName), + listPullRequests(c.env, fullName), + ]); + const installationId = repo?.installationId ?? null; + const healthRecord = installationId !== null ? await getInstallationHealth(c.env, installationId) : null; + const enriched = healthRecord ? enrichInstallationHealth(healthRecord) : null; + const installation = enriched + ? { + installationId: enriched.installationId, + status: enriched.status, + missingPermissions: enriched.missingPermissions, + missingEvents: enriched.missingEvents, + permissionRemediation: enriched.permissionRemediation, + } + : null; + return c.json( + buildRepoSettingsPreview({ + repoFullName: fullName, + repo, + settings, + installation, + issues, + pullRequests, + sample: parsed.data.sample ?? {}, + }), + ); + }); + app.get("/v1/repos/:owner/:repo/pulls/:number/maintainer-packet", async (c) => { const fullName = `${c.req.param("owner")}/${c.req.param("repo")}`; const number = Number(c.req.param("number")); diff --git a/src/openapi/schemas.ts b/src/openapi/schemas.ts index 469e67fd47..f03f27711b 100644 --- a/src/openapi/schemas.ts +++ b/src/openapi/schemas.ts @@ -449,6 +449,71 @@ export const RepositorySettingsSchema = z }) .openapi("RepositorySettings"); +export const RepoSettingsPreviewSchema = z + .object({ + repoFullName: z.string(), + generatedAt: z.string(), + settings: z.object({ + publicSurface: z.enum(["off", "comment_and_label", "comment_only", "label_only"]), + commentMode: z.enum(["off", "detected_contributors_only", "all_prs"]), + publicSignalLevel: z.enum(["minimal", "standard"]), + checkRunMode: z.enum(["off", "enabled"]), + checkRunDetailLevel: z.enum(["minimal", "standard", "deep"]), + autoLabelEnabled: z.boolean(), + gittensorLabel: z.string(), + createMissingLabel: z.boolean(), + includeMaintainerAuthors: z.boolean(), + requireLinkedIssue: z.boolean(), + }), + installation: z + .object({ + installationId: z.number(), + status: z.enum(["healthy", "needs_attention", "broken"]), + missingPermissions: z.array(z.string()), + missingEvents: z.array(z.string()), + permissionRemediation: z.array( + z.object({ + permission: z.string(), + requiredAccess: z.string(), + currentAccess: z.string(), + ok: z.boolean(), + action: z.string(), + }), + ), + }) + .nullable(), + sample: z.object({ + authorLogin: z.string(), + authorType: z.string(), + authorAssociation: z.string(), + minerStatus: z.enum(["confirmed", "not_found", "unavailable"]), + title: z.string(), + labels: z.array(z.string()), + linkedIssues: z.array(z.number()), + }), + decision: z.object({ + willComment: z.boolean(), + willLabel: z.boolean(), + willCheckRun: z.boolean(), + skipped: z.boolean(), + skipReason: z.enum(["surface_off", "missing_author", "bot_author", "maintainer_author", "miner_detection_unavailable", "not_official_gittensor_miner"]).nullable(), + actions: z.array(z.enum(["skip", "comment", "label", "check_run", "none"])), + summary: z.string(), + }), + previewComment: z.string().nullable(), + appliedLabel: z.string().nullable(), + checkRun: z + .object({ + willCreate: z.boolean(), + title: z.string(), + detailLevel: z.enum(["minimal", "standard", "deep"]), + }) + .nullable(), + warnings: z.array(z.string()), + summary: z.string(), + }) + .openapi("RepoSettingsPreview"); + export const RepoSyncStateSchema = z .object({ repoFullName: z.string(), diff --git a/src/openapi/spec.ts b/src/openapi/spec.ts index cdd74d7619..0dd578e5d6 100644 --- a/src/openapi/spec.ts +++ b/src/openapi/spec.ts @@ -51,6 +51,7 @@ import { GitHubRateLimitObservationSchema, RepoSyncSegmentSchema, RepoSyncStateSchema, + RepoSettingsPreviewSchema, RepositorySchema, RepositorySettingsSchema, RoleContextSchema, @@ -99,6 +100,7 @@ export function buildOpenApiSpec() { registry.register("Bounty", BountySchema); registry.register("BountyAdvisory", BountyAdvisorySchema); registry.register("RepositorySettings", RepositorySettingsSchema); + registry.register("RepoSettingsPreview", RepoSettingsPreviewSchema); registry.register("AgentRun", AgentRunSchema); registry.register("AgentAction", AgentActionSchema); registry.register("AgentContextSnapshot", AgentContextSnapshotSchema); @@ -242,6 +244,14 @@ export function buildOpenApiSpec() { 200: { description: "Gittensory repository automation settings", content: { "application/json": { schema: RepositorySettingsSchema } } }, }, }); + registry.registerPath({ + method: "post", + path: "/v1/repos/{owner}/{repo}/settings-preview", + responses: { + 200: { description: "Maintainer dry-run preview of the public surface decision for a sample PR (no GitHub mutation)", content: { "application/json": { schema: RepoSettingsPreviewSchema } } }, + 400: { description: "Invalid settings preview request" }, + }, + }); registry.registerPath({ method: "get", path: "/v1/repos/{owner}/{repo}/pulls/{number}/maintainer-packet", diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 03d688d0d9..c655f36aab 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -76,6 +76,7 @@ import { buildQueueHealth, detectGittensorContributor, } from "../signals/engine"; +import { decidePublicSurface } from "../signals/settings-preview"; import type { ContributorEvidenceRecord, GitHubWebhookPayload, JobMessage, JsonValue } from "../types"; import { errorMessage } from "../utils/json"; @@ -520,20 +521,21 @@ async function maybePublishPrPublicSurface( advisory: Awaited>, webhook: { deliveryId: string; authorType?: string | undefined }, ): Promise { - if (!hasVisiblePrSurface(settings)) return; - const author = pr.authorLogin; - if (!author) { - await auditPrVisibilitySkip(env, repoFullName, pr.number, null, "missing_author", webhook.deliveryId); - return; - } - if (webhook.authorType === "Bot" || /\[bot\]$/i.test(author)) { - await auditPrVisibilitySkip(env, repoFullName, pr.number, author, "bot_author", webhook.deliveryId); - return; - } - if (!settings.includeMaintainerAuthors && pr.authorAssociation && ["OWNER", "MEMBER", "COLLABORATOR"].includes(pr.authorAssociation)) { - await auditPrVisibilitySkip(env, repoFullName, pr.number, author, "maintainer_author", webhook.deliveryId); + const author = pr.authorLogin ?? null; + // Cheap, network-free skip checks (also avoids the miner lookup when it would be wasted). + const prelim = decidePublicSurface({ + settings, + authorLogin: author, + authorType: webhook.authorType ?? null, + authorAssociation: pr.authorAssociation ?? null, + minerStatus: "not_checked", + }); + if (prelim.skipped) { + if (prelim.skipReason === "surface_off") return; + await auditPrVisibilitySkip(env, repoFullName, pr.number, author, prelim.skipReason ?? "skipped", webhook.deliveryId); return; } + if (!author) return; const official = await fetchOfficialGittensorMiner(author); if (official.status === "unavailable") { @@ -547,10 +549,17 @@ async function maybePublishPrPublicSurface( }); return; } - if (official.status === "not_found") { + if (official.status !== "confirmed") { await auditPrVisibilitySkip(env, repoFullName, pr.number, author, "not_official_gittensor_miner", webhook.deliveryId); return; } + const decision = decidePublicSurface({ + settings, + authorLogin: author, + authorType: webhook.authorType ?? null, + authorAssociation: pr.authorAssociation ?? null, + minerStatus: "confirmed", + }); const [contributorPullRequests, contributorIssues, repoIssues, repoPullRequests, github, cachedRepoStats] = await Promise.all([ listContributorPullRequests(env, author), @@ -580,7 +589,7 @@ async function maybePublishPrPublicSurface( repoIssues, repoPullRequests, ); - if (shouldPublishPrComment(settings)) { + if (decision.willComment) { const body = buildPublicPrIntelligenceComment({ repo, pr, @@ -593,12 +602,12 @@ async function maybePublishPrPublicSurface( }); await createOrUpdatePrIntelligenceComment(env, installationId, repoFullName, pr.number, body); } - if (shouldApplyPrLabel(settings)) { + if (decision.willLabel) { await ensurePullRequestLabel(env, installationId, repoFullName, pr.number, settings.gittensorLabel, { createMissingLabel: settings.createMissingLabel, }); } - if (settings.checkRunMode === "enabled" && advisory.headSha) { + if (decision.willCheckRun && advisory.headSha) { await createOrUpdateCheckRun(env, installationId, repoFullName, { ...advisory, conclusion: "success", @@ -616,7 +625,7 @@ async function maybePublishPrPublicSurface( metadata: { deliveryId: webhook.deliveryId, publicSurface: settings.publicSurface, - label: shouldApplyPrLabel(settings) ? settings.gittensorLabel : null, + label: decision.willLabel ? settings.gittensorLabel : null, checkRunMode: settings.checkRunMode, }, }); @@ -716,19 +725,6 @@ async function maybeProcessGittensoryMentionCommand(env: Env, deliveryId: string return true; } -function hasVisiblePrSurface(settings: Awaited>): boolean { - return settings.publicSurface !== "off" || settings.checkRunMode === "enabled"; -} - -function shouldPublishPrComment(settings: Awaited>): boolean { - if (settings.commentMode === "off") return false; - return settings.publicSurface === "comment_and_label" || settings.publicSurface === "comment_only"; -} - -function shouldApplyPrLabel(settings: Awaited>): boolean { - return settings.autoLabelEnabled && (settings.publicSurface === "comment_and_label" || settings.publicSurface === "label_only"); -} - async function auditPrVisibilitySkip( env: Env, repoFullName: string, diff --git a/src/signals/settings-preview.ts b/src/signals/settings-preview.ts new file mode 100644 index 0000000000..9e485632a5 --- /dev/null +++ b/src/signals/settings-preview.ts @@ -0,0 +1,285 @@ +import type { IssueRecord, PullRequestRecord, RepositoryRecord, RepositorySettings } from "../types"; +import { nowIso } from "../utils/json"; +import { + buildCollisionReport, + buildContributorProfile, + buildPreflightResult, + buildPublicPrIntelligenceComment, + buildQueueHealth, + type ContributorDetection, +} from "./engine"; + +export function hasVisiblePrSurface(settings: RepositorySettings): boolean { + return settings.publicSurface !== "off" || settings.checkRunMode === "enabled"; +} + +export function shouldPublishPrComment(settings: RepositorySettings): boolean { + if (settings.commentMode === "off") return false; + return settings.publicSurface === "comment_and_label" || settings.publicSurface === "comment_only"; +} + +export function shouldApplyPrLabel(settings: RepositorySettings): boolean { + return settings.autoLabelEnabled && (settings.publicSurface === "comment_and_label" || settings.publicSurface === "label_only"); +} + +export type PublicSurfaceMinerStatus = "confirmed" | "not_found" | "unavailable" | "not_checked"; + +export type PublicSurfaceSkipReason = + | "surface_off" + | "missing_author" + | "bot_author" + | "maintainer_author" + | "miner_detection_unavailable" + | "not_official_gittensor_miner"; + +export type PublicSurfaceAction = "skip" | "comment" | "label" | "check_run" | "none"; + +export type PublicSurfaceDecisionInput = { + settings: RepositorySettings; + authorLogin?: string | null | undefined; + authorType?: string | null | undefined; + authorAssociation?: string | null | undefined; + minerStatus: PublicSurfaceMinerStatus; +}; + +export type PublicSurfaceDecision = { + willComment: boolean; + willLabel: boolean; + willCheckRun: boolean; + skipped: boolean; + skipReason: PublicSurfaceSkipReason | null; + actions: PublicSurfaceAction[]; + summary: string; +}; + +const SKIP_SUMMARY: Record = { + surface_off: "Public surface and check runs are both disabled for this repo; nothing would be posted.", + missing_author: "The pull request has no resolvable author login; Gittensory would skip it.", + bot_author: "The author is a bot account; Gittensory would skip it.", + maintainer_author: "The author is a maintainer (owner/member/collaborator) and maintainer authors are excluded by this repo's settings.", + miner_detection_unavailable: "Official Gittensor miner detection is unavailable, so Gittensory would skip rather than guess.", + not_official_gittensor_miner: "The author is not a confirmed Gittensor miner; Gittensory would stay quiet.", +}; + +function skipDecision(reason: PublicSurfaceSkipReason): PublicSurfaceDecision { + return { willComment: false, willLabel: false, willCheckRun: false, skipped: true, skipReason: reason, actions: ["skip"], summary: SKIP_SUMMARY[reason] }; +} + +/** + * Pure decision for what the GitHub App's public surface would do for a PR. + * This is the single source of truth shared by the live webhook processor and the + * maintainer-facing dry-run preview, so the preview can never drift from real behavior. + */ +export function decidePublicSurface(input: PublicSurfaceDecisionInput): PublicSurfaceDecision { + const { settings } = input; + if (!hasVisiblePrSurface(settings)) return skipDecision("surface_off"); + if (!input.authorLogin) return skipDecision("missing_author"); + if (input.authorType === "Bot" || /\[bot\]$/i.test(input.authorLogin)) return skipDecision("bot_author"); + if (!settings.includeMaintainerAuthors && input.authorAssociation && ["OWNER", "MEMBER", "COLLABORATOR"].includes(input.authorAssociation)) { + return skipDecision("maintainer_author"); + } + if (input.minerStatus === "unavailable") return skipDecision("miner_detection_unavailable"); + if (input.minerStatus === "not_found") return skipDecision("not_official_gittensor_miner"); + + const willComment = shouldPublishPrComment(settings); + const willLabel = shouldApplyPrLabel(settings); + const willCheckRun = settings.checkRunMode === "enabled"; + const actions: PublicSurfaceAction[] = [ + ...(willComment ? (["comment"] as const) : []), + ...(willLabel ? (["label"] as const) : []), + ...(willCheckRun ? (["check_run"] as const) : []), + ]; + const surfaceActions = actions.length > 0 ? actions : (["none"] as PublicSurfaceAction[]); + return { + willComment, + willLabel, + willCheckRun, + skipped: false, + skipReason: null, + actions: surfaceActions, + summary: surfaceActions.includes("none") + ? "The author qualifies, but no surface action is enabled by the current settings." + : `Gittensory would ${surfaceActions.join(" + ").replace("check_run", "post a minimal check run")} for this PR.`, + }; +} + +export type PublicSurfaceSample = { + authorLogin?: string | null | undefined; + authorType?: string | null | undefined; + authorAssociation?: string | null | undefined; + minerStatus?: "confirmed" | "not_found" | "unavailable" | undefined; + title?: string | undefined; + body?: string | null | undefined; + labels?: string[] | undefined; + linkedIssues?: number[] | undefined; +}; + +export type InstallationHealthSummary = { + installationId: number; + status: "healthy" | "needs_attention" | "broken"; + missingPermissions: string[]; + missingEvents: string[]; + permissionRemediation: Array<{ permission: string; requiredAccess: string; currentAccess: string; ok: boolean; action: string }>; +}; + +export type RepoSettingsPreview = { + repoFullName: string; + generatedAt: string; + settings: { + publicSurface: RepositorySettings["publicSurface"]; + commentMode: RepositorySettings["commentMode"]; + publicSignalLevel: RepositorySettings["publicSignalLevel"]; + checkRunMode: RepositorySettings["checkRunMode"]; + checkRunDetailLevel: RepositorySettings["checkRunDetailLevel"]; + autoLabelEnabled: boolean; + gittensorLabel: string; + createMissingLabel: boolean; + includeMaintainerAuthors: boolean; + requireLinkedIssue: boolean; + }; + installation: InstallationHealthSummary | null; + sample: { + authorLogin: string; + authorType: string; + authorAssociation: string; + minerStatus: "confirmed" | "not_found" | "unavailable"; + title: string; + labels: string[]; + linkedIssues: number[]; + }; + decision: PublicSurfaceDecision; + previewComment: string | null; + appliedLabel: string | null; + checkRun: { willCreate: boolean; title: string; detailLevel: RepositorySettings["checkRunDetailLevel"] } | null; + warnings: string[]; + summary: string; +}; + +/** + * Assemble a maintainer-facing dry-run preview of the public surface for a sample PR. + * Pure and read-only: it never posts to or mutates GitHub. + */ +export function buildRepoSettingsPreview(args: { + repoFullName: string; + repo: RepositoryRecord | null; + settings: RepositorySettings; + installation: InstallationHealthSummary | null; + issues: IssueRecord[]; + pullRequests: PullRequestRecord[]; + sample: PublicSurfaceSample; +}): RepoSettingsPreview { + const { settings, repo, repoFullName } = args; + const sample = { + authorLogin: args.sample.authorLogin?.trim() || "sample-contributor", + authorType: args.sample.authorType || "User", + authorAssociation: args.sample.authorAssociation || "NONE", + minerStatus: args.sample.minerStatus ?? ("confirmed" as const), + title: args.sample.title?.trim() || "Sample pull request", + labels: args.sample.labels ?? [], + linkedIssues: args.sample.linkedIssues ?? [], + }; + + const decision = decidePublicSurface({ + settings, + authorLogin: sample.authorLogin, + authorType: sample.authorType, + authorAssociation: sample.authorAssociation, + minerStatus: sample.minerStatus, + }); + + const previewComment = decision.willComment + ? buildSamplePreviewComment({ repoFullName, repo, settings, issues: args.issues, pullRequests: args.pullRequests, sample, body: args.sample.body ?? null }) + : null; + + const warnings = buildWarnings(settings, decision, args.installation); + + return { + repoFullName, + generatedAt: nowIso(), + settings: { + publicSurface: settings.publicSurface, + commentMode: settings.commentMode, + publicSignalLevel: settings.publicSignalLevel, + checkRunMode: settings.checkRunMode, + checkRunDetailLevel: settings.checkRunDetailLevel, + autoLabelEnabled: settings.autoLabelEnabled, + gittensorLabel: settings.gittensorLabel, + createMissingLabel: settings.createMissingLabel, + includeMaintainerAuthors: settings.includeMaintainerAuthors, + requireLinkedIssue: settings.requireLinkedIssue, + }, + installation: args.installation, + sample, + decision, + previewComment, + appliedLabel: decision.willLabel ? settings.gittensorLabel : null, + checkRun: decision.willCheckRun ? { willCreate: true, title: "Gittensory context posted", detailLevel: settings.checkRunDetailLevel } : null, + warnings, + summary: decision.skipped + ? `Sample PR would be skipped: ${decision.summary}` + : `Sample PR would result in: ${decision.actions.join(", ")}.${warnings.length > 0 ? ` ${warnings.length} permission/config warning(s).` : ""}`, + }; +} + +function buildWarnings(settings: RepositorySettings, decision: PublicSurfaceDecision, installation: InstallationHealthSummary | null): string[] { + const warnings: string[] = []; + if (!installation) { + warnings.push("Installation health is unknown for this repo; run refresh-installation-health to verify GitHub App permissions and subscribed events."); + return warnings; + } + const missing = new Set(installation.missingPermissions); + if ((decision.willComment || decision.willLabel) && missing.has("issues")) { + warnings.push("Comments and labels require GitHub App permission Issues: write, which is currently missing. Set repository permission issues to write, then approve the change."); + } + if (settings.checkRunMode === "enabled" && missing.has("checks")) { + warnings.push("Check runs are enabled but GitHub App permission Checks: write is missing. Set repository permission checks to write, then approve the change."); + } + for (const event of installation.missingEvents) { + warnings.push(`The GitHub App is not subscribed to the ${event} webhook event; subscribe to it so Gittensory receives the relevant deliveries.`); + } + if (installation.status !== "healthy" && warnings.length === 0) { + warnings.push(`Installation status is ${installation.status}; review the installation health endpoint for remediation steps.`); + } + return warnings; +} + +function buildSamplePreviewComment(args: { + repoFullName: string; + repo: RepositoryRecord | null; + settings: RepositorySettings; + issues: IssueRecord[]; + pullRequests: PullRequestRecord[]; + sample: { authorLogin: string; authorAssociation: string; minerStatus: "confirmed" | "not_found" | "unavailable"; title: string; labels: string[]; linkedIssues: number[] }; + body: string | null; +}): string { + const samplePr: PullRequestRecord = { + repoFullName: args.repoFullName, + number: 0, + title: args.sample.title, + state: "open", + authorLogin: args.sample.authorLogin, + authorAssociation: args.sample.authorAssociation, + labels: args.sample.labels, + linkedIssues: args.sample.linkedIssues, + body: args.body, + }; + const profile = buildContributorProfile(args.sample.authorLogin, { login: args.sample.authorLogin, topLanguages: [], source: "unavailable" }, [], []); + const detection: ContributorDetection = { detected: true, reason: "Confirmed Gittensor miner (simulated for preview).", source: "official_gittensor_api", priorPullRequests: 0, priorMergedPullRequests: 0, priorIssues: 0 }; + const collisions = buildCollisionReport(args.repoFullName, args.issues, args.pullRequests); + const queueHealth = buildQueueHealth(args.repo, args.issues, args.pullRequests, collisions); + const preflight = buildPreflightResult( + { + repoFullName: args.repoFullName, + contributorLogin: args.sample.authorLogin, + title: args.sample.title, + body: args.body ?? undefined, + labels: args.sample.labels, + linkedIssues: args.sample.linkedIssues, + authorAssociation: args.sample.authorAssociation, + }, + args.repo, + args.issues, + args.pullRequests, + ); + return buildPublicPrIntelligenceComment({ repo: args.repo, pr: samplePr, profile, detection, queueHealth, collisions, preflight, settings: args.settings }); +} diff --git a/test/integration/api.test.ts b/test/integration/api.test.ts index 0857fca3b6..51a2009cec 100644 --- a/test/integration/api.test.ts +++ b/test/integration/api.test.ts @@ -236,6 +236,43 @@ describe("api routes", () => { dataQuality: expect.any(Object), }); + const settingsPreviewUnauthenticated = await app.request("/v1/repos/entrius/allways-ui/settings-preview", { method: "POST", body: "{}" }, env); + expect(settingsPreviewUnauthenticated.status).toBe(401); + + const minerPreview = await app.request( + "/v1/repos/entrius/allways-ui/settings-preview", + { method: "POST", headers: apiHeaders(env), body: JSON.stringify({ sample: { authorLogin: "oktofeesh1", minerStatus: "confirmed", title: "Fix cache", labels: ["bug"], linkedIssues: [7] } }) }, + env, + ); + expect(minerPreview.status).toBe(200); + const minerPreviewBody = (await minerPreview.json()) as { decision: { willComment: boolean; skipped: boolean }; previewComment: string | null; settings: { publicSurface: string } }; + expect(minerPreviewBody.decision.skipped).toBe(false); + expect(minerPreviewBody.decision.willComment).toBe(true); + expect(minerPreviewBody.previewComment).toContain("Gittensory contribution context"); + expect(minerPreviewBody.previewComment).not.toMatch(/wallet|hotkey|trust score|scoreability|payout/i); + + const invalidPreview = await app.request( + "/v1/repos/entrius/allways-ui/settings-preview", + { method: "POST", headers: apiHeaders(env), body: JSON.stringify({ sample: { minerStatus: "maybe" } }) }, + env, + ); + expect(invalidPreview.status).toBe(400); + + const unknownRepoPreview = await app.request("/v1/repos/missing/repo/settings-preview", { method: "POST", headers: apiHeaders(env), body: "{" }, env); + expect(unknownRepoPreview.status).toBe(200); + await expect(unknownRepoPreview.json()).resolves.toMatchObject({ + installation: null, + sample: { authorLogin: "sample-contributor", minerStatus: "confirmed" }, + }); + + const botPreview = await app.request( + "/v1/repos/entrius/allways-ui/settings-preview", + { method: "POST", headers: apiHeaders(env), body: JSON.stringify({ sample: { authorLogin: "robot", authorType: "Bot", minerStatus: "confirmed" } }) }, + env, + ); + expect(botPreview.status).toBe(200); + await expect(botPreview.json()).resolves.toMatchObject({ decision: { skipped: true, skipReason: "bot_author" }, previewComment: null }); + const registrationReadiness = await app.request("/v1/repos/entrius/allways-ui/registration-readiness", { headers: apiHeaders(env) }, env); expect(registrationReadiness.status).toBe(200); await expect(registrationReadiness.json()).resolves.toMatchObject({ @@ -690,6 +727,28 @@ describe("api routes", () => { } }); + it("settings-preview never mutates GitHub state", async () => { + const app = createApp(); + const env = createTestEnv(); + await seedSignalData(env); + const calls: Array<{ method: string; url: string }> = []; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + calls.push({ method: (init?.method ?? "GET").toUpperCase(), url: input.toString() }); + return new Response("not found", { status: 404 }); + }); + const response = await app.request( + "/v1/repos/entrius/allways-ui/settings-preview", + { method: "POST", headers: apiHeaders(env), body: JSON.stringify({ sample: { authorLogin: "oktofeesh1", minerStatus: "confirmed", labels: ["bug"], linkedIssues: [7] } }) }, + env, + ); + expect(response.status).toBe(200); + // The dry-run preview is fully offline: it must make no GitHub calls at all, and certainly no mutating ones. + const githubCalls = calls.filter((call) => /github\.com/.test(call.url)); + expect(githubCalls).toEqual([]); + const mutatingCalls = calls.filter((call) => call.method !== "GET" && call.method !== "HEAD"); + expect(mutatingCalls).toEqual([]); + }); + it("reports ready status when required public-review dependencies are present", async () => { const app = createApp(); const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "public-token" }); @@ -1894,6 +1953,19 @@ async function seedSignalData(env: Env): Promise { events: ["issues", "pull_request", "repository"], }, }); + await upsertInstallationHealth(env, { + installationId: 123, + accountLogin: "entrius", + repositorySelection: "selected", + installedReposCount: 1, + registeredInstalledCount: 1, + status: "healthy", + missingPermissions: [], + missingEvents: [], + permissions: { metadata: "read", pull_requests: "read", issues: "write" }, + events: ["issues", "pull_request", "repository"], + checkedAt: "2026-05-23T00:00:00.000Z", + }); const snapshot = normalizeRegistryPayload( { "entrius/allways-ui": { @@ -1930,7 +2002,7 @@ async function seedSignalData(env: Env): Promise { private: false, default_branch: "test", owner: { login: "entrius" }, - }); + }, 123); await persistScoringModelSnapshot(env, { id: "scoring-1", sourceKind: "test", diff --git a/test/unit/openapi.test.ts b/test/unit/openapi.test.ts index 3b6177907e..f1d656f100 100644 --- a/test/unit/openapi.test.ts +++ b/test/unit/openapi.test.ts @@ -29,6 +29,7 @@ describe("OpenAPI contract", () => { expect(spec.paths["/v1/scoring/model"]).toBeDefined(); expect(spec.paths["/v1/scoring/preview"]).toBeDefined(); expect(spec.paths["/v1/bounties/{id}/advisory"]).toBeDefined(); + expect(spec.paths["/v1/repos/{owner}/{repo}/settings-preview"]).toBeDefined(); expect(spec.paths["/v1/auth/github/device/start"]).toBeDefined(); expect(spec.paths["/v1/auth/session"]).toBeDefined(); expect(spec.paths["/v1/internal/jobs/repair-data-fidelity"]).toBeDefined(); @@ -68,6 +69,7 @@ describe("OpenAPI contract", () => { expect(spec.components?.schemas?.PullRequestMaintainerPacket).toBeDefined(); expect(spec.components?.schemas?.PullRequestReviewability).toBeDefined(); expect(spec.components?.schemas?.LocalBranchAnalysis).toBeDefined(); + expect(spec.components?.schemas?.RepoSettingsPreview).toBeDefined(); expect(spec.components?.schemas?.AgentRunBundle).toBeDefined(); expect(spec.components?.schemas?.AgentAction).toBeDefined(); expect(JSON.stringify(spec.components?.schemas?.ScorePreviewResult)).toContain("scenarioPreviews"); diff --git a/test/unit/settings-preview.test.ts b/test/unit/settings-preview.test.ts new file mode 100644 index 0000000000..e39899906d --- /dev/null +++ b/test/unit/settings-preview.test.ts @@ -0,0 +1,203 @@ +import { describe, expect, it } from "vitest"; +import { buildRepoSettingsPreview, decidePublicSurface, type InstallationHealthSummary } from "../../src/signals/settings-preview"; +import type { IssueRecord, PullRequestRecord, RepositoryRecord, RepositorySettings } from "../../src/types"; + +const repo: RepositoryRecord = { + fullName: "entrius/allways-ui", + owner: "entrius", + name: "allways-ui", + installationId: 1, + isInstalled: true, + isRegistered: true, + isPrivate: false, + registryConfig: { + repo: "entrius/allways-ui", + emissionShare: 0.01, + issueDiscoveryShare: 0, + labelMultipliers: { bug: 1.1 }, + trustedLabelPipeline: true, + maintainerCut: 0, + raw: {}, + }, +}; + +const issues: IssueRecord[] = [ + { repoFullName: repo.fullName, number: 7, title: "Cache refresh fails", state: "open", authorLogin: "reporter", labels: ["bug"], linkedPrs: [] }, +]; +const pullRequests: PullRequestRecord[] = []; + +function settings(overrides: Partial = {}): RepositorySettings { + return { + repoFullName: repo.fullName, + commentMode: "detected_contributors_only", + publicSignalLevel: "standard", + checkRunMode: "off", + checkRunDetailLevel: "standard", + autoLabelEnabled: true, + gittensorLabel: "gittensor", + createMissingLabel: true, + publicSurface: "comment_and_label", + includeMaintainerAuthors: false, + requireLinkedIssue: false, + backfillEnabled: true, + privateTrustEnabled: true, + ...overrides, + }; +} + +const healthyInstall: InstallationHealthSummary = { + installationId: 1, + status: "healthy", + missingPermissions: [], + missingEvents: [], + permissionRemediation: [{ permission: "issues", requiredAccess: "write", currentAccess: "write", ok: true, action: "No change needed." }], +}; + +describe("decidePublicSurface", () => { + it("comments and labels for a confirmed miner when the surface is enabled", () => { + const decision = decidePublicSurface({ settings: settings(), authorLogin: "miner", authorType: "User", authorAssociation: "NONE", minerStatus: "confirmed" }); + expect(decision).toMatchObject({ skipped: false, willComment: true, willLabel: true, willCheckRun: false }); + expect(decision.actions).toEqual(["comment", "label"]); + }); + + it("skips disabled surfaces, bots, maintainer authors, non-miners, and unavailable detection", () => { + expect(decidePublicSurface({ settings: settings({ publicSurface: "off", checkRunMode: "off" }), authorLogin: "miner", minerStatus: "confirmed" }).skipReason).toBe("surface_off"); + expect(decidePublicSurface({ settings: settings(), authorLogin: null, minerStatus: "confirmed" }).skipReason).toBe("missing_author"); + expect(decidePublicSurface({ settings: settings(), authorLogin: "robot", authorType: "Bot", minerStatus: "confirmed" }).skipReason).toBe("bot_author"); + expect(decidePublicSurface({ settings: settings(), authorLogin: "app[bot]", minerStatus: "confirmed" }).skipReason).toBe("bot_author"); + expect(decidePublicSurface({ settings: settings(), authorLogin: "owner", authorAssociation: "OWNER", minerStatus: "confirmed" }).skipReason).toBe("maintainer_author"); + expect(decidePublicSurface({ settings: settings(), authorLogin: "x", minerStatus: "not_found" }).skipReason).toBe("not_official_gittensor_miner"); + expect(decidePublicSurface({ settings: settings(), authorLogin: "x", minerStatus: "unavailable" }).skipReason).toBe("miner_detection_unavailable"); + }); + + it("includes maintainer authors when configured", () => { + const decision = decidePublicSurface({ settings: settings({ includeMaintainerAuthors: true }), authorLogin: "owner", authorAssociation: "OWNER", minerStatus: "confirmed" }); + expect(decision.skipped).toBe(false); + }); + + it("supports a check-run-only surface even when public comments are off", () => { + const decision = decidePublicSurface({ settings: settings({ publicSurface: "off", checkRunMode: "enabled" }), authorLogin: "miner", minerStatus: "confirmed" }); + expect(decision).toMatchObject({ skipped: false, willComment: false, willLabel: false, willCheckRun: true }); + expect(decision.actions).toEqual(["check_run"]); + }); + + it("reports no action when the surface is visible but every action is disabled", () => { + const decision = decidePublicSurface({ + settings: settings({ publicSurface: "label_only", autoLabelEnabled: false, commentMode: "off", checkRunMode: "off" }), + authorLogin: "miner", + minerStatus: "confirmed", + }); + expect(decision).toMatchObject({ skipped: false, willComment: false, willLabel: false, willCheckRun: false, actions: ["none"] }); + expect(decision.summary).toMatch(/no surface action is enabled/); + }); +}); + +describe("buildRepoSettingsPreview", () => { + const base = { repoFullName: repo.fullName, repo, issues, pullRequests }; + + it("previews a confirmed-miner PR on a healthy install with no warnings", () => { + const preview = buildRepoSettingsPreview({ ...base, settings: settings(), installation: healthyInstall, sample: { authorLogin: "miner", minerStatus: "confirmed" } }); + expect(preview.decision.willComment).toBe(true); + expect(preview.appliedLabel).toBe("gittensor"); + expect(preview.previewComment).toContain("Gittensory contribution context"); + expect(preview.warnings).toHaveLength(0); + }); + + it("uses safe defaults for an empty sample preview", () => { + const preview = buildRepoSettingsPreview({ ...base, settings: settings(), installation: healthyInstall, sample: {} }); + expect(preview.sample).toMatchObject({ authorLogin: "sample-contributor", authorType: "User", authorAssociation: "NONE", minerStatus: "confirmed", title: "Sample pull request" }); + expect(preview.decision.skipped).toBe(false); + }); + + it("explains a missing Issues: write permission", () => { + const preview = buildRepoSettingsPreview({ + ...base, + settings: settings(), + installation: { ...healthyInstall, status: "needs_attention", missingPermissions: ["issues"] }, + sample: { authorLogin: "miner", minerStatus: "confirmed" }, + }); + expect(preview.warnings.some((warning) => /Issues: write/.test(warning))).toBe(true); + }); + + it("explains a missing optional Checks: write permission only when check runs are enabled", () => { + const withChecks = buildRepoSettingsPreview({ + ...base, + settings: settings({ checkRunMode: "enabled" }), + installation: { ...healthyInstall, status: "needs_attention", missingPermissions: ["checks"] }, + sample: { authorLogin: "miner", minerStatus: "confirmed" }, + }); + expect(withChecks.checkRun).toMatchObject({ willCreate: true }); + expect(withChecks.warnings.some((warning) => /Checks: write/.test(warning))).toBe(true); + + const withoutChecks = buildRepoSettingsPreview({ + ...base, + settings: settings({ checkRunMode: "off" }), + installation: { ...healthyInstall, missingPermissions: ["checks"] }, + sample: { authorLogin: "miner", minerStatus: "confirmed" }, + }); + expect(withoutChecks.checkRun).toBeNull(); + expect(withoutChecks.warnings.some((warning) => /Checks: write/.test(warning))).toBe(false); + }); + + it("shows a quiet skip for a non-miner author with no rendered comment", () => { + const preview = buildRepoSettingsPreview({ ...base, settings: settings(), installation: healthyInstall, sample: { authorLogin: "drive-by", minerStatus: "not_found" } }); + expect(preview.decision).toMatchObject({ skipped: true, skipReason: "not_official_gittensor_miner" }); + expect(preview.previewComment).toBeNull(); + expect(preview.appliedLabel).toBeNull(); + }); + + it("warns that label-only mode still needs Issues: write", () => { + const preview = buildRepoSettingsPreview({ + ...base, + settings: settings({ publicSurface: "label_only", autoLabelEnabled: true, commentMode: "off" }), + installation: { ...healthyInstall, status: "needs_attention", missingPermissions: ["issues"] }, + sample: { authorLogin: "miner", minerStatus: "confirmed" }, + }); + // Labels are applied through the GitHub Issues API, so label-only mode still requires Issues: write. + expect(preview.decision).toMatchObject({ willComment: false, willLabel: true }); + expect(preview.appliedLabel).toBe("gittensor"); + expect(preview.warnings.some((warning) => /Issues: write/.test(warning))).toBe(true); + }); + + it("shows the default maintainer-author skip", () => { + const preview = buildRepoSettingsPreview({ ...base, settings: settings(), installation: healthyInstall, sample: { authorLogin: "owner", authorAssociation: "OWNER", minerStatus: "confirmed" } }); + expect(preview.decision.skipReason).toBe("maintainer_author"); + expect(preview.previewComment).toBeNull(); + }); + + it("warns when installation health is unknown", () => { + const preview = buildRepoSettingsPreview({ ...base, settings: settings(), installation: null, sample: { authorLogin: "miner", minerStatus: "confirmed" } }); + expect(preview.warnings.some((warning) => /Installation health is unknown/.test(warning))).toBe(true); + }); + + it("explains missing webhook event subscriptions", () => { + const preview = buildRepoSettingsPreview({ + ...base, + settings: settings(), + installation: { ...healthyInstall, status: "needs_attention", missingEvents: ["pull_request"] }, + sample: { authorLogin: "miner", minerStatus: "confirmed" }, + }); + expect(preview.warnings).toEqual(expect.arrayContaining([expect.stringMatching(/pull_request webhook event/)])); + }); + + it("falls back to the installation status warning when no specific remediation is available", () => { + const preview = buildRepoSettingsPreview({ + ...base, + settings: settings(), + installation: { ...healthyInstall, status: "broken" }, + sample: { authorLogin: "miner", minerStatus: "confirmed" }, + }); + expect(preview.warnings).toEqual(["Installation status is broken; review the installation health endpoint for remediation steps."]); + }); + + it("never leaks private scoring/trust terms into the preview comment (sanitizer regression)", () => { + const preview = buildRepoSettingsPreview({ + ...base, + settings: settings(), + installation: healthyInstall, + sample: { authorLogin: "miner", minerStatus: "confirmed", title: "Improve wallet hotkey trust score payout", body: "raw trust and scoreability /100 reviewability 5", labels: ["bug"], linkedIssues: [7] }, + }); + expect(preview.previewComment).not.toBeNull(); + expect(preview.previewComment ?? "").not.toMatch(/wallet|hotkey|trust score|raw trust|scoreability|payout|reward|farming|\/100|reviewability\s*\d/i); + }); +});