diff --git a/.gittensory.yml.example b/.gittensory.yml.example index c64f781690..51c61614ea 100644 --- a/.gittensory.yml.example +++ b/.gittensory.yml.example @@ -899,6 +899,7 @@ settings: # e2eTestGen: false # Auto-generated E2E test scaffolding. Default: false. # planner: false # The @gittensory plan / issue-planner completion. Default: false. # summaries: false # AI summaries/rewrite text. Default: false. + # chatQa: false # @gittensory chat grounded LLM Q&A. Ollama-ONLY (never the frontier env.AI); needs env.AI_ADVISORY set. Co-requisite: commandRateLimitPolicy: hold (defaults off). Default: false. # Maintainer AI review tuning (`.gittensory.yml` top-level `review:` block). These knobs shape the advisory AI # review prompt and file selection only — gate/slop/secret-scan are unaffected. diff --git a/apps/gittensory-ui/public/openapi.json b/apps/gittensory-ui/public/openapi.json index a94f4b8658..d7e1d7a103 100644 --- a/apps/gittensory-ui/public/openapi.json +++ b/apps/gittensory-ui/public/openapi.json @@ -9167,13 +9167,18 @@ }, "summaries": { "type": "boolean" + }, + "chatQa": { + "type": "boolean", + "description": "Opt the `@gittensory chat ` grounded Q&A surface (#4595) into local Ollama inference. Ollama-only: unlike the four capabilities above it NEVER falls back to the frontier env.AI when off; it declines unless this is true AND env.AI_ADVISORY is configured. Co-requisite: set `commandRateLimitPolicy` to `hold` (it defaults to `off` fleet-wide) so the tighter `commandRateLimitAiMaxPerWindow` ceiling actually throttles this cost-bearing command." } }, "required": [ "slop", "e2eTestGen", "planner", - "summaries" + "summaries", + "chatQa" ] }, "gittensorLabel": { diff --git a/apps/gittensory-ui/src/lib/command-reference.ts b/apps/gittensory-ui/src/lib/command-reference.ts index 38b6515f05..4eed8e63eb 100644 --- a/apps/gittensory-ui/src/lib/command-reference.ts +++ b/apps/gittensory-ui/src/lib/command-reference.ts @@ -12,6 +12,12 @@ export const PUBLIC_COMMAND_ENTRIES = [ description: "Answer contribution-quality questions from connected cached sources with citations.", }, + { + id: "chat", + title: "Gittensory grounded chat Q&A", + description: + "Answer a question in natural prose from cached decision-pack facts via local Ollama (maintainer/collaborator; read-only).", + }, { id: "preflight", title: "Gittensory preflight", @@ -55,7 +61,7 @@ export const PUBLIC_COMMAND_ENTRIES = [ ] as const; export const PUBLIC_COMMAND_LIST = - "@gittensory help\n@gittensory ask\n@gittensory preflight\n@gittensory blockers\n@gittensory duplicate-check\n@gittensory miner-context\n@gittensory next-action\n@gittensory reviewability\n@gittensory repo-fit\n@gittensory packet"; + "@gittensory help\n@gittensory ask\n@gittensory chat\n@gittensory preflight\n@gittensory blockers\n@gittensory duplicate-check\n@gittensory miner-context\n@gittensory next-action\n@gittensory reviewability\n@gittensory repo-fit\n@gittensory packet"; export const MAINTAINER_COMMAND_ENTRIES = [ { diff --git a/config/examples/gittensory.full.yml b/config/examples/gittensory.full.yml index 95e8fee051..b749475499 100644 --- a/config/examples/gittensory.full.yml +++ b/config/examples/gittensory.full.yml @@ -912,6 +912,7 @@ settings: # e2eTestGen: false # Auto-generated E2E test scaffolding. Default: false. # planner: false # The @gittensory plan / issue-planner completion. Default: false. # summaries: false # AI summaries/rewrite text. Default: false. + # chatQa: false # @gittensory chat grounded LLM Q&A. Ollama-ONLY (never the frontier env.AI); needs env.AI_ADVISORY set. Co-requisite: commandRateLimitPolicy: hold (defaults off). Default: false. # Maintainer AI review tuning (`.gittensory.yml` top-level `review:` block). These knobs shape the advisory AI # review prompt and file selection only — gate/slop/secret-scan are unaffected. diff --git a/packages/gittensory-engine/src/focus-manifest.ts b/packages/gittensory-engine/src/focus-manifest.ts index 2d40087233..0b0114d302 100644 --- a/packages/gittensory-engine/src/focus-manifest.ts +++ b/packages/gittensory-engine/src/focus-manifest.ts @@ -1959,6 +1959,7 @@ function parseSettingsOverride(value: JsonValue | undefined, warnings: string[], if (typeof rawRouting.e2eTestGen === "boolean") sparseRouting.e2eTestGen = validated.e2eTestGen; if (typeof rawRouting.planner === "boolean") sparseRouting.planner = validated.planner; if (typeof rawRouting.summaries === "boolean") sparseRouting.summaries = validated.summaries; + if (typeof rawRouting.chatQa === "boolean") sparseRouting.chatQa = validated.chatQa; out.advisoryAiRouting = sparseRouting; } else if (r.advisoryAiRouting !== undefined) { warnings.push(`Manifest "settings.advisoryAiRouting" must be an object; ignoring it and keeping any existing policy.`); diff --git a/packages/gittensory-engine/src/review/advisory-ai-routing-config.ts b/packages/gittensory-engine/src/review/advisory-ai-routing-config.ts index fb4bf7aea1..5ea588697f 100644 --- a/packages/gittensory-engine/src/review/advisory-ai-routing-config.ts +++ b/packages/gittensory-engine/src/review/advisory-ai-routing-config.ts @@ -5,6 +5,7 @@ export const DEFAULT_ADVISORY_AI_ROUTING: AdvisoryAiRoutingConfig = { e2eTestGen: false, planner: false, summaries: false, + chatQa: false, }; function normalizeField(value: unknown, field: keyof AdvisoryAiRoutingConfig, warnings: string[]): boolean { @@ -31,5 +32,6 @@ export function normalizeAdvisoryAiRoutingConfig(input: unknown, warnings: strin e2eTestGen: normalizeField(record.e2eTestGen, "e2eTestGen", warnings), planner: normalizeField(record.planner, "planner", warnings), summaries: normalizeField(record.summaries, "summaries", warnings), + chatQa: normalizeField(record.chatQa, "chatQa", warnings), }; } diff --git a/packages/gittensory-engine/src/settings/command-authorization.ts b/packages/gittensory-engine/src/settings/command-authorization.ts index 7f4c0de1a0..7bad7d13ef 100644 --- a/packages/gittensory-engine/src/settings/command-authorization.ts +++ b/packages/gittensory-engine/src/settings/command-authorization.ts @@ -14,6 +14,12 @@ export const DEFAULT_COMMAND_AUTHORIZATION_POLICY: RepositoryCommandAuthorizatio "noise-report": ["maintainer", "collaborator"], "gate-override": ["maintainer", "collaborator"], plan: ["maintainer", "collaborator"], + // #4595: deliberately narrower than "ask"'s default (which allows confirmed_miner) -- chat is Ollama-only + // grounded LLM generation, a materially larger surface than ask's deterministic-only answer, so v1 starts + // maintainer/collaborator-only. Explicit registration here (rather than falling through to `default`) also + // activates the pr_author-widening guard below, so a self-hoster can't accidentally yml themselves into + // "anyone commenting on their own PR" without it. + chat: ["maintainer", "collaborator"], // #1960 PR control-surface verbs. "review" is deliberately widenable to confirmed_miner (same self-rerun // precedent already applied to review-now, #824) — a confirmed miner may re-trigger review on their own PR. // The rest (pause/resume/resolve/configuration/explain) are conservative maintainer/collaborator-only diff --git a/packages/gittensory-engine/src/types/manifest-deps-types.ts b/packages/gittensory-engine/src/types/manifest-deps-types.ts index 454b5f8635..7ea7e43318 100644 --- a/packages/gittensory-engine/src/types/manifest-deps-types.ts +++ b/packages/gittensory-engine/src/types/manifest-deps-types.ts @@ -112,6 +112,9 @@ export type AdvisoryAiRoutingConfig = { e2eTestGen: boolean; planner: boolean; summaries: boolean; + /** Grounded `@gittensory chat ` LLM Q&A (#4595). Ollama-only: unlike the four fields above it NEVER + * falls back to the frontier env.AI when off -- it simply declines. Default false. */ + chatQa: boolean; }; export type ContributorBlacklistEntry = { diff --git a/src/github/commands.ts b/src/github/commands.ts index 1fe6150eae..d3b86b19a8 100644 --- a/src/github/commands.ts +++ b/src/github/commands.ts @@ -6,6 +6,7 @@ import { } from "./command-suggest"; import { gittensoryFooter, GITTENSORY_SITE_URL, type GittensoryFooterEnv } from "./footer"; import type { AgentRunBundle } from "../services/agent-orchestrator"; +import type { ChatQaResult } from "../services/ai-chat-qa"; import type { GittensorContributorSnapshot, OfficialGittensorMinerDetection } from "../gittensor/api"; import type { AgentActionRecord, RepositoryCommandAuthorizationPolicy } from "../types"; import type { CheckSummaryRecord, GitHubIssuePayload, IssueRecord, PullRequestRecord, RecentMergedPullRequestRecord, RepositoryRecord } from "../types"; @@ -28,6 +29,7 @@ import { buildMaintainerNoiseReport, type MaintainerNoiseReport } from "../signa const PUBLIC_MENTION_COMMAND_CATALOG = [ { id: "help", title: "Gittensory command help", description: "Show public-safe @gittensory command help." }, { id: "ask", title: "Gittensory contribution context Q&A", description: "Answer contribution-quality questions from connected cached sources with citations." }, + { id: "chat", title: "Gittensory grounded chat Q&A", description: "Answer a question in natural prose from cached decision-pack facts via local Ollama (maintainer/collaborator; read-only)." }, { id: "preflight", title: "Gittensory preflight", description: "Summarize public PR hygiene and validation readiness." }, { id: "blockers", title: "Gittensory readiness blockers", description: "Explain public-safe readiness blockers." }, { id: "duplicate-check", title: "Gittensory duplicate & WIP check", description: "Summarize duplicate and in-progress overlap caution." }, @@ -54,7 +56,9 @@ export const GITTENSORY_MENTION_COMMAND_CATALOG = [...PUBLIC_MENTION_COMMAND_CAT export type GittensoryMentionCommandName = (typeof GITTENSORY_MENTION_COMMAND_CATALOG)[number]["id"]; export type MaintainerQueueDigestCommandName = (typeof MAINTAINER_QUEUE_DIGEST_COMMAND_CATALOG)[number]["id"]; -type SnapshotCommandName = Exclude; +// `chat` (#4595) is excluded like help/miner-context: it renders a bespoke LLM-answer card (buildChatPublicAnswerCard), +// not the deterministic snapshot-section path, so it needs no REFRESH_/EMPTY_SECTION_TITLES entry. +type SnapshotCommandName = Exclude; // Action commands are NOT Q&A: they perform a side effect (handled before the mention-command path) rather // than producing a public answer card. They are intentionally kept OUT of the Q&A catalog/unions so the @@ -135,6 +139,8 @@ type PublicAnswerCard = { nextActions: string[]; sourceNotes: string[]; safeDetails?: string[] | undefined; + /** Fixed, non-LLM footer stamped verbatim on the card (only `@gittensory chat` sets it, #4595 req 9). */ + disclaimer?: string | undefined; }; export type AgentCommandFeedbackContext = { @@ -268,7 +274,7 @@ export function parseGittensoryMentionCommand(body: string | null | undefined): const name = requested as GittensoryMentionCommandName; // match[2] is always defined for the same reason as the action-command path above. /* v8 ignore next */ - const question = name === "ask" ? (match[2] ?? "").trim() : undefined; + const question = name === "ask" || name === "chat" ? (match[2] ?? "").trim() : undefined; return { name, raw: match[0].trim(), @@ -320,6 +326,7 @@ export function isGittensoryActionCommand(name: GittensoryMentionCommandName | G // limit to the AI-cost-bearing surface than the cheap one. const AI_COST_BEARING_COMMANDS = new Set([ "ask", + "chat", "blockers", "preflight", "reviewability", @@ -352,6 +359,12 @@ export function isAuthorizedCommandActor(args: { return { authorized: decision.authorized, reason: decision.reason, actorKind: decision.actorKind }; } +/** Fixed, non-LLM disclaimer stamped on every `@gittensory chat` answer card (#4595 req 9). Deliberately NOT run + * through neutralizePublicMarkdownText so the `@gittensory review` code span renders; it carries no forbidden + * terms, so the whole-body sanitizePublicComment pass leaves it byte-for-byte intact. */ +export const CHAT_QA_DISCLAIMER = + "Read-only informational reply — cannot change review outcomes, gate state, or trigger a re-review. To retrigger a review, comment `@gittensory review`."; + export function buildPublicAgentCommandComment(args: { command: GittensoryMentionCommand; repo: RepositoryRecord | null; @@ -362,6 +375,9 @@ export function buildPublicAgentCommandComment(args: { officialMiner?: GittensorContributorSnapshot | null | undefined; bundle?: AgentRunBundle | null | undefined; maintainerDigest?: MaintainerQueueDigest | null | undefined; + /** Grounded `@gittensory chat` answer (#4595). Only read when `command.name === "chat"`; the dispatcher + * resolves it via generateChatQaAnswer before composing the card. */ + chatAnswer?: ChatQaResult | null | undefined; /** Resolved by the caller from `env.PUBLIC_SITE_ORIGIN` -- see `gittensoryFooter` (#4613). */ env: GittensoryFooterEnv; }): string { @@ -384,7 +400,8 @@ export function buildPublicAgentCommandComment(args: { bundle: args.bundle, officialMiner: args.officialMiner, actorKind: args.actorKind, - question: commandName === "ask" ? args.command.question : undefined, + question: commandName === "ask" || commandName === "chat" ? args.command.question : undefined, + chatAnswer: args.chatAnswer, }); const body = [ AGENT_COMMAND_COMMENT_MARKER, @@ -422,10 +439,14 @@ function buildPublicAnswerCard(args: { officialMiner: GittensorContributorSnapshot | null | undefined; actorKind: "maintainer" | "author"; question?: string | undefined; + chatAnswer?: ChatQaResult | null | undefined; }): PublicAnswerCard { if (args.command === "ask") { return buildAskPublicAnswerCard(args); } + if (args.command === "chat") { + return buildChatPublicAnswerCard(args); + } const [titleLine, ...contentLines] = args.sections; const safeContent = contentLines.map(stripBulletPrefix).filter((line) => line.length > 0); const findings = safeContent.length > 0 ? safeContent.slice(0, 5) : ["No public-safe findings are available from the current cached context."]; @@ -503,11 +524,91 @@ function buildAskPublicAnswerCard(args: { }; } +function buildChatPublicAnswerCard(args: { + bundle: AgentRunBundle | null | undefined; + officialMiner: GittensorContributorSnapshot | null | undefined; + actorKind: "maintainer" | "author"; + question?: string | undefined; + chatAnswer?: ChatQaResult | null | undefined; +}): PublicAnswerCard { + // (#4595 req 8) The question is free-form contributor text and the answer is MODEL output -- the first surface + // that echoes model output into a trusted bot comment. Run BOTH through sanitizePublicComment (redact private + // terms) then neutralizePublicMarkdownText (escape markdown/HTML, zero-width @mentions + URLs) before they land + // in the card, exactly like ask does for its question (#2457). + const questionLine = `Question: ${neutralizePublicMarkdownText( + sanitizePublicComment(args.question?.trim() || "No question was provided."), + )}`; + const answer = chatAnswerContent(args.chatAnswer); + return { + title: "Grounded chat Q&A", + summary: commandSummary("chat"), + findings: [questionLine, ...answer.findings], + evidence: commandEvidence("chat", args.bundle, args.officialMiner, args.actorKind), + nextActions: answer.nextActions, + sourceNotes: commandSourceNotes("chat", args.bundle, args.officialMiner), + disclaimer: CHAT_QA_DISCLAIMER, + }; +} + +// Maps a ChatQaResult into the card's answer findings + next actions. `ok` neutralizes the MODEL prose (#4595 +// req 8); every other status renders a fixed, safe, deterministic line (never the model) — so a +// disabled/unavailable/declined/over-budget/unsafe/error path always posts a grounded, non-leaking reply. +function chatAnswerContent(chatAnswer: ChatQaResult | null | undefined): { findings: string[]; nextActions: string[] } { + if (!chatAnswer) { + return { + findings: ["Chat Q&A could not produce a grounded answer for this request."], + nextActions: ["Run `@gittensory preflight` or `@gittensory blockers` for the deterministic readiness facts."], + }; + } + switch (chatAnswer.status) { + case "ok": + return { + findings: chatAnswerProseLines(chatAnswer.text), + nextActions: ["Ask one concrete question per invocation; chat only rewrites the same cached decision-pack facts and cannot change review outcomes."], + }; + case "disabled": + case "unavailable": + return { + findings: ["Chat Q&A is not enabled on this instance. It runs only on local advisory inference (Ollama) and never falls back to the frontier model."], + nextActions: ["A maintainer can enable it via `settings.advisoryAiRouting.chatQa` with `env.AI_ADVISORY` configured; use `@gittensory ask` in the meantime."], + }; + case "declined": + // `reason`/`suggestion` are fixed, trusted strings authored in ai-chat-qa.ts (no user/model interpolation), + // so they keep their `@gittensory ...` code spans -- redact-only, not markdown-escaped. + return { + findings: [sanitizePublicComment(chatAnswer.reason)], + nextActions: [sanitizePublicComment(chatAnswer.suggestion)], + }; + case "quota_exceeded": + return { + findings: ["The shared daily AI budget is exhausted, so chat Q&A declined this request rather than spending over budget."], + nextActions: ["Try again after the daily budget resets, or run `@gittensory preflight` for the deterministic readiness facts."], + }; + case "unsafe": + case "error": + return { + findings: ["Chat Q&A could not produce a grounded answer for this request."], + nextActions: ["Run `@gittensory preflight` or `@gittensory blockers` for the deterministic readiness facts."], + }; + } +} + +function chatAnswerProseLines(text: string): string[] { + const lines = text + .split(/\r?\n+/) + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .slice(0, 6) + .map((line) => neutralizePublicMarkdownText(sanitizePublicComment(line))); + return lines.length > 0 ? lines : ["The grounded answer was empty after sanitization. Run `@gittensory preflight` for the deterministic readiness facts."]; +} + function renderPublicAnswerCard(card: PublicAnswerCard): string[] { const lines = [ `**${sanitizePublicComment(card.title)}**`, "", `- ${sanitizePublicComment(card.summary)}`, + ...(card.disclaimer ? ["", `> ${sanitizePublicComment(card.disclaimer)}`] : []), "", "**Findings**", "", @@ -540,6 +641,8 @@ function commandSummary(command: GittensoryMentionCommandName): string { return "Available public commands and their safest use on a PR thread."; case "ask": return "Contribution-context Q&A from connected cached sources, scoped to contribution quality and repository policy."; + case "chat": + return "Grounded natural-prose answer sourced from the same cached decision-pack facts, via local advisory inference (read-only)."; case "miner-context": return "Public miner context from official Gittensor data when available."; case "preflight": @@ -588,6 +691,10 @@ function commandEvidence( evidence.push("Answer scope is limited to contribution quality and repository policy."); evidence.push("Sources are cited with freshness and public-boundary redaction."); } + if (command === "chat") { + evidence.push("Answer is a natural-prose rewrite of the same cached decision-pack facts, adding no new claims."); + evidence.push("Generated by local advisory inference; it never reaches the frontier model or any write/action path."); + } if (command === "miner-context") { evidence.push(officialMiner ? "Official Gittensor miner context was available." : "Official Gittensor miner context was unavailable."); } @@ -612,6 +719,8 @@ function commandNextActions(command: GittensoryMentionCommandName, bundle: Agent return ["Comment one listed command on the PR thread when more context is needed."]; case "ask": return ["Ask one concrete contribution-quality question per command for clearer cited guidance."]; + case "chat": + return ["Ask one concrete question; chat rewrites the same cached decision-pack facts and cannot change review outcomes or trigger a re-review."]; case "miner-context": return ["Use MCP or the authenticated control panel for private contributor planning."]; case "preflight": @@ -659,6 +768,8 @@ function commandSourceNotes( ? "static command catalog" : command === "ask" ? askCommandSourceSummary(bundle) + : command === "chat" + ? "cached decision-pack facts rewritten by local advisory inference" : command === "miner-context" ? officialMiner ? "official Gittensor miner API" @@ -721,6 +832,10 @@ function commandSections( return helpSections(env, unknownVerb); case "ask": return askSections(bundle, question); + case "chat": + // chat renders a bespoke LLM-answer card (buildChatPublicAnswerCard) that ignores these sections; the case + // only keeps the exhaustive switch total, mirroring how ask's sections are discarded by buildPublicAnswerCard. + return ["**Grounded chat Q&A**"]; case "miner-context": return minerContextSections(officialMiner); case "preflight": @@ -784,6 +899,7 @@ function helpSections(env: GittensoryFooterEnv, unknownVerb?: string | undefined ...buildDidYouMeanSections(unknownVerb, suggestCommand), "- `@gittensory help` shows this command list.", "- `@gittensory ask ` answers contribution-quality Q&A with source citations and freshness.", + "- `@gittensory chat ` answers in natural prose from cached decision-pack facts via local inference (maintainer/collaborator; read-only).", "- `@gittensory preflight` summarizes public PR hygiene.", "- `@gittensory blockers` explains public readiness blockers.", "- `@gittensory duplicate-check` summarizes duplicate/WIP caution.", @@ -1714,4 +1830,5 @@ export const githubCommandsInternals = { helpSections, actionCommandHelpSections, commandReferenceUrl, + commandNextActions, }; diff --git a/src/openapi/schemas.ts b/src/openapi/schemas.ts index 98c0459b53..b6454f1e42 100644 --- a/src/openapi/schemas.ts +++ b/src/openapi/schemas.ts @@ -759,6 +759,11 @@ export const RepositorySettingsSchema = z e2eTestGen: z.boolean(), planner: z.boolean(), summaries: z.boolean(), + chatQa: z + .boolean() + .describe( + "Opt the `@gittensory chat ` grounded Q&A surface (#4595) into local Ollama inference. Ollama-only: unlike the four capabilities above it NEVER falls back to the frontier env.AI when off; it declines unless this is true AND env.AI_ADVISORY is configured. Co-requisite: set `commandRateLimitPolicy` to `hold` (it defaults to `off` fleet-wide) so the tighter `commandRateLimitAiMaxPerWindow` ceiling actually throttles this cost-bearing command.", + ), }) .optional(), gittensorLabel: z.string(), diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 828e74ff71..68ab097ef3 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -545,6 +545,7 @@ import { shouldEmitFixHandoff } from "../review/fix-handoff"; import { buildFixHandoffBlocks } from "../review/fix-handoff-render"; import { buildE2eTestGenCommentBody, type E2eTestGenCommitOutcome } from "../review/e2e-test-gen-render"; import { resolveE2eTestGenInstructions, runGittensoryE2eTestGeneration } from "../services/ai-e2e-test-gen"; +import { generateChatQaAnswer } from "../services/ai-chat-qa"; import { commitE2eTestToPrBranch } from "../github/e2e-test-commit"; import { shouldApplyRepoCultureProfile } from "../review/repo-culture-profile-wire"; import { applyReviewMemorySuppression, getCachedReviewSuppressions, invalidateReviewSuppressionCache, shouldApplyReviewMemory } from "../review/review-memory-wire"; @@ -12477,6 +12478,21 @@ async function maybeProcessGittensoryMentionCommand( }, command.question, ); + // #4595: resolved BEFORE the (synchronous) card renderer, mirroring how `bundle` above is fetched first -- + // generateChatQaAnswer is Ollama-only and never falls back to the frontier chain (a hard requirement, unlike + // the other four advisoryAiRouting capabilities), so no withAdvisoryAiEnv() swap belongs here. + const chatAnswer = + command.name === "chat" + ? await generateChatQaAnswer(env, { + bundle, + question: command.question, + advisoryAiRouting: settings.advisoryAiRouting, + repoFullName, + issueNumber: issue.number, + actor: commenter, + route: "github_app.chat_qa", + }) + : null; const body = buildPublicAgentCommandComment({ command, repo, @@ -12488,6 +12504,7 @@ async function maybeProcessGittensoryMentionCommand( officialMiner: official?.status === "confirmed" ? official.snapshot : null, bundle, maintainerDigest, + chatAnswer, env, }); const responseComment = await createOrUpdateAgentCommandComment( @@ -12643,8 +12660,8 @@ async function buildMentionCommandBundle( repoFullName: context.repoFullName, surface: "github_comment", objective: - commandName === "ask" && question && question.trim().length > 0 - ? `Respond to @gittensory ask for ${context.repoFullName}#${context.issue.number}. Question: ${question.trim().slice(0, 280)}` + (commandName === "ask" || commandName === "chat") && question && question.trim().length > 0 + ? `Respond to @gittensory ${commandName} for ${context.repoFullName}#${context.issue.number}. Question: ${question.trim().slice(0, 280)}` : `Respond to @gittensory ${commandName} for ${context.repoFullName}#${context.issue.number}.`, }); } diff --git a/src/review/advisory-ai-routing-config.ts b/src/review/advisory-ai-routing-config.ts index b63dbe356c..e6032cbedf 100644 --- a/src/review/advisory-ai-routing-config.ts +++ b/src/review/advisory-ai-routing-config.ts @@ -5,6 +5,7 @@ export const DEFAULT_ADVISORY_AI_ROUTING: AdvisoryAiRoutingConfig = { e2eTestGen: false, planner: false, summaries: false, + chatQa: false, }; function normalizeField(value: unknown, field: keyof AdvisoryAiRoutingConfig, warnings: string[]): boolean { @@ -31,5 +32,6 @@ export function normalizeAdvisoryAiRoutingConfig(input: unknown, warnings: strin e2eTestGen: normalizeField(record.e2eTestGen, "e2eTestGen", warnings), planner: normalizeField(record.planner, "planner", warnings), summaries: normalizeField(record.summaries, "summaries", warnings), + chatQa: normalizeField(record.chatQa, "chatQa", warnings), }; } diff --git a/src/services/ai-chat-qa.ts b/src/services/ai-chat-qa.ts new file mode 100644 index 0000000000..67ec63f54f --- /dev/null +++ b/src/services/ai-chat-qa.ts @@ -0,0 +1,274 @@ +import { recordAiUsageEvent, recordAuditEvent, sumAiEstimatedNeuronsSince } from "../db/repositories"; +import { sanitizePublicComment } from "../queue-intelligence"; +import type { AdvisoryAiRoutingConfig } from "../types"; +import type { AgentRunBundle } from "./agent-orchestrator"; + +// Grounded @gittensory chat LLM Q&A (#4595), powered ENTIRELY by local Ollama (env.AI_ADVISORY). +// +// This is modeled on summarizeAgentBundleWithAi / rewriteSignalBundleWithAi (src/services/ai-summaries.ts): it +// reuses their enable-flag-check → shared-neuron-budget-gate → provider-call → guaranteed-safe-fallback shape. +// It deliberately does NOT import that module (nor github/commands, nor any action-command handler), so this +// generation surface can never reach a write/action path -- the isolation asserted by +// test/unit/ai-chat-qa-import-isolation.test.ts (#4595 requirement 10). It only narrowly rewrites the +// ALREADY-deterministic decision-pack facts in the bundle (PR verdict, which checks/findings are blocking, +// what a finding means) into natural prose; it never synthesizes new claims. +// +// Ollama-ONLY, by hard requirement (#4595 requirement 5): unlike the four sibling advisoryAiRouting +// capabilities (slop/e2eTestGen/planner/summaries), which silently fall back to the shared frontier env.AI when +// their flag is off, this surface NEVER touches the frontier. It declines whenever advisoryAiRouting.chatQa is +// not true or env.AI_ADVISORY is unconfigured -- it does not call withAdvisoryAiEnv(env, false) and let a +// frontier token be spent. + +export type ChatQaResult = + | { status: "disabled"; reason: string } + | { status: "unavailable"; reason: string } + | { status: "declined"; reason: string; suggestion: string } + | { status: "quota_exceeded"; model: string; estimatedNeurons: number; remainingBudget: number } + | { status: "unsafe"; model: string; estimatedNeurons: number; reason: string } + | { status: "error"; model: string; estimatedNeurons: number; reason: string } + | { status: "ok"; model: string; estimatedNeurons: number; text: string }; + +export type ChatQaRequest = { + bundle: AgentRunBundle | null | undefined; + question: string | undefined; + /** Resolved repository settings' `advisoryAiRouting` block; `chatQa === true` is the enable gate. */ + advisoryAiRouting: AdvisoryAiRoutingConfig | undefined; + repoFullName: string; + issueNumber: number; + actor?: string | null | undefined; + route?: string | null | undefined; +}; + +/** The existing deterministic command a declined answer points the reader at, rather than guessing (#4595 req 3). */ +export const CHAT_QA_FALLBACK_COMMAND = "@gittensory preflight"; + +const CHAT_QA_SYSTEM_PROMPT = + "You are answering a contributor's question about a GitHub pull request using ONLY the deterministic Gittensory " + + "facts provided in the user message. Restate and explain those facts in clear, friendly prose (under 6 sentences). " + + "Do not invent facts, do not claim a guaranteed outcome, and never mention rewards, rankings, payouts, wallets, " + + "hotkeys, raw or estimated trust scores, scoreability, or reviewability. If the provided facts do not answer the " + + "question, say so plainly and suggest running `@gittensory preflight` or `@gittensory blockers`."; + +// Private decision-pack blocker codes and boundary terms are redacted (not thrown on) before the grounding +// bundle is ever put in a prompt -- publicSafeSummary is already public-safe, but raw `blockedBy`/`why` can +// carry these. Mirrors github/commands.ts's publicBlockerDetail redaction intent without importing it. +const PRIVATE_DECISION_BLOCKER_PATTERN = + /\b(?:open_pr_pressure|closed_pr_credibility|low_credibility|maintainer_lane|inactive_or_unknown_lane|issue_discovery_only|merged_pr_history_floor|issue_discovery_validity_floor)\b/gi; +const PRIVATE_BOUNDARY_TERM_PATTERN = + /\b(?:wallets?|hotkeys?|coldkeys?|seed phrases?|mnemonics?|raw trust scores?|trust scores?|scoreability|reviewability|payouts?|rewards?|reward estimates?|farming|rankings?)\b/gi; + +// Public-safe forbidden-term guard on the MODEL's OWN output, mirroring ai-summaries' containsPublicForbiddenText: +// near-miss phrasings the throwing word-list validator narrows (e.g. bare "estimated score") are also caught. +const PUBLIC_FORBIDDEN_TEXT_PATTERN = + /\b(wallets?|hotkeys?|coldkeys?|seed phrases?|mnemonics?|raw trust scores?|trust scores?|estimated scores?|score estimates?|scoreability|score preview|public score estimates?|estimated rewards?|rewards?|reward estimates?|payouts?|farming|reviewability(?: internals?)?|private reviewability|private scoreability|private rankings?|rankings?|reward optimization)\b/i; + +type ChatGroundingAction = { + actionType: string; + status: string; + publicSafeSummary: string; + why: string[]; + blockedBy: string[]; +}; + +type ChatGroundingBundle = { + objective: string; + status: string; + dataQualityStatus: string; + summary: string; + actions: ChatGroundingAction[]; + freshnessWarnings: string[]; +}; + +export async function generateChatQaAnswer(env: Env, req: ChatQaRequest): Promise { + // (#4595 req 5) Ollama-only enablement. BOTH gates are hard declines, never a frontier fallback. + if (req.advisoryAiRouting?.chatQa !== true) { + return { status: "disabled", reason: "Chat Q&A is not enabled on this instance (settings.advisoryAiRouting.chatQa is off)." }; + } + if (!env.AI_ADVISORY) { + return { + status: "unavailable", + reason: "Local advisory inference (env.AI_ADVISORY) is not configured; chat Q&A never falls back to the frontier model.", + }; + } + + // (#4595 req 3) Decline rather than guess when there is nothing deterministic to ground an answer in. + const question = req.question?.trim(); + if (!question) { + return { + status: "declined", + reason: "No question was supplied.", + suggestion: "Ask a specific question, for example `@gittensory chat why is this PR blocked?`.", + }; + } + if (!req.bundle || req.bundle.run.status === "needs_snapshot_refresh") { + return { + status: "declined", + reason: "The cached contribution-context snapshot is still refreshing.", + suggestion: `Try again shortly, or run \`${CHAT_QA_FALLBACK_COMMAND}\` for the deterministic readiness facts.`, + }; + } + const grounding = compactChatSignalBundle(req.bundle); + if (grounding.actions.length === 0) { + return { + status: "declined", + reason: "No cached deterministic facts are available to ground an answer for this PR.", + suggestion: `Run \`${CHAT_QA_FALLBACK_COMMAND}\` or \`@gittensory blockers\` for the deterministic readiness facts.`, + }; + } + + // Empty string (not a Workers-AI `@cf/...` id): the advisory provider's own per-provider default wins when no + // override is set. Mirrors ai-summaries.ts. + const model = env.WORKERS_AI_SUMMARY_MODEL || ""; + const maxOutputTokens = clampNumber(Number(env.AI_MAX_OUTPUT_TOKENS || 256), 64, 512); + const prompt = buildChatPrompt(question, grounding); + const estimatedNeurons = estimateNeurons(prompt, maxOutputTokens); + // Shared daily neuron budget: the SAME counter every AI feature sums into (ai-review / ai-slop / ai-summaries, + // #1369). Default HIGH (10M) and clamp to 10M so chat Q&A never starves — or is starved by — the shared pool. + const rawNeuronBudget = Number(env.AI_DAILY_NEURON_BUDGET); + const budget = clampNumber(env.AI_DAILY_NEURON_BUDGET && Number.isFinite(rawNeuronBudget) ? rawNeuronBudget : 10_000_000, 0, 10_000_000); + const used = await sumAiEstimatedNeuronsSince(env, utcDayStartIso()); + const remainingBudget = Math.max(0, budget - used); + if (estimatedNeurons > remainingBudget) { + await recordChatAi(env, req, { + model, + status: "quota_exceeded", + estimatedNeurons: 0, + detail: `estimated ${estimatedNeurons} neurons exceeds remaining budget ${remainingBudget}`, + }); + return { status: "quota_exceeded", model, estimatedNeurons, remainingBudget }; + } + + try { + const response = await env.AI_ADVISORY.run(model, { + messages: [ + { role: "system", content: CHAT_QA_SYSTEM_PROMPT }, + { role: "user", content: prompt }, + ], + max_tokens: maxOutputTokens, + temperature: 0.1, + }); + const rawText = extractAiText(response); + if (!rawText) throw new Error("empty_chat_answer"); + if (containsPublicForbiddenText(rawText)) { + await recordChatAi(env, req, { model, status: "unsafe", estimatedNeurons, detail: "chat answer failed public sanitizer" }); + return { status: "unsafe", model, estimatedNeurons, reason: "chat answer failed public sanitizer" }; + } + await recordChatAi(env, req, { model, status: "ok", estimatedNeurons, detail: "chat answer generated" }); + return { status: "ok", model, estimatedNeurons, text: rawText.trim() }; + } catch (error) { + const reason = error instanceof Error ? error.message : "chat_answer_failed"; + await recordChatAi(env, req, { model, status: "error", estimatedNeurons: 0, detail: reason }); + return { status: "error", model, estimatedNeurons, reason }; + } +} + +function compactChatSignalBundle(bundle: AgentRunBundle): ChatGroundingBundle { + return { + objective: redactGroundingText(bundle.run.objective), + status: bundle.run.status, + dataQualityStatus: bundle.run.dataQualityStatus, + summary: redactGroundingText(bundle.summary), + actions: bundle.actions.slice(0, 5).map((action) => ({ + actionType: action.actionType, + status: action.status, + publicSafeSummary: redactGroundingText(action.publicSafeSummary), + why: action.why.slice(0, 4).map(redactGroundingText).filter((line) => line.length > 0), + blockedBy: action.blockedBy.slice(0, 4).map(redactGroundingText).filter((line) => line.length > 0), + })), + freshnessWarnings: bundle.contextSnapshots.flatMap((snapshot) => snapshot.freshnessWarnings).slice(0, 8).map(redactGroundingText), + }; +} + +function redactGroundingText(value: string): string { + return value + .replace(/\blikely_duplicate\b/gi, "possible overlap with existing work") + .replace(PRIVATE_DECISION_BLOCKER_PATTERN, "private readiness context") + .replace(PRIVATE_BOUNDARY_TERM_PATTERN, "private context") + .trim(); +} + +function buildChatPrompt(question: string, grounding: ChatGroundingBundle): string { + return [ + `Contributor question: ${question}`, + "Deterministic Gittensory facts for this pull request (answer using only these):", + JSON.stringify(grounding), + ].join("\n"); +} + +function containsPublicForbiddenText(value: string): boolean { + // The queue-intelligence sanitizePublicComment THROWS on any forbidden public word; treat a throw as a fail. + try { + sanitizePublicComment(value); + } catch { + return true; + } + return PUBLIC_FORBIDDEN_TEXT_PATTERN.test(value); +} + +function estimateNeurons(prompt: string, maxOutputTokens: number): number { + const inputTokens = Math.ceil(prompt.length / 4); + return Math.max(1, Math.ceil((inputTokens + maxOutputTokens) * 0.035)); +} + +function extractAiText(response: unknown): string { + if (typeof response === "string") return response; + if (!response || typeof response !== "object") return ""; + const record = response as Record; + if (typeof record.response === "string") return record.response; + if (typeof record.text === "string") return record.text; + if (typeof record.result === "string") return record.result; + return ""; +} + +function clampNumber(value: number, min: number, max: number): number { + if (!Number.isFinite(value)) return min; + return Math.min(max, Math.max(min, Math.floor(value))); +} + +function utcDayStartIso(): string { + const now = new Date(); + return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())).toISOString(); +} + +function auditOutcomeForAiStatus(status: string): "success" | "denied" | "error" | "completed" { + if (status === "ok") return "success"; + if (status === "quota_exceeded" || status === "unsafe") return "denied"; + if (status === "error") return "error"; + return "completed"; +} + +async function recordChatAi( + env: Env, + req: ChatQaRequest, + event: { model: string; status: string; estimatedNeurons: number; detail: string }, +): Promise { + await recordAiUsageEvent(env, { + feature: "chat_qa", + actor: req.actor, + route: req.route, + model: event.model, + status: event.status, + estimatedNeurons: event.estimatedNeurons, + detail: event.detail, + metadata: { repoFullName: req.repoFullName, issueNumber: req.issueNumber }, + }); + await recordAuditEvent(env, { + eventType: "ai.chat_qa", + actor: req.actor, + route: req.route, + outcome: auditOutcomeForAiStatus(event.status), + detail: event.detail, + metadata: { repoFullName: req.repoFullName, issueNumber: req.issueNumber, model: event.model, estimatedNeurons: event.estimatedNeurons }, + }); +} + +/** @internal Exported for unit tests of the pure chat-Q&A helpers. */ +export const __chatQaInternals = { + compactChatSignalBundle, + redactGroundingText, + buildChatPrompt, + containsPublicForbiddenText, + estimateNeurons, + extractAiText, + auditOutcomeForAiStatus, +}; diff --git a/src/settings/command-authorization.ts b/src/settings/command-authorization.ts index 84d935c32f..1dcc1e8ae0 100644 --- a/src/settings/command-authorization.ts +++ b/src/settings/command-authorization.ts @@ -14,6 +14,12 @@ export const DEFAULT_COMMAND_AUTHORIZATION_POLICY: RepositoryCommandAuthorizatio "noise-report": ["maintainer", "collaborator"], "gate-override": ["maintainer", "collaborator"], plan: ["maintainer", "collaborator"], + // #4595: deliberately narrower than "ask"'s default (which allows confirmed_miner) -- chat is Ollama-only + // grounded LLM generation, a materially larger surface than ask's deterministic-only answer, so v1 starts + // maintainer/collaborator-only. Explicit registration here (rather than falling through to `default`) also + // activates the pr_author-widening guard below, so a self-hoster can't accidentally yml themselves into + // "anyone commenting on their own PR" without it. + chat: ["maintainer", "collaborator"], // #1960 PR control-surface verbs. "review" is deliberately widenable to confirmed_miner (same self-rerun // precedent already applied to review-now, #824) — a confirmed miner may re-trigger review on their own PR. // The rest (pause/resume/resolve/configuration/explain) are conservative maintainer/collaborator-only diff --git a/src/signals/focus-manifest.ts b/src/signals/focus-manifest.ts index 82e754c4d1..e52c6cd4d2 100644 --- a/src/signals/focus-manifest.ts +++ b/src/signals/focus-manifest.ts @@ -598,6 +598,7 @@ export function resolveEffectiveSettings( e2eTestGen: advisoryAiRoutingOverride.e2eTestGen ?? base.e2eTestGen, planner: advisoryAiRoutingOverride.planner ?? base.planner, summaries: advisoryAiRoutingOverride.summaries ?? base.summaries, + chatQa: advisoryAiRoutingOverride.chatQa ?? base.chatQa, }; } applyGateConfigOverrides(effective, manifest.gate); diff --git a/src/types.ts b/src/types.ts index 20133512c8..d9235bc316 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1275,16 +1275,24 @@ export type UnlinkedIssueGuardrailConfig = { minConfidence: number; }; -/** Per-capability opt-in to the local-inference AI_ADVISORY binding (#4364): each of these four ADVISORY-ONLY +/** Per-capability opt-in to the local-inference AI_ADVISORY binding (#4364): each of these ADVISORY-ONLY * (never gate-blocking) capabilities independently decides whether it routes through env.AI_ADVISORY (when * configured) instead of the shared frontier env.AI chain. Config-as-code only, `.gittensory.yml * settings.advisoryAiRouting` (global default in shared/root config, per-repo override); defaults all-false - * so an operator must deliberately opt each capability in. */ + * so an operator must deliberately opt each capability in. + * + * `chatQa` (#4595) is the ONE capability that does NOT share the others' silent-frontier fallback: the four + * cost-optimizing capabilities above quietly fall back to the shared frontier env.AI when their flag is off, + * but the `@gittensory chat` grounded Q&A surface is "Ollama only" -- it declines/skips whenever + * `chatQa !== true` or `env.AI_ADVISORY` is unconfigured rather than ever spending a frontier token. */ export type AdvisoryAiRoutingConfig = { slop: boolean; e2eTestGen: boolean; planner: boolean; summaries: boolean; + /** Grounded `@gittensory chat ` LLM Q&A (#4595). Ollama-only: unlike the four fields above it NEVER + * falls back to the frontier env.AI when off -- it simply declines. Default false. */ + chatQa: boolean; }; /** A blocked contributor (#1425, anti-abuse): a GitHub `login` plus optional maintainer metadata. The converged diff --git a/test/unit/advisory-ai-routing-call-sites.test.ts b/test/unit/advisory-ai-routing-call-sites.test.ts index c4fd61789e..d0aa430882 100644 --- a/test/unit/advisory-ai-routing-call-sites.test.ts +++ b/test/unit/advisory-ai-routing-call-sites.test.ts @@ -35,7 +35,7 @@ describe("runAiSlopForAdvisory routes through AI_ADVISORY (#4364)", () => { }); await runAiSlopForAdvisory(env, { mode: "live", - settings: settingsFixture({ slop: true, e2eTestGen: false, planner: false, summaries: false }), + settings: settingsFixture({ slop: true, e2eTestGen: false, planner: false, summaries: false, chatQa: false }), advisory, repoFullName: "owner/repo", pr: { number: 1, title: "t" }, @@ -80,7 +80,7 @@ describe("runAiSlopForAdvisory routes through AI_ADVISORY (#4364)", () => { const env = createTestEnv({ AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true", AI: { run: frontierRun } as unknown as Ai }); await runAiSlopForAdvisory(env, { mode: "live", - settings: settingsFixture({ slop: true, e2eTestGen: false, planner: false, summaries: false }), + settings: settingsFixture({ slop: true, e2eTestGen: false, planner: false, summaries: false, chatQa: false }), advisory, repoFullName: "owner/repo", pr: { number: 3, title: "t" }, diff --git a/test/unit/advisory-ai-routing-config-engine.test.ts b/test/unit/advisory-ai-routing-config-engine.test.ts index f67344d843..752b7058b3 100644 --- a/test/unit/advisory-ai-routing-config-engine.test.ts +++ b/test/unit/advisory-ai-routing-config-engine.test.ts @@ -11,22 +11,23 @@ describe("normalizeAdvisoryAiRoutingConfig", () => { it("normalizes a fully-valid config", () => { const warnings: string[] = []; - expect(normalizeAdvisoryAiRoutingConfig({ slop: true, e2eTestGen: true, planner: true, summaries: true }, warnings)).toEqual({ + expect(normalizeAdvisoryAiRoutingConfig({ slop: true, e2eTestGen: true, planner: true, summaries: true, chatQa: true }, warnings)).toEqual({ slop: true, e2eTestGen: true, planner: true, summaries: true, + chatQa: true, }); expect(warnings).toEqual([]); }); - it.each(["slop", "e2eTestGen", "planner", "summaries"] as const)("defaults %s to false when omitted", (field) => { + it.each(["slop", "e2eTestGen", "planner", "summaries", "chatQa"] as const)("defaults %s to false when omitted", (field) => { const warnings: string[] = []; expect(normalizeAdvisoryAiRoutingConfig({}, warnings)[field]).toBe(false); expect(warnings).toEqual([]); }); - it.each(["slop", "e2eTestGen", "planner", "summaries"] as const)("falls back to false and warns on a non-boolean %s", (field) => { + it.each(["slop", "e2eTestGen", "planner", "summaries", "chatQa"] as const)("falls back to false and warns on a non-boolean %s", (field) => { const warnings: string[] = []; const cfg = normalizeAdvisoryAiRoutingConfig({ [field]: "yes" }, warnings); expect(cfg[field]).toBe(false); diff --git a/test/unit/advisory-ai-routing-config.test.ts b/test/unit/advisory-ai-routing-config.test.ts index 2b777eb9e8..d9db0f038d 100644 --- a/test/unit/advisory-ai-routing-config.test.ts +++ b/test/unit/advisory-ai-routing-config.test.ts @@ -10,22 +10,23 @@ describe("normalizeAdvisoryAiRoutingConfig", () => { it("normalizes a fully-valid config", () => { const warnings: string[] = []; - expect(normalizeAdvisoryAiRoutingConfig({ slop: true, e2eTestGen: true, planner: true, summaries: true }, warnings)).toEqual({ + expect(normalizeAdvisoryAiRoutingConfig({ slop: true, e2eTestGen: true, planner: true, summaries: true, chatQa: true }, warnings)).toEqual({ slop: true, e2eTestGen: true, planner: true, summaries: true, + chatQa: true, }); expect(warnings).toEqual([]); }); - it.each(["slop", "e2eTestGen", "planner", "summaries"] as const)("defaults %s to false when omitted", (field) => { + it.each(["slop", "e2eTestGen", "planner", "summaries", "chatQa"] as const)("defaults %s to false when omitted", (field) => { const warnings: string[] = []; expect(normalizeAdvisoryAiRoutingConfig({}, warnings)[field]).toBe(false); expect(warnings).toEqual([]); }); - it.each(["slop", "e2eTestGen", "planner", "summaries"] as const)("falls back to false and warns on a non-boolean %s", (field) => { + it.each(["slop", "e2eTestGen", "planner", "summaries", "chatQa"] as const)("falls back to false and warns on a non-boolean %s", (field) => { const warnings: string[] = []; const cfg = normalizeAdvisoryAiRoutingConfig({ [field]: "yes" }, warnings); expect(cfg[field]).toBe(false); @@ -35,7 +36,7 @@ describe("normalizeAdvisoryAiRoutingConfig", () => { it("normalizes one valid field alongside one invalid field independently", () => { const warnings: string[] = []; const cfg = normalizeAdvisoryAiRoutingConfig({ slop: true, planner: "nope" }, warnings); - expect(cfg).toEqual({ slop: true, e2eTestGen: false, planner: false, summaries: false }); + expect(cfg).toEqual({ slop: true, e2eTestGen: false, planner: false, summaries: false, chatQa: false }); expect(warnings).toEqual([`settings.advisoryAiRouting.planner must be a boolean; using the default "false".`]); }); diff --git a/test/unit/ai-chat-qa-import-isolation.test.ts b/test/unit/ai-chat-qa-import-isolation.test.ts new file mode 100644 index 0000000000..fd5d06d2c7 --- /dev/null +++ b/test/unit/ai-chat-qa-import-isolation.test.ts @@ -0,0 +1,84 @@ +import { readFileSync, statSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +// #4595 req 10: `ai-chat-qa.ts` must never be able to reach a write/action-command handler or an +// undeclared DB-mutation helper -- it only ever rewrites already-deterministic cached facts into prose. +// Both `maybeProcessReviewCommand`/`maybeProcessGateOverrideCommand`/`maybeProcessPauseCommand` live, +// un-exported, in src/queue/processors.ts, so the only way this module could ever reach them is by +// importing that file (or the command dispatcher/catalog in src/github/commands.ts) directly -- which +// this test forbids at the import-specifier level, independent of what those files currently export. + +const root = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); +const srcRoot = join(root, "src"); +const CHAT_QA_MODULE = join(srcRoot, "services/ai-chat-qa.ts"); + +const FORBIDDEN_MODULES = [join(srcRoot, "queue/processors.ts"), join(srcRoot, "github/commands.ts")]; + +// The only db/repositories exports this module may use: the AI-usage/audit-event recorders and the +// shared daily neuron-budget reader. Any other name (e.g. an issue/PR/comment upsert) would mean this +// "advisory rewrite" module gained a write capability beyond recording its own usage. +const ALLOWED_DB_REPOSITORY_IMPORTS = new Set(["recordAiUsageEvent", "recordAuditEvent", "sumAiEstimatedNeuronsSince"]); + +function resolveLocalImport(fromFile: string, specifier: string): string | null { + if (!specifier.startsWith(".")) return null; + const base = dirname(fromFile); + const candidates = [join(base, specifier), join(base, `${specifier}.ts`), join(base, `${specifier}.tsx`), join(base, specifier, "index.ts")]; + for (const candidate of candidates) { + try { + statSync(candidate); + return candidate; + } catch { + // try next candidate + } + } + return null; +} + +function parseImportSpecifiers(filePath: string): string[] { + const content = readFileSync(filePath, "utf8"); + const specifiers = new Set(); + for (const match of content.matchAll(/(?:import|export)\s+[\s\S]*?\sfrom\s+["']([^"']+)["']/g)) { + specifiers.add(match[1]!); + } + for (const match of content.matchAll(/import\s*\(\s*["']([^"']+)["']\s*\)/g)) { + specifiers.add(match[1]!); + } + return [...specifiers]; +} + +function namedImportsFrom(filePath: string, specifier: string): string[] { + const content = readFileSync(filePath, "utf8"); + const escaped = specifier.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const match = content.match(new RegExp(`import\\s+(?:type\\s+)?\\{([^}]*)\\}\\s+from\\s+["']${escaped}["']`)); + if (!match?.[1]) return []; + return match[1] + .split(",") + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0) + .map((entry) => entry.replace(/^type\s+/, "").split(/\s+as\s+/)[0]!.trim()); +} + +describe("ai-chat-qa.ts import isolation (#4595 req 10)", () => { + it("never directly imports the action-command dispatcher or handler modules", () => { + const specifiers = parseImportSpecifiers(CHAT_QA_MODULE); + const resolved = specifiers.map((specifier) => resolveLocalImport(CHAT_QA_MODULE, specifier)).filter((path): path is string => path !== null); + const forbiddenHits = resolved.filter((path) => FORBIDDEN_MODULES.includes(path)); + expect(forbiddenHits, `ai-chat-qa.ts must not import: ${forbiddenHits.join(", ")}`).toEqual([]); + }); + + it("only imports the allow-listed db/repositories helpers (no undeclared DB-mutation capability)", () => { + const imported = namedImportsFrom(CHAT_QA_MODULE, "../db/repositories"); + expect(imported.length).toBeGreaterThan(0); + const disallowed = imported.filter((name) => !ALLOWED_DB_REPOSITORY_IMPORTS.has(name)); + expect(disallowed, `ai-chat-qa.ts imported an unexpected db/repositories helper: ${disallowed.join(", ")}`).toEqual([]); + }); + + it("does not reference the action-command handler function names anywhere in its source", () => { + const content = readFileSync(CHAT_QA_MODULE, "utf8"); + for (const name of ["maybeProcessReviewCommand", "maybeProcessGateOverrideCommand", "maybeProcessPauseCommand"]) { + expect(content, `ai-chat-qa.ts must not reference ${name}`).not.toContain(name); + } + }); +}); diff --git a/test/unit/ai-chat-qa.test.ts b/test/unit/ai-chat-qa.test.ts new file mode 100644 index 0000000000..1530f64ee4 --- /dev/null +++ b/test/unit/ai-chat-qa.test.ts @@ -0,0 +1,312 @@ +import { describe, expect, it, vi } from "vitest"; +import { __chatQaInternals, CHAT_QA_FALLBACK_COMMAND, generateChatQaAnswer } from "../../src/services/ai-chat-qa"; +import type { AgentRunBundle } from "../../src/services/agent-orchestrator"; +import { createTestEnv } from "../helpers/d1"; + +const ADVISORY_ON = { slop: false, e2eTestGen: false, planner: false, summaries: false, chatQa: true }; +const ADVISORY_OFF = { slop: false, e2eTestGen: false, planner: false, summaries: false, chatQa: false }; + +function bundleFixture(runOverrides?: Partial, actionOverrides?: Partial): AgentRunBundle { + return { + run: { + id: "run-chat", + objective: "Respond to @gittensory chat for owner/repo#1", + actorLogin: "octofeesh1", + surface: "github_comment", + mode: "copilot", + status: "completed", + dataQualityStatus: "complete", + payload: {}, + createdAt: "2026-07-11T00:00:00.000Z", + updatedAt: "2026-07-11T00:00:00.000Z", + ...runOverrides, + }, + actions: [ + { + id: "action-chat", + runId: "run-chat", + actionType: "cleanup_existing_prs", + status: "recommended", + recommendation: "Clean up open PR pressure before opening new work.", + why: ["Open PR pressure blocks current scoreability.", " ", "Mentions a wallet that must be redacted."], + blockedBy: ["open_pr_pressure"], + publicSafeSummary: "Clean up open PR pressure before opening new work.", + approvalRequired: true, + safetyClass: "private", + payload: {}, + createdAt: "2026-07-11T00:00:00.000Z", + ...actionOverrides, + }, + ], + contextSnapshots: [ + { + id: "ctx-chat", + runId: "run-chat", + repoSignalSnapshotIds: [], + freshnessWarnings: ["fresh enough"], + payload: {}, + createdAt: "2026-07-11T00:00:00.000Z", + }, + ], + summary: "likely_duplicate of an existing open PR.", + }; +} + +describe("generateChatQaAnswer", () => { + it("declines when chatQa is off (does not call the advisory provider)", async () => { + const advisoryRun = vi.fn(); + const env = createTestEnv({ AI_ADVISORY: { run: advisoryRun } as unknown as Ai }); + const result = await generateChatQaAnswer(env, { + bundle: bundleFixture(), + question: "why is this blocked?", + advisoryAiRouting: ADVISORY_OFF, + repoFullName: "owner/repo", + issueNumber: 1, + }); + expect(result).toEqual({ status: "disabled", reason: "Chat Q&A is not enabled on this instance (settings.advisoryAiRouting.chatQa is off)." }); + expect(advisoryRun).not.toHaveBeenCalled(); + }); + + it("declines when advisoryAiRouting is undefined entirely", async () => { + const env = createTestEnv({}); + const result = await generateChatQaAnswer(env, { + bundle: bundleFixture(), + question: "why is this blocked?", + advisoryAiRouting: undefined, + repoFullName: "owner/repo", + issueNumber: 1, + }); + expect(result.status).toBe("disabled"); + }); + + it("never falls back to the frontier chain: reports unavailable when chatQa is on but AI_ADVISORY is unconfigured", async () => { + const frontierRun = vi.fn(); + const env = createTestEnv({ AI: { run: frontierRun } as unknown as Ai }); + const result = await generateChatQaAnswer(env, { + bundle: bundleFixture(), + question: "why is this blocked?", + advisoryAiRouting: ADVISORY_ON, + repoFullName: "owner/repo", + issueNumber: 1, + }); + expect(result).toMatchObject({ status: "unavailable" }); + expect(frontierRun).not.toHaveBeenCalled(); + }); + + it("declines when no question is supplied", async () => { + const advisoryRun = vi.fn(); + const env = createTestEnv({ AI_ADVISORY: { run: advisoryRun } as unknown as Ai }); + const result = await generateChatQaAnswer(env, { + bundle: bundleFixture(), + question: " ", + advisoryAiRouting: ADVISORY_ON, + repoFullName: "owner/repo", + issueNumber: 1, + }); + expect(result).toMatchObject({ status: "declined", reason: "No question was supplied.", suggestion: expect.stringContaining("@gittensory chat") }); + expect(advisoryRun).not.toHaveBeenCalled(); + }); + + it("declines and points at the fallback command when there is no bundle at all", async () => { + const env = createTestEnv({ AI_ADVISORY: { run: vi.fn() } as unknown as Ai }); + const result = await generateChatQaAnswer(env, { + bundle: null, + question: "why is this blocked?", + advisoryAiRouting: ADVISORY_ON, + repoFullName: "owner/repo", + issueNumber: 1, + }); + expect(result).toMatchObject({ status: "declined", reason: "The cached contribution-context snapshot is still refreshing." }); + expect((result as { suggestion: string }).suggestion).toContain(CHAT_QA_FALLBACK_COMMAND); + }); + + it("declines when the cached bundle is still refreshing", async () => { + const env = createTestEnv({ AI_ADVISORY: { run: vi.fn() } as unknown as Ai }); + const result = await generateChatQaAnswer(env, { + bundle: bundleFixture({ status: "needs_snapshot_refresh" }), + question: "why is this blocked?", + advisoryAiRouting: ADVISORY_ON, + repoFullName: "owner/repo", + issueNumber: 1, + }); + expect(result).toMatchObject({ status: "declined", reason: "The cached contribution-context snapshot is still refreshing." }); + }); + + it("declines when the bundle has no actions to ground an answer in", async () => { + const env = createTestEnv({ AI_ADVISORY: { run: vi.fn() } as unknown as Ai }); + const bundle = bundleFixture(); + bundle.actions = []; + const result = await generateChatQaAnswer(env, { + bundle, + question: "why is this blocked?", + advisoryAiRouting: ADVISORY_ON, + repoFullName: "owner/repo", + issueNumber: 1, + }); + expect(result).toMatchObject({ status: "declined", reason: "No cached deterministic facts are available to ground an answer for this PR." }); + }); + + it("reports quota_exceeded and never calls the provider when the shared daily neuron budget is exhausted", async () => { + const run = vi.fn(); + const env = createTestEnv({ AI_ADVISORY: { run } as unknown as Ai, AI_DAILY_NEURON_BUDGET: "1" }); + const result = await generateChatQaAnswer(env, { + bundle: bundleFixture(), + question: "why is this blocked?", + advisoryAiRouting: ADVISORY_ON, + repoFullName: "owner/repo", + issueNumber: 1, + actor: "alice", + }); + expect(result).toMatchObject({ status: "quota_exceeded" }); + expect(run).not.toHaveBeenCalled(); + }); + + it("falls back to the shared 10M default budget when unset, and again when the configured value is non-finite", async () => { + const run1 = vi.fn(async () => ({ response: "Grounded answer one." })); + const env1 = createTestEnv({ AI_ADVISORY: { run: run1 } as unknown as Ai }); + const result1 = await generateChatQaAnswer(env1, { bundle: bundleFixture(), question: "why?", advisoryAiRouting: ADVISORY_ON, repoFullName: "owner/repo", issueNumber: 1 }); + expect(result1).toMatchObject({ status: "ok" }); + + const run2 = vi.fn(async () => ({ response: "Grounded answer two." })); + const env2 = createTestEnv({ AI_ADVISORY: { run: run2 } as unknown as Ai, AI_DAILY_NEURON_BUDGET: "not-a-number" }); + const result2 = await generateChatQaAnswer(env2, { bundle: bundleFixture(), question: "why?", advisoryAiRouting: ADVISORY_ON, repoFullName: "owner/repo", issueNumber: 1 }); + expect(result2).toMatchObject({ status: "ok" }); + }); + + it("generates a grounded answer, redacting private terms before they ever reach the prompt", async () => { + const run = vi.fn(async () => ({ response: "Here is the readiness answer." })); + const env = createTestEnv({ AI_ADVISORY: { run } as unknown as Ai, AI_DAILY_NEURON_BUDGET: "10000" }); + const result = await generateChatQaAnswer(env, { + bundle: bundleFixture(), + question: "why is this blocked?", + advisoryAiRouting: ADVISORY_ON, + repoFullName: "owner/repo", + issueNumber: 42, + actor: "alice", + route: "github_comment", + }); + expect(result).toMatchObject({ status: "ok", text: "Here is the readiness answer." }); + expect(run).toHaveBeenCalledWith( + "", + expect.objectContaining({ + messages: [expect.objectContaining({ role: "system" }), expect.objectContaining({ role: "user", content: expect.stringContaining("why is this blocked?") })], + }), + ); + const call = run.mock.calls[0] as unknown as [string, { messages: Array<{ content: string }> }]; + const userMessage = call[1].messages[1]?.content ?? ""; + expect(userMessage).not.toMatch(/\bopen_pr_pressure\b/); + expect(userMessage).not.toMatch(/\bwallet\b/i); + expect(userMessage).not.toMatch(/\blikely_duplicate\b/); + }); + + it("honors a custom model override and clamps output tokens", async () => { + const run = vi.fn(async () => ({ response: "Custom-model answer." })); + const env = createTestEnv({ + AI_ADVISORY: { run } as unknown as Ai, + WORKERS_AI_SUMMARY_MODEL: "@cf/test/chat-model", + AI_DAILY_NEURON_BUDGET: "10000", + AI_MAX_OUTPUT_TOKENS: "99999", + }); + const result = await generateChatQaAnswer(env, { bundle: bundleFixture(), question: "why?", advisoryAiRouting: ADVISORY_ON, repoFullName: "owner/repo", issueNumber: 1 }); + expect(result).toMatchObject({ status: "ok", model: "@cf/test/chat-model" }); + expect(run).toHaveBeenCalledWith("@cf/test/chat-model", expect.objectContaining({ max_tokens: 512 })); + }); + + it("clamps max output tokens to the floor when AI_MAX_OUTPUT_TOKENS is non-numeric", async () => { + const run = vi.fn(async () => ({ response: "Answer within the floor." })); + const env = createTestEnv({ AI_ADVISORY: { run } as unknown as Ai, AI_MAX_OUTPUT_TOKENS: "not-a-number", AI_DAILY_NEURON_BUDGET: "10000" }); + const result = await generateChatQaAnswer(env, { bundle: bundleFixture(), question: "why?", advisoryAiRouting: ADVISORY_ON, repoFullName: "owner/repo", issueNumber: 1 }); + expect(result).toMatchObject({ status: "ok" }); + expect(run).toHaveBeenCalledWith("", expect.objectContaining({ max_tokens: 64 })); + }); + + it("withholds an unsafe model answer instead of ever returning it", async () => { + const run = vi.fn(async () => ({ response: "Mentions a wallet address directly." })); + const env = createTestEnv({ AI_ADVISORY: { run } as unknown as Ai, AI_DAILY_NEURON_BUDGET: "10000" }); + const result = await generateChatQaAnswer(env, { bundle: bundleFixture(), question: "why?", advisoryAiRouting: ADVISORY_ON, repoFullName: "owner/repo", issueNumber: 1 }); + expect(result).toMatchObject({ status: "unsafe" }); + }); + + it("reports an error status with the underlying message when the provider throws an Error", async () => { + const run = vi.fn(async () => { + throw new Error("provider_down"); + }); + const env = createTestEnv({ AI_ADVISORY: { run } as unknown as Ai, AI_DAILY_NEURON_BUDGET: "10000" }); + const result = await generateChatQaAnswer(env, { bundle: bundleFixture(), question: "why?", advisoryAiRouting: ADVISORY_ON, repoFullName: "owner/repo", issueNumber: 1 }); + expect(result).toMatchObject({ status: "error", reason: "provider_down" }); + }); + + it("reports a generic error reason when the provider throws a non-Error value", async () => { + const run = vi.fn(async () => { + throw "boom"; + }); + const env = createTestEnv({ AI_ADVISORY: { run } as unknown as Ai, AI_DAILY_NEURON_BUDGET: "10000" }); + const result = await generateChatQaAnswer(env, { bundle: bundleFixture(), question: "why?", advisoryAiRouting: ADVISORY_ON, repoFullName: "owner/repo", issueNumber: 1 }); + expect(result).toMatchObject({ status: "error", reason: "chat_answer_failed" }); + }); + + it("reports an error status when the provider returns an empty/unrecognized response shape", async () => { + const run = vi.fn(async () => ({ unexpected: "shape" })); + const env = createTestEnv({ AI_ADVISORY: { run } as unknown as Ai, AI_DAILY_NEURON_BUDGET: "10000" }); + const result = await generateChatQaAnswer(env, { bundle: bundleFixture(), question: "why?", advisoryAiRouting: ADVISORY_ON, repoFullName: "owner/repo", issueNumber: 1 }); + expect(result).toMatchObject({ status: "error", reason: "empty_chat_answer" }); + }); +}); + +describe("__chatQaInternals", () => { + const { compactChatSignalBundle, redactGroundingText, buildChatPrompt, containsPublicForbiddenText, estimateNeurons, extractAiText, auditOutcomeForAiStatus } = __chatQaInternals; + + it("redacts private decision-pack blocker codes and boundary terms, leaving safe text untouched", () => { + expect(redactGroundingText("blocked by open_pr_pressure")).toBe("blocked by private readiness context"); + expect(redactGroundingText("do not mention a wallet or hotkey")).toBe("do not mention a private context or private context"); + expect(redactGroundingText("likely_duplicate of #123")).toBe("possible overlap with existing work of #123"); + expect(redactGroundingText("perfectly safe text")).toBe("perfectly safe text"); + }); + + it("compacts a bundle to at most 5 actions and filters out blank why/blockedBy lines after redaction", () => { + const compact = compactChatSignalBundle(bundleFixture()); + expect(compact.actions).toHaveLength(1); + expect(compact.actions[0]?.why).toHaveLength(2); + expect(compact.actions[0]?.why.every((line) => line.length > 0)).toBe(true); + expect(compact.freshnessWarnings).toEqual(["fresh enough"]); + }); + + it("caps compacted actions at 5 even when the bundle has more", () => { + const bundle = bundleFixture(); + bundle.actions = Array.from({ length: 7 }, (_, i) => ({ ...bundle.actions[0]!, id: `action-${i}` })); + expect(compactChatSignalBundle(bundle).actions).toHaveLength(5); + }); + + it("builds a prompt embedding the question and the grounding JSON", () => { + const prompt = buildChatPrompt("why?", { objective: "o", status: "s", dataQualityStatus: "complete", summary: "sum", actions: [], freshnessWarnings: [] }); + expect(prompt).toContain("Contributor question: why?"); + expect(prompt).toContain('"objective":"o"'); + }); + + it("flags forbidden public terms via the shared sanitizer and the local near-miss pattern", () => { + expect(containsPublicForbiddenText("mentions a wallet")).toBe(true); + expect(containsPublicForbiddenText("perfectly safe prose")).toBe(false); + }); + + it("estimates neurons from prompt length and output tokens, with a floor of 1", () => { + expect(estimateNeurons("a".repeat(400), 256)).toBe(13); + expect(estimateNeurons("", 0)).toBe(1); + }); + + it("extracts text from every recognized response shape and falls back to empty otherwise", () => { + expect(extractAiText("plain string")).toBe("plain string"); + expect(extractAiText({ response: "r" })).toBe("r"); + expect(extractAiText({ text: "t" })).toBe("t"); + expect(extractAiText({ result: "res" })).toBe("res"); + expect(extractAiText({ nothing: "here" })).toBe(""); + expect(extractAiText(null)).toBe(""); + }); + + it("maps every ChatQaResult status to its audit outcome, including the unreachable-in-practice default", () => { + expect(auditOutcomeForAiStatus("ok")).toBe("success"); + expect(auditOutcomeForAiStatus("quota_exceeded")).toBe("denied"); + expect(auditOutcomeForAiStatus("unsafe")).toBe("denied"); + expect(auditOutcomeForAiStatus("error")).toBe("error"); + expect(auditOutcomeForAiStatus("disabled")).toBe("completed"); + }); +}); diff --git a/test/unit/command-authorization-engine.test.ts b/test/unit/command-authorization-engine.test.ts index e3a0606353..bfdbd66590 100644 --- a/test/unit/command-authorization-engine.test.ts +++ b/test/unit/command-authorization-engine.test.ts @@ -186,6 +186,22 @@ describe("repo command authorization policy", () => { expect(clamped.policy.commands.review).toEqual(["confirmed_miner"]); }); + it("#4595: chat defaults to maintainer/collaborator-only, deliberately excluding confirmed_miner (unlike ask's default)", () => { + expect(commandAuthorizationAllowedRoles(undefined, "chat")).toEqual(["maintainer", "collaborator"]); + expect(evaluateCommandAuthorization({ commandName: "chat", commenterAssociation: "OWNER" })).toMatchObject({ authorized: true, reason: "maintainer_invocation", actorKind: "maintainer" }); + expect(evaluateCommandAuthorization({ commandName: "chat", commenterAssociation: "COLLABORATOR" })).toMatchObject({ authorized: true, reason: "collaborator_invocation", actorKind: "maintainer" }); + // A confirmed-miner PR author is denied on chat (unlike "review"): confirmed_miner is not in chat's default + // allowed-roles list, so the pr_author-widening guard denies it the same as any other non-maintainer author. + expect( + evaluateCommandAuthorization({ commandName: "chat", commenterLogin: "miner", pullRequestAuthorLogin: "miner", minerStatus: "confirmed" }), + ).toMatchObject({ authorized: false, reason: "maintainer_command_requires_maintainer", actorKind: "author" }); + // A spoofable pr_author role added via override is clamped off with a warning, same as every other + // maintainer-only default command. + const clamped = normalizeCommandAuthorizationPolicy({ commands: { chat: ["collaborator", "pr_author"] } }); + expect(clamped.warnings).toContain("Ignored author command authorization roles for maintainer-only command: chat."); + expect(clamped.policy.commands.chat).toEqual(["collaborator"]); + }); + it("falls back to default roles for inherited object property command names", () => { for (const commandName of ["constructor", "toString", "__proto__", "hasOwnProperty"]) { expect(commandAuthorizationAllowedRoles(undefined, commandName)).toEqual(["maintainer", "collaborator", "confirmed_miner"]); diff --git a/test/unit/command-authorization.test.ts b/test/unit/command-authorization.test.ts index f931bc6d7b..5550ae889f 100644 --- a/test/unit/command-authorization.test.ts +++ b/test/unit/command-authorization.test.ts @@ -157,6 +157,22 @@ describe("repo command authorization policy", () => { expect(clamped.policy.commands.review).toEqual(["confirmed_miner"]); }); + it("#4595: chat defaults to maintainer/collaborator-only, deliberately excluding confirmed_miner (unlike ask's default)", () => { + expect(commandAuthorizationAllowedRoles(undefined, "chat")).toEqual(["maintainer", "collaborator"]); + expect(evaluateCommandAuthorization({ commandName: "chat", commenterAssociation: "OWNER" })).toMatchObject({ authorized: true, reason: "maintainer_invocation", actorKind: "maintainer" }); + expect(evaluateCommandAuthorization({ commandName: "chat", commenterAssociation: "COLLABORATOR" })).toMatchObject({ authorized: true, reason: "collaborator_invocation", actorKind: "maintainer" }); + // A confirmed-miner PR author is denied on chat (unlike "review"): confirmed_miner is not in chat's default + // allowed-roles list, so the pr_author-widening guard denies it the same as any other non-maintainer author. + expect( + evaluateCommandAuthorization({ commandName: "chat", commenterLogin: "miner", pullRequestAuthorLogin: "miner", minerStatus: "confirmed" }), + ).toMatchObject({ authorized: false, reason: "maintainer_command_requires_maintainer", actorKind: "author" }); + // A spoofable pr_author role added via override is clamped off with a warning, same as every other + // maintainer-only default command. + const clamped = normalizeCommandAuthorizationPolicy({ commands: { chat: ["collaborator", "pr_author"] } }); + expect(clamped.warnings).toContain("Ignored author command authorization roles for maintainer-only command: chat."); + expect(clamped.policy.commands.chat).toEqual(["collaborator"]); + }); + it("falls back to default roles for inherited object property command names", () => { for (const commandName of ["constructor", "toString", "__proto__", "hasOwnProperty"]) { expect(commandAuthorizationAllowedRoles(undefined, commandName)).toEqual(["maintainer", "collaborator", "confirmed_miner"]); diff --git a/test/unit/focus-manifest.test.ts b/test/unit/focus-manifest.test.ts index 2d54f6d7f7..c3f5520d77 100644 --- a/test/unit/focus-manifest.test.ts +++ b/test/unit/focus-manifest.test.ts @@ -3048,7 +3048,25 @@ describe("parseFocusManifest settings override + resolveEffectiveSettings", () = it("resolveEffectiveSettings falls back to the all-off built-in default when the DB layer has no advisoryAiRouting at all (#4364)", () => { const db = {} as unknown as RepositorySettings; const eff = resolveEffectiveSettings(db, parseFocusManifest({ settings: { advisoryAiRouting: { planner: true } } })); - expect(eff.advisoryAiRouting).toEqual({ slop: false, e2eTestGen: false, planner: true, summaries: false }); + expect(eff.advisoryAiRouting).toEqual({ slop: false, e2eTestGen: false, planner: true, summaries: false, chatQa: false }); + }); + + it("wires settings.advisoryAiRouting.chatQa into the manifest parser as a sparse override (#4595)", () => { + const parsed = parseFocusManifest({ settings: { advisoryAiRouting: { chatQa: true } } }); + expect(parsed.settings.advisoryAiRouting).toEqual({ chatQa: true }); + expect(parsed.warnings).toEqual([]); + }); + + it("resolveEffectiveSettings merges an explicit chatQa override over the DB layer's value (#4595)", () => { + const db = { advisoryAiRouting: { slop: false, e2eTestGen: false, planner: false, summaries: false, chatQa: false } } as unknown as RepositorySettings; + const eff = resolveEffectiveSettings(db, parseFocusManifest({ settings: { advisoryAiRouting: { chatQa: true } } })); + expect(eff.advisoryAiRouting).toEqual({ slop: false, e2eTestGen: false, planner: false, summaries: false, chatQa: true }); + }); + + it("resolveEffectiveSettings keeps the DB layer's chatQa when the manifest override omits it (#4595)", () => { + const db = { advisoryAiRouting: { slop: false, e2eTestGen: false, planner: false, summaries: false, chatQa: true } } as unknown as RepositorySettings; + const eff = resolveEffectiveSettings(db, parseFocusManifest({ settings: { advisoryAiRouting: { slop: true } } })); + expect(eff.advisoryAiRouting).toEqual({ slop: true, e2eTestGen: false, planner: false, summaries: false, chatQa: true }); }); it("drops a malformed advisoryAiRouting.slop field instead of replacing existing policy with defaults (#4364)", () => { diff --git a/test/unit/gen-command-reference-script.test.ts b/test/unit/gen-command-reference-script.test.ts index 619361a4e1..d5183e003e 100644 --- a/test/unit/gen-command-reference-script.test.ts +++ b/test/unit/gen-command-reference-script.test.ts @@ -94,15 +94,16 @@ describe("gen-command-reference script (#3046)", () => { expect(actionCommands).toHaveLength(7); }); - it("extracts the real 10 public + 9 maintainer-only + 8 action commands from the real repo source", () => { + it("extracts the real 11 public + 9 maintainer-only + 8 action commands from the real repo source", () => { const { publicCommands, maintainerCommands, actionCommands } = collectCommandCatalogs({ rootDir: process.cwd() }); - expect(publicCommands).toHaveLength(10); + expect(publicCommands).toHaveLength(11); expect(maintainerCommands).toHaveLength(9); expect(actionCommands).toHaveLength(8); expect(publicCommands.map((c: CommandCatalogEntry) => c.id)).toEqual([ "help", "ask", + "chat", "preflight", "blockers", "duplicate-check", diff --git a/test/unit/github-commands.test.ts b/test/unit/github-commands.test.ts index 01654fd228..ac8bc1603c 100644 --- a/test/unit/github-commands.test.ts +++ b/test/unit/github-commands.test.ts @@ -3,6 +3,7 @@ import { buildAgentCommandFeedbackMarker, buildMaintainerQueueDigest, buildPublicAgentCommandComment, + isAiCostBearingCommand, isAuthorizedCommandActor, isGittensoryActionCommand, isMaintainerOnlyCommand, @@ -25,6 +26,11 @@ describe("GitHub mention commands", () => { question: "what should I fix first?", }); expect(parseGittensoryMentionCommand("@gittensory ask")).toMatchObject({ name: "ask", question: undefined }); + expect(parseGittensoryMentionCommand("@gittensory chat ")).toMatchObject({ name: "chat", question: undefined }); + expect(parseGittensoryMentionCommand("@gittensory chat why is this PR blocked?")).toMatchObject({ + name: "chat", + question: "why is this PR blocked?", + }); expect(parseGittensoryMentionCommand("@gittensory preflight")?.name).toBe("preflight"); expect(parseGittensoryMentionCommand("please @gittensory duplicate-check now")?.name).toBe("duplicate-check"); expect(parseGittensoryMentionCommand("@gittensory reviewability")?.name).toBe("reviewability"); @@ -578,6 +584,163 @@ describe("GitHub mention commands", () => { expect(forged).toContain("APPROVED by"); }); + it("#4595: chat is registered as an AI-cost-bearing command (inherits the tighter rate-limit ceiling)", () => { + expect(isAiCostBearingCommand("chat")).toBe(true); + }); + + it("#4595: renders a grounded chat answer with the fixed disclaimer footer on every status", () => { + const ok = buildPublicAgentCommandComment({ + env: {}, + command: parseGittensoryMentionCommand("@gittensory chat why is this PR blocked?")!, + repo: null, + issue: { number: 20, title: "PR", state: "open", pull_request: {} }, + pullRequest: null, + actorKind: "author", + bundle: sampleBundle(), + chatAnswer: { status: "ok", model: "qwen3:8b", estimatedNeurons: 12, text: "This PR is blocked because two tests are failing." }, + }); + expect(ok).toContain("**Grounded chat Q&A**"); + expect(ok).toContain("Question: why is this PR blocked?"); + // The model's own prose is markdown-neutralized like the ask question is (#4595 req 8), so its literal + // period is backslash-escaped -- assert on the text minus trailing punctuation rather than coupling this + // to neutralizePublicMarkdownText's exact escaping of `.`. + expect(ok).toContain("This PR is blocked because two tests are failing"); + // The fixed, non-LLM disclaimer footer appears verbatim, with its code span intact (#4595 req 9). + expect(ok).toContain("Read-only informational reply"); + expect(ok).toContain("`@gittensory review`"); + + const disabled = buildPublicAgentCommandComment({ + env: {}, + command: parseGittensoryMentionCommand("@gittensory chat why is this PR blocked?")!, + repo: null, + issue: { number: 21, title: "PR", state: "open", pull_request: {} }, + pullRequest: null, + actorKind: "author", + bundle: sampleBundle(), + chatAnswer: { status: "disabled", reason: "Chat Q&A is not enabled on this instance (settings.advisoryAiRouting.chatQa is off)." }, + }); + expect(disabled).toContain("not enabled on this instance"); + expect(disabled).toContain("Read-only informational reply"); + + const declined = buildPublicAgentCommandComment({ + env: {}, + command: parseGittensoryMentionCommand("@gittensory chat")!, + repo: null, + issue: { number: 22, title: "PR", state: "open", pull_request: {} }, + pullRequest: null, + actorKind: "author", + bundle: sampleBundle(), + chatAnswer: { status: "declined", reason: "No question was supplied.", suggestion: "Ask a specific question, for example `@gittensory chat why is this PR blocked?`." }, + }); + expect(declined).toContain("No question was supplied."); + expect(declined).toContain("Read-only informational reply"); + + const noAnswer = buildPublicAgentCommandComment({ + env: {}, + command: parseGittensoryMentionCommand("@gittensory chat why is this PR blocked?")!, + repo: null, + issue: { number: 23, title: "PR", state: "open", pull_request: {} }, + pullRequest: null, + actorKind: "author", + bundle: sampleBundle(), + chatAnswer: null, + }); + expect(noAnswer).toContain("Chat Q&A could not produce a grounded answer"); + expect(noAnswer).toContain("Read-only informational reply"); + + const unavailable = buildPublicAgentCommandComment({ + env: {}, + command: parseGittensoryMentionCommand("@gittensory chat why is this PR blocked?")!, + repo: null, + issue: { number: 25, title: "PR", state: "open", pull_request: {} }, + pullRequest: null, + actorKind: "author", + bundle: sampleBundle(), + chatAnswer: { status: "unavailable", reason: "Local advisory inference (env.AI_ADVISORY) is not configured; chat Q&A never falls back to the frontier model." }, + }); + expect(unavailable).toContain("not enabled on this instance"); + expect(unavailable).toContain("Read-only informational reply"); + + const quotaExceeded = buildPublicAgentCommandComment({ + env: {}, + command: parseGittensoryMentionCommand("@gittensory chat why is this PR blocked?")!, + repo: null, + issue: { number: 26, title: "PR", state: "open", pull_request: {} }, + pullRequest: null, + actorKind: "author", + bundle: sampleBundle(), + chatAnswer: { status: "quota_exceeded", model: "qwen3:8b", estimatedNeurons: 12, remainingBudget: 0 }, + }); + expect(quotaExceeded).toContain("shared daily AI budget is exhausted"); + expect(quotaExceeded).toContain("Read-only informational reply"); + + const unsafe = buildPublicAgentCommandComment({ + env: {}, + command: parseGittensoryMentionCommand("@gittensory chat why is this PR blocked?")!, + repo: null, + issue: { number: 27, title: "PR", state: "open", pull_request: {} }, + pullRequest: null, + actorKind: "author", + bundle: sampleBundle(), + chatAnswer: { status: "unsafe", model: "qwen3:8b", estimatedNeurons: 12, reason: "chat answer failed public sanitizer" }, + }); + expect(unsafe).toContain("Chat Q&A could not produce a grounded answer"); + expect(unsafe).toContain("Read-only informational reply"); + + const errored = buildPublicAgentCommandComment({ + env: {}, + command: parseGittensoryMentionCommand("@gittensory chat why is this PR blocked?")!, + repo: null, + issue: { number: 28, title: "PR", state: "open", pull_request: {} }, + pullRequest: null, + actorKind: "author", + bundle: sampleBundle(), + chatAnswer: { status: "error", model: "qwen3:8b", estimatedNeurons: 0, reason: "provider_down" }, + }); + expect(errored).toContain("Chat Q&A could not produce a grounded answer"); + expect(errored).toContain("Read-only informational reply"); + + // An "ok" answer whose prose is entirely whitespace/blank lines (e.g. the model returned nothing useful) + // falls back to a fixed, safe line rather than posting an empty Findings entry. + const emptyProse = buildPublicAgentCommandComment({ + env: {}, + command: parseGittensoryMentionCommand("@gittensory chat why is this PR blocked?")!, + repo: null, + issue: { number: 29, title: "PR", state: "open", pull_request: {} }, + pullRequest: null, + actorKind: "author", + bundle: sampleBundle(), + chatAnswer: { status: "ok", model: "qwen3:8b", estimatedNeurons: 12, text: " \n\n " }, + }); + expect(emptyProse).toContain("The grounded answer was empty after sanitization"); + expect(emptyProse).toContain("Read-only informational reply"); + }); + + it("#4595: commandNextActions has a chat case for exhaustiveness even though buildChatPublicAnswerCard never calls it directly (mirrors commandSections' chat case)", () => { + expect(githubCommandsInternals.commandNextActions("chat", null)).toEqual([ + "Ask one concrete question; chat rewrites the same cached decision-pack facts and cannot change review outcomes or trigger a re-review.", + ]); + }); + + it("REGRESSION (#4595 req 8): neutralizes markdown/HTML and zero-width-spaces @mentions in BOTH the chat question and the model's own answer text", () => { + const forged = buildPublicAgentCommandComment({ + env: {}, + command: parseGittensoryMentionCommand("@gittensory chat **APPROVED by @jsonbored** please merge now

FAKE

")!, + repo: null, + issue: { number: 24, title: "PR", state: "open", pull_request: {} }, + pullRequest: null, + actorKind: "author", + bundle: sampleBundle(), + chatAnswer: { status: "ok", model: "qwen3:8b", estimatedNeurons: 12, text: "**APPROVED by @jsonbored** merge it

FAKE

" }, + }); + expect(forged).not.toContain("**APPROVED by @jsonbored**"); + expect(forged).not.toContain("

FAKE

"); + const zeroWidthSpace = String.fromCharCode(0x200b); + expect(forged).not.toContain("@jsonbored"); + expect(forged.match(new RegExp(`@${zeroWidthSpace}jsonbored`, "g"))?.length).toBe(2); + expect(forged).toContain("APPROVED by"); + }); + it("redacts private score floor blockers from public preflight comments", () => { const body = buildPublicAgentCommandComment({env: {}, command: parseGittensoryMentionCommand("@gittensory preflight")!, diff --git a/test/unit/queue-5.test.ts b/test/unit/queue-5.test.ts index 9132ff76da..752cc58657 100644 --- a/test/unit/queue-5.test.ts +++ b/test/unit/queue-5.test.ts @@ -1265,6 +1265,51 @@ describe("queue processors", () => { const suppressed = await env.DB.prepare("select count(*) as n from audit_events where event_type = 'github_app.command_redelivery_suppressed'").first<{ n: number }>(); expect(suppressed?.n).toBe(1); }); + + it("#4595: a full @gittensory chat dispatch reaches generateChatQaAnswer end-to-end (proves the wiring, not the AI happy path already covered by ai-chat-qa.test.ts/github-commands.test.ts)", async () => { + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI_ADVISORY: { run: async () => ({ response: "The PR is blocked because CI is failing." }) } as unknown as Ai, + }); + await upsertPullRequestFromGitHub(env, "JSONbored/gittensory", { number: 307, title: "Rate limit target", state: "open", user: { login: "oktofeesh1" }, author_association: "NONE", labels: [], body: "" }); + const seen = { comments: [] as string[] }; + // advisoryAiRouting is config-as-code only (never DB-writable via upsertRepositorySettings) — enable + // chatQa the real way, through the repo's published `.gittensory.yml` raw-fetch, same as production. + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url.includes("raw.githubusercontent.com") && url.includes(".gittensory.yml")) { + return new Response("settings:\n advisoryAiRouting:\n chatQa: true\n", { status: 200 }); + } + if (url.includes("/access_tokens")) return Response.json({ token: "fake-installation-token" }); + if (url.includes("/collaborators/") && url.includes("/permission")) return Response.json({ permission: "maintain" }); + if (url.includes("/issues/307/comments") && method === "GET") return Response.json([]); + if (url.includes("/issues/307/comments") && method === "POST") { + seen.comments.push(String(JSON.parse(String(init?.body ?? "{}")).body ?? "")); + return Response.json({ id: seen.comments.length }, { status: 201 }); + } + return new Response("not found", { status: 404 }); + }); + await processJob(env, { type: "github-webhook", deliveryId: "chat-full-dispatch", eventName: "issue_comment", payload: mentionPayload(307, "@gittensory chat why is this blocked?") }); + expect(seen.comments).toHaveLength(1); + expect(seen.comments[0]).toContain("Grounded chat Q&A"); + // A brand-new synthetic PR has no pre-existing decision-pack snapshot, so the bundle is naturally + // "needs_snapshot_refresh" here -- that's fine: this test's job is proving processors.ts actually reaches + // and calls generateChatQaAnswer for a real "chat" webhook (chatQa enabled, not the "disabled" text), not + // exercising the AI happy path (already covered directly in ai-chat-qa.test.ts and github-commands.test.ts). + expect(seen.comments[0]).toContain("The cached contribution-context snapshot is still refreshing"); + expect(seen.comments[0]).not.toContain("not enabled on this instance"); + }); + + it("#4595: chat declines gracefully end-to-end (never posts model text) when chatQa is off, the default", async () => { + const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem() }); + await upsertPullRequestFromGitHub(env, "JSONbored/gittensory", { number: 308, title: "Rate limit target", state: "open", user: { login: "oktofeesh1" }, author_association: "NONE", labels: [], body: "" }); + const seen = { comments: [] as string[] }; + stubCommandRateLimitFetch(308, seen); + await processJob(env, { type: "github-webhook", deliveryId: "chat-default-off", eventName: "issue_comment", payload: mentionPayload(308, "@gittensory chat why is this blocked?") }); + expect(seen.comments).toHaveLength(1); + expect(seen.comments[0]).toContain("not enabled on this instance"); + }); }); it("denies a maintainer Q&A command from an org member without real repo permission (#788)", async () => {