diff --git a/apps/gittensory-ui/public/openapi.json b/apps/gittensory-ui/public/openapi.json index ed28ec3fd5..2bfa244355 100644 --- a/apps/gittensory-ui/public/openapi.json +++ b/apps/gittensory-ui/public/openapi.json @@ -11301,6 +11301,36 @@ ] } }, + "/v1/app/commands/usefulness": { + "get": { + "responses": { + "200": { + "description": "Live app API response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": { + "nullable": true + } + } + } + } + }, + "401": { + "description": "Unauthorized" + } + }, + "security": [ + { + "GittensoryBearer": [] + }, + { + "GittensorySessionCookie": [] + } + ] + } + }, "/v1/app/digest": { "get": { "responses": { @@ -11331,6 +11361,36 @@ ] } }, + "/v1/app/analytics/daily-rollups": { + "get": { + "responses": { + "200": { + "description": "Live app API response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": { + "nullable": true + } + } + } + } + }, + "401": { + "description": "Unauthorized" + } + }, + "security": [ + { + "GittensoryBearer": [] + }, + { + "GittensorySessionCookie": [] + } + ] + } + }, "/v1/app/analytics/mcp-compatibility": { "get": { "responses": { @@ -11437,6 +11497,52 @@ ] } }, + "/v1/app/commands/feedback": { + "post": { + "responses": { + "200": { + "description": "Live app mutation or preview response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": { + "nullable": true + } + } + } + } + }, + "201": { + "description": "Created", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": { + "nullable": true + } + } + } + } + }, + "400": { + "description": "Invalid request" + }, + "401": { + "description": "Unauthorized" + } + }, + "security": [ + { + "GittensoryBearer": [] + }, + { + "GittensorySessionCookie": [] + } + ] + } + }, "/v1/app/digest/subscriptions": { "post": { "responses": { diff --git a/migrations/0015_github_agent_command_feedback.sql b/migrations/0015_github_agent_command_feedback.sql new file mode 100644 index 0000000000..d5a8f5c9ea --- /dev/null +++ b/migrations/0015_github_agent_command_feedback.sql @@ -0,0 +1,44 @@ +CREATE TABLE IF NOT EXISTS github_agent_command_answers ( + id TEXT PRIMARY KEY, + repo_full_name TEXT NOT NULL, + issue_number INTEGER NOT NULL, + command TEXT NOT NULL, + request_comment_id INTEGER, + response_comment_id INTEGER, + response_url TEXT, + actor_kind TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + metadata_json TEXT NOT NULL DEFAULT '{}' +); + +CREATE INDEX IF NOT EXISTS github_agent_command_answers_repo_issue_idx + ON github_agent_command_answers(repo_full_name, issue_number); + +CREATE INDEX IF NOT EXISTS github_agent_command_answers_command_updated_idx + ON github_agent_command_answers(command, updated_at); + +CREATE TABLE IF NOT EXISTS github_agent_command_feedback ( + id TEXT PRIMARY KEY, + answer_id TEXT NOT NULL, + repo_full_name TEXT NOT NULL, + issue_number INTEGER NOT NULL, + command TEXT NOT NULL, + actor_hash TEXT NOT NULL, + vote TEXT NOT NULL, + source TEXT NOT NULL, + actor_kind TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + metadata_json TEXT NOT NULL DEFAULT '{}', + FOREIGN KEY(answer_id) REFERENCES github_agent_command_answers(id) +); + +CREATE UNIQUE INDEX IF NOT EXISTS github_agent_command_feedback_actor_answer_unique + ON github_agent_command_feedback(answer_id, actor_hash); + +CREATE INDEX IF NOT EXISTS github_agent_command_feedback_command_updated_idx + ON github_agent_command_feedback(command, updated_at); + +CREATE INDEX IF NOT EXISTS github_agent_command_feedback_repo_issue_idx + ON github_agent_command_feedback(repo_full_name, issue_number); diff --git a/src/api/routes.ts b/src/api/routes.ts index efbc6a6f32..2909f8e389 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -28,6 +28,9 @@ import { countActiveAuthSessions, countActiveDigestSubscriptions, getBounty, + getAgentCommandAnswer, + getCommandUsefulnessSummary, + getFreshOfficialMinerDetection, getIssue, getInstallationHealth, getLatestRepoGithubTotalsSnapshot, @@ -71,6 +74,7 @@ import { persistBountyLifecycleEvent, persistScorePreview, persistSignalSnapshot, + recordAgentCommandFeedback, recordProductUsageEvent, rollupProductUsageDaily, summarizeMcpCompatibilityAdoption, @@ -398,6 +402,13 @@ const commandPreviewSchema = z }) .strict(); +const commandFeedbackSchema = z + .object({ + answerId: z.string().min(8).max(120).regex(/^[A-Za-z0-9_.:-]+$/), + vote: z.enum(["useful", "not_useful"]), + }) + .strict(); + const digestSubscriptionSchema = z .object({ email: z.string().email().max(320), @@ -774,6 +785,7 @@ export function createApp() { usageRollups, usageRollupStatus, mcpCompatibilityAdoption, + commandUsefulness, ] = await Promise.all([ listRepositories(c.env), listInstallations(c.env), @@ -788,6 +800,7 @@ export function createApp() { listProductUsageDailyRollups(c.env, { limit: 14 }), getProductUsageRollupStatus(c.env), summarizeMcpCompatibilityAdoption(c.env, usageSince), + getCommandUsefulnessSummary(c.env), ]); const weeklyValueReport = buildWeeklyValueReport({ generatedAt: nowIso(), @@ -818,6 +831,7 @@ export function createApp() { { label: "Active users", value: String(usageSummary.activeActors), delta: "hashed, last 7 days" }, { label: "Activation rollups", value: usageRollupStatus.status, delta: usageRollupStatus.latestRollupDay ?? "not generated" }, { label: "MCP stale clients", value: String(mcpCompatibilityAdoption.staleEvents + mcpCompatibilityAdoption.incompatibleEvents), delta: `${mcpCompatibilityAdoption.totalEvents} MCP event(s)` }, + { label: "Command usefulness", value: `${commandUsefulness.totals.usefulCount}/${commandUsefulness.totals.feedbackCount}`, delta: usefulnessDelta(commandUsefulness.totals.usefulnessRate) }, { label: "Install issues", value: String(health.filter((record) => record.status !== "healthy").length), delta: "current health cache" }, { label: "Rate-limit events", value: String(rateLimits.length), delta: "latest observations" }, ], @@ -832,6 +846,7 @@ export function createApp() { usageRollups, usageRollupStatus, mcpCompatibilityAdoption, + commandUsefulness, registry, scoringModel: scoring, upstreamDrift, @@ -938,6 +953,53 @@ export function createApp() { }); }); + app.get("/v1/app/commands/usefulness", async (c) => { + const identity = await authenticateRequestIdentity(c); + if (!identity) return c.json({ error: "unauthorized" }, 401); + const days = Number(c.req.query("days") ?? 30); + return c.json(await getCommandUsefulnessSummary(c.env, { windowDays: clampInteger(days, 1, 180) })); + }); + + app.post("/v1/app/commands/feedback", async (c) => { + const identity = await authenticateRequestIdentity(c); + if (!identity) return c.json({ error: "unauthorized" }, 401); + const body = await c.req.json().catch(() => null); + const parsed = commandFeedbackSchema.safeParse(body); + if (!parsed.success) return c.json({ error: "invalid_command_feedback", issues: parsed.error.issues }, 400); + const answer = await getAgentCommandAnswer(c.env, parsed.data.answerId); + if (!answer) return c.json({ error: "command_answer_not_found" }, 404); + const actorLogin = identity.actor; + await recordAgentCommandFeedback(c.env, { + answerId: answer.id, + repoFullName: answer.repoFullName, + issueNumber: answer.issueNumber, + command: answer.command, + actorLogin, + vote: parsed.data.vote, + source: "app", + actorKind: "maintainer", + metadata: { surface: "app", identityKind: identity.kind }, + }); + await recordAuditEvent(c.env, { + eventType: "github_app.agent_command_feedback_recorded", + actor: actorLogin, + targetKey: `${answer.repoFullName}#${answer.issueNumber}`, + outcome: "completed", + metadata: { answerId: answer.id, command: answer.command, vote: parsed.data.vote, source: "app", identityKind: identity.kind }, + }); + return c.json({ + ok: true, + generatedAt: nowIso(), + answer: { + id: answer.id, + repoFullName: answer.repoFullName, + issueNumber: answer.issueNumber, + command: answer.command, + }, + vote: parsed.data.vote, + }); + }); + app.get("/v1/app/digest", async (c) => { const forbidden = await requireAppRole(c, ["maintainer", "owner", "operator"]); if (forbidden) return forbidden; @@ -2152,6 +2214,15 @@ function buildCommandPreview(command: (typeof APP_COMMANDS)[number], request: z. }; } +function usefulnessDelta(rate: number | null): string { + return rate === null ? "no feedback yet" : `${Math.round(rate * 100)}% useful over 30 days`; +} + +function clampInteger(value: number, min: number, max: number): number { + if (!Number.isFinite(value)) return min; + return Math.min(max, Math.max(min, Math.round(value))); +} + function buildDigestItems(args: { repositories: RepositoryRecord[]; health: InstallationHealthRecord[]; diff --git a/src/db/repositories.ts b/src/db/repositories.ts index e98aa343d7..80edf62b9b 100644 --- a/src/db/repositories.ts +++ b/src/db/repositories.ts @@ -18,6 +18,8 @@ import { contributorScoringProfiles, contributors, digestSubscriptions, + githubAgentCommandAnswers, + githubAgentCommandFeedback, installationHealth, installations, issueQualityReports, @@ -52,6 +54,8 @@ import type { AgentActionRecord, AgentActionStatus, AgentActionType, + AgentCommandAnswerRecord, + AgentCommandFeedbackRecord, AgentContextSnapshotRecord, AgentMode, AgentRunRecord, @@ -65,6 +69,7 @@ import type { BurdenForecastRecord, CheckSummaryRecord, CollisionEdgeRecord, + CommandUsefulnessSummary, ContributorEvidenceRecord, ContributorRecord, ContributorRepoStatRecord, @@ -1080,6 +1085,132 @@ export async function summarizeProductUsageEvents(env: Env, sinceIso?: string): }; } +export async function upsertAgentCommandAnswer(env: Env, answer: AgentCommandAnswerRecord): Promise { + const now = answer.updatedAt ?? nowIso(); + const createdAt = answer.createdAt ?? now; + const values = { + id: answer.id, + repoFullName: boundedString(answer.repoFullName, 200), + issueNumber: Math.max(0, Math.round(answer.issueNumber)), + command: boundedString(answer.command, 64), + requestCommentId: optionalNumber(answer.requestCommentId), + responseCommentId: optionalNumber(answer.responseCommentId), + responseUrl: answer.responseUrl ? boundedString(answer.responseUrl, 500) : null, + actorKind: answer.actorKind, + createdAt, + updatedAt: now, + metadataJson: jsonString(answer.metadata), + }; + await getDb(env.DB) + .insert(githubAgentCommandAnswers) + .values(values) + .onConflictDoUpdate({ + target: githubAgentCommandAnswers.id, + set: { + repoFullName: values.repoFullName, + issueNumber: values.issueNumber, + command: values.command, + requestCommentId: values.requestCommentId, + responseCommentId: values.responseCommentId, + responseUrl: values.responseUrl, + actorKind: values.actorKind, + updatedAt: values.updatedAt, + metadataJson: values.metadataJson, + }, + }); + return (await getAgentCommandAnswer(env, answer.id))!; +} + +export async function getAgentCommandAnswer(env: Env, answerId: string): Promise { + const [row] = await getDb(env.DB).select().from(githubAgentCommandAnswers).where(eq(githubAgentCommandAnswers.id, answerId)).limit(1); + return row ? toAgentCommandAnswer(row) : null; +} + +export async function recordAgentCommandFeedback(env: Env, feedback: AgentCommandFeedbackRecord): Promise { + const actorHash = await hashCommandFeedbackActor(feedback.repoFullName, feedback.actorLogin); + const now = feedback.updatedAt ?? nowIso(); + const values = { + id: feedback.id ?? crypto.randomUUID(), + answerId: feedback.answerId, + repoFullName: boundedString(feedback.repoFullName, 200), + issueNumber: Math.max(0, Math.round(feedback.issueNumber)), + command: boundedString(feedback.command, 64), + actorHash, + vote: feedback.vote, + source: feedback.source, + actorKind: feedback.actorKind, + createdAt: feedback.createdAt ?? now, + updatedAt: now, + metadataJson: jsonString(feedback.metadata ?? {}), + }; + await getDb(env.DB) + .insert(githubAgentCommandFeedback) + .values(values) + .onConflictDoUpdate({ + target: [githubAgentCommandFeedback.answerId, githubAgentCommandFeedback.actorHash], + set: { + vote: values.vote, + source: values.source, + actorKind: values.actorKind, + updatedAt: values.updatedAt, + metadataJson: values.metadataJson, + }, + }); +} + +export async function getCommandUsefulnessSummary(env: Env, options: { windowDays?: number; now?: string } = {}): Promise { + const windowDays = clampInteger(options.windowDays ?? 30, 1, 180); + const now = options.now ?? nowIso(); + const sinceIso = new Date(Date.parse(now) - windowDays * 24 * 60 * 60 * 1000).toISOString(); + const rows = await getDb(env.DB) + .select({ + command: githubAgentCommandFeedback.command, + feedbackCount: sql`count(*)`, + usefulCount: sql`coalesce(sum(case when ${githubAgentCommandFeedback.vote} = 'useful' then 1 else 0 end), 0)`, + notUsefulCount: sql`coalesce(sum(case when ${githubAgentCommandFeedback.vote} = 'not_useful' then 1 else 0 end), 0)`, + answerCount: sql`count(distinct ${githubAgentCommandFeedback.answerId})`, + latestFeedbackAt: sql`max(${githubAgentCommandFeedback.updatedAt})`, + }) + .from(githubAgentCommandFeedback) + .where(gte(githubAgentCommandFeedback.updatedAt, sinceIso)) + .groupBy(githubAgentCommandFeedback.command); + const commands = rows + .map((row) => { + const feedbackCount = Number(row.feedbackCount); + const usefulCount = Number(row.usefulCount); + const notUsefulCount = Number(row.notUsefulCount); + return { + command: row.command, + feedbackCount, + usefulCount, + notUsefulCount, + answerCount: Number(row.answerCount), + usefulnessRate: usefulCount / feedbackCount, + latestFeedbackAt: row.latestFeedbackAt, + }; + }) + .sort((left, right) => right.feedbackCount - left.feedbackCount || left.command.localeCompare(right.command)); + const totals = commands.reduce( + (acc, row) => ({ + feedbackCount: acc.feedbackCount + row.feedbackCount, + usefulCount: acc.usefulCount + row.usefulCount, + notUsefulCount: acc.notUsefulCount + row.notUsefulCount, + answerCount: acc.answerCount + row.answerCount, + latestFeedbackAt: maxIso(acc.latestFeedbackAt, row.latestFeedbackAt), + }), + { feedbackCount: 0, usefulCount: 0, notUsefulCount: 0, answerCount: 0, latestFeedbackAt: null as string | null }, + ); + return { + windowDays, + generatedAt: now, + totals: { + ...totals, + usefulnessRate: totals.feedbackCount > 0 ? totals.usefulCount / totals.feedbackCount : null, + }, + commands, + }; +} + export async function summarizeMcpCompatibilityAdoption( env: Env, sinceIso?: string, @@ -1315,6 +1446,21 @@ function boundedString(value: unknown, maxLength: number): string { return String(value ?? "").slice(0, maxLength); } +async function hashCommandFeedbackActor(repoFullName: string, actorLogin: string): Promise { + return `sha256:${await sha256Hex(`gittensory-command-feedback:v1:${repoFullName.toLowerCase()}:${actorLogin.toLowerCase()}`)}`; +} + +function clampInteger(value: number, min: number, max: number): number { + if (!Number.isFinite(value)) return min; + return Math.min(max, Math.max(min, Math.round(value))); +} + +function maxIso(left: string | null | undefined, right: string | null | undefined): string | null { + if (!left) return right ?? null; + if (!right) return left; + return right > left ? right : left; +} + function finiteNumber(value: unknown): number { return typeof value === "number" && Number.isFinite(value) ? value : 0; } @@ -3312,6 +3458,22 @@ function sanitizeProductUsageString(value: string, maxLength: number): string { return redacted.slice(0, maxLength); } +function toAgentCommandAnswer(row: typeof githubAgentCommandAnswers.$inferSelect): AgentCommandAnswerRecord { + return { + id: row.id, + repoFullName: row.repoFullName, + issueNumber: row.issueNumber, + command: row.command, + requestCommentId: row.requestCommentId, + responseCommentId: row.responseCommentId, + responseUrl: row.responseUrl, + actorKind: row.actorKind === "maintainer" ? "maintainer" : "author", + createdAt: row.createdAt, + updatedAt: row.updatedAt, + metadata: parseJson>(row.metadataJson, {}), + }; +} + function parseAgentSurface(value: string): AgentSurface { if (value === "mcp" || value === "github_comment") return value; return "api"; diff --git a/src/db/schema.ts b/src/db/schema.ts index fcf90f019d..45d37fe943 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -729,6 +729,52 @@ export const digestSubscriptions = sqliteTable( }), ); +export const githubAgentCommandAnswers = sqliteTable( + "github_agent_command_answers", + { + id: text("id").primaryKey(), + repoFullName: text("repo_full_name").notNull(), + issueNumber: integer("issue_number").notNull(), + command: text("command").notNull(), + requestCommentId: integer("request_comment_id"), + responseCommentId: integer("response_comment_id"), + responseUrl: text("response_url"), + actorKind: text("actor_kind").notNull(), + createdAt: text("created_at").notNull().default("CURRENT_TIMESTAMP"), + updatedAt: text("updated_at").notNull().default("CURRENT_TIMESTAMP"), + metadataJson: text("metadata_json").notNull().default("{}"), + }, + (table) => ({ + repoIssue: index("github_agent_command_answers_repo_issue_idx").on(table.repoFullName, table.issueNumber), + commandUpdated: index("github_agent_command_answers_command_updated_idx").on(table.command, table.updatedAt), + }), +); + +export const githubAgentCommandFeedback = sqliteTable( + "github_agent_command_feedback", + { + id: text("id").primaryKey(), + answerId: text("answer_id") + .notNull() + .references(() => githubAgentCommandAnswers.id), + repoFullName: text("repo_full_name").notNull(), + issueNumber: integer("issue_number").notNull(), + command: text("command").notNull(), + actorHash: text("actor_hash").notNull(), + vote: text("vote").notNull(), + source: text("source").notNull(), + actorKind: text("actor_kind").notNull(), + createdAt: text("created_at").notNull().default("CURRENT_TIMESTAMP"), + updatedAt: text("updated_at").notNull().default("CURRENT_TIMESTAMP"), + metadataJson: text("metadata_json").notNull().default("{}"), + }, + (table) => ({ + actorAnswer: uniqueIndex("github_agent_command_feedback_actor_answer_unique").on(table.answerId, table.actorHash), + commandUpdated: index("github_agent_command_feedback_command_updated_idx").on(table.command, table.updatedAt), + repoIssue: index("github_agent_command_feedback_repo_issue_idx").on(table.repoFullName, table.issueNumber), + }), +); + export const auditEvents = sqliteTable( "audit_events", { diff --git a/src/github/commands.ts b/src/github/commands.ts index 620dc31a5e..4a554f560e 100644 --- a/src/github/commands.ts +++ b/src/github/commands.ts @@ -46,9 +46,15 @@ type PublicAnswerCard = { safeDetails?: string[] | undefined; }; +export type AgentCommandFeedbackContext = { + answerId: string; + command: GittensoryMentionCommandName | null; +}; + const COMMANDS = new Set(GITTENSORY_MENTION_COMMAND_CATALOG.map((command) => command.id)); const MAINTAINER_QUEUE_DIGEST_COMMANDS = new Set(MAINTAINER_QUEUE_DIGEST_COMMAND_CATALOG.map((command) => command.id)); const MAINTAINER_ASSOCIATIONS = new Set(["OWNER", "MEMBER", "COLLABORATOR"]); +const AGENT_COMMAND_FEEDBACK_MARKER = "gittensory-agent-command-answer"; const COMMAND_TITLES = Object.fromEntries(GITTENSORY_MENTION_COMMAND_CATALOG.map((command) => [command.id, command.title])) as Record; @@ -132,6 +138,20 @@ export function isMaintainerAssociation(association: string | null | undefined): return Boolean(association && MAINTAINER_ASSOCIATIONS.has(association)); } +export function buildAgentCommandFeedbackMarker(answerId: string): string { + return ``; +} + +export function parseAgentCommandFeedbackContext(body: string | null | undefined): AgentCommandFeedbackContext | null { + if (!body) return null; + const answerMatch = body.match(//); + if (!answerMatch?.[1]) return null; + const commandMatch = body.match(/Command:\s*`@gittensory\s+([a-z-]+)`/i); + const requestedCommand = commandMatch?.[1]?.toLowerCase() as GittensoryMentionCommandName | undefined; + const command = requestedCommand && COMMANDS.has(requestedCommand) ? requestedCommand : null; + return { answerId: answerMatch[1], command }; +} + export function isMaintainerQueueDigestCommand(command: GittensoryMentionCommandName): command is MaintainerQueueDigestCommandName { return MAINTAINER_QUEUE_DIGEST_COMMANDS.has(command as MaintainerQueueDigestCommandName); } @@ -169,6 +189,7 @@ export function buildPublicAgentCommandComment(args: { issue: GitHubIssuePayload; pullRequest: PullRequestRecord | null; actorKind: "maintainer" | "author"; + answerId?: string | null | undefined; officialMiner?: GittensorContributorSnapshot | null | undefined; bundle?: AgentRunBundle | null | undefined; maintainerDigest?: MaintainerQueueDigest | null | undefined; @@ -190,6 +211,7 @@ export function buildPublicAgentCommandComment(args: { `Scope: ${repoFullName}#${args.issue.number}`, "", ...renderPublicAnswerCard(card), + ...feedbackPromptSections(args.answerId), "", "_Advisory context only. Public comments exclude non-public contributor signals and private planning internals._", ].join("\n"); @@ -377,6 +399,17 @@ function stripEmphasis(value: string): string { return value.replace(/^\*\*/, "").replace(/\*\*$/, "").trim(); } +function feedbackPromptSections(answerId: string | null | undefined): string[] { + if (!answerId) return []; + return [ + "", + buildAgentCommandFeedbackMarker(answerId), + "**Feedback**", + "", + "- Use a thumbs-up or thumbs-down reaction to mark whether this answer helped. Feedback is aggregate-only and never changes deterministic results.", + ]; +} + function commandSections( command: GittensoryMentionCommandName, bundle: AgentRunBundle | null | undefined, @@ -896,6 +929,10 @@ function dedupeBulletLines(lines: string[]): string[] { }); } +function sanitizeFeedbackAnswerId(answerId: string): string { + return answerId.replace(/[^A-Za-z0-9_.:-]/g, "").slice(0, 120); +} + export function sanitizePublicComment(value: string): string { const sanitized = value .replace(/\b(raw trust score|trust score|wallet|hotkey|coldkey|seed phrase|mnemonic)\b/gi, "private context") diff --git a/src/openapi/spec.ts b/src/openapi/spec.ts index b95bd1a5aa..ff0929ad38 100644 --- a/src/openapi/spec.ts +++ b/src/openapi/spec.ts @@ -579,7 +579,9 @@ export function buildOpenApiSpec() { "/v1/app/maintainer-dashboard", "/v1/app/operator-dashboard", "/v1/app/commands", + "/v1/app/commands/usefulness", "/v1/app/digest", + "/v1/app/analytics/daily-rollups", "/v1/app/analytics/mcp-compatibility", "/v1/app/analytics/weekly-value-report", ]) { @@ -592,7 +594,7 @@ export function buildOpenApiSpec() { }, }); } - for (const path of ["/v1/app/commands/preview", "/v1/app/digest/subscriptions"]) { + for (const path of ["/v1/app/commands/preview", "/v1/app/commands/feedback", "/v1/app/digest/subscriptions"]) { registry.registerPath({ method: "post", path, diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 98ae3e8947..0ce08495e2 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -1,6 +1,7 @@ import { countOpenIssues, countOpenPullRequests, + getAgentCommandAnswer, getLatestRepoGithubTotalsSnapshot, getFreshOfficialMinerDetection, getPullRequest, @@ -27,11 +28,13 @@ import { listRepositories, markInstallationDeleted, persistAdvisory, + recordAgentCommandFeedback, recordAuditEvent, recordProductUsageEvent, persistSignalSnapshot, recordWebhookEvent, replaceCollisionEdges, + upsertAgentCommandAnswer, upsertOfficialMinerDetection, rollupProductUsageDaily, upsertBurdenForecast, @@ -61,6 +64,7 @@ import { isMaintainerAssociation, isMaintainerOnlyCommand, isMaintainerQueueDigestCommand, + parseAgentCommandFeedbackContext, parseGittensoryMentionCommand, } from "../github/commands"; import { ensurePullRequestLabel } from "../github/labels"; @@ -70,6 +74,7 @@ import { buildIssueAdvisory, buildPullRequestAdvisory } from "../rules/advisory" import { getOrCreateScoringModelSnapshot, refreshScoringModelSnapshot } from "../scoring/model"; import { buildAndPersistContributorDecisionPack, loadDecisionPackSharedInputs } from "../services/decision-pack"; import { executeAgentRun, explainBlockersWithAgent, planNextWork, preflightBranchWithAgent, preparePrPacketWithAgent } from "../services/agent-orchestrator"; +import { isAuthorizedGitHubSessionLogin } from "../auth/security"; import { loadIssueQualityReportMap } from "../services/issue-quality"; import { generateWeeklyValueReport } from "../services/weekly-value-report"; import { REPO_OUTCOME_PATTERNS_SIGNAL, computeRepoOutcomePatterns } from "../services/repo-outcome-patterns"; @@ -536,6 +541,19 @@ async function processGitHubWebhook(env: Env, deliveryId: string, eventName: str } if (payload.repository) await upsertRepositoryFromGitHub(env, payload.repository, installationId ?? undefined); + if (eventName === "reaction" && (await maybeProcessAgentCommandFeedbackReaction(env, deliveryId, payload))) { + await recordWebhookEvent(env, { + deliveryId, + eventName, + action: payload.action, + installationId: payload.installation?.id, + repositoryFullName: payload.repository?.full_name, + payloadHash: "processed", + status: "processed", + }); + return; + } + if (eventName === "issue_comment" && (await maybeProcessGittensoryMentionCommand(env, deliveryId, payload))) { await recordWebhookEvent(env, { deliveryId, @@ -905,6 +923,7 @@ async function maybeProcessGittensoryMentionCommand(env: Env, deliveryId: string return true; } + const answerId = crypto.randomUUID(); const login = pullRequestAuthor ?? commenter; const maintainerDigest = isMaintainerQueueDigestCommand(command.name) ? await buildMaintainerQueueDigestForCommand(env, repo, repoFullName) @@ -923,17 +942,32 @@ async function maybeProcessGittensoryMentionCommand(env: Env, deliveryId: string issue, pullRequest: cachedPullRequest, actorKind: authorization.actorKind === "maintainer" ? "maintainer" : "author", + answerId, officialMiner: official?.status === "confirmed" ? official.snapshot : null, bundle, maintainerDigest, }); - await createOrUpdateAgentCommandComment(env, installationId, repoFullName, issue.number, body); + const responseComment = await createOrUpdateAgentCommandComment(env, installationId, repoFullName, issue.number, body); + await upsertAgentCommandAnswer(env, { + id: answerId, + repoFullName, + issueNumber: issue.number, + command: command.name, + requestCommentId: payload.comment?.id ?? null, + responseCommentId: responseComment?.id ?? null, + responseUrl: responseComment?.html_url ?? null, + actorKind: authorization.actorKind === "maintainer" ? "maintainer" : "author", + metadata: { + publicSurface: "github_comment", + responseCommentStored: Boolean(responseComment?.id), + }, + }); await recordAuditEvent(env, { eventType: "github_app.agent_command_replied", actor: commenter, targetKey: `${repoFullName}#${issue.number}`, outcome: "completed", - metadata: { deliveryId, command: command.name, actorKind: authorization.actorKind, runId: bundle?.run.id ?? null }, + metadata: { deliveryId, command: command.name, actorKind: authorization.actorKind, runId: bundle?.run.id ?? null, answerId }, }); await recordAgentCommandUsage(env, { repoFullName, @@ -1129,6 +1163,157 @@ async function recordAgentCommandFeedbackPrompt( }); } +async function maybeProcessAgentCommandFeedbackReaction(env: Env, deliveryId: string, payload: GitHubWebhookPayload): Promise { + const repoFullName = payload.repository?.full_name; + const issue = payload.issue; + const actor = payload.reaction?.user?.login ?? payload.sender?.login; + const vote = reactionVote(payload.reaction?.content); + const feedback = parseAgentCommandFeedbackContext(payload.comment?.body); + if (!repoFullName || !issue || !actor || !feedback || !vote) return false; + + const targetKey = `${repoFullName}#${issue.number}`; + if (payload.action !== "created") { + await recordAuditEvent(env, { + eventType: "github_app.agent_command_feedback_skipped", + actor, + targetKey, + outcome: "completed", + detail: "unsupported_reaction_action", + metadata: { deliveryId, action: payload.action ?? null, answerId: feedback.answerId }, + }); + return true; + } + if (payload.reaction?.user?.type === "Bot" || /\[bot\]$/i.test(actor)) { + await recordAuditEvent(env, { + eventType: "github_app.agent_command_feedback_skipped", + actor, + targetKey, + outcome: "completed", + detail: "bot_reaction", + metadata: { deliveryId, answerId: feedback.answerId }, + }); + return true; + } + const [answer, cachedPullRequest] = await Promise.all([ + getAgentCommandAnswer(env, feedback.answerId), + getPullRequest(env, repoFullName, issue.number), + ]); + const command = answer?.command ?? feedback.command ?? "unknown"; + if (!answer) { + await recordAuditEvent(env, { + eventType: "github_app.agent_command_feedback_skipped", + actor, + targetKey, + outcome: "completed", + detail: "unknown_answer", + metadata: { deliveryId, answerId: feedback.answerId, command, vote }, + }); + return true; + } + const contextMismatch = answer.repoFullName.toLowerCase() !== repoFullName.toLowerCase() || answer.issueNumber !== issue.number; + if (contextMismatch) { + await recordAuditEvent(env, { + eventType: "github_app.agent_command_feedback_skipped", + actor, + targetKey, + outcome: "completed", + detail: "answer_context_mismatch", + metadata: { deliveryId, answerId: feedback.answerId, command, vote }, + }); + return true; + } + if (!answer.responseCommentId || answer.responseCommentId !== payload.comment?.id) { + await recordAuditEvent(env, { + eventType: "github_app.agent_command_feedback_skipped", + actor, + targetKey, + outcome: "completed", + detail: "answer_comment_mismatch", + metadata: { deliveryId, answerId: feedback.answerId, command, vote, commentId: payload.comment?.id ?? null }, + }); + return true; + } + const pullRequestAuthor = cachedPullRequest?.authorLogin ?? issue.user?.login ?? null; + const official = pullRequestAuthor && actor.toLowerCase() === pullRequestAuthor.toLowerCase() + ? await getCachedOfficialMinerDetection(env, actor, { targetKey, deliveryId }) + : undefined; + const authorization = authorizeFeedbackActor(env, { + actor, + repoFullName, + pullRequestAuthor, + officialAuthorDetection: official, + }); + if (!authorization.authorized) { + await recordAuditEvent(env, { + eventType: "github_app.agent_command_feedback_denied", + actor, + targetKey, + outcome: "denied", + detail: authorization.reason, + metadata: { deliveryId, answerId: feedback.answerId, command, vote }, + }); + return true; + } + + await recordAgentCommandFeedback(env, { + answerId: feedback.answerId, + repoFullName, + issueNumber: issue.number, + command, + actorLogin: actor, + vote, + source: "github_reaction", + actorKind: authorization.actorKind, + metadata: { + deliveryId, + reactionId: payload.reaction?.id ?? null, + }, + }); + await recordAuditEvent(env, { + eventType: "github_app.agent_command_feedback_recorded", + actor, + targetKey, + outcome: "completed", + metadata: { deliveryId, answerId: feedback.answerId, command, vote, source: "github_reaction", actorKind: authorization.actorKind }, + }); + return true; +} + +function reactionVote(content: string | null | undefined): "useful" | "not_useful" | null { + if (content === "+1") return "useful"; + if (content === "-1") return "not_useful"; + return null; +} + +function authorizeFeedbackActor( + env: Env, + args: { + actor: string; + repoFullName: string; + pullRequestAuthor?: string | null | undefined; + officialAuthorDetection?: OfficialGittensorMinerDetection | undefined; + }, +): { authorized: boolean; reason: string; actorKind: "maintainer" | "author" } { + const [owner] = args.repoFullName.split("/"); + if (owner && owner.toLowerCase() === args.actor.toLowerCase()) { + return { authorized: true, reason: "repo_owner_feedback", actorKind: "maintainer" }; + } + if (isAuthorizedGitHubSessionLogin(env, args.actor)) { + return { authorized: true, reason: "operator_feedback", actorKind: "maintainer" }; + } + const authorAuthorization = isAuthorizedCommandActor({ + commenterLogin: args.actor, + commenterAssociation: null, + pullRequestAuthorLogin: args.pullRequestAuthor, + officialAuthorDetection: args.officialAuthorDetection, + }); + return { + authorized: authorAuthorization.authorized, + reason: authorAuthorization.reason, + actorKind: "author", + }; +} + async function auditPrVisibilitySkip( env: Env, repoFullName: string, diff --git a/src/types.ts b/src/types.ts index c17e7ff8b3..afcde5474c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -131,11 +131,19 @@ export type GitHubWebhookPayload = { pull_request?: GitHubPullRequestPayload; issue?: GitHubIssuePayload; comment?: GitHubIssueCommentPayload; + reaction?: GitHubReactionPayload; + sender?: GitHubWebhookUserPayload; label?: { name?: string; }; }; +export type GitHubWebhookUserPayload = { + login?: string; + type?: string; + id?: number; +}; + export type GitHubRepositoryPayload = { id?: number; name: string; @@ -194,6 +202,13 @@ export type GitHubIssuePayload = { pull_request?: unknown; }; +export type GitHubReactionPayload = { + id?: number; + content?: string; + user?: GitHubWebhookUserPayload; + created_at?: string | null; +}; + export type GitHubIssueCommentPayload = { id: number; body?: string | null; @@ -862,6 +877,55 @@ export type DigestSubscriptionRecord = { updatedAt: string; }; +export type CommandFeedbackVote = "useful" | "not_useful"; +export type CommandFeedbackSource = "github_reaction" | "app"; + +export type AgentCommandAnswerRecord = { + id: string; + repoFullName: string; + issueNumber: number; + command: string; + requestCommentId?: number | null | undefined; + responseCommentId?: number | null | undefined; + responseUrl?: string | null | undefined; + actorKind: "maintainer" | "author"; + createdAt?: string | null | undefined; + updatedAt?: string | null | undefined; + metadata: Record; +}; + +export type AgentCommandFeedbackRecord = { + id?: string | undefined; + answerId: string; + repoFullName: string; + issueNumber: number; + command: string; + actorLogin: string; + vote: CommandFeedbackVote; + source: CommandFeedbackSource; + actorKind: "maintainer" | "author"; + createdAt?: string | null | undefined; + updatedAt?: string | null | undefined; + metadata?: Record | undefined; +}; + +export type CommandUsefulnessBucket = { + command: string; + feedbackCount: number; + usefulCount: number; + notUsefulCount: number; + answerCount: number; + usefulnessRate: number | null; + latestFeedbackAt?: string | null | undefined; +}; + +export type CommandUsefulnessSummary = { + windowDays: number; + generatedAt: string; + totals: Omit; + commands: CommandUsefulnessBucket[]; +}; + export type AuditEventRecord = { id?: string | undefined; eventType: string; diff --git a/test/integration/api.test.ts b/test/integration/api.test.ts index c8bd9c98ac..1abe7846d2 100644 --- a/test/integration/api.test.ts +++ b/test/integration/api.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createSessionForGitHubUser, hashToken } from "../../src/auth/security"; import { upsertBounty, + upsertAgentCommandAnswer, upsertBurdenForecast, upsertCheckSummary, upsertInstallation, @@ -1442,12 +1443,72 @@ describe("api routes", () => { settingsPreview: { added: expect.any(Array), removed: expect.any(Array) }, }); + await upsertAgentCommandAnswer(env, { + id: "api-answer-feedback", + repoFullName: "entrius/allways-ui", + issueNumber: 14, + command: "preflight", + requestCommentId: 100, + responseCommentId: 101, + responseUrl: "https://github.com/entrius/allways-ui/pull/14#issuecomment-101", + actorKind: "maintainer", + createdAt: "2026-05-28T00:00:00.000Z", + updatedAt: "2026-05-28T00:00:00.000Z", + metadata: {}, + }); + const unauthenticatedFeedback = await app.request( + "/v1/app/commands/feedback", + { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ answerId: "api-answer-feedback", vote: "useful" }) }, + env, + ); + expect(unauthenticatedFeedback.status).toBe(401); + const invalidFeedback = await app.request( + "/v1/app/commands/feedback", + { method: "POST", headers: cookieHeaders, body: JSON.stringify({ answerId: "bad