diff --git a/src/db/repositories.ts b/src/db/repositories.ts index 5393df9f21..cac992eba9 100644 --- a/src/db/repositories.ts +++ b/src/db/repositories.ts @@ -2647,6 +2647,11 @@ export async function upsertPullRequestFile(env: Env, file: PullRequestFileRecor }); } +export async function deletePullRequestFiles(env: Env, fullName: string, pullNumber: number): Promise { + const db = getDb(env.DB); + await db.delete(pullRequestFiles).where(and(eq(pullRequestFiles.repoFullName, fullName), eq(pullRequestFiles.pullNumber, pullNumber))); +} + export async function listPullRequestFiles(env: Env, fullName: string, pullNumber: number): Promise { const db = getDb(env.DB); const rows = await db diff --git a/src/github/backfill.ts b/src/github/backfill.ts index ac8fff2ade..1264fbb9aa 100644 --- a/src/github/backfill.ts +++ b/src/github/backfill.ts @@ -1,9 +1,11 @@ import { getRepositorySettings, getRepository, + getPullRequest, countOpenIssues, countOpenPullRequests, countRecentMergedPullRequests, + deletePullRequestFiles, countRepoLabels, getInstallation, getLatestRepoGithubTotalsSnapshot, @@ -45,6 +47,7 @@ import type { InstallationHealthRecord, InstallationRecord, JsonValue, + PullRequestDetailSyncStateRecord, PullRequestRecord, RecentMergedPullRequestRecord, RepoGithubTotalsSnapshotRecord, @@ -560,6 +563,34 @@ export async function backfillOpenPullRequestDetails( }; } +export async function refreshPullRequestDetails( + env: Env, + repoFullName: string, + pullNumber: number, +): Promise<{ ok: true; repoFullName: string; pullNumber: number; status: PullRequestDetailSyncStateRecord["status"]; warnings: string[] }> { + const [repo, pr] = await Promise.all([getRepository(env, repoFullName), getPullRequest(env, repoFullName, pullNumber)]); + if (!repo || !pr) { + return { ok: true, repoFullName, pullNumber, status: "partial", warnings: ["Repository or pull request was not found."] }; + } + const token = await tokenForRepo(env, repo); + const warnings: string[] = []; + await upsertPullRequestDetailSyncState(env, { repoFullName, pullNumber, status: "running" }); + await fetchAndStorePullRequestDetails(env, repoFullName, pr, token, warnings); + const syncedAt = nowIso(); + const status: PullRequestDetailSyncStateRecord["status"] = warnings.length > 0 ? "partial" : "complete"; + await upsertPullRequestDetailSyncState(env, { + repoFullName, + pullNumber, + status, + filesSyncedAt: syncedAt, + reviewsSyncedAt: syncedAt, + checksSyncedAt: syncedAt, + lastSyncedAt: syncedAt, + errorSummary: warnings.at(-1), + }); + return { ok: true, repoFullName, pullNumber, status, warnings }; +} + export async function refreshContributorActivity( env: Env, login: string, @@ -1698,20 +1729,25 @@ async function fetchAndStorePullRequestDetails( token: string | undefined, warnings: string[], ): Promise { + const warningStart = warnings.length; const [files, reviews, checks] = await Promise.all([fetchPullRequestFiles(env, repoFullName, pr.number, token, warnings), fetchPullRequestReviews(env, repoFullName, pr.number, token, warnings), fetchPullRequestChecks(env, repoFullName, pr, token, warnings)]); - - for (const file of files) { - await upsertPullRequestFile(env, { - repoFullName, - pullNumber: pr.number, - path: file.filename, - status: file.status, - additions: file.additions ?? 0, - deletions: file.deletions ?? 0, - changes: file.changes ?? 0, - previousFilename: file.previous_filename, - payload: file as unknown as Record, - }); + const fileSyncFailed = warnings.slice(warningStart).some((warning) => warning.startsWith(`File sync failed for #${pr.number}:`)); + + if (!fileSyncFailed) { + await deletePullRequestFiles(env, repoFullName, pr.number); + for (const file of files) { + await upsertPullRequestFile(env, { + repoFullName, + pullNumber: pr.number, + path: file.filename, + status: file.status, + additions: file.additions ?? 0, + deletions: file.deletions ?? 0, + changes: file.changes ?? 0, + previousFilename: file.previous_filename, + payload: file as unknown as Record, + }); + } } for (const review of reviews) { await upsertPullRequestReview(env, { diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 0abc37e874..1738f3c5c1 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -65,6 +65,7 @@ import { enqueueRepositoryOpenDataBackfill, refreshContributorActivity, refreshInstallationHealth, + refreshPullRequestDetails, } from "../github/backfill"; import { contributorRepoStatsFromGittensor, fetchGittensorContributorSnapshot, fetchOfficialGittensorMiner, type GittensorContributorSnapshot, type OfficialGittensorMinerDetection } from "../gittensor/api"; import { createOrUpdateCheckRun, createOrUpdateErroredGateCheckRun, createOrUpdateGateCheckRun, createOrUpdateOverriddenGateCheckRun, createOrUpdatePendingGateCheckRun, createOrUpdateSkippedGateCheckRun, getInstallationId, getRepositoryCollaboratorPermission } from "../github/app"; @@ -923,6 +924,9 @@ async function processGitHubWebhook(env: Env, deliveryId: string, eventName: str }); await persistAdvisory(env, advisory); if (installationId && shouldProcessPullRequestPublicSurface(payload.action)) { + if (settings.slopGateMode !== "off" || settings.manifestPolicyGateMode !== "off") { + await refreshPullRequestDetails(env, repoFullName, pr.number); + } const gate = await maybePublishPrPublicSurface(env, installationId, repoFullName, pr, repo, settings, advisory, { deliveryId, authorType: payload.pull_request.user?.type, diff --git a/test/unit/backfill.test.ts b/test/unit/backfill.test.ts index 1a64839799..d32820b459 100644 --- a/test/unit/backfill.test.ts +++ b/test/unit/backfill.test.ts @@ -18,6 +18,7 @@ import { upsertInstallation, upsertRepoSyncSegment, upsertRepoSyncState, + upsertPullRequestFile, upsertPullRequestFromGitHub, upsertIssueFromGitHub, upsertRepositoryFromGitHub, @@ -32,6 +33,7 @@ import { enrichInstallationHealth, refreshContributorActivity, refreshInstallationHealth, + refreshPullRequestDetails, } from "../../src/github/backfill"; import { normalizeRegistryPayload } from "../../src/registry/normalize"; import { persistRegistrySnapshot } from "../../src/registry/sync"; @@ -1792,6 +1794,84 @@ describe("GitHub backfill", () => { ); }); + + it("refreshes one pull request's files before gate evaluation and drops stale cached paths", async () => { + const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "public-token" }); + await seedRegisteredRepo(env); + await upsertPullRequestFromGitHub(env, "JSONbored/gittensory", { + number: 12, + title: "Refresh files", + state: "open", + user: { login: "oktofeesh1" }, + head: { sha: "new-head" }, + labels: [], + body: "", + }); + await upsertPullRequestFile(env, { + repoFullName: "JSONbored/gittensory", + pullNumber: 12, + path: "stale/old-secret.txt", + status: "modified", + additions: 1, + deletions: 0, + changes: 1, + payload: { filename: "stale/old-secret.txt" }, + }); + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes("/pulls/12/files")) return Response.json([{ filename: "src/current.ts", status: "modified", additions: 2, deletions: 1, changes: 3 }]); + if (url.includes("/pulls/12/reviews")) return Response.json([]); + if (url.includes("/commits/new-head/check-runs")) return Response.json({ check_runs: [] }); + return Response.json([]); + }); + + const result = await refreshPullRequestDetails(env, "JSONbored/gittensory", 12); + + expect(result).toMatchObject({ status: "complete", pullNumber: 12 }); + expect(await listPullRequestFiles(env, "JSONbored/gittensory", 12)).toEqual([expect.objectContaining({ path: "src/current.ts", changes: 3 })]); + }); + + it("preserves cached pull request files when refresh cannot reload the current file list", async () => { + const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "public-token" }); + await seedRegisteredRepo(env); + await upsertPullRequestFromGitHub(env, "JSONbored/gittensory", { + number: 12, + title: "Refresh unavailable", + state: "open", + user: { login: "oktofeesh1" }, + head: { sha: "new-head" }, + labels: [], + body: "", + }); + await upsertPullRequestFile(env, { + repoFullName: "JSONbored/gittensory", + pullNumber: 12, + path: "src/cached.ts", + status: "modified", + additions: 1, + deletions: 0, + changes: 1, + payload: { filename: "src/cached.ts" }, + }); + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + if (url === "https://api.github.com/graphql") { + const query = JSON.parse(String(init?.body ?? "{}")).query as string; + if (query.includes("GittensoryPullRequestDetails")) return Response.json({ data: { repository: { pullRequest: null } } }); + } + if (url.includes("/pulls/12/files")) return new Response("files unavailable", { status: 503 }); + if (url.includes("/pulls/12/reviews")) return Response.json([]); + if (url.includes("/commits/new-head/check-runs")) return Response.json({ check_runs: [] }); + return Response.json([]); + }); + + const result = await refreshPullRequestDetails(env, "JSONbored/gittensory", 12); + + expect(result).toMatchObject({ status: "partial", pullNumber: 12 }); + expect(result.warnings).toEqual([expect.stringContaining("File sync failed for #12")]); + expect(await listPullRequestFiles(env, "JSONbored/gittensory", 12)).toEqual([expect.objectContaining({ path: "src/cached.ts", changes: 1 })]); + }); + it("records partial PR detail state and check summary segment when check-run fetches fail", async () => { const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "public-token" }); await seedRegisteredRepo(env);