From 1dfafb32621bf3e44b586e36e7ac1d3fe021c31c Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sat, 13 Jun 2026 15:19:56 -0700 Subject: [PATCH 1/2] fix(gate): never leave the Gittensory Gate check stuck in_progress MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Production incident: the 'Gittensory Gate is evaluating' check on PR #650 stayed in_progress for ~8h. Root cause: maybePublishPrPublicSurface posts the pending (in_progress) gate check, then does D1 + GitHub + Gittensor work before the completing PATCH — with no request timeouts and no failure finalization. If anything in that gap hangs (a bare fetch to a slow upstream) or throws, the completing PATCH never runs and the check is orphaned forever; the caller's .catch only logs it. Confirmed via prod: miner detection was a cache hit at 13:35:08, the pending check posted at 13:35:11, then nothing — no completion, no error row (a hang, not a caught throw), while prod D1 was intermittently overloaded. Fix: - Bound every external call in the gate window with a request timeout so a hang becomes a catchable error: AbortSignal.timeout on the GitHub App fetches + the Octokit instance (src/github/app.ts) and on the Gittensor API client (src/gittensor/api.ts, the single fetchJson chokepoint). - Wrap the pending-post -> completion window in try/catch. On any failure, finalize the SAME check run to a neutral, non-blocking 'could not finish evaluating — will re-run' state (createOrUpdateErroredGateCheckRun) and audit it, so the Gate is always terminal and never hangs. Only finalizes when a real conclusion was not already published (no clobbering verdicts). Test proves a failed completion PATCH is followed by a neutral finalize of the same check id. Coverage holds above the 97% gate. --- src/github/app.ts | 48 ++++++++++++-- src/gittensor/api.ts | 6 ++ src/queue/processors.ts | 135 ++++++++++++++++++++++++---------------- test/unit/queue.test.ts | 63 +++++++++++++++++++ 4 files changed, 195 insertions(+), 57 deletions(-) diff --git a/src/github/app.ts b/src/github/app.ts index da498c63f5..8b5b972716 100644 --- a/src/github/app.ts +++ b/src/github/app.ts @@ -26,9 +26,20 @@ export const GITTENSORY_GATE_CHECK_NAME = "Gittensory Gate"; type GitHubCheckConclusion = Advisory["conclusion"] | GateCheckConclusion | "skipped"; type GitHubCheckStatus = "queued" | "in_progress" | "completed"; +/** Hard cap on a single GitHub API request. Without it a slow/half-open GitHub connection can hang the + * Worker — e.g. the Gate's own completing PATCH stalling after the pending check was posted, which leaves + * the check in_progress forever. A bounded timeout turns a hang into a catchable error the caller can + * finalize. Applied to every raw fetch here and to the Octokit instances (via a timeout-injecting fetch). */ +const GITHUB_FETCH_TIMEOUT_MS = 12_000; + +function timeoutFetch(input: RequestInfo | URL, init?: RequestInit): Promise { + if (init?.signal) return fetch(input, init); + return fetch(input, { ...(init ?? {}), signal: AbortSignal.timeout(GITHUB_FETCH_TIMEOUT_MS) }); +} + export async function createInstallationToken(env: Env, installationId: number): Promise { const jwt = await createAppJwt(env); - const response = await fetch(`https://api.github.com/app/installations/${installationId}/access_tokens`, { + const response = await timeoutFetch(`https://api.github.com/app/installations/${installationId}/access_tokens`, { method: "POST", headers: githubHeaders(`Bearer ${jwt}`), }); @@ -43,7 +54,7 @@ export async function createInstallationToken(env: Env, installationId: number): export async function getAppInstallation(env: Env, installationId: number): Promise> { const jwt = await createAppJwt(env); - const response = await fetch(`https://api.github.com/app/installations/${installationId}`, { + const response = await timeoutFetch(`https://api.github.com/app/installations/${installationId}`, { headers: githubHeaders(`Bearer ${jwt}`), }); if (!response.ok) { @@ -66,7 +77,7 @@ export async function getRepositoryCollaboratorPermission( const [owner, name] = repoFullName.split("/"); if (!owner || !name || !login) return null; const token = await createInstallationToken(env, installationId); - const response = await fetch( + const response = await timeoutFetch( `https://api.github.com/repos/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/collaborators/${encodeURIComponent(login)}/permission`, { headers: githubHeaders(`Bearer ${token}`) }, ); @@ -163,6 +174,33 @@ export async function createOrUpdateSkippedGateCheckRun( }); } +/** + * Finalize a previously-posted pending Gate check to a NEUTRAL (non-blocking) terminal state when the + * evaluation could not finish (a transient error/timeout in the work between posting the pending check and + * completing it). This guarantees the "Gittensory Gate is evaluating" run never hangs in_progress forever; + * it does not block the PR and re-runs on the next push. Targets the known pending check_run id so it + * updates the SAME run rather than creating a second one. + */ +export async function createOrUpdateErroredGateCheckRun( + env: Env, + installationId: number, + repoFullName: string, + advisory: Advisory, + options: { checkRunId?: number | undefined } = {}, +): Promise { + return createOrUpdateNamedCheckRun(env, installationId, repoFullName, advisory, { + name: GITTENSORY_GATE_CHECK_NAME, + status: "completed", + conclusion: "neutral", + output: { + title: "Gittensory Gate — could not finish evaluating", + summary: "A transient error interrupted gate evaluation. This does NOT block the PR and re-runs automatically on the next push.", + text: "Gittensory finalizes the Gate to a neutral, non-blocking state when evaluation is interrupted, so the check never hangs in_progress. Push a new commit or use the 'Re-run Gittensory review' checkbox to re-evaluate.", + }, + checkRunId: options.checkRunId, + }); +} + async function createOrUpdateNamedCheckRun( env: Env, installationId: number, @@ -181,7 +219,9 @@ async function createOrUpdateNamedCheckRun( if (!owner || !repo) throw new Error(`Invalid repository full name: ${repoFullName}`); const token = await createInstallationToken(env, installationId); - const octokit = new Octokit({ auth: token }); + // Inject a per-request timeout so a stalled GitHub API call (e.g. the Gate's completing PATCH) can never + // hang the Worker and orphan the in_progress check. + const octokit = new Octokit({ auth: token, request: { fetch: timeoutFetch } }); try { if (check.checkRunId) { diff --git a/src/gittensor/api.ts b/src/gittensor/api.ts index 4427770932..355564c5fa 100644 --- a/src/gittensor/api.ts +++ b/src/gittensor/api.ts @@ -251,12 +251,18 @@ async function buildGittensorContributorSnapshot(miner: ConfirmedGittensorMinerS }; } +/** Hard cap on a single Gittensor API request so a slow/half-open upstream connection can never hang the + * Worker indefinitely (it would otherwise stall the webhook between posting and completing the Gate + * check, leaving the check in_progress forever — see the gate-finalization fix). */ +const GITTENSOR_FETCH_TIMEOUT_MS = 10_000; + async function fetchJson(url: string): Promise { const response = await fetch(url, { headers: { accept: "application/json", "user-agent": "gittensory/0.1", }, + signal: AbortSignal.timeout(GITTENSOR_FETCH_TIMEOUT_MS), }); if (!response.ok) throw new Error(`Gittensor API failed for ${url} (${response.status})`); return (await response.json()) as T; diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 368f0d55c9..64894489ae 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -61,7 +61,7 @@ import { refreshInstallationHealth, } from "../github/backfill"; import { contributorRepoStatsFromGittensor, fetchGittensorContributorSnapshot, fetchOfficialGittensorMiner, type GittensorContributorSnapshot, type OfficialGittensorMinerDetection } from "../gittensor/api"; -import { createOrUpdateCheckRun, createOrUpdateGateCheckRun, createOrUpdatePendingGateCheckRun, createOrUpdateSkippedGateCheckRun, getInstallationId, getRepositoryCollaboratorPermission } from "../github/app"; +import { createOrUpdateCheckRun, createOrUpdateErroredGateCheckRun, createOrUpdateGateCheckRun, createOrUpdatePendingGateCheckRun, createOrUpdateSkippedGateCheckRun, getInstallationId, getRepositoryCollaboratorPermission } from "../github/app"; import { createOrUpdateAgentCommandComment, createOrUpdatePrIntelligenceComment, PR_PANEL_COMMENT_MARKER } from "../github/comments"; import { gittensoryFooter, gittensorRepoEarnUrl } from "../github/footer"; import { @@ -944,62 +944,91 @@ async function maybePublishPrPublicSurface( } } - const [repoIssues, repoPullRequests, repoBounties] = await Promise.all([ - listIssues(env, repoFullName), - listPullRequests(env, repoFullName), - listBountiesByRepo(env, repoFullName), - ]); - const collisions = buildCollisionReport(repoFullName, repoIssues, repoPullRequests); - const queueHealth = buildQueueHealth(repo, repoIssues, repoPullRequests, collisions); - const preflight = buildPreflightResult( - { - repoFullName, - contributorLogin: author ?? undefined, - title: pr.title, - body: pr.body ?? undefined, - labels: pr.labels, - linkedIssues: pr.linkedIssues, - authorAssociation: pr.authorAssociation ?? undefined, - }, - repo, - repoIssues, - repoPullRequests, - repoBounties, - ); - const readiness = buildPublicReadinessScore({ - pr, - preflight, - queueHealth, - linkedDuplicatePrs: linkedIssueDuplicatePullRequestsForGate(pr, repoPullRequests), - scopedOverlapCount: unionScopedOverlapClusters(collisions, pr, preflight.collisions).length, - }); - - if (gateEnabled && author && !publicSurfaceSkipped && !official) { - official = await getCachedOfficialMinerDetection(env, author, { - targetKey: `${repoFullName}#${pr.number}`, - deliveryId: webhook.deliveryId, - }); - } - - // Only CONFIRMED gittensor contributors can be hard-blocked; everyone else (or an unavailable - // detection) gets a neutral, non-blocking gate. Gate-only runs still verify confirmation before - // evaluating blockers so confirmed contributors cannot bypass a required Gate check. - const confirmedContributor = official?.status === "confirmed"; - const gateEvaluation = gateEnabled ? evaluateGateCheck(advisory, gateCheckPolicy(settings, readiness.total, confirmedContributor)) : undefined; - if (gateEnabled) { - const gateCheckResult = await createOrUpdateGateCheckRun( - env, - installationId, - repoFullName, - advisory, - gateCheckPolicy(settings, readiness.total, confirmedContributor), + // The pending Gate check is now posted (status in_progress). Everything from here until the gate is + // completed runs inside a try so that ANY failure/timeout (a slow Gittensor or GitHub call, a D1 error) + // still finalizes the check to a neutral, non-blocking state instead of orphaning it in_progress forever + // (the cause of the multi-hour stuck Gate). External calls in this window are bounded by request timeouts + // (GitHub App + Gittensor API), so a hang becomes a catchable error here. + let collisions!: ReturnType; + let queueHealth!: ReturnType; + let preflight!: ReturnType; + let gateEvaluation: ReturnType | undefined; + let gateFinalized = false; + try { + const [repoIssues, repoPullRequests, repoBounties] = await Promise.all([ + listIssues(env, repoFullName), + listPullRequests(env, repoFullName), + listBountiesByRepo(env, repoFullName), + ]); + collisions = buildCollisionReport(repoFullName, repoIssues, repoPullRequests); + queueHealth = buildQueueHealth(repo, repoIssues, repoPullRequests, collisions); + preflight = buildPreflightResult( { - checkRunId: pendingGateCheckRunId, + repoFullName, + contributorLogin: author ?? undefined, + title: pr.title, + body: pr.body ?? undefined, + labels: pr.labels, + linkedIssues: pr.linkedIssues, + authorAssociation: pr.authorAssociation ?? undefined, }, + repo, + repoIssues, + repoPullRequests, + repoBounties, ); - if (gateCheckResult?.kind === "permission_missing") { - await auditGateCheckPermissionMissing(env, author, repoFullName, pr.number, webhook.deliveryId, gateCheckResult.warning); + const readiness = buildPublicReadinessScore({ + pr, + preflight, + queueHealth, + linkedDuplicatePrs: linkedIssueDuplicatePullRequestsForGate(pr, repoPullRequests), + scopedOverlapCount: unionScopedOverlapClusters(collisions, pr, preflight.collisions).length, + }); + + if (gateEnabled && author && !publicSurfaceSkipped && !official) { + official = await getCachedOfficialMinerDetection(env, author, { + targetKey: `${repoFullName}#${pr.number}`, + deliveryId: webhook.deliveryId, + }); + } + + // Only CONFIRMED gittensor contributors can be hard-blocked; everyone else (or an unavailable + // detection) gets a neutral, non-blocking gate. Gate-only runs still verify confirmation before + // evaluating blockers so confirmed contributors cannot bypass a required Gate check. + const confirmedContributor = official?.status === "confirmed"; + gateEvaluation = gateEnabled ? evaluateGateCheck(advisory, gateCheckPolicy(settings, readiness.total, confirmedContributor)) : undefined; + if (gateEnabled) { + const gateCheckResult = await createOrUpdateGateCheckRun( + env, + installationId, + repoFullName, + advisory, + gateCheckPolicy(settings, readiness.total, confirmedContributor), + { + checkRunId: pendingGateCheckRunId, + }, + ); + if (gateCheckResult?.kind === "published") gateFinalized = true; + if (gateCheckResult?.kind === "permission_missing") { + await auditGateCheckPermissionMissing(env, author, repoFullName, pr.number, webhook.deliveryId, gateCheckResult.warning); + } + } + } catch (error) { + // The pending Gate check was posted but evaluation could not finish. Finalize it to a neutral + // (non-blocking) terminal state so it never hangs in_progress; it re-runs on the next push. Only when + // the gate was enabled, a pending check id exists, and a real conclusion was not already published. + if (gateEnabled && pendingGateCheckRunId !== undefined && !gateFinalized) { + await createOrUpdateErroredGateCheckRun(env, installationId, repoFullName, advisory, { checkRunId: pendingGateCheckRunId }).catch(() => undefined); + await recordAuditEvent(env, { + eventType: "github_app.gate_finalized_on_error", + actor: author, + targetKey: `${repoFullName}#${pr.number}`, + outcome: "error", + detail: errorMessage(error), + metadata: { deliveryId: webhook.deliveryId, repoFullName }, + }).catch(() => undefined); } + throw error; } if (!prelimHasPublicOutput) return; diff --git a/test/unit/queue.test.ts b/test/unit/queue.test.ts index 37abcaa861..2228f8e1fd 100644 --- a/test/unit/queue.test.ts +++ b/test/unit/queue.test.ts @@ -968,6 +968,69 @@ describe("queue processors", () => { expect(gatePatchBody.output?.title).toBe("Gittensory Gate: No linked issue detected"); }); + it("finalizes the Gate to neutral instead of leaving it in_progress when gate completion fails", async () => { + const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem() }); + await persistRegistrySnapshot( + env, + normalizeRegistryPayload( + { "JSONbored/gittensory": { emission_share: 0.01, issue_discovery_share: 0 } }, + { kind: "raw-github", url: "https://example.test" }, + "2026-05-23T00:00:00.000Z", + ), + ); + await upsertRepositoryFromGitHub(env, { name: "gittensory", full_name: "JSONbored/gittensory", private: false, owner: { login: "JSONbored" } }, 123); + await upsertRepositorySettings(env, { + repoFullName: "JSONbored/gittensory", + commentMode: "off", + publicSurface: "off", + autoLabelEnabled: false, + checkRunMode: "off", + gateCheckMode: "enabled", + linkedIssueGateMode: "off", + }); + const patchBodies: Array<{ status?: string; conclusion?: string; output?: { title?: string } }> = []; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url === "https://api.gittensor.io/miners") return Response.json([]); + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/commits/finalize123/check-runs")) return Response.json({ total_count: 0, check_runs: [] }); + if (url.includes("/check-runs") && method === "POST") return Response.json({ id: 970 }, { status: 201 }); // pending + if (url.includes("/check-runs/970") && method === "PATCH") { + const body = JSON.parse(String(init?.body ?? "{}")) as { status?: string; conclusion?: string; output?: { title?: string } }; + patchBodies.push(body); + // First PATCH = the gate completion; fail it transiently so the catch must finalize the check. + if (patchBodies.length === 1) return new Response(JSON.stringify({ message: "server error" }), { status: 500 }); + return Response.json({ id: 970 }); + } + return new Response("not found", { status: 404 }); + }); + + await processJob(env, { + type: "github-webhook", + deliveryId: "gate-finalize-on-error", + eventName: "pull_request", + payload: { + action: "opened", + installation: { id: 123, account: { login: "JSONbored", id: 1, type: "User" } }, + repository: { name: "gittensory", full_name: "JSONbored/gittensory", private: false, owner: { login: "JSONbored" } }, + pull_request: { number: 80, title: "Some change", state: "open", user: { login: "contributor" }, head: { sha: "finalize123" }, labels: [], body: "No issue link." }, + }, + }); + + // The completion PATCH failed (500), so the catch finalized the SAME check run (id 970) to a neutral, + // non-blocking terminal state — never left hanging in_progress. + expect(patchBodies.length).toBe(2); + const finalize = patchBodies[1]; + expect(finalize?.status).toBe("completed"); + expect(finalize?.conclusion).toBe("neutral"); + expect(finalize?.output?.title).toBe("Gittensory Gate — could not finish evaluating"); + const audit = await env.DB.prepare("select outcome from audit_events where event_type = ? and target_key = ?") + .bind("github_app.gate_finalized_on_error", "JSONbored/gittensory#80") + .first<{ outcome: string }>(); + expect(audit?.outcome).toBe("error"); + }); + it("disables the gate from .gittensory.yml (gate.enabled: false) even when repo settings enable it", async () => { const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem() }); await persistRegistrySnapshot( From 7371a4dd3c9ce001fefa13833c77b6708692a8d0 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sat, 13 Jun 2026 15:31:53 -0700 Subject: [PATCH 2/2] fix(queue): dead-letter exhausted gittensory-jobs instead of dropping them silently The gittensory-jobs consumer had no dead_letter_queue, so a webhook job that fails its retries was silently dropped (no record once webhook_events isn't reached). Route exhausted jobs to a new gittensory-jobs-dlq landing queue (created on the account; no consumer, matching the house pattern) and set max_retries explicitly. Validated with wrangler deploy --dry-run. Co-requisite infra (already provisioned): `wrangler queues create gittensory-jobs-dlq`. --- wrangler.jsonc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/wrangler.jsonc b/wrangler.jsonc index efde65e876..e5b295f628 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -84,6 +84,11 @@ "queue": "gittensory-jobs", "max_batch_size": 10, "max_batch_timeout": 5, + // After max_retries failed attempts a job is dead-lettered instead of silently dropped, so a + // persistently-failing webhook is observable (inspect via `wrangler queues` / the dashboard) + // rather than vanishing. The DLQ is a landing pad with no consumer (house pattern). + "max_retries": 3, + "dead_letter_queue": "gittensory-jobs-dlq", }, ], },