From 526a6384d3a1d4812bf2e30897efa7bb63fdc338 Mon Sep 17 00:00:00 2001 From: Bruno Date: Fri, 6 Nov 2026 22:23:18 -0800 Subject: [PATCH 1/6] test(host-service): reproduce GitHub unreachable from an overloaded PR sweep A sweep over 96 projects spawns every gh lookup at once. With gh's timeout running from spawn, calls that would each answer in 0.4 s wait behind the others, get killed, and trip the unreachable hold (#8998). A second test pins that a real outage still trips the gate, and that it does so without one gh spawn per project. --- .../pull-requests/pull-requests.test.ts | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/packages/host-service/src/runtime/pull-requests/pull-requests.test.ts b/packages/host-service/src/runtime/pull-requests/pull-requests.test.ts index 4bc80d615f8..00f95d2a452 100644 --- a/packages/host-service/src/runtime/pull-requests/pull-requests.test.ts +++ b/packages/host-service/src/runtime/pull-requests/pull-requests.test.ts @@ -2490,3 +2490,150 @@ describe("PullRequestRuntimeManager GitHub traffic", () => { expect(getWorkspace(db, "ws")?.pullRequestId).toBeNull(); }); }); + +describe("PullRequestRuntimeManager fleet sweep under load", () => { + const PROJECTS = 96; + const WORKSPACES_PER_PROJECT = 2; + + function seedFleet(db: HostDb) { + const prsByHeadRef: Record> = {}; + for (let p = 0; p < PROJECTS; p++) { + const projectId = `project-${p}`; + db.insert(schema.projects) + .values({ + id: projectId, + repoPath: `/repo-${p}`, + createdAt: Date.now(), + repoProvider: "github", + repoOwner: REPO.owner, + repoName: REPO.name, + repoUrl: `https://github.com/${REPO.owner}/${REPO.name}.git`, + remoteName: "origin", + }) + .run(); + for (let w = 0; w < WORKSPACES_PER_PROJECT; w++) { + const branch = `p${p}-w${w}`; + db.insert(schema.workspaces) + .values({ + id: `ws-${branch}`, + projectId, + worktreePath: `/repo-${p}/.worktrees/${branch}`, + branch, + createdAt: Date.now(), + headSha: `sha-${branch}`, + upstreamOwner: REPO.owner, + upstreamRepo: REPO.name, + upstreamBranch: branch, + }) + .run(); + prsByHeadRef[branch] = makePrNode({ + number: Object.keys(prsByHeadRef).length + 1, + headRef: branch, + headSha: `sha-${branch}`, + }); + } + } + return prsByHeadRef; + } + + function killedByTimeout(args: string[]) { + return Object.assign(new Error(`Command failed: gh ${args.join(" ")}`), { + killed: true, + signal: "SIGTERM", + code: null, + }); + } + + // A machine that runs `slots` gh processes at once, each needing `callMs`. + // Like execFile, the timeout starts at spawn, so time spent waiting behind + // other spawns counts against it. Virtual time: nothing really waits. + function contendedGh( + answer: (args: string[]) => Promise, + machine: { slots: number; callMs: number; timeoutMs: number }, + ) { + const stats = { spawns: 0, killed: 0 }; + let inFlight = 0; + const execGh = async (args: string[]) => { + stats.spawns += 1; + const ahead = inFlight++; + try { + await new Promise((resolve) => setImmediate(resolve)); + const elapsedMs = + (Math.floor(ahead / machine.slots) + 1) * machine.callMs; + if (elapsedMs > machine.timeoutMs) { + stats.killed += 1; + throw killedByTimeout(args); + } + return await answer(args); + } finally { + inFlight--; + } + }; + return { execGh, stats }; + } + + function fleetSweep(manager: PullRequestRuntimeManager) { + const accessible = manager as unknown as { + refreshEligibleProjects(): Promise; + }; + return accessible.refreshEligibleProjects.bind(accessible); + } + + test("a healthy GitHub stays reachable when the sweep covers many projects", async () => { + const db = createRealDb(); + const prsByHeadRef = seedFleet(db); + const warns: string[] = []; + const originalWarn = console.warn; + console.warn = (...args: unknown[]) => { + warns.push(String(args[0])); + }; + // The reporter's numbers: one PR call takes 0.4 s, gh times out at 10 s. + const gh = contendedGh(routeGh(prsByHeadRef), { + slots: 4, + callMs: 400, + timeoutMs: 10_000, + }); + const manager = createManager(db, { + execGh: gh.execGh, + github: (async () => { + throw new Error("octokit must not run"); + }) as never, + }); + try { + await fleetSweep(manager)(); + + expect(manager.getGithubStatus()).toBeNull(); + expect(warns.filter((w) => w.includes("GitHub unreachable"))).toEqual([]); + expect(gh.stats.killed).toBe(0); + expect(getWorkspace(db, "ws-p95-w1")?.pullRequestId).not.toBeNull(); + } finally { + manager.stop(); + console.warn = originalWarn; + } + }); + + test("a GitHub that hangs on every call trips the gate without a spawn per project", async () => { + const db = createRealDb(); + seedFleet(db); + let spawns = 0; + const manager = createManager(db, { + execGh: async (args) => { + spawns += 1; + await new Promise((resolve) => setImmediate(resolve)); + throw killedByTimeout(args); + }, + github: (async () => { + throw new Error("octokit must not run on a dead network"); + }) as never, + }); + try { + await withSilencedWarnings(() => fleetSweep(manager)()); + + expect(manager.getGithubStatus()?.reason).toBe("unreachable"); + expect(spawns).toBeGreaterThan(0); + expect(spawns).toBeLessThan(PROJECTS); + } finally { + manager.stop(); + } + }); +}); From 0a56ab878ad5cca51f01a8b0d952b44a88b0a8f0 Mon Sep 17 00:00:00 2001 From: Bruno Date: Fri, 6 Nov 2026 22:24:52 -0800 Subject: [PATCH 2/6] fix(host-service): bound concurrent GitHub lookups in the PR runtime The fleet sweep refreshed every project at once, and each project fanned out a gh spawn per workspace and three per linked PR. gh's 10 s timeout runs from spawn, so on a host with ~100 projects calls that answer in 0.4 s alone spent their budget waiting behind each other, were killed with SIGTERM, and tripped the unreachable hold. The hold-recovery sweep then fired the same burst again, so the sidebar warning flickered all day. Every GitHub lookup now takes one of 8 slots before it spawns gh, so a queued lookup's timeout has not started yet. The gate is checked after a slot is granted: in a real outage the first lookups time out, trip the hold, and the queued ones fail fast instead of each spawning a gh that hangs. Explicit refreshes (probes) go to the front of the queue so a user action does not wait behind a sweep. Fixes #8998 --- .../runtime/pull-requests/pull-requests.ts | 72 +++++++++++++------ 1 file changed, 49 insertions(+), 23 deletions(-) diff --git a/packages/host-service/src/runtime/pull-requests/pull-requests.ts b/packages/host-service/src/runtime/pull-requests/pull-requests.ts index a88a1de2ff1..eded9181266 100644 --- a/packages/host-service/src/runtime/pull-requests/pull-requests.ts +++ b/packages/host-service/src/runtime/pull-requests/pull-requests.ts @@ -59,6 +59,8 @@ import { type WorkspaceRefsSnapshot, } from "./utils/workspace-refs"; +// gh's timeout runs from spawn, so an unbounded burst times out healthy calls. +const MAX_CONCURRENT_GITHUB_LOOKUPS = 8; // Long-cadence sweep that catches anything `GitWatcher` might miss // (overflow, fs.watch errors, transient watcher failures). Steady-state // branch syncs are event-driven via `GitWatcher.onChanged`; this is a @@ -236,6 +238,8 @@ export class PullRequestRuntimeManager { // appear once the limit resets" is true within seconds rather than at // the next 5-minute sweep. private holdRecoveryTimer: ReturnType | null = null; + private githubLookupsRunning = 0; + private readonly githubLookupQueue: (() => void)[] = []; private readonly workspaceSyncState = new Map< string, { running: Promise; rerunPending: boolean; bypassCache: boolean } @@ -1281,34 +1285,56 @@ export class PullRequestRuntimeManager { viaOctokit: () => Promise, options: { probe?: boolean } = {}, ): Promise { - // An explicit refresh (PR just created, user asked) is allowed through a - // hold: it is one call, and its success is what reopens the gate early - // once the network is back. - if (!options.probe) this.githubGate.assertReachable(); - try { - const result = await viaGh(); - this.githubGate.recordSuccess(); - return result; - } catch (ghError) { - if (isGitHubUnreachableError(ghError)) { - this.noteGitHubFailure(ghError); - throw ghError; - } - console.warn( - `[host-service:pull-request-runtime] gh ${what} failed; falling back to Octokit`, - { ...context, error: ghError }, - ); + if (this.githubLookupsRunning < MAX_CONCURRENT_GITHUB_LOOKUPS) { + this.githubLookupsRunning += 1; + } else { + await this.waitForGitHubSlot(options.probe === true); } try { - const result = await viaOctokit(); - this.githubGate.recordSuccess(); - return result; - } catch (octokitError) { - this.noteGitHubFailure(octokitError); - throw octokitError; + // An explicit refresh (PR just created, user asked) is allowed through a + // hold: it is one call, and its success is what reopens the gate early + // once the network is back. + if (!options.probe) this.githubGate.assertReachable(); + try { + const result = await viaGh(); + this.githubGate.recordSuccess(); + return result; + } catch (ghError) { + if (isGitHubUnreachableError(ghError)) { + this.noteGitHubFailure(ghError); + throw ghError; + } + console.warn( + `[host-service:pull-request-runtime] gh ${what} failed; falling back to Octokit`, + { ...context, error: ghError }, + ); + } + try { + const result = await viaOctokit(); + this.githubGate.recordSuccess(); + return result; + } catch (octokitError) { + this.noteGitHubFailure(octokitError); + throw octokitError; + } + } finally { + this.releaseGitHubSlot(); } } + private waitForGitHubSlot(urgent: boolean): Promise { + return new Promise((resolve) => { + if (urgent) this.githubLookupQueue.unshift(resolve); + else this.githubLookupQueue.push(resolve); + }); + } + + private releaseGitHubSlot(): void { + const next = this.githubLookupQueue.shift(); + if (next) next(); + else this.githubLookupsRunning -= 1; + } + /** Trips the gate on a host-wide failure and says so once per hold. */ private noteGitHubFailure(error: unknown): void { const hold = this.githubGate.recordFailure(error); From 465e16e05e5857cf3c60590d8c40123f69b8faf7 Mon Sep 17 00:00:00 2001 From: Bruno Date: Fri, 6 Nov 2026 22:43:15 -0800 Subject: [PATCH 3/6] test(host-service): reproduce an explicit refresh stuck behind hung Octokit fallbacks Eight sweep lookups whose gh fails over HTTP fall through to an Octokit call that never settles. They hold every GitHub lookup slot, so a later explicit refresh waits forever. --- .../pull-requests/pull-requests.test.ts | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/packages/host-service/src/runtime/pull-requests/pull-requests.test.ts b/packages/host-service/src/runtime/pull-requests/pull-requests.test.ts index 00f95d2a452..316db1844b1 100644 --- a/packages/host-service/src/runtime/pull-requests/pull-requests.test.ts +++ b/packages/host-service/src/runtime/pull-requests/pull-requests.test.ts @@ -155,6 +155,7 @@ function createManager( worktreePath: string, ) => Promise; worktreeExists?: (worktreePath: string) => boolean; + octokitTimeoutMs?: number; } = {}, ) { return new PullRequestRuntimeManager({ @@ -179,6 +180,7 @@ function createManager( // Seeded worktree paths are fabricated; default the disk gate open so // sync-path tests exercise the git read, not the missing-dir skip. worktreeExists: overrides.worktreeExists ?? (() => true), + octokitTimeoutMs: overrides.octokitTimeoutMs, }); } @@ -2612,6 +2614,52 @@ describe("PullRequestRuntimeManager fleet sweep under load", () => { } }); + test("an Octokit fallback that never answers frees its slot for an explicit refresh", async () => { + const db = createRealDb(); + const prsByHeadRef = seedFleet(db); + const answer = routeGh(prsByHeadRef); + let ghAnswers = false; + let hungOctokitCalls = 0; + const manager = createManager(db, { + execGh: async (args) => { + await new Promise((resolve) => setImmediate(resolve)); + if (ghAnswers) return answer(args); + throw new Error("gh: Not Found (HTTP 404)"); + }, + github: () => { + hungOctokitCalls += 1; + return new Promise(() => {}); + }, + octokitTimeoutMs: 5, + }); + let stuckTimer: ReturnType | undefined; + try { + await withSilencedWarnings(async () => { + const sweep = fleetSweep(manager)(); + while (hungOctokitCalls < 8) { + await new Promise((resolve) => setImmediate(resolve)); + } + ghAnswers = true; + + const outcome = await Promise.race([ + manager + .refreshPullRequestsByWorkspaces(["ws-p95-w1"]) + .then(() => "refreshed"), + new Promise((resolve) => { + stuckTimer = setTimeout(resolve, 1_000, "stuck"); + }), + ]); + + expect(outcome).toBe("refreshed"); + expect(getWorkspace(db, "ws-p95-w1")?.pullRequestId).not.toBeNull(); + await sweep; + }); + } finally { + clearTimeout(stuckTimer); + manager.stop(); + } + }); + test("a GitHub that hangs on every call trips the gate without a spawn per project", async () => { const db = createRealDb(); seedFleet(db); From d4037050179c711b9fca720688ef8ab4906d7c6f Mon Sep 17 00:00:00 2001 From: Bruno Date: Fri, 6 Nov 2026 22:43:53 -0800 Subject: [PATCH 4/6] fix(host-service): give the Octokit fallback the same 10 s deadline as gh Octokit is built with no request timeout, so a fallback request that never settles held its lookup slot forever. Eight of them blocked every later lookup, explicit refreshes included. Before the limiter a hung request blocked only its own call. The deadline rejects with ETIMEDOUT, which noteGitHubFailure already classifies as unreachable, so a hung fallback trips the hold the same way a hung gh does. The request is abandoned, not aborted: an AbortSignal would have to go through five github-query helpers and would not cover the token lookup in github(), and the hold stops new requests from starting. --- .../runtime/pull-requests/pull-requests.ts | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/packages/host-service/src/runtime/pull-requests/pull-requests.ts b/packages/host-service/src/runtime/pull-requests/pull-requests.ts index eded9181266..901d5c31334 100644 --- a/packages/host-service/src/runtime/pull-requests/pull-requests.ts +++ b/packages/host-service/src/runtime/pull-requests/pull-requests.ts @@ -61,6 +61,8 @@ import { // gh's timeout runs from spawn, so an unbounded burst times out healthy calls. const MAX_CONCURRENT_GITHUB_LOOKUPS = 8; +// Octokit has no request timeout; this matches gh's so a hung fallback frees its slot. +const OCTOKIT_TIMEOUT_MS = 10_000; // Long-cadence sweep that catches anything `GitWatcher` might miss // (overflow, fs.watch errors, transient watcher failures). Steady-state // branch syncs are event-driven via `GitWatcher.onChanged`; this is a @@ -153,6 +155,7 @@ export interface PullRequestRuntimeManagerOptions { readWorkspaceRefs?: (worktreePath: string) => Promise; /** Test seam for the missing-worktree gate. Defaults to `existsSync`. */ worktreeExists?: (worktreePath: string) => boolean; + octokitTimeoutMs?: number; } interface NormalizedRepoIdentity { @@ -182,6 +185,33 @@ export interface CheckoutPullRequestMetadata { isCrossRepository: boolean; } +// ponytail: the request is not aborted, only abandoned. The ETIMEDOUT trips the +// unreachable hold, so no new requests start; thread an AbortSignal into the +// github-query helpers if abandoned sockets ever matter. +async function rejectAfter( + work: Promise, + ms: number, + what: string, +): Promise { + let timer: ReturnType | undefined; + const deadline = new Promise((_, reject) => { + timer = setTimeout( + () => + reject( + Object.assign(new Error(`${what} timed out after ${ms}ms`), { + code: "ETIMEDOUT", + }), + ), + ms, + ); + }); + try { + return await Promise.race([work, deadline]); + } finally { + clearTimeout(timer); + } +} + function mapCheckoutPullRequestState( state: CheckoutPullRequestMetadata["state"], isDraft: boolean, @@ -238,6 +268,7 @@ export class PullRequestRuntimeManager { // appear once the limit resets" is true within seconds rather than at // the next 5-minute sweep. private holdRecoveryTimer: ReturnType | null = null; + private readonly octokitTimeoutMs: number; private githubLookupsRunning = 0; private readonly githubLookupQueue: (() => void)[] = []; private readonly workspaceSyncState = new Map< @@ -297,6 +328,7 @@ export class PullRequestRuntimeManager { options.readWorkspaceRefs ?? (async (worktreePath) => readWorkspaceRefs(await this.git(worktreePath))); this.worktreeExists = options.worktreeExists ?? existsSync; + this.octokitTimeoutMs = options.octokitTimeoutMs ?? OCTOKIT_TIMEOUT_MS; } start() { @@ -1310,7 +1342,11 @@ export class PullRequestRuntimeManager { ); } try { - const result = await viaOctokit(); + const result = await rejectAfter( + viaOctokit(), + this.octokitTimeoutMs, + `Octokit ${what}`, + ); this.githubGate.recordSuccess(); return result; } catch (octokitError) { From f464e3f4b078ea30dc203e9cd99385c24144d3ad Mon Sep 17 00:00:00 2001 From: Bruno Date: Fri, 6 Nov 2026 22:44:20 -0800 Subject: [PATCH 5/6] fix(host-service): reject a held GitHub lookup before it queues for a slot A sweep lookup that starts during a hold now fails at once instead of waiting behind the running lookups only to fail after it gets a slot. The check after the slot is granted stays, for a hold that opens while the lookup waits. The early check throws before the slot count changes. --- packages/host-service/src/runtime/pull-requests/pull-requests.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/host-service/src/runtime/pull-requests/pull-requests.ts b/packages/host-service/src/runtime/pull-requests/pull-requests.ts index 901d5c31334..e71df25df68 100644 --- a/packages/host-service/src/runtime/pull-requests/pull-requests.ts +++ b/packages/host-service/src/runtime/pull-requests/pull-requests.ts @@ -1317,6 +1317,7 @@ export class PullRequestRuntimeManager { viaOctokit: () => Promise, options: { probe?: boolean } = {}, ): Promise { + if (!options.probe) this.githubGate.assertReachable(); if (this.githubLookupsRunning < MAX_CONCURRENT_GITHUB_LOOKUPS) { this.githubLookupsRunning += 1; } else { From df2408244e3f3c00b017e1d23461869288163d01 Mon Sep 17 00:00:00 2001 From: Bruno Date: Sat, 7 Nov 2026 00:03:37 -0800 Subject: [PATCH 6/6] refactor(host-service): share the gh timeout default with the Octokit fallback --- .../src/runtime/pull-requests/pull-requests.ts | 9 +++++---- .../src/trpc/router/workspace-creation/utils/exec-gh.ts | 4 +++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/host-service/src/runtime/pull-requests/pull-requests.ts b/packages/host-service/src/runtime/pull-requests/pull-requests.ts index e71df25df68..7f1637493d6 100644 --- a/packages/host-service/src/runtime/pull-requests/pull-requests.ts +++ b/packages/host-service/src/runtime/pull-requests/pull-requests.ts @@ -13,7 +13,10 @@ import { } from "../../db/schema"; import type { EventBus } from "../../events/event-bus"; import type { GitWatcher } from "../../events/git-watcher"; -import type { ExecGh } from "../../trpc/router/workspace-creation/utils/exec-gh"; +import { + type ExecGh, + GH_TIMEOUT_MS, +} from "../../trpc/router/workspace-creation/utils/exec-gh"; import { type GitFactory, resolveDefaultBranchName } from "../git"; import { ConditionalGh } from "./utils/conditional-gh"; import { @@ -61,8 +64,6 @@ import { // gh's timeout runs from spawn, so an unbounded burst times out healthy calls. const MAX_CONCURRENT_GITHUB_LOOKUPS = 8; -// Octokit has no request timeout; this matches gh's so a hung fallback frees its slot. -const OCTOKIT_TIMEOUT_MS = 10_000; // Long-cadence sweep that catches anything `GitWatcher` might miss // (overflow, fs.watch errors, transient watcher failures). Steady-state // branch syncs are event-driven via `GitWatcher.onChanged`; this is a @@ -328,7 +329,7 @@ export class PullRequestRuntimeManager { options.readWorkspaceRefs ?? (async (worktreePath) => readWorkspaceRefs(await this.git(worktreePath))); this.worktreeExists = options.worktreeExists ?? existsSync; - this.octokitTimeoutMs = options.octokitTimeoutMs ?? OCTOKIT_TIMEOUT_MS; + this.octokitTimeoutMs = options.octokitTimeoutMs ?? GH_TIMEOUT_MS; } start() { diff --git a/packages/host-service/src/trpc/router/workspace-creation/utils/exec-gh.ts b/packages/host-service/src/trpc/router/workspace-creation/utils/exec-gh.ts index 5d4cf6c0f2a..81249bcd079 100644 --- a/packages/host-service/src/trpc/router/workspace-creation/utils/exec-gh.ts +++ b/packages/host-service/src/trpc/router/workspace-creation/utils/exec-gh.ts @@ -4,6 +4,8 @@ import { getToolEnvironment } from "../../../../terminal/clean-shell-env"; const execFileAsync = promisify(execFile); +export const GH_TIMEOUT_MS = 10_000; + export interface ExecGhOptions { cwd?: string; timeout?: number; @@ -28,7 +30,7 @@ export const execGh: ExecGh = async (args, options) => { const env = await getToolEnvironment(); const child = execFileAsync("gh", args, { encoding: "utf8", - timeout: options?.timeout ?? 10_000, + timeout: options?.timeout ?? GH_TIMEOUT_MS, // Node's 1MB default dies on large REST payloads (open-PR sweeps of // busy repos exceed it even paginated). maxBuffer: options?.maxBuffer ?? 10 * 1024 * 1024,