Skip to content

fix(selfhost): fast-fail codex on a stdin-read hang before the full timeout #3443

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JSONbored merged 2 commits into from
Jul 5, 2026
Merged

fix(selfhost): fast-fail codex on a stdin-read hang before the full timeout #3443

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e7b2c2a
fix(selfhost): fast-fail codex when it never produces any output
JSONbored Jul 5, 2026
d32e831
fix(selfhost): clear the codex fast-fail deadline on stdout only, not…
JSONbored Jul 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 33 additions & 28 deletions apps/gittensory-ui/src/lib/selfhost-env-reference.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@ export type SelfHostEnvReferenceRow = {
export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
{
name: "AI_COMBINE",
firstReference: "src/selfhost/ai.ts:1080",
firstReference: "src/selfhost/ai.ts:1160",
},
{
name: "AI_DUAL_REVIEW",
firstReference: "src/selfhost/ai.ts:1055",
firstReference: "src/selfhost/ai.ts:1135",
},
{
name: "AI_EMBED_API_KEY",
Expand All @@ -23,27 +23,27 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
},
{
name: "AI_EMBED_MODEL",
firstReference: "src/selfhost/ai.ts:952",
firstReference: "src/selfhost/ai.ts:1032",
},
{
name: "AI_ON_MERGE",
firstReference: "src/selfhost/ai.ts:1082",
firstReference: "src/selfhost/ai.ts:1162",
},
{
name: "AI_PROVIDER",
firstReference: "src/selfhost/ai-config.ts:43",
},
{
name: "ANTHROPIC_AI_BASE_URL",
firstReference: "src/selfhost/ai.ts:956",
firstReference: "src/selfhost/ai.ts:1036",
},
{
name: "ANTHROPIC_AI_MODEL",
firstReference: "src/selfhost/ai.ts:96",
},
{
name: "ANTHROPIC_API_KEY",
firstReference: "src/selfhost/ai.ts:955",
firstReference: "src/selfhost/ai.ts:1035",
},
{
name: "BACKUP_ACKNOWLEDGED",
Expand All @@ -69,6 +69,10 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
name: "CODEX_AI_EFFORT",
firstReference: "src/selfhost/ai.ts:151",
},
{
name: "CODEX_AI_FIRST_OUTPUT_TIMEOUT_MS",
firstReference: "src/selfhost/ai.ts:171",
},
{
name: "CODEX_AI_MODEL",
firstReference: "src/selfhost/ai.ts:92",
Expand All @@ -79,7 +83,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
},
{
name: "CODEX_HOME",
firstReference: "src/selfhost/ai.ts:318",
firstReference: "src/selfhost/ai.ts:340",
},
{
name: "CRON_INTERVAL_MS",
Expand Down Expand Up @@ -151,7 +155,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
},
{
name: "HOME",
firstReference: "src/selfhost/ai.ts:318",
firstReference: "src/selfhost/ai.ts:340",
},
{
name: "MAINTENANCE_ADMISSION_DEFER_MS",
Expand Down Expand Up @@ -199,35 +203,35 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
},
{
name: "OLLAMA_AI_API_KEY",
firstReference: "src/selfhost/ai.ts:949",
firstReference: "src/selfhost/ai.ts:1029",
},
{
name: "OLLAMA_AI_BASE_URL",
firstReference: "src/selfhost/ai.ts:945",
firstReference: "src/selfhost/ai.ts:1025",
},
{
name: "OLLAMA_AI_MODEL",
firstReference: "src/selfhost/ai.ts:100",
},
{
name: "OPENAI_AI_BASE_URL",
firstReference: "src/selfhost/ai.ts:947",
firstReference: "src/selfhost/ai.ts:1027",
},
{
name: "OPENAI_AI_MODEL",
firstReference: "src/selfhost/ai.ts:101",
},
{
name: "OPENAI_API_KEY",
firstReference: "src/selfhost/ai.ts:949",
firstReference: "src/selfhost/ai.ts:1029",
},
{
name: "OPENAI_COMPATIBLE_AI_API_KEY",
firstReference: "src/selfhost/ai.ts:949",
firstReference: "src/selfhost/ai.ts:1029",
},
{
name: "OPENAI_COMPATIBLE_AI_BASE_URL",
firstReference: "src/selfhost/ai.ts:948",
firstReference: "src/selfhost/ai.ts:1028",
},
{
name: "OPENAI_COMPATIBLE_AI_MODEL",
Expand Down Expand Up @@ -390,25 +394,26 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
"| Name | First reference |",
"| --- | --- |",
"| `AI_COMBINE` | `src/selfhost/ai.ts:1080` |",
"| `AI_DUAL_REVIEW` | `src/selfhost/ai.ts:1055` |",
"| `AI_COMBINE` | `src/selfhost/ai.ts:1160` |",
"| `AI_DUAL_REVIEW` | `src/selfhost/ai.ts:1135` |",
"| `AI_EMBED_API_KEY` | `src/server.ts:441` |",
"| `AI_EMBED_BASE_URL` | `src/server.ts:438` |",
"| `AI_EMBED_MODEL` | `src/selfhost/ai.ts:952` |",
"| `AI_ON_MERGE` | `src/selfhost/ai.ts:1082` |",
"| `AI_EMBED_MODEL` | `src/selfhost/ai.ts:1032` |",
"| `AI_ON_MERGE` | `src/selfhost/ai.ts:1162` |",
"| `AI_PROVIDER` | `src/selfhost/ai-config.ts:43` |",
"| `ANTHROPIC_AI_BASE_URL` | `src/selfhost/ai.ts:956` |",
"| `ANTHROPIC_AI_BASE_URL` | `src/selfhost/ai.ts:1036` |",
"| `ANTHROPIC_AI_MODEL` | `src/selfhost/ai.ts:96` |",
"| `ANTHROPIC_API_KEY` | `src/selfhost/ai.ts:955` |",
"| `ANTHROPIC_API_KEY` | `src/selfhost/ai.ts:1035` |",
"| `BACKUP_ACKNOWLEDGED` | `src/server.ts:380` |",
"| `BROWSER_WS_ENDPOINT` | `src/selfhost/stubs/puppeteer.ts:11` |",
"| `CLAUDE_AI_EFFORT` | `src/selfhost/ai.ts:147` |",
"| `CLAUDE_AI_MODEL` | `src/selfhost/ai.ts:88` |",
"| `CLAUDE_AI_TIMEOUT_MS` | `src/selfhost/ai.ts:147` |",
"| `CODEX_AI_EFFORT` | `src/selfhost/ai.ts:151` |",
"| `CODEX_AI_FIRST_OUTPUT_TIMEOUT_MS` | `src/selfhost/ai.ts:171` |",
"| `CODEX_AI_MODEL` | `src/selfhost/ai.ts:92` |",
"| `CODEX_AI_TIMEOUT_MS` | `src/selfhost/ai.ts:151` |",
"| `CODEX_HOME` | `src/selfhost/ai.ts:318` |",
"| `CODEX_HOME` | `src/selfhost/ai.ts:340` |",
"| `CRON_INTERVAL_MS` | `src/server.ts:919` |",
"| `DATABASE_PATH` | `src/server.ts:250` |",
"| `DATABASE_URL` | `src/selfhost/preflight.ts:201` |",
Expand All @@ -426,7 +431,7 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
"| `GITHUB_INSTALLATION_CONCURRENCY_LIMIT` | `src/selfhost/installation-concurrency-admission.ts:43` |",
"| `GITTENSORY_REPO_CONFIG_DIR` | `src/server.ts:289` |",
"| `GITTENSORY_VERSION` | `src/selfhost/otel.ts:62` |",
"| `HOME` | `src/selfhost/ai.ts:318` |",
"| `HOME` | `src/selfhost/ai.ts:340` |",
"| `MAINTENANCE_ADMISSION_DEFER_MS` | `src/selfhost/maintenance-admission.ts:171` |",
"| `MAINTENANCE_ADMISSION_DRAIN_AGE_MS` | `src/selfhost/maintenance-admission.ts:145` |",
"| `MAINTENANCE_ADMISSION_ENABLED` | `src/selfhost/maintenance-admission.ts:126` |",
Expand All @@ -438,14 +443,14 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
"| `MIGRATIONS_DIR` | `src/server.ts:393` |",
"| `OBSERVABILITY_SMOKE_POLL_MS` | `scripts/smoke-observability-traces.mjs:8` |",
"| `OBSERVABILITY_SMOKE_TIMEOUT_MS` | `scripts/smoke-observability-traces.mjs:6` |",
"| `OLLAMA_AI_API_KEY` | `src/selfhost/ai.ts:949` |",
"| `OLLAMA_AI_BASE_URL` | `src/selfhost/ai.ts:945` |",
"| `OLLAMA_AI_API_KEY` | `src/selfhost/ai.ts:1029` |",
"| `OLLAMA_AI_BASE_URL` | `src/selfhost/ai.ts:1025` |",
"| `OLLAMA_AI_MODEL` | `src/selfhost/ai.ts:100` |",
"| `OPENAI_AI_BASE_URL` | `src/selfhost/ai.ts:947` |",
"| `OPENAI_AI_BASE_URL` | `src/selfhost/ai.ts:1027` |",
"| `OPENAI_AI_MODEL` | `src/selfhost/ai.ts:101` |",
"| `OPENAI_API_KEY` | `src/selfhost/ai.ts:949` |",
"| `OPENAI_COMPATIBLE_AI_API_KEY` | `src/selfhost/ai.ts:949` |",
"| `OPENAI_COMPATIBLE_AI_BASE_URL` | `src/selfhost/ai.ts:948` |",
"| `OPENAI_API_KEY` | `src/selfhost/ai.ts:1029` |",
"| `OPENAI_COMPATIBLE_AI_API_KEY` | `src/selfhost/ai.ts:1029` |",
"| `OPENAI_COMPATIBLE_AI_BASE_URL` | `src/selfhost/ai.ts:1028` |",
"| `OPENAI_COMPATIBLE_AI_MODEL` | `src/selfhost/ai.ts:102` |",
"| `ORB_AIR_GAP` | `src/selfhost/orb-collector.ts:161` |",
"| `ORB_ANONYMIZE` | `src/selfhost/orb-collector.ts:174` |",
Expand Down
90 changes: 85 additions & 5 deletions src/selfhost/ai.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,28 @@ export function resolveCodexCliTimeoutMs(env: Record<string, string | undefined>
return resolveCliTimeoutFrom(firstConfigured(env.CODEX_AI_TIMEOUT_MS), resolveCodexEffort(firstConfigured(env.CODEX_AI_EFFORT)));
}

// Fast-fail deadline for Codex's "Reading prompt from stdin..." hang (GITTENSORY-K/GITTENSORY-M): observed in prod
// as `codex exec` printing ONLY its own startup banner to stderr and then never producing a single byte of JSONL
// on stdout before the FULL timeoutMs (up to 600_000ms at max effort) elapses and the process is SIGKILLed. That
// full timeout is sized for a legitimately long-running review, so waiting it out to detect a completely dead
// subprocess stalls the codex → claude-code fallback chain for up to 10 minutes per attempt. This is a SEPARATE,
// much shorter deadline: if not one single byte has arrived on STDOUT by this point, the process is almost
// certainly hung at the stdin-read step, not merely thinking. Deliberately STDOUT-ONLY, not "either stream" —
// the startup banner itself is unconditional stderr output on every invocation, so treating it as "alive" would
// let it satisfy this deadline forever and never catch the exact hang it exists to detect; real JSONL progress
// from `codex --json` always lands on stdout, so stdout is the only reliable liveness signal. 30s default: long
// enough that a busy host (cold container start, contended CPU) doesn't false-positive on a merely-slow-to-start
// real call, short enough that the codex→claude-code fallback (or a caller retry) kicks in almost immediately
// instead of after a 10-minute stall. Independent of CODEX_AI_EFFORT/CODEX_AI_TIMEOUT_MS on purpose: a higher
// effort makes a COMPLETION take longer, it does not make the CLI slower to print its FIRST stdout byte, so this
// must not scale with effort the way the full timeout does. Bounds mirror resolveCliTimeoutFrom's floor but cap
// well under the shortest full timeout (120_000ms) so this can never itself become the effective timeout.
export function resolveCodexFirstOutputTimeoutMs(env: Record<string, string | undefined>): number {
const raw = Number(firstConfigured(env.CODEX_AI_FIRST_OUTPUT_TIMEOUT_MS));
if (Number.isFinite(raw) && raw > 0) return Math.min(120_000, Math.max(1_000, raw));
return 30_000;
}

/** OpenAI-compatible endpoint (Ollama's /v1, OpenAI, vLLM, LM Studio, …) — chat + embeddings. */
export function createOpenAiCompatibleAi(opts: {
baseUrl: string;
Expand Down Expand Up @@ -515,8 +537,19 @@ export function codexErrorFromStdout(stdout: string): string | null {
type SpawnFn = (
cmd: string,
args: string[],
opts: { env: Record<string, string | undefined>; input?: string; timeoutMs: number; cwd?: string },
) => Promise<{ stdout: string; code: number | null; stderr?: string; timedOut?: boolean }>;
opts: {
env: Record<string, string | undefined>;
input?: string;
timeoutMs: number;
cwd?: string;
// Optional, generic on SpawnFn (not codex-specific) so any CLI whose real progress lands on STDOUT (not
// stderr banners/logs) could opt in later — but ONLY codex wires it up today (see
// resolveCodexFirstOutputTimeoutMs): Claude Code has no comparable prod-observed dead-air hang, so leaving
// this undefined for that caller keeps its spawn path byte-identical to before this option existed. See the
// stdout-only rationale on the timer construction below — this deadline is cleared by stdout data ONLY.
firstOutputTimeoutMs?: number;
},
) => Promise<{ stdout: string; code: number | null; stderr?: string; timedOut?: boolean; stalledNoOutput?: boolean }>;

async function defaultSpawn(): Promise<SpawnFn> {
const cp = await import("node:child_process");
Expand All @@ -528,6 +561,7 @@ async function defaultSpawn(): Promise<SpawnFn> {
// Capture stderr too — the CLI's actual error (auth, rate limit, model-not-supported, OOM) lands here, and
// it's what makes a `claude_code_exit_1` / `codex_exit_1` diagnosable instead of an opaque exit code (#26).
let stderr = "";
let sawStdout = false;
/* v8 ignore start */ // a 120s subprocess timeout is not unit-testable without a 2-minute wait
const timer = setTimeout(() => {
child.kill("SIGKILL");
Expand All @@ -536,14 +570,44 @@ async function defaultSpawn(): Promise<SpawnFn> {
resolve({ stdout, code: null, stderr, timedOut: true });
}, o.timeoutMs);
/* v8 ignore stop */
child.stdout?.on("data", (d: Buffer) => (stdout += d.toString("utf8")));
child.stderr?.on("data", (d: Buffer) => (stderr += d.toString("utf8")));
// Fast-fail deadline (GITTENSORY-K/GITTENSORY-M): a SEPARATE, shorter timer that only fires if STDOUT has
// not produced a single byte by firstOutputTimeoutMs — cleared the instant any data arrives on stdout, same
// as the full timer is cleared on `close`/`error`. Deliberately STDOUT-ONLY, not "either stream": codex's
// own "Reading prompt from stdin..." startup banner is written to STDERR unconditionally, on every
// invocation, whether or not it goes on to actually process anything — clearing on stderr too would let
// that banner alone satisfy the deadline forever, which is exactly the real hang this exists to catch
// (confirmed as a defect during review: the first version of this fix cleared on either stream and would
// never have fired for the actual "banner then silence" failure mode). Real JSONL progress from codex
// (`--json`) always lands on stdout, so stdout is the only reliable "codex is genuinely alive" signal. If
// output DOES start flowing on stdout but then stalls later, only the full timeoutMs above still governs —
// this timer has already been cleared by the first stdout byte and never fires.
const firstOutputTimer =
o.firstOutputTimeoutMs != null
? /* v8 ignore start */ // real-timer path; tests inject a fake spawnImpl instead of racing setTimeout
setTimeout(() => {
child.kill("SIGKILL");
resolve({ stdout, code: null, stderr, timedOut: true, stalledNoOutput: true });
}, o.firstOutputTimeoutMs)
: /* v8 ignore stop */
undefined;
child.stdout?.on("data", (d: Buffer) => {
if (!sawStdout) {
sawStdout = true;
if (firstOutputTimer) clearTimeout(firstOutputTimer);
}
stdout += d.toString("utf8");
});
child.stderr?.on("data", (d: Buffer) => {
stderr += d.toString("utf8");
});
child.on("error", (e) => {
clearTimeout(timer);
if (firstOutputTimer) clearTimeout(firstOutputTimer);
reject(e);
});
child.on("close", (code) => {
clearTimeout(timer);
if (firstOutputTimer) clearTimeout(firstOutputTimer);
resolve({ stdout, code, stderr });
});
if (o.input != null) {
Expand Down Expand Up @@ -693,6 +757,10 @@ export function createCodexAi(
const codexModel = resolveModel(configuredCodexModel(parentEnv), model, "");
const effort = resolveCodexEffort(firstConfigured(parentEnv.CODEX_AI_EFFORT));
const timeoutMs = resolveCodexCliTimeoutMs(parentEnv);
// Clamp below timeoutMs so a misconfigured/low CODEX_AI_TIMEOUT_MS (its own floor is 30_000ms, the same as
// this deadline's default) can never make the fast-fail deadline equal or exceed the outer safety net —
// that would make the "outer" timeout unreachable and defeat the point of having two distinct signals.
const firstOutputTimeoutMs = Math.min(resolveCodexFirstOutputTimeoutMs(parentEnv), Math.max(1, timeoutMs - 1));
let attempted = false;
let stdoutForMetrics = "";
try {
Expand All @@ -706,14 +774,26 @@ export function createCodexAi(
if (codexModel) args.push("--model", codexModel);
args.push("-c", `model_reasoning_effort="${effort}"`);
attempted = true;
const { stdout, code, stderr, timedOut } = await spawn("codex", args, {
const { stdout, code, stderr, timedOut, stalledNoOutput } = await spawn("codex", args, {
env,
// `codex exec` reads stdin when no prompt argv is provided; keep PR prompts/diffs out of process listings.
input: prompt,
timeoutMs,
firstOutputTimeoutMs,
cwd: await isolatedCliCwd(),
});
stdoutForMetrics = stdout;
if (timedOut && stalledNoOutput) {
// Fast-fail path (GITTENSORY-K/GITTENSORY-M): killed at firstOutputTimeoutMs, well before the full
// timeoutMs, because STDOUT produced no bytes at all — the "Reading prompt from stdin..." hang where
// codex prints its own startup banner to STDERR and then never emits any JSONL. Stdout-only is
// deliberate: that banner would otherwise satisfy an "either stream" deadline on every single
// invocation, defeating the point. A DISTINCT error (never reusing `codex_timeout`) so this fast-fail
// is separately countable in Sentry/logs from a genuine full-timeout case where the process was at
// least emitting JSONL before it was killed — that distinction is what lets an operator tell "codex
// never started" apart from "codex hung mid-review".
throw new Error("codex_stalled_no_output: no stdout within firstOutputTimeoutMs — codex likely hung reading stdin");
}
if (timedOut) {
// Include whatever the JSONL stream captured before the kill — codex writes errors there, not to stderr.
const detail = codexErrorFromStdout(stdout) ?? (redactSecrets(stderr ?? "").slice(0, 200) || "no output");
Expand Down
Loading
Loading