fix(session): bound transient transport failures

Author: literally-dan

packages/core/src/v1/config/provider.ts

@@ -108,9 +108,14 @@ export const Info = Schema.Struct({
           description:
             "Timeout in milliseconds to wait for response headers. Provider integrations may set defaults. Set to false to disable timeout.",
         }),
-        chunkTimeout: Schema.optional(PositiveInt).annotate({
+        chunkTimeout: Schema.optional(
+          Schema.Union([PositiveInt, Schema.Literal(false)]).annotate({
+            description:
+              "Timeout in milliseconds between streamed SSE chunks for this provider. If no chunk arrives within this window, the request is aborted. Default is 120000 (2 minutes). Set to false to disable.",
+          }),
+        ).annotate({
           description:
-            "Timeout in milliseconds between streamed SSE chunks for this provider. If no chunk arrives within this window, the request is aborted.",
+            "Timeout in milliseconds between streamed SSE chunks for this provider. If no chunk arrives within this window, the request is aborted. Default is 120000 (2 minutes). Set to false to disable.",
         }),
       }),
       [Schema.Record(Schema.String, Schema.Any)],

packages/opencode/src/provider/provider.ts

@@ -34,6 +34,16 @@ import { ProviderError } from "./error"
 
 const OPENAI_HEADER_TIMEOUT_DEFAULT = 300_000
 
+// Default between-chunk timeout for streamed SSE responses. Without this,
+// a TCP socket that goes silent (wifi hiccup, NAT eviction, server crash
+// with no FIN/RST) parks the AI SDK's fullStream forever — Linux TCP
+// keepalive defaults to ~2 hours of idle before probing. 120s is short
+// enough to surface a freeze quickly, long enough that even slow
+// reasoning models emit *something* (token, ping, thinking chunk) in
+// between. Per-provider `chunkTimeout` overrides this; set to `false` to
+// disable entirely for providers with legitimately silent long pauses.
+const DEFAULT_CHUNK_TIMEOUT_MS = 120_000
+
 function wrapSSE(res: Response, ms: number, ctl: AbortController) {
   if (typeof ms !== "number" || ms <= 0) return res
   if (!res.body) return res
@@ -1730,15 +1740,23 @@ const layer = Layer.effect(
         if (existing) return existing
 
         const customFetch = options["fetch"]
-        const chunkTimeout = options["chunkTimeout"]
+        const configuredChunkTimeout = options["chunkTimeout"]
         const headerTimeout = options["headerTimeout"]
         delete options["chunkTimeout"]
         delete options["headerTimeout"]
+        // Default to DEFAULT_CHUNK_TIMEOUT_MS unless the provider config
+        // explicitly sets a value (number) or disables it (false).
+        const chunkTimeout =
+          configuredChunkTimeout === false
+            ? 0
+            : typeof configuredChunkTimeout === "number"
+              ? configuredChunkTimeout
+              : DEFAULT_CHUNK_TIMEOUT_MS
 
         options["fetch"] = async (input: any, init?: BunFetchRequestInit) => {
           const fetchFn = customFetch ?? fetch
           const opts = init ?? {}
-          const chunkAbortCtl = typeof chunkTimeout === "number" && chunkTimeout > 0 ? new AbortController() : undefined
+          const chunkAbortCtl = chunkTimeout > 0 ? new AbortController() : undefined
           const headerTimeoutMs = headerTimeout === false ? undefined : headerTimeout
           const headerTimeoutCtl = typeof headerTimeoutMs === "number" ? timeoutController(headerTimeoutMs) : undefined
           const signals: AbortSignal[] = []

packages/opencode/src/session/message-v2.ts

@@ -600,17 +600,78 @@ export function latest(msgs: WithParts[]) {
   return { user, assistant, finished, tasks }
 }
 
+// Transport failures that can recover without changing the request. Keep
+// permanent endpoint failures such as ENOTFOUND and ECONNREFUSED out of this
+// list so a bad provider configuration surfaces immediately.
+const TRANSIENT_SYS_CODES = new Set([
+  "ECONNRESET",
+  "ETIMEDOUT",
+  "EAI_AGAIN",
+  "EHOSTUNREACH",
+  "ENETUNREACH",
+  "EPIPE",
+  "UND_ERR_CONNECT_TIMEOUT",
+  "UND_ERR_HEADERS_TIMEOUT",
+  "UND_ERR_BODY_TIMEOUT",
+  "UND_ERR_SOCKET",
+])
+const PERMANENT_SYS_CODES = new Set(["ENOTFOUND", "ECONNREFUSED"])
+const CAUSE_MAX_DEPTH = 8
+
+export function isPermanentTransportCode(code: unknown) {
+  return typeof code === "string" && PERMANENT_SYS_CODES.has(code)
+}
+
+// Exact messages emitted for a prematurely closed transport. Generic text
+// such as "network error" or "fetch failed" cannot distinguish a transient
+// disconnect from a permanent DNS, TLS, authentication, or endpoint failure.
+const TRANSIENT_MESSAGES = new Set(["socket hang up", "sse read timed out", "other side closed", "terminated"])
+
+function classifyTransportError(error: unknown) {
+  const seen = new Set<Error>()
+  let current = error
+  let transient: SystemError | undefined
+  for (let depth = 0; depth < CAUSE_MAX_DEPTH; depth++) {
+    if (!(current instanceof Error) || seen.has(current)) break
+    seen.add(current)
+    const code = "code" in current && typeof current.code === "string" ? current.code : undefined
+    if (isPermanentTransportCode(code)) return { type: "permanent" as const, error: current as SystemError }
+    if (!transient && code && TRANSIENT_SYS_CODES.has(code)) transient = current as SystemError
+    current = current.cause
+  }
+  if (transient) return { type: "transient" as const, error: transient }
+  return undefined
+}
+
 export function fromError(
   e: unknown,
   ctx: { providerID: ProviderV2.ID; aborted?: boolean },
 ): NonNullable<Assistant["error"]> {
+  const transport = classifyTransportError(e)
+  const transient = transport?.type === "transient" ? transport.error : undefined
+  const permanent = transport?.type === "permanent" ? transport.error : undefined
+  const sysCode = transient?.code
   switch (true) {
     case e instanceof DOMException && e.name === "AbortError":
-      return new AbortedError(
-        { message: e.message },
-        {
-          cause: e,
-        },
+      // A *bare* AbortError only ever originates from `controller.abort()`
+      // with no reason — i.e. a user/parent cancel (see provider.ts fetch
+      // wrapper + processor onInterrupt). Every transport timeout we control
+      // aborts with a *specific* reason and therefore surfaces as its own
+      // identifiable error instead: AbortSignal.timeout() -> "TimeoutError"
+      // DOMException (next case), the header-timeout -> HeaderTimeoutError,
+      // and the SSE chunk-timeout -> ResponseStreamError ("SSE read timed
+      // out"). Those are all classified retryable on their own below, so we
+      // do NOT need to second-guess a bare AbortError here. Classifying by
+      // error identity (rather than the externally-set `ctx.aborted` flag,
+      // which races the abort propagating through the failure channel) keeps
+      // a genuine cancel from being retried.
+      return new AbortedError({ message: e.message }, { cause: e }).toObject()
+    case e instanceof DOMException && e.name === "TimeoutError":
+      // Modern fetch surfaces AbortSignal.timeout() as a DOMException with
+      // name "TimeoutError" rather than "AbortError". Always retryable.
+      return new APIError(
+        { message: e.message || "Request timed out", isRetryable: true, metadata: { code: "TimeoutError" } },
+        { cause: e },
       ).toObject()
     case OutputLengthError.isInstance(e):
       return e
@@ -622,15 +683,15 @@ export function fromError(
         },
         { cause: e },
       ).toObject()
-    case (e as SystemError)?.code === "ECONNRESET":
+    case typeof sysCode === "string" && TRANSIENT_SYS_CODES.has(sysCode):
       return new APIError(
         {
-          message: "Connection reset by server",
+          message: sysCode === "ECONNRESET" ? "Connection reset by server" : `Network error (${sysCode})`,
           isRetryable: true,
           metadata: {
-            code: (e as SystemError).code ?? "",
-            syscall: (e as SystemError).syscall ?? "",
-            message: (e as SystemError).message ?? "",
+            code: sysCode ?? "",
+            syscall: transient?.syscall ?? "",
+            message: transient?.message ?? "",
           },
         },
         { cause: e },
@@ -692,13 +753,25 @@ export function fromError(
         {
           message: parsed.message,
           statusCode: parsed.statusCode,
-          isRetryable: parsed.isRetryable,
+          isRetryable: permanent ? false : parsed.isRetryable,
           responseHeaders: parsed.responseHeaders,
           responseBody: parsed.responseBody,
-          metadata: parsed.metadata,
+          metadata: permanent
+            ? {
+                ...parsed.metadata,
+                code: permanent.code ?? "",
+                syscall: permanent.syscall ?? "",
+                message: permanent.message ?? "",
+              }
+            : parsed.metadata,
         },
         { cause: e },
       ).toObject()
+    case e instanceof Error && TRANSIENT_MESSAGES.has(e.message.toLowerCase()):
+      return new APIError(
+        { message: e.message, isRetryable: true, metadata: { code: "NETWORK_ERROR", message: e.message } },
+        { cause: e },
+      ).toObject()
     case e instanceof Error:
       return new NamedError.Unknown({ message: errorMessage(e) }, { cause: e }).toObject()
     default:

packages/opencode/src/session/retry.ts

@@ -25,6 +25,7 @@ export type Retryable = {
 
 export const RETRY_INITIAL_DELAY = 2000
 export const RETRY_BACKOFF_FACTOR = 2
+export const RETRY_MAX_ATTEMPTS = 5
 export const RETRY_MAX_DELAY_NO_HEADERS = 30_000 // 30 seconds
 export const RETRY_MAX_DELAY = 2_147_483_647 // max 32-bit signed integer for setTimeout
 
@@ -69,6 +70,7 @@ export function retryable(error: Err, provider: string) {
   // context overflow errors should not be retried
   if (SessionV1.ContextOverflowError.isInstance(error)) return undefined
   if (SessionV1.APIError.isInstance(error)) {
+    if (MessageV2.isPermanentTransportCode(error.data.metadata?.code)) return undefined
     const status = error.data.statusCode
     // 5xx errors are transient server failures and should always be retried,
     // even when the provider SDK doesn't explicitly mark them as retryable.
@@ -180,6 +182,7 @@ export function policy(opts: {
 }) {
   return Schedule.fromStepWithMetadata(
     Effect.succeed((meta: Schedule.InputMetadata<unknown>) => {
+      if (meta.attempt > RETRY_MAX_ATTEMPTS) return Cause.done(meta.attempt)
       const error = opts.parse(meta.input)
       const retry = retryable(error, opts.provider)
       if (!retry) return Cause.done(meta.attempt)

packages/opencode/test/session/message-v2.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test"
 import { SessionV1 } from "@opencode-ai/core/v1/session"
 import { APICallError } from "ai"
 import { MessageV2 } from "../../src/session/message-v2"
+import { SessionRetry } from "../../src/session/retry"
 import { ProviderTransform } from "@/provider/transform"
 import type { Provider } from "@/provider/provider"
 
@@ -1552,6 +1553,130 @@ describe("session.message-v2.fromError", () => {
 
     expect(result.name).toBe("MessageAbortedError")
   })
+
+  test("classifies user-cancelled AbortError as AbortedError when ctx.aborted is true", () => {
+    const err = new DOMException("Aborted", "AbortError")
+    const result = MessageV2.fromError(err, { providerID, aborted: true })
+    expect(result.name).toBe("MessageAbortedError")
+  })
+
+  test("classifies a bare AbortError as a cancel regardless of ctx.aborted", () => {
+    // A bare AbortError only comes from controller.abort() with no reason —
+    // i.e. a user/parent cancel. Transport timeouts surface as their own
+    // specific errors (TimeoutError/HeaderTimeoutError/ResponseStreamError),
+    // so a bare AbortError must NOT be reclassified as retryable — otherwise
+    // a cancel races the failure channel and gets retried.
+    const err = new DOMException("The operation was aborted", "AbortError")
+    const result = MessageV2.fromError(err, { providerID })
+    expect(result.name).toBe("MessageAbortedError")
+  })
+
+  test("classifies TimeoutError DOMException as retryable APIError", () => {
+    // AbortSignal.timeout() in modern fetch surfaces as TimeoutError.
+    const err = new DOMException("The operation timed out", "TimeoutError")
+    const result = MessageV2.fromError(err, { providerID })
+    expect(SessionV1.APIError.isInstance(result)).toBe(true)
+    expect((result as SessionV1.APIError).data.isRetryable).toBe(true)
+  })
+
+  test.each([
+    ["ETIMEDOUT", "Network error (ETIMEDOUT)"],
+    ["EAI_AGAIN", "Network error (EAI_AGAIN)"],
+    ["EHOSTUNREACH", "Network error (EHOSTUNREACH)"],
+    ["ENETUNREACH", "Network error (ENETUNREACH)"],
+    ["EPIPE", "Network error (EPIPE)"],
+    ["UND_ERR_CONNECT_TIMEOUT", "Network error (UND_ERR_CONNECT_TIMEOUT)"],
+    ["UND_ERR_HEADERS_TIMEOUT", "Network error (UND_ERR_HEADERS_TIMEOUT)"],
+    ["UND_ERR_BODY_TIMEOUT", "Network error (UND_ERR_BODY_TIMEOUT)"],
+    ["UND_ERR_SOCKET", "Network error (UND_ERR_SOCKET)"],
+  ])("classifies %s SystemError as retryable APIError", (code, expectedMessage) => {
+    const err = Object.assign(new Error(`${code} from test`), { code })
+    const result = MessageV2.fromError(err, { providerID })
+    expect(SessionV1.APIError.isInstance(result)).toBe(true)
+    expect((result as SessionV1.APIError).data.isRetryable).toBe(true)
+    expect((result as SessionV1.APIError).data.message).toBe(expectedMessage)
+  })
+
+  test("classifies a fetch failure with a transient cause as retryable", () => {
+    const cause = Object.assign(new Error("read ECONNRESET"), { code: "ECONNRESET" })
+    const wrapper = Object.assign(new TypeError("fetch failed", { cause }), { code: "ERR_FETCH_FAILED" })
+    const result = MessageV2.fromError(wrapper, { providerID })
+    expect(SessionV1.APIError.isInstance(result)).toBe(true)
+    expect((result as SessionV1.APIError).data.metadata?.code).toBe("ECONNRESET")
+  })
+
+  test.each([
+    ["ENOTFOUND", "getaddrinfo"],
+    ["ECONNREFUSED", "connect"],
+  ])("overrides retryable APICallError caused by %s", (code, syscall) => {
+    const cause = Object.assign(new Error(`${syscall} ${code} api.invalid`), { code, syscall })
+    const wrapper = Object.assign(new TypeError("fetch failed", { cause }), { code: "ERR_FETCH_FAILED" })
+    const error = new APICallError({
+      message: "Cannot connect to API",
+      url: "https://api.invalid/v1/messages",
+      requestBodyValues: { model: "test" },
+      cause: wrapper,
+      isRetryable: true,
+    })
+
+    const result = MessageV2.fromError(error, { providerID })
+    expect(SessionV1.APIError.isInstance(result)).toBe(true)
+    if (!SessionV1.APIError.isInstance(result)) throw new Error("expected APIError")
+    expect(result.data.isRetryable).toBe(false)
+    expect(result.data.metadata).toMatchObject({ code, syscall, url: "https://api.invalid/v1/messages" })
+    expect(SessionRetry.retryable(result, "test")).toBeUndefined()
+  })
+
+  test("gives a nested permanent code precedence over a transient wrapper", () => {
+    const cause = Object.assign(new Error("getaddrinfo ENOTFOUND api.invalid"), { code: "ENOTFOUND" })
+    const wrapper = Object.assign(new Error("socket reset", { cause }), { code: "ECONNRESET" })
+    const result = MessageV2.fromError(wrapper, { providerID })
+    expect(result.name).toBe("UnknownError")
+  })
+
+  test("stops traversing cyclic error causes", () => {
+    const first = Object.assign(new Error("first wrapper"), { code: "ERR_FIRST" })
+    const second = Object.assign(new Error("second wrapper", { cause: first }), { code: "ERR_SECOND" })
+    first.cause = second
+    const result = MessageV2.fromError(first, { providerID })
+    expect(result.name).toBe("UnknownError")
+  })
+
+  test("bounds deeply nested error causes", () => {
+    const cause = Object.assign(new Error("read ECONNRESET"), { code: "ECONNRESET" })
+    const wrapper = Array.from({ length: 32 }, (_, index) => index).reduce<Error>(
+      (current, index) => new Error(`wrapper ${index}`, { cause: current }),
+      cause,
+    )
+    const result = MessageV2.fromError(wrapper, { providerID })
+    expect(result.name).toBe("UnknownError")
+  })
+
+  test.each(["socket hang up", "SSE read timed out", "other side closed", "terminated"])(
+    "classifies '%s' bare Error message as retryable APIError",
+    (message) => {
+      const result = MessageV2.fromError(new Error(message), { providerID })
+      expect(SessionV1.APIError.isInstance(result)).toBe(true)
+      expect((result as SessionV1.APIError).data.isRetryable).toBe(true)
+    },
+  )
+
+  test.each([
+    "fetch failed",
+    "Failed to fetch account configuration",
+    "network error: certificate has expired",
+    "Network request failed because access is forbidden",
+    "connect error: invalid provider base URL",
+    "request terminated because the API key is invalid",
+  ])("does not treat '%s' as a transient transport error", (message) => {
+    const result = MessageV2.fromError(new Error(message), { providerID })
+    expect(result.name).toBe("UnknownError")
+  })
+
+  test("leaves unrelated Error messages classified as Unknown", () => {
+    const result = MessageV2.fromError(new Error("Some unrelated bug"), { providerID })
+    expect(result.name).toBe("UnknownError")
+  })
 })
 
 describe("session.message-v2.latest", () => {