diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 94f9a1c121..93bdfdc6f5 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -8203,6 +8203,11 @@ async function runSelfHostVisualVision(env: Env, system: string, user: string, i { role: "user", content: [{ type: "text", text: user }, ...images] }, ], max_tokens: 600, + // Bounds per-request KV cache on a concurrency-constrained GPU (#4327/#4335 concurrency tuning docs + // this exact figure) -- without a cap, vision's larger-than-text context can exhaust VRAM under + // concurrent load faster than the embed model does, degrading to latency collapse rather than a clean + // OOM. Ignored by every non-Ollama provider (embeddings, subscription CLIs, Anthropic). + providerOptions: { num_ctx: 4096 }, })) as { response?: string } | null; return result?.response?.trim() || null; } catch { diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts index 6b473ea47e..039027176e 100644 --- a/src/selfhost/ai.ts +++ b/src/selfhost/ai.ts @@ -25,6 +25,12 @@ interface AiRunOptions { text?: string[]; // embedding input — the core's embedTexts passes { text: string[] } max_tokens?: number; temperature?: number; + // Ollama-specific runtime options (e.g. `{ num_ctx: 4096 }` to bound per-request KV cache on a + // concurrency-constrained GPU, #4327/#4335) — forwarded verbatim as the OpenAI-compatible endpoint's + // `options` extension field. Only createOpenAiCompatibleAi's chat path reads this; every other provider + // (embeddings, the subscription CLIs, Anthropic) ignores it, so it is safe to set unconditionally on a + // call that ONLY ever targets an Ollama-backed binding (e.g. AI_VISION). + providerOptions?: Record; // Correlation context for a provider-failure log (#codex-timeout-fields): purely observational, never read by a // provider's own request logic. The caller (runWorkersOpinion) passes whatever of these it already has in scope // for THIS review — job id and attempt are per-attempt, repoFullName/pullNumber identify the PR being reviewed — @@ -281,6 +287,7 @@ export function createOpenAiCompatibleAi(opts: { messages: toMessages(options).map((message) => ({ role: message.role, content: toOpenAiMessageContent(message.content) })), max_tokens: options.max_tokens, temperature: options.temperature, + ...(options.providerOptions ? { options: options.providerOptions } : {}), }), signal: AbortSignal.timeout(120_000), }); diff --git a/test/unit/selfhost-ai.test.ts b/test/unit/selfhost-ai.test.ts index 0192faa04a..6cffa8b810 100644 --- a/test/unit/selfhost-ai.test.ts +++ b/test/unit/selfhost-ai.test.ts @@ -117,6 +117,28 @@ describe("createOpenAiCompatibleAi (#979)", () => { expect(first?.body.model).toBe("llama3.1"); }); + it("forwards providerOptions verbatim as the request's `options` field (#4327/#4335 num_ctx capping)", async () => { + let body: Record | undefined; + vi.stubGlobal("fetch", vi.fn(async (_u: string, init: { body: string }) => { + body = JSON.parse(init.body); + return new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), { status: 200 }); + })); + const ai = createOpenAiCompatibleAi({ baseUrl: "http://ollama:11434/v1" }); + await ai.run("qwen3-vl:8b-instruct", { messages: [{ role: "user", content: "x" }], providerOptions: { num_ctx: 4096 } }); + expect(body?.options).toEqual({ num_ctx: 4096 }); + }); + + it("omits `options` entirely from the request body when providerOptions is unset (byte-identical to before)", async () => { + let body: Record | undefined; + vi.stubGlobal("fetch", vi.fn(async (_u: string, init: { body: string }) => { + body = JSON.parse(init.body); + return new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), { status: 200 }); + })); + const ai = createOpenAiCompatibleAi({ baseUrl: "http://ollama:11434/v1" }); + await ai.run("llama3.1", { messages: [{ role: "user", content: "x" }] }); + expect("options" in (body ?? {})).toBe(false); + }); + it("throws on a non-OK response so the caller degrades", async () => { vi.stubGlobal("fetch", vi.fn(async () => new Response("err", { status: 500 }))); await expect(createOpenAiCompatibleAi({ baseUrl: "http://x/v1" }).run("m", { prompt: "p" })).rejects.toThrow(/ai_http_500/); diff --git a/test/unit/visual-vision-wiring.test.ts b/test/unit/visual-vision-wiring.test.ts index 75bf22f4b9..dac4053505 100644 --- a/test/unit/visual-vision-wiring.test.ts +++ b/test/unit/visual-vision-wiring.test.ts @@ -426,6 +426,7 @@ describe("runVisualVisionForAdvisory: self-host local vision provider (#4335)", const [, options] = runMock.mock.calls[0]!; expect(options.messages[0]).toMatchObject({ role: "system" }); expect(options.messages[1]).toMatchObject({ role: "user" }); + expect((options as unknown as { providerOptions?: { num_ctx?: number } }).providerOptions).toEqual({ num_ctx: 4096 }); expect(adv.findings).toEqual([ { code: "visual_regression_finding",