diff --git a/.env.example b/.env.example index de474e33d1..0493b7b37e 100644 --- a/.env.example +++ b/.env.example @@ -600,6 +600,16 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # # embeddings use the same provider as everything else; setting # # AI_EMBED_MODEL above alone does nothing without this. # AI_EMBED_API_KEY= # bearer credential for AI_EMBED_BASE_URL, if it requires one. +# AI_VISION_MODEL=qwen3-vl:8b-instruct # vision-language model for the visual-vision advisory (#4111/#4335) +# # analyzing before/after PR screenshots. Only used when +# # AI_VISION_BASE_URL is also set. +# AI_VISION_BASE_URL= # route visual-vision to a SEPARATE openai-compatible endpoint that +# # can actually see images — e.g. a local Ollama running a VLM. The +# # subscription CLIs (claude-code/codex) cannot consume inline image +# # bytes, so before this was set, visual-vision required a maintainer +# # BYOK (anthropic/openai) key; this gives self-host a local option. +# # Unset = visual-vision falls back to BYOK-only. +# AI_VISION_API_KEY= # bearer credential for AI_VISION_BASE_URL, if it requires one. # --- Gittensory Orb (#1255; ALWAYS-ON fleet-calibration telemetry) --- # TELEMETRY NOTICE: running this self-hosted image contributes anonymized gate-calibration data to diff --git a/apps/gittensory-ui/src/lib/selfhost-env-reference.ts b/apps/gittensory-ui/src/lib/selfhost-env-reference.ts index 2f59b03eb6..953cc6600c 100644 --- a/apps/gittensory-ui/src/lib/selfhost-env-reference.ts +++ b/apps/gittensory-ui/src/lib/selfhost-env-reference.ts @@ -33,6 +33,18 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ name: "AI_PROVIDER", firstReference: "src/selfhost/ai-config.ts", }, + { + name: "AI_VISION_API_KEY", + firstReference: "src/server.ts", + }, + { + name: "AI_VISION_BASE_URL", + firstReference: "src/server.ts", + }, + { + name: "AI_VISION_MODEL", + firstReference: "src/server.ts", + }, { name: "ANTHROPIC_AI_BASE_URL", firstReference: "src/selfhost/ai.ts", @@ -441,6 +453,9 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `AI_EMBED_MODEL` | `src/selfhost/ai.ts` |", "| `AI_ON_MERGE` | `src/selfhost/ai.ts` |", "| `AI_PROVIDER` | `src/selfhost/ai-config.ts` |", + "| `AI_VISION_API_KEY` | `src/server.ts` |", + "| `AI_VISION_BASE_URL` | `src/server.ts` |", + "| `AI_VISION_MODEL` | `src/server.ts` |", "| `ANTHROPIC_AI_BASE_URL` | `src/selfhost/ai.ts` |", "| `ANTHROPIC_AI_MODEL` | `src/selfhost/ai.ts` |", "| `ANTHROPIC_API_KEY` | `src/selfhost/ai.ts` |", diff --git a/src/env.d.ts b/src/env.d.ts index 854a6586a2..beda0df325 100644 --- a/src/env.d.ts +++ b/src/env.d.ts @@ -11,6 +11,11 @@ declare global { * local/openai-compatible endpoint (ollama). Built at boot from AI_EMBED_BASE_URL/AI_EMBED_MODEL. Absent ⇒ * `createReviewAdapters` falls back to `env.AI` (byte-identical to before). */ AI_EMBED?: Ai; + /** Self-host (visual-vision, #4111/#4335): a DEDICATED vision-capable provider, separate from both the + * review chain and the embed provider — a local/openai-compatible endpoint (ollama + a vision-language + * model). Built at boot from AI_VISION_BASE_URL/AI_VISION_MODEL. Absent ⇒ visual-vision advisory falls + * back to requiring a maintainer BYOK key (the only option before this binding existed). */ + AI_VISION?: Ai; /** Self-host RAG vector adapter. Cloudflare no longer binds Vectorize for hosted reviews; the Node runtime * injects Qdrant/sqlite/pg adapters here when configured. Absent ⇒ no RAG, review proceeds with no retrieved * context. */ diff --git a/src/queue/processors.ts b/src/queue/processors.ts index c837e4dea2..94f9a1c121 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -8181,6 +8181,35 @@ class RetryablePublicSurfacePublishFailedError extends RetryableJobError { } } +/** A vision-capable self-host provider's `.run()` — mirrors ai-slop.ts's `AiRunner` (same loose shape for a + * `env.AI`-family binding called outside the SelfHostAi/RagInfra type boundaries), scoped locally since it + * is not exported. */ +type SelfHostVisionRunner = { run?: (model: string, options: Record) => Promise }; + +/** Self-host local vision (#4335): calls the dedicated `env.AI_VISION` binding (ollama + a vision-language + * model) the SAME way `env.AI_EMBED` is called for embeddings — a binding kept separate from the review + * chain so a vision request never competes with/degrades review-model routing. The `model` argument is a + * placeholder: `createOpenAiCompatibleAi`'s chat path prefers its own construction-time-configured model + * (AI_VISION_MODEL) over whatever string is passed here (see `resolveModel` in `selfhost/ai.ts`). Fail-safe + * on every path, exactly like `callAiProvider`'s BYOK sibling: no binding / no `.run` / a thrown error / an + * unparseable response all degrade to `null`, never a thrown error reaching the caller. */ +async function runSelfHostVisualVision(env: Env, system: string, user: string, images: readonly AiContentBlock[]): Promise { + const ai = env.AI_VISION as unknown as SelfHostVisionRunner | undefined; + if (!ai || typeof ai.run !== "function") return null; + try { + const result = (await ai.run("visual-vision", { + messages: [ + { role: "system", content: system }, + { role: "user", content: [{ type: "text", text: user }, ...images] }, + ], + max_tokens: 600, + })) as { response?: string } | null; + return result?.response?.trim() || null; + } catch { + return null; + } +} + /** * AI-vision analysis of a confirmed visual regression (#4111 wiring): the existing pixel-diff threshold can * tell "the pixels changed" but not "does it look broken" — a route the capture pipeline already flagged @@ -8221,17 +8250,22 @@ export async function runVisualVisionForAdvisory( model: args.settings.aiReviewModel ?? storedVisionKey.model, } : null; + // Self-host local vision (#4335): a dedicated ollama+VLM binding lets vision run WITHOUT a maintainer + // BYOK key -- see evaluateVisualVisionGate's header for why only an HTTP-capable provider (BYOK or this) + // can see the screenshots at all. + const selfHostVisionAvailable = Boolean(env.AI_VISION); const visionGate = evaluateVisualVisionGate({ routes: args.routes, reputationSignal: visionReputation.signal, providerKey: visionProviderKey, + selfHostVisionAvailable, }); if (!visionGate.run) return; - // evaluateVisualVisionGate only ever returns run:true when its own providerKey input (the SAME - // visionProviderKey resolved above) was non-null -- this is a defensive type-narrowing guard for the - // callAiProvider call below, not a reachable false case. - /* v8 ignore next -- see comment above */ - if (!visionProviderKey) return; + // evaluateVisualVisionGate only ever returns run:true when providerKey OR selfHostVisionAvailable (the + // SAME two values resolved above) was truthy -- this is a defensive type-narrowing guard, not a reachable + // false case: if neither is set here, the gate itself would already have returned run:false above. + /* v8 ignore next 2 -- see comment above */ + if (!visionProviderKey && !selfHostVisionAvailable) return; const images: AiContentBlock[] = []; for (const route of visionGate.routes) { // Show the model the viewport that actually crossed the pixel-diff threshold — a route can qualify via @@ -8250,15 +8284,18 @@ export async function runVisualVisionForAdvisory( if (afterBlock) images.push(afterBlock); } if (images.length === 0) return; - const visionResponse = await callAiProvider( - visionProviderKey, - VISUAL_VISION_SYSTEM_PROMPT, - buildVisualVisionUserPrompt(visionGate.routes), - 600, - images, - ); - if (!visionResponse.text) return; - const visionFindings = parseVisualVisionResponse(visionResponse.text); + // BYOK (a maintainer's own anthropic/openai key) takes priority when both are configured -- matches + // every other dual-path AI call site's convention (BYOK bills the maintainer's own account, so it's + // preferred over the shared/free local resource when the operator has explicitly set one up). + let visionText: string | null; + if (visionProviderKey) { + const visionResponse = await callAiProvider(visionProviderKey, VISUAL_VISION_SYSTEM_PROMPT, buildVisualVisionUserPrompt(visionGate.routes), 600, images); + visionText = visionResponse.text; + } else { + visionText = await runSelfHostVisualVision(env, VISUAL_VISION_SYSTEM_PROMPT, buildVisualVisionUserPrompt(visionGate.routes), images); + } + if (!visionText) return; + const visionFindings = parseVisualVisionResponse(visionText); args.advisory.findings.push(...buildVisualRegressionFindings(visionFindings)); } catch (error) { console.log( diff --git a/src/review/visual/visual-findings.ts b/src/review/visual/visual-findings.ts index ad491f16b7..3e76129078 100644 --- a/src/review/visual/visual-findings.ts +++ b/src/review/visual/visual-findings.ts @@ -62,20 +62,23 @@ export type VisualVisionGateResult = * exactly like the other AI neurons already skip for a low-reputation/burst submitter * (`shouldSkipAiForReputation`, `../reputation-wire.ts`); checked FIRST so a low-reputation submitter is * never even told which reason applies to their capture. - * 3. BYOK — vision rides the maintainer's OWN provider key (`providerKey` non-null): Workers AI is fully - * retired (no free vision-capable path exists) and the self-host subscription CLIs (claude-code/codex) - * cannot consume inline image bytes through their stdin-JSON invocation (see `../../selfhost/ai.ts`'s - * `contentText`), so only an HTTP BYOK provider (anthropic/openai) can actually see the screenshots. - * Pure + total: the caller resolves the reputation signal / provider key (D1 + decryption both live outside - * this file) and passes the results in. + * 3. a provider that can actually SEE the screenshots — either BYOK (`providerKey` non-null: the + * maintainer's own anthropic/openai key) or a self-host local vision provider (`selfHostVisionAvailable`, + * #4335: a dedicated ollama+VLM binding, `env.AI_VISION`). Workers AI is fully retired (no free + * vision-capable path exists) and the self-host subscription CLIs (claude-code/codex) cannot consume + * inline image bytes through their stdin-JSON invocation (see `../../selfhost/ai.ts`'s `contentText`), + * so only an HTTP-capable provider — BYOK or self-host's dedicated AI_VISION binding — can see them. + * Pure + total: the caller resolves the reputation signal / provider key / self-host vision availability (D1, + * decryption, and env all live outside this file) and passes the results in. */ export function evaluateVisualVisionGate(input: { routes: readonly CaptureRoute[]; reputationSignal: ReputationSignal; providerKey: AiReviewProviderKey | null; + selfHostVisionAvailable?: boolean; }): VisualVisionGateResult { if (input.reputationSignal === "low") return { run: false, reason: "low_reputation" }; - if (!input.providerKey) return { run: false, reason: "byok_not_configured" }; + if (!input.providerKey && !input.selfHostVisionAvailable) return { run: false, reason: "byok_not_configured" }; const routes = selectRoutesForVisualVision(input.routes); if (routes.length === 0) return { run: false, reason: "no_confirmed_regression" }; return { run: true, routes }; diff --git a/src/server.ts b/src/server.ts index ee4d3c7954..0d3ff97909 100644 --- a/src/server.ts +++ b/src/server.ts @@ -488,6 +488,26 @@ async function main(): Promise { model: process.env.AI_EMBED_MODEL ?? "bge-m3", }), ); + // Dedicated visual-vision provider (#4111/#4335): when AI_VISION_BASE_URL is set, the visual-vision + // advisory routes to a SEPARATE openai-compatible endpoint (e.g. ollama at http://ollama:11434/v1, a + // vision-language model) instead of requiring a maintainer BYOK key -- kept separate from AI_EMBED (a + // different model, a different capability) the same way AI_EMBED is kept separate from the review chain. + // Unset ⇒ absent ⇒ visual-vision falls back to BYOK-only (byte-identical to before this binding existed). + const visionAi = process.env.AI_VISION_BASE_URL + ? createOpenAiCompatibleAi({ + baseUrl: process.env.AI_VISION_BASE_URL, + apiKey: process.env.AI_VISION_API_KEY ?? process.env.OPENAI_API_KEY, + model: process.env.AI_VISION_MODEL, + }) + : undefined; + if (visionAi) + console.log( + JSON.stringify({ + event: "selfhost_vision_provider", + baseUrl: process.env.AI_VISION_BASE_URL, + model: process.env.AI_VISION_MODEL, + }), + ); // Dual-review plan (#dual-ai-combiner): resolve which provider(s) review + how to combine, attached to env // below so the review call site uses it. Undefined for a single provider's default review or no AI. const aiReviewPlan = resolveAiReviewerPlan(process.env); @@ -594,6 +614,7 @@ async function main(): Promise { WEBHOOKS: backend.queue.binding, // the brokered relay receiver enqueues via WEBHOOKS; both lanes share the in-process queue AI: ai, ...(embedAi ? { AI_EMBED: embedAi as unknown as Ai } : {}), + ...(visionAi ? { AI_VISION: visionAi as unknown as Ai } : {}), ...(aiReviewPlan ? { AI_REVIEW_PLAN: aiReviewPlan } : {}), SELFHOST_TRANSIENT_CACHE: webhookCache, // Qdrant takes priority; falls back to the backend's built-in vectorize (pgvector or sqlite-vec) diff --git a/test/unit/visual-findings.test.ts b/test/unit/visual-findings.test.ts index e80117dea0..a37b9d8eaa 100644 --- a/test/unit/visual-findings.test.ts +++ b/test/unit/visual-findings.test.ts @@ -91,6 +91,18 @@ describe("evaluateVisualVisionGate", () => { routes: [changedRoute("/a")], }); }); + + it("runs via a self-host local vision provider even with NO BYOK key configured (#4335)", () => { + expect( + evaluateVisualVisionGate({ routes: [changedRoute("/a")], reputationSignal: "neutral", providerKey: null, selfHostVisionAvailable: true }), + ).toEqual({ run: true, routes: [changedRoute("/a")] }); + }); + + it("still skips when self-host vision is explicitly unavailable and there is no BYOK key either", () => { + expect( + evaluateVisualVisionGate({ routes: [changedRoute("/a")], reputationSignal: "neutral", providerKey: null, selfHostVisionAvailable: false }), + ).toEqual({ run: false, reason: "byok_not_configured" }); + }); }); describe("buildVisualVisionUserPrompt", () => { diff --git a/test/unit/visual-vision-wiring.test.ts b/test/unit/visual-vision-wiring.test.ts index 1d05df6c75..75bf22f4b9 100644 --- a/test/unit/visual-vision-wiring.test.ts +++ b/test/unit/visual-vision-wiring.test.ts @@ -389,3 +389,140 @@ describe("runVisualVisionForAdvisory", () => { expect(fetchMock).not.toHaveBeenCalled(); }); }); + +/** Only the shot-fetch side of stubShotsAndProvider — the self-host vision path never calls `fetch` for the + * AI call itself (it calls `env.AI_VISION.run` directly), so no provider URL needs mocking here. */ +function stubShots() { + vi.stubGlobal("fetch", vi.fn(async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes("/gittensory/shot")) return new Response(new Uint8Array([1, 2, 3]), { status: 200, headers: { "content-type": "image/png" } }); + return new Response("not found", { status: 404 }); + })); +} + +function selfHostVisionRoutes() { + return [route({ path: "/app", diffUrl: "https://x/gittensory/shot?key=diff", beforeUrl: "https://x/gittensory/shot?key=before", afterUrl: "https://x/gittensory/shot?key=after" })]; +} + +describe("runVisualVisionForAdvisory: self-host local vision provider (#4335)", () => { + it("runs via env.AI_VISION when NO BYOK key is configured at all", async () => { + const runMock = vi.fn(async (_model: string, _options: { messages: Array<{ role: string; content: unknown }> }) => ({ + response: findingsResponse([{ path: "/app", body: "Nav bar overlaps the logo on the AFTER screenshot." }]), + })); + const env = byokEnv(); + (env as unknown as { AI_VISION: unknown }).AI_VISION = { run: runMock }; + stubShots(); + const adv = findingsHolder(); + await runVisualVisionForAdvisory(env, { + repoFullName, + pr, + author: "alice", + confirmedContributor: true, + settings: byokSettings({ aiReviewByok: false }), // no BYOK configured + advisory: adv, + routes: selfHostVisionRoutes(), + }); + expect(runMock).toHaveBeenCalledTimes(1); + const [, options] = runMock.mock.calls[0]!; + expect(options.messages[0]).toMatchObject({ role: "system" }); + expect(options.messages[1]).toMatchObject({ role: "user" }); + expect(adv.findings).toEqual([ + { + code: "visual_regression_finding", + severity: "warning", + title: "Possible visual regression: /app", + detail: "Nav bar overlaps the logo on the AFTER screenshot.", + action: "Advisory only — verify against the Visual preview screenshots before deciding.", + }, + ]); + }); + + it("prefers a configured BYOK key over env.AI_VISION when both are available", async () => { + const env = byokEnv(); + await upsertRepositoryAiKey(env, { repoFullName, provider: "anthropic", key: "sk-ant-vision-key", model: null }); + const runMock = vi.fn(async () => ({ response: findingsResponse([{ path: "/app", body: "should not be used" }]) })); + (env as unknown as { AI_VISION: unknown }).AI_VISION = { run: runMock }; + stubShotsAndProvider(findingsResponse([{ path: "/app", body: "BYOK finding wins." }])); + const adv = findingsHolder(); + await runVisualVisionForAdvisory(env, { + repoFullName, + pr, + author: "alice", + confirmedContributor: true, + settings: byokSettings(), + advisory: adv, + routes: selfHostVisionRoutes(), + }); + expect(runMock).not.toHaveBeenCalled(); + expect(adv.findings[0]).toMatchObject({ detail: "BYOK finding wins." }); + }); + + it("adds no finding (fail-safe) when env.AI_VISION.run throws", async () => { + const env = byokEnv(); + (env as unknown as { AI_VISION: unknown }).AI_VISION = { run: vi.fn(async () => { throw new Error("ollama connection refused"); }) }; + stubShots(); + const adv = findingsHolder(); + await runVisualVisionForAdvisory(env, { + repoFullName, + pr, + author: "alice", + confirmedContributor: true, + settings: byokSettings({ aiReviewByok: false }), + advisory: adv, + routes: selfHostVisionRoutes(), + }); + expect(adv.findings).toEqual([]); + }); + + it("adds no finding when env.AI_VISION.run resolves to an empty/whitespace-only response", async () => { + const env = byokEnv(); + (env as unknown as { AI_VISION: unknown }).AI_VISION = { run: vi.fn(async () => ({ response: " " })) }; + stubShots(); + const adv = findingsHolder(); + await runVisualVisionForAdvisory(env, { + repoFullName, + pr, + author: "alice", + confirmedContributor: true, + settings: byokSettings({ aiReviewByok: false }), + advisory: adv, + routes: selfHostVisionRoutes(), + }); + expect(adv.findings).toEqual([]); + }); + + it("adds no finding when env.AI_VISION is present but has no callable .run (a malformed binding)", async () => { + const env = byokEnv(); + (env as unknown as { AI_VISION: unknown }).AI_VISION = {}; + stubShots(); + const adv = findingsHolder(); + await runVisualVisionForAdvisory(env, { + repoFullName, + pr, + author: "alice", + confirmedContributor: true, + settings: byokSettings({ aiReviewByok: false }), + advisory: adv, + routes: selfHostVisionRoutes(), + }); + expect(adv.findings).toEqual([]); + }); + + it("still declines entirely when NEITHER BYOK nor env.AI_VISION is configured", async () => { + const env = byokEnv(); + const fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + const adv = findingsHolder(); + await runVisualVisionForAdvisory(env, { + repoFullName, + pr, + author: "alice", + confirmedContributor: true, + settings: byokSettings({ aiReviewByok: false }), + advisory: adv, + routes: selfHostVisionRoutes(), + }); + expect(adv.findings).toEqual([]); + expect(fetchMock).not.toHaveBeenCalled(); + }); +});