diff --git a/apps/web/src/components/preview/PreviewView.test.tsx b/apps/web/src/components/preview/PreviewView.test.tsx index d9671e2f2d98..c880280db676 100644 --- a/apps/web/src/components/preview/PreviewView.test.tsx +++ b/apps/web/src/components/preview/PreviewView.test.tsx @@ -414,7 +414,7 @@ describe("PreviewView navigation", () => { expect(mocks.addPreviewAnnotation).toHaveBeenCalledWith(TEST_THREAD_REF, annotation); }); - it("still sends when screenshot attachment conversion fails", async () => { + it("drops the screenshot claim when attachment conversion fails", async () => { const annotation = { id: "annotation-2", pageUrl: "https://example.com/dashboard", @@ -434,7 +434,9 @@ describe("PreviewView navigation", () => { }; const onSendAnnotation = vi.fn(); mocks.pickElement.mockResolvedValue({ annotation, submission: "send" }); - mocks.previewAnnotationScreenshotFile.mockRejectedValue(new Error("conversion failed")); + mocks.previewAnnotationScreenshotFile.mockImplementation(() => { + throw new Error("conversion failed"); + }); renderToStaticMarkup( { ); mocks.toggleAnnotation?.(); - await vi.waitFor(() => expect(onSendAnnotation).toHaveBeenCalledWith(annotation, null)); + // The prompt advertises the crop off `screenshot`, so a failed conversion + // has to clear it — otherwise the agent is told to look at an image that + // was never attached. + const withoutScreenshot = { ...annotation, screenshot: null }; + await vi.waitFor(() => expect(onSendAnnotation).toHaveBeenCalledWith(withoutScreenshot, null)); + expect(mocks.addPreviewAnnotation).toHaveBeenCalledWith(TEST_THREAD_REF, withoutScreenshot); expect(mocks.addImage).not.toHaveBeenCalled(); }); }); diff --git a/apps/web/src/components/preview/PreviewView.tsx b/apps/web/src/components/preview/PreviewView.tsx index 6979a1a4006d..a08085211990 100644 --- a/apps/web/src/components/preview/PreviewView.tsx +++ b/apps/web/src/components/preview/PreviewView.tsx @@ -557,15 +557,23 @@ export function PreviewView({ try { const result = await previewBridge.pickElement(runtimeTabId); if (!result) return; - const { annotation, submission } = result; - addPreviewAnnotation(threadRef, annotation); + const { annotation: capturedAnnotation, submission } = result; let screenshotFile: File | null = null; try { - screenshotFile = await previewAnnotationScreenshotFile(annotation); - } catch { + screenshotFile = previewAnnotationScreenshotFile(capturedAnnotation); + } catch (error) { // The structured annotation is still sendable when converting its // optional screenshot into a composer attachment fails. + console.error("Could not convert preview annotation screenshot.", error); } + // The prompt advertises the crop off `screenshot`, so an annotation + // whose attachment did not survive must not keep it — otherwise the + // agent is told to look at an image it never received. + const annotation = + capturedAnnotation.screenshot && !screenshotFile + ? { ...capturedAnnotation, screenshot: null } + : capturedAnnotation; + addPreviewAnnotation(threadRef, annotation); const image = screenshotFile && annotation.screenshot ? ({ diff --git a/apps/web/src/lib/imageCompression.ts b/apps/web/src/lib/imageCompression.ts index be45024f38c4..8a62ac0a84cd 100644 --- a/apps/web/src/lib/imageCompression.ts +++ b/apps/web/src/lib/imageCompression.ts @@ -85,7 +85,7 @@ function dataUrlByteLength(dataUrl: string): number { } /** Base64 payload of a data URL decoded back into a `File`. */ -function dataUrlToFile(dataUrl: string, name: string, mimeType: string): File { +export function dataUrlToFile(dataUrl: string, name: string, mimeType: string): File { const payload = dataUrl.slice(dataUrl.indexOf(",") + 1); const binary = atob(payload); const bytes = new Uint8Array(binary.length); diff --git a/apps/web/src/lib/previewAnnotation.test.ts b/apps/web/src/lib/previewAnnotation.test.ts index 05f4b8d62731..eb9700879fe6 100644 --- a/apps/web/src/lib/previewAnnotation.test.ts +++ b/apps/web/src/lib/previewAnnotation.test.ts @@ -5,6 +5,7 @@ import { appendPreviewAnnotationPrompt, buildPreviewAnnotationPrompt, extractTrailingPreviewAnnotation, + previewAnnotationScreenshotFile, } from "./previewAnnotation"; const annotation: PreviewAnnotationPayload = { @@ -85,3 +86,40 @@ describe("preview annotations", () => { expect(extractedFirst.promptText).toBe("Fix this"); }); }); + +describe("preview annotation screenshots", () => { + it("decodes the crop into a named file", () => { + const file = previewAnnotationScreenshotFile(annotation); + expect(file?.name).toBe("preview-annotation-annotation_1.png"); + expect(file?.type).toBe("image/png"); + expect(file?.size).toBe(1); + }); + + it("returns null when the annotation carries no crop", () => { + expect(previewAnnotationScreenshotFile({ ...annotation, screenshot: null })).toBe(null); + }); + + it("returns null for a data URL that is not base64", () => { + expect( + previewAnnotationScreenshotFile({ + ...annotation, + screenshot: { ...annotation.screenshot!, dataUrl: "data:image/png,notbase64" }, + }), + ).toBe(null); + }); + + // The desktop renderer's CSP allows `data:` for `img-src` but not for + // `connect-src`, so fetching the crop is blocked there and the screenshot + // silently never reaches the composer. Decode it in place instead. + it("never fetches the data URL", () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = (() => { + throw new Error("connect-src blocks data: URLs in the desktop renderer"); + }) as typeof globalThis.fetch; + try { + expect(previewAnnotationScreenshotFile(annotation)).not.toBe(null); + } finally { + globalThis.fetch = originalFetch; + } + }); +}); diff --git a/apps/web/src/lib/previewAnnotation.ts b/apps/web/src/lib/previewAnnotation.ts index f1723dd93aad..fc480b74ac7e 100644 --- a/apps/web/src/lib/previewAnnotation.ts +++ b/apps/web/src/lib/previewAnnotation.ts @@ -1,5 +1,6 @@ import type { PreviewAnnotationPayload } from "@t3tools/contracts"; import { buildElementContextBlock, normalizeElementContextSelection } from "./elementContext"; +import { dataUrlToFile } from "./imageCompression"; const TRAILING_PREVIEW_ANNOTATION_BLOCK_PATTERN = /\n*\n((?:(?!)[\s\S])*)\n<\/preview_annotation>\s*$/; @@ -99,13 +100,20 @@ export function extractTrailingPreviewAnnotation(prompt: string): ExtractedPrevi }; } -export async function previewAnnotationScreenshotFile( - annotation: PreviewAnnotationPayload, -): Promise { +const BASE64_DATA_URL_PATTERN = /^data:([^;,]*);base64,/; + +/** + * Decodes the captured crop in place rather than `fetch`ing its data URL. The + * desktop renderer is served over a custom scheme whose CSP allows `data:` for + * `img-src` but not for `connect-src`, so a fetch is blocked there and the + * screenshot never reaches the composer. Returns null when the crop is missing + * or is not a base64 data URL; callers must then drop the annotation's + * screenshot so the prompt does not advertise an attachment that never landed. + */ +export function previewAnnotationScreenshotFile(annotation: PreviewAnnotationPayload): File | null { if (!annotation.screenshot) return null; - const response = await fetch(annotation.screenshot.dataUrl); - const blob = await response.blob(); - return new File([blob], `preview-annotation-${annotation.id}.png`, { - type: blob.type || "image/png", - }); + const { dataUrl } = annotation.screenshot; + const match = BASE64_DATA_URL_PATTERN.exec(dataUrl); + if (!match) return null; + return dataUrlToFile(dataUrl, `preview-annotation-${annotation.id}.png`, match[1] || "image/png"); }