fix(web): attach preview annotation screenshots in the desktop app

apps/web/src/components/preview/PreviewView.test.tsx

@@ -414,7 +414,7 @@ describe("PreviewView navigation", () => {
     expect(mocks.addPreviewAnnotation).toHaveBeenCalledWith(TEST_THREAD_REF, annotation);
   });
 
-  it("still sends when screenshot attachment conversion fails", async () => {
+  it("drops the screenshot claim when attachment conversion fails", async () => {
     const annotation = {
       id: "annotation-2",
       pageUrl: "https://example.com/dashboard",
@@ -434,7 +434,9 @@ describe("PreviewView navigation", () => {
     };
     const onSendAnnotation = vi.fn();
     mocks.pickElement.mockResolvedValue({ annotation, submission: "send" });
-    mocks.previewAnnotationScreenshotFile.mockRejectedValue(new Error("conversion failed"));
+    mocks.previewAnnotationScreenshotFile.mockImplementation(() => {
+      throw new Error("conversion failed");
+    });
 
     renderToStaticMarkup(
       <PreviewView
@@ -446,7 +448,12 @@ describe("PreviewView navigation", () => {
     );
     mocks.toggleAnnotation?.();
 
-    await vi.waitFor(() => expect(onSendAnnotation).toHaveBeenCalledWith(annotation, null));
+    // The prompt advertises the crop off `screenshot`, so a failed conversion
+    // has to clear it — otherwise the agent is told to look at an image that
+    // was never attached.
+    const withoutScreenshot = { ...annotation, screenshot: null };
+    await vi.waitFor(() => expect(onSendAnnotation).toHaveBeenCalledWith(withoutScreenshot, null));
+    expect(mocks.addPreviewAnnotation).toHaveBeenCalledWith(TEST_THREAD_REF, withoutScreenshot);
     expect(mocks.addImage).not.toHaveBeenCalled();
   });
 });

apps/web/src/components/preview/PreviewView.tsx

@@ -557,15 +557,23 @@ export function PreviewView({
       try {
         const result = await previewBridge.pickElement(runtimeTabId);
         if (!result) return;
-        const { annotation, submission } = result;
-        addPreviewAnnotation(threadRef, annotation);
+        const { annotation: capturedAnnotation, submission } = result;
         let screenshotFile: File | null = null;
         try {
-          screenshotFile = await previewAnnotationScreenshotFile(annotation);
-        } catch {
+          screenshotFile = previewAnnotationScreenshotFile(capturedAnnotation);
+        } catch (error) {
           // The structured annotation is still sendable when converting its
           // optional screenshot into a composer attachment fails.
+          console.error("Could not convert preview annotation screenshot.", error);
         }
+        // The prompt advertises the crop off `screenshot`, so an annotation
+        // whose attachment did not survive must not keep it — otherwise the
+        // agent is told to look at an image it never received.
+        const annotation =
+          capturedAnnotation.screenshot && !screenshotFile
+            ? { ...capturedAnnotation, screenshot: null }
+            : capturedAnnotation;
+        addPreviewAnnotation(threadRef, annotation);
         const image =
           screenshotFile && annotation.screenshot
             ? ({

apps/web/src/lib/imageCompression.ts

@@ -85,7 +85,7 @@ function dataUrlByteLength(dataUrl: string): number {
 }
 
 /** Base64 payload of a data URL decoded back into a `File`. */
-function dataUrlToFile(dataUrl: string, name: string, mimeType: string): File {
+export function dataUrlToFile(dataUrl: string, name: string, mimeType: string): File {
   const payload = dataUrl.slice(dataUrl.indexOf(",") + 1);
   const binary = atob(payload);
   const bytes = new Uint8Array(binary.length);

apps/web/src/lib/previewAnnotation.test.ts

@@ -5,6 +5,7 @@ import {
   appendPreviewAnnotationPrompt,
   buildPreviewAnnotationPrompt,
   extractTrailingPreviewAnnotation,
+  previewAnnotationScreenshotFile,
 } from "./previewAnnotation";
 
 const annotation: PreviewAnnotationPayload = {
@@ -85,3 +86,40 @@ describe("preview annotations", () => {
     expect(extractedFirst.promptText).toBe("Fix this");
   });
 });
+
+describe("preview annotation screenshots", () => {
+  it("decodes the crop into a named file", () => {
+    const file = previewAnnotationScreenshotFile(annotation);
+    expect(file?.name).toBe("preview-annotation-annotation_1.png");
+    expect(file?.type).toBe("image/png");
+    expect(file?.size).toBe(1);
+  });
+
+  it("returns null when the annotation carries no crop", () => {
+    expect(previewAnnotationScreenshotFile({ ...annotation, screenshot: null })).toBe(null);
+  });
+
+  it("returns null for a data URL that is not base64", () => {
+    expect(
+      previewAnnotationScreenshotFile({
+        ...annotation,
+        screenshot: { ...annotation.screenshot!, dataUrl: "data:image/png,notbase64" },
+      }),
+    ).toBe(null);
+  });
+
+  // The desktop renderer's CSP allows `data:` for `img-src` but not for
+  // `connect-src`, so fetching the crop is blocked there and the screenshot
+  // silently never reaches the composer. Decode it in place instead.
+  it("never fetches the data URL", () => {
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = (() => {
+      throw new Error("connect-src blocks data: URLs in the desktop renderer");
+    }) as typeof globalThis.fetch;
+    try {
+      expect(previewAnnotationScreenshotFile(annotation)).not.toBe(null);
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+});

apps/web/src/lib/previewAnnotation.ts

@@ -1,5 +1,6 @@
 import type { PreviewAnnotationPayload } from "@t3tools/contracts";
 import { buildElementContextBlock, normalizeElementContextSelection } from "./elementContext";
+import { dataUrlToFile } from "./imageCompression";
 
 const TRAILING_PREVIEW_ANNOTATION_BLOCK_PATTERN =
   /\n*<preview_annotation>\n((?:(?!<preview_annotation>)[\s\S])*)\n<\/preview_annotation>\s*$/;
@@ -99,13 +100,20 @@ export function extractTrailingPreviewAnnotation(prompt: string): ExtractedPrevi
   };
 }
 
-export async function previewAnnotationScreenshotFile(
-  annotation: PreviewAnnotationPayload,
-): Promise<File | null> {
+const BASE64_DATA_URL_PATTERN = /^data:([^;,]*);base64,/;
+
+/**
+ * Decodes the captured crop in place rather than `fetch`ing its data URL. The
+ * desktop renderer is served over a custom scheme whose CSP allows `data:` for
+ * `img-src` but not for `connect-src`, so a fetch is blocked there and the
+ * screenshot never reaches the composer. Returns null when the crop is missing
+ * or is not a base64 data URL; callers must then drop the annotation's
+ * screenshot so the prompt does not advertise an attachment that never landed.
+ */
+export function previewAnnotationScreenshotFile(annotation: PreviewAnnotationPayload): File | null {
   if (!annotation.screenshot) return null;
-  const response = await fetch(annotation.screenshot.dataUrl);
-  const blob = await response.blob();
-  return new File([blob], `preview-annotation-${annotation.id}.png`, {
-    type: blob.type || "image/png",
-  });
+  const { dataUrl } = annotation.screenshot;
+  const match = BASE64_DATA_URL_PATTERN.exec(dataUrl);
+  if (!match) return null;
+  return dataUrlToFile(dataUrl, `preview-annotation-${annotation.id}.png`, match[1] || "image/png");
 }