fix(clients): large pasted images no longer break later Claude image reads

apps/mobile/src/features/sharing/incoming-share-model.ts

@@ -12,6 +12,7 @@ import * as Schema from "effect/Schema";
 import type { ResolvedSharePayload, SharePayload } from "expo-sharing";
 
 import { DraftComposerAttachmentSchema } from "../../lib/composer-image-schema";
+import { fitComposerImage } from "../../lib/composerImageFit";
 import type { DraftComposerAttachment } from "../../lib/composerImages";
 import { estimateBase64ByteSize } from "../../lib/base64";
 
@@ -406,8 +407,18 @@ export async function buildIncomingShareDraft(input: {
     }
 
     try {
-      const base64 = await input.fileReader.readBase64(uri);
-      const sizeBytes = resolved?.contentSize ?? estimateBase64ByteSize(base64);
+      const sharedBase64 = await input.fileReader.readBase64(uri);
+      // Rendered from the bytes already read, since the provider's file may not be readable
+      // by the native image loader.
+      const fitted = await fitComposerImage({
+        uri: `data:${mimeType};base64,${sharedBase64}`,
+        mimeType,
+        base64: sharedBase64,
+      });
+      const base64 = fitted?.base64 ?? sharedBase64;
+      const sizeBytes = fitted
+        ? estimateBase64ByteSize(base64)
+        : (resolved?.contentSize ?? estimateBase64ByteSize(base64));
       if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) {
         warnings.push(
           `'${resolved?.originalName ?? fallbackName(uri, index, mimeType)}' exceeds the 10 MB attachment limit.`,

apps/mobile/src/lib/composerFiles.test.ts

@@ -185,10 +185,10 @@ describe("composer file attachments", () => {
     );
 
     it.each([
-      { size: { width: 4032, height: 3024 }, resizes: [{ width: 2048 }] },
-      { size: { width: 3024, height: 4032 }, resizes: [{ height: 2048 }] },
-      { size: { width: 2048, height: 1536 }, resizes: [] },
-    ])("bounds a $size.width x $size.height photo to a 2048 px longest edge", async (input) => {
+      { size: { width: 4032, height: 3024 }, resizes: [{ width: 2000, height: 1500 }] },
+      { size: { width: 3024, height: 4032 }, resizes: [{ width: 1500, height: 2000 }] },
+      { size: { width: 2000, height: 1500 }, resizes: [] },
+    ])("bounds a $size.width x $size.height photo to a 2000 px longest edge", async (input) => {
       native.size = input.size;
       mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [photo] });
 

apps/mobile/src/lib/composerImageFit.ts

@@ -0,0 +1,70 @@
+import {
+  downscalableImageMimeType,
+  fitImageAttachmentSize,
+} from "@t3tools/client-runtime/image-attachment-size";
+import {
+  IMAGE_DIMENSIONS_HEADER_BYTES,
+  readImageDimensions,
+} from "@t3tools/shared/imageDimensions";
+
+const PHOTO_JPEG_QUALITY = 0.85;
+
+/**
+ * Renders an image to a JPEG, or a PNG when `format` asks for one, with its long edge capped at
+ * `IMAGE_ATTACHMENT_MAX_EDGE`. Decode, downscale,
+ * and encode run natively; only the bounded result crosses the bridge. Camera photos are
+ * 12-48 MP HEIC files, so a full-size conversion is both slow to transfer and far more than a
+ * model can use.
+ */
+export async function renderPhoto(
+  uri: string,
+  format: "jpeg" | "png" = "jpeg",
+): Promise<{ base64: string; uri: string }> {
+  const { ImageManipulator, SaveFormat } = await import("expo-image-manipulator");
+  let image = await ImageManipulator.manipulate(uri).renderAsync();
+  try {
+    const fittedSize = fitImageAttachmentSize(image);
+    if (fittedSize !== null) {
+      const resized = await ImageManipulator.manipulate(image).resize(fittedSize).renderAsync();
+      image.release();
+      image = resized;
+    }
+    const saved = await image.saveAsync({
+      format: format === "png" ? SaveFormat.PNG : SaveFormat.JPEG,
+      compress: PHOTO_JPEG_QUALITY,
+      base64: true,
+    });
+    if (!saved.base64) {
+      throw new Error("The rendered photo has no bytes.");
+    }
+    return { base64: saved.base64, uri: saved.uri };
+  } finally {
+    image.release();
+  }
+}
+
+/**
+ * Scales a PNG or JPEG whose long edge is over `IMAGE_ATTACHMENT_MAX_EDGE` down to that edge
+ * in its own format. Returns null when the image fits, is another format (GIF, WebP), or the
+ * native renderer fails, so the caller keeps the original bytes. Images inside the limit are
+ * never re-encoded: only the header is read to decide.
+ */
+export async function fitComposerImage(image: {
+  readonly uri: string;
+  readonly mimeType: string;
+  readonly base64: string;
+}): Promise<{ base64: string; uri: string } | null> {
+  const mimeType = downscalableImageMimeType(image.mimeType);
+  if (mimeType === null) return null;
+  try {
+    const header = atob(image.base64.slice(0, Math.ceil(IMAGE_DIMENSIONS_HEADER_BYTES / 3) * 4));
+    const bytes = new Uint8Array(header.length);
+    for (let index = 0; index < header.length; index += 1) bytes[index] = header.charCodeAt(index);
+    const dimensions = readImageDimensions(bytes);
+    if (dimensions === null || fitImageAttachmentSize(dimensions) === null) return null;
+    return await renderPhoto(image.uri, mimeType === "image/png" ? "png" : "jpeg");
+  } catch (error) {
+    console.warn("[composer-images] could not scale an oversized image", error);
+    return null;
+  }
+}

apps/mobile/src/lib/composerImages.test.ts

@@ -12,6 +12,35 @@ const clipboard = vi.hoisted(() => ({
 
 vi.mock("expo-clipboard", () => clipboard);
 
+/** A native image renderer that reports the source size the test gives it. */
+const manipulator = vi.hoisted(() => {
+  const state = { sourceSize: { width: 0, height: 0 } };
+  const manipulate = vi.fn((source: string | { width: number; height: number }) => {
+    let size = typeof source === "string" ? state.sourceSize : source;
+    const context = {
+      resize: vi.fn((next: { width: number; height: number }) => {
+        size = next;
+        return context;
+      }),
+      renderAsync: vi.fn(async () => ({
+        ...size,
+        release: vi.fn(),
+        saveAsync: vi.fn(async (options: { format: string }) => ({
+          uri: `file:///cache/rendered.${options.format}`,
+          base64: `cmVuZGVyZWQ=`,
+        })),
+      })),
+    };
+    return context;
+  });
+  return { state, manipulate };
+});
+
+vi.mock("expo-image-manipulator", () => ({
+  ImageManipulator: { manipulate: manipulator.manipulate },
+  SaveFormat: { JPEG: "jpeg", PNG: "png" },
+}));
+
 vi.mock("expo-file-system", () => ({
   File: class {
     readonly uri: string;
@@ -128,6 +157,67 @@ describe("composer clipboard paste", () => {
   });
 });
 
+describe("oversized pasted images", () => {
+  const u32 = (n: number) => [(n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff];
+  const pngDataUrl = (width: number, height: number) =>
+    `data:image/png;base64,${Buffer.from([
+      0x89,
+      0x50,
+      0x4e,
+      0x47,
+      0x0d,
+      0x0a,
+      0x1a,
+      0x0a,
+      ...u32(13),
+      0x49,
+      0x48,
+      0x44,
+      0x52,
+      ...u32(width),
+      ...u32(height),
+      8,
+      6,
+      0,
+      0,
+      0,
+    ]).toString("base64")}`;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    clipboard.hasImageAsync.mockResolvedValue(true);
+  });
+
+  it("scales a clipboard screenshot over 2000px to 2000px and keeps it a PNG", async () => {
+    manipulator.state.sourceSize = { width: 3024, height: 1964 };
+    const pasted = pngDataUrl(3024, 1964);
+    clipboard.getImageAsync.mockResolvedValue({ data: pasted });
+
+    const result = await pasteComposerClipboard({ existingCount: 0 });
+
+    expect(manipulator.manipulate).toHaveBeenCalledWith(pasted);
+    const resize = manipulator.manipulate.mock.results.at(-1)?.value.resize;
+    expect(resize).toHaveBeenCalledWith({ width: 2000, height: 1299 });
+    expect(result.images).toEqual([
+      expect.objectContaining({
+        mimeType: "image/png",
+        dataUrl: "data:image/png;base64,cmVuZGVyZWQ=",
+        previewUri: "data:image/png;base64,cmVuZGVyZWQ=",
+      }),
+    ]);
+  });
+
+  it("keeps a 2000px clipboard image byte for byte without rendering it", async () => {
+    const pasted = pngDataUrl(2000, 1200);
+    clipboard.getImageAsync.mockResolvedValue({ data: pasted });
+
+    const result = await pasteComposerClipboard({ existingCount: 0 });
+
+    expect(manipulator.manipulate).not.toHaveBeenCalled();
+    expect(result.images).toEqual([expect.objectContaining({ dataUrl: pasted })]);
+  });
+});
+
 describe("native pasted image cleanup", () => {
   beforeEach(() => {
     files.clear();

apps/mobile/src/lib/composerImages.ts

@@ -20,6 +20,7 @@ import {
 } from "./composerAttachmentFiles";
 import { imageMimeType } from "@t3tools/shared/image";
 import { videoMimeType } from "@t3tools/shared/video";
+import { fitComposerImage, renderPhoto } from "./composerImageFit";
 import { beginForegroundHandoff } from "./foreground-handoff";
 import { uuidv4 } from "./uuid";
 import { writeFileAtomically } from "./atomic-file";
@@ -341,46 +342,6 @@ export async function pickComposerFiles(input: {
   return { files: attachments, error };
 }
 
-/**
- * Longest edge kept when a photo has to be re-encoded. Matches the web composer's
- * MAX_DIMENSION so every client hands providers the same resolution.
- */
-const PHOTO_MAX_EDGE = 2048;
-const PHOTO_JPEG_QUALITY = 0.85;
-
-/**
- * Renders a photo-library pick to a provider-readable JPEG. Decode, downscale, and encode run
- * natively; only the bounded result crosses the bridge. Camera photos are 12-48 MP HEIC files,
- * so a full-size conversion is both slow to transfer and far more than a model can use.
- */
-async function renderPhotoAsJpeg(uri: string): Promise<{ base64: string; uri: string }> {
-  const { ImageManipulator, SaveFormat } = await import("expo-image-manipulator");
-  let image = await ImageManipulator.manipulate(uri).renderAsync();
-  try {
-    const longestEdge = Math.max(image.width, image.height);
-    if (longestEdge > PHOTO_MAX_EDGE) {
-      const resized = await ImageManipulator.manipulate(image)
-        .resize(
-          image.width >= image.height ? { width: PHOTO_MAX_EDGE } : { height: PHOTO_MAX_EDGE },
-        )
-        .renderAsync();
-      image.release();
-      image = resized;
-    }
-    const saved = await image.saveAsync({
-      format: SaveFormat.JPEG,
-      compress: PHOTO_JPEG_QUALITY,
-      base64: true,
-    });
-    if (!saved.base64) {
-      throw new Error("The rendered photo has no bytes.");
-    }
-    return { base64: saved.base64, uri: saved.uri };
-  } finally {
-    image.release();
-  }
-}
-
 async function loadImagePicker() {
   try {
     return await import("expo-image-picker");
@@ -530,14 +491,20 @@ export async function pickComposerMedia(input: {
     try {
       if (originalMimeType !== null) {
         const { File } = await import("expo-file-system");
+        const base64 = await new File(asset.uri).base64();
+        const fitted = await fitComposerImage({
+          uri: asset.uri,
+          mimeType: originalMimeType,
+          base64,
+        });
         image = {
-          base64: await new File(asset.uri).base64(),
+          base64: fitted?.base64 ?? base64,
           mimeType: originalMimeType,
           name,
-          previewUri: asset.uri,
+          previewUri: fitted?.uri ?? asset.uri,
         };
       } else {
-        const rendered = await renderPhotoAsJpeg(asset.uri);
+        const rendered = await renderPhoto(asset.uri);
         image = {
           base64: rendered.base64,
           mimeType: "image/jpeg",
@@ -616,7 +583,14 @@ export async function pasteComposerClipboard(input: { readonly existingCount: nu
       };
     }
 
-    const base64 = image.data.split(",")[1] ?? "";
+    const pastedBase64 = image.data.split(",")[1] ?? "";
+    const fitted = await fitComposerImage({
+      uri: image.data,
+      mimeType: "image/png",
+      base64: pastedBase64,
+    });
+    const base64 = fitted?.base64 ?? pastedBase64;
+    const dataUrl = fitted ? `data:image/png;base64,${base64}` : image.data;
     const sizeBytes = estimateBase64ByteSize(base64);
     if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) {
       return {
@@ -634,8 +608,8 @@ export async function pasteComposerClipboard(input: { readonly existingCount: nu
           name: "pasted-image.png",
           mimeType: "image/png",
           sizeBytes,
-          dataUrl: image.data,
-          previewUri: image.data,
+          dataUrl,
+          previewUri: dataUrl,
         },
       ],
       text: null,
@@ -707,20 +681,22 @@ export async function convertPastedImagesToAttachments(input: {
         continue;
       }
       const file = new File(uri);
-      const base64 = await file.base64();
+      const mimeType = mimeTypeFromUri(uri);
+      const pastedBase64 = await file.base64();
+      const fitted = await fitComposerImage({ uri, mimeType, base64: pastedBase64 });
+      const base64 = fitted?.base64 ?? pastedBase64;
       const sizeBytes = estimateBase64ByteSize(base64);
       if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) {
         continue;
       }
-      const mimeType = mimeTypeFromUri(uri);
       results.push({
         id: uuidv4(),
         type: "image",
         name: `pasted-image.${mimeType.split("/")[1] ?? "png"}`,
         mimeType,
         sizeBytes,
         dataUrl: `data:${mimeType};base64,${base64}`,
-        previewUri: ownedTemporaryFile ? `data:${mimeType};base64,${base64}` : uri,
+        previewUri: ownedTemporaryFile ? `data:${mimeType};base64,${base64}` : (fitted?.uri ?? uri),
       });
     } catch (error) {
       console.warn("Failed to read pasted image", uri, error);