From d87038608c7ed1fc1af9b78bde79dc027543260f Mon Sep 17 00:00:00 2001 From: Oscar Swanros Date: Thu, 15 Oct 2026 20:09:03 -0600 Subject: [PATCH] fix(clients): images over Claude's 5 MB limit no longer end a running Claude turn Claude Code sends images queued into a running turn as they are, and Anthropic rejects an image whose base64 is over 5 MB. #19364 scaled attachments to 2000px, but an image within 2000px still passed through byte for byte up to the 10 MB wire cap and ended the turn with image_error. Clients now also fit image attachments to IMAGE_ATTACHMENT_MAX_BYTES (5 MB of base64) when they are attached. Fixes #20106 Co-Authored-By: Claude Opus 5.5 --- .../features/sharing/incoming-share-model.ts | 10 ++- apps/mobile/src/lib/composerImageFit.ts | 46 +++++++--- apps/mobile/src/lib/composerImages.test.ts | 83 +++++++++++++++++-- apps/mobile/src/lib/composerImages.ts | 23 +++-- apps/web/src/components/chat/ChatComposer.tsx | 9 +- .../desktop/SnapShotCoordinator.tsx | 9 +- apps/web/src/lib/imageCompression.ts | 6 +- .../client-runtime/src/imageAttachmentSize.ts | 8 ++ 8 files changed, 147 insertions(+), 47 deletions(-) diff --git a/apps/mobile/src/features/sharing/incoming-share-model.ts b/apps/mobile/src/features/sharing/incoming-share-model.ts index 6b125f006a3e..765e4a0f2317 100644 --- a/apps/mobile/src/features/sharing/incoming-share-model.ts +++ b/apps/mobile/src/features/sharing/incoming-share-model.ts @@ -12,7 +12,7 @@ import * as Schema from "effect/Schema"; import type { ResolvedSharePayload, SharePayload } from "expo-sharing"; import { DraftComposerAttachmentSchema } from "../../lib/composer-image-schema"; -import { fitComposerImage } from "../../lib/composerImageFit"; +import { fitComposerImage, jpegAttachmentName } from "../../lib/composerImageFit"; import type { DraftComposerAttachment } from "../../lib/composerImages"; import { estimateBase64ByteSize } from "../../lib/base64"; @@ -416,6 +416,7 @@ export async function buildIncomingShareDraft(input: { base64: sharedBase64, }); const base64 = fitted?.base64 ?? sharedBase64; + const attachmentMimeType = fitted?.mimeType ?? mimeType; const sizeBytes = fitted ? estimateBase64ByteSize(base64) : (resolved?.contentSize ?? estimateBase64ByteSize(base64)); @@ -425,12 +426,13 @@ export async function buildIncomingShareDraft(input: { ); continue; } - const dataUrl = `data:${mimeType};base64,${base64}`; + const dataUrl = `data:${attachmentMimeType};base64,${base64}`; + const name = resolved?.originalName ?? fallbackName(uri, index, mimeType); attachments.push({ id: `${input.id}:image:${index}`, type: "image", - name: resolved?.originalName ?? fallbackName(uri, index, mimeType), - mimeType, + name: attachmentMimeType === mimeType ? name : jpegAttachmentName(name), + mimeType: attachmentMimeType, sizeBytes, dataUrl, // The share provider's file is temporary. A data-backed preview keeps diff --git a/apps/mobile/src/lib/composerImageFit.ts b/apps/mobile/src/lib/composerImageFit.ts index ccc2f5ebaf1c..0ef291088402 100644 --- a/apps/mobile/src/lib/composerImageFit.ts +++ b/apps/mobile/src/lib/composerImageFit.ts @@ -1,12 +1,15 @@ import { downscalableImageMimeType, fitImageAttachmentSize, + IMAGE_ATTACHMENT_MAX_BYTES, } from "@t3tools/client-runtime/image-attachment-size"; import { IMAGE_DIMENSIONS_HEADER_BYTES, readImageDimensions, } from "@t3tools/shared/imageDimensions"; +import { estimateBase64ByteSize } from "./base64"; + const PHOTO_JPEG_QUALITY = 0.85; /** @@ -43,26 +46,45 @@ export async function renderPhoto( } } +/** Keeps an attachment's name truthful once it has been rendered to a JPEG. */ +export function jpegAttachmentName(name: string): string { + return /\.jpe?g$/i.test(name) ? name : `${name.replace(/\.[^.]+$/, "")}.jpg`; +} + +function isOverMaxEdge(base64: string): boolean { + const header = atob(base64.slice(0, Math.ceil(IMAGE_DIMENSIONS_HEADER_BYTES / 3) * 4)); + const bytes = new Uint8Array(header.length); + for (let index = 0; index < header.length; index += 1) bytes[index] = header.charCodeAt(index); + const dimensions = readImageDimensions(bytes); + return dimensions !== null && fitImageAttachmentSize(dimensions) !== null; +} + /** - * Scales a PNG or JPEG whose long edge is over `IMAGE_ATTACHMENT_MAX_EDGE` down to that edge - * in its own format. Returns null when the image fits, is another format (GIF, WebP), or the - * native renderer fails, so the caller keeps the original bytes. Images inside the limit are - * never re-encoded: only the header is read to decide. + * Fits an attachment to `IMAGE_ATTACHMENT_MAX_EDGE` and `IMAGE_ATTACHMENT_MAX_BYTES`. A PNG or + * JPEG over the edge is scaled to it in its own format. An image still over the byte limit, + * including a GIF or WebP, is rendered to a JPEG instead, so the result can change type. Returns + * null when the image already fits or the native renderer fails, so the caller keeps the + * original bytes. Images inside both limits are never re-encoded: only the header is read. */ export async function fitComposerImage(image: { readonly uri: string; readonly mimeType: string; readonly base64: string; -}): Promise<{ base64: string; uri: string } | null> { +}): Promise<{ base64: string; uri: string; mimeType: "image/png" | "image/jpeg" } | null> { const mimeType = downscalableImageMimeType(image.mimeType); - if (mimeType === null) return null; try { - const header = atob(image.base64.slice(0, Math.ceil(IMAGE_DIMENSIONS_HEADER_BYTES / 3) * 4)); - const bytes = new Uint8Array(header.length); - for (let index = 0; index < header.length; index += 1) bytes[index] = header.charCodeAt(index); - const dimensions = readImageDimensions(bytes); - if (dimensions === null || fitImageAttachmentSize(dimensions) === null) return null; - return await renderPhoto(image.uri, mimeType === "image/png" ? "png" : "jpeg"); + if (mimeType !== null && isOverMaxEdge(image.base64)) { + const scaled = await renderPhoto(image.uri, mimeType === "image/png" ? "png" : "jpeg"); + if ( + mimeType === "image/jpeg" || + estimateBase64ByteSize(scaled.base64) <= IMAGE_ATTACHMENT_MAX_BYTES + ) { + return { ...scaled, mimeType }; + } + } else if (estimateBase64ByteSize(image.base64) <= IMAGE_ATTACHMENT_MAX_BYTES) { + return null; + } + return { ...(await renderPhoto(image.uri)), mimeType: "image/jpeg" }; } catch (error) { console.warn("[composer-images] could not scale an oversized image", error); return null; diff --git a/apps/mobile/src/lib/composerImages.test.ts b/apps/mobile/src/lib/composerImages.test.ts index 52b6de41b709..3f8500ffdd5b 100644 --- a/apps/mobile/src/lib/composerImages.test.ts +++ b/apps/mobile/src/lib/composerImages.test.ts @@ -1,4 +1,5 @@ import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; +import { IMAGE_ATTACHMENT_MAX_BYTES } from "@t3tools/client-runtime/image-attachment-size"; import { PROVIDER_SEND_TURN_MAX_ATTACHMENTS } from "@t3tools/contracts"; const files = new Map(); @@ -14,7 +15,11 @@ vi.mock("expo-clipboard", () => clipboard); /** A native image renderer that reports the source size the test gives it. */ const manipulator = vi.hoisted(() => { - const state = { sourceSize: { width: 0, height: 0 } }; + const state: { + sourceSize: { width: number; height: number }; + renderedPngBase64?: string; + savedFormats: string[]; + } = { sourceSize: { width: 0, height: 0 }, savedFormats: [] }; const manipulate = vi.fn((source: string | { width: number; height: number }) => { let size = typeof source === "string" ? state.sourceSize : source; const context = { @@ -25,10 +30,16 @@ const manipulator = vi.hoisted(() => { renderAsync: vi.fn(async () => ({ ...size, release: vi.fn(), - saveAsync: vi.fn(async (options: { format: string }) => ({ - uri: `file:///cache/rendered.${options.format}`, - base64: `cmVuZGVyZWQ=`, - })), + saveAsync: vi.fn(async (options: { format: string }) => { + state.savedFormats.push(options.format); + return { + uri: `file:///cache/rendered.${options.format}`, + base64: + options.format === "png" && state.renderedPngBase64 !== undefined + ? state.renderedPngBase64 + : `cmVuZGVyZWQ=`, + }; + }), })), }; return context; @@ -159,8 +170,13 @@ describe("composer clipboard paste", () => { describe("oversized pasted images", () => { const u32 = (n: number) => [(n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff]; - const pngDataUrl = (width: number, height: number) => - `data:image/png;base64,${Buffer.from([ + const pngDataUrl = (width: number, height: number, totalBytes = 0) => { + const header = pngHeader(width, height); + const padding = Buffer.alloc(Math.max(0, totalBytes - header.length)); + return `data:image/png;base64,${Buffer.concat([header, padding]).toString("base64")}`; + }; + const pngHeader = (width: number, height: number) => + Buffer.from([ 0x89, 0x50, 0x4e, @@ -181,10 +197,12 @@ describe("oversized pasted images", () => { 0, 0, 0, - ]).toString("base64")}`; + ]); beforeEach(() => { vi.clearAllMocks(); + manipulator.state.renderedPngBase64 = undefined; + manipulator.state.savedFormats = []; clipboard.hasImageAsync.mockResolvedValue(true); }); @@ -207,6 +225,55 @@ describe("oversized pasted images", () => { ]); }); + it("renders a clipboard image within 2000px but over the byte limit to a JPEG", async () => { + manipulator.state.sourceSize = { width: 1800, height: 1200 }; + const pasted = pngDataUrl(1800, 1200, IMAGE_ATTACHMENT_MAX_BYTES + 1); + clipboard.getImageAsync.mockResolvedValue({ data: pasted }); + + const result = await pasteComposerClipboard({ existingCount: 0 }); + + expect(manipulator.manipulate).toHaveBeenCalledWith(pasted); + expect(manipulator.manipulate.mock.results.at(-1)?.value.resize).not.toHaveBeenCalled(); + expect(manipulator.state.savedFormats).toEqual(["jpeg"]); + expect(result.images).toEqual([ + expect.objectContaining({ + name: "pasted-image.jpg", + mimeType: "image/jpeg", + dataUrl: "data:image/jpeg;base64,cmVuZGVyZWQ=", + }), + ]); + }); + + it("renders a scaled PNG that is still over the byte limit to a JPEG", async () => { + manipulator.state.sourceSize = { width: 3024, height: 1964 }; + manipulator.state.renderedPngBase64 = Buffer.alloc(IMAGE_ATTACHMENT_MAX_BYTES + 1).toString( + "base64", + ); + clipboard.getImageAsync.mockResolvedValue({ data: pngDataUrl(3024, 1964) }); + + const result = await pasteComposerClipboard({ existingCount: 0 }); + + expect(manipulator.state.savedFormats).toEqual(["png", "jpeg"]); + expect(result.images).toEqual([ + expect.objectContaining({ + mimeType: "image/jpeg", + dataUrl: "data:image/jpeg;base64,cmVuZGVyZWQ=", + }), + ]); + }); + + it("keeps an image exactly at the byte limit byte for byte", async () => { + const pasted = pngDataUrl(1800, 1200, IMAGE_ATTACHMENT_MAX_BYTES); + clipboard.getImageAsync.mockResolvedValue({ data: pasted }); + + const result = await pasteComposerClipboard({ existingCount: 0 }); + + expect(manipulator.manipulate).not.toHaveBeenCalled(); + expect(result.images).toEqual([ + expect.objectContaining({ mimeType: "image/png", dataUrl: pasted }), + ]); + }); + it("keeps a 2000px clipboard image byte for byte without rendering it", async () => { const pasted = pngDataUrl(2000, 1200); clipboard.getImageAsync.mockResolvedValue({ data: pasted }); diff --git a/apps/mobile/src/lib/composerImages.ts b/apps/mobile/src/lib/composerImages.ts index f8e07704a24d..9c0e76a13163 100644 --- a/apps/mobile/src/lib/composerImages.ts +++ b/apps/mobile/src/lib/composerImages.ts @@ -20,7 +20,7 @@ import { } from "./composerAttachmentFiles"; import { imageMimeType } from "@t3tools/shared/image"; import { videoMimeType } from "@t3tools/shared/video"; -import { fitComposerImage, renderPhoto } from "./composerImageFit"; +import { fitComposerImage, jpegAttachmentName, renderPhoto } from "./composerImageFit"; import { beginForegroundHandoff } from "./foreground-handoff"; import { uuidv4 } from "./uuid"; import { writeFileAtomically } from "./atomic-file"; @@ -497,10 +497,11 @@ export async function pickComposerMedia(input: { mimeType: originalMimeType, base64, }); + const fittedMimeType = fitted?.mimeType ?? originalMimeType; image = { base64: fitted?.base64 ?? base64, - mimeType: originalMimeType, - name, + mimeType: fittedMimeType, + name: fittedMimeType === originalMimeType ? name : jpegAttachmentName(name), previewUri: fitted?.uri ?? asset.uri, }; } else { @@ -590,7 +591,8 @@ export async function pasteComposerClipboard(input: { readonly existingCount: nu base64: pastedBase64, }); const base64 = fitted?.base64 ?? pastedBase64; - const dataUrl = fitted ? `data:image/png;base64,${base64}` : image.data; + const mimeType = fitted?.mimeType ?? "image/png"; + const dataUrl = fitted ? `data:${mimeType};base64,${base64}` : image.data; const sizeBytes = estimateBase64ByteSize(base64); if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) { return { @@ -605,8 +607,8 @@ export async function pasteComposerClipboard(input: { readonly existingCount: nu { id: uuidv4(), type: "image", - name: "pasted-image.png", - mimeType: "image/png", + name: mimeType === "image/png" ? "pasted-image.png" : "pasted-image.jpg", + mimeType, sizeBytes, dataUrl, previewUri: dataUrl, @@ -681,9 +683,14 @@ export async function convertPastedImagesToAttachments(input: { continue; } const file = new File(uri); - const mimeType = mimeTypeFromUri(uri); + const pastedMimeType = mimeTypeFromUri(uri); const pastedBase64 = await file.base64(); - const fitted = await fitComposerImage({ uri, mimeType, base64: pastedBase64 }); + const fitted = await fitComposerImage({ + uri, + mimeType: pastedMimeType, + base64: pastedBase64, + }); + const mimeType = fitted?.mimeType ?? pastedMimeType; const base64 = fitted?.base64 ?? pastedBase64; const sizeBytes = estimateBase64ByteSize(base64); if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) { diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index 822e79da8dbd..2df9707d6af3 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -48,11 +48,11 @@ import { ProviderDriverKind, ProviderInstanceId, PROVIDER_SEND_TURN_MAX_ATTACHMENTS, - PROVIDER_SEND_TURN_MAX_IMAGE_BYTES, PROVIDER_SEND_TURN_MAX_INPUT_CHARS, PROVIDER_WORKSPACE_SNAPSHOT_TTL_MS, } from "@t3tools/contracts"; import type { EnvironmentConnectionPresentation } from "@t3tools/client-runtime/connection"; +import { IMAGE_ATTACHMENT_MAX_BYTES } from "@t3tools/client-runtime/image-attachment-size"; import { squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime"; import * as Schema from "effect/Schema"; import { @@ -5795,12 +5795,9 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) const nextImages: ComposerImageAttachment[] = []; let compressionError: string | null = null; for (const file of acceptedImages) { - // Images over the wire cap are downscaled to fit rather than + // Images over the per-image limit are downscaled to fit rather than // refused; files already within it pass through byte-for-byte. - const compressed = await prepareImageForAttachment( - file, - PROVIDER_SEND_TURN_MAX_IMAGE_BYTES, - ); + const compressed = await prepareImageForAttachment(file, IMAGE_ATTACHMENT_MAX_BYTES); if (!compressed.ok) { compressionError = compressed.reason === "unreadable" diff --git a/apps/web/src/components/desktop/SnapShotCoordinator.tsx b/apps/web/src/components/desktop/SnapShotCoordinator.tsx index e7ac04b0d068..30fb0a23fa5f 100644 --- a/apps/web/src/components/desktop/SnapShotCoordinator.tsx +++ b/apps/web/src/components/desktop/SnapShotCoordinator.tsx @@ -1,8 +1,5 @@ -import { - type DesktopPendingSnapShot, - PROVIDER_SEND_TURN_MAX_IMAGE_BYTES, - type ScopedThreadRef, -} from "@t3tools/contracts"; +import { IMAGE_ATTACHMENT_MAX_BYTES } from "@t3tools/client-runtime/image-attachment-size"; +import { type DesktopPendingSnapShot, type ScopedThreadRef } from "@t3tools/contracts"; import { useCallback, useEffect, useRef } from "react"; import { @@ -140,7 +137,7 @@ export async function deliverSnapShot( updateSnapShotAnimationSource(item.id, item.source); const capture = await bridge.readSnapShot(item.id); const original = dataUrlToFile(capture.dataUrl, capture.name, capture.mimeType); - const compressed = await compressImageToByteLimit(original, PROVIDER_SEND_TURN_MAX_IMAGE_BYTES); + const compressed = await compressImageToByteLimit(original, IMAGE_ATTACHMENT_MAX_BYTES); if (!compressed.ok) { finishSnapShotAnimation(item.id); throw new Error("The captured window is too large to attach."); diff --git a/apps/web/src/lib/imageCompression.ts b/apps/web/src/lib/imageCompression.ts index b80cc4bfec0c..5a01dac5599f 100644 --- a/apps/web/src/lib/imageCompression.ts +++ b/apps/web/src/lib/imageCompression.ts @@ -4,9 +4,9 @@ * * - The prompt stash persists images as base64 in localStorage (~5MB origin * quota), so `compressImageForStash` targets a per-image character budget. - * - The composer accepts pasted/dropped images larger than the provider's - * `PROVIDER_SEND_TURN_MAX_IMAGE_BYTES` wire cap and shrinks them to fit - * via `compressImageToByteLimit` instead of rejecting the paste. + * - The composer accepts pasted/dropped images larger than the per-image + * `IMAGE_ATTACHMENT_MAX_BYTES` limit and shrinks them to fit via + * `compressImageToByteLimit` instead of rejecting the paste. * * Supported images already within budget pass through untouched, except that * a PNG or JPEG longer than `IMAGE_ATTACHMENT_MAX_EDGE` on a side is scaled down diff --git a/packages/client-runtime/src/imageAttachmentSize.ts b/packages/client-runtime/src/imageAttachmentSize.ts index 400a991cec46..60bd36255d07 100644 --- a/packages/client-runtime/src/imageAttachmentSize.ts +++ b/packages/client-runtime/src/imageAttachmentSize.ts @@ -6,6 +6,14 @@ */ export const IMAGE_ATTACHMENT_MAX_EDGE = 2000; +/** + * Largest file, in bytes, an image attachment keeps. Anthropic rejects an image whose base64 + * is over 5 MB, and for the same mid-turn reason a heavier image ends the running Claude turn. + * This is that limit in decoded bytes; web and mobile re-encode anything heavier when it is + * attached, well below the `PROVIDER_SEND_TURN_MAX_IMAGE_BYTES` wire cap. + */ +export const IMAGE_ATTACHMENT_MAX_BYTES = ((5 * 1024 * 1024) / 4) * 3; + /** Formats a client re-encodes in place. GIF, WebP, and SVG are left as attached. */ export type DownscalableImageMimeType = "image/png" | "image/jpeg";