diff --git a/apps/mobile/src/features/sharing/incoming-share-model.ts b/apps/mobile/src/features/sharing/incoming-share-model.ts index a12343dfa4e2..6b125f006a3e 100644 --- a/apps/mobile/src/features/sharing/incoming-share-model.ts +++ b/apps/mobile/src/features/sharing/incoming-share-model.ts @@ -12,6 +12,7 @@ import * as Schema from "effect/Schema"; import type { ResolvedSharePayload, SharePayload } from "expo-sharing"; import { DraftComposerAttachmentSchema } from "../../lib/composer-image-schema"; +import { fitComposerImage } from "../../lib/composerImageFit"; import type { DraftComposerAttachment } from "../../lib/composerImages"; import { estimateBase64ByteSize } from "../../lib/base64"; @@ -406,8 +407,18 @@ export async function buildIncomingShareDraft(input: { } try { - const base64 = await input.fileReader.readBase64(uri); - const sizeBytes = resolved?.contentSize ?? estimateBase64ByteSize(base64); + const sharedBase64 = await input.fileReader.readBase64(uri); + // Rendered from the bytes already read, since the provider's file may not be readable + // by the native image loader. + const fitted = await fitComposerImage({ + uri: `data:${mimeType};base64,${sharedBase64}`, + mimeType, + base64: sharedBase64, + }); + const base64 = fitted?.base64 ?? sharedBase64; + const sizeBytes = fitted + ? estimateBase64ByteSize(base64) + : (resolved?.contentSize ?? estimateBase64ByteSize(base64)); if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) { warnings.push( `'${resolved?.originalName ?? fallbackName(uri, index, mimeType)}' exceeds the 10 MB attachment limit.`, diff --git a/apps/mobile/src/lib/composerFiles.test.ts b/apps/mobile/src/lib/composerFiles.test.ts index a99c2c6064e7..332c0ad9c3d9 100644 --- a/apps/mobile/src/lib/composerFiles.test.ts +++ b/apps/mobile/src/lib/composerFiles.test.ts @@ -185,10 +185,10 @@ describe("composer file attachments", () => { ); it.each([ - { size: { width: 4032, height: 3024 }, resizes: [{ width: 2048 }] }, - { size: { width: 3024, height: 4032 }, resizes: [{ height: 2048 }] }, - { size: { width: 2048, height: 1536 }, resizes: [] }, - ])("bounds a $size.width x $size.height photo to a 2048 px longest edge", async (input) => { + { size: { width: 4032, height: 3024 }, resizes: [{ width: 2000, height: 1500 }] }, + { size: { width: 3024, height: 4032 }, resizes: [{ width: 1500, height: 2000 }] }, + { size: { width: 2000, height: 1500 }, resizes: [] }, + ])("bounds a $size.width x $size.height photo to a 2000 px longest edge", async (input) => { native.size = input.size; mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [photo] }); diff --git a/apps/mobile/src/lib/composerImageFit.ts b/apps/mobile/src/lib/composerImageFit.ts new file mode 100644 index 000000000000..ccc2f5ebaf1c --- /dev/null +++ b/apps/mobile/src/lib/composerImageFit.ts @@ -0,0 +1,70 @@ +import { + downscalableImageMimeType, + fitImageAttachmentSize, +} from "@t3tools/client-runtime/image-attachment-size"; +import { + IMAGE_DIMENSIONS_HEADER_BYTES, + readImageDimensions, +} from "@t3tools/shared/imageDimensions"; + +const PHOTO_JPEG_QUALITY = 0.85; + +/** + * Renders an image to a JPEG, or a PNG when `format` asks for one, with its long edge capped at + * `IMAGE_ATTACHMENT_MAX_EDGE`. Decode, downscale, + * and encode run natively; only the bounded result crosses the bridge. Camera photos are + * 12-48 MP HEIC files, so a full-size conversion is both slow to transfer and far more than a + * model can use. + */ +export async function renderPhoto( + uri: string, + format: "jpeg" | "png" = "jpeg", +): Promise<{ base64: string; uri: string }> { + const { ImageManipulator, SaveFormat } = await import("expo-image-manipulator"); + let image = await ImageManipulator.manipulate(uri).renderAsync(); + try { + const fittedSize = fitImageAttachmentSize(image); + if (fittedSize !== null) { + const resized = await ImageManipulator.manipulate(image).resize(fittedSize).renderAsync(); + image.release(); + image = resized; + } + const saved = await image.saveAsync({ + format: format === "png" ? SaveFormat.PNG : SaveFormat.JPEG, + compress: PHOTO_JPEG_QUALITY, + base64: true, + }); + if (!saved.base64) { + throw new Error("The rendered photo has no bytes."); + } + return { base64: saved.base64, uri: saved.uri }; + } finally { + image.release(); + } +} + +/** + * Scales a PNG or JPEG whose long edge is over `IMAGE_ATTACHMENT_MAX_EDGE` down to that edge + * in its own format. Returns null when the image fits, is another format (GIF, WebP), or the + * native renderer fails, so the caller keeps the original bytes. Images inside the limit are + * never re-encoded: only the header is read to decide. + */ +export async function fitComposerImage(image: { + readonly uri: string; + readonly mimeType: string; + readonly base64: string; +}): Promise<{ base64: string; uri: string } | null> { + const mimeType = downscalableImageMimeType(image.mimeType); + if (mimeType === null) return null; + try { + const header = atob(image.base64.slice(0, Math.ceil(IMAGE_DIMENSIONS_HEADER_BYTES / 3) * 4)); + const bytes = new Uint8Array(header.length); + for (let index = 0; index < header.length; index += 1) bytes[index] = header.charCodeAt(index); + const dimensions = readImageDimensions(bytes); + if (dimensions === null || fitImageAttachmentSize(dimensions) === null) return null; + return await renderPhoto(image.uri, mimeType === "image/png" ? "png" : "jpeg"); + } catch (error) { + console.warn("[composer-images] could not scale an oversized image", error); + return null; + } +} diff --git a/apps/mobile/src/lib/composerImages.test.ts b/apps/mobile/src/lib/composerImages.test.ts index 3e05c578f3f3..52b6de41b709 100644 --- a/apps/mobile/src/lib/composerImages.test.ts +++ b/apps/mobile/src/lib/composerImages.test.ts @@ -12,6 +12,35 @@ const clipboard = vi.hoisted(() => ({ vi.mock("expo-clipboard", () => clipboard); +/** A native image renderer that reports the source size the test gives it. */ +const manipulator = vi.hoisted(() => { + const state = { sourceSize: { width: 0, height: 0 } }; + const manipulate = vi.fn((source: string | { width: number; height: number }) => { + let size = typeof source === "string" ? state.sourceSize : source; + const context = { + resize: vi.fn((next: { width: number; height: number }) => { + size = next; + return context; + }), + renderAsync: vi.fn(async () => ({ + ...size, + release: vi.fn(), + saveAsync: vi.fn(async (options: { format: string }) => ({ + uri: `file:///cache/rendered.${options.format}`, + base64: `cmVuZGVyZWQ=`, + })), + })), + }; + return context; + }); + return { state, manipulate }; +}); + +vi.mock("expo-image-manipulator", () => ({ + ImageManipulator: { manipulate: manipulator.manipulate }, + SaveFormat: { JPEG: "jpeg", PNG: "png" }, +})); + vi.mock("expo-file-system", () => ({ File: class { readonly uri: string; @@ -128,6 +157,67 @@ describe("composer clipboard paste", () => { }); }); +describe("oversized pasted images", () => { + const u32 = (n: number) => [(n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff]; + const pngDataUrl = (width: number, height: number) => + `data:image/png;base64,${Buffer.from([ + 0x89, + 0x50, + 0x4e, + 0x47, + 0x0d, + 0x0a, + 0x1a, + 0x0a, + ...u32(13), + 0x49, + 0x48, + 0x44, + 0x52, + ...u32(width), + ...u32(height), + 8, + 6, + 0, + 0, + 0, + ]).toString("base64")}`; + + beforeEach(() => { + vi.clearAllMocks(); + clipboard.hasImageAsync.mockResolvedValue(true); + }); + + it("scales a clipboard screenshot over 2000px to 2000px and keeps it a PNG", async () => { + manipulator.state.sourceSize = { width: 3024, height: 1964 }; + const pasted = pngDataUrl(3024, 1964); + clipboard.getImageAsync.mockResolvedValue({ data: pasted }); + + const result = await pasteComposerClipboard({ existingCount: 0 }); + + expect(manipulator.manipulate).toHaveBeenCalledWith(pasted); + const resize = manipulator.manipulate.mock.results.at(-1)?.value.resize; + expect(resize).toHaveBeenCalledWith({ width: 2000, height: 1299 }); + expect(result.images).toEqual([ + expect.objectContaining({ + mimeType: "image/png", + dataUrl: "data:image/png;base64,cmVuZGVyZWQ=", + previewUri: "data:image/png;base64,cmVuZGVyZWQ=", + }), + ]); + }); + + it("keeps a 2000px clipboard image byte for byte without rendering it", async () => { + const pasted = pngDataUrl(2000, 1200); + clipboard.getImageAsync.mockResolvedValue({ data: pasted }); + + const result = await pasteComposerClipboard({ existingCount: 0 }); + + expect(manipulator.manipulate).not.toHaveBeenCalled(); + expect(result.images).toEqual([expect.objectContaining({ dataUrl: pasted })]); + }); +}); + describe("native pasted image cleanup", () => { beforeEach(() => { files.clear(); diff --git a/apps/mobile/src/lib/composerImages.ts b/apps/mobile/src/lib/composerImages.ts index 8fc821d2d26f..f8e07704a24d 100644 --- a/apps/mobile/src/lib/composerImages.ts +++ b/apps/mobile/src/lib/composerImages.ts @@ -20,6 +20,7 @@ import { } from "./composerAttachmentFiles"; import { imageMimeType } from "@t3tools/shared/image"; import { videoMimeType } from "@t3tools/shared/video"; +import { fitComposerImage, renderPhoto } from "./composerImageFit"; import { beginForegroundHandoff } from "./foreground-handoff"; import { uuidv4 } from "./uuid"; import { writeFileAtomically } from "./atomic-file"; @@ -341,46 +342,6 @@ export async function pickComposerFiles(input: { return { files: attachments, error }; } -/** - * Longest edge kept when a photo has to be re-encoded. Matches the web composer's - * MAX_DIMENSION so every client hands providers the same resolution. - */ -const PHOTO_MAX_EDGE = 2048; -const PHOTO_JPEG_QUALITY = 0.85; - -/** - * Renders a photo-library pick to a provider-readable JPEG. Decode, downscale, and encode run - * natively; only the bounded result crosses the bridge. Camera photos are 12-48 MP HEIC files, - * so a full-size conversion is both slow to transfer and far more than a model can use. - */ -async function renderPhotoAsJpeg(uri: string): Promise<{ base64: string; uri: string }> { - const { ImageManipulator, SaveFormat } = await import("expo-image-manipulator"); - let image = await ImageManipulator.manipulate(uri).renderAsync(); - try { - const longestEdge = Math.max(image.width, image.height); - if (longestEdge > PHOTO_MAX_EDGE) { - const resized = await ImageManipulator.manipulate(image) - .resize( - image.width >= image.height ? { width: PHOTO_MAX_EDGE } : { height: PHOTO_MAX_EDGE }, - ) - .renderAsync(); - image.release(); - image = resized; - } - const saved = await image.saveAsync({ - format: SaveFormat.JPEG, - compress: PHOTO_JPEG_QUALITY, - base64: true, - }); - if (!saved.base64) { - throw new Error("The rendered photo has no bytes."); - } - return { base64: saved.base64, uri: saved.uri }; - } finally { - image.release(); - } -} - async function loadImagePicker() { try { return await import("expo-image-picker"); @@ -530,14 +491,20 @@ export async function pickComposerMedia(input: { try { if (originalMimeType !== null) { const { File } = await import("expo-file-system"); + const base64 = await new File(asset.uri).base64(); + const fitted = await fitComposerImage({ + uri: asset.uri, + mimeType: originalMimeType, + base64, + }); image = { - base64: await new File(asset.uri).base64(), + base64: fitted?.base64 ?? base64, mimeType: originalMimeType, name, - previewUri: asset.uri, + previewUri: fitted?.uri ?? asset.uri, }; } else { - const rendered = await renderPhotoAsJpeg(asset.uri); + const rendered = await renderPhoto(asset.uri); image = { base64: rendered.base64, mimeType: "image/jpeg", @@ -616,7 +583,14 @@ export async function pasteComposerClipboard(input: { readonly existingCount: nu }; } - const base64 = image.data.split(",")[1] ?? ""; + const pastedBase64 = image.data.split(",")[1] ?? ""; + const fitted = await fitComposerImage({ + uri: image.data, + mimeType: "image/png", + base64: pastedBase64, + }); + const base64 = fitted?.base64 ?? pastedBase64; + const dataUrl = fitted ? `data:image/png;base64,${base64}` : image.data; const sizeBytes = estimateBase64ByteSize(base64); if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) { return { @@ -634,8 +608,8 @@ export async function pasteComposerClipboard(input: { readonly existingCount: nu name: "pasted-image.png", mimeType: "image/png", sizeBytes, - dataUrl: image.data, - previewUri: image.data, + dataUrl, + previewUri: dataUrl, }, ], text: null, @@ -707,12 +681,14 @@ export async function convertPastedImagesToAttachments(input: { continue; } const file = new File(uri); - const base64 = await file.base64(); + const mimeType = mimeTypeFromUri(uri); + const pastedBase64 = await file.base64(); + const fitted = await fitComposerImage({ uri, mimeType, base64: pastedBase64 }); + const base64 = fitted?.base64 ?? pastedBase64; const sizeBytes = estimateBase64ByteSize(base64); if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) { continue; } - const mimeType = mimeTypeFromUri(uri); results.push({ id: uuidv4(), type: "image", @@ -720,7 +696,7 @@ export async function convertPastedImagesToAttachments(input: { mimeType, sizeBytes, dataUrl: `data:${mimeType};base64,${base64}`, - previewUri: ownedTemporaryFile ? `data:${mimeType};base64,${base64}` : uri, + previewUri: ownedTemporaryFile ? `data:${mimeType};base64,${base64}` : (fitted?.uri ?? uri), }); } catch (error) { console.warn("Failed to read pasted image", uri, error); diff --git a/apps/web/src/lib/imageCompression.test.ts b/apps/web/src/lib/imageCompression.test.ts index e82fc9f89288..db0c0f01f679 100644 --- a/apps/web/src/lib/imageCompression.test.ts +++ b/apps/web/src/lib/imageCompression.test.ts @@ -598,6 +598,118 @@ describe("HEIC attachment preparation", () => { }); }); +describe("attachment max edge", () => { + const u32 = (n: number) => [(n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff]; + const u16 = (n: number) => [(n >>> 8) & 0xff, n & 0xff]; + const pngHeader = (width: number, height: number) => [ + 0x89, + 0x50, + 0x4e, + 0x47, + 0x0d, + 0x0a, + 0x1a, + 0x0a, + ...u32(13), + 0x49, + 0x48, + 0x44, + 0x52, + ...u32(width), + ...u32(height), + 8, + 6, + 0, + 0, + 0, + ]; + const jpegHeader = (width: number, height: number) => [ + 0xff, + 0xd8, + 0xff, + 0xc0, + ...u16(17), + 8, + ...u16(height), + ...u16(width), + ]; + const image = (header: number[], name: string, type: string) => + new File([new Uint8Array([...header, ...new Uint8Array(2048).fill(7)])], name, { type }); + + /** Records the decoded size, the canvas sizes, and the encode calls. */ + function stubDecoder(bitmapSize: { width: number; height: number }) { + const decode = vi.fn(async () => ({ ...bitmapSize, close: vi.fn() })); + const canvases: Array<{ width: number; height: number }> = []; + const encodes: Array<{ type: string; quality: number }> = []; + const fillRect = vi.fn(); + vi.stubGlobal("createImageBitmap", decode); + vi.stubGlobal( + "OffscreenCanvas", + class { + constructor(width: number, height: number) { + canvases.push({ width, height }); + } + getContext() { + return { fillRect, drawImage: vi.fn() }; + } + async convertToBlob(options: { type: string; quality: number }) { + encodes.push(options); + return new Blob([new Uint8Array(500_000)], { type: options.type }); + } + }, + ); + return { decode, canvases, encodes, fillRect }; + } + + it("scales a large PNG screenshot under the byte cap to 2000px and keeps it a PNG", async () => { + const { decode, canvases, encodes, fillRect } = stubDecoder({ width: 3024, height: 1964 }); + const original = image(pngHeader(3024, 1964), "shot.png", "image/png"); + + const result = await prepareImageForAttachment(original, 10 * 1024 * 1024); + + expect(decode).toHaveBeenCalledExactlyOnceWith(original, { imageOrientation: "from-image" }); + expect(canvases).toEqual([{ width: 2000, height: 1299 }]); + expect(encodes.map(({ type }) => type)).toEqual(["image/png"]); + // PNG keeps alpha, so nothing paints a matte behind it. + expect(fillRect).not.toHaveBeenCalled(); + expect(result.ok && result.recompressed).toBe(true); + expect(result.ok && result.imageSize).toEqual({ width: 2000, height: 1299 }); + expect(result.ok && result.file.name).toBe("shot.png"); + expect(result.ok && result.file.type).toBe("image/png"); + }); + + it("scales a rotated JPEG by its displayed size and keeps it a JPEG", async () => { + // Stored landscape, displayed portrait after the EXIF rotation. + const { canvases, encodes } = stubDecoder({ width: 3024, height: 4032 }); + const original = image(jpegHeader(4032, 3024), "photo.jpg", "image/jpeg"); + + const result = await prepareImageForAttachment(original, 10 * 1024 * 1024); + + expect(canvases).toEqual([{ width: 1500, height: 2000 }]); + expect(encodes).toEqual([{ type: "image/jpeg", quality: 0.92 }]); + expect(result.ok && result.file.type).toBe("image/jpeg"); + expect(result.ok && result.imageSize).toEqual({ width: 1500, height: 2000 }); + }); + + it.each([ + { label: "a 2000px PNG", header: pngHeader(2000, 1200), type: "image/png" }, + { + label: "a 3000px GIF", + header: [0x47, 0x49, 0x46, 0x38, 0x39, 0x61, 0xb8, 0x0b, 0x10, 0x04], + type: "image/gif", + }, + ])("passes $label through without decoding it", async ({ header, type }) => { + const { decode } = stubDecoder({ width: 3000, height: 1040 }); + const original = image(header, "image", type); + + const result = await prepareImageForAttachment(original, 10 * 1024 * 1024); + + expect(result.ok && result.file).toBe(original); + expect(result.ok && result.recompressed).toBe(false); + expect(decode).not.toHaveBeenCalled(); + }); +}); + describe("snapshot coordinates after compression", () => { const source: SnapShotSource = { kind: "snap-shot", @@ -649,7 +761,7 @@ describe("snapshot coordinates after compression", () => { sizeBytes: compressed.file.size, dataUrl: `data:${compressed.file.type};base64,${Buffer.from(await compressed.file.arrayBuffer()).toString("base64")}`, }; - expect(image.imageSize).toEqual({ width: 2048, height: 1280 }); + expect(image.imageSize).toEqual({ width: 2000, height: 1250 }); const resized = resizeSnapShotSource(source, image.imageSize); const [restored] = hydrateImagesFromPersisted([ { @@ -662,10 +774,10 @@ describe("snapshot coordinates after compression", () => { }, ]); expect(restored?.source?.accessibility).toMatchObject({ - imageSize: { width: 2048, height: 1280 }, + imageSize: { width: 2000, height: 1250 }, root: { - bounds: { x: 0, y: 0, width: 2048, height: 1280 }, - children: [{ bounds: { x: 1920, y: 1120, width: 80, height: 80 } }, { bounds: null }], + bounds: { x: 0, y: 0, width: 2000, height: 1250 }, + children: [{ bounds: { x: 1875, y: 1094, width: 78, height: 78 } }, { bounds: null }], }, }); expect(source.accessibility).toMatchObject({ imageSize: { width: 2560, height: 1600 } }); diff --git a/apps/web/src/lib/imageCompression.ts b/apps/web/src/lib/imageCompression.ts index 36a05f646d16..b80cc4bfec0c 100644 --- a/apps/web/src/lib/imageCompression.ts +++ b/apps/web/src/lib/imageCompression.ts @@ -8,15 +8,24 @@ * `PROVIDER_SEND_TURN_MAX_IMAGE_BYTES` wire cap and shrinks them to fit * via `compressImageToByteLimit` instead of rejecting the paste. * - * Supported images already within budget pass through untouched. HEIC/HEIF - * photos are decoded to JPEG first because providers cannot consume them. + * Supported images already within budget pass through untouched, except that + * a PNG or JPEG longer than `IMAGE_ATTACHMENT_MAX_EDGE` on a side is scaled down + * to it in its own format before upload. HEIC/HEIF photos are decoded to JPEG + * first because providers cannot consume them. */ -/** - * Longest edge kept when an image has to be re-encoded. Sized so a typical - * retina screenshot (3024px wide) stays legible rather than being halved. - */ -const MAX_DIMENSION = 2048; +import { + downscalableImageMimeType, + fitImageAttachmentSize, + IMAGE_ATTACHMENT_MAX_EDGE, +} from "@t3tools/client-runtime/image-attachment-size"; +import { + IMAGE_DIMENSIONS_HEADER_BYTES, + readImageDimensions, +} from "@t3tools/shared/imageDimensions"; + +/** Longest edge kept when an image has to be re-encoded. */ +const MAX_DIMENSION = IMAGE_ATTACHMENT_MAX_EDGE; /** Base64 budget for a single stashed image (~975KB of binary). */ export const MAX_STASH_IMAGE_DATA_URL_CHARS = 1_300_000; /** @@ -423,10 +432,47 @@ export async function compressImageForStash( } } +/** + * Scales a PNG or JPEG whose long edge is over `IMAGE_ATTACHMENT_MAX_EDGE` + * down to that edge in its own format, so a PNG keeps its transparency. + * Returns null when the image already fits, is another format (GIF, WebP, + * SVG), or cannot be decoded or encoded here; the caller keeps the original. + */ +async function fitImageToMaxEdge(file: File): Promise<{ file: File; imageSize: ImageSize } | null> { + const mimeType = downscalableImageMimeType(file.type); + if (!mimeType || !canRecompress() || file.size > MAX_COMPRESSIBLE_SOURCE_BYTES) return null; + let bitmap: ImageBitmap | undefined; + try { + // The header answers "does it fit?" without decoding every attachment. + const header = readImageDimensions( + new Uint8Array(await file.slice(0, IMAGE_DIMENSIONS_HEADER_BYTES).arrayBuffer()), + ); + if (!header || !fitImageAttachmentSize(header)) return null; + bitmap = await createImageBitmap(file, { imageOrientation: "from-image" }); + const imageSize = fitImageAttachmentSize(bitmap); + if (!imageSize) return null; + const target = createCanvas(imageSize.width, imageSize.height); + if (!target) return null; + target.context.imageSmoothingQuality = "high"; + target.context.drawImage(bitmap, 0, 0, imageSize.width, imageSize.height); + const blob = await encodeCanvas(target.canvas, QUALITY_STEPS[0], mimeType); + if (!blob) return null; + return { + file: new File([blob], file.name, { type: mimeType, lastModified: file.lastModified }), + imageSize, + }; + } catch { + return null; + } finally { + bitmap?.close(); + } +} + /** * Shrinks `file` until its binary size fits `maxBytes`, returning a new - * `File` (WebP or JPEG). Files already within the limit pass through - * untouched, preserving their exact bytes and format. Sources above + * `File` (WebP or JPEG). A PNG or JPEG over `IMAGE_ATTACHMENT_MAX_EDGE` is + * first scaled to that edge in its own format. Other files already within the + * limit pass through untouched, preserving their exact bytes and format. Sources above * `MAX_COMPRESSIBLE_SOURCE_BYTES` are refused outright — decoding them is * the risk, so no amount of output budget makes them safe. An internally * converted image can provide its original source size when the intermediate @@ -437,7 +483,11 @@ export async function compressImageToByteLimit( maxBytes: number, options?: { preferredMimeType?: "image/jpeg"; sourceSizeBytes?: number }, ): Promise { - if (file.size <= maxBytes) { + const fitted = await fitImageToMaxEdge(file); + if (fitted && fitted.file.size <= maxBytes) { + return { ok: true, file: fitted.file, recompressed: true, imageSize: fitted.imageSize }; + } + if (!fitted && file.size <= maxBytes) { return { ok: true, file, recompressed: false }; } if ((options?.sourceSizeBytes ?? file.size) > MAX_COMPRESSIBLE_SOURCE_BYTES) { @@ -447,7 +497,11 @@ export async function compressImageToByteLimit( // into 4 chars; flooring keeps the budget a hair conservative instead of // admitting an encoding right at the byte cap. const budgetChars = Math.floor(maxBytes / 3) * 4; - const reencoded = await reencodeWithinBudget(file, budgetChars, options?.preferredMimeType); + const reencoded = await reencodeWithinBudget( + fitted?.file ?? file, + budgetChars, + options?.preferredMimeType, + ); if (!reencoded.ok) { return reencoded; } diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index 0366c65b1ed2..703d418ac93a 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -7,6 +7,10 @@ "types": "./src/composerThreadItems.ts", "default": "./src/composerThreadItems.ts" }, + "./image-attachment-size": { + "types": "./src/imageAttachmentSize.ts", + "default": "./src/imageAttachmentSize.ts" + }, "./handoff": { "types": "./src/handoff.ts", "default": "./src/handoff.ts" diff --git a/packages/client-runtime/src/imageAttachmentSize.ts b/packages/client-runtime/src/imageAttachmentSize.ts new file mode 100644 index 000000000000..400a991cec46 --- /dev/null +++ b/packages/client-runtime/src/imageAttachmentSize.ts @@ -0,0 +1,34 @@ +/** + * Longest edge, in pixels, a PNG or JPEG attachment keeps. Anthropic rejects an image over + * 2000px on a side once a request holds more than 20 images, and Claude Code forwards images + * sent into a running turn unchanged, so one large paste breaks every later request in that + * thread. Web and mobile scale attachments down to this edge when they are attached. + */ +export const IMAGE_ATTACHMENT_MAX_EDGE = 2000; + +/** Formats a client re-encodes in place. GIF, WebP, and SVG are left as attached. */ +export type DownscalableImageMimeType = "image/png" | "image/jpeg"; + +export function downscalableImageMimeType(mimeType: string): DownscalableImageMimeType | null { + const normalized = mimeType.toLowerCase(); + if (normalized === "image/png") return "image/png"; + if (normalized === "image/jpeg" || normalized === "image/jpg") return "image/jpeg"; + return null; +} + +/** + * The size an image should be scaled to so its long edge fits `IMAGE_ATTACHMENT_MAX_EDGE`, + * keeping its aspect ratio. Returns null when it already fits. + */ +export function fitImageAttachmentSize(size: { + readonly width: number; + readonly height: number; +}): { width: number; height: number } | null { + const longestEdge = Math.max(size.width, size.height); + if (longestEdge <= IMAGE_ATTACHMENT_MAX_EDGE) return null; + const scale = IMAGE_ATTACHMENT_MAX_EDGE / longestEdge; + return { + width: Math.max(1, Math.min(IMAGE_ATTACHMENT_MAX_EDGE, Math.round(size.width * scale))), + height: Math.max(1, Math.min(IMAGE_ATTACHMENT_MAX_EDGE, Math.round(size.height * scale))), + }; +}