authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-10-14 03:20:19-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-14 03:20:19-07:00
logadb04202b25c1c45255e76be9f03f0b346b9c5c4
treef24912982581b8b46eb0dbb185f34e5b23223c39
parent26c48bbc0a9ddc23acdd20a733fe209d423e848a
signature Signed by PGP key B5690EEEBB952194

fix(clients): large pasted images no longer break later Claude image reads (#19364)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

9 files changed, 421 insertions(+), 70 deletions(-)

apps/mobile/src/features/sharing/incoming-share-model.ts+13-2
...@@ -12,6 +12,7 @@ import * as Schema from "effect/Schema";...@@ -12,6 +12,7 @@ import * as Schema from "effect/Schema";
12import type { ResolvedSharePayload, SharePayload } from "expo-sharing";12import type { ResolvedSharePayload, SharePayload } from "expo-sharing";
1313
14import { DraftComposerAttachmentSchema } from "../../lib/composer-image-schema";14import { DraftComposerAttachmentSchema } from "../../lib/composer-image-schema";
15import { fitComposerImage } from "../../lib/composerImageFit";
15import type { DraftComposerAttachment } from "../../lib/composerImages";16import type { DraftComposerAttachment } from "../../lib/composerImages";
16import { estimateBase64ByteSize } from "../../lib/base64";17import { estimateBase64ByteSize } from "../../lib/base64";
1718
...@@ -406,8 +407,18 @@ export async function buildIncomingShareDraft(input: {...@@ -406,8 +407,18 @@ export async function buildIncomingShareDraft(input: {
406 }407 }
407408
408 try {409 try {
409 const base64 = await input.fileReader.readBase64(uri);410 const sharedBase64 = await input.fileReader.readBase64(uri);
410 const sizeBytes = resolved?.contentSize ?? estimateBase64ByteSize(base64);411 // Rendered from the bytes already read, since the provider's file may not be readable
412 // by the native image loader.
413 const fitted = await fitComposerImage({
414 uri: `data:${mimeType};base64,${sharedBase64}`,
415 mimeType,
416 base64: sharedBase64,
417 });
418 const base64 = fitted?.base64 ?? sharedBase64;
419 const sizeBytes = fitted
420 ? estimateBase64ByteSize(base64)
421 : (resolved?.contentSize ?? estimateBase64ByteSize(base64));
411 if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) {422 if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) {
412 warnings.push(423 warnings.push(
413 `'${resolved?.originalName ?? fallbackName(uri, index, mimeType)}' exceeds the 10 MB attachment limit.`,424 `'${resolved?.originalName ?? fallbackName(uri, index, mimeType)}' exceeds the 10 MB attachment limit.`,
apps/mobile/src/lib/composerFiles.test.ts+4-4
...@@ -185,10 +185,10 @@ describe("composer file attachments", () => {...@@ -185,10 +185,10 @@ describe("composer file attachments", () => {
185 );185 );
186186
187 it.each([187 it.each([
188 { size: { width: 4032, height: 3024 }, resizes: [{ width: 2048 }] },188 { size: { width: 4032, height: 3024 }, resizes: [{ width: 2000, height: 1500 }] },
189 { size: { width: 3024, height: 4032 }, resizes: [{ height: 2048 }] },189 { size: { width: 3024, height: 4032 }, resizes: [{ width: 1500, height: 2000 }] },
190 { size: { width: 2048, height: 1536 }, resizes: [] },190 { size: { width: 2000, height: 1500 }, resizes: [] },
191 ])("bounds a $size.width x $size.height photo to a 2048 px longest edge", async (input) => {191 ])("bounds a $size.width x $size.height photo to a 2000 px longest edge", async (input) => {
192 native.size = input.size;192 native.size = input.size;
193 mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [photo] });193 mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [photo] });
194194
apps/mobile/src/lib/composerImageFit.ts created+70
...@@ -0,0 +1,70 @@
1import {
2 downscalableImageMimeType,
3 fitImageAttachmentSize,
4} from "@t3tools/client-runtime/image-attachment-size";
5import {
6 IMAGE_DIMENSIONS_HEADER_BYTES,
7 readImageDimensions,
8} from "@t3tools/shared/imageDimensions";
9
10const PHOTO_JPEG_QUALITY = 0.85;
11
12/**
13 * Renders an image to a JPEG, or a PNG when `format` asks for one, with its long edge capped at
14 * `IMAGE_ATTACHMENT_MAX_EDGE`. Decode, downscale,
15 * and encode run natively; only the bounded result crosses the bridge. Camera photos are
16 * 12-48 MP HEIC files, so a full-size conversion is both slow to transfer and far more than a
17 * model can use.
18 */
19export async function renderPhoto(
20 uri: string,
21 format: "jpeg" | "png" = "jpeg",
22): Promise<{ base64: string; uri: string }> {
23 const { ImageManipulator, SaveFormat } = await import("expo-image-manipulator");
24 let image = await ImageManipulator.manipulate(uri).renderAsync();
25 try {
26 const fittedSize = fitImageAttachmentSize(image);
27 if (fittedSize !== null) {
28 const resized = await ImageManipulator.manipulate(image).resize(fittedSize).renderAsync();
29 image.release();
30 image = resized;
31 }
32 const saved = await image.saveAsync({
33 format: format === "png" ? SaveFormat.PNG : SaveFormat.JPEG,
34 compress: PHOTO_JPEG_QUALITY,
35 base64: true,
36 });
37 if (!saved.base64) {
38 throw new Error("The rendered photo has no bytes.");
39 }
40 return { base64: saved.base64, uri: saved.uri };
41 } finally {
42 image.release();
43 }
44}
45
46/**
47 * Scales a PNG or JPEG whose long edge is over `IMAGE_ATTACHMENT_MAX_EDGE` down to that edge
48 * in its own format. Returns null when the image fits, is another format (GIF, WebP), or the
49 * native renderer fails, so the caller keeps the original bytes. Images inside the limit are
50 * never re-encoded: only the header is read to decide.
51 */
52export async function fitComposerImage(image: {
53 readonly uri: string;
54 readonly mimeType: string;
55 readonly base64: string;
56}): Promise<{ base64: string; uri: string } | null> {
57 const mimeType = downscalableImageMimeType(image.mimeType);
58 if (mimeType === null) return null;
59 try {
60 const header = atob(image.base64.slice(0, Math.ceil(IMAGE_DIMENSIONS_HEADER_BYTES / 3) * 4));
61 const bytes = new Uint8Array(header.length);
62 for (let index = 0; index < header.length; index += 1) bytes[index] = header.charCodeAt(index);
63 const dimensions = readImageDimensions(bytes);
64 if (dimensions === null || fitImageAttachmentSize(dimensions) === null) return null;
65 return await renderPhoto(image.uri, mimeType === "image/png" ? "png" : "jpeg");
66 } catch (error) {
67 console.warn("[composer-images] could not scale an oversized image", error);
68 return null;
69 }
70}
apps/mobile/src/lib/composerImages.test.ts+90
...@@ -12,6 +12,35 @@ const clipboard = vi.hoisted(() => ({...@@ -12,6 +12,35 @@ const clipboard = vi.hoisted(() => ({
1212
13vi.mock("expo-clipboard", () => clipboard);13vi.mock("expo-clipboard", () => clipboard);
1414
15/** A native image renderer that reports the source size the test gives it. */
16const manipulator = vi.hoisted(() => {
17 const state = { sourceSize: { width: 0, height: 0 } };
18 const manipulate = vi.fn((source: string | { width: number; height: number }) => {
19 let size = typeof source === "string" ? state.sourceSize : source;
20 const context = {
21 resize: vi.fn((next: { width: number; height: number }) => {
22 size = next;
23 return context;
24 }),
25 renderAsync: vi.fn(async () => ({
26 ...size,
27 release: vi.fn(),
28 saveAsync: vi.fn(async (options: { format: string }) => ({
29 uri: `file:///cache/rendered.${options.format}`,
30 base64: `cmVuZGVyZWQ=`,
31 })),
32 })),
33 };
34 return context;
35 });
36 return { state, manipulate };
37});
38
39vi.mock("expo-image-manipulator", () => ({
40 ImageManipulator: { manipulate: manipulator.manipulate },
41 SaveFormat: { JPEG: "jpeg", PNG: "png" },
42}));
43
15vi.mock("expo-file-system", () => ({44vi.mock("expo-file-system", () => ({
16 File: class {45 File: class {
17 readonly uri: string;46 readonly uri: string;
...@@ -128,6 +157,67 @@ describe("composer clipboard paste", () => {...@@ -128,6 +157,67 @@ describe("composer clipboard paste", () => {
128 });157 });
129});158});
130159
160describe("oversized pasted images", () => {
161 const u32 = (n: number) => [(n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff];
162 const pngDataUrl = (width: number, height: number) =>
163 `data:image/png;base64,${Buffer.from([
164 0x89,
165 0x50,
166 0x4e,
167 0x47,
168 0x0d,
169 0x0a,
170 0x1a,
171 0x0a,
172 ...u32(13),
173 0x49,
174 0x48,
175 0x44,
176 0x52,
177 ...u32(width),
178 ...u32(height),
179 8,
180 6,
181 0,
182 0,
183 0,
184 ]).toString("base64")}`;
185
186 beforeEach(() => {
187 vi.clearAllMocks();
188 clipboard.hasImageAsync.mockResolvedValue(true);
189 });
190
191 it("scales a clipboard screenshot over 2000px to 2000px and keeps it a PNG", async () => {
192 manipulator.state.sourceSize = { width: 3024, height: 1964 };
193 const pasted = pngDataUrl(3024, 1964);
194 clipboard.getImageAsync.mockResolvedValue({ data: pasted });
195
196 const result = await pasteComposerClipboard({ existingCount: 0 });
197
198 expect(manipulator.manipulate).toHaveBeenCalledWith(pasted);
199 const resize = manipulator.manipulate.mock.results.at(-1)?.value.resize;
200 expect(resize).toHaveBeenCalledWith({ width: 2000, height: 1299 });
201 expect(result.images).toEqual([
202 expect.objectContaining({
203 mimeType: "image/png",
204 dataUrl: "data:image/png;base64,cmVuZGVyZWQ=",
205 previewUri: "data:image/png;base64,cmVuZGVyZWQ=",
206 }),
207 ]);
208 });
209
210 it("keeps a 2000px clipboard image byte for byte without rendering it", async () => {
211 const pasted = pngDataUrl(2000, 1200);
212 clipboard.getImageAsync.mockResolvedValue({ data: pasted });
213
214 const result = await pasteComposerClipboard({ existingCount: 0 });
215
216 expect(manipulator.manipulate).not.toHaveBeenCalled();
217 expect(result.images).toEqual([expect.objectContaining({ dataUrl: pasted })]);
218 });
219});
220
131describe("native pasted image cleanup", () => {221describe("native pasted image cleanup", () => {
132 beforeEach(() => {222 beforeEach(() => {
133 files.clear();223 files.clear();
apps/mobile/src/lib/composerImages.ts+25-49
...@@ -20,6 +20,7 @@ import {...@@ -20,6 +20,7 @@ import {
20} from "./composerAttachmentFiles";20} from "./composerAttachmentFiles";
21import { imageMimeType } from "@t3tools/shared/image";21import { imageMimeType } from "@t3tools/shared/image";
22import { videoMimeType } from "@t3tools/shared/video";22import { videoMimeType } from "@t3tools/shared/video";
23import { fitComposerImage, renderPhoto } from "./composerImageFit";
23import { beginForegroundHandoff } from "./foreground-handoff";24import { beginForegroundHandoff } from "./foreground-handoff";
24import { uuidv4 } from "./uuid";25import { uuidv4 } from "./uuid";
25import { writeFileAtomically } from "./atomic-file";26import { writeFileAtomically } from "./atomic-file";
...@@ -341,46 +342,6 @@ export async function pickComposerFiles(input: {...@@ -341,46 +342,6 @@ export async function pickComposerFiles(input: {
341 return { files: attachments, error };342 return { files: attachments, error };
342}343}
343344
344/**
345 * Longest edge kept when a photo has to be re-encoded. Matches the web composer's
346 * MAX_DIMENSION so every client hands providers the same resolution.
347 */
348const PHOTO_MAX_EDGE = 2048;
349const PHOTO_JPEG_QUALITY = 0.85;
350
351/**
352 * Renders a photo-library pick to a provider-readable JPEG. Decode, downscale, and encode run
353 * natively; only the bounded result crosses the bridge. Camera photos are 12-48 MP HEIC files,
354 * so a full-size conversion is both slow to transfer and far more than a model can use.
355 */
356async function renderPhotoAsJpeg(uri: string): Promise<{ base64: string; uri: string }> {
357 const { ImageManipulator, SaveFormat } = await import("expo-image-manipulator");
358 let image = await ImageManipulator.manipulate(uri).renderAsync();
359 try {
360 const longestEdge = Math.max(image.width, image.height);
361 if (longestEdge > PHOTO_MAX_EDGE) {
362 const resized = await ImageManipulator.manipulate(image)
363 .resize(
364 image.width >= image.height ? { width: PHOTO_MAX_EDGE } : { height: PHOTO_MAX_EDGE },
365 )
366 .renderAsync();
367 image.release();
368 image = resized;
369 }
370 const saved = await image.saveAsync({
371 format: SaveFormat.JPEG,
372 compress: PHOTO_JPEG_QUALITY,
373 base64: true,
374 });
375 if (!saved.base64) {
376 throw new Error("The rendered photo has no bytes.");
377 }
378 return { base64: saved.base64, uri: saved.uri };
379 } finally {
380 image.release();
381 }
382}
383
384async function loadImagePicker() {345async function loadImagePicker() {
385 try {346 try {
386 return await import("expo-image-picker");347 return await import("expo-image-picker");
...@@ -530,14 +491,20 @@ export async function pickComposerMedia(input: {...@@ -530,14 +491,20 @@ export async function pickComposerMedia(input: {
530 try {491 try {
531 if (originalMimeType !== null) {492 if (originalMimeType !== null) {
532 const { File } = await import("expo-file-system");493 const { File } = await import("expo-file-system");
494 const base64 = await new File(asset.uri).base64();
495 const fitted = await fitComposerImage({
496 uri: asset.uri,
497 mimeType: originalMimeType,
498 base64,
499 });
533 image = {500 image = {
534 base64: await new File(asset.uri).base64(),501 base64: fitted?.base64 ?? base64,
535 mimeType: originalMimeType,502 mimeType: originalMimeType,
536 name,503 name,
537 previewUri: asset.uri,504 previewUri: fitted?.uri ?? asset.uri,
538 };505 };
539 } else {506 } else {
540 const rendered = await renderPhotoAsJpeg(asset.uri);507 const rendered = await renderPhoto(asset.uri);
541 image = {508 image = {
542 base64: rendered.base64,509 base64: rendered.base64,
543 mimeType: "image/jpeg",510 mimeType: "image/jpeg",
...@@ -616,7 +583,14 @@ export async function pasteComposerClipboard(input: { readonly existingCount: nu...@@ -616,7 +583,14 @@ export async function pasteComposerClipboard(input: { readonly existingCount: nu
616 };583 };
617 }584 }
618585
619 const base64 = image.data.split(",")[1] ?? "";586 const pastedBase64 = image.data.split(",")[1] ?? "";
587 const fitted = await fitComposerImage({
588 uri: image.data,
589 mimeType: "image/png",
590 base64: pastedBase64,
591 });
592 const base64 = fitted?.base64 ?? pastedBase64;
593 const dataUrl = fitted ? `data:image/png;base64,${base64}` : image.data;
620 const sizeBytes = estimateBase64ByteSize(base64);594 const sizeBytes = estimateBase64ByteSize(base64);
621 if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) {595 if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) {
622 return {596 return {
...@@ -634,8 +608,8 @@ export async function pasteComposerClipboard(input: { readonly existingCount: nu...@@ -634,8 +608,8 @@ export async function pasteComposerClipboard(input: { readonly existingCount: nu
634 name: "pasted-image.png",608 name: "pasted-image.png",
635 mimeType: "image/png",609 mimeType: "image/png",
636 sizeBytes,610 sizeBytes,
637 dataUrl: image.data,611 dataUrl,
638 previewUri: image.data,612 previewUri: dataUrl,
639 },613 },
640 ],614 ],
641 text: null,615 text: null,
...@@ -707,12 +681,14 @@ export async function convertPastedImagesToAttachments(input: {...@@ -707,12 +681,14 @@ export async function convertPastedImagesToAttachments(input: {
707 continue;681 continue;
708 }682 }
709 const file = new File(uri);683 const file = new File(uri);
710 const base64 = await file.base64();684 const mimeType = mimeTypeFromUri(uri);
685 const pastedBase64 = await file.base64();
686 const fitted = await fitComposerImage({ uri, mimeType, base64: pastedBase64 });
687 const base64 = fitted?.base64 ?? pastedBase64;
711 const sizeBytes = estimateBase64ByteSize(base64);688 const sizeBytes = estimateBase64ByteSize(base64);
712 if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) {689 if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) {
713 continue;690 continue;
714 }691 }
715 const mimeType = mimeTypeFromUri(uri);
716 results.push({692 results.push({
717 id: uuidv4(),693 id: uuidv4(),
718 type: "image",694 type: "image",
...@@ -720,7 +696,7 @@ export async function convertPastedImagesToAttachments(input: {...@@ -720,7 +696,7 @@ export async function convertPastedImagesToAttachments(input: {
720 mimeType,696 mimeType,
721 sizeBytes,697 sizeBytes,
722 dataUrl: `data:${mimeType};base64,${base64}`,698 dataUrl: `data:${mimeType};base64,${base64}`,
723 previewUri: ownedTemporaryFile ? `data:${mimeType};base64,${base64}` : uri,699 previewUri: ownedTemporaryFile ? `data:${mimeType};base64,${base64}` : (fitted?.uri ?? uri),
724 });700 });
725 } catch (error) {701 } catch (error) {
726 console.warn("Failed to read pasted image", uri, error);702 console.warn("Failed to read pasted image", uri, error);
apps/web/src/lib/imageCompression.test.ts+116-4
...@@ -598,6 +598,118 @@ describe("HEIC attachment preparation", () => {...@@ -598,6 +598,118 @@ describe("HEIC attachment preparation", () => {
598 });598 });
599});599});
600600
601describe("attachment max edge", () => {
602 const u32 = (n: number) => [(n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff];
603 const u16 = (n: number) => [(n >>> 8) & 0xff, n & 0xff];
604 const pngHeader = (width: number, height: number) => [
605 0x89,
606 0x50,
607 0x4e,
608 0x47,
609 0x0d,
610 0x0a,
611 0x1a,
612 0x0a,
613 ...u32(13),
614 0x49,
615 0x48,
616 0x44,
617 0x52,
618 ...u32(width),
619 ...u32(height),
620 8,
621 6,
622 0,
623 0,
624 0,
625 ];
626 const jpegHeader = (width: number, height: number) => [
627 0xff,
628 0xd8,
629 0xff,
630 0xc0,
631 ...u16(17),
632 8,
633 ...u16(height),
634 ...u16(width),
635 ];
636 const image = (header: number[], name: string, type: string) =>
637 new File([new Uint8Array([...header, ...new Uint8Array(2048).fill(7)])], name, { type });
638
639 /** Records the decoded size, the canvas sizes, and the encode calls. */
640 function stubDecoder(bitmapSize: { width: number; height: number }) {
641 const decode = vi.fn(async () => ({ ...bitmapSize, close: vi.fn() }));
642 const canvases: Array<{ width: number; height: number }> = [];
643 const encodes: Array<{ type: string; quality: number }> = [];
644 const fillRect = vi.fn();
645 vi.stubGlobal("createImageBitmap", decode);
646 vi.stubGlobal(
647 "OffscreenCanvas",
648 class {
649 constructor(width: number, height: number) {
650 canvases.push({ width, height });
651 }
652 getContext() {
653 return { fillRect, drawImage: vi.fn() };
654 }
655 async convertToBlob(options: { type: string; quality: number }) {
656 encodes.push(options);
657 return new Blob([new Uint8Array(500_000)], { type: options.type });
658 }
659 },
660 );
661 return { decode, canvases, encodes, fillRect };
662 }
663
664 it("scales a large PNG screenshot under the byte cap to 2000px and keeps it a PNG", async () => {
665 const { decode, canvases, encodes, fillRect } = stubDecoder({ width: 3024, height: 1964 });
666 const original = image(pngHeader(3024, 1964), "shot.png", "image/png");
667
668 const result = await prepareImageForAttachment(original, 10 * 1024 * 1024);
669
670 expect(decode).toHaveBeenCalledExactlyOnceWith(original, { imageOrientation: "from-image" });
671 expect(canvases).toEqual([{ width: 2000, height: 1299 }]);
672 expect(encodes.map(({ type }) => type)).toEqual(["image/png"]);
673 // PNG keeps alpha, so nothing paints a matte behind it.
674 expect(fillRect).not.toHaveBeenCalled();
675 expect(result.ok && result.recompressed).toBe(true);
676 expect(result.ok && result.imageSize).toEqual({ width: 2000, height: 1299 });
677 expect(result.ok && result.file.name).toBe("shot.png");
678 expect(result.ok && result.file.type).toBe("image/png");
679 });
680
681 it("scales a rotated JPEG by its displayed size and keeps it a JPEG", async () => {
682 // Stored landscape, displayed portrait after the EXIF rotation.
683 const { canvases, encodes } = stubDecoder({ width: 3024, height: 4032 });
684 const original = image(jpegHeader(4032, 3024), "photo.jpg", "image/jpeg");
685
686 const result = await prepareImageForAttachment(original, 10 * 1024 * 1024);
687
688 expect(canvases).toEqual([{ width: 1500, height: 2000 }]);
689 expect(encodes).toEqual([{ type: "image/jpeg", quality: 0.92 }]);
690 expect(result.ok && result.file.type).toBe("image/jpeg");
691 expect(result.ok && result.imageSize).toEqual({ width: 1500, height: 2000 });
692 });
693
694 it.each([
695 { label: "a 2000px PNG", header: pngHeader(2000, 1200), type: "image/png" },
696 {
697 label: "a 3000px GIF",
698 header: [0x47, 0x49, 0x46, 0x38, 0x39, 0x61, 0xb8, 0x0b, 0x10, 0x04],
699 type: "image/gif",
700 },
701 ])("passes $label through without decoding it", async ({ header, type }) => {
702 const { decode } = stubDecoder({ width: 3000, height: 1040 });
703 const original = image(header, "image", type);
704
705 const result = await prepareImageForAttachment(original, 10 * 1024 * 1024);
706
707 expect(result.ok && result.file).toBe(original);
708 expect(result.ok && result.recompressed).toBe(false);
709 expect(decode).not.toHaveBeenCalled();
710 });
711});
712
601describe("snapshot coordinates after compression", () => {713describe("snapshot coordinates after compression", () => {
602 const source: SnapShotSource = {714 const source: SnapShotSource = {
603 kind: "snap-shot",715 kind: "snap-shot",
...@@ -649,7 +761,7 @@ describe("snapshot coordinates after compression", () => {...@@ -649,7 +761,7 @@ describe("snapshot coordinates after compression", () => {
649 sizeBytes: compressed.file.size,761 sizeBytes: compressed.file.size,
650 dataUrl: `data:${compressed.file.type};base64,${Buffer.from(await compressed.file.arrayBuffer()).toString("base64")}`,762 dataUrl: `data:${compressed.file.type};base64,${Buffer.from(await compressed.file.arrayBuffer()).toString("base64")}`,
651 };763 };
652 expect(image.imageSize).toEqual({ width: 2048, height: 1280 });764 expect(image.imageSize).toEqual({ width: 2000, height: 1250 });
653 const resized = resizeSnapShotSource(source, image.imageSize);765 const resized = resizeSnapShotSource(source, image.imageSize);
654 const [restored] = hydrateImagesFromPersisted([766 const [restored] = hydrateImagesFromPersisted([
655 {767 {
...@@ -662,10 +774,10 @@ describe("snapshot coordinates after compression", () => {...@@ -662,10 +774,10 @@ describe("snapshot coordinates after compression", () => {
662 },774 },
663 ]);775 ]);
664 expect(restored?.source?.accessibility).toMatchObject({776 expect(restored?.source?.accessibility).toMatchObject({
665 imageSize: { width: 2048, height: 1280 },777 imageSize: { width: 2000, height: 1250 },
666 root: {778 root: {
667 bounds: { x: 0, y: 0, width: 2048, height: 1280 },779 bounds: { x: 0, y: 0, width: 2000, height: 1250 },
668 children: [{ bounds: { x: 1920, y: 1120, width: 80, height: 80 } }, { bounds: null }],780 children: [{ bounds: { x: 1875, y: 1094, width: 78, height: 78 } }, { bounds: null }],
669 },781 },
670 });782 });
671 expect(source.accessibility).toMatchObject({ imageSize: { width: 2560, height: 1600 } });783 expect(source.accessibility).toMatchObject({ imageSize: { width: 2560, height: 1600 } });
apps/web/src/lib/imageCompression.ts+65-11
...@@ -8,15 +8,24 @@...@@ -8,15 +8,24 @@
8 * `PROVIDER_SEND_TURN_MAX_IMAGE_BYTES` wire cap and shrinks them to fit8 * `PROVIDER_SEND_TURN_MAX_IMAGE_BYTES` wire cap and shrinks them to fit
9 * via `compressImageToByteLimit` instead of rejecting the paste.9 * via `compressImageToByteLimit` instead of rejecting the paste.
10 *10 *
11 * Supported images already within budget pass through untouched. HEIC/HEIF11 * Supported images already within budget pass through untouched, except that
12 * photos are decoded to JPEG first because providers cannot consume them.12 * a PNG or JPEG longer than `IMAGE_ATTACHMENT_MAX_EDGE` on a side is scaled down
13 * to it in its own format before upload. HEIC/HEIF photos are decoded to JPEG
14 * first because providers cannot consume them.
13 */15 */
1416
15/**17import {
16 * Longest edge kept when an image has to be re-encoded. Sized so a typical18 downscalableImageMimeType,
17 * retina screenshot (3024px wide) stays legible rather than being halved.19 fitImageAttachmentSize,
18 */20 IMAGE_ATTACHMENT_MAX_EDGE,
19const MAX_DIMENSION = 2048;21} from "@t3tools/client-runtime/image-attachment-size";
22import {
23 IMAGE_DIMENSIONS_HEADER_BYTES,
24 readImageDimensions,
25} from "@t3tools/shared/imageDimensions";
26
27/** Longest edge kept when an image has to be re-encoded. */
28const MAX_DIMENSION = IMAGE_ATTACHMENT_MAX_EDGE;
20/** Base64 budget for a single stashed image (~975KB of binary). */29/** Base64 budget for a single stashed image (~975KB of binary). */
21export const MAX_STASH_IMAGE_DATA_URL_CHARS = 1_300_000;30export const MAX_STASH_IMAGE_DATA_URL_CHARS = 1_300_000;
22/**31/**
...@@ -423,10 +432,47 @@ export async function compressImageForStash(...@@ -423,10 +432,47 @@ export async function compressImageForStash(
423 }432 }
424}433}
425434
435/**
436 * Scales a PNG or JPEG whose long edge is over `IMAGE_ATTACHMENT_MAX_EDGE`
437 * down to that edge in its own format, so a PNG keeps its transparency.
438 * Returns null when the image already fits, is another format (GIF, WebP,
439 * SVG), or cannot be decoded or encoded here; the caller keeps the original.
440 */
441async function fitImageToMaxEdge(file: File): Promise<{ file: File; imageSize: ImageSize } | null> {
442 const mimeType = downscalableImageMimeType(file.type);
443 if (!mimeType || !canRecompress() || file.size > MAX_COMPRESSIBLE_SOURCE_BYTES) return null;
444 let bitmap: ImageBitmap | undefined;
445 try {
446 // The header answers "does it fit?" without decoding every attachment.
447 const header = readImageDimensions(
448 new Uint8Array(await file.slice(0, IMAGE_DIMENSIONS_HEADER_BYTES).arrayBuffer()),
449 );
450 if (!header || !fitImageAttachmentSize(header)) return null;
451 bitmap = await createImageBitmap(file, { imageOrientation: "from-image" });
452 const imageSize = fitImageAttachmentSize(bitmap);
453 if (!imageSize) return null;
454 const target = createCanvas(imageSize.width, imageSize.height);
455 if (!target) return null;
456 target.context.imageSmoothingQuality = "high";
457 target.context.drawImage(bitmap, 0, 0, imageSize.width, imageSize.height);
458 const blob = await encodeCanvas(target.canvas, QUALITY_STEPS[0], mimeType);
459 if (!blob) return null;
460 return {
461 file: new File([blob], file.name, { type: mimeType, lastModified: file.lastModified }),
462 imageSize,
463 };
464 } catch {
465 return null;
466 } finally {
467 bitmap?.close();
468 }
469}
470
426/**471/**
427 * Shrinks `file` until its binary size fits `maxBytes`, returning a new472 * Shrinks `file` until its binary size fits `maxBytes`, returning a new
428 * `File` (WebP or JPEG). Files already within the limit pass through473 * `File` (WebP or JPEG). A PNG or JPEG over `IMAGE_ATTACHMENT_MAX_EDGE` is
429 * untouched, preserving their exact bytes and format. Sources above474 * first scaled to that edge in its own format. Other files already within the
475 * limit pass through untouched, preserving their exact bytes and format. Sources above
430 * `MAX_COMPRESSIBLE_SOURCE_BYTES` are refused outright — decoding them is476 * `MAX_COMPRESSIBLE_SOURCE_BYTES` are refused outright — decoding them is
431 * the risk, so no amount of output budget makes them safe. An internally477 * the risk, so no amount of output budget makes them safe. An internally
432 * converted image can provide its original source size when the intermediate478 * converted image can provide its original source size when the intermediate
...@@ -437,7 +483,11 @@ export async function compressImageToByteLimit(...@@ -437,7 +483,11 @@ export async function compressImageToByteLimit(
437 maxBytes: number,483 maxBytes: number,
438 options?: { preferredMimeType?: "image/jpeg"; sourceSizeBytes?: number },484 options?: { preferredMimeType?: "image/jpeg"; sourceSizeBytes?: number },
439): Promise<CompressImageFileResult> {485): Promise<CompressImageFileResult> {
440 if (file.size <= maxBytes) {486 const fitted = await fitImageToMaxEdge(file);
487 if (fitted && fitted.file.size <= maxBytes) {
488 return { ok: true, file: fitted.file, recompressed: true, imageSize: fitted.imageSize };
489 }
490 if (!fitted && file.size <= maxBytes) {
441 return { ok: true, file, recompressed: false };491 return { ok: true, file, recompressed: false };
442 }492 }
443 if ((options?.sourceSizeBytes ?? file.size) > MAX_COMPRESSIBLE_SOURCE_BYTES) {493 if ((options?.sourceSizeBytes ?? file.size) > MAX_COMPRESSIBLE_SOURCE_BYTES) {
...@@ -447,7 +497,11 @@ export async function compressImageToByteLimit(...@@ -447,7 +497,11 @@ export async function compressImageToByteLimit(
447 // into 4 chars; flooring keeps the budget a hair conservative instead of497 // into 4 chars; flooring keeps the budget a hair conservative instead of
448 // admitting an encoding right at the byte cap.498 // admitting an encoding right at the byte cap.
449 const budgetChars = Math.floor(maxBytes / 3) * 4;499 const budgetChars = Math.floor(maxBytes / 3) * 4;
450 const reencoded = await reencodeWithinBudget(file, budgetChars, options?.preferredMimeType);500 const reencoded = await reencodeWithinBudget(
501 fitted?.file ?? file,
502 budgetChars,
503 options?.preferredMimeType,
504 );
451 if (!reencoded.ok) {505 if (!reencoded.ok) {
452 return reencoded;506 return reencoded;
453 }507 }
packages/client-runtime/package.json+4
...@@ -7,6 +7,10 @@...@@ -7,6 +7,10 @@
7 "types": "./src/composerThreadItems.ts",7 "types": "./src/composerThreadItems.ts",
8 "default": "./src/composerThreadItems.ts"8 "default": "./src/composerThreadItems.ts"
9 },9 },
10 "./image-attachment-size": {
11 "types": "./src/imageAttachmentSize.ts",
12 "default": "./src/imageAttachmentSize.ts"
13 },
10 "./handoff": {14 "./handoff": {
11 "types": "./src/handoff.ts",15 "types": "./src/handoff.ts",
12 "default": "./src/handoff.ts"16 "default": "./src/handoff.ts"
packages/client-runtime/src/imageAttachmentSize.ts created+34
...@@ -0,0 +1,34 @@
1/**
2 * Longest edge, in pixels, a PNG or JPEG attachment keeps. Anthropic rejects an image over
3 * 2000px on a side once a request holds more than 20 images, and Claude Code forwards images
4 * sent into a running turn unchanged, so one large paste breaks every later request in that
5 * thread. Web and mobile scale attachments down to this edge when they are attached.
6 */
7export const IMAGE_ATTACHMENT_MAX_EDGE = 2000;
8
9/** Formats a client re-encodes in place. GIF, WebP, and SVG are left as attached. */
10export type DownscalableImageMimeType = "image/png" | "image/jpeg";
11
12export function downscalableImageMimeType(mimeType: string): DownscalableImageMimeType | null {
13 const normalized = mimeType.toLowerCase();
14 if (normalized === "image/png") return "image/png";
15 if (normalized === "image/jpeg" || normalized === "image/jpg") return "image/jpeg";
16 return null;
17}
18
19/**
20 * The size an image should be scaled to so its long edge fits `IMAGE_ATTACHMENT_MAX_EDGE`,
21 * keeping its aspect ratio. Returns null when it already fits.
22 */
23export function fitImageAttachmentSize(size: {
24 readonly width: number;
25 readonly height: number;
26}): { width: number; height: number } | null {
27 const longestEdge = Math.max(size.width, size.height);
28 if (longestEdge <= IMAGE_ATTACHMENT_MAX_EDGE) return null;
29 const scale = IMAGE_ATTACHMENT_MAX_EDGE / longestEdge;
30 return {
31 width: Math.max(1, Math.min(IMAGE_ATTACHMENT_MAX_EDGE, Math.round(size.width * scale))),
32 height: Math.max(1, Math.min(IMAGE_ATTACHMENT_MAX_EDGE, Math.round(size.height * scale))),
33 };
34}