authorgravatar for nadagrava@gmail.comNelglor <nadagrava@gmail.com> 2026-09-12 14:40:48-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-12 11:40:48-07:00
logbbedad0278bbf753503184c00e0c09a0eab6679c
tree246f96b01914049d0491ab0f3650d64c9eb37ebd
parenta43f9b45ae85caf37e0be8270ad3d27365ece2bd
signature Signed by PGP key B5690EEEBB952194

fix(mobile): render photo library picks to a bounded JPEG off the JS thread (#11440)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>

5 files changed, 246 insertions(+), 65 deletions(-)

apps/mobile/package.json+1
......@@ -91,6 +91,7 @@
9191 "expo-glass-effect": "~57.0.1",
9292 "expo-haptics": "~57.0.2",
9393 "expo-image": "~57.0.3",
94 "expo-image-manipulator": "~57.0.17",
9495 "expo-image-picker": "~57.0.14",
9596 "expo-linking": "~57.0.8",
9697 "expo-network": "~57.0.1",
apps/mobile/src/lib/composerFiles.test.ts+137-24
......@@ -11,6 +11,8 @@ const mocks = vi.hoisted(() => ({
1111 open: vi.fn(),
1212 size: vi.fn(),
1313 readBase64: vi.fn(),
14 manipulate: vi.fn(),
15 release: vi.fn(),
1416}));
1517
1618vi.mock("expo-file-system", () => {
......@@ -80,6 +82,10 @@ vi.mock("expo-file-system", () => {
8082
8183vi.mock("expo-image-picker", () => ({ launchImageLibraryAsync: mocks.pickMedia }));
8284vi.mock("expo-document-picker", () => ({ getDocumentAsync: mocks.pickFile }));
85vi.mock("expo-image-manipulator", () => ({
86 SaveFormat: { JPEG: "jpeg", PNG: "png", WEBP: "webp" },
87 ImageManipulator: { manipulate: mocks.manipulate },
88}));
8389vi.mock("./uuid", () => ({ uuidv4: () => "attachment-id" }));
8490
8591import {
......@@ -106,20 +112,50 @@ describe("composer file attachments", () => {
106112 });
107113
108114 describe("photo library image conversion", () => {
109 const jpeg = "/9j/2Q==";
115 const rendered = { uri: "file:///cache/ImageManipulator/photo.jpg", base64: "/9j/2Q==" };
110116 const photo: ImagePickerAsset = {
111117 uri: "file:///picker/photo.heic",
112118 type: "image",
113119 fileName: "photo.HEIC",
114120 mimeType: "image/heic",
115 fileSize: 20 * 1024 * 1024,
116 base64: jpeg,
117 width: 1,
118 height: 1,
121 fileSize: 4 * 1024 * 1024,
122 width: 4032,
123 height: 3024,
124 };
125 /**
126 * Stands in for the native manipulator: `size` is what decoding the source yields,
127 * `resizes` records every requested resize, and `saved` is what saving returns.
128 */
129 const native = {
130 size: { width: 1, height: 1 },
131 resizes: [] as Array<{ width?: number | null; height?: number | null }>,
132 saved: rendered as { uri: string; base64?: string },
119133 };
120134
135 beforeEach(() => {
136 native.size = { width: 1, height: 1 };
137 native.resizes = [];
138 native.saved = rendered;
139 mocks.manipulate.mockReset();
140 mocks.release.mockReset();
141 mocks.manipulate.mockImplementation(() => {
142 const context = {
143 resize(size: { width?: number | null; height?: number | null }) {
144 native.resizes.push(size);
145 return context;
146 },
147 renderAsync: async () => ({
148 ...native.size,
149 release: mocks.release,
150 saveAsync: async () => native.saved,
151 }),
152 };
153 return context;
154 });
155 });
156
121157 it.each(["image/heic", "image/heif", undefined])(
122 "attaches the native JPEG conversion with matching metadata when the source MIME is %s",
158 "renders a %s photo to JPEG natively and previews the rendered file",
123159 async (mimeType) => {
124160 mocks.pickMedia.mockResolvedValue({
125161 canceled: false,
......@@ -128,6 +164,9 @@ describe("composer file attachments", () => {
128164
129165 const result = await pickComposerImages({ existingCount: 0 });
130166
167 expect(mocks.pickMedia).toHaveBeenCalledWith(expect.objectContaining({ base64: false }));
168 expect(mocks.manipulate).toHaveBeenCalledWith(photo.uri);
169 expect(mocks.readBase64).not.toHaveBeenCalled();
131170 expect(result).toEqual({
132171 images: [
133172 {
......@@ -136,8 +175,8 @@ describe("composer file attachments", () => {
136175 name: "photo.jpg",
137176 mimeType: "image/jpeg",
138177 sizeBytes: 4,
139 dataUrl: `data:image/jpeg;base64,${jpeg}`,
140 previewUri: `data:image/jpeg;base64,${jpeg}`,
178 dataUrl: `data:image/jpeg;base64,${rendered.base64}`,
179 previewUri: rendered.uri,
141180 },
142181 ],
143182 error: null,
......@@ -145,11 +184,26 @@ describe("composer file attachments", () => {
145184 },
146185 );
147186
187 it.each([
188 { size: { width: 4032, height: 3024 }, resizes: [{ width: 2048 }] },
189 { size: { width: 3024, height: 4032 }, resizes: [{ height: 2048 }] },
190 { size: { width: 2048, height: 1536 }, resizes: [] },
191 ])("bounds a $size.width x $size.height photo to a 2048 px longest edge", async (input) => {
192 native.size = input.size;
193 mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [photo] });
194
195 await pickComposerImages({ existingCount: 0 });
196
197 expect(native.resizes).toEqual(input.resizes);
198 // Every decoded bitmap is released, including the full-size one a resize replaces.
199 expect(mocks.release).toHaveBeenCalledTimes(input.resizes.length + 1);
200 });
201
148202 it.each([
149203 { extension: "png", mimeType: "image/png", base64: "iVBORw0KGgo=" },
150204 { extension: "gif", mimeType: "image/gif", base64: "R0lGODlh" },
151205 { extension: "webp", mimeType: "image/webp", base64: "UklGRgQAAABXRUJQ" },
152 ])("preserves original $extension bytes instead of the picker's JPEG", async (original) => {
206 ])("keeps original $extension bytes from the picker file", async (original) => {
153207 const name = `photo.${original.extension}`;
154208 mocks.pickMedia.mockResolvedValue({
155209 canceled: false,
......@@ -159,6 +213,8 @@ describe("composer file attachments", () => {
159213
160214 const result = await pickComposerImages({ existingCount: 0 });
161215
216 expect(mocks.manipulate).not.toHaveBeenCalled();
217 expect(mocks.readBase64).toHaveBeenCalledWith(photo.uri);
162218 expect(result.error).toBeNull();
163219 expect(result.images).toEqual([
164220 expect.objectContaining({
......@@ -166,37 +222,74 @@ describe("composer file attachments", () => {
166222 mimeType: original.mimeType,
167223 dataUrl: `data:${original.mimeType};base64,${original.base64}`,
168224 sizeBytes: Buffer.from(original.base64, "base64").byteLength,
225 previewUri: photo.uri,
169226 }),
170227 ]);
171228 });
172229
173 it("checks the converted JPEG size even when the HEIC source was smaller", async () => {
174 const oversized =
175 jpeg.slice(0, 4) + "A".repeat(Math.ceil(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES / 3) * 4);
230 it("renders a supported original that exceeds the image limit instead of rejecting it", async () => {
176231 mocks.pickMedia.mockResolvedValue({
177232 canceled: false,
178 assets: [{ ...photo, fileSize: 42, base64: oversized }],
233 assets: [{ ...photo, fileName: "photo.jpg", mimeType: "image/jpeg" }],
179234 });
235 mocks.size.mockReturnValue(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES + 1);
180236
181 await expect(pickComposerImages({ existingCount: 0 })).resolves.toEqual({
182 images: [],
183 error: "'photo.HEIC' exceeds the 10 MB attachment limit.",
237 const result = await pickComposerImages({ existingCount: 0 });
238
239 expect(mocks.manipulate).toHaveBeenCalledWith(photo.uri);
240 expect(result.error).toBeNull();
241 expect(result.images).toEqual([
242 expect.objectContaining({ name: "photo.jpg", mimeType: "image/jpeg", sizeBytes: 4 }),
243 ]);
244 });
245
246 it("measures the picker file instead of trusting the reported size", async () => {
247 // A content stream can deliver more bytes than the picker advertises; a supported
248 // original only skips rendering when the file itself measures within the limit.
249 mocks.pickMedia.mockResolvedValue({
250 canceled: false,
251 assets: [{ ...photo, fileName: "photo.png", mimeType: "image/png", fileSize: 42 }],
184252 });
253 mocks.size.mockReturnValue(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES + 1);
254
255 const result = await pickComposerImages({ existingCount: 0 });
256
257 expect(mocks.readBase64).not.toHaveBeenCalled();
258 expect(mocks.manipulate).toHaveBeenCalledWith(photo.uri);
259 expect(result.images).toEqual([expect.objectContaining({ mimeType: "image/jpeg" })]);
185260 });
186261
187 it("does not relabel unconverted HEIC bytes as JPEG", async () => {
262 it("renders a supported original whose size cannot be measured", async () => {
188263 mocks.pickMedia.mockResolvedValue({
189264 canceled: false,
190 assets: [{ ...photo, base64: "AAAAGGZ0eXBoZWlj" }],
265 assets: [
266 { ...photo, uri: "content://media/1", fileName: "photo.png", mimeType: "image/png" },
267 ],
191268 });
192269
193270 const result = await pickComposerImages({ existingCount: 0 });
194271
195 expect(result.images).toEqual([]);
196 expect(result.error).toContain("not a supported image type");
272 expect(mocks.readBase64).not.toHaveBeenCalled();
273 expect(mocks.manipulate).toHaveBeenCalledWith("content://media/1");
274 expect(result.images).toEqual([expect.objectContaining({ mimeType: "image/jpeg" })]);
197275 });
198276
199 it("retains a converted photo when another original cannot be read", async () => {
277 it("checks the rendered JPEG against the image limit", async () => {
278 native.saved = {
279 uri: rendered.uri,
280 base64:
281 rendered.base64.slice(0, 4) +
282 "A".repeat(Math.ceil(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES / 3) * 4),
283 };
284 mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [photo] });
285
286 await expect(pickComposerImages({ existingCount: 0 })).resolves.toEqual({
287 images: [],
288 error: "'photo.HEIC' exceeds the 10 MB attachment limit.",
289 });
290 });
291
292 it("retains a rendered photo when another original cannot be read", async () => {
200293 mocks.pickMedia.mockResolvedValue({
201294 canceled: false,
202295 assets: [{ ...photo, fileName: "missing.gif", mimeType: "image/gif" }, photo],
......@@ -208,6 +301,21 @@ describe("composer file attachments", () => {
208301 expect(result.images).toEqual([expect.objectContaining({ name: "photo.jpg" })]);
209302 expect(result.error).toBe("Failed to read 'missing.gif'.");
210303 });
304
305 it("reports a photo the native renderer cannot decode", async () => {
306 mocks.manipulate.mockImplementation(() => ({
307 resize: () => {
308 throw new Error("unreachable");
309 },
310 renderAsync: () => Promise.reject(new Error("corrupt")),
311 }));
312 mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [photo] });
313
314 await expect(pickComposerImages({ existingCount: 0 })).resolves.toEqual({
315 images: [],
316 error: "Failed to read 'photo.HEIC'.",
317 });
318 });
211319 });
212320
213321 describe("photo library videos", () => {
......@@ -217,10 +325,13 @@ describe("composer file attachments", () => {
217325 fileName: "photo.png",
218326 mimeType: "image/png",
219327 fileSize: 3,
220 base64: "YWJj",
221328 width: 1,
222329 height: 1,
223330 };
331
332 beforeEach(() => {
333 mocks.readBase64.mockResolvedValue("YWJj");
334 });
224335 const video: ImagePickerAsset = {
225336 uri: "file:///picker/clip.mov",
226337 type: "video",
......@@ -234,7 +345,9 @@ describe("composer file attachments", () => {
234345
235346 it("retains mixed photos and videos, keeping video bytes in durable file storage", async () => {
236347 mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [image, video] });
237 mocks.size.mockReturnValue(video.fileSize);
348 mocks.size.mockImplementation((uri: string) =>
349 uri.endsWith("clip.mov") ? video.fileSize : 3,
350 );
238351
239352 const result = await pickComposerMedia({ existingCount: 0, maxVideoBytes: 50 * 1024 * 1024 });
240353
......@@ -345,7 +458,7 @@ describe("composer file attachments", () => {
345458 canceled: false,
346459 assets: [{ ...video, fileSize: reported }, image],
347460 });
348 mocks.size.mockReturnValue(stored);
461 mocks.size.mockImplementation((uri: string) => (uri.endsWith("clip.mov") ? stored : 3));
349462
350463 const result = await pickComposerMedia({ existingCount: 0, maxVideoBytes: limit });
351464
apps/mobile/src/lib/composerImages.ts+92-38
......@@ -307,6 +307,46 @@ export async function pickComposerFiles(input: {
307307 return { files: attachments, error };
308308}
309309
310/**
311 * Longest edge kept when a photo has to be re-encoded. Matches the web composer's
312 * MAX_DIMENSION so every client hands providers the same resolution.
313 */
314const PHOTO_MAX_EDGE = 2048;
315const PHOTO_JPEG_QUALITY = 0.85;
316
317/**
318 * Renders a photo-library pick to a provider-readable JPEG. Decode, downscale, and encode run
319 * natively; only the bounded result crosses the bridge. Camera photos are 12-48 MP HEIC files,
320 * so a full-size conversion is both slow to transfer and far more than a model can use.
321 */
322async function renderPhotoAsJpeg(uri: string): Promise<{ base64: string; uri: string }> {
323 const { ImageManipulator, SaveFormat } = await import("expo-image-manipulator");
324 let image = await ImageManipulator.manipulate(uri).renderAsync();
325 try {
326 const longestEdge = Math.max(image.width, image.height);
327 if (longestEdge > PHOTO_MAX_EDGE) {
328 const resized = await ImageManipulator.manipulate(image)
329 .resize(
330 image.width >= image.height ? { width: PHOTO_MAX_EDGE } : { height: PHOTO_MAX_EDGE },
331 )
332 .renderAsync();
333 image.release();
334 image = resized;
335 }
336 const saved = await image.saveAsync({
337 format: SaveFormat.JPEG,
338 compress: PHOTO_JPEG_QUALITY,
339 base64: true,
340 });
341 if (!saved.base64) {
342 throw new Error("The rendered photo has no bytes.");
343 }
344 return { base64: saved.base64, uri: saved.uri };
345 } finally {
346 image.release();
347 }
348}
349
310350async function loadImagePicker() {
311351 try {
312352 return await import("expo-image-picker");
......@@ -369,7 +409,10 @@ export async function pickComposerMedia(input: {
369409 mediaTypes: input.maxVideoBytes === undefined ? ["images"] : ["images", "videos"],
370410 allowsMultipleSelection: true,
371411 selectionLimit: remainingSlots,
372 base64: true,
412 // Bytes stay in the picker's file until we know how much of them we need. Asking for
413 // base64 here made iOS decode and re-encode every camera photo at full resolution and
414 // hand JS a 10 MB+ string, which stalled the composer for seconds.
415 base64: false,
373416 quality: 1,
374417 shouldDownloadFromNetwork: true,
375418 });
......@@ -397,7 +440,7 @@ export async function pickComposerMedia(input: {
397440 error = `You can attach up to ${PROVIDER_SEND_TURN_MAX_ATTACHMENTS} attachments per message.`;
398441 break;
399442 }
400 let mimeType = asset.mimeType?.toLowerCase();
443 const mimeType = asset.mimeType?.toLowerCase();
401444 if (asset.type === "video" || mimeType?.startsWith("video/")) {
402445 if (input.maxVideoBytes === undefined) {
403446 error = "Video attachments are unavailable here.";
......@@ -426,56 +469,67 @@ export async function pickComposerMedia(input: {
426469 continue;
427470 }
428471
429 let base64 = asset.base64;
430 if (!base64) {
431 error = `Failed to read '${asset.fileName ?? "image"}'.`;
432 continue;
472 const name = asset.fileName?.trim() || "image";
473 // The picker's reported size is a hint, not a measurement: Android content streams can
474 // deliver more bytes than they advertise. Only a size read from the file itself decides
475 // whether the original bytes are safe to load into JS.
476 let sourceBytes: number | null = null;
477 try {
478 const { File } = await import("expo-file-system");
479 sourceBytes = new File(asset.uri).size;
480 } catch {
481 sourceBytes = null;
433482 }
434
435 let name = asset.fileName?.trim() || "image";
436 // The iOS picker returns JPEG base64 even when its metadata describes HEIC,
437 // PNG, or GIF. Keep supported originals so transparency and animation survive;
438 // use the native JPEG conversion for formats providers cannot accept.
439 if (base64.startsWith("/9j/")) {
440 if (
441 mimeType &&
442 mimeType !== "image/jpeg" &&
443 isProviderSendTurnSupportedImageMimeType(mimeType)
444 ) {
445 try {
446 const { File } = await import("expo-file-system");
447 base64 = await new File(asset.uri).base64();
448 } catch {
449 error = `Failed to read '${name}'.`;
450 continue;
451 }
483 // Originals the provider can read and that fit the cap pass through byte for byte so
484 // transparency and animation survive. Everything else (HEIC/HEIF, oversized JPEGs,
485 // unmeasurable sources) is rendered to a bounded JPEG off the JS thread.
486 const originalMimeType =
487 mimeType !== undefined &&
488 isProviderSendTurnSupportedImageMimeType(mimeType) &&
489 sourceBytes !== null &&
490 sourceBytes > 0 &&
491 sourceBytes <= PROVIDER_SEND_TURN_MAX_IMAGE_BYTES
492 ? mimeType
493 : null;
494
495 let image: { base64: string; mimeType: string; name: string; previewUri: string };
496 try {
497 if (originalMimeType !== null) {
498 const { File } = await import("expo-file-system");
499 image = {
500 base64: await new File(asset.uri).base64(),
501 mimeType: originalMimeType,
502 name,
503 previewUri: asset.uri,
504 };
452505 } else {
453 mimeType = "image/jpeg";
454 if (!/\.jpe?g$/i.test(name)) {
455 name = `${name.replace(/\.[^.]+$/, "")}.jpg`;
456 }
506 const rendered = await renderPhotoAsJpeg(asset.uri);
507 image = {
508 base64: rendered.base64,
509 mimeType: "image/jpeg",
510 name: /\.jpe?g$/i.test(name) ? name : `${name.replace(/\.[^.]+$/, "")}.jpg`,
511 previewUri: rendered.uri,
512 };
457513 }
458 }
459 if (!mimeType || !isProviderSendTurnSupportedImageMimeType(mimeType)) {
460 error = `'${name}' is not a supported image type. Attach GIF, JPEG, PNG, or WebP images.`;
514 } catch {
515 error = `Failed to read '${name}'.`;
461516 continue;
462517 }
463518
464 const sizeBytes = estimateBase64ByteSize(base64);
519 const sizeBytes = estimateBase64ByteSize(image.base64);
465520 if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) {
466 error = `'${asset.fileName ?? "image"}' exceeds the 10 MB attachment limit.`;
521 error = `'${name}' exceeds the 10 MB attachment limit.`;
467522 continue;
468523 }
469524
470 const dataUrl = `data:${mimeType};base64,${base64}`;
471525 attachments.push({
472526 id: uuidv4(),
473527 type: "image",
474 name,
475 mimeType,
528 name: image.name,
529 mimeType: image.mimeType,
476530 sizeBytes,
477 dataUrl,
478 previewUri: mimeType === asset.mimeType?.toLowerCase() ? asset.uri : dataUrl,
531 dataUrl: `data:${image.mimeType};base64,${image.base64}`,
532 previewUri: image.previewUri,
479533 });
480534 }
481535
docs/user/composer.md+3-3
......@@ -17,9 +17,9 @@ message can send. Retry or remove a failed upload. On web and desktop, reloading
1717before an upload finishes requires you to attach that file again.
1818
1919You can drag or paste images into the web or desktop composer. HEIC and HEIF
20photos are converted to JPEG there and when selected from the iOS photo library;
21the image limit applies after conversion. On mobile, you can also send files to
22T3 Code through another app's system share sheet.
20photos are converted to JPEG there and when selected from the mobile photo
21library; photos over the image limit are also resized to fit. On mobile, you can
22also send files to T3 Code through another app's system share sheet.
2323
2424See [images and videos](#images-and-videos-in-messages) for previewing and saving media.
2525
pnpm-lock.yaml+13
......@@ -368,6 +368,9 @@ importers:
368368 expo-image:
369369 specifier: ~57.0.3
370370 version: 57.0.3(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)
371 expo-image-manipulator:
372 specifier: ~57.0.17
373 version: 57.0.17(expo@57.0.18)
371374 expo-image-picker:
372375 specifier: ~57.0.14
373376 version: 57.0.14(expo@57.0.18)
......@@ -7443,6 +7446,11 @@ packages:
74437446 peerDependencies:
74447447 expo: '*'
74457448
7449 expo-image-manipulator@57.0.17:
7450 resolution: {integrity: sha512-VpM8qAotTeSIobIAL8RAIDrZKL0jQBK/6O41NnwmdT96sd2wT263DlTbkvQ6RkcFrR9+NZiC5Um7FDBK+jFneg==}
7451 peerDependencies:
7452 expo: '*'
7453
74467454 expo-image-picker@57.0.14:
74477455 resolution: {integrity: sha512-NK9XBQqOtscbB/uRts1Gm7Oki6odMN3FQPWD6fSKmNUfKM69O6kcgU25lEXFTSLZ7sLl+AopHa2X0uILd8rjaQ==}
74487456 peerDependencies:
......@@ -17957,6 +17965,11 @@ snapshots:
1795717965 dependencies:
1795817966 expo: 57.0.18(fc5a731e35a0144aab60c7305f29cbed)
1795917967
17968 expo-image-manipulator@57.0.17(expo@57.0.18):
17969 dependencies:
17970 expo: 57.0.18(fc5a731e35a0144aab60c7305f29cbed)
17971 expo-image-loader: 57.0.1(expo@57.0.18)
17972
1796017973 expo-image-picker@57.0.14(expo@57.0.18):
1796117974 dependencies:
1796217975 expo: 57.0.18(fc5a731e35a0144aab60c7305f29cbed)