diff --git a/apps/mobile/package.json b/apps/mobile/package.json index 461e7edf56cf3fb5d6d5efdb3227503c9d014fb6..0fd35faf7528966fd9322c553516eaea0e4b3a3b 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -91,6 +91,7 @@ "expo-glass-effect": "~57.0.1", "expo-haptics": "~57.0.2", "expo-image": "~57.0.3", + "expo-image-manipulator": "~57.0.17", "expo-image-picker": "~57.0.14", "expo-linking": "~57.0.8", "expo-network": "~57.0.1", diff --git a/apps/mobile/src/lib/composerFiles.test.ts b/apps/mobile/src/lib/composerFiles.test.ts index b38c0813c6a1b1dd82ad34bf1300bdc5293f0013..962be35013f2d1ed60bdad4ded7f0877df71eb83 100644 --- a/apps/mobile/src/lib/composerFiles.test.ts +++ b/apps/mobile/src/lib/composerFiles.test.ts @@ -11,6 +11,8 @@ const mocks = vi.hoisted(() => ({ open: vi.fn(), size: vi.fn(), readBase64: vi.fn(), + manipulate: vi.fn(), + release: vi.fn(), })); vi.mock("expo-file-system", () => { @@ -80,6 +82,10 @@ vi.mock("expo-file-system", () => { vi.mock("expo-image-picker", () => ({ launchImageLibraryAsync: mocks.pickMedia })); vi.mock("expo-document-picker", () => ({ getDocumentAsync: mocks.pickFile })); +vi.mock("expo-image-manipulator", () => ({ + SaveFormat: { JPEG: "jpeg", PNG: "png", WEBP: "webp" }, + ImageManipulator: { manipulate: mocks.manipulate }, +})); vi.mock("./uuid", () => ({ uuidv4: () => "attachment-id" })); import { @@ -106,20 +112,50 @@ describe("composer file attachments", () => { }); describe("photo library image conversion", () => { - const jpeg = "/9j/2Q=="; + const rendered = { uri: "file:///cache/ImageManipulator/photo.jpg", base64: "/9j/2Q==" }; const photo: ImagePickerAsset = { uri: "file:///picker/photo.heic", type: "image", fileName: "photo.HEIC", mimeType: "image/heic", - fileSize: 20 * 1024 * 1024, - base64: jpeg, - width: 1, - height: 1, + fileSize: 4 * 1024 * 1024, + width: 4032, + height: 3024, }; + /** + * Stands in for the native manipulator: `size` is what decoding the source yields, + * `resizes` records every requested resize, and `saved` is what saving returns. + */ + const native = { + size: { width: 1, height: 1 }, + resizes: [] as Array<{ width?: number | null; height?: number | null }>, + saved: rendered as { uri: string; base64?: string }, + }; + + beforeEach(() => { + native.size = { width: 1, height: 1 }; + native.resizes = []; + native.saved = rendered; + mocks.manipulate.mockReset(); + mocks.release.mockReset(); + mocks.manipulate.mockImplementation(() => { + const context = { + resize(size: { width?: number | null; height?: number | null }) { + native.resizes.push(size); + return context; + }, + renderAsync: async () => ({ + ...native.size, + release: mocks.release, + saveAsync: async () => native.saved, + }), + }; + return context; + }); + }); it.each(["image/heic", "image/heif", undefined])( - "attaches the native JPEG conversion with matching metadata when the source MIME is %s", + "renders a %s photo to JPEG natively and previews the rendered file", async (mimeType) => { mocks.pickMedia.mockResolvedValue({ canceled: false, @@ -128,6 +164,9 @@ describe("composer file attachments", () => { const result = await pickComposerImages({ existingCount: 0 }); + expect(mocks.pickMedia).toHaveBeenCalledWith(expect.objectContaining({ base64: false })); + expect(mocks.manipulate).toHaveBeenCalledWith(photo.uri); + expect(mocks.readBase64).not.toHaveBeenCalled(); expect(result).toEqual({ images: [ { @@ -136,8 +175,8 @@ describe("composer file attachments", () => { name: "photo.jpg", mimeType: "image/jpeg", sizeBytes: 4, - dataUrl: `data:image/jpeg;base64,${jpeg}`, - previewUri: `data:image/jpeg;base64,${jpeg}`, + dataUrl: `data:image/jpeg;base64,${rendered.base64}`, + previewUri: rendered.uri, }, ], error: null, @@ -145,11 +184,26 @@ describe("composer file attachments", () => { }, ); + it.each([ + { size: { width: 4032, height: 3024 }, resizes: [{ width: 2048 }] }, + { size: { width: 3024, height: 4032 }, resizes: [{ height: 2048 }] }, + { size: { width: 2048, height: 1536 }, resizes: [] }, + ])("bounds a $size.width x $size.height photo to a 2048 px longest edge", async (input) => { + native.size = input.size; + mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [photo] }); + + await pickComposerImages({ existingCount: 0 }); + + expect(native.resizes).toEqual(input.resizes); + // Every decoded bitmap is released, including the full-size one a resize replaces. + expect(mocks.release).toHaveBeenCalledTimes(input.resizes.length + 1); + }); + it.each([ { extension: "png", mimeType: "image/png", base64: "iVBORw0KGgo=" }, { extension: "gif", mimeType: "image/gif", base64: "R0lGODlh" }, { extension: "webp", mimeType: "image/webp", base64: "UklGRgQAAABXRUJQ" }, - ])("preserves original $extension bytes instead of the picker's JPEG", async (original) => { + ])("keeps original $extension bytes from the picker file", async (original) => { const name = `photo.${original.extension}`; mocks.pickMedia.mockResolvedValue({ canceled: false, @@ -159,6 +213,8 @@ describe("composer file attachments", () => { const result = await pickComposerImages({ existingCount: 0 }); + expect(mocks.manipulate).not.toHaveBeenCalled(); + expect(mocks.readBase64).toHaveBeenCalledWith(photo.uri); expect(result.error).toBeNull(); expect(result.images).toEqual([ expect.objectContaining({ @@ -166,17 +222,66 @@ describe("composer file attachments", () => { mimeType: original.mimeType, dataUrl: `data:${original.mimeType};base64,${original.base64}`, sizeBytes: Buffer.from(original.base64, "base64").byteLength, + previewUri: photo.uri, }), ]); }); - it("checks the converted JPEG size even when the HEIC source was smaller", async () => { - const oversized = - jpeg.slice(0, 4) + "A".repeat(Math.ceil(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES / 3) * 4); + it("renders a supported original that exceeds the image limit instead of rejecting it", async () => { mocks.pickMedia.mockResolvedValue({ canceled: false, - assets: [{ ...photo, fileSize: 42, base64: oversized }], + assets: [{ ...photo, fileName: "photo.jpg", mimeType: "image/jpeg" }], }); + mocks.size.mockReturnValue(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES + 1); + + const result = await pickComposerImages({ existingCount: 0 }); + + expect(mocks.manipulate).toHaveBeenCalledWith(photo.uri); + expect(result.error).toBeNull(); + expect(result.images).toEqual([ + expect.objectContaining({ name: "photo.jpg", mimeType: "image/jpeg", sizeBytes: 4 }), + ]); + }); + + it("measures the picker file instead of trusting the reported size", async () => { + // A content stream can deliver more bytes than the picker advertises; a supported + // original only skips rendering when the file itself measures within the limit. + mocks.pickMedia.mockResolvedValue({ + canceled: false, + assets: [{ ...photo, fileName: "photo.png", mimeType: "image/png", fileSize: 42 }], + }); + mocks.size.mockReturnValue(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES + 1); + + const result = await pickComposerImages({ existingCount: 0 }); + + expect(mocks.readBase64).not.toHaveBeenCalled(); + expect(mocks.manipulate).toHaveBeenCalledWith(photo.uri); + expect(result.images).toEqual([expect.objectContaining({ mimeType: "image/jpeg" })]); + }); + + it("renders a supported original whose size cannot be measured", async () => { + mocks.pickMedia.mockResolvedValue({ + canceled: false, + assets: [ + { ...photo, uri: "content://media/1", fileName: "photo.png", mimeType: "image/png" }, + ], + }); + + const result = await pickComposerImages({ existingCount: 0 }); + + expect(mocks.readBase64).not.toHaveBeenCalled(); + expect(mocks.manipulate).toHaveBeenCalledWith("content://media/1"); + expect(result.images).toEqual([expect.objectContaining({ mimeType: "image/jpeg" })]); + }); + + it("checks the rendered JPEG against the image limit", async () => { + native.saved = { + uri: rendered.uri, + base64: + rendered.base64.slice(0, 4) + + "A".repeat(Math.ceil(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES / 3) * 4), + }; + mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [photo] }); await expect(pickComposerImages({ existingCount: 0 })).resolves.toEqual({ images: [], @@ -184,19 +289,7 @@ describe("composer file attachments", () => { }); }); - it("does not relabel unconverted HEIC bytes as JPEG", async () => { - mocks.pickMedia.mockResolvedValue({ - canceled: false, - assets: [{ ...photo, base64: "AAAAGGZ0eXBoZWlj" }], - }); - - const result = await pickComposerImages({ existingCount: 0 }); - - expect(result.images).toEqual([]); - expect(result.error).toContain("not a supported image type"); - }); - - it("retains a converted photo when another original cannot be read", async () => { + it("retains a rendered photo when another original cannot be read", async () => { mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [{ ...photo, fileName: "missing.gif", mimeType: "image/gif" }, photo], @@ -208,6 +301,21 @@ describe("composer file attachments", () => { expect(result.images).toEqual([expect.objectContaining({ name: "photo.jpg" })]); expect(result.error).toBe("Failed to read 'missing.gif'."); }); + + it("reports a photo the native renderer cannot decode", async () => { + mocks.manipulate.mockImplementation(() => ({ + resize: () => { + throw new Error("unreachable"); + }, + renderAsync: () => Promise.reject(new Error("corrupt")), + })); + mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [photo] }); + + await expect(pickComposerImages({ existingCount: 0 })).resolves.toEqual({ + images: [], + error: "Failed to read 'photo.HEIC'.", + }); + }); }); describe("photo library videos", () => { @@ -217,10 +325,13 @@ describe("composer file attachments", () => { fileName: "photo.png", mimeType: "image/png", fileSize: 3, - base64: "YWJj", width: 1, height: 1, }; + + beforeEach(() => { + mocks.readBase64.mockResolvedValue("YWJj"); + }); const video: ImagePickerAsset = { uri: "file:///picker/clip.mov", type: "video", @@ -234,7 +345,9 @@ describe("composer file attachments", () => { it("retains mixed photos and videos, keeping video bytes in durable file storage", async () => { mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [image, video] }); - mocks.size.mockReturnValue(video.fileSize); + mocks.size.mockImplementation((uri: string) => + uri.endsWith("clip.mov") ? video.fileSize : 3, + ); const result = await pickComposerMedia({ existingCount: 0, maxVideoBytes: 50 * 1024 * 1024 }); @@ -345,7 +458,7 @@ describe("composer file attachments", () => { canceled: false, assets: [{ ...video, fileSize: reported }, image], }); - mocks.size.mockReturnValue(stored); + mocks.size.mockImplementation((uri: string) => (uri.endsWith("clip.mov") ? stored : 3)); const result = await pickComposerMedia({ existingCount: 0, maxVideoBytes: limit }); diff --git a/apps/mobile/src/lib/composerImages.ts b/apps/mobile/src/lib/composerImages.ts index 5a45825c3a4324cbc8cfb3aa88fddfd152ebc7ed..86658db8dc3bc1580c60fc805f3a6f4abc146436 100644 --- a/apps/mobile/src/lib/composerImages.ts +++ b/apps/mobile/src/lib/composerImages.ts @@ -307,6 +307,46 @@ export async function pickComposerFiles(input: { return { files: attachments, error }; } +/** + * Longest edge kept when a photo has to be re-encoded. Matches the web composer's + * MAX_DIMENSION so every client hands providers the same resolution. + */ +const PHOTO_MAX_EDGE = 2048; +const PHOTO_JPEG_QUALITY = 0.85; + +/** + * Renders a photo-library pick to a provider-readable JPEG. Decode, downscale, and encode run + * natively; only the bounded result crosses the bridge. Camera photos are 12-48 MP HEIC files, + * so a full-size conversion is both slow to transfer and far more than a model can use. + */ +async function renderPhotoAsJpeg(uri: string): Promise<{ base64: string; uri: string }> { + const { ImageManipulator, SaveFormat } = await import("expo-image-manipulator"); + let image = await ImageManipulator.manipulate(uri).renderAsync(); + try { + const longestEdge = Math.max(image.width, image.height); + if (longestEdge > PHOTO_MAX_EDGE) { + const resized = await ImageManipulator.manipulate(image) + .resize( + image.width >= image.height ? { width: PHOTO_MAX_EDGE } : { height: PHOTO_MAX_EDGE }, + ) + .renderAsync(); + image.release(); + image = resized; + } + const saved = await image.saveAsync({ + format: SaveFormat.JPEG, + compress: PHOTO_JPEG_QUALITY, + base64: true, + }); + if (!saved.base64) { + throw new Error("The rendered photo has no bytes."); + } + return { base64: saved.base64, uri: saved.uri }; + } finally { + image.release(); + } +} + async function loadImagePicker() { try { return await import("expo-image-picker"); @@ -369,7 +409,10 @@ export async function pickComposerMedia(input: { mediaTypes: input.maxVideoBytes === undefined ? ["images"] : ["images", "videos"], allowsMultipleSelection: true, selectionLimit: remainingSlots, - base64: true, + // Bytes stay in the picker's file until we know how much of them we need. Asking for + // base64 here made iOS decode and re-encode every camera photo at full resolution and + // hand JS a 10 MB+ string, which stalled the composer for seconds. + base64: false, quality: 1, shouldDownloadFromNetwork: true, }); @@ -397,7 +440,7 @@ export async function pickComposerMedia(input: { error = `You can attach up to ${PROVIDER_SEND_TURN_MAX_ATTACHMENTS} attachments per message.`; break; } - let mimeType = asset.mimeType?.toLowerCase(); + const mimeType = asset.mimeType?.toLowerCase(); if (asset.type === "video" || mimeType?.startsWith("video/")) { if (input.maxVideoBytes === undefined) { error = "Video attachments are unavailable here."; @@ -426,56 +469,67 @@ export async function pickComposerMedia(input: { continue; } - let base64 = asset.base64; - if (!base64) { - error = `Failed to read '${asset.fileName ?? "image"}'.`; - continue; + const name = asset.fileName?.trim() || "image"; + // The picker's reported size is a hint, not a measurement: Android content streams can + // deliver more bytes than they advertise. Only a size read from the file itself decides + // whether the original bytes are safe to load into JS. + let sourceBytes: number | null = null; + try { + const { File } = await import("expo-file-system"); + sourceBytes = new File(asset.uri).size; + } catch { + sourceBytes = null; } + // Originals the provider can read and that fit the cap pass through byte for byte so + // transparency and animation survive. Everything else (HEIC/HEIF, oversized JPEGs, + // unmeasurable sources) is rendered to a bounded JPEG off the JS thread. + const originalMimeType = + mimeType !== undefined && + isProviderSendTurnSupportedImageMimeType(mimeType) && + sourceBytes !== null && + sourceBytes > 0 && + sourceBytes <= PROVIDER_SEND_TURN_MAX_IMAGE_BYTES + ? mimeType + : null; - let name = asset.fileName?.trim() || "image"; - // The iOS picker returns JPEG base64 even when its metadata describes HEIC, - // PNG, or GIF. Keep supported originals so transparency and animation survive; - // use the native JPEG conversion for formats providers cannot accept. - if (base64.startsWith("/9j/")) { - if ( - mimeType && - mimeType !== "image/jpeg" && - isProviderSendTurnSupportedImageMimeType(mimeType) - ) { - try { - const { File } = await import("expo-file-system"); - base64 = await new File(asset.uri).base64(); - } catch { - error = `Failed to read '${name}'.`; - continue; - } + let image: { base64: string; mimeType: string; name: string; previewUri: string }; + try { + if (originalMimeType !== null) { + const { File } = await import("expo-file-system"); + image = { + base64: await new File(asset.uri).base64(), + mimeType: originalMimeType, + name, + previewUri: asset.uri, + }; } else { - mimeType = "image/jpeg"; - if (!/\.jpe?g$/i.test(name)) { - name = `${name.replace(/\.[^.]+$/, "")}.jpg`; - } + const rendered = await renderPhotoAsJpeg(asset.uri); + image = { + base64: rendered.base64, + mimeType: "image/jpeg", + name: /\.jpe?g$/i.test(name) ? name : `${name.replace(/\.[^.]+$/, "")}.jpg`, + previewUri: rendered.uri, + }; } - } - if (!mimeType || !isProviderSendTurnSupportedImageMimeType(mimeType)) { - error = `'${name}' is not a supported image type. Attach GIF, JPEG, PNG, or WebP images.`; + } catch { + error = `Failed to read '${name}'.`; continue; } - const sizeBytes = estimateBase64ByteSize(base64); + const sizeBytes = estimateBase64ByteSize(image.base64); if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) { - error = `'${asset.fileName ?? "image"}' exceeds the 10 MB attachment limit.`; + error = `'${name}' exceeds the 10 MB attachment limit.`; continue; } - const dataUrl = `data:${mimeType};base64,${base64}`; attachments.push({ id: uuidv4(), type: "image", - name, - mimeType, + name: image.name, + mimeType: image.mimeType, sizeBytes, - dataUrl, - previewUri: mimeType === asset.mimeType?.toLowerCase() ? asset.uri : dataUrl, + dataUrl: `data:${image.mimeType};base64,${image.base64}`, + previewUri: image.previewUri, }); } diff --git a/docs/user/composer.md b/docs/user/composer.md index a49b147a30af2fa642a2eb36e926b02ea48db1cc..186bbf034aa0c60b0131260a0c73a7a858c2a71e 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -17,9 +17,9 @@ message can send. Retry or remove a failed upload. On web and desktop, reloading before an upload finishes requires you to attach that file again. You can drag or paste images into the web or desktop composer. HEIC and HEIF -photos are converted to JPEG there and when selected from the iOS photo library; -the image limit applies after conversion. On mobile, you can also send files to -T3 Code through another app's system share sheet. +photos are converted to JPEG there and when selected from the mobile photo +library; photos over the image limit are also resized to fit. On mobile, you can +also send files to T3 Code through another app's system share sheet. See [images and videos](#images-and-videos-in-messages) for previewing and saving media. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1e33a5d03964623c634e53bf2e1d7af7bf83f669..a8dbaf39692448ac6ffda2c252ee57fb52e3dadd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -368,6 +368,9 @@ importers: expo-image: specifier: ~57.0.3 version: 57.0.3(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3) + expo-image-manipulator: + specifier: ~57.0.17 + version: 57.0.17(expo@57.0.18) expo-image-picker: specifier: ~57.0.14 version: 57.0.14(expo@57.0.18) @@ -7443,6 +7446,11 @@ packages: peerDependencies: expo: '*' + expo-image-manipulator@57.0.17: + resolution: {integrity: sha512-VpM8qAotTeSIobIAL8RAIDrZKL0jQBK/6O41NnwmdT96sd2wT263DlTbkvQ6RkcFrR9+NZiC5Um7FDBK+jFneg==} + peerDependencies: + expo: '*' + expo-image-picker@57.0.14: resolution: {integrity: sha512-NK9XBQqOtscbB/uRts1Gm7Oki6odMN3FQPWD6fSKmNUfKM69O6kcgU25lEXFTSLZ7sLl+AopHa2X0uILd8rjaQ==} peerDependencies: @@ -17957,6 +17965,11 @@ snapshots: dependencies: expo: 57.0.18(fc5a731e35a0144aab60c7305f29cbed) + expo-image-manipulator@57.0.17(expo@57.0.18): + dependencies: + expo: 57.0.18(fc5a731e35a0144aab60c7305f29cbed) + expo-image-loader: 57.0.1(expo@57.0.18) + expo-image-picker@57.0.14(expo@57.0.18): dependencies: expo: 57.0.18(fc5a731e35a0144aab60c7305f29cbed)