From bd9ed2b4bbda3dd6e468df1cb06233e29c4a9f5c Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 24 Aug 2026 22:00:28 -0700 Subject: [PATCH] feat(web): safely attach HEIC photos as JPEG images (#8161) Co-authored-by: mweinbach --- apps/web/package.json | 1 + apps/web/src/components/chat/ChatComposer.tsx | 22 +- apps/web/src/lib/imageCompression.test.ts | 195 ++++++++++++++++++ apps/web/src/lib/imageCompression.ts | 152 +++++++++++++- docs/user/composer.md | 3 + pnpm-lock.yaml | 8 + 6 files changed, 367 insertions(+), 14 deletions(-) diff --git a/apps/web/package.json b/apps/web/package.json index 598feaec0ce94b473296b0423832eeda5a64b700..ea6b00effb3b59b249230447a9fd2fd62bc3ef7a 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -34,6 +34,7 @@ "class-variance-authority": "^0.7.1", "culori": "^4.0.2", "effect": "catalog:", + "heic-to": "^1.5.2", "jose": "catalog:", "jsonc-parser": "3.3.1", "jszip": "3.10.1", diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index 1ec58e0de702937e0f8c028c16dba3a472af9ec1..841bbb8f40112f15a19dcd13111d61fa37c420af 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -73,7 +73,11 @@ import { type ComposerTaskStep, type ComposerTasksProgress, } from "./ComposerTasksBadge"; -import { compressImageForStash, compressImageToByteLimit } from "../../lib/imageCompression"; +import { + compressImageForStash, + isHeicImageFile, + prepareImageForAttachment, +} from "../../lib/imageCompression"; import { releaseAttachmentUpload, retryAttachmentUpload, @@ -2537,12 +2541,13 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) const acceptedFiles: File[] = []; let error: string | null = null; for (const file of files) { - if (!file.type.startsWith("image/")) { + const isHeicImage = isHeicImageFile(file); + if (!file.type.startsWith("image/") && !isHeicImage) { error = `Unsupported file type for '${file.name}'. Please attach image files only.`; continue; } - if (!isProviderSendTurnSupportedImageMimeType(file.type)) { - error = `'${file.name}' is not a supported image type. Attach GIF, JPEG, PNG, or WebP images.`; + if (!isHeicImage && !isProviderSendTurnSupportedImageMimeType(file.type)) { + error = `'${file.name}' is not a supported image type. Attach GIF, HEIC, HEIF, JPEG, PNG, or WebP images.`; continue; } if (reservedCount >= PROVIDER_SEND_TURN_MAX_ATTACHMENTS) { @@ -2562,7 +2567,10 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) for (const file of acceptedFiles) { // Images over the wire cap are downscaled to fit rather than // refused; files already within it pass through byte-for-byte. - const compressed = await compressImageToByteLimit(file, PROVIDER_SEND_TURN_MAX_IMAGE_BYTES); + const compressed = await prepareImageForAttachment( + file, + PROVIDER_SEND_TURN_MAX_IMAGE_BYTES, + ); if (!compressed.ok) { compressionError = compressed.reason === "unreadable" @@ -2615,7 +2623,9 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) const onComposerPaste = (event: React.ClipboardEvent) => { const files = Array.from(event.clipboardData.files); if (files.length === 0) return; - const imageFiles = files.filter((file) => file.type.startsWith("image/")); + const imageFiles = files.filter( + (file) => file.type.startsWith("image/") || isHeicImageFile(file), + ); if (imageFiles.length === 0) return; event.preventDefault(); void addComposerImages(imageFiles); diff --git a/apps/web/src/lib/imageCompression.test.ts b/apps/web/src/lib/imageCompression.test.ts index 63712ca7e2950c75d747bb7865bdefb04d5b8811..435c51ef86ee8af86522f89b6b192176e6dcc8ba 100644 --- a/apps/web/src/lib/imageCompression.test.ts +++ b/apps/web/src/lib/imageCompression.test.ts @@ -3,10 +3,20 @@ import { afterEach, describe, expect, it, vi } from "vite-plus/test"; import { compressImageForStash, compressImageToByteLimit, + isHeicImageFile, MAX_COMPRESSIBLE_SOURCE_BYTES, MAX_STASH_IMAGE_DATA_URL_CHARS, + prepareImageForAttachment, } from "./imageCompression"; +const mocks = vi.hoisted(() => ({ + heicTo: vi.fn(), +})); + +vi.mock("heic-to/csp", () => ({ + heicTo: mocks.heicTo, +})); + /** * jsdom has no real canvas/codec, so the re-encode path is exercised with * stubbed `createImageBitmap` + `OffscreenCanvas`. The encoder stub returns a @@ -22,6 +32,45 @@ function makeFile(sizeBytes: number, type = "image/png"): File { return new File([new Uint8Array(sizeBytes).fill(7)], "shot.png", { type }); } +function makeHeicFile(options?: { + name?: string; + type?: string; + width?: number; + height?: number; + lastModified?: number; +}): File { + const encoder = new TextEncoder(); + const makeBox = (name: string, ...contents: Uint8Array[]) => { + const bytes = new Uint8Array(8 + contents.reduce((size, content) => size + content.length, 0)); + new DataView(bytes.buffer).setUint32(0, bytes.length); + bytes.set(encoder.encode(name), 4); + let offset = 8; + for (const content of contents) { + bytes.set(content, offset); + offset += content.length; + } + return bytes; + }; + + const dimensions = new Uint8Array(12); + const view = new DataView(dimensions.buffer); + view.setUint32(4, options?.width ?? 4000); + view.setUint32(8, options?.height ?? 3000); + const properties = makeBox("iprp", makeBox("ipco", makeBox("ispe", dimensions))); + + return new File( + [ + makeBox("ftyp", encoder.encode("heic"), new Uint8Array(4)), + makeBox("meta", new Uint8Array(4), properties), + ], + options?.name ?? "photo.heic", + { + type: options?.type ?? "image/heic", + ...(options?.lastModified !== undefined ? { lastModified: options.lastModified } : {}), + }, + ); +} + /** * Installs a fake bitmap + canvas whose encoded size follows `sizeForQuality`. * `supportsWebp: false` makes `convertToBlob` hand back a differently-typed @@ -63,6 +112,7 @@ function stubCanvasPipeline( } afterEach(() => { + mocks.heicTo.mockReset(); vi.unstubAllGlobals(); globalThis.createImageBitmap = originalCreateImageBitmap; globalThis.OffscreenCanvas = originalOffscreenCanvas; @@ -251,3 +301,148 @@ describe("compressImageForStash", () => { expect(smallestRequested).toBeLessThan(800); }); }); + +describe("HEIC attachment preparation", () => { + it("recognizes HEIC and HEIF MIME types and case-insensitive file extensions", () => { + expect(isHeicImageFile({ name: "photo.bin", type: "image/heic" })).toBe(true); + expect(isHeicImageFile({ name: "photo.bin", type: "image/heif" })).toBe(true); + expect(isHeicImageFile({ name: "photo.heic", type: "image/heic-sequence" })).toBe(false); + expect(isHeicImageFile({ name: "photo.heif", type: "image/heif-sequence" })).toBe(false); + expect(isHeicImageFile({ name: "IMG_1234.HEIC", type: "" })).toBe(true); + expect(isHeicImageFile({ name: "photo.heif", type: "application/octet-stream" })).toBe(true); + expect(isHeicImageFile({ name: "photo.png", type: "image/png" })).toBe(false); + expect(isHeicImageFile({ name: "photo.heic", type: "image/png" })).toBe(false); + expect(isHeicImageFile({ name: "photo.heif", type: "image/jpeg" })).toBe(false); + }); + + it("converts a HEIC photo with a missing MIME type into a named JPEG", async () => { + const original = makeHeicFile({ + name: "IMG_1234.HEIC", + type: "", + lastModified: 123, + }); + mocks.heicTo.mockResolvedValueOnce( + new Blob([new Uint8Array([4, 5, 6, 7])], { type: "image/jpeg" }), + ); + + const result = await prepareImageForAttachment(original, 1024); + + expect(mocks.heicTo).toHaveBeenCalledWith({ + blob: original, + type: "image/jpeg", + quality: 0.92, + }); + expect(result.ok && result.file.name).toBe("IMG_1234.jpg"); + expect(result.ok && result.file.type).toBe("image/jpeg"); + expect(result.ok && result.file.size).toBe(4); + expect(result.ok && result.file.lastModified).toBe(123); + expect(result.ok && result.recompressed).toBe(true); + }); + + it("keeps oversized converted photos in JPEG format while shrinking them", async () => { + const original = makeHeicFile({ + name: "photo.heif", + type: "image/heif", + }); + mocks.heicTo.mockResolvedValueOnce( + new Blob([new Uint8Array(2_000_000)], { type: "image/jpeg" }), + ); + const { fillRect } = stubCanvasPipeline(() => 200_000); + + const result = await prepareImageForAttachment(original, 1_000_000); + + expect(result.ok && result.file.name).toBe("photo.jpg"); + expect(result.ok && result.file.type).toBe("image/jpeg"); + expect(result.ok && result.file.size).toBeLessThanOrEqual(1_000_000); + expect(fillRect).toHaveBeenCalled(); + }); + + it("compresses JPEG intermediates above the source safety ceiling", async () => { + const original = makeHeicFile({ + name: "large.heic", + type: "image/heic", + }); + mocks.heicTo.mockResolvedValueOnce( + new Blob([new Uint8Array(MAX_COMPRESSIBLE_SOURCE_BYTES + 1)], { + type: "image/jpeg", + }), + ); + const { close } = stubCanvasPipeline(() => 200_000); + + const result = await prepareImageForAttachment(original, 1_000_000); + + expect(result.ok && result.file.name).toBe("large.jpg"); + expect(result.ok && result.file.type).toBe("image/jpeg"); + expect(result.ok && result.file.size).toBeLessThanOrEqual(1_000_000); + expect(close).toHaveBeenCalled(); + }); + + it.each([ + { label: "24 MP", width: 5712, height: 4284 }, + { label: "48 MP", width: 8064, height: 6048 }, + ])("accepts $label HEIC photos", async ({ width, height }) => { + const original = makeHeicFile({ width, height }); + mocks.heicTo.mockResolvedValueOnce(new Blob(["jpeg"], { type: "image/jpeg" })); + + const result = await prepareImageForAttachment(original, 1024); + + expect(result.ok && result.file.type).toBe("image/jpeg"); + expect(mocks.heicTo).toHaveBeenCalledOnce(); + }); + + it("rejects oversized HEIC dimensions before loading the decoder", async () => { + const original = makeHeicFile({ width: 16_000, height: 4001 }); + + expect(await prepareImageForAttachment(original, 1024)).toEqual({ + ok: false, + reason: "too-large", + }); + expect(mocks.heicTo).not.toHaveBeenCalled(); + }); + + it("rejects invalid HEIC metadata before loading the decoder", async () => { + const original = new File([new Uint8Array([1, 2, 3])], "broken.heic", { + type: "image/heic", + }); + + expect(await prepareImageForAttachment(original, 1024)).toEqual({ + ok: false, + reason: "unreadable", + }); + expect(mocks.heicTo).not.toHaveBeenCalled(); + }); + + it("reports unreadable when HEIC decoding fails", async () => { + const original = makeHeicFile({ + name: "broken.heic", + type: "image/heic", + }); + mocks.heicTo.mockRejectedValueOnce(new Error("Invalid HEIC image")); + + expect(await prepareImageForAttachment(original, 1024)).toEqual({ + ok: false, + reason: "unreadable", + }); + }); + + it("rejects unsafe HEIC sources before loading the decoder", async () => { + const original = new File(["photo"], "large.heic", { type: "image/heic" }); + Object.defineProperty(original, "size", { value: MAX_COMPRESSIBLE_SOURCE_BYTES + 1 }); + + expect(await prepareImageForAttachment(original, 1024)).toEqual({ + ok: false, + reason: "too-large", + }); + expect(mocks.heicTo).not.toHaveBeenCalled(); + }); + + it("leaves supported images untouched without loading the HEIC decoder", async () => { + const original = makeFile(1024); + + const result = await prepareImageForAttachment(original, 2048); + + expect(result.ok && result.file).toBe(original); + expect(result.ok && result.recompressed).toBe(false); + expect(mocks.heicTo).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/src/lib/imageCompression.ts b/apps/web/src/lib/imageCompression.ts index be45024f38c4d4ad2020641677ba29a78c14d66b..c699886c44ebe8a3d22a743b0d5ec4ff7ab737bb 100644 --- a/apps/web/src/lib/imageCompression.ts +++ b/apps/web/src/lib/imageCompression.ts @@ -8,7 +8,8 @@ * `PROVIDER_SEND_TURN_MAX_IMAGE_BYTES` wire cap and shrinks them to fit * via `compressImageToByteLimit` instead of rejecting the paste. * - * Images already within budget pass through untouched. + * Supported images already within budget pass through untouched. HEIC/HEIF + * photos are decoded to JPEG first because providers cannot consume them. */ /** @@ -24,6 +25,8 @@ export const MAX_STASH_IMAGE_DATA_URL_CHARS = 1_300_000; * ImageBitmap can OOM the tab — beyond this we refuse rather than risk it. */ export const MAX_COMPRESSIBLE_SOURCE_BYTES = 50 * 1024 * 1024; +const MAX_HEIC_DECODE_PIXELS = 64_000_000; +const MAX_HEIC_METADATA_BYTES = 1024 * 1024; /** * Quality ladder tried in order until the encoded image fits the budget. * The floor stays high enough to avoid visible blocking on UI screenshots; @@ -32,6 +35,8 @@ export const MAX_COMPRESSIBLE_SOURCE_BYTES = 50 * 1024 * 1024; const QUALITY_STEPS = [0.92, 0.85, 0.78, 0.68] as const; /** Extra downscale passes applied when even the lowest quality overflows. */ const FALLBACK_SCALE_STEPS = [0.75, 0.55] as const; +const HEIC_IMAGE_MIME_TYPE = /^image\/hei(?:c|f)$/i; +const HEIC_IMAGE_EXTENSION = /\.(?:heic|heif)$/i; export interface CompressedStashImage { dataUrl: string; @@ -55,6 +60,86 @@ export type CompressImageFileResult = | { ok: true; file: File; recompressed: boolean } | { ok: false; reason: ImageCompressionFailureReason }; +/** Finder and some browsers omit the MIME type when dragging HEIC photos. */ +export function isHeicImageFile(file: Pick): boolean { + if (HEIC_IMAGE_MIME_TYPE.test(file.type)) { + return true; + } + return ( + (file.type === "" || file.type.toLowerCase() === "application/octet-stream") && + HEIC_IMAGE_EXTENSION.test(file.name) + ); +} + +interface HeicMetadataBox { + payloadOffset: number; + endOffset: number; +} + +function findHeicMetadataBox( + view: DataView, + startOffset: number, + endOffset: number, + type: number, +): HeicMetadataBox | null { + let offset = startOffset; + while (offset + 8 <= endOffset) { + let size = view.getUint32(offset); + let headerSize = 8; + if (size === 1) { + if (offset + 16 > endOffset) return null; + const extendedSize = view.getBigUint64(offset + 8); + if (extendedSize > BigInt(Number.MAX_SAFE_INTEGER)) return null; + size = Number(extendedSize); + headerSize = 16; + } else if (size === 0) { + size = endOffset - offset; + } + if (size < headerSize || size > endOffset - offset) return null; + + const nextOffset = offset + size; + if (view.getUint32(offset + 4) === type) { + return { payloadOffset: offset + headerSize, endOffset: nextOffset }; + } + offset = nextOffset; + } + return null; +} + +/** Read HEIC image dimensions before the decoder allocates full RGBA buffers. */ +async function validateHeicImageDimensions( + file: File, +): Promise { + const metadata = await file.slice(0, MAX_HEIC_METADATA_BYTES).arrayBuffer(); + const view = new DataView(metadata); + const meta = findHeicMetadataBox(view, 0, view.byteLength, 0x6d657461); + if (!meta || meta.payloadOffset + 4 > meta.endOffset) return "unreadable"; + const properties = findHeicMetadataBox(view, meta.payloadOffset + 4, meta.endOffset, 0x69707270); + if (!properties) return "unreadable"; + const containers = findHeicMetadataBox( + view, + properties.payloadOffset, + properties.endOffset, + 0x6970636f, + ); + if (!containers) return "unreadable"; + + let offset = containers.payloadOffset; + let foundImageDimensions = false; + while (offset < containers.endOffset) { + const image = findHeicMetadataBox(view, offset, containers.endOffset, 0x69737065); + if (!image) break; + if (image.payloadOffset + 12 > image.endOffset) return "unreadable"; + const width = view.getUint32(image.payloadOffset + 4); + const height = view.getUint32(image.payloadOffset + 8); + if (width === 0 || height === 0) return "unreadable"; + if (width > MAX_HEIC_DECODE_PIXELS / height) return "too-large"; + foundImageDimensions = true; + offset = image.endOffset; + } + return foundImageDimensions ? null : "unreadable"; +} + /** Chunked so a large image can't blow the argument limit of `fromCharCode`. */ const BASE64_CHUNK_SIZE = 0x8000; @@ -167,6 +252,7 @@ async function encodeWithinBudget( bitmap: ImageBitmap, maxDimension: number, budgetChars: number, + preferredMimeType?: "image/jpeg", ): Promise<{ dataUrl: string; mimeType: string } | null> { const scale = Math.min(1, maxDimension / Math.max(bitmap.width, bitmap.height)); const width = Math.max(1, Math.round(bitmap.width * scale)); @@ -176,8 +262,11 @@ async function encodeWithinBudget( // Probe WebP once; JPEG (no alpha) needs a white matte, so the fill has to // happen before drawing and depends on which codec we end up using. - const probe = await encodeCanvas(target.canvas, QUALITY_STEPS[0], "image/webp", 0); - const mimeType = probe ? "image/webp" : "image/jpeg"; + const mimeType = + preferredMimeType ?? + ((await encodeCanvas(target.canvas, QUALITY_STEPS[0], "image/webp", 0)) + ? "image/webp" + : "image/jpeg"); if (mimeType === "image/jpeg") { target.context.fillStyle = "#ffffff"; @@ -203,7 +292,11 @@ type ReencodeResult = * Shared re-encode loop: decodes `file`, then walks the quality ladder and * fallback downscale passes until an encoding fits `budgetChars`. */ -async function reencodeWithinBudget(file: File, budgetChars: number): Promise { +async function reencodeWithinBudget( + file: File, + budgetChars: number, + preferredMimeType?: "image/jpeg", +): Promise { if (!canRecompress()) { return { ok: false, reason: "too-large" }; } @@ -229,7 +322,7 @@ async function reencodeWithinBudget(file: File, budgetChars: number): Promise { if (file.size <= maxBytes) { return { ok: true, file, recompressed: false }; } - if (file.size > MAX_COMPRESSIBLE_SOURCE_BYTES) { + if ((options?.sourceSizeBytes ?? file.size) > MAX_COMPRESSIBLE_SOURCE_BYTES) { return { ok: false, reason: "too-large" }; } // The re-encode loop budgets in data-URL characters. Base64 turns 3 bytes // into 4 chars; flooring keeps the budget a hair conservative instead of // admitting an encoding right at the byte cap. const budgetChars = Math.floor(maxBytes / 3) * 4; - const reencoded = await reencodeWithinBudget(file, budgetChars); + const reencoded = await reencodeWithinBudget(file, budgetChars, options?.preferredMimeType); if (!reencoded.ok) { return reencoded; } @@ -330,3 +426,43 @@ export async function compressImageToByteLimit( recompressed: true, }; } + +/** + * Converts HEIC/HEIF photos to provider-compatible JPEG before applying the + * attachment size limit. The decoder is loaded only when such a photo arrives. + */ +export async function prepareImageForAttachment( + file: File, + maxBytes: number, +): Promise { + if (!isHeicImageFile(file)) { + return compressImageToByteLimit(file, maxBytes); + } + + if (file.size > MAX_COMPRESSIBLE_SOURCE_BYTES) { + return { ok: false, reason: "too-large" }; + } + + let converted: Blob; + try { + const dimensionError = await validateHeicImageDimensions(file); + if (dimensionError) { + return { ok: false, reason: dimensionError }; + } + const { heicTo } = await import("heic-to/csp"); + converted = await heicTo({ blob: file, type: "image/jpeg", quality: QUALITY_STEPS[0] }); + } catch { + return { ok: false, reason: "unreadable" }; + } + + const jpeg = new File([converted], fileNameForMimeType(file.name || "image", "image/jpeg"), { + type: "image/jpeg", + lastModified: file.lastModified, + }); + const result = await compressImageToByteLimit(jpeg, maxBytes, { + preferredMimeType: "image/jpeg", + sourceSizeBytes: file.size, + }); + + return result.ok ? { ...result, recompressed: true } : result; +} diff --git a/docs/user/composer.md b/docs/user/composer.md index b7ef57a56015488ad57c7ac77d4b1a7fea2f91bc..35d634556d88d9cb49905042c372d9edebf6e835 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -7,6 +7,9 @@ multiple messages, then send again in the same thread. On servers that support direct uploads, images upload as soon as you add them. The send button becomes available after every upload finishes. Failed uploads can be retried or removed. +On web and desktop, HEIC and HEIF photos are automatically converted to JPEG when you drag them into +the composer or paste them into a message. + ## Commands and skills Type `/` to open the command menu. Type `$` to find and add a skill. Skill rows show their source, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3dadd8da20e3ee1197d8a9254929b38ba44cd615..490d9ca9d4caf650a7cc9eeaa27a80aab2aa8f58 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -590,6 +590,9 @@ importers: effect: specifier: 4.0.0-beta.103 version: 4.0.0-beta.103(patch_hash=af36b7948b6f9c56623074662b51dade5699880c1a7c71245de73e13c3185fb6) + heic-to: + specifier: ^1.5.2 + version: 1.5.2 jose: specifier: 'catalog:' version: 6.2.2 @@ -7159,6 +7162,9 @@ packages: headers-polyfill@4.0.3: resolution: {integrity: sha512-IScLbePpkvO846sIwOtOTDjutRMWdXdJmXdMvk6gCBHxFO8d+QKOQedyZSxFTTFYRSmlgSTDtXqqq4pcenBXLQ==} + heic-to@1.5.2: + resolution: {integrity: sha512-8Fns+lZHAWmz5U5IUxDeXKwIf3foBoKNPLxxFY4B0MkLjNuomEIHCoDbDE+x/llFK3NCEO1cu4+n3iUKY+Svmw==} + hermes-compiler@250829098.0.10: resolution: {integrity: sha512-TcRlZ0/TlyfJqquRFAWoyElVNnkdYRi/sEp4/Qy8/GYxjg8j2cS9D4MjuaQ+qimkmLN7AmO+44IznRf06mAr0w==} @@ -17839,6 +17845,8 @@ snapshots: headers-polyfill@4.0.3: {} + heic-to@1.5.2: {} + hermes-compiler@250829098.0.10: {} hermes-estree@0.33.3: {} -- 2.54.0