diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index b89521a932da136a397ac157a1569537e1d82648..e92ecd497e3db289ad92bbb06374c4c5738b89c1 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -64,7 +64,7 @@ import { } from "../../promptStashStore"; import { ComposerStashBadge } from "./ComposerStashBadge"; import { ComposerStashMenu } from "./ComposerStashMenu"; -import { compressImageForStash } from "../../lib/stashImageCompression"; +import { compressImageForStash, compressImageToByteLimit } from "../../lib/imageCompression"; import { isCommandPaletteOpen } from "../../commandPaletteBus"; import { getTerminalFocusOwner } from "../../lib/terminalFocus"; import { resolveShortcutCommand } from "../../keybindings"; @@ -206,8 +206,6 @@ import { searchProviderSkills } from "../../providerSkillSearch"; import { useMediaQuery } from "../../hooks/useMediaQuery"; import type { ReviewCommentContext } from "../../reviewCommentContext"; -const IMAGE_SIZE_LIMIT_LABEL = `${Math.round(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES / (1024 * 1024))}MB`; - const runtimeModeConfig: Record< RuntimeMode, { label: string; description: string; icon: LucideIcon } @@ -1006,6 +1004,13 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) * thread) can still be stashed while an earlier encode is running. */ const stashInFlightRef = useRef>(new Set()); + /** + * Count of pasted images still being compressed, per thread. Reserved + * against the attachment limit so concurrent pastes can't overshoot it, + * and checked by `submitComposer` so a send can't race an image into the + * next draft. + */ + const pendingImageCompressionsRef = useRef>(new Map()); // ------------------------------------------------------------------ // Derived: composer send state @@ -1821,12 +1826,26 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) event?.preventDefault(); return; } + // A send while a pasted image is still compressing would strand that + // image: the turn snapshot wouldn't include it, and it would surface + // in the *next* draft instead. Only oversized images hit this — small + // files clear the pending counter within a microtask. + if (activeThreadId && (pendingImageCompressionsRef.current.get(activeThreadId) ?? 0) > 0) { + event?.preventDefault(); + toastManager.add({ + type: "info", + title: "Still compressing a pasted image.", + description: "Send again once its thumbnail appears.", + }); + return; + } onSend(event); if (shouldBlurMobileComposerOnSubmit()) { blurMobileComposerAfterSend(); } }, [ + activeThreadId, blurMobileComposerAfterSend, isSendDisabled, noProviderAvailable, @@ -2273,7 +2292,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) // ------------------------------------------------------------------ // Callbacks: images // ------------------------------------------------------------------ - const addComposerImages = (files: File[]) => { + const addComposerImages = async (files: File[]) => { if (!activeThreadId || files.length === 0) return; if (pendingUserInputs.length > 0) { toastManager.add({ @@ -2282,40 +2301,81 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) }); return; } - const nextImages: ComposerImageAttachment[] = []; - let nextImageCount = composerImagesRef.current.length; + // Captured before the awaits below: the user may switch threads while a + // large image is being compressed, and the attachments and errors belong + // to the thread the paste happened in. + const threadId = activeThreadId; + + // Validation happens synchronously so concurrent pastes see each other: + // accepted files reserve their attachment slots (via the pending counter) + // before the first await, keeping the total under the limit. + const pendingCount = pendingImageCompressionsRef.current.get(threadId) ?? 0; + let reservedCount = composerImagesRef.current.length + pendingCount; + const acceptedFiles: File[] = []; let error: string | null = null; for (const file of files) { if (!file.type.startsWith("image/")) { error = `Unsupported file type for '${file.name}'. Please attach image files only.`; continue; } - if (file.size > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) { - error = `'${file.name}' exceeds the ${IMAGE_SIZE_LIMIT_LABEL} attachment limit.`; - continue; - } - if (nextImageCount >= PROVIDER_SEND_TURN_MAX_ATTACHMENTS) { + if (reservedCount >= PROVIDER_SEND_TURN_MAX_ATTACHMENTS) { error = `You can attach up to ${PROVIDER_SEND_TURN_MAX_ATTACHMENTS} images per message.`; break; } - const previewUrl = URL.createObjectURL(file); - nextImages.push({ - type: "image", - id: randomUUID(), - name: file.name || "image", - mimeType: file.type, - sizeBytes: file.size, - previewUrl, - file, - }); - nextImageCount += 1; + acceptedFiles.push(file); + reservedCount += 1; } - if (nextImages.length === 1 && nextImages[0]) { - addComposerImage(nextImages[0]); - } else if (nextImages.length > 1) { - addComposerImagesToDraft(nextImages); + setThreadError(threadId, error); + if (acceptedFiles.length === 0) return; + + pendingImageCompressionsRef.current.set(threadId, pendingCount + acceptedFiles.length); + try { + const nextImages: ComposerImageAttachment[] = []; + let compressionError: string | null = null; + for (const file of acceptedFiles) { + // Images over the wire cap are downscaled to fit rather than + // refused; files already within it pass through byte-for-byte. + const compressed = await compressImageToByteLimit(file, PROVIDER_SEND_TURN_MAX_IMAGE_BYTES); + if (!compressed.ok) { + compressionError = + compressed.reason === "unreadable" + ? `'${file.name}' could not be read as an image.` + : `'${file.name}' is too large to attach, even after compression.`; + continue; + } + const attachmentFile = compressed.file; + const previewUrl = URL.createObjectURL(attachmentFile); + nextImages.push({ + type: "image", + id: randomUUID(), + name: attachmentFile.name || "image", + mimeType: attachmentFile.type, + sizeBytes: attachmentFile.size, + previewUrl, + file: attachmentFile, + }); + } + if (nextImages.length === 1 && nextImages[0]) { + addComposerImage(nextImages[0]); + } else if (nextImages.length > 1) { + addComposerImagesToDraft(nextImages); + } + // Only failures are reported here. Success must not pass `null`: by + // now other work (a failed send, an overlapping paste) may have set a + // thread error this call knows nothing about, and clearing it would + // swallow that message. + if (compressionError !== null) { + setThreadError(threadId, compressionError); + } + } finally { + const remaining = + (pendingImageCompressionsRef.current.get(threadId) ?? 0) - acceptedFiles.length; + if (remaining > 0) { + pendingImageCompressionsRef.current.set(threadId, remaining); + } else { + pendingImageCompressionsRef.current.delete(threadId); + } } - setThreadError(activeThreadId, error); }; const removeComposerImage = (imageId: string) => { @@ -2331,7 +2391,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) const imageFiles = files.filter((file) => file.type.startsWith("image/")); if (imageFiles.length === 0) return; event.preventDefault(); - addComposerImages(imageFiles); + void addComposerImages(imageFiles); }; const onComposerDragEnter = (event: React.DragEvent) => { @@ -2365,7 +2425,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) dragDepthRef.current = 0; setIsDragOverComposer(false); const files = Array.from(event.dataTransfer.files); - addComposerImages(files); + void addComposerImages(files); focusComposer(); }; diff --git a/apps/web/src/lib/imageCompression.test.ts b/apps/web/src/lib/imageCompression.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..63712ca7e2950c75d747bb7865bdefb04d5b8811 --- /dev/null +++ b/apps/web/src/lib/imageCompression.test.ts @@ -0,0 +1,253 @@ +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; + +import { + compressImageForStash, + compressImageToByteLimit, + MAX_COMPRESSIBLE_SOURCE_BYTES, + MAX_STASH_IMAGE_DATA_URL_CHARS, +} from "./imageCompression"; + +/** + * jsdom has no real canvas/codec, so the re-encode path is exercised with + * stubbed `createImageBitmap` + `OffscreenCanvas`. The encoder stub returns a + * payload whose size scales with quality, mirroring how a real JPEG encoder + * shrinks as quality drops — enough to verify the ladder logic and budget + * enforcement without pulling in a native canvas. + */ + +const originalCreateImageBitmap = globalThis.createImageBitmap; +const originalOffscreenCanvas = globalThis.OffscreenCanvas; + +function makeFile(sizeBytes: number, type = "image/png"): File { + return new File([new Uint8Array(sizeBytes).fill(7)], "shot.png", { type }); +} + +/** + * Installs a fake bitmap + canvas whose encoded size follows `sizeForQuality`. + * `supportsWebp: false` makes `convertToBlob` hand back a differently-typed + * blob for WebP requests, which is how a real browser signals it cannot + * encode that format. + */ +function stubCanvasPipeline( + sizeForQuality: (quality: number) => number, + options?: { supportsWebp?: boolean }, +) { + const supportsWebp = options?.supportsWebp ?? true; + const close = vi.fn(); + const fillRect = vi.fn(); + vi.stubGlobal( + "createImageBitmap", + vi.fn(async () => ({ width: 4000, height: 3000, close })), + ); + vi.stubGlobal( + "OffscreenCanvas", + class { + constructor( + public width: number, + public height: number, + ) {} + getContext() { + return { + fillStyle: "", + fillRect, + drawImage: vi.fn(), + }; + } + async convertToBlob({ type, quality }: { type: string; quality: number }) { + const resolvedType = type === "image/webp" && !supportsWebp ? "image/png" : type; + return new Blob([new Uint8Array(sizeForQuality(quality))], { type: resolvedType }); + } + }, + ); + return { close, fillRect }; +} + +afterEach(() => { + vi.unstubAllGlobals(); + globalThis.createImageBitmap = originalCreateImageBitmap; + globalThis.OffscreenCanvas = originalOffscreenCanvas; +}); + +describe("compressImageForStash", () => { + it("stores a small image verbatim without re-encoding", async () => { + const bitmapSpy = vi.fn(); + vi.stubGlobal("createImageBitmap", bitmapSpy); + + const result = await compressImageForStash(makeFile(1024)); + + expect(result.ok).toBe(true); + expect(result.ok && result.image.recompressed).toBe(false); + expect(result.ok && result.image.mimeType).toBe("image/png"); + expect(result.ok && result.image.dataUrl.startsWith("data:image/png")).toBe(true); + // Untouched payloads must not pay for a decode. + expect(bitmapSpy).not.toHaveBeenCalled(); + }); + + it("re-encodes an oversized image to WebP within the budget", async () => { + // Comfortably under budget at the very first quality step. + const { close, fillRect } = stubCanvasPipeline(() => 120_000); + + const result = await compressImageForStash(makeFile(4_000_000)); + + expect(result.ok).toBe(true); + expect(result.ok && result.image.recompressed).toBe(true); + expect(result.ok && result.image.mimeType).toBe("image/webp"); + expect(result.ok && result.image.dataUrl.length <= MAX_STASH_IMAGE_DATA_URL_CHARS).toBe(true); + // sizeBytes should describe the re-encoded payload, not the 4MB original. + expect(result.ok && result.image.sizeBytes).toBeLessThan(4_000_000); + // WebP keeps alpha, so no white matte should be painted. + expect(fillRect).not.toHaveBeenCalled(); + expect(close).toHaveBeenCalled(); + }); + + it("falls back to JPEG with a white matte when WebP encoding is unavailable", async () => { + const { fillRect } = stubCanvasPipeline(() => 120_000, { supportsWebp: false }); + + const result = await compressImageForStash(makeFile(4_000_000)); + + expect(result.ok && result.image.recompressed).toBe(true); + expect(result.ok && result.image.mimeType).toBe("image/jpeg"); + // JPEG has no alpha, so transparent regions must be matted white. + expect(fillRect).toHaveBeenCalled(); + }); + + it("steps quality down until the encoded image fits", async () => { + // Only the lowest quality step (0.68) lands under the budget. + const { close } = stubCanvasPipeline((quality) => (quality <= 0.68 ? 400_000 : 3_000_000)); + + const result = await compressImageForStash(makeFile(9_000_000)); + + expect(result.ok && result.image.recompressed).toBe(true); + expect(result.ok && result.image.dataUrl.length <= MAX_STASH_IMAGE_DATA_URL_CHARS).toBe(true); + expect(close).toHaveBeenCalled(); + }); + + it("reports too-large when even the smallest encoding overflows the budget", async () => { + const { close } = stubCanvasPipeline(() => 8_000_000); + + const result = await compressImageForStash(makeFile(9_000_000)); + + expect(result).toEqual({ ok: false, reason: "too-large" }); + // The bitmap must still be released on the give-up path. + expect(close).toHaveBeenCalled(); + }); + + it("reports too-large for an oversized image when the browser cannot re-encode", async () => { + vi.stubGlobal("createImageBitmap", undefined); + vi.stubGlobal("OffscreenCanvas", undefined); + + expect(await compressImageForStash(makeFile(4_000_000))).toEqual({ + ok: false, + reason: "too-large", + }); + }); + + it("reports unreadable when the image fails to decode", async () => { + vi.stubGlobal( + "createImageBitmap", + vi.fn(async () => { + throw new Error("corrupt image"); + }), + ); + vi.stubGlobal( + "OffscreenCanvas", + class { + getContext() { + return null; + } + }, + ); + + expect(await compressImageForStash(makeFile(4_000_000))).toEqual({ + ok: false, + reason: "unreadable", + }); + }); + + it("compressImageToByteLimit passes small files through byte-for-byte", async () => { + const bitmapSpy = vi.fn(); + vi.stubGlobal("createImageBitmap", bitmapSpy); + + const original = makeFile(1024); + const result = await compressImageToByteLimit(original, 10 * 1024 * 1024); + + expect(result.ok).toBe(true); + expect(result.ok && result.recompressed).toBe(false); + // Pass-through must be the same File object, not a copy. + expect(result.ok && result.file).toBe(original); + expect(bitmapSpy).not.toHaveBeenCalled(); + }); + + it("compressImageToByteLimit re-encodes an oversized file under the byte cap", async () => { + stubCanvasPipeline(() => 200_000); + + const result = await compressImageToByteLimit(makeFile(2_000_000), 1_000_000); + + expect(result.ok).toBe(true); + expect(result.ok && result.recompressed).toBe(true); + expect(result.ok && result.file.type).toBe("image/webp"); + // The re-encoded name must match the new container format. + expect(result.ok && result.file.name).toBe("shot.webp"); + expect(result.ok && result.file.size).toBeLessThanOrEqual(1_000_000); + }); + + it("compressImageToByteLimit refuses sources above the decode-safety ceiling", async () => { + const bitmapSpy = vi.fn(); + vi.stubGlobal("createImageBitmap", bitmapSpy); + + const result = await compressImageToByteLimit( + makeFile(MAX_COMPRESSIBLE_SOURCE_BYTES + 1), + 10 * 1024 * 1024, + ); + + expect(result).toEqual({ ok: false, reason: "too-large" }); + // The whole point of the ceiling is to never decode such a file. + expect(bitmapSpy).not.toHaveBeenCalled(); + }); + + it("compressImageToByteLimit reports too-large when no encoding fits", async () => { + const { close } = stubCanvasPipeline(() => 3_000_000); + + const result = await compressImageToByteLimit(makeFile(2_000_000), 1_000_000); + + expect(result).toEqual({ ok: false, reason: "too-large" }); + expect(close).toHaveBeenCalled(); + }); + + it("shrinks below the source size when the image is already under MAX_DIMENSION", async () => { + // A small-but-heavy source (e.g. a dense PNG): only a real downscale can + // get it under budget, since quality alone is stubbed to never suffice. + let smallestRequested = Number.POSITIVE_INFINITY; + const close = vi.fn(); + vi.stubGlobal( + "createImageBitmap", + vi.fn(async () => ({ width: 800, height: 600, close })), + ); + vi.stubGlobal( + "OffscreenCanvas", + class { + constructor( + public width: number, + public height: number, + ) { + smallestRequested = Math.min(smallestRequested, width); + } + getContext() { + return { fillStyle: "", fillRect: vi.fn(), drawImage: vi.fn() }; + } + async convertToBlob({ type }: { type: string; quality: number }) { + // Only a genuinely downscaled pass fits the budget. + const size = smallestRequested < 800 ? 100_000 : 5_000_000; + return new Blob([new Uint8Array(size)], { type }); + } + }, + ); + + const result = await compressImageForStash(makeFile(4_000_000)); + + expect(result.ok).toBe(true); + // Fallback passes must scale off the bitmap, not a fixed 2048 ceiling + // that would never go below an 800px source. + expect(smallestRequested).toBeLessThan(800); + }); +}); diff --git a/apps/web/src/lib/imageCompression.ts b/apps/web/src/lib/imageCompression.ts new file mode 100644 index 0000000000000000000000000000000000000000..1c57fdad1a55c764ef075b4a481df883a22bb4dc --- /dev/null +++ b/apps/web/src/lib/imageCompression.ts @@ -0,0 +1,335 @@ +/** + * Downscale + re-encode for image attachments that are too big for where + * they're headed. Two consumers share the same pipeline: + * + * - The prompt stash persists images as base64 in localStorage (~5MB origin + * quota), so `compressImageForStash` targets a per-image character budget. + * - The composer accepts pasted/dropped images larger than the provider's + * `PROVIDER_SEND_TURN_MAX_IMAGE_BYTES` wire cap and shrinks them to fit + * via `compressImageToByteLimit` instead of rejecting the paste. + * + * Images already within budget pass through untouched. + */ + +/** + * Longest edge kept when an image has to be re-encoded. Sized so a typical + * retina screenshot (3024px wide) stays legible rather than being halved. + */ +const MAX_DIMENSION = 2048; +/** Base64 budget for a single stashed image (~975KB of binary). */ +export const MAX_STASH_IMAGE_DATA_URL_CHARS = 1_300_000; +/** + * Ceiling on the *source* file handed to the re-encoder. File size is a + * proxy for pixel count, and decoding hundreds of megapixels into an + * ImageBitmap can OOM the tab — beyond this we refuse rather than risk it. + */ +export const MAX_COMPRESSIBLE_SOURCE_BYTES = 50 * 1024 * 1024; +/** + * Quality ladder tried in order until the encoded image fits the budget. + * The floor stays high enough to avoid visible blocking on UI screenshots; + * if even that overflows we drop resolution instead of quality. + */ +const QUALITY_STEPS = [0.92, 0.85, 0.78, 0.68] as const; +/** Extra downscale passes applied when even the lowest quality overflows. */ +const FALLBACK_SCALE_STEPS = [0.75, 0.55] as const; + +export interface CompressedStashImage { + dataUrl: string; + mimeType: string; + sizeBytes: number; + /** True when the payload was re-encoded rather than stored verbatim. */ + recompressed: boolean; +} + +/** + * Why an image could not be compressed. Callers report these differently: + * "too large" is a budget outcome, "unreadable" is a decode failure. + */ +export type ImageCompressionFailureReason = "too-large" | "unreadable"; + +export type CompressStashImageResult = + | { ok: true; image: CompressedStashImage } + | { ok: false; reason: ImageCompressionFailureReason }; + +export type CompressImageFileResult = + | { ok: true; file: File; recompressed: boolean } + | { ok: false; reason: ImageCompressionFailureReason }; + +/** Chunked so a large image can't blow the argument limit of `fromCharCode`. */ +const BASE64_CHUNK_SIZE = 0x8000; + +function bytesToBase64(bytes: Uint8Array): string { + let binary = ""; + for (let offset = 0; offset < bytes.length; offset += BASE64_CHUNK_SIZE) { + binary += String.fromCharCode(...bytes.subarray(offset, offset + BASE64_CHUNK_SIZE)); + } + return btoa(binary); +} + +/** + * Blob → base64 data URL. Uses `arrayBuffer()` rather than `FileReader` so + * the module works anywhere `Blob` does (including non-DOM test runners). + */ +async function blobToDataUrl(blob: File | Blob, mimeTypeOverride?: string): Promise { + const buffer = await blob.arrayBuffer(); + const mimeType = mimeTypeOverride || blob.type || "application/octet-stream"; + return `data:${mimeType};base64,${bytesToBase64(new Uint8Array(buffer))}`; +} + +/** Approximate decoded byte count for a base64 data URL. */ +function dataUrlByteLength(dataUrl: string): number { + const commaIndex = dataUrl.indexOf(","); + const payload = commaIndex === -1 ? dataUrl : dataUrl.slice(commaIndex + 1); + const padding = payload.endsWith("==") ? 2 : payload.endsWith("=") ? 1 : 0; + return Math.max(0, Math.floor((payload.length * 3) / 4) - padding); +} + +/** Base64 payload of a data URL decoded back into a `File`. */ +function dataUrlToFile(dataUrl: string, name: string, mimeType: string): File { + const payload = dataUrl.slice(dataUrl.indexOf(",") + 1); + const binary = atob(payload); + const bytes = new Uint8Array(binary.length); + for (let index = 0; index < binary.length; index += 1) { + bytes[index] = binary.charCodeAt(index); + } + return new File([bytes], name, { type: mimeType }); +} + +/** + * Re-encoding changes the container, so a name like `shot.png` would lie + * about its contents. Swap the extension to match the encoded mime type. + */ +function fileNameForMimeType(name: string, mimeType: string): string { + const extension = mimeType === "image/webp" ? ".webp" : ".jpg"; + const dotIndex = name.lastIndexOf("."); + const base = dotIndex > 0 ? name.slice(0, dotIndex) : name; + return `${base}${extension}`; +} + +function canRecompress(): boolean { + return ( + typeof createImageBitmap === "function" && + (typeof OffscreenCanvas === "function" || typeof document !== "undefined") + ); +} + +interface Canvas2D { + canvas: OffscreenCanvas | HTMLCanvasElement; + context: OffscreenCanvasRenderingContext2D | CanvasRenderingContext2D; +} + +function createCanvas(width: number, height: number): Canvas2D | null { + if (typeof OffscreenCanvas === "function") { + const canvas = new OffscreenCanvas(width, height); + const context = canvas.getContext("2d"); + if (!context) return null; + return { canvas, context }; + } + if (typeof document === "undefined") return null; + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + const context = canvas.getContext("2d"); + if (!context) return null; + return { canvas, context }; +} + +/** + * WebP is preferred: at matched visual quality it lands roughly 25-35% + * smaller than JPEG, so the same budget buys more resolution and detail — + * and it keeps alpha, so screenshots with transparency survive intact. + * Browsers that can't encode it silently fall back to JPEG. + */ +async function encodeToDataUrl( + canvas: OffscreenCanvas | HTMLCanvasElement, + quality: number, + mimeType: string, +): Promise<{ dataUrl: string; mimeType: string } | null> { + if (typeof HTMLCanvasElement !== "undefined" && canvas instanceof HTMLCanvasElement) { + const dataUrl = canvas.toDataURL(mimeType, quality); + // toDataURL silently returns a PNG when the requested type is unsupported. + if (!dataUrl.startsWith(`data:${mimeType}`)) return null; + return { dataUrl, mimeType }; + } + const blob = await (canvas as OffscreenCanvas).convertToBlob({ type: mimeType, quality }); + if (blob.type && blob.type !== mimeType) return null; + return { dataUrl: await blobToDataUrl(blob, mimeType), mimeType }; +} + +/** + * Draws `bitmap` scaled to fit `maxDimension` and encodes it, stepping + * quality down until the data URL fits `budgetChars`. Returns the smallest + * encoding produced, even if it still exceeds the budget, so the caller can + * decide whether to keep or drop it. + */ +async function encodeWithinBudget( + bitmap: ImageBitmap, + maxDimension: number, + budgetChars: number, +): Promise<{ dataUrl: string; mimeType: string } | null> { + const scale = Math.min(1, maxDimension / Math.max(bitmap.width, bitmap.height)); + const width = Math.max(1, Math.round(bitmap.width * scale)); + const height = Math.max(1, Math.round(bitmap.height * scale)); + const target = createCanvas(width, height); + if (!target) return null; + + // Probe WebP once; JPEG (no alpha) needs a white matte, so the fill has to + // happen before drawing and depends on which codec we end up using. + const probe = await encodeToDataUrl(target.canvas, QUALITY_STEPS[0], "image/webp"); + const mimeType = probe ? "image/webp" : "image/jpeg"; + + if (mimeType === "image/jpeg") { + target.context.fillStyle = "#ffffff"; + target.context.fillRect(0, 0, width, height); + } + target.context.drawImage(bitmap, 0, 0, width, height); + + let smallest: { dataUrl: string; mimeType: string } | null = null; + for (const quality of QUALITY_STEPS) { + const encoded = await encodeToDataUrl(target.canvas, quality, mimeType); + if (!encoded) break; + if (smallest === null || encoded.dataUrl.length < smallest.dataUrl.length) { + smallest = encoded; + } + if (encoded.dataUrl.length <= budgetChars) { + return encoded; + } + } + return smallest; +} + +type ReencodeResult = + | { ok: true; dataUrl: string; mimeType: string } + | { ok: false; reason: ImageCompressionFailureReason }; + +/** + * Shared re-encode loop: decodes `file`, then walks the quality ladder and + * fallback downscale passes until an encoding fits `budgetChars`. + */ +async function reencodeWithinBudget(file: File, budgetChars: number): Promise { + if (!canRecompress()) { + return { ok: false, reason: "too-large" }; + } + + let bitmap: ImageBitmap; + try { + bitmap = await createImageBitmap(file); + } catch { + return { ok: false, reason: "unreadable" }; + } + + try { + // Each pass shrinks relative to the *previous target*, capped by + // MAX_DIMENSION. Scaling a fixed ceiling instead would be a no-op for + // images already smaller than that ceiling — the fallback passes would + // all resolve to the source size and never actually reduce resolution. + const baseDimension = Math.min(MAX_DIMENSION, Math.max(bitmap.width, bitmap.height)); + // Tracks whether the *last* attempt threw, so a run of encoder failures + // is reported as unreadable while a run of merely-too-big results is + // reported as too-large. + let encodeFailed = false; + for (const dimensionScale of [1, ...FALLBACK_SCALE_STEPS]) { + const targetDimension = Math.max(1, Math.round(baseDimension * dimensionScale)); + let encoded: { dataUrl: string; mimeType: string } | null; + try { + encoded = await encodeWithinBudget(bitmap, targetDimension, budgetChars); + } catch { + // Canvas allocation, drawing, or the codec itself can throw — often + // precisely *because* the target is too big (OOM on a large bitmap). + // Keep trying the smaller fallback scales rather than giving up: a + // reduced pass may well succeed. The exception must never escape, + // though, since callers finalize state after this returns and a + // throw would strand it (e.g. a stash entry stuck "still saving"). + encodeFailed = true; + continue; + } + encodeFailed = false; + if (encoded && encoded.dataUrl.length <= budgetChars) { + return { ok: true, dataUrl: encoded.dataUrl, mimeType: encoded.mimeType }; + } + } + return { ok: false, reason: encodeFailed ? "unreadable" : "too-large" }; + } finally { + bitmap.close(); + } +} + +/** + * Produces the payload to persist for a stashed image. + * + * Small images are stored verbatim (preserving PNG transparency and exact + * pixels). Anything over budget is downscaled and re-encoded; if it still + * doesn't fit after the fallback passes, reports a failure so the caller + * can record it as dropped. + */ +export async function compressImageForStash( + file: File, + budgetChars: number = MAX_STASH_IMAGE_DATA_URL_CHARS, +): Promise { + let originalDataUrl: string; + try { + originalDataUrl = await blobToDataUrl(file); + } catch { + return { ok: false, reason: "unreadable" }; + } + if (originalDataUrl.length <= budgetChars) { + return { + ok: true, + image: { + dataUrl: originalDataUrl, + mimeType: file.type, + sizeBytes: file.size, + recompressed: false, + }, + }; + } + const reencoded = await reencodeWithinBudget(file, budgetChars); + if (!reencoded.ok) { + return reencoded; + } + return { + ok: true, + image: { + dataUrl: reencoded.dataUrl, + mimeType: reencoded.mimeType, + sizeBytes: dataUrlByteLength(reencoded.dataUrl), + recompressed: true, + }, + }; +} + +/** + * Shrinks `file` until its binary size fits `maxBytes`, returning a new + * `File` (WebP or JPEG). Files already within the limit pass through + * untouched, preserving their exact bytes and format. Sources above + * `MAX_COMPRESSIBLE_SOURCE_BYTES` are refused outright — decoding them is + * the risk, so no amount of output budget makes them safe. + */ +export async function compressImageToByteLimit( + file: File, + maxBytes: number, +): Promise { + if (file.size <= maxBytes) { + return { ok: true, file, recompressed: false }; + } + if (file.size > MAX_COMPRESSIBLE_SOURCE_BYTES) { + return { ok: false, reason: "too-large" }; + } + // The re-encode loop budgets in data-URL characters. Base64 turns 3 bytes + // into 4 chars; flooring keeps the budget a hair conservative instead of + // admitting an encoding right at the byte cap. + const budgetChars = Math.floor(maxBytes / 3) * 4; + const reencoded = await reencodeWithinBudget(file, budgetChars); + if (!reencoded.ok) { + return reencoded; + } + return { + ok: true, + file: dataUrlToFile( + reencoded.dataUrl, + fileNameForMimeType(file.name || "image", reencoded.mimeType), + reencoded.mimeType, + ), + recompressed: true, + }; +} diff --git a/apps/web/src/lib/stashImageCompression.test.ts b/apps/web/src/lib/stashImageCompression.test.ts deleted file mode 100644 index aaffa284cbb33010fde330b54a4322cf53b1a34a..0000000000000000000000000000000000000000 --- a/apps/web/src/lib/stashImageCompression.test.ts +++ /dev/null @@ -1,198 +0,0 @@ -import { afterEach, describe, expect, it, vi } from "vite-plus/test"; - -import { compressImageForStash, MAX_STASH_IMAGE_DATA_URL_CHARS } from "./stashImageCompression"; - -/** - * jsdom has no real canvas/codec, so the re-encode path is exercised with - * stubbed `createImageBitmap` + `OffscreenCanvas`. The encoder stub returns a - * payload whose size scales with quality, mirroring how a real JPEG encoder - * shrinks as quality drops — enough to verify the ladder logic and budget - * enforcement without pulling in a native canvas. - */ - -const originalCreateImageBitmap = globalThis.createImageBitmap; -const originalOffscreenCanvas = globalThis.OffscreenCanvas; - -function makeFile(sizeBytes: number, type = "image/png"): File { - return new File([new Uint8Array(sizeBytes).fill(7)], "shot.png", { type }); -} - -/** - * Installs a fake bitmap + canvas whose encoded size follows `sizeForQuality`. - * `supportsWebp: false` makes `convertToBlob` hand back a differently-typed - * blob for WebP requests, which is how a real browser signals it cannot - * encode that format. - */ -function stubCanvasPipeline( - sizeForQuality: (quality: number) => number, - options?: { supportsWebp?: boolean }, -) { - const supportsWebp = options?.supportsWebp ?? true; - const close = vi.fn(); - const fillRect = vi.fn(); - vi.stubGlobal( - "createImageBitmap", - vi.fn(async () => ({ width: 4000, height: 3000, close })), - ); - vi.stubGlobal( - "OffscreenCanvas", - class { - constructor( - public width: number, - public height: number, - ) {} - getContext() { - return { - fillStyle: "", - fillRect, - drawImage: vi.fn(), - }; - } - async convertToBlob({ type, quality }: { type: string; quality: number }) { - const resolvedType = type === "image/webp" && !supportsWebp ? "image/png" : type; - return new Blob([new Uint8Array(sizeForQuality(quality))], { type: resolvedType }); - } - }, - ); - return { close, fillRect }; -} - -afterEach(() => { - vi.unstubAllGlobals(); - globalThis.createImageBitmap = originalCreateImageBitmap; - globalThis.OffscreenCanvas = originalOffscreenCanvas; -}); - -describe("compressImageForStash", () => { - it("stores a small image verbatim without re-encoding", async () => { - const bitmapSpy = vi.fn(); - vi.stubGlobal("createImageBitmap", bitmapSpy); - - const result = await compressImageForStash(makeFile(1024)); - - expect(result.ok).toBe(true); - expect(result.ok && result.image.recompressed).toBe(false); - expect(result.ok && result.image.mimeType).toBe("image/png"); - expect(result.ok && result.image.dataUrl.startsWith("data:image/png")).toBe(true); - // Untouched payloads must not pay for a decode. - expect(bitmapSpy).not.toHaveBeenCalled(); - }); - - it("re-encodes an oversized image to WebP within the budget", async () => { - // Comfortably under budget at the very first quality step. - const { close, fillRect } = stubCanvasPipeline(() => 120_000); - - const result = await compressImageForStash(makeFile(4_000_000)); - - expect(result.ok).toBe(true); - expect(result.ok && result.image.recompressed).toBe(true); - expect(result.ok && result.image.mimeType).toBe("image/webp"); - expect(result.ok && result.image.dataUrl.length <= MAX_STASH_IMAGE_DATA_URL_CHARS).toBe(true); - // sizeBytes should describe the re-encoded payload, not the 4MB original. - expect(result.ok && result.image.sizeBytes).toBeLessThan(4_000_000); - // WebP keeps alpha, so no white matte should be painted. - expect(fillRect).not.toHaveBeenCalled(); - expect(close).toHaveBeenCalled(); - }); - - it("falls back to JPEG with a white matte when WebP encoding is unavailable", async () => { - const { fillRect } = stubCanvasPipeline(() => 120_000, { supportsWebp: false }); - - const result = await compressImageForStash(makeFile(4_000_000)); - - expect(result.ok && result.image.recompressed).toBe(true); - expect(result.ok && result.image.mimeType).toBe("image/jpeg"); - // JPEG has no alpha, so transparent regions must be matted white. - expect(fillRect).toHaveBeenCalled(); - }); - - it("steps quality down until the encoded image fits", async () => { - // Only the lowest quality step (0.68) lands under the budget. - const { close } = stubCanvasPipeline((quality) => (quality <= 0.68 ? 400_000 : 3_000_000)); - - const result = await compressImageForStash(makeFile(9_000_000)); - - expect(result.ok && result.image.recompressed).toBe(true); - expect(result.ok && result.image.dataUrl.length <= MAX_STASH_IMAGE_DATA_URL_CHARS).toBe(true); - expect(close).toHaveBeenCalled(); - }); - - it("reports too-large when even the smallest encoding overflows the budget", async () => { - const { close } = stubCanvasPipeline(() => 8_000_000); - - const result = await compressImageForStash(makeFile(9_000_000)); - - expect(result).toEqual({ ok: false, reason: "too-large" }); - // The bitmap must still be released on the give-up path. - expect(close).toHaveBeenCalled(); - }); - - it("reports too-large for an oversized image when the browser cannot re-encode", async () => { - vi.stubGlobal("createImageBitmap", undefined); - vi.stubGlobal("OffscreenCanvas", undefined); - - expect(await compressImageForStash(makeFile(4_000_000))).toEqual({ - ok: false, - reason: "too-large", - }); - }); - - it("reports unreadable when the image fails to decode", async () => { - vi.stubGlobal( - "createImageBitmap", - vi.fn(async () => { - throw new Error("corrupt image"); - }), - ); - vi.stubGlobal( - "OffscreenCanvas", - class { - getContext() { - return null; - } - }, - ); - - expect(await compressImageForStash(makeFile(4_000_000))).toEqual({ - ok: false, - reason: "unreadable", - }); - }); - - it("shrinks below the source size when the image is already under MAX_DIMENSION", async () => { - // A small-but-heavy source (e.g. a dense PNG): only a real downscale can - // get it under budget, since quality alone is stubbed to never suffice. - let smallestRequested = Number.POSITIVE_INFINITY; - const close = vi.fn(); - vi.stubGlobal( - "createImageBitmap", - vi.fn(async () => ({ width: 800, height: 600, close })), - ); - vi.stubGlobal( - "OffscreenCanvas", - class { - constructor( - public width: number, - public height: number, - ) { - smallestRequested = Math.min(smallestRequested, width); - } - getContext() { - return { fillStyle: "", fillRect: vi.fn(), drawImage: vi.fn() }; - } - async convertToBlob({ type }: { type: string; quality: number }) { - // Only a genuinely downscaled pass fits the budget. - const size = smallestRequested < 800 ? 100_000 : 5_000_000; - return new Blob([new Uint8Array(size)], { type }); - } - }, - ); - - const result = await compressImageForStash(makeFile(4_000_000)); - - expect(result.ok).toBe(true); - // Fallback passes must scale off the bitmap, not a fixed 2048 ceiling - // that would never go below an 800px source. - expect(smallestRequested).toBeLessThan(800); - }); -}); diff --git a/apps/web/src/lib/stashImageCompression.ts b/apps/web/src/lib/stashImageCompression.ts deleted file mode 100644 index 7f133fb292624de2f4154a67944bec7c43e3e3a7..0000000000000000000000000000000000000000 --- a/apps/web/src/lib/stashImageCompression.ts +++ /dev/null @@ -1,250 +0,0 @@ -/** - * Re-encoding for stashed image attachments. - * - * The composer accepts images up to `PROVIDER_SEND_TURN_MAX_IMAGE_BYTES` - * (10MB), but the stash persists them as base64 in localStorage, where the - * whole origin shares a ~5MB quota. Rather than refuse large screenshots, - * downscale + re-encode them to JPEG so a stashed prompt keeps its images. - * - * Only the *stashed copy* is compressed; the live composer attachment is - * untouched, so sending without stashing still uploads the original file. - */ - -/** - * Longest edge kept when an image has to be re-encoded. Sized so a typical - * retina screenshot (3024px wide) stays legible rather than being halved. - */ -const MAX_DIMENSION = 2048; -/** Base64 budget for a single stashed image (~975KB of binary). */ -export const MAX_STASH_IMAGE_DATA_URL_CHARS = 1_300_000; -/** - * Quality ladder tried in order until the encoded image fits the budget. - * The floor stays high enough to avoid visible blocking on UI screenshots; - * if even that overflows we drop resolution instead of quality. - */ -const QUALITY_STEPS = [0.92, 0.85, 0.78, 0.68] as const; -/** Extra downscale passes applied when even the lowest quality overflows. */ -const FALLBACK_SCALE_STEPS = [0.75, 0.55] as const; - -export interface CompressedStashImage { - dataUrl: string; - mimeType: string; - sizeBytes: number; - /** True when the payload was re-encoded rather than stored verbatim. */ - recompressed: boolean; -} - -/** - * Why an image could not be stashed. Callers report these differently: - * "too large" is a budget outcome, "unreadable" is a decode failure. - */ -export type StashImageFailureReason = "too-large" | "unreadable"; - -export type CompressStashImageResult = - | { ok: true; image: CompressedStashImage } - | { ok: false; reason: StashImageFailureReason }; - -/** Chunked so a large image can't blow the argument limit of `fromCharCode`. */ -const BASE64_CHUNK_SIZE = 0x8000; - -function bytesToBase64(bytes: Uint8Array): string { - let binary = ""; - for (let offset = 0; offset < bytes.length; offset += BASE64_CHUNK_SIZE) { - binary += String.fromCharCode(...bytes.subarray(offset, offset + BASE64_CHUNK_SIZE)); - } - return btoa(binary); -} - -/** - * Blob → base64 data URL. Uses `arrayBuffer()` rather than `FileReader` so - * the module works anywhere `Blob` does (including non-DOM test runners). - */ -async function blobToDataUrl(blob: File | Blob, mimeTypeOverride?: string): Promise { - const buffer = await blob.arrayBuffer(); - const mimeType = mimeTypeOverride || blob.type || "application/octet-stream"; - return `data:${mimeType};base64,${bytesToBase64(new Uint8Array(buffer))}`; -} - -/** Approximate decoded byte count for a base64 data URL. */ -function dataUrlByteLength(dataUrl: string): number { - const commaIndex = dataUrl.indexOf(","); - const payload = commaIndex === -1 ? dataUrl : dataUrl.slice(commaIndex + 1); - const padding = payload.endsWith("==") ? 2 : payload.endsWith("=") ? 1 : 0; - return Math.max(0, Math.floor((payload.length * 3) / 4) - padding); -} - -function canRecompress(): boolean { - return ( - typeof createImageBitmap === "function" && - (typeof OffscreenCanvas === "function" || typeof document !== "undefined") - ); -} - -interface Canvas2D { - canvas: OffscreenCanvas | HTMLCanvasElement; - context: OffscreenCanvasRenderingContext2D | CanvasRenderingContext2D; -} - -function createCanvas(width: number, height: number): Canvas2D | null { - if (typeof OffscreenCanvas === "function") { - const canvas = new OffscreenCanvas(width, height); - const context = canvas.getContext("2d"); - if (!context) return null; - return { canvas, context }; - } - if (typeof document === "undefined") return null; - const canvas = document.createElement("canvas"); - canvas.width = width; - canvas.height = height; - const context = canvas.getContext("2d"); - if (!context) return null; - return { canvas, context }; -} - -/** - * WebP is preferred: at matched visual quality it lands roughly 25-35% - * smaller than JPEG, so the same budget buys more resolution and detail — - * and it keeps alpha, so screenshots with transparency survive intact. - * Browsers that can't encode it silently fall back to JPEG. - */ -async function encodeToDataUrl( - canvas: OffscreenCanvas | HTMLCanvasElement, - quality: number, - mimeType: string, -): Promise<{ dataUrl: string; mimeType: string } | null> { - if (typeof HTMLCanvasElement !== "undefined" && canvas instanceof HTMLCanvasElement) { - const dataUrl = canvas.toDataURL(mimeType, quality); - // toDataURL silently returns a PNG when the requested type is unsupported. - if (!dataUrl.startsWith(`data:${mimeType}`)) return null; - return { dataUrl, mimeType }; - } - const blob = await (canvas as OffscreenCanvas).convertToBlob({ type: mimeType, quality }); - if (blob.type && blob.type !== mimeType) return null; - return { dataUrl: await blobToDataUrl(blob, mimeType), mimeType }; -} - -/** - * Draws `bitmap` scaled to fit `maxDimension` and encodes it as JPEG, - * stepping quality down until the data URL fits `budgetChars`. Returns the - * smallest encoding produced, even if it still exceeds the budget, so the - * caller can decide whether to keep or drop it. - */ -async function encodeWithinBudget( - bitmap: ImageBitmap, - maxDimension: number, - budgetChars: number, -): Promise<{ dataUrl: string; mimeType: string } | null> { - const scale = Math.min(1, maxDimension / Math.max(bitmap.width, bitmap.height)); - const width = Math.max(1, Math.round(bitmap.width * scale)); - const height = Math.max(1, Math.round(bitmap.height * scale)); - const target = createCanvas(width, height); - if (!target) return null; - - // Probe WebP once; JPEG (no alpha) needs a white matte, so the fill has to - // happen before drawing and depends on which codec we end up using. - const probe = await encodeToDataUrl(target.canvas, QUALITY_STEPS[0], "image/webp"); - const mimeType = probe ? "image/webp" : "image/jpeg"; - - if (mimeType === "image/jpeg") { - target.context.fillStyle = "#ffffff"; - target.context.fillRect(0, 0, width, height); - } - target.context.drawImage(bitmap, 0, 0, width, height); - - let smallest: { dataUrl: string; mimeType: string } | null = null; - for (const quality of QUALITY_STEPS) { - const encoded = await encodeToDataUrl(target.canvas, quality, mimeType); - if (!encoded) break; - if (smallest === null || encoded.dataUrl.length < smallest.dataUrl.length) { - smallest = encoded; - } - if (encoded.dataUrl.length <= budgetChars) { - return encoded; - } - } - return smallest; -} - -/** - * Produces the payload to persist for a stashed image. - * - * Small images are stored verbatim (preserving PNG transparency and exact - * pixels). Anything over budget is downscaled and JPEG-encoded; if it still - * doesn't fit after the fallback passes, returns `null` so the caller can - * record it as dropped. - */ -export async function compressImageForStash( - file: File, - budgetChars: number = MAX_STASH_IMAGE_DATA_URL_CHARS, -): Promise { - let originalDataUrl: string; - try { - originalDataUrl = await blobToDataUrl(file); - } catch { - return { ok: false, reason: "unreadable" }; - } - if (originalDataUrl.length <= budgetChars) { - return { - ok: true, - image: { - dataUrl: originalDataUrl, - mimeType: file.type, - sizeBytes: file.size, - recompressed: false, - }, - }; - } - if (!canRecompress()) { - return { ok: false, reason: "too-large" }; - } - - let bitmap: ImageBitmap; - try { - bitmap = await createImageBitmap(file); - } catch { - return { ok: false, reason: "unreadable" }; - } - - try { - // Each pass shrinks relative to the *previous target*, capped by - // MAX_DIMENSION. Scaling a fixed ceiling instead would be a no-op for - // images already smaller than that ceiling — the fallback passes would - // all resolve to the source size and never actually reduce resolution. - const baseDimension = Math.min(MAX_DIMENSION, Math.max(bitmap.width, bitmap.height)); - // Tracks whether the *last* attempt threw, so a run of encoder failures - // is reported as unreadable while a run of merely-too-big results is - // reported as too-large. - let encodeFailed = false; - for (const dimensionScale of [1, ...FALLBACK_SCALE_STEPS]) { - const targetDimension = Math.max(1, Math.round(baseDimension * dimensionScale)); - let encoded: { dataUrl: string; mimeType: string } | null; - try { - encoded = await encodeWithinBudget(bitmap, targetDimension, budgetChars); - } catch { - // Canvas allocation, drawing, or the codec itself can throw — often - // precisely *because* the target is too big (OOM on a large bitmap). - // Keep trying the smaller fallback scales rather than giving up: a - // reduced pass may well succeed. The exception must never escape, - // though, since the caller finalizes the entry after this returns and - // a throw would strand it as permanently "still saving". - encodeFailed = true; - continue; - } - encodeFailed = false; - if (encoded && encoded.dataUrl.length <= budgetChars) { - return { - ok: true, - image: { - dataUrl: encoded.dataUrl, - mimeType: encoded.mimeType, - sizeBytes: dataUrlByteLength(encoded.dataUrl), - recompressed: true, - }, - }; - } - } - return { ok: false, reason: encodeFailed ? "unreadable" : "too-large" }; - } finally { - bitmap.close(); - } -}