1/**
2 * Downscale + re-encode for image attachments that are too big for where
3 * they're headed. Two consumers share the same pipeline:
4 *
5 * - The prompt stash persists images as base64 in localStorage (~5MB origin
6 * quota), so `compressImageForStash` targets a per-image character budget.
7 * - The composer accepts pasted/dropped images larger than the provider's
8 * `PROVIDER_SEND_TURN_MAX_IMAGE_BYTES` wire cap and shrinks them to fit
9 * via `compressImageToByteLimit` instead of rejecting the paste.
10 *
11 * Supported images already within budget pass through untouched, except that
12 * a PNG or JPEG longer than `IMAGE_ATTACHMENT_MAX_EDGE` on a side is scaled down
13 * to it in its own format before upload. HEIC/HEIF photos are decoded to JPEG
14 * first because providers cannot consume them.
15 */
16
17import {
18 downscalableImageMimeType,
19 fitImageAttachmentSize,
20 IMAGE_ATTACHMENT_MAX_EDGE,
21} from "@t3tools/client-runtime/image-attachment-size";
22import {
23 IMAGE_DIMENSIONS_HEADER_BYTES,
24 readImageDimensions,
25} from "@t3tools/shared/imageDimensions";
26
27/** Longest edge kept when an image has to be re-encoded. */
28const MAX_DIMENSION = IMAGE_ATTACHMENT_MAX_EDGE;
29/** Base64 budget for a single stashed image (~975KB of binary). */
30export const MAX_STASH_IMAGE_DATA_URL_CHARS = 1_300_000;
31/**
32 * Ceiling on the *source* file handed to the re-encoder. File size is a
33 * proxy for pixel count, and decoding hundreds of megapixels into an
34 * ImageBitmap can OOM the tab — beyond this we refuse rather than risk it.
35 */
36export const MAX_COMPRESSIBLE_SOURCE_BYTES = 50 * 1024 * 1024;
37const MAX_HEIC_DECODE_PIXELS = 64_000_000;
38const MAX_HEIC_METADATA_BYTES = 1024 * 1024;
39/**
40 * Quality ladder tried in order until the encoded image fits the budget.
41 * The floor stays high enough to avoid visible blocking on UI screenshots;
42 * if even that overflows we drop resolution instead of quality.
43 */
44const QUALITY_STEPS = [0.92, 0.85, 0.78, 0.68] as const;
45/** Extra downscale passes applied when even the lowest quality overflows. */
46const FALLBACK_SCALE_STEPS = [0.75, 0.55] as const;
47const HEIC_IMAGE_MIME_TYPE = /^image\/hei(?:c|f)$/i;
48const HEIC_IMAGE_EXTENSION = /\.(?:heic|heif)$/i;
49
50type ImageSize = { width: number; height: number };
51
52export interface CompressedStashImage {
53 imageSize?: ImageSize;
54 dataUrl: string;
55 mimeType: string;
56 sizeBytes: number;
57 /** True when the payload was re-encoded rather than stored verbatim. */
58 recompressed: boolean;
59}
60
61/**
62 * Why an image could not be compressed. Callers report these differently:
63 * "too large" is a budget outcome, "unreadable" is a decode failure.
64 */
65export type ImageCompressionFailureReason = "too-large" | "unreadable";
66
67export type CompressStashImageResult =
68 | { ok: true; image: CompressedStashImage }
69 | { ok: false; reason: ImageCompressionFailureReason };
70
71export type CompressImageFileResult =
72 | { ok: true; file: File; recompressed: boolean; imageSize?: ImageSize }
73 | { ok: false; reason: ImageCompressionFailureReason };
74
75/** Finder and some browsers omit the MIME type when dragging HEIC photos. */
76export function isHeicImageFile(file: Pick<File, "name" | "type">): boolean {
77 if (HEIC_IMAGE_MIME_TYPE.test(file.type)) {
78 return true;
79 }
80 return (
81 (file.type === "" || file.type.toLowerCase() === "application/octet-stream") &&
82 HEIC_IMAGE_EXTENSION.test(file.name)
83 );
84}
85
86interface HeicMetadataBox {
87 payloadOffset: number;
88 endOffset: number;
89}
90
91function findHeicMetadataBox(
92 view: DataView,
93 startOffset: number,
94 endOffset: number,
95 type: number,
96): HeicMetadataBox | null {
97 let offset = startOffset;
98 while (offset + 8 <= endOffset) {
99 let size = view.getUint32(offset);
100 let headerSize = 8;
101 if (size === 1) {
102 if (offset + 16 > endOffset) return null;
103 const extendedSize = view.getBigUint64(offset + 8);
104 if (extendedSize > BigInt(Number.MAX_SAFE_INTEGER)) return null;
105 size = Number(extendedSize);
106 headerSize = 16;
107 } else if (size === 0) {
108 size = endOffset - offset;
109 }
110 if (size < headerSize || size > endOffset - offset) return null;
111
112 const nextOffset = offset + size;
113 if (view.getUint32(offset + 4) === type) {
114 return { payloadOffset: offset + headerSize, endOffset: nextOffset };
115 }
116 offset = nextOffset;
117 }
118 return null;
119}
120
121/** Read HEIC image dimensions before the decoder allocates full RGBA buffers. */
122async function validateHeicImageDimensions(
123 file: File,
124): Promise<ImageCompressionFailureReason | null> {
125 const metadata = await file.slice(0, MAX_HEIC_METADATA_BYTES).arrayBuffer();
126 const view = new DataView(metadata);
127 const meta = findHeicMetadataBox(view, 0, view.byteLength, 0x6d657461);
128 if (!meta || meta.payloadOffset + 4 > meta.endOffset) return "unreadable";
129 const properties = findHeicMetadataBox(view, meta.payloadOffset + 4, meta.endOffset, 0x69707270);
130 if (!properties) return "unreadable";
131 const containers = findHeicMetadataBox(
132 view,
133 properties.payloadOffset,
134 properties.endOffset,
135 0x6970636f,
136 );
137 if (!containers) return "unreadable";
138
139 let offset = containers.payloadOffset;
140 let foundImageDimensions = false;
141 while (offset < containers.endOffset) {
142 const image = findHeicMetadataBox(view, offset, containers.endOffset, 0x69737065);
143 if (!image) break;
144 if (image.payloadOffset + 12 > image.endOffset) return "unreadable";
145 const width = view.getUint32(image.payloadOffset + 4);
146 const height = view.getUint32(image.payloadOffset + 8);
147 if (width === 0 || height === 0) return "unreadable";
148 if (width > MAX_HEIC_DECODE_PIXELS / height) return "too-large";
149 foundImageDimensions = true;
150 offset = image.endOffset;
151 }
152 return foundImageDimensions ? null : "unreadable";
153}
154
155export function readFileAsDataUrl(file: Blob): Promise<string> {
156 return new Promise((resolve, reject) => {
157 const reader = new FileReader();
158 reader.addEventListener("load", () => {
159 if (typeof reader.result === "string") {
160 resolve(reader.result);
161 return;
162 }
163 reject(new Error("Could not read image data."));
164 });
165 reader.addEventListener("error", () => {
166 reject(reader.error ?? new Error("Failed to read image."));
167 });
168 reader.readAsDataURL(file);
169 });
170}
171
172function dataUrlLength(blob: Blob): number {
173 return (
174 `data:${blob.type || "application/octet-stream"};base64,`.length + 4 * Math.ceil(blob.size / 3)
175 );
176}
177
178/** Base64 payload of a data URL decoded back into a `File`. */
179export function dataUrlToFile(dataUrl: string, name: string, mimeType: string): File {
180 const payload = dataUrl.slice(dataUrl.indexOf(",") + 1);
181 const binary = atob(payload);
182 const bytes = new Uint8Array(binary.length);
183 for (let index = 0; index < binary.length; index += 1) {
184 bytes[index] = binary.charCodeAt(index);
185 }
186 return new File([bytes], name, { type: mimeType });
187}
188
189/**
190 * Re-encoding changes the container, so a name like `shot.png` would lie
191 * about its contents. Swap the extension to match the encoded mime type.
192 */
193function fileNameForMimeType(name: string, mimeType: string): string {
194 const extension = mimeType === "image/webp" ? ".webp" : ".jpg";
195 const dotIndex = name.lastIndexOf(".");
196 const base = dotIndex > 0 ? name.slice(0, dotIndex) : name;
197 return `${base}${extension}`;
198}
199
200function canRecompress(): boolean {
201 return (
202 typeof createImageBitmap === "function" &&
203 (typeof OffscreenCanvas === "function" || typeof document !== "undefined")
204 );
205}
206
207interface Canvas2D {
208 canvas: OffscreenCanvas | HTMLCanvasElement;
209 context: OffscreenCanvasRenderingContext2D | CanvasRenderingContext2D;
210}
211
212function createCanvas(width: number, height: number): Canvas2D | null {
213 if (typeof OffscreenCanvas === "function") {
214 const canvas = new OffscreenCanvas(width, height);
215 const context = canvas.getContext("2d");
216 if (!context) return null;
217 return { canvas, context };
218 }
219 if (typeof document === "undefined") return null;
220 const canvas = document.createElement("canvas");
221 canvas.width = width;
222 canvas.height = height;
223 const context = canvas.getContext("2d");
224 if (!context) return null;
225 return { canvas, context };
226}
227
228/**
229 * WebP is preferred: at matched visual quality it lands roughly 25-35%
230 * smaller than JPEG, so the same budget buys more resolution and detail —
231 * and it keeps alpha, so screenshots with transparency survive intact.
232 * Browsers that can't encode it silently fall back to JPEG.
233 */
234async function encodeCanvas(
235 canvas: OffscreenCanvas | HTMLCanvasElement,
236 quality: number,
237 mimeType: string,
238): Promise<Blob | null> {
239 let blob: Blob | null;
240 if (typeof HTMLCanvasElement !== "undefined" && canvas instanceof HTMLCanvasElement) {
241 blob = await new Promise<Blob | null>((resolve) => canvas.toBlob(resolve, mimeType, quality));
242 } else {
243 blob = await (canvas as OffscreenCanvas).convertToBlob({ type: mimeType, quality });
244 }
245 return blob?.type === mimeType ? blob : null;
246}
247
248const composerThumbnails = new WeakMap<File, Promise<string | null>>();
249
250/** Cache a centered square crop for the composer's object-cover image tiles. */
251export function createComposerImageThumbnail(file: File): Promise<string | null> {
252 const cached = composerThumbnails.get(file);
253 if (cached) return cached;
254 const thumbnail = (async () => {
255 if (!canRecompress()) return null;
256 let bitmap: ImageBitmap | undefined;
257 try {
258 bitmap = await createImageBitmap(file);
259 const side = Math.min(bitmap.width, bitmap.height);
260 if (side <= 0) return null;
261 const dimension = Math.min(256, side);
262 const surface = createCanvas(dimension, dimension);
263 if (!surface) return null;
264 surface.context.drawImage(
265 bitmap,
266 (bitmap.width - side) / 2,
267 (bitmap.height - side) / 2,
268 side,
269 side,
270 0,
271 0,
272 dimension,
273 dimension,
274 );
275 const blob = await encodeCanvas(surface.canvas, 1, "image/png");
276 return blob ? await readFileAsDataUrl(blob) : null;
277 } catch {
278 return null;
279 } finally {
280 bitmap?.close();
281 }
282 })();
283 composerThumbnails.set(file, thumbnail);
284 return thumbnail;
285}
286
287/**
288 * Draws `bitmap` scaled to fit `maxDimension` and encodes it, stepping
289 * quality down until the data URL fits `budgetChars`.
290 */
291async function encodeWithinBudget(
292 bitmap: ImageBitmap,
293 maxDimension: number,
294 budgetChars: number,
295 preferredMimeType?: "image/jpeg",
296): Promise<{ blob: Blob; mimeType: string; imageSize: ImageSize } | null> {
297 const scale = Math.min(1, maxDimension / Math.max(bitmap.width, bitmap.height));
298 const width = Math.max(1, Math.round(bitmap.width * scale));
299 const height = Math.max(1, Math.round(bitmap.height * scale));
300 const target = createCanvas(width, height);
301 if (!target) return null;
302
303 // Probe WebP once; JPEG (no alpha) needs a white matte, so the fill has to
304 // happen before drawing and depends on which codec we end up using.
305 const mimeType =
306 preferredMimeType ??
307 ((await encodeCanvas(target.canvas, QUALITY_STEPS[0], "image/webp"))
308 ? "image/webp"
309 : "image/jpeg");
310
311 if (mimeType === "image/jpeg") {
312 target.context.fillStyle = "#ffffff";
313 target.context.fillRect(0, 0, width, height);
314 }
315 target.context.drawImage(bitmap, 0, 0, width, height);
316
317 for (const quality of QUALITY_STEPS) {
318 const encoded = await encodeCanvas(target.canvas, quality, mimeType);
319 if (!encoded) break;
320 if (dataUrlLength(encoded) <= budgetChars) {
321 return { blob: encoded, mimeType, imageSize: { width, height } };
322 }
323 }
324 return null;
325}
326
327type ReencodeResult =
328 | { ok: true; blob: Blob; mimeType: string; imageSize: ImageSize }
329 | { ok: false; reason: ImageCompressionFailureReason };
330
331/**
332 * Shared re-encode loop: decodes `file`, then walks the quality ladder and
333 * fallback downscale passes until an encoding fits `budgetChars`.
334 */
335async function reencodeWithinBudget(
336 file: File,
337 budgetChars: number,
338 preferredMimeType?: "image/jpeg",
339): Promise<ReencodeResult> {
340 if (!canRecompress()) {
341 return { ok: false, reason: "too-large" };
342 }
343
344 let bitmap: ImageBitmap;
345 try {
346 bitmap = await createImageBitmap(file);
347 } catch {
348 return { ok: false, reason: "unreadable" };
349 }
350
351 try {
352 // Each pass shrinks relative to the *previous target*, capped by
353 // MAX_DIMENSION. Scaling a fixed ceiling instead would be a no-op for
354 // images already smaller than that ceiling — the fallback passes would
355 // all resolve to the source size and never actually reduce resolution.
356 const baseDimension = Math.min(MAX_DIMENSION, Math.max(bitmap.width, bitmap.height));
357 // Tracks whether the *last* attempt threw, so a run of encoder failures
358 // is reported as unreadable while a run of merely-too-big results is
359 // reported as too-large.
360 let encodeFailed = false;
361 for (const dimensionScale of [1, ...FALLBACK_SCALE_STEPS]) {
362 const targetDimension = Math.max(1, Math.round(baseDimension * dimensionScale));
363 let encoded: Awaited<ReturnType<typeof encodeWithinBudget>>;
364 try {
365 encoded = await encodeWithinBudget(bitmap, targetDimension, budgetChars, preferredMimeType);
366 } catch {
367 // Canvas allocation, drawing, or the codec itself can throw — often
368 // precisely *because* the target is too big (OOM on a large bitmap).
369 // Keep trying the smaller fallback scales rather than giving up: a
370 // reduced pass may well succeed. The exception must never escape,
371 // though, since callers finalize state after this returns and a
372 // throw would strand it (e.g. a stash entry stuck "still saving").
373 encodeFailed = true;
374 continue;
375 }
376 encodeFailed = false;
377 if (encoded) {
378 return {
379 ok: true,
380 blob: encoded.blob,
381 mimeType: encoded.mimeType,
382 imageSize: encoded.imageSize,
383 };
384 }
385 }
386 return { ok: false, reason: encodeFailed ? "unreadable" : "too-large" };
387 } finally {
388 bitmap.close();
389 }
390}
391
392/**
393 * Produces the payload to persist for a stashed image.
394 *
395 * Small images are stored verbatim (preserving PNG transparency and exact
396 * pixels). Anything over budget is downscaled and re-encoded; if it still
397 * doesn't fit after the fallback passes, reports a failure so the caller
398 * can record it as dropped.
399 */
400export async function compressImageForStash(
401 file: File,
402 budgetChars: number = MAX_STASH_IMAGE_DATA_URL_CHARS,
403): Promise<CompressStashImageResult> {
404 try {
405 if (dataUrlLength(file) <= budgetChars) {
406 return {
407 ok: true,
408 image: {
409 dataUrl: await readFileAsDataUrl(file),
410 mimeType: file.type,
411 sizeBytes: file.size,
412 recompressed: false,
413 },
414 };
415 }
416 const reencoded = await reencodeWithinBudget(file, budgetChars);
417 if (!reencoded.ok) {
418 return reencoded;
419 }
420 return {
421 ok: true,
422 image: {
423 dataUrl: await readFileAsDataUrl(reencoded.blob),
424 mimeType: reencoded.mimeType,
425 sizeBytes: reencoded.blob.size,
426 recompressed: true,
427 imageSize: reencoded.imageSize,
428 },
429 };
430 } catch {
431 return { ok: false, reason: "unreadable" };
432 }
433}
434
435/**
436 * Scales a PNG or JPEG whose long edge is over `IMAGE_ATTACHMENT_MAX_EDGE`
437 * down to that edge in its own format, so a PNG keeps its transparency.
438 * Returns null when the image already fits, is another format (GIF, WebP,
439 * SVG), or cannot be decoded or encoded here; the caller keeps the original.
440 */
441async function fitImageToMaxEdge(file: File): Promise<{ file: File; imageSize: ImageSize } | null> {
442 const mimeType = downscalableImageMimeType(file.type);
443 if (!mimeType || !canRecompress() || file.size > MAX_COMPRESSIBLE_SOURCE_BYTES) return null;
444 let bitmap: ImageBitmap | undefined;
445 try {
446 // The header answers "does it fit?" without decoding every attachment.
447 const header = readImageDimensions(
448 new Uint8Array(await file.slice(0, IMAGE_DIMENSIONS_HEADER_BYTES).arrayBuffer()),
449 );
450 if (!header || !fitImageAttachmentSize(header)) return null;
451 bitmap = await createImageBitmap(file, { imageOrientation: "from-image" });
452 const imageSize = fitImageAttachmentSize(bitmap);
453 if (!imageSize) return null;
454 const target = createCanvas(imageSize.width, imageSize.height);
455 if (!target) return null;
456 target.context.imageSmoothingQuality = "high";
457 target.context.drawImage(bitmap, 0, 0, imageSize.width, imageSize.height);
458 const blob = await encodeCanvas(target.canvas, QUALITY_STEPS[0], mimeType);
459 if (!blob) return null;
460 return {
461 file: new File([blob], file.name, { type: mimeType, lastModified: file.lastModified }),
462 imageSize,
463 };
464 } catch {
465 return null;
466 } finally {
467 bitmap?.close();
468 }
469}
470
471/**
472 * Shrinks `file` until its binary size fits `maxBytes`, returning a new
473 * `File` (WebP or JPEG). A PNG or JPEG over `IMAGE_ATTACHMENT_MAX_EDGE` is
474 * first scaled to that edge in its own format. Other files already within the
475 * limit pass through untouched, preserving their exact bytes and format. Sources above
476 * `MAX_COMPRESSIBLE_SOURCE_BYTES` are refused outright — decoding them is
477 * the risk, so no amount of output budget makes them safe. An internally
478 * converted image can provide its original source size when the intermediate
479 * format expands beyond that ceiling.
480 */
481export async function compressImageToByteLimit(
482 file: File,
483 maxBytes: number,
484 options?: { preferredMimeType?: "image/jpeg"; sourceSizeBytes?: number },
485): Promise<CompressImageFileResult> {
486 const fitted = await fitImageToMaxEdge(file);
487 if (fitted && fitted.file.size <= maxBytes) {
488 return { ok: true, file: fitted.file, recompressed: true, imageSize: fitted.imageSize };
489 }
490 if (!fitted && file.size <= maxBytes) {
491 return { ok: true, file, recompressed: false };
492 }
493 if ((options?.sourceSizeBytes ?? file.size) > MAX_COMPRESSIBLE_SOURCE_BYTES) {
494 return { ok: false, reason: "too-large" };
495 }
496 // The re-encode loop budgets in data-URL characters. Base64 turns 3 bytes
497 // into 4 chars; flooring keeps the budget a hair conservative instead of
498 // admitting an encoding right at the byte cap.
499 const budgetChars = Math.floor(maxBytes / 3) * 4;
500 const reencoded = await reencodeWithinBudget(
501 fitted?.file ?? file,
502 budgetChars,
503 options?.preferredMimeType,
504 );
505 if (!reencoded.ok) {
506 return reencoded;
507 }
508 return {
509 ok: true,
510 file: new File(
511 [reencoded.blob],
512 fileNameForMimeType(file.name || "image", reencoded.mimeType),
513 { type: reencoded.mimeType },
514 ),
515 recompressed: true,
516 imageSize: reencoded.imageSize,
517 };
518}
519
520/**
521 * Converts HEIC/HEIF photos to provider-compatible JPEG before applying the
522 * attachment size limit. The decoder is loaded only when such a photo arrives.
523 */
524export async function prepareImageForAttachment(
525 file: File,
526 maxBytes: number,
527): Promise<CompressImageFileResult> {
528 if (!isHeicImageFile(file)) {
529 return compressImageToByteLimit(file, maxBytes);
530 }
531
532 if (file.size > MAX_COMPRESSIBLE_SOURCE_BYTES) {
533 return { ok: false, reason: "too-large" };
534 }
535
536 let converted: Blob;
537 try {
538 const dimensionError = await validateHeicImageDimensions(file);
539 if (dimensionError) {
540 return { ok: false, reason: dimensionError };
541 }
542 const { heicTo } = await import("heic-to/csp");
543 converted = await heicTo({ blob: file, type: "image/jpeg", quality: QUALITY_STEPS[0] });
544 } catch {
545 return { ok: false, reason: "unreadable" };
546 }
547
548 const jpeg = new File([converted], fileNameForMimeType(file.name || "image", "image/jpeg"), {
549 type: "image/jpeg",
550 lastModified: file.lastModified,
551 });
552 const result = await compressImageToByteLimit(jpeg, maxBytes, {
553 preferredMimeType: "image/jpeg",
554 sourceSizeBytes: file.size,
555 });
556
557 return result.ok ? { ...result, recompressed: true } : result;
558}