authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-09-04 12:13:24-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-04 12:13:24-07:00
log15eda897d37aad232fca5089f5eb6ef7b726c31c
tree434de06d2fc2e87cb85ecc65bbf7d37c17d449d4
parent6365919f2e5bcfb4fa4020b95e19af26ae40979f
signature Signed by PGP key B5690EEEBB952194

perf(web): defer image URL requests for thread history (#9760)

4 files changed, 196 insertions(+), 30 deletions(-)

apps/web/src/components/ChatView.tsx+6-20
...@@ -104,6 +104,7 @@ import {...@@ -104,6 +104,7 @@ import {
104 deriveWorkLogEntries,104 deriveWorkLogEntries,
105 hasActionableProposedPlan,105 hasActionableProposedPlan,
106 isLatestTurnSettled,106 isLatestTurnSettled,
107 selectHandoffImageResources,
107 type TimelineEntriesProjection,108 type TimelineEntriesProjection,
108} from "../session-logic";109} from "../session-logic";
109import { type LegendListRef } from "@legendapp/list/react";110import { type LegendListRef } from "@legendapp/list/react";
...@@ -2763,35 +2764,20 @@ export default function ChatView(props: ChatViewProps) {...@@ -2763,35 +2764,20 @@ export default function ChatView(props: ChatViewProps) {
2763 },2764 },
2764 [activeThreadRef, downloadFileAttachment],2765 [activeThreadRef, downloadFileAttachment],
2765 );2766 );
2766 const serverAttachmentIds = useMemo(() => {
2767 const attachmentIds = new Set<string>();
2768 for (const message of serverMessages ?? []) {
2769 for (const attachment of message.attachments ?? []) {
2770 if (isImageAttachment(attachment)) {
2771 attachmentIds.add(attachment.id);
2772 }
2773 }
2774 }
2775 return [...attachmentIds];
2776 }, [serverMessages]);
2777 const serverAttachmentResources = useMemo(2767 const serverAttachmentResources = useMemo(
2778 () =>2768 () => selectHandoffImageResources(serverMessages, attachmentPreviewHandoffByMessageId),
2779 serverAttachmentIds.map((attachmentId) => ({2769 [serverMessages, attachmentPreviewHandoffByMessageId],
2780 _tag: "attachment" as const,
2781 attachmentId,
2782 })),
2783 [serverAttachmentIds],
2784 );2770 );
2785 const serverAttachmentUrls = useAssetUrls(environmentId, serverAttachmentResources);2771 const serverAttachmentUrls = useAssetUrls(environmentId, serverAttachmentResources);
2786 const serverAttachmentUrlById = useMemo(2772 const serverAttachmentUrlById = useMemo(
2787 () =>2773 () =>
2788 new Map(2774 new Map(
2789 serverAttachmentIds.flatMap((attachmentId, index) => {2775 serverAttachmentResources.flatMap((resource, index) => {
2790 const url = serverAttachmentUrls[index];2776 const url = serverAttachmentUrls[index];
2791 return url ? [[attachmentId, url] as const] : [];2777 return url ? [[resource.attachmentId, url] as const] : [];
2792 }),2778 }),
2793 ),2779 ),
2794 [serverAttachmentIds, serverAttachmentUrls],2780 [serverAttachmentResources, serverAttachmentUrls],
2795 );2781 );
2796 const displayServerMessages = useMemo<ReadonlyArray<ChatMessage>>(() => {2782 const displayServerMessages = useMemo<ReadonlyArray<ChatMessage>>(() => {
2797 if (!serverMessages) return [];2783 if (!serverMessages) return [];
apps/web/src/components/chat/MessagesTimeline.tsx+19-2
...@@ -52,7 +52,9 @@ import {...@@ -52,7 +52,9 @@ import {
52import { FileDiff } from "@pierre/diffs/react";52import { FileDiff } from "@pierre/diffs/react";
53import { DiffWorkerPoolProvider } from "../DiffWorkerPoolProvider";53import { DiffWorkerPoolProvider } from "../DiffWorkerPoolProvider";
54import {54import {
55 createMessageAttachmentPreviewProjector,
55 deriveTimelineEntries,56 deriveTimelineEntries,
57 selectMessageImageResources,
56 workEntryDisplayIndicatesToolFailure,58 workEntryDisplayIndicatesToolFailure,
57 workEntrySignalsSevereFailure,59 workEntrySignalsSevereFailure,
58 workLogEntryIsToolLike,60 workLogEntryIsToolLike,
...@@ -99,7 +101,7 @@ import {...@@ -99,7 +101,7 @@ import {
99 ZapIcon,101 ZapIcon,
100} from "lucide-react";102} from "lucide-react";
101import { Button } from "../ui/button";103import { Button } from "../ui/button";
102import { useAssetUrlRefresh, useAssetUrlState } from "../../assets/assetUrls";104import { useAssetUrlRefresh, useAssetUrls, useAssetUrlState } from "../../assets/assetUrls";
103import { MediaVideoPlayer } from "../media/MediaVideoPlayer";105import { MediaVideoPlayer } from "../media/MediaVideoPlayer";
104import { getVirtualizedScrollFadeClassName } from "../ui/scroll-area";106import { getVirtualizedScrollFadeClassName } from "../ui/scroll-area";
105import {107import {
...@@ -1227,9 +1229,24 @@ function UserVideoAttachment({ file }: { readonly file: ChatFileAttachment }) {...@@ -1227,9 +1229,24 @@ function UserVideoAttachment({ file }: { readonly file: ChatFileAttachment }) {
12271229
1228function UserTimelineRow({ row }: { row: Extract<TimelineRow, { kind: "message" }> }) {1230function UserTimelineRow({ row }: { row: Extract<TimelineRow, { kind: "message" }> }) {
1229 const ctx = use(TimelineRowCtx);1231 const ctx = use(TimelineRowCtx);
1232 const resources = useMemo(
1233 () => selectMessageImageResources(row.message.attachments),
1234 [row.message.attachments],
1235 );
1236 const previewUrls = useAssetUrls(ctx.activeThreadEnvironmentId, resources);
1237 const [projectPreviews] = useState(createMessageAttachmentPreviewProjector);
1238 const messageWithPreviews = useMemo(() => {
1239 const urlsById = new Map(
1240 resources.flatMap((resource, index) => {
1241 const url = previewUrls[index];
1242 return url ? [[resource.attachmentId, url] as const] : [];
1243 }),
1244 );
1245 return projectPreviews(row.message, (attachment) => urlsById.get(attachment.id));
1246 }, [previewUrls, projectPreviews, resources, row.message]);
1230 // The attachment union has an open member, so guards (not literal type1247 // The attachment union has an open member, so guards (not literal type
1231 // comparisons) split it. Unknown types render as inert rows below the files.1248 // comparisons) split it. Unknown types render as inert rows below the files.
1232 const userImages = (row.message.attachments ?? []).filter(isImageAttachment);1249 const userImages = (messageWithPreviews.attachments ?? []).filter(isImageAttachment);
1233 const userFiles = (row.message.attachments ?? []).filter(isFileAttachment);1250 const userFiles = (row.message.attachments ?? []).filter(isFileAttachment);
1234 const userVideos = userFiles.filter(isVideoAttachment);1251 const userVideos = userFiles.filter(isVideoAttachment);
1235 const otherUserFiles = userFiles.filter((file) => !isVideoAttachment(file));1252 const otherUserFiles = userFiles.filter((file) => !isVideoAttachment(file));
apps/web/src/session-logic.test.ts+124
...@@ -21,6 +21,8 @@ import {...@@ -21,6 +21,8 @@ import {
21 findLatestProposedPlan,21 findLatestProposedPlan,
22 hasActionableProposedPlan,22 hasActionableProposedPlan,
23 isLatestTurnSettled,23 isLatestTurnSettled,
24 selectHandoffImageResources,
25 selectMessageImageResources,
24 workEntryIndicatesToolFailure,26 workEntryIndicatesToolFailure,
25 workEntryIndicatesToolNeutralStatus,27 workEntryIndicatesToolNeutralStatus,
26 workEntryIndicatesToolSuccess,28 workEntryIndicatesToolSuccess,
...@@ -2075,6 +2077,128 @@ describe("deriveWorkLogEntries", () => {...@@ -2075,6 +2077,128 @@ describe("deriveWorkLogEntries", () => {
2075 });2077 });
2076});2078});
20772079
2080describe("image asset requests", () => {
2081 const image = {
2082 type: "image" as const,
2083 id: "image",
2084 name: "image.png",
2085 mimeType: "image/png",
2086 sizeBytes: 42,
2087 };
2088 const message = {
2089 id: MessageId.make("image-message"),
2090 role: "user" as const,
2091 text: "Inspect these images",
2092 turnId: null,
2093 createdAt: "2026-09-04T00:00:00.000Z",
2094 updatedAt: "2026-09-04T00:00:00.000Z",
2095 streaming: false,
2096 attachments: [image],
2097 };
2098
2099 it("requests the whole row's gallery and crops without signing local preview IDs", () => {
2100 const attachments = Object.freeze([
2101 image,
2102 { ...image, id: "second" },
2103 { ...image, id: "crop", name: "preview-annotation-1.png" },
2104 { ...image, id: "local", previewUrl: "blob:local" },
2105 { ...image, id: "inline", previewUrl: "data:image/png;base64,AA==" },
2106 { ...image, id: "provided", previewUrl: "https://preview.test/image" },
2107 { ...image, type: "file" as const, id: "file", mimeType: "application/pdf" },
2108 { ...image, type: "future", id: "unknown" },
2109 image,
2110 ]);
2111
2112 expect(selectMessageImageResources(attachments)).toEqual([
2113 { _tag: "attachment", attachmentId: "image" },
2114 { _tag: "attachment", attachmentId: "second" },
2115 { _tag: "attachment", attachmentId: "crop" },
2116 { _tag: "attachment", attachmentId: "provided" },
2117 ]);
2118 });
2119
2120 it("requests offscreen handoffs without signing the rest of the loaded history", () => {
2121 const history = {
2122 ...message,
2123 id: MessageId.make("history"),
2124 attachments: [{ ...image, id: "history-image" }],
2125 };
2126 const offscreen = {
2127 ...message,
2128 id: MessageId.make("offscreen"),
2129 attachments: [image, { ...image, id: "crop", name: "preview-annotation-1.png" }],
2130 };
2131 const empty = {
2132 ...message,
2133 id: MessageId.make("empty"),
2134 attachments: [{ ...image, id: "empty" }],
2135 };
2136 const assistant = {
2137 ...message,
2138 id: MessageId.make("assistant"),
2139 role: "assistant" as const,
2140 attachments: [{ ...image, id: "assistant-image" }],
2141 };
2142 expect(
2143 selectHandoffImageResources([history, message, offscreen, empty, assistant], {
2144 [message.id]: ["blob:message"],
2145 [offscreen.id]: ["blob:offscreen", "blob:crop"],
2146 [empty.id]: [],
2147 [assistant.id]: ["blob:unused"],
2148 }),
2149 ).toEqual([
2150 { _tag: "attachment", attachmentId: "image" },
2151 { _tag: "attachment", attachmentId: "crop" },
2152 ]);
2153 });
2154
2155 it("does not scan history when no handoff is pending", () => {
2156 let reads = 0;
2157 const messages = new Proxy([message], {
2158 get(target, property, receiver) {
2159 if (property === "0") reads += 1;
2160 return Reflect.get(target, property, receiver);
2161 },
2162 });
2163 const empty = selectHandoffImageResources(messages, {});
2164 expect(reads).toBe(0);
2165 expect(empty).toHaveLength(0);
2166 expect(selectHandoffImageResources(undefined, { missing: ["blob:missing"] })).toBe(empty);
2167 expect(selectMessageImageResources(undefined)).toBe(empty);
2168 });
2169
2170 it("hands signed URLs to a mounted row only after the local preview is released", () => {
2171 const server = createMessageAttachmentPreviewProjector();
2172 const handoff = createMessageAttachmentPreviewProjector();
2173 const row = createMessageAttachmentPreviewProjector();
2174 const pending = handoff(
2175 server(message, () => undefined),
2176 () => "blob:pending",
2177 );
2178 expect(selectMessageImageResources(pending.attachments)).toEqual([]);
2179 expect(selectHandoffImageResources([message], { [message.id]: ["blob:pending"] })).toEqual([
2180 { _tag: "attachment", attachmentId: image.id },
2181 ]);
2182
2183 const ready = server(message, () => "https://server.test/image");
2184 expect(selectMessageImageResources(handoff(ready, () => "blob:pending").attachments)).toEqual(
2185 [],
2186 );
2187 const released = server(message, () => undefined);
2188 expect(selectMessageImageResources(released.attachments)).toEqual([
2189 { _tag: "attachment", attachmentId: image.id },
2190 ]);
2191 const displayed = row(released, () => "https://server.test/image");
2192 expect(displayed).toEqual(ready);
2193 expect(pending.attachments?.[0]).toMatchObject({ previewUrl: "blob:pending" });
2194 expect(row(released, () => "https://server.test/renewed").attachments?.[0]).toMatchObject({
2195 previewUrl: "https://server.test/renewed",
2196 });
2197 expect(displayed.attachments?.[0]).toMatchObject({ previewUrl: "https://server.test/image" });
2198 expect(row(released, () => undefined)).toBe(message);
2199 });
2200});
2201
2078describe("deriveTimelineEntries", () => {2202describe("deriveTimelineEntries", () => {
2079 const streamingMessage = {2203 const streamingMessage = {
2080 id: MessageId.make("streaming-message"),2204 id: MessageId.make("streaming-message"),
apps/web/src/session-logic.ts+47-8
...@@ -12,6 +12,7 @@ import { extractToolActivityPresentation } from "@t3tools/client-runtime/work-lo...@@ -12,6 +12,7 @@ import { extractToolActivityPresentation } from "@t3tools/client-runtime/work-lo
12import {12import {
13 ApprovalRequestId,13 ApprovalRequestId,
14 isToolLifecycleItemType,14 isToolLifecycleItemType,
15 type AssetResource,
15 type OrchestrationLatestTurn,16 type OrchestrationLatestTurn,
16 type OrchestrationThreadActivity,17 type OrchestrationThreadActivity,
17 type OrchestrationProposedPlanId,18 type OrchestrationProposedPlanId,
...@@ -24,14 +25,15 @@ import {...@@ -24,14 +25,15 @@ import {
24 type TurnId,25 type TurnId,
25} from "@t3tools/contracts";26} from "@t3tools/contracts";
2627
27import type {28import {
28 ChatAttachment,29 isImageAttachment,
29 ChatMessage,30 type ChatAttachment,
30 ProposedPlan,31 type ChatMessage,
31 SessionPhase,32 type ProposedPlan,
32 Thread,33 type SessionPhase,
33 ThreadSession,34 type Thread,
34 TurnDiffSummary,35 type ThreadSession,
36 type TurnDiffSummary,
35} from "./types";37} from "./types";
3638
37export type ProviderPickerKind = ProviderDriverKind;39export type ProviderPickerKind = ProviderDriverKind;
...@@ -1883,6 +1885,43 @@ function mergeTimelineEntrySuffix(...@@ -1883,6 +1885,43 @@ function mergeTimelineEntrySuffix(
1883 return merged;1885 return merged;
1884}1886}
18851887
1888type AttachmentResource = Extract<AssetResource, { readonly _tag: "attachment" }>;
1889const EMPTY_IMAGE_RESOURCES = Object.freeze<ReadonlyArray<AttachmentResource>>([]);
1890
1891/** A mounted row requests its stored images. Local previews keep their existing URLs. */
1892export function selectMessageImageResources(
1893 attachments: ChatMessage["attachments"],
1894): ReadonlyArray<AttachmentResource> {
1895 const attachmentIds = new Set<string>();
1896 for (const attachment of attachments ?? []) {
1897 if (!isImageAttachment(attachment)) continue;
1898 const previewUrl = attachment.previewUrl;
1899 if (previewUrl?.startsWith("blob:") || previewUrl?.startsWith("data:")) continue;
1900 attachmentIds.add(attachment.id);
1901 }
1902 return attachmentIds.size === 0
1903 ? EMPTY_IMAGE_RESOURCES
1904 : Array.from(attachmentIds, (attachmentId) => ({ _tag: "attachment", attachmentId }));
1905}
1906
1907/** Handoffs need server URLs even while their message rows are unmounted. */
1908export function selectHandoffImageResources(
1909 messages: ReadonlyArray<ChatMessage> | undefined,
1910 handoffs: Readonly<Record<string, ReadonlyArray<string>>>,
1911): ReadonlyArray<AttachmentResource> {
1912 if (Object.keys(handoffs).length === 0) return EMPTY_IMAGE_RESOURCES;
1913 const attachmentIds = new Set<string>();
1914 for (const message of messages ?? []) {
1915 if (message.role !== "user" || !handoffs[message.id]?.length) continue;
1916 for (const attachment of message.attachments ?? []) {
1917 if (isImageAttachment(attachment)) attachmentIds.add(attachment.id);
1918 }
1919 }
1920 return attachmentIds.size === 0
1921 ? EMPTY_IMAGE_RESOURCES
1922 : Array.from(attachmentIds, (attachmentId) => ({ _tag: "attachment", attachmentId }));
1923}
1924
1886/** Own one mapper per preview stage. Immutable messages retain unchanged preview objects. */1925/** Own one mapper per preview stage. Immutable messages retain unchanged preview objects. */
1887export function createMessageAttachmentPreviewProjector() {1926export function createMessageAttachmentPreviewProjector() {
1888 const attachmentsBySource = new WeakMap<1927 const attachmentsBySource = new WeakMap<