authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-10-05 14:49:14-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-05 14:49:14-07:00
log5eb87730871ac6125266b48b98d067de00a19ec7
tree24f77464787b8e0530bb38396d1255040b51c42f
parent2a68a777afda95aa23538b9954c5a15221ed2857
signature Signed by PGP key B5690EEEBB952194

fix: tool screenshots show as images, not base64 text (#16199)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

17 files changed, 551 insertions(+), 21 deletions(-)

apps/mobile/src/features/threads/ThreadFeed.tsx+1
......@@ -1933,6 +1933,7 @@ function renderFeedEntry(
19331933 onToggleRow={props.onToggleWorkRow}
19341934 renderImage={props.renderViewedImage}
19351935 renderReasoning={props.renderReasoning}
1936 onPressPreview={props.onPressPreview}
19361937 />
19371938 );
19381939}
apps/mobile/src/features/threads/ThreadMarkdownImage.tsx+7-2
......@@ -178,7 +178,10 @@ function ThreadMarkdownImageRequest(props: {
178178/** Environment-hosted image that loads through a signed asset URL. */
179179export function ThreadMarkdownImage(props: {
180180 readonly environmentId: EnvironmentId;
181 readonly resource: Extract<AssetResource, { readonly _tag: "attachment" | "media-file" }>;
181 readonly resource: Extract<
182 AssetResource,
183 { readonly _tag: "attachment" | "media-file" | "tool-output-image" }
184 >;
182185 readonly alt: string | null;
183186 readonly srcFragment?: string;
184187 readonly actionsSource?: MediaActionsSource;
......@@ -192,7 +195,9 @@ export function ThreadMarkdownImage(props: {
192195 sourceKey={
193196 props.resource._tag === "attachment"
194197 ? `attachment:${props.resource.attachmentId}`
195 : `workspace:${props.resource.path}`
198 : props.resource._tag === "tool-output-image"
199 ? `tool-output:${props.resource.itemId}:${props.resource.index}`
200 : `workspace:${props.resource.path}`
196201 }
197202 unavailable={assetUrl._tag === "Failure"}
198203 knownSize={assetUrl._tag === "Success" ? assetUrl.imageDimensions : undefined}
apps/mobile/src/features/threads/thread-work-log.tsx+23-2
......@@ -64,7 +64,11 @@ import {
6464 type ThreadFeedActivity,
6565 workEntryRowLabel,
6666} from "../../lib/threadActivity";
67import { toolCallLines, turnItemOutputText } from "@t3tools/client-runtime/work-log/item-detail";
67import {
68 toolCallLines,
69 turnItemOutputImages,
70 turnItemOutputText,
71} from "@t3tools/client-runtime/work-log/item-detail";
6872import { useTurnItemDetail } from "../../state/queries";
6973import {
7074 resolveThreadWorkGroupInitialScroll,
......@@ -80,6 +84,8 @@ import {
8084import { resolveWorkGroupScrollAnchor } from "@t3tools/client-runtime/work-log/scroll-anchor";
8185import { notificationChildThreadId } from "@t3tools/client-runtime/state/thread-execution";
8286import type { MarkdownImageRenderer } from "../../native/SelectableMarkdownText";
87import type { FilePreviewSource } from "../../components/FilePreviewModal";
88import { ThreadMarkdownImage } from "./ThreadMarkdownImage";
8389import Animated, {
8490 cancelAnimation,
8591 Easing,
......@@ -462,6 +468,7 @@ interface ThreadWorkLogProps {
462468 readonly onToggleRow: (rowId: string, anchorKey: string) => void;
463469 readonly renderImage: MarkdownImageRenderer;
464470 readonly renderReasoning: (text: string) => ReactNode;
471 readonly onPressPreview: (source: FilePreviewSource) => void;
465472}
466473
467474export function ThreadWorkLog(props: ThreadWorkLogProps) {
......@@ -479,6 +486,7 @@ export function ThreadWorkLog(props: ThreadWorkLogProps) {
479486 onToggleRow={props.onToggleRow}
480487 renderImage={props.renderImage}
481488 renderReasoning={props.renderReasoning}
489 onPressPreview={props.onPressPreview}
482490 themeAppearance={props.themeAppearance}
483491 />
484492 ),
......@@ -492,6 +500,7 @@ export function ThreadWorkLog(props: ThreadWorkLogProps) {
492500 props.onToggleRow,
493501 props.renderImage,
494502 props.renderReasoning,
503 props.onPressPreview,
495504 props.themeAppearance,
496505 ],
497506 );
......@@ -951,12 +960,13 @@ const ThreadWorkLogRow = memo(function ThreadWorkLogRow(
951960 ? formatItemFullDetail(row.projectedItem, fetchedItem)
952961 : row.getFullDetail()
953962 : null;
963 const outputImages = expanded && fetchedItem ? turnItemOutputImages(fetchedItem) : [];
954964 const fetchedOutput = !expanded
955965 ? null
956966 : shownItem.type === "file_search" || shownItem.type === "web_search"
957967 ? turnItemOutputText(shownItem)
958968 : fetchedItem
959 ? (turnItemOutputText(fetchedItem) ?? "No output.")
969 ? (turnItemOutputText(fetchedItem) ?? (outputImages.length > 0 ? null : "No output."))
960970 : fetchedDetail.error
961971 ? `Couldn't load output: ${fetchedDetail.error}`
962972 : row.fetchesDetail
......@@ -1116,6 +1126,7 @@ const ThreadWorkLogRow = memo(function ThreadWorkLogRow(
11161126 call ||
11171127 fetchedOutput ||
11181128 viewedImagePath ||
1129 outputImages.length > 0 ||
11191130 row.workEntry.questionAnswer) ? (
11201131 <Animated.View
11211132 entering={WORK_LOG_DETAIL_ENTER_TRANSITION}
......@@ -1134,6 +1145,16 @@ const ThreadWorkLogRow = memo(function ThreadWorkLogRow(
11341145 {props.renderImage({ href: viewedImagePath, alt: null, title: null })}
11351146 </View>
11361147 ) : null}
1148 {outputImages.map((resource) => (
1149 <View key={resource.index} className="pb-1.5">
1150 <ThreadMarkdownImage
1151 environmentId={props.environmentId}
1152 resource={resource}
1153 alt={null}
1154 onPressPreview={props.onPressPreview}
1155 />
1156 </View>
1157 ))}
11371158 <ScrollView
11381159 nestedScrollEnabled
11391160 directionalLockEnabled
apps/server/src/assets/AssetAccess.test.ts+99-1
......@@ -3,10 +3,16 @@ import * as NodeServices from "@effect/platform-node/NodeServices";
33import * as NodeHttpPlatform from "@effect/platform-node/NodeHttpPlatform";
44import * as NodeFSP from "node:fs/promises";
55import * as NodeOS from "node:os";
6import { AssetAccessError, AssetPreviewTypeValidationError, ThreadId } from "@t3tools/contracts";
6import {
7 AssetAccessError,
8 AssetPreviewTypeValidationError,
9 ThreadId,
10 TurnItemId,
11} from "@t3tools/contracts";
712import { PROJECT_FAVICON_FALLBACK_MARKER } from "@t3tools/shared/projectFavicon";
813import { describe, expect, it } from "@effect/vitest";
914import * as Crypto from "effect/Crypto";
15import * as DateTime from "effect/DateTime";
1016import * as Effect from "effect/Effect";
1117import * as FileSystem from "effect/FileSystem";
1218import * as Layer from "effect/Layer";
......@@ -20,6 +26,7 @@ import { vi } from "vite-plus/test";
2026
2127import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
2228import * as ServerConfig from "../config.ts";
29import * as Orchestrator from "../orchestration-v2/Orchestrator.ts";
2330import * as ProjectFaviconResolver from "../project/ProjectFaviconResolver.ts";
2431import * as T3ProjectFileLoader from "../project/T3ProjectFileLoader.ts";
2532import * as WorkspacePaths from "../workspace/WorkspacePaths.ts";
......@@ -44,8 +51,70 @@ vi.mock("node:os", async (importOriginal) => {
4451const configLayer = ServerConfig.ServerConfig.layerTest(process.cwd(), {
4552 prefix: "t3-asset-access-test-",
4653});
54// A PNG header is enough for the dimension read: signature, then IHDR width and height.
55const screenshotPng = new Uint8Array(24);
56screenshotPng.set([
57 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 13, 0x49, 0x48, 0x44, 0x52,
58]);
59new DataView(screenshotPng.buffer).setUint32(16, 390);
60new DataView(screenshotPng.buffer).setUint32(20, 844);
61const screenshotItem = {
62 id: TurnItemId.make("tool-screenshot"),
63 type: "dynamic_tool" as const,
64 threadId: ThreadId.make("thread-1"),
65 runId: null,
66 nodeId: null,
67 providerThreadId: null,
68 providerTurnId: null,
69 nativeItemRef: null,
70 parentItemId: null,
71 ordinal: 1,
72 status: "completed" as const,
73 title: null,
74 toolName: "mcp__t3-code__device_screenshot",
75 input: { deviceId: "phone" },
76 output: {
77 content: [
78 {
79 type: "image",
80 source: {
81 type: "base64",
82 media_type: "image/png",
83 data: Buffer.from(screenshotPng).toString("base64"),
84 },
85 },
86 ],
87 },
88 startedAt: DateTime.makeUnsafe("2026-10-05T00:00:00.000Z"),
89 completedAt: DateTime.makeUnsafe("2026-10-05T00:00:01.000Z"),
90 updatedAt: DateTime.makeUnsafe("2026-10-05T00:00:01.000Z"),
91};
92
93const oversizedScreenshotItem = {
94 ...screenshotItem,
95 id: TurnItemId.make("tool-screenshot-oversized"),
96 output: {
97 content: [
98 {
99 type: "image",
100 source: { type: "base64", media_type: "image/png", data: "A".repeat(14 * 1024 * 1024) },
101 },
102 ],
103 },
104};
105
47106const testLayer = Layer.mergeAll(
48107 NodeHttpPlatform.layer,
108 Layer.mock(Orchestrator.OrchestratorV2)({
109 getTurnItem: ({ itemId }) =>
110 Effect.succeed(
111 itemId === screenshotItem.id
112 ? screenshotItem
113 : itemId === oversizedScreenshotItem.id
114 ? oversizedScreenshotItem
115 : null,
116 ),
117 }),
49118 configLayer,
50119 WorkspacePaths.layer,
51120 ProjectFaviconResolver.layer.pipe(
......@@ -103,6 +172,35 @@ describe("AssetAccess", () => {
103172 );
104173 });
105174
175 it.effect("serves an image a tool returned inline from the stored item", () =>
176 Effect.gen(function* () {
177 const resource = {
178 _tag: "tool-output-image" as const,
179 threadId: screenshotItem.threadId,
180 itemId: screenshotItem.id,
181 index: 0,
182 };
183 const result = yield* issueAssetUrl({ resource });
184 expect(result.imageDimensions).toEqual({ width: 390, height: 844 });
185 const suffix = result.relativeUrl.slice(`${ASSET_ROUTE_PREFIX}/`.length);
186 const separator = suffix.indexOf("/");
187 expect(yield* resolveAsset(suffix.slice(0, separator), suffix.slice(separator + 1))).toEqual({
188 kind: "bytes",
189 mimeType: "image/png",
190 bytes: Buffer.from(screenshotPng),
191 });
192 const missing = yield* issueAssetUrl({ resource: { ...resource, index: 1 } }).pipe(
193 Effect.flip,
194 );
195 expect(missing._tag).toBe("AssetWorkspaceAssetNotFoundError");
196 // Larger than a provider turn accepts, so it is never decoded.
197 const oversized = yield* issueAssetUrl({
198 resource: { ...resource, itemId: oversizedScreenshotItem.id },
199 }).pipe(Effect.flip);
200 expect(oversized._tag).toBe("AssetWorkspaceAssetNotFoundError");
201 }).pipe(Effect.provide(testLayer)),
202 );
203
106204 it.effect("issues exact URLs for media and browser documents outside the workspace", () =>
107205 Effect.gen(function* () {
108206 const fs = yield* FileSystem.FileSystem;
apps/server/src/assets/AssetAccess.ts+75
......@@ -1,6 +1,7 @@
11import type { AssetResource } from "@t3tools/contracts";
22import {
33 AssetAttachmentNotFoundError,
4 PROVIDER_SEND_TURN_MAX_IMAGE_BYTES,
45 AssetGitHubMediaUrlValidationError,
56 AssetPreviewTypeValidationError,
67 AssetProjectFaviconInspectionError,
......@@ -13,7 +14,9 @@ import {
1314 AssetWorkspacePathValidationError,
1415 AssetWorkspaceResolutionError,
1516 AssetWorkspaceRootNormalizationError,
17 ThreadId,
1618 ToolActivityNativeAppReference,
19 TurnItemId,
1720} from "@t3tools/contracts";
1821import {
1922 audioMimeTypeFromExtension,
......@@ -30,6 +33,7 @@ import {
3033} from "@t3tools/shared/imageDimensions";
3134import { githubMediaFetchUrl, githubMediaFileName } from "@t3tools/shared/githubMedia";
3235import { PROJECT_FAVICON_FALLBACK_MARKER } from "@t3tools/shared/projectFavicon";
36import { toolOutputImages } from "@t3tools/shared/toolOutput";
3337import * as Clock from "effect/Clock";
3438import * as Crypto from "effect/Crypto";
3539import * as Effect from "effect/Effect";
......@@ -49,6 +53,7 @@ import {
4953import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
5054import { parseAttachmentFileExtension, resolveAttachmentPathById } from "../attachmentStore.ts";
5155import * as ServerConfig from "../config.ts";
56import * as Orchestrator from "../orchestration-v2/Orchestrator.ts";
5257import { expandHomePathWith } from "../pathExpansion.ts";
5358import * as ProjectFaviconResolver from "../project/ProjectFaviconResolver.ts";
5459import * as WorkspacePaths from "../workspace/WorkspacePaths.ts";
......@@ -138,6 +143,14 @@ const AssetClaimsSchema = Schema.Union([
138143 app: ToolActivityNativeAppReference,
139144 expiresAt: Schema.Number,
140145 }),
146 Schema.Struct({
147 version: Schema.Literal(1),
148 kind: Schema.Literal("tool-output-image"),
149 threadId: ThreadId,
150 itemId: TurnItemId,
151 index: Schema.Number,
152 expiresAt: Schema.Number,
153 }),
141154 Schema.Struct({
142155 version: Schema.Literal(1),
143156 kind: Schema.Literal("github-media"),
......@@ -162,6 +175,11 @@ export type ResolvedAsset =
162175 readonly mimeType?: string;
163176 readonly file?: OpenMediaFile;
164177 }
178 | {
179 readonly kind: "bytes";
180 readonly bytes: Uint8Array;
181 readonly mimeType: string;
182 }
165183 | {
166184 readonly kind: "github-media";
167185 readonly url: string;
......@@ -198,6 +216,27 @@ const optionOnNotFound = <A, R>(
198216 }),
199217 );
200218
219// The largest image a provider turn accepts, as base64 (4 characters per 3 bytes).
220const MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH = Math.ceil(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES / 3) * 4;
221
222/**
223 * Decodes one image a tool returned inline; null when the stored item has no
224 * such image, or it is larger than a provider turn accepts.
225 */
226const readToolOutputImage = Effect.fn("AssetAccess.readToolOutputImage")(function* (input: {
227 readonly threadId: ThreadId;
228 readonly itemId: TurnItemId;
229 readonly index: number;
230}) {
231 const orchestrator = yield* Orchestrator.OrchestratorV2;
232 const item = yield* orchestrator.getTurnItem(input);
233 const image =
234 item?.type === "dynamic_tool" ? toolOutputImages(item.output)[input.index] : undefined;
235 return image?.data === undefined || image.data.length > MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH
236 ? null
237 : { mimeType: image.mimeType, bytes: Buffer.from(image.data, "base64") };
238});
239
201240const resolveCanonicalFile = Effect.fn("AssetAccess.resolveCanonicalFile")(function* (
202241 filePath: string,
203242) {
......@@ -672,6 +711,28 @@ export const issueAssetUrl = Effect.fn("AssetAccess.issueAssetUrl")(function* (i
672711 }
673712 break;
674713 }
714 case "tool-output-image": {
715 const image = yield* readToolOutputImage(input.resource).pipe(
716 Effect.mapError(
717 (cause) => new AssetWorkspaceAssetInspectionError({ resource: input.resource, cause }),
718 ),
719 );
720 if (image === null) {
721 return yield* new AssetWorkspaceAssetNotFoundError({ resource: input.resource });
722 }
723 claims = {
724 version: 1,
725 kind: "tool-output-image",
726 threadId: input.resource.threadId,
727 itemId: input.resource.itemId,
728 index: input.resource.index,
729 expiresAt,
730 };
731 // The allowed types are all `image/<extension>`.
732 fileName = `image-${input.resource.index + 1}.${image.mimeType.slice("image/".length)}`;
733 imageDimensions = readImageDimensions(image.bytes);
734 break;
735 }
675736 case "native-app-icon": {
676737 claims = {
677738 version: 1,
......@@ -806,6 +867,20 @@ export const resolveAsset = Effect.fn("AssetAccess.resolveAsset")(function* (
806867 } satisfies ResolvedAsset;
807868 }
808869
870 if (claims.kind === "tool-output-image") {
871 const image = yield* readToolOutputImage(claims).pipe(
872 Effect.tapError((cause) =>
873 Effect.logError("Failed to read tool output image.", {
874 threadId: claims.threadId,
875 itemId: claims.itemId,
876 cause,
877 }),
878 ),
879 Effect.orElseSucceed(() => null),
880 );
881 return image ? ({ kind: "bytes", ...image } satisfies ResolvedAsset) : null;
882 }
883
809884 if (claims.kind === "native-app-icon") {
810885 const nativeAppIconResolver = yield* NativeAppIconResolver.NativeAppIconResolver;
811886 const iconPath = yield* nativeAppIconResolver.resolve(claims.app);
apps/server/src/http.ts+6
......@@ -411,6 +411,12 @@ export const assetRouteLayer = HttpRouter.add(
411411 if (!asset) {
412412 return HttpServerResponse.text("Not Found", { status: 404 });
413413 }
414 if (asset.kind === "bytes") {
415 return HttpServerResponse.uint8Array(asset.bytes, {
416 contentType: asset.mimeType,
417 headers: { "cache-control": "private, max-age=3600", "x-content-type-options": "nosniff" },
418 });
419 }
414420 if (asset.kind === "github-media") {
415421 return yield* githubMediaResponse(asset, request.headers).pipe(
416422 Effect.tapError((cause) =>
apps/server/src/orchestration-v2/WireProjection.test.ts+51
......@@ -23,6 +23,7 @@ import {
2323 projectDomainEventForWire,
2424} from "./WireProjection.ts";
2525import { threadShellFromProjection } from "./ProjectionStore.ts";
26import { MAX_TOOL_OUTPUT_IMAGES, toolOutputImages } from "@t3tools/shared/toolOutput";
2627
2728const decodeTurnItem = Schema.decodeUnknownSync(OrchestrationV2TurnItem);
2829const encodeTurnItemJson = Schema.encodeSync(OrchestrationV2TurnItemJson);
......@@ -274,6 +275,56 @@ describe("orchestration V2 wire projection", () => {
274275 expect(item.input).toBe(input);
275276 });
276277
278 it("leaves a screenshot's bytes out of the fetched tool output", () => {
279 const data = "A".repeat(600_000);
280 const item = {
281 ...base,
282 type: "dynamic_tool" as const,
283 toolName: "mcp__t3-code__device_screenshot",
284 input: { deviceId: "phone" },
285 output: {
286 content: [{ type: "image", source: { type: "base64", media_type: "image/png", data } }],
287 },
288 };
289 expect(projectTurnItemForDetail(item)).toMatchObject({
290 output: { content: [{ type: "image", mimeType: "image/png" }] },
291 });
292 expect(item.output.content[0]?.source.data).toBe(data);
293 });
294
295 it("keeps image markers when the rest of a tool output is too large to send", () => {
296 const image = {
297 type: "image",
298 source: { type: "base64", media_type: "image/png", data: "AAAA" },
299 };
300 const item = {
301 ...base,
302 type: "dynamic_tool" as const,
303 output: { content: [{ type: "text", text: "x".repeat(300_000) }, image] },
304 };
305 const projected = projectTurnItemForDetail(item);
306 const output = projected.type === "dynamic_tool" ? projected.output : null;
307 expect(Array.isArray(output) ? output.slice(1) : null).toEqual([
308 { type: "image", mimeType: "image/png" },
309 ]);
310 expect(JSON.stringify(output).length).toBeLessThan(270_000);
311 });
312
313 it("bounds a tool output made of many images", () => {
314 const image = { type: "image", data: "AAAA", mimeType: "image/png" };
315 const item = {
316 ...base,
317 type: "dynamic_tool" as const,
318 output: Array.from({ length: 10_000 }, () => image),
319 };
320 const projected = projectTurnItemForDetail(item);
321 const output = projected.type === "dynamic_tool" ? projected.output : null;
322 // The truncated text block plus a fixed number of markers, however many images there are.
323 expect(Array.isArray(output) ? output.length : null).toBe(1 + MAX_TOOL_OUTPUT_IMAGES);
324 expect(Array.isArray(output) ? String(output[0]?.text).length : null).toBeLessThan(263_000);
325 expect(toolOutputImages(output)).toHaveLength(MAX_TOOL_OUTPUT_IMAGES);
326 });
327
277328 it("keeps failure evidence without retaining command output", () => {
278329 const item = {
279330 ...base,
apps/server/src/orchestration-v2/WireProjection.ts+26-2
......@@ -4,7 +4,12 @@ import type {
44 OrchestrationV2ThreadProjection,
55 OrchestrationV2TurnItem,
66} from "@t3tools/contracts";
7import { compactDynamicToolOutput, toolOutputIndicatesFailure } from "@t3tools/shared/toolOutput";
7import {
8 compactDynamicToolOutput,
9 omitToolOutputImageData,
10 toolOutputImages,
11 toolOutputIndicatesFailure,
12} from "@t3tools/shared/toolOutput";
813
914const MAX_DETAIL_STRING_BYTES = 32_768;
1015const MAX_DYNAMIC_VALUE_BYTES = 16_384;
......@@ -142,9 +147,28 @@ function boundDynamicValue(value: unknown): unknown {
142147 : truncateDetail(json, MAX_ON_DEMAND_BYTES);
143148}
144149
150/**
151 * A tool output for a detail read, without image bytes. When the rest is too
152 * large to send whole, its truncated text still comes with the image markers,
153 * in order, so clients can load each image by index.
154 */
155function boundToolOutput(output: unknown): unknown {
156 const omitted = omitToolOutputImageData(output);
157 const bounded = boundDynamicValue(omitted);
158 if (typeof bounded !== "string" || typeof omitted === "string") return bounded;
159 const images = toolOutputImages(omitted);
160 return images.length === 0
161 ? bounded
162 : [
163 { type: "text", text: bounded },
164 ...images.map((image) => ({ type: "image", mimeType: image.mimeType })),
165 ];
166}
167
145168/**
146169 * Projects one item for an on-demand detail read: keeps the input and output
147170 * the timeline withholds, bounded so a huge result cannot stall the socket.
171 * Image bytes are left out; clients load them as `tool-output-image` assets.
148172 */
149173export function projectTurnItemForDetail(item: OrchestrationV2TurnItem): OrchestrationV2TurnItem {
150174 switch (item.type) {
......@@ -158,7 +182,7 @@ export function projectTurnItemForDetail(item: OrchestrationV2TurnItem): Orchest
158182 return {
159183 ...item,
160184 input: boundDynamicValue(item.input),
161 output: boundDynamicValue(item.output),
185 output: boundToolOutput(item.output),
162186 };
163187 case "subagent":
164188 return {
apps/server/src/ws.ts+1
......@@ -3148,6 +3148,7 @@ const makeWsRpcLayer = (
31483148 if (
31493149 input.resource._tag === "attachment" ||
31503150 input.resource._tag === "native-app-icon" ||
3151 input.resource._tag === "tool-output-image" ||
31513152 // GitHub media names the repository it authenticates through itself.
31523153 input.resource._tag === "github-media" ||
31533154 (input.resource._tag === "media-file" && path.isAbsolute(input.resource.path))
apps/web/src/components/ChatMarkdown.tsx+8-1
......@@ -1762,7 +1762,14 @@ export const ChatMarkdownAssetImage = memo(function ChatMarkdownAssetImage(props
17621762 readonly environmentId: EnvironmentId;
17631763 readonly resource: Extract<
17641764 AssetResource,
1765 { readonly _tag: "attachment" | "workspace-file" | "media-file" | "github-media" }
1765 {
1766 readonly _tag:
1767 | "attachment"
1768 | "workspace-file"
1769 | "media-file"
1770 | "github-media"
1771 | "tool-output-image";
1772 }
17661773 >;
17671774 readonly kind?: "image" | "video";
17681775 readonly alt: string;
apps/web/src/components/chat/MessagesTimeline.tsx+4
......@@ -2878,6 +2878,7 @@ function V2EventTimelineRow({ row }: { row: Extract<TimelineRow, { kind: "event"
28782878 onOpenThread={ctx.onOpenThread}
28792879 onOpenTurnDiff={ctx.onOpenTurnDiff}
28802880 onRollbackCheckpoint={ctx.onRollbackCheckpoint}
2881 onImageExpand={ctx.onImageExpand}
28812882 />
28822883 </div>
28832884 </div>
......@@ -2952,6 +2953,7 @@ function V2EventTimelineRow({ row }: { row: Extract<TimelineRow, { kind: "event"
29522953 onOpenThread={ctx.onOpenThread}
29532954 onOpenTurnDiff={ctx.onOpenTurnDiff}
29542955 onRollbackCheckpoint={ctx.onRollbackCheckpoint}
2956 onImageExpand={ctx.onImageExpand}
29552957 />
29562958 </div>
29572959 </div>
......@@ -5357,6 +5359,7 @@ function WorkEntryLogRow(props: WorkEntryRowProps) {
53575359 onOpenThread={ctx.onOpenThread}
53585360 onOpenTurnDiff={ctx.onOpenTurnDiff}
53595361 onRollbackCheckpoint={ctx.onRollbackCheckpoint}
5362 onImageExpand={ctx.onImageExpand}
53605363 />
53615364 ) : (
53625365 <>
......@@ -5367,6 +5370,7 @@ function WorkEntryLogRow(props: WorkEntryRowProps) {
53675370 <FetchedToolOutput
53685371 projectedItem={workEntry.projectedItem}
53695372 environmentId={ctx.activeThreadEnvironmentId}
5373 onImageExpand={onImageExpand}
53705374 />
53715375 ) : null}
53725376 </>
apps/web/src/components/chat/V2ItemInspector.tsx+39-7
......@@ -8,6 +8,7 @@ import {
88 toolCallLines,
99 turnItemDetailRevision,
1010 turnItemNeedsDetailFetch,
11 turnItemOutputImages,
1112 turnItemOutputText,
1213} from "@t3tools/client-runtime/work-log/item-detail";
1314import { ExternalLinkIcon, GitBranchIcon, RotateCcwIcon } from "lucide-react";
......@@ -21,9 +22,10 @@ import { useTurnItemDetail } from "../../state/queries";
2122import { useV2ItemSupport } from "../../state/v2ItemSupport";
2223import { formatWorkspaceRelativePath } from "../../filePathDisplay";
2324import { Button } from "../ui/button";
24import ChatMarkdown from "../ChatMarkdown";
25import ChatMarkdown, { ChatMarkdownAssetImage } from "../ChatMarkdown";
2526import { RenderErrorBoundary } from "../RenderErrorBoundary";
2627import { resolveExternalWebLinkHref } from "./externalLinkContextMenu";
28import type { ExpandedImagePreview } from "./ExpandedImagePreview";
2729import { ShellCommandBlock } from "./ShellCommandBlock";
2830
2931interface V2ItemInspectorProps {
......@@ -37,6 +39,7 @@ interface V2ItemInspectorProps {
3739 readonly checkpointId: string;
3840 readonly scopeId: string;
3941 }) => void;
42 readonly onImageExpand?: ((preview: ExpandedImagePreview) => void) | undefined;
4043}
4144
4245function JsonTokens({ text }: { readonly text: string }) {
......@@ -119,6 +122,8 @@ function useFetchedTurnItem(
119122 item,
120123 output: {
121124 output: turnItemOutputText(item),
125 images: turnItemOutputImages(item),
126 environmentId,
122127 pending: item === wireItem && detail.isPending,
123128 error:
124129 item !== wireItem
......@@ -133,32 +138,52 @@ function useFetchedTurnItem(
133138
134139interface ToolOutputState {
135140 readonly output: string | null;
141 readonly images: ReturnType<typeof turnItemOutputImages>;
142 readonly environmentId: EnvironmentId;
136143 readonly pending: boolean;
137144 readonly error: string | null;
138145 readonly empty: boolean;
146 readonly onImageExpand?: ((preview: ExpandedImagePreview) => void) | undefined;
139147}
140148
141149function ToolOutput(props: ToolOutputState) {
142 return props.output ? (
150 const images = props.images.map((resource) => (
151 <ChatMarkdownAssetImage
152 key={resource.index}
153 environmentId={props.environmentId}
154 resource={resource}
155 alt="Tool output image"
156 maxHeightRem={16}
157 onImageExpand={props.onImageExpand}
158 />
159 ));
160 const text = props.output ? (
143161 <div className="max-h-80 overflow-auto text-muted-foreground">{props.output}</div>
144162 ) : props.pending ? (
145163 <div className="text-muted-foreground italic">Loading output…</div>
146164 ) : props.error ? (
147165 <div className="text-destructive">Couldn&apos;t load output: {props.error}</div>
148 ) : props.empty ? (
166 ) : props.empty && images.length === 0 ? (
149167 <div className="text-muted-foreground italic">No output.</div>
150168 ) : null;
169 return (
170 <>
171 {images}
172 {text}
173 </>
174 );
151175}
152176
153177/** Fetched output for rows that show their own plain text instead of the inspector. */
154178export function FetchedToolOutput(props: {
155179 readonly projectedItem: OrchestrationV2ProjectedTurnItem;
156180 readonly environmentId: EnvironmentId;
181 readonly onImageExpand?: ((preview: ExpandedImagePreview) => void) | undefined;
157182}) {
158183 const { output } = useFetchedTurnItem(props.projectedItem, props.environmentId);
159184 return (
160 <div className={cn("mt-1.5", monoClassName)}>
161 <ToolOutput {...output} />
185 <div className={cn("mt-1.5 space-y-1.5", monoClassName)}>
186 <ToolOutput {...output} onImageExpand={props.onImageExpand} />
162187 </div>
163188 );
164189}
......@@ -220,7 +245,12 @@ export const V2ItemInspector = memo(function V2ItemInspector(props: V2ItemInspec
220245 ) : null}
221246
222247 {item.type === "command_execution" ? (
223 <ToolCallBody command={item.input} exitCode={item.exitCode} {...outputState} />
248 <ToolCallBody
249 command={item.input}
250 exitCode={item.exitCode}
251 {...outputState}
252 onImageExpand={props.onImageExpand}
253 />
224254 ) : null}
225255
226256 {item.type === "file_change" ? (
......@@ -317,7 +347,9 @@ export const V2ItemInspector = memo(function V2ItemInspector(props: V2ItemInspec
317347 </ul>
318348 ) : null}
319349
320 {item.type === "dynamic_tool" ? <ToolCallBody args={item.input} {...outputState} /> : null}
350 {item.type === "dynamic_tool" ? (
351 <ToolCallBody args={item.input} {...outputState} onImageExpand={props.onImageExpand} />
352 ) : null}
321353
322354 {item.type === "approval_request" ? <StructuredValue value={item.prompt} /> : null}
323355 {item.type === "user_input_request" ? (
packages/client-runtime/src/work-log/itemDetail.test.ts created+55
......@@ -0,0 +1,55 @@
1import { ThreadId, TurnItemId } from "@t3tools/contracts";
2import * as DateTime from "effect/DateTime";
3import { describe, expect, it } from "vite-plus/test";
4
5import { turnItemOutputImages, turnItemOutputText } from "./itemDetail.ts";
6
7const screenshot = {
8 id: TurnItemId.make("tool-screenshot"),
9 type: "dynamic_tool" as const,
10 threadId: ThreadId.make("thread-1"),
11 runId: null,
12 nodeId: null,
13 providerThreadId: null,
14 providerTurnId: null,
15 nativeItemRef: null,
16 parentItemId: null,
17 ordinal: 1,
18 status: "completed" as const,
19 title: null,
20 toolName: "mcp__t3-code__device_screenshot",
21 input: { deviceId: "phone" },
22 // What a detail read returns: the image's position without its bytes.
23 output: {
24 content: [
25 { type: "text", text: "Captured the home screen." },
26 { type: "image", mimeType: "image/png" },
27 ],
28 },
29 startedAt: DateTime.makeUnsafe("2026-10-05T00:00:00.000Z"),
30 completedAt: DateTime.makeUnsafe("2026-10-05T00:00:01.000Z"),
31 updatedAt: DateTime.makeUnsafe("2026-10-05T00:00:01.000Z"),
32};
33
34describe("tool output images", () => {
35 it("shows a screenshot as an image asset, not as text", () => {
36 expect(turnItemOutputImages(screenshot)).toEqual([
37 {
38 _tag: "tool-output-image",
39 threadId: screenshot.threadId,
40 itemId: screenshot.id,
41 index: 0,
42 },
43 ]);
44 expect(turnItemOutputText(screenshot)).toBe("Captured the home screen.");
45 expect(
46 turnItemOutputText({ ...screenshot, output: { content: [screenshot.output.content[1]] } }),
47 ).toBeNull();
48 });
49
50 it("keeps a placeholder for images it cannot show", () => {
51 const output = { content: [{ type: "image", mimeType: "image/svg+xml" }] };
52 expect(turnItemOutputImages({ ...screenshot, output })).toEqual([]);
53 expect(turnItemOutputText({ ...screenshot, output })).toBe("[image]");
54 });
55});
packages/client-runtime/src/work-log/itemDetail.ts+23-3
......@@ -1,4 +1,5 @@
1import type { OrchestrationV2TurnItem } from "@t3tools/contracts";
1import type { AssetResource, OrchestrationV2TurnItem } from "@t3tools/contracts";
2import { readToolOutputImage, toolOutputImages } from "@t3tools/shared/toolOutput";
23import * as DateTime from "effect/DateTime";
34
45const MAX_TEXT_BLOCK_DEPTH = 4;
......@@ -17,11 +18,14 @@ function textFromBlocks(value: unknown, depth: number): string | null {
1718 if (typeof value === "string") return value;
1819 if (Array.isArray(value)) {
1920 const parts = value.map((block) => textFromBlocks(block, depth + 1));
20 return parts.every((part) => part !== null) ? parts.join("\n") : null;
21 return parts.every((part) => part !== null)
22 ? parts.filter((part) => part !== "").join("\n")
23 : null;
2124 }
2225 if (!isRecord(value)) return null;
2326 if (value.type === "text" && typeof value.text === "string") return value.text;
24 if (value.type === "image") return "[image]";
27 // Images clients can show render on their own (see turnItemOutputImages).
28 if (value.type === "image") return readToolOutputImage(value) ? "" : "[image]";
2529 if (value.type === "resource_link" && typeof value.uri === "string") return value.uri;
2630 if (value.type === "resource" && isRecord(value.resource)) {
2731 const resource = value.resource;
......@@ -200,6 +204,22 @@ export function turnItemOutputText(item: OrchestrationV2TurnItem): string | null
200204 }
201205}
202206
207/**
208 * Images in a fetched item's tool output, as assets. The detail read leaves
209 * the bytes out, so each loads over HTTP by its index.
210 */
211export function turnItemOutputImages(
212 item: OrchestrationV2TurnItem,
213): ReadonlyArray<Extract<AssetResource, { readonly _tag: "tool-output-image" }>> {
214 if (item.type !== "dynamic_tool" || item.outputOmitted === true) return [];
215 return toolOutputImages(item.output).map((_, index) => ({
216 _tag: "tool-output-image",
217 threadId: item.threadId,
218 itemId: item.id,
219 index,
220 }));
221}
222
203223/**
204224 * Whether expanding the item shows anything. Rows without content must not
205225 * offer a disclosure, otherwise they open to an empty panel.
packages/contracts/src/assets.ts+11-2
......@@ -1,6 +1,6 @@
11import * as Schema from "effect/Schema";
22
3import { NonNegativeInt, ThreadId, TrimmedNonEmptyString } from "./baseSchemas.ts";
3import { NonNegativeInt, ThreadId, TrimmedNonEmptyString, TurnItemId } from "./baseSchemas.ts";
44import {
55 PROVIDER_SEND_TURN_MAX_FILE_BYTES,
66 PROVIDER_SEND_TURN_MAX_IMAGE_BYTES,
......@@ -41,6 +41,13 @@ export const AssetResource = Schema.Union([
4141 inline response after deciding the file type is safe to preview. */
4242 disposition: Schema.optionalKey(Schema.Literals(["inline", "attachment"])),
4343 }),
44 // An image a tool returned inline, such as a device screenshot, by its order
45 // in the stored output. The timeline never carries these bytes.
46 Schema.TaggedStruct("tool-output-image", {
47 threadId: ThreadId,
48 itemId: TurnItemId,
49 index: NonNegativeInt,
50 }),
4451 Schema.TaggedStruct("project-favicon", {
4552 cwd: TrimmedNonEmptyString.check(Schema.isMaxLength(ASSET_PATH_MAX_LENGTH)),
4653 // A cache-key hint only. The server reads the authoritative path from the
......@@ -218,7 +225,9 @@ export class AssetWorkspaceAssetNotFoundError extends Schema.TaggedError<AssetWo
218225 override get message(): string {
219226 return this.resource._tag === "media-file"
220227 ? "Media file was not found."
221 : "Workspace asset was not found.";
228 : this.resource._tag === "tool-output-image"
229 ? "Tool output image was not found."
230 : "Workspace asset was not found.";
222231 }
223232}
224233
packages/shared/src/toolOutput.test.ts+55-1
......@@ -1,6 +1,12 @@
11import { describe, expect, it } from "vite-plus/test";
22
3import { compactDynamicToolOutput, toolOutputIndicatesFailure } from "./toolOutput.ts";
3import {
4 compactDynamicToolOutput,
5 MAX_TOOL_OUTPUT_IMAGES,
6 omitToolOutputImageData,
7 toolOutputImages,
8 toolOutputIndicatesFailure,
9} from "./toolOutput.ts";
410
511describe("compactDynamicToolOutput", () => {
612 it("extracts IDs through the MCP result envelopes used by T3 summaries", () => {
......@@ -173,3 +179,51 @@ describe("toolOutputIndicatesFailure", () => {
173179 }
174180 });
175181});
182
183describe("toolOutputImages", () => {
184 const mcpImage = { type: "image", data: "AAAA", mimeType: "image/png" };
185 const claudeImage = {
186 type: "image",
187 source: { type: "base64", media_type: "image/JPEG", data: "BBBB" },
188 };
189 const text = { type: "text", text: "{}" };
190
191 it("finds images in each place providers store them, in order", () => {
192 expect(toolOutputImages({ content: [text, claudeImage, mcpImage] })).toEqual([
193 { mimeType: "image/jpeg", data: "BBBB" },
194 { mimeType: "image/png", data: "AAAA" },
195 ]);
196 expect(toolOutputImages([mcpImage])).toEqual([{ mimeType: "image/png", data: "AAAA" }]);
197 expect(toolOutputImages(mcpImage)).toEqual([{ mimeType: "image/png", data: "AAAA" }]);
198 });
199
200 it("skips types the server must not serve inline", () => {
201 expect(
202 toolOutputImages([
203 { ...mcpImage, mimeType: "image/svg+xml" },
204 { ...mcpImage, mimeType: "text/html" },
205 { type: "image", source: { type: "url", url: "https://example.com/a.png" } },
206 ]),
207 ).toEqual([]);
208 });
209
210 it("stops after the first few images", () => {
211 const images = toolOutputImages(Array.from({ length: 10_000 }, () => mcpImage));
212 expect(images).toHaveLength(MAX_TOOL_OUTPUT_IMAGES);
213 });
214
215 it("drops the bytes but keeps each image's position", () => {
216 const omitted = omitToolOutputImageData({ content: [text, claudeImage, mcpImage] });
217 expect(omitted).toEqual({
218 content: [
219 text,
220 { type: "image", mimeType: "image/jpeg" },
221 { type: "image", mimeType: "image/png" },
222 ],
223 });
224 expect(toolOutputImages(omitted)).toEqual([
225 { mimeType: "image/jpeg" },
226 { mimeType: "image/png" },
227 ]);
228 });
229});
packages/shared/src/toolOutput.ts+67
......@@ -1,3 +1,4 @@
1import { isProviderSendTurnSupportedImageMimeType } from "@t3tools/contracts";
12import * as Predicate from "effect/Predicate";
23
34const MAX_PARSED_BYTES = 16_384;
......@@ -166,3 +167,69 @@ export function toolOutputIndicatesFailure(text: string): boolean {
166167 /exit code\s*[:\s]\s*[1-9]\d*\b/i.test(text)
167168 );
168169}
170
171/** An image a tool returned inline. `data` is base64; a detail read omits it. */
172export interface ToolOutputImage {
173 readonly mimeType: string;
174 readonly data?: string;
175}
176
177/**
178 * Reads one image block in the MCP `{ data, mimeType }` shape or the Anthropic
179 * `{ source: { type: "base64", media_type, data } }` shape Claude stores.
180 * Only raster types are recognized, so the server never serves an agent's SVG
181 * or HTML inline.
182 */
183export function readToolOutputImage(block: unknown): ToolOutputImage | null {
184 if (!Predicate.isObject(block) || block.type !== "image") return null;
185 const source = Predicate.isObject(block.source) ? block.source : undefined;
186 if (source !== undefined && source.type !== "base64") return null;
187 const mimeType = source === undefined ? block.mimeType : source.media_type;
188 const data = source === undefined ? block.data : source.data;
189 if (typeof mimeType !== "string" || !isProviderSendTurnSupportedImageMimeType(mimeType)) {
190 return null;
191 }
192 const image = { mimeType: mimeType.toLowerCase() };
193 return typeof data === "string" ? { ...image, data } : image;
194}
195
196/** Tools return a block, a list of blocks, or an MCP result with a `content` list. */
197function outputBlocks(value: unknown): ReadonlyArray<unknown> {
198 if (Array.isArray(value)) return value;
199 if (Predicate.isObject(value) && Array.isArray(value.content)) return value.content;
200 return [value];
201}
202
203/** A tool returns one screenshot or a few frames; more would only flood the timeline. */
204export const MAX_TOOL_OUTPUT_IMAGES = 8;
205
206/**
207 * The first images in a tool output, in order. The order is the
208 * `tool-output-image` asset index, so servers and clients agree on it.
209 */
210export function toolOutputImages(value: unknown): ReadonlyArray<ToolOutputImage> {
211 const images: ToolOutputImage[] = [];
212 for (const block of outputBlocks(value)) {
213 const image = readToolOutputImage(block);
214 if (image === null) continue;
215 images.push(image);
216 if (images.length === MAX_TOOL_OUTPUT_IMAGES) break;
217 }
218 return images;
219}
220
221/**
222 * Replaces each image's bytes with `{ type: "image", mimeType }` and keeps its
223 * position, so detail reads stay small and clients load the bytes as assets.
224 */
225export function omitToolOutputImageData(value: unknown): unknown {
226 const omit = (block: unknown) => {
227 const image = readToolOutputImage(block);
228 return image?.data === undefined ? block : { type: "image", mimeType: image.mimeType };
229 };
230 if (Array.isArray(value)) return value.map(omit);
231 if (Predicate.isObject(value) && Array.isArray(value.content)) {
232 return { ...value, content: value.content.map(omit) };
233 }
234 return omit(value);
235}