authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-10-05 14:49:14-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-05 14:49:14-07:00
log5eb87730871ac6125266b48b98d067de00a19ec7
tree24f77464787b8e0530bb38396d1255040b51c42f
parent2a68a777afda95aa23538b9954c5a15221ed2857
signature Signed by PGP key B5690EEEBB952194

fix: tool screenshots show as images, not base64 text (#16199)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

17 files changed, 551 insertions(+), 21 deletions(-)

apps/mobile/src/features/threads/ThreadFeed.tsx+1
...@@ -1933,6 +1933,7 @@ function renderFeedEntry(...@@ -1933,6 +1933,7 @@ function renderFeedEntry(
1933 onToggleRow={props.onToggleWorkRow}1933 onToggleRow={props.onToggleWorkRow}
1934 renderImage={props.renderViewedImage}1934 renderImage={props.renderViewedImage}
1935 renderReasoning={props.renderReasoning}1935 renderReasoning={props.renderReasoning}
1936 onPressPreview={props.onPressPreview}
1936 />1937 />
1937 );1938 );
1938}1939}
apps/mobile/src/features/threads/ThreadMarkdownImage.tsx+7-2
...@@ -178,7 +178,10 @@ function ThreadMarkdownImageRequest(props: {...@@ -178,7 +178,10 @@ function ThreadMarkdownImageRequest(props: {
178/** Environment-hosted image that loads through a signed asset URL. */178/** Environment-hosted image that loads through a signed asset URL. */
179export function ThreadMarkdownImage(props: {179export function ThreadMarkdownImage(props: {
180 readonly environmentId: EnvironmentId;180 readonly environmentId: EnvironmentId;
181 readonly resource: Extract<AssetResource, { readonly _tag: "attachment" | "media-file" }>;181 readonly resource: Extract<
182 AssetResource,
183 { readonly _tag: "attachment" | "media-file" | "tool-output-image" }
184 >;
182 readonly alt: string | null;185 readonly alt: string | null;
183 readonly srcFragment?: string;186 readonly srcFragment?: string;
184 readonly actionsSource?: MediaActionsSource;187 readonly actionsSource?: MediaActionsSource;
...@@ -192,7 +195,9 @@ export function ThreadMarkdownImage(props: {...@@ -192,7 +195,9 @@ export function ThreadMarkdownImage(props: {
192 sourceKey={195 sourceKey={
193 props.resource._tag === "attachment"196 props.resource._tag === "attachment"
194 ? `attachment:${props.resource.attachmentId}`197 ? `attachment:${props.resource.attachmentId}`
195 : `workspace:${props.resource.path}`198 : props.resource._tag === "tool-output-image"
199 ? `tool-output:${props.resource.itemId}:${props.resource.index}`
200 : `workspace:${props.resource.path}`
196 }201 }
197 unavailable={assetUrl._tag === "Failure"}202 unavailable={assetUrl._tag === "Failure"}
198 knownSize={assetUrl._tag === "Success" ? assetUrl.imageDimensions : undefined}203 knownSize={assetUrl._tag === "Success" ? assetUrl.imageDimensions : undefined}
apps/mobile/src/features/threads/thread-work-log.tsx+23-2
...@@ -64,7 +64,11 @@ import {...@@ -64,7 +64,11 @@ import {
64 type ThreadFeedActivity,64 type ThreadFeedActivity,
65 workEntryRowLabel,65 workEntryRowLabel,
66} from "../../lib/threadActivity";66} from "../../lib/threadActivity";
67import { toolCallLines, turnItemOutputText } from "@t3tools/client-runtime/work-log/item-detail";67import {
68 toolCallLines,
69 turnItemOutputImages,
70 turnItemOutputText,
71} from "@t3tools/client-runtime/work-log/item-detail";
68import { useTurnItemDetail } from "../../state/queries";72import { useTurnItemDetail } from "../../state/queries";
69import {73import {
70 resolveThreadWorkGroupInitialScroll,74 resolveThreadWorkGroupInitialScroll,
...@@ -80,6 +84,8 @@ import {...@@ -80,6 +84,8 @@ import {
80import { resolveWorkGroupScrollAnchor } from "@t3tools/client-runtime/work-log/scroll-anchor";84import { resolveWorkGroupScrollAnchor } from "@t3tools/client-runtime/work-log/scroll-anchor";
81import { notificationChildThreadId } from "@t3tools/client-runtime/state/thread-execution";85import { notificationChildThreadId } from "@t3tools/client-runtime/state/thread-execution";
82import type { MarkdownImageRenderer } from "../../native/SelectableMarkdownText";86import type { MarkdownImageRenderer } from "../../native/SelectableMarkdownText";
87import type { FilePreviewSource } from "../../components/FilePreviewModal";
88import { ThreadMarkdownImage } from "./ThreadMarkdownImage";
83import Animated, {89import Animated, {
84 cancelAnimation,90 cancelAnimation,
85 Easing,91 Easing,
...@@ -462,6 +468,7 @@ interface ThreadWorkLogProps {...@@ -462,6 +468,7 @@ interface ThreadWorkLogProps {
462 readonly onToggleRow: (rowId: string, anchorKey: string) => void;468 readonly onToggleRow: (rowId: string, anchorKey: string) => void;
463 readonly renderImage: MarkdownImageRenderer;469 readonly renderImage: MarkdownImageRenderer;
464 readonly renderReasoning: (text: string) => ReactNode;470 readonly renderReasoning: (text: string) => ReactNode;
471 readonly onPressPreview: (source: FilePreviewSource) => void;
465}472}
466473
467export function ThreadWorkLog(props: ThreadWorkLogProps) {474export function ThreadWorkLog(props: ThreadWorkLogProps) {
...@@ -479,6 +486,7 @@ export function ThreadWorkLog(props: ThreadWorkLogProps) {...@@ -479,6 +486,7 @@ export function ThreadWorkLog(props: ThreadWorkLogProps) {
479 onToggleRow={props.onToggleRow}486 onToggleRow={props.onToggleRow}
480 renderImage={props.renderImage}487 renderImage={props.renderImage}
481 renderReasoning={props.renderReasoning}488 renderReasoning={props.renderReasoning}
489 onPressPreview={props.onPressPreview}
482 themeAppearance={props.themeAppearance}490 themeAppearance={props.themeAppearance}
483 />491 />
484 ),492 ),
...@@ -492,6 +500,7 @@ export function ThreadWorkLog(props: ThreadWorkLogProps) {...@@ -492,6 +500,7 @@ export function ThreadWorkLog(props: ThreadWorkLogProps) {
492 props.onToggleRow,500 props.onToggleRow,
493 props.renderImage,501 props.renderImage,
494 props.renderReasoning,502 props.renderReasoning,
503 props.onPressPreview,
495 props.themeAppearance,504 props.themeAppearance,
496 ],505 ],
497 );506 );
...@@ -951,12 +960,13 @@ const ThreadWorkLogRow = memo(function ThreadWorkLogRow(...@@ -951,12 +960,13 @@ const ThreadWorkLogRow = memo(function ThreadWorkLogRow(
951 ? formatItemFullDetail(row.projectedItem, fetchedItem)960 ? formatItemFullDetail(row.projectedItem, fetchedItem)
952 : row.getFullDetail()961 : row.getFullDetail()
953 : null;962 : null;
963 const outputImages = expanded && fetchedItem ? turnItemOutputImages(fetchedItem) : [];
954 const fetchedOutput = !expanded964 const fetchedOutput = !expanded
955 ? null965 ? null
956 : shownItem.type === "file_search" || shownItem.type === "web_search"966 : shownItem.type === "file_search" || shownItem.type === "web_search"
957 ? turnItemOutputText(shownItem)967 ? turnItemOutputText(shownItem)
958 : fetchedItem968 : fetchedItem
959 ? (turnItemOutputText(fetchedItem) ?? "No output.")969 ? (turnItemOutputText(fetchedItem) ?? (outputImages.length > 0 ? null : "No output."))
960 : fetchedDetail.error970 : fetchedDetail.error
961 ? `Couldn't load output: ${fetchedDetail.error}`971 ? `Couldn't load output: ${fetchedDetail.error}`
962 : row.fetchesDetail972 : row.fetchesDetail
...@@ -1116,6 +1126,7 @@ const ThreadWorkLogRow = memo(function ThreadWorkLogRow(...@@ -1116,6 +1126,7 @@ const ThreadWorkLogRow = memo(function ThreadWorkLogRow(
1116 call ||1126 call ||
1117 fetchedOutput ||1127 fetchedOutput ||
1118 viewedImagePath ||1128 viewedImagePath ||
1129 outputImages.length > 0 ||
1119 row.workEntry.questionAnswer) ? (1130 row.workEntry.questionAnswer) ? (
1120 <Animated.View1131 <Animated.View
1121 entering={WORK_LOG_DETAIL_ENTER_TRANSITION}1132 entering={WORK_LOG_DETAIL_ENTER_TRANSITION}
...@@ -1134,6 +1145,16 @@ const ThreadWorkLogRow = memo(function ThreadWorkLogRow(...@@ -1134,6 +1145,16 @@ const ThreadWorkLogRow = memo(function ThreadWorkLogRow(
1134 {props.renderImage({ href: viewedImagePath, alt: null, title: null })}1145 {props.renderImage({ href: viewedImagePath, alt: null, title: null })}
1135 </View>1146 </View>
1136 ) : null}1147 ) : null}
1148 {outputImages.map((resource) => (
1149 <View key={resource.index} className="pb-1.5">
1150 <ThreadMarkdownImage
1151 environmentId={props.environmentId}
1152 resource={resource}
1153 alt={null}
1154 onPressPreview={props.onPressPreview}
1155 />
1156 </View>
1157 ))}
1137 <ScrollView1158 <ScrollView
1138 nestedScrollEnabled1159 nestedScrollEnabled
1139 directionalLockEnabled1160 directionalLockEnabled
apps/server/src/assets/AssetAccess.test.ts+99-1
...@@ -3,10 +3,16 @@ import * as NodeServices from "@effect/platform-node/NodeServices";...@@ -3,10 +3,16 @@ import * as NodeServices from "@effect/platform-node/NodeServices";
3import * as NodeHttpPlatform from "@effect/platform-node/NodeHttpPlatform";3import * as NodeHttpPlatform from "@effect/platform-node/NodeHttpPlatform";
4import * as NodeFSP from "node:fs/promises";4import * as NodeFSP from "node:fs/promises";
5import * as NodeOS from "node:os";5import * as NodeOS from "node:os";
6import { AssetAccessError, AssetPreviewTypeValidationError, ThreadId } from "@t3tools/contracts";6import {
7 AssetAccessError,
8 AssetPreviewTypeValidationError,
9 ThreadId,
10 TurnItemId,
11} from "@t3tools/contracts";
7import { PROJECT_FAVICON_FALLBACK_MARKER } from "@t3tools/shared/projectFavicon";12import { PROJECT_FAVICON_FALLBACK_MARKER } from "@t3tools/shared/projectFavicon";
8import { describe, expect, it } from "@effect/vitest";13import { describe, expect, it } from "@effect/vitest";
9import * as Crypto from "effect/Crypto";14import * as Crypto from "effect/Crypto";
15import * as DateTime from "effect/DateTime";
10import * as Effect from "effect/Effect";16import * as Effect from "effect/Effect";
11import * as FileSystem from "effect/FileSystem";17import * as FileSystem from "effect/FileSystem";
12import * as Layer from "effect/Layer";18import * as Layer from "effect/Layer";
...@@ -20,6 +26,7 @@ import { vi } from "vite-plus/test";...@@ -20,6 +26,7 @@ import { vi } from "vite-plus/test";
2026
21import * as ServerSecretStore from "../auth/ServerSecretStore.ts";27import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
22import * as ServerConfig from "../config.ts";28import * as ServerConfig from "../config.ts";
29import * as Orchestrator from "../orchestration-v2/Orchestrator.ts";
23import * as ProjectFaviconResolver from "../project/ProjectFaviconResolver.ts";30import * as ProjectFaviconResolver from "../project/ProjectFaviconResolver.ts";
24import * as T3ProjectFileLoader from "../project/T3ProjectFileLoader.ts";31import * as T3ProjectFileLoader from "../project/T3ProjectFileLoader.ts";
25import * as WorkspacePaths from "../workspace/WorkspacePaths.ts";32import * as WorkspacePaths from "../workspace/WorkspacePaths.ts";
...@@ -44,8 +51,70 @@ vi.mock("node:os", async (importOriginal) => {...@@ -44,8 +51,70 @@ vi.mock("node:os", async (importOriginal) => {
44const configLayer = ServerConfig.ServerConfig.layerTest(process.cwd(), {51const configLayer = ServerConfig.ServerConfig.layerTest(process.cwd(), {
45 prefix: "t3-asset-access-test-",52 prefix: "t3-asset-access-test-",
46});53});
54// A PNG header is enough for the dimension read: signature, then IHDR width and height.
55const screenshotPng = new Uint8Array(24);
56screenshotPng.set([
57 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 13, 0x49, 0x48, 0x44, 0x52,
58]);
59new DataView(screenshotPng.buffer).setUint32(16, 390);
60new DataView(screenshotPng.buffer).setUint32(20, 844);
61const screenshotItem = {
62 id: TurnItemId.make("tool-screenshot"),
63 type: "dynamic_tool" as const,
64 threadId: ThreadId.make("thread-1"),
65 runId: null,
66 nodeId: null,
67 providerThreadId: null,
68 providerTurnId: null,
69 nativeItemRef: null,
70 parentItemId: null,
71 ordinal: 1,
72 status: "completed" as const,
73 title: null,
74 toolName: "mcp__t3-code__device_screenshot",
75 input: { deviceId: "phone" },
76 output: {
77 content: [
78 {
79 type: "image",
80 source: {
81 type: "base64",
82 media_type: "image/png",
83 data: Buffer.from(screenshotPng).toString("base64"),
84 },
85 },
86 ],
87 },
88 startedAt: DateTime.makeUnsafe("2026-10-05T00:00:00.000Z"),
89 completedAt: DateTime.makeUnsafe("2026-10-05T00:00:01.000Z"),
90 updatedAt: DateTime.makeUnsafe("2026-10-05T00:00:01.000Z"),
91};
92
93const oversizedScreenshotItem = {
94 ...screenshotItem,
95 id: TurnItemId.make("tool-screenshot-oversized"),
96 output: {
97 content: [
98 {
99 type: "image",
100 source: { type: "base64", media_type: "image/png", data: "A".repeat(14 * 1024 * 1024) },
101 },
102 ],
103 },
104};
105
47const testLayer = Layer.mergeAll(106const testLayer = Layer.mergeAll(
48 NodeHttpPlatform.layer,107 NodeHttpPlatform.layer,
108 Layer.mock(Orchestrator.OrchestratorV2)({
109 getTurnItem: ({ itemId }) =>
110 Effect.succeed(
111 itemId === screenshotItem.id
112 ? screenshotItem
113 : itemId === oversizedScreenshotItem.id
114 ? oversizedScreenshotItem
115 : null,
116 ),
117 }),
49 configLayer,118 configLayer,
50 WorkspacePaths.layer,119 WorkspacePaths.layer,
51 ProjectFaviconResolver.layer.pipe(120 ProjectFaviconResolver.layer.pipe(
...@@ -103,6 +172,35 @@ describe("AssetAccess", () => {...@@ -103,6 +172,35 @@ describe("AssetAccess", () => {
103 );172 );
104 });173 });
105174
175 it.effect("serves an image a tool returned inline from the stored item", () =>
176 Effect.gen(function* () {
177 const resource = {
178 _tag: "tool-output-image" as const,
179 threadId: screenshotItem.threadId,
180 itemId: screenshotItem.id,
181 index: 0,
182 };
183 const result = yield* issueAssetUrl({ resource });
184 expect(result.imageDimensions).toEqual({ width: 390, height: 844 });
185 const suffix = result.relativeUrl.slice(`${ASSET_ROUTE_PREFIX}/`.length);
186 const separator = suffix.indexOf("/");
187 expect(yield* resolveAsset(suffix.slice(0, separator), suffix.slice(separator + 1))).toEqual({
188 kind: "bytes",
189 mimeType: "image/png",
190 bytes: Buffer.from(screenshotPng),
191 });
192 const missing = yield* issueAssetUrl({ resource: { ...resource, index: 1 } }).pipe(
193 Effect.flip,
194 );
195 expect(missing._tag).toBe("AssetWorkspaceAssetNotFoundError");
196 // Larger than a provider turn accepts, so it is never decoded.
197 const oversized = yield* issueAssetUrl({
198 resource: { ...resource, itemId: oversizedScreenshotItem.id },
199 }).pipe(Effect.flip);
200 expect(oversized._tag).toBe("AssetWorkspaceAssetNotFoundError");
201 }).pipe(Effect.provide(testLayer)),
202 );
203
106 it.effect("issues exact URLs for media and browser documents outside the workspace", () =>204 it.effect("issues exact URLs for media and browser documents outside the workspace", () =>
107 Effect.gen(function* () {205 Effect.gen(function* () {
108 const fs = yield* FileSystem.FileSystem;206 const fs = yield* FileSystem.FileSystem;
apps/server/src/assets/AssetAccess.ts+75
...@@ -1,6 +1,7 @@...@@ -1,6 +1,7 @@
1import type { AssetResource } from "@t3tools/contracts";1import type { AssetResource } from "@t3tools/contracts";
2import {2import {
3 AssetAttachmentNotFoundError,3 AssetAttachmentNotFoundError,
4 PROVIDER_SEND_TURN_MAX_IMAGE_BYTES,
4 AssetGitHubMediaUrlValidationError,5 AssetGitHubMediaUrlValidationError,
5 AssetPreviewTypeValidationError,6 AssetPreviewTypeValidationError,
6 AssetProjectFaviconInspectionError,7 AssetProjectFaviconInspectionError,
...@@ -13,7 +14,9 @@ import {...@@ -13,7 +14,9 @@ import {
13 AssetWorkspacePathValidationError,14 AssetWorkspacePathValidationError,
14 AssetWorkspaceResolutionError,15 AssetWorkspaceResolutionError,
15 AssetWorkspaceRootNormalizationError,16 AssetWorkspaceRootNormalizationError,
17 ThreadId,
16 ToolActivityNativeAppReference,18 ToolActivityNativeAppReference,
19 TurnItemId,
17} from "@t3tools/contracts";20} from "@t3tools/contracts";
18import {21import {
19 audioMimeTypeFromExtension,22 audioMimeTypeFromExtension,
...@@ -30,6 +33,7 @@ import {...@@ -30,6 +33,7 @@ import {
30} from "@t3tools/shared/imageDimensions";33} from "@t3tools/shared/imageDimensions";
31import { githubMediaFetchUrl, githubMediaFileName } from "@t3tools/shared/githubMedia";34import { githubMediaFetchUrl, githubMediaFileName } from "@t3tools/shared/githubMedia";
32import { PROJECT_FAVICON_FALLBACK_MARKER } from "@t3tools/shared/projectFavicon";35import { PROJECT_FAVICON_FALLBACK_MARKER } from "@t3tools/shared/projectFavicon";
36import { toolOutputImages } from "@t3tools/shared/toolOutput";
33import * as Clock from "effect/Clock";37import * as Clock from "effect/Clock";
34import * as Crypto from "effect/Crypto";38import * as Crypto from "effect/Crypto";
35import * as Effect from "effect/Effect";39import * as Effect from "effect/Effect";
...@@ -49,6 +53,7 @@ import {...@@ -49,6 +53,7 @@ import {
49import * as ServerSecretStore from "../auth/ServerSecretStore.ts";53import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
50import { parseAttachmentFileExtension, resolveAttachmentPathById } from "../attachmentStore.ts";54import { parseAttachmentFileExtension, resolveAttachmentPathById } from "../attachmentStore.ts";
51import * as ServerConfig from "../config.ts";55import * as ServerConfig from "../config.ts";
56import * as Orchestrator from "../orchestration-v2/Orchestrator.ts";
52import { expandHomePathWith } from "../pathExpansion.ts";57import { expandHomePathWith } from "../pathExpansion.ts";
53import * as ProjectFaviconResolver from "../project/ProjectFaviconResolver.ts";58import * as ProjectFaviconResolver from "../project/ProjectFaviconResolver.ts";
54import * as WorkspacePaths from "../workspace/WorkspacePaths.ts";59import * as WorkspacePaths from "../workspace/WorkspacePaths.ts";
...@@ -138,6 +143,14 @@ const AssetClaimsSchema = Schema.Union([...@@ -138,6 +143,14 @@ const AssetClaimsSchema = Schema.Union([
138 app: ToolActivityNativeAppReference,143 app: ToolActivityNativeAppReference,
139 expiresAt: Schema.Number,144 expiresAt: Schema.Number,
140 }),145 }),
146 Schema.Struct({
147 version: Schema.Literal(1),
148 kind: Schema.Literal("tool-output-image"),
149 threadId: ThreadId,
150 itemId: TurnItemId,
151 index: Schema.Number,
152 expiresAt: Schema.Number,
153 }),
141 Schema.Struct({154 Schema.Struct({
142 version: Schema.Literal(1),155 version: Schema.Literal(1),
143 kind: Schema.Literal("github-media"),156 kind: Schema.Literal("github-media"),
...@@ -162,6 +175,11 @@ export type ResolvedAsset =...@@ -162,6 +175,11 @@ export type ResolvedAsset =
162 readonly mimeType?: string;175 readonly mimeType?: string;
163 readonly file?: OpenMediaFile;176 readonly file?: OpenMediaFile;
164 }177 }
178 | {
179 readonly kind: "bytes";
180 readonly bytes: Uint8Array;
181 readonly mimeType: string;
182 }
165 | {183 | {
166 readonly kind: "github-media";184 readonly kind: "github-media";
167 readonly url: string;185 readonly url: string;
...@@ -198,6 +216,27 @@ const optionOnNotFound = <A, R>(...@@ -198,6 +216,27 @@ const optionOnNotFound = <A, R>(
198 }),216 }),
199 );217 );
200218
219// The largest image a provider turn accepts, as base64 (4 characters per 3 bytes).
220const MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH = Math.ceil(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES / 3) * 4;
221
222/**
223 * Decodes one image a tool returned inline; null when the stored item has no
224 * such image, or it is larger than a provider turn accepts.
225 */
226const readToolOutputImage = Effect.fn("AssetAccess.readToolOutputImage")(function* (input: {
227 readonly threadId: ThreadId;
228 readonly itemId: TurnItemId;
229 readonly index: number;
230}) {
231 const orchestrator = yield* Orchestrator.OrchestratorV2;
232 const item = yield* orchestrator.getTurnItem(input);
233 const image =
234 item?.type === "dynamic_tool" ? toolOutputImages(item.output)[input.index] : undefined;
235 return image?.data === undefined || image.data.length > MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH
236 ? null
237 : { mimeType: image.mimeType, bytes: Buffer.from(image.data, "base64") };
238});
239
201const resolveCanonicalFile = Effect.fn("AssetAccess.resolveCanonicalFile")(function* (240const resolveCanonicalFile = Effect.fn("AssetAccess.resolveCanonicalFile")(function* (
202 filePath: string,241 filePath: string,
203) {242) {
...@@ -672,6 +711,28 @@ export const issueAssetUrl = Effect.fn("AssetAccess.issueAssetUrl")(function* (i...@@ -672,6 +711,28 @@ export const issueAssetUrl = Effect.fn("AssetAccess.issueAssetUrl")(function* (i
672 }711 }
673 break;712 break;
674 }713 }
714 case "tool-output-image": {
715 const image = yield* readToolOutputImage(input.resource).pipe(
716 Effect.mapError(
717 (cause) => new AssetWorkspaceAssetInspectionError({ resource: input.resource, cause }),
718 ),
719 );
720 if (image === null) {
721 return yield* new AssetWorkspaceAssetNotFoundError({ resource: input.resource });
722 }
723 claims = {
724 version: 1,
725 kind: "tool-output-image",
726 threadId: input.resource.threadId,
727 itemId: input.resource.itemId,
728 index: input.resource.index,
729 expiresAt,
730 };
731 // The allowed types are all `image/<extension>`.
732 fileName = `image-${input.resource.index + 1}.${image.mimeType.slice("image/".length)}`;
733 imageDimensions = readImageDimensions(image.bytes);
734 break;
735 }
675 case "native-app-icon": {736 case "native-app-icon": {
676 claims = {737 claims = {
677 version: 1,738 version: 1,
...@@ -806,6 +867,20 @@ export const resolveAsset = Effect.fn("AssetAccess.resolveAsset")(function* (...@@ -806,6 +867,20 @@ export const resolveAsset = Effect.fn("AssetAccess.resolveAsset")(function* (
806 } satisfies ResolvedAsset;867 } satisfies ResolvedAsset;
807 }868 }
808869
870 if (claims.kind === "tool-output-image") {
871 const image = yield* readToolOutputImage(claims).pipe(
872 Effect.tapError((cause) =>
873 Effect.logError("Failed to read tool output image.", {
874 threadId: claims.threadId,
875 itemId: claims.itemId,
876 cause,
877 }),
878 ),
879 Effect.orElseSucceed(() => null),
880 );
881 return image ? ({ kind: "bytes", ...image } satisfies ResolvedAsset) : null;
882 }
883
809 if (claims.kind === "native-app-icon") {884 if (claims.kind === "native-app-icon") {
810 const nativeAppIconResolver = yield* NativeAppIconResolver.NativeAppIconResolver;885 const nativeAppIconResolver = yield* NativeAppIconResolver.NativeAppIconResolver;
811 const iconPath = yield* nativeAppIconResolver.resolve(claims.app);886 const iconPath = yield* nativeAppIconResolver.resolve(claims.app);
apps/server/src/http.ts+6
...@@ -411,6 +411,12 @@ export const assetRouteLayer = HttpRouter.add(...@@ -411,6 +411,12 @@ export const assetRouteLayer = HttpRouter.add(
411 if (!asset) {411 if (!asset) {
412 return HttpServerResponse.text("Not Found", { status: 404 });412 return HttpServerResponse.text("Not Found", { status: 404 });
413 }413 }
414 if (asset.kind === "bytes") {
415 return HttpServerResponse.uint8Array(asset.bytes, {
416 contentType: asset.mimeType,
417 headers: { "cache-control": "private, max-age=3600", "x-content-type-options": "nosniff" },
418 });
419 }
414 if (asset.kind === "github-media") {420 if (asset.kind === "github-media") {
415 return yield* githubMediaResponse(asset, request.headers).pipe(421 return yield* githubMediaResponse(asset, request.headers).pipe(
416 Effect.tapError((cause) =>422 Effect.tapError((cause) =>
apps/server/src/orchestration-v2/WireProjection.test.ts+51
...@@ -23,6 +23,7 @@ import {...@@ -23,6 +23,7 @@ import {
23 projectDomainEventForWire,23 projectDomainEventForWire,
24} from "./WireProjection.ts";24} from "./WireProjection.ts";
25import { threadShellFromProjection } from "./ProjectionStore.ts";25import { threadShellFromProjection } from "./ProjectionStore.ts";
26import { MAX_TOOL_OUTPUT_IMAGES, toolOutputImages } from "@t3tools/shared/toolOutput";
2627
27const decodeTurnItem = Schema.decodeUnknownSync(OrchestrationV2TurnItem);28const decodeTurnItem = Schema.decodeUnknownSync(OrchestrationV2TurnItem);
28const encodeTurnItemJson = Schema.encodeSync(OrchestrationV2TurnItemJson);29const encodeTurnItemJson = Schema.encodeSync(OrchestrationV2TurnItemJson);
...@@ -274,6 +275,56 @@ describe("orchestration V2 wire projection", () => {...@@ -274,6 +275,56 @@ describe("orchestration V2 wire projection", () => {
274 expect(item.input).toBe(input);275 expect(item.input).toBe(input);
275 });276 });
276277
278 it("leaves a screenshot's bytes out of the fetched tool output", () => {
279 const data = "A".repeat(600_000);
280 const item = {
281 ...base,
282 type: "dynamic_tool" as const,
283 toolName: "mcp__t3-code__device_screenshot",
284 input: { deviceId: "phone" },
285 output: {
286 content: [{ type: "image", source: { type: "base64", media_type: "image/png", data } }],
287 },
288 };
289 expect(projectTurnItemForDetail(item)).toMatchObject({
290 output: { content: [{ type: "image", mimeType: "image/png" }] },
291 });
292 expect(item.output.content[0]?.source.data).toBe(data);
293 });
294
295 it("keeps image markers when the rest of a tool output is too large to send", () => {
296 const image = {
297 type: "image",
298 source: { type: "base64", media_type: "image/png", data: "AAAA" },
299 };
300 const item = {
301 ...base,
302 type: "dynamic_tool" as const,
303 output: { content: [{ type: "text", text: "x".repeat(300_000) }, image] },
304 };
305 const projected = projectTurnItemForDetail(item);
306 const output = projected.type === "dynamic_tool" ? projected.output : null;
307 expect(Array.isArray(output) ? output.slice(1) : null).toEqual([
308 { type: "image", mimeType: "image/png" },
309 ]);
310 expect(JSON.stringify(output).length).toBeLessThan(270_000);
311 });
312
313 it("bounds a tool output made of many images", () => {
314 const image = { type: "image", data: "AAAA", mimeType: "image/png" };
315 const item = {
316 ...base,
317 type: "dynamic_tool" as const,
318 output: Array.from({ length: 10_000 }, () => image),
319 };
320 const projected = projectTurnItemForDetail(item);
321 const output = projected.type === "dynamic_tool" ? projected.output : null;
322 // The truncated text block plus a fixed number of markers, however many images there are.
323 expect(Array.isArray(output) ? output.length : null).toBe(1 + MAX_TOOL_OUTPUT_IMAGES);
324 expect(Array.isArray(output) ? String(output[0]?.text).length : null).toBeLessThan(263_000);
325 expect(toolOutputImages(output)).toHaveLength(MAX_TOOL_OUTPUT_IMAGES);
326 });
327
277 it("keeps failure evidence without retaining command output", () => {328 it("keeps failure evidence without retaining command output", () => {
278 const item = {329 const item = {
279 ...base,330 ...base,
apps/server/src/orchestration-v2/WireProjection.ts+26-2
...@@ -4,7 +4,12 @@ import type {...@@ -4,7 +4,12 @@ import type {
4 OrchestrationV2ThreadProjection,4 OrchestrationV2ThreadProjection,
5 OrchestrationV2TurnItem,5 OrchestrationV2TurnItem,
6} from "@t3tools/contracts";6} from "@t3tools/contracts";
7import { compactDynamicToolOutput, toolOutputIndicatesFailure } from "@t3tools/shared/toolOutput";7import {
8 compactDynamicToolOutput,
9 omitToolOutputImageData,
10 toolOutputImages,
11 toolOutputIndicatesFailure,
12} from "@t3tools/shared/toolOutput";
813
9const MAX_DETAIL_STRING_BYTES = 32_768;14const MAX_DETAIL_STRING_BYTES = 32_768;
10const MAX_DYNAMIC_VALUE_BYTES = 16_384;15const MAX_DYNAMIC_VALUE_BYTES = 16_384;
...@@ -142,9 +147,28 @@ function boundDynamicValue(value: unknown): unknown {...@@ -142,9 +147,28 @@ function boundDynamicValue(value: unknown): unknown {
142 : truncateDetail(json, MAX_ON_DEMAND_BYTES);147 : truncateDetail(json, MAX_ON_DEMAND_BYTES);
143}148}
144149
150/**
151 * A tool output for a detail read, without image bytes. When the rest is too
152 * large to send whole, its truncated text still comes with the image markers,
153 * in order, so clients can load each image by index.
154 */
155function boundToolOutput(output: unknown): unknown {
156 const omitted = omitToolOutputImageData(output);
157 const bounded = boundDynamicValue(omitted);
158 if (typeof bounded !== "string" || typeof omitted === "string") return bounded;
159 const images = toolOutputImages(omitted);
160 return images.length === 0
161 ? bounded
162 : [
163 { type: "text", text: bounded },
164 ...images.map((image) => ({ type: "image", mimeType: image.mimeType })),
165 ];
166}
167
145/**168/**
146 * Projects one item for an on-demand detail read: keeps the input and output169 * Projects one item for an on-demand detail read: keeps the input and output
147 * the timeline withholds, bounded so a huge result cannot stall the socket.170 * the timeline withholds, bounded so a huge result cannot stall the socket.
171 * Image bytes are left out; clients load them as `tool-output-image` assets.
148 */172 */
149export function projectTurnItemForDetail(item: OrchestrationV2TurnItem): OrchestrationV2TurnItem {173export function projectTurnItemForDetail(item: OrchestrationV2TurnItem): OrchestrationV2TurnItem {
150 switch (item.type) {174 switch (item.type) {
...@@ -158,7 +182,7 @@ export function projectTurnItemForDetail(item: OrchestrationV2TurnItem): Orchest...@@ -158,7 +182,7 @@ export function projectTurnItemForDetail(item: OrchestrationV2TurnItem): Orchest
158 return {182 return {
159 ...item,183 ...item,
160 input: boundDynamicValue(item.input),184 input: boundDynamicValue(item.input),
161 output: boundDynamicValue(item.output),185 output: boundToolOutput(item.output),
162 };186 };
163 case "subagent":187 case "subagent":
164 return {188 return {
apps/server/src/ws.ts+1
...@@ -3148,6 +3148,7 @@ const makeWsRpcLayer = (...@@ -3148,6 +3148,7 @@ const makeWsRpcLayer = (
3148 if (3148 if (
3149 input.resource._tag === "attachment" ||3149 input.resource._tag === "attachment" ||
3150 input.resource._tag === "native-app-icon" ||3150 input.resource._tag === "native-app-icon" ||
3151 input.resource._tag === "tool-output-image" ||
3151 // GitHub media names the repository it authenticates through itself.3152 // GitHub media names the repository it authenticates through itself.
3152 input.resource._tag === "github-media" ||3153 input.resource._tag === "github-media" ||
3153 (input.resource._tag === "media-file" && path.isAbsolute(input.resource.path))3154 (input.resource._tag === "media-file" && path.isAbsolute(input.resource.path))
apps/web/src/components/ChatMarkdown.tsx+8-1
...@@ -1762,7 +1762,14 @@ export const ChatMarkdownAssetImage = memo(function ChatMarkdownAssetImage(props...@@ -1762,7 +1762,14 @@ export const ChatMarkdownAssetImage = memo(function ChatMarkdownAssetImage(props
1762 readonly environmentId: EnvironmentId;1762 readonly environmentId: EnvironmentId;
1763 readonly resource: Extract<1763 readonly resource: Extract<
1764 AssetResource,1764 AssetResource,
1765 { readonly _tag: "attachment" | "workspace-file" | "media-file" | "github-media" }1765 {
1766 readonly _tag:
1767 | "attachment"
1768 | "workspace-file"
1769 | "media-file"
1770 | "github-media"
1771 | "tool-output-image";
1772 }
1766 >;1773 >;
1767 readonly kind?: "image" | "video";1774 readonly kind?: "image" | "video";
1768 readonly alt: string;1775 readonly alt: string;
apps/web/src/components/chat/MessagesTimeline.tsx+4
...@@ -2878,6 +2878,7 @@ function V2EventTimelineRow({ row }: { row: Extract<TimelineRow, { kind: "event"...@@ -2878,6 +2878,7 @@ function V2EventTimelineRow({ row }: { row: Extract<TimelineRow, { kind: "event"
2878 onOpenThread={ctx.onOpenThread}2878 onOpenThread={ctx.onOpenThread}
2879 onOpenTurnDiff={ctx.onOpenTurnDiff}2879 onOpenTurnDiff={ctx.onOpenTurnDiff}
2880 onRollbackCheckpoint={ctx.onRollbackCheckpoint}2880 onRollbackCheckpoint={ctx.onRollbackCheckpoint}
2881 onImageExpand={ctx.onImageExpand}
2881 />2882 />
2882 </div>2883 </div>
2883 </div>2884 </div>
...@@ -2952,6 +2953,7 @@ function V2EventTimelineRow({ row }: { row: Extract<TimelineRow, { kind: "event"...@@ -2952,6 +2953,7 @@ function V2EventTimelineRow({ row }: { row: Extract<TimelineRow, { kind: "event"
2952 onOpenThread={ctx.onOpenThread}2953 onOpenThread={ctx.onOpenThread}
2953 onOpenTurnDiff={ctx.onOpenTurnDiff}2954 onOpenTurnDiff={ctx.onOpenTurnDiff}
2954 onRollbackCheckpoint={ctx.onRollbackCheckpoint}2955 onRollbackCheckpoint={ctx.onRollbackCheckpoint}
2956 onImageExpand={ctx.onImageExpand}
2955 />2957 />
2956 </div>2958 </div>
2957 </div>2959 </div>
...@@ -5357,6 +5359,7 @@ function WorkEntryLogRow(props: WorkEntryRowProps) {...@@ -5357,6 +5359,7 @@ function WorkEntryLogRow(props: WorkEntryRowProps) {
5357 onOpenThread={ctx.onOpenThread}5359 onOpenThread={ctx.onOpenThread}
5358 onOpenTurnDiff={ctx.onOpenTurnDiff}5360 onOpenTurnDiff={ctx.onOpenTurnDiff}
5359 onRollbackCheckpoint={ctx.onRollbackCheckpoint}5361 onRollbackCheckpoint={ctx.onRollbackCheckpoint}
5362 onImageExpand={ctx.onImageExpand}
5360 />5363 />
5361 ) : (5364 ) : (
5362 <>5365 <>
...@@ -5367,6 +5370,7 @@ function WorkEntryLogRow(props: WorkEntryRowProps) {...@@ -5367,6 +5370,7 @@ function WorkEntryLogRow(props: WorkEntryRowProps) {
5367 <FetchedToolOutput5370 <FetchedToolOutput
5368 projectedItem={workEntry.projectedItem}5371 projectedItem={workEntry.projectedItem}
5369 environmentId={ctx.activeThreadEnvironmentId}5372 environmentId={ctx.activeThreadEnvironmentId}
5373 onImageExpand={onImageExpand}
5370 />5374 />
5371 ) : null}5375 ) : null}
5372 </>5376 </>
apps/web/src/components/chat/V2ItemInspector.tsx+39-7
...@@ -8,6 +8,7 @@ import {...@@ -8,6 +8,7 @@ import {
8 toolCallLines,8 toolCallLines,
9 turnItemDetailRevision,9 turnItemDetailRevision,
10 turnItemNeedsDetailFetch,10 turnItemNeedsDetailFetch,
11 turnItemOutputImages,
11 turnItemOutputText,12 turnItemOutputText,
12} from "@t3tools/client-runtime/work-log/item-detail";13} from "@t3tools/client-runtime/work-log/item-detail";
13import { ExternalLinkIcon, GitBranchIcon, RotateCcwIcon } from "lucide-react";14import { ExternalLinkIcon, GitBranchIcon, RotateCcwIcon } from "lucide-react";
...@@ -21,9 +22,10 @@ import { useTurnItemDetail } from "../../state/queries";...@@ -21,9 +22,10 @@ import { useTurnItemDetail } from "../../state/queries";
21import { useV2ItemSupport } from "../../state/v2ItemSupport";22import { useV2ItemSupport } from "../../state/v2ItemSupport";
22import { formatWorkspaceRelativePath } from "../../filePathDisplay";23import { formatWorkspaceRelativePath } from "../../filePathDisplay";
23import { Button } from "../ui/button";24import { Button } from "../ui/button";
24import ChatMarkdown from "../ChatMarkdown";25import ChatMarkdown, { ChatMarkdownAssetImage } from "../ChatMarkdown";
25import { RenderErrorBoundary } from "../RenderErrorBoundary";26import { RenderErrorBoundary } from "../RenderErrorBoundary";
26import { resolveExternalWebLinkHref } from "./externalLinkContextMenu";27import { resolveExternalWebLinkHref } from "./externalLinkContextMenu";
28import type { ExpandedImagePreview } from "./ExpandedImagePreview";
27import { ShellCommandBlock } from "./ShellCommandBlock";29import { ShellCommandBlock } from "./ShellCommandBlock";
2830
29interface V2ItemInspectorProps {31interface V2ItemInspectorProps {
...@@ -37,6 +39,7 @@ interface V2ItemInspectorProps {...@@ -37,6 +39,7 @@ interface V2ItemInspectorProps {
37 readonly checkpointId: string;39 readonly checkpointId: string;
38 readonly scopeId: string;40 readonly scopeId: string;
39 }) => void;41 }) => void;
42 readonly onImageExpand?: ((preview: ExpandedImagePreview) => void) | undefined;
40}43}
4144
42function JsonTokens({ text }: { readonly text: string }) {45function JsonTokens({ text }: { readonly text: string }) {
...@@ -119,6 +122,8 @@ function useFetchedTurnItem(...@@ -119,6 +122,8 @@ function useFetchedTurnItem(
119 item,122 item,
120 output: {123 output: {
121 output: turnItemOutputText(item),124 output: turnItemOutputText(item),
125 images: turnItemOutputImages(item),
126 environmentId,
122 pending: item === wireItem && detail.isPending,127 pending: item === wireItem && detail.isPending,
123 error:128 error:
124 item !== wireItem129 item !== wireItem
...@@ -133,32 +138,52 @@ function useFetchedTurnItem(...@@ -133,32 +138,52 @@ function useFetchedTurnItem(
133138
134interface ToolOutputState {139interface ToolOutputState {
135 readonly output: string | null;140 readonly output: string | null;
141 readonly images: ReturnType<typeof turnItemOutputImages>;
142 readonly environmentId: EnvironmentId;
136 readonly pending: boolean;143 readonly pending: boolean;
137 readonly error: string | null;144 readonly error: string | null;
138 readonly empty: boolean;145 readonly empty: boolean;
146 readonly onImageExpand?: ((preview: ExpandedImagePreview) => void) | undefined;
139}147}
140148
141function ToolOutput(props: ToolOutputState) {149function ToolOutput(props: ToolOutputState) {
142 return props.output ? (150 const images = props.images.map((resource) => (
151 <ChatMarkdownAssetImage
152 key={resource.index}
153 environmentId={props.environmentId}
154 resource={resource}
155 alt="Tool output image"
156 maxHeightRem={16}
157 onImageExpand={props.onImageExpand}
158 />
159 ));
160 const text = props.output ? (
143 <div className="max-h-80 overflow-auto text-muted-foreground">{props.output}</div>161 <div className="max-h-80 overflow-auto text-muted-foreground">{props.output}</div>
144 ) : props.pending ? (162 ) : props.pending ? (
145 <div className="text-muted-foreground italic">Loading output…</div>163 <div className="text-muted-foreground italic">Loading output…</div>
146 ) : props.error ? (164 ) : props.error ? (
147 <div className="text-destructive">Couldn&apos;t load output: {props.error}</div>165 <div className="text-destructive">Couldn&apos;t load output: {props.error}</div>
148 ) : props.empty ? (166 ) : props.empty && images.length === 0 ? (
149 <div className="text-muted-foreground italic">No output.</div>167 <div className="text-muted-foreground italic">No output.</div>
150 ) : null;168 ) : null;
169 return (
170 <>
171 {images}
172 {text}
173 </>
174 );
151}175}
152176
153/** Fetched output for rows that show their own plain text instead of the inspector. */177/** Fetched output for rows that show their own plain text instead of the inspector. */
154export function FetchedToolOutput(props: {178export function FetchedToolOutput(props: {
155 readonly projectedItem: OrchestrationV2ProjectedTurnItem;179 readonly projectedItem: OrchestrationV2ProjectedTurnItem;
156 readonly environmentId: EnvironmentId;180 readonly environmentId: EnvironmentId;
181 readonly onImageExpand?: ((preview: ExpandedImagePreview) => void) | undefined;
157}) {182}) {
158 const { output } = useFetchedTurnItem(props.projectedItem, props.environmentId);183 const { output } = useFetchedTurnItem(props.projectedItem, props.environmentId);
159 return (184 return (
160 <div className={cn("mt-1.5", monoClassName)}>185 <div className={cn("mt-1.5 space-y-1.5", monoClassName)}>
161 <ToolOutput {...output} />186 <ToolOutput {...output} onImageExpand={props.onImageExpand} />
162 </div>187 </div>
163 );188 );
164}189}
...@@ -220,7 +245,12 @@ export const V2ItemInspector = memo(function V2ItemInspector(props: V2ItemInspec...@@ -220,7 +245,12 @@ export const V2ItemInspector = memo(function V2ItemInspector(props: V2ItemInspec
220 ) : null}245 ) : null}
221246
222 {item.type === "command_execution" ? (247 {item.type === "command_execution" ? (
223 <ToolCallBody command={item.input} exitCode={item.exitCode} {...outputState} />248 <ToolCallBody
249 command={item.input}
250 exitCode={item.exitCode}
251 {...outputState}
252 onImageExpand={props.onImageExpand}
253 />
224 ) : null}254 ) : null}
225255
226 {item.type === "file_change" ? (256 {item.type === "file_change" ? (
...@@ -317,7 +347,9 @@ export const V2ItemInspector = memo(function V2ItemInspector(props: V2ItemInspec...@@ -317,7 +347,9 @@ export const V2ItemInspector = memo(function V2ItemInspector(props: V2ItemInspec
317 </ul>347 </ul>
318 ) : null}348 ) : null}
319349
320 {item.type === "dynamic_tool" ? <ToolCallBody args={item.input} {...outputState} /> : null}350 {item.type === "dynamic_tool" ? (
351 <ToolCallBody args={item.input} {...outputState} onImageExpand={props.onImageExpand} />
352 ) : null}
321353
322 {item.type === "approval_request" ? <StructuredValue value={item.prompt} /> : null}354 {item.type === "approval_request" ? <StructuredValue value={item.prompt} /> : null}
323 {item.type === "user_input_request" ? (355 {item.type === "user_input_request" ? (
packages/client-runtime/src/work-log/itemDetail.test.ts created+55
...@@ -0,0 +1,55 @@
1import { ThreadId, TurnItemId } from "@t3tools/contracts";
2import * as DateTime from "effect/DateTime";
3import { describe, expect, it } from "vite-plus/test";
4
5import { turnItemOutputImages, turnItemOutputText } from "./itemDetail.ts";
6
7const screenshot = {
8 id: TurnItemId.make("tool-screenshot"),
9 type: "dynamic_tool" as const,
10 threadId: ThreadId.make("thread-1"),
11 runId: null,
12 nodeId: null,
13 providerThreadId: null,
14 providerTurnId: null,
15 nativeItemRef: null,
16 parentItemId: null,
17 ordinal: 1,
18 status: "completed" as const,
19 title: null,
20 toolName: "mcp__t3-code__device_screenshot",
21 input: { deviceId: "phone" },
22 // What a detail read returns: the image's position without its bytes.
23 output: {
24 content: [
25 { type: "text", text: "Captured the home screen." },
26 { type: "image", mimeType: "image/png" },
27 ],
28 },
29 startedAt: DateTime.makeUnsafe("2026-10-05T00:00:00.000Z"),
30 completedAt: DateTime.makeUnsafe("2026-10-05T00:00:01.000Z"),
31 updatedAt: DateTime.makeUnsafe("2026-10-05T00:00:01.000Z"),
32};
33
34describe("tool output images", () => {
35 it("shows a screenshot as an image asset, not as text", () => {
36 expect(turnItemOutputImages(screenshot)).toEqual([
37 {
38 _tag: "tool-output-image",
39 threadId: screenshot.threadId,
40 itemId: screenshot.id,
41 index: 0,
42 },
43 ]);
44 expect(turnItemOutputText(screenshot)).toBe("Captured the home screen.");
45 expect(
46 turnItemOutputText({ ...screenshot, output: { content: [screenshot.output.content[1]] } }),
47 ).toBeNull();
48 });
49
50 it("keeps a placeholder for images it cannot show", () => {
51 const output = { content: [{ type: "image", mimeType: "image/svg+xml" }] };
52 expect(turnItemOutputImages({ ...screenshot, output })).toEqual([]);
53 expect(turnItemOutputText({ ...screenshot, output })).toBe("[image]");
54 });
55});
packages/client-runtime/src/work-log/itemDetail.ts+23-3
...@@ -1,4 +1,5 @@...@@ -1,4 +1,5 @@
1import type { OrchestrationV2TurnItem } from "@t3tools/contracts";1import type { AssetResource, OrchestrationV2TurnItem } from "@t3tools/contracts";
2import { readToolOutputImage, toolOutputImages } from "@t3tools/shared/toolOutput";
2import * as DateTime from "effect/DateTime";3import * as DateTime from "effect/DateTime";
34
4const MAX_TEXT_BLOCK_DEPTH = 4;5const MAX_TEXT_BLOCK_DEPTH = 4;
...@@ -17,11 +18,14 @@ function textFromBlocks(value: unknown, depth: number): string | null {...@@ -17,11 +18,14 @@ function textFromBlocks(value: unknown, depth: number): string | null {
17 if (typeof value === "string") return value;18 if (typeof value === "string") return value;
18 if (Array.isArray(value)) {19 if (Array.isArray(value)) {
19 const parts = value.map((block) => textFromBlocks(block, depth + 1));20 const parts = value.map((block) => textFromBlocks(block, depth + 1));
20 return parts.every((part) => part !== null) ? parts.join("\n") : null;21 return parts.every((part) => part !== null)
22 ? parts.filter((part) => part !== "").join("\n")
23 : null;
21 }24 }
22 if (!isRecord(value)) return null;25 if (!isRecord(value)) return null;
23 if (value.type === "text" && typeof value.text === "string") return value.text;26 if (value.type === "text" && typeof value.text === "string") return value.text;
24 if (value.type === "image") return "[image]";27 // Images clients can show render on their own (see turnItemOutputImages).
28 if (value.type === "image") return readToolOutputImage(value) ? "" : "[image]";
25 if (value.type === "resource_link" && typeof value.uri === "string") return value.uri;29 if (value.type === "resource_link" && typeof value.uri === "string") return value.uri;
26 if (value.type === "resource" && isRecord(value.resource)) {30 if (value.type === "resource" && isRecord(value.resource)) {
27 const resource = value.resource;31 const resource = value.resource;
...@@ -200,6 +204,22 @@ export function turnItemOutputText(item: OrchestrationV2TurnItem): string | null...@@ -200,6 +204,22 @@ export function turnItemOutputText(item: OrchestrationV2TurnItem): string | null
200 }204 }
201}205}
202206
207/**
208 * Images in a fetched item's tool output, as assets. The detail read leaves
209 * the bytes out, so each loads over HTTP by its index.
210 */
211export function turnItemOutputImages(
212 item: OrchestrationV2TurnItem,
213): ReadonlyArray<Extract<AssetResource, { readonly _tag: "tool-output-image" }>> {
214 if (item.type !== "dynamic_tool" || item.outputOmitted === true) return [];
215 return toolOutputImages(item.output).map((_, index) => ({
216 _tag: "tool-output-image",
217 threadId: item.threadId,
218 itemId: item.id,
219 index,
220 }));
221}
222
203/**223/**
204 * Whether expanding the item shows anything. Rows without content must not224 * Whether expanding the item shows anything. Rows without content must not
205 * offer a disclosure, otherwise they open to an empty panel.225 * offer a disclosure, otherwise they open to an empty panel.
packages/contracts/src/assets.ts+11-2
...@@ -1,6 +1,6 @@...@@ -1,6 +1,6 @@
1import * as Schema from "effect/Schema";1import * as Schema from "effect/Schema";
22
3import { NonNegativeInt, ThreadId, TrimmedNonEmptyString } from "./baseSchemas.ts";3import { NonNegativeInt, ThreadId, TrimmedNonEmptyString, TurnItemId } from "./baseSchemas.ts";
4import {4import {
5 PROVIDER_SEND_TURN_MAX_FILE_BYTES,5 PROVIDER_SEND_TURN_MAX_FILE_BYTES,
6 PROVIDER_SEND_TURN_MAX_IMAGE_BYTES,6 PROVIDER_SEND_TURN_MAX_IMAGE_BYTES,
...@@ -41,6 +41,13 @@ export const AssetResource = Schema.Union([...@@ -41,6 +41,13 @@ export const AssetResource = Schema.Union([
41 inline response after deciding the file type is safe to preview. */41 inline response after deciding the file type is safe to preview. */
42 disposition: Schema.optionalKey(Schema.Literals(["inline", "attachment"])),42 disposition: Schema.optionalKey(Schema.Literals(["inline", "attachment"])),
43 }),43 }),
44 // An image a tool returned inline, such as a device screenshot, by its order
45 // in the stored output. The timeline never carries these bytes.
46 Schema.TaggedStruct("tool-output-image", {
47 threadId: ThreadId,
48 itemId: TurnItemId,
49 index: NonNegativeInt,
50 }),
44 Schema.TaggedStruct("project-favicon", {51 Schema.TaggedStruct("project-favicon", {
45 cwd: TrimmedNonEmptyString.check(Schema.isMaxLength(ASSET_PATH_MAX_LENGTH)),52 cwd: TrimmedNonEmptyString.check(Schema.isMaxLength(ASSET_PATH_MAX_LENGTH)),
46 // A cache-key hint only. The server reads the authoritative path from the53 // A cache-key hint only. The server reads the authoritative path from the
...@@ -218,7 +225,9 @@ export class AssetWorkspaceAssetNotFoundError extends Schema.TaggedError<AssetWo...@@ -218,7 +225,9 @@ export class AssetWorkspaceAssetNotFoundError extends Schema.TaggedError<AssetWo
218 override get message(): string {225 override get message(): string {
219 return this.resource._tag === "media-file"226 return this.resource._tag === "media-file"
220 ? "Media file was not found."227 ? "Media file was not found."
221 : "Workspace asset was not found.";228 : this.resource._tag === "tool-output-image"
229 ? "Tool output image was not found."
230 : "Workspace asset was not found.";
222 }231 }
223}232}
224233
packages/shared/src/toolOutput.test.ts+55-1
...@@ -1,6 +1,12 @@...@@ -1,6 +1,12 @@
1import { describe, expect, it } from "vite-plus/test";1import { describe, expect, it } from "vite-plus/test";
22
3import { compactDynamicToolOutput, toolOutputIndicatesFailure } from "./toolOutput.ts";3import {
4 compactDynamicToolOutput,
5 MAX_TOOL_OUTPUT_IMAGES,
6 omitToolOutputImageData,
7 toolOutputImages,
8 toolOutputIndicatesFailure,
9} from "./toolOutput.ts";
410
5describe("compactDynamicToolOutput", () => {11describe("compactDynamicToolOutput", () => {
6 it("extracts IDs through the MCP result envelopes used by T3 summaries", () => {12 it("extracts IDs through the MCP result envelopes used by T3 summaries", () => {
...@@ -173,3 +179,51 @@ describe("toolOutputIndicatesFailure", () => {...@@ -173,3 +179,51 @@ describe("toolOutputIndicatesFailure", () => {
173 }179 }
174 });180 });
175});181});
182
183describe("toolOutputImages", () => {
184 const mcpImage = { type: "image", data: "AAAA", mimeType: "image/png" };
185 const claudeImage = {
186 type: "image",
187 source: { type: "base64", media_type: "image/JPEG", data: "BBBB" },
188 };
189 const text = { type: "text", text: "{}" };
190
191 it("finds images in each place providers store them, in order", () => {
192 expect(toolOutputImages({ content: [text, claudeImage, mcpImage] })).toEqual([
193 { mimeType: "image/jpeg", data: "BBBB" },
194 { mimeType: "image/png", data: "AAAA" },
195 ]);
196 expect(toolOutputImages([mcpImage])).toEqual([{ mimeType: "image/png", data: "AAAA" }]);
197 expect(toolOutputImages(mcpImage)).toEqual([{ mimeType: "image/png", data: "AAAA" }]);
198 });
199
200 it("skips types the server must not serve inline", () => {
201 expect(
202 toolOutputImages([
203 { ...mcpImage, mimeType: "image/svg+xml" },
204 { ...mcpImage, mimeType: "text/html" },
205 { type: "image", source: { type: "url", url: "https://example.com/a.png" } },
206 ]),
207 ).toEqual([]);
208 });
209
210 it("stops after the first few images", () => {
211 const images = toolOutputImages(Array.from({ length: 10_000 }, () => mcpImage));
212 expect(images).toHaveLength(MAX_TOOL_OUTPUT_IMAGES);
213 });
214
215 it("drops the bytes but keeps each image's position", () => {
216 const omitted = omitToolOutputImageData({ content: [text, claudeImage, mcpImage] });
217 expect(omitted).toEqual({
218 content: [
219 text,
220 { type: "image", mimeType: "image/jpeg" },
221 { type: "image", mimeType: "image/png" },
222 ],
223 });
224 expect(toolOutputImages(omitted)).toEqual([
225 { mimeType: "image/jpeg" },
226 { mimeType: "image/png" },
227 ]);
228 });
229});
packages/shared/src/toolOutput.ts+67
...@@ -1,3 +1,4 @@...@@ -1,3 +1,4 @@
1import { isProviderSendTurnSupportedImageMimeType } from "@t3tools/contracts";
1import * as Predicate from "effect/Predicate";2import * as Predicate from "effect/Predicate";
23
3const MAX_PARSED_BYTES = 16_384;4const MAX_PARSED_BYTES = 16_384;
...@@ -166,3 +167,69 @@ export function toolOutputIndicatesFailure(text: string): boolean {...@@ -166,3 +167,69 @@ export function toolOutputIndicatesFailure(text: string): boolean {
166 /exit code\s*[:\s]\s*[1-9]\d*\b/i.test(text)167 /exit code\s*[:\s]\s*[1-9]\d*\b/i.test(text)
167 );168 );
168}169}
170
171/** An image a tool returned inline. `data` is base64; a detail read omits it. */
172export interface ToolOutputImage {
173 readonly mimeType: string;
174 readonly data?: string;
175}
176
177/**
178 * Reads one image block in the MCP `{ data, mimeType }` shape or the Anthropic
179 * `{ source: { type: "base64", media_type, data } }` shape Claude stores.
180 * Only raster types are recognized, so the server never serves an agent's SVG
181 * or HTML inline.
182 */
183export function readToolOutputImage(block: unknown): ToolOutputImage | null {
184 if (!Predicate.isObject(block) || block.type !== "image") return null;
185 const source = Predicate.isObject(block.source) ? block.source : undefined;
186 if (source !== undefined && source.type !== "base64") return null;
187 const mimeType = source === undefined ? block.mimeType : source.media_type;
188 const data = source === undefined ? block.data : source.data;
189 if (typeof mimeType !== "string" || !isProviderSendTurnSupportedImageMimeType(mimeType)) {
190 return null;
191 }
192 const image = { mimeType: mimeType.toLowerCase() };
193 return typeof data === "string" ? { ...image, data } : image;
194}
195
196/** Tools return a block, a list of blocks, or an MCP result with a `content` list. */
197function outputBlocks(value: unknown): ReadonlyArray<unknown> {
198 if (Array.isArray(value)) return value;
199 if (Predicate.isObject(value) && Array.isArray(value.content)) return value.content;
200 return [value];
201}
202
203/** A tool returns one screenshot or a few frames; more would only flood the timeline. */
204export const MAX_TOOL_OUTPUT_IMAGES = 8;
205
206/**
207 * The first images in a tool output, in order. The order is the
208 * `tool-output-image` asset index, so servers and clients agree on it.
209 */
210export function toolOutputImages(value: unknown): ReadonlyArray<ToolOutputImage> {
211 const images: ToolOutputImage[] = [];
212 for (const block of outputBlocks(value)) {
213 const image = readToolOutputImage(block);
214 if (image === null) continue;
215 images.push(image);
216 if (images.length === MAX_TOOL_OUTPUT_IMAGES) break;
217 }
218 return images;
219}
220
221/**
222 * Replaces each image's bytes with `{ type: "image", mimeType }` and keeps its
223 * position, so detail reads stay small and clients load the bytes as assets.
224 */
225export function omitToolOutputImageData(value: unknown): unknown {
226 const omit = (block: unknown) => {
227 const image = readToolOutputImage(block);
228 return image?.data === undefined ? block : { type: "image", mimeType: image.mimeType };
229 };
230 if (Array.isArray(value)) return value.map(omit);
231 if (Predicate.isObject(value) && Array.isArray(value.content)) {
232 return { ...value, content: value.content.map(omit) };
233 }
234 return omit(value);
235}