authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-09-01 18:26:41-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-01 18:26:41-07:00
logd0b19b32e01d7abc8829c01353d759fed3d304ed
tree450a1bdd5056399eedaf9ddee624ab8e137f2adc
parent716069f40f000887cb01685099a16e19d9194307
signature Signed by PGP key B5690EEEBB952194

fix(claude): preview images read from the workspace (#9119)

Classify Claude Read calls on image files as image_view and carry the full path through projection and both clients so the existing image renderer can load it. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

10 files changed, 384 insertions(+), 3 deletions(-)

apps/mobile/src/lib/threadActivity.test.ts+55
......@@ -504,6 +504,61 @@ describe("buildThreadFeed", () => {
504504 );
505505 });
506506
507 it("keeps viewed image metadata while collapsing a streamed Claude Read", () => {
508 const turnId = TurnId.make("turn-image-read");
509 const imagePath = `/workspace/${"nested folder/".repeat(16)}reference image.webp`;
510 const thread = makeThread({
511 id: ThreadId.make("thread-image-read"),
512 projectId: ProjectId.make("project-1"),
513 title: "Image read",
514 activities: [
515 makeActivity({
516 id: EventId.make("image-read-update"),
517 kind: "tool.updated",
518 tone: "tool",
519 summary: "Image view",
520 createdAt: "2026-04-01T00:00:01.000Z",
521 turnId,
522 payload: {
523 toolCallId: "tool-read-image",
524 itemType: "image_view",
525 status: "inProgress",
526 detail: `${imagePath.slice(0, 177)}...`,
527 data: { imagePath },
528 },
529 }),
530 makeActivity({
531 id: EventId.make("image-read-completed"),
532 kind: "tool.completed",
533 tone: "tool",
534 summary: "Image view",
535 createdAt: "2026-04-01T00:00:02.000Z",
536 turnId,
537 payload: {
538 toolCallId: "tool-read-image",
539 itemType: "image_view",
540 status: "completed",
541 detail: `${imagePath.slice(0, 177)}...`,
542 data: {},
543 },
544 }),
545 ],
546 });
547
548 const group = buildThreadFeed(thread)[0];
549 expect(group).toMatchObject({
550 type: "activity-group",
551 activities: [
552 {
553 workEntry: {
554 itemType: "image_view",
555 viewedImagePath: imagePath,
556 },
557 },
558 ],
559 });
560 });
561
507562 it("keeps MCP inputs available to expanded mobile work rows", () => {
508563 const turnId = TurnId.make("turn-mcp");
509564 const thread = makeThread({
apps/mobile/src/lib/threadActivity.ts+7
......@@ -88,6 +88,7 @@ export interface WorkLogEntry {
8888 turnId: TurnId | null;
8989 label: string;
9090 detail?: string;
91 viewedImagePath?: string;
9192 command?: string;
9293 rawCommand?: string;
9394 changedFiles?: ReadonlyArray<string>;
......@@ -438,6 +439,7 @@ function toDerivedWorkLogEntry(activity: OrchestrationThreadActivity): DerivedWo
438439 }
439440 const itemType = extractWorkLogItemType(payload);
440441 const requestKind = extractWorkLogRequestKind(payload);
442 const viewedImagePath = asTrimmedString(asRecord(payload?.data)?.imagePath);
441443 if (
442444 !taskDetailAsLabel &&
443445 payload &&
......@@ -449,6 +451,9 @@ function toDerivedWorkLogEntry(activity: OrchestrationThreadActivity): DerivedWo
449451 entry.detail = detail;
450452 }
451453 }
454 if (viewedImagePath) {
455 entry.viewedImagePath = viewedImagePath;
456 }
452457 if (commandPreview.command) {
453458 entry.command = commandPreview.command;
454459 }
......@@ -591,6 +596,7 @@ function mergeDerivedWorkLogEntries(
591596): DerivedWorkLogEntry {
592597 const changedFiles = mergeChangedFiles(previous.changedFiles, next.changedFiles);
593598 const detail = next.detail ?? previous.detail;
599 const viewedImagePath = next.viewedImagePath ?? previous.viewedImagePath;
594600 const command = next.command ?? previous.command;
595601 const rawCommand = next.rawCommand ?? previous.rawCommand;
596602 const toolTitle = next.toolTitle ?? previous.toolTitle;
......@@ -606,6 +612,7 @@ function mergeDerivedWorkLogEntries(
606612 id: previous.id,
607613 createdAt: previous.createdAt,
608614 ...(detail ? { detail } : {}),
615 ...(viewedImagePath ? { viewedImagePath } : {}),
609616 ...(command ? { command } : {}),
610617 ...(rawCommand ? { rawCommand } : {}),
611618 ...(changedFiles.length > 0 ? { changedFiles } : {}),
apps/server/src/orchestration/ActivityPayloadProjection.test.ts+27
......@@ -158,6 +158,33 @@ describe("projectActivityPayload", () => {
158158 expect(JSON.stringify(openCode.payload).length).toBeLessThan(200);
159159 });
160160
161 it("keeps full Claude Read image paths through repeated projection", () => {
162 const imagePath = `/workspace/${"nested folder/".repeat(16)}reference image.webp`;
163 const projected = projectActivityPayload(
164 activity({
165 itemType: "dynamic_tool_call",
166 detail: 'Read: {"file_path":"truncated..."}',
167 data: {
168 toolName: "Read",
169 input: { file_path: imagePath },
170 result: { content: "Image Size: 1280x720." },
171 },
172 }),
173 );
174 const projectedAgain = projectActivityPayload(projected);
175
176 expect(projected.payload).toMatchObject({ data: { imagePath } });
177 expect(projectedAgain.payload).toMatchObject({ data: { imagePath } });
178
179 const textRead = projectActivityPayload(
180 activity({
181 itemType: "dynamic_tool_call",
182 data: { toolName: "Read", input: { file_path: "/workspace/src/index.ts" } },
183 }),
184 );
185 expect(textRead.payload).not.toMatchObject({ data: { imagePath: expect.anything() } });
186 });
187
161188 it("slims Codex-shaped mcp_tool_call items to rendered fields plus a result summary", () => {
162189 const projected = projectActivityPayload(
163190 activity({
apps/server/src/orchestration/ActivityPayloadProjection.ts+20
......@@ -3,6 +3,7 @@ import type {
33 OrchestrationThreadActivity,
44 OrchestrationThreadDetailSnapshot,
55} from "@t3tools/contracts";
6import { isWorkspaceImagePreviewPath } from "@t3tools/shared/filePreview";
67
78function asRecord(value: unknown): Record<string, unknown> | null {
89 return value !== null && typeof value === "object" && !Array.isArray(value)
......@@ -143,6 +144,21 @@ function projectCommandValue(data: Record<string, unknown>): unknown {
143144 return undefined;
144145}
145146
147function projectViewedImagePath(data: Record<string, unknown>): string | undefined {
148 const directPath = asTrimmedString(data.imagePath);
149 if (directPath && isWorkspaceImagePreviewPath(directPath)) {
150 return directPath;
151 }
152
153 const toolName = asTrimmedString(data.toolName)?.toLowerCase();
154 if (toolName !== "read" && toolName !== "read file") {
155 return undefined;
156 }
157 const input = asRecord(data.input);
158 const inputPath = asTrimmedString(input?.file_path) ?? asTrimmedString(input?.path);
159 return inputPath && isWorkspaceImagePreviewPath(inputPath) ? inputPath : undefined;
160}
161
146162function summarizeToolTextOutput(value: string): string | null {
147163 let meaningfulLineCount = 0;
148164 let offset = 0;
......@@ -374,6 +390,10 @@ export function projectActivityPayload(
374390 if (command !== undefined) {
375391 projectedData.command = command;
376392 }
393 const imagePath = projectViewedImagePath(data);
394 if (imagePath) {
395 projectedData.imagePath = imagePath;
396 }
377397
378398 const changedFiles: string[] = [];
379399 collectChangedFiles(data, changedFiles, new Set<string>(), 0);
apps/server/src/provider/Layers/ClaudeAdapter.test.ts+162
......@@ -46,6 +46,7 @@ import { ProviderAdapterProcessError, ProviderAdapterValidationError } from "../
4646import type { ClaudeAdapterShape } from "../Services/ClaudeAdapter.ts";
4747import { makeClaudeAdapter, type ClaudeAdapterLiveOptions } from "./ClaudeAdapter.ts";
4848const decodeClaudeSettings = Schema.decodeSync(ClaudeSettings);
49const encodeUnknownJsonString = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));
4950
5051// Test-local service tag so the rest of the file can keep using `yield* ClaudeAdapter`.
5152class ClaudeAdapter extends Context.Service<ClaudeAdapter, ClaudeAdapterShape>()(
......@@ -1434,6 +1435,167 @@ describe("ClaudeAdapterLive", () => {
14341435 );
14351436 });
14361437
1438 it.effect("classifies only streamed Read image inputs as image views", () => {
1439 const harness = makeHarness();
1440 return Effect.gen(function* () {
1441 const adapter = yield* ClaudeAdapter;
1442 const runtimeEventsFiber = yield* Stream.takeUntil(
1443 adapter.streamEvents,
1444 (event) => event.type === "turn.completed",
1445 ).pipe(Stream.runCollect, Effect.forkChild);
1446
1447 const session = yield* adapter.startSession({
1448 threadId: THREAD_ID,
1449 provider: ProviderDriverKind.make("claudeAgent"),
1450 runtimeMode: "full-access",
1451 });
1452 yield* adapter.sendTurn({
1453 threadId: session.threadId,
1454 input: "inspect both files",
1455 attachments: [],
1456 });
1457
1458 const imagePath = `/workspace/${"nested folder/".repeat(16)}reference image.webp`;
1459 harness.query.emit({
1460 type: "stream_event",
1461 session_id: "sdk-session-read-image",
1462 uuid: "read-image-start",
1463 parent_tool_use_id: null,
1464 event: {
1465 type: "content_block_start",
1466 index: 0,
1467 content_block: {
1468 type: "tool_use",
1469 id: "tool-read-image",
1470 name: "Read",
1471 input: {},
1472 },
1473 },
1474 } as unknown as SDKMessage);
1475 harness.query.emit({
1476 type: "stream_event",
1477 session_id: "sdk-session-read-image",
1478 uuid: "read-image-input",
1479 parent_tool_use_id: null,
1480 event: {
1481 type: "content_block_delta",
1482 index: 0,
1483 delta: {
1484 type: "input_json_delta",
1485 partial_json: encodeUnknownJsonString({ file_path: imagePath }),
1486 },
1487 },
1488 } as unknown as SDKMessage);
1489 harness.query.emit({
1490 type: "user",
1491 session_id: "sdk-session-read-image",
1492 uuid: "read-image-result",
1493 parent_tool_use_id: null,
1494 message: {
1495 role: "user",
1496 content: [
1497 {
1498 type: "tool_result",
1499 tool_use_id: "tool-read-image",
1500 content: "Image Size: 1280x720.",
1501 },
1502 ],
1503 },
1504 } as unknown as SDKMessage);
1505
1506 harness.query.emit({
1507 type: "stream_event",
1508 session_id: "sdk-session-read-image",
1509 uuid: "read-text-start",
1510 parent_tool_use_id: null,
1511 event: {
1512 type: "content_block_start",
1513 index: 1,
1514 content_block: {
1515 type: "tool_use",
1516 id: "tool-read-text",
1517 name: "Read",
1518 input: { file_path: "/workspace/src/index.ts" },
1519 },
1520 },
1521 } as unknown as SDKMessage);
1522 harness.query.emit({
1523 type: "user",
1524 session_id: "sdk-session-read-image",
1525 uuid: "read-text-result",
1526 parent_tool_use_id: null,
1527 message: {
1528 role: "user",
1529 content: [
1530 {
1531 type: "tool_result",
1532 tool_use_id: "tool-read-text",
1533 content: "export {};",
1534 },
1535 ],
1536 },
1537 } as unknown as SDKMessage);
1538 harness.query.emit({
1539 type: "result",
1540 subtype: "success",
1541 is_error: false,
1542 errors: [],
1543 session_id: "sdk-session-read-image",
1544 uuid: "read-image-turn-result",
1545 } as unknown as SDKMessage);
1546
1547 const runtimeEvents = Array.from(yield* Fiber.join(runtimeEventsFiber));
1548 const imageEvents = runtimeEvents.filter(
1549 (
1550 event,
1551 ): event is Extract<
1552 ProviderRuntimeEvent,
1553 { type: "item.started" | "item.updated" | "item.completed" }
1554 > =>
1555 (event.type === "item.started" ||
1556 event.type === "item.updated" ||
1557 event.type === "item.completed") &&
1558 String(event.itemId) === "tool-read-image",
1559 );
1560 assert.deepEqual(
1561 imageEvents.map((event) => [event.type, event.payload.itemType]),
1562 [
1563 ["item.started", "dynamic_tool_call"],
1564 ["item.updated", "image_view"],
1565 ["item.updated", "image_view"],
1566 ["item.completed", "image_view"],
1567 ],
1568 );
1569 for (const event of imageEvents.slice(1)) {
1570 assert.equal(event.payload.detail, imagePath);
1571 assert.equal(
1572 (event.payload.data as { input?: { file_path?: string } } | undefined)?.input?.file_path,
1573 imagePath,
1574 );
1575 }
1576
1577 const textEvents = runtimeEvents.filter(
1578 (
1579 event,
1580 ): event is Extract<
1581 ProviderRuntimeEvent,
1582 { type: "item.started" | "item.updated" | "item.completed" }
1583 > =>
1584 (event.type === "item.started" ||
1585 event.type === "item.updated" ||
1586 event.type === "item.completed") &&
1587 String(event.itemId) === "tool-read-text",
1588 );
1589 assert.deepEqual(
1590 textEvents.map((event) => event.payload.itemType),
1591 ["dynamic_tool_call", "dynamic_tool_call", "dynamic_tool_call"],
1592 );
1593 }).pipe(
1594 Effect.provideService(Random.Random, makeDeterministicRandomService()),
1595 Effect.provide(harness.layer),
1596 );
1597 });
1598
14371599 it.effect("falls back to a default plan step label for blank TodoWrite content", () => {
14381600 const harness = makeHarness();
14391601 return Effect.gen(function* () {
apps/server/src/provider/Layers/ClaudeAdapter.ts+32-2
......@@ -20,6 +20,7 @@ import {
2020 type ModelUsage,
2121} from "@anthropic-ai/claude-agent-sdk";
2222import { parseCliArgs } from "@t3tools/shared/cliArgs";
23import { isWorkspaceImagePreviewPath } from "@t3tools/shared/filePreview";
2324import {
2425 ApprovalRequestId,
2526 type CanonicalItemType,
......@@ -713,8 +714,27 @@ function readClaudeResumeState(resumeCursor: unknown): ClaudeResumeState | undef
713714 };
714715}
715716
716function classifyToolItemType(toolName: string): CanonicalItemType {
717function readToolImagePath(toolName: string, input: Record<string, unknown>): string | undefined {
718 const normalized = toolName.trim().toLowerCase();
719 if (normalized !== "read" && normalized !== "read file") {
720 return undefined;
721 }
722 const pathValue = input.file_path ?? input.path;
723 if (typeof pathValue !== "string") {
724 return undefined;
725 }
726 const path = pathValue.trim();
727 return path.length > 0 && isWorkspaceImagePreviewPath(path) ? path : undefined;
728}
729
730function classifyToolItemType(
731 toolName: string,
732 input: Record<string, unknown> = {},
733): CanonicalItemType {
717734 const normalized = toolName.toLowerCase();
735 if (readToolImagePath(toolName, input)) {
736 return "image_view";
737 }
718738 if (normalized.includes("agent")) {
719739 return "collab_agent_tool_call";
720740 }
......@@ -1166,6 +1186,11 @@ function workflowAgentStatus(entry: ClaudeWorkflowAgentEntry): RuntimeTaskStatus
11661186}
11671187
11681188function summarizeToolRequest(toolName: string, input: Record<string, unknown>): string {
1189 const imagePath = readToolImagePath(toolName, input);
1190 if (imagePath) {
1191 return imagePath;
1192 }
1193
11691194 const commandValue = input.command ?? input.cmd;
11701195 const command = typeof commandValue === "string" ? commandValue : undefined;
11711196 if (command && command.trim().length > 0) {
......@@ -2529,9 +2554,14 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* (
25292554
25302555 const partialInputJson = tool.partialInputJson + event.delta.partial_json;
25312556 const parsedInput = tryParseJsonRecord(partialInputJson);
2557 const itemType = parsedInput
2558 ? classifyToolItemType(tool.toolName, parsedInput)
2559 : tool.itemType;
25322560 const detail = parsedInput ? summarizeToolRequest(tool.toolName, parsedInput) : tool.detail;
25332561 let nextTool: ToolInFlight = {
25342562 ...tool,
2563 itemType,
2564 title: titleForTool(itemType),
25352565 partialInputJson,
25362566 ...(parsedInput ? { input: parsedInput } : {}),
25372567 ...(detail ? { detail } : {}),
......@@ -2636,11 +2666,11 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* (
26362666 }
26372667
26382668 const toolName = block.name;
2639 const itemType = classifyToolItemType(toolName);
26402669 const toolInput =
26412670 typeof block.input === "object" && block.input !== null
26422671 ? (block.input as Record<string, unknown>)
26432672 : {};
2673 const itemType = classifyToolItemType(toolName, toolInput);
26442674 const itemId = block.id;
26452675 const detail = summarizeToolRequest(toolName, toolInput);
26462676 const inputFingerprint =
apps/web/src/session-logic.test.ts+38
......@@ -1653,6 +1653,44 @@ describe("deriveWorkLogEntries", () => {
16531653 });
16541654 });
16551655
1656 it("keeps viewed image metadata while collapsing a streamed Claude Read", () => {
1657 const imagePath = `/workspace/${"nested folder/".repeat(16)}reference image.webp`;
1658 const activities: OrchestrationThreadActivity[] = [
1659 makeActivity({
1660 id: "image-read-update",
1661 createdAt: "2026-02-23T00:00:01.000Z",
1662 kind: "tool.updated",
1663 summary: "Image view",
1664 payload: {
1665 toolCallId: "tool-read-image",
1666 itemType: "image_view",
1667 detail: `${imagePath.slice(0, 177)}...`,
1668 data: { imagePath },
1669 },
1670 }),
1671 makeActivity({
1672 id: "image-read-complete",
1673 createdAt: "2026-02-23T00:00:02.000Z",
1674 kind: "tool.completed",
1675 summary: "Image view",
1676 payload: {
1677 toolCallId: "tool-read-image",
1678 itemType: "image_view",
1679 detail: `${imagePath.slice(0, 177)}...`,
1680 data: {},
1681 },
1682 }),
1683 ];
1684
1685 const entries = deriveWorkLogEntries(activities);
1686 expect(entries).toHaveLength(1);
1687 expect(entries[0]).toMatchObject({
1688 id: "image-read-complete",
1689 itemType: "image_view",
1690 viewedImagePath: imagePath,
1691 });
1692 });
1693
16561694 it("does not use command stdout as the detail when Cursor omits the command input", () => {
16571695 const activities: OrchestrationThreadActivity[] = [
16581696 makeActivity({
apps/web/src/session-logic.ts+7
......@@ -73,6 +73,7 @@ export interface WorkLogEntry {
7373 toolCallId?: string;
7474 label: string;
7575 detail?: string;
76 viewedImagePath?: string;
7677 command?: string;
7778 rawCommand?: string;
7879 changedFiles?: ReadonlyArray<string>;
......@@ -924,9 +925,13 @@ function toDerivedWorkLogEntry(activity: OrchestrationThreadActivity): DerivedWo
924925 };
925926 const itemType = extractWorkLogItemType(payload);
926927 const requestKind = extractWorkLogRequestKind(payload);
928 const viewedImagePath = asTrimmedString(asRecord(payload?.data)?.imagePath);
927929 if (detail) {
928930 entry.detail = detail;
929931 }
932 if (viewedImagePath) {
933 entry.viewedImagePath = viewedImagePath;
934 }
930935 if (commandPreview.command) {
931936 entry.command = commandPreview.command;
932937 }
......@@ -1156,6 +1161,7 @@ function mergeDerivedWorkLogEntries(
11561161): DerivedWorkLogEntry {
11571162 const changedFiles = mergeChangedFiles(previous.changedFiles, next.changedFiles);
11581163 const detail = next.detail ?? previous.detail;
1164 const viewedImagePath = next.viewedImagePath ?? previous.viewedImagePath;
11591165 const command = next.command ?? previous.command;
11601166 const rawCommand = next.rawCommand ?? previous.rawCommand;
11611167 const toolTitle = next.toolTitle ?? previous.toolTitle;
......@@ -1169,6 +1175,7 @@ function mergeDerivedWorkLogEntries(
11691175 ...previous,
11701176 ...next,
11711177 ...(detail ? { detail } : {}),
1178 ...(viewedImagePath ? { viewedImagePath } : {}),
11721179 ...(command ? { command } : {}),
11731180 ...(rawCommand ? { rawCommand } : {}),
11741181 ...(changedFiles.length > 0 ? { changedFiles } : {}),
packages/client-runtime/src/work-log/presentation.test.ts+26-1
......@@ -2,7 +2,11 @@ import { describe, expect, it } from "vite-plus/test";
22
33import { ThreadId } from "@t3tools/contracts";
44
5import { resolveViewedImageAsset, workEntryViewedImagePath } from "./presentation.js";
5import {
6 resolveViewedImageAsset,
7 toolGroupAction,
8 workEntryViewedImagePath,
9} from "./presentation.js";
610
711describe("workEntryViewedImagePath", () => {
812 const entry = { label: "Read", tone: "tool" } as const;
......@@ -19,6 +23,14 @@ describe("workEntryViewedImagePath", () => {
1923 detail: "C:\\workspace\\a.webp",
2024 }),
2125 ).toBe("C:\\workspace\\a.webp");
26 expect(
27 workEntryViewedImagePath({
28 ...entry,
29 itemType: "dynamic_tool_call",
30 detail: 'Read: {"file_path":"truncated..."}',
31 viewedImagePath: " /workspace/reference image.webp ",
32 }),
33 ).toBe("/workspace/reference image.webp");
2234 });
2335
2436 it("rejects non-image, multi-line, and non-read details", () => {
......@@ -32,6 +44,19 @@ describe("workEntryViewedImagePath", () => {
3244 });
3345});
3446
47describe("toolGroupAction", () => {
48 it("groups legacy Claude image reads with other reads", () => {
49 expect(
50 toolGroupAction({
51 label: "Tool call",
52 tone: "tool",
53 itemType: "dynamic_tool_call",
54 viewedImagePath: "/workspace/reference.png",
55 }),
56 ).toBe("read");
57 });
58});
59
3560describe("resolveViewedImageAsset", () => {
3661 const threadId = ThreadId.make("thread-1");
3762
packages/client-runtime/src/work-log/presentation.ts+10
......@@ -20,6 +20,7 @@ export interface WorkLogPresentationEntry {
2020 readonly tone: "thinking" | "tool" | "info" | "error";
2121 readonly command?: string;
2222 readonly detail?: string;
23 readonly viewedImagePath?: string;
2324 readonly changedFiles?: ReadonlyArray<string>;
2425 readonly itemType?: ToolLifecycleItemType;
2526 readonly requestKind?: string;
......@@ -68,6 +69,7 @@ export function toolGroupAction(entry: WorkLogPresentationEntry): ToolGroupActio
6869 if (
6970 entry.requestKind === "file-read" ||
7071 entry.itemType === "image_view" ||
72 entry.viewedImagePath !== undefined ||
7173 (entry.itemType === "dynamic_tool_call" &&
7274 entry.toolTitle?.trim().toLowerCase() === "read file")
7375 ) {
......@@ -89,6 +91,14 @@ export function toolGroupAction(entry: WorkLogPresentationEntry): ToolGroupActio
8991}
9092
9193export function workEntryViewedImagePath(entry: WorkLogPresentationEntry): string | null {
94 const viewedImagePath = entry.viewedImagePath?.trim();
95 if (
96 viewedImagePath !== undefined &&
97 !/[\r\n]/.test(viewedImagePath) &&
98 isWorkspaceImagePreviewPath(viewedImagePath)
99 ) {
100 return viewedImagePath;
101 }
92102 const detail = entry.detail?.trim();
93103 return toolGroupAction(entry) === "read" &&
94104 detail !== undefined &&