authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-09-08 05:41:10-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-08 05:41:10-07:00
log061543e9e5b54ec0048725c37d52fef2962df173
treee082e26d2e997d0eaab372d50b14691f6ed21aa2
parentb5d89038ae72142038dfa8cf69d49b7a607fe98e
signature Signed by PGP key B5690EEEBB952194

fix(mcp): keep preview snapshots usable by the agent and let it save them (#10501)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>

6 files changed, 594 insertions(+), 66 deletions(-)

apps/desktop/src/preview/Manager.ts+7-1
......@@ -117,6 +117,12 @@ const ZOOM_EPSILON = 0.001;
117117const MAX_EVALUATION_BYTES = 64_000;
118118const MAX_VISIBLE_TEXT_LENGTH = 20_000;
119119const MAX_INTERACTIVE_ELEMENTS = 200;
120/**
121 * A `[role]` container's innerText is its whole subtree, which turned one
122 * snapshot's element list into 60 KB of repeated page text. Names are labels,
123 * not content, so cap them where they are read.
124 */
125const MAX_INTERACTIVE_ELEMENT_NAME_LENGTH = 200;
120126const MAX_SCREENSHOT_WIDTH = 1280;
121127/** How long an armed tab keeps the exclusive display-media slot before another tab may take it. */
122128const RECORDING_ARM_GRACE_MS = 10_000;
......@@ -3582,7 +3588,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
35823588 return {
35833589 tag: element.tagName.toLowerCase(),
35843590 role: element.getAttribute("role"),
3585 name: element.getAttribute("aria-label") || element.innerText || element.getAttribute("name") || "",
3591 name: (element.getAttribute("aria-label") || element.innerText || element.getAttribute("name") || "").slice(0, ${MAX_INTERACTIVE_ELEMENT_NAME_LENGTH}),
35863592 selector: selectorFor(element),
35873593 x: rect.x,
35883594 y: rect.y,
apps/server/src/config.ts+3
......@@ -38,6 +38,8 @@ export interface ServerDerivedPaths {
3838 readonly providerStatusCacheDir: string;
3939 readonly worktreesDir: string;
4040 readonly attachmentsDir: string;
41 /** Screenshots the agent asks the collaborative browser to keep for the user. */
42 readonly browserArtifactsDir: string;
4143 readonly logsDir: string;
4244 readonly serverLogPath: string;
4345 readonly serverTracePath: string;
......@@ -125,6 +127,7 @@ export const deriveServerPaths = Effect.fn(function* (
125127 providerStatusCacheDir,
126128 worktreesDir: join(baseDir, "worktrees"),
127129 attachmentsDir,
130 browserArtifactsDir: join(stateDir, "browser-artifacts"),
128131 logsDir,
129132 serverLogPath: join(logsDir, "server.log"),
130133 serverTracePath: join(logsDir, "server.trace.ndjson"),
apps/server/src/mcp/McpHttpServer.test.ts+316-34
......@@ -4,12 +4,15 @@ import * as NodeServices from "@effect/platform-node/NodeServices";
44import { EnvironmentId, PreviewTabId, ProviderInstanceId, ThreadId } from "@t3tools/contracts";
55import * as Deferred from "effect/Deferred";
66import * as Effect from "effect/Effect";
7import * as FileSystem from "effect/FileSystem";
78import * as Layer from "effect/Layer";
9import * as Path from "effect/Path";
810import * as Schema from "effect/Schema";
911import * as Stream from "effect/Stream";
1012import { McpProtocol, McpSchema, McpServer } from "effect/unstable/ai";
1113import { HttpBody, HttpClient, HttpRouter, HttpServerResponse } from "effect/unstable/http";
1214
15import * as ServerConfig from "../config.ts";
1316import * as McpHttpServer from "./McpHttpServer.ts";
1417import * as McpInvocationContext from "./McpInvocationContext.ts";
1518import * as PreviewAutomationBroker from "./PreviewAutomationBroker.ts";
......@@ -18,7 +21,7 @@ const environmentId = EnvironmentId.make("environment-mcp-test");
1821const threadId = ThreadId.make("thread-mcp-test");
1922const tabId = PreviewTabId.make("tab-mcp-test");
2023const alternateTabId = PreviewTabId.make("tab-mcp-alternate");
21const encodeJsonText = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));
24const decodeJsonText = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown));
2225const invocation = {
2326 environmentId,
2427 threadId,
......@@ -41,9 +44,62 @@ const client = McpSchema.McpServerClient.of({
4144});
4245const TestLayer = McpHttpServer.PreviewToolkitRegistrationLive.pipe(
4346 Layer.provideMerge(McpServer.McpServer.layer),
44 Layer.provideMerge(PreviewAutomationBroker.layer.pipe(Layer.provide(NodeServices.layer))),
47 Layer.provideMerge(PreviewAutomationBroker.layer),
48 Layer.provideMerge(ServerConfig.layerTest(process.cwd(), { prefix: "t3-mcp-http-server-test-" })),
49 Layer.provideMerge(NodeServices.layer),
4550);
4651
52const snapshotResult = {
53 url: "http://example.test/",
54 title: "Example",
55 loading: false,
56 visibleText: "Example",
57 interactiveElements: [],
58 accessibilityTree: {},
59 consoleEntries: [],
60 networkEntries: [],
61 actionTimeline: [],
62 screenshot: {
63 mimeType: "image/png",
64 data: Buffer.from("png").toString("base64"),
65 width: 10,
66 height: 5,
67 },
68};
69
70/** Answers every snapshot request on a fresh broker host with the given result. */
71const serveSnapshots = (clientId: string, result: unknown) =>
72 Effect.gen(function* () {
73 const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker;
74 const connected = yield* Deferred.make<void>();
75 const inputs: Array<unknown> = [];
76 const events = yield* broker.connect({ clientId, environmentId });
77 yield* Stream.runForEach(events, (event) => {
78 if (event.type === "connected") return Deferred.succeed(connected, undefined);
79 inputs.push(event.request.input);
80 return broker.respond({
81 clientId,
82 connectionId: event.connectionId,
83 requestId: event.request.requestId,
84 ok: true,
85 result,
86 });
87 }).pipe(Effect.forkScoped);
88 yield* Deferred.await(connected);
89 return inputs;
90 });
91
92const callSnapshot = (args: Record<string, unknown>) =>
93 Effect.gen(function* () {
94 const server = yield* McpServer.McpServer;
95 return yield* server
96 .callTool({ name: "preview_snapshot", arguments: args })
97 .pipe(
98 Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
99 Effect.provideService(McpSchema.McpServerClient, client),
100 );
101 });
102
47103it("normalizes empty successful notification responses to accepted", () => {
48104 const notificationResponse = McpHttpServer.normalizeMcpHttpResponse(
49105 HttpServerResponse.text("", { status: 200, contentType: "application/json" }),
......@@ -92,7 +148,9 @@ it.effect.each([{}, { includeImage: false }])(
92148 );
93149
94150 expect(snapshot.isError).toBe(true);
95 expect(snapshot.content).toEqual([{ type: "text", text: "Preview snapshot failed." }]);
151 expect(snapshot.content).toEqual([
152 { type: "text", text: "Preview snapshot failed: PreviewAutomationExecutionError." },
153 ]);
96154 expect(snapshot.structuredContent).toEqual({
97155 error: {
98156 _tag: "PreviewAutomationExecutionError",
......@@ -174,10 +232,16 @@ it.effect.each([
174232 Effect.provideService(McpSchema.McpServerClient, client),
175233 );
176234 const metadata = { ...page, title: `Snapshot ${call}`, screenshot };
235 const { accessibilityTree: _tree, ...boundedMetadata } = metadata;
177236 expect(snapshot.isError).toBe(false);
178237 expect(snapshot.structuredContent).toEqual(metadata);
179 expect(snapshot.content).toEqual([
180 { type: "text", text: encodeJsonText(snapshot.structuredContent) },
238 const [text, ...rest] = snapshot.content;
239 expect(text?.type === "text" ? decodeJsonText(text.text) : null).toEqual(boundedMetadata);
240 expect(rest).toEqual([
241 {
242 type: "text",
243 text: "Snapshot text was bounded. Omitted: accessibilityTree (use interactiveElements locators or preview_evaluate).",
244 },
181245 ...(images
182246 ? [
183247 {
......@@ -200,7 +264,7 @@ it.effect.each([
200264 Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
201265 Effect.provideService(McpSchema.McpServerClient, client),
202266 );
203 expect(nextDefault.content.map((content) => content.type)).toEqual(["text", "image"]);
267 expect(nextDefault.content.map((content) => content.type)).toEqual(["text", "text", "image"]);
204268 expect(nextDefault.structuredContent).toEqual({ ...page, title: "Snapshot 7", screenshot });
205269 expect(requests).toBe(7);
206270 }),
......@@ -221,7 +285,7 @@ it.effect("rejects non-boolean snapshot image options before selecting a browser
221285 Effect.provideService(McpSchema.McpServerClient, client),
222286 );
223287 expect(result.isError).toBe(true);
224 expect(result.content).toEqual([{ type: "text", text: "Preview snapshot failed." }]);
288 expect(result.content).toEqual([{ type: "text", text: "Preview snapshot failed: AiError." }]);
225289 expect(result.structuredContent).toEqual({
226290 error: { _tag: "AiError", operation: "snapshot", failureCount: 1 },
227291 });
......@@ -229,6 +293,222 @@ it.effect("rejects non-boolean snapshot image options before selecting a browser
229293 }).pipe(Effect.provide(TestLayer)),
230294);
231295
296it.effect("saves the snapshot PNG on request and reports its path", () =>
297 Effect.scoped(
298 Effect.gen(function* () {
299 const config = yield* ServerConfig.ServerConfig;
300 const fileSystem = yield* FileSystem.FileSystem;
301 const path = yield* Path.Path;
302 const inputs = yield* serveSnapshots("mcp-save-client", snapshotResult);
303
304 const snapshot = yield* callSnapshot({ save: true });
305
306 expect(snapshot.isError).toBe(false);
307 // The browser never receives the server-only `save` flag.
308 expect(inputs).toEqual([{}]);
309 const structured = snapshot.structuredContent as { readonly screenshotPath?: string };
310 const screenshotPath = structured.screenshotPath;
311 expect(typeof screenshotPath).toBe("string");
312 expect(path.dirname(screenshotPath!)).toBe(config.browserArtifactsDir);
313 expect(path.basename(screenshotPath!)).toMatch(
314 /^browser-screenshot-example-test-[0-9a-z]+-[0-9a-f]{8}\.png$/,
315 );
316 expect(Buffer.from(yield* fileSystem.readFile(screenshotPath!)).toString()).toBe("png");
317 const text = snapshot.content.find((content) => content.type === "text");
318 expect(text?.type === "text" ? text.text : "").toContain(screenshotPath);
319
320 const unsaved = yield* callSnapshot({});
321 expect(unsaved.structuredContent).not.toHaveProperty("screenshotPath");
322 }),
323 ).pipe(Effect.provide(TestLayer)),
324);
325
326it.effect("reports a tagged error when the screenshot cannot be saved", () =>
327 Effect.scoped(
328 Effect.gen(function* () {
329 const config = yield* ServerConfig.ServerConfig;
330 const fileSystem = yield* FileSystem.FileSystem;
331 // A regular file where the artifacts directory should be makes every write fail.
332 yield* fileSystem.writeFileString(config.browserArtifactsDir, "");
333 yield* serveSnapshots("mcp-save-failure-client", snapshotResult);
334
335 const snapshot = yield* callSnapshot({ save: true });
336
337 expect(snapshot.isError).toBe(true);
338 expect(snapshot.content).toEqual([
339 { type: "text", text: "Preview snapshot failed: PreviewScreenshotSaveError." },
340 ]);
341 expect(snapshot.structuredContent).toEqual({
342 error: { _tag: "PreviewScreenshotSaveError", operation: "snapshot", failureCount: 1 },
343 });
344 }),
345 ).pipe(Effect.provide(TestLayer)),
346);
347
348it.effect("keeps the snapshot text under the agent's output ceiling", () =>
349 Effect.scoped(
350 Effect.gen(function* () {
351 // Mirrors the real failure: a [role] container whose innerText is the whole
352 // project list, repeated for several elements, plus a big AX tree.
353 const pageText = "/Users/theo/Code/project\nClaude, Codex · 79 threads\n".repeat(600);
354 const element = (name: string, index: number) => ({
355 tag: "div",
356 role: "presentation",
357 name,
358 selector: `div:nth-of-type(${index})`,
359 x: 0,
360 y: 0,
361 width: 10,
362 height: 10,
363 });
364 const oversized = {
365 ...snapshotResult,
366 visibleText: pageText,
367 interactiveElements: [
368 element(pageText, 1),
369 element(pageText, 2),
370 element(pageText, 3),
371 element("Continue", 4),
372 ],
373 accessibilityTree: { nodes: Array.from({ length: 2_000 }, (_, i) => ({ nodeId: `${i}` })) },
374 consoleEntries: Array.from({ length: 100 }, (_, i) => ({
375 level: "log",
376 text: `entry ${i}`,
377 timestamp: "t",
378 })),
379 };
380 yield* serveSnapshots("mcp-bounded-client", oversized);
381
382 const snapshot = yield* callSnapshot({ includeImage: false });
383
384 expect(snapshot.isError).toBe(false);
385 const [text, notice] = snapshot.content;
386 expect(text?.type).toBe("text");
387 const body = text?.type === "text" ? text.text : "";
388 expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual(
389 McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES,
390 );
391 const parsed = decodeJsonText(body) as {
392 readonly accessibilityTree?: unknown;
393 readonly visibleText: string;
394 readonly interactiveElements: ReadonlyArray<{ readonly name: string }>;
395 readonly consoleEntries: ReadonlyArray<{ readonly text: string }>;
396 };
397 expect(parsed.accessibilityTree).toBeUndefined();
398 expect(parsed.visibleText.length).toBeLessThanOrEqual(8_001);
399 expect(parsed.interactiveElements).toHaveLength(4);
400 expect(parsed.interactiveElements[0]?.name.length).toBeLessThanOrEqual(201);
401 expect(parsed.interactiveElements[3]?.name).toBe("Continue");
402 expect(parsed.consoleEntries).toHaveLength(40);
403 expect(parsed.consoleEntries[0]?.text).toBe("entry 60");
404 expect(notice?.type === "text" ? notice.text : "").toContain("accessibilityTree");
405 expect(notice?.type === "text" ? notice.text : "").toContain("60 older console entries");
406 // The structured result is untouched; only the text the agent reads is bounded.
407 expect(snapshot.structuredContent).toMatchObject({
408 accessibilityTree: oversized.accessibilityTree,
409 });
410 }),
411 ).pipe(Effect.provide(TestLayer)),
412);
413
414it.effect("bounds the snapshot text even when nothing but logs and the title are large", () =>
415 Effect.scoped(
416 Effect.gen(function* () {
417 const oversized = {
418 ...snapshotResult,
419 title: "t".repeat(70_000),
420 interactiveElements: [],
421 consoleEntries: [{ level: "log", text: "x".repeat(70_000), timestamp: "t" }],
422 };
423 yield* serveSnapshots("mcp-bounded-logs-client", oversized);
424
425 const snapshot = yield* callSnapshot({ includeImage: false });
426
427 const [text] = snapshot.content;
428 const body = text?.type === "text" ? text.text : "";
429 expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual(
430 McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES,
431 );
432 const parsed = decodeJsonText(body) as {
433 readonly title: string;
434 readonly consoleEntries: ReadonlyArray<{ readonly text: string }>;
435 };
436 expect(parsed.title.length).toBe(2_049);
437 expect(parsed.consoleEntries[0]?.text.length).toBe(501);
438 const notice = snapshot.content[1];
439 const noticeText = notice?.type === "text" ? notice.text : "";
440 expect(noticeText).toContain("url or title after 2048 characters");
441 expect(noticeText).toContain("console entries text after 500 characters");
442 }),
443 ).pipe(Effect.provide(TestLayer)),
444);
445
446it.effect("sheds log entries before locators when every list is full", () =>
447 Effect.scoped(
448 Effect.gen(function* () {
449 const long = "x".repeat(2_000);
450 const oversized = {
451 ...snapshotResult,
452 interactiveElements: Array.from({ length: 20 }, (_, i) => ({
453 tag: "button",
454 role: "button",
455 name: `Button ${i}`,
456 selector: `#button-${i}`,
457 x: 0,
458 y: 0,
459 width: 10,
460 height: 10,
461 })),
462 consoleEntries: Array.from({ length: 200 }, () => ({
463 level: long,
464 text: long,
465 timestamp: long,
466 source: long,
467 })),
468 networkEntries: Array.from({ length: 200 }, () => ({
469 url: long,
470 method: long,
471 status: 200,
472 failed: false,
473 errorText: long,
474 timestamp: long,
475 })),
476 actionTimeline: Array.from({ length: 200 }, () => ({
477 id: long,
478 action: long,
479 status: "succeeded",
480 startedAt: long,
481 completedAt: long,
482 error: long,
483 })),
484 };
485 yield* serveSnapshots("mcp-full-logs-client", oversized);
486
487 const snapshot = yield* callSnapshot({ includeImage: false });
488
489 const [text, notice] = snapshot.content;
490 const body = text?.type === "text" ? text.text : "";
491 expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual(
492 McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES,
493 );
494 const parsed = decodeJsonText(body) as {
495 readonly interactiveElements: ReadonlyArray<unknown>;
496 readonly consoleEntries: ReadonlyArray<unknown>;
497 readonly networkEntries: ReadonlyArray<unknown>;
498 readonly actionTimeline: ReadonlyArray<unknown>;
499 };
500 // Locators survive; the log lists take the cut.
501 expect(parsed.interactiveElements).toHaveLength(20);
502 expect(
503 parsed.consoleEntries.length + parsed.networkEntries.length + parsed.actionTimeline.length,
504 ).toBeLessThan(120);
505 const noticeText = notice?.type === "text" ? notice.text : "";
506 expect(noticeText).toContain("40 of 40 actionTimeline");
507 expect(noticeText).not.toMatch(/\d+ of \d+ interactiveElements/);
508 }),
509 ).pipe(Effect.provide(TestLayer)),
510);
511
232512it.effect("terminates HTTP MCP sessions with DELETE", () =>
233513 Effect.scoped(
234514 Effect.gen(function* () {
......@@ -306,33 +586,19 @@ it.effect("registers annotated tools and preserves authenticated request context
306586 ok: true,
307587 result:
308588 event.request.operation === "snapshot"
309 ? {
310 url: "http://example.test/",
311 title: "Example",
312 loading: false,
313 visibleText: "Example",
314 interactiveElements: [],
315 accessibilityTree: {},
316 consoleEntries: [],
317 networkEntries: [],
318 actionTimeline: [],
319 screenshot: {
320 mimeType: "image/png",
321 data: Buffer.from("png").toString("base64"),
322 width: 10,
323 height: 5,
324 },
325 }
326 : event.request.operation === "press"
327 ? undefined
328 : {
329 available: true,
330 visible: true,
331 tabId,
332 url: "http://example.test/",
333 title: "Example",
334 loading: false,
335 },
589 ? snapshotResult
590 : event.request.operation === "evaluate"
591 ? ["Connect", "Continue"]
592 : event.request.operation === "press"
593 ? undefined
594 : {
595 available: true,
596 visible: true,
597 tabId,
598 url: "http://example.test/",
599 title: "Example",
600 loading: false,
601 },
336602 });
337603 }).pipe(Effect.forkScoped);
338604 yield* Effect.yieldNow;
......@@ -397,6 +663,22 @@ it.effect("registers annotated tools and preserves authenticated request context
397663 alternateTabId,
398664 );
399665
666 // Arrays and primitives are wrapped so structuredContent stays a JSON object.
667 // Claude Code rejects the whole result otherwise.
668 const evaluateTool = server.tools.find(({ tool }) => tool.name === "preview_evaluate");
669 expect(evaluateTool?.tool.outputSchema).toMatchObject({ type: "object" });
670 const evaluated = yield* server
671 .callTool({ name: "preview_evaluate", arguments: { expression: "buttons()" } })
672 .pipe(
673 Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
674 Effect.provideService(McpSchema.McpServerClient, client),
675 );
676 expect(evaluated.isError).toBe(false);
677 expect(evaluated.structuredContent).toEqual({ value: ["Connect", "Continue"] });
678 expect(evaluated.content).toEqual([
679 { type: "text", text: '{"value":["Connect","Continue"]}' },
680 ]);
681
400682 const actionRequests = [
401683 { name: "preview_click", arguments: { x: 10, y: 10 } },
402684 { name: "preview_type", arguments: { text: "Hello" } },
apps/server/src/mcp/McpHttpServer.ts+243-25
......@@ -1,8 +1,13 @@
1import * as NodeCrypto from "node:crypto";
12import * as Cause from "effect/Cause";
3import * as Clock from "effect/Clock";
24import * as Context from "effect/Context";
35import * as Effect from "effect/Effect";
6import * as FileSystem from "effect/FileSystem";
47import * as Layer from "effect/Layer";
58import * as Option from "effect/Option";
9import * as Path from "effect/Path";
10import * as Schema from "effect/Schema";
611import * as Sink from "effect/Sink";
712import * as Stream from "effect/Stream";
813import type * as Types from "effect/Types";
......@@ -10,6 +15,7 @@ import { McpProtocol, McpSchema, McpServer, Tool } from "effect/unstable/ai";
1015import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
1116
1217import packageJson from "../../package.json" with { type: "json" };
18import * as ServerConfig from "../config.ts";
1319import * as McpInvocationContext from "./McpInvocationContext.ts";
1420import * as McpSessionRegistry from "./McpSessionRegistry.ts";
1521import * as PreviewAutomationBroker from "./PreviewAutomationBroker.ts";
......@@ -95,6 +101,204 @@ const McpAuthMiddlewareLive = HttpRouter.middleware<{
95101 provides: McpInvocationContext.McpInvocationContext;
96102}>()(makeMcpAuthMiddleware).layer;
97103
104/**
105 * Claude Code drops every MCP result above 25k tokens (~100 KB of text) and
106 * hands the agent a truncation notice instead, so a snapshot that carries the
107 * full accessibility tree and 20 KB of page text loses its locators too. Keep
108 * the text under that ceiling and tell the agent what was cut.
109 */
110export const MAX_SNAPSHOT_TEXT_BYTES = 60_000;
111const MAX_SNAPSHOT_VISIBLE_TEXT_CHARS = 8_000;
112const MAX_SNAPSHOT_ELEMENT_NAME_CHARS = 200;
113const MAX_SNAPSHOT_LOG_ENTRIES = 40;
114const MAX_SNAPSHOT_LOG_TEXT_CHARS = 500;
115const MAX_SNAPSHOT_IDENTIFIER_CHARS = 2_048;
116
117const encodeJsonText = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));
118const utf8Length = (text: string) => Buffer.byteLength(text, "utf8");
119const cutText = (text: string, max: number) =>
120 text.length > max ? `${text.slice(0, max)}…` : text;
121
122/** Shortens every string field of a log entry; other fields pass through. */
123const cutEntryStrings = <A>(entry: A): A =>
124 typeof entry === "object" && entry !== null
125 ? (Object.fromEntries(
126 Object.entries(entry).map(([key, value]) => [
127 key,
128 typeof value === "string" ? cutText(value, MAX_SNAPSHOT_LOG_TEXT_CHARS) : value,
129 ]),
130 ) as A)
131 : entry;
132
133const hasLongString = (entry: unknown, max: number) =>
134 typeof entry === "object" &&
135 entry !== null &&
136 Object.values(entry).some((value) => typeof value === "string" && value.length > max);
137
138type SnapshotMetadata = {
139 readonly url: string;
140 readonly title: string;
141 readonly visibleText: string;
142 readonly interactiveElements: ReadonlyArray<{
143 readonly name: string;
144 readonly [key: string]: unknown;
145 }>;
146 readonly consoleEntries: ReadonlyArray<unknown>;
147 readonly networkEntries: ReadonlyArray<unknown>;
148 readonly actionTimeline: ReadonlyArray<unknown>;
149 readonly [key: string]: unknown;
150};
151
152/**
153 * Drops the accessibility tree, shortens page text, element names, identifiers,
154 * and log strings, keeps only the newest log entries, and finally sheds
155 * interactive elements until the JSON fits. Returns the text plus notes on
156 * what is missing so the agent can reach for preview_evaluate.
157 */
158const boundSnapshotMetadata = (
159 metadata: SnapshotMetadata,
160): { readonly text: string; readonly omitted: ReadonlyArray<string> } => {
161 const omitted: Array<string> = [];
162 const { accessibilityTree, ...withoutTree } = metadata;
163 if (accessibilityTree !== undefined) {
164 omitted.push("accessibilityTree (use interactiveElements locators or preview_evaluate)");
165 }
166 const tail = <A>(entries: ReadonlyArray<A>, label: string) => {
167 if (entries.length > MAX_SNAPSHOT_LOG_ENTRIES) {
168 omitted.push(`${entries.length - MAX_SNAPSHOT_LOG_ENTRIES} older ${label}`);
169 }
170 const kept = entries.slice(-MAX_SNAPSHOT_LOG_ENTRIES);
171 if (kept.some((entry) => hasLongString(entry, MAX_SNAPSHOT_LOG_TEXT_CHARS))) {
172 omitted.push(`${label} text after ${MAX_SNAPSHOT_LOG_TEXT_CHARS} characters`);
173 }
174 return kept.map(cutEntryStrings);
175 };
176 if (
177 metadata.url.length > MAX_SNAPSHOT_IDENTIFIER_CHARS ||
178 metadata.title.length > MAX_SNAPSHOT_IDENTIFIER_CHARS
179 ) {
180 omitted.push(`url or title after ${MAX_SNAPSHOT_IDENTIFIER_CHARS} characters`);
181 }
182 if (
183 metadata.interactiveElements.some(
184 (element) => element.name.length > MAX_SNAPSHOT_ELEMENT_NAME_CHARS,
185 )
186 ) {
187 omitted.push(`element names longer than ${MAX_SNAPSHOT_ELEMENT_NAME_CHARS} characters`);
188 }
189 if (metadata.visibleText.length > MAX_SNAPSHOT_VISIBLE_TEXT_CHARS) {
190 omitted.push(
191 `visibleText after ${MAX_SNAPSHOT_VISIBLE_TEXT_CHARS} characters (use preview_evaluate for more)`,
192 );
193 }
194 const bounded = {
195 ...withoutTree,
196 url: cutText(metadata.url, MAX_SNAPSHOT_IDENTIFIER_CHARS),
197 title: cutText(metadata.title, MAX_SNAPSHOT_IDENTIFIER_CHARS),
198 visibleText: cutText(metadata.visibleText, MAX_SNAPSHOT_VISIBLE_TEXT_CHARS),
199 interactiveElements: metadata.interactiveElements.map((element) => ({
200 ...element,
201 name: cutText(element.name, MAX_SNAPSHOT_ELEMENT_NAME_CHARS),
202 })),
203 consoleEntries: tail(metadata.consoleEntries, "console entries"),
204 networkEntries: tail(metadata.networkEntries, "network entries"),
205 actionTimeline: tail(metadata.actionTimeline, "action timeline entries"),
206 };
207
208 // Per-field caps do not sum below the ceiling: three log arrays of 40 capped
209 // entries alone can pass 60 KB. Shed the least useful lists first, halving
210 // one list per round, until the JSON fits. With every list empty the rest
211 // is bounded by the identifier and visibleText caps, so this terminates.
212 const shedOrder = [
213 "actionTimeline",
214 "networkEntries",
215 "consoleEntries",
216 "interactiveElements",
217 ] as const;
218 const lists: Record<(typeof shedOrder)[number], ReadonlyArray<unknown>> = {
219 interactiveElements: bounded.interactiveElements,
220 consoleEntries: bounded.consoleEntries,
221 networkEntries: bounded.networkEntries,
222 actionTimeline: bounded.actionTimeline,
223 };
224 const dropped: Record<(typeof shedOrder)[number], number> = {
225 interactiveElements: 0,
226 consoleEntries: 0,
227 networkEntries: 0,
228 actionTimeline: 0,
229 };
230 let text = encodeJsonText({ ...bounded, ...lists });
231 while (utf8Length(text) > MAX_SNAPSHOT_TEXT_BYTES) {
232 // Elements carry the locators, so they go last; logs shed newest-last.
233 const key =
234 shedOrder.find(
235 (candidate) => candidate !== "interactiveElements" && lists[candidate].length > 0,
236 ) ?? (lists.interactiveElements.length > 0 ? "interactiveElements" : undefined);
237 if (key === undefined) break;
238 const keep = Math.floor(lists[key].length / 2);
239 dropped[key] += lists[key].length - keep;
240 // slice(-0) keeps everything, so spell out the empty case.
241 lists[key] =
242 keep === 0
243 ? []
244 : key === "interactiveElements"
245 ? lists[key].slice(0, keep)
246 : lists[key].slice(-keep);
247 text = encodeJsonText({ ...bounded, ...lists });
248 }
249 for (const key of shedOrder) {
250 if (dropped[key] > 0) {
251 omitted.push(`${dropped[key]} of ${bounded[key].length} ${key}`);
252 }
253 }
254 return { text, omitted };
255};
256
257export class PreviewScreenshotSaveError extends Schema.TaggedError<PreviewScreenshotSaveError>()(
258 "PreviewScreenshotSaveError",
259 { screenshotPath: Schema.String, cause: Schema.Defect() },
260) {
261 override get message(): string {
262 return `Could not save preview screenshot to ${this.screenshotPath}.`;
263 }
264}
265
266const MAX_SCREENSHOT_SITE_SLUG_LENGTH = 40;
267
268/** Hostname reduced to a filename-safe slug, matching the desktop's own screenshot names. */
269const screenshotSiteSlug = (rawUrl: string): string => {
270 try {
271 const slug = new URL(rawUrl).hostname
272 .toLowerCase()
273 .replace(/[^a-z0-9]+/g, "-")
274 .replace(/^-+|-+$/g, "")
275 .slice(0, MAX_SCREENSHOT_SITE_SLUG_LENGTH)
276 .replace(/-+$/g, "");
277 return slug || "site";
278 } catch {
279 return "site";
280 }
281};
282
283/** Writes the snapshot PNG under the browser artifacts directory and returns its path. */
284const saveScreenshot = Effect.fn("McpHttpServer.saveScreenshot")(function* (
285 pageUrl: string,
286 data: Uint8Array,
287) {
288 const config = yield* ServerConfig.ServerConfig;
289 const fileSystem = yield* FileSystem.FileSystem;
290 const path = yield* Path.Path;
291 const millis = yield* Clock.currentTimeMillis;
292 // Two saves in the same millisecond must not overwrite each other.
293 const fileName = `browser-screenshot-${screenshotSiteSlug(pageUrl)}-${millis.toString(36)}-${NodeCrypto.randomUUID().slice(0, 8)}.png`;
294 const screenshotPath = path.join(config.browserArtifactsDir, fileName);
295 yield* fileSystem.makeDirectory(config.browserArtifactsDir, { recursive: true }).pipe(
296 Effect.andThen(fileSystem.writeFile(screenshotPath, data)),
297 Effect.mapError((cause) => new PreviewScreenshotSaveError({ screenshotPath, cause })),
298 );
299 return screenshotPath;
300});
301
98302const previewSnapshotFailure = <E>(cause: Cause.Cause<E>) => {
99303 if (Cause.hasInterrupts(cause) || cause.reasons.some(Cause.isDieReason)) {
100304 return Effect.failCause(cause).pipe(Effect.orDie);
......@@ -117,7 +321,8 @@ const previewSnapshotFailure = <E>(cause: Cause.Cause<E>) => {
117321 failureCount: failures.length,
118322 },
119323 },
120 content: [{ type: "text", text: "Preview snapshot failed." }],
324 // Agents usually see only the text content, so name the tag there too.
325 content: [{ type: "text", text: `Preview snapshot failed: ${errorTag}.` }],
121326 });
122327 return Effect.logWarning("preview snapshot failed", {
123328 operation: "snapshot",
......@@ -129,6 +334,10 @@ const previewSnapshotFailure = <E>(cause: Cause.Cause<E>) => {
129334const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot")(function* () {
130335 const server = yield* McpServer.McpServer;
131336 const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker;
337 // The MCP tool runner only supplies the client, so hand the save path its services here.
338 const saveServices = yield* Effect.context<
339 ServerConfig.ServerConfig | FileSystem.FileSystem | Path.Path
340 >();
132341 const built = yield* PreviewSnapshotToolkit;
133342 const tool = PreviewSnapshotTool;
134343 yield* server.addTool({
......@@ -160,19 +369,21 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
160369 Effect.flatMap(Effect.fromOption),
161370 Effect.provideService(PreviewAutomationBroker.PreviewAutomationBroker, broker),
162371 Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
163 Effect.matchCauseEffect({
164 onFailure: previewSnapshotFailure,
165 onSuccess: ({ encodedResult }) => {
166 const snapshot = encodedResult as {
372 Effect.flatMap(({ encodedResult }) =>
373 Effect.gen(function* () {
374 const snapshot = encodedResult as SnapshotMetadata & {
375 readonly url: string;
167376 readonly screenshot: {
168377 readonly mimeType: "image/png";
169378 readonly data: string;
170379 readonly width: number;
171380 readonly height: number;
172381 };
173 readonly [key: string]: unknown;
174382 };
175383 const { screenshot, ...page } = snapshot;
384 const png = new Uint8Array(Buffer.from(screenshot.data, "base64"));
385 const screenshotPath =
386 payload?.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined;
176387 const metadata = {
177388 ...page,
178389 screenshot: {
......@@ -180,26 +391,33 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
180391 width: screenshot.width,
181392 height: screenshot.height,
182393 },
394 ...(screenshotPath === undefined ? {} : { screenshotPath }),
183395 };
184 return Effect.succeed(
185 new McpSchema.CallToolResult({
186 isError: false,
187 structuredContent: metadata,
188 content: [
189 { type: "text", text: JSON.stringify(metadata) },
190 ...(payload?.includeImage === false
191 ? []
192 : [
193 {
194 type: "image" as const,
195 data: new Uint8Array(Buffer.from(screenshot.data, "base64")),
196 mimeType: screenshot.mimeType,
197 },
198 ]),
199 ],
200 }),
201 );
202 },
396 const bounded = boundSnapshotMetadata(metadata);
397 return new McpSchema.CallToolResult({
398 isError: false,
399 structuredContent: metadata,
400 content: [
401 { type: "text", text: bounded.text },
402 ...(bounded.omitted.length === 0
403 ? []
404 : [
405 {
406 type: "text" as const,
407 text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`,
408 },
409 ]),
410 ...(payload?.includeImage === false
411 ? []
412 : [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }]),
413 ],
414 });
415 }),
416 ),
417 Effect.provide(saveServices),
418 Effect.matchCauseEffect({
419 onFailure: previewSnapshotFailure,
420 onSuccess: Effect.succeed,
203421 }),
204422 );
205423 }),
apps/server/src/mcp/toolkits/preview/handlers.ts+5-3
......@@ -166,8 +166,8 @@ const handlers = {
166166 preview_set_appearance: (input) =>
167167 invokeTargeted<PreviewAutomationSetColorSchemeResult>("setColorScheme", input),
168168 preview_snapshot: (input) => {
169 // Output selection is MCP-only; the browser still produces a complete snapshot.
170 const { includeImage: _includeImage, ...operationInput } = input ?? {};
169 // Output selection and saving are MCP-only; the browser still produces a complete snapshot.
170 const { includeImage: _includeImage, save: _save, ...operationInput } = input ?? {};
171171 return invokeTargeted<PreviewAutomationSnapshot>("snapshot", operationInput);
172172 },
173173 preview_click: (input) =>
......@@ -176,7 +176,9 @@ const handlers = {
176176 preview_press: (input) => invokeTargeted<void>("press", input).pipe(Effect.as({})),
177177 preview_scroll: (input) => invokeTargeted<void>("scroll", input).pipe(Effect.as({})),
178178 preview_evaluate: (input) =>
179 invokeTargeted<unknown>("evaluate", input).pipe(Effect.map((result) => result ?? null)),
179 invokeTargeted<unknown>("evaluate", input).pipe(
180 Effect.map((result) => ({ value: result ?? null })),
181 ),
180182 preview_wait_for: (input) =>
181183 invokeTargeted<void>("waitFor", input, input.timeoutMs).pipe(Effect.as({})),
182184 preview_recording_start: (input) =>
apps/server/src/mcp/toolkits/preview/tools.ts+20-3
......@@ -113,7 +113,7 @@ const PreviewSetAppearanceTool = safeBrowserTool(
113113export const PreviewSnapshotTool = readonlyBrowserTool(
114114 Tool.make("preview_snapshot", {
115115 description:
116 "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. Set includeImage=false for text-only output with the same page metadata.",
116 "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. Set includeImage=false for text-only output with the same page metadata. Set save=true to also write the PNG to disk and get screenshotPath back; embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.",
117117 parameters: Schema.Struct({
118118 ...PreviewAutomationTabTargetInput.fields,
119119 includeImage: Schema.optional(
......@@ -122,6 +122,12 @@ export const PreviewSnapshotTool = readonlyBrowserTool(
122122 "Include the PNG image in the tool response. Defaults to true. Set false for text-only output.",
123123 }),
124124 ),
125 save: Schema.optional(
126 Schema.Boolean.annotate({
127 description:
128 "Write the screenshot PNG to disk and return its absolute path as screenshotPath. Defaults to false.",
129 }),
130 ),
125131 }),
126132 success: PreviewAutomationSnapshot,
127133 failure: PreviewAutomationError,
......@@ -173,12 +179,23 @@ const PreviewScrollTool = safeBrowserTool(
173179 }).annotate(Tool.Title, "Scroll preview page"),
174180);
175181
182/**
183 * MCP `structuredContent` must be a JSON object, and Claude Code rejects the
184 * whole result when it is not. Wrapping keeps arrays, strings, numbers, and
185 * null valid instead of failing only for non-object expressions.
186 */
187export const PreviewEvaluateResult = Schema.Struct({
188 value: Schema.Unknown.annotate({
189 description: "The JSON-serializable value the expression produced, or null.",
190 }),
191}).annotate({ description: "The evaluated expression result." });
192
176193const PreviewEvaluateTool = browserTool(
177194 Tool.make("preview_evaluate", {
178195 description:
179 "Evaluate JavaScript in the tab selected by tabId, or this agent session's current tab when omitted. Returns a serializable result up to 64 KB; the expression may mutate page state.",
196 "Evaluate JavaScript in the tab selected by tabId, or this agent session's current tab when omitted. Returns {value} with a serializable result up to 64 KB; the expression may mutate page state.",
180197 parameters: PreviewAutomationEvaluateInput,
181 success: Schema.Unknown,
198 success: PreviewEvaluateResult,
182199 failure: PreviewAutomationError,
183200 dependencies,
184201 }).annotate(Tool.Title, "Evaluate JavaScript in preview"),