authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-09-24 21:52:01-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-24 21:52:01-07:00
log4f27a8463105662ddfc834e01bca375d5124be79
tree645c1a1561b0d095f277bd39d1efbb9238acb633
parent9957349c0a066a9efc657600fcb901c9b9cca34b
signature Signed by PGP key B5690EEEBB952194

fix(mcp): preview snapshots fit in the agent's tool output again (#13558)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

3 files changed, 125 insertions(+), 40 deletions(-)

apps/server/src/mcp/McpHttpServer.test.ts+58-5
......@@ -248,7 +248,10 @@ it.effect.each([
248248 const metadata = { ...page, title: `Snapshot ${call}`, screenshot };
249249 const { accessibilityTree: _tree, ...boundedMetadata } = metadata;
250250 expect(snapshot.isError).toBe(false);
251 expect(snapshot.structuredContent).toEqual(metadata);
251 expect(snapshot.structuredContent).toEqual({
252 ...boundedMetadata,
253 omitted: ["accessibilityTree (use interactiveElements locators or preview_evaluate)"],
254 });
252255 const [identity, text, ...rest] = snapshot.content;
253256 expect(identity?.type === "text" ? decodeJsonText(identity.text) : null).toEqual({
254257 url: page.url,
......@@ -287,7 +290,8 @@ it.effect.each([
287290 "text",
288291 "image",
289292 ]);
290 expect(nextDefault.structuredContent).toEqual({ ...page, title: "Snapshot 7", screenshot });
293 expect(nextDefault.structuredContent).toMatchObject({ title: "Snapshot 7", screenshot });
294 expect(nextDefault.structuredContent).not.toHaveProperty("accessibilityTree");
291295 expect(requests).toBe(7);
292296 }),
293297 ).pipe(Effect.provide(TestLayer)),
......@@ -341,6 +345,15 @@ it.effect("saves the snapshot PNG on request and reports its path", () =>
341345
342346 const unsaved = yield* callSnapshot({});
343347 expect(unsaved.structuredContent).not.toHaveProperty("screenshotPath");
348
349 // A save without the image skips the page dump.
350 const pathOnly = yield* callSnapshot({ save: true, includeImage: false });
351 const saved = pathOnly.structuredContent as { readonly screenshotPath: string };
352 expect(saved).toEqual({ url: snapshotResult.url, screenshotPath: expect.any(String) });
353 expect(Buffer.from(yield* fileSystem.readFile(saved.screenshotPath)).toString()).toBe("png");
354 const [only, ...others] = pathOnly.content;
355 expect(others).toEqual([]);
356 expect(only?.type === "text" ? decodeJsonText(only.text) : null).toEqual(saved);
344357 }),
345358 ).pipe(Effect.provide(TestLayer)),
346359);
......@@ -460,9 +473,10 @@ it.effect("keeps the snapshot text under the agent's output ceiling", () =>
460473 expect(parsed.consoleEntries[0]?.text).toBe("entry 60");
461474 expect(notice?.type === "text" ? notice.text : "").toContain("accessibilityTree");
462475 expect(notice?.type === "text" ? notice.text : "").toContain("60 older console entries");
463 // The structured result is untouched; only the text the agent reads is bounded.
464 expect(snapshot.structuredContent).toMatchObject({
465 accessibilityTree: oversized.accessibilityTree,
476 // Claude Code shows the model structuredContent instead of the text, so it is bounded too.
477 expect(snapshot.structuredContent).toEqual({
478 ...parsed,
479 omitted: expect.arrayContaining(["60 older console entries"]),
466480 });
467481 }),
468482 ).pipe(Effect.provide(TestLayer)),
......@@ -500,6 +514,45 @@ it.effect("bounds the snapshot text even when nothing but logs and the title are
500514 ).pipe(Effect.provide(TestLayer)),
501515);
502516
517it.effect("bounds page text made of wide characters before dropping locators", () =>
518 Effect.scoped(
519 Effect.gen(function* () {
520 // The character caps alone leave 8,000 three-byte characters, about 24 KB.
521 yield* serveSnapshots("mcp-wide-text-client", {
522 ...snapshotResult,
523 visibleText: "界".repeat(9_000),
524 interactiveElements: Array.from({ length: 20 }, (_, i) => ({
525 tag: "button",
526 role: "button",
527 name: `Button ${i}`,
528 selector: `#button-${i}`,
529 x: 0,
530 y: 0,
531 width: 10,
532 height: 10,
533 })),
534 });
535
536 const snapshot = yield* callSnapshot({ includeImage: false });
537
538 const [, text, notice] = snapshot.content;
539 const body = text?.type === "text" ? text.text : "";
540 expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual(
541 McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES,
542 );
543 const parsed = decodeJsonText(body) as {
544 readonly visibleText: string;
545 readonly interactiveElements: ReadonlyArray<unknown>;
546 };
547 expect(parsed.visibleText).toMatch(/^界+…$/);
548 expect(parsed.interactiveElements).toHaveLength(20);
549 expect(notice?.type === "text" ? notice.text : "").toContain(
550 "visibleText after 4000 characters",
551 );
552 }),
553 ).pipe(Effect.provide(TestLayer)),
554);
555
503556it.effect("sheds log entries before locators when every list is full", () =>
504557 Effect.scoped(
505558 Effect.gen(function* () {
apps/server/src/mcp/McpHttpServer.ts+65-33
......@@ -114,12 +114,15 @@ const McpAuthMiddlewareLive = HttpRouter.middleware<{
114114}>()(makeMcpAuthMiddleware).layer;
115115
116116/**
117 * Claude Code drops every MCP result above 25k tokens (~100 KB of text) and
118 * hands the agent a truncation notice instead, so a snapshot that carries the
119 * full accessibility tree and 20 KB of page text loses its locators too. Keep
120 * the text under that ceiling and tell the agent what was cut.
117 * Claude Code moves an MCP result above its output limit to a file and hands
118 * the agent a notice instead, so a snapshot that carries the full
119 * accessibility tree and page text loses its locators too. Claude Code also
120 * shows the model `structuredContent` in place of the text blocks when a
121 * result has both, so both carry the same bounded snapshot. Keep it near
122 * 20 KB and tell the agent what was cut. The short `omitted` notes may go a
123 * little over; the provider limit is far above this.
121124 */
122export const MAX_SNAPSHOT_TEXT_BYTES = 60_000;
125export const MAX_SNAPSHOT_TEXT_BYTES = 20_000;
123126const MAX_SNAPSHOT_VISIBLE_TEXT_CHARS = 8_000;
124127const MAX_SNAPSHOT_ELEMENT_NAME_CHARS = 200;
125128const MAX_SNAPSHOT_LOG_ENTRIES = 40;
......@@ -164,12 +167,11 @@ type SnapshotMetadata = {
164167/**
165168 * Drops the accessibility tree, shortens page text, element names, identifiers,
166169 * and log strings, keeps only the newest log entries, and finally sheds
167 * interactive elements until the JSON fits. Returns the text plus notes on
168 * what is missing so the agent can reach for preview_evaluate.
170 * interactive elements until the JSON fits. Returns the bounded value, its
171 * text, and notes on what is missing so the agent can reach for
172 * preview_evaluate.
169173 */
170const boundSnapshotMetadata = (
171 metadata: SnapshotMetadata,
172): { readonly text: string; readonly omitted: ReadonlyArray<string> } => {
174const boundSnapshotMetadata = (metadata: SnapshotMetadata) => {
173175 const omitted: Array<string> = [];
174176 const { accessibilityTree, ...withoutTree } = metadata;
175177 if (accessibilityTree !== undefined) {
......@@ -198,16 +200,10 @@ const boundSnapshotMetadata = (
198200 ) {
199201 omitted.push(`element names longer than ${MAX_SNAPSHOT_ELEMENT_NAME_CHARS} characters`);
200202 }
201 if (metadata.visibleText.length > MAX_SNAPSHOT_VISIBLE_TEXT_CHARS) {
202 omitted.push(
203 `visibleText after ${MAX_SNAPSHOT_VISIBLE_TEXT_CHARS} characters (use preview_evaluate for more)`,
204 );
205 }
206203 const bounded = {
207204 ...withoutTree,
208205 url: cutText(metadata.url, MAX_SNAPSHOT_IDENTIFIER_CHARS),
209206 title: cutText(metadata.title, MAX_SNAPSHOT_IDENTIFIER_CHARS),
210 visibleText: cutText(metadata.visibleText, MAX_SNAPSHOT_VISIBLE_TEXT_CHARS),
211207 interactiveElements: metadata.interactiveElements.map((element) => ({
212208 ...element,
213209 name: cutText(element.name, MAX_SNAPSHOT_ELEMENT_NAME_CHARS),
......@@ -218,9 +214,10 @@ const boundSnapshotMetadata = (
218214 };
219215
220216 // Per-field caps do not sum below the ceiling: three log arrays of 40 capped
221 // entries alone can pass 60 KB. Shed the least useful lists first, halving
222 // one list per round, until the JSON fits. With every list empty the rest
223 // is bounded by the identifier and visibleText caps, so this terminates.
217 // entries alone can pass 60 KB, and the caps count characters, not bytes.
218 // Halve one thing per round until the JSON fits: logs first, then page
219 // text, then the locators. The identifier caps bound the rest, so this
220 // terminates.
224221 const shedOrder = [
225222 "actionTimeline",
226223 "networkEntries",
......@@ -239,31 +236,51 @@ const boundSnapshotMetadata = (
239236 networkEntries: 0,
240237 actionTimeline: 0,
241238 };
242 let text = encodeJsonText({ ...bounded, ...lists });
239 let visibleTextChars = Math.min(metadata.visibleText.length, MAX_SNAPSHOT_VISIBLE_TEXT_CHARS);
240 const value = () => ({
241 ...bounded,
242 visibleText: cutText(metadata.visibleText, visibleTextChars),
243 ...lists,
244 });
245 let text = encodeJsonText(value());
243246 while (utf8Length(text) > MAX_SNAPSHOT_TEXT_BYTES) {
244247 // Elements carry the locators, so they go last; logs shed newest-last.
245248 const key =
246249 shedOrder.find(
247250 (candidate) => candidate !== "interactiveElements" && lists[candidate].length > 0,
248 ) ?? (lists.interactiveElements.length > 0 ? "interactiveElements" : undefined);
251 ) ??
252 (visibleTextChars > 0
253 ? "visibleText"
254 : lists.interactiveElements.length > 0
255 ? "interactiveElements"
256 : undefined);
249257 if (key === undefined) break;
250 const keep = Math.floor(lists[key].length / 2);
251 dropped[key] += lists[key].length - keep;
252 // slice(-0) keeps everything, so spell out the empty case.
253 lists[key] =
254 keep === 0
255 ? []
256 : key === "interactiveElements"
257 ? lists[key].slice(0, keep)
258 : lists[key].slice(-keep);
259 text = encodeJsonText({ ...bounded, ...lists });
258 if (key === "visibleText") {
259 visibleTextChars = Math.floor(visibleTextChars / 2);
260 } else {
261 const keep = Math.floor(lists[key].length / 2);
262 dropped[key] += lists[key].length - keep;
263 // slice(-0) keeps everything, so spell out the empty case.
264 lists[key] =
265 keep === 0
266 ? []
267 : key === "interactiveElements"
268 ? lists[key].slice(0, keep)
269 : lists[key].slice(-keep);
270 }
271 text = encodeJsonText(value());
272 }
273 if (visibleTextChars < metadata.visibleText.length) {
274 omitted.push(
275 `visibleText after ${visibleTextChars} characters (use preview_evaluate for more)`,
276 );
260277 }
261278 for (const key of shedOrder) {
262279 if (dropped[key] > 0) {
263280 omitted.push(`${dropped[key]} of ${bounded[key].length} ${key}`);
264281 }
265282 }
266 return { text, omitted };
283 return { value: value(), text, omitted };
267284};
268285
269286export class PreviewScreenshotSaveError extends Schema.TaggedError<PreviewScreenshotSaveError>()(
......@@ -396,6 +413,18 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
396413 const png = new Uint8Array(Buffer.from(screenshot.data, "base64"));
397414 const screenshotPath =
398415 payload?.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined;
416 if (screenshotPath !== undefined && payload?.includeImage === false) {
417 // The agent only wants a file to show the user. The url keeps the site icon on the tool row.
418 const saved = {
419 url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS),
420 screenshotPath,
421 };
422 return new McpSchema.CallToolResult({
423 isError: false,
424 structuredContent: saved,
425 content: [{ type: "text", text: encodeJsonText(saved) }],
426 });
427 }
399428 const metadata = {
400429 ...page,
401430 screenshot: {
......@@ -408,7 +437,10 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
408437 const bounded = boundSnapshotMetadata(metadata);
409438 return new McpSchema.CallToolResult({
410439 isError: false,
411 structuredContent: metadata,
440 structuredContent:
441 bounded.omitted.length === 0
442 ? bounded.value
443 : { ...bounded.value, omitted: bounded.omitted },
412444 content: [
413445 // Keep the page identity readable even if a provider truncates the snapshot.
414446 {
apps/server/src/mcp/toolkits/preview/tools.ts+2-2
......@@ -119,7 +119,7 @@ const PreviewSetAppearanceTool = safeBrowserTool(
119119export const PreviewSnapshotTool = readonlyBrowserTool(
120120 Tool.make("preview_snapshot", {
121121 description:
122 "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. Set includeImage=false for text-only output with the same page metadata. Set save=true to also write the PNG to disk and get screenshotPath back; embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.",
122 "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. The text is capped near 20 KB and lists what it omitted; use preview_evaluate to read more. Set includeImage=false for text-only output with the same page metadata. Set save=true to also write the PNG to disk and get screenshotPath back; with includeImage=false, save=true returns only the url and screenshotPath. Embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.",
123123 parameters: Schema.Struct({
124124 ...PreviewAutomationTabTargetInput.fields,
125125 includeImage: Schema.optional(
......@@ -131,7 +131,7 @@ export const PreviewSnapshotTool = readonlyBrowserTool(
131131 save: Schema.optional(
132132 Schema.Boolean.annotate({
133133 description:
134 "Write the screenshot PNG to disk and return its absolute path as screenshotPath. Defaults to false.",
134 "Write the screenshot PNG to disk and return its absolute path as screenshotPath. With includeImage=false, return only the url and screenshotPath. Defaults to false.",
135135 }),
136136 ),
137137 }),