| author | |
| committer | |
| log | dbd83435347d13d5f456271f4f1f5efe294a111d |
| tree | f71af277987e8fda5fc00bcc2127f691b7aa0d4e |
| parent | c724c7cb02aaeb0a23ff6e5e224d17c0fdb9a36b |
| signature | Signed by PGP key B5690EEEBB952194 |
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>24 files changed, 743 insertions(+), 122 deletions(-)
apps/server/src/auth/RpcAuthorization.test.ts+1| ... | @@ -152,6 +152,7 @@ describe("RPC authorization scopes", () => { | ... | @@ -152,6 +152,7 @@ describe("RPC authorization scopes", () => { |
| 152 | WS_METHODS.previewReportStatus, | 152 | WS_METHODS.previewReportStatus, |
| 153 | WS_METHODS.previewAdjust, | 153 | WS_METHODS.previewAdjust, |
| 154 | WS_METHODS.previewClearProfile, | 154 | WS_METHODS.previewClearProfile, |
| 155 | WS_METHODS.previewReportProfiles, | ||
| 155 | ]) { | 156 | ]) { |
| 156 | expect(requiredScopeForRpcMethod(method)).toBe(AuthPreviewOperateScope); | 157 | expect(requiredScopeForRpcMethod(method)).toBe(AuthPreviewOperateScope); |
| 157 | } | 158 | } |
apps/server/src/auth/RpcAuthorization.ts+1| ... | @@ -189,6 +189,7 @@ export const RPC_REQUIRED_SCOPES = { | ... | @@ -189,6 +189,7 @@ export const RPC_REQUIRED_SCOPES = { |
| 189 | [WS_METHODS.previewClose]: AuthPreviewOperateScope, | 189 | [WS_METHODS.previewClose]: AuthPreviewOperateScope, |
| 190 | [WS_METHODS.previewList]: AuthOrchestrationReadScope, | 190 | [WS_METHODS.previewList]: AuthOrchestrationReadScope, |
| 191 | [WS_METHODS.previewClearProfile]: AuthPreviewOperateScope, | 191 | [WS_METHODS.previewClearProfile]: AuthPreviewOperateScope, |
| 192 | [WS_METHODS.previewReportProfiles]: AuthPreviewOperateScope, | ||
| 192 | [WS_METHODS.previewReportStatus]: AuthPreviewOperateScope, | 193 | [WS_METHODS.previewReportStatus]: AuthPreviewOperateScope, |
| 193 | [WS_METHODS.subscribePreviewEvents]: AuthOrchestrationReadScope, | 194 | [WS_METHODS.subscribePreviewEvents]: AuthOrchestrationReadScope, |
| 194 | [WS_METHODS.subscribeDiscoveredLocalServers]: AuthOrchestrationReadScope, | 195 | [WS_METHODS.subscribeDiscoveredLocalServers]: AuthOrchestrationReadScope, |
apps/server/src/mcp/McpDeviceToolkit.test.ts+22-1| ... | @@ -2,6 +2,7 @@ import { expect, it } from "@effect/vitest"; | ... | @@ -2,6 +2,7 @@ import { expect, it } from "@effect/vitest"; |
| 2 | import * as NodeServices from "@effect/platform-node/NodeServices"; | 2 | import * as NodeServices from "@effect/platform-node/NodeServices"; |
| 3 | import { | 3 | import { |
| 4 | DeviceHostUnavailableError, | 4 | DeviceHostUnavailableError, |
| 5 | DeviceId, | ||
| 5 | EnvironmentId, | 6 | EnvironmentId, |
| 6 | ProviderInstanceId, | 7 | ProviderInstanceId, |
| 7 | ThreadId, | 8 | ThreadId, |
| ... | @@ -93,7 +94,21 @@ const layerDeviceServiceMock = Layer.mock(DeviceService.DeviceService)({ | ... | @@ -93,7 +94,21 @@ const layerDeviceServiceMock = Layer.mock(DeviceService.DeviceService)({ |
| 93 | platform: input.platform, | 94 | platform: input.platform, |
| 94 | openedAt: "2026-09-08T00:00:00.000Z", | 95 | openedAt: "2026-09-08T00:00:00.000Z", |
| 95 | }), | 96 | }), |
| 96 | sessionsForThread: () => Effect.succeed([]), | 97 | // UDID-1 is open in the test thread; UDID-2 exists but belongs to another thread. |
| 98 | sessionsForThread: (id) => | ||
| 99 | Effect.succeed( | ||
| 100 | id === threadId | ||
| 101 | ? [ | ||
| 102 | { | ||
| 103 | threadId, | ||
| 104 | hostId: "local", | ||
| 105 | deviceId: DeviceId.make("UDID-1"), | ||
| 106 | platform: "ios" as const, | ||
| 107 | openedAt: "2026-09-08T00:00:00.000Z", | ||
| 108 | }, | ||
| 109 | ] | ||
| 110 | : [], | ||
| 111 | ), | ||
| 97 | screenshot: () => Effect.succeed({ device, png }), | 112 | screenshot: () => Effect.succeed({ device, png }), |
| 98 | close: () => Effect.void, | 113 | close: () => Effect.void, |
| 99 | agentCli: Effect.succeed("/cli"), | 114 | agentCli: Effect.succeed("/cli"), |
| ... | @@ -135,6 +150,12 @@ it.effect("registers the device tools and returns the screenshot as image conten | ... | @@ -135,6 +150,12 @@ it.effect("registers the device tools and returns the screenshot as image conten |
| 135 | screenshot: { mimeType: "image/png", width: 1206, height: 2622 }, | 150 | screenshot: { mimeType: "image/png", width: 1206, height: 2622 }, |
| 136 | }); | 151 | }); |
| 137 | 152 | ||
| 153 | const foreign = yield* server | ||
| 154 | .callTool({ name: "device_screenshot", arguments: { deviceId: "UDID-2" } }) | ||
| 155 | .pipe(callWith(["device"]), Effect.provideService(McpSchema.McpServerClient, client)); | ||
| 156 | expect(foreign.isError).toBe(true); | ||
| 157 | expect(foreign.content.map((entry) => entry.type)).toEqual(["text"]); | ||
| 158 | |||
| 138 | const denied = yield* server | 159 | const denied = yield* server |
| 139 | .callTool({ name: "device_list", arguments: {} }) | 160 | .callTool({ name: "device_list", arguments: {} }) |
| 140 | .pipe(callWith(["preview"]), Effect.provideService(McpSchema.McpServerClient, client)); | 161 | .pipe(callWith(["preview"]), Effect.provideService(McpSchema.McpServerClient, client)); |
apps/server/src/mcp/McpHttpServer.test.ts+11-12| ... | @@ -244,7 +244,7 @@ it.effect("tells the agent how to fall back when no desktop app can run the snap | ... | @@ -244,7 +244,7 @@ it.effect("tells the agent how to fall back when no desktop app can run the snap |
| 244 | ); | 244 | ); |
| 245 | 245 | ||
| 246 | it.effect.each([ | 246 | it.effect.each([ |
| 247 | { mode: "default", input: {}, images: true }, | 247 | { mode: "default", input: {}, images: false }, |
| 248 | { mode: "explicit image", input: { includeImage: true }, images: true }, | 248 | { mode: "explicit image", input: { includeImage: true }, images: true }, |
| 249 | { mode: "text only", input: { includeImage: false }, images: false }, | 249 | { mode: "text only", input: { includeImage: false }, images: false }, |
| 250 | ])("returns fresh $mode snapshots on repeated MCP calls", ({ input, images }) => | 250 | ])("returns fresh $mode snapshots on repeated MCP calls", ({ input, images }) => |
| ... | @@ -351,12 +351,7 @@ it.effect.each([ | ... | @@ -351,12 +351,7 @@ it.effect.each([ |
| 351 | Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), | 351 | Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), |
| 352 | Effect.provideService(McpSchema.McpServerClient, client), | 352 | Effect.provideService(McpSchema.McpServerClient, client), |
| 353 | ); | 353 | ); |
| 354 | expect(nextDefault.content.map((content) => content.type)).toEqual([ | 354 | expect(nextDefault.content.map((content) => content.type)).toEqual(["text", "text", "text"]); |
| 355 | "text", | ||
| 356 | "text", | ||
| 357 | "text", | ||
| 358 | "image", | ||
| 359 | ]); | ||
| 360 | expect(nextDefault.structuredContent).toMatchObject({ title: "Snapshot 7", screenshot }); | 355 | expect(nextDefault.structuredContent).toMatchObject({ title: "Snapshot 7", screenshot }); |
| 361 | expect(nextDefault.structuredContent).not.toHaveProperty("accessibilityTree"); | 356 | expect(nextDefault.structuredContent).not.toHaveProperty("accessibilityTree"); |
| 362 | expect(requests).toBe(7); | 357 | expect(requests).toBe(7); |
| ... | @@ -416,11 +411,12 @@ it.effect("saves the snapshot PNG on request and reports its path", () => | ... | @@ -416,11 +411,12 @@ it.effect("saves the snapshot PNG on request and reports its path", () => |
| 416 | const path = yield* Path.Path; | 411 | const path = yield* Path.Path; |
| 417 | const inputs = yield* serveSnapshots("mcp-save-client", snapshotResult); | 412 | const inputs = yield* serveSnapshots("mcp-save-client", snapshotResult); |
| 418 | 413 | ||
| 419 | const snapshot = yield* callSnapshot({ save: true }); | 414 | const snapshot = yield* callSnapshot({ save: true, includeImage: true }); |
| 420 | 415 | ||
| 421 | expect(snapshot.isError).toBe(false); | 416 | expect(snapshot.isError).toBe(false); |
| 422 | // The browser never receives the server-only `save` flag. | 417 | // The browser never receives the server-only `save` flag. |
| 423 | expect(inputs).toEqual([{}]); | 418 | expect(inputs).toEqual([{}]); |
| 419 | expect(snapshot.content.map((content) => content.type)).toContain("image"); | ||
| 424 | const structured = snapshot.structuredContent as { readonly screenshotPath?: string }; | 420 | const structured = snapshot.structuredContent as { readonly screenshotPath?: string }; |
| 425 | const screenshotPath = structured.screenshotPath; | 421 | const screenshotPath = structured.screenshotPath; |
| 426 | expect(typeof screenshotPath).toBe("string"); | 422 | expect(typeof screenshotPath).toBe("string"); |
| ... | @@ -436,7 +432,7 @@ it.effect("saves the snapshot PNG on request and reports its path", () => | ... | @@ -436,7 +432,7 @@ it.effect("saves the snapshot PNG on request and reports its path", () => |
| 436 | expect(unsaved.structuredContent).not.toHaveProperty("screenshotPath"); | 432 | expect(unsaved.structuredContent).not.toHaveProperty("screenshotPath"); |
| 437 | 433 | ||
| 438 | // A save without the image skips the page dump. | 434 | // A save without the image skips the page dump. |
| 439 | const pathOnly = yield* callSnapshot({ save: true, includeImage: false }); | 435 | const pathOnly = yield* callSnapshot({ save: true }); |
| 440 | const saved = pathOnly.structuredContent as { readonly screenshotPath: string }; | 436 | const saved = pathOnly.structuredContent as { readonly screenshotPath: string }; |
| 441 | expect(saved).toEqual({ url: snapshotResult.url, screenshotPath: expect.any(String) }); | 437 | expect(saved).toEqual({ url: snapshotResult.url, screenshotPath: expect.any(String) }); |
| 442 | expect(Buffer.from(yield* fileSystem.readFile(saved.screenshotPath)).toString()).toBe("png"); | 438 | expect(Buffer.from(yield* fileSystem.readFile(saved.screenshotPath)).toString()).toBe("png"); |
| ... | @@ -851,8 +847,8 @@ it.effect("registers annotated tools and preserves authenticated request context | ... | @@ -851,8 +847,8 @@ it.effect("registers annotated tools and preserves authenticated request context |
| 851 | expect(statusTool?.tool.annotations?.destructiveHint).toBe(false); | 847 | expect(statusTool?.tool.annotations?.destructiveHint).toBe(false); |
| 852 | 848 | ||
| 853 | const snapshotTool = server.tools.find(({ tool }) => tool.name === "preview_snapshot"); | 849 | const snapshotTool = server.tools.find(({ tool }) => tool.name === "preview_snapshot"); |
| 854 | expect(snapshotTool?.tool.annotations?.readOnlyHint).toBe(true); | 850 | expect(snapshotTool?.tool.annotations?.readOnlyHint).toBe(false); |
| 855 | expect(snapshotTool?.tool.annotations?.idempotentHint).toBe(true); | 851 | expect(snapshotTool?.tool.annotations?.idempotentHint).toBe(false); |
| 856 | expect(snapshotTool?.tool.annotations?.openWorldHint).toBe(true); | 852 | expect(snapshotTool?.tool.annotations?.openWorldHint).toBe(true); |
| 857 | 853 | ||
| 858 | const clickTool = server.tools.find(({ tool }) => tool.name === "preview_click"); | 854 | const clickTool = server.tools.find(({ tool }) => tool.name === "preview_click"); |
| ... | @@ -891,7 +887,10 @@ it.effect("registers annotated tools and preserves authenticated request context | ... | @@ -891,7 +887,10 @@ it.effect("registers annotated tools and preserves authenticated request context |
| 891 | expect(malformed._tag).toBe("InvalidParams"); | 887 | expect(malformed._tag).toBe("InvalidParams"); |
| 892 | 888 | ||
| 893 | const snapshot = yield* server | 889 | const snapshot = yield* server |
| 894 | .callTool({ name: "preview_snapshot", arguments: { tabId: alternateTabId } }) | 890 | .callTool({ |
| 891 | name: "preview_snapshot", | ||
| 892 | arguments: { tabId: alternateTabId, includeImage: true }, | ||
| 893 | }) | ||
| 895 | .pipe( | 894 | .pipe( |
| 896 | Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), | 895 | Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), |
| 897 | Effect.provideService(McpSchema.McpServerClient, client), | 896 | Effect.provideService(McpSchema.McpServerClient, client), |
apps/server/src/mcp/McpHttpServer.ts+9-6| ... | @@ -503,7 +503,10 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot | ... | @@ -503,7 +503,10 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot |
| 503 | const png = new Uint8Array(Buffer.from(screenshot.data, "base64")); | 503 | const png = new Uint8Array(Buffer.from(screenshot.data, "base64")); |
| 504 | const screenshotPath = | 504 | const screenshotPath = |
| 505 | payload?.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined; | 505 | payload?.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined; |
| 506 | if (screenshotPath !== undefined && payload?.includeImage === false) { | 506 | // Images stay out of tool history unless asked for: providers replay them on every |
| 507 | // later request, and some reject inline images outright. | ||
| 508 | const includeImage = payload?.includeImage === true; | ||
| 509 | if (screenshotPath !== undefined && !includeImage) { | ||
| 507 | // The agent only wants a file to show the user. The url keeps the site icon on the tool row. | 510 | // The agent only wants a file to show the user. The url keeps the site icon on the tool row. |
| 508 | const saved = { | 511 | const saved = { |
| 509 | url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS), | 512 | url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS), |
| ... | @@ -548,9 +551,9 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot | ... | @@ -548,9 +551,9 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot |
| 548 | text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`, | 551 | text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`, |
| 549 | }, | 552 | }, |
| 550 | ]), | 553 | ]), |
| 551 | ...(payload?.includeImage === false | 554 | ...(includeImage |
| 552 | ? [] | 555 | ? [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }] |
| 553 | : [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }]), | 556 | : []), |
| 554 | ], | 557 | ], |
| 555 | }); | 558 | }); |
| 556 | }), | 559 | }), |
| ... | @@ -815,7 +818,7 @@ const layerEnvironmentRegistration = toolkitRegistration( | ... | @@ -815,7 +818,7 @@ const layerEnvironmentRegistration = toolkitRegistration( |
| 815 | 818 | ||
| 816 | const layerProjectRegistration = toolkitRegistration(ProjectToolkit, ProjectHandlers.layer); | 819 | const layerProjectRegistration = toolkitRegistration(ProjectToolkit, ProjectHandlers.layer); |
| 817 | 820 | ||
| 818 | const layerAttachmentRegistration = toolkitRegistration( | 821 | export const layerAttachmentToolkit = toolkitRegistration( |
| 819 | AttachmentToolkit, | 822 | AttachmentToolkit, |
| 820 | AttachmentHandlers.layer, | 823 | AttachmentHandlers.layer, |
| 821 | ); | 824 | ); |
| ... | @@ -851,7 +854,7 @@ export const layer = Layer.mergeAll( | ... | @@ -851,7 +854,7 @@ export const layer = Layer.mergeAll( |
| 851 | layerPreviewToolkit, | 854 | layerPreviewToolkit, |
| 852 | layerOrchestratorToolkit, | 855 | layerOrchestratorToolkit, |
| 853 | layerThreadToolkit, | 856 | layerThreadToolkit, |
| 854 | layerAttachmentRegistration, | 857 | layerAttachmentToolkit, |
| 855 | layerProjectRegistration, | 858 | layerProjectRegistration, |
| 856 | layerEnvironmentRegistration, | 859 | layerEnvironmentRegistration, |
| 857 | layerPreviewControlsRegistration, | 860 | layerPreviewControlsRegistration, |
apps/server/src/mcp/toolkits/attachment/handlers.ts+99-51| ... | @@ -1,5 +1,12 @@ | ... | @@ -1,5 +1,12 @@ |
| 1 | import { type ChatAttachment, MessageId, OrchestratorMcpFailure } from "@t3tools/contracts"; | 1 | import { |
| 2 | ATTACHMENT_UPLOAD_URL_TTL_MS, | ||
| 3 | type ChatAttachment, | ||
| 4 | MessageId, | ||
| 5 | OrchestratorMcpFailure, | ||
| 6 | } from "@t3tools/contracts"; | ||
| 7 | import * as Clock from "effect/Clock"; | ||
| 2 | import * as Effect from "effect/Effect"; | 8 | import * as Effect from "effect/Effect"; |
| 9 | import * as McpInvocationContext from "../../McpInvocationContext.ts"; | ||
| 3 | import * as Upload from "../../../assets/AttachmentUpload.ts"; | 10 | import * as Upload from "../../../assets/AttachmentUpload.ts"; |
| 4 | import * as Claims from "../../../orchestration-v2/AttachmentClaims.ts"; | 11 | import * as Claims from "../../../orchestration-v2/AttachmentClaims.ts"; |
| 5 | import * as ThreadMessageIntake from "../../../orchestration-v2/ThreadMessageIntake.ts"; | 12 | import * as ThreadMessageIntake from "../../../orchestration-v2/ThreadMessageIntake.ts"; |
| ... | @@ -27,53 +34,94 @@ export function resolveAttachmentReferences( | ... | @@ -27,53 +34,94 @@ export function resolveAttachmentReferences( |
| 27 | }); | 34 | }); |
| 28 | } | 35 | } |
| 29 | 36 | ||
| 30 | export const layer = McpToolAccess.toLayer(AttachmentToolkit, { | 37 | /** |
| 31 | t3_attachment_prepare_upload: McpToolAccess.writes((input) => | 38 | * As long as the pending file can live. The sweep counts its 24 hours from when |
| 32 | Upload.issueAttachmentUploadUrl(input.upload).pipe(Effect.mapError(unavailable)), | 39 | * the upload finishes, which can be up to the upload URL's lifetime after issue. |
| 33 | ), | 40 | */ |
| 34 | t3_attachment_discard: McpToolAccess.writes((input) => | 41 | const UPLOAD_OWNER_TTL_MS = 24 * 60 * 60 * 1000 + ATTACHMENT_UPLOAD_URL_TTL_MS; |
| 35 | Upload.deletePendingAttachment(input.attachmentId).pipe(Effect.as({})), | 42 | |
| 36 | ), | 43 | /** A thread owns its uploads across provider sessions; an outside client per MCP session. */ |
| 37 | t3_thread_send_attachments: McpToolAccess.writesThreads( | 44 | const uploadOwner = McpInvocationContext.McpInvocationContext.pipe( |
| 38 | (input) => [input.threadId], | 45 | Effect.map((scope) => scope.thread?.threadId ?? scope.requestNamespace), |
| 39 | (input) => | 46 | ); |
| 40 | Effect.gen(function* () { | 47 | |
| 41 | const { caller, projection } = yield* readThread(input.threadId, ["messages"]); | 48 | export const layer = McpToolAccess.toLayer( |
| 42 | if (projection.thread.archivedAt !== null) | 49 | AttachmentToolkit, |
| 43 | return yield* new OrchestratorMcpFailure({ | 50 | Effect.sync(() => { |
| 44 | code: "invalid_request", | 51 | // Which caller prepared each pending upload, so only that caller can discard it. |
| 45 | message: "Unarchive the target thread before sending attachments.", | 52 | const uploadOwners = new Map<string, { readonly owner: string; readonly issuedAt: number }>(); |
| 46 | }); | 53 | return { |
| 47 | const attachments = yield* resolveAttachmentReferences( | 54 | t3_attachment_prepare_upload: McpToolAccess.writes((input) => |
| 48 | input.attachments, | 55 | Effect.gen(function* () { |
| 49 | projection.messages.flatMap((message) => message.attachments), | 56 | const result = yield* Upload.issueAttachmentUploadUrl(input.upload).pipe( |
| 50 | ); | 57 | Effect.mapError(unavailable), |
| 51 | const commandId = yield* newCommandId(); | 58 | ); |
| 52 | const messageId = MessageId.make(commandId); | 59 | const now = yield* Clock.currentTimeMillis; |
| 53 | const result = yield* ThreadMessageIntake.sendToThread({ | 60 | for (const [id, entry] of uploadOwners) { |
| 54 | projectId: projection.thread.projectId, | 61 | if (now - entry.issuedAt > UPLOAD_OWNER_TTL_MS) uploadOwners.delete(id); |
| 55 | threadId: projection.thread.id, | 62 | } |
| 56 | commandId, | 63 | uploadOwners.set(result.attachmentId, { owner: yield* uploadOwner, issuedAt: now }); |
| 57 | messageId, | 64 | return result; |
| 58 | ...(caller === undefined ? {} : { senderThreadId: caller.id }), | 65 | }), |
| 59 | text: input.message ?? "", | 66 | ), |
| 60 | attachments, | 67 | t3_attachment_discard: McpToolAccess.writes((input) => |
| 61 | mode: "auto", | 68 | Effect.gen(function* () { |
| 62 | createdBy: "agent", | 69 | if (uploadOwners.get(input.attachmentId)?.owner !== (yield* uploadOwner)) { |
| 63 | creationSource: "mcp", | 70 | return yield* new OrchestratorMcpFailure({ |
| 64 | }).pipe( | 71 | code: "invalid_request", |
| 65 | Effect.mapError((error) => | 72 | message: "Only the caller that prepared a pending upload can discard it.", |
| 66 | error._tag === "AttachmentClaimError" | 73 | }); |
| 67 | ? new OrchestratorMcpFailure({ code: "orchestration_error", message: error.message }) | 74 | } |
| 68 | : unavailable(), | 75 | yield* Upload.deletePendingAttachment(input.attachmentId); |
| 69 | ), | 76 | uploadOwners.delete(input.attachmentId); |
| 70 | ); | 77 | return {}; |
| 71 | return { | 78 | }), |
| 72 | threadId: projection.thread.id, | 79 | ), |
| 73 | messageId, | 80 | t3_thread_send_attachments: McpToolAccess.writesThreads( |
| 74 | runId: result.run.id, | 81 | (input) => [input.threadId], |
| 75 | status: result.run.status, | 82 | (input) => |
| 76 | }; | 83 | Effect.gen(function* () { |
| 77 | }), | 84 | const { caller, projection } = yield* readThread(input.threadId, ["messages"]); |
| 78 | ), | 85 | if (projection.thread.archivedAt !== null) |
| 79 | }); | 86 | return yield* new OrchestratorMcpFailure({ |
| 87 | code: "invalid_request", | ||
| 88 | message: "Unarchive the target thread before sending attachments.", | ||
| 89 | }); | ||
| 90 | const attachments = yield* resolveAttachmentReferences( | ||
| 91 | input.attachments, | ||
| 92 | projection.messages.flatMap((message) => message.attachments), | ||
| 93 | ); | ||
| 94 | const commandId = yield* newCommandId(); | ||
| 95 | const messageId = MessageId.make(commandId); | ||
| 96 | const result = yield* ThreadMessageIntake.sendToThread({ | ||
| 97 | projectId: projection.thread.projectId, | ||
| 98 | threadId: projection.thread.id, | ||
| 99 | commandId, | ||
| 100 | messageId, | ||
| 101 | ...(caller === undefined ? {} : { senderThreadId: caller.id }), | ||
| 102 | text: input.message ?? "", | ||
| 103 | attachments, | ||
| 104 | mode: "auto", | ||
| 105 | createdBy: "agent", | ||
| 106 | creationSource: "mcp", | ||
| 107 | }).pipe( | ||
| 108 | Effect.mapError((error) => | ||
| 109 | error._tag === "AttachmentClaimError" | ||
| 110 | ? new OrchestratorMcpFailure({ | ||
| 111 | code: "orchestration_error", | ||
| 112 | message: error.message, | ||
| 113 | }) | ||
| 114 | : unavailable(), | ||
| 115 | ), | ||
| 116 | ); | ||
| 117 | return { | ||
| 118 | threadId: projection.thread.id, | ||
| 119 | messageId, | ||
| 120 | runId: result.run.id, | ||
| 121 | status: result.run.status, | ||
| 122 | }; | ||
| 123 | }), | ||
| 124 | ), | ||
| 125 | }; | ||
| 126 | }), | ||
| 127 | ); |
apps/server/src/mcp/toolkits/core.test.ts+60| ... | @@ -14,6 +14,7 @@ import { | ... | @@ -14,6 +14,7 @@ import { |
| 14 | type OrchestrationV2ThreadShell, | 14 | type OrchestrationV2ThreadShell, |
| 15 | } from "@t3tools/contracts"; | 15 | } from "@t3tools/contracts"; |
| 16 | import * as Effect from "effect/Effect"; | 16 | import * as Effect from "effect/Effect"; |
| 17 | import * as TestClock from "effect/testing/TestClock"; | ||
| 17 | import * as Layer from "effect/Layer"; | 18 | import * as Layer from "effect/Layer"; |
| 18 | import * as Schema from "effect/Schema"; | 19 | import * as Schema from "effect/Schema"; |
| 19 | import { McpAttachmentInput } from "./attachment/input.ts"; | 20 | import { McpAttachmentInput } from "./attachment/input.ts"; |
| ... | @@ -26,6 +27,7 @@ import { | ... | @@ -26,6 +27,7 @@ import { |
| 26 | OrchestratorProjectionError, | 27 | OrchestratorProjectionError, |
| 27 | } from "../../orchestration-v2/Orchestrator.ts"; | 28 | } from "../../orchestration-v2/Orchestrator.ts"; |
| 28 | 29 | ||
| 30 | import * as ServerSecretStore from "../../auth/ServerSecretStore.ts"; | ||
| 29 | import * as ServerConfig from "../../config.ts"; | 31 | import * as ServerConfig from "../../config.ts"; |
| 30 | import * as ProviderAdapterRegistry from "../../orchestration-v2/ProviderAdapterRegistry.ts"; | 32 | import * as ProviderAdapterRegistry from "../../orchestration-v2/ProviderAdapterRegistry.ts"; |
| 31 | import * as ThreadManagement from "../../orchestration-v2/ThreadManagementService.ts"; | 33 | import * as ThreadManagement from "../../orchestration-v2/ThreadManagementService.ts"; |
| ... | @@ -696,3 +698,61 @@ it.effect("a caller cannot interrupt a thread that runs above its own modes", () | ... | @@ -696,3 +698,61 @@ it.effect("a caller cannot interrupt a thread that runs above its own modes", () |
| 696 | ), | 698 | ), |
| 697 | ), | 699 | ), |
| 698 | ); | 700 | ); |
| 701 | |||
| 702 | it.effect("only the caller that prepared a pending upload can discard it", () => | ||
| 703 | Effect.gen(function* () { | ||
| 704 | const server = yield* McpServer.McpServer; | ||
| 705 | const call = ( | ||
| 706 | name: string, | ||
| 707 | args: Record<string, unknown>, | ||
| 708 | invocation: McpInvocationContext.McpInvocationScope, | ||
| 709 | ) => | ||
| 710 | server | ||
| 711 | .callTool({ name, arguments: args }) | ||
| 712 | .pipe( | ||
| 713 | Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), | ||
| 714 | Effect.provideService(McpSchema.McpServerClient, client), | ||
| 715 | ); | ||
| 716 | const prepared = yield* call( | ||
| 717 | "t3_attachment_prepare_upload", | ||
| 718 | { upload: { name: "shot.png", mimeType: "image/png", sizeBytes: 4 } }, | ||
| 719 | scope, | ||
| 720 | ); | ||
| 721 | const { attachmentId } = prepared.structuredContent as { readonly attachmentId: string }; | ||
| 722 | const otherThread = { | ||
| 723 | ...scope, | ||
| 724 | requestNamespace: "other-session", | ||
| 725 | thread: { ...scope.thread!, threadId: ThreadId.make("other-thread") }, | ||
| 726 | }; | ||
| 727 | |||
| 728 | const refused = yield* call("t3_attachment_discard", { attachmentId }, otherThread); | ||
| 729 | expect(declaredFailure(refused)).toMatchObject({ code: "invalid_request" }); | ||
| 730 | |||
| 731 | // Another prepare a day later keeps it: the file outlives its URL's 24 hours | ||
| 732 | // by as long as the upload took, up to the URL's own lifetime. | ||
| 733 | yield* TestClock.adjust(24 * 60 * 60 * 1000 + 5 * 60_000); | ||
| 734 | yield* call( | ||
| 735 | "t3_attachment_prepare_upload", | ||
| 736 | { upload: { name: "later.png", mimeType: "image/png", sizeBytes: 4 } }, | ||
| 737 | scope, | ||
| 738 | ); | ||
| 739 | |||
| 740 | // A new provider session of the same thread still owns the upload. | ||
| 741 | const discarded = yield* call( | ||
| 742 | "t3_attachment_discard", | ||
| 743 | { attachmentId }, | ||
| 744 | { ...scope, requestNamespace: "mcp-core-session-2" }, | ||
| 745 | ); | ||
| 746 | expect(discarded.isError).toBe(false); | ||
| 747 | }).pipe( | ||
| 748 | Effect.provide( | ||
| 749 | McpHttpServer.layerAttachmentToolkit.pipe( | ||
| 750 | Layer.provideMerge(McpServer.McpServer.layer), | ||
| 751 | Layer.provide(McpToolAccessTestkit.liveThreadsLayer), | ||
| 752 | Layer.provide(ServerSecretStore.layer), | ||
| 753 | Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-mcp-attachment-" })), | ||
| 754 | Layer.provide(NodeServices.layer), | ||
| 755 | ), | ||
| 756 | ), | ||
| 757 | ), | ||
| 758 | ); |
apps/server/src/mcp/toolkits/device/handlers.ts+12-7| ... | @@ -211,15 +211,20 @@ const handlers = { | ... | @@ -211,15 +211,20 @@ const handlers = { |
| 211 | const scope = yield* requireDeviceAccess; | 211 | const scope = yield* requireDeviceAccess; |
| 212 | const devices = yield* DeviceService.DeviceService; | 212 | const devices = yield* DeviceService.DeviceService; |
| 213 | const sessions = yield* devices.sessionsForThread(scope.thread.threadId); | 213 | const sessions = yield* devices.sessionsForThread(scope.thread.threadId); |
| 214 | const target = | 214 | // Only devices this thread opened: another thread's device is not this agent's to watch. |
| 215 | input.deviceId !== undefined | 215 | const hostId = |
| 216 | ? { hostId: input.hostId ?? LOCAL_DEVICE_HOST_ID, deviceId: input.deviceId } | 216 | input.deviceId === undefined ? input.hostId : (input.hostId ?? LOCAL_DEVICE_HOST_ID); |
| 217 | : sessions | 217 | const target = sessions.findLast( |
| 218 | .filter((session) => input.hostId === undefined || session.hostId === input.hostId) | 218 | (session) => |
| 219 | .at(-1); | 219 | (hostId === undefined || session.hostId === hostId) && |
| 220 | (input.deviceId === undefined || session.deviceId === input.deviceId), | ||
| 221 | ); | ||
| 220 | if (!target) { | 222 | if (!target) { |
| 221 | return yield* new DeviceToolUnavailableError({ | 223 | return yield* new DeviceToolUnavailableError({ |
| 222 | reason: "No device is open in this thread. Call device_open first.", | 224 | reason: |
| 225 | input.deviceId === undefined | ||
| 226 | ? "No device is open in this thread. Call device_open first." | ||
| 227 | : `Device ${input.deviceId} on host ${hostId} is not open in this thread. Call device_open first.`, | ||
| 223 | }); | 228 | }); |
| 224 | } | 229 | } |
| 225 | const shot = yield* devices.screenshot(target); | 230 | const shot = yield* devices.screenshot(target); |
apps/server/src/mcp/toolkits/preview/tools.ts+9-8| ... | @@ -64,7 +64,7 @@ const readonlyBrowserTool = <T extends Tool.Any>(tool: T): T => | ... | @@ -64,7 +64,7 @@ const readonlyBrowserTool = <T extends Tool.Any>(tool: T): T => |
| 64 | 64 | ||
| 65 | const PreviewStatusTool = Tool.make("preview_status", { | 65 | const PreviewStatusTool = Tool.make("preview_status", { |
| 66 | description: | 66 | description: |
| 67 | "Report whether a collaborative browser tab is automation-capable, including its control owner, pending dialog, URL, title, visibility, loading state, viewport mode, and measured CSS-pixel size. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab.", | 67 | "Report whether a collaborative browser tab is automation-capable, including its control owner, pending dialog, URL, title, visibility, loading state, viewport mode, and measured CSS-pixel size. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab, or the tab the user is viewing when you have none. Server hosts also list every tab in the thread (tabs) with its owner, including tabs the user opened, and the browser profiles preview_open accepts. You can read any listed tab; act only on your own tabs, or on an unclaimed tab while no human controls it.", |
| 68 | parameters: PreviewAutomationTabTargetInput, | 68 | parameters: PreviewAutomationTabTargetInput, |
| 69 | success: PreviewAutomationStatus, | 69 | success: PreviewAutomationStatus, |
| 70 | failure: PreviewToolFailure, | 70 | failure: PreviewToolFailure, |
| ... | @@ -78,7 +78,7 @@ const PreviewStatusTool = Tool.make("preview_status", { | ... | @@ -78,7 +78,7 @@ const PreviewStatusTool = Tool.make("preview_status", { |
| 78 | const PreviewOpenTool = browserTool( | 78 | const PreviewOpenTool = browserTool( |
| 79 | Tool.make("preview_open", { | 79 | Tool.make("preview_open", { |
| 80 | description: | 80 | description: |
| 81 | "Initialize a collaborative browser tab and open its thread-bound inline preview by default. Set open=false for background-only automation. Pass tabId to reuse a specific existing tab, set reuseExistingTab=false to create another tab, or omit both to use this agent session's current tab. Parallel subagents sharing a provider session must each open with reuseExistingTab=false and pass their returned tabId on every call. Server tabs use isolated storage and cannot be operated by a different agent session.", | 81 | "Initialize a collaborative browser tab and open its thread-bound inline preview by default. Set open=false for background-only automation. Pass tabId to reuse a specific existing tab, set reuseExistingTab=false to create another tab, or omit both to use this agent session's current tab. Parallel subagents sharing a provider session must each open with reuseExistingTab=false and pass their returned tabId on every call. Pass profileId (an id or name from preview_status profiles) to open under a browser profile and its saved logins; omit it for the user's default profile. Another agent session's tabs can be read but not operated.", |
| 82 | parameters: PreviewAutomationOpenInput, | 82 | parameters: PreviewAutomationOpenInput, |
| 83 | success: PreviewAutomationStatus, | 83 | success: PreviewAutomationStatus, |
| 84 | failure: PreviewToolFailure, | 84 | failure: PreviewToolFailure, |
| ... | @@ -91,7 +91,7 @@ const PreviewOpenTool = browserTool( | ... | @@ -91,7 +91,7 @@ const PreviewOpenTool = browserTool( |
| 91 | const PreviewDialogTool = browserTool( | 91 | const PreviewDialogTool = browserTool( |
| 92 | Tool.make("preview_dialog", { | 92 | Tool.make("preview_dialog", { |
| 93 | description: | 93 | description: |
| 94 | "Accept or dismiss the server browser dialog reported by preview_status. For a prompt, supply promptText when accepting. Requires this agent to own the tab. Desktop hosts may not support this operation.", | 94 | "Accept or dismiss the server browser dialog reported by preview_status. For a prompt, supply promptText when accepting. Requires this agent to own the tab, or the tab to be unclaimed with no human in control. Desktop hosts may not support this operation.", |
| 95 | parameters: PreviewAutomationDialogInput, | 95 | parameters: PreviewAutomationDialogInput, |
| 96 | success: PreviewAutomationStatus, | 96 | success: PreviewAutomationStatus, |
| 97 | failure: PreviewToolFailure, | 97 | failure: PreviewToolFailure, |
| ... | @@ -139,22 +139,23 @@ const PreviewSetAppearanceTool = safeBrowserTool( | ... | @@ -139,22 +139,23 @@ const PreviewSetAppearanceTool = safeBrowserTool( |
| 139 | .annotate(Tool.Idempotent, true), | 139 | .annotate(Tool.Idempotent, true), |
| 140 | ); | 140 | ); |
| 141 | 141 | ||
| 142 | export const PreviewSnapshotTool = readonlyBrowserTool( | 142 | // Not read-only: save=true writes the screenshot to disk. |
| 143 | export const PreviewSnapshotTool = safeBrowserTool( | ||
| 143 | Tool.make("preview_snapshot", { | 144 | Tool.make("preview_snapshot", { |
| 144 | description: | 145 | description: |
| 145 | "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. Server snapshots include an accessibilityTree with refs; pass locator=aria-ref=<ref> to target one exact element, including inside frames. Refresh refs after navigation, another snapshot, or human takeover. The text is capped near 20 KB and lists what it omitted; use preview_evaluate to read more. Set includeImage=false for text-only output with the same page metadata. Set save=true to also write the PNG to disk and get screenshotPath back; with includeImage=false, save=true returns only the url and screenshotPath. Embed that path in your reply as  so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.", | 146 | "Inspect a page before interacting. Pass tabId to inspect a specific tab, including one the user opened (see preview_status tabs), even while they control it; omit it to use this agent session's current tab, or the tab the user is viewing when you have none. Returns page state, semantic elements, diagnostics, action history, and screenshot dimensions. Server snapshots include an accessibilityTree with refs; pass locator=aria-ref=<ref> to target one exact element, including inside frames. Refresh refs after navigation, another snapshot, or human takeover. The text is capped near 20 KB and lists what it omitted; use preview_evaluate to read more. Set includeImage=true only when you need to see the page; the image stays in tool history. Set save=true to write the PNG to disk and get back only the url and screenshotPath. Embed that path in your reply as  so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.", |
| 146 | parameters: Schema.Struct({ | 147 | parameters: Schema.Struct({ |
| 147 | ...PreviewAutomationTabTargetInput.fields, | 148 | ...PreviewAutomationTabTargetInput.fields, |
| 148 | includeImage: Schema.optional( | 149 | includeImage: Schema.optional( |
| 149 | Schema.Boolean.annotate({ | 150 | Schema.Boolean.annotate({ |
| 150 | description: | 151 | description: |
| 151 | "Include the PNG image in the tool response. Defaults to true. Set false for text-only output.", | 152 | "Include the PNG image in the tool response. Defaults to false. Set true only when you need to see the page.", |
| 152 | }), | 153 | }), |
| 153 | ), | 154 | ), |
| 154 | save: Schema.optional( | 155 | save: Schema.optional( |
| 155 | Schema.Boolean.annotate({ | 156 | Schema.Boolean.annotate({ |
| 156 | description: | 157 | description: |
| 157 | "Write the screenshot PNG to disk and return its absolute path as screenshotPath. With includeImage=false, return only the url and screenshotPath. Defaults to false.", | 158 | "Write the screenshot PNG to disk and return its absolute path as screenshotPath. Unless includeImage=true, return only the url and screenshotPath. Defaults to false.", |
| 158 | }), | 159 | }), |
| 159 | ), | 160 | ), |
| 160 | }), | 161 | }), |
| ... | @@ -270,7 +271,7 @@ export const PreviewEvaluateResult = Schema.Struct({ | ... | @@ -270,7 +271,7 @@ export const PreviewEvaluateResult = Schema.Struct({ |
| 270 | const PreviewEvaluateTool = browserTool( | 271 | const PreviewEvaluateTool = browserTool( |
| 271 | Tool.make("preview_evaluate", { | 272 | Tool.make("preview_evaluate", { |
| 272 | description: | 273 | description: |
| 273 | "Evaluate JavaScript in the tab selected by tabId, or this agent session's current tab when omitted. Returns {value} with a serializable result up to 64 KB; the expression may mutate page state.", | 274 | "Evaluate JavaScript in the tab selected by tabId, or this agent session's current tab when omitted. Needs the same control as clicking: it works on your own tabs and on the user's tab only while nobody controls it. To read a tab the user is driving, use preview_snapshot. Returns {value} with a serializable result up to 64 KB; the expression may mutate page state.", |
| 274 | parameters: PreviewAutomationEvaluateInput, | 275 | parameters: PreviewAutomationEvaluateInput, |
| 275 | success: PreviewEvaluateResult, | 276 | success: PreviewEvaluateResult, |
| 276 | failure: PreviewToolFailure, | 277 | failure: PreviewToolFailure, |
apps/server/src/observability/RpcInstrumentation.ts+1| ... | @@ -183,6 +183,7 @@ const RPC_AGGREGATES = { | ... | @@ -183,6 +183,7 @@ const RPC_AGGREGATES = { |
| 183 | [WS_METHODS.previewClose]: "preview", | 183 | [WS_METHODS.previewClose]: "preview", |
| 184 | [WS_METHODS.previewList]: "preview", | 184 | [WS_METHODS.previewList]: "preview", |
| 185 | [WS_METHODS.previewClearProfile]: "preview", | 185 | [WS_METHODS.previewClearProfile]: "preview", |
| 186 | [WS_METHODS.previewReportProfiles]: "preview", | ||
| 186 | [WS_METHODS.previewReportStatus]: "preview", | 187 | [WS_METHODS.previewReportStatus]: "preview", |
| 187 | [WS_METHODS.subscribePreviewEvents]: "preview", | 188 | [WS_METHODS.subscribePreviewEvents]: "preview", |
| 188 | [WS_METHODS.subscribeDiscoveredLocalServers]: "preview", | 189 | [WS_METHODS.subscribeDiscoveredLocalServers]: "preview", |
apps/server/src/preview/ServerBrowser.test.ts+201-5| ... | @@ -38,7 +38,8 @@ vi.mock("./ServerBrowserContexts.ts", () => ({ | ... | @@ -38,7 +38,8 @@ vi.mock("./ServerBrowserContexts.ts", () => ({ |
| 38 | constructor(options: { onContextClose?: (context: BrowserContext) => void }) { | 38 | constructor(options: { onContextClose?: (context: BrowserContext) => void }) { |
| 39 | this.onClose = options.onContextClose; | 39 | this.onClose = options.onContextClose; |
| 40 | } | 40 | } |
| 41 | async contextFor() { | 41 | async contextFor(profileId: string, isolationKey?: string) { |
| 42 | contextRequests.push({ profileId, isolated: isolationKey !== undefined }); | ||
| 42 | if (contextFailure) throw contextFailure; | 43 | if (contextFailure) throw contextFailure; |
| 43 | await contextGate?.promise; | 44 | await contextGate?.promise; |
| 44 | const context = makeContext(this.onClose); | 45 | const context = makeContext(this.onClose); |
| ... | @@ -149,6 +150,8 @@ function makeContext(onClose?: (context: BrowserContext) => void) { | ... | @@ -149,6 +150,8 @@ function makeContext(onClose?: (context: BrowserContext) => void) { |
| 149 | } | 150 | } |
| 150 | 151 | ||
| 151 | const contexts: ReturnType<typeof makeContext>[] = []; | 152 | const contexts: ReturnType<typeof makeContext>[] = []; |
| 153 | /** The profile and isolation each headless tab asked its context for. */ | ||
| 154 | const contextRequests: Array<{ profileId: string; isolated: boolean }> = []; | ||
| 152 | let contextGate: PromiseWithResolvers<void> | null = null; | 155 | let contextGate: PromiseWithResolvers<void> | null = null; |
| 153 | type ClipboardBinding = (source: { page: unknown }, text: unknown) => void; | 156 | type ClipboardBinding = (source: { page: unknown }, text: unknown) => void; |
| 154 | let clipboardBinding: ClipboardBinding | null = null; | 157 | let clipboardBinding: ClipboardBinding | null = null; |
| ... | @@ -266,6 +269,7 @@ const viewerInput = (tabId: string, canOperate: boolean) => ({ | ... | @@ -266,6 +269,7 @@ const viewerInput = (tabId: string, canOperate: boolean) => ({ |
| 266 | 269 | ||
| 267 | beforeEach(() => { | 270 | beforeEach(() => { |
| 268 | contexts.length = 0; | 271 | contexts.length = 0; |
| 272 | contextRequests.length = 0; | ||
| 269 | contextGate = null; | 273 | contextGate = null; |
| 270 | contextFailure = null; | 274 | contextFailure = null; |
| 271 | desktopTabs.clear(); | 275 | desktopTabs.clear(); |
| ... | @@ -380,18 +384,37 @@ it.live("enforces provider ownership and explicit targets when a session has mul | ... | @@ -380,18 +384,37 @@ it.live("enforces provider ownership and explicit targets when a session has mul |
| 380 | .invoke<void>({ | 384 | .invoke<void>({ |
| 381 | scope: asSession("agent-b"), | 385 | scope: asSession("agent-b"), |
| 382 | tabId, | 386 | tabId, |
| 383 | operation: "evaluate", | 387 | operation: "navigate", |
| 384 | input: { expression: "foreign()" }, | 388 | input: { url: "http://localhost:5173/foreign" }, |
| 385 | }) | 389 | }) |
| 386 | .pipe(Effect.flip); | 390 | .pipe(Effect.flip); |
| 387 | expect(foreign).toMatchObject({ | 391 | expect(foreign).toMatchObject({ |
| 388 | _tag: "PreviewAutomationControlInterruptedError", | 392 | _tag: "PreviewAutomationControlInterruptedError", |
| 389 | reason: "agentMismatch", | 393 | reason: "agentMismatch", |
| 390 | }); | 394 | }); |
| 391 | expect(contexts[0]!.sessions[0]!.send).not.toHaveBeenCalledWith( | 395 | expect(contexts[0]!.page.goto).not.toHaveBeenCalledWith( |
| 392 | "Runtime.evaluate", | 396 | "http://localhost:5173/foreign", |
| 393 | expect.anything(), | 397 | expect.anything(), |
| 394 | ); | 398 | ); |
| 399 | // Another session may still read the tab, but not run page script in it. | ||
| 400 | yield* broker.invoke<PreviewAutomationSnapshot>({ | ||
| 401 | scope: asSession("agent-b"), | ||
| 402 | tabId, | ||
| 403 | operation: "snapshot", | ||
| 404 | input: {}, | ||
| 405 | }); | ||
| 406 | const foreignEvaluate = yield* broker | ||
| 407 | .invoke<void>({ | ||
| 408 | scope: asSession("agent-b"), | ||
| 409 | tabId, | ||
| 410 | operation: "evaluate", | ||
| 411 | input: { expression: "read()" }, | ||
| 412 | }) | ||
| 413 | .pipe(Effect.flip); | ||
| 414 | expect(foreignEvaluate).toMatchObject({ | ||
| 415 | _tag: "PreviewAutomationControlInterruptedError", | ||
| 416 | reason: "agentMismatch", | ||
| 417 | }); | ||
| 395 | yield* broker.invoke({ | 418 | yield* broker.invoke({ |
| 396 | scope, | 419 | scope, |
| 397 | operation: "open", | 420 | operation: "open", |
| ... | @@ -1323,6 +1346,179 @@ it.live("a desktop page the desktop takes back reconnects instead of closing", ( | ... | @@ -1323,6 +1346,179 @@ it.live("a desktop page the desktop takes back reconnects instead of closing", ( |
| 1323 | ).pipe(Effect.provide(layer)), | 1346 | ).pipe(Effect.provide(layer)), |
| 1324 | ); | 1347 | ); |
| 1325 | 1348 | ||
| 1349 | it.live("agents read a human's tab with no arguments and act on it only while nobody drives", () => | ||
| 1350 | Effect.scoped( | ||
| 1351 | Effect.gen(function* () { | ||
| 1352 | const browser = yield* ServerBrowser.ServerBrowser; | ||
| 1353 | const broker = yield* Broker.PreviewAutomationBroker; | ||
| 1354 | const manager = yield* Manager.PreviewManager; | ||
| 1355 | yield* Effect.yieldNow; | ||
| 1356 | const opened = yield* manager.open({ | ||
| 1357 | threadId: scope.thread.threadId, | ||
| 1358 | url: "http://localhost:5173/mine", | ||
| 1359 | runtime: "server", | ||
| 1360 | }); | ||
| 1361 | const tabId = PreviewTabId.make(opened.tabId); | ||
| 1362 | // The user opened and is looking at the tab; attaching takes control. | ||
| 1363 | const viewer = yield* browser.attachViewer(viewerInput(tabId, true)); | ||
| 1364 | const status = yield* broker.invoke<PreviewAutomationStatus>({ | ||
| 1365 | scope, | ||
| 1366 | operation: "status", | ||
| 1367 | input: {}, | ||
| 1368 | }); | ||
| 1369 | expect(status).toMatchObject({ | ||
| 1370 | tabId, | ||
| 1371 | control: { owner: "human", ownedByCaller: false }, | ||
| 1372 | }); | ||
| 1373 | expect(status.tabs).toEqual([ | ||
| 1374 | expect.objectContaining({ tabId, owner: "human", ownedByCaller: false, visible: true }), | ||
| 1375 | ]); | ||
| 1376 | // Reads work while the user drives; page script does not, since it can change the page. | ||
| 1377 | const evaluated = yield* broker | ||
| 1378 | .invoke<void>({ scope, operation: "evaluate", input: { expression: "read()" } }) | ||
| 1379 | .pipe(Effect.flip); | ||
| 1380 | expect(evaluated).toMatchObject({ | ||
| 1381 | _tag: "PreviewAutomationControlInterruptedError", | ||
| 1382 | reason: "humanControl", | ||
| 1383 | }); | ||
| 1384 | yield* broker.invoke<PreviewAutomationSnapshot>({ | ||
| 1385 | scope, | ||
| 1386 | tabId, | ||
| 1387 | operation: "snapshot", | ||
| 1388 | input: {}, | ||
| 1389 | }); | ||
| 1390 | // Acting is refused while the user controls the tab. | ||
| 1391 | const refused = yield* broker | ||
| 1392 | .invoke<void>({ | ||
| 1393 | scope, | ||
| 1394 | tabId, | ||
| 1395 | operation: "navigate", | ||
| 1396 | input: { url: "http://localhost:5173/agent" }, | ||
| 1397 | }) | ||
| 1398 | .pipe(Effect.flip); | ||
| 1399 | expect(refused).toMatchObject({ | ||
| 1400 | _tag: "PreviewAutomationControlInterruptedError", | ||
| 1401 | reason: "humanControl", | ||
| 1402 | }); | ||
| 1403 | // Once they let go, the tab is unclaimed and the agent may act on it. | ||
| 1404 | yield* viewer.input({ type: "releaseControl" }); | ||
| 1405 | const released = yield* broker.invoke<PreviewAutomationStatus>({ | ||
| 1406 | scope, | ||
| 1407 | tabId, | ||
| 1408 | operation: "status", | ||
| 1409 | input: {}, | ||
| 1410 | }); | ||
| 1411 | expect(released.control).toMatchObject({ owner: "unclaimed", ownedByCaller: false }); | ||
| 1412 | yield* broker.invoke({ | ||
| 1413 | scope, | ||
| 1414 | tabId, | ||
| 1415 | operation: "navigate", | ||
| 1416 | input: { url: "http://localhost:5173/agent" }, | ||
| 1417 | }); | ||
| 1418 | expect(contexts[0]!.page.goto).toHaveBeenCalledWith( | ||
| 1419 | "http://localhost:5173/agent", | ||
| 1420 | expect.anything(), | ||
| 1421 | ); | ||
| 1422 | // Taking control back refuses the agent again. | ||
| 1423 | yield* viewer.input({ type: "takeControl" }); | ||
| 1424 | const retaken = yield* broker | ||
| 1425 | .invoke<void>({ scope, tabId, operation: "press", input: { key: "Enter" } }) | ||
| 1426 | .pipe(Effect.flip); | ||
| 1427 | expect(retaken).toMatchObject({ reason: "humanControl" }); | ||
| 1428 | }), | ||
| 1429 | ).pipe(Effect.provide(layer)), | ||
| 1430 | ); | ||
| 1431 | |||
| 1432 | it.live("a session's own tab stays its default while other sessions' tabs are listed", () => | ||
| 1433 | Effect.scoped( | ||
| 1434 | Effect.gen(function* () { | ||
| 1435 | const { broker, tabId } = yield* ready; | ||
| 1436 | const other = yield* broker.invoke<PreviewAutomationStatus>({ | ||
| 1437 | scope: asSession("agent-b"), | ||
| 1438 | operation: "open", | ||
| 1439 | input: { reuseExistingTab: false, show: false }, | ||
| 1440 | }); | ||
| 1441 | const status = yield* broker.invoke<PreviewAutomationStatus>({ | ||
| 1442 | scope, | ||
| 1443 | operation: "status", | ||
| 1444 | input: {}, | ||
| 1445 | }); | ||
| 1446 | expect(status.tabId).toBe(tabId); | ||
| 1447 | expect(status.tabs).toEqual( | ||
| 1448 | expect.arrayContaining([ | ||
| 1449 | expect.objectContaining({ tabId, owner: "agent", ownedByCaller: true }), | ||
| 1450 | expect.objectContaining({ tabId: other.tabId, owner: "agent", ownedByCaller: false }), | ||
| 1451 | ]), | ||
| 1452 | ); | ||
| 1453 | }), | ||
| 1454 | ).pipe(Effect.provide(layer)), | ||
| 1455 | ); | ||
| 1456 | |||
| 1457 | it.live( | ||
| 1458 | "preview_open picks a reported profile by id or name and defaults to the reported one", | ||
| 1459 | () => | ||
| 1460 | Effect.scoped( | ||
| 1461 | Effect.gen(function* () { | ||
| 1462 | const browser = yield* ServerBrowser.ServerBrowser; | ||
| 1463 | const broker = yield* Broker.PreviewAutomationBroker; | ||
| 1464 | const manager = yield* Manager.PreviewManager; | ||
| 1465 | yield* Effect.yieldNow; | ||
| 1466 | yield* browser.reportProfiles({ | ||
| 1467 | profiles: [{ id: "profile-work", name: "Work", kind: "persistent" }], | ||
| 1468 | defaultProfileId: "profile-work", | ||
| 1469 | }); | ||
| 1470 | const openWith = (profileId?: string) => | ||
| 1471 | broker.invoke<PreviewAutomationStatus>({ | ||
| 1472 | scope, | ||
| 1473 | operation: "open", | ||
| 1474 | input: { | ||
| 1475 | reuseExistingTab: false, | ||
| 1476 | show: false, | ||
| 1477 | ...(profileId === undefined ? {} : { profileId }), | ||
| 1478 | }, | ||
| 1479 | }); | ||
| 1480 | const byDefault = yield* openWith(); | ||
| 1481 | const byName = yield* openWith("WORK"); | ||
| 1482 | const byId = yield* openWith("incognito"); | ||
| 1483 | const { sessions } = yield* manager.list({ threadId: scope.thread.threadId }); | ||
| 1484 | const profileOf = (tabId: string | null) => | ||
| 1485 | sessions.find((session) => session.tabId === tabId)?.profileId; | ||
| 1486 | expect(profileOf(byDefault.tabId)).toBe("profile-work"); | ||
| 1487 | expect(profileOf(byName.tabId)).toBe("profile-work"); | ||
| 1488 | expect(profileOf(byId.tabId)).toBe("incognito"); | ||
| 1489 | // The chosen profile reaches the headless tab's storage. | ||
| 1490 | expect(contextRequests).toEqual([ | ||
| 1491 | { profileId: "profile-work", isolated: false }, | ||
| 1492 | { profileId: "profile-work", isolated: false }, | ||
| 1493 | { profileId: "incognito", isolated: true }, | ||
| 1494 | ]); | ||
| 1495 | expect(byDefault).toMatchObject({ | ||
| 1496 | defaultProfileId: "profile-work", | ||
| 1497 | profiles: [ | ||
| 1498 | { id: "default", name: "Default" }, | ||
| 1499 | { id: "incognito", name: "Incognito", incognito: true }, | ||
| 1500 | { id: "profile-work", name: "Work" }, | ||
| 1501 | ], | ||
| 1502 | }); | ||
| 1503 | const unknown = yield* openWith("Personal").pipe(Effect.flip); | ||
| 1504 | expect(unknown).toMatchObject({ | ||
| 1505 | _tag: "PreviewAutomationExecutionError", | ||
| 1506 | reason: | ||
| 1507 | 'No browser profile is named "Personal". Use one of: Default (id default), Incognito (id incognito), Work (id profile-work).', | ||
| 1508 | }); | ||
| 1509 | }), | ||
| 1510 | ).pipe(Effect.provide(layer)), | ||
| 1511 | ); | ||
| 1512 | |||
| 1513 | it.live("an agent tab keeps throwaway storage when no client reported profiles", () => | ||
| 1514 | Effect.scoped( | ||
| 1515 | Effect.gen(function* () { | ||
| 1516 | yield* ready; | ||
| 1517 | expect(contextRequests).toEqual([{ profileId: "default", isolated: true }]); | ||
| 1518 | }), | ||
| 1519 | ).pipe(Effect.provide(layer)), | ||
| 1520 | ); | ||
| 1521 | |||
| 1326 | it.live("a desktop page that comes back reconnects without waiting for a viewer", () => | 1522 | it.live("a desktop page that comes back reconnects without waiting for a viewer", () => |
| 1327 | Effect.scoped( | 1523 | Effect.scoped( |
| 1328 | Effect.gen(function* () { | 1524 | Effect.gen(function* () { |
apps/server/src/preview/ServerBrowser.ts+148-22| ... | @@ -2,8 +2,14 @@ | ... | @@ -2,8 +2,14 @@ |
| 2 | // Screencasts ignore emulated device scale; real 2x keeps captures sharp. | 2 | // Screencasts ignore emulated device scale; real 2x keeps captures sharp. |
| 3 | // --disable-gpu uses cheaper software compositing while preserving SwiftShader WebGL. | 3 | // --disable-gpu uses cheaper software compositing while preserving SwiftShader WebGL. |
| 4 | import { | 4 | import { |
| 5 | BUILT_IN_BROWSER_PROFILES, | ||
| 6 | DEFAULT_BROWSER_PROFILE_ID, | ||
| 5 | FILL_PREVIEW_VIEWPORT, | 7 | FILL_PREVIEW_VIEWPORT, |
| 8 | findBrowserProfile, | ||
| 6 | INCOGNITO_BROWSER_PROFILE_ID, | 9 | INCOGNITO_BROWSER_PROFILE_ID, |
| 10 | resolveBrowserProfiles, | ||
| 11 | type BrowserProfile, | ||
| 12 | type PreviewReportProfilesInput, | ||
| 7 | PREVIEW_AUTOMATION_SERVER_OPERATIONS, | 13 | PREVIEW_AUTOMATION_SERVER_OPERATIONS, |
| 8 | PreviewViewportSetting as PreviewViewportSettingSchema, | 14 | PreviewViewportSetting as PreviewViewportSettingSchema, |
| 9 | PROVIDER_SEND_TURN_MAX_FILE_BYTES, | 15 | PROVIDER_SEND_TURN_MAX_FILE_BYTES, |
| ... | @@ -243,6 +249,11 @@ export class ServerBrowser extends Context.Service< | ... | @@ -243,6 +249,11 @@ export class ServerBrowser extends Context.Service< |
| 243 | }) => Effect.Effect<Option.Option<{ readonly path: string; readonly fileName: string }>>; | 249 | }) => Effect.Effect<Option.Option<{ readonly path: string; readonly fileName: string }>>; |
| 244 | /** Deletes a human profile's server-side storage, closing its open tabs first. */ | 250 | /** Deletes a human profile's server-side storage, closing its open tabs first. */ |
| 245 | readonly clearProfile: (profileId: string) => Effect.Effect<void, PreviewClearProfileError>; | 251 | readonly clearProfile: (profileId: string) => Effect.Effect<void, PreviewClearProfileError>; |
| 252 | /** | ||
| 253 | * Records a client's browser profiles for agents' preview_open. In memory: | ||
| 254 | * clients report again on connect and whenever the list changes. | ||
| 255 | */ | ||
| 256 | readonly reportProfiles: (input: PreviewReportProfilesInput) => Effect.Effect<void>; | ||
| 246 | } | 257 | } |
| 247 | >()("t3/preview/ServerBrowser") {} | 258 | >()("t3/preview/ServerBrowser") {} |
| 248 | 259 | ||
| ... | @@ -346,6 +357,17 @@ interface ServerDownload { | ... | @@ -346,6 +357,17 @@ interface ServerDownload { |
| 346 | } | 357 | } |
| 347 | 358 | ||
| 348 | /** One agent session and the whole server; each tab holds a renderer process. */ | 359 | /** One agent session and the whole server; each tab holds a renderer process. */ |
| 360 | /** | ||
| 361 | * Operations an agent may run on any tab in its thread, even while a human | ||
| 362 | * drives it. Evaluate is not one: page script can change anything, so it needs | ||
| 363 | * the same control as clicking. | ||
| 364 | */ | ||
| 365 | const READ_OPERATIONS: ReadonlySet<PreviewAutomationRequest["operation"]> = new Set([ | ||
| 366 | "status", | ||
| 367 | "snapshot", | ||
| 368 | "waitFor", | ||
| 369 | ]); | ||
| 370 | |||
| 349 | const AGENT_TAB_LIMIT = 8; | 371 | const AGENT_TAB_LIMIT = 8; |
| 350 | const SERVER_TAB_LIMIT = 32; | 372 | const SERVER_TAB_LIMIT = 32; |
| 351 | /** Agent tabs nobody watches close after this long without an agent request. */ | 373 | /** Agent tabs nobody watches close after this long without an agent request. */ |
| ... | @@ -727,10 +749,11 @@ const make = Effect.gen(function* () { | ... | @@ -727,10 +749,11 @@ const make = Effect.gen(function* () { |
| 727 | adopted === undefined && (await desktopRenders(snapshot)) | 749 | adopted === undefined && (await desktopRenders(snapshot)) |
| 728 | ? await connectDesktop(snapshot) | 750 | ? await connectDesktop(snapshot) |
| 729 | : null; | 751 | : null; |
| 752 | // An agent tab without a profile (no client reported one) keeps throwaway storage. | ||
| 730 | const isolatedContext = | 753 | const isolatedContext = |
| 731 | adopted === undefined && | 754 | adopted === undefined && |
| 732 | desktop === null && | 755 | desktop === null && |
| 733 | (snapshot.automationOwner !== undefined || | 756 | ((snapshot.automationOwner !== undefined && snapshot.profileId === undefined) || |
| 734 | snapshot.profileId === INCOGNITO_BROWSER_PROFILE_ID); | 757 | snapshot.profileId === INCOGNITO_BROWSER_PROFILE_ID); |
| 735 | const context = | 758 | const context = |
| 736 | adopted?.page.context() ?? | 759 | adopted?.page.context() ?? |
| ... | @@ -1199,10 +1222,75 @@ const make = Effect.gen(function* () { | ... | @@ -1199,10 +1222,75 @@ const make = Effect.gen(function* () { |
| 1199 | } | 1222 | } |
| 1200 | }; | 1223 | }; |
| 1201 | 1224 | ||
| 1202 | const latestThreadTab = (threadId: string, agentSessionId?: string) => | 1225 | /** |
| 1203 | [...tabs.values()] | 1226 | * The tab a request without a tabId means: the caller's newest tab, else the |
| 1204 | .filter((tab) => tab.threadId === threadId && tab.control.agentId === agentSessionId) | 1227 | * thread's tab the user is looking at or used last, so "this page" works. |
| 1205 | .sort((left, right) => right.createdAt - left.createdAt)[0]; | 1228 | */ |
| 1229 | const latestThreadTab = (threadId: string, agentSessionId?: string) => { | ||
| 1230 | const threadTabs = [...tabs.values()].filter((tab) => tab.threadId === threadId); | ||
| 1231 | return ( | ||
| 1232 | threadTabs | ||
| 1233 | .filter((tab) => tab.control.agentId === agentSessionId) | ||
| 1234 | .sort((left, right) => right.createdAt - left.createdAt)[0] ?? | ||
| 1235 | threadTabs.sort( | ||
| 1236 | (left, right) => | ||
| 1237 | Number(right.viewers.size > 0) - Number(left.viewers.size > 0) || | ||
| 1238 | right.usedAt - left.usedAt, | ||
| 1239 | )[0] | ||
| 1240 | ); | ||
| 1241 | }; | ||
| 1242 | |||
| 1243 | const tabOwner = (tab: ServerTab) => | ||
| 1244 | tab.control.controller !== null | ||
| 1245 | ? ("human" as const) | ||
| 1246 | : tab.control.agentId !== null | ||
| 1247 | ? ("agent" as const) | ||
| 1248 | : ("unclaimed" as const); | ||
| 1249 | |||
| 1250 | /** The latest profile list a client reported; agents choose from it. */ | ||
| 1251 | let reportedProfiles: { | ||
| 1252 | readonly profiles: ReadonlyArray<BrowserProfile>; | ||
| 1253 | readonly defaultProfileId: string; | ||
| 1254 | } | null = null; | ||
| 1255 | |||
| 1256 | const reportProfiles: ServerBrowser["Service"]["reportProfiles"] = (input) => | ||
| 1257 | Effect.sync(() => { | ||
| 1258 | const profiles = resolveBrowserProfiles(input.profiles); | ||
| 1259 | reportedProfiles = { | ||
| 1260 | profiles, | ||
| 1261 | defaultProfileId: | ||
| 1262 | findBrowserProfile(profiles, input.defaultProfileId)?.id ?? DEFAULT_BROWSER_PROFILE_ID, | ||
| 1263 | }; | ||
| 1264 | }); | ||
| 1265 | |||
| 1266 | const profileStatus = () => { | ||
| 1267 | const profiles = reportedProfiles?.profiles ?? BUILT_IN_BROWSER_PROFILES; | ||
| 1268 | return { | ||
| 1269 | profiles: profiles.map((profile) => ({ | ||
| 1270 | id: profile.id, | ||
| 1271 | name: profile.name, | ||
| 1272 | ...(profile.kind === "incognito" ? { incognito: true } : {}), | ||
| 1273 | })), | ||
| 1274 | ...(reportedProfiles ? { defaultProfileId: reportedProfiles.defaultProfileId } : {}), | ||
| 1275 | }; | ||
| 1276 | }; | ||
| 1277 | |||
| 1278 | /** Matches an id, then a case-insensitive name. Omitted means the reported default. */ | ||
| 1279 | const resolveOpenProfile = (requested: string | undefined): string | undefined => { | ||
| 1280 | if (requested === undefined) return reportedProfiles?.defaultProfileId; | ||
| 1281 | const profiles = reportedProfiles?.profiles ?? BUILT_IN_BROWSER_PROFILES; | ||
| 1282 | const wanted = requested.toLowerCase(); | ||
| 1283 | const match = | ||
| 1284 | findBrowserProfile(profiles, requested) ?? | ||
| 1285 | profiles.find((profile) => profile.name.toLowerCase() === wanted); | ||
| 1286 | if (match) return match.id; | ||
| 1287 | throw new ServerBrowserPage.ServerBrowserOperationError( | ||
| 1288 | "PreviewAutomationUnknownProfileError", | ||
| 1289 | `No browser profile is named "${requested}". Use one of: ${profiles | ||
| 1290 | .map((profile) => `${profile.name} (id ${profile.id})`) | ||
| 1291 | .join(", ")}.`, | ||
| 1292 | ); | ||
| 1293 | }; | ||
| 1206 | 1294 | ||
| 1207 | const statusWithTitle = async ( | 1295 | const statusWithTitle = async ( |
| 1208 | tab: ServerTab | undefined, | 1296 | tab: ServerTab | undefined, |
| ... | @@ -1216,6 +1304,7 @@ const make = Effect.gen(function* () { | ... | @@ -1216,6 +1304,7 @@ const make = Effect.gen(function* () { |
| 1216 | url: null, | 1304 | url: null, |
| 1217 | title: null, | 1305 | title: null, |
| 1218 | loading: false, | 1306 | loading: false, |
| 1307 | ...profileStatus(), | ||
| 1219 | }; | 1308 | }; |
| 1220 | } | 1309 | } |
| 1221 | const url = tab.page.url(); | 1310 | const url = tab.page.url(); |
| ... | @@ -1228,12 +1317,7 @@ const make = Effect.gen(function* () { | ... | @@ -1228,12 +1317,7 @@ const make = Effect.gen(function* () { |
| 1228 | title: null, | 1317 | title: null, |
| 1229 | loading: tab.loading, | 1318 | loading: tab.loading, |
| 1230 | control: { | 1319 | control: { |
| 1231 | owner: | 1320 | owner: tabOwner(tab), |
| 1232 | tab.control.controller !== null | ||
| 1233 | ? ("human" as const) | ||
| 1234 | : tab.control.agentId !== null | ||
| 1235 | ? ("agent" as const) | ||
| 1236 | : ("unclaimed" as const), | ||
| 1237 | ownedByCaller: tab.control.agentId === agentSessionId, | 1321 | ownedByCaller: tab.control.agentId === agentSessionId, |
| 1238 | generation: tab.control.generation, | 1322 | generation: tab.control.generation, |
| 1239 | }, | 1323 | }, |
| ... | @@ -1244,15 +1328,17 @@ const make = Effect.gen(function* () { | ... | @@ -1244,15 +1328,17 @@ const make = Effect.gen(function* () { |
| 1244 | viewportSetting: tab.setting, | 1328 | viewportSetting: tab.setting, |
| 1245 | ...(viewport ? { viewport } : {}), | 1329 | ...(viewport ? { viewport } : {}), |
| 1246 | tabs: [...tabs.values()] | 1330 | tabs: [...tabs.values()] |
| 1247 | .filter( | 1331 | .filter((candidate) => candidate.threadId === tab.threadId) |
| 1248 | (candidate) => | ||
| 1249 | candidate.threadId === tab.threadId && candidate.control.agentId === agentSessionId, | ||
| 1250 | ) | ||
| 1251 | .map((candidate) => ({ | 1332 | .map((candidate) => ({ |
| 1252 | tabId: candidate.tabId, | 1333 | tabId: candidate.tabId, |
| 1253 | url: candidate.page.url() === "about:blank" ? null : candidate.page.url(), | 1334 | url: candidate.page.url() === "about:blank" ? null : candidate.page.url(), |
| 1254 | ...(candidate.openerTabId === undefined ? {} : { openerTabId: candidate.openerTabId }), | 1335 | ...(candidate.openerTabId === undefined ? {} : { openerTabId: candidate.openerTabId }), |
| 1336 | owner: tabOwner(candidate), | ||
| 1337 | ownedByCaller: candidate.control.agentId === agentSessionId, | ||
| 1338 | visible: candidate.viewers.size > 0, | ||
| 1339 | ...(candidate.profileId === undefined ? {} : { profileId: candidate.profileId }), | ||
| 1255 | })), | 1340 | })), |
| 1341 | ...profileStatus(), | ||
| 1256 | downloads: tab.downloads.map(({ fileName, path, sizeBytes, url, completedAt }) => ({ | 1342 | downloads: tab.downloads.map(({ fileName, path, sizeBytes, url, completedAt }) => ({ |
| 1257 | fileName, | 1343 | fileName, |
| 1258 | path, | 1344 | path, |
| ... | @@ -1554,19 +1640,26 @@ const make = Effect.gen(function* () { | ... | @@ -1554,19 +1640,26 @@ const make = Effect.gen(function* () { |
| 1554 | "No server preview tab is open for this thread. Call preview_open first.", | 1640 | "No server preview tab is open for this thread. Call preview_open first.", |
| 1555 | ); | 1641 | ); |
| 1556 | } | 1642 | } |
| 1557 | if (tab.control.agentId !== request.agentSessionId) | 1643 | // Any tab in the thread can be read; acting is checked by its control. |
| 1644 | if (!READ_OPERATIONS.has(request.operation) && !tab.control.agentMayAct(request.agentSessionId)) | ||
| 1558 | throw new BrowserControlInterrupted( | 1645 | throw new BrowserControlInterrupted( |
| 1559 | "This tab belongs to another agent session or a human. Open your own tab.", | 1646 | "This tab belongs to another agent session. You can read it, but open your own tab to act.", |
| 1560 | "agentMismatch", | 1647 | "agentMismatch", |
| 1561 | ); | 1648 | ); |
| 1562 | return tab; | 1649 | return tab; |
| 1563 | }; | 1650 | }; |
| 1564 | 1651 | ||
| 1565 | /** Any request, even a failed one, keeps the agent's tabs on the thread from idling out. */ | 1652 | /** |
| 1653 | * Any request, even a failed one, keeps the agent's tabs on the thread from | ||
| 1654 | * idling out. The tab it targeted becomes the thread's most recently used. | ||
| 1655 | */ | ||
| 1566 | const markUsed = (request: PreviewAutomationRequest) => { | 1656 | const markUsed = (request: PreviewAutomationRequest) => { |
| 1567 | const now = Date.now(); | 1657 | const now = Date.now(); |
| 1568 | for (const tab of tabs.values()) { | 1658 | for (const tab of tabs.values()) { |
| 1569 | if (tab.threadId === request.threadId && tab.control.agentId === request.agentSessionId) | 1659 | if ( |
| 1660 | tab.threadId === request.threadId && | ||
| 1661 | (tab.control.agentId === request.agentSessionId || tab.tabId === request.tabId) | ||
| 1662 | ) | ||
| 1570 | tab.usedAt = now; | 1663 | tab.usedAt = now; |
| 1571 | } | 1664 | } |
| 1572 | }; | 1665 | }; |
| ... | @@ -1602,7 +1695,7 @@ const make = Effect.gen(function* () { | ... | @@ -1602,7 +1695,7 @@ const make = Effect.gen(function* () { |
| 1602 | "tabRequired", | 1695 | "tabRequired", |
| 1603 | ); | 1696 | ); |
| 1604 | // A tab still launching exists only as a session, so resolve it like a viewer would. | 1697 | // A tab still launching exists only as a session, so resolve it like a viewer would. |
| 1605 | const existing = | 1698 | const found = |
| 1606 | reuse && request.tabId !== undefined | 1699 | reuse && request.tabId !== undefined |
| 1607 | ? await Effect.runPromise( | 1700 | ? await Effect.runPromise( |
| 1608 | findTab(request.threadId, request.tabId).pipe( | 1701 | findTab(request.threadId, request.tabId).pipe( |
| ... | @@ -1612,7 +1705,14 @@ const make = Effect.gen(function* () { | ... | @@ -1612,7 +1705,14 @@ const make = Effect.gen(function* () { |
| 1612 | ), | 1705 | ), |
| 1613 | ) | 1706 | ) |
| 1614 | : undefined; | 1707 | : undefined; |
| 1708 | // Only an explicit tabId reuses a tab this session did not open; a tab | ||
| 1709 | // it last read (such as the user's) is not taken over implicitly. | ||
| 1710 | const existing = | ||
| 1711 | found && (request.tabIdExplicit || found.control.agentId === request.agentSessionId) | ||
| 1712 | ? found | ||
| 1713 | : undefined; | ||
| 1615 | const navigationTimeout = Math.min(request.timeoutMs, NAVIGATION_TIMEOUT_MS); | 1714 | const navigationTimeout = Math.min(request.timeoutMs, NAVIGATION_TIMEOUT_MS); |
| 1715 | const profileId = existing ? undefined : resolveOpenProfile(open.profileId); | ||
| 1616 | if (!existing) { | 1716 | if (!existing) { |
| 1617 | closeIdleAgentTabs(); | 1717 | closeIdleAgentTabs(); |
| 1618 | assertTabCapacity(request.agentSessionId); | 1718 | assertTabCapacity(request.agentSessionId); |
| ... | @@ -1624,6 +1724,7 @@ const make = Effect.gen(function* () { | ... | @@ -1624,6 +1724,7 @@ const make = Effect.gen(function* () { |
| 1624 | manager.open({ | 1724 | manager.open({ |
| 1625 | threadId: request.threadId, | 1725 | threadId: request.threadId, |
| 1626 | ...(url ? { url } : {}), | 1726 | ...(url ? { url } : {}), |
| 1727 | ...(profileId === undefined ? {} : { profileId }), | ||
| 1627 | runtime: "server", | 1728 | runtime: "server", |
| 1628 | reveal: false, | 1729 | reveal: false, |
| 1629 | automationOwner: request.agentSessionId, | 1730 | automationOwner: request.agentSessionId, |
| ... | @@ -1681,7 +1782,7 @@ const make = Effect.gen(function* () { | ... | @@ -1681,7 +1782,7 @@ const make = Effect.gen(function* () { |
| 1681 | const recordings = [...tabs.values()].filter( | 1782 | const recordings = [...tabs.values()].filter( |
| 1682 | (candidate) => | 1783 | (candidate) => |
| 1683 | candidate.threadId === request.threadId && | 1784 | candidate.threadId === request.threadId && |
| 1684 | candidate.control.agentId === request.agentSessionId && | 1785 | candidate.control.agentMayAct(request.agentSessionId ?? "") && |
| 1685 | (candidate.recording || candidate.recordingStart), | 1786 | (candidate.recording || candidate.recordingStart), |
| 1686 | ); | 1787 | ); |
| 1687 | const targetTabId = | 1788 | const targetTabId = |
| ... | @@ -1701,6 +1802,11 @@ const make = Effect.gen(function* () { | ... | @@ -1701,6 +1802,11 @@ const make = Effect.gen(function* () { |
| 1701 | const tab = await requireTab(request); | 1802 | const tab = await requireTab(request); |
| 1702 | // Closing must unblock an action waiting on a dialog, without queueing behind it. | 1803 | // Closing must unblock an action waiting on a dialog, without queueing behind it. |
| 1703 | if (request.operation === "close") { | 1804 | if (request.operation === "close") { |
| 1805 | if (tab.control.agentId !== request.agentSessionId) | ||
| 1806 | throw new BrowserControlInterrupted( | ||
| 1807 | "Only the agent session that opened this tab can close it.", | ||
| 1808 | "agentMismatch", | ||
| 1809 | ); | ||
| 1704 | if (tab.control.controller !== null) | 1810 | if (tab.control.controller !== null) |
| 1705 | throw new BrowserControlInterrupted("A human controls this tab.", "humanControl"); | 1811 | throw new BrowserControlInterrupted("A human controls this tab.", "humanControl"); |
| 1706 | void tab.control.close().catch(constVoid); | 1812 | void tab.control.close().catch(constVoid); |
| ... | @@ -1716,7 +1822,26 @@ const make = Effect.gen(function* () { | ... | @@ -1716,7 +1822,26 @@ const make = Effect.gen(function* () { |
| 1716 | await resolveDialog(tab, input as PreviewAutomationDialogInput); | 1822 | await resolveDialog(tab, input as PreviewAutomationDialogInput); |
| 1717 | return statusWithTitle(tab, request.agentSessionId); | 1823 | return statusWithTitle(tab, request.agentSessionId); |
| 1718 | } | 1824 | } |
| 1719 | return tab.control.agent(request.agentSessionId!, async () => { | 1825 | const agentSessionId = request.agentSessionId!; |
| 1826 | // A tab this session did not open, while it cannot act there (another | ||
| 1827 | // agent's, or the user's while they drive): read it in place, without | ||
| 1828 | // queueing behind the human's input. Its own tabs keep the takeover | ||
| 1829 | // contract: a human taking control interrupts every agent call. | ||
| 1830 | if ( | ||
| 1831 | READ_OPERATIONS.has(request.operation) && | ||
| 1832 | tab.control.agentId !== agentSessionId && | ||
| 1833 | !tab.control.agentCanActNow(agentSessionId) | ||
| 1834 | ) { | ||
| 1835 | return tab.control.observe(async () => { | ||
| 1836 | if (tab.dialog) | ||
| 1837 | throw new BrowserControlInterrupted( | ||
| 1838 | "A browser dialog is pending. Read preview_status.", | ||
| 1839 | "dialogPending", | ||
| 1840 | ); | ||
| 1841 | return executeTabOperation(tab, request); | ||
| 1842 | }); | ||
| 1843 | } | ||
| 1844 | return tab.control.agent(agentSessionId, async () => { | ||
| 1720 | if (tab.dialog) | 1845 | if (tab.dialog) |
| 1721 | throw new BrowserControlInterrupted( | 1846 | throw new BrowserControlInterrupted( |
| 1722 | "A browser dialog is pending. Read preview_status and use preview_dialog first.", | 1847 | "A browser dialog is pending. Read preview_status and use preview_dialog first.", |
| ... | @@ -2328,6 +2453,7 @@ const make = Effect.gen(function* () { | ... | @@ -2328,6 +2453,7 @@ const make = Effect.gen(function* () { |
| 2328 | return ServerBrowser.of({ | 2453 | return ServerBrowser.of({ |
| 2329 | attachViewer, | 2454 | attachViewer, |
| 2330 | clearProfile, | 2455 | clearProfile, |
| 2456 | reportProfiles, | ||
| 2331 | openDownload, | 2457 | openDownload, |
| 2332 | answerFileChooser, | 2458 | answerFileChooser, |
| 2333 | }); | 2459 | }); |
apps/server/src/preview/ServerBrowserStream.test.ts+5| ... | @@ -87,6 +87,7 @@ it.effect.each([ | ... | @@ -87,6 +87,7 @@ it.effect.each([ |
| 87 | }); | 87 | }); |
| 88 | const browser = ServerBrowser.ServerBrowser.of({ | 88 | const browser = ServerBrowser.ServerBrowser.of({ |
| 89 | clearProfile: () => Effect.void, | 89 | clearProfile: () => Effect.void, |
| 90 | reportProfiles: () => Effect.void, | ||
| 90 | openDownload: () => Effect.succeedNone, | 91 | openDownload: () => Effect.succeedNone, |
| 91 | answerFileChooser: () => Effect.succeed(false), | 92 | answerFileChooser: () => Effect.succeed(false), |
| 92 | attachViewer: (input) => | 93 | attachViewer: (input) => |
| ... | @@ -163,6 +164,7 @@ it.effect.each([ | ... | @@ -163,6 +164,7 @@ it.effect.each([ |
| 163 | let attachments = 0; | 164 | let attachments = 0; |
| 164 | const browser = ServerBrowser.ServerBrowser.of({ | 165 | const browser = ServerBrowser.ServerBrowser.of({ |
| 165 | clearProfile: () => Effect.void, | 166 | clearProfile: () => Effect.void, |
| 167 | reportProfiles: () => Effect.void, | ||
| 166 | openDownload: () => Effect.succeedNone, | 168 | openDownload: () => Effect.succeedNone, |
| 167 | answerFileChooser: () => Effect.succeed(false), | 169 | answerFileChooser: () => Effect.succeed(false), |
| 168 | attachViewer: () => { | 170 | attachViewer: () => { |
| ... | @@ -203,6 +205,7 @@ it.effect("serves a tab's download only to an authorized session", () => | ... | @@ -203,6 +205,7 @@ it.effect("serves a tab's download only to an authorized session", () => |
| 203 | const requests: Array<unknown> = []; | 205 | const requests: Array<unknown> = []; |
| 204 | const browser = ServerBrowser.ServerBrowser.of({ | 206 | const browser = ServerBrowser.ServerBrowser.of({ |
| 205 | clearProfile: () => Effect.void, | 207 | clearProfile: () => Effect.void, |
| 208 | reportProfiles: () => Effect.void, | ||
| 206 | openDownload: (input) => | 209 | openDownload: (input) => |
| 207 | Effect.sync(() => { | 210 | Effect.sync(() => { |
| 208 | requests.push(input); | 211 | requests.push(input); |
| ... | @@ -250,6 +253,7 @@ it.effect("passes uploaded files to the page's open picker and needs operate sco | ... | @@ -250,6 +253,7 @@ it.effect("passes uploaded files to the page's open picker and needs operate sco |
| 250 | const answers: Array<{ chooserId: string; files: Array<{ name: string; text: string }> }> = []; | 253 | const answers: Array<{ chooserId: string; files: Array<{ name: string; text: string }> }> = []; |
| 251 | const browser = ServerBrowser.ServerBrowser.of({ | 254 | const browser = ServerBrowser.ServerBrowser.of({ |
| 252 | clearProfile: () => Effect.void, | 255 | clearProfile: () => Effect.void, |
| 256 | reportProfiles: () => Effect.void, | ||
| 253 | openDownload: () => Effect.succeedNone, | 257 | openDownload: () => Effect.succeedNone, |
| 254 | answerFileChooser: (input) => | 258 | answerFileChooser: (input) => |
| 255 | Effect.sync(() => { | 259 | Effect.sync(() => { |
| ... | @@ -317,6 +321,7 @@ it.effect.each([ | ... | @@ -317,6 +321,7 @@ it.effect.each([ |
| 317 | Effect.gen(function* () { | 321 | Effect.gen(function* () { |
| 318 | const browser = ServerBrowser.ServerBrowser.of({ | 322 | const browser = ServerBrowser.ServerBrowser.of({ |
| 319 | clearProfile: () => Effect.void, | 323 | clearProfile: () => Effect.void, |
| 324 | reportProfiles: () => Effect.void, | ||
| 320 | openDownload: () => Effect.succeedNone, | 325 | openDownload: () => Effect.succeedNone, |
| 321 | answerFileChooser: () => Effect.succeed(false), | 326 | answerFileChooser: () => Effect.succeed(false), |
| 322 | attachViewer: () => Effect.fail(new ServerBrowser.ServerBrowserLaunchError({ cause: error })), | 327 | attachViewer: () => Effect.fail(new ServerBrowser.ServerBrowserLaunchError({ cause: error })), |
apps/server/src/preview/SessionControl.test.ts+12| ... | @@ -166,4 +166,16 @@ describe("SessionControl", () => { | ... | @@ -166,4 +166,16 @@ describe("SessionControl", () => { |
| 166 | ).rejects.toThrow("navigation failed"); | 166 | ).rejects.toThrow("navigation failed"); |
| 167 | await expect(control.agent("agent", async () => "recovered")).resolves.toBe("recovered"); | 167 | await expect(control.agent("agent", async () => "recovered")).resolves.toBe("recovered"); |
| 168 | }); | 168 | }); |
| 169 | |||
| 170 | it("lets any agent act on a tab no agent opened, but only while no human controls it", async () => { | ||
| 171 | const control = new SessionControl(null); | ||
| 172 | await expect(control.agent("agent-a", async () => "acted")).resolves.toBe("acted"); | ||
| 173 | await control.take("viewer-a"); | ||
| 174 | await expect(control.agent("agent-a", async () => "racing")).rejects.toMatchObject({ | ||
| 175 | reason: "humanControl", | ||
| 176 | }); | ||
| 177 | await expect(control.observe(async () => "read")).resolves.toBe("read"); | ||
| 178 | await control.release("viewer-a"); | ||
| 179 | await expect(control.agent("agent-b", async () => "acted")).resolves.toBe("acted"); | ||
| 180 | }); | ||
| 169 | }); | 181 | }); |
apps/server/src/preview/SessionControl.ts+25-2| ... | @@ -73,8 +73,17 @@ export class SessionControl { | ... | @@ -73,8 +73,17 @@ export class SessionControl { |
| 73 | }); | 73 | }); |
| 74 | } | 74 | } |
| 75 | 75 | ||
| 76 | /** Whether an agent may act here: its own tab, or a tab no agent opened. */ | ||
| 77 | agentMayAct(agentId: string) { | ||
| 78 | return this.agentId === null || this.agentId === agentId; | ||
| 79 | } | ||
| 80 | |||
| 81 | /** | ||
| 82 | * An agent action. It may act on its own tab or on one a human opened, but | ||
| 83 | * never while a human controls the tab; taking control interrupts it. | ||
| 84 | */ | ||
| 76 | agent<A>(agentId: string, run: () => Promise<A>) { | 85 | agent<A>(agentId: string, run: () => Promise<A>) { |
| 77 | if (this.agentId !== agentId) | 86 | if (!this.agentMayAct(agentId)) |
| 78 | return Promise.reject( | 87 | return Promise.reject( |
| 79 | new BrowserControlInterrupted("This tab belongs to another agent.", "agentMismatch"), | 88 | new BrowserControlInterrupted("This tab belongs to another agent.", "agentMismatch"), |
| 80 | ); | 89 | ); |
| ... | @@ -82,7 +91,21 @@ export class SessionControl { | ... | @@ -82,7 +91,21 @@ export class SessionControl { |
| 82 | return Promise.reject( | 91 | return Promise.reject( |
| 83 | new BrowserControlInterrupted("A human controls this tab.", "humanControl"), | 92 | new BrowserControlInterrupted("A human controls this tab.", "humanControl"), |
| 84 | ); | 93 | ); |
| 85 | return this.action(() => this.agentId === agentId && this.owner === null, run); | 94 | return this.action(() => this.agentMayAct(agentId) && this.owner === null, run); |
| 95 | } | ||
| 96 | |||
| 97 | /** Whether this agent's action would run now rather than be refused. */ | ||
| 98 | agentCanActNow(agentId: string) { | ||
| 99 | return this.agentMayAct(agentId) && this.owner === null; | ||
| 100 | } | ||
| 101 | |||
| 102 | /** | ||
| 103 | * A read that needs no control and does not queue, so an agent can look at | ||
| 104 | * a page while a human drives it without holding up the human's input. | ||
| 105 | */ | ||
| 106 | observe<A>(run: () => Promise<A>) { | ||
| 107 | this.assertOpen(); | ||
| 108 | return run(); | ||
| 86 | } | 109 | } |
| 87 | 110 | ||
| 88 | /** | 111 | /** |
apps/server/src/ws.ts+1| ... | @@ -2899,6 +2899,7 @@ const layerWsRpc = ( | ... | @@ -2899,6 +2899,7 @@ const layerWsRpc = ( |
| 2899 | [WS_METHODS.previewClose]: (input) => previewManager.close(input), | 2899 | [WS_METHODS.previewClose]: (input) => previewManager.close(input), |
| 2900 | [WS_METHODS.previewList]: (input) => previewManager.list(input), | 2900 | [WS_METHODS.previewList]: (input) => previewManager.list(input), |
| 2901 | [WS_METHODS.previewClearProfile]: (input) => serverBrowser.clearProfile(input.profileId), | 2901 | [WS_METHODS.previewClearProfile]: (input) => serverBrowser.clearProfile(input.profileId), |
| 2902 | [WS_METHODS.previewReportProfiles]: (input) => serverBrowser.reportProfiles(input), | ||
| 2902 | [WS_METHODS.previewReportStatus]: (input) => previewManager.reportStatus(input), | 2903 | [WS_METHODS.previewReportStatus]: (input) => previewManager.reportStatus(input), |
| 2903 | [WS_METHODS.subscribePreviewEvents]: (_input) => previewManager.events, | 2904 | [WS_METHODS.subscribePreviewEvents]: (_input) => previewManager.events, |
| 2904 | [WS_METHODS.deviceConfigure]: (input) => deviceService.configure(input), | 2905 | [WS_METHODS.deviceConfigure]: (input) => deviceService.configure(input), |
apps/web/src/AppRoot.tsx+2| ... | @@ -1,5 +1,6 @@ | ... | @@ -1,5 +1,6 @@ |
| 1 | import { RouterProvider } from "@tanstack/react-router"; | 1 | import { RouterProvider } from "@tanstack/react-router"; |
| 2 | 2 | ||
| 3 | import { BrowserProfileReporter } from "./browser/BrowserProfileReporter"; | ||
| 3 | import { ElectronBrowserHost } from "./browser/ElectronBrowserHost"; | 4 | import { ElectronBrowserHost } from "./browser/ElectronBrowserHost"; |
| 4 | import { QuitHoldOverlay } from "./components/QuitHoldOverlay"; | 5 | import { QuitHoldOverlay } from "./components/QuitHoldOverlay"; |
| 5 | import { AppAtomRegistryProvider } from "./rpc/atomRegistry"; | 6 | import { AppAtomRegistryProvider } from "./rpc/atomRegistry"; |
| ... | @@ -15,6 +16,7 @@ export function AppRoot({ router }: { readonly router: AppRouter }) { | ... | @@ -15,6 +16,7 @@ export function AppRoot({ router }: { readonly router: AppRouter }) { |
| 15 | <AppAtomRegistryProvider> | 16 | <AppAtomRegistryProvider> |
| 16 | <RouterProvider router={router} /> | 17 | <RouterProvider router={router} /> |
| 17 | <ElectronBrowserHost /> | 18 | <ElectronBrowserHost /> |
| 19 | <BrowserProfileReporter /> | ||
| 18 | <QuitHoldOverlay /> | 20 | <QuitHoldOverlay /> |
| 19 | </AppAtomRegistryProvider> | 21 | </AppAtomRegistryProvider> |
| 20 | ); | 22 | ); |
apps/web/src/browser/BrowserProfileReporter.tsx created+40| ... | @@ -0,0 +1,40 @@ | ||
| 1 | import { AuthPreviewOperateScope, type BrowserProfile } from "@t3tools/contracts"; | ||
| 2 | import { useEffect } from "react"; | ||
| 3 | |||
| 4 | import { useClientSettings, useClientSettingsHydrated } from "~/hooks/useSettings"; | ||
| 5 | import { useServerConfigs } from "~/state/entities"; | ||
| 6 | import { useEnvironments } from "~/state/environments"; | ||
| 7 | import { previewEnvironment } from "~/state/preview"; | ||
| 8 | import { useEnvironmentsWithScope } from "~/state/session"; | ||
| 9 | import { useAtomCommand } from "~/state/use-atom-command"; | ||
| 10 | |||
| 11 | const selectProfiles = (settings: { readonly browserProfiles: ReadonlyArray<BrowserProfile> }) => | ||
| 12 | settings.browserProfiles; | ||
| 13 | const selectDefaultProfileId = (settings: { readonly browserDefaultProfileId: string }) => | ||
| 14 | settings.browserDefaultProfileId; | ||
| 15 | |||
| 16 | /** | ||
| 17 | * Tells each environment that hosts browser tabs which profiles this client | ||
| 18 | * has, so agents can open tabs under them. Profiles live in client settings; | ||
| 19 | * the server keeps only the latest report in memory. A reconnect brings a new | ||
| 20 | * server config, which sends the report again. | ||
| 21 | */ | ||
| 22 | export function BrowserProfileReporter() { | ||
| 23 | const hydrated = useClientSettingsHydrated(); | ||
| 24 | const profiles = useClientSettings(selectProfiles); | ||
| 25 | const defaultProfileId = useClientSettings(selectDefaultProfileId); | ||
| 26 | const { environments } = useEnvironments(); | ||
| 27 | const serverConfigs = useServerConfigs(); | ||
| 28 | const operable = useEnvironmentsWithScope(environments, AuthPreviewOperateScope); | ||
| 29 | const report = useAtomCommand(previewEnvironment.reportProfiles, { reportFailure: false }); | ||
| 30 | |||
| 31 | useEffect(() => { | ||
| 32 | if (!hydrated) return; | ||
| 33 | for (const [environmentId, config] of serverConfigs) { | ||
| 34 | if (!config.environment.capabilities.serverBrowser || !operable.has(environmentId)) continue; | ||
| 35 | void report({ environmentId, input: { profiles, defaultProfileId } }); | ||
| 36 | } | ||
| 37 | }, [defaultProfileId, hydrated, operable, profiles, report, serverConfigs]); | ||
| 38 | |||
| 39 | return null; | ||
| 40 | } | ||
apps/web/src/browser/ElectronBrowserHost.tsx+22-2| ... | @@ -3,11 +3,12 @@ | ... | @@ -3,11 +3,12 @@ |
| 3 | import { parseScopedThreadKey } from "@t3tools/client-runtime/environment"; | 3 | import { parseScopedThreadKey } from "@t3tools/client-runtime/environment"; |
| 4 | import { AuthPreviewOperateScope, FILL_PREVIEW_VIEWPORT } from "@t3tools/contracts"; | 4 | import { AuthPreviewOperateScope, FILL_PREVIEW_VIEWPORT } from "@t3tools/contracts"; |
| 5 | import { useAtomValue } from "@effect/atom-react"; | 5 | import { useAtomValue } from "@effect/atom-react"; |
| 6 | import { type ComponentProps, useEffect, useMemo, useRef } from "react"; | 6 | import { type ComponentProps, useEffect, useMemo, useRef, useState } from "react"; |
| 7 | 7 | ||
| 8 | import { primaryEnvironmentIdAtom } from "~/state/primaryEnvironment"; | 8 | import { primaryEnvironmentIdAtom } from "~/state/primaryEnvironment"; |
| 9 | 9 | ||
| 10 | import { isElectron } from "~/env"; | 10 | import { isElectron } from "~/env"; |
| 11 | import { useClientSettingsHydrated } from "~/hooks/useSettings"; | ||
| 11 | import { useTheme } from "~/hooks/useTheme"; | 12 | import { useTheme } from "~/hooks/useTheme"; |
| 12 | import { useActivePreviewSessions } from "~/previewStateStore"; | 13 | import { useActivePreviewSessions } from "~/previewStateStore"; |
| 13 | import { previewEnvironment } from "~/state/preview"; | 14 | import { previewEnvironment } from "~/state/preview"; |
| ... | @@ -15,6 +16,7 @@ import { useEnvironmentScope } from "~/state/session"; | ... | @@ -15,6 +16,7 @@ import { useEnvironmentScope } from "~/state/session"; |
| 15 | import { useAtomCommand } from "~/state/use-atom-command"; | 16 | import { useAtomCommand } from "~/state/use-atom-command"; |
| 16 | 17 | ||
| 17 | import { readPreviewAnnotationTheme } from "./annotationTheme"; | 18 | import { readPreviewAnnotationTheme } from "./annotationTheme"; |
| 19 | import { useBrowserDefaults } from "./browserDefaults"; | ||
| 18 | import { useBrowserPointerStore } from "./browserPointerStore"; | 20 | import { useBrowserPointerStore } from "./browserPointerStore"; |
| 19 | import { HostedBrowserWebview } from "./HostedBrowserWebview"; | 21 | import { HostedBrowserWebview } from "./HostedBrowserWebview"; |
| 20 | import { openUrlInPreview } from "./openFileInPreview"; | 22 | import { openUrlInPreview } from "./openFileInPreview"; |
| ... | @@ -158,5 +160,23 @@ function AuthorizedBrowserWebview(props: ComponentProps<typeof HostedBrowserWebv | ... | @@ -158,5 +160,23 @@ function AuthorizedBrowserWebview(props: ComponentProps<typeof HostedBrowserWebv |
| 158 | props.threadRef.environmentId, | 160 | props.threadRef.environmentId, |
| 159 | AuthPreviewOperateScope, | 161 | AuthPreviewOperateScope, |
| 160 | ); | 162 | ); |
| 161 | return canOperatePreview ? <HostedBrowserWebview {...props} /> : null; | 163 | const profileId = useTabProfileId(props.profileId); |
| 164 | return canOperatePreview ? <HostedBrowserWebview {...props} profileId={profileId} /> : null; | ||
| 165 | } | ||
| 166 | |||
| 167 | /** | ||
| 168 | * Agent `preview_open` normally carries the profile clients reported (see | ||
| 169 | * BrowserProfileReporter). An agent tab opened before any client reported has | ||
| 170 | * none, so it falls back to the configured default here. It is latched once | ||
| 171 | * settings load: Electron fixes the partition when the guest attaches, so a | ||
| 172 | * later settings change must not move a live tab. | ||
| 173 | */ | ||
| 174 | function useTabProfileId(profileId: string | undefined): string | undefined { | ||
| 175 | const hydrated = useClientSettingsHydrated(); | ||
| 176 | const defaultProfileId = useBrowserDefaults().profileId; | ||
| 177 | const [fallback, setFallback] = useState<string | undefined>(undefined); | ||
| 178 | if (profileId === undefined && hydrated && fallback === undefined) { | ||
| 179 | setFallback(defaultProfileId); | ||
| 180 | } | ||
| 181 | return profileId ?? fallback; | ||
| 162 | } | 182 | } |
packages/client-runtime/src/state/preview.ts+5| ... | @@ -76,6 +76,11 @@ export function createPreviewEnvironmentAtoms<R, E>( | ... | @@ -76,6 +76,11 @@ export function createPreviewEnvironmentAtoms<R, E>( |
| 76 | label: "environment-data:preview:clear-profile", | 76 | label: "environment-data:preview:clear-profile", |
| 77 | tag: WS_METHODS.previewClearProfile, | 77 | tag: WS_METHODS.previewClearProfile, |
| 78 | }), | 78 | }), |
| 79 | reportProfiles: createEnvironmentRpcCommand(runtime, { | ||
| 80 | label: "environment-data:preview:report-profiles", | ||
| 81 | tag: WS_METHODS.previewReportProfiles, | ||
| 82 | concurrency: { mode: "latest", key: ({ environmentId }) => environmentId }, | ||
| 83 | }), | ||
| 79 | reportStatus: createEnvironmentRpcCommand(runtime, { | 84 | reportStatus: createEnvironmentRpcCommand(runtime, { |
| 80 | label: "environment-data:preview:report-status", | 85 | label: "environment-data:preview:report-status", |
| 81 | tag: WS_METHODS.previewReportStatus, | 86 | tag: WS_METHODS.previewReportStatus, |
packages/contracts/src/preview.ts+9-1| ... | @@ -10,7 +10,7 @@ | ... | @@ -10,7 +10,7 @@ |
| 10 | */ | 10 | */ |
| 11 | import { Schema } from "effect"; | 11 | import { Schema } from "effect"; |
| 12 | import { NonNegativeInt, PositiveInt, ThreadId, TrimmedNonEmptyString } from "./baseSchemas.ts"; | 12 | import { NonNegativeInt, PositiveInt, ThreadId, TrimmedNonEmptyString } from "./baseSchemas.ts"; |
| 13 | import { BrowserProfileId } from "./browserProfile.ts"; | 13 | import { BROWSER_PROFILE_MAX_COUNT, BrowserProfile, BrowserProfileId } from "./browserProfile.ts"; |
| 14 | 14 | ||
| 15 | export const PREVIEW_URL_MAX_LENGTH = 2_048; | 15 | export const PREVIEW_URL_MAX_LENGTH = 2_048; |
| 16 | export const CONFIGURED_LOCAL_SERVER_URLS_MAX_ITEMS = 32; | 16 | export const CONFIGURED_LOCAL_SERVER_URLS_MAX_ITEMS = 32; |
| ... | @@ -295,6 +295,14 @@ export const PreviewCloseInput = Schema.Struct({ | ... | @@ -295,6 +295,14 @@ export const PreviewCloseInput = Schema.Struct({ |
| 295 | }); | 295 | }); |
| 296 | export type PreviewCloseInput = typeof PreviewCloseInput.Type; | 296 | export type PreviewCloseInput = typeof PreviewCloseInput.Type; |
| 297 | 297 | ||
| 298 | /** A client's browser profiles, which agents choose from when they open a tab. */ | ||
| 299 | export const PreviewReportProfilesInput = Schema.Struct({ | ||
| 300 | /** The user's own profiles; the server adds the built-ins. */ | ||
| 301 | profiles: Schema.Array(BrowserProfile).check(Schema.isMaxLength(BROWSER_PROFILE_MAX_COUNT)), | ||
| 302 | defaultProfileId: BrowserProfileId, | ||
| 303 | }); | ||
| 304 | export type PreviewReportProfilesInput = typeof PreviewReportProfilesInput.Type; | ||
| 305 | |||
| 298 | export const PreviewClearProfileInput = Schema.Struct({ | 306 | export const PreviewClearProfileInput = Schema.Struct({ |
| 299 | profileId: BrowserProfileId, | 307 | profileId: BrowserProfileId, |
| 300 | }); | 308 | }); |
packages/contracts/src/previewAutomation.ts+34-4| ... | @@ -10,12 +10,15 @@ import { | ... | @@ -10,12 +10,15 @@ import { |
| 10 | PreviewViewportSize, | 10 | PreviewViewportSize, |
| 11 | } from "./preview.ts"; | 11 | } from "./preview.ts"; |
| 12 | import { ProviderInstanceId } from "./providerInstance.ts"; | 12 | import { ProviderInstanceId } from "./providerInstance.ts"; |
| 13 | import { BrowserProfileId } from "./browserProfile.ts"; | ||
| 13 | 14 | ||
| 14 | const BoundedUrl = Schema.String.check(Schema.isTrimmed()) | 15 | const BoundedUrl = Schema.String.check(Schema.isTrimmed()) |
| 15 | .check(Schema.isNonEmpty()) | 16 | .check(Schema.isNonEmpty()) |
| 16 | .check(Schema.isMaxLength(2048)); | 17 | .check(Schema.isMaxLength(2048)); |
| 17 | const URL_GUIDANCE = | 18 | const URL_GUIDANCE = |
| 18 | "Absolute http(s) URL or a schemeless host such as t3.chat or localhost:5173. Schemeless public hosts use https; loopback hosts use http."; | 19 | "Absolute http(s) URL or a schemeless host such as t3.chat or localhost:5173. Schemeless public hosts use https; loopback hosts use http."; |
| 20 | const PROFILE_GUIDANCE = | ||
| 21 | "Browser profile for a new tab, by id or name (case-insensitive) from preview_status profiles. Omit to use the user's default profile. A tab keeps the profile it opened with."; | ||
| 19 | const OptionalTimeoutMs = Schema.optional( | 22 | const OptionalTimeoutMs = Schema.optional( |
| 20 | Schema.Int.check(Schema.isGreaterThan(0)) | 23 | Schema.Int.check(Schema.isGreaterThan(0)) |
| 21 | .check(Schema.isLessThanOrEqualTo(60_000)) | 24 | .check(Schema.isLessThanOrEqualTo(60_000)) |
| ... | @@ -61,17 +64,27 @@ const PreviewAutomationTabTargetFields = { | ... | @@ -61,17 +64,27 @@ const PreviewAutomationTabTargetFields = { |
| 61 | tabId: Schema.optional( | 64 | tabId: Schema.optional( |
| 62 | PreviewTabId.annotate({ | 65 | PreviewTabId.annotate({ |
| 63 | description: | 66 | description: |
| 64 | "Exact collaborative browser tab to target. Omit to use this agent session's current tab.", | 67 | "Exact collaborative browser tab to target, including a tab the user opened (see preview_status tabs). Omit to use this agent session's current tab, or the user's visible tab when this session has none.", |
| 65 | }), | 68 | }), |
| 66 | ).annotate({ | 69 | ).annotate({ |
| 67 | description: | 70 | description: |
| 68 | "Exact collaborative browser tab to target. Omit to use this agent session's current tab.", | 71 | "Exact collaborative browser tab to target, including a tab the user opened (see preview_status tabs). Omit to use this agent session's current tab, or the user's visible tab when this session has none.", |
| 69 | }), | 72 | }), |
| 70 | }; | 73 | }; |
| 71 | 74 | ||
| 72 | export const PreviewAutomationTabTargetInput = Schema.Struct(PreviewAutomationTabTargetFields); | 75 | export const PreviewAutomationTabTargetInput = Schema.Struct(PreviewAutomationTabTargetFields); |
| 73 | export type PreviewAutomationTabTargetInput = typeof PreviewAutomationTabTargetInput.Type; | 76 | export type PreviewAutomationTabTargetInput = typeof PreviewAutomationTabTargetInput.Type; |
| 74 | 77 | ||
| 78 | /** `human`: a person controls the tab now; `agent`: an agent session opened it; `unclaimed`: neither. */ | ||
| 79 | export const PreviewAutomationTabOwner = Schema.Literals(["agent", "human", "unclaimed"]); | ||
| 80 | |||
| 81 | export const PreviewAutomationProfile = Schema.Struct({ | ||
| 82 | id: BrowserProfileId, | ||
| 83 | name: Schema.String, | ||
| 84 | incognito: Schema.optional(Schema.Boolean), | ||
| 85 | }); | ||
| 86 | export type PreviewAutomationProfile = typeof PreviewAutomationProfile.Type; | ||
| 87 | |||
| 75 | export const PreviewAutomationStatus = Schema.Struct({ | 88 | export const PreviewAutomationStatus = Schema.Struct({ |
| 76 | available: Schema.Boolean, | 89 | available: Schema.Boolean, |
| 77 | visible: Schema.Boolean, | 90 | visible: Schema.Boolean, |
| ... | @@ -81,7 +94,7 @@ export const PreviewAutomationStatus = Schema.Struct({ | ... | @@ -81,7 +94,7 @@ export const PreviewAutomationStatus = Schema.Struct({ |
| 81 | loading: Schema.Boolean, | 94 | loading: Schema.Boolean, |
| 82 | control: Schema.optional( | 95 | control: Schema.optional( |
| 83 | Schema.Struct({ | 96 | Schema.Struct({ |
| 84 | owner: Schema.Literals(["agent", "human", "unclaimed"]), | 97 | owner: PreviewAutomationTabOwner, |
| 85 | ownedByCaller: Schema.Boolean, | 98 | ownedByCaller: Schema.Boolean, |
| 86 | generation: Schema.Number, | 99 | generation: Schema.Number, |
| 87 | }), | 100 | }), |
| ... | @@ -115,16 +128,28 @@ export const PreviewAutomationStatus = Schema.Struct({ | ... | @@ -115,16 +128,28 @@ export const PreviewAutomationStatus = Schema.Struct({ |
| 115 | }), | 128 | }), |
| 116 | ), | 129 | ), |
| 117 | ), | 130 | ), |
| 118 | /** Server hosts: every tab this agent session owns, including popups its pages opened. */ | 131 | /** |
| 132 | * Server hosts: every tab in the thread, including ones the user or another | ||
| 133 | * agent session opened. Any of them can be read; acting needs `ownedByCaller`, | ||
| 134 | * or an `unclaimed` tab while no human controls it. | ||
| 135 | */ | ||
| 119 | tabs: Schema.optional( | 136 | tabs: Schema.optional( |
| 120 | Schema.Array( | 137 | Schema.Array( |
| 121 | Schema.Struct({ | 138 | Schema.Struct({ |
| 122 | tabId: PreviewTabId, | 139 | tabId: PreviewTabId, |
| 123 | url: Schema.NullOr(Schema.String), | 140 | url: Schema.NullOr(Schema.String), |
| 124 | openerTabId: Schema.optional(PreviewTabId), | 141 | openerTabId: Schema.optional(PreviewTabId), |
| 142 | owner: Schema.optional(PreviewAutomationTabOwner), | ||
| 143 | ownedByCaller: Schema.optional(Schema.Boolean), | ||
| 144 | visible: Schema.optional(Schema.Boolean), | ||
| 145 | profileId: Schema.optional(BrowserProfileId), | ||
| 125 | }), | 146 | }), |
| 126 | ), | 147 | ), |
| 127 | ), | 148 | ), |
| 149 | /** Server hosts: the browser profiles preview_open accepts, as the user's client reported them. */ | ||
| 150 | profiles: Schema.optional(Schema.Array(PreviewAutomationProfile)), | ||
| 151 | /** Server hosts: the profile preview_open uses when given none. */ | ||
| 152 | defaultProfileId: Schema.optional(BrowserProfileId), | ||
| 128 | }); | 153 | }); |
| 129 | export type PreviewAutomationStatus = typeof PreviewAutomationStatus.Type; | 154 | export type PreviewAutomationStatus = typeof PreviewAutomationStatus.Type; |
| 130 | 155 | ||
| ... | @@ -162,6 +187,11 @@ export const PreviewAutomationOpenInput = Schema.Struct({ | ... | @@ -162,6 +187,11 @@ export const PreviewAutomationOpenInput = Schema.Struct({ |
| 162 | "Reuse tabId when supplied, otherwise this agent session's current tab. Defaults to true; set false to create a new tab.", | 187 | "Reuse tabId when supplied, otherwise this agent session's current tab. Defaults to true; set false to create a new tab.", |
| 163 | }), | 188 | }), |
| 164 | ), | 189 | ), |
| 190 | profileId: Schema.optional( | ||
| 191 | TrimmedNonEmptyString.check(Schema.isMaxLength(64)).annotate({ | ||
| 192 | description: PROFILE_GUIDANCE, | ||
| 193 | }), | ||
| 194 | ).annotate({ description: PROFILE_GUIDANCE }), | ||
| 165 | }) | 195 | }) |
| 166 | .check( | 196 | .check( |
| 167 | Schema.makeFilter( | 197 | Schema.makeFilter( |
packages/contracts/src/rpc.ts+8| ... | @@ -256,6 +256,7 @@ import { | ... | @@ -256,6 +256,7 @@ import { |
| 256 | PreviewListResult, | 256 | PreviewListResult, |
| 257 | PreviewClearProfileError, | 257 | PreviewClearProfileError, |
| 258 | PreviewClearProfileInput, | 258 | PreviewClearProfileInput, |
| 259 | PreviewReportProfilesInput, | ||
| 259 | PreviewNavigateInput, | 260 | PreviewNavigateInput, |
| 260 | PreviewOpenInput, | 261 | PreviewOpenInput, |
| 261 | PreviewRefreshInput, | 262 | PreviewRefreshInput, |
| ... | @@ -444,6 +445,7 @@ export const WS_METHODS = { | ... | @@ -444,6 +445,7 @@ export const WS_METHODS = { |
| 444 | previewClose: "preview.close", | 445 | previewClose: "preview.close", |
| 445 | previewList: "preview.list", | 446 | previewList: "preview.list", |
| 446 | previewClearProfile: "preview.clearProfile", | 447 | previewClearProfile: "preview.clearProfile", |
| 448 | previewReportProfiles: "preview.reportProfiles", | ||
| 447 | previewReportStatus: "preview.reportStatus", | 449 | previewReportStatus: "preview.reportStatus", |
| 448 | 450 | ||
| 449 | // Device methods | 451 | // Device methods |
| ... | @@ -1475,6 +1477,11 @@ const WsPreviewClearProfileRpc = Rpc.make(WS_METHODS.previewClearProfile, { | ... | @@ -1475,6 +1477,11 @@ const WsPreviewClearProfileRpc = Rpc.make(WS_METHODS.previewClearProfile, { |
| 1475 | error: Schema.Union([PreviewClearProfileError, EnvironmentAuthorizationError]), | 1477 | error: Schema.Union([PreviewClearProfileError, EnvironmentAuthorizationError]), |
| 1476 | }); | 1478 | }); |
| 1477 | 1479 | ||
| 1480 | const WsPreviewReportProfilesRpc = Rpc.make(WS_METHODS.previewReportProfiles, { | ||
| 1481 | payload: PreviewReportProfilesInput, | ||
| 1482 | error: EnvironmentAuthorizationError, | ||
| 1483 | }); | ||
| 1484 | |||
| 1478 | const WsPreviewReportStatusRpc = Rpc.make(WS_METHODS.previewReportStatus, { | 1485 | const WsPreviewReportStatusRpc = Rpc.make(WS_METHODS.previewReportStatus, { |
| 1479 | payload: PreviewReportStatusInput, | 1486 | payload: PreviewReportStatusInput, |
| 1480 | error: Schema.Union([PreviewError, EnvironmentAuthorizationError]), | 1487 | error: Schema.Union([PreviewError, EnvironmentAuthorizationError]), |
| ... | @@ -1955,6 +1962,7 @@ export const WsRpcGroup = RpcGroup.make( | ... | @@ -1955,6 +1962,7 @@ export const WsRpcGroup = RpcGroup.make( |
| 1955 | WsPreviewCloseRpc, | 1962 | WsPreviewCloseRpc, |
| 1956 | WsPreviewListRpc, | 1963 | WsPreviewListRpc, |
| 1957 | WsPreviewClearProfileRpc, | 1964 | WsPreviewClearProfileRpc, |
| 1965 | WsPreviewReportProfilesRpc, | ||
| 1958 | WsPreviewReportStatusRpc, | 1966 | WsPreviewReportStatusRpc, |
| 1959 | WsSubscribePreviewEventsRpc, | 1967 | WsSubscribePreviewEventsRpc, |
| 1960 | WsSubscribeDiscoveredLocalServersRpc, | 1968 | WsSubscribeDiscoveredLocalServersRpc, |
packages/provider-core/src/server/orchestrationInstructions.ts+6-1| ... | @@ -44,7 +44,12 @@ You are running inside T3 Code. The \`t3-code\` MCP server is the product-native | ... | @@ -44,7 +44,12 @@ You are running inside T3 Code. The \`t3-code\` MCP server is the product-native |
| 44 | 44 | ||
| 45 | For browser work, first call \`preview_status\`. If no automation-capable preview is attached, call \`preview_open\` before concluding that the browser is unavailable. Then use \`preview_navigate\`, \`preview_snapshot\`, and the focused interaction tools. Prefer snapshot-provided locators over coordinates. | 45 | For browser work, first call \`preview_status\`. If no automation-capable preview is attached, call \`preview_open\` before concluding that the browser is unavailable. Then use \`preview_navigate\`, \`preview_snapshot\`, and the focused interaction tools. Prefer snapshot-provided locators over coordinates. |
| 46 | 46 | ||
| 47 | Do not switch to global browser skills, Chrome, Node REPL browser automation, standalone Playwright, or agent-browser merely because the preview is initially closed or a first call fails. Use an alternative browser system only when the T3 preview tools are absent, the user explicitly requests another browser, or \`preview_open\` returns an explicit unsupported/unavailable error. A failed T3 preview tool call should be inspected and retried with corrected arguments when the error is actionable. | 47 | \`preview_status\` lists every browser tab in this thread, including tabs the user opened. When the user asks about "this page" or a page they have open, read their tab: pass its \`tabId\` to \`preview_snapshot\` or \`preview_wait_for\`, or omit \`tabId\` when you have no tab of your own. You may act on the user's tab, including \`preview_evaluate\`, only while its owner is \`unclaimed\`; while it is \`human\`, the user is driving, so read it with \`preview_snapshot\` or open your own tab. To use a browser profile (a set of saved logins), pass \`profileId\` from \`preview_status\` profiles to \`preview_open\`. |
| 48 | |||
| 49 | Do not switch to global browser skills, Chrome, Node REPL browser automation, standalone Playwright, or agent-browser merely because the preview is initially closed or a first call fails. Inspect a failed preview call and retry with corrected arguments when the error is actionable. Use another browser system when: | ||
| 50 | - the T3 preview tools are absent, or \`preview_open\` returns an explicit unsupported/unavailable error; | ||
| 51 | - the user asks for another browser, or invokes a skill or documented repository workflow that names one; follow that workflow and report any prerequisite it is missing; | ||
| 52 | - preview calls on an open tab have failed twice on the same step (timeouts, \`chrome-error://\` pages, a different client answering). Quote the raw error and switch without asking the user which browser to use. | ||
| 48 | `; | 53 | `; |
| 49 | 54 | ||
| 50 | const T3_CODE_ACP_DEFAULT_MODE_INSTRUCTIONS = `## T3 Code interaction mode: Default | 55 | const T3_CODE_ACP_DEFAULT_MODE_INSTRUCTIONS = `## T3 Code interaction mode: Default |