authorgravatar for maria@kuuro.netmaria <maria@kuuro.net> 2026-09-08 00:34:43-03:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-08 00:34:43-03:00
log9e37f0c291974d084a59d7b9f165a1cb056e6043
treed6b52b8e251eeef060bba9d2e2cf49a684e8f27a
parent15193df9fb915837f7215909a7d741789db0a273
signature Signed by PGP key B5690EEEBB952194

fix(preview): transfer recordings to the agent environment (#10572)

11 files changed, 517 insertions(+), 17 deletions(-)

apps/server/src/mcp/PreviewAutomationBroker.test.ts+43
...@@ -404,6 +404,49 @@ it.effect("classifies a remote non-editable target without collapsing it to exec...@@ -404,6 +404,49 @@ it.effect("classifies a remote non-editable target without collapsing it to exec
404 );404 );
405});405});
406406
407it.effect.each([
408 "PreviewAutomationRecordingTransferError",
409 "PreviewAutomationRecordingDesktopUpdateRequiredError",
410 "PreviewAutomationRecordingTooLargeError",
411 "PreviewAutomationRecordingDeadlineExpiredError",
412] as const)("preserves recording failure %s", (tag) =>
413 Effect.scoped(
414 Effect.gen(function* () {
415 const broker = yield* makeBroker;
416 const remoteError = {
417 _tag: tag,
418 message: "remote recording details",
419 detail: { reason: "untrusted-reason", threadId: "untrusted-thread" },
420 };
421 const requests = requestsFrom(yield* broker.connect(makeHost()));
422 yield* Stream.runForEach(requests, (request) =>
423 broker.respond({
424 clientId: "client-1",
425 connectionId: request.connectionId,
426 requestId: request.requestId,
427 ok: false,
428 error: remoteError,
429 }),
430 ).pipe(Effect.forkScoped);
431 yield* Effect.yieldNow;
432 const error = yield* broker
433 .invoke<void>({
434 scope,
435 operation: "recordingStop",
436 input: {},
437 })
438 .pipe(Effect.flip);
439 expect(error).toMatchObject({
440 _tag: tag,
441 threadId: scope.threadId,
442 });
443 expect(error.cause).toBe(remoteError);
444 expect(error.message).toContain("remains on the desktop");
445 expect(error.message).not.toContain("remote recording details");
446 }),
447 ),
448);
449
407it.effect("distinguishes malformed remote failures", () =>450it.effect("distinguishes malformed remote failures", () =>
408 Effect.scoped(451 Effect.scoped(
409 Effect.gen(function* () {452 Effect.gen(function* () {
apps/server/src/mcp/PreviewAutomationBroker.ts+24
...@@ -7,6 +7,10 @@ import {...@@ -7,6 +7,10 @@ import {
7 PreviewAutomationMalformedResponseError,7 PreviewAutomationMalformedResponseError,
8 PreviewAutomationNoAvailableHostError,8 PreviewAutomationNoAvailableHostError,
9 PreviewAutomationRemoteUnavailableError,9 PreviewAutomationRemoteUnavailableError,
10 PreviewAutomationRecordingTransferError,
11 PreviewAutomationRecordingDesktopUpdateRequiredError,
12 PreviewAutomationRecordingTooLargeError,
13 PreviewAutomationRecordingDeadlineExpiredError,
10 PreviewAutomationRequestQueueClosedError,14 PreviewAutomationRequestQueueClosedError,
11 PreviewAutomationResultTooLargeError,15 PreviewAutomationResultTooLargeError,
12 PreviewAutomationTabNotFoundError,16 PreviewAutomationTabNotFoundError,
...@@ -194,6 +198,26 @@ const classifyResponseError = (...@@ -194,6 +198,26 @@ const classifyResponseError = (
194 cause: error,198 cause: error,
195 };199 };
196 switch (error._tag) {200 switch (error._tag) {
201 case "PreviewAutomationRecordingDesktopUpdateRequiredError":
202 return new PreviewAutomationRecordingDesktopUpdateRequiredError({
203 threadId: context.threadId,
204 cause: error,
205 });
206 case "PreviewAutomationRecordingTooLargeError":
207 return new PreviewAutomationRecordingTooLargeError({
208 threadId: context.threadId,
209 cause: error,
210 });
211 case "PreviewAutomationRecordingDeadlineExpiredError":
212 return new PreviewAutomationRecordingDeadlineExpiredError({
213 threadId: context.threadId,
214 cause: error,
215 });
216 case "PreviewAutomationRecordingTransferError":
217 return new PreviewAutomationRecordingTransferError({
218 threadId: context.threadId,
219 cause: error,
220 });
197 case "PreviewAutomationNoAvailableHostError":221 case "PreviewAutomationNoAvailableHostError":
198 return new PreviewAutomationNoAvailableHostError({222 return new PreviewAutomationNoAvailableHostError({
199 ...context,223 ...context,
apps/server/src/mcp/toolkits/preview/handlers.test.ts+125-2
...@@ -1,6 +1,17 @@...@@ -1,6 +1,17 @@
1import { describe, expect, it } from "vite-plus/test";1import { describe, expect, it } from "@effect/vitest";
2import * as NodeServices from "@effect/platform-node/NodeServices";
3import { ThreadId } from "@t3tools/contracts";
4import * as Effect from "effect/Effect";
5import * as FileSystem from "effect/FileSystem";
6import * as Layer from "effect/Layer";
7import * as Path from "effect/Path";
28
3import { normalizePreviewOpenInput } from "./handlers.ts";9import {
10 createPendingAttachmentId,
11 parseThreadSegmentFromAttachmentId,
12} from "../../../attachmentStore.ts";
13import * as ServerConfig from "../../../config.ts";
14import { claimPreviewRecording, normalizePreviewOpenInput } from "./handlers.ts";
415
5describe("normalizePreviewOpenInput", () => {16describe("normalizePreviewOpenInput", () => {
6 it("leaves an unstated visibility for the client preference to decide", () => {17 it("leaves an unstated visibility for the client preference to decide", () => {
...@@ -30,3 +41,115 @@ describe("normalizePreviewOpenInput", () => {...@@ -30,3 +41,115 @@ describe("normalizePreviewOpenInput", () => {
30 });41 });
31 });42 });
32});43});
44
45describe("claimPreviewRecording", () => {
46 it.effect("overlapping and repeated claims return the same retained recording", () =>
47 Effect.gen(function* () {
48 const config = yield* ServerConfig.ServerConfig;
49 const fileSystem = yield* FileSystem.FileSystem;
50 const path = yield* Path.Path;
51 const uploadedAttachmentId = createPendingAttachmentId(".webm");
52 const pendingPath = path.join(config.attachmentsDir, `${uploadedAttachmentId}.webm`);
53 yield* fileSystem.makeDirectory(config.attachmentsDir, { recursive: true });
54 yield* fileSystem.writeFileString(pendingPath, "video!");
55 const response = {
56 id: "desktop-recording",
57 tabId: "tab-1",
58 path: "/desktop/recording.webm",
59 mimeType: "video/webm",
60 sizeBytes: 6,
61 createdAt: "2026-09-07T00:00:00.000Z",
62 uploadedAttachmentId,
63 };
64 const claim = claimPreviewRecording(ThreadId.make("thread-1"), response);
65 const [first, second] = yield* Effect.all([claim, claim], { concurrency: "unbounded" });
66 expect(first).toEqual(second);
67 expect(yield* claim).toEqual(first);
68 expect(yield* fileSystem.readFileString(first.path)).toBe("video!");
69 expect(yield* fileSystem.exists(pendingPath)).toBe(false);
70 const wrongThread = yield* claimPreviewRecording(ThreadId.make("thread-2"), response).pipe(
71 Effect.result,
72 );
73 expect(wrongThread._tag).toBe("Failure");
74 const wrongPath = yield* claimPreviewRecording(ThreadId.make("thread-1"), {
75 ...response,
76 uploadedAttachmentId: `../${uploadedAttachmentId}`,
77 }).pipe(Effect.result);
78 expect(wrongPath._tag).toBe("Failure");
79 }).pipe(
80 Effect.provide(
81 ServerConfig.layerTest(process.cwd(), { prefix: "t3-preview-recording-" }).pipe(
82 Layer.provideMerge(NodeServices.layer),
83 ),
84 ),
85 ),
86 );
87
88 it.effect.each([6, 5])(
89 "claims only a complete uploaded recording (reported bytes: %s)",
90 (sizeBytes) =>
91 Effect.gen(function* () {
92 const config = yield* ServerConfig.ServerConfig;
93 const fileSystem = yield* FileSystem.FileSystem;
94 const path = yield* Path.Path;
95 const uploadedAttachmentId = createPendingAttachmentId(".webm");
96 const pendingPath = path.join(config.attachmentsDir, `${uploadedAttachmentId}.webm`);
97 yield* fileSystem.makeDirectory(config.attachmentsDir, { recursive: true });
98 yield* fileSystem.writeFileString(pendingPath, "video!");
99 const response = {
100 id: "desktop-recording",
101 tabId: "tab-1",
102 path: "/desktop/recording.webm",
103 mimeType: "video/webm",
104 sizeBytes,
105 createdAt: "2026-09-07T00:00:00.000Z",
106 uploadedAttachmentId,
107 };
108 const result = yield* claimPreviewRecording(ThreadId.make("thread-1"), response).pipe(
109 Effect.result,
110 );
111 if (sizeBytes === 6) {
112 expect(result._tag).toBe("Success");
113 if (result._tag !== "Success") return;
114 expect(result.success.path).not.toBe(response.path);
115 expect(parseThreadSegmentFromAttachmentId(result.success.id)).toBe("thread-1");
116 expect(yield* fileSystem.readFileString(result.success.path)).toBe("video!");
117 expect(yield* fileSystem.exists(pendingPath)).toBe(false);
118 } else {
119 expect(result._tag).toBe("Failure");
120 if (result._tag !== "Failure") return;
121 expect(result.failure._tag).toBe("PreviewAutomationRecordingTransferError");
122 expect(yield* fileSystem.exists(pendingPath)).toBe(true);
123 }
124 }).pipe(
125 Effect.provide(
126 ServerConfig.layerTest(process.cwd(), { prefix: "t3-preview-recording-" }).pipe(
127 Layer.provideMerge(NodeServices.layer),
128 ),
129 ),
130 ),
131 );
132
133 it.effect("reports an older desktop without returning its inaccessible path", () =>
134 Effect.gen(function* () {
135 const result = yield* claimPreviewRecording(ThreadId.make("thread-1"), {
136 id: "desktop-recording",
137 tabId: "tab-1",
138 path: "/desktop/recording.webm",
139 mimeType: "video/webm",
140 sizeBytes: 6,
141 createdAt: "2026-09-07T00:00:00.000Z",
142 }).pipe(Effect.result);
143 expect(result._tag).toBe("Failure");
144 if (result._tag !== "Failure") return;
145 expect(result.failure._tag).toBe("PreviewAutomationRecordingDesktopUpdateRequiredError");
146 expect(result.failure.message).toContain("Update the desktop app");
147 }).pipe(
148 Effect.provide(
149 ServerConfig.layerTest(process.cwd(), { prefix: "t3-preview-recording-" }).pipe(
150 Layer.provideMerge(NodeServices.layer),
151 ),
152 ),
153 ),
154 );
155});
apps/server/src/mcp/toolkits/preview/handlers.ts+106-10
...@@ -1,16 +1,31 @@...@@ -1,16 +1,31 @@
1import * as Effect from "effect/Effect";1import * as Effect from "effect/Effect";
2import type {2import * as FileSystem from "effect/FileSystem";
3 PreviewAutomationOperation,3import * as Schema from "effect/Schema";
4 PreviewAutomationOpenInput,4import {
5 PROVIDER_SEND_TURN_MAX_FILE_BYTES,
6 PREVIEW_RECORDING_STOP_TIMEOUT_MS,
7 PreviewAutomationRecordingTransferError,
8 PreviewAutomationRecordingDesktopUpdateRequiredError,
5 PreviewAutomationRecordingArtifact,9 PreviewAutomationRecordingArtifact,
6 PreviewAutomationRecordingStatus,10 type ThreadId,
7 PreviewAutomationResizeResult,11 type PreviewAutomationOperation,
8 PreviewAutomationSetColorSchemeResult,12 type PreviewAutomationOpenInput,
9 PreviewAutomationSnapshot,13 type PreviewAutomationRecordingStatus,
10 PreviewAutomationStatus,14 type PreviewAutomationResizeResult,
11 PreviewTabId,15 type PreviewAutomationSetColorSchemeResult,
16 type PreviewAutomationSnapshot,
17 type PreviewAutomationStatus,
18 type PreviewTabId,
12} from "@t3tools/contracts";19} from "@t3tools/contracts";
1320
21import {
22 parseAttachmentUuid,
23 parseAttachmentFileExtension,
24 PENDING_ATTACHMENT_THREAD_SEGMENT,
25 toSafeThreadAttachmentSegment,
26} from "../../../attachmentStore.ts";
27import { resolveAttachmentRelativePath } from "../../../attachmentPaths.ts";
28import * as ServerConfig from "../../../config.ts";
14import * as McpInvocationContext from "../../McpInvocationContext.ts";29import * as McpInvocationContext from "../../McpInvocationContext.ts";
15import * as PreviewAutomationBroker from "../../PreviewAutomationBroker.ts";30import * as PreviewAutomationBroker from "../../PreviewAutomationBroker.ts";
16import { PreviewSnapshotToolkit, PreviewStandardToolkit, PreviewToolkit } from "./tools.ts";31import { PreviewSnapshotToolkit, PreviewStandardToolkit, PreviewToolkit } from "./tools.ts";
...@@ -67,6 +82,79 @@ const invokeTargeted = <A>(...@@ -67,6 +82,79 @@ const invokeTargeted = <A>(
67 return invoke<A>(operation, operationInput, timeoutMs, tabId);82 return invoke<A>(operation, operationInput, timeoutMs, tabId);
68};83};
6984
85const UploadedRecordingArtifact = Schema.Struct({
86 ...PreviewAutomationRecordingArtifact.fields,
87 uploadedAttachmentId: Schema.optional(Schema.String),
88});
89const decodeUploadedRecordingArtifact = Schema.decodeUnknownEffect(UploadedRecordingArtifact);
90
91export const claimPreviewRecording = Effect.fn("PreviewToolkit.claimRecording")(function* (
92 threadId: ThreadId,
93 response: unknown,
94) {
95 const artifact = yield* decodeUploadedRecordingArtifact(response).pipe(
96 Effect.mapError(
97 (cause) =>
98 new PreviewAutomationRecordingTransferError({
99 threadId,
100 cause,
101 }),
102 ),
103 );
104 if (!artifact.uploadedAttachmentId) {
105 return yield* new PreviewAutomationRecordingDesktopUpdateRequiredError({ threadId });
106 }
107 const config = yield* ServerConfig.ServerConfig;
108 const uuid = parseAttachmentUuid(artifact.uploadedAttachmentId);
109 const extension = parseAttachmentFileExtension(artifact.uploadedAttachmentId);
110 const threadSegment = toSafeThreadAttachmentSegment(threadId);
111 const pendingId = `${PENDING_ATTACHMENT_THREAD_SEGMENT}-${uuid}-${extension}`;
112 if (!uuid || !extension || !threadSegment || artifact.uploadedAttachmentId !== pendingId) {
113 return yield* new PreviewAutomationRecordingTransferError({
114 threadId,
115 });
116 }
117 // The same completed upload can be returned to overlapping stop requests.
118 const finalId = `${threadSegment}-${uuid}-${extension}`;
119 const currentPath = resolveAttachmentRelativePath({
120 attachmentsDir: config.attachmentsDir,
121 relativePath: `${pendingId}.${extension}`,
122 });
123 const finalPath = resolveAttachmentRelativePath({
124 attachmentsDir: config.attachmentsDir,
125 relativePath: `${finalId}.${extension}`,
126 });
127 if (!currentPath || !finalPath) {
128 return yield* new PreviewAutomationRecordingTransferError({ threadId });
129 }
130 const fileSystem = yield* FileSystem.FileSystem;
131 const validateFile = (filePath: string) =>
132 fileSystem.stat(filePath).pipe(
133 Effect.filterOrFail(
134 (stat) =>
135 stat.type === "File" &&
136 Number(stat.size) === artifact.sizeBytes &&
137 artifact.sizeBytes > 0 &&
138 artifact.sizeBytes <= PROVIDER_SEND_TURN_MAX_FILE_BYTES,
139 () => new PreviewAutomationRecordingTransferError({ threadId }),
140 ),
141 );
142 yield* Effect.gen(function* () {
143 yield* validateFile(currentPath);
144 yield* fileSystem.rename(currentPath, finalPath);
145 }).pipe(
146 // Another stop may already have claimed this exact upload for this thread.
147 Effect.catch((cause) =>
148 cause._tag !== "PreviewAutomationRecordingTransferError" && cause.reason._tag === "NotFound"
149 ? validateFile(finalPath)
150 : Effect.fail(cause),
151 ),
152 Effect.mapError((cause) => new PreviewAutomationRecordingTransferError({ threadId, cause })),
153 );
154 const { uploadedAttachmentId: _uploadedAttachmentId, ...recording } = artifact;
155 return { ...recording, id: finalId, path: finalPath };
156});
157
70const handlers = {158const handlers = {
71 preview_status: (input) => invokeTargeted<PreviewAutomationStatus>("status", input ?? {}),159 preview_status: (input) => invokeTargeted<PreviewAutomationStatus>("status", input ?? {}),
72 preview_open: (input) =>160 preview_open: (input) =>
...@@ -94,7 +182,15 @@ const handlers = {...@@ -94,7 +182,15 @@ const handlers = {
94 preview_recording_start: (input) =>182 preview_recording_start: (input) =>
95 invokeTargeted<PreviewAutomationRecordingStatus>("recordingStart", input ?? {}),183 invokeTargeted<PreviewAutomationRecordingStatus>("recordingStart", input ?? {}),
96 preview_recording_stop: (input) =>184 preview_recording_stop: (input) =>
97 invokeTargeted<PreviewAutomationRecordingArtifact>("recordingStop", input ?? {}),185 Effect.gen(function* () {
186 const scope = yield* McpInvocationContext.requireMcpCapability("preview");
187 const response = yield* invokeTargeted<unknown>(
188 "recordingStop",
189 { ...input, transferToEnvironment: true },
190 PREVIEW_RECORDING_STOP_TIMEOUT_MS,
191 );
192 return yield* claimPreviewRecording(scope.threadId, response);
193 }),
98} satisfies Parameters<typeof PreviewToolkit.toLayer>[0];194} satisfies Parameters<typeof PreviewToolkit.toLayer>[0];
99195
100const { preview_snapshot, ...standardHandlers } = handlers;196const { preview_snapshot, ...standardHandlers } = handlers;
apps/server/src/mcp/toolkits/preview/tools.ts+4-2
...@@ -19,10 +19,12 @@ import {...@@ -19,10 +19,12 @@ import {
19 PreviewAutomationWaitForInput,19 PreviewAutomationWaitForInput,
20} from "@t3tools/contracts";20} from "@t3tools/contracts";
21import * as Schema from "effect/Schema";21import * as Schema from "effect/Schema";
22import * as FileSystem from "effect/FileSystem";
22import { Tool, Toolkit } from "effect/unstable/ai";23import { Tool, Toolkit } from "effect/unstable/ai";
2324
24import * as McpInvocationContext from "../../McpInvocationContext.ts";25import * as McpInvocationContext from "../../McpInvocationContext.ts";
25import * as PreviewAutomationBroker from "../../PreviewAutomationBroker.ts";26import * as PreviewAutomationBroker from "../../PreviewAutomationBroker.ts";
27import * as ServerConfig from "../../../config.ts";
2628
27const dependencies = [29const dependencies = [
28 McpInvocationContext.McpInvocationContext,30 McpInvocationContext.McpInvocationContext,
...@@ -207,11 +209,11 @@ export const PreviewRecordingStartTool = safeBrowserTool(...@@ -207,11 +209,11 @@ export const PreviewRecordingStartTool = safeBrowserTool(
207export const PreviewRecordingStopTool = safeBrowserTool(209export const PreviewRecordingStopTool = safeBrowserTool(
208 Tool.make("preview_recording_stop", {210 Tool.make("preview_recording_stop", {
209 description:211 description:
210 "Stop recording the collaborative browser tab selected by tabId, or this agent session's current tab when omitted, and save it as a local evidence artifact.",212 "Stop recording the collaborative browser tab selected by tabId, or this agent session's current tab when omitted, and transfer the compressed recording once (up to 50 MiB) to an evidence file readable in this agent's environment. Returns its environment-local path after transfer succeeds.",
211 parameters: PreviewAutomationTabTargetInput,213 parameters: PreviewAutomationTabTargetInput,
212 success: PreviewAutomationRecordingArtifact,214 success: PreviewAutomationRecordingArtifact,
213 failure: PreviewAutomationError,215 failure: PreviewAutomationError,
214 dependencies,216 dependencies: [...dependencies, FileSystem.FileSystem, ServerConfig.ServerConfig],
215 }).annotate(Tool.Title, "Stop browser recording"),217 }).annotate(Tool.Title, "Stop browser recording"),
216);218);
217219
apps/web/src/browser/browserRecording.test.ts+43
...@@ -78,6 +78,7 @@ import {...@@ -78,6 +78,7 @@ import {
78 readActiveBrowserRecordingTargets,78 readActiveBrowserRecordingTargets,
79 startBrowserRecording,79 startBrowserRecording,
80 stopBrowserRecording,80 stopBrowserRecording,
81 stopBrowserRecordingForUpload,
81} from "./browserRecording";82} from "./browserRecording";
82import { useBrowserSurfaceStore } from "./browserSurfaceStore";83import { useBrowserSurfaceStore } from "./browserSurfaceStore";
83import { previewRuntimeTabId } from "./previewRuntimeTabId";84import { previewRuntimeTabId } from "./previewRuntimeTabId";
...@@ -180,6 +181,48 @@ describe("browser recording", () => {...@@ -180,6 +181,48 @@ describe("browser recording", () => {
180 await stopBrowserRecording("automation-recording-tab");181 await stopBrowserRecording("automation-recording-tab");
181 });182 });
182183
184 it("saves locally and releases capture before transferring the encoded recording once", async () => {
185 const stopTrack = vi.fn();
186 getDisplayMedia.mockResolvedValue({
187 getVideoTracks: () => [],
188 getTracks: () => [{ stop: stopTrack }],
189 });
190 await startBrowserRecording("transfer-tab");
191 let finishUpload!: () => void;
192 const uploaded = new Promise<void>((resolve) => {
193 finishUpload = resolve;
194 });
195 const transfer = vi.fn(async (artifact, blob: Blob) => {
196 expect(save).toHaveBeenCalledOnce();
197 expect(stopTrack).toHaveBeenCalled();
198 expect(artifact.path).toBe("/tmp/recording-test.webm");
199 expect(blob.type).toBe("video/webm;codecs=vp9");
200 await uploaded;
201 return "uploaded-recording";
202 });
203 const localStop = stopBrowserRecording("transfer-tab");
204 const firstStop = stopBrowserRecordingForUpload("transfer-tab", transfer);
205 const secondStop = stopBrowserRecordingForUpload("transfer-tab", transfer);
206 finishUpload();
207 expect(await firstStop).toEqual(await secondStop);
208 expect((await firstStop)?.uploadedAttachmentId).toBe("uploaded-recording");
209 expect((await localStop)?.path).toBe("/tmp/recording-test.webm");
210 expect(transfer).toHaveBeenCalledOnce();
211 });
212
213 it("keeps the saved desktop file and releases the recording when transfer fails", async () => {
214 await startBrowserRecording("failed-transfer-tab");
215 await expect(
216 stopBrowserRecordingForUpload("failed-transfer-tab", async () => {
217 throw new Error("Connection interrupted");
218 }),
219 ).rejects.toThrow("Connection interrupted");
220 expect(save).toHaveBeenCalledOnce();
221 expect(readActiveBrowserRecordingTabIds().has("failed-transfer-tab")).toBe(false);
222 await startBrowserRecording("failed-transfer-tab");
223 await stopBrowserRecording("failed-transfer-tab");
224 });
225
183 it("paints and holds a hidden browser surface for the recording lifetime", async () => {226 it("paints and holds a hidden browser surface for the recording lifetime", async () => {
184 startScreencast.mockImplementationOnce(async (tabId: string) => {227 startScreencast.mockImplementationOnce(async (tabId: string) => {
185 expect(animationFrameCount).toBe(2);228 expect(animationFrameCount).toBe(2);
apps/web/src/browser/browserRecording.ts+16
...@@ -123,6 +123,8 @@ interface ActiveRecording {...@@ -123,6 +123,8 @@ interface ActiveRecording {
123 releaseSurfaceActivity: (() => void) | null;123 releaseSurfaceActivity: (() => void) | null;
124 stream: MediaStream | null;124 stream: MediaStream | null;
125 recorder: MediaRecorder | null;125 recorder: MediaRecorder | null;
126 savedBlob?: Blob;
127 uploadPromise?: Promise<string>;
126 lifecycle: BrowserRecordingLifecycle;128 lifecycle: BrowserRecordingLifecycle;
127}129}
128130
...@@ -708,6 +710,7 @@ const finalizeBrowserRecording = async (...@@ -708,6 +710,7 @@ const finalizeBrowserRecording = async (
708 mimeType,710 mimeType,
709 new Uint8Array(await blob.arrayBuffer()),711 new Uint8Array(await blob.arrayBuffer()),
710 );712 );
713 recording.savedBlob = blob;
711 result = { _tag: "Success", artifact };714 result = { _tag: "Success", artifact };
712 } catch (cause) {715 } catch (cause) {
713 throw new BrowserRecordingOperationError({716 throw new BrowserRecordingOperationError({
...@@ -814,3 +817,16 @@ export function stopBrowserRecording(...@@ -814,3 +817,16 @@ export function stopBrowserRecording(
814 recording.lifecycle = { phase: "stopping", stopPromise };817 recording.lifecycle = { phase: "stopping", stopPromise };
815 return stopPromise;818 return stopPromise;
816}819}
820
821/** Joins local stops and shares one upload among concurrent automation requests. */
822export async function stopBrowserRecordingForUpload(
823 tabId: string,
824 upload: (artifact: DesktopPreviewRecordingArtifact, blob: Blob) => Promise<string>,
825): Promise<(DesktopPreviewRecordingArtifact & { uploadedAttachmentId: string }) | null> {
826 const recording = activeRecordings.get(tabId);
827 if (!recording) return null;
828 const artifact = await stopBrowserRecording(tabId);
829 if (!artifact || !recording.savedBlob) return null;
830 recording.uploadPromise ??= upload(artifact, recording.savedBlob);
831 return { ...artifact, uploadedAttachmentId: await recording.uploadPromise };
832}
apps/web/src/browser/browserRecordingUpload.ts created+84
...@@ -0,0 +1,84 @@
1import {
2 PROVIDER_SEND_TURN_MAX_FILE_BYTES,
3 PreviewAutomationRecordingTransferError,
4 PreviewAutomationRecordingTooLargeError,
5 PreviewAutomationRecordingDeadlineExpiredError,
6 type DesktopPreviewRecordingArtifact,
7 type ScopedThreadRef,
8} from "@t3tools/contracts";
9import { resolveAssetUrl } from "@t3tools/client-runtime/state/assets";
10import {
11 deletePendingAttachmentUpload,
12 runAttachmentUploadCycle,
13} from "@t3tools/client-runtime/state/attachments";
14
15import { appAtomRegistry } from "~/rpc/atomRegistry";
16import { attachmentEnvironment } from "~/state/attachments";
17import { readPreparedConnection } from "~/state/session";
18
19/** Sends the finished encoded file once; capture frames never cross the environment connection. */
20export async function uploadBrowserRecording(
21 { environmentId, threadId }: ScopedThreadRef,
22 artifact: DesktopPreviewRecordingArtifact,
23 blob: Blob,
24 deadlineMs: number,
25): Promise<string> {
26 if (blob.size > PROVIDER_SEND_TURN_MAX_FILE_BYTES) {
27 throw new PreviewAutomationRecordingTooLargeError({ threadId });
28 }
29 const result = await runAttachmentUploadCycle({
30 registry: appAtomRegistry,
31 createUploadUrl: attachmentEnvironment.createUploadUrl,
32 remove: attachmentEnvironment.remove,
33 environmentId,
34 upload: {
35 type: "file",
36 name: artifact.path.split(/[\\/]/).at(-1) ?? artifact.id,
37 mimeType: artifact.mimeType,
38 sizeBytes: blob.size,
39 },
40 resolveUploadUrl: (relativeUrl) => {
41 const connection = readPreparedConnection(environmentId);
42 return connection ? resolveAssetUrl(connection.httpBaseUrl, relativeUrl) : null;
43 },
44 transport: (url) => {
45 const controller = new AbortController();
46 // Encoding, saving and minting consume the same request budget. Leave time to reply.
47 const remainingMs = deadlineMs - Date.now() - 1_000;
48 return {
49 abort: () => controller.abort(),
50 done:
51 remainingMs <= 0
52 ? Promise.reject(new Error("Recording transfer deadline expired."))
53 : fetch(url, {
54 method: "POST",
55 headers: { "Content-Type": artifact.mimeType },
56 body: blob,
57 signal: AbortSignal.any([controller.signal, AbortSignal.timeout(remainingMs)]),
58 }).then((response) => {
59 if (!response.ok)
60 throw new Error(`Recording upload rejected (${response.status}).`);
61 }),
62 };
63 },
64 });
65 if (result.status !== "uploaded") {
66 if (result.attachmentId) {
67 deletePendingAttachmentUpload({
68 registry: appAtomRegistry,
69 remove: attachmentEnvironment.remove,
70 environmentId,
71 attachmentId: result.attachmentId,
72 });
73 }
74 const cause = result.status === "failed" ? result.error : undefined;
75 if (Date.now() >= deadlineMs - 1_000) {
76 throw new PreviewAutomationRecordingDeadlineExpiredError({ threadId, cause });
77 }
78 throw new PreviewAutomationRecordingTransferError({
79 threadId,
80 cause,
81 });
82 }
83 return result.attachmentId;
84}
apps/web/src/components/preview/PreviewAutomationHosts.tsx+18-2
...@@ -35,8 +35,10 @@ import {...@@ -35,8 +35,10 @@ import {
35 readActiveBrowserRecordingTargets,35 readActiveBrowserRecordingTargets,
36 startBrowserRecording,36 startBrowserRecording,
37 stopBrowserRecording,37 stopBrowserRecording,
38 stopBrowserRecordingForUpload,
38} from "~/browser/browserRecording";39} from "~/browser/browserRecording";
39import { resolveBrowserRecordingStopTarget } from "~/browser/browserRecordingScope";40import { resolveBrowserRecordingStopTarget } from "~/browser/browserRecordingScope";
41import { uploadBrowserRecording } from "~/browser/browserRecordingUpload";
40import {42import {
41 acquireBrowserSurfaceActivity,43 acquireBrowserSurfaceActivity,
42 useBrowserSurfaceStore,44 useBrowserSurfaceStore,
...@@ -716,7 +718,18 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId })...@@ -716,7 +718,18 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId })
716 const stopRuntimeTabId =718 const stopRuntimeTabId =
717 activeRecordings.find((recording) => recording.serverTabId === stopTabId)719 activeRecordings.find((recording) => recording.serverTabId === stopTabId)
718 ?.runtimeTabId ?? null;720 ?.runtimeTabId ?? null;
719 const artifact = stopRuntimeTabId ? await stopBrowserRecording(stopRuntimeTabId) : null;721 const transferToEnvironment =
722 typeof request.input === "object" &&
723 request.input !== null &&
724 "transferToEnvironment" in request.input &&
725 request.input.transferToEnvironment === true;
726 const artifact = stopRuntimeTabId
727 ? transferToEnvironment
728 ? await stopBrowserRecordingForUpload(stopRuntimeTabId, (saved, blob) =>
729 uploadBrowserRecording(threadRef, saved, blob, hostDeadlineMs),
730 )
731 : await stopBrowserRecording(stopRuntimeTabId)
732 : null;
720 if (!artifact || !stopTabId) {733 if (!artifact || !stopTabId) {
721 return raisePreviewAutomationHostError(734 return raisePreviewAutomationHostError(
722 new PreviewAutomationRecordingNotActiveError({735 new PreviewAutomationRecordingNotActiveError({
...@@ -727,7 +740,10 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId })...@@ -727,7 +740,10 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId })
727 }),740 }),
728 );741 );
729 }742 }
730 return { ...artifact, tabId: stopTabId };743 return {
744 ...artifact,
745 tabId: stopTabId,
746 };
731 }747 }
732 }748 }
733 } catch (cause) {749 } catch (cause) {
apps/web/src/components/preview/previewAutomationErrors.ts+9-1
...@@ -2,6 +2,10 @@ import {...@@ -2,6 +2,10 @@ import {
2 EnvironmentId,2 EnvironmentId,
3 type PreviewAutomationHost,3 type PreviewAutomationHost,
4 PreviewAutomationOperation,4 PreviewAutomationOperation,
5 PreviewAutomationRecordingTransferError,
6 PreviewAutomationRecordingDesktopUpdateRequiredError,
7 PreviewAutomationRecordingTooLargeError,
8 PreviewAutomationRecordingDeadlineExpiredError,
5 type PreviewAutomationRequest,9 type PreviewAutomationRequest,
6 type PreviewAutomationResponse,10 type PreviewAutomationResponse,
7 PreviewTabId,11 PreviewTabId,
...@@ -206,6 +210,10 @@ export class PreviewAutomationOperationError extends Schema.TaggedErrorClass<Pre...@@ -206,6 +210,10 @@ export class PreviewAutomationOperationError extends Schema.TaggedErrorClass<Pre
206}210}
207211
208export const PreviewAutomationHostError = Schema.Union([212export const PreviewAutomationHostError = Schema.Union([
213 PreviewAutomationRecordingTransferError,
214 PreviewAutomationRecordingDesktopUpdateRequiredError,
215 PreviewAutomationRecordingTooLargeError,
216 PreviewAutomationRecordingDeadlineExpiredError,
209 PreviewAutomationOverlayTimeoutError,217 PreviewAutomationOverlayTimeoutError,
210 PreviewAutomationNavigationTimeoutError,218 PreviewAutomationNavigationTimeoutError,
211 PreviewAutomationViewportTimeoutError,219 PreviewAutomationViewportTimeoutError,
...@@ -228,7 +236,7 @@ export function serializePreviewAutomationHostError(...@@ -228,7 +236,7 @@ export function serializePreviewAutomationHostError(
228 ),236 ),
229 );237 );
230 return {238 return {
231 _tag: error.responseTag,239 _tag: "responseTag" in error ? error.responseTag : error._tag,
232 message: error.message,240 message: error.message,
233 ...(Object.keys(detail).length === 0 ? {} : { detail }),241 ...(Object.keys(detail).length === 0 ? {} : { detail }),
234 };242 };
packages/contracts/src/previewAutomation.ts+45
...@@ -553,6 +553,8 @@ export const PreviewAutomationRecordingStatus = Schema.Struct({...@@ -553,6 +553,8 @@ export const PreviewAutomationRecordingStatus = Schema.Struct({
553});553});
554export type PreviewAutomationRecordingStatus = typeof PreviewAutomationRecordingStatus.Type;554export type PreviewAutomationRecordingStatus = typeof PreviewAutomationRecordingStatus.Type;
555555
556export const PREVIEW_RECORDING_STOP_TIMEOUT_MS = 120_000;
557
556export const PreviewAutomationRecordingArtifact = Schema.Struct({558export const PreviewAutomationRecordingArtifact = Schema.Struct({
557 id: Schema.String,559 id: Schema.String,
558 tabId: PreviewTabId,560 tabId: PreviewTabId,
...@@ -856,7 +858,50 @@ export class PreviewAutomationMalformedResponseError extends Schema.TaggedErrorC...@@ -856,7 +858,50 @@ export class PreviewAutomationMalformedResponseError extends Schema.TaggedErrorC
856 }858 }
857}859}
858860
861export class PreviewAutomationRecordingTransferError extends Schema.TaggedErrorClass<PreviewAutomationRecordingTransferError>()(
862 "PreviewAutomationRecordingTransferError",
863 {
864 threadId: ThreadId,
865 cause: Schema.optional(Schema.Defect()),
866 },
867) {
868 override get message(): string {
869 return "Preview recording could not be saved to the agent environment. The saved copy remains on the desktop.";
870 }
871}
872
873export class PreviewAutomationRecordingDesktopUpdateRequiredError extends Schema.TaggedErrorClass<PreviewAutomationRecordingDesktopUpdateRequiredError>()(
874 "PreviewAutomationRecordingDesktopUpdateRequiredError",
875 { threadId: ThreadId, cause: Schema.optional(Schema.Defect()) },
876) {
877 override get message(): string {
878 return "Update the desktop app to transfer recordings. The recording remains on the desktop.";
879 }
880}
881
882export class PreviewAutomationRecordingTooLargeError extends Schema.TaggedErrorClass<PreviewAutomationRecordingTooLargeError>()(
883 "PreviewAutomationRecordingTooLargeError",
884 { threadId: ThreadId, cause: Schema.optional(Schema.Defect()) },
885) {
886 override get message(): string {
887 return "The recording exceeds 50 MiB. The saved copy remains on the desktop.";
888 }
889}
890
891export class PreviewAutomationRecordingDeadlineExpiredError extends Schema.TaggedErrorClass<PreviewAutomationRecordingDeadlineExpiredError>()(
892 "PreviewAutomationRecordingDeadlineExpiredError",
893 { threadId: ThreadId, cause: Schema.optional(Schema.Defect()) },
894) {
895 override get message(): string {
896 return "The recording transfer deadline expired. The saved copy remains on the desktop.";
897 }
898}
899
859export const PreviewAutomationError = Schema.Union([900export const PreviewAutomationError = Schema.Union([
901 PreviewAutomationRecordingTransferError,
902 PreviewAutomationRecordingDesktopUpdateRequiredError,
903 PreviewAutomationRecordingTooLargeError,
904 PreviewAutomationRecordingDeadlineExpiredError,
860 PreviewAutomationUnavailableError,905 PreviewAutomationUnavailableError,
861 PreviewAutomationNoAvailableHostError,906 PreviewAutomationNoAvailableHostError,
862 PreviewAutomationUnsupportedClientError,907 PreviewAutomationUnsupportedClientError,