1// @effect-diagnostics globalTimers:off globalFetch:off - This browser and WebView transport runs without an Effect runtime.
2import { type DeviceHubAccess, withDeviceHubQuery } from "../device/hubAccess.ts";
3import {
4 PREVIEW_STREAM_HOST_SETUP_CLOSE_CODE,
5 type PreviewStreamHostSetup,
6 type PreviewViewportSetting,
7} from "@t3tools/contracts";
8
9export const PREVIEW_STREAM_BASE_PATH = "/api/preview-stream";
10
11export interface PreviewStreamViewport {
12 readonly width: number;
13 readonly height: number;
14}
15
16export type PreviewStreamMouseButton = "none" | "left" | "middle" | "right";
17
18export interface PreviewStreamControl {
19 readonly canOperate: boolean;
20 readonly controller: "agent" | "you" | "another-viewer" | "unclaimed";
21 readonly generation: number;
22 readonly dialog: null | {
23 readonly type: string;
24 readonly message: string;
25 readonly defaultValue: string;
26 };
27}
28
29/** Status line shown above a viewer; `null` control means the socket is not connected. */
30export const previewStreamControlLabel = (control: PreviewStreamControl | null): string =>
31 !control
32 ? "Connecting..."
33 : !control.canOperate
34 ? "Read-only"
35 : control.controller === "you"
36 ? "You have control"
37 : control.controller === "agent"
38 ? "Agent has control"
39 : control.controller === "another-viewer"
40 ? "Another viewer has control"
41 : "Watching";
42
43const isPreviewStreamDialog = (value: unknown): value is PreviewStreamControl["dialog"] =>
44 value === null ||
45 (typeof value === "object" &&
46 "type" in value &&
47 typeof value.type === "string" &&
48 "message" in value &&
49 typeof value.message === "string" &&
50 "defaultValue" in value &&
51 typeof value.defaultValue === "string");
52
53/** Client-to-server messages. Coordinates are page CSS px. */
54export type PreviewStreamInput =
55 | { readonly type: "takeControl" }
56 | { readonly type: "releaseControl" }
57 | { readonly type: "dialog"; readonly accept: boolean; readonly promptText?: string }
58 | { readonly type: "viewport"; readonly setting: PreviewViewportSetting }
59 | {
60 readonly type: "mouse";
61 readonly action: "move" | "down" | "up";
62 readonly x: number;
63 readonly y: number;
64 readonly button: PreviewStreamMouseButton;
65 /** Pressed buttons: left 1, right 2, middle 4. */
66 readonly buttons: number;
67 readonly clickCount: number;
68 readonly modifiers: number;
69 }
70 | {
71 readonly type: "wheel";
72 readonly x: number;
73 readonly y: number;
74 readonly deltaX: number;
75 readonly deltaY: number;
76 readonly modifiers: number;
77 }
78 | {
79 readonly type: "key";
80 readonly action: "down" | "up";
81 readonly key: string;
82 readonly code: string;
83 /** Windows virtual key code (DOM `keyCode`); Enter, Backspace, and arrows need it. */
84 readonly keyCode?: number;
85 /** Only for printable keys without Ctrl or Meta. */
86 readonly text?: string;
87 readonly modifiers: number;
88 }
89 | { readonly type: "text"; readonly text: string }
90 | { readonly type: "resize"; readonly width: number; readonly height: number }
91 | { readonly type: "navigate"; readonly url: string }
92 | { readonly type: "history"; readonly delta: -1 | 1 }
93 | { readonly type: "reload"; readonly ignoreCache?: boolean }
94 /** Asks whether the page point takes text. Touch viewers send it on touch start. */
95 | { readonly type: "probe"; readonly x: number; readonly y: number };
96
97export interface PreviewStreamDownload {
98 readonly fileName: string;
99 readonly sizeBytes: number;
100 /** Authenticated with the stream's own access; cookie sessions must send credentials. */
101 readonly url: string;
102}
103
104export interface PreviewStreamFileChooser {
105 readonly multiple: boolean;
106 /** The input's `accept` attribute, ready for a local `<input type=file>`. */
107 readonly accept: string;
108 /** POST multipart `file` parts here; an empty form cancels the page's picker. */
109 readonly uploadUrl: string;
110 /** Cookie sessions must send credentials with the upload. */
111 readonly credentials: boolean;
112}
113
114const previewStreamUploadUrl = (
115 target: Pick<PreviewStreamTarget, "access" | "threadId" | "tabId">,
116 chooser: string,
117): string =>
118 withDeviceHubQuery(
119 `${target.access.httpBase}/upload?${new URLSearchParams({
120 threadId: target.threadId,
121 tabId: target.tabId,
122 chooser,
123 }).toString()}`,
124 target.access,
125 );
126
127/** Sends files to a page's open picker. Rejects when the server refuses them. */
128export async function uploadPreviewStreamFiles(
129 chooser: PreviewStreamFileChooser,
130 files: ReadonlyArray<Blob & { readonly name?: string }>,
131): Promise<void> {
132 const body = new FormData();
133 for (const file of files) body.append("file", file, file.name ?? "file");
134 const response = await fetch(chooser.uploadUrl, {
135 method: "POST",
136 body,
137 credentials: chooser.credentials ? "include" : "omit",
138 });
139 if (!response.ok) throw new Error((await response.text()) || "The upload was refused.");
140}
141
142/** A file a server tab downloaded, served with the stream's own access. */
143export const previewStreamDownloadUrl = (
144 target: Pick<PreviewStreamTarget, "access" | "threadId" | "tabId">,
145 id: string,
146): string =>
147 withDeviceHubQuery(
148 `${target.access.httpBase}/download?${new URLSearchParams({
149 threadId: target.threadId,
150 tabId: target.tabId,
151 id,
152 }).toString()}`,
153 target.access,
154 );
155
156/** Answer to a `probe`, echoing its point. */
157export interface PreviewStreamProbe {
158 readonly x: number;
159 readonly y: number;
160 readonly editable: boolean;
161}
162
163/** Where an agent action is about to land, in page CSS pixels. */
164export interface PreviewStreamPointer {
165 readonly phase: "move" | "click";
166 readonly x: number;
167 readonly y: number;
168 readonly sequence: number;
169}
170
171/** CDP modifier bitmask: Alt 1, Ctrl 2, Meta 4, Shift 8. */
172export const previewStreamModifiers = (event: {
173 readonly altKey: boolean;
174 readonly ctrlKey: boolean;
175 readonly metaKey: boolean;
176 readonly shiftKey: boolean;
177}): number =>
178 (event.altKey ? 1 : 0) |
179 (event.ctrlKey ? 2 : 0) |
180 (event.metaKey ? 4 : 0) |
181 (event.shiftKey ? 8 : 0);
182
183export interface PreviewStreamTarget {
184 readonly access: DeviceHubAccess;
185 readonly threadId: string;
186 readonly tabId: string;
187 /** Viewer backing-store size in device px. The server never sends larger frames. */
188 readonly maxWidth: number;
189 readonly maxHeight: number;
190 /** Passive viewers reduce their own access, including automatic control grants. */
191 readonly interactive?: boolean;
192}
193
194export interface PreviewStreamEvents {
195 /** One complete JPEG frame. */
196 readonly onFrame: (jpeg: ArrayBuffer) => void;
197 readonly onViewport: (viewport: PreviewStreamViewport) => void;
198 readonly onProbe?: (probe: PreviewStreamProbe) => void;
199 /** The agent's cursor moved to, or clicked at, a page point. */
200 readonly onPointer?: (pointer: PreviewStreamPointer) => void;
201 readonly onControl?: (control: PreviewStreamControl) => void;
202 /** Text the page just copied or cut while this viewer had control. */
203 readonly onClipboard?: (text: string) => void;
204 /** A file the page downloaded while this viewer had control. */
205 readonly onDownload?: (download: PreviewStreamDownload) => void;
206 /** The page this viewer controls opened a new tab (a link or `window.open`). */
207 readonly onPopup?: (tabId: string) => void;
208 /** The page opened a file picker (`null` once answered or replaced). */
209 readonly onFileChooser?: (chooser: PreviewStreamFileChooser | null) => void;
210 /** Input sent while disconnected is dropped. */
211 readonly onConnectedChange: (connected: boolean) => void;
212 /** The upgrade was refused; refresh access and start a new client. */
213 readonly onUnauthorized: () => void;
214 /** The tab was closed on the server. The client has stopped. */
215 readonly onGone?: () => void;
216 /** The server's browser cannot start until its host is set up. The client has stopped. */
217 readonly onHostSetup?: (setup: PreviewStreamHostSetup) => void;
218}
219
220export interface PreviewStreamClient {
221 /** False when the socket is not open and the message was dropped. */
222 readonly send: (input: PreviewStreamInput) => boolean;
223 readonly stop: () => void;
224}
225
226const ACK_MESSAGE = JSON.stringify({ type: "ack" });
227export function createPreviewStreamClient(
228 target: PreviewStreamTarget,
229 events: PreviewStreamEvents,
230): PreviewStreamClient {
231 const query = new URLSearchParams({
232 threadId: target.threadId,
233 tabId: target.tabId,
234 maxWidth: String(Math.max(1, Math.round(target.maxWidth))),
235 maxHeight: String(Math.max(1, Math.round(target.maxHeight))),
236 });
237 if (target.interactive === false) query.set("interactive", "false");
238 const url = withDeviceHubQuery(`${target.access.wsBase}/ws?${query.toString()}`, target.access);
239 let stopped = false;
240 let socket: WebSocket | null = null;
241 let retryTimer: ReturnType<typeof setTimeout> | null = null;
242 let failures = 0;
243 let control: PreviewStreamControl | null = null;
244 let fileChooser: string | null = null;
245
246 const connect = () => {
247 if (stopped) return;
248 const ws = new WebSocket(url);
249 ws.binaryType = "arraybuffer";
250 socket = ws;
251 let opened = false;
252 ws.addEventListener("open", () => {
253 if (socket !== ws) return;
254 opened = true;
255 events.onConnectedChange(true);
256 });
257 ws.addEventListener("message", (event) => {
258 if (socket !== ws) return;
259 if (event.data instanceof ArrayBuffer) {
260 failures = 0;
261 ws.send(ACK_MESSAGE);
262 events.onFrame(event.data);
263 return;
264 }
265 if (typeof event.data !== "string") return;
266 let message: unknown;
267 try {
268 message = JSON.parse(event.data);
269 } catch {
270 return;
271 }
272 if (typeof message !== "object" || message === null) return;
273 const {
274 type,
275 x,
276 y,
277 width,
278 height,
279 editable,
280 canOperate,
281 controller,
282 generation,
283 dialog,
284 text,
285 id,
286 fileName,
287 sizeBytes,
288 multiple,
289 accept,
290 phase,
291 sequence,
292 tabId,
293 } = message as Record<string, unknown>;
294 if (
295 type === "fileChooser" &&
296 typeof id === "string" &&
297 typeof multiple === "boolean" &&
298 typeof accept === "string"
299 ) {
300 fileChooser = id;
301 events.onFileChooser?.({
302 multiple,
303 accept,
304 uploadUrl: previewStreamUploadUrl(target, id),
305 credentials: target.access.credentials,
306 });
307 } else if (type === "fileChooserClosed" && typeof id === "string") {
308 if (fileChooser !== id) return;
309 fileChooser = null;
310 events.onFileChooser?.(null);
311 } else if (
312 type === "pointer" &&
313 (phase === "move" || phase === "click") &&
314 typeof x === "number" &&
315 typeof y === "number" &&
316 typeof sequence === "number"
317 ) {
318 events.onPointer?.({ phase, x, y, sequence });
319 } else if (type === "popup" && typeof tabId === "string") {
320 events.onPopup?.(tabId);
321 } else if (type === "clipboard" && typeof text === "string") {
322 events.onClipboard?.(text);
323 } else if (
324 type === "download" &&
325 typeof id === "string" &&
326 typeof fileName === "string" &&
327 typeof sizeBytes === "number"
328 ) {
329 events.onDownload?.({
330 fileName,
331 sizeBytes,
332 url: previewStreamDownloadUrl(target, id),
333 });
334 } else if (type === "viewport" && typeof width === "number" && typeof height === "number") {
335 failures = 0;
336 events.onViewport({ width, height });
337 } else if (
338 type === "probe" &&
339 typeof x === "number" &&
340 typeof y === "number" &&
341 typeof editable === "boolean"
342 ) {
343 events.onProbe?.({ x, y, editable });
344 } else if (
345 type === "control" &&
346 typeof canOperate === "boolean" &&
347 (controller === "agent" ||
348 controller === "you" ||
349 controller === "another-viewer" ||
350 controller === "unclaimed") &&
351 typeof generation === "number" &&
352 isPreviewStreamDialog(dialog)
353 ) {
354 const nextControl: PreviewStreamControl = {
355 canOperate,
356 controller,
357 generation,
358 dialog,
359 };
360 control = nextControl;
361 events.onControl?.(nextControl);
362 }
363 });
364 ws.addEventListener("close", (event) => {
365 if (socket !== ws) return;
366 socket = null;
367 control = null;
368 if (opened) events.onConnectedChange(false);
369 if (stopped) return;
370 if (event.code === 4404) {
371 stopped = true;
372 events.onGone?.();
373 return;
374 }
375 if (event.code === PREVIEW_STREAM_HOST_SETUP_CLOSE_CODE) {
376 stopped = true;
377 events.onHostSetup?.(decodeHostSetup(event.reason));
378 return;
379 }
380 // Rejected upgrades surface as 1006 before open for both cookies and tickets.
381 if (event.code === 1008 || event.code === 4401 || (!opened && event.code === 1006)) {
382 stopped = true;
383 events.onUnauthorized();
384 return;
385 }
386 retryTimer = setTimeout(connect, Math.min(500 * 2 ** failures++, 10_000));
387 });
388 ws.addEventListener("error", () => ws.close());
389 };
390
391 connect();
392
393 return {
394 send: (input) => {
395 if (socket?.readyState !== WebSocket.OPEN) return false;
396 if (!control?.canOperate) return false;
397 if (input.type !== "takeControl" && control.controller !== "you") return false;
398 socket.send(JSON.stringify(input));
399 return true;
400 },
401 stop: () => {
402 stopped = true;
403 if (retryTimer !== null) clearTimeout(retryTimer);
404 const ws = socket;
405 socket = null;
406 control = null;
407 ws?.close();
408 },
409 };
410}
411
412export interface PreviewFramePainter {
413 readonly paint: (jpeg: ArrayBuffer) => void;
414 readonly stop: () => void;
415}
416
417/** One decode at a time, keeping only the latest waiting frame for slow viewers. */
418export function createPreviewFramePainter(
419 canvas: HTMLCanvasElement,
420 onPainted?: () => void,
421): PreviewFramePainter {
422 const context = canvas.getContext("2d");
423 let stopped = false;
424 let decoding = false;
425 let waiting: ArrayBuffer | null = null;
426 const draw = (jpeg: ArrayBuffer) => {
427 decoding = true;
428 void createImageBitmap(new Blob([jpeg], { type: "image/jpeg" }))
429 .then((bitmap) => {
430 if (stopped || !context) {
431 bitmap.close();
432 return;
433 }
434 if (canvas.width !== bitmap.width || canvas.height !== bitmap.height) {
435 canvas.width = bitmap.width;
436 canvas.height = bitmap.height;
437 }
438 context.drawImage(bitmap, 0, 0);
439 bitmap.close();
440 onPainted?.();
441 })
442 // A frame that fails to decode is skipped; the next one repaints.
443 .catch(() => undefined)
444 .finally(() => {
445 decoding = false;
446 const next = waiting;
447 waiting = null;
448 if (next && !stopped) draw(next);
449 });
450 };
451 return {
452 paint: (jpeg) => {
453 if (stopped) return;
454 if (decoding) waiting = jpeg;
455 else draw(jpeg);
456 },
457 stop: () => {
458 stopped = true;
459 waiting = null;
460 },
461 };
462}
463
464/** A malformed reason still stops the viewer and offers the usual command. */
465const decodeHostSetup = (reason: string): PreviewStreamHostSetup => {
466 try {
467 const value = JSON.parse(reason) as Partial<PreviewStreamHostSetup>;
468 if (
469 (value.need === "sandbox" || value.need === "libraries") &&
470 typeof value.command === "string"
471 ) {
472 return { need: value.need, command: value.command };
473 }
474 } catch {}
475 return { need: "sandbox", command: "sudo npx t3 browser setup" };
476};
477
478/** What a viewer tells the person; `command` is shown beside it, ready to copy. */
479export const previewStreamHostSetupMessage = (setup: PreviewStreamHostSetup) =>
480 setup.need === "sandbox"
481 ? "This server's host blocks the sandbox its browser runs in. Run this once on the host, then try again:"
482 : "This server's host is missing libraries its browser needs. Run this once on the host, then try again:";