1import {
2 createPreviewFramePainter,
3 createPreviewStreamClient,
4 previewStreamModifiers,
5 type PreviewStreamClient,
6 type PreviewStreamControl,
7 type PreviewStreamInput,
8 type PreviewStreamPointer,
9 type PreviewStreamViewport,
10} from "@t3tools/client-runtime/preview/server-browser-stream";
11
12import type { PreviewStreamConfiguration, PreviewStreamMessage } from "./preview-stream-document";
13
14declare global {
15 interface Window {
16 ReactNativeWebView: { postMessage: (message: string) => void };
17 }
18}
19
20/** WebKit's presentation API, the only picture in picture entry on older iOS. */
21interface PresentationVideo extends HTMLVideoElement {
22 webkitSetPresentationMode?: (mode: "inline" | "picture-in-picture") => void;
23 webkitPresentationMode?: string;
24}
25
26type WheelInput = Extract<PreviewStreamInput, { type: "wheel" }>;
27type MouseInput = Extract<PreviewStreamInput, { type: "mouse" }>;
28const mouseButton = (button: number): MouseInput["button"] =>
29 button === 0 ? "left" : button === 1 ? "middle" : button === 2 ? "right" : "none";
30
31const RESIZE_DEBOUNCE_MS = 150;
32const TAP_SLOP_PX = 8;
33const MULTI_CLICK_MS = 500;
34const MULTI_CLICK_SLOP_PX = 4;
35const WHEEL_LINE_PX = 16;
36// JPEG frames past 2x cost bandwidth without a visible gain on a phone.
37const MAX_PIXEL_RATIO = 2;
38// A tap this close to an answered probe, this soon, reuses its answer. On a slow
39// link the answer lands after touch end, too late for iOS to raise the keyboard.
40const PROBE_REUSE_PX = 24;
41const PROBE_REUSE_MS = 10_000;
42// Kept in the hidden textarea so a soft keyboard's backspace has something to
43// delete and fires `input`; Gboard's keydown carries keyCode 229 and no key.
44const SENTINEL = "\u200b";
45// The agent cursor stays bright while the agent acts, then dims like the web panel's.
46const AGENT_CURSOR_ACTIVE_MS = 700;
47const AGENT_CURSOR_SVG =
48 '<svg width="20" height="20" viewBox="0 0 24 24" fill="#fff" stroke="#3b82f6" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M4.037 4.688a.495.495 0 0 1 .651-.651l16 6.5a.5.5 0 0 1-.063.947l-6.124 1.58a2 2 0 0 0-1.438 1.435l-1.579 6.126a.5.5 0 0 1-.947.063z"/></svg>';
49
50interface Viewer {
51 readonly stop: () => void;
52 readonly command: (input: PreviewStreamInput) => void;
53 readonly togglePictureInPicture: () => void;
54}
55
56let activeViewer: Viewer | null = null;
57
58export function stop() {
59 activeViewer?.stop();
60 activeViewer = null;
61}
62
63/** Navigation and history from the native chrome require current browser control. */
64export function command(input: PreviewStreamInput) {
65 activeViewer?.command(input);
66}
67
68export function pictureInPicture() {
69 activeViewer?.togglePictureInPicture();
70}
71
72export function start(configuration: PreviewStreamConfiguration) {
73 stop();
74 const post = (message: PreviewStreamMessage) => {
75 // oxlint-disable-next-line unicorn/require-post-message-target-origin -- The native WebView bridge takes one string.
76 window.ReactNativeWebView.postMessage(JSON.stringify(message));
77 };
78 const { interactive } = configuration;
79 document.body.style.background = configuration.background;
80 const container = document.createElement("div");
81 const canvas = document.createElement("canvas");
82 canvas.setAttribute("role", "img");
83 canvas.setAttribute("aria-label", "Browser page");
84 const agentCursor = document.createElement("div");
85 agentCursor.className = "agent-cursor";
86 agentCursor.setAttribute("aria-hidden", "true");
87 agentCursor.innerHTML = AGENT_CURSOR_SVG;
88 container.append(canvas, agentCursor);
89 const input = document.createElement("textarea");
90 input.setAttribute("aria-label", "Browser page input");
91 input.autocapitalize = "off";
92 input.autocomplete = "off";
93 input.spellcheck = false;
94 input.setAttribute("autocorrect", "off");
95 input.disabled = true;
96 document.body.replaceChildren(container, ...(interactive ? [input] : []));
97
98 let stopped = false;
99 let streaming = false;
100 let viewport: PreviewStreamViewport | null = null;
101 let size: { width: number; height: number } | null = null;
102 // Frame cap in device px, fixed per socket. It only grows, so only outgrowing it reconnects.
103 let cap: { width: number; height: number } | null = null;
104 let client: PreviewStreamClient | null = null;
105 let control: PreviewStreamControl | null = null;
106 let resizeTimer: ReturnType<typeof setTimeout> | null = null;
107 let wheelFrame: number | null = null;
108 let pendingWheel: WheelInput | null = null;
109 let mouseFrame: number | null = null;
110 let pendingMouse: MouseInput | null = null;
111 let mouseButtons = 0;
112 const mouseClicks = { left: 1, middle: 1, right: 1, none: 1 };
113 // The latest probe. `editable` is the answer its tap acts on, null until known;
114 // `end` records what touch end did, so a late answer can still act or correct it.
115 let probe: {
116 readonly x: number;
117 readonly y: number;
118 readonly clientX: number;
119 readonly clientY: number;
120 editable: boolean | null;
121 end: "touching" | "acted" | "late" | "panned";
122 } | null = null;
123 let probeCache: {
124 readonly clientX: number;
125 readonly clientY: number;
126 readonly editable: boolean;
127 readonly time: number;
128 } | null = null;
129 // A keydown already sent this key; its `input` must not send it again.
130 let keySent = false;
131 let touch: {
132 start: PointerEvent;
133 last: PointerEvent;
134 panning: boolean;
135 } | null = null;
136 let lastTap: {
137 time: number;
138 x: number;
139 y: number;
140 count: number;
141 button: MouseInput["button"];
142 } | null = null;
143
144 const reportStatus = (status: "connecting" | "streaming") => {
145 streaming = status === "streaming";
146 post({ type: "status", status });
147 };
148 const painter = createPreviewFramePainter(canvas, () => {
149 if (!streaming && !stopped) reportStatus("streaming");
150 sendPictureInPictureFrame();
151 });
152 const send = (message: PreviewStreamInput) => client?.send(message) ?? false;
153 const clearInput = () => {
154 if (wheelFrame !== null) cancelAnimationFrame(wheelFrame);
155 if (mouseFrame !== null) cancelAnimationFrame(mouseFrame);
156 wheelFrame = mouseFrame = null;
157 pendingWheel = pendingMouse = null;
158 mouseButtons = 0;
159 Object.assign(mouseClicks, { left: 1, middle: 1, right: 1, none: 1 });
160 probe = probeCache = touch = lastTap = null;
161 keySent = false;
162 input.value = SENTINEL;
163 input.disabled = true;
164 input.blur();
165 };
166
167 const connect = () => {
168 // Stopping suppresses the old socket's disconnect report; native also drops its control.
169 if (client) reportStatus("connecting");
170 client?.stop();
171 control = null;
172 clearInput();
173 if (!cap || stopped) return;
174 const next = createPreviewStreamClient(
175 {
176 access: configuration.access,
177 threadId: configuration.threadId,
178 tabId: configuration.tabId,
179 maxWidth: cap.width,
180 maxHeight: cap.height,
181 interactive,
182 },
183 {
184 onFrame: (jpeg) => painter.paint(jpeg),
185 onPointer: showAgentCursor,
186 onClipboard: (text) => post({ type: "clipboard", text }),
187 onDownload: (download) => post({ type: "download", ...download }),
188 onFileChooser: (chooser) => post({ type: "fileChooser", chooser }),
189 onViewport: (page) => {
190 if (viewport?.width === page.width && viewport.height === page.height) return;
191 viewport = page;
192 post({ type: "viewport", width: page.width, height: page.height });
193 },
194 onProbe: (result) => {
195 if (control?.controller !== "you") return;
196 const current = probe;
197 if (!current || current.x !== result.x || current.y !== result.y) return;
198 probeCache = {
199 clientX: current.clientX,
200 clientY: current.clientY,
201 editable: result.editable,
202 time: performance.now(),
203 };
204 if (current.end === "touching") {
205 current.editable = result.editable;
206 return;
207 }
208 probe = null;
209 if (current.end === "panned") return;
210 if (current.end === "acted" && current.editable === result.editable) return;
211 // Late or corrected answer: Android still raises the keyboard; iOS waits
212 // for the next tap, which can reuse this answer.
213 if (result.editable) input.focus({ preventScroll: true });
214 else input.blur();
215 },
216 onControl: (nextControl) => {
217 const previous = control;
218 control = nextControl;
219 post({ type: "control", ...nextControl });
220 // Taking over hides the agent cursor; the person's own touch is the pointer now.
221 if (nextControl.controller === "you") agentCursor.style.opacity = "0";
222 if (nextControl.controller !== "you") clearInput();
223 else {
224 input.disabled = !interactive;
225 if (interactive && size && previous?.controller !== "you")
226 next.send({ type: "resize", ...size });
227 }
228 },
229 onConnectedChange: (connected) => {
230 if (!connected) {
231 control = null;
232 clearInput();
233 if (streaming) reportStatus("connecting");
234 return;
235 }
236 },
237 onUnauthorized: () => post({ type: "unauthorized" }),
238 onGone: () => post({ type: "gone" }),
239 onHostSetup: (setup) => post({ type: "hostSetup", ...setup }),
240 },
241 );
242 client = next;
243 };
244
245 const measure = () => {
246 resizeTimer = null;
247 const rect = container.getBoundingClientRect();
248 if (rect.width < 1 || rect.height < 1) return;
249 const next = { width: Math.round(rect.width), height: Math.round(rect.height) };
250 if (size?.width === next.width && size.height === next.height) return;
251 size = next;
252 if (interactive) send({ type: "resize", ...next });
253 const ratio = Math.min(window.devicePixelRatio || 1, MAX_PIXEL_RATIO);
254 // The floating player scales a page of any shape into its box, so its cap is square.
255 const side = Math.max(next.width, next.height);
256 const width = Math.round((interactive ? next.width : side) * ratio);
257 const height = Math.round((interactive ? next.height : side) * ratio);
258 if (cap && cap.width >= width && cap.height >= height) return;
259 cap = { width: Math.max(width, cap?.width ?? 0), height: Math.max(height, cap?.height ?? 0) };
260 connect();
261 };
262 const observer = new ResizeObserver(() => {
263 if (resizeTimer !== null) clearTimeout(resizeTimer);
264 // The first size connects right away; later ones settle before resizing the page.
265 resizeTimer = setTimeout(measure, size === null ? 0 : RESIZE_DEBOUNCE_MS);
266 });
267 observer.observe(container);
268
269 // iOS keeps the layout viewport under the soft keyboard; follow the visible area
270 // so the page resizes above it.
271 const visualViewport = window.visualViewport;
272 const followVisualViewport = () => {
273 if (!visualViewport) return;
274 container.style.top = `${visualViewport.offsetTop}px`;
275 container.style.height = `${visualViewport.height}px`;
276 };
277 if (interactive) {
278 visualViewport?.addEventListener("resize", followVisualViewport);
279 visualViewport?.addEventListener("scroll", followVisualViewport);
280 }
281
282 let agentCursorTimer: ReturnType<typeof setTimeout> | null = null;
283 /** Mirrors the frame's letterbox so the cursor lands on the element the agent targets. */
284 const showAgentCursor = (pointer: PreviewStreamPointer) => {
285 if (!viewport || canvas.width === 0 || canvas.height === 0 || control?.controller === "you")
286 return;
287 const box = { width: canvas.clientWidth, height: canvas.clientHeight };
288 const fit = Math.min(box.width / canvas.width, box.height / canvas.height);
289 const width = canvas.width * fit;
290 const height = canvas.height * fit;
291 const left = (box.width - width) / 2 + (pointer.x * width) / viewport.width;
292 const top = (box.height - height) / 2 + (pointer.y * height) / viewport.height;
293 agentCursor.style.transform = `translate3d(${left}px, ${top}px, 0)`;
294 agentCursor.style.opacity = "1";
295 if (pointer.phase === "click") {
296 agentCursor.querySelector(".ping")?.remove();
297 const ping = document.createElement("span");
298 ping.className = "ping";
299 agentCursor.prepend(ping);
300 }
301 if (agentCursorTimer !== null) clearTimeout(agentCursorTimer);
302 agentCursorTimer = setTimeout(() => {
303 agentCursor.style.opacity = control?.controller === "agent" ? "0.35" : "0";
304 }, AGENT_CURSOR_ACTIVE_MS);
305 };
306
307 const pagePoint = (clientX: number, clientY: number, clamp: boolean) => {
308 if (!viewport || canvas.width === 0 || canvas.height === 0 || control?.controller !== "you")
309 return null;
310 const rect = canvas.getBoundingClientRect();
311 // `object-fit: contain` letterboxes the frame inside the canvas box.
312 const fit = Math.min(rect.width / canvas.width, rect.height / canvas.height);
313 const width = canvas.width * fit;
314 const height = canvas.height * fit;
315 if (!(width > 0 && height > 0)) return null;
316 const scale = viewport.width / width;
317 const x = (clientX - rect.left - (rect.width - width) / 2) * scale;
318 const y = (clientY - rect.top - (rect.height - height) / 2) * (viewport.height / height);
319 const inside = x >= 0 && y >= 0 && x <= viewport.width && y <= viewport.height;
320 if (!inside && !clamp) return null;
321 return {
322 x: Math.min(Math.max(x, 0), viewport.width),
323 y: Math.min(Math.max(y, 0), viewport.height),
324 scale,
325 };
326 };
327
328 // Wheel deltas coalesce to one message per animation frame.
329 const flushWheel = () => {
330 if (wheelFrame !== null) cancelAnimationFrame(wheelFrame);
331 wheelFrame = null;
332 const wheel = pendingWheel;
333 pendingWheel = null;
334 if (wheel) send(wheel);
335 };
336 const queueWheel = (point: { x: number; y: number }, deltaX: number, deltaY: number) => {
337 pendingWheel = {
338 type: "wheel",
339 x: point.x,
340 y: point.y,
341 deltaX: (pendingWheel?.deltaX ?? 0) + deltaX,
342 deltaY: (pendingWheel?.deltaY ?? 0) + deltaY,
343 modifiers: 0,
344 };
345 wheelFrame ??= requestAnimationFrame(flushWheel);
346 };
347
348 const flushMouse = () => {
349 if (mouseFrame !== null) cancelAnimationFrame(mouseFrame);
350 mouseFrame = null;
351 if (pendingMouse) send(pendingMouse);
352 pendingMouse = null;
353 };
354 const countClick = (event: PointerEvent, button: MouseInput["button"]) => {
355 const last = lastTap;
356 const count =
357 last &&
358 last.button === button &&
359 event.timeStamp - last.time < MULTI_CLICK_MS &&
360 Math.hypot(event.clientX - last.x, event.clientY - last.y) < MULTI_CLICK_SLOP_PX
361 ? last.count + 1
362 : 1;
363 lastTap = { time: event.timeStamp, x: event.clientX, y: event.clientY, count, button };
364 return count;
365 };
366 const mouseInput = (
367 event: PointerEvent,
368 point: { x: number; y: number },
369 action: MouseInput["action"],
370 button: MouseInput["button"],
371 buttons = event.buttons,
372 clickCount = mouseClicks[button],
373 ): MouseInput => ({
374 type: "mouse",
375 action,
376 x: point.x,
377 y: point.y,
378 button,
379 buttons,
380 clickCount,
381 modifiers: previewStreamModifiers(event),
382 });
383 const onPointerDown = (event: PointerEvent) => {
384 if (control?.controller !== "you") return;
385 if (!event.isPrimary) return;
386 event.preventDefault();
387 if (event.pointerType !== "touch") {
388 const point = pagePoint(event.clientX, event.clientY, false);
389 if (!point) return;
390 canvas.setPointerCapture(event.pointerId);
391 input.focus({ preventScroll: true });
392 flushMouse();
393 flushWheel();
394 mouseButtons = event.buttons & 7;
395 const button = mouseButton(event.button);
396 mouseClicks[button] = countClick(event, button);
397 send(mouseInput(event, point, "down", button));
398 return;
399 }
400 touch = { start: event, last: event, panning: false };
401 const point = pagePoint(event.clientX, event.clientY, false);
402 if (!point) {
403 probe = null;
404 return;
405 }
406 const cached = probeCache;
407 const reuse =
408 cached !== null &&
409 performance.now() - cached.time < PROBE_REUSE_MS &&
410 Math.hypot(event.clientX - cached.clientX, event.clientY - cached.clientY) <= PROBE_REUSE_PX;
411 probe = {
412 x: point.x,
413 y: point.y,
414 clientX: event.clientX,
415 clientY: event.clientY,
416 editable: reuse ? cached.editable : null,
417 end: "touching",
418 };
419 // Sent even with a cached answer, so the next tap reuses a fresh one.
420 send({ type: "probe", x: point.x, y: point.y });
421 };
422 const onPointerMove = (event: PointerEvent) => {
423 if (control?.controller !== "you") return;
424 if (event.pointerType !== "touch") {
425 const point = pagePoint(event.clientX, event.clientY, mouseButtons !== 0);
426 if (!point) return;
427 // Chorded presses and releases arrive as pointermove while another button is held.
428 const changed = mouseButtons ^ (event.buttons & 7);
429 if (mouseButtons !== 0 && changed !== 0) {
430 flushMouse();
431 flushWheel();
432 for (const bit of [1, 2, 4]) {
433 if (!(changed & bit)) continue;
434 const button = bit === 1 ? "left" : bit === 2 ? "right" : "middle";
435 const down = (event.buttons & bit) !== 0;
436 mouseButtons ^= bit;
437 if (down) mouseClicks[button] = countClick(event, button);
438 send(mouseInput(event, point, down ? "down" : "up", button, mouseButtons));
439 }
440 }
441 const button =
442 mouseButtons & 1
443 ? "left"
444 : mouseButtons & 2
445 ? "right"
446 : mouseButtons & 4
447 ? "middle"
448 : "none";
449 pendingMouse = mouseInput(event, point, "move", button, event.buttons, 0);
450 mouseFrame ??= requestAnimationFrame(flushMouse);
451 return;
452 }
453 if (!touch || touch.start.pointerId !== event.pointerId) return;
454 if (
455 !touch.panning &&
456 Math.hypot(event.clientX - touch.start.clientX, event.clientY - touch.start.clientY) <
457 TAP_SLOP_PX
458 ) {
459 return;
460 }
461 touch.panning = true;
462 const point = pagePoint(event.clientX, event.clientY, true);
463 // Dragging the page up scrolls it down, following the finger.
464 if (point) {
465 queueWheel(
466 point,
467 (touch.last.clientX - event.clientX) * point.scale,
468 (touch.last.clientY - event.clientY) * point.scale,
469 );
470 }
471 touch.last = event;
472 };
473 const releaseMouse = (event: PointerEvent, cancelled: boolean) => {
474 let buttons = mouseButtons;
475 if (buttons === 0) return;
476 mouseButtons = cancelled ? 0 : event.buttons & 7;
477 flushMouse();
478 flushWheel();
479 const point = pagePoint(event.clientX, event.clientY, true);
480 if (!point) return;
481 for (const bit of [1, 2, 4]) {
482 if (!(buttons & bit) || (mouseButtons & bit) !== 0) continue;
483 buttons &= ~bit;
484 const button = bit === 1 ? "left" : bit === 2 ? "right" : "middle";
485 send(mouseInput(event, point, "up", button, buttons));
486 }
487 };
488 const onPointerEnd = (event: PointerEvent) => {
489 const cancelled = event.type === "pointercancel";
490 if (event.pointerType !== "touch") {
491 releaseMouse(event, cancelled);
492 return;
493 }
494 const ended = touch;
495 if (!ended || ended.start.pointerId !== event.pointerId) return;
496 touch = null;
497 const answered = probe;
498 if (cancelled || ended.panning) {
499 if (answered) answered.end = "panned";
500 return;
501 }
502 const point = pagePoint(event.clientX, event.clientY, false);
503 if (!point) return;
504 // Focusing inside the tap's user activation is what lets iOS raise the keyboard.
505 if (answered?.editable === true) input.focus({ preventScroll: true });
506 else if (answered?.editable === false) input.blur();
507 if (answered) answered.end = answered.editable === null ? "late" : "acted";
508 flushWheel();
509 const clickCount = countClick(event, "left");
510 const at = { x: point.x, y: point.y, modifiers: 0 };
511 send({ type: "mouse", action: "move", ...at, button: "none", buttons: 0, clickCount: 0 });
512 send({ type: "mouse", action: "down", ...at, button: "left", buttons: 1, clickCount });
513 send({ type: "mouse", action: "up", ...at, button: "left", buttons: 0, clickCount });
514 };
515 // Trackpads and mice on tablets scroll with wheel events.
516 const onWheel = (event: WheelEvent) => {
517 const point = pagePoint(event.clientX, event.clientY, false);
518 if (!point) return;
519 event.preventDefault();
520 const unit =
521 event.deltaMode === WheelEvent.DOM_DELTA_LINE
522 ? WHEEL_LINE_PX
523 : event.deltaMode === WheelEvent.DOM_DELTA_PAGE
524 ? (viewport?.height ?? 0)
525 : 1;
526 queueWheel(point, event.deltaX * unit, event.deltaY * unit);
527 };
528 // Keeps focus in the page input and stops native selection and callouts.
529 const preventDefault = (event: Event) => event.preventDefault();
530
531 const onKey = (action: "down" | "up", event: KeyboardEvent) => {
532 if (control?.controller !== "you") return;
533 // IME and soft keyboards deliver text through composition and input events.
534 if (
535 event.isComposing ||
536 event.keyCode === 229 ||
537 event.key === "Process" ||
538 event.key === "Unidentified"
539 ) {
540 return;
541 }
542 const shortcut = event.ctrlKey || event.metaKey;
543 // Paste arrives as input text from this device's clipboard. Copy and cut run in
544 // the page, which sends the copied text back. Shift+Insert is the same paste.
545 if (shortcut && event.key.toLowerCase() === "v") return;
546 if (event.shiftKey && event.key === "Insert") return;
547 // Enter carries "\r" like Puppeteer's key table, so forms submit and textareas break lines.
548 const text = shortcut
549 ? undefined
550 : [...event.key].length === 1
551 ? event.key
552 : event.key === "Enter"
553 ? "\r"
554 : undefined;
555 send({
556 type: "key",
557 action,
558 key: event.key,
559 code: event.code,
560 keyCode: event.keyCode,
561 ...(action === "down" && text !== undefined ? { text } : {}),
562 modifiers: previewStreamModifiers(event),
563 });
564 // Some Android keyboards edit the textarea even when keydown is prevented.
565 // Shortcuts keep their default, so a paste still arrives as text.
566 keySent =
567 action === "down" &&
568 !shortcut &&
569 (text !== undefined || event.key === "Backspace" || event.key === "Delete");
570 if (!shortcut) event.preventDefault();
571 };
572 const resetInput = () => {
573 input.value = SENTINEL;
574 input.setSelectionRange(SENTINEL.length, SENTINEL.length);
575 };
576 const pressKey = (key: "Backspace" | "Delete", keyCode: number) => {
577 for (const action of ["down", "up"] as const) {
578 send({ type: "key", action, key, code: key, keyCode, modifiers: 0 });
579 }
580 };
581 const onInput = (event: Event) => {
582 if (event instanceof InputEvent && event.isComposing) return;
583 const inputType = event instanceof InputEvent ? event.inputType : "";
584 if (keySent) keySent = false;
585 else if (inputType === "deleteContentBackward") pressKey("Backspace", 8);
586 else if (inputType === "deleteContentForward") pressKey("Delete", 46);
587 else {
588 const value = input.value;
589 const text = value.startsWith(SENTINEL) ? value.slice(SENTINEL.length) : value;
590 if (text) send({ type: "text", text });
591 }
592 resetInput();
593 };
594 const onCompositionEnd = (event: CompositionEvent) => {
595 const text = event.data.startsWith(SENTINEL) ? event.data.slice(SENTINEL.length) : event.data;
596 if (text) send({ type: "text", text });
597 resetInput();
598 };
599
600 if (interactive) {
601 canvas.addEventListener("pointerdown", onPointerDown);
602 canvas.addEventListener("pointermove", onPointerMove);
603 canvas.addEventListener("pointerup", onPointerEnd);
604 canvas.addEventListener("pointercancel", onPointerEnd);
605 canvas.addEventListener("wheel", onWheel, { passive: false });
606 canvas.addEventListener("mousedown", preventDefault);
607 canvas.addEventListener("contextmenu", preventDefault);
608 input.addEventListener("keydown", (event) => onKey("down", event));
609 input.addEventListener("keyup", (event) => onKey("up", event));
610 input.addEventListener("input", onInput);
611 input.addEventListener("compositionend", onCompositionEnd);
612 input.addEventListener("focus", resetInput);
613 // Copying the input would put its sentinel on this device's clipboard.
614 input.addEventListener("copy", preventDefault);
615 input.addEventListener("cut", preventDefault);
616 resetInput();
617 }
618
619 // WebKit reports no support on a video that has not loaded yet; check the prototype.
620 const videoPrototype: PresentationVideo = HTMLVideoElement.prototype;
621 const pictureInPictureSupported =
622 interactive &&
623 typeof canvas.captureStream === "function" &&
624 typeof CanvasCaptureMediaStreamTrack === "function" &&
625 ((document.pictureInPictureEnabled === true &&
626 typeof videoPrototype.requestPictureInPicture === "function") ||
627 typeof videoPrototype.webkitSetPresentationMode === "function");
628 // Since the iOS 17 SDK, WebKit revokes the activation it grants an injected script when
629 // the script returns, so the native button's request must start before any await. The
630 // video is ready ahead of the tap: paused, and fed a frame only when it has none or
631 // floats, so the inline view captures nothing.
632 let video: PresentationVideo | null = null;
633 let track: CanvasCaptureMediaStreamTrack | null = null;
634 const pictureInPictureActive = () =>
635 video !== null &&
636 (document.pictureInPictureElement === video ||
637 video.webkitPresentationMode === "picture-in-picture");
638 const reportPictureInPicture = (detail?: string) =>
639 post({
640 type: "pictureInPicture",
641 supported: pictureInPictureSupported,
642 active: pictureInPictureActive(),
643 ...(detail ? { detail } : {}),
644 });
645 // Called after each paint. A frame drawn before the player attached is lost, so retry.
646 const sendPictureInPictureFrame = () => {
647 if (video && (video.readyState === HTMLMediaElement.HAVE_NOTHING || pictureInPictureActive())) {
648 track?.requestFrame();
649 }
650 };
651 if (pictureInPictureSupported) {
652 const stream = canvas.captureStream(0);
653 const [captured] = stream.getVideoTracks();
654 const element: PresentationVideo = document.createElement("video");
655 element.muted = true;
656 element.playsInline = true;
657 // Under the canvas at full size: WebKit pauses muted video it considers off screen.
658 element.srcObject = stream;
659 for (const name of [
660 "enterpictureinpicture",
661 "leavepictureinpicture",
662 "webkitpresentationmodechanged",
663 ]) {
664 element.addEventListener(name, () => {
665 if (!pictureInPictureActive()) element.pause();
666 reportPictureInPicture();
667 });
668 }
669 container.prepend(element);
670 // Covers the letterbox, where a stale frame of another shape would show through.
671 canvas.style.background = configuration.background;
672 video = element;
673 track = captured instanceof CanvasCaptureMediaStreamTrack ? captured : null;
674 }
675 const togglePictureInPicture = () => {
676 const element = video;
677 if (!element) return;
678 if (pictureInPictureActive()) {
679 if (document.pictureInPictureElement) void document.exitPictureInPicture().catch(() => {});
680 else element.webkitSetPresentationMode?.("inline");
681 return;
682 }
683 if (element.readyState === HTMLMediaElement.HAVE_NOTHING) {
684 reportPictureInPicture("The page has not shown yet. Try again in a moment.");
685 return;
686 }
687 // The page may have changed since the video's last frame. WebKit captures only on a draw.
688 track?.requestFrame();
689 canvas.getContext("2d")?.drawImage(canvas, 0, 0);
690 const fail = (error: unknown) => {
691 element.pause();
692 reportPictureInPicture(
693 error instanceof Error ? error.message : "Picture in picture is unavailable.",
694 );
695 };
696 if (document.pictureInPictureEnabled) {
697 const entering = element.requestPictureInPicture();
698 void Promise.all([entering, element.play()]).catch(fail);
699 } else {
700 element.webkitSetPresentationMode?.("picture-in-picture");
701 void element.play().catch(fail);
702 }
703 };
704
705 activeViewer = {
706 stop: () => {
707 stopped = true;
708 observer.disconnect();
709 visualViewport?.removeEventListener("resize", followVisualViewport);
710 visualViewport?.removeEventListener("scroll", followVisualViewport);
711 if (resizeTimer !== null) clearTimeout(resizeTimer);
712 if (wheelFrame !== null) cancelAnimationFrame(wheelFrame);
713 if (mouseFrame !== null) cancelAnimationFrame(mouseFrame);
714 if (agentCursorTimer !== null) clearTimeout(agentCursorTimer);
715 painter.stop();
716 client?.stop();
717 client = null;
718 track?.stop();
719 if (video) video.srcObject = null;
720 },
721 command: (message) => {
722 send(message);
723 },
724 togglePictureInPicture,
725 };
726 window.addEventListener("pagehide", stop, { once: true });
727 reportStatus("connecting");
728 if (interactive) reportPictureInPicture();
729}