1import type { DevicePlatform, EnvironmentId } from "@t3tools/contracts";
2import { useCallback, useEffect, useMemo, useRef, useState, type ReactNode } from "react";
3
4import { cn } from "~/lib/utils";
5import { Button } from "~/components/ui/button";
6import { refreshDeviceHubAccess, useDeviceHubAccess } from "~/state/device";
7import { createCanvasFrameSink } from "@t3tools/client-runtime/device/frame";
8import { resolveDeviceShape } from "@t3tools/client-runtime/device/shape-profile";
9import { deviceKeyboard, deviceModel } from "./deviceModels";
10import { fitDeviceFrame } from "./deviceFrameLayout";
11import { DeviceDuoViewport } from "./DeviceDuoViewport";
12import { DeviceDuoControls } from "./DeviceDuoControls";
13import { DeviceAndroidFoldControls } from "./DeviceAndroidFoldControls";
14import type { DuoControlState } from "@t3tools/client-runtime/device/duo-control";
15import { DevicePhoneViewport } from "./DevicePhoneViewport";
16import { DeviceLoadingView } from "./DeviceLoadingView";
17import { type DeviceAxElement, fetchDeviceAxTree } from "./deviceHubApi";
18import {
19 createDeviceStreamClient,
20 type DeviceHardwareButton,
21 type DeviceScreenSize,
22 type DeviceStreamClient,
23 type DeviceStreamStatus,
24} from "@t3tools/client-runtime/device/stream";
25import { observeResize } from "~/lib/observeResize";
26
27const AX_POLL_INTERVAL_MS = 2_000;
28const CONTROLS_RAIL_WIDTH = 56;
29
30export interface DeviceViewControls {
31 readonly phone: boolean;
32 readonly streaming: boolean;
33 readonly phoneUnavailableReason: string | null;
34 readonly showPhone: () => void;
35 readonly showFlat: () => void;
36 readonly resetView: () => void;
37 readonly foldingControls?: ReactNode;
38 readonly keyboard: { readonly attached: boolean; readonly toggle: () => void } | null;
39}
40
41export interface DeviceStreamHandle {
42 readonly pressButton: (button: DeviceHardwareButton) => void;
43 readonly rotate: () => void;
44 /** False while the input socket is down; controls should disable. */
45 readonly inputConnected: boolean;
46}
47
48/**
49 * The live device screen. Pointer events map onto normalized coordinates in
50 * the displayed frame and go to the device; keyboard input is forwarded while
51 * the surface is focused. `visible=false` tears the stream down so a hidden
52 * panel decodes nothing.
53 */
54export function DeviceStreamView(props: {
55 readonly environmentId: EnvironmentId;
56 readonly platform: DevicePlatform;
57 readonly deviceId: string;
58 readonly deviceName?: string;
59 readonly deviceDescription?: string;
60 readonly visible: boolean;
61 readonly hostId: string;
62 /** The full panel opts into the phone spike; compact viewers retain their flat presentation. */
63 readonly allowPhoneView?: boolean;
64 readonly renderControls?: (view: DeviceViewControls) => ReactNode;
65 /** Draw accessibility element frames over the screen. */
66 readonly axOverlay?: boolean;
67 readonly onHandle?: (handle: DeviceStreamHandle | null) => void;
68 readonly onScreen?: (screen: DeviceScreenSize | null) => void;
69}) {
70 const [duoControl, setDuoControl] = useState<DuoControlState>({
71 pending: false,
72 requested: null,
73 error: null,
74 });
75 const model = deviceModel(props.platform, props.deviceName ?? "");
76 const isDuo = model?.id === "iphone-duo";
77 const [presentation, setPresentation] = useState<"phone" | "flat">("phone");
78 const [keyboardAttached, setKeyboardAttached] = useState(false);
79 const [phoneUnavailable, setPhoneUnavailable] = useState(false);
80 const onPhoneUnavailable = useCallback(() => setPhoneUnavailable(true), []);
81 const cancelPhoneInputRef = useRef<(() => void) | null>(null);
82 const cancelPhoneInput = useCallback(() => cancelPhoneInputRef.current?.(), []);
83 const resetViewRef = useRef<(() => void) | null>(null);
84 const onResetReady = useCallback((reset: (() => void) | null) => {
85 resetViewRef.current = reset;
86 }, []);
87 const frameListenerRef = useRef<(() => void) | null>(null);
88 const onFrameListener = useCallback((listener: (() => void) | null) => {
89 frameListenerRef.current = listener;
90 }, []);
91 const onInputCancel = useCallback((cancel: (() => void) | null) => {
92 cancelPhoneInputRef.current = cancel;
93 }, []);
94 const access = useDeviceHubAccess(props.environmentId, props.hostId);
95 const canvasRef = useRef<HTMLCanvasElement | null>(null);
96 const clientRef = useRef<DeviceStreamClient | null>(null);
97 const [status, setStatus] = useState<DeviceStreamStatus>("connecting");
98 const [hasFrame, setHasFrame] = useState(false);
99 const [detail, setDetail] = useState<string | undefined>(undefined);
100 const [showRestartNotice, setShowRestartNotice] = useState(false);
101 const [screen, setScreen] = useState<DeviceScreenSize | null>(null);
102 const [foldAngle, setFoldAngle] = useState<number | null>(null);
103 const [mjpegUrl, setMjpegUrl] = useState<string | null>(null);
104 const [mjpegGeneration, setMjpegGeneration] = useState(0);
105 const attachMjpegImage = useCallback((image: HTMLImageElement | null) => {
106 clientRef.current?.setMjpegImage(image);
107 }, []);
108 const [inputState, setInputState] = useState<{ connected: boolean; detail?: string }>({
109 connected: false,
110 });
111 const { onHandle, onScreen } = props;
112
113 useEffect(() => {
114 const canvas = canvasRef.current;
115 if (!access || !canvas || !props.visible) {
116 setStatus("connecting");
117 onHandle?.(null);
118 return;
119 }
120 const client = createDeviceStreamClient(
121 { platform: props.platform, deviceId: props.deviceId, access },
122 createCanvasFrameSink(canvas, () => frameListenerRef.current?.()),
123 {
124 onDuoControl: setDuoControl,
125 onDuoUnavailable: onPhoneUnavailable,
126 onStatus: (next, nextDetail) => {
127 setStatus(next);
128 if (next === "streaming") setHasFrame(true);
129 setDetail(nextDetail);
130 if (next !== "connecting") setShowRestartNotice(false);
131 },
132 onScreen: (next) => {
133 setScreen(next);
134 onScreen?.(next);
135 },
136 onUnauthorized: () => {
137 // A fresh ticket re-runs this effect through the access dependency.
138 refreshDeviceHubAccess(props.environmentId);
139 },
140 onMjpegFallback: (url) => {
141 setMjpegUrl(url);
142 setMjpegGeneration((generation) => generation + 1);
143 },
144 onInputConnected: (connected, detail) => {
145 setInputState({ connected, ...(detail ? { detail } : {}) });
146 if (!connected) setShowRestartNotice(false);
147 onHandle?.({
148 pressButton: client.pressButton,
149 rotate: client.rotate,
150 inputConnected: connected,
151 });
152 },
153 },
154 );
155 clientRef.current = client;
156 setPhoneUnavailable(false);
157 setHasFrame(false);
158 setMjpegUrl(null);
159 setInputState({ connected: false });
160 client.start();
161 onHandle?.({ pressButton: client.pressButton, rotate: client.rotate, inputConnected: false });
162 return () => {
163 cancelPhoneInput();
164 client.stop();
165 clientRef.current = null;
166 onHandle?.(null);
167 onScreen?.(null);
168 setScreen(null);
169 };
170 }, [
171 access,
172 cancelPhoneInput,
173 onHandle,
174 onPhoneUnavailable,
175 onScreen,
176 props.deviceId,
177 props.environmentId,
178 props.platform,
179 props.visible,
180 ]);
181
182 // Displayed aspect ratio (width / height) of the device as the user sees it.
183 const aspect = useMemo(() => {
184 if (!screen) return props.platform === "ios" ? 9 / 19.5 : 9 / 20;
185 const landscape =
186 screen.orientation === "landscape_left" || screen.orientation === "landscape_right";
187 const w = landscape
188 ? Math.max(screen.width, screen.height)
189 : Math.min(screen.width, screen.height);
190 const h = landscape
191 ? Math.min(screen.width, screen.height)
192 : Math.max(screen.width, screen.height);
193 return w / h;
194 }, [props.platform, screen]);
195
196 // Encoder restarts and video reconnects retain the decoded frame and viewer
197 // while input is still connected and the next keyframe is on its way.
198 const retainingFrame =
199 status === "connecting" && hasFrame && inputState.connected && screen !== null;
200 const showPhone =
201 props.allowPhoneView &&
202 (status === "streaming" || retainingFrame) &&
203 props.visible &&
204 presentation === "phone" &&
205 !phoneUnavailable &&
206 !mjpegUrl &&
207 !props.axOverlay &&
208 (!isDuo || screen?.supportsHingeAngle === true);
209 useEffect(() => {
210 if (!retainingFrame || !showPhone) return;
211 const timeout = window.setTimeout(() => setShowRestartNotice(true), 2_000);
212 return () => window.clearTimeout(timeout);
213 }, [retainingFrame, showPhone]);
214 const controlsInset = props.renderControls && !showPhone ? CONTROLS_RAIL_WIDTH : 0;
215
216 // The frame is the largest box at `aspect` that fits the container, so a
217 // narrow panel shows a shorter phone rather than a squeezed one. CSS
218 // `aspect-ratio` alone cannot do this: with the height pinned to 100% the
219 // width clamp wins and distorts the drawn frame.
220 const hostRef = useRef<HTMLDivElement | null>(null);
221 const [host, setHost] = useState({ width: 0, height: 0 });
222 useEffect(() => {
223 const element = hostRef.current;
224 if (!element) return;
225 const update = () => {
226 const rect = element.getBoundingClientRect();
227 setHost((current) =>
228 current.width === rect.width && current.height === rect.height
229 ? current
230 : { width: rect.width, height: rect.height },
231 );
232 };
233 update();
234 return observeResize(element, update);
235 }, []);
236 const frame = useMemo(() => {
237 return fitDeviceFrame(aspect, host.width, host.height, controlsInset);
238 }, [aspect, controlsInset, host]);
239
240 // serve-sim streams the raw framebuffer; rotate the display for a device
241 // that reports landscape while its frames stay portrait.
242 const rotation = useMemo(() => {
243 if (props.platform !== "ios" || !screen || screen.width > screen.height) return 0;
244 switch (screen.orientation) {
245 case "landscape_left":
246 return 90;
247 case "landscape_right":
248 return -90;
249 case "portrait_upside_down":
250 return 180;
251 default:
252 return 0;
253 }
254 }, [props.platform, screen]);
255
256 // A sideways rotation draws the raw portrait frame into a landscape box:
257 // the media element takes the transposed size and is rotated about the
258 // box's center.
259 const sideways = rotation === 90 || rotation === -90;
260 const mediaStyle: React.CSSProperties = sideways
261 ? {
262 width: frame.height,
263 height: frame.width,
264 left: (frame.width - frame.height) / 2,
265 top: (frame.height - frame.width) / 2,
266 transform: `rotate(${rotation}deg)`,
267 }
268 : {
269 width: frame.width,
270 height: frame.height,
271 ...(rotation ? { transform: `rotate(${rotation}deg)` } : {}),
272 };
273
274 // The accessibility tree is polled while the overlay is on; each poll is
275 // one JSON fetch, so there is nothing to repaint between polls.
276 const [axElements, setAxElements] = useState<ReadonlyArray<DeviceAxElement>>([]);
277 useEffect(() => {
278 if (!props.axOverlay || !access || !props.visible) return;
279 const target = { access, platform: props.platform, deviceId: props.deviceId };
280 let controller: AbortController | null = null;
281 let timer: ReturnType<typeof setTimeout> | null = null;
282 let stopped = false;
283 const poll = async () => {
284 controller = new AbortController();
285 try {
286 const tree = await fetchDeviceAxTree(target, controller.signal);
287 if (!stopped) setAxElements(tree.elements);
288 } catch {
289 // Keep the last good tree; the next poll retries.
290 }
291 if (!stopped) timer = setTimeout(() => void poll(), AX_POLL_INTERVAL_MS);
292 };
293 void poll();
294 return () => {
295 stopped = true;
296 controller?.abort();
297 if (timer) clearTimeout(timer);
298 setAxElements([]);
299 };
300 }, [access, props.axOverlay, props.deviceId, props.platform, props.visible]);
301
302 const pointerActive = useRef(false);
303 const normalizedPoint = (event: React.PointerEvent<HTMLElement>) => {
304 const rect = event.currentTarget.getBoundingClientRect();
305 const x = (event.clientX - rect.left) / rect.width;
306 const y = (event.clientY - rect.top) / rect.height;
307 return { x: Math.min(1, Math.max(0, x)), y: Math.min(1, Math.max(0, y)) };
308 };
309
310 const phoneUnavailableReason =
311 isDuo && !screen?.supportsHingeAngle
312 ? "iPhone Duo 3D requires Device Hub 0.11.0 or newer"
313 : phoneUnavailable
314 ? "3D is unavailable on this browser"
315 : mjpegUrl
316 ? "3D requires the H.264 stream"
317 : props.axOverlay
318 ? "Turn off accessibility frames to use 3D"
319 : null;
320
321 const keyboardSource = deviceKeyboard(props.platform, props.deviceName ?? "");
322 const resetView = useCallback(() => {
323 if (!isDuo) {
324 const orientation = keyboardAttached ? "landscape_right" : "portrait";
325 if (screen?.orientation !== orientation) clientRef.current?.setOrientation(orientation);
326 }
327 resetViewRef.current?.();
328 }, [isDuo, keyboardAttached, screen?.orientation]);
329 const profile = resolveDeviceShape({
330 platform: props.platform,
331 name: props.deviceName ?? "",
332 portraitAspect: Math.min(aspect, 1 / aspect),
333 });
334
335 return (
336 <div
337 className={cn(
338 "relative flex size-full min-h-0 min-w-0",
339 props.allowPhoneView ? "bg-background" : "bg-black/90",
340 )}
341 >
342 {props.renderControls ? (
343 <DeviceControlsSlot
344 renderControls={props.renderControls}
345 view={{
346 phone: !!showPhone,
347 streaming: status === "streaming",
348 phoneUnavailableReason,
349 foldingControls:
350 props.platform === "android" && access ? (
351 <DeviceAndroidFoldControls
352 key={`${props.hostId}:${props.deviceId}`}
353 access={access}
354 deviceId={props.deviceId}
355 visible={props.visible}
356 enabled={status === "streaming"}
357 screenWidth={screen?.width}
358 screenHeight={screen?.height}
359 onFoldAngle={setFoldAngle}
360 />
361 ) : showPhone && isDuo && screen?.supportsHingeAngle ? (
362 <DeviceDuoControls
363 screen={screen}
364 state={duoControl}
365 enabled={inputState.connected}
366 onCommand={(command) => {
367 cancelPhoneInput();
368 clientRef.current?.controlDuo(command);
369 }}
370 />
371 ) : null,
372 keyboard:
373 showPhone && keyboardSource
374 ? {
375 attached: keyboardAttached,
376 toggle: () => {
377 cancelPhoneInput();
378 if (!keyboardAttached && screen?.orientation !== "landscape_right")
379 clientRef.current?.setOrientation("landscape_right");
380 setKeyboardAttached(!keyboardAttached);
381 },
382 }
383 : null,
384 showPhone: () => setPresentation("phone"),
385 showFlat: () => setPresentation("flat"),
386 }}
387 onResetView={resetView}
388 />
389 ) : null}
390 <div
391 ref={hostRef}
392 className={cn(
393 "relative flex min-h-0 min-w-0 flex-1 items-center justify-center overflow-hidden outline-none",
394 controlsInset && "pr-14",
395 )}
396 tabIndex={0}
397 role="application"
398 aria-label={`${props.platform === "ios" ? "iOS Simulator" : "Android Emulator"} screen`}
399 onKeyDown={(event) => {
400 if (event.target !== event.currentTarget) return;
401 if (event.metaKey && !["r", "R"].includes(event.key)) return;
402 event.preventDefault();
403 clientRef.current?.sendKey(event.nativeEvent, "down");
404 }}
405 onKeyUp={(event) => {
406 if (event.target !== event.currentTarget) return;
407 clientRef.current?.sendKey(event.nativeEvent, "up");
408 }}
409 >
410 <div
411 className={cn("relative select-none", showPhone && "invisible pointer-events-none")}
412 style={{ width: frame.width, height: frame.height }}
413 onPointerDown={(event) => {
414 event.currentTarget.setPointerCapture(event.pointerId);
415 (event.currentTarget.parentElement as HTMLElement | null)?.focus();
416 pointerActive.current = true;
417 const { x, y } = normalizedPoint(event);
418 clientRef.current?.sendTouch("begin", x, y);
419 }}
420 onPointerMove={(event) => {
421 if (!pointerActive.current) return;
422 const { x, y } = normalizedPoint(event);
423 clientRef.current?.sendTouch("move", x, y);
424 }}
425 onPointerUp={(event) => {
426 if (!pointerActive.current) return;
427 pointerActive.current = false;
428 const { x, y } = normalizedPoint(event);
429 clientRef.current?.sendTouch("end", x, y);
430 }}
431 onPointerCancel={(event) => {
432 if (!pointerActive.current) return;
433 pointerActive.current = false;
434 const { x, y } = normalizedPoint(event);
435 clientRef.current?.sendTouch("end", x, y);
436 }}
437 >
438 <canvas
439 ref={canvasRef}
440 className={cn("absolute top-0 left-0", mjpegUrl && "hidden")}
441 style={mediaStyle}
442 />
443 {props.visible && access && mjpegUrl ? (
444 <img
445 key={mjpegGeneration}
446 ref={attachMjpegImage}
447 alt=""
448 draggable={false}
449 className="absolute top-0 left-0 object-contain"
450 style={mediaStyle}
451 />
452 ) : null}
453 {axElements.length > 0 ? (
454 <div className="pointer-events-none absolute inset-0" aria-hidden>
455 {axElements.map((element) => (
456 <div
457 key={element.id}
458 className="absolute border border-info/80 bg-info/10"
459 style={{
460 left: `${element.x * 100}%`,
461 top: `${element.y * 100}%`,
462 width: `${element.width * 100}%`,
463 height: `${element.height * 100}%`,
464 }}
465 >
466 {element.label ? (
467 <span className="absolute -top-3.5 left-0 max-w-full truncate rounded-sm bg-info px-1 text-3xs leading-3.5 text-white">
468 {element.label}
469 </span>
470 ) : null}
471 </div>
472 ))}
473 </div>
474 ) : null}
475 </div>
476 {showPhone && isDuo && model ? (
477 <DeviceDuoViewport
478 onFrameListener={onFrameListener}
479 model={model}
480 controlError={duoControl.error}
481 hingePreview={
482 duoControl.requested?.control === "angle" ? duoControl.requested.value : null
483 }
484 source={canvasRef}
485 client={clientRef}
486 onInputCancel={onInputCancel}
487 onResetReady={onResetReady}
488 screen={screen}
489 onUnavailable={onPhoneUnavailable}
490 />
491 ) : showPhone ? (
492 <DevicePhoneViewport
493 profile={profile}
494 model={deviceModel(props.platform, props.deviceName ?? "")}
495 accessory={keyboardAttached ? keyboardSource : null}
496 source={canvasRef}
497 onFrameListener={onFrameListener}
498 client={clientRef}
499 onInputCancel={onInputCancel}
500 onResetReady={onResetReady}
501 screen={screen}
502 foldAngle={foldAngle}
503 onUnavailable={onPhoneUnavailable}
504 />
505 ) : null}
506 {props.allowPhoneView && !props.renderControls && status === "streaming" ? (
507 <div className="absolute top-3 left-3 flex gap-1 rounded-lg border border-border/50 bg-background/90 p-1 shadow-sm">
508 <Button
509 variant={showPhone ? "secondary" : "ghost"}
510 size="xs"
511 aria-pressed={!!showPhone}
512 disabled={!!phoneUnavailableReason}
513 title={phoneUnavailableReason ?? "Show 3D phone"}
514 onClick={() => setPresentation("phone")}
515 >
516 3D
517 </Button>
518 <Button
519 variant={!showPhone ? "secondary" : "ghost"}
520 size="xs"
521 aria-pressed={!showPhone}
522 onClick={() => setPresentation("flat")}
523 >
524 Flat
525 </Button>
526 </div>
527 ) : null}
528 {status === "streaming" && !inputState.connected ? (
529 <div className="pointer-events-none absolute inset-x-0 bottom-0 flex justify-center p-2">
530 <span className="rounded-md bg-background/85 px-2 py-1 text-xs text-muted-foreground">
531 Input disconnected{inputState.detail ? ` (${inputState.detail})` : ""}, reconnecting…
532 </span>
533 </div>
534 ) : null}
535 {retainingFrame && showPhone && showRestartNotice ? (
536 <div className="pointer-events-none absolute inset-x-0 bottom-0 flex justify-center p-2">
537 <span className="rounded-md bg-background/85 px-2 py-1 text-xs text-muted-foreground">
538 Waiting for device video…
539 </span>
540 </div>
541 ) : null}
542 {status !== "streaming" && !(retainingFrame && showPhone) ? (
543 <div className="absolute inset-0">
544 <DeviceLoadingView
545 name={props.deviceName ?? "Device"}
546 description={props.deviceDescription ?? ""}
547 stage="stream"
548 message={status === "error" ? (detail ?? "Stream failed.") : "Connecting video…"}
549 error={status === "error"}
550 >
551 {status === "error" ? (
552 <Button
553 size="sm"
554 variant="outline"
555 onClick={() => {
556 // An expired ticket surfaces as unauthorized on restart and
557 // refreshes access through the effect; no need to mint one here.
558 clientRef.current?.stop();
559 setPhoneUnavailable(false);
560 clientRef.current?.start();
561 }}
562 >
563 Reconnect
564 </Button>
565 ) : null}
566 </DeviceLoadingView>
567 </div>
568 ) : null}
569 </div>
570 </div>
571 );
572}
573
574function DeviceControlsSlot(props: {
575 renderControls: (view: DeviceViewControls) => ReactNode;
576 view: Omit<DeviceViewControls, "resetView">;
577 onResetView: () => void;
578}) {
579 return props.renderControls({ ...props.view, resetView: props.onResetView });
580}