authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-09-24 18:07:33-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-24 18:07:33-07:00
log46f3c2ca2c99ceab0547452ac863232944621fce
treec91f46d82e70aef3f58aa7482f9d5a09e3dfe49f
parentfd465100581055383126aab922a8da2c5d7952fd
signature Signed by PGP key B5690EEEBB952194

feat(web): add iPhone Duo 3D controls (#12813)

25 files changed, 2951 insertions(+), 44 deletions(-)

apps/server/src/device/DeviceHubProxy.test.ts+26
......@@ -139,3 +139,29 @@ it.each([
139139 expect(await response.text()).not.toContain("private credential diagnostic");
140140 expect(requests).toEqual([]);
141141});
142
143it.each([1, 3])(
144 "forwards fixed Duo display %s through the authenticated read proxy",
145 async (panel) => {
146 const { handler, requests } = fixture([AuthOrchestrationReadScope]);
147 const route = `/vendor/serve-sim/helper/duo/panel/${panel}/stream.avcc`;
148 const response = await handler(
149 new Request(`http://t3.test/api/device-hub${route}?wsTicket=secret`),
150 );
151 expect(response.status).toBe(200);
152 await response.text();
153 expect(requests).toEqual([`http://hub.test${route}`]);
154 },
155);
156
157it.each(["/panel/2/stream.avcc", "/panel/1/webrtc/offer", "/panel/3/exec"])(
158 "rejects unsupported Duo route %s",
159 async (route) => {
160 const { handler, requests } = fixture([AuthOrchestrationReadScope]);
161 const response = await handler(
162 new Request(`http://t3.test/api/device-hub/vendor/serve-sim/helper/duo${route}`),
163 );
164 expect(response.status).toBe(404);
165 expect(requests).toEqual([]);
166 },
167);
apps/server/src/device/DeviceHubProxy.ts+1
......@@ -42,6 +42,7 @@ const ALLOWED_PATHS: ReadonlyArray<RegExp> = [
4242 /^\/vendor\/serve-sim\/api\/screenshot$/,
4343 /^\/vendor\/serve-sim\/api\/event-log(\/events)?$/,
4444 /^\/vendor\/serve-sim\/helper\/[^/]+\/(stream\.mjpeg|stream\.avcc|config|health|ax|foreground)$/,
45 /^\/vendor\/serve-sim\/helper\/[^/]+\/panel\/(1|3)\/stream\.avcc$/,
4546 /^\/vendor\/serve-sim\/appstate$/,
4647 /^\/vendor\/serve-emu\/api\/(devices|screenshot|stream-mode|stream-settings|accessibility)$/,
4748 /^\/vendor\/serve-emu\/health$/,
apps/server/src/device/DeviceToolchain.ts+1-1
......@@ -26,7 +26,7 @@ import * as Semaphore from "effect/Semaphore";
2626import * as ProcessRunner from "../processRunner.ts";
2727
2828const DEVICE_HUB_PACKAGE = "expo-device-hub";
29export const DEVICE_HUB_VERSION = "0.10.1";
29export const DEVICE_HUB_VERSION = "0.12.0";
3030const AGENT_DEVICE_PACKAGE = "agent-device";
3131export const AGENT_DEVICE_VERSION = "0.21.12";
3232
apps/web/src/components/device/DeviceControlsRail.tsx+4-3
......@@ -57,9 +57,9 @@ export function DeviceControlsRail(props: {
5757 <aside
5858 aria-label="Device controls"
5959 data-layout="rail"
60 className="pointer-events-none absolute inset-y-0 right-0 z-10 flex w-14 flex-col items-center justify-center py-3 pr-2"
60 className="pointer-events-none absolute inset-y-0 right-0 z-10 flex w-14 flex-col items-center gap-2 overflow-y-auto [justify-content:safe_center] py-3 pr-2 [scrollbar-width:none]"
6161 >
62 <div className="pointer-events-auto flex max-h-full shrink-0 flex-col items-center gap-1 overflow-y-auto rounded-full border border-border/50 bg-background/80 p-2 shadow-sm [scrollbar-width:none]">
62 <div className="pointer-events-auto flex shrink-0 flex-col items-center gap-1 overflow-y-auto rounded-full border border-border/50 bg-background/80 p-2 shadow-sm [scrollbar-width:none]">
6363 <RailButton
6464 tooltipSide={popupSide}
6565 label="Home"
......@@ -255,11 +255,12 @@ export function DeviceControlsRail(props: {
255255 </RailButton>
256256 ) : null}
257257 {view.phone ? (
258 <RailButton tooltipSide={popupSide} label="Reset phone view" onClick={view.resetView}>
258 <RailButton tooltipSide={popupSide} label="Restore 3D view" onClick={view.resetView}>
259259 <Maximize />
260260 </RailButton>
261261 ) : null}
262262 </div>
263 {view.foldingControls}
263264 </aside>
264265 );
265266}
apps/web/src/components/device/DeviceDuoControls.tsx created+75
......@@ -0,0 +1,75 @@
1import {
2 DUO_POSES,
3 type DuoCommand,
4 type DuoControlState,
5} from "@t3tools/client-runtime/device/duo-control";
6import type { DeviceScreenSize } from "@t3tools/client-runtime/device/stream";
7import { DeviceDuoGlyph } from "./DeviceDuoGlyph";
8import { Button } from "~/components/ui/button";
9import { Tooltip, TooltipPopup, TooltipTrigger } from "~/components/ui/tooltip";
10
11/** Physical presets live beside the device. Pinching supplies continuous hinge control. */
12export function DeviceDuoControls(props: {
13 screen: DeviceScreenSize;
14 state: DuoControlState;
15 enabled: boolean;
16 onCommand: (command: DuoCommand) => void;
17}) {
18 const angle = props.screen.hingeAngle;
19 const fold = angle == null ? null : angle === 0 ? "closed" : angle === 180 ? "open" : "book";
20 const selected = (id: (typeof DUO_POSES)[number]["id"]) =>
21 id === "laptop" || id === "tent" ? props.screen.hingePose === id : fold === id;
22 return (
23 <div aria-label="iPhone Duo stands" className="flex flex-col items-center gap-2">
24 {([DUO_POSES.slice(0, 3), DUO_POSES.slice(3)] as const).map((poses, index) => (
25 <div
26 key={poses[0]?.id}
27 role="group"
28 aria-label={index === 0 ? "Fold shape" : "Device stance"}
29 className="pointer-events-auto flex shrink-0 flex-col items-center gap-1 rounded-full border border-border/50 bg-background/80 p-1 shadow-sm"
30 >
31 {poses.map((pose) => (
32 <Tooltip key={pose.id}>
33 <TooltipTrigger
34 render={
35 <Button
36 size="icon"
37 variant={selected(pose.id) ? "secondary" : "ghost"}
38 disabled={!props.enabled}
39 aria-label={`${pose.label} stand`}
40 aria-pressed={selected(pose.id)}
41 data-pressed={selected(pose.id) ? "" : undefined}
42 onClick={() => props.onCommand({ control: "pose", value: pose.id })}
43 />
44 }
45 >
46 <DeviceDuoGlyph pose={pose.id} />
47 </TooltipTrigger>
48 <TooltipPopup side="left">
49 {pose.label}
50 {pose.id === "book" ? " / bookshelf" : ""}
51 </TooltipPopup>
52 </Tooltip>
53 ))}
54 </div>
55 ))}
56 {props.state.error ? (
57 <Tooltip>
58 <TooltipTrigger
59 render={
60 <span
61 tabIndex={0}
62 role="alert"
63 aria-label={props.state.error}
64 className="pointer-events-auto text-xs text-destructive"
65 >
66 !
67 </span>
68 }
69 />
70 <TooltipPopup side="left">{props.state.error}</TooltipPopup>
71 </Tooltip>
72 ) : null}
73 </div>
74 );
75}
apps/web/src/components/device/DeviceDuoGlyph.tsx created+43
......@@ -0,0 +1,43 @@
1import type { DuoPose } from "@t3tools/client-runtime/device/duo-control";
2
3// Bitrig 0.25's toolbar glyphs. The fold outlines follow the native toolbar;
4// the rounded stance contours are ported from SimulatorFoldingPoseGlyph's SwiftUI paths.
5const stancePaths = {
6 laptop:
7 "M2.7351 12.9857 L15.8128 11.4159 L14.4114 3.3787 C14.1943 2.134 13.1275 1.2319 12.0283 1.3639 L2.9308 2.4559 C1.8317 2.5879 1.1167 3.7038 1.3337 4.9485 L2.7351 12.9857 Z M2.7351 12.9857 L15.8128 11.4159 L19.9103 12.8914 C20.5449 13.1199 20.1683 13.4121 19.0692 13.5441 L9.9717 14.6361 C8.8725 14.7681 7.4672 14.6898 6.8326 14.4612 L2.7351 12.9857 Z",
8 tent: "M3.2522 2.7267 L18.7478 1.4816 L16.696 9.7017 C16.3782 10.9748 15.0649 12.0915 13.7626 12.1962 L2.983 13.0623 C1.6807 13.167 0.8826 12.2199 1.2004 10.9468 L3.2522 2.7267 Z M3.2522 2.7267 L18.7478 1.4816 L20.7996 10.6909 C21.1174 12.1172 20.3193 13.3581 19.017 13.4628 L8.2374 14.3289 C6.9351 14.4336 5.6218 13.3623 5.304 11.936 L3.2522 2.7267 Z",
9} as const;
10
11export function DeviceDuoGlyph({ pose }: { pose: DuoPose }) {
12 const stance = pose === "laptop" || pose === "tent";
13 return (
14 <svg
15 viewBox={stance ? "0 0 22 16" : "0 0 24 32"}
16 fill="none"
17 stroke="currentColor"
18 strokeWidth={stance ? 1.25 : 1.8}
19 strokeLinecap="round"
20 strokeLinejoin="round"
21 className="size-7 shrink-0"
22 aria-hidden
23 >
24 {stance ? <path d={stancePaths[pose]} /> : null}
25 {pose === "closed" ? (
26 <>
27 <rect x="2" y="9" width="20" height="13" rx="2.2" />
28 <path d="M19.3 12.5v6" />
29 <circle cx="5" cy="12" r=".5" fill="currentColor" stroke="none" />
30 </>
31 ) : null}
32 {pose === "book" ? (
33 <path d="M4.2 6.5h15.6c1 0 1.6.8 1.2 1.7L17.7 16l3.3 7.8c.4.9-.2 1.7-1.2 1.7H4.2c-1 0-1.6-.8-1.2-1.7L6.3 16 3 8.2c-.4-.9.2-1.7 1.2-1.7Z" />
34 ) : null}
35 {pose === "open" ? (
36 <>
37 <rect x="2" y="2" width="20" height="28" rx="2.4" />
38 <path d="M10 27h4" />
39 </>
40 ) : null}
41 </svg>
42 );
43}
apps/web/src/components/device/DeviceDuoViewport.tsx created+205
......@@ -0,0 +1,205 @@
1import { useEffect, useRef, type RefObject } from "react";
2import type { DuoViewer } from "@t3tools/client-runtime/device/duo-viewer";
3import type { DeviceModelSource } from "@t3tools/client-runtime/device/model";
4import { createPhoneInteraction } from "@t3tools/client-runtime/device/phone-interaction";
5import { createCanvasFrameSink } from "@t3tools/client-runtime/device/frame";
6import type { DeviceScreenSize, DeviceStreamClient } from "@t3tools/client-runtime/device/stream";
7import { createDuoPinch } from "@t3tools/client-runtime/device/duo-control";
8import { bindPhoneTrackpad } from "./phoneTrackpad";
9
10const loadDuoViewer = () => import("@t3tools/client-runtime/device/duo-viewer");
11
12/** Web shell for the framework-independent viewer. The decoded screen and input connection remain owned by DeviceStreamView. */
13export function DeviceDuoViewport(props: {
14 readonly model: DeviceModelSource;
15 readonly source: RefObject<HTMLCanvasElement | null>;
16 readonly onFrameListener: (listener: (() => void) | null) => void;
17 readonly onResetReady: (reset: (() => void) | null) => void;
18 readonly onInputCancel: (cancel: (() => void) | null) => void;
19 readonly client: RefObject<DeviceStreamClient | null>;
20 readonly screen: DeviceScreenSize | null;
21 readonly hingePreview: number | null;
22 readonly controlError: string | null;
23 readonly onUnavailable: () => void;
24}) {
25 const hostRef = useRef<HTMLDivElement | null>(null);
26 const canvasRef = useRef<HTMLCanvasElement | null>(null);
27 const viewerRef = useRef<DuoViewer | null>(null);
28 const interactionRef = useRef<ReturnType<typeof createPhoneInteraction> | null>(null);
29 const pinchRef = useRef<ReturnType<typeof createDuoPinch> | null>(null);
30 const trackpadRef = useRef<ReturnType<typeof bindPhoneTrackpad> | null>(null);
31 const screenRef = useRef(props.screen);
32 const modelRef = useRef(props.model);
33 const previewRef = useRef(props.hingePreview);
34 const { source, client, onInputCancel, onResetReady, onUnavailable, onFrameListener } = props;
35
36 useEffect(() => {
37 screenRef.current = props.screen;
38 modelRef.current = props.model;
39 interactionRef.current?.end();
40 viewerRef.current?.setScreen(props.screen);
41 }, [props.screen, props.model]);
42
43 useEffect(() => {
44 if (props.controlError || !props.screen) {
45 trackpadRef.current?.cancel();
46 viewerRef.current?.rejectOrientation();
47 }
48 }, [props.controlError, props.screen]);
49
50 useEffect(() => {
51 previewRef.current = props.hingePreview;
52 if (!pinchRef.current?.active) viewerRef.current?.setHingePreview(props.hingePreview);
53 }, [props.hingePreview]);
54
55 useEffect(() => {
56 const host = hostRef.current;
57 const canvas = canvasRef.current;
58 const decoded = source.current;
59 if (!host || !canvas || !decoded) return;
60 let disposed = false;
61 let trackpad: ReturnType<typeof bindPhoneTrackpad> | null = null;
62 let stopTrackpadEnd: (() => void) | undefined;
63 const resize = () => {
64 const { width, height } = host.getBoundingClientRect();
65 viewerRef.current?.resize(width, height, window.devicePixelRatio);
66 };
67 const observer = new ResizeObserver(resize);
68 observer.observe(host);
69 const blur = () => {
70 interactionRef.current?.end();
71 trackpad?.cancel();
72 viewerRef.current?.cancelInput();
73 };
74 onInputCancel(blur);
75 window.addEventListener("blur", blur);
76 void loadDuoViewer()
77 .then(({ createDuoViewer }) => {
78 if (disposed) return;
79 const sources = {
80 1: document.createElement("canvas"),
81 3: document.createElement("canvas"),
82 };
83 const viewer = createDuoViewer({
84 canvas,
85 sources,
86 onUnavailable,
87 onPanelRequested: (panel) =>
88 client.current?.controlDuo({
89 control: "physical",
90 value: panel === 1 ? "facedown" : "faceup",
91 }),
92 onOrientationRequested: (value) =>
93 client.current?.controlDuo({ control: "orientation", value }),
94 onModelError: (cause) => console.warn("Device 3D asset could not load", cause),
95 model: modelRef.current,
96 });
97 viewerRef.current = viewer;
98 onResetReady(viewer.resetPose);
99 viewer.setScreen(screenRef.current);
100 const primaryFrame = () => {
101 const panel = screenRef.current?.screenId;
102 if (panel === 1 || panel === 3) viewer.frameUpdated(panel, decoded);
103 };
104 onFrameListener(primaryFrame);
105 primaryFrame();
106 viewer.setHingePreview(previewRef.current);
107 client.current?.setDuoPanels({
108 onScreen(next) {
109 screenRef.current = next;
110 interactionRef.current?.end();
111 viewer.setScreen(next);
112 },
113 cover: createCanvasFrameSink(sources[1], () => viewer.frameUpdated(1)),
114 inner: createCanvasFrameSink(sources[3], () => viewer.frameUpdated(3)),
115 });
116 interactionRef.current = createPhoneInteraction({
117 screenPoint: (point, captured) => viewer.screenPoint(point.x, point.y, captured),
118 touch: (phase, point) => client.current?.sendRawTouch(phase, point.x, point.y),
119 orbit: viewer.orbit,
120 zoomBy: () => {},
121 onInteractionActive: viewer.setInteractionActive,
122 });
123 const pinch = createDuoPinch({
124 angle: () =>
125 previewRef.current ??
126 screenRef.current?.hingeAngle ??
127 (screenRef.current?.screenId === 1 ? 0 : 180),
128 contains: (x, y) => !!screenRef.current && viewer.beginHinge(x, y),
129 change: (angle) => {
130 viewer.setHingePreview(angle ?? previewRef.current);
131 if (angle !== null) client.current?.controlDuo({ control: "angle", value: angle });
132 },
133 });
134 pinchRef.current = pinch;
135 trackpad = bindPhoneTrackpad(canvas, interactionRef.current, {
136 begin(x, y) {
137 interactionRef.current?.end();
138 return pinch.begin(x, y);
139 },
140 move: pinch.move,
141 end: pinch.end,
142 });
143 trackpadRef.current = trackpad;
144 stopTrackpadEnd = window.desktopBridge?.onTrackpadScrollEnd?.(() => trackpad?.endOrbit());
145 resize();
146 })
147 .catch(() => {
148 if (!disposed) onUnavailable();
149 });
150 return () => {
151 disposed = true;
152 stopTrackpadEnd?.();
153 trackpad?.dispose();
154 trackpadRef.current = null;
155 pinchRef.current = null;
156 interactionRef.current?.end();
157 interactionRef.current = null;
158 onInputCancel(null);
159 onResetReady(null);
160 onFrameListener(null);
161 client.current?.setDuoPanels(null);
162 observer.disconnect();
163 window.removeEventListener("blur", blur);
164 viewerRef.current?.dispose();
165 viewerRef.current = null;
166 };
167 }, [onInputCancel, onResetReady, client, onUnavailable, source, onFrameListener]);
168
169 const point = (event: React.PointerEvent<HTMLCanvasElement>) => {
170 const rect = event.currentTarget.getBoundingClientRect();
171 return {
172 x: (event.clientX - rect.left) / rect.width,
173 y: (event.clientY - rect.top) / rect.height,
174 };
175 };
176 return (
177 <div className="absolute inset-0">
178 <div ref={hostRef} className="absolute inset-0">
179 <div
180 aria-hidden
181 className="pointer-events-none absolute bottom-[6%] left-1/2 h-5 w-2/5 -translate-x-1/2 rounded-full bg-foreground/10 blur-xl"
182 />
183 <canvas
184 ref={canvasRef}
185 aria-label="Interactive 3D iPhone Duo. Drag the screen to interact. Drag outside it or swipe with two fingers to turn. Pinch over the device to open or close its hinge."
186 className="size-full touch-none"
187 onPointerDown={(event) => {
188 if (event.button !== 0) return;
189 if (!interactionRef.current?.begin(event.pointerId, point(event), event.altKey)) return;
190 event.preventDefault();
191 event.currentTarget.setPointerCapture(event.pointerId);
192 (event.currentTarget.closest('[role="application"]') as HTMLElement | null)?.focus();
193 }}
194 onPointerMove={(event) => interactionRef.current?.move(event.pointerId, point(event))}
195 onPointerUp={(event) => {
196 interactionRef.current?.move(event.pointerId, point(event));
197 interactionRef.current?.end(event.pointerId);
198 }}
199 onPointerCancel={(event) => interactionRef.current?.end(event.pointerId)}
200 onLostPointerCapture={(event) => interactionRef.current?.end(event.pointerId)}
201 />
202 </div>
203 </div>
204 );
205}
apps/web/src/components/device/DeviceStreamView.tsx+61-22
......@@ -8,6 +8,9 @@ import { createCanvasFrameSink } from "@t3tools/client-runtime/device/frame";
88import { resolveDeviceShape } from "@t3tools/client-runtime/device/shape-profile";
99import { deviceKeyboard, deviceModel } from "./deviceModels";
1010import { fitDeviceFrame } from "./deviceFrameLayout";
11import { DeviceDuoViewport } from "./DeviceDuoViewport";
12import { DeviceDuoControls } from "./DeviceDuoControls";
13import type { DuoControlState } from "@t3tools/client-runtime/device/duo-control";
1114import { DevicePhoneViewport } from "./DevicePhoneViewport";
1215import { DeviceLoadingView } from "./DeviceLoadingView";
1316import { type DeviceAxElement, fetchDeviceAxTree } from "./deviceHubApi";
......@@ -29,6 +32,7 @@ export interface DeviceViewControls {
2932 readonly showPhone: () => void;
3033 readonly showFlat: () => void;
3134 readonly resetView: () => void;
35 readonly foldingControls?: ReactNode;
3236 readonly keyboard: { readonly attached: boolean; readonly toggle: () => void } | null;
3337}
3438
......@@ -61,6 +65,13 @@ export function DeviceStreamView(props: {
6165 readonly onHandle?: (handle: DeviceStreamHandle | null) => void;
6266 readonly onScreen?: (screen: DeviceScreenSize | null) => void;
6367}) {
68 const [duoControl, setDuoControl] = useState<DuoControlState>({
69 pending: false,
70 requested: null,
71 error: null,
72 });
73 const model = deviceModel(props.platform, props.deviceName ?? "");
74 const isDuo = model?.id === "iphone-duo";
6475 const [presentation, setPresentation] = useState<"phone" | "flat">("phone");
6576 const [keyboardAttached, setKeyboardAttached] = useState(false);
6677 const [phoneUnavailable, setPhoneUnavailable] = useState(false);
......@@ -105,6 +116,8 @@ export function DeviceStreamView(props: {
105116 { platform: props.platform, deviceId: props.deviceId, access },
106117 createCanvasFrameSink(canvas, () => frameListenerRef.current?.()),
107118 {
119 onDuoControl: setDuoControl,
120 onDuoUnavailable: onPhoneUnavailable,
108121 onStatus: (next, nextDetail) => {
109122 setStatus(next);
110123 setDetail(nextDetail);
......@@ -148,6 +161,7 @@ export function DeviceStreamView(props: {
148161 access,
149162 cancelPhoneInput,
150163 onHandle,
164 onPhoneUnavailable,
151165 onScreen,
152166 props.deviceId,
153167 props.environmentId,
......@@ -176,7 +190,8 @@ export function DeviceStreamView(props: {
176190 presentation === "phone" &&
177191 !phoneUnavailable &&
178192 !mjpegUrl &&
179 !props.axOverlay;
193 !props.axOverlay &&
194 (!isDuo || screen?.supportsHingeAngle === true);
180195 const controlsInset = props.renderControls && !showPhone ? CONTROLS_RAIL_WIDTH : 0;
181196
182197 // The frame is the largest box at `aspect` that fits the container, so a
......@@ -275,20 +290,25 @@ export function DeviceStreamView(props: {
275290 return { x: Math.min(1, Math.max(0, x)), y: Math.min(1, Math.max(0, y)) };
276291 };
277292
278 const phoneUnavailableReason = phoneUnavailable
279 ? "3D is unavailable on this browser"
280 : mjpegUrl
281 ? "3D requires the H.264 stream"
282 : props.axOverlay
283 ? "Turn off accessibility frames to use 3D"
284 : null;
293 const phoneUnavailableReason =
294 isDuo && !screen?.supportsHingeAngle
295 ? "iPhone Duo 3D requires Device Hub 0.11.0 or newer"
296 : phoneUnavailable
297 ? "3D is unavailable on this browser"
298 : mjpegUrl
299 ? "3D requires the H.264 stream"
300 : props.axOverlay
301 ? "Turn off accessibility frames to use 3D"
302 : null;
285303
286304 const keyboardSource = deviceKeyboard(props.platform, props.deviceName ?? "");
287305 const resetView = useCallback(() => {
288 const orientation = keyboardAttached ? "landscape_right" : "portrait";
289 if (screen?.orientation !== orientation) clientRef.current?.setOrientation(orientation);
306 if (!isDuo) {
307 const orientation = keyboardAttached ? "landscape_right" : "portrait";
308 if (screen?.orientation !== orientation) clientRef.current?.setOrientation(orientation);
309 }
290310 resetViewRef.current?.();
291 }, [keyboardAttached, screen?.orientation]);
311 }, [isDuo, keyboardAttached, screen?.orientation]);
292312 const profile = resolveDeviceShape({
293313 platform: props.platform,
294314 name: props.deviceName ?? "",
......@@ -309,6 +329,18 @@ export function DeviceStreamView(props: {
309329 phone: !!showPhone,
310330 streaming: status === "streaming",
311331 phoneUnavailableReason,
332 foldingControls:
333 showPhone && isDuo && screen?.supportsHingeAngle ? (
334 <DeviceDuoControls
335 screen={screen}
336 state={duoControl}
337 enabled={inputState.connected}
338 onCommand={(command) => {
339 cancelPhoneInput();
340 clientRef.current?.controlDuo(command);
341 }}
342 />
343 ) : null,
312344 keyboard:
313345 showPhone && keyboardSource
314346 ? {
......@@ -413,7 +445,22 @@ export function DeviceStreamView(props: {
413445 </div>
414446 ) : null}
415447 </div>
416 {showPhone ? (
448 {showPhone && isDuo && model ? (
449 <DeviceDuoViewport
450 onFrameListener={onFrameListener}
451 model={model}
452 controlError={duoControl.error}
453 hingePreview={
454 duoControl.requested?.control === "angle" ? duoControl.requested.value : null
455 }
456 source={canvasRef}
457 client={clientRef}
458 onInputCancel={onInputCancel}
459 onResetReady={onResetReady}
460 screen={screen}
461 onUnavailable={onPhoneUnavailable}
462 />
463 ) : showPhone ? (
417464 <DevicePhoneViewport
418465 profile={profile}
419466 model={deviceModel(props.platform, props.deviceName ?? "")}
......@@ -433,16 +480,8 @@ export function DeviceStreamView(props: {
433480 variant={showPhone ? "secondary" : "ghost"}
434481 size="xs"
435482 aria-pressed={!!showPhone}
436 disabled={phoneUnavailable || !!mjpegUrl || !!props.axOverlay}
437 title={
438 phoneUnavailable
439 ? "3D is unavailable on this browser"
440 : mjpegUrl
441 ? "3D requires the H.264 stream"
442 : props.axOverlay
443 ? "Turn off accessibility frames to use 3D"
444 : "Show 3D phone"
445 }
483 disabled={!!phoneUnavailableReason}
484 title={phoneUnavailableReason ?? "Show 3D phone"}
446485 onClick={() => setPresentation("phone")}
447486 >
448487 3D
apps/web/src/components/device/deviceModels.ts+2
......@@ -4,6 +4,7 @@ import {
44 type DeviceModelSource,
55} from "@t3tools/client-runtime/device/model";
66import type { DevicePlatform } from "@t3tools/contracts";
7import iphoneDuo from "./models/iphone-duo.glb?url";
78import iphone18Pro from "./models/iphone-18-pro.glb?url";
89import iphone18ProMax from "./models/iphone-18-pro-max.glb?url";
910import magicKeyboard from "./models/ipad-pro-13-m5-magic-keyboard.glb?url";
......@@ -11,6 +12,7 @@ import ipadPro13M5 from "./models/ipad-pro-13-m5.glb?url";
1112
1213// Bundled URLs follow the client origin in local, desktop, hosted and remote sessions.
1314const models: Record<DeviceModelSource["id"], DeviceModelSource> = {
15 "iphone-duo": { id: "iphone-duo", url: iphoneDuo },
1416 "iphone-18-pro": { id: "iphone-18-pro", url: iphone18Pro },
1517 "iphone-18-pro-max": { id: "iphone-18-pro-max", url: iphone18ProMax },
1618 "ipad-pro-13-m5": { id: "ipad-pro-13-m5", url: ipadPro13M5 },
apps/web/src/components/device/models/iphone-duo.glb created
Binary files /dev/null and b/apps/web/src/components/device/models/iphone-duo.glb differ
apps/web/src/components/device/models/sources.json+9-1
......@@ -6,7 +6,7 @@
66 "gltfTransform": "4.2.1",
77 "changes": [
88 "Extracted device bodies and a separate Magic Keyboard accessory; excluded unrelated parts and presentation duplicates.",
9 "Normalized portrait axes, centered the screen and uniformly scaled its height to 2.2 scene units.",
9 "Normalized conventional device portrait axes, centered their screen and uniformly scaled its height to 2.2 scene units. The Duo retains its upstream centimeter hinge rig.",
1010 "Replaced the baked marketing screen with a named placeholder for the live framebuffer.",
1111 "Resized textures to at most 1024 pixels and encoded them as WebP; retained body geometry."
1212 ]
......@@ -44,6 +44,14 @@
4444 "bodyNode": "zRrSLDpdYmKeRJQ",
4545 "screenNode": "lsDiIbtoSGSmWWZ",
4646 "accessoryNode": "PoBqSMmyhhcJsBX"
47 },
48 {
49 "id": "iphone-duo",
50 "file": "iphone-duo.glb",
51 "sourceUrl": "https://www.apple.com/105/media/us/iphone-duo/2026/9305e4b9-72d9-4c05-9381-b572adadd5e5/ar/iPhone_Duo_e-sim_Star-White_Variant.usdz",
52 "sourceSha256": "5cab2ea636da0bc0b06c8abf4809843498f7c1d680042b4f95e383b5c6a718b2",
53 "upstreamConversionCommit": "bb265b11c13b395e5302d121458e2d42224a2e9f",
54 "conversion": "Preserved the centimeter split hinge rig and three display meshes from serve-sim; resized textures to 1024 pixels and encoded them as WebP without simplifying or flattening the rig."
4755 }
4856 ]
4957}
docs/user/devices.md+6
......@@ -39,6 +39,12 @@ iOS, and power off. Close the tab to stop watching; the device keeps running
3939unless you power it off. Closed tabs stay closed after a reload. To watch the
4040device again, choose it from **+ → Device**.
4141
42Choose **3D view** to inspect supported devices while the live screen stays
43interactive. On iPhone Duo, use the fold and stance controls to change its
44physical pose, or pinch over the device to adjust the hinge. Turning the model
45to the other screen switches the live display and touch input to that screen.
46**Restore 3D view** returns the device to a screen-facing position.
47
4248## Tools
4349
4450The toolbar's **Tools** button opens a drawer for the open device. It shows the
packages/client-runtime/package.json+8
......@@ -290,6 +290,14 @@
290290 "./device/model": {
291291 "types": "./src/device/model.ts",
292292 "default": "./src/device/model.ts"
293 },
294 "./device/duo-viewer": {
295 "types": "./src/device/duoViewer.ts",
296 "default": "./src/device/duoViewer.ts"
297 },
298 "./device/duo-control": {
299 "types": "./src/device/duoControl.ts",
300 "default": "./src/device/duoControl.ts"
293301 }
294302 },
295303 "scripts": {
packages/client-runtime/src/device/duoControl.test.ts created+109
......@@ -0,0 +1,109 @@
1import { afterEach, expect, it, vi } from "vite-plus/test";
2import { createDuoControl, createDuoPinch, type DuoCommand } from "./duoControl.ts";
3afterEach(() => vi.useRealTimers());
4
5it("keeps a failed send visible, including a disconnect while draining queued motion", () => {
6 const send = vi.fn((_request: { requestId: number; command: DuoCommand }) => false);
7 const onChange = vi.fn();
8 const queue = createDuoControl({ send, onChange });
9 queue.enqueue({ control: "angle", value: 40 });
10 expect(onChange).toHaveBeenLastCalledWith({
11 pending: false,
12 requested: null,
13 error: "Device is disconnected.",
14 });
15 send.mockReturnValueOnce(true);
16 queue.enqueue({ control: "pose", value: "book" });
17 queue.enqueue({ control: "angle", value: 60 });
18 queue.receive({ requestId: 2, ok: true });
19 expect(onChange).toHaveBeenLastCalledWith({
20 pending: false,
21 requested: null,
22 error: "Device is disconnected.",
23 });
24});
25
26it("coalesces hinge edits behind acknowledgements and lets a preset replace queued edits", () => {
27 const send = vi.fn((_request: { requestId: number; command: DuoCommand }) => true);
28 const onChange = vi.fn();
29 const queue = createDuoControl({ send, onChange });
30 queue.enqueue({ control: "angle", value: 40 });
31 queue.enqueue({ control: "angle", value: 50 });
32 queue.enqueue({ control: "angle", value: 60 });
33 expect(send).toHaveBeenCalledTimes(1);
34 queue.receive({ requestId: 1, ok: true });
35 expect(send.mock.calls[1]?.[0]).toEqual({
36 requestId: 2,
37 command: { control: "angle", value: 60 },
38 });
39 queue.enqueue({ control: "angle", value: 100 });
40 queue.enqueue({ control: "pose", value: "tent" });
41 queue.receive({ requestId: 2, ok: true });
42 expect(send.mock.calls[2]?.[0]).toEqual({
43 requestId: 3,
44 command: { control: "pose", value: "tent" },
45 });
46 queue.receive({ requestId: 3, ok: true });
47 expect(onChange).toHaveBeenLastCalledWith({ pending: false, requested: null, error: null });
48 queue.clear();
49});
50
51it("drops queued commands on failure, timeout and disconnect; late replies cannot acknowledge later work", () => {
52 vi.useFakeTimers();
53 const send = vi.fn((_request: { requestId: number; command: DuoCommand }) => true);
54 const onChange = vi.fn();
55 const queue = createDuoControl({ send, onChange, timeoutMs: 100 });
56 queue.enqueue({ control: "angle", value: 90 });
57 queue.enqueue({ control: "angle", value: 100 });
58 vi.advanceTimersByTime(100);
59 expect(onChange.mock.lastCall?.[0].error).toContain("timed out");
60 queue.enqueue({ control: "pose", value: "open" });
61 queue.receive({ requestId: 1, ok: true });
62 expect(onChange.mock.lastCall?.[0].pending).toBe(true);
63 queue.enqueue({ control: "angle", value: 130 });
64 queue.receive({ requestId: 2, ok: false, error: "native refused" });
65 expect(onChange.mock.lastCall?.[0]).toEqual({
66 pending: false,
67 requested: null,
68 error: "native refused",
69 });
70 queue.enqueue({ control: "pose", value: "book" });
71 queue.enqueue({ control: "pose", value: "closed" });
72 queue.clear();
73 queue.receive({ requestId: 3, ok: true });
74 expect(send).toHaveBeenCalledTimes(3);
75 queue.enqueue({ control: "angle", value: Infinity });
76 expect(send).toHaveBeenCalledTimes(3);
77 queue.clear();
78});
79
80it("pinches only a hit device, accumulates independently of native readback, clamps and cancels", () => {
81 let confirmed = 90;
82 const change = vi.fn();
83 const pinch = createDuoPinch({
84 angle: () => confirmed,
85 contains: (x, y) => x > 0.2 && y > 0.2,
86 change,
87 });
88 expect(pinch.begin(0.1, 0.5)).toBe(false);
89 pinch.move(1);
90 expect(change).not.toHaveBeenCalled();
91 expect(pinch.begin(0.5, 0.5)).toBe(true);
92 pinch.move(0.25);
93 expect(change).toHaveBeenLastCalledWith(120);
94 confirmed = 100;
95 pinch.move(0.25);
96 expect(change).toHaveBeenLastCalledWith(150);
97 pinch.move(2);
98 expect(change).toHaveBeenLastCalledWith(180);
99 pinch.move(-3);
100 expect(change).toHaveBeenLastCalledWith(0);
101 pinch.move(NaN);
102 pinch.end();
103 expect(change).toHaveBeenLastCalledWith(null);
104 const count = change.mock.calls.length;
105 pinch.move(1);
106 pinch.end();
107 expect(change).toHaveBeenCalledTimes(count);
108 expect(pinch.active).toBe(false);
109});
packages/client-runtime/src/device/duoControl.ts created+113
......@@ -0,0 +1,113 @@
1// @effect-diagnostics globalTimers:off - The stream owns this browser control queue and its timeout.
2export const DUO_POSES = [
3 { id: "closed", label: "Closed", angle: 0 },
4 { id: "book", label: "Book", angle: 90 },
5 { id: "open", label: "Open", angle: 180 },
6 { id: "laptop", label: "Laptop", angle: 90 },
7 { id: "tent", label: "Tent", angle: 80 },
8] as const;
9export type DuoPose = (typeof DUO_POSES)[number]["id"];
10export type DuoOrientation =
11 | "portrait"
12 | "landscape_left"
13 | "portrait_upside_down"
14 | "landscape_right";
15export type DuoCommand =
16 | { control: "angle"; value: number }
17 | { control: "pose"; value: DuoPose }
18 | { control: "table"; value: boolean }
19 | { control: "physical"; value: "faceup" | "facedown" }
20 | { control: "orientation"; value: DuoOrientation };
21export type DuoControlState = {
22 pending: boolean;
23 requested: DuoCommand | null;
24 error: string | null;
25};
26
27/** One in-flight native transaction. Hinge motion coalesces; presets replace queued motion. Nothing replays after reconnect. */
28export function createDuoControl(options: {
29 send: (request: { requestId: number; command: DuoCommand }) => boolean;
30 onChange: (state: DuoControlState) => void;
31 timeoutMs?: number;
32}) {
33 let nextId = 1;
34 let active: { requestId: number; command: DuoCommand } | null = null;
35 let queued: DuoCommand | null = null;
36 let timer: ReturnType<typeof setTimeout> | null = null;
37 const publish = (error: string | null = null) =>
38 options.onChange({ pending: !!active, requested: queued ?? active?.command ?? null, error });
39 const clear = (error: string | null = null) => {
40 if (timer) clearTimeout(timer);
41 timer = null;
42 active = null;
43 queued = null;
44 publish(error);
45 };
46 const drain = () => {
47 if (active || !queued) return;
48 active = { requestId: nextId++, command: queued };
49 queued = null;
50 const id = active.requestId;
51 timer = setTimeout(() => {
52 if (active?.requestId === id) clear("Device control timed out. Its position is unknown.");
53 }, options.timeoutMs ?? 5_000);
54 publish();
55 if (!options.send(active)) clear("Device is disconnected.");
56 };
57 return {
58 enqueue(command: DuoCommand) {
59 if (
60 command.control === "angle" &&
61 (!Number.isFinite(command.value) || command.value < 0 || command.value > 180)
62 )
63 return;
64 queued = command;
65 if (active) publish();
66 else drain();
67 },
68 receive(reply: { requestId: number; ok: boolean; error?: string }) {
69 if (!active || reply.requestId !== active.requestId) return;
70 if (timer) clearTimeout(timer);
71 timer = null;
72 active = null;
73 if (!reply.ok) {
74 clear(reply.error ?? "Device control failed. Its position is unknown.");
75 return;
76 }
77 if (queued) drain();
78 else publish();
79 },
80 clear,
81 };
82}
83
84/** A pinch keeps its own accumulator across asynchronous native acknowledgements. */
85export function createDuoPinch(options: {
86 angle: () => number;
87 contains: (x: number, y: number) => boolean;
88 change: (angle: number | null) => void;
89}) {
90 let angle: number | null = null;
91 return {
92 begin(x: number, y: number) {
93 if (!options.contains(x, y)) return false;
94 angle = Math.max(0, Math.min(180, options.angle()));
95 return true;
96 },
97 move(logScale: number) {
98 if (angle === null || !Number.isFinite(logScale)) return;
99 const next = Math.max(0, Math.min(180, angle + logScale * 120));
100 if (next === angle) return;
101 angle = next;
102 options.change(next);
103 },
104 end() {
105 if (angle === null) return;
106 angle = null;
107 options.change(null);
108 },
109 get active() {
110 return angle !== null;
111 },
112 };
113}
packages/client-runtime/src/device/duoScene.test.ts created+108
......@@ -0,0 +1,108 @@
1import {
2 BoxGeometry,
3 Group,
4 Mesh,
5 MeshBasicMaterial,
6 PerspectiveCamera,
7 PlaneGeometry,
8 Texture,
9 Vector3,
10} from "three";
11import { expect, it } from "vite-plus/test";
12import { createDuoScene, duoDisplayKey, duoRawPoint } from "./duoScene.ts";
13
14function fixture() {
15 const asset = new Group();
16 const left = new Group();
17 left.name = "left-half";
18 const right = new Group();
19 right.name = "right-half";
20 const screen = (name: string, x: number, z: number, rear = false) => {
21 const mesh = new Mesh(new PlaneGeometry(1, 2), new MeshBasicMaterial());
22 if (rear) mesh.geometry.rotateY(Math.PI);
23 mesh.geometry.translate(x, 0, z);
24 mesh.name = name;
25 return mesh;
26 };
27 const innerLeft = screen("inner-display-left", -0.5, 0.06);
28 const innerRight = screen("inner-display-right", 0.5, 0.06);
29 const cover = screen("cover-display", -0.5, -0.06, true);
30 left.add(innerLeft, cover);
31 right.add(innerRight);
32 for (const [group, x] of [
33 [left, -0.5],
34 [right, 0.5],
35 ] as const) {
36 const body = new Mesh(new BoxGeometry(1, 2, 0.1), new MeshBasicMaterial());
37 body.geometry.translate(x, 0, 0);
38 group.add(body);
39 }
40 asset.add(left, right);
41 const scene = createDuoScene(asset, { 1: new Texture(), 3: new Texture() });
42 const camera = new PerspectiveCamera(36, 1, 0.1, 50);
43 camera.position.z = 6;
44 camera.updateMatrixWorld();
45 return { scene, camera, left, right, innerLeft, innerRight };
46}
47
48it("keeps one continuous inner UV map across the hinge and folds opposite leaves", () => {
49 const { scene, left, right, innerLeft, innerRight } = fixture();
50 scene.setAngle(90);
51 expect(left.rotation.y).toBeCloseTo(Math.PI / 4);
52 expect(right.rotation.y).toBeCloseTo(-Math.PI / 4);
53 const values = (mesh: Mesh) =>
54 Array.from({ length: mesh.geometry.getAttribute("uv").count }, (_, i) =>
55 mesh.geometry.getAttribute("uv").getX(i),
56 );
57 expect(Math.max(...values(innerLeft))).toBe(0.5);
58 expect(Math.min(...values(innerRight))).toBe(0.5);
59 scene.dispose();
60});
61
62it("derives resting views from the hinged display planes independently of the inspection orbit", () => {
63 const { scene } = fixture();
64 scene.setAngle(90);
65 const frames = scene.restFrames(3);
66 expect(frames.map((frame) => frame.face)).toEqual(["inside", "left", "right"]);
67 expect(frames[1]!.normal.x).toBeGreaterThan(0.5);
68 expect(frames[2]!.normal.x).toBeLessThan(-0.5);
69 scene.root.rotation.set(0.5, 1.2, -0.7);
70 scene.root.position.set(3, -2, 1);
71 const rotated = scene.restFrames(3);
72 for (let index = 0; index < frames.length; index++) {
73 expect(rotated[index]!.normal.distanceTo(frames[index]!.normal)).toBeLessThan(1e-6);
74 expect(rotated[index]!.center.distanceTo(frames[index]!.center)).toBeLessThan(1e-6);
75 }
76 scene.setAngle(180);
77 expect(scene.restFrames(3).map((frame) => frame.face)).toEqual(["inside"]);
78 expect(scene.restFrames(1).map((frame) => frame.face)).toEqual(["cover"]);
79 scene.dispose();
80});
81
82it("maps active display input through hardware mounting and blocks rear, inactive and stale surfaces", () => {
83 const { scene, camera, innerLeft } = fixture();
84 scene.setAngle(180);
85 const screen = { width: 2007, height: 2853, orientation: "portrait" as const, screenId: 3 };
86 const key = duoDisplayKey(screen);
87 scene.root.updateMatrixWorld();
88 const world = innerLeft.localToWorld(new Vector3(-0.5, -0.5, 0.06)).project(camera);
89 const x = (world.x + 1) / 2,
90 y = (1 - world.y) / 2;
91 const hit = scene.screenPoint(x, y, camera, screen, key);
92 expect(hit?.x).toBeCloseTo(0.75);
93 expect(hit?.y).toBeCloseTo(0.75);
94 expect(
95 scene.screenPoint(
96 x,
97 y,
98 camera,
99 { ...screen, screenId: 1 },
100 duoDisplayKey({ ...screen, screenId: 1 }),
101 ),
102 ).toBeNull();
103 expect(scene.screenPoint(x, y, camera, screen, "old")).toBeNull();
104 expect(scene.screenPoint(x, y, camera, screen, key)).not.toBeNull();
105 expect(scene.screenPoint(x, 1.1, camera, screen, key, true)?.x).toBeGreaterThan(1);
106 expect(duoRawPoint(1, 0.2, 0.7)).toEqual({ x: 0.2, y: 0.7 });
107 scene.dispose();
108});
packages/client-runtime/src/device/duoScene.ts created+244
......@@ -0,0 +1,244 @@
1import {
2 Box3,
3 BufferAttribute,
4 Group,
5 Mesh,
6 MeshBasicMaterial,
7 Matrix4,
8 Quaternion,
9 Plane,
10 Raycaster,
11 Triangle,
12 Vector2,
13 Vector3,
14 type PerspectiveCamera,
15 type Texture,
16} from "three";
17import type { DuoRestFrame } from "./duoSnap.ts";
18import type { DeviceScreenSize } from "./stream.ts";
19
20export type DuoPanelId = 1 | 3;
21export type DuoHingeLeaf = "left" | "right";
22export type DuoFrameLayout = { width: number; height: number };
23
24/** Hardware mounting is independent of app orientation and the model's orbit. The inverse is shared by taps and drags. */
25export function duoRawPoint(panel: DuoPanelId, x: number, y: number) {
26 return panel === 1 ? { x, y } : { x: y, y: 1 - x };
27}
28export function duoFrameMatches(frame: DuoFrameLayout, screen: DeviceScreenSize) {
29 return (
30 frame.width > 0 &&
31 frame.height > 0 &&
32 Math.abs(frame.width / frame.height - screen.width / screen.height) <=
33 1 / frame.height + 1 / screen.height
34 );
35}
36export function duoDisplayKey(screen: DeviceScreenSize | null) {
37 return screen ? `${screen.screenId}:${screen.width}:${screen.height}:${screen.orientation}` : "";
38}
39
40/** Asset resources belong to its model slot; this scene owns only live-screen replacement materials. */
41export function createDuoScene(asset: Group, textures: Record<DuoPanelId, Texture>) {
42 const left = asset.getObjectByName("left-half");
43 const right = asset.getObjectByName("right-half");
44 const cover = asset.getObjectByName("cover-display");
45 const innerLeft = asset.getObjectByName("inner-display-left");
46 const innerRight = asset.getObjectByName("inner-display-right");
47 if (
48 !left ||
49 !right ||
50 !(cover instanceof Mesh) ||
51 !(innerLeft instanceof Mesh) ||
52 !(innerRight instanceof Mesh)
53 )
54 throw new Error("Duo model must have two hinge groups and all three display meshes");
55 const surfaces = { 1: [cover], 3: [innerLeft, innerRight] };
56 const boundsByPanel = ([1, 3] as const).map((id) => {
57 const bounds = new Box3();
58 for (const mesh of surfaces[id]) {
59 mesh.geometry.computeBoundingBox();
60 bounds.union(mesh.geometry.boundingBox!);
61 }
62 const size = bounds.getSize(new Vector3());
63 if (![size.x, size.y].every((value) => Number.isFinite(value) && value > 0))
64 throw new Error("Invalid Duo display bounds");
65 return { bounds, size };
66 });
67 const root = new Group();
68 const content = new Group();
69 content.add(asset);
70 root.add(content);
71 let currentAngle = Number.NaN;
72 const localBounds = new Box3();
73 const relative = new Matrix4();
74 const inverse = new Matrix4();
75 const materials = {
76 1: new MeshBasicMaterial({ map: textures[1], toneMapped: false }),
77 3: new MeshBasicMaterial({ map: textures[3], toneMapped: false }),
78 };
79 const originals = new Map<Mesh, Mesh["material"]>();
80 // The two inner meshes sample halves of one framebuffer. Cover UVs face outward on the rear leaf.
81 for (const id of [1, 3] as const) {
82 const { bounds, size } = boundsByPanel[id === 1 ? 0 : 1]!;
83 for (const mesh of surfaces[id]) {
84 const position = mesh.geometry.getAttribute("position");
85 const uv = new Float32Array(position.count * 2);
86 for (let i = 0; i < position.count; i++) {
87 const x = (position.getX(i) - bounds.min.x) / size.x;
88 uv[i * 2] = id === 1 ? 1 - x : x;
89 uv[i * 2 + 1] = (position.getY(i) - bounds.min.y) / size.y;
90 }
91 mesh.geometry.setAttribute("uv", new BufferAttribute(uv, 2));
92 originals.set(mesh, mesh.material);
93 mesh.material = materials[id];
94 }
95 }
96 const ray = new Raycaster();
97 let captured: {
98 panel: DuoPanelId;
99 plane: Plane;
100 triangle: Triangle;
101 uv: [Vector2, Vector2, Vector2];
102 key: string;
103 } | null = null;
104 return {
105 root,
106 setAngle(angle: number) {
107 if (angle === currentAngle) return;
108 currentAngle = angle;
109 const radians = ((180 - Math.min(180, Math.max(0, angle))) * Math.PI) / 360;
110 left.rotation.y = radians;
111 right.rotation.y = -radians;
112 // Orbit the folded body's center, while retaining the authored hinge pivots.
113 content.position.set(0, 0, 0);
114 root.updateWorldMatrix(true, true);
115 inverse.copy(root.matrixWorld).invert();
116 localBounds.makeEmpty();
117 asset.traverse((object) => {
118 if (!(object instanceof Mesh)) return;
119 if (!object.geometry.boundingBox) object.geometry.computeBoundingBox();
120 if (!object.geometry.boundingBox) return;
121 relative.multiplyMatrices(inverse, object.matrixWorld);
122 localBounds.union(object.geometry.boundingBox.clone().applyMatrix4(relative));
123 });
124 content.position.copy(localBounds.getCenter(new Vector3())).negate();
125 },
126 restFrames(panel: DuoPanelId): DuoRestFrame[] {
127 root.updateWorldMatrix(true, true);
128 inverse.copy(root.matrixWorld).invert();
129 const frame = (meshes: Mesh[], face: DuoRestFrame["face"]): DuoRestFrame => {
130 const bounds = new Box3();
131 const normal = new Vector3();
132 const up = new Vector3();
133 for (const mesh of meshes) {
134 if (!mesh.geometry.boundingBox) mesh.geometry.computeBoundingBox();
135 relative.multiplyMatrices(inverse, mesh.matrixWorld);
136 bounds.union(mesh.geometry.boundingBox!.clone().applyMatrix4(relative));
137 normal.add(
138 new Vector3()
139 .fromBufferAttribute(mesh.geometry.getAttribute("normal"), 0)
140 .transformDirection(relative),
141 );
142 up.add(new Vector3(0, 1, 0).transformDirection(relative));
143 }
144 return {
145 face,
146 center: bounds.getCenter(new Vector3()),
147 normal: normal.normalize(),
148 up: up.normalize(),
149 };
150 };
151 if (panel === 1) return [frame([cover], "cover")];
152 const inside = frame([innerLeft, innerRight], "inside");
153 // Leaf views only make sense when both displays form a useful open fold.
154 return currentAngle > 20 && currentAngle < 165
155 ? [inside, frame([innerLeft], "left"), frame([innerRight], "right")]
156 : [inside];
157 },
158 leafRotation(leaf: DuoHingeLeaf) {
159 root.updateWorldMatrix(true, true);
160 relative.multiplyMatrices(
161 root.matrixWorld.clone().invert(),
162 (leaf === "left" ? left : right).matrixWorld,
163 );
164 return new Quaternion().setFromRotationMatrix(relative.extractRotation(relative));
165 },
166 hingeLeafAt(x: number, y: number, camera: PerspectiveCamera): DuoHingeLeaf | null {
167 if (!Number.isFinite(x) || !Number.isFinite(y)) return null;
168 root.updateMatrixWorld(true);
169 camera.updateMatrixWorld(true);
170 ray.setFromCamera(new Vector2(x * 2 - 1, 1 - y * 2), camera);
171 let object = ray.intersectObject(root, true)[0]?.object;
172 while (object) {
173 if (object === left) return "left";
174 if (object === right) return "right";
175 object = object.parent ?? undefined;
176 }
177 return null;
178 },
179 cancelInput() {
180 captured = null;
181 },
182 screenPoint(
183 x: number,
184 y: number,
185 camera: PerspectiveCamera,
186 screen: DeviceScreenSize | null,
187 readyKey: string,
188 extend = false,
189 ) {
190 const key = duoDisplayKey(screen);
191 if (!screen || !readyKey || key !== readyKey) {
192 captured = null;
193 return null;
194 }
195 root.updateMatrixWorld(true);
196 camera.updateMatrixWorld(true);
197 ray.setFromCamera(new Vector2(x * 2 - 1, 1 - y * 2), camera);
198 if (extend && captured?.key === key) {
199 const point = ray.ray.intersectPlane(captured.plane, new Vector3());
200 const bary = point ? captured.triangle.getBarycoord(point, new Vector3()) : null;
201 if (!bary) return null;
202 const uv = captured.uv[0]
203 .clone()
204 .multiplyScalar(bary.x)
205 .addScaledVector(captured.uv[1], bary.y)
206 .addScaledVector(captured.uv[2], bary.z);
207 return duoRawPoint(captured.panel, uv.x, 1 - uv.y);
208 }
209 // The chassis occludes rear displays. Only the first visible hit can own a contact.
210 const hit = ray.intersectObject(root, true)[0];
211 if (!hit?.uv || !hit.face || !(hit.object instanceof Mesh)) return null;
212 const panel: DuoPanelId | null = surfaces[1].includes(hit.object)
213 ? 1
214 : surfaces[3].includes(hit.object)
215 ? 3
216 : null;
217 if (!panel || panel !== screen.screenId) return null;
218 const indices = [hit.face.a, hit.face.b, hit.face.c] as const;
219 const positions = hit.object.geometry.getAttribute("position");
220 const triangle = new Triangle(
221 ...(indices.map((index) =>
222 new Vector3().fromBufferAttribute(positions, index).applyMatrix4(hit.object.matrixWorld),
223 ) as [Vector3, Vector3, Vector3]),
224 );
225 const hitUvs = hit.object.geometry.getAttribute("uv");
226 const uv = indices.map((index) => new Vector2().fromBufferAttribute(hitUvs, index)) as [
227 Vector2,
228 Vector2,
229 Vector2,
230 ];
231 captured = { panel, triangle, uv, plane: triangle.getPlane(new Plane()), key };
232 return duoRawPoint(panel, hit.uv.x, 1 - hit.uv.y);
233 },
234 dispose() {
235 for (const [mesh, material] of originals) mesh.material = material;
236 for (const material of Object.values(materials)) {
237 material.map = null;
238 material.dispose();
239 }
240 content.remove(asset);
241 captured = null;
242 },
243 };
244}
packages/client-runtime/src/device/duoSnap.test.ts created+54
......@@ -0,0 +1,54 @@
1import { expect, it } from "vite-plus/test";
2import { Quaternion, Vector3 } from "three";
3import { duoViewSnaps, nearestDuoView } from "./duoSnap.ts";
4
5const frames = [
6 {
7 face: "inside" as const,
8 normal: new Vector3(0, 0, 1),
9 up: new Vector3(0, 1, 0),
10 center: new Vector3(),
11 },
12];
13const rotation = (x: number, y: number, z = 0) =>
14 new Quaternion().setFromAxisAngle(new Vector3(x, y, z).normalize(), Math.hypot(x, y, z));
15
16it("chooses a leaf-focused seated view when it is closer than the middle of the fold", () => {
17 const leaf = {
18 face: "right" as const,
19 normal: new Vector3(0.6, 0, 0.8),
20 up: new Vector3(0, 1, 0),
21 center: new Vector3(1, 0, 0),
22 };
23 const candidates = duoViewSnaps([...frames, leaf], 3);
24 const seat = candidates.find(
25 (candidate) => candidate.face === "right" && candidate.orientation === "portrait",
26 )!;
27 const released = seat.rotation.clone().premultiply(rotation(0.04, 0.05));
28 const chosen = nearestDuoView(released, candidates)!;
29 expect(chosen.face).toBe("right");
30 expect(chosen.orientation).toBe("portrait");
31 expect(chosen.center.x).toBe(1);
32});
33
34it("a seated view exposes the base, and upright yaw keeps both inner displays facing the camera", () => {
35 const folded = [
36 frames[0]!,
37 ...(["left", "right"] as const).map((face) => ({
38 face,
39 normal: new Vector3(face === "left" ? Math.SQRT1_2 : -Math.SQRT1_2, 0, Math.SQRT1_2),
40 up: new Vector3(0, 1, 0),
41 center: new Vector3(face === "left" ? -1 : 1, 0, 0),
42 })),
43 ];
44 const candidates = duoViewSnaps(folded, 3);
45 expect(candidates.filter((candidate) => candidate.face === "right")).toHaveLength(1);
46 for (const candidate of candidates) {
47 for (const frame of folded.slice(1))
48 expect(frame.normal.clone().applyQuaternion(candidate.rotation).z).toBeGreaterThan(0.2);
49 const side = candidate.rotation.clone().premultiply(rotation(0, 1.5));
50 const chosen = nearestDuoView(side, [candidate])!;
51 for (const frame of folded.slice(1))
52 expect(frame.normal.clone().applyQuaternion(chosen.rotation).z).toBeGreaterThan(0.02);
53 }
54});
packages/client-runtime/src/device/duoSnap.ts created+86
......@@ -0,0 +1,86 @@
1import { Matrix4, Quaternion, Vector3 } from "three";
2import { nearestDeviceView } from "./deviceViewSnap.ts";
3import type { DeviceScreenSize } from "./stream.ts";
4
5export type DuoRestFace = "cover" | "inside" | "left" | "right";
6export type DuoRestFrame = { face: DuoRestFace; normal: Vector3; up: Vector3; center: Vector3 };
7export type DuoViewSnap = {
8 rotation: Quaternion;
9 face: DuoRestFace;
10 orientation: DeviceScreenSize["orientation"];
11 center: Vector3;
12 yawLimit: number;
13};
14const orientations = [
15 "portrait",
16 "landscape_left",
17 "portrait_upside_down",
18 "landscape_right",
19] as const;
20const z = new Vector3(0, 0, 1);
21const y = new Vector3(0, 1, 0);
22
23/** Build views from the actual hinged display planes, rather than model-specific Euler offsets. */
24export function duoViewSnaps(frames: readonly DuoRestFrame[], panel: 1 | 3) {
25 const snaps: DuoViewSnap[] = [];
26 for (const frame of frames) {
27 const normal = frame.normal.clone().normalize();
28 if (normal.lengthSq() < 0.5) continue;
29 const right = frame.up.clone().cross(normal).normalize();
30 const up = normal.clone().cross(right).normalize();
31 const faceRotation = new Quaternion()
32 .setFromRotationMatrix(new Matrix4().makeBasis(right, up, normal))
33 .invert();
34 for (let index = 0; index < orientations.length; index++) {
35 // Only these rolls put the partner below the focused lid. Other quarter
36 // turns belong to the upright, whole-display family.
37 if (frame.face === "right" && index !== 0) continue;
38 if (frame.face === "left" && index !== 2) continue;
39 const roll = (panel === 3 ? Math.PI / 2 : 0) - (index * Math.PI) / 2;
40 const rotation = faceRotation.clone().premultiply(new Quaternion().setFromAxisAngle(z, roll));
41 // A leaf-focused view looks slightly down onto its partner, as in a seated laptop.
42 if (frame.face === "left" || frame.face === "right")
43 rotation.premultiply(new Quaternion().setFromAxisAngle(new Vector3(1, 0, 0), Math.PI / 12));
44 if (
45 panel === 3 &&
46 frames.some(
47 (leaf) =>
48 (leaf.face === "left" || leaf.face === "right") &&
49 leaf.normal.clone().applyQuaternion(rotation).z < 0.04,
50 )
51 )
52 continue;
53 // Bound yaw using both real leaf normals. A narrow fold cannot tolerate
54 // the same side view as a fully open display.
55 let yawLimit = frame.face === "cover" ? Math.PI / 9 : Math.PI / 3;
56 if (panel === 3) {
57 for (const sign of [-1, 1]) {
58 for (let step = 1; step <= 60; step++) {
59 const turn = new Quaternion().setFromAxisAngle(y, (sign * step * Math.PI) / 180);
60 if (
61 frames.some(
62 (leaf) =>
63 (leaf.face === "left" || leaf.face === "right") &&
64 leaf.normal.clone().applyQuaternion(rotation).applyQuaternion(turn).z < 0.04,
65 )
66 ) {
67 yawLimit = Math.min(yawLimit, ((step - 1) * Math.PI) / 180);
68 break;
69 }
70 }
71 }
72 }
73 snaps.push({
74 rotation,
75 face: frame.face,
76 orientation: orientations[index]!,
77 center: frame.center.clone(),
78 yawLimit,
79 });
80 }
81 }
82 return snaps;
83}
84
85/** Fold-specific candidates use the base viewer's nearest-family selection. */
86export const nearestDuoView = nearestDeviceView<DuoViewSnap>;
packages/client-runtime/src/device/duoStream.test.ts created+285
......@@ -0,0 +1,285 @@
1import { afterEach, expect, it, vi } from "vite-plus/test";
2import { createDeviceStreamClient, type DeviceScreenSize } from "./stream.ts";
3afterEach(() => vi.unstubAllGlobals());
4
5it("switches to fixed authenticated feeds while retaining one HID socket, routes the active panel and rejects superseded decoded output", async () => {
6 const feeds: {
7 url: string;
8 signal: AbortSignal;
9 controller: ReadableStreamDefaultController<Uint8Array>;
10 }[] = [];
11 const outputs: VideoFrameOutputCallback[] = [];
12 const errors: VideoDecoderInit["error"][] = [];
13 let supported = true;
14 let panelHttpStatus = 200;
15 let expected = 0;
16 let decoderReady = () => {};
17 const waitDecoders = (count: number) => {
18 expected = count;
19 return new Promise<void>((resolve) => {
20 decoderReady = resolve;
21 });
22 };
23 class Decoder {
24 static isConfigSupported = async () => ({ supported });
25 static instances: Decoder[] = [];
26 state = "unconfigured";
27 constructor(options: VideoDecoderInit) {
28 outputs.push(options.output);
29 errors.push(options.error);
30 Decoder.instances.push(this);
31 if (outputs.length === expected) decoderReady();
32 }
33 configure() {
34 this.state = "configured";
35 }
36 close() {
37 this.state = "closed";
38 }
39 }
40 let socketReady = () => {};
41 const socketConstructed = new Promise<void>((resolve) => {
42 socketReady = resolve;
43 });
44 class Socket {
45 static OPEN = 1;
46 static instances: Socket[] = [];
47 readyState = 1;
48 binaryType = "arraybuffer";
49 onopen?: () => void;
50 onmessage?: (event: { data: ArrayBuffer }) => void;
51 onclose?: (event: { code: number; reason: string }) => void;
52 send = vi.fn();
53 close = vi.fn();
54 constructor() {
55 Socket.instances.push(this);
56 socketReady();
57 }
58 }
59 vi.stubGlobal("VideoDecoder", Decoder);
60 vi.stubGlobal("EncodedVideoChunk", vi.fn());
61 vi.stubGlobal("WebSocket", Socket);
62 vi.stubGlobal(
63 "fetch",
64 vi.fn(async (url: string, options: { signal: AbortSignal }) => {
65 if (!url.includes("avcc")) return new Response("prime");
66 if (url.includes("/panel/") && panelHttpStatus !== 200)
67 return new Response("unsupported panel", { status: panelHttpStatus });
68 const stream = new ReadableStream<Uint8Array>({
69 start(controller) {
70 feeds.push({ url, signal: options.signal, controller });
71 options.signal.addEventListener("abort", () =>
72 controller.error(new DOMException("Aborted", "AbortError")),
73 );
74 },
75 });
76 return new Response(stream);
77 }),
78 );
79 const present = vi.fn((_source: CanvasImageSource, _width: number, _height: number) => true),
80 cover = vi.fn((_source: CanvasImageSource, _width: number, _height: number) => true),
81 inner = vi.fn((_source: CanvasImageSource, _width: number, _height: number) => true);
82 let panelFailed = () => {};
83 const panelFailure = new Promise<void>((resolve) => {
84 panelFailed = resolve;
85 });
86 const onStatus = vi.fn();
87 const onDuoControl = vi.fn();
88 const onDuoUnavailable = vi.fn((_detail?: string) => panelFailed());
89 const client = createDeviceStreamClient(
90 {
91 platform: "ios",
92 deviceId: "duo",
93 access: {
94 httpBase: "https://t3.test/api/device-hub",
95 wsBase: "wss://t3.test/api/device-hub",
96 credentials: true,
97 query: { hostId: "remote", wsTicket: "ticket" },
98 },
99 },
100 { present },
101 {
102 onStatus,
103 onDuoControl,
104 onDuoUnavailable,
105 onScreen: vi.fn(),
106 onInputConnected: vi.fn(),
107 onUnauthorized: vi.fn(),
108 onMjpegFallback: vi.fn(),
109 },
110 );
111 client.start();
112 await socketConstructed;
113 const ws = Socket.instances[0]!;
114 const config = (
115 id: number,
116 orientation: DeviceScreenSize["orientation"] = "portrait",
117 physical = false,
118 ) => {
119 const json = new TextEncoder().encode(
120 JSON.stringify({
121 width: id === 1 ? 1398 : 2007,
122 height: id === 1 ? 2034 : 2853,
123 orientation,
124 screenId: id,
125 supportsHingeAngle: true,
126 supportsPhysicalOrientation: physical,
127 hingePose: "open",
128 }),
129 );
130 const packet = new Uint8Array(json.length + 1);
131 packet[0] = 0x82;
132 packet.set(json, 1);
133 ws.onmessage?.({ data: packet.buffer });
134 };
135 config(3);
136 const requestedOrientation = () =>
137 JSON.parse(new TextDecoder().decode(ws.send.mock.lastCall?.[0].subarray(1))).orientation;
138 client.rotate();
139 expect(requestedOrientation()).toBe("landscape_left");
140 config(3); // An orientation-locked app keeps its framebuffer orientation after the sensor rotates.
141 expect(onDuoControl.mock.lastCall?.[0].error).toBeNull();
142 client.rotate();
143 expect(requestedOrientation()).toBe("portrait_upside_down");
144 config(3, "portrait_upside_down");
145 client.rotate();
146 expect(requestedOrientation()).toBe("landscape_right");
147 config(3); // An external native orientation change becomes authoritative again.
148 client.rotate();
149 expect(requestedOrientation()).toBe("landscape_left");
150 config(3, "landscape_left");
151 expect(onDuoControl.mock.lastCall?.[0].pending).toBe(false);
152 client.controlDuo({ control: "angle", value: 40 });
153 const angleRequest = JSON.parse(new TextDecoder().decode(ws.send.mock.lastCall?.[0].subarray(1)));
154 const before = ws.send.mock.calls.length;
155 client.controlDuo({ control: "orientation", value: "portrait" });
156 expect(ws.send.mock.calls.length).toBe(before);
157 config(3, "landscape_left"); // The angle's config precedes its receipt; it cannot acknowledge a queued rotation.
158 const reply = new TextEncoder().encode(
159 JSON.stringify({ requestId: angleRequest.requestId, ok: true }),
160 );
161 const receipt = new Uint8Array(reply.length + 1);
162 receipt[0] = 0x90;
163 receipt.set(reply, 1);
164 ws.onmessage?.({ data: receipt.buffer });
165 expect(requestedOrientation()).toBe("portrait");
166 expect(ws.send.mock.lastCall?.[0][0]).toBe(0x07);
167 client.controlDuo({ control: "angle", value: 55 });
168 const orientationSends = ws.send.mock.calls.length;
169 expect(onDuoControl.mock.lastCall?.[0].pending).toBe(true);
170 config(3);
171 expect(ws.send.mock.calls.length).toBe(orientationSends + 1);
172 const after = JSON.parse(new TextDecoder().decode(ws.send.mock.lastCall?.[0].subarray(1)));
173 expect(after.command).toEqual({ control: "angle", value: 55 });
174 const finalReply = new TextEncoder().encode(
175 JSON.stringify({ requestId: after.requestId, ok: true }),
176 );
177 const finalReceipt = new Uint8Array(finalReply.length + 1);
178 finalReceipt[0] = 0x90;
179 finalReceipt.set(finalReply, 1);
180 ws.onmessage?.({ data: finalReceipt.buffer });
181 expect(onDuoControl.mock.lastCall?.[0].pending).toBe(false);
182 const description = new Uint8Array([0, 0, 0, 5, 1, 1, 0x64, 0, 0x1f]);
183 let ready = waitDecoders(1);
184 feeds[0]!.controller.enqueue(description);
185 await ready;
186 client.setDuoPanels({ cover: { present: cover }, inner: { present: inner } });
187 expect(feeds[0]!.signal.aborted).toBe(true);
188 expect(Socket.instances).toHaveLength(1);
189 for (const [index, id] of [
190 [1, 1],
191 [2, 3],
192 ]) {
193 const url = new URL(feeds[index!]!.url);
194 expect(url.pathname.endsWith(`/panel/${id}/stream.avcc`)).toBe(true);
195 expect(url.searchParams.get("hostId")).toBe("remote");
196 expect(url.searchParams.get("wsTicket")).toBe("ticket");
197 }
198 ready = waitDecoders(3);
199 feeds[1]!.controller.enqueue(description);
200 feeds[2]!.controller.enqueue(description);
201 await ready;
202 const frame = {
203 displayWidth: 2007,
204 displayHeight: 2853,
205 close: vi.fn(),
206 } as unknown as VideoFrame;
207 outputs[0]!(frame);
208 expect(present).not.toHaveBeenCalled();
209 outputs[1]!(frame);
210 expect(cover).not.toHaveBeenCalled();
211 outputs[2]!(frame);
212 expect(inner).toHaveBeenCalledOnce();
213 expect(present).toHaveBeenCalledOnce();
214 config(1);
215 outputs[2]!(frame);
216 expect(inner).toHaveBeenCalledOnce();
217 outputs[1]!(frame);
218 expect(cover).toHaveBeenCalledOnce();
219 client.setDuoPanels(null);
220 expect(feeds[1]!.signal.aborted).toBe(true);
221 expect(feeds[2]!.signal.aborted).toBe(true);
222 expect(feeds[3]!.url).not.toContain("/panel/");
223 outputs[1]!(frame);
224 expect(cover).toHaveBeenCalledOnce();
225 ready = waitDecoders(4);
226 feeds[3]!.controller.enqueue(description);
227 await ready;
228 errors[0]!(new DOMException("Late decoder failure"));
229 expect(Decoder.instances[3]!.state).toBe("configured");
230 config(1, "portrait", true);
231 client.setDuoPanels({ cover: { present: cover }, inner: { present: inner } });
232 expect(feeds).toHaveLength(5); // One active feed replaces the two fixed feeds.
233 expect(feeds[4]!.url).not.toContain("/panel/");
234 expect(new URL(feeds[4]!.url).searchParams.get("hostId")).toBe("remote");
235 ready = waitDecoders(5);
236 feeds[4]!.controller.enqueue(description);
237 await ready;
238 const primary = { ...frame, displayWidth: 1398, displayHeight: 2034 } as VideoFrame;
239 const painted = present.mock.calls.length;
240 outputs[4]!(primary);
241 expect(present).toHaveBeenCalledTimes(painted + 1);
242 expect(present.mock.lastCall?.[0]).toBe(primary);
243 expect(cover).toHaveBeenCalledOnce(); // The main sink owns primary texture delivery.
244 config(3, "portrait", true);
245 expect(feeds[4]!.signal.aborted).toBe(true);
246 expect(feeds).toHaveLength(6);
247 ready = waitDecoders(6);
248 feeds[5]!.controller.enqueue(description);
249 await ready;
250 outputs[4]!(primary); // The old elected display cannot paint after handoff.
251 expect(present).toHaveBeenCalledTimes(painted + 1);
252 outputs[5]!(frame);
253 expect(present).toHaveBeenCalledTimes(painted + 2);
254 expect(present.mock.lastCall?.[0]).toBe(frame);
255 config(3, "portrait", true);
256 expect(feeds).toHaveLength(6); // Duplicate native readback does not reconnect.
257 supported = false;
258 client.setDuoPanels(null);
259 client.setDuoPanels({ cover: { present: cover }, inner: { present: inner } });
260 expect(feeds[7]!.url).not.toContain("/panel/");
261 feeds[7]!.controller.enqueue(description);
262 await panelFailure;
263 expect(onDuoUnavailable.mock.lastCall?.[0]).toContain("cannot decode");
264 config(3);
265 const fixedPanelFailure = new Promise<void>((resolve) => {
266 panelFailed = resolve;
267 });
268 client.setDuoPanels({ cover: { present: cover }, inner: { present: inner } });
269 feeds[8]!.controller.enqueue(description);
270 await fixedPanelFailure;
271 expect(onDuoUnavailable).toHaveBeenCalled();
272 expect(onStatus.mock.calls.some(([status]) => status === "error")).toBe(false);
273 panelHttpStatus = 404;
274 const missingPanel = new Promise<void>((resolve) => {
275 panelFailed = resolve;
276 });
277 client.setDuoPanels({ cover: { present: cover }, inner: { present: inner } });
278 await missingPanel;
279 expect(onDuoUnavailable.mock.lastCall?.[0]).toContain("does not provide fixed Duo");
280 expect(Socket.instances).toHaveLength(1);
281 client.stop();
282 expect(feeds[3]!.signal.aborted).toBe(true);
283 expect(ws.close).toHaveBeenCalledOnce();
284 expect(frame.close).toHaveBeenCalledTimes(9);
285});
packages/client-runtime/src/device/duoViewer.test.ts created+703
......@@ -0,0 +1,703 @@
1import { afterEach, expect, it, vi } from "vite-plus/test";
2import type { Scene, PerspectiveCamera, Box3, Quaternion } from "three";
3
4const gpu = vi.hoisted(() => ({
5 instances: [] as {
6 blank: boolean;
7 allocations: number;
8 frames: Scene[];
9 views: { camera: PerspectiveCamera; bounds: Box3; quaternion: Quaternion }[];
10 dispose: ReturnType<typeof vi.fn>;
11 forceContextLoss: ReturnType<typeof vi.fn>;
12 }[],
13 environmentDispose: vi.fn(),
14}));
15vi.mock("three", async () => {
16 const actual = await vi.importActual<typeof import("three")>("three");
17 return {
18 ...actual,
19 WebGLRenderer: class {
20 state = {
21 blank: true,
22 allocations: 0,
23 frames: [] as Scene[],
24 views: [] as { camera: PerspectiveCamera; bounds: Box3; quaternion: Quaternion }[],
25 dispose: vi.fn(),
26 forceContextLoss: vi.fn(),
27 };
28 constructor() {
29 gpu.instances.push(this.state);
30 }
31 setDrawingBufferSize() {
32 this.state.blank = true;
33 this.state.allocations++;
34 }
35 render(scene: Scene, camera: PerspectiveCamera) {
36 this.state.blank = false;
37 this.state.frames.push(scene);
38 const root = scene.children.find((child) => child instanceof actual.Group)!;
39 root.updateMatrixWorld(true);
40 camera.updateMatrixWorld(true);
41 this.state.views.push({
42 camera: camera.clone(),
43 bounds: new actual.Box3().setFromObject(root),
44 quaternion: root.quaternion.clone(),
45 });
46 }
47 dispose() {
48 this.state.dispose();
49 }
50 forceContextLoss() {
51 this.state.forceContextLoss();
52 }
53 },
54 PMREMGenerator: class {
55 fromScene() {
56 return { texture: new actual.Texture(), dispose: gpu.environmentDispose };
57 }
58 dispose() {}
59 },
60 };
61});
62const models = vi.hoisted(() => ({
63 resolve: (_model: { asset: import("three").Group; dispose: () => void }) => {},
64 signal: null as AbortSignal | null,
65}));
66vi.mock("./modelScene.ts", () => ({
67 loadDeviceModel: (_source: unknown, signal: AbortSignal) => {
68 models.signal = signal;
69 return new Promise((resolve) => {
70 models.resolve = resolve;
71 });
72 },
73}));
74import {
75 Group,
76 Mesh,
77 MeshBasicMaterial,
78 PlaneGeometry,
79 Vector3,
80 Euler,
81 Quaternion as Rotation,
82} from "three";
83import { createDuoViewer } from "./duoViewer.ts";
84
85afterEach(() => {
86 vi.useRealTimers();
87 vi.unstubAllGlobals();
88 gpu.instances.length = 0;
89 gpu.environmentDispose.mockClear();
90});
91
92function fixture(
93 reduced = true,
94 onOrientationRequested = vi.fn(),
95 onPanelRequested?: (panel: 1 | 3) => void,
96) {
97 const pending = new Map<number, FrameRequestCallback>();
98 let id = 0;
99 vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => {
100 pending.set(++id, callback);
101 return id;
102 });
103 vi.stubGlobal("cancelAnimationFrame", (id: number) => pending.delete(id));
104 vi.stubGlobal("matchMedia", () => ({ matches: reduced }));
105 const drawImage = vi.fn();
106 vi.stubGlobal("document", {
107 createElement: () => ({
108 width: 0,
109 height: 0,
110 getContext: () => ({
111 fillRect() {},
112 save() {},
113 restore() {},
114 translate() {},
115 rotate() {},
116 drawImage,
117 getImageData: () => ({ data: new Uint8ClampedArray(8 * 8 * 4).fill(255) }),
118 }),
119 }),
120 });
121 const canvas = new EventTarget() as HTMLCanvasElement;
122 const viewer = createDuoViewer({
123 canvas,
124 sources: {
125 1: { width: 1398, height: 2034 } as HTMLCanvasElement,
126 3: { width: 2007, height: 2853 } as HTMLCanvasElement,
127 },
128 model: { id: "iphone-duo", url: "/duo.glb" },
129 onUnavailable: vi.fn(),
130 onOrientationRequested,
131 ...(onPanelRequested ? { onPanelRequested } : {}),
132 });
133 const draw = () => {
134 const callbacks = [...pending.values()];
135 pending.clear();
136 callbacks.forEach((callback) => callback(0));
137 };
138 return { viewer, draw, pending, state: gpu.instances[0]!, onOrientationRequested, drawImage };
139}
140
141function asset() {
142 const group = new Group();
143 for (const [name, names] of [
144 ["left-half", ["cover-display", "inner-display-left"]],
145 ["right-half", ["inner-display-right"]],
146 ] as const) {
147 const leaf = new Group();
148 leaf.name = name;
149 for (const name of names) {
150 const mesh = new Mesh(new PlaneGeometry(1, 2), new MeshBasicMaterial());
151 if (name === "cover-display") mesh.geometry.rotateY(Math.PI);
152 mesh.geometry.translate(
153 name === "inner-display-right" ? 0.5 : -0.5,
154 0,
155 name === "cover-display" ? -0.05 : 0.05,
156 );
157 mesh.name = name;
158 leaf.add(mesh);
159 }
160 group.add(leaf);
161 }
162 return group;
163}
164
165it("coalesces resize with redraw, retains the renderer and scene, and settles without an idle animation loop", async () => {
166 let now = 0;
167 vi.stubGlobal("performance", { now: () => now });
168 const { viewer, draw, pending, state } = fixture();
169 const dispose = vi.fn();
170 models.resolve({ asset: asset(), dispose });
171 await Promise.resolve();
172 viewer.setScreen({
173 width: 1398,
174 height: 2034,
175 orientation: "portrait",
176 screenId: 1,
177 hingeAngle: 0,
178 });
179 viewer.resize(500, 700, 2);
180 draw();
181 const scene = state.frames.at(-1);
182 viewer.resize(450, 700, 2);
183 viewer.resize(400, 700, 2);
184 viewer.orbit(0.1, 0.04);
185 expect(state.blank).toBe(false);
186 expect(state.allocations).toBe(1);
187 expect(pending.size).toBe(1);
188 draw();
189 expect(state.blank).toBe(false);
190 expect(state.allocations).toBe(2);
191 expect(state.frames.at(-1)).toBe(scene);
192 expect(gpu.instances).toHaveLength(1);
193 now += 1000;
194 draw();
195 draw();
196 expect(pending.size).toBe(0);
197 viewer.dispose();
198 viewer.dispose();
199 expect(dispose).toHaveBeenCalledOnce();
200 expect(state.dispose).toHaveBeenCalledOnce();
201 expect(gpu.environmentDispose).toHaveBeenCalledOnce();
202 expect(pending.size).toBe(0);
203});
204
205it("cancels loading and disposes a late model after unmount without installing or rendering it", async () => {
206 const { viewer, draw, state } = fixture();
207 viewer.resize(400, 700, 2);
208 viewer.dispose();
209 expect(models.signal?.aborted).toBe(true);
210 const dispose = vi.fn();
211 models.resolve({ asset: asset(), dispose });
212 await Promise.resolve();
213 draw();
214 expect(dispose).toHaveBeenCalledOnce();
215 expect(state.frames).toHaveLength(0);
216});
217
218it("keeps the chosen release view when an app locks orientation, freezes it during contact, and stops drawing after release", async () => {
219 let now = 0;
220 vi.stubGlobal("performance", { now: () => now });
221 const { viewer, draw, state, pending, onOrientationRequested } = fixture();
222 models.resolve({ asset: asset(), dispose: vi.fn() });
223 await Promise.resolve();
224 const screen = {
225 width: 1398,
226 height: 2034,
227 orientation: "portrait" as const,
228 screenId: 1,
229 hingeAngle: 0,
230 };
231 viewer.setScreen(screen);
232 viewer.resize(500, 700, 2);
233 draw();
234 const initial = state.views.at(-1)!.quaternion.clone();
235 viewer.setInteractionActive(true, "orbit");
236 viewer.orbit(0, Math.PI / 3);
237 draw();
238 const held = state.views.at(-1)!.quaternion.clone();
239 now = 1000;
240 viewer.frameUpdated(1);
241 draw();
242 expect(state.views.at(-1)!.quaternion.angleTo(held)).toBeLessThan(1e-6);
243 expect(onOrientationRequested).not.toHaveBeenCalled();
244 viewer.setInteractionActive(false, "orbit");
245 draw();
246 expect(onOrientationRequested).toHaveBeenCalledOnce();
247 const chosen = state.views.at(-1)!.quaternion.clone();
248 expect(chosen.angleTo(initial)).toBeGreaterThan(1);
249 viewer.setScreen({ ...screen }); // Native config confirms the sensor command, even if the app stays portrait.
250 draw();
251 expect(state.views.at(-1)!.quaternion.angleTo(chosen)).toBeLessThan(1e-6);
252 expect(pending.size).toBe(0);
253 viewer.resetPose();
254 draw();
255 expect(onOrientationRequested).toHaveBeenCalledTimes(2);
256 expect(onOrientationRequested.mock.lastCall?.[0]).toBe("portrait");
257 viewer.setScreen({ ...screen });
258 draw();
259 expect(pending.size).toBe(0);
260 viewer.dispose();
261});
262
263it("settles a hinge transition after a throttled frame instead of stretching time", async () => {
264 let now = 100;
265 vi.stubGlobal("performance", { now: () => now });
266 const { viewer, draw, pending } = fixture(false);
267 models.resolve({ asset: asset(), dispose: vi.fn() });
268 await Promise.resolve();
269 viewer.resize(400, 700, 2);
270 viewer.setScreen({
271 width: 1398,
272 height: 2034,
273 orientation: "portrait",
274 screenId: 1,
275 hingeAngle: 0,
276 });
277 draw();
278 viewer.setScreen({
279 width: 2007,
280 height: 2853,
281 orientation: "portrait",
282 screenId: 3,
283 hingeAngle: 180,
284 });
285 now += 1_000;
286 draw();
287 draw();
288 expect(pending.size).toBe(0);
289 viewer.dispose();
290});
291
292it("faces a display handed off by native rotation without requesting another sensor rotation", async () => {
293 let now = 0;
294 vi.stubGlobal("performance", { now: () => now });
295 const { viewer, draw, pending, onOrientationRequested } = fixture();
296 const loaded = asset();
297 models.resolve({ asset: loaded, dispose: vi.fn() });
298 await Promise.resolve();
299 viewer.resize(500, 700, 2);
300 viewer.setScreen({
301 width: 2007,
302 height: 2853,
303 orientation: "portrait",
304 screenId: 3,
305 hingeAngle: 90,
306 });
307 draw();
308 viewer.setInteractionActive(true, "orbit");
309 viewer.orbit(0, Math.PI / 6);
310 viewer.setInteractionActive(false, "orbit");
311 now = 1000;
312 draw();
313 const requests = onOrientationRequested.mock.calls.length;
314 expect(requests).toBe(1);
315 viewer.setScreen({
316 width: 1398,
317 height: 2034,
318 orientation: "landscape_left",
319 screenId: 1,
320 hingeAngle: 90,
321 });
322 draw();
323 draw();
324 const cover = loaded.getObjectByName("cover-display") as Mesh;
325 const normal = new Vector3()
326 .fromBufferAttribute(cover.geometry.getAttribute("normal"), 0)
327 .transformDirection(cover.matrixWorld);
328 expect(normal.z).toBeGreaterThan(0.45);
329 expect(onOrientationRequested).toHaveBeenCalledTimes(requests);
330 expect(pending.size).toBe(0);
331 viewer.dispose();
332});
333
334it.each(["book", "laptop", "open"] as const)(
335 "faces native cover readback for %s even when configuration arrives before the model",
336 async (hingePose) => {
337 const { viewer, draw } = fixture();
338 viewer.resize(500, 700, 2);
339 viewer.setScreen({
340 width: 1398,
341 height: 2034,
342 orientation: "landscape_left",
343 screenId: 1,
344 hingeAngle: hingePose === "open" ? 180 : 90,
345 hingePose,
346 });
347 const loaded = asset();
348 models.resolve({ asset: loaded, dispose: vi.fn() });
349 await Promise.resolve();
350 draw();
351 draw();
352 const cover = loaded.getObjectByName("cover-display") as Mesh;
353 const normal = new Vector3()
354 .fromBufferAttribute(cover.geometry.getAttribute("normal"), 0)
355 .transformDirection(cover.matrixWorld);
356 expect(normal.z).toBeGreaterThan(0.45);
357 viewer.dispose();
358 },
359);
360
361it("keeps default-zoom orbits framed across folds and fits the current assembly", async () => {
362 const { viewer, draw, state } = fixture();
363 models.resolve({ asset: asset(), dispose: vi.fn() });
364 await Promise.resolve();
365 viewer.resize(380, 620, 2);
366 viewer.resetPose();
367 viewer.setInteractionActive(true, "orbit");
368 let minimumDistance = Infinity;
369 let maximumDistance = 0;
370 for (const hingeAngle of [0, 30, 90, 127, 180]) {
371 viewer.setScreen({
372 width: 2007,
373 height: 2853,
374 orientation: "portrait",
375 screenId: 3,
376 hingeAngle,
377 });
378 for (let turn = 0; turn < 16; turn++) {
379 viewer.orbit(0.1, turn % 2 ? -0.08 : 0.08);
380 draw();
381 const { camera, bounds } = state.views.at(-1)!;
382 const center = bounds.getCenter(new Vector3()).project(camera);
383 expect(Math.abs(center.x)).toBeLessThan(0.18);
384 expect(Math.abs(center.y)).toBeLessThan(0.18);
385 state.frames.at(-1)!.traverse((object) => {
386 if (!(object instanceof Mesh)) return;
387 const positions = object.geometry.getAttribute("position");
388 for (let index = 0; index < positions.count; index++) {
389 const point = new Vector3()
390 .fromBufferAttribute(positions, index)
391 .applyMatrix4(object.matrixWorld)
392 .project(camera);
393 expect(Math.abs(point.x)).toBeLessThan(1);
394 expect(Math.abs(point.y)).toBeLessThan(1);
395 }
396 });
397 minimumDistance = Math.min(minimumDistance, camera.position.z);
398 maximumDistance = Math.max(maximumDistance, camera.position.z);
399 }
400 }
401 expect(maximumDistance).toBeGreaterThan(minimumDistance);
402 viewer.dispose();
403});
404
405it("previews hinge articulation while preserving the laptop lid orientation and blocking unconfirmed input", async () => {
406 const { viewer, draw, pending } = fixture();
407 const loaded = asset();
408 models.resolve({ asset: loaded, dispose: vi.fn() });
409 await Promise.resolve();
410 viewer.resize(500, 700, 2);
411 viewer.setScreen({
412 width: 2853,
413 height: 2007,
414 orientation: "landscape_left",
415 screenId: 3,
416 hingeAngle: 90,
417 hingePose: "laptop",
418 });
419 draw();
420 const lid = loaded.getObjectByName("right-half")!;
421 const presentation = lid.getWorldQuaternion(new Rotation());
422 for (const value of [127, 50, 170]) {
423 viewer.setHingePreview(value);
424 draw();
425 expect(loaded.getObjectByName("left-half")!.rotation.y).toBeCloseTo(
426 ((180 - value) * Math.PI) / 360,
427 );
428 expect(lid.getWorldQuaternion(new Rotation()).angleTo(presentation)).toBeLessThan(0.00001);
429 expect(viewer.screenPoint(0.5, 0.5)).toBeNull();
430 viewer.setScreen({
431 width: 2007,
432 height: 2853,
433 orientation: "portrait",
434 screenId: 3,
435 hingeAngle: value,
436 hingePose: null,
437 });
438 draw();
439 expect(lid.getWorldQuaternion(new Rotation()).angleTo(presentation)).toBeLessThan(0.00001);
440 }
441 viewer.setHingePreview(null);
442 draw();
443 expect(pending.size).toBe(0);
444 expect(lid.getWorldQuaternion(new Rotation()).angleTo(presentation)).toBeLessThan(0.00001);
445 viewer.dispose();
446});
447
448it("does not restart an animated preset on duplicate native configurations", async () => {
449 let now = 100;
450 vi.stubGlobal("performance", { now: () => now });
451 const { viewer, draw, pending } = fixture(false);
452 models.resolve({ asset: asset(), dispose: vi.fn() });
453 await Promise.resolve();
454 viewer.resize(400, 700, 2);
455 viewer.setScreen({
456 width: 1398,
457 height: 2034,
458 orientation: "portrait",
459 screenId: 1,
460 hingeAngle: 0,
461 hingePose: "closed",
462 });
463 draw();
464 const next = {
465 width: 2007,
466 height: 2853,
467 orientation: "portrait" as const,
468 screenId: 3,
469 hingeAngle: 180,
470 hingePose: "open" as const,
471 };
472 viewer.setScreen(next);
473 now += 1_000;
474 viewer.setScreen(next);
475 draw();
476 now += 1000;
477 draw();
478 draw();
479 expect(pending.size).toBe(0);
480 viewer.dispose();
481});
482
483it("lets standalone rotation leave Laptop and Tent and stand a closed device upright after hinge edits", async () => {
484 const { viewer, draw, state } = fixture();
485 models.resolve({ asset: asset(), dispose: vi.fn() });
486 await Promise.resolve();
487 viewer.resize(500, 700, 2);
488 viewer.resetPose();
489 for (const hingePose of ["laptop", "tent"] as const) {
490 viewer.setScreen({
491 width: 1398,
492 height: 2034,
493 orientation: "landscape_left",
494 screenId: 1,
495 hingeAngle: 90,
496 hingePose,
497 });
498 draw();
499 viewer.setHingePreview(0);
500 viewer.setScreen({
501 width: 1398,
502 height: 2034,
503 orientation: "landscape_left",
504 screenId: 1,
505 hingeAngle: 0,
506 hingePose: null,
507 });
508 viewer.setHingePreview(null);
509 draw();
510 viewer.cancelInput(); // Explicit toolbar rotation ends folding ownership.
511 viewer.setScreen({
512 width: 1398,
513 height: 2034,
514 orientation: "portrait",
515 screenId: 1,
516 hingeAngle: 0,
517 hingePose: null,
518 });
519 draw();
520 const upright = new Rotation().setFromEuler(new Euler(0, Math.PI / 2, 0, "YXZ"));
521 expect(state.views.at(-1)!.quaternion.angleTo(upright)).toBeLessThan(0.00001);
522 const bounds = state.views.at(-1)!.bounds;
523 expect(bounds.max.y - bounds.min.y).toBeGreaterThan(bounds.max.x - bounds.min.x);
524 }
525 viewer.dispose();
526});
527
528it("snaps onto the opposite screen, requests native handoff once and retains the chosen view on readback", async () => {
529 vi.useFakeTimers();
530 let now = 0;
531 vi.stubGlobal("performance", { now: () => now });
532 const onPanelRequested = vi.fn();
533 const { viewer, draw, state, pending, onOrientationRequested } = fixture(
534 true,
535 vi.fn(),
536 onPanelRequested,
537 );
538 models.resolve({ asset: asset(), dispose: vi.fn() });
539 await Promise.resolve();
540 viewer.resize(500, 700, 2);
541 const inner = {
542 width: 2007,
543 height: 2853,
544 orientation: "portrait" as const,
545 screenId: 3,
546 hingeAngle: 90,
547 supportsPhysicalOrientation: true,
548 };
549 viewer.setScreen(inner);
550 draw();
551 viewer.setInteractionActive(true, "orbit");
552 viewer.orbit(Math.PI / 3, 0);
553 draw();
554 viewer.setInteractionActive(false, "orbit");
555 draw();
556 expect(onPanelRequested).toHaveBeenCalledExactlyOnceWith(1);
557 expect(onOrientationRequested).not.toHaveBeenCalled();
558 expect(viewer.screenPoint(0.5, 0.5)).toBeNull();
559 const chosen = state.views.at(-1)!.quaternion.clone();
560 viewer.setScreen({ ...inner, hingePose: null }); // Command acknowledgement precedes sensor readback.
561 draw();
562 expect(state.views.at(-1)!.quaternion.angleTo(chosen)).toBeLessThan(1e-6);
563 viewer.setScreen({ ...inner, width: 1398, height: 2034, screenId: 1 });
564 draw();
565 expect(state.views.at(-1)!.quaternion.angleTo(chosen)).toBeLessThan(1e-6);
566 vi.advanceTimersByTime(5000);
567 expect(onPanelRequested).toHaveBeenCalledTimes(1);
568 expect(pending.size).toBe(0);
569 viewer.setInteractionActive(true, "orbit");
570 viewer.orbit(-Math.PI / 3, 0);
571 draw();
572 viewer.setInteractionActive(false, "orbit");
573 draw();
574 expect(onPanelRequested).toHaveBeenLastCalledWith(3);
575 const reverse = state.views.at(-1)!.quaternion.clone();
576 viewer.setScreen(inner);
577 draw();
578 expect(state.views.at(-1)!.quaternion.angleTo(reverse)).toBeLessThan(1e-6);
579 vi.advanceTimersByTime(5000);
580 draw();
581 expect(pending.size).toBe(0);
582 expect(onPanelRequested).toHaveBeenCalledTimes(2);
583 viewer.dispose();
584 vi.useRealTimers();
585});
586
587it("rolls an unconfirmed opposite-screen snap back to the native active screen after timeout", async () => {
588 vi.useFakeTimers();
589 const request = vi.fn();
590 const { viewer, draw, state, pending } = fixture(true, vi.fn(), request);
591 models.resolve({ asset: asset(), dispose: vi.fn() });
592 await Promise.resolve();
593 viewer.resize(500, 700, 2);
594 viewer.setScreen({
595 width: 2007,
596 height: 2853,
597 orientation: "portrait",
598 screenId: 3,
599 hingeAngle: 90,
600 supportsPhysicalOrientation: true,
601 });
602 draw();
603 viewer.setInteractionActive(true, "orbit");
604 viewer.orbit(Math.PI / 3, 0);
605 draw();
606 viewer.setInteractionActive(false, "orbit");
607 draw();
608 const unconfirmed = state.views.at(-1)!.quaternion.clone();
609 expect(request).toHaveBeenCalledExactlyOnceWith(1);
610 vi.advanceTimersByTime(5000);
611 draw();
612 expect(state.views.at(-1)!.quaternion.angleTo(unconfirmed)).toBeGreaterThan(0.5);
613 expect(request).toHaveBeenCalledOnce();
614 draw();
615 expect(pending.size).toBe(0);
616 viewer.dispose();
617});
618
619it("uses the elected primary frame during handoff and never lets a stale fixed-panel feed overwrite it", async () => {
620 const { viewer, drawImage } = fixture();
621 const cover = {
622 width: 1398,
623 height: 2034,
624 orientation: "portrait" as const,
625 screenId: 1,
626 hingeAngle: 90,
627 };
628 viewer.setScreen(cover);
629 const primary = { width: 1398, height: 2034 } as HTMLCanvasElement;
630 viewer.frameUpdated(1, primary);
631 expect(drawImage.mock.calls.at(-1)?.[0]).toBe(primary);
632 const uploads = drawImage.mock.calls.length;
633 viewer.frameUpdated(1);
634 expect(drawImage).toHaveBeenCalledTimes(uploads);
635 viewer.frameUpdated(3, primary);
636 expect(drawImage).toHaveBeenCalledTimes(uploads);
637 viewer.setScreen({ ...cover, width: 2007, height: 2853, screenId: 3 });
638 const inner = { width: 2007, height: 2853 } as HTMLCanvasElement;
639 viewer.frameUpdated(3, inner);
640 expect(drawImage.mock.calls.at(-1)?.[0]).toBe(inner);
641 viewer.setScreen(cover);
642 viewer.frameUpdated(1);
643 expect(drawImage.mock.calls.length).toBeGreaterThan(uploads + 1);
644 viewer.dispose();
645});
646
647it.each(["left", "right"] as const)(
648 "keeps the primary surface oriented when pinching over the %s half through closure and reopening",
649 async (face) => {
650 const { viewer, draw, state, pending, onOrientationRequested } = fixture();
651 const loaded = asset();
652 models.resolve({ asset: loaded, dispose: vi.fn() });
653 await Promise.resolve();
654 viewer.resize(500, 700, 2);
655 const inner = {
656 width: 2007,
657 height: 2853,
658 orientation: "portrait" as const,
659 screenId: 3,
660 hingeAngle: 180,
661 hingePose: "open" as const,
662 };
663 viewer.setScreen(inner);
664 draw();
665 expect(viewer.beginHinge(-1, -1)).toBe(false);
666 const leaf = loaded.getObjectByName("right-half")!;
667 const mesh = loaded.getObjectByName(`inner-display-${face}`) as Mesh;
668 const center = mesh.geometry.boundingBox!.getCenter(new Vector3());
669 const point = mesh.localToWorld(center).project(state.views.at(-1)!.camera);
670 expect(viewer.beginHinge((point.x + 1) / 2, (1 - point.y) / 2)).toBe(true);
671 const orientation = leaf.getWorldQuaternion(new Rotation());
672 const partner = loaded.getObjectByName("left-half")!;
673 const partnerStart = partner.getWorldQuaternion(new Rotation());
674 for (const value of [150, 90, 30, 0, 30, 90, 180]) {
675 viewer.setHingePreview(value);
676 draw();
677 expect(leaf.getWorldQuaternion(new Rotation()).angleTo(orientation)).toBeLessThan(1e-6);
678 if (value === 90)
679 expect(partner.getWorldQuaternion(new Rotation()).angleTo(partnerStart)).toBeGreaterThan(1);
680 const next =
681 value === 0
682 ? { ...inner, width: 1398, height: 2034, screenId: 1, hingeAngle: value, hingePose: null }
683 : { ...inner, hingeAngle: value, hingePose: null };
684 viewer.setScreen(next);
685 viewer.setHingePreview(null);
686 draw();
687 viewer.setScreen({ ...next }); // Late native readback cannot change the view.
688 draw();
689 expect(leaf.getWorldQuaternion(new Rotation()).angleTo(orientation)).toBeLessThan(1e-6);
690 expect(viewer.screenPoint(0.5, 0.5)).toBeNull(); // No fresh native frame yet.
691 if (value === 0) {
692 const cover = loaded.getObjectByName("cover-display") as Mesh;
693 const center = cover.geometry.boundingBox!.getCenter(new Vector3());
694 const hit = cover.localToWorld(center).project(state.views.at(-1)!.camera);
695 expect(viewer.beginHinge((hit.x + 1) / 2, (1 - hit.y) / 2)).toBe(true);
696 }
697 }
698 expect(onOrientationRequested).not.toHaveBeenCalled();
699 draw();
700 expect(pending.size).toBe(0);
701 viewer.dispose();
702 },
703);
packages/client-runtime/src/device/duoViewer.ts created+566
......@@ -0,0 +1,566 @@
1// @effect-diagnostics globalTimers:off - Native display handoff has a bounded acknowledgement window.
2import {
3 AmbientLight,
4 Box3,
5 CanvasTexture,
6 DirectionalLight,
7 Euler,
8 LinearFilter,
9 PerspectiveCamera,
10 PMREMGenerator,
11 Quaternion,
12 Scene,
13 SRGBColorSpace,
14 Vector3,
15 WebGLRenderer,
16} from "three";
17import { RoomEnvironment } from "three/addons/environments/RoomEnvironment.js";
18import {
19 createDuoScene,
20 duoDisplayKey,
21 duoFrameMatches,
22 type DuoPanelId,
23 type DuoHingeLeaf,
24} from "./duoScene.ts";
25import { loadDeviceModel } from "./modelScene.ts";
26import { createDeviceModelSlot, type DeviceModelSource } from "./model.ts";
27import { createDeviceFraming } from "./deviceFraming.ts";
28import { createDeviceMotion } from "./deviceMotion.ts";
29import { duoViewSnaps, nearestDuoView, type DuoRestFace } from "./duoSnap.ts";
30import { createRenderScheduler } from "./renderScheduler.ts";
31import type { DeviceScreenSize } from "./stream.ts";
32import type { DuoPose } from "./duoControl.ts";
33
34export interface DuoViewer {
35 readonly setScreen: (screen: DeviceScreenSize | null) => void;
36 readonly setHingePreview: (angle: number | null) => void;
37 readonly setInteractionActive: (active: boolean, mode?: "touch" | "orbit") => void;
38 readonly rejectOrientation: () => void;
39 readonly frameUpdated: (panel: DuoPanelId, primary?: HTMLCanvasElement) => void;
40 readonly resize: (width: number, height: number, pixelRatio: number) => void;
41 readonly screenPoint: (
42 x: number,
43 y: number,
44 captured?: boolean,
45 ) => { x: number; y: number } | null;
46 readonly orbit: (x: number, y: number) => void;
47 readonly beginHinge: (x: number, y: number) => boolean;
48 readonly resetPose: () => void;
49 readonly cancelInput: () => void;
50 readonly dispose: () => void;
51}
52
53/** On-demand renderer for the articulated body. One inner framebuffer spans both leaves; HID belongs to the stream. */
54export function createDuoViewer(options: {
55 canvas: HTMLCanvasElement;
56 sources: Record<DuoPanelId, HTMLCanvasElement>;
57 model: DeviceModelSource;
58 onUnavailable: () => void;
59 onModelError?: (cause: unknown) => void;
60 onPanelRequested?: (panel: DuoPanelId) => void;
61 onOrientationRequested?: (orientation: DeviceScreenSize["orientation"]) => void;
62}): DuoViewer {
63 const surfaces = ([1, 3] as const)
64 .map((id) => {
65 const canvas = document.createElement("canvas");
66 canvas.width = id === 1 ? 784 : 1600;
67 canvas.height = id === 1 ? 1140 : 1125;
68 const context = canvas.getContext("2d", { alpha: false });
69 if (!context) throw new Error("Duo display canvas is unavailable");
70 context.fillStyle = "#080a10";
71 context.fillRect(0, 0, canvas.width, canvas.height);
72 return { id, canvas, context };
73 })
74 .map(({ id, canvas, context }) => {
75 const texture = new CanvasTexture(canvas);
76 texture.colorSpace = SRGBColorSpace;
77 texture.minFilter = texture.magFilter = LinearFilter;
78 texture.generateMipmaps = false;
79 return { id, canvas, context, texture };
80 });
81 const renderer = new WebGLRenderer({
82 canvas: options.canvas,
83 alpha: true,
84 antialias: true,
85 powerPreference: "low-power",
86 });
87 renderer.outputColorSpace = SRGBColorSpace;
88 const scene = new Scene();
89 const environment = (() => {
90 const generator = new PMREMGenerator(renderer);
91 const room = new RoomEnvironment();
92 try {
93 return generator.fromScene(room, 0.04);
94 } catch (cause) {
95 for (const surface of surfaces) surface.texture.dispose();
96 renderer.dispose();
97 renderer.forceContextLoss();
98 throw cause;
99 } finally {
100 room.dispose();
101 generator.dispose();
102 }
103 })();
104 scene.environment = environment.texture;
105 const camera = new PerspectiveCamera(36, 1, 0.1, 150);
106 const key = new DirectionalLight(0xffffff, 4);
107 key.position.set(-6, 10, 14);
108 const fill = new DirectionalLight(0xc7dcff, 2);
109 fill.position.set(8, -3, -5);
110 scene.add(new AmbientLight(0xffffff, 2.4), key, fill);
111 let model: ReturnType<typeof createDuoScene> | null = null;
112 let screen: DeviceScreenSize | null = null;
113 let readyKey = "";
114 let primaryKey = "";
115 let activationAt = 0;
116 let disposed = false;
117 let viewport = { width: 0, height: 0, ratio: 1 };
118 let buffer = { width: 0, height: 0, ratio: 0 };
119 let restFace: DuoRestFace = "inside";
120 let requestedOrientation: DeviceScreenSize["orientation"] | null = null;
121 let viewOrientation: DeviceScreenSize["orientation"] | null = null;
122 const pivot = new Vector3();
123 const targetPivot = new Vector3();
124 let requestedPanel: DuoPanelId | null = null;
125 let handoffTimer: ReturnType<typeof setTimeout> | null = null;
126 const clearHandoff = () => {
127 if (handoffTimer) clearTimeout(handoffTimer);
128 handoffTimer = null;
129 requestedPanel = null;
130 };
131 const activeSnaps = () =>
132 model
133 ? duoViewSnaps(
134 model.restFrames(screen?.screenId === 1 ? 1 : 3),
135 screen?.screenId === 1 ? 1 : 3,
136 )
137 : [];
138 const snaps = () => {
139 if (!model || !screen?.supportsPhysicalOrientation || !options.onPanelRequested)
140 return activeSnaps();
141 const cover = duoViewSnaps(model.restFrames(1), 1);
142 // A shut inner display is occluded and cannot become a useful rest view.
143 return angle > 20 && angle < 180
144 ? [...duoViewSnaps(model.restFrames(3), 3), ...cover]
145 : activeSnaps();
146 };
147 const orbit = createDeviceMotion({
148 choose(rotation) {
149 const snap = nearestDuoView(rotation, snaps());
150 if (!snap) return rotation;
151 restFace = snap.face;
152 const panel = snap.face === "cover" ? 1 : 3;
153 if (screen && panel !== (requestedPanel ?? screen.screenId) && options.onPanelRequested) {
154 clearHandoff();
155 requestedPanel = panel;
156 model?.cancelInput();
157 options.onPanelRequested(panel);
158 handoffTimer = setTimeout(() => {
159 clearHandoff();
160 const confirmed = nearestDuoView(orbit.rotation, activeSnaps());
161 if (confirmed) {
162 restFace = confirmed.face;
163 orbit.setPose(confirmed.rotation, performance.now());
164 }
165 scheduler.invalidate();
166 }, 5000);
167 } else if (
168 screen &&
169 requestedPanel === null &&
170 snap.orientation !== viewOrientation &&
171 options.onOrientationRequested
172 ) {
173 viewOrientation = snap.orientation;
174 requestedOrientation = snap.orientation;
175 options.onOrientationRequested(snap.orientation);
176 }
177 return snap.rotation;
178 },
179 });
180 let angle = 180;
181 let targetAngle = 180;
182 let previewAngle: number | null = null;
183 let hingeLeaf: DuoHingeLeaf | null = null;
184 let appliedAngle = Number.NaN;
185 let interactionActive = false;
186 const framing = createDeviceFraming();
187 const framingBounds = new Box3();
188 let firstPose = true;
189 let physicalPose: DuoPose = "open";
190 let presentationAngle = 180;
191 let targetPresentation = new Quaternion();
192 let lastTime = 0;
193 const reduced =
194 typeof matchMedia === "function" && matchMedia("(prefers-reduced-motion: reduce)").matches;
195 const applyPose = () => {
196 if (!model) return;
197 const before = hingeLeaf && appliedAngle !== angle ? model.leafRotation(hingeLeaf) : null;
198 model.setAngle(angle);
199 appliedAngle = angle;
200 if (before && hingeLeaf) {
201 // The primary surface stays in camera space; its partner supplies the fold.
202 // Rebase both the displayed rotation and its rest target so releasing a
203 // pinch cannot resume the pre-fold body rotation.
204 const correction = before.multiply(model.leafRotation(hingeLeaf).invert());
205 if (correction.angleTo(new Quaternion()) > 1e-8)
206 orbit.setPose(orbit.rotation.clone().multiply(correction), performance.now(), true);
207 }
208 model.root.quaternion.copy(orbit.rotation);
209 model.root.position.set(0, 0, 0);
210 const frames = model.restFrames(restFace === "cover" ? 1 : 3);
211 const frame = frames.find((frame) => frame.face === restFace) ?? frames[0];
212 targetPivot.copy(frame?.center ?? new Vector3());
213 model.root.position.copy(pivot).applyQuaternion(model.root.quaternion).negate();
214 };
215 const updateCamera = () => {
216 camera.position.set(framing.center.x, framing.center.y, framing.distance());
217 camera.lookAt(framing.center.x, framing.center.y, 0);
218 camera.updateProjectionMatrix();
219 };
220 const fit = (immediate = false) => {
221 if (!model || !viewport.width || !viewport.height) return;
222 camera.aspect = viewport.width / viewport.height;
223 framing.setBounds(
224 framingBounds.setFromObject(model.root),
225 (camera.fov * Math.PI) / 360,
226 camera.aspect,
227 performance.now(),
228 immediate,
229 );
230 updateCamera();
231 };
232 const moving = () =>
233 Math.abs(angle - targetAngle) > 0.01 || pivot.distanceTo(targetPivot) > 0.001;
234 const scheduler = createRenderScheduler(() => {
235 if (disposed || !model || !viewport.width || !viewport.height) return;
236 try {
237 const now = performance.now();
238 const elapsed = Math.max(0, (now - lastTime) / 1000);
239 const amount = reduced
240 ? 1
241 : 1 - Math.exp(-14 * Math.max(0, (now - lastTime) / 1000 || 0.016));
242 lastTime = now;
243 const inMotion = moving();
244 const orbitChanged = orbit.advance(now, reduced);
245 if (inMotion) {
246 angle += (targetAngle - angle) * amount;
247
248 if (Math.abs(angle - targetAngle) <= 0.01) angle = targetAngle;
249 applyPose();
250 pivot.lerp(targetPivot, amount);
251 if (pivot.distanceTo(targetPivot) <= 0.001) pivot.copy(targetPivot);
252 applyPose();
253 fit(reduced || elapsed > 0.5);
254 }
255 if (orbitChanged) {
256 applyPose();
257 fit(reduced || elapsed > 0.5);
258 }
259 if (framing.advance(now, reduced)) updateCamera();
260 if (
261 buffer.width !== viewport.width ||
262 buffer.height !== viewport.height ||
263 buffer.ratio !== viewport.ratio
264 ) {
265 renderer.setDrawingBufferSize(viewport.width, viewport.height, viewport.ratio);
266 buffer = viewport;
267 }
268 renderer.render(scene, camera);
269 if (inMotion || moving() || orbit.needsFrame() || framing.needsFrame())
270 scheduler.invalidate();
271 } catch {
272 options.onUnavailable();
273 }
274 });
275 const slot = createDeviceModelSlot({
276 load: loadDeviceModel,
277 onError(cause) {
278 options.onModelError?.(cause);
279 options.onUnavailable();
280 },
281 install(loaded) {
282 const next = loaded
283 ? createDuoScene(loaded.asset, { 1: surfaces[0]!.texture, 3: surfaces[1]!.texture })
284 : null;
285 if (model) {
286 scene.remove(model.root);
287 model.dispose();
288 }
289 model = next;
290 appliedAngle = Number.NaN;
291 if (!model || disposed) return;
292 scene.add(model.root);
293 applyPose();
294 if (screen?.screenId === 1) {
295 const snap = nearestDuoView(orbit.rotation, activeSnaps());
296 if (snap) {
297 restFace = snap.face;
298 orbit.setPose(snap.rotation, performance.now(), true);
299 applyPose();
300 }
301 }
302 pivot.copy(targetPivot);
303 applyPose();
304 fit();
305 scheduler.invalidate();
306 },
307 });
308 const lost = (event: Event) => {
309 event.preventDefault();
310 options.onUnavailable();
311 };
312 options.canvas.addEventListener("webglcontextlost", lost);
313 slot.set(options.model);
314 return {
315 setScreen(next) {
316 if (disposed) return;
317 const wasMoving = moving() || orbit.needsFrame();
318 if (duoDisplayKey(screen) !== duoDisplayKey(next)) {
319 readyKey = "";
320 activationAt = performance.now();
321 model?.cancelInput();
322 }
323 const previous = screen;
324 const ownedHandoff = requestedPanel !== null;
325 if (!next || next.screenId === requestedPanel) clearHandoff();
326 const ownedRotation = requestedOrientation !== null && next?.screenId === previous?.screenId;
327 const changedDisplay = next?.screenId !== previous?.screenId;
328 requestedOrientation = null;
329 screen = next;
330 const changedPose = next?.hingePose && next.hingePose !== previous?.hingePose;
331 const folding = hingeLeaf !== null && !changedPose;
332 const rotated =
333 next?.screenId === previous?.screenId &&
334 next?.hingeAngle === previous?.hingeAngle &&
335 next?.orientation !== previous?.orientation;
336 const leftPhysicalPose =
337 rotated &&
338 !ownedRotation &&
339 !ownedHandoff &&
340 !folding &&
341 !next?.hingePose &&
342 previewAngle === null;
343 if (firstPose || changedPose || leftPhysicalPose) {
344 physicalPose = next?.hingePose ?? (next?.screenId === 1 ? "closed" : "open");
345 presentationAngle = next?.hingeAngle ?? (next?.screenId === 1 ? 0 : 180);
346 }
347 // A command reply/config confirms hinge state. Display identity, never an angle heuristic, owns input.
348 targetAngle = previewAngle ?? next?.hingeAngle ?? (next?.screenId === 1 ? 0 : 180);
349 // Native angle commands clear hingePose. They change articulation only;
350 // preserve the viewing pose, including laptop/tent and user orbit. A
351 // separate rotation clears the physical preset and follows panel orientation.
352 if (firstPose || changedPose || (rotated && !ownedRotation && !ownedHandoff && !folding)) {
353 hingeLeaf = null;
354 viewOrientation = next?.orientation ?? null;
355 const poseAngle = presentationAngle;
356 const fold = ((180 - poseAngle) * Math.PI) / 360;
357 let roll = physicalPose === "closed" ? 0 : Math.PI / 2;
358 if (next && next.width < next.height) {
359 if (next.orientation === "landscape_left") roll -= Math.PI / 2;
360 if (next.orientation === "landscape_right") roll += Math.PI / 2;
361 }
362 if (next?.orientation === "portrait_upside_down") roll -= Math.PI;
363 const physical =
364 physicalPose === "laptop"
365 ? new Euler(-fold + Math.PI / 9, -Math.PI / 9, Math.PI / 2, "YXZ")
366 : physicalPose === "tent"
367 ? new Euler(Math.PI / 2 + Math.PI / 18, -Math.PI / 9, -Math.PI / 2, "YXZ")
368 : new Euler(0, (Math.PI / 2) * Math.pow(1 - poseAngle / 180, 3), 0, "YXZ");
369 targetPresentation = new Quaternion().setFromEuler(physical);
370 if (physicalPose !== "laptop" && physicalPose !== "tent")
371 targetPresentation.premultiply(
372 new Quaternion().setFromAxisAngle(new Vector3(0, 0, 1), roll),
373 );
374 if (next?.screenId === 1) {
375 // Native readback can retain the cover even for an open physical preset.
376 // A screen-facing rest must use that display's actual hinged normal.
377 const snap = nearestDuoView(targetPresentation, activeSnaps());
378 if (snap) targetPresentation.copy(snap.rotation);
379 }
380 if (firstPose) angle = targetAngle;
381 restFace = next?.screenId === 1 ? "cover" : physicalPose === "laptop" ? "right" : "inside";
382 orbit.setPose(targetPresentation, performance.now(), firstPose);
383 firstPose = false;
384 } else if (next && changedDisplay && !ownedHandoff && !folding) {
385 // A sensor rotation can hand ownership to the other native display.
386 // Face that display without sending another rotation and creating a feedback loop.
387 const snap = nearestDuoView(orbit.rotation, activeSnaps());
388 if (snap) {
389 restFace = snap.face;
390 orbit.setPose(snap.rotation, performance.now());
391 }
392 }
393 if (!wasMoving) lastTime = performance.now();
394 applyPose();
395 fit();
396 scheduler.invalidate();
397 },
398 setHingePreview(next) {
399 if (disposed || (next !== null && (!Number.isFinite(next) || next < 0 || next > 180))) return;
400 if (!moving()) lastTime = performance.now();
401 previewAngle = next;
402 if (next !== null && !hingeLeaf && model) {
403 clearHandoff();
404 requestedOrientation = null;
405 hingeLeaf = screen?.screenId === 1 && angle > 20 ? "left" : "right";
406 orbit.setPose(orbit.rotation.clone(), performance.now(), true);
407 }
408 orbit.hold(interactionActive || next !== null, performance.now());
409 targetAngle = next ?? screen?.hingeAngle ?? (screen?.screenId === 1 ? 0 : 180);
410 if (next !== null) {
411 angle = next;
412 applyPose();
413 fit();
414 }
415 model?.cancelInput();
416 scheduler.invalidate();
417 },
418 setInteractionActive(active, mode = "touch") {
419 if (disposed) return;
420 if (mode === "orbit") {
421 if (active) hingeLeaf = null;
422 orbit.dragActive(active, performance.now());
423 } else {
424 interactionActive = active;
425 orbit.hold(active || previewAngle !== null, performance.now());
426 framing.hold(active, performance.now());
427 }
428 scheduler.invalidate();
429 },
430 rejectOrientation() {
431 if (disposed || (requestedOrientation === null && requestedPanel === null)) return;
432 clearHandoff();
433 requestedOrientation = null;
434 viewOrientation = screen?.orientation ?? null;
435 const confirmed = activeSnaps().filter((snap) => snap.orientation === screen?.orientation);
436 const snap = nearestDuoView(orbit.rotation, confirmed);
437 if (snap) {
438 restFace = snap.face;
439 orbit.setPose(snap.rotation, performance.now());
440 }
441 scheduler.invalidate();
442 },
443 frameUpdated(id, primary) {
444 if (disposed || screen?.screenId !== id) return;
445 const key = duoDisplayKey(screen);
446 // The elected native feed is authoritative during handoff. Fixed-panel
447 // encoders can retain an inactive blank until that surface changes again.
448 if (!primary && primaryKey === key) return;
449 const source = primary ?? options.sources[id];
450 if (!duoFrameMatches(source, screen)) return;
451 const surface = surfaces[id === 1 ? 0 : 1]!;
452 const { context, canvas } = surface;
453 // Ignore native shutdown blanks only while waiting for an activation. Steady black application content remains valid.
454 if (!primary && !readyKey && performance.now() - activationAt < 1500) {
455 const probe = document.createElement("canvas");
456 probe.width = probe.height = 8;
457 const probeContext = probe.getContext("2d", { willReadFrequently: true });
458 if (probeContext) {
459 probeContext.drawImage(source, 0, 0, 8, 8);
460 if (
461 !probeContext
462 .getImageData(0, 0, 8, 8)
463 .data.some((value, index) => index % 4 !== 3 && value > 3)
464 )
465 return;
466 }
467 }
468 context.save();
469 context.fillStyle = "#080a10";
470 context.fillRect(0, 0, canvas.width, canvas.height);
471 context.translate(canvas.width / 2, canvas.height / 2);
472 // The inner panel is mounted a quarter turn from the native framebuffer.
473 const rotate = id === 3;
474 if (rotate) context.rotate(Math.PI / 2);
475 const scale = Math.min(
476 canvas.width / (rotate ? source.height : source.width),
477 canvas.height / (rotate ? source.width : source.height),
478 );
479 context.drawImage(
480 source,
481 (-source.width * scale) / 2,
482 (-source.height * scale) / 2,
483 source.width * scale,
484 source.height * scale,
485 );
486 context.restore();
487 surface.texture.needsUpdate = true;
488 readyKey = duoDisplayKey(screen);
489 if (primary) primaryKey = readyKey;
490 scheduler.invalidate();
491 },
492 resize(width, height, ratio) {
493 if (disposed || ![width, height, ratio].every(Number.isFinite) || width <= 0 || height <= 0)
494 return;
495 viewport = { width, height, ratio: Math.min(2, Math.max(1, ratio)) };
496 fit(true);
497 scheduler.invalidate();
498 },
499 screenPoint(x, y, captured = false) {
500 if (
501 disposed ||
502 moving() ||
503 previewAngle !== null ||
504 requestedOrientation !== null ||
505 requestedPanel !== null
506 )
507 return null;
508 applyPose();
509 return model?.screenPoint(x, y, camera, screen, readyKey, captured) ?? null;
510 },
511 cancelInput() {
512 hingeLeaf = null;
513 model?.cancelInput();
514 },
515 orbit(x, y) {
516 if (disposed) return;
517 hingeLeaf = null;
518 orbit.orbit(x * viewport.width, y * viewport.height, performance.now());
519 scheduler.invalidate();
520 },
521 beginHinge(x, y) {
522 if (disposed || !model || !screen) return false;
523 applyPose();
524 const leaf = model.hingeLeafAt(x, y, camera);
525 if (!leaf) return false;
526 clearHandoff();
527 requestedOrientation = null;
528 // The right inner leaf and the shut cover share the front-facing plane.
529 // Holding that leaf lets the cover replace the inner image without a
530 // half turn, even when the pinch starts over the moving partner.
531 hingeLeaf = screen.screenId === 1 && angle > 20 ? "left" : "right";
532 orbit.setPose(orbit.rotation.clone(), performance.now(), true);
533 model.cancelInput();
534 return true;
535 },
536 resetPose() {
537 if (disposed) return;
538 hingeLeaf = null;
539 requestedOrientation = null;
540 clearHandoff();
541 const snap = activeSnaps().find((snap) => snap.orientation === screen?.orientation);
542 if (snap && snap.orientation !== viewOrientation && options.onOrientationRequested) {
543 viewOrientation = snap.orientation;
544 requestedOrientation = snap.orientation;
545 options.onOrientationRequested(snap.orientation);
546 }
547 orbit.reset(snap?.rotation ?? targetPresentation, performance.now());
548 applyPose();
549 fit();
550 scheduler.invalidate();
551 },
552 dispose() {
553 clearHandoff();
554 if (disposed) return;
555 disposed = true;
556 scheduler.dispose();
557 slot.dispose();
558 options.canvas.removeEventListener("webglcontextlost", lost);
559 for (const surface of surfaces) surface.texture.dispose();
560 scene.environment = null;
561 environment.dispose();
562 renderer.dispose();
563 renderer.forceContextLoss();
564 },
565 };
566}
packages/client-runtime/src/device/model.test.ts+1
......@@ -24,6 +24,7 @@ function fixture() {
2424it("matches only exact supported hardware, including iPad size and generation", () => {
2525 expect(resolveDeviceModelId("ios", "iPhone 18 Pro")).toBe(pro.id);
2626 expect(resolveDeviceModelId("ios", "iPhone 18 Pro Max")).toBe(max.id);
27 expect(resolveDeviceModelId("ios", "iPhone Duo")).toBe("iphone-duo");
2728 expect(resolveDeviceModelId("ios", "iPad Pro 13-inch (M5)")).toBe("ipad-pro-13-m5");
2829 for (const name of [
2930 "iPhone 17 Pro",
packages/client-runtime/src/device/model.ts+2-1
......@@ -1,6 +1,6 @@
11import type { DevicePlatform } from "@t3tools/contracts";
22
3export type DeviceModelId = "iphone-18-pro" | "iphone-18-pro-max" | "ipad-pro-13-m5";
3export type DeviceModelId = "iphone-18-pro" | "iphone-18-pro-max" | "ipad-pro-13-m5" | "iphone-duo";
44
55export interface DeviceModelSource {
66 readonly id: DeviceModelId;
......@@ -18,6 +18,7 @@ export type DeviceAssetSource = DeviceModelSource | DeviceAccessorySource;
1818/** Match actual hardware, never stretch an available model to impersonate another device. */
1919export function resolveDeviceModelId(platform: DevicePlatform, name: string): DeviceModelId | null {
2020 if (platform !== "ios") return null;
21 if (/^iPhone Duo$/i.test(name)) return "iphone-duo";
2122 if (/^iPhone 18 Pro Max$/i.test(name)) return "iphone-18-pro-max";
2223 if (/^iPhone 18 Pro$/i.test(name)) return "iphone-18-pro";
2324 if (/^iPad Pro 13-inch \(M5\)$/i.test(name)) return "ipad-pro-13-m5";
packages/client-runtime/src/device/stream.ts+239-16
......@@ -19,6 +19,14 @@
1919 * The decoder only runs while frames arrive and the viewer is attached; a
2020 * hidden panel calls `stop()` so an idle device costs nothing on the GPU.
2121 */
22import {
23 createDuoControl,
24 type DuoCommand,
25 type DuoControlState,
26 type DuoPose,
27} from "./duoControl.ts";
28import * as Schema from "effect/Schema";
29import * as Option from "effect/Option";
2230import { createCanvasFrameSink, type DeviceFrameSink } from "./frame.ts";
2331import { type DeviceHubAccess, withDeviceHubQuery } from "./hubAccess.ts";
2432import type { DevicePlatform } from "@t3tools/contracts";
......@@ -29,9 +37,55 @@ export interface DeviceScreenSize {
2937 readonly width: number;
3038 readonly height: number;
3139 readonly orientation: "portrait" | "portrait_upside_down" | "landscape_left" | "landscape_right";
40 readonly screenId?: number;
41 readonly supportsHingeAngle?: boolean;
42 readonly supportsPhysicalOrientation?: boolean;
43 readonly hingeAngle?: number;
44 readonly hingePose?: DuoPose | null;
45 readonly tableMode?: boolean;
46 readonly tableModeAvailable?: boolean;
47}
48
49const screenConfigSchema = Schema.Struct({
50 width: Schema.Finite.check(Schema.isGreaterThan(0)),
51 height: Schema.Finite.check(Schema.isGreaterThan(0)),
52 orientation: Schema.Literals([
53 "portrait",
54 "portrait_upside_down",
55 "landscape_left",
56 "landscape_right",
57 ]),
58 screenId: Schema.optionalKey(Schema.Number),
59 supportsHingeAngle: Schema.optionalKey(Schema.Boolean),
60 supportsPhysicalOrientation: Schema.optionalKey(Schema.Boolean),
61 hingeAngle: Schema.optionalKey(
62 Schema.Finite.check(Schema.isBetween({ minimum: 0, maximum: 180 })),
63 ),
64 hingePose: Schema.optionalKey(
65 Schema.NullOr(Schema.Literals(["closed", "book", "open", "laptop", "tent"])),
66 ),
67 tableMode: Schema.optionalKey(Schema.Boolean),
68 tableModeAvailable: Schema.optionalKey(Schema.Boolean),
69});
70const controlReplySchema = Schema.Struct({
71 requestId: Schema.Int,
72 ok: Schema.Boolean,
73 error: Schema.optionalKey(Schema.String),
74});
75const decodeScreenConfig = Schema.decodeUnknownOption(screenConfigSchema);
76const decodeControlReply = Schema.decodeUnknownOption(controlReplySchema);
77
78export interface DuoPanelSinks {
79 readonly cover: DeviceFrameSink;
80 readonly inner: DeviceFrameSink;
81 /** Invalidate captured input synchronously, before React can commit the new layout. */
82 readonly onScreen?: (screen: DeviceScreenSize) => void;
3283}
3384
3485export interface DeviceStreamEvents {
86 readonly onDuoControl?: (state: DuoControlState) => void;
87 /** A fixed panel cannot be decoded; the owner should return to the active flat feed. */
88 readonly onDuoUnavailable?: (detail?: string) => void;
3589 readonly onStatus: (status: DeviceStreamStatus, detail?: string) => void;
3690 readonly onScreen: (screen: DeviceScreenSize) => void;
3791 /** The proxy rejected the credential; the owner should refresh access and reconnect. */
......@@ -52,6 +106,9 @@ export interface DeviceStreamTarget {
52106 readonly access: DeviceHubAccess;
53107 /** Native iOS WebViews can use MJPEG without cross-origin fetch or secure-context support. */
54108 readonly preferMjpeg?: boolean;
109 /** Internal fixed-panel feeds share their parent's input session. */
110 readonly panelId?: 1 | 3;
111 readonly videoOnly?: boolean;
55112}
56113
57114export type DeviceHardwareButton = "home" | "back" | "recents" | "power" | "appSwitcher";
......@@ -219,6 +276,11 @@ export interface DeviceStreamClient {
219276 readonly pressButton: (button: DeviceHardwareButton) => void;
220277 readonly rotate: () => void;
221278 readonly setOrientation: (orientation: DeviceScreenSize["orientation"]) => void;
279 readonly controlDuo: (command: DuoCommand) => void;
280 /** Switch between one active feed and two fixed-panel feeds without replacing HID. */
281 readonly setDuoPanels: (panels: DuoPanelSinks | null) => void;
282 /** Model UVs already map to the hardware framebuffer. */
283 readonly sendRawTouch: (phase: "begin" | "move" | "end", x: number, y: number) => void;
222284}
223285
224286const HID_USAGE_BY_CODE: Readonly<Record<string, number>> = {
......@@ -314,6 +376,37 @@ export function createDeviceStreamClient(
314376 let videoReadTimer: ReturnType<typeof setTimeout> | null = null;
315377 let mjpegImage: HTMLImageElement | null = null;
316378 let releaseImage: (() => void) | null = null;
379 let videoGeneration = 0;
380 let panelClients: DeviceStreamClient[] = [];
381 let panelSinks: DuoPanelSinks | null = null;
382 let rotationCursor: DeviceScreenSize["orientation"] | null = null;
383 let pendingOrientation: { requestId: number } | null = null;
384 const duoControl = createDuoControl({
385 send(request) {
386 if (socket?.readyState !== WebSocket.OPEN || !screen?.supportsHingeAngle) return false;
387 if (request.command.control === "physical" && !screen.supportsPhysicalOrientation)
388 return false;
389 try {
390 pendingOrientation = null;
391 if (request.command.control === "orientation") {
392 const value = request.command.value;
393 pendingOrientation = { requestId: request.requestId };
394 rotationCursor = value;
395 // Upstream serializes orientation with hinge commands, then broadcasts config.
396 // An orientation-locked app can keep its framebuffer orientation after the sensor rotates.
397 socket.send(taggedJson(IOS_MSG_ORIENTATION, { orientation: value }));
398 } else socket.send(taggedJson(0x10, request));
399 return true;
400 } catch {
401 return false;
402 }
403 },
404 onChange(state) {
405 if (!state.pending) pendingOrientation = null;
406 events.onDuoControl?.(state);
407 },
408 });
409 const videoPath = `/helper/${device}${target.panelId ? `/panel/${target.panelId}` : ""}/stream.avcc`;
317410
318411 const mjpegUrl = () => httpUrl(`/helper/${device}/stream.mjpeg`);
319412
......@@ -437,16 +530,26 @@ export function createDeviceStreamClient(
437530 };
438531
439532 const makeDecoder = () => {
533 const feedGeneration = videoGeneration;
440534 const decoder = new VideoDecoder({
441535 output: (frame) => {
442536 try {
443 if (videoDecoder === decoder) paint(frame, frame.displayWidth, frame.displayHeight);
537 if (
538 videoDecoder === decoder &&
539 (platform !== "ios" || feedGeneration === videoGeneration)
540 )
541 paint(frame, frame.displayWidth, frame.displayHeight);
444542 } finally {
445543 frame.close();
446544 }
447545 },
448546 error: () => {
449 if (stopped || videoDecoder !== decoder) return;
547 if (
548 stopped ||
549 videoDecoder !== decoder ||
550 (platform === "ios" && feedGeneration !== videoGeneration)
551 )
552 return;
450553 recoverDecoder();
451554 },
452555 });
......@@ -456,7 +559,7 @@ export function createDeviceStreamClient(
456559 /** iOS can fall back to MJPEG when the stream's H.264 profile is unsupported. */
457560 const configureDecoder = async (
458561 config: VideoDecoderConfig,
459 isCurrent: () => boolean,
562 isCurrent = () => !stopped,
460563 ): Promise<boolean> => {
461564 const epoch = decoderEpoch;
462565 const full: VideoDecoderConfig = { ...config, optimizeForLatency: true };
......@@ -525,19 +628,32 @@ export function createDeviceStreamClient(
525628
526629 // iOS video: fetch the AVCC body and demux into the decoder.
527630 const readIosVideo = async () => {
631 const lifecycle = generation;
632 const feedGeneration = ++videoGeneration;
528633 const demuxer = new AvccDemuxer();
529 const session = generation;
530634 const videoController = new AbortController();
531635 controller = videoController;
532 const isCurrent = () => !stopped && generation === session && controller === videoController;
636 const isCurrent = () =>
637 !stopped &&
638 generation === lifecycle &&
639 videoGeneration === feedGeneration &&
640 controller === videoController;
533641 let retryDetail: string | undefined;
534642 try {
535 const response = await fetch(httpUrl(`/helper/${device}/stream.avcc`), {
643 const response = await fetch(httpUrl(videoPath), {
536644 signal: videoController.signal,
537645 credentials: access.credentials ? "include" : "same-origin",
538646 });
539 if (!isCurrent()) return;
647 if (!isCurrent()) {
648 await response.body?.cancel();
649 return;
650 }
540651 if (response.status === 401 || response.status === 403) return handleUnauthorized();
652 if (target.panelId && [400, 404, 405, 410].includes(response.status)) {
653 await response.body?.cancel();
654 setStatus("error", "This Device Hub does not provide fixed Duo display feeds.");
655 return;
656 }
541657 if (!response.ok || !response.body) throw new Error(`stream ${response.status}`);
542658 const reader = response.body.getReader();
543659 for (;;) {
......@@ -579,7 +695,12 @@ export function createDeviceStreamClient(
579695 if (!configured) {
580696 await reader.cancel().catch(() => {});
581697 if (!isCurrent()) return;
582 fallBackToMjpeg();
698 if (target.videoOnly)
699 setStatus(
700 "error",
701 `This browser cannot decode the Duo panel's ${avcCodecString(chunk.payload)} stream.`,
702 );
703 else fallBackToMjpeg();
583704 return;
584705 }
585706 break;
......@@ -630,6 +751,42 @@ export function createDeviceStreamClient(
630751 }
631752 };
632753
754 const startDuoVideo = (panels: DuoPanelSinks) => {
755 for (const panel of panelClients) panel.stop();
756 // Physical handoff elects a native surface. Fixed-panel encoders can keep
757 // an inactive shutdown frame after election, so this build uses one active
758 // feed instead of decoding a third stream alongside the two fixed feeds.
759 const ids = screen?.supportsPhysicalOrientation ? ([null] as const) : ([1, 3] as const);
760 panelClients = ids.map((id) => {
761 const output = id === 1 ? panels.cover : panels.inner;
762 return createDeviceStreamClient(
763 { ...target, ...(id === null ? {} : { panelId: id }), videoOnly: true },
764 {
765 present(source, width, height) {
766 if (id === null) {
767 return sink.present(source, width, height);
768 }
769 // An inactive native LCD can emit its shutdown black frame. Retain its last useful image.
770 if (screen?.screenId !== id) return true;
771 const retained = output.present(source, width, height);
772 const primary = sink.present(source, width, height);
773 return retained && primary;
774 },
775 },
776 {
777 onStatus: (status, detail) => {
778 if (status === "error") events.onDuoUnavailable?.(detail);
779 },
780 onScreen: () => {},
781 onInputConnected: () => {},
782 onMjpegFallback: () => {},
783 onUnauthorized: handleUnauthorized,
784 },
785 );
786 });
787 for (const panel of panelClients) panel.start();
788 };
789
633790 // iOS input socket; also carries the screen config the helper pushes.
634791 const connectIosInput = async () => {
635792 if (stopped) return;
......@@ -648,12 +805,42 @@ export function createDeviceStreamClient(
648805 if (stopped || socket !== ws) return;
649806 if (!(event.data instanceof ArrayBuffer)) return;
650807 const bytes = new Uint8Array(event.data);
651 if (bytes.length < 1 || bytes[0] !== IOS_TAG_SCREEN_CONFIG) return;
808 if (socket !== ws || stopped || bytes.length < 1) return;
652809 try {
653 const config = JSON.parse(decoder.decode(bytes.subarray(1))) as DeviceScreenSize;
654 if (config.width > 0 && config.height > 0) {
655 screen = config;
656 events.onScreen(config);
810 const payload: unknown = JSON.parse(decoder.decode(bytes.subarray(1)));
811 if (bytes[0] === 0x90) {
812 const reply = decodeControlReply(payload);
813 if (Option.isSome(reply)) duoControl.receive(reply.value);
814 } else if (bytes[0] === IOS_TAG_SCREEN_CONFIG) {
815 const config = decodeScreenConfig(payload);
816 if (Option.isSome(config)) {
817 const previous = screen;
818 screen = config.value;
819 if (screen.hingePose && screen.hingePose !== previous?.hingePose)
820 rotationCursor = screen.hingePose === "laptop" ? "landscape_left" : "portrait";
821 else if (screen.orientation !== previous?.orientation)
822 rotationCursor = screen.orientation;
823 panelSinks?.onScreen?.(screen);
824 events.onScreen(screen);
825 // A surface election can leave an existing decoder on the former
826 // encoder description. Reopen only video to acquire the elected
827 // surface's seed and codec configuration; HID and the viewer stay.
828 if (
829 panelSinks &&
830 screen.supportsPhysicalOrientation &&
831 previous &&
832 screen.screenId !== previous.screenId
833 )
834 startDuoVideo(panelSinks);
835 if (pendingOrientation) {
836 const receipt = pendingOrientation;
837 pendingOrientation = null;
838 duoControl.receive({
839 requestId: receipt.requestId,
840 ok: true,
841 });
842 }
843 }
657844 }
658845 } catch {
659846 // Ignore malformed config frames.
......@@ -662,6 +849,8 @@ export function createDeviceStreamClient(
662849 ws.onclose = (event) => {
663850 if (socket !== ws) return;
664851 socket = null;
852 duoControl.clear();
853 rotationCursor = null;
665854 if (stopped) return;
666855 events.onInputConnected(
667856 false,
......@@ -757,7 +946,7 @@ export function createDeviceStreamClient(
757946 configuring = false;
758947 connecting();
759948 if (platform === "ios") {
760 void connectIosInput();
949 if (!target.videoOnly) void connectIosInput();
761950 if (useWebCodecs) void readIosVideo();
762951 else fallBackToMjpeg();
763952 } else if (useWebCodecs) {
......@@ -771,6 +960,12 @@ export function createDeviceStreamClient(
771960 if (stopped) return;
772961 stopped = true;
773962 generation++;
963 videoGeneration++;
964 duoControl.clear();
965 rotationCursor = null;
966 for (const panel of panelClients) panel.stop();
967 panelClients = [];
968 panelSinks = null;
774969 mjpeg = false;
775970 clearFrameTimer();
776971 if (videoReadTimer !== null) clearTimeout(videoReadTimer);
......@@ -814,6 +1009,29 @@ export function createDeviceStreamClient(
8141009 start,
8151010 stop,
8161011 setMjpegImage,
1012 controlDuo: duoControl.enqueue,
1013 sendRawTouch: (phase, x, y) => {
1014 if (platform === "ios") send(taggedJson(IOS_MSG_TOUCH, { type: phase, x, y }));
1015 },
1016 setDuoPanels(panels) {
1017 if (platform !== "ios" || target.videoOnly || stopped || panelSinks === panels) return;
1018 if (panels && !screen?.supportsHingeAngle) return;
1019 panelSinks = panels;
1020 videoGeneration++;
1021 controller?.abort();
1022 controller = null;
1023 closeDecoder();
1024 const retry = retryTimers.get("video");
1025 if (retry) clearTimeout(retry);
1026 retryTimers.delete("video");
1027 for (const panel of panelClients) panel.stop();
1028 panelClients = [];
1029 if (!panels) {
1030 if (useWebCodecs) void readIosVideo();
1031 return;
1032 }
1033 startDuoVideo(panels);
1034 },
8171035 sendTouch: (phase, x, y) => {
8181036 if (platform === "ios") {
8191037 send(taggedJson(IOS_MSG_TOUCH, { type: phase, ...rawPoint(x, y) }));
......@@ -856,10 +1074,15 @@ export function createDeviceStreamClient(
8561074 },
8571075 rotate: () => {
8581076 if (platform !== "ios") return;
859 const current = screen?.orientation ?? "portrait";
1077 const current = screen?.supportsHingeAngle
1078 ? (rotationCursor ?? screen.orientation)
1079 : (screen?.orientation ?? "portrait");
8601080 const next =
8611081 IOS_ORIENTATIONS[(IOS_ORIENTATIONS.indexOf(current) + 1) % IOS_ORIENTATIONS.length]!;
862 send(taggedJson(IOS_MSG_ORIENTATION, { orientation: next }));
1082 if (screen?.supportsHingeAngle) {
1083 rotationCursor = next;
1084 duoControl.enqueue({ control: "orientation", value: next });
1085 } else send(taggedJson(IOS_MSG_ORIENTATION, { orientation: next }));
8631086 },
8641087 setOrientation: (orientation) => {
8651088 if (platform === "ios") send(taggedJson(IOS_MSG_ORIENTATION, { orientation }));