Commit 78af372cf46a80147f05183b69d7a8339259ca37 | the-knife

apps/desktop/src/electron/ElectronProtocol.test.ts+9-2

 @@ -105,7 +105,7 @@ describe("ElectronProtocol", () => {
           );
           assert.include(
             response.headers.get("content-security-policy") ?? "",
-            "connect-src 'self' http: https: ws: wss:",
+            "connect-src 'self' blob: http: https: ws: wss:",
           );
           assert.include(
             response.headers.get("content-security-policy") ?? "",
 @@ -255,7 +255,14 @@ describe("ElectronProtocol", () => {
       "https://clerk.t3.codes",
       "https://challenges.cloudflare.com",
     ]);
-    assert.deepEqual(directives["connect-src"], ["'self'", "http:", "https:", "ws:", "wss:"]);
+    assert.deepEqual(directives["connect-src"], [
+      "'self'",
+      "blob:",
+      "http:",
+      "https:",
+      "ws:",
+      "wss:",
+    ]);
     assert.deepEqual(directives["img-src"], [
       "'self'",
       "t3code:",

apps/desktop/src/electron/ElectronProtocol.ts+2-1

...	@@ -84,7 +84,8 @@ export function makeDesktopContentSecurityPolicy(input: DesktopProtocolRegistrat	...	@@ -84,7 +84,8 @@ export function makeDesktopContentSecurityPolicy(input: DesktopProtocolRegistrat
84	// the build-configured Clerk, relay, and OTLP endpoints. Those environment	84	// the build-configured Clerk, relay, and OTLP endpoints. Those environment
85	// origins are not known when this response policy is created, so restrict	85	// origins are not known when this response policy is created, so restrict
86	// connections by the network schemes the client supports instead of by host.	86	// connections by the network schemes the client supports instead of by host.
87	const connectSources = ["'self'", "http:", "https:", "ws:", "wss:"];	87	// GLTFLoader fetches embedded textures through blob URLs after parsing the model.
		88	const connectSources = ["'self'", "blob:", "http:", "https:", "ws:", "wss:"];
88		89
89	return [	90	return [
90	"default-src 'self'",	91	"default-src 'self'",

apps/desktop/src/ipc/channels.ts+1

...	@@ -11,6 +11,7 @@ export const MENU_ACTION_CHANNEL = "desktop:menu-action";	...	@@ -11,6 +11,7 @@ export const MENU_ACTION_CHANNEL = "desktop:menu-action";
11	export const PASTE_AS_TEXT_CHANNEL = "desktop:paste-as-text";	11	export const PASTE_AS_TEXT_CHANNEL = "desktop:paste-as-text";
12	export const SNAP_SHOT_EVENT_CHANNEL = "desktop:snap-shot-event";	12	export const SNAP_SHOT_EVENT_CHANNEL = "desktop:snap-shot-event";
13	export const QUIT_SHORTCUT_CHANNEL = "desktop:quit-shortcut";	13	export const QUIT_SHORTCUT_CHANNEL = "desktop:quit-shortcut";
		14	export const TRACKPAD_SCROLL_END_CHANNEL = "desktop:trackpad-scroll-end";
14	export const GET_WINDOW_FULLSCREEN_STATE_CHANNEL = "desktop:get-window-fullscreen-state";	15	export const GET_WINDOW_FULLSCREEN_STATE_CHANNEL = "desktop:get-window-fullscreen-state";
15	export const WINDOW_FULLSCREEN_STATE_CHANNEL = "desktop:window-fullscreen-state";	16	export const WINDOW_FULLSCREEN_STATE_CHANNEL = "desktop:window-fullscreen-state";
16	export const DESKTOP_APP_ACTIVATION_READY_CHANNEL = "desktop:app-activation-ready";	17	export const DESKTOP_APP_ACTIVATION_READY_CHANNEL = "desktop:app-activation-ready";

apps/desktop/src/preload.ts+5

 @@ -79,6 +79,11 @@ contextBridge.exposeInMainWorld("desktopBridge", {
     ipcRenderer.on(IpcChannels.SET_NOTIFICATION_BADGE_CHANNEL, handler);
     return () => ipcRenderer.removeListener(IpcChannels.SET_NOTIFICATION_BADGE_CHANNEL, handler);
   },
+  onTrackpadScrollEnd: (listener) => {
+    const handler = () => listener();
+    ipcRenderer.on(IpcChannels.TRACKPAD_SCROLL_END_CHANNEL, handler);
+    return () => ipcRenderer.removeListener(IpcChannels.TRACKPAD_SCROLL_END_CHANNEL, handler);
+  },
   getSystemLocale: () => {
     const result = ipcRenderer.sendSync(IpcChannels.GET_SYSTEM_LOCALE_CHANNEL);
     return typeof result === "string" ? result : null;

apps/desktop/src/window/DesktopWindow.test.ts+25

...	@@ -49,6 +49,7 @@ import * as ElectronWindow from "../electron/ElectronWindow.ts";	...	@@ -49,6 +49,7 @@ import * as ElectronWindow from "../electron/ElectronWindow.ts";
49	import {	49	import {
50	MENU_ACTION_CHANNEL,	50	MENU_ACTION_CHANNEL,
51	SNAP_SHOT_EVENT_CHANNEL,	51	SNAP_SHOT_EVENT_CHANNEL,
		52	TRACKPAD_SCROLL_END_CHANNEL,
52	WINDOW_FULLSCREEN_STATE_CHANNEL,	53	WINDOW_FULLSCREEN_STATE_CHANNEL,
53	} from "../ipc/channels.ts";	54	} from "../ipc/channels.ts";
54	import * as DesktopServerExposure from "../backend/DesktopServerExposure.ts";	55	import * as DesktopServerExposure from "../backend/DesktopServerExposure.ts";
...	@@ -709,6 +710,30 @@ describe("DesktopWindow", () => {	...	@@ -709,6 +710,30 @@ describe("DesktopWindow", () => {
709	}),	710	}),
710	);	711	);
711		712
		713	it.effect("forwards native trackpad release to the renderer", () =>
		714	Effect.gen(function* () {
		715	const fakeWindow = makeFakeBrowserWindow();
		716	const send = vi.spyOn(fakeWindow.window.webContents, "send");
		717	const createCount = yield* Ref.make(0);
		718	const mainWindow = yield* Ref.make<Option.Option<Electron.BrowserWindow>>(Option.none());
		719	const layer = makeTestLayer({ window: fakeWindow.window, createCount, mainWindow });
		720
		721	yield* Effect.gen(function* () {
		722	const desktopWindow = yield* DesktopWindow.DesktopWindow;
		723	yield* desktopWindow.handleBackendReady(new URL("http://127.0.0.1:3773"));
		724	const onInput = fakeWindow.webContentsListeners.get("input-event");
		725	if (!onInput) return yield* Effect.die("input-event listener was not registered");
		726	onInput({}, { type: "gestureScrollUpdate" });
		727	assert.notInclude(
		728	send.mock.calls.map(([channel]) => channel),
		729	TRACKPAD_SCROLL_END_CHANNEL,
		730	);
		731	onInput({}, { type: "gestureScrollEnd" });
		732	assert.isTrue(send.mock.calls.some(([channel]) => channel === TRACKPAD_SCROLL_END_CHANNEL));
		733	}).pipe(Effect.provide(layer));
		734	}),
		735	);
		736
712	// Chromium hands the main window's zoom level down to embedded preview	737	// Chromium hands the main window's zoom level down to embedded preview
713	// guests, so every app zoom has to put the preview browser back at its own	738	// guests, so every app zoom has to put the preview browser back at its own
714	// zoom or zooming the UI drags the previewed page with it.	739	// zoom or zooming the UI drags the previewed page with it.

apps/desktop/src/window/DesktopWindow.ts+4

...	@@ -22,6 +22,7 @@ import {	...	@@ -22,6 +22,7 @@ import {
22	MENU_ACTION_CHANNEL,	22	MENU_ACTION_CHANNEL,
23	QUIT_SHORTCUT_CHANNEL,	23	QUIT_SHORTCUT_CHANNEL,
24	SNAP_SHOT_EVENT_CHANNEL,	24	SNAP_SHOT_EVENT_CHANNEL,
		25	TRACKPAD_SCROLL_END_CHANNEL,
25	WINDOW_FULLSCREEN_STATE_CHANNEL,	26	WINDOW_FULLSCREEN_STATE_CHANNEL,
26	} from "../ipc/channels.ts";	27	} from "../ipc/channels.ts";
27	import * as PreviewManager from "../preview/Manager.ts";	28	import * as PreviewManager from "../preview/Manager.ts";
...	@@ -661,6 +662,9 @@ export const make = Effect.gen(function* () {	...	@@ -661,6 +662,9 @@ export const make = Effect.gen(function* () {
661	event.preventDefault();	662	event.preventDefault();
662	}	663	}
663	});	664	});
		665	window.webContents.on("input-event", (_event, input) => {
		666	if (input.type === "gestureScrollEnd") window.webContents.send(TRACKPAD_SCROLL_END_CHANNEL);
		667	});
664		668
665	window.on("page-title-updated", (event) => {	669	window.on("page-title-updated", (event) => {
666	event.preventDefault();	670	event.preventDefault();

apps/web/src/components/RightPanelTabs.tsx+36-1

...	@@ -63,6 +63,7 @@ import { ScrollArea } from "~/components/ui/scroll-area";	...	@@ -63,6 +63,7 @@ import { ScrollArea } from "~/components/ui/scroll-area";
63	import { PanelTabCloseButton } from "~/components/ui/panel-tab-close-button";	63	import { PanelTabCloseButton } from "~/components/ui/panel-tab-close-button";
64	import { faviconUrlForOrigin } from "~/lib/favicon";	64	import { faviconUrlForOrigin } from "~/lib/favicon";
65	import { useTheme } from "~/hooks/useTheme";	65	import { useTheme } from "~/hooks/useTheme";
		66	import { useDeviceState } from "~/state/device";
66	import {	67	import {
67	newestPullRequestSummary,	68	newestPullRequestSummary,
68	pullRequestEnvironment,	69	pullRequestEnvironment,
...	@@ -1244,7 +1245,17 @@ export function RightPanelTabs(props: RightPanelTabsProps) {	...	@@ -1244,7 +1245,17 @@ export function RightPanelTabs(props: RightPanelTabsProps) {
1244	</button>	1245	</button>
1245	}	1246	}
1246	/>	1247	/>
1247	<TooltipPopup>{title}</TooltipPopup>	1248	<TooltipPopup>
		1249	{surface.kind === "device" ? (
		1250	<DeviceTabTooltip
		1251	surface={surface}
		1252	environmentId={props.environmentId}
		1253	title={title}
		1254	/>
		1255	) : (
		1256	title
		1257	)}
		1258	</TooltipPopup>
1248	</Tooltip>	1259	</Tooltip>
1249	)}	1260	)}
1250	</div>	1261	</div>
...	@@ -1421,3 +1432,27 @@ export function RightPanelTabs(props: RightPanelTabsProps) {	...	@@ -1421,3 +1432,27 @@ export function RightPanelTabs(props: RightPanelTabsProps) {
1421	</PreviewPanelShell>	1432	</PreviewPanelShell>
1422	);	1433	);
1423	}	1434	}
		1435
		1436	function DeviceTabTooltip(props: {
		1437	surface: Extract<RightPanelSurface, { kind: "device" }>;
		1438	environmentId: EnvironmentId \| null;
		1439	title: string;
		1440	}) {
		1441	const target = props.surface.target;
		1442	const { state } = useDeviceState(target ? props.environmentId : null);
		1443	const device = target
		1444	? state.devices.find((entry) => entry.hostId === target.hostId && entry.id === target.deviceId)
		1445	: undefined;
		1446	const host = target ? state.hosts.find((entry) => entry.id === target.hostId) : undefined;
		1447	return (
		1448	<div className="flex flex-col gap-0.5">
		1449	<span>{props.title}</span>
		1450	{target ? (
		1451	<span className="text-muted-foreground">
		1452	{host?.label ?? "Device host"} ·{" "}
		1453	{device?.version ?? (target.platform === "ios" ? "iOS" : "Android")}
		1454	</span>
		1455	) : null}
		1456	</div>
		1457	);
		1458	}

apps/web/src/components/device/DeviceControlsRail.tsx created+308

...	@@ -0,0 +1,308 @@
1	import type { DevicePlatform } from "@t3tools/contracts";
2	import {
3	Camera,
4	ChevronLeft,
5	Home,
6	Keyboard,
7	Box,
8	Maximize,
9	Moon,
10	MoreHorizontal,
11	PictureInPicture2,
12	Power,
13	RotateCcw,
14	SlidersHorizontal,
15	Smartphone,
16	Square,
17	Sun,
18	Type,
19	X,
20	} from "lucide-react";
21	import type { ReactNode } from "react";
22	import { Button } from "~/components/ui/button";
23	import {
24	Menu,
25	MenuItem,
26	MenuPopup,
27	MenuSeparator,
28	MenuTrigger,
29	MenuRadioGroup,
30	MenuRadioItem,
31	MenuRadioItemIndicator,
32	} from "~/components/ui/menu";
33	import { Tooltip, TooltipPopup, TooltipTrigger } from "~/components/ui/tooltip";
34	import type { DeviceStreamHandle, DeviceViewControls } from "./DeviceStreamView";
35	import type { DeviceControls } from "./useDeviceControls";
36
37	/** Stable floating controls for both presentations, regardless of panel width. */
38	export function DeviceControlsRail(props: {
39	platform: DevicePlatform;
40	handle: DeviceStreamHandle \| null;
41	view: DeviceViewControls;
42	controls: DeviceControls;
43	screenshotPending: boolean;
44	onScreenshot: () => void;
45	toolsOpen: boolean;
46	onTools: () => void;
47	onFloat: () => void;
48	onClose: () => void;
49	onPowerOff: () => void;
50	}) {
51	const { view, handle, controls } = props;
52	const popupSide = "left";
53	const settings = controls.detail?.settings;
54	const inputDisabled = !handle?.inputConnected;
55	const nextAppearance = settings?.appearance === "dark" ? "light" : "dark";
56	return (
57	<aside
58	aria-label="Device controls"
59	data-layout="rail"
60	className="pointer-events-none absolute inset-y-0 right-0 z-10 flex w-14 flex-col items-center justify-center py-3 pr-2"
61	>
62	<div className="pointer-events-auto flex max-h-full shrink-0 flex-col items-center gap-1 overflow-y-auto rounded-full border border-border/50 bg-background/80 p-2 shadow-sm [scrollbar-width:none]">
63	<RailButton
64	tooltipSide={popupSide}
65	label="Home"
66	disabled={inputDisabled}
67	onClick={() => handle?.pressButton("home")}
68	>
69	<Home />
70	</RailButton>
71	{props.platform === "android" ? (
72	<>
73	<RailButton
74	tooltipSide={popupSide}
75	label="Back"
76	disabled={inputDisabled}
77	onClick={() => handle?.pressButton("back")}
78	>
79	<ChevronLeft />
80	</RailButton>
81	<RailButton
82	tooltipSide={popupSide}
83	label="Recents"
84	disabled={inputDisabled}
85	onClick={() => handle?.pressButton("recents")}
86	>
87	<Square />
88	</RailButton>
89	<Menu>
90	<MenuTrigger
91	render={
92	<Button
93	size="icon-sm"
94	variant="ghost"
95	aria-label="Rotate device"
96	disabled={controls.disabled}
97	/>
98	}
99	>
100	<RotateCcw />
101	</MenuTrigger>
102	<MenuPopup side={popupSide}>
103	<MenuItem
104	onClick={() => void controls.act({ type: "setOrientation", value: "portrait" })}
105	>
106	Portrait
107	</MenuItem>
108	<MenuItem
109	onClick={() =>
110	void controls.act({ type: "setOrientation", value: "landscape_left" })
111	}
112	>
113	Landscape
114	</MenuItem>
115	</MenuPopup>
116	</Menu>
117	</>
118	) : (
119	<RailButton
120	tooltipSide={popupSide}
121	label="Rotate device"
122	disabled={inputDisabled \|\| !!view.keyboard?.attached}
123	onClick={() => handle?.rotate()}
124	>
125	<RotateCcw />
126	</RailButton>
127	)}
128	<RailDivider />
129	<RailButton
130	tooltipSide={popupSide}
131	label={`Switch device to ${nextAppearance} mode`}
132	disabled={controls.disabled \|\| !settings?.appearance}
133	onClick={() => void controls.act({ type: "setAppearance", value: nextAppearance })}
134	>
135	{settings?.appearance === "dark" ? <Sun /> : <Moon />}
136	</RailButton>
137	<Menu>
138	<MenuTrigger
139	render={
140	<Button
141	size="icon-sm"
142	variant="ghost"
143	aria-label="Device text size"
144	title="Device text size"
145	disabled={controls.disabled \|\| !settings?.textSize}
146	/>
147	}
148	>
149	<Type />
150	</MenuTrigger>
151	<MenuPopup side={popupSide} className="min-w-40">
152	<MenuRadioGroup
153	value={settings?.textSize ?? ""}
154	onValueChange={(value) => {
155	if (
156	value === "small" \|\|
157	value === "default" \|\|
158	value === "large" \|\|
159	value === "extra-large"
160	)
161	void controls.act({ type: "setTextSize", value });
162	}}
163	>
164	{(
165	[
166	["small", "Small"],
167	["default", "Default"],
168	["large", "Large"],
169	["extra-large", "Extra large"],
170	] as const
171	).map(([value, label]) => (
172	<MenuRadioItem key={value} value={value}>
173	<span className="flex items-center gap-2">
174	<span className="flex-1">{label}</span>
175	<MenuRadioItemIndicator />
176	</span>
177	</MenuRadioItem>
178	))}
179	</MenuRadioGroup>
180	</MenuPopup>
181	</Menu>
182	<RailButton
183	tooltipSide={popupSide}
184	label="Device tools"
185	pressed={props.toolsOpen}
186	onClick={props.onTools}
187	>
188	<SlidersHorizontal />
189	</RailButton>
190	<RailButton
191	tooltipSide={popupSide}
192	label={props.screenshotPending ? "Capturing screenshot" : "Save screenshot"}
193	disabled={!view.streaming \|\| props.screenshotPending}
194	onClick={props.onScreenshot}
195	>
196	<Camera />
197	</RailButton>
198	<Menu>
199	<MenuTrigger
200	render={
201	<Button
202	size="icon-sm"
203	variant="ghost"
204	aria-label="More device actions"
205	title="More device actions"
206	/>
207	}
208	>
209	<MoreHorizontal />
210	</MenuTrigger>
211	<MenuPopup side={popupSide} align="end">
212	<MenuItem onClick={props.onFloat}>
213	<PictureInPicture2 />
214	Float device over chat
215	</MenuItem>
216	<MenuItem onClick={props.onClose}>
217	<X />
218	Close device panel
219	</MenuItem>
220	<MenuSeparator />
221	<MenuItem variant="destructive" onClick={props.onPowerOff}>
222	<Power />
223	Power off device
224	</MenuItem>
225	</MenuPopup>
226	</Menu>
227	<RailDivider />
228	<RailButton
229	tooltipSide={popupSide}
230	label="3D view"
231	pressed={view.phone}
232	disabled={!view.streaming \|\| !!view.phoneUnavailableReason}
233	description={view.phoneUnavailableReason ?? undefined}
234	onClick={view.showPhone}
235	>
236	<Box />
237	</RailButton>
238	<RailButton
239	tooltipSide={popupSide}
240	label="Flat view"
241	pressed={!view.phone}
242	disabled={!view.streaming}
243	onClick={view.showFlat}
244	>
245	<Smartphone />
246	</RailButton>
247	{view.keyboard ? (
248	<RailButton
249	tooltipSide={popupSide}
250	label={view.keyboard.attached ? "Detach Magic Keyboard" : "Attach Magic Keyboard"}
251	pressed={view.keyboard.attached}
252	onClick={view.keyboard.toggle}
253	>
254	<Keyboard />
255	</RailButton>
256	) : null}
257	{view.phone ? (
258	<RailButton tooltipSide={popupSide} label="Reset phone view" onClick={view.resetView}>
259	<Maximize />
260	</RailButton>
261	) : null}
262	</div>
263	</aside>
264	);
265	}
266
267	function RailDivider() {
268	return <div aria-hidden className="my-1 h-px w-5 shrink-0 bg-border/70" />;
269	}
270
271	function RailButton(props: {
272	tooltipSide: "left" \| "bottom";
273	label: string;
274	description?: string \| undefined;
275	disabled?: boolean;
276	pressed?: boolean;
277	onClick: () => void;
278	children: ReactNode;
279	}) {
280	const button = (
281	<Button
282	size="icon-sm"
283	variant={props.pressed ? "secondary" : "ghost"}
284	aria-label={props.label}
285	aria-pressed={props.pressed}
286	disabled={props.disabled}
287	onClick={props.onClick}
288	>
289	{props.children}
290	</Button>
291	);
292	return (
293	<Tooltip>
294	{props.disabled && props.description ? (
295	<TooltipTrigger
296	render={
297	<span className="inline-flex" tabIndex={0} role="group" aria-label={props.label} />
298	}
299	>
300	{button}
301	</TooltipTrigger>
302	) : (
303	<TooltipTrigger render={button} />
304	)}
305	<TooltipPopup side={props.tooltipSide}>{props.description ?? props.label}</TooltipPopup>
306	</Tooltip>
307	);
308	}

apps/web/src/components/device/DevicePanel.tsx+17-137

...	@@ -5,17 +5,7 @@ import type {	...	@@ -5,17 +5,7 @@ import type {
5	DeviceSummary,	5	DeviceSummary,
6	ScopedThreadRef,	6	ScopedThreadRef,
7	} from "@t3tools/contracts";	7	} from "@t3tools/contracts";
8	import {	8	import { Smartphone, X } from "lucide-react";
9	ChevronLeft,
10	Home,
11	PictureInPicture2,
12	Power,
13	RotateCcw,
14	SlidersHorizontal,
15	Smartphone,
16	Square,
17	X,
18	} from "lucide-react";
19	import { useEffect, useMemo, useState } from "react";	9	import { useEffect, useMemo, useState } from "react";
20		10
21	import { usePreviewMiniPlayerStore } from "~/previewMiniPlayerStore";	11	import { usePreviewMiniPlayerStore } from "~/previewMiniPlayerStore";
...	@@ -25,16 +15,13 @@ import { DiscoveryList, DiscoveryListRow } from "~/components/ui/discovery-list"	...	@@ -25,16 +15,13 @@ import { DiscoveryList, DiscoveryListRow } from "~/components/ui/discovery-list"
25	import { Dialog } from "~/components/ui/dialog";	15	import { Dialog } from "~/components/ui/dialog";
26	import { WizardPopup } from "~/components/ui/wizard";	16	import { WizardPopup } from "~/components/ui/wizard";
27	import { Spinner } from "~/components/ui/spinner";	17	import { Spinner } from "~/components/ui/spinner";
28	import { Toggle } from "~/components/ui/toggle";
29	import { Tooltip, TooltipPopup, TooltipTrigger } from "~/components/ui/tooltip";
30	import { cn } from "~/lib/utils";	18	import { cn } from "~/lib/utils";
31	import { deviceEnvironment, useDeviceHubAccess, useDeviceState } from "~/state/device";	19	import { deviceEnvironment, useDeviceState } from "~/state/device";
32	import { formatEnvironmentQueryError } from "~/state/query";	20	import { formatEnvironmentQueryError } from "~/state/query";
33	import { useAtomCommand } from "~/state/use-atom-command";	21	import { useAtomCommand } from "~/state/use-atom-command";
34	import { DeviceStreamView, type DeviceStreamHandle } from "./DeviceStreamView";
35	import { DeviceLoadingView } from "./DeviceLoadingView";	22	import { DeviceLoadingView } from "./DeviceLoadingView";
36	import { DeviceSetup } from "./DeviceSetup";	23	import { DeviceSetup } from "./DeviceSetup";
37	import { DeviceToolsPanel } from "./DeviceToolsPanel";	24	import { DeviceWorkspace } from "./DeviceWorkspace";
38	import { PreviewPanelShell, type PreviewPanelMode } from "../preview/PreviewPanelShell";	25	import { PreviewPanelShell, type PreviewPanelMode } from "../preview/PreviewPanelShell";
39		26
40	const platformLabel = (platform: DevicePlatform) =>	27	const platformLabel = (platform: DevicePlatform) =>
...	@@ -59,10 +46,6 @@ export function DevicePanel(props: {	...	@@ -59,10 +46,6 @@ export function DevicePanel(props: {
59	const [operationError, setOperationError] = useState<string \| null>(null);	46	const [operationError, setOperationError] = useState<string \| null>(null);
60	const [pendingDevice, setPendingDevice] = useState<DeviceSummary \| null>(null);	47	const [pendingDevice, setPendingDevice] = useState<DeviceSummary \| null>(null);
61	const pendingDeviceKey = pendingDevice ? deviceKey(pendingDevice) : null;	48	const pendingDeviceKey = pendingDevice ? deviceKey(pendingDevice) : null;
62	const [handle, setHandle] = useState<DeviceStreamHandle \| null>(null);
63	const [toolsOpen, setToolsOpen] = useState(false);
64	const [axOverlay, setAxOverlay] = useState(false);
65	const access = useDeviceHubAccess(environmentId);
66		49
67	const hostDisabled = state.hostStatus === "disabled";	50	const hostDisabled = state.hostStatus === "disabled";
68		51
...	@@ -184,69 +167,7 @@ export function DevicePanel(props: {	...	@@ -184,69 +167,7 @@ export function DevicePanel(props: {
184		167
185	return (	168	return (
186	<PreviewPanelShell mode={props.mode}>	169	<PreviewPanelShell mode={props.mode}>
187	<div className="flex h-9 shrink-0 items-center gap-1.5 border-b px-2">	170	{hostReady && !activeDevice && state.hostStatusDetail ? (
188	<span className="min-w-0 flex-1 truncate text-xs text-muted-foreground">
189	{props.surface.target
190	? `${state.hosts.find((host) => host.id === props.surface.target?.hostId)?.label ?? "Device host"} · ${activeDevice?.version ?? props.surface.target.platform}`
191	: (pendingDevice?.name ?? "Choose a device")}
192	</span>
193	{activeDevice ? (
194	<>
195	<DeviceButton
196	label="Home"
197	onClick={() => handle?.pressButton("home")}
198	disabled={!handle?.inputConnected}
199	>
200	<Home />
201	</DeviceButton>
202	{activeDevice.platform === "android" ? (
203	<>
204	<DeviceButton
205	label="Back"
206	onClick={() => handle?.pressButton("back")}
207	disabled={!handle?.inputConnected}
208	>
209	<ChevronLeft />
210	</DeviceButton>
211	<DeviceButton
212	label="Recents"
213	onClick={() => handle?.pressButton("recents")}
214	disabled={!handle?.inputConnected}
215	>
216	<Square />
217	</DeviceButton>
218	</>
219	) : (
220	<DeviceButton
221	label="Rotate"
222	onClick={() => handle?.rotate()}
223	disabled={!handle?.inputConnected}
224	>
225	<RotateCcw />
226	</DeviceButton>
227	)}
228	<Toggle
229	aria-label="Tools"
230	variant="ghost"
231	size="xs"
232	pressed={toolsOpen}
233	onPressedChange={(pressed) => setToolsOpen(Boolean(pressed))}
234	>
235	<SlidersHorizontal />
236	</Toggle>
237	<DeviceButton label="Float device over chat" onClick={floatActive}>
238	<PictureInPicture2 />
239	</DeviceButton>
240	<DeviceButton label="Power off" onClick={() => closeActive(true)}>
241	<Power />
242	</DeviceButton>
243	<DeviceButton label="Close" onClick={() => closeActive(false)}>
244	<X />
245	</DeviceButton>
246	</>
247	) : null}
248	</div>
249	{hostReady && state.hostStatusDetail ? (
250	<div	171	<div
251	role="status"	172	role="status"
252	className="whitespace-pre-line border-b px-3 py-2 text-xs text-muted-foreground"	173	className="whitespace-pre-line border-b px-3 py-2 text-xs text-muted-foreground"
...	@@ -278,34 +199,19 @@ export function DevicePanel(props: {	...	@@ -278,34 +199,19 @@ export function DevicePanel(props: {
278	) : null}	199	) : null}
279	<div className="@container relative flex min-h-0 flex-1">	200	<div className="@container relative flex min-h-0 flex-1">
280	{activeDevice && activeSession ? (	201	{activeDevice && activeSession ? (
281	<>	202	<DeviceWorkspace
282	<div className="relative min-h-0 min-w-0 flex-1">	203	key={`${environmentId}\u0000${deviceKey(activeDevice)}`}
283	<DeviceStreamView	204	environmentId={environmentId}
284	key={deviceKey(activeDevice)}	205	device={activeDevice}
285	environmentId={environmentId}	206	hostLabel={
286	platform={activeDevice.platform}	207	state.hosts.find((host) => host.id === activeDevice.hostId)?.label ?? "Device host"
287	deviceName={activeDevice.name}	208	}
288	deviceDescription={`${state.hosts.find((host) => host.id === activeDevice.hostId)?.label ?? "Device host"} · ${activeDevice.version}`}	209	hostDiagnostics={state.hostStatusDetail}
289	deviceId={activeDevice.id}	210	visible={props.visible}
290	hostId={activeDevice.hostId}	211	onFloat={floatActive}
291	visible={props.visible}	212	onClose={() => closeActive(false)}
292	axOverlay={axOverlay}	213	onPowerOff={() => closeActive(true)}
293	onHandle={setHandle}	214	/>
294	/>
295	</div>
296	{toolsOpen ? (
297	<DeviceToolsPanel
298	key={deviceKey(activeDevice)}
299	environmentId={environmentId}
300	device={activeDevice}
301	access={access}
302	axOverlay={axOverlay}
303	onAxOverlayChange={setAxOverlay}
304	onClose={() => setToolsOpen(false)}
305	className="absolute inset-y-0 right-0 z-10 w-full max-w-72 border-l shadow-lg @[560px]:static @[560px]:w-72 @[560px]:shrink-0 @[560px]:shadow-none"
306	/>
307	) : null}
308	</>
309	) : pendingDevice \|\| hostBusy \|\| !loaded ? (	215	) : pendingDevice \|\| hostBusy \|\| !loaded ? (
310	<DeviceLoadingView	216	<DeviceLoadingView
311	name={pendingDevice?.name ?? "Devices"}	217	name={pendingDevice?.name ?? "Devices"}
...	@@ -407,32 +313,6 @@ export function DevicePanel(props: {	...	@@ -407,32 +313,6 @@ export function DevicePanel(props: {
407	);	313	);
408	}	314	}
409		315
410	function DeviceButton(props: {
411	readonly label: string;
412	readonly onClick: () => void;
413	readonly disabled?: boolean;
414	readonly children: React.ReactNode;
415	}) {
416	return (
417	<Tooltip>
418	<TooltipTrigger
419	render={
420	<Button
421	size="icon-xs"
422	variant="ghost-muted"
423	aria-label={props.label}
424	onClick={props.onClick}
425	disabled={props.disabled ?? false}
426	/>
427	}
428	>
429	{props.children}
430	</TooltipTrigger>
431	<TooltipPopup>{props.label}</TooltipPopup>
432	</Tooltip>
433	);
434	}
435
436	function groupDevices(state: DeviceServiceState) {	316	function groupDevices(state: DeviceServiceState) {
437	const groups: Array<{ platform: DevicePlatform; devices: DeviceSummary[] }> = [];	317	const groups: Array<{ platform: DevicePlatform; devices: DeviceSummary[] }> = [];
438	for (const platform of ["ios", "android"] as const) {	318	for (const platform of ["ios", "android"] as const) {

apps/web/src/components/device/DevicePhoneViewport.tsx created+151

...	@@ -0,0 +1,151 @@
1	import { useEffect, useRef, type RefObject } from "react";
2	import type { PhoneViewer } from "@t3tools/client-runtime/device/phone-viewer";
3	import type {
4	DeviceAccessorySource,
5	DeviceModelSource,
6	} from "@t3tools/client-runtime/device/model";
7	import type { DeviceShapeProfile } from "@t3tools/client-runtime/device/shape-profile";
8	import { createPhoneInteraction } from "@t3tools/client-runtime/device/phone-interaction";
9	import type { DeviceScreenSize, DeviceStreamClient } from "@t3tools/client-runtime/device/stream";
10	import { bindPhoneTrackpad } from "./phoneTrackpad";
11
12	const loadPhoneViewer = () => import("@t3tools/client-runtime/device/phone-viewer");
13
14	/** Web shell for the framework-independent viewer. The decoded screen and input connection remain owned by DeviceStreamView. */
15	export function DevicePhoneViewport(props: {
16	readonly profile: DeviceShapeProfile;
17	readonly model: DeviceModelSource \| null;
18	readonly accessory: DeviceAccessorySource \| null;
19	readonly source: RefObject<HTMLCanvasElement \| null>;
20	readonly onFrameListener: (listener: (() => void) \| null) => void;
21	readonly onResetReady: (reset: (() => void) \| null) => void;
22	readonly onInputCancel: (cancel: (() => void) \| null) => void;
23	readonly client: RefObject<DeviceStreamClient \| null>;
24	readonly screen: DeviceScreenSize \| null;
25	readonly onUnavailable: () => void;
26	}) {
27	const hostRef = useRef<HTMLDivElement \| null>(null);
28	const canvasRef = useRef<HTMLCanvasElement \| null>(null);
29	const viewerRef = useRef<PhoneViewer \| null>(null);
30	const interactionRef = useRef<ReturnType<typeof createPhoneInteraction> \| null>(null);
31	const screenRef = useRef(props.screen);
32	const profileRef = useRef(props.profile);
33	const modelRef = useRef(props.model);
34	const accessoryRef = useRef(props.accessory);
35	const { source, onFrameListener, client, onInputCancel, onResetReady, onUnavailable } = props;
36
37	useEffect(() => {
38	screenRef.current = props.screen;
39	profileRef.current = props.profile;
40	modelRef.current = props.model;
41	accessoryRef.current = props.accessory;
42	interactionRef.current?.end();
43	viewerRef.current?.setModel(props.model);
44	viewerRef.current?.setAccessory(props.accessory);
45	viewerRef.current?.setScreen(props.screen, props.profile);
46	}, [props.screen, props.profile, props.model, props.accessory]);
47
48	useEffect(() => {
49	const host = hostRef.current;
50	const canvas = canvasRef.current;
51	const decoded = source.current;
52	if (!host \|\| !canvas \|\| !decoded) return;
53	let disposed = false;
54	let trackpad: ReturnType<typeof bindPhoneTrackpad> \| null = null;
55	let stopTrackpadEnd: (() => void) \| undefined;
56	const resize = () => {
57	const { width, height } = host.getBoundingClientRect();
58	viewerRef.current?.resize(width, height, window.devicePixelRatio);
59	};
60	const observer = new ResizeObserver(resize);
61	observer.observe(host);
62	const blur = () => {
63	interactionRef.current?.end();
64	trackpad?.cancel();
65	};
66	onInputCancel(blur);
67	window.addEventListener("blur", blur);
68	void loadPhoneViewer()
69	.then(({ createPhoneViewer }) => {
70	if (disposed) return;
71	const viewer = createPhoneViewer({
72	canvas,
73	source: decoded,
74	onUnavailable,
75	onModelError: (cause) => console.warn("Device 3D asset could not load", cause),
76	profile: profileRef.current,
77	model: modelRef.current,
78	accessory: accessoryRef.current,
79	});
80	viewerRef.current = viewer;
81	onResetReady(viewer.resetPose);
82	viewer.setScreen(screenRef.current);
83	onFrameListener(viewer.frameUpdated);
84	interactionRef.current = createPhoneInteraction({
85	screenPoint: (point, captured) => viewer.screenPoint(point.x, point.y, captured),
86	touch: (phase, point) => client.current?.sendTouch(phase, point.x, point.y),
87	orbit: viewer.orbit,
88	zoomBy: () => {},
89	onInteractionActive: viewer.setInteractionActive,
90	});
91	trackpad = bindPhoneTrackpad(canvas, interactionRef.current);
92	stopTrackpadEnd = window.desktopBridge?.onTrackpadScrollEnd?.(() => trackpad?.endOrbit());
93	resize();
94	viewer.frameUpdated();
95	})
96	.catch(() => {
97	if (!disposed) onUnavailable();
98	});
99	return () => {
100	disposed = true;
101	stopTrackpadEnd?.();
102	trackpad?.dispose();
103	interactionRef.current?.end();
104	interactionRef.current = null;
105	onInputCancel(null);
106	onResetReady(null);
107	onFrameListener(null);
108	observer.disconnect();
109	window.removeEventListener("blur", blur);
110	viewerRef.current?.dispose();
111	viewerRef.current = null;
112	};
113	}, [onInputCancel, onResetReady, client, onFrameListener, onUnavailable, source]);
114
115	const point = (event: React.PointerEvent<HTMLCanvasElement>) => {
116	const rect = event.currentTarget.getBoundingClientRect();
117	return {
118	x: (event.clientX - rect.left) / rect.width,
119	y: (event.clientY - rect.top) / rect.height,
120	};
121	};
122	return (
123	<div className="absolute inset-0">
124	<div ref={hostRef} className="absolute inset-0">
125	<div
126	aria-hidden
127	className="pointer-events-none absolute bottom-[6%] left-1/2 h-5 w-2/5 -translate-x-1/2 rounded-full bg-foreground/10 blur-xl"
128	/>
129	<canvas
130	ref={canvasRef}
131	aria-label="Interactive 3D device. Drag the screen to interact. Drag outside it or swipe with two fingers to turn."
132	className="size-full touch-none"
133	onPointerDown={(event) => {
134	if (event.button !== 0) return;
135	if (!interactionRef.current?.begin(event.pointerId, point(event), event.altKey)) return;
136	event.preventDefault();
137	event.currentTarget.setPointerCapture(event.pointerId);
138	(event.currentTarget.closest('[role="application"]') as HTMLElement \| null)?.focus();
139	}}
140	onPointerMove={(event) => interactionRef.current?.move(event.pointerId, point(event))}
141	onPointerUp={(event) => {
142	interactionRef.current?.move(event.pointerId, point(event));
143	interactionRef.current?.end(event.pointerId);
144	}}
145	onPointerCancel={(event) => interactionRef.current?.end(event.pointerId)}
146	onLostPointerCapture={(event) => interactionRef.current?.end(event.pointerId)}
147	/>
148	</div>
149	</div>
150	);
151	}

apps/web/src/components/device/DeviceStreamView.tsx+263-112

...	@@ -1,9 +1,14 @@	...	@@ -1,9 +1,14 @@
1	import type { DevicePlatform, EnvironmentId } from "@t3tools/contracts";	1	import type { DevicePlatform, EnvironmentId } from "@t3tools/contracts";
2	import { useCallback, useEffect, useMemo, useRef, useState } from "react";	2	import { useCallback, useEffect, useMemo, useRef, useState, type ReactNode } from "react";
3		3
4	import { cn } from "~/lib/utils";	4	import { cn } from "~/lib/utils";
5	import { Button } from "~/components/ui/button";	5	import { Button } from "~/components/ui/button";
6	import { refreshDeviceHubAccess, useDeviceHubAccess } from "~/state/device";	6	import { refreshDeviceHubAccess, useDeviceHubAccess } from "~/state/device";
		7	import { createCanvasFrameSink } from "@t3tools/client-runtime/device/frame";
		8	import { resolveDeviceShape } from "@t3tools/client-runtime/device/shape-profile";
		9	import { deviceKeyboard, deviceModel } from "./deviceModels";
		10	import { fitDeviceFrame } from "./deviceFrameLayout";
		11	import { DevicePhoneViewport } from "./DevicePhoneViewport";
7	import { DeviceLoadingView } from "./DeviceLoadingView";	12	import { DeviceLoadingView } from "./DeviceLoadingView";
8	import { type DeviceAxElement, fetchDeviceAxTree } from "./deviceHubApi";	13	import { type DeviceAxElement, fetchDeviceAxTree } from "./deviceHubApi";
9	import {	14	import {
...	@@ -15,6 +20,17 @@ import {	...	@@ -15,6 +20,17 @@ import {
15	} from "@t3tools/client-runtime/device/stream";	20	} from "@t3tools/client-runtime/device/stream";
16		21
17	const AX_POLL_INTERVAL_MS = 2_000;	22	const AX_POLL_INTERVAL_MS = 2_000;
		23	const CONTROLS_RAIL_WIDTH = 56;
		24
		25	export interface DeviceViewControls {
		26	readonly phone: boolean;
		27	readonly streaming: boolean;
		28	readonly phoneUnavailableReason: string \| null;
		29	readonly showPhone: () => void;
		30	readonly showFlat: () => void;
		31	readonly resetView: () => void;
		32	readonly keyboard: { readonly attached: boolean; readonly toggle: () => void } \| null;
		33	}
18		34
19	export interface DeviceStreamHandle {	35	export interface DeviceStreamHandle {
20	readonly pressButton: (button: DeviceHardwareButton) => void;	36	readonly pressButton: (button: DeviceHardwareButton) => void;
...	@@ -37,11 +53,31 @@ export function DeviceStreamView(props: {	...	@@ -37,11 +53,31 @@ export function DeviceStreamView(props: {
37	readonly deviceDescription?: string;	53	readonly deviceDescription?: string;
38	readonly visible: boolean;	54	readonly visible: boolean;
39	readonly hostId: string;	55	readonly hostId: string;
		56	/** The full panel opts into the phone spike; compact viewers retain their flat presentation. */
		57	readonly allowPhoneView?: boolean;
		58	readonly renderControls?: (view: DeviceViewControls) => ReactNode;
40	/** Draw accessibility element frames over the screen. */	59	/** Draw accessibility element frames over the screen. */
41	readonly axOverlay?: boolean;	60	readonly axOverlay?: boolean;
42	readonly onHandle?: (handle: DeviceStreamHandle \| null) => void;	61	readonly onHandle?: (handle: DeviceStreamHandle \| null) => void;
43	readonly onScreen?: (screen: DeviceScreenSize \| null) => void;	62	readonly onScreen?: (screen: DeviceScreenSize \| null) => void;
44	}) {	63	}) {
		64	const [presentation, setPresentation] = useState<"phone" \| "flat">("phone");
		65	const [keyboardAttached, setKeyboardAttached] = useState(false);
		66	const [phoneUnavailable, setPhoneUnavailable] = useState(false);
		67	const onPhoneUnavailable = useCallback(() => setPhoneUnavailable(true), []);
		68	const cancelPhoneInputRef = useRef<(() => void) \| null>(null);
		69	const cancelPhoneInput = useCallback(() => cancelPhoneInputRef.current?.(), []);
		70	const resetViewRef = useRef<(() => void) \| null>(null);
		71	const onResetReady = useCallback((reset: (() => void) \| null) => {
		72	resetViewRef.current = reset;
		73	}, []);
		74	const frameListenerRef = useRef<(() => void) \| null>(null);
		75	const onFrameListener = useCallback((listener: (() => void) \| null) => {
		76	frameListenerRef.current = listener;
		77	}, []);
		78	const onInputCancel = useCallback((cancel: (() => void) \| null) => {
		79	cancelPhoneInputRef.current = cancel;
		80	}, []);
45	const access = useDeviceHubAccess(props.environmentId, props.hostId);	81	const access = useDeviceHubAccess(props.environmentId, props.hostId);
46	const canvasRef = useRef<HTMLCanvasElement \| null>(null);	82	const canvasRef = useRef<HTMLCanvasElement \| null>(null);
47	const clientRef = useRef<DeviceStreamClient \| null>(null);	83	const clientRef = useRef<DeviceStreamClient \| null>(null);
...	@@ -67,7 +103,7 @@ export function DeviceStreamView(props: {	...	@@ -67,7 +103,7 @@ export function DeviceStreamView(props: {
67	}	103	}
68	const client = createDeviceStreamClient(	104	const client = createDeviceStreamClient(
69	{ platform: props.platform, deviceId: props.deviceId, access },	105	{ platform: props.platform, deviceId: props.deviceId, access },
70	canvas,	106	createCanvasFrameSink(canvas, () => frameListenerRef.current?.()),
71	{	107	{
72	onStatus: (next, nextDetail) => {	108	onStatus: (next, nextDetail) => {
73	setStatus(next);	109	setStatus(next);
...	@@ -101,6 +137,7 @@ export function DeviceStreamView(props: {	...	@@ -101,6 +137,7 @@ export function DeviceStreamView(props: {
101	client.start();	137	client.start();
102	onHandle?.({ pressButton: client.pressButton, rotate: client.rotate, inputConnected: false });	138	onHandle?.({ pressButton: client.pressButton, rotate: client.rotate, inputConnected: false });
103	return () => {	139	return () => {
		140	cancelPhoneInput();
104	client.stop();	141	client.stop();
105	clientRef.current = null;	142	clientRef.current = null;
106	onHandle?.(null);	143	onHandle?.(null);
...	@@ -109,6 +146,7 @@ export function DeviceStreamView(props: {	...	@@ -109,6 +146,7 @@ export function DeviceStreamView(props: {
109	};	146	};
110	}, [	147	}, [
111	access,	148	access,
		149	cancelPhoneInput,
112	onHandle,	150	onHandle,
113	onScreen,	151	onScreen,
114	props.deviceId,	152	props.deviceId,
...	@@ -131,6 +169,16 @@ export function DeviceStreamView(props: {	...	@@ -131,6 +169,16 @@ export function DeviceStreamView(props: {
131	return w / h;	169	return w / h;
132	}, [props.platform, screen]);	170	}, [props.platform, screen]);
133		171
		172	const showPhone =
		173	props.allowPhoneView &&
		174	status === "streaming" &&
		175	props.visible &&
		176	presentation === "phone" &&
		177	!phoneUnavailable &&
		178	!mjpegUrl &&
		179	!props.axOverlay;
		180	const controlsInset = props.renderControls && !showPhone ? CONTROLS_RAIL_WIDTH : 0;
		181
134	// The frame is the largest box at `aspect` that fits the container, so a	182	// The frame is the largest box at `aspect` that fits the container, so a
135	// narrow panel shows a shorter phone rather than a squeezed one. CSS	183	// narrow panel shows a shorter phone rather than a squeezed one. CSS
136	// `aspect-ratio` alone cannot do this: with the height pinned to 100% the	184	// `aspect-ratio` alone cannot do this: with the height pinned to 100% the
...	@@ -154,12 +202,8 @@ export function DeviceStreamView(props: {	...	@@ -154,12 +202,8 @@ export function DeviceStreamView(props: {
154	return () => observer.disconnect();	202	return () => observer.disconnect();
155	}, []);	203	}, []);
156	const frame = useMemo(() => {	204	const frame = useMemo(() => {
157	if (host.width === 0 \|\| host.height === 0) return { width: 0, height: 0 };	205	return fitDeviceFrame(aspect, host.width, host.height, controlsInset);
158	const byHeight = { width: host.height * aspect, height: host.height };	206	}, [aspect, controlsInset, host]);
159	return byHeight.width <= host.width
160	? byHeight
161	: { width: host.width, height: host.width / aspect };
162	}, [aspect, host]);
163		207
164	// serve-sim streams the raw framebuffer; rotate the display for a device	208	// serve-sim streams the raw framebuffer; rotate the display for a device
165	// that reports landscape while its frames stay portrait.	209	// that reports landscape while its frames stay portrait.
...	@@ -231,123 +275,230 @@ export function DeviceStreamView(props: {	...	@@ -231,123 +275,230 @@ export function DeviceStreamView(props: {
231	return { x: Math.min(1, Math.max(0, x)), y: Math.min(1, Math.max(0, y)) };	275	return { x: Math.min(1, Math.max(0, x)), y: Math.min(1, Math.max(0, y)) };
232	};	276	};
233		277
		278	const phoneUnavailableReason = phoneUnavailable
		279	? "3D is unavailable on this browser"
		280	: mjpegUrl
		281	? "3D requires the H.264 stream"
		282	: props.axOverlay
		283	? "Turn off accessibility frames to use 3D"
		284	: null;
		285
		286	const keyboardSource = deviceKeyboard(props.platform, props.deviceName ?? "");
		287	const resetView = useCallback(() => {
		288	const orientation = keyboardAttached ? "landscape_right" : "portrait";
		289	if (screen?.orientation !== orientation) clientRef.current?.setOrientation(orientation);
		290	resetViewRef.current?.();
		291	}, [keyboardAttached, screen?.orientation]);
		292	const profile = resolveDeviceShape({
		293	platform: props.platform,
		294	name: props.deviceName ?? "",
		295	portraitAspect: Math.min(aspect, 1 / aspect),
		296	});
		297
234	return (	298	return (
235	<div	299	<div
236	ref={hostRef}	300	className={cn(
237	className="relative flex size-full items-center justify-center overflow-hidden bg-black/90 outline-none"	301	"relative flex size-full min-h-0 min-w-0",
238	tabIndex={0}	302	props.allowPhoneView ? "bg-background" : "bg-black/90",
239	role="application"	303	)}
240	aria-label={`${props.platform === "ios" ? "iOS Simulator" : "Android Emulator"} screen`}
241	onKeyDown={(event) => {
242	if (event.target !== event.currentTarget) return;
243	if (event.metaKey && !["r", "R"].includes(event.key)) return;
244	event.preventDefault();
245	clientRef.current?.sendKey(event.nativeEvent, "down");
246	}}
247	onKeyUp={(event) => {
248	if (event.target !== event.currentTarget) return;
249	clientRef.current?.sendKey(event.nativeEvent, "up");
250	}}
251	>	304	>
		305	{props.renderControls ? (
		306	<DeviceControlsSlot
		307	renderControls={props.renderControls}
		308	view={{
		309	phone: !!showPhone,
		310	streaming: status === "streaming",
		311	phoneUnavailableReason,
		312	keyboard:
		313	showPhone && keyboardSource
		314	? {
		315	attached: keyboardAttached,
		316	toggle: () => {
		317	cancelPhoneInput();
		318	if (!keyboardAttached && screen?.orientation !== "landscape_right")
		319	clientRef.current?.setOrientation("landscape_right");
		320	setKeyboardAttached(!keyboardAttached);
		321	},
		322	}
		323	: null,
		324	showPhone: () => setPresentation("phone"),
		325	showFlat: () => setPresentation("flat"),
		326	}}
		327	onResetView={resetView}
		328	/>
		329	) : null}
252	<div	330	<div
253	className="relative select-none"	331	ref={hostRef}
254	style={{ width: frame.width, height: frame.height }}	332	className={cn(
255	onPointerDown={(event) => {	333	"relative flex min-h-0 min-w-0 flex-1 items-center justify-center overflow-hidden outline-none",
256	event.currentTarget.setPointerCapture(event.pointerId);	334	controlsInset && "pr-14",
257	(event.currentTarget.parentElement as HTMLElement \| null)?.focus();	335	)}
258	pointerActive.current = true;	336	tabIndex={0}
259	const { x, y } = normalizedPoint(event);	337	role="application"
260	clientRef.current?.sendTouch("begin", x, y);	338	aria-label={`${props.platform === "ios" ? "iOS Simulator" : "Android Emulator"} screen`}
		339	onKeyDown={(event) => {
		340	if (event.target !== event.currentTarget) return;
		341	if (event.metaKey && !["r", "R"].includes(event.key)) return;
		342	event.preventDefault();
		343	clientRef.current?.sendKey(event.nativeEvent, "down");
261	}}	344	}}
262	onPointerMove={(event) => {	345	onKeyUp={(event) => {
263	if (!pointerActive.current) return;	346	if (event.target !== event.currentTarget) return;
264	const { x, y } = normalizedPoint(event);	347	clientRef.current?.sendKey(event.nativeEvent, "up");
265	clientRef.current?.sendTouch("move", x, y);
266	}}
267	onPointerUp={(event) => {
268	if (!pointerActive.current) return;
269	pointerActive.current = false;
270	const { x, y } = normalizedPoint(event);
271	clientRef.current?.sendTouch("end", x, y);
272	}}
273	onPointerCancel={(event) => {
274	if (!pointerActive.current) return;
275	pointerActive.current = false;
276	const { x, y } = normalizedPoint(event);
277	clientRef.current?.sendTouch("end", x, y);
278	}}	348	}}
279	>	349	>
280	<canvas	350	<div
281	ref={canvasRef}	351	className={cn("relative select-none", showPhone && "invisible pointer-events-none")}
282	className={cn("absolute top-0 left-0", mjpegUrl && "hidden")}	352	style={{ width: frame.width, height: frame.height }}
283	style={mediaStyle}	353	onPointerDown={(event) => {
284	/>	354	event.currentTarget.setPointerCapture(event.pointerId);
285	{props.visible && access && mjpegUrl ? (	355	(event.currentTarget.parentElement as HTMLElement \| null)?.focus();
286	<img	356	pointerActive.current = true;
287	key={mjpegGeneration}	357	const { x, y } = normalizedPoint(event);
288	ref={attachMjpegImage}	358	clientRef.current?.sendTouch("begin", x, y);
289	alt=""	359	}}
290	draggable={false}	360	onPointerMove={(event) => {
291	className="absolute top-0 left-0 object-contain"	361	if (!pointerActive.current) return;
		362	const { x, y } = normalizedPoint(event);
		363	clientRef.current?.sendTouch("move", x, y);
		364	}}
		365	onPointerUp={(event) => {
		366	if (!pointerActive.current) return;
		367	pointerActive.current = false;
		368	const { x, y } = normalizedPoint(event);
		369	clientRef.current?.sendTouch("end", x, y);
		370	}}
		371	onPointerCancel={(event) => {
		372	if (!pointerActive.current) return;
		373	pointerActive.current = false;
		374	const { x, y } = normalizedPoint(event);
		375	clientRef.current?.sendTouch("end", x, y);
		376	}}
		377	>
		378	<canvas
		379	ref={canvasRef}
		380	className={cn("absolute top-0 left-0", mjpegUrl && "hidden")}
292	style={mediaStyle}	381	style={mediaStyle}
293	/>	382	/>
		383	{props.visible && access && mjpegUrl ? (
		384	<img
		385	key={mjpegGeneration}
		386	ref={attachMjpegImage}
		387	alt=""
		388	draggable={false}
		389	className="absolute top-0 left-0 object-contain"
		390	style={mediaStyle}
		391	/>
		392	) : null}
		393	{axElements.length > 0 ? (
		394	<div className="pointer-events-none absolute inset-0" aria-hidden>
		395	{axElements.map((element) => (
		396	<div
		397	key={element.id}
		398	className="absolute border border-sky-400/80 bg-sky-400/10"
		399	style={{
		400	left: `${element.x * 100}%`,
		401	top: `${element.y * 100}%`,
		402	width: `${element.width * 100}%`,
		403	height: `${element.height * 100}%`,
		404	}}
		405	>
		406	{element.label ? (
		407	<span className="absolute -top-3.5 left-0 max-w-full truncate rounded-sm bg-sky-500 px-1 text-[9px] leading-3.5 text-white">
		408	{element.label}
		409	</span>
		410	) : null}
		411	</div>
		412	))}
		413	</div>
		414	) : null}
		415	</div>
		416	{showPhone ? (
		417	<DevicePhoneViewport
		418	profile={profile}
		419	model={deviceModel(props.platform, props.deviceName ?? "")}
		420	accessory={keyboardAttached ? keyboardSource : null}
		421	source={canvasRef}
		422	onFrameListener={onFrameListener}
		423	client={clientRef}
		424	onInputCancel={onInputCancel}
		425	onResetReady={onResetReady}
		426	screen={screen}
		427	onUnavailable={onPhoneUnavailable}
		428	/>
		429	) : null}
		430	{props.allowPhoneView && !props.renderControls && status === "streaming" ? (
		431	<div className="absolute top-3 left-3 flex gap-1 rounded-lg border border-border/50 bg-background/90 p-1 shadow-sm">
		432	<Button
		433	variant={showPhone ? "secondary" : "ghost"}
		434	size="xs"
		435	aria-pressed={!!showPhone}
		436	disabled={phoneUnavailable \|\| !!mjpegUrl \|\| !!props.axOverlay}
		437	title={
		438	phoneUnavailable
		439	? "3D is unavailable on this browser"
		440	: mjpegUrl
		441	? "3D requires the H.264 stream"
		442	: props.axOverlay
		443	? "Turn off accessibility frames to use 3D"
		444	: "Show 3D phone"
		445	}
		446	onClick={() => setPresentation("phone")}
		447	>
		448	3D
		449	</Button>
		450	<Button
		451	variant={!showPhone ? "secondary" : "ghost"}
		452	size="xs"
		453	aria-pressed={!showPhone}
		454	onClick={() => setPresentation("flat")}
		455	>
		456	Flat
		457	</Button>
		458	</div>
294	) : null}	459	) : null}
295	{axElements.length > 0 ? (	460	{status === "streaming" && !inputState.connected ? (
296	<div className="pointer-events-none absolute inset-0" aria-hidden>	461	<div className="pointer-events-none absolute inset-x-0 bottom-0 flex justify-center p-2">
297	{axElements.map((element) => (	462	<span className="rounded-md bg-background/85 px-2 py-1 text-xs text-muted-foreground">
298	<div	463	Input disconnected{inputState.detail ? ` (${inputState.detail})` : ""}, reconnecting…
299	key={element.id}	464	</span>
300	className="absolute border border-sky-400/80 bg-sky-400/10"	465	</div>
301	style={{	466	) : null}
302	left: `${element.x * 100}%`,	467	{status !== "streaming" ? (
303	top: `${element.y * 100}%`,	468	<div className="absolute inset-0">
304	width: `${element.width * 100}%`,	469	<DeviceLoadingView
305	height: `${element.height * 100}%`,	470	name={props.deviceName ?? "Device"}
306	}}	471	description={props.deviceDescription ?? ""}
307	>	472	stage="stream"
308	{element.label ? (	473	message={status === "error" ? (detail ?? "Stream failed.") : "Connecting video…"}
309	<span className="absolute -top-3.5 left-0 max-w-full truncate rounded-sm bg-sky-500 px-1 text-[9px] leading-3.5 text-white">	474	error={status === "error"}
310	{element.label}	475	>
311	</span>	476	{status === "error" ? (
312	) : null}	477	<Button
313	</div>	478	size="sm"
314	))}	479	variant="outline"
		480	onClick={() => {
		481	// An expired ticket surfaces as unauthorized on restart and
		482	// refreshes access through the effect; no need to mint one here.
		483	clientRef.current?.stop();
		484	clientRef.current?.start();
		485	}}
		486	>
		487	Reconnect
		488	</Button>
		489	) : null}
		490	</DeviceLoadingView>
315	</div>	491	</div>
316	) : null}	492	) : null}
317	</div>	493	</div>
318	{status === "streaming" && !inputState.connected ? (
319	<div className="pointer-events-none absolute inset-x-0 bottom-0 flex justify-center p-2">
320	<span className="rounded-md bg-background/85 px-2 py-1 text-xs text-muted-foreground">
321	Input disconnected{inputState.detail ? ` (${inputState.detail})` : ""}, reconnecting…
322	</span>
323	</div>
324	) : null}
325	{status !== "streaming" ? (
326	<div className="absolute inset-0">
327	<DeviceLoadingView
328	name={props.deviceName ?? "Device"}
329	description={props.deviceDescription ?? ""}
330	stage="stream"
331	message={status === "error" ? (detail ?? "Stream failed.") : "Connecting video…"}
332	error={status === "error"}
333	>
334	{status === "error" ? (
335	<Button
336	size="sm"
337	variant="outline"
338	onClick={() => {
339	// An expired ticket surfaces as unauthorized on restart and
340	// refreshes access through the effect; no need to mint one here.
341	clientRef.current?.stop();
342	clientRef.current?.start();
343	}}
344	>
345	Reconnect
346	</Button>
347	) : null}
348	</DeviceLoadingView>
349	</div>
350	) : null}
351	</div>	494	</div>
352	);	495	);
353	}	496	}
		497
		498	function DeviceControlsSlot(props: {
		499	renderControls: (view: DeviceViewControls) => ReactNode;
		500	view: Omit<DeviceViewControls, "resetView">;
		501	onResetView: () => void;
		502	}) {
		503	return props.renderControls({ ...props.view, resetView: props.onResetView });
		504	}

apps/web/src/components/device/DeviceToolsPanel.tsx+19-84

...	@@ -1,14 +1,7 @@	...	@@ -1,14 +1,7 @@
1	import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess";	1	import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess";
2	import type {	2	import type { DevicePermission, DeviceSummary, DeviceTextSize } from "@t3tools/contracts";
3	DeviceActionInput,
4	DeviceDetail,
5	DevicePermission,
6	DeviceSummary,
7	DeviceTextSize,
8	EnvironmentId,
9	} from "@t3tools/contracts";
10	import { ChevronDown, X } from "lucide-react";	3	import { ChevronDown, X } from "lucide-react";
11	import { useCallback, useEffect, useMemo, useState } from "react";	4	import { useEffect, useState } from "react";
12		5
13	import { Button } from "~/components/ui/button";	6	import { Button } from "~/components/ui/button";
14	import { Collapsible, CollapsiblePanel, CollapsibleTrigger } from "~/components/ui/collapsible";	7	import { Collapsible, CollapsiblePanel, CollapsibleTrigger } from "~/components/ui/collapsible";
...	@@ -24,21 +17,8 @@ import { Spinner } from "~/components/ui/spinner";	...	@@ -24,21 +17,8 @@ import { Spinner } from "~/components/ui/spinner";
24	import { Switch } from "~/components/ui/switch";	17	import { Switch } from "~/components/ui/switch";
25	import { Toggle, ToggleGroup } from "~/components/ui/toggle-group";	18	import { Toggle, ToggleGroup } from "~/components/ui/toggle-group";
26	import { cn } from "~/lib/utils";	19	import { cn } from "~/lib/utils";
27	import { deviceEnvironment } from "~/state/device";	20	import type { DeviceControls } from "./useDeviceControls";
28	import { formatEnvironmentQueryError } from "~/state/query";	21	import { type DeviceEventLogEntry, subscribeDeviceEventLog } from "./deviceHubApi";
29	import { useAtomCommand } from "~/state/use-atom-command";
30	import {
31	type DeviceEventLogEntry,
32	type DeviceForegroundInfo,
33	subscribeDeviceEventLog,
34	subscribeDeviceForeground,
35	} from "./deviceHubApi";
36
37	type ActionBody = DeviceActionInput extends infer A
38	? A extends { readonly type: string }
39	? Omit<A, "hostId" \| "deviceId">
40	: never
41	: never;
42		22
43	const TEXT_SIZES: ReadonlyArray<{ value: DeviceTextSize; label: string }> = [	23	const TEXT_SIZES: ReadonlyArray<{ value: DeviceTextSize; label: string }> = [
44	{ value: "small", label: "Small" },	24	{ value: "small", label: "Small" },
...	@@ -102,7 +82,8 @@ const LOCATION_PRESETS = [	...	@@ -102,7 +82,8 @@ const LOCATION_PRESETS = [
102	* local state so the controls never show a value the device did not confirm.	82	* local state so the controls never show a value the device did not confirm.
103	*/	83	*/
104	export function DeviceToolsPanel(props: {	84	export function DeviceToolsPanel(props: {
105	readonly environmentId: EnvironmentId;	85	readonly controls: DeviceControls;
		86	readonly hostDiagnostics: string \| undefined;
106	readonly device: DeviceSummary;	87	readonly device: DeviceSummary;
107	readonly access: DeviceHubAccess \| null;	88	readonly access: DeviceHubAccess \| null;
108	readonly axOverlay: boolean;	89	readonly axOverlay: boolean;
...	@@ -110,62 +91,10 @@ export function DeviceToolsPanel(props: {	...	@@ -110,62 +91,10 @@ export function DeviceToolsPanel(props: {
110	readonly onClose: () => void;	91	readonly onClose: () => void;
111	readonly className?: string;	92	readonly className?: string;
112	}) {	93	}) {
113	const { environmentId, device } = props;	94	const { device, controls } = props;
114	const readDetail = useAtomCommand(deviceEnvironment.detail, { reportFailure: false });	95	const { detail, pending, error, foregroundApp, disabled, act } = controls;
115	const runAction = useAtomCommand(deviceEnvironment.action, { reportFailure: false });
116	const [detail, setDetail] = useState<DeviceDetail \| null>(null);
117	const [pending, setPending] = useState(false);
118	const [error, setError] = useState<string \| null>(null);
119	const [foreground, setForeground] = useState<DeviceForegroundInfo \| null \| undefined>(undefined);
120	const isIos = device.platform === "ios";
121
122	const target = useMemo(
123	() => ({ hostId: device.hostId, deviceId: device.id }),
124	[device.hostId, device.id],
125	);
126
127	// The panel is keyed by device, so a mount is always a fresh device.
128	useEffect(() => {
129	let cancelled = false;
130	void readDetail({ environmentId, input: target }).then((result) => {
131	if (cancelled) return;
132	if (result._tag === "Success") setDetail(result.value);
133	else setError(formatEnvironmentQueryError(result.cause));
134	});
135	return () => {
136	cancelled = true;
137	};
138	}, [environmentId, readDetail, target]);
139
140	useEffect(() => {
141	if (!props.access) return;
142	return subscribeDeviceForeground(
143	{ access: props.access, platform: device.platform, deviceId: device.id },
144	setForeground,
145	);
146	}, [device.id, device.platform, props.access]);
147
148	const act = useCallback(
149	async (body: ActionBody) => {
150	setPending(true);
151	setError(null);
152	try {
153	const result = await runAction({
154	environmentId,
155	input: { ...target, ...body } as DeviceActionInput,
156	});
157	if (result._tag === "Success") setDetail(result.value);
158	else setError(formatEnvironmentQueryError(result.cause));
159	} finally {
160	setPending(false);
161	}
162	},
163	[environmentId, runAction, target],
164	);
165
166	const settings = detail?.settings;	96	const settings = detail?.settings;
167	const foregroundApp = foreground === undefined ? (detail?.foregroundApp ?? null) : foreground;	97	const isIos = device.platform === "ios";
168	const disabled = pending \|\| detail === null;
169		98
170	return (	99	return (
171	<div	100	<div
...	@@ -194,6 +123,14 @@ export function DeviceToolsPanel(props: {	...	@@ -194,6 +123,14 @@ export function DeviceToolsPanel(props: {
194	</div>	123	</div>
195	) : null}	124	) : null}
196		125
		126	{props.hostDiagnostics ? (
		127	<Section title="Host diagnostics">
		128	<p className="whitespace-pre-line text-xs text-muted-foreground">
		129	{props.hostDiagnostics}
		130	</p>
		131	</Section>
		132	) : null}
		133
197	<Section title="App">	134	<Section title="App">
198	<Row label="Foreground">	135	<Row label="Foreground">
199	<span className="truncate font-mono text-xs">{foregroundApp?.id ?? "—"}</span>	136	<span className="truncate font-mono text-xs">{foregroundApp?.id ?? "—"}</span>
...	@@ -398,9 +335,7 @@ export function DeviceToolsPanel(props: {	...	@@ -398,9 +335,7 @@ export function DeviceToolsPanel(props: {
398	function Section(props: { readonly title: string; readonly children: React.ReactNode }) {	335	function Section(props: { readonly title: string; readonly children: React.ReactNode }) {
399	return (	336	return (
400	<section className="flex flex-col gap-2 border-b px-3 py-2.5 last:border-b-0">	337	<section className="flex flex-col gap-2 border-b px-3 py-2.5 last:border-b-0">
401	<h3 className="text-xs font-medium tracking-wide text-muted-foreground uppercase">	338	<h3 className="text-xs font-medium text-muted-foreground">{props.title}</h3>
402	{props.title}
403	</h3>
404	{props.children}	339	{props.children}
405	</section>	340	</section>
406	);	341	);
...	@@ -698,7 +633,7 @@ function EventLogSection(props: {	...	@@ -698,7 +633,7 @@ function EventLogSection(props: {
698		633
699	return (	634	return (
700	<Collapsible open={open} onOpenChange={setOpen}>	635	<Collapsible open={open} onOpenChange={setOpen}>
701	<CollapsibleTrigger className="flex w-full items-center gap-1.5 border-b px-3 py-2.5 text-left text-xs font-medium tracking-wide text-muted-foreground uppercase">	636	<CollapsibleTrigger className="flex w-full items-center gap-1.5 border-b px-3 py-2.5 text-left text-xs font-medium text-muted-foreground">
702	Event log	637	Event log
703	<ChevronDown	638	<ChevronDown
704	className={cn("ml-auto size-3.5 transition-transform", open && "rotate-180")}	639	className={cn("ml-auto size-3.5 transition-transform", open && "rotate-180")}

apps/web/src/components/device/DeviceWorkspace.tsx created+131

...	@@ -0,0 +1,131 @@
1	import type { DeviceSummary, EnvironmentId } from "@t3tools/contracts";
2	import { useEffect, useRef, useState } from "react";
3	import {
4	captureDeviceScreenshot,
5	DeviceScreenshotError,
6	} from "@t3tools/client-runtime/device/screenshot";
7	import { refreshDeviceHubAccess, useDeviceHubAccess } from "~/state/device";
8	import { DeviceControlsRail } from "./DeviceControlsRail";
9	import { DeviceStreamView, type DeviceStreamHandle } from "./DeviceStreamView";
10	import { DeviceToolsPanel } from "./DeviceToolsPanel";
11	import { useDeviceControls } from "./useDeviceControls";
12
13	/** Keyed by environment and device; the screen, quick controls and drawer share the same session. */
14	export function DeviceWorkspace(props: {
15	environmentId: EnvironmentId;
16	device: DeviceSummary;
17	hostLabel: string;
18	hostDiagnostics: string \| undefined;
19	visible: boolean;
20	onFloat: () => void;
21	onClose: () => void;
22	onPowerOff: () => void;
23	}) {
24	const [handle, setHandle] = useState<DeviceStreamHandle \| null>(null);
25	const [toolsOpen, setToolsOpen] = useState(false);
26	const [axOverlay, setAxOverlay] = useState(false);
27	const [screenshotPending, setScreenshotPending] = useState(false);
28	const [screenshotError, setScreenshotError] = useState<string \| null>(null);
29	const captureRef = useRef<AbortController \| null>(null);
30	const downloadRef = useRef<string \| null>(null);
31	const access = useDeviceHubAccess(props.environmentId, props.device.hostId);
32	const controls = useDeviceControls({ ...props, access });
33	useEffect(() => {
34	if (!access \|\| !props.visible) captureRef.current?.abort();
35	return () => {
36	captureRef.current?.abort();
37	if (downloadRef.current) URL.revokeObjectURL(downloadRef.current);
38	downloadRef.current = null;
39	};
40	}, [access, props.visible]);
41	const saveScreenshot = () => {
42	if (!access \|\| captureRef.current \|\| !props.visible) return;
43	const controller = new AbortController();
44	captureRef.current = controller;
45	setScreenshotPending(true);
46	setScreenshotError(null);
47	return captureDeviceScreenshot(
48	{ access, platform: props.device.platform, deviceId: props.device.id },
49	controller.signal,
50	)
51	.then((image) => {
52	if (controller.signal.aborted) return;
53	if (downloadRef.current) URL.revokeObjectURL(downloadRef.current);
54	const url = URL.createObjectURL(image);
55	downloadRef.current = url;
56	const link = document.createElement("a");
57	link.href = url;
58	link.download = `${props.device.name.replace(/[^a-z0-9-]/gi, "-")}-${Date.now()}.png`;
59	document.body.append(link);
60	link.click();
61	link.remove();
62	})
63	.catch((cause: unknown) => {
64	if (!controller.signal.aborted) {
65	if (cause instanceof DeviceScreenshotError && cause.status === 401)
66	refreshDeviceHubAccess(props.environmentId);
67	setScreenshotError(
68	cause instanceof Error ? cause.message : "Screenshot capture failed. Try again.",
69	);
70	}
71	})
72	.finally(() => {
73	if (captureRef.current === controller) {
74	captureRef.current = null;
75	setScreenshotPending(false);
76	}
77	});
78	};
79	return (
80	<>
81	<div className="relative flex min-h-0 min-w-0 flex-1 flex-col">
82	{screenshotError \|\| (controls.error && !toolsOpen) ? (
83	<p role="alert" className="px-3 py-2 text-xs text-destructive">
84	{screenshotError ?? controls.error}
85	</p>
86	) : null}
87	<div className="min-h-0 flex-1">
88	<DeviceStreamView
89	environmentId={props.environmentId}
90	platform={props.device.platform}
91	deviceName={props.device.name}
92	deviceDescription={`${props.hostLabel} · ${props.device.version}`}
93	deviceId={props.device.id}
94	hostId={props.device.hostId}
95	visible={props.visible}
96	axOverlay={axOverlay}
97	allowPhoneView
98	onHandle={setHandle}
99	renderControls={(view) => (
100	<DeviceControlsRail
101	platform={props.device.platform}
102	handle={handle}
103	view={view}
104	controls={controls}
105	screenshotPending={screenshotPending}
106	onScreenshot={() => void saveScreenshot()}
107	toolsOpen={toolsOpen}
108	onTools={() => setToolsOpen(!toolsOpen)}
109	onFloat={props.onFloat}
110	onClose={props.onClose}
111	onPowerOff={props.onPowerOff}
112	/>
113	)}
114	/>
115	</div>
116	</div>
117	{toolsOpen ? (
118	<DeviceToolsPanel
119	device={props.device}
120	controls={controls}
121	hostDiagnostics={props.hostDiagnostics}
122	access={access}
123	axOverlay={axOverlay}
124	onAxOverlayChange={setAxOverlay}
125	onClose={() => setToolsOpen(false)}
126	className="absolute inset-y-0 right-0 z-10 w-full max-w-72 border-l shadow-lg @[700px]:static @[700px]:w-72 @[700px]:shrink-0 @[700px]:shadow-none"
127	/>
128	) : null}
129	</>
130	);
131	}

apps/web/src/components/device/deviceFrameLayout.test.ts created+9

...	@@ -0,0 +1,9 @@
1	import { expect, it } from "vite-plus/test";
2	import { fitDeviceFrame } from "./deviceFrameLayout";
3
4	it("fits a flat device beside the fixed controls rail", () => {
5	const frame = fitDeviceFrame(9 / 19.5, 480, 1_000, 56);
6	expect(frame.width + 56).toBeLessThanOrEqual(480);
7	expect(frame.height).toBeLessThanOrEqual(1_000);
8	expect(frame.width / frame.height).toBeCloseTo(9 / 19.5);
9	});

apps/web/src/components/device/deviceFrameLayout.ts created+8

...	@@ -0,0 +1,8 @@
1	export function fitDeviceFrame(aspect: number, width: number, height: number, rightInset = 0) {
2	const availableWidth = Math.max(0, width - rightInset);
3	if (availableWidth === 0 \|\| height === 0) return { width: 0, height: 0 };
4	const byHeight = { width: height * aspect, height };
5	return byHeight.width <= availableWidth
6	? byHeight
7	: { width: availableWidth, height: availableWidth / aspect };
8	}

apps/web/src/components/device/deviceModels.ts created+35

...	@@ -0,0 +1,35 @@
1	import {
2	resolveDeviceModelId,
3	type DeviceAccessorySource,
4	type DeviceModelSource,
5	} from "@t3tools/client-runtime/device/model";
6	import type { DevicePlatform } from "@t3tools/contracts";
7	import iphone18Pro from "./models/iphone-18-pro.glb?url";
8	import iphone18ProMax from "./models/iphone-18-pro-max.glb?url";
9	import magicKeyboard from "./models/ipad-pro-13-m5-magic-keyboard.glb?url";
10	import ipadPro13M5 from "./models/ipad-pro-13-m5.glb?url";
11
12	// Bundled URLs follow the client origin in local, desktop, hosted and remote sessions.
13	const models: Record<DeviceModelSource["id"], DeviceModelSource> = {
14	"iphone-18-pro": { id: "iphone-18-pro", url: iphone18Pro },
15	"iphone-18-pro-max": { id: "iphone-18-pro-max", url: iphone18ProMax },
16	"ipad-pro-13-m5": { id: "ipad-pro-13-m5", url: ipadPro13M5 },
17	};
18
19	export function deviceModel(platform: DevicePlatform, name: string): DeviceModelSource \| null {
20	const id = resolveDeviceModelId(platform, name);
21	return id ? models[id] : null;
22	}
23
24	const keyboard: DeviceAccessorySource = {
25	id: "ipad-pro-13-m5-magic-keyboard",
26	modelId: "ipad-pro-13-m5",
27	url: magicKeyboard,
28	};
29
30	export function deviceKeyboard(
31	platform: DevicePlatform,
32	name: string,
33	): DeviceAccessorySource \| null {
34	return resolveDeviceModelId(platform, name) === keyboard.modelId ? keyboard : null;
35	}

apps/web/src/components/device/models/ipad-pro-13-m5-magic-keyboard.glb created

Binary files /dev/null and b/apps/web/src/components/device/models/ipad-pro-13-m5-magic-keyboard.glb differ

apps/web/src/components/device/models/ipad-pro-13-m5.glb created

Binary files /dev/null and b/apps/web/src/components/device/models/ipad-pro-13-m5.glb differ

apps/web/src/components/device/models/iphone-18-pro-max.glb created

Binary files /dev/null and b/apps/web/src/components/device/models/iphone-18-pro-max.glb differ

apps/web/src/components/device/models/iphone-18-pro.glb created

Binary files /dev/null and b/apps/web/src/components/device/models/iphone-18-pro.glb differ

apps/web/src/components/device/models/sources.json created+49

...	@@ -0,0 +1,49 @@
1	{
2	"author": "Apple Inc.",
3	"assetLicense": "No open-source or redistribution license has been established.",
4	"conversion": {
5	"blender": "4.5.9",
6	"gltfTransform": "4.2.1",
7	"changes": [
8	"Extracted device bodies and a separate Magic Keyboard accessory; excluded unrelated parts and presentation duplicates.",
9	"Normalized portrait axes, centered the screen and uniformly scaled its height to 2.2 scene units.",
10	"Replaced the baked marketing screen with a named placeholder for the live framebuffer.",
11	"Resized textures to at most 1024 pixels and encoded them as WebP; retained body geometry."
12	]
13	},
14	"models": [
15	{
16	"id": "iphone-18-pro",
17	"file": "iphone-18-pro.glb",
18	"sourceUrl": "https://www.apple.com/105/media/us/iphone-18-pro/2026/591df885-5ee2-4173-86e6-401021249f7c/ar/iphone-18-pro-e-sim.usdz",
19	"sourceSha256": "0f2dac354fbc6505e7917983de460f2cc68e253f96aabf52e9f0468a4803f317",
20	"bodyNode": "UBGArkKGrAMRRnj",
21	"screenNode": "EHjrGDHNHzgNjtE.001"
22	},
23	{
24	"id": "iphone-18-pro-max",
25	"file": "iphone-18-pro-max.glb",
26	"sourceUrl": "https://www.apple.com/105/media/us/iphone-18-pro/2026/591df885-5ee2-4173-86e6-401021249f7c/ar/iphone-18-pro-e-sim.usdz",
27	"sourceSha256": "0f2dac354fbc6505e7917983de460f2cc68e253f96aabf52e9f0468a4803f317",
28	"bodyNode": "wUOhcMgiBmCgGaw",
29	"screenNode": "EHjrGDHNHzgNjtE"
30	},
31	{
32	"id": "ipad-pro-13-m5",
33	"file": "ipad-pro-13-m5.glb",
34	"sourceUrl": "https://www.apple.com/105/media/us/ipad-pro/2025/adee90db-c01e-430d-b726-fe64c0063f08/ar/ipad-pro-space-black.usdz",
35	"sourceSha256": "48098daf7c49117c81a4e4e1808899b93fad081faeba6c8b65124946b83b5ac6",
36	"bodyNode": "zRrSLDpdYmKeRJQ",
37	"screenNode": "lsDiIbtoSGSmWWZ"
38	},
39	{
40	"id": "ipad-pro-13-m5-magic-keyboard",
41	"file": "ipad-pro-13-m5-magic-keyboard.glb",
42	"sourceUrl": "https://www.apple.com/105/media/us/ipad-pro/2025/adee90db-c01e-430d-b726-fe64c0063f08/ar/ipad-pro-space-black.usdz",
43	"sourceSha256": "48098daf7c49117c81a4e4e1808899b93fad081faeba6c8b65124946b83b5ac6",
44	"bodyNode": "zRrSLDpdYmKeRJQ",
45	"screenNode": "lsDiIbtoSGSmWWZ",
46	"accessoryNode": "PoBqSMmyhhcJsBX"
47	}
48	]
49	}

apps/web/src/components/device/phoneTrackpad.test.ts created+105

...	@@ -0,0 +1,105 @@
1	import { afterEach, expect, it, vi } from "vite-plus/test";
2	import { bindPhoneTrackpad } from "./phoneTrackpad";
3
4	afterEach(() => vi.useRealTimers());
5
6	class Canvas extends EventTarget {
7	getBoundingClientRect() {
8	return { width: 400, height: 800 } as DOMRect;
9	}
10	}
11	const wheel = (ctrlKey = false) =>
12	Object.assign(new Event("wheel", { cancelable: true }), {
13	deltaX: 16,
14	deltaY: -20,
15	deltaMode: 0,
16	ctrlKey,
17	});
18	const gesture = (type: string, scale: number) =>
19	Object.assign(new Event(type, { cancelable: true }), { scale });
20
21	it("consumes scrolling and page zoom only on the bound canvas and releases listeners on detach", () => {
22	const canvas = new Canvas();
23	const navigate = vi.fn(() => true);
24	const listeners = vi.spyOn(canvas, "addEventListener");
25	const binding = bindPhoneTrackpad(canvas, { navigate, endWheel: vi.fn() });
26	const swipe = wheel();
27	canvas.dispatchEvent(swipe);
28	const pinch = wheel(true);
29	canvas.dispatchEvent(pinch);
30	expect(swipe.defaultPrevented).toBe(true);
31	expect(pinch.defaultPrevented).toBe(true);
32	expect(navigate.mock.calls).toEqual([[{ type: "orbit", x: -0.04, y: 0.025 }]]);
33	expect(
34	listeners.mock.calls.every(
35	([, , options]) => typeof options === "object" && options.passive === false,
36	),
37	).toBe(true);
38	binding.dispose();
39	const detached = wheel(true);
40	canvas.dispatchEvent(detached);
41	expect(detached.defaultPrevented).toBe(false);
42	expect(navigate).toHaveBeenCalledOnce();
43	});
44
45	it("consumes Safari pinch without zooming the model and resumes orbit after cancellation", () => {
46	const canvas = new Canvas();
47	const navigate = vi.fn(() => true);
48	const binding = bindPhoneTrackpad(canvas, { navigate, endWheel: vi.fn() });
49	canvas.dispatchEvent(gesture("gesturestart", 1));
50	canvas.dispatchEvent(gesture("gesturechange", 1.2));
51	canvas.dispatchEvent(wheel(true));
52	expect(navigate).not.toHaveBeenCalled();
53	binding.cancel();
54	canvas.dispatchEvent(wheel());
55	expect(navigate).toHaveBeenCalledOnce();
56	binding.dispose();
57	});
58
59	it("normalizes Chrome and Safari pinch, expires wheel sequences and cancels on detach", () => {
60	vi.useFakeTimers();
61	const canvas = new Canvas();
62	const pinch = { begin: vi.fn(() => true), move: vi.fn(), end: vi.fn() };
63	const navigate = vi.fn(() => true);
64	const binding = bindPhoneTrackpad(canvas, { navigate, endWheel: vi.fn() }, pinch);
65	canvas.dispatchEvent(wheel(true));
66	canvas.dispatchEvent(wheel(true));
67	expect(pinch.begin).toHaveBeenCalledOnce();
68	expect(pinch.move.mock.calls).toEqual([[0.2], [0.2]]);
69	vi.advanceTimersByTime(180);
70	expect(pinch.end).toHaveBeenCalledOnce();
71	canvas.dispatchEvent(gesture("gesturestart", 1));
72	canvas.dispatchEvent(gesture("gesturechange", 1.2));
73	expect(pinch.move).toHaveBeenLastCalledWith(Math.log(1.2));
74	canvas.dispatchEvent(wheel(true));
75	expect(pinch.move).toHaveBeenCalledTimes(3);
76	binding.dispose();
77	const count = pinch.move.mock.calls.length;
78	vi.advanceTimersByTime(500);
79	canvas.dispatchEvent(wheel(true));
80	expect(pinch.move).toHaveBeenCalledTimes(count);
81	expect(navigate).not.toHaveBeenCalled();
82	});
83
84	it("keeps a paused wheel orbit active until native release or browser fallback", () => {
85	vi.useFakeTimers();
86	const canvas = new Canvas();
87	const navigate = vi.fn(() => true);
88	const endWheel = vi.fn();
89	const binding = bindPhoneTrackpad(canvas, { navigate, endWheel });
90	canvas.dispatchEvent(wheel());
91	vi.advanceTimersByTime(500);
92	expect(endWheel).not.toHaveBeenCalled();
93	binding.endOrbit();
94	expect(endWheel).toHaveBeenCalledOnce();
95	canvas.dispatchEvent(wheel());
96	vi.advanceTimersByTime(1199);
97	expect(endWheel).toHaveBeenCalledOnce();
98	vi.advanceTimersByTime(1);
99	expect(endWheel).toHaveBeenCalledTimes(2);
100	canvas.dispatchEvent(wheel());
101	canvas.dispatchEvent(Object.assign(wheel(), { momentum: true }));
102	expect(endWheel).toHaveBeenCalledTimes(3);
103	expect(navigate).toHaveBeenCalledTimes(3);
104	binding.dispose();
105	});

apps/web/src/components/device/phoneTrackpad.ts created+137

...	@@ -0,0 +1,137 @@
1	// @effect-diagnostics globalTimers:off - Canvas-local wheel gesture expiry.
2	import {
3	phoneWheelNavigation,
4	type createPhoneInteraction,
5	} from "@t3tools/client-runtime/device/phone-interaction";
6
7	/** Canvas-local, non-passive listeners consume browser zoom. Safari reports cumulative pinch scale instead of Ctrl-wheel. */
8	export function bindPhoneTrackpad(
9	canvas: Pick<
10	HTMLCanvasElement,
11	"addEventListener" \| "removeEventListener" \| "getBoundingClientRect"
12	>,
13	interaction: Pick<ReturnType<typeof createPhoneInteraction>, "navigate" \| "endWheel">,
14	pinch?: {
15	begin: (x: number, y: number) => boolean;
16	move: (logScale: number) => void;
17	end: () => void;
18	},
19	) {
20	let scale: number \| null = null;
21	let wheelActive = false;
22	let timer: ReturnType<typeof setTimeout> \| null = null;
23	let orbitActive = false;
24	let orbitTimer: ReturnType<typeof setTimeout> \| null = null;
25	const endOrbit = () => {
26	if (orbitTimer) clearTimeout(orbitTimer);
27	orbitTimer = null;
28	if (!orbitActive) return;
29	orbitActive = false;
30	interaction.endWheel();
31	};
32	const finish = () => {
33	if (timer) clearTimeout(timer);
34	timer = null;
35	wheelActive = false;
36	scale = null;
37	pinch?.end();
38	};
39	const begin = (event: Event) => {
40	const rect = canvas.getBoundingClientRect();
41	const x = "clientX" in event && typeof event.clientX === "number" ? event.clientX : NaN;
42	const y = "clientY" in event && typeof event.clientY === "number" ? event.clientY : NaN;
43	pinch?.begin((x - rect.left) / rect.width, (y - rect.top) / rect.height);
44	};
45	const consume = (event: Event) => {
46	event.preventDefault();
47	event.stopPropagation();
48	};
49	const wheel = (event: WheelEvent) => {
50	consume(event);
51	if (scale !== null) return;
52	if (event.ctrlKey) {
53	endOrbit();
54	if (!wheelActive) {
55	begin(event);
56	wheelActive = true;
57	}
58	const unit =
59	event.deltaMode === 1
60	? 16
61	: event.deltaMode === 2
62	? canvas.getBoundingClientRect().height
63	: 1;
64	pinch?.move((-event.deltaY * unit) / 100);
65	if (timer) clearTimeout(timer);
66	timer = setTimeout(finish, 180);
67	return;
68	}
69	if (wheelActive) finish();
70	if ("momentum" in event && event.momentum === true) {
71	endOrbit();
72	return;
73	}
74	const rect = canvas.getBoundingClientRect();
75	const navigation = phoneWheelNavigation({
76	width: rect.width,
77	height: rect.height,
78	deltaX: event.deltaX,
79	deltaY: event.deltaY,
80	deltaMode: event.deltaMode,
81	ctrlKey: event.ctrlKey,
82	});
83	if (navigation && interaction.navigate(navigation)) {
84	orbitActive = true;
85	if (orbitTimer) clearTimeout(orbitTimer);
86	// Browsers without a release signal still return to a useful view.
87	orbitTimer = setTimeout(endOrbit, 1200);
88	}
89	};
90	const gestureScale = (event: Event) => {
91	if (
92	!("scale" in event) \|\|
93	typeof event.scale !== "number" \|\|
94	!Number.isFinite(event.scale) \|\|
95	event.scale <= 0
96	)
97	return null;
98	return event.scale;
99	};
100	const start = (event: Event) => {
101	consume(event);
102	endOrbit();
103	finish();
104	begin(event);
105	scale = gestureScale(event) ?? 1;
106	};
107	const change = (event: Event) => {
108	consume(event);
109	const next = gestureScale(event);
110	if (scale === null \|\| next === null) return;
111	pinch?.move(Math.log(next / scale));
112	scale = next;
113	};
114	const end = (event: Event) => {
115	consume(event);
116	finish();
117	};
118	canvas.addEventListener("wheel", wheel, { passive: false });
119	canvas.addEventListener("gesturestart", start, { passive: false });
120	canvas.addEventListener("gesturechange", change, { passive: false });
121	canvas.addEventListener("gestureend", end, { passive: false });
122	return {
123	endOrbit,
124	cancel() {
125	endOrbit();
126	finish();
127	},
128	dispose() {
129	canvas.removeEventListener("wheel", wheel);
130	canvas.removeEventListener("gesturestart", start);
131	canvas.removeEventListener("gesturechange", change);
132	canvas.removeEventListener("gestureend", end);
133	endOrbit();
134	finish();
135	},
136	};
137	}

apps/web/src/components/device/useDeviceControls.test.tsx created+232

...	@@ -0,0 +1,232 @@
1	import { EnvironmentId, type DeviceDetail, type DeviceSummary } from "@t3tools/contracts";
2	import type { DeviceHubAccess } from "@t3tools/client-runtime/device/hub-access";
3	import { act, useEffect } from "react";
4	import { create, type ReactTestRenderer } from "react-test-renderer";
5	import { afterEach, beforeEach, expect, it, vi } from "vite-plus/test";
6	import { useDeviceControls, type DeviceControls } from "./useDeviceControls";
7
8	type Result = { _tag: "Success"; value: DeviceDetail } \| { _tag: "Failure"; cause: unknown };
9	const { read, action, subscribe } = vi.hoisted(() => ({
10	read: vi.fn<() => Promise<Result>>(),
11	action: vi.fn<() => Promise<Result>>(),
12	subscribe:
13	vi.fn<(_target: unknown, onChange: (app: { id: string } \| null) => void) => () => void>(),
14	}));
15	vi.mock("~/state/device", () => ({ deviceEnvironment: { detail: "detail", action: "action" } }));
16	vi.mock("~/state/use-atom-command", () => ({
17	useAtomCommand: (command: string) => (command === "detail" ? read : action),
18	}));
19	vi.mock("~/state/query", () => ({ formatEnvironmentQueryError: () => "Device action failed" }));
20	vi.mock("./deviceHubApi", () => ({ subscribeDeviceForeground: subscribe }));
21
22	const device: DeviceSummary = {
23	hostId: "remote",
24	id: "phone",
25	name: "Phone",
26	platform: "ios",
27	version: "iOS",
28	booted: true,
29	physical: false,
30	};
31	const snapshot = (appearance: "light" \| "dark"): Result => ({
32	_tag: "Success",
33	value: {
34	hostId: "remote",
35	deviceId: "phone",
36	settings: { appearance },
37	foregroundApp: null,
38	readAt: "2026-09-20T00:00:00Z",
39	},
40	});
41	function deferred() {
42	let resolve!: (value: Result) => void;
43	const promise = new Promise<Result>((done) => {
44	resolve = done;
45	});
46	return { promise, resolve };
47	}
48	let renderer: ReactTestRenderer \| undefined;
49	let controls: DeviceControls;
50	function Probe({ visible, access = null }: { visible: boolean; access?: DeviceHubAccess \| null }) {
51	const next = useDeviceControls({
52	environmentId: EnvironmentId.make("environment"),
53	device,
54	access,
55	visible,
56	});
57	useEffect(() => {
58	controls = next;
59	}, [next]);
60	return null;
61	}
62	beforeEach(() => {
63	vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true);
64	read.mockReset();
65	action.mockReset();
66	subscribe.mockReset();
67	subscribe.mockReturnValue(() => {});
68	read.mockResolvedValue(snapshot("light"));
69	});
70	afterEach(async () => {
71	await act(async () => renderer?.unmount());
72	vi.unstubAllGlobals();
73	});
74	async function mount() {
75	await act(async () => {
76	renderer = create(<Probe visible />);
77	});
78	}
79
80	it("serializes rapid actions and retains confirmed settings while pending or after failure", async () => {
81	await mount();
82	const response = deferred();
83	action.mockReturnValueOnce(response.promise);
84	let pending!: Promise<void>;
85	await act(async () => {
86	pending = controls.act({ type: "setAppearance", value: "dark" });
87	await controls.act({ type: "setAppearance", value: "light" });
88	});
89	expect(action).toHaveBeenCalledOnce();
90	expect(controls.disabled).toBe(true);
91	expect(controls.detail?.settings.appearance).toBe("light");
92	await act(async () => {
93	response.resolve(snapshot("dark"));
94	await pending;
95	});
96	expect(controls.detail?.settings.appearance).toBe("dark");
97	action.mockResolvedValueOnce({ _tag: "Failure", cause: "offline" });
98	await act(async () => controls.act({ type: "setAppearance", value: "light" }));
99	expect(controls.detail?.settings.appearance).toBe("dark");
100	expect(controls.error).toBe("Device action failed");
101	expect(controls.disabled).toBe(false);
102	});
103
104	it("serializes host actions across hide/reopen and discards detached results", async () => {
105	await mount();
106	const response = deferred();
107	action.mockReturnValueOnce(response.promise);
108	let pending!: Promise<void>;
109	await act(async () => {
110	pending = controls.act({ type: "setAppearance", value: "dark" });
111	});
112	await act(async () => renderer!.update(<Probe visible={false} />));
113	expect(controls.disabled).toBe(true);
114	await act(async () => renderer!.update(<Probe visible />));
115	await act(async () => controls.act({ type: "setAppearance", value: "light" }));
116	expect(action).toHaveBeenCalledOnce();
117	expect(controls.disabled).toBe(true);
118	const confirmed = deferred();
119	read.mockReturnValueOnce(confirmed.promise);
120	await act(async () => {
121	response.resolve(snapshot("dark"));
122	await response.promise;
123	});
124	expect(controls.detail?.settings.appearance).toBe("light");
125	expect(controls.disabled).toBe(true);
126	await act(async () => {
127	confirmed.resolve(snapshot("dark"));
128	await pending;
129	});
130	expect(controls.detail?.settings.appearance).toBe("dark");
131	expect(controls.disabled).toBe(false);
132	action.mockResolvedValueOnce(snapshot("dark"));
133	await act(async () => controls.act({ type: "setAppearance", value: "dark" }));
134	expect(action).toHaveBeenCalledTimes(2);
135	expect(controls.detail?.settings.appearance).toBe("dark");
136	});
137
138	it("keeps actions disabled when settings cannot be confirmed after a detached command", async () => {
139	await mount();
140	const response = deferred();
141	action.mockReturnValueOnce(response.promise);
142	let pending!: Promise<void>;
143	await act(async () => {
144	pending = controls.act({ type: "setAppearance", value: "dark" });
145	});
146	await act(async () => renderer!.update(<Probe visible={false} />));
147	await act(async () => renderer!.update(<Probe visible />));
148	read.mockResolvedValueOnce({ _tag: "Failure", cause: "offline" });
149	await act(async () => {
150	response.resolve(snapshot("dark"));
151	await pending;
152	});
153	expect(controls.detail).toBeNull();
154	expect(controls.disabled).toBe(true);
155	expect(controls.error).toBe("Device action failed");
156	await act(async () => controls.act({ type: "setAppearance", value: "light" }));
157	expect(action).toHaveBeenCalledOnce();
158	await act(async () => renderer!.update(<Probe visible={false} />));
159	await act(async () => renderer!.update(<Probe visible />));
160	expect(controls.disabled).toBe(false);
161	expect(controls.error).toBeNull();
162	});
163
164	it("disables stale settings after a failed reopen read and recovers on a successful read", async () => {
165	await mount();
166	await act(async () => renderer!.update(<Probe visible={false} />));
167	read.mockResolvedValueOnce({ _tag: "Failure", cause: "offline" });
168	await act(async () => renderer!.update(<Probe visible />));
169	expect(controls.detail).toBeNull();
170	expect(controls.disabled).toBe(true);
171	expect(controls.error).toBe("Device action failed");
172	await act(async () => controls.act({ type: "setAppearance", value: "dark" }));
173	expect(action).not.toHaveBeenCalled();
174	await act(async () => renderer!.update(<Probe visible={false} />));
175	await act(async () => renderer!.update(<Probe visible />));
176	expect(controls.detail?.settings.appearance).toBe("light");
177	expect(controls.disabled).toBe(false);
178	expect(controls.error).toBeNull();
179	});
180
181	it("does not let an older refresh overwrite a newer confirmed action", async () => {
182	await mount();
183	await act(async () => renderer!.update(<Probe visible={false} />));
184	const refresh = deferred();
185	read.mockReturnValueOnce(refresh.promise);
186	await act(async () => renderer!.update(<Probe visible />));
187	action.mockResolvedValueOnce(snapshot("dark"));
188	await act(async () => controls.act({ type: "setAppearance", value: "dark" }));
189	await act(async () => {
190	refresh.resolve(snapshot("light"));
191	await refresh.promise;
192	});
193	expect(controls.detail?.settings.appearance).toBe("dark");
194	});
195
196	it.each(["hidden", "access renewed"] as const)(
197	"uses the detail snapshot until a new foreground event after the subscription is %s",
198	async (restart) => {
199	const access: DeviceHubAccess = {
200	httpBase: "https://remote.test/api/device-hub",
201	wsBase: "wss://remote.test/api/device-hub",
202	query: {},
203	credentials: false,
204	};
205	const detail = snapshot("light");
206	if (detail._tag !== "Success") throw new Error("Expected a successful snapshot");
207	read.mockResolvedValue({
208	...detail,
209	value: { ...detail.value, foregroundApp: { id: "snapshot.app" } },
210	});
211	await act(async () => {
212	renderer = create(<Probe visible access={access} />);
213	});
214	const oldEvent = subscribe.mock.calls[0]![1];
215	await act(async () => oldEvent({ id: "previous.live.app" }));
216	expect(controls.foregroundApp?.id).toBe("previous.live.app");
217	if (restart === "hidden") {
218	await act(async () => renderer!.update(<Probe visible={false} access={access} />));
219	await act(async () => renderer!.update(<Probe visible access={access} />));
220	} else {
221	await act(async () =>
222	renderer!.update(<Probe visible access={{ ...access, query: { ticket: "renewed" } }} />),
223	);
224	}
225	expect(controls.foregroundApp?.id).toBe("snapshot.app");
226	await act(async () => oldEvent({ id: "late.previous.app" }));
227	expect(controls.foregroundApp?.id).toBe("snapshot.app");
228	const newEvent = subscribe.mock.calls[1]![1];
229	await act(async () => newEvent({ id: "current.live.app" }));
230	expect(controls.foregroundApp?.id).toBe("current.live.app");
231	},
232	);

apps/web/src/components/device/useDeviceControls.ts created+130

...	@@ -0,0 +1,130 @@
1	import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess";
2	import type {
3	DeviceActionInput,
4	DeviceDetail,
5	DeviceSummary,
6	EnvironmentId,
7	} from "@t3tools/contracts";
8	import { useEffect, useMemo, useRef, useState } from "react";
9	import { deviceEnvironment } from "~/state/device";
10	import { formatEnvironmentQueryError } from "~/state/query";
11	import { useAtomCommand } from "~/state/use-atom-command";
12	import { subscribeDeviceForeground, type DeviceForegroundInfo } from "./deviceHubApi";
13
14	type ActionBody = DeviceActionInput extends infer A
15	? A extends { readonly type: string }
16	? Omit<A, "hostId" \| "deviceId">
17	: never
18	: never;
19
20	/** One confirmed settings snapshot and serialized actions for the rail and drawer. Mount keyed by device. */
21	export function useDeviceControls(options: {
22	environmentId: EnvironmentId;
23	device: DeviceSummary;
24	access: DeviceHubAccess \| null;
25	visible: boolean;
26	}) {
27	const { environmentId, device, access, visible } = options;
28	const readDetail = useAtomCommand(deviceEnvironment.detail, { reportFailure: false });
29	const runAction = useAtomCommand(deviceEnvironment.action, { reportFailure: false });
30	const [detail, setDetail] = useState<DeviceDetail \| null>(null);
31	const [pending, setPending] = useState(false);
32	const [error, setError] = useState<string \| null>(null);
33	const [foreground, setForeground] = useState<DeviceForegroundInfo \| null \| undefined>();
34	// Hiding invalidates UI results, but does not cancel host commands. Keep them serialized
35	// until settlement rather than allowing a reopened panel to race the previous command.
36	const busy = useRef(false);
37	const generation = useRef(0);
38	const visibleRead = useRef<Parameters<typeof readDetail>[0] \| null>(null);
39	const target = useMemo(
40	() => ({ hostId: device.hostId, deviceId: device.id }),
41	[device.hostId, device.id],
42	);
43
44	useEffect(() => {
45	const revision = ++generation.current;
46	if (!visible) return;
47	const request = { environmentId, input: target };
48	visibleRead.current = request;
49	void readDetail(request).then((result) => {
50	if (generation.current !== revision) return;
51	if (result._tag === "Success") {
52	setDetail(result.value);
53	setError(null);
54	} else {
55	setDetail(null);
56	setError(formatEnvironmentQueryError(result.cause));
57	}
58	});
59	return () => {
60	visibleRead.current = null;
61	generation.current++;
62	};
63	}, [environmentId, readDetail, target, visible]);
64
65	useEffect(() => {
66	if (!access \|\| !visible) return;
67	let active = true;
68	const unsubscribe = subscribeDeviceForeground(
69	{ access, platform: device.platform, deviceId: device.id },
70	(app) => {
71	if (active) setForeground(app);
72	},
73	);
74	return () => {
75	active = false;
76	unsubscribe();
77	setForeground(undefined);
78	};
79	}, [access, device.id, device.platform, visible]);
80
81	const available = detail !== null && visible;
82	const act = async (body: ActionBody) => {
83	if (busy.current \|\| !available) return;
84	busy.current = true;
85	// A settings read started before this action must not overwrite its confirmed result.
86	const revision = ++generation.current;
87	setPending(true);
88	setError(null);
89	return runAction({
90	environmentId,
91	input: { ...target, ...body } as DeviceActionInput,
92	})
93	.then((result) => {
94	if (generation.current !== revision) {
95	const request = visibleRead.current;
96	if (!request) return;
97	// Reopening can read settings before the host command finishes. Confirm them again
98	// after it settles, keeping actions serialized through this refresh.
99	const refreshRevision = ++generation.current;
100	return readDetail(request).then((refreshed) => {
101	if (generation.current !== refreshRevision) return;
102	if (refreshed._tag === "Success") {
103	setDetail(refreshed.value);
104	setError(null);
105	} else {
106	setDetail(null);
107	setError(formatEnvironmentQueryError(refreshed.cause));
108	}
109	});
110	}
111	if (result._tag === "Success") setDetail(result.value);
112	else setError(formatEnvironmentQueryError(result.cause));
113	})
114	.finally(() => {
115	busy.current = false;
116	setPending(false);
117	});
118	};
119
120	return {
121	detail,
122	pending,
123	error,
124	act,
125	disabled: pending \|\| !detail \|\| !visible,
126	foregroundApp: foreground === undefined ? (detail?.foregroundApp ?? null) : foreground,
127	};
128	}
129
130	export type DeviceControls = ReturnType<typeof useDeviceControls>;

packages/client-runtime/package.json+26

...	@@ -266,6 +266,30 @@	...	@@ -266,6 +266,30 @@
266	"./file-preview": {	266	"./file-preview": {
267	"types": "./src/filePreview.ts",	267	"types": "./src/filePreview.ts",
268	"default": "./src/filePreview.ts"	268	"default": "./src/filePreview.ts"
		269	},
		270	"./device/frame": {
		271	"types": "./src/device/frame.ts",
		272	"default": "./src/device/frame.ts"
		273	},
		274	"./device/phone-viewer": {
		275	"types": "./src/device/phoneViewer.ts",
		276	"default": "./src/device/phoneViewer.ts"
		277	},
		278	"./device/phone-interaction": {
		279	"types": "./src/device/phoneInteraction.ts",
		280	"default": "./src/device/phoneInteraction.ts"
		281	},
		282	"./device/shape-profile": {
		283	"types": "./src/device/shapeProfile.ts",
		284	"default": "./src/device/shapeProfile.ts"
		285	},
		286	"./device/screenshot": {
		287	"types": "./src/device/screenshot.ts",
		288	"default": "./src/device/screenshot.ts"
		289	},
		290	"./device/model": {
		291	"types": "./src/device/model.ts",
		292	"default": "./src/device/model.ts"
269	}	293	}
270	},	294	},
271	"scripts": {	295	"scripts": {
...	@@ -280,10 +304,12 @@	...	@@ -280,10 +304,12 @@
280	"micromark-extension-directive": "^4.0.0",	304	"micromark-extension-directive": "^4.0.0",
281	"micromark-util-character": "^2.1.1",	305	"micromark-util-character": "^2.1.1",
282	"remark-parse": "^11.0.0",	306	"remark-parse": "^11.0.0",
		307	"three": "0.180.0",
283	"unified": "^11.0.5"	308	"unified": "^11.0.5"
284	},	309	},
285	"devDependencies": {	310	"devDependencies": {
286	"@effect/vitest": "catalog:",	311	"@effect/vitest": "catalog:",
		312	"@types/three": "0.180.0",
287	"micromark-util-types": "^2.0.2",	313	"micromark-util-types": "^2.0.2",
288	"vite-plus": "catalog:"	314	"vite-plus": "catalog:"
289	}	315	}

packages/client-runtime/src/device/deviceFraming.test.ts created+53

...	@@ -0,0 +1,53 @@
1	import { expect, it } from "vite-plus/test";
2	import { Box3, Vector3 } from "three";
3	import { createDeviceFraming } from "./deviceFraming.ts";
4
5	const box = (x: number, width: number) =>
6	new Box3(new Vector3(x - width / 2, -1, -0.2), new Vector3(x + width / 2, 1, 0.2));
7	it("fits both axes through a moving pivot, freezes for contact, and stops once framing settles", () => {
8	const framing = createDeviceFraming();
9	framing.setBounds(box(0, 1), Math.PI / 8, 0.7, 0);
10	const initialDistance = framing.distance();
11	const next = box(1.5, 3);
12	framing.setBounds(next, Math.PI / 8, 0.7, 10);
13	framing.advance(50);
14	expect(framing.center.x).toBeGreaterThan(0);
15	expect(framing.center.x).toBeLessThan(1.5);
16	for (const x of [next.min.x, next.max.x])
17	expect(
18	Math.abs(x - framing.center.x) /
19	((framing.distance() - next.max.z) * Math.tan(Math.PI / 8) * 0.7),
20	).toBeLessThan(1);
21	framing.hold(true, 50);
22	const center = framing.center.clone(),
23	distance = framing.distance();
24	framing.advance(10000);
25	expect(framing.center.equals(center)).toBe(true);
26	expect(framing.distance()).toBe(distance);
27	expect(framing.needsFrame()).toBe(false);
28	framing.hold(false, 10000);
29	framing.advance(10016);
30	expect(framing.center.x).toBeLessThan(1.5);
31	framing.advance(12000);
32	expect(framing.center.x).toBe(1.5);
33	expect(framing.distance()).toBeGreaterThan(initialDistance);
34	expect(framing.needsFrame()).toBe(false);
35	});
36
37	it("uses elapsed time independently of frame partitions and fits a changed viewport immediately", () => {
38	const run = (step: number) => {
39	const framing = createDeviceFraming();
40	framing.setBounds(box(0, 1), Math.PI / 8, 1, 0);
41	framing.setBounds(box(-1, 2), Math.PI / 8, 1, 0);
42	for (let time = step; time < 400; time += step) framing.advance(time);
43	framing.advance(400);
44	return framing;
45	};
46	const fast = run(8),
47	slow = run(33);
48	expect(fast.center.distanceTo(slow.center)).toBeLessThan(1e-8);
49	expect(fast.distance()).toBeCloseTo(slow.distance(), 8);
50	fast.setBounds(box(-1, 2), Math.PI / 8, 0.4, 400, true);
51	expect(fast.distance()).toBeGreaterThan(slow.distance());
52	expect(fast.needsFrame()).toBe(false);
53	});

packages/client-runtime/src/device/deviceFraming.ts created+82

...	@@ -0,0 +1,82 @@
1	import { Box3, Vector3 } from "three";
2
3	/** Fit the current assembly, with independent framing dynamics so a leaf pivot does not drag it off-screen. */
4	export function createDeviceFraming() {
5	const center = new Vector3();
6	const target = new Vector3();
7	const velocity = new Vector3();
8	const bounds = new Box3();
9	let distance = 1;
10	let targetDistance = 1;
11	let distanceVelocity = 0;
12	let tanX = 1;
13	let tanY = 1;
14	let at = 0;
15	let initialized = false;
16	let moving = false;
17	let held = false;
18	const advance = (now: number, immediate = false) => {
19	if (held) return false;
20	const dt = Math.max(0, (now - at) / 1000);
21	at = now;
22	if (!moving) return false;
23	const omega = 14;
24	const decay = Math.exp(-omega * dt);
25	const error = center.clone().sub(target);
26	const coefficient = velocity.clone().addScaledVector(error, omega);
27	center.copy(error).addScaledVector(coefficient, dt).multiplyScalar(decay).add(target);
28	velocity.addScaledVector(coefficient, -omega * dt).multiplyScalar(decay);
29	const distanceError = distance - targetDistance;
30	const distanceCoefficient = distanceVelocity + omega * distanceError;
31	distance = targetDistance + (distanceError + distanceCoefficient * dt) * decay;
32	distanceVelocity = (distanceVelocity - omega * distanceCoefficient * dt) * decay;
33	moving =
34	center.distanceTo(target) > 0.0005 \|\|
35	velocity.length() > 0.005 \|\|
36	Math.abs(distance - targetDistance) > 0.0005 \|\|
37	Math.abs(distanceVelocity) > 0.005;
38	if (immediate \|\| !moving) {
39	center.copy(target);
40	distance = targetDistance;
41	velocity.set(0, 0, 0);
42	distanceVelocity = 0;
43	moving = false;
44	}
45	return true;
46	};
47	return {
48	center,
49	setBounds(next: Box3, vertical: number, aspect: number, now: number, immediate = false) {
50	if (next.isEmpty()) return;
51	advance(now);
52	bounds.copy(next);
53	tanY = Math.tan(vertical);
54	tanX = tanY * aspect;
55	bounds.getCenter(target);
56	target.z = 0;
57	const size = bounds.getSize(new Vector3());
58	targetDistance = Math.max(1, Math.max(size.x / tanX, size.y / tanY) * 0.565 + bounds.max.z);
59	moving = true;
60	advance(now, immediate \|\| !initialized);
61	initialized = true;
62	},
63	advance,
64	distance() {
65	// Spring lag must never clip the fitted assembly.
66	const clearance = Math.max(
67	Math.abs(bounds.min.x - center.x) / tanX,
68	Math.abs(bounds.max.x - center.x) / tanX,
69	Math.abs(bounds.min.y - center.y) / tanY,
70	Math.abs(bounds.max.y - center.y) / tanY,
71	);
72	return Math.max(distance, clearance * 1.02 + bounds.max.z, 1);
73	},
74	hold(active: boolean, now: number) {
75	held = active;
76	at = now;
77	},
78	needsFrame() {
79	return moving && !held;
80	},
81	};
82	}

packages/client-runtime/src/device/deviceMotion.test.ts created+175

...	@@ -0,0 +1,175 @@
1	import { expect, it, vi } from "vite-plus/test";
2	import { Quaternion, Vector3 } from "three";
3	import { createDeviceMotion, rotationVector } from "./deviceMotion.ts";
4	import { nearestDeviceView } from "./deviceViewSnap.ts";
5
6	const snaps = ["portrait", "landscape_left", "portrait_upside_down", "landscape_right"].map(
7	(orientation, index) => ({
8	orientation,
9	rotation: new Quaternion().setFromAxisAngle(new Vector3(0, 0, 1), (-index * Math.PI) / 2),
10	yawLimit: Math.PI / 3,
11	}),
12	);
13	const rotation = (x: number, y: number, z = 0) =>
14	new Quaternion().setFromAxisAngle(new Vector3(x, y, z).normalize(), Math.hypot(x, y, z));
15
16	it("selects the nearest quarter turn, retains a nearby yaw, and treats quaternion signs identically", () => {
17	const side = new Quaternion().setFromAxisAngle(new Vector3(0, 0, 1), -Math.PI / 2);
18	const released = side.clone().premultiply(rotation(0, 0.4));
19	const nearest = nearestDeviceView(released, snaps)!;
20	expect(nearest.orientation).toBe("landscape_left");
21	expect(nearest.rotation.angleTo(released)).toBeLessThan(1e-6);
22	const negative = released.clone().set(-released.x, -released.y, -released.z, -released.w);
23	expect(nearestDeviceView(negative, snaps)!.rotation.angleTo(nearest.rotation)).toBeLessThan(1e-6);
24	expect(
25	nearestDeviceView(rotation(0, 3), snaps)!.rotation.angleTo(new Quaternion()),
26	).toBeLessThanOrEqual(Math.PI / 3 + 1e-6);
27	});
28
29	it("springs toward a cumulative drag without teleporting and keeps the same target across event partitions", () => {
30	const run = (parts: number) => {
31	const choose = vi.fn((q: Quaternion) => q);
32	const motion = createDeviceMotion({ choose });
33	motion.dragActive(true, 0);
34	for (let part = 0; part < parts; part++) motion.orbit(160 / parts, 80 / parts, 0);
35	expect(motion.rotation.angleTo(new Quaternion())).toBeLessThan(1e-6);
36	for (let time = 8; time <= 400; time += 8) motion.advance(time);
37	expect(motion.rotation.angleTo(new Quaternion())).toBeGreaterThan(0.5);
38	motion.dragActive(false, 400);
39	for (let time = 408; time <= 2400; time += 8) motion.advance(time);
40	expect(choose).toHaveBeenCalledOnce();
41	expect(motion.needsFrame()).toBe(false);
42	return motion.rotation;
43	};
44	expect(run(1).angleTo(run(20))).toBeLessThan(1e-6);
45	});
46
47	it("selects the predicted nearest view once and preserves velocity when interrupted", () => {
48	const choose = vi.fn((q: Quaternion) => nearestDeviceView(q, snaps)!.rotation);
49	const motion = createDeviceMotion({ choose });
50	motion.dragActive(true, 0);
51	motion.orbit(0, 240, 0);
52	motion.orbit(0, 20, 20);
53	motion.advance(40);
54	motion.dragActive(false, 40);
55	const before = motion.rotation.clone();
56	motion.advance(80);
57	expect(motion.rotation.angleTo(before)).toBeGreaterThan(0.01);
58	const interrupted = motion.rotation.clone();
59	motion.dragActive(true, 80);
60	expect(motion.rotation.angleTo(interrupted)).toBeLessThan(1e-6);
61	motion.orbit(30, -30, 80);
62	motion.advance(96);
63	motion.dragActive(false, 100);
64	for (let time = 116; time <= 2500; time += 16) motion.advance(time);
65	expect(choose).toHaveBeenCalledTimes(2);
66	expect(motion.needsFrame()).toBe(false);
67	});
68
69	it("lets a hard flick coast through multiple turns before settling on a visible view", () => {
70	const choose = vi.fn(() => new Quaternion());
71	const motion = createDeviceMotion({ choose });
72	motion.dragActive(true, 0);
73	motion.orbit(0, 80, 16);
74	motion.advance(16);
75	motion.orbit(0, 80, 32);
76	motion.advance(32);
77	motion.dragActive(false, 32);
78
79	let previous = motion.rotation.clone();
80	let travel = 0;
81	let lateSpeed = 0;
82	for (let time = 40; time <= 8500; time += 8) {
83	motion.advance(time);
84	const distance = motion.rotation.angleTo(previous);
85	travel += distance;
86	if (time >= 2500) lateSpeed = Math.max(lateSpeed, distance / 0.008);
87	previous = motion.rotation.clone();
88	}
89	expect(travel).toBeGreaterThan(4 * Math.PI);
90	expect(lateSpeed).toBeLessThan(2.5);
91	expect(choose).toHaveBeenCalledOnce();
92	expect(motion.rotation.angleTo(new Quaternion())).toBeLessThan(1e-6);
93	expect(motion.needsFrame()).toBe(false);
94	});
95
96	it("release uses elapsed time equally at different frame rates and a captured contact freezes projection", () => {
97	const run = (step: number) => {
98	const motion = createDeviceMotion({ choose: () => new Quaternion() });
99	motion.dragActive(true, 0);
100	motion.orbit(160, 80, 0);
101	motion.advance(50);
102	motion.dragActive(false, 50);
103	for (let time = 50; time < 450; time += step) motion.advance(time);
104	motion.advance(450);
105	return motion;
106	};
107	const fast = run(8),
108	slow = run(33),
109	throttled = run(1000);
110	expect(fast.rotation.angleTo(slow.rotation)).toBeLessThan(1e-6);
111	expect(fast.rotation.angleTo(throttled.rotation)).toBeLessThan(1e-6);
112	fast.hold(true, 450);
113	const contact = fast.rotation.clone();
114	fast.orbit(100, 100, 500);
115	fast.advance(10000);
116	expect(fast.rotation.angleTo(contact)).toBeLessThan(1e-6);
117	fast.hold(false, 10000);
118	fast.advance(12000);
119	expect(fast.needsFrame()).toBe(false);
120	});
121
122	it("holds a trackpad orbit through a pause and snaps only when the gesture ends", () => {
123	const choose = vi.fn(() => new Quaternion());
124	const motion = createDeviceMotion({ choose });
125	motion.orbit(NaN, 0, 0);
126	expect(motion.needsFrame()).toBe(false);
127	motion.orbit(200, 100, 0);
128	motion.advance(100);
129	expect(choose).not.toHaveBeenCalled();
130	expect(motion.rotation.angleTo(new Quaternion())).toBeGreaterThan(0);
131	motion.advance(140, true);
132	expect(choose).not.toHaveBeenCalled();
133	const held = motion.rotation.clone();
134	expect(held.angleTo(new Quaternion())).toBeGreaterThan(0.5);
135	expect(motion.needsFrame()).toBe(false);
136	motion.orbit(80, 0, 200);
137	motion.advance(340, true);
138	expect(motion.rotation.angleTo(held)).toBeGreaterThan(0.1);
139	expect(choose).not.toHaveBeenCalled();
140	expect(motion.needsFrame()).toBe(false);
141	motion.dragActive(false, 340);
142	motion.advance(2500);
143	expect(choose).toHaveBeenCalledOnce();
144	expect(motion.rotation.angleTo(new Quaternion())).toBeLessThan(1e-6);
145	expect(motion.needsFrame()).toBe(false);
146	expect(rotationVector(rotation(0, Math.PI)).length()).toBeCloseTo(Math.PI);
147	});
148
149	it("keeps release angular speed bounded even across a back-facing half turn", () => {
150	const motion = createDeviceMotion({ choose: () => rotation(0, Math.PI) });
151	motion.setPose(rotation(0, Math.PI), 0);
152	let previous = motion.rotation.clone();
153	for (let time = 8; time <= 1600; time += 8) {
154	motion.advance(time);
155	expect(motion.rotation.angleTo(previous)).toBeLessThanOrEqual(9 * 0.008 + 1e-5);
156	previous = motion.rotation.clone();
157	}
158	expect(motion.rotation.angleTo(rotation(0, Math.PI))).toBeLessThan(1e-6);
159	expect(motion.needsFrame()).toBe(false);
160	});
161
162	it("a click without dragging resumes the interrupted resting view instead of stranding the device", () => {
163	const choose = vi.fn(() => new Quaternion());
164	const motion = createDeviceMotion({ choose });
165	const rest = rotation(0, 0.8);
166	motion.setPose(rest, 0);
167	motion.advance(50);
168	motion.dragActive(true, 50);
169	motion.advance(60);
170	motion.dragActive(false, 60);
171	motion.advance(2100);
172	expect(motion.rotation.angleTo(rest)).toBeLessThan(1e-6);
173	expect(choose).not.toHaveBeenCalled();
174	expect(motion.needsFrame()).toBe(false);
175	});

packages/client-runtime/src/device/deviceMotion.ts created+279

...	@@ -0,0 +1,279 @@
1	import { Quaternion, Vector3 } from "three";
2
3	/** Shortest rotation vector. q and -q represent the same orientation. */
4	export function rotationVector(rotation: Quaternion) {
5	const q = rotation.clone().normalize();
6	if (q.w < 0) q.set(-q.x, -q.y, -q.z, -q.w);
7	const length = Math.hypot(q.x, q.y, q.z);
8	return length < 1e-8
9	? new Vector3()
10	: new Vector3(q.x, q.y, q.z).multiplyScalar((2 * Math.atan2(length, q.w)) / length);
11	}
12	function fromVector(vector: Vector3) {
13	const angle = vector.length();
14	return angle < 1e-8
15	? new Quaternion()
16	: new Quaternion().setFromAxisAngle(vector.clone().divideScalar(angle), angle);
17	}
18
19	// Recovered from Bitrig 0.25's rotation dynamics: point gain, drag force limit,
20	// velocity limits, release prediction/blend, and the 0.5 / 0.78 release spring.
21	const gain = 0.006;
22	const response = 0.5;
23	const damping = 0.78;
24	const frequency = (2 * Math.PI) / response;
25	const decay = frequency * damping;
26	const stepSeconds = 1 / 120;
27	// Fast releases combine exponential friction with a critically damped correction.
28	const flickThreshold = 4.5;
29	const flickLimit = 22;
30	const spinDecay = 1.1;
31	const settleFrequency = 2;
32
33	/** A moving spring target follows cumulative camera-space gestures; release latches the nearest useful view. */
34	export function createDeviceMotion(options: { choose: (rotation: Quaternion) => Quaternion }) {
35	const rotation = new Quaternion();
36	const target = new Quaternion();
37	const velocity = new Vector3();
38	const gestureVelocity = new Vector3();
39	let spring: { at: number; rotation: Quaternion; velocity: Vector3; steps: number } \| null = null;
40	let spin: { at: number; rotation: Quaternion; velocity: Vector3; correction: Vector3 } \| null =
41	null;
42	let drag: {
43	start: Quaternion;
44	rest: Quaternion;
45	x: number;
46	y: number;
47	error: Vector3;
48	at: number;
49	} \| null = null;
50	let held = false;
51	let interruptedDrag = false;
52	let lastInput = -Infinity;
53	const beginSpring = (next: Quaternion, now: number) => {
54	target.copy(next).normalize();
55	spring = {
56	at: now,
57	rotation: rotation.clone(),
58	velocity: velocity.clone(),
59	steps: 0,
60	};
61	};
62	const release = (now: number) => {
63	if (!drag) return;
64	// A pause before lifting a pointer should not resurrect an old flick.
65	if (now - lastInput > 100) gestureVelocity.set(0, 0, 0);
66	const prediction = target
67	.clone()
68	.premultiply(fromVector(gestureVelocity.clone().multiplyScalar(0.085)));
69	velocity.multiplyScalar(0.2).addScaledVector(gestureVelocity, 0.8).clampLength(0, flickLimit);
70	const moved = drag.x !== 0 \|\| drag.y !== 0;
71	const rest = drag.rest;
72	drag = null;
73	if (moved && velocity.length() >= flickThreshold) {
74	// Choose where the free spin would finish, then correct toward that view
75	// throughout the coast rather than handing off to a spring at low speed.
76	const projected = rotation
77	.clone()
78	.premultiply(fromVector(velocity.clone().multiplyScalar(1 / spinDecay)));
79	target.copy(options.choose(projected)).normalize();
80	spin = {
81	at: now,
82	rotation: rotation.clone(),
83	velocity: velocity.clone(),
84	correction: rotationVector(target.clone().multiply(projected.invert())),
85	};
86	return;
87	}
88	if (moved) beginSpring(options.choose(prediction), now);
89	else beginSpring(rest, now);
90	};
91	const advance = (now: number, reduced = false) => {
92	if (held \|\| !Number.isFinite(now)) return false;
93	if (spin) {
94	const seconds = Math.max(0, (now - spin.at) / 1000);
95	const coast = Math.exp(-spinDecay * seconds);
96	const settle = Math.exp(-settleFrequency * seconds);
97	const correction = fromVector(
98	spin.correction.clone().multiplyScalar(1 - (1 + settleFrequency * seconds) * settle),
99	);
100	rotation
101	.copy(spin.rotation)
102	.premultiply(fromVector(spin.velocity.clone().multiplyScalar((1 - coast) / spinDecay)))
103	.premultiply(correction)
104	.normalize();
105	velocity
106	.copy(spin.velocity)
107	.multiplyScalar(coast)
108	.applyQuaternion(correction)
109	.addScaledVector(spin.correction, settleFrequency ** 2 * seconds * settle);
110	if (reduced \|\| (rotation.angleTo(target) < 0.01 && velocity.length() < 0.08)) {
111	spin = null;
112	rotation.copy(target);
113	velocity.set(0, 0, 0);
114	}
115	return true;
116	}
117	if (drag) {
118	// Follow the target during the gesture. Retain the logarithm's winding so
119	// a long drag cannot reverse its spring force when crossing a half turn.
120	const seconds = Math.min(0.05, Math.max(0, (now - drag.at) / 1000));
121	drag.at = now;
122	const steps = Math.ceil(seconds * 120);
123	const dt = steps ? seconds / steps : 0;
124	for (let step = 0; step < steps; step++) {
125	const error = rotationVector(target.clone().multiply(rotation.clone().invert()));
126	if (error.lengthSq() > 1e-10) {
127	const axis = error.clone().normalize();
128	const turns = Math.round((drag.error.dot(axis) - error.length()) / (2 * Math.PI));
129	error.addScaledVector(axis, turns * 2 * Math.PI);
130	}
131	drag.error.copy(error);
132	const omega = (2 * Math.PI) / 0.68;
133	const acceleration = error
134	.clampLength(0, 1.2)
135	.multiplyScalar(omega * omega)
136	.addScaledVector(velocity, -2 * 1.12 * omega);
137	velocity.addScaledVector(acceleration, dt).clampLength(0, 9);
138	rotation.premultiply(fromVector(velocity.clone().multiplyScalar(dt))).normalize();
139	}
140	if (reduced) {
141	rotation.copy(target);
142	velocity.set(0, 0, 0);
143	}
144	return steps > 0 \|\| reduced;
145	}
146	if (!spring) return false;
147	const seconds = Math.max(0, (now - spring.at) / 1000);
148	const steps = Math.min(240, Math.floor(seconds / stepSeconds));
149	const integrate = (q: Quaternion, speed: Vector3, dt: number) => {
150	const error = rotationVector(target.clone().multiply(q.clone().invert()));
151	const acceleration = error
152	.multiplyScalar(frequency * frequency)
153	.addScaledVector(speed, -2 * decay);
154	speed.addScaledVector(acceleration, dt).clampLength(0, 9);
155	q.premultiply(fromVector(speed.clone().multiplyScalar(dt))).normalize();
156	};
157	// Fixed ticks make release independent of display refresh rate. The final
158	// partial tick interpolates the next tick without committing it. This also
159	// keeps displayed angular speed bounded between tick boundaries.
160	for (; spring.steps < steps; spring.steps++)
161	integrate(spring.rotation, spring.velocity, stepSeconds);
162	rotation.copy(spring.rotation);
163	velocity.copy(spring.velocity);
164	const nextRotation = spring.rotation.clone();
165	const nextVelocity = spring.velocity.clone();
166	integrate(nextRotation, nextVelocity, stepSeconds);
167	const fraction = Math.min(1, (seconds - steps * stepSeconds) / stepSeconds);
168	rotation.slerp(nextRotation, fraction);
169	velocity.lerp(nextVelocity, fraction);
170	if (
171	reduced \|\|
172	steps === 240 \|\|
173	(rotation.angleTo(target) < 0.0005 && velocity.length() < 0.005)
174	) {
175	rotation.copy(target);
176	velocity.set(0, 0, 0);
177	spring = null;
178	}
179	return true;
180	};
181	const beginDrag = (now: number) => {
182	if (held \|\| drag) return;
183	advance(now);
184	drag = {
185	start: rotation.clone(),
186	rest: target.clone(),
187	x: 0,
188	y: 0,
189	error: new Vector3(),
190	at: now,
191	};
192	target.copy(rotation);
193	gestureVelocity.set(0, 0, 0);
194	lastInput = -Infinity;
195	spring = null;
196	spin = null;
197	};
198	return {
199	rotation,
200	setPose(next: Quaternion, now: number, immediate = false) {
201	advance(now);
202	drag = null;
203	spin = null;
204	beginSpring(next, now);
205	if (immediate) {
206	rotation.copy(target);
207	velocity.set(0, 0, 0);
208	spring = null;
209	}
210	},
211	dragActive(active: boolean, now: number) {
212	if (active) {
213	// A pointer takes over at the displayed pose, even if a wheel gesture
214	// was paused with its target still ahead of the spring.
215	if (drag \|\| spin) {
216	advance(now);
217	drag = null;
218	spin = null;
219	}
220	beginDrag(now);
221	} else {
222	advance(now);
223	release(now);
224	}
225	},
226	/** Deltas are CSS pixels, independent of panel size and event partitioning. */
227	orbit(x: number, y: number, now: number) {
228	if (held \|\| ![x, y, now].every(Number.isFinite) \|\| (!x && !y)) return;
229	advance(now);
230	beginDrag(now);
231	if (!drag) return;
232	const previous = target.clone();
233	drag.x += x;
234	drag.y += y;
235	target
236	.copy(fromVector(new Vector3(drag.y * gain, drag.x * gain, 0)))
237	.multiply(drag.start)
238	.normalize();
239	const seconds = (now - lastInput) / 1000;
240	gestureVelocity.copy(rotationVector(target.clone().multiply(previous.invert())));
241	if (seconds > 0 && seconds < 0.1)
242	gestureVelocity.divideScalar(seconds).clampLength(0, flickLimit);
243	else gestureVelocity.set(0, 0, 0);
244	lastInput = now;
245	},
246	/** Captured device touches and hinge edits freeze projection, independently of orbit dragging. */
247	hold(active: boolean, now: number) {
248	if (held === active) return;
249	if (active) {
250	interruptedDrag = drag !== null \|\| spin !== null;
251	drag = null;
252	spin = null;
253	spring = null;
254	velocity.set(0, 0, 0);
255	}
256	held = active;
257	if (!active) {
258	if (interruptedDrag) beginSpring(options.choose(rotation.clone()), now);
259	else if (rotation.angleTo(target) > 0.0005) beginSpring(target, now);
260	interruptedDrag = false;
261	}
262	},
263	advance,
264	needsFrame() {
265	return (
266	!held &&
267	(spring !== null \|\|
268	spin !== null \|\|
269	(drag !== null && (rotation.angleTo(target) > 0.0005 \|\| velocity.length() > 0.005)))
270	);
271	},
272	reset(next: Quaternion, now: number) {
273	advance(now);
274	drag = null;
275	spin = null;
276	beginSpring(next, now);
277	},
278	};
279	}

packages/client-runtime/src/device/deviceViewSnap.ts created+26

...	@@ -0,0 +1,26 @@
1	import { Quaternion, Vector3 } from "three";
2	const y = new Vector3(0, 1, 0);
3
4	/** Each useful view permits a small yaw. Pick the closest member of each family, then the closest family. */
5	export function nearestDeviceView<T extends { rotation: Quaternion; yawLimit: number }>(
6	rotation: Quaternion,
7	snaps: readonly T[],
8	) {
9	let closest: T \| null = null;
10	let distance = Infinity;
11	for (const snap of snaps) {
12	const relative = rotation.clone().multiply(snap.rotation.clone().invert());
13	const turn = 2 * Math.atan2(relative.y, relative.w);
14	const yaw = Math.max(
15	-snap.yawLimit,
16	Math.min(snap.yawLimit, Math.atan2(Math.sin(turn), Math.cos(turn))),
17	);
18	const candidate = snap.rotation.clone().premultiply(new Quaternion().setFromAxisAngle(y, yaw));
19	const nextDistance = candidate.angleTo(rotation);
20	if (nextDistance < distance - 1e-8) {
21	distance = nextDistance;
22	closest = { ...snap, rotation: candidate };
23	}
24	}
25	return closest;
26	}

packages/client-runtime/src/device/frame.test.ts created+32

...	@@ -0,0 +1,32 @@
1	import { expect, it, vi } from "vite-plus/test";
2	import { createCanvasFrameSink } from "./frame.ts";
3
4	it("retains a borrowed frame at its native size before notifying the viewer", () => {
5	const drawImage = vi.fn();
6	const canvas = {
7	width: 300,
8	height: 150,
9	getContext: () => ({ drawImage }),
10	} as unknown as HTMLCanvasElement;
11	const source = {} as CanvasImageSource;
12	const notify = vi.fn(() => {
13	expect(canvas.width).toBe(1170);
14	expect(canvas.height).toBe(2532);
15	expect(drawImage).toHaveBeenLastCalledWith(source, 0, 0, 1170, 2532);
16	});
17	createCanvasFrameSink(canvas, notify).present(source, 1170, 2532);
18	expect(notify).toHaveBeenCalledOnce();
19	});
20
21	it("reports when a canvas cannot present a frame", () => {
22	const canvas = {
23	width: 300,
24	height: 150,
25	getContext: () => null,
26	} as unknown as HTMLCanvasElement;
27	const notify = vi.fn();
28	expect(createCanvasFrameSink(canvas, notify).present({} as CanvasImageSource, 1170, 2532)).toBe(
29	false,
30	);
31	expect(notify).not.toHaveBeenCalled();
32	});

packages/client-runtime/src/device/frame.ts created+24

...	@@ -0,0 +1,24 @@
1	/** A synchronous, borrowed frame. The producer releases its source after present returns. */
2	export interface DeviceFrameSink {
3	readonly present: (source: CanvasImageSource, width: number, height: number) => boolean;
4	}
5
6	/** Retains the latest frame in a canvas; consumers can invalidate textures after each draw. */
7	export function createCanvasFrameSink(
8	canvas: HTMLCanvasElement,
9	onFrame?: (width: number, height: number) => void,
10	): DeviceFrameSink {
11	return {
12	present(source, width, height) {
13	const context = canvas.getContext("2d");
14	if (!context) return false;
15	if (canvas.width !== width \|\| canvas.height !== height) {
16	canvas.width = width;
17	canvas.height = height;
18	}
19	context.drawImage(source, 0, 0, width, height);
20	onFrame?.(width, height);
21	return true;
22	},
23	};
24	}

packages/client-runtime/src/device/model.test.ts created+125

...	@@ -0,0 +1,125 @@
1	import { expect, it, vi } from "vite-plus/test";
2	import { createDeviceModelSlot, resolveDeviceModelId, type DeviceModelSource } from "./model.ts";
3
4	const pro: DeviceModelSource = { id: "iphone-18-pro", url: "/pro.glb" };
5	const max: DeviceModelSource = { id: "iphone-18-pro-max", url: "/max.glb" };
6
7	function fixture() {
8	const pending: {
9	signal: AbortSignal;
10	resolve: (model: { dispose: () => void }) => void;
11	reject: (error: Error) => void;
12	}[] = [];
13	const install = vi.fn();
14	const load = vi.fn(
15	(_source: DeviceModelSource, signal: AbortSignal) =>
16	new Promise<{ dispose: () => void }>((resolve, reject) =>
17	pending.push({ signal, resolve, reject }),
18	),
19	);
20	const slot = createDeviceModelSlot({ load, install });
21	return { slot, pending, install, load };
22	}
23
24	it("matches only exact supported hardware, including iPad size and generation", () => {
25	expect(resolveDeviceModelId("ios", "iPhone 18 Pro")).toBe(pro.id);
26	expect(resolveDeviceModelId("ios", "iPhone 18 Pro Max")).toBe(max.id);
27	expect(resolveDeviceModelId("ios", "iPad Pro 13-inch (M5)")).toBe("ipad-pro-13-m5");
28	for (const name of [
29	"iPhone 17 Pro",
30	"iPhone 18",
31	"iPad Pro 11-inch (M5)",
32	"iPad Pro 13-inch (M4)",
33	"Pixel_10_Pro",
34	]) {
35	expect(resolveDeviceModelId("ios", name)).toBeNull();
36	}
37	expect(resolveDeviceModelId("android", "iPhone 18 Pro")).toBeNull();
38	});
39
40	it("cancels superseded downloads and releases a late decoded model without installing it", async () => {
41	const { slot, pending, install, load } = fixture();
42	slot.set(pro);
43	slot.set({ ...pro });
44	expect(load).toHaveBeenCalledTimes(1);
45	slot.set(max);
46	expect(pending[0]!.signal.aborted).toBe(true);
47	const latest = { dispose: vi.fn() };
48	const stale = { dispose: vi.fn() };
49	pending[1]!.resolve(latest);
50	await Promise.resolve();
51	pending[0]!.resolve(stale);
52	await Promise.resolve();
53	expect(install).toHaveBeenCalledExactlyOnceWith(latest);
54	expect(stale.dispose).toHaveBeenCalledOnce();
55	slot.set(null);
56	expect(install).toHaveBeenLastCalledWith(null);
57	expect(latest.dispose).toHaveBeenCalledOnce();
58	slot.dispose();
59	expect(latest.dispose).toHaveBeenCalledOnce();
60	});
61
62	it("keeps the fallback on request failure and disposes results that finish after teardown", async () => {
63	const { slot, pending, install } = fixture();
64	slot.set(pro);
65	pending[0]!.reject(new Error("offline"));
66	await Promise.resolve();
67	expect(install).not.toHaveBeenCalled();
68	slot.set(max);
69	slot.dispose();
70	expect(pending[1]!.signal.aborted).toBe(true);
71	const late = { dispose: vi.fn() };
72	pending[1]!.resolve(late);
73	await Promise.resolve();
74	expect(late.dispose).toHaveBeenCalledOnce();
75	expect(install).not.toHaveBeenCalled();
76	});
77
78	it("returns the current scene to the fallback before releasing its imported resources", async () => {
79	const { slot, pending, install } = fixture();
80	slot.set(pro);
81	const model = { dispose: vi.fn(() => expect(install).toHaveBeenLastCalledWith(null)) };
82	pending[0]!.resolve(model);
83	await Promise.resolve();
84	slot.dispose();
85	slot.dispose();
86	expect(model.dispose).toHaveBeenCalledOnce();
87	});
88
89	it("retries a failed source while ignoring rejection of a superseded request", async () => {
90	const { slot, pending, install, load } = fixture();
91	slot.set(pro);
92	pending[0]!.reject(new Error("offline"));
93	await Promise.resolve();
94	slot.set({ ...pro });
95	expect(load).toHaveBeenCalledTimes(2);
96	slot.set(max);
97	pending[1]!.reject(new Error("cancelled"));
98	await Promise.resolve();
99	slot.set({ ...max });
100	expect(load).toHaveBeenCalledTimes(3);
101	const recovered = { dispose: vi.fn() };
102	pending[2]!.resolve(recovered);
103	await Promise.resolve();
104	expect(install).toHaveBeenCalledExactlyOnceWith(recovered);
105	slot.dispose();
106	});
107
108	it("releases an invalid model and restores the fallback if scene preparation rejects it", async () => {
109	const { slot, pending, install } = fixture();
110	install.mockImplementationOnce(() => {
111	throw new Error("invalid display");
112	});
113	slot.set(pro);
114	const model = { dispose: vi.fn() };
115	pending[0]!.resolve(model);
116	await Promise.resolve();
117	expect(model.dispose).toHaveBeenCalledOnce();
118	expect(install).toHaveBeenLastCalledWith(null);
119	slot.set({ ...pro });
120	const recovered = { dispose: vi.fn() };
121	pending[1]!.resolve(recovered);
122	await Promise.resolve();
123	expect(install).toHaveBeenLastCalledWith(recovered);
124	slot.dispose();
125	});

packages/client-runtime/src/device/model.ts created+87

...	@@ -0,0 +1,87 @@
1	import type { DevicePlatform } from "@t3tools/contracts";
2
3	export type DeviceModelId = "iphone-18-pro" \| "iphone-18-pro-max" \| "ipad-pro-13-m5";
4
5	export interface DeviceModelSource {
6	readonly id: DeviceModelId;
7	readonly url: string;
8	}
9
10	export interface DeviceAccessorySource {
11	readonly id: "ipad-pro-13-m5-magic-keyboard";
12	readonly modelId: "ipad-pro-13-m5";
13	readonly url: string;
14	}
15
16	export type DeviceAssetSource = DeviceModelSource \| DeviceAccessorySource;
17
18	/** Match actual hardware, never stretch an available model to impersonate another device. */
19	export function resolveDeviceModelId(platform: DevicePlatform, name: string): DeviceModelId \| null {
20	if (platform !== "ios") return null;
21	if (/^iPhone 18 Pro Max$/i.test(name)) return "iphone-18-pro-max";
22	if (/^iPhone 18 Pro$/i.test(name)) return "iphone-18-pro";
23	if (/^iPad Pro 13-inch $M5$$/i.test(name)) return "ipad-pro-13-m5";
24	return null;
25	}
26
27	/** Each request owns its result. A cancelled or superseded parse must still release its resources. */
28	export function createDeviceModelSlot<
29	T extends { dispose: () => void },
30	Source extends DeviceAssetSource = DeviceModelSource,
31	>(options: {
32	load: (source: Source, signal: AbortSignal) => Promise<T>;
33	install: (model: T \| null) => void;
34	onError?: ((cause: unknown) => void) \| undefined;
35	}) {
36	let request: { source: Source; controller: AbortController } \| null = null;
37	let current: T \| null = null;
38	let disposed = false;
39	const clear = () => {
40	if (current) options.install(null);
41	current?.dispose();
42	current = null;
43	};
44	return {
45	set(source: Source \| null) {
46	if (disposed) return;
47	if (source?.id === request?.source.id && source?.url === request?.source.url) return;
48	request?.controller.abort();
49	request = null;
50	clear();
51	if (!source) return;
52	const next = { source, controller: new AbortController() };
53	request = next;
54	void options.load(source, next.controller.signal).then(
55	(model) => {
56	if (disposed \|\| request !== next) {
57	model.dispose();
58	return;
59	}
60	try {
61	options.install(model);
62	current = model;
63	} catch (cause) {
64	options.install(null);
65	model.dispose();
66	request = null;
67	options.onError?.(cause);
68	}
69	},
70	(cause: unknown) => {
71	if (!disposed && request === next) {
72	request = null;
73	options.onError?.(cause);
74	}
75	// The procedural body stays installed on download, decoding or validation failure.
76	},
77	);
78	},
79	dispose() {
80	if (disposed) return;
81	disposed = true;
82	request?.controller.abort();
83	request = null;
84	clear();
85	},
86	};
87	}

packages/client-runtime/src/device/modelScene.test.ts created+121

...	@@ -0,0 +1,121 @@
1	import {
2	BoxGeometry,
3	Group,
4	Mesh,
5	MeshBasicMaterial,
6	PerspectiveCamera,
7	PlaneGeometry,
8	Texture,
9	Vector3,
10	} from "three";
11	import { expect, it, vi } from "vite-plus/test";
12	import { createImportedPhoneScene, disposeDeviceModel } from "./modelScene.ts";
13	import { phoneDisplayLayout } from "./phoneScene.ts";
14
15	function asset() {
16	const root = new Group();
17	const body = new Mesh(new BoxGeometry(1.15, 2.3, 0.1), new MeshBasicMaterial());
18	body.position.z = -0.02;
19	const screen = new Mesh(new PlaneGeometry(1, 2.2), new MeshBasicMaterial());
20	screen.geometry.translate(0, 0, 0.043);
21	screen.name = "device-screen";
22	root.add(body, screen);
23	return { root, screen, body };
24	}
25
26	it.each(["portrait", "landscape_left", "landscape_right", "portrait_upside_down"] as const)(
27	"retains the imported geometry while mapping screen input and framebuffer UVs in %s",
28	(orientation) => {
29	const { root, screen } = asset();
30	const texture = new Texture();
31	const original = screen.material;
32	// Optimization strips UVs from the placeholder material; the viewer generates framebuffer UVs.
33	screen.geometry.deleteAttribute("uv");
34	const phone = createImportedPhoneScene(root, texture, phoneDisplayLayout(null, 1206, 2622));
35	const layout = phoneDisplayLayout({ width: 1206, height: 2622, orientation }, 1206, 2622);
36	phone.setDisplay(texture, layout);
37	phone.root.rotation.set(0.2, -0.3, 0, "YXZ");
38	phone.orientation.rotation.z = layout.rotation;
39	phone.root.updateMatrixWorld(true);
40	const camera = new PerspectiveCamera(32, 1, 0.1, 30);
41	camera.position.z = 6;
42	camera.updateMatrixWorld(true);
43	const point = phone.orientation.localToWorld(new Vector3(-0.25, -0.55, 0.043)).project(camera);
44	const hit = phone.screenPoint((point.x + 1) / 2, (1 - point.y) / 2, camera);
45	const expected =
46	orientation === "landscape_left"
47	? [0.25, 0.25]
48	: orientation === "landscape_right"
49	? [0.75, 0.75]
50	: orientation === "portrait_upside_down"
51	? [0.75, 0.25]
52	: [0.25, 0.75];
53	expect(hit?.x).toBeCloseTo(expected[0]!);
54	expect(hit?.y).toBeCloseTo(expected[1]!);
55	const geometry = screen.geometry;
56	const rotated = phoneDisplayLayout(
57	{ width: 2622, height: 1206, orientation: "landscape_left" },
58	2622,
59	1206,
60	);
61	phone.setDisplay(texture, rotated);
62	expect(screen.geometry).toBe(geometry);
63	const pos = geometry.getAttribute("position");
64	const uv = geometry.getAttribute("uv");
65	for (let i = 0; i < pos.count; i++) {
66	expect(uv.getX(i)).toBeCloseTo(pos.getY(i) / 2.2 + 0.5);
67	expect(uv.getY(i)).toBeCloseTo(0.5 - pos.getX(i));
68	}
69	phone.dispose();
70	expect(screen.material).toBe(original);
71	disposeDeviceModel(root);
72	texture.dispose();
73	},
74	);
75
76	it("rejects touches from the rear and preserves borrowed textures when releasing imported resources", () => {
77	const { root, screen, body } = asset();
78	const texture = new Texture();
79	const borrowed = vi.spyOn(texture, "dispose");
80	const ownedTexture = new Texture();
81	const owned = vi.spyOn(ownedTexture, "dispose");
82	body.material.map = ownedTexture;
83	const geometry = vi.spyOn(body.geometry, "dispose");
84	const phone = createImportedPhoneScene(root, texture, phoneDisplayLayout(null, 1206, 2622));
85	const camera = new PerspectiveCamera(32, 1, 0.1, 30);
86	camera.position.z = 6;
87	phone.root.rotation.y = Math.PI;
88	expect(phone.screenPoint(0.5, 0.5, camera)).toBeNull();
89	phone.dispose();
90	expect(borrowed).not.toHaveBeenCalled();
91	expect(geometry).not.toHaveBeenCalled();
92	expect(screen.material.map).toBeNull();
93	disposeDeviceModel(root);
94	expect(owned).toHaveBeenCalledOnce();
95	expect(geometry).toHaveBeenCalledOnce();
96	expect(borrowed).not.toHaveBeenCalled();
97	texture.dispose();
98	});
99
100	it("rejects missing and incorrectly normalized displays before changing their materials", () => {
101	const { root, screen } = asset();
102	const original = screen.material;
103	const texture = new Texture();
104	screen.geometry.scale(1, 2, 1);
105	expect(() =>
106	createImportedPhoneScene(root, texture, phoneDisplayLayout(null, 1206, 2622)),
107	).toThrow("not normalized");
108	expect(screen.material).toBe(original);
109	screen.geometry.scale(1, 0.5, 1);
110	root.rotation.y = Math.PI;
111	expect(() =>
112	createImportedPhoneScene(root, texture, phoneDisplayLayout(null, 1206, 2622)),
113	).toThrow("not normalized");
114	expect(screen.material).toBe(original);
115	screen.name = "wrong-screen";
116	expect(() =>
117	createImportedPhoneScene(root, texture, phoneDisplayLayout(null, 1206, 2622)),
118	).toThrow("one device-screen");
119	disposeDeviceModel(root);
120	texture.dispose();
121	});

packages/client-runtime/src/device/modelScene.ts created+118

...	@@ -0,0 +1,118 @@
1	// @effect-diagnostics globalFetch:off - This browser-only renderer owns abortable asset requests without an Effect runtime, like the live stream.
2	import { Box3, Group, Mesh, MeshBasicMaterial, Texture, type Material, type Object3D } from "three";
3	import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js";
4	import type { DeviceAssetSource } from "./model.ts";
5	import { createDisplayProjection, updateDisplayUv, type PhoneDisplayLayout } from "./phoneScene.ts";
6
7	/** GLB resources belong to one viewer; the live framebuffer texture belongs to its stream presentation. */
8	export function disposeDeviceModel(root: Object3D) {
9	const geometries = new Set<Mesh["geometry"]>();
10	const materials = new Set<Material>();
11	const textures = new Set<Texture>();
12	root.traverse((object) => {
13	if (!(object instanceof Mesh)) return;
14	geometries.add(object.geometry);
15	for (const material of Array.isArray(object.material) ? object.material : [object.material]) {
16	materials.add(material);
17	for (const value of Object.values(material)) {
18	if (value instanceof Texture) textures.add(value);
19	}
20	}
21	});
22	for (const geometry of geometries) geometry.dispose();
23	for (const material of materials) material.dispose();
24	const images = new Set<ImageBitmap>();
25	for (const texture of textures) {
26	texture.dispose();
27	if (typeof ImageBitmap !== "undefined" && texture.image instanceof ImageBitmap)
28	images.add(texture.image);
29	}
30	for (const image of images) image.close();
31	}
32
33	/** Assets are converted offline: portrait, front +Z, centered display, height 2.2 and planar display UVs. */
34	export function createImportedPhoneScene(
35	asset: Group,
36	texture: Texture,
37	initial: PhoneDisplayLayout,
38	) {
39	const screens: Mesh[] = [];
40	asset.traverse((object) => {
41	if (object instanceof Mesh && object.name === "device-screen") screens.push(object);
42	});
43	const display = screens[0];
44	if (!display \|\| screens.length !== 1)
45	throw new Error("Device model must have one device-screen mesh");
46	asset.updateMatrixWorld(true);
47	const screenBounds = new Box3().setFromObject(display);
48	const screenWidth = screenBounds.max.x - screenBounds.min.x;
49	const screenHeight = screenBounds.max.y - screenBounds.min.y;
50	if (
51	display.matrixWorld.elements.some(
52	(value, index) =>
53	!Number.isFinite(value) \|\| Math.abs(value - (index % 5 === 0 ? 1 : 0)) > 0.001,
54	) \|\|
55	!Number.isFinite(screenWidth) \|\|
56	!Number.isFinite(screenHeight) \|\|
57	!Number.isFinite(screenBounds.min.z) \|\|
58	!Number.isFinite(screenBounds.max.z) \|\|
59	screenWidth <= 0 \|\|
60	Math.abs(screenHeight - 2.2) > 0.001 \|\|
61	Math.abs(screenBounds.min.x + screenBounds.max.x) > 0.001 \|\|
62	Math.abs(screenBounds.min.y + screenBounds.max.y) > 0.001 \|\|
63	screenBounds.max.z - screenBounds.min.z > 0.001
64	)
65	throw new Error("Device model display is not normalized");
66	updateDisplayUv(display.geometry, screenWidth, screenHeight, initial);
67	const originalMaterial = display.material;
68	const screenMaterial = new MeshBasicMaterial({ map: texture, toneMapped: false });
69	display.material = screenMaterial;
70	const root = new Group();
71	const orientation = new Group();
72	orientation.add(asset);
73	root.add(orientation);
74	const bounds = new Box3().setFromObject(asset);
75	let layout = initial;
76	const projection = createDisplayProjection(
77	display,
78	orientation,
79	screenWidth,
80	screenHeight,
81	() => layout,
82	);
83	return {
84	root,
85	orientation,
86	width: bounds.max.x - bounds.min.x,
87	height: bounds.max.y - bounds.min.y,
88	setDisplay(nextTexture: Texture, nextLayout: PhoneDisplayLayout) {
89	layout = nextLayout;
90	screenMaterial.map = nextTexture;
91	updateDisplayUv(display.geometry, screenWidth, screenHeight, layout);
92	},
93	screenPoint: projection,
94	dispose() {
95	// The model slot owns imported resources. This scene owns only its replacement screen material.
96	display.material = originalMaterial;
97	screenMaterial.map = null;
98	screenMaterial.dispose();
99	orientation.remove(asset);
100	},
101	};
102	}
103
104	export async function loadDeviceModel(source: DeviceAssetSource, signal: AbortSignal) {
105	const response = await fetch(source.url, { signal });
106	if (!response.ok) throw new Error(`Device model request failed: ${response.status}`);
107	const data = await response.arrayBuffer();
108	signal.throwIfAborted();
109	const gltf = await new GLTFLoader().parseAsync(
110	data,
111	new URL(".", new URL(source.url, globalThis.location.href)).href,
112	);
113	if (signal.aborted) {
114	disposeDeviceModel(gltf.scene);
115	signal.throwIfAborted();
116	}
117	return { asset: gltf.scene, dispose: () => disposeDeviceModel(gltf.scene) };
118	}

packages/client-runtime/src/device/phoneInteraction.test.ts created+105

...	@@ -0,0 +1,105 @@
1	import { expect, it, vi } from "vite-plus/test";
2	import { createPhoneInteraction, phoneWheelNavigation } from "./phoneInteraction.ts";
3
4	it("keeps a captured screen gesture separate from orbit and ends it once on cancellation", () => {
5	const touch = vi.fn();
6	const orbit = vi.fn();
7	const screenPoint = vi
8	.fn()
9	.mockReturnValueOnce({ x: 0.2, y: 0.3 })
10	.mockReturnValueOnce({ x: 1, y: 0.6 });
11	const input = createPhoneInteraction({ screenPoint, touch, orbit, zoomBy: vi.fn() });
12	expect(input.begin(1, { x: 0.4, y: 0.3 })).toBe(true);
13	expect(input.begin(2, { x: 0.2, y: 0.2 })).toBe(false);
14	input.move(2, { x: 0.8, y: 0.8 });
15	input.move(1, { x: 1.3, y: 0.6 });
16	input.end(2);
17	input.end();
18	input.end(1);
19	expect(touch.mock.calls).toEqual([
20	["begin", { x: 0.2, y: 0.3 }],
21	["move", { x: 1, y: 0.6 }],
22	["end", { x: 1, y: 0.6 }],
23	]);
24	expect(orbit).not.toHaveBeenCalled();
25	expect(screenPoint.mock.calls[1]?.[1]).toBe(true);
26	});
27
28	it("orbits without sending touches when dragged outside the screen or with the orbit modifier", () => {
29	const touch = vi.fn();
30	const orbit = vi.fn();
31	const input = createPhoneInteraction({ screenPoint: () => null, touch, orbit, zoomBy: vi.fn() });
32	input.begin(1, { x: 0.1, y: 0.2 });
33	input.move(1, { x: 0.4, y: 0.5 });
34	input.end();
35	input.begin(2, { x: 0.5, y: 0.5 }, true);
36	input.move(2, { x: 0.6, y: 0.5 });
37	input.end();
38	expect(orbit).toHaveBeenCalledTimes(2);
39	expect(orbit.mock.calls[0]?.[0]).toBeCloseTo(0.3);
40	expect(touch).not.toHaveBeenCalled();
41	});
42
43	it("blocks navigation during either captured gesture, then resumes without sending device touches", () => {
44	const touch = vi.fn();
45	const orbit = vi.fn();
46	const zoomBy = vi.fn();
47	const input = createPhoneInteraction({
48	screenPoint: () => ({ x: 0.5, y: 0.5 }),
49	touch,
50	orbit,
51	zoomBy,
52	});
53	input.begin(1, { x: 0.5, y: 0.5 });
54	expect(input.navigate({ type: "zoom", delta: 0.2 })).toBe(false);
55	expect(input.navigate({ type: "orbit", x: 0.1, y: 0.2 })).toBe(false);
56	input.end();
57	input.begin(2, { x: 0.5, y: 0.5 }, true);
58	expect(input.navigate({ type: "zoom", delta: 0.2 })).toBe(false);
59	input.end();
60	expect(input.navigate({ type: "zoom", delta: 0.2 })).toBe(true);
61	expect(input.navigate({ type: "orbit", x: 0.1, y: 0.2 })).toBe(true);
62	expect(zoomBy.mock.calls).toEqual([[0.2]]);
63	expect(orbit.mock.calls).toEqual([[0.1, 0.2]]);
64	expect(touch.mock.calls.map(([phase]) => phase)).toEqual(["begin", "end"]);
65	});
66
67	it("ends wheel orbit without ending a captured screen touch", () => {
68	const onInteractionActive = vi.fn();
69	const input = createPhoneInteraction({
70	screenPoint: () => ({ x: 0.5, y: 0.5 }),
71	touch: vi.fn(),
72	orbit: vi.fn(),
73	zoomBy: vi.fn(),
74	onInteractionActive,
75	});
76	input.endWheel();
77	expect(onInteractionActive).toHaveBeenCalledWith(false, "orbit");
78	input.begin(1, { x: 0.5, y: 0.5 });
79	onInteractionActive.mockClear();
80	input.endWheel();
81	expect(onInteractionActive).not.toHaveBeenCalled();
82	input.end(1);
83	expect(onInteractionActive).toHaveBeenCalledWith(false, "touch");
84	});
85
86	it("normalizes wheel units, distinguishes pinch, and bounds jumps from coarse scroll wheels", () => {
87	const wheel = { width: 400, height: 800, deltaX: 16, deltaY: 32, deltaMode: 0, ctrlKey: false };
88	expect(phoneWheelNavigation(wheel)).toEqual({ type: "orbit", x: -0.04, y: -0.04 });
89	expect(phoneWheelNavigation({ ...wheel, deltaX: 1, deltaY: 2, deltaMode: 1 })).toEqual(
90	phoneWheelNavigation(wheel),
91	);
92	expect(phoneWheelNavigation({ ...wheel, deltaX: 1, deltaY: 1, deltaMode: 2 })).toEqual({
93	type: "orbit",
94	x: -0.25,
95	y: -0.25,
96	});
97	expect(phoneWheelNavigation({ ...wheel, ctrlKey: true })).toEqual({ type: "zoom", delta: -0.32 });
98	expect(phoneWheelNavigation({ ...wheel, deltaY: -10000, ctrlKey: true })).toEqual({
99	type: "zoom",
100	delta: 1,
101	});
102	expect(phoneWheelNavigation({ ...wheel, width: 0 })).toBeNull();
103	expect(phoneWheelNavigation({ ...wheel, deltaY: NaN })).toBeNull();
104	expect(phoneWheelNavigation({ ...wheel, deltaMode: 9 })).toBeNull();
105	});

packages/client-runtime/src/device/phoneInteraction.ts created+83

...	@@ -0,0 +1,83 @@
1	interface Point {
2	readonly x: number;
3	readonly y: number;
4	}
5
6	export type PhoneNavigation =
7	\| { readonly type: "orbit"; readonly x: number; readonly y: number }
8	\| { readonly type: "zoom"; readonly delta: number };
9
10	/** Browser wheel units vary by device. Navigation uses viewport fractions and logarithmic zoom. */
11	export function phoneWheelNavigation(input: {
12	readonly deltaX: number;
13	readonly deltaY: number;
14	readonly deltaMode: number;
15	readonly ctrlKey: boolean;
16	readonly width: number;
17	readonly height: number;
18	}): PhoneNavigation \| null {
19	const { deltaX, deltaY, deltaMode, ctrlKey, width, height } = input;
20	if (![deltaX, deltaY, width, height].every(Number.isFinite) \|\| width <= 0 \|\| height <= 0)
21	return null;
22	if (deltaMode !== 0 && deltaMode !== 1 && deltaMode !== 2) return null;
23	const unit = deltaMode === 1 ? 16 : deltaMode === 2 ? height : 1;
24	const clamp = (value: number, limit: number) => Math.min(limit, Math.max(-limit, value));
25	if (ctrlKey) return { type: "zoom", delta: clamp(-deltaY * unit * 0.01, 1) };
26	return {
27	type: "orbit",
28	x: clamp((-deltaX * unit) / width, 0.25),
29	y: clamp((-deltaY * unit) / height, 0.25),
30	};
31	}
32
33	/** A single pointer owns either a device gesture or an orbit until released or cancelled. */
34	export function createPhoneInteraction(options: {
35	readonly screenPoint: (point: Point, captured: boolean) => Point \| null;
36	readonly touch: (phase: "begin" \| "move" \| "end", point: Point) => void;
37	readonly orbit: (deltaX: number, deltaY: number) => void;
38	readonly zoomBy: (logDelta: number) => void;
39	readonly onInteractionActive?: (active: boolean, mode: "touch" \| "orbit") => void;
40	}) {
41	let active: { id: number; mode: "touch" \| "orbit"; last: Point; screen: Point \| null } \| null =
42	null;
43	return {
44	navigate(gesture: PhoneNavigation) {
45	// Moving the camera during a captured device touch would change its projected coordinates.
46	if (active) return false;
47	if (gesture.type === "zoom") options.zoomBy(gesture.delta);
48	else options.orbit(gesture.x, gesture.y);
49	return true;
50	},
51	endWheel() {
52	if (!active) options.onInteractionActive?.(false, "orbit");
53	},
54	begin(id: number, point: Point, forceOrbit = false) {
55	if (active) return false;
56	const screen = forceOrbit ? null : options.screenPoint(point, false);
57	active = { id, mode: screen ? "touch" : "orbit", last: point, screen };
58	options.onInteractionActive?.(true, active.mode);
59	if (screen) options.touch("begin", screen);
60	return true;
61	},
62	move(id: number, point: Point) {
63	if (active?.id !== id) return;
64	if (active.mode === "touch") {
65	const screen = options.screenPoint(point, true);
66	if (screen) {
67	active.screen = screen;
68	options.touch("move", screen);
69	}
70	} else {
71	options.orbit(point.x - active.last.x, point.y - active.last.y);
72	}
73	active.last = point;
74	},
75	end(id?: number) {
76	if (!active \|\| (id !== undefined && active.id !== id)) return;
77	const previous = active;
78	active = null;
79	if (previous.mode === "touch" && previous.screen) options.touch("end", previous.screen);
80	options.onInteractionActive?.(false, previous.mode);
81	},
82	};
83	}

packages/client-runtime/src/device/phoneScene.test.ts created+150

...	@@ -0,0 +1,150 @@
1	import { Box3, PerspectiveCamera, Texture, Vector3 } from "three";
2	import { describe, expect, it } from "vite-plus/test";
3	import { createPhoneScene, phoneDisplayLayout } from "./phoneScene.ts";
4	import { IOS_TABLET_SHAPE, IOS_PHONE_SHAPE, ANDROID_PHONE_SHAPE } from "./shapeProfile.ts";
5
6	const orientations = [
7	"portrait",
8	"landscape_left",
9	"landscape_right",
10	"portrait_upside_down",
11	] as const;
12
13	describe("3D phone input", () => {
14	it.each([IOS_TABLET_SHAPE, IOS_PHONE_SHAPE, ANDROID_PHONE_SHAPE])(
15	"preserves input projection for the $id silhouette in every orientation",
16	(profile) => {
17	for (const orientation of orientations) {
18	const texture = new Texture();
19	const aspect = profile.id === "ios-tablet" ? 0.75 : 0.45;
20	const layout = phoneDisplayLayout(
21	{ width: 1200 * aspect, height: 1200, orientation },
22	1200 * aspect,
23	1200,
24	);
25	const phone = createPhoneScene(texture, layout, profile);
26	const camera = new PerspectiveCamera(32, 1, 0.1, 30);
27	camera.position.z = 6;
28	camera.updateMatrixWorld(true);
29	phone.root.rotation.set(0.2, -0.3, 0, "YXZ");
30	phone.orientation.rotation.z = layout.rotation;
31	phone.root.updateMatrixWorld(true);
32	const point = new Vector3((-2.2 * aspect) / 4, -2.2 / 4, 0.043);
33	phone.orientation.localToWorld(point);
34	point.project(camera);
35	const hit = phone.screenPoint((point.x + 1) / 2, (1 - point.y) / 2, camera);
36	const expected =
37	orientation === "landscape_left"
38	? { x: 0.25, y: 0.25 }
39	: orientation === "landscape_right"
40	? { x: 0.75, y: 0.75 }
41	: orientation === "portrait_upside_down"
42	? { x: 0.75, y: 0.25 }
43	: { x: 0.25, y: 0.75 };
44	expect(hit?.x).toBeCloseTo(expected.x);
45	expect(hit?.y).toBeCloseTo(expected.y);
46	phone.dispose();
47	texture.dispose();
48	}
49	},
50	);
51	it.each([0.6, 2.2])("keeps device coordinates accurate at %s camera zoom", (zoom) => {
52	const texture = new Texture();
53	const layout = phoneDisplayLayout(null, 900, 1950);
54	const phone = createPhoneScene(texture, layout);
55	const camera = new PerspectiveCamera(32, 0.65, 0.1, 30);
56	camera.position.z = 6 / zoom;
57	camera.updateMatrixWorld(true);
58	phone.root.rotation.set(0.2, -0.3, 0, "YXZ");
59	phone.root.updateMatrixWorld(true);
60	const point = new Vector3((-2.2 * layout.aspect) / 4, -2.2 / 4, 0.043);
61	phone.orientation.localToWorld(point);
62	point.project(camera);
63	const hit = phone.screenPoint((point.x + 1) / 2, (1 - point.y) / 2, camera);
64	expect(hit?.x).toBeCloseTo(0.25);
65	expect(hit?.y).toBeCloseTo(0.75);
66	phone.dispose();
67	texture.dispose();
68	});
69
70	it.each(orientations)("maps perspective hits to displayed coordinates in %s", (orientation) => {
71	const texture = new Texture();
72	const layout = phoneDisplayLayout({ width: 1170, height: 2532, orientation }, 1170, 2532);
73	const phone = createPhoneScene(texture, layout);
74	const camera = new PerspectiveCamera(32, 0.65, 0.1, 30);
75	camera.position.z = 6;
76	camera.updateMatrixWorld(true);
77	phone.root.rotation.set(0.3, -0.45, 0, "YXZ");
78	phone.orientation.rotation.z = layout.rotation;
79	phone.root.updateMatrixWorld(true);
80	// A point one quarter across and three quarters down the portrait display.
81	const point = new Vector3((-2.2 * layout.aspect) / 4, -2.2 / 4, 0.043);
82	phone.orientation.localToWorld(point);
83	point.project(camera);
84	const hit = phone.screenPoint((point.x + 1) / 2, (1 - point.y) / 2, camera);
85	const expected =
86	orientation === "landscape_left"
87	? { x: 0.25, y: 0.25 }
88	: orientation === "landscape_right"
89	? { x: 0.75, y: 0.75 }
90	: orientation === "portrait_upside_down"
91	? { x: 0.75, y: 0.25 }
92	: { x: 0.25, y: 0.75 };
93	expect(hit?.x).toBeCloseTo(expected.x);
94	expect(hit?.y).toBeCloseTo(expected.y);
95	phone.dispose();
96	texture.dispose();
97	});
98
99	it("rejects new gestures outside the display, but clamps captured drags at its edge", () => {
100	const texture = new Texture();
101	const phone = createPhoneScene(texture, phoneDisplayLayout(null, 900, 1950));
102	const camera = new PerspectiveCamera(32, 1, 0.1, 30);
103	camera.position.z = 5;
104	expect(phone.screenPoint(1, 0.5, camera)).toBeNull();
105	expect(phone.screenPoint(1, 0.5, camera, true)).toEqual({ x: 1, y: 0.5 });
106	phone.root.rotation.y = Math.PI;
107	expect(phone.screenPoint(0.5, 0.5, camera)).toBeNull();
108	phone.dispose();
109	texture.dispose();
110	});
111
112	it.each(["landscape_left", "landscape_right"] as const)(
113	"unrotates an already-landscape framebuffer in %s",
114	(orientation) => {
115	const texture = new Texture();
116	const layout = phoneDisplayLayout({ width: 1950, height: 900, orientation }, 1950, 900);
117	const phone = createPhoneScene(texture, layout);
118	phone.orientation.rotation.z = layout.rotation;
119	const camera = new PerspectiveCamera(32, 1, 0.1, 30);
120	camera.position.z = 5;
121	expect(phone.screenPoint(0.5, 0.5, camera)).toEqual({ x: 0.5, y: 0.5 });
122	phone.dispose();
123	texture.dispose();
124	},
125	);
126	});
127
128	it.each([IOS_PHONE_SHAPE, IOS_TABLET_SHAPE, ANDROID_PHONE_SHAPE])(
129	"attaches the $id rear camera plate to the back and seats its lenses on the housing",
130	(profile) => {
131	const texture = new Texture();
132	const phone = createPhoneScene(texture, phoneDisplayLayout(null, 900, 1950), profile);
133	phone.root.updateMatrixWorld(true);
134	const back = new Box3().setFromObject(phone.root.getObjectByName("device-back")!);
135	const plate = new Box3().setFromObject(phone.root.getObjectByName("camera-plate")!);
136	expect(plate.min.z).toBeLessThan(back.min.z);
137	expect(plate.max.z).toBeGreaterThanOrEqual(back.min.z);
138	const assembly = phone.root.getObjectByName("rear-camera")!;
139	const rings = assembly.children.filter((part) => part.name === "camera-ring");
140	const lenses = assembly.children.filter((part) => part.name === "camera-lens");
141	for (const [index, ring] of rings.entries()) {
142	const ringBounds = new Box3().setFromObject(ring);
143	expect(ringBounds.intersectsBox(plate)).toBe(true);
144	const lensBounds = new Box3().setFromObject(lenses[index]!);
145	expect(Math.abs(lensBounds.max.z - ringBounds.min.z)).toBeLessThan(0.001);
146	}
147	phone.dispose();
148	texture.dispose();
149	},
150	);

packages/client-runtime/src/device/phoneScene.ts created+302

...	@@ -0,0 +1,302 @@
1	import {
2	BoxGeometry,
3	CircleGeometry,
4	CylinderGeometry,
5	ExtrudeGeometry,
6	Float32BufferAttribute,
7	Group,
8	Mesh,
9	MeshBasicMaterial,
10	MeshPhysicalMaterial,
11	MeshStandardMaterial,
12	Plane,
13	Raycaster,
14	Shape,
15	ShapeGeometry,
16	Vector2,
17	Vector3,
18	type Camera,
19	type BufferGeometry,
20	type Texture,
21	} from "three";
22	import type { DeviceScreenSize } from "./stream.ts";
23	import { IOS_PHONE_SHAPE, type DeviceShapeProfile } from "./shapeProfile.ts";
24
25	const SCREEN_HEIGHT = 2.2;
26
27	function roundedPath(width: number, height: number, radius: number, path = new Shape()) {
28	const x = -width / 2;
29	const y = -height / 2;
30	path.moveTo(x + radius, y);
31	path.lineTo(x + width - radius, y);
32	path.quadraticCurveTo(x + width, y, x + width, y + radius);
33	path.lineTo(x + width, y + height - radius);
34	path.quadraticCurveTo(x + width, y + height, x + width - radius, y + height);
35	path.lineTo(x + radius, y + height);
36	path.quadraticCurveTo(x, y + height, x, y + height - radius);
37	path.lineTo(x, y + radius);
38	path.quadraticCurveTo(x, y, x + radius, y);
39	return path;
40	}
41
42	/** Display orientation is independent of orbit. Texture coordinates stay in raw framebuffer space. */
43	export function phoneDisplayLayout(
44	screen: DeviceScreenSize \| null,
45	rawWidth: number,
46	rawHeight: number,
47	) {
48	const width = rawWidth \|\| screen?.width \|\| 900;
49	const height = rawHeight \|\| screen?.height \|\| 1950;
50	const landscape =
51	screen?.orientation === "landscape_left" \|\| screen?.orientation === "landscape_right";
52	const rotation =
53	screen?.orientation === "landscape_left"
54	? -Math.PI / 2
55	: screen?.orientation === "landscape_right"
56	? Math.PI / 2
57	: screen?.orientation === "portrait_upside_down"
58	? Math.PI
59	: 0;
60	return {
61	aspect: Math.min(width, height) / Math.max(width, height),
62	rotation,
63	landscape,
64	rawLandscape: width > height,
65	};
66	}
67
68	/** An original procedural device body. It makes no claim to reproduce a particular hardware model. */
69	export function createPhoneScene(
70	texture: Texture,
71	layout: ReturnType<typeof phoneDisplayLayout>,
72	profile: DeviceShapeProfile = IOS_PHONE_SHAPE,
73	) {
74	const { aspect } = layout;
75	const root = new Group();
76	const orientation = new Group();
77	root.add(orientation);
78	const screenWidth = SCREEN_HEIGHT * aspect;
79	const width = screenWidth + profile.bezel * 2;
80	const height = SCREEN_HEIGHT + profile.bezel * 2;
81	const backZ = 0.01 - profile.depth;
82	const metal = new MeshStandardMaterial({ color: 0xb5bcc7, metalness: 0.88, roughness: 0.27 });
83	const glass = new MeshPhysicalMaterial({
84	color: 0x141820,
85	metalness: 0.15,
86	roughness: 0.2,
87	clearcoat: 1,
88	});
89	const backMaterial = new MeshStandardMaterial({
90	color: profile.backColor,
91	metalness: 0.45,
92	roughness: 0.32,
93	});
94	const lensMaterial = new MeshPhysicalMaterial({
95	color: 0x071326,
96	metalness: 0.6,
97	roughness: 0.12,
98	clearcoat: 1,
99	});
100	const materials = [metal, glass, backMaterial, lensMaterial];
101
102	const body = new Mesh(
103	new ExtrudeGeometry(roundedPath(width, height, profile.bodyRadius), {
104	depth: profile.depth,
105	bevelEnabled: true,
106	bevelSize: 0.012,
107	bevelThickness: 0.012,
108	bevelSegments: 3,
109	steps: 1,
110	curveSegments: 12,
111	}),
112	metal,
113	);
114	body.position.z = 0.025 - profile.depth;
115	orientation.add(body);
116	const face = new Mesh(
117	new ShapeGeometry(roundedPath(width - 0.014, height - 0.014, profile.bodyRadius - 0.01), 16),
118	glass,
119	);
120	face.position.z = 0.04;
121	orientation.add(face);
122	const back = new Mesh(
123	new ShapeGeometry(roundedPath(width - 0.012, height - 0.012, profile.bodyRadius - 0.01), 16),
124	backMaterial,
125	);
126	back.name = "device-back";
127	back.rotation.y = Math.PI;
128	back.position.z = backZ;
129	orientation.add(back);
130
131	const screenGeometry = new ShapeGeometry(
132	roundedPath(screenWidth, SCREEN_HEIGHT, profile.screenRadius),
133	20,
134	);
135	updateDisplayUv(screenGeometry, screenWidth, SCREEN_HEIGHT, layout);
136	const screenMaterial = new MeshBasicMaterial({ map: texture, toneMapped: false });
137	const display = new Mesh(screenGeometry, screenMaterial);
138	display.position.z = 0.043;
139	orientation.add(display);
140
141	for (const { edge, offset, length } of profile.buttons) {
142	const button = new Mesh(
143	new BoxGeometry(
144	edge === "top" ? length : 0.026,
145	edge === "top" ? 0.026 : length,
146	profile.depth * 0.65,
147	),
148	metal,
149	);
150	button.position.set(
151	edge === "top" ? offset : (edge === "left" ? -1 : 1) * (width / 2 + 0.015),
152	edge === "top" ? height / 2 + 0.015 : offset,
153	(0.025 + backZ) / 2,
154	);
155	orientation.add(button);
156	}
157	// Rear components share a surface-relative coordinate system, with outward positive Z.
158	const rearCamera = new Group();
159	rearCamera.name = "rear-camera";
160	rearCamera.position.set(
161	width / 2 - profile.camera.insetX,
162	height / 2 - profile.camera.insetY,
163	back.position.z + 0.001,
164	);
165	rearCamera.rotation.y = Math.PI;
166	orientation.add(rearCamera);
167	const plateFront = 0.025 + 0.007;
168	const ringDepth = 0.025;
169	const cameraPlate = new Mesh(
170	new ExtrudeGeometry(
171	roundedPath(
172	profile.camera.width,
173	profile.camera.height,
174	Math.min(profile.camera.width, profile.camera.height) / 4,
175	),
176	{
177	depth: 0.025,
178	bevelEnabled: true,
179	bevelSize: 0.009,
180	bevelThickness: 0.007,
181	bevelSegments: 2,
182	},
183	),
184	backMaterial,
185	);
186	cameraPlate.name = "camera-plate";
187	rearCamera.add(cameraPlate);
188	for (const [x, y] of profile.camera.lenses) {
189	const radius = profile.camera.lensRadius;
190	const ring = new Mesh(new CylinderGeometry(radius + 0.014, radius + 0.014, 0.025, 32), metal);
191	ring.rotation.x = Math.PI / 2;
192	ring.name = "camera-ring";
193	ring.position.set(x, y, plateFront + ringDepth / 2 - 0.003);
194	rearCamera.add(ring);
195	const lens = new Mesh(new CircleGeometry(radius, 32), lensMaterial);
196	lens.name = "camera-lens";
197	lens.position.set(x, y, ring.position.z + ringDepth / 2 + 0.0005);
198	rearCamera.add(lens);
199	}
200	const flashMaterial = new MeshBasicMaterial({ color: 0xf2ead6 });
201	if (profile.camera.flash) {
202	const flash = new Mesh(new CircleGeometry(0.022, 20), flashMaterial);
203	flash.position.set(profile.camera.flash[0], profile.camera.flash[1], plateFront + 0.0005);
204	rearCamera.add(flash);
205	}
206
207	let activeLayout = layout;
208	const screenPoint = createDisplayProjection(
209	display,
210	orientation,
211	screenWidth,
212	SCREEN_HEIGHT,
213	() => activeLayout,
214	);
215	return {
216	root,
217	orientation,
218	width,
219	height,
220	screenPoint,
221	setDisplay(nextTexture: Texture, nextLayout: PhoneDisplayLayout) {
222	activeLayout = nextLayout;
223	screenMaterial.map = nextTexture;
224	updateDisplayUv(screenGeometry, screenWidth, SCREEN_HEIGHT, activeLayout);
225	},
226	dispose() {
227	root.traverse((object) => {
228	if (object instanceof Mesh) object.geometry.dispose();
229	});
230	for (const material of materials) material.dispose();
231	screenMaterial.dispose();
232	flashMaterial.dispose();
233	},
234	};
235	}
236
237	export type PhoneDisplayLayout = ReturnType<typeof phoneDisplayLayout>;
238
239	/** Canonical portrait geometry maps to raw framebuffer coordinates in every OS orientation. */
240	export function updateDisplayUv(
241	geometry: BufferGeometry,
242	width: number,
243	height: number,
244	layout: PhoneDisplayLayout,
245	) {
246	const position = geometry.getAttribute("position");
247	if (!geometry.hasAttribute("uv")) {
248	geometry.setAttribute(
249	"uv",
250	new Float32BufferAttribute(new Float32Array(position.count * 2), 2),
251	);
252	}
253	const uv = geometry.getAttribute("uv");
254	for (let i = 0; i < position.count; i++) {
255	const u = (position.getX(i) + width / 2) / width;
256	const v = (position.getY(i) + height / 2) / height;
257	uv.setXY(
258	i,
259	layout.rawLandscape ? (layout.rotation > 0 ? 1 - v : v) : u,
260	layout.rawLandscape ? (layout.rotation > 0 ? u : 1 - u) : v,
261	);
262	}
263	uv.needsUpdate = true;
264	}
265
266	/** New touches hit only the front display; captured drags project onto its plane and clamp. */
267	export function createDisplayProjection(
268	display: Mesh,
269	orientation: Group,
270	width: number,
271	height: number,
272	getLayout: () => PhoneDisplayLayout,
273	) {
274	const raycaster = new Raycaster();
275	const pointer = new Vector2();
276	const local = new Vector3();
277	display.geometry.computeBoundingBox();
278	const z = display.position.z + (display.geometry.boundingBox?.max.z ?? 0);
279	const plane = new Plane(new Vector3(0, 0, 1), -z);
280	return (x: number, y: number, camera: Camera, captured = false) => {
281	orientation.updateWorldMatrix(true, true);
282	camera.updateMatrixWorld(true);
283	pointer.set(x * 2 - 1, 1 - y * 2);
284	raycaster.setFromCamera(pointer, camera);
285	if (!captured) {
286	const hit = raycaster.intersectObject(display, false)[0];
287	if (!hit) return null;
288	local.copy(hit.point);
289	orientation.worldToLocal(local);
290	} else {
291	const ray = raycaster.ray.clone().applyMatrix4(orientation.matrixWorld.clone().invert());
292	if (!ray.intersectPlane(plane, local)) return null;
293	}
294	const u = Math.min(1, Math.max(0, (local.x + width / 2) / width));
295	const v = Math.min(1, Math.max(0, (local.y + height / 2) / height));
296	const { rotation } = getLayout();
297	if (rotation === -Math.PI / 2) return { x: v, y: u };
298	if (rotation === Math.PI / 2) return { x: 1 - v, y: 1 - u };
299	if (rotation === Math.PI) return { x: 1 - u, y: v };
300	return { x: u, y: 1 - v };
301	};
302	}

packages/client-runtime/src/device/phoneViewer.test.ts created+350

...	@@ -0,0 +1,350 @@
1	import type { PerspectiveCamera, Scene, Object3D } from "three";
2	import { afterEach, expect, it, vi } from "vite-plus/test";
3
4	const gpu = vi.hoisted(() => ({
5	instances: [] as {
6	blank: boolean;
7	allocations: number;
8	disposed: boolean;
9	size: { width: number; height: number; pixelRatio: number };
10	frames: {
11	scene: Scene;
12	phone: Object3D \| undefined;
13	rotation: Quaternion \| undefined;
14	yaw: number \| undefined;
15	cameraZ: number;
16	}[];
17	}[],
18	}));
19
20	vi.mock("three", async () => {
21	const actual = await vi.importActual<typeof import("three")>("three");
22	return {
23	...actual,
24	WebGLRenderer: class {
25	outputColorSpace = "";
26	state: (typeof gpu.instances)[number] = {
27	blank: true,
28	allocations: 0,
29	disposed: false,
30	size: { width: 0, height: 0, pixelRatio: 1 },
31	frames: [],
32	};
33	constructor() {
34	gpu.instances.push(this.state);
35	}
36	setDrawingBufferSize(width: number, height: number, pixelRatio: number) {
37	this.state.size = { width, height, pixelRatio };
38	this.state.allocations++;
39	this.state.blank = true;
40	}
41	setSize(width: number, height: number) {
42	this.setDrawingBufferSize(width, height, this.state.size.pixelRatio);
43	}
44	setPixelRatio(pixelRatio: number) {
45	this.setDrawingBufferSize(this.state.size.width, this.state.size.height, pixelRatio);
46	}
47	render(scene: Scene, camera: PerspectiveCamera) {
48	const phone = scene.children.find((child) => child.type === "Group");
49	this.state.frames.push({
50	scene,
51	phone,
52	rotation: phone?.quaternion.clone(),
53	yaw: phone?.rotation.y,
54	cameraZ: camera.position.z,
55	});
56	this.state.blank = false;
57	}
58	dispose() {
59	this.state.disposed = true;
60	}
61	forceContextLoss() {}
62	},
63	};
64	});
65
66	const models = vi.hoisted(() => ({
67	pending: [] as {
68	signal: AbortSignal;
69	resolve: (model: { asset: import("three").Group; dispose: () => void }) => void;
70	}[],
71	}));
72	vi.mock("./modelScene.ts", async () => {
73	const actual = await vi.importActual<typeof import("./modelScene.ts")>("./modelScene.ts");
74	return {
75	...actual,
76	loadDeviceModel: (_source: unknown, signal: AbortSignal) =>
77	new Promise((resolve) => models.pending.push({ signal, resolve })),
78	};
79	});
80
81	import { BoxGeometry, Group, Mesh, MeshBasicMaterial, PlaneGeometry, Quaternion } from "three";
82	import { disposeDeviceModel } from "./modelScene.ts";
83	import { createPhoneViewer } from "./phoneViewer.ts";
84	import { IOS_TABLET_SHAPE } from "./shapeProfile.ts";
85
86	afterEach(() => {
87	vi.restoreAllMocks();
88	vi.unstubAllGlobals();
89	gpu.instances.length = 0;
90	models.pending.length = 0;
91	});
92
93	function fixture() {
94	const pending = new Map<number, FrameRequestCallback>();
95	let id = 0;
96	let now = 0;
97	vi.spyOn(performance, "now").mockImplementation(() => now);
98	vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => {
99	pending.set(++id, callback);
100	return id;
101	});
102	vi.stubGlobal("cancelAnimationFrame", (id: number) => pending.delete(id));
103	const canvas = Object.assign(new EventTarget(), { width: 0, height: 0 }) as HTMLCanvasElement;
104	const source = { width: 1206, height: 2622 } as HTMLCanvasElement;
105	const onUnavailable = vi.fn();
106	const onFramingAspect = vi.fn();
107	const viewer = createPhoneViewer({ canvas, source, onUnavailable, onFramingAspect });
108	const draw = (time = now) => {
109	now = time;
110	const callbacks = [...pending.values()];
111	pending.clear();
112	callbacks.forEach((callback) => callback(0));
113	};
114	viewer.resize(400, 700, 2);
115	draw();
116	return {
117	viewer,
118	draw,
119	pending,
120	source,
121	onUnavailable,
122	onFramingAspect,
123	state: gpu.instances[0]!,
124	};
125	}
126
127	it("retains the drawn phone until resize and redraw commit together, without replacing scene or pose", () => {
128	const { viewer, draw, pending, source, state, onUnavailable } = fixture();
129	viewer.orbit(0.08, 0.04);
130	draw();
131	const previous = state.frames.at(-1)!;
132	const allocations = state.allocations;
133	viewer.resize(500, 700, 2);
134	viewer.resize(450, 700, 2);
135	viewer.frameUpdated();
136	// ResizeObserver runs after rAF. A cleared buffer here would reach the browser's next paint.
137	expect(state.blank).toBe(false);
138	expect(state.allocations).toBe(allocations);
139	expect(pending.size).toBe(1);
140	draw();
141	expect(state.size).toEqual({ width: 450, height: 700, pixelRatio: 2 });
142	expect(state.blank).toBe(false);
143	expect(state.allocations).toBe(allocations + 1);
144	expect(state.frames.at(-1)?.scene).toBe(previous.scene);
145	expect(state.frames.at(-1)?.phone).toBe(previous.phone);
146	expect(state.frames.at(-1)?.yaw).toBe(previous.yaw);
147	viewer.resize(400, 700, 2);
148	draw();
149	expect(state.frames.at(-1)?.cameraZ).toBeCloseTo(previous.cameraZ);
150	expect(source).toMatchObject({ width: 1206, height: 2622 });
151	expect(gpu.instances).toHaveLength(1);
152	expect(onUnavailable).not.toHaveBeenCalled();
153	viewer.dispose();
154	});
155
156	it("ignores redundant and invalid sizes and combines a DPR change into one allocation and draw", () => {
157	const { viewer, draw, pending, state } = fixture();
158	const allocations = state.allocations;
159	viewer.resize(400, 700, 2);
160	viewer.resize(400, 700, 5);
161	viewer.resize(0, 700, 2);
162	viewer.resize(NaN, 700, 2);
163	expect(pending.size).toBe(0);
164	viewer.resize(400, 700, 1);
165	expect(state.blank).toBe(false);
166	draw();
167	expect(state.size.pixelRatio).toBe(1);
168	expect(state.allocations).toBe(allocations + 1);
169	viewer.resize(450, 700, 1);
170	viewer.dispose();
171	draw();
172	expect(state.allocations).toBe(allocations + 1);
173	expect(state.disposed).toBe(true);
174	});
175
176	it("changes device shape without replacing the renderer, decoded source or pose", () => {
177	const { viewer, draw, source, state } = fixture();
178	viewer.orbit(0.1, 0.05);
179	draw();
180	const previous = state.frames.at(-1)!;
181	const allocations = state.allocations;
182	viewer.setScreen(null, IOS_TABLET_SHAPE);
183	draw();
184	expect(state.frames.at(-1)?.phone).not.toBe(previous.phone);
185	expect(state.frames.at(-1)?.yaw).toBe(previous.yaw);
186	expect(state.frames.at(-1)?.scene).toBe(previous.scene);
187	expect(state.allocations).toBe(allocations);
188	expect(source.width).toBe(1206);
189	expect(gpu.instances).toHaveLength(1);
190	const tablet = state.frames.at(-1)?.phone;
191	viewer.frameUpdated();
192	draw();
193	expect(state.frames.at(-1)?.phone).toBe(tablet);
194	viewer.dispose();
195	});
196
197	it("retains the loaded model and pose through rotation and framebuffer resolution changes, then releases it once", async () => {
198	const { viewer, draw, source, state } = fixture();
199	viewer.orbit(0.08, 0.04);
200	draw();
201	const yaw = state.frames.at(-1)!.yaw;
202	viewer.setModel({ id: "iphone-18-pro", url: "/pro.glb" });
203	const asset = new Group();
204	const body = new Mesh(new BoxGeometry(1.15, 2.3, 0.1), new MeshBasicMaterial());
205	body.position.z = -0.02;
206	const display = new Mesh(new PlaneGeometry(1, 2.2), new MeshBasicMaterial());
207	display.geometry.translate(0, 0, 0.043);
208	display.name = "device-screen";
209	asset.add(body, display);
210	const release = vi.spyOn(body.geometry, "dispose");
211	const dispose = vi.fn(() => disposeDeviceModel(asset));
212	models.pending[0]!.resolve({ asset, dispose });
213	await Promise.resolve();
214	draw();
215	const loaded = state.frames.at(-1)!.phone;
216	expect(loaded?.getObjectByName("device-screen")).toBe(display);
217	expect(state.frames.at(-1)!.yaw).toBe(yaw);
218	source.width = 2622;
219	source.height = 1206;
220	viewer.setScreen({ width: 2622, height: 1206, orientation: "landscape_left" });
221	viewer.frameUpdated();
222	viewer.resize(700, 400, 1);
223	draw();
224	expect(state.frames.at(-1)!.phone).toBe(loaded);
225	expect(state.frames.at(-1)!.yaw).toBe(yaw);
226	expect(display.material.map?.image).toBe(source);
227	expect(release).not.toHaveBeenCalled();
228	expect(gpu.instances).toHaveLength(1);
229	viewer.dispose();
230	viewer.dispose();
231	expect(dispose).toHaveBeenCalledOnce();
232	expect(release).toHaveBeenCalledOnce();
233	});
234
235	it("attaches and detaches an accessory without replacing the device, stream texture, renderer or pose", async () => {
236	const { viewer, draw, source, state, onFramingAspect } = fixture();
237	viewer.setModel({ id: "ipad-pro-13-m5", url: "/ipad.glb" });
238	const asset = new Group();
239	const body = new Mesh(new BoxGeometry(1.8, 2.3, 0.1), new MeshBasicMaterial());
240	const display = new Mesh(new PlaneGeometry(1.7, 2.2), new MeshBasicMaterial());
241	display.geometry.translate(0, 0, 0.043);
242	display.name = "device-screen";
243	asset.add(body, display);
244	const releaseBody = vi.fn(() => disposeDeviceModel(asset));
245	models.pending[0]!.resolve({ asset, dispose: releaseBody });
246	await Promise.resolve();
247	viewer.orbit(0.1, 0.05);
248	draw();
249	const before = state.frames.at(-1)!;
250	const bareAspect = onFramingAspect.mock.lastCall![0];
251	const keyboard = new Group();
252	const deck = new Mesh(new BoxGeometry(2, 1, 1), new MeshBasicMaterial());
253	deck.position.set(1, 0, 0.5);
254	keyboard.add(deck);
255	const releaseKeyboard = vi.fn(() => disposeDeviceModel(keyboard));
256	viewer.setAccessory({
257	id: "ipad-pro-13-m5-magic-keyboard",
258	modelId: "ipad-pro-13-m5",
259	url: "/keyboard.glb",
260	});
261	models.pending[1]!.resolve({ asset: keyboard, dispose: releaseKeyboard });
262	await Promise.resolve();
263	draw();
264	expect(state.frames.at(-1)!.phone).toBe(before.phone);
265	expect(keyboard.parent).toBe(asset.parent);
266	expect(state.frames.at(-1)!.yaw).toBe(before.yaw);
267	expect(display.material.map?.image).toBe(source);
268	expect(state.frames.at(-1)!.cameraZ).toBeGreaterThan(before.cameraZ);
269	expect(onFramingAspect.mock.lastCall![0]).toBeGreaterThan(bareAspect);
270	const calls = onFramingAspect.mock.calls.length;
271	viewer.orbit(0.1, 0.05);
272	viewer.frameUpdated();
273	draw();
274	expect(onFramingAspect).toHaveBeenCalledTimes(calls);
275	viewer.setAccessory(null);
276	draw();
277	expect(keyboard.parent).toBeNull();
278	expect(onFramingAspect.mock.lastCall![0]).toBeCloseTo(bareAspect);
279	expect(releaseKeyboard).toHaveBeenCalledOnce();
280	expect(releaseBody).not.toHaveBeenCalled();
281	expect(state.frames.at(-1)!.phone).toBe(before.phone);
282	expect(state.frames.at(-1)!.cameraZ).toBeCloseTo(before.cameraZ);
283	expect(gpu.instances).toHaveLength(1);
284	viewer.dispose();
285	expect(releaseBody).toHaveBeenCalledOnce();
286	expect(releaseKeyboard).toHaveBeenCalledOnce();
287	});
288
289	it("rejects incompatible accessories and releases an accessory that finishes after detaching", async () => {
290	const { viewer } = fixture();
291	const keyboard = {
292	id: "ipad-pro-13-m5-magic-keyboard",
293	modelId: "ipad-pro-13-m5",
294	url: "/keyboard.glb",
295	} as const;
296	viewer.setAccessory(keyboard);
297	expect(models.pending).toHaveLength(0);
298	viewer.setModel({ id: "ipad-pro-13-m5", url: "/ipad.glb" });
299	viewer.setAccessory(keyboard);
300	viewer.setAccessory(null);
301	expect(models.pending[1]!.signal.aborted).toBe(true);
302	const dispose = vi.fn();
303	const asset = new Group();
304	models.pending[1]!.resolve({ asset, dispose });
305	await Promise.resolve();
306	expect(asset.parent).toBeNull();
307	expect(dispose).toHaveBeenCalledOnce();
308	viewer.dispose();
309	});
310
311	it("springs an ordinary device back toward its screen, freezes captured input, and stops rendering at rest", () => {
312	const { viewer, draw, pending, state } = fixture();
313	const initialYaw = state.frames.at(-1)!.yaw!;
314	viewer.setInteractionActive(true, "orbit");
315	viewer.orbit(0.8, 0.4);
316	draw(80);
317	const dragged = state.frames.at(-1)!;
318	expect(Math.abs(dragged.yaw! - initialYaw)).toBeGreaterThan(0.05);
319	viewer.setInteractionActive(false, "orbit");
320	draw(160);
321	viewer.setInteractionActive(true, "touch");
322	draw(180);
323	const captured = state.frames.at(-1)!;
324	viewer.orbit(1, 1);
325	viewer.frameUpdated();
326	draw(1000);
327	expect(state.frames.at(-1)!.yaw).toBe(captured.yaw);
328	expect(state.frames.at(-1)!.cameraZ).toBe(captured.cameraZ);
329	expect(pending.size).toBe(0);
330	viewer.setInteractionActive(false, "touch");
331	for (let time = 1016; time <= 4000 && pending.size; time += 16) draw(time);
332	expect(pending.size).toBe(0);
333	expect(Math.abs(state.frames.at(-1)!.yaw!)).toBeLessThanOrEqual(Math.PI / 3 + 1e-6);
334	expect(state.frames.at(-1)!.phone).toBe(dragged.phone);
335	viewer.dispose();
336	});
337
338	it("resets the device to a square front view after orbiting", () => {
339	const { viewer, draw, pending, state } = fixture();
340	const front = state.frames.at(-1)!.rotation!;
341	expect(front.angleTo(new Quaternion())).toBeLessThan(1e-6);
342	viewer.orbit(0.8, 0.4);
343	draw(80);
344	expect(state.frames.at(-1)!.rotation!.angleTo(front)).toBeGreaterThan(0.05);
345	viewer.resetPose();
346	for (let time = 96; time <= 3000 && pending.size; time += 16) draw(time);
347	expect(pending.size).toBe(0);
348	expect(state.frames.at(-1)!.rotation!.angleTo(new Quaternion())).toBeLessThan(1e-6);
349	viewer.dispose();
350	});

packages/client-runtime/src/device/phoneViewer.ts created+332

...	@@ -0,0 +1,332 @@
1	import {
2	AmbientLight,
3	Box3,
4	CanvasTexture,
5	DirectionalLight,
6	LinearFilter,
7	Matrix4,
8	PerspectiveCamera,
9	Quaternion,
10	Scene,
11	SRGBColorSpace,
12	Vector3,
13	WebGLRenderer,
14	} from "three";
15	import {
16	createDeviceModelSlot,
17	type DeviceModelSource,
18	type DeviceAccessorySource,
19	} from "./model.ts";
20	import { createImportedPhoneScene, loadDeviceModel } from "./modelScene.ts";
21	import { createPhoneScene, phoneDisplayLayout } from "./phoneScene.ts";
22	import { createRenderScheduler } from "./renderScheduler.ts";
23	import { createDeviceMotion } from "./deviceMotion.ts";
24	import { createDeviceFraming } from "./deviceFraming.ts";
25	import { nearestDeviceView } from "./deviceViewSnap.ts";
26	import type { DeviceScreenSize } from "./stream.ts";
27	import { IOS_PHONE_SHAPE, type DeviceShapeProfile } from "./shapeProfile.ts";
28
29	export interface PhoneViewer {
30	readonly setModel: (source: DeviceModelSource \| null) => void;
31	readonly setAccessory: (source: DeviceAccessorySource \| null) => void;
32	readonly frameUpdated: () => void;
33	readonly setScreen: (screen: DeviceScreenSize \| null, profile?: DeviceShapeProfile) => void;
34	readonly resize: (width: number, height: number, pixelRatio: number) => void;
35	readonly screenPoint: (
36	x: number,
37	y: number,
38	captured?: boolean,
39	) => { x: number; y: number } \| null;
40	readonly orbit: (deltaX: number, deltaY: number) => void;
41	readonly setInteractionActive: (active: boolean, mode: "touch" \| "orbit") => void;
42	readonly resetPose: () => void;
43	readonly dispose: () => void;
44	}
45
46	/** Owns only presentation resources. The caller retains the decoded canvas and the stream connection. */
47	export function createPhoneViewer(options: {
48	readonly canvas: HTMLCanvasElement;
49	readonly source: HTMLCanvasElement;
50	readonly onUnavailable: () => void;
51	readonly onModelError?: (cause: unknown) => void;
52	readonly onFramingAspect?: (aspect: number) => void;
53	readonly profile?: DeviceShapeProfile;
54	readonly model?: DeviceModelSource \| null;
55	readonly accessory?: DeviceAccessorySource \| null;
56	}): PhoneViewer {
57	const renderer = new WebGLRenderer({
58	canvas: options.canvas,
59	alpha: true,
60	antialias: true,
61	powerPreference: "low-power",
62	});
63	renderer.outputColorSpace = SRGBColorSpace;
64	const makeTexture = () => {
65	const next = new CanvasTexture(options.source);
66	next.colorSpace = SRGBColorSpace;
67	next.minFilter = LinearFilter;
68	next.magFilter = LinearFilter;
69	next.generateMipmaps = false;
70	return next;
71	};
72	let texture = makeTexture();
73	let textureWidth = options.source.width;
74	let textureHeight = options.source.height;
75	const scene = new Scene();
76	const camera = new PerspectiveCamera(32, 1, 0.1, 30);
77	camera.position.z = 5.5;
78	const ambient = new AmbientLight(0xffffff, 2.4);
79	const key = new DirectionalLight(0xe4edff, 5);
80	key.position.set(-3, 4, 5);
81	const rim = new DirectionalLight(0xffffff, 4);
82	rim.position.set(3, 1, -3);
83	const fill = new DirectionalLight(0x9facd4, 2);
84	fill.position.set(-2, -2, -4);
85	scene.add(ambient, key, rim, fill);
86
87	let screen: DeviceScreenSize \| null = null;
88	let layout = phoneDisplayLayout(screen, options.source.width, options.source.height);
89	let profile = options.profile ?? IOS_PHONE_SHAPE;
90	let imported: Awaited<ReturnType<typeof loadDeviceModel>> \| null = null;
91	let modelSource = options.model ?? null;
92	let accessory: Awaited<ReturnType<typeof loadDeviceModel>> \| null = null;
93	let accessoryBounds: Box3 \| null = null;
94	let phone = createPhoneScene(texture, layout, profile);
95	scene.add(phone.root);
96	let disposed = false;
97	const rest = new Quaternion();
98	const motion = createDeviceMotion({
99	choose: (rotation) =>
100	nearestDeviceView(rotation, [{ rotation: new Quaternion(), yawLimit: Math.PI / 3 }])!
101	.rotation,
102	});
103	motion.setPose(rest, performance.now(), true);
104	const framing = createDeviceFraming();
105	const reducedMotion = () =>
106	globalThis.matchMedia?.("(prefers-reduced-motion: reduce)").matches ?? false;
107	let viewport = { width: 0, height: 0, pixelRatio: 1 };
108	let drawingBuffer = { width: 0, height: 0, pixelRatio: 0 };
109
110	let framingAspect: number \| null = null;
111	const fit = (immediate = false) => {
112	if (!viewport.width \|\| !viewport.height) return;
113	camera.aspect = viewport.width / viewport.height;
114	const bounds = new Box3(
115	new Vector3(-phone.width / 2, -phone.height / 2, 0),
116	new Vector3(phone.width / 2, phone.height / 2, 0),
117	);
118	if (imported && accessoryBounds) bounds.union(accessoryBounds);
119	bounds.applyMatrix4(new Matrix4().makeRotationZ(layout.rotation));
120	const size = bounds.getSize(new Vector3());
121	const aspect = size.x / size.y;
122	if (aspect !== framingAspect) {
123	framingAspect = aspect;
124	options.onFramingAspect?.(aspect);
125	}
126	phone.root.updateMatrixWorld(true);
127	framing.setBounds(
128	new Box3().setFromObject(phone.root),
129	(camera.fov * Math.PI) / 360,
130	camera.aspect,
131	performance.now(),
132	immediate,
133	);
134	applyCamera();
135	};
136	const applyCamera = () => {
137	camera.position.set(framing.center.x, framing.center.y, framing.distance());
138	camera.lookAt(framing.center.x, framing.center.y, 0);
139	camera.updateProjectionMatrix();
140	};
141	const applyPose = () => {
142	phone.root.quaternion.copy(motion.rotation);
143	phone.orientation.rotation.z = layout.rotation;
144	};
145	const scheduler = createRenderScheduler(() => {
146	if (disposed \|\| !viewport.width \|\| !viewport.height) return;
147	try {
148	if (
149	drawingBuffer.width !== viewport.width \|\|
150	drawingBuffer.height !== viewport.height \|\|
151	drawingBuffer.pixelRatio !== viewport.pixelRatio
152	) {
153	// Canvas allocation clears the previous image. Commit it with the redraw,
154	// rather than exposing an empty buffer between ResizeObserver and the next frame.
155	renderer.setDrawingBufferSize(viewport.width, viewport.height, viewport.pixelRatio);
156	drawingBuffer = viewport;
157	}
158	const now = performance.now();
159	if (motion.advance(now, reducedMotion())) {
160	applyPose();
161	fit(reducedMotion());
162	}
163	framing.advance(now, reducedMotion());
164	applyCamera();
165	renderer.render(scene, camera);
166	if (motion.needsFrame() \|\| framing.needsFrame()) scheduler.invalidate();
167	} catch {
168	options.onUnavailable();
169	}
170	});
171	const updateLayout = (nextProfile = profile) => {
172	const next = phoneDisplayLayout(screen, options.source.width, options.source.height);
173	const resized =
174	textureWidth !== options.source.width \|\| textureHeight !== options.source.height;
175	if (
176	resized \|\|
177	nextProfile !== profile \|\|
178	next.aspect !== layout.aspect \|\|
179	next.rawLandscape !== layout.rawLandscape \|\|
180	next.rotation !== layout.rotation
181	) {
182	// The model and renderer survive framebuffer rotation and native resolution changes.
183	if (resized) {
184	const previous = texture;
185	texture = makeTexture();
186	textureWidth = options.source.width;
187	textureHeight = options.source.height;
188	phone.setDisplay(texture, next);
189	previous.dispose();
190	}
191	if (!imported && (nextProfile !== profile \|\| next.aspect !== layout.aspect)) {
192	scene.remove(phone.root);
193	phone.dispose();
194	phone = createPhoneScene(texture, next, nextProfile);
195	scene.add(phone.root);
196	} else {
197	phone.setDisplay(texture, next);
198	}
199	layout = next;
200	profile = nextProfile;
201	applyPose();
202	fit(true);
203	}
204	applyPose();
205	};
206	const modelSlot = createDeviceModelSlot({
207	load: loadDeviceModel,
208	onError: options.onModelError,
209	install(model) {
210	// Validate and prepare the next scene before releasing the visible one.
211	const next = disposed
212	? null
213	: model
214	? createImportedPhoneScene(model.asset, texture, layout)
215	: createPhoneScene(texture, layout, profile);
216	scene.remove(phone.root);
217	phone.dispose();
218	imported = model;
219	if (!next) return;
220	phone = next;
221	scene.add(phone.root);
222	if (accessory) {
223	if (imported) phone.orientation.add(accessory.asset);
224	else accessory.asset.removeFromParent();
225	}
226	applyPose();
227	fit(true);
228	scheduler.invalidate();
229	},
230	});
231	const accessorySlot = createDeviceModelSlot({
232	load: loadDeviceModel,
233	onError: options.onModelError,
234	install(model) {
235	const bounds = model ? new Box3().setFromObject(model.asset) : null;
236	if (
237	bounds &&
238	(bounds.isEmpty() \|\|
239	![...bounds.min.toArray(), ...bounds.max.toArray()].every(Number.isFinite))
240	)
241	throw new Error("Device accessory has invalid bounds");
242	accessory?.asset.removeFromParent();
243	accessory = model;
244	accessoryBounds = bounds;
245	if (disposed) return;
246	if (imported && accessory) phone.orientation.add(accessory.asset);
247	fit(true);
248	scheduler.invalidate();
249	},
250	});
251	const setAccessory = (source: DeviceAccessorySource \| null) => {
252	accessorySlot.set(source?.modelId === modelSource?.id ? source : null);
253	};
254	const contextLost = (event: Event) => {
255	event.preventDefault();
256	options.onUnavailable();
257	};
258	options.canvas.addEventListener("webglcontextlost", contextLost);
259	applyPose();
260	modelSlot.set(options.model ?? null);
261	setAccessory(options.accessory ?? null);
262	return {
263	setModel(source) {
264	if (source?.id !== modelSource?.id \|\| source?.url !== modelSource?.url)
265	accessorySlot.set(null);
266	modelSource = source;
267	modelSlot.set(source);
268	},
269	setAccessory,
270	frameUpdated() {
271	if (disposed) return;
272	updateLayout();
273	texture.needsUpdate = true;
274	scheduler.invalidate();
275	},
276	setScreen(next, nextProfile = profile) {
277	if (disposed) return;
278	screen = next;
279	updateLayout(nextProfile);
280	scheduler.invalidate();
281	},
282	resize(width, height, pixelRatio) {
283	if (disposed) return;
284	if (![width, height, pixelRatio].every(Number.isFinite) \|\| width <= 0 \|\| height <= 0) return;
285	const ratio = Math.min(2, Math.max(1, pixelRatio));
286	if (viewport.width === width && viewport.height === height && viewport.pixelRatio === ratio)
287	return;
288	viewport = { width, height, pixelRatio: ratio };
289	fit(true);
290	scheduler.invalidate();
291	},
292	screenPoint(x, y, captured = false) {
293	if (disposed) return null;
294	applyPose();
295	return phone.screenPoint(x, y, camera, captured);
296	},
297	orbit(deltaX, deltaY) {
298	if (disposed) return;
299	motion.orbit(deltaX * viewport.width, deltaY * viewport.height, performance.now());
300	scheduler.invalidate();
301	},
302	setInteractionActive(active, mode) {
303	if (disposed) return;
304	const now = performance.now();
305	if (mode === "orbit") motion.dragActive(active, now);
306	else {
307	motion.hold(active, now);
308	framing.hold(active, now);
309	}
310	scheduler.invalidate();
311	},
312	resetPose() {
313	if (disposed) return;
314	motion.reset(rest, performance.now());
315	scheduler.invalidate();
316	},
317	dispose() {
318	if (disposed) return;
319	disposed = true;
320	scheduler.dispose();
321	const hadImported = imported !== null;
322	accessorySlot.dispose();
323	modelSlot.dispose();
324	options.canvas.removeEventListener("webglcontextlost", contextLost);
325	scene.remove(phone.root);
326	if (!hadImported) phone.dispose();
327	texture.dispose();
328	renderer.dispose();
329	renderer.forceContextLoss();
330	},
331	};
332	}

packages/client-runtime/src/device/renderScheduler.test.ts created+27

...	@@ -0,0 +1,27 @@
1	import { expect, it, vi } from "vite-plus/test";
2	import { createRenderScheduler } from "./renderScheduler.ts";
3
4	it("coalesces frame and pointer invalidations, then stops scheduling when idle or disposed", () => {
5	const callbacks: FrameRequestCallback[] = [];
6	const request = vi.fn((callback: FrameRequestCallback) => {
7	callbacks.push(callback);
8	return callbacks.length;
9	});
10	const cancel = vi.fn();
11	const render = vi.fn();
12	const scheduler = createRenderScheduler(render, request, cancel);
13	scheduler.invalidate();
14	scheduler.invalidate();
15	expect(request).toHaveBeenCalledTimes(1);
16	callbacks[0]?.(0);
17	expect(render).toHaveBeenCalledTimes(1);
18	expect(request).toHaveBeenCalledTimes(1);
19	scheduler.invalidate();
20	scheduler.dispose();
21	scheduler.dispose();
22	scheduler.invalidate();
23	callbacks[1]?.(0);
24	expect(cancel).toHaveBeenCalledExactlyOnceWith(2);
25	expect(render).toHaveBeenCalledTimes(1);
26	expect(request).toHaveBeenCalledTimes(2);
27	});

packages/client-runtime/src/device/renderScheduler.ts created+23

...	@@ -0,0 +1,23 @@
1	/** Coalesces invalidations into one render. No work is scheduled while the view is idle. */
2	export function createRenderScheduler(
3	render: () => void,
4	request: (callback: FrameRequestCallback) => number = requestAnimationFrame,
5	cancel: (id: number) => void = cancelAnimationFrame,
6	) {
7	let pending: number \| null = null;
8	let disposed = false;
9	return {
10	invalidate() {
11	if (disposed \|\| pending !== null) return;
12	pending = request(() => {
13	pending = null;
14	if (!disposed) render();
15	});
16	},
17	dispose() {
18	disposed = true;
19	if (pending !== null) cancel(pending);
20	pending = null;
21	},
22	};
23	}

packages/client-runtime/src/device/screenshot.test.ts created+50

...	@@ -0,0 +1,50 @@
1	import { afterEach, expect, it, vi } from "vite-plus/test";
2	import { captureDeviceScreenshot, DeviceScreenshotError } from "./screenshot.ts";
3
4	afterEach(() => vi.unstubAllGlobals());
5	const target = {
6	platform: "android" as const,
7	deviceId: "phone with spaces",
8	access: {
9	httpBase: "https://remote.example/api/device-hub/hosts/remote",
10	wsBase: "wss://remote.example",
11	query: { ticket: "capture-ticket" },
12	credentials: false,
13	},
14	};
15
16	it("captures native bytes from the selected remote host with its media credentials", async () => {
17	const fetch = vi.fn(
18	async () =>
19	new Response(new Uint8Array([137, 80, 78, 71]), { headers: { "Content-Type": "image/png" } }),
20	);
21	vi.stubGlobal("fetch", fetch);
22	const image = await captureDeviceScreenshot(target, new AbortController().signal);
23	expect(image.type).toBe("image/png");
24	expect(Array.from(new Uint8Array(await image.arrayBuffer()))).toEqual([137, 80, 78, 71]);
25	expect(fetch).toHaveBeenCalledWith(
26	"https://remote.example/api/device-hub/hosts/remote/vendor/serve-emu/api/screenshot?device=phone%20with%20spaces&ticket=capture-ticket",
27	expect.objectContaining({ method: "POST", credentials: "omit" }),
28	);
29	});
30
31	it("preserves authorization failures and cancels an in-flight capture when its owner detaches", async () => {
32	vi.stubGlobal(
33	"fetch",
34	vi.fn(async () => new Response("expired", { status: 401 })),
35	);
36	await expect(captureDeviceScreenshot(target, new AbortController().signal)).rejects.toMatchObject(
37	new DeviceScreenshotError(401),
38	);
39	vi.stubGlobal(
40	"fetch",
41	(_url: string, init: RequestInit) =>
42	new Promise((_resolve, reject) => {
43	init.signal?.addEventListener("abort", () => reject(init.signal?.reason), { once: true });
44	}),
45	);
46	const controller = new AbortController();
47	const capture = captureDeviceScreenshot(target, controller.signal);
48	controller.abort(new Error("detached"));
49	await expect(capture).rejects.toThrow("detached");
50	});

packages/client-runtime/src/device/screenshot.ts created+30

...	@@ -0,0 +1,30 @@
1	// @effect-diagnostics globalFetch:off - This browser transport runs without an Effect runtime, like the live stream.
2	import { withDeviceHubQuery } from "./hubAccess.ts";
3	import type { DeviceStreamTarget } from "./stream.ts";
4
5	export class DeviceScreenshotError extends Error {
6	readonly status: number;
7
8	constructor(status: number) {
9	super(`Screenshot capture failed (${status}). Try again.`);
10	this.status = status;
11	}
12	}
13
14	/** Captures native pixels through the same host and credentials as the live stream, in either presentation. */
15	export async function captureDeviceScreenshot(target: DeviceStreamTarget, signal: AbortSignal) {
16	const vendor = target.platform === "ios" ? "serve-sim" : "serve-emu";
17	const url = withDeviceHubQuery(
18	`${target.access.httpBase}/vendor/${vendor}/api/screenshot?device=${encodeURIComponent(target.deviceId)}`,
19	target.access,
20	);
21	const response = await fetch(url, {
22	method: "POST",
23	credentials: target.access.credentials ? "include" : "omit",
24	signal: AbortSignal.any([signal, AbortSignal.timeout(20_000)]),
25	});
26	if (!response.ok) throw new DeviceScreenshotError(response.status);
27	const image = await response.blob();
28	if (!image.size) throw new Error("The device returned an empty screenshot. Try again.");
29	return image;
30	}

packages/client-runtime/src/device/shapeProfile.test.ts created+27

...	@@ -0,0 +1,27 @@
1	import { expect, it } from "vite-plus/test";
2	import { resolveDeviceShape } from "./shapeProfile.ts";
3
4	it("chooses tablet and phone families without mistaking display rotation for device shape", () => {
5	expect(
6	resolveDeviceShape({ platform: "ios", name: "iPad Pro 11-inch (M5)", portraitAspect: 0.75 }).id,
7	).toBe("ios-tablet");
8	expect(
9	resolveDeviceShape({ platform: "ios", name: "iPhone 18 Pro", portraitAspect: 0.46 }).id,
10	).toBe("ios-phone");
11	expect(
12	resolveDeviceShape({ platform: "android", name: "Pixel 9", portraitAspect: 0.45 }).id,
13	).toBe("android-phone");
14	expect(
15	resolveDeviceShape({ platform: "android", name: "Pixel Tablet", portraitAspect: 0.625 }).id,
16	).toBe("android-tablet");
17	});
18
19	it("uses the screen shape for renamed devices and a phone fallback before metadata arrives", () => {
20	expect(resolveDeviceShape({ platform: "ios", name: "Julius", portraitAspect: 0.75 }).id).toBe(
21	"ios-tablet",
22	);
23	expect(resolveDeviceShape({ platform: "android", portraitAspect: 0.45 }).id).toBe(
24	"android-phone",
25	);
26	expect(resolveDeviceShape({ platform: "ios", portraitAspect: Number.NaN }).id).toBe("ios-phone");
27	});

packages/client-runtime/src/device/shapeProfile.ts created+124

...	@@ -0,0 +1,124 @@
1	import type { DevicePlatform } from "@t3tools/contracts";
2
3	export interface DeviceShapeProfile {
4	readonly id: "ios-phone" \| "ios-tablet" \| "android-phone" \| "android-tablet";
5	readonly bezel: number;
6	readonly bodyRadius: number;
7	readonly screenRadius: number;
8	readonly depth: number;
9	readonly backColor: number;
10	readonly buttons: ReadonlyArray<{
11	readonly edge: "left" \| "right" \| "top";
12	readonly offset: number;
13	readonly length: number;
14	}>;
15	readonly camera: {
16	readonly width: number;
17	readonly height: number;
18	readonly insetX: number;
19	readonly insetY: number;
20	readonly lensRadius: number;
21	readonly lenses: ReadonlyArray<readonly [number, number]>;
22	readonly flash: readonly [number, number] \| null;
23	};
24	}
25
26	/** Original family silhouettes, rather than claims to reproduce individual hardware models. */
27	export const IOS_PHONE_SHAPE: DeviceShapeProfile = {
28	id: "ios-phone",
29	bezel: 0.055,
30	bodyRadius: 0.15,
31	screenRadius: 0.105,
32	depth: 0.085,
33	backColor: 0x424b5d,
34	buttons: [
35	{ edge: "right", offset: 0.35, length: 0.3 },
36	{ edge: "left", offset: 0.48, length: 0.18 },
37	{ edge: "left", offset: 0.22, length: 0.18 },
38	],
39	camera: {
40	width: 0.39,
41	height: 0.44,
42	insetX: 0.25,
43	insetY: 0.29,
44	lensRadius: 0.068,
45	lenses: [
46	[-0.08, 0.095],
47	[0.08, -0.095],
48	],
49	flash: [0.085, 0.11],
50	},
51	};
52
53	export const IOS_TABLET_SHAPE: DeviceShapeProfile = {
54	id: "ios-tablet",
55	bezel: 0.065,
56	bodyRadius: 0.105,
57	screenRadius: 0.045,
58	depth: 0.055,
59	backColor: 0x9ca5af,
60	buttons: [
61	{ edge: "top", offset: 0.5, length: 0.15 },
62	{ edge: "right", offset: 0.78, length: 0.13 },
63	{ edge: "right", offset: 0.58, length: 0.13 },
64	],
65	camera: {
66	width: 0.19,
67	height: 0.19,
68	insetX: 0.15,
69	insetY: 0.15,
70	lensRadius: 0.045,
71	lenses: [[0, 0]],
72	flash: null,
73	},
74	};
75
76	export const ANDROID_PHONE_SHAPE: DeviceShapeProfile = {
77	id: "android-phone",
78	bezel: 0.035,
79	bodyRadius: 0.115,
80	screenRadius: 0.08,
81	depth: 0.085,
82	backColor: 0x344449,
83	buttons: [
84	{ edge: "right", offset: 0.2, length: 0.24 },
85	{ edge: "right", offset: 0.65, length: 0.32 },
86	],
87	camera: {
88	width: 0.24,
89	height: 0.47,
90	insetX: 0.18,
91	insetY: 0.29,
92	lensRadius: 0.056,
93	lenses: [
94	[0, 0.11],
95	[0, -0.11],
96	],
97	flash: [0.1, 0],
98	},
99	};
100
101	const ANDROID_TABLET_SHAPE: DeviceShapeProfile = {
102	...IOS_TABLET_SHAPE,
103	id: "android-tablet",
104	backColor: 0x697b80,
105	};
106
107	/** Names identify a family when available; wide unknown screens get a generic tablet shell. */
108	export function resolveDeviceShape(options: {
109	platform: DevicePlatform;
110	name?: string;
111	portraitAspect: number;
112	}): DeviceShapeProfile {
113	const name = options.name ?? "";
114	const namedTablet = /\b(ipad\|tablet)\b/i.test(name);
115	const namedPhone = /\b(iphone\|phone)\b/i.test(name);
116	const tablet = namedTablet \|\| (!namedPhone && options.portraitAspect >= 0.6);
117	return options.platform === "ios"
118	? tablet
119	? IOS_TABLET_SHAPE
120	: IOS_PHONE_SHAPE
121	: tablet
122	? ANDROID_TABLET_SHAPE
123	: ANDROID_PHONE_SHAPE;
124	}

packages/client-runtime/src/device/stream.test.ts+3

packages/client-runtime/src/device/stream.ts+14-11

 @@ -19,6 +19,7 @@
  * The decoder only runs while frames arrive and the viewer is attached; a
  * hidden panel calls `stop()` so an idle device costs nothing on the GPU.
  */
+import { createCanvasFrameSink, type DeviceFrameSink } from "./frame.ts";
 import { type DeviceHubAccess, withDeviceHubQuery } from "./hubAccess.ts";
 import type { DevicePlatform } from "@t3tools/contracts";
 @@ -217,6 +218,7 @@ export interface DeviceStreamClient {
   readonly sendKey: (event: KeyboardEvent, phase: "down" | "up") => void;
   readonly pressButton: (button: DeviceHardwareButton) => void;
   readonly rotate: () => void;
+  readonly setOrientation: (orientation: DeviceScreenSize["orientation"]) => void;
 }
 const HID_USAGE_BY_CODE: Readonly<Record<string, number>> = {
 @@ -282,10 +284,11 @@ const IOS_ORIENTATIONS: ReadonlyArray<DeviceScreenSize["orientation"]> = [
 export function createDeviceStreamClient(
   target: DeviceStreamTarget,
-  canvas: HTMLCanvasElement,
+  output: HTMLCanvasElement | DeviceFrameSink,
   events: DeviceStreamEvents,
 ): DeviceStreamClient {
   const { access, platform, deviceId } = target;
+  const sink = "present" in output ? output : createCanvasFrameSink(output);
   const vendor = platform === "ios" ? "/vendor/serve-sim" : "/vendor/serve-emu";
   const device = encodeURIComponent(deviceId);
   const httpUrl = (path: string) =>
 @@ -402,17 +405,14 @@ export function createDeviceStreamClient(
   const paint = (source: CanvasImageSource, width: number, height: number) => {
     if (stopped) return;
-    if (canvas.width !== width || canvas.height !== height) {
+    if (platform === "android" && (screen?.width !== width || screen.height !== height)) {
-      canvas.width = width;
+      screen = { width, height, orientation: width > height ? "landscape_left" : "portrait" };
-      canvas.height = height;
+      events.onScreen(screen);
-      if (platform === "android") {
+    }
-        screen = { width, height, orientation: width > height ? "landscape_left" : "portrait" };
+    if (!sink.present(source, width, height)) {
-        events.onScreen(screen);
+      fail("Could not display the device stream. Reconnect to try again.");
-      }
+      return;
     }
-    const context = canvas.getContext("2d");
-    if (!context) return fail("Could not display the device stream. Reconnect to try again.");
-    context.drawImage(source, 0, 0, width, height);
     frameReceived();
   };
 @@ -861,5 +861,8 @@ export function createDeviceStreamClient(
         IOS_ORIENTATIONS[(IOS_ORIENTATIONS.indexOf(current) + 1) % IOS_ORIENTATIONS.length]!;
       send(taggedJson(IOS_MSG_ORIENTATION, { orientation: next }));
     },
+    setOrientation: (orientation) => {
+      if (platform === "ios") send(taggedJson(IOS_MSG_ORIENTATION, { orientation }));
+    },
   };
 }

packages/client-runtime/src/device/streamFrames.test.ts created+97

...	@@ -0,0 +1,97 @@
1	import { afterEach, expect, it, vi } from "vite-plus/test";
2	import { createDeviceStreamClient } from "./stream.ts";
3
4	afterEach(() => vi.unstubAllGlobals());
5
6	it("delivers borrowed decoded frames, reports presentation failure, and discards late output", async () => {
7	let decoderOutput: VideoFrameOutputCallback \| undefined;
8	let resolveOutput!: () => void;
9	const outputReady = new Promise<void>((resolve) => {
10	resolveOutput = resolve;
11	});
12	class Decoder {
13	static isConfigSupported = async () => ({ supported: true });
14	state = "unconfigured";
15	constructor(options: VideoDecoderInit) {
16	decoderOutput = options.output;
17	resolveOutput();
18	}
19	configure() {
20	this.state = "configured";
21	}
22	close() {
23	this.state = "closed";
24	}
25	}
26	vi.stubGlobal("VideoDecoder", Decoder);
27	vi.stubGlobal("EncodedVideoChunk", vi.fn());
28	vi.stubGlobal(
29	"WebSocket",
30	class {
31	close() {}
32	},
33	);
34	const envelope = new Uint8Array([0, 0, 0, 5, 1, 1, 0x64, 0, 0x1f]);
35	vi.stubGlobal(
36	"fetch",
37	vi.fn(
38	async (url: string) =>
39	new Response(
40	url.includes("avcc")
41	? new ReadableStream({
42	start(controller) {
43	controller.enqueue(envelope);
44	},
45	})
46	: "seed",
47	),
48	),
49	);
50	const present = vi.fn(() => true);
51	const events = {
52	onStatus: vi.fn(),
53	onScreen: vi.fn(),
54	onUnauthorized: vi.fn(),
55	onMjpegFallback: vi.fn(),
56	onInputConnected: vi.fn(),
57	};
58	const client = createDeviceStreamClient(
59	{
60	platform: "ios",
61	deviceId: "phone",
62	access: {
63	httpBase: "https://test",
64	wsBase: "wss://test",
65	credentials: false,
66	query: {},
67	},
68	},
69	{ present },
70	events,
71	);
72	client.start();
73	await outputReady;
74	const frame = {
75	displayWidth: 1170,
76	displayHeight: 2532,
77	close: vi.fn(),
78	} as unknown as VideoFrame;
79	decoderOutput?.(frame);
80	expect(present).toHaveBeenCalledExactlyOnceWith(frame, 1170, 2532);
81	expect(frame.close).toHaveBeenCalledOnce();
82	present.mockImplementationOnce(() => {
83	throw new Error("renderer failed");
84	});
85	expect(() => decoderOutput?.(frame)).toThrow("renderer failed");
86	expect(frame.close).toHaveBeenCalledTimes(2);
87	present.mockReturnValueOnce(false);
88	decoderOutput?.(frame);
89	expect(events.onStatus).toHaveBeenLastCalledWith(
90	"error",
91	"Could not display the device stream. Reconnect to try again.",
92	);
93	expect(frame.close).toHaveBeenCalledTimes(3);
94	decoderOutput?.(frame);
95	expect(present).toHaveBeenCalledTimes(3);
96	expect(frame.close).toHaveBeenCalledTimes(4);
97	});

packages/contracts/src/ipc.ts+1

...	@@ -1125,6 +1125,7 @@ export interface DesktopBridge {	...	@@ -1125,6 +1125,7 @@ export interface DesktopBridge {
1125	getClientPlatform?: () => string;	1125	getClientPlatform?: () => string;
1126	setNotificationBadge?: (badge: { count: number; image: string \| null }) => Promise<void>;	1126	setNotificationBadge?: (badge: { count: number; image: string \| null }) => Promise<void>;
1127	onNotificationBadgeClear?: (listener: () => void) => () => void;	1127	onNotificationBadgeClear?: (listener: () => void) => () => void;
		1128	onTrackpadScrollEnd?: (listener: () => void) => () => void;
1128	/**	1129	/**
1129	* The OS locale as a BCP-47 tag, which the renderer cannot read for itself:	1130	* The OS locale as a BCP-47 tag, which the renderer cannot read for itself:
1130	* the packaged app ships only the `en-US` Chromium locale pak, so	1131	* the packaged app ships only the `en-US` Chromium locale pak, so

pnpm-lock.yaml+54

 @@ -874,6 +874,9 @@ importers:
       remark-parse:
         specifier: ^11.0.0
         version: 11.0.0
+      three:
+        specifier: 0.180.0
+        version: 0.180.0
       unified:
         specifier: ^11.0.5
         version: 11.0.5
 @@ -881,6 +884,9 @@ importers:
       '@effect/vitest':
         specifier: 4.0.0-rc.115
         version: 4.0.0-rc.115(patch_hash=172bd95ef73a13d322888984415d5d18d9d2aa8e13fa529eac235efac837e902)(effect@4.0.0-rc.115(patch_hash=0dfc4bb8ebd80fb3e06b91ef61346f5259517ab0f2437644fe95ae531084b1f5))
+      '@types/three':
+        specifier: 0.180.0
+        version: 0.180.0
       micromark-util-types:
         specifier: ^2.0.2
         version: 2.0.2
 @@ -2030,6 +2036,9 @@ packages:
       '@types/react':
         optional: true
+  '@dimforge/rapier3d-compat@0.12.0':
+    resolution: {integrity: sha512-uekIGetywIgopfD97oDL5PfeezkFpNhwlzlaEYNOA0N6ghdsOvh/HYjSMek5Q2O1PYvRSDFcqFVJl4r4ZBwOow==}
   '@distilled.cloud/aws@1.0.0-rc.12':
     resolution: {integrity: sha512-L4TTEwMKB1ymXbYUqvYeoy5UWgmJtqQSzry2qWtAvD2ccwVGy0F7cFViEIYQk7boF4ZIP6SfhAls4+y1fVIi/g==}
     peerDependencies:
 @@ -5222,6 +5231,9 @@ packages:
   '@tiptap/starter-kit@3.31.3':
     resolution: {integrity: sha512-WKof9RewdmGHvWJ1wn0/HVNG2mV+HOgVRyJkKekuM9fgr6BZAAH/xZsWE1eon+94JnQ+KtK2ThydXQM/qc6b2A==}
+  '@tweenjs/tween.js@23.1.3':
+    resolution: {integrity: sha512-vJmvvwFxYuGnF2axRtPYocag6Clbb5YS7kLL+SO/TeVFzHqDIWrNKYtcsPMibjDx9O+bu+psAy9NKfWklassUA==}
   '@tybys/wasm-util@0.10.3':
     resolution: {integrity: sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==}
 @@ -5359,9 +5371,15 @@ packages:
   '@types/serve-static@2.2.0':
     resolution: {integrity: sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ==}
+  '@types/stats.js@0.17.4':
+    resolution: {integrity: sha512-jIBvWWShCvlBqBNIZt0KAshWpvSjhkwkEu4ZUcASoAvhmrgAUI2t1dXrjSL4xXVLB4FznPrIsX3nKXFl/Dt4vA==}
   '@types/statuses@2.0.6':
     resolution: {integrity: sha512-xMAgYwceFhRA2zY+XbEA7mxYbA093wdiW8Vu6gZPGWy9cmOyU9XesH1tNcEWsKFd5Vzrqx5T3D38PWx1FIIXkA==}
+  '@types/three@0.180.0':
+    resolution: {integrity: sha512-ykFtgCqNnY0IPvDro7h+9ZeLY+qjgUWv+qEvUt84grhenO60Hqd4hScHE7VTB9nOQ/3QM8lkbNE+4vKjEpUxKg==}
   '@types/unist@2.0.11':
     resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==}
 @@ -5374,6 +5392,9 @@ packages:
   '@types/webidl-conversions@7.0.3':
     resolution: {integrity: sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==}
+  '@types/webxr@0.5.24':
+    resolution: {integrity: sha512-h8fgEd/DpoS9CBrjEQXR+dIDraopAEfu4wYVNY2tEPwk60stPWhvZMf4Foo5FakuQ7HFZoa8WceaWFervK2Ovg==}
   '@types/whatwg-url@11.0.5':
     resolution: {integrity: sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==}
 @@ -5706,6 +5727,9 @@ packages:
   '@vscode/l10n@0.0.18':
     resolution: {integrity: sha512-KYSIHVmslkaCDyw013pphY+d7x1qV8IZupYfeIfzNA+nsaWHbn5uPuQRvdRFsa9zFzGeudPuoGoZ1Op4jrJXIQ==}
+  '@webgpu/types@0.1.74':
+    resolution: {integrity: sha512-lgiI4hbuLcI9unnm2cL/tvCaQU45dp0xcLWJh5uB/9MBGvIA4F8XIk7nSiBeg+K4xWGYwgZKn80LmERI5CxoTA==}
   '@xmldom/xmldom@0.8.15':
     resolution: {integrity: sha512-/5NV/vDALVFDXgLmfsy9TRCBlKwO2LNBFzpzvb9iIj+jR+eSc6DLYYvVOdivT/jm7MtU6TebYuRmzEOI7w40UA==}
     engines: {node: '>=10.0.0'}
 @@ -8463,6 +8487,9 @@ packages:
   merge-stream@2.0.0:
     resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==}
+  meshoptimizer@0.22.0:
+    resolution: {integrity: sha512-IebiK79sqIy+E4EgOr+CAw+Ke8hAspXKzBd0JdgEmPHiAwmvEj2S4h1rfvo+o/BnfEYd/jAOg5IeeIjzlzSnDg==}
   metro-babel-transformer@0.84.4:
     resolution: {integrity: sha512-rvCfz8snl9h20VcvpOHxZuHP1SlAkv4HXbzw7nyyVwu6Eqo5PRerbakQ9XmUCOsRy70spJ37O+G1TK8oMzo48g==}
     engines: {node: ^20.19.4 || ^22.13.0 || ^24.3.0 || >= 25.0.0}
 @@ -10224,6 +10251,9 @@ packages:
     engines: {node: '>=10'}
     hasBin: true
+  three@0.180.0:
+    resolution: {integrity: sha512-o+qycAMZrh+TsE01GqWUxUIKR1AL0S8pq7zDkYOQw8GqfX8b8VoCKYUoHbhiX5j+7hr8XsuHDVU6+gkQJQKg9w==}
   throat@5.0.0:
     resolution: {integrity: sha512-fcwX4mndzpLQKBS1DVYhGAcYaYt7vsHNIvQV+WXMvnow5cgjPphq5CaayLaGsjRdSCKZFNGt7/GYAuXaNOiYCA==}
 @@ -12223,6 +12253,8 @@ snapshots:
     optionalDependencies:
       '@types/react': 19.2.16
+  '@dimforge/rapier3d-compat@0.12.0': {}
   '@distilled.cloud/aws@1.0.0-rc.12(effect@4.0.0-rc.115(patch_hash=0dfc4bb8ebd80fb3e06b91ef61346f5259517ab0f2437644fe95ae531084b1f5))':
     dependencies:
       '@aws-sdk/types': 3.974.5
 @@ -15438,6 +15470,8 @@ snapshots:
       '@tiptap/extensions': 3.31.3(@tiptap/core@3.31.3(@tiptap/pm@3.31.3))(@tiptap/pm@3.31.3)
       '@tiptap/pm': 3.31.3
+  '@tweenjs/tween.js@23.1.3': {}
   '@tybys/wasm-util@0.10.3':
     dependencies:
       tslib: 2.8.1
 @@ -15609,9 +15643,21 @@ snapshots:
       '@types/http-errors': 2.0.5
       '@types/node': 24.12.4
+  '@types/stats.js@0.17.4': {}
   '@types/statuses@2.0.6':
     optional: true
+  '@types/three@0.180.0':
+    dependencies:
+      '@dimforge/rapier3d-compat': 0.12.0
+      '@tweenjs/tween.js': 23.1.3
+      '@types/stats.js': 0.17.4
+      '@types/webxr': 0.5.24
+      '@webgpu/types': 0.1.74
+      fflate: 0.8.3
+      meshoptimizer: 0.22.0
   '@types/unist@2.0.11': {}
   '@types/unist@3.0.3': {}
 @@ -15621,6 +15667,8 @@ snapshots:
   '@types/webidl-conversions@7.0.3':
     optional: true
+  '@types/webxr@0.5.24': {}
   '@types/whatwg-url@11.0.5':
     dependencies:
       '@types/webidl-conversions': 7.0.3
 @@ -15958,6 +16006,8 @@ snapshots:
   '@vscode/l10n@0.0.18': {}
+  '@webgpu/types@0.1.74': {}
   '@xmldom/xmldom@0.8.15': {}
   '@xmldom/xmldom@0.9.12': {}
 @@ -19070,6 +19120,8 @@ snapshots:
   merge-stream@2.0.0: {}
+  meshoptimizer@0.22.0: {}
   metro-babel-transformer@0.84.4:
     dependencies:
       '@babel/core': 7.29.7
 @@ -21599,6 +21651,8 @@ snapshots:
       commander: 2.20.3
       source-map-support: 0.5.21
+  three@0.180.0: {}
   throat@5.0.0: {}
   through@2.3.8: {}

scripts/convert-device-model.py created+108

...	@@ -0,0 +1,108 @@
1	# Run with Blender 4.5.9:
2	# blender --background --python scripts/convert-device-model.py -- <model-id> <source.usdz> <output.glb>
3	# Then optimize using @gltf-transform/cli 4.2.1 with --compress false --simplify false
4	# --join false --palette false --flatten false --texture-size 1024 --texture-compress webp.
5	import hashlib
6	import json
7	import sys
8	from pathlib import Path
9
10	import bpy
11	import numpy as np
12	from mathutils import Matrix, Vector
13
14	model_id, source, output = sys.argv[sys.argv.index("--") + 1 :]
15	manifest_path = (
16	Path(__file__).resolve().parents[1]
17	/ "apps/web/src/components/device/models/sources.json"
18	)
19	manifest = json.loads(manifest_path.read_text())
20	model = next(m for m in manifest["models"] if m["id"] == model_id)
21	if hashlib.sha256(Path(source).read_bytes()).hexdigest() != model["sourceSha256"]:
22	raise ValueError("Source checksum differs from the recorded Apple asset")
23	body_name, screen_name = model["bodyNode"], model["screenNode"]
24	bpy.ops.wm.read_factory_settings(use_empty=True)
25	bpy.ops.wm.usd_import(filepath=source, import_materials=True)
26	body = bpy.data.objects[model.get("accessoryNode", body_name)]
27	screen = bpy.data.objects[screen_name]
28	objects = [o for o in body.children_recursive if o.type == "MESH"]
29	if body.type == "MESH":
30	objects.append(body)
31	uv = screen.data.uv_layers.active
32	points = []
33	coords = []
34	for loop in screen.data.loops:
35	points.append(
36	tuple(screen.matrix_world @ screen.data.vertices[loop.vertex_index].co)
37	)
38	coords.append((*uv.data[loop.index].uv, 1))
39	coef = np.linalg.lstsq(np.array(coords), np.array(points), rcond=None)[0]
40	right = Vector(coef[0]).normalized()
41	up = Vector(coef[1]).normalized()
42	normal = right.cross(up).normalized()
43	# The authored UVs establish the device's front and its physical portrait axes.
44	# No body proportions are changed. The runtime uses a standard planar framebuffer UV.
45	rot = Matrix((right, up, normal)).to_4x4()
46	world = [rot @ Vector(p) for p in points]
47	lo = Vector(tuple(min(p[i] for p in world) for i in range(3)))
48	hi = Vector(tuple(max(p[i] for p in world) for i in range(3)))
49	center = (lo + hi) / 2
50	scale = 2.2 / (hi.y - lo.y)
51	transform = (
52	Matrix.Translation(Vector((0, 0, 0.043)))
53	@ Matrix.Diagonal(Vector((scale, scale, scale, 1)))
54	@ Matrix.Translation(-center)
55	@ rot
56	)
57	for o in objects:
58	matrix = transform @ o.matrix_world
59	o.parent = None
60	o.matrix_world = Matrix.Identity(4)
61	o.data.transform(matrix)
62	# Blender is Z-up; glTF export converts to Y-up. Store canonical Y-up geometry as Blender Z-up.
63	o.data.transform(Matrix.Rotation(np.pi / 2, 4, "X"))
64	if o == screen:
65	o.name = "device-screen"
66	o.data.name = "device-screen"
67	for loop in o.data.loops:
68	p = o.data.vertices[loop.vertex_index].co
69	uv.data[loop.index].uv = (
70	(p.x / ((hi.x - lo.x) * scale)) + 0.5,
71	p.z / 2.2 + 0.5,
72	)
73	# This material is replaced by the viewer and must not retain the marketing screenshot.
74	mat = bpy.data.materials.new("device-screen")
75	mat.diffuse_color = (0, 0, 0, 1)
76	o.data.materials.clear()
77	o.data.materials.append(mat)
78	for o in list(bpy.data.objects):
79	if o not in objects:
80	bpy.data.objects.remove(o, do_unlink=True)
81	for image in bpy.data.images:
82	if image.size[0] > 1024 or image.size[1] > 1024:
83	k = 1024 / max(image.size)
84	image.scale(max(1, round(image.size[0] * k)), max(1, round(image.size[1] * k)))
85	for o in objects:
86	o.select_set(True)
87	bpy.ops.export_scene.gltf(
88	filepath=output,
89	export_format="GLB",
90	use_selection=True,
91	export_yup=True,
92	export_animations=False,
93	export_extras=False,
94	export_image_format="AUTO",
95	export_copyright="Apple Inc. Original device assets; see models/sources.json for provenance.",
96	)
97	print(
98	"MODEL_METRICS",
99	json.dumps(
100	{
101	"width": (hi.x - lo.x) * scale,
102	"height": 2.2,
103	"bodyMeshes": len(objects),
104	"screen": screen.name,
105	"source": source,
106	}
107	),
108	)

author	Julius Marminge <julius0216@outlook.com> 2026-09-23 20:42:53-07:00
committer	GitHub <noreply@github.com> 2026-09-23 20:42:53-07:00
log	78af372cf46a80147f05183b69d7a8339259ca37
tree	2f5dfec9769840659ff693edcaa353eee61efc22
parent	f1add18ae1560f509405a5eece07f3cfb1cc850f
signature	Signed by PGP key B5690EEEBB952194

...	@@ -111,6 +111,7 @@ describe("native device stream transport", () => {	...	@@ -111,6 +111,7 @@ describe("native device stream transport", () => {
111	expect(imageUrl.searchParams.get("hostId")).toBe("ssh-host");	111	expect(imageUrl.searchParams.get("hostId")).toBe("ssh-host");
112	expect(fetch).toHaveBeenCalledTimes(1);	112	expect(fetch).toHaveBeenCalledTimes(1);
113	socket.onopen?.();	113	socket.onopen?.();
		114	client.setOrientation("landscape_right");
114	client.pressButton("home");	115	client.pressButton("home");
115	client.sendTouch("begin", 0.25, 0.75);	116	client.sendTouch("begin", 0.25, 0.75);
116	const messages = socket.send.mock.calls.slice(1).map(([bytes]) => {	117	const messages = socket.send.mock.calls.slice(1).map(([bytes]) => {
...	@@ -121,6 +122,7 @@ describe("native device stream transport", () => {	...	@@ -121,6 +122,7 @@ describe("native device stream transport", () => {
121	};	122	};
122	});	123	});
123	expect(messages).toEqual([	124	expect(messages).toEqual([
		125	{ tag: 0x07, body: { orientation: "landscape_right" } },
124	{ tag: 0x04, body: { button: "home" } },	126	{ tag: 0x04, body: { button: "home" } },
125	{ tag: 0x03, body: { type: "begin", x: 0.25, y: 0.75 } },	127	{ tag: 0x03, body: { type: "begin", x: 0.25, y: 0.75 } },
126	]);	128	]);
...	@@ -140,6 +142,7 @@ describe("native device stream transport", () => {	...	@@ -140,6 +142,7 @@ describe("native device stream transport", () => {
140	client.sendTouch("begin", 0.2, 0.8);	142	client.sendTouch("begin", 0.2, 0.8);
141	client.sendTouch("move", 0.2, 0.4);	143	client.sendTouch("move", 0.2, 0.4);
142	client.sendTouch("end", 0.2, 0.1);	144	client.sendTouch("end", 0.2, 0.1);
		145	client.setOrientation("landscape_right");
143	client.pressButton("home");	146	client.pressButton("home");
144	client.pressButton("appSwitcher");	147	client.pressButton("appSwitcher");
145	expect(	148	expect(

feat(web): add an interactive 3D device workspace (#12787)