authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-09-24 21:50:58-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-24 21:50:58-07:00
log9957349c0a066a9efc657600fcb901c9b9cca34b
treea14db3e7576ca423c16415f1acee9bbd3c3dc5e2
parent6391be2729566a1fe1045ff8617f003580b37bfd
signature Signed by PGP key B5690EEEBB952194

feat(web): control Android foldables in the Device panel (#13534)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

11 files changed, 985 insertions(+), 15 deletions(-)

apps/server/src/device/DeviceHubProxy.test.ts+26
......@@ -111,6 +111,32 @@ describe("device hub proxy", () => {
111111 await response.text();
112112 });
113113
114 it("reads Android fold state but requires operate scope to change it", async () => {
115 const path = "http://t3.test/api/device-hub/vendor/serve-emu/api/fold?device=emulator-5554";
116 const reader = fixture([AuthOrchestrationReadScope]);
117 const read = await reader.handler(new Request(path));
118 expect(read.status).toBe(200);
119 await read.text();
120 expect(reader.requests).toEqual([
121 "http://hub.test/vendor/serve-emu/api/fold?device=emulator-5554",
122 ]);
123 const denied = await reader.handler(
124 new Request(path, { method: "POST", body: '{"posture":"closed"}' }),
125 );
126 expect(denied.status).toBe(403);
127 expect(reader.requests).toHaveLength(1);
128
129 const operator = fixture([AuthOrchestrationOperateScope]);
130 const changed = await operator.handler(
131 new Request(path, { method: "POST", body: '{"posture":"closed"}' }),
132 );
133 expect(changed.status).toBe(200);
134 await changed.text();
135 expect(operator.requests).toEqual([
136 "http://hub.test/vendor/serve-emu/api/fold?device=emulator-5554",
137 ]);
138 });
139
114140 it("never forwards the vendor shell endpoint", async () => {
115141 const { handler, requests } = fixture([AuthOrchestrationOperateScope]);
116142 expect(
apps/server/src/device/DeviceHubProxy.ts+3-2
......@@ -44,7 +44,7 @@ const ALLOWED_PATHS: ReadonlyArray<RegExp> = [
4444 /^\/vendor\/serve-sim\/helper\/[^/]+\/(stream\.mjpeg|stream\.avcc|config|health|ax|foreground)$/,
4545 /^\/vendor\/serve-sim\/helper\/[^/]+\/panel\/(1|3)\/stream\.avcc$/,
4646 /^\/vendor\/serve-sim\/appstate$/,
47 /^\/vendor\/serve-emu\/api\/(devices|screenshot|stream-mode|stream-settings|accessibility)$/,
47 /^\/vendor\/serve-emu\/api\/(devices|screenshot|stream-mode|stream-settings|accessibility|fold)$/,
4848 /^\/vendor\/serve-emu\/health$/,
4949];
5050
......@@ -52,6 +52,7 @@ const ALLOWED_PATHS: ReadonlyArray<RegExp> = [
5252const MUTABLE_PATHS: ReadonlyArray<RegExp> = [
5353 /^\/vendor\/serve-sim\/api\/screenshot$/,
5454 /^\/vendor\/serve-emu\/api\/(screenshot|stream-mode|stream-settings)$/,
55 /^\/vendor\/serve-emu\/api\/fold$/,
5556];
5657
5758const ALLOWED_WS_PATHS: ReadonlyArray<RegExp> = [
......@@ -204,7 +205,7 @@ const handler = Effect.gen(function* () {
204205 }
205206 const controlsDevice =
206207 (upgrade && hubPath !== "/api/devices/ws") ||
207 (!readOnly && /\/api\/stream-(mode|settings)$/.test(hubPath));
208 (!readOnly && /\/api\/(stream-(mode|settings)|fold)$/.test(hubPath));
208209 yield* authenticate(controlsDevice ? AuthOrchestrationOperateScope : AuthOrchestrationReadScope);
209210 const devices = yield* DeviceService.DeviceService;
210211 const ready = yield* devices.currentReadiness(url.value.searchParams.get("hostId") ?? undefined);
apps/web/src/components/device/DeviceAndroidFoldControls.tsx created+155
......@@ -0,0 +1,155 @@
1import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess";
2import { useEffect, useRef, useState } from "react";
3import { Button } from "~/components/ui/button";
4import { Tooltip, TooltipPopup, TooltipTrigger } from "~/components/ui/tooltip";
5import {
6 readAndroidFold,
7 setAndroidFold,
8 type AndroidFoldPosture,
9 type AndroidFoldState,
10} from "./deviceFold";
11import { DeviceDuoGlyph } from "./DeviceDuoGlyph";
12
13/** Capability comes from the emulator, not its AVD name or screen dimensions. */
14export function DeviceAndroidFoldControls(props: {
15 readonly access: DeviceHubAccess;
16 readonly deviceId: string;
17 readonly visible: boolean;
18 readonly enabled: boolean;
19 readonly screenWidth: number | undefined;
20 readonly screenHeight: number | undefined;
21 readonly onFoldAngle: (angle: number | null) => void;
22}) {
23 const [fold, setFold] = useState<AndroidFoldState | null>(null);
24 const [pending, setPending] = useState(false);
25 const [error, setError] = useState<string | null>(null);
26 const busy = useRef(false);
27 const mounted = useRef(true);
28 const { access, deviceId, visible, enabled, screenWidth, screenHeight, onFoldAngle } = props;
29
30 useEffect(() => {
31 mounted.current = true;
32 return () => {
33 mounted.current = false;
34 };
35 }, []);
36
37 useEffect(() => {
38 if (!visible || !enabled || pending || !screenWidth || !screenHeight) return;
39 const controller = new AbortController();
40 let retry: number | undefined;
41 const read = () => {
42 void readAndroidFold(access, deviceId, controller.signal)
43 .then((next) => {
44 if (!controller.signal.aborted) {
45 setFold(next);
46 onFoldAngle(
47 next.supported ? (next.hingeAngle ?? (next.posture === "closed" ? 0 : 180)) : null,
48 );
49 }
50 })
51 .catch(() => {
52 if (!controller.signal.aborted) retry = window.setTimeout(read, 3_000);
53 });
54 };
55 read();
56 return () => {
57 controller.abort();
58 window.clearTimeout(retry);
59 };
60 }, [access, deviceId, visible, enabled, screenWidth, screenHeight, pending, onFoldAngle]);
61
62 if (!fold?.supported || !props.visible) return null;
63
64 const change = (posture: AndroidFoldPosture) => {
65 if (busy.current || !props.enabled) return;
66 busy.current = true;
67 setPending(true);
68 setError(null);
69 props.onFoldAngle(posture === "closed" ? 0 : 180);
70 const controller = new AbortController();
71 const timeout = window.setTimeout(() => controller.abort(), 12_000);
72 void setAndroidFold(props.access, props.deviceId, posture, controller.signal)
73 .then((next) => {
74 if (mounted.current) {
75 setFold(next);
76 props.onFoldAngle(next.hingeAngle ?? (next.posture === "closed" ? 0 : 180));
77 }
78 })
79 .catch((cause: unknown) => {
80 if (mounted.current) {
81 props.onFoldAngle(fold.hingeAngle ?? (fold.posture === "closed" ? 0 : 180));
82 setError(
83 controller.signal.aborted
84 ? "Fold command timed out."
85 : cause instanceof Error
86 ? cause.message
87 : "Could not change fold posture.",
88 );
89 }
90 })
91 .finally(() => {
92 window.clearTimeout(timeout);
93 busy.current = false;
94 if (mounted.current) setPending(false);
95 });
96 };
97
98 return (
99 <div aria-label="Android fold controls" className="flex flex-col items-center gap-2">
100 <div className="pointer-events-auto flex shrink-0 flex-col items-center gap-1 rounded-full border border-border/50 bg-background/80 p-1 shadow-sm">
101 <Tooltip>
102 <TooltipTrigger
103 render={
104 <Button
105 size="icon"
106 variant={fold.posture === "closed" ? "secondary" : "ghost"}
107 aria-label="Fold device"
108 aria-pressed={fold.posture === "closed"}
109 disabled={pending || !props.enabled}
110 onClick={() => change("closed")}
111 />
112 }
113 >
114 <DeviceDuoGlyph pose="closed" />
115 </TooltipTrigger>
116 <TooltipPopup side="left">Fold device</TooltipPopup>
117 </Tooltip>
118 <Tooltip>
119 <TooltipTrigger
120 render={
121 <Button
122 size="icon"
123 variant={fold.posture === "opened" ? "secondary" : "ghost"}
124 aria-label="Unfold device"
125 aria-pressed={fold.posture === "opened"}
126 disabled={pending || !props.enabled}
127 onClick={() => change("opened")}
128 />
129 }
130 >
131 <DeviceDuoGlyph pose="open" />
132 </TooltipTrigger>
133 <TooltipPopup side="left">Unfold device</TooltipPopup>
134 </Tooltip>
135 </div>
136 {error ? (
137 <Tooltip>
138 <TooltipTrigger
139 render={
140 <span
141 tabIndex={0}
142 role="alert"
143 aria-label={error}
144 className="pointer-events-auto text-xs text-destructive"
145 >
146 !
147 </span>
148 }
149 />
150 <TooltipPopup side="left">{error}</TooltipPopup>
151 </Tooltip>
152 ) : null}
153 </div>
154 );
155}
apps/web/src/components/device/DevicePhoneViewport.tsx+8
......@@ -22,6 +22,7 @@ export function DevicePhoneViewport(props: {
2222 readonly onInputCancel: (cancel: (() => void) | null) => void;
2323 readonly client: RefObject<DeviceStreamClient | null>;
2424 readonly screen: DeviceScreenSize | null;
25 readonly foldAngle: number | null;
2526 readonly onUnavailable: () => void;
2627}) {
2728 const hostRef = useRef<HTMLDivElement | null>(null);
......@@ -29,11 +30,17 @@ export function DevicePhoneViewport(props: {
2930 const viewerRef = useRef<PhoneViewer | null>(null);
3031 const interactionRef = useRef<ReturnType<typeof createPhoneInteraction> | null>(null);
3132 const screenRef = useRef(props.screen);
33 const foldAngleRef = useRef(props.foldAngle);
3234 const profileRef = useRef(props.profile);
3335 const modelRef = useRef(props.model);
3436 const accessoryRef = useRef(props.accessory);
3537 const { source, onFrameListener, client, onInputCancel, onResetReady, onUnavailable } = props;
3638
39 useEffect(() => {
40 foldAngleRef.current = props.foldAngle;
41 viewerRef.current?.setFoldAngle(props.foldAngle);
42 }, [props.foldAngle]);
43
3744 useEffect(() => {
3845 screenRef.current = props.screen;
3946 profileRef.current = props.profile;
......@@ -76,6 +83,7 @@ export function DevicePhoneViewport(props: {
7683 profile: profileRef.current,
7784 model: modelRef.current,
7885 accessory: accessoryRef.current,
86 foldAngle: foldAngleRef.current,
7987 });
8088 viewerRef.current = viewer;
8189 onResetReady(viewer.resetPose);
apps/web/src/components/device/DeviceStreamView.tsx+39-3
......@@ -10,6 +10,7 @@ import { deviceKeyboard, deviceModel } from "./deviceModels";
1010import { fitDeviceFrame } from "./deviceFrameLayout";
1111import { DeviceDuoViewport } from "./DeviceDuoViewport";
1212import { DeviceDuoControls } from "./DeviceDuoControls";
13import { DeviceAndroidFoldControls } from "./DeviceAndroidFoldControls";
1314import type { DuoControlState } from "@t3tools/client-runtime/device/duo-control";
1415import { DevicePhoneViewport } from "./DevicePhoneViewport";
1516import { DeviceLoadingView } from "./DeviceLoadingView";
......@@ -94,7 +95,9 @@ export function DeviceStreamView(props: {
9495 const clientRef = useRef<DeviceStreamClient | null>(null);
9596 const [status, setStatus] = useState<DeviceStreamStatus>("connecting");
9697 const [detail, setDetail] = useState<string | undefined>(undefined);
98 const [showRestartNotice, setShowRestartNotice] = useState(false);
9799 const [screen, setScreen] = useState<DeviceScreenSize | null>(null);
100 const [foldAngle, setFoldAngle] = useState<number | null>(null);
98101 const [mjpegUrl, setMjpegUrl] = useState<string | null>(null);
99102 const [mjpegGeneration, setMjpegGeneration] = useState(0);
100103 const attachMjpegImage = useCallback((image: HTMLImageElement | null) => {
......@@ -121,6 +124,7 @@ export function DeviceStreamView(props: {
121124 onStatus: (next, nextDetail) => {
122125 setStatus(next);
123126 setDetail(nextDetail);
127 if (next !== "connecting") setShowRestartNotice(false);
124128 },
125129 onScreen: (next) => {
126130 setScreen(next);
......@@ -136,6 +140,7 @@ export function DeviceStreamView(props: {
136140 },
137141 onInputConnected: (connected, detail) => {
138142 setInputState({ connected, ...(detail ? { detail } : {}) });
143 if (!connected) setShowRestartNotice(false);
139144 onHandle?.({
140145 pressButton: client.pressButton,
141146 rotate: client.rotate,
......@@ -183,15 +188,27 @@ export function DeviceStreamView(props: {
183188 return w / h;
184189 }, [props.platform, screen]);
185190
191 // Android restarts its encoder when a fold changes the framebuffer size.
192 // Keep the last decoded frame and viewer mounted while the next keyframe arrives.
193 const retainingAndroidFrame =
194 props.platform === "android" &&
195 status === "connecting" &&
196 inputState.connected &&
197 screen !== null;
186198 const showPhone =
187199 props.allowPhoneView &&
188 status === "streaming" &&
200 (status === "streaming" || retainingAndroidFrame) &&
189201 props.visible &&
190202 presentation === "phone" &&
191203 !phoneUnavailable &&
192204 !mjpegUrl &&
193205 !props.axOverlay &&
194206 (!isDuo || screen?.supportsHingeAngle === true);
207 useEffect(() => {
208 if (!retainingAndroidFrame || !showPhone) return;
209 const timeout = window.setTimeout(() => setShowRestartNotice(true), 2_000);
210 return () => window.clearTimeout(timeout);
211 }, [retainingAndroidFrame, showPhone]);
195212 const controlsInset = props.renderControls && !showPhone ? CONTROLS_RAIL_WIDTH : 0;
196213
197214 // The frame is the largest box at `aspect` that fits the container, so a
......@@ -330,7 +347,18 @@ export function DeviceStreamView(props: {
330347 streaming: status === "streaming",
331348 phoneUnavailableReason,
332349 foldingControls:
333 showPhone && isDuo && screen?.supportsHingeAngle ? (
350 props.platform === "android" && access ? (
351 <DeviceAndroidFoldControls
352 key={`${props.hostId}:${props.deviceId}`}
353 access={access}
354 deviceId={props.deviceId}
355 visible={props.visible}
356 enabled={status === "streaming"}
357 screenWidth={screen?.width}
358 screenHeight={screen?.height}
359 onFoldAngle={setFoldAngle}
360 />
361 ) : showPhone && isDuo && screen?.supportsHingeAngle ? (
334362 <DeviceDuoControls
335363 screen={screen}
336364 state={duoControl}
......@@ -471,6 +499,7 @@ export function DeviceStreamView(props: {
471499 onInputCancel={onInputCancel}
472500 onResetReady={onResetReady}
473501 screen={screen}
502 foldAngle={foldAngle}
474503 onUnavailable={onPhoneUnavailable}
475504 />
476505 ) : null}
......@@ -503,7 +532,14 @@ export function DeviceStreamView(props: {
503532 </span>
504533 </div>
505534 ) : null}
506 {status !== "streaming" ? (
535 {retainingAndroidFrame && showPhone && showRestartNotice ? (
536 <div className="pointer-events-none absolute inset-x-0 bottom-0 flex justify-center p-2">
537 <span className="rounded-md bg-background/85 px-2 py-1 text-xs text-muted-foreground">
538 Waiting for device video…
539 </span>
540 </div>
541 ) : null}
542 {status !== "streaming" && !(retainingAndroidFrame && showPhone) ? (
507543 <div className="absolute inset-0">
508544 <DeviceLoadingView
509545 name={props.deviceName ?? "Device"}
apps/web/src/components/device/deviceFold.ts created+72
......@@ -0,0 +1,72 @@
1import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess";
2import { withDeviceHubQuery } from "@t3tools/client-runtime/state/deviceHubAccess";
3
4export type AndroidFoldPosture = "closed" | "opened";
5
6export interface AndroidFoldState {
7 readonly supported: boolean;
8 readonly posture: "closed" | "half_opened" | "opened" | "flipped" | "tent" | null;
9 readonly hingeAngle: number | null;
10}
11
12const isRecord = (value: unknown): value is Record<string, unknown> =>
13 typeof value === "object" && value !== null && !Array.isArray(value);
14
15const parseFold = (payload: unknown): AndroidFoldState => {
16 if (!isRecord(payload) || payload.ok !== true || !isRecord(payload.fold)) {
17 throw new Error("Unexpected Android fold response.");
18 }
19 const { supported, posture, hingeAngle } = payload.fold;
20 if (
21 typeof supported !== "boolean" ||
22 (posture !== null &&
23 posture !== "closed" &&
24 posture !== "half_opened" &&
25 posture !== "opened" &&
26 posture !== "flipped" &&
27 posture !== "tent") ||
28 (hingeAngle !== null && (typeof hingeAngle !== "number" || !Number.isFinite(hingeAngle)))
29 ) {
30 throw new Error("Unexpected Android fold response.");
31 }
32 return { supported, posture, hingeAngle };
33};
34
35const foldRequest = async (
36 access: DeviceHubAccess,
37 deviceId: string,
38 posture?: AndroidFoldPosture,
39 signal?: AbortSignal,
40): Promise<AndroidFoldState> => {
41 const url = withDeviceHubQuery(
42 `${access.httpBase}/vendor/serve-emu/api/fold?${new URLSearchParams({ device: deviceId })}`,
43 access,
44 );
45 const response = await fetch(url, {
46 method: posture ? "POST" : "GET",
47 cache: "no-store",
48 credentials: access.credentials ? "include" : "same-origin",
49 ...(posture
50 ? { headers: { "content-type": "application/json" }, body: JSON.stringify({ posture }) }
51 : {}),
52 ...(signal ? { signal } : {}),
53 });
54 if (!response.ok) {
55 const payload: unknown = await response.json().catch(() => null);
56 const error = isRecord(payload) ? payload.error : null;
57 throw new Error(
58 typeof error === "string" ? error : `Fold command failed (${response.status}).`,
59 );
60 }
61 return parseFold(await response.json());
62};
63
64export const readAndroidFold = (access: DeviceHubAccess, deviceId: string, signal?: AbortSignal) =>
65 foldRequest(access, deviceId, undefined, signal);
66
67export const setAndroidFold = (
68 access: DeviceHubAccess,
69 deviceId: string,
70 posture: AndroidFoldPosture,
71 signal?: AbortSignal,
72) => foldRequest(access, deviceId, posture, signal);
docs/user/devices.md+2
......@@ -44,6 +44,8 @@ interactive. On iPhone Duo, use the fold and stance controls to change its
4444physical pose, or pinch over the device to adjust the hinge. Turning the model
4545to the other screen switches the live display and touch input to that screen.
4646**Restore 3D view** returns the device to a screen-facing position.
47On supported Android foldables, use **Fold device** and **Unfold device** beside
48the screen to change its posture in either view.
4749
4850## Tools
4951
packages/client-runtime/src/device/androidFoldScene.test.ts created+57
......@@ -0,0 +1,57 @@
1import { Box3, Mesh, PerspectiveCamera, Texture, Vector3 } from "three";
2import { describe, expect, it } from "vite-plus/test";
3import { createAndroidFoldScene } from "./androidFoldScene.ts";
4import { phoneDisplayLayout } from "./phoneScene.ts";
5
6describe("Android fold scene", () => {
7 it("moves one physical half around the hinge while preserving both screen halves", () => {
8 const texture = new Texture();
9 const scene = createAndroidFoldScene(
10 texture,
11 phoneDisplayLayout({ width: 2200, height: 1840, orientation: "landscape_left" }, 2200, 1840),
12 180,
13 );
14 const moving = scene.orientation.children[0]!;
15 const fixed = scene.orientation.children[1]!;
16 const openWidth = new Box3().setFromObject(scene.root).getSize(new Vector3()).x;
17 const leftScreen = scene.root.getObjectByName("left-inner-screen") as Mesh;
18 const rightScreen = scene.root.getObjectByName("right-inner-screen") as Mesh;
19 leftScreen.geometry.computeBoundingBox();
20 rightScreen.geometry.computeBoundingBox();
21 const creaseWidth =
22 rightScreen.geometry.boundingBox!.min.x - leftScreen.geometry.boundingBox!.max.x;
23 expect(creaseWidth).toBeLessThan(0.01);
24 const continuousScreen = scene.root.getObjectByName("continuous-inner-screen") as Mesh;
25 const positions = continuousScreen.geometry.getAttribute("position");
26 expect(continuousScreen.geometry.index).not.toBeNull();
27 expect(Array.from({ length: positions.count }, (_, i) => positions.getX(i))).toContain(0);
28 scene.setAngle(90);
29 expect(moving.rotation.y).toBeCloseTo(Math.PI / 2);
30 expect(fixed.rotation.y).toBe(0);
31 scene.setAngle(0);
32 const closedWidth = new Box3().setFromObject(scene.root).getSize(new Vector3()).x;
33 expect(closedWidth).toBeLessThan(openWidth * 0.7);
34 expect(scene.root.getObjectByName("cover-screen")?.visible).toBe(true);
35 scene.dispose();
36 texture.dispose();
37 });
38
39 it("maps touches on each open half and the closed cover to the live frame", () => {
40 const texture = new Texture();
41 const scene = createAndroidFoldScene(texture, phoneDisplayLayout(null, 2200, 1840), 180);
42 const camera = new PerspectiveCamera(32, 1, 0.1, 30);
43 camera.position.z = 6;
44 camera.updateMatrixWorld(true);
45 const project = (x: number) => {
46 const point = new Vector3(x, 0, 0.041).project(camera);
47 return scene.screenPoint((point.x + 1) / 2, (1 - point.y) / 2, camera);
48 };
49 expect(project(-0.52)?.x).toBeCloseTo(0.25, 1);
50 expect(project(0.52)?.x).toBeCloseTo(0.75, 1);
51 expect(scene.screenPoint(0.99, 0.5, camera, true)?.x).toBe(1);
52 scene.setAngle(0);
53 expect(project(0.52)?.x).toBeCloseTo(0.5, 1);
54 scene.dispose();
55 texture.dispose();
56 });
57});
packages/client-runtime/src/device/androidFoldScene.ts created+395
......@@ -0,0 +1,395 @@
1import {
2 BoxGeometry,
3 CircleGeometry,
4 CylinderGeometry,
5 ExtrudeGeometry,
6 Group,
7 Mesh,
8 MeshBasicMaterial,
9 MeshPhysicalMaterial,
10 MeshStandardMaterial,
11 PlaneGeometry,
12 Raycaster,
13 Shape,
14 ShapeGeometry,
15 Vector2,
16 Vector3,
17 type Camera,
18 type Texture,
19} from "three";
20import type { PhoneDisplayLayout } from "./phoneScene.ts";
21
22const HALF_WIDTH = 1.04;
23const HEIGHT = 2.2;
24const DEPTH = 0.075;
25const INSET = 0.026;
26const CREASE = 0.004;
27const BEVEL = 0.008;
28// The hinge axis sits just above the inner screens so closed halves meet face to face.
29const PIVOT_Z = DEPTH / 2 + 0.005;
30const SPINE_RADIUS = PIVOT_Z + DEPTH / 2 - 0.002;
31
32function panelPath(side: "left" | "right", inset: number, radius: number, hingeInset = 0) {
33 const left = side === "left" ? -HALF_WIDTH + inset : CREASE / 2 + hingeInset;
34 const right = side === "left" ? -CREASE / 2 - hingeInset : HALF_WIDTH - inset;
35 const bottom = -HEIGHT / 2 + inset;
36 const top = HEIGHT / 2 - inset;
37 const path = new Shape();
38 if (side === "left") {
39 path.moveTo(left + radius, bottom);
40 path.lineTo(right, bottom);
41 path.lineTo(right, top);
42 path.lineTo(left + radius, top);
43 path.quadraticCurveTo(left, top, left, top - radius);
44 path.lineTo(left, bottom + radius);
45 path.quadraticCurveTo(left, bottom, left + radius, bottom);
46 } else {
47 path.moveTo(left, bottom);
48 path.lineTo(right - radius, bottom);
49 path.quadraticCurveTo(right, bottom, right, bottom + radius);
50 path.lineTo(right, top - radius);
51 path.quadraticCurveTo(right, top, right - radius, top);
52 path.lineTo(left, top);
53 }
54 path.closePath();
55 return path;
56}
57
58function roundedRectPath(width: number, height: number, radius: number) {
59 const path = new Shape();
60 path.moveTo(-width / 2 + radius, -height / 2);
61 path.lineTo(width / 2 - radius, -height / 2);
62 path.quadraticCurveTo(width / 2, -height / 2, width / 2, -height / 2 + radius);
63 path.lineTo(width / 2, height / 2 - radius);
64 path.quadraticCurveTo(width / 2, height / 2, width / 2 - radius, height / 2);
65 path.lineTo(-width / 2 + radius, height / 2);
66 path.quadraticCurveTo(-width / 2, height / 2, -width / 2, height / 2 - radius);
67 path.lineTo(-width / 2, -height / 2 + radius);
68 path.quadraticCurveTo(-width / 2, -height / 2, -width / 2 + radius, -height / 2);
69 path.closePath();
70 return path;
71}
72
73function coverPath() {
74 return roundedRectPath(HALF_WIDTH - INSET * 2 - 0.02, HEIGHT - INSET * 2 - 0.04, 0.07);
75}
76
77/** A procedural book-style foldable: one fixed half, one half rotating around a shared hinge. */
78export function createAndroidFoldScene(
79 texture: Texture,
80 layout: PhoneDisplayLayout,
81 initialAngle: number,
82) {
83 const root = new Group();
84 const orientation = new Group();
85 root.add(orientation);
86 const left = new Group();
87 const right = new Group();
88 orientation.add(left, right);
89 const frameMetal = new MeshStandardMaterial({
90 color: 0xa3abb2,
91 metalness: 0.9,
92 roughness: 0.28,
93 });
94 const polishedMetal = new MeshStandardMaterial({
95 color: 0xc4cad0,
96 metalness: 0.95,
97 roughness: 0.16,
98 });
99 const bezel = new MeshPhysicalMaterial({
100 color: 0x0b0d10,
101 metalness: 0.1,
102 roughness: 0.2,
103 clearcoat: 1,
104 });
105 const backGlass = new MeshPhysicalMaterial({
106 color: 0x2c3237,
107 metalness: 0.35,
108 roughness: 0.52,
109 clearcoat: 0.4,
110 clearcoatRoughness: 0.6,
111 });
112 const island = new MeshPhysicalMaterial({
113 color: 0x1a1e22,
114 metalness: 0.55,
115 roughness: 0.3,
116 clearcoat: 1,
117 });
118 const lensMaterial = new MeshPhysicalMaterial({
119 color: 0x061022,
120 metalness: 0.6,
121 roughness: 0.1,
122 clearcoat: 1,
123 });
124 const flashMaterial = new MeshBasicMaterial({ color: 0xf2ead6 });
125 const displayMaterial = new MeshBasicMaterial({ map: texture, toneMapped: false });
126 const hitMaterial = new MeshBasicMaterial({ colorWrite: false, depthWrite: false });
127 const coverMaterial = new MeshBasicMaterial({ map: texture, toneMapped: false });
128 const materials = [
129 frameMetal,
130 polishedMetal,
131 bezel,
132 backGlass,
133 island,
134 lensMaterial,
135 flashMaterial,
136 displayMaterial,
137 hitMaterial,
138 coverMaterial,
139 ];
140
141 // Each half's meshes live in body coordinates; `left` pivots them around the hinge axis.
142 left.position.z = PIVOT_Z;
143 const leftBody = new Group();
144 leftBody.position.z = -PIVOT_Z;
145 left.add(leftBody);
146
147 function half(group: Group, side: "left" | "right", back: MeshPhysicalMaterial) {
148 const body = new Mesh(
149 new ExtrudeGeometry(panelPath(side, BEVEL, 0.1, BEVEL), {
150 depth: DEPTH - BEVEL * 2,
151 bevelEnabled: true,
152 bevelSize: BEVEL,
153 bevelThickness: BEVEL,
154 bevelSegments: 3,
155 curveSegments: 12,
156 }),
157 frameMetal,
158 );
159 body.position.z = -DEPTH / 2 + BEVEL;
160 group.add(body);
161 const frame = new Mesh(new ShapeGeometry(panelPath(side, 0.01, 0.095, 0.002), 12), bezel);
162 frame.position.z = DEPTH / 2 + 0.001;
163 group.add(frame);
164 // A back-facing shape mirrors X, so it is drawn from the opposite side's outline.
165 const rear = new Mesh(
166 new ShapeGeometry(panelPath(side === "left" ? "right" : "left", 0.01, 0.095, 0.002), 12),
167 back,
168 );
169 rear.name = `${side}-back`;
170 rear.rotation.y = Math.PI;
171 rear.position.set(0, 0, -DEPTH / 2 - 0.001);
172 group.add(rear);
173 const geometry = new ShapeGeometry(panelPath(side, INSET, 0.076));
174 geometry.computeBoundingBox();
175 const display = new Mesh(geometry, hitMaterial);
176 display.name = `${side}-inner-screen`;
177 display.position.z = DEPTH / 2 + 0.003;
178 group.add(display);
179 return display;
180 }
181
182 const innerLeft = half(leftBody, "left", bezel);
183 const innerRight = half(right, "right", backGlass);
184 // One indexed surface keeps adjacent pixels joined at the crease. The
185 // physical halves move separately underneath it.
186 const screenWidth = 2 * (HALF_WIDTH - INSET);
187 const screenHeight = HEIGHT - 2 * INSET;
188 const screenGeometry = new PlaneGeometry(screenWidth, screenHeight, 40, 48);
189 const screenPositions = screenGeometry.getAttribute("position");
190 const screenUvs = screenGeometry.getAttribute("uv");
191 const baseX = new Float32Array(screenPositions.count);
192 for (let i = 0; i < screenPositions.count; i++) {
193 const y = screenPositions.getY(i);
194 const outerX = screenWidth / 2;
195 const outerY = screenHeight / 2;
196 const radius = 0.076;
197 const cornerY = Math.max(0, Math.abs(y) - (outerY - radius));
198 const limit = outerX - radius + Math.sqrt(Math.max(0, radius * radius - cornerY * cornerY));
199 const x = Math.max(-limit, Math.min(limit, screenPositions.getX(i)));
200 baseX[i] = x;
201 screenUvs.setXY(i, x / screenWidth + 0.5, y / screenHeight + 0.5);
202 }
203 screenUvs.needsUpdate = true;
204 const innerSurface = new Mesh(screenGeometry, displayMaterial);
205 innerSurface.name = "continuous-inner-screen";
206 orientation.add(innerSurface);
207 // The outer half of the hinge housing. It tucks behind the back glass when
208 // open and becomes the rounded spine when closed.
209 const spineSlack = 0.15;
210 const spine = new Mesh(
211 new CylinderGeometry(
212 SPINE_RADIUS,
213 SPINE_RADIUS,
214 HEIGHT - 0.012,
215 32,
216 1,
217 false,
218 Math.PI / 2 + spineSlack,
219 Math.PI - spineSlack * 2,
220 ),
221 polishedMetal,
222 );
223 spine.name = "hinge-spine";
224 spine.position.z = PIVOT_Z;
225 orientation.add(spine);
226
227 const cover = new Mesh(new ShapeGeometry(coverPath()), coverMaterial);
228 cover.name = "cover-screen";
229 cover.position.set(-HALF_WIDTH / 2, 0, -DEPTH / 2 - 0.003);
230 cover.rotation.y = Math.PI;
231 leftBody.add(cover);
232
233 // Rear components use back-surface coordinates, with outward positive Z.
234 const rearCamera = new Group();
235 rearCamera.name = "rear-camera";
236 const islandWidth = 0.46;
237 const islandHeight = 0.2;
238 rearCamera.position.set(
239 HALF_WIDTH - 0.07 - islandWidth / 2,
240 HEIGHT / 2 - 0.08 - islandHeight / 2,
241 -DEPTH / 2 - 0.002,
242 );
243 rearCamera.rotation.y = Math.PI;
244 right.add(rearCamera);
245 const plateDepth = 0.02;
246 const plate = new Mesh(
247 new ExtrudeGeometry(roundedRectPath(islandWidth, islandHeight, 0.07), {
248 depth: plateDepth,
249 bevelEnabled: true,
250 bevelSize: 0.008,
251 bevelThickness: 0.006,
252 bevelSegments: 3,
253 curveSegments: 12,
254 }),
255 island,
256 );
257 plate.name = "camera-plate";
258 rearCamera.add(plate);
259 const plateFront = plateDepth + 0.006;
260 for (const [x, radius] of [
261 [-0.14, 0.05],
262 [-0.01, 0.05],
263 [0.105, 0.036],
264 ] as const) {
265 const ring = new Mesh(
266 new CylinderGeometry(radius + 0.012, radius + 0.012, 0.012, 32),
267 frameMetal,
268 );
269 ring.rotation.x = Math.PI / 2;
270 ring.position.set(x, 0, plateFront + 0.004);
271 rearCamera.add(ring);
272 const lens = new Mesh(new CircleGeometry(radius, 32), lensMaterial);
273 lens.name = "camera-lens";
274 lens.position.set(x, 0, plateFront + 0.0105);
275 rearCamera.add(lens);
276 }
277 const flash = new Mesh(new CircleGeometry(0.018, 20), flashMaterial);
278 flash.position.set(0.185, 0.045, plateFront + 0.0005);
279 rearCamera.add(flash);
280
281 // Power and volume keys sit on the fixed half's outer edge.
282 for (const [y, length] of [
283 [0.52, 0.16],
284 [0.2, 0.3],
285 ] as const) {
286 const key = new Mesh(new BoxGeometry(0.02, length, DEPTH * 0.45), frameMetal);
287 key.position.set(HALF_WIDTH + 0.008, y, 0);
288 right.add(key);
289 }
290
291 const raycaster = new Raycaster();
292 const pointer = new Vector2();
293 const local = new Vector3();
294 let capturedDisplay: Mesh | null = null;
295 let activeLayout = layout;
296 let angle = initialAngle;
297 const updateVisibleScreen = () => {
298 const innerActive = angle >= 90;
299 innerSurface.visible = innerActive;
300 innerLeft.visible = innerActive;
301 innerRight.visible = innerActive;
302 cover.visible = !innerActive;
303 };
304 const setAngle = (next: number) => {
305 angle = Math.max(0, Math.min(180, next));
306 left.rotation.y = Math.PI * (1 - angle / 180);
307 spine.rotation.y = left.rotation.y / 2;
308 const radians = left.rotation.y;
309 const cosine = Math.cos(radians);
310 const sine = Math.sin(radians);
311 const frontZ = DEPTH / 2 + 0.004 - PIVOT_Z;
312 for (let i = 0; i < screenPositions.count; i++) {
313 const x = baseX[i]!;
314 if (x < 0) {
315 screenPositions.setXYZ(
316 i,
317 x * cosine + frontZ * sine,
318 screenPositions.getY(i),
319 -x * sine + frontZ * cosine + PIVOT_Z,
320 );
321 } else {
322 screenPositions.setXYZ(i, x, screenPositions.getY(i), frontZ + PIVOT_Z);
323 }
324 }
325 screenPositions.needsUpdate = true;
326 screenGeometry.computeBoundingBox();
327 screenGeometry.computeBoundingSphere();
328 updateVisibleScreen();
329 };
330 const setDisplay = (nextTexture: Texture, nextLayout: PhoneDisplayLayout) => {
331 activeLayout = nextLayout;
332 displayMaterial.map = nextTexture;
333 coverMaterial.map = nextTexture;
334 cover.geometry.computeBoundingBox();
335 const bounds = cover.geometry.boundingBox!;
336 const uv = cover.geometry.getAttribute("uv");
337 const position = cover.geometry.getAttribute("position");
338 for (let i = 0; i < uv.count; i++) {
339 const u = (position.getX(i) - bounds.min.x) / (bounds.max.x - bounds.min.x);
340 const v = (position.getY(i) - bounds.min.y) / (bounds.max.y - bounds.min.y);
341 uv.setXY(i, u, v);
342 }
343 uv.needsUpdate = true;
344 updateVisibleScreen();
345 };
346 setAngle(initialAngle);
347 setDisplay(texture, layout);
348
349 return {
350 root,
351 orientation,
352 width: HALF_WIDTH * 2,
353 height: HEIGHT,
354 setAngle,
355 setDisplay,
356 screenPoint(x: number, y: number, camera: Camera, captured = false) {
357 orientation.updateWorldMatrix(true, true);
358 camera.updateMatrixWorld(true);
359 pointer.set(x * 2 - 1, 1 - y * 2);
360 raycaster.setFromCamera(pointer, camera);
361 const screens = cover.visible ? [cover] : [innerLeft, innerRight];
362 const hit = raycaster.intersectObjects(screens, false)[0];
363 if (!hit && !captured) {
364 capturedDisplay = null;
365 return null;
366 }
367 const display =
368 (hit?.object as Mesh | undefined) ??
369 (capturedDisplay?.visible ? capturedDisplay : screens[0]);
370 if (!display) return null;
371 if (hit) capturedDisplay = display;
372 if (hit) local.copy(hit.point);
373 else {
374 const plane = new Vector3(0, 0, 1).transformDirection(display.matrixWorld);
375 const point = new Vector3().setFromMatrixPosition(display.matrixWorld);
376 const distance =
377 plane.dot(point.clone().sub(raycaster.ray.origin)) / plane.dot(raycaster.ray.direction);
378 if (!Number.isFinite(distance)) return null;
379 local.copy(raycaster.ray.direction).multiplyScalar(distance).add(raycaster.ray.origin);
380 }
381 display.worldToLocal(local);
382 const bounds = display.geometry.boundingBox!;
383 const u = Math.max(0, Math.min(1, (local.x - bounds.min.x) / (bounds.max.x - bounds.min.x)));
384 const v = Math.max(0, Math.min(1, (bounds.max.y - local.y) / (bounds.max.y - bounds.min.y)));
385 const across = display === cover ? u : (display === innerLeft ? u : 1 + u) / 2;
386 return activeLayout.rotation === Math.PI ? { x: 1 - across, y: 1 - v } : { x: across, y: v };
387 },
388 dispose() {
389 root.traverse((object) => {
390 if (object instanceof Mesh) object.geometry.dispose();
391 });
392 for (const material of materials) material.dispose();
393 },
394 };
395}
packages/client-runtime/src/device/phoneViewer.test.ts+127-3
......@@ -11,6 +11,7 @@ const gpu = vi.hoisted(() => ({
1111 scene: Scene;
1212 phone: Object3D | undefined;
1313 rotation: Quaternion | undefined;
14 displayAngle: number | undefined;
1415 yaw: number | undefined;
1516 cameraZ: number;
1617 }[];
......@@ -50,6 +51,7 @@ vi.mock("three", async () => {
5051 scene,
5152 phone,
5253 rotation: phone?.quaternion.clone(),
54 displayAngle: phone?.children[0]?.rotation.z,
5355 yaw: phone?.rotation.y,
5456 cameraZ: camera.position.z,
5557 });
......@@ -81,7 +83,12 @@ vi.mock("./modelScene.ts", async () => {
8183import { BoxGeometry, Group, Mesh, MeshBasicMaterial, PlaneGeometry, Quaternion } from "three";
8284import { disposeDeviceModel } from "./modelScene.ts";
8385import { createPhoneViewer } from "./phoneViewer.ts";
84import { IOS_TABLET_SHAPE } from "./shapeProfile.ts";
86import {
87 ANDROID_PHONE_SHAPE,
88 IOS_TABLET_SHAPE,
89 resolveDeviceShape,
90 type DeviceShapeProfile,
91} from "./shapeProfile.ts";
8592
8693afterEach(() => {
8794 vi.restoreAllMocks();
......@@ -90,7 +97,7 @@ afterEach(() => {
9097 models.pending.length = 0;
9198});
9299
93function fixture() {
100function fixture(profile?: DeviceShapeProfile) {
94101 const pending = new Map<number, FrameRequestCallback>();
95102 let id = 0;
96103 let now = 0;
......@@ -104,7 +111,13 @@ function fixture() {
104111 const source = { width: 1206, height: 2622 } as HTMLCanvasElement;
105112 const onUnavailable = vi.fn();
106113 const onFramingAspect = vi.fn();
107 const viewer = createPhoneViewer({ canvas, source, onUnavailable, onFramingAspect });
114 const viewer = createPhoneViewer({
115 canvas,
116 source,
117 onUnavailable,
118 onFramingAspect,
119 ...(profile ? { profile } : {}),
120 });
108121 const draw = (time = now) => {
109122 now = time;
110123 const callbacks = [...pending.values()];
......@@ -194,6 +207,117 @@ it("changes device shape without replacing the renderer, decoded source or pose"
194207 viewer.dispose();
195208});
196209
210it("keeps the Android viewer while the resized framebuffer turns between fold postures", () => {
211 const openProfile = resolveDeviceShape({ platform: "android", portraitAspect: 0.96 });
212 const { viewer, draw, source, state } = fixture(openProfile);
213 const scene = state.frames.at(-1)!.scene;
214 source.width = 2076;
215 source.height = 2152;
216 viewer.setScreen({ width: 2076, height: 2152, orientation: "landscape_left" });
217 viewer.frameUpdated();
218 draw(0);
219 expect(state.frames.at(-1)!.displayAngle).toBeCloseTo(0);
220 draw(225);
221 expect(state.frames.at(-1)!.displayAngle).toBeCloseTo(-Math.PI / 4);
222 draw(450);
223 expect(state.frames.at(-1)!.displayAngle).toBeCloseTo(-Math.PI / 2);
224 expect(state.frames.at(-1)!.scene).toBe(scene);
225 expect(gpu.instances).toHaveLength(1);
226
227 source.width = 1080;
228 source.height = 2424;
229 viewer.setScreen({ width: 1080, height: 2424, orientation: "portrait" }, ANDROID_PHONE_SHAPE);
230 viewer.frameUpdated();
231 draw(450);
232 expect(state.frames.at(-1)!.displayAngle).toBeCloseTo(-Math.PI / 2);
233 draw(675);
234 expect(state.frames.at(-1)!.displayAngle).toBeCloseTo(-Math.PI / 4);
235 draw(900);
236 expect(state.frames.at(-1)!.displayAngle).toBeCloseTo(0);
237 expect(state.frames.at(-1)!.scene).toBe(scene);
238 viewer.dispose();
239});
240
241it("animates the Android hinge on the same scene through an encoder resize", () => {
242 const openProfile = resolveDeviceShape({ platform: "android", portraitAspect: 0.96 });
243 const { viewer, draw, source, state } = fixture(openProfile);
244 viewer.setFoldAngle(180);
245 draw(0);
246 const shell = state.frames.at(-1)!.phone!;
247 const moving = shell.children[0]!.children[0]!;
248 viewer.setFoldAngle(0);
249 draw(425);
250 expect(moving.rotation.y).toBeCloseTo(Math.PI / 2);
251 source.width = 1080;
252 source.height = 2424;
253 viewer.setScreen({ width: 1080, height: 2424, orientation: "portrait" }, ANDROID_PHONE_SHAPE);
254 viewer.frameUpdated();
255 draw(850);
256 expect(moving.rotation.y).toBeCloseTo(Math.PI);
257 expect(state.frames.at(-1)!.phone).toBe(shell);
258 expect(gpu.instances).toHaveLength(1);
259 viewer.dispose();
260});
261
262it("retargets an unfinished hinge turn from its visible angle", () => {
263 const { viewer, draw, state } = fixture(ANDROID_PHONE_SHAPE);
264 viewer.setFoldAngle(180);
265 draw(0);
266 const moving = state.frames.at(-1)!.phone!.children[0]!.children[0]!;
267 viewer.setFoldAngle(0);
268 draw(200);
269 const visibleAngle = moving.rotation.y;
270 viewer.setFoldAngle(180);
271 draw(200);
272 expect(moving.rotation.y).toBeCloseTo(visibleAngle);
273 draw(1050);
274 expect(moving.rotation.y).toBeCloseTo(0);
275 viewer.dispose();
276});
277
278it("stops a hinge turn when a loaded model replaces the fold scene", async () => {
279 const { viewer, draw, pending } = fixture(ANDROID_PHONE_SHAPE);
280 viewer.setFoldAngle(180);
281 draw(0);
282 viewer.setFoldAngle(0);
283 viewer.setModel({ id: "iphone-18-pro", url: "/fold.glb" });
284 const asset = new Group();
285 const body = new Mesh(new BoxGeometry(1, 2, 0.1), new MeshBasicMaterial());
286 const display = new Mesh(new PlaneGeometry(0.9, 1.9), new MeshBasicMaterial());
287 display.name = "device-screen";
288 asset.add(body, display);
289 models.pending[0]!.resolve({ asset, dispose: () => disposeDeviceModel(asset) });
290 await Promise.resolve();
291 draw(200);
292 draw(1050);
293 expect(pending.size).toBe(0);
294 viewer.dispose();
295});
296
297it("keeps a loaded model when the fold angle changes and releases it once", async () => {
298 const { viewer, draw, state } = fixture(ANDROID_PHONE_SHAPE);
299 viewer.setModel({ id: "iphone-18-pro", url: "/pro.glb" });
300 const asset = new Group();
301 const body = new Mesh(new BoxGeometry(1.15, 2.3, 0.1), new MeshBasicMaterial());
302 body.position.z = -0.02;
303 const display = new Mesh(new PlaneGeometry(1, 2.2), new MeshBasicMaterial());
304 display.geometry.translate(0, 0, 0.043);
305 display.name = "device-screen";
306 asset.add(body, display);
307 const dispose = vi.fn(() => disposeDeviceModel(asset));
308 models.pending[0]!.resolve({ asset, dispose });
309 await Promise.resolve();
310 draw();
311 const loaded = state.frames.at(-1)!.phone;
312 expect(loaded?.getObjectByName("device-screen")).toBe(display);
313 viewer.setFoldAngle(180);
314 viewer.setFoldAngle(0);
315 draw();
316 expect(state.frames.at(-1)!.phone).toBe(loaded);
317 viewer.dispose();
318 expect(dispose).toHaveBeenCalledOnce();
319});
320
197321it("retains the loaded model and pose through rotation and framebuffer resolution changes, then releases it once", async () => {
198322 const { viewer, draw, source, state } = fixture();
199323 viewer.orbit(0.08, 0.04);
packages/client-runtime/src/device/phoneViewer.ts+101-7
......@@ -19,6 +19,7 @@ import {
1919} from "./model.ts";
2020import { createImportedPhoneScene, loadDeviceModel } from "./modelScene.ts";
2121import { createPhoneScene, phoneDisplayLayout } from "./phoneScene.ts";
22import { createAndroidFoldScene } from "./androidFoldScene.ts";
2223import { createRenderScheduler } from "./renderScheduler.ts";
2324import { createDeviceMotion } from "./deviceMotion.ts";
2425import { createDeviceFraming } from "./deviceFraming.ts";
......@@ -31,6 +32,7 @@ export interface PhoneViewer {
3132 readonly setAccessory: (source: DeviceAccessorySource | null) => void;
3233 readonly frameUpdated: () => void;
3334 readonly setScreen: (screen: DeviceScreenSize | null, profile?: DeviceShapeProfile) => void;
35 readonly setFoldAngle: (angle: number | null) => void;
3436 readonly resize: (width: number, height: number, pixelRatio: number) => void;
3537 readonly screenPoint: (
3638 x: number,
......@@ -43,6 +45,9 @@ export interface PhoneViewer {
4345 readonly dispose: () => void;
4446}
4547
48const ANDROID_ORIENTATION_TURN_MS = 450;
49const ANDROID_FOLD_TURN_MS = 850;
50
4651/** Owns only presentation resources. The caller retains the decoded canvas and the stream connection. */
4752export function createPhoneViewer(options: {
4853 readonly canvas: HTMLCanvasElement;
......@@ -53,6 +58,7 @@ export function createPhoneViewer(options: {
5358 readonly profile?: DeviceShapeProfile;
5459 readonly model?: DeviceModelSource | null;
5560 readonly accessory?: DeviceAccessorySource | null;
61 readonly foldAngle?: number | null;
5662}): PhoneViewer {
5763 const renderer = new WebGLRenderer({
5864 canvas: options.canvas,
......@@ -87,11 +93,19 @@ export function createPhoneViewer(options: {
8793 let screen: DeviceScreenSize | null = null;
8894 let layout = phoneDisplayLayout(screen, options.source.width, options.source.height);
8995 let profile = options.profile ?? IOS_PHONE_SHAPE;
96 let foldAngle = options.foldAngle ?? null;
97 let orientationAngle =
98 foldAngle !== null && profile.id.startsWith("android") ? 0 : layout.rotation;
99 let orientationTurn: { from: number; to: number; startedAt: number } | null = null;
90100 let imported: Awaited<ReturnType<typeof loadDeviceModel>> | null = null;
91101 let modelSource = options.model ?? null;
92102 let accessory: Awaited<ReturnType<typeof loadDeviceModel>> | null = null;
93103 let accessoryBounds: Box3 | null = null;
94 let phone = createPhoneScene(texture, layout, profile);
104 let foldTurn: { from: number; to: number; startedAt: number } | null = null;
105 let phone: ReturnType<typeof createPhoneScene> | ReturnType<typeof createAndroidFoldScene> =
106 foldAngle !== null && profile.id.startsWith("android")
107 ? createAndroidFoldScene(texture, layout, foldAngle)
108 : createPhoneScene(texture, layout, profile);
95109 scene.add(phone.root);
96110 let disposed = false;
97111 const rest = new Quaternion();
......@@ -116,7 +130,7 @@ export function createPhoneViewer(options: {
116130 new Vector3(phone.width / 2, phone.height / 2, 0),
117131 );
118132 if (imported && accessoryBounds) bounds.union(accessoryBounds);
119 bounds.applyMatrix4(new Matrix4().makeRotationZ(layout.rotation));
133 bounds.applyMatrix4(new Matrix4().makeRotationZ(orientationAngle));
120134 const size = bounds.getSize(new Vector3());
121135 const aspect = size.x / size.y;
122136 if (aspect !== framingAspect) {
......@@ -140,7 +154,7 @@ export function createPhoneViewer(options: {
140154 };
141155 const applyPose = () => {
142156 phone.root.quaternion.copy(motion.rotation);
143 phone.orientation.rotation.z = layout.rotation;
157 phone.orientation.rotation.z = orientationAngle;
144158 };
145159 const scheduler = createRenderScheduler(() => {
146160 if (disposed || !viewport.width || !viewport.height) return;
......@@ -160,10 +174,30 @@ export function createPhoneViewer(options: {
160174 applyPose();
161175 fit(reducedMotion());
162176 }
177 if (orientationTurn) {
178 const progress = Math.min(
179 1,
180 (now - orientationTurn.startedAt) / ANDROID_ORIENTATION_TURN_MS,
181 );
182 const eased = progress * progress * (3 - 2 * progress);
183 orientationAngle =
184 orientationTurn.from + (orientationTurn.to - orientationTurn.from) * eased;
185 if (progress === 1) orientationTurn = null;
186 applyPose();
187 fit(reducedMotion());
188 }
189 if (foldTurn && "setAngle" in phone) {
190 const progress = Math.min(1, (now - foldTurn.startedAt) / ANDROID_FOLD_TURN_MS);
191 const eased = progress * progress * (3 - 2 * progress);
192 phone.setAngle(foldTurn.from + (foldTurn.to - foldTurn.from) * eased);
193 if (progress === 1) foldTurn = null;
194 fit(reducedMotion());
195 }
163196 framing.advance(now, reducedMotion());
164197 applyCamera();
165198 renderer.render(scene, camera);
166 if (motion.needsFrame() || framing.needsFrame()) scheduler.invalidate();
199 if (motion.needsFrame() || framing.needsFrame() || orientationTurn || foldTurn)
200 scheduler.invalidate();
167201 } catch {
168202 options.onUnavailable();
169203 }
......@@ -188,7 +222,11 @@ export function createPhoneViewer(options: {
188222 phone.setDisplay(texture, next);
189223 previous.dispose();
190224 }
191 if (!imported && (nextProfile !== profile || next.aspect !== layout.aspect)) {
225 if (
226 !imported &&
227 !("setAngle" in phone) &&
228 (nextProfile !== profile || next.aspect !== layout.aspect)
229 ) {
192230 scene.remove(phone.root);
193231 phone.dispose();
194232 phone = createPhoneScene(texture, next, nextProfile);
......@@ -196,10 +234,26 @@ export function createPhoneViewer(options: {
196234 } else {
197235 phone.setDisplay(texture, next);
198236 }
237 if (next.rotation !== layout.rotation) {
238 if (nextProfile.id.startsWith("android") && !("setAngle" in phone) && !reducedMotion()) {
239 const difference = Math.atan2(
240 Math.sin(next.rotation - orientationAngle),
241 Math.cos(next.rotation - orientationAngle),
242 );
243 orientationTurn = {
244 from: orientationAngle,
245 to: orientationAngle + difference,
246 startedAt: performance.now(),
247 };
248 } else {
249 orientationTurn = null;
250 orientationAngle = "setAngle" in phone ? 0 : next.rotation;
251 }
252 }
199253 layout = next;
200254 profile = nextProfile;
201255 applyPose();
202 fit(true);
256 fit(!orientationTurn);
203257 }
204258 applyPose();
205259 };
......@@ -212,7 +266,10 @@ export function createPhoneViewer(options: {
212266 ? null
213267 : model
214268 ? createImportedPhoneScene(model.asset, texture, layout)
215 : createPhoneScene(texture, layout, profile);
269 : foldAngle !== null && profile.id.startsWith("android")
270 ? createAndroidFoldScene(texture, layout, foldAngle)
271 : createPhoneScene(texture, layout, profile);
272 foldTurn = null;
216273 scene.remove(phone.root);
217274 phone.dispose();
218275 imported = model;
......@@ -267,6 +324,43 @@ export function createPhoneViewer(options: {
267324 modelSlot.set(source);
268325 },
269326 setAccessory,
327 setFoldAngle(next) {
328 if (disposed || next === foldAngle) return;
329 const previous = foldAngle;
330 foldAngle = next;
331 // A loaded model owns the scene; install() reads foldAngle if it is removed.
332 if (imported) return;
333 if (next === null || !("setAngle" in phone)) {
334 scene.remove(phone.root);
335 phone.dispose();
336 phone =
337 next === null
338 ? createPhoneScene(texture, layout, profile)
339 : createAndroidFoldScene(texture, layout, next);
340 scene.add(phone.root);
341 orientationTurn = null;
342 orientationAngle = next === null ? layout.rotation : 0;
343 foldTurn = null;
344 applyPose();
345 fit(true);
346 } else {
347 const progress = foldTurn
348 ? Math.min(1, (performance.now() - foldTurn.startedAt) / ANDROID_FOLD_TURN_MS)
349 : 1;
350 const eased = progress * progress * (3 - 2 * progress);
351 const from = foldTurn
352 ? foldTurn.from + (foldTurn.to - foldTurn.from) * eased
353 : (previous ?? next);
354 if (reducedMotion()) {
355 foldTurn = null;
356 phone.setAngle(next);
357 fit(true);
358 } else {
359 foldTurn = { from, to: next, startedAt: performance.now() };
360 }
361 }
362 scheduler.invalidate();
363 },
270364 frameUpdated() {
271365 if (disposed) return;
272366 updateLayout();