authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-09-24 22:14:36-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-24 22:14:36-07:00
log7b84431161a64c9f9e9b57637dec23474a1b4236
tree56066b7cc10da759a55908bb16af217d3197c7bb
parentdf9826f0806cc4874cd79f24a42a1e9e506030c2
signature Signed by PGP key B5690EEEBB952194

fix(web): size the Android fold model from the inner display (#13574)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

4 files changed, 154 insertions(+), 34 deletions(-)

packages/client-runtime/src/device/androidFoldScene.test.ts+21
...@@ -54,4 +54,25 @@ describe("Android fold scene", () => {...@@ -54,4 +54,25 @@ describe("Android fold scene", () => {
54 scene.dispose();54 scene.dispose();
55 texture.dispose();55 texture.dispose();
56 });56 });
57
58 it("shapes the inner display to the raw frame, portrait or landscape", () => {
59 const texture = new Texture();
60 for (const [width, height] of [
61 [2076, 2152],
62 [2208, 1840],
63 ] as const) {
64 const scene = createAndroidFoldScene(
65 texture,
66 phoneDisplayLayout(null, width, height),
67 180,
68 width / height,
69 );
70 const screen = scene.root.getObjectByName("continuous-inner-screen") as Mesh;
71 screen.geometry.computeBoundingBox();
72 const size = screen.geometry.boundingBox!.getSize(new Vector3());
73 expect(size.x / size.y).toBeCloseTo(width / height, 2);
74 scene.dispose();
75 }
76 texture.dispose();
77 });
57});78});
packages/client-runtime/src/device/androidFoldScene.ts+43-18
...@@ -19,7 +19,6 @@ import {...@@ -19,7 +19,6 @@ import {
19} from "three";19} from "three";
20import type { PhoneDisplayLayout } from "./phoneScene.ts";20import type { PhoneDisplayLayout } from "./phoneScene.ts";
2121
22const HALF_WIDTH = 1.04;
23const HEIGHT = 2.2;22const HEIGHT = 2.2;
24const DEPTH = 0.075;23const DEPTH = 0.075;
25const INSET = 0.026;24const INSET = 0.026;
...@@ -28,10 +27,24 @@ const BEVEL = 0.008;...@@ -28,10 +27,24 @@ const BEVEL = 0.008;
28// The hinge axis sits just above the inner screens so closed halves meet face to face.27// The hinge axis sits just above the inner screens so closed halves meet face to face.
29const PIVOT_Z = DEPTH / 2 + 0.005;28const PIVOT_Z = DEPTH / 2 + 0.005;
30const SPINE_RADIUS = PIVOT_Z + DEPTH / 2 - 0.002;29const SPINE_RADIUS = PIVOT_Z + DEPTH / 2 - 0.002;
30/** Width over height of the unfolded inner display until a live frame reports its own. */
31export const DEFAULT_FOLD_INNER_ASPECT = 2076 / 2152;
32/**
33 * Unfolded inner displays are near square in either orientation. Cover displays are
34 * phone shaped (about 0.4-0.5, or 2-2.6 when rotated), so they never retune the body.
35 */
36export const isFoldInnerAspect = (aspect: number) =>
37 Number.isFinite(aspect) && aspect > 0.75 && aspect < 1.5;
3138
32function panelPath(side: "left" | "right", inset: number, radius: number, hingeInset = 0) {39function panelPath(
33 const left = side === "left" ? -HALF_WIDTH + inset : CREASE / 2 + hingeInset;40 halfWidth: number,
34 const right = side === "left" ? -CREASE / 2 - hingeInset : HALF_WIDTH - inset;41 side: "left" | "right",
42 inset: number,
43 radius: number,
44 hingeInset = 0,
45) {
46 const left = side === "left" ? -halfWidth + inset : CREASE / 2 + hingeInset;
47 const right = side === "left" ? -CREASE / 2 - hingeInset : halfWidth - inset;
35 const bottom = -HEIGHT / 2 + inset;48 const bottom = -HEIGHT / 2 + inset;
36 const top = HEIGHT / 2 - inset;49 const top = HEIGHT / 2 - inset;
37 const path = new Shape();50 const path = new Shape();
...@@ -70,16 +83,22 @@ function roundedRectPath(width: number, height: number, radius: number) {...@@ -70,16 +83,22 @@ function roundedRectPath(width: number, height: number, radius: number) {
70 return path;83 return path;
71}84}
7285
73function coverPath() {86function coverPath(halfWidth: number) {
74 return roundedRectPath(HALF_WIDTH - INSET * 2 - 0.02, HEIGHT - INSET * 2 - 0.04, 0.07);87 return roundedRectPath(halfWidth - INSET * 2 - 0.02, HEIGHT - INSET * 2 - 0.04, 0.07);
75}88}
7689
77/** A procedural book-style foldable: one fixed half, one half rotating around a shared hinge. */90/**
91 * A procedural book-style foldable: one fixed half, one half rotating around a shared hinge.
92 * The inner display keeps the raw framebuffer's native aspect, portrait or landscape.
93 */
78export function createAndroidFoldScene(94export function createAndroidFoldScene(
79 texture: Texture,95 texture: Texture,
80 layout: PhoneDisplayLayout,96 layout: PhoneDisplayLayout,
81 initialAngle: number,97 initialAngle: number,
98 innerAspect = DEFAULT_FOLD_INNER_ASPECT,
82) {99) {
100 const screenHeight = HEIGHT - 2 * INSET;
101 const halfWidth = (innerAspect * screenHeight) / 2 + INSET;
83 const root = new Group();102 const root = new Group();
84 const orientation = new Group();103 const orientation = new Group();
85 root.add(orientation);104 root.add(orientation);
...@@ -146,7 +165,7 @@ export function createAndroidFoldScene(...@@ -146,7 +165,7 @@ export function createAndroidFoldScene(
146165
147 function half(group: Group, side: "left" | "right", back: MeshPhysicalMaterial) {166 function half(group: Group, side: "left" | "right", back: MeshPhysicalMaterial) {
148 const body = new Mesh(167 const body = new Mesh(
149 new ExtrudeGeometry(panelPath(side, BEVEL, 0.1, BEVEL), {168 new ExtrudeGeometry(panelPath(halfWidth, side, BEVEL, 0.1, BEVEL), {
150 depth: DEPTH - BEVEL * 2,169 depth: DEPTH - BEVEL * 2,
151 bevelEnabled: true,170 bevelEnabled: true,
152 bevelSize: BEVEL,171 bevelSize: BEVEL,
...@@ -158,19 +177,25 @@ export function createAndroidFoldScene(...@@ -158,19 +177,25 @@ export function createAndroidFoldScene(
158 );177 );
159 body.position.z = -DEPTH / 2 + BEVEL;178 body.position.z = -DEPTH / 2 + BEVEL;
160 group.add(body);179 group.add(body);
161 const frame = new Mesh(new ShapeGeometry(panelPath(side, 0.01, 0.095, 0.002), 12), bezel);180 const frame = new Mesh(
181 new ShapeGeometry(panelPath(halfWidth, side, 0.01, 0.095, 0.002), 12),
182 bezel,
183 );
162 frame.position.z = DEPTH / 2 + 0.001;184 frame.position.z = DEPTH / 2 + 0.001;
163 group.add(frame);185 group.add(frame);
164 // A back-facing shape mirrors X, so it is drawn from the opposite side's outline.186 // A back-facing shape mirrors X, so it is drawn from the opposite side's outline.
165 const rear = new Mesh(187 const rear = new Mesh(
166 new ShapeGeometry(panelPath(side === "left" ? "right" : "left", 0.01, 0.095, 0.002), 12),188 new ShapeGeometry(
189 panelPath(halfWidth, side === "left" ? "right" : "left", 0.01, 0.095, 0.002),
190 12,
191 ),
167 back,192 back,
168 );193 );
169 rear.name = `${side}-back`;194 rear.name = `${side}-back`;
170 rear.rotation.y = Math.PI;195 rear.rotation.y = Math.PI;
171 rear.position.set(0, 0, -DEPTH / 2 - 0.001);196 rear.position.set(0, 0, -DEPTH / 2 - 0.001);
172 group.add(rear);197 group.add(rear);
173 const geometry = new ShapeGeometry(panelPath(side, INSET, 0.076));198 const geometry = new ShapeGeometry(panelPath(halfWidth, side, INSET, 0.076));
174 geometry.computeBoundingBox();199 geometry.computeBoundingBox();
175 const display = new Mesh(geometry, hitMaterial);200 const display = new Mesh(geometry, hitMaterial);
176 display.name = `${side}-inner-screen`;201 display.name = `${side}-inner-screen`;
...@@ -183,8 +208,7 @@ export function createAndroidFoldScene(...@@ -183,8 +208,7 @@ export function createAndroidFoldScene(
183 const innerRight = half(right, "right", backGlass);208 const innerRight = half(right, "right", backGlass);
184 // One indexed surface keeps adjacent pixels joined at the crease. The209 // One indexed surface keeps adjacent pixels joined at the crease. The
185 // physical halves move separately underneath it.210 // physical halves move separately underneath it.
186 const screenWidth = 2 * (HALF_WIDTH - INSET);211 const screenWidth = 2 * (halfWidth - INSET);
187 const screenHeight = HEIGHT - 2 * INSET;
188 const screenGeometry = new PlaneGeometry(screenWidth, screenHeight, 40, 48);212 const screenGeometry = new PlaneGeometry(screenWidth, screenHeight, 40, 48);
189 const screenPositions = screenGeometry.getAttribute("position");213 const screenPositions = screenGeometry.getAttribute("position");
190 const screenUvs = screenGeometry.getAttribute("uv");214 const screenUvs = screenGeometry.getAttribute("uv");
...@@ -224,9 +248,9 @@ export function createAndroidFoldScene(...@@ -224,9 +248,9 @@ export function createAndroidFoldScene(
224 spine.position.z = PIVOT_Z;248 spine.position.z = PIVOT_Z;
225 orientation.add(spine);249 orientation.add(spine);
226250
227 const cover = new Mesh(new ShapeGeometry(coverPath()), coverMaterial);251 const cover = new Mesh(new ShapeGeometry(coverPath(halfWidth)), coverMaterial);
228 cover.name = "cover-screen";252 cover.name = "cover-screen";
229 cover.position.set(-HALF_WIDTH / 2, 0, -DEPTH / 2 - 0.003);253 cover.position.set(-halfWidth / 2, 0, -DEPTH / 2 - 0.003);
230 cover.rotation.y = Math.PI;254 cover.rotation.y = Math.PI;
231 leftBody.add(cover);255 leftBody.add(cover);
232256
...@@ -236,7 +260,7 @@ export function createAndroidFoldScene(...@@ -236,7 +260,7 @@ export function createAndroidFoldScene(
236 const islandWidth = 0.46;260 const islandWidth = 0.46;
237 const islandHeight = 0.2;261 const islandHeight = 0.2;
238 rearCamera.position.set(262 rearCamera.position.set(
239 HALF_WIDTH - 0.07 - islandWidth / 2,263 halfWidth - 0.07 - islandWidth / 2,
240 HEIGHT / 2 - 0.08 - islandHeight / 2,264 HEIGHT / 2 - 0.08 - islandHeight / 2,
241 -DEPTH / 2 - 0.002,265 -DEPTH / 2 - 0.002,
242 );266 );
...@@ -284,7 +308,7 @@ export function createAndroidFoldScene(...@@ -284,7 +308,7 @@ export function createAndroidFoldScene(
284 [0.2, 0.3],308 [0.2, 0.3],
285 ] as const) {309 ] as const) {
286 const key = new Mesh(new BoxGeometry(0.02, length, DEPTH * 0.45), frameMetal);310 const key = new Mesh(new BoxGeometry(0.02, length, DEPTH * 0.45), frameMetal);
287 key.position.set(HALF_WIDTH + 0.008, y, 0);311 key.position.set(halfWidth + 0.008, y, 0);
288 right.add(key);312 right.add(key);
289 }313 }
290314
...@@ -349,8 +373,9 @@ export function createAndroidFoldScene(...@@ -349,8 +373,9 @@ export function createAndroidFoldScene(
349 return {373 return {
350 root,374 root,
351 orientation,375 orientation,
352 width: HALF_WIDTH * 2,376 width: halfWidth * 2,
353 height: HEIGHT,377 height: HEIGHT,
378 innerAspect,
354 setAngle,379 setAngle,
355 setDisplay,380 setDisplay,
356 screenPoint(x: number, y: number, camera: Camera, captured = false) {381 screenPoint(x: number, y: number, camera: Camera, captured = false) {
packages/client-runtime/src/device/phoneViewer.test.ts+57-1
...@@ -80,7 +80,16 @@ vi.mock("./modelScene.ts", async () => {...@@ -80,7 +80,16 @@ vi.mock("./modelScene.ts", async () => {
80 };80 };
81});81});
8282
83import { BoxGeometry, Group, Mesh, MeshBasicMaterial, PlaneGeometry, Quaternion } from "three";83import {
84 Box3,
85 BoxGeometry,
86 Group,
87 Mesh,
88 MeshBasicMaterial,
89 PlaneGeometry,
90 Quaternion,
91 Vector3,
92} from "three";
84import { disposeDeviceModel } from "./modelScene.ts";93import { disposeDeviceModel } from "./modelScene.ts";
85import { createPhoneViewer } from "./phoneViewer.ts";94import { createPhoneViewer } from "./phoneViewer.ts";
86import {95import {
...@@ -259,6 +268,53 @@ it("animates the Android hinge on the same scene through an encoder resize", ()...@@ -259,6 +268,53 @@ it("animates the Android hinge on the same scene through an encoder resize", ()
259 viewer.dispose();268 viewer.dispose();
260});269});
261270
271it("resizes the fold body for a landscape inner display and keeps it through the cover frame", () => {
272 const openProfile = resolveDeviceShape({ platform: "android", portraitAspect: 0.83 });
273 const { viewer, draw, source, state } = fixture(openProfile);
274 viewer.setFoldAngle(180);
275 draw(0);
276 const portraitWidth = new Box3()
277 .setFromObject(state.frames.at(-1)!.phone!)
278 .getSize(new Vector3()).x;
279 source.width = 2208;
280 source.height = 1840;
281 viewer.setScreen({ width: 2208, height: 1840, orientation: "portrait" });
282 viewer.frameUpdated();
283 draw(10);
284 const landscape = state.frames.at(-1)!.phone!;
285 const landscapeWidth = new Box3().setFromObject(landscape).getSize(new Vector3()).x;
286 expect(landscapeWidth / portraitWidth).toBeGreaterThan(1.15);
287 source.width = 1080;
288 source.height = 2092;
289 viewer.setScreen({ width: 1080, height: 2092, orientation: "portrait" }, ANDROID_PHONE_SHAPE);
290 viewer.frameUpdated();
291 draw(20);
292 expect(state.frames.at(-1)!.phone).toBe(landscape);
293 expect(gpu.instances).toHaveLength(1);
294 viewer.dispose();
295});
296
297it("keeps the fold body through a rotated cover frame and learns the inner shape before fold mode", () => {
298 const { viewer, draw, source, state } = fixture(ANDROID_PHONE_SHAPE);
299 source.width = 2208;
300 source.height = 1840;
301 viewer.setScreen({ width: 2208, height: 1840, orientation: "portrait" });
302 viewer.frameUpdated();
303 draw(0);
304 viewer.setFoldAngle(180);
305 draw(10);
306 const landscape = state.frames.at(-1)!.phone!;
307 const width = new Box3().setFromObject(landscape).getSize(new Vector3()).x;
308 expect(width / new Box3().setFromObject(landscape).getSize(new Vector3()).y).toBeGreaterThan(1.1);
309 source.width = 2092;
310 source.height = 1080;
311 viewer.setScreen({ width: 2092, height: 1080, orientation: "landscape_left" });
312 viewer.frameUpdated();
313 draw(20);
314 expect(state.frames.at(-1)!.phone).toBe(landscape);
315 viewer.dispose();
316});
317
262it("retargets an unfinished hinge turn from its visible angle", () => {318it("retargets an unfinished hinge turn from its visible angle", () => {
263 const { viewer, draw, state } = fixture(ANDROID_PHONE_SHAPE);319 const { viewer, draw, state } = fixture(ANDROID_PHONE_SHAPE);
264 viewer.setFoldAngle(180);320 viewer.setFoldAngle(180);
packages/client-runtime/src/device/phoneViewer.ts+33-15
...@@ -19,7 +19,11 @@ import {...@@ -19,7 +19,11 @@ import {
19} from "./model.ts";19} from "./model.ts";
20import { createImportedPhoneScene, loadDeviceModel } from "./modelScene.ts";20import { createImportedPhoneScene, loadDeviceModel } from "./modelScene.ts";
21import { createPhoneScene, phoneDisplayLayout } from "./phoneScene.ts";21import { createPhoneScene, phoneDisplayLayout } from "./phoneScene.ts";
22import { createAndroidFoldScene } from "./androidFoldScene.ts";22import {
23 createAndroidFoldScene,
24 DEFAULT_FOLD_INNER_ASPECT,
25 isFoldInnerAspect,
26} from "./androidFoldScene.ts";
23import { createRenderScheduler } from "./renderScheduler.ts";27import { createRenderScheduler } from "./renderScheduler.ts";
24import { createDeviceMotion } from "./deviceMotion.ts";28import { createDeviceMotion } from "./deviceMotion.ts";
25import { createDeviceFraming } from "./deviceFraming.ts";29import { createDeviceFraming } from "./deviceFraming.ts";
...@@ -102,9 +106,21 @@ export function createPhoneViewer(options: {...@@ -102,9 +106,21 @@ export function createPhoneViewer(options: {
102 let accessory: Awaited<ReturnType<typeof loadDeviceModel>> | null = null;106 let accessory: Awaited<ReturnType<typeof loadDeviceModel>> | null = null;
103 let accessoryBounds: Box3 | null = null;107 let accessoryBounds: Box3 | null = null;
104 let foldTurn: { from: number; to: number; startedAt: number } | null = null;108 let foldTurn: { from: number; to: number; startedAt: number } | null = null;
109 // The inner display's raw width over height. Cover frames leave the last unfolded shape.
110 const rawAspect = () => options.source.width / options.source.height;
111 let foldAspect = isFoldInnerAspect(rawAspect()) ? rawAspect() : DEFAULT_FOLD_INNER_ASPECT;
112 const createFoldScene = (angle: number, displayLayout = layout) =>
113 createAndroidFoldScene(texture, displayLayout, angle, foldAspect);
114 /** The hinge angle currently on screen, including an unfinished turn. */
115 const visibleFoldAngle = (fallback: number) => {
116 if (!foldTurn) return fallback;
117 const progress = Math.min(1, (performance.now() - foldTurn.startedAt) / ANDROID_FOLD_TURN_MS);
118 const eased = progress * progress * (3 - 2 * progress);
119 return foldTurn.from + (foldTurn.to - foldTurn.from) * eased;
120 };
105 let phone: ReturnType<typeof createPhoneScene> | ReturnType<typeof createAndroidFoldScene> =121 let phone: ReturnType<typeof createPhoneScene> | ReturnType<typeof createAndroidFoldScene> =
106 foldAngle !== null && profile.id.startsWith("android")122 foldAngle !== null && profile.id.startsWith("android")
107 ? createAndroidFoldScene(texture, layout, foldAngle)123 ? createFoldScene(foldAngle)
108 : createPhoneScene(texture, layout, profile);124 : createPhoneScene(texture, layout, profile);
109 scene.add(phone.root);125 scene.add(phone.root);
110 let disposed = false;126 let disposed = false;
...@@ -222,7 +238,18 @@ export function createPhoneViewer(options: {...@@ -222,7 +238,18 @@ export function createPhoneViewer(options: {
222 phone.setDisplay(texture, next);238 phone.setDisplay(texture, next);
223 previous.dispose();239 previous.dispose();
224 }240 }
225 if (241 // Learn the inner display shape from any unfolded frame, including before fold mode.
242 const frameAspect = rawAspect();
243 const innerChanged = isFoldInnerAspect(frameAspect) && frameAspect !== foldAspect;
244 if (innerChanged) foldAspect = frameAspect;
245 if (!imported && "setAngle" in phone && innerChanged) {
246 // A new inner display shape resizes the body; the hinge keeps its visible angle.
247 const angle = visibleFoldAngle(foldAngle ?? 180);
248 scene.remove(phone.root);
249 phone.dispose();
250 phone = createFoldScene(angle, next);
251 scene.add(phone.root);
252 } else if (
226 !imported &&253 !imported &&
227 !("setAngle" in phone) &&254 !("setAngle" in phone) &&
228 (nextProfile !== profile || next.aspect !== layout.aspect)255 (nextProfile !== profile || next.aspect !== layout.aspect)
...@@ -267,7 +294,7 @@ export function createPhoneViewer(options: {...@@ -267,7 +294,7 @@ export function createPhoneViewer(options: {
267 : model294 : model
268 ? createImportedPhoneScene(model.asset, texture, layout)295 ? createImportedPhoneScene(model.asset, texture, layout)
269 : foldAngle !== null && profile.id.startsWith("android")296 : foldAngle !== null && profile.id.startsWith("android")
270 ? createAndroidFoldScene(texture, layout, foldAngle)297 ? createFoldScene(foldAngle)
271 : createPhoneScene(texture, layout, profile);298 : createPhoneScene(texture, layout, profile);
272 foldTurn = null;299 foldTurn = null;
273 scene.remove(phone.root);300 scene.remove(phone.root);
...@@ -333,10 +360,7 @@ export function createPhoneViewer(options: {...@@ -333,10 +360,7 @@ export function createPhoneViewer(options: {
333 if (next === null || !("setAngle" in phone)) {360 if (next === null || !("setAngle" in phone)) {
334 scene.remove(phone.root);361 scene.remove(phone.root);
335 phone.dispose();362 phone.dispose();
336 phone =363 phone = next === null ? createPhoneScene(texture, layout, profile) : createFoldScene(next);
337 next === null
338 ? createPhoneScene(texture, layout, profile)
339 : createAndroidFoldScene(texture, layout, next);
340 scene.add(phone.root);364 scene.add(phone.root);
341 orientationTurn = null;365 orientationTurn = null;
342 orientationAngle = next === null ? layout.rotation : 0;366 orientationAngle = next === null ? layout.rotation : 0;
...@@ -344,13 +368,7 @@ export function createPhoneViewer(options: {...@@ -344,13 +368,7 @@ export function createPhoneViewer(options: {
344 applyPose();368 applyPose();
345 fit(true);369 fit(true);
346 } else {370 } else {
347 const progress = foldTurn371 const from = visibleFoldAngle(previous ?? next);
348 ? Math.min(1, (performance.now() - foldTurn.startedAt) / ANDROID_FOLD_TURN_MS)
349 : 1;
350 const eased = progress * progress * (3 - 2 * progress);
351 const from = foldTurn
352 ? foldTurn.from + (foldTurn.to - foldTurn.from) * eased
353 : (previous ?? next);
354 if (reducedMotion()) {372 if (reducedMotion()) {
355 foldTurn = null;373 foldTurn = null;
356 phone.setAngle(next);374 phone.setAngle(next);