authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-10-09 12:18:27-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-09 12:18:27-07:00
logecfb7342edde9160b7df4231d4f145c7e4a30530
tree71938659d3b762680bce1ba8d64a7e9aba92c8b5
parent91a6646efd98622dc9cef7061387e20bc389463f
signature Signed by PGP key B5690EEEBB952194

fix(devices): recover stalled video without losing simulator input (#17566)

6 files changed, 272 insertions(+), 31 deletions(-)

apps/mobile/src/features/devices/device-stream.browser.test.ts+3
...@@ -38,6 +38,7 @@ async function setup() {...@@ -38,6 +38,7 @@ async function setup() {
38 static OPEN = 1;38 static OPEN = 1;
39 readyState = 1;39 readyState = 1;
40 onopen: (() => void) | null = null;40 onopen: (() => void) | null = null;
41 onmessage: ((event: { data: ArrayBuffer }) => void) | null = null;
41 close = vi.fn();42 close = vi.fn();
42 send = vi.fn();43 send = vi.fn();
43 constructor() {44 constructor() {
...@@ -86,6 +87,8 @@ it("bridges shared first-frame readiness, image failure, and a successful fresh...@@ -86,6 +87,8 @@ it("bridges shared first-frame readiness, image failure, and a successful fresh
86 const { elements, messages, sockets, configuration } = await setup();87 const { elements, messages, sockets, configuration } = await setup();
87 sockets[0]!.onopen?.();88 sockets[0]!.onopen?.();
88 expect(messages()).not.toContainEqual({ type: "status", status: "streaming" });89 expect(messages()).not.toContainEqual({ type: "status", status: "streaming" });
90 expect(messages()).not.toContainEqual({ type: "input", connected: true });
91 sockets[0]!.onmessage?.({ data: new Uint8Array([0x83]).buffer });
89 expect(messages()).toContainEqual({ type: "input", connected: true });92 expect(messages()).toContainEqual({ type: "input", connected: true });
90 const image = elements.find((element) => element.tag === "img")!;93 const image = elements.find((element) => element.tag === "img")!;
91 image.naturalWidth = 400;94 image.naturalWidth = 400;
apps/server/src/device/DeviceToolchain.ts+2-2
...@@ -26,9 +26,9 @@ import * as Semaphore from "effect/Semaphore";...@@ -26,9 +26,9 @@ import * as Semaphore from "effect/Semaphore";
26import * as ProcessRunner from "../processRunner.ts";26import * as ProcessRunner from "../processRunner.ts";
2727
28const DEVICE_HUB_PACKAGE = "expo-device-hub";28const DEVICE_HUB_PACKAGE = "expo-device-hub";
29export const DEVICE_HUB_VERSION = "0.12.0";29export const DEVICE_HUB_VERSION = "0.15.3";
30const AGENT_DEVICE_PACKAGE = "agent-device";30const AGENT_DEVICE_PACKAGE = "agent-device";
31export const AGENT_DEVICE_VERSION = "0.21.12";31export const AGENT_DEVICE_VERSION = "0.21.23";
3232
33const INSTALL_TIMEOUT = Duration.minutes(10);33const INSTALL_TIMEOUT = Duration.minutes(10);
34const installLock = Semaphore.makeUnsafe(1);34const installLock = Semaphore.makeUnsafe(1);
apps/web/src/components/device/DeviceStreamView.test.tsx+99-4
...@@ -1,4 +1,4 @@...@@ -1,4 +1,4 @@
1import { act, useSyncExternalStore } from "react";1import { act, useEffect, useSyncExternalStore } from "react";
2import { create, type ReactTestRenderer } from "react-test-renderer";2import { create, type ReactTestRenderer } from "react-test-renderer";
3import { EnvironmentId } from "@t3tools/contracts";3import { EnvironmentId } from "@t3tools/contracts";
4import { afterEach, beforeEach, expect, it, vi } from "vite-plus/test";4import { afterEach, beforeEach, expect, it, vi } from "vite-plus/test";
...@@ -20,6 +20,20 @@ vi.mock("~/state/device", () => ({...@@ -20,6 +20,20 @@ vi.mock("~/state/device", () => ({
20 useDeviceHubAccess: () => useSyncExternalStore(accessStore.subscribe, () => accessStore.value),20 useDeviceHubAccess: () => useSyncExternalStore(accessStore.subscribe, () => accessStore.value),
21 refreshDeviceHubAccess: () => accessStore.refresh(),21 refreshDeviceHubAccess: () => accessStore.refresh(),
22}));22}));
23// Replace GPU allocation while keeping the real React and stream lifecycles.
24let viewerMounts = 0;
25let viewerUnmounts = 0;
26vi.mock("./DevicePhoneViewport", () => ({
27 DevicePhoneViewport: function Viewer() {
28 useEffect(() => {
29 viewerMounts++;
30 return () => {
31 viewerUnmounts++;
32 };
33 }, []);
34 return null;
35 },
36}));
23import { DeviceStreamView } from "./DeviceStreamView";37import { DeviceStreamView } from "./DeviceStreamView";
2438
25class Image extends EventTarget {39class Image extends EventTarget {
...@@ -34,6 +48,8 @@ let renderer: ReactTestRenderer | undefined;...@@ -34,6 +48,8 @@ let renderer: ReactTestRenderer | undefined;
34let primes = 0;48let primes = 0;
35beforeEach(() => {49beforeEach(() => {
36 primes = 0;50 primes = 0;
51 viewerMounts = 0;
52 viewerUnmounts = 0;
37});53});
38afterEach(async () => {54afterEach(async () => {
39 await act(async () => renderer?.unmount());55 await act(async () => renderer?.unmount());
...@@ -42,18 +58,57 @@ afterEach(async () => {...@@ -42,18 +58,57 @@ afterEach(async () => {
42 vi.unstubAllGlobals();58 vi.unstubAllGlobals();
43});59});
4460
45async function setup() {61async function setup(h264 = false) {
46 vi.useFakeTimers();62 vi.useFakeTimers();
63 vi.stubGlobal("window", globalThis);
47 vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true);64 vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true);
48 vi.stubGlobal("fetch", () => {65 let videoBody: ReadableStreamDefaultController<Uint8Array> | undefined;
66 let output: VideoFrameOutputCallback | undefined;
67 if (h264) {
68 vi.stubGlobal(
69 "VideoDecoder",
70 class {
71 static isConfigSupported = async () => ({ supported: true });
72 state = "unconfigured";
73 constructor(callbacks: VideoDecoderInit) {
74 output = callbacks.output;
75 }
76 configure() {
77 this.state = "configured";
78 }
79 close() {
80 this.state = "closed";
81 }
82 },
83 );
84 vi.stubGlobal("EncodedVideoChunk", vi.fn());
85 }
86 vi.stubGlobal("fetch", (url: string, init: RequestInit) => {
87 if (url.endsWith("stream.avcc")) {
88 return Promise.resolve(
89 new Response(
90 new ReadableStream<Uint8Array>({
91 start(controller) {
92 videoBody = controller;
93 init.signal?.addEventListener("abort", () => controller.error(new Error("aborted")));
94 },
95 }),
96 ),
97 );
98 }
49 primes++;99 primes++;
50 return Promise.resolve(new Response("prime"));100 return Promise.resolve(new Response("prime"));
51 });101 });
102 const sockets: Array<{ onmessage?: (event: { data: ArrayBuffer }) => void }> = [];
52 vi.stubGlobal(103 vi.stubGlobal(
53 "WebSocket",104 "WebSocket",
54 class {105 class {
55 static OPEN = 1;106 static OPEN = 1;
56 readyState = 1;107 readyState = 1;
108 constructor() {
109 sockets.push(this);
110 }
111 onmessage?: (event: { data: ArrayBuffer }) => void;
57 send() {}112 send() {}
58 close() {}113 close() {}
59 },114 },
...@@ -73,6 +128,7 @@ async function setup() {...@@ -73,6 +128,7 @@ async function setup() {
73 deviceId="test"128 deviceId="test"
74 platform="ios"129 platform="ios"
75 visible={visible}130 visible={visible}
131 allowPhoneView={h264}
76 />132 />
77 );133 );
78 await act(async () => {134 await act(async () => {
...@@ -84,13 +140,30 @@ async function setup() {...@@ -84,13 +140,30 @@ async function setup() {
84 return image;140 return image;
85 }141 }
86 return {142 return {
143 getContext: () => ({ drawImage() {} }),
87 style: { setProperty() {} },144 style: { setProperty() {} },
88 getBoundingClientRect: () => ({ width: 400, height: 800 }),145 getBoundingClientRect: () => ({ width: 400, height: 800 }),
89 };146 };
90 },147 },
91 });148 });
92 });149 });
93 return { images, view };150 return {
151 images,
152 view,
153 configure() {
154 const json = new TextEncoder().encode(
155 JSON.stringify({ width: 400, height: 800, orientation: "portrait" }),
156 );
157 const packet = new Uint8Array(1 + json.length);
158 packet[0] = 0x82;
159 packet.set(json, 1);
160 sockets[0]?.onmessage?.({ data: packet.buffer });
161 videoBody?.enqueue(new Uint8Array([0, 0, 0, 5, 1, 1, 0x64, 0, 0x1f]));
162 },
163 frame() {
164 output?.({ displayWidth: 400, displayHeight: 800, close() {} } as VideoFrame);
165 },
166 };
94}167}
95168
96it("removes MJPEG requests while hidden and reconnects when shown", async () => {169it("removes MJPEG requests while hidden and reconnects when shown", async () => {
...@@ -135,3 +208,25 @@ it("starts exactly one new stream per Reconnect press", async () => {...@@ -135,3 +208,25 @@ it("starts exactly one new stream per Reconnect press", async () => {
135 await act(async () => renderer!.root.findByType("button").props.onClick());208 await act(async () => renderer!.root.findByType("button").props.onClick());
136 expect(primes).toBe(2);209 expect(primes).toBe(2);
137});210});
211
212it("keeps the 3D viewer mounted across iOS video recovery and releases it when hidden", async () => {
213 const { configure, frame, view } = await setup(true);
214 await act(async () => configure());
215 expect(viewerMounts).toBe(0);
216 await act(async () => frame());
217 expect(viewerMounts).toBe(1);
218 await act(async () => {
219 await vi.advanceTimersByTimeAsync(15_000);
220 });
221 expect(viewerUnmounts).toBe(0);
222 expect(viewerMounts).toBe(1);
223 await act(async () => {
224 await vi.advanceTimersByTimeAsync(1_000);
225 configure();
226 });
227 await act(async () => frame());
228 expect(viewerMounts).toBe(1);
229 await act(async () => renderer!.update(view(false)));
230 expect(viewerUnmounts).toBe(1);
231 expect(vi.getTimerCount()).toBe(0);
232});
apps/web/src/components/device/DeviceStreamView.tsx+14-12
...@@ -94,6 +94,7 @@ export function DeviceStreamView(props: {...@@ -94,6 +94,7 @@ export function DeviceStreamView(props: {
94 const canvasRef = useRef<HTMLCanvasElement | null>(null);94 const canvasRef = useRef<HTMLCanvasElement | null>(null);
95 const clientRef = useRef<DeviceStreamClient | null>(null);95 const clientRef = useRef<DeviceStreamClient | null>(null);
96 const [status, setStatus] = useState<DeviceStreamStatus>("connecting");96 const [status, setStatus] = useState<DeviceStreamStatus>("connecting");
97 const [hasFrame, setHasFrame] = useState(false);
97 const [detail, setDetail] = useState<string | undefined>(undefined);98 const [detail, setDetail] = useState<string | undefined>(undefined);
98 const [showRestartNotice, setShowRestartNotice] = useState(false);99 const [showRestartNotice, setShowRestartNotice] = useState(false);
99 const [screen, setScreen] = useState<DeviceScreenSize | null>(null);100 const [screen, setScreen] = useState<DeviceScreenSize | null>(null);
...@@ -123,6 +124,7 @@ export function DeviceStreamView(props: {...@@ -123,6 +124,7 @@ export function DeviceStreamView(props: {
123 onDuoUnavailable: onPhoneUnavailable,124 onDuoUnavailable: onPhoneUnavailable,
124 onStatus: (next, nextDetail) => {125 onStatus: (next, nextDetail) => {
125 setStatus(next);126 setStatus(next);
127 if (next === "streaming") setHasFrame(true);
126 setDetail(nextDetail);128 setDetail(nextDetail);
127 if (next !== "connecting") setShowRestartNotice(false);129 if (next !== "connecting") setShowRestartNotice(false);
128 },130 },
...@@ -150,6 +152,8 @@ export function DeviceStreamView(props: {...@@ -150,6 +152,8 @@ export function DeviceStreamView(props: {
150 },152 },
151 );153 );
152 clientRef.current = client;154 clientRef.current = client;
155 setPhoneUnavailable(false);
156 setHasFrame(false);
153 setMjpegUrl(null);157 setMjpegUrl(null);
154 setInputState({ connected: false });158 setInputState({ connected: false });
155 client.start();159 client.start();
...@@ -188,16 +192,13 @@ export function DeviceStreamView(props: {...@@ -188,16 +192,13 @@ export function DeviceStreamView(props: {
188 return w / h;192 return w / h;
189 }, [props.platform, screen]);193 }, [props.platform, screen]);
190194
191 // Android restarts its encoder when a fold changes the framebuffer size.195 // Encoder restarts and video reconnects retain the decoded frame and viewer
192 // Keep the last decoded frame and viewer mounted while the next keyframe arrives.196 // while input is still connected and the next keyframe is on its way.
193 const retainingAndroidFrame =197 const retainingFrame =
194 props.platform === "android" &&198 status === "connecting" && hasFrame && inputState.connected && screen !== null;
195 status === "connecting" &&
196 inputState.connected &&
197 screen !== null;
198 const showPhone =199 const showPhone =
199 props.allowPhoneView &&200 props.allowPhoneView &&
200 (status === "streaming" || retainingAndroidFrame) &&201 (status === "streaming" || retainingFrame) &&
201 props.visible &&202 props.visible &&
202 presentation === "phone" &&203 presentation === "phone" &&
203 !phoneUnavailable &&204 !phoneUnavailable &&
...@@ -205,10 +206,10 @@ export function DeviceStreamView(props: {...@@ -205,10 +206,10 @@ export function DeviceStreamView(props: {
205 !props.axOverlay &&206 !props.axOverlay &&
206 (!isDuo || screen?.supportsHingeAngle === true);207 (!isDuo || screen?.supportsHingeAngle === true);
207 useEffect(() => {208 useEffect(() => {
208 if (!retainingAndroidFrame || !showPhone) return;209 if (!retainingFrame || !showPhone) return;
209 const timeout = window.setTimeout(() => setShowRestartNotice(true), 2_000);210 const timeout = window.setTimeout(() => setShowRestartNotice(true), 2_000);
210 return () => window.clearTimeout(timeout);211 return () => window.clearTimeout(timeout);
211 }, [retainingAndroidFrame, showPhone]);212 }, [retainingFrame, showPhone]);
212 const controlsInset = props.renderControls && !showPhone ? CONTROLS_RAIL_WIDTH : 0;213 const controlsInset = props.renderControls && !showPhone ? CONTROLS_RAIL_WIDTH : 0;
213214
214 // The frame is the largest box at `aspect` that fits the container, so a215 // The frame is the largest box at `aspect` that fits the container, so a
...@@ -532,14 +533,14 @@ export function DeviceStreamView(props: {...@@ -532,14 +533,14 @@ export function DeviceStreamView(props: {
532 </span>533 </span>
533 </div>534 </div>
534 ) : null}535 ) : null}
535 {retainingAndroidFrame && showPhone && showRestartNotice ? (536 {retainingFrame && showPhone && showRestartNotice ? (
536 <div className="pointer-events-none absolute inset-x-0 bottom-0 flex justify-center p-2">537 <div className="pointer-events-none absolute inset-x-0 bottom-0 flex justify-center p-2">
537 <span className="rounded-md bg-background/85 px-2 py-1 text-xs text-muted-foreground">538 <span className="rounded-md bg-background/85 px-2 py-1 text-xs text-muted-foreground">
538 Waiting for device video…539 Waiting for device video…
539 </span>540 </span>
540 </div>541 </div>
541 ) : null}542 ) : null}
542 {status !== "streaming" && !(retainingAndroidFrame && showPhone) ? (543 {status !== "streaming" && !(retainingFrame && showPhone) ? (
543 <div className="absolute inset-0">544 <div className="absolute inset-0">
544 <DeviceLoadingView545 <DeviceLoadingView
545 name={props.deviceName ?? "Device"}546 name={props.deviceName ?? "Device"}
...@@ -556,6 +557,7 @@ export function DeviceStreamView(props: {...@@ -556,6 +557,7 @@ export function DeviceStreamView(props: {
556 // An expired ticket surfaces as unauthorized on restart and557 // An expired ticket surfaces as unauthorized on restart and
557 // refreshes access through the effect; no need to mint one here.558 // refreshes access through the effect; no need to mint one here.
558 clientRef.current?.stop();559 clientRef.current?.stop();
560 setPhoneUnavailable(false);
559 clientRef.current?.start();561 clientRef.current?.start();
560 }}562 }}
561 >563 >
packages/client-runtime/src/device/stream.test.ts+109-5
...@@ -385,10 +385,14 @@ function recoveryFixture(platform: "ios" | "android" = "ios", preferMjpeg = true...@@ -385,10 +385,14 @@ function recoveryFixture(platform: "ios" | "android" = "ios", preferMjpeg = true
385 configure() {385 configure() {
386 this.state = "configured";386 this.state = "configured";
387 }387 }
388 reset = vi.fn(() => {
389 this.state = "unconfigured";
390 this.decodeQueueSize = 0;
391 });
388 close() {392 close() {
389 this.state = "closed";393 this.state = "closed";
390 }394 }
391 decode() {}395 decode = vi.fn();
392 readonly callbacks: { output: (frame: VideoFrame) => void; error: () => void };396 readonly callbacks: { output: (frame: VideoFrame) => void; error: () => void };
393 constructor(callbacks: { output: (frame: VideoFrame) => void; error: () => void }) {397 constructor(callbacks: { output: (frame: VideoFrame) => void; error: () => void }) {
394 this.callbacks = callbacks;398 this.callbacks = callbacks;
...@@ -480,6 +484,93 @@ describe("shared device stream readiness and recovery", () => {...@@ -480,6 +484,93 @@ describe("shared device stream readiness and recovery", () => {
480 vi.unstubAllGlobals();484 vi.unstubAllGlobals();
481 });485 });
482486
487 it("waits for iOS input admission and respects native input loss and recovery", async () => {
488 const { client, sockets, events } = recoveryFixture();
489 client.start();
490 await vi.advanceTimersByTimeAsync(0);
491 const socket = sockets[0]!;
492 socket.onopen?.();
493 expect(events.onInputConnected).not.toHaveBeenCalledWith(true);
494 socket.onmessage?.({ data: new Uint8Array([0x83]).buffer });
495 expect(events.onInputConnected).toHaveBeenLastCalledWith(true);
496 const config = (inputUnavailable: boolean) => {
497 const json = new TextEncoder().encode(
498 JSON.stringify({ width: 400, height: 800, orientation: "portrait", inputUnavailable }),
499 );
500 const packet = new Uint8Array(1 + json.length);
501 packet[0] = 0x82;
502 packet.set(json, 1);
503 socket.onmessage?.({ data: packet.buffer });
504 };
505 config(true);
506 expect(events.onInputConnected).toHaveBeenLastCalledWith(
507 false,
508 "Simulator input is unavailable",
509 );
510 socket.send.mockClear();
511 client.sendTouch("begin", 0.2, 0.3);
512 client.pressButton("home");
513 expect(socket.send).not.toHaveBeenCalled();
514 config(false);
515 expect(events.onInputConnected).toHaveBeenLastCalledWith(true, undefined);
516 client.sendTouch("begin", 0.2, 0.3);
517 expect(socket.send).toHaveBeenCalledOnce();
518 client.stop();
519 expect(vi.getTimerCount()).toBe(0);
520 });
521
522 it("drops an iOS decode backlog and resumes at a keyframe without switching to MJPEG", async () => {
523 const { client, videoBodies, decoders, events, decodedFrame } = recoveryFixture("ios", false);
524 client.start();
525 await vi.advanceTimersByTimeAsync(0);
526 const body = videoBodies[0]!;
527 body.enqueue(new Uint8Array([...envelope(1, [1, 0x64, 0, 0x1f]), ...envelope(2, [1])]));
528 await vi.advanceTimersByTimeAsync(0);
529 decodedFrame();
530 const decoder = decoders[0]!;
531 expect(decoder.decode).toHaveBeenCalledOnce();
532 decoder.decodeQueueSize = 9;
533 body.enqueue(new Uint8Array([...envelope(3, [2]), ...envelope(3, [3])]));
534 await vi.advanceTimersByTimeAsync(0);
535 expect(decoder.reset).toHaveBeenCalledOnce();
536 expect(decoder.decode).toHaveBeenCalledOnce();
537 body.enqueue(new Uint8Array([...envelope(2, [4]), ...envelope(3, [5])]));
538 await vi.advanceTimersByTimeAsync(0);
539 expect(decoder.decode).toHaveBeenCalledTimes(3);
540 expect(events.onMjpegFallback).not.toHaveBeenCalled();
541 expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);
542 client.stop();
543 });
544
545 it("does not repaint a delayed JPEG seed over newer decoded video", async () => {
546 const { client, videoBodies, drawImage, decodedFrame } = recoveryFixture("ios", false);
547 let resolveBitmap!: (bitmap: ImageBitmap) => void;
548 vi.stubGlobal(
549 "createImageBitmap",
550 () =>
551 new Promise((resolve) => {
552 resolveBitmap = resolve;
553 }),
554 );
555 client.start();
556 await vi.advanceTimersByTimeAsync(0);
557 videoBodies[0]!.enqueue(
558 new Uint8Array([
559 ...envelope(4, [1]),
560 ...envelope(1, [1, 0x64, 0, 0x1f]),
561 ...envelope(2, [2]),
562 ]),
563 );
564 await vi.advanceTimersByTimeAsync(0);
565 decodedFrame();
566 const close = vi.fn();
567 resolveBitmap({ width: 400, height: 800, close } as unknown as ImageBitmap);
568 await vi.advanceTimersByTimeAsync(0);
569 expect(drawImage).toHaveBeenCalledOnce();
570 expect(close).toHaveBeenCalledOnce();
571 client.stop();
572 });
573
483 it("waits for actual MJPEG dimensions without requiring a multipart load event", async () => {574 it("waits for actual MJPEG dimensions without requiring a multipart load event", async () => {
484 const { client, events, image } = recoveryFixture();575 const { client, events, image } = recoveryFixture();
485 client.start();576 client.start();
...@@ -713,22 +804,35 @@ describe("shared device stream readiness and recovery", () => {...@@ -713,22 +804,35 @@ describe("shared device stream readiness and recovery", () => {
713 });804 });
714});805});
715806
716it("reports a stalled AVCC body after its initial image instead of leaving a frozen streaming state", async () => {807it("reopens a stalled AVCC body without disconnecting input, then receives fresh video", async () => {
717 const { client, videoBodies, events, signals } = recoveryFixture("ios", false);808 const { client, videoBodies, events, signals, sockets, decodedFrame } = recoveryFixture(
809 "ios",
810 false,
811 );
718 const close = vi.fn();812 const close = vi.fn();
719 vi.stubGlobal("createImageBitmap", () => Promise.resolve({ width: 400, height: 800, close }));813 vi.stubGlobal("createImageBitmap", () => Promise.resolve({ width: 400, height: 800, close }));
720 try {814 try {
721 client.start();815 client.start();
722 await vi.advanceTimersByTimeAsync(0);816 await vi.advanceTimersByTimeAsync(0);
817 sockets[0]!.onmessage?.({ data: new Uint8Array([0x83]).buffer });
723 videoBodies[0]!.enqueue(new Uint8Array(envelope(4, [1])));818 videoBodies[0]!.enqueue(new Uint8Array(envelope(4, [1])));
724 await vi.advanceTimersByTimeAsync(0);819 await vi.advanceTimersByTimeAsync(0);
725 expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);820 expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);
726 await vi.advanceTimersByTimeAsync(15_000);821 await vi.advanceTimersByTimeAsync(15_000);
727 expect(events.onStatus).toHaveBeenLastCalledWith(822 expect(events.onStatus).toHaveBeenLastCalledWith(
728 "error",823 "connecting",
729 expect.stringContaining("stopped receiving video"),824 expect.stringContaining("stopped receiving video"),
730 );825 );
731 expect(signals.every((signal) => signal.aborted)).toBe(true);826 expect(signals[1]!.aborted).toBe(true);
827 expect(sockets[0]!.close).not.toHaveBeenCalled();
828 expect(events.onInputConnected).toHaveBeenLastCalledWith(true);
829 await vi.advanceTimersByTimeAsync(1_000);
830 expect(videoBodies).toHaveLength(2);
831 videoBodies[1]!.enqueue(new Uint8Array(envelope(1, [1, 0x64, 0, 0x1f])));
832 await vi.advanceTimersByTimeAsync(0);
833 decodedFrame();
834 expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);
835 client.stop();
732 expect(vi.getTimerCount()).toBe(0);836 expect(vi.getTimerCount()).toBe(0);
733 } finally {837 } finally {
734 client.stop();838 client.stop();
packages/client-runtime/src/device/stream.ts+45-8
...@@ -55,6 +55,7 @@ const screenConfigSchema = Schema.Struct({...@@ -55,6 +55,7 @@ const screenConfigSchema = Schema.Struct({
55 "landscape_left",55 "landscape_left",
56 "landscape_right",56 "landscape_right",
57 ]),57 ]),
58 inputUnavailable: Schema.optionalKey(Schema.Boolean),
58 screenId: Schema.optionalKey(Schema.Number),59 screenId: Schema.optionalKey(Schema.Number),
59 supportsHingeAngle: Schema.optionalKey(Schema.Boolean),60 supportsHingeAngle: Schema.optionalKey(Schema.Boolean),
60 supportsPhysicalOrientation: Schema.optionalKey(Schema.Boolean),61 supportsPhysicalOrientation: Schema.optionalKey(Schema.Boolean),
...@@ -130,6 +131,7 @@ const IOS_MSG_ORIENTATION = 0x07;...@@ -130,6 +131,7 @@ const IOS_MSG_ORIENTATION = 0x07;
130const IOS_MSG_HARDWARE_KEYBOARD = 0x0d;131const IOS_MSG_HARDWARE_KEYBOARD = 0x0d;
131// helper -> browser.132// helper -> browser.
132const IOS_TAG_SCREEN_CONFIG = 0x82;133const IOS_TAG_SCREEN_CONFIG = 0x82;
134const IOS_TAG_INPUT_ADMITTED = 0x83;
133135
134const encoder = new TextEncoder();136const encoder = new TextEncoder();
135const decoder = new TextDecoder();137const decoder = new TextDecoder();
...@@ -364,6 +366,9 @@ export function createDeviceStreamClient(...@@ -364,6 +366,9 @@ export function createDeviceStreamClient(
364 const retryTimers = new Map<"video" | "input", ReturnType<typeof setTimeout>>();366 const retryTimers = new Map<"video" | "input", ReturnType<typeof setTimeout>>();
365 let primeController: AbortController | null = null;367 let primeController: AbortController | null = null;
366 let videoDecoder: VideoDecoder | null = null;368 let videoDecoder: VideoDecoder | null = null;
369 let decoderConfig: VideoDecoderConfig | null = null;
370 let decodedVideoGeneration: number | null = null;
371 let iosInputUnavailable = false;
367 let timestamp = 0;372 let timestamp = 0;
368 let awaitingKeyframe = true;373 let awaitingKeyframe = true;
369 let screen: DeviceScreenSize | null = null;374 let screen: DeviceScreenSize | null = null;
...@@ -517,6 +522,7 @@ export function createDeviceStreamClient(...@@ -517,6 +522,7 @@ export function createDeviceStreamClient(
517 // Already closed.522 // Already closed.
518 }523 }
519 videoDecoder = null;524 videoDecoder = null;
525 decoderConfig = null;
520 awaitingKeyframe = true;526 awaitingKeyframe = true;
521 };527 };
522528
...@@ -537,8 +543,10 @@ export function createDeviceStreamClient(...@@ -537,8 +543,10 @@ export function createDeviceStreamClient(
537 if (543 if (
538 videoDecoder === decoder &&544 videoDecoder === decoder &&
539 (platform !== "ios" || feedGeneration === videoGeneration)545 (platform !== "ios" || feedGeneration === videoGeneration)
540 )546 ) {
547 decodedVideoGeneration = feedGeneration;
541 paint(frame, frame.displayWidth, frame.displayHeight);548 paint(frame, frame.displayWidth, frame.displayHeight);
549 }
542 } finally {550 } finally {
543 frame.close();551 frame.close();
544 }552 }
...@@ -572,6 +580,7 @@ export function createDeviceStreamClient(...@@ -572,6 +580,7 @@ export function createDeviceStreamClient(
572 try {580 try {
573 if (!videoDecoder || videoDecoder.state === "closed") videoDecoder = makeDecoder();581 if (!videoDecoder || videoDecoder.state === "closed") videoDecoder = makeDecoder();
574 videoDecoder.configure(full);582 videoDecoder.configure(full);
583 decoderConfig = full;
575 return true;584 return true;
576 } catch (cause) {585 } catch (cause) {
577 if (platform === "android") fail(`Video decoder: ${(cause as Error).message}`);586 if (platform === "android") fail(`Video decoder: ${(cause as Error).message}`);
...@@ -586,8 +595,22 @@ export function createDeviceStreamClient(...@@ -586,8 +595,22 @@ export function createDeviceStreamClient(
586 awaitingKeyframe = false;595 awaitingKeyframe = false;
587 }596 }
588 if (videoDecoder.decodeQueueSize > SOFT_DECODE_QUEUE) {597 if (videoDecoder.decodeQueueSize > SOFT_DECODE_QUEUE) {
589 recoverDecoder();598 if (platform !== "ios" || !decoderConfig) {
590 return;599 recoverDecoder();
600 return;
601 }
602 // A slow viewer can accumulate healthy H.264 frames. Drop that backlog
603 // and resume at an IDR instead of permanently giving up its 3D view.
604 try {
605 videoDecoder.reset();
606 videoDecoder.configure(decoderConfig);
607 awaitingKeyframe = true;
608 if (!isKey) return;
609 awaitingKeyframe = false;
610 } catch {
611 recoverDecoder();
612 return;
613 }
591 }614 }
592 try {615 try {
593 videoDecoder.decode(616 videoDecoder.decode(
...@@ -659,7 +682,9 @@ export function createDeviceStreamClient(...@@ -659,7 +682,9 @@ export function createDeviceStreamClient(
659 for (;;) {682 for (;;) {
660 // An AVCC body can stay open after its helper stops producing frames.683 // An AVCC body can stay open after its helper stops producing frames.
661 const timer = setTimeout(() => {684 const timer = setTimeout(() => {
662 if (isCurrent()) fail("Device stream stopped receiving video. Reconnect to try again.");685 if (!isCurrent()) return;
686 retryDetail = "Device stream stopped receiving video. Reconnecting…";
687 videoController.abort();
663 }, FIRST_FRAME_TIMEOUT_MS);688 }, FIRST_FRAME_TIMEOUT_MS);
664 videoReadTimer = timer;689 videoReadTimer = timer;
665 let result: ReadableStreamReadResult<Uint8Array>;690 let result: ReadableStreamReadResult<Uint8Array>;
...@@ -678,7 +703,8 @@ export function createDeviceStreamClient(...@@ -678,7 +703,8 @@ export function createDeviceStreamClient(
678 void createImageBitmap(new Blob([chunk.payload as BlobPart], { type: "image/jpeg" }))703 void createImageBitmap(new Blob([chunk.payload as BlobPart], { type: "image/jpeg" }))
679 .then((bitmap) => {704 .then((bitmap) => {
680 try {705 try {
681 if (isCurrent()) paint(bitmap, bitmap.width, bitmap.height);706 if (isCurrent() && decodedVideoGeneration !== feedGeneration)
707 paint(bitmap, bitmap.width, bitmap.height);
682 } finally {708 } finally {
683 bitmap.close();709 bitmap.close();
684 }710 }
...@@ -714,7 +740,7 @@ export function createDeviceStreamClient(...@@ -714,7 +740,7 @@ export function createDeviceStreamClient(
714 }740 }
715 } catch (cause) {741 } catch (cause) {
716 if (!isCurrent()) return;742 if (!isCurrent()) return;
717 retryDetail = (cause as Error).message;743 retryDetail ??= (cause as Error).message;
718 }744 }
719 if (isCurrent()) {745 if (isCurrent()) {
720 controller = null;746 controller = null;
...@@ -799,13 +825,17 @@ export function createDeviceStreamClient(...@@ -799,13 +825,17 @@ export function createDeviceStreamClient(
799 ws.onopen = () => {825 ws.onopen = () => {
800 if (stopped || socket !== ws) return;826 if (stopped || socket !== ws) return;
801 ws.send(taggedJson(IOS_MSG_HARDWARE_KEYBOARD, { enabled: false }));827 ws.send(taggedJson(IOS_MSG_HARDWARE_KEYBOARD, { enabled: false }));
802 events.onInputConnected(true);
803 };828 };
804 ws.onmessage = (event) => {829 ws.onmessage = (event) => {
805 if (stopped || socket !== ws) return;830 if (stopped || socket !== ws) return;
806 if (!(event.data instanceof ArrayBuffer)) return;831 if (!(event.data instanceof ArrayBuffer)) return;
807 const bytes = new Uint8Array(event.data);832 const bytes = new Uint8Array(event.data);
808 if (socket !== ws || stopped || bytes.length < 1) return;833 if (socket !== ws || stopped || bytes.length < 1) return;
834 if (bytes[0] === IOS_TAG_INPUT_ADMITTED) {
835 iosInputUnavailable = false;
836 events.onInputConnected(true);
837 return;
838 }
809 try {839 try {
810 const payload: unknown = JSON.parse(decoder.decode(bytes.subarray(1)));840 const payload: unknown = JSON.parse(decoder.decode(bytes.subarray(1)));
811 if (bytes[0] === 0x90) {841 if (bytes[0] === 0x90) {
...@@ -814,6 +844,11 @@ export function createDeviceStreamClient(...@@ -814,6 +844,11 @@ export function createDeviceStreamClient(
814 } else if (bytes[0] === IOS_TAG_SCREEN_CONFIG) {844 } else if (bytes[0] === IOS_TAG_SCREEN_CONFIG) {
815 const config = decodeScreenConfig(payload);845 const config = decodeScreenConfig(payload);
816 if (Option.isSome(config)) {846 if (Option.isSome(config)) {
847 iosInputUnavailable = config.value.inputUnavailable === true;
848 events.onInputConnected(
849 !iosInputUnavailable,
850 iosInputUnavailable ? "Simulator input is unavailable" : undefined,
851 );
817 const previous = screen;852 const previous = screen;
818 screen = config.value;853 screen = config.value;
819 if (screen.hingePose && screen.hingePose !== previous?.hingePose)854 if (screen.hingePose && screen.hingePose !== previous?.hingePose)
...@@ -944,6 +979,7 @@ export function createDeviceStreamClient(...@@ -944,6 +979,7 @@ export function createDeviceStreamClient(
944 stopped = false;979 stopped = false;
945 generation++;980 generation++;
946 configuring = false;981 configuring = false;
982 iosInputUnavailable = false;
947 connecting();983 connecting();
948 if (platform === "ios") {984 if (platform === "ios") {
949 if (!target.videoOnly) void connectIosInput();985 if (!target.videoOnly) void connectIosInput();
...@@ -986,7 +1022,8 @@ export function createDeviceStreamClient(...@@ -986,7 +1022,8 @@ export function createDeviceStreamClient(
986 };1022 };
9871023
988 const send = (payload: Uint8Array<ArrayBuffer> | string) => {1024 const send = (payload: Uint8Array<ArrayBuffer> | string) => {
989 if (!stopped && socket?.readyState === WebSocket.OPEN) socket.send(payload);1025 if (!stopped && !iosInputUnavailable && socket?.readyState === WebSocket.OPEN)
1026 socket.send(payload);
990 };1027 };
9911028
992 const rawPoint = (x: number, y: number) => {1029 const rawPoint = (x: number, y: number) => {