Commit 9f73ca3677faa975be00f0ac55460279671df726 | the-knife

apps/mobile/src/features/devices/DevicePreviewRouteScreen.tsx+2-2

...	@@ -68,7 +68,7 @@ function DevicePreviewScreen({	...	@@ -68,7 +68,7 @@ function DevicePreviewScreen({
68	const insets = useSafeAreaInsets();	68	const insets = useSafeAreaInsets();
69	const { themeVariables } = useAppearancePreferences();	69	const { themeVariables } = useAppearancePreferences();
70	const focused = useIsFocused();	70	const focused = useIsFocused();
71	const [foreground, setForeground] = useState(AppState.currentState === "active");	71	const [foreground, setForeground] = useState(AppState.currentState !== "background");
72	const [selectedKey, setSelectedKey] = useState<string \| null>(null);	72	const [selectedKey, setSelectedKey] = useState<string \| null>(null);
73	const [inputConnected, setInputConnected] = useState(false);	73	const [inputConnected, setInputConnected] = useState(false);
74	const [streamAttempt, setStreamAttempt] = useState(0);	74	const [streamAttempt, setStreamAttempt] = useState(0);
...	@@ -87,7 +87,7 @@ function DevicePreviewScreen({	...	@@ -87,7 +87,7 @@ function DevicePreviewScreen({
87	);	87	);
88	useEffect(() => {	88	useEffect(() => {
89	const subscription = AppState.addEventListener("change", (state) =>	89	const subscription = AppState.addEventListener("change", (state) =>
90	setForeground(state === "active"),	90	setForeground(state !== "background"),
91	);	91	);
92	return () => subscription.remove();	92	return () => subscription.remove();
93	}, []);	93	}, []);

apps/mobile/src/features/devices/DeviceStreamWebView.tsx+127-26

...	@@ -1,7 +1,19 @@	...	@@ -1,7 +1,19 @@
1	import deviceStreamScript from "@t3tools/mobile-device-stream";	1	import deviceStreamScript from "@t3tools/mobile-device-stream";
2	import { useImperativeHandle, useLayoutEffect, useMemo, useRef, useState, type Ref } from "react";	2	import {
3	import { Platform } from "react-native";	3	useEffect,
		4	useEffectEvent,
		5	useImperativeHandle,
		6	useLayoutEffect,
		7	useMemo,
		8	useRef,
		9	useState,
		10	type Ref,
		11	} from "react";
		12	import { ActivityIndicator, Platform, Pressable, View } from "react-native";
4	import { WebView } from "react-native-webview";	13	import { WebView } from "react-native-webview";
		14	import type { DeviceStreamStatus } from "@t3tools/client-runtime/device/stream";
		15
		16	import { AppText } from "../../components/AppText";
5		17
6	import {	18	import {
7	deviceStreamDocument,	19	deviceStreamDocument,
...	@@ -27,6 +39,7 @@ export function DeviceStreamWebView({	...	@@ -27,6 +39,7 @@ export function DeviceStreamWebView({
27	...props	39	...props
28	}: DeviceStreamConfiguration & NativeStreamBridge) {	40	}: DeviceStreamConfiguration & NativeStreamBridge) {
29	const [attempt, setAttempt] = useState(0);	41	const [attempt, setAttempt] = useState(0);
		42	const processRetried = useRef(false);
30	const configuration = JSON.stringify({	43	const configuration = JSON.stringify({
31	access: props.access,	44	access: props.access,
32	platform: props.platform,	45	platform: props.platform,
...	@@ -41,7 +54,21 @@ export function DeviceStreamWebView({	...	@@ -41,7 +54,21 @@ export function DeviceStreamWebView({
41	background={props.colors.background}	54	background={props.colors.background}
42	onUnauthorized={props.onUnauthorized}	55	onUnauthorized={props.onUnauthorized}
43	onInputConnected={props.onInputConnected}	56	onInputConnected={props.onInputConnected}
44	onRetry={() => setAttempt((attempt) => attempt + 1)}	57	onRetry={() => {
		58	processRetried.current = false;
		59	setAttempt((attempt) => attempt + 1);
		60	void props.onUnauthorized();
		61	}}
		62	onStreaming={() => {
		63	processRetried.current = false;
		64	}}
		65	onRecoverProcess={() => {
		66	if (processRetried.current) return false;
		67	processRetried.current = true;
		68	setAttempt((attempt) => attempt + 1);
		69	void props.onUnauthorized();
		70	return true;
		71	}}
45	/>	72	/>
46	);	73	);
47	}	74	}
...	@@ -53,12 +80,38 @@ function DeviceStreamDocumentView({	...	@@ -53,12 +80,38 @@ function DeviceStreamDocumentView({
53	onUnauthorized,	80	onUnauthorized,
54	onInputConnected,	81	onInputConnected,
55	onRetry,	82	onRetry,
		83	onStreaming,
		84	onRecoverProcess,
56	}: NativeStreamBridge & {	85	}: NativeStreamBridge & {
57	readonly configuration: string;	86	readonly configuration: string;
58	readonly background: string;	87	readonly background: string;
59	readonly onRetry: () => void;	88	readonly onRetry: () => void;
		89	readonly onStreaming: () => void;
		90	readonly onRecoverProcess: () => boolean;
60	}) {	91	}) {
61	const webView = useRef<WebView>(null);	92	const webView = useRef<WebView>(null);
		93	const active = useRef(true);
		94	const failed = useRef(false);
		95	const [status, setStatus] = useState<DeviceStreamStatus>("connecting");
		96	const [error, setError] = useState<string \| null>(null);
		97	const [started, setStarted] = useState(false);
		98	const fail = (message: string) => {
		99	if (!active.current \|\| failed.current) return;
		100	failed.current = true;
		101	void onInputConnected(false);
		102	webView.current?.injectJavaScript("window.T3DeviceStream?.stop(); true;");
		103	setError(message);
		104	setStatus("error");
		105	};
		106	// The shared transport owns video timeouts once the document acknowledges startup.
		107	const bootstrapTimedOut = useEffectEvent(() =>
		108	fail("Device viewer could not start. Reconnect to try again."),
		109	);
		110	useEffect(() => {
		111	if (started) return;
		112	const timer = setTimeout(bootstrapTimedOut, 15_000);
		113	return () => clearTimeout(timer);
		114	}, [started]);
62	const source = useMemo(	115	const source = useMemo(
63	() => ({	116	() => ({
64	html: deviceStreamDocument(configuration, deviceStreamScript),	117	html: deviceStreamDocument(configuration, deviceStreamScript),
...	@@ -78,32 +131,80 @@ function DeviceStreamDocumentView({	...	@@ -78,32 +131,80 @@ function DeviceStreamDocumentView({
78	appSwitcher: () => command("appSwitcher"),	131	appSwitcher: () => command("appSwitcher"),
79	rotate: () => command("rotate"),	132	rotate: () => command("rotate"),
80	}));	133	}));
		134	const resetInput = useEffectEvent(() => void onInputConnected(false));
81	useLayoutEffect(() => {	135	useLayoutEffect(() => {
		136	active.current = true;
		137	resetInput();
82	const view = webView.current;	138	const view = webView.current;
83	return () => view?.injectJavaScript("window.T3DeviceStream?.stop(); true;");	139	return () => {
		140	active.current = false;
		141	view?.injectJavaScript("window.T3DeviceStream?.stop(); true;");
		142	};
84	}, []);	143	}, []);
		144	const processTerminated = () => {
		145	if (!active.current \|\| failed.current) return;
		146	void onInputConnected(false);
		147	if (!onRecoverProcess()) fail("Device viewer stopped. Reconnect to try again.");
		148	};
85	return (	149	return (
86	<WebView	150	<View className="flex-1" style={{ backgroundColor: background }}>
87	ref={webView}	151	<WebView
88	source={source}	152	ref={webView}
89	originWhitelist={["*"]}	153	source={source}
90	scrollEnabled={false}	154	originWhitelist={["*"]}
91	bounces={false}	155	scrollEnabled={false}
92	mixedContentMode="always"	156	bounces={false}
93	allowUniversalAccessFromFileURLs	157	mixedContentMode="always"
94	contentInsetAdjustmentBehavior="never"	158	allowUniversalAccessFromFileURLs
95	setSupportMultipleWindows={false}	159	contentInsetAdjustmentBehavior="never"
96	style={{ flex: 1, backgroundColor: background }}	160	setSupportMultipleWindows={false}
97	onLoadStart={() => void onInputConnected(false)}	161	style={{ flex: 1, backgroundColor: background }}
98	onShouldStartLoadWithRequest={(request) =>	162	onError={() => fail("Device viewer could not load. Reconnect to try again.")}
99	request.url === "about:blank" \|\| request.url === source.baseUrl	163	onHttpError={() => fail("Device viewer could not load. Reconnect to try again.")}
100	}	164	onContentProcessDidTerminate={processTerminated}
101	onMessage={(event) => {	165	onRenderProcessGone={processTerminated}
102	const message = deviceStreamMessage(event.nativeEvent.data);	166	onShouldStartLoadWithRequest={(request) =>
103	if (message?.type === "unauthorized") void onUnauthorized();	167	request.url === "about:blank" \|\| request.url === source.baseUrl
104	else if (message?.type === "input") void onInputConnected(message.connected);	168	}
105	else if (message?.type === "retry") onRetry();	169	onMessage={(event) => {
106	}}	170	if (!active.current \|\| failed.current) return;
107	/>	171	const message = deviceStreamMessage(event.nativeEvent.data);
		172	if (message?.type === "unauthorized") void onUnauthorized();
		173	else if (message?.type === "input") void onInputConnected(message.connected);
		174	else if (message?.type === "retry") onRetry();
		175	else if (message?.type === "status") {
		176	setStarted(true);
		177	if (message.status === "error") fail(message.detail ?? "Device stream failed.");
		178	else {
		179	setStatus(message.status);
		180	if (message.status === "streaming") onStreaming();
		181	}
		182	}
		183	}}
		184	/>
		185	{status !== "streaming" ? (
		186	<View
		187	className="absolute inset-0 items-center justify-center gap-4 px-6"
		188	style={{ backgroundColor: background }}
		189	>
		190	{status === "connecting" ? <ActivityIndicator colorClassName="accent-icon" /> : null}
		191	<AppText
		192	accessibilityLiveRegion="polite"
		193	className="text-center text-sm text-foreground-muted"
		194	>
		195	{status === "error" ? error : "Connecting to device..."}
		196	</AppText>
		197	{status === "error" ? (
		198	<Pressable
		199	accessibilityRole="button"
		200	className="rounded-full border border-secondary-border bg-secondary px-6 py-3"
		201	onPress={onRetry}
		202	>
		203	<AppText className="text-secondary-foreground">Reconnect</AppText>
		204	</Pressable>
		205	) : null}
		206	</View>
		207	) : null}
		208	</View>
108	);	209	);
109	}	210	}

apps/mobile/src/features/devices/device-stream-document.test.ts+44-2

...	@@ -1,5 +1,5 @@	...	@@ -1,5 +1,5 @@
1	import * as NodeVM from "node:vm";	1	import * as NodeVM from "node:vm";
2	import { describe, expect, it } from "vite-plus/test";	2	import { describe, expect, it, vi } from "vite-plus/test";
3		3
4	import { deviceStreamDocument, deviceStreamMessage } from "./device-stream-document";	4	import { deviceStreamDocument, deviceStreamMessage } from "./device-stream-document";
5		5
...	@@ -13,7 +13,7 @@ describe("native device stream document", () => {	...	@@ -13,7 +13,7 @@ describe("native device stream document", () => {
13	);	13	);
14	const script = html.match(/<script>([\s\S]*)<\/script>/)?.[1];	14	const script = html.match(/<script>([\s\S]*)<\/script>/)?.[1];
15	expect(html.match(/<script>/g)).toHaveLength(1);	15	expect(html.match(/<script>/g)).toHaveLength(1);
16	const input: unknown = NodeVM.runInNewContext(script!);	16	const input: unknown = NodeVM.runInNewContext(script!, { window: { addEventListener() {} } });
17	expect(input).toEqual({ deviceId, ticket: "<ticket>" });	17	expect(input).toEqual({ deviceId, ticket: "<ticket>" });
18	});	18	});
19		19
...	@@ -22,6 +22,39 @@ describe("native device stream document", () => {	...	@@ -22,6 +22,39 @@ describe("native device stream document", () => {
22	expect(html.match(/<\/script>/g)).toHaveLength(1);	22	expect(html.match(/<\/script>/g)).toHaveLength(1);
23	expect(html).toContain('var text="<\\/script>";');	23	expect(html).toContain('var text="<\\/script>";');
24	});	24	});
		25
		26	it("reports a bootstrap exception to the native recovery UI", () => {
		27	const postMessage = vi.fn();
		28	const html = deviceStreamDocument(
		29	"{}",
		30	'var T3DeviceStream={start(){throw new Error("startup")}};',
		31	);
		32	NodeVM.runInNewContext(html.match(/<script>([\s\S]*)<\/script>/)![1]!, {
		33	window: { addEventListener() {}, ReactNativeWebView: { postMessage } },
		34	});
		35	expect(JSON.parse(postMessage.mock.calls[0]![0] as string)).toEqual({
		36	type: "status",
		37	status: "error",
		38	detail: "Device viewer stopped unexpectedly.",
		39	});
		40	});
		41
		42	it.each(["error", "unhandledrejection"])("reports later %s failures to native", (event) => {
		43	const postMessage = vi.fn();
		44	const listeners = new Map<string, () => void>();
		45	const html = deviceStreamDocument("{}", "var T3DeviceStream={start(){}};");
		46	NodeVM.runInNewContext(html.match(/<script>([\s\S]*)<\/script>/)![1]!, {
		47	window: {
		48	addEventListener: (name: string, callback: () => void) => listeners.set(name, callback),
		49	ReactNativeWebView: { postMessage },
		50	},
		51	});
		52	listeners.get(event)!();
		53	expect(JSON.parse(postMessage.mock.calls[0]![0] as string)).toMatchObject({
		54	type: "status",
		55	status: "error",
		56	});
		57	});
25	});	58	});
26		59
27	describe("native device stream messages", () => {	60	describe("native device stream messages", () => {
...	@@ -38,6 +71,12 @@ describe("native device stream messages", () => {	...	@@ -38,6 +71,12 @@ describe("native device stream messages", () => {
38	expect(deviceStreamMessage('{"type":"retry"}')).toEqual({ type: "retry" });	71	expect(deviceStreamMessage('{"type":"retry"}')).toEqual({ type: "retry" });
39	});	72	});
40		73
		74	it.each(["connecting", "streaming", "error"])("accepts stream %s feedback", (status) => {
		75	expect(
		76	deviceStreamMessage(JSON.stringify({ type: "status", status, detail: "Stream feedback" })),
		77	).toEqual({ type: "status", status, detail: "Stream feedback" });
		78	});
		79
41	it.each([	80	it.each([
42	"invalid JSON",	81	"invalid JSON",
43	"null",	82	"null",
...	@@ -45,5 +84,8 @@ describe("native device stream messages", () => {	...	@@ -45,5 +84,8 @@ describe("native device stream messages", () => {
45	"{}",	84	"{}",
46	'{"type":"input","connected":"yes"}',	85	'{"type":"input","connected":"yes"}',
47	'{"type":"unknown"}',	86	'{"type":"unknown"}',
		87	'{"type":"status","status":"unknown"}',
		88	'{"type":"status","status":"error","detail":42}',
		89	'{"type":"status"}',
48	])("ignores invalid bridge messages: %s", (data) => expect(deviceStreamMessage(data)).toBeNull());	90	])("ignores invalid bridge messages: %s", (data) => expect(deviceStreamMessage(data)).toBeNull());
49	});	91	});

apps/mobile/src/features/devices/device-stream-document.ts+17-1

...	@@ -19,7 +19,8 @@ export function deviceStreamDocument(configuration: string, script: string) {	...	@@ -19,7 +19,8 @@ export function deviceStreamDocument(configuration: string, script: string) {
19	// Tickets and device names are data, including any HTML delimiter characters.	19	// Tickets and device names are data, including any HTML delimiter characters.
20	const safeConfiguration = configuration.replace(/</g, "\\u003c");	20	const safeConfiguration = configuration.replace(/</g, "\\u003c");
21	const safeScript = script.replace(/<\/script/gi, "<\\/script");	21	const safeScript = script.replace(/<\/script/gi, "<\\/script");
22	return `<!doctype html><html><head><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"></head><body><script>${safeScript}\nT3DeviceStream.start(${safeConfiguration});</script></body></html>`;	22	const failure = `window.ReactNativeWebView.postMessage(JSON.stringify({type:"status",status:"error",detail:"Device viewer stopped unexpectedly."}));`;
		23	return `<!doctype html><html><head><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"></head><body><script>window.addEventListener("error",function(){${failure}});window.addEventListener("unhandledrejection",function(){${failure}});\n${safeScript}\ntry{T3DeviceStream.start(${safeConfiguration});}catch{${failure}}</script></body></html>`;
23	}	24	}
24		25
25	export function deviceStreamMessage(data: string) {	26	export function deviceStreamMessage(data: string) {
...	@@ -36,6 +37,21 @@ export function deviceStreamMessage(data: string) {	...	@@ -36,6 +37,21 @@ export function deviceStreamMessage(data: string) {
36	) {	37	) {
37	return { type: message.type, connected: message.connected } as const;	38	return { type: message.type, connected: message.connected } as const;
38	}	39	}
		40	if (
		41	message.type === "status" &&
		42	"status" in message &&
		43	(message.status === "connecting" \|\|
		44	message.status === "streaming" \|\|
		45	message.status === "error") &&
		46	(!("detail" in message) \|\| typeof message.detail === "string")
		47	) {
		48	return {
		49	type: message.type,
		50	status: message.status,
		51	detail:
		52	"detail" in message && typeof message.detail === "string" ? message.detail : undefined,
		53	} as const;
		54	}
39	} catch {	55	} catch {
40	// Ignore messages that are not part of the stream bridge.	56	// Ignore messages that are not part of the stream bridge.
41	}	57	}

apps/mobile/src/features/devices/device-stream.browser.test.ts created+117

...	@@ -0,0 +1,117 @@
1	import { afterEach, expect, it, vi } from "vite-plus/test";
2	import { start, stop } from "./device-stream.browser";
3
4	class Element extends EventTarget {
5	readonly style = {};
6	naturalWidth = 0;
7	naturalHeight = 0;
8	src = "";
9	readonly tag: string;
10	constructor(tag: string) {
11	super();
12	this.tag = tag;
13	}
14	setAttribute() {}
15	removeAttribute(name: string) {
16	if (name === "src") this.src = "";
17	}
18	append() {}
19	}
20
21	async function setup() {
22	vi.useFakeTimers();
23	const elements: Element[] = [];
24	vi.stubGlobal("document", {
25	documentElement: { style: {} },
26	body: { style: {}, replaceChildren() {} },
27	createElement: (tag: string) => {
28	const element = new Element(tag);
29	elements.push(element);
30	return element;
31	},
32	});
33	const postMessage = vi.fn();
34	vi.stubGlobal("window", { ReactNativeWebView: { postMessage }, addEventListener() {} });
35	vi.stubGlobal("fetch", () => Promise.resolve(new Response("prime")));
36	const sockets: Socket[] = [];
37	class Socket {
38	static OPEN = 1;
39	readyState = 1;
40	onopen: (() => void) \| null = null;
41	close = vi.fn();
42	send = vi.fn();
43	constructor() {
44	sockets.push(this);
45	}
46	}
47	vi.stubGlobal("WebSocket", Socket);
48	const configuration = {
49	platform: "ios" as const,
50	deviceId: "fixture-device",
51	access: {
52	httpBase: "https://device.test",
53	wsBase: "wss://device.test",
54	credentials: false,
55	query: {},
56	},
57	colors: {
58	background: "white",
59	foreground: "black",
60	muted: "gray",
61	buttonBackground: "gray",
62	buttonForeground: "black",
63	buttonBorder: "gray",
64	},
65	};
66	start(configuration);
67	await vi.advanceTimersByTimeAsync(0);
68	return {
69	configuration,
70	elements,
71	sockets,
72	messages: () =>
73	postMessage.mock.calls.map(
74	([message]) => JSON.parse(message as string) as { type: string; status?: string },
75	),
76	};
77	}
78
79	afterEach(() => {
80	stop();
81	vi.useRealTimers();
82	vi.unstubAllGlobals();
83	});
84
85	it("bridges shared first-frame readiness, image failure, and a successful fresh attempt to native", async () => {
86	const { elements, messages, sockets, configuration } = await setup();
87	sockets[0]!.onopen?.();
88	expect(messages()).not.toContainEqual({ type: "status", status: "streaming" });
89	expect(messages()).toContainEqual({ type: "input", connected: true });
90	const image = elements.find((element) => element.tag === "img")!;
91	image.naturalWidth = 400;
92	image.naturalHeight = 800;
93	await vi.advanceTimersByTimeAsync(250);
94	expect(messages()).toContainEqual({ type: "status", status: "streaming" });
95	image.dispatchEvent(new Event("error"));
96	expect(messages()).toContainEqual({
97	type: "status",
98	status: "error",
99	detail: "Could not receive the device stream. Reconnect to try again.",
100	});
101	expect(messages()).toContainEqual({ type: "input", connected: false });
102	expect(messages()).not.toContainEqual({ type: "unauthorized" });
103	expect(image.src).toBe("");
104	expect(sockets[0]!.close).toHaveBeenCalledOnce();
105	start(configuration);
106	await vi.advanceTimersByTimeAsync(0);
107	const replacement = elements.findLast((element) => element.tag === "img")!;
108	replacement.naturalWidth = 400;
109	replacement.naturalHeight = 800;
110	replacement.dispatchEvent(new Event("load"));
111	expect(messages().at(-1)).toEqual({ type: "status", status: "streaming" });
112	image.dispatchEvent(new Event("error"));
113	expect(sockets[1]!.close).not.toHaveBeenCalled();
114	stop();
115	expect(replacement.src).toBe("");
116	expect(vi.getTimerCount()).toBe(0);
117	});

apps/mobile/src/features/devices/device-stream.browser.ts+10-44

 @@ -12,13 +12,10 @@ declare global {
 }
 let activeClient: ReturnType<typeof createDeviceStreamClient> | null = null;
-let activeImage: HTMLImageElement | null = null;
 export function stop() {
   activeClient?.stop();
   activeClient = null;
-  activeImage?.removeAttribute("src");
-  activeImage = null;
 }
 export function command(button: "home" | "back" | "appSwitcher" | "rotate") {
 @@ -70,34 +67,6 @@ export function start(configuration: DeviceStreamConfiguration) {
   image.alt = "";
   image.draggable = false;
   image.style.display = "none";
-  const overlay = document.createElement("div");
-  overlay.setAttribute("role", "status");
-  Object.assign(overlay.style, {
-    position: "fixed",
-    inset: "0",
-    display: "flex",
-    flexDirection: "column",
-    alignItems: "center",
-    justifyContent: "center",
-    gap: "16px",
-    padding: "24px",
-    textAlign: "center",
-    background: colors.background,
-  });
-  const detail = document.createElement("span");
-  const retry = document.createElement("button");
-  retry.textContent = "Retry";
-  Object.assign(retry.style, {
-    padding: "12px 24px",
-    borderRadius: "20px",
-    border: `1px solid ${colors.buttonBorder}`,
-    background: colors.buttonBackground,
-    color: colors.buttonForeground,
-    font: "inherit",
-    display: "none",
-  });
-  retry.addEventListener("click", () => post({ type: "retry" }));
-  overlay.append(detail, retry);
   const inputStatus = document.createElement("div");
   inputStatus.setAttribute("role", "status");
   inputStatus.textContent = "Reconnecting device controls...";
 @@ -114,11 +83,17 @@ export function start(configuration: DeviceStreamConfiguration) {
   });
   frame.append(canvas, image);
   container.append(frame);
-  document.body.replaceChildren(container, overlay, inputStatus);
+  document.body.replaceChildren(container, inputStatus);
   let pointerId: number | null = null;
   let inputConnected = false;
   let streaming = false;
+  const reportStatus = (status: "connecting" | "streaming" | "error", detail?: string) => {
+    if (activeClient !== client) return;
+    streaming = status === "streaming";
+    inputStatus.style.display = streaming && !inputConnected ? "block" : "none";
+    post({ type: "status", status, detail });
+  };
   const layout = (screen: DeviceScreenSize | null) => {
     const landscape =
       screen?.orientation === "landscape_left" || screen?.orientation === "landscape_right";
 @@ -160,19 +135,11 @@ export function start(configuration: DeviceStreamConfiguration) {
     { ...configuration, preferMjpeg: platform === "ios" },
     canvas,
     {
-      onStatus: (status, message) => {
+      onStatus: reportStatus,
-        streaming = status === "streaming";
-        overlay.style.display = streaming ? "none" : "flex";
-        inputStatus.style.display = streaming && !inputConnected ? "block" : "none";
-        detail.textContent =
-          status === "error" ? (message ?? "Device stream failed.") : "Connecting to device...";
-        retry.style.display = status === "error" ? "block" : "none";
-      },
       onScreen: layout,
-      onMjpegFallback: (url) => {
+      onMjpegFallback: () => {
         canvas.style.display = "none";
         image.style.display = "block";
-        image.src = url;
       },
       onUnauthorized: unauthorized,
       onInputConnected: (connected) => {
 @@ -183,8 +150,7 @@ export function start(configuration: DeviceStreamConfiguration) {
     },
   );
   activeClient = client;
-  activeImage = image;
+  client.setMjpegImage(image);
-  image.addEventListener("error", unauthorized);
   const touch = (event: PointerEvent, phase: "begin" | "move" | "end") => {
     const rect = frame.getBoundingClientRect();
     client.sendTouch(

apps/server/src/device/DeviceService.test.ts+91

...	@@ -11,6 +11,7 @@ import * as Deferred from "effect/Deferred";	...	@@ -11,6 +11,7 @@ import * as Deferred from "effect/Deferred";
11	import * as Fiber from "effect/Fiber";	11	import * as Fiber from "effect/Fiber";
12	import * as PubSub from "effect/PubSub";	12	import * as PubSub from "effect/PubSub";
13	import * as Ref from "effect/Ref";	13	import * as Ref from "effect/Ref";
		14	import * as Schema from "effect/Schema";
14	import * as Stream from "effect/Stream";	15	import * as Stream from "effect/Stream";
15	import { HttpClient, HttpClientResponse } from "effect/unstable/http";	16	import { HttpClient, HttpClientResponse } from "effect/unstable/http";
16	import { ServerSettingsService } from "../serverSettings.ts";	17	import { ServerSettingsService } from "../serverSettings.ts";
...	@@ -19,6 +20,8 @@ import { NodeRuntimeUnavailableError } from "@t3tools/shared/nodeRuntime";	...	@@ -19,6 +20,8 @@ import { NodeRuntimeUnavailableError } from "@t3tools/shared/nodeRuntime";
19		20
20	import { type DeviceService, makeWithHosts, stateStream } from "./DeviceService.ts";	21	import { type DeviceService, makeWithHosts, stateStream } from "./DeviceService.ts";
21		22
		23	const decodeJson = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown));
		24
22	const baseState: DeviceServiceState = {	25	const baseState: DeviceServiceState = {
23	hosts: [],	26	hosts: [],
24	hostStatus: "idle",	27	hostStatus: "idle",
...	@@ -378,3 +381,91 @@ it.effect("keeps shutdown successful when subsequent discovery fails", () =>	...	@@ -378,3 +381,91 @@ it.effect("keeps shutdown successful when subsequent discovery fails", () =>
378	expect(state.devices.find((device) => device.id === session.deviceId)?.booted).toBe(false);	381	expect(state.devices.find((device) => device.id === session.deviceId)?.booted).toBe(false);
379	}).pipe(Effect.scoped),	382	}).pipe(Effect.scoped),
380	);	383	);
		384
		385	it.effect.each(["shutdown", "close"] as const)(
		386	"%s releases iOS capture so reopening uses a fresh session",
		387	(operation) =>
		388	Effect.gen(function* () {
		389	const deviceId = DeviceId.make("11111111-1111-1111-1111-111111111111");
		390	const threadId = ThreadId.make("capture-recovery");
		391	let booted = true;
		392	let capture: number \| null = null;
		393	let generation = 0;
		394	const ready: DeviceHost.DeviceHostReady = {
		395	nodePath: process.execPath,
		396	hub: { origin: "http://device.test" },
		397	helpers: { serveSimAxSettings: null, serveSimCli: null },
		398	run: () => Effect.succeed({ code: 0, stdout: "", stderr: "" }),
		399	};
		400	const host: DeviceHost.DeviceHost["Service"] = {
		401	id: LOCAL_DEVICE_HOST_ID,
		402	summary: Effect.succeed({
		403	id: LOCAL_DEVICE_HOST_ID,
		404	kind: "local",
		405	label: "Simulator host",
		406	platforms: [{ platform: "ios", available: true }],
		407	hubInstalled: true,
		408	agentDeviceInstalled: false,
		409	}),
		410	platformAvailability: (platform) => Effect.succeed({ platform, available: true }),
		411	ensureReady: () => Effect.succeed(ready),
		412	ensureAgentReady: () => Effect.die("Agent access is not used in this test"),
		413	current: Effect.succeed(ready),
		414	stopAgent: Effect.void,
		415	stop: Effect.void,
		416	};
		417	const http = HttpClient.make((request) =>
		418	Effect.sync(() => {
		419	const path = new URL(request.url).pathname;
		420	if (path === "/api/devices") {
		421	return HttpClientResponse.fromWeb(
		422	request,
		423	Response.json({
		424	emulators: [],
		425	simulators: [
		426	{
		427	id: deviceId,
		428	name: "iPhone",
		429	platform: "ios",
		430	version: "26",
		431	physical: false,
		432	booted,
		433	},
		434	],
		435	}),
		436	);
		437	}
		438	if (path === "/vendor/serve-sim/grid/api/start") capture ??= ++generation;
		439	else if (path === "/vendor/serve-sim/grid/api/shutdown") {
		440	if (request.body._tag !== "Uint8Array") throw new Error("Missing shutdown body");
		441	expect(decodeJson(new TextDecoder().decode(request.body.body))).toEqual({
		442	udid: deviceId,
		443	});
		444	capture = null;
		445	booted = false;
		446	} else if (path === "/api/devices/shutdown") {
		447	// This route powers off without releasing serve-sim's cached capture.
		448	booted = false;
		449	} else if (path === "/api/devices/boot") booted = true;
		450	else throw new Error(`Unexpected hub path: ${path}`);
		451	return HttpClientResponse.fromWeb(request, Response.json({ ok: true, id: deviceId }));
		452	}),
		453	);
		454	const service = yield* makeWithHosts(new Map([[host.id, host]])).pipe(
		455	Effect.provideService(HttpClient.HttpClient, http),
		456	);
		457	const input = { threadId, deviceId, platform: "ios" as const };
		458	yield* service.open(input);
		459	expect(capture).toBe(1);
		460	if (operation === "shutdown") yield* service.shutdown(input);
		461	else yield* service.close({ threadId, deviceId, shutdown: true });
		462	expect(capture).toBeNull();
		463	expect((yield* service.state).sessions).toEqual([]);
		464	yield* service.open(input);
		465	expect(capture).toBe(2);
		466	expect((yield* service.state).sessions).toHaveLength(1);
		467	}).pipe(
		468	Effect.provide(ServerSettingsService.layerTest({ enableDeviceSupport: true })),
		469	Effect.scoped,
		470	),
		471	);

apps/server/src/device/DeviceService.ts+8-2

 @@ -670,8 +670,14 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function*
     platform: DevicePlatform,
   ) {
     const ready = yield* readiness(hostId);
-    yield* HttpClientRequest.post(`${ready.hub.origin}/api/devices/shutdown`).pipe(
+    // serve-sim's shutdown closes its in-process capture session before powering off.
-      HttpClientRequest.bodyJson({ platform, id: deviceId }),
+    // The hub's generic shutdown can leave that session cached across a reboot.
+    const path =
+      platform === "ios" ? `${vendorPrefix("ios")}/grid/api/shutdown` : "/api/devices/shutdown";
+    yield* HttpClientRequest.post(`${ready.hub.origin}${path}`).pipe(
+      HttpClientRequest.bodyJson(
+        platform === "ios" ? { udid: deviceId } : { platform, id: deviceId },
+      ),
       Effect.mapError(
         (cause) =>
           new DeviceOperationError({ operation: "shutdown", reason: "invalid_payload", cause }),

apps/web/src/components/device/DeviceLoadingView.tsx+3

...	@@ -1,3 +1,4 @@	...	@@ -1,3 +1,4 @@
		1	import type { ReactNode } from "react";
1	import { Smartphone } from "lucide-react";	2	import { Smartphone } from "lucide-react";
2		3
3	import { Spinner } from "~/components/ui/spinner";	4	import { Spinner } from "~/components/ui/spinner";
...	@@ -8,6 +9,7 @@ export function DeviceLoadingView(props: {	...	@@ -8,6 +9,7 @@ export function DeviceLoadingView(props: {
8	readonly stage: "opening" \| "stream";	9	readonly stage: "opening" \| "stream";
9	readonly message: string;	10	readonly message: string;
10	readonly error?: boolean;	11	readonly error?: boolean;
		12	readonly children?: ReactNode;
11	}) {	13	}) {
12	return (	14	return (
13	<div	15	<div
...	@@ -28,6 +30,7 @@ export function DeviceLoadingView(props: {	...	@@ -28,6 +30,7 @@ export function DeviceLoadingView(props: {
28	{!props.error ? <Spinner className="size-3" /> : null}	30	{!props.error ? <Spinner className="size-3" /> : null}
29	<span>{props.message}</span>	31	<span>{props.message}</span>
30	</div>	32	</div>
		33	{props.children}
31	{!props.error ? (	34	{!props.error ? (
32	<div	35	<div
33	className="flex w-24 gap-1"	36	className="flex w-24 gap-1"

apps/web/src/components/device/DeviceStreamView.test.tsx+66-17

...	@@ -8,25 +8,37 @@ vi.mock("~/state/device", () => ({	...	@@ -8,25 +8,37 @@ vi.mock("~/state/device", () => ({
8	refreshDeviceHubAccess: vi.fn(),	8	refreshDeviceHubAccess: vi.fn(),
9	}));	9	}));
10	const access = { httpBase: "http://test", wsBase: "ws://test", query: {}, credentials: true };	10	const access = { httpBase: "http://test", wsBase: "ws://test", query: {}, credentials: true };
11	vi.mock("@t3tools/client-runtime/device/stream", () => ({
12	createDeviceStreamClient: (
13	_target: unknown,
14	_canvas: unknown,
15	events: { onMjpegFallback: (url: string) => void },
16	) => ({
17	start: () => events.onMjpegFallback("http://test/stream.mjpeg"),
18	stop: vi.fn(),
19	}),
20	}));
21	import { DeviceStreamView } from "./DeviceStreamView";	11	import { DeviceStreamView } from "./DeviceStreamView";
		12
		13	class Image extends EventTarget {
		14	src = "";
		15	naturalWidth = 0;
		16	naturalHeight = 0;
		17	removeAttribute(name: string) {
		18	if (name === "src") this.src = "";
		19	}
		20	}
22	let renderer: ReactTestRenderer \| undefined;	21	let renderer: ReactTestRenderer \| undefined;
23	afterEach(async () => {	22	afterEach(async () => {
24	await act(async () => renderer?.unmount());	23	await act(async () => renderer?.unmount());
		24	renderer = undefined;
		25	vi.useRealTimers();
25	vi.unstubAllGlobals();	26	vi.unstubAllGlobals();
26	});	27	});
27		28
28	it("removes MJPEG requests while hidden and reconnects when shown", async () => {	29	async function setup() {
		30	vi.useFakeTimers();
29	vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true);	31	vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true);
		32	vi.stubGlobal("fetch", () => Promise.resolve(new Response("prime")));
		33	vi.stubGlobal(
		34	"WebSocket",
		35	class {
		36	static OPEN = 1;
		37	readyState = 1;
		38	send() {}
		39	close() {}
		40	},
		41	);
30	vi.stubGlobal(	42	vi.stubGlobal(
31	"ResizeObserver",	43	"ResizeObserver",
32	class {	44	class {
...	@@ -34,6 +46,7 @@ it("removes MJPEG requests while hidden and reconnects when shown", async () =>	...	@@ -34,6 +46,7 @@ it("removes MJPEG requests while hidden and reconnects when shown", async () =>
34	disconnect() {}	46	disconnect() {}
35	},	47	},
36	);	48	);
		49	const images: Image[] = [];
37	const view = (visible: boolean) => (	50	const view = (visible: boolean) => (
38	<DeviceStreamView	51	<DeviceStreamView
39	hostId="local"	52	hostId="local"
...	@@ -45,15 +58,51 @@ it("removes MJPEG requests while hidden and reconnects when shown", async () =>	...	@@ -45,15 +58,51 @@ it("removes MJPEG requests while hidden and reconnects when shown", async () =>
45	);	58	);
46	await act(async () => {	59	await act(async () => {
47	renderer = create(view(true), {	60	renderer = create(view(true), {
48	createNodeMock: () => ({	61	createNodeMock: (node) => {
49	style: { setProperty() {} },	62	if (node.type === "img") {
50	getBoundingClientRect: () => ({ width: 400, height: 800 }),	63	const image = new Image();
51	}),	64	images.push(image);
		65	return image;
		66	}
		67	return {
		68	style: { setProperty() {} },
		69	getBoundingClientRect: () => ({ width: 400, height: 800 }),
		70	};
		71	},
52	});	72	});
53	});	73	});
54	expect(renderer!.root.findAllByType("img")).toHaveLength(1);	74	return { images, view };
		75	}
		76
		77	it("removes MJPEG requests while hidden and reconnects when shown", async () => {
		78	const { images, view } = await setup();
		79	expect(images[0]!.src).toContain("stream.mjpeg");
55	await act(async () => renderer!.update(view(false)));	80	await act(async () => renderer!.update(view(false)));
56	expect(renderer!.root.findAllByType("img")).toHaveLength(0);	81	expect(renderer!.root.findAllByType("img")).toHaveLength(0);
		82	expect(images[0]!.src).toBe("");
		83	expect(vi.getTimerCount()).toBe(0);
57	await act(async () => renderer!.update(view(true)));	84	await act(async () => renderer!.update(view(true)));
58	expect(renderer!.root.findAllByType("img")).toHaveLength(1);	85	expect(images.at(-1)!.src).toContain("stream.mjpeg");
		86	});
		87
		88	it("offers Reconnect after the shared timeout and receives a frame after retry with unchanged access", async () => {
		89	const { images } = await setup();
		90	await act(async () => {
		91	await vi.advanceTimersByTimeAsync(15_000);
		92	});
		93	expect(
		94	renderer!.root.findByProps({ role: "alert" }).findByType("span").children.join(""),
		95	).toContain("No video");
		96	expect(images[0]!.src).toBe("");
		97	await act(async () => renderer!.root.findByType("button").props.onClick());
		98	expect(renderer!.root.findAllByProps({ role: "alert" })).toHaveLength(0);
		99	expect(images[1]!.src).toContain("stream.mjpeg");
		100	await act(async () => {
		101	images[1]!.naturalWidth = 400;
		102	images[1]!.naturalHeight = 800;
		103	images[1]!.dispatchEvent(new Event("load"));
		104	});
		105	expect(renderer!.root.findAllByProps({ role: "status" })).toHaveLength(0);
		106	expect(renderer!.root.findAllByType("button")).toHaveLength(0);
		107	expect(vi.getTimerCount()).toBe(0);
59	});	108	});

apps/web/src/components/device/DeviceStreamView.tsx+24-4

 @@ -1,7 +1,8 @@
 import type { DevicePlatform, EnvironmentId } from "@t3tools/contracts";
-import { useEffect, useMemo, useRef, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { cn } from "~/lib/utils";
+import { Button } from "~/components/ui/button";
 import { refreshDeviceHubAccess, useDeviceHubAccess } from "~/state/device";
 import { DeviceLoadingView } from "./DeviceLoadingView";
 import { type DeviceAxElement, fetchDeviceAxTree } from "./deviceHubApi";
 @@ -49,6 +50,9 @@ export function DeviceStreamView(props: {
   const [screen, setScreen] = useState<DeviceScreenSize | null>(null);
   const [mjpegUrl, setMjpegUrl] = useState<string | null>(null);
   const [mjpegGeneration, setMjpegGeneration] = useState(0);
+  const attachMjpegImage = useCallback((image: HTMLImageElement | null) => {
+    clientRef.current?.setMjpegImage(image);
+  }, []);
   const [inputState, setInputState] = useState<{ connected: boolean; detail?: string }>({
     connected: false,
   });
 @@ -235,11 +239,13 @@ export function DeviceStreamView(props: {
       role="application"
       aria-label={`${props.platform === "ios" ? "iOS Simulator" : "Android Emulator"} screen`}
       onKeyDown={(event) => {
+        if (event.target !== event.currentTarget) return;
         if (event.metaKey && !["r", "R"].includes(event.key)) return;
         event.preventDefault();
         clientRef.current?.sendKey(event.nativeEvent, "down");
       }}
       onKeyUp={(event) => {
+        if (event.target !== event.currentTarget) return;
         clientRef.current?.sendKey(event.nativeEvent, "up");
       }}
     >
 @@ -279,7 +285,7 @@ export function DeviceStreamView(props: {
         {props.visible && access && mjpegUrl ? (
           <img
             key={mjpegGeneration}
-            src={mjpegUrl}
+            ref={attachMjpegImage}
             alt=""
             draggable={false}
             className="absolute top-0 left-0 object-contain"
 @@ -317,14 +323,28 @@ export function DeviceStreamView(props: {
         </div>
       ) : null}
       {status !== "streaming" ? (
-        <div className="pointer-events-none absolute inset-0">
+        <div className="absolute inset-0">
           <DeviceLoadingView
             name={props.deviceName ?? "Device"}
             description={props.deviceDescription ?? ""}
             stage="stream"
             message={status === "error" ? (detail ?? "Stream failed.") : "Connecting video…"}
             error={status === "error"}
-          />
+          >
+            {status === "error" ? (
+              <Button
+                size="sm"
+                variant="outline"
+                onClick={() => {
+                  refreshDeviceHubAccess(props.environmentId);
+                  clientRef.current?.stop();
+                  clientRef.current?.start();
+                }}
+              >
+                Reconnect
+              </Button>
+            ) : null}
+          </DeviceLoadingView>
         </div>
       ) : null}
     </div>

apps/web/src/components/device/deviceStream.test.ts deleted-345

...	@@ -1,345 +0,0 @@
1	import { afterEach, describe, expect, it, vi } from "vite-plus/test";
2
3	import {
4	createDeviceStreamClient,
5	AvccDemuxer,
6	avcCodecString,
7	parseSemuPacket,
8	scanAccessUnit,
9	} from "@t3tools/client-runtime/device/stream";
10
11	const envelope = (tag: number, payload: number[]) => {
12	const length = 1 + payload.length;
13	return [
14	(length >>> 24) & 0xff,
15	(length >>> 16) & 0xff,
16	(length >>> 8) & 0xff,
17	length & 0xff,
18	tag,
19	...payload,
20	];
21	};
22
23	describe("AvccDemuxer", () => {
24	it("reassembles envelopes split across reads", () => {
25	const demuxer = new AvccDemuxer();
26	const bytes = new Uint8Array([
27	...envelope(1, [1, 0x64, 0x00, 0x1f]),
28	...envelope(2, [9, 9, 9]),
29	...envelope(0x7f, [0]),
30	...envelope(3, [4]),
31	]);
32	const first = demuxer.push(bytes.subarray(0, 7));
33	const rest = demuxer.push(bytes.subarray(7));
34	const chunks = [...first, ...rest];
35	expect(chunks.map((chunk) => chunk.type)).toEqual(["description", "keyframe", "delta"]);
36	expect(Array.from(chunks[0]!.payload)).toEqual([1, 0x64, 0x00, 0x1f]);
37	expect(Array.from(chunks[1]!.payload)).toEqual([9, 9, 9]);
38	});
39
40	it("derives the WebCodecs codec string from the avcC record", () => {
41	expect(avcCodecString(new Uint8Array([1, 0x64, 0x00, 0x1f]))).toBe("avc1.64001f");
42	expect(avcCodecString(new Uint8Array([1]))).toBe("avc1.42E01E");
43	});
44	});
45
46	describe("native device stream transport", () => {
47	afterEach(() => {
48	vi.useRealTimers();
49	vi.unstubAllGlobals();
50	});
51
52	function setup(platform: "ios" \| "android") {
53	vi.useFakeTimers();
54	vi.stubGlobal("VideoDecoder", vi.fn());
55	vi.stubGlobal("EncodedVideoChunk", vi.fn());
56	let resolveOpened!: (socket: FakeSocket) => void;
57	const waitForSocket = () =>
58	new Promise<FakeSocket>((resolve) => {
59	resolveOpened = resolve;
60	});
61	const opened = waitForSocket();
62	class FakeSocket {
63	static OPEN = 1;
64	readyState = 1;
65	binaryType = "";
66	onopen: (() => void) \| null = null;
67	onclose: ((event: { code: number; reason: string }) => void) \| null = null;
68	send = vi.fn();
69	close = vi.fn();
70	readonly url: string;
71	constructor(url: string) {
72	this.url = url;
73	resolveOpened(this);
74	}
75	}
76	vi.stubGlobal("WebSocket", FakeSocket);
77	const fetch = vi.fn(() => Promise.resolve(new Response("frame")));
78	vi.stubGlobal("fetch", fetch);
79	const events = {
80	onStatus: vi.fn(),
81	onScreen: vi.fn(),
82	onUnauthorized: vi.fn(),
83	onMjpegFallback: vi.fn(),
84	onInputConnected: vi.fn(),
85	};
86	const client = createDeviceStreamClient(
87	{
88	platform,
89	deviceId: "test device",
90	preferMjpeg: platform === "ios",
91	access: {
92	httpBase: "https://environment.test/api/device-hub",
93	wsBase: "wss://environment.test/api/device-hub",
94	credentials: false,
95	query: { wsTicket: "stream-ticket", hostId: "ssh-host" },
96	},
97	},
98	{ getContext: () => null } as unknown as HTMLCanvasElement,
99	events,
100	);
101	return { client, opened, waitForSocket, events, fetch };
102	}
103
104	it("uses authenticated iOS MJPEG and forwards controls without a cross-origin video fetch", async () => {
105	const { client, opened, events, fetch } = setup("ios");
106	client.start();
107	const socket = await opened;
108	const imageUrl = new URL(events.onMjpegFallback.mock.calls[0]![0] as string);
109	expect(imageUrl.pathname).toContain("test%20device/stream.mjpeg");
110	expect(imageUrl.searchParams.get("wsTicket")).toBe("stream-ticket");
111	expect(imageUrl.searchParams.get("hostId")).toBe("ssh-host");
112	expect(fetch).toHaveBeenCalledTimes(1);
113	socket.onopen?.();
114	client.pressButton("home");
115	client.sendTouch("begin", 0.25, 0.75);
116	const messages = socket.send.mock.calls.slice(1).map(([bytes]) => {
117	const packet = bytes as Uint8Array;
118	return {
119	tag: packet[0],
120	body: JSON.parse(new TextDecoder().decode(packet.subarray(1))) as unknown,
121	};
122	});
123	expect(messages).toEqual([
124	{ tag: 0x04, body: { button: "home" } },
125	{ tag: 0x03, body: { type: "begin", x: 0.25, y: 0.75 } },
126	]);
127	client.stop();
128	expect(socket.close).toHaveBeenCalledTimes(1);
129	expect(vi.getTimerCount()).toBe(0);
130	});
131
132	it("forwards Android gestures and device controls through the ticketed host socket", async () => {
133	const { client, opened } = setup("android");
134	client.start();
135	const socket = await opened;
136	const url = new URL(socket.url);
137	expect(url.searchParams.get("device")).toBe("test device");
138	expect(url.searchParams.get("wsTicket")).toBe("stream-ticket");
139	expect(url.searchParams.get("hostId")).toBe("ssh-host");
140	client.sendTouch("begin", 0.2, 0.8);
141	client.sendTouch("move", 0.2, 0.4);
142	client.sendTouch("end", 0.2, 0.1);
143	client.pressButton("home");
144	client.pressButton("appSwitcher");
145	expect(
146	socket.send.mock.calls.map(([message]) => JSON.parse(message as string) as unknown),
147	).toEqual([
148	{ type: "touch", action: "down", x: 0.2, y: 0.8 },
149	{ type: "touch", action: "move", x: 0.2, y: 0.4 },
150	{ type: "touch", action: "up", x: 0.2, y: 0.1 },
151	{ type: "home" },
152	{ type: "recents" },
153	]);
154	client.stop();
155	expect(socket.close).toHaveBeenCalledTimes(1);
156	});
157
158	it("renews an expired Android ticket when the HTTP upgrade is rejected instead of retrying it forever", async () => {
159	const { client, opened, events } = setup("android");
160	client.start();
161	const socket = await opened;
162	socket.onclose?.({ code: 1006, reason: "" });
163	expect(events.onUnauthorized).toHaveBeenCalledTimes(1);
164	expect(vi.getTimerCount()).toBe(0);
165	client.stop();
166	});
167
168	it.each(["ios", "android"] as const)(
169	"keeps a restarted %s stream connected when the discarded socket closes late",
170	async (platform) => {
171	const { client, opened, waitForSocket, events } = setup(platform);
172	client.start();
173	const discarded = await opened;
174	client.stop();
175	const reopened = waitForSocket();
176	client.start();
177	const replacement = await reopened;
178	replacement.onopen?.();
179	replacement.send.mockClear();
180	events.onInputConnected.mockClear();
181
182	discarded.onclose?.({ code: 1006, reason: "" });
183
184	expect(events.onUnauthorized).not.toHaveBeenCalled();
185	expect(events.onInputConnected).not.toHaveBeenCalled();
186	expect(replacement.close).not.toHaveBeenCalled();
187	client.pressButton("home");
188	expect(replacement.send).toHaveBeenCalledTimes(1);
189	client.stop();
190	expect(replacement.close).toHaveBeenCalledTimes(1);
191	expect(vi.getTimerCount()).toBe(0);
192	},
193	);
194
195	it.each(["ios", "android"] as const)(
196	"does not renew credentials or reconnect a stopped %s stream",
197	async (platform) => {
198	const { client, opened, events } = setup(platform);
199	client.start();
200	const socket = await opened;
201	events.onInputConnected.mockClear();
202	client.stop();
203	socket.onclose?.({ code: 1006, reason: "" });
204	expect(events.onUnauthorized).not.toHaveBeenCalled();
205	expect(events.onInputConnected).not.toHaveBeenCalled();
206	expect(vi.getTimerCount()).toBe(0);
207	},
208	);
209	});
210
211	describe("serve-emu frames", () => {
212	it("strips the SEMU header and reads the keyframe flag and timestamp", () => {
213	const buffer = new ArrayBuffer(16 + 3);
214	const view = new DataView(buffer);
215	view.setUint32(0, 0x53454d55);
216	view.setUint8(4, 1);
217	view.setUint8(5, 1);
218	view.setBigUint64(8, 123456n);
219	new Uint8Array(buffer).set([7, 8, 9], 16);
220	const packet = parseSemuPacket(buffer);
221	expect(packet.isKey).toBe(true);
222	expect(packet.timestamp).toBe(123456);
223	expect(Array.from(packet.data)).toEqual([7, 8, 9]);
224	});
225
226	it("treats a frame without the header as raw data", () => {
227	const packet = parseSemuPacket(new Uint8Array([0, 0, 1, 0x65]).buffer);
228	expect(packet.isKey).toBeNull();
229	expect(packet.data.length).toBe(4);
230	});
231
232	it("finds the SPS and IDR NAL units in an Annex-B access unit", () => {
233	const unit = new Uint8Array([0, 0, 0, 1, 0x67, 0x64, 0x00, 0x1f, 0, 0, 1, 0x65, 0xaa]);
234	const scanned = scanAccessUnit(unit);
235	expect(scanned.isKey).toBe(true);
236	expect(scanned.sps && avcCodecString(scanned.sps)).toBe("avc1.64001f");
237	expect(scanAccessUnit(new Uint8Array([0, 0, 1, 0x41, 0x00])).isKey).toBe(false);
238	});
239	});
240
241	describe("iOS input startup", () => {
242	afterEach(() => {
243	vi.useRealTimers();
244	vi.unstubAllGlobals();
245	});
246
247	const setup = () => {
248	vi.useFakeTimers();
249	const sockets: FakeSocket[] = [];
250	class FakeSocket {
251	static OPEN = 1;
252	readyState = 1;
253	binaryType = "";
254	onopen: (() => void) \| null = null;
255	onmessage: ((event: { data: ArrayBuffer }) => void) \| null = null;
256	onclose: ((event: { code: number; reason: string }) => void) \| null = null;
257	send = vi.fn();
258	close = vi.fn();
259	constructor() {
260	sockets.push(this);
261	}
262	}
263	vi.stubGlobal("WebSocket", FakeSocket);
264	const signals: AbortSignal[] = [];
265	vi.stubGlobal(
266	"fetch",
267	vi.fn((_url, init: RequestInit) => {
268	const signal = init.signal!;
269	signals.push(signal);
270	return Promise.resolve(
271	new Response(
272	new ReadableStream({
273	start(controller) {
274	signal.addEventListener("abort", () => controller.error(new Error("aborted")));
275	},
276	}),
277	),
278	);
279	}),
280	);
281	const client = createDeviceStreamClient(
282	{
283	platform: "ios",
284	deviceId: "test-device",
285	access: {
286	httpBase: "http://test/api/device-hub",
287	wsBase: "ws://test/api/device-hub",
288	credentials: true,
289	query: {},
290	},
291	},
292	{ getContext: () => null } as unknown as HTMLCanvasElement,
293	{
294	onStatus: vi.fn(),
295	onScreen: vi.fn(),
296	onUnauthorized: vi.fn(),
297	onMjpegFallback: vi.fn(),
298	onInputConnected: vi.fn(),
299	},
300	);
301	return { client, sockets, signals };
302	};
303
304	it("connects input when the MJPEG prime never produces a frame", async () => {
305	const { client, sockets, signals } = setup();
306	client.start();
307	await vi.advanceTimersByTimeAsync(2_000);
308	expect(signals[0]?.aborted).toBe(true);
309	expect(sockets).toHaveLength(1);
310	client.stop();
311	});
312
313	it("aborts priming immediately when hidden without opening a socket later", async () => {
314	const { client, sockets, signals } = setup();
315	client.start();
316	await vi.advanceTimersByTimeAsync(0);
317	client.stop();
318	expect(signals[0]?.aborted).toBe(true);
319	await vi.advanceTimersByTimeAsync(2_000);
320	expect(sockets).toHaveLength(0);
321	});
322
323	it.each([
324	["landscape_left", 0.7, 1 - 0.2],
325	["landscape_right", 1 - 0.7, 0.2],
326	])("maps %s touches back to the raw iOS framebuffer", async (orientation, x, y) => {
327	const { client, sockets } = setup();
328	client.start();
329	await vi.advanceTimersByTimeAsync(2_000);
330	const socket = sockets[0]!;
331	const json = new TextEncoder().encode(JSON.stringify({ width: 400, height: 800, orientation }));
332	const packet = new Uint8Array(1 + json.length);
333	packet[0] = 0x82;
334	packet.set(json, 1);
335	socket.onmessage?.({ data: packet.buffer });
336	client.sendTouch("begin", 0.2, 0.7);
337	const sent = socket.send.mock.calls[0]![0] as Uint8Array;
338	expect(JSON.parse(new TextDecoder().decode(sent.subarray(1)))).toEqual({
339	type: "begin",
340	x,
341	y,
342	});
343	client.stop();
344	});
345	});

packages/client-runtime/src/device/stream.test.ts created+735

...	@@ -0,0 +1,735 @@
1	import { afterEach, describe, expect, it, vi } from "vite-plus/test";
2
3	import {
4	createDeviceStreamClient,
5	AvccDemuxer,
6	avcCodecString,
7	parseSemuPacket,
8	scanAccessUnit,
9	} from "./stream.ts";
10
11	const envelope = (tag: number, payload: number[]) => {
12	const length = 1 + payload.length;
13	return [
14	(length >>> 24) & 0xff,
15	(length >>> 16) & 0xff,
16	(length >>> 8) & 0xff,
17	length & 0xff,
18	tag,
19	...payload,
20	];
21	};
22
23	describe("AvccDemuxer", () => {
24	it("reassembles envelopes split across reads", () => {
25	const demuxer = new AvccDemuxer();
26	const bytes = new Uint8Array([
27	...envelope(1, [1, 0x64, 0x00, 0x1f]),
28	...envelope(2, [9, 9, 9]),
29	...envelope(0x7f, [0]),
30	...envelope(3, [4]),
31	]);
32	const first = demuxer.push(bytes.subarray(0, 7));
33	const rest = demuxer.push(bytes.subarray(7));
34	const chunks = [...first, ...rest];
35	expect(chunks.map((chunk) => chunk.type)).toEqual(["description", "keyframe", "delta"]);
36	expect(Array.from(chunks[0]!.payload)).toEqual([1, 0x64, 0x00, 0x1f]);
37	expect(Array.from(chunks[1]!.payload)).toEqual([9, 9, 9]);
38	});
39
40	it("derives the WebCodecs codec string from the avcC record", () => {
41	expect(avcCodecString(new Uint8Array([1, 0x64, 0x00, 0x1f]))).toBe("avc1.64001f");
42	expect(avcCodecString(new Uint8Array([1]))).toBe("avc1.42E01E");
43	});
44	});
45
46	describe("native device stream transport", () => {
47	afterEach(() => {
48	vi.useRealTimers();
49	vi.unstubAllGlobals();
50	});
51
52	function setup(platform: "ios" \| "android") {
53	vi.useFakeTimers();
54	vi.stubGlobal("VideoDecoder", vi.fn());
55	vi.stubGlobal("EncodedVideoChunk", vi.fn());
56	let resolveOpened!: (socket: FakeSocket) => void;
57	const waitForSocket = () =>
58	new Promise<FakeSocket>((resolve) => {
59	resolveOpened = resolve;
60	});
61	const opened = waitForSocket();
62	class FakeSocket {
63	static OPEN = 1;
64	readyState = 1;
65	binaryType = "";
66	onopen: (() => void) \| null = null;
67	onclose: ((event: { code: number; reason: string }) => void) \| null = null;
68	send = vi.fn();
69	close = vi.fn();
70	readonly url: string;
71	constructor(url: string) {
72	this.url = url;
73	resolveOpened(this);
74	}
75	}
76	vi.stubGlobal("WebSocket", FakeSocket);
77	const fetch = vi.fn(() => Promise.resolve(new Response("frame")));
78	vi.stubGlobal("fetch", fetch);
79	const events = {
80	onStatus: vi.fn(),
81	onScreen: vi.fn(),
82	onUnauthorized: vi.fn(),
83	onMjpegFallback: vi.fn(),
84	onInputConnected: vi.fn(),
85	};
86	const client = createDeviceStreamClient(
87	{
88	platform,
89	deviceId: "test device",
90	preferMjpeg: platform === "ios",
91	access: {
92	httpBase: "https://environment.test/api/device-hub",
93	wsBase: "wss://environment.test/api/device-hub",
94	credentials: false,
95	query: { wsTicket: "stream-ticket", hostId: "ssh-host" },
96	},
97	},
98	{ getContext: () => null } as unknown as HTMLCanvasElement,
99	events,
100	);
101	return { client, opened, waitForSocket, events, fetch };
102	}
103
104	it("uses authenticated iOS MJPEG and forwards controls without a cross-origin video fetch", async () => {
105	const { client, opened, events, fetch } = setup("ios");
106	client.start();
107	const socket = await opened;
108	const imageUrl = new URL(events.onMjpegFallback.mock.calls[0]![0] as string);
109	expect(imageUrl.pathname).toContain("test%20device/stream.mjpeg");
110	expect(imageUrl.searchParams.get("wsTicket")).toBe("stream-ticket");
111	expect(imageUrl.searchParams.get("hostId")).toBe("ssh-host");
112	expect(fetch).toHaveBeenCalledTimes(1);
113	socket.onopen?.();
114	client.pressButton("home");
115	client.sendTouch("begin", 0.25, 0.75);
116	const messages = socket.send.mock.calls.slice(1).map(([bytes]) => {
117	const packet = bytes as Uint8Array;
118	return {
119	tag: packet[0],
120	body: JSON.parse(new TextDecoder().decode(packet.subarray(1))) as unknown,
121	};
122	});
123	expect(messages).toEqual([
124	{ tag: 0x04, body: { button: "home" } },
125	{ tag: 0x03, body: { type: "begin", x: 0.25, y: 0.75 } },
126	]);
127	client.stop();
128	expect(socket.close).toHaveBeenCalledTimes(1);
129	expect(vi.getTimerCount()).toBe(0);
130	});
131
132	it("forwards Android gestures and device controls through the ticketed host socket", async () => {
133	const { client, opened } = setup("android");
134	client.start();
135	const socket = await opened;
136	const url = new URL(socket.url);
137	expect(url.searchParams.get("device")).toBe("test device");
138	expect(url.searchParams.get("wsTicket")).toBe("stream-ticket");
139	expect(url.searchParams.get("hostId")).toBe("ssh-host");
140	client.sendTouch("begin", 0.2, 0.8);
141	client.sendTouch("move", 0.2, 0.4);
142	client.sendTouch("end", 0.2, 0.1);
143	client.pressButton("home");
144	client.pressButton("appSwitcher");
145	expect(
146	socket.send.mock.calls.map(([message]) => JSON.parse(message as string) as unknown),
147	).toEqual([
148	{ type: "touch", action: "down", x: 0.2, y: 0.8 },
149	{ type: "touch", action: "move", x: 0.2, y: 0.4 },
150	{ type: "touch", action: "up", x: 0.2, y: 0.1 },
151	{ type: "home" },
152	{ type: "recents" },
153	]);
154	client.stop();
155	expect(socket.close).toHaveBeenCalledTimes(1);
156	});
157
158	it("renews an expired Android ticket when the HTTP upgrade is rejected instead of retrying it forever", async () => {
159	const { client, opened, events } = setup("android");
160	client.start();
161	const socket = await opened;
162	socket.onclose?.({ code: 1006, reason: "" });
163	expect(events.onUnauthorized).toHaveBeenCalledTimes(1);
164	expect(vi.getTimerCount()).toBe(0);
165	client.stop();
166	});
167
168	it.each(["ios", "android"] as const)(
169	"keeps a restarted %s stream connected when the discarded socket closes late",
170	async (platform) => {
171	const { client, opened, waitForSocket, events } = setup(platform);
172	client.start();
173	const discarded = await opened;
174	client.stop();
175	const reopened = waitForSocket();
176	client.start();
177	const replacement = await reopened;
178	replacement.onopen?.();
179	replacement.send.mockClear();
180	events.onInputConnected.mockClear();
181
182	discarded.onclose?.({ code: 1006, reason: "" });
183
184	expect(events.onUnauthorized).not.toHaveBeenCalled();
185	expect(events.onInputConnected).not.toHaveBeenCalled();
186	expect(replacement.close).not.toHaveBeenCalled();
187	client.pressButton("home");
188	expect(replacement.send).toHaveBeenCalledTimes(1);
189	client.stop();
190	expect(replacement.close).toHaveBeenCalledTimes(1);
191	expect(vi.getTimerCount()).toBe(0);
192	},
193	);
194
195	it.each(["ios", "android"] as const)(
196	"does not renew credentials or reconnect a stopped %s stream",
197	async (platform) => {
198	const { client, opened, events } = setup(platform);
199	client.start();
200	const socket = await opened;
201	events.onInputConnected.mockClear();
202	client.stop();
203	socket.onclose?.({ code: 1006, reason: "" });
204	expect(events.onUnauthorized).not.toHaveBeenCalled();
205	expect(events.onInputConnected).not.toHaveBeenCalled();
206	expect(vi.getTimerCount()).toBe(0);
207	},
208	);
209	});
210
211	describe("serve-emu frames", () => {
212	it("strips the SEMU header and reads the keyframe flag and timestamp", () => {
213	const buffer = new ArrayBuffer(16 + 3);
214	const view = new DataView(buffer);
215	view.setUint32(0, 0x53454d55);
216	view.setUint8(4, 1);
217	view.setUint8(5, 1);
218	view.setBigUint64(8, 123456n);
219	new Uint8Array(buffer).set([7, 8, 9], 16);
220	const packet = parseSemuPacket(buffer);
221	expect(packet.isKey).toBe(true);
222	expect(packet.timestamp).toBe(123456);
223	expect(Array.from(packet.data)).toEqual([7, 8, 9]);
224	});
225
226	it("treats a frame without the header as raw data", () => {
227	const packet = parseSemuPacket(new Uint8Array([0, 0, 1, 0x65]).buffer);
228	expect(packet.isKey).toBeNull();
229	expect(packet.data.length).toBe(4);
230	});
231
232	it("finds the SPS and IDR NAL units in an Annex-B access unit", () => {
233	const unit = new Uint8Array([0, 0, 0, 1, 0x67, 0x64, 0x00, 0x1f, 0, 0, 1, 0x65, 0xaa]);
234	const scanned = scanAccessUnit(unit);
235	expect(scanned.isKey).toBe(true);
236	expect(scanned.sps && avcCodecString(scanned.sps)).toBe("avc1.64001f");
237	expect(scanAccessUnit(new Uint8Array([0, 0, 1, 0x41, 0x00])).isKey).toBe(false);
238	});
239	});
240
241	describe("iOS input startup", () => {
242	afterEach(() => {
243	vi.useRealTimers();
244	vi.unstubAllGlobals();
245	});
246
247	const setup = () => {
248	vi.useFakeTimers();
249	const sockets: FakeSocket[] = [];
250	class FakeSocket {
251	static OPEN = 1;
252	readyState = 1;
253	binaryType = "";
254	onopen: (() => void) \| null = null;
255	onmessage: ((event: { data: ArrayBuffer }) => void) \| null = null;
256	onclose: ((event: { code: number; reason: string }) => void) \| null = null;
257	send = vi.fn();
258	close = vi.fn();
259	constructor() {
260	sockets.push(this);
261	}
262	}
263	vi.stubGlobal("WebSocket", FakeSocket);
264	const signals: AbortSignal[] = [];
265	vi.stubGlobal(
266	"fetch",
267	vi.fn((_url, init: RequestInit) => {
268	const signal = init.signal!;
269	signals.push(signal);
270	return Promise.resolve(
271	new Response(
272	new ReadableStream({
273	start(controller) {
274	signal.addEventListener("abort", () => controller.error(new Error("aborted")));
275	},
276	}),
277	),
278	);
279	}),
280	);
281	const client = createDeviceStreamClient(
282	{
283	platform: "ios",
284	deviceId: "test-device",
285	access: {
286	httpBase: "http://test/api/device-hub",
287	wsBase: "ws://test/api/device-hub",
288	credentials: true,
289	query: {},
290	},
291	},
292	{ getContext: () => null } as unknown as HTMLCanvasElement,
293	{
294	onStatus: vi.fn(),
295	onScreen: vi.fn(),
296	onUnauthorized: vi.fn(),
297	onMjpegFallback: vi.fn(),
298	onInputConnected: vi.fn(),
299	},
300	);
301	return { client, sockets, signals };
302	};
303
304	it("connects input when the MJPEG prime never produces a frame", async () => {
305	const { client, sockets, signals } = setup();
306	client.start();
307	await vi.advanceTimersByTimeAsync(2_000);
308	expect(signals[0]?.aborted).toBe(true);
309	expect(sockets).toHaveLength(1);
310	client.stop();
311	});
312
313	it("aborts priming immediately when hidden without opening a socket later", async () => {
314	const { client, sockets, signals } = setup();
315	client.start();
316	await vi.advanceTimersByTimeAsync(0);
317	client.stop();
318	expect(signals[0]?.aborted).toBe(true);
319	await vi.advanceTimersByTimeAsync(2_000);
320	expect(sockets).toHaveLength(0);
321	});
322
323	it.each([
324	["landscape_left", 0.7, 1 - 0.2],
325	["landscape_right", 1 - 0.7, 0.2],
326	])("maps %s touches back to the raw iOS framebuffer", async (orientation, x, y) => {
327	const { client, sockets } = setup();
328	client.start();
329	await vi.advanceTimersByTimeAsync(2_000);
330	const socket = sockets[0]!;
331	const json = new TextEncoder().encode(JSON.stringify({ width: 400, height: 800, orientation }));
332	const packet = new Uint8Array(1 + json.length);
333	packet[0] = 0x82;
334	packet.set(json, 1);
335	socket.onmessage?.({ data: packet.buffer });
336	client.sendTouch("begin", 0.2, 0.7);
337	const sent = socket.send.mock.calls[0]![0] as Uint8Array;
338	expect(JSON.parse(new TextDecoder().decode(sent.subarray(1)))).toEqual({
339	type: "begin",
340	x,
341	y,
342	});
343	client.stop();
344	});
345	});
346
347	class MjpegImage extends EventTarget {
348	naturalWidth = 0;
349	naturalHeight = 0;
350	src = "";
351	removeAttribute(name: string) {
352	if (name === "src") {
353	this.src = "";
354	this.naturalWidth = 0;
355	this.naturalHeight = 0;
356	}
357	}
358	}
359
360	function recoveryFixture(platform: "ios" \| "android" = "ios", preferMjpeg = true) {
361	vi.useFakeTimers();
362	const sockets: Socket[] = [];
363	class Socket {
364	static OPEN = 1;
365	readyState = 1;
366	binaryType = "";
367	onopen: (() => void) \| null = null;
368	onmessage: ((event: { data: ArrayBuffer \| string }) => void) \| null = null;
369	onclose: ((event: { code: number; reason: string }) => void) \| null = null;
370	send = vi.fn();
371	close = vi.fn();
372	constructor() {
373	sockets.push(this);
374	}
375	}
376	vi.stubGlobal("WebSocket", Socket);
377	const decoders: Decoder[] = [];
378	class Decoder {
379	static isConfigSupported = vi.fn(async () => ({ supported: true }));
380	state = "unconfigured";
381	decodeQueueSize = 0;
382	configure() {
383	this.state = "configured";
384	}
385	close() {
386	this.state = "closed";
387	}
388	decode() {}
389	readonly callbacks: { output: (frame: VideoFrame) => void; error: () => void };
390	constructor(callbacks: { output: (frame: VideoFrame) => void; error: () => void }) {
391	this.callbacks = callbacks;
392	decoders.push(this);
393	}
394	}
395	vi.stubGlobal("VideoDecoder", Decoder);
396	vi.stubGlobal("EncodedVideoChunk", vi.fn());
397	const videoBodies: ReadableStreamDefaultController<Uint8Array>[] = [];
398	const signals: AbortSignal[] = [];
399	const fetch = vi.fn((url: string, init: RequestInit) => {
400	const signal = init.signal!;
401	signals.push(signal);
402	if (!url.includes("stream.avcc")) return Promise.resolve(new Response("prime"));
403	return Promise.resolve(
404	new Response(
405	new ReadableStream<Uint8Array>({
406	start(body) {
407	videoBodies.push(body);
408	signal.addEventListener("abort", () => body.error(new Error("aborted")), {
409	once: true,
410	});
411	},
412	}),
413	),
414	);
415	});
416	vi.stubGlobal("fetch", fetch);
417	const events = {
418	onStatus: vi.fn(),
419	onScreen: vi.fn(),
420	onUnauthorized: vi.fn(),
421	onMjpegFallback: vi.fn(),
422	onInputConnected: vi.fn(),
423	};
424	const drawImage = vi.fn();
425	const image = new MjpegImage();
426	const client = createDeviceStreamClient(
427	{
428	platform,
429	preferMjpeg,
430	deviceId: "test-device",
431	access: {
432	httpBase: "https://test/api/device-hub",
433	wsBase: "wss://test/api/device-hub",
434	credentials: true,
435	query: {},
436	},
437	},
438	{ getContext: () => ({ drawImage }) } as unknown as HTMLCanvasElement,
439	events,
440	);
441	client.setMjpegImage(image as unknown as HTMLImageElement);
442	const imageReady = () => {
443	image.naturalWidth = 400;
444	image.naturalHeight = 800;
445	image.dispatchEvent(new Event("load"));
446	};
447	const decodedFrame = (index = decoders.length - 1) => {
448	const close = vi.fn();
449	decoders[index]!.callbacks.output({
450	displayWidth: 400,
451	displayHeight: 800,
452	close,
453	} as unknown as VideoFrame);
454	return close;
455	};
456	const sps = new Uint8Array([0, 0, 0, 1, 0x67, 0x64, 0, 0x1f, 0, 0, 1, 0x65]).buffer;
457	return {
458	client,
459	events,
460	image,
461	imageReady,
462	sockets,
463	signals,
464	fetch,
465	videoBodies,
466	decoders,
467	Decoder,
468	drawImage,
469	decodedFrame,
470	sps,
471	};
472	}
473
474	describe("shared device stream readiness and recovery", () => {
475	afterEach(() => {
476	vi.useRealTimers();
477	vi.unstubAllGlobals();
478	});
479
480	it("waits for actual MJPEG dimensions without requiring a multipart load event", async () => {
481	const { client, events, image } = recoveryFixture();
482	client.start();
483	await vi.advanceTimersByTimeAsync(0);
484	expect(image.src).toContain("stream.mjpeg");
485	expect(events.onStatus).not.toHaveBeenCalledWith("streaming", undefined);
486	image.naturalWidth = 400;
487	image.naturalHeight = 800;
488	await vi.advanceTimersByTimeAsync(250);
489	expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);
490	expect(vi.getTimerCount()).toBe(0);
491	await vi.advanceTimersByTimeAsync(20_000);
492	expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);
493	client.stop();
494	});
495
496	it("observes a web image attached after MJPEG fallback has already begun", async () => {
497	const { client, image, imageReady, events } = recoveryFixture();
498	client.setMjpegImage(null);
499	client.start();
500	expect(image.src).toBe("");
501	await vi.advanceTimersByTimeAsync(1000);
502	client.setMjpegImage(image as unknown as HTMLImageElement);
503	imageReady();
504	expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);
505	client.stop();
506	expect(image.src).toBe("");
507	expect(vi.getTimerCount()).toBe(0);
508	});
509
510	it("stops input and video on an image error without treating it as expired credentials", async () => {
511	const { client, image, imageReady, sockets, events } = recoveryFixture();
512	client.start();
513	await vi.advanceTimersByTimeAsync(0);
514	sockets[0]!.onopen?.();
515	imageReady();
516	image.dispatchEvent(new Event("error"));
517	expect(events.onStatus).toHaveBeenLastCalledWith("error", expect.stringContaining("Reconnect"));
518	expect(events.onInputConnected).toHaveBeenLastCalledWith(false, expect.any(String));
519	expect(events.onUnauthorized).not.toHaveBeenCalled();
520	expect(sockets[0]!.close).toHaveBeenCalledOnce();
521	expect(image.src).toBe("");
522	client.pressButton("home");
523	expect(sockets[0]!.send).toHaveBeenCalledTimes(1);
524	expect(vi.getTimerCount()).toBe(0);
525	});
526
527	it.each([
528	["ios", true],
529	["ios", false],
530	["android", false],
531	] as const)(
532	"bounds missing first frames for %s with preferMjpeg=%s",
533	async (platform, preferMjpeg) => {
534	const { client, events, signals, sockets, image } = recoveryFixture(platform, preferMjpeg);
535	client.start();
536	await vi.advanceTimersByTimeAsync(14_999);
537	expect(events.onStatus.mock.calls.some(([status]) => status === "error")).toBe(false);
538	await vi.advanceTimersByTimeAsync(1);
539	expect(events.onStatus).toHaveBeenLastCalledWith(
540	"error",
541	expect.stringContaining("No video"),
542	);
543	expect(events.onInputConnected).toHaveBeenLastCalledWith(false, expect.any(String));
544	expect(signals.every((signal) => signal.aborted)).toBe(true);
545	expect(sockets[0]!.close).toHaveBeenCalledOnce();
546	expect(image.src).toBe("");
547	expect(vi.getTimerCount()).toBe(0);
548	},
549	);
550
551	it("does not extend the first-frame deadline each time a socket reconnects", async () => {
552	const { client, sockets, events } = recoveryFixture("android");
553	client.start();
554	sockets[0]!.onopen?.();
555	sockets[0]!.onclose?.({ code: 1000, reason: "restart" });
556	await vi.advanceTimersByTimeAsync(14_999);
557	sockets[1]!.onopen?.();
558	await vi.advanceTimersByTimeAsync(1);
559	expect(events.onStatus).toHaveBeenLastCalledWith("error", expect.stringContaining("No video"));
560	expect(sockets[1]!.close).toHaveBeenCalledOnce();
561	expect(vi.getTimerCount()).toBe(0);
562	});
563
564	it("gives a manual restart a fresh deadline and ignores queued image events from its old binding", async () => {
565	const { client, image, events } = recoveryFixture();
566	const listeners: EventListener[] = [];
567	const add = image.addEventListener.bind(image);
568	vi.spyOn(image, "addEventListener").mockImplementation((name, callback, options) => {
569	if (typeof callback === "function") listeners.push(callback);
570	add(name, callback, options);
571	});
572	client.start();
573	await vi.advanceTimersByTimeAsync(10_000);
574	client.stop();
575	client.start();
576	events.onStatus.mockClear();
577	for (const listener of listeners.slice(0, 2)) listener(new Event("error"));
578	await vi.advanceTimersByTimeAsync(10_000);
579	expect(events.onStatus).not.toHaveBeenCalled();
580	const replacement = new MjpegImage();
581	client.setMjpegImage(replacement as unknown as HTMLImageElement);
582	replacement.naturalWidth = 400;
583	replacement.naturalHeight = 800;
584	replacement.dispatchEvent(new Event("load"));
585	expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);
586	client.stop();
587	expect(vi.getTimerCount()).toBe(0);
588	});
589
590	it("falls back from unsupported iOS H.264 without publishing a terminal error", async () => {
591	const { client, videoBodies, Decoder, events, imageReady } = recoveryFixture("ios", false);
592	Decoder.isConfigSupported.mockResolvedValue({ supported: false });
593	client.start();
594	await vi.advanceTimersByTimeAsync(0);
595	videoBodies[0]!.enqueue(new Uint8Array(envelope(1, [1, 0x64, 0, 0x1f])));
596	await vi.advanceTimersByTimeAsync(0);
597	expect(events.onMjpegFallback).toHaveBeenCalledOnce();
598	expect(events.onStatus.mock.calls.some(([status]) => status === "error")).toBe(false);
599	imageReady();
600	expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);
601	client.stop();
602	});
603
604	it("publishes a terminal unsupported-profile error for Android and closes its socket", async () => {
605	const { client, sockets, sps, Decoder, events } = recoveryFixture("android");
606	Decoder.isConfigSupported.mockResolvedValue({ supported: false });
607	client.start();
608	sockets[0]!.onmessage?.({ data: sps });
609	await vi.advanceTimersByTimeAsync(0);
610	expect(events.onStatus).toHaveBeenLastCalledWith(
611	"error",
612	expect.stringContaining("cannot decode"),
613	);
614	expect(sockets[0]!.close).toHaveBeenCalledOnce();
615	expect(events.onMjpegFallback).not.toHaveBeenCalled();
616	expect(vi.getTimerCount()).toBe(0);
617	});
618
619	it("waits for decoded Android output and bounds a rotation that never produces another frame", async () => {
620	const { client, sockets, sps, events, decodedFrame, drawImage } = recoveryFixture("android");
621	client.start();
622	sockets[0]!.onmessage?.({ data: sps });
623	await vi.advanceTimersByTimeAsync(0);
624	expect(events.onStatus.mock.calls.some(([status]) => status === "streaming")).toBe(false);
625	expect(decodedFrame()).toHaveBeenCalledOnce();
626	expect(drawImage).toHaveBeenCalledOnce();
627	expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);
628	sockets[0]!.onmessage?.({ data: JSON.stringify({ type: "video-session" }) });
629	await vi.advanceTimersByTimeAsync(15_000);
630	expect(events.onStatus).toHaveBeenLastCalledWith("error", expect.stringContaining("No video"));
631	expect(vi.getTimerCount()).toBe(0);
632	});
633
634	it("discards decoder support results and output from a previous Android socket", async () => {
635	const { client, sockets, sps, events, Decoder, decoders, decodedFrame, drawImage } =
636	recoveryFixture("android");
637	let resolveSupport!: (support: { supported: boolean }) => void;
638	Decoder.isConfigSupported.mockImplementationOnce(
639	() =>
640	new Promise((resolve) => {
641	resolveSupport = resolve;
642	}),
643	);
644	client.start();
645	sockets[0]!.onmessage?.({ data: sps });
646	client.stop();
647	client.start();
648	resolveSupport({ supported: true });
649	await vi.advanceTimersByTimeAsync(0);
650	expect(decoders).toHaveLength(0);
651	sockets[1]!.onmessage?.({ data: sps });
652	await vi.advanceTimersByTimeAsync(0);
653	client.stop();
654	client.start();
655	events.onStatus.mockClear();
656	expect(decodedFrame()).toHaveBeenCalledOnce();
657	expect(drawImage).not.toHaveBeenCalled();
658	expect(events.onStatus).not.toHaveBeenCalled();
659	client.stop();
660	});
661
662	it("ignores an old AVCC response rejected after a new attempt is connected", async () => {
663	const { client, fetch, events } = recoveryFixture("ios", false);
664	let resolveResponse!: (response: Response) => void;
665	fetch.mockImplementationOnce(() => Promise.resolve(new Response("prime")));
666	fetch.mockImplementationOnce(
667	() =>
668	new Promise((resolve) => {
669	resolveResponse = resolve;
670	}),
671	);
672	client.start();
673	await vi.advanceTimersByTimeAsync(0);
674	client.stop();
675	client.start();
676	await vi.advanceTimersByTimeAsync(0);
677	events.onStatus.mockClear();
678	resolveResponse(new Response(null, { status: 401 }));
679	await vi.advanceTimersByTimeAsync(0);
680	expect(events.onUnauthorized).not.toHaveBeenCalled();
681	expect(events.onStatus).not.toHaveBeenCalled();
682	client.stop();
683	expect(vi.getTimerCount()).toBe(0);
684	});
685
686	it("closes a delayed JPEG seed from a stopped AVCC attempt without painting it", async () => {
687	const { client, videoBodies, drawImage, events } = recoveryFixture("ios", false);
688	let resolveBitmap!: (bitmap: ImageBitmap) => void;
689	vi.stubGlobal(
690	"createImageBitmap",
691	() =>
692	new Promise((resolve) => {
693	resolveBitmap = resolve;
694	}),
695	);
696	client.start();
697	await vi.advanceTimersByTimeAsync(0);
698	videoBodies[0]!.enqueue(new Uint8Array(envelope(4, [1])));
699	await vi.advanceTimersByTimeAsync(0);
700	client.stop();
701	client.start();
702	events.onStatus.mockClear();
703	const close = vi.fn();
704	resolveBitmap({ width: 400, height: 800, close } as unknown as ImageBitmap);
705	await vi.advanceTimersByTimeAsync(0);
706	expect(close).toHaveBeenCalledOnce();
707	expect(drawImage).not.toHaveBeenCalled();
708	expect(events.onStatus).not.toHaveBeenCalled();
709	client.stop();
710	});
711	});
712
713	it("reports a stalled AVCC body after its initial image instead of leaving a frozen streaming state", async () => {
714	const { client, videoBodies, events, signals } = recoveryFixture("ios", false);
715	const close = vi.fn();
716	vi.stubGlobal("createImageBitmap", () => Promise.resolve({ width: 400, height: 800, close }));
717	try {
718	client.start();
719	await vi.advanceTimersByTimeAsync(0);
720	videoBodies[0]!.enqueue(new Uint8Array(envelope(4, [1])));
721	await vi.advanceTimersByTimeAsync(0);
722	expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);
723	await vi.advanceTimersByTimeAsync(15_000);
724	expect(events.onStatus).toHaveBeenLastCalledWith(
725	"error",
726	expect.stringContaining("stopped receiving video"),
727	);
728	expect(signals.every((signal) => signal.aborted)).toBe(true);
729	expect(vi.getTimerCount()).toBe(0);
730	} finally {
731	client.stop();
732	vi.useRealTimers();
733	vi.unstubAllGlobals();
734	}
735	});

packages/client-runtime/src/device/stream.ts+213-60

 @@ -37,8 +37,8 @@ export interface DeviceStreamEvents {
   readonly onUnauthorized: () => void;
   /**
    * H.264 cannot be decoded here (no WebCodecs, or the simulator's profile is
-   * unsupported); the owner should show this MJPEG URL in an `<img>` instead of
+   * unsupported); the owner should show an `<img>` instead of the canvas
-   * the canvas.
+   * and attach it with `setMjpegImage` so the client can observe real frames.
    */
   readonly onMjpegFallback: (url: string) => void;
   /** Whether touches and keys can currently reach the device. */
 @@ -56,6 +56,8 @@ export interface DeviceStreamTarget {
 export type DeviceHardwareButton = "home" | "back" | "recents" | "power" | "appSwitcher";
 const RETRY_DELAY_MS = 1_000;
+const FIRST_FRAME_TIMEOUT_MS = 15_000;
+const MJPEG_FRAME_CHECK_MS = 250;
 const FRAME_DURATION_US = 16_667;
 const SEMU_MAGIC = 0x53454d55;
 const SEMU_HEADER_BYTES = 16;
 @@ -205,6 +207,11 @@ export class AvccDemuxer {
 export interface DeviceStreamClient {
   readonly start: () => void;
   readonly stop: () => void;
+  /**
+   * Own the displayed MJPEG image's source and frame/error observation.
+   * `stop()` detaches it; attach a fresh image for each restart.
+   */
+  readonly setMjpegImage: (image: HTMLImageElement | null) => void;
   /** Normalized 0..1 coordinates in the displayed frame. */
   readonly sendTouch: (phase: "begin" | "move" | "end", x: number, y: number) => void;
   readonly sendKey: (event: KeyboardEvent, phase: "down" | "up") => void;
 @@ -298,19 +305,99 @@ export function createDeviceStreamClient(
   let firstFrame = false;
   let configuring = false;
   let mjpeg = false;
+  let generation = 0;
+  let decoderEpoch = 0;
+  let frameTimer: ReturnType<typeof setTimeout> | null = null;
+  let videoReadTimer: ReturnType<typeof setTimeout> | null = null;
+  let mjpegImage: HTMLImageElement | null = null;
+  let releaseImage: (() => void) | null = null;
   const mjpegUrl = () => httpUrl(`/helper/${device}/stream.mjpeg`);
+  const setStatus = (status: DeviceStreamStatus, detail?: string) => {
+    if (!stopped) events.onStatus(status, detail);
+  };
+  const clearFrameTimer = () => {
+    if (frameTimer !== null) clearTimeout(frameTimer);
+    frameTimer = null;
+  };
+  const fail = (detail: string) => {
+    if (stopped) return;
+    stop();
+    events.onInputConnected(false, detail);
+    events.onStatus("error", detail);
+  };
+  const connecting = (detail?: string) => {
+    firstFrame = false;
+    if (frameTimer === null) {
+      frameTimer = setTimeout(
+        () => fail("No video received from the device. Reconnect to try again."),
+        FIRST_FRAME_TIMEOUT_MS,
+      );
+    }
+    setStatus("connecting", detail);
+  };
+  const frameReceived = () => {
+    if (firstFrame) return;
+    firstFrame = true;
+    clearFrameTimer();
+    setStatus("streaming");
+  };
+  const observeMjpegImage = () => {
+    releaseImage?.();
+    releaseImage = null;
+    const image = mjpegImage;
+    if (!image || stopped || !mjpeg) return;
+    let timer: ReturnType<typeof setTimeout> | null = null;
+    let released = false;
+    const check = () => {
+      if (released || stopped) return;
+      if (timer !== null) clearTimeout(timer);
+      timer = null;
+      if (image.naturalWidth > 0 && image.naturalHeight > 0) frameReceived();
+      else {
+        // Multipart images may not emit load until the response ends. Stop checking after the first frame.
+        timer = setTimeout(check, MJPEG_FRAME_CHECK_MS);
+      }
+    };
+    const error = () => {
+      if (!released) fail("Could not receive the device stream. Reconnect to try again.");
+    };
+    image.addEventListener("load", check);
+    image.addEventListener("error", error);
+    releaseImage = () => {
+      released = true;
+      if (timer !== null) clearTimeout(timer);
+      image.removeEventListener("load", check);
+      image.removeEventListener("error", error);
+      image.removeAttribute("src");
+    };
+    image.src = mjpegUrl();
+    check();
+  };
+  const setMjpegImage = (image: HTMLImageElement | null) => {
+    if (mjpegImage === image) return;
+    releaseImage?.();
+    releaseImage = null;
+    mjpegImage = image;
+    observeMjpegImage();
+  };
   const fallBackToMjpeg = () => {
     if (stopped || mjpeg) return;
     mjpeg = true;
+    controller?.abort();
+    controller = null;
     closeDecoder();
+    connecting();
     events.onMjpegFallback(mjpegUrl());
-    setStatus("streaming");
+    if (!releaseImage) observeMjpegImage();
-  };
-  const setStatus = (status: DeviceStreamStatus, detail?: string) => {
-    if (!stopped) events.onStatus(status, detail);
   };
   const paint = (source: CanvasImageSource, width: number, height: number) => {
 @@ -323,14 +410,14 @@ export function createDeviceStreamClient(
         events.onScreen(screen);
       }
     }
-    canvas.getContext("2d")?.drawImage(source, 0, 0, width, height);
+    const context = canvas.getContext("2d");
-    if (!firstFrame) {
+    if (!context) return fail("Could not display the device stream. Reconnect to try again.");
-      firstFrame = true;
+    context.drawImage(source, 0, 0, width, height);
-      setStatus("streaming");
+    frameReceived();
-    }
   };
   const closeDecoder = () => {
+    decoderEpoch++;
     try {
       videoDecoder?.close();
     } catch {
 @@ -340,40 +427,51 @@ export function createDeviceStreamClient(
     awaitingKeyframe = true;
   };
-  const makeDecoder = () =>
+  const recoverDecoder = () => {
-    new VideoDecoder({
+    if (platform === "ios") fallBackToMjpeg();
+    else {
+      closeDecoder();
+      connecting("Video decoder restarted.");
+      requestKeyframe();
+    }
+  };
+  const makeDecoder = () => {
+    const decoder = new VideoDecoder({
       output: (frame) => {
         try {
-          paint(frame, frame.displayWidth, frame.displayHeight);
+          if (videoDecoder === decoder) paint(frame, frame.displayWidth, frame.displayHeight);
         } finally {
           frame.close();
         }
       },
       error: () => {
-        closeDecoder();
+        if (stopped || videoDecoder !== decoder) return;
-        requestKeyframe();
+        recoverDecoder();
       },
     });
+    return decoder;
+  };
-  /**
+  /** iOS can fall back to MJPEG when the stream's H.264 profile is unsupported. */
-   * Resolves false when this browser cannot decode the stream's profile
+  const configureDecoder = async (
-   * (simulators encode High 5.1, which headless and some hardware decoders
+    config: VideoDecoderConfig,
-   * reject). iOS then falls back to MJPEG; Android has no MJPEG.
+    isCurrent: () => boolean,
-   */
+  ): Promise<boolean> => {
-  const configureDecoder = async (config: VideoDecoderConfig): Promise<boolean> => {
+    const epoch = decoderEpoch;
     const full: VideoDecoderConfig = { ...config, optimizeForLatency: true };
     const support = await VideoDecoder.isConfigSupported(full).catch(() => ({ supported: false }));
-    if (stopped) return false;
+    if (!isCurrent() || epoch !== decoderEpoch) return false;
     if (!support.supported) {
-      setStatus("error", `This browser cannot decode ${config.codec}.`);
+      if (platform === "android") fail(`This browser cannot decode ${config.codec}.`);
       return false;
     }
-    if (!videoDecoder || videoDecoder.state === "closed") videoDecoder = makeDecoder();
     try {
+      if (!videoDecoder || videoDecoder.state === "closed") videoDecoder = makeDecoder();
       videoDecoder.configure(full);
       return true;
     } catch (cause) {
-      setStatus("error", `Video decoder: ${(cause as Error).message}`);
+      if (platform === "android") fail(`Video decoder: ${(cause as Error).message}`);
       return false;
     }
   };
 @@ -385,8 +483,7 @@ export function createDeviceStreamClient(
       awaitingKeyframe = false;
     }
     if (videoDecoder.decodeQueueSize > SOFT_DECODE_QUEUE) {
-      closeDecoder();
+      recoverDecoder();
-      requestKeyframe();
       return;
     }
     try {
 @@ -399,8 +496,7 @@ export function createDeviceStreamClient(
       );
       timestamp += FRAME_DURATION_US;
     } catch {
-      closeDecoder();
+      recoverDecoder();
-      requestKeyframe();
     }
   };
 @@ -423,42 +519,66 @@ export function createDeviceStreamClient(
   const handleUnauthorized = () => {
     stop();
+    events.onInputConnected(false);
     events.onUnauthorized();
   };
   // iOS video: fetch the AVCC body and demux into the decoder.
   const readIosVideo = async () => {
     const demuxer = new AvccDemuxer();
-    controller = new AbortController();
+    const session = generation;
+    const videoController = new AbortController();
+    controller = videoController;
+    const isCurrent = () => !stopped && generation === session && controller === videoController;
+    let retryDetail: string | undefined;
     try {
       const response = await fetch(httpUrl(`/helper/${device}/stream.avcc`), {
-        signal: controller.signal,
+        signal: videoController.signal,
         credentials: access.credentials ? "include" : "same-origin",
       });
+      if (!isCurrent()) return;
       if (response.status === 401 || response.status === 403) return handleUnauthorized();
       if (!response.ok || !response.body) throw new Error(`stream ${response.status}`);
       const reader = response.body.getReader();
       for (;;) {
-        const { done, value } = await reader.read();
+        // An AVCC body can stay open after its helper stops producing frames.
-        if (done || stopped) break;
+        const timer = setTimeout(() => {
+          if (isCurrent()) fail("Device stream stopped receiving video. Reconnect to try again.");
+        }, FIRST_FRAME_TIMEOUT_MS);
+        videoReadTimer = timer;
+        let result: ReadableStreamReadResult<Uint8Array>;
+        try {
+          result = await reader.read();
+        } finally {
+          clearTimeout(timer);
+          if (videoReadTimer === timer) videoReadTimer = null;
+        }
+        const { done, value } = result;
+        if (!isCurrent()) return;
+        if (done) break;
         for (const chunk of demuxer.push(value)) {
           switch (chunk.type) {
             case "seed":
               void createImageBitmap(new Blob([chunk.payload as BlobPart], { type: "image/jpeg" }))
                 .then((bitmap) => {
-                  paint(bitmap, bitmap.width, bitmap.height);
+                  try {
-                  bitmap.close();
+                    if (isCurrent()) paint(bitmap, bitmap.width, bitmap.height);
+                  } finally {
+                    bitmap.close();
+                  }
                 })
                 .catch(() => {});
               break;
             case "description": {
               awaitingKeyframe = true;
-              const configured = await configureDecoder({
+              const configured = await configureDecoder(
-                codec: avcCodecString(chunk.payload),
+                { codec: avcCodecString(chunk.payload), description: chunk.payload },
-                description: chunk.payload,
+                isCurrent,
-              });
+              );
+              if (!isCurrent()) return;
               if (!configured) {
                 await reader.cancel().catch(() => {});
+                if (!isCurrent()) return;
                 fallBackToMjpeg();
                 return;
               }
 @@ -472,10 +592,16 @@ export function createDeviceStreamClient(
         }
       }
     } catch (cause) {
-      if (stopped) return;
+      if (!isCurrent()) return;
-      setStatus("connecting", (cause as Error).message);
+      retryDetail = (cause as Error).message;
+    }
+    if (isCurrent()) {
+      controller = null;
+      videoController.abort();
+      closeDecoder();
+      connecting(retryDetail);
+      scheduleRetry("video", () => void readIosVideo());
     }
-    if (!stopped) scheduleRetry("video", () => void readIosVideo());
   };
   /**
 @@ -483,7 +609,7 @@ export function createDeviceStreamClient(
    * screen capture is running, and the AVCC stream does not reliably start
    * it. Touching the MJPEG endpoint does; one aborted request is enough.
    */
-  const primeIosHelper = async () => {
+  const primeIosHelper = async (session: number) => {
     const controller = new AbortController();
     primeController = controller;
     const timeout = setTimeout(() => controller.abort(), 2_000);
 @@ -492,6 +618,7 @@ export function createDeviceStreamClient(
         signal: controller.signal,
         credentials: access.credentials ? "include" : "same-origin",
       });
+      if (stopped || generation !== session) return;
       if (response.status === 401 || response.status === 403) return handleUnauthorized();
       await response.body?.getReader().read();
     } catch {
 @@ -506,16 +633,19 @@ export function createDeviceStreamClient(
   // iOS input socket; also carries the screen config the helper pushes.
   const connectIosInput = async () => {
     if (stopped) return;
-    await primeIosHelper();
+    const session = generation;
-    if (stopped) return;
+    await primeIosHelper(session);
+    if (stopped || generation !== session) return;
     const ws = new WebSocket(wsUrl(`/helper/ws?device=${device}`));
     ws.binaryType = "arraybuffer";
     socket = ws;
     ws.onopen = () => {
+      if (stopped || socket !== ws) return;
       ws.send(taggedJson(IOS_MSG_HARDWARE_KEYBOARD, { enabled: false }));
       events.onInputConnected(true);
     };
     ws.onmessage = (event) => {
+      if (stopped || socket !== ws) return;
       if (!(event.data instanceof ArrayBuffer)) return;
       const bytes = new Uint8Array(event.data);
       if (bytes.length < 1 || bytes[0] !== IOS_TAG_SCREEN_CONFIG) return;
 @@ -556,14 +686,21 @@ export function createDeviceStreamClient(
     ws.binaryType = "arraybuffer";
     socket = ws;
     ws.onopen = () => {
-      setStatus("connecting");
+      if (stopped || socket !== ws) return;
+      connecting();
       events.onInputConnected(true);
     };
     ws.onmessage = (event) => {
+      if (stopped || socket !== ws) return;
       if (typeof event.data === "string") {
         // The encoder restarts at a new size when the device rotates; the
         // next keyframe carries a fresh SPS, so the decoder is rebuilt from it.
-        if (isVideoSessionMessage(event.data)) closeDecoder();
+        if (isVideoSessionMessage(event.data)) {
+          closeDecoder();
+          configuring = false;
+          connecting();
+          requestKeyframe();
+        }
         return;
       }
       if (!(event.data instanceof ArrayBuffer)) return;
 @@ -576,11 +713,16 @@ export function createDeviceStreamClient(
       if (scanned?.sps && (!videoDecoder || videoDecoder.state !== "configured")) {
         if (configuring) return;
         configuring = true;
-        void configureDecoder({ codec: avcCodecString(scanned.sps) }).then((configured) => {
+        const epoch = decoderEpoch;
-          configuring = false;
+        const isCurrent = () => !stopped && socket === ws;
-          awaitingKeyframe = true;
+        void configureDecoder({ codec: avcCodecString(scanned.sps) }, isCurrent).then(
-          if (configured) requestKeyframe();
+          (configured) => {
-        });
+            if (!isCurrent() || epoch !== decoderEpoch) return;
+            configuring = false;
+            awaitingKeyframe = true;
+            if (configured) requestKeyframe();
+          },
+        );
         return;
       }
       if (!videoDecoder || videoDecoder.state !== "configured") {
 @@ -601,7 +743,8 @@ export function createDeviceStreamClient(
         (event.code === 1006 && access.query.wsTicket)
       )
         return handleUnauthorized();
-      setStatus("connecting", event.reason || undefined);
+      configuring = false;
+      connecting(event.reason || undefined);
       scheduleRetry("input", connectAndroid);
     };
     ws.onerror = () => ws.close();
 @@ -610,8 +753,9 @@ export function createDeviceStreamClient(
   const start = () => {
     if (!stopped) return;
     stopped = false;
-    firstFrame = false;
+    generation++;
-    events.onStatus("connecting");
+    configuring = false;
+    connecting();
     if (platform === "ios") {
       void connectIosInput();
       if (useWebCodecs) void readIosVideo();
 @@ -619,27 +763,35 @@ export function createDeviceStreamClient(
     } else if (useWebCodecs) {
       connectAndroid();
     } else {
-      setStatus("error", "This browser cannot decode the Android stream (WebCodecs unavailable).");
+      fail("This browser cannot decode the Android stream (WebCodecs unavailable).");
     }
   };
   const stop = () => {
     if (stopped) return;
     stopped = true;
+    generation++;
     mjpeg = false;
+    clearFrameTimer();
+    if (videoReadTimer !== null) clearTimeout(videoReadTimer);
+    videoReadTimer = null;
+    releaseImage?.();
+    releaseImage = null;
+    mjpegImage = null;
     for (const timer of retryTimers.values()) clearTimeout(timer);
     retryTimers.clear();
     primeController?.abort();
     primeController = null;
     controller?.abort();
     controller = null;
-    socket?.close();
+    const discarded = socket;
     socket = null;
+    discarded?.close();
     closeDecoder();
   };
   const send = (payload: Uint8Array<ArrayBuffer> | string) => {
-    if (socket?.readyState === WebSocket.OPEN) socket.send(payload);
+    if (!stopped && socket?.readyState === WebSocket.OPEN) socket.send(payload);
   };
   const rawPoint = (x: number, y: number) => {
 @@ -661,6 +813,7 @@ export function createDeviceStreamClient(
   return {
     start,
     stop,
+    setMjpegImage,
     sendTouch: (phase, x, y) => {
       if (platform === "ios") {
         send(taggedJson(IOS_MSG_TOUCH, { type: phase, ...rawPoint(x, y) }));

author	Julius Marminge <julius0216@outlook.com> 2026-09-20 16:44:32-07:00
committer	GitHub <noreply@github.com> 2026-09-20 16:44:32-07:00
log	9f73ca3677faa975be00f0ac55460279671df726
tree	262f08d8c5f36d76214b1a79dfbe8d0302266d62
parent	f4d979a6c6139ab10aa1013335bd94a313a5b56e
signature	Signed by PGP key B5690EEEBB952194

fix(devices): recover simulator streams after failures (#12639)