authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-10-11 13:35:57-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-11 13:35:57-07:00
log5cbe82a87006a7ee472f0f9c9fce5d5f24caa795
tree1430736b0710801f07155572eb12e1c6755cd9fb
parentbed923986fbcbe8279f23b9367efde1596bde1f9
signature Signed by PGP key B5690EEEBB952194

fix(devices): refresh backlogged iOS video and require input admission (#18326)

6 files changed, 209 insertions(+), 37 deletions(-)

apps/server/src/device/DeviceHubProxy.stream.test.ts created+89
......@@ -0,0 +1,89 @@
1import { expect, it } from "@effect/vitest";
2import { NodeWS } from "@effect/platform-node/NodeSocket";
3import * as Context from "effect/Context";
4import * as Effect from "effect/Effect";
5import * as Layer from "effect/Layer";
6import { HttpRouter, HttpServer } from "effect/http";
7import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer";
8import {
9 AuthOrchestrationOperateScope,
10 AuthSessionId,
11 LOCAL_DEVICE_HOST_ID,
12} from "@t3tools/contracts";
13import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts";
14import * as DeviceService from "./DeviceService.ts";
15import * as DeviceHubProxy from "./DeviceHubProxy.ts";
16
17it.effect("preserves binary input and admission frames through the real socket proxy", () =>
18 Effect.gen(function* () {
19 const hub = yield* Effect.acquireRelease(
20 Effect.sync(() => new NodeWS.WebSocketServer({ port: 0 })),
21 (server) =>
22 Effect.promise(
23 () =>
24 new Promise<void>((resolve) => {
25 for (const client of server.clients) client.terminate();
26 server.close(() => resolve());
27 }),
28 ),
29 );
30 if (!hub.address())
31 yield* Effect.promise(() => new Promise<void>((resolve) => hub.once("listening", resolve)));
32 const address = hub.address();
33 if (typeof address !== "object" || !address) throw new Error("Missing hub port");
34 hub.on("connection", (socket) => {
35 socket.send(Uint8Array.of(0x83));
36 socket.on("message", (data, binary) => socket.send(data, { binary }));
37 });
38 const services = yield* Layer.build(
39 HttpRouter.serve(DeviceHubProxy.layer, { disableListenLog: true }).pipe(
40 Layer.provideMerge(NodeHttpServer.layerTest),
41 Layer.provide(
42 Layer.succeed(EnvironmentAuth.EnvironmentAuth, {
43 authenticateWebSocketUpgrade: () =>
44 Effect.succeed({
45 sessionId: AuthSessionId.make("test"),
46 subject: "test",
47 method: "bearer-access-token",
48 scopes: [AuthOrchestrationOperateScope],
49 }),
50 } as unknown as EnvironmentAuth.EnvironmentAuth["Service"]),
51 ),
52 Layer.provide(
53 Layer.succeed(DeviceService.DeviceService, {
54 currentReadiness: () =>
55 Effect.succeed({
56 hostId: LOCAL_DEVICE_HOST_ID,
57 hub: { origin: `http://127.0.0.1:${address.port}` },
58 }),
59 } as DeviceService.DeviceService["Service"]),
60 ),
61 ),
62 );
63 const origin = HttpServer.formatAddress(
64 Context.get(services, HttpServer.HttpServer).address,
65 ).replace(/^http/, "ws");
66 const client = yield* Effect.acquireRelease(
67 Effect.sync(
68 () =>
69 new WebSocket(
70 `${origin}/api/device-hub/vendor/serve-sim/helper/ws?device=test&wsTicket=test`,
71 ),
72 ),
73 (socket) => Effect.sync(() => socket.close()),
74 );
75 client.binaryType = "arraybuffer";
76 const frames: number[][] = [];
77 const received = Promise.withResolvers<void>();
78 client.addEventListener("error", () => received.reject(new Error("Proxy failed")));
79 client.addEventListener("message", ({ data }) => {
80 frames.push(Array.from(new Uint8Array(data as ArrayBuffer)));
81 if (frames.length === 1) {
82 client.send(Uint8Array.of(0x03, 1));
83 client.send(Uint8Array.of(0x03, 2));
84 } else if (frames.length === 3) received.resolve();
85 });
86 yield* Effect.promise(() => received.promise);
87 expect(frames).toEqual([[0x83], [0x03, 1], [0x03, 2]]);
88 }),
89);
apps/server/src/device/deviceToolVersions.json+1-1
......@@ -1,4 +1,4 @@
11{
2 "agent-device": "0.21.24",
2 "agent-device": "0.21.25",
33 "expo-device-hub": "0.15.3"
44}
apps/web/src/components/device/DeviceStreamView.test.tsx+1
......@@ -157,6 +157,7 @@ async function setup(h264 = false) {
157157 const packet = new Uint8Array(1 + json.length);
158158 packet[0] = 0x82;
159159 packet.set(json, 1);
160 sockets[0]?.onmessage?.({ data: Uint8Array.of(0x83).buffer });
160161 sockets[0]?.onmessage?.({ data: packet.buffer });
161162 videoBody?.enqueue(new Uint8Array([0, 0, 0, 5, 1, 1, 0x64, 0, 0x1f]));
162163 },
packages/client-runtime/src/device/duoStream.test.ts+1
......@@ -111,6 +111,7 @@ it("switches to fixed authenticated feeds while retaining one HID socket, routes
111111 client.start();
112112 await socketConstructed;
113113 const ws = Socket.instances[0]!;
114 ws.onmessage?.({ data: Uint8Array.of(0x83).buffer });
114115 const config = (
115116 id: number,
116117 orientation: DeviceScreenSize["orientation"] = "portrait",
packages/client-runtime/src/device/stream.test.ts+87-9
......@@ -64,6 +64,7 @@ describe("native device stream transport", () => {
6464 readyState = 1;
6565 binaryType = "";
6666 onopen: (() => void) | null = null;
67 onmessage: ((event: { data: ArrayBuffer }) => void) | null = null;
6768 onclose: ((event: { code: number; reason: string }) => void) | null = null;
6869 send = vi.fn();
6970 close = vi.fn();
......@@ -111,10 +112,12 @@ describe("native device stream transport", () => {
111112 expect(imageUrl.searchParams.get("hostId")).toBe("ssh-host");
112113 expect(fetch).toHaveBeenCalledTimes(1);
113114 socket.onopen?.();
115 expect(socket.send).not.toHaveBeenCalled();
116 socket.onmessage?.({ data: Uint8Array.of(0x83).buffer });
114117 client.setOrientation("landscape_right");
115118 client.pressButton("home");
116119 client.sendTouch("begin", 0.25, 0.75);
117 const messages = socket.send.mock.calls.slice(1).map(([bytes]) => {
120 const messages = socket.send.mock.calls.map(([bytes]) => {
118121 const packet = bytes as Uint8Array;
119122 return {
120123 tag: packet[0],
......@@ -179,6 +182,7 @@ describe("native device stream transport", () => {
179182 client.start();
180183 const replacement = await reopened;
181184 replacement.onopen?.();
185 if (platform === "ios") replacement.onmessage?.({ data: Uint8Array.of(0x83).buffer });
182186 replacement.send.mockClear();
183187 events.onInputConnected.mockClear();
184188
......@@ -336,6 +340,7 @@ describe("iOS input startup", () => {
336340 packet[0] = 0x82;
337341 packet.set(json, 1);
338342 socket.onmessage?.({ data: packet.buffer });
343 socket.onmessage?.({ data: Uint8Array.of(0x83).buffer });
339344 client.sendTouch("begin", 0.2, 0.7);
340345 const sent = socket.send.mock.calls[0]![0] as Uint8Array;
341346 expect(JSON.parse(new TextDecoder().decode(sent.subarray(1)))).toEqual({
......@@ -519,27 +524,100 @@ describe("shared device stream readiness and recovery", () => {
519524 expect(vi.getTimerCount()).toBe(0);
520525 });
521526
522 it("drops an iOS decode backlog and resumes at a keyframe without switching to MJPEG", async () => {
523 const { client, videoBodies, decoders, events, decodedFrame } = recoveryFixture("ios", false);
527 it("requires admission on each iOS socket and does not mistake config for admission", async () => {
528 const { client, sockets, events, imageReady } = recoveryFixture();
524529 client.start();
525530 await vi.advanceTimersByTimeAsync(0);
531 imageReady();
532 const config = (socket: (typeof sockets)[number], unavailable: boolean) => {
533 const json = new TextEncoder().encode(
534 JSON.stringify({
535 width: 400,
536 height: 800,
537 orientation: "portrait",
538 inputUnavailable: unavailable,
539 supportsHingeAngle: true,
540 }),
541 );
542 socket.onmessage?.({ data: Uint8Array.of(0x82, ...json).buffer });
543 };
544 const controls = () => {
545 client.sendTouch("begin", 0.2, 0.3);
546 client.pressButton("home");
547 client.sendKey({ code: "KeyA" } as KeyboardEvent, "down");
548 client.setOrientation("landscape_left");
549 client.controlDuo({ control: "angle", value: 90 });
550 };
551 const first = sockets[0]!;
552 config(first, false);
553 controls();
554 expect(first.send).not.toHaveBeenCalled();
555 expect(events.onInputConnected).not.toHaveBeenCalledWith(true, undefined);
556 config(first, true);
557 first.onmessage?.({ data: Uint8Array.of(0x83).buffer });
558 controls();
559 expect(first.send).not.toHaveBeenCalled();
560 expect(events.onInputConnected).toHaveBeenLastCalledWith(false);
561 config(first, false);
562 client.pressButton("home");
563 expect(first.send).toHaveBeenCalled();
564 first.onclose?.({ code: 1013, reason: "retry" });
565 await vi.advanceTimersByTimeAsync(1_000);
566 const replacement = sockets[1]!;
567 config(replacement, false);
568 controls();
569 expect(replacement.send).not.toHaveBeenCalled();
570 first.onmessage?.({ data: Uint8Array.of(0x83).buffer });
571 expect(events.onInputConnected).toHaveBeenLastCalledWith(false, undefined);
572 replacement.onmessage?.({ data: Uint8Array.of(0x83).buffer });
573 client.pressButton("home");
574 expect(replacement.send).toHaveBeenCalledOnce();
575 client.stop();
576 expect(vi.getTimerCount()).toBe(0);
577 });
578
579 it("reopens a backlogged iOS feed for a fresh keyframe while keeping input connected", async () => {
580 const { client, videoBodies, decoders, events, decodedFrame, signals, sockets, drawImage } =
581 recoveryFixture("ios", false);
582 client.start();
583 await vi.advanceTimersByTimeAsync(0);
584 const socket = sockets[0]!;
585 socket.onmessage?.({ data: Uint8Array.of(0x83).buffer });
526586 const body = videoBodies[0]!;
527587 body.enqueue(new Uint8Array([...envelope(1, [1, 0x64, 0, 0x1f]), ...envelope(2, [1])]));
528588 await vi.advanceTimersByTimeAsync(0);
529589 decodedFrame();
530590 const decoder = decoders[0]!;
531 expect(decoder.decode).toHaveBeenCalledOnce();
532591 decoder.decodeQueueSize = 9;
533592 body.enqueue(new Uint8Array([...envelope(3, [2]), ...envelope(3, [3])]));
534593 await vi.advanceTimersByTimeAsync(0);
535 expect(decoder.reset).toHaveBeenCalledOnce();
536 expect(decoder.decode).toHaveBeenCalledOnce();
537 body.enqueue(new Uint8Array([...envelope(2, [4]), ...envelope(3, [5])]));
594 expect(signals[1]!.aborted).toBe(true);
595 expect(decoder.state).toBe("closed");
596 expect(events.onStatus).toHaveBeenLastCalledWith(
597 "connecting",
598 expect.stringContaining("fell behind"),
599 );
600 expect(socket.close).not.toHaveBeenCalled();
601 client.sendTouch("begin", 0.2, 0.3);
602 expect(socket.send).toHaveBeenCalledOnce();
603 await vi.advanceTimersByTimeAsync(1_000);
604 expect(videoBodies).toHaveLength(2);
605 // The old feed never produces another IDR. Only a fresh subscription can recover.
606 videoBodies[1]!.enqueue(
607 new Uint8Array([...envelope(1, [1, 0x64, 0, 0x1f]), ...envelope(2, [4])]),
608 );
538609 await vi.advanceTimersByTimeAsync(0);
539 expect(decoder.decode).toHaveBeenCalledTimes(3);
610 expect(decoders[1]!.decode).toHaveBeenCalledOnce();
611 drawImage.mockClear();
612 decodedFrame(0);
613 expect(drawImage).not.toHaveBeenCalled();
614 decodedFrame();
615 expect(drawImage).toHaveBeenCalledOnce();
540616 expect(events.onMjpegFallback).not.toHaveBeenCalled();
541617 expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);
618 expect(events.onInputConnected).toHaveBeenLastCalledWith(true);
542619 client.stop();
620 expect(vi.getTimerCount()).toBe(0);
543621 });
544622
545623 it("does not repaint a delayed JPEG seed over newer decoded video", async () => {
......@@ -614,7 +692,7 @@ describe("shared device stream readiness and recovery", () => {
614692 expect(sockets[0]!.close).toHaveBeenCalledOnce();
615693 expect(image.src).toBe("");
616694 client.pressButton("home");
617 expect(sockets[0]!.send).toHaveBeenCalledTimes(1);
695 expect(sockets[0]!.send).not.toHaveBeenCalled();
618696 expect(vi.getTimerCount()).toBe(0);
619697 });
620698
packages/client-runtime/src/device/stream.ts+30-27
......@@ -128,7 +128,6 @@ const IOS_MSG_TOUCH = 0x03;
128128const IOS_MSG_BUTTON = 0x04;
129129const IOS_MSG_KEY = 0x06;
130130const IOS_MSG_ORIENTATION = 0x07;
131const IOS_MSG_HARDWARE_KEYBOARD = 0x0d;
132131// helper -> browser.
133132const IOS_TAG_SCREEN_CONFIG = 0x82;
134133const IOS_TAG_INPUT_ADMITTED = 0x83;
......@@ -366,8 +365,8 @@ export function createDeviceStreamClient(
366365 const retryTimers = new Map<"video" | "input", ReturnType<typeof setTimeout>>();
367366 let primeController: AbortController | null = null;
368367 let videoDecoder: VideoDecoder | null = null;
369 let decoderConfig: VideoDecoderConfig | null = null;
370368 let decodedVideoGeneration: number | null = null;
369 let iosInputAdmitted = false;
371370 let iosInputUnavailable = false;
372371 let timestamp = 0;
373372 let awaitingKeyframe = true;
......@@ -388,7 +387,13 @@ export function createDeviceStreamClient(
388387 let pendingOrientation: { requestId: number } | null = null;
389388 const duoControl = createDuoControl({
390389 send(request) {
391 if (socket?.readyState !== WebSocket.OPEN || !screen?.supportsHingeAngle) return false;
390 if (
391 !iosInputAdmitted ||
392 iosInputUnavailable ||
393 socket?.readyState !== WebSocket.OPEN ||
394 !screen?.supportsHingeAngle
395 )
396 return false;
392397 if (request.command.control === "physical" && !screen.supportsPhysicalOrientation)
393398 return false;
394399 try {
......@@ -522,7 +527,6 @@ export function createDeviceStreamClient(
522527 // Already closed.
523528 }
524529 videoDecoder = null;
525 decoderConfig = null;
526530 awaitingKeyframe = true;
527531 };
528532
......@@ -580,7 +584,6 @@ export function createDeviceStreamClient(
580584 try {
581585 if (!videoDecoder || videoDecoder.state === "closed") videoDecoder = makeDecoder();
582586 videoDecoder.configure(full);
583 decoderConfig = full;
584587 return true;
585588 } catch (cause) {
586589 if (platform === "android") fail(`Video decoder: ${(cause as Error).message}`);
......@@ -595,22 +598,15 @@ export function createDeviceStreamClient(
595598 awaitingKeyframe = false;
596599 }
597600 if (videoDecoder.decodeQueueSize > SOFT_DECODE_QUEUE) {
598 if (platform !== "ios" || !decoderConfig) {
599 recoverDecoder();
600 return;
601 }
602 // A slow viewer can accumulate healthy H.264 frames. Drop that backlog
603 // and resume at an IDR instead of permanently giving up its 3D view.
604 try {
605 videoDecoder.reset();
606 videoDecoder.configure(decoderConfig);
607 awaitingKeyframe = true;
608 if (!isKey) return;
609 awaitingKeyframe = false;
610 } catch {
601 if (platform !== "ios") {
611602 recoverDecoder();
612603 return;
613604 }
605 // Native IDRs are spaced by frame count, so an idle simulator may take
606 // minutes to produce one. A new AVCC subscription forces a fresh IDR.
607 controller?.abort();
608 closeDecoder();
609 return;
614610 }
615611 try {
616612 videoDecoder.decode(
......@@ -698,6 +694,7 @@ export function createDeviceStreamClient(
698694 if (!isCurrent()) return;
699695 if (done) break;
700696 for (const chunk of demuxer.push(value)) {
697 if (videoController.signal.aborted) break;
701698 switch (chunk.type) {
702699 case "seed":
703700 void createImageBitmap(new Blob([chunk.payload as BlobPart], { type: "image/jpeg" }))
......@@ -740,7 +737,9 @@ export function createDeviceStreamClient(
740737 }
741738 } catch (cause) {
742739 if (!isCurrent()) return;
743 retryDetail ??= (cause as Error).message;
740 retryDetail ??= videoController.signal.aborted
741 ? "Video decoder fell behind. Reconnecting…"
742 : (cause as Error).message;
744743 }
745744 if (isCurrent()) {
746745 controller = null;
......@@ -822,18 +821,16 @@ export function createDeviceStreamClient(
822821 const ws = new WebSocket(wsUrl(`/helper/ws?device=${device}`));
823822 ws.binaryType = "arraybuffer";
824823 socket = ws;
825 ws.onopen = () => {
826 if (stopped || socket !== ws) return;
827 ws.send(taggedJson(IOS_MSG_HARDWARE_KEYBOARD, { enabled: false }));
828 };
824 iosInputAdmitted = false;
825 iosInputUnavailable = false;
829826 ws.onmessage = (event) => {
830827 if (stopped || socket !== ws) return;
831828 if (!(event.data instanceof ArrayBuffer)) return;
832829 const bytes = new Uint8Array(event.data);
833830 if (socket !== ws || stopped || bytes.length < 1) return;
834831 if (bytes[0] === IOS_TAG_INPUT_ADMITTED) {
835 iosInputUnavailable = false;
836 events.onInputConnected(true);
832 iosInputAdmitted = true;
833 events.onInputConnected(!iosInputUnavailable);
837834 return;
838835 }
839836 try {
......@@ -846,7 +843,7 @@ export function createDeviceStreamClient(
846843 if (Option.isSome(config)) {
847844 iosInputUnavailable = config.value.inputUnavailable === true;
848845 events.onInputConnected(
849 !iosInputUnavailable,
846 iosInputAdmitted && !iosInputUnavailable,
850847 iosInputUnavailable ? "Simulator input is unavailable" : undefined,
851848 );
852849 const previous = screen;
......@@ -884,6 +881,7 @@ export function createDeviceStreamClient(
884881 ws.onclose = (event) => {
885882 if (socket !== ws) return;
886883 socket = null;
884 iosInputAdmitted = false;
887885 duoControl.clear();
888886 rotationCursor = null;
889887 if (stopped) return;
......@@ -979,6 +977,7 @@ export function createDeviceStreamClient(
979977 stopped = false;
980978 generation++;
981979 configuring = false;
980 iosInputAdmitted = false;
982981 iosInputUnavailable = false;
983982 connecting();
984983 if (platform === "ios") {
......@@ -1022,7 +1021,11 @@ export function createDeviceStreamClient(
10221021 };
10231022
10241023 const send = (payload: Uint8Array<ArrayBuffer> | string) => {
1025 if (!stopped && !iosInputUnavailable && socket?.readyState === WebSocket.OPEN)
1024 if (
1025 !stopped &&
1026 (platform !== "ios" || (iosInputAdmitted && !iosInputUnavailable)) &&
1027 socket?.readyState === WebSocket.OPEN
1028 )
10261029 socket.send(payload);
10271030 };
10281031