diff --git a/apps/mobile/metro.config.js b/apps/mobile/metro.config.js index 02a6e2c8666cd2238bc018e19d0565f4f2709732..9025dc4f6b283152f3a43abaa0ad383953c8366e 100644 --- a/apps/mobile/metro.config.js +++ b/apps/mobile/metro.config.js @@ -17,6 +17,7 @@ const licenseGeneratorSource = path.join( ); const escapedWorkspaceRoot = workspaceRoot.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); const mobileShikiRoot = path.dirname(require.resolve("shiki/package.json", { paths: [__dirname] })); +const generatedDeviceStreamRoot = path.join(__dirname, ".generated", "device-stream"); const resolveShikiDependencyRoot = (packageName) => { const entryPath = require.resolve(packageName, { paths: [mobileShikiRoot] }); let currentDir = path.dirname(entryPath); @@ -46,6 +47,7 @@ config.resolver = { extraNodeModules: { ...config.resolver?.extraNodeModules, "@t3tools/mobile-third-party-licenses": generatedLicenseModuleRoot, + "@t3tools/mobile-device-stream": generatedDeviceStreamRoot, shiki: mobileShikiRoot, "@shikijs/core": resolveShikiDependencyRoot("@shikijs/core"), "@shikijs/engine-javascript": resolveShikiDependencyRoot("@shikijs/engine-javascript"), @@ -93,7 +95,32 @@ async function generateMobileThirdPartyLicenses() { ]); } -module.exports = generateMobileThirdPartyLicenses().then(() => +async function prepareDeviceStream() { + const { generateDeviceStreamScript } = await import( + pathToFileURL(path.join(__dirname, "scripts", "generate-device-stream.mts")).href + ); + await generateDeviceStreamScript(); + if (process.env.NODE_ENV !== "production") { + let rebuild = Promise.resolve(); + for (const [directory, files] of [ + [path.join(__dirname, "src/features/devices"), ["device-stream.browser.ts"]], + [ + path.join(workspaceRoot, "packages/client-runtime/src/device"), + ["stream.ts", "hubAccess.ts"], + ], + ]) { + // The generated module participates in Metro's normal Fast Refresh. + fs.watch(directory, { persistent: false }, (_event, filename) => { + if (filename && !files.includes(String(filename))) return; + rebuild = rebuild.then(generateDeviceStreamScript).catch((error) => { + console.error("Could not rebuild the device stream:", error); + }); + }); + } + } +} + +module.exports = Promise.all([generateMobileThirdPartyLicenses(), prepareDeviceStream()]).then(() => withUniwindConfig(config, { cssEntryFile: "./global.css", extraThemes, diff --git a/apps/mobile/scripts/generate-device-stream.mts b/apps/mobile/scripts/generate-device-stream.mts new file mode 100644 index 0000000000000000000000000000000000000000..249bbab6ca58b7900ff5901b7bbf61dac6bc61e9 --- /dev/null +++ b/apps/mobile/scripts/generate-device-stream.mts @@ -0,0 +1,39 @@ +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; +import * as NodeURL from "node:url"; +import { build } from "vite-plus"; + +const mobileRoot = NodePath.resolve(NodePath.dirname(NodeURL.fileURLToPath(import.meta.url)), ".."); + +/** Metro embeds the shared browser transport as a small script in the native WebView. */ +export async function generateDeviceStreamScript() { + const result = await build({ + configFile: false, + logLevel: "silent", + build: { + write: false, + target: "es2022", + minify: true, + lib: { + entry: NodePath.join(mobileRoot, "src/features/devices/device-stream.browser.ts"), + name: "T3DeviceStream", + formats: ["iife"], + }, + }, + }); + const bundles = Array.isArray(result) ? result : [result]; + const chunk = bundles + .flatMap((bundle) => ("output" in bundle ? bundle.output : [])) + .find((output) => output.type === "chunk"); + if (!chunk) throw new Error("Device stream build did not emit a script."); + const root = NodePath.join(mobileRoot, ".generated/device-stream"); + await NodeFSP.mkdir(root, { recursive: true }); + for (const [name, contents] of [ + ["index.js", `module.exports = ${JSON.stringify(chunk.code)};\n`], + ["package.json", '{"main":"index.js"}\n'], + ] as const) { + const destination = NodePath.join(root, name); + const previous = await NodeFSP.readFile(destination, "utf8").catch(() => null); + if (previous !== contents) await NodeFSP.writeFile(destination, contents); + } +} diff --git a/apps/mobile/src/Stack.tsx b/apps/mobile/src/Stack.tsx index c750e366c1d3c542952d74d0ef5681fbccae21af..adad6f8e1f18d6bfa3f57f4b25819d1f676e7b4d 100644 --- a/apps/mobile/src/Stack.tsx +++ b/apps/mobile/src/Stack.tsx @@ -30,6 +30,7 @@ import { import { ReviewCommentComposerSheet } from "./features/review/ReviewCommentComposerSheet"; import { ReviewSheet } from "./features/review/ReviewSheet"; import { ThreadTerminalRouteScreen } from "./features/terminal/ThreadTerminalRouteScreen"; +import { DevicePreviewRouteScreen } from "./features/devices/DevicePreviewRouteScreen"; import { GitBranchesSheet } from "./features/threads/git/GitBranchesSheet"; import { GitCommitSheet } from "./features/threads/git/GitCommitSheet"; import { GitConfirmSheet } from "./features/threads/git/GitConfirmSheet"; @@ -449,6 +450,7 @@ const WORKSPACE_OVERLAY_ROUTES = new Set([ "SettingsLegal", "SettingsSheet", "ThreadReviewComment", + "ThreadDevicePreview", "ThreadSettingsSheet", ]); @@ -595,6 +597,15 @@ export const RootStack = createNativeStackNavigator({ linking: `${THREAD_LINKING_PREFIX}/terminal`, options: SOLID_HEADER_OPTIONS, }), + ThreadDevicePreview: createNativeStackScreen({ + screen: DevicePreviewRouteScreen, + linking: `${THREAD_LINKING_PREFIX}/devices`, + options: { + presentation: "fullScreenModal", + headerShown: false, + gestureEnabled: false, + }, + }), ThreadReview: createNativeStackScreen({ screen: ReviewSheet, linking: `${THREAD_LINKING_PREFIX}/review`, diff --git a/apps/mobile/src/components/AppSymbol.tsx b/apps/mobile/src/components/AppSymbol.tsx index 6002dc48cd918fb7fa37d1cece7deef236e6988a..5540d5b5dfb85da42aaf4fef7b69925753e54717 100644 --- a/apps/mobile/src/components/AppSymbol.tsx +++ b/apps/mobile/src/components/AppSymbol.tsx @@ -60,6 +60,7 @@ import IconGitBranch from "@tabler/icons-react-native/IconGitBranch"; import IconGitMerge from "@tabler/icons-react-native/IconGitMerge"; import IconGitPullRequest from "@tabler/icons-react-native/IconGitPullRequest"; import IconHammer from "@tabler/icons-react-native/IconHammer"; +import IconHome from "@tabler/icons-react-native/IconHome"; import IconInfoCircle from "@tabler/icons-react-native/IconInfoCircle"; import IconKeyboard from "@tabler/icons-react-native/IconKeyboard"; import IconKeyboardHide from "@tabler/icons-react-native/IconKeyboardHide"; @@ -161,6 +162,7 @@ const ANDROID_ICON_BY_SF_SYMBOL = { gearshape: IconSettings, globe: IconWorld, hammer: IconHammer, + house: IconHome, "info.circle": IconInfoCircle, internaldrive: IconDatabase, keyboard: IconKeyboard, @@ -194,6 +196,7 @@ const ANDROID_ICON_BY_SF_SYMBOL = { "sidebar.right": IconLayoutSidebarRight, "slider.horizontal.3": IconAdjustmentsHorizontal, "square.and.pencil": IconEdit, + "square.on.square": IconCopy, "square.grid.2x2": IconApps, "square.split.2x1": IconLayoutColumns, star: IconStar, diff --git a/apps/mobile/src/features/devices/DevicePreviewRouteScreen.tsx b/apps/mobile/src/features/devices/DevicePreviewRouteScreen.tsx new file mode 100644 index 0000000000000000000000000000000000000000..2d85a71f14707c159bb72e7bfdb19369051e4bca --- /dev/null +++ b/apps/mobile/src/features/devices/DevicePreviewRouteScreen.tsx @@ -0,0 +1,309 @@ +import { useIsFocused, useNavigation, type StaticScreenProps } from "@react-navigation/native"; +import { createNativeStackNavigator } from "@react-navigation/native-stack"; +import { EnvironmentId, ThreadId } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import { useCallback, useEffect, useMemo, useRef, useState, type RefObject } from "react"; +import { ActivityIndicator, Alert, AppState, Platform, Pressable, View } from "react-native"; +import { useSafeAreaInsets } from "react-native-safe-area-context"; + +import { AppText } from "../../components/AppText"; +import { ScreenHeader, type ScreenHeaderMenuItem } from "../../components/ScreenHeader"; +import { NativeHeaderToolbar } from "../../native/StackHeader"; +import { deviceEnvironment, refreshDeviceHubAccess, useDeviceHubAccess } from "../../state/device"; +import { useEnvironmentQuery } from "../../state/query"; +import { useAtomCommand } from "../../state/use-atom-command"; +import { useAppearancePreferences } from "../settings/appearance/AppearancePreferencesProvider"; +import { DeviceStreamWebView, type DeviceStreamRef } from "./DeviceStreamWebView"; +import { + selectedThreadDevicePreview, + threadDevicePreviews, + type ThreadDevicePreview, +} from "./threadDevicePreviews"; + +const DevicePreviewStack = createNativeStackNavigator<{ DevicePreview: undefined }>(); + +type DevicePreviewRouteScreenProps = StaticScreenProps<{ + readonly environmentId: string; + readonly threadId: string; +}>; + +/** The nested native stack supplies the navigation bar inside the modal. */ +export function DevicePreviewRouteScreen({ route }: DevicePreviewRouteScreenProps) { + const navigation = useNavigation(); + const onClose = useCallback(() => navigation.goBack(), [navigation]); + return ( + + + + {() => ( + + )} + + + + ); +} + +function DevicePreviewScreen({ + environmentId, + threadId, + onClose, +}: { + readonly environmentId: EnvironmentId; + readonly threadId: ThreadId; + readonly onClose: () => void; +}) { + const insets = useSafeAreaInsets(); + const { themeVariables } = useAppearancePreferences(); + const focused = useIsFocused(); + const [foreground, setForeground] = useState(AppState.currentState === "active"); + const [selectedKey, setSelectedKey] = useState(null); + const [inputConnected, setInputConnected] = useState(false); + const [streamAttempt, setStreamAttempt] = useState(0); + const [shuttingDown, setShuttingDown] = useState(false); + const shutdown = useAtomCommand(deviceEnvironment.shutdown, { reportFailure: false }); + const streamRef = useRef(null); + const state = useEnvironmentQuery(deviceEnvironment.state({ environmentId, input: {} })); + const previews = useMemo( + () => threadDevicePreviews(state.data, threadId), + [state.data, threadId], + ); + const preview = selectedThreadDevicePreview(previews, selectedKey); + const onInputConnected = useCallback( + async (connected: boolean) => setInputConnected(connected), + [], + ); + useEffect(() => { + const subscription = AppState.addEventListener("change", (state) => + setForeground(state === "active"), + ); + return () => subscription.remove(); + }, []); + useEffect(() => { + if (focused && state.data !== null && previews.length === 0) onClose(); + }, [focused, state.data, previews.length, onClose]); + + const shutDownDevice = async () => { + if (!preview || shuttingDown) return; + setShuttingDown(true); + try { + const result = await shutdown({ + environmentId, + input: { + hostId: preview.session.hostId, + deviceId: preview.session.deviceId, + platform: preview.session.platform, + }, + }); + if (result._tag === "Failure") { + Alert.alert("Could not shut down device", String(Cause.squash(result.cause))); + } + } finally { + setShuttingDown(false); + } + }; + + const controls: ScreenHeaderMenuItem[] = [ + { + id: "reload", + title: "Reload stream", + icon: "arrow.clockwise", + disabled: !preview || shuttingDown, + onPress: () => { + setInputConnected(false); + setStreamAttempt((attempt) => attempt + 1); + }, + }, + ...(preview?.session.platform === "android" + ? [ + { + id: "back", + title: "Back", + icon: "arrow.left", + disabled: !inputConnected, + onPress: () => streamRef.current?.back(), + }, + ] + : []), + { + id: "app-switcher", + title: "App switcher", + icon: "square.on.square", + disabled: !inputConnected, + onPress: () => streamRef.current?.appSwitcher(), + }, + ...(preview?.session.platform === "ios" + ? [ + { + id: "rotate", + title: "Rotate device", + icon: "arrow.clockwise", + disabled: !inputConnected, + onPress: () => streamRef.current?.rotate(), + }, + ] + : []), + { + id: "shutdown", + title: shuttingDown ? "Shutting down…" : "Shut down device", + icon: "power", + disabled: !preview || shuttingDown, + onPress: () => void shutDownDevice(), + }, + ]; + return ( + + streamRef.current?.home(), + }, + ]} + menus={[ + { + title: "Device options", + icon: "ellipsis", + items: [ + ...(previews.length > 1 + ? [ + { + id: "devices", + title: "Devices", + inline: true, + items: previews.map((device) => ({ + id: device.key, + title: device.name, + subtitle: device.description, + selected: device.key === preview?.key, + onPress: () => setSelectedKey(device.key), + })), + }, + ] + : []), + ...controls, + ], + }, + ]} + /> + {Platform.OS === "ios" ? ( + + + + ) : null} + {preview && focused && foreground ? ( + + ) : ( + + {state.error ? ( + <> + + {state.error} + + + Retry + + + ) : focused && foreground ? ( + + ) : null} + + )} + + ); +} + +function OpenDevicePreview({ + environmentId, + preview, + streamRef, + onInputConnected, +}: { + readonly environmentId: EnvironmentId; + readonly preview: ThreadDevicePreview; + readonly streamRef: RefObject; + readonly onInputConnected: (connected: boolean) => Promise; +}) { + const { session } = preview; + const { themeVariables } = useAppearancePreferences(); + const { access, error, refresh } = useDeviceHubAccess(environmentId, session.hostId); + const onUnauthorized = useCallback( + async () => refreshDeviceHubAccess(environmentId), + [environmentId], + ); + useEffect(() => { + refreshDeviceHubAccess(environmentId); + return () => void onInputConnected(false); + }, [environmentId, onInputConnected]); + return access ? ( + + ) : ( + + {error ? ( + <> + + {error} + + + Retry + + + ) : ( + <> + + Connecting to device... + + )} + + ); +} diff --git a/apps/mobile/src/features/devices/DeviceStreamWebView.tsx b/apps/mobile/src/features/devices/DeviceStreamWebView.tsx new file mode 100644 index 0000000000000000000000000000000000000000..9a5cef83ae3fb4e0099f4a3f915fe51145f61880 --- /dev/null +++ b/apps/mobile/src/features/devices/DeviceStreamWebView.tsx @@ -0,0 +1,109 @@ +import deviceStreamScript from "@t3tools/mobile-device-stream"; +import { useImperativeHandle, useLayoutEffect, useMemo, useRef, useState, type Ref } from "react"; +import { Platform } from "react-native"; +import { WebView } from "react-native-webview"; + +import { + deviceStreamDocument, + deviceStreamMessage, + type DeviceStreamConfiguration, +} from "./device-stream-document"; + +export interface DeviceStreamRef { + home: () => void; + back: () => void; + appSwitcher: () => void; + rotate: () => void; +} + +type NativeStreamBridge = { + readonly ref?: Ref; + readonly onUnauthorized: () => Promise; + readonly onInputConnected: (connected: boolean) => Promise; +}; + +export function DeviceStreamWebView({ + ref, + ...props +}: DeviceStreamConfiguration & NativeStreamBridge) { + const [attempt, setAttempt] = useState(0); + const configuration = JSON.stringify({ + access: props.access, + platform: props.platform, + deviceId: props.deviceId, + colors: props.colors, + }); + return ( + setAttempt((attempt) => attempt + 1)} + /> + ); +} + +function DeviceStreamDocumentView({ + ref, + configuration, + background, + onUnauthorized, + onInputConnected, + onRetry, +}: NativeStreamBridge & { + readonly configuration: string; + readonly background: string; + readonly onRetry: () => void; +}) { + const webView = useRef(null); + const source = useMemo( + () => ({ + html: deviceStreamDocument(configuration, deviceStreamScript), + // Android WebCodecs needs a secure document; streams still use the environment's URLs. + baseUrl: Platform.OS === "android" ? "https://localhost/" : "file:///", + }), + [configuration], + ); + const command = (button: keyof DeviceStreamRef) => { + webView.current?.injectJavaScript( + `window.T3DeviceStream?.command(${JSON.stringify(button)}); true;`, + ); + }; + useImperativeHandle(ref, () => ({ + home: () => command("home"), + back: () => command("back"), + appSwitcher: () => command("appSwitcher"), + rotate: () => command("rotate"), + })); + useLayoutEffect(() => { + const view = webView.current; + return () => view?.injectJavaScript("window.T3DeviceStream?.stop(); true;"); + }, []); + return ( + void onInputConnected(false)} + onShouldStartLoadWithRequest={(request) => + request.url === "about:blank" || request.url === source.baseUrl + } + onMessage={(event) => { + const message = deviceStreamMessage(event.nativeEvent.data); + if (message?.type === "unauthorized") void onUnauthorized(); + else if (message?.type === "input") void onInputConnected(message.connected); + else if (message?.type === "retry") onRetry(); + }} + /> + ); +} diff --git a/apps/mobile/src/features/devices/device-preview-button.tsx b/apps/mobile/src/features/devices/device-preview-button.tsx new file mode 100644 index 0000000000000000000000000000000000000000..01e789946711d1b654f6328799ffbee2a25e7855 --- /dev/null +++ b/apps/mobile/src/features/devices/device-preview-button.tsx @@ -0,0 +1,41 @@ +import { Pressable, View } from "react-native"; + +import { AppText } from "../../components/AppText"; +import { SymbolView } from "../../components/AppSymbol"; + +export function DevicePreviewButton(props: { + readonly count: number; + readonly onPress: () => void; + readonly compact?: boolean; +}) { + const compact = props.compact ?? true; + return ( + + + {!compact ? ( + + {props.count === 1 ? "One device open" : `${props.count} devices open`} + + ) : props.count > 1 ? ( + + {props.count} + + ) : null} + + ); +} diff --git a/apps/mobile/src/features/devices/device-stream-document.test.ts b/apps/mobile/src/features/devices/device-stream-document.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..ffc08667a147237daf6b51045bed3de0c9634fde --- /dev/null +++ b/apps/mobile/src/features/devices/device-stream-document.test.ts @@ -0,0 +1,49 @@ +import * as NodeVM from "node:vm"; +import { describe, expect, it } from "vite-plus/test"; + +import { deviceStreamDocument, deviceStreamMessage } from "./device-stream-document"; + +describe("native device stream document", () => { + it("keeps ticket and device values from terminating the embedded script", () => { + const deviceId = ''; + const configuration = JSON.stringify({ deviceId, ticket: "" }); + const html = deviceStreamDocument( + configuration, + "var T3DeviceStream={start(input){return input}};", + ); + const script = html.match(/";'); + expect(html.match(/<\/script>/g)).toHaveLength(1); + expect(html).toContain('var text="<\\/script>";'); + }); +}); + +describe("native device stream messages", () => { + it("accepts authentication renewal and input connection changes", () => { + expect(deviceStreamMessage('{"type":"unauthorized"}')).toEqual({ type: "unauthorized" }); + expect(deviceStreamMessage('{"type":"input","connected":true}')).toEqual({ + type: "input", + connected: true, + }); + expect(deviceStreamMessage('{"type":"input","connected":false}')).toEqual({ + type: "input", + connected: false, + }); + expect(deviceStreamMessage('{"type":"retry"}')).toEqual({ type: "retry" }); + }); + + it.each([ + "invalid JSON", + "null", + '"input"', + "{}", + '{"type":"input","connected":"yes"}', + '{"type":"unknown"}', + ])("ignores invalid bridge messages: %s", (data) => expect(deviceStreamMessage(data)).toBeNull()); +}); diff --git a/apps/mobile/src/features/devices/device-stream-document.ts b/apps/mobile/src/features/devices/device-stream-document.ts new file mode 100644 index 0000000000000000000000000000000000000000..a919e43218c31f6e36a4157dd152e562d6b39ddc --- /dev/null +++ b/apps/mobile/src/features/devices/device-stream-document.ts @@ -0,0 +1,41 @@ +import type { DeviceHubAccess } from "@t3tools/client-runtime/device/hub-access"; +import type { DevicePlatform } from "@t3tools/contracts"; + +export interface DeviceStreamConfiguration { + readonly access: DeviceHubAccess; + readonly platform: DevicePlatform; + readonly deviceId: string; + readonly colors: { + readonly background: string; + readonly foreground: string; + readonly muted: string; + readonly buttonBackground: string; + }; +} + +export function deviceStreamDocument(configuration: string, script: string) { + // Tickets and device names are data, including any HTML delimiter characters. + const safeConfiguration = configuration.replace(/`; +} + +export function deviceStreamMessage(data: string) { + try { + const message: unknown = JSON.parse(data); + if (typeof message !== "object" || message === null || !("type" in message)) return null; + if (message.type === "unauthorized" || message.type === "retry") { + return { type: message.type } as const; + } + if ( + message.type === "input" && + "connected" in message && + typeof message.connected === "boolean" + ) { + return { type: message.type, connected: message.connected } as const; + } + } catch { + // Ignore messages that are not part of the stream bridge. + } + return null; +} diff --git a/apps/mobile/src/features/devices/device-stream.browser.ts b/apps/mobile/src/features/devices/device-stream.browser.ts new file mode 100644 index 0000000000000000000000000000000000000000..727ea3e8fceb88ba272cb30cf57c4cc79bbfe83d --- /dev/null +++ b/apps/mobile/src/features/devices/device-stream.browser.ts @@ -0,0 +1,223 @@ +import { + createDeviceStreamClient, + type DeviceScreenSize, +} from "@t3tools/client-runtime/device/stream"; + +import type { DeviceStreamConfiguration } from "./device-stream-document"; + +declare global { + interface Window { + ReactNativeWebView: { postMessage: (message: string) => void }; + } +} + +let activeClient: ReturnType | null = null; +let activeImage: HTMLImageElement | null = null; + +export function stop() { + activeClient?.stop(); + activeClient = null; + activeImage?.removeAttribute("src"); + activeImage = null; +} + +export function command(button: "home" | "back" | "appSwitcher" | "rotate") { + if (button === "rotate") activeClient?.rotate(); + else activeClient?.pressButton(button); +} + +/** Bundled into the existing native WebView without React or Expo's web runtime. */ +export function start(configuration: DeviceStreamConfiguration) { + stop(); + // oxlint-disable-next-line unicorn/require-post-message-target-origin -- The native WebView bridge takes one string. + const post = (message: object) => window.ReactNativeWebView.postMessage(JSON.stringify(message)); + const { colors, platform } = configuration; + Object.assign(document.documentElement.style, { height: "100%", overflow: "hidden" }); + Object.assign(document.body.style, { + margin: "0", + height: "100%", + overflow: "hidden", + background: colors.background, + color: colors.foreground, + fontFamily: "system-ui", + }); + const container = document.createElement("div"); + Object.assign(container.style, { + position: "fixed", + inset: "0", + containerType: "size", + display: "flex", + alignItems: "center", + justifyContent: "center", + }); + const frame = document.createElement("div"); + frame.setAttribute("role", "application"); + frame.setAttribute( + "aria-label", + platform === "ios" ? "iOS Simulator screen" : "Android Emulator screen", + ); + frame.tabIndex = 0; + Object.assign(frame.style, { + position: "relative", + touchAction: "none", + userSelect: "none", + webkitUserSelect: "none", + webkitTouchCallout: "none", + outline: "none", + }); + const canvas = document.createElement("canvas"); + const image = document.createElement("img"); + image.alt = ""; + image.draggable = false; + image.style.display = "none"; + const overlay = document.createElement("div"); + overlay.setAttribute("role", "status"); + Object.assign(overlay.style, { + position: "fixed", + inset: "0", + display: "flex", + flexDirection: "column", + alignItems: "center", + justifyContent: "center", + gap: "16px", + padding: "24px", + textAlign: "center", + background: colors.background, + }); + const detail = document.createElement("span"); + const retry = document.createElement("button"); + retry.textContent = "Retry"; + Object.assign(retry.style, { + padding: "12px 24px", + borderRadius: "20px", + border: "0", + background: colors.buttonBackground, + color: colors.foreground, + font: "inherit", + display: "none", + }); + retry.addEventListener("click", () => post({ type: "retry" })); + overlay.append(detail, retry); + const inputStatus = document.createElement("div"); + inputStatus.setAttribute("role", "status"); + inputStatus.textContent = "Reconnecting device controls..."; + Object.assign(inputStatus.style, { + position: "fixed", + bottom: "12px", + left: "0", + right: "0", + textAlign: "center", + pointerEvents: "none", + fontSize: "13px", + color: colors.muted, + background: colors.background, + }); + frame.append(canvas, image); + container.append(frame); + document.body.replaceChildren(container, overlay, inputStatus); + + let pointerId: number | null = null; + let inputConnected = false; + let streaming = false; + const layout = (screen: DeviceScreenSize | null) => { + const landscape = + screen?.orientation === "landscape_left" || screen?.orientation === "landscape_right"; + const aspect = screen + ? landscape + ? Math.max(screen.width, screen.height) / Math.min(screen.width, screen.height) + : Math.min(screen.width, screen.height) / Math.max(screen.width, screen.height) + : 9 / 19.5; + const rotation = + platform === "ios" && screen && screen.width <= screen.height + ? screen.orientation === "landscape_left" + ? 90 + : screen.orientation === "landscape_right" + ? -90 + : screen.orientation === "portrait_upside_down" + ? 180 + : 0 + : 0; + const sideways = Math.abs(rotation) === 90; + frame.style.width = `min(100cqw, ${aspect * 100}cqh)`; + frame.style.height = `min(100cqh, ${100 / aspect}cqw)`; + for (const media of [canvas, image]) { + Object.assign(media.style, { + position: "absolute", + width: sideways ? `${100 / aspect}%` : "100%", + height: sideways ? `${100 * aspect}%` : "100%", + left: "50%", + top: "50%", + transform: `translate(-50%, -50%) rotate(${rotation}deg)`, + pointerEvents: "none", + }); + } + }; + layout(null); + const unauthorized = () => { + if (activeClient === client) post({ type: "unauthorized" }); + }; + const client = createDeviceStreamClient( + { ...configuration, preferMjpeg: platform === "ios" }, + canvas, + { + onStatus: (status, message) => { + streaming = status === "streaming"; + overlay.style.display = streaming ? "none" : "flex"; + inputStatus.style.display = streaming && !inputConnected ? "block" : "none"; + detail.textContent = + status === "error" ? (message ?? "Device stream failed.") : "Connecting to device..."; + retry.style.display = status === "error" ? "block" : "none"; + }, + onScreen: layout, + onMjpegFallback: (url) => { + canvas.style.display = "none"; + image.style.display = "block"; + image.src = url; + }, + onUnauthorized: unauthorized, + onInputConnected: (connected) => { + inputConnected = connected; + inputStatus.style.display = streaming && !connected ? "block" : "none"; + post({ type: "input", connected }); + }, + }, + ); + activeClient = client; + activeImage = image; + image.addEventListener("error", unauthorized); + const touch = (event: PointerEvent, phase: "begin" | "move" | "end") => { + const rect = frame.getBoundingClientRect(); + client.sendTouch( + phase, + Math.max(0, Math.min(1, (event.clientX - rect.left) / rect.width)), + Math.max(0, Math.min(1, (event.clientY - rect.top) / rect.height)), + ); + }; + frame.addEventListener("pointerdown", (event) => { + if (!inputConnected || pointerId !== null) return; + event.preventDefault(); + pointerId = event.pointerId; + frame.setPointerCapture(event.pointerId); + frame.focus(); + touch(event, "begin"); + }); + frame.addEventListener("pointermove", (event) => { + if (pointerId === event.pointerId) touch(event, "move"); + }); + const endTouch = (event: PointerEvent) => { + if (pointerId !== event.pointerId) return; + pointerId = null; + touch(event, "end"); + }; + frame.addEventListener("pointerup", endTouch); + frame.addEventListener("pointercancel", endTouch); + frame.addEventListener("lostpointercapture", endTouch); + frame.addEventListener("keydown", (event) => { + event.preventDefault(); + client.sendKey(event, "down"); + }); + frame.addEventListener("keyup", (event) => client.sendKey(event, "up")); + window.addEventListener("pagehide", stop, { once: true }); + post({ type: "input", connected: false }); + client.start(); +} diff --git a/apps/mobile/src/features/devices/threadDevicePreviews.test.ts b/apps/mobile/src/features/devices/threadDevicePreviews.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..90582dbdc572eb0d2d800c8c71820662674e7ec5 --- /dev/null +++ b/apps/mobile/src/features/devices/threadDevicePreviews.test.ts @@ -0,0 +1,106 @@ +import { ThreadId, type DeviceServiceState } from "@t3tools/contracts"; +import { describe, expect, it } from "vite-plus/test"; + +import { selectedThreadDevicePreview, threadDevicePreviews } from "./threadDevicePreviews"; + +const threadId = ThreadId.make("thread"); +const state: DeviceServiceState = { + hosts: [ + { + id: "local", + kind: "local", + label: "Local", + platforms: [], + hubInstalled: true, + agentDeviceInstalled: true, + }, + { + id: "remote", + kind: "ssh", + label: "Mac mini", + platforms: [], + hubInstalled: true, + agentDeviceInstalled: true, + }, + ], + hostStatus: "ready", + hostStatuses: {}, + devices: [ + { + hostId: "local", + id: "emulator-5554", + platform: "android", + name: "Pixel 9", + version: "Android 16", + booted: true, + physical: false, + }, + { + hostId: "remote", + id: "emulator-5554", + platform: "android", + name: "Pixel 8", + version: "Android 15", + booted: true, + physical: false, + }, + ], + sessions: [ + { + threadId, + hostId: "local", + deviceId: "emulator-5554", + platform: "android", + openedAt: "2026-09-18T00:00:00Z", + }, + { + threadId, + hostId: "remote", + deviceId: "emulator-5554", + platform: "android", + openedAt: "2026-09-18T00:01:00Z", + }, + { + threadId: ThreadId.make("another-thread"), + hostId: "local", + deviceId: "iphone", + platform: "ios", + openedAt: "2026-09-18T00:02:00Z", + }, + ], + onboardingCompleted: true, + agentAccessEnabled: true, + hubBasePath: "/api/device-hub", + revision: 1, +}; + +describe("thread device previews", () => { + it("keeps device names and selection distinct when hosts have the same Android serial", () => { + const previews = threadDevicePreviews(state, threadId); + expect(previews.map((preview) => preview.name)).toEqual(["Pixel 9", "Pixel 8"]); + expect(new Set(previews.map((preview) => preview.key)).size).toBe(2); + expect(selectedThreadDevicePreview(previews, previews[1]!.key)?.description).toBe( + "Android 15 · Mac mini", + ); + }); + + it("selects another open device when the agent closes the selected session", () => { + const previews = threadDevicePreviews(state, threadId); + const selectedKey = previews[1]!.key; + const afterClose = threadDevicePreviews({ ...state, sessions: [state.sessions[0]!] }, threadId); + expect(selectedThreadDevicePreview(afterClose, selectedKey)?.name).toBe("Pixel 9"); + expect(selectedThreadDevicePreview([], selectedKey)).toBeNull(); + }); + + it("can view an open session before device discovery metadata arrives", () => { + const previews = threadDevicePreviews({ ...state, hosts: [], devices: [] }, threadId); + expect(previews[0]?.name).toBe("Android Emulator"); + expect(previews[0]?.session.deviceId).toBe("emulator-5554"); + expect(previews[0]?.description).toBe(""); + }); + + it("offers no devices before state arrives or in a thread without sessions", () => { + expect(threadDevicePreviews(null, threadId)).toEqual([]); + expect(threadDevicePreviews(state, ThreadId.make("empty-thread"))).toEqual([]); + }); +}); diff --git a/apps/mobile/src/features/devices/threadDevicePreviews.ts b/apps/mobile/src/features/devices/threadDevicePreviews.ts new file mode 100644 index 0000000000000000000000000000000000000000..aaf87ca9784e1a94751ada3aa6194c4568526fc2 --- /dev/null +++ b/apps/mobile/src/features/devices/threadDevicePreviews.ts @@ -0,0 +1,28 @@ +import type { DeviceServiceState, ThreadId } from "@t3tools/contracts"; + +/** Host identity is part of the selection because Android serials repeat across hosts. */ +export function threadDevicePreviews(state: DeviceServiceState | null, threadId: ThreadId) { + return (state?.sessions ?? []) + .filter((session) => session.threadId === threadId) + .map((session) => { + const device = state?.devices.find( + (device) => device.hostId === session.hostId && device.id === session.deviceId, + ); + const host = state?.hosts.find((host) => host.id === session.hostId); + return { + key: JSON.stringify([session.hostId, session.deviceId]), + session, + name: device?.name ?? (session.platform === "ios" ? "iOS Simulator" : "Android Emulator"), + description: [device?.version, host?.label].filter(Boolean).join(" · "), + }; + }); +} + +export type ThreadDevicePreview = ReturnType[number]; + +export function selectedThreadDevicePreview( + previews: ReadonlyArray, + selectedKey: string | null, +) { + return previews.find((preview) => preview.key === selectedKey) ?? previews[0] ?? null; +} diff --git a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx index e7b4650e2117ce69d1e5f89ddce9e54d13e6390d..ad0872ea9116d232d95d297f3dd44b74377ff525 100644 --- a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx @@ -13,6 +13,7 @@ import { useKeyboardChatComposerInset, useKeyboardScrollToEnd } from "@legendapp import { resolveProviderSkillsForCwd } from "@t3tools/client-runtime/providerSkills"; import type { LegendListRef } from "@legendapp/list/react-native"; import { HeaderHeightContext } from "@react-navigation/elements"; +import { useNavigation } from "@react-navigation/native"; import type { ApprovalRequestId, EnvironmentId, @@ -72,6 +73,9 @@ import type { DraftComposerAttachment } from "../../lib/composerImages"; import { CHAT_CONTENT_MAX_WIDTH, type LayoutVariant } from "../../lib/layout"; import { IOS_NAV_BAR_HEIGHT } from "../../lib/layoutMetrics"; import { editPendingThreadMessage } from "../../state/edit-pending-thread-message"; +import { deviceEnvironment } from "../../state/device"; +import { useEnvironmentQuery } from "../../state/query"; +import { threadDevicePreviews } from "../devices/threadDevicePreviews"; import type { QueuedThreadMessage } from "../../state/thread-outbox-model"; import { scopedThreadKey } from "../../lib/scopedEntities"; import type { @@ -259,6 +263,21 @@ const USER_INPUT_TOGGLE_TIMING = { }; export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: ThreadDetailScreenProps) { + const navigation = useNavigation(); + const deviceState = useEnvironmentQuery( + deviceEnvironment.state({ environmentId: props.environmentId, input: {} }), + ); + const devicePreviews = useMemo( + () => threadDevicePreviews(deviceState.data, props.selectedThread.id), + [deviceState.data, props.selectedThread.id], + ); + const openDevicePreview = useCallback(() => { + Keyboard.dismiss(); + navigation.navigate("ThreadDevicePreview", { + environmentId: props.environmentId, + threadId: props.selectedThread.id, + }); + }, [navigation, props.environmentId, props.selectedThread.id]); const insets = useSafeAreaInsets(); const isKeyboardVisible = useKeyboardState((state) => state.isVisible); const liveKeyboardHeight = useKeyboardState((state) => state.height); @@ -390,6 +409,7 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread // stable when reconnecting hands off to syncing and then to a running turn. const showFloatingStatus = showWorkingControl || + devicePreviews.length > 0 || props.connectionStateLabel !== "connected" || props.queuedMessages.length > 0 || props.selectedThreadFeed.some( @@ -936,6 +956,11 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread 0 + ? { count: devicePreviews.length, onPress: openDevicePreview } + : null + } showScrollToEnd={showScrollToEndButton} onScrollToEnd={handleScrollToEnd} /> diff --git a/apps/mobile/src/features/threads/floating-working-control.tsx b/apps/mobile/src/features/threads/floating-working-control.tsx index 89cf39e89e39c423a180dcca1dd78debe2d50eb4..d124b138676856c127aa6744489c5f8fcdb8e4fa 100644 --- a/apps/mobile/src/features/threads/floating-working-control.tsx +++ b/apps/mobile/src/features/threads/floating-working-control.tsx @@ -24,6 +24,7 @@ import { AppText as Text } from "../../components/AppText"; import { SymbolView } from "../../components/AppSymbol"; import { ControlPill } from "../../components/ControlPill"; import { NATIVE_LIQUID_GLASS_SUPPORTED } from "../../native/native-glass"; +import { DevicePreviewButton } from "../devices/device-preview-button"; import type { FloatingWorkingStatus } from "./floating-working-status"; import { ShimmeringWorkContent } from "./thread-work-log"; @@ -63,12 +64,18 @@ export const FLOATING_WORKING_CONTROL_COVERAGE = CONTROL_OVERLAY_OFFSET + CONTRO export function FloatingWorkingControl(props: { readonly colorScheme: "light" | "dark"; readonly status: FloatingWorkingStatus | null; + readonly devicePreview: { readonly count: number; readonly onPress: () => void } | null; readonly showScrollToEnd: boolean; readonly onScrollToEnd: () => void; }) { const { width: windowWidth } = useWindowDimensions(); const [overlayWidth, setOverlayWidth] = useState(windowWidth); - const labelWidth = Math.max(0, Math.min(overlayWidth, windowWidth) - CONTROL_HEIGHT - 16); + const deviceControlWidth = props.devicePreview !== null ? CONTROL_HEIGHT : 0; + const hasCapsule = props.status !== null || props.devicePreview !== null; + const labelWidth = Math.max( + 0, + Math.min(overlayWidth, windowWidth) - CONTROL_HEIGHT - 16 - deviceControlWidth, + ); const separationProgress = useSharedValue(props.showScrollToEnd ? 1 : 0); useEffect(() => { @@ -111,25 +118,30 @@ export function FloatingWorkingControl(props: { })); // Zero until the first measurement lands, so the capsule never paints around // a label it has not sized to yet. - const capsuleSizerStyle = useAnimatedStyle(() => ({ width: capsuleWidth.value ?? 0 })); + const capsuleSizerStyle = useAnimatedStyle(() => ({ + width: (capsuleWidth.value ?? 0) + deviceControlWidth, + })); - if (props.status === null && !props.showScrollToEnd) { + if (!hasCapsule && !props.showScrollToEnd) { return null; } // Only the connection label is a button (tap to reconnect); the others // pass touches through to the feed like before. const statusInteractive = props.status?.kind === "connection"; + const capsuleInteractive = statusInteractive || props.devicePreview !== null; // The host stays centered on the capsule, but its measurement constraint // comes from the overlay, independent of the capsule's current width. - const statusContent = - props.status !== null ? ( + const capsuleContent = + props.status === null && props.devicePreview !== null ? ( + + ) : props.status !== null ? ( <> + {props.devicePreview !== null ? ( + <> + + + + + + ) : null} ) : null; @@ -153,7 +177,7 @@ export function FloatingWorkingControl(props: { entering={NATIVE_LIQUID_GLASS_SUPPORTED ? undefined : CONTROL_ENTERING} exiting={NATIVE_LIQUID_GLASS_SUPPORTED ? undefined : CONTROL_EXITING} > - {props.status !== null && NATIVE_LIQUID_GLASS_SUPPORTED ? ( + {hasCapsule && NATIVE_LIQUID_GLASS_SUPPORTED ? ( - {statusContent} + {capsuleContent} - ) : props.status !== null ? ( + ) : hasCapsule ? ( - {statusContent} + {capsuleContent} + connectionAtomRuntime + .atom((get) => { + const prepared = Option.getOrNull( + get(environmentSession.preparedConnectionValueAtom(environmentId)), + ); + return prepared === null + ? Effect.never + : resolveDeviceHubAccess({ prepared, hubBasePath: "/api/device-hub" }); + }) + .pipe(Atom.setIdleTTL(60_000), Atom.withLabel(`mobile-device-hub-access:${environmentId}`)), +); + +export function refreshDeviceHubAccess(environmentId: EnvironmentId) { + appAtomRegistry.refresh(deviceHubAccessAtom(environmentId)); +} + +export function useDeviceHubAccess(environmentId: EnvironmentId, hostId: string) { + const prepared = usePreparedConnection(environmentId); + const query = useEnvironmentQuery(deviceHubAccessAtom(environmentId)); + const access = useMemo( + () => + query.data && query.error === null && Option.isSome(prepared) + ? { ...query.data, query: { ...query.data.query, hostId } } + : null, + [query.data, query.error, prepared, hostId], + ); + return { access, error: query.error, refresh: query.refresh }; +} diff --git a/apps/mobile/src/types/mobile-device-stream.d.ts b/apps/mobile/src/types/mobile-device-stream.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..eddedfac361c7c8fad0b0e8898713efff6889862 --- /dev/null +++ b/apps/mobile/src/types/mobile-device-stream.d.ts @@ -0,0 +1,4 @@ +declare module "@t3tools/mobile-device-stream" { + const script: string; + export default script; +} diff --git a/apps/web/src/components/device/DeviceStreamView.test.tsx b/apps/web/src/components/device/DeviceStreamView.test.tsx index 9c8a1197926a34bd1b086769b5b5d8309a3f89dc..78556078313102b867f860ec63c02a3e919659af 100644 --- a/apps/web/src/components/device/DeviceStreamView.test.tsx +++ b/apps/web/src/components/device/DeviceStreamView.test.tsx @@ -8,7 +8,7 @@ vi.mock("~/state/device", () => ({ refreshDeviceHubAccess: vi.fn(), })); const access = { httpBase: "http://test", wsBase: "ws://test", query: {}, credentials: true }; -vi.mock("./deviceStream", () => ({ +vi.mock("@t3tools/client-runtime/device/stream", () => ({ createDeviceStreamClient: ( _target: unknown, _canvas: unknown, diff --git a/apps/web/src/components/device/DeviceStreamView.tsx b/apps/web/src/components/device/DeviceStreamView.tsx index 994ee1d6d1ec3b123f52661c0de966c2051591bb..60e8414fb060718f6267919b4b788ca7a16c4020 100644 --- a/apps/web/src/components/device/DeviceStreamView.tsx +++ b/apps/web/src/components/device/DeviceStreamView.tsx @@ -11,7 +11,7 @@ import { type DeviceScreenSize, type DeviceStreamClient, type DeviceStreamStatus, -} from "./deviceStream"; +} from "@t3tools/client-runtime/device/stream"; const AX_POLL_INTERVAL_MS = 2_000; diff --git a/apps/web/src/components/device/deviceStream.test.ts b/apps/web/src/components/device/deviceStream.test.ts index 362bd44fedba2c28b63bf7e888fa89ae98964635..13a57d44ffcf32318994079a9228d37158cae4ae 100644 --- a/apps/web/src/components/device/deviceStream.test.ts +++ b/apps/web/src/components/device/deviceStream.test.ts @@ -6,7 +6,7 @@ import { avcCodecString, parseSemuPacket, scanAccessUnit, -} from "./deviceStream"; +} from "@t3tools/client-runtime/device/stream"; const envelope = (tag: number, payload: number[]) => { const length = 1 + payload.length; @@ -43,6 +43,171 @@ describe("AvccDemuxer", () => { }); }); +describe("native device stream transport", () => { + afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); + + function setup(platform: "ios" | "android") { + vi.useFakeTimers(); + vi.stubGlobal("VideoDecoder", vi.fn()); + vi.stubGlobal("EncodedVideoChunk", vi.fn()); + let resolveOpened!: (socket: FakeSocket) => void; + const waitForSocket = () => + new Promise((resolve) => { + resolveOpened = resolve; + }); + const opened = waitForSocket(); + class FakeSocket { + static OPEN = 1; + readyState = 1; + binaryType = ""; + onopen: (() => void) | null = null; + onclose: ((event: { code: number; reason: string }) => void) | null = null; + send = vi.fn(); + close = vi.fn(); + readonly url: string; + constructor(url: string) { + this.url = url; + resolveOpened(this); + } + } + vi.stubGlobal("WebSocket", FakeSocket); + const fetch = vi.fn(() => Promise.resolve(new Response("frame"))); + vi.stubGlobal("fetch", fetch); + const events = { + onStatus: vi.fn(), + onScreen: vi.fn(), + onUnauthorized: vi.fn(), + onMjpegFallback: vi.fn(), + onInputConnected: vi.fn(), + }; + const client = createDeviceStreamClient( + { + platform, + deviceId: "test device", + preferMjpeg: platform === "ios", + access: { + httpBase: "https://environment.test/api/device-hub", + wsBase: "wss://environment.test/api/device-hub", + credentials: false, + query: { wsTicket: "stream-ticket", hostId: "ssh-host" }, + }, + }, + { getContext: () => null } as unknown as HTMLCanvasElement, + events, + ); + return { client, opened, waitForSocket, events, fetch }; + } + + it("uses authenticated iOS MJPEG and forwards controls without a cross-origin video fetch", async () => { + const { client, opened, events, fetch } = setup("ios"); + client.start(); + const socket = await opened; + const imageUrl = new URL(events.onMjpegFallback.mock.calls[0]![0] as string); + expect(imageUrl.pathname).toContain("test%20device/stream.mjpeg"); + expect(imageUrl.searchParams.get("wsTicket")).toBe("stream-ticket"); + expect(imageUrl.searchParams.get("hostId")).toBe("ssh-host"); + expect(fetch).toHaveBeenCalledTimes(1); + socket.onopen?.(); + client.pressButton("home"); + client.sendTouch("begin", 0.25, 0.75); + const messages = socket.send.mock.calls.slice(1).map(([bytes]) => { + const packet = bytes as Uint8Array; + return { + tag: packet[0], + body: JSON.parse(new TextDecoder().decode(packet.subarray(1))) as unknown, + }; + }); + expect(messages).toEqual([ + { tag: 0x04, body: { button: "home" } }, + { tag: 0x03, body: { type: "begin", x: 0.25, y: 0.75 } }, + ]); + client.stop(); + expect(socket.close).toHaveBeenCalledTimes(1); + expect(vi.getTimerCount()).toBe(0); + }); + + it("forwards Android gestures and device controls through the ticketed host socket", async () => { + const { client, opened } = setup("android"); + client.start(); + const socket = await opened; + const url = new URL(socket.url); + expect(url.searchParams.get("device")).toBe("test device"); + expect(url.searchParams.get("wsTicket")).toBe("stream-ticket"); + expect(url.searchParams.get("hostId")).toBe("ssh-host"); + client.sendTouch("begin", 0.2, 0.8); + client.sendTouch("move", 0.2, 0.4); + client.sendTouch("end", 0.2, 0.1); + client.pressButton("home"); + client.pressButton("appSwitcher"); + expect( + socket.send.mock.calls.map(([message]) => JSON.parse(message as string) as unknown), + ).toEqual([ + { type: "touch", action: "down", x: 0.2, y: 0.8 }, + { type: "touch", action: "move", x: 0.2, y: 0.4 }, + { type: "touch", action: "up", x: 0.2, y: 0.1 }, + { type: "home" }, + { type: "recents" }, + ]); + client.stop(); + expect(socket.close).toHaveBeenCalledTimes(1); + }); + + it("renews an expired Android ticket when the HTTP upgrade is rejected instead of retrying it forever", async () => { + const { client, opened, events } = setup("android"); + client.start(); + const socket = await opened; + socket.onclose?.({ code: 1006, reason: "" }); + expect(events.onUnauthorized).toHaveBeenCalledTimes(1); + expect(vi.getTimerCount()).toBe(0); + client.stop(); + }); + + it.each(["ios", "android"] as const)( + "keeps a restarted %s stream connected when the discarded socket closes late", + async (platform) => { + const { client, opened, waitForSocket, events } = setup(platform); + client.start(); + const discarded = await opened; + client.stop(); + const reopened = waitForSocket(); + client.start(); + const replacement = await reopened; + replacement.onopen?.(); + replacement.send.mockClear(); + events.onInputConnected.mockClear(); + + discarded.onclose?.({ code: 1006, reason: "" }); + + expect(events.onUnauthorized).not.toHaveBeenCalled(); + expect(events.onInputConnected).not.toHaveBeenCalled(); + expect(replacement.close).not.toHaveBeenCalled(); + client.pressButton("home"); + expect(replacement.send).toHaveBeenCalledTimes(1); + client.stop(); + expect(replacement.close).toHaveBeenCalledTimes(1); + expect(vi.getTimerCount()).toBe(0); + }, + ); + + it.each(["ios", "android"] as const)( + "does not renew credentials or reconnect a stopped %s stream", + async (platform) => { + const { client, opened, events } = setup(platform); + client.start(); + const socket = await opened; + events.onInputConnected.mockClear(); + client.stop(); + socket.onclose?.({ code: 1006, reason: "" }); + expect(events.onUnauthorized).not.toHaveBeenCalled(); + expect(events.onInputConnected).not.toHaveBeenCalled(); + expect(vi.getTimerCount()).toBe(0); + }, + ); +}); + describe("serve-emu frames", () => { it("strips the SEMU header and reads the keyframe flag and timestamp", () => { const buffer = new ArrayBuffer(16 + 3); diff --git a/apps/web/src/components/device/deviceStream.ts b/apps/web/src/components/device/deviceStream.ts deleted file mode 100644 index d6ae632202ddaafff91f398b7741f639c33fecd2..0000000000000000000000000000000000000000 --- a/apps/web/src/components/device/deviceStream.ts +++ /dev/null @@ -1,697 +0,0 @@ -/** - * Framework-free client for expo-device-hub's per-device streams, reached - * through the T3 proxy. One class handles both platforms because the hub - * vendors two servers with different wire formats: - * - * - iOS (serve-sim): video is an HTTP `stream.avcc` body of length-prefixed - * envelopes (`u32be length, u8 tag, payload`; tag 1 avcC description, - * 2 keyframe, 3 delta, 4 JPEG seed) decoded with WebCodecs; input goes over - * `helper/ws?device=` as `[tag][json]` packets. When WebCodecs is - * unavailable (plain-http remote origins) the MJPEG endpoint is used as an - * `` source instead. - * - Android (serve-emu): one WebSocket at `ws?device=&frame-meta=1` - * carries H.264 access units prefixed with a 16-byte "SEMU" header - * (magic, version, key flag, pts) and accepts JSON gestures upstream. - * - * The decoder only runs while frames arrive and the viewer is attached; a - * hidden panel calls `stop()` so an idle device costs nothing on the GPU. - */ -import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess"; -import { withDeviceHubQuery } from "@t3tools/client-runtime/state/deviceHubAccess"; -import type { DevicePlatform } from "@t3tools/contracts"; - -export type DeviceStreamStatus = "connecting" | "streaming" | "error"; - -export interface DeviceScreenSize { - readonly width: number; - readonly height: number; - readonly orientation: "portrait" | "portrait_upside_down" | "landscape_left" | "landscape_right"; -} - -export interface DeviceStreamEvents { - readonly onStatus: (status: DeviceStreamStatus, detail?: string) => void; - readonly onScreen: (screen: DeviceScreenSize) => void; - /** The proxy rejected the credential; the owner should refresh access and reconnect. */ - readonly onUnauthorized: () => void; - /** - * H.264 cannot be decoded here (no WebCodecs, or the simulator's profile is - * unsupported); the owner should show this MJPEG URL in an `` instead of - * the canvas. - */ - readonly onMjpegFallback: (url: string) => void; - /** Whether touches and keys can currently reach the device. */ - readonly onInputConnected: (connected: boolean, detail?: string) => void; -} - -export interface DeviceStreamTarget { - readonly platform: DevicePlatform; - readonly deviceId: string; - readonly access: DeviceHubAccess; -} - -export type DeviceHardwareButton = "home" | "back" | "recents" | "power" | "appSwitcher"; - -const RETRY_DELAY_MS = 1_000; -const FRAME_DURATION_US = 16_667; -const SEMU_MAGIC = 0x53454d55; -const SEMU_HEADER_BYTES = 16; -const SEMU_FLAG_KEY = 1; -const SOFT_DECODE_QUEUE = 8; - -// serve-sim binary WS message tags (browser -> helper). -const IOS_MSG_TOUCH = 0x03; -const IOS_MSG_BUTTON = 0x04; -const IOS_MSG_KEY = 0x06; -const IOS_MSG_ORIENTATION = 0x07; -const IOS_MSG_HARDWARE_KEYBOARD = 0x0d; -// helper -> browser. -const IOS_TAG_SCREEN_CONFIG = 0x82; - -const encoder = new TextEncoder(); -const decoder = new TextDecoder(); - -const isWebCodecsSupported = (): boolean => - typeof globalThis !== "undefined" && - "VideoDecoder" in globalThis && - "EncodedVideoChunk" in globalThis; - -function taggedJson(tag: number, payload: unknown): Uint8Array { - const json = encoder.encode(JSON.stringify(payload)); - const out = new Uint8Array(1 + json.length); - out[0] = tag; - out.set(json, 1); - return out; -} - -/** Build the WebCodecs `avc1.PPCCLL` string from an avcC record or an SPS NAL. */ -export function avcCodecString(bytes: Uint8Array): string { - if (bytes.length < 4) return "avc1.42E01E"; - const hex = (byte: number) => byte.toString(16).padStart(2, "0"); - return `avc1.${hex(bytes[1]!)}${hex(bytes[2]!)}${hex(bytes[3]!)}`; -} - -/** Split serve-emu's SEMU-framed message into metadata and the Annex-B payload. */ -export function parseSemuPacket(raw: ArrayBuffer): { - readonly data: Uint8Array; - readonly isKey: boolean | null; - readonly timestamp: number | null; -} { - const bytes = new Uint8Array(raw); - if (bytes.byteLength > SEMU_HEADER_BYTES) { - const view = new DataView(raw, 0, SEMU_HEADER_BYTES); - if (view.getUint32(0, false) === SEMU_MAGIC && view.getUint8(4) === 1) { - const pts = view.getBigUint64(8, false); - return { - data: bytes.subarray(SEMU_HEADER_BYTES), - isKey: (view.getUint8(5) & SEMU_FLAG_KEY) !== 0, - timestamp: pts <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(pts) : null, - }; - } - } - return { data: bytes, isKey: null, timestamp: null }; -} - -const isVideoSessionMessage = (text: string) => { - try { - const message = JSON.parse(text) as { type?: unknown }; - return message.type === "video-session"; - } catch { - return false; - } -}; - -/** Walk an Annex-B access unit for its keyframe flag and SPS bytes. */ -export function scanAccessUnit(buf: Uint8Array): { isKey: boolean; sps: Uint8Array | null } { - let isKey = false; - let sps: Uint8Array | null = null; - const len = buf.length; - let i = 0; - while (i + 2 < len) { - if (buf[i] === 0 && buf[i + 1] === 0) { - let codeLen = 0; - if (buf[i + 2] === 1) codeLen = 3; - else if (i + 3 < len && buf[i + 2] === 0 && buf[i + 3] === 1) codeLen = 4; - if (codeLen) { - const nalType = buf[i + codeLen]! & 0x1f; - if (nalType === 7 && !sps) sps = buf.subarray(i + codeLen); - if (nalType === 5) isKey = true; - i += codeLen + 1; - continue; - } - } - i++; - } - return { isKey, sps }; -} - -export type AvccChunk = { - readonly type: "description" | "keyframe" | "delta" | "seed"; - readonly payload: Uint8Array; -}; - -const AVCC_TAGS: Record = { - 1: "description", - 2: "keyframe", - 3: "delta", - 4: "seed", -}; - -/** Turns a fragmented AVCC byte stream into complete envelopes. */ -export class AvccDemuxer { - private buffer = new Uint8Array(64 * 1024); - private length = 0; - - push(bytes: Uint8Array): AvccChunk[] { - if (this.length + bytes.length > this.buffer.length) { - let capacity = this.buffer.length; - while (capacity < this.length + bytes.length) capacity *= 2; - const grown = new Uint8Array(capacity); - grown.set(this.buffer.subarray(0, this.length)); - this.buffer = grown; - } - this.buffer.set(bytes, this.length); - this.length += bytes.length; - - const chunks: AvccChunk[] = []; - let offset = 0; - while (this.length - offset >= 4) { - const view = new DataView(this.buffer.buffer, this.buffer.byteOffset + offset, 4); - const frameLength = view.getUint32(0, false); - if (this.length - offset - 4 < frameLength) break; - if (frameLength >= 1) { - const type = AVCC_TAGS[this.buffer[offset + 4]!]; - if (type) { - chunks.push({ type, payload: this.buffer.slice(offset + 5, offset + 4 + frameLength) }); - } - } - offset += 4 + frameLength; - } - if (offset > 0) { - this.buffer.copyWithin(0, offset, this.length); - this.length -= offset; - } - return chunks; - } - - reset(): void { - this.length = 0; - } -} - -export interface DeviceStreamClient { - readonly start: () => void; - readonly stop: () => void; - /** Normalized 0..1 coordinates in the displayed frame. */ - readonly sendTouch: (phase: "begin" | "move" | "end", x: number, y: number) => void; - readonly sendKey: (event: KeyboardEvent, phase: "down" | "up") => void; - readonly pressButton: (button: DeviceHardwareButton) => void; - readonly rotate: () => void; -} - -const HID_USAGE_BY_CODE: Readonly> = { - Enter: 0x28, - Escape: 0x29, - Backspace: 0x2a, - Tab: 0x2b, - Space: 0x2c, - Minus: 0x2d, - Equal: 0x2e, - BracketLeft: 0x2f, - BracketRight: 0x30, - Backslash: 0x31, - Semicolon: 0x33, - Quote: 0x34, - Backquote: 0x35, - Comma: 0x36, - Period: 0x37, - Slash: 0x38, - Delete: 0x4c, - ArrowRight: 0x4f, - ArrowLeft: 0x50, - ArrowDown: 0x51, - ArrowUp: 0x52, - ControlLeft: 0xe0, - ShiftLeft: 0xe1, - AltLeft: 0xe2, - MetaLeft: 0xe3, - ControlRight: 0xe4, - ShiftRight: 0xe5, - AltRight: 0xe6, - MetaRight: 0xe7, -}; - -function hidUsageForCode(code: string): number | null { - if (/^Key[A-Z]$/.test(code)) return 0x04 + (code.charCodeAt(3) - 65); - if (/^Digit[1-9]$/.test(code)) return 0x1e + (code.charCodeAt(5) - 49); - if (code === "Digit0") return 0x27; - return HID_USAGE_BY_CODE[code] ?? null; -} - -const ANDROID_KEYCODE_BY_KEY: Readonly> = { - ArrowUp: 19, - ArrowDown: 20, - ArrowLeft: 21, - ArrowRight: 22, - Tab: 61, - Enter: 66, - Backspace: 67, - Delete: 112, - Home: 122, - End: 123, - PageUp: 92, - PageDown: 93, -}; - -const IOS_ORIENTATIONS: ReadonlyArray = [ - "portrait", - "landscape_left", - "portrait_upside_down", - "landscape_right", -]; - -export function createDeviceStreamClient( - target: DeviceStreamTarget, - canvas: HTMLCanvasElement, - events: DeviceStreamEvents, -): DeviceStreamClient { - const { access, platform, deviceId } = target; - const vendor = platform === "ios" ? "/vendor/serve-sim" : "/vendor/serve-emu"; - const device = encodeURIComponent(deviceId); - const httpUrl = (path: string) => - withDeviceHubQuery(`${access.httpBase}${vendor}${path}`, access); - const wsUrl = (path: string) => withDeviceHubQuery(`${access.wsBase}${vendor}${path}`, access); - const useWebCodecs = isWebCodecsSupported(); - - let stopped = true; - let socket: WebSocket | null = null; - let controller: AbortController | null = null; - const retryTimers = new Map<"video" | "input", ReturnType>(); - let primeController: AbortController | null = null; - let videoDecoder: VideoDecoder | null = null; - let timestamp = 0; - let awaitingKeyframe = true; - let screen: DeviceScreenSize | null = null; - let firstFrame = false; - let configuring = false; - let mjpeg = false; - - const mjpegUrl = () => httpUrl(`/helper/${device}/stream.mjpeg`); - - const fallBackToMjpeg = () => { - if (stopped || mjpeg) return; - mjpeg = true; - closeDecoder(); - events.onMjpegFallback(mjpegUrl()); - setStatus("streaming"); - }; - - const setStatus = (status: DeviceStreamStatus, detail?: string) => { - if (!stopped) events.onStatus(status, detail); - }; - - const paint = (source: CanvasImageSource, width: number, height: number) => { - if (stopped) return; - if (canvas.width !== width || canvas.height !== height) { - canvas.width = width; - canvas.height = height; - if (platform === "android") { - screen = { width, height, orientation: width > height ? "landscape_left" : "portrait" }; - events.onScreen(screen); - } - } - canvas.getContext("2d")?.drawImage(source, 0, 0, width, height); - if (!firstFrame) { - firstFrame = true; - setStatus("streaming"); - } - }; - - const closeDecoder = () => { - try { - videoDecoder?.close(); - } catch { - // Already closed. - } - videoDecoder = null; - awaitingKeyframe = true; - }; - - const makeDecoder = () => - new VideoDecoder({ - output: (frame) => { - try { - paint(frame, frame.displayWidth, frame.displayHeight); - } finally { - frame.close(); - } - }, - error: () => { - closeDecoder(); - requestKeyframe(); - }, - }); - - /** - * Resolves false when this browser cannot decode the stream's profile - * (simulators encode High 5.1, which headless and some hardware decoders - * reject). iOS then falls back to MJPEG; Android has no MJPEG. - */ - const configureDecoder = async (config: VideoDecoderConfig): Promise => { - const full: VideoDecoderConfig = { ...config, optimizeForLatency: true }; - const support = await VideoDecoder.isConfigSupported(full).catch(() => ({ supported: false })); - if (stopped) return false; - if (!support.supported) { - setStatus("error", `This browser cannot decode ${config.codec}.`); - return false; - } - if (!videoDecoder || videoDecoder.state === "closed") videoDecoder = makeDecoder(); - try { - videoDecoder.configure(full); - return true; - } catch (cause) { - setStatus("error", `Video decoder: ${(cause as Error).message}`); - return false; - } - }; - - const decode = (isKey: boolean, data: Uint8Array, pts?: number | null) => { - if (!videoDecoder || videoDecoder.state !== "configured") return; - if (awaitingKeyframe) { - if (!isKey) return; - awaitingKeyframe = false; - } - if (videoDecoder.decodeQueueSize > SOFT_DECODE_QUEUE) { - closeDecoder(); - requestKeyframe(); - return; - } - try { - videoDecoder.decode( - new EncodedVideoChunk({ - type: isKey ? "key" : "delta", - timestamp: pts ?? timestamp, - data, - }), - ); - timestamp += FRAME_DURATION_US; - } catch { - closeDecoder(); - requestKeyframe(); - } - }; - - const requestKeyframe = () => { - if (platform === "android" && socket?.readyState === WebSocket.OPEN) { - socket.send(JSON.stringify({ type: "reset-video", ack: false })); - } - }; - - const scheduleRetry = (channel: "video" | "input", run: () => void) => { - if (stopped || retryTimers.has(channel)) return; - retryTimers.set( - channel, - setTimeout(() => { - retryTimers.delete(channel); - run(); - }, RETRY_DELAY_MS), - ); - }; - - const handleUnauthorized = () => { - stop(); - events.onUnauthorized(); - }; - - // iOS video: fetch the AVCC body and demux into the decoder. - const readIosVideo = async () => { - const demuxer = new AvccDemuxer(); - controller = new AbortController(); - try { - const response = await fetch(httpUrl(`/helper/${device}/stream.avcc`), { - signal: controller.signal, - credentials: access.credentials ? "include" : "same-origin", - }); - if (response.status === 401 || response.status === 403) return handleUnauthorized(); - if (!response.ok || !response.body) throw new Error(`stream ${response.status}`); - const reader = response.body.getReader(); - for (;;) { - const { done, value } = await reader.read(); - if (done || stopped) break; - for (const chunk of demuxer.push(value)) { - switch (chunk.type) { - case "seed": - void createImageBitmap(new Blob([chunk.payload as BlobPart], { type: "image/jpeg" })) - .then((bitmap) => { - paint(bitmap, bitmap.width, bitmap.height); - bitmap.close(); - }) - .catch(() => {}); - break; - case "description": { - awaitingKeyframe = true; - const configured = await configureDecoder({ - codec: avcCodecString(chunk.payload), - description: chunk.payload, - }); - if (!configured) { - await reader.cancel().catch(() => {}); - fallBackToMjpeg(); - return; - } - break; - } - case "keyframe": - case "delta": - decode(chunk.type === "keyframe", chunk.payload); - break; - } - } - } - } catch (cause) { - if (stopped) return; - setStatus("connecting", (cause as Error).message); - } - if (!stopped) scheduleRetry("video", () => void readIosVideo()); - }; - - /** - * serve-sim's helper only accepts HID and pushes its screen config once - * screen capture is running, and the AVCC stream does not reliably start - * it. Touching the MJPEG endpoint does; one aborted request is enough. - */ - const primeIosHelper = async () => { - const controller = new AbortController(); - primeController = controller; - const timeout = setTimeout(() => controller.abort(), 2_000); - try { - const response = await fetch(httpUrl(`/helper/${device}/stream.mjpeg`), { - signal: controller.signal, - credentials: access.credentials ? "include" : "same-origin", - }); - if (response.status === 401 || response.status === 403) return handleUnauthorized(); - await response.body?.getReader().read(); - } catch { - // A failed prime just means the socket may take a retry to come up. - } finally { - clearTimeout(timeout); - controller.abort(); - if (primeController === controller) primeController = null; - } - }; - - // iOS input socket; also carries the screen config the helper pushes. - const connectIosInput = async () => { - if (stopped) return; - await primeIosHelper(); - if (stopped) return; - const ws = new WebSocket(wsUrl(`/helper/ws?device=${device}`)); - ws.binaryType = "arraybuffer"; - socket = ws; - ws.onopen = () => { - ws.send(taggedJson(IOS_MSG_HARDWARE_KEYBOARD, { enabled: false })); - events.onInputConnected(true); - }; - ws.onmessage = (event) => { - if (!(event.data instanceof ArrayBuffer)) return; - const bytes = new Uint8Array(event.data); - if (bytes.length < 1 || bytes[0] !== IOS_TAG_SCREEN_CONFIG) return; - try { - const config = JSON.parse(decoder.decode(bytes.subarray(1))) as DeviceScreenSize; - if (config.width > 0 && config.height > 0) { - screen = config; - events.onScreen(config); - } - } catch { - // Ignore malformed config frames. - } - }; - ws.onclose = (event) => { - if (socket === ws) socket = null; - if (!stopped) { - events.onInputConnected( - false, - event.reason || (event.code === 1006 ? "input socket refused" : `closed ${event.code}`), - ); - } - if (event.code === 1008 || event.code === 4401) return handleUnauthorized(); - scheduleRetry("input", () => void connectIosInput()); - }; - ws.onerror = () => ws.close(); - }; - - // Android: one socket for video and input. - const connectAndroid = () => { - if (stopped) return; - const ws = new WebSocket(wsUrl(`/ws?device=${device}&frame-meta=1`)); - ws.binaryType = "arraybuffer"; - socket = ws; - ws.onopen = () => { - setStatus("connecting"); - events.onInputConnected(true); - }; - ws.onmessage = (event) => { - if (typeof event.data === "string") { - // The encoder restarts at a new size when the device rotates; the - // next keyframe carries a fresh SPS, so the decoder is rebuilt from it. - if (isVideoSessionMessage(event.data)) closeDecoder(); - return; - } - if (!(event.data instanceof ArrayBuffer)) return; - const packet = parseSemuPacket(event.data); - const needsScan = - packet.isKey === null || - (packet.isKey && (!videoDecoder || videoDecoder.state !== "configured")); - const scanned = needsScan ? scanAccessUnit(packet.data) : null; - const isKey = packet.isKey ?? scanned?.isKey ?? false; - if (scanned?.sps && (!videoDecoder || videoDecoder.state !== "configured")) { - if (configuring) return; - configuring = true; - void configureDecoder({ codec: avcCodecString(scanned.sps) }).then((configured) => { - configuring = false; - awaitingKeyframe = true; - if (configured) requestKeyframe(); - }); - return; - } - if (!videoDecoder || videoDecoder.state !== "configured") { - if (!isKey) requestKeyframe(); - return; - } - decode(isKey, packet.data, packet.timestamp); - }; - ws.onclose = (event) => { - if (socket === ws) socket = null; - closeDecoder(); - if (!stopped) events.onInputConnected(false, event.reason || `closed ${event.code}`); - if (event.code === 1008 || event.code === 4401) return handleUnauthorized(); - if (!stopped) { - setStatus("connecting", event.reason || undefined); - scheduleRetry("input", connectAndroid); - } - }; - ws.onerror = () => ws.close(); - }; - - const start = () => { - if (!stopped) return; - stopped = false; - firstFrame = false; - events.onStatus("connecting"); - if (platform === "ios") { - void connectIosInput(); - if (useWebCodecs) void readIosVideo(); - else fallBackToMjpeg(); - } else if (useWebCodecs) { - connectAndroid(); - } else { - setStatus("error", "This browser cannot decode the Android stream (WebCodecs unavailable)."); - } - }; - - const stop = () => { - if (stopped) return; - stopped = true; - mjpeg = false; - for (const timer of retryTimers.values()) clearTimeout(timer); - retryTimers.clear(); - primeController?.abort(); - primeController = null; - controller?.abort(); - controller = null; - socket?.close(); - socket = null; - closeDecoder(); - }; - - const send = (payload: Uint8Array | string) => { - if (socket?.readyState === WebSocket.OPEN) socket.send(payload); - }; - - const rawPoint = (x: number, y: number) => { - // serve-sim streams the raw framebuffer; rotated devices need input - // remapped into that raw space. - if (platform !== "ios" || !screen || screen.width > screen.height) return { x, y }; - switch (screen.orientation) { - case "landscape_left": - return { x: y, y: 1 - x }; - case "landscape_right": - return { x: 1 - y, y: x }; - case "portrait_upside_down": - return { x: 1 - x, y: 1 - y }; - default: - return { x, y }; - } - }; - - return { - start, - stop, - sendTouch: (phase, x, y) => { - if (platform === "ios") { - send(taggedJson(IOS_MSG_TOUCH, { type: phase, ...rawPoint(x, y) })); - return; - } - const action = phase === "begin" ? "down" : phase === "move" ? "move" : "up"; - send(JSON.stringify({ type: "touch", action, x, y })); - }, - sendKey: (event, phase) => { - if (platform === "ios") { - const usage = hidUsageForCode(event.code); - if (usage !== null) send(taggedJson(IOS_MSG_KEY, { type: phase, usage })); - return; - } - if (phase !== "down") return; - if (event.key === "Escape") return send(JSON.stringify({ type: "back" })); - const keycode = ANDROID_KEYCODE_BY_KEY[event.key]; - if (keycode !== undefined) return send(JSON.stringify({ type: "key", keycode })); - if (event.key.length === 1 && !event.metaKey && !event.ctrlKey) { - send(JSON.stringify({ type: "text", text: event.key })); - } - }, - pressButton: (button) => { - if (platform === "ios") { - const name = - button === "home" - ? "home" - : button === "appSwitcher" - ? "app_switcher" - : button === "power" - ? "lock" - : null; - if (name) send(taggedJson(IOS_MSG_BUTTON, { button: name })); - return; - } - const type = button === "appSwitcher" ? "recents" : button; - if (type === "home" || type === "back" || type === "recents" || type === "power") { - send(JSON.stringify({ type })); - } - }, - rotate: () => { - if (platform !== "ios") return; - const current = screen?.orientation ?? "portrait"; - const next = - IOS_ORIENTATIONS[(IOS_ORIENTATIONS.indexOf(current) + 1) % IOS_ORIENTATIONS.length]!; - send(taggedJson(IOS_MSG_ORIENTATION, { orientation: next })); - }, - }; -} diff --git a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx index 5249fe37401a5b4aaaf899aaa68dbf583659dfb9..3e42c39715ee7704b5d1d1fcd7eab019b9acd515 100644 --- a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx +++ b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx @@ -35,7 +35,7 @@ import { useRightPanelStore } from "~/rightPanelStore"; import { useDeviceState } from "~/state/device"; import { DeviceStreamView } from "../device/DeviceStreamView"; -import type { DeviceScreenSize } from "../device/deviceStream"; +import type { DeviceScreenSize } from "@t3tools/client-runtime/device/stream"; import { previewBridge } from "./previewBridge"; import { clampPreviewMiniPlayerPosition, diff --git a/apps/web/src/components/preview/previewMiniPlayerLayout.ts b/apps/web/src/components/preview/previewMiniPlayerLayout.ts index 13ba26f06fcae7dc376f25b0843d4e703085e0cc..faa529ae91c2b14d1f7423eb140e99d9739c3f9e 100644 --- a/apps/web/src/components/preview/previewMiniPlayerLayout.ts +++ b/apps/web/src/components/preview/previewMiniPlayerLayout.ts @@ -7,7 +7,7 @@ import { } from "~/browser/browserViewportLayout"; import type { PreviewMiniPlayerPosition, PreviewMiniPlayerSize } from "~/previewMiniPlayerStore"; -import type { DeviceScreenSize } from "../device/deviceStream"; +import type { DeviceScreenSize } from "@t3tools/client-runtime/device/stream"; export const PREVIEW_MINI_PLAYER_EDGE_GAP = 12; export const PREVIEW_MINI_PLAYER_CORNER_RADIUS = 12; diff --git a/docs/user/devices.md b/docs/user/devices.md index f40d119044595fff7f95fd4761569ca4fba346e3..b388688bae29883738ce94c64ff311dad9f75c61 100644 --- a/docs/user/devices.md +++ b/docs/user/devices.md @@ -55,8 +55,13 @@ back from the device after a change. When an agent opens a device, it floats over the chat in web and desktop clients connected to the thread, the same way an agent-driven browser does. Turn off **Auto-show floating preview** in **Settings → Integrations → Browser** to open a -right-panel tab instead. Mobile clients show device activity in the thread -timeline. Agents drive the device through the `agent-device` command line. T3 +right-panel tab instead. In the mobile app, open the agent's thread and tap the +device button above the composer to watch the live screen and control it. +If the thread has several devices open, choose one in the viewer. Closing the +viewer stops streaming and leaves the device available to the agent. Device +activity also appears in the thread timeline. + +Agents drive the device through the `agent-device` command line. T3 Code installs and starts it only after **Agent device access** is enabled. iOS taps build a small test runner on first use, which takes a couple of minutes once per server. Restart an existing agent session after granting access so it diff --git a/knip.jsonc b/knip.jsonc index a06e0885a2f8c18d9f177cf8c9b76171450fafa9..5e87b4d5bc55f14318a89bcdd55e1a8343e446d4 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -58,8 +58,13 @@ "ignoreIssues": { "src/components/ui/*.tsx": ["exports", "nsExports", "duplicates"] }, }, "apps/mobile": { - // Expo loads local config plugins by string; Metro handles platform variants. - "entry": ["index.ts!", "plugins/*.cjs"], + // Expo loads plugins by string; Metro embeds the browser entry in the native WebView. + "entry": [ + "index.ts!", + "plugins/*.cjs", + "scripts/generate-device-stream.mts", + "src/features/devices/device-stream.browser.ts!", + ], // Fonts are configured as asset paths; Expo autolinks the native packages. "ignoreDependencies": [ "@expo-google-fonts/dm-sans", diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index 65e13c251edcead74096b308920ec00624d69c4c..63dfc4ab344caff33e916d833b30a6feaec03e59 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -3,6 +3,14 @@ "private": true, "type": "module", "exports": { + "./device/stream": { + "types": "./src/device/stream.ts", + "default": "./src/device/stream.ts" + }, + "./device/hub-access": { + "types": "./src/device/hubAccess.ts", + "default": "./src/device/hubAccess.ts" + }, "./worktree-setup": { "types": "./src/worktreeSetup.ts", "default": "./src/worktreeSetup.ts" diff --git a/packages/client-runtime/src/device/hubAccess.ts b/packages/client-runtime/src/device/hubAccess.ts new file mode 100644 index 0000000000000000000000000000000000000000..9c010a2583f78c08527ddfd181977b66c4e56714 --- /dev/null +++ b/packages/client-runtime/src/device/hubAccess.ts @@ -0,0 +1,18 @@ +/** Credentials for media requests that cannot set bearer or DPoP headers. */ +export interface DeviceHubAccess { + /** Absolute environment URL ending in `/api/device-hub`. */ + readonly httpBase: string; + /** Same base with the `ws(s)` scheme. */ + readonly wsBase: string; + /** Empty for cookie sessions; includes a short-lived ticket for bearer and DPoP sessions. */ + readonly query: Readonly>; + /** Whether requests must include session cookies. */ + readonly credentials: boolean; +} + +export const withDeviceHubQuery = (url: string, access: DeviceHubAccess): string => { + const entries = Object.entries(access.query); + if (entries.length === 0) return url; + const separator = url.includes("?") ? "&" : "?"; + return `${url}${separator}${new URLSearchParams(entries).toString()}`; +}; diff --git a/packages/client-runtime/src/device/stream.ts b/packages/client-runtime/src/device/stream.ts new file mode 100644 index 0000000000000000000000000000000000000000..02aa46c67e1da8435ed2050654fee59519b059a5 --- /dev/null +++ b/packages/client-runtime/src/device/stream.ts @@ -0,0 +1,712 @@ +// @effect-diagnostics globalFetch:off globalTimers:off - This browser and WebView transport runs without an Effect runtime. +/* oxlint-disable unicorn/prefer-add-event-listener -- Each client owns its sockets and their handlers. */ + +/** + * Framework-free client for expo-device-hub's per-device streams, reached + * through the T3 proxy. One class handles both platforms because the hub + * vendors two servers with different wire formats: + * + * - iOS (serve-sim): video is an HTTP `stream.avcc` body of length-prefixed + * envelopes (`u32be length, u8 tag, payload`; tag 1 avcC description, + * 2 keyframe, 3 delta, 4 JPEG seed) decoded with WebCodecs; input goes over + * `helper/ws?device=` as `[tag][json]` packets. When WebCodecs is + * unavailable (plain-http remote origins) the MJPEG endpoint is used as an + * `` source instead. + * - Android (serve-emu): one WebSocket at `ws?device=&frame-meta=1` + * carries H.264 access units prefixed with a 16-byte "SEMU" header + * (magic, version, key flag, pts) and accepts JSON gestures upstream. + * + * The decoder only runs while frames arrive and the viewer is attached; a + * hidden panel calls `stop()` so an idle device costs nothing on the GPU. + */ +import { type DeviceHubAccess, withDeviceHubQuery } from "./hubAccess.ts"; +import type { DevicePlatform } from "@t3tools/contracts"; + +export type DeviceStreamStatus = "connecting" | "streaming" | "error"; + +export interface DeviceScreenSize { + readonly width: number; + readonly height: number; + readonly orientation: "portrait" | "portrait_upside_down" | "landscape_left" | "landscape_right"; +} + +export interface DeviceStreamEvents { + readonly onStatus: (status: DeviceStreamStatus, detail?: string) => void; + readonly onScreen: (screen: DeviceScreenSize) => void; + /** The proxy rejected the credential; the owner should refresh access and reconnect. */ + readonly onUnauthorized: () => void; + /** + * H.264 cannot be decoded here (no WebCodecs, or the simulator's profile is + * unsupported); the owner should show this MJPEG URL in an `` instead of + * the canvas. + */ + readonly onMjpegFallback: (url: string) => void; + /** Whether touches and keys can currently reach the device. */ + readonly onInputConnected: (connected: boolean, detail?: string) => void; +} + +export interface DeviceStreamTarget { + readonly platform: DevicePlatform; + readonly deviceId: string; + readonly access: DeviceHubAccess; + /** Native iOS WebViews can use MJPEG without cross-origin fetch or secure-context support. */ + readonly preferMjpeg?: boolean; +} + +export type DeviceHardwareButton = "home" | "back" | "recents" | "power" | "appSwitcher"; + +const RETRY_DELAY_MS = 1_000; +const FRAME_DURATION_US = 16_667; +const SEMU_MAGIC = 0x53454d55; +const SEMU_HEADER_BYTES = 16; +const SEMU_FLAG_KEY = 1; +const SOFT_DECODE_QUEUE = 8; + +// serve-sim binary WS message tags (browser -> helper). +const IOS_MSG_TOUCH = 0x03; +const IOS_MSG_BUTTON = 0x04; +const IOS_MSG_KEY = 0x06; +const IOS_MSG_ORIENTATION = 0x07; +const IOS_MSG_HARDWARE_KEYBOARD = 0x0d; +// helper -> browser. +const IOS_TAG_SCREEN_CONFIG = 0x82; + +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +const isWebCodecsSupported = (): boolean => + typeof globalThis !== "undefined" && + "VideoDecoder" in globalThis && + "EncodedVideoChunk" in globalThis; + +function taggedJson(tag: number, payload: unknown): Uint8Array { + const json = encoder.encode(JSON.stringify(payload)); + const out = new Uint8Array(1 + json.length); + out[0] = tag; + out.set(json, 1); + return out; +} + +/** Build the WebCodecs `avc1.PPCCLL` string from an avcC record or an SPS NAL. */ +export function avcCodecString(bytes: Uint8Array): string { + if (bytes.length < 4) return "avc1.42E01E"; + const hex = (byte: number) => byte.toString(16).padStart(2, "0"); + return `avc1.${hex(bytes[1]!)}${hex(bytes[2]!)}${hex(bytes[3]!)}`; +} + +/** Split serve-emu's SEMU-framed message into metadata and the Annex-B payload. */ +export function parseSemuPacket(raw: ArrayBuffer): { + readonly data: Uint8Array; + readonly isKey: boolean | null; + readonly timestamp: number | null; +} { + const bytes = new Uint8Array(raw); + if (bytes.byteLength > SEMU_HEADER_BYTES) { + const view = new DataView(raw, 0, SEMU_HEADER_BYTES); + if (view.getUint32(0, false) === SEMU_MAGIC && view.getUint8(4) === 1) { + const pts = view.getBigUint64(8, false); + return { + data: bytes.subarray(SEMU_HEADER_BYTES), + isKey: (view.getUint8(5) & SEMU_FLAG_KEY) !== 0, + timestamp: pts <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(pts) : null, + }; + } + } + return { data: bytes, isKey: null, timestamp: null }; +} + +const isVideoSessionMessage = (text: string) => { + try { + const message = JSON.parse(text) as { type?: unknown }; + return message.type === "video-session"; + } catch { + return false; + } +}; + +/** Walk an Annex-B access unit for its keyframe flag and SPS bytes. */ +export function scanAccessUnit(buf: Uint8Array): { isKey: boolean; sps: Uint8Array | null } { + let isKey = false; + let sps: Uint8Array | null = null; + const len = buf.length; + let i = 0; + while (i + 2 < len) { + if (buf[i] === 0 && buf[i + 1] === 0) { + let codeLen = 0; + if (buf[i + 2] === 1) codeLen = 3; + else if (i + 3 < len && buf[i + 2] === 0 && buf[i + 3] === 1) codeLen = 4; + if (codeLen) { + const nalType = buf[i + codeLen]! & 0x1f; + if (nalType === 7 && !sps) sps = buf.subarray(i + codeLen); + if (nalType === 5) isKey = true; + i += codeLen + 1; + continue; + } + } + i++; + } + return { isKey, sps }; +} + +export type AvccChunk = { + readonly type: "description" | "keyframe" | "delta" | "seed"; + readonly payload: Uint8Array; +}; + +const AVCC_TAGS: Record = { + 1: "description", + 2: "keyframe", + 3: "delta", + 4: "seed", +}; + +/** Turns a fragmented AVCC byte stream into complete envelopes. */ +export class AvccDemuxer { + private buffer = new Uint8Array(64 * 1024); + private length = 0; + + push(bytes: Uint8Array): AvccChunk[] { + if (this.length + bytes.length > this.buffer.length) { + let capacity = this.buffer.length; + while (capacity < this.length + bytes.length) capacity *= 2; + const grown = new Uint8Array(capacity); + grown.set(this.buffer.subarray(0, this.length)); + this.buffer = grown; + } + this.buffer.set(bytes, this.length); + this.length += bytes.length; + + const chunks: AvccChunk[] = []; + let offset = 0; + while (this.length - offset >= 4) { + const view = new DataView(this.buffer.buffer, this.buffer.byteOffset + offset, 4); + const frameLength = view.getUint32(0, false); + if (this.length - offset - 4 < frameLength) break; + if (frameLength >= 1) { + const type = AVCC_TAGS[this.buffer[offset + 4]!]; + if (type) { + chunks.push({ type, payload: this.buffer.slice(offset + 5, offset + 4 + frameLength) }); + } + } + offset += 4 + frameLength; + } + if (offset > 0) { + this.buffer.copyWithin(0, offset, this.length); + this.length -= offset; + } + return chunks; + } + + reset(): void { + this.length = 0; + } +} + +export interface DeviceStreamClient { + readonly start: () => void; + readonly stop: () => void; + /** Normalized 0..1 coordinates in the displayed frame. */ + readonly sendTouch: (phase: "begin" | "move" | "end", x: number, y: number) => void; + readonly sendKey: (event: KeyboardEvent, phase: "down" | "up") => void; + readonly pressButton: (button: DeviceHardwareButton) => void; + readonly rotate: () => void; +} + +const HID_USAGE_BY_CODE: Readonly> = { + Enter: 0x28, + Escape: 0x29, + Backspace: 0x2a, + Tab: 0x2b, + Space: 0x2c, + Minus: 0x2d, + Equal: 0x2e, + BracketLeft: 0x2f, + BracketRight: 0x30, + Backslash: 0x31, + Semicolon: 0x33, + Quote: 0x34, + Backquote: 0x35, + Comma: 0x36, + Period: 0x37, + Slash: 0x38, + Delete: 0x4c, + ArrowRight: 0x4f, + ArrowLeft: 0x50, + ArrowDown: 0x51, + ArrowUp: 0x52, + ControlLeft: 0xe0, + ShiftLeft: 0xe1, + AltLeft: 0xe2, + MetaLeft: 0xe3, + ControlRight: 0xe4, + ShiftRight: 0xe5, + AltRight: 0xe6, + MetaRight: 0xe7, +}; + +function hidUsageForCode(code: string): number | null { + if (/^Key[A-Z]$/.test(code)) return 0x04 + (code.charCodeAt(3) - 65); + if (/^Digit[1-9]$/.test(code)) return 0x1e + (code.charCodeAt(5) - 49); + if (code === "Digit0") return 0x27; + return HID_USAGE_BY_CODE[code] ?? null; +} + +const ANDROID_KEYCODE_BY_KEY: Readonly> = { + ArrowUp: 19, + ArrowDown: 20, + ArrowLeft: 21, + ArrowRight: 22, + Tab: 61, + Enter: 66, + Backspace: 67, + Delete: 112, + Home: 122, + End: 123, + PageUp: 92, + PageDown: 93, +}; + +const IOS_ORIENTATIONS: ReadonlyArray = [ + "portrait", + "landscape_left", + "portrait_upside_down", + "landscape_right", +]; + +export function createDeviceStreamClient( + target: DeviceStreamTarget, + canvas: HTMLCanvasElement, + events: DeviceStreamEvents, +): DeviceStreamClient { + const { access, platform, deviceId } = target; + const vendor = platform === "ios" ? "/vendor/serve-sim" : "/vendor/serve-emu"; + const device = encodeURIComponent(deviceId); + const httpUrl = (path: string) => + withDeviceHubQuery(`${access.httpBase}${vendor}${path}`, access); + const wsUrl = (path: string) => withDeviceHubQuery(`${access.wsBase}${vendor}${path}`, access); + const useWebCodecs = isWebCodecsSupported() && !(platform === "ios" && target.preferMjpeg); + + let stopped = true; + let socket: WebSocket | null = null; + let controller: AbortController | null = null; + const retryTimers = new Map<"video" | "input", ReturnType>(); + let primeController: AbortController | null = null; + let videoDecoder: VideoDecoder | null = null; + let timestamp = 0; + let awaitingKeyframe = true; + let screen: DeviceScreenSize | null = null; + let firstFrame = false; + let configuring = false; + let mjpeg = false; + + const mjpegUrl = () => httpUrl(`/helper/${device}/stream.mjpeg`); + + const fallBackToMjpeg = () => { + if (stopped || mjpeg) return; + mjpeg = true; + closeDecoder(); + events.onMjpegFallback(mjpegUrl()); + setStatus("streaming"); + }; + + const setStatus = (status: DeviceStreamStatus, detail?: string) => { + if (!stopped) events.onStatus(status, detail); + }; + + const paint = (source: CanvasImageSource, width: number, height: number) => { + if (stopped) return; + if (canvas.width !== width || canvas.height !== height) { + canvas.width = width; + canvas.height = height; + if (platform === "android") { + screen = { width, height, orientation: width > height ? "landscape_left" : "portrait" }; + events.onScreen(screen); + } + } + canvas.getContext("2d")?.drawImage(source, 0, 0, width, height); + if (!firstFrame) { + firstFrame = true; + setStatus("streaming"); + } + }; + + const closeDecoder = () => { + try { + videoDecoder?.close(); + } catch { + // Already closed. + } + videoDecoder = null; + awaitingKeyframe = true; + }; + + const makeDecoder = () => + new VideoDecoder({ + output: (frame) => { + try { + paint(frame, frame.displayWidth, frame.displayHeight); + } finally { + frame.close(); + } + }, + error: () => { + closeDecoder(); + requestKeyframe(); + }, + }); + + /** + * Resolves false when this browser cannot decode the stream's profile + * (simulators encode High 5.1, which headless and some hardware decoders + * reject). iOS then falls back to MJPEG; Android has no MJPEG. + */ + const configureDecoder = async (config: VideoDecoderConfig): Promise => { + const full: VideoDecoderConfig = { ...config, optimizeForLatency: true }; + const support = await VideoDecoder.isConfigSupported(full).catch(() => ({ supported: false })); + if (stopped) return false; + if (!support.supported) { + setStatus("error", `This browser cannot decode ${config.codec}.`); + return false; + } + if (!videoDecoder || videoDecoder.state === "closed") videoDecoder = makeDecoder(); + try { + videoDecoder.configure(full); + return true; + } catch (cause) { + setStatus("error", `Video decoder: ${(cause as Error).message}`); + return false; + } + }; + + const decode = (isKey: boolean, data: Uint8Array, pts?: number | null) => { + if (!videoDecoder || videoDecoder.state !== "configured") return; + if (awaitingKeyframe) { + if (!isKey) return; + awaitingKeyframe = false; + } + if (videoDecoder.decodeQueueSize > SOFT_DECODE_QUEUE) { + closeDecoder(); + requestKeyframe(); + return; + } + try { + videoDecoder.decode( + new EncodedVideoChunk({ + type: isKey ? "key" : "delta", + timestamp: pts ?? timestamp, + data, + }), + ); + timestamp += FRAME_DURATION_US; + } catch { + closeDecoder(); + requestKeyframe(); + } + }; + + const requestKeyframe = () => { + if (platform === "android" && socket?.readyState === WebSocket.OPEN) { + socket.send(JSON.stringify({ type: "reset-video", ack: false })); + } + }; + + const scheduleRetry = (channel: "video" | "input", run: () => void) => { + if (stopped || retryTimers.has(channel)) return; + retryTimers.set( + channel, + setTimeout(() => { + retryTimers.delete(channel); + run(); + }, RETRY_DELAY_MS), + ); + }; + + const handleUnauthorized = () => { + stop(); + events.onUnauthorized(); + }; + + // iOS video: fetch the AVCC body and demux into the decoder. + const readIosVideo = async () => { + const demuxer = new AvccDemuxer(); + controller = new AbortController(); + try { + const response = await fetch(httpUrl(`/helper/${device}/stream.avcc`), { + signal: controller.signal, + credentials: access.credentials ? "include" : "same-origin", + }); + if (response.status === 401 || response.status === 403) return handleUnauthorized(); + if (!response.ok || !response.body) throw new Error(`stream ${response.status}`); + const reader = response.body.getReader(); + for (;;) { + const { done, value } = await reader.read(); + if (done || stopped) break; + for (const chunk of demuxer.push(value)) { + switch (chunk.type) { + case "seed": + void createImageBitmap(new Blob([chunk.payload as BlobPart], { type: "image/jpeg" })) + .then((bitmap) => { + paint(bitmap, bitmap.width, bitmap.height); + bitmap.close(); + }) + .catch(() => {}); + break; + case "description": { + awaitingKeyframe = true; + const configured = await configureDecoder({ + codec: avcCodecString(chunk.payload), + description: chunk.payload, + }); + if (!configured) { + await reader.cancel().catch(() => {}); + fallBackToMjpeg(); + return; + } + break; + } + case "keyframe": + case "delta": + decode(chunk.type === "keyframe", chunk.payload); + break; + } + } + } + } catch (cause) { + if (stopped) return; + setStatus("connecting", (cause as Error).message); + } + if (!stopped) scheduleRetry("video", () => void readIosVideo()); + }; + + /** + * serve-sim's helper only accepts HID and pushes its screen config once + * screen capture is running, and the AVCC stream does not reliably start + * it. Touching the MJPEG endpoint does; one aborted request is enough. + */ + const primeIosHelper = async () => { + const controller = new AbortController(); + primeController = controller; + const timeout = setTimeout(() => controller.abort(), 2_000); + try { + const response = await fetch(httpUrl(`/helper/${device}/stream.mjpeg`), { + signal: controller.signal, + credentials: access.credentials ? "include" : "same-origin", + }); + if (response.status === 401 || response.status === 403) return handleUnauthorized(); + await response.body?.getReader().read(); + } catch { + // A failed prime just means the socket may take a retry to come up. + } finally { + clearTimeout(timeout); + controller.abort(); + if (primeController === controller) primeController = null; + } + }; + + // iOS input socket; also carries the screen config the helper pushes. + const connectIosInput = async () => { + if (stopped) return; + await primeIosHelper(); + if (stopped) return; + const ws = new WebSocket(wsUrl(`/helper/ws?device=${device}`)); + ws.binaryType = "arraybuffer"; + socket = ws; + ws.onopen = () => { + ws.send(taggedJson(IOS_MSG_HARDWARE_KEYBOARD, { enabled: false })); + events.onInputConnected(true); + }; + ws.onmessage = (event) => { + if (!(event.data instanceof ArrayBuffer)) return; + const bytes = new Uint8Array(event.data); + if (bytes.length < 1 || bytes[0] !== IOS_TAG_SCREEN_CONFIG) return; + try { + const config = JSON.parse(decoder.decode(bytes.subarray(1))) as DeviceScreenSize; + if (config.width > 0 && config.height > 0) { + screen = config; + events.onScreen(config); + } + } catch { + // Ignore malformed config frames. + } + }; + ws.onclose = (event) => { + if (socket !== ws) return; + socket = null; + if (stopped) return; + events.onInputConnected( + false, + event.reason || (event.code === 1006 ? "input socket refused" : `closed ${event.code}`), + ); + // A rejected HTTP upgrade surfaces as 1006, including an expired stream ticket. + if ( + event.code === 1008 || + event.code === 4401 || + (event.code === 1006 && access.query.wsTicket) + ) + return handleUnauthorized(); + scheduleRetry("input", () => void connectIosInput()); + }; + ws.onerror = () => ws.close(); + }; + + // Android: one socket for video and input. + const connectAndroid = () => { + if (stopped) return; + const ws = new WebSocket(wsUrl(`/ws?device=${device}&frame-meta=1`)); + ws.binaryType = "arraybuffer"; + socket = ws; + ws.onopen = () => { + setStatus("connecting"); + events.onInputConnected(true); + }; + ws.onmessage = (event) => { + if (typeof event.data === "string") { + // The encoder restarts at a new size when the device rotates; the + // next keyframe carries a fresh SPS, so the decoder is rebuilt from it. + if (isVideoSessionMessage(event.data)) closeDecoder(); + return; + } + if (!(event.data instanceof ArrayBuffer)) return; + const packet = parseSemuPacket(event.data); + const needsScan = + packet.isKey === null || + (packet.isKey && (!videoDecoder || videoDecoder.state !== "configured")); + const scanned = needsScan ? scanAccessUnit(packet.data) : null; + const isKey = packet.isKey ?? scanned?.isKey ?? false; + if (scanned?.sps && (!videoDecoder || videoDecoder.state !== "configured")) { + if (configuring) return; + configuring = true; + void configureDecoder({ codec: avcCodecString(scanned.sps) }).then((configured) => { + configuring = false; + awaitingKeyframe = true; + if (configured) requestKeyframe(); + }); + return; + } + if (!videoDecoder || videoDecoder.state !== "configured") { + if (!isKey) requestKeyframe(); + return; + } + decode(isKey, packet.data, packet.timestamp); + }; + ws.onclose = (event) => { + if (socket !== ws) return; + socket = null; + closeDecoder(); + if (stopped) return; + events.onInputConnected(false, event.reason || `closed ${event.code}`); + if ( + event.code === 1008 || + event.code === 4401 || + (event.code === 1006 && access.query.wsTicket) + ) + return handleUnauthorized(); + setStatus("connecting", event.reason || undefined); + scheduleRetry("input", connectAndroid); + }; + ws.onerror = () => ws.close(); + }; + + const start = () => { + if (!stopped) return; + stopped = false; + firstFrame = false; + events.onStatus("connecting"); + if (platform === "ios") { + void connectIosInput(); + if (useWebCodecs) void readIosVideo(); + else fallBackToMjpeg(); + } else if (useWebCodecs) { + connectAndroid(); + } else { + setStatus("error", "This browser cannot decode the Android stream (WebCodecs unavailable)."); + } + }; + + const stop = () => { + if (stopped) return; + stopped = true; + mjpeg = false; + for (const timer of retryTimers.values()) clearTimeout(timer); + retryTimers.clear(); + primeController?.abort(); + primeController = null; + controller?.abort(); + controller = null; + socket?.close(); + socket = null; + closeDecoder(); + }; + + const send = (payload: Uint8Array | string) => { + if (socket?.readyState === WebSocket.OPEN) socket.send(payload); + }; + + const rawPoint = (x: number, y: number) => { + // serve-sim streams the raw framebuffer; rotated devices need input + // remapped into that raw space. + if (platform !== "ios" || !screen || screen.width > screen.height) return { x, y }; + switch (screen.orientation) { + case "landscape_left": + return { x: y, y: 1 - x }; + case "landscape_right": + return { x: 1 - y, y: x }; + case "portrait_upside_down": + return { x: 1 - x, y: 1 - y }; + default: + return { x, y }; + } + }; + + return { + start, + stop, + sendTouch: (phase, x, y) => { + if (platform === "ios") { + send(taggedJson(IOS_MSG_TOUCH, { type: phase, ...rawPoint(x, y) })); + return; + } + const action = phase === "begin" ? "down" : phase === "move" ? "move" : "up"; + send(JSON.stringify({ type: "touch", action, x, y })); + }, + sendKey: (event, phase) => { + if (platform === "ios") { + const usage = hidUsageForCode(event.code); + if (usage !== null) send(taggedJson(IOS_MSG_KEY, { type: phase, usage })); + return; + } + if (phase !== "down") return; + if (event.key === "Escape") return send(JSON.stringify({ type: "back" })); + const keycode = ANDROID_KEYCODE_BY_KEY[event.key]; + if (keycode !== undefined) return send(JSON.stringify({ type: "key", keycode })); + if (event.key.length === 1 && !event.metaKey && !event.ctrlKey) { + send(JSON.stringify({ type: "text", text: event.key })); + } + }, + pressButton: (button) => { + if (platform === "ios") { + const name = + button === "home" + ? "home" + : button === "appSwitcher" + ? "app_switcher" + : button === "power" + ? "lock" + : null; + if (name) send(taggedJson(IOS_MSG_BUTTON, { button: name })); + return; + } + const type = button === "appSwitcher" ? "recents" : button; + if (type === "home" || type === "back" || type === "recents" || type === "power") { + send(JSON.stringify({ type })); + } + }, + rotate: () => { + if (platform !== "ios") return; + const current = screen?.orientation ?? "portrait"; + const next = + IOS_ORIENTATIONS[(IOS_ORIENTATIONS.indexOf(current) + 1) % IOS_ORIENTATIONS.length]!; + send(taggedJson(IOS_MSG_ORIENTATION, { orientation: next })); + }, + }; +} diff --git a/packages/client-runtime/src/state/deviceHubAccess.ts b/packages/client-runtime/src/state/deviceHubAccess.ts index 4c87aa538ac101fc6b54e1dedc05148df3a4c141..9ab65b25e6345665c40fe6e87c333f0aac0aad3f 100644 --- a/packages/client-runtime/src/state/deviceHubAccess.ts +++ b/packages/client-runtime/src/state/deviceHubAccess.ts @@ -20,20 +20,12 @@ import { environmentEndpointUrl } from "../environment/endpoint.ts"; import { ManagedRelayDpopSigner } from "../relay/managedRelay.ts"; import type { RemoteEnvironmentRequestError } from "../rpc/http.ts"; import { executeAuthenticatedEnvironmentHttpRequest } from "./environmentHttpAuth.ts"; +import type { DeviceHubAccess } from "../device/hubAccess.ts"; + +export { type DeviceHubAccess, withDeviceHubQuery } from "../device/hubAccess.ts"; const TICKET_TIMEOUT_MS = 8_000; -export interface DeviceHubAccess { - /** Absolute origin-relative base, e.g. `https://env.example/api/device-hub`. */ - readonly httpBase: string; - /** Same base with the `ws(s)` scheme. */ - readonly wsBase: string; - /** Query parameters to append to every hub request; empty for cookie sessions. */ - readonly query: Readonly>; - /** Whether requests must include cookies (same-origin session). */ - readonly credentials: boolean; -} - export const resolveDeviceHubAccess = Effect.fn("clientRuntime.state.resolveDeviceHubAccess")( function* (input: { readonly prepared: PreparedConnection; @@ -64,10 +56,3 @@ export const resolveDeviceHubAccess = Effect.fn("clientRuntime.state.resolveDevi }; }, ); - -export const withDeviceHubQuery = (url: string, access: DeviceHubAccess): string => { - const entries = Object.entries(access.query); - if (entries.length === 0) return url; - const separator = url.includes("?") ? "&" : "?"; - return `${url}${separator}${new URLSearchParams(entries).toString()}`; -};