| author |  Julius Marminge <julius0216@outlook.com> 2026-09-18 20:36:50-07:00 |
| committer |  GitHub <noreply@github.com> 2026-09-18 20:36:50-07:00 |
| log | 823119350ee29cca434b1295aa5c560f4c9ef98a |
| tree | f3fe805130624552f5942f0716229113ba81516f |
| parent | cb3d95c17487f3d834a5537f7c3aa0106bc9b5ec |
| signature | Signed by PGP key B5690EEEBB952194 |
feat(mobile): view and control agent devices (#12531)
28 files changed, 2023 insertions(+), 739 deletions(-)
apps
mobile
- metro.config.js
scripts
src
- Stack.tsx
components
state
docs
user
- knip.jsonc
packages
client-runtime
- package.json
src
device
state
apps/mobile/metro.config.js+28-1
| ... | ... | @@ -17,6 +17,7 @@ const licenseGeneratorSource = path.join( |
| 17 | 17 | ); |
| 18 | 18 | const escapedWorkspaceRoot = workspaceRoot.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); |
| 19 | 19 | const mobileShikiRoot = path.dirname(require.resolve("shiki/package.json", { paths: [__dirname] })); |
| 20 | const generatedDeviceStreamRoot = path.join(__dirname, ".generated", "device-stream"); | |
| 20 | 21 | const resolveShikiDependencyRoot = (packageName) => { |
| 21 | 22 | const entryPath = require.resolve(packageName, { paths: [mobileShikiRoot] }); |
| 22 | 23 | let currentDir = path.dirname(entryPath); |
| ... | ... | @@ -46,6 +47,7 @@ config.resolver = { |
| 46 | 47 | extraNodeModules: { |
| 47 | 48 | ...config.resolver?.extraNodeModules, |
| 48 | 49 | "@t3tools/mobile-third-party-licenses": generatedLicenseModuleRoot, |
| 50 | "@t3tools/mobile-device-stream": generatedDeviceStreamRoot, | |
| 49 | 51 | shiki: mobileShikiRoot, |
| 50 | 52 | "@shikijs/core": resolveShikiDependencyRoot("@shikijs/core"), |
| 51 | 53 | "@shikijs/engine-javascript": resolveShikiDependencyRoot("@shikijs/engine-javascript"), |
| ... | ... | @@ -93,7 +95,32 @@ async function generateMobileThirdPartyLicenses() { |
| 93 | 95 | ]); |
| 94 | 96 | } |
| 95 | 97 | |
| 96 | module.exports = generateMobileThirdPartyLicenses().then(() => | |
| 98 | async function prepareDeviceStream() { | |
| 99 | const { generateDeviceStreamScript } = await import( | |
| 100 | pathToFileURL(path.join(__dirname, "scripts", "generate-device-stream.mts")).href | |
| 101 | ); | |
| 102 | await generateDeviceStreamScript(); | |
| 103 | if (process.env.NODE_ENV !== "production") { | |
| 104 | let rebuild = Promise.resolve(); | |
| 105 | for (const [directory, files] of [ | |
| 106 | [path.join(__dirname, "src/features/devices"), ["device-stream.browser.ts"]], | |
| 107 | [ | |
| 108 | path.join(workspaceRoot, "packages/client-runtime/src/device"), | |
| 109 | ["stream.ts", "hubAccess.ts"], | |
| 110 | ], | |
| 111 | ]) { | |
| 112 | // The generated module participates in Metro's normal Fast Refresh. | |
| 113 | fs.watch(directory, { persistent: false }, (_event, filename) => { | |
| 114 | if (filename && !files.includes(String(filename))) return; | |
| 115 | rebuild = rebuild.then(generateDeviceStreamScript).catch((error) => { | |
| 116 | console.error("Could not rebuild the device stream:", error); | |
| 117 | }); | |
| 118 | }); | |
| 119 | } | |
| 120 | } | |
| 121 | } | |
| 122 | ||
| 123 | module.exports = Promise.all([generateMobileThirdPartyLicenses(), prepareDeviceStream()]).then(() => | |
| 97 | 124 | withUniwindConfig(config, { |
| 98 | 125 | cssEntryFile: "./global.css", |
| 99 | 126 | extraThemes, |
apps/mobile/scripts/generate-device-stream.mts created+39
| ... | ... | @@ -0,0 +1,39 @@ |
| 1 | import * as NodeFSP from "node:fs/promises"; | |
| 2 | import * as NodePath from "node:path"; | |
| 3 | import * as NodeURL from "node:url"; | |
| 4 | import { build } from "vite-plus"; | |
| 5 | ||
| 6 | const mobileRoot = NodePath.resolve(NodePath.dirname(NodeURL.fileURLToPath(import.meta.url)), ".."); | |
| 7 | ||
| 8 | /** Metro embeds the shared browser transport as a small script in the native WebView. */ | |
| 9 | export async function generateDeviceStreamScript() { | |
| 10 | const result = await build({ | |
| 11 | configFile: false, | |
| 12 | logLevel: "silent", | |
| 13 | build: { | |
| 14 | write: false, | |
| 15 | target: "es2022", | |
| 16 | minify: true, | |
| 17 | lib: { | |
| 18 | entry: NodePath.join(mobileRoot, "src/features/devices/device-stream.browser.ts"), | |
| 19 | name: "T3DeviceStream", | |
| 20 | formats: ["iife"], | |
| 21 | }, | |
| 22 | }, | |
| 23 | }); | |
| 24 | const bundles = Array.isArray(result) ? result : [result]; | |
| 25 | const chunk = bundles | |
| 26 | .flatMap((bundle) => ("output" in bundle ? bundle.output : [])) | |
| 27 | .find((output) => output.type === "chunk"); | |
| 28 | if (!chunk) throw new Error("Device stream build did not emit a script."); | |
| 29 | const root = NodePath.join(mobileRoot, ".generated/device-stream"); | |
| 30 | await NodeFSP.mkdir(root, { recursive: true }); | |
| 31 | for (const [name, contents] of [ | |
| 32 | ["index.js", `module.exports = ${JSON.stringify(chunk.code)};\n`], | |
| 33 | ["package.json", '{"main":"index.js"}\n'], | |
| 34 | ] as const) { | |
| 35 | const destination = NodePath.join(root, name); | |
| 36 | const previous = await NodeFSP.readFile(destination, "utf8").catch(() => null); | |
| 37 | if (previous !== contents) await NodeFSP.writeFile(destination, contents); | |
| 38 | } | |
| 39 | } |
apps/mobile/src/Stack.tsx+11
| ... | ... | @@ -30,6 +30,7 @@ import { |
| 30 | 30 | import { ReviewCommentComposerSheet } from "./features/review/ReviewCommentComposerSheet"; |
| 31 | 31 | import { ReviewSheet } from "./features/review/ReviewSheet"; |
| 32 | 32 | import { ThreadTerminalRouteScreen } from "./features/terminal/ThreadTerminalRouteScreen"; |
| 33 | import { DevicePreviewRouteScreen } from "./features/devices/DevicePreviewRouteScreen"; | |
| 33 | 34 | import { GitBranchesSheet } from "./features/threads/git/GitBranchesSheet"; |
| 34 | 35 | import { GitCommitSheet } from "./features/threads/git/GitCommitSheet"; |
| 35 | 36 | import { GitConfirmSheet } from "./features/threads/git/GitConfirmSheet"; |
| ... | ... | @@ -449,6 +450,7 @@ const WORKSPACE_OVERLAY_ROUTES = new Set([ |
| 449 | 450 | "SettingsLegal", |
| 450 | 451 | "SettingsSheet", |
| 451 | 452 | "ThreadReviewComment", |
| 453 | "ThreadDevicePreview", | |
| 452 | 454 | "ThreadSettingsSheet", |
| 453 | 455 | ]); |
| 454 | 456 | |
| ... | ... | @@ -595,6 +597,15 @@ export const RootStack = createNativeStackNavigator({ |
| 595 | 597 | linking: `${THREAD_LINKING_PREFIX}/terminal`, |
| 596 | 598 | options: SOLID_HEADER_OPTIONS, |
| 597 | 599 | }), |
| 600 | ThreadDevicePreview: createNativeStackScreen({ | |
| 601 | screen: DevicePreviewRouteScreen, | |
| 602 | linking: `${THREAD_LINKING_PREFIX}/devices`, | |
| 603 | options: { | |
| 604 | presentation: "fullScreenModal", | |
| 605 | headerShown: false, | |
| 606 | gestureEnabled: false, | |
| 607 | }, | |
| 608 | }), | |
| 598 | 609 | ThreadReview: createNativeStackScreen({ |
| 599 | 610 | screen: ReviewSheet, |
| 600 | 611 | linking: `${THREAD_LINKING_PREFIX}/review`, |
apps/mobile/src/components/AppSymbol.tsx+3
| ... | ... | @@ -60,6 +60,7 @@ import IconGitBranch from "@tabler/icons-react-native/IconGitBranch"; |
| 60 | 60 | import IconGitMerge from "@tabler/icons-react-native/IconGitMerge"; |
| 61 | 61 | import IconGitPullRequest from "@tabler/icons-react-native/IconGitPullRequest"; |
| 62 | 62 | import IconHammer from "@tabler/icons-react-native/IconHammer"; |
| 63 | import IconHome from "@tabler/icons-react-native/IconHome"; | |
| 63 | 64 | import IconInfoCircle from "@tabler/icons-react-native/IconInfoCircle"; |
| 64 | 65 | import IconKeyboard from "@tabler/icons-react-native/IconKeyboard"; |
| 65 | 66 | import IconKeyboardHide from "@tabler/icons-react-native/IconKeyboardHide"; |
| ... | ... | @@ -161,6 +162,7 @@ const ANDROID_ICON_BY_SF_SYMBOL = { |
| 161 | 162 | gearshape: IconSettings, |
| 162 | 163 | globe: IconWorld, |
| 163 | 164 | hammer: IconHammer, |
| 165 | house: IconHome, | |
| 164 | 166 | "info.circle": IconInfoCircle, |
| 165 | 167 | internaldrive: IconDatabase, |
| 166 | 168 | keyboard: IconKeyboard, |
| ... | ... | @@ -194,6 +196,7 @@ const ANDROID_ICON_BY_SF_SYMBOL = { |
| 194 | 196 | "sidebar.right": IconLayoutSidebarRight, |
| 195 | 197 | "slider.horizontal.3": IconAdjustmentsHorizontal, |
| 196 | 198 | "square.and.pencil": IconEdit, |
| 199 | "square.on.square": IconCopy, | |
| 197 | 200 | "square.grid.2x2": IconApps, |
| 198 | 201 | "square.split.2x1": IconLayoutColumns, |
| 199 | 202 | star: IconStar, |
apps/mobile/src/features/devices/DevicePreviewRouteScreen.tsx created+309
| ... | ... | @@ -0,0 +1,309 @@ |
| 1 | import { useIsFocused, useNavigation, type StaticScreenProps } from "@react-navigation/native"; | |
| 2 | import { createNativeStackNavigator } from "@react-navigation/native-stack"; | |
| 3 | import { EnvironmentId, ThreadId } from "@t3tools/contracts"; | |
| 4 | import * as Cause from "effect/Cause"; | |
| 5 | import { useCallback, useEffect, useMemo, useRef, useState, type RefObject } from "react"; | |
| 6 | import { ActivityIndicator, Alert, AppState, Platform, Pressable, View } from "react-native"; | |
| 7 | import { useSafeAreaInsets } from "react-native-safe-area-context"; | |
| 8 | ||
| 9 | import { AppText } from "../../components/AppText"; | |
| 10 | import { ScreenHeader, type ScreenHeaderMenuItem } from "../../components/ScreenHeader"; | |
| 11 | import { NativeHeaderToolbar } from "../../native/StackHeader"; | |
| 12 | import { deviceEnvironment, refreshDeviceHubAccess, useDeviceHubAccess } from "../../state/device"; | |
| 13 | import { useEnvironmentQuery } from "../../state/query"; | |
| 14 | import { useAtomCommand } from "../../state/use-atom-command"; | |
| 15 | import { useAppearancePreferences } from "../settings/appearance/AppearancePreferencesProvider"; | |
| 16 | import { DeviceStreamWebView, type DeviceStreamRef } from "./DeviceStreamWebView"; | |
| 17 | import { | |
| 18 | selectedThreadDevicePreview, | |
| 19 | threadDevicePreviews, | |
| 20 | type ThreadDevicePreview, | |
| 21 | } from "./threadDevicePreviews"; | |
| 22 | ||
| 23 | const DevicePreviewStack = createNativeStackNavigator<{ DevicePreview: undefined }>(); | |
| 24 | ||
| 25 | type DevicePreviewRouteScreenProps = StaticScreenProps<{ | |
| 26 | readonly environmentId: string; | |
| 27 | readonly threadId: string; | |
| 28 | }>; | |
| 29 | ||
| 30 | /** The nested native stack supplies the navigation bar inside the modal. */ | |
| 31 | export function DevicePreviewRouteScreen({ route }: DevicePreviewRouteScreenProps) { | |
| 32 | const navigation = useNavigation(); | |
| 33 | const onClose = useCallback(() => navigation.goBack(), [navigation]); | |
| 34 | return ( | |
| 35 | <View collapsable={false} className="flex-1 bg-sheet"> | |
| 36 | <DevicePreviewStack.Navigator | |
| 37 | screenOptions={{ | |
| 38 | headerShown: Platform.OS === "ios", | |
| 39 | headerBackVisible: false, | |
| 40 | headerShadowVisible: false, | |
| 41 | headerTransparent: false, | |
| 42 | headerTitleStyle: { fontSize: 17, fontWeight: "600" }, | |
| 43 | }} | |
| 44 | > | |
| 45 | <DevicePreviewStack.Screen name="DevicePreview"> | |
| 46 | {() => ( | |
| 47 | <DevicePreviewScreen | |
| 48 | environmentId={EnvironmentId.make(route.params.environmentId)} | |
| 49 | threadId={ThreadId.make(route.params.threadId)} | |
| 50 | onClose={onClose} | |
| 51 | /> | |
| 52 | )} | |
| 53 | </DevicePreviewStack.Screen> | |
| 54 | </DevicePreviewStack.Navigator> | |
| 55 | </View> | |
| 56 | ); | |
| 57 | } | |
| 58 | ||
| 59 | function DevicePreviewScreen({ | |
| 60 | environmentId, | |
| 61 | threadId, | |
| 62 | onClose, | |
| 63 | }: { | |
| 64 | readonly environmentId: EnvironmentId; | |
| 65 | readonly threadId: ThreadId; | |
| 66 | readonly onClose: () => void; | |
| 67 | }) { | |
| 68 | const insets = useSafeAreaInsets(); | |
| 69 | const { themeVariables } = useAppearancePreferences(); | |
| 70 | const focused = useIsFocused(); | |
| 71 | const [foreground, setForeground] = useState(AppState.currentState === "active"); | |
| 72 | const [selectedKey, setSelectedKey] = useState<string | null>(null); | |
| 73 | const [inputConnected, setInputConnected] = useState(false); | |
| 74 | const [streamAttempt, setStreamAttempt] = useState(0); | |
| 75 | const [shuttingDown, setShuttingDown] = useState(false); | |
| 76 | const shutdown = useAtomCommand(deviceEnvironment.shutdown, { reportFailure: false }); | |
| 77 | const streamRef = useRef<DeviceStreamRef>(null); | |
| 78 | const state = useEnvironmentQuery(deviceEnvironment.state({ environmentId, input: {} })); | |
| 79 | const previews = useMemo( | |
| 80 | () => threadDevicePreviews(state.data, threadId), | |
| 81 | [state.data, threadId], | |
| 82 | ); | |
| 83 | const preview = selectedThreadDevicePreview(previews, selectedKey); | |
| 84 | const onInputConnected = useCallback( | |
| 85 | async (connected: boolean) => setInputConnected(connected), | |
| 86 | [], | |
| 87 | ); | |
| 88 | useEffect(() => { | |
| 89 | const subscription = AppState.addEventListener("change", (state) => | |
| 90 | setForeground(state === "active"), | |
| 91 | ); | |
| 92 | return () => subscription.remove(); | |
| 93 | }, []); | |
| 94 | useEffect(() => { | |
| 95 | if (focused && state.data !== null && previews.length === 0) onClose(); | |
| 96 | }, [focused, state.data, previews.length, onClose]); | |
| 97 | ||
| 98 | const shutDownDevice = async () => { | |
| 99 | if (!preview || shuttingDown) return; | |
| 100 | setShuttingDown(true); | |
| 101 | try { | |
| 102 | const result = await shutdown({ | |
| 103 | environmentId, | |
| 104 | input: { | |
| 105 | hostId: preview.session.hostId, | |
| 106 | deviceId: preview.session.deviceId, | |
| 107 | platform: preview.session.platform, | |
| 108 | }, | |
| 109 | }); | |
| 110 | if (result._tag === "Failure") { | |
| 111 | Alert.alert("Could not shut down device", String(Cause.squash(result.cause))); | |
| 112 | } | |
| 113 | } finally { | |
| 114 | setShuttingDown(false); | |
| 115 | } | |
| 116 | }; | |
| 117 | ||
| 118 | const controls: ScreenHeaderMenuItem[] = [ | |
| 119 | { | |
| 120 | id: "reload", | |
| 121 | title: "Reload stream", | |
| 122 | icon: "arrow.clockwise", | |
| 123 | disabled: !preview || shuttingDown, | |
| 124 | onPress: () => { | |
| 125 | setInputConnected(false); | |
| 126 | setStreamAttempt((attempt) => attempt + 1); | |
| 127 | }, | |
| 128 | }, | |
| 129 | ...(preview?.session.platform === "android" | |
| 130 | ? [ | |
| 131 | { | |
| 132 | id: "back", | |
| 133 | title: "Back", | |
| 134 | icon: "arrow.left", | |
| 135 | disabled: !inputConnected, | |
| 136 | onPress: () => streamRef.current?.back(), | |
| 137 | }, | |
| 138 | ] | |
| 139 | : []), | |
| 140 | { | |
| 141 | id: "app-switcher", | |
| 142 | title: "App switcher", | |
| 143 | icon: "square.on.square", | |
| 144 | disabled: !inputConnected, | |
| 145 | onPress: () => streamRef.current?.appSwitcher(), | |
| 146 | }, | |
| 147 | ...(preview?.session.platform === "ios" | |
| 148 | ? [ | |
| 149 | { | |
| 150 | id: "rotate", | |
| 151 | title: "Rotate device", | |
| 152 | icon: "arrow.clockwise", | |
| 153 | disabled: !inputConnected, | |
| 154 | onPress: () => streamRef.current?.rotate(), | |
| 155 | }, | |
| 156 | ] | |
| 157 | : []), | |
| 158 | { | |
| 159 | id: "shutdown", | |
| 160 | title: shuttingDown ? "Shutting down…" : "Shut down device", | |
| 161 | icon: "power", | |
| 162 | disabled: !preview || shuttingDown, | |
| 163 | onPress: () => void shutDownDevice(), | |
| 164 | }, | |
| 165 | ]; | |
| 166 | return ( | |
| 167 | <View className="flex-1 bg-sheet" style={{ paddingBottom: insets.bottom }}> | |
| 168 | <ScreenHeader | |
| 169 | title={preview?.name ?? "Devices"} | |
| 170 | sidebar={false} | |
| 171 | onBack={onClose} | |
| 172 | options={{ headerBackVisible: false }} | |
| 173 | actions={[ | |
| 174 | { | |
| 175 | accessibilityLabel: "Home", | |
| 176 | icon: "house", | |
| 177 | disabled: !inputConnected, | |
| 178 | onPress: () => streamRef.current?.home(), | |
| 179 | }, | |
| 180 | ]} | |
| 181 | menus={[ | |
| 182 | { | |
| 183 | title: "Device options", | |
| 184 | icon: "ellipsis", | |
| 185 | items: [ | |
| 186 | ...(previews.length > 1 | |
| 187 | ? [ | |
| 188 | { | |
| 189 | id: "devices", | |
| 190 | title: "Devices", | |
| 191 | inline: true, | |
| 192 | items: previews.map((device) => ({ | |
| 193 | id: device.key, | |
| 194 | title: device.name, | |
| 195 | subtitle: device.description, | |
| 196 | selected: device.key === preview?.key, | |
| 197 | onPress: () => setSelectedKey(device.key), | |
| 198 | })), | |
| 199 | }, | |
| 200 | ] | |
| 201 | : []), | |
| 202 | ...controls, | |
| 203 | ], | |
| 204 | }, | |
| 205 | ]} | |
| 206 | /> | |
| 207 | {Platform.OS === "ios" ? ( | |
| 208 | <NativeHeaderToolbar placement="left"> | |
| 209 | <NativeHeaderToolbar.Button | |
| 210 | icon="xmark" | |
| 211 | accessibilityLabel="Close device preview" | |
| 212 | onPress={onClose} | |
| 213 | separateBackground | |
| 214 | /> | |
| 215 | </NativeHeaderToolbar> | |
| 216 | ) : null} | |
| 217 | {preview && focused && foreground ? ( | |
| 218 | <OpenDevicePreview | |
| 219 | key={`${preview.key}:${streamAttempt}`} | |
| 220 | environmentId={environmentId} | |
| 221 | preview={preview} | |
| 222 | streamRef={streamRef} | |
| 223 | onInputConnected={onInputConnected} | |
| 224 | /> | |
| 225 | ) : ( | |
| 226 | <View className="flex-1 items-center justify-center gap-4 px-6"> | |
| 227 | {state.error ? ( | |
| 228 | <> | |
| 229 | <AppText selectable className="text-center text-sm text-foreground-muted"> | |
| 230 | {state.error} | |
| 231 | </AppText> | |
| 232 | <Pressable | |
| 233 | accessibilityRole="button" | |
| 234 | className="rounded-full bg-subtle px-6 py-3" | |
| 235 | onPress={state.refresh} | |
| 236 | > | |
| 237 | <AppText>Retry</AppText> | |
| 238 | </Pressable> | |
| 239 | </> | |
| 240 | ) : focused && foreground ? ( | |
| 241 | <ActivityIndicator color={themeVariables["--color-icon"]} /> | |
| 242 | ) : null} | |
| 243 | </View> | |
| 244 | )} | |
| 245 | </View> | |
| 246 | ); | |
| 247 | } | |
| 248 | ||
| 249 | function OpenDevicePreview({ | |
| 250 | environmentId, | |
| 251 | preview, | |
| 252 | streamRef, | |
| 253 | onInputConnected, | |
| 254 | }: { | |
| 255 | readonly environmentId: EnvironmentId; | |
| 256 | readonly preview: ThreadDevicePreview; | |
| 257 | readonly streamRef: RefObject<DeviceStreamRef | null>; | |
| 258 | readonly onInputConnected: (connected: boolean) => Promise<void>; | |
| 259 | }) { | |
| 260 | const { session } = preview; | |
| 261 | const { themeVariables } = useAppearancePreferences(); | |
| 262 | const { access, error, refresh } = useDeviceHubAccess(environmentId, session.hostId); | |
| 263 | const onUnauthorized = useCallback( | |
| 264 | async () => refreshDeviceHubAccess(environmentId), | |
| 265 | [environmentId], | |
| 266 | ); | |
| 267 | useEffect(() => { | |
| 268 | refreshDeviceHubAccess(environmentId); | |
| 269 | return () => void onInputConnected(false); | |
| 270 | }, [environmentId, onInputConnected]); | |
| 271 | return access ? ( | |
| 272 | <DeviceStreamWebView | |
| 273 | ref={streamRef} | |
| 274 | access={access} | |
| 275 | platform={session.platform} | |
| 276 | deviceId={session.deviceId} | |
| 277 | colors={{ | |
| 278 | background: themeVariables["--color-sheet-solid"], | |
| 279 | foreground: themeVariables["--color-foreground"], | |
| 280 | muted: themeVariables["--color-foreground-muted"], | |
| 281 | buttonBackground: themeVariables["--color-subtle"], | |
| 282 | }} | |
| 283 | onUnauthorized={onUnauthorized} | |
| 284 | onInputConnected={onInputConnected} | |
| 285 | /> | |
| 286 | ) : ( | |
| 287 | <View className="flex-1 items-center justify-center gap-4 px-6"> | |
| 288 | {error ? ( | |
| 289 | <> | |
| 290 | <AppText selectable className="text-center text-sm text-foreground-muted"> | |
| 291 | {error} | |
| 292 | </AppText> | |
| 293 | <Pressable | |
| 294 | accessibilityRole="button" | |
| 295 | className="rounded-full bg-subtle px-6 py-3" | |
| 296 | onPress={refresh} | |
| 297 | > | |
| 298 | <AppText>Retry</AppText> | |
| 299 | </Pressable> | |
| 300 | </> | |
| 301 | ) : ( | |
| 302 | <> | |
| 303 | <ActivityIndicator color={themeVariables["--color-icon"]} /> | |
| 304 | <AppText className="text-sm text-foreground-muted">Connecting to device...</AppText> | |
| 305 | </> | |
| 306 | )} | |
| 307 | </View> | |
| 308 | ); | |
| 309 | } |
apps/mobile/src/features/devices/DeviceStreamWebView.tsx created+109
| ... | ... | @@ -0,0 +1,109 @@ |
| 1 | import deviceStreamScript from "@t3tools/mobile-device-stream"; | |
| 2 | import { useImperativeHandle, useLayoutEffect, useMemo, useRef, useState, type Ref } from "react"; | |
| 3 | import { Platform } from "react-native"; | |
| 4 | import { WebView } from "react-native-webview"; | |
| 5 | ||
| 6 | import { | |
| 7 | deviceStreamDocument, | |
| 8 | deviceStreamMessage, | |
| 9 | type DeviceStreamConfiguration, | |
| 10 | } from "./device-stream-document"; | |
| 11 | ||
| 12 | export interface DeviceStreamRef { | |
| 13 | home: () => void; | |
| 14 | back: () => void; | |
| 15 | appSwitcher: () => void; | |
| 16 | rotate: () => void; | |
| 17 | } | |
| 18 | ||
| 19 | type NativeStreamBridge = { | |
| 20 | readonly ref?: Ref<DeviceStreamRef>; | |
| 21 | readonly onUnauthorized: () => Promise<void>; | |
| 22 | readonly onInputConnected: (connected: boolean) => Promise<void>; | |
| 23 | }; | |
| 24 | ||
| 25 | export function DeviceStreamWebView({ | |
| 26 | ref, | |
| 27 | ...props | |
| 28 | }: DeviceStreamConfiguration & NativeStreamBridge) { | |
| 29 | const [attempt, setAttempt] = useState(0); | |
| 30 | const configuration = JSON.stringify({ | |
| 31 | access: props.access, | |
| 32 | platform: props.platform, | |
| 33 | deviceId: props.deviceId, | |
| 34 | colors: props.colors, | |
| 35 | }); | |
| 36 | return ( | |
| 37 | <DeviceStreamDocumentView | |
| 38 | key={`${attempt}:${configuration}`} | |
| 39 | ref={ref} | |
| 40 | configuration={configuration} | |
| 41 | background={props.colors.background} | |
| 42 | onUnauthorized={props.onUnauthorized} | |
| 43 | onInputConnected={props.onInputConnected} | |
| 44 | onRetry={() => setAttempt((attempt) => attempt + 1)} | |
| 45 | /> | |
| 46 | ); | |
| 47 | } | |
| 48 | ||
| 49 | function DeviceStreamDocumentView({ | |
| 50 | ref, | |
| 51 | configuration, | |
| 52 | background, | |
| 53 | onUnauthorized, | |
| 54 | onInputConnected, | |
| 55 | onRetry, | |
| 56 | }: NativeStreamBridge & { | |
| 57 | readonly configuration: string; | |
| 58 | readonly background: string; | |
| 59 | readonly onRetry: () => void; | |
| 60 | }) { | |
| 61 | const webView = useRef<WebView>(null); | |
| 62 | const source = useMemo( | |
| 63 | () => ({ | |
| 64 | html: deviceStreamDocument(configuration, deviceStreamScript), | |
| 65 | // Android WebCodecs needs a secure document; streams still use the environment's URLs. | |
| 66 | baseUrl: Platform.OS === "android" ? "https://localhost/" : "file:///", | |
| 67 | }), | |
| 68 | [configuration], | |
| 69 | ); | |
| 70 | const command = (button: keyof DeviceStreamRef) => { | |
| 71 | webView.current?.injectJavaScript( | |
| 72 | `window.T3DeviceStream?.command(${JSON.stringify(button)}); true;`, | |
| 73 | ); | |
| 74 | }; | |
| 75 | useImperativeHandle(ref, () => ({ | |
| 76 | home: () => command("home"), | |
| 77 | back: () => command("back"), | |
| 78 | appSwitcher: () => command("appSwitcher"), | |
| 79 | rotate: () => command("rotate"), | |
| 80 | })); | |
| 81 | useLayoutEffect(() => { | |
| 82 | const view = webView.current; | |
| 83 | return () => view?.injectJavaScript("window.T3DeviceStream?.stop(); true;"); | |
| 84 | }, []); | |
| 85 | return ( | |
| 86 | <WebView | |
| 87 | ref={webView} | |
| 88 | source={source} | |
| 89 | originWhitelist={["*"]} | |
| 90 | scrollEnabled={false} | |
| 91 | bounces={false} | |
| 92 | mixedContentMode="always" | |
| 93 | allowUniversalAccessFromFileURLs | |
| 94 | contentInsetAdjustmentBehavior="never" | |
| 95 | setSupportMultipleWindows={false} | |
| 96 | style={{ flex: 1, backgroundColor: background }} | |
| 97 | onLoadStart={() => void onInputConnected(false)} | |
| 98 | onShouldStartLoadWithRequest={(request) => | |
| 99 | request.url === "about:blank" || request.url === source.baseUrl | |
| 100 | } | |
| 101 | onMessage={(event) => { | |
| 102 | const message = deviceStreamMessage(event.nativeEvent.data); | |
| 103 | if (message?.type === "unauthorized") void onUnauthorized(); | |
| 104 | else if (message?.type === "input") void onInputConnected(message.connected); | |
| 105 | else if (message?.type === "retry") onRetry(); | |
| 106 | }} | |
| 107 | /> | |
| 108 | ); | |
| 109 | } |
apps/mobile/src/features/devices/device-preview-button.tsx created+41
| ... | ... | @@ -0,0 +1,41 @@ |
| 1 | import { Pressable, View } from "react-native"; | |
| 2 | ||
| 3 | import { AppText } from "../../components/AppText"; | |
| 4 | import { SymbolView } from "../../components/AppSymbol"; | |
| 5 | ||
| 6 | export function DevicePreviewButton(props: { | |
| 7 | readonly count: number; | |
| 8 | readonly onPress: () => void; | |
| 9 | readonly compact?: boolean; | |
| 10 | }) { | |
| 11 | const compact = props.compact ?? true; | |
| 12 | return ( | |
| 13 | <Pressable | |
| 14 | accessibilityRole="button" | |
| 15 | accessibilityLabel={props.count === 1 ? "View device" : `View ${props.count} devices`} | |
| 16 | accessibilityHint="Watch and control devices open in this thread" | |
| 17 | onPress={props.onPress} | |
| 18 | className={ | |
| 19 | compact | |
| 20 | ? "size-11 shrink-0 items-center justify-center rounded-full active:bg-subtle" | |
| 21 | : "h-11 shrink-0 flex-row items-center justify-center gap-2 rounded-full px-4 active:bg-subtle" | |
| 22 | } | |
| 23 | > | |
| 24 | <SymbolView | |
| 25 | name={{ ios: "iphone", android: "smartphone" }} | |
| 26 | size={20} | |
| 27 | tintColorClassName="accent-icon" | |
| 28 | type="monochrome" | |
| 29 | /> | |
| 30 | {!compact ? ( | |
| 31 | <AppText className="font-t3-medium text-xs text-foreground"> | |
| 32 | {props.count === 1 ? "One device open" : `${props.count} devices open`} | |
| 33 | </AppText> | |
| 34 | ) : props.count > 1 ? ( | |
| 35 | <View className="absolute right-0.5 top-0.5 min-w-4 items-center rounded-full bg-primary px-1"> | |
| 36 | <AppText className="text-2xs font-t3-bold text-primary-foreground">{props.count}</AppText> | |
| 37 | </View> | |
| 38 | ) : null} | |
| 39 | </Pressable> | |
| 40 | ); | |
| 41 | } |
apps/mobile/src/features/devices/device-stream-document.test.ts created+49
| ... | ... | @@ -0,0 +1,49 @@ |
| 1 | import * as NodeVM from "node:vm"; | |
| 2 | import { describe, expect, it } from "vite-plus/test"; | |
| 3 | ||
| 4 | import { deviceStreamDocument, deviceStreamMessage } from "./device-stream-document"; | |
| 5 | ||
| 6 | describe("native device stream document", () => { | |
| 7 | it("keeps ticket and device values from terminating the embedded script", () => { | |
| 8 | const deviceId = '</script><script>alert("device")</script>'; | |
| 9 | const configuration = JSON.stringify({ deviceId, ticket: "<ticket>" }); | |
| 10 | const html = deviceStreamDocument( | |
| 11 | configuration, | |
| 12 | "var T3DeviceStream={start(input){return input}};", | |
| 13 | ); | |
| 14 | const script = html.match(/<script>([\s\S]*)<\/script>/)?.[1]; | |
| 15 | expect(html.match(/<script>/g)).toHaveLength(1); | |
| 16 | const input: unknown = NodeVM.runInNewContext(script!); | |
| 17 | expect(input).toEqual({ deviceId, ticket: "<ticket>" }); | |
| 18 | }); | |
| 19 | ||
| 20 | it("preserves script strings containing an HTML closing delimiter", () => { | |
| 21 | const html = deviceStreamDocument("{}", 'var text="</script>";'); | |
| 22 | expect(html.match(/<\/script>/g)).toHaveLength(1); | |
| 23 | expect(html).toContain('var text="<\\/script>";'); | |
| 24 | }); | |
| 25 | }); | |
| 26 | ||
| 27 | describe("native device stream messages", () => { | |
| 28 | it("accepts authentication renewal and input connection changes", () => { | |
| 29 | expect(deviceStreamMessage('{"type":"unauthorized"}')).toEqual({ type: "unauthorized" }); | |
| 30 | expect(deviceStreamMessage('{"type":"input","connected":true}')).toEqual({ | |
| 31 | type: "input", | |
| 32 | connected: true, | |
| 33 | }); | |
| 34 | expect(deviceStreamMessage('{"type":"input","connected":false}')).toEqual({ | |
| 35 | type: "input", | |
| 36 | connected: false, | |
| 37 | }); | |
| 38 | expect(deviceStreamMessage('{"type":"retry"}')).toEqual({ type: "retry" }); | |
| 39 | }); | |
| 40 | ||
| 41 | it.each([ | |
| 42 | "invalid JSON", | |
| 43 | "null", | |
| 44 | '"input"', | |
| 45 | "{}", | |
| 46 | '{"type":"input","connected":"yes"}', | |
| 47 | '{"type":"unknown"}', | |
| 48 | ])("ignores invalid bridge messages: %s", (data) => expect(deviceStreamMessage(data)).toBeNull()); | |
| 49 | }); |
apps/mobile/src/features/devices/device-stream-document.ts created+41
| ... | ... | @@ -0,0 +1,41 @@ |
| 1 | import type { DeviceHubAccess } from "@t3tools/client-runtime/device/hub-access"; | |
| 2 | import type { DevicePlatform } from "@t3tools/contracts"; | |
| 3 | ||
| 4 | export interface DeviceStreamConfiguration { | |
| 5 | readonly access: DeviceHubAccess; | |
| 6 | readonly platform: DevicePlatform; | |
| 7 | readonly deviceId: string; | |
| 8 | readonly colors: { | |
| 9 | readonly background: string; | |
| 10 | readonly foreground: string; | |
| 11 | readonly muted: string; | |
| 12 | readonly buttonBackground: string; | |
| 13 | }; | |
| 14 | } | |
| 15 | ||
| 16 | export function deviceStreamDocument(configuration: string, script: string) { | |
| 17 | // Tickets and device names are data, including any HTML delimiter characters. | |
| 18 | const safeConfiguration = configuration.replace(/</g, "\\u003c"); | |
| 19 | const safeScript = script.replace(/<\/script/gi, "<\\/script"); | |
| 20 | return `<!doctype html><html><head><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"></head><body><script>${safeScript}\nT3DeviceStream.start(${safeConfiguration});</script></body></html>`; | |
| 21 | } | |
| 22 | ||
| 23 | export function deviceStreamMessage(data: string) { | |
| 24 | try { | |
| 25 | const message: unknown = JSON.parse(data); | |
| 26 | if (typeof message !== "object" || message === null || !("type" in message)) return null; | |
| 27 | if (message.type === "unauthorized" || message.type === "retry") { | |
| 28 | return { type: message.type } as const; | |
| 29 | } | |
| 30 | if ( | |
| 31 | message.type === "input" && | |
| 32 | "connected" in message && | |
| 33 | typeof message.connected === "boolean" | |
| 34 | ) { | |
| 35 | return { type: message.type, connected: message.connected } as const; | |
| 36 | } | |
| 37 | } catch { | |
| 38 | // Ignore messages that are not part of the stream bridge. | |
| 39 | } | |
| 40 | return null; | |
| 41 | } |
apps/mobile/src/features/devices/device-stream.browser.ts created+223
| ... | ... | @@ -0,0 +1,223 @@ |
| 1 | import { | |
| 2 | createDeviceStreamClient, | |
| 3 | type DeviceScreenSize, | |
| 4 | } from "@t3tools/client-runtime/device/stream"; | |
| 5 | ||
| 6 | import type { DeviceStreamConfiguration } from "./device-stream-document"; | |
| 7 | ||
| 8 | declare global { | |
| 9 | interface Window { | |
| 10 | ReactNativeWebView: { postMessage: (message: string) => void }; | |
| 11 | } | |
| 12 | } | |
| 13 | ||
| 14 | let activeClient: ReturnType<typeof createDeviceStreamClient> | null = null; | |
| 15 | let activeImage: HTMLImageElement | null = null; | |
| 16 | ||
| 17 | export function stop() { | |
| 18 | activeClient?.stop(); | |
| 19 | activeClient = null; | |
| 20 | activeImage?.removeAttribute("src"); | |
| 21 | activeImage = null; | |
| 22 | } | |
| 23 | ||
| 24 | export function command(button: "home" | "back" | "appSwitcher" | "rotate") { | |
| 25 | if (button === "rotate") activeClient?.rotate(); | |
| 26 | else activeClient?.pressButton(button); | |
| 27 | } | |
| 28 | ||
| 29 | /** Bundled into the existing native WebView without React or Expo's web runtime. */ | |
| 30 | export function start(configuration: DeviceStreamConfiguration) { | |
| 31 | stop(); | |
| 32 | // oxlint-disable-next-line unicorn/require-post-message-target-origin -- The native WebView bridge takes one string. | |
| 33 | const post = (message: object) => window.ReactNativeWebView.postMessage(JSON.stringify(message)); | |
| 34 | const { colors, platform } = configuration; | |
| 35 | Object.assign(document.documentElement.style, { height: "100%", overflow: "hidden" }); | |
| 36 | Object.assign(document.body.style, { | |
| 37 | margin: "0", | |
| 38 | height: "100%", | |
| 39 | overflow: "hidden", | |
| 40 | background: colors.background, | |
| 41 | color: colors.foreground, | |
| 42 | fontFamily: "system-ui", | |
| 43 | }); | |
| 44 | const container = document.createElement("div"); | |
| 45 | Object.assign(container.style, { | |
| 46 | position: "fixed", | |
| 47 | inset: "0", | |
| 48 | containerType: "size", | |
| 49 | display: "flex", | |
| 50 | alignItems: "center", | |
| 51 | justifyContent: "center", | |
| 52 | }); | |
| 53 | const frame = document.createElement("div"); | |
| 54 | frame.setAttribute("role", "application"); | |
| 55 | frame.setAttribute( | |
| 56 | "aria-label", | |
| 57 | platform === "ios" ? "iOS Simulator screen" : "Android Emulator screen", | |
| 58 | ); | |
| 59 | frame.tabIndex = 0; | |
| 60 | Object.assign(frame.style, { | |
| 61 | position: "relative", | |
| 62 | touchAction: "none", | |
| 63 | userSelect: "none", | |
| 64 | webkitUserSelect: "none", | |
| 65 | webkitTouchCallout: "none", | |
| 66 | outline: "none", | |
| 67 | }); | |
| 68 | const canvas = document.createElement("canvas"); | |
| 69 | const image = document.createElement("img"); | |
| 70 | image.alt = ""; | |
| 71 | image.draggable = false; | |
| 72 | image.style.display = "none"; | |
| 73 | const overlay = document.createElement("div"); | |
| 74 | overlay.setAttribute("role", "status"); | |
| 75 | Object.assign(overlay.style, { | |
| 76 | position: "fixed", | |
| 77 | inset: "0", | |
| 78 | display: "flex", | |
| 79 | flexDirection: "column", | |
| 80 | alignItems: "center", | |
| 81 | justifyContent: "center", | |
| 82 | gap: "16px", | |
| 83 | padding: "24px", | |
| 84 | textAlign: "center", | |
| 85 | background: colors.background, | |
| 86 | }); | |
| 87 | const detail = document.createElement("span"); | |
| 88 | const retry = document.createElement("button"); | |
| 89 | retry.textContent = "Retry"; | |
| 90 | Object.assign(retry.style, { | |
| 91 | padding: "12px 24px", | |
| 92 | borderRadius: "20px", | |
| 93 | border: "0", | |
| 94 | background: colors.buttonBackground, | |
| 95 | color: colors.foreground, | |
| 96 | font: "inherit", | |
| 97 | display: "none", | |
| 98 | }); | |
| 99 | retry.addEventListener("click", () => post({ type: "retry" })); | |
| 100 | overlay.append(detail, retry); | |
| 101 | const inputStatus = document.createElement("div"); | |
| 102 | inputStatus.setAttribute("role", "status"); | |
| 103 | inputStatus.textContent = "Reconnecting device controls..."; | |
| 104 | Object.assign(inputStatus.style, { | |
| 105 | position: "fixed", | |
| 106 | bottom: "12px", | |
| 107 | left: "0", | |
| 108 | right: "0", | |
| 109 | textAlign: "center", | |
| 110 | pointerEvents: "none", | |
| 111 | fontSize: "13px", | |
| 112 | color: colors.muted, | |
| 113 | background: colors.background, | |
| 114 | }); | |
| 115 | frame.append(canvas, image); | |
| 116 | container.append(frame); | |
| 117 | document.body.replaceChildren(container, overlay, inputStatus); | |
| 118 | ||
| 119 | let pointerId: number | null = null; | |
| 120 | let inputConnected = false; | |
| 121 | let streaming = false; | |
| 122 | const layout = (screen: DeviceScreenSize | null) => { | |
| 123 | const landscape = | |
| 124 | screen?.orientation === "landscape_left" || screen?.orientation === "landscape_right"; | |
| 125 | const aspect = screen | |
| 126 | ? landscape | |
| 127 | ? Math.max(screen.width, screen.height) / Math.min(screen.width, screen.height) | |
| 128 | : Math.min(screen.width, screen.height) / Math.max(screen.width, screen.height) | |
| 129 | : 9 / 19.5; | |
| 130 | const rotation = | |
| 131 | platform === "ios" && screen && screen.width <= screen.height | |
| 132 | ? screen.orientation === "landscape_left" | |
| 133 | ? 90 | |
| 134 | : screen.orientation === "landscape_right" | |
| 135 | ? -90 | |
| 136 | : screen.orientation === "portrait_upside_down" | |
| 137 | ? 180 | |
| 138 | : 0 | |
| 139 | : 0; | |
| 140 | const sideways = Math.abs(rotation) === 90; | |
| 141 | frame.style.width = `min(100cqw, ${aspect * 100}cqh)`; | |
| 142 | frame.style.height = `min(100cqh, ${100 / aspect}cqw)`; | |
| 143 | for (const media of [canvas, image]) { | |
| 144 | Object.assign(media.style, { | |
| 145 | position: "absolute", | |
| 146 | width: sideways ? `${100 / aspect}%` : "100%", | |
| 147 | height: sideways ? `${100 * aspect}%` : "100%", | |
| 148 | left: "50%", | |
| 149 | top: "50%", | |
| 150 | transform: `translate(-50%, -50%) rotate(${rotation}deg)`, | |
| 151 | pointerEvents: "none", | |
| 152 | }); | |
| 153 | } | |
| 154 | }; | |
| 155 | layout(null); | |
| 156 | const unauthorized = () => { | |
| 157 | if (activeClient === client) post({ type: "unauthorized" }); | |
| 158 | }; | |
| 159 | const client = createDeviceStreamClient( | |
| 160 | { ...configuration, preferMjpeg: platform === "ios" }, | |
| 161 | canvas, | |
| 162 | { | |
| 163 | onStatus: (status, message) => { | |
| 164 | streaming = status === "streaming"; | |
| 165 | overlay.style.display = streaming ? "none" : "flex"; | |
| 166 | inputStatus.style.display = streaming && !inputConnected ? "block" : "none"; | |
| 167 | detail.textContent = | |
| 168 | status === "error" ? (message ?? "Device stream failed.") : "Connecting to device..."; | |
| 169 | retry.style.display = status === "error" ? "block" : "none"; | |
| 170 | }, | |
| 171 | onScreen: layout, | |
| 172 | onMjpegFallback: (url) => { | |
| 173 | canvas.style.display = "none"; | |
| 174 | image.style.display = "block"; | |
| 175 | image.src = url; | |
| 176 | }, | |
| 177 | onUnauthorized: unauthorized, | |
| 178 | onInputConnected: (connected) => { | |
| 179 | inputConnected = connected; | |
| 180 | inputStatus.style.display = streaming && !connected ? "block" : "none"; | |
| 181 | post({ type: "input", connected }); | |
| 182 | }, | |
| 183 | }, | |
| 184 | ); | |
| 185 | activeClient = client; | |
| 186 | activeImage = image; | |
| 187 | image.addEventListener("error", unauthorized); | |
| 188 | const touch = (event: PointerEvent, phase: "begin" | "move" | "end") => { | |
| 189 | const rect = frame.getBoundingClientRect(); | |
| 190 | client.sendTouch( | |
| 191 | phase, | |
| 192 | Math.max(0, Math.min(1, (event.clientX - rect.left) / rect.width)), | |
| 193 | Math.max(0, Math.min(1, (event.clientY - rect.top) / rect.height)), | |
| 194 | ); | |
| 195 | }; | |
| 196 | frame.addEventListener("pointerdown", (event) => { | |
| 197 | if (!inputConnected || pointerId !== null) return; | |
| 198 | event.preventDefault(); | |
| 199 | pointerId = event.pointerId; | |
| 200 | frame.setPointerCapture(event.pointerId); | |
| 201 | frame.focus(); | |
| 202 | touch(event, "begin"); | |
| 203 | }); | |
| 204 | frame.addEventListener("pointermove", (event) => { | |
| 205 | if (pointerId === event.pointerId) touch(event, "move"); | |
| 206 | }); | |
| 207 | const endTouch = (event: PointerEvent) => { | |
| 208 | if (pointerId !== event.pointerId) return; | |
| 209 | pointerId = null; | |
| 210 | touch(event, "end"); | |
| 211 | }; | |
| 212 | frame.addEventListener("pointerup", endTouch); | |
| 213 | frame.addEventListener("pointercancel", endTouch); | |
| 214 | frame.addEventListener("lostpointercapture", endTouch); | |
| 215 | frame.addEventListener("keydown", (event) => { | |
| 216 | event.preventDefault(); | |
| 217 | client.sendKey(event, "down"); | |
| 218 | }); | |
| 219 | frame.addEventListener("keyup", (event) => client.sendKey(event, "up")); | |
| 220 | window.addEventListener("pagehide", stop, { once: true }); | |
| 221 | post({ type: "input", connected: false }); | |
| 222 | client.start(); | |
| 223 | } |
apps/mobile/src/features/devices/threadDevicePreviews.test.ts created+106
| ... | ... | @@ -0,0 +1,106 @@ |
| 1 | import { ThreadId, type DeviceServiceState } from "@t3tools/contracts"; | |
| 2 | import { describe, expect, it } from "vite-plus/test"; | |
| 3 | ||
| 4 | import { selectedThreadDevicePreview, threadDevicePreviews } from "./threadDevicePreviews"; | |
| 5 | ||
| 6 | const threadId = ThreadId.make("thread"); | |
| 7 | const state: DeviceServiceState = { | |
| 8 | hosts: [ | |
| 9 | { | |
| 10 | id: "local", | |
| 11 | kind: "local", | |
| 12 | label: "Local", | |
| 13 | platforms: [], | |
| 14 | hubInstalled: true, | |
| 15 | agentDeviceInstalled: true, | |
| 16 | }, | |
| 17 | { | |
| 18 | id: "remote", | |
| 19 | kind: "ssh", | |
| 20 | label: "Mac mini", | |
| 21 | platforms: [], | |
| 22 | hubInstalled: true, | |
| 23 | agentDeviceInstalled: true, | |
| 24 | }, | |
| 25 | ], | |
| 26 | hostStatus: "ready", | |
| 27 | hostStatuses: {}, | |
| 28 | devices: [ | |
| 29 | { | |
| 30 | hostId: "local", | |
| 31 | id: "emulator-5554", | |
| 32 | platform: "android", | |
| 33 | name: "Pixel 9", | |
| 34 | version: "Android 16", | |
| 35 | booted: true, | |
| 36 | physical: false, | |
| 37 | }, | |
| 38 | { | |
| 39 | hostId: "remote", | |
| 40 | id: "emulator-5554", | |
| 41 | platform: "android", | |
| 42 | name: "Pixel 8", | |
| 43 | version: "Android 15", | |
| 44 | booted: true, | |
| 45 | physical: false, | |
| 46 | }, | |
| 47 | ], | |
| 48 | sessions: [ | |
| 49 | { | |
| 50 | threadId, | |
| 51 | hostId: "local", | |
| 52 | deviceId: "emulator-5554", | |
| 53 | platform: "android", | |
| 54 | openedAt: "2026-09-18T00:00:00Z", | |
| 55 | }, | |
| 56 | { | |
| 57 | threadId, | |
| 58 | hostId: "remote", | |
| 59 | deviceId: "emulator-5554", | |
| 60 | platform: "android", | |
| 61 | openedAt: "2026-09-18T00:01:00Z", | |
| 62 | }, | |
| 63 | { | |
| 64 | threadId: ThreadId.make("another-thread"), | |
| 65 | hostId: "local", | |
| 66 | deviceId: "iphone", | |
| 67 | platform: "ios", | |
| 68 | openedAt: "2026-09-18T00:02:00Z", | |
| 69 | }, | |
| 70 | ], | |
| 71 | onboardingCompleted: true, | |
| 72 | agentAccessEnabled: true, | |
| 73 | hubBasePath: "/api/device-hub", | |
| 74 | revision: 1, | |
| 75 | }; | |
| 76 | ||
| 77 | describe("thread device previews", () => { | |
| 78 | it("keeps device names and selection distinct when hosts have the same Android serial", () => { | |
| 79 | const previews = threadDevicePreviews(state, threadId); | |
| 80 | expect(previews.map((preview) => preview.name)).toEqual(["Pixel 9", "Pixel 8"]); | |
| 81 | expect(new Set(previews.map((preview) => preview.key)).size).toBe(2); | |
| 82 | expect(selectedThreadDevicePreview(previews, previews[1]!.key)?.description).toBe( | |
| 83 | "Android 15 · Mac mini", | |
| 84 | ); | |
| 85 | }); | |
| 86 | ||
| 87 | it("selects another open device when the agent closes the selected session", () => { | |
| 88 | const previews = threadDevicePreviews(state, threadId); | |
| 89 | const selectedKey = previews[1]!.key; | |
| 90 | const afterClose = threadDevicePreviews({ ...state, sessions: [state.sessions[0]!] }, threadId); | |
| 91 | expect(selectedThreadDevicePreview(afterClose, selectedKey)?.name).toBe("Pixel 9"); | |
| 92 | expect(selectedThreadDevicePreview([], selectedKey)).toBeNull(); | |
| 93 | }); | |
| 94 | ||
| 95 | it("can view an open session before device discovery metadata arrives", () => { | |
| 96 | const previews = threadDevicePreviews({ ...state, hosts: [], devices: [] }, threadId); | |
| 97 | expect(previews[0]?.name).toBe("Android Emulator"); | |
| 98 | expect(previews[0]?.session.deviceId).toBe("emulator-5554"); | |
| 99 | expect(previews[0]?.description).toBe(""); | |
| 100 | }); | |
| 101 | ||
| 102 | it("offers no devices before state arrives or in a thread without sessions", () => { | |
| 103 | expect(threadDevicePreviews(null, threadId)).toEqual([]); | |
| 104 | expect(threadDevicePreviews(state, ThreadId.make("empty-thread"))).toEqual([]); | |
| 105 | }); | |
| 106 | }); |
apps/mobile/src/features/devices/threadDevicePreviews.ts created+28
| ... | ... | @@ -0,0 +1,28 @@ |
| 1 | import type { DeviceServiceState, ThreadId } from "@t3tools/contracts"; | |
| 2 | ||
| 3 | /** Host identity is part of the selection because Android serials repeat across hosts. */ | |
| 4 | export function threadDevicePreviews(state: DeviceServiceState | null, threadId: ThreadId) { | |
| 5 | return (state?.sessions ?? []) | |
| 6 | .filter((session) => session.threadId === threadId) | |
| 7 | .map((session) => { | |
| 8 | const device = state?.devices.find( | |
| 9 | (device) => device.hostId === session.hostId && device.id === session.deviceId, | |
| 10 | ); | |
| 11 | const host = state?.hosts.find((host) => host.id === session.hostId); | |
| 12 | return { | |
| 13 | key: JSON.stringify([session.hostId, session.deviceId]), | |
| 14 | session, | |
| 15 | name: device?.name ?? (session.platform === "ios" ? "iOS Simulator" : "Android Emulator"), | |
| 16 | description: [device?.version, host?.label].filter(Boolean).join(" · "), | |
| 17 | }; | |
| 18 | }); | |
| 19 | } | |
| 20 | ||
| 21 | export type ThreadDevicePreview = ReturnType<typeof threadDevicePreviews>[number]; | |
| 22 | ||
| 23 | export function selectedThreadDevicePreview( | |
| 24 | previews: ReadonlyArray<ThreadDevicePreview>, | |
| 25 | selectedKey: string | null, | |
| 26 | ) { | |
| 27 | return previews.find((preview) => preview.key === selectedKey) ?? previews[0] ?? null; | |
| 28 | } |
apps/mobile/src/features/threads/ThreadDetailScreen.tsx+25
| ... | ... | @@ -13,6 +13,7 @@ import { useKeyboardChatComposerInset, useKeyboardScrollToEnd } from "@legendapp |
| 13 | 13 | import { resolveProviderSkillsForCwd } from "@t3tools/client-runtime/providerSkills"; |
| 14 | 14 | import type { LegendListRef } from "@legendapp/list/react-native"; |
| 15 | 15 | import { HeaderHeightContext } from "@react-navigation/elements"; |
| 16 | import { useNavigation } from "@react-navigation/native"; | |
| 16 | 17 | import type { |
| 17 | 18 | ApprovalRequestId, |
| 18 | 19 | EnvironmentId, |
| ... | ... | @@ -72,6 +73,9 @@ import type { DraftComposerAttachment } from "../../lib/composerImages"; |
| 72 | 73 | import { CHAT_CONTENT_MAX_WIDTH, type LayoutVariant } from "../../lib/layout"; |
| 73 | 74 | import { IOS_NAV_BAR_HEIGHT } from "../../lib/layoutMetrics"; |
| 74 | 75 | import { editPendingThreadMessage } from "../../state/edit-pending-thread-message"; |
| 76 | import { deviceEnvironment } from "../../state/device"; | |
| 77 | import { useEnvironmentQuery } from "../../state/query"; | |
| 78 | import { threadDevicePreviews } from "../devices/threadDevicePreviews"; | |
| 75 | 79 | import type { QueuedThreadMessage } from "../../state/thread-outbox-model"; |
| 76 | 80 | import { scopedThreadKey } from "../../lib/scopedEntities"; |
| 77 | 81 | import type { |
| ... | ... | @@ -259,6 +263,21 @@ const USER_INPUT_TOGGLE_TIMING = { |
| 259 | 263 | }; |
| 260 | 264 | |
| 261 | 265 | export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: ThreadDetailScreenProps) { |
| 266 | const navigation = useNavigation(); | |
| 267 | const deviceState = useEnvironmentQuery( | |
| 268 | deviceEnvironment.state({ environmentId: props.environmentId, input: {} }), | |
| 269 | ); | |
| 270 | const devicePreviews = useMemo( | |
| 271 | () => threadDevicePreviews(deviceState.data, props.selectedThread.id), | |
| 272 | [deviceState.data, props.selectedThread.id], | |
| 273 | ); | |
| 274 | const openDevicePreview = useCallback(() => { | |
| 275 | Keyboard.dismiss(); | |
| 276 | navigation.navigate("ThreadDevicePreview", { | |
| 277 | environmentId: props.environmentId, | |
| 278 | threadId: props.selectedThread.id, | |
| 279 | }); | |
| 280 | }, [navigation, props.environmentId, props.selectedThread.id]); | |
| 262 | 281 | const insets = useSafeAreaInsets(); |
| 263 | 282 | const isKeyboardVisible = useKeyboardState((state) => state.isVisible); |
| 264 | 283 | const liveKeyboardHeight = useKeyboardState((state) => state.height); |
| ... | ... | @@ -390,6 +409,7 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread |
| 390 | 409 | // stable when reconnecting hands off to syncing and then to a running turn. |
| 391 | 410 | const showFloatingStatus = |
| 392 | 411 | showWorkingControl || |
| 412 | devicePreviews.length > 0 || | |
| 393 | 413 | props.connectionStateLabel !== "connected" || |
| 394 | 414 | props.queuedMessages.length > 0 || |
| 395 | 415 | props.selectedThreadFeed.some( |
| ... | ... | @@ -936,6 +956,11 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread |
| 936 | 956 | <FloatingWorkingControl |
| 937 | 957 | colorScheme={isDarkMode ? "dark" : "light"} |
| 938 | 958 | status={floatingStatus} |
| 959 | devicePreview={ | |
| 960 | devicePreviews.length > 0 | |
| 961 | ? { count: devicePreviews.length, onPress: openDevicePreview } | |
| 962 | : null | |
| 963 | } | |
| 939 | 964 | showScrollToEnd={showScrollToEndButton} |
| 940 | 965 | onScrollToEnd={handleScrollToEnd} |
| 941 | 966 | /> |
apps/mobile/src/features/threads/floating-working-control.tsx+38-14
| ... | ... | @@ -24,6 +24,7 @@ import { AppText as Text } from "../../components/AppText"; |
| 24 | 24 | import { SymbolView } from "../../components/AppSymbol"; |
| 25 | 25 | import { ControlPill } from "../../components/ControlPill"; |
| 26 | 26 | import { NATIVE_LIQUID_GLASS_SUPPORTED } from "../../native/native-glass"; |
| 27 | import { DevicePreviewButton } from "../devices/device-preview-button"; | |
| 27 | 28 | import type { FloatingWorkingStatus } from "./floating-working-status"; |
| 28 | 29 | import { ShimmeringWorkContent } from "./thread-work-log"; |
| 29 | 30 | |
| ... | ... | @@ -63,12 +64,18 @@ export const FLOATING_WORKING_CONTROL_COVERAGE = CONTROL_OVERLAY_OFFSET + CONTRO |
| 63 | 64 | export function FloatingWorkingControl(props: { |
| 64 | 65 | readonly colorScheme: "light" | "dark"; |
| 65 | 66 | readonly status: FloatingWorkingStatus | null; |
| 67 | readonly devicePreview: { readonly count: number; readonly onPress: () => void } | null; | |
| 66 | 68 | readonly showScrollToEnd: boolean; |
| 67 | 69 | readonly onScrollToEnd: () => void; |
| 68 | 70 | }) { |
| 69 | 71 | const { width: windowWidth } = useWindowDimensions(); |
| 70 | 72 | const [overlayWidth, setOverlayWidth] = useState(windowWidth); |
| 71 | const labelWidth = Math.max(0, Math.min(overlayWidth, windowWidth) - CONTROL_HEIGHT - 16); | |
| 73 | const deviceControlWidth = props.devicePreview !== null ? CONTROL_HEIGHT : 0; | |
| 74 | const hasCapsule = props.status !== null || props.devicePreview !== null; | |
| 75 | const labelWidth = Math.max( | |
| 76 | 0, | |
| 77 | Math.min(overlayWidth, windowWidth) - CONTROL_HEIGHT - 16 - deviceControlWidth, | |
| 78 | ); | |
| 72 | 79 | const separationProgress = useSharedValue(props.showScrollToEnd ? 1 : 0); |
| 73 | 80 | |
| 74 | 81 | useEffect(() => { |
| ... | ... | @@ -111,25 +118,30 @@ export function FloatingWorkingControl(props: { |
| 111 | 118 | })); |
| 112 | 119 | // Zero until the first measurement lands, so the capsule never paints around |
| 113 | 120 | // a label it has not sized to yet. |
| 114 | const capsuleSizerStyle = useAnimatedStyle(() => ({ width: capsuleWidth.value ?? 0 })); | |
| 121 | const capsuleSizerStyle = useAnimatedStyle(() => ({ | |
| 122 | width: (capsuleWidth.value ?? 0) + deviceControlWidth, | |
| 123 | })); | |
| 115 | 124 | |
| 116 | if (props.status === null && !props.showScrollToEnd) { | |
| 125 | if (!hasCapsule && !props.showScrollToEnd) { | |
| 117 | 126 | return null; |
| 118 | 127 | } |
| 119 | 128 | |
| 120 | 129 | // Only the connection label is a button (tap to reconnect); the others |
| 121 | 130 | // pass touches through to the feed like before. |
| 122 | 131 | const statusInteractive = props.status?.kind === "connection"; |
| 132 | const capsuleInteractive = statusInteractive || props.devicePreview !== null; | |
| 123 | 133 | // The host stays centered on the capsule, but its measurement constraint |
| 124 | 134 | // comes from the overlay, independent of the capsule's current width. |
| 125 | const statusContent = | |
| 126 | props.status !== null ? ( | |
| 135 | const capsuleContent = | |
| 136 | props.status === null && props.devicePreview !== null ? ( | |
| 137 | <DevicePreviewButton {...props.devicePreview} compact={false} /> | |
| 138 | ) : props.status !== null ? ( | |
| 127 | 139 | <> |
| 128 | 140 | <Animated.View className="h-11" style={capsuleSizerStyle} /> |
| 129 | 141 | <View |
| 130 | pointerEvents="box-none" | |
| 142 | pointerEvents={statusInteractive ? "box-none" : "none"} | |
| 131 | 143 | className="absolute h-11 items-center justify-center" |
| 132 | style={{ width: labelWidth }} | |
| 144 | style={{ width: labelWidth, transform: [{ translateX: -deviceControlWidth / 2 }] }} | |
| 133 | 145 | > |
| 134 | 146 | <FloatingStatusLabel |
| 135 | 147 | key={ |
| ... | ... | @@ -141,6 +153,18 @@ export function FloatingWorkingControl(props: { |
| 141 | 153 | onLayout={handleLabelLayout} |
| 142 | 154 | /> |
| 143 | 155 | </View> |
| 156 | {props.devicePreview !== null ? ( | |
| 157 | <> | |
| 158 | <View | |
| 159 | pointerEvents="none" | |
| 160 | className="absolute h-4 w-px bg-border" | |
| 161 | style={{ right: deviceControlWidth }} | |
| 162 | /> | |
| 163 | <View className="absolute right-0"> | |
| 164 | <DevicePreviewButton {...props.devicePreview} /> | |
| 165 | </View> | |
| 166 | </> | |
| 167 | ) : null} | |
| 144 | 168 | </> |
| 145 | 169 | ) : null; |
| 146 | 170 | |
| ... | ... | @@ -153,7 +177,7 @@ export function FloatingWorkingControl(props: { |
| 153 | 177 | entering={NATIVE_LIQUID_GLASS_SUPPORTED ? undefined : CONTROL_ENTERING} |
| 154 | 178 | exiting={NATIVE_LIQUID_GLASS_SUPPORTED ? undefined : CONTROL_EXITING} |
| 155 | 179 | > |
| 156 | {props.status !== null && NATIVE_LIQUID_GLASS_SUPPORTED ? ( | |
| 180 | {hasCapsule && NATIVE_LIQUID_GLASS_SUPPORTED ? ( | |
| 157 | 181 | <UniwindGlassContainer |
| 158 | 182 | spacing={GLASS_MERGE_SPACING} |
| 159 | 183 | pointerEvents="box-none" |
| ... | ... | @@ -162,12 +186,12 @@ export function FloatingWorkingControl(props: { |
| 162 | 186 | <AnimatedGlassView |
| 163 | 187 | colorScheme={props.colorScheme} |
| 164 | 188 | glassEffectStyle="regular" |
| 165 | isInteractive={statusInteractive} | |
| 166 | pointerEvents={statusInteractive ? "box-none" : "none"} | |
| 189 | isInteractive={capsuleInteractive} | |
| 190 | pointerEvents={capsuleInteractive ? "box-none" : "none"} | |
| 167 | 191 | className="h-11 items-center justify-center overflow-hidden rounded-full" |
| 168 | 192 | style={capsuleStyle} |
| 169 | 193 | > |
| 170 | {statusContent} | |
| 194 | {capsuleContent} | |
| 171 | 195 | </AnimatedGlassView> |
| 172 | 196 | |
| 173 | 197 | <AnimatedGlassView |
| ... | ... | @@ -185,14 +209,14 @@ export function FloatingWorkingControl(props: { |
| 185 | 209 | </Animated.View> |
| 186 | 210 | </AnimatedGlassView> |
| 187 | 211 | </UniwindGlassContainer> |
| 188 | ) : props.status !== null ? ( | |
| 212 | ) : hasCapsule ? ( | |
| 189 | 213 | <View pointerEvents="box-none" className="flex-row items-center gap-4"> |
| 190 | 214 | <Animated.View |
| 191 | pointerEvents={statusInteractive ? "box-none" : "none"} | |
| 215 | pointerEvents={capsuleInteractive ? "box-none" : "none"} | |
| 192 | 216 | className="h-11 items-center justify-center overflow-hidden rounded-full border border-border bg-card shadow-md shadow-black/10" |
| 193 | 217 | style={capsuleStyle} |
| 194 | 218 | > |
| 195 | {statusContent} | |
| 219 | {capsuleContent} | |
| 196 | 220 | </Animated.View> |
| 197 | 221 | |
| 198 | 222 | <Animated.View |
apps/mobile/src/state/device.ts created+44
| ... | ... | @@ -0,0 +1,44 @@ |
| 1 | import { createDeviceEnvironmentAtoms } from "@t3tools/client-runtime/state/device"; | |
| 2 | import { resolveDeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess"; | |
| 3 | import type { EnvironmentId } from "@t3tools/contracts"; | |
| 4 | import * as Effect from "effect/Effect"; | |
| 5 | import * as Option from "effect/Option"; | |
| 6 | import { Atom } from "effect/unstable/reactivity"; | |
| 7 | import { useMemo } from "react"; | |
| 8 | ||
| 9 | import { connectionAtomRuntime } from "../connection/runtime"; | |
| 10 | import { appAtomRegistry } from "./atom-registry"; | |
| 11 | import { useEnvironmentQuery } from "./query"; | |
| 12 | import { environmentSession, usePreparedConnection } from "./session"; | |
| 13 | ||
| 14 | export const deviceEnvironment = createDeviceEnvironmentAtoms(connectionAtomRuntime); | |
| 15 | ||
| 16 | const deviceHubAccessAtom = Atom.family((environmentId: EnvironmentId) => | |
| 17 | connectionAtomRuntime | |
| 18 | .atom((get) => { | |
| 19 | const prepared = Option.getOrNull( | |
| 20 | get(environmentSession.preparedConnectionValueAtom(environmentId)), | |
| 21 | ); | |
| 22 | return prepared === null | |
| 23 | ? Effect.never | |
| 24 | : resolveDeviceHubAccess({ prepared, hubBasePath: "/api/device-hub" }); | |
| 25 | }) | |
| 26 | .pipe(Atom.setIdleTTL(60_000), Atom.withLabel(`mobile-device-hub-access:${environmentId}`)), | |
| 27 | ); | |
| 28 | ||
| 29 | export function refreshDeviceHubAccess(environmentId: EnvironmentId) { | |
| 30 | appAtomRegistry.refresh(deviceHubAccessAtom(environmentId)); | |
| 31 | } | |
| 32 | ||
| 33 | export function useDeviceHubAccess(environmentId: EnvironmentId, hostId: string) { | |
| 34 | const prepared = usePreparedConnection(environmentId); | |
| 35 | const query = useEnvironmentQuery(deviceHubAccessAtom(environmentId)); | |
| 36 | const access = useMemo( | |
| 37 | () => | |
| 38 | query.data && query.error === null && Option.isSome(prepared) | |
| 39 | ? { ...query.data, query: { ...query.data.query, hostId } } | |
| 40 | : null, | |
| 41 | [query.data, query.error, prepared, hostId], | |
| 42 | ); | |
| 43 | return { access, error: query.error, refresh: query.refresh }; | |
| 44 | } |
apps/mobile/src/types/mobile-device-stream.d.ts created+4
| ... | ... | @@ -0,0 +1,4 @@ |
| 1 | declare module "@t3tools/mobile-device-stream" { | |
| 2 | const script: string; | |
| 3 | export default script; | |
| 4 | } |
apps/web/src/components/device/DeviceStreamView.test.tsx+1-1
| ... | ... | @@ -8,7 +8,7 @@ vi.mock("~/state/device", () => ({ |
| 8 | 8 | refreshDeviceHubAccess: vi.fn(), |
| 9 | 9 | })); |
| 10 | 10 | const access = { httpBase: "http://test", wsBase: "ws://test", query: {}, credentials: true }; |
| 11 | vi.mock("./deviceStream", () => ({ | |
| 11 | vi.mock("@t3tools/client-runtime/device/stream", () => ({ | |
| 12 | 12 | createDeviceStreamClient: ( |
| 13 | 13 | _target: unknown, |
| 14 | 14 | _canvas: unknown, |
apps/web/src/components/device/DeviceStreamView.tsx+1-1
| ... | ... | @@ -11,7 +11,7 @@ import { |
| 11 | 11 | type DeviceScreenSize, |
| 12 | 12 | type DeviceStreamClient, |
| 13 | 13 | type DeviceStreamStatus, |
| 14 | } from "./deviceStream"; | |
| 14 | } from "@t3tools/client-runtime/device/stream"; | |
| 15 | 15 | |
| 16 | 16 | const AX_POLL_INTERVAL_MS = 2_000; |
| 17 | 17 |
apps/web/src/components/device/deviceStream.test.ts+166-1
| ... | ... | @@ -6,7 +6,7 @@ import { |
| 6 | 6 | avcCodecString, |
| 7 | 7 | parseSemuPacket, |
| 8 | 8 | scanAccessUnit, |
| 9 | } from "./deviceStream"; | |
| 9 | } from "@t3tools/client-runtime/device/stream"; | |
| 10 | 10 | |
| 11 | 11 | const envelope = (tag: number, payload: number[]) => { |
| 12 | 12 | const length = 1 + payload.length; |
| ... | ... | @@ -43,6 +43,171 @@ describe("AvccDemuxer", () => { |
| 43 | 43 | }); |
| 44 | 44 | }); |
| 45 | 45 | |
| 46 | describe("native device stream transport", () => { | |
| 47 | afterEach(() => { | |
| 48 | vi.useRealTimers(); | |
| 49 | vi.unstubAllGlobals(); | |
| 50 | }); | |
| 51 | ||
| 52 | function setup(platform: "ios" | "android") { | |
| 53 | vi.useFakeTimers(); | |
| 54 | vi.stubGlobal("VideoDecoder", vi.fn()); | |
| 55 | vi.stubGlobal("EncodedVideoChunk", vi.fn()); | |
| 56 | let resolveOpened!: (socket: FakeSocket) => void; | |
| 57 | const waitForSocket = () => | |
| 58 | new Promise<FakeSocket>((resolve) => { | |
| 59 | resolveOpened = resolve; | |
| 60 | }); | |
| 61 | const opened = waitForSocket(); | |
| 62 | class FakeSocket { | |
| 63 | static OPEN = 1; | |
| 64 | readyState = 1; | |
| 65 | binaryType = ""; | |
| 66 | onopen: (() => void) | null = null; | |
| 67 | onclose: ((event: { code: number; reason: string }) => void) | null = null; | |
| 68 | send = vi.fn(); | |
| 69 | close = vi.fn(); | |
| 70 | readonly url: string; | |
| 71 | constructor(url: string) { | |
| 72 | this.url = url; | |
| 73 | resolveOpened(this); | |
| 74 | } | |
| 75 | } | |
| 76 | vi.stubGlobal("WebSocket", FakeSocket); | |
| 77 | const fetch = vi.fn(() => Promise.resolve(new Response("frame"))); | |
| 78 | vi.stubGlobal("fetch", fetch); | |
| 79 | const events = { | |
| 80 | onStatus: vi.fn(), | |
| 81 | onScreen: vi.fn(), | |
| 82 | onUnauthorized: vi.fn(), | |
| 83 | onMjpegFallback: vi.fn(), | |
| 84 | onInputConnected: vi.fn(), | |
| 85 | }; | |
| 86 | const client = createDeviceStreamClient( | |
| 87 | { | |
| 88 | platform, | |
| 89 | deviceId: "test device", | |
| 90 | preferMjpeg: platform === "ios", | |
| 91 | access: { | |
| 92 | httpBase: "https://environment.test/api/device-hub", | |
| 93 | wsBase: "wss://environment.test/api/device-hub", | |
| 94 | credentials: false, | |
| 95 | query: { wsTicket: "stream-ticket", hostId: "ssh-host" }, | |
| 96 | }, | |
| 97 | }, | |
| 98 | { getContext: () => null } as unknown as HTMLCanvasElement, | |
| 99 | events, | |
| 100 | ); | |
| 101 | return { client, opened, waitForSocket, events, fetch }; | |
| 102 | } | |
| 103 | ||
| 104 | it("uses authenticated iOS MJPEG and forwards controls without a cross-origin video fetch", async () => { | |
| 105 | const { client, opened, events, fetch } = setup("ios"); | |
| 106 | client.start(); | |
| 107 | const socket = await opened; | |
| 108 | const imageUrl = new URL(events.onMjpegFallback.mock.calls[0]![0] as string); | |
| 109 | expect(imageUrl.pathname).toContain("test%20device/stream.mjpeg"); | |
| 110 | expect(imageUrl.searchParams.get("wsTicket")).toBe("stream-ticket"); | |
| 111 | expect(imageUrl.searchParams.get("hostId")).toBe("ssh-host"); | |
| 112 | expect(fetch).toHaveBeenCalledTimes(1); | |
| 113 | socket.onopen?.(); | |
| 114 | client.pressButton("home"); | |
| 115 | client.sendTouch("begin", 0.25, 0.75); | |
| 116 | const messages = socket.send.mock.calls.slice(1).map(([bytes]) => { | |
| 117 | const packet = bytes as Uint8Array; | |
| 118 | return { | |
| 119 | tag: packet[0], | |
| 120 | body: JSON.parse(new TextDecoder().decode(packet.subarray(1))) as unknown, | |
| 121 | }; | |
| 122 | }); | |
| 123 | expect(messages).toEqual([ | |
| 124 | { tag: 0x04, body: { button: "home" } }, | |
| 125 | { tag: 0x03, body: { type: "begin", x: 0.25, y: 0.75 } }, | |
| 126 | ]); | |
| 127 | client.stop(); | |
| 128 | expect(socket.close).toHaveBeenCalledTimes(1); | |
| 129 | expect(vi.getTimerCount()).toBe(0); | |
| 130 | }); | |
| 131 | ||
| 132 | it("forwards Android gestures and device controls through the ticketed host socket", async () => { | |
| 133 | const { client, opened } = setup("android"); | |
| 134 | client.start(); | |
| 135 | const socket = await opened; | |
| 136 | const url = new URL(socket.url); | |
| 137 | expect(url.searchParams.get("device")).toBe("test device"); | |
| 138 | expect(url.searchParams.get("wsTicket")).toBe("stream-ticket"); | |
| 139 | expect(url.searchParams.get("hostId")).toBe("ssh-host"); | |
| 140 | client.sendTouch("begin", 0.2, 0.8); | |
| 141 | client.sendTouch("move", 0.2, 0.4); | |
| 142 | client.sendTouch("end", 0.2, 0.1); | |
| 143 | client.pressButton("home"); | |
| 144 | client.pressButton("appSwitcher"); | |
| 145 | expect( | |
| 146 | socket.send.mock.calls.map(([message]) => JSON.parse(message as string) as unknown), | |
| 147 | ).toEqual([ | |
| 148 | { type: "touch", action: "down", x: 0.2, y: 0.8 }, | |
| 149 | { type: "touch", action: "move", x: 0.2, y: 0.4 }, | |
| 150 | { type: "touch", action: "up", x: 0.2, y: 0.1 }, | |
| 151 | { type: "home" }, | |
| 152 | { type: "recents" }, | |
| 153 | ]); | |
| 154 | client.stop(); | |
| 155 | expect(socket.close).toHaveBeenCalledTimes(1); | |
| 156 | }); | |
| 157 | ||
| 158 | it("renews an expired Android ticket when the HTTP upgrade is rejected instead of retrying it forever", async () => { | |
| 159 | const { client, opened, events } = setup("android"); | |
| 160 | client.start(); | |
| 161 | const socket = await opened; | |
| 162 | socket.onclose?.({ code: 1006, reason: "" }); | |
| 163 | expect(events.onUnauthorized).toHaveBeenCalledTimes(1); | |
| 164 | expect(vi.getTimerCount()).toBe(0); | |
| 165 | client.stop(); | |
| 166 | }); | |
| 167 | ||
| 168 | it.each(["ios", "android"] as const)( | |
| 169 | "keeps a restarted %s stream connected when the discarded socket closes late", | |
| 170 | async (platform) => { | |
| 171 | const { client, opened, waitForSocket, events } = setup(platform); | |
| 172 | client.start(); | |
| 173 | const discarded = await opened; | |
| 174 | client.stop(); | |
| 175 | const reopened = waitForSocket(); | |
| 176 | client.start(); | |
| 177 | const replacement = await reopened; | |
| 178 | replacement.onopen?.(); | |
| 179 | replacement.send.mockClear(); | |
| 180 | events.onInputConnected.mockClear(); | |
| 181 | ||
| 182 | discarded.onclose?.({ code: 1006, reason: "" }); | |
| 183 | ||
| 184 | expect(events.onUnauthorized).not.toHaveBeenCalled(); | |
| 185 | expect(events.onInputConnected).not.toHaveBeenCalled(); | |
| 186 | expect(replacement.close).not.toHaveBeenCalled(); | |
| 187 | client.pressButton("home"); | |
| 188 | expect(replacement.send).toHaveBeenCalledTimes(1); | |
| 189 | client.stop(); | |
| 190 | expect(replacement.close).toHaveBeenCalledTimes(1); | |
| 191 | expect(vi.getTimerCount()).toBe(0); | |
| 192 | }, | |
| 193 | ); | |
| 194 | ||
| 195 | it.each(["ios", "android"] as const)( | |
| 196 | "does not renew credentials or reconnect a stopped %s stream", | |
| 197 | async (platform) => { | |
| 198 | const { client, opened, events } = setup(platform); | |
| 199 | client.start(); | |
| 200 | const socket = await opened; | |
| 201 | events.onInputConnected.mockClear(); | |
| 202 | client.stop(); | |
| 203 | socket.onclose?.({ code: 1006, reason: "" }); | |
| 204 | expect(events.onUnauthorized).not.toHaveBeenCalled(); | |
| 205 | expect(events.onInputConnected).not.toHaveBeenCalled(); | |
| 206 | expect(vi.getTimerCount()).toBe(0); | |
| 207 | }, | |
| 208 | ); | |
| 209 | }); | |
| 210 | ||
| 46 | 211 | describe("serve-emu frames", () => { |
| 47 | 212 | it("strips the SEMU header and reads the keyframe flag and timestamp", () => { |
| 48 | 213 | const buffer = new ArrayBuffer(16 + 3); |
apps/web/src/components/device/deviceStream.ts deleted-697
| ... | ... | @@ -1,697 +0,0 @@ |
| 1 | /** | |
| 2 | * Framework-free client for expo-device-hub's per-device streams, reached | |
| 3 | * through the T3 proxy. One class handles both platforms because the hub | |
| 4 | * vendors two servers with different wire formats: | |
| 5 | * | |
| 6 | * - iOS (serve-sim): video is an HTTP `stream.avcc` body of length-prefixed | |
| 7 | * envelopes (`u32be length, u8 tag, payload`; tag 1 avcC description, | |
| 8 | * 2 keyframe, 3 delta, 4 JPEG seed) decoded with WebCodecs; input goes over | |
| 9 | * `helper/ws?device=<udid>` as `[tag][json]` packets. When WebCodecs is | |
| 10 | * unavailable (plain-http remote origins) the MJPEG endpoint is used as an | |
| 11 | * `<img>` source instead. | |
| 12 | * - Android (serve-emu): one WebSocket at `ws?device=<serial>&frame-meta=1` | |
| 13 | * carries H.264 access units prefixed with a 16-byte "SEMU" header | |
| 14 | * (magic, version, key flag, pts) and accepts JSON gestures upstream. | |
| 15 | * | |
| 16 | * The decoder only runs while frames arrive and the viewer is attached; a | |
| 17 | * hidden panel calls `stop()` so an idle device costs nothing on the GPU. | |
| 18 | */ | |
| 19 | import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess"; | |
| 20 | import { withDeviceHubQuery } from "@t3tools/client-runtime/state/deviceHubAccess"; | |
| 21 | import type { DevicePlatform } from "@t3tools/contracts"; | |
| 22 | ||
| 23 | export type DeviceStreamStatus = "connecting" | "streaming" | "error"; | |
| 24 | ||
| 25 | export interface DeviceScreenSize { | |
| 26 | readonly width: number; | |
| 27 | readonly height: number; | |
| 28 | readonly orientation: "portrait" | "portrait_upside_down" | "landscape_left" | "landscape_right"; | |
| 29 | } | |
| 30 | ||
| 31 | export interface DeviceStreamEvents { | |
| 32 | readonly onStatus: (status: DeviceStreamStatus, detail?: string) => void; | |
| 33 | readonly onScreen: (screen: DeviceScreenSize) => void; | |
| 34 | /** The proxy rejected the credential; the owner should refresh access and reconnect. */ | |
| 35 | readonly onUnauthorized: () => void; | |
| 36 | /** | |
| 37 | * H.264 cannot be decoded here (no WebCodecs, or the simulator's profile is | |
| 38 | * unsupported); the owner should show this MJPEG URL in an `<img>` instead of | |
| 39 | * the canvas. | |
| 40 | */ | |
| 41 | readonly onMjpegFallback: (url: string) => void; | |
| 42 | /** Whether touches and keys can currently reach the device. */ | |
| 43 | readonly onInputConnected: (connected: boolean, detail?: string) => void; | |
| 44 | } | |
| 45 | ||
| 46 | export interface DeviceStreamTarget { | |
| 47 | readonly platform: DevicePlatform; | |
| 48 | readonly deviceId: string; | |
| 49 | readonly access: DeviceHubAccess; | |
| 50 | } | |
| 51 | ||
| 52 | export type DeviceHardwareButton = "home" | "back" | "recents" | "power" | "appSwitcher"; | |
| 53 | ||
| 54 | const RETRY_DELAY_MS = 1_000; | |
| 55 | const FRAME_DURATION_US = 16_667; | |
| 56 | const SEMU_MAGIC = 0x53454d55; | |
| 57 | const SEMU_HEADER_BYTES = 16; | |
| 58 | const SEMU_FLAG_KEY = 1; | |
| 59 | const SOFT_DECODE_QUEUE = 8; | |
| 60 | ||
| 61 | // serve-sim binary WS message tags (browser -> helper). | |
| 62 | const IOS_MSG_TOUCH = 0x03; | |
| 63 | const IOS_MSG_BUTTON = 0x04; | |
| 64 | const IOS_MSG_KEY = 0x06; | |
| 65 | const IOS_MSG_ORIENTATION = 0x07; | |
| 66 | const IOS_MSG_HARDWARE_KEYBOARD = 0x0d; | |
| 67 | // helper -> browser. | |
| 68 | const IOS_TAG_SCREEN_CONFIG = 0x82; | |
| 69 | ||
| 70 | const encoder = new TextEncoder(); | |
| 71 | const decoder = new TextDecoder(); | |
| 72 | ||
| 73 | const isWebCodecsSupported = (): boolean => | |
| 74 | typeof globalThis !== "undefined" && | |
| 75 | "VideoDecoder" in globalThis && | |
| 76 | "EncodedVideoChunk" in globalThis; | |
| 77 | ||
| 78 | function taggedJson(tag: number, payload: unknown): Uint8Array<ArrayBuffer> { | |
| 79 | const json = encoder.encode(JSON.stringify(payload)); | |
| 80 | const out = new Uint8Array(1 + json.length); | |
| 81 | out[0] = tag; | |
| 82 | out.set(json, 1); | |
| 83 | return out; | |
| 84 | } | |
| 85 | ||
| 86 | /** Build the WebCodecs `avc1.PPCCLL` string from an avcC record or an SPS NAL. */ | |
| 87 | export function avcCodecString(bytes: Uint8Array): string { | |
| 88 | if (bytes.length < 4) return "avc1.42E01E"; | |
| 89 | const hex = (byte: number) => byte.toString(16).padStart(2, "0"); | |
| 90 | return `avc1.${hex(bytes[1]!)}${hex(bytes[2]!)}${hex(bytes[3]!)}`; | |
| 91 | } | |
| 92 | ||
| 93 | /** Split serve-emu's SEMU-framed message into metadata and the Annex-B payload. */ | |
| 94 | export function parseSemuPacket(raw: ArrayBuffer): { | |
| 95 | readonly data: Uint8Array; | |
| 96 | readonly isKey: boolean | null; | |
| 97 | readonly timestamp: number | null; | |
| 98 | } { | |
| 99 | const bytes = new Uint8Array(raw); | |
| 100 | if (bytes.byteLength > SEMU_HEADER_BYTES) { | |
| 101 | const view = new DataView(raw, 0, SEMU_HEADER_BYTES); | |
| 102 | if (view.getUint32(0, false) === SEMU_MAGIC && view.getUint8(4) === 1) { | |
| 103 | const pts = view.getBigUint64(8, false); | |
| 104 | return { | |
| 105 | data: bytes.subarray(SEMU_HEADER_BYTES), | |
| 106 | isKey: (view.getUint8(5) & SEMU_FLAG_KEY) !== 0, | |
| 107 | timestamp: pts <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(pts) : null, | |
| 108 | }; | |
| 109 | } | |
| 110 | } | |
| 111 | return { data: bytes, isKey: null, timestamp: null }; | |
| 112 | } | |
| 113 | ||
| 114 | const isVideoSessionMessage = (text: string) => { | |
| 115 | try { | |
| 116 | const message = JSON.parse(text) as { type?: unknown }; | |
| 117 | return message.type === "video-session"; | |
| 118 | } catch { | |
| 119 | return false; | |
| 120 | } | |
| 121 | }; | |
| 122 | ||
| 123 | /** Walk an Annex-B access unit for its keyframe flag and SPS bytes. */ | |
| 124 | export function scanAccessUnit(buf: Uint8Array): { isKey: boolean; sps: Uint8Array | null } { | |
| 125 | let isKey = false; | |
| 126 | let sps: Uint8Array | null = null; | |
| 127 | const len = buf.length; | |
| 128 | let i = 0; | |
| 129 | while (i + 2 < len) { | |
| 130 | if (buf[i] === 0 && buf[i + 1] === 0) { | |
| 131 | let codeLen = 0; | |
| 132 | if (buf[i + 2] === 1) codeLen = 3; | |
| 133 | else if (i + 3 < len && buf[i + 2] === 0 && buf[i + 3] === 1) codeLen = 4; | |
| 134 | if (codeLen) { | |
| 135 | const nalType = buf[i + codeLen]! & 0x1f; | |
| 136 | if (nalType === 7 && !sps) sps = buf.subarray(i + codeLen); | |
| 137 | if (nalType === 5) isKey = true; | |
| 138 | i += codeLen + 1; | |
| 139 | continue; | |
| 140 | } | |
| 141 | } | |
| 142 | i++; | |
| 143 | } | |
| 144 | return { isKey, sps }; | |
| 145 | } | |
| 146 | ||
| 147 | export type AvccChunk = { | |
| 148 | readonly type: "description" | "keyframe" | "delta" | "seed"; | |
| 149 | readonly payload: Uint8Array; | |
| 150 | }; | |
| 151 | ||
| 152 | const AVCC_TAGS: Record<number, AvccChunk["type"] | undefined> = { | |
| 153 | 1: "description", | |
| 154 | 2: "keyframe", | |
| 155 | 3: "delta", | |
| 156 | 4: "seed", | |
| 157 | }; | |
| 158 | ||
| 159 | /** Turns a fragmented AVCC byte stream into complete envelopes. */ | |
| 160 | export class AvccDemuxer { | |
| 161 | private buffer = new Uint8Array(64 * 1024); | |
| 162 | private length = 0; | |
| 163 | ||
| 164 | push(bytes: Uint8Array): AvccChunk[] { | |
| 165 | if (this.length + bytes.length > this.buffer.length) { | |
| 166 | let capacity = this.buffer.length; | |
| 167 | while (capacity < this.length + bytes.length) capacity *= 2; | |
| 168 | const grown = new Uint8Array(capacity); | |
| 169 | grown.set(this.buffer.subarray(0, this.length)); | |
| 170 | this.buffer = grown; | |
| 171 | } | |
| 172 | this.buffer.set(bytes, this.length); | |
| 173 | this.length += bytes.length; | |
| 174 | ||
| 175 | const chunks: AvccChunk[] = []; | |
| 176 | let offset = 0; | |
| 177 | while (this.length - offset >= 4) { | |
| 178 | const view = new DataView(this.buffer.buffer, this.buffer.byteOffset + offset, 4); | |
| 179 | const frameLength = view.getUint32(0, false); | |
| 180 | if (this.length - offset - 4 < frameLength) break; | |
| 181 | if (frameLength >= 1) { | |
| 182 | const type = AVCC_TAGS[this.buffer[offset + 4]!]; | |
| 183 | if (type) { | |
| 184 | chunks.push({ type, payload: this.buffer.slice(offset + 5, offset + 4 + frameLength) }); | |
| 185 | } | |
| 186 | } | |
| 187 | offset += 4 + frameLength; | |
| 188 | } | |
| 189 | if (offset > 0) { | |
| 190 | this.buffer.copyWithin(0, offset, this.length); | |
| 191 | this.length -= offset; | |
| 192 | } | |
| 193 | return chunks; | |
| 194 | } | |
| 195 | ||
| 196 | reset(): void { | |
| 197 | this.length = 0; | |
| 198 | } | |
| 199 | } | |
| 200 | ||
| 201 | export interface DeviceStreamClient { | |
| 202 | readonly start: () => void; | |
| 203 | readonly stop: () => void; | |
| 204 | /** Normalized 0..1 coordinates in the displayed frame. */ | |
| 205 | readonly sendTouch: (phase: "begin" | "move" | "end", x: number, y: number) => void; | |
| 206 | readonly sendKey: (event: KeyboardEvent, phase: "down" | "up") => void; | |
| 207 | readonly pressButton: (button: DeviceHardwareButton) => void; | |
| 208 | readonly rotate: () => void; | |
| 209 | } | |
| 210 | ||
| 211 | const HID_USAGE_BY_CODE: Readonly<Record<string, number>> = { | |
| 212 | Enter: 0x28, | |
| 213 | Escape: 0x29, | |
| 214 | Backspace: 0x2a, | |
| 215 | Tab: 0x2b, | |
| 216 | Space: 0x2c, | |
| 217 | Minus: 0x2d, | |
| 218 | Equal: 0x2e, | |
| 219 | BracketLeft: 0x2f, | |
| 220 | BracketRight: 0x30, | |
| 221 | Backslash: 0x31, | |
| 222 | Semicolon: 0x33, | |
| 223 | Quote: 0x34, | |
| 224 | Backquote: 0x35, | |
| 225 | Comma: 0x36, | |
| 226 | Period: 0x37, | |
| 227 | Slash: 0x38, | |
| 228 | Delete: 0x4c, | |
| 229 | ArrowRight: 0x4f, | |
| 230 | ArrowLeft: 0x50, | |
| 231 | ArrowDown: 0x51, | |
| 232 | ArrowUp: 0x52, | |
| 233 | ControlLeft: 0xe0, | |
| 234 | ShiftLeft: 0xe1, | |
| 235 | AltLeft: 0xe2, | |
| 236 | MetaLeft: 0xe3, | |
| 237 | ControlRight: 0xe4, | |
| 238 | ShiftRight: 0xe5, | |
| 239 | AltRight: 0xe6, | |
| 240 | MetaRight: 0xe7, | |
| 241 | }; | |
| 242 | ||
| 243 | function hidUsageForCode(code: string): number | null { | |
| 244 | if (/^Key[A-Z]$/.test(code)) return 0x04 + (code.charCodeAt(3) - 65); | |
| 245 | if (/^Digit[1-9]$/.test(code)) return 0x1e + (code.charCodeAt(5) - 49); | |
| 246 | if (code === "Digit0") return 0x27; | |
| 247 | return HID_USAGE_BY_CODE[code] ?? null; | |
| 248 | } | |
| 249 | ||
| 250 | const ANDROID_KEYCODE_BY_KEY: Readonly<Record<string, number>> = { | |
| 251 | ArrowUp: 19, | |
| 252 | ArrowDown: 20, | |
| 253 | ArrowLeft: 21, | |
| 254 | ArrowRight: 22, | |
| 255 | Tab: 61, | |
| 256 | Enter: 66, | |
| 257 | Backspace: 67, | |
| 258 | Delete: 112, | |
| 259 | Home: 122, | |
| 260 | End: 123, | |
| 261 | PageUp: 92, | |
| 262 | PageDown: 93, | |
| 263 | }; | |
| 264 | ||
| 265 | const IOS_ORIENTATIONS: ReadonlyArray<DeviceScreenSize["orientation"]> = [ | |
| 266 | "portrait", | |
| 267 | "landscape_left", | |
| 268 | "portrait_upside_down", | |
| 269 | "landscape_right", | |
| 270 | ]; | |
| 271 | ||
| 272 | export function createDeviceStreamClient( | |
| 273 | target: DeviceStreamTarget, | |
| 274 | canvas: HTMLCanvasElement, | |
| 275 | events: DeviceStreamEvents, | |
| 276 | ): DeviceStreamClient { | |
| 277 | const { access, platform, deviceId } = target; | |
| 278 | const vendor = platform === "ios" ? "/vendor/serve-sim" : "/vendor/serve-emu"; | |
| 279 | const device = encodeURIComponent(deviceId); | |
| 280 | const httpUrl = (path: string) => | |
| 281 | withDeviceHubQuery(`${access.httpBase}${vendor}${path}`, access); | |
| 282 | const wsUrl = (path: string) => withDeviceHubQuery(`${access.wsBase}${vendor}${path}`, access); | |
| 283 | const useWebCodecs = isWebCodecsSupported(); | |
| 284 | ||
| 285 | let stopped = true; | |
| 286 | let socket: WebSocket | null = null; | |
| 287 | let controller: AbortController | null = null; | |
| 288 | const retryTimers = new Map<"video" | "input", ReturnType<typeof setTimeout>>(); | |
| 289 | let primeController: AbortController | null = null; | |
| 290 | let videoDecoder: VideoDecoder | null = null; | |
| 291 | let timestamp = 0; | |
| 292 | let awaitingKeyframe = true; | |
| 293 | let screen: DeviceScreenSize | null = null; | |
| 294 | let firstFrame = false; | |
| 295 | let configuring = false; | |
| 296 | let mjpeg = false; | |
| 297 | ||
| 298 | const mjpegUrl = () => httpUrl(`/helper/${device}/stream.mjpeg`); | |
| 299 | ||
| 300 | const fallBackToMjpeg = () => { | |
| 301 | if (stopped || mjpeg) return; | |
| 302 | mjpeg = true; | |
| 303 | closeDecoder(); | |
| 304 | events.onMjpegFallback(mjpegUrl()); | |
| 305 | setStatus("streaming"); | |
| 306 | }; | |
| 307 | ||
| 308 | const setStatus = (status: DeviceStreamStatus, detail?: string) => { | |
| 309 | if (!stopped) events.onStatus(status, detail); | |
| 310 | }; | |
| 311 | ||
| 312 | const paint = (source: CanvasImageSource, width: number, height: number) => { | |
| 313 | if (stopped) return; | |
| 314 | if (canvas.width !== width || canvas.height !== height) { | |
| 315 | canvas.width = width; | |
| 316 | canvas.height = height; | |
| 317 | if (platform === "android") { | |
| 318 | screen = { width, height, orientation: width > height ? "landscape_left" : "portrait" }; | |
| 319 | events.onScreen(screen); | |
| 320 | } | |
| 321 | } | |
| 322 | canvas.getContext("2d")?.drawImage(source, 0, 0, width, height); | |
| 323 | if (!firstFrame) { | |
| 324 | firstFrame = true; | |
| 325 | setStatus("streaming"); | |
| 326 | } | |
| 327 | }; | |
| 328 | ||
| 329 | const closeDecoder = () => { | |
| 330 | try { | |
| 331 | videoDecoder?.close(); | |
| 332 | } catch { | |
| 333 | // Already closed. | |
| 334 | } | |
| 335 | videoDecoder = null; | |
| 336 | awaitingKeyframe = true; | |
| 337 | }; | |
| 338 | ||
| 339 | const makeDecoder = () => | |
| 340 | new VideoDecoder({ | |
| 341 | output: (frame) => { | |
| 342 | try { | |
| 343 | paint(frame, frame.displayWidth, frame.displayHeight); | |
| 344 | } finally { | |
| 345 | frame.close(); | |
| 346 | } | |
| 347 | }, | |
| 348 | error: () => { | |
| 349 | closeDecoder(); | |
| 350 | requestKeyframe(); | |
| 351 | }, | |
| 352 | }); | |
| 353 | ||
| 354 | /** | |
| 355 | * Resolves false when this browser cannot decode the stream's profile | |
| 356 | * (simulators encode High 5.1, which headless and some hardware decoders | |
| 357 | * reject). iOS then falls back to MJPEG; Android has no MJPEG. | |
| 358 | */ | |
| 359 | const configureDecoder = async (config: VideoDecoderConfig): Promise<boolean> => { | |
| 360 | const full: VideoDecoderConfig = { ...config, optimizeForLatency: true }; | |
| 361 | const support = await VideoDecoder.isConfigSupported(full).catch(() => ({ supported: false })); | |
| 362 | if (stopped) return false; | |
| 363 | if (!support.supported) { | |
| 364 | setStatus("error", `This browser cannot decode ${config.codec}.`); | |
| 365 | return false; | |
| 366 | } | |
| 367 | if (!videoDecoder || videoDecoder.state === "closed") videoDecoder = makeDecoder(); | |
| 368 | try { | |
| 369 | videoDecoder.configure(full); | |
| 370 | return true; | |
| 371 | } catch (cause) { | |
| 372 | setStatus("error", `Video decoder: ${(cause as Error).message}`); | |
| 373 | return false; | |
| 374 | } | |
| 375 | }; | |
| 376 | ||
| 377 | const decode = (isKey: boolean, data: Uint8Array, pts?: number | null) => { | |
| 378 | if (!videoDecoder || videoDecoder.state !== "configured") return; | |
| 379 | if (awaitingKeyframe) { | |
| 380 | if (!isKey) return; | |
| 381 | awaitingKeyframe = false; | |
| 382 | } | |
| 383 | if (videoDecoder.decodeQueueSize > SOFT_DECODE_QUEUE) { | |
| 384 | closeDecoder(); | |
| 385 | requestKeyframe(); | |
| 386 | return; | |
| 387 | } | |
| 388 | try { | |
| 389 | videoDecoder.decode( | |
| 390 | new EncodedVideoChunk({ | |
| 391 | type: isKey ? "key" : "delta", | |
| 392 | timestamp: pts ?? timestamp, | |
| 393 | data, | |
| 394 | }), | |
| 395 | ); | |
| 396 | timestamp += FRAME_DURATION_US; | |
| 397 | } catch { | |
| 398 | closeDecoder(); | |
| 399 | requestKeyframe(); | |
| 400 | } | |
| 401 | }; | |
| 402 | ||
| 403 | const requestKeyframe = () => { | |
| 404 | if (platform === "android" && socket?.readyState === WebSocket.OPEN) { | |
| 405 | socket.send(JSON.stringify({ type: "reset-video", ack: false })); | |
| 406 | } | |
| 407 | }; | |
| 408 | ||
| 409 | const scheduleRetry = (channel: "video" | "input", run: () => void) => { | |
| 410 | if (stopped || retryTimers.has(channel)) return; | |
| 411 | retryTimers.set( | |
| 412 | channel, | |
| 413 | setTimeout(() => { | |
| 414 | retryTimers.delete(channel); | |
| 415 | run(); | |
| 416 | }, RETRY_DELAY_MS), | |
| 417 | ); | |
| 418 | }; | |
| 419 | ||
| 420 | const handleUnauthorized = () => { | |
| 421 | stop(); | |
| 422 | events.onUnauthorized(); | |
| 423 | }; | |
| 424 | ||
| 425 | // iOS video: fetch the AVCC body and demux into the decoder. | |
| 426 | const readIosVideo = async () => { | |
| 427 | const demuxer = new AvccDemuxer(); | |
| 428 | controller = new AbortController(); | |
| 429 | try { | |
| 430 | const response = await fetch(httpUrl(`/helper/${device}/stream.avcc`), { | |
| 431 | signal: controller.signal, | |
| 432 | credentials: access.credentials ? "include" : "same-origin", | |
| 433 | }); | |
| 434 | if (response.status === 401 || response.status === 403) return handleUnauthorized(); | |
| 435 | if (!response.ok || !response.body) throw new Error(`stream ${response.status}`); | |
| 436 | const reader = response.body.getReader(); | |
| 437 | for (;;) { | |
| 438 | const { done, value } = await reader.read(); | |
| 439 | if (done || stopped) break; | |
| 440 | for (const chunk of demuxer.push(value)) { | |
| 441 | switch (chunk.type) { | |
| 442 | case "seed": | |
| 443 | void createImageBitmap(new Blob([chunk.payload as BlobPart], { type: "image/jpeg" })) | |
| 444 | .then((bitmap) => { | |
| 445 | paint(bitmap, bitmap.width, bitmap.height); | |
| 446 | bitmap.close(); | |
| 447 | }) | |
| 448 | .catch(() => {}); | |
| 449 | break; | |
| 450 | case "description": { | |
| 451 | awaitingKeyframe = true; | |
| 452 | const configured = await configureDecoder({ | |
| 453 | codec: avcCodecString(chunk.payload), | |
| 454 | description: chunk.payload, | |
| 455 | }); | |
| 456 | if (!configured) { | |
| 457 | await reader.cancel().catch(() => {}); | |
| 458 | fallBackToMjpeg(); | |
| 459 | return; | |
| 460 | } | |
| 461 | break; | |
| 462 | } | |
| 463 | case "keyframe": | |
| 464 | case "delta": | |
| 465 | decode(chunk.type === "keyframe", chunk.payload); | |
| 466 | break; | |
| 467 | } | |
| 468 | } | |
| 469 | } | |
| 470 | } catch (cause) { | |
| 471 | if (stopped) return; | |
| 472 | setStatus("connecting", (cause as Error).message); | |
| 473 | } | |
| 474 | if (!stopped) scheduleRetry("video", () => void readIosVideo()); | |
| 475 | }; | |
| 476 | ||
| 477 | /** | |
| 478 | * serve-sim's helper only accepts HID and pushes its screen config once | |
| 479 | * screen capture is running, and the AVCC stream does not reliably start | |
| 480 | * it. Touching the MJPEG endpoint does; one aborted request is enough. | |
| 481 | */ | |
| 482 | const primeIosHelper = async () => { | |
| 483 | const controller = new AbortController(); | |
| 484 | primeController = controller; | |
| 485 | const timeout = setTimeout(() => controller.abort(), 2_000); | |
| 486 | try { | |
| 487 | const response = await fetch(httpUrl(`/helper/${device}/stream.mjpeg`), { | |
| 488 | signal: controller.signal, | |
| 489 | credentials: access.credentials ? "include" : "same-origin", | |
| 490 | }); | |
| 491 | if (response.status === 401 || response.status === 403) return handleUnauthorized(); | |
| 492 | await response.body?.getReader().read(); | |
| 493 | } catch { | |
| 494 | // A failed prime just means the socket may take a retry to come up. | |
| 495 | } finally { | |
| 496 | clearTimeout(timeout); | |
| 497 | controller.abort(); | |
| 498 | if (primeController === controller) primeController = null; | |
| 499 | } | |
| 500 | }; | |
| 501 | ||
| 502 | // iOS input socket; also carries the screen config the helper pushes. | |
| 503 | const connectIosInput = async () => { | |
| 504 | if (stopped) return; | |
| 505 | await primeIosHelper(); | |
| 506 | if (stopped) return; | |
| 507 | const ws = new WebSocket(wsUrl(`/helper/ws?device=${device}`)); | |
| 508 | ws.binaryType = "arraybuffer"; | |
| 509 | socket = ws; | |
| 510 | ws.onopen = () => { | |
| 511 | ws.send(taggedJson(IOS_MSG_HARDWARE_KEYBOARD, { enabled: false })); | |
| 512 | events.onInputConnected(true); | |
| 513 | }; | |
| 514 | ws.onmessage = (event) => { | |
| 515 | if (!(event.data instanceof ArrayBuffer)) return; | |
| 516 | const bytes = new Uint8Array(event.data); | |
| 517 | if (bytes.length < 1 || bytes[0] !== IOS_TAG_SCREEN_CONFIG) return; | |
| 518 | try { | |
| 519 | const config = JSON.parse(decoder.decode(bytes.subarray(1))) as DeviceScreenSize; | |
| 520 | if (config.width > 0 && config.height > 0) { | |
| 521 | screen = config; | |
| 522 | events.onScreen(config); | |
| 523 | } | |
| 524 | } catch { | |
| 525 | // Ignore malformed config frames. | |
| 526 | } | |
| 527 | }; | |
| 528 | ws.onclose = (event) => { | |
| 529 | if (socket === ws) socket = null; | |
| 530 | if (!stopped) { | |
| 531 | events.onInputConnected( | |
| 532 | false, | |
| 533 | event.reason || (event.code === 1006 ? "input socket refused" : `closed ${event.code}`), | |
| 534 | ); | |
| 535 | } | |
| 536 | if (event.code === 1008 || event.code === 4401) return handleUnauthorized(); | |
| 537 | scheduleRetry("input", () => void connectIosInput()); | |
| 538 | }; | |
| 539 | ws.onerror = () => ws.close(); | |
| 540 | }; | |
| 541 | ||
| 542 | // Android: one socket for video and input. | |
| 543 | const connectAndroid = () => { | |
| 544 | if (stopped) return; | |
| 545 | const ws = new WebSocket(wsUrl(`/ws?device=${device}&frame-meta=1`)); | |
| 546 | ws.binaryType = "arraybuffer"; | |
| 547 | socket = ws; | |
| 548 | ws.onopen = () => { | |
| 549 | setStatus("connecting"); | |
| 550 | events.onInputConnected(true); | |
| 551 | }; | |
| 552 | ws.onmessage = (event) => { | |
| 553 | if (typeof event.data === "string") { | |
| 554 | // The encoder restarts at a new size when the device rotates; the | |
| 555 | // next keyframe carries a fresh SPS, so the decoder is rebuilt from it. | |
| 556 | if (isVideoSessionMessage(event.data)) closeDecoder(); | |
| 557 | return; | |
| 558 | } | |
| 559 | if (!(event.data instanceof ArrayBuffer)) return; | |
| 560 | const packet = parseSemuPacket(event.data); | |
| 561 | const needsScan = | |
| 562 | packet.isKey === null || | |
| 563 | (packet.isKey && (!videoDecoder || videoDecoder.state !== "configured")); | |
| 564 | const scanned = needsScan ? scanAccessUnit(packet.data) : null; | |
| 565 | const isKey = packet.isKey ?? scanned?.isKey ?? false; | |
| 566 | if (scanned?.sps && (!videoDecoder || videoDecoder.state !== "configured")) { | |
| 567 | if (configuring) return; | |
| 568 | configuring = true; | |
| 569 | void configureDecoder({ codec: avcCodecString(scanned.sps) }).then((configured) => { | |
| 570 | configuring = false; | |
| 571 | awaitingKeyframe = true; | |
| 572 | if (configured) requestKeyframe(); | |
| 573 | }); | |
| 574 | return; | |
| 575 | } | |
| 576 | if (!videoDecoder || videoDecoder.state !== "configured") { | |
| 577 | if (!isKey) requestKeyframe(); | |
| 578 | return; | |
| 579 | } | |
| 580 | decode(isKey, packet.data, packet.timestamp); | |
| 581 | }; | |
| 582 | ws.onclose = (event) => { | |
| 583 | if (socket === ws) socket = null; | |
| 584 | closeDecoder(); | |
| 585 | if (!stopped) events.onInputConnected(false, event.reason || `closed ${event.code}`); | |
| 586 | if (event.code === 1008 || event.code === 4401) return handleUnauthorized(); | |
| 587 | if (!stopped) { | |
| 588 | setStatus("connecting", event.reason || undefined); | |
| 589 | scheduleRetry("input", connectAndroid); | |
| 590 | } | |
| 591 | }; | |
| 592 | ws.onerror = () => ws.close(); | |
| 593 | }; | |
| 594 | ||
| 595 | const start = () => { | |
| 596 | if (!stopped) return; | |
| 597 | stopped = false; | |
| 598 | firstFrame = false; | |
| 599 | events.onStatus("connecting"); | |
| 600 | if (platform === "ios") { | |
| 601 | void connectIosInput(); | |
| 602 | if (useWebCodecs) void readIosVideo(); | |
| 603 | else fallBackToMjpeg(); | |
| 604 | } else if (useWebCodecs) { | |
| 605 | connectAndroid(); | |
| 606 | } else { | |
| 607 | setStatus("error", "This browser cannot decode the Android stream (WebCodecs unavailable)."); | |
| 608 | } | |
| 609 | }; | |
| 610 | ||
| 611 | const stop = () => { | |
| 612 | if (stopped) return; | |
| 613 | stopped = true; | |
| 614 | mjpeg = false; | |
| 615 | for (const timer of retryTimers.values()) clearTimeout(timer); | |
| 616 | retryTimers.clear(); | |
| 617 | primeController?.abort(); | |
| 618 | primeController = null; | |
| 619 | controller?.abort(); | |
| 620 | controller = null; | |
| 621 | socket?.close(); | |
| 622 | socket = null; | |
| 623 | closeDecoder(); | |
| 624 | }; | |
| 625 | ||
| 626 | const send = (payload: Uint8Array<ArrayBuffer> | string) => { | |
| 627 | if (socket?.readyState === WebSocket.OPEN) socket.send(payload); | |
| 628 | }; | |
| 629 | ||
| 630 | const rawPoint = (x: number, y: number) => { | |
| 631 | // serve-sim streams the raw framebuffer; rotated devices need input | |
| 632 | // remapped into that raw space. | |
| 633 | if (platform !== "ios" || !screen || screen.width > screen.height) return { x, y }; | |
| 634 | switch (screen.orientation) { | |
| 635 | case "landscape_left": | |
| 636 | return { x: y, y: 1 - x }; | |
| 637 | case "landscape_right": | |
| 638 | return { x: 1 - y, y: x }; | |
| 639 | case "portrait_upside_down": | |
| 640 | return { x: 1 - x, y: 1 - y }; | |
| 641 | default: | |
| 642 | return { x, y }; | |
| 643 | } | |
| 644 | }; | |
| 645 | ||
| 646 | return { | |
| 647 | start, | |
| 648 | stop, | |
| 649 | sendTouch: (phase, x, y) => { | |
| 650 | if (platform === "ios") { | |
| 651 | send(taggedJson(IOS_MSG_TOUCH, { type: phase, ...rawPoint(x, y) })); | |
| 652 | return; | |
| 653 | } | |
| 654 | const action = phase === "begin" ? "down" : phase === "move" ? "move" : "up"; | |
| 655 | send(JSON.stringify({ type: "touch", action, x, y })); | |
| 656 | }, | |
| 657 | sendKey: (event, phase) => { | |
| 658 | if (platform === "ios") { | |
| 659 | const usage = hidUsageForCode(event.code); | |
| 660 | if (usage !== null) send(taggedJson(IOS_MSG_KEY, { type: phase, usage })); | |
| 661 | return; | |
| 662 | } | |
| 663 | if (phase !== "down") return; | |
| 664 | if (event.key === "Escape") return send(JSON.stringify({ type: "back" })); | |
| 665 | const keycode = ANDROID_KEYCODE_BY_KEY[event.key]; | |
| 666 | if (keycode !== undefined) return send(JSON.stringify({ type: "key", keycode })); | |
| 667 | if (event.key.length === 1 && !event.metaKey && !event.ctrlKey) { | |
| 668 | send(JSON.stringify({ type: "text", text: event.key })); | |
| 669 | } | |
| 670 | }, | |
| 671 | pressButton: (button) => { | |
| 672 | if (platform === "ios") { | |
| 673 | const name = | |
| 674 | button === "home" | |
| 675 | ? "home" | |
| 676 | : button === "appSwitcher" | |
| 677 | ? "app_switcher" | |
| 678 | : button === "power" | |
| 679 | ? "lock" | |
| 680 | : null; | |
| 681 | if (name) send(taggedJson(IOS_MSG_BUTTON, { button: name })); | |
| 682 | return; | |
| 683 | } | |
| 684 | const type = button === "appSwitcher" ? "recents" : button; | |
| 685 | if (type === "home" || type === "back" || type === "recents" || type === "power") { | |
| 686 | send(JSON.stringify({ type })); | |
| 687 | } | |
| 688 | }, | |
| 689 | rotate: () => { | |
| 690 | if (platform !== "ios") return; | |
| 691 | const current = screen?.orientation ?? "portrait"; | |
| 692 | const next = | |
| 693 | IOS_ORIENTATIONS[(IOS_ORIENTATIONS.indexOf(current) + 1) % IOS_ORIENTATIONS.length]!; | |
| 694 | send(taggedJson(IOS_MSG_ORIENTATION, { orientation: next })); | |
| 695 | }, | |
| 696 | }; | |
| 697 | } |
apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx+1-1
| ... | ... | @@ -35,7 +35,7 @@ import { useRightPanelStore } from "~/rightPanelStore"; |
| 35 | 35 | import { useDeviceState } from "~/state/device"; |
| 36 | 36 | |
| 37 | 37 | import { DeviceStreamView } from "../device/DeviceStreamView"; |
| 38 | import type { DeviceScreenSize } from "../device/deviceStream"; | |
| 38 | import type { DeviceScreenSize } from "@t3tools/client-runtime/device/stream"; | |
| 39 | 39 | import { previewBridge } from "./previewBridge"; |
| 40 | 40 | import { |
| 41 | 41 | clampPreviewMiniPlayerPosition, |
apps/web/src/components/preview/previewMiniPlayerLayout.ts+1-1
| ... | ... | @@ -7,7 +7,7 @@ import { |
| 7 | 7 | } from "~/browser/browserViewportLayout"; |
| 8 | 8 | import type { PreviewMiniPlayerPosition, PreviewMiniPlayerSize } from "~/previewMiniPlayerStore"; |
| 9 | 9 | |
| 10 | import type { DeviceScreenSize } from "../device/deviceStream"; | |
| 10 | import type { DeviceScreenSize } from "@t3tools/client-runtime/device/stream"; | |
| 11 | 11 | |
| 12 | 12 | export const PREVIEW_MINI_PLAYER_EDGE_GAP = 12; |
| 13 | 13 | export const PREVIEW_MINI_PLAYER_CORNER_RADIUS = 12; |
docs/user/devices.md+7-2
| ... | ... | @@ -55,8 +55,13 @@ back from the device after a change. |
| 55 | 55 | When an agent opens a device, it floats over the chat in web and desktop clients |
| 56 | 56 | connected to the thread, the same way an agent-driven browser does. Turn off |
| 57 | 57 | **Auto-show floating preview** in **Settings → Integrations → Browser** to open a |
| 58 | right-panel tab instead. Mobile clients show device activity in the thread | |
| 59 | timeline. Agents drive the device through the `agent-device` command line. T3 | |
| 58 | right-panel tab instead. In the mobile app, open the agent's thread and tap the | |
| 59 | device button above the composer to watch the live screen and control it. | |
| 60 | If the thread has several devices open, choose one in the viewer. Closing the | |
| 61 | viewer stops streaming and leaves the device available to the agent. Device | |
| 62 | activity also appears in the thread timeline. | |
| 63 | ||
| 64 | Agents drive the device through the `agent-device` command line. T3 | |
| 60 | 65 | Code installs and starts it only after **Agent device access** is enabled. iOS |
| 61 | 66 | taps build a small test runner on first use, which takes a couple of minutes |
| 62 | 67 | once per server. Restart an existing agent session after granting access so it |
knip.jsonc+7-2
| ... | ... | @@ -58,8 +58,13 @@ |
| 58 | 58 | "ignoreIssues": { "src/components/ui/*.tsx": ["exports", "nsExports", "duplicates"] }, |
| 59 | 59 | }, |
| 60 | 60 | "apps/mobile": { |
| 61 | // Expo loads local config plugins by string; Metro handles platform variants. | |
| 62 | "entry": ["index.ts!", "plugins/*.cjs"], | |
| 61 | // Expo loads plugins by string; Metro embeds the browser entry in the native WebView. | |
| 62 | "entry": [ | |
| 63 | "index.ts!", | |
| 64 | "plugins/*.cjs", | |
| 65 | "scripts/generate-device-stream.mts", | |
| 66 | "src/features/devices/device-stream.browser.ts!", | |
| 67 | ], | |
| 63 | 68 | // Fonts are configured as asset paths; Expo autolinks the native packages. |
| 64 | 69 | "ignoreDependencies": [ |
| 65 | 70 | "@expo-google-fonts/dm-sans", |
packages/client-runtime/package.json+8
| ... | ... | @@ -3,6 +3,14 @@ |
| 3 | 3 | "private": true, |
| 4 | 4 | "type": "module", |
| 5 | 5 | "exports": { |
| 6 | "./device/stream": { | |
| 7 | "types": "./src/device/stream.ts", | |
| 8 | "default": "./src/device/stream.ts" | |
| 9 | }, | |
| 10 | "./device/hub-access": { | |
| 11 | "types": "./src/device/hubAccess.ts", | |
| 12 | "default": "./src/device/hubAccess.ts" | |
| 13 | }, | |
| 6 | 14 | "./worktree-setup": { |
| 7 | 15 | "types": "./src/worktreeSetup.ts", |
| 8 | 16 | "default": "./src/worktreeSetup.ts" |
packages/client-runtime/src/device/hubAccess.ts created+18
| ... | ... | @@ -0,0 +1,18 @@ |
| 1 | /** Credentials for media requests that cannot set bearer or DPoP headers. */ | |
| 2 | export interface DeviceHubAccess { | |
| 3 | /** Absolute environment URL ending in `/api/device-hub`. */ | |
| 4 | readonly httpBase: string; | |
| 5 | /** Same base with the `ws(s)` scheme. */ | |
| 6 | readonly wsBase: string; | |
| 7 | /** Empty for cookie sessions; includes a short-lived ticket for bearer and DPoP sessions. */ | |
| 8 | readonly query: Readonly<Record<string, string>>; | |
| 9 | /** Whether requests must include session cookies. */ | |
| 10 | readonly credentials: boolean; | |
| 11 | } | |
| 12 | ||
| 13 | export const withDeviceHubQuery = (url: string, access: DeviceHubAccess): string => { | |
| 14 | const entries = Object.entries(access.query); | |
| 15 | if (entries.length === 0) return url; | |
| 16 | const separator = url.includes("?") ? "&" : "?"; | |
| 17 | return `${url}${separator}${new URLSearchParams(entries).toString()}`; | |
| 18 | }; |
packages/client-runtime/src/device/stream.ts created+712
| ... | ... | @@ -0,0 +1,712 @@ |
| 1 | // @effect-diagnostics globalFetch:off globalTimers:off - This browser and WebView transport runs without an Effect runtime. | |
| 2 | /* oxlint-disable unicorn/prefer-add-event-listener -- Each client owns its sockets and their handlers. */ | |
| 3 | ||
| 4 | /** | |
| 5 | * Framework-free client for expo-device-hub's per-device streams, reached | |
| 6 | * through the T3 proxy. One class handles both platforms because the hub | |
| 7 | * vendors two servers with different wire formats: | |
| 8 | * | |
| 9 | * - iOS (serve-sim): video is an HTTP `stream.avcc` body of length-prefixed | |
| 10 | * envelopes (`u32be length, u8 tag, payload`; tag 1 avcC description, | |
| 11 | * 2 keyframe, 3 delta, 4 JPEG seed) decoded with WebCodecs; input goes over | |
| 12 | * `helper/ws?device=<udid>` as `[tag][json]` packets. When WebCodecs is | |
| 13 | * unavailable (plain-http remote origins) the MJPEG endpoint is used as an | |
| 14 | * `<img>` source instead. | |
| 15 | * - Android (serve-emu): one WebSocket at `ws?device=<serial>&frame-meta=1` | |
| 16 | * carries H.264 access units prefixed with a 16-byte "SEMU" header | |
| 17 | * (magic, version, key flag, pts) and accepts JSON gestures upstream. | |
| 18 | * | |
| 19 | * The decoder only runs while frames arrive and the viewer is attached; a | |
| 20 | * hidden panel calls `stop()` so an idle device costs nothing on the GPU. | |
| 21 | */ | |
| 22 | import { type DeviceHubAccess, withDeviceHubQuery } from "./hubAccess.ts"; | |
| 23 | import type { DevicePlatform } from "@t3tools/contracts"; | |
| 24 | ||
| 25 | export type DeviceStreamStatus = "connecting" | "streaming" | "error"; | |
| 26 | ||
| 27 | export interface DeviceScreenSize { | |
| 28 | readonly width: number; | |
| 29 | readonly height: number; | |
| 30 | readonly orientation: "portrait" | "portrait_upside_down" | "landscape_left" | "landscape_right"; | |
| 31 | } | |
| 32 | ||
| 33 | export interface DeviceStreamEvents { | |
| 34 | readonly onStatus: (status: DeviceStreamStatus, detail?: string) => void; | |
| 35 | readonly onScreen: (screen: DeviceScreenSize) => void; | |
| 36 | /** The proxy rejected the credential; the owner should refresh access and reconnect. */ | |
| 37 | readonly onUnauthorized: () => void; | |
| 38 | /** | |
| 39 | * H.264 cannot be decoded here (no WebCodecs, or the simulator's profile is | |
| 40 | * unsupported); the owner should show this MJPEG URL in an `<img>` instead of | |
| 41 | * the canvas. | |
| 42 | */ | |
| 43 | readonly onMjpegFallback: (url: string) => void; | |
| 44 | /** Whether touches and keys can currently reach the device. */ | |
| 45 | readonly onInputConnected: (connected: boolean, detail?: string) => void; | |
| 46 | } | |
| 47 | ||
| 48 | export interface DeviceStreamTarget { | |
| 49 | readonly platform: DevicePlatform; | |
| 50 | readonly deviceId: string; | |
| 51 | readonly access: DeviceHubAccess; | |
| 52 | /** Native iOS WebViews can use MJPEG without cross-origin fetch or secure-context support. */ | |
| 53 | readonly preferMjpeg?: boolean; | |
| 54 | } | |
| 55 | ||
| 56 | export type DeviceHardwareButton = "home" | "back" | "recents" | "power" | "appSwitcher"; | |
| 57 | ||
| 58 | const RETRY_DELAY_MS = 1_000; | |
| 59 | const FRAME_DURATION_US = 16_667; | |
| 60 | const SEMU_MAGIC = 0x53454d55; | |
| 61 | const SEMU_HEADER_BYTES = 16; | |
| 62 | const SEMU_FLAG_KEY = 1; | |
| 63 | const SOFT_DECODE_QUEUE = 8; | |
| 64 | ||
| 65 | // serve-sim binary WS message tags (browser -> helper). | |
| 66 | const IOS_MSG_TOUCH = 0x03; | |
| 67 | const IOS_MSG_BUTTON = 0x04; | |
| 68 | const IOS_MSG_KEY = 0x06; | |
| 69 | const IOS_MSG_ORIENTATION = 0x07; | |
| 70 | const IOS_MSG_HARDWARE_KEYBOARD = 0x0d; | |
| 71 | // helper -> browser. | |
| 72 | const IOS_TAG_SCREEN_CONFIG = 0x82; | |
| 73 | ||
| 74 | const encoder = new TextEncoder(); | |
| 75 | const decoder = new TextDecoder(); | |
| 76 | ||
| 77 | const isWebCodecsSupported = (): boolean => | |
| 78 | typeof globalThis !== "undefined" && | |
| 79 | "VideoDecoder" in globalThis && | |
| 80 | "EncodedVideoChunk" in globalThis; | |
| 81 | ||
| 82 | function taggedJson(tag: number, payload: unknown): Uint8Array<ArrayBuffer> { | |
| 83 | const json = encoder.encode(JSON.stringify(payload)); | |
| 84 | const out = new Uint8Array(1 + json.length); | |
| 85 | out[0] = tag; | |
| 86 | out.set(json, 1); | |
| 87 | return out; | |
| 88 | } | |
| 89 | ||
| 90 | /** Build the WebCodecs `avc1.PPCCLL` string from an avcC record or an SPS NAL. */ | |
| 91 | export function avcCodecString(bytes: Uint8Array): string { | |
| 92 | if (bytes.length < 4) return "avc1.42E01E"; | |
| 93 | const hex = (byte: number) => byte.toString(16).padStart(2, "0"); | |
| 94 | return `avc1.${hex(bytes[1]!)}${hex(bytes[2]!)}${hex(bytes[3]!)}`; | |
| 95 | } | |
| 96 | ||
| 97 | /** Split serve-emu's SEMU-framed message into metadata and the Annex-B payload. */ | |
| 98 | export function parseSemuPacket(raw: ArrayBuffer): { | |
| 99 | readonly data: Uint8Array; | |
| 100 | readonly isKey: boolean | null; | |
| 101 | readonly timestamp: number | null; | |
| 102 | } { | |
| 103 | const bytes = new Uint8Array(raw); | |
| 104 | if (bytes.byteLength > SEMU_HEADER_BYTES) { | |
| 105 | const view = new DataView(raw, 0, SEMU_HEADER_BYTES); | |
| 106 | if (view.getUint32(0, false) === SEMU_MAGIC && view.getUint8(4) === 1) { | |
| 107 | const pts = view.getBigUint64(8, false); | |
| 108 | return { | |
| 109 | data: bytes.subarray(SEMU_HEADER_BYTES), | |
| 110 | isKey: (view.getUint8(5) & SEMU_FLAG_KEY) !== 0, | |
| 111 | timestamp: pts <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(pts) : null, | |
| 112 | }; | |
| 113 | } | |
| 114 | } | |
| 115 | return { data: bytes, isKey: null, timestamp: null }; | |
| 116 | } | |
| 117 | ||
| 118 | const isVideoSessionMessage = (text: string) => { | |
| 119 | try { | |
| 120 | const message = JSON.parse(text) as { type?: unknown }; | |
| 121 | return message.type === "video-session"; | |
| 122 | } catch { | |
| 123 | return false; | |
| 124 | } | |
| 125 | }; | |
| 126 | ||
| 127 | /** Walk an Annex-B access unit for its keyframe flag and SPS bytes. */ | |
| 128 | export function scanAccessUnit(buf: Uint8Array): { isKey: boolean; sps: Uint8Array | null } { | |
| 129 | let isKey = false; | |
| 130 | let sps: Uint8Array | null = null; | |
| 131 | const len = buf.length; | |
| 132 | let i = 0; | |
| 133 | while (i + 2 < len) { | |
| 134 | if (buf[i] === 0 && buf[i + 1] === 0) { | |
| 135 | let codeLen = 0; | |
| 136 | if (buf[i + 2] === 1) codeLen = 3; | |
| 137 | else if (i + 3 < len && buf[i + 2] === 0 && buf[i + 3] === 1) codeLen = 4; | |
| 138 | if (codeLen) { | |
| 139 | const nalType = buf[i + codeLen]! & 0x1f; | |
| 140 | if (nalType === 7 && !sps) sps = buf.subarray(i + codeLen); | |
| 141 | if (nalType === 5) isKey = true; | |
| 142 | i += codeLen + 1; | |
| 143 | continue; | |
| 144 | } | |
| 145 | } | |
| 146 | i++; | |
| 147 | } | |
| 148 | return { isKey, sps }; | |
| 149 | } | |
| 150 | ||
| 151 | export type AvccChunk = { | |
| 152 | readonly type: "description" | "keyframe" | "delta" | "seed"; | |
| 153 | readonly payload: Uint8Array; | |
| 154 | }; | |
| 155 | ||
| 156 | const AVCC_TAGS: Record<number, AvccChunk["type"] | undefined> = { | |
| 157 | 1: "description", | |
| 158 | 2: "keyframe", | |
| 159 | 3: "delta", | |
| 160 | 4: "seed", | |
| 161 | }; | |
| 162 | ||
| 163 | /** Turns a fragmented AVCC byte stream into complete envelopes. */ | |
| 164 | export class AvccDemuxer { | |
| 165 | private buffer = new Uint8Array(64 * 1024); | |
| 166 | private length = 0; | |
| 167 | ||
| 168 | push(bytes: Uint8Array): AvccChunk[] { | |
| 169 | if (this.length + bytes.length > this.buffer.length) { | |
| 170 | let capacity = this.buffer.length; | |
| 171 | while (capacity < this.length + bytes.length) capacity *= 2; | |
| 172 | const grown = new Uint8Array(capacity); | |
| 173 | grown.set(this.buffer.subarray(0, this.length)); | |
| 174 | this.buffer = grown; | |
| 175 | } | |
| 176 | this.buffer.set(bytes, this.length); | |
| 177 | this.length += bytes.length; | |
| 178 | ||
| 179 | const chunks: AvccChunk[] = []; | |
| 180 | let offset = 0; | |
| 181 | while (this.length - offset >= 4) { | |
| 182 | const view = new DataView(this.buffer.buffer, this.buffer.byteOffset + offset, 4); | |
| 183 | const frameLength = view.getUint32(0, false); | |
| 184 | if (this.length - offset - 4 < frameLength) break; | |
| 185 | if (frameLength >= 1) { | |
| 186 | const type = AVCC_TAGS[this.buffer[offset + 4]!]; | |
| 187 | if (type) { | |
| 188 | chunks.push({ type, payload: this.buffer.slice(offset + 5, offset + 4 + frameLength) }); | |
| 189 | } | |
| 190 | } | |
| 191 | offset += 4 + frameLength; | |
| 192 | } | |
| 193 | if (offset > 0) { | |
| 194 | this.buffer.copyWithin(0, offset, this.length); | |
| 195 | this.length -= offset; | |
| 196 | } | |
| 197 | return chunks; | |
| 198 | } | |
| 199 | ||
| 200 | reset(): void { | |
| 201 | this.length = 0; | |
| 202 | } | |
| 203 | } | |
| 204 | ||
| 205 | export interface DeviceStreamClient { | |
| 206 | readonly start: () => void; | |
| 207 | readonly stop: () => void; | |
| 208 | /** Normalized 0..1 coordinates in the displayed frame. */ | |
| 209 | readonly sendTouch: (phase: "begin" | "move" | "end", x: number, y: number) => void; | |
| 210 | readonly sendKey: (event: KeyboardEvent, phase: "down" | "up") => void; | |
| 211 | readonly pressButton: (button: DeviceHardwareButton) => void; | |
| 212 | readonly rotate: () => void; | |
| 213 | } | |
| 214 | ||
| 215 | const HID_USAGE_BY_CODE: Readonly<Record<string, number>> = { | |
| 216 | Enter: 0x28, | |
| 217 | Escape: 0x29, | |
| 218 | Backspace: 0x2a, | |
| 219 | Tab: 0x2b, | |
| 220 | Space: 0x2c, | |
| 221 | Minus: 0x2d, | |
| 222 | Equal: 0x2e, | |
| 223 | BracketLeft: 0x2f, | |
| 224 | BracketRight: 0x30, | |
| 225 | Backslash: 0x31, | |
| 226 | Semicolon: 0x33, | |
| 227 | Quote: 0x34, | |
| 228 | Backquote: 0x35, | |
| 229 | Comma: 0x36, | |
| 230 | Period: 0x37, | |
| 231 | Slash: 0x38, | |
| 232 | Delete: 0x4c, | |
| 233 | ArrowRight: 0x4f, | |
| 234 | ArrowLeft: 0x50, | |
| 235 | ArrowDown: 0x51, | |
| 236 | ArrowUp: 0x52, | |
| 237 | ControlLeft: 0xe0, | |
| 238 | ShiftLeft: 0xe1, | |
| 239 | AltLeft: 0xe2, | |
| 240 | MetaLeft: 0xe3, | |
| 241 | ControlRight: 0xe4, | |
| 242 | ShiftRight: 0xe5, | |
| 243 | AltRight: 0xe6, | |
| 244 | MetaRight: 0xe7, | |
| 245 | }; | |
| 246 | ||
| 247 | function hidUsageForCode(code: string): number | null { | |
| 248 | if (/^Key[A-Z]$/.test(code)) return 0x04 + (code.charCodeAt(3) - 65); | |
| 249 | if (/^Digit[1-9]$/.test(code)) return 0x1e + (code.charCodeAt(5) - 49); | |
| 250 | if (code === "Digit0") return 0x27; | |
| 251 | return HID_USAGE_BY_CODE[code] ?? null; | |
| 252 | } | |
| 253 | ||
| 254 | const ANDROID_KEYCODE_BY_KEY: Readonly<Record<string, number>> = { | |
| 255 | ArrowUp: 19, | |
| 256 | ArrowDown: 20, | |
| 257 | ArrowLeft: 21, | |
| 258 | ArrowRight: 22, | |
| 259 | Tab: 61, | |
| 260 | Enter: 66, | |
| 261 | Backspace: 67, | |
| 262 | Delete: 112, | |
| 263 | Home: 122, | |
| 264 | End: 123, | |
| 265 | PageUp: 92, | |
| 266 | PageDown: 93, | |
| 267 | }; | |
| 268 | ||
| 269 | const IOS_ORIENTATIONS: ReadonlyArray<DeviceScreenSize["orientation"]> = [ | |
| 270 | "portrait", | |
| 271 | "landscape_left", | |
| 272 | "portrait_upside_down", | |
| 273 | "landscape_right", | |
| 274 | ]; | |
| 275 | ||
| 276 | export function createDeviceStreamClient( | |
| 277 | target: DeviceStreamTarget, | |
| 278 | canvas: HTMLCanvasElement, | |
| 279 | events: DeviceStreamEvents, | |
| 280 | ): DeviceStreamClient { | |
| 281 | const { access, platform, deviceId } = target; | |
| 282 | const vendor = platform === "ios" ? "/vendor/serve-sim" : "/vendor/serve-emu"; | |
| 283 | const device = encodeURIComponent(deviceId); | |
| 284 | const httpUrl = (path: string) => | |
| 285 | withDeviceHubQuery(`${access.httpBase}${vendor}${path}`, access); | |
| 286 | const wsUrl = (path: string) => withDeviceHubQuery(`${access.wsBase}${vendor}${path}`, access); | |
| 287 | const useWebCodecs = isWebCodecsSupported() && !(platform === "ios" && target.preferMjpeg); | |
| 288 | ||
| 289 | let stopped = true; | |
| 290 | let socket: WebSocket | null = null; | |
| 291 | let controller: AbortController | null = null; | |
| 292 | const retryTimers = new Map<"video" | "input", ReturnType<typeof setTimeout>>(); | |
| 293 | let primeController: AbortController | null = null; | |
| 294 | let videoDecoder: VideoDecoder | null = null; | |
| 295 | let timestamp = 0; | |
| 296 | let awaitingKeyframe = true; | |
| 297 | let screen: DeviceScreenSize | null = null; | |
| 298 | let firstFrame = false; | |
| 299 | let configuring = false; | |
| 300 | let mjpeg = false; | |
| 301 | ||
| 302 | const mjpegUrl = () => httpUrl(`/helper/${device}/stream.mjpeg`); | |
| 303 | ||
| 304 | const fallBackToMjpeg = () => { | |
| 305 | if (stopped || mjpeg) return; | |
| 306 | mjpeg = true; | |
| 307 | closeDecoder(); | |
| 308 | events.onMjpegFallback(mjpegUrl()); | |
| 309 | setStatus("streaming"); | |
| 310 | }; | |
| 311 | ||
| 312 | const setStatus = (status: DeviceStreamStatus, detail?: string) => { | |
| 313 | if (!stopped) events.onStatus(status, detail); | |
| 314 | }; | |
| 315 | ||
| 316 | const paint = (source: CanvasImageSource, width: number, height: number) => { | |
| 317 | if (stopped) return; | |
| 318 | if (canvas.width !== width || canvas.height !== height) { | |
| 319 | canvas.width = width; | |
| 320 | canvas.height = height; | |
| 321 | if (platform === "android") { | |
| 322 | screen = { width, height, orientation: width > height ? "landscape_left" : "portrait" }; | |
| 323 | events.onScreen(screen); | |
| 324 | } | |
| 325 | } | |
| 326 | canvas.getContext("2d")?.drawImage(source, 0, 0, width, height); | |
| 327 | if (!firstFrame) { | |
| 328 | firstFrame = true; | |
| 329 | setStatus("streaming"); | |
| 330 | } | |
| 331 | }; | |
| 332 | ||
| 333 | const closeDecoder = () => { | |
| 334 | try { | |
| 335 | videoDecoder?.close(); | |
| 336 | } catch { | |
| 337 | // Already closed. | |
| 338 | } | |
| 339 | videoDecoder = null; | |
| 340 | awaitingKeyframe = true; | |
| 341 | }; | |
| 342 | ||
| 343 | const makeDecoder = () => | |
| 344 | new VideoDecoder({ | |
| 345 | output: (frame) => { | |
| 346 | try { | |
| 347 | paint(frame, frame.displayWidth, frame.displayHeight); | |
| 348 | } finally { | |
| 349 | frame.close(); | |
| 350 | } | |
| 351 | }, | |
| 352 | error: () => { | |
| 353 | closeDecoder(); | |
| 354 | requestKeyframe(); | |
| 355 | }, | |
| 356 | }); | |
| 357 | ||
| 358 | /** | |
| 359 | * Resolves false when this browser cannot decode the stream's profile | |
| 360 | * (simulators encode High 5.1, which headless and some hardware decoders | |
| 361 | * reject). iOS then falls back to MJPEG; Android has no MJPEG. | |
| 362 | */ | |
| 363 | const configureDecoder = async (config: VideoDecoderConfig): Promise<boolean> => { | |
| 364 | const full: VideoDecoderConfig = { ...config, optimizeForLatency: true }; | |
| 365 | const support = await VideoDecoder.isConfigSupported(full).catch(() => ({ supported: false })); | |
| 366 | if (stopped) return false; | |
| 367 | if (!support.supported) { | |
| 368 | setStatus("error", `This browser cannot decode ${config.codec}.`); | |
| 369 | return false; | |
| 370 | } | |
| 371 | if (!videoDecoder || videoDecoder.state === "closed") videoDecoder = makeDecoder(); | |
| 372 | try { | |
| 373 | videoDecoder.configure(full); | |
| 374 | return true; | |
| 375 | } catch (cause) { | |
| 376 | setStatus("error", `Video decoder: ${(cause as Error).message}`); | |
| 377 | return false; | |
| 378 | } | |
| 379 | }; | |
| 380 | ||
| 381 | const decode = (isKey: boolean, data: Uint8Array, pts?: number | null) => { | |
| 382 | if (!videoDecoder || videoDecoder.state !== "configured") return; | |
| 383 | if (awaitingKeyframe) { | |
| 384 | if (!isKey) return; | |
| 385 | awaitingKeyframe = false; | |
| 386 | } | |
| 387 | if (videoDecoder.decodeQueueSize > SOFT_DECODE_QUEUE) { | |
| 388 | closeDecoder(); | |
| 389 | requestKeyframe(); | |
| 390 | return; | |
| 391 | } | |
| 392 | try { | |
| 393 | videoDecoder.decode( | |
| 394 | new EncodedVideoChunk({ | |
| 395 | type: isKey ? "key" : "delta", | |
| 396 | timestamp: pts ?? timestamp, | |
| 397 | data, | |
| 398 | }), | |
| 399 | ); | |
| 400 | timestamp += FRAME_DURATION_US; | |
| 401 | } catch { | |
| 402 | closeDecoder(); | |
| 403 | requestKeyframe(); | |
| 404 | } | |
| 405 | }; | |
| 406 | ||
| 407 | const requestKeyframe = () => { | |
| 408 | if (platform === "android" && socket?.readyState === WebSocket.OPEN) { | |
| 409 | socket.send(JSON.stringify({ type: "reset-video", ack: false })); | |
| 410 | } | |
| 411 | }; | |
| 412 | ||
| 413 | const scheduleRetry = (channel: "video" | "input", run: () => void) => { | |
| 414 | if (stopped || retryTimers.has(channel)) return; | |
| 415 | retryTimers.set( | |
| 416 | channel, | |
| 417 | setTimeout(() => { | |
| 418 | retryTimers.delete(channel); | |
| 419 | run(); | |
| 420 | }, RETRY_DELAY_MS), | |
| 421 | ); | |
| 422 | }; | |
| 423 | ||
| 424 | const handleUnauthorized = () => { | |
| 425 | stop(); | |
| 426 | events.onUnauthorized(); | |
| 427 | }; | |
| 428 | ||
| 429 | // iOS video: fetch the AVCC body and demux into the decoder. | |
| 430 | const readIosVideo = async () => { | |
| 431 | const demuxer = new AvccDemuxer(); | |
| 432 | controller = new AbortController(); | |
| 433 | try { | |
| 434 | const response = await fetch(httpUrl(`/helper/${device}/stream.avcc`), { | |
| 435 | signal: controller.signal, | |
| 436 | credentials: access.credentials ? "include" : "same-origin", | |
| 437 | }); | |
| 438 | if (response.status === 401 || response.status === 403) return handleUnauthorized(); | |
| 439 | if (!response.ok || !response.body) throw new Error(`stream ${response.status}`); | |
| 440 | const reader = response.body.getReader(); | |
| 441 | for (;;) { | |
| 442 | const { done, value } = await reader.read(); | |
| 443 | if (done || stopped) break; | |
| 444 | for (const chunk of demuxer.push(value)) { | |
| 445 | switch (chunk.type) { | |
| 446 | case "seed": | |
| 447 | void createImageBitmap(new Blob([chunk.payload as BlobPart], { type: "image/jpeg" })) | |
| 448 | .then((bitmap) => { | |
| 449 | paint(bitmap, bitmap.width, bitmap.height); | |
| 450 | bitmap.close(); | |
| 451 | }) | |
| 452 | .catch(() => {}); | |
| 453 | break; | |
| 454 | case "description": { | |
| 455 | awaitingKeyframe = true; | |
| 456 | const configured = await configureDecoder({ | |
| 457 | codec: avcCodecString(chunk.payload), | |
| 458 | description: chunk.payload, | |
| 459 | }); | |
| 460 | if (!configured) { | |
| 461 | await reader.cancel().catch(() => {}); | |
| 462 | fallBackToMjpeg(); | |
| 463 | return; | |
| 464 | } | |
| 465 | break; | |
| 466 | } | |
| 467 | case "keyframe": | |
| 468 | case "delta": | |
| 469 | decode(chunk.type === "keyframe", chunk.payload); | |
| 470 | break; | |
| 471 | } | |
| 472 | } | |
| 473 | } | |
| 474 | } catch (cause) { | |
| 475 | if (stopped) return; | |
| 476 | setStatus("connecting", (cause as Error).message); | |
| 477 | } | |
| 478 | if (!stopped) scheduleRetry("video", () => void readIosVideo()); | |
| 479 | }; | |
| 480 | ||
| 481 | /** | |
| 482 | * serve-sim's helper only accepts HID and pushes its screen config once | |
| 483 | * screen capture is running, and the AVCC stream does not reliably start | |
| 484 | * it. Touching the MJPEG endpoint does; one aborted request is enough. | |
| 485 | */ | |
| 486 | const primeIosHelper = async () => { | |
| 487 | const controller = new AbortController(); | |
| 488 | primeController = controller; | |
| 489 | const timeout = setTimeout(() => controller.abort(), 2_000); | |
| 490 | try { | |
| 491 | const response = await fetch(httpUrl(`/helper/${device}/stream.mjpeg`), { | |
| 492 | signal: controller.signal, | |
| 493 | credentials: access.credentials ? "include" : "same-origin", | |
| 494 | }); | |
| 495 | if (response.status === 401 || response.status === 403) return handleUnauthorized(); | |
| 496 | await response.body?.getReader().read(); | |
| 497 | } catch { | |
| 498 | // A failed prime just means the socket may take a retry to come up. | |
| 499 | } finally { | |
| 500 | clearTimeout(timeout); | |
| 501 | controller.abort(); | |
| 502 | if (primeController === controller) primeController = null; | |
| 503 | } | |
| 504 | }; | |
| 505 | ||
| 506 | // iOS input socket; also carries the screen config the helper pushes. | |
| 507 | const connectIosInput = async () => { | |
| 508 | if (stopped) return; | |
| 509 | await primeIosHelper(); | |
| 510 | if (stopped) return; | |
| 511 | const ws = new WebSocket(wsUrl(`/helper/ws?device=${device}`)); | |
| 512 | ws.binaryType = "arraybuffer"; | |
| 513 | socket = ws; | |
| 514 | ws.onopen = () => { | |
| 515 | ws.send(taggedJson(IOS_MSG_HARDWARE_KEYBOARD, { enabled: false })); | |
| 516 | events.onInputConnected(true); | |
| 517 | }; | |
| 518 | ws.onmessage = (event) => { | |
| 519 | if (!(event.data instanceof ArrayBuffer)) return; | |
| 520 | const bytes = new Uint8Array(event.data); | |
| 521 | if (bytes.length < 1 || bytes[0] !== IOS_TAG_SCREEN_CONFIG) return; | |
| 522 | try { | |
| 523 | const config = JSON.parse(decoder.decode(bytes.subarray(1))) as DeviceScreenSize; | |
| 524 | if (config.width > 0 && config.height > 0) { | |
| 525 | screen = config; | |
| 526 | events.onScreen(config); | |
| 527 | } | |
| 528 | } catch { | |
| 529 | // Ignore malformed config frames. | |
| 530 | } | |
| 531 | }; | |
| 532 | ws.onclose = (event) => { | |
| 533 | if (socket !== ws) return; | |
| 534 | socket = null; | |
| 535 | if (stopped) return; | |
| 536 | events.onInputConnected( | |
| 537 | false, | |
| 538 | event.reason || (event.code === 1006 ? "input socket refused" : `closed ${event.code}`), | |
| 539 | ); | |
| 540 | // A rejected HTTP upgrade surfaces as 1006, including an expired stream ticket. | |
| 541 | if ( | |
| 542 | event.code === 1008 || | |
| 543 | event.code === 4401 || | |
| 544 | (event.code === 1006 && access.query.wsTicket) | |
| 545 | ) | |
| 546 | return handleUnauthorized(); | |
| 547 | scheduleRetry("input", () => void connectIosInput()); | |
| 548 | }; | |
| 549 | ws.onerror = () => ws.close(); | |
| 550 | }; | |
| 551 | ||
| 552 | // Android: one socket for video and input. | |
| 553 | const connectAndroid = () => { | |
| 554 | if (stopped) return; | |
| 555 | const ws = new WebSocket(wsUrl(`/ws?device=${device}&frame-meta=1`)); | |
| 556 | ws.binaryType = "arraybuffer"; | |
| 557 | socket = ws; | |
| 558 | ws.onopen = () => { | |
| 559 | setStatus("connecting"); | |
| 560 | events.onInputConnected(true); | |
| 561 | }; | |
| 562 | ws.onmessage = (event) => { | |
| 563 | if (typeof event.data === "string") { | |
| 564 | // The encoder restarts at a new size when the device rotates; the | |
| 565 | // next keyframe carries a fresh SPS, so the decoder is rebuilt from it. | |
| 566 | if (isVideoSessionMessage(event.data)) closeDecoder(); | |
| 567 | return; | |
| 568 | } | |
| 569 | if (!(event.data instanceof ArrayBuffer)) return; | |
| 570 | const packet = parseSemuPacket(event.data); | |
| 571 | const needsScan = | |
| 572 | packet.isKey === null || | |
| 573 | (packet.isKey && (!videoDecoder || videoDecoder.state !== "configured")); | |
| 574 | const scanned = needsScan ? scanAccessUnit(packet.data) : null; | |
| 575 | const isKey = packet.isKey ?? scanned?.isKey ?? false; | |
| 576 | if (scanned?.sps && (!videoDecoder || videoDecoder.state !== "configured")) { | |
| 577 | if (configuring) return; | |
| 578 | configuring = true; | |
| 579 | void configureDecoder({ codec: avcCodecString(scanned.sps) }).then((configured) => { | |
| 580 | configuring = false; | |
| 581 | awaitingKeyframe = true; | |
| 582 | if (configured) requestKeyframe(); | |
| 583 | }); | |
| 584 | return; | |
| 585 | } | |
| 586 | if (!videoDecoder || videoDecoder.state !== "configured") { | |
| 587 | if (!isKey) requestKeyframe(); | |
| 588 | return; | |
| 589 | } | |
| 590 | decode(isKey, packet.data, packet.timestamp); | |
| 591 | }; | |
| 592 | ws.onclose = (event) => { | |
| 593 | if (socket !== ws) return; | |
| 594 | socket = null; | |
| 595 | closeDecoder(); | |
| 596 | if (stopped) return; | |
| 597 | events.onInputConnected(false, event.reason || `closed ${event.code}`); | |
| 598 | if ( | |
| 599 | event.code === 1008 || | |
| 600 | event.code === 4401 || | |
| 601 | (event.code === 1006 && access.query.wsTicket) | |
| 602 | ) | |
| 603 | return handleUnauthorized(); | |
| 604 | setStatus("connecting", event.reason || undefined); | |
| 605 | scheduleRetry("input", connectAndroid); | |
| 606 | }; | |
| 607 | ws.onerror = () => ws.close(); | |
| 608 | }; | |
| 609 | ||
| 610 | const start = () => { | |
| 611 | if (!stopped) return; | |
| 612 | stopped = false; | |
| 613 | firstFrame = false; | |
| 614 | events.onStatus("connecting"); | |
| 615 | if (platform === "ios") { | |
| 616 | void connectIosInput(); | |
| 617 | if (useWebCodecs) void readIosVideo(); | |
| 618 | else fallBackToMjpeg(); | |
| 619 | } else if (useWebCodecs) { | |
| 620 | connectAndroid(); | |
| 621 | } else { | |
| 622 | setStatus("error", "This browser cannot decode the Android stream (WebCodecs unavailable)."); | |
| 623 | } | |
| 624 | }; | |
| 625 | ||
| 626 | const stop = () => { | |
| 627 | if (stopped) return; | |
| 628 | stopped = true; | |
| 629 | mjpeg = false; | |
| 630 | for (const timer of retryTimers.values()) clearTimeout(timer); | |
| 631 | retryTimers.clear(); | |
| 632 | primeController?.abort(); | |
| 633 | primeController = null; | |
| 634 | controller?.abort(); | |
| 635 | controller = null; | |
| 636 | socket?.close(); | |
| 637 | socket = null; | |
| 638 | closeDecoder(); | |
| 639 | }; | |
| 640 | ||
| 641 | const send = (payload: Uint8Array<ArrayBuffer> | string) => { | |
| 642 | if (socket?.readyState === WebSocket.OPEN) socket.send(payload); | |
| 643 | }; | |
| 644 | ||
| 645 | const rawPoint = (x: number, y: number) => { | |
| 646 | // serve-sim streams the raw framebuffer; rotated devices need input | |
| 647 | // remapped into that raw space. | |
| 648 | if (platform !== "ios" || !screen || screen.width > screen.height) return { x, y }; | |
| 649 | switch (screen.orientation) { | |
| 650 | case "landscape_left": | |
| 651 | return { x: y, y: 1 - x }; | |
| 652 | case "landscape_right": | |
| 653 | return { x: 1 - y, y: x }; | |
| 654 | case "portrait_upside_down": | |
| 655 | return { x: 1 - x, y: 1 - y }; | |
| 656 | default: | |
| 657 | return { x, y }; | |
| 658 | } | |
| 659 | }; | |
| 660 | ||
| 661 | return { | |
| 662 | start, | |
| 663 | stop, | |
| 664 | sendTouch: (phase, x, y) => { | |
| 665 | if (platform === "ios") { | |
| 666 | send(taggedJson(IOS_MSG_TOUCH, { type: phase, ...rawPoint(x, y) })); | |
| 667 | return; | |
| 668 | } | |
| 669 | const action = phase === "begin" ? "down" : phase === "move" ? "move" : "up"; | |
| 670 | send(JSON.stringify({ type: "touch", action, x, y })); | |
| 671 | }, | |
| 672 | sendKey: (event, phase) => { | |
| 673 | if (platform === "ios") { | |
| 674 | const usage = hidUsageForCode(event.code); | |
| 675 | if (usage !== null) send(taggedJson(IOS_MSG_KEY, { type: phase, usage })); | |
| 676 | return; | |
| 677 | } | |
| 678 | if (phase !== "down") return; | |
| 679 | if (event.key === "Escape") return send(JSON.stringify({ type: "back" })); | |
| 680 | const keycode = ANDROID_KEYCODE_BY_KEY[event.key]; | |
| 681 | if (keycode !== undefined) return send(JSON.stringify({ type: "key", keycode })); | |
| 682 | if (event.key.length === 1 && !event.metaKey && !event.ctrlKey) { | |
| 683 | send(JSON.stringify({ type: "text", text: event.key })); | |
| 684 | } | |
| 685 | }, | |
| 686 | pressButton: (button) => { | |
| 687 | if (platform === "ios") { | |
| 688 | const name = | |
| 689 | button === "home" | |
| 690 | ? "home" | |
| 691 | : button === "appSwitcher" | |
| 692 | ? "app_switcher" | |
| 693 | : button === "power" | |
| 694 | ? "lock" | |
| 695 | : null; | |
| 696 | if (name) send(taggedJson(IOS_MSG_BUTTON, { button: name })); | |
| 697 | return; | |
| 698 | } | |
| 699 | const type = button === "appSwitcher" ? "recents" : button; | |
| 700 | if (type === "home" || type === "back" || type === "recents" || type === "power") { | |
| 701 | send(JSON.stringify({ type })); | |
| 702 | } | |
| 703 | }, | |
| 704 | rotate: () => { | |
| 705 | if (platform !== "ios") return; | |
| 706 | const current = screen?.orientation ?? "portrait"; | |
| 707 | const next = | |
| 708 | IOS_ORIENTATIONS[(IOS_ORIENTATIONS.indexOf(current) + 1) % IOS_ORIENTATIONS.length]!; | |
| 709 | send(taggedJson(IOS_MSG_ORIENTATION, { orientation: next })); | |
| 710 | }, | |
| 711 | }; | |
| 712 | } |
packages/client-runtime/src/state/deviceHubAccess.ts+3-18
| ... | ... | @@ -20,19 +20,11 @@ import { environmentEndpointUrl } from "../environment/endpoint.ts"; |
| 20 | 20 | import { ManagedRelayDpopSigner } from "../relay/managedRelay.ts"; |
| 21 | 21 | import type { RemoteEnvironmentRequestError } from "../rpc/http.ts"; |
| 22 | 22 | import { executeAuthenticatedEnvironmentHttpRequest } from "./environmentHttpAuth.ts"; |
| 23 | import type { DeviceHubAccess } from "../device/hubAccess.ts"; | |
| 23 | 24 | |
| 24 | const TICKET_TIMEOUT_MS = 8_000; | |
| 25 | export { type DeviceHubAccess, withDeviceHubQuery } from "../device/hubAccess.ts"; | |
| 25 | 26 | |
| 26 | export interface DeviceHubAccess { | |
| 27 | /** Absolute origin-relative base, e.g. `https://env.example/api/device-hub`. */ | |
| 28 | readonly httpBase: string; | |
| 29 | /** Same base with the `ws(s)` scheme. */ | |
| 30 | readonly wsBase: string; | |
| 31 | /** Query parameters to append to every hub request; empty for cookie sessions. */ | |
| 32 | readonly query: Readonly<Record<string, string>>; | |
| 33 | /** Whether requests must include cookies (same-origin session). */ | |
| 34 | readonly credentials: boolean; | |
| 35 | } | |
| 27 | const TICKET_TIMEOUT_MS = 8_000; | |
| 36 | 28 | |
| 37 | 29 | export const resolveDeviceHubAccess = Effect.fn("clientRuntime.state.resolveDeviceHubAccess")( |
| 38 | 30 | function* (input: { |
| ... | ... | @@ -64,10 +56,3 @@ export const resolveDeviceHubAccess = Effect.fn("clientRuntime.state.resolveDevi |
| 64 | 56 | }; |
| 65 | 57 | }, |
| 66 | 58 | ); |
| 67 | ||
| 68 | export const withDeviceHubQuery = (url: string, access: DeviceHubAccess): string => { | |
| 69 | const entries = Object.entries(access.query); | |
| 70 | if (entries.length === 0) return url; | |
| 71 | const separator = url.includes("?") ? "&" : "?"; | |
| 72 | return `${url}${separator}${new URLSearchParams(entries).toString()}`; | |
| 73 | }; |