From 9e9eefa9e34d0067f716831ef397d23ecae5c895 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Mon, 5 Oct 2026 19:57:26 -0700 Subject: [PATCH] Enable separate Windows surfaces and reliable native cursor reconnects Add bounded Windows Graphics Capture surfaces and dirty rectangles, synchronized browser composition, cached caption dragging, and observed stream metrics. Keep the experiment opt-in on Windows 10 build 19041+ and Windows 11, with desktop fallback. Use WebRTC congestion feedback and retransmission, coalesce video work, preserve reliable held-button input, and refresh native cursors on connect. Retire guest channels safely and release CUDA contexts when viewers close. Correct libvirt channel paths for long VM names. Validated 25 native GStreamer/WebRTC tests, 14 VM boundary tests, Windows x86/x64/ARM64 builds and cursor sanitizer checks, frontend check/build, production Windows 10/11 composition and reconnects, GPU release, and unchanged inference processes/driver/Nomad. Assisted-by: gpt-6.1-sol --- dashboard/web/components/VMConsole.tsx | 155 +++++++++- dashboard/web/pages/VMs.css | 5 +- dashboard/web/styles.css | 2 +- guest/DESIGN.md | 27 +- guest/build.py | 4 +- guest/capture.c | 350 ++++++++++++++++++++++ guest/capture.h | 9 + guest/protocol.json | 12 +- guest/windows.c | 152 ++++++++-- nixos/configuration.nix | 2 +- tools/dashboard-vms-test.py | 12 + tools/vm-screen-test.py | 222 +++++++++++++- tools/vm-screen.py | 384 ++++++++++++++++++++++--- tools/vms.py | 2 +- 14 files changed, 1259 insertions(+), 79 deletions(-) create mode 100644 guest/capture.c create mode 100644 guest/capture.h diff --git a/dashboard/web/components/VMConsole.tsx b/dashboard/web/components/VMConsole.tsx index 09413ba5b00acc3f8d0136f1a404ef4106368a0b..602099de40a6935c43359cf99f3ebaf389917cbc 100644 --- a/dashboard/web/components/VMConsole.tsx +++ b/dashboard/web/components/VMConsole.tsx @@ -5,16 +5,30 @@ import Keyboard from "lucide-solid/icons/keyboard"; import Clipboard from "lucide-solid/icons/clipboard"; import Maximize from "lucide-solid/icons/maximize"; import RefreshCw from "lucide-solid/icons/refresh-cw"; +import Check from "lucide-solid/icons/check"; import RemoteKeyboard from "@novnc/keyboard"; +type WindowSurface = { + id: number; x: number; y: number; width: number; height: number; atlas_x: number; atlas_y: number; + caption_left: number; caption_top: number; caption_right: number; caption_bottom: number; +}; +type WindowFrame = { windows: WindowSurface[]; rtpTimestamp: number }; + export function VMConsole(props: { name: string; enabled: boolean; active: boolean; toolbar: JSX.Element; content: (toolbar: HTMLDivElement) => JSX.Element }) { const enabled = createMemo(() => props.enabled); let attempt = 0; let video!: HTMLVideoElement; + let canvas!: HTMLCanvasElement; let cursorImage!: HTMLImageElement; let cursorInvert!: HTMLImageElement; let panel!: HTMLDivElement; let input: RTCDataChannel | undefined; + let movement: RTCDataChannel | undefined; + let pointerSequence = 0; + let windowFrame: WindowFrame | undefined; + let pendingFrames: WindowFrame[] = []; + let frameTimestamp: number | undefined; + let drag: { id: number; startX: number; startY: number; x: number; y: number; dx: number; dy: number; released: number } | undefined; let disconnect = () => {}; let width = 0; let height = 0; @@ -55,9 +69,38 @@ export function VMConsole(props: { name: string; enabled: boolean; active: boole const [problem, setProblem] = createSignal(""); const [clipboard, setClipboard] = createSignal(""); const [showClipboard, setShowClipboard] = createSignal(false); + const [windowsAvailable, setWindowsAvailable] = createSignal(false); + const [separateWindows, setSeparateWindows] = createSignal(false); + const [frameSync, setFrameSync] = createSignal(false); + const [windowCapture, setWindowCapture] = createSignal(false); + const [showStats, setShowStats] = createSignal(false); + const [stats, setStats] = createSignal<{ fps: number; bitrate: number; lost: number; dropped: number; delay: number; rtt: number }>(); + const [capture, setCapture] = createSignal<{ fps: number; capture: number; scan: number; write: number }>(); const [tools, setTools] = createSignal(); const send = (message: object) => { if (input?.readyState === "open") input.send(JSON.stringify(message)); }; - const release = () => { buttons = 0; send({ type: "release" }); }; + const compose = () => { + const frame = windowFrame; + const composed = !!frame?.windows.length; + video.classList.toggle("vm-video-composed", composed); + canvas.hidden = !composed; + if (!frame || !composed || !width || !height || video.readyState < 2) return; + if (canvas.width !== width || canvas.height !== height) { canvas.width = width; canvas.height = height; } + const context = canvas.getContext("2d", { alpha: false }); + if (!context) return; + context.fillStyle = "#101014"; + context.fillRect(0, 0, width, height); + for (const window of [...frame.windows].reverse()) { + let x = window.x, y = window.y; + if (drag?.id === window.id) { + const acknowledged = Math.abs(window.x - drag.x - drag.dx) <= 3 && Math.abs(window.y - drag.y - drag.dy) <= 3; + if (drag.released && (acknowledged || performance.now() - drag.released > 500)) drag = undefined; + else { x = drag.x + drag.dx; y = drag.y + drag.dy; } + } + context.drawImage(video, window.atlas_x, window.atlas_y, window.width, window.height, x, y, window.width, window.height); + } + canvas.dataset.windows = String(frame.windows.length); + }; + const release = () => { buttons = 0; drag = undefined; compose(); send({ type: "release", sequence: ++pointerSequence }); }; const connect = async () => { const current = ++attempt; disconnect(); @@ -81,6 +124,45 @@ export function VMConsole(props: { name: string; enabled: boolean; active: boole video.addEventListener("focus", focus); video.addEventListener("blur", blur); let peer: RTCPeerConnection | undefined; + let frameCallback: number | undefined; + let statsTimer: number | undefined; + let previousStats: { time: number; bytes: number; frames: number } | undefined; + const frameReady: VideoFrameRequestCallback = (_, metadata) => { + if (current !== attempt || disposed) return; + frameTimestamp = (metadata as VideoFrameCallbackMetadata & { rtpTimestamp?: number }).rtpTimestamp; + setFrameSync(Number.isFinite(frameTimestamp)); + if (!frameSync()) { + frameTimestamp = undefined; windowFrame = undefined; pendingFrames = []; + if (separateWindows()) { setSeparateWindows(false); send({ type: "experiment", enabled: false }); } + } + while (pendingFrames[0] && frameTimestamp !== undefined && ((frameTimestamp - pendingFrames[0].rtpTimestamp) | 0) >= 0) { + windowFrame = pendingFrames.shift(); + } + compose(); + frameCallback = video.requestVideoFrameCallback(frameReady); + }; + const measure = async () => { + if (!peer || current !== attempt || disposed) return; + try { + const report = await peer.getStats(); + if (current !== attempt || disposed || peer.connectionState === "closed") return; + let rtt = 0; + report.forEach((value) => { if (value.type === "candidate-pair" && value.state === "succeeded" && value.nominated) rtt = (value.currentRoundTripTime ?? 0) * 1000; }); + report.forEach((value) => { + if (value.type !== "inbound-rtp" || value.kind !== "video") return; + const elapsed = previousStats ? (value.timestamp - previousStats.time) / 1000 : 0; + setStats({ fps: elapsed ? (value.framesDecoded - previousStats!.frames) / elapsed : 0, + bitrate: elapsed ? (value.bytesReceived - previousStats!.bytes) * 8 / elapsed / 1000000 : 0, + lost: value.packetsLost ?? 0, dropped: value.framesDropped ?? 0, + delay: value.jitterBufferEmittedCount ? value.jitterBufferDelay / value.jitterBufferEmittedCount * 1000 : 0, rtt }); + previousStats = { time: value.timestamp, bytes: value.bytesReceived, frames: value.framesDecoded }; + }); + } catch { + if (peer.connectionState !== "closed") return; + } finally { + if (current === attempt && !disposed && peer.connectionState !== "closed") statsTimer = window.setTimeout(measure, 1000); + } + }; let pending = ""; let messages = Promise.resolve(); const decoder = new TextDecoder(); @@ -101,6 +183,12 @@ export function VMConsole(props: { name: string; enabled: boolean; active: boole socket.onclose = null; socket.close(); peer?.close(); + clearTimeout(statsTimer); + if (frameCallback !== undefined) video.cancelVideoFrameCallback(frameCallback); + windowFrame = undefined; pendingFrames = []; frameTimestamp = undefined; drag = undefined; + movement = undefined; pointerSequence = 0; + setFrameSync(false); setWindowsAvailable(false); setWindowCapture(false); setStats(undefined); setCapture(undefined); + compose(); video.onplaying = null; video.srcObject = null; clearTimeout(cursorTimer); @@ -125,11 +213,15 @@ export function VMConsole(props: { name: string; enabled: boolean; active: boole if (message.type === "ready") { peer = new RTCPeerConnection({ iceServers: message.iceServers }); const transceiver = peer.addTransceiver("video", { direction: "recvonly" }); - const codecs = RTCRtpReceiver.getCapabilities("video")?.codecs.filter((codec) => codec.mimeType === "video/H264"); + const codecs = RTCRtpReceiver.getCapabilities("video")?.codecs.filter((codec) => codec.mimeType === "video/H264" || codec.mimeType === "video/rtx"); if (codecs?.length) transceiver.setCodecPreferences(codecs); input = peer.createDataChannel("input"); + movement = peer.createDataChannel("pointer", { ordered: false, maxRetransmits: 0 }); keyboard.onkeyevent = (key, _code, down) => send({ type: "key", key, down }); - input.onopen = () => { if (document.activeElement === video) focus(); }; + input.onopen = () => { + if (document.activeElement === video) focus(); + if (windowsAvailable()) send({ type: "experiment", enabled: separateWindows() }); + }; peer.onicecandidate = ({ candidate }) => { if (candidate) signal({ type: "candidate", ...candidate.toJSON() }); }; peer.onconnectionstatechange = () => { if (peer?.connectionState === "failed" || peer?.connectionState === "disconnected") { @@ -137,7 +229,11 @@ export function VMConsole(props: { name: string; enabled: boolean; active: boole } }; peer.ontrack = ({ track }) => { + const receiver = transceiver.receiver as RTCRtpReceiver & { jitterBufferTarget?: number }; + if ("jitterBufferTarget" in receiver) receiver.jitterBufferTarget = 0; video.srcObject = new MediaStream([track]); + if (video.requestVideoFrameCallback) frameCallback = video.requestVideoFrameCallback(frameReady); + void measure(); void video.play().catch(() => { if (current === attempt && video.srcObject) fail("The browser couldn't play the screen. Reconnect to try again."); }); @@ -155,6 +251,27 @@ export function VMConsole(props: { name: string; enabled: boolean; active: boole height = message.height; video.dataset.encoder = message.encoder; video.dataset.source = message.source ?? "qemu"; + } else if (message.type === "capabilities") { + setWindowsAvailable(message.windows); + if (message.windows) send({ type: "experiment", enabled: separateWindows() }); + } else if (message.type === "window-capture") { + setWindowCapture(message.available); + } else if (message.type === "windows") { + pendingFrames.push(message); + if (pendingFrames.length > 16) pendingFrames.shift(); + if (frameTimestamp !== undefined && ((frameTimestamp - message.rtpTimestamp) | 0) >= 0) { + windowFrame = message; pendingFrames = []; compose(); + } + } else if (message.type === "window-positions") { + if (windowFrame?.windows.length && windowFrame.windows.length === message.windows.length && + windowFrame.windows.every((window) => message.windows.some((next: WindowSurface) => window.id === next.id && window.width === next.width && window.height === next.height))) { + windowFrame = { ...windowFrame, windows: message.windows }; compose(); + } + } else if (message.type === "capture-stats") { + setCapture({ fps: message.frames * 1000 / message.elapsed_ms, + capture: message.frames ? message.capture_us / message.frames / 1000 : 0, + scan: message.frames ? message.scan_us / message.frames / 1000 : 0, + write: message.frames ? message.write_us / message.frames / 1000 : 0 }); } else if (message.type === "cursor") { if (message.frame === 0) { clearTimeout(cursorTimer); @@ -193,6 +310,7 @@ export function VMConsole(props: { name: string; enabled: boolean; active: boole if (current !== attempt || disposed) return; clearTimeout(timeout); setState("Connected"); + compose(); if (props.active) video.focus(); }; } catch (failure) { @@ -225,7 +343,18 @@ export function VMConsole(props: { name: string; enabled: boolean; active: boole cursorInside = x >= 0 && y >= 0 && x < width && y < height; drawCursor(); if ((event.type === "pointerdown" || event.type === "wheel") && (x < 0 || y < 0 || x >= width || y >= height)) return false; - send({ type: "pointer", x, y, buttons: mask }); + if (event.type === "pointerdown" && mask === 1) { + const window = windowFrame?.windows.find((window) => x >= window.x && y >= window.y && x < window.x + window.width && y < window.y + window.height); + if (window && x >= window.caption_left && x < window.caption_right && y >= window.caption_top && y < window.caption_bottom) { + drag = { id: window.id, startX: x, startY: y, x: window.x, y: window.y, dx: 0, dy: 0, released: 0 }; + } + } + if (drag && event.type === "pointermove" && buttons === 1) { drag.dx = x - drag.startX; drag.dy = y - drag.startY; compose(); } + if (drag && event.type === "pointerup" && !mask) { drag.released = performance.now(); window.setTimeout(compose, 510); } + const message = JSON.stringify({ type: "pointer", x, y, buttons: mask, sequence: ++pointerSequence }); + if (event.type === "pointermove" && !mask && movement?.readyState === "open") { + if (movement.bufferedAmount < 16384) movement.send(message); + } else if (input?.readyState === "open" && (event.type !== "pointermove" || input.bufferedAmount < 16384)) input.send(message); return true; }; const ctrlAltDel = () => { @@ -241,6 +370,12 @@ export function VMConsole(props: { name: string; enabled: boolean; active: boole + + @@ -248,6 +383,13 @@ export function VMConsole(props: { name: string; enabled: boolean; active: boole {props.enabled ? state() : "Off"} + +
+ {(value) => <>{value().fps.toFixed(1)} FPS{value().bitrate.toFixed(2)} Mbps{value().lost} packets lost{value().dropped} frames dropped{value().rtt.toFixed(0)} ms round trip{value().delay.toFixed(0)} ms buffer} + {(value) => <>Guest {value().fps.toFixed(1)} FPSDesktop capture {value().capture.toFixed(1)} msCompare {value().scan.toFixed(1)} msStream work {value().write.toFixed(1)} ms} + {windowCapture() ? "Separate windows" : "Desktop fallback"} +
+
@@ -260,6 +402,7 @@ export function VMConsole(props: { name: string; enabled: boolean; active: boole