diff --git a/dashboard/web/components/VMConsole.tsx b/dashboard/web/components/VMConsole.tsx index 09413ba5b00acc3f8d0136f1a404ef4106368a0b..602099de40a6935c43359cf99f3ebaf389917cbc 100644 --- a/dashboard/web/components/VMConsole.tsx +++ b/dashboard/web/components/VMConsole.tsx @@ -5,16 +5,30 @@ import Keyboard from "lucide-solid/icons/keyboard"; import Clipboard from "lucide-solid/icons/clipboard"; import Maximize from "lucide-solid/icons/maximize"; import RefreshCw from "lucide-solid/icons/refresh-cw"; +import Check from "lucide-solid/icons/check"; import RemoteKeyboard from "@novnc/keyboard"; +type WindowSurface = { + id: number; x: number; y: number; width: number; height: number; atlas_x: number; atlas_y: number; + caption_left: number; caption_top: number; caption_right: number; caption_bottom: number; +}; +type WindowFrame = { windows: WindowSurface[]; rtpTimestamp: number }; + export function VMConsole(props: { name: string; enabled: boolean; active: boolean; toolbar: JSX.Element; content: (toolbar: HTMLDivElement) => JSX.Element }) { const enabled = createMemo(() => props.enabled); let attempt = 0; let video!: HTMLVideoElement; + let canvas!: HTMLCanvasElement; let cursorImage!: HTMLImageElement; let cursorInvert!: HTMLImageElement; let panel!: HTMLDivElement; let input: RTCDataChannel | undefined; + let movement: RTCDataChannel | undefined; + let pointerSequence = 0; + let windowFrame: WindowFrame | undefined; + let pendingFrames: WindowFrame[] = []; + let frameTimestamp: number | undefined; + let drag: { id: number; startX: number; startY: number; x: number; y: number; dx: number; dy: number; released: number } | undefined; let disconnect = () => {}; let width = 0; let height = 0; @@ -55,9 +69,38 @@ export function VMConsole(props: { name: string; enabled: boolean; active: boole const [problem, setProblem] = createSignal(""); const [clipboard, setClipboard] = createSignal(""); const [showClipboard, setShowClipboard] = createSignal(false); + const [windowsAvailable, setWindowsAvailable] = createSignal(false); + const [separateWindows, setSeparateWindows] = createSignal(false); + const [frameSync, setFrameSync] = createSignal(false); + const [windowCapture, setWindowCapture] = createSignal(false); + const [showStats, setShowStats] = createSignal(false); + const [stats, setStats] = createSignal<{ fps: number; bitrate: number; lost: number; dropped: number; delay: number; rtt: number }>(); + const [capture, setCapture] = createSignal<{ fps: number; capture: number; scan: number; write: number }>(); const [tools, setTools] = createSignal(); const send = (message: object) => { if (input?.readyState === "open") input.send(JSON.stringify(message)); }; - const release = () => { buttons = 0; send({ type: "release" }); }; + const compose = () => { + const frame = windowFrame; + const composed = !!frame?.windows.length; + video.classList.toggle("vm-video-composed", composed); + canvas.hidden = !composed; + if (!frame || !composed || !width || !height || video.readyState < 2) return; + if (canvas.width !== width || canvas.height !== height) { canvas.width = width; canvas.height = height; } + const context = canvas.getContext("2d", { alpha: false }); + if (!context) return; + context.fillStyle = "#101014"; + context.fillRect(0, 0, width, height); + for (const window of [...frame.windows].reverse()) { + let x = window.x, y = window.y; + if (drag?.id === window.id) { + const acknowledged = Math.abs(window.x - drag.x - drag.dx) <= 3 && Math.abs(window.y - drag.y - drag.dy) <= 3; + if (drag.released && (acknowledged || performance.now() - drag.released > 500)) drag = undefined; + else { x = drag.x + drag.dx; y = drag.y + drag.dy; } + } + context.drawImage(video, window.atlas_x, window.atlas_y, window.width, window.height, x, y, window.width, window.height); + } + canvas.dataset.windows = String(frame.windows.length); + }; + const release = () => { buttons = 0; drag = undefined; compose(); send({ type: "release", sequence: ++pointerSequence }); }; const connect = async () => { const current = ++attempt; disconnect(); @@ -81,6 +124,45 @@ export function VMConsole(props: { name: string; enabled: boolean; active: boole video.addEventListener("focus", focus); video.addEventListener("blur", blur); let peer: RTCPeerConnection | undefined; + let frameCallback: number | undefined; + let statsTimer: number | undefined; + let previousStats: { time: number; bytes: number; frames: number } | undefined; + const frameReady: VideoFrameRequestCallback = (_, metadata) => { + if (current !== attempt || disposed) return; + frameTimestamp = (metadata as VideoFrameCallbackMetadata & { rtpTimestamp?: number }).rtpTimestamp; + setFrameSync(Number.isFinite(frameTimestamp)); + if (!frameSync()) { + frameTimestamp = undefined; windowFrame = undefined; pendingFrames = []; + if (separateWindows()) { setSeparateWindows(false); send({ type: "experiment", enabled: false }); } + } + while (pendingFrames[0] && frameTimestamp !== undefined && ((frameTimestamp - pendingFrames[0].rtpTimestamp) | 0) >= 0) { + windowFrame = pendingFrames.shift(); + } + compose(); + frameCallback = video.requestVideoFrameCallback(frameReady); + }; + const measure = async () => { + if (!peer || current !== attempt || disposed) return; + try { + const report = await peer.getStats(); + if (current !== attempt || disposed || peer.connectionState === "closed") return; + let rtt = 0; + report.forEach((value) => { if (value.type === "candidate-pair" && value.state === "succeeded" && value.nominated) rtt = (value.currentRoundTripTime ?? 0) * 1000; }); + report.forEach((value) => { + if (value.type !== "inbound-rtp" || value.kind !== "video") return; + const elapsed = previousStats ? (value.timestamp - previousStats.time) / 1000 : 0; + setStats({ fps: elapsed ? (value.framesDecoded - previousStats!.frames) / elapsed : 0, + bitrate: elapsed ? (value.bytesReceived - previousStats!.bytes) * 8 / elapsed / 1000000 : 0, + lost: value.packetsLost ?? 0, dropped: value.framesDropped ?? 0, + delay: value.jitterBufferEmittedCount ? value.jitterBufferDelay / value.jitterBufferEmittedCount * 1000 : 0, rtt }); + previousStats = { time: value.timestamp, bytes: value.bytesReceived, frames: value.framesDecoded }; + }); + } catch { + if (peer.connectionState !== "closed") return; + } finally { + if (current === attempt && !disposed && peer.connectionState !== "closed") statsTimer = window.setTimeout(measure, 1000); + } + }; let pending = ""; let messages = Promise.resolve(); const decoder = new TextDecoder(); @@ -101,6 +183,12 @@ export function VMConsole(props: { name: string; enabled: boolean; active: boole socket.onclose = null; socket.close(); peer?.close(); + clearTimeout(statsTimer); + if (frameCallback !== undefined) video.cancelVideoFrameCallback(frameCallback); + windowFrame = undefined; pendingFrames = []; frameTimestamp = undefined; drag = undefined; + movement = undefined; pointerSequence = 0; + setFrameSync(false); setWindowsAvailable(false); setWindowCapture(false); setStats(undefined); setCapture(undefined); + compose(); video.onplaying = null; video.srcObject = null; clearTimeout(cursorTimer); @@ -125,11 +213,15 @@ export function VMConsole(props: { name: string; enabled: boolean; active: boole if (message.type === "ready") { peer = new RTCPeerConnection({ iceServers: message.iceServers }); const transceiver = peer.addTransceiver("video", { direction: "recvonly" }); - const codecs = RTCRtpReceiver.getCapabilities("video")?.codecs.filter((codec) => codec.mimeType === "video/H264"); + const codecs = RTCRtpReceiver.getCapabilities("video")?.codecs.filter((codec) => codec.mimeType === "video/H264" || codec.mimeType === "video/rtx"); if (codecs?.length) transceiver.setCodecPreferences(codecs); input = peer.createDataChannel("input"); + movement = peer.createDataChannel("pointer", { ordered: false, maxRetransmits: 0 }); keyboard.onkeyevent = (key, _code, down) => send({ type: "key", key, down }); - input.onopen = () => { if (document.activeElement === video) focus(); }; + input.onopen = () => { + if (document.activeElement === video) focus(); + if (windowsAvailable()) send({ type: "experiment", enabled: separateWindows() }); + }; peer.onicecandidate = ({ candidate }) => { if (candidate) signal({ type: "candidate", ...candidate.toJSON() }); }; peer.onconnectionstatechange = () => { if (peer?.connectionState === "failed" || peer?.connectionState === "disconnected") { @@ -137,7 +229,11 @@ export function VMConsole(props: { name: string; enabled: boolean; active: boole } }; peer.ontrack = ({ track }) => { + const receiver = transceiver.receiver as RTCRtpReceiver & { jitterBufferTarget?: number }; + if ("jitterBufferTarget" in receiver) receiver.jitterBufferTarget = 0; video.srcObject = new MediaStream([track]); + if (video.requestVideoFrameCallback) frameCallback = video.requestVideoFrameCallback(frameReady); + void measure(); void video.play().catch(() => { if (current === attempt && video.srcObject) fail("The browser couldn't play the screen. Reconnect to try again."); }); @@ -155,6 +251,27 @@ export function VMConsole(props: { name: string; enabled: boolean; active: boole height = message.height; video.dataset.encoder = message.encoder; video.dataset.source = message.source ?? "qemu"; + } else if (message.type === "capabilities") { + setWindowsAvailable(message.windows); + if (message.windows) send({ type: "experiment", enabled: separateWindows() }); + } else if (message.type === "window-capture") { + setWindowCapture(message.available); + } else if (message.type === "windows") { + pendingFrames.push(message); + if (pendingFrames.length > 16) pendingFrames.shift(); + if (frameTimestamp !== undefined && ((frameTimestamp - message.rtpTimestamp) | 0) >= 0) { + windowFrame = message; pendingFrames = []; compose(); + } + } else if (message.type === "window-positions") { + if (windowFrame?.windows.length && windowFrame.windows.length === message.windows.length && + windowFrame.windows.every((window) => message.windows.some((next: WindowSurface) => window.id === next.id && window.width === next.width && window.height === next.height))) { + windowFrame = { ...windowFrame, windows: message.windows }; compose(); + } + } else if (message.type === "capture-stats") { + setCapture({ fps: message.frames * 1000 / message.elapsed_ms, + capture: message.frames ? message.capture_us / message.frames / 1000 : 0, + scan: message.frames ? message.scan_us / message.frames / 1000 : 0, + write: message.frames ? message.write_us / message.frames / 1000 : 0 }); } else if (message.type === "cursor") { if (message.frame === 0) { clearTimeout(cursorTimer); @@ -193,6 +310,7 @@ export function VMConsole(props: { name: string; enabled: boolean; active: boole if (current !== attempt || disposed) return; clearTimeout(timeout); setState("Connected"); + compose(); if (props.active) video.focus(); }; } catch (failure) { @@ -225,7 +343,18 @@ export function VMConsole(props: { name: string; enabled: boolean; active: boole cursorInside = x >= 0 && y >= 0 && x < width && y < height; drawCursor(); if ((event.type === "pointerdown" || event.type === "wheel") && (x < 0 || y < 0 || x >= width || y >= height)) return false; - send({ type: "pointer", x, y, buttons: mask }); + if (event.type === "pointerdown" && mask === 1) { + const window = windowFrame?.windows.find((window) => x >= window.x && y >= window.y && x < window.x + window.width && y < window.y + window.height); + if (window && x >= window.caption_left && x < window.caption_right && y >= window.caption_top && y < window.caption_bottom) { + drag = { id: window.id, startX: x, startY: y, x: window.x, y: window.y, dx: 0, dy: 0, released: 0 }; + } + } + if (drag && event.type === "pointermove" && buttons === 1) { drag.dx = x - drag.startX; drag.dy = y - drag.startY; compose(); } + if (drag && event.type === "pointerup" && !mask) { drag.released = performance.now(); window.setTimeout(compose, 510); } + const message = JSON.stringify({ type: "pointer", x, y, buttons: mask, sequence: ++pointerSequence }); + if (event.type === "pointermove" && !mask && movement?.readyState === "open") { + if (movement.bufferedAmount < 16384) movement.send(message); + } else if (input?.readyState === "open" && (event.type !== "pointermove" || input.bufferedAmount < 16384)) input.send(message); return true; }; const ctrlAltDel = () => { @@ -241,6 +370,12 @@ export function VMConsole(props: { name: string; enabled: boolean; active: boole + + @@ -248,6 +383,13 @@ export function VMConsole(props: { name: string; enabled: boolean; active: boole {props.enabled ? state() : "Off"} + +
+ {(value) => <>{value().fps.toFixed(1)} FPS{value().bitrate.toFixed(2)} Mbps{value().lost} packets lost{value().dropped} frames dropped{value().rtt.toFixed(0)} ms round trip{value().delay.toFixed(0)} ms buffer} + {(value) => <>Guest {value().fps.toFixed(1)} FPSDesktop capture {value().capture.toFixed(1)} msCompare {value().scan.toFixed(1)} msStream work {value().write.toFixed(1)} ms} + {windowCapture() ? "Separate windows" : "Desktop fallback"} +
+
@@ -260,6 +402,7 @@ export function VMConsole(props: { name: string; enabled: boolean; active: boole