1import { VMMenu } from "./VMMenu.tsx";
2import { createEffect, createMemo, createSignal, type JSX, onCleanup, Show } from "solid-js";
3import { reason } from "../api.ts";
4import Keyboard from "lucide-solid/icons/keyboard";
5import Clipboard from "lucide-solid/icons/clipboard";
6import Maximize from "lucide-solid/icons/maximize";
7import RefreshCw from "lucide-solid/icons/refresh-cw";
8import Check from "lucide-solid/icons/check";
9import RemoteKeyboard from "@novnc/keyboard";
10
11type WindowSurface = {
12 id: number; x: number; y: number; width: number; height: number; atlas_x: number; atlas_y: number;
13 caption_left: number; caption_top: number; caption_right: number; caption_bottom: number;
14};
15type WindowFrame = { windows: WindowSurface[]; rtpTimestamp: number };
16
17export function VMConsole(props: { name: string; enabled: boolean; active: boolean; toolbar: JSX.Element; content: (toolbar: HTMLDivElement) => JSX.Element }) {
18 const enabled = createMemo(() => props.enabled);
19 let attempt = 0;
20 let video!: HTMLVideoElement;
21 let canvas!: HTMLCanvasElement;
22 let cursorImage!: HTMLImageElement;
23 let cursorInvert!: HTMLImageElement;
24 let panel!: HTMLDivElement;
25 let input: RTCDataChannel | undefined;
26 let movement: RTCDataChannel | undefined;
27 let pointerSequence = 0;
28 let windowFrame: WindowFrame | undefined;
29 let pendingFrames: WindowFrame[] = [];
30 let frameTimestamp: number | undefined;
31 let drag: { id: number; startX: number; startY: number; x: number; y: number; dx: number; dy: number; released: number } | undefined;
32 let disconnect = () => {};
33 let width = 0;
34 let height = 0;
35 let buttons = 0;
36 let disposed = false;
37 let cursorFrames: { image: string; invert?: string; duration: number }[] = [];
38 let cursorFrame = 0;
39 let cursorTimer: number | undefined;
40 let cursorX = 0;
41 let cursorY = 0;
42 let pointerX = 0;
43 let pointerY = 0;
44 let cursorInside = false;
45 let cursorVisible = true;
46 const drawCursor = () => {
47 const frame = cursorFrames[cursorFrame];
48 const overlay = !!frame?.invert && cursorInside && cursorVisible;
49 cursorImage.style.display = cursorInvert.style.display = overlay ? "block" : "none";
50 if (!frame || !cursorVisible) { video.style.cursor = cursorVisible ? "default" : "none"; return; }
51 video.style.cursor = frame.invert ? (cursorInside ? "none" : "default") : `url(${frame.image}) ${cursorX} ${cursorY}, default`;
52 if (overlay) {
53 cursorImage.src = frame.image;
54 cursorInvert.src = frame.invert!;
55 cursorImage.style.transform = cursorInvert.style.transform = `translate(${pointerX - cursorX}px, ${pointerY - cursorY}px)`;
56 }
57 };
58 const animateCursor = () => {
59 const frame = cursorFrames[cursorFrame];
60 if (!frame) return;
61 cursorTimer = window.setTimeout(() => {
62 cursorFrame = (cursorFrame + 1) % cursorFrames.length;
63 drawCursor();
64 animateCursor();
65 }, frame.duration);
66 };
67 const [state, setState] = createSignal<"Connecting…" | "Connected" | "Disconnected">("Connecting…");
68 const connected = () => state() === "Connected";
69 const [problem, setProblem] = createSignal("");
70 const [clipboard, setClipboard] = createSignal("");
71 const [showClipboard, setShowClipboard] = createSignal(false);
72 const [windowsAvailable, setWindowsAvailable] = createSignal(false);
73 const [separateWindows, setSeparateWindows] = createSignal(false);
74 const [frameSync, setFrameSync] = createSignal(false);
75 const [windowCapture, setWindowCapture] = createSignal(false);
76 const [showStats, setShowStats] = createSignal(false);
77 const [stats, setStats] = createSignal<{ fps: number; bitrate: number; lost: number; dropped: number; delay: number; rtt: number }>();
78 const [capture, setCapture] = createSignal<{ fps: number; capture: number; scan: number; write: number }>();
79 const [tools, setTools] = createSignal<HTMLDivElement>();
80 const send = (message: object) => { if (input?.readyState === "open") input.send(JSON.stringify(message)); };
81 const compose = () => {
82 const frame = windowFrame;
83 const composed = !!frame?.windows.length;
84 video.classList.toggle("vm-video-composed", composed);
85 canvas.hidden = !composed;
86 if (!frame || !composed || !width || !height || video.readyState < 2) return;
87 if (canvas.width !== width || canvas.height !== height) { canvas.width = width; canvas.height = height; }
88 const context = canvas.getContext("2d", { alpha: false });
89 if (!context) return;
90 context.fillStyle = "#101014";
91 context.fillRect(0, 0, width, height);
92 for (const window of [...frame.windows].reverse()) {
93 let x = window.x, y = window.y;
94 if (drag?.id === window.id) {
95 const acknowledged = Math.abs(window.x - drag.x - drag.dx) <= 3 && Math.abs(window.y - drag.y - drag.dy) <= 3;
96 if (drag.released && (acknowledged || performance.now() - drag.released > 500)) drag = undefined;
97 else { x = drag.x + drag.dx; y = drag.y + drag.dy; }
98 }
99 context.drawImage(video, window.atlas_x, window.atlas_y, window.width, window.height, x, y, window.width, window.height);
100 }
101 canvas.dataset.windows = String(frame.windows.length);
102 };
103 const release = () => { buttons = 0; drag = undefined; compose(); send({ type: "release", sequence: ++pointerSequence }); };
104 const connect = async () => {
105 const current = ++attempt;
106 disconnect();
107 input = undefined;
108 setProblem("");
109 setState("Connecting…");
110 try {
111 if (disposed || !enabled() || current !== attempt) return;
112 if (!window.RTCPeerConnection) {
113 setProblem("This browser cannot stream VM screens. Open it in a current browser.");
114 setState("Disconnected");
115 return;
116 }
117 const url = new URL(`/api/vms/${encodeURIComponent(props.name)}/console`, location.href);
118 url.protocol = location.protocol === "https:" ? "wss:" : "ws:";
119 const socket = new WebSocket(url.href, ["binary"]);
120 socket.binaryType = "arraybuffer";
121 const keyboard = new RemoteKeyboard(video);
122 const focus = () => { if (input?.readyState === "open") keyboard.grab(); };
123 const blur = () => { keyboard.ungrab(); release(); };
124 video.addEventListener("focus", focus);
125 video.addEventListener("blur", blur);
126 let peer: RTCPeerConnection | undefined;
127 let frameCallback: number | undefined;
128 let statsTimer: number | undefined;
129 let previousStats: { time: number; bytes: number; frames: number } | undefined;
130 const frameReady: VideoFrameRequestCallback = (_, metadata) => {
131 if (current !== attempt || disposed) return;
132 frameTimestamp = (metadata as VideoFrameCallbackMetadata & { rtpTimestamp?: number }).rtpTimestamp;
133 setFrameSync(Number.isFinite(frameTimestamp));
134 if (!frameSync()) {
135 frameTimestamp = undefined; windowFrame = undefined; pendingFrames = [];
136 if (separateWindows()) { setSeparateWindows(false); send({ type: "experiment", enabled: false }); }
137 }
138 while (pendingFrames[0] && frameTimestamp !== undefined && ((frameTimestamp - pendingFrames[0].rtpTimestamp) | 0) >= 0) {
139 windowFrame = pendingFrames.shift();
140 }
141 compose();
142 frameCallback = video.requestVideoFrameCallback(frameReady);
143 };
144 const measure = async () => {
145 if (!peer || current !== attempt || disposed) return;
146 try {
147 const report = await peer.getStats();
148 if (current !== attempt || disposed || peer.connectionState === "closed") return;
149 let rtt = 0;
150 report.forEach((value) => { if (value.type === "candidate-pair" && value.state === "succeeded" && value.nominated) rtt = (value.currentRoundTripTime ?? 0) * 1000; });
151 report.forEach((value) => {
152 if (value.type !== "inbound-rtp" || value.kind !== "video") return;
153 const elapsed = previousStats ? (value.timestamp - previousStats.time) / 1000 : 0;
154 setStats({ fps: elapsed ? (value.framesDecoded - previousStats!.frames) / elapsed : 0,
155 bitrate: elapsed ? (value.bytesReceived - previousStats!.bytes) * 8 / elapsed / 1000000 : 0,
156 lost: value.packetsLost ?? 0, dropped: value.framesDropped ?? 0,
157 delay: value.jitterBufferEmittedCount ? value.jitterBufferDelay / value.jitterBufferEmittedCount * 1000 : 0, rtt });
158 previousStats = { time: value.timestamp, bytes: value.bytesReceived, frames: value.framesDecoded };
159 });
160 } catch {
161 if (peer.connectionState !== "closed") return;
162 } finally {
163 if (current === attempt && !disposed && peer.connectionState !== "closed") statsTimer = window.setTimeout(measure, 1000);
164 }
165 };
166 let pending = "";
167 let messages = Promise.resolve();
168 const decoder = new TextDecoder();
169 const candidates: RTCIceCandidateInit[] = [];
170 const signal = (message: object) => socket.send(JSON.stringify(message) + "\n");
171 const fail = (message: string) => {
172 if (current !== attempt || disposed) return;
173 setState("Disconnected");
174 setProblem(message);
175 disconnect();
176 };
177 const timeout = window.setTimeout(() => fail("The screen couldn't connect. Reconnect to try again."), 20000);
178 disconnect = () => {
179 clearTimeout(timeout);
180 blur();
181 video.removeEventListener("focus", focus);
182 video.removeEventListener("blur", blur);
183 socket.onclose = null;
184 socket.close();
185 peer?.close();
186 clearTimeout(statsTimer);
187 if (frameCallback !== undefined) video.cancelVideoFrameCallback(frameCallback);
188 windowFrame = undefined; pendingFrames = []; frameTimestamp = undefined; drag = undefined;
189 movement = undefined; pointerSequence = 0;
190 setFrameSync(false); setWindowsAvailable(false); setWindowCapture(false); setStats(undefined); setCapture(undefined);
191 compose();
192 video.onplaying = null;
193 video.srcObject = null;
194 clearTimeout(cursorTimer);
195 cursorFrames = [];
196 cursorFrame = 0;
197 cursorVisible = true;
198 cursorImage.style.display = cursorInvert.style.display = "none";
199 video.style.cursor = "default";
200 };
201 socket.onclose = () => fail("The screen disconnected. Check that the VM is running, then reconnect.");
202 socket.onerror = () => fail("Unable to connect the screen. Reconnect to try again.");
203 socket.onmessage = (event: MessageEvent<ArrayBuffer>) => {
204 pending += decoder.decode(event.data, { stream: true });
205 if (pending.length > 262144) { fail("The screen connection stopped. Reconnect to try again."); return; }
206 let end: number;
207 while ((end = pending.indexOf("\n")) !== -1) {
208 const line = pending.slice(0, end);
209 pending = pending.slice(end + 1);
210 messages = messages.then(async () => {
211 if (current !== attempt || disposed) return;
212 const message = JSON.parse(line);
213 if (message.type === "ready") {
214 peer = new RTCPeerConnection({ iceServers: message.iceServers });
215 const transceiver = peer.addTransceiver("video", { direction: "recvonly" });
216 const codecs = RTCRtpReceiver.getCapabilities("video")?.codecs.filter((codec) => codec.mimeType === "video/H264" || codec.mimeType === "video/rtx");
217 if (codecs?.length) transceiver.setCodecPreferences(codecs);
218 input = peer.createDataChannel("input");
219 movement = peer.createDataChannel("pointer", { ordered: false, maxRetransmits: 0 });
220 keyboard.onkeyevent = (key, _code, down) => send({ type: "key", key, down });
221 input.onopen = () => {
222 if (document.activeElement === video) focus();
223 if (windowsAvailable()) send({ type: "experiment", enabled: separateWindows() });
224 };
225 peer.onicecandidate = ({ candidate }) => { if (candidate) signal({ type: "candidate", ...candidate.toJSON() }); };
226 peer.onconnectionstatechange = () => {
227 if (peer?.connectionState === "failed" || peer?.connectionState === "disconnected") {
228 fail("The screen disconnected. Reconnect to try again.");
229 }
230 };
231 peer.ontrack = ({ track }) => {
232 const receiver = transceiver.receiver as RTCRtpReceiver & { jitterBufferTarget?: number };
233 if ("jitterBufferTarget" in receiver) receiver.jitterBufferTarget = 0;
234 video.srcObject = new MediaStream([track]);
235 if (video.requestVideoFrameCallback) frameCallback = video.requestVideoFrameCallback(frameReady);
236 void measure();
237 void video.play().catch(() => {
238 if (current === attempt && video.srcObject) fail("The browser couldn't play the screen. Reconnect to try again.");
239 });
240 };
241 await peer.setLocalDescription(await peer.createOffer());
242 signal({ type: "offer", sdp: peer.localDescription!.sdp });
243 } else if (message.type === "answer" && peer) {
244 await peer.setRemoteDescription({ type: "answer", sdp: message.sdp });
245 for (const candidate of candidates.splice(0)) await peer.addIceCandidate(candidate);
246 } else if (message.type === "candidate") {
247 if (peer?.remoteDescription) await peer.addIceCandidate(message);
248 else candidates.push(message);
249 } else if (message.type === "display") {
250 width = message.width;
251 height = message.height;
252 video.dataset.encoder = message.encoder;
253 video.dataset.source = message.source ?? "qemu";
254 } else if (message.type === "capabilities") {
255 setWindowsAvailable(message.windows);
256 if (message.windows) send({ type: "experiment", enabled: separateWindows() });
257 } else if (message.type === "window-capture") {
258 setWindowCapture(message.available);
259 } else if (message.type === "windows") {
260 pendingFrames.push(message);
261 if (pendingFrames.length > 16) pendingFrames.shift();
262 if (frameTimestamp !== undefined && ((frameTimestamp - message.rtpTimestamp) | 0) >= 0) {
263 windowFrame = message; pendingFrames = []; compose();
264 }
265 } else if (message.type === "window-positions") {
266 if (windowFrame?.windows.length && windowFrame.windows.length === message.windows.length &&
267 windowFrame.windows.every((window) => message.windows.some((next: WindowSurface) => window.id === next.id && window.width === next.width && window.height === next.height))) {
268 windowFrame = { ...windowFrame, windows: message.windows }; compose();
269 }
270 } else if (message.type === "capture-stats") {
271 setCapture({ fps: message.frames * 1000 / message.elapsed_ms,
272 capture: message.frames ? message.capture_us / message.frames / 1000 : 0,
273 scan: message.frames ? message.scan_us / message.frames / 1000 : 0,
274 write: message.frames ? message.write_us / message.frames / 1000 : 0 });
275 } else if (message.type === "cursor") {
276 if (message.frame === 0) {
277 clearTimeout(cursorTimer);
278 cursorFrames = [];
279 cursorFrame = 0;
280 cursorX = message.x;
281 cursorY = message.y;
282 if (message.source !== "guest") cursorVisible = !!message.fallback || (!!message.width && !!message.height);
283 video.dataset.cursorSource = message.source;
284 }
285 if (!message.image) { drawCursor(); return; }
286 for (const url of [message.image, message.invert].filter(Boolean)) {
287 const image = new Image();
288 image.src = url;
289 await image.decode();
290 }
291 if (current !== attempt || disposed) return;
292 cursorFrames.push({ image: message.image, invert: message.invert, duration: message.duration });
293 if (message.frame === 0) drawCursor();
294 if (cursorFrames.length === message.frames && message.frames > 1) animateCursor();
295 } else if (message.type === "cursor-position") {
296 cursorVisible = message.visible;
297 drawCursor();
298 } else if (message.type === "clipboard") setClipboard(message.text);
299 else if (message.type === "error") {
300 if (message.fatal) fail(message.message);
301 else setProblem(message.message);
302 }
303 }).catch((failure) => {
304 console.error("VM screen connection", failure);
305 fail("The browser couldn't connect the screen. Reconnect to try again.");
306 });
307 }
308 };
309 video.onplaying = () => {
310 if (current !== attempt || disposed) return;
311 clearTimeout(timeout);
312 setState("Connected");
313 compose();
314 if (props.active) video.focus();
315 };
316 } catch (failure) {
317 console.error("VM screen connection", failure);
318 disconnect();
319 setProblem("The browser couldn't connect the screen. Reconnect to try again.");
320 setState("Disconnected");
321 }
322 };
323 createEffect(() => {
324 if (enabled()) void connect();
325 else {
326 attempt++;
327 disconnect();
328 input = undefined;
329 setState("Disconnected");
330 setProblem("");
331 }
332 });
333 createEffect(() => { if (props.active && connected()) video.focus(); else release(); });
334 onCleanup(() => { disposed = true; disconnect(); });
335 const pointer = (event: PointerEvent | WheelEvent, mask = buttons) => {
336 if (!connected() || !width || !height) return;
337 const rect = video.getBoundingClientRect();
338 const scale = Math.min(rect.width / width, rect.height / height);
339 const x = Math.floor((event.clientX - rect.left - (rect.width - width * scale) / 2) / scale);
340 const y = Math.floor((event.clientY - rect.top - (rect.height - height * scale) / 2) / scale);
341 pointerX = event.clientX;
342 pointerY = event.clientY;
343 cursorInside = x >= 0 && y >= 0 && x < width && y < height;
344 drawCursor();
345 if ((event.type === "pointerdown" || event.type === "wheel") && (x < 0 || y < 0 || x >= width || y >= height)) return false;
346 if (event.type === "pointerdown" && mask === 1) {
347 const window = windowFrame?.windows.find((window) => x >= window.x && y >= window.y && x < window.x + window.width && y < window.y + window.height);
348 if (window && x >= window.caption_left && x < window.caption_right && y >= window.caption_top && y < window.caption_bottom) {
349 drag = { id: window.id, startX: x, startY: y, x: window.x, y: window.y, dx: 0, dy: 0, released: 0 };
350 }
351 }
352 if (drag && event.type === "pointermove" && buttons === 1) { drag.dx = x - drag.startX; drag.dy = y - drag.startY; compose(); }
353 if (drag && event.type === "pointerup" && !mask) { drag.released = performance.now(); window.setTimeout(compose, 510); }
354 const message = JSON.stringify({ type: "pointer", x, y, buttons: mask, sequence: ++pointerSequence });
355 if (event.type === "pointermove" && !mask && movement?.readyState === "open") {
356 if (movement.bufferedAmount < 16384) movement.send(message);
357 } else if (input?.readyState === "open" && (event.type !== "pointermove" || input.bufferedAmount < 16384)) input.send(message);
358 return true;
359 };
360 const ctrlAltDel = () => {
361 for (const key of [0xffe3, 0xffe9, 0xffff]) send({ type: "key", key, down: true });
362 for (const key of [0xffff, 0xffe9, 0xffe3]) send({ type: "key", key, down: false });
363 };
364 return <div class="vm-console" ref={panel}>
365 <div class="vm-console-toolbar" ref={setTools}>
366 {props.toolbar}
367 <Show when={props.active}>
368 <VMMenu label="Input">
369 <button disabled={!connected()} onClick={ctrlAltDel}><Keyboard size={14} aria-hidden="true" />Ctrl+Alt+Del</button>
370 <button disabled={!connected()} onClick={() => setShowClipboard(!showClipboard())}><Clipboard size={14} aria-hidden="true" />Clipboard</button>
371 </VMMenu>
372 <VMMenu label="View">
373 <button disabled={!connected() || !windowsAvailable() || !frameSync()} aria-pressed={separateWindows()}
374 onClick={() => { const enabled = !separateWindows(); setSeparateWindows(enabled); send({ type: "experiment", enabled }); }}>
375 <Check size={14} aria-hidden="true" style={{ visibility: separateWindows() ? "visible" : "hidden" }} />
376 Separate windows (experimental)
377 </button>
378 <button aria-pressed={showStats()} onClick={() => setShowStats(!showStats())}><Check size={14} aria-hidden="true" style={{ visibility: showStats() ? "visible" : "hidden" }} />Stream statistics</button>
379 <button onClick={() => panel.requestFullscreen().catch((failure) => setProblem(reason(failure)))}><Maximize size={14} aria-hidden="true" />Fullscreen</button>
380 <button disabled={!props.enabled || state() === "Connecting…"} onClick={connect}><RefreshCw size={14} aria-hidden="true" />Reconnect</button>
381 </VMMenu>
382 <span class="spacer" />
383 <span class="vm-console-status" role="status">{props.enabled ? state() : "Off"}</span>
384 </Show>
385 </div>
386 <Show when={props.active && showStats()}>
387 <div class="vm-stream-stats">
388 <Show when={stats()}>{(value) => <><span>{value().fps.toFixed(1)} FPS</span><span>{value().bitrate.toFixed(2)} Mbps</span><span>{value().lost} packets lost</span><span>{value().dropped} frames dropped</span><span>{value().rtt.toFixed(0)} ms round trip</span><span>{value().delay.toFixed(0)} ms buffer</span></>}</Show>
389 <Show when={capture()}>{(value) => <><span>Guest {value().fps.toFixed(1)} FPS</span><span>Desktop capture {value().capture.toFixed(1)} ms</span><span>Compare {value().scan.toFixed(1)} ms</span><span>Stream work {value().write.toFixed(1)} ms</span></>}</Show>
390 <Show when={separateWindows()}><span>{windowCapture() ? "Separate windows" : "Desktop fallback"}</span></Show>
391 </div>
392 </Show>
393 <Show when={props.active && problem()}><p class="error vm-console-error" role="alert">{problem()}</p></Show>
394 <Show when={props.active && showClipboard()}>
395 <div class="vm-clipboard">
396 <label class="field">clipboard<textarea class="search" rows="3" value={clipboard()} onInput={(event) => setClipboard(event.currentTarget.value)} /></label>
397 <span class="hint">Clipboard sharing requires support in the guest</span>
398 <div class="row">
399 <button class="button small" disabled={!connected()} onClick={() => send({ type: "clipboard", text: clipboard() })}>send text</button>
400 <button class="button small" onClick={() => navigator.clipboard.writeText(clipboard()).catch((failure) => setProblem(reason(failure)))}>copy text</button>
401 </div>
402 </div>
403 </Show>
404 <div class="vm-console-screen" hidden={!props.active || !props.enabled}>
405 <canvas ref={canvas} class="vm-window-canvas" hidden aria-hidden="true" />
406 <video ref={video} autoplay muted playsinline tabindex="0" aria-label="VM screen"
407 onContextMenu={(event) => event.preventDefault()}
408 onPointerMove={pointer}
409 onPointerLeave={() => { cursorInside = false; drawCursor(); }}
410 onPointerDown={(event) => {
411 event.preventDefault();
412 const mask = buttons | (event.button === 0 ? 1 : event.button === 1 ? 2 : event.button === 2 ? 4 : 0);
413 if (!pointer(event)) return;
414 pointer(event, mask);
415 buttons = mask;
416 video.focus(); video.setPointerCapture(event.pointerId);
417 }}
418 onPointerUp={(event) => {
419 if (buttons) pointer(event);
420 buttons &= ~(event.button === 0 ? 1 : event.button === 1 ? 2 : event.button === 2 ? 4 : 0);
421 pointer(event);
422 if (!event.buttons && video.hasPointerCapture(event.pointerId)) video.releasePointerCapture(event.pointerId);
423 }}
424 onPointerCancel={release}
425 onWheel={(event) => {
426 event.preventDefault();
427 const mask = event.deltaY ? (event.deltaY < 0 ? 8 : 16) : event.deltaX < 0 ? 32 : 64;
428 pointer(event, buttons | mask); pointer(event);
429 }} />
430 <img ref={cursorImage} class="vm-cursor" alt="" aria-hidden="true" />
431 <img ref={cursorInvert} class="vm-cursor vm-cursor-invert" alt="" aria-hidden="true" />
432 </div>
433 <Show when={tools()}>{(toolbar) => props.content(toolbar())}</Show>
434 <Show when={props.active && !props.enabled}><div class="empty"><p>The VM is off. Start it from the Machine menu.</p></div></Show>
435 </div>;
436}