diff --git a/dashboard/src/vms.rs b/dashboard/src/vms.rs index 1c1513b8cf20b252851939307f4f202000ec484a..a1c311e6cad72aea7683183be64379dd278e3274 100644 --- a/dashboard/src/vms.rs +++ b/dashboard/src/vms.rs @@ -34,12 +34,29 @@ pub async fn console(request: Request, name: &str, serial: bool) -> Result bridge(socket, stream).await, + let stream = async { + if serial { + return host::open(json!({"operation":"vm.serial","payload":{"name":name}})) + .await + .map(|(_, stream)| stream); + } + let mut stream = tokio::net::UnixStream::connect(env( + "STUDIO_VM_SCREEN_SOCKET", + "/run/studio-vm-screen/screen.sock", + )) + .await?; + stream + .write_all(format!("{}\n", json!({"name":name})).as_bytes()) + .await?; + Ok(stream) + } + .await; + match stream { + Ok(stream) => bridge(socket, stream).await, Err(error) => { let reason = if error.message.len() <= 120 { error.message @@ -89,9 +106,9 @@ async fn bridge(socket: WebSocket, stream: tokio::net::UnixStream) { while let Some(Ok(message)) = receive.next().await { match message { Message::Binary(data) => write.write_all(&data).await?, + Message::Text(data) => write.write_all(data.as_bytes()).await?, Message::Close(_) => break, Message::Ping(_) | Message::Pong(_) => {} - _ => break, } } Ok::<_, std::io::Error>(()) diff --git a/dashboard/vite.config.ts b/dashboard/vite.config.ts index 54d16d109b20c762382ddba2ac19e9ed2adab4ad..8ecb5ec28d49272e8de4dd00d5e06168e732777b 100644 --- a/dashboard/vite.config.ts +++ b/dashboard/vite.config.ts @@ -1,9 +1,12 @@ import { defineConfig } from "vite"; import solid from "vite-plugin-solid"; +import { createRequire } from "node:module"; +import { dirname, resolve } from "node:path"; export default defineConfig({ root: "web", plugins: [solid()], + resolve: { alias: { "@novnc/keyboard": resolve(dirname(createRequire(import.meta.url).resolve("@novnc/novnc")), "input/keyboard.js") } }, build: { outDir: "../dist", emptyOutDir: true }, server: { host: true, diff --git a/dashboard/web/components/VMConsole.tsx b/dashboard/web/components/VMConsole.tsx index ef1aa59401aeea4cd5156dff7a7d9d538439d7bd..09413ba5b00acc3f8d0136f1a404ef4106368a0b 100644 --- a/dashboard/web/components/VMConsole.tsx +++ b/dashboard/web/components/VMConsole.tsx @@ -5,56 +5,200 @@ import Keyboard from "lucide-solid/icons/keyboard"; import Clipboard from "lucide-solid/icons/clipboard"; import Maximize from "lucide-solid/icons/maximize"; import RefreshCw from "lucide-solid/icons/refresh-cw"; +import RemoteKeyboard from "@novnc/keyboard"; export function VMConsole(props: { name: string; enabled: boolean; active: boolean; toolbar: JSX.Element; content: (toolbar: HTMLDivElement) => JSX.Element }) { const enabled = createMemo(() => props.enabled); let attempt = 0; - let screen!: HTMLDivElement; + let video!: HTMLVideoElement; + let cursorImage!: HTMLImageElement; + let cursorInvert!: HTMLImageElement; let panel!: HTMLDivElement; - let connection: import("@novnc/novnc").default | undefined; + let input: RTCDataChannel | undefined; + let disconnect = () => {}; + let width = 0; + let height = 0; + let buttons = 0; let disposed = false; + let cursorFrames: { image: string; invert?: string; duration: number }[] = []; + let cursorFrame = 0; + let cursorTimer: number | undefined; + let cursorX = 0; + let cursorY = 0; + let pointerX = 0; + let pointerY = 0; + let cursorInside = false; + let cursorVisible = true; + const drawCursor = () => { + const frame = cursorFrames[cursorFrame]; + const overlay = !!frame?.invert && cursorInside && cursorVisible; + cursorImage.style.display = cursorInvert.style.display = overlay ? "block" : "none"; + if (!frame || !cursorVisible) { video.style.cursor = cursorVisible ? "default" : "none"; return; } + video.style.cursor = frame.invert ? (cursorInside ? "none" : "default") : `url(${frame.image}) ${cursorX} ${cursorY}, default`; + if (overlay) { + cursorImage.src = frame.image; + cursorInvert.src = frame.invert!; + cursorImage.style.transform = cursorInvert.style.transform = `translate(${pointerX - cursorX}px, ${pointerY - cursorY}px)`; + } + }; + const animateCursor = () => { + const frame = cursorFrames[cursorFrame]; + if (!frame) return; + cursorTimer = window.setTimeout(() => { + cursorFrame = (cursorFrame + 1) % cursorFrames.length; + drawCursor(); + animateCursor(); + }, frame.duration); + }; const [state, setState] = createSignal<"Connecting…" | "Connected" | "Disconnected">("Connecting…"); const connected = () => state() === "Connected"; const [problem, setProblem] = createSignal(""); const [clipboard, setClipboard] = createSignal(""); const [showClipboard, setShowClipboard] = createSignal(false); const [tools, setTools] = createSignal(); + const send = (message: object) => { if (input?.readyState === "open") input.send(JSON.stringify(message)); }; + const release = () => { buttons = 0; send({ type: "release" }); }; const connect = async () => { const current = ++attempt; - const previous = connection; - connection = undefined; - previous?.disconnect(); + disconnect(); + input = undefined; setProblem(""); setState("Connecting…"); try { - const { default: RFB } = await import("@novnc/novnc"); if (disposed || !enabled() || current !== attempt) return; + if (!window.RTCPeerConnection) { + setProblem("This browser cannot stream VM screens. Open it in a current browser."); + setState("Disconnected"); + return; + } const url = new URL(`/api/vms/${encodeURIComponent(props.name)}/console`, location.href); url.protocol = location.protocol === "https:" ? "wss:" : "ws:"; - const rfb = new RFB(screen, url.href, { shared: true, wsProtocols: ["binary"] }); - connection = rfb; - rfb.scaleViewport = true; - rfb.background = "#101014"; - rfb.addEventListener("connect", () => { - if (connection !== rfb) return; + const socket = new WebSocket(url.href, ["binary"]); + socket.binaryType = "arraybuffer"; + const keyboard = new RemoteKeyboard(video); + const focus = () => { if (input?.readyState === "open") keyboard.grab(); }; + const blur = () => { keyboard.ungrab(); release(); }; + video.addEventListener("focus", focus); + video.addEventListener("blur", blur); + let peer: RTCPeerConnection | undefined; + let pending = ""; + let messages = Promise.resolve(); + const decoder = new TextDecoder(); + const candidates: RTCIceCandidateInit[] = []; + const signal = (message: object) => socket.send(JSON.stringify(message) + "\n"); + const fail = (message: string) => { + if (current !== attempt || disposed) return; + setState("Disconnected"); + setProblem(message); + disconnect(); + }; + const timeout = window.setTimeout(() => fail("The screen couldn't connect. Reconnect to try again."), 20000); + disconnect = () => { + clearTimeout(timeout); + blur(); + video.removeEventListener("focus", focus); + video.removeEventListener("blur", blur); + socket.onclose = null; + socket.close(); + peer?.close(); + video.onplaying = null; + video.srcObject = null; + clearTimeout(cursorTimer); + cursorFrames = []; + cursorFrame = 0; + cursorVisible = true; + cursorImage.style.display = cursorInvert.style.display = "none"; + video.style.cursor = "default"; + }; + socket.onclose = () => fail("The screen disconnected. Check that the VM is running, then reconnect."); + socket.onerror = () => fail("Unable to connect the screen. Reconnect to try again."); + socket.onmessage = (event: MessageEvent) => { + pending += decoder.decode(event.data, { stream: true }); + if (pending.length > 262144) { fail("The screen connection stopped. Reconnect to try again."); return; } + let end: number; + while ((end = pending.indexOf("\n")) !== -1) { + const line = pending.slice(0, end); + pending = pending.slice(end + 1); + messages = messages.then(async () => { + if (current !== attempt || disposed) return; + const message = JSON.parse(line); + if (message.type === "ready") { + peer = new RTCPeerConnection({ iceServers: message.iceServers }); + const transceiver = peer.addTransceiver("video", { direction: "recvonly" }); + const codecs = RTCRtpReceiver.getCapabilities("video")?.codecs.filter((codec) => codec.mimeType === "video/H264"); + if (codecs?.length) transceiver.setCodecPreferences(codecs); + input = peer.createDataChannel("input"); + keyboard.onkeyevent = (key, _code, down) => send({ type: "key", key, down }); + input.onopen = () => { if (document.activeElement === video) focus(); }; + peer.onicecandidate = ({ candidate }) => { if (candidate) signal({ type: "candidate", ...candidate.toJSON() }); }; + peer.onconnectionstatechange = () => { + if (peer?.connectionState === "failed" || peer?.connectionState === "disconnected") { + fail("The screen disconnected. Reconnect to try again."); + } + }; + peer.ontrack = ({ track }) => { + video.srcObject = new MediaStream([track]); + void video.play().catch(() => { + if (current === attempt && video.srcObject) fail("The browser couldn't play the screen. Reconnect to try again."); + }); + }; + await peer.setLocalDescription(await peer.createOffer()); + signal({ type: "offer", sdp: peer.localDescription!.sdp }); + } else if (message.type === "answer" && peer) { + await peer.setRemoteDescription({ type: "answer", sdp: message.sdp }); + for (const candidate of candidates.splice(0)) await peer.addIceCandidate(candidate); + } else if (message.type === "candidate") { + if (peer?.remoteDescription) await peer.addIceCandidate(message); + else candidates.push(message); + } else if (message.type === "display") { + width = message.width; + height = message.height; + video.dataset.encoder = message.encoder; + video.dataset.source = message.source ?? "qemu"; + } else if (message.type === "cursor") { + if (message.frame === 0) { + clearTimeout(cursorTimer); + cursorFrames = []; + cursorFrame = 0; + cursorX = message.x; + cursorY = message.y; + if (message.source !== "guest") cursorVisible = !!message.fallback || (!!message.width && !!message.height); + video.dataset.cursorSource = message.source; + } + if (!message.image) { drawCursor(); return; } + for (const url of [message.image, message.invert].filter(Boolean)) { + const image = new Image(); + image.src = url; + await image.decode(); + } + if (current !== attempt || disposed) return; + cursorFrames.push({ image: message.image, invert: message.invert, duration: message.duration }); + if (message.frame === 0) drawCursor(); + if (cursorFrames.length === message.frames && message.frames > 1) animateCursor(); + } else if (message.type === "cursor-position") { + cursorVisible = message.visible; + drawCursor(); + } else if (message.type === "clipboard") setClipboard(message.text); + else if (message.type === "error") { + if (message.fatal) fail(message.message); + else setProblem(message.message); + } + }).catch((failure) => { + console.error("VM screen connection", failure); + fail("The browser couldn't connect the screen. Reconnect to try again."); + }); + } + }; + video.onplaying = () => { + if (current !== attempt || disposed) return; + clearTimeout(timeout); setState("Connected"); - }); - rfb.addEventListener("disconnect", () => { - if (connection !== rfb || disposed) return; - setState("Disconnected"); - setProblem("The screen disconnected. Check that the VM is running, then reconnect."); - }); - rfb.addEventListener("credentialsrequired", () => { - if (connection !== rfb || disposed) return; - setProblem("This screen requires a VNC password. Remove it in the VM's display settings, then reconnect."); - rfb.disconnect(); - }); - rfb.addEventListener("clipboard", (event: Event) => { - if (connection !== rfb || disposed) return; - setClipboard((event as CustomEvent<{ text: string }>).detail.text); - }); + if (props.active) video.focus(); + }; } catch (failure) { - setProblem(`Unable to connect. ${reason(failure)}`); + console.error("VM screen connection", failure); + disconnect(); + setProblem("The browser couldn't connect the screen. Reconnect to try again."); setState("Disconnected"); } }; @@ -62,21 +206,38 @@ export function VMConsole(props: { name: string; enabled: boolean; active: boole if (enabled()) void connect(); else { attempt++; - const previous = connection; - connection = undefined; - previous?.disconnect(); + disconnect(); + input = undefined; setState("Disconnected"); setProblem(""); } }); - createEffect(() => { if (props.active && connected()) connection?.focus(); }); - onCleanup(() => { disposed = true; connection?.disconnect(); }); + createEffect(() => { if (props.active && connected()) video.focus(); else release(); }); + onCleanup(() => { disposed = true; disconnect(); }); + const pointer = (event: PointerEvent | WheelEvent, mask = buttons) => { + if (!connected() || !width || !height) return; + const rect = video.getBoundingClientRect(); + const scale = Math.min(rect.width / width, rect.height / height); + const x = Math.floor((event.clientX - rect.left - (rect.width - width * scale) / 2) / scale); + const y = Math.floor((event.clientY - rect.top - (rect.height - height * scale) / 2) / scale); + pointerX = event.clientX; + pointerY = event.clientY; + cursorInside = x >= 0 && y >= 0 && x < width && y < height; + drawCursor(); + if ((event.type === "pointerdown" || event.type === "wheel") && (x < 0 || y < 0 || x >= width || y >= height)) return false; + send({ type: "pointer", x, y, buttons: mask }); + return true; + }; + const ctrlAltDel = () => { + for (const key of [0xffe3, 0xffe9, 0xffff]) send({ type: "key", key, down: true }); + for (const key of [0xffff, 0xffe9, 0xffe3]) send({ type: "key", key, down: false }); + }; return
{props.toolbar} - + @@ -93,12 +254,37 @@ export function VMConsole(props: { name: string; enabled: boolean; active: boole