From 400964c2b06d4624703b6f2a7aab7abd15392fd9 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Sun, 22 Nov 2026 08:59:52 +0000 Subject: [PATCH] Enable Spacebar voice signaling and Pion media Build the SFU from the running Snowcord source pin, supervise a single voice signaling process, and publish the voice host and UDP mapping. Keep the API healthy if voice fails and keep preview voice disabled. Assisted-by: gpt-6.2-sol --- nixos/snowglobe.nix | 10 ++- service/spacebar/build/Dockerfile | 2 + service/spacebar/build/supervisor.mjs | 110 +++++++++++++++++++++++++- service/spacebar/service.pkl | 28 +++++++ service/spacebar/voice-network.json | 5 ++ tools/spacebar.md | 10 +++ 6 files changed, 159 insertions(+), 6 deletions(-) create mode 100644 service/spacebar/voice-network.json diff --git a/nixos/snowglobe.nix b/nixos/snowglobe.nix index 80e28cfbe1e52d617665946c4ef5e37839f28aa4..f15f37741df2c757707e3ef7f8f3258afeab1f2d 100644 --- a/nixos/snowglobe.nix +++ b/nixos/snowglobe.nix @@ -6,6 +6,8 @@ let xmppTCPPorts = map (name: xmppNetwork.${name}) [ "c2s" "c2s_tls" "s2s" "turn" "turn_tls" ]; xmppUDPPorts = [ xmppNetwork.turn xmppNetwork.turn_tls ]; irohNetwork = builtins.fromJSON (builtins.readFile ../service/iroh-relay/network.json); + spacebarVoice = builtins.fromJSON (builtins.readFile ../service/spacebar/voice-network.json); + udpPorts = xmppUDPPorts ++ [ irohNetwork.quic spacebarVoice.media ]; encryptionRoots = { clover = "clover"; media = "clover/Media"; @@ -24,7 +26,7 @@ in networking.hostId = "4fa19ccb"; services.nomad.settings.name = config.networking.hostName; networking.firewall.allowedTCPPorts = [ 80 443 ] ++ xmppTCPPorts; - networking.firewall.allowedUDPPorts = xmppUDPPorts ++ [ irohNetwork.quic ]; + networking.firewall.allowedUDPPorts = udpPorts; networking.firewall.allowedUDPPortRanges = [{ from = xmppNetwork.relay_first; to = xmppNetwork.relay_last; }]; networking.firewall.extraCommands = lib.concatMapStrings ({ address, prefixLength, ... }: '' iptables -A nixos-fw -i enp4s0 -s ${address}/${toString prefixLength} -p tcp --dport 445 -j nixos-fw-accept @@ -34,6 +36,10 @@ in ip6tables -A nixos-fw -i enp4s0 -s fc00::/7 -p tcp --dport 445 -j nixos-fw-accept ''; + services.caddy.virtualHosts."${spacebarVoice.subdomain}.${config.environment.variables.SNOWGLOBE_DOMAIN}".extraConfig = '' + reverse_proxy 127.0.0.1:${toString spacebarVoice.signaling} + ''; + systemd.services.public-ports = { wantedBy = [ "multi-user.target" ]; wants = [ "network-online.target" ]; @@ -44,7 +50,7 @@ in description="$(${pkgs.python3}/bin/python3 ${./upnp-discover.py} ${lib.escapeShellArg config.networking.defaultGateway.address})" exec ${pkgs.miniupnpc}/bin/upnpc -u "$description" -e "Snow Globe services" -r \ ${lib.concatMapStringsSep " " (port: "${toString port} TCP") xmppTCPPorts} \ - ${lib.concatMapStringsSep " " (port: "${toString port} UDP") (xmppUDPPorts ++ [ irohNetwork.quic ] ++ lib.range xmppNetwork.relay_first xmppNetwork.relay_last)} + ${lib.concatMapStringsSep " " (port: "${toString port} UDP") (udpPorts ++ lib.range xmppNetwork.relay_first xmppNetwork.relay_last)} ''; }; diff --git a/service/spacebar/build/Dockerfile b/service/spacebar/build/Dockerfile index 364837521609c0ffb39caab0015a26d3e40548ea..1e78df0346b8b6dd2f5670ce2ab1dff921e6764d 100644 --- a/service/spacebar/build/Dockerfile +++ b/service/spacebar/build/Dockerfile @@ -1,3 +1,5 @@ +FROM docker.io/library/golang:1.25.11-bookworm@sha256:b96f24a8d7d010ea0acb9c3ba99064740f02b6b984612b28bd3c9c5ab9453e38 AS go FROM docker.io/library/node:26.13.1-bookworm@sha256:b7087bc4d973034b30854b378ba107f9fab622b47514cb7e2592e41b8bab8d49 +COPY --from=go /usr/local/go /usr/local/go COPY supervisor.mjs /supervisor.mjs CMD ["node", "/supervisor.mjs"] diff --git a/service/spacebar/build/supervisor.mjs b/service/spacebar/build/supervisor.mjs index dcf2d4fcdbaca7b4387c68b0944d2053fc1eb645..a23ad23da7341f835e0853763cb201449c6032ae 100644 --- a/service/spacebar/build/supervisor.mjs +++ b/service/spacebar/build/supervisor.mjs @@ -1,13 +1,15 @@ import { execFile, spawn } from "node:child_process"; import { createHash } from "node:crypto"; -import { copyFileSync, createWriteStream, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; +import { resolve4 } from "node:dns/promises"; +import { once } from "node:events"; +import { copyFileSync, createWriteStream, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; import { connect, createServer } from "node:net"; import { basename } from "node:path"; import { createInterface } from "node:readline"; import { setTimeout as sleep } from "node:timers/promises"; import { promisify } from "node:util"; -const { BRANCH, PORT } = process.env; +const { BRANCH, PORT, VOICE_PUBLIC_HOST, VOICE_SIGNALING_PORT, VOICE_MEDIA_PORT } = process.env; const WORK = "/work"; const REPO = "/source.git"; const POLL_MS = 250; @@ -68,6 +70,9 @@ let wanted = null; let recentModules = null; const releases = new Set(); let failed = { rev: null, at: 0 }; +let voice = null; +let retryVoiceAt = 0; +let stopping = false; createServer((client) => { const target = live; @@ -126,7 +131,7 @@ const freePort = () => function sweep() { const used = { rel: new Set(), modules: new Set([recentModules]) }; - for (const release of releases) { + for (const release of [...releases, ...(voice ? [voice.release] : [])]) { used.rel.add(release.rev); if (release.modules) used.modules.add(basename(release.modules)).add(`${basename(release.modules)}.partial`); } @@ -135,6 +140,97 @@ function sweep() { } } +const running = (proc) => proc?.pid && proc.exitCode === null && proc.signalCode === null; + +async function stop(proc) { + if (!running(proc)) return; + const exited = once(proc, "exit"); + proc.kill("SIGTERM"); + await Promise.race([exited, sleep(3000, undefined, { ref: false })]); + if (running(proc)) { + proc.kill("SIGKILL"); + await exited; + } +} + +async function stopVoice() { + if (!voice) return; + await Promise.all([stop(voice.signaling), stop(voice.sfu)]); + rmSync("/tmp/sfu-ipc.sock", { force: true }); + voice = null; +} + +async function prepareSfu(release) { + const pin = JSON.parse(readFileSync(`${release.dir}/flake.lock`, "utf8")).nodes["pion-webrtc"].locked; + if (pin.owner !== "spacebarchat" || pin.repo !== "pion-webrtc" || !/^[a-f0-9]{40}$/.test(pin.rev)) { + throw new Error("invalid Snowcord Pion SFU pin"); + } + const binary = `${WORK}/pion-sfu-${pin.rev}`; + if (existsSync(binary)) return binary; + + const scratch = mkdtempSync(`${WORK}/build-sfu-`); + try { + const source = await fetch(`https://codeload.github.com/spacebarchat/pion-webrtc/tar.gz/${pin.rev}`, { signal: AbortSignal.timeout(60000) }); + if (!source.ok) throw new Error(`Pion SFU download failed with HTTP ${source.status}`); + writeFileSync(`${scratch}/source.tar.gz`, new Uint8Array(await source.arrayBuffer())); + await run("tar", ["-xzf", `${scratch}/source.tar.gz`, "-C", scratch]); + await run("/usr/local/go/bin/go", ["build", "-trimpath", "-o", `${scratch}/pion-sfu`, "."], { + cwd: `${scratch}/pion-webrtc-${pin.rev}/pion-sfu`, + env: { ...process.env, GOTOOLCHAIN: "local", CGO_ENABLED: "0" }, + }); + renameSync(`${scratch}/pion-sfu`, binary); + return binary; + } finally { + rmSync(scratch, { recursive: true, force: true }); + } +} + +async function waitForVoice(proc, ready) { + for (const deadline = Date.now() + 15000; Date.now() < deadline; await sleep(100)) { + if (!running(proc)) throw new Error("voice process exited during startup"); + if (await ready()) return; + } + throw new Error("voice process did not become ready"); +} + +async function startVoice(release) { + try { + const addresses = await resolve4(VOICE_PUBLIC_HOST); + if (addresses.length !== 1) throw new Error("voice hostname must resolve to one IPv4 address"); + const binary = await prepareSfu(release); + if (stopping) return; + await stopVoice(); + if (stopping || live !== release) return; + + const [ip] = addresses; + const start = (command, args, env = process.env) => capture(spawn(command, args, { + cwd: "/data", stdio: ["ignore", "pipe", "pipe"], env, + })).on("error", (error) => log(`voice process: ${error.message}`)); + voice = { release, sfu: start(binary, ["-ip", ip, "-port", VOICE_MEDIA_PORT]), signaling: null }; + await waitForVoice(voice.sfu, () => statSync("/tmp/sfu-ipc.sock", { throwIfNoEntry: false })?.isSocket()); + if (stopping) return; + + voice.signaling = start(process.execPath, ["--enable-source-maps", `${release.dir}/dist/webrtc/start.js`], { + ...process.env, + PORT: VOICE_SIGNALING_PORT, + APPLY_DB_MIGRATIONS: "false", + WRTC_LIBRARY: "@spacebarchat/pion-webrtc", + WRTC_PUBLIC_IP: ip, + WRTC_PORT_MIN: VOICE_MEDIA_PORT, + WRTC_PORT_MAX: VOICE_MEDIA_PORT, + }); + await waitForVoice(voice.signaling, () => fetch(`http://127.0.0.1:${VOICE_SIGNALING_PORT}/`, { + signal: AbortSignal.timeout(1000), + }).then((response) => response.ok, () => false)); + retryVoiceAt = 0; + log(`voice live ${release.rev.slice(0, 7)} with UDP ${VOICE_MEDIA_PORT}`); + } catch (error) { + if (voice?.release === release) await stopVoice(); + retryVoiceAt = Date.now() + RETRY_FAILED_MS; + log(`voice ${release.rev.slice(0, 7)} failed: ${error.message}`); + } +} + function retire(old) { setTimeout(() => { for (const socket of old.sockets) socket.destroy(); @@ -220,7 +316,11 @@ async function deploy() { mkdirSync(`${WORK}/rel`, { recursive: true }); mkdirSync(`${WORK}/modules`, { recursive: true }); -process.on("SIGTERM", () => process.exit(0)); +process.once("SIGTERM", async () => { + stopping = true; + await stopVoice(); + process.exit(0); +}); (async () => { let problem = ""; @@ -239,5 +339,7 @@ process.on("SIGTERM", () => process.exit(0)); for (;;) { const retry = wanted === failed.rev && !live && Date.now() - failed.at > RETRY_FAILED_MS; if (wanted && wanted !== live?.rev && (wanted !== failed.rev || retry)) await deploy(); + else if (VOICE_SIGNALING_PORT && live && Date.now() >= retryVoiceAt + && (voice?.release !== live || !running(voice.sfu) || !running(voice.signaling))) await startVoice(live); else await sleep(100); } diff --git a/service/spacebar/service.pkl b/service/spacebar/service.pkl index f91f16ab4cad578b73f877c8e49b381899a91802..f0692976947ee04ced04d42b243bdaeea4019d2e 100644 --- a/service/spacebar/service.pkl +++ b/service/spacebar/service.pkl @@ -2,8 +2,11 @@ amends "../../config/Service.pkl" import "../../config/site.pkl" as site import "../postgres/service.pkl" as postgres +import "pkl:json" local host = module.container.http.hostname +local voice = new json.Parser {}.parse(read("voice-network.json").text) +local voiceHost = "\(voice.subdomain).\(site.domain)" name = "Spacebar" tagline = "Discord-compatible chat" @@ -35,6 +38,13 @@ container { plainHostnames { "phoenix-selfhosted.\(site.domain)"; "px.\(site.domain)" } } + tcp { + when (!site.preview) { + new { name = "voice"; containerPort = voice.signaling; hostPort = containerPort } + new { name = "voice-media"; containerPort = voice.media; hostPort = containerPort; loopback = false } + } + } + volumes { ["/data"] {} // Shale keeps each repository under its UUID, which renaming and visibility changes leave alone. @@ -51,6 +61,11 @@ container { ["CONFIG_READONLY"] = "true" ["STORAGE_LOCATION"] = "/data/files" ["PORT"] = "3001" + when (!site.preview) { + ["VOICE_PUBLIC_HOST"] = voiceHost + ["VOICE_SIGNALING_PORT"] = voice.signaling.toString() + ["VOICE_MEDIA_PORT"] = voice.media.toString() + } } envTemplate = """ @@ -74,6 +89,19 @@ container { "gateway": { "endpointPublic": "wss://\(host)" }, + \(if (site.preview) "" else """ + "regions": { + "default": "spacebar", + "available": [{ + "id": "spacebar", + "name": "Spacebar", + "endpoint": "\(voiceHost):443", + "vip": false, + "custom": false, + "deprecated": false + }] + }, + """) "register": { "requireCaptcha": false }, diff --git a/service/spacebar/voice-network.json b/service/spacebar/voice-network.json new file mode 100644 index 0000000000000000000000000000000000000000..1ee97792773e1a6480c108f9a3977d9450ad78ea --- /dev/null +++ b/service/spacebar/voice-network.json @@ -0,0 +1,5 @@ +{ + "subdomain": "spacebar-voice", + "signaling": 17404, + "media": 6000 +} diff --git a/tools/spacebar.md b/tools/spacebar.md index ece84aa63cca7cbdf98b9f2a82bab2f552d418a9..534a47efa4514344ba01f0f0fd4cee9e5959b107 100644 --- a/tools/spacebar.md +++ b/tools/spacebar.md @@ -49,3 +49,13 @@ start that is still waiting. Rolling back is pushing the old commit again. start and a revert does not undo it. The old and new server briefly overlap. - Changing the supervisor, image or environment redeploys the container, which takes about 9 s to rebuild and start from the branch. + +## Voice + +The production service publishes the voice WebSocket at `spacebar-voice.paperclover.net:443`. +[`voice-network.json`](../service/spacebar/voice-network.json) configures the +loopback WebSocket mapping and the UDP media port. The supervisor builds Pion +from the Snowcord revision's `flake.lock` and caches the binary for that pin. +It starts one SFU and one standalone voice process after the API is healthy. +Each app release restarts voice, and an unavailable voice process retries +while the API continues serving. Previews leave voice disabled. -- 2.54.1