authoravatarclover caruso<git@paperclover.net>2026-11-22 08:59:52+00:00
committeravatarclover caruso<git@paperclover.net>2026-11-22 09:04:21+00:00
log400964c2b06d4624703b6f2a7aab7abd15392fd9
tree8a87faba85bfa644b2db7b79c20e960e1f35e5f7
parent3c3a110db0400a7c3e2d139f2aff60577ed1dd10

Enable Spacebar voice signaling and Pion media

Build the SFU from the running Snowcord source pin, supervise a single voice signaling process, and publish the voice host and UDP mapping. Keep the API healthy if voice fails and keep preview voice disabled. Assisted-by: gpt-6.2-sol

6 files changed, 159 insertions(+), 6 deletions(-)

nixos/snowglobe.nix+8-2
......@@ -6,6 +6,8 @@ let
66 xmppTCPPorts = map (name: xmppNetwork.${name}) [ "c2s" "c2s_tls" "s2s" "turn" "turn_tls" ];
77 xmppUDPPorts = [ xmppNetwork.turn xmppNetwork.turn_tls ];
88 irohNetwork = builtins.fromJSON (builtins.readFile ../service/iroh-relay/network.json);
9 spacebarVoice = builtins.fromJSON (builtins.readFile ../service/spacebar/voice-network.json);
10 udpPorts = xmppUDPPorts ++ [ irohNetwork.quic spacebarVoice.media ];
911 encryptionRoots = {
1012 clover = "clover";
1113 media = "clover/Media";
......@@ -24,7 +26,7 @@ in
2426 networking.hostId = "4fa19ccb";
2527 services.nomad.settings.name = config.networking.hostName;
2628 networking.firewall.allowedTCPPorts = [ 80 443 ] ++ xmppTCPPorts;
27 networking.firewall.allowedUDPPorts = xmppUDPPorts ++ [ irohNetwork.quic ];
29 networking.firewall.allowedUDPPorts = udpPorts;
2830 networking.firewall.allowedUDPPortRanges = [{ from = xmppNetwork.relay_first; to = xmppNetwork.relay_last; }];
2931 networking.firewall.extraCommands = lib.concatMapStrings ({ address, prefixLength, ... }: ''
3032 iptables -A nixos-fw -i enp4s0 -s ${address}/${toString prefixLength} -p tcp --dport 445 -j nixos-fw-accept
......@@ -34,6 +36,10 @@ in
3436 ip6tables -A nixos-fw -i enp4s0 -s fc00::/7 -p tcp --dport 445 -j nixos-fw-accept
3537 '';
3638
39 services.caddy.virtualHosts."${spacebarVoice.subdomain}.${config.environment.variables.SNOWGLOBE_DOMAIN}".extraConfig = ''
40 reverse_proxy 127.0.0.1:${toString spacebarVoice.signaling}
41 '';
42
3743 systemd.services.public-ports = {
3844 wantedBy = [ "multi-user.target" ];
3945 wants = [ "network-online.target" ];
......@@ -44,7 +50,7 @@ in
4450 description="$(${pkgs.python3}/bin/python3 ${./upnp-discover.py} ${lib.escapeShellArg config.networking.defaultGateway.address})"
4551 exec ${pkgs.miniupnpc}/bin/upnpc -u "$description" -e "Snow Globe services" -r \
4652 ${lib.concatMapStringsSep " " (port: "${toString port} TCP") xmppTCPPorts} \
47 ${lib.concatMapStringsSep " " (port: "${toString port} UDP") (xmppUDPPorts ++ [ irohNetwork.quic ] ++ lib.range xmppNetwork.relay_first xmppNetwork.relay_last)}
53 ${lib.concatMapStringsSep " " (port: "${toString port} UDP") (udpPorts ++ lib.range xmppNetwork.relay_first xmppNetwork.relay_last)}
4854 '';
4955 };
5056
service/spacebar/build/Dockerfile+2
......@@ -1,3 +1,5 @@
1FROM docker.io/library/golang:1.25.11-bookworm@sha256:b96f24a8d7d010ea0acb9c3ba99064740f02b6b984612b28bd3c9c5ab9453e38 AS go
12FROM docker.io/library/node:26.13.1-bookworm@sha256:b7087bc4d973034b30854b378ba107f9fab622b47514cb7e2592e41b8bab8d49
3COPY --from=go /usr/local/go /usr/local/go
24COPY supervisor.mjs /supervisor.mjs
35CMD ["node", "/supervisor.mjs"]
service/spacebar/build/supervisor.mjs+106-4
......@@ -1,13 +1,15 @@
11import { execFile, spawn } from "node:child_process";
22import { createHash } from "node:crypto";
3import { copyFileSync, createWriteStream, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs";
3import { resolve4 } from "node:dns/promises";
4import { once } from "node:events";
5import { copyFileSync, createWriteStream, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs";
46import { connect, createServer } from "node:net";
57import { basename } from "node:path";
68import { createInterface } from "node:readline";
79import { setTimeout as sleep } from "node:timers/promises";
810import { promisify } from "node:util";
911
10const { BRANCH, PORT } = process.env;
12const { BRANCH, PORT, VOICE_PUBLIC_HOST, VOICE_SIGNALING_PORT, VOICE_MEDIA_PORT } = process.env;
1113const WORK = "/work";
1214const REPO = "/source.git";
1315const POLL_MS = 250;
......@@ -68,6 +70,9 @@ let wanted = null;
6870let recentModules = null;
6971const releases = new Set();
7072let failed = { rev: null, at: 0 };
73let voice = null;
74let retryVoiceAt = 0;
75let stopping = false;
7176
7277createServer((client) => {
7378 const target = live;
......@@ -126,7 +131,7 @@ const freePort = () =>
126131
127132function sweep() {
128133 const used = { rel: new Set(), modules: new Set([recentModules]) };
129 for (const release of releases) {
134 for (const release of [...releases, ...(voice ? [voice.release] : [])]) {
130135 used.rel.add(release.rev);
131136 if (release.modules) used.modules.add(basename(release.modules)).add(`${basename(release.modules)}.partial`);
132137 }
......@@ -135,6 +140,97 @@ function sweep() {
135140 }
136141}
137142
143const running = (proc) => proc?.pid && proc.exitCode === null && proc.signalCode === null;
144
145async function stop(proc) {
146 if (!running(proc)) return;
147 const exited = once(proc, "exit");
148 proc.kill("SIGTERM");
149 await Promise.race([exited, sleep(3000, undefined, { ref: false })]);
150 if (running(proc)) {
151 proc.kill("SIGKILL");
152 await exited;
153 }
154}
155
156async function stopVoice() {
157 if (!voice) return;
158 await Promise.all([stop(voice.signaling), stop(voice.sfu)]);
159 rmSync("/tmp/sfu-ipc.sock", { force: true });
160 voice = null;
161}
162
163async function prepareSfu(release) {
164 const pin = JSON.parse(readFileSync(`${release.dir}/flake.lock`, "utf8")).nodes["pion-webrtc"].locked;
165 if (pin.owner !== "spacebarchat" || pin.repo !== "pion-webrtc" || !/^[a-f0-9]{40}$/.test(pin.rev)) {
166 throw new Error("invalid Snowcord Pion SFU pin");
167 }
168 const binary = `${WORK}/pion-sfu-${pin.rev}`;
169 if (existsSync(binary)) return binary;
170
171 const scratch = mkdtempSync(`${WORK}/build-sfu-`);
172 try {
173 const source = await fetch(`https://codeload.github.com/spacebarchat/pion-webrtc/tar.gz/${pin.rev}`, { signal: AbortSignal.timeout(60000) });
174 if (!source.ok) throw new Error(`Pion SFU download failed with HTTP ${source.status}`);
175 writeFileSync(`${scratch}/source.tar.gz`, new Uint8Array(await source.arrayBuffer()));
176 await run("tar", ["-xzf", `${scratch}/source.tar.gz`, "-C", scratch]);
177 await run("/usr/local/go/bin/go", ["build", "-trimpath", "-o", `${scratch}/pion-sfu`, "."], {
178 cwd: `${scratch}/pion-webrtc-${pin.rev}/pion-sfu`,
179 env: { ...process.env, GOTOOLCHAIN: "local", CGO_ENABLED: "0" },
180 });
181 renameSync(`${scratch}/pion-sfu`, binary);
182 return binary;
183 } finally {
184 rmSync(scratch, { recursive: true, force: true });
185 }
186}
187
188async function waitForVoice(proc, ready) {
189 for (const deadline = Date.now() + 15000; Date.now() < deadline; await sleep(100)) {
190 if (!running(proc)) throw new Error("voice process exited during startup");
191 if (await ready()) return;
192 }
193 throw new Error("voice process did not become ready");
194}
195
196async function startVoice(release) {
197 try {
198 const addresses = await resolve4(VOICE_PUBLIC_HOST);
199 if (addresses.length !== 1) throw new Error("voice hostname must resolve to one IPv4 address");
200 const binary = await prepareSfu(release);
201 if (stopping) return;
202 await stopVoice();
203 if (stopping || live !== release) return;
204
205 const [ip] = addresses;
206 const start = (command, args, env = process.env) => capture(spawn(command, args, {
207 cwd: "/data", stdio: ["ignore", "pipe", "pipe"], env,
208 })).on("error", (error) => log(`voice process: ${error.message}`));
209 voice = { release, sfu: start(binary, ["-ip", ip, "-port", VOICE_MEDIA_PORT]), signaling: null };
210 await waitForVoice(voice.sfu, () => statSync("/tmp/sfu-ipc.sock", { throwIfNoEntry: false })?.isSocket());
211 if (stopping) return;
212
213 voice.signaling = start(process.execPath, ["--enable-source-maps", `${release.dir}/dist/webrtc/start.js`], {
214 ...process.env,
215 PORT: VOICE_SIGNALING_PORT,
216 APPLY_DB_MIGRATIONS: "false",
217 WRTC_LIBRARY: "@spacebarchat/pion-webrtc",
218 WRTC_PUBLIC_IP: ip,
219 WRTC_PORT_MIN: VOICE_MEDIA_PORT,
220 WRTC_PORT_MAX: VOICE_MEDIA_PORT,
221 });
222 await waitForVoice(voice.signaling, () => fetch(`http://127.0.0.1:${VOICE_SIGNALING_PORT}/`, {
223 signal: AbortSignal.timeout(1000),
224 }).then((response) => response.ok, () => false));
225 retryVoiceAt = 0;
226 log(`voice live ${release.rev.slice(0, 7)} with UDP ${VOICE_MEDIA_PORT}`);
227 } catch (error) {
228 if (voice?.release === release) await stopVoice();
229 retryVoiceAt = Date.now() + RETRY_FAILED_MS;
230 log(`voice ${release.rev.slice(0, 7)} failed: ${error.message}`);
231 }
232}
233
138234function retire(old) {
139235 setTimeout(() => {
140236 for (const socket of old.sockets) socket.destroy();
......@@ -220,7 +316,11 @@ async function deploy() {
220316
221317mkdirSync(`${WORK}/rel`, { recursive: true });
222318mkdirSync(`${WORK}/modules`, { recursive: true });
223process.on("SIGTERM", () => process.exit(0));
319process.once("SIGTERM", async () => {
320 stopping = true;
321 await stopVoice();
322 process.exit(0);
323});
224324
225325(async () => {
226326 let problem = "";
......@@ -239,5 +339,7 @@ process.on("SIGTERM", () => process.exit(0));
239339for (;;) {
240340 const retry = wanted === failed.rev && !live && Date.now() - failed.at > RETRY_FAILED_MS;
241341 if (wanted && wanted !== live?.rev && (wanted !== failed.rev || retry)) await deploy();
342 else if (VOICE_SIGNALING_PORT && live && Date.now() >= retryVoiceAt
343 && (voice?.release !== live || !running(voice.sfu) || !running(voice.signaling))) await startVoice(live);
242344 else await sleep(100);
243345}
service/spacebar/service.pkl+28
......@@ -2,8 +2,11 @@ amends "../../config/Service.pkl"
22
33import "../../config/site.pkl" as site
44import "../postgres/service.pkl" as postgres
5import "pkl:json"
56
67local host = module.container.http.hostname
8local voice = new json.Parser {}.parse(read("voice-network.json").text)
9local voiceHost = "\(voice.subdomain).\(site.domain)"
710
811name = "Spacebar"
912tagline = "Discord-compatible chat"
......@@ -35,6 +38,13 @@ container {
3538 plainHostnames { "phoenix-selfhosted.\(site.domain)"; "px.\(site.domain)" }
3639 }
3740
41 tcp {
42 when (!site.preview) {
43 new { name = "voice"; containerPort = voice.signaling; hostPort = containerPort }
44 new { name = "voice-media"; containerPort = voice.media; hostPort = containerPort; loopback = false }
45 }
46 }
47
3848 volumes {
3949 ["/data"] {}
4050 // Shale keeps each repository under its UUID, which renaming and visibility changes leave alone.
......@@ -51,6 +61,11 @@ container {
5161 ["CONFIG_READONLY"] = "true"
5262 ["STORAGE_LOCATION"] = "/data/files"
5363 ["PORT"] = "3001"
64 when (!site.preview) {
65 ["VOICE_PUBLIC_HOST"] = voiceHost
66 ["VOICE_SIGNALING_PORT"] = voice.signaling.toString()
67 ["VOICE_MEDIA_PORT"] = voice.media.toString()
68 }
5469 }
5570
5671 envTemplate = """
......@@ -74,6 +89,19 @@ container {
7489 "gateway": {
7590 "endpointPublic": "wss://\(host)"
7691 },
92 \(if (site.preview) "" else """
93 "regions": {
94 "default": "spacebar",
95 "available": [{
96 "id": "spacebar",
97 "name": "Spacebar",
98 "endpoint": "\(voiceHost):443",
99 "vip": false,
100 "custom": false,
101 "deprecated": false
102 }]
103 },
104 """)
77105 "register": {
78106 "requireCaptcha": false
79107 },
service/spacebar/voice-network.json created+5
......@@ -0,0 +1,5 @@
1{
2 "subdomain": "spacebar-voice",
3 "signaling": 17404,
4 "media": 6000
5}
tools/spacebar.md+10
......@@ -49,3 +49,13 @@ start that is still waiting. Rolling back is pushing the old commit again.
4949 start and a revert does not undo it. The old and new server briefly overlap.
5050- Changing the supervisor, image or environment redeploys the container, which
5151 takes about 9 s to rebuild and start from the branch.
52
53## Voice
54
55The production service publishes the voice WebSocket at `spacebar-voice.paperclover.net:443`.
56[`voice-network.json`](../service/spacebar/voice-network.json) configures the
57loopback WebSocket mapping and the UDP media port. The supervisor builds Pion
58from the Snowcord revision's `flake.lock` and caches the binary for that pin.
59It starts one SFU and one standalone voice process after the API is healthy.
60Each app release restarts voice, and an unavailable voice process retries
61while the API continues serving. Previews leave voice disabled.