authoravatarclover caruso<git@paperclover.net>2026-11-22 08:59:52+00:00
committeravatarclover caruso<git@paperclover.net>2026-11-22 09:04:21+00:00
log400964c2b06d4624703b6f2a7aab7abd15392fd9
tree8a87faba85bfa644b2db7b79c20e960e1f35e5f7
parent3c3a110db0400a7c3e2d139f2aff60577ed1dd10

Enable Spacebar voice signaling and Pion media

Build the SFU from the running Snowcord source pin, supervise a single voice signaling process, and publish the voice host and UDP mapping. Keep the API healthy if voice fails and keep preview voice disabled. Assisted-by: gpt-6.2-sol

6 files changed, 159 insertions(+), 6 deletions(-)

nixos/snowglobe.nix+8-2
...@@ -6,6 +6,8 @@ let...@@ -6,6 +6,8 @@ let
6 xmppTCPPorts = map (name: xmppNetwork.${name}) [ "c2s" "c2s_tls" "s2s" "turn" "turn_tls" ];6 xmppTCPPorts = map (name: xmppNetwork.${name}) [ "c2s" "c2s_tls" "s2s" "turn" "turn_tls" ];
7 xmppUDPPorts = [ xmppNetwork.turn xmppNetwork.turn_tls ];7 xmppUDPPorts = [ xmppNetwork.turn xmppNetwork.turn_tls ];
8 irohNetwork = builtins.fromJSON (builtins.readFile ../service/iroh-relay/network.json);8 irohNetwork = builtins.fromJSON (builtins.readFile ../service/iroh-relay/network.json);
9 spacebarVoice = builtins.fromJSON (builtins.readFile ../service/spacebar/voice-network.json);
10 udpPorts = xmppUDPPorts ++ [ irohNetwork.quic spacebarVoice.media ];
9 encryptionRoots = {11 encryptionRoots = {
10 clover = "clover";12 clover = "clover";
11 media = "clover/Media";13 media = "clover/Media";
...@@ -24,7 +26,7 @@ in...@@ -24,7 +26,7 @@ in
24 networking.hostId = "4fa19ccb";26 networking.hostId = "4fa19ccb";
25 services.nomad.settings.name = config.networking.hostName;27 services.nomad.settings.name = config.networking.hostName;
26 networking.firewall.allowedTCPPorts = [ 80 443 ] ++ xmppTCPPorts;28 networking.firewall.allowedTCPPorts = [ 80 443 ] ++ xmppTCPPorts;
27 networking.firewall.allowedUDPPorts = xmppUDPPorts ++ [ irohNetwork.quic ];29 networking.firewall.allowedUDPPorts = udpPorts;
28 networking.firewall.allowedUDPPortRanges = [{ from = xmppNetwork.relay_first; to = xmppNetwork.relay_last; }];30 networking.firewall.allowedUDPPortRanges = [{ from = xmppNetwork.relay_first; to = xmppNetwork.relay_last; }];
29 networking.firewall.extraCommands = lib.concatMapStrings ({ address, prefixLength, ... }: ''31 networking.firewall.extraCommands = lib.concatMapStrings ({ address, prefixLength, ... }: ''
30 iptables -A nixos-fw -i enp4s0 -s ${address}/${toString prefixLength} -p tcp --dport 445 -j nixos-fw-accept32 iptables -A nixos-fw -i enp4s0 -s ${address}/${toString prefixLength} -p tcp --dport 445 -j nixos-fw-accept
...@@ -34,6 +36,10 @@ in...@@ -34,6 +36,10 @@ in
34 ip6tables -A nixos-fw -i enp4s0 -s fc00::/7 -p tcp --dport 445 -j nixos-fw-accept36 ip6tables -A nixos-fw -i enp4s0 -s fc00::/7 -p tcp --dport 445 -j nixos-fw-accept
35 '';37 '';
3638
39 services.caddy.virtualHosts."${spacebarVoice.subdomain}.${config.environment.variables.SNOWGLOBE_DOMAIN}".extraConfig = ''
40 reverse_proxy 127.0.0.1:${toString spacebarVoice.signaling}
41 '';
42
37 systemd.services.public-ports = {43 systemd.services.public-ports = {
38 wantedBy = [ "multi-user.target" ];44 wantedBy = [ "multi-user.target" ];
39 wants = [ "network-online.target" ];45 wants = [ "network-online.target" ];
...@@ -44,7 +50,7 @@ in...@@ -44,7 +50,7 @@ in
44 description="$(${pkgs.python3}/bin/python3 ${./upnp-discover.py} ${lib.escapeShellArg config.networking.defaultGateway.address})"50 description="$(${pkgs.python3}/bin/python3 ${./upnp-discover.py} ${lib.escapeShellArg config.networking.defaultGateway.address})"
45 exec ${pkgs.miniupnpc}/bin/upnpc -u "$description" -e "Snow Globe services" -r \51 exec ${pkgs.miniupnpc}/bin/upnpc -u "$description" -e "Snow Globe services" -r \
46 ${lib.concatMapStringsSep " " (port: "${toString port} TCP") xmppTCPPorts} \52 ${lib.concatMapStringsSep " " (port: "${toString port} TCP") xmppTCPPorts} \
47 ${lib.concatMapStringsSep " " (port: "${toString port} UDP") (xmppUDPPorts ++ [ irohNetwork.quic ] ++ lib.range xmppNetwork.relay_first xmppNetwork.relay_last)}53 ${lib.concatMapStringsSep " " (port: "${toString port} UDP") (udpPorts ++ lib.range xmppNetwork.relay_first xmppNetwork.relay_last)}
48 '';54 '';
49 };55 };
5056
service/spacebar/build/Dockerfile+2
...@@ -1,3 +1,5 @@...@@ -1,3 +1,5 @@
1FROM docker.io/library/golang:1.25.11-bookworm@sha256:b96f24a8d7d010ea0acb9c3ba99064740f02b6b984612b28bd3c9c5ab9453e38 AS go
1FROM docker.io/library/node:26.13.1-bookworm@sha256:b7087bc4d973034b30854b378ba107f9fab622b47514cb7e2592e41b8bab8d492FROM docker.io/library/node:26.13.1-bookworm@sha256:b7087bc4d973034b30854b378ba107f9fab622b47514cb7e2592e41b8bab8d49
3COPY --from=go /usr/local/go /usr/local/go
2COPY supervisor.mjs /supervisor.mjs4COPY supervisor.mjs /supervisor.mjs
3CMD ["node", "/supervisor.mjs"]5CMD ["node", "/supervisor.mjs"]
service/spacebar/build/supervisor.mjs+106-4
...@@ -1,13 +1,15 @@...@@ -1,13 +1,15 @@
1import { execFile, spawn } from "node:child_process";1import { execFile, spawn } from "node:child_process";
2import { createHash } from "node:crypto";2import { createHash } from "node:crypto";
3import { copyFileSync, createWriteStream, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs";3import { resolve4 } from "node:dns/promises";
4import { once } from "node:events";
5import { copyFileSync, createWriteStream, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs";
4import { connect, createServer } from "node:net";6import { connect, createServer } from "node:net";
5import { basename } from "node:path";7import { basename } from "node:path";
6import { createInterface } from "node:readline";8import { createInterface } from "node:readline";
7import { setTimeout as sleep } from "node:timers/promises";9import { setTimeout as sleep } from "node:timers/promises";
8import { promisify } from "node:util";10import { promisify } from "node:util";
911
10const { BRANCH, PORT } = process.env;12const { BRANCH, PORT, VOICE_PUBLIC_HOST, VOICE_SIGNALING_PORT, VOICE_MEDIA_PORT } = process.env;
11const WORK = "/work";13const WORK = "/work";
12const REPO = "/source.git";14const REPO = "/source.git";
13const POLL_MS = 250;15const POLL_MS = 250;
...@@ -68,6 +70,9 @@ let wanted = null;...@@ -68,6 +70,9 @@ let wanted = null;
68let recentModules = null;70let recentModules = null;
69const releases = new Set();71const releases = new Set();
70let failed = { rev: null, at: 0 };72let failed = { rev: null, at: 0 };
73let voice = null;
74let retryVoiceAt = 0;
75let stopping = false;
7176
72createServer((client) => {77createServer((client) => {
73 const target = live;78 const target = live;
...@@ -126,7 +131,7 @@ const freePort = () =>...@@ -126,7 +131,7 @@ const freePort = () =>
126131
127function sweep() {132function sweep() {
128 const used = { rel: new Set(), modules: new Set([recentModules]) };133 const used = { rel: new Set(), modules: new Set([recentModules]) };
129 for (const release of releases) {134 for (const release of [...releases, ...(voice ? [voice.release] : [])]) {
130 used.rel.add(release.rev);135 used.rel.add(release.rev);
131 if (release.modules) used.modules.add(basename(release.modules)).add(`${basename(release.modules)}.partial`);136 if (release.modules) used.modules.add(basename(release.modules)).add(`${basename(release.modules)}.partial`);
132 }137 }
...@@ -135,6 +140,97 @@ function sweep() {...@@ -135,6 +140,97 @@ function sweep() {
135 }140 }
136}141}
137142
143const running = (proc) => proc?.pid && proc.exitCode === null && proc.signalCode === null;
144
145async function stop(proc) {
146 if (!running(proc)) return;
147 const exited = once(proc, "exit");
148 proc.kill("SIGTERM");
149 await Promise.race([exited, sleep(3000, undefined, { ref: false })]);
150 if (running(proc)) {
151 proc.kill("SIGKILL");
152 await exited;
153 }
154}
155
156async function stopVoice() {
157 if (!voice) return;
158 await Promise.all([stop(voice.signaling), stop(voice.sfu)]);
159 rmSync("/tmp/sfu-ipc.sock", { force: true });
160 voice = null;
161}
162
163async function prepareSfu(release) {
164 const pin = JSON.parse(readFileSync(`${release.dir}/flake.lock`, "utf8")).nodes["pion-webrtc"].locked;
165 if (pin.owner !== "spacebarchat" || pin.repo !== "pion-webrtc" || !/^[a-f0-9]{40}$/.test(pin.rev)) {
166 throw new Error("invalid Snowcord Pion SFU pin");
167 }
168 const binary = `${WORK}/pion-sfu-${pin.rev}`;
169 if (existsSync(binary)) return binary;
170
171 const scratch = mkdtempSync(`${WORK}/build-sfu-`);
172 try {
173 const source = await fetch(`https://codeload.github.com/spacebarchat/pion-webrtc/tar.gz/${pin.rev}`, { signal: AbortSignal.timeout(60000) });
174 if (!source.ok) throw new Error(`Pion SFU download failed with HTTP ${source.status}`);
175 writeFileSync(`${scratch}/source.tar.gz`, new Uint8Array(await source.arrayBuffer()));
176 await run("tar", ["-xzf", `${scratch}/source.tar.gz`, "-C", scratch]);
177 await run("/usr/local/go/bin/go", ["build", "-trimpath", "-o", `${scratch}/pion-sfu`, "."], {
178 cwd: `${scratch}/pion-webrtc-${pin.rev}/pion-sfu`,
179 env: { ...process.env, GOTOOLCHAIN: "local", CGO_ENABLED: "0" },
180 });
181 renameSync(`${scratch}/pion-sfu`, binary);
182 return binary;
183 } finally {
184 rmSync(scratch, { recursive: true, force: true });
185 }
186}
187
188async function waitForVoice(proc, ready) {
189 for (const deadline = Date.now() + 15000; Date.now() < deadline; await sleep(100)) {
190 if (!running(proc)) throw new Error("voice process exited during startup");
191 if (await ready()) return;
192 }
193 throw new Error("voice process did not become ready");
194}
195
196async function startVoice(release) {
197 try {
198 const addresses = await resolve4(VOICE_PUBLIC_HOST);
199 if (addresses.length !== 1) throw new Error("voice hostname must resolve to one IPv4 address");
200 const binary = await prepareSfu(release);
201 if (stopping) return;
202 await stopVoice();
203 if (stopping || live !== release) return;
204
205 const [ip] = addresses;
206 const start = (command, args, env = process.env) => capture(spawn(command, args, {
207 cwd: "/data", stdio: ["ignore", "pipe", "pipe"], env,
208 })).on("error", (error) => log(`voice process: ${error.message}`));
209 voice = { release, sfu: start(binary, ["-ip", ip, "-port", VOICE_MEDIA_PORT]), signaling: null };
210 await waitForVoice(voice.sfu, () => statSync("/tmp/sfu-ipc.sock", { throwIfNoEntry: false })?.isSocket());
211 if (stopping) return;
212
213 voice.signaling = start(process.execPath, ["--enable-source-maps", `${release.dir}/dist/webrtc/start.js`], {
214 ...process.env,
215 PORT: VOICE_SIGNALING_PORT,
216 APPLY_DB_MIGRATIONS: "false",
217 WRTC_LIBRARY: "@spacebarchat/pion-webrtc",
218 WRTC_PUBLIC_IP: ip,
219 WRTC_PORT_MIN: VOICE_MEDIA_PORT,
220 WRTC_PORT_MAX: VOICE_MEDIA_PORT,
221 });
222 await waitForVoice(voice.signaling, () => fetch(`http://127.0.0.1:${VOICE_SIGNALING_PORT}/`, {
223 signal: AbortSignal.timeout(1000),
224 }).then((response) => response.ok, () => false));
225 retryVoiceAt = 0;
226 log(`voice live ${release.rev.slice(0, 7)} with UDP ${VOICE_MEDIA_PORT}`);
227 } catch (error) {
228 if (voice?.release === release) await stopVoice();
229 retryVoiceAt = Date.now() + RETRY_FAILED_MS;
230 log(`voice ${release.rev.slice(0, 7)} failed: ${error.message}`);
231 }
232}
233
138function retire(old) {234function retire(old) {
139 setTimeout(() => {235 setTimeout(() => {
140 for (const socket of old.sockets) socket.destroy();236 for (const socket of old.sockets) socket.destroy();
...@@ -220,7 +316,11 @@ async function deploy() {...@@ -220,7 +316,11 @@ async function deploy() {
220316
221mkdirSync(`${WORK}/rel`, { recursive: true });317mkdirSync(`${WORK}/rel`, { recursive: true });
222mkdirSync(`${WORK}/modules`, { recursive: true });318mkdirSync(`${WORK}/modules`, { recursive: true });
223process.on("SIGTERM", () => process.exit(0));319process.once("SIGTERM", async () => {
320 stopping = true;
321 await stopVoice();
322 process.exit(0);
323});
224324
225(async () => {325(async () => {
226 let problem = "";326 let problem = "";
...@@ -239,5 +339,7 @@ process.on("SIGTERM", () => process.exit(0));...@@ -239,5 +339,7 @@ process.on("SIGTERM", () => process.exit(0));
239for (;;) {339for (;;) {
240 const retry = wanted === failed.rev && !live && Date.now() - failed.at > RETRY_FAILED_MS;340 const retry = wanted === failed.rev && !live && Date.now() - failed.at > RETRY_FAILED_MS;
241 if (wanted && wanted !== live?.rev && (wanted !== failed.rev || retry)) await deploy();341 if (wanted && wanted !== live?.rev && (wanted !== failed.rev || retry)) await deploy();
342 else if (VOICE_SIGNALING_PORT && live && Date.now() >= retryVoiceAt
343 && (voice?.release !== live || !running(voice.sfu) || !running(voice.signaling))) await startVoice(live);
242 else await sleep(100);344 else await sleep(100);
243}345}
service/spacebar/service.pkl+28
...@@ -2,8 +2,11 @@ amends "../../config/Service.pkl"...@@ -2,8 +2,11 @@ amends "../../config/Service.pkl"
22
3import "../../config/site.pkl" as site3import "../../config/site.pkl" as site
4import "../postgres/service.pkl" as postgres4import "../postgres/service.pkl" as postgres
5import "pkl:json"
56
6local host = module.container.http.hostname7local host = module.container.http.hostname
8local voice = new json.Parser {}.parse(read("voice-network.json").text)
9local voiceHost = "\(voice.subdomain).\(site.domain)"
710
8name = "Spacebar"11name = "Spacebar"
9tagline = "Discord-compatible chat"12tagline = "Discord-compatible chat"
...@@ -35,6 +38,13 @@ container {...@@ -35,6 +38,13 @@ container {
35 plainHostnames { "phoenix-selfhosted.\(site.domain)"; "px.\(site.domain)" }38 plainHostnames { "phoenix-selfhosted.\(site.domain)"; "px.\(site.domain)" }
36 }39 }
3740
41 tcp {
42 when (!site.preview) {
43 new { name = "voice"; containerPort = voice.signaling; hostPort = containerPort }
44 new { name = "voice-media"; containerPort = voice.media; hostPort = containerPort; loopback = false }
45 }
46 }
47
38 volumes {48 volumes {
39 ["/data"] {}49 ["/data"] {}
40 // Shale keeps each repository under its UUID, which renaming and visibility changes leave alone.50 // Shale keeps each repository under its UUID, which renaming and visibility changes leave alone.
...@@ -51,6 +61,11 @@ container {...@@ -51,6 +61,11 @@ container {
51 ["CONFIG_READONLY"] = "true"61 ["CONFIG_READONLY"] = "true"
52 ["STORAGE_LOCATION"] = "/data/files"62 ["STORAGE_LOCATION"] = "/data/files"
53 ["PORT"] = "3001"63 ["PORT"] = "3001"
64 when (!site.preview) {
65 ["VOICE_PUBLIC_HOST"] = voiceHost
66 ["VOICE_SIGNALING_PORT"] = voice.signaling.toString()
67 ["VOICE_MEDIA_PORT"] = voice.media.toString()
68 }
54 }69 }
5570
56 envTemplate = """71 envTemplate = """
...@@ -74,6 +89,19 @@ container {...@@ -74,6 +89,19 @@ container {
74 "gateway": {89 "gateway": {
75 "endpointPublic": "wss://\(host)"90 "endpointPublic": "wss://\(host)"
76 },91 },
92 \(if (site.preview) "" else """
93 "regions": {
94 "default": "spacebar",
95 "available": [{
96 "id": "spacebar",
97 "name": "Spacebar",
98 "endpoint": "\(voiceHost):443",
99 "vip": false,
100 "custom": false,
101 "deprecated": false
102 }]
103 },
104 """)
77 "register": {105 "register": {
78 "requireCaptcha": false106 "requireCaptcha": false
79 },107 },
service/spacebar/voice-network.json created+5
...@@ -0,0 +1,5 @@
1{
2 "subdomain": "spacebar-voice",
3 "signaling": 17404,
4 "media": 6000
5}
tools/spacebar.md+10
...@@ -49,3 +49,13 @@ start that is still waiting. Rolling back is pushing the old commit again....@@ -49,3 +49,13 @@ start that is still waiting. Rolling back is pushing the old commit again.
49 start and a revert does not undo it. The old and new server briefly overlap.49 start and a revert does not undo it. The old and new server briefly overlap.
50- Changing the supervisor, image or environment redeploys the container, which50- Changing the supervisor, image or environment redeploys the container, which
51 takes about 9 s to rebuild and start from the branch.51 takes about 9 s to rebuild and start from the branch.
52
53## Voice
54
55The production service publishes the voice WebSocket at `spacebar-voice.paperclover.net:443`.
56[`voice-network.json`](../service/spacebar/voice-network.json) configures the
57loopback WebSocket mapping and the UDP media port. The supervisor builds Pion
58from the Snowcord revision's `flake.lock` and caches the binary for that pin.
59It starts one SFU and one standalone voice process after the API is healthy.
60Each app release restarts voice, and an unavailable voice process retries
61while the API continues serving. Previews leave voice disabled.