| author | |
| committer | |
| log | d80e473e7de68b35767ef38d2264bb4bbf01d685 |
| tree | cf9d41ac35c33ee08920778d40ed00fc094c61dd |
| parent | 91df6befa2b0bd5a5d0ea7da1ee70e708950dd61 |
| parent | 400964c2b06d4624703b6f2a7aab7abd15392fd9 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Assisted-by: claude-sonnet-5.56 files changed, 159 insertions(+), 6 deletions(-)
nixos/snowglobe.nix+8-2| ... | @@ -6,6 +6,8 @@ let | ... | @@ -6,6 +6,8 @@ let |
| 6 | xmppTCPPorts = map (name: xmppNetwork.${name}) [ "c2s" "c2s_tls" "s2s" "turn" "turn_tls" ]; | 6 | xmppTCPPorts = map (name: xmppNetwork.${name}) [ "c2s" "c2s_tls" "s2s" "turn" "turn_tls" ]; |
| 7 | xmppUDPPorts = [ xmppNetwork.turn xmppNetwork.turn_tls ]; | 7 | xmppUDPPorts = [ xmppNetwork.turn xmppNetwork.turn_tls ]; |
| 8 | irohNetwork = builtins.fromJSON (builtins.readFile ../service/iroh-relay/network.json); | 8 | irohNetwork = builtins.fromJSON (builtins.readFile ../service/iroh-relay/network.json); |
| 9 | spacebarVoice = builtins.fromJSON (builtins.readFile ../service/spacebar/voice-network.json); | ||
| 10 | udpPorts = xmppUDPPorts ++ [ irohNetwork.quic spacebarVoice.media ]; | ||
| 9 | encryptionRoots = { | 11 | encryptionRoots = { |
| 10 | clover = "clover"; | 12 | clover = "clover"; |
| 11 | media = "clover/Media"; | 13 | media = "clover/Media"; |
| ... | @@ -24,7 +26,7 @@ in | ... | @@ -24,7 +26,7 @@ in |
| 24 | networking.hostId = "4fa19ccb"; | 26 | networking.hostId = "4fa19ccb"; |
| 25 | services.nomad.settings.name = config.networking.hostName; | 27 | services.nomad.settings.name = config.networking.hostName; |
| 26 | networking.firewall.allowedTCPPorts = [ 80 443 ] ++ xmppTCPPorts; | 28 | networking.firewall.allowedTCPPorts = [ 80 443 ] ++ xmppTCPPorts; |
| 27 | networking.firewall.allowedUDPPorts = xmppUDPPorts ++ [ irohNetwork.quic ]; | 29 | networking.firewall.allowedUDPPorts = udpPorts; |
| 28 | networking.firewall.allowedUDPPortRanges = [{ from = xmppNetwork.relay_first; to = xmppNetwork.relay_last; }]; | 30 | networking.firewall.allowedUDPPortRanges = [{ from = xmppNetwork.relay_first; to = xmppNetwork.relay_last; }]; |
| 29 | networking.firewall.extraCommands = lib.concatMapStrings ({ address, prefixLength, ... }: '' | 31 | networking.firewall.extraCommands = lib.concatMapStrings ({ address, prefixLength, ... }: '' |
| 30 | iptables -A nixos-fw -i enp4s0 -s ${address}/${toString prefixLength} -p tcp --dport 445 -j nixos-fw-accept | 32 | iptables -A nixos-fw -i enp4s0 -s ${address}/${toString prefixLength} -p tcp --dport 445 -j nixos-fw-accept |
| ... | @@ -34,6 +36,10 @@ in | ... | @@ -34,6 +36,10 @@ in |
| 34 | ip6tables -A nixos-fw -i enp4s0 -s fc00::/7 -p tcp --dport 445 -j nixos-fw-accept | 36 | ip6tables -A nixos-fw -i enp4s0 -s fc00::/7 -p tcp --dport 445 -j nixos-fw-accept |
| 35 | ''; | 37 | ''; |
| 36 | 38 | ||
| 39 | services.caddy.virtualHosts."${spacebarVoice.subdomain}.${config.environment.variables.SNOWGLOBE_DOMAIN}".extraConfig = '' | ||
| 40 | reverse_proxy 127.0.0.1:${toString spacebarVoice.signaling} | ||
| 41 | ''; | ||
| 42 | |||
| 37 | systemd.services.public-ports = { | 43 | systemd.services.public-ports = { |
| 38 | wantedBy = [ "multi-user.target" ]; | 44 | wantedBy = [ "multi-user.target" ]; |
| 39 | wants = [ "network-online.target" ]; | 45 | wants = [ "network-online.target" ]; |
| ... | @@ -44,7 +50,7 @@ in | ... | @@ -44,7 +50,7 @@ in |
| 44 | description="$(${pkgs.python3}/bin/python3 ${./upnp-discover.py} ${lib.escapeShellArg config.networking.defaultGateway.address})" | 50 | description="$(${pkgs.python3}/bin/python3 ${./upnp-discover.py} ${lib.escapeShellArg config.networking.defaultGateway.address})" |
| 45 | exec ${pkgs.miniupnpc}/bin/upnpc -u "$description" -e "Snow Globe services" -r \ | 51 | exec ${pkgs.miniupnpc}/bin/upnpc -u "$description" -e "Snow Globe services" -r \ |
| 46 | ${lib.concatMapStringsSep " " (port: "${toString port} TCP") xmppTCPPorts} \ | 52 | ${lib.concatMapStringsSep " " (port: "${toString port} TCP") xmppTCPPorts} \ |
| 47 | ${lib.concatMapStringsSep " " (port: "${toString port} UDP") (xmppUDPPorts ++ [ irohNetwork.quic ] ++ lib.range xmppNetwork.relay_first xmppNetwork.relay_last)} | 53 | ${lib.concatMapStringsSep " " (port: "${toString port} UDP") (udpPorts ++ lib.range xmppNetwork.relay_first xmppNetwork.relay_last)} |
| 48 | ''; | 54 | ''; |
| 49 | }; | 55 | }; |
| 50 | 56 |
service/spacebar/build/Dockerfile+2| ... | @@ -1,3 +1,5 @@ | ... | @@ -1,3 +1,5 @@ |
| 1 | FROM docker.io/library/golang:1.25.11-bookworm@sha256:b96f24a8d7d010ea0acb9c3ba99064740f02b6b984612b28bd3c9c5ab9453e38 AS go | ||
| 1 | FROM docker.io/library/node:26.13.1-bookworm@sha256:b7087bc4d973034b30854b378ba107f9fab622b47514cb7e2592e41b8bab8d49 | 2 | FROM docker.io/library/node:26.13.1-bookworm@sha256:b7087bc4d973034b30854b378ba107f9fab622b47514cb7e2592e41b8bab8d49 |
| 3 | COPY --from=go /usr/local/go /usr/local/go | ||
| 2 | COPY supervisor.mjs /supervisor.mjs | 4 | COPY supervisor.mjs /supervisor.mjs |
| 3 | CMD ["node", "/supervisor.mjs"] | 5 | CMD ["node", "/supervisor.mjs"] |
service/spacebar/build/supervisor.mjs+106-4| ... | @@ -1,13 +1,15 @@ | ... | @@ -1,13 +1,15 @@ |
| 1 | import { execFile, spawn } from "node:child_process"; | 1 | import { execFile, spawn } from "node:child_process"; |
| 2 | import { createHash } from "node:crypto"; | 2 | import { createHash } from "node:crypto"; |
| 3 | import { copyFileSync, createWriteStream, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; | 3 | import { resolve4 } from "node:dns/promises"; |
| 4 | import { once } from "node:events"; | ||
| 5 | import { copyFileSync, createWriteStream, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; | ||
| 4 | import { connect, createServer } from "node:net"; | 6 | import { connect, createServer } from "node:net"; |
| 5 | import { basename } from "node:path"; | 7 | import { basename } from "node:path"; |
| 6 | import { createInterface } from "node:readline"; | 8 | import { createInterface } from "node:readline"; |
| 7 | import { setTimeout as sleep } from "node:timers/promises"; | 9 | import { setTimeout as sleep } from "node:timers/promises"; |
| 8 | import { promisify } from "node:util"; | 10 | import { promisify } from "node:util"; |
| 9 | 11 | ||
| 10 | const { BRANCH, PORT } = process.env; | 12 | const { BRANCH, PORT, VOICE_PUBLIC_HOST, VOICE_SIGNALING_PORT, VOICE_MEDIA_PORT } = process.env; |
| 11 | const WORK = "/work"; | 13 | const WORK = "/work"; |
| 12 | const REPO = "/source.git"; | 14 | const REPO = "/source.git"; |
| 13 | const POLL_MS = 250; | 15 | const POLL_MS = 250; |
| ... | @@ -68,6 +70,9 @@ let wanted = null; | ... | @@ -68,6 +70,9 @@ let wanted = null; |
| 68 | let recentModules = null; | 70 | let recentModules = null; |
| 69 | const releases = new Set(); | 71 | const releases = new Set(); |
| 70 | let failed = { rev: null, at: 0 }; | 72 | let failed = { rev: null, at: 0 }; |
| 73 | let voice = null; | ||
| 74 | let retryVoiceAt = 0; | ||
| 75 | let stopping = false; | ||
| 71 | 76 | ||
| 72 | createServer((client) => { | 77 | createServer((client) => { |
| 73 | const target = live; | 78 | const target = live; |
| ... | @@ -126,7 +131,7 @@ const freePort = () => | ... | @@ -126,7 +131,7 @@ const freePort = () => |
| 126 | 131 | ||
| 127 | function sweep() { | 132 | function sweep() { |
| 128 | const used = { rel: new Set(), modules: new Set([recentModules]) }; | 133 | const used = { rel: new Set(), modules: new Set([recentModules]) }; |
| 129 | for (const release of releases) { | 134 | for (const release of [...releases, ...(voice ? [voice.release] : [])]) { |
| 130 | used.rel.add(release.rev); | 135 | used.rel.add(release.rev); |
| 131 | if (release.modules) used.modules.add(basename(release.modules)).add(`${basename(release.modules)}.partial`); | 136 | if (release.modules) used.modules.add(basename(release.modules)).add(`${basename(release.modules)}.partial`); |
| 132 | } | 137 | } |
| ... | @@ -135,6 +140,97 @@ function sweep() { | ... | @@ -135,6 +140,97 @@ function sweep() { |
| 135 | } | 140 | } |
| 136 | } | 141 | } |
| 137 | 142 | ||
| 143 | const running = (proc) => proc?.pid && proc.exitCode === null && proc.signalCode === null; | ||
| 144 | |||
| 145 | async function stop(proc) { | ||
| 146 | if (!running(proc)) return; | ||
| 147 | const exited = once(proc, "exit"); | ||
| 148 | proc.kill("SIGTERM"); | ||
| 149 | await Promise.race([exited, sleep(3000, undefined, { ref: false })]); | ||
| 150 | if (running(proc)) { | ||
| 151 | proc.kill("SIGKILL"); | ||
| 152 | await exited; | ||
| 153 | } | ||
| 154 | } | ||
| 155 | |||
| 156 | async function stopVoice() { | ||
| 157 | if (!voice) return; | ||
| 158 | await Promise.all([stop(voice.signaling), stop(voice.sfu)]); | ||
| 159 | rmSync("/tmp/sfu-ipc.sock", { force: true }); | ||
| 160 | voice = null; | ||
| 161 | } | ||
| 162 | |||
| 163 | async function prepareSfu(release) { | ||
| 164 | const pin = JSON.parse(readFileSync(`${release.dir}/flake.lock`, "utf8")).nodes["pion-webrtc"].locked; | ||
| 165 | if (pin.owner !== "spacebarchat" || pin.repo !== "pion-webrtc" || !/^[a-f0-9]{40}$/.test(pin.rev)) { | ||
| 166 | throw new Error("invalid Snowcord Pion SFU pin"); | ||
| 167 | } | ||
| 168 | const binary = `${WORK}/pion-sfu-${pin.rev}`; | ||
| 169 | if (existsSync(binary)) return binary; | ||
| 170 | |||
| 171 | const scratch = mkdtempSync(`${WORK}/build-sfu-`); | ||
| 172 | try { | ||
| 173 | const source = await fetch(`https://codeload.github.com/spacebarchat/pion-webrtc/tar.gz/${pin.rev}`, { signal: AbortSignal.timeout(60000) }); | ||
| 174 | if (!source.ok) throw new Error(`Pion SFU download failed with HTTP ${source.status}`); | ||
| 175 | writeFileSync(`${scratch}/source.tar.gz`, new Uint8Array(await source.arrayBuffer())); | ||
| 176 | await run("tar", ["-xzf", `${scratch}/source.tar.gz`, "-C", scratch]); | ||
| 177 | await run("/usr/local/go/bin/go", ["build", "-trimpath", "-o", `${scratch}/pion-sfu`, "."], { | ||
| 178 | cwd: `${scratch}/pion-webrtc-${pin.rev}/pion-sfu`, | ||
| 179 | env: { ...process.env, GOTOOLCHAIN: "local", CGO_ENABLED: "0" }, | ||
| 180 | }); | ||
| 181 | renameSync(`${scratch}/pion-sfu`, binary); | ||
| 182 | return binary; | ||
| 183 | } finally { | ||
| 184 | rmSync(scratch, { recursive: true, force: true }); | ||
| 185 | } | ||
| 186 | } | ||
| 187 | |||
| 188 | async function waitForVoice(proc, ready) { | ||
| 189 | for (const deadline = Date.now() + 15000; Date.now() < deadline; await sleep(100)) { | ||
| 190 | if (!running(proc)) throw new Error("voice process exited during startup"); | ||
| 191 | if (await ready()) return; | ||
| 192 | } | ||
| 193 | throw new Error("voice process did not become ready"); | ||
| 194 | } | ||
| 195 | |||
| 196 | async function startVoice(release) { | ||
| 197 | try { | ||
| 198 | const addresses = await resolve4(VOICE_PUBLIC_HOST); | ||
| 199 | if (addresses.length !== 1) throw new Error("voice hostname must resolve to one IPv4 address"); | ||
| 200 | const binary = await prepareSfu(release); | ||
| 201 | if (stopping) return; | ||
| 202 | await stopVoice(); | ||
| 203 | if (stopping || live !== release) return; | ||
| 204 | |||
| 205 | const [ip] = addresses; | ||
| 206 | const start = (command, args, env = process.env) => capture(spawn(command, args, { | ||
| 207 | cwd: "/data", stdio: ["ignore", "pipe", "pipe"], env, | ||
| 208 | })).on("error", (error) => log(`voice process: ${error.message}`)); | ||
| 209 | voice = { release, sfu: start(binary, ["-ip", ip, "-port", VOICE_MEDIA_PORT]), signaling: null }; | ||
| 210 | await waitForVoice(voice.sfu, () => statSync("/tmp/sfu-ipc.sock", { throwIfNoEntry: false })?.isSocket()); | ||
| 211 | if (stopping) return; | ||
| 212 | |||
| 213 | voice.signaling = start(process.execPath, ["--enable-source-maps", `${release.dir}/dist/webrtc/start.js`], { | ||
| 214 | ...process.env, | ||
| 215 | PORT: VOICE_SIGNALING_PORT, | ||
| 216 | APPLY_DB_MIGRATIONS: "false", | ||
| 217 | WRTC_LIBRARY: "@spacebarchat/pion-webrtc", | ||
| 218 | WRTC_PUBLIC_IP: ip, | ||
| 219 | WRTC_PORT_MIN: VOICE_MEDIA_PORT, | ||
| 220 | WRTC_PORT_MAX: VOICE_MEDIA_PORT, | ||
| 221 | }); | ||
| 222 | await waitForVoice(voice.signaling, () => fetch(`http://127.0.0.1:${VOICE_SIGNALING_PORT}/`, { | ||
| 223 | signal: AbortSignal.timeout(1000), | ||
| 224 | }).then((response) => response.ok, () => false)); | ||
| 225 | retryVoiceAt = 0; | ||
| 226 | log(`voice live ${release.rev.slice(0, 7)} with UDP ${VOICE_MEDIA_PORT}`); | ||
| 227 | } catch (error) { | ||
| 228 | if (voice?.release === release) await stopVoice(); | ||
| 229 | retryVoiceAt = Date.now() + RETRY_FAILED_MS; | ||
| 230 | log(`voice ${release.rev.slice(0, 7)} failed: ${error.message}`); | ||
| 231 | } | ||
| 232 | } | ||
| 233 | |||
| 138 | function retire(old) { | 234 | function retire(old) { |
| 139 | setTimeout(() => { | 235 | setTimeout(() => { |
| 140 | for (const socket of old.sockets) socket.destroy(); | 236 | for (const socket of old.sockets) socket.destroy(); |
| ... | @@ -220,7 +316,11 @@ async function deploy() { | ... | @@ -220,7 +316,11 @@ async function deploy() { |
| 220 | 316 | ||
| 221 | mkdirSync(`${WORK}/rel`, { recursive: true }); | 317 | mkdirSync(`${WORK}/rel`, { recursive: true }); |
| 222 | mkdirSync(`${WORK}/modules`, { recursive: true }); | 318 | mkdirSync(`${WORK}/modules`, { recursive: true }); |
| 223 | process.on("SIGTERM", () => process.exit(0)); | 319 | process.once("SIGTERM", async () => { |
| 320 | stopping = true; | ||
| 321 | await stopVoice(); | ||
| 322 | process.exit(0); | ||
| 323 | }); | ||
| 224 | 324 | ||
| 225 | (async () => { | 325 | (async () => { |
| 226 | let problem = ""; | 326 | let problem = ""; |
| ... | @@ -239,5 +339,7 @@ process.on("SIGTERM", () => process.exit(0)); | ... | @@ -239,5 +339,7 @@ process.on("SIGTERM", () => process.exit(0)); |
| 239 | for (;;) { | 339 | for (;;) { |
| 240 | const retry = wanted === failed.rev && !live && Date.now() - failed.at > RETRY_FAILED_MS; | 340 | const retry = wanted === failed.rev && !live && Date.now() - failed.at > RETRY_FAILED_MS; |
| 241 | if (wanted && wanted !== live?.rev && (wanted !== failed.rev || retry)) await deploy(); | 341 | if (wanted && wanted !== live?.rev && (wanted !== failed.rev || retry)) await deploy(); |
| 342 | else if (VOICE_SIGNALING_PORT && live && Date.now() >= retryVoiceAt | ||
| 343 | && (voice?.release !== live || !running(voice.sfu) || !running(voice.signaling))) await startVoice(live); | ||
| 242 | else await sleep(100); | 344 | else await sleep(100); |
| 243 | } | 345 | } |
service/spacebar/service.pkl+28| ... | @@ -2,8 +2,11 @@ amends "../../config/Service.pkl" | ... | @@ -2,8 +2,11 @@ amends "../../config/Service.pkl" |
| 2 | 2 | ||
| 3 | import "../../config/site.pkl" as site | 3 | import "../../config/site.pkl" as site |
| 4 | import "../postgres/service.pkl" as postgres | 4 | import "../postgres/service.pkl" as postgres |
| 5 | import "pkl:json" | ||
| 5 | 6 | ||
| 6 | local host = module.container.http.hostname | 7 | local host = module.container.http.hostname |
| 8 | local voice = new json.Parser {}.parse(read("voice-network.json").text) | ||
| 9 | local voiceHost = "\(voice.subdomain).\(site.domain)" | ||
| 7 | 10 | ||
| 8 | name = "Spacebar" | 11 | name = "Spacebar" |
| 9 | tagline = "Discord-compatible chat" | 12 | tagline = "Discord-compatible chat" |
| ... | @@ -35,6 +38,13 @@ container { | ... | @@ -35,6 +38,13 @@ container { |
| 35 | plainHostnames { "phoenix-selfhosted.\(site.domain)" } | 38 | plainHostnames { "phoenix-selfhosted.\(site.domain)" } |
| 36 | } | 39 | } |
| 37 | 40 | ||
| 41 | tcp { | ||
| 42 | when (!site.preview) { | ||
| 43 | new { name = "voice"; containerPort = voice.signaling; hostPort = containerPort } | ||
| 44 | new { name = "voice-media"; containerPort = voice.media; hostPort = containerPort; loopback = false } | ||
| 45 | } | ||
| 46 | } | ||
| 47 | |||
| 38 | volumes { | 48 | volumes { |
| 39 | ["/data"] {} | 49 | ["/data"] {} |
| 40 | // Shale keeps each repository under its UUID, which renaming and visibility changes leave alone. | 50 | // Shale keeps each repository under its UUID, which renaming and visibility changes leave alone. |
| ... | @@ -51,6 +61,11 @@ container { | ... | @@ -51,6 +61,11 @@ container { |
| 51 | ["CONFIG_READONLY"] = "true" | 61 | ["CONFIG_READONLY"] = "true" |
| 52 | ["STORAGE_LOCATION"] = "/data/files" | 62 | ["STORAGE_LOCATION"] = "/data/files" |
| 53 | ["PORT"] = "3001" | 63 | ["PORT"] = "3001" |
| 64 | when (!site.preview) { | ||
| 65 | ["VOICE_PUBLIC_HOST"] = voiceHost | ||
| 66 | ["VOICE_SIGNALING_PORT"] = voice.signaling.toString() | ||
| 67 | ["VOICE_MEDIA_PORT"] = voice.media.toString() | ||
| 68 | } | ||
| 54 | } | 69 | } |
| 55 | 70 | ||
| 56 | envTemplate = """ | 71 | envTemplate = """ |
| ... | @@ -74,6 +89,19 @@ container { | ... | @@ -74,6 +89,19 @@ container { |
| 74 | "gateway": { | 89 | "gateway": { |
| 75 | "endpointPublic": "wss://\(host)" | 90 | "endpointPublic": "wss://\(host)" |
| 76 | }, | 91 | }, |
| 92 | \(if (site.preview) "" else """ | ||
| 93 | "regions": { | ||
| 94 | "default": "spacebar", | ||
| 95 | "available": [{ | ||
| 96 | "id": "spacebar", | ||
| 97 | "name": "Spacebar", | ||
| 98 | "endpoint": "\(voiceHost):443", | ||
| 99 | "vip": false, | ||
| 100 | "custom": false, | ||
| 101 | "deprecated": false | ||
| 102 | }] | ||
| 103 | }, | ||
| 104 | """) | ||
| 77 | "register": { | 105 | "register": { |
| 78 | "requireCaptcha": false | 106 | "requireCaptcha": false |
| 79 | }, | 107 | }, |
service/spacebar/voice-network.json created+5| ... | @@ -0,0 +1,5 @@ | ||
| 1 | { | ||
| 2 | "subdomain": "spacebar-voice", | ||
| 3 | "signaling": 17404, | ||
| 4 | "media": 6000 | ||
| 5 | } | ||
tools/spacebar.md+10| ... | @@ -49,3 +49,13 @@ start that is still waiting. Rolling back is pushing the old commit again. | ... | @@ -49,3 +49,13 @@ start that is still waiting. Rolling back is pushing the old commit again. |
| 49 | start and a revert does not undo it. The old and new server briefly overlap. | 49 | start and a revert does not undo it. The old and new server briefly overlap. |
| 50 | - Changing the supervisor, image or environment redeploys the container, which | 50 | - Changing the supervisor, image or environment redeploys the container, which |
| 51 | takes about 9 s to rebuild and start from the branch. | 51 | takes about 9 s to rebuild and start from the branch. |
| 52 | |||
| 53 | ## Voice | ||
| 54 | |||
| 55 | The production service publishes the voice WebSocket at `spacebar-voice.paperclover.net:443`. | ||
| 56 | [`voice-network.json`](../service/spacebar/voice-network.json) configures the | ||
| 57 | loopback WebSocket mapping and the UDP media port. The supervisor builds Pion | ||
| 58 | from the Snowcord revision's `flake.lock` and caches the binary for that pin. | ||
| 59 | It starts one SFU and one standalone voice process after the API is healthy. | ||
| 60 | Each app release restarts voice, and an unavailable voice process retries | ||
| 61 | while the API continues serving. Previews leave voice disabled. |