| author | |
| committer | |
| log | c1158b6f7b5deb3cd0bc0c27460b4bb42ca8556d |
| tree | 2d8543e65e90ba43601c77934983f68880bdeba0 |
| parent | d80e473e7de68b35767ef38d2264bb4bbf01d685 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Move the API, gateway and CDN to phoenix-selfhosted.paperclover.net, serve
voice from voice.phoenix-selfhosted, give /work a persistent volume instead of
a memory-backed tmpfs, fetch the Pion pin through git, and run the Go build
and SFU with a minimal environment.
Assisted-by: claude-sonnet-5.55 files changed, 36 insertions(+), 22 deletions(-)
service/spacebar/build/supervisor.mjs+25-12| ... | ... | @@ -2,7 +2,7 @@ import { execFile, spawn } from "node:child_process"; |
| 2 | 2 | import { createHash } from "node:crypto"; |
| 3 | 3 | import { resolve4 } from "node:dns/promises"; |
| 4 | 4 | import { once } from "node:events"; |
| 5 | import { copyFileSync, createWriteStream, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; | |
| 5 | import { copyFileSync, createWriteStream, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; | |
| 6 | 6 | import { connect, createServer } from "node:net"; |
| 7 | 7 | import { basename } from "node:path"; |
| 8 | 8 | import { createInterface } from "node:readline"; |
| ... | ... | @@ -165,20 +165,33 @@ async function prepareSfu(release) { |
| 165 | 165 | if (pin.owner !== "spacebarchat" || pin.repo !== "pion-webrtc" || !/^[a-f0-9]{40}$/.test(pin.rev)) { |
| 166 | 166 | throw new Error("invalid Snowcord Pion SFU pin"); |
| 167 | 167 | } |
| 168 | const binary = `${WORK}/pion-sfu-${pin.rev}`; | |
| 168 | const name = `pion-sfu-${pin.rev}`; | |
| 169 | const binary = `${WORK}/${name}`; | |
| 169 | 170 | if (existsSync(binary)) return binary; |
| 170 | 171 | |
| 171 | const scratch = mkdtempSync(`${WORK}/build-sfu-`); | |
| 172 | const scratch = `${WORK}/sfu-build`; | |
| 173 | const source = `${scratch}/source`; | |
| 174 | rmSync(scratch, { recursive: true, force: true }); | |
| 175 | mkdirSync(source, { recursive: true }); | |
| 172 | 176 | try { |
| 173 | const source = await fetch(`https://codeload.github.com/spacebarchat/pion-webrtc/tar.gz/${pin.rev}`, { signal: AbortSignal.timeout(60000) }); | |
| 174 | if (!source.ok) throw new Error(`Pion SFU download failed with HTTP ${source.status}`); | |
| 175 | writeFileSync(`${scratch}/source.tar.gz`, new Uint8Array(await source.arrayBuffer())); | |
| 176 | await run("tar", ["-xzf", `${scratch}/source.tar.gz`, "-C", scratch]); | |
| 177 | await run("/usr/local/go/bin/go", ["build", "-trimpath", "-o", `${scratch}/pion-sfu`, "."], { | |
| 178 | cwd: `${scratch}/pion-webrtc-${pin.rev}/pion-sfu`, | |
| 179 | env: { ...process.env, GOTOOLCHAIN: "local", CGO_ENABLED: "0" }, | |
| 177 | // Fetching by full object id makes git verify the commit; go.sum pins every module. | |
| 178 | await run("git", ["-C", source, "init", "--quiet"]); | |
| 179 | await run("git", ["-C", source, "fetch", "--quiet", "--depth", "1", `https://github.com/${pin.owner}/${pin.repo}`, pin.rev]); | |
| 180 | await run("git", ["-C", source, "-c", "advice.detachedHead=false", "checkout", "--quiet", "FETCH_HEAD"]); | |
| 181 | await run("/usr/local/go/bin/go", ["build", "-trimpath", "-buildvcs=false", "-o", `${scratch}/pion-sfu`, "."], { | |
| 182 | cwd: `${source}/pion-sfu`, | |
| 183 | env: { | |
| 184 | PATH: process.env.PATH, | |
| 185 | HOME: scratch, | |
| 186 | GOPATH: `${scratch}/go`, | |
| 187 | GOCACHE: `${scratch}/cache`, | |
| 188 | GOFLAGS: "-mod=readonly -modcacherw", | |
| 189 | GOTOOLCHAIN: "local", | |
| 190 | CGO_ENABLED: "0", | |
| 191 | }, | |
| 180 | 192 | }); |
| 181 | 193 | renameSync(`${scratch}/pion-sfu`, binary); |
| 194 | for (const entry of readdirSync(WORK)) if (entry.startsWith("pion-sfu-") && entry !== name) rmSync(`${WORK}/${entry}`, { force: true }); | |
| 182 | 195 | return binary; |
| 183 | 196 | } finally { |
| 184 | 197 | rmSync(scratch, { recursive: true, force: true }); |
| ... | ... | @@ -203,10 +216,10 @@ async function startVoice(release) { |
| 203 | 216 | if (stopping || live !== release) return; |
| 204 | 217 | |
| 205 | 218 | const [ip] = addresses; |
| 206 | const start = (command, args, env = process.env) => capture(spawn(command, args, { | |
| 219 | const start = (command, args, env) => capture(spawn(command, args, { | |
| 207 | 220 | cwd: "/data", stdio: ["ignore", "pipe", "pipe"], env, |
| 208 | 221 | })).on("error", (error) => log(`voice process: ${error.message}`)); |
| 209 | voice = { release, sfu: start(binary, ["-ip", ip, "-port", VOICE_MEDIA_PORT]), signaling: null }; | |
| 222 | voice = { release, sfu: start(binary, ["-ip", ip, "-port", VOICE_MEDIA_PORT], {}), signaling: null }; | |
| 210 | 223 | await waitForVoice(voice.sfu, () => statSync("/tmp/sfu-ipc.sock", { throwIfNoEntry: false })?.isSocket()); |
| 211 | 224 | if (stopping) return; |
| 212 | 225 |
service/spacebar/service.pkl+2-3| ... | ... | @@ -32,10 +32,9 @@ container { |
| 32 | 32 | |
| 33 | 33 | http { |
| 34 | 34 | containerPort = 3001 |
| 35 | subdomain = "spacebar" | |
| 35 | subdomain = "phoenix-selfhosted" | |
| 36 | 36 | checkPath = "/api/ping" |
| 37 | 37 | forwardRealIp = true |
| 38 | plainHostnames { "phoenix-selfhosted.\(site.domain)" } | |
| 39 | 38 | } |
| 40 | 39 | |
| 41 | 40 | tcp { |
| ... | ... | @@ -47,12 +46,12 @@ container { |
| 47 | 46 | |
| 48 | 47 | volumes { |
| 49 | 48 | ["/data"] {} |
| 49 | ["/work"] {} | |
| 50 | 50 | // Shale keeps each repository under its UUID, which renaming and visibility changes leave alone. |
| 51 | 51 | ["/source.git"] { src = "\(site.root)/prod/shale/repositories_owned/006AH3H3QKH2KN318QXRNFBH4B"; readOnly = true } |
| 52 | 52 | // Production writes the server log into the sandbox workspace; previews keep their own. |
| 53 | 53 | ["/logs"] { when (!site.preview) { src = "\(site.root)/work/snowcord-server/logs" } } |
| 54 | 54 | } |
| 55 | tmpfs { "/work:rw,exec,size=3g" } | |
| 56 | 55 | |
| 57 | 56 | env { |
| 58 | 57 | ["BRANCH"] = "main" |
service/spacebar/voice-network.json+1-1| ... | ... | @@ -1,5 +1,5 @@ |
| 1 | 1 | { |
| 2 | "subdomain": "spacebar-voice", | |
| 2 | "subdomain": "voice.phoenix-selfhosted", | |
| 3 | 3 | "signaling": 17404, |
| 4 | 4 | "media": 6000 |
| 5 | 5 | } |
tools/phoenix-box.md+1-1| ... | ... | @@ -17,7 +17,7 @@ on the tailnet. |
| 17 | 17 | (any start clears it). Restarts log to `~/Library/Logs/m3-phoenix-box-revive.log`. |
| 18 | 18 | After a host reboot nothing runs until Clover unlocks FileVault and logs in. |
| 19 | 19 | - **Names:** the guest has no DNS (its gateway is blocked), so `/etc/hosts` maps |
| 20 | `spacebar.paperclover.net` to `100.64.0.1`. Go ignores `/etc/hosts` on macOS unless | |
| 20 | `phoenix-selfhosted.paperclover.net` to `100.64.0.1`. Go ignores `/etc/hosts` on macOS unless | |
| 21 | 21 | `GODEBUG=netdns=go`, which `run-phoenix-spacebar` sets for the Caddy relay. |
| 22 | 22 | - **Access:** Softnet also blocks host to guest, so there is no SSH or Screen Sharing. |
| 23 | 23 | Tart's own VNC server (random password per run, printed to `run.log` in |
tools/spacebar.md+7-5| ... | ... | @@ -1,6 +1,6 @@ |
| 1 | 1 | # Spacebar |
| 2 | 2 | |
| 3 | `https://spacebar.paperclover.net` runs Clover's fork of | |
| 3 | `https://phoenix-selfhosted.paperclover.net` runs Clover's fork of | |
| 4 | 4 | [spacebarchat/server](https://github.com/spacebarchat/server) from branch `main` of |
| 5 | 5 | the private Shale repository `https://shale.paperclover.net/snowcord-server.git`. |
| 6 | 6 | Pushing to that branch is the deploy; `tools/deploy.py` is only needed to change the service definition. |
| ... | ... | @@ -9,7 +9,7 @@ Pushing to that branch is the deploy; `tools/deploy.py` is only needed to change |
| 9 | 9 | ~/dev/snowcord-server jj checkout, remotes: origin = Shale, upstream = GitHub |
| 10 | 10 | (`master` mirrors upstream, `main` is what runs) |
| 11 | 11 | jj bookmark set main -r @- && jj git push -b main |
| 12 | curl https://spacebar.paperclover.net/_spacebar/api/version # commit that is live | |
| 12 | curl https://phoenix-selfhosted.paperclover.net/_spacebar/api/version # commit that is live | |
| 13 | 13 | ``` |
| 14 | 14 | |
| 15 | 15 | A push is live about 4 seconds later (3.4 to 4.6 s measured from the start of the push). Dependency changes |
| ... | ... | @@ -19,7 +19,7 @@ supervisor against the same branch with a fresh database. |
| 19 | 19 | ## How it works |
| 20 | 20 | |
| 21 | 21 | [`supervisor.mjs`](../service/spacebar/build/supervisor.mjs) is the container's |
| 22 | only process, on a stock Node image with its working files in tmpfs `/work`. | |
| 22 | only process, on a stock Node image with its working files in the persistent `/work` volume. | |
| 23 | 23 | |
| 24 | 24 | 1. Every 250 ms it reads the branch tip from the repository's Shale storage, |
| 25 | 25 | mounted read-only at `/source.git`. Shale keeps a repository under its UUID, |
| ... | ... | @@ -52,10 +52,12 @@ start that is still waiting. Rolling back is pushing the old commit again. |
| 52 | 52 | |
| 53 | 53 | ## Voice |
| 54 | 54 | |
| 55 | The production service publishes the voice WebSocket at `spacebar-voice.paperclover.net:443`. | |
| 55 | The production service publishes the voice WebSocket at `voice.phoenix-selfhosted.paperclover.net:443`. | |
| 56 | 56 | [`voice-network.json`](../service/spacebar/voice-network.json) configures the |
| 57 | 57 | loopback WebSocket mapping and the UDP media port. The supervisor builds Pion |
| 58 | from the Snowcord revision's `flake.lock` and caches the binary for that pin. | |
| 58 | from the Snowcord revision's `flake.lock`: git fetches the pinned commit, `go.sum` | |
| 59 | pins the modules, and the build and SFU run with a minimal environment. The binary | |
| 60 | is kept in `/work` for that pin, so a restart needs no network. | |
| 59 | 61 | It starts one SFU and one standalone voice process after the API is healthy. |
| 60 | 62 | Each app release restarts voice, and an unavailable voice process retries |
| 61 | 63 | while the API continues serving. Previews leave voice disabled. |