| author | |
| committer | |
| log | c1158b6f7b5deb3cd0bc0c27460b4bb42ca8556d |
| tree | 2d8543e65e90ba43601c77934983f68880bdeba0 |
| parent | d80e473e7de68b35767ef38d2264bb4bbf01d685 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Move the API, gateway and CDN to phoenix-selfhosted.paperclover.net, serve
voice from voice.phoenix-selfhosted, give /work a persistent volume instead of
a memory-backed tmpfs, fetch the Pion pin through git, and run the Go build
and SFU with a minimal environment.
Assisted-by: claude-sonnet-5.55 files changed, 36 insertions(+), 22 deletions(-)
service/spacebar/build/supervisor.mjs+25-12| ... | @@ -2,7 +2,7 @@ import { execFile, spawn } from "node:child_process"; | ... | @@ -2,7 +2,7 @@ import { execFile, spawn } from "node:child_process"; |
| 2 | import { createHash } from "node:crypto"; | 2 | import { createHash } from "node:crypto"; |
| 3 | import { resolve4 } from "node:dns/promises"; | 3 | import { resolve4 } from "node:dns/promises"; |
| 4 | import { once } from "node:events"; | 4 | import { once } from "node:events"; |
| 5 | import { copyFileSync, createWriteStream, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; | 5 | import { copyFileSync, createWriteStream, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; |
| 6 | import { connect, createServer } from "node:net"; | 6 | import { connect, createServer } from "node:net"; |
| 7 | import { basename } from "node:path"; | 7 | import { basename } from "node:path"; |
| 8 | import { createInterface } from "node:readline"; | 8 | import { createInterface } from "node:readline"; |
| ... | @@ -165,20 +165,33 @@ async function prepareSfu(release) { | ... | @@ -165,20 +165,33 @@ async function prepareSfu(release) { |
| 165 | if (pin.owner !== "spacebarchat" || pin.repo !== "pion-webrtc" || !/^[a-f0-9]{40}$/.test(pin.rev)) { | 165 | if (pin.owner !== "spacebarchat" || pin.repo !== "pion-webrtc" || !/^[a-f0-9]{40}$/.test(pin.rev)) { |
| 166 | throw new Error("invalid Snowcord Pion SFU pin"); | 166 | throw new Error("invalid Snowcord Pion SFU pin"); |
| 167 | } | 167 | } |
| 168 | const binary = `${WORK}/pion-sfu-${pin.rev}`; | 168 | const name = `pion-sfu-${pin.rev}`; |
| 169 | const binary = `${WORK}/${name}`; | ||
| 169 | if (existsSync(binary)) return binary; | 170 | if (existsSync(binary)) return binary; |
| 170 | 171 | ||
| 171 | const scratch = mkdtempSync(`${WORK}/build-sfu-`); | 172 | const scratch = `${WORK}/sfu-build`; |
| 173 | const source = `${scratch}/source`; | ||
| 174 | rmSync(scratch, { recursive: true, force: true }); | ||
| 175 | mkdirSync(source, { recursive: true }); | ||
| 172 | try { | 176 | try { |
| 173 | const source = await fetch(`https://codeload.github.com/spacebarchat/pion-webrtc/tar.gz/${pin.rev}`, { signal: AbortSignal.timeout(60000) }); | 177 | // Fetching by full object id makes git verify the commit; go.sum pins every module. |
| 174 | if (!source.ok) throw new Error(`Pion SFU download failed with HTTP ${source.status}`); | 178 | await run("git", ["-C", source, "init", "--quiet"]); |
| 175 | writeFileSync(`${scratch}/source.tar.gz`, new Uint8Array(await source.arrayBuffer())); | 179 | await run("git", ["-C", source, "fetch", "--quiet", "--depth", "1", `https://github.com/${pin.owner}/${pin.repo}`, pin.rev]); |
| 176 | await run("tar", ["-xzf", `${scratch}/source.tar.gz`, "-C", scratch]); | 180 | await run("git", ["-C", source, "-c", "advice.detachedHead=false", "checkout", "--quiet", "FETCH_HEAD"]); |
| 177 | await run("/usr/local/go/bin/go", ["build", "-trimpath", "-o", `${scratch}/pion-sfu`, "."], { | 181 | await run("/usr/local/go/bin/go", ["build", "-trimpath", "-buildvcs=false", "-o", `${scratch}/pion-sfu`, "."], { |
| 178 | cwd: `${scratch}/pion-webrtc-${pin.rev}/pion-sfu`, | 182 | cwd: `${source}/pion-sfu`, |
| 179 | env: { ...process.env, GOTOOLCHAIN: "local", CGO_ENABLED: "0" }, | 183 | env: { |
| 184 | PATH: process.env.PATH, | ||
| 185 | HOME: scratch, | ||
| 186 | GOPATH: `${scratch}/go`, | ||
| 187 | GOCACHE: `${scratch}/cache`, | ||
| 188 | GOFLAGS: "-mod=readonly -modcacherw", | ||
| 189 | GOTOOLCHAIN: "local", | ||
| 190 | CGO_ENABLED: "0", | ||
| 191 | }, | ||
| 180 | }); | 192 | }); |
| 181 | renameSync(`${scratch}/pion-sfu`, binary); | 193 | renameSync(`${scratch}/pion-sfu`, binary); |
| 194 | for (const entry of readdirSync(WORK)) if (entry.startsWith("pion-sfu-") && entry !== name) rmSync(`${WORK}/${entry}`, { force: true }); | ||
| 182 | return binary; | 195 | return binary; |
| 183 | } finally { | 196 | } finally { |
| 184 | rmSync(scratch, { recursive: true, force: true }); | 197 | rmSync(scratch, { recursive: true, force: true }); |
| ... | @@ -203,10 +216,10 @@ async function startVoice(release) { | ... | @@ -203,10 +216,10 @@ async function startVoice(release) { |
| 203 | if (stopping || live !== release) return; | 216 | if (stopping || live !== release) return; |
| 204 | 217 | ||
| 205 | const [ip] = addresses; | 218 | const [ip] = addresses; |
| 206 | const start = (command, args, env = process.env) => capture(spawn(command, args, { | 219 | const start = (command, args, env) => capture(spawn(command, args, { |
| 207 | cwd: "/data", stdio: ["ignore", "pipe", "pipe"], env, | 220 | cwd: "/data", stdio: ["ignore", "pipe", "pipe"], env, |
| 208 | })).on("error", (error) => log(`voice process: ${error.message}`)); | 221 | })).on("error", (error) => log(`voice process: ${error.message}`)); |
| 209 | voice = { release, sfu: start(binary, ["-ip", ip, "-port", VOICE_MEDIA_PORT]), signaling: null }; | 222 | voice = { release, sfu: start(binary, ["-ip", ip, "-port", VOICE_MEDIA_PORT], {}), signaling: null }; |
| 210 | await waitForVoice(voice.sfu, () => statSync("/tmp/sfu-ipc.sock", { throwIfNoEntry: false })?.isSocket()); | 223 | await waitForVoice(voice.sfu, () => statSync("/tmp/sfu-ipc.sock", { throwIfNoEntry: false })?.isSocket()); |
| 211 | if (stopping) return; | 224 | if (stopping) return; |
| 212 | 225 |
service/spacebar/service.pkl+2-3| ... | @@ -32,10 +32,9 @@ container { | ... | @@ -32,10 +32,9 @@ container { |
| 32 | 32 | ||
| 33 | http { | 33 | http { |
| 34 | containerPort = 3001 | 34 | containerPort = 3001 |
| 35 | subdomain = "spacebar" | 35 | subdomain = "phoenix-selfhosted" |
| 36 | checkPath = "/api/ping" | 36 | checkPath = "/api/ping" |
| 37 | forwardRealIp = true | 37 | forwardRealIp = true |
| 38 | plainHostnames { "phoenix-selfhosted.\(site.domain)" } | ||
| 39 | } | 38 | } |
| 40 | 39 | ||
| 41 | tcp { | 40 | tcp { |
| ... | @@ -47,12 +46,12 @@ container { | ... | @@ -47,12 +46,12 @@ container { |
| 47 | 46 | ||
| 48 | volumes { | 47 | volumes { |
| 49 | ["/data"] {} | 48 | ["/data"] {} |
| 49 | ["/work"] {} | ||
| 50 | // Shale keeps each repository under its UUID, which renaming and visibility changes leave alone. | 50 | // Shale keeps each repository under its UUID, which renaming and visibility changes leave alone. |
| 51 | ["/source.git"] { src = "\(site.root)/prod/shale/repositories_owned/006AH3H3QKH2KN318QXRNFBH4B"; readOnly = true } | 51 | ["/source.git"] { src = "\(site.root)/prod/shale/repositories_owned/006AH3H3QKH2KN318QXRNFBH4B"; readOnly = true } |
| 52 | // Production writes the server log into the sandbox workspace; previews keep their own. | 52 | // Production writes the server log into the sandbox workspace; previews keep their own. |
| 53 | ["/logs"] { when (!site.preview) { src = "\(site.root)/work/snowcord-server/logs" } } | 53 | ["/logs"] { when (!site.preview) { src = "\(site.root)/work/snowcord-server/logs" } } |
| 54 | } | 54 | } |
| 55 | tmpfs { "/work:rw,exec,size=3g" } | ||
| 56 | 55 | ||
| 57 | env { | 56 | env { |
| 58 | ["BRANCH"] = "main" | 57 | ["BRANCH"] = "main" |
service/spacebar/voice-network.json+1-1| ... | @@ -1,5 +1,5 @@ | ... | @@ -1,5 +1,5 @@ |
| 1 | { | 1 | { |
| 2 | "subdomain": "spacebar-voice", | 2 | "subdomain": "voice.phoenix-selfhosted", |
| 3 | "signaling": 17404, | 3 | "signaling": 17404, |
| 4 | "media": 6000 | 4 | "media": 6000 |
| 5 | } | 5 | } |
tools/phoenix-box.md+1-1| ... | @@ -17,7 +17,7 @@ on the tailnet. | ... | @@ -17,7 +17,7 @@ on the tailnet. |
| 17 | (any start clears it). Restarts log to `~/Library/Logs/m3-phoenix-box-revive.log`. | 17 | (any start clears it). Restarts log to `~/Library/Logs/m3-phoenix-box-revive.log`. |
| 18 | After a host reboot nothing runs until Clover unlocks FileVault and logs in. | 18 | After a host reboot nothing runs until Clover unlocks FileVault and logs in. |
| 19 | - **Names:** the guest has no DNS (its gateway is blocked), so `/etc/hosts` maps | 19 | - **Names:** the guest has no DNS (its gateway is blocked), so `/etc/hosts` maps |
| 20 | `spacebar.paperclover.net` to `100.64.0.1`. Go ignores `/etc/hosts` on macOS unless | 20 | `phoenix-selfhosted.paperclover.net` to `100.64.0.1`. Go ignores `/etc/hosts` on macOS unless |
| 21 | `GODEBUG=netdns=go`, which `run-phoenix-spacebar` sets for the Caddy relay. | 21 | `GODEBUG=netdns=go`, which `run-phoenix-spacebar` sets for the Caddy relay. |
| 22 | - **Access:** Softnet also blocks host to guest, so there is no SSH or Screen Sharing. | 22 | - **Access:** Softnet also blocks host to guest, so there is no SSH or Screen Sharing. |
| 23 | Tart's own VNC server (random password per run, printed to `run.log` in | 23 | Tart's own VNC server (random password per run, printed to `run.log` in |
tools/spacebar.md+7-5| ... | @@ -1,6 +1,6 @@ | ... | @@ -1,6 +1,6 @@ |
| 1 | # Spacebar | 1 | # Spacebar |
| 2 | 2 | ||
| 3 | `https://spacebar.paperclover.net` runs Clover's fork of | 3 | `https://phoenix-selfhosted.paperclover.net` runs Clover's fork of |
| 4 | [spacebarchat/server](https://github.com/spacebarchat/server) from branch `main` of | 4 | [spacebarchat/server](https://github.com/spacebarchat/server) from branch `main` of |
| 5 | the private Shale repository `https://shale.paperclover.net/snowcord-server.git`. | 5 | the private Shale repository `https://shale.paperclover.net/snowcord-server.git`. |
| 6 | Pushing to that branch is the deploy; `tools/deploy.py` is only needed to change the service definition. | 6 | Pushing to that branch is the deploy; `tools/deploy.py` is only needed to change the service definition. |
| ... | @@ -9,7 +9,7 @@ Pushing to that branch is the deploy; `tools/deploy.py` is only needed to change | ... | @@ -9,7 +9,7 @@ Pushing to that branch is the deploy; `tools/deploy.py` is only needed to change |
| 9 | ~/dev/snowcord-server jj checkout, remotes: origin = Shale, upstream = GitHub | 9 | ~/dev/snowcord-server jj checkout, remotes: origin = Shale, upstream = GitHub |
| 10 | (`master` mirrors upstream, `main` is what runs) | 10 | (`master` mirrors upstream, `main` is what runs) |
| 11 | jj bookmark set main -r @- && jj git push -b main | 11 | jj bookmark set main -r @- && jj git push -b main |
| 12 | curl https://spacebar.paperclover.net/_spacebar/api/version # commit that is live | 12 | curl https://phoenix-selfhosted.paperclover.net/_spacebar/api/version # commit that is live |
| 13 | ``` | 13 | ``` |
| 14 | 14 | ||
| 15 | A push is live about 4 seconds later (3.4 to 4.6 s measured from the start of the push). Dependency changes | 15 | A push is live about 4 seconds later (3.4 to 4.6 s measured from the start of the push). Dependency changes |
| ... | @@ -19,7 +19,7 @@ supervisor against the same branch with a fresh database. | ... | @@ -19,7 +19,7 @@ supervisor against the same branch with a fresh database. |
| 19 | ## How it works | 19 | ## How it works |
| 20 | 20 | ||
| 21 | [`supervisor.mjs`](../service/spacebar/build/supervisor.mjs) is the container's | 21 | [`supervisor.mjs`](../service/spacebar/build/supervisor.mjs) is the container's |
| 22 | only process, on a stock Node image with its working files in tmpfs `/work`. | 22 | only process, on a stock Node image with its working files in the persistent `/work` volume. |
| 23 | 23 | ||
| 24 | 1. Every 250 ms it reads the branch tip from the repository's Shale storage, | 24 | 1. Every 250 ms it reads the branch tip from the repository's Shale storage, |
| 25 | mounted read-only at `/source.git`. Shale keeps a repository under its UUID, | 25 | mounted read-only at `/source.git`. Shale keeps a repository under its UUID, |
| ... | @@ -52,10 +52,12 @@ start that is still waiting. Rolling back is pushing the old commit again. | ... | @@ -52,10 +52,12 @@ start that is still waiting. Rolling back is pushing the old commit again. |
| 52 | 52 | ||
| 53 | ## Voice | 53 | ## Voice |
| 54 | 54 | ||
| 55 | The production service publishes the voice WebSocket at `spacebar-voice.paperclover.net:443`. | 55 | The production service publishes the voice WebSocket at `voice.phoenix-selfhosted.paperclover.net:443`. |
| 56 | [`voice-network.json`](../service/spacebar/voice-network.json) configures the | 56 | [`voice-network.json`](../service/spacebar/voice-network.json) configures the |
| 57 | loopback WebSocket mapping and the UDP media port. The supervisor builds Pion | 57 | loopback WebSocket mapping and the UDP media port. The supervisor builds Pion |
| 58 | from the Snowcord revision's `flake.lock` and caches the binary for that pin. | 58 | from the Snowcord revision's `flake.lock`: git fetches the pinned commit, `go.sum` |
| 59 | pins the modules, and the build and SFU run with a minimal environment. The binary | ||
| 60 | is kept in `/work` for that pin, so a restart needs no network. | ||
| 59 | It starts one SFU and one standalone voice process after the API is healthy. | 61 | It starts one SFU and one standalone voice process after the API is healthy. |
| 60 | Each app release restarts voice, and an unavailable voice process retries | 62 | Each app release restarts voice, and an unavailable voice process retries |
| 61 | while the API continues serving. Previews leave voice disabled. | 63 | while the API continues serving. Previews leave voice disabled. |