| 1 | import { execFile, spawn } from "node:child_process"; |
| 2 | import { createHash } from "node:crypto"; |
| 3 | import { resolve4 } from "node:dns/promises"; |
| 4 | import { once } from "node:events"; |
| 5 | import { copyFileSync, createWriteStream, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; |
| 6 | import { connect, createServer } from "node:net"; |
| 7 | import { basename } from "node:path"; |
| 8 | import { createInterface } from "node:readline"; |
| 9 | import { setTimeout as sleep } from "node:timers/promises"; |
| 10 | import { promisify } from "node:util"; |
| 11 | |
| 12 | const { BRANCH, PORT, VOICE_PUBLIC_HOST, VOICE_SIGNALING_PORT, VOICE_MEDIA_PORT } = process.env; |
| 13 | const WORK = "/work"; |
| 14 | const REPO = "/source.git"; |
| 15 | const POLL_MS = 250; |
| 16 | const DRAIN_MS = 500; |
| 17 | const RETRY_FAILED_MS = 15000; |
| 18 | const GIT = ["-c", "safe.directory=*", "--git-dir", REPO]; |
| 19 | const LOG = "/logs/server.log"; |
| 20 | const LOG_MAX_BYTES = 10 * 1024 * 1024; |
| 21 | |
| 22 | const out = promisify(execFile); |
| 23 | |
| 24 | let sink; |
| 25 | let sinkBytes = 0; |
| 26 | function openSink() { |
| 27 | sink = createWriteStream(LOG, { flags: "a" }).on("error", () => {}); |
| 28 | sinkBytes = existsSync(LOG) ? statSync(LOG).size : 0; |
| 29 | } |
| 30 | openSink(); |
| 31 | |
| 32 | /** Everything the container prints is also kept in LOG, with one rotated copy. */ |
| 33 | function record(line) { |
| 34 | const text = `${new Date().toISOString()} ${line}\n`; |
| 35 | if (sinkBytes + text.length > LOG_MAX_BYTES) { |
| 36 | sink.end(); |
| 37 | try { |
| 38 | renameSync(LOG, `${LOG}.1`); |
| 39 | } catch {} |
| 40 | openSink(); |
| 41 | } |
| 42 | sink.write(text); |
| 43 | sinkBytes += text.length; |
| 44 | } |
| 45 | |
| 46 | function capture(proc) { |
| 47 | for (const [stream, output] of [[proc.stdout, process.stdout], [proc.stderr, process.stderr]]) { |
| 48 | createInterface({ input: stream }).on("line", (line) => { |
| 49 | output.write(`${line}\n`); |
| 50 | record(line); |
| 51 | }); |
| 52 | } |
| 53 | return proc; |
| 54 | } |
| 55 | |
| 56 | const log = (...args) => { |
| 57 | console.log("[deploy]", ...args); |
| 58 | record(["[deploy]", ...args].join(" ")); |
| 59 | }; |
| 60 | const run = (cmd, args, opts = {}, ok = [0]) => |
| 61 | new Promise((resolve, reject) => |
| 62 | capture(spawn(cmd, args, { stdio: ["ignore", "pipe", "pipe"], ...opts })) |
| 63 | .once("error", reject) |
| 64 | .once("exit", (code) => (ok.includes(code) ? resolve() : reject(new Error(`${basename(cmd)} ${args[0]} exited with ${code}`)))), |
| 65 | ); |
| 66 | const git = (...args) => out("git", [...GIT, ...args]).then((result) => result.stdout.trim()); |
| 67 | |
| 68 | let live = null; |
| 69 | let wanted = null; |
| 70 | let recentModules = null; |
| 71 | const releases = new Set(); |
| 72 | let failed = { rev: null, at: 0 }; |
| 73 | let voice = null; |
| 74 | let retryVoiceAt = 0; |
| 75 | let stopping = false; |
| 76 | |
| 77 | createServer((client) => { |
| 78 | const target = live; |
| 79 | if (!target) return client.destroy(); |
| 80 | const upstream = connect(target.port, "127.0.0.1"); |
| 81 | target.sockets.add(client); |
| 82 | client.setNoDelay(true); |
| 83 | upstream.setNoDelay(true); |
| 84 | const end = () => { |
| 85 | client.destroy(); |
| 86 | upstream.destroy(); |
| 87 | target.sockets.delete(client); |
| 88 | }; |
| 89 | client.pipe(upstream).pipe(client); |
| 90 | client.on("error", end).on("close", end); |
| 91 | upstream.on("error", end).on("close", end); |
| 92 | }).listen(Number(PORT)); |
| 93 | |
| 94 | const branchTip = () => |
| 95 | git("rev-parse", "--verify", "--quiet", `refs/heads/${BRANCH}`).catch((error) => { |
| 96 | if (error.code !== 1) throw error; |
| 97 | return null; |
| 98 | }); |
| 99 | |
| 100 | async function install(dir, modules) { |
| 101 | const staging = `${modules}.partial`; |
| 102 | rmSync(staging, { recursive: true, force: true }); |
| 103 | mkdirSync(staging, { recursive: true }); |
| 104 | for (const file of ["package.json", "package-lock.json"]) copyFileSync(`${dir}/${file}`, `${staging}/${file}`); |
| 105 | await run("npm", ["ci", "--ignore-scripts", "--no-audit", "--no-fund", "--loglevel=error"], { cwd: staging }); |
| 106 | const { packages } = JSON.parse(readFileSync(`${staging}/package-lock.json`)); |
| 107 | const native = Object.entries(packages) |
| 108 | .filter(([path, pkg]) => path && pkg.hasInstallScript) |
| 109 | .map(([path]) => path.split("node_modules/").pop()); |
| 110 | if (native.length) await run("npm", ["rebuild", "--loglevel=error", ...native], { cwd: staging }); |
| 111 | renameSync(staging, modules); |
| 112 | } |
| 113 | |
| 114 | async function healthy(port, proc, current) { |
| 115 | for (const deadline = Date.now() + 90000; Date.now() < deadline; await sleep(100)) { |
| 116 | if (proc.exitCode !== null || proc.signalCode !== null) throw new Error("server exited during startup"); |
| 117 | current(); |
| 118 | const ok = await fetch(`http://127.0.0.1:${port}/api/ping`, { signal: AbortSignal.timeout(1000) }).then((response) => response.ok, () => false); |
| 119 | if (ok) return; |
| 120 | } |
| 121 | throw new Error("server did not become healthy"); |
| 122 | } |
| 123 | |
| 124 | const freePort = () => |
| 125 | new Promise((resolve) => { |
| 126 | const probe = createServer().listen(0, "127.0.0.1", () => { |
| 127 | const { port } = probe.address(); |
| 128 | probe.close(() => resolve(port)); |
| 129 | }); |
| 130 | }); |
| 131 | |
| 132 | function sweep() { |
| 133 | const used = { rel: new Set(), modules: new Set([recentModules]) }; |
| 134 | for (const release of [...releases, ...(voice ? [voice.release] : [])]) { |
| 135 | used.rel.add(release.rev); |
| 136 | if (release.modules) used.modules.add(basename(release.modules)).add(`${basename(release.modules)}.partial`); |
| 137 | } |
| 138 | for (const [name, keep] of Object.entries(used)) { |
| 139 | for (const entry of readdirSync(`${WORK}/${name}`)) if (!keep.has(entry)) rmSync(`${WORK}/${name}/${entry}`, { recursive: true, force: true }); |
| 140 | } |
| 141 | } |
| 142 | |
| 143 | const running = (proc) => proc?.pid && proc.exitCode === null && proc.signalCode === null; |
| 144 | |
| 145 | async function stop(proc) { |
| 146 | if (!running(proc)) return; |
| 147 | const exited = once(proc, "exit"); |
| 148 | proc.kill("SIGTERM"); |
| 149 | await Promise.race([exited, sleep(3000, undefined, { ref: false })]); |
| 150 | if (running(proc)) { |
| 151 | proc.kill("SIGKILL"); |
| 152 | await exited; |
| 153 | } |
| 154 | } |
| 155 | |
| 156 | async function stopVoice() { |
| 157 | if (!voice) return; |
| 158 | await Promise.all([stop(voice.signaling), stop(voice.sfu)]); |
| 159 | rmSync("/tmp/sfu-ipc.sock", { force: true }); |
| 160 | voice = null; |
| 161 | } |
| 162 | |
| 163 | async function prepareSfu(release) { |
| 164 | const pin = JSON.parse(readFileSync(`${release.dir}/flake.lock`, "utf8")).nodes["pion-webrtc"].locked; |
| 165 | if (pin.owner !== "spacebarchat" || pin.repo !== "pion-webrtc" || !/^[a-f0-9]{40}$/.test(pin.rev)) { |
| 166 | throw new Error("invalid Snowcord Pion SFU pin"); |
| 167 | } |
| 168 | const name = `pion-sfu-${pin.rev}`; |
| 169 | const binary = `${WORK}/${name}`; |
| 170 | if (existsSync(binary)) return binary; |
| 171 | |
| 172 | const scratch = `${WORK}/sfu-build`; |
| 173 | const source = `${scratch}/source`; |
| 174 | rmSync(scratch, { recursive: true, force: true }); |
| 175 | mkdirSync(source, { recursive: true }); |
| 176 | try { |
| 177 | // Fetching by full object id makes git verify the commit; go.sum pins every module. |
| 178 | await run("git", ["-C", source, "init", "--quiet"]); |
| 179 | await run("git", ["-C", source, "fetch", "--quiet", "--depth", "1", `https://github.com/${pin.owner}/${pin.repo}`, pin.rev]); |
| 180 | await run("git", ["-C", source, "-c", "advice.detachedHead=false", "checkout", "--quiet", "FETCH_HEAD"]); |
| 181 | await run("/usr/local/go/bin/go", ["build", "-trimpath", "-buildvcs=false", "-o", `${scratch}/pion-sfu`, "."], { |
| 182 | cwd: `${source}/pion-sfu`, |
| 183 | env: { |
| 184 | PATH: process.env.PATH, |
| 185 | HOME: scratch, |
| 186 | GOPATH: `${scratch}/go`, |
| 187 | GOCACHE: `${scratch}/cache`, |
| 188 | GOFLAGS: "-mod=readonly -modcacherw", |
| 189 | GOTOOLCHAIN: "local", |
| 190 | CGO_ENABLED: "0", |
| 191 | }, |
| 192 | }); |
| 193 | renameSync(`${scratch}/pion-sfu`, binary); |
| 194 | for (const entry of readdirSync(WORK)) if (entry.startsWith("pion-sfu-") && entry !== name) rmSync(`${WORK}/${entry}`, { force: true }); |
| 195 | return binary; |
| 196 | } finally { |
| 197 | rmSync(scratch, { recursive: true, force: true }); |
| 198 | } |
| 199 | } |
| 200 | |
| 201 | async function waitForVoice(proc, ready) { |
| 202 | for (const deadline = Date.now() + 15000; Date.now() < deadline; await sleep(100)) { |
| 203 | if (!running(proc)) throw new Error("voice process exited during startup"); |
| 204 | if (await ready()) return; |
| 205 | } |
| 206 | throw new Error("voice process did not become ready"); |
| 207 | } |
| 208 | |
| 209 | async function startVoice(release) { |
| 210 | try { |
| 211 | const addresses = await resolve4(VOICE_PUBLIC_HOST); |
| 212 | if (addresses.length !== 1) throw new Error("voice hostname must resolve to one IPv4 address"); |
| 213 | const binary = await prepareSfu(release); |
| 214 | if (stopping) return; |
| 215 | await stopVoice(); |
| 216 | if (stopping || live !== release) return; |
| 217 | |
| 218 | const [ip] = addresses; |
| 219 | const start = (command, args, env) => capture(spawn(command, args, { |
| 220 | cwd: "/data", stdio: ["ignore", "pipe", "pipe"], env, |
| 221 | })).on("error", (error) => log(`voice process: ${error.message}`)); |
| 222 | voice = { release, sfu: start(binary, ["-ip", ip, "-port", VOICE_MEDIA_PORT], {}), signaling: null }; |
| 223 | await waitForVoice(voice.sfu, () => statSync("/tmp/sfu-ipc.sock", { throwIfNoEntry: false })?.isSocket()); |
| 224 | if (stopping) return; |
| 225 | |
| 226 | voice.signaling = start(process.execPath, ["--enable-source-maps", `${release.dir}/dist/webrtc/start.js`], { |
| 227 | ...process.env, |
| 228 | PORT: VOICE_SIGNALING_PORT, |
| 229 | APPLY_DB_MIGRATIONS: "false", |
| 230 | WRTC_LIBRARY: "@spacebarchat/pion-webrtc", |
| 231 | WRTC_PUBLIC_IP: ip, |
| 232 | WRTC_PORT_MIN: VOICE_MEDIA_PORT, |
| 233 | WRTC_PORT_MAX: VOICE_MEDIA_PORT, |
| 234 | }); |
| 235 | await waitForVoice(voice.signaling, () => fetch(`http://127.0.0.1:${VOICE_SIGNALING_PORT}/`, { |
| 236 | signal: AbortSignal.timeout(1000), |
| 237 | }).then((response) => response.ok, () => false)); |
| 238 | retryVoiceAt = 0; |
| 239 | log(`voice live ${release.rev.slice(0, 7)} with UDP ${VOICE_MEDIA_PORT}`); |
| 240 | } catch (error) { |
| 241 | if (voice?.release === release) await stopVoice(); |
| 242 | retryVoiceAt = Date.now() + RETRY_FAILED_MS; |
| 243 | log(`voice ${release.rev.slice(0, 7)} failed: ${error.message}`); |
| 244 | } |
| 245 | } |
| 246 | |
| 247 | function retire(old) { |
| 248 | setTimeout(() => { |
| 249 | for (const socket of old.sockets) socket.destroy(); |
| 250 | old.proc.kill("SIGTERM"); |
| 251 | setTimeout(() => old.proc.kill("SIGKILL"), 5000).unref(); |
| 252 | }, DRAIN_MS); |
| 253 | } |
| 254 | |
| 255 | async function deploy() { |
| 256 | const rev = wanted; |
| 257 | const current = () => { |
| 258 | if (wanted !== rev) throw new Error("superseded by a newer push"); |
| 259 | }; |
| 260 | let release; |
| 261 | let last = performance.now(); |
| 262 | const start = last; |
| 263 | const laps = []; |
| 264 | const lap = (name) => { |
| 265 | const now = performance.now(); |
| 266 | laps.push(`${name} ${Math.round(now - last)}ms`); |
| 267 | last = now; |
| 268 | }; |
| 269 | |
| 270 | try { |
| 271 | const [committedAt, subject] = (await git("log", "-1", "--format=%ct%n%s", rev)).split("\n"); |
| 272 | log(`deploying ${rev.slice(0, 7)} ${subject}`); |
| 273 | |
| 274 | release = { rev, dir: `${WORK}/rel/${rev}`, modules: null, port: 0, proc: null, sockets: new Set() }; |
| 275 | releases.add(release); |
| 276 | rmSync(release.dir, { recursive: true, force: true }); |
| 277 | mkdirSync(release.dir, { recursive: true }); |
| 278 | await run("bash", ["-c", 'set -o pipefail; git -c safe.directory="*" --git-dir "$0" archive "$1" | tar -x -C "$2"', REPO, rev, release.dir]); |
| 279 | lap("extract"); |
| 280 | |
| 281 | const lock = createHash("sha256"); |
| 282 | for (const file of ["package.json", "package-lock.json"]) lock.update(readFileSync(`${release.dir}/${file}`)); |
| 283 | release.modules = `${WORK}/modules/${lock.digest("hex").slice(0, 16)}`; |
| 284 | const cached = existsSync(release.modules); |
| 285 | if (!cached) await install(release.dir, release.modules); |
| 286 | symlinkSync(`${release.modules}/node_modules`, `${release.dir}/node_modules`); |
| 287 | lap(cached ? "deps cached" : "deps installed"); |
| 288 | |
| 289 | await run(`${release.dir}/node_modules/.bin/tsc`, ["-p", "tsconfig.json", "--declaration", "false", "--declarationMap", "false", "--incremental", "false"], { cwd: release.dir }); |
| 290 | writeFileSync(`${release.dir}/.rev`, JSON.stringify({ rev, shortRev: rev.slice(0, 7), lastModified: Number(committedAt) })); |
| 291 | lap("build"); |
| 292 | |
| 293 | current(); |
| 294 | release.port = await freePort(); |
| 295 | // The server keeps jwt.key and other state in its working directory. |
| 296 | release.proc = capture(spawn(process.execPath, ["--enable-source-maps", `${release.dir}/dist/bundle/start.js`], { |
| 297 | cwd: "/data", |
| 298 | stdio: ["ignore", "pipe", "pipe"], |
| 299 | env: { ...process.env, PORT: String(release.port) }, |
| 300 | })); |
| 301 | release.proc.once("exit", (code, signal) => { |
| 302 | if (live === release) { |
| 303 | live = null; |
| 304 | log(`server ${rev.slice(0, 7)} exited (${code ?? signal})`); |
| 305 | } |
| 306 | releases.delete(release); |
| 307 | sweep(); |
| 308 | }); |
| 309 | try { |
| 310 | await healthy(release.port, release.proc, current); |
| 311 | } catch (error) { |
| 312 | release.proc.kill("SIGKILL"); |
| 313 | throw error; |
| 314 | } |
| 315 | lap("start"); |
| 316 | |
| 317 | const old = live; |
| 318 | live = release; |
| 319 | recentModules = basename(release.modules); |
| 320 | if (old) retire(old); |
| 321 | log(`live ${rev.slice(0, 7)} in ${((performance.now() - start) / 1000).toFixed(1)}s: ${laps.join(", ")}`); |
| 322 | } catch (error) { |
| 323 | log(`${rev.slice(0, 7)} failed: ${error.message}`); |
| 324 | failed = { rev, at: Date.now() }; |
| 325 | if (release) releases.delete(release); |
| 326 | sweep(); |
| 327 | } |
| 328 | } |
| 329 | |
| 330 | mkdirSync(`${WORK}/rel`, { recursive: true }); |
| 331 | mkdirSync(`${WORK}/modules`, { recursive: true }); |
| 332 | process.once("SIGTERM", async () => { |
| 333 | stopping = true; |
| 334 | await stopVoice(); |
| 335 | process.exit(0); |
| 336 | }); |
| 337 | |
| 338 | (async () => { |
| 339 | let problem = ""; |
| 340 | for (;; await sleep(POLL_MS)) { |
| 341 | try { |
| 342 | const head = await branchTip(); |
| 343 | if (!head) throw new Error(`${REPO} has no branch ${BRANCH}`); |
| 344 | wanted = head; |
| 345 | problem = ""; |
| 346 | } catch (error) { |
| 347 | if (error.message !== problem) log((problem = error.message)); |
| 348 | } |
| 349 | } |
| 350 | })(); |
| 351 | |
| 352 | for (;;) { |
| 353 | const retry = wanted === failed.rev && !live && Date.now() - failed.at > RETRY_FAILED_MS; |
| 354 | if (wanted && wanted !== live?.rev && (wanted !== failed.rev || retry)) await deploy(); |
| 355 | else if (VOICE_SIGNALING_PORT && live && Date.now() >= retryVoiceAt |
| 356 | && (voice?.release !== live || !running(voice.sfu) || !running(voice.signaling))) await startVoice(live); |
| 357 | else await sleep(100); |
| 358 | } |