1import { execFile, spawn } from "node:child_process";
2import { createHash } from "node:crypto";
3import { resolve4 } from "node:dns/promises";
4import { once } from "node:events";
5import { copyFileSync, createWriteStream, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs";
6import { connect, createServer } from "node:net";
7import { basename } from "node:path";
8import { createInterface } from "node:readline";
9import { setTimeout as sleep } from "node:timers/promises";
10import { promisify } from "node:util";
11
12const { BRANCH, PORT, VOICE_PUBLIC_HOST, VOICE_SIGNALING_PORT, VOICE_MEDIA_PORT } = process.env;
13const WORK = "/work";
14const REPO = "/source.git";
15const POLL_MS = 250;
16const DRAIN_MS = 500;
17const RETRY_FAILED_MS = 15000;
18const GIT = ["-c", "safe.directory=*", "--git-dir", REPO];
19const LOG = "/logs/server.log";
20const LOG_MAX_BYTES = 10 * 1024 * 1024;
21
22const out = promisify(execFile);
23
24let sink;
25let sinkBytes = 0;
26function openSink() {
27 sink = createWriteStream(LOG, { flags: "a" }).on("error", () => {});
28 sinkBytes = existsSync(LOG) ? statSync(LOG).size : 0;
29}
30openSink();
31
32/** Everything the container prints is also kept in LOG, with one rotated copy. */
33function record(line) {
34 const text = `${new Date().toISOString()} ${line}\n`;
35 if (sinkBytes + text.length > LOG_MAX_BYTES) {
36 sink.end();
37 try {
38 renameSync(LOG, `${LOG}.1`);
39 } catch {}
40 openSink();
41 }
42 sink.write(text);
43 sinkBytes += text.length;
44}
45
46function capture(proc) {
47 for (const [stream, output] of [[proc.stdout, process.stdout], [proc.stderr, process.stderr]]) {
48 createInterface({ input: stream }).on("line", (line) => {
49 output.write(`${line}\n`);
50 record(line);
51 });
52 }
53 return proc;
54}
55
56const log = (...args) => {
57 console.log("[deploy]", ...args);
58 record(["[deploy]", ...args].join(" "));
59};
60const run = (cmd, args, opts = {}, ok = [0]) =>
61 new Promise((resolve, reject) =>
62 capture(spawn(cmd, args, { stdio: ["ignore", "pipe", "pipe"], ...opts }))
63 .once("error", reject)
64 .once("exit", (code) => (ok.includes(code) ? resolve() : reject(new Error(`${basename(cmd)} ${args[0]} exited with ${code}`)))),
65 );
66const git = (...args) => out("git", [...GIT, ...args]).then((result) => result.stdout.trim());
67
68let live = null;
69let wanted = null;
70let recentModules = null;
71const releases = new Set();
72let failed = { rev: null, at: 0 };
73let voice = null;
74let retryVoiceAt = 0;
75let stopping = false;
76
77createServer((client) => {
78 const target = live;
79 if (!target) return client.destroy();
80 const upstream = connect(target.port, "127.0.0.1");
81 target.sockets.add(client);
82 client.setNoDelay(true);
83 upstream.setNoDelay(true);
84 const end = () => {
85 client.destroy();
86 upstream.destroy();
87 target.sockets.delete(client);
88 };
89 client.pipe(upstream).pipe(client);
90 client.on("error", end).on("close", end);
91 upstream.on("error", end).on("close", end);
92}).listen(Number(PORT));
93
94const branchTip = () =>
95 git("rev-parse", "--verify", "--quiet", `refs/heads/${BRANCH}`).catch((error) => {
96 if (error.code !== 1) throw error;
97 return null;
98 });
99
100async function install(dir, modules) {
101 const staging = `${modules}.partial`;
102 rmSync(staging, { recursive: true, force: true });
103 mkdirSync(staging, { recursive: true });
104 for (const file of ["package.json", "package-lock.json"]) copyFileSync(`${dir}/${file}`, `${staging}/${file}`);
105 await run("npm", ["ci", "--ignore-scripts", "--no-audit", "--no-fund", "--loglevel=error"], { cwd: staging });
106 const { packages } = JSON.parse(readFileSync(`${staging}/package-lock.json`));
107 const native = Object.entries(packages)
108 .filter(([path, pkg]) => path && pkg.hasInstallScript)
109 .map(([path]) => path.split("node_modules/").pop());
110 if (native.length) await run("npm", ["rebuild", "--loglevel=error", ...native], { cwd: staging });
111 renameSync(staging, modules);
112}
113
114async function healthy(port, proc, current) {
115 for (const deadline = Date.now() + 90000; Date.now() < deadline; await sleep(100)) {
116 if (proc.exitCode !== null || proc.signalCode !== null) throw new Error("server exited during startup");
117 current();
118 const ok = await fetch(`http://127.0.0.1:${port}/api/ping`, { signal: AbortSignal.timeout(1000) }).then((response) => response.ok, () => false);
119 if (ok) return;
120 }
121 throw new Error("server did not become healthy");
122}
123
124const freePort = () =>
125 new Promise((resolve) => {
126 const probe = createServer().listen(0, "127.0.0.1", () => {
127 const { port } = probe.address();
128 probe.close(() => resolve(port));
129 });
130 });
131
132function sweep() {
133 const used = { rel: new Set(), modules: new Set([recentModules]) };
134 for (const release of [...releases, ...(voice ? [voice.release] : [])]) {
135 used.rel.add(release.rev);
136 if (release.modules) used.modules.add(basename(release.modules)).add(`${basename(release.modules)}.partial`);
137 }
138 for (const [name, keep] of Object.entries(used)) {
139 for (const entry of readdirSync(`${WORK}/${name}`)) if (!keep.has(entry)) rmSync(`${WORK}/${name}/${entry}`, { recursive: true, force: true });
140 }
141}
142
143const running = (proc) => proc?.pid && proc.exitCode === null && proc.signalCode === null;
144
145async function stop(proc) {
146 if (!running(proc)) return;
147 const exited = once(proc, "exit");
148 proc.kill("SIGTERM");
149 await Promise.race([exited, sleep(3000, undefined, { ref: false })]);
150 if (running(proc)) {
151 proc.kill("SIGKILL");
152 await exited;
153 }
154}
155
156async function stopVoice() {
157 if (!voice) return;
158 await Promise.all([stop(voice.signaling), stop(voice.sfu)]);
159 rmSync("/tmp/sfu-ipc.sock", { force: true });
160 voice = null;
161}
162
163async function prepareSfu(release) {
164 const pin = JSON.parse(readFileSync(`${release.dir}/flake.lock`, "utf8")).nodes["pion-webrtc"].locked;
165 if (pin.owner !== "spacebarchat" || pin.repo !== "pion-webrtc" || !/^[a-f0-9]{40}$/.test(pin.rev)) {
166 throw new Error("invalid Snowcord Pion SFU pin");
167 }
168 const name = `pion-sfu-${pin.rev}`;
169 const binary = `${WORK}/${name}`;
170 if (existsSync(binary)) return binary;
171
172 const scratch = `${WORK}/sfu-build`;
173 const source = `${scratch}/source`;
174 rmSync(scratch, { recursive: true, force: true });
175 mkdirSync(source, { recursive: true });
176 try {
177 // Fetching by full object id makes git verify the commit; go.sum pins every module.
178 await run("git", ["-C", source, "init", "--quiet"]);
179 await run("git", ["-C", source, "fetch", "--quiet", "--depth", "1", `https://github.com/${pin.owner}/${pin.repo}`, pin.rev]);
180 await run("git", ["-C", source, "-c", "advice.detachedHead=false", "checkout", "--quiet", "FETCH_HEAD"]);
181 await run("/usr/local/go/bin/go", ["build", "-trimpath", "-buildvcs=false", "-o", `${scratch}/pion-sfu`, "."], {
182 cwd: `${source}/pion-sfu`,
183 env: {
184 PATH: process.env.PATH,
185 HOME: scratch,
186 GOPATH: `${scratch}/go`,
187 GOCACHE: `${scratch}/cache`,
188 GOFLAGS: "-mod=readonly -modcacherw",
189 GOTOOLCHAIN: "local",
190 CGO_ENABLED: "0",
191 },
192 });
193 renameSync(`${scratch}/pion-sfu`, binary);
194 for (const entry of readdirSync(WORK)) if (entry.startsWith("pion-sfu-") && entry !== name) rmSync(`${WORK}/${entry}`, { force: true });
195 return binary;
196 } finally {
197 rmSync(scratch, { recursive: true, force: true });
198 }
199}
200
201async function waitForVoice(proc, ready) {
202 for (const deadline = Date.now() + 15000; Date.now() < deadline; await sleep(100)) {
203 if (!running(proc)) throw new Error("voice process exited during startup");
204 if (await ready()) return;
205 }
206 throw new Error("voice process did not become ready");
207}
208
209async function startVoice(release) {
210 try {
211 const addresses = await resolve4(VOICE_PUBLIC_HOST);
212 if (addresses.length !== 1) throw new Error("voice hostname must resolve to one IPv4 address");
213 const binary = await prepareSfu(release);
214 if (stopping) return;
215 await stopVoice();
216 if (stopping || live !== release) return;
217
218 const [ip] = addresses;
219 const start = (command, args, env) => capture(spawn(command, args, {
220 cwd: "/data", stdio: ["ignore", "pipe", "pipe"], env,
221 })).on("error", (error) => log(`voice process: ${error.message}`));
222 voice = { release, sfu: start(binary, ["-ip", ip, "-port", VOICE_MEDIA_PORT], {}), signaling: null };
223 await waitForVoice(voice.sfu, () => statSync("/tmp/sfu-ipc.sock", { throwIfNoEntry: false })?.isSocket());
224 if (stopping) return;
225
226 voice.signaling = start(process.execPath, ["--enable-source-maps", `${release.dir}/dist/webrtc/start.js`], {
227 ...process.env,
228 PORT: VOICE_SIGNALING_PORT,
229 APPLY_DB_MIGRATIONS: "false",
230 WRTC_LIBRARY: "@spacebarchat/pion-webrtc",
231 WRTC_PUBLIC_IP: ip,
232 WRTC_PORT_MIN: VOICE_MEDIA_PORT,
233 WRTC_PORT_MAX: VOICE_MEDIA_PORT,
234 });
235 await waitForVoice(voice.signaling, () => fetch(`http://127.0.0.1:${VOICE_SIGNALING_PORT}/`, {
236 signal: AbortSignal.timeout(1000),
237 }).then((response) => response.ok, () => false));
238 retryVoiceAt = 0;
239 log(`voice live ${release.rev.slice(0, 7)} with UDP ${VOICE_MEDIA_PORT}`);
240 } catch (error) {
241 if (voice?.release === release) await stopVoice();
242 retryVoiceAt = Date.now() + RETRY_FAILED_MS;
243 log(`voice ${release.rev.slice(0, 7)} failed: ${error.message}`);
244 }
245}
246
247function retire(old) {
248 setTimeout(() => {
249 for (const socket of old.sockets) socket.destroy();
250 old.proc.kill("SIGTERM");
251 setTimeout(() => old.proc.kill("SIGKILL"), 5000).unref();
252 }, DRAIN_MS);
253}
254
255async function deploy() {
256 const rev = wanted;
257 const current = () => {
258 if (wanted !== rev) throw new Error("superseded by a newer push");
259 };
260 let release;
261 let last = performance.now();
262 const start = last;
263 const laps = [];
264 const lap = (name) => {
265 const now = performance.now();
266 laps.push(`${name} ${Math.round(now - last)}ms`);
267 last = now;
268 };
269
270 try {
271 const [committedAt, subject] = (await git("log", "-1", "--format=%ct%n%s", rev)).split("\n");
272 log(`deploying ${rev.slice(0, 7)} ${subject}`);
273
274 release = { rev, dir: `${WORK}/rel/${rev}`, modules: null, port: 0, proc: null, sockets: new Set() };
275 releases.add(release);
276 rmSync(release.dir, { recursive: true, force: true });
277 mkdirSync(release.dir, { recursive: true });
278 await run("bash", ["-c", 'set -o pipefail; git -c safe.directory="*" --git-dir "$0" archive "$1" | tar -x -C "$2"', REPO, rev, release.dir]);
279 lap("extract");
280
281 const lock = createHash("sha256");
282 for (const file of ["package.json", "package-lock.json"]) lock.update(readFileSync(`${release.dir}/${file}`));
283 release.modules = `${WORK}/modules/${lock.digest("hex").slice(0, 16)}`;
284 const cached = existsSync(release.modules);
285 if (!cached) await install(release.dir, release.modules);
286 symlinkSync(`${release.modules}/node_modules`, `${release.dir}/node_modules`);
287 lap(cached ? "deps cached" : "deps installed");
288
289 await run(`${release.dir}/node_modules/.bin/tsc`, ["-p", "tsconfig.json", "--declaration", "false", "--declarationMap", "false", "--incremental", "false"], { cwd: release.dir });
290 writeFileSync(`${release.dir}/.rev`, JSON.stringify({ rev, shortRev: rev.slice(0, 7), lastModified: Number(committedAt) }));
291 lap("build");
292
293 current();
294 release.port = await freePort();
295 // The server keeps jwt.key and other state in its working directory.
296 release.proc = capture(spawn(process.execPath, ["--enable-source-maps", `${release.dir}/dist/bundle/start.js`], {
297 cwd: "/data",
298 stdio: ["ignore", "pipe", "pipe"],
299 env: { ...process.env, PORT: String(release.port) },
300 }));
301 release.proc.once("exit", (code, signal) => {
302 if (live === release) {
303 live = null;
304 log(`server ${rev.slice(0, 7)} exited (${code ?? signal})`);
305 }
306 releases.delete(release);
307 sweep();
308 });
309 try {
310 await healthy(release.port, release.proc, current);
311 } catch (error) {
312 release.proc.kill("SIGKILL");
313 throw error;
314 }
315 lap("start");
316
317 const old = live;
318 live = release;
319 recentModules = basename(release.modules);
320 if (old) retire(old);
321 log(`live ${rev.slice(0, 7)} in ${((performance.now() - start) / 1000).toFixed(1)}s: ${laps.join(", ")}`);
322 } catch (error) {
323 log(`${rev.slice(0, 7)} failed: ${error.message}`);
324 failed = { rev, at: Date.now() };
325 if (release) releases.delete(release);
326 sweep();
327 }
328}
329
330mkdirSync(`${WORK}/rel`, { recursive: true });
331mkdirSync(`${WORK}/modules`, { recursive: true });
332process.once("SIGTERM", async () => {
333 stopping = true;
334 await stopVoice();
335 process.exit(0);
336});
337
338(async () => {
339 let problem = "";
340 for (;; await sleep(POLL_MS)) {
341 try {
342 const head = await branchTip();
343 if (!head) throw new Error(`${REPO} has no branch ${BRANCH}`);
344 wanted = head;
345 problem = "";
346 } catch (error) {
347 if (error.message !== problem) log((problem = error.message));
348 }
349 }
350})();
351
352for (;;) {
353 const retry = wanted === failed.rev && !live && Date.now() - failed.at > RETRY_FAILED_MS;
354 if (wanted && wanted !== live?.rev && (wanted !== failed.rev || retry)) await deploy();
355 else if (VOICE_SIGNALING_PORT && live && Date.now() >= retryVoiceAt
356 && (voice?.release !== live || !running(voice.sfu) || !running(voice.signaling))) await startVoice(live);
357 else await sleep(100);
358}