diff --git a/service/spacebar/build/supervisor.mjs b/service/spacebar/build/supervisor.mjs index 53d5b3338c85dc21d1a922d92500fbc8424de56b..dcf2d4fcdbaca7b4387c68b0944d2053fc1eb645 100644 --- a/service/spacebar/build/supervisor.mjs +++ b/service/spacebar/build/supervisor.mjs @@ -1,8 +1,9 @@ import { execFile, spawn } from "node:child_process"; import { createHash } from "node:crypto"; -import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { copyFileSync, createWriteStream, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; import { connect, createServer } from "node:net"; import { basename } from "node:path"; +import { createInterface } from "node:readline"; import { setTimeout as sleep } from "node:timers/promises"; import { promisify } from "node:util"; @@ -13,12 +14,50 @@ const POLL_MS = 250; const DRAIN_MS = 500; const RETRY_FAILED_MS = 15000; const GIT = ["-c", "safe.directory=*", "--git-dir", REPO]; +const LOG = "/logs/server.log"; +const LOG_MAX_BYTES = 10 * 1024 * 1024; const out = promisify(execFile); -const log = (...args) => console.log("[deploy]", ...args); + +let sink; +let sinkBytes = 0; +function openSink() { + sink = createWriteStream(LOG, { flags: "a" }).on("error", () => {}); + sinkBytes = existsSync(LOG) ? statSync(LOG).size : 0; +} +openSink(); + +/** Everything the container prints is also kept in LOG, with one rotated copy. */ +function record(line) { + const text = `${new Date().toISOString()} ${line}\n`; + if (sinkBytes + text.length > LOG_MAX_BYTES) { + sink.end(); + try { + renameSync(LOG, `${LOG}.1`); + } catch {} + openSink(); + } + sink.write(text); + sinkBytes += text.length; +} + +function capture(proc) { + for (const [stream, output] of [[proc.stdout, process.stdout], [proc.stderr, process.stderr]]) { + createInterface({ input: stream }).on("line", (line) => { + output.write(`${line}\n`); + record(line); + }); + } + return proc; +} + +const log = (...args) => { + console.log("[deploy]", ...args); + record(["[deploy]", ...args].join(" ")); +}; const run = (cmd, args, opts = {}, ok = [0]) => new Promise((resolve, reject) => - spawn(cmd, args, { stdio: "inherit", ...opts }) + capture(spawn(cmd, args, { stdio: ["ignore", "pipe", "pipe"], ...opts })) .once("error", reject) .once("exit", (code) => (ok.includes(code) ? resolve() : reject(new Error(`${basename(cmd)} ${args[0]} exited with ${code}`)))), ); @@ -145,11 +184,11 @@ async function deploy() { current(); release.port = await freePort(); // The server keeps jwt.key and other state in its working directory. - release.proc = spawn(process.execPath, ["--enable-source-maps", `${release.dir}/dist/bundle/start.js`], { + release.proc = capture(spawn(process.execPath, ["--enable-source-maps", `${release.dir}/dist/bundle/start.js`], { cwd: "/data", - stdio: "inherit", + stdio: ["ignore", "pipe", "pipe"], env: { ...process.env, PORT: String(release.port) }, - }); + })); release.proc.once("exit", (code, signal) => { if (live === release) { live = null; diff --git a/service/spacebar/prepare.py b/service/spacebar/prepare.py new file mode 100755 index 0000000000000000000000000000000000000000..2909945953ac6750465fd8391053c96f96e6dbe3 --- /dev/null +++ b/service/spacebar/prepare.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +"""Create the server log directory in the sandbox workspace, owned by the service.""" +import json +import os +from pathlib import Path +import sys + +uid = json.load(sys.stdin)["uid"] +work = Path("/srv/work") +if not os.path.ismount(work): + raise ValueError("The work SSD is not mounted; refusing to create the log directory on the root disk.") +logs = work / "snowcord-server/logs" +if not logs.parent.is_dir(): + raise ValueError(f"{logs.parent} does not exist; create the sandbox workspace first.") +logs.mkdir(mode=0o755, exist_ok=True) +os.chown(logs, uid, uid) +logs.chmod(0o755) diff --git a/service/spacebar/service.pkl b/service/spacebar/service.pkl index 6e2112e62395e74097b45f36a6ad13644cb3f8b0..a85dc26a82cf593944e2eea50e27ad34f16ef602 100644 --- a/service/spacebar/service.pkl +++ b/service/spacebar/service.pkl @@ -10,6 +10,7 @@ tagline = "Discord-compatible chat" launcher = false stageIsolation = "fresh" healthyDeadline = "10m" +prepare = if (site.preview) null else "prepare.py" requirements { new postgres.Database { name = "spacebar" } @@ -37,6 +38,8 @@ container { ["/data"] {} // Shale keeps each repository under its UUID, which renaming and visibility changes leave alone. ["/source.git"] { src = "\(site.root)/prod/shale/repositories_owned/006AH3H3QKH2KN318QXRNFBH4B"; readOnly = true } + // Production writes the server log into the sandbox workspace; previews keep their own. + ["/logs"] { when (!site.preview) { src = "\(site.root)/work/snowcord-server/logs" } } } tmpfs { "/work:rw,exec,size=3g" } diff --git a/tools/spacebar.md b/tools/spacebar.md index 0dc17ca62a736bdb9a08ce4bbccf46412636ed66..ece84aa63cca7cbdf98b9f2a82bab2f552d418a9 100644 --- a/tools/spacebar.md +++ b/tools/spacebar.md @@ -36,6 +36,10 @@ A tsc error, a crash, or a server that never answers `/api/ping` leaves the previous server running; the reason is in the service log. A newer push cancels a start that is still waiting. Rolling back is pushing the old commit again. +- The supervisor also writes everything the container prints (npm, tsc, the server, deploy + results) to `/logs/server.log`, with one rotated copy at 10 MB. Production mounts that + from `/srv/work/snowcord-server/logs`, so `server-logs.sh` beside `push.sh` in the sandbox + prints it (`-f` follows); previews keep their own. - `assets/schemas.json` and `assets/openapi.json` deploy as committed. After changing request or response schemas, run `npm run generate:schema` and `npm run generate:openapi` and commit the results, as upstream does.