diff --git a/dashboard/src/core.rs b/dashboard/src/core.rs index d5fe219239a7a8beca4f095f67a5e5b5b1cf130b..3cdc741d67baeb20cb601acf2956cf4123efb9e9 100644 --- a/dashboard/src/core.rs +++ b/dashboard/src/core.rs @@ -622,6 +622,7 @@ pub async fn route( json!({"cores":sample.value["cores"],"memory":sample.value["memory"]["total"],"bootedAt":sample.value["bootedAt"],"outages":null}) } ["services"] if method == Method::GET => return Ok(summaries(app).await?.response()), + ["vps"] if method == Method::GET => telemetry::vps(app).await?, ["launcher"] if method == Method::GET => { let found = launcher(app.clone()).await?; let jobs = app.cache.peek("nomad-scan", Duration::from_secs(10)); @@ -723,11 +724,12 @@ pub async fn route( .response()); } ["services", id, "metrics", name] if method == Method::GET => { - let detail = service(app.clone(), id).await?; - if !array(&detail.value["applicationMetrics"]) - .iter() - .any(|v| v == *name) - { + let names = if id.starts_with("vps.") { + telemetry::metric_names(app.clone(), id).await? + } else { + service(app.clone(), id).await?.value["applicationMetrics"].clone() + }; + if !array(&names).iter().any(|v| v == *name) { return Err(Error::new(404, "Metric unavailable for this service.")); } return Ok(telemetry::metrics(app, name, query, Some(id), None) diff --git a/dashboard/src/main.rs b/dashboard/src/main.rs index 000fcd65aad75acc5c49626d195914aed20b1d83..0a2a034c898c523f503826a55dd1b154476bae4d 100644 --- a/dashboard/src/main.rs +++ b/dashboard/src/main.rs @@ -279,7 +279,7 @@ async fn api(State(app): State>, request: Request) -> Result "services" if parts.len() == 1 => Some("metrics"), "storage" if parts.len() == 1 => Some("metrics"), "ai" | "mcp" => Some("ai"), - "services" | "traces" | "storage" | "users" | "shares" | "deploys" | "paper-clover" => { + "services" | "vps" | "traces" | "storage" | "users" | "shares" | "deploys" | "paper-clover" => { Some("admin") } "media" | "seedbox" | "youtube" => Some("media"), diff --git a/dashboard/src/telemetry.rs b/dashboard/src/telemetry.rs index 88b1a8376edc3b020ea9b7843fe1f61714be0edb..07357542ae5f06123132e39c5ba36dec22127f06 100644 --- a/dashboard/src/telemetry.rs +++ b/dashboard/src/telemetry.rs @@ -228,7 +228,7 @@ pub async fn metric_names(app: Arc, id: &str) -> Result { ( "match[]", format!( - "{{{label}={},__name__!~\"snowglobe_.*\"}}", + "{{{label}={},__name__!~\"snowglobe_.*|up|scrape_.*\"}}", serde_json::to_string(id).unwrap() ), ) @@ -248,6 +248,49 @@ pub async fn metric_names(app: Arc, id: &str) -> Result { Ok(json!(values)) } +/// Services the VPS agents report, newest state first; quiet for two minutes counts as down. +pub async fn vps(app: Arc) -> Result { + let base = endpoint(app.clone(), "victoria-metrics").await?; + let instant = |query: &str| { + let url = format!("{base}/api/v1/query?{}", params(&[("query", query.to_owned())])); + let app = app.clone(); + async move { + let response = read(app, url, None).await?; + Ok::<_, Error>( + array(&response.value["data"]["result"]) + .iter() + .map(|row| (string(&row["metric"]["service"]).to_owned(), number(&row["value"][1]))) + .collect::>(), + ) + } + }; + let up = r#"{__name__=~"up|snowglobe_service_up",source="vps"}"#; + let (known, current, cpu, memory, restarts, started) = futures::try_join!( + instant(&format!("max by (service) (max_over_time({up}[7d]))")), + instant(&format!("max by (service) (last_over_time({up}[2m]))")), + instant(r#"max by (service) (last_over_time(snowglobe_service_cpu_cores{source="vps"}[2m]))"#), + instant(r#"max by (service) (last_over_time(snowglobe_service_memory_bytes{source="vps"}[2m]))"#), + instant(r#"max by (service) (last_over_time(snowglobe_service_restarts_total{source="vps"}[7d]))"#), + instant(r#"max by (service) (last_over_time(snowglobe_service_started_seconds{source="vps"}[2m]))"#), + )?; + let mut ids: Vec<_> = known.into_keys().filter(|id| id.starts_with("vps.")).collect(); + ids.sort(); + let mut services = Vec::new(); + for id in ids { + services.push(json!({ + "id": id, + "name": &id[4..], + "health": match current.get(&id) { Some(v) if *v >= 1.0 => "healthy", _ => "down" }, + "cpu": cpu.get(&id), + "memory": memory.get(&id), + "restarts": restarts.get(&id), + "startedAt": started.get(&id).filter(|v| **v > 0.0), + "applicationMetrics": metric_names(app.clone(), &id).await?, + })); + } + Ok(json!(services)) +} + #[derive(Debug, PartialEq)] struct Term { field: Option, @@ -299,9 +342,10 @@ fn terms(query: &str, trace: bool) -> Result> { Ok(found) } fn log_query(id: &str, query: &HashMap) -> Result { + let (source, job) = id.strip_prefix("vps.").map_or(("nomad", id), |job| ("vps", job)); let mut filters = vec![ - "source:=nomad".to_owned(), - format!("job:={}", serde_json::to_string(id)?), + format!("source:={source}"), + format!("job:={}", serde_json::to_string(job)?), ]; if let Some(level) = query.get("level") { filters.push( @@ -1001,6 +1045,12 @@ pub fn start(app: Arc) { mod tests { use super::*; #[test] + fn vps_ids_read_the_vps_source() { + let none = HashMap::new(); + assert_eq!(log_query("vps.site", &none).unwrap(), r#"source:=vps job:="site""#); + assert_eq!(log_query("shale", &none).unwrap(), r#"source:=nomad job:="shale""#); + } + #[test] fn search_fields_comparisons_exclusions_and_unknown_fields() { let parsed = terms( r#"status:5xx -path:/health duration:>1s "GET /x" user:42 -noise"#, diff --git a/dashboard/web/api.contract.ts b/dashboard/web/api.contract.ts index 98834fa0df4b2e4429d6bfcbdbe92b2d01a34f6d..a82e8a50d83b4f637e9f754bd36c919889d2eaac 100644 --- a/dashboard/web/api.contract.ts +++ b/dashboard/web/api.contract.ts @@ -6,7 +6,7 @@ import type { User, Group, Session, Credential } from "./types/users.ts"; import type { Hono } from "hono"; import type { Health } from "./types/model.ts"; import type { Domain, History, NewDomain, VMLibrary } from "./types/vms.ts"; -import type { Me, ServiceSummary, ServiceDetail, ServiceDefinition, HostInfo, Issue, Series, LogLine, TraceSummary, Trace } from "./types/model.ts"; +import type { Me, ServiceSummary, ServiceDetail, VpsService, ServiceDefinition, HostInfo, Issue, Series, LogLine, TraceSummary, Trace } from "./types/model.ts"; import type { MapNode } from "./types/storage.index.ts"; import type { ProgressNode, PaperCloverStats } from "./types/paperClover.ts"; import type { AccountSecret, FolderGrant, LinkInput, ShareState } from "./types/shares.ts"; @@ -102,6 +102,9 @@ export type Api = Hono<{}, { "/services": { $get: Endpoint; }; + "/vps": { + $get: Endpoint; + }; "/services/:id": { $get: Endpoint; }; diff --git a/dashboard/web/api.ts b/dashboard/web/api.ts index c3c99c3f76df045bfb6996ba1f04911aa3b9a32e..a249588a9408147d37842ea6e186041d8438fccc 100644 --- a/dashboard/web/api.ts +++ b/dashboard/web/api.ts @@ -80,6 +80,7 @@ export const queries = { storage: query("storage", () => parseResponse(api.storage.$get())), deploys: query("deploys", () => parseResponse(api.deploys.$get())), vms: query("vms", () => parseResponse(api.vms.$get())), + vps: query("vps", () => parseResponse(api.vps.$get())), metric: query("metric", ({ metric, range, service }: { metric: Metric; range: number; service?: string }) => parseResponse(api.metrics[":metric"].$get({ param: { metric }, query: { range: String(range), ...(service ? { service } : {}) } }))), seedbox: query("seedbox", () => parseResponse(api.seedbox.$get())), diff --git a/dashboard/web/components/Sidebar.tsx b/dashboard/web/components/Sidebar.tsx index 3af6518a3640544a1c895125e313ad2f034d22fb..94f24296a324c1ca67be5cd1149a95ad8d982fc0 100644 --- a/dashboard/web/components/Sidebar.tsx +++ b/dashboard/web/components/Sidebar.tsx @@ -10,6 +10,7 @@ import House from "lucide-solid/icons/house"; import LogOut from "lucide-solid/icons/log-out"; import Monitor from "lucide-solid/icons/monitor"; import Rocket from "lucide-solid/icons/rocket"; +import Server from "lucide-solid/icons/server"; import SquarePlay from "lucide-solid/icons/square-play"; import UserRound from "lucide-solid/icons/user-round"; import Plug from "lucide-solid/icons/plug"; @@ -247,6 +248,37 @@ function Services() { ); } +function VpsServices() { + const [open, toggle] = remembered("sidebar.vps", true); + const [services, { refetch }] = queries.vps.use(); + const timer = setInterval(() => !services.loading && refetch(), 15_000); + onCleanup(() => clearInterval(timer)); + const loaded = lastGood(services); + const down = () => (loaded() ?? []).filter((service) => service.health !== "healthy").map((service) => service.name); + return ( + <> + + + + ); +} + /** Previews the dashboard as someone in fewer groups; the server only honors it for admins. */ export function viewAs(groups: string[] | null) { document.cookie = groups ? `${VIEW_AS}=${encodeURIComponent(groups.join(","))}; path=/; samesite=strict` : `${VIEW_AS}=; path=/; max-age=0`; @@ -310,6 +342,7 @@ export function Sidebar(props: { me: Me }) { {(page) => } + {(page) => page.section === "vms" ? : } diff --git a/dashboard/web/main.tsx b/dashboard/web/main.tsx index e77ab74909898f9298d24a020b58662b6a953bfc..ce72d7deafe826c1f5fc949c53f60b882fb444d3 100644 --- a/dashboard/web/main.tsx +++ b/dashboard/web/main.tsx @@ -20,6 +20,7 @@ import { shares, Shares } from "./pages/Shares.tsx"; import { Deploys } from "./pages/Deploys.tsx"; import { Deploy } from "./pages/Deploy.tsx"; import { VMs } from "./pages/VMs.tsx"; +import { Vps } from "./pages/Vps.tsx"; import { MCP } from "./pages/MCP.tsx"; import { MCPConsent } from "./pages/MCPConsent.tsx"; import { SignIn } from "./pages/SignIn.tsx"; @@ -34,7 +35,7 @@ const preload = (...list: { preload(): void }[]) => () => list.forEach((query) = function Shell(props: RouteSectionProps) { const location = useLocation(); const consent = () => location.pathname.startsWith("/connect/") || location.pathname === "/mcp" && new URLSearchParams(location.search).has("request"); - const section = () => location.pathname.startsWith("/services/") ? "admin" + const section = () => /^\/(services|vps)\//.test(location.pathname) ? "admin" : PAGES.find((page) => page.href !== "/" && location.pathname.startsWith(page.href))?.section; return ( <> @@ -78,6 +79,7 @@ render(() => ( queries.service.preload(params.id!); queries.deploys.preload(); }} /> + diff --git a/dashboard/web/pages/Service.tsx b/dashboard/web/pages/Service.tsx index 1b6b1343514b7115a14270a5e4724784fe5d9505..d84933b6baf07882917a2ee4cc21b94ee33da2c5 100644 --- a/dashboard/web/pages/Service.tsx +++ b/dashboard/web/pages/Service.tsx @@ -44,7 +44,7 @@ const LEVELS = [[undefined, "all"], ["warn", "warn"], ["error", "error"]] as con type Query = { q?: string; level?: "warn" | "error"; at?: string; from?: string; to?: string; errors?: "1"; sort?: TraceSort; trace?: string }; /** Newest at the bottom; follows new lines while scrolled to the end, and loads older ones near the top. */ -function Logs(props: { service: ServiceDetail; tools: HTMLElement }) { +export function Logs(props: { service: Pick; tools: HTMLElement }) { const [params, setParams] = useSearchParams(); const at = () => (params.at ? Number(params.at) : null); const bounded = () => at() !== null || !!params.to; @@ -272,7 +272,7 @@ const traceList = query("traces", ({ id, ...query }: { id: string; q: string; er const oneTrace = query("trace", (id: string) => parseResponse(api.traces[":id"].$get({ param: { id } }))); const PAGE = 50; -function AppMetrics(props: { service: ServiceDetail; range: number }) { +export function AppMetrics(props: { service: Pick; range: number }) { const [name, setName] = createSignal(props.service.applicationMetrics[0]!); const [data, { refetch }] = createResource( () => [props.service.id, name(), props.range] as const, diff --git a/dashboard/web/pages/Vps.tsx b/dashboard/web/pages/Vps.tsx new file mode 100644 index 0000000000000000000000000000000000000000..79c7f646158c70eddabc2b59be06afcf6e36aa67 --- /dev/null +++ b/dashboard/web/pages/Vps.tsx @@ -0,0 +1,90 @@ +import { A, useParams } from "@solidjs/router"; +import RotateCcw from "lucide-solid/icons/rotate-ccw"; +import Server from "lucide-solid/icons/server"; +import { createSignal, Match, onCleanup, Show, Switch } from "solid-js"; +import { queries } from "../api.ts"; +import { AppIcon } from "../components/AppIcon.tsx"; +import { ListPage } from "../components/ListPage.tsx"; +import { lastGood, Loaded } from "../components/Loaded.tsx"; +import { RangePicker } from "../components/Metric.tsx"; +import { StatusLabel } from "../components/Status.tsx"; +import { Strip } from "../components/Strip.tsx"; +import { TabBar } from "../components/TabBar.tsx"; +import { bytes, cores, datetime, duration, plural } from "../format.ts"; +import { AppMetrics, Logs } from "./Service.tsx"; + +/** Remounts per app, so switching never shows the last one's logs. */ +export function Vps() { + const params = useParams<{ name: string }>(); + return ; +} + +function VpsPage() { + const params = useParams<{ name: string; tab?: string }>(); + const [services, { refetch }] = queries.vps.use(); + const timer = setInterval(refetch, 15_000); + onCleanup(() => clearInterval(timer)); + const loaded = lastGood(services); + const [range, setRange] = createSignal(3600); + return ( +
}> + {() => ( + service.name === params.name)} fallback={ +
The VPS hasn't reported {params.name} in the last week.
+ }> + {(data) => { + const tabHref = (tab: string) => `/vps/${data().name}${tab ? "/" + tab : ""}`; + let tools!: HTMLDivElement; + return ( + + +

{data().name}

+
+ + }> + {(t) => ( + + up {duration(Date.now() / 1000 - t())} + + )} + + + + {(restarts) => } + +
+ + } summary={ + +
+ } + now={() => ({ value: cores(data().cpu ?? 0) })} /> + ({ value: bytes(data().memory ?? 0) })} /> +
+
+ }> +
+ + logs + metrics + +
+ + + + + + +
+ ); + }} +
+ )} +
+ ); +} + diff --git a/dashboard/web/types/model.ts b/dashboard/web/types/model.ts index f8540bca4367e954c2ba9a1d492893cbca4e84c4..5b1e6a89d6325356ff985e7983816d919bb48584 100644 --- a/dashboard/web/types/model.ts +++ b/dashboard/web/types/model.ts @@ -51,6 +51,18 @@ export interface ServiceSummary { memoryLimit: number; } +/** An app the VPS agents report, keyed `vps.`; usage, restarts and start time are pm2's. */ +export interface VpsService { + id: string; + name: string; + health: Health; + cpu: number | null; + memory: number | null; + restarts: number | null; + startedAt: number | null; + applicationMetrics: string[]; +} + /** States that need someone to look, unlike stopped (on purpose) or the passing ones like restarting. */ export const trouble = (health: Health) => health === "down" || health === "degraded"; diff --git a/service/ingest/build/Dockerfile b/service/ingest/build/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..2a7f015843d4641a859c9f317f25aad2afa51acd --- /dev/null +++ b/service/ingest/build/Dockerfile @@ -0,0 +1,3 @@ +FROM docker.io/library/python:3.14-slim@sha256:0214bda9af2652a418f2bf34373a4d3ca00a5f4784081e040049433ac9cc79d5 +COPY app.py /app/app.py +CMD ["python3", "-u", "/app/app.py"] diff --git a/service/ingest/build/app.py b/service/ingest/build/app.py new file mode 100644 index 0000000000000000000000000000000000000000..0c3cb614aeec725421ab104036331ade7214a15e --- /dev/null +++ b/service/ingest/build/app.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Files logs and metrics pushed by the VPS agents in Victoria under source=vps.""" +from compression import zstd +import gzip +import hmac +import json +import os +import re +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +TOKEN = os.environ["INGEST_TOKEN"].encode() +LOGS = os.environ["LOGS_URL"] +METRICS = os.environ["METRICS_URL"] +PM2 = re.compile(r".*/\.pm2/logs/(.+)-(out|error)\.log") +ERROR = re.compile(r"\b(ERROR|ERR|FATAL|CRITICAL|PANIC)\b|level=(error|fatal)", re.I) +WARN = re.compile(r"\b(WARN|WARNING|WRN)\b|level=warn", re.I) +LIMIT = 32 << 20 +JOURNAL = ("_time", "_msg", "_HOSTNAME", "_SYSTEMD_UNIT", "SYSLOG_IDENTIFIER", "_PID", "PRIORITY") + +sequence_lock = threading.Lock() +last_sequence = 0 + + +def next_sequence(): + """Microsecond clock, so lines interleave with the Nomad shipper's sequence that live logs follow.""" + global last_sequence + with sequence_lock: + last_sequence = max(last_sequence + 1, time.time_ns() // 1000) + return last_sequence + + +def shape(row): + if match := PM2.fullmatch(row.get("file", "")): + job, stream = match[1], "stdout" if match[2] == "out" else "stderr" + message = row.get("_msg", "") + level = "error" if ERROR.search(message) else "warn" if WARN.search(message) else "info" + else: + job, stream = (row.get("_SYSTEMD_UNIT") or row.get("SYSLOG_IDENTIFIER") or "system").removesuffix(".service"), "journal" + priority = int(row.get("PRIORITY") or 6) + row = {key: row[key] for key in JOURNAL if key in row} + level = "error" if priority <= 3 else "warn" if priority == 4 else "info" + row.update(source="vps", host=row.pop("hostname", None) or row.pop("_HOSTNAME", ""), job=job, stream=stream, + level=level, sequence=next_sequence()) + return row + + +def forward(url, body, headers): + with urlopen(Request(url, data=body, headers=headers), timeout=30) as response: + response.read() + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def reply(self, status, text=""): + body = text.encode() + self.send_response(status) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + self.reply(204 if self.path == "/health" else 404) + + def do_POST(self): + length = int(self.headers.get("Content-Length") or 0) + if length > LIMIT: + self.close_connection = True + return self.reply(413, "batch too large") + body = self.rfile.read(length) + if not hmac.compare_digest(self.headers.get("Authorization", "").removeprefix("Bearer ").encode(), TOKEN): + return self.reply(401, "bad or missing token") + try: + if self.path == "/insert/jsonline": + encoding = self.headers.get("Content-Encoding", "") + text = zstd.decompress(body) if encoding == "zstd" else gzip.decompress(body) if encoding == "gzip" else body + rows = [shape(json.loads(line)) for line in text.splitlines() if line.strip()] + forward(LOGS + "/insert/jsonline", b"".join(json.dumps(row).encode() + b"\n" for row in rows), + {"Content-Type": "application/stream+json", "VL-Stream-Fields": "source,host,job,stream"}) + forward(LOGS + "/internal/force_flush", b"", {}) + elif self.path == "/api/v1/write": + forward(METRICS + "/api/v1/write?extra_label=source=vps", body, + {key: self.headers[key] for key in ("Content-Type", "Content-Encoding") if key in self.headers}) + else: + return self.reply(404) + except HTTPError as error: + return self.reply(502, f"victoria answered {error.code}") + except (OSError, ValueError, zstd.ZstdError) as error: + return self.reply(400 if isinstance(error, (ValueError, zstd.ZstdError)) else 502, str(error)) + self.reply(204) + + def log_request(self, code="-", size="-"): + if not str(int(code)).startswith("2"): + super().log_request(code, size) + + +ThreadingHTTPServer(("0.0.0.0", 8000), Handler).serve_forever() diff --git a/service/ingest/service.pkl b/service/ingest/service.pkl new file mode 100644 index 0000000000000000000000000000000000000000..220c67dcde41236724c792233b201e780f1279f7 --- /dev/null +++ b/service/ingest/service.pkl @@ -0,0 +1,30 @@ +amends "../../config/Service.pkl" + +name = "Ingest" +tagline = "Logs and metrics pushed from the VPS" +launcher = false +dependsOn { "victoria" } + +secrets { ["token"] {} } + +container { + build = "build" + readOnlyRoot = true + cpu = 100 + memory = 128 + + http { + containerPort = 8000 + subdomain = "ingest" + checkPath = "/health" + } + + env { + ["INGEST_TOKEN"] = "${secret.own.token}" + } + + envTemplate = """ + {{ range nomadService 1 (env "NOMAD_ALLOC_ID") "victoria-logs" }}LOGS_URL=http://\(module.nomadHostPort){{ end }} + {{ range nomadService 1 (env "NOMAD_ALLOC_ID") "victoria-metrics" }}METRICS_URL=http://\(module.nomadHostPort){{ end }} + """ +} diff --git a/service/iroh-relay/iroh-relay.service b/service/iroh-relay/iroh-relay.service index d16095a145268f543996fbae48ac580f6e195ba8..2440608b80387bd8f0a7a7d547335636f0c857bb 100644 --- a/service/iroh-relay/iroh-relay.service +++ b/service/iroh-relay/iroh-relay.service @@ -6,6 +6,7 @@ After=network-online.target [Service] DynamicUser=true Environment=HTTP_PORT=3340 +Environment=METRICS_ADDR=127.0.0.1:9091 LoadCredential=fullchain.pem:/etc/letsencrypt/live/iroh-relay/fullchain.pem LoadCredential=privkey.pem:/etc/letsencrypt/live/iroh-relay/privkey.pem ExecStart=/usr/bin/python3 /usr/local/lib/iroh-relay/run.py relay diff --git a/service/iroh-relay/run.py b/service/iroh-relay/run.py index 6140475cd0eb5d3fe6ff2d460ba34f3f5dffea8b..443bbe93511471b4edb3fa2a6cced51b3a7108f7 100644 --- a/service/iroh-relay/run.py +++ b/service/iroh-relay/run.py @@ -36,8 +36,10 @@ if mode == "relay": raise RuntimeError("Relay certificate was not issued within three minutes") quic_port = int(os.environ["QUIC_PORT"]) if "QUIC_PORT" in os.environ else json.loads( Path(__file__).with_name("network.json").read_text())["quic"] + metrics = os.environ.get("METRICS_ADDR") config.write_text( - 'enable_metrics = false\nenable_quic_addr_discovery = true\n' + f'enable_metrics = {json.dumps(bool(metrics))}\nenable_quic_addr_discovery = true\n' + + (f'metrics_bind_addr = {json.dumps(metrics)}\n' if metrics else '') + f'http_bind_addr = "127.0.0.1:{int(os.environ["HTTP_PORT"])}"\n' '[tls]\ncert_mode = "Reloading"\n' f'quic_bind_addr = "0.0.0.0:{quic_port}"\n' diff --git a/service/victoria/service.pkl b/service/victoria/service.pkl index 783ada6df75137b36d30d6527820b9c29acefa60..5a4ae3c9bbde47a418d5502182387648b69f6dfc 100644 --- a/service/victoria/service.pkl +++ b/service/victoria/service.pkl @@ -11,6 +11,7 @@ containers { http { containerPort = 9428 + hostPort = 19428 subdomain = "logs" authRole = "infra-admin" checkPath = "/health" diff --git a/tools/iroh-relay.md b/tools/iroh-relay.md index e3124e0d479b84d1b6dfe0783704a07c04ceb67c..0c4926e116746480a8dff0b10c038aba3530aa58 100644 --- a/tools/iroh-relay.md +++ b/tools/iroh-relay.md @@ -33,6 +33,9 @@ Install `run.py` and `network.json` into `/usr/local/lib/iroh-relay/`, and [`iroh-relay.service`](../service/iroh-relay/iroh-relay.service) into `/etc/systemd/system/`. +The unit sets `METRICS_ADDR`, so the relay serves metrics on `127.0.0.1:9091` for +[vmagent](vps.md). + [`r1.caddy`](../service/iroh-relay/r1.caddy) is installed as `/etc/caddy/Caddyfile.d/iroh-relay.caddy`, explicitly imported by the main Caddyfile. Validate before reloading Caddy. The ACME webroot is diff --git a/tools/vps.md b/tools/vps.md new file mode 100644 index 0000000000000000000000000000000000000000..ae1f0a366e926f0ba11485255c0f6f18e798ab70 --- /dev/null +++ b/tools/vps.md @@ -0,0 +1,39 @@ +# VPS logs and metrics + +The VPS (`ssh root@paperclover.net`, hostname `usw-server-1`) pushes its logs and +metrics to Snow Globe's [ingest](../service/ingest/service.pkl) service at +`https://ingest.paperclover.net`, which files them in Victoria under `source=vps`. +The dashboard lists them under **vps services**; their ids are `vps.`. + +| Agent | Collects | Sends | +| --- | --- | --- | +| [vlagent](vps/vlagent.service) | `~clo/.pm2/logs/*.log`, plus the journal through `systemd-journal-upload` | `/insert/jsonline` | +| [vmagent](vps/vmagent.service) | the iroh relay's metrics on `127.0.0.1:9091`, plus [pm2-metrics.py](vps/pm2-metrics.py) pushes | `/api/v1/write` | + +Ingest names each log line's `job` from its pm2 file (`site-out.log` is `site` on +stdout) or its systemd unit. pm2 writes no timestamps, so a line's time is when +vlagent read it. Both agents buffer up to 1 GB on disk while Snow Globe is +unreachable. + +```logsql +source:=vps job:=site level:=error +``` + +## Install + +Agents are the VictoriaMetrics release binaries, checksum-verified: + +| Binary | Release tarball | SHA256 | +| --- | --- | --- | +| `/usr/local/bin/vlagent` | `vlutils-linux-amd64-v1.52.0.tar.gz` | `caa315764df0d11a77ba61d4ee96b783ddb98455b167da8326940dda62914bc9` | +| `/usr/local/bin/vmagent` | `vmutils-linux-amd64-v1.152.0.tar.gz` | `8eee4a98ff1665c60682475e8a8b292b8d718b63a2f023124384dd2f6a220c79` | + +The ingest service's generated `token` secret lives in `/etc/snowglobe-ingest.token` +(root, mode 600). `vmagent.yml` and `pm2-metrics.py` go in +`/usr/local/lib/snowglobe-agents/`, the units in `/etc/systemd/system/`, and +[journal-upload.conf](vps/journal-upload.conf) in +`/etc/systemd/journal-upload.conf.d/snowglobe.conf` (package +`systemd-journal-remote`). Seed `/var/lib/systemd/journal-upload/state` with +`LAST_CURSOR=` and the current cursor, and run vlagent once against a sink until +its checkpoints reach the ends of the pm2 files; otherwise both send their whole +history, and the pm2 lines all get the same time. diff --git a/tools/vps/journal-upload.conf b/tools/vps/journal-upload.conf new file mode 100644 index 0000000000000000000000000000000000000000..008a25f46bf95fc032e8670f6c35fd88dac60cc5 --- /dev/null +++ b/tools/vps/journal-upload.conf @@ -0,0 +1,2 @@ +[Upload] +URL=http://127.0.0.1:9429/insert/journald diff --git a/tools/vps/pm2-metrics.py b/tools/vps/pm2-metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..b41cd407e87aa3973385ac385241c6e7cd048699 --- /dev/null +++ b/tools/vps/pm2-metrics.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +"""Pushes each pm2 app's state to vmagent as Snow Globe service metrics.""" +import json +import subprocess +import time +from urllib.request import Request, urlopen + +while True: + apps = json.loads(subprocess.run(["pm2", "jlist"], capture_output=True, check=True, text=True).stdout) + lines = [] + for app in apps: + env, monit = app["pm2_env"], app.get("monit") or {} + labels = "{service=%s}" % json.dumps("vps." + app["name"]) + lines += [ + f"snowglobe_service_up{labels} {int(env['status'] == 'online')}", + f"snowglobe_service_cpu_cores{labels} {monit.get('cpu', 0) / 100}", + f"snowglobe_service_memory_bytes{labels} {monit.get('memory', 0)}", + f"snowglobe_service_restarts_total{labels} {env.get('restart_time', 0)}", + f"snowglobe_service_started_seconds{labels} {env.get('pm_uptime', 0) / 1000}", + ] + urlopen(Request("http://127.0.0.1:8429/api/v1/import/prometheus", data="\n".join(lines).encode() + b"\n"), timeout=10).read() + time.sleep(15) diff --git a/tools/vps/pm2-metrics.service b/tools/vps/pm2-metrics.service new file mode 100644 index 0000000000000000000000000000000000000000..fc499fd40656888e151186105f5777906cf9b7d2 --- /dev/null +++ b/tools/vps/pm2-metrics.service @@ -0,0 +1,14 @@ +[Unit] +Description=Report pm2 apps to vmagent +After=vmagent.service + +[Service] +User=clo +Environment=PATH=/home/clo/.local/bin:/usr/bin +ExecStart=/usr/bin/python3 /usr/local/lib/snowglobe-agents/pm2-metrics.py +Restart=always +RestartSec=15 +NoNewPrivileges=true + +[Install] +WantedBy=multi-user.target diff --git a/tools/vps/vlagent.service b/tools/vps/vlagent.service new file mode 100644 index 0000000000000000000000000000000000000000..a3aca78fade0e5684c43fa22867e25c8ff3759e7 --- /dev/null +++ b/tools/vps/vlagent.service @@ -0,0 +1,23 @@ +[Unit] +Description=Ship VPS logs to Snow Globe +Wants=network-online.target +After=network-online.target + +[Service] +LoadCredential=token:/etc/snowglobe-ingest.token +ExecStart=/usr/local/bin/vlagent -httpListenAddr=127.0.0.1:9429 -tmpDataPath=/var/lib/vlagent \ + -fileCollector.glob=/home/clo/.pm2/logs/*.log \ + -remoteWrite.url=https://ingest.paperclover.net/insert/jsonline -remoteWrite.format=jsonline \ + -remoteWrite.bearerTokenFile=%d/token -remoteWrite.maxDiskUsagePerURL=1GB +StateDirectory=vlagent +Restart=always +RestartSec=5 +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=read-only +PrivateTmp=true +PrivateDevices=true +MemoryMax=256M + +[Install] +WantedBy=multi-user.target diff --git a/tools/vps/vmagent.service b/tools/vps/vmagent.service new file mode 100644 index 0000000000000000000000000000000000000000..b50ea67a51bc35860f47db656d0d5392902ea2fd --- /dev/null +++ b/tools/vps/vmagent.service @@ -0,0 +1,25 @@ +[Unit] +Description=Ship VPS metrics to Snow Globe +Wants=network-online.target +After=network-online.target + +[Service] +DynamicUser=true +LoadCredential=token:/etc/snowglobe-ingest.token +ExecStart=/usr/local/bin/vmagent -httpListenAddr=127.0.0.1:8429 \ + -promscrape.config=/usr/local/lib/snowglobe-agents/vmagent.yml \ + -remoteWrite.url=https://ingest.paperclover.net/api/v1/write -remoteWrite.label=host=usw-server-1 \ + -remoteWrite.bearerTokenFile=%d/token \ + -remoteWrite.tmpDataPath=/var/lib/vmagent -remoteWrite.maxDiskUsagePerURL=1GB +StateDirectory=vmagent +Restart=always +RestartSec=5 +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +PrivateDevices=true +MemoryMax=256M + +[Install] +WantedBy=multi-user.target diff --git a/tools/vps/vmagent.yml b/tools/vps/vmagent.yml new file mode 100644 index 0000000000000000000000000000000000000000..d8baa0dc9b785046e86a2d7d6128b3f871c73677 --- /dev/null +++ b/tools/vps/vmagent.yml @@ -0,0 +1,7 @@ +global: + scrape_interval: 30s +scrape_configs: + - job_name: iroh-relay + static_configs: + - targets: ["127.0.0.1:9091"] + labels: { service: "vps.iroh-relay" }