authoravatarclover caruso<git@paperclover.net>2026-10-19 17:28:16-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-21 02:52:29-07:00
logb00a4bc62d1b4f4972f879356490535a4b4ea9dc
tree038d563d955aba396ea8f0e0f9dbb57b99ddde40
parent9980b1fb6b031ee32f458b5f17b8fc33b536c1cf
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Stream service logs with durable searchable checkpoints

Retain application output and lifecycle events across reconnects, delivery retries and allocation cleanup. Gate streaming on verified search visibility; preserve timestamp ties in paged history and use read-only requests in previews. Correct the dashboard dependency hash required by the current locked Nix build. Assisted-by: gpt-6-astra

11 files changed, 772 insertions(+), 100 deletions(-)

dashboard/src/main.rs+7
......@@ -101,6 +101,7 @@ struct App {
101101 internal: Option<(url::Url, reqwest::Client)>,
102102 cache: cache::Cache,
103103 nomad_slots: Semaphore,
104 log_slots: Semaphore,
104105 specs: Mutex<HashMap<String, Value>>,
105106 repo: PathBuf,
106107 data: PathBuf,
......@@ -314,6 +315,11 @@ async fn api(State(app): State<Arc<App>>, request: Request) -> Result<Response>
314315 });
315316 return Ok(Sse::new(stream).into_response());
316317 }
318 if let ["services", id, "logs", "stream"] = parts.as_slice()
319 && method == Method::GET
320 {
321 return telemetry::log_stream(app, request, id, &query).await;
322 }
317323 if let ["vms", name, kind @ ("console" | "serial")] = parts.as_slice() {
318324 if method == Method::GET {
319325 return vms::console(request, name, *kind == "serial").await;
......@@ -546,6 +552,7 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
546552 internal,
547553 cache: cache::Cache::default(),
548554 nomad_slots: Semaphore::new(4),
555 log_slots: Semaphore::new(32),
549556 specs: Mutex::new(HashMap::new()),
550557 repo: env("SNOWGLOBE_REPO", "..").into(),
551558 data: env("SNOWGLOBE_DATA_DIR", "data").into(),
dashboard/src/telemetry.rs+106-7
......@@ -1,5 +1,6 @@
11use crate::*;
2use futures::{StreamExt, stream};
2use futures::{SinkExt, StreamExt, stream};
3use axum::extract::{FromRequestParts, ws::{Message, WebSocketUpgrade}};
34
45pub async fn endpoint(app: Arc<App>, service: &str) -> Result<String> {
56 if let Some((base, _)) = &app.internal {
......@@ -297,8 +298,7 @@ fn terms(query: &str, trace: bool) -> Result<Vec<Term>> {
297298 }
298299 Ok(found)
299300}
300pub async fn logs(app: Arc<App>, id: &str, query: &HashMap<String, String>) -> Result<Value> {
301 let limit = query_number(query, "limit", 200.0, 1.0, 5000.0)?;
301fn log_query(id: &str, query: &HashMap<String, String>) -> Result<String> {
302302 let mut filters = vec![
303303 "source:=nomad".to_owned(),
304304 format!("job:={}", serde_json::to_string(id)?),
......@@ -332,15 +332,36 @@ pub async fn logs(app: Arc<App>, id: &str, query: &HashMap<String, String>) -> R
332332 ))?
333333 ));
334334 }
335 Ok(filters.join(" "))
336}
337
338fn log_line(row: Value) -> Value {
339 json!({"id":row["id"].as_str().map(str::to_owned).unwrap_or_else(|| format!("{}:{}:{}", string(&row["_stream_id"]), string(&row["_time"]), string(&row["_msg"]))),
340 "sequence":number(&row["sequence"]),
341 "t":core::seconds(&row["_time"]),"container":row["task"].as_str().unwrap_or("app"),
342 "stream":row["stream"].as_str().filter(|s| ["stderr", "event"].contains(s)).unwrap_or("stdout"),
343 "level":row["level"],"text":row["_msg"].as_str().unwrap_or_default(),
344 "event":row["event"],"allocation":row["allocation"]})
345}
346
347pub async fn logs(app: Arc<App>, id: &str, query: &HashMap<String, String>) -> Result<Value> {
348 let limit = query_number(query, "limit", 200.0, 1.0, 5000.0)?;
349 if query.contains_key("after") && query.contains_key("before")
350 && query_number(query, "after", 0.0, f64::NEG_INFINITY, f64::INFINITY)?
351 > query_number(query, "before", 0.0, f64::NEG_INFINITY, f64::INFINITY)?
352 {
353 return Err(Error::new(400, "Choose a “to” time at or after “from”."));
354 }
355 let offset = query_number(query, "offset", 0.0, 0.0, 1_000_000_000.0)?;
335356 let mut form = vec![
336 ("query".into(), filters.join(" ")),
357 ("query".into(), format!("{} | sort by (_time desc, id desc, _stream_id desc, _msg desc) offset {offset:.0} limit {limit:.0}", log_query(id, query)?)),
337358 ("limit".into(), limit.to_string()),
338359 ];
339 for (key, field, add) in [("after", "start", 0.000001), ("before", "end", 0.0)] {
360 for (key, field) in [("after", "start"), ("before", "end")] {
340361 if query.contains_key(key) {
341362 form.push((
342363 field.into(),
343 (query_number(query, key, 0.0, f64::NEG_INFINITY, f64::INFINITY)? + add)
364 query_number(query, key, 0.0, f64::NEG_INFINITY, f64::INFINITY)?
344365 .to_string(),
345366 ));
346367 }
......@@ -352,12 +373,90 @@ pub async fn logs(app: Arc<App>, id: &str, query: &HashMap<String, String>) -> R
352373 let response = read(app, url, Some(form)).await?;
353374 let mut rows = string(&response.value).lines().filter(|l| !l.trim().is_empty()).map(|line| {
354375 let row: Value = serde_json::from_str(line)?;
355 Ok::<_,Error>(json!({"t":core::seconds(&row["_time"]),"container":row["task"].as_str().unwrap_or("app"),"stream":if row["stream"] == "stderr" {"stderr"} else {"stdout"},"level":if row["level"] == "error" || row["level"] == "warn" {row["level"].clone()} else {Value::Null},"text":row["_msg"].as_str().unwrap_or_default()}))
376 Ok::<_,Error>(log_line(row))
356377 }).collect::<Result<Vec<_>>>()?;
357378 rows.sort_by(|a, b| number(&b["t"]).total_cmp(&number(&a["t"])));
358379 Ok(json!(rows))
359380}
360381
382pub async fn log_stream(app: Arc<App>, request: Request, id: &str, query: &HashMap<String, String>) -> Result<Response> {
383 vms::same_origin(&request)?;
384 let filter = log_query(id, query)?;
385 let mut cursor = query_number(query, "cursor", 0.0, 0.0, 9_007_199_254_740_991.0)?;
386 let from = query.get("from").map(|_| query_number(query, "from", 0.0, 0.0, f64::INFINITY)).transpose()?;
387 let base = endpoint(app.clone(), "victoria-logs").await?;
388 let url = format!("{base}/select/logsql/query");
389 let headers = request.headers().clone();
390 let (mut parts, _) = request.into_parts();
391 let upgrade = WebSocketUpgrade::from_request_parts(&mut parts, &()).await
392 .map_err(|_| Error::new(400, "Open the service log to connect."))?
393 .max_message_size(1024).max_frame_size(1024);
394 Ok(upgrade.on_upgrade(move |mut socket| async move {
395 let run = async {
396 let _slot = app.log_slots.try_acquire().map_err(|_| Error::new(429, "Too many live logs are open. Close one and retry."))?;
397 let mut first = true;
398 let mut poll = tokio::time::interval(Duration::from_secs(1));
399 poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
400 let mut heartbeat = tokio::time::interval(Duration::from_secs(15));
401 loop {
402 tokio::select! {
403 message = socket.recv() => match message {
404 Some(Ok(Message::Ping(bytes))) => socket.send(Message::Pong(bytes)).await?,
405 Some(Ok(Message::Pong(_))) => {},
406 _ => break,
407 },
408 _ = heartbeat.tick() => {
409 if app.auth.ready() {
410 let account = app.auth.session(&headers, "dashboard")?;
411 if !array(&account["groups"]).iter().any(|g| g["name"] == "infra-admin") {
412 return Err(Error::new(401, "Sign in as an administrator to continue reading logs."));
413 }
414 }
415 tokio::time::timeout(Duration::from_secs(5), socket.send(Message::Ping(Bytes::new()))).await
416 .map_err(|_| Error::new(504, "The log connection stopped responding."))??;
417 },
418 _ = poll.tick() => {
419 let checkpoint = read(app.clone(), url.clone(), Some(vec![("query".into(), "source:=log-shipper | stats max(sequence) as sequence".into())])).await?;
420 let checkpoint: Value = serde_json::from_str(string(&checkpoint.value).lines().find(|line| !line.trim().is_empty()).unwrap_or("{}"))?;
421 let settled = number(&checkpoint["sequence"]);
422 if first { cursor = cursor.min(settled); first = false; }
423 let mut form = vec![("query", format!("{filter} sequence:>{cursor:.0} sequence:<={settled:.0} | sort by (sequence) limit 5000"))];
424 if let Some(from) = from { form.push(("start", from.to_string())); }
425 let request = if env("SNOWGLOBE_PREVIEW", "") == "1" {
426 app.request(Method::GET, &url)?.query(&form)
427 } else {
428 app.request(Method::POST, &url)?.form(&form)
429 };
430 let response = request.timeout(Duration::from_secs(8)).send().await?;
431 if !response.status().is_success() { return Err(Error::new(502, "The log store isn't answering. Retrying…")); }
432 let mut chunks = response.bytes_stream();
433 let mut body = Vec::new();
434 while let Some(chunk) = chunks.next().await {
435 body.extend_from_slice(&chunk?);
436 if body.len() > 8 * 1024 * 1024 {
437 let end = body[..8 * 1024 * 1024].iter().rposition(|b| *b == b'\n')
438 .ok_or_else(|| Error::new(502, "A log entry is too large. Narrow the search and retry."))?;
439 body.truncate(end + 1);
440 break;
441 }
442 }
443 let lines = std::str::from_utf8(&body)?.lines().filter(|l| !l.trim().is_empty())
444 .map(|raw| serde_json::from_str(raw).map(log_line)).collect::<std::result::Result<Vec<_>, _>>()?;
445 for line in &lines { cursor = cursor.max(number(&line["sequence"])); }
446 tokio::time::timeout(Duration::from_secs(5), socket.send(Message::Text(json!({"lines":lines,"cursor":cursor}).to_string().into()))).await
447 .map_err(|_| Error::new(504, "The log connection stopped responding."))??;
448 }
449 }
450 }
451 Ok::<_, Error>(())
452 }.await;
453 if let Err(error) = run {
454 let _ = tokio::time::timeout(Duration::from_secs(2), socket.send(Message::Text(json!({"error":error.message}).to_string().into()))).await;
455 }
456 let _ = tokio::time::timeout(Duration::from_secs(1), socket.close()).await;
457 }).into_response())
458}
459
361460fn span(row: &Value) -> Value {
362461 let mut attributes: serde_json::Map<String, Value> = row
363462 .as_object()
dashboard/src/vms.rs+2-2
......@@ -75,7 +75,7 @@ pub async fn console(request: Request, name: &str, serial: bool) -> Result<Respo
7575 .into_response())
7676}
7777
78fn same_origin(request: &Request) -> Result<()> {
78pub(crate) fn same_origin(request: &Request) -> Result<()> {
7979 let supplied = request
8080 .headers()
8181 .get("origin")
......@@ -94,7 +94,7 @@ fn same_origin(request: &Request) -> Result<()> {
9494 };
9595 authority != host || !matches!(origin.scheme(), "http" | "https")
9696 }) {
97 return Err(Error::new(403, "Open the VM page on this site to connect."));
97 return Err(Error::new(403, "Open this page on the same site to connect."));
9898 }
9999 Ok(())
100100}
dashboard/web/components/LogView.css+2
......@@ -18,6 +18,8 @@
1818.log-view .log { grid-template-columns: 62px 1fr; padding: 0 var(--gutter); }
1919.log-view .log.warn:hover { background: color-mix(in srgb, var(--warning) 16%, transparent); }
2020.log-view .log.error:hover { background: color-mix(in srgb, var(--critical) 20%, transparent); }
21.log-view .task-event { background: color-mix(in srgb, var(--text) 4%, transparent); }
22.log-view .event-label { margin-right: 8px; font: 600 11px var(--sans); }
2123.log-view .ansi {
2224 color: color-mix(in srgb, var(--fg, currentColor) var(--ansi-mix), var(--text));
2325 background: color-mix(in srgb, var(--bg, transparent) 45%, transparent);
dashboard/web/components/LogView.tsx+2-1
......@@ -132,9 +132,10 @@ export function LogView(props: {
132132 <div class="log-edge day">{date(line.t)}</div>
133133 </Show>
134134 {props.before?.(line, index)}
135 <div class={`log ${line.level ?? ""}`}>
135 <div class={`log ${line.level ?? ""}`} classList={{ "task-event": line.stream === "event" }}>
136136 <time data-tip={datetime(line.t)}>{clock(line.t)}</time>
137137 <span>
138 <Show when={line.event}><strong class="event-label" data-tip={line.allocation ? `Allocation ${line.allocation}` : undefined}>{line.event}</strong>{" "}</Show>
138139 <Show when={mixed()}>
139140 <Show when={props.onContainer} fallback={<span class="container">{line.container}</span>}>
140141 {(filter) => (
dashboard/web/pages/Service.tsx+117-53
......@@ -8,7 +8,7 @@ import RotateCcw from "lucide-solid/icons/rotate-ccw";
88import Search from "lucide-solid/icons/search";
99import X from "lucide-solid/icons/x";
1010import {
11 createEffect, createMemo, createResource, createSignal, createUniqueId, For, type JSX, Match, onCleanup, Show, Switch,
11 createEffect, createMemo, createResource, createSignal, createUniqueId, For, type JSX, Match, onCleanup, Show, Switch, untrack,
1212} from "solid-js";
1313import { Portal } from "solid-js/web";
1414import {
......@@ -43,29 +43,37 @@ const LEVELS = [[undefined, "all"], ["warn", "warn"], ["error", "error"]] as con
4343/** URL state of the logs and traces tabs; `at` opens the logs at a moment instead of following. */
4444type Query = { q?: string; level?: "warn" | "error"; at?: string; from?: string; to?: string; errors?: "1"; sort?: TraceSort; trace?: string };
4545
46/** Each service's latest unfiltered lines, so returning to a service draws its logs at once. */
47const recent = new Map<string, LogLine[]>();
48
4946/** Newest at the bottom; follows new lines while scrolled to the end, and loads older ones near the top. */
5047function Logs(props: { service: ServiceDetail; tools: HTMLElement }) {
5148 const [params, setParams] = useSearchParams<Query>();
5249 const at = () => (params.at ? Number(params.at) : null);
5350 const bounded = () => at() !== null || !!params.to;
54 const unfiltered = () => !params.q && !params.level && !params.from && !params.to && at() === null;
5551 const dateValue = (value?: string) => {
5652 if (!value) return "";
5753 const date = new Date(Number(value) * 1000);
5854 return new Date(date.getTime() - date.getTimezoneOffset() * 60_000).toISOString().slice(0, 16);
5955 };
60 const [older, setOlder] = createSignal<LogLine[]>([]);
61 const [newer, setNewer] = createSignal<LogLine[]>([]);
56 const [all, setLines] = createSignal<LogLine[]>([]);
57 let pending: LogLine[] = [];
58 let generation = 0;
59 let older: { before: string; offset: number } | undefined;
6260 const [exhausted, setExhausted] = createSignal(false);
6361 const [loadingOlder, setLoadingOlder] = createSignal(false);
6462 const [follow, setFollow] = createSignal(true);
6563 const [unseen, setUnseen] = createSignal(0);
66 /** Why the last poll failed, until one succeeds. */
6764 const [stalled, setStalled] = createSignal<string>();
68 const fetchLines = (limit: number, extra: { before?: string; after?: string }) =>
65 const [connected, setConnected] = createSignal(false);
66 const merge = (before: LogLine[], after: LogLine[]) => [...new Map([...before, ...after]
67 .map((line) => [line.id ?? JSON.stringify(line), line])).values()].sort((a, b) => a.t - b.t);
68 const resume = () => {
69 older = undefined;
70 setLines((lines) => merge(lines, pending).slice(-10_000));
71 pending = [];
72 setUnseen(0);
73 setExhausted(false);
74 setFollow(true);
75 };
76 const fetchLines = (limit: number, extra: { before?: string; after?: string; offset?: string }) =>
6977 parseResponse(api.services[":id"].logs.$get({
7078 param: { id: props.service.id },
7179 query: { q: params.q ?? "", ...(params.level ? { level: params.level } : {}), limit: String(limit),
......@@ -74,64 +82,120 @@ function Logs(props: { service: ServiceDetail; tools: HTMLElement }) {
7482 const [first, { refetch }] = createResource<LogLine[], string>(
7583 // A string key, since a fresh array from an unrelated rerun would count as a change and refetch.
7684 () => JSON.stringify([props.service.id, params.q, params.level, params.at, params.from, params.to]),
77 (_, { refetching }) => {
78 setOlder([]);
79 setNewer([]);
85 () => {
86 const current = ++generation;
87 older = undefined;
88 setLines([]);
89 pending = [];
90 setLoadingOlder(false);
8091 setExhausted(false);
8192 setFollow(!bounded());
8293 setUnseen(0);
8394 setStalled();
84 const cached = !refetching && unfiltered() && recent.get(props.service.id);
85 if (cached) {
86 queueMicrotask(() => poll());
87 return cached;
88 }
89 return fetchLines(1000, at() === null ? {} : { before: String(at()! + 30) });
95 return fetchLines(1000, at() === null ? {} : { before: String(at()! + 30) }).then((lines) => {
96 if (current === generation) setLines(lines);
97 return lines;
98 });
9099 },
91100 );
92 const loaded = lastGood(first);
93 const all = createMemo(() => [...older(), ...(loaded() ?? []), ...newer()]);
94 createEffect(() => first.state === "ready" && unfiltered() && recent.set(props.service.id, all().slice(-1000)));
95101
96102 const loadOlder = async () => {
97103 const oldest = all()[0];
98104 if (!oldest || loadingOlder() || exhausted()) return;
99 if (params.from && oldest.t <= Number(params.from)) return setExhausted(true);
105
100106 setLoadingOlder(true);
107 const current = generation;
101108 try {
102 const lines = await fetchLines(5000, { before: String(oldest.t) });
103 if (!lines.length) setExhausted(true);
104 setOlder([...lines, ...older()]);
109 const page = older ??= { before: String(oldest.t + 0.000001), offset: 0 };
110 const fetched = await fetchLines(5000, { before: page.before, offset: String(page.offset) });
111 if (current !== generation || older !== page) return;
112 const known = new Set(all().map((line) => line.id ?? JSON.stringify(line)));
113 const lines = fetched.filter((line) => !known.has(line.id ?? JSON.stringify(line)));
114 page.offset += fetched.length;
115 if (fetched.length < 5000) setExhausted(true);
116 setLines((previous) => merge(lines, previous));
105117 } catch (failure) {
106 toast(`Couldn't load older lines. ${reason(failure)}`);
118 if (current === generation) toast(`Couldn't load older lines. ${reason(failure)}`);
107119 } finally {
108 setLoadingOlder(false);
120 if (current === generation) setLoadingOlder(false);
109121 }
110122 };
111123
112 const poll = async () => {
113 const last = all().at(-1);
114 if (bounded() || !last || first.loading) return;
115 let lines: LogLine[];
116 try {
117 lines = await fetchLines(1000, { after: String(last.t) });
118 setStalled();
119 } catch (failure) {
120 setStalled(reason(failure));
121 return;
122 }
123 // A search changed while this was under way, so the lines belong to the old one.
124 if (!lines.length || first.loading || last !== all().at(-1)) return;
125 setNewer([...newer(), ...lines]);
126 if (!follow()) setUnseen(unseen() + lines.length);
127 };
128 const timer = setInterval(poll, 2000);
129 onCleanup(() => clearInterval(timer));
124 createEffect(() => {
125 const id = props.service.id;
126 const q = params.q ?? "";
127 const level = params.level;
128 const from = params.from;
129 setConnected(false);
130 if (bounded() || first.state !== "ready") return;
131 let disposed = false;
132 let failures = 0;
133 let cursor = Math.max(0, ...untrack(all).map((line) => line.sequence ?? 0));
134 let socket: WebSocket;
135 let retry: ReturnType<typeof setTimeout> | undefined;
136 let stale: ReturnType<typeof setTimeout> | undefined;
137 const connect = () => {
138 const url = new URL(`/api/services/${encodeURIComponent(id)}/logs/stream`, location.href);
139 url.protocol = location.protocol === "https:" ? "wss:" : "ws:";
140 url.searchParams.set("q", q);
141 if (level) url.searchParams.set("level", level);
142 url.searchParams.set("cursor", String(cursor));
143 if (from) url.searchParams.set("from", from);
144 socket = new WebSocket(url);
145 socket.onmessage = (event) => {
146 if (disposed) return;
147 const message: { lines?: LogLine[]; cursor?: number; error?: string } = JSON.parse(event.data);
148 if (message.error) {
149 setConnected(false);
150 setStalled(message.error);
151 return;
152 }
153 failures = 0;
154 setConnected(true);
155 setStalled();
156 clearTimeout(stale);
157 stale = setTimeout(() => socket.close(), 35_000);
158 if (message.lines?.length) {
159 const known = new Set([...all(), ...pending].map((line) => line.id ?? JSON.stringify(line)));
160 const lines = message.lines.filter((line) => {
161 const id = line.id ?? JSON.stringify(line);
162 if (known.has(id)) return false;
163 known.add(id);
164 return true;
165 });
166 if (follow()) {
167 older = undefined;
168 setLines((previous) => merge(previous, lines).slice(-10_000));
169 }
170 else {
171 pending = merge(pending, lines).slice(-10_000);
172 setUnseen((previous) => previous + lines.length);
173 }
174 }
175 cursor = message.cursor ?? cursor;
176 };
177 socket.onclose = () => {
178 if (disposed) return;
179 clearTimeout(stale);
180 setConnected(false);
181 setStalled((previous) => previous ?? "Live logs disconnected. Retrying…");
182 retry = setTimeout(connect, Math.min(30, 2 ** failures++) * 1000);
183 };
184 };
185 connect();
186 onCleanup(() => {
187 disposed = true;
188 clearTimeout(retry);
189 clearTimeout(stale);
190 socket.close();
191 });
192 });
130193 const marked = createMemo(() => (at() === null ? undefined : all().find((line) => line.t > at()!)));
131194 const hints = () => [
132195 ...props.service.containers.filter((container) => !container.hook && props.service.containers.length > 1).slice(0, 1)
133196 .map((container) => [`container:${container.name}`, "one container's lines"] as const),
134197 ["stream:stderr", "error output only"],
198 ["stream:event", "starts, restarts, and OOM kills"],
135199 ["-health", "hide lines with a word"],
136200 ['"connection refused"', "an exact phrase"],
137201 ] as const;
......@@ -155,7 +219,7 @@ function Logs(props: { service: ServiceDetail; tools: HTMLElement }) {
155219 <label>to <input type="datetime-local" value={dateValue(params.to)}
156220 onChange={(event) => setParams({ to: event.currentTarget.value ? String(Date.parse(event.currentTarget.value) / 1000) : undefined }, { replace: true })} /></label>
157221 <Show when={props.service.logs}>{(link) => <OpenApp app={link().app} href={link().url}>{link().app.name}</OpenApp>}</Show>
158 <Liveness every="lines every 2 s" paused={bounded() ? "Showing the selected time range" : undefined}
222 <Liveness every="lines as they arrive" connecting={!connected()} paused={bounded() ? "Showing the selected time range" : undefined}
159223 failure={stalled() ?? (first.state === "errored" ? reason(first.error) : undefined)} />
160224 </Filters>
161225 </Portal>
......@@ -178,10 +242,10 @@ function Logs(props: { service: ServiceDetail; tools: HTMLElement }) {
178242 onStart={loadOlder}
179243 onContainer={(name) => setParams({ q: [params.q, `container:${name}`].filter(Boolean).join(" ") }, { replace: true })}
180244 onFollow={(end) => {
181 setFollow(end && at() === null);
182 if (end) setUnseen(0);
245 if (end && !bounded()) resume();
246 else setFollow(false);
183247 }}
184 onLatest={() => (at() === null ? setFollow(true) : setParams({ at: undefined }))}
248 onLatest={() => (at() === null ? resume() : setParams({ at: undefined }))}
185249 before={(line, index) => (
186250 <>
187251 <Show when={index() === 0}>
......@@ -387,12 +451,12 @@ function Traces(props: { service: ServiceDetail; tools: HTMLElement }) {
387451}
388452
389453/** Whether a tab still takes in new data; `paused` and `failure` say why it doesn't. */
390function Liveness(props: { every: string; paused?: string; failure?: string }) {
391 const health = () => (props.failure ? "degraded" : props.paused ? "stopped" : "healthy");
454function Liveness(props: { every: string; paused?: string; failure?: string; connecting?: boolean }) {
455 const health = () => (props.failure ? "degraded" : props.paused || props.connecting ? "stopped" : "healthy");
392456 return (
393457 <span class="liveness" classList={{ live: health() === "healthy" }} role="status" tabIndex={0}
394 data-tip={props.failure ?? props.paused ?? `Checks for new ${props.every}`}>
395 <StatusLabel health={health()}>{props.failure ? "reconnecting" : props.paused ? "paused" : "live"}</StatusLabel>
458 data-tip={props.failure ?? props.paused ?? (props.connecting ? "Connecting to live logs…" : `New ${props.every}`)}>
459 <StatusLabel health={health()}>{props.failure ? props.paused ? "unavailable" : "reconnecting" : props.paused ? "paused" : props.connecting ? "connecting" : "live"}</StatusLabel>
396460 </span>
397461 );
398462}
dashboard/web/types/model.ts+5-1
......@@ -206,11 +206,15 @@ export interface TraceSummary {
206206}
207207
208208export interface LogLine {
209 id?: string;
210 sequence?: number;
209211 t: number;
210212 container: string;
211 stream: "stdout" | "stderr";
213 stream: "stdout" | "stderr" | "event";
212214 level: "debug" | "info" | "warn" | "error" | null;
213215 text: string;
216 event?: string | null;
217 allocation?: string | null;
214218}
215219
216220export interface HostInfo {
nixos/dashboard.nix+1-1
......@@ -24,7 +24,7 @@ let
2424 inherit (finalAttrs) pname version src sourceRoot;
2525 pnpm = pnpm_10;
2626 fetcherVersion = 3;
27 hash = "sha256-zux7++pfJ9YJAMPjnvW94Z2m884Ypnc6BK9iHJxRFnA=";
27 hash = "sha256-Jhk4mAOCqoljelFJfC7o+3t66x9GPFYb2e928jeaPSk=";
2828 };
2929 nativeBuildInputs = [ nodejs_24 pnpm_10 pnpmConfigHook ];
3030 buildPhase = "pnpm run build";
tools/dashboard-logs-test.py created+270
......@@ -0,0 +1,270 @@
1#!/usr/bin/env python3
2import argparse
3import base64
4from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
5import json
6import os
7from pathlib import Path
8import socket
9import struct
10import re
11import sqlite3
12import hashlib
13import subprocess
14import tempfile
15import threading
16import time
17from urllib.parse import parse_qs, urlsplit
18from urllib.request import Request, urlopen
19from urllib.error import HTTPError
20
21
22def main():
23 parser = argparse.ArgumentParser()
24 parser.add_argument('--preview', action='store_true')
25 parser.add_argument('--binary', type=Path, default=Path('dashboard/target/debug/home-dashboard'))
26 args = parser.parse_args()
27 forms = []
28 failed = threading.Event()
29 row = {'_time': '2026-10-14T02:50:00Z', '_msg': 'Out of memory (OOM): exit 137',
30 'id': 'allocation:app:1:Terminated', 'task': 'app', 'stream': 'event', 'event': 'OOM',
31 'level': 'error', 'allocation': 'allocation', 'job': 'shale', 'source': 'nomad', 'sequence': 1}
32 rows = [row]
33 checkpoint = 0
34
35 class Fixture(BaseHTTPRequestHandler):
36 protocol_version = 'HTTP/1.1'
37
38 def handle(self):
39 try:
40 super().handle()
41 except (ConnectionResetError, BrokenPipeError):
42 pass
43
44 def do_GET(self):
45 if urlsplit(self.path).path == '/select/logsql/query':
46 self.query(parse_qs(urlsplit(self.path).query))
47 else:
48 self.reply([{'Address': '127.0.0.1', 'Port': self.server.server_port}])
49
50 def reply(self, value):
51 data = json.dumps(value).encode()
52 self.send_response(200)
53 self.send_header('Content-Length', str(len(data)))
54 self.end_headers()
55 self.wfile.write(data)
56
57 def do_POST(self):
58 form = parse_qs(self.rfile.read(int(self.headers['Content-Length'])).decode())
59 self.query(form)
60
61 def query(self, form):
62 forms.append((self.path, form))
63 if failed.is_set():
64 self.send_error(503)
65 return
66 if 'source:=log-shipper' in form['query'][0]:
67 if checkpoint == 0:
68 self.send_response(200)
69 self.send_header('Content-Length', '0')
70 self.end_headers()
71 else:
72 self.reply({'sequence': checkpoint if checkpoint is not None else max(r['sequence'] for r in rows)})
73 return
74 selected = rows[:]
75 match = re.search(r'sequence:>([0-9]+)', form['query'][0])
76 if match:
77 selected = [r for r in selected if r['sequence'] > int(match[1])]
78 match = re.search(r'sequence:<=(\d+)', form['query'][0])
79 if match:
80 selected = [r for r in selected if r['sequence'] <= int(match[1])]
81 if '| sort by (sequence)' in form['query'][0]:
82 selected.sort(key=lambda r: r['sequence'])
83 elif '| sort by (_time desc' in form['query'][0]:
84 selected.sort(key=lambda r: (r['_time'], r['id']), reverse=True)
85 offset = int(re.search(r'offset (\d+)', form['query'][0])[1])
86 limit = int(re.search(r'limit (\d+)', form['query'][0])[1])
87 selected = selected[offset:offset + limit]
88 self.send_response(200)
89 data = b''.join((json.dumps(r) + '\n').encode() for r in selected)
90 self.send_header('Content-Length', str(len(data)))
91 self.end_headers()
92 self.wfile.write(data[:17])
93 self.wfile.flush()
94 try:
95 self.wfile.write(data[17:])
96 self.wfile.flush()
97 except (BrokenPipeError, ConnectionResetError):
98 pass
99
100 def log_message(self, *args):
101 pass
102
103 with tempfile.TemporaryDirectory(prefix='dashboard-logs-') as temporary, ThreadingHTTPServer(('127.0.0.1', 0), Fixture) as fixture:
104 fixture.daemon_threads = True
105 threading.Thread(target=fixture.serve_forever, daemon=True).start()
106 root = Path(temporary).resolve()
107 (root / 'data').mkdir()
108 (root / 'token').write_text('disposable')
109 with socket.socket() as reservation:
110 reservation.bind(('127.0.0.1', 0))
111 port = reservation.getsockname()[1]
112 origin = f'http://127.0.0.1:{port}'
113 environment = {**os.environ, 'PORT': str(port), 'SNOWGLOBE_DATA_DIR': str(root / 'data'),
114 'SNOWGLOBE_PUBLIC_ORIGIN': 'https://globe.studio.test', 'SNOWGLOBE_NOMAD_TOKEN_FILE': str(root / 'token'),
115 'NOMAD_ADDR': f'http://127.0.0.1:{fixture.server_port}'}
116 environment['SNOWGLOBE_PREVIEW'] = '1' if args.preview else '0'
117 for name in ['SNOWGLOBE_PROXY_TOKEN_FILE', 'SNOWGLOBE_INTERNAL_URL', 'SNOWGLOBE_AUTH_REQUIRED', 'SNOWGLOBE_INDEX_POOL', 'SNOWGLOBE_YT_STATE']:
118 environment.pop(name, None)
119 with (root / 'server.log').open('w') as log:
120 server = subprocess.Popen([str(args.binary.resolve())], env=environment, stdout=log, stderr=log)
121 try:
122 deadline = time.monotonic() + 10
123 while True:
124 try:
125 with socket.create_connection(('127.0.0.1', port), timeout=.1): break
126 except OSError:
127 assert server.poll() is None, (root / 'server.log').read_text()
128 assert time.monotonic() < deadline
129 time.sleep(.05)
130 path = '/api/services/shale/logs'
131 with urlopen(Request(origin + path + '?level=error&q=stream:event', headers={'User-Name': 'fixture', 'User-Groups': 'infra-admin'})) as response:
132 lines = json.load(response)
133 assert lines[0]['stream'] == 'event' and lines[0]['event'] == 'OOM'
134 assert lines[0]['level'] == 'error' and lines[0]['id'] == row['id']
135 assert 'level:=error' in forms[-1][1]['query'][0]
136 assert 'stream:~"(?i)^event"' in forms[-1][1]['query'][0]
137 try:
138 urlopen(Request(origin + path + '?after=2&before=1', headers={'User-Name': 'fixture', 'User-Groups': 'infra-admin'}))
139 raise AssertionError('reversed bounds accepted')
140 except HTTPError as error:
141 assert error.code == 400 and b'at or after' in error.read()
142
143 def connect(expected, supplied_origin=origin, groups='infra-admin', actor='fixture', cursor=0, cookie=None):
144 sock = socket.create_connection(('127.0.0.1', port), timeout=5)
145 sock.settimeout(20)
146 stream = sock.makefile('rb')
147 headers = {'Host': urlsplit(origin).netloc, 'Upgrade': 'websocket', 'Connection': 'Upgrade',
148 'Sec-WebSocket-Version': '13', 'Sec-WebSocket-Key': base64.b64encode(os.urandom(16)).decode()}
149 if actor is not None: headers['User-Name'] = actor
150 if cookie is not None: headers['Cookie'] = '__Host-snow-session=' + cookie
151 if groups is not None: headers['User-Groups'] = groups
152 if supplied_origin is not None: headers['Origin'] = supplied_origin
153 sock.sendall(('GET ' + path + f'/stream?q=stream:event&cursor={cursor}&from=1 HTTP/1.1\r\n' + ''.join(f'{k}: {v}\r\n' for k, v in headers.items()) + '\r\n').encode())
154 status = int(stream.readline().split()[1])
155 assert status == expected, status
156 while stream.readline().strip(): pass
157 if expected != 101:
158 stream.close()
159 sock.close()
160 return sock, stream
161
162 connect(401, actor=None)
163 connect(403, groups='metrics')
164 connect(403, supplied_origin=None)
165 connect(403, supplied_origin='http://other.invalid')
166 sock, stream = connect(101)
167 try:
168 def receive():
169 head = stream.read(2)
170 assert len(head) == 2, 'stream closed unexpectedly'
171 size = head[1] & 127
172 if size == 126: size = struct.unpack('!H', stream.read(2))[0]
173 if size == 127: size = struct.unpack('!Q', stream.read(8))[0]
174 if head[0] & 15 == 9:
175 payload = stream.read(size)
176 mask = os.urandom(4)
177 sock.sendall(bytes([138, 128 | size]) + mask + bytes(v ^ mask[i % 4] for i, v in enumerate(payload)))
178 return receive()
179 assert head[0] & 15 == 1, head
180 return json.loads(stream.read(size))
181 message = receive()
182 assert message == {'lines': [], 'cursor': 0}
183 checkpoint = None
184 while not (message := receive()).get('lines'): pass
185 assert message['lines'] == lines and message['cursor'] == 1
186 assert forms[-1][1]['start'] == ['1']
187 rows.append({**row, 'id': 'same-time', 'sequence': 2})
188 while not (message := receive()).get('lines'): pass
189 assert message['lines'][0]['id'] == 'same-time' and message['cursor'] == 2
190 rows.append({**row, '_time': '2026-10-14T01:00:00Z', 'id': 'late-oom', 'sequence': 3})
191 while not (message := receive()).get('lines'): pass
192 assert message['lines'][0]['id'] == 'late-oom' and message['cursor'] == 3
193 failed.set()
194 while not (message := receive()).get('error'): pass
195 assert 'store' in message['error']
196 finally:
197 stream.close()
198 sock.close()
199 failed.clear()
200 sock, stream = connect(101, cursor=2)
201 try:
202 while not (message := receive()).get('lines'): pass
203 assert [r['id'] for r in message['lines']] == ['late-oom']
204 finally:
205 stream.close()
206 sock.close()
207 bulk = [{**row, 'id': f'bulk-{i}', 'sequence': i + 4, '_msg': 'x' * 3500} for i in range(3000)]
208 rows.extend(bulk)
209 sock, stream = connect(101, cursor=3)
210 try:
211 received = []
212 while len(received) < 3000:
213 message = receive()
214 assert not message.get('error'), message
215 received.extend(r['id'] for r in message['lines'])
216 assert received == [r['id'] for r in bulk]
217 finally:
218 stream.close()
219 sock.close()
220 accepted = time.time()
221 sequence = int(accepted * 1_000_000)
222 delayed = {**row, 'id': 'delayed-search-visibility', 'sequence': sequence}
223 checkpoint = 3003
224 rows.append({**row, 'id': 'initial-unsettled', 'sequence': sequence + 1})
225 def publish():
226 nonlocal checkpoint
227 rows.append(delayed)
228 checkpoint = sequence + 1
229 threading.Timer(7, publish).start()
230 sock, stream = connect(101, cursor=sequence + 1)
231 try:
232 while not (message := receive()).get('lines'): pass
233 assert [r['id'] for r in message['lines']] == [delayed['id'], 'initial-unsettled']
234 assert time.time() - accepted >= 7
235 finally:
236 stream.close()
237 sock.close()
238 rows[:] = [{**row, 'id': f'tie-{i:05}', 'sequence': i + 1} for i in range(6000)] + [{**row, '_time': '2026-10-13T00:00:00Z', 'id': 'older-history', 'sequence': 6001}]
239 history = []
240 for offset in (0, 5000):
241 with urlopen(Request(origin + path + f'?limit=5000&offset={offset}', headers={'User-Name': 'fixture', 'User-Groups': 'infra-admin'})) as response:
242 history.extend(json.load(response))
243 assert len({line['id'] for line in history}) == 6001
244 assert history[-1]['id'] == 'older-history'
245 cookie = 'disposable-native-session'
246 with sqlite3.connect(root / 'data/accounts.sqlite') as db:
247 db.execute('INSERT INTO users VALUES (?,?)', ('fixture', json.dumps({'username':'fixture','enabled':True,'requiredActions':[]})))
248 role = db.execute('SELECT id FROM roles WHERE name=?', ('infra-admin',)).fetchone()[0]
249 db.execute('INSERT INTO memberships VALUES (?,?)', ('fixture', role))
250 db.execute('INSERT INTO sessions VALUES (?,?,?,?,?,?,?,?)', (hashlib.sha256(cookie.encode()).hexdigest(),'fixture','dashboard',int(time.time())+120,'127.0.0.1',0,0,0))
251 connect(401)
252 sock, stream = connect(101, cursor=sequence, cookie=cookie)
253 try:
254 while 'lines' not in (message := receive()): pass
255 with sqlite3.connect(root / 'data/accounts.sqlite') as db:
256 db.execute('DELETE FROM sessions')
257 while not (message := receive()).get('error'): pass
258 assert 'administrator' in message['error']
259 finally:
260 stream.close()
261 sock.close()
262 print('REST events/filters, WebSocket auth/Origin, fragmented responses, timestamp ties, delayed OOM, reconnect cursor, oversized backlog, delayed visibility, session revocation, upstream failure: passed')
263 finally:
264 server.terminate()
265 server.wait(timeout=10)
266 fixture.shutdown()
267
268
269if __name__ == '__main__':
270 main()
tools/log-shipper-test.py created+137
......@@ -0,0 +1,137 @@
1#!/usr/bin/env python3
2import importlib.util
3import io
4import json
5from pathlib import Path
6import tempfile
7import unittest
8from unittest.mock import patch
9
10
11spec = importlib.util.spec_from_file_location("shipper", Path(__file__).with_name("log-shipper.py"))
12shipper = importlib.util.module_from_spec(spec)
13spec.loader.exec_module(shipper)
14
15
16class Journal:
17 def __init__(self, *args, **kwargs):
18 self.stdout = io.StringIO("")
19 self.returncode = 0
20
21 def wait(self):
22 return self.returncode
23
24
25class Shipping(unittest.TestCase):
26 def setUp(self):
27 self.temporary = tempfile.TemporaryDirectory()
28 self.addCleanup(self.temporary.cleanup)
29 root = Path(self.temporary.name)
30 self.log = root / "alloc" / "allocation" / "alloc/logs/app.stdout.0"
31 self.log.parent.mkdir(parents=True)
32 self.log.write_bytes(b"2026-10-14T02:50:00Z stderr F before crash\n")
33 self.allocations = [{"ID": "allocation", "JobID": "shale", "TaskStates": {"app": {"Events": [
34 {"Time": 1791946241000000000, "Type": "Terminated", "ExitCode": 137,
35 "DisplayMessage": "Exit Code: 137", "Details": {"oom_killed": "true"}},
36 {"Time": 1791946242000000000, "Type": "Restarting", "DisplayMessage": "Task restarting in 17s"},
37 {"Time": 1791946265000000000, "Type": "Started", "DisplayMessage": "Task started by client"},
38 ]}}}]
39 self.batches = []
40 self.checkpoints = []
41 self.visible = None
42 replacement = patch.object(shipper, "retention_bounds", return_value=(0, float("inf")))
43 replacement.start()
44 self.addCleanup(replacement.stop)
45 replacement = patch.object(shipper, "urlopen", side_effect=lambda *a, **k: io.BytesIO(json.dumps({"rows": self.visible if self.visible is not None else len(self.batches) - (self.checkpoints[-1] if self.checkpoints else 0)}).encode()))
46 replacement.start()
47 self.addCleanup(replacement.stop)
48 for target, value in [("LOGS", root / "alloc"), ("STATE", root / "state.json")]:
49 replacement = patch.object(shipper, target, value)
50 replacement.start()
51 self.addCleanup(replacement.stop)
52 replacement = patch.object(shipper, "nomad", side_effect=lambda path: self.allocations if path == "/v1/allocations" else [{"Address": "127.0.0.1", "Port": 1}])
53 replacement.start()
54 self.addCleanup(replacement.stop)
55 replacement = patch.object(shipper.subprocess, "Popen", Journal)
56 replacement.start()
57 self.addCleanup(replacement.stop)
58
59 def once(self):
60 state = json.loads(shipper.STATE.read_text()) if shipper.STATE.exists() else {}
61 def deliver(base, rows, fields):
62 if rows and rows[0]["source"] == "log-shipper":
63 self.checkpoints.append(len(self.batches))
64 else:
65 self.batches.extend(rows)
66 with patch.object(shipper, "send", side_effect=deliver):
67 shipper.once(state)
68
69 def test_oom_and_restarts_are_retained_once_across_process_restarts_and_allocation_gc(self):
70 self.once()
71 self.assertEqual([row.get("event") for row in self.batches], [None, "OOM", "Restarting", "Started"])
72 self.assertEqual(self.batches[1]["level"], "error")
73 self.assertIn("Out of memory (OOM)", self.batches[1]["_msg"])
74 self.assertEqual(self.batches[2]["level"], "warn")
75 self.assertEqual(len({row["id"] for row in self.batches}), 4)
76 self.assertEqual(len({row["sequence"] for row in self.batches}), 4)
77 self.once()
78 self.assertEqual(len(self.batches), 4)
79 self.allocations.clear()
80 self.once()
81 self.assertEqual(json.loads(shipper.STATE.read_text())["events"], {})
82 self.assertEqual(len(self.batches), 4)
83
84 def test_same_timestamp_lines_and_split_writes_keep_distinct_ids(self):
85 self.once()
86 line = b"2026-10-14T02:50:01Z stdout F same message"
87 with self.log.open("ab") as file:
88 file.write(line)
89 self.once()
90 self.assertEqual(len(self.batches), 4)
91 with self.log.open("ab") as file:
92 file.write(b"\n" + line + b"\n")
93 self.once()
94 self.assertEqual(len(self.batches), 6)
95 self.assertNotEqual(self.batches[-1]["id"], self.batches[-2]["id"])
96 self.assertGreater(self.batches[-2]["sequence"], self.batches[-3]["sequence"])
97 self.assertGreater(self.batches[-1]["sequence"], self.batches[-2]["sequence"])
98
99 def test_failed_delivery_does_not_advance_file_or_event_cursors(self):
100 with patch.object(shipper, "send", side_effect=OSError("store unavailable")):
101 with self.assertRaises(OSError):
102 shipper.once({})
103 self.assertNotIn("events", json.loads(shipper.STATE.read_text()))
104 self.assertFalse(json.loads(shipper.STATE.read_text()).get("files"))
105 self.once()
106 self.assertEqual(len(self.batches), 4)
107
108 def test_expired_rows_advance_source_cursors_without_blocking_checkpoint(self):
109 with patch.object(shipper, "retention_bounds", return_value=(2_000_000_000, 3_000_000_000)):
110 self.once()
111 self.assertEqual(self.batches, [])
112 self.assertEqual(self.checkpoints, [])
113 self.assertTrue(json.loads(shipper.STATE.read_text())["events"])
114 self.once()
115 self.assertEqual(self.batches, [])
116
117 def test_invisible_rows_do_not_commit_checkpoint_or_offsets(self):
118 self.visible = 3
119 with patch.object(shipper.time, "monotonic", side_effect=[0, 31]):
120 with self.assertRaises(TimeoutError):
121 self.once()
122 self.assertEqual(self.checkpoints, [])
123 reserved = json.loads(shipper.STATE.read_text())
124 self.assertFalse(reserved.get("files"))
125 self.assertNotIn("events", reserved)
126 self.visible = 4
127 self.log.unlink()
128 self.allocations.clear()
129 with patch.object(shipper.time, "time_ns", return_value=0):
130 self.once()
131 self.assertEqual(self.batches[-1]["sequence"], reserved["sequence"])
132 self.assertEqual(self.batches[:4], self.batches[-4:])
133 self.assertEqual(len(self.checkpoints), 1)
134
135
136if __name__ == "__main__":
137 unittest.main()
tools/log-shipper.py+123-35
......@@ -1,11 +1,13 @@
11#!/usr/bin/env python3
22import datetime
33import json
4import math
45import os
56from pathlib import Path
67import re
78import subprocess
89import time
10from urllib.parse import urlencode
911from urllib.request import Request, urlopen
1012
1113
......@@ -31,44 +33,132 @@ def send(base, rows, fields):
3133 response.read()
3234
3335
36def retention_bounds(base):
37 with urlopen(base + "/metrics", timeout=10) as response:
38 metrics = response.read().decode()
39 flags = dict(re.findall(r'flag\{name="(retentionPeriod|futureRetention|maxBackfillAge)", value="([^"]+)"', metrics))
40 durations = {}
41 units = {"ms": .001, "s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800, "y": 31536000}
42 for name in ("retentionPeriod", "futureRetention", "maxBackfillAge"):
43 value = flags[name]
44 if re.fullmatch(r"[0-9.]+M?", value):
45 seconds = float(value.rstrip("M")) * 2678400
46 else:
47 parts = re.findall(r"([0-9.]+)(ms|[smhdwy])", value.lower())
48 if "".join(number + unit for number, unit in parts) != value.lower():
49 raise ValueError("unsupported log retention duration")
50 seconds = sum(float(number) * units[unit] for number, unit in parts)
51 if not math.isfinite(seconds) or seconds < 0:
52 raise ValueError("invalid log retention duration")
53 durations[name] = int(seconds * 1000) / 1000
54 retention = max(86400, durations["retentionPeriod"])
55 backfill = durations["maxBackfillAge"]
56 backfill = backfill if 0 < backfill <= retention else retention
57 current = time.time()
58 return current - backfill, (math.floor((current + max(86400, durations["futureRetention"])) / 86400) + 1) * 86400
59
60
61def save(state):
62 pending_state = STATE.with_suffix(".pending")
63 with os.fdopen(os.open(pending_state, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), "w") as file:
64 json.dump(state, file)
65 os.replace(pending_state, STATE)
66
67
3468def once(state):
3569 services = nomad("/v1/service/victoria-logs")
3670 if not services:
3771 return
3872 target = services[0]
3973 base = f"http://{target['Address']}:{target['Port']}"
40 jobs = {item["ID"]: item["JobID"] for item in nomad("/v1/allocations")}
41 offsets = state.setdefault("files", {})
42 rows = []
43 pending = {}
44 for file in sorted(LOGS.glob("*/alloc/logs/*")):
45 match = re.fullmatch(r"(.+)\.(stdout|stderr)\.\d+", file.name)
46 job = jobs.get(file.parent.parent.parent.name)
47 if not match or not job:
48 continue
49 info = file.stat()
50 old = offsets.get(str(file), {})
51 offset = old.get("offset", 0) if old.get("inode") == info.st_ino and old.get("offset", 0) <= info.st_size else 0
52 with file.open("rb") as stream:
53 stream.seek(offset)
54 chunk = stream.read(262144)
55 if not chunk:
74 if "pending" not in state:
75 allocations = nomad("/v1/allocations")
76 jobs = {item["ID"]: item["JobID"] for item in allocations}
77 offsets = state.get("files", {}).copy()
78 rows = []
79 pending = {}
80 for file in sorted(LOGS.glob("*/alloc/logs/*")):
81 match = re.fullmatch(r"(.+)\.(stdout|stderr)\.\d+", file.name)
82 job = jobs.get(file.parent.parent.parent.name)
83 if not match or not job:
5684 continue
57 if not chunk.endswith(b"\n"):
58 chunk = chunk[:chunk.rfind(b"\n") + 1]
59 if not chunk:
60 continue
61 pending[str(file)] = {"inode": info.st_ino, "offset": offset + len(chunk)}
62 for raw in chunk.decode("utf8", "replace").splitlines():
63 parsed = CRI.match(raw)
64 if not parsed:
65 continue
66 stamp, stream, message = parsed.groups()
67 level = "error" if re.search(r"\b(ERROR|ERR|FATAL|CRITICAL|PANIC)\b|level=(error|fatal)", message, re.I) else "warn" if re.search(r"\b(WARN|WARNING|WRN)\b|level=warn", message, re.I) else "info"
68 rows.append({"_time": stamp, "_msg": message, "source": "nomad", "job": job,
69 "task": match.group(1), "stream": stream, "level": level})
70 send(base, rows, "source,job,task,stream")
71 offsets.update(pending)
85 info = file.stat()
86 old = offsets.get(str(file), {})
87 offset = old.get("offset", 0) if old.get("inode") == info.st_ino and old.get("offset", 0) <= info.st_size else 0
88 with file.open("rb") as stream:
89 stream.seek(offset)
90 chunk = stream.read(262144)
91 if not chunk:
92 continue
93 if not chunk.endswith(b"\n"):
94 chunk = chunk[:chunk.rfind(b"\n") + 1]
95 if not chunk:
96 continue
97 pending[str(file)] = {"inode": info.st_ino, "offset": offset + len(chunk)}
98 position = offset
99 for raw in chunk.splitlines(keepends=True):
100 identity = f"{file.parent.parent.parent.name}:{file.name}:{info.st_ino}:{position}"
101 position += len(raw)
102 parsed = CRI.match(raw.decode("utf8", "replace").rstrip("\r\n"))
103 if not parsed:
104 continue
105 stamp, stream, message = parsed.groups()
106 level = "error" if re.search(r"\b(ERROR|ERR|FATAL|CRITICAL|PANIC)\b|level=(error|fatal)", message, re.I) else "warn" if re.search(r"\b(WARN|WARNING|WRN)\b|level=warn", message, re.I) else "info"
107 rows.append({"_time": stamp, "_msg": message, "source": "nomad", "job": job,
108 "task": match.group(1), "stream": stream, "level": level,
109 "id": identity, "allocation": file.parent.parent.parent.name})
110 events = {}
111 for allocation in allocations:
112 seen = set(state.get("events", {}).get(allocation["ID"], []))
113 current = events[allocation["ID"]] = []
114 for task, task_state in (allocation.get("TaskStates") or {}).items():
115 for event in task_state.get("Events") or []:
116 kind = event["Type"]
117 identity = f"{allocation['ID']}:{task}:{event['Time']}:{kind}"
118 current.append(identity)
119 if identity in seen:
120 continue
121 oom = str((event.get("Details") or {}).get("oom_killed", "")).lower() == "true"
122 message = event.get("DisplayMessage") or event.get("Message") or kind
123 if oom:
124 message = "Out of memory (OOM): " + message
125 stamp = datetime.datetime.fromtimestamp(event["Time"] / 1e9, datetime.timezone.utc).isoformat()
126 level = "error" if oom or event.get("FailsTask") or kind in {"Driver Failure", "Failed Validation"} else "warn" if kind in {"Restarting", "Not Restarting"} or kind == "Terminated" and event.get("ExitCode") else "info"
127 rows.append({"_time": stamp, "_msg": message, "source": "nomad", "job": allocation["JobID"],
128 "task": task, "stream": "event", "level": level, "event": "OOM" if oom else kind,
129 "id": identity, "allocation": allocation["ID"]})
130 sequence = max(state.get("sequence", 0), time.time_ns() // 1000)
131 for row in rows:
132 sequence += 1
133 row["sequence"] = sequence
134 offsets.update(pending)
135 state["sequence"] = sequence
136 state["pending"] = {"rows": rows, "files": offsets, "events": events}
137 save(state)
138 batch = state["pending"]
139 rows = batch["rows"]
140 sequence = state["sequence"]
141 if rows:
142 minimum, maximum = retention_bounds(base)
143 rows = [row for row in rows if minimum <= datetime.datetime.fromisoformat(row["_time"].replace("Z", "+00:00")).timestamp() < maximum]
144 if rows:
145 send(base, rows, "source,job,task,stream")
146 query = f"source:=nomad sequence:>={rows[0]['sequence']} sequence:<={sequence} | stats count_uniq(sequence) as rows"
147 deadline = time.monotonic() + 30
148 while True:
149 request = Request(base + "/select/logsql/query", data=urlencode({"query": query, "start": minimum, "end": maximum}).encode())
150 with urlopen(request, timeout=10) as response:
151 visible = json.loads(response.read() or b"{}")
152 if int(visible.get("rows", 0)) == len(rows):
153 break
154 if time.monotonic() >= deadline:
155 raise TimeoutError("log batch is not searchable yet")
156 time.sleep(.25)
157 send(base, [{"_msg": "searchable log checkpoint", "source": "log-shipper", "sequence": sequence}], "source")
158 state["files"] = {path: value for path, value in batch["files"].items() if Path(path).is_file()}
159 state["events"] = batch["events"]
160 del state["pending"]
161 save(state)
72162
73163 command = ["journalctl", "--no-pager", "--output=json"]
74164 command += ["--after-cursor=" + state["cursor"]] if state.get("cursor") else ["--since=-1 hour"]
......@@ -97,9 +187,7 @@ def once(state):
97187 send(base, system, "source,unit")
98188 if cursor:
99189 state["cursor"] = cursor
100 pending_state = STATE.with_suffix(".pending")
101 pending_state.write_text(json.dumps(state))
102 os.replace(pending_state, STATE)
190 save(state)
103191
104192
105193def main():
......@@ -110,7 +198,7 @@ def main():
110198 once(state)
111199 except Exception as error:
112200 print(f"log shipping paused: {error}", flush=True)
113 time.sleep(5)
201 time.sleep(1)
114202
115203
116204if __name__ == "__main__":