authoravatarclover caruso<git@paperclover.net>2026-10-19 17:28:16-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-21 02:52:29-07:00
logb00a4bc62d1b4f4972f879356490535a4b4ea9dc
tree038d563d955aba396ea8f0e0f9dbb57b99ddde40
parent9980b1fb6b031ee32f458b5f17b8fc33b536c1cf
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Stream service logs with durable searchable checkpoints

Retain application output and lifecycle events across reconnects, delivery retries and allocation cleanup. Gate streaming on verified search visibility; preserve timestamp ties in paged history and use read-only requests in previews. Correct the dashboard dependency hash required by the current locked Nix build. Assisted-by: gpt-6-astra

11 files changed, 772 insertions(+), 100 deletions(-)

dashboard/src/main.rs+7
...@@ -101,6 +101,7 @@ struct App {...@@ -101,6 +101,7 @@ struct App {
101 internal: Option<(url::Url, reqwest::Client)>,101 internal: Option<(url::Url, reqwest::Client)>,
102 cache: cache::Cache,102 cache: cache::Cache,
103 nomad_slots: Semaphore,103 nomad_slots: Semaphore,
104 log_slots: Semaphore,
104 specs: Mutex<HashMap<String, Value>>,105 specs: Mutex<HashMap<String, Value>>,
105 repo: PathBuf,106 repo: PathBuf,
106 data: PathBuf,107 data: PathBuf,
...@@ -314,6 +315,11 @@ async fn api(State(app): State<Arc<App>>, request: Request) -> Result<Response>...@@ -314,6 +315,11 @@ async fn api(State(app): State<Arc<App>>, request: Request) -> Result<Response>
314 });315 });
315 return Ok(Sse::new(stream).into_response());316 return Ok(Sse::new(stream).into_response());
316 }317 }
318 if let ["services", id, "logs", "stream"] = parts.as_slice()
319 && method == Method::GET
320 {
321 return telemetry::log_stream(app, request, id, &query).await;
322 }
317 if let ["vms", name, kind @ ("console" | "serial")] = parts.as_slice() {323 if let ["vms", name, kind @ ("console" | "serial")] = parts.as_slice() {
318 if method == Method::GET {324 if method == Method::GET {
319 return vms::console(request, name, *kind == "serial").await;325 return vms::console(request, name, *kind == "serial").await;
...@@ -546,6 +552,7 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {...@@ -546,6 +552,7 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
546 internal,552 internal,
547 cache: cache::Cache::default(),553 cache: cache::Cache::default(),
548 nomad_slots: Semaphore::new(4),554 nomad_slots: Semaphore::new(4),
555 log_slots: Semaphore::new(32),
549 specs: Mutex::new(HashMap::new()),556 specs: Mutex::new(HashMap::new()),
550 repo: env("SNOWGLOBE_REPO", "..").into(),557 repo: env("SNOWGLOBE_REPO", "..").into(),
551 data: env("SNOWGLOBE_DATA_DIR", "data").into(),558 data: env("SNOWGLOBE_DATA_DIR", "data").into(),
dashboard/src/telemetry.rs+106-7
...@@ -1,5 +1,6 @@...@@ -1,5 +1,6 @@
1use crate::*;1use crate::*;
2use futures::{StreamExt, stream};2use futures::{SinkExt, StreamExt, stream};
3use axum::extract::{FromRequestParts, ws::{Message, WebSocketUpgrade}};
34
4pub async fn endpoint(app: Arc<App>, service: &str) -> Result<String> {5pub async fn endpoint(app: Arc<App>, service: &str) -> Result<String> {
5 if let Some((base, _)) = &app.internal {6 if let Some((base, _)) = &app.internal {
...@@ -297,8 +298,7 @@ fn terms(query: &str, trace: bool) -> Result<Vec<Term>> {...@@ -297,8 +298,7 @@ fn terms(query: &str, trace: bool) -> Result<Vec<Term>> {
297 }298 }
298 Ok(found)299 Ok(found)
299}300}
300pub async fn logs(app: Arc<App>, id: &str, query: &HashMap<String, String>) -> Result<Value> {301fn log_query(id: &str, query: &HashMap<String, String>) -> Result<String> {
301 let limit = query_number(query, "limit", 200.0, 1.0, 5000.0)?;
302 let mut filters = vec![302 let mut filters = vec![
303 "source:=nomad".to_owned(),303 "source:=nomad".to_owned(),
304 format!("job:={}", serde_json::to_string(id)?),304 format!("job:={}", serde_json::to_string(id)?),
...@@ -332,15 +332,36 @@ pub async fn logs(app: Arc<App>, id: &str, query: &HashMap<String, String>) -> R...@@ -332,15 +332,36 @@ pub async fn logs(app: Arc<App>, id: &str, query: &HashMap<String, String>) -> R
332 ))?332 ))?
333 ));333 ));
334 }334 }
335 Ok(filters.join(" "))
336}
337
338fn log_line(row: Value) -> Value {
339 json!({"id":row["id"].as_str().map(str::to_owned).unwrap_or_else(|| format!("{}:{}:{}", string(&row["_stream_id"]), string(&row["_time"]), string(&row["_msg"]))),
340 "sequence":number(&row["sequence"]),
341 "t":core::seconds(&row["_time"]),"container":row["task"].as_str().unwrap_or("app"),
342 "stream":row["stream"].as_str().filter(|s| ["stderr", "event"].contains(s)).unwrap_or("stdout"),
343 "level":row["level"],"text":row["_msg"].as_str().unwrap_or_default(),
344 "event":row["event"],"allocation":row["allocation"]})
345}
346
347pub async fn logs(app: Arc<App>, id: &str, query: &HashMap<String, String>) -> Result<Value> {
348 let limit = query_number(query, "limit", 200.0, 1.0, 5000.0)?;
349 if query.contains_key("after") && query.contains_key("before")
350 && query_number(query, "after", 0.0, f64::NEG_INFINITY, f64::INFINITY)?
351 > query_number(query, "before", 0.0, f64::NEG_INFINITY, f64::INFINITY)?
352 {
353 return Err(Error::new(400, "Choose a “to” time at or after “from”."));
354 }
355 let offset = query_number(query, "offset", 0.0, 0.0, 1_000_000_000.0)?;
335 let mut form = vec![356 let mut form = vec![
336 ("query".into(), filters.join(" ")),357 ("query".into(), format!("{} | sort by (_time desc, id desc, _stream_id desc, _msg desc) offset {offset:.0} limit {limit:.0}", log_query(id, query)?)),
337 ("limit".into(), limit.to_string()),358 ("limit".into(), limit.to_string()),
338 ];359 ];
339 for (key, field, add) in [("after", "start", 0.000001), ("before", "end", 0.0)] {360 for (key, field) in [("after", "start"), ("before", "end")] {
340 if query.contains_key(key) {361 if query.contains_key(key) {
341 form.push((362 form.push((
342 field.into(),363 field.into(),
343 (query_number(query, key, 0.0, f64::NEG_INFINITY, f64::INFINITY)? + add)364 query_number(query, key, 0.0, f64::NEG_INFINITY, f64::INFINITY)?
344 .to_string(),365 .to_string(),
345 ));366 ));
346 }367 }
...@@ -352,12 +373,90 @@ pub async fn logs(app: Arc<App>, id: &str, query: &HashMap<String, String>) -> R...@@ -352,12 +373,90 @@ pub async fn logs(app: Arc<App>, id: &str, query: &HashMap<String, String>) -> R
352 let response = read(app, url, Some(form)).await?;373 let response = read(app, url, Some(form)).await?;
353 let mut rows = string(&response.value).lines().filter(|l| !l.trim().is_empty()).map(|line| {374 let mut rows = string(&response.value).lines().filter(|l| !l.trim().is_empty()).map(|line| {
354 let row: Value = serde_json::from_str(line)?;375 let row: Value = serde_json::from_str(line)?;
355 Ok::<_,Error>(json!({"t":core::seconds(&row["_time"]),"container":row["task"].as_str().unwrap_or("app"),"stream":if row["stream"] == "stderr" {"stderr"} else {"stdout"},"level":if row["level"] == "error" || row["level"] == "warn" {row["level"].clone()} else {Value::Null},"text":row["_msg"].as_str().unwrap_or_default()}))376 Ok::<_,Error>(log_line(row))
356 }).collect::<Result<Vec<_>>>()?;377 }).collect::<Result<Vec<_>>>()?;
357 rows.sort_by(|a, b| number(&b["t"]).total_cmp(&number(&a["t"])));378 rows.sort_by(|a, b| number(&b["t"]).total_cmp(&number(&a["t"])));
358 Ok(json!(rows))379 Ok(json!(rows))
359}380}
360381
382pub async fn log_stream(app: Arc<App>, request: Request, id: &str, query: &HashMap<String, String>) -> Result<Response> {
383 vms::same_origin(&request)?;
384 let filter = log_query(id, query)?;
385 let mut cursor = query_number(query, "cursor", 0.0, 0.0, 9_007_199_254_740_991.0)?;
386 let from = query.get("from").map(|_| query_number(query, "from", 0.0, 0.0, f64::INFINITY)).transpose()?;
387 let base = endpoint(app.clone(), "victoria-logs").await?;
388 let url = format!("{base}/select/logsql/query");
389 let headers = request.headers().clone();
390 let (mut parts, _) = request.into_parts();
391 let upgrade = WebSocketUpgrade::from_request_parts(&mut parts, &()).await
392 .map_err(|_| Error::new(400, "Open the service log to connect."))?
393 .max_message_size(1024).max_frame_size(1024);
394 Ok(upgrade.on_upgrade(move |mut socket| async move {
395 let run = async {
396 let _slot = app.log_slots.try_acquire().map_err(|_| Error::new(429, "Too many live logs are open. Close one and retry."))?;
397 let mut first = true;
398 let mut poll = tokio::time::interval(Duration::from_secs(1));
399 poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
400 let mut heartbeat = tokio::time::interval(Duration::from_secs(15));
401 loop {
402 tokio::select! {
403 message = socket.recv() => match message {
404 Some(Ok(Message::Ping(bytes))) => socket.send(Message::Pong(bytes)).await?,
405 Some(Ok(Message::Pong(_))) => {},
406 _ => break,
407 },
408 _ = heartbeat.tick() => {
409 if app.auth.ready() {
410 let account = app.auth.session(&headers, "dashboard")?;
411 if !array(&account["groups"]).iter().any(|g| g["name"] == "infra-admin") {
412 return Err(Error::new(401, "Sign in as an administrator to continue reading logs."));
413 }
414 }
415 tokio::time::timeout(Duration::from_secs(5), socket.send(Message::Ping(Bytes::new()))).await
416 .map_err(|_| Error::new(504, "The log connection stopped responding."))??;
417 },
418 _ = poll.tick() => {
419 let checkpoint = read(app.clone(), url.clone(), Some(vec![("query".into(), "source:=log-shipper | stats max(sequence) as sequence".into())])).await?;
420 let checkpoint: Value = serde_json::from_str(string(&checkpoint.value).lines().find(|line| !line.trim().is_empty()).unwrap_or("{}"))?;
421 let settled = number(&checkpoint["sequence"]);
422 if first { cursor = cursor.min(settled); first = false; }
423 let mut form = vec![("query", format!("{filter} sequence:>{cursor:.0} sequence:<={settled:.0} | sort by (sequence) limit 5000"))];
424 if let Some(from) = from { form.push(("start", from.to_string())); }
425 let request = if env("SNOWGLOBE_PREVIEW", "") == "1" {
426 app.request(Method::GET, &url)?.query(&form)
427 } else {
428 app.request(Method::POST, &url)?.form(&form)
429 };
430 let response = request.timeout(Duration::from_secs(8)).send().await?;
431 if !response.status().is_success() { return Err(Error::new(502, "The log store isn't answering. Retrying…")); }
432 let mut chunks = response.bytes_stream();
433 let mut body = Vec::new();
434 while let Some(chunk) = chunks.next().await {
435 body.extend_from_slice(&chunk?);
436 if body.len() > 8 * 1024 * 1024 {
437 let end = body[..8 * 1024 * 1024].iter().rposition(|b| *b == b'\n')
438 .ok_or_else(|| Error::new(502, "A log entry is too large. Narrow the search and retry."))?;
439 body.truncate(end + 1);
440 break;
441 }
442 }
443 let lines = std::str::from_utf8(&body)?.lines().filter(|l| !l.trim().is_empty())
444 .map(|raw| serde_json::from_str(raw).map(log_line)).collect::<std::result::Result<Vec<_>, _>>()?;
445 for line in &lines { cursor = cursor.max(number(&line["sequence"])); }
446 tokio::time::timeout(Duration::from_secs(5), socket.send(Message::Text(json!({"lines":lines,"cursor":cursor}).to_string().into()))).await
447 .map_err(|_| Error::new(504, "The log connection stopped responding."))??;
448 }
449 }
450 }
451 Ok::<_, Error>(())
452 }.await;
453 if let Err(error) = run {
454 let _ = tokio::time::timeout(Duration::from_secs(2), socket.send(Message::Text(json!({"error":error.message}).to_string().into()))).await;
455 }
456 let _ = tokio::time::timeout(Duration::from_secs(1), socket.close()).await;
457 }).into_response())
458}
459
361fn span(row: &Value) -> Value {460fn span(row: &Value) -> Value {
362 let mut attributes: serde_json::Map<String, Value> = row461 let mut attributes: serde_json::Map<String, Value> = row
363 .as_object()462 .as_object()
dashboard/src/vms.rs+2-2
...@@ -75,7 +75,7 @@ pub async fn console(request: Request, name: &str, serial: bool) -> Result<Respo...@@ -75,7 +75,7 @@ pub async fn console(request: Request, name: &str, serial: bool) -> Result<Respo
75 .into_response())75 .into_response())
76}76}
7777
78fn same_origin(request: &Request) -> Result<()> {78pub(crate) fn same_origin(request: &Request) -> Result<()> {
79 let supplied = request79 let supplied = request
80 .headers()80 .headers()
81 .get("origin")81 .get("origin")
...@@ -94,7 +94,7 @@ fn same_origin(request: &Request) -> Result<()> {...@@ -94,7 +94,7 @@ fn same_origin(request: &Request) -> Result<()> {
94 };94 };
95 authority != host || !matches!(origin.scheme(), "http" | "https")95 authority != host || !matches!(origin.scheme(), "http" | "https")
96 }) {96 }) {
97 return Err(Error::new(403, "Open the VM page on this site to connect."));97 return Err(Error::new(403, "Open this page on the same site to connect."));
98 }98 }
99 Ok(())99 Ok(())
100}100}
dashboard/web/components/LogView.css+2
...@@ -18,6 +18,8 @@...@@ -18,6 +18,8 @@
18.log-view .log { grid-template-columns: 62px 1fr; padding: 0 var(--gutter); }18.log-view .log { grid-template-columns: 62px 1fr; padding: 0 var(--gutter); }
19.log-view .log.warn:hover { background: color-mix(in srgb, var(--warning) 16%, transparent); }19.log-view .log.warn:hover { background: color-mix(in srgb, var(--warning) 16%, transparent); }
20.log-view .log.error:hover { background: color-mix(in srgb, var(--critical) 20%, transparent); }20.log-view .log.error:hover { background: color-mix(in srgb, var(--critical) 20%, transparent); }
21.log-view .task-event { background: color-mix(in srgb, var(--text) 4%, transparent); }
22.log-view .event-label { margin-right: 8px; font: 600 11px var(--sans); }
21.log-view .ansi {23.log-view .ansi {
22 color: color-mix(in srgb, var(--fg, currentColor) var(--ansi-mix), var(--text));24 color: color-mix(in srgb, var(--fg, currentColor) var(--ansi-mix), var(--text));
23 background: color-mix(in srgb, var(--bg, transparent) 45%, transparent);25 background: color-mix(in srgb, var(--bg, transparent) 45%, transparent);
dashboard/web/components/LogView.tsx+2-1
...@@ -132,9 +132,10 @@ export function LogView(props: {...@@ -132,9 +132,10 @@ export function LogView(props: {
132 <div class="log-edge day">{date(line.t)}</div>132 <div class="log-edge day">{date(line.t)}</div>
133 </Show>133 </Show>
134 {props.before?.(line, index)}134 {props.before?.(line, index)}
135 <div class={`log ${line.level ?? ""}`}>135 <div class={`log ${line.level ?? ""}`} classList={{ "task-event": line.stream === "event" }}>
136 <time data-tip={datetime(line.t)}>{clock(line.t)}</time>136 <time data-tip={datetime(line.t)}>{clock(line.t)}</time>
137 <span>137 <span>
138 <Show when={line.event}><strong class="event-label" data-tip={line.allocation ? `Allocation ${line.allocation}` : undefined}>{line.event}</strong>{" "}</Show>
138 <Show when={mixed()}>139 <Show when={mixed()}>
139 <Show when={props.onContainer} fallback={<span class="container">{line.container}</span>}>140 <Show when={props.onContainer} fallback={<span class="container">{line.container}</span>}>
140 {(filter) => (141 {(filter) => (
dashboard/web/pages/Service.tsx+117-53
...@@ -8,7 +8,7 @@ import RotateCcw from "lucide-solid/icons/rotate-ccw";...@@ -8,7 +8,7 @@ import RotateCcw from "lucide-solid/icons/rotate-ccw";
8import Search from "lucide-solid/icons/search";8import Search from "lucide-solid/icons/search";
9import X from "lucide-solid/icons/x";9import X from "lucide-solid/icons/x";
10import {10import {
11 createEffect, createMemo, createResource, createSignal, createUniqueId, For, type JSX, Match, onCleanup, Show, Switch,11 createEffect, createMemo, createResource, createSignal, createUniqueId, For, type JSX, Match, onCleanup, Show, Switch, untrack,
12} from "solid-js";12} from "solid-js";
13import { Portal } from "solid-js/web";13import { Portal } from "solid-js/web";
14import {14import {
...@@ -43,29 +43,37 @@ const LEVELS = [[undefined, "all"], ["warn", "warn"], ["error", "error"]] as con...@@ -43,29 +43,37 @@ const LEVELS = [[undefined, "all"], ["warn", "warn"], ["error", "error"]] as con
43/** URL state of the logs and traces tabs; `at` opens the logs at a moment instead of following. */43/** URL state of the logs and traces tabs; `at` opens the logs at a moment instead of following. */
44type Query = { q?: string; level?: "warn" | "error"; at?: string; from?: string; to?: string; errors?: "1"; sort?: TraceSort; trace?: string };44type Query = { q?: string; level?: "warn" | "error"; at?: string; from?: string; to?: string; errors?: "1"; sort?: TraceSort; trace?: string };
4545
46/** Each service's latest unfiltered lines, so returning to a service draws its logs at once. */
47const recent = new Map<string, LogLine[]>();
48
49/** Newest at the bottom; follows new lines while scrolled to the end, and loads older ones near the top. */46/** Newest at the bottom; follows new lines while scrolled to the end, and loads older ones near the top. */
50function Logs(props: { service: ServiceDetail; tools: HTMLElement }) {47function Logs(props: { service: ServiceDetail; tools: HTMLElement }) {
51 const [params, setParams] = useSearchParams<Query>();48 const [params, setParams] = useSearchParams<Query>();
52 const at = () => (params.at ? Number(params.at) : null);49 const at = () => (params.at ? Number(params.at) : null);
53 const bounded = () => at() !== null || !!params.to;50 const bounded = () => at() !== null || !!params.to;
54 const unfiltered = () => !params.q && !params.level && !params.from && !params.to && at() === null;
55 const dateValue = (value?: string) => {51 const dateValue = (value?: string) => {
56 if (!value) return "";52 if (!value) return "";
57 const date = new Date(Number(value) * 1000);53 const date = new Date(Number(value) * 1000);
58 return new Date(date.getTime() - date.getTimezoneOffset() * 60_000).toISOString().slice(0, 16);54 return new Date(date.getTime() - date.getTimezoneOffset() * 60_000).toISOString().slice(0, 16);
59 };55 };
60 const [older, setOlder] = createSignal<LogLine[]>([]);56 const [all, setLines] = createSignal<LogLine[]>([]);
61 const [newer, setNewer] = createSignal<LogLine[]>([]);57 let pending: LogLine[] = [];
58 let generation = 0;
59 let older: { before: string; offset: number } | undefined;
62 const [exhausted, setExhausted] = createSignal(false);60 const [exhausted, setExhausted] = createSignal(false);
63 const [loadingOlder, setLoadingOlder] = createSignal(false);61 const [loadingOlder, setLoadingOlder] = createSignal(false);
64 const [follow, setFollow] = createSignal(true);62 const [follow, setFollow] = createSignal(true);
65 const [unseen, setUnseen] = createSignal(0);63 const [unseen, setUnseen] = createSignal(0);
66 /** Why the last poll failed, until one succeeds. */
67 const [stalled, setStalled] = createSignal<string>();64 const [stalled, setStalled] = createSignal<string>();
68 const fetchLines = (limit: number, extra: { before?: string; after?: string }) =>65 const [connected, setConnected] = createSignal(false);
66 const merge = (before: LogLine[], after: LogLine[]) => [...new Map([...before, ...after]
67 .map((line) => [line.id ?? JSON.stringify(line), line])).values()].sort((a, b) => a.t - b.t);
68 const resume = () => {
69 older = undefined;
70 setLines((lines) => merge(lines, pending).slice(-10_000));
71 pending = [];
72 setUnseen(0);
73 setExhausted(false);
74 setFollow(true);
75 };
76 const fetchLines = (limit: number, extra: { before?: string; after?: string; offset?: string }) =>
69 parseResponse(api.services[":id"].logs.$get({77 parseResponse(api.services[":id"].logs.$get({
70 param: { id: props.service.id },78 param: { id: props.service.id },
71 query: { q: params.q ?? "", ...(params.level ? { level: params.level } : {}), limit: String(limit),79 query: { q: params.q ?? "", ...(params.level ? { level: params.level } : {}), limit: String(limit),
...@@ -74,64 +82,120 @@ function Logs(props: { service: ServiceDetail; tools: HTMLElement }) {...@@ -74,64 +82,120 @@ function Logs(props: { service: ServiceDetail; tools: HTMLElement }) {
74 const [first, { refetch }] = createResource<LogLine[], string>(82 const [first, { refetch }] = createResource<LogLine[], string>(
75 // A string key, since a fresh array from an unrelated rerun would count as a change and refetch.83 // A string key, since a fresh array from an unrelated rerun would count as a change and refetch.
76 () => JSON.stringify([props.service.id, params.q, params.level, params.at, params.from, params.to]),84 () => JSON.stringify([props.service.id, params.q, params.level, params.at, params.from, params.to]),
77 (_, { refetching }) => {85 () => {
78 setOlder([]);86 const current = ++generation;
79 setNewer([]);87 older = undefined;
88 setLines([]);
89 pending = [];
90 setLoadingOlder(false);
80 setExhausted(false);91 setExhausted(false);
81 setFollow(!bounded());92 setFollow(!bounded());
82 setUnseen(0);93 setUnseen(0);
83 setStalled();94 setStalled();
84 const cached = !refetching && unfiltered() && recent.get(props.service.id);95 return fetchLines(1000, at() === null ? {} : { before: String(at()! + 30) }).then((lines) => {
85 if (cached) {96 if (current === generation) setLines(lines);
86 queueMicrotask(() => poll());97 return lines;
87 return cached;98 });
88 }
89 return fetchLines(1000, at() === null ? {} : { before: String(at()! + 30) });
90 },99 },
91 );100 );
92 const loaded = lastGood(first);
93 const all = createMemo(() => [...older(), ...(loaded() ?? []), ...newer()]);
94 createEffect(() => first.state === "ready" && unfiltered() && recent.set(props.service.id, all().slice(-1000)));
95101
96 const loadOlder = async () => {102 const loadOlder = async () => {
97 const oldest = all()[0];103 const oldest = all()[0];
98 if (!oldest || loadingOlder() || exhausted()) return;104 if (!oldest || loadingOlder() || exhausted()) return;
99 if (params.from && oldest.t <= Number(params.from)) return setExhausted(true);105
100 setLoadingOlder(true);106 setLoadingOlder(true);
107 const current = generation;
101 try {108 try {
102 const lines = await fetchLines(5000, { before: String(oldest.t) });109 const page = older ??= { before: String(oldest.t + 0.000001), offset: 0 };
103 if (!lines.length) setExhausted(true);110 const fetched = await fetchLines(5000, { before: page.before, offset: String(page.offset) });
104 setOlder([...lines, ...older()]);111 if (current !== generation || older !== page) return;
112 const known = new Set(all().map((line) => line.id ?? JSON.stringify(line)));
113 const lines = fetched.filter((line) => !known.has(line.id ?? JSON.stringify(line)));
114 page.offset += fetched.length;
115 if (fetched.length < 5000) setExhausted(true);
116 setLines((previous) => merge(lines, previous));
105 } catch (failure) {117 } catch (failure) {
106 toast(`Couldn't load older lines. ${reason(failure)}`);118 if (current === generation) toast(`Couldn't load older lines. ${reason(failure)}`);
107 } finally {119 } finally {
108 setLoadingOlder(false);120 if (current === generation) setLoadingOlder(false);
109 }121 }
110 };122 };
111123
112 const poll = async () => {124 createEffect(() => {
113 const last = all().at(-1);125 const id = props.service.id;
114 if (bounded() || !last || first.loading) return;126 const q = params.q ?? "";
115 let lines: LogLine[];127 const level = params.level;
116 try {128 const from = params.from;
117 lines = await fetchLines(1000, { after: String(last.t) });129 setConnected(false);
118 setStalled();130 if (bounded() || first.state !== "ready") return;
119 } catch (failure) {131 let disposed = false;
120 setStalled(reason(failure));132 let failures = 0;
121 return;133 let cursor = Math.max(0, ...untrack(all).map((line) => line.sequence ?? 0));
122 }134 let socket: WebSocket;
123 // A search changed while this was under way, so the lines belong to the old one.135 let retry: ReturnType<typeof setTimeout> | undefined;
124 if (!lines.length || first.loading || last !== all().at(-1)) return;136 let stale: ReturnType<typeof setTimeout> | undefined;
125 setNewer([...newer(), ...lines]);137 const connect = () => {
126 if (!follow()) setUnseen(unseen() + lines.length);138 const url = new URL(`/api/services/${encodeURIComponent(id)}/logs/stream`, location.href);
127 };139 url.protocol = location.protocol === "https:" ? "wss:" : "ws:";
128 const timer = setInterval(poll, 2000);140 url.searchParams.set("q", q);
129 onCleanup(() => clearInterval(timer));141 if (level) url.searchParams.set("level", level);
142 url.searchParams.set("cursor", String(cursor));
143 if (from) url.searchParams.set("from", from);
144 socket = new WebSocket(url);
145 socket.onmessage = (event) => {
146 if (disposed) return;
147 const message: { lines?: LogLine[]; cursor?: number; error?: string } = JSON.parse(event.data);
148 if (message.error) {
149 setConnected(false);
150 setStalled(message.error);
151 return;
152 }
153 failures = 0;
154 setConnected(true);
155 setStalled();
156 clearTimeout(stale);
157 stale = setTimeout(() => socket.close(), 35_000);
158 if (message.lines?.length) {
159 const known = new Set([...all(), ...pending].map((line) => line.id ?? JSON.stringify(line)));
160 const lines = message.lines.filter((line) => {
161 const id = line.id ?? JSON.stringify(line);
162 if (known.has(id)) return false;
163 known.add(id);
164 return true;
165 });
166 if (follow()) {
167 older = undefined;
168 setLines((previous) => merge(previous, lines).slice(-10_000));
169 }
170 else {
171 pending = merge(pending, lines).slice(-10_000);
172 setUnseen((previous) => previous + lines.length);
173 }
174 }
175 cursor = message.cursor ?? cursor;
176 };
177 socket.onclose = () => {
178 if (disposed) return;
179 clearTimeout(stale);
180 setConnected(false);
181 setStalled((previous) => previous ?? "Live logs disconnected. Retrying…");
182 retry = setTimeout(connect, Math.min(30, 2 ** failures++) * 1000);
183 };
184 };
185 connect();
186 onCleanup(() => {
187 disposed = true;
188 clearTimeout(retry);
189 clearTimeout(stale);
190 socket.close();
191 });
192 });
130 const marked = createMemo(() => (at() === null ? undefined : all().find((line) => line.t > at()!)));193 const marked = createMemo(() => (at() === null ? undefined : all().find((line) => line.t > at()!)));
131 const hints = () => [194 const hints = () => [
132 ...props.service.containers.filter((container) => !container.hook && props.service.containers.length > 1).slice(0, 1)195 ...props.service.containers.filter((container) => !container.hook && props.service.containers.length > 1).slice(0, 1)
133 .map((container) => [`container:${container.name}`, "one container's lines"] as const),196 .map((container) => [`container:${container.name}`, "one container's lines"] as const),
134 ["stream:stderr", "error output only"],197 ["stream:stderr", "error output only"],
198 ["stream:event", "starts, restarts, and OOM kills"],
135 ["-health", "hide lines with a word"],199 ["-health", "hide lines with a word"],
136 ['"connection refused"', "an exact phrase"],200 ['"connection refused"', "an exact phrase"],
137 ] as const;201 ] as const;
...@@ -155,7 +219,7 @@ function Logs(props: { service: ServiceDetail; tools: HTMLElement }) {...@@ -155,7 +219,7 @@ function Logs(props: { service: ServiceDetail; tools: HTMLElement }) {
155 <label>to <input type="datetime-local" value={dateValue(params.to)}219 <label>to <input type="datetime-local" value={dateValue(params.to)}
156 onChange={(event) => setParams({ to: event.currentTarget.value ? String(Date.parse(event.currentTarget.value) / 1000) : undefined }, { replace: true })} /></label>220 onChange={(event) => setParams({ to: event.currentTarget.value ? String(Date.parse(event.currentTarget.value) / 1000) : undefined }, { replace: true })} /></label>
157 <Show when={props.service.logs}>{(link) => <OpenApp app={link().app} href={link().url}>{link().app.name}</OpenApp>}</Show>221 <Show when={props.service.logs}>{(link) => <OpenApp app={link().app} href={link().url}>{link().app.name}</OpenApp>}</Show>
158 <Liveness every="lines every 2 s" paused={bounded() ? "Showing the selected time range" : undefined}222 <Liveness every="lines as they arrive" connecting={!connected()} paused={bounded() ? "Showing the selected time range" : undefined}
159 failure={stalled() ?? (first.state === "errored" ? reason(first.error) : undefined)} />223 failure={stalled() ?? (first.state === "errored" ? reason(first.error) : undefined)} />
160 </Filters>224 </Filters>
161 </Portal>225 </Portal>
...@@ -178,10 +242,10 @@ function Logs(props: { service: ServiceDetail; tools: HTMLElement }) {...@@ -178,10 +242,10 @@ function Logs(props: { service: ServiceDetail; tools: HTMLElement }) {
178 onStart={loadOlder}242 onStart={loadOlder}
179 onContainer={(name) => setParams({ q: [params.q, `container:${name}`].filter(Boolean).join(" ") }, { replace: true })}243 onContainer={(name) => setParams({ q: [params.q, `container:${name}`].filter(Boolean).join(" ") }, { replace: true })}
180 onFollow={(end) => {244 onFollow={(end) => {
181 setFollow(end && at() === null);245 if (end && !bounded()) resume();
182 if (end) setUnseen(0);246 else setFollow(false);
183 }}247 }}
184 onLatest={() => (at() === null ? setFollow(true) : setParams({ at: undefined }))}248 onLatest={() => (at() === null ? resume() : setParams({ at: undefined }))}
185 before={(line, index) => (249 before={(line, index) => (
186 <>250 <>
187 <Show when={index() === 0}>251 <Show when={index() === 0}>
...@@ -387,12 +451,12 @@ function Traces(props: { service: ServiceDetail; tools: HTMLElement }) {...@@ -387,12 +451,12 @@ function Traces(props: { service: ServiceDetail; tools: HTMLElement }) {
387}451}
388452
389/** Whether a tab still takes in new data; `paused` and `failure` say why it doesn't. */453/** Whether a tab still takes in new data; `paused` and `failure` say why it doesn't. */
390function Liveness(props: { every: string; paused?: string; failure?: string }) {454function Liveness(props: { every: string; paused?: string; failure?: string; connecting?: boolean }) {
391 const health = () => (props.failure ? "degraded" : props.paused ? "stopped" : "healthy");455 const health = () => (props.failure ? "degraded" : props.paused || props.connecting ? "stopped" : "healthy");
392 return (456 return (
393 <span class="liveness" classList={{ live: health() === "healthy" }} role="status" tabIndex={0}457 <span class="liveness" classList={{ live: health() === "healthy" }} role="status" tabIndex={0}
394 data-tip={props.failure ?? props.paused ?? `Checks for new ${props.every}`}>458 data-tip={props.failure ?? props.paused ?? (props.connecting ? "Connecting to live logs…" : `New ${props.every}`)}>
395 <StatusLabel health={health()}>{props.failure ? "reconnecting" : props.paused ? "paused" : "live"}</StatusLabel>459 <StatusLabel health={health()}>{props.failure ? props.paused ? "unavailable" : "reconnecting" : props.paused ? "paused" : props.connecting ? "connecting" : "live"}</StatusLabel>
396 </span>460 </span>
397 );461 );
398}462}
dashboard/web/types/model.ts+5-1
...@@ -206,11 +206,15 @@ export interface TraceSummary {...@@ -206,11 +206,15 @@ export interface TraceSummary {
206}206}
207207
208export interface LogLine {208export interface LogLine {
209 id?: string;
210 sequence?: number;
209 t: number;211 t: number;
210 container: string;212 container: string;
211 stream: "stdout" | "stderr";213 stream: "stdout" | "stderr" | "event";
212 level: "debug" | "info" | "warn" | "error" | null;214 level: "debug" | "info" | "warn" | "error" | null;
213 text: string;215 text: string;
216 event?: string | null;
217 allocation?: string | null;
214}218}
215219
216export interface HostInfo {220export interface HostInfo {
nixos/dashboard.nix+1-1
...@@ -24,7 +24,7 @@ let...@@ -24,7 +24,7 @@ let
24 inherit (finalAttrs) pname version src sourceRoot;24 inherit (finalAttrs) pname version src sourceRoot;
25 pnpm = pnpm_10;25 pnpm = pnpm_10;
26 fetcherVersion = 3;26 fetcherVersion = 3;
27 hash = "sha256-zux7++pfJ9YJAMPjnvW94Z2m884Ypnc6BK9iHJxRFnA=";27 hash = "sha256-Jhk4mAOCqoljelFJfC7o+3t66x9GPFYb2e928jeaPSk=";
28 };28 };
29 nativeBuildInputs = [ nodejs_24 pnpm_10 pnpmConfigHook ];29 nativeBuildInputs = [ nodejs_24 pnpm_10 pnpmConfigHook ];
30 buildPhase = "pnpm run build";30 buildPhase = "pnpm run build";
tools/dashboard-logs-test.py created+270
...@@ -0,0 +1,270 @@
1#!/usr/bin/env python3
2import argparse
3import base64
4from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
5import json
6import os
7from pathlib import Path
8import socket
9import struct
10import re
11import sqlite3
12import hashlib
13import subprocess
14import tempfile
15import threading
16import time
17from urllib.parse import parse_qs, urlsplit
18from urllib.request import Request, urlopen
19from urllib.error import HTTPError
20
21
22def main():
23 parser = argparse.ArgumentParser()
24 parser.add_argument('--preview', action='store_true')
25 parser.add_argument('--binary', type=Path, default=Path('dashboard/target/debug/home-dashboard'))
26 args = parser.parse_args()
27 forms = []
28 failed = threading.Event()
29 row = {'_time': '2026-10-14T02:50:00Z', '_msg': 'Out of memory (OOM): exit 137',
30 'id': 'allocation:app:1:Terminated', 'task': 'app', 'stream': 'event', 'event': 'OOM',
31 'level': 'error', 'allocation': 'allocation', 'job': 'shale', 'source': 'nomad', 'sequence': 1}
32 rows = [row]
33 checkpoint = 0
34
35 class Fixture(BaseHTTPRequestHandler):
36 protocol_version = 'HTTP/1.1'
37
38 def handle(self):
39 try:
40 super().handle()
41 except (ConnectionResetError, BrokenPipeError):
42 pass
43
44 def do_GET(self):
45 if urlsplit(self.path).path == '/select/logsql/query':
46 self.query(parse_qs(urlsplit(self.path).query))
47 else:
48 self.reply([{'Address': '127.0.0.1', 'Port': self.server.server_port}])
49
50 def reply(self, value):
51 data = json.dumps(value).encode()
52 self.send_response(200)
53 self.send_header('Content-Length', str(len(data)))
54 self.end_headers()
55 self.wfile.write(data)
56
57 def do_POST(self):
58 form = parse_qs(self.rfile.read(int(self.headers['Content-Length'])).decode())
59 self.query(form)
60
61 def query(self, form):
62 forms.append((self.path, form))
63 if failed.is_set():
64 self.send_error(503)
65 return
66 if 'source:=log-shipper' in form['query'][0]:
67 if checkpoint == 0:
68 self.send_response(200)
69 self.send_header('Content-Length', '0')
70 self.end_headers()
71 else:
72 self.reply({'sequence': checkpoint if checkpoint is not None else max(r['sequence'] for r in rows)})
73 return
74 selected = rows[:]
75 match = re.search(r'sequence:>([0-9]+)', form['query'][0])
76 if match:
77 selected = [r for r in selected if r['sequence'] > int(match[1])]
78 match = re.search(r'sequence:<=(\d+)', form['query'][0])
79 if match:
80 selected = [r for r in selected if r['sequence'] <= int(match[1])]
81 if '| sort by (sequence)' in form['query'][0]:
82 selected.sort(key=lambda r: r['sequence'])
83 elif '| sort by (_time desc' in form['query'][0]:
84 selected.sort(key=lambda r: (r['_time'], r['id']), reverse=True)
85 offset = int(re.search(r'offset (\d+)', form['query'][0])[1])
86 limit = int(re.search(r'limit (\d+)', form['query'][0])[1])
87 selected = selected[offset:offset + limit]
88 self.send_response(200)
89 data = b''.join((json.dumps(r) + '\n').encode() for r in selected)
90 self.send_header('Content-Length', str(len(data)))
91 self.end_headers()
92 self.wfile.write(data[:17])
93 self.wfile.flush()
94 try:
95 self.wfile.write(data[17:])
96 self.wfile.flush()
97 except (BrokenPipeError, ConnectionResetError):
98 pass
99
100 def log_message(self, *args):
101 pass
102
103 with tempfile.TemporaryDirectory(prefix='dashboard-logs-') as temporary, ThreadingHTTPServer(('127.0.0.1', 0), Fixture) as fixture:
104 fixture.daemon_threads = True
105 threading.Thread(target=fixture.serve_forever, daemon=True).start()
106 root = Path(temporary).resolve()
107 (root / 'data').mkdir()
108 (root / 'token').write_text('disposable')
109 with socket.socket() as reservation:
110 reservation.bind(('127.0.0.1', 0))
111 port = reservation.getsockname()[1]
112 origin = f'http://127.0.0.1:{port}'
113 environment = {**os.environ, 'PORT': str(port), 'SNOWGLOBE_DATA_DIR': str(root / 'data'),
114 'SNOWGLOBE_PUBLIC_ORIGIN': 'https://globe.studio.test', 'SNOWGLOBE_NOMAD_TOKEN_FILE': str(root / 'token'),
115 'NOMAD_ADDR': f'http://127.0.0.1:{fixture.server_port}'}
116 environment['SNOWGLOBE_PREVIEW'] = '1' if args.preview else '0'
117 for name in ['SNOWGLOBE_PROXY_TOKEN_FILE', 'SNOWGLOBE_INTERNAL_URL', 'SNOWGLOBE_AUTH_REQUIRED', 'SNOWGLOBE_INDEX_POOL', 'SNOWGLOBE_YT_STATE']:
118 environment.pop(name, None)
119 with (root / 'server.log').open('w') as log:
120 server = subprocess.Popen([str(args.binary.resolve())], env=environment, stdout=log, stderr=log)
121 try:
122 deadline = time.monotonic() + 10
123 while True:
124 try:
125 with socket.create_connection(('127.0.0.1', port), timeout=.1): break
126 except OSError:
127 assert server.poll() is None, (root / 'server.log').read_text()
128 assert time.monotonic() < deadline
129 time.sleep(.05)
130 path = '/api/services/shale/logs'
131 with urlopen(Request(origin + path + '?level=error&q=stream:event', headers={'User-Name': 'fixture', 'User-Groups': 'infra-admin'})) as response:
132 lines = json.load(response)
133 assert lines[0]['stream'] == 'event' and lines[0]['event'] == 'OOM'
134 assert lines[0]['level'] == 'error' and lines[0]['id'] == row['id']
135 assert 'level:=error' in forms[-1][1]['query'][0]
136 assert 'stream:~"(?i)^event"' in forms[-1][1]['query'][0]
137 try:
138 urlopen(Request(origin + path + '?after=2&before=1', headers={'User-Name': 'fixture', 'User-Groups': 'infra-admin'}))
139 raise AssertionError('reversed bounds accepted')
140 except HTTPError as error:
141 assert error.code == 400 and b'at or after' in error.read()
142
143 def connect(expected, supplied_origin=origin, groups='infra-admin', actor='fixture', cursor=0, cookie=None):
144 sock = socket.create_connection(('127.0.0.1', port), timeout=5)
145 sock.settimeout(20)
146 stream = sock.makefile('rb')
147 headers = {'Host': urlsplit(origin).netloc, 'Upgrade': 'websocket', 'Connection': 'Upgrade',
148 'Sec-WebSocket-Version': '13', 'Sec-WebSocket-Key': base64.b64encode(os.urandom(16)).decode()}
149 if actor is not None: headers['User-Name'] = actor
150 if cookie is not None: headers['Cookie'] = '__Host-snow-session=' + cookie
151 if groups is not None: headers['User-Groups'] = groups
152 if supplied_origin is not None: headers['Origin'] = supplied_origin
153 sock.sendall(('GET ' + path + f'/stream?q=stream:event&cursor={cursor}&from=1 HTTP/1.1\r\n' + ''.join(f'{k}: {v}\r\n' for k, v in headers.items()) + '\r\n').encode())
154 status = int(stream.readline().split()[1])
155 assert status == expected, status
156 while stream.readline().strip(): pass
157 if expected != 101:
158 stream.close()
159 sock.close()
160 return sock, stream
161
162 connect(401, actor=None)
163 connect(403, groups='metrics')
164 connect(403, supplied_origin=None)
165 connect(403, supplied_origin='http://other.invalid')
166 sock, stream = connect(101)
167 try:
168 def receive():
169 head = stream.read(2)
170 assert len(head) == 2, 'stream closed unexpectedly'
171 size = head[1] & 127
172 if size == 126: size = struct.unpack('!H', stream.read(2))[0]
173 if size == 127: size = struct.unpack('!Q', stream.read(8))[0]
174 if head[0] & 15 == 9:
175 payload = stream.read(size)
176 mask = os.urandom(4)
177 sock.sendall(bytes([138, 128 | size]) + mask + bytes(v ^ mask[i % 4] for i, v in enumerate(payload)))
178 return receive()
179 assert head[0] & 15 == 1, head
180 return json.loads(stream.read(size))
181 message = receive()
182 assert message == {'lines': [], 'cursor': 0}
183 checkpoint = None
184 while not (message := receive()).get('lines'): pass
185 assert message['lines'] == lines and message['cursor'] == 1
186 assert forms[-1][1]['start'] == ['1']
187 rows.append({**row, 'id': 'same-time', 'sequence': 2})
188 while not (message := receive()).get('lines'): pass
189 assert message['lines'][0]['id'] == 'same-time' and message['cursor'] == 2
190 rows.append({**row, '_time': '2026-10-14T01:00:00Z', 'id': 'late-oom', 'sequence': 3})
191 while not (message := receive()).get('lines'): pass
192 assert message['lines'][0]['id'] == 'late-oom' and message['cursor'] == 3
193 failed.set()
194 while not (message := receive()).get('error'): pass
195 assert 'store' in message['error']
196 finally:
197 stream.close()
198 sock.close()
199 failed.clear()
200 sock, stream = connect(101, cursor=2)
201 try:
202 while not (message := receive()).get('lines'): pass
203 assert [r['id'] for r in message['lines']] == ['late-oom']
204 finally:
205 stream.close()
206 sock.close()
207 bulk = [{**row, 'id': f'bulk-{i}', 'sequence': i + 4, '_msg': 'x' * 3500} for i in range(3000)]
208 rows.extend(bulk)
209 sock, stream = connect(101, cursor=3)
210 try:
211 received = []
212 while len(received) < 3000:
213 message = receive()
214 assert not message.get('error'), message
215 received.extend(r['id'] for r in message['lines'])
216 assert received == [r['id'] for r in bulk]
217 finally:
218 stream.close()
219 sock.close()
220 accepted = time.time()
221 sequence = int(accepted * 1_000_000)
222 delayed = {**row, 'id': 'delayed-search-visibility', 'sequence': sequence}
223 checkpoint = 3003
224 rows.append({**row, 'id': 'initial-unsettled', 'sequence': sequence + 1})
225 def publish():
226 nonlocal checkpoint
227 rows.append(delayed)
228 checkpoint = sequence + 1
229 threading.Timer(7, publish).start()
230 sock, stream = connect(101, cursor=sequence + 1)
231 try:
232 while not (message := receive()).get('lines'): pass
233 assert [r['id'] for r in message['lines']] == [delayed['id'], 'initial-unsettled']
234 assert time.time() - accepted >= 7
235 finally:
236 stream.close()
237 sock.close()
238 rows[:] = [{**row, 'id': f'tie-{i:05}', 'sequence': i + 1} for i in range(6000)] + [{**row, '_time': '2026-10-13T00:00:00Z', 'id': 'older-history', 'sequence': 6001}]
239 history = []
240 for offset in (0, 5000):
241 with urlopen(Request(origin + path + f'?limit=5000&offset={offset}', headers={'User-Name': 'fixture', 'User-Groups': 'infra-admin'})) as response:
242 history.extend(json.load(response))
243 assert len({line['id'] for line in history}) == 6001
244 assert history[-1]['id'] == 'older-history'
245 cookie = 'disposable-native-session'
246 with sqlite3.connect(root / 'data/accounts.sqlite') as db:
247 db.execute('INSERT INTO users VALUES (?,?)', ('fixture', json.dumps({'username':'fixture','enabled':True,'requiredActions':[]})))
248 role = db.execute('SELECT id FROM roles WHERE name=?', ('infra-admin',)).fetchone()[0]
249 db.execute('INSERT INTO memberships VALUES (?,?)', ('fixture', role))
250 db.execute('INSERT INTO sessions VALUES (?,?,?,?,?,?,?,?)', (hashlib.sha256(cookie.encode()).hexdigest(),'fixture','dashboard',int(time.time())+120,'127.0.0.1',0,0,0))
251 connect(401)
252 sock, stream = connect(101, cursor=sequence, cookie=cookie)
253 try:
254 while 'lines' not in (message := receive()): pass
255 with sqlite3.connect(root / 'data/accounts.sqlite') as db:
256 db.execute('DELETE FROM sessions')
257 while not (message := receive()).get('error'): pass
258 assert 'administrator' in message['error']
259 finally:
260 stream.close()
261 sock.close()
262 print('REST events/filters, WebSocket auth/Origin, fragmented responses, timestamp ties, delayed OOM, reconnect cursor, oversized backlog, delayed visibility, session revocation, upstream failure: passed')
263 finally:
264 server.terminate()
265 server.wait(timeout=10)
266 fixture.shutdown()
267
268
269if __name__ == '__main__':
270 main()
tools/log-shipper-test.py created+137
...@@ -0,0 +1,137 @@
1#!/usr/bin/env python3
2import importlib.util
3import io
4import json
5from pathlib import Path
6import tempfile
7import unittest
8from unittest.mock import patch
9
10
11spec = importlib.util.spec_from_file_location("shipper", Path(__file__).with_name("log-shipper.py"))
12shipper = importlib.util.module_from_spec(spec)
13spec.loader.exec_module(shipper)
14
15
16class Journal:
17 def __init__(self, *args, **kwargs):
18 self.stdout = io.StringIO("")
19 self.returncode = 0
20
21 def wait(self):
22 return self.returncode
23
24
25class Shipping(unittest.TestCase):
26 def setUp(self):
27 self.temporary = tempfile.TemporaryDirectory()
28 self.addCleanup(self.temporary.cleanup)
29 root = Path(self.temporary.name)
30 self.log = root / "alloc" / "allocation" / "alloc/logs/app.stdout.0"
31 self.log.parent.mkdir(parents=True)
32 self.log.write_bytes(b"2026-10-14T02:50:00Z stderr F before crash\n")
33 self.allocations = [{"ID": "allocation", "JobID": "shale", "TaskStates": {"app": {"Events": [
34 {"Time": 1791946241000000000, "Type": "Terminated", "ExitCode": 137,
35 "DisplayMessage": "Exit Code: 137", "Details": {"oom_killed": "true"}},
36 {"Time": 1791946242000000000, "Type": "Restarting", "DisplayMessage": "Task restarting in 17s"},
37 {"Time": 1791946265000000000, "Type": "Started", "DisplayMessage": "Task started by client"},
38 ]}}}]
39 self.batches = []
40 self.checkpoints = []
41 self.visible = None
42 replacement = patch.object(shipper, "retention_bounds", return_value=(0, float("inf")))
43 replacement.start()
44 self.addCleanup(replacement.stop)
45 replacement = patch.object(shipper, "urlopen", side_effect=lambda *a, **k: io.BytesIO(json.dumps({"rows": self.visible if self.visible is not None else len(self.batches) - (self.checkpoints[-1] if self.checkpoints else 0)}).encode()))
46 replacement.start()
47 self.addCleanup(replacement.stop)
48 for target, value in [("LOGS", root / "alloc"), ("STATE", root / "state.json")]:
49 replacement = patch.object(shipper, target, value)
50 replacement.start()
51 self.addCleanup(replacement.stop)
52 replacement = patch.object(shipper, "nomad", side_effect=lambda path: self.allocations if path == "/v1/allocations" else [{"Address": "127.0.0.1", "Port": 1}])
53 replacement.start()
54 self.addCleanup(replacement.stop)
55 replacement = patch.object(shipper.subprocess, "Popen", Journal)
56 replacement.start()
57 self.addCleanup(replacement.stop)
58
59 def once(self):
60 state = json.loads(shipper.STATE.read_text()) if shipper.STATE.exists() else {}
61 def deliver(base, rows, fields):
62 if rows and rows[0]["source"] == "log-shipper":
63 self.checkpoints.append(len(self.batches))
64 else:
65 self.batches.extend(rows)
66 with patch.object(shipper, "send", side_effect=deliver):
67 shipper.once(state)
68
69 def test_oom_and_restarts_are_retained_once_across_process_restarts_and_allocation_gc(self):
70 self.once()
71 self.assertEqual([row.get("event") for row in self.batches], [None, "OOM", "Restarting", "Started"])
72 self.assertEqual(self.batches[1]["level"], "error")
73 self.assertIn("Out of memory (OOM)", self.batches[1]["_msg"])
74 self.assertEqual(self.batches[2]["level"], "warn")
75 self.assertEqual(len({row["id"] for row in self.batches}), 4)
76 self.assertEqual(len({row["sequence"] for row in self.batches}), 4)
77 self.once()
78 self.assertEqual(len(self.batches), 4)
79 self.allocations.clear()
80 self.once()
81 self.assertEqual(json.loads(shipper.STATE.read_text())["events"], {})
82 self.assertEqual(len(self.batches), 4)
83
84 def test_same_timestamp_lines_and_split_writes_keep_distinct_ids(self):
85 self.once()
86 line = b"2026-10-14T02:50:01Z stdout F same message"
87 with self.log.open("ab") as file:
88 file.write(line)
89 self.once()
90 self.assertEqual(len(self.batches), 4)
91 with self.log.open("ab") as file:
92 file.write(b"\n" + line + b"\n")
93 self.once()
94 self.assertEqual(len(self.batches), 6)
95 self.assertNotEqual(self.batches[-1]["id"], self.batches[-2]["id"])
96 self.assertGreater(self.batches[-2]["sequence"], self.batches[-3]["sequence"])
97 self.assertGreater(self.batches[-1]["sequence"], self.batches[-2]["sequence"])
98
99 def test_failed_delivery_does_not_advance_file_or_event_cursors(self):
100 with patch.object(shipper, "send", side_effect=OSError("store unavailable")):
101 with self.assertRaises(OSError):
102 shipper.once({})
103 self.assertNotIn("events", json.loads(shipper.STATE.read_text()))
104 self.assertFalse(json.loads(shipper.STATE.read_text()).get("files"))
105 self.once()
106 self.assertEqual(len(self.batches), 4)
107
108 def test_expired_rows_advance_source_cursors_without_blocking_checkpoint(self):
109 with patch.object(shipper, "retention_bounds", return_value=(2_000_000_000, 3_000_000_000)):
110 self.once()
111 self.assertEqual(self.batches, [])
112 self.assertEqual(self.checkpoints, [])
113 self.assertTrue(json.loads(shipper.STATE.read_text())["events"])
114 self.once()
115 self.assertEqual(self.batches, [])
116
117 def test_invisible_rows_do_not_commit_checkpoint_or_offsets(self):
118 self.visible = 3
119 with patch.object(shipper.time, "monotonic", side_effect=[0, 31]):
120 with self.assertRaises(TimeoutError):
121 self.once()
122 self.assertEqual(self.checkpoints, [])
123 reserved = json.loads(shipper.STATE.read_text())
124 self.assertFalse(reserved.get("files"))
125 self.assertNotIn("events", reserved)
126 self.visible = 4
127 self.log.unlink()
128 self.allocations.clear()
129 with patch.object(shipper.time, "time_ns", return_value=0):
130 self.once()
131 self.assertEqual(self.batches[-1]["sequence"], reserved["sequence"])
132 self.assertEqual(self.batches[:4], self.batches[-4:])
133 self.assertEqual(len(self.checkpoints), 1)
134
135
136if __name__ == "__main__":
137 unittest.main()
tools/log-shipper.py+123-35
...@@ -1,11 +1,13 @@...@@ -1,11 +1,13 @@
1#!/usr/bin/env python31#!/usr/bin/env python3
2import datetime2import datetime
3import json3import json
4import math
4import os5import os
5from pathlib import Path6from pathlib import Path
6import re7import re
7import subprocess8import subprocess
8import time9import time
10from urllib.parse import urlencode
9from urllib.request import Request, urlopen11from urllib.request import Request, urlopen
1012
1113
...@@ -31,44 +33,132 @@ def send(base, rows, fields):...@@ -31,44 +33,132 @@ def send(base, rows, fields):
31 response.read()33 response.read()
3234
3335
36def retention_bounds(base):
37 with urlopen(base + "/metrics", timeout=10) as response:
38 metrics = response.read().decode()
39 flags = dict(re.findall(r'flag\{name="(retentionPeriod|futureRetention|maxBackfillAge)", value="([^"]+)"', metrics))
40 durations = {}
41 units = {"ms": .001, "s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800, "y": 31536000}
42 for name in ("retentionPeriod", "futureRetention", "maxBackfillAge"):
43 value = flags[name]
44 if re.fullmatch(r"[0-9.]+M?", value):
45 seconds = float(value.rstrip("M")) * 2678400
46 else:
47 parts = re.findall(r"([0-9.]+)(ms|[smhdwy])", value.lower())
48 if "".join(number + unit for number, unit in parts) != value.lower():
49 raise ValueError("unsupported log retention duration")
50 seconds = sum(float(number) * units[unit] for number, unit in parts)
51 if not math.isfinite(seconds) or seconds < 0:
52 raise ValueError("invalid log retention duration")
53 durations[name] = int(seconds * 1000) / 1000
54 retention = max(86400, durations["retentionPeriod"])
55 backfill = durations["maxBackfillAge"]
56 backfill = backfill if 0 < backfill <= retention else retention
57 current = time.time()
58 return current - backfill, (math.floor((current + max(86400, durations["futureRetention"])) / 86400) + 1) * 86400
59
60
61def save(state):
62 pending_state = STATE.with_suffix(".pending")
63 with os.fdopen(os.open(pending_state, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), "w") as file:
64 json.dump(state, file)
65 os.replace(pending_state, STATE)
66
67
34def once(state):68def once(state):
35 services = nomad("/v1/service/victoria-logs")69 services = nomad("/v1/service/victoria-logs")
36 if not services:70 if not services:
37 return71 return
38 target = services[0]72 target = services[0]
39 base = f"http://{target['Address']}:{target['Port']}"73 base = f"http://{target['Address']}:{target['Port']}"
40 jobs = {item["ID"]: item["JobID"] for item in nomad("/v1/allocations")}74 if "pending" not in state:
41 offsets = state.setdefault("files", {})75 allocations = nomad("/v1/allocations")
42 rows = []76 jobs = {item["ID"]: item["JobID"] for item in allocations}
43 pending = {}77 offsets = state.get("files", {}).copy()
44 for file in sorted(LOGS.glob("*/alloc/logs/*")):78 rows = []
45 match = re.fullmatch(r"(.+)\.(stdout|stderr)\.\d+", file.name)79 pending = {}
46 job = jobs.get(file.parent.parent.parent.name)80 for file in sorted(LOGS.glob("*/alloc/logs/*")):
47 if not match or not job:81 match = re.fullmatch(r"(.+)\.(stdout|stderr)\.\d+", file.name)
48 continue82 job = jobs.get(file.parent.parent.parent.name)
49 info = file.stat()83 if not match or not job:
50 old = offsets.get(str(file), {})
51 offset = old.get("offset", 0) if old.get("inode") == info.st_ino and old.get("offset", 0) <= info.st_size else 0
52 with file.open("rb") as stream:
53 stream.seek(offset)
54 chunk = stream.read(262144)
55 if not chunk:
56 continue84 continue
57 if not chunk.endswith(b"\n"):85 info = file.stat()
58 chunk = chunk[:chunk.rfind(b"\n") + 1]86 old = offsets.get(str(file), {})
59 if not chunk:87 offset = old.get("offset", 0) if old.get("inode") == info.st_ino and old.get("offset", 0) <= info.st_size else 0
60 continue88 with file.open("rb") as stream:
61 pending[str(file)] = {"inode": info.st_ino, "offset": offset + len(chunk)}89 stream.seek(offset)
62 for raw in chunk.decode("utf8", "replace").splitlines():90 chunk = stream.read(262144)
63 parsed = CRI.match(raw)91 if not chunk:
64 if not parsed:92 continue
65 continue93 if not chunk.endswith(b"\n"):
66 stamp, stream, message = parsed.groups()94 chunk = chunk[:chunk.rfind(b"\n") + 1]
67 level = "error" if re.search(r"\b(ERROR|ERR|FATAL|CRITICAL|PANIC)\b|level=(error|fatal)", message, re.I) else "warn" if re.search(r"\b(WARN|WARNING|WRN)\b|level=warn", message, re.I) else "info"95 if not chunk:
68 rows.append({"_time": stamp, "_msg": message, "source": "nomad", "job": job,96 continue
69 "task": match.group(1), "stream": stream, "level": level})97 pending[str(file)] = {"inode": info.st_ino, "offset": offset + len(chunk)}
70 send(base, rows, "source,job,task,stream")98 position = offset
71 offsets.update(pending)99 for raw in chunk.splitlines(keepends=True):
100 identity = f"{file.parent.parent.parent.name}:{file.name}:{info.st_ino}:{position}"
101 position += len(raw)
102 parsed = CRI.match(raw.decode("utf8", "replace").rstrip("\r\n"))
103 if not parsed:
104 continue
105 stamp, stream, message = parsed.groups()
106 level = "error" if re.search(r"\b(ERROR|ERR|FATAL|CRITICAL|PANIC)\b|level=(error|fatal)", message, re.I) else "warn" if re.search(r"\b(WARN|WARNING|WRN)\b|level=warn", message, re.I) else "info"
107 rows.append({"_time": stamp, "_msg": message, "source": "nomad", "job": job,
108 "task": match.group(1), "stream": stream, "level": level,
109 "id": identity, "allocation": file.parent.parent.parent.name})
110 events = {}
111 for allocation in allocations:
112 seen = set(state.get("events", {}).get(allocation["ID"], []))
113 current = events[allocation["ID"]] = []
114 for task, task_state in (allocation.get("TaskStates") or {}).items():
115 for event in task_state.get("Events") or []:
116 kind = event["Type"]
117 identity = f"{allocation['ID']}:{task}:{event['Time']}:{kind}"
118 current.append(identity)
119 if identity in seen:
120 continue
121 oom = str((event.get("Details") or {}).get("oom_killed", "")).lower() == "true"
122 message = event.get("DisplayMessage") or event.get("Message") or kind
123 if oom:
124 message = "Out of memory (OOM): " + message
125 stamp = datetime.datetime.fromtimestamp(event["Time"] / 1e9, datetime.timezone.utc).isoformat()
126 level = "error" if oom or event.get("FailsTask") or kind in {"Driver Failure", "Failed Validation"} else "warn" if kind in {"Restarting", "Not Restarting"} or kind == "Terminated" and event.get("ExitCode") else "info"
127 rows.append({"_time": stamp, "_msg": message, "source": "nomad", "job": allocation["JobID"],
128 "task": task, "stream": "event", "level": level, "event": "OOM" if oom else kind,
129 "id": identity, "allocation": allocation["ID"]})
130 sequence = max(state.get("sequence", 0), time.time_ns() // 1000)
131 for row in rows:
132 sequence += 1
133 row["sequence"] = sequence
134 offsets.update(pending)
135 state["sequence"] = sequence
136 state["pending"] = {"rows": rows, "files": offsets, "events": events}
137 save(state)
138 batch = state["pending"]
139 rows = batch["rows"]
140 sequence = state["sequence"]
141 if rows:
142 minimum, maximum = retention_bounds(base)
143 rows = [row for row in rows if minimum <= datetime.datetime.fromisoformat(row["_time"].replace("Z", "+00:00")).timestamp() < maximum]
144 if rows:
145 send(base, rows, "source,job,task,stream")
146 query = f"source:=nomad sequence:>={rows[0]['sequence']} sequence:<={sequence} | stats count_uniq(sequence) as rows"
147 deadline = time.monotonic() + 30
148 while True:
149 request = Request(base + "/select/logsql/query", data=urlencode({"query": query, "start": minimum, "end": maximum}).encode())
150 with urlopen(request, timeout=10) as response:
151 visible = json.loads(response.read() or b"{}")
152 if int(visible.get("rows", 0)) == len(rows):
153 break
154 if time.monotonic() >= deadline:
155 raise TimeoutError("log batch is not searchable yet")
156 time.sleep(.25)
157 send(base, [{"_msg": "searchable log checkpoint", "source": "log-shipper", "sequence": sequence}], "source")
158 state["files"] = {path: value for path, value in batch["files"].items() if Path(path).is_file()}
159 state["events"] = batch["events"]
160 del state["pending"]
161 save(state)
72162
73 command = ["journalctl", "--no-pager", "--output=json"]163 command = ["journalctl", "--no-pager", "--output=json"]
74 command += ["--after-cursor=" + state["cursor"]] if state.get("cursor") else ["--since=-1 hour"]164 command += ["--after-cursor=" + state["cursor"]] if state.get("cursor") else ["--since=-1 hour"]
...@@ -97,9 +187,7 @@ def once(state):...@@ -97,9 +187,7 @@ def once(state):
97 send(base, system, "source,unit")187 send(base, system, "source,unit")
98 if cursor:188 if cursor:
99 state["cursor"] = cursor189 state["cursor"] = cursor
100 pending_state = STATE.with_suffix(".pending")190 save(state)
101 pending_state.write_text(json.dumps(state))
102 os.replace(pending_state, STATE)
103191
104192
105def main():193def main():
...@@ -110,7 +198,7 @@ def main():...@@ -110,7 +198,7 @@ def main():
110 once(state)198 once(state)
111 except Exception as error:199 except Exception as error:
112 print(f"log shipping paused: {error}", flush=True)200 print(f"log shipping paused: {error}", flush=True)
113 time.sleep(5)201 time.sleep(1)
114202
115203
116if __name__ == "__main__":204if __name__ == "__main__":