authoravatarclover caruso<git@paperclover.net>2026-11-11 15:26:19-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-12 16:08:55-08:00
log9cd6ce8c7d9ad4395f6deb4ef710c060029c077f
tree1122679175f96e3ddd5ac542b9413b7ce17c0692
parent53fd9c969ec080da95877749caebd841358fe8c7
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Retain Substation transport incidents with TCP progress

A recurring 60-second failure received only 965720 of 2814027 upload bytes. Keep a restart-safe journal watcher and two minutes of TCP counters so the next failure identifies stalled sending, network loss or server backpressure. Record the API client endpoint and retain each edge failure with upload completion and surrounding connection counters in restricted state. Persist the journal cursor, bound incident retention, recover cursor gaps visibly, and run the watcher under systemd across deployments and reboots. Validated metadata filtering, connection selection, upload accounting, atomic state permissions, actual isolated systemd restart and cursor resume, API routing and privacy, release activation tests, deployment export tests, and healthy Substation staging. Assisted-by: gpt-6.1-sol

7 files changed, 239 insertions(+), 0 deletions(-)

nixos/configuration.nix+15
......@@ -150,6 +150,21 @@ in
150150 requires = [ "podman.socket" ];
151151 };
152152
153 systemd.services.snowglobe-substation-watch = {
154 wantedBy = [ "multi-user.target" ];
155 wants = [ "nomad.service" "caddy.service" ];
156 after = [ "nomad.service" "caddy.service" ];
157 unitConfig.ConditionPathExists = [ "/var/lib/snowglobe/nomad.token" "/opt/snowglobe/current/tools/substation-watch.py" ];
158 environment.SNOWGLOBE_DOMAIN = config.environment.variables.SNOWGLOBE_DOMAIN;
159 path = [ pkgs.systemd pkgs.iproute2 ];
160 serviceConfig = {
161 ExecStart = "${pkgs.python3}/bin/python3 /opt/snowglobe/current/tools/substation-watch.py";
162 Restart = "always";
163 RestartSec = 3;
164 UMask = "0077";
165 };
166 };
167
153168 systemd.services.snowglobe-router = {
154169 wantedBy = [ "multi-user.target" ];
155170 wants = [ "nomad.service" "caddy.service" ];
tools/release.py+2
......@@ -230,6 +230,8 @@ def activate(release, initial=False):
230230 pending.replace(managed)
231231 if (path / "tools/log-shipper.py").exists():
232232 subprocess.run(["systemctl", "restart", "snowglobe-log-shipper.service"], check=True)
233 if (path / "tools/substation-watch.py").exists():
234 subprocess.run(["systemctl", "restart", "snowglobe-substation-watch.service"], check=True)
233235 manifest = path / ".snowglobe-release.json"
234236 if json.loads(manifest.read_text()).get("version", 1) >= 2:
235237 subprocess.run(["python3", str(script), "check"], check=True, env=environment)
tools/router-api-test.py+2
......@@ -5,6 +5,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
55import json
66import os
77from pathlib import Path
8import re
89import socket
910import subprocess
1011import tempfile
......@@ -206,6 +207,7 @@ def main():
206207 time.sleep(.05)
207208 assert all("request" not in entry and "resp_headers" not in entry for entry in entries)
208209 assert all(entry.get("request_id") and entry.get("path", "").startswith(("/v1/", "/v0/management/")) for entry in entries)
210 assert all(re.fullmatch(r"127\.0\.0\.1:[0-9]+", entry["client"]) for entry in entries)
209211 assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200)
210212 assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200)
211213 assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499)
tools/router.py+1
......@@ -370,6 +370,7 @@ def render(token):
370370 f" log_append service {service}", " log_append request_id {http.request.uuid}",
371371 " log_append method {method}", " log_append path {path}", " log_append protocol {http.request.proto}",
372372 " log_append content_length {http.request.header.Content-Length}",
373 " log_append client {remote}",
373374 " log_append upstream {http.reverse_proxy.upstream.hostport}",
374375 " log_append upstream_status {http.reverse_proxy.status_code}",
375376 " log_append upstream_header_ms {http.reverse_proxy.upstream.latency_ms}",
tools/substation-watch-test.py created+65
......@@ -0,0 +1,65 @@
1#!/usr/bin/env python3
2import importlib.util
3import json
4from pathlib import Path
5import tempfile
6import unittest
7
8
9spec = importlib.util.spec_from_file_location("watch", Path(__file__).with_name("substation-watch.py"))
10watch = importlib.util.module_from_spec(spec)
11spec.loader.exec_module(watch)
12
13
14class TransportEvidence(unittest.TestCase):
15 def test_filters_private_headers_bodies_and_other_services(self):
16 original = {"service": "substation", "status": 504, "request_id": "fixture", "content_length": "100", "bytes_read": 40,
17 "client": "192.0.2.1:1234", "request": {"headers": {"X-Api-Key": "private"}}, "body": "private"}
18 edge, client = watch.event({"MESSAGE": json.dumps(original)}, "substation.example")
19 self.assertEqual(client, "192.0.2.1:1234")
20 self.assertNotIn("private", json.dumps(edge))
21 self.assertEqual(watch.event({"MESSAGE": json.dumps({"service": "another"})}, "substation.example"), (None, None))
22 self.assertEqual(watch.event({"MESSAGE": "not json"}, "substation.example"), (None, None))
23 peer = {"request": {"host": "substation.example", "remote_ip": "2001:db8::1", "remote_port": "44300", "headers": {"Authorization": "private"}}}
24 self.assertEqual(watch.event({"MESSAGE": json.dumps(peer)}, "substation.example"), (None, "[2001:db8::1]:44300"))
25
26 def test_connection_selection_and_counters(self):
27 output = "\n".join([
28 "0 100 127.0.0.1:24000 127.0.0.1:45000 cubic bytes_received:40000 rtt:0.1/0.2 snd_wnd:0 ignored:private",
29 "0 0 192.0.2.5:443 192.0.2.1:1234 cubic bytes_received:41000 lastrcv:60000",
30 "0 0 192.0.2.5:443 192.0.2.2:1234 cubic bytes_received:99999",
31 "0 0 [::ffff:192.0.2.5]:443 [::ffff:192.0.2.1]:5678 cubic bytes_received:42000",
32 ])
33 rows = watch.connections(output, {24000}, {"192.0.2.1"})
34 self.assertEqual(len(rows), 3)
35 self.assertEqual(rows[0]["recv_q"], 0)
36 self.assertEqual(rows[0]["send_q"], 100)
37 self.assertEqual(rows[0]["snd_wnd"], "0")
38 self.assertEqual(rows[1]["lastrcv"], "60000")
39 self.assertNotIn("private", json.dumps(rows))
40 self.assertEqual(watch.endpoint("[2001:db8::1]:1234"), ("2001:db8::1", 1234))
41 self.assertEqual(watch.endpoint("[::ffff:192.0.2.1]:1234"), ("192.0.2.1", 1234))
42
43 def test_completion_and_window(self):
44 edge = {"ts": 200, "duration": 60, "content_length": "100", "bytes_read": 40}
45 samples = [{"ts": 130}, {"ts": 140}, {"ts": 195}, {"ts": 210}]
46 record = watch.incident(edge, samples)
47 self.assertFalse(record["upload_complete"])
48 self.assertEqual(record["tcp"], [{"ts": 140}, {"ts": 195}])
49 edge["bytes_read"] = 100
50 self.assertTrue(watch.incident(edge, samples)["upload_complete"])
51 edge["content_length"] = ""
52 self.assertIsNone(watch.incident(edge, samples)["upload_complete"])
53
54 def test_private_atomic_state_survives_reload(self):
55 with tempfile.TemporaryDirectory() as temporary:
56 path = Path(temporary) / "state.json"
57 watch.save(path, {"cursor": "first"})
58 watch.save(path, {"cursor": "second"})
59 self.assertEqual(json.loads(path.read_text()), {"cursor": "second"})
60 self.assertEqual(path.stat().st_mode & 0o777, 0o600)
61 self.assertFalse(path.with_suffix(".pending").exists())
62
63
64if __name__ == "__main__":
65 unittest.main()
tools/substation-watch.py created+141
......@@ -0,0 +1,141 @@
1#!/usr/bin/env python3
2from collections import deque
3import ipaddress
4import json
5import os
6from pathlib import Path
7import re
8import subprocess
9import time
10from urllib.request import Request, urlopen
11
12
13ROOT = Path("/var/lib/snowglobe/substation-watch")
14EDGE_FIELDS = ("ts", "status", "bytes_read", "content_length", "duration", "request_id",
15 "method", "path", "protocol", "upstream", "upstream_status", "upstream_header_ms", "client")
16TCP_FIELDS = {"bytes_sent", "bytes_acked", "bytes_received", "bytes_retrans", "lastsnd", "lastrcv", "lastack",
17 "rtt", "rto", "retrans", "snd_wnd", "rcv_wnd", "rcv_space", "rcv_ssthresh", "cwnd", "unacked"}
18
19
20def save(path, value):
21 pending = path.with_suffix(".pending")
22 with pending.open("w") as file:
23 json.dump(value, file, separators=(",", ":"))
24 pending.chmod(0o600)
25 pending.replace(path)
26
27
28def endpoint(value):
29 address, _, port = value.rpartition(":")
30 address = ipaddress.ip_address(address.strip("[]"))
31 if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped:
32 address = address.ipv4_mapped
33 return str(address), int(port)
34
35
36def connections(output, ports, clients):
37 rows = []
38 for line in output.splitlines():
39 parts = line.split()
40 if len(parts) < 4:
41 continue
42 local, peer = endpoint(parts[2]), endpoint(parts[3])
43 if not (local[1] in ports or peer[1] in ports or local[1] == 443 and peer[0] in clients):
44 continue
45 row = {"local": parts[2], "peer": parts[3], "recv_q": int(parts[0]), "send_q": int(parts[1])}
46 for part in parts[4:]:
47 key, _, value = part.partition(":")
48 if key in TCP_FIELDS and re.fullmatch(r"[0-9./]+", value):
49 row[key] = value
50 rows.append(row)
51 return rows
52
53
54def event(entry, host):
55 try:
56 message = json.loads(entry.get("MESSAGE", ""))
57 except (ValueError, TypeError):
58 return None, None
59 if not isinstance(message, dict):
60 return None, None
61 if message.get("service") == "substation":
62 return {key: message.get(key) for key in EDGE_FIELDS}, message.get("client")
63 request = message.get("request", {})
64 if request.get("host") == host and request.get("remote_ip"):
65 address = str(ipaddress.ip_address(request["remote_ip"]))
66 return None, f"[{address}]:{int(request['remote_port'])}"
67 return None, None
68
69
70def incident(edge, samples):
71 length = edge.get("content_length")
72 complete = int(length) == edge["bytes_read"] if str(length).isdigit() else None
73 return {"edge": edge, "upload_complete": complete,
74 "tcp": [sample for sample in samples if edge["ts"] - edge["duration"] - 5 <= sample["ts"] <= edge["ts"] + 5]}
75
76
77def main():
78 os.umask(0o077)
79 ROOT.mkdir(mode=0o700, parents=True, exist_ok=True)
80 state_file = ROOT / "state.json"
81 state = json.loads(state_file.read_text()) if state_file.exists() else {"clients": {}}
82 recent = ROOT / "recent.json"
83 samples = deque(json.loads(recent.read_text()) if recent.exists() and time.time() - recent.stat().st_mtime < 120 else [], maxlen=120)
84 ports = set()
85 refreshed = saved = reported = 0
86 host = "substation.ai." + os.environ["SNOWGLOBE_DOMAIN"]
87 while True:
88 try:
89 now = time.time()
90 if now - refreshed >= 10:
91 token = Path("/var/lib/snowglobe/nomad.token").read_text().strip()
92 request = Request("http://127.0.0.1:4646/v1/service/substation-hub", headers={"X-Nomad-Token": token})
93 with urlopen(request, timeout=5) as response:
94 ports = {item["Port"] for item in json.load(response)
95 if item.get("JobID") == "substation" and ipaddress.ip_address(item["Address"]).is_loopback}
96 refreshed = now
97 command = ["journalctl", "-u", "caddy", "--no-pager", "--output=json"]
98 command += ["--after-cursor=" + state["cursor"]] if state.get("cursor") else ["--since=-1 hour"]
99 journal = subprocess.run(command, capture_output=True, text=True, timeout=10)
100 if journal.returncode:
101 if "cursor" in journal.stderr.lower() and state.pop("cursor", None):
102 print(json.dumps({"event": "substation-watcher-journal-gap"}), flush=True)
103 journal.check_returncode()
104 for line in journal.stdout.splitlines():
105 entry = json.loads(line)
106 edge, client = event(entry, host)
107 if client:
108 state["clients"][endpoint(client)[0]] = now
109 if edge and edge.get("status", 0) >= 499 and re.fullmatch(r"[0-9a-f-]{36}", str(edge.get("request_id", ""))):
110 path = ROOT / (edge["request_id"] + ".json")
111 if not path.exists():
112 record = incident(edge, samples)
113 save(path, record)
114 print(json.dumps({"event": "substation-incident", "request_id": edge["request_id"],
115 "status": edge["status"], "upload_complete": record["upload_complete"],
116 "samples": len(record["tcp"])}), flush=True)
117 state["cursor"] = entry["__CURSOR"]
118 state["clients"] = {client: stamp for client, stamp in state["clients"].items() if now - stamp < 3600}
119 if ports:
120 filters = "( sport = :443 or " + " or ".join(f"sport = :{port} or dport = :{port}" for port in sorted(ports)) + " )"
121 result = subprocess.run(["ss", "-HtinO", "state", "established", filters], check=True, capture_output=True, text=True, timeout=5)
122 rows = connections(result.stdout, ports, state["clients"])
123 samples.append({"ts": now, "connections": rows})
124 save(state_file, state)
125 if now - saved >= 5:
126 save(recent, list(samples))
127 saved = now
128 old = sorted((p for p in ROOT.glob("*.json") if re.fullmatch(r"[0-9a-f-]{36}", p.stem)), key=lambda p: p.stat().st_mtime)
129 for path in old[:-100]:
130 path.unlink()
131 if now - reported >= 60:
132 print(json.dumps({"event": "substation-watcher", "ports": sorted(ports),
133 "clients": len(state["clients"]), "samples": len(samples)}), flush=True)
134 reported = now
135 except Exception as error:
136 print(json.dumps({"event": "substation-watcher-error", "type": type(error).__name__}), flush=True)
137 time.sleep(1)
138
139
140if __name__ == "__main__":
141 main()
tools/substation.md+13
......@@ -118,3 +118,16 @@ uploads. A missing length can mean a chunked request; it does not prove completi
118118A 200 followed by a truncated stream needs the edge byte count and the hub's
119119stream outcome. A successful probe does not establish that every generation
120120stream completed.
121
122`snowglobe-substation-watch.service` follows Caddy with a persisted journal
123cursor and samples TCP counters once per second for the hub and known API
124clients. It survives service restarts and reboots. The latest two minutes are
125retained in `/var/lib/snowglobe/substation-watch/recent.json`; each 499 or 5xx
126retains its request metadata and surrounding TCP progress in a private file
127named by request ID. The directory keeps the latest 100 incidents. No packet
128payloads, request bodies, cookies, or credentials are collected.
129
130Watcher heartbeat and incident summaries appear in its journal and Victoria
131Logs. `upload_complete: false` proves an incomplete fixed-length upload;
132the saved receive/send queues, byte counters, receive windows, and retransmits
133distinguish stalled sending from server backpressure and network loss.