authoravatarclover caruso<git@paperclover.net>2026-11-11 15:26:19-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-12 16:08:55-08:00
log9cd6ce8c7d9ad4395f6deb4ef710c060029c077f
tree1122679175f96e3ddd5ac542b9413b7ce17c0692
parent53fd9c969ec080da95877749caebd841358fe8c7
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Retain Substation transport incidents with TCP progress

A recurring 60-second failure received only 965720 of 2814027 upload bytes. Keep a restart-safe journal watcher and two minutes of TCP counters so the next failure identifies stalled sending, network loss or server backpressure. Record the API client endpoint and retain each edge failure with upload completion and surrounding connection counters in restricted state. Persist the journal cursor, bound incident retention, recover cursor gaps visibly, and run the watcher under systemd across deployments and reboots. Validated metadata filtering, connection selection, upload accounting, atomic state permissions, actual isolated systemd restart and cursor resume, API routing and privacy, release activation tests, deployment export tests, and healthy Substation staging. Assisted-by: gpt-6.1-sol

7 files changed, 239 insertions(+), 0 deletions(-)

nixos/configuration.nix+15
...@@ -150,6 +150,21 @@ in...@@ -150,6 +150,21 @@ in
150 requires = [ "podman.socket" ];150 requires = [ "podman.socket" ];
151 };151 };
152152
153 systemd.services.snowglobe-substation-watch = {
154 wantedBy = [ "multi-user.target" ];
155 wants = [ "nomad.service" "caddy.service" ];
156 after = [ "nomad.service" "caddy.service" ];
157 unitConfig.ConditionPathExists = [ "/var/lib/snowglobe/nomad.token" "/opt/snowglobe/current/tools/substation-watch.py" ];
158 environment.SNOWGLOBE_DOMAIN = config.environment.variables.SNOWGLOBE_DOMAIN;
159 path = [ pkgs.systemd pkgs.iproute2 ];
160 serviceConfig = {
161 ExecStart = "${pkgs.python3}/bin/python3 /opt/snowglobe/current/tools/substation-watch.py";
162 Restart = "always";
163 RestartSec = 3;
164 UMask = "0077";
165 };
166 };
167
153 systemd.services.snowglobe-router = {168 systemd.services.snowglobe-router = {
154 wantedBy = [ "multi-user.target" ];169 wantedBy = [ "multi-user.target" ];
155 wants = [ "nomad.service" "caddy.service" ];170 wants = [ "nomad.service" "caddy.service" ];
tools/release.py+2
...@@ -230,6 +230,8 @@ def activate(release, initial=False):...@@ -230,6 +230,8 @@ def activate(release, initial=False):
230 pending.replace(managed)230 pending.replace(managed)
231 if (path / "tools/log-shipper.py").exists():231 if (path / "tools/log-shipper.py").exists():
232 subprocess.run(["systemctl", "restart", "snowglobe-log-shipper.service"], check=True)232 subprocess.run(["systemctl", "restart", "snowglobe-log-shipper.service"], check=True)
233 if (path / "tools/substation-watch.py").exists():
234 subprocess.run(["systemctl", "restart", "snowglobe-substation-watch.service"], check=True)
233 manifest = path / ".snowglobe-release.json"235 manifest = path / ".snowglobe-release.json"
234 if json.loads(manifest.read_text()).get("version", 1) >= 2:236 if json.loads(manifest.read_text()).get("version", 1) >= 2:
235 subprocess.run(["python3", str(script), "check"], check=True, env=environment)237 subprocess.run(["python3", str(script), "check"], check=True, env=environment)
tools/router-api-test.py+2
...@@ -5,6 +5,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer...@@ -5,6 +5,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
5import json5import json
6import os6import os
7from pathlib import Path7from pathlib import Path
8import re
8import socket9import socket
9import subprocess10import subprocess
10import tempfile11import tempfile
...@@ -206,6 +207,7 @@ def main():...@@ -206,6 +207,7 @@ def main():
206 time.sleep(.05)207 time.sleep(.05)
207 assert all("request" not in entry and "resp_headers" not in entry for entry in entries)208 assert all("request" not in entry and "resp_headers" not in entry for entry in entries)
208 assert all(entry.get("request_id") and entry.get("path", "").startswith(("/v1/", "/v0/management/")) for entry in entries)209 assert all(entry.get("request_id") and entry.get("path", "").startswith(("/v1/", "/v0/management/")) for entry in entries)
210 assert all(re.fullmatch(r"127\.0\.0\.1:[0-9]+", entry["client"]) for entry in entries)
209 assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200)211 assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200)
210 assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200)212 assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200)
211 assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499)213 assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499)
tools/router.py+1
...@@ -370,6 +370,7 @@ def render(token):...@@ -370,6 +370,7 @@ def render(token):
370 f" log_append service {service}", " log_append request_id {http.request.uuid}",370 f" log_append service {service}", " log_append request_id {http.request.uuid}",
371 " log_append method {method}", " log_append path {path}", " log_append protocol {http.request.proto}",371 " log_append method {method}", " log_append path {path}", " log_append protocol {http.request.proto}",
372 " log_append content_length {http.request.header.Content-Length}",372 " log_append content_length {http.request.header.Content-Length}",
373 " log_append client {remote}",
373 " log_append upstream {http.reverse_proxy.upstream.hostport}",374 " log_append upstream {http.reverse_proxy.upstream.hostport}",
374 " log_append upstream_status {http.reverse_proxy.status_code}",375 " log_append upstream_status {http.reverse_proxy.status_code}",
375 " log_append upstream_header_ms {http.reverse_proxy.upstream.latency_ms}",376 " log_append upstream_header_ms {http.reverse_proxy.upstream.latency_ms}",
tools/substation-watch-test.py created+65
...@@ -0,0 +1,65 @@
1#!/usr/bin/env python3
2import importlib.util
3import json
4from pathlib import Path
5import tempfile
6import unittest
7
8
9spec = importlib.util.spec_from_file_location("watch", Path(__file__).with_name("substation-watch.py"))
10watch = importlib.util.module_from_spec(spec)
11spec.loader.exec_module(watch)
12
13
14class TransportEvidence(unittest.TestCase):
15 def test_filters_private_headers_bodies_and_other_services(self):
16 original = {"service": "substation", "status": 504, "request_id": "fixture", "content_length": "100", "bytes_read": 40,
17 "client": "192.0.2.1:1234", "request": {"headers": {"X-Api-Key": "private"}}, "body": "private"}
18 edge, client = watch.event({"MESSAGE": json.dumps(original)}, "substation.example")
19 self.assertEqual(client, "192.0.2.1:1234")
20 self.assertNotIn("private", json.dumps(edge))
21 self.assertEqual(watch.event({"MESSAGE": json.dumps({"service": "another"})}, "substation.example"), (None, None))
22 self.assertEqual(watch.event({"MESSAGE": "not json"}, "substation.example"), (None, None))
23 peer = {"request": {"host": "substation.example", "remote_ip": "2001:db8::1", "remote_port": "44300", "headers": {"Authorization": "private"}}}
24 self.assertEqual(watch.event({"MESSAGE": json.dumps(peer)}, "substation.example"), (None, "[2001:db8::1]:44300"))
25
26 def test_connection_selection_and_counters(self):
27 output = "\n".join([
28 "0 100 127.0.0.1:24000 127.0.0.1:45000 cubic bytes_received:40000 rtt:0.1/0.2 snd_wnd:0 ignored:private",
29 "0 0 192.0.2.5:443 192.0.2.1:1234 cubic bytes_received:41000 lastrcv:60000",
30 "0 0 192.0.2.5:443 192.0.2.2:1234 cubic bytes_received:99999",
31 "0 0 [::ffff:192.0.2.5]:443 [::ffff:192.0.2.1]:5678 cubic bytes_received:42000",
32 ])
33 rows = watch.connections(output, {24000}, {"192.0.2.1"})
34 self.assertEqual(len(rows), 3)
35 self.assertEqual(rows[0]["recv_q"], 0)
36 self.assertEqual(rows[0]["send_q"], 100)
37 self.assertEqual(rows[0]["snd_wnd"], "0")
38 self.assertEqual(rows[1]["lastrcv"], "60000")
39 self.assertNotIn("private", json.dumps(rows))
40 self.assertEqual(watch.endpoint("[2001:db8::1]:1234"), ("2001:db8::1", 1234))
41 self.assertEqual(watch.endpoint("[::ffff:192.0.2.1]:1234"), ("192.0.2.1", 1234))
42
43 def test_completion_and_window(self):
44 edge = {"ts": 200, "duration": 60, "content_length": "100", "bytes_read": 40}
45 samples = [{"ts": 130}, {"ts": 140}, {"ts": 195}, {"ts": 210}]
46 record = watch.incident(edge, samples)
47 self.assertFalse(record["upload_complete"])
48 self.assertEqual(record["tcp"], [{"ts": 140}, {"ts": 195}])
49 edge["bytes_read"] = 100
50 self.assertTrue(watch.incident(edge, samples)["upload_complete"])
51 edge["content_length"] = ""
52 self.assertIsNone(watch.incident(edge, samples)["upload_complete"])
53
54 def test_private_atomic_state_survives_reload(self):
55 with tempfile.TemporaryDirectory() as temporary:
56 path = Path(temporary) / "state.json"
57 watch.save(path, {"cursor": "first"})
58 watch.save(path, {"cursor": "second"})
59 self.assertEqual(json.loads(path.read_text()), {"cursor": "second"})
60 self.assertEqual(path.stat().st_mode & 0o777, 0o600)
61 self.assertFalse(path.with_suffix(".pending").exists())
62
63
64if __name__ == "__main__":
65 unittest.main()
tools/substation-watch.py created+141
...@@ -0,0 +1,141 @@
1#!/usr/bin/env python3
2from collections import deque
3import ipaddress
4import json
5import os
6from pathlib import Path
7import re
8import subprocess
9import time
10from urllib.request import Request, urlopen
11
12
13ROOT = Path("/var/lib/snowglobe/substation-watch")
14EDGE_FIELDS = ("ts", "status", "bytes_read", "content_length", "duration", "request_id",
15 "method", "path", "protocol", "upstream", "upstream_status", "upstream_header_ms", "client")
16TCP_FIELDS = {"bytes_sent", "bytes_acked", "bytes_received", "bytes_retrans", "lastsnd", "lastrcv", "lastack",
17 "rtt", "rto", "retrans", "snd_wnd", "rcv_wnd", "rcv_space", "rcv_ssthresh", "cwnd", "unacked"}
18
19
20def save(path, value):
21 pending = path.with_suffix(".pending")
22 with pending.open("w") as file:
23 json.dump(value, file, separators=(",", ":"))
24 pending.chmod(0o600)
25 pending.replace(path)
26
27
28def endpoint(value):
29 address, _, port = value.rpartition(":")
30 address = ipaddress.ip_address(address.strip("[]"))
31 if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped:
32 address = address.ipv4_mapped
33 return str(address), int(port)
34
35
36def connections(output, ports, clients):
37 rows = []
38 for line in output.splitlines():
39 parts = line.split()
40 if len(parts) < 4:
41 continue
42 local, peer = endpoint(parts[2]), endpoint(parts[3])
43 if not (local[1] in ports or peer[1] in ports or local[1] == 443 and peer[0] in clients):
44 continue
45 row = {"local": parts[2], "peer": parts[3], "recv_q": int(parts[0]), "send_q": int(parts[1])}
46 for part in parts[4:]:
47 key, _, value = part.partition(":")
48 if key in TCP_FIELDS and re.fullmatch(r"[0-9./]+", value):
49 row[key] = value
50 rows.append(row)
51 return rows
52
53
54def event(entry, host):
55 try:
56 message = json.loads(entry.get("MESSAGE", ""))
57 except (ValueError, TypeError):
58 return None, None
59 if not isinstance(message, dict):
60 return None, None
61 if message.get("service") == "substation":
62 return {key: message.get(key) for key in EDGE_FIELDS}, message.get("client")
63 request = message.get("request", {})
64 if request.get("host") == host and request.get("remote_ip"):
65 address = str(ipaddress.ip_address(request["remote_ip"]))
66 return None, f"[{address}]:{int(request['remote_port'])}"
67 return None, None
68
69
70def incident(edge, samples):
71 length = edge.get("content_length")
72 complete = int(length) == edge["bytes_read"] if str(length).isdigit() else None
73 return {"edge": edge, "upload_complete": complete,
74 "tcp": [sample for sample in samples if edge["ts"] - edge["duration"] - 5 <= sample["ts"] <= edge["ts"] + 5]}
75
76
77def main():
78 os.umask(0o077)
79 ROOT.mkdir(mode=0o700, parents=True, exist_ok=True)
80 state_file = ROOT / "state.json"
81 state = json.loads(state_file.read_text()) if state_file.exists() else {"clients": {}}
82 recent = ROOT / "recent.json"
83 samples = deque(json.loads(recent.read_text()) if recent.exists() and time.time() - recent.stat().st_mtime < 120 else [], maxlen=120)
84 ports = set()
85 refreshed = saved = reported = 0
86 host = "substation.ai." + os.environ["SNOWGLOBE_DOMAIN"]
87 while True:
88 try:
89 now = time.time()
90 if now - refreshed >= 10:
91 token = Path("/var/lib/snowglobe/nomad.token").read_text().strip()
92 request = Request("http://127.0.0.1:4646/v1/service/substation-hub", headers={"X-Nomad-Token": token})
93 with urlopen(request, timeout=5) as response:
94 ports = {item["Port"] for item in json.load(response)
95 if item.get("JobID") == "substation" and ipaddress.ip_address(item["Address"]).is_loopback}
96 refreshed = now
97 command = ["journalctl", "-u", "caddy", "--no-pager", "--output=json"]
98 command += ["--after-cursor=" + state["cursor"]] if state.get("cursor") else ["--since=-1 hour"]
99 journal = subprocess.run(command, capture_output=True, text=True, timeout=10)
100 if journal.returncode:
101 if "cursor" in journal.stderr.lower() and state.pop("cursor", None):
102 print(json.dumps({"event": "substation-watcher-journal-gap"}), flush=True)
103 journal.check_returncode()
104 for line in journal.stdout.splitlines():
105 entry = json.loads(line)
106 edge, client = event(entry, host)
107 if client:
108 state["clients"][endpoint(client)[0]] = now
109 if edge and edge.get("status", 0) >= 499 and re.fullmatch(r"[0-9a-f-]{36}", str(edge.get("request_id", ""))):
110 path = ROOT / (edge["request_id"] + ".json")
111 if not path.exists():
112 record = incident(edge, samples)
113 save(path, record)
114 print(json.dumps({"event": "substation-incident", "request_id": edge["request_id"],
115 "status": edge["status"], "upload_complete": record["upload_complete"],
116 "samples": len(record["tcp"])}), flush=True)
117 state["cursor"] = entry["__CURSOR"]
118 state["clients"] = {client: stamp for client, stamp in state["clients"].items() if now - stamp < 3600}
119 if ports:
120 filters = "( sport = :443 or " + " or ".join(f"sport = :{port} or dport = :{port}" for port in sorted(ports)) + " )"
121 result = subprocess.run(["ss", "-HtinO", "state", "established", filters], check=True, capture_output=True, text=True, timeout=5)
122 rows = connections(result.stdout, ports, state["clients"])
123 samples.append({"ts": now, "connections": rows})
124 save(state_file, state)
125 if now - saved >= 5:
126 save(recent, list(samples))
127 saved = now
128 old = sorted((p for p in ROOT.glob("*.json") if re.fullmatch(r"[0-9a-f-]{36}", p.stem)), key=lambda p: p.stat().st_mtime)
129 for path in old[:-100]:
130 path.unlink()
131 if now - reported >= 60:
132 print(json.dumps({"event": "substation-watcher", "ports": sorted(ports),
133 "clients": len(state["clients"]), "samples": len(samples)}), flush=True)
134 reported = now
135 except Exception as error:
136 print(json.dumps({"event": "substation-watcher-error", "type": type(error).__name__}), flush=True)
137 time.sleep(1)
138
139
140if __name__ == "__main__":
141 main()
tools/substation.md+13
...@@ -118,3 +118,16 @@ uploads. A missing length can mean a chunked request; it does not prove completi...@@ -118,3 +118,16 @@ uploads. A missing length can mean a chunked request; it does not prove completi
118A 200 followed by a truncated stream needs the edge byte count and the hub's118A 200 followed by a truncated stream needs the edge byte count and the hub's
119stream outcome. A successful probe does not establish that every generation119stream outcome. A successful probe does not establish that every generation
120stream completed.120stream completed.
121
122`snowglobe-substation-watch.service` follows Caddy with a persisted journal
123cursor and samples TCP counters once per second for the hub and known API
124clients. It survives service restarts and reboots. The latest two minutes are
125retained in `/var/lib/snowglobe/substation-watch/recent.json`; each 499 or 5xx
126retains its request metadata and surrounding TCP progress in a private file
127named by request ID. The directory keeps the latest 100 incidents. No packet
128payloads, request bodies, cookies, or credentials are collected.
129
130Watcher heartbeat and incident summaries appear in its journal and Victoria
131Logs. `upload_complete: false` proves an incomplete fixed-length upload;
132the saved receive/send queues, byte counters, receive windows, and retransmits
133distinguish stalled sending from server backpressure and network loss.