authoravatarclover caruso<git@paperclover.net>2026-11-11 15:56:04-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-12 16:08:55-08:00
logc65330b68ae0ff9f783e00a339d61006cf9c8ff0
treec86163368fa3946cb6c0fcb3f238d1cceef8d949
parent3bc44ef75c76cdead70fd5f67b61325c5618f368
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Retain TCP close and acknowledgement metadata for Substation

Keep bounded transport-only packet metadata for known API clients. Attach matching sequence numbers, ACK/SACK ranges, windows and FIN/RST timing to failed-request evidence so sender loss and server cancellation can be told apart. Validated parser privacy/filtering, five watcher tests, ten deployment tests, staging health and an isolated transient service on Snow Globe. A disposable partial upload retained the client RST before its Caddy error. Assisted-by: gpt-6.1-sol

4 files changed, 68 insertions(+), 9 deletions(-)

nixos/configuration.nix+1-1
......@@ -156,7 +156,7 @@ in
156156 after = [ "nomad.service" "caddy.service" ];
157157 unitConfig.ConditionPathExists = [ "/var/lib/snowglobe/nomad.token" "/opt/snowglobe/current/tools/substation-watch.py" ];
158158 environment.SNOWGLOBE_DOMAIN = config.environment.variables.SNOWGLOBE_DOMAIN;
159 path = [ pkgs.systemd pkgs.iproute2 ];
159 path = [ pkgs.systemd pkgs.iproute2 pkgs.tcpdump ];
160160 serviceConfig = {
161161 ExecStart = "${pkgs.python3}/bin/python3 /opt/snowglobe/current/tools/substation-watch.py";
162162 Restart = "always";
tools/substation-watch-test.py+14-3
......@@ -43,13 +43,24 @@ class TransportEvidence(unittest.TestCase):
4343 def test_completion_and_window(self):
4444 edge = {"ts": 200, "duration": 60, "content_length": "100", "bytes_read": 40}
4545 samples = [{"ts": 130}, {"ts": 140}, {"ts": 195}, {"ts": 210}]
46 record = watch.incident(edge, samples)
46 record = watch.incident(edge, samples, [])
4747 self.assertFalse(record["upload_complete"])
4848 self.assertEqual(record["tcp"], [{"ts": 140}, {"ts": 195}])
4949 edge["bytes_read"] = 100
50 self.assertTrue(watch.incident(edge, samples)["upload_complete"])
50 self.assertTrue(watch.incident(edge, samples, [])["upload_complete"])
5151 edge["content_length"] = ""
52 self.assertIsNone(watch.incident(edge, samples)["upload_complete"])
52 self.assertIsNone(watch.incident(edge, samples, [])["upload_complete"])
53
54 def test_packet_metadata_excludes_payload_and_other_connections(self):
55 row = watch.packet("190.123 lo In IP 192.0.2.1.50000 > 192.0.2.5.443: Flags [R.], seq 10:20, ack 30, win 40, options [sack 1 {50:60}], length 10 private-payload")
56 self.assertEqual(row["flags"], "R.")
57 self.assertEqual(row["sack"], ["50:60"])
58 self.assertNotIn("private", json.dumps(row))
59 edge = {"ts": 200, "duration": 60, "client": "192.0.2.1:50000"}
60 self.assertEqual(watch.incident(edge, [], [row])["packets"], [row])
61 edge["client"] = "192.0.2.1:50001"
62 self.assertEqual(watch.incident(edge, [], [row])["packets"], [])
63 self.assertIsNone(watch.packet("private application payload"))
5364
5465 def test_private_atomic_state_survives_reload(self):
5566 with tempfile.TemporaryDirectory() as temporary:
tools/substation-watch.py+50-5
......@@ -6,6 +6,7 @@ import os
66from pathlib import Path
77import re
88import subprocess
9import threading
910import time
1011from urllib.request import Request, urlopen
1112
......@@ -67,11 +68,35 @@ def event(entry, host):
6768 return None, None
6869
6970
70def incident(edge, samples):
71def packet(line):
72 match = re.fullmatch(r"([0-9.]+) \S+\s+(?:In|Out)\s+IP6? (\S+) > (\S+): Flags \[([A-Za-z.*]+)\], (.*)", line.strip())
73 if not match:
74 return None
75 stamp, source, destination, flags, details = match.groups()
76 endpoints = {}
77 try:
78 for key, value in (("source", source), ("destination", destination)):
79 host, _, port = value.rpartition(".")
80 host = ipaddress.ip_address(host)
81 endpoints[key] = f"[{host}]:{int(port)}"
82 except ValueError:
83 return None
84 row = {"ts": float(stamp), **endpoints, "flags": flags}
85 for key in ("seq", "ack", "win", "length"):
86 if value := re.search(r"\b" + key + r" ([0-9]+(?::[0-9]+)?)", details):
87 row[key] = value[1]
88 row["sack"] = re.findall(r"\{([0-9]+:[0-9]+)\}", details)
89 return row
90
91
92def incident(edge, samples, packets):
7193 length = edge.get("content_length")
7294 complete = int(length) == edge["bytes_read"] if str(length).isdigit() else None
95 peer = endpoint(edge["client"]) if edge.get("client") else None
7396 return {"edge": edge, "upload_complete": complete,
74 "tcp": [sample for sample in samples if edge["ts"] - edge["duration"] - 5 <= sample["ts"] <= edge["ts"] + 5]}
97 "tcp": [sample for sample in samples if edge["ts"] - edge["duration"] - 5 <= sample["ts"] <= edge["ts"] + 5],
98 "packets": [row for row in list(packets) if edge["ts"] - edge["duration"] - 5 <= row["ts"] <= edge["ts"] + 5
99 and peer in (endpoint(row["source"]), endpoint(row["destination"]))]}
75100
76101
77102def main():
......@@ -81,12 +106,32 @@ def main():
81106 state = json.loads(state_file.read_text()) if state_file.exists() else {"clients": {}}
82107 recent = ROOT / "recent.json"
83108 samples = deque(json.loads(recent.read_text()) if recent.exists() and time.time() - recent.stat().st_mtime < 120 else [], maxlen=120)
109 packets = deque(maxlen=100000)
110 capture = None
111 captured_clients = None
112
113 def read_packets(stream):
114 for line in stream:
115 row = packet(line)
116 if row and any(endpoint(row[key])[0] in state["clients"] for key in ("source", "destination")):
117 packets.append(row)
118
84119 ports = set()
85120 refreshed = saved = reported = 0
86121 host = "substation.ai." + os.environ["SNOWGLOBE_DOMAIN"]
87122 while True:
88123 try:
89124 now = time.time()
125 clients = tuple(sorted(state["clients"]))
126 if capture is None or capture.poll() is not None or clients != captured_clients:
127 if capture is not None:
128 capture.terminate()
129 capture.wait(timeout=2)
130 filters = "tcp port 443 and (" + " or ".join("host " + client for client in (clients or ("127.0.0.1",))) + ")"
131 capture = subprocess.Popen(["tcpdump", "-l", "-tt", "-nn", "-S", "-i", "any", "-s", "128", filters],
132 stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True)
133 threading.Thread(target=read_packets, args=(capture.stdout,), daemon=True).start()
134 captured_clients = clients
90135 if now - refreshed >= 10:
91136 token = Path("/var/lib/snowglobe/nomad.token").read_text().strip()
92137 request = Request("http://127.0.0.1:4646/v1/service/substation-hub", headers={"X-Nomad-Token": token})
......@@ -109,11 +154,11 @@ def main():
109154 if edge and edge.get("status", 0) >= 499 and re.fullmatch(r"[0-9a-f-]{36}", str(edge.get("request_id", ""))):
110155 path = ROOT / (edge["request_id"] + ".json")
111156 if not path.exists():
112 record = incident(edge, samples)
157 record = incident(edge, samples, packets)
113158 save(path, record)
114159 print(json.dumps({"event": "substation-incident", "request_id": edge["request_id"],
115160 "status": edge["status"], "upload_complete": record["upload_complete"],
116 "samples": len(record["tcp"])}), flush=True)
161 "samples": len(record["tcp"]), "packets": len(record["packets"])}), flush=True)
117162 state["cursor"] = entry["__CURSOR"]
118163 state["clients"] = {client: stamp for client, stamp in state["clients"].items() if now - stamp < 3600}
119164 if ports:
......@@ -130,7 +175,7 @@ def main():
130175 path.unlink()
131176 if now - reported >= 60:
132177 print(json.dumps({"event": "substation-watcher", "ports": sorted(ports),
133 "clients": len(state["clients"]), "samples": len(samples)}), flush=True)
178 "clients": len(state["clients"]), "samples": len(samples), "packets": len(packets)}), flush=True)
134179 reported = now
135180 except Exception as error:
136181 print(json.dumps({"event": "substation-watcher-error", "type": type(error).__name__}), flush=True)
tools/substation.md+3
......@@ -132,6 +132,9 @@ Watcher heartbeat and incident summaries appear in its journal and Victoria
132132Logs. `upload_complete: false` proves an incomplete fixed-length upload;
133133the saved receive/send queues, byte counters, receive windows, and retransmits
134134distinguish stalled sending from server backpressure and network loss.
135Incident files also retain matching TCP sequence numbers, acknowledgements,
136SACK ranges, windows and flags. These establish packet loss and which side
137sent FIN/RST first. Only parsed transport fields are retained.
135138
136139Worktop's `net.paperclover.substation-watch` LaunchAgent records matching
137140Substation sockets under `~/.local/state/substation-watch`. Minute files retain