authoravatarclover caruso<git@paperclover.net>2026-11-12 10:23:02-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-12 16:08:55-08:00
log42fe3bf6420ef43eef3bd23ca7093532cf255c1b
tree31dc6ce8271413a4093ad4761add9d4c8ed6151c
parent3b735d29265d065eff0ee72b7943eb7795f87bbb
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Retain client packet evidence for stalled Substation uploads

Share the transport-only tcpdump parser between the server and Worktop watchers. Accept macOS capture summaries and retain matching packet metadata in each client stall archive. Consume the append log in bounded chunks and truncate it at 64 MiB; mark truncation instants because concurrent writes can be lost. Eleven watcher tests passed locally and six client tests passed on Worktop. A disposable staging connection with scoped packet loss retained 1,745 client packet records with its stall before the connection aborted. The temporary firewall rule and capture were removed. Staging client/management auth checks passed. The Worktop LaunchAgent runs the exported client revision. Assisted-by: gpt-6.1-sol

6 files changed, 102 insertions(+), 22 deletions(-)

tools/substation-client-watch-test.py+32
......@@ -2,6 +2,7 @@
22import importlib.util
33import json
44from pathlib import Path
5import tempfile
56import unittest
67
78
......@@ -11,6 +12,37 @@ spec.loader.exec_module(watch)
1112
1213
1314class ClientEvidence(unittest.TestCase):
15 def test_capture_keeps_partial_lines_and_excludes_payload(self):
16 with tempfile.TemporaryDirectory() as temporary:
17 path = Path(temporary) / "capture.log"
18 line = "190.123 IP 192.0.2.1.50000 > 192.0.2.5.443: Flags [P.], seq 10:20, ack 30, win 40, length 10 private-payload"
19 self.assertEqual(watch.capture(path, 0), ([], 0, False))
20 path.write_text(line)
21 self.assertEqual(watch.capture(path, 0), ([], 0, False))
22 with path.open("a") as writer:
23 writer.write("\n")
24 rows, offset, rotated = watch.capture(path, 0)
25 self.assertEqual(rows[0]["seq"], "10:20")
26 self.assertNotIn("private", json.dumps(rows))
27 self.assertFalse(rotated)
28 self.assertEqual(watch.capture(path, offset), ([], offset, False))
29
30 def test_capture_rotation_preserves_the_open_append_writer(self):
31 with tempfile.TemporaryDirectory() as temporary:
32 path = Path(temporary) / "capture.log"
33 with path.open("a") as writer:
34 writer.truncate(64 * 1024 * 1024)
35 writer.write("\n")
36 writer.flush()
37 _, offset, rotated = watch.capture(path, 64 * 1024 * 1024)
38 self.assertTrue(rotated)
39 self.assertEqual(offset, 0)
40 writer.write("200.0 IP 192.0.2.5.443 > 192.0.2.1.50000: Flags [.], ack 20, win 40, length 0\n")
41 writer.flush()
42 rows, _, rotated = watch.capture(path, offset)
43 self.assertEqual(rows[0]["ack"], "20")
44 self.assertFalse(rotated)
45
1446 def test_socket_counters_and_pids_exclude_other_destinations_and_program_names(self):
1547 text = "\n".join([
1648 "tcp4 0 1234 192.0.2.1.50000 198.51.100.1.443 ESTABLISHED 10 20 1000 2000 private-name:42 0 0",
tools/substation-client-watch.py+36-1
......@@ -11,6 +11,8 @@ import socket
1111import subprocess
1212import time
1313
14from substation_packets import packet
15
1416
1517ROOT = Path.home() / ".local/state/substation-watch"
1618COLUMNS = "bytes_in,bytes_out,rx_dupe,rx_ooo,re-tx,rtt_avg,rcvsize,tx_win"
......@@ -82,12 +84,37 @@ def stalls(samples):
8284 return stalled
8385
8486
87def capture(path, offset):
88 rows = []
89 rotated = False
90 if path.exists():
91 with path.open("r+b") as file:
92 if path.stat().st_size < offset:
93 offset = 0
94 file.seek(offset)
95 data = file.read(8 * 1024 * 1024)
96 end = data.rfind(b"\n") + 1
97 offset += end
98 for line in data[:end].splitlines():
99 if row := packet(line.decode("ascii", errors="replace")):
100 rows.append(row)
101 if offset >= 64 * 1024 * 1024 and offset == path.stat().st_size:
102 # tcpdump holds an append descriptor; truncation preserves its capture privileges.
103 file.seek(0)
104 file.truncate()
105 offset = 0
106 rotated = True
107 return rows, offset, rotated
108
109
85110def main():
86111 os.umask(0o077)
87112 ROOT.mkdir(mode=0o700, parents=True, exist_ok=True)
88113 targets = set()
89114 samples = deque(maxlen=120)
115 packets = deque(maxlen=100000)
90116 stalled = set()
117 capture_offset = 0
91118 resolved = measured = 0
92119 while True:
93120 now = time.time()
......@@ -99,6 +126,12 @@ def main():
99126 result = subprocess.run(["netstat", "-anv", "-p", "tcp"], check=True, capture_output=True, text=True, timeout=5)
100127 rows = sockets(result.stdout, targets)
101128 record["connections"] = rows
129 captured, capture_offset, rotated = capture(ROOT / "tcp-headers.log", capture_offset)
130 packets.extend(captured)
131 while packets and packets[0]["ts"] < now - 120:
132 packets.popleft()
133 if rotated:
134 record["capture_rotated"] = True
102135 if now - measured >= 5:
103136 pids = sorted({row["pid"] for row in rows if row["pid"]})
104137 if pids:
......@@ -121,7 +154,9 @@ def main():
121154 if added := current - stalled:
122155 path = ROOT / ("stall-" + str(int(now * 1000)) + ".json")
123156 pending = path.with_suffix(".pending")
124 pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples)}, separators=(",", ":")))
157 endpoints = {f"[{local.rpartition('.')[0]}]:{local.rpartition('.')[2]}" for local, _, _ in added}
158 pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples),
159 "packets": [row for row in packets if endpoints.intersection((row["source"], row["destination"]))]}, separators=(",", ":")))
125160 pending.replace(path)
126161 for old in sorted(ROOT.glob("stall-*.json"))[:-100]:
127162 old.unlink()
tools/substation-watch-test.py+4
......@@ -56,6 +56,10 @@ class TransportEvidence(unittest.TestCase):
5656 self.assertEqual(row["flags"], "R.")
5757 self.assertEqual(row["sack"], ["50:60"])
5858 self.assertNotIn("private", json.dumps(row))
59 mac = watch.packet("190.123 IP 192.0.2.1.50000 > 192.0.2.5.443: Flags [R.], seq 10:20, ack 30, win 40, options [sack 1 {50:60}], length 10")
60 self.assertEqual(mac, row)
61 ipv6 = watch.packet("190.123 IP6 2001:db8::1.50000 > 2001:db8::5.443: Flags [.], ack 30, win 40, length 0")
62 self.assertEqual(ipv6["source"], "[2001:db8::1]:50000")
5963 edge = {"ts": 200, "duration": 60, "client": "192.0.2.1:50000"}
6064 self.assertEqual(watch.incident(edge, [], [row])["packets"], [row])
6165 edge["client"] = "192.0.2.1:50001"
tools/substation-watch.py+2-21
......@@ -10,6 +10,8 @@ import threading
1010import time
1111from urllib.request import Request, urlopen
1212
13from substation_packets import packet
14
1315
1416ROOT = Path("/var/lib/snowglobe/substation-watch")
1517EDGE_FIELDS = ("ts", "status", "bytes_read", "content_length", "duration", "request_id",
......@@ -68,27 +70,6 @@ def event(entry, host):
6870 return None, None
6971
7072
71def packet(line):
72 match = re.fullmatch(r"([0-9.]+) \S+\s+(?:In|Out)\s+IP6? (\S+) > (\S+): Flags \[([A-Za-z.*]+)\], (.*)", line.strip())
73 if not match:
74 return None
75 stamp, source, destination, flags, details = match.groups()
76 endpoints = {}
77 try:
78 for key, value in (("source", source), ("destination", destination)):
79 host, _, port = value.rpartition(".")
80 host = ipaddress.ip_address(host)
81 endpoints[key] = f"[{host}]:{int(port)}"
82 except ValueError:
83 return None
84 row = {"ts": float(stamp), **endpoints, "flags": flags}
85 for key in ("seq", "ack", "win", "length"):
86 if value := re.search(r"\b" + key + r" ([0-9]+(?::[0-9]+)?)", details):
87 row[key] = value[1]
88 row["sack"] = re.findall(r"\{([0-9]+:[0-9]+)\}", details)
89 return row
90
91
9273def incident(edge, samples, packets):
9374 length = edge.get("content_length")
9475 complete = int(length) == edge["bytes_read"] if str(length).isdigit() else None
tools/substation.md+5
......@@ -148,3 +148,8 @@ Match UTC timestamps to the server incident before comparing TCP progress.
148148An upload with queued bytes and no acknowledgement progress for 10 seconds
149149retains the preceding two minutes in a private `stall-<timestamp>.json` file.
150150The latest 100 stalls survive rolling-history expiry and watcher restarts.
151When an administrator starts a TCP summary capture into `tcp-headers.log`, the
152client watcher retains matching sequence/ACK/SACK metadata with each stall.
153It consumes and truncates that append-only log at 64 MiB without restarting
154tcpdump. Samples mark each truncation; exclude that instant from packet-loss
155conclusions because concurrent writes can be lost during truncation.
tools/substation_packets.py created+23
......@@ -0,0 +1,23 @@
1import ipaddress
2import re
3
4
5def packet(line):
6 match = re.fullmatch(r"([0-9.]+) (?:\S+\s+(?:In|Out)\s+)?IP6? (\S+) > (\S+): Flags \[([A-Za-z.*]+)\], (.*)", line.strip())
7 if not match:
8 return None
9 stamp, source, destination, flags, details = match.groups()
10 endpoints = {}
11 try:
12 for key, value in (("source", source), ("destination", destination)):
13 host, _, port = value.rpartition(".")
14 host = ipaddress.ip_address(host)
15 endpoints[key] = f"[{host}]:{int(port)}"
16 except ValueError:
17 return None
18 row = {"ts": float(stamp), **endpoints, "flags": flags}
19 for key in ("seq", "ack", "win", "length"):
20 if value := re.search(r"\b" + key + r" ([0-9]+(?::[0-9]+)?)", details):
21 row[key] = value[1]
22 row["sack"] = re.findall(r"\{([0-9]+:[0-9]+)\}", details)
23 return row