authoravatarclover caruso<git@paperclover.net>2026-11-12 10:41:10-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-12 16:08:55-08:00
log029fe21eabd885646110ce6c0991a7dc1be99bfe
tree43af4de3fdd388f87ca7420430f2d9501a4515df
parent42fe3bf6420ef43eef3bd23ca7093532cf255c1b
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Keep Worktop socket sampling independent of packet ingestion

Read and parse the capture log on a separate thread so upload bursts and macOS resource limits cannot delay TCP queue samples. Record ingestion backlog, busy state and errors so absent packets are not interpreted without capture coverage. Truncate before parsing and retain the truncation timestamp with stall evidence. The installed LaunchAgent now uses the Standard process type. Twelve watcher tests passed locally and seven client tests passed on Worktop. Live sampling stayed within 2.2 seconds while clearing a 55 MB packet backlog. Staging health passed. Disposable persistent TLS controls sent 198 MB each through Python and Bun 1.3.5 under 143 and 359 scoped packet drops; all 32 bodies were read fully and returned 200 on their original connections. Those controls did not reproduce the natural fault. The fixture and drop rules were removed; private diagnostic metadata remains on Snow Globe. Assisted-by: gpt-6.1-sol

3 files changed, 69 insertions(+), 18 deletions(-)

tools/substation-client-watch-test.py+27-3
......@@ -4,6 +4,7 @@ import json
44from pathlib import Path
55import tempfile
66import unittest
7from unittest.mock import patch
78
89
910spec = importlib.util.spec_from_file_location("watch", Path(__file__).with_name("substation-client-watch.py"))
......@@ -16,16 +17,16 @@ class ClientEvidence(unittest.TestCase):
1617 with tempfile.TemporaryDirectory() as temporary:
1718 path = Path(temporary) / "capture.log"
1819 line = "190.123 IP 192.0.2.1.50000 > 192.0.2.5.443: Flags [P.], seq 10:20, ack 30, win 40, length 10 private-payload"
19 self.assertEqual(watch.capture(path, 0), ([], 0, False))
20 self.assertEqual(watch.capture(path, 0), ([], 0, None))
2021 path.write_text(line)
21 self.assertEqual(watch.capture(path, 0), ([], 0, False))
22 self.assertEqual(watch.capture(path, 0), ([], 0, None))
2223 with path.open("a") as writer:
2324 writer.write("\n")
2425 rows, offset, rotated = watch.capture(path, 0)
2526 self.assertEqual(rows[0]["seq"], "10:20")
2627 self.assertNotIn("private", json.dumps(rows))
2728 self.assertFalse(rotated)
28 self.assertEqual(watch.capture(path, offset), ([], offset, False))
29 self.assertEqual(watch.capture(path, offset), ([], offset, None))
2930
3031 def test_capture_rotation_preserves_the_open_append_writer(self):
3132 with tempfile.TemporaryDirectory() as temporary:
......@@ -43,6 +44,29 @@ class ClientEvidence(unittest.TestCase):
4344 self.assertEqual(rows[0]["ack"], "20")
4445 self.assertFalse(rotated)
4546
47 def test_capture_rotates_before_parsing_while_the_writer_remains_active(self):
48 with tempfile.TemporaryDirectory() as temporary:
49 path = Path(temporary) / "capture.log"
50 first = "200.0 IP 192.0.2.5.443 > 192.0.2.1.50000: Flags [.], ack 20, win 40, length 0\n"
51 second = first.replace("200.0", "201.0")
52 with path.open("a") as writer:
53 writer.truncate(64 * 1024 * 1024)
54 writer.write(first)
55 writer.flush()
56 original = watch.packet
57
58 def parse(line):
59 writer.write(second)
60 writer.flush()
61 return original(line)
62
63 with patch.object(watch, "packet", side_effect=parse):
64 rows, offset, rotated = watch.capture(path, 64 * 1024 * 1024)
65 self.assertTrue(rotated)
66 self.assertEqual(rows[0]["ts"], 200.0)
67 rows, _, _ = watch.capture(path, offset)
68 self.assertEqual(rows[0]["ts"], 201.0)
69
4670 def test_socket_counters_and_pids_exclude_other_destinations_and_program_names(self):
4771 text = "\n".join([
4872 "tcp4 0 1234 192.0.2.1.50000 198.51.100.1.443 ESTABLISHED 10 20 1000 2000 private-name:42 0 0",
tools/substation-client-watch.py+37-14
......@@ -9,6 +9,7 @@ from pathlib import Path
99import re
1010import socket
1111import subprocess
12import threading
1213import time
1314
1415from substation_packets import packet
......@@ -86,7 +87,7 @@ def stalls(samples):
8687
8788def capture(path, offset):
8889 rows = []
89 rotated = False
90 truncated_at = None
9091 if path.exists():
9192 with path.open("r+b") as file:
9293 if path.stat().st_size < offset:
......@@ -95,16 +96,16 @@ def capture(path, offset):
9596 data = file.read(8 * 1024 * 1024)
9697 end = data.rfind(b"\n") + 1
9798 offset += end
98 for line in data[:end].splitlines():
99 if row := packet(line.decode("ascii", errors="replace")):
100 rows.append(row)
10199 if offset >= 64 * 1024 * 1024 and offset == path.stat().st_size:
102100 # tcpdump holds an append descriptor; truncation preserves its capture privileges.
103101 file.seek(0)
104102 file.truncate()
105103 offset = 0
106 rotated = True
107 return rows, offset, rotated
104 truncated_at = time.time()
105 for line in data[:end].splitlines():
106 if row := packet(line.decode("ascii", errors="replace")):
107 rows.append(row)
108 return rows, offset, truncated_at
108109
109110
110111def main():
......@@ -114,7 +115,29 @@ def main():
114115 samples = deque(maxlen=120)
115116 packets = deque(maxlen=100000)
116117 stalled = set()
117 capture_offset = 0
118 capture_state = {"offset": 0, "busy": False}
119 capture_gaps = deque(maxlen=100)
120 capture_path = ROOT / "tcp-headers.log"
121
122 def read_capture():
123 while True:
124 capture_state["busy"] = True
125 try:
126 rows, offset, truncated_at = capture(capture_path, capture_state["offset"])
127 packets.extend(rows)
128 while packets and packets[0]["ts"] < time.time() - 120:
129 packets.popleft()
130 capture_state["offset"] = offset
131 capture_state.pop("error", None)
132 if truncated_at is not None:
133 capture_gaps.append(truncated_at)
134 except Exception as error:
135 capture_state["error"] = type(error).__name__
136 finally:
137 capture_state["busy"] = False
138 time.sleep(0.1)
139
140 threading.Thread(target=read_capture, daemon=True).start()
118141 resolved = measured = 0
119142 while True:
120143 now = time.time()
......@@ -126,12 +149,11 @@ def main():
126149 result = subprocess.run(["netstat", "-anv", "-p", "tcp"], check=True, capture_output=True, text=True, timeout=5)
127150 rows = sockets(result.stdout, targets)
128151 record["connections"] = rows
129 captured, capture_offset, rotated = capture(ROOT / "tcp-headers.log", capture_offset)
130 packets.extend(captured)
131 while packets and packets[0]["ts"] < now - 120:
132 packets.popleft()
133 if rotated:
134 record["capture_rotated"] = True
152 if capture_path.exists():
153 record["capture_backlog"] = max(0, capture_path.stat().st_size - capture_state["offset"])
154 record["capture_busy"] = capture_state["busy"]
155 if "error" in capture_state:
156 record["capture_error"] = capture_state["error"]
135157 if now - measured >= 5:
136158 pids = sorted({row["pid"] for row in rows if row["pid"]})
137159 if pids:
......@@ -156,7 +178,8 @@ def main():
156178 pending = path.with_suffix(".pending")
157179 endpoints = {f"[{local.rpartition('.')[0]}]:{local.rpartition('.')[2]}" for local, _, _ in added}
158180 pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples),
159 "packets": [row for row in packets if endpoints.intersection((row["source"], row["destination"]))]}, separators=(",", ":")))
181 "capture_gaps": [ts for ts in list(capture_gaps) if ts >= now - 120],
182 "packets": [row for row in list(packets) if endpoints.intersection((row["source"], row["destination"]))]}, separators=(",", ":")))
160183 pending.replace(path)
161184 for old in sorted(ROOT.glob("stall-*.json"))[:-100]:
162185 old.unlink()
tools/substation.md+5-1
......@@ -151,5 +151,9 @@ The latest 100 stalls survive rolling-history expiry and watcher restarts.
151151When an administrator starts a TCP summary capture into `tcp-headers.log`, the
152152client watcher retains matching sequence/ACK/SACK metadata with each stall.
153153It consumes and truncates that append-only log at 64 MiB without restarting
154tcpdump. Samples mark each truncation; exclude that instant from packet-loss
154tcpdump. Archives mark each truncation; exclude that instant from packet-loss
155155conclusions because concurrent writes can be lost during truncation.
156Packet ingestion runs independently of socket sampling. Samples record unread
157capture bytes, active ingestion and errors. Infer missing sender packets only
158after ingestion finishes and the backlog clears, with no capture error or
159truncation gap in that interval. The LaunchAgent uses `ProcessType: Standard`.