| author | |
| committer | |
| log | 029fe21eabd885646110ce6c0991a7dc1be99bfe |
| tree | 43af4de3fdd388f87ca7420430f2d9501a4515df |
| parent | 42fe3bf6420ef43eef3bd23ca7093532cf255c1b |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Read and parse the capture log on a separate thread so upload bursts and macOS
resource limits cannot delay TCP queue samples. Record ingestion backlog, busy
state and errors so absent packets are not interpreted without capture coverage.
Truncate before parsing and retain the truncation timestamp with stall evidence.
The installed LaunchAgent now uses the Standard process type.
Twelve watcher tests passed locally and seven client tests passed on Worktop.
Live sampling stayed within 2.2 seconds while clearing a 55 MB packet backlog.
Staging health passed. Disposable persistent TLS controls sent 198 MB each
through Python and Bun 1.3.5 under 143 and 359 scoped packet drops; all 32 bodies
were read fully and returned 200 on their original connections. Those controls
did not reproduce the natural fault. The fixture and drop rules were removed;
private diagnostic metadata remains on Snow Globe.
Assisted-by: gpt-6.1-sol3 files changed, 69 insertions(+), 18 deletions(-)
tools/substation-client-watch-test.py+27-3| ... | @@ -4,6 +4,7 @@ import json | ... | @@ -4,6 +4,7 @@ import json |
| 4 | from pathlib import Path | 4 | from pathlib import Path |
| 5 | import tempfile | 5 | import tempfile |
| 6 | import unittest | 6 | import unittest |
| 7 | from unittest.mock import patch | ||
| 7 | 8 | ||
| 8 | 9 | ||
| 9 | spec = importlib.util.spec_from_file_location("watch", Path(__file__).with_name("substation-client-watch.py")) | 10 | spec = importlib.util.spec_from_file_location("watch", Path(__file__).with_name("substation-client-watch.py")) |
| ... | @@ -16,16 +17,16 @@ class ClientEvidence(unittest.TestCase): | ... | @@ -16,16 +17,16 @@ class ClientEvidence(unittest.TestCase): |
| 16 | with tempfile.TemporaryDirectory() as temporary: | 17 | with tempfile.TemporaryDirectory() as temporary: |
| 17 | path = Path(temporary) / "capture.log" | 18 | path = Path(temporary) / "capture.log" |
| 18 | line = "190.123 IP 192.0.2.1.50000 > 192.0.2.5.443: Flags [P.], seq 10:20, ack 30, win 40, length 10 private-payload" | 19 | line = "190.123 IP 192.0.2.1.50000 > 192.0.2.5.443: Flags [P.], seq 10:20, ack 30, win 40, length 10 private-payload" |
| 19 | self.assertEqual(watch.capture(path, 0), ([], 0, False)) | 20 | self.assertEqual(watch.capture(path, 0), ([], 0, None)) |
| 20 | path.write_text(line) | 21 | path.write_text(line) |
| 21 | self.assertEqual(watch.capture(path, 0), ([], 0, False)) | 22 | self.assertEqual(watch.capture(path, 0), ([], 0, None)) |
| 22 | with path.open("a") as writer: | 23 | with path.open("a") as writer: |
| 23 | writer.write("\n") | 24 | writer.write("\n") |
| 24 | rows, offset, rotated = watch.capture(path, 0) | 25 | rows, offset, rotated = watch.capture(path, 0) |
| 25 | self.assertEqual(rows[0]["seq"], "10:20") | 26 | self.assertEqual(rows[0]["seq"], "10:20") |
| 26 | self.assertNotIn("private", json.dumps(rows)) | 27 | self.assertNotIn("private", json.dumps(rows)) |
| 27 | self.assertFalse(rotated) | 28 | self.assertFalse(rotated) |
| 28 | self.assertEqual(watch.capture(path, offset), ([], offset, False)) | 29 | self.assertEqual(watch.capture(path, offset), ([], offset, None)) |
| 29 | 30 | ||
| 30 | def test_capture_rotation_preserves_the_open_append_writer(self): | 31 | def test_capture_rotation_preserves_the_open_append_writer(self): |
| 31 | with tempfile.TemporaryDirectory() as temporary: | 32 | with tempfile.TemporaryDirectory() as temporary: |
| ... | @@ -43,6 +44,29 @@ class ClientEvidence(unittest.TestCase): | ... | @@ -43,6 +44,29 @@ class ClientEvidence(unittest.TestCase): |
| 43 | self.assertEqual(rows[0]["ack"], "20") | 44 | self.assertEqual(rows[0]["ack"], "20") |
| 44 | self.assertFalse(rotated) | 45 | self.assertFalse(rotated) |
| 45 | 46 | ||
| 47 | def test_capture_rotates_before_parsing_while_the_writer_remains_active(self): | ||
| 48 | with tempfile.TemporaryDirectory() as temporary: | ||
| 49 | path = Path(temporary) / "capture.log" | ||
| 50 | first = "200.0 IP 192.0.2.5.443 > 192.0.2.1.50000: Flags [.], ack 20, win 40, length 0\n" | ||
| 51 | second = first.replace("200.0", "201.0") | ||
| 52 | with path.open("a") as writer: | ||
| 53 | writer.truncate(64 * 1024 * 1024) | ||
| 54 | writer.write(first) | ||
| 55 | writer.flush() | ||
| 56 | original = watch.packet | ||
| 57 | |||
| 58 | def parse(line): | ||
| 59 | writer.write(second) | ||
| 60 | writer.flush() | ||
| 61 | return original(line) | ||
| 62 | |||
| 63 | with patch.object(watch, "packet", side_effect=parse): | ||
| 64 | rows, offset, rotated = watch.capture(path, 64 * 1024 * 1024) | ||
| 65 | self.assertTrue(rotated) | ||
| 66 | self.assertEqual(rows[0]["ts"], 200.0) | ||
| 67 | rows, _, _ = watch.capture(path, offset) | ||
| 68 | self.assertEqual(rows[0]["ts"], 201.0) | ||
| 69 | |||
| 46 | def test_socket_counters_and_pids_exclude_other_destinations_and_program_names(self): | 70 | def test_socket_counters_and_pids_exclude_other_destinations_and_program_names(self): |
| 47 | text = "\n".join([ | 71 | text = "\n".join([ |
| 48 | "tcp4 0 1234 192.0.2.1.50000 198.51.100.1.443 ESTABLISHED 10 20 1000 2000 private-name:42 0 0", | 72 | "tcp4 0 1234 192.0.2.1.50000 198.51.100.1.443 ESTABLISHED 10 20 1000 2000 private-name:42 0 0", |
tools/substation-client-watch.py+37-14| ... | @@ -9,6 +9,7 @@ from pathlib import Path | ... | @@ -9,6 +9,7 @@ from pathlib import Path |
| 9 | import re | 9 | import re |
| 10 | import socket | 10 | import socket |
| 11 | import subprocess | 11 | import subprocess |
| 12 | import threading | ||
| 12 | import time | 13 | import time |
| 13 | 14 | ||
| 14 | from substation_packets import packet | 15 | from substation_packets import packet |
| ... | @@ -86,7 +87,7 @@ def stalls(samples): | ... | @@ -86,7 +87,7 @@ def stalls(samples): |
| 86 | 87 | ||
| 87 | def capture(path, offset): | 88 | def capture(path, offset): |
| 88 | rows = [] | 89 | rows = [] |
| 89 | rotated = False | 90 | truncated_at = None |
| 90 | if path.exists(): | 91 | if path.exists(): |
| 91 | with path.open("r+b") as file: | 92 | with path.open("r+b") as file: |
| 92 | if path.stat().st_size < offset: | 93 | if path.stat().st_size < offset: |
| ... | @@ -95,16 +96,16 @@ def capture(path, offset): | ... | @@ -95,16 +96,16 @@ def capture(path, offset): |
| 95 | data = file.read(8 * 1024 * 1024) | 96 | data = file.read(8 * 1024 * 1024) |
| 96 | end = data.rfind(b"\n") + 1 | 97 | end = data.rfind(b"\n") + 1 |
| 97 | offset += end | 98 | offset += end |
| 98 | for line in data[:end].splitlines(): | ||
| 99 | if row := packet(line.decode("ascii", errors="replace")): | ||
| 100 | rows.append(row) | ||
| 101 | if offset >= 64 * 1024 * 1024 and offset == path.stat().st_size: | 99 | if offset >= 64 * 1024 * 1024 and offset == path.stat().st_size: |
| 102 | # tcpdump holds an append descriptor; truncation preserves its capture privileges. | 100 | # tcpdump holds an append descriptor; truncation preserves its capture privileges. |
| 103 | file.seek(0) | 101 | file.seek(0) |
| 104 | file.truncate() | 102 | file.truncate() |
| 105 | offset = 0 | 103 | offset = 0 |
| 106 | rotated = True | 104 | truncated_at = time.time() |
| 107 | return rows, offset, rotated | 105 | for line in data[:end].splitlines(): |
| 106 | if row := packet(line.decode("ascii", errors="replace")): | ||
| 107 | rows.append(row) | ||
| 108 | return rows, offset, truncated_at | ||
| 108 | 109 | ||
| 109 | 110 | ||
| 110 | def main(): | 111 | def main(): |
| ... | @@ -114,7 +115,29 @@ def main(): | ... | @@ -114,7 +115,29 @@ def main(): |
| 114 | samples = deque(maxlen=120) | 115 | samples = deque(maxlen=120) |
| 115 | packets = deque(maxlen=100000) | 116 | packets = deque(maxlen=100000) |
| 116 | stalled = set() | 117 | stalled = set() |
| 117 | capture_offset = 0 | 118 | capture_state = {"offset": 0, "busy": False} |
| 119 | capture_gaps = deque(maxlen=100) | ||
| 120 | capture_path = ROOT / "tcp-headers.log" | ||
| 121 | |||
| 122 | def read_capture(): | ||
| 123 | while True: | ||
| 124 | capture_state["busy"] = True | ||
| 125 | try: | ||
| 126 | rows, offset, truncated_at = capture(capture_path, capture_state["offset"]) | ||
| 127 | packets.extend(rows) | ||
| 128 | while packets and packets[0]["ts"] < time.time() - 120: | ||
| 129 | packets.popleft() | ||
| 130 | capture_state["offset"] = offset | ||
| 131 | capture_state.pop("error", None) | ||
| 132 | if truncated_at is not None: | ||
| 133 | capture_gaps.append(truncated_at) | ||
| 134 | except Exception as error: | ||
| 135 | capture_state["error"] = type(error).__name__ | ||
| 136 | finally: | ||
| 137 | capture_state["busy"] = False | ||
| 138 | time.sleep(0.1) | ||
| 139 | |||
| 140 | threading.Thread(target=read_capture, daemon=True).start() | ||
| 118 | resolved = measured = 0 | 141 | resolved = measured = 0 |
| 119 | while True: | 142 | while True: |
| 120 | now = time.time() | 143 | now = time.time() |
| ... | @@ -126,12 +149,11 @@ def main(): | ... | @@ -126,12 +149,11 @@ def main(): |
| 126 | result = subprocess.run(["netstat", "-anv", "-p", "tcp"], check=True, capture_output=True, text=True, timeout=5) | 149 | result = subprocess.run(["netstat", "-anv", "-p", "tcp"], check=True, capture_output=True, text=True, timeout=5) |
| 127 | rows = sockets(result.stdout, targets) | 150 | rows = sockets(result.stdout, targets) |
| 128 | record["connections"] = rows | 151 | record["connections"] = rows |
| 129 | captured, capture_offset, rotated = capture(ROOT / "tcp-headers.log", capture_offset) | 152 | if capture_path.exists(): |
| 130 | packets.extend(captured) | 153 | record["capture_backlog"] = max(0, capture_path.stat().st_size - capture_state["offset"]) |
| 131 | while packets and packets[0]["ts"] < now - 120: | 154 | record["capture_busy"] = capture_state["busy"] |
| 132 | packets.popleft() | 155 | if "error" in capture_state: |
| 133 | if rotated: | 156 | record["capture_error"] = capture_state["error"] |
| 134 | record["capture_rotated"] = True | ||
| 135 | if now - measured >= 5: | 157 | if now - measured >= 5: |
| 136 | pids = sorted({row["pid"] for row in rows if row["pid"]}) | 158 | pids = sorted({row["pid"] for row in rows if row["pid"]}) |
| 137 | if pids: | 159 | if pids: |
| ... | @@ -156,7 +178,8 @@ def main(): | ... | @@ -156,7 +178,8 @@ def main(): |
| 156 | pending = path.with_suffix(".pending") | 178 | pending = path.with_suffix(".pending") |
| 157 | endpoints = {f"[{local.rpartition('.')[0]}]:{local.rpartition('.')[2]}" for local, _, _ in added} | 179 | endpoints = {f"[{local.rpartition('.')[0]}]:{local.rpartition('.')[2]}" for local, _, _ in added} |
| 158 | pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples), | 180 | pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples), |
| 159 | "packets": [row for row in packets if endpoints.intersection((row["source"], row["destination"]))]}, separators=(",", ":"))) | 181 | "capture_gaps": [ts for ts in list(capture_gaps) if ts >= now - 120], |
| 182 | "packets": [row for row in list(packets) if endpoints.intersection((row["source"], row["destination"]))]}, separators=(",", ":"))) | ||
| 160 | pending.replace(path) | 183 | pending.replace(path) |
| 161 | for old in sorted(ROOT.glob("stall-*.json"))[:-100]: | 184 | for old in sorted(ROOT.glob("stall-*.json"))[:-100]: |
| 162 | old.unlink() | 185 | old.unlink() |
tools/substation.md+5-1| ... | @@ -151,5 +151,9 @@ The latest 100 stalls survive rolling-history expiry and watcher restarts. | ... | @@ -151,5 +151,9 @@ The latest 100 stalls survive rolling-history expiry and watcher restarts. |
| 151 | When an administrator starts a TCP summary capture into `tcp-headers.log`, the | 151 | When an administrator starts a TCP summary capture into `tcp-headers.log`, the |
| 152 | client watcher retains matching sequence/ACK/SACK metadata with each stall. | 152 | client watcher retains matching sequence/ACK/SACK metadata with each stall. |
| 153 | It consumes and truncates that append-only log at 64 MiB without restarting | 153 | It consumes and truncates that append-only log at 64 MiB without restarting |
| 154 | tcpdump. Samples mark each truncation; exclude that instant from packet-loss | 154 | tcpdump. Archives mark each truncation; exclude that instant from packet-loss |
| 155 | conclusions because concurrent writes can be lost during truncation. | 155 | conclusions because concurrent writes can be lost during truncation. |
| 156 | Packet ingestion runs independently of socket sampling. Samples record unread | ||
| 157 | capture bytes, active ingestion and errors. Infer missing sender packets only | ||
| 158 | after ingestion finishes and the backlog clears, with no capture error or | ||
| 159 | truncation gap in that interval. The LaunchAgent uses `ProcessType: Standard`. |