| author | |
| committer | |
| log | 42fe3bf6420ef43eef3bd23ca7093532cf255c1b |
| tree | 31dc6ce8271413a4093ad4761add9d4c8ed6151c |
| parent | 3b735d29265d065eff0ee72b7943eb7795f87bbb |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Share the transport-only tcpdump parser between the server and Worktop watchers.
Accept macOS capture summaries and retain matching packet metadata in each
client stall archive. Consume the append log in bounded chunks and truncate it
at 64 MiB; mark truncation instants because concurrent writes can be lost.
Eleven watcher tests passed locally and six client tests passed on Worktop.
A disposable staging connection with scoped packet loss retained 1,745 client
packet records with its stall before the connection aborted. The temporary
firewall rule and capture were removed. Staging client/management auth checks
passed. The Worktop LaunchAgent runs the exported client revision.
Assisted-by: gpt-6.1-sol6 files changed, 102 insertions(+), 22 deletions(-)
tools/substation-client-watch-test.py+32| ... | @@ -2,6 +2,7 @@ | ... | @@ -2,6 +2,7 @@ |
| 2 | import importlib.util | 2 | import importlib.util |
| 3 | import json | 3 | import json |
| 4 | from pathlib import Path | 4 | from pathlib import Path |
| 5 | import tempfile | ||
| 5 | import unittest | 6 | import unittest |
| 6 | 7 | ||
| 7 | 8 | ||
| ... | @@ -11,6 +12,37 @@ spec.loader.exec_module(watch) | ... | @@ -11,6 +12,37 @@ spec.loader.exec_module(watch) |
| 11 | 12 | ||
| 12 | 13 | ||
| 13 | class ClientEvidence(unittest.TestCase): | 14 | class ClientEvidence(unittest.TestCase): |
| 15 | def test_capture_keeps_partial_lines_and_excludes_payload(self): | ||
| 16 | with tempfile.TemporaryDirectory() as temporary: | ||
| 17 | path = Path(temporary) / "capture.log" | ||
| 18 | line = "190.123 IP 192.0.2.1.50000 > 192.0.2.5.443: Flags [P.], seq 10:20, ack 30, win 40, length 10 private-payload" | ||
| 19 | self.assertEqual(watch.capture(path, 0), ([], 0, False)) | ||
| 20 | path.write_text(line) | ||
| 21 | self.assertEqual(watch.capture(path, 0), ([], 0, False)) | ||
| 22 | with path.open("a") as writer: | ||
| 23 | writer.write("\n") | ||
| 24 | rows, offset, rotated = watch.capture(path, 0) | ||
| 25 | self.assertEqual(rows[0]["seq"], "10:20") | ||
| 26 | self.assertNotIn("private", json.dumps(rows)) | ||
| 27 | self.assertFalse(rotated) | ||
| 28 | self.assertEqual(watch.capture(path, offset), ([], offset, False)) | ||
| 29 | |||
| 30 | def test_capture_rotation_preserves_the_open_append_writer(self): | ||
| 31 | with tempfile.TemporaryDirectory() as temporary: | ||
| 32 | path = Path(temporary) / "capture.log" | ||
| 33 | with path.open("a") as writer: | ||
| 34 | writer.truncate(64 * 1024 * 1024) | ||
| 35 | writer.write("\n") | ||
| 36 | writer.flush() | ||
| 37 | _, offset, rotated = watch.capture(path, 64 * 1024 * 1024) | ||
| 38 | self.assertTrue(rotated) | ||
| 39 | self.assertEqual(offset, 0) | ||
| 40 | writer.write("200.0 IP 192.0.2.5.443 > 192.0.2.1.50000: Flags [.], ack 20, win 40, length 0\n") | ||
| 41 | writer.flush() | ||
| 42 | rows, _, rotated = watch.capture(path, offset) | ||
| 43 | self.assertEqual(rows[0]["ack"], "20") | ||
| 44 | self.assertFalse(rotated) | ||
| 45 | |||
| 14 | def test_socket_counters_and_pids_exclude_other_destinations_and_program_names(self): | 46 | def test_socket_counters_and_pids_exclude_other_destinations_and_program_names(self): |
| 15 | text = "\n".join([ | 47 | text = "\n".join([ |
| 16 | "tcp4 0 1234 192.0.2.1.50000 198.51.100.1.443 ESTABLISHED 10 20 1000 2000 private-name:42 0 0", | 48 | "tcp4 0 1234 192.0.2.1.50000 198.51.100.1.443 ESTABLISHED 10 20 1000 2000 private-name:42 0 0", |
tools/substation-client-watch.py+36-1| ... | @@ -11,6 +11,8 @@ import socket | ... | @@ -11,6 +11,8 @@ import socket |
| 11 | import subprocess | 11 | import subprocess |
| 12 | import time | 12 | import time |
| 13 | 13 | ||
| 14 | from substation_packets import packet | ||
| 15 | |||
| 14 | 16 | ||
| 15 | ROOT = Path.home() / ".local/state/substation-watch" | 17 | ROOT = Path.home() / ".local/state/substation-watch" |
| 16 | COLUMNS = "bytes_in,bytes_out,rx_dupe,rx_ooo,re-tx,rtt_avg,rcvsize,tx_win" | 18 | COLUMNS = "bytes_in,bytes_out,rx_dupe,rx_ooo,re-tx,rtt_avg,rcvsize,tx_win" |
| ... | @@ -82,12 +84,37 @@ def stalls(samples): | ... | @@ -82,12 +84,37 @@ def stalls(samples): |
| 82 | return stalled | 84 | return stalled |
| 83 | 85 | ||
| 84 | 86 | ||
| 87 | def capture(path, offset): | ||
| 88 | rows = [] | ||
| 89 | rotated = False | ||
| 90 | if path.exists(): | ||
| 91 | with path.open("r+b") as file: | ||
| 92 | if path.stat().st_size < offset: | ||
| 93 | offset = 0 | ||
| 94 | file.seek(offset) | ||
| 95 | data = file.read(8 * 1024 * 1024) | ||
| 96 | end = data.rfind(b"\n") + 1 | ||
| 97 | offset += end | ||
| 98 | for line in data[:end].splitlines(): | ||
| 99 | if row := packet(line.decode("ascii", errors="replace")): | ||
| 100 | rows.append(row) | ||
| 101 | if offset >= 64 * 1024 * 1024 and offset == path.stat().st_size: | ||
| 102 | # tcpdump holds an append descriptor; truncation preserves its capture privileges. | ||
| 103 | file.seek(0) | ||
| 104 | file.truncate() | ||
| 105 | offset = 0 | ||
| 106 | rotated = True | ||
| 107 | return rows, offset, rotated | ||
| 108 | |||
| 109 | |||
| 85 | def main(): | 110 | def main(): |
| 86 | os.umask(0o077) | 111 | os.umask(0o077) |
| 87 | ROOT.mkdir(mode=0o700, parents=True, exist_ok=True) | 112 | ROOT.mkdir(mode=0o700, parents=True, exist_ok=True) |
| 88 | targets = set() | 113 | targets = set() |
| 89 | samples = deque(maxlen=120) | 114 | samples = deque(maxlen=120) |
| 115 | packets = deque(maxlen=100000) | ||
| 90 | stalled = set() | 116 | stalled = set() |
| 117 | capture_offset = 0 | ||
| 91 | resolved = measured = 0 | 118 | resolved = measured = 0 |
| 92 | while True: | 119 | while True: |
| 93 | now = time.time() | 120 | now = time.time() |
| ... | @@ -99,6 +126,12 @@ def main(): | ... | @@ -99,6 +126,12 @@ def main(): |
| 99 | result = subprocess.run(["netstat", "-anv", "-p", "tcp"], check=True, capture_output=True, text=True, timeout=5) | 126 | result = subprocess.run(["netstat", "-anv", "-p", "tcp"], check=True, capture_output=True, text=True, timeout=5) |
| 100 | rows = sockets(result.stdout, targets) | 127 | rows = sockets(result.stdout, targets) |
| 101 | record["connections"] = rows | 128 | record["connections"] = rows |
| 129 | captured, capture_offset, rotated = capture(ROOT / "tcp-headers.log", capture_offset) | ||
| 130 | packets.extend(captured) | ||
| 131 | while packets and packets[0]["ts"] < now - 120: | ||
| 132 | packets.popleft() | ||
| 133 | if rotated: | ||
| 134 | record["capture_rotated"] = True | ||
| 102 | if now - measured >= 5: | 135 | if now - measured >= 5: |
| 103 | pids = sorted({row["pid"] for row in rows if row["pid"]}) | 136 | pids = sorted({row["pid"] for row in rows if row["pid"]}) |
| 104 | if pids: | 137 | if pids: |
| ... | @@ -121,7 +154,9 @@ def main(): | ... | @@ -121,7 +154,9 @@ def main(): |
| 121 | if added := current - stalled: | 154 | if added := current - stalled: |
| 122 | path = ROOT / ("stall-" + str(int(now * 1000)) + ".json") | 155 | path = ROOT / ("stall-" + str(int(now * 1000)) + ".json") |
| 123 | pending = path.with_suffix(".pending") | 156 | pending = path.with_suffix(".pending") |
| 124 | pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples)}, separators=(",", ":"))) | 157 | endpoints = {f"[{local.rpartition('.')[0]}]:{local.rpartition('.')[2]}" for local, _, _ in added} |
| 158 | pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples), | ||
| 159 | "packets": [row for row in packets if endpoints.intersection((row["source"], row["destination"]))]}, separators=(",", ":"))) | ||
| 125 | pending.replace(path) | 160 | pending.replace(path) |
| 126 | for old in sorted(ROOT.glob("stall-*.json"))[:-100]: | 161 | for old in sorted(ROOT.glob("stall-*.json"))[:-100]: |
| 127 | old.unlink() | 162 | old.unlink() |
tools/substation-watch-test.py+4| ... | @@ -56,6 +56,10 @@ class TransportEvidence(unittest.TestCase): | ... | @@ -56,6 +56,10 @@ class TransportEvidence(unittest.TestCase): |
| 56 | self.assertEqual(row["flags"], "R.") | 56 | self.assertEqual(row["flags"], "R.") |
| 57 | self.assertEqual(row["sack"], ["50:60"]) | 57 | self.assertEqual(row["sack"], ["50:60"]) |
| 58 | self.assertNotIn("private", json.dumps(row)) | 58 | self.assertNotIn("private", json.dumps(row)) |
| 59 | mac = watch.packet("190.123 IP 192.0.2.1.50000 > 192.0.2.5.443: Flags [R.], seq 10:20, ack 30, win 40, options [sack 1 {50:60}], length 10") | ||
| 60 | self.assertEqual(mac, row) | ||
| 61 | ipv6 = watch.packet("190.123 IP6 2001:db8::1.50000 > 2001:db8::5.443: Flags [.], ack 30, win 40, length 0") | ||
| 62 | self.assertEqual(ipv6["source"], "[2001:db8::1]:50000") | ||
| 59 | edge = {"ts": 200, "duration": 60, "client": "192.0.2.1:50000"} | 63 | edge = {"ts": 200, "duration": 60, "client": "192.0.2.1:50000"} |
| 60 | self.assertEqual(watch.incident(edge, [], [row])["packets"], [row]) | 64 | self.assertEqual(watch.incident(edge, [], [row])["packets"], [row]) |
| 61 | edge["client"] = "192.0.2.1:50001" | 65 | edge["client"] = "192.0.2.1:50001" |
tools/substation-watch.py+2-21| ... | @@ -10,6 +10,8 @@ import threading | ... | @@ -10,6 +10,8 @@ import threading |
| 10 | import time | 10 | import time |
| 11 | from urllib.request import Request, urlopen | 11 | from urllib.request import Request, urlopen |
| 12 | 12 | ||
| 13 | from substation_packets import packet | ||
| 14 | |||
| 13 | 15 | ||
| 14 | ROOT = Path("/var/lib/snowglobe/substation-watch") | 16 | ROOT = Path("/var/lib/snowglobe/substation-watch") |
| 15 | EDGE_FIELDS = ("ts", "status", "bytes_read", "content_length", "duration", "request_id", | 17 | EDGE_FIELDS = ("ts", "status", "bytes_read", "content_length", "duration", "request_id", |
| ... | @@ -68,27 +70,6 @@ def event(entry, host): | ... | @@ -68,27 +70,6 @@ def event(entry, host): |
| 68 | return None, None | 70 | return None, None |
| 69 | 71 | ||
| 70 | 72 | ||
| 71 | def packet(line): | ||
| 72 | match = re.fullmatch(r"([0-9.]+) \S+\s+(?:In|Out)\s+IP6? (\S+) > (\S+): Flags \[([A-Za-z.*]+)\], (.*)", line.strip()) | ||
| 73 | if not match: | ||
| 74 | return None | ||
| 75 | stamp, source, destination, flags, details = match.groups() | ||
| 76 | endpoints = {} | ||
| 77 | try: | ||
| 78 | for key, value in (("source", source), ("destination", destination)): | ||
| 79 | host, _, port = value.rpartition(".") | ||
| 80 | host = ipaddress.ip_address(host) | ||
| 81 | endpoints[key] = f"[{host}]:{int(port)}" | ||
| 82 | except ValueError: | ||
| 83 | return None | ||
| 84 | row = {"ts": float(stamp), **endpoints, "flags": flags} | ||
| 85 | for key in ("seq", "ack", "win", "length"): | ||
| 86 | if value := re.search(r"\b" + key + r" ([0-9]+(?::[0-9]+)?)", details): | ||
| 87 | row[key] = value[1] | ||
| 88 | row["sack"] = re.findall(r"\{([0-9]+:[0-9]+)\}", details) | ||
| 89 | return row | ||
| 90 | |||
| 91 | |||
| 92 | def incident(edge, samples, packets): | 73 | def incident(edge, samples, packets): |
| 93 | length = edge.get("content_length") | 74 | length = edge.get("content_length") |
| 94 | complete = int(length) == edge["bytes_read"] if str(length).isdigit() else None | 75 | complete = int(length) == edge["bytes_read"] if str(length).isdigit() else None |
tools/substation.md+5| ... | @@ -148,3 +148,8 @@ Match UTC timestamps to the server incident before comparing TCP progress. | ... | @@ -148,3 +148,8 @@ Match UTC timestamps to the server incident before comparing TCP progress. |
| 148 | An upload with queued bytes and no acknowledgement progress for 10 seconds | 148 | An upload with queued bytes and no acknowledgement progress for 10 seconds |
| 149 | retains the preceding two minutes in a private `stall-<timestamp>.json` file. | 149 | retains the preceding two minutes in a private `stall-<timestamp>.json` file. |
| 150 | The latest 100 stalls survive rolling-history expiry and watcher restarts. | 150 | The latest 100 stalls survive rolling-history expiry and watcher restarts. |
| 151 | When an administrator starts a TCP summary capture into `tcp-headers.log`, the | ||
| 152 | client watcher retains matching sequence/ACK/SACK metadata with each stall. | ||
| 153 | It consumes and truncates that append-only log at 64 MiB without restarting | ||
| 154 | tcpdump. Samples mark each truncation; exclude that instant from packet-loss | ||
| 155 | conclusions because concurrent writes can be lost during truncation. |
tools/substation_packets.py created+23| ... | @@ -0,0 +1,23 @@ | ||
| 1 | import ipaddress | ||
| 2 | import re | ||
| 3 | |||
| 4 | |||
| 5 | def packet(line): | ||
| 6 | match = re.fullmatch(r"([0-9.]+) (?:\S+\s+(?:In|Out)\s+)?IP6? (\S+) > (\S+): Flags \[([A-Za-z.*]+)\], (.*)", line.strip()) | ||
| 7 | if not match: | ||
| 8 | return None | ||
| 9 | stamp, source, destination, flags, details = match.groups() | ||
| 10 | endpoints = {} | ||
| 11 | try: | ||
| 12 | for key, value in (("source", source), ("destination", destination)): | ||
| 13 | host, _, port = value.rpartition(".") | ||
| 14 | host = ipaddress.ip_address(host) | ||
| 15 | endpoints[key] = f"[{host}]:{int(port)}" | ||
| 16 | except ValueError: | ||
| 17 | return None | ||
| 18 | row = {"ts": float(stamp), **endpoints, "flags": flags} | ||
| 19 | for key in ("seq", "ack", "win", "length"): | ||
| 20 | if value := re.search(r"\b" + key + r" ([0-9]+(?::[0-9]+)?)", details): | ||
| 21 | row[key] = value[1] | ||
| 22 | row["sack"] = re.findall(r"\{([0-9]+:[0-9]+)\}", details) | ||
| 23 | return row | ||