| author | |
| committer | |
| log | 9f7ec5c28c4d001c4d36e7c68550acb86e533950 |
| tree | c3791dc164947efd2c354c648f2bdd12a9ba7deb |
| parent | 029fe21eabd885646110ce6c0991a7dc1be99bfe |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Give each stalled socket one archive, then update it after stall detection ends
and packet ingestion catches up. Retain late retransmissions and connection
closure instead of only the first ten seconds. Skip expired packet records
before parsing so restart catch-up does not spend CPU on discarded history.
Thirteen watcher tests passed locally and eight client tests passed on Worktop.
A disposable staging connection with scoped packet loss updated the same
archive through its reset, retaining 2,473 matching packets with zero backlog
at finalization. The firewall fixture was removed and private evidence retained.
Staging health passed. Worktop runs the exported client revision.
Assisted-by: gpt-6.1-sol3 files changed, 39 insertions(+), 9 deletions(-)
tools/substation-client-watch-test.py+13| ... | @@ -13,6 +13,19 @@ spec.loader.exec_module(watch) | ... | @@ -13,6 +13,19 @@ spec.loader.exec_module(watch) |
| 13 | 13 | ||
| 14 | 14 | ||
| 15 | class ClientEvidence(unittest.TestCase): | 15 | class ClientEvidence(unittest.TestCase): |
| 16 | def setUp(self): | ||
| 17 | clock = patch.object(watch.time, "time", return_value=200) | ||
| 18 | clock.start() | ||
| 19 | self.addCleanup(clock.stop) | ||
| 20 | |||
| 21 | def test_capture_skips_expired_metadata_and_still_advances(self): | ||
| 22 | with tempfile.TemporaryDirectory() as temporary: | ||
| 23 | path = Path(temporary) / "capture.log" | ||
| 24 | path.write_text("invalid line\n1.0 IP 192.0.2.5.443 > 192.0.2.1.50000: Flags [.], ack 20, win 40, length 0\n") | ||
| 25 | rows, offset, _ = watch.capture(path, 0) | ||
| 26 | self.assertEqual(rows, []) | ||
| 27 | self.assertEqual(offset, path.stat().st_size) | ||
| 28 | |||
| 16 | def test_capture_keeps_partial_lines_and_excludes_payload(self): | 29 | def test_capture_keeps_partial_lines_and_excludes_payload(self): |
| 17 | with tempfile.TemporaryDirectory() as temporary: | 30 | with tempfile.TemporaryDirectory() as temporary: |
| 18 | path = Path(temporary) / "capture.log" | 31 | path = Path(temporary) / "capture.log" |
tools/substation-client-watch.py+21-7| ... | @@ -102,7 +102,13 @@ def capture(path, offset): | ... | @@ -102,7 +102,13 @@ def capture(path, offset): |
| 102 | file.truncate() | 102 | file.truncate() |
| 103 | offset = 0 | 103 | offset = 0 |
| 104 | truncated_at = time.time() | 104 | truncated_at = time.time() |
| 105 | cutoff = time.time() - 120 | ||
| 105 | for line in data[:end].splitlines(): | 106 | for line in data[:end].splitlines(): |
| 107 | try: | ||
| 108 | if float(line.partition(b" ")[0]) < cutoff: | ||
| 109 | continue | ||
| 110 | except ValueError: | ||
| 111 | continue | ||
| 106 | if row := packet(line.decode("ascii", errors="replace")): | 112 | if row := packet(line.decode("ascii", errors="replace")): |
| 107 | rows.append(row) | 113 | rows.append(row) |
| 108 | return rows, offset, truncated_at | 114 | return rows, offset, truncated_at |
| ... | @@ -114,7 +120,7 @@ def main(): | ... | @@ -114,7 +120,7 @@ def main(): |
| 114 | targets = set() | 120 | targets = set() |
| 115 | samples = deque(maxlen=120) | 121 | samples = deque(maxlen=120) |
| 116 | packets = deque(maxlen=100000) | 122 | packets = deque(maxlen=100000) |
| 117 | stalled = set() | 123 | stalled = {} |
| 118 | capture_state = {"offset": 0, "busy": False} | 124 | capture_state = {"offset": 0, "busy": False} |
| 119 | capture_gaps = deque(maxlen=100) | 125 | capture_gaps = deque(maxlen=100) |
| 120 | capture_path = ROOT / "tcp-headers.log" | 126 | capture_path = ROOT / "tcp-headers.log" |
| ... | @@ -173,17 +179,25 @@ def main(): | ... | @@ -173,17 +179,25 @@ def main(): |
| 173 | file.write(json.dumps(record, separators=(",", ":")) + "\n") | 179 | file.write(json.dumps(record, separators=(",", ":")) + "\n") |
| 174 | samples.append(record) | 180 | samples.append(record) |
| 175 | current = stalls(samples) | 181 | current = stalls(samples) |
| 176 | if added := current - stalled: | 182 | for key in current | stalled.keys(): |
| 177 | path = ROOT / ("stall-" + str(int(now * 1000)) + ".json") | 183 | if key in current: |
| 184 | if key in stalled: | ||
| 185 | continue | ||
| 186 | path = ROOT / f"stall-{int(now * 1000)}-{key[2]}-{key[0].rpartition('.')[2]}.json" | ||
| 187 | stalled[key] = path | ||
| 188 | else: | ||
| 189 | if any(record.get(field) for field in ("error", "capture_error", "capture_busy", "capture_backlog")): | ||
| 190 | continue | ||
| 191 | path = stalled.pop(key) | ||
| 178 | pending = path.with_suffix(".pending") | 192 | pending = path.with_suffix(".pending") |
| 179 | endpoints = {f"[{local.rpartition('.')[0]}]:{local.rpartition('.')[2]}" for local, _, _ in added} | 193 | local, _, port = key[0].rpartition(".") |
| 180 | pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples), | 194 | endpoint = f"[{local}]:{port}" |
| 195 | pending.write_text(json.dumps({"ts": now, "connections": [key], "samples": list(samples), | ||
| 181 | "capture_gaps": [ts for ts in list(capture_gaps) if ts >= now - 120], | 196 | "capture_gaps": [ts for ts in list(capture_gaps) if ts >= now - 120], |
| 182 | "packets": [row for row in list(packets) if endpoints.intersection((row["source"], row["destination"]))]}, separators=(",", ":"))) | 197 | "packets": [row for row in list(packets) if endpoint in (row["source"], row["destination"])]}, separators=(",", ":"))) |
| 183 | pending.replace(path) | 198 | pending.replace(path) |
| 184 | for old in sorted(ROOT.glob("stall-*.json"))[:-100]: | 199 | for old in sorted(ROOT.glob("stall-*.json"))[:-100]: |
| 185 | old.unlink() | 200 | old.unlink() |
| 186 | stalled = current | ||
| 187 | for path in sorted(ROOT.glob("????????T????.jsonl"))[:-720]: | 201 | for path in sorted(ROOT.glob("????????T????.jsonl"))[:-720]: |
| 188 | path.unlink() | 202 | path.unlink() |
| 189 | time.sleep(1) | 203 | time.sleep(1) |
tools/substation.md+5-2| ... | @@ -146,10 +146,13 @@ The installed script is an exported revision of `tools/substation-client-watch.p | ... | @@ -146,10 +146,13 @@ The installed script is an exported revision of `tools/substation-client-watch.p |
| 146 | the LaunchAgent follows its `~/.local/share/substation-watch/current.py` link. | 146 | the LaunchAgent follows its `~/.local/share/substation-watch/current.py` link. |
| 147 | Match UTC timestamps to the server incident before comparing TCP progress. | 147 | Match UTC timestamps to the server incident before comparing TCP progress. |
| 148 | An upload with queued bytes and no acknowledgement progress for 10 seconds | 148 | An upload with queued bytes and no acknowledgement progress for 10 seconds |
| 149 | retains the preceding two minutes in a private `stall-<timestamp>.json` file. | 149 | retains the preceding 120 socket samples in a private `stall-<timestamp>.json` file. |
| 150 | The latest 100 stalls survive rolling-history expiry and watcher restarts. | 150 | The latest 100 stalls survive rolling-history expiry and watcher restarts. |
| 151 | Each stalled socket has one archive, updated after the stall and after packet | ||
| 152 | ingestion catches up, retaining later retransmissions and closure. | ||
| 151 | When an administrator starts a TCP summary capture into `tcp-headers.log`, the | 153 | When an administrator starts a TCP summary capture into `tcp-headers.log`, the |
| 152 | client watcher retains matching sequence/ACK/SACK metadata with each stall. | 154 | client watcher retains up to two minutes of matching sequence/ACK/SACK metadata |
| 155 | with each stall, bounded to 100,000 packets. | ||
| 153 | It consumes and truncates that append-only log at 64 MiB without restarting | 156 | It consumes and truncates that append-only log at 64 MiB without restarting |
| 154 | tcpdump. Archives mark each truncation; exclude that instant from packet-loss | 157 | tcpdump. Archives mark each truncation; exclude that instant from packet-loss |
| 155 | conclusions because concurrent writes can be lost during truncation. | 158 | conclusions because concurrent writes can be lost during truncation. |