authoravatarclover caruso<git@paperclover.net>2026-11-12 11:00:46-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-12 16:08:55-08:00
log9f7ec5c28c4d001c4d36e7c68550acb86e533950
treec3791dc164947efd2c354c648f2bdd12a9ba7deb
parent029fe21eabd885646110ce6c0991a7dc1be99bfe
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Retain complete Worktop upload stall episodes

Give each stalled socket one archive, then update it after stall detection ends and packet ingestion catches up. Retain late retransmissions and connection closure instead of only the first ten seconds. Skip expired packet records before parsing so restart catch-up does not spend CPU on discarded history. Thirteen watcher tests passed locally and eight client tests passed on Worktop. A disposable staging connection with scoped packet loss updated the same archive through its reset, retaining 2,473 matching packets with zero backlog at finalization. The firewall fixture was removed and private evidence retained. Staging health passed. Worktop runs the exported client revision. Assisted-by: gpt-6.1-sol

3 files changed, 39 insertions(+), 9 deletions(-)

tools/substation-client-watch-test.py+13
......@@ -13,6 +13,19 @@ spec.loader.exec_module(watch)
1313
1414
1515class ClientEvidence(unittest.TestCase):
16 def setUp(self):
17 clock = patch.object(watch.time, "time", return_value=200)
18 clock.start()
19 self.addCleanup(clock.stop)
20
21 def test_capture_skips_expired_metadata_and_still_advances(self):
22 with tempfile.TemporaryDirectory() as temporary:
23 path = Path(temporary) / "capture.log"
24 path.write_text("invalid line\n1.0 IP 192.0.2.5.443 > 192.0.2.1.50000: Flags [.], ack 20, win 40, length 0\n")
25 rows, offset, _ = watch.capture(path, 0)
26 self.assertEqual(rows, [])
27 self.assertEqual(offset, path.stat().st_size)
28
1629 def test_capture_keeps_partial_lines_and_excludes_payload(self):
1730 with tempfile.TemporaryDirectory() as temporary:
1831 path = Path(temporary) / "capture.log"
tools/substation-client-watch.py+21-7
......@@ -102,7 +102,13 @@ def capture(path, offset):
102102 file.truncate()
103103 offset = 0
104104 truncated_at = time.time()
105 cutoff = time.time() - 120
105106 for line in data[:end].splitlines():
107 try:
108 if float(line.partition(b" ")[0]) < cutoff:
109 continue
110 except ValueError:
111 continue
106112 if row := packet(line.decode("ascii", errors="replace")):
107113 rows.append(row)
108114 return rows, offset, truncated_at
......@@ -114,7 +120,7 @@ def main():
114120 targets = set()
115121 samples = deque(maxlen=120)
116122 packets = deque(maxlen=100000)
117 stalled = set()
123 stalled = {}
118124 capture_state = {"offset": 0, "busy": False}
119125 capture_gaps = deque(maxlen=100)
120126 capture_path = ROOT / "tcp-headers.log"
......@@ -173,17 +179,25 @@ def main():
173179 file.write(json.dumps(record, separators=(",", ":")) + "\n")
174180 samples.append(record)
175181 current = stalls(samples)
176 if added := current - stalled:
177 path = ROOT / ("stall-" + str(int(now * 1000)) + ".json")
182 for key in current | stalled.keys():
183 if key in current:
184 if key in stalled:
185 continue
186 path = ROOT / f"stall-{int(now * 1000)}-{key[2]}-{key[0].rpartition('.')[2]}.json"
187 stalled[key] = path
188 else:
189 if any(record.get(field) for field in ("error", "capture_error", "capture_busy", "capture_backlog")):
190 continue
191 path = stalled.pop(key)
178192 pending = path.with_suffix(".pending")
179 endpoints = {f"[{local.rpartition('.')[0]}]:{local.rpartition('.')[2]}" for local, _, _ in added}
180 pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples),
193 local, _, port = key[0].rpartition(".")
194 endpoint = f"[{local}]:{port}"
195 pending.write_text(json.dumps({"ts": now, "connections": [key], "samples": list(samples),
181196 "capture_gaps": [ts for ts in list(capture_gaps) if ts >= now - 120],
182 "packets": [row for row in list(packets) if endpoints.intersection((row["source"], row["destination"]))]}, separators=(",", ":")))
197 "packets": [row for row in list(packets) if endpoint in (row["source"], row["destination"])]}, separators=(",", ":")))
183198 pending.replace(path)
184199 for old in sorted(ROOT.glob("stall-*.json"))[:-100]:
185200 old.unlink()
186 stalled = current
187201 for path in sorted(ROOT.glob("????????T????.jsonl"))[:-720]:
188202 path.unlink()
189203 time.sleep(1)
tools/substation.md+5-2
......@@ -146,10 +146,13 @@ The installed script is an exported revision of `tools/substation-client-watch.p
146146the LaunchAgent follows its `~/.local/share/substation-watch/current.py` link.
147147Match UTC timestamps to the server incident before comparing TCP progress.
148148An upload with queued bytes and no acknowledgement progress for 10 seconds
149retains the preceding two minutes in a private `stall-<timestamp>.json` file.
149retains the preceding 120 socket samples in a private `stall-<timestamp>.json` file.
150150The latest 100 stalls survive rolling-history expiry and watcher restarts.
151Each stalled socket has one archive, updated after the stall and after packet
152ingestion catches up, retaining later retransmissions and closure.
151153When an administrator starts a TCP summary capture into `tcp-headers.log`, the
152client watcher retains matching sequence/ACK/SACK metadata with each stall.
154client watcher retains up to two minutes of matching sequence/ACK/SACK metadata
155with each stall, bounded to 100,000 packets.
153156It consumes and truncates that append-only log at 64 MiB without restarting
154157tcpdump. Archives mark each truncation; exclude that instant from packet-loss
155158conclusions because concurrent writes can be lost during truncation.