authoravatarclover caruso<git@paperclover.net>2026-11-11 16:43:57-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-12 16:08:55-08:00
log3b735d29265d065eff0ee72b7943eb7795f87bbb
tree67913067d74065aa840c9f184bd7267e193ec54a
parent9ffb8cb4f4f5c06bc6240d3f27ef6e0b71e54640
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Retain Worktop upload stalls before edge timeouts

Archive sender-side evidence when queued upload bytes make no acknowledgement progress for ten seconds. macOS txbytes includes queued bytes; derive ACK progress by subtracting Send-Q so newly buffered data cannot hide a stall. Keep the preceding 120 samples in a private atomic archive on each stall transition, bounded to 100 incidents independently of the rolling history. Four client-watcher tests passed locally and on Worktop. A disposable staging connection with scoped, automatically cleaned packet loss produced one private archive before its TCP connection closed; the archive survived watcher restart. Staging health passed. Separate native Claude controls confirmed recovery from an early empty-body 499 using the harness retry defaults and the production hint. Assisted-by: gpt-6.1-sol

3 files changed, 55 insertions(+), 0 deletions(-)

tools/substation-client-watch-test.py+20
......@@ -38,6 +38,26 @@ class ClientEvidence(unittest.TestCase):
3838 self.assertEqual(rows[0]["tx_win"], "1000000")
3939 self.assertEqual(watch.address("::ffff:198.51.100.1"), "198.51.100.1")
4040
41 def test_stall_retains_unacknowledged_upload_despite_new_buffered_bytes(self):
42 row = {"local": "192.0.2.1.50000", "peer": "198.51.100.1.443", "pid": 42, "send_q": 100, "bytes_sent": 200}
43 samples = [{"ts": 0, "connections": [row]},
44 {"ts": 5, "connections": [{**row, "send_q": 150, "bytes_sent": 250}]},
45 {"ts": 10, "connections": [{**row, "send_q": 200, "bytes_sent": 300}]}]
46 self.assertEqual(watch.stalls(samples), {(row["local"], row["peer"], 42)})
47 self.assertEqual(watch.stalls(samples[:-1]), set())
48 samples[-1]["connections"][0]["send_q"] = 190
49 self.assertEqual(watch.stalls(samples), set())
50
51 def test_idle_socket_and_reused_connection_do_not_count_as_stalls(self):
52 row = {"local": "192.0.2.1.50000", "peer": "198.51.100.1.443", "pid": 42, "send_q": 0, "bytes_sent": 200}
53 samples = [{"ts": 0, "connections": [row]}, {"ts": 10, "connections": [row]}]
54 self.assertEqual(watch.stalls(samples), set())
55 samples[0]["connections"] = [{**row, "send_q": 100, "pid": 41}]
56 samples[-1]["connections"] = [{**row, "send_q": 100}]
57 self.assertEqual(watch.stalls(samples), set())
58 samples.insert(1, {"ts": 5, "connections": []})
59 self.assertEqual(watch.stalls(samples), set())
60
4161
4262if __name__ == "__main__":
4363 unittest.main()
tools/substation-client-watch.py+32
......@@ -1,5 +1,6 @@
11#!/usr/bin/env python3
22import csv
3from collections import deque
34from datetime import datetime, timezone
45import ipaddress
56import json
......@@ -62,10 +63,31 @@ def counters(output, targets):
6263 return rows
6364
6465
66def stalls(samples):
67 latest = samples[-1]
68 stalled = set()
69 for row in latest.get("connections", []):
70 if not row["pid"] or not row["send_q"]:
71 continue
72 key = (row["local"], row["peer"], row["pid"])
73 acknowledged = row["bytes_sent"] - row["send_q"]
74 since = latest["ts"]
75 for sample in reversed(samples):
76 prior = next((r for r in sample.get("connections", []) if (r["local"], r["peer"], r["pid"]) == key), None)
77 if not prior or not prior["send_q"] or prior["bytes_sent"] - prior["send_q"] != acknowledged:
78 break
79 since = sample["ts"]
80 if latest["ts"] - since >= 10:
81 stalled.add(key)
82 return stalled
83
84
6585def main():
6686 os.umask(0o077)
6787 ROOT.mkdir(mode=0o700, parents=True, exist_ok=True)
6888 targets = set()
89 samples = deque(maxlen=120)
90 stalled = set()
6991 resolved = measured = 0
7092 while True:
7193 now = time.time()
......@@ -94,6 +116,16 @@ def main():
94116 stamp = datetime.fromtimestamp(now, timezone.utc).strftime("%Y%m%dT%H%M")
95117 with (ROOT / (stamp + ".jsonl")).open("a") as file:
96118 file.write(json.dumps(record, separators=(",", ":")) + "\n")
119 samples.append(record)
120 current = stalls(samples)
121 if added := current - stalled:
122 path = ROOT / ("stall-" + str(int(now * 1000)) + ".json")
123 pending = path.with_suffix(".pending")
124 pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples)}, separators=(",", ":")))
125 pending.replace(path)
126 for old in sorted(ROOT.glob("stall-*.json"))[:-100]:
127 old.unlink()
128 stalled = current
97129 for path in sorted(ROOT.glob("????????T????.jsonl"))[:-720]:
98130 path.unlink()
99131 time.sleep(1)
tools/substation.md+3
......@@ -145,3 +145,6 @@ retransmissions and windows.
145145The installed script is an exported revision of `tools/substation-client-watch.py`;
146146the LaunchAgent follows its `~/.local/share/substation-watch/current.py` link.
147147Match UTC timestamps to the server incident before comparing TCP progress.
148An upload with queued bytes and no acknowledgement progress for 10 seconds
149retains the preceding two minutes in a private `stall-<timestamp>.json` file.
150The latest 100 stalls survive rolling-history expiry and watcher restarts.