| author | |
| committer | |
| log | 3bc44ef75c76cdead70fd5f67b61325c5618f368 |
| tree | dcbb4a790569e1584d16c085bdb7a00ad97cd4d9 |
| parent | 9cd6ce8c7d9ad4395f6deb4ef710c060029c077f |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Retain sender-side socket counters, process IDs, retransmissions and windows
for Substation connections, without command lines or application payloads.
Bound private minute archives to 12 hours and supervise the exported script
with a Worktop LaunchAgent so captures survive app and agent restarts.
Validated parser privacy and filtering, PID handling, real netstat and nettop
output, source installation, LaunchAgent startup and ongoing private capture.
Assisted-by: gpt-6.1-sol3 files changed, 157 insertions(+), 2 deletions(-)
tools/substation-client-watch-test.py created+43| ... | @@ -0,0 +1,43 @@ | ||
| 1 | #!/usr/bin/env python3 | ||
| 2 | import importlib.util | ||
| 3 | import json | ||
| 4 | from pathlib import Path | ||
| 5 | import unittest | ||
| 6 | |||
| 7 | |||
| 8 | spec = importlib.util.spec_from_file_location("watch", Path(__file__).with_name("substation-client-watch.py")) | ||
| 9 | watch = importlib.util.module_from_spec(spec) | ||
| 10 | spec.loader.exec_module(watch) | ||
| 11 | |||
| 12 | |||
| 13 | class ClientEvidence(unittest.TestCase): | ||
| 14 | def test_socket_counters_and_pids_exclude_other_destinations_and_program_names(self): | ||
| 15 | text = "\n".join([ | ||
| 16 | "tcp4 0 1234 192.0.2.1.50000 198.51.100.1.443 ESTABLISHED 10 20 1000 2000 private-name:42 0 0", | ||
| 17 | "tcp4 0 0 192.0.2.1.50001 198.51.100.2.443 ESTABLISHED 10 20 1000 2000 private-name:43 0 0", | ||
| 18 | "tcp4 0 0 192.0.2.1.50002 198.51.100.1.443 ESTABLISHED 30 40 1000 2000 The Knife:44 0 0", | ||
| 19 | ]) | ||
| 20 | rows = watch.sockets(text, {"198.51.100.1"}) | ||
| 21 | self.assertEqual(len(rows), 2) | ||
| 22 | self.assertEqual(rows[0]["send_q"], 1234) | ||
| 23 | self.assertEqual(rows[0]["pid"], 42) | ||
| 24 | self.assertEqual(rows[1]["pid"], 44) | ||
| 25 | self.assertNotIn("private-name", json.dumps(rows)) | ||
| 26 | |||
| 27 | def test_nettop_retransmissions_windows_and_owner(self): | ||
| 28 | text = "\n".join([ | ||
| 29 | ",bytes_in,bytes_out,rx_dupe,rx_ooo,re-tx,rtt_avg,rcvsize,tx_win,", | ||
| 30 | "2.1.322.42,10,20,0,0,0,,,,", | ||
| 31 | "tcp4 192.0.2.1:50000<->198.51.100.1:443,10,2000000,0,0,123456,20.5 ms,131072,1000000,", | ||
| 32 | "tcp4 192.0.2.1:50001<->198.51.100.2:443,10,20,0,0,0,2 ms,131072,1000000,", | ||
| 33 | ]) | ||
| 34 | rows = watch.counters(text, {"198.51.100.1"}) | ||
| 35 | self.assertEqual(len(rows), 1) | ||
| 36 | self.assertEqual(rows[0]["pid"], 42) | ||
| 37 | self.assertEqual(rows[0]["re-tx"], "123456") | ||
| 38 | self.assertEqual(rows[0]["tx_win"], "1000000") | ||
| 39 | self.assertEqual(watch.address("::ffff:198.51.100.1"), "198.51.100.1") | ||
| 40 | |||
| 41 | |||
| 42 | if __name__ == "__main__": | ||
| 43 | unittest.main() | ||
tools/substation-client-watch.py created+103| ... | @@ -0,0 +1,103 @@ | ||
| 1 | #!/usr/bin/env python3 | ||
| 2 | import csv | ||
| 3 | from datetime import datetime, timezone | ||
| 4 | import ipaddress | ||
| 5 | import json | ||
| 6 | import os | ||
| 7 | from pathlib import Path | ||
| 8 | import re | ||
| 9 | import socket | ||
| 10 | import subprocess | ||
| 11 | import time | ||
| 12 | |||
| 13 | |||
| 14 | ROOT = Path.home() / ".local/state/substation-watch" | ||
| 15 | COLUMNS = "bytes_in,bytes_out,rx_dupe,rx_ooo,re-tx,rtt_avg,rcvsize,tx_win" | ||
| 16 | |||
| 17 | |||
| 18 | def address(value): | ||
| 19 | parsed = ipaddress.ip_address(value.strip("[]")) | ||
| 20 | if isinstance(parsed, ipaddress.IPv6Address) and parsed.ipv4_mapped: | ||
| 21 | parsed = parsed.ipv4_mapped | ||
| 22 | return str(parsed) | ||
| 23 | |||
| 24 | |||
| 25 | def sockets(output, targets): | ||
| 26 | rows = [] | ||
| 27 | for line in output.splitlines(): | ||
| 28 | parts = line.split() | ||
| 29 | if len(parts) < 11 or parts[0] not in {"tcp4", "tcp6"}: | ||
| 30 | continue | ||
| 31 | peer, _, port = parts[4].rpartition(".") | ||
| 32 | if port != "443" or address(peer) not in targets: | ||
| 33 | continue | ||
| 34 | pid = next((int(match[1]) for part in parts[10:] if (match := re.fullmatch(r".*:([0-9]+)", part))), None) | ||
| 35 | rows.append({"local": parts[3], "peer": parts[4], "state": parts[5], "pid": pid, | ||
| 36 | **{key: int(parts[index]) for key, index in | ||
| 37 | (("recv_q", 1), ("send_q", 2), ("bytes_received", 6), ("bytes_sent", 7), ("recv_hiwat", 8), ("send_hiwat", 9))}}) | ||
| 38 | return rows | ||
| 39 | |||
| 40 | |||
| 41 | def counters(output, targets): | ||
| 42 | reader = csv.reader(output.splitlines()) | ||
| 43 | header = next(reader, []) | ||
| 44 | rows = [] | ||
| 45 | pid = None | ||
| 46 | for values in reader: | ||
| 47 | if not values: | ||
| 48 | continue | ||
| 49 | if not values[0].startswith(("tcp4 ", "tcp6 ")): | ||
| 50 | last = values[0].rpartition(".")[2] | ||
| 51 | pid = int(last) if last.isdigit() else None | ||
| 52 | continue | ||
| 53 | peer = values[0].partition("<->")[2] | ||
| 54 | host, _, port = peer.rpartition(":") | ||
| 55 | if port != "443" or address(host) not in targets: | ||
| 56 | continue | ||
| 57 | row = {"connection": values[0], "pid": pid} | ||
| 58 | for key, value in zip(header[1:], values[1:]): | ||
| 59 | if key in COLUMNS.split(",") and re.fullmatch(r"[0-9. ]+(?:ms)?", value): | ||
| 60 | row[key] = value | ||
| 61 | rows.append(row) | ||
| 62 | return rows | ||
| 63 | |||
| 64 | |||
| 65 | def main(): | ||
| 66 | os.umask(0o077) | ||
| 67 | ROOT.mkdir(mode=0o700, parents=True, exist_ok=True) | ||
| 68 | targets = set() | ||
| 69 | resolved = measured = 0 | ||
| 70 | while True: | ||
| 71 | now = time.time() | ||
| 72 | record = {"ts": now} | ||
| 73 | try: | ||
| 74 | if now - resolved > 300: | ||
| 75 | targets = {address(item[4][0]) for item in socket.getaddrinfo("substation.ai.paperclover.net", 443, type=socket.SOCK_STREAM)} | ||
| 76 | resolved = now | ||
| 77 | result = subprocess.run(["netstat", "-anv", "-p", "tcp"], check=True, capture_output=True, text=True, timeout=5) | ||
| 78 | rows = sockets(result.stdout, targets) | ||
| 79 | record["connections"] = rows | ||
| 80 | if now - measured >= 5: | ||
| 81 | pids = sorted({row["pid"] for row in rows if row["pid"]}) | ||
| 82 | if pids: | ||
| 83 | result = subprocess.run(["ps", "-p", ",".join(map(str, pids)), "-o", "pid=,pcpu=,time=,state="], | ||
| 84 | capture_output=True, text=True, timeout=5) | ||
| 85 | record["processes"] = [{"pid": int(parts[0]), "cpu_percent": float(parts[1]), "cpu_time": parts[2], "state": parts[3]} | ||
| 86 | for line in result.stdout.splitlines() if len(parts := line.split()) == 4] | ||
| 87 | result = subprocess.run(["nettop", "-L", "1", "-n", "-x", "-m", "tcp", "-J", COLUMNS, | ||
| 88 | *(part for pid in pids for part in ("-p", str(pid)))], | ||
| 89 | check=True, capture_output=True, text=True, timeout=10) | ||
| 90 | record["tcp"] = counters(result.stdout, targets) | ||
| 91 | measured = now | ||
| 92 | except Exception as error: | ||
| 93 | record["error"] = type(error).__name__ | ||
| 94 | stamp = datetime.fromtimestamp(now, timezone.utc).strftime("%Y%m%dT%H%M") | ||
| 95 | with (ROOT / (stamp + ".jsonl")).open("a") as file: | ||
| 96 | file.write(json.dumps(record, separators=(",", ":")) + "\n") | ||
| 97 | for path in sorted(ROOT.glob("????????T????.jsonl"))[:-720]: | ||
| 98 | path.unlink() | ||
| 99 | time.sleep(1) | ||
| 100 | |||
| 101 | |||
| 102 | if __name__ == "__main__": | ||
| 103 | main() | ||
tools/substation.md+11-2| ... | @@ -111,8 +111,9 @@ request ID and rejects redirects, so an OAuth login page cannot pass as a health | ... | @@ -111,8 +111,9 @@ request ID and rejects redirects, so an OAuth login page cannot pass as a health |
| 111 | API. It sends no generation requests and logs only check outcomes and IDs. | 111 | API. It sends no generation requests and logs only check outcomes and IDs. |
| 112 | 112 | ||
| 113 | Inspect `substation` task `monitor` in the service logs, and search Caddy's | 113 | Inspect `substation` task `monitor` in the service logs, and search Caddy's |
| 114 | journal for the response's request ID. A 499 with no upstream status marks a | 114 | journal for the response's request ID. A 499 with no upstream status can mean |
| 115 | cancellation before upstream headers; an upstream 499 is recorded separately. | 115 | client cancellation or Caddy's request-body idle timeout; an upstream 499 is |
| 116 | recorded separately. | ||
| 116 | Compare `bytes_read` with `content_length` to identify incomplete fixed-length | 117 | Compare `bytes_read` with `content_length` to identify incomplete fixed-length |
| 117 | uploads. A missing length can mean a chunked request; it does not prove completion. | 118 | uploads. A missing length can mean a chunked request; it does not prove completion. |
| 118 | A 200 followed by a truncated stream needs the edge byte count and the hub's | 119 | A 200 followed by a truncated stream needs the edge byte count and the hub's |
| ... | @@ -131,3 +132,11 @@ Watcher heartbeat and incident summaries appear in its journal and Victoria | ... | @@ -131,3 +132,11 @@ Watcher heartbeat and incident summaries appear in its journal and Victoria |
| 131 | Logs. `upload_complete: false` proves an incomplete fixed-length upload; | 132 | Logs. `upload_complete: false` proves an incomplete fixed-length upload; |
| 132 | the saved receive/send queues, byte counters, receive windows, and retransmits | 133 | the saved receive/send queues, byte counters, receive windows, and retransmits |
| 133 | distinguish stalled sending from server backpressure and network loss. | 134 | distinguish stalled sending from server backpressure and network loss. |
| 135 | |||
| 136 | Worktop's `net.paperclover.substation-watch` LaunchAgent records matching | ||
| 137 | Substation sockets under `~/.local/state/substation-watch`. Minute files retain | ||
| 138 | 12 hours of process IDs, CPU time/state, TCP queues, byte counters, | ||
| 139 | retransmissions and windows. | ||
| 140 | The installed script is an exported revision of `tools/substation-client-watch.py`; | ||
| 141 | the LaunchAgent follows its `~/.local/share/substation-watch/current.py` link. | ||
| 142 | Match UTC timestamps to the server incident before comparing TCP progress. |