diff --git a/tools/substation-client-watch-test.py b/tools/substation-client-watch-test.py new file mode 100644 index 0000000000000000000000000000000000000000..c4d71b804d392abb138fd42360d81aa118951808 --- /dev/null +++ b/tools/substation-client-watch-test.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +import importlib.util +import json +from pathlib import Path +import unittest + + +spec = importlib.util.spec_from_file_location("watch", Path(__file__).with_name("substation-client-watch.py")) +watch = importlib.util.module_from_spec(spec) +spec.loader.exec_module(watch) + + +class ClientEvidence(unittest.TestCase): + def test_socket_counters_and_pids_exclude_other_destinations_and_program_names(self): + text = "\n".join([ + "tcp4 0 1234 192.0.2.1.50000 198.51.100.1.443 ESTABLISHED 10 20 1000 2000 private-name:42 0 0", + "tcp4 0 0 192.0.2.1.50001 198.51.100.2.443 ESTABLISHED 10 20 1000 2000 private-name:43 0 0", + "tcp4 0 0 192.0.2.1.50002 198.51.100.1.443 ESTABLISHED 30 40 1000 2000 The Knife:44 0 0", + ]) + rows = watch.sockets(text, {"198.51.100.1"}) + self.assertEqual(len(rows), 2) + self.assertEqual(rows[0]["send_q"], 1234) + self.assertEqual(rows[0]["pid"], 42) + self.assertEqual(rows[1]["pid"], 44) + self.assertNotIn("private-name", json.dumps(rows)) + + def test_nettop_retransmissions_windows_and_owner(self): + text = "\n".join([ + ",bytes_in,bytes_out,rx_dupe,rx_ooo,re-tx,rtt_avg,rcvsize,tx_win,", + "2.1.322.42,10,20,0,0,0,,,,", + "tcp4 192.0.2.1:50000<->198.51.100.1:443,10,2000000,0,0,123456,20.5 ms,131072,1000000,", + "tcp4 192.0.2.1:50001<->198.51.100.2:443,10,20,0,0,0,2 ms,131072,1000000,", + ]) + rows = watch.counters(text, {"198.51.100.1"}) + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["pid"], 42) + self.assertEqual(rows[0]["re-tx"], "123456") + self.assertEqual(rows[0]["tx_win"], "1000000") + self.assertEqual(watch.address("::ffff:198.51.100.1"), "198.51.100.1") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/substation-client-watch.py b/tools/substation-client-watch.py new file mode 100644 index 0000000000000000000000000000000000000000..6e43f70c394b0e54df93506fd9fd77923361755c --- /dev/null +++ b/tools/substation-client-watch.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +import csv +from datetime import datetime, timezone +import ipaddress +import json +import os +from pathlib import Path +import re +import socket +import subprocess +import time + + +ROOT = Path.home() / ".local/state/substation-watch" +COLUMNS = "bytes_in,bytes_out,rx_dupe,rx_ooo,re-tx,rtt_avg,rcvsize,tx_win" + + +def address(value): + parsed = ipaddress.ip_address(value.strip("[]")) + if isinstance(parsed, ipaddress.IPv6Address) and parsed.ipv4_mapped: + parsed = parsed.ipv4_mapped + return str(parsed) + + +def sockets(output, targets): + rows = [] + for line in output.splitlines(): + parts = line.split() + if len(parts) < 11 or parts[0] not in {"tcp4", "tcp6"}: + continue + peer, _, port = parts[4].rpartition(".") + if port != "443" or address(peer) not in targets: + continue + pid = next((int(match[1]) for part in parts[10:] if (match := re.fullmatch(r".*:([0-9]+)", part))), None) + rows.append({"local": parts[3], "peer": parts[4], "state": parts[5], "pid": pid, + **{key: int(parts[index]) for key, index in + (("recv_q", 1), ("send_q", 2), ("bytes_received", 6), ("bytes_sent", 7), ("recv_hiwat", 8), ("send_hiwat", 9))}}) + return rows + + +def counters(output, targets): + reader = csv.reader(output.splitlines()) + header = next(reader, []) + rows = [] + pid = None + for values in reader: + if not values: + continue + if not values[0].startswith(("tcp4 ", "tcp6 ")): + last = values[0].rpartition(".")[2] + pid = int(last) if last.isdigit() else None + continue + peer = values[0].partition("<->")[2] + host, _, port = peer.rpartition(":") + if port != "443" or address(host) not in targets: + continue + row = {"connection": values[0], "pid": pid} + for key, value in zip(header[1:], values[1:]): + if key in COLUMNS.split(",") and re.fullmatch(r"[0-9. ]+(?:ms)?", value): + row[key] = value + rows.append(row) + return rows + + +def main(): + os.umask(0o077) + ROOT.mkdir(mode=0o700, parents=True, exist_ok=True) + targets = set() + resolved = measured = 0 + while True: + now = time.time() + record = {"ts": now} + try: + if now - resolved > 300: + targets = {address(item[4][0]) for item in socket.getaddrinfo("substation.ai.paperclover.net", 443, type=socket.SOCK_STREAM)} + resolved = now + result = subprocess.run(["netstat", "-anv", "-p", "tcp"], check=True, capture_output=True, text=True, timeout=5) + rows = sockets(result.stdout, targets) + record["connections"] = rows + if now - measured >= 5: + pids = sorted({row["pid"] for row in rows if row["pid"]}) + if pids: + result = subprocess.run(["ps", "-p", ",".join(map(str, pids)), "-o", "pid=,pcpu=,time=,state="], + capture_output=True, text=True, timeout=5) + record["processes"] = [{"pid": int(parts[0]), "cpu_percent": float(parts[1]), "cpu_time": parts[2], "state": parts[3]} + for line in result.stdout.splitlines() if len(parts := line.split()) == 4] + result = subprocess.run(["nettop", "-L", "1", "-n", "-x", "-m", "tcp", "-J", COLUMNS, + *(part for pid in pids for part in ("-p", str(pid)))], + check=True, capture_output=True, text=True, timeout=10) + record["tcp"] = counters(result.stdout, targets) + measured = now + except Exception as error: + record["error"] = type(error).__name__ + stamp = datetime.fromtimestamp(now, timezone.utc).strftime("%Y%m%dT%H%M") + with (ROOT / (stamp + ".jsonl")).open("a") as file: + file.write(json.dumps(record, separators=(",", ":")) + "\n") + for path in sorted(ROOT.glob("????????T????.jsonl"))[:-720]: + path.unlink() + time.sleep(1) + + +if __name__ == "__main__": + main() diff --git a/tools/substation.md b/tools/substation.md index 7fbd7a8141983fd8b23b585ee7511c919d793280..cb0cc729943dedbfa5a3b3f539f99f7fb39cf9e6 100644 --- a/tools/substation.md +++ b/tools/substation.md @@ -111,8 +111,9 @@ request ID and rejects redirects, so an OAuth login page cannot pass as a health API. It sends no generation requests and logs only check outcomes and IDs. Inspect `substation` task `monitor` in the service logs, and search Caddy's -journal for the response's request ID. A 499 with no upstream status marks a -cancellation before upstream headers; an upstream 499 is recorded separately. +journal for the response's request ID. A 499 with no upstream status can mean +client cancellation or Caddy's request-body idle timeout; an upstream 499 is +recorded separately. Compare `bytes_read` with `content_length` to identify incomplete fixed-length uploads. A missing length can mean a chunked request; it does not prove completion. A 200 followed by a truncated stream needs the edge byte count and the hub's @@ -131,3 +132,11 @@ Watcher heartbeat and incident summaries appear in its journal and Victoria Logs. `upload_complete: false` proves an incomplete fixed-length upload; the saved receive/send queues, byte counters, receive windows, and retransmits distinguish stalled sending from server backpressure and network loss. + +Worktop's `net.paperclover.substation-watch` LaunchAgent records matching +Substation sockets under `~/.local/state/substation-watch`. Minute files retain +12 hours of process IDs, CPU time/state, TCP queues, byte counters, +retransmissions and windows. +The installed script is an exported revision of `tools/substation-client-watch.py`; +the LaunchAgent follows its `~/.local/share/substation-watch/current.py` link. +Match UTC timestamps to the server incident before comparing TCP progress.