| 1 | #!/usr/bin/env python3 |
| 2 | from collections import deque |
| 3 | import ipaddress |
| 4 | import json |
| 5 | import os |
| 6 | from pathlib import Path |
| 7 | import re |
| 8 | import subprocess |
| 9 | import threading |
| 10 | import time |
| 11 | from urllib.request import Request, urlopen |
| 12 | |
| 13 | from substation_packets import packet |
| 14 | |
| 15 | |
| 16 | ROOT = Path("/var/lib/snowglobe/substation-watch") |
| 17 | EDGE_FIELDS = ("ts", "status", "bytes_read", "content_length", "duration", "request_id", |
| 18 | "method", "path", "protocol", "upstream", "upstream_status", "upstream_header_ms", "client") |
| 19 | TCP_FIELDS = {"bytes_sent", "bytes_acked", "bytes_received", "bytes_retrans", "lastsnd", "lastrcv", "lastack", |
| 20 | "rtt", "rto", "retrans", "snd_wnd", "rcv_wnd", "rcv_space", "rcv_ssthresh", "cwnd", "unacked"} |
| 21 | |
| 22 | |
| 23 | def save(path, value): |
| 24 | pending = path.with_suffix(".pending") |
| 25 | with pending.open("w") as file: |
| 26 | json.dump(value, file, separators=(",", ":")) |
| 27 | pending.chmod(0o600) |
| 28 | pending.replace(path) |
| 29 | |
| 30 | |
| 31 | def endpoint(value): |
| 32 | address, _, port = value.rpartition(":") |
| 33 | address = ipaddress.ip_address(address.strip("[]")) |
| 34 | if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped: |
| 35 | address = address.ipv4_mapped |
| 36 | return str(address), int(port) |
| 37 | |
| 38 | |
| 39 | def connections(output, ports, clients): |
| 40 | rows = [] |
| 41 | for line in output.splitlines(): |
| 42 | parts = line.split() |
| 43 | if len(parts) < 4: |
| 44 | continue |
| 45 | local, peer = endpoint(parts[2]), endpoint(parts[3]) |
| 46 | if not (local[1] in ports or peer[1] in ports or local[1] == 443 and peer[0] in clients): |
| 47 | continue |
| 48 | row = {"local": parts[2], "peer": parts[3], "recv_q": int(parts[0]), "send_q": int(parts[1])} |
| 49 | for part in parts[4:]: |
| 50 | key, _, value = part.partition(":") |
| 51 | if key in TCP_FIELDS and re.fullmatch(r"[0-9./]+", value): |
| 52 | row[key] = value |
| 53 | rows.append(row) |
| 54 | return rows |
| 55 | |
| 56 | |
| 57 | def event(entry, host): |
| 58 | try: |
| 59 | message = json.loads(entry.get("MESSAGE", "")) |
| 60 | except (ValueError, TypeError): |
| 61 | return None, None |
| 62 | if not isinstance(message, dict): |
| 63 | return None, None |
| 64 | if message.get("service") == "substation": |
| 65 | return {key: message.get(key) for key in EDGE_FIELDS}, message.get("client") |
| 66 | request = message.get("request", {}) |
| 67 | if request.get("host") == host and request.get("remote_ip"): |
| 68 | address = str(ipaddress.ip_address(request["remote_ip"])) |
| 69 | return None, f"[{address}]:{int(request['remote_port'])}" |
| 70 | return None, None |
| 71 | |
| 72 | |
| 73 | def incident(edge, samples, packets): |
| 74 | length = edge.get("content_length") |
| 75 | complete = int(length) == edge["bytes_read"] if str(length).isdigit() else None |
| 76 | peer = endpoint(edge["client"]) if edge.get("client") else None |
| 77 | return {"edge": edge, "upload_complete": complete, |
| 78 | "tcp": [sample for sample in samples if edge["ts"] - edge["duration"] - 5 <= sample["ts"] <= edge["ts"] + 5], |
| 79 | "packets": [row for row in list(packets) if edge["ts"] - edge["duration"] - 5 <= row["ts"] <= edge["ts"] + 5 |
| 80 | and peer in (endpoint(row["source"]), endpoint(row["destination"]))]} |
| 81 | |
| 82 | |
| 83 | def main(): |
| 84 | os.umask(0o077) |
| 85 | ROOT.mkdir(mode=0o700, parents=True, exist_ok=True) |
| 86 | state_file = ROOT / "state.json" |
| 87 | state = json.loads(state_file.read_text()) if state_file.exists() else {"clients": {}} |
| 88 | recent = ROOT / "recent.json" |
| 89 | samples = deque(json.loads(recent.read_text()) if recent.exists() and time.time() - recent.stat().st_mtime < 120 else [], maxlen=120) |
| 90 | packets = deque(maxlen=100000) |
| 91 | capture = None |
| 92 | captured_clients = None |
| 93 | |
| 94 | def read_packets(stream): |
| 95 | for line in stream: |
| 96 | row = packet(line) |
| 97 | if row and any(endpoint(row[key])[0] in state["clients"] for key in ("source", "destination")): |
| 98 | packets.append(row) |
| 99 | |
| 100 | ports = set() |
| 101 | refreshed = saved = reported = 0 |
| 102 | host = "substation.ai." + os.environ["SNOWGLOBE_DOMAIN"] |
| 103 | while True: |
| 104 | try: |
| 105 | now = time.time() |
| 106 | clients = tuple(sorted(state["clients"])) |
| 107 | if capture is None or capture.poll() is not None or clients != captured_clients: |
| 108 | if capture is not None: |
| 109 | capture.terminate() |
| 110 | capture.wait(timeout=2) |
| 111 | filters = "tcp port 443 and (" + " or ".join("host " + client for client in (clients or ("127.0.0.1",))) + ")" |
| 112 | capture = subprocess.Popen(["tcpdump", "-l", "-tt", "-nn", "-S", "-i", "any", "-s", "128", filters], |
| 113 | stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) |
| 114 | threading.Thread(target=read_packets, args=(capture.stdout,), daemon=True).start() |
| 115 | captured_clients = clients |
| 116 | if now - refreshed >= 10: |
| 117 | token = Path("/var/lib/snowglobe/nomad.token").read_text().strip() |
| 118 | request = Request("http://127.0.0.1:4646/v1/service/substation-hub", headers={"X-Nomad-Token": token}) |
| 119 | with urlopen(request, timeout=5) as response: |
| 120 | ports = {item["Port"] for item in json.load(response) |
| 121 | if item.get("JobID") == "substation" and ipaddress.ip_address(item["Address"]).is_loopback} |
| 122 | refreshed = now |
| 123 | command = ["journalctl", "-u", "caddy", "--no-pager", "--output=json"] |
| 124 | command += ["--after-cursor=" + state["cursor"]] if state.get("cursor") else ["--since=-1 hour"] |
| 125 | journal = subprocess.run(command, capture_output=True, text=True, timeout=10) |
| 126 | if journal.returncode: |
| 127 | if "cursor" in journal.stderr.lower() and state.pop("cursor", None): |
| 128 | print(json.dumps({"event": "substation-watcher-journal-gap"}), flush=True) |
| 129 | journal.check_returncode() |
| 130 | for line in journal.stdout.splitlines(): |
| 131 | entry = json.loads(line) |
| 132 | edge, client = event(entry, host) |
| 133 | if client: |
| 134 | state["clients"][endpoint(client)[0]] = now |
| 135 | if edge and edge.get("status", 0) >= 499 and re.fullmatch(r"[0-9a-f-]{36}", str(edge.get("request_id", ""))): |
| 136 | path = ROOT / (edge["request_id"] + ".json") |
| 137 | if not path.exists(): |
| 138 | record = incident(edge, samples, packets) |
| 139 | save(path, record) |
| 140 | print(json.dumps({"event": "substation-incident", "request_id": edge["request_id"], |
| 141 | "status": edge["status"], "upload_complete": record["upload_complete"], |
| 142 | "samples": len(record["tcp"]), "packets": len(record["packets"])}), flush=True) |
| 143 | state["cursor"] = entry["__CURSOR"] |
| 144 | state["clients"] = {client: stamp for client, stamp in state["clients"].items() if now - stamp < 3600} |
| 145 | if ports: |
| 146 | filters = "( sport = :443 or " + " or ".join(f"sport = :{port} or dport = :{port}" for port in sorted(ports)) + " )" |
| 147 | result = subprocess.run(["ss", "-HtinO", "state", "established", filters], check=True, capture_output=True, text=True, timeout=5) |
| 148 | rows = connections(result.stdout, ports, state["clients"]) |
| 149 | samples.append({"ts": now, "connections": rows}) |
| 150 | save(state_file, state) |
| 151 | if now - saved >= 5: |
| 152 | save(recent, list(samples)) |
| 153 | saved = now |
| 154 | old = sorted((p for p in ROOT.glob("*.json") if re.fullmatch(r"[0-9a-f-]{36}", p.stem)), key=lambda p: p.stat().st_mtime) |
| 155 | for path in old[:-100]: |
| 156 | path.unlink() |
| 157 | if now - reported >= 60: |
| 158 | print(json.dumps({"event": "substation-watcher", "ports": sorted(ports), |
| 159 | "clients": len(state["clients"]), "samples": len(samples), "packets": len(packets)}), flush=True) |
| 160 | reported = now |
| 161 | except Exception as error: |
| 162 | print(json.dumps({"event": "substation-watcher-error", "type": type(error).__name__}), flush=True) |
| 163 | time.sleep(1) |
| 164 | |
| 165 | |
| 166 | if __name__ == "__main__": |
| 167 | main() |