1#!/usr/bin/env python3
2from collections import deque
3import ipaddress
4import json
5import os
6from pathlib import Path
7import re
8import subprocess
9import threading
10import time
11from urllib.request import Request, urlopen
12
13from substation_packets import packet
14
15
16ROOT = Path("/var/lib/snowglobe/substation-watch")
17EDGE_FIELDS = ("ts", "status", "bytes_read", "content_length", "duration", "request_id",
18 "method", "path", "protocol", "upstream", "upstream_status", "upstream_header_ms", "client")
19TCP_FIELDS = {"bytes_sent", "bytes_acked", "bytes_received", "bytes_retrans", "lastsnd", "lastrcv", "lastack",
20 "rtt", "rto", "retrans", "snd_wnd", "rcv_wnd", "rcv_space", "rcv_ssthresh", "cwnd", "unacked"}
21
22
23def save(path, value):
24 pending = path.with_suffix(".pending")
25 with pending.open("w") as file:
26 json.dump(value, file, separators=(",", ":"))
27 pending.chmod(0o600)
28 pending.replace(path)
29
30
31def endpoint(value):
32 address, _, port = value.rpartition(":")
33 address = ipaddress.ip_address(address.strip("[]"))
34 if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped:
35 address = address.ipv4_mapped
36 return str(address), int(port)
37
38
39def connections(output, ports, clients):
40 rows = []
41 for line in output.splitlines():
42 parts = line.split()
43 if len(parts) < 4:
44 continue
45 local, peer = endpoint(parts[2]), endpoint(parts[3])
46 if not (local[1] in ports or peer[1] in ports or local[1] == 443 and peer[0] in clients):
47 continue
48 row = {"local": parts[2], "peer": parts[3], "recv_q": int(parts[0]), "send_q": int(parts[1])}
49 for part in parts[4:]:
50 key, _, value = part.partition(":")
51 if key in TCP_FIELDS and re.fullmatch(r"[0-9./]+", value):
52 row[key] = value
53 rows.append(row)
54 return rows
55
56
57def event(entry, host):
58 try:
59 message = json.loads(entry.get("MESSAGE", ""))
60 except (ValueError, TypeError):
61 return None, None
62 if not isinstance(message, dict):
63 return None, None
64 if message.get("service") == "substation":
65 return {key: message.get(key) for key in EDGE_FIELDS}, message.get("client")
66 request = message.get("request", {})
67 if request.get("host") == host and request.get("remote_ip"):
68 address = str(ipaddress.ip_address(request["remote_ip"]))
69 return None, f"[{address}]:{int(request['remote_port'])}"
70 return None, None
71
72
73def incident(edge, samples, packets):
74 length = edge.get("content_length")
75 complete = int(length) == edge["bytes_read"] if str(length).isdigit() else None
76 peer = endpoint(edge["client"]) if edge.get("client") else None
77 return {"edge": edge, "upload_complete": complete,
78 "tcp": [sample for sample in samples if edge["ts"] - edge["duration"] - 5 <= sample["ts"] <= edge["ts"] + 5],
79 "packets": [row for row in list(packets) if edge["ts"] - edge["duration"] - 5 <= row["ts"] <= edge["ts"] + 5
80 and peer in (endpoint(row["source"]), endpoint(row["destination"]))]}
81
82
83def main():
84 os.umask(0o077)
85 ROOT.mkdir(mode=0o700, parents=True, exist_ok=True)
86 state_file = ROOT / "state.json"
87 state = json.loads(state_file.read_text()) if state_file.exists() else {"clients": {}}
88 recent = ROOT / "recent.json"
89 samples = deque(json.loads(recent.read_text()) if recent.exists() and time.time() - recent.stat().st_mtime < 120 else [], maxlen=120)
90 packets = deque(maxlen=100000)
91 capture = None
92 captured_clients = None
93
94 def read_packets(stream):
95 for line in stream:
96 row = packet(line)
97 if row and any(endpoint(row[key])[0] in state["clients"] for key in ("source", "destination")):
98 packets.append(row)
99
100 ports = set()
101 refreshed = saved = reported = 0
102 host = "substation.ai." + os.environ["SNOWGLOBE_DOMAIN"]
103 while True:
104 try:
105 now = time.time()
106 clients = tuple(sorted(state["clients"]))
107 if capture is None or capture.poll() is not None or clients != captured_clients:
108 if capture is not None:
109 capture.terminate()
110 capture.wait(timeout=2)
111 filters = "tcp port 443 and (" + " or ".join("host " + client for client in (clients or ("127.0.0.1",))) + ")"
112 capture = subprocess.Popen(["tcpdump", "-l", "-tt", "-nn", "-S", "-i", "any", "-s", "128", filters],
113 stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True)
114 threading.Thread(target=read_packets, args=(capture.stdout,), daemon=True).start()
115 captured_clients = clients
116 if now - refreshed >= 10:
117 token = Path("/var/lib/snowglobe/nomad.token").read_text().strip()
118 request = Request("http://127.0.0.1:4646/v1/service/substation-hub", headers={"X-Nomad-Token": token})
119 with urlopen(request, timeout=5) as response:
120 ports = {item["Port"] for item in json.load(response)
121 if item.get("JobID") == "substation" and ipaddress.ip_address(item["Address"]).is_loopback}
122 refreshed = now
123 command = ["journalctl", "-u", "caddy", "--no-pager", "--output=json"]
124 command += ["--after-cursor=" + state["cursor"]] if state.get("cursor") else ["--since=-1 hour"]
125 journal = subprocess.run(command, capture_output=True, text=True, timeout=10)
126 if journal.returncode:
127 if "cursor" in journal.stderr.lower() and state.pop("cursor", None):
128 print(json.dumps({"event": "substation-watcher-journal-gap"}), flush=True)
129 journal.check_returncode()
130 for line in journal.stdout.splitlines():
131 entry = json.loads(line)
132 edge, client = event(entry, host)
133 if client:
134 state["clients"][endpoint(client)[0]] = now
135 if edge and edge.get("status", 0) >= 499 and re.fullmatch(r"[0-9a-f-]{36}", str(edge.get("request_id", ""))):
136 path = ROOT / (edge["request_id"] + ".json")
137 if not path.exists():
138 record = incident(edge, samples, packets)
139 save(path, record)
140 print(json.dumps({"event": "substation-incident", "request_id": edge["request_id"],
141 "status": edge["status"], "upload_complete": record["upload_complete"],
142 "samples": len(record["tcp"]), "packets": len(record["packets"])}), flush=True)
143 state["cursor"] = entry["__CURSOR"]
144 state["clients"] = {client: stamp for client, stamp in state["clients"].items() if now - stamp < 3600}
145 if ports:
146 filters = "( sport = :443 or " + " or ".join(f"sport = :{port} or dport = :{port}" for port in sorted(ports)) + " )"
147 result = subprocess.run(["ss", "-HtinO", "state", "established", filters], check=True, capture_output=True, text=True, timeout=5)
148 rows = connections(result.stdout, ports, state["clients"])
149 samples.append({"ts": now, "connections": rows})
150 save(state_file, state)
151 if now - saved >= 5:
152 save(recent, list(samples))
153 saved = now
154 old = sorted((p for p in ROOT.glob("*.json") if re.fullmatch(r"[0-9a-f-]{36}", p.stem)), key=lambda p: p.stat().st_mtime)
155 for path in old[:-100]:
156 path.unlink()
157 if now - reported >= 60:
158 print(json.dumps({"event": "substation-watcher", "ports": sorted(ports),
159 "clients": len(state["clients"]), "samples": len(samples), "packets": len(packets)}), flush=True)
160 reported = now
161 except Exception as error:
162 print(json.dumps({"event": "substation-watcher-error", "type": type(error).__name__}), flush=True)
163 time.sleep(1)
164
165
166if __name__ == "__main__":
167 main()