| ... | @@ -6,6 +6,7 @@ import os | ... | @@ -6,6 +6,7 @@ import os |
| 6 | from pathlib import Path | 6 | from pathlib import Path |
| 7 | import re | 7 | import re |
| 8 | import subprocess | 8 | import subprocess |
| | 9 | import threading |
| 9 | import time | 10 | import time |
| 10 | from urllib.request import Request, urlopen | 11 | from urllib.request import Request, urlopen |
| 11 | | 12 | |
| ... | @@ -67,11 +68,35 @@ def event(entry, host): | ... | @@ -67,11 +68,35 @@ def event(entry, host): |
| 67 | return None, None | 68 | return None, None |
| 68 | | 69 | |
| 69 | | 70 | |
| 70 | def incident(edge, samples): | 71 | def packet(line): |
| | 72 | match = re.fullmatch(r"([0-9.]+) \S+\s+(?:In|Out)\s+IP6? (\S+) > (\S+): Flags \[([A-Za-z.*]+)\], (.*)", line.strip()) |
| | 73 | if not match: |
| | 74 | return None |
| | 75 | stamp, source, destination, flags, details = match.groups() |
| | 76 | endpoints = {} |
| | 77 | try: |
| | 78 | for key, value in (("source", source), ("destination", destination)): |
| | 79 | host, _, port = value.rpartition(".") |
| | 80 | host = ipaddress.ip_address(host) |
| | 81 | endpoints[key] = f"[{host}]:{int(port)}" |
| | 82 | except ValueError: |
| | 83 | return None |
| | 84 | row = {"ts": float(stamp), **endpoints, "flags": flags} |
| | 85 | for key in ("seq", "ack", "win", "length"): |
| | 86 | if value := re.search(r"\b" + key + r" ([0-9]+(?::[0-9]+)?)", details): |
| | 87 | row[key] = value[1] |
| | 88 | row["sack"] = re.findall(r"\{([0-9]+:[0-9]+)\}", details) |
| | 89 | return row |
| | 90 | |
| | 91 | |
| | 92 | def incident(edge, samples, packets): |
| 71 | length = edge.get("content_length") | 93 | length = edge.get("content_length") |
| 72 | complete = int(length) == edge["bytes_read"] if str(length).isdigit() else None | 94 | complete = int(length) == edge["bytes_read"] if str(length).isdigit() else None |
| | 95 | peer = endpoint(edge["client"]) if edge.get("client") else None |
| 73 | return {"edge": edge, "upload_complete": complete, | 96 | return {"edge": edge, "upload_complete": complete, |
| 74 | "tcp": [sample for sample in samples if edge["ts"] - edge["duration"] - 5 <= sample["ts"] <= edge["ts"] + 5]} | 97 | "tcp": [sample for sample in samples if edge["ts"] - edge["duration"] - 5 <= sample["ts"] <= edge["ts"] + 5], |
| | 98 | "packets": [row for row in list(packets) if edge["ts"] - edge["duration"] - 5 <= row["ts"] <= edge["ts"] + 5 |
| | 99 | and peer in (endpoint(row["source"]), endpoint(row["destination"]))]} |
| 75 | | 100 | |
| 76 | | 101 | |
| 77 | def main(): | 102 | def main(): |
| ... | @@ -81,12 +106,32 @@ def main(): | ... | @@ -81,12 +106,32 @@ def main(): |
| 81 | state = json.loads(state_file.read_text()) if state_file.exists() else {"clients": {}} | 106 | state = json.loads(state_file.read_text()) if state_file.exists() else {"clients": {}} |
| 82 | recent = ROOT / "recent.json" | 107 | recent = ROOT / "recent.json" |
| 83 | samples = deque(json.loads(recent.read_text()) if recent.exists() and time.time() - recent.stat().st_mtime < 120 else [], maxlen=120) | 108 | samples = deque(json.loads(recent.read_text()) if recent.exists() and time.time() - recent.stat().st_mtime < 120 else [], maxlen=120) |
| | 109 | packets = deque(maxlen=100000) |
| | 110 | capture = None |
| | 111 | captured_clients = None |
| | 112 | |
| | 113 | def read_packets(stream): |
| | 114 | for line in stream: |
| | 115 | row = packet(line) |
| | 116 | if row and any(endpoint(row[key])[0] in state["clients"] for key in ("source", "destination")): |
| | 117 | packets.append(row) |
| | 118 | |
| 84 | ports = set() | 119 | ports = set() |
| 85 | refreshed = saved = reported = 0 | 120 | refreshed = saved = reported = 0 |
| 86 | host = "substation.ai." + os.environ["SNOWGLOBE_DOMAIN"] | 121 | host = "substation.ai." + os.environ["SNOWGLOBE_DOMAIN"] |
| 87 | while True: | 122 | while True: |
| 88 | try: | 123 | try: |
| 89 | now = time.time() | 124 | now = time.time() |
| | 125 | clients = tuple(sorted(state["clients"])) |
| | 126 | if capture is None or capture.poll() is not None or clients != captured_clients: |
| | 127 | if capture is not None: |
| | 128 | capture.terminate() |
| | 129 | capture.wait(timeout=2) |
| | 130 | filters = "tcp port 443 and (" + " or ".join("host " + client for client in (clients or ("127.0.0.1",))) + ")" |
| | 131 | capture = subprocess.Popen(["tcpdump", "-l", "-tt", "-nn", "-S", "-i", "any", "-s", "128", filters], |
| | 132 | stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) |
| | 133 | threading.Thread(target=read_packets, args=(capture.stdout,), daemon=True).start() |
| | 134 | captured_clients = clients |
| 90 | if now - refreshed >= 10: | 135 | if now - refreshed >= 10: |
| 91 | token = Path("/var/lib/snowglobe/nomad.token").read_text().strip() | 136 | token = Path("/var/lib/snowglobe/nomad.token").read_text().strip() |
| 92 | request = Request("http://127.0.0.1:4646/v1/service/substation-hub", headers={"X-Nomad-Token": token}) | 137 | request = Request("http://127.0.0.1:4646/v1/service/substation-hub", headers={"X-Nomad-Token": token}) |
| ... | @@ -109,11 +154,11 @@ def main(): | ... | @@ -109,11 +154,11 @@ def main(): |
| 109 | if edge and edge.get("status", 0) >= 499 and re.fullmatch(r"[0-9a-f-]{36}", str(edge.get("request_id", ""))): | 154 | if edge and edge.get("status", 0) >= 499 and re.fullmatch(r"[0-9a-f-]{36}", str(edge.get("request_id", ""))): |
| 110 | path = ROOT / (edge["request_id"] + ".json") | 155 | path = ROOT / (edge["request_id"] + ".json") |
| 111 | if not path.exists(): | 156 | if not path.exists(): |
| 112 | record = incident(edge, samples) | 157 | record = incident(edge, samples, packets) |
| 113 | save(path, record) | 158 | save(path, record) |
| 114 | print(json.dumps({"event": "substation-incident", "request_id": edge["request_id"], | 159 | print(json.dumps({"event": "substation-incident", "request_id": edge["request_id"], |
| 115 | "status": edge["status"], "upload_complete": record["upload_complete"], | 160 | "status": edge["status"], "upload_complete": record["upload_complete"], |
| 116 | "samples": len(record["tcp"])}), flush=True) | 161 | "samples": len(record["tcp"]), "packets": len(record["packets"])}), flush=True) |
| 117 | state["cursor"] = entry["__CURSOR"] | 162 | state["cursor"] = entry["__CURSOR"] |
| 118 | state["clients"] = {client: stamp for client, stamp in state["clients"].items() if now - stamp < 3600} | 163 | state["clients"] = {client: stamp for client, stamp in state["clients"].items() if now - stamp < 3600} |
| 119 | if ports: | 164 | if ports: |
| ... | @@ -130,7 +175,7 @@ def main(): | ... | @@ -130,7 +175,7 @@ def main(): |
| 130 | path.unlink() | 175 | path.unlink() |
| 131 | if now - reported >= 60: | 176 | if now - reported >= 60: |
| 132 | print(json.dumps({"event": "substation-watcher", "ports": sorted(ports), | 177 | print(json.dumps({"event": "substation-watcher", "ports": sorted(ports), |
| 133 | "clients": len(state["clients"]), "samples": len(samples)}), flush=True) | 178 | "clients": len(state["clients"]), "samples": len(samples), "packets": len(packets)}), flush=True) |
| 134 | reported = now | 179 | reported = now |
| 135 | except Exception as error: | 180 | except Exception as error: |
| 136 | print(json.dumps({"event": "substation-watcher-error", "type": type(error).__name__}), flush=True) | 181 | print(json.dumps({"event": "substation-watcher-error", "type": type(error).__name__}), flush=True) |